LLVM 24.0.0git
SIISelLowering.cpp
Go to the documentation of this file.
1//===-- SIISelLowering.cpp - SI DAG Lowering Implementation ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Custom DAG lowering for SI
11//
12//===----------------------------------------------------------------------===//
13
14#include "SIISelLowering.h"
15#include "AMDGPU.h"
16#include "AMDGPUIGroupLP.h"
17#include "AMDGPUInstrInfo.h"
18#include "AMDGPULaneMaskUtils.h"
19#include "AMDGPUMachineInstrs.h"
20#include "AMDGPUMemoryUtils.h"
22#include "AMDGPUTargetMachine.h"
23#include "GCNSubtarget.h"
26#include "SIRegisterInfo.h"
27#include "llvm/ADT/APFloat.h"
28#include "llvm/ADT/APInt.h"
30#include "llvm/ADT/Statistic.h"
46#include "llvm/IR/IRBuilder.h"
48#include "llvm/IR/IntrinsicsAMDGPU.h"
49#include "llvm/IR/IntrinsicsR600.h"
50#include "llvm/IR/MDBuilder.h"
53#include "llvm/Support/ModRef.h"
56#include <optional>
57
58using namespace llvm;
59using namespace llvm::SDPatternMatch;
60
61#define DEBUG_TYPE "si-lower"
62
63STATISTIC(NumTailCalls, "Number of tail calls");
64
66 return MF.getInfo<SIMachineFunctionInfo>()->getMode().getDenormalFPEnv();
67}
68
73
78
79static unsigned findFirstFreeSGPR(CCState &CCInfo) {
80 unsigned NumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs();
81 for (unsigned Reg = 0; Reg < NumSGPRs; ++Reg) {
82 if (!CCInfo.isAllocated(AMDGPU::SGPR0 + Reg)) {
83 return AMDGPU::SGPR0 + Reg;
84 }
85 }
86 llvm_unreachable("Cannot allocate sgpr");
87}
88
90 const GCNSubtarget &STI)
91 : AMDGPUTargetLowering(TM, STI, STI), Subtarget(&STI) {
92 addRegisterClass(MVT::i1, &AMDGPU::VReg_1RegClass);
93 addRegisterClass(MVT::i64, &AMDGPU::SReg_64RegClass);
94
95 addRegisterClass(MVT::i32, &AMDGPU::SReg_32RegClass);
96
97 const SIRegisterInfo *TRI = STI.getRegisterInfo();
98 const TargetRegisterClass *V32RegClass =
99 TRI->getDefaultVectorSuperClassForBitWidth(32);
100 addRegisterClass(MVT::f32, V32RegClass);
101
102 addRegisterClass(MVT::v2i32, &AMDGPU::SReg_64RegClass);
103
104 const TargetRegisterClass *V64RegClass =
105 TRI->getDefaultVectorSuperClassForBitWidth(64);
106
107 addRegisterClass(MVT::f64, V64RegClass);
108 addRegisterClass(MVT::v2f32, V64RegClass);
109 addRegisterClass(MVT::Untyped, V64RegClass);
110
111 addRegisterClass(MVT::v3i32, &AMDGPU::SGPR_96RegClass);
112 addRegisterClass(MVT::v3f32, TRI->getDefaultVectorSuperClassForBitWidth(96));
113
114 addRegisterClass(MVT::v2i64, &AMDGPU::SGPR_128RegClass);
115 addRegisterClass(MVT::v2f64, &AMDGPU::SGPR_128RegClass);
116
117 addRegisterClass(MVT::v4i32, &AMDGPU::SGPR_128RegClass);
118 addRegisterClass(MVT::v4f32, TRI->getDefaultVectorSuperClassForBitWidth(128));
119
120 addRegisterClass(MVT::v5i32, &AMDGPU::SGPR_160RegClass);
121 addRegisterClass(MVT::v5f32, TRI->getDefaultVectorSuperClassForBitWidth(160));
122
123 addRegisterClass(MVT::v6i32, &AMDGPU::SGPR_192RegClass);
124 addRegisterClass(MVT::v6f32, TRI->getDefaultVectorSuperClassForBitWidth(192));
125
126 addRegisterClass(MVT::v3i64, &AMDGPU::SGPR_192RegClass);
127 addRegisterClass(MVT::v3f64, TRI->getDefaultVectorSuperClassForBitWidth(192));
128
129 addRegisterClass(MVT::v7i32, &AMDGPU::SGPR_224RegClass);
130 addRegisterClass(MVT::v7f32, TRI->getDefaultVectorSuperClassForBitWidth(224));
131
132 addRegisterClass(MVT::v8i32, &AMDGPU::SGPR_256RegClass);
133 addRegisterClass(MVT::v8f32, TRI->getDefaultVectorSuperClassForBitWidth(256));
134
135 addRegisterClass(MVT::v4i64, &AMDGPU::SGPR_256RegClass);
136 addRegisterClass(MVT::v4f64, TRI->getDefaultVectorSuperClassForBitWidth(256));
137
138 addRegisterClass(MVT::v9i32, &AMDGPU::SGPR_288RegClass);
139 addRegisterClass(MVT::v9f32, TRI->getDefaultVectorSuperClassForBitWidth(288));
140
141 addRegisterClass(MVT::v10i32, &AMDGPU::SGPR_320RegClass);
142 addRegisterClass(MVT::v10f32,
143 TRI->getDefaultVectorSuperClassForBitWidth(320));
144
145 addRegisterClass(MVT::v11i32, &AMDGPU::SGPR_352RegClass);
146 addRegisterClass(MVT::v11f32,
147 TRI->getDefaultVectorSuperClassForBitWidth(352));
148
149 addRegisterClass(MVT::v12i32, &AMDGPU::SGPR_384RegClass);
150 addRegisterClass(MVT::v12f32,
151 TRI->getDefaultVectorSuperClassForBitWidth(384));
152
153 addRegisterClass(MVT::v16i32, &AMDGPU::SGPR_512RegClass);
154 addRegisterClass(MVT::v16f32,
155 TRI->getDefaultVectorSuperClassForBitWidth(512));
156
157 addRegisterClass(MVT::v8i64, &AMDGPU::SGPR_512RegClass);
158 addRegisterClass(MVT::v8f64, TRI->getDefaultVectorSuperClassForBitWidth(512));
159
160 addRegisterClass(MVT::v16i64, &AMDGPU::SGPR_1024RegClass);
161 addRegisterClass(MVT::v16f64,
162 TRI->getDefaultVectorSuperClassForBitWidth(1024));
163
164 if (Subtarget->has16BitInsts()) {
165 if (Subtarget->useRealTrue16Insts()) {
166 addRegisterClass(MVT::i16, &AMDGPU::VGPR_16RegClass);
167 addRegisterClass(MVT::f16, &AMDGPU::VGPR_16RegClass);
168 addRegisterClass(MVT::bf16, &AMDGPU::VGPR_16RegClass);
169 } else {
170 addRegisterClass(MVT::i16, &AMDGPU::SReg_32RegClass);
171 addRegisterClass(MVT::f16, &AMDGPU::SReg_32RegClass);
172 addRegisterClass(MVT::bf16, &AMDGPU::SReg_32RegClass);
173 }
174
175 // Unless there are also VOP3P operations, not operations are really legal.
176 addRegisterClass(MVT::v2i16, &AMDGPU::SReg_32RegClass);
177 addRegisterClass(MVT::v2f16, &AMDGPU::SReg_32RegClass);
178 addRegisterClass(MVT::v2bf16, &AMDGPU::SReg_32RegClass);
179 addRegisterClass(MVT::v4i16, &AMDGPU::SReg_64RegClass);
180 addRegisterClass(MVT::v4f16, &AMDGPU::SReg_64RegClass);
181 addRegisterClass(MVT::v4bf16, &AMDGPU::SReg_64RegClass);
182 addRegisterClass(MVT::v8i16, &AMDGPU::SGPR_128RegClass);
183 addRegisterClass(MVT::v8f16, &AMDGPU::SGPR_128RegClass);
184 addRegisterClass(MVT::v8bf16, &AMDGPU::SGPR_128RegClass);
185 addRegisterClass(MVT::v16i16, &AMDGPU::SGPR_256RegClass);
186 addRegisterClass(MVT::v16f16, &AMDGPU::SGPR_256RegClass);
187 addRegisterClass(MVT::v16bf16, &AMDGPU::SGPR_256RegClass);
188 addRegisterClass(MVT::v32i16, &AMDGPU::SGPR_512RegClass);
189 addRegisterClass(MVT::v32f16, &AMDGPU::SGPR_512RegClass);
190 addRegisterClass(MVT::v32bf16, &AMDGPU::SGPR_512RegClass);
191 }
192
193 addRegisterClass(MVT::v32i32, &AMDGPU::VReg_1024RegClass);
194 addRegisterClass(MVT::v32f32,
195 TRI->getDefaultVectorSuperClassForBitWidth(1024));
196
197 computeRegisterProperties(Subtarget->getRegisterInfo());
198
201
202 // The boolean content concept here is too inflexible. Compares only ever
203 // really produce a 1-bit result. Any copy/extend from these will turn into a
204 // select, and zext/1 or sext/-1 are equally cheap. Arbitrarily choose 0/1, as
205 // it's what most targets use.
208
209 // We need to custom lower vector stores from local memory
211 {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
212 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
213 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32,
214 MVT::i1, MVT::v32i32},
215 Custom);
216
218 {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
219 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
220 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32,
221 MVT::i1, MVT::v32i32},
222 Custom);
223
224 if (isTypeLegal(MVT::bf16)) {
225 for (unsigned Opc :
234 ISD::SETCC}) {
235 setOperationAction(Opc, MVT::bf16, Promote);
236 }
237
238 // Only targets with packed bf16 instructions, e.g. gfx13.
239 if (Subtarget->hasBF16PackedInsts()) {
240 // Don't use Expand for fsub - the DAG combiner will undo fadd+fneg back
241 // to fsub, causing a libcall (which doesn't exist for bf16). Instead,
242 // directly expand to widened v2bf16 operations.
245 Promote);
246 // Promote scalar operations to a v2bf16 operation with an unused high
247 // lane.
248 for (unsigned Opc :
251 AddPromotedToType(Opc, MVT::bf16, MVT::v2bf16);
252 }
253
255
257 AddPromotedToType(ISD::SELECT, MVT::bf16, MVT::i16);
258
262
263 // We only need to custom lower because we can't specify an action for bf16
264 // sources.
267 }
268
269 setTruncStoreAction(MVT::v2i32, MVT::v2i16, Expand);
270 setTruncStoreAction(MVT::v3i32, MVT::v3i16, Expand);
271 setTruncStoreAction(MVT::v4i32, MVT::v4i16, Expand);
272 setTruncStoreAction(MVT::v8i32, MVT::v8i16, Expand);
273 setTruncStoreAction(MVT::v16i32, MVT::v16i16, Expand);
274 setTruncStoreAction(MVT::v32i32, MVT::v32i16, Expand);
275 setTruncStoreAction(MVT::v2i32, MVT::v2i8, Expand);
276 setTruncStoreAction(MVT::v4i32, MVT::v4i8, Expand);
277 setTruncStoreAction(MVT::v8i32, MVT::v8i8, Expand);
278 setTruncStoreAction(MVT::v16i32, MVT::v16i8, Expand);
279 setTruncStoreAction(MVT::v32i32, MVT::v32i8, Expand);
280 setTruncStoreAction(MVT::v2i16, MVT::v2i8, Expand);
281 setTruncStoreAction(MVT::v4i16, MVT::v4i8, Expand);
282 setTruncStoreAction(MVT::v8i16, MVT::v8i8, Expand);
283 setTruncStoreAction(MVT::v16i16, MVT::v16i8, Expand);
284 setTruncStoreAction(MVT::v32i16, MVT::v32i8, Expand);
285
286 setTruncStoreAction(MVT::v3i64, MVT::v3i16, Expand);
287 setTruncStoreAction(MVT::v3i64, MVT::v3i32, Expand);
288 setTruncStoreAction(MVT::v4i64, MVT::v4i8, Expand);
289 setTruncStoreAction(MVT::v8i64, MVT::v8i8, Expand);
290 setTruncStoreAction(MVT::v8i64, MVT::v8i16, Expand);
291 setTruncStoreAction(MVT::v8i64, MVT::v8i32, Expand);
292 setTruncStoreAction(MVT::v16i64, MVT::v16i32, Expand);
293
294 setOperationAction(ISD::GlobalAddress, {MVT::i32, MVT::i64}, Custom);
295
296 // Only for an alloca in the VGPR ("as memory") address space.
298 setOperationAction(ISD::BlockAddress, {MVT::i32, MVT::i64}, Custom);
299 setOperationAction(ISD::ExternalSymbol, {MVT::i32, MVT::i64}, Custom);
300
304 AddPromotedToType(ISD::SELECT, MVT::f64, MVT::i64);
305
306 setOperationAction(ISD::FSQRT, {MVT::f32, MVT::f64}, Custom);
307
309 {MVT::f32, MVT::i32, MVT::i64, MVT::f64, MVT::i1}, Expand);
310
312 setOperationAction(ISD::SETCC, {MVT::v2i1, MVT::v4i1}, Expand);
313 AddPromotedToType(ISD::SETCC, MVT::i1, MVT::i32);
314
316 {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
317 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
318 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32},
319 Expand);
321 {MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32,
322 MVT::v6f32, MVT::v7f32, MVT::v8f32, MVT::v9f32,
323 MVT::v10f32, MVT::v11f32, MVT::v12f32, MVT::v16f32},
324 Expand);
325
327 {MVT::v2i1, MVT::v4i1, MVT::v2i8, MVT::v4i8, MVT::v2i16,
328 MVT::v3i16, MVT::v4i16, MVT::Other},
329 Custom);
330
333 {MVT::i1, MVT::i32, MVT::i64, MVT::f32, MVT::f64}, Expand);
334
337
340
342 Expand);
343
345
346 // We only support LOAD/STORE and vector manipulation ops for vectors
347 // with > 4 elements.
348 for (MVT VT :
349 {MVT::v8i32, MVT::v8f32, MVT::v9i32, MVT::v9f32, MVT::v10i32,
350 MVT::v10f32, MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32,
351 MVT::v16i32, MVT::v16f32, MVT::v2i64, MVT::v2f64, MVT::v4i16,
352 MVT::v4f16, MVT::v4bf16, MVT::v3i64, MVT::v3f64, MVT::v6i32,
353 MVT::v6f32, MVT::v4i64, MVT::v4f64, MVT::v8i64, MVT::v8f64,
354 MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16, MVT::v16f16,
355 MVT::v16bf16, MVT::v16i64, MVT::v16f64, MVT::v32i32, MVT::v32f32,
356 MVT::v32i16, MVT::v32f16, MVT::v32bf16}) {
357 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) {
358 switch (Op) {
359 case ISD::LOAD:
360 case ISD::STORE:
361 case ISD::ATOMIC_LOAD:
364 case ISD::BITCAST:
365 case ISD::UNDEF:
366 case ISD::POISON:
370 case ISD::IS_FPCLASS:
371 break;
376 break;
377 default:
379 break;
380 }
381 }
382 }
383
385
386 // TODO: For dynamic 64-bit vector inserts/extracts, should emit a pseudo that
387 // is expanded to avoid having two separate loops in case the index is a VGPR.
388
389 // Most operations are naturally 32-bit vector operations. We only support
390 // load and store of i64 vectors, so promote v2i64 vector operations to v4i32.
391 for (MVT Vec64 : {MVT::v2i64, MVT::v2f64}) {
393 AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v4i32);
394
396 AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v4i32);
397
399 AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v4i32);
400
402 AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v4i32);
403 }
404
405 for (MVT Vec64 : {MVT::v3i64, MVT::v3f64}) {
407 AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v6i32);
408
410 AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v6i32);
411
413 AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v6i32);
414
416 AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v6i32);
417 }
418
419 for (MVT Vec64 : {MVT::v4i64, MVT::v4f64}) {
421 AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v8i32);
422
424 AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v8i32);
425
427 AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v8i32);
428
430 AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v8i32);
431 }
432
433 for (MVT Vec64 : {MVT::v8i64, MVT::v8f64}) {
435 AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v16i32);
436
438 AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v16i32);
439
441 AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v16i32);
442
444 AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v16i32);
445 }
446
447 for (MVT Vec64 : {MVT::v16i64, MVT::v16f64}) {
449 AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v32i32);
450
452 AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v32i32);
453
455 AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v32i32);
456
458 AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v32i32);
459 }
460
462 {MVT::v4i32, MVT::v4f32, MVT::v8i32, MVT::v8f32,
463 MVT::v16i32, MVT::v16f32, MVT::v32i32, MVT::v32f32},
464 Custom);
465
466 if (Subtarget->hasPkMovB32()) {
467 // TODO: 16-bit element vectors should be legal with even aligned elements.
468 // TODO: Can be legal with wider source types than the result with
469 // subregister extracts.
470 setOperationAction(ISD::VECTOR_SHUFFLE, {MVT::v2i32, MVT::v2f32}, Legal);
471 }
472
474 // Prevent SELECT v2i32 from being implemented with the above bitwise ops and
475 // instead lower to cndmask in SITargetLowering::LowerSELECT().
477 // Enable MatchRotate to produce ISD::ROTR, which is later transformed to
478 // alignbit.
479 setOperationAction(ISD::ROTR, MVT::v2i32, Custom);
480
481 setOperationAction(ISD::BUILD_VECTOR, {MVT::v4f16, MVT::v4i16, MVT::v4bf16},
482 Custom);
483
484 // Avoid stack access for these.
485 // TODO: Generalize to wider i8 vectors.
487 {MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::v2i8, MVT::v4i8,
488 MVT::v8i8, MVT::v4i16, MVT::v4f16, MVT::v4bf16},
489 Custom);
490
491 // Deal with vec3 vector operations when widened to vec4.
493 {MVT::v3i32, MVT::v3f32, MVT::v4i32, MVT::v4f32}, Custom);
494
495 // Deal with vec5/6/7 vector operations when widened to vec8.
497 {MVT::v5i32, MVT::v5f32, MVT::v6i32, MVT::v6f32,
498 MVT::v7i32, MVT::v7f32, MVT::v8i32, MVT::v8f32,
499 MVT::v9i32, MVT::v9f32, MVT::v10i32, MVT::v10f32,
500 MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32},
501 Custom);
502
503 // BUFFER/FLAT_ATOMIC_CMP_SWAP on GCN GPUs needs input marshalling,
504 // and output demarshalling
505 setOperationAction(ISD::ATOMIC_CMP_SWAP, {MVT::i32, MVT::i64}, Custom);
506
507 // We can't return success/failure, only the old value,
508 // let LLVM add the comparison
510 Expand);
511
512 setOperationAction(ISD::ADDRSPACECAST, {MVT::i32, MVT::i64}, Custom);
513
514 setOperationAction(ISD::BITREVERSE, {MVT::i32, MVT::i64}, Legal);
515
516 // FIXME: This should be narrowed to i32, but that only happens if i64 is
517 // illegal.
518 // FIXME: Should lower sub-i32 bswaps to bit-ops without v_perm_b32.
519 setOperationAction(ISD::BSWAP, {MVT::i64, MVT::i32}, Legal);
520
521 // On SI this is s_memtime and s_memrealtime on VI.
523
524 if (Subtarget->hasSMemRealTime() ||
525 Subtarget->getGeneration() >= AMDGPUSubtarget::GFX11)
528
529 if (Subtarget->has16BitInsts()) {
532 setOperationAction(ISD::IS_FPCLASS, {MVT::f16, MVT::f32, MVT::f64}, Legal);
535 } else {
537 }
538
539 if (Subtarget->hasMadMacF32Insts())
541
545
546 // We only really have 32-bit BFE instructions (and 16-bit on VI).
547 //
548 // On SI+ there are 64-bit BFEs, but they are scalar only and there isn't any
549 // effort to match them now. We want this to be false for i64 cases when the
550 // extraction isn't restricted to the upper or lower half. Ideally we would
551 // have some pass reduce 64-bit extracts to 32-bit if possible. Extracts that
552 // span the midpoint are probably relatively rare, so don't worry about them
553 // for now.
555
556 // Clamp modifier on add/sub
557 if (Subtarget->hasIntClamp())
559
560 if (Subtarget->hasAddNoCarryInsts())
561 setOperationAction({ISD::SADDSAT, ISD::SSUBSAT}, {MVT::i16, MVT::i32},
562 Legal);
563
564 // Do not have s_{min|max}_*f64 instruction f64 will only be lowered to
565 // v_{min|max}_*f64
566 if (Subtarget->hasIEEEMinimumMaximumInsts()) {
569 {MVT::f64, MVT::f32}, Legal);
570 } else {
573 {MVT::f64, MVT::f32}, Custom);
574 // These are really only legal for ieee_mode functions. We should be
575 // avoiding them for functions that don't have ieee_mode enabled, so just
576 // say they are legal.
578 {MVT::f64, MVT::f32}, Legal);
579 }
580
581 if (Subtarget->haveRoundOpsF64())
583 Legal);
584 else
586 MVT::f64, Custom);
587
589 setOperationAction({ISD::FLDEXP, ISD::STRICT_FLDEXP}, {MVT::f32, MVT::f64},
590 Legal);
591 setOperationAction(ISD::FFREXP, {MVT::f32, MVT::f64}, Custom);
592
595
596 setOperationAction(ISD::BF16_TO_FP, {MVT::i16, MVT::f32, MVT::f64}, Expand);
597 setOperationAction(ISD::FP_TO_BF16, {MVT::i16, MVT::f32, MVT::f64}, Expand);
598
600 Custom);
602 Custom);
604 Custom);
605
606 // Custom lower these because we can't specify a rule based on an illegal
607 // source bf16.
610
611 if (Subtarget->has16BitInsts()) {
614 MVT::i16, Legal);
615
616 AddPromotedToType(ISD::SIGN_EXTEND, MVT::i16, MVT::i32);
617
619 MVT::i16, Expand);
620
624 ISD::CTPOP},
625 MVT::i16, Promote);
626
628
629 setTruncStoreAction(MVT::i64, MVT::i16, Expand);
630
632 AddPromotedToType(ISD::FP16_TO_FP, MVT::i16, MVT::i32);
634 AddPromotedToType(ISD::FP_TO_FP16, MVT::i16, MVT::i32);
635
640
642
643 // F16 - Constant Actions.
646
647 // F16 - Load/Store Actions.
649 AddPromotedToType(ISD::LOAD, MVT::f16, MVT::i16);
651 AddPromotedToType(ISD::STORE, MVT::f16, MVT::i16);
652
653 // BF16 - Load/Store Actions.
655 AddPromotedToType(ISD::LOAD, MVT::bf16, MVT::i16);
657 AddPromotedToType(ISD::STORE, MVT::bf16, MVT::i16);
658
659 if (!Subtarget->useRealTrue16Insts() ||
660 !Subtarget->d16PreservesUnusedBits()) {
661 for (MVT VT : {MVT::i16, MVT::f16, MVT::bf16}) {
663 AddPromotedToType(ISD::ATOMIC_LOAD, VT, MVT::i32);
664 }
665 }
666
667 if (!Subtarget->useRealTrue16Insts()) {
668 for (MVT VT : {MVT::i16, MVT::f16, MVT::bf16}) {
671 }
672 }
673
674 // F16 - VOP1 Actions.
677 MVT::f16, Custom);
678
679 // BF16 - VOP1 Actions.
680 if (Subtarget->hasBF16TransInsts())
682
683 // F16 - VOP2 Actions.
684 setOperationAction({ISD::BR_CC, ISD::SELECT_CC}, {MVT::f16, MVT::bf16},
685 Expand);
689
690 // F16 - VOP3 Actions.
692 if (STI.hasMadF16())
694
695 for (MVT VT :
696 {MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::v4i16, MVT::v4f16,
697 MVT::v4bf16, MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16,
698 MVT::v16f16, MVT::v16bf16, MVT::v32i16, MVT::v32f16}) {
699 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) {
700 switch (Op) {
701 case ISD::LOAD:
702 case ISD::STORE:
703 case ISD::ATOMIC_LOAD:
706 case ISD::BITCAST:
707 case ISD::UNDEF:
708 case ISD::POISON:
713 case ISD::IS_FPCLASS:
714 break;
717 case ISD::FSIN:
718 case ISD::FCOS:
720 break;
721 default:
723 break;
724 }
725 }
726 }
727
728 // v_perm_b32 can handle either of these.
729 setOperationAction(ISD::BSWAP, {MVT::i16, MVT::v2i16}, Legal);
731
732 // Legalize vector types for sat conversions to select v_cvt_pk_[iu]16_f32.
733 if (Subtarget->hasVCvtPkIU16F32())
736 {MVT::v2i16, MVT::v4i16, MVT::v8i16, MVT::v16i16, MVT::v32i16},
737 Custom);
738
739 // XXX - Do these do anything? Vector constants turn into build_vector.
740 setOperationAction(ISD::Constant, {MVT::v2i16, MVT::v2f16}, Legal);
741
743 {MVT::v2i16, MVT::v2f16, MVT::v2bf16}, Legal);
744
746 AddPromotedToType(ISD::STORE, MVT::v2i16, MVT::i32);
748 AddPromotedToType(ISD::STORE, MVT::v2f16, MVT::i32);
749
751 AddPromotedToType(ISD::LOAD, MVT::v2i16, MVT::i32);
753 AddPromotedToType(ISD::LOAD, MVT::v2f16, MVT::i32);
754
755 setOperationAction(ISD::AND, MVT::v2i16, Promote);
756 AddPromotedToType(ISD::AND, MVT::v2i16, MVT::i32);
757 setOperationAction(ISD::OR, MVT::v2i16, Promote);
758 AddPromotedToType(ISD::OR, MVT::v2i16, MVT::i32);
759 setOperationAction(ISD::XOR, MVT::v2i16, Promote);
760 AddPromotedToType(ISD::XOR, MVT::v2i16, MVT::i32);
761
763 AddPromotedToType(ISD::LOAD, MVT::v4i16, MVT::v2i32);
765 AddPromotedToType(ISD::LOAD, MVT::v4f16, MVT::v2i32);
766 setOperationAction(ISD::LOAD, MVT::v4bf16, Promote);
767 AddPromotedToType(ISD::LOAD, MVT::v4bf16, MVT::v2i32);
768
770 AddPromotedToType(ISD::STORE, MVT::v4i16, MVT::v2i32);
772 AddPromotedToType(ISD::STORE, MVT::v4f16, MVT::v2i32);
774 AddPromotedToType(ISD::STORE, MVT::v4bf16, MVT::v2i32);
775
777 AddPromotedToType(ISD::LOAD, MVT::v8i16, MVT::v4i32);
779 AddPromotedToType(ISD::LOAD, MVT::v8f16, MVT::v4i32);
780 setOperationAction(ISD::LOAD, MVT::v8bf16, Promote);
781 AddPromotedToType(ISD::LOAD, MVT::v8bf16, MVT::v4i32);
782
784 AddPromotedToType(ISD::STORE, MVT::v4i16, MVT::v2i32);
786 AddPromotedToType(ISD::STORE, MVT::v4f16, MVT::v2i32);
787
789 AddPromotedToType(ISD::STORE, MVT::v8i16, MVT::v4i32);
791 AddPromotedToType(ISD::STORE, MVT::v8f16, MVT::v4i32);
793 AddPromotedToType(ISD::STORE, MVT::v8bf16, MVT::v4i32);
794
795 setOperationAction(ISD::LOAD, MVT::v16i16, Promote);
796 AddPromotedToType(ISD::LOAD, MVT::v16i16, MVT::v8i32);
797 setOperationAction(ISD::LOAD, MVT::v16f16, Promote);
798 AddPromotedToType(ISD::LOAD, MVT::v16f16, MVT::v8i32);
799 setOperationAction(ISD::LOAD, MVT::v16bf16, Promote);
800 AddPromotedToType(ISD::LOAD, MVT::v16bf16, MVT::v8i32);
801
803 AddPromotedToType(ISD::STORE, MVT::v16i16, MVT::v8i32);
805 AddPromotedToType(ISD::STORE, MVT::v16f16, MVT::v8i32);
806 setOperationAction(ISD::STORE, MVT::v16bf16, Promote);
807 AddPromotedToType(ISD::STORE, MVT::v16bf16, MVT::v8i32);
808
809 setOperationAction(ISD::LOAD, MVT::v32i16, Promote);
810 AddPromotedToType(ISD::LOAD, MVT::v32i16, MVT::v16i32);
811 setOperationAction(ISD::LOAD, MVT::v32f16, Promote);
812 AddPromotedToType(ISD::LOAD, MVT::v32f16, MVT::v16i32);
813 setOperationAction(ISD::LOAD, MVT::v32bf16, Promote);
814 AddPromotedToType(ISD::LOAD, MVT::v32bf16, MVT::v16i32);
815
817 AddPromotedToType(ISD::STORE, MVT::v32i16, MVT::v16i32);
819 AddPromotedToType(ISD::STORE, MVT::v32f16, MVT::v16i32);
820 setOperationAction(ISD::STORE, MVT::v32bf16, Promote);
821 AddPromotedToType(ISD::STORE, MVT::v32bf16, MVT::v16i32);
822
824 MVT::v2i32, Expand);
826
828 MVT::v4i32, Expand);
829
831 MVT::v8i32, Expand);
832
833 setOperationAction(ISD::BUILD_VECTOR, {MVT::v2i16, MVT::v2f16, MVT::v2bf16},
834 Subtarget->hasVOP3PInsts() ? Legal : Custom);
835
836 setOperationAction(ISD::FNEG, {MVT::v2f16, MVT::v2bf16}, Legal);
837 // This isn't really legal, but this avoids the legalizer unrolling it (and
838 // allows matching fneg (fabs x) patterns)
839 setOperationAction(ISD::FABS, {MVT::v2f16, MVT::v2bf16}, Legal);
840
841 // Can do this in one BFI plus a constant materialize.
843 {MVT::v2f16, MVT::v2bf16, MVT::v4f16, MVT::v4bf16,
844 MVT::v8f16, MVT::v8bf16, MVT::v16f16, MVT::v16bf16,
845 MVT::v32f16, MVT::v32bf16},
846 Custom);
847 if (Subtarget->hasIEEEMinimumMaximumInsts()) {
850 MVT::f16, Legal);
851
854 {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16}, Custom);
855 } else {
858 MVT::f16, Custom);
859
861 Legal);
862
865 {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
866 Custom);
867
869 {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
870 Expand);
871 }
872
873 for (MVT Vec16 :
874 {MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16, MVT::v16f16,
875 MVT::v16bf16, MVT::v32i16, MVT::v32f16, MVT::v32bf16}) {
878 Vec16, Custom);
879 }
880 }
881
882 if (Subtarget->hasVOP3PInsts()) {
886 MVT::v2i16, Legal);
887
890 MVT::v2f16, Legal);
891
893 {MVT::v2i16, MVT::v2f16, MVT::v2bf16}, Custom);
894
896 {MVT::v4f16, MVT::v4i16, MVT::v4bf16, MVT::v8f16,
897 MVT::v8i16, MVT::v8bf16, MVT::v16f16, MVT::v16i16,
898 MVT::v16bf16, MVT::v32f16, MVT::v32i16, MVT::v32bf16},
899 Custom);
900
901 for (MVT VT : {MVT::v4i16, MVT::v8i16, MVT::v16i16, MVT::v32i16})
902 // Split vector operations.
907 VT, Custom);
908
909 for (MVT VT : {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16})
910 // Split vector operations.
913 VT, Custom);
914
915 if (Subtarget->hasIEEEMinimumMaximumInsts()) {
918 MVT::v2f16, Legal);
919 } else {
921 Legal);
922
925 {MVT::v2f16, MVT::v4f16}, Custom);
926 }
927 setOperationAction(ISD::FEXP, MVT::v2f16, Custom);
928 setOperationAction(ISD::SELECT, {MVT::v4i16, MVT::v4f16, MVT::v4bf16},
929 Custom);
930
931 if (Subtarget->hasBF16PackedInsts()) {
935 MVT::v2bf16, Legal);
936
937 for (MVT VT : {MVT::v4bf16, MVT::v8bf16, MVT::v16bf16, MVT::v32bf16})
938 // Split vector operations.
942 VT, Custom);
943 }
944
945 if (Subtarget->hasAnyPackedFP32Ops()) {
947 MVT::v2f32, Legal);
949 {MVT::v4f32, MVT::v8f32, MVT::v16f32, MVT::v32f32},
950 Custom);
951 }
952 if (Subtarget->hasAnyPackedFP64Ops()) {
955 MVT::v2f64, Legal);
958 {MVT::v4f64, MVT::v8f64, MVT::v16f64, MVT::v32f64}, Custom);
959
960 if (Subtarget->hasIEEEMinimumMaximumInsts()) {
963 MVT::v2f64, Legal);
964
967 {MVT::v4f64, MVT::v8f64, MVT::v16f64, MVT::v32f64}, Custom);
968 } else {
970 Legal);
973 MVT::v2f64, Custom);
976 {MVT::v4f64, MVT::v8f64, MVT::v16f64, MVT::v32f64},
977 Custom);
978 }
979 }
980
981 if (Subtarget->hasAnyPackedU64Ops()) {
983 MVT::v2i64, Legal);
985 {MVT::v4i64, MVT::v8i64, MVT::v16i64, MVT::v32i64},
986 Custom);
987 }
988 }
989
991
992 if (Subtarget->has16BitInsts()) {
994 AddPromotedToType(ISD::SELECT, MVT::v2i16, MVT::i32);
996 AddPromotedToType(ISD::SELECT, MVT::v2f16, MVT::i32);
998 AddPromotedToType(ISD::SELECT, MVT::v2bf16, MVT::i32);
999 } else {
1000 // Legalization hack.
1001 setOperationAction(ISD::SELECT, {MVT::v2i16, MVT::v2f16}, Custom);
1002
1004 }
1005
1007 {MVT::v4i16, MVT::v4f16, MVT::v4bf16, MVT::v2i8, MVT::v4i8,
1008 MVT::v8i8, MVT::v8i16, MVT::v8f16, MVT::v8bf16,
1009 MVT::v16i16, MVT::v16f16, MVT::v16bf16, MVT::v32i16,
1010 MVT::v32f16, MVT::v32bf16},
1011 Custom);
1012
1014
1015 if (Subtarget->useVMulU64Inst())
1016 setOperationAction(ISD::MUL, MVT::i64, Legal);
1017 else if (Subtarget->hasScalarSMulU64())
1019
1020 if (Subtarget->hasMad64_32())
1022
1023 if (Subtarget->hasSafeSmemPrefetch() || Subtarget->hasVmemPrefInsts())
1025 // Type legalization cannot expand the i128 pointer of a buffer resource
1026 // prefetch; lowerPREFETCH drops the prefetch instead.
1028
1029 if (Subtarget->hasIEEEMinimumMaximumInsts()) {
1031 {MVT::f16, MVT::f32, MVT::f64, MVT::v2f16}, Legal);
1032 } else {
1033 // FIXME: For nnan fmaximum, emit the fmaximum3 instead of fmaxnum
1034 if (Subtarget->hasMinimum3Maximum3F32())
1036
1037 if (Subtarget->hasMinimum3Maximum3PKF16()) {
1039
1040 // If only the vector form is available, we need to widen to a vector.
1041 if (!Subtarget->hasMinimum3Maximum3F16())
1043 MVT::v2f16);
1044 }
1045 }
1046
1047 if (Subtarget->hasVOP3PInsts()) {
1048 // We want to break these into v2f16 pieces, not scalarize.
1050 {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
1051 Custom);
1052 }
1053
1054 if (Subtarget->useMinMaxI64Insts())
1056 Legal);
1057
1059 {MVT::Other, MVT::f32, MVT::v4f32, MVT::i16, MVT::f16,
1060 MVT::bf16, MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::i128,
1061 MVT::i8},
1062 Custom);
1063
1065 {MVT::v2f16, MVT::v2i16, MVT::v2bf16, MVT::v3f16,
1066 MVT::v3i16, MVT::v4f16, MVT::v4i16, MVT::v4bf16,
1067 MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::Other, MVT::f16,
1068 MVT::i16, MVT::bf16, MVT::i8, MVT::i128},
1069 Custom);
1070
1071 // The s_buffer_load intrinsics accept any result type in IR, but only a few
1072 // of them can be selected. Mark the remaining illegal result types Custom so
1073 // ReplaceNodeResults gets a chance to diagnose them instead of letting the
1074 // type legalizer abort. Its INTRINSIC_WO_CHAIN case dispatches on the
1075 // intrinsic ID, but INTRINSIC_W_CHAIN does not, so remember the types added
1076 // here to keep other chained intrinsics on generic legalization.
1077 for (MVT VT : MVT::all_valuetypes()) {
1078 if (VT.isScalableVector() || isTypeLegal(VT))
1079 continue;
1083 SBufferLoadDiagnosticVTs.set(VT.SimpleTy);
1084 }
1085 }
1086
1088 {MVT::Other, MVT::v2i16, MVT::v2f16, MVT::v2bf16,
1089 MVT::v3i16, MVT::v3f16, MVT::v4f16, MVT::v4i16,
1090 MVT::v4bf16, MVT::v8i16, MVT::v8f16, MVT::v8bf16,
1091 MVT::f16, MVT::i16, MVT::bf16, MVT::i8, MVT::i128},
1092 Custom);
1093
1099
1100 // TODO: Could move this to custom lowering, could benefit from combines on
1101 // extract of relevant bits.
1103
1105
1106 if (Subtarget->hasBF16ConversionInsts()) {
1108 {MVT::bf16, MVT::v2bf16}, Custom);
1110 }
1111
1112 if (Subtarget->hasBF16TransInsts()) {
1114 }
1115
1116 const bool HasE5M3ConversionInsts =
1117 Subtarget->hasFP8ConversionInsts() && Subtarget->hasFP8E5M3Insts();
1118 if (Subtarget->hasOCPFP8ConversionInsts() || HasE5M3ConversionInsts) {
1119 setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP, {MVT::f32, MVT::v2f32},
1120 Custom);
1122
1123 // i8 result promotes to i16, wider vectors split down to v2i8, and v2i8 is
1124 // handled in ReplaceNodeResults before the legalizer splits it per lane.
1125 setOperationAction(ISD::CONVERT_TO_ARBITRARY_FP, {MVT::i16, MVT::v2i8},
1126 Custom);
1127 }
1128
1129 if (Subtarget->hasFP8F16ConversionInsts()) {
1130 setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP, {MVT::f16, MVT::v2f16},
1131 Custom);
1132 }
1133
1134 if (Subtarget->hasCvtPkF16F32Inst()) {
1136 {MVT::v2f16, MVT::v4f16, MVT::v8f16, MVT::v16f16},
1137 Custom);
1138 }
1139
1142 ISD::SUB,
1143 ISD::MUL,
1144 ISD::FADD,
1145 ISD::FSUB,
1146 ISD::FDIV,
1147 ISD::FMUL,
1156 ISD::FMA,
1157 ISD::ABS,
1158 ISD::SMIN,
1159 ISD::SMAX,
1160 ISD::UMIN,
1161 ISD::UMAX,
1162 ISD::SETCC,
1164 ISD::SMIN,
1165 ISD::SMAX,
1166 ISD::UMIN,
1167 ISD::UMAX,
1170 ISD::AND,
1171 ISD::OR,
1172 ISD::XOR,
1173 ISD::SHL,
1174 ISD::SRL,
1175 ISD::SRA,
1176 ISD::FSHR,
1188
1189 if (Subtarget->has16BitInsts() && !Subtarget->hasMed3_16())
1191
1192 // All memory operations. Some folding on the pointer operand is done to help
1193 // matching the constant offsets in the addressing modes.
1195 ISD::STORE,
1220
1221 // FIXME: In other contexts we pretend this is a per-function property.
1223
1225}
1226
1227const GCNSubtarget *SITargetLowering::getSubtarget() const { return Subtarget; }
1228
1230 static const MCPhysReg RCRegs[] = {AMDGPU::MODE};
1231 return RCRegs;
1232}
1233
1234//===----------------------------------------------------------------------===//
1235// TargetLowering queries
1236//===----------------------------------------------------------------------===//
1237
1238// v_mad_mix* support a conversion from f16 to f32.
1239//
1240// There is only one special case when denormals are enabled we don't currently,
1241// where this is OK to use.
1242bool SITargetLowering::isFPExtFoldable(const SelectionDAG &DAG, unsigned Opcode,
1243 EVT DestVT, EVT SrcVT) const {
1244 return DestVT.getScalarType() == MVT::f32 &&
1245 ((((Opcode == ISD::FMAD && Subtarget->hasMadMixInsts()) ||
1246 (Opcode == ISD::FMA && Subtarget->hasFmaMixInsts())) &&
1247 SrcVT.getScalarType() == MVT::f16) ||
1248 (Opcode == ISD::FMA && Subtarget->hasFmaMixBF16Insts() &&
1249 SrcVT.getScalarType() == MVT::bf16)) &&
1250 // TODO: This probably only requires no input flushing?
1252}
1253
1255 LLT DestTy, LLT SrcTy) const {
1256 return ((Opcode == TargetOpcode::G_FMAD && Subtarget->hasMadMixInsts()) ||
1257 (Opcode == TargetOpcode::G_FMA && Subtarget->hasFmaMixInsts())) &&
1258 DestTy.getScalarSizeInBits() == 32 &&
1259 SrcTy.getScalarSizeInBits() == 16 &&
1260 // TODO: This probably only requires no input flushing?
1261 denormalModeIsFlushAllF32(*MI.getMF());
1262}
1263
1265 // SI has some legal vector types, but no legal vector operations. Say no
1266 // shuffles are legal in order to prefer scalarizing some vector operations.
1267 return false;
1268}
1269
1272 EVT VT) const {
1275
1276 if (VT.isVector()) {
1277 EVT ScalarVT = VT.getScalarType();
1278 unsigned Size = ScalarVT.getSizeInBits();
1279 if (Size == 16) {
1280 return Subtarget->has16BitInsts()
1281 ? MVT::getVectorVT(ScalarVT.getSimpleVT(), 2)
1282 : MVT::i32;
1283 }
1284
1285 if (Size < 16)
1286 return Subtarget->has16BitInsts() ? MVT::i16 : MVT::i32;
1287 return Size == 32 ? ScalarVT.getSimpleVT() : MVT::i32;
1288 }
1289
1290 if (!Subtarget->has16BitInsts() && VT.getSizeInBits() == 16)
1291 return MVT::i32;
1292
1293 if (VT.getSizeInBits() > 32)
1294 return MVT::i32;
1295
1297}
1298
1301 EVT VT) const {
1304
1305 if (VT.isVector()) {
1306 unsigned NumElts = VT.getVectorNumElements();
1307 EVT ScalarVT = VT.getScalarType();
1308 unsigned Size = ScalarVT.getSizeInBits();
1309
1310 // FIXME: Should probably promote 8-bit vectors to i16.
1311 if (Size == 16)
1312 return (NumElts + 1) / 2;
1313
1314 if (Size <= 32)
1315 return NumElts;
1316
1317 if (Size > 32)
1318 return NumElts * ((Size + 31) / 32);
1319 } else if (VT.getSizeInBits() > 32)
1320 return (VT.getSizeInBits() + 31) / 32;
1321
1323}
1324
1326 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
1327 unsigned &NumIntermediates, MVT &RegisterVT) const {
1328 if (CC != CallingConv::AMDGPU_KERNEL && VT.isVector()) {
1329 unsigned NumElts = VT.getVectorNumElements();
1330 EVT ScalarVT = VT.getScalarType();
1331 unsigned Size = ScalarVT.getSizeInBits();
1332 // FIXME: We should fix the ABI to be the same on targets without 16-bit
1333 // support, but unless we can properly handle 3-vectors, it will be still be
1334 // inconsistent.
1335 if (Size == 16) {
1336 MVT SimpleIntermediateVT =
1338 IntermediateVT = SimpleIntermediateVT;
1339 RegisterVT = Subtarget->has16BitInsts() ? SimpleIntermediateVT : MVT::i32;
1340 NumIntermediates = (NumElts + 1) / 2;
1341 return (NumElts + 1) / 2;
1342 }
1343
1344 if (Size == 32) {
1345 RegisterVT = ScalarVT.getSimpleVT();
1346 IntermediateVT = RegisterVT;
1347 NumIntermediates = NumElts;
1348 return NumIntermediates;
1349 }
1350
1351 if (Size < 16 && Subtarget->has16BitInsts()) {
1352 // FIXME: Should probably form v2i16 pieces
1353 RegisterVT = MVT::i16;
1354 IntermediateVT = ScalarVT;
1355 NumIntermediates = NumElts;
1356 return NumIntermediates;
1357 }
1358
1359 if (Size != 16 && Size <= 32) {
1360 RegisterVT = MVT::i32;
1361 IntermediateVT = ScalarVT;
1362 NumIntermediates = NumElts;
1363 return NumIntermediates;
1364 }
1365
1366 if (Size > 32) {
1367 RegisterVT = MVT::i32;
1368 IntermediateVT = RegisterVT;
1369 NumIntermediates = NumElts * ((Size + 31) / 32);
1370 return NumIntermediates;
1371 }
1372 }
1373
1375 Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
1376}
1377
1379 const DataLayout &DL, Type *Ty,
1380 unsigned MaxNumLanes) {
1381 assert(MaxNumLanes != 0);
1382
1383 LLVMContext &Ctx = Ty->getContext();
1384 if (auto *VT = dyn_cast<FixedVectorType>(Ty)) {
1385 unsigned NumElts = std::min(MaxNumLanes, VT->getNumElements());
1386 return EVT::getVectorVT(Ctx, TLI.getValueType(DL, VT->getElementType()),
1387 NumElts);
1388 }
1389
1390 return TLI.getValueType(DL, Ty);
1391}
1392
1393// Peek through TFE struct returns to only use the data size.
1395 const DataLayout &DL, Type *Ty,
1396 unsigned MaxNumLanes) {
1397 auto *ST = dyn_cast<StructType>(Ty);
1398 if (!ST)
1399 return memVTFromLoadIntrData(TLI, DL, Ty, MaxNumLanes);
1400
1401 // TFE intrinsics return an aggregate type.
1402 assert(ST->getNumContainedTypes() == 2 &&
1403 ST->getContainedType(1)->isIntegerTy(32));
1404 return memVTFromLoadIntrData(TLI, DL, ST->getContainedType(0), MaxNumLanes);
1405}
1406
1407/// Map address space 7 to MVT::amdgpuBufferFatPointer because that's its
1408/// in-memory representation. This return value is a custom type because there
1409/// is no MVT::i160 and adding one breaks integer promotion logic. While this
1410/// could cause issues during codegen, these address space 7 pointers will be
1411/// rewritten away by then. Therefore, we can return MVT::amdgpuBufferFatPointer
1412/// in order to allow pre-codegen passes that query TargetTransformInfo, often
1413/// for cost modeling, to work. (This also sets us up decently for doing the
1414/// buffer lowering in GlobalISel if SelectionDAG ever goes away.)
1416 if (AMDGPUAS::BUFFER_FAT_POINTER == AS && DL.getPointerSizeInBits(AS) == 160)
1417 return MVT::amdgpuBufferFatPointer;
1419 DL.getPointerSizeInBits(AS) == 192)
1420 return MVT::amdgpuBufferStridedPointer;
1422}
1423/// Similarly, the in-memory representation of a p7 is {p8, i32}, aka
1424/// v8i32 when padding is added.
1425/// The in-memory representation of a p9 is {p8, i32, i32}, which is
1426/// also v8i32 with padding.
1428 if ((AMDGPUAS::BUFFER_FAT_POINTER == AS &&
1429 DL.getPointerSizeInBits(AS) == 160) ||
1431 DL.getPointerSizeInBits(AS) == 192))
1432 return MVT::v8i32;
1434}
1435
1436static unsigned getIntrMemWidth(unsigned IntrID) {
1437 switch (IntrID) {
1438 case Intrinsic::amdgcn_global_load_async_to_lds_b8:
1439 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
1440 case Intrinsic::amdgcn_global_store_async_from_lds_b8:
1441 return 8;
1442 case Intrinsic::amdgcn_global_load_async_to_lds_b32:
1443 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
1444 case Intrinsic::amdgcn_global_store_async_from_lds_b32:
1445 case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
1446 case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
1447 case Intrinsic::amdgcn_flat_load_monitor_b32:
1448 case Intrinsic::amdgcn_global_load_monitor_b32:
1449 return 32;
1450 case Intrinsic::amdgcn_global_load_async_to_lds_b64:
1451 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
1452 case Intrinsic::amdgcn_global_store_async_from_lds_b64:
1453 case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
1454 case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
1455 case Intrinsic::amdgcn_flat_load_monitor_b64:
1456 case Intrinsic::amdgcn_global_load_monitor_b64:
1457 return 64;
1458 case Intrinsic::amdgcn_global_load_async_to_lds_b128:
1459 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128:
1460 case Intrinsic::amdgcn_global_store_async_from_lds_b128:
1461 case Intrinsic::amdgcn_cooperative_atomic_load_8x16B:
1462 case Intrinsic::amdgcn_cooperative_atomic_store_8x16B:
1463 case Intrinsic::amdgcn_flat_load_monitor_b128:
1464 case Intrinsic::amdgcn_global_load_monitor_b128:
1465 return 128;
1466 default:
1467 llvm_unreachable("Unknown width");
1468 }
1469}
1470
1472 unsigned ArgIdx) {
1473 Value *OrderingArg = CI.getArgOperand(ArgIdx);
1474 unsigned Ord = cast<ConstantInt>(OrderingArg)->getZExtValue();
1475 switch (AtomicOrderingCABI(Ord)) {
1478 break;
1481 break;
1484 break;
1485 default:
1487 }
1488}
1489
1490static unsigned parseSyncscopeMDArg(const CallBase &CI, unsigned ArgIdx) {
1491 MDNode *ScopeMD = cast<MDNode>(
1492 cast<MetadataAsValue>(CI.getArgOperand(ArgIdx))->getMetadata());
1493 StringRef Scope = cast<MDString>(ScopeMD->getOperand(0))->getString();
1494 return CI.getContext().getOrInsertSyncScopeID(Scope);
1495}
1496
1498 const CallBase &CI,
1499 MachineFunction &MF,
1500 unsigned IntrID) const {
1502 if (CI.hasMetadata(LLVMContext::MD_invariant_load))
1504 if (CI.hasMetadata(LLVMContext::MD_nontemporal))
1506 Flags |= getTargetMMOFlags(CI);
1507
1508 if (const AMDGPU::RsrcIntrinsic *RsrcIntr =
1510 AttributeSet Attr =
1512 MemoryEffects ME = Attr.getMemoryEffects();
1513 if (ME.doesNotAccessMemory())
1514 return;
1515
1516 const AMDGPU::BufferIntrinsicInfo *BufInfo =
1518 bool IsSPrefetch = BufInfo && BufInfo->Kind == AMDGPU::BufferKindPrefetch;
1519 std::optional<unsigned> AuxArg =
1520 BufInfo ? BufInfo->getAuxArg() : CI.arg_size() - 1;
1521 if (AuxArg) {
1522 auto *Aux = cast<ConstantInt>(CI.getArgOperand(*AuxArg));
1523 if (Aux->getZExtValue() & AMDGPU::CPol::VOLATILE)
1525 }
1526
1528
1529 IntrinsicInfo Info;
1530 // TODO: Should images get their own address space?
1532
1533 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode = nullptr;
1534 if (RsrcIntr->IsImage) {
1535 const AMDGPU::ImageDimIntrinsicInfo *Intr =
1537 BaseOpcode = AMDGPU::getMIMGBaseOpcodeInfo(Intr->BaseOpcode);
1538 Info.align.reset();
1539 }
1540
1541 Value *RsrcArg = CI.getArgOperand(RsrcIntr->RsrcArg);
1542 if (auto *RsrcPtrTy = dyn_cast<PointerType>(RsrcArg->getType())) {
1543 if (RsrcPtrTy->getAddressSpace() == AMDGPUAS::BUFFER_RESOURCE)
1544 // We conservatively set the memory operand of a buffer intrinsic to the
1545 // base resource pointer, so that we can access alias information about
1546 // those pointers. Cases like "this points at the same value
1547 // but with a different offset" are handled in
1548 // areMemAccessesTriviallyDisjoint.
1549 Info.ptrVal = RsrcArg;
1550 }
1551
1552 if (ME.onlyReadsMemory()) {
1553 if (RsrcIntr->IsImage) {
1554 unsigned MaxNumLanes = 4;
1555
1556 if (!BaseOpcode->Gather4) {
1557 // If this isn't a gather, we may have excess loaded elements in the
1558 // IR type. Check the dmask for the real number of elements loaded.
1559 unsigned DMask =
1560 cast<ConstantInt>(CI.getArgOperand(0))->getZExtValue();
1561 MaxNumLanes = DMask == 0 ? 1 : llvm::popcount(DMask);
1562 }
1563
1564 Info.memVT = memVTFromLoadIntrReturn(*this, MF.getDataLayout(),
1565 CI.getType(), MaxNumLanes);
1566 } else {
1567 Info.memVT =
1569 std::numeric_limits<unsigned>::max());
1570 }
1571
1572 // FIXME: What does alignment mean for an image?
1573 Info.opc = ISD::INTRINSIC_W_CHAIN;
1574 Info.flags = Flags | MachineMemOperand::MOLoad;
1575 } else if (ME.onlyWritesMemory()) {
1576 Info.opc = ISD::INTRINSIC_VOID;
1577
1578 Type *DataTy = CI.getArgOperand(0)->getType();
1579 if (RsrcIntr->IsImage) {
1580 unsigned DMask = cast<ConstantInt>(CI.getArgOperand(1))->getZExtValue();
1581 unsigned DMaskLanes = DMask == 0 ? 1 : llvm::popcount(DMask);
1582 Info.memVT = memVTFromLoadIntrData(*this, MF.getDataLayout(), DataTy,
1583 DMaskLanes);
1584 } else
1585 Info.memVT = getValueType(MF.getDataLayout(), DataTy);
1586
1587 Info.flags = Flags | MachineMemOperand::MOStore;
1588 } else {
1589 // Atomic, NoReturn Sampler or prefetch
1590 Info.opc = CI.getType()->isVoidTy() ? ISD::INTRINSIC_VOID
1592
1593 if (BufInfo && BufInfo->Kind == AMDGPU::BufferKindLoadToLDS) {
1594 unsigned Width =
1596 ->getZExtValue();
1597
1598 // Entry 0: Load from buffer.
1599 // Don't set an offset, since the pointer value always represents the
1600 // base of the buffer.
1601 Info.memVT = EVT::getIntegerVT(CI.getContext(), Width * 8);
1602 Info.flags = Flags | MachineMemOperand::MOLoad;
1603 Infos.push_back(Info);
1604
1605 // Entry 1: Store to LDS.
1606 // Instruction offset is applied, and an additional per-lane offset
1607 // which we simulate using a larger memory type.
1608 Info.memVT = EVT::getIntegerVT(
1609 CI.getContext(), Width * 8 * Subtarget->getWavefrontSize());
1610 Info.ptrVal = CI.getArgOperand(*BufInfo->getLDSPtrArg());
1611 Info.offset =
1613 ->getZExtValue();
1614 Info.fallbackAddressSpace = AMDGPUAS::LOCAL_ADDRESS;
1615 Info.flags = Flags | MachineMemOperand::MOStore;
1616 Infos.push_back(Info);
1617 return;
1618 }
1619 if (BufInfo && BufInfo->Kind == AMDGPU::BufferKindAtomicLoad) {
1620 Info.memVT =
1622 std::numeric_limits<unsigned>::max());
1623 Info.flags = Flags | MachineMemOperand::MOLoad;
1624 Infos.push_back(Info);
1625 return;
1626 }
1627
1628 Info.flags = Flags | MachineMemOperand::MOLoad;
1629 if (!IsSPrefetch)
1630 Info.flags |= MachineMemOperand::MOStore;
1631
1632 if ((RsrcIntr->IsImage && BaseOpcode->NoReturn) || IsSPrefetch) {
1633 // Fake memory access type for no return sampler intrinsics
1634 Info.memVT = MVT::i32;
1635 } else {
1636 // XXX - Should this be volatile without known ordering?
1637 Info.flags |= MachineMemOperand::MOVolatile;
1638 Info.memVT = MVT::getVT(CI.getArgOperand(0)->getType());
1639 }
1640 }
1641 Infos.push_back(Info);
1642 return;
1643 }
1644
1645 IntrinsicInfo Info;
1646 switch (IntrID) {
1647 case Intrinsic::amdgcn_ds_ordered_add:
1648 case Intrinsic::amdgcn_ds_ordered_swap: {
1649 Info.opc = ISD::INTRINSIC_W_CHAIN;
1650 Info.memVT = MVT::getVT(CI.getType());
1651 Info.ptrVal = CI.getOperand(0);
1652 Info.align.reset();
1654
1655 const ConstantInt *Vol = cast<ConstantInt>(CI.getOperand(4));
1656 if (!Vol->isZero())
1657 Info.flags |= MachineMemOperand::MOVolatile;
1658
1659 Infos.push_back(Info);
1660 return;
1661 }
1662 case Intrinsic::amdgcn_ds_add_gs_reg_rtn:
1663 case Intrinsic::amdgcn_ds_sub_gs_reg_rtn: {
1664 Info.opc = ISD::INTRINSIC_W_CHAIN;
1665 Info.memVT = MVT::getVT(CI.getOperand(0)->getType());
1666 Info.ptrVal = nullptr;
1667 Info.fallbackAddressSpace = AMDGPUAS::STREAMOUT_REGISTER;
1669 Infos.push_back(Info);
1670 return;
1671 }
1672 case Intrinsic::amdgcn_ds_append:
1673 case Intrinsic::amdgcn_ds_consume: {
1674 Info.opc = ISD::INTRINSIC_W_CHAIN;
1675 Info.memVT = MVT::getVT(CI.getType());
1676 Info.ptrVal = CI.getOperand(0);
1677 Info.align.reset();
1679
1680 const ConstantInt *Vol = cast<ConstantInt>(CI.getOperand(1));
1681 if (!Vol->isZero())
1682 Info.flags |= MachineMemOperand::MOVolatile;
1683
1684 Infos.push_back(Info);
1685 return;
1686 }
1687 case Intrinsic::amdgcn_ds_atomic_async_barrier_arrive_b64:
1688 case Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64: {
1689 Info.opc = (IntrID == Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64)
1692 Info.memVT = MVT::getVT(CI.getType());
1693 Info.ptrVal = CI.getOperand(0);
1694 Info.memVT = MVT::i64;
1695 Info.size = 8;
1696 Info.align.reset();
1698 Info.order = AtomicOrdering::Monotonic;
1699 Infos.push_back(Info);
1700 return;
1701 }
1702 case Intrinsic::amdgcn_image_bvh_dual_intersect_ray:
1703 case Intrinsic::amdgcn_image_bvh_intersect_ray:
1704 case Intrinsic::amdgcn_image_bvh8_intersect_ray: {
1705 Info.opc = ISD::INTRINSIC_W_CHAIN;
1706 Info.memVT =
1707 MVT::getVT(IntrID == Intrinsic::amdgcn_image_bvh_intersect_ray
1708 ? CI.getType()
1710 ->getElementType(0)); // XXX: what is correct VT?
1711
1712 Info.fallbackAddressSpace = AMDGPUAS::BUFFER_RESOURCE;
1713 Info.align.reset();
1714 Info.flags = Flags | MachineMemOperand::MOLoad |
1716 Infos.push_back(Info);
1717 return;
1718 }
1719 case Intrinsic::amdgcn_global_atomic_fmin_num:
1720 case Intrinsic::amdgcn_global_atomic_fmax_num:
1721 case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
1722 case Intrinsic::amdgcn_flat_atomic_fmin_num:
1723 case Intrinsic::amdgcn_flat_atomic_fmax_num: {
1724 Info.opc = ISD::INTRINSIC_W_CHAIN;
1725 Info.memVT = MVT::getVT(CI.getType());
1726 Info.ptrVal = CI.getOperand(0);
1727 Info.align.reset();
1728 Info.flags =
1731 Infos.push_back(Info);
1732 return;
1733 }
1734 case Intrinsic::amdgcn_cluster_load_b32:
1735 case Intrinsic::amdgcn_cluster_load_b64:
1736 case Intrinsic::amdgcn_cluster_load_b128:
1737 case Intrinsic::amdgcn_ds_load_tr6_b96:
1738 case Intrinsic::amdgcn_ds_load_tr4_b64:
1739 case Intrinsic::amdgcn_ds_load_tr8_b64:
1740 case Intrinsic::amdgcn_ds_load_tr16_b128:
1741 case Intrinsic::amdgcn_global_load_tr6_b96:
1742 case Intrinsic::amdgcn_global_load_tr4_b64:
1743 case Intrinsic::amdgcn_global_load_tr_b64:
1744 case Intrinsic::amdgcn_global_load_tr_b128:
1745 case Intrinsic::amdgcn_ds_read_tr4_b64:
1746 case Intrinsic::amdgcn_ds_read_tr6_b96:
1747 case Intrinsic::amdgcn_ds_read_tr8_b64:
1748 case Intrinsic::amdgcn_ds_read_tr16_b64: {
1749 Info.opc = ISD::INTRINSIC_W_CHAIN;
1750 Info.memVT = MVT::getVT(CI.getType());
1751 Info.ptrVal = CI.getOperand(0);
1752 Info.align.reset();
1753 Info.flags = Flags | MachineMemOperand::MOLoad;
1754 Infos.push_back(Info);
1755 return;
1756 }
1757 case Intrinsic::amdgcn_flat_load_monitor_b32:
1758 case Intrinsic::amdgcn_flat_load_monitor_b64:
1759 case Intrinsic::amdgcn_flat_load_monitor_b128:
1760 case Intrinsic::amdgcn_global_load_monitor_b32:
1761 case Intrinsic::amdgcn_global_load_monitor_b64:
1762 case Intrinsic::amdgcn_global_load_monitor_b128: {
1763 Info.opc = ISD::INTRINSIC_W_CHAIN;
1764 Info.memVT = EVT::getIntegerVT(CI.getContext(), getIntrMemWidth(IntrID));
1765 Info.ptrVal = CI.getOperand(0);
1766 Info.align.reset();
1767 Info.flags = MachineMemOperand::MOLoad;
1768 Info.order = parseAtomicOrderingCABIArg(CI, 1);
1769 Info.ssid = parseSyncscopeMDArg(CI, 2);
1770 Infos.push_back(Info);
1771 return;
1772 }
1773 case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
1774 case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
1775 case Intrinsic::amdgcn_cooperative_atomic_load_8x16B: {
1776 Info.opc = ISD::INTRINSIC_W_CHAIN;
1777 Info.memVT = MVT::getVT(CI.getType());
1778 Info.ptrVal = CI.getOperand(0);
1779 Info.align.reset();
1781 Info.order = parseAtomicOrderingCABIArg(CI, 1);
1782 Info.ssid = parseSyncscopeMDArg(CI, 2);
1783 Infos.push_back(Info);
1784 return;
1785 }
1786 case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
1787 case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
1788 case Intrinsic::amdgcn_cooperative_atomic_store_8x16B: {
1789 Info.opc = ISD::INTRINSIC_VOID;
1790 Info.memVT = MVT::getVT(CI.getArgOperand(1)->getType());
1791 Info.ptrVal = CI.getArgOperand(0);
1792 Info.align.reset();
1794 Info.order = parseAtomicOrderingCABIArg(CI, 2);
1795 Info.ssid = parseSyncscopeMDArg(CI, 3);
1796 Infos.push_back(Info);
1797 return;
1798 }
1799 case Intrinsic::amdgcn_ds_gws_init:
1800 case Intrinsic::amdgcn_ds_gws_barrier:
1801 case Intrinsic::amdgcn_ds_gws_sema_v:
1802 case Intrinsic::amdgcn_ds_gws_sema_br:
1803 case Intrinsic::amdgcn_ds_gws_sema_p:
1804 case Intrinsic::amdgcn_ds_gws_sema_release_all: {
1805 Info.opc = ISD::INTRINSIC_VOID;
1806
1807 const GCNTargetMachine &TM =
1808 static_cast<const GCNTargetMachine &>(getTargetMachine());
1809
1811 Info.ptrVal = MFI->getGWSPSV(TM);
1812
1813 // This is an abstract access, but we need to specify a type and size.
1814 Info.memVT = MVT::i32;
1815 Info.size = 4;
1816 Info.align = Align(4);
1817
1818 if (IntrID == Intrinsic::amdgcn_ds_gws_barrier)
1819 Info.flags = Flags | MachineMemOperand::MOLoad;
1820 else
1821 Info.flags = Flags | MachineMemOperand::MOStore;
1822 Infos.push_back(Info);
1823 return;
1824 }
1825 case Intrinsic::amdgcn_global_load_async_to_lds_b8:
1826 case Intrinsic::amdgcn_global_load_async_to_lds_b32:
1827 case Intrinsic::amdgcn_global_load_async_to_lds_b64:
1828 case Intrinsic::amdgcn_global_load_async_to_lds_b128:
1829 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
1830 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
1831 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
1832 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128: {
1833 // Entry 0: Load from source (global/flat).
1834 Info.opc = ISD::INTRINSIC_VOID;
1835 Info.memVT = EVT::getIntegerVT(CI.getContext(), getIntrMemWidth(IntrID));
1836 Info.ptrVal = CI.getArgOperand(0); // Global pointer
1837 Info.offset = cast<ConstantInt>(CI.getArgOperand(2))->getSExtValue();
1838 Info.flags = Flags | MachineMemOperand::MOLoad;
1839 Infos.push_back(Info);
1840
1841 // Entry 1: Store to LDS (same offset).
1842 Info.flags = Flags | MachineMemOperand::MOStore;
1843 Info.ptrVal = CI.getArgOperand(1); // LDS pointer
1844 Infos.push_back(Info);
1845 return;
1846 }
1847 case Intrinsic::amdgcn_global_store_async_from_lds_b8:
1848 case Intrinsic::amdgcn_global_store_async_from_lds_b32:
1849 case Intrinsic::amdgcn_global_store_async_from_lds_b64:
1850 case Intrinsic::amdgcn_global_store_async_from_lds_b128: {
1851 // Entry 0: Load from LDS.
1852 Info.opc = ISD::INTRINSIC_VOID;
1853 Info.memVT = EVT::getIntegerVT(CI.getContext(), getIntrMemWidth(IntrID));
1854 Info.ptrVal = CI.getArgOperand(1); // LDS pointer
1855 Info.offset = cast<ConstantInt>(CI.getArgOperand(2))->getSExtValue();
1856 Info.flags = Flags | MachineMemOperand::MOLoad;
1857 Infos.push_back(Info);
1858
1859 // Entry 1: Store to global (same offset).
1860 Info.flags = Flags | MachineMemOperand::MOStore;
1861 Info.ptrVal = CI.getArgOperand(0); // Global pointer
1862 Infos.push_back(Info);
1863 return;
1864 }
1865 case Intrinsic::amdgcn_av_load_b8:
1866 case Intrinsic::amdgcn_av_load_b16:
1867 case Intrinsic::amdgcn_av_load_b32:
1868 case Intrinsic::amdgcn_av_load_b64:
1869 case Intrinsic::amdgcn_av_load_b128:
1870 case Intrinsic::amdgcn_av_store_b8:
1871 case Intrinsic::amdgcn_av_store_b16:
1872 case Intrinsic::amdgcn_av_store_b32:
1873 case Intrinsic::amdgcn_av_store_b64:
1874 case Intrinsic::amdgcn_av_store_b128: {
1875 bool IsStore = CI.getType()->isVoidTy();
1876 Type *DataTy = IsStore ? CI.getArgOperand(1)->getType() : CI.getType();
1877 Info.opc = IsStore ? ISD::INTRINSIC_VOID : ISD::INTRINSIC_W_CHAIN;
1878 Info.memVT = MVT::getVT(DataTy);
1879 Info.ptrVal = CI.getArgOperand(0);
1880 Info.align = Align(Info.memVT.getStoreSize());
1881 Info.flags |=
1883 // Pretend to be atomic so that SIMemoryLegalizer::expandStore sets cache
1884 // flags appropriately.
1885 Info.order = AtomicOrdering::Monotonic;
1886
1887 LLVMContext &Ctx = CI.getContext();
1888 unsigned ScopeIdx = CI.arg_size() - 1;
1889 MDNode *ScopeMD = cast<MDNode>(
1890 cast<MetadataAsValue>(CI.getArgOperand(ScopeIdx))->getMetadata());
1891 StringRef Scope = cast<MDString>(ScopeMD->getOperand(0))->getString();
1892 Info.ssid = Ctx.getOrInsertSyncScopeID(Scope);
1893 Infos.push_back(Info);
1894 return;
1895 }
1896 case Intrinsic::amdgcn_load_to_lds:
1897 case Intrinsic::amdgcn_load_async_to_lds:
1898 case Intrinsic::amdgcn_global_load_lds:
1899 case Intrinsic::amdgcn_global_load_async_lds: {
1900 unsigned Width = cast<ConstantInt>(CI.getArgOperand(2))->getZExtValue();
1901 auto *Aux = cast<ConstantInt>(CI.getArgOperand(CI.arg_size() - 1));
1902 bool IsVolatile = Aux->getZExtValue() & AMDGPU::CPol::VOLATILE;
1903 if (IsVolatile)
1905
1906 // Entry 0: Load from source (global/flat).
1907 Info.opc = ISD::INTRINSIC_VOID;
1908 Info.memVT = EVT::getIntegerVT(CI.getContext(), Width * 8);
1909 Info.ptrVal = CI.getArgOperand(0); // Source pointer
1910 Info.offset = cast<ConstantInt>(CI.getArgOperand(3))->getSExtValue();
1911 Info.flags = Flags | MachineMemOperand::MOLoad;
1912 Infos.push_back(Info);
1913
1914 // Entry 1: Store to LDS.
1915 // Same offset from the instruction, but an additional per-lane offset is
1916 // added. Represent that using a wider memory type.
1917 Info.memVT = EVT::getIntegerVT(CI.getContext(),
1918 Width * 8 * Subtarget->getWavefrontSize());
1919 Info.ptrVal = CI.getArgOperand(1); // LDS destination pointer
1920 Info.flags = Flags | MachineMemOperand::MOStore;
1921 Infos.push_back(Info);
1922 return;
1923 }
1924 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
1925 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
1926 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
1927 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn: {
1928 Info.opc = ISD::INTRINSIC_W_CHAIN;
1929
1930 const GCNTargetMachine &TM =
1931 static_cast<const GCNTargetMachine &>(getTargetMachine());
1932
1934 Info.ptrVal = MFI->getGWSPSV(TM);
1935
1936 // This is an abstract access, but we need to specify a type and size.
1937 Info.memVT = MVT::i32;
1938 Info.size = 4;
1939 Info.align = Align(4);
1940
1942 Infos.push_back(Info);
1943 return;
1944 }
1945 case Intrinsic::amdgcn_s_prefetch_data:
1946 case Intrinsic::amdgcn_s_prefetch_inst:
1947 case Intrinsic::amdgcn_flat_prefetch:
1948 case Intrinsic::amdgcn_global_prefetch: {
1949 Info.opc = ISD::INTRINSIC_VOID;
1950 Info.memVT = EVT::getIntegerVT(CI.getContext(), 8);
1951 Info.ptrVal = CI.getArgOperand(0);
1952 Info.flags = Flags | MachineMemOperand::MOLoad;
1953 Infos.push_back(Info);
1954 return;
1955 }
1956 default:
1957 return;
1958 }
1959}
1960
1963 Type *&AccessTy) const {
1964 Value *Ptr = nullptr;
1965 switch (II->getIntrinsicID()) {
1966 case Intrinsic::amdgcn_cluster_load_b128:
1967 case Intrinsic::amdgcn_cluster_load_b64:
1968 case Intrinsic::amdgcn_cluster_load_b32:
1969 case Intrinsic::amdgcn_ds_append:
1970 case Intrinsic::amdgcn_ds_consume:
1971 case Intrinsic::amdgcn_ds_load_tr8_b64:
1972 case Intrinsic::amdgcn_ds_load_tr16_b128:
1973 case Intrinsic::amdgcn_ds_load_tr4_b64:
1974 case Intrinsic::amdgcn_ds_load_tr6_b96:
1975 case Intrinsic::amdgcn_ds_read_tr4_b64:
1976 case Intrinsic::amdgcn_ds_read_tr6_b96:
1977 case Intrinsic::amdgcn_ds_read_tr8_b64:
1978 case Intrinsic::amdgcn_ds_read_tr16_b64:
1979 case Intrinsic::amdgcn_ds_ordered_add:
1980 case Intrinsic::amdgcn_ds_ordered_swap:
1981 case Intrinsic::amdgcn_ds_atomic_async_barrier_arrive_b64:
1982 case Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64:
1983 case Intrinsic::amdgcn_flat_atomic_fmax_num:
1984 case Intrinsic::amdgcn_flat_atomic_fmin_num:
1985 case Intrinsic::amdgcn_global_atomic_fmax_num:
1986 case Intrinsic::amdgcn_global_atomic_fmin_num:
1987 case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
1988 case Intrinsic::amdgcn_global_load_tr_b64:
1989 case Intrinsic::amdgcn_global_load_tr_b128:
1990 case Intrinsic::amdgcn_global_load_tr4_b64:
1991 case Intrinsic::amdgcn_global_load_tr6_b96:
1992 case Intrinsic::amdgcn_global_store_async_from_lds_b8:
1993 case Intrinsic::amdgcn_global_store_async_from_lds_b32:
1994 case Intrinsic::amdgcn_global_store_async_from_lds_b64:
1995 case Intrinsic::amdgcn_global_store_async_from_lds_b128:
1996 case Intrinsic::amdgcn_av_load_b8:
1997 case Intrinsic::amdgcn_av_load_b16:
1998 case Intrinsic::amdgcn_av_load_b32:
1999 case Intrinsic::amdgcn_av_load_b64:
2000 case Intrinsic::amdgcn_av_load_b128:
2001 case Intrinsic::amdgcn_av_store_b8:
2002 case Intrinsic::amdgcn_av_store_b16:
2003 case Intrinsic::amdgcn_av_store_b32:
2004 case Intrinsic::amdgcn_av_store_b64:
2005 case Intrinsic::amdgcn_av_store_b128:
2006 Ptr = II->getArgOperand(0);
2007 break;
2008 case Intrinsic::amdgcn_load_to_lds:
2009 case Intrinsic::amdgcn_load_async_to_lds:
2010 case Intrinsic::amdgcn_global_load_lds:
2011 case Intrinsic::amdgcn_global_load_async_lds:
2012 case Intrinsic::amdgcn_global_load_async_to_lds_b8:
2013 case Intrinsic::amdgcn_global_load_async_to_lds_b32:
2014 case Intrinsic::amdgcn_global_load_async_to_lds_b64:
2015 case Intrinsic::amdgcn_global_load_async_to_lds_b128:
2016 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
2017 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
2018 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
2019 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128:
2020 Ptr = II->getArgOperand(1);
2021 break;
2022 default:
2023 return false;
2024 }
2025 AccessTy = II->getType();
2026 Ops.push_back(Ptr);
2027 return true;
2028}
2029
2031 unsigned AddrSpace) const {
2032 if (!Subtarget->hasFlatInstOffsets()) {
2033 // Flat instructions do not have offsets, and only have the register
2034 // address.
2035 return AM.BaseOffs == 0 && AM.Scale == 0;
2036 }
2037
2039 FlatAddrSpace FlatVariant =
2040 AddrSpace == AMDGPUAS::GLOBAL_ADDRESS ? FlatAddrSpace::FlatGlobal
2041 : AddrSpace == AMDGPUAS::PRIVATE_ADDRESS ? FlatAddrSpace::FlatScratch
2042 : FlatAddrSpace::FLAT;
2043
2044 return AM.Scale == 0 &&
2045 (AM.BaseOffs == 0 || Subtarget->getInstrInfo()->isLegalFLATOffset(
2046 AM.BaseOffs, AddrSpace, FlatVariant));
2047}
2048
2050 if (Subtarget->hasFlatGlobalInsts())
2052
2053 if (!Subtarget->hasAddr64() || Subtarget->useFlatForGlobal()) {
2054 // Assume the we will use FLAT for all global memory accesses
2055 // on VI.
2056 // FIXME: This assumption is currently wrong. On VI we still use
2057 // MUBUF instructions for the r + i addressing mode. As currently
2058 // implemented, the MUBUF instructions only work on buffer < 4GB.
2059 // It may be possible to support > 4GB buffers with MUBUF instructions,
2060 // by setting the stride value in the resource descriptor which would
2061 // increase the size limit to (stride * 4GB). However, this is risky,
2062 // because it has never been validated.
2064 }
2065
2066 return isLegalMUBUFAddressingMode(AM);
2067}
2068
2069bool SITargetLowering::isLegalMUBUFAddressingMode(const AddrMode &AM) const {
2070 // MUBUF / MTBUF instructions have a 12-bit unsigned byte offset, and
2071 // additionally can do r + r + i with addr64. 32-bit has more addressing
2072 // mode options. Depending on the resource constant, it can also do
2073 // (i64 r0) + (i32 r1) * (i14 i).
2074 //
2075 // Private arrays end up using a scratch buffer most of the time, so also
2076 // assume those use MUBUF instructions. Scratch loads / stores are currently
2077 // implemented as mubuf instructions with offen bit set, so slightly
2078 // different than the normal addr64.
2079 const SIInstrInfo *TII = Subtarget->getInstrInfo();
2080 if (!TII->isLegalMUBUFImmOffset(AM.BaseOffs))
2081 return false;
2082
2083 // FIXME: Since we can split immediate into soffset and immediate offset,
2084 // would it make sense to allow any immediate?
2085
2086 switch (AM.Scale) {
2087 case 0: // r + i or just i, depending on HasBaseReg.
2088 return true;
2089 case 1:
2090 return true; // We have r + r or r + i.
2091 case 2:
2092 if (AM.HasBaseReg) {
2093 // Reject 2 * r + r.
2094 return false;
2095 }
2096
2097 // Allow 2 * r as r + r
2098 // Or 2 * r + i is allowed as r + r + i.
2099 return true;
2100 default: // Don't allow n * r
2101 return false;
2102 }
2103}
2104
2106 const AddrMode &AM, Type *Ty,
2107 unsigned AS,
2108 Instruction *I) const {
2109 // No global is ever allowed as a base.
2110 if (AM.BaseGV)
2111 return false;
2112
2113 if (AS == AMDGPUAS::GLOBAL_ADDRESS)
2114 return isLegalGlobalAddressingMode(AM);
2115
2116 if (AS == AMDGPUAS::CONSTANT_ADDRESS ||
2120 // If the offset isn't a multiple of 4, it probably isn't going to be
2121 // correctly aligned.
2122 // FIXME: Can we get the real alignment here?
2123 if (AM.BaseOffs % 4 != 0)
2124 return isLegalMUBUFAddressingMode(AM);
2125
2126 if (!Subtarget->hasScalarSubwordLoads()) {
2127 // There are no SMRD extloads, so if we have to do a small type access we
2128 // will use a MUBUF load.
2129 // FIXME?: We also need to do this if unaligned, but we don't know the
2130 // alignment here.
2131 if (Ty->isSized() && DL.getTypeStoreSize(Ty) < 4)
2132 return isLegalGlobalAddressingMode(AM);
2133 }
2134
2135 if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS) {
2136 // SMRD instructions have an 8-bit, dword offset on SI.
2137 if (!isUInt<8>(AM.BaseOffs / 4))
2138 return false;
2139 } else if (Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS) {
2140 // On CI+, this can also be a 32-bit literal constant offset. If it fits
2141 // in 8-bits, it can use a smaller encoding.
2142 if (!isUInt<32>(AM.BaseOffs / 4))
2143 return false;
2144 } else if (Subtarget->getGeneration() < AMDGPUSubtarget::GFX9) {
2145 // On VI, these use the SMEM format and the offset is 20-bit in bytes.
2146 if (!isUInt<20>(AM.BaseOffs))
2147 return false;
2148 } else if (Subtarget->getGeneration() < AMDGPUSubtarget::GFX12) {
2149 // On GFX9 the offset is signed 21-bit in bytes (but must not be negative
2150 // for S_BUFFER_* instructions).
2151 if (!isInt<21>(AM.BaseOffs))
2152 return false;
2153 } else {
2154 // On GFX12, all offsets are signed 24-bit in bytes.
2155 if (!isInt<24>(AM.BaseOffs))
2156 return false;
2157 }
2158
2159 if ((AS == AMDGPUAS::CONSTANT_ADDRESS ||
2161 AM.BaseOffs < 0) {
2162 // Scalar (non-buffer) loads can only use a negative offset if
2163 // soffset+offset is non-negative. Since the compiler can only prove that
2164 // in a few special cases, it is safer to claim that negative offsets are
2165 // not supported.
2166 return false;
2167 }
2168
2169 if (AM.Scale == 0) // r + i or just i, depending on HasBaseReg.
2170 return true;
2171
2172 if (AM.Scale == 1 && AM.HasBaseReg)
2173 return true;
2174
2175 return false;
2176 }
2177
2178 if (AS == AMDGPUAS::PRIVATE_ADDRESS)
2179 return Subtarget->hasFlatScratchEnabled()
2181 : isLegalMUBUFAddressingMode(AM);
2182
2183 if (AS == AMDGPUAS::LOCAL_ADDRESS ||
2184 (AS == AMDGPUAS::REGION_ADDRESS && Subtarget->hasGDS())) {
2185 // Basic, single offset DS instructions allow a 16-bit unsigned immediate
2186 // field.
2187 // XXX - If doing a 4-byte aligned 8-byte type access, we effectively have
2188 // an 8-bit dword offset but we don't know the alignment here.
2189 if (!isUInt<16>(AM.BaseOffs))
2190 return false;
2191
2192 if (AM.Scale == 0) // r + i or just i, depending on HasBaseReg.
2193 return true;
2194
2195 if (AM.Scale == 1 && AM.HasBaseReg)
2196 return true;
2197
2198 return false;
2199 }
2200
2202 // For an unknown address space, this usually means that this is for some
2203 // reason being used for pure arithmetic, and not based on some addressing
2204 // computation. We don't have instructions that compute pointers with any
2205 // addressing modes, so treat them as having no offset like flat
2206 // instructions.
2208 }
2209
2210 // Assume a user alias of global for unknown address spaces.
2211 return isLegalGlobalAddressingMode(AM);
2212}
2213
2215 const MachineFunction &MF) const {
2217 return (MemVT.getSizeInBits() <= 4 * 32);
2218 if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
2219 unsigned MaxPrivateBits = 8 * getSubtarget()->getMaxPrivateElementSize();
2220 return (MemVT.getSizeInBits() <= MaxPrivateBits);
2221 }
2223 return (MemVT.getSizeInBits() <= 2 * 32);
2224 return true;
2225}
2226
2228 unsigned Size, unsigned AddrSpace, Align Alignment,
2229 MachineMemOperand::Flags Flags, unsigned *IsFast) const {
2230 if (IsFast)
2231 *IsFast = 0;
2232
2233 if (AddrSpace == AMDGPUAS::LOCAL_ADDRESS ||
2234 AddrSpace == AMDGPUAS::REGION_ADDRESS) {
2235 // Check if alignment requirements for ds_read/write instructions are
2236 // disabled.
2237 if (!Subtarget->hasUnalignedDSAccessEnabled() && Alignment < Align(4))
2238 return false;
2239
2240 Align RequiredAlignment(
2241 PowerOf2Ceil(divideCeil(Size, 8))); // Natural alignment.
2242 if (Subtarget->hasLDSMisalignedBugInWGPMode() && Size > 32 &&
2243 Alignment < RequiredAlignment)
2244 return false;
2245
2246 // Either, the alignment requirements are "enabled", or there is an
2247 // unaligned LDS access related hardware bug though alignment requirements
2248 // are "disabled". In either case, we need to check for proper alignment
2249 // requirements.
2250 //
2251 switch (Size) {
2252 case 64:
2253 // SI has a hardware bug in the LDS / GDS bounds checking: if the base
2254 // address is negative, then the instruction is incorrectly treated as
2255 // out-of-bounds even if base + offsets is in bounds. Split vectorized
2256 // loads here to avoid emitting ds_read2_b32. We may re-combine the
2257 // load later in the SILoadStoreOptimizer.
2258 if (!Subtarget->hasUsableDSOffset() && Alignment < Align(8))
2259 return false;
2260
2261 // 8 byte accessing via ds_read/write_b64 require 8-byte alignment, but we
2262 // can do a 4 byte aligned, 8 byte access in a single operation using
2263 // ds_read2/write2_b32 with adjacent offsets.
2264 RequiredAlignment = Align(4);
2265
2266 if (Subtarget->hasUnalignedDSAccessEnabled()) {
2267 // We will either select ds_read_b64/ds_write_b64 or ds_read2_b32/
2268 // ds_write2_b32 depending on the alignment. In either case with either
2269 // alignment there is no faster way of doing this.
2270
2271 // The numbers returned here and below are not additive, it is a 'speed
2272 // rank'. They are just meant to be compared to decide if a certain way
2273 // of lowering an operation is faster than another. For that purpose
2274 // naturally aligned operation gets it bitsize to indicate that "it
2275 // operates with a speed comparable to N-bit wide load". With the full
2276 // alignment ds128 is slower than ds96 for example. If underaligned it
2277 // is comparable to a speed of a single dword access, which would then
2278 // mean 32 < 128 and it is faster to issue a wide load regardless.
2279 // 1 is simply "slow, don't do it". I.e. comparing an aligned load to a
2280 // wider load which will not be aligned anymore the latter is slower.
2281 if (IsFast)
2282 *IsFast = (Alignment >= RequiredAlignment) ? 64
2283 : (Alignment < Align(4)) ? 32
2284 : 1;
2285 return true;
2286 }
2287
2288 break;
2289 case 96:
2290 if (!Subtarget->hasDS96AndDS128())
2291 return false;
2292
2293 // 12 byte accessing via ds_read/write_b96 require 16-byte alignment on
2294 // gfx8 and older.
2295
2296 if (Subtarget->hasUnalignedDSAccessEnabled()) {
2297 // Naturally aligned access is fastest. However, also report it is Fast
2298 // if memory is aligned less than DWORD. A narrow load or store will be
2299 // be equally slow as a single ds_read_b96/ds_write_b96, but there will
2300 // be more of them, so overall we will pay less penalty issuing a single
2301 // instruction.
2302
2303 // See comment on the values above.
2304 if (IsFast)
2305 *IsFast = (Alignment >= RequiredAlignment) ? 96
2306 : (Alignment < Align(4)) ? 32
2307 : 1;
2308 return true;
2309 }
2310
2311 break;
2312 case 128:
2313 if (!Subtarget->hasDS96AndDS128() || !Subtarget->useDS128())
2314 return false;
2315
2316 // 16 byte accessing via ds_read/write_b128 require 16-byte alignment on
2317 // gfx8 and older, but we can do a 8 byte aligned, 16 byte access in a
2318 // single operation using ds_read2/write2_b64.
2319 RequiredAlignment = Align(8);
2320
2321 if (Subtarget->hasUnalignedDSAccessEnabled()) {
2322 // Naturally aligned access is fastest. However, also report it is Fast
2323 // if memory is aligned less than DWORD. A narrow load or store will be
2324 // be equally slow as a single ds_read_b128/ds_write_b128, but there
2325 // will be more of them, so overall we will pay less penalty issuing a
2326 // single instruction.
2327
2328 // See comment on the values above.
2329 if (IsFast)
2330 *IsFast = (Alignment >= RequiredAlignment) ? 128
2331 : (Alignment < Align(4)) ? 32
2332 : 1;
2333 return true;
2334 }
2335
2336 break;
2337 default:
2338 if (Size > 32)
2339 return false;
2340
2341 break;
2342 }
2343
2344 // See comment on the values above.
2345 // Note that we have a single-dword or sub-dword here, so if underaligned
2346 // it is a slowest possible access, hence returned value is 0.
2347 if (IsFast)
2348 *IsFast = (Alignment >= RequiredAlignment) ? Size : 0;
2349
2350 return Alignment >= RequiredAlignment ||
2351 Subtarget->hasUnalignedDSAccessEnabled();
2352 }
2353
2354 // FIXME: We have to be conservative here and assume that flat operations
2355 // will access scratch. If we had access to the IR function, then we
2356 // could determine if any private memory was used in the function.
2357 if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS ||
2358 AddrSpace == AMDGPUAS::FLAT_ADDRESS) {
2359 bool AlignedBy4 = Alignment >= Align(4);
2360 if (Subtarget->hasUnalignedScratchAccessEnabled()) {
2361 if (IsFast)
2362 *IsFast = AlignedBy4 ? Size : 1;
2363 return true;
2364 }
2365
2366 if (IsFast)
2367 *IsFast = AlignedBy4;
2368
2369 return AlignedBy4;
2370 }
2371
2372 // So long as they are correct, wide global memory operations perform better
2373 // than multiple smaller memory ops -- even when misaligned
2374 if (AMDGPU::isExtendedGlobalAddrSpace(AddrSpace)) {
2375 if (IsFast)
2376 *IsFast = Size;
2377
2378 return Alignment >= Align(4) ||
2379 Subtarget->hasUnalignedBufferAccessEnabled();
2380 }
2381
2382 // Ensure robust out-of-bounds guarantees for buffer accesses are met when the
2383 // "amdgpu.buffer.oob.mode" module flag has not enabled relaxed untyped-buffer
2384 // OOB semantics. Normally hardware will ensure proper
2385 // out-of-bounds behavior, but in the edge case where an access starts
2386 // out-of-bounds and then enters in-bounds, the entire access would be treated
2387 // as out-of-bounds. Prevent misaligned memory accesses by requiring the
2388 // natural alignment of buffer accesses.
2389 if (AddrSpace == AMDGPUAS::BUFFER_FAT_POINTER ||
2390 AddrSpace == AMDGPUAS::BUFFER_RESOURCE ||
2391 AddrSpace == AMDGPUAS::BUFFER_STRIDED_POINTER) {
2392 if (!Subtarget->hasRelaxedBufferOOBMode() &&
2393 Alignment < Align(PowerOf2Ceil(divideCeil(Size, 8))))
2394 return false;
2395 }
2396
2397 // Smaller than dword value must be aligned.
2398 if (Size < 32)
2399 return false;
2400
2401 // 8.1.6 - For Dword or larger reads or writes, the two LSBs of the
2402 // byte-address are ignored, thus forcing Dword alignment.
2403 // This applies to private, global, and constant memory.
2404 if (IsFast)
2405 *IsFast = 1;
2406
2407 return Size >= 32 && Alignment >= Align(4);
2408}
2409
2411 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
2412 unsigned *IsFast) const {
2414 Alignment, Flags, IsFast);
2415}
2416
2418 const DataLayout &DL, EVT VT,
2419 unsigned AddrSpace, Align Alignment,
2421 unsigned *IsFast) const {
2422 // Sub-dword accesses aren't lowered yet, so combines must not narrow an
2423 // access to one.
2424 if (AddrSpace == AMDGPUAS::VGPR && VT.getSizeInBits() < 32)
2425 return false;
2426 return TargetLowering::allowsMemoryAccess(Context, DL, VT, AddrSpace,
2427 Alignment, Flags, IsFast);
2428}
2429
2431 LLVMContext &Context, const MemOp &Op,
2432 const AttributeList &FuncAttributes) const {
2433 // FIXME: Should account for address space here.
2434
2435 // The default fallback uses the private pointer size as a guess for a type to
2436 // use. Make sure we switch these to 64-bit accesses.
2437
2438 if (Op.size() >= 16 &&
2439 Op.isDstAligned(Align(4))) // XXX: Should only do for global
2440 return MVT::v4i32;
2441
2442 if (Op.size() >= 8 && Op.isDstAligned(Align(4)))
2443 return MVT::v2i32;
2444
2445 // Use the default.
2446 return MVT::Other;
2447}
2448
2450 const MemSDNode *MemNode = cast<MemSDNode>(N);
2451 return MemNode->getMemOperand()->getFlags() & MONoClobber;
2452}
2453
2458
2460 unsigned DestAS) const {
2461 if (SrcAS == AMDGPUAS::FLAT_ADDRESS) {
2462 if (DestAS == AMDGPUAS::PRIVATE_ADDRESS &&
2463 Subtarget->hasGloballyAddressableScratch()) {
2464 // Flat -> private requires subtracting src_flat_scratch_base_lo.
2465 return false;
2466 }
2467
2468 // Flat -> private/local is a simple truncate.
2469 // Flat -> global is no-op
2470 return true;
2471 }
2472
2473 const GCNTargetMachine &TM =
2474 static_cast<const GCNTargetMachine &>(getTargetMachine());
2475 return TM.isNoopAddrSpaceCast(DL, SrcAS, DestAS);
2476}
2477
2485
2487 Type *Ty) const {
2488 // FIXME: Could be smarter if called for vector constants.
2489 return true;
2490}
2491
2494 unsigned Index) const {
2497
2498 // TODO: Add more cases that are cheap.
2499 if (Index == 0)
2502}
2503
2504bool SITargetLowering::isExtractVecEltCheap(EVT VT, unsigned Index) const {
2505 // TODO: This should be more aggressive, particular for 16-bit element
2506 // vectors. However there are some mixed improvements and regressions.
2507 EVT EltTy = VT.getVectorElementType();
2508 unsigned MinAlign = Subtarget->useRealTrue16Insts() ? 16 : 32;
2509 return EltTy.getSizeInBits() % MinAlign == 0;
2510}
2511
2513 if (Subtarget->has16BitInsts() && VT == MVT::i16) {
2514 switch (Op) {
2515 case ISD::LOAD:
2516 case ISD::STORE:
2517 return true;
2518 default:
2519 return false;
2520 }
2521 }
2522
2523 // SimplifySetCC uses this function to determine whether or not it should
2524 // create setcc with i1 operands. We don't have instructions for i1 setcc.
2525 if (VT == MVT::i1 && Op == ISD::SETCC)
2526 return false;
2527
2528 if (VT.getScalarType() == MVT::i64 &&
2530 return false;
2531
2532 // The DAGCombiner replaces a conversion from i1 with a select of the two
2533 // exact results. The select avoids an f32 conversion plus rounding for
2534 // f16/bf16, a 64-bit conversion for f64 and a missing libcall for f128, and
2535 // is the same single v_cndmask for f32.
2536 if (VT == MVT::i1 && (Op == ISD::SINT_TO_FP || Op == ISD::UINT_TO_FP))
2537 return false;
2538
2540}
2541
2543 // Do not convert uniform loads to 16-bit.
2544 // Uniform 16-bit loads are legalized to i16 = trunc (zextload i16->i32)
2545 // to match subword load patterns. Allowing conversion back to a 16-bit
2546 // load would create an infinite loop.
2547 if (Subtarget->hasScalarSubwordLoads() && N->getOpcode() == ISD::LOAD &&
2548 !VT.isVector() && VT.getSizeInBits() == 16) {
2549 auto *Load = dyn_cast<LoadSDNode>(N);
2550 if (Load && isUniformLoad(Load)) {
2551 return false;
2552 }
2553 }
2554
2555 return isTypeDesirableForOp(N->getOpcode(), VT);
2556}
2557
2559 const MachineMemOperand *MMO = Load->getMemOperand();
2560
2561 // FIXME: We ought to able able to take the direct isDivergent result. We
2562 // cannot rely on the MMO for a uniformity check, and should stop using
2563 // it. This is a hack for 2 ways that the IR divergence analysis is superior
2564 // to the DAG divergence: Recognizing shift-of-workitem-id as always
2565 // uniform, and isSingleLaneExecution. These should be handled in the DAG
2566 // version, and then this can be dropped.
2567 if (Load->isDivergent() && !AMDGPU::isUniformMMO(MMO))
2568 return false;
2569
2570 return MMO->getSize().hasValue() &&
2571 Load->getAlign() >=
2572 Align(std::min(MMO->getSize().getValue().getKnownMinValue(),
2573 uint64_t(4))) &&
2574 (MMO->isInvariant() ||
2575 (Load->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
2576 Load->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) ||
2577 (Load->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS &&
2578 Load->isSimple() && isMemOpHasNoClobberedMemOperand(Load)));
2579}
2580
2583 // This isn't really a constant pool but close enough.
2586 return PtrInfo;
2587}
2588
2589SDValue SITargetLowering::lowerKernArgParameterPtr(SelectionDAG &DAG,
2590 const SDLoc &SL,
2591 SDValue Chain,
2592 uint64_t Offset) const {
2593 const DataLayout &DL = DAG.getDataLayout();
2597
2598 auto [InputPtrReg, RC, ArgTy] =
2599 Info->getPreloadedValue(AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
2600
2601 // We may not have the kernarg segment argument if we have no kernel
2602 // arguments.
2603 if (!InputPtrReg)
2604 return DAG.getConstant(Offset, SL, PtrVT);
2605
2607 SDValue BasePtr = DAG.getCopyFromReg(
2608 Chain, SL, MRI.getLiveInVirtReg(InputPtrReg->getRegister()), PtrVT);
2609
2610 return DAG.getObjectPtrOffset(SL, BasePtr, TypeSize::getFixed(Offset));
2611}
2612
2613SDValue SITargetLowering::getImplicitArgPtr(SelectionDAG &DAG,
2614 const SDLoc &SL) const {
2617 return lowerKernArgParameterPtr(DAG, SL, DAG.getEntryNode(), Offset);
2618}
2619
2620SDValue SITargetLowering::getLDSKernelId(SelectionDAG &DAG,
2621 const SDLoc &SL) const {
2622
2624 std::optional<uint32_t> KnownSize =
2626 if (KnownSize.has_value())
2627 return DAG.getConstant(*KnownSize, SL, MVT::i32);
2628 return SDValue();
2629}
2630
2631SDValue SITargetLowering::convertArgType(SelectionDAG &DAG, EVT VT, EVT MemVT,
2632 const SDLoc &SL, SDValue Val,
2633 bool Signed,
2634 const ISD::InputArg *Arg) const {
2635 // First, if it is a widened vector, narrow it.
2636 if (VT.isVector() &&
2638 EVT NarrowedVT =
2641 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, NarrowedVT, Val,
2642 DAG.getConstant(0, SL, MVT::i32));
2643 }
2644
2645 // Then convert the vector elements or scalar value.
2646 if (Arg && (Arg->Flags.isSExt() || Arg->Flags.isZExt()) && VT.bitsLT(MemVT)) {
2647 unsigned Opc = Arg->Flags.isZExt() ? ISD::AssertZext : ISD::AssertSext;
2648 Val = DAG.getNode(Opc, SL, MemVT, Val, DAG.getValueType(VT));
2649 }
2650
2651 if (MemVT.isFloatingPoint()) {
2652 if (VT.isFloatingPoint()) {
2653 Val = getFPExtOrFPRound(DAG, Val, SL, VT);
2654 } else {
2655 assert(!MemVT.isVector());
2656 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), MemVT.getSizeInBits());
2657 SDValue Cast = DAG.getBitcast(IntVT, Val);
2658 Val = DAG.getAnyExtOrTrunc(Cast, SL, VT);
2659 }
2660 } else if (Signed)
2661 Val = DAG.getSExtOrTrunc(Val, SL, VT);
2662 else
2663 Val = DAG.getZExtOrTrunc(Val, SL, VT);
2664
2665 return Val;
2666}
2667
2668SDValue SITargetLowering::lowerKernargMemParameter(
2669 SelectionDAG &DAG, EVT VT, EVT MemVT, const SDLoc &SL, SDValue Chain,
2670 uint64_t Offset, Align Alignment, bool Signed,
2671 const ISD::InputArg *Arg) const {
2672
2673 MachinePointerInfo PtrInfo =
2675
2676 // Try to avoid using an extload by loading earlier than the argument address,
2677 // and extracting the relevant bits. The load should hopefully be merged with
2678 // the previous argument.
2679 if (MemVT.getStoreSize() < 4 && Alignment < 4) {
2680 // TODO: Handle align < 4 and size >= 4 (can happen with packed structs).
2681 int64_t AlignDownOffset = alignDown(Offset, 4);
2682 int64_t OffsetDiff = Offset - AlignDownOffset;
2683
2684 EVT IntVT = MemVT.changeTypeToInteger();
2685
2686 // TODO: If we passed in the base kernel offset we could have a better
2687 // alignment than 4, but we don't really need it.
2688 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, AlignDownOffset);
2689 SDValue Load = DAG.getLoad(MVT::i32, SL, Chain, Ptr,
2690 PtrInfo.getWithOffset(AlignDownOffset), Align(4),
2693
2694 SDValue ShiftAmt = DAG.getConstant(OffsetDiff * 8, SL, MVT::i32);
2695 SDValue Extract = DAG.getNode(ISD::SRL, SL, MVT::i32, Load, ShiftAmt);
2696
2697 SDValue ArgVal = DAG.getNode(ISD::TRUNCATE, SL, IntVT, Extract);
2698 ArgVal = DAG.getNode(ISD::BITCAST, SL, MemVT, ArgVal);
2699 ArgVal = convertArgType(DAG, VT, MemVT, SL, ArgVal, Signed, Arg);
2700
2701 return DAG.getMergeValues({ArgVal, Load.getValue(1)}, SL);
2702 }
2703
2704 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, Offset);
2705 SDValue Load = DAG.getLoad(
2706 MemVT, SL, Chain, Ptr, PtrInfo.getWithOffset(Offset), Alignment,
2708
2709 SDValue Val = convertArgType(DAG, VT, MemVT, SL, Load, Signed, Arg);
2710 return DAG.getMergeValues({Val, Load.getValue(1)}, SL);
2711}
2712
2713/// Coerce an argument which was passed in a different ABI type to the original
2714/// expected value type.
2715SDValue SITargetLowering::convertABITypeToValueType(SelectionDAG &DAG,
2716 SDValue Val,
2717 CCValAssign &VA,
2718 const SDLoc &SL) const {
2719 EVT ValVT = VA.getValVT();
2720
2721 // If this is an 8 or 16-bit value, it is really passed promoted
2722 // to 32 bits. Insert an assert[sz]ext to capture this, then
2723 // truncate to the right size.
2724 switch (VA.getLocInfo()) {
2725 case CCValAssign::Full:
2726 return Val;
2727 case CCValAssign::BCvt:
2728 return DAG.getNode(ISD::BITCAST, SL, ValVT, Val);
2729 case CCValAssign::SExt:
2730 Val = DAG.getNode(ISD::AssertSext, SL, VA.getLocVT(), Val,
2731 DAG.getValueType(ValVT));
2732 return DAG.getNode(ISD::TRUNCATE, SL, ValVT, Val);
2733 case CCValAssign::ZExt:
2734 Val = DAG.getNode(ISD::AssertZext, SL, VA.getLocVT(), Val,
2735 DAG.getValueType(ValVT));
2736 return DAG.getNode(ISD::TRUNCATE, SL, ValVT, Val);
2737 case CCValAssign::AExt:
2738 Val = DAG.getNode(ISD::TRUNCATE, SL, ValVT.changeTypeToInteger(), Val);
2739 return DAG.getBitcast(ValVT, Val);
2740 default:
2741 llvm_unreachable("Unknown loc info!");
2742 }
2743}
2744
2745SDValue SITargetLowering::lowerStackParameter(SelectionDAG &DAG,
2746 CCValAssign &VA, const SDLoc &SL,
2747 SDValue Chain,
2748 const ISD::InputArg &Arg) const {
2750 MachineFrameInfo &MFI = MF.getFrameInfo();
2751
2752 if (Arg.Flags.isByVal()) {
2753 unsigned Size = Arg.Flags.getByValSize();
2754 int FrameIdx = MFI.CreateFixedObject(Size, VA.getLocMemOffset(), false);
2755 return DAG.getFrameIndex(FrameIdx, MVT::i32);
2756 }
2757
2758 unsigned ArgOffset = VA.getLocMemOffset();
2759 unsigned ArgSize = VA.getValVT().getStoreSize();
2760
2761 int FI = MFI.CreateFixedObject(ArgSize, ArgOffset, true);
2762
2763 // Create load nodes to retrieve arguments from the stack.
2764 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
2765 MachinePointerInfo PtrInfo =
2767 if (VA.getLocInfo() == CCValAssign::AExt)
2768 return DAG.getLoad(VA.getValVT(), SL, Chain, FIN, PtrInfo);
2769
2770 // For NON_EXTLOAD, generic code in getLoad assert(ValVT == MemVT)
2772 MVT MemVT = VA.getValVT();
2773
2774 switch (VA.getLocInfo()) {
2775 default:
2776 break;
2777 case CCValAssign::BCvt:
2778 MemVT = VA.getLocVT();
2779 break;
2780 case CCValAssign::SExt:
2781 ExtType = ISD::SEXTLOAD;
2782 break;
2783 case CCValAssign::ZExt:
2784 ExtType = ISD::ZEXTLOAD;
2785 break;
2786 }
2787
2788 SDValue ArgValue =
2789 DAG.getExtLoad(ExtType, SL, VA.getLocVT(), Chain, FIN, PtrInfo, MemVT);
2790
2791 SDValue ConvertedVal = convertABITypeToValueType(DAG, ArgValue, VA, SL);
2792 if (ConvertedVal == ArgValue)
2793 return ConvertedVal;
2794
2795 return DAG.getMergeValues({ConvertedVal, ArgValue.getValue(1)}, SL);
2796}
2797
2798SDValue SITargetLowering::lowerWorkGroupId(
2799 SelectionDAG &DAG, const SIMachineFunctionInfo &MFI, EVT VT,
2802 AMDGPUFunctionArgInfo::PreloadedValue ClusterWorkGroupIdPV) const {
2803 if (!Subtarget->hasClusters())
2804 return getPreloadedValue(DAG, MFI, VT, WorkGroupIdPV);
2805
2806 // Clusters are supported. Return the global position in the grid. If clusters
2807 // are enabled, WorkGroupIdPV returns the cluster ID not the workgroup ID.
2808
2809 // WorkGroupIdXYZ = ClusterId == 0 ?
2810 // ClusterIdXYZ :
2811 // ClusterIdXYZ * (ClusterMaxIdXYZ + 1) + ClusterWorkGroupIdXYZ
2812 SDValue ClusterIdXYZ = getPreloadedValue(DAG, MFI, VT, WorkGroupIdPV);
2813 SDLoc SL(ClusterIdXYZ);
2814 SDValue ClusterMaxIdXYZ = getPreloadedValue(DAG, MFI, VT, ClusterMaxIdPV);
2815 SDValue One = DAG.getConstant(1, SL, VT);
2816 SDValue ClusterSizeXYZ = DAG.getNode(ISD::ADD, SL, VT, ClusterMaxIdXYZ, One);
2817 SDValue ClusterWorkGroupIdXYZ =
2818 getPreloadedValue(DAG, MFI, VT, ClusterWorkGroupIdPV);
2819 SDValue GlobalIdXYZ =
2820 DAG.getNode(ISD::ADD, SL, VT, ClusterWorkGroupIdXYZ,
2821 DAG.getNode(ISD::MUL, SL, VT, ClusterIdXYZ, ClusterSizeXYZ));
2822
2823 switch (MFI.getClusterDims().getKind()) {
2826 return GlobalIdXYZ;
2828 return ClusterIdXYZ;
2830 using namespace AMDGPU::Hwreg;
2831 SDValue ClusterIdField =
2832 DAG.getTargetConstant(HwregEncoding::encode(ID_IB_STS2, 6, 4), SL, VT);
2833 SDNode *GetReg =
2834 DAG.getMachineNode(AMDGPU::S_GETREG_B32_const, SL, VT, ClusterIdField);
2835 SDValue ClusterId(GetReg, 0);
2836 SDValue Zero = DAG.getConstant(0, SL, VT);
2837 return DAG.getNode(ISD::SELECT_CC, SL, VT, ClusterId, Zero, ClusterIdXYZ,
2838 GlobalIdXYZ, DAG.getCondCode(ISD::SETEQ));
2839 }
2840 }
2841
2842 llvm_unreachable("nothing should reach here");
2843}
2844
2845SDValue SITargetLowering::getPreloadedValue(
2846 SelectionDAG &DAG, const SIMachineFunctionInfo &MFI, EVT VT,
2848 const ArgDescriptor *Reg = nullptr;
2849 const TargetRegisterClass *RC = nullptr;
2850 LLT Ty;
2851
2853 const ArgDescriptor WorkGroupIDX =
2854 ArgDescriptor::createRegister(AMDGPU::TTMP9);
2855 // If GridZ is not programmed in an entry function then the hardware will set
2856 // it to all zeros, so there is no need to mask the GridY value in the low
2857 // order bits.
2858 const ArgDescriptor WorkGroupIDY = ArgDescriptor::createRegister(
2859 AMDGPU::TTMP7,
2860 AMDGPU::isEntryFunctionCC(CC) && !MFI.hasWorkGroupIDZ() ? ~0u : 0xFFFFu);
2861 const ArgDescriptor WorkGroupIDZ =
2862 ArgDescriptor::createRegister(AMDGPU::TTMP7, 0xFFFF0000u);
2863 const ArgDescriptor ClusterWorkGroupIDX =
2864 ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x0000000Fu);
2865 const ArgDescriptor ClusterWorkGroupIDY =
2866 ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x000000F0u);
2867 const ArgDescriptor ClusterWorkGroupIDZ =
2868 ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x00000F00u);
2869 const ArgDescriptor ClusterWorkGroupMaxIDX =
2870 ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x0000F000u);
2871 const ArgDescriptor ClusterWorkGroupMaxIDY =
2872 ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x000F0000u);
2873 const ArgDescriptor ClusterWorkGroupMaxIDZ =
2874 ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x00F00000u);
2875 const ArgDescriptor ClusterWorkGroupMaxFlatID =
2876 ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x0F000000u);
2877
2878 auto LoadConstant = [&](unsigned N) {
2879 return DAG.getConstant(N, SDLoc(), VT);
2880 };
2881
2882 if (Subtarget->hasArchitectedSGPRs() &&
2885 AMDGPU::ClusterDimsAttr ClusterDims = MFI.getClusterDims();
2886 bool HasFixedDims = ClusterDims.isFixedDims();
2887
2888 switch (PVID) {
2890 Reg = &WorkGroupIDX;
2891 RC = &AMDGPU::SReg_32RegClass;
2892 Ty = LLT::scalar(32);
2893 break;
2895 Reg = &WorkGroupIDY;
2896 RC = &AMDGPU::SReg_32RegClass;
2897 Ty = LLT::scalar(32);
2898 break;
2900 Reg = &WorkGroupIDZ;
2901 RC = &AMDGPU::SReg_32RegClass;
2902 Ty = LLT::scalar(32);
2903 break;
2905 if (HasFixedDims && ClusterDims.getDims()[0] == 1)
2906 return LoadConstant(0);
2907 Reg = &ClusterWorkGroupIDX;
2908 RC = &AMDGPU::SReg_32RegClass;
2909 Ty = LLT::scalar(32);
2910 break;
2912 if (HasFixedDims && ClusterDims.getDims()[1] == 1)
2913 return LoadConstant(0);
2914 Reg = &ClusterWorkGroupIDY;
2915 RC = &AMDGPU::SReg_32RegClass;
2916 Ty = LLT::scalar(32);
2917 break;
2919 if (HasFixedDims && ClusterDims.getDims()[2] == 1)
2920 return LoadConstant(0);
2921 Reg = &ClusterWorkGroupIDZ;
2922 RC = &AMDGPU::SReg_32RegClass;
2923 Ty = LLT::scalar(32);
2924 break;
2926 if (HasFixedDims)
2927 return LoadConstant(ClusterDims.getDims()[0] - 1);
2928 Reg = &ClusterWorkGroupMaxIDX;
2929 RC = &AMDGPU::SReg_32RegClass;
2930 Ty = LLT::scalar(32);
2931 break;
2933 if (HasFixedDims)
2934 return LoadConstant(ClusterDims.getDims()[1] - 1);
2935 Reg = &ClusterWorkGroupMaxIDY;
2936 RC = &AMDGPU::SReg_32RegClass;
2937 Ty = LLT::scalar(32);
2938 break;
2940 if (HasFixedDims)
2941 return LoadConstant(ClusterDims.getDims()[2] - 1);
2942 Reg = &ClusterWorkGroupMaxIDZ;
2943 RC = &AMDGPU::SReg_32RegClass;
2944 Ty = LLT::scalar(32);
2945 break;
2947 Reg = &ClusterWorkGroupMaxFlatID;
2948 RC = &AMDGPU::SReg_32RegClass;
2949 Ty = LLT::scalar(32);
2950 break;
2951 default:
2952 break;
2953 }
2954 }
2955
2956 if (!Reg)
2957 std::tie(Reg, RC, Ty) = MFI.getPreloadedValue(PVID);
2958 if (!Reg) {
2960 // It's possible for a kernarg intrinsic call to appear in a kernel with
2961 // no allocated segment, in which case we do not add the user sgpr
2962 // argument, so just return null.
2963 return DAG.getConstant(0, SDLoc(), VT);
2964 }
2965
2966 // It's undefined behavior if a function marked with the amdgpu-no-*
2967 // attributes uses the corresponding intrinsic.
2968 return DAG.getPOISON(VT);
2969 }
2970
2971 return loadInputValue(DAG, RC, VT, SDLoc(DAG.getEntryNode()), *Reg);
2972}
2973
2975 CallingConv::ID CallConv,
2976 ArrayRef<ISD::InputArg> Ins, BitVector &Skipped,
2977 FunctionType *FType,
2978 SIMachineFunctionInfo *Info) {
2979 for (unsigned I = 0, E = Ins.size(), PSInputNum = 0; I != E; ++I) {
2980 const ISD::InputArg *Arg = &Ins[I];
2981
2982 assert((!Arg->VT.isVector() || Arg->VT.getScalarSizeInBits() == 16) &&
2983 "vector type argument should have been split");
2984
2985 // First check if it's a PS input addr.
2986 if (CallConv == CallingConv::AMDGPU_PS && !Arg->Flags.isInReg() &&
2987 PSInputNum <= 15) {
2988 bool SkipArg = !Arg->Used && !Info->isPSInputAllocated(PSInputNum);
2989
2990 // Inconveniently only the first part of the split is marked as isSplit,
2991 // so skip to the end. We only want to increment PSInputNum once for the
2992 // entire split argument.
2993 if (Arg->Flags.isSplit()) {
2994 while (!Arg->Flags.isSplitEnd()) {
2995 assert((!Arg->VT.isVector() || Arg->VT.getScalarSizeInBits() == 16) &&
2996 "unexpected vector split in ps argument type");
2997 if (!SkipArg)
2998 Splits.push_back(*Arg);
2999 Arg = &Ins[++I];
3000 }
3001 }
3002
3003 if (SkipArg) {
3004 // We can safely skip PS inputs.
3005 Skipped.set(Arg->getOrigArgIndex());
3006 ++PSInputNum;
3007 continue;
3008 }
3009
3010 Info->markPSInputAllocated(PSInputNum);
3011 if (Arg->Used)
3012 Info->markPSInputEnabled(PSInputNum);
3013
3014 ++PSInputNum;
3015 }
3016
3017 Splits.push_back(*Arg);
3018 }
3019}
3020
3021// Allocate special inputs passed in VGPRs.
3023 CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI,
3024 SIMachineFunctionInfo &Info) const {
3025 const LLT I32 = LLT::integer(32);
3027
3028 if (Info.hasWorkItemIDX()) {
3029 Register Reg = AMDGPU::VGPR0;
3030 MRI.setType(MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass), I32);
3031
3032 CCInfo.AllocateReg(Reg);
3033 unsigned Mask =
3034 (Subtarget->hasPackedTID() && Info.hasWorkItemIDY()) ? 0x3ff : ~0u;
3035 Info.setWorkItemIDX(ArgDescriptor::createRegister(Reg, Mask));
3036 }
3037
3038 if (Info.hasWorkItemIDY()) {
3039 assert(Info.hasWorkItemIDX());
3040 if (Subtarget->hasPackedTID()) {
3041 Info.setWorkItemIDY(
3042 ArgDescriptor::createRegister(AMDGPU::VGPR0, 0x3ff << 10));
3043 } else {
3044 unsigned Reg = AMDGPU::VGPR1;
3045 MRI.setType(MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass), I32);
3046
3047 CCInfo.AllocateReg(Reg);
3048 Info.setWorkItemIDY(ArgDescriptor::createRegister(Reg));
3049 }
3050 }
3051
3052 if (Info.hasWorkItemIDZ()) {
3053 assert(Info.hasWorkItemIDX() && Info.hasWorkItemIDY());
3054 if (Subtarget->hasPackedTID()) {
3055 Info.setWorkItemIDZ(
3056 ArgDescriptor::createRegister(AMDGPU::VGPR0, 0x3ff << 20));
3057 } else {
3058 unsigned Reg = AMDGPU::VGPR2;
3059 MRI.setType(MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass), I32);
3060
3061 CCInfo.AllocateReg(Reg);
3062 Info.setWorkItemIDZ(ArgDescriptor::createRegister(Reg));
3063 }
3064 }
3065}
3066
3068 const TargetRegisterClass *RC,
3069 unsigned NumArgRegs) {
3070 ArrayRef<MCPhysReg> ArgSGPRs = ArrayRef(RC->begin(), 32);
3071 unsigned RegIdx = CCInfo.getFirstUnallocated(ArgSGPRs);
3072 if (RegIdx == ArgSGPRs.size())
3073 report_fatal_error("ran out of SGPRs for arguments");
3074
3075 unsigned Reg = ArgSGPRs[RegIdx];
3076 Reg = CCInfo.AllocateReg(Reg);
3077 assert(Reg != AMDGPU::NoRegister);
3078
3079 MachineFunction &MF = CCInfo.getMachineFunction();
3080 MF.addLiveIn(Reg, RC);
3082}
3083
3084// If this has a fixed position, we still should allocate the register in the
3085// CCInfo state. Technically we could get away with this for values passed
3086// outside of the normal argument range.
3088 const TargetRegisterClass *RC,
3089 MCRegister Reg) {
3090 Reg = CCInfo.AllocateReg(Reg);
3091 assert(Reg != AMDGPU::NoRegister);
3092 MachineFunction &MF = CCInfo.getMachineFunction();
3093 MF.addLiveIn(Reg, RC);
3094}
3095
3096static void allocateSGPR32Input(CCState &CCInfo, ArgDescriptor &Arg) {
3097 if (Arg) {
3098 allocateFixedSGPRInputImpl(CCInfo, &AMDGPU::SGPR_32RegClass,
3099 Arg.getRegister());
3100 } else
3101 Arg = allocateSGPR32InputImpl(CCInfo, &AMDGPU::SGPR_32RegClass, 32);
3102}
3103
3104static void allocateSGPR64Input(CCState &CCInfo, ArgDescriptor &Arg) {
3105 if (Arg) {
3106 allocateFixedSGPRInputImpl(CCInfo, &AMDGPU::SGPR_64RegClass,
3107 Arg.getRegister());
3108 } else
3109 Arg = allocateSGPR32InputImpl(CCInfo, &AMDGPU::SGPR_64RegClass, 16);
3110}
3111
3112/// Allocate implicit function VGPR arguments in fixed registers.
3114 CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI,
3115 SIMachineFunctionInfo &Info) const {
3116 Register Reg = CCInfo.AllocateReg(AMDGPU::VGPR31);
3117 if (!Reg)
3118 report_fatal_error("failed to allocate VGPR for implicit arguments");
3119
3120 const unsigned Mask = 0x3ff;
3121 Info.setWorkItemIDX(ArgDescriptor::createRegister(Reg, Mask));
3122 Info.setWorkItemIDY(ArgDescriptor::createRegister(Reg, Mask << 10));
3123 Info.setWorkItemIDZ(ArgDescriptor::createRegister(Reg, Mask << 20));
3124}
3125
3127 CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI,
3128 SIMachineFunctionInfo &Info) const {
3129 auto &ArgInfo = Info.getArgInfo();
3130 const GCNUserSGPRUsageInfo &UserSGPRInfo = Info.getUserSGPRInfo();
3131
3132 // TODO: Unify handling with private memory pointers.
3133 if (UserSGPRInfo.hasDispatchPtr())
3134 allocateSGPR64Input(CCInfo, ArgInfo.DispatchPtr);
3135
3136 if (UserSGPRInfo.hasQueuePtr())
3137 allocateSGPR64Input(CCInfo, ArgInfo.QueuePtr);
3138
3139 // Implicit arg ptr takes the place of the kernarg segment pointer. This is a
3140 // constant offset from the kernarg segment.
3141 if (Info.hasImplicitArgPtr())
3142 allocateSGPR64Input(CCInfo, ArgInfo.ImplicitArgPtr);
3143
3144 if (UserSGPRInfo.hasDispatchID())
3145 allocateSGPR64Input(CCInfo, ArgInfo.DispatchID);
3146
3147 // flat_scratch_init is not applicable for non-kernel functions.
3148
3149 if (Info.hasWorkGroupIDX())
3150 allocateSGPR32Input(CCInfo, ArgInfo.WorkGroupIDX);
3151
3152 if (Info.hasWorkGroupIDY())
3153 allocateSGPR32Input(CCInfo, ArgInfo.WorkGroupIDY);
3154
3155 if (Info.hasWorkGroupIDZ())
3156 allocateSGPR32Input(CCInfo, ArgInfo.WorkGroupIDZ);
3157
3158 if (Info.hasLDSKernelId())
3159 allocateSGPR32Input(CCInfo, ArgInfo.LDSKernelId);
3160}
3161
3162// Allocate special inputs passed in user SGPRs.
3164 MachineFunction &MF,
3165 const SIRegisterInfo &TRI,
3166 SIMachineFunctionInfo &Info) const {
3167 const GCNUserSGPRUsageInfo &UserSGPRInfo = Info.getUserSGPRInfo();
3168 if (UserSGPRInfo.hasImplicitBufferPtr()) {
3169 Register ImplicitBufferPtrReg = Info.addImplicitBufferPtr(TRI);
3170 MF.addLiveIn(ImplicitBufferPtrReg, &AMDGPU::SGPR_64RegClass);
3171 CCInfo.AllocateReg(ImplicitBufferPtrReg);
3172 }
3173
3174 // FIXME: How should these inputs interact with inreg / custom SGPR inputs?
3175 if (UserSGPRInfo.hasPrivateSegmentBuffer()) {
3176 Register PrivateSegmentBufferReg = Info.addPrivateSegmentBuffer(TRI);
3177 MF.addLiveIn(PrivateSegmentBufferReg, &AMDGPU::SGPR_128RegClass);
3178 CCInfo.AllocateReg(PrivateSegmentBufferReg);
3179 }
3180
3181 if (UserSGPRInfo.hasDispatchPtr()) {
3182 Register DispatchPtrReg = Info.addDispatchPtr(TRI);
3183 MF.addLiveIn(DispatchPtrReg, &AMDGPU::SGPR_64RegClass);
3184 CCInfo.AllocateReg(DispatchPtrReg);
3185 }
3186
3187 if (UserSGPRInfo.hasQueuePtr()) {
3188 Register QueuePtrReg = Info.addQueuePtr(TRI);
3189 MF.addLiveIn(QueuePtrReg, &AMDGPU::SGPR_64RegClass);
3190 CCInfo.AllocateReg(QueuePtrReg);
3191 }
3192
3193 if (UserSGPRInfo.hasKernargSegmentPtr()) {
3195 Register InputPtrReg = Info.addKernargSegmentPtr(TRI);
3196 CCInfo.AllocateReg(InputPtrReg);
3197
3198 Register VReg = MF.addLiveIn(InputPtrReg, &AMDGPU::SGPR_64RegClass);
3199 MRI.setType(VReg, LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));
3200 }
3201
3202 if (UserSGPRInfo.hasDispatchID()) {
3203 Register DispatchIDReg = Info.addDispatchID(TRI);
3204 MF.addLiveIn(DispatchIDReg, &AMDGPU::SGPR_64RegClass);
3205 CCInfo.AllocateReg(DispatchIDReg);
3206 }
3207
3208 if (UserSGPRInfo.hasFlatScratchInit() && !getSubtarget()->isAmdPalOS()) {
3209 Register FlatScratchInitReg = Info.addFlatScratchInit(TRI);
3210 MF.addLiveIn(FlatScratchInitReg, &AMDGPU::SGPR_64RegClass);
3211 CCInfo.AllocateReg(FlatScratchInitReg);
3212 }
3213
3214 if (UserSGPRInfo.hasPrivateSegmentSize()) {
3215 Register PrivateSegmentSizeReg = Info.addPrivateSegmentSize(TRI);
3216 MF.addLiveIn(PrivateSegmentSizeReg, &AMDGPU::SGPR_32RegClass);
3217 CCInfo.AllocateReg(PrivateSegmentSizeReg);
3218 }
3219
3220 // TODO: Add GridWorkGroupCount user SGPRs when used. For now with HSA we read
3221 // these from the dispatch pointer.
3222}
3223
3224// Allocate pre-loaded kernel arguemtns. Arguments to be preloading must be
3225// sequential starting from the first argument.
3227 CCState &CCInfo, SmallVectorImpl<CCValAssign> &ArgLocs,
3229 const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const {
3230 Function &F = MF.getFunction();
3231 unsigned LastExplicitArgOffset = Subtarget->getExplicitKernelArgOffset();
3232 GCNUserSGPRUsageInfo &SGPRInfo = Info.getUserSGPRInfo();
3233 bool InPreloadSequence = true;
3234 unsigned InIdx = 0;
3235 bool AlignedForImplictArgs = false;
3236 unsigned ImplicitArgOffset = 0;
3237 for (auto &Arg : F.args()) {
3238 if (!InPreloadSequence || !Arg.hasInRegAttr())
3239 break;
3240
3241 unsigned ArgIdx = Arg.getArgNo();
3242 // Don't preload non-original args or parts not in the current preload
3243 // sequence.
3244 if (InIdx < Ins.size() &&
3245 (!Ins[InIdx].isOrigArg() || Ins[InIdx].getOrigArgIndex() != ArgIdx))
3246 break;
3247
3248 for (; InIdx < Ins.size() && Ins[InIdx].isOrigArg() &&
3249 Ins[InIdx].getOrigArgIndex() == ArgIdx;
3250 InIdx++) {
3251 assert(ArgLocs[ArgIdx].isMemLoc());
3252 auto &ArgLoc = ArgLocs[InIdx];
3253 const Align KernelArgBaseAlign = Align(16);
3254 unsigned ArgOffset = ArgLoc.getLocMemOffset();
3255 Align Alignment = commonAlignment(KernelArgBaseAlign, ArgOffset);
3256 unsigned NumAllocSGPRs =
3257 alignTo(ArgLoc.getLocVT().getFixedSizeInBits(), 32) / 32;
3258
3259 // Fix alignment for hidden arguments.
3260 if (Arg.hasAttribute("amdgpu-hidden-argument")) {
3261 if (!AlignedForImplictArgs) {
3262 ImplicitArgOffset =
3263 alignTo(LastExplicitArgOffset,
3264 Subtarget->getAlignmentForImplicitArgPtr()) -
3265 LastExplicitArgOffset;
3266 AlignedForImplictArgs = true;
3267 }
3268 ArgOffset += ImplicitArgOffset;
3269 }
3270
3271 // Arg is preloaded into the previous SGPR.
3272 if (ArgLoc.getLocVT().getStoreSize() < 4 && Alignment < 4) {
3273 assert(InIdx >= 1 && "No previous SGPR");
3274 Info.getArgInfo().PreloadKernArgs[InIdx].Regs.push_back(
3275 Info.getArgInfo().PreloadKernArgs[InIdx - 1].Regs[0]);
3276 continue;
3277 }
3278
3279 unsigned Padding = ArgOffset - LastExplicitArgOffset;
3280 unsigned PaddingSGPRs = alignTo(Padding, 4) / 4;
3281 // Check for free user SGPRs for preloading.
3282 if (PaddingSGPRs + NumAllocSGPRs > SGPRInfo.getNumFreeUserSGPRs()) {
3283 InPreloadSequence = false;
3284 break;
3285 }
3286
3287 // Preload this argument.
3288 const TargetRegisterClass *RC =
3289 TRI.getSGPRClassForBitWidth(NumAllocSGPRs * 32);
3290 SmallVectorImpl<MCRegister> *PreloadRegs =
3291 Info.addPreloadedKernArg(TRI, RC, NumAllocSGPRs, InIdx, PaddingSGPRs);
3292
3293 if (PreloadRegs->size() > 1)
3294 RC = &AMDGPU::SGPR_32RegClass;
3295 for (auto &Reg : *PreloadRegs) {
3296 assert(Reg);
3297 MF.addLiveIn(Reg, RC);
3298 CCInfo.AllocateReg(Reg);
3299 }
3300
3301 LastExplicitArgOffset = NumAllocSGPRs * 4 + ArgOffset;
3302 }
3303 }
3304}
3305
3307 const SIRegisterInfo &TRI,
3308 SIMachineFunctionInfo &Info) const {
3309 // Always allocate this last since it is a synthetic preload.
3310 if (Info.hasLDSKernelId()) {
3311 Register Reg = Info.addLDSKernelId();
3312 MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
3313 CCInfo.AllocateReg(Reg);
3314 }
3315}
3316
3317// Allocate special input registers that are initialized per-wave.
3320 CallingConv::ID CallConv,
3321 bool IsShader) const {
3322 bool HasArchitectedSGPRs = Subtarget->hasArchitectedSGPRs();
3323 if (Subtarget->hasUserSGPRInit16BugInWave32() && !IsShader) {
3324 // Note: user SGPRs are handled by the front-end for graphics shaders
3325 // Pad up the used user SGPRs with dead inputs.
3326
3327 // TODO: NumRequiredSystemSGPRs computation should be adjusted appropriately
3328 // before enabling architected SGPRs for workgroup IDs.
3329 assert(!HasArchitectedSGPRs && "Unhandled feature for the subtarget");
3330
3331 unsigned CurrentUserSGPRs = Info.getNumUserSGPRs();
3332 // Note we do not count the PrivateSegmentWaveByteOffset. We do not want to
3333 // rely on it to reach 16 since if we end up having no stack usage, it will
3334 // not really be added.
3335 unsigned NumRequiredSystemSGPRs =
3336 Info.hasWorkGroupIDX() + Info.hasWorkGroupIDY() +
3337 Info.hasWorkGroupIDZ() + Info.hasWorkGroupInfo();
3338 for (unsigned i = NumRequiredSystemSGPRs + CurrentUserSGPRs; i < 16; ++i) {
3339 Register Reg = Info.addReservedUserSGPR();
3340 MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
3341 CCInfo.AllocateReg(Reg);
3342 }
3343 }
3344
3345 if (!HasArchitectedSGPRs) {
3346 if (Info.hasWorkGroupIDX()) {
3347 Register Reg = Info.addWorkGroupIDX();
3348 MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
3349 CCInfo.AllocateReg(Reg);
3350 }
3351
3352 if (Info.hasWorkGroupIDY()) {
3353 Register Reg = Info.addWorkGroupIDY();
3354 MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
3355 CCInfo.AllocateReg(Reg);
3356 }
3357
3358 if (Info.hasWorkGroupIDZ()) {
3359 Register Reg = Info.addWorkGroupIDZ();
3360 MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
3361 CCInfo.AllocateReg(Reg);
3362 }
3363 }
3364
3365 if (Info.hasWorkGroupInfo()) {
3366 Register Reg = Info.addWorkGroupInfo();
3367 MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
3368 CCInfo.AllocateReg(Reg);
3369 }
3370
3371 if (Info.hasPrivateSegmentWaveByteOffset()) {
3372 // Scratch wave offset passed in system SGPR.
3373 unsigned PrivateSegmentWaveByteOffsetReg;
3374
3375 if (IsShader) {
3376 PrivateSegmentWaveByteOffsetReg =
3377 Info.getPrivateSegmentWaveByteOffsetSystemSGPR();
3378
3379 // This is true if the scratch wave byte offset doesn't have a fixed
3380 // location.
3381 if (PrivateSegmentWaveByteOffsetReg == AMDGPU::NoRegister) {
3382 PrivateSegmentWaveByteOffsetReg = findFirstFreeSGPR(CCInfo);
3383 Info.setPrivateSegmentWaveByteOffset(PrivateSegmentWaveByteOffsetReg);
3384 }
3385 } else
3386 PrivateSegmentWaveByteOffsetReg = Info.addPrivateSegmentWaveByteOffset();
3387
3388 MF.addLiveIn(PrivateSegmentWaveByteOffsetReg, &AMDGPU::SGPR_32RegClass);
3389 CCInfo.AllocateReg(PrivateSegmentWaveByteOffsetReg);
3390 }
3391
3392 assert(!Subtarget->hasUserSGPRInit16BugInWave32() || IsShader ||
3393 Info.getNumPreloadedSGPRs() >= 16);
3394}
3395
3397 MachineFunction &MF,
3398 const SIRegisterInfo &TRI,
3399 SIMachineFunctionInfo &Info) {
3400 // Now that we've figured out where the scratch register inputs are, see if
3401 // should reserve the arguments and use them directly.
3402 MachineFrameInfo &MFI = MF.getFrameInfo();
3403 bool HasStackObjects = MFI.hasStackObjects();
3404 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
3405
3406 // Record that we know we have non-spill stack objects so we don't need to
3407 // check all stack objects later.
3408 if (HasStackObjects)
3409 Info.setHasNonSpillStackObjects(true);
3410
3411 // Everything live out of a block is spilled with fast regalloc, so it's
3412 // almost certain that spilling will be required.
3414 HasStackObjects = true;
3415
3416 // For now assume stack access is needed in any callee functions, so we need
3417 // the scratch registers to pass in.
3418 bool RequiresStackAccess = HasStackObjects || MFI.hasCalls();
3419
3420 if (!ST.hasFlatScratchEnabled()) {
3421 if (RequiresStackAccess && ST.isAmdHsaOrMesa(MF.getFunction())) {
3422 // If we have stack objects, we unquestionably need the private buffer
3423 // resource. For the Code Object V2 ABI, this will be the first 4 user
3424 // SGPR inputs. We can reserve those and use them directly.
3425
3426 Register PrivateSegmentBufferReg =
3428 Info.setScratchRSrcReg(PrivateSegmentBufferReg);
3429 } else {
3430 unsigned ReservedBufferReg = TRI.reservedPrivateSegmentBufferReg(MF);
3431 // We tentatively reserve the last registers (skipping the last registers
3432 // which may contain VCC, FLAT_SCR, and XNACK). After register allocation,
3433 // we'll replace these with the ones immediately after those which were
3434 // really allocated. In the prologue copies will be inserted from the
3435 // argument to these reserved registers.
3436
3437 // Without HSA, relocations are used for the scratch pointer and the
3438 // buffer resource setup is always inserted in the prologue. Scratch wave
3439 // offset is still in an input SGPR.
3440 Info.setScratchRSrcReg(ReservedBufferReg);
3441 }
3442 }
3443
3445
3446 // For entry functions we have to set up the stack pointer if we use it,
3447 // whereas non-entry functions get this "for free". This means there is no
3448 // intrinsic advantage to using S32 over S34 in cases where we do not have
3449 // calls but do need a frame pointer (i.e. if we are requested to have one
3450 // because frame pointer elimination is disabled). To keep things simple we
3451 // only ever use S32 as the call ABI stack pointer, and so using it does not
3452 // imply we need a separate frame pointer.
3453 //
3454 // Try to use s32 as the SP, but move it if it would interfere with input
3455 // arguments. This won't work with calls though.
3456 //
3457 // FIXME: Move SP to avoid any possible inputs, or find a way to spill input
3458 // registers.
3459 if (!MRI.isLiveIn(AMDGPU::SGPR32)) {
3460 Info.setStackPtrOffsetReg(AMDGPU::SGPR32);
3461 } else {
3463
3464 if (MFI.hasCalls())
3465 report_fatal_error("call in graphics shader with too many input SGPRs");
3466
3467 for (unsigned Reg : AMDGPU::SGPR_32RegClass) {
3468 if (!MRI.isLiveIn(Reg)) {
3469 Info.setStackPtrOffsetReg(Reg);
3470 break;
3471 }
3472 }
3473
3474 if (Info.getStackPtrOffsetReg() == AMDGPU::SP_REG)
3475 report_fatal_error("failed to find register for SP");
3476 }
3477
3478 // hasFP should be accurate for entry functions even before the frame is
3479 // finalized, because it does not rely on the known stack size, only
3480 // properties like whether variable sized objects are present.
3481 if (ST.getFrameLowering()->hasFP(MF)) {
3482 Info.setFrameOffsetReg(AMDGPU::SGPR33);
3483 }
3484}
3485
3488 return !Info->isEntryFunction();
3489}
3490
3492
3494 MachineBasicBlock *Entry,
3495 const SmallVectorImpl<MachineBasicBlock *> &Exits) const {
3497
3498 const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(Entry->getParent());
3499 if (!IStart)
3500 return;
3501
3502 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
3503 MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo();
3504 MachineBasicBlock::iterator MBBI = Entry->begin();
3505 for (const MCPhysReg *I = IStart; *I; ++I) {
3506 const TargetRegisterClass *RC = nullptr;
3507 if (AMDGPU::SReg_64RegClass.contains(*I))
3508 RC = &AMDGPU::SGPR_64RegClass;
3509 else if (AMDGPU::SReg_32RegClass.contains(*I))
3510 RC = &AMDGPU::SGPR_32RegClass;
3511 else
3512 llvm_unreachable("Unexpected register class in CSRsViaCopy!");
3513
3514 Register NewVR = MRI->createVirtualRegister(RC);
3515 // Create copy from CSR to a virtual register.
3516 Entry->addLiveIn(*I);
3517 BuildMI(*Entry, MBBI, DebugLoc(), TII->get(TargetOpcode::COPY), NewVR)
3518 .addReg(*I);
3519
3520 // Insert the copy-back instructions right before the terminator.
3521 for (auto *Exit : Exits)
3522 BuildMI(*Exit, Exit->getFirstTerminator(), DebugLoc(),
3523 TII->get(TargetOpcode::COPY), *I)
3524 .addReg(NewVR);
3525 }
3526}
3527
3529 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
3530 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
3531 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
3533
3535 const Function &Fn = MF.getFunction();
3538 bool IsError = false;
3539
3540 if (Subtarget->isAmdHsaOS() && AMDGPU::isGraphics(CallConv)) {
3542 Fn, "unsupported non-compute shaders with HSA", DL.getDebugLoc()));
3543 IsError = true;
3544 }
3545
3548 BitVector Skipped(Fn.arg_size());
3549 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
3550 *DAG.getContext());
3551
3552 bool IsGraphics = AMDGPU::isGraphics(CallConv);
3553 bool IsKernel = AMDGPU::isKernel(CallConv);
3554 bool IsEntryFunc = AMDGPU::isEntryFunctionCC(CallConv);
3555
3556 if (IsGraphics) {
3557 const GCNUserSGPRUsageInfo &UserSGPRInfo = Info->getUserSGPRInfo();
3558 assert(!UserSGPRInfo.hasDispatchPtr() &&
3559 !UserSGPRInfo.hasKernargSegmentPtr() && !Info->hasWorkGroupInfo() &&
3560 !Info->hasLDSKernelId() && !Info->hasWorkItemIDX() &&
3561 !Info->hasWorkItemIDY() && !Info->hasWorkItemIDZ());
3562 (void)UserSGPRInfo;
3563 if (!Subtarget->hasFlatScratchEnabled())
3564 assert(!UserSGPRInfo.hasFlatScratchInit());
3565 if ((CallConv != CallingConv::AMDGPU_CS &&
3566 CallConv != CallingConv::AMDGPU_Gfx &&
3567 CallConv != CallingConv::AMDGPU_Gfx_WholeWave) ||
3568 !Subtarget->hasArchitectedSGPRs())
3569 assert(!Info->hasWorkGroupIDX() && !Info->hasWorkGroupIDY() &&
3570 !Info->hasWorkGroupIDZ());
3571 }
3572
3573 bool IsWholeWaveFunc = Info->isWholeWaveFunction();
3574
3575 if (CallConv == CallingConv::AMDGPU_PS) {
3576 processPSInputArgs(Splits, CallConv, Ins, Skipped, FType, Info);
3577
3578 // At least one interpolation mode must be enabled or else the GPU will
3579 // hang.
3580 //
3581 // Check PSInputAddr instead of PSInputEnable. The idea is that if the user
3582 // set PSInputAddr, the user wants to enable some bits after the compilation
3583 // based on run-time states. Since we can't know what the final PSInputEna
3584 // will look like, so we shouldn't do anything here and the user should take
3585 // responsibility for the correct programming.
3586 //
3587 // Otherwise, the following restrictions apply:
3588 // - At least one of PERSP_* (0xF) or LINEAR_* (0x70) must be enabled.
3589 // - If POS_W_FLOAT (11) is enabled, at least one of PERSP_* must be
3590 // enabled too.
3591 if ((Info->getPSInputAddr() & 0x7F) == 0 ||
3592 ((Info->getPSInputAddr() & 0xF) == 0 && Info->isPSInputAllocated(11))) {
3593 CCInfo.AllocateReg(AMDGPU::VGPR0);
3594 CCInfo.AllocateReg(AMDGPU::VGPR1);
3595 Info->markPSInputAllocated(0);
3596 Info->markPSInputEnabled(0);
3597 }
3598 if (Subtarget->isAmdPalOS()) {
3599 // For isAmdPalOS, the user does not enable some bits after compilation
3600 // based on run-time states; the register values being generated here are
3601 // the final ones set in hardware. Therefore we need to apply the
3602 // workaround to PSInputAddr and PSInputEnable together. (The case where
3603 // a bit is set in PSInputAddr but not PSInputEnable is where the
3604 // frontend set up an input arg for a particular interpolation mode, but
3605 // nothing uses that input arg. Really we should have an earlier pass
3606 // that removes such an arg.)
3607 unsigned PsInputBits = Info->getPSInputAddr() & Info->getPSInputEnable();
3608 if ((PsInputBits & 0x7F) == 0 ||
3609 ((PsInputBits & 0xF) == 0 && (PsInputBits >> 11 & 1)))
3610 Info->markPSInputEnabled(llvm::countr_zero(Info->getPSInputAddr()));
3611 }
3612 } else if (IsKernel) {
3613 assert(Info->hasWorkGroupIDX() && Info->hasWorkItemIDX());
3614 } else {
3615 Splits.append(IsWholeWaveFunc ? std::next(Ins.begin()) : Ins.begin(),
3616 Ins.end());
3617 }
3618
3619 if (IsKernel)
3620 analyzeFormalArgumentsCompute(CCInfo, Ins);
3621
3622 if (IsEntryFunc) {
3623 allocateSpecialEntryInputVGPRs(CCInfo, MF, *TRI, *Info);
3624 allocateHSAUserSGPRs(CCInfo, MF, *TRI, *Info);
3625 if (IsKernel && Subtarget->hasKernargPreload())
3626 allocatePreloadKernArgSGPRs(CCInfo, ArgLocs, Ins, MF, *TRI, *Info);
3627
3628 allocateLDSKernelId(CCInfo, MF, *TRI, *Info);
3629 } else if (!IsGraphics) {
3630 // For the fixed ABI, pass workitem IDs in the last argument register.
3631 allocateSpecialInputVGPRsFixed(CCInfo, MF, *TRI, *Info);
3632
3633 // FIXME: Sink this into allocateSpecialInputSGPRs
3634 if (!Subtarget->hasFlatScratchEnabled())
3635 CCInfo.AllocateReg(Info->getScratchRSrcReg());
3636
3637 allocateSpecialInputSGPRs(CCInfo, MF, *TRI, *Info);
3638 }
3639
3640 if (!IsKernel) {
3641 CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, isVarArg);
3642 CCInfo.AnalyzeFormalArguments(Splits, AssignFn);
3643
3644 // This assumes the registers are allocated by CCInfo in ascending order
3645 // with no gaps.
3646 Info->setNumWaveDispatchSGPRs(
3647 CCInfo.getFirstUnallocated(AMDGPU::SGPR_32RegClass.getRegisters()));
3648 Info->setNumWaveDispatchVGPRs(
3649 CCInfo.getFirstUnallocated(AMDGPU::VGPR_32RegClass.getRegisters()));
3650 } else if (Info->getNumKernargPreloadedSGPRs()) {
3651 Info->setNumWaveDispatchSGPRs(Info->getNumUserSGPRs());
3652 }
3653
3655
3656 if (IsWholeWaveFunc) {
3657 SDValue Setup = DAG.getNode(AMDGPUISD::WHOLE_WAVE_SETUP, DL,
3658 {MVT::i1, MVT::Other}, Chain);
3659 InVals.push_back(Setup.getValue(0));
3660 Chains.push_back(Setup.getValue(1));
3661 }
3662
3663 // FIXME: This is the minimum kernel argument alignment. We should improve
3664 // this to the maximum alignment of the arguments.
3665 //
3666 // FIXME: Alignment of explicit arguments totally broken with non-0 explicit
3667 // kern arg offset.
3668 const Align KernelArgBaseAlign = Align(16);
3669
3670 for (unsigned i = IsWholeWaveFunc ? 1 : 0, e = Ins.size(), ArgIdx = 0; i != e;
3671 ++i) {
3672 const ISD::InputArg &Arg = Ins[i];
3673 if ((Arg.isOrigArg() && Skipped[Arg.getOrigArgIndex()]) || IsError) {
3674 InVals.push_back(DAG.getPOISON(Arg.VT));
3675 continue;
3676 }
3677
3678 CCValAssign &VA = ArgLocs[ArgIdx++];
3679 MVT VT = VA.getLocVT();
3680
3681 if (IsEntryFunc && VA.isMemLoc()) {
3682 VT = Ins[i].VT;
3683 EVT MemVT = VA.getLocVT();
3684
3685 const uint64_t Offset = VA.getLocMemOffset();
3686 Align Alignment = commonAlignment(KernelArgBaseAlign, Offset);
3687
3688 if (Arg.Flags.isByRef()) {
3689 SDValue Ptr = lowerKernArgParameterPtr(DAG, DL, Chain, Offset);
3690
3691 const GCNTargetMachine &TM =
3692 static_cast<const GCNTargetMachine &>(getTargetMachine());
3693 if (!TM.isNoopAddrSpaceCast(DAG.getDataLayout(),
3695 Arg.Flags.getPointerAddrSpace())) {
3698 }
3699
3700 InVals.push_back(Ptr);
3701 continue;
3702 }
3703
3704 SDValue NewArg;
3705 if (Arg.isOrigArg() && Info->getArgInfo().PreloadKernArgs.count(i)) {
3706 if (MemVT.getStoreSize() < 4 && Alignment < 4) {
3707 // In this case the argument is packed into the previous preload SGPR.
3708 int64_t AlignDownOffset = alignDown(Offset, 4);
3709 int64_t OffsetDiff = Offset - AlignDownOffset;
3710 EVT IntVT = MemVT.changeTypeToInteger();
3711
3712 const SIMachineFunctionInfo *Info =
3715 Register Reg =
3716 Info->getArgInfo().PreloadKernArgs.find(i)->getSecond().Regs[0];
3717
3718 assert(Reg);
3719 Register VReg = MRI.getLiveInVirtReg(Reg);
3720 SDValue Copy = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i32);
3721
3722 SDValue ShiftAmt = DAG.getConstant(OffsetDiff * 8, DL, MVT::i32);
3723 SDValue Extract = DAG.getNode(ISD::SRL, DL, MVT::i32, Copy, ShiftAmt);
3724
3725 SDValue ArgVal = DAG.getNode(ISD::TRUNCATE, DL, IntVT, Extract);
3726 ArgVal = DAG.getNode(ISD::BITCAST, DL, MemVT, ArgVal);
3727 NewArg = convertArgType(DAG, VT, MemVT, DL, ArgVal,
3728 Ins[i].Flags.isSExt(), &Ins[i]);
3729
3730 NewArg = DAG.getMergeValues({NewArg, Copy.getValue(1)}, DL);
3731 } else {
3732 const SIMachineFunctionInfo *Info =
3735 const SmallVectorImpl<MCRegister> &PreloadRegs =
3736 Info->getArgInfo().PreloadKernArgs.find(i)->getSecond().Regs;
3737
3738 SDValue Copy;
3739 if (PreloadRegs.size() == 1) {
3740 Register VReg = MRI.getLiveInVirtReg(PreloadRegs[0]);
3741 const TargetRegisterClass *RC = MRI.getRegClass(VReg);
3742 NewArg = DAG.getCopyFromReg(
3743 Chain, DL, VReg,
3745 TRI->getRegSizeInBits(*RC)));
3746
3747 } else {
3748 // If the kernarg alignment does not match the alignment of the SGPR
3749 // tuple RC that can accommodate this argument, it will be built up
3750 // via copies from from the individual SGPRs that the argument was
3751 // preloaded to.
3753 for (auto Reg : PreloadRegs) {
3754 Register VReg = MRI.getLiveInVirtReg(Reg);
3755 Copy = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i32);
3756 Elts.push_back(Copy);
3757 }
3758 NewArg =
3759 DAG.getBuildVector(EVT::getVectorVT(*DAG.getContext(), MVT::i32,
3760 PreloadRegs.size()),
3761 DL, Elts);
3762 }
3763
3764 // If the argument was preloaded to multiple consecutive 32-bit
3765 // registers because of misalignment between addressable SGPR tuples
3766 // and the argument size, we can still assume that because of kernarg
3767 // segment alignment restrictions that NewArg's size is the same as
3768 // MemVT and just do a bitcast. If MemVT is less than 32-bits we add a
3769 // truncate since we cannot preload to less than a single SGPR and the
3770 // MemVT may be smaller.
3771 EVT MemVTInt =
3773 if (MemVT.bitsLT(NewArg.getSimpleValueType()))
3774 NewArg = DAG.getNode(ISD::TRUNCATE, DL, MemVTInt, NewArg);
3775
3776 NewArg = DAG.getBitcast(MemVT, NewArg);
3777 NewArg = convertArgType(DAG, VT, MemVT, DL, NewArg,
3778 Ins[i].Flags.isSExt(), &Ins[i]);
3779 NewArg = DAG.getMergeValues({NewArg, Chain}, DL);
3780 }
3781 } else {
3782 // Hidden arguments that are in the kernel signature must be preloaded
3783 // to user SGPRs. Print a diagnostic error if a hidden argument is in
3784 // the argument list and is not preloaded.
3785 if (Arg.isOrigArg()) {
3786 Argument *OrigArg = Fn.getArg(Arg.getOrigArgIndex());
3787 if (OrigArg->hasAttribute("amdgpu-hidden-argument")) {
3789 *OrigArg->getParent(),
3790 "hidden argument in kernel signature was not preloaded",
3791 DL.getDebugLoc()));
3792 }
3793 }
3794
3795 NewArg =
3796 lowerKernargMemParameter(DAG, VT, MemVT, DL, Chain, Offset,
3797 Alignment, Ins[i].Flags.isSExt(), &Ins[i]);
3798 }
3799 Chains.push_back(NewArg.getValue(1));
3800
3801 auto *ParamTy =
3802 dyn_cast<PointerType>(FType->getParamType(Ins[i].getOrigArgIndex()));
3803 if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS &&
3804 ParamTy &&
3805 (ParamTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
3806 ParamTy->getAddressSpace() == AMDGPUAS::REGION_ADDRESS)) {
3807 // On SI local pointers are just offsets into LDS, so they are always
3808 // less than 16-bits. On CI and newer they could potentially be
3809 // real pointers, so we can't guarantee their size.
3810 NewArg = DAG.getNode(ISD::AssertZext, DL, NewArg.getValueType(), NewArg,
3811 DAG.getValueType(MVT::i16));
3812 }
3813
3814 InVals.push_back(NewArg);
3815 continue;
3816 }
3817 if (!IsEntryFunc && VA.isMemLoc()) {
3818 SDValue Val = lowerStackParameter(DAG, VA, DL, Chain, Arg);
3819 InVals.push_back(Val);
3820 if (!Arg.Flags.isByVal())
3821 Chains.push_back(Val.getValue(1));
3822 continue;
3823 }
3824
3825 assert(VA.isRegLoc() && "Parameter must be in a register!");
3826
3827 Register Reg = VA.getLocReg();
3828 const TargetRegisterClass *RC = nullptr;
3829 if (AMDGPU::VGPR_32RegClass.contains(Reg))
3830 RC = &AMDGPU::VGPR_32RegClass;
3831 else if (AMDGPU::SGPR_32RegClass.contains(Reg))
3832 RC = &AMDGPU::SGPR_32RegClass;
3833 else
3834 llvm_unreachable("Unexpected register class in LowerFormalArguments!");
3835
3836 Reg = MF.addLiveIn(Reg, RC);
3837 SDValue Val = DAG.getCopyFromReg(Chain, DL, Reg, VT);
3838 if (Arg.Flags.isInReg() && RC == &AMDGPU::VGPR_32RegClass) {
3839 // FIXME: Need to forward the chains created by `CopyFromReg`s, make sure
3840 // they will read physical regs before any side effect instructions.
3841 SDValue ReadFirstLane =
3842 DAG.getTargetConstant(Intrinsic::amdgcn_readfirstlane, DL, MVT::i32);
3844 ReadFirstLane, Val);
3845 }
3846
3847 if (Arg.Flags.isSRet()) {
3848 // The return object should be reasonably addressable.
3849 Val = annotateStackObjectPointer(Val, DAG, DL,
3851 }
3852
3853 Val = convertABITypeToValueType(DAG, Val, VA, DL);
3854 InVals.push_back(Val);
3855 }
3856
3857 // Start adding system SGPRs.
3858 if (IsEntryFunc)
3859 allocateSystemSGPRs(CCInfo, MF, *Info, CallConv, IsGraphics);
3860
3861 unsigned StackArgSize = CCInfo.getStackSize();
3862 Info->setBytesInStackArgArea(StackArgSize);
3863
3864 return Chains.empty() ? Chain
3865 : DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
3866}
3867
3868// TODO: If return values can't fit in registers, we should return as many as
3869// possible in registers before passing on stack.
3871 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
3872 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context,
3873 const Type *RetTy) const {
3874 // Replacing returns with sret/stack usage doesn't make sense for shaders.
3875 // FIXME: Also sort of a workaround for custom vector splitting in LowerReturn
3876 // for shaders. Vector types should be explicitly handled by CC.
3877 if (AMDGPU::isEntryFunctionCC(CallConv))
3878 return true;
3879
3881 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
3882 if (!CCInfo.CheckReturn(Outs, CCAssignFnForReturn(CallConv, IsVarArg)))
3883 return false;
3884
3885 // We must use the stack if return would require unavailable registers.
3886 unsigned MaxNumVGPRs = Subtarget->getMaxNumVGPRs(MF);
3887 unsigned TotalNumVGPRs = Subtarget->getAddressableNumArchVGPRs();
3888 for (unsigned i = MaxNumVGPRs; i < TotalNumVGPRs; ++i)
3889 if (CCInfo.isAllocated(AMDGPU::VGPR_32RegClass.getRegister(i)))
3890 return false;
3891
3892 return true;
3893}
3894
3895SDValue
3897 bool isVarArg,
3899 const SmallVectorImpl<SDValue> &OutVals,
3900 const SDLoc &DL, SelectionDAG &DAG) const {
3904
3905 if (AMDGPU::isKernel(CallConv)) {
3906 return AMDGPUTargetLowering::LowerReturn(Chain, CallConv, isVarArg, Outs,
3907 OutVals, DL, DAG);
3908 }
3909
3910 bool IsShader = AMDGPU::isShader(CallConv);
3911
3912 Info->setIfReturnsVoid(Outs.empty());
3913 bool IsWaveEnd = Info->returnsVoid() && IsShader;
3914
3915 // CCValAssign - represent the assignment of the return value to a location.
3917
3918 // CCState - Info about the registers and stack slots.
3919 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
3920 *DAG.getContext());
3921
3922 // Analyze outgoing return values.
3923 CCInfo.AnalyzeReturn(Outs, CCAssignFnForReturn(CallConv, isVarArg));
3924
3925 SDValue Glue;
3927 RetOps.push_back(Chain); // Operand #0 = Chain (updated below)
3928
3929 SDValue ReadFirstLane =
3930 DAG.getTargetConstant(Intrinsic::amdgcn_readfirstlane, DL, MVT::i32);
3931 // Copy the result values into the output registers.
3932 for (unsigned I = 0, RealRVLocIdx = 0, E = RVLocs.size(); I != E;
3933 ++I, ++RealRVLocIdx) {
3934 CCValAssign &VA = RVLocs[I];
3935 assert(VA.isRegLoc() && "Can only return in registers!");
3936 // TODO: Partially return in registers if return values don't fit.
3937 SDValue Arg = OutVals[RealRVLocIdx];
3938
3939 // Copied from other backends.
3940 switch (VA.getLocInfo()) {
3941 case CCValAssign::Full:
3942 break;
3943 case CCValAssign::BCvt:
3944 Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg);
3945 break;
3946 case CCValAssign::SExt:
3947 Arg = DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Arg);
3948 break;
3949 case CCValAssign::ZExt:
3950 Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg);
3951 break;
3952 case CCValAssign::AExt:
3953 Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg);
3954 break;
3955 default:
3956 llvm_unreachable("Unknown loc info!");
3957 }
3958 if (TRI->isSGPRPhysReg(VA.getLocReg()))
3960 ReadFirstLane, Arg);
3961 Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Arg, Glue);
3962 Glue = Chain.getValue(1);
3963 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
3964 }
3965
3966 // FIXME: Does sret work properly?
3967 if (!Info->isEntryFunction()) {
3968 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
3969 const MCPhysReg *I =
3970 TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction());
3971 if (I) {
3972 for (; *I; ++I) {
3973 if (AMDGPU::SReg_64RegClass.contains(*I))
3974 RetOps.push_back(DAG.getRegister(*I, MVT::i64));
3975 else if (AMDGPU::SReg_32RegClass.contains(*I))
3976 RetOps.push_back(DAG.getRegister(*I, MVT::i32));
3977 else
3978 llvm_unreachable("Unexpected register class in CSRsViaCopy!");
3979 }
3980 }
3981 }
3982
3983 // Update chain and glue.
3984 RetOps[0] = Chain;
3985 if (Glue.getNode())
3986 RetOps.push_back(Glue);
3987
3988 unsigned Opc = AMDGPUISD::ENDPGM;
3989 if (!IsWaveEnd)
3990 Opc = Info->isWholeWaveFunction() ? AMDGPUISD::WHOLE_WAVE_RETURN
3991 : IsShader ? AMDGPUISD::RETURN_TO_EPILOG
3992 : AMDGPUISD::RET_GLUE;
3993 return DAG.getNode(Opc, DL, MVT::Other, RetOps);
3994}
3995
3997 SDValue Chain, SDValue InGlue, CallingConv::ID CallConv, bool IsVarArg,
3998 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
3999 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool IsThisReturn,
4000 SDValue ThisVal) const {
4001 CCAssignFn *RetCC = CCAssignFnForReturn(CallConv, IsVarArg);
4002
4003 // Assign locations to each value returned by this call.
4005 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
4006 *DAG.getContext());
4007 CCInfo.AnalyzeCallResult(Ins, RetCC);
4008
4009 // Copy all of the result registers out of their specified physreg.
4010 for (CCValAssign VA : RVLocs) {
4011 SDValue Val;
4012
4013 if (VA.isRegLoc()) {
4014 Val =
4015 DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), InGlue);
4016 Chain = Val.getValue(1);
4017 InGlue = Val.getValue(2);
4018 } else if (VA.isMemLoc()) {
4019 report_fatal_error("TODO: return values in memory");
4020 } else
4021 llvm_unreachable("unknown argument location type");
4022
4023 switch (VA.getLocInfo()) {
4024 case CCValAssign::Full:
4025 break;
4026 case CCValAssign::BCvt:
4027 Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
4028 break;
4029 case CCValAssign::ZExt:
4030 Val = DAG.getNode(ISD::AssertZext, DL, VA.getLocVT(), Val,
4031 DAG.getValueType(VA.getValVT()));
4032 Val = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Val);
4033 break;
4034 case CCValAssign::SExt:
4035 Val = DAG.getNode(ISD::AssertSext, DL, VA.getLocVT(), Val,
4036 DAG.getValueType(VA.getValVT()));
4037 Val = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Val);
4038 break;
4039 case CCValAssign::AExt:
4040 Val = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Val);
4041 break;
4042 default:
4043 llvm_unreachable("Unknown loc info!");
4044 }
4045
4046 InVals.push_back(Val);
4047 }
4048
4049 return Chain;
4050}
4051
4052// Add code to pass special inputs required depending on used features separate
4053// from the explicit user arguments present in the IR.
4055 CallLoweringInfo &CLI, CCState &CCInfo, const SIMachineFunctionInfo &Info,
4056 SmallVectorImpl<std::pair<unsigned, SDValue>> &RegsToPass,
4057 SmallVectorImpl<SDValue> &MemOpChains, SDValue Chain) const {
4058 // If we don't have a call site, this was a call inserted by
4059 // legalization. These can never use special inputs.
4060 if (!CLI.CB)
4061 return;
4062
4063 SelectionDAG &DAG = CLI.DAG;
4064 const SDLoc &DL = CLI.DL;
4065 const Function &F = DAG.getMachineFunction().getFunction();
4066
4067 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
4068 const AMDGPUFunctionArgInfo &CallerArgInfo = Info.getArgInfo();
4069
4070 const AMDGPUFunctionArgInfo &CalleeArgInfo =
4072
4073 // TODO: Unify with private memory register handling. This is complicated by
4074 // the fact that at least in kernels, the input argument is not necessarily
4075 // in the same location as the input.
4076 // clang-format off
4077 static constexpr std::pair<AMDGPUFunctionArgInfo::PreloadedValue,
4078 std::array<StringLiteral, 2>> ImplicitAttrs[] = {
4079 {AMDGPUFunctionArgInfo::DISPATCH_PTR, {"amdgpu-no-dispatch-ptr", ""}},
4080 {AMDGPUFunctionArgInfo::QUEUE_PTR, {"amdgpu-no-queue-ptr", ""}},
4081 {AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR, {"amdgpu-no-implicitarg-ptr", ""}},
4082 {AMDGPUFunctionArgInfo::DISPATCH_ID, {"amdgpu-no-dispatch-id", ""}},
4083 {AMDGPUFunctionArgInfo::WORKGROUP_ID_X, {"amdgpu-no-workgroup-id-x", "amdgpu-no-cluster-id-x"}},
4084 {AMDGPUFunctionArgInfo::WORKGROUP_ID_Y, {"amdgpu-no-workgroup-id-y", "amdgpu-no-cluster-id-y"}},
4085 {AMDGPUFunctionArgInfo::WORKGROUP_ID_Z, {"amdgpu-no-workgroup-id-z", "amdgpu-no-cluster-id-z"}},
4086 {AMDGPUFunctionArgInfo::LDS_KERNEL_ID, {"amdgpu-no-lds-kernel-id", ""}},
4087 };
4088 // clang-format on
4089
4090 for (auto [InputID, Attrs] : ImplicitAttrs) {
4091 // If the callee does not use the attribute value, skip copying the value.
4092 if (all_of(Attrs, [&](StringRef Attr) {
4093 return Attr.empty() || CLI.CB->hasFnAttr(Attr);
4094 }))
4095 continue;
4096
4097 const auto [OutgoingArg, ArgRC, ArgTy] =
4098 CalleeArgInfo.getPreloadedValue(InputID);
4099 if (!OutgoingArg)
4100 continue;
4101
4102 const auto [IncomingArg, IncomingArgRC, Ty] =
4103 CallerArgInfo.getPreloadedValue(InputID);
4104 assert(IncomingArgRC == ArgRC);
4105
4106 // All special arguments are ints for now.
4107 EVT ArgVT = TRI->getSpillSize(*ArgRC) == 8 ? MVT::i64 : MVT::i32;
4108 SDValue InputReg;
4109
4110 if (IncomingArg) {
4111 InputReg = loadInputValue(DAG, ArgRC, ArgVT, DL, *IncomingArg);
4112 } else if (InputID == AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR) {
4113 // The implicit arg ptr is special because it doesn't have a corresponding
4114 // input for kernels, and is computed from the kernarg segment pointer.
4115 InputReg = getImplicitArgPtr(DAG, DL);
4116 } else if (InputID == AMDGPUFunctionArgInfo::LDS_KERNEL_ID) {
4117 std::optional<uint32_t> Id =
4119 if (Id.has_value()) {
4120 InputReg = DAG.getConstant(*Id, DL, ArgVT);
4121 } else {
4122 InputReg = DAG.getPOISON(ArgVT);
4123 }
4124 } else {
4125 // We may have proven the input wasn't needed, although the ABI is
4126 // requiring it. We just need to allocate the register appropriately.
4127 InputReg = DAG.getPOISON(ArgVT);
4128 }
4129
4130 if (OutgoingArg->isRegister()) {
4131 RegsToPass.emplace_back(OutgoingArg->getRegister(), InputReg);
4132 if (!CCInfo.AllocateReg(OutgoingArg->getRegister()))
4133 report_fatal_error("failed to allocate implicit input argument");
4134 } else {
4135 unsigned SpecialArgOffset =
4136 CCInfo.AllocateStack(ArgVT.getStoreSize(), Align(4));
4137 SDValue ArgStore =
4138 storeStackInputValue(DAG, DL, Chain, InputReg, SpecialArgOffset);
4139 MemOpChains.push_back(ArgStore);
4140 }
4141 }
4142
4143 // Pack workitem IDs into a single register or pass it as is if already
4144 // packed.
4145
4146 auto [OutgoingArg, ArgRC, Ty] =
4148 if (!OutgoingArg)
4149 std::tie(OutgoingArg, ArgRC, Ty) =
4151 if (!OutgoingArg)
4152 std::tie(OutgoingArg, ArgRC, Ty) =
4154 if (!OutgoingArg)
4155 return;
4156
4157 const ArgDescriptor *IncomingArgX = std::get<0>(
4159 const ArgDescriptor *IncomingArgY = std::get<0>(
4161 const ArgDescriptor *IncomingArgZ = std::get<0>(
4163
4164 SDValue InputReg;
4165 SDLoc SL;
4166
4167 const bool NeedWorkItemIDX = !CLI.CB->hasFnAttr("amdgpu-no-workitem-id-x");
4168 const bool NeedWorkItemIDY = !CLI.CB->hasFnAttr("amdgpu-no-workitem-id-y");
4169 const bool NeedWorkItemIDZ = !CLI.CB->hasFnAttr("amdgpu-no-workitem-id-z");
4170
4171 // If incoming ids are not packed we need to pack them.
4172 if (IncomingArgX && !IncomingArgX->isMasked() && CalleeArgInfo.WorkItemIDX &&
4173 NeedWorkItemIDX) {
4174 if (Subtarget->getMaxWorkitemID(F, 0) != 0) {
4175 InputReg = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgX);
4176 } else {
4177 InputReg = DAG.getConstant(0, DL, MVT::i32);
4178 }
4179 }
4180
4181 if (IncomingArgY && !IncomingArgY->isMasked() && CalleeArgInfo.WorkItemIDY &&
4182 NeedWorkItemIDY && Subtarget->getMaxWorkitemID(F, 1) != 0) {
4183 SDValue Y = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgY);
4184 Y = DAG.getNode(ISD::SHL, SL, MVT::i32, Y,
4185 DAG.getShiftAmountConstant(10, MVT::i32, SL));
4186 InputReg = InputReg.getNode()
4187 ? DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Y)
4188 : Y;
4189 }
4190
4191 if (IncomingArgZ && !IncomingArgZ->isMasked() && CalleeArgInfo.WorkItemIDZ &&
4192 NeedWorkItemIDZ && Subtarget->getMaxWorkitemID(F, 2) != 0) {
4193 SDValue Z = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgZ);
4194 Z = DAG.getNode(ISD::SHL, SL, MVT::i32, Z,
4195 DAG.getShiftAmountConstant(20, MVT::i32, SL));
4196 InputReg = InputReg.getNode()
4197 ? DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Z)
4198 : Z;
4199 }
4200
4201 if (!InputReg && (NeedWorkItemIDX || NeedWorkItemIDY || NeedWorkItemIDZ)) {
4202 if (!IncomingArgX && !IncomingArgY && !IncomingArgZ) {
4203 // We're in a situation where the outgoing function requires the workitem
4204 // ID, but the calling function does not have it (e.g a graphics function
4205 // calling a C calling convention function). This is illegal, but we need
4206 // to produce something.
4207 InputReg = DAG.getPOISON(MVT::i32);
4208 } else {
4209 // Workitem ids are already packed, any of present incoming arguments
4210 // will carry all required fields.
4211 ArgDescriptor IncomingArg =
4212 ArgDescriptor::createArg(IncomingArgX ? *IncomingArgX
4213 : IncomingArgY ? *IncomingArgY
4214 : *IncomingArgZ,
4215 ~0u);
4216 InputReg = loadInputValue(DAG, ArgRC, MVT::i32, DL, IncomingArg);
4217 }
4218 }
4219
4220 if (OutgoingArg->isRegister()) {
4221 if (InputReg)
4222 RegsToPass.emplace_back(OutgoingArg->getRegister(), InputReg);
4223
4224 CCInfo.AllocateReg(OutgoingArg->getRegister());
4225 } else {
4226 unsigned SpecialArgOffset = CCInfo.AllocateStack(4, Align(4));
4227 if (InputReg) {
4228 SDValue ArgStore =
4229 storeStackInputValue(DAG, DL, Chain, InputReg, SpecialArgOffset);
4230 MemOpChains.push_back(ArgStore);
4231 }
4232 }
4233}
4234
4236 SDValue Callee, CallingConv::ID CalleeCC, bool IsVarArg,
4238 const SmallVectorImpl<SDValue> &OutVals,
4239 const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const {
4240 if (AMDGPU::isChainCC(CalleeCC))
4241 return true;
4242
4243 if (!AMDGPU::mayTailCallThisCC(CalleeCC))
4244 return false;
4245
4246 // For a divergent call target, we need to do a waterfall loop over the
4247 // possible callees which precludes us from using a simple jump.
4248 if (Callee->isDivergent())
4249 return false;
4250
4252 const Function &CallerF = MF.getFunction();
4253 CallingConv::ID CallerCC = CallerF.getCallingConv();
4255 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
4256
4257 // Kernels aren't callable, and don't have a live in return address so it
4258 // doesn't make sense to do a tail call with entry functions.
4259 if (!CallerPreserved)
4260 return false;
4261
4262 bool CCMatch = CallerCC == CalleeCC;
4263
4265 if (AMDGPU::canGuaranteeTCO(CalleeCC) && CCMatch)
4266 return true;
4267 return false;
4268 }
4269
4270 // TODO: Can we handle var args?
4271 if (IsVarArg)
4272 return false;
4273
4274 for (const Argument &Arg : CallerF.args()) {
4275 if (Arg.hasByValAttr())
4276 return false;
4277 }
4278
4279 LLVMContext &Ctx = *DAG.getContext();
4280
4281 // Check that the call results are passed in the same way.
4282 if (!CCState::resultsCompatible(CalleeCC, CallerCC, MF, Ctx, Ins,
4283 CCAssignFnForCall(CalleeCC, IsVarArg),
4284 CCAssignFnForCall(CallerCC, IsVarArg)))
4285 return false;
4286
4287 // The callee has to preserve all registers the caller needs to preserve.
4288 if (!CCMatch) {
4289 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
4290 if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
4291 return false;
4292 }
4293
4294 // Nothing more to check if the callee is taking no arguments.
4295 if (Outs.empty())
4296 return true;
4297
4299 CCState CCInfo(CalleeCC, IsVarArg, MF, ArgLocs, Ctx);
4300
4301 // FIXME: We are not allocating special input registers, so we will be
4302 // deciding based on incorrect register assignments.
4303 CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CalleeCC, IsVarArg));
4304
4305 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
4306 // If the stack arguments for this call do not fit into our own save area then
4307 // the call cannot be made tail.
4308 // TODO: Is this really necessary?
4309 if (CCInfo.getStackSize() > FuncInfo->getBytesInStackArgArea())
4310 return false;
4311
4312 for (const auto &[CCVA, ArgVal] : zip_equal(ArgLocs, OutVals)) {
4313 // FIXME: What about inreg arguments that end up passed in memory?
4314 if (!CCVA.isRegLoc())
4315 continue;
4316
4317 // If we are passing an argument in an SGPR, and the value is divergent,
4318 // this call requires a waterfall loop.
4319 if (ArgVal->isDivergent() && TRI->isSGPRPhysReg(CCVA.getLocReg())) {
4320 LLVM_DEBUG(
4321 dbgs() << "Cannot tail call due to divergent outgoing argument in "
4322 << printReg(CCVA.getLocReg(), TRI) << '\n');
4323 return false;
4324 }
4325 }
4326
4327 const MachineRegisterInfo &MRI = MF.getRegInfo();
4328 return parametersInCSRMatch(MRI, CallerPreserved, ArgLocs, OutVals);
4329}
4330
4332 if (!CI->isTailCall())
4333 return false;
4334
4335 const Function *ParentFn = CI->getFunction();
4337 return false;
4338 return true;
4339}
4340
4341namespace {
4342// Chain calls have special arguments that we need to handle. These are
4343// tagging along at the end of the arguments list(s), after the SGPR and VGPR
4344// arguments (index 0 and 1 respectively).
4345enum ChainCallArgIdx {
4346 Exec = 2,
4347 Flags,
4348 NumVGPRs,
4349 FallbackExec,
4350 FallbackCallee
4351};
4352} // anonymous namespace
4353
4354// The wave scratch offset register is used as the global base pointer.
4356 SmallVectorImpl<SDValue> &InVals) const {
4357 CallingConv::ID CallConv = CLI.CallConv;
4358 bool IsChainCallConv = AMDGPU::isChainCC(CallConv);
4359
4360 SelectionDAG &DAG = CLI.DAG;
4361
4362 const SDLoc &DL = CLI.DL;
4363 SDValue Chain = CLI.Chain;
4364 SDValue Callee = CLI.Callee;
4365
4366 llvm::SmallVector<SDValue, 6> ChainCallSpecialArgs;
4367 bool UsesDynamicVGPRs = false;
4368 if (IsChainCallConv) {
4369 // The last arguments should be the value that we need to put in EXEC,
4370 // followed by the flags and any other arguments with special meanings.
4371 // Pop them out of CLI.Outs and CLI.OutVals before we do any processing so
4372 // we don't treat them like the "real" arguments.
4373 auto RequestedExecIt =
4374 llvm::find_if(CLI.Outs, [](const ISD::OutputArg &Arg) {
4375 return Arg.OrigArgIndex == 2;
4376 });
4377 assert(RequestedExecIt != CLI.Outs.end() && "No node for EXEC");
4378
4379 size_t SpecialArgsBeginIdx = RequestedExecIt - CLI.Outs.begin();
4380 CLI.OutVals.erase(CLI.OutVals.begin() + SpecialArgsBeginIdx,
4381 CLI.OutVals.end());
4382 CLI.Outs.erase(RequestedExecIt, CLI.Outs.end());
4383
4384 assert(CLI.Outs.back().OrigArgIndex < 2 &&
4385 "Haven't popped all the special args");
4386
4387 TargetLowering::ArgListEntry RequestedExecArg =
4388 CLI.Args[ChainCallArgIdx::Exec];
4389 if (!RequestedExecArg.Ty->isIntegerTy(Subtarget->getWavefrontSize()))
4390 return lowerUnhandledCall(CLI, InVals, "Invalid value for EXEC");
4391
4392 // Convert constants into TargetConstants, so they become immediate operands
4393 // instead of being selected into S_MOV.
4394 auto PushNodeOrTargetConstant = [&](TargetLowering::ArgListEntry Arg) {
4395 if (const auto *ArgNode = dyn_cast<ConstantSDNode>(Arg.Node)) {
4396 ChainCallSpecialArgs.push_back(DAG.getTargetConstant(
4397 ArgNode->getAPIntValue(), DL, ArgNode->getValueType(0)));
4398 } else
4399 ChainCallSpecialArgs.push_back(Arg.Node);
4400 };
4401
4402 PushNodeOrTargetConstant(RequestedExecArg);
4403
4404 // Process any other special arguments depending on the value of the flags.
4405 TargetLowering::ArgListEntry Flags = CLI.Args[ChainCallArgIdx::Flags];
4406
4407 const APInt &FlagsValue = cast<ConstantSDNode>(Flags.Node)->getAPIntValue();
4408 if (FlagsValue.isZero()) {
4409 if (CLI.Args.size() > ChainCallArgIdx::Flags + 1)
4410 return lowerUnhandledCall(CLI, InVals,
4411 "no additional args allowed if flags == 0");
4412 } else if (FlagsValue.isOneBitSet(0)) {
4413 if (CLI.Args.size() != ChainCallArgIdx::FallbackCallee + 1) {
4414 return lowerUnhandledCall(CLI, InVals, "expected 3 additional args");
4415 }
4416
4417 if (!Subtarget->isWave32()) {
4418 return lowerUnhandledCall(
4419 CLI, InVals, "dynamic VGPR mode is only supported for wave32");
4420 }
4421
4422 UsesDynamicVGPRs = true;
4423 std::for_each(CLI.Args.begin() + ChainCallArgIdx::NumVGPRs,
4424 CLI.Args.end(), PushNodeOrTargetConstant);
4425 }
4426 }
4427
4429 SmallVector<SDValue, 32> &OutVals = CLI.OutVals;
4431 bool &IsTailCall = CLI.IsTailCall;
4432 bool IsVarArg = CLI.IsVarArg;
4433 bool IsSibCall = false;
4435
4436 if (Callee.isUndef() || isNullConstant(Callee)) {
4437 if (!CLI.IsTailCall) {
4438 for (ISD::InputArg &Arg : CLI.Ins)
4439 InVals.push_back(DAG.getPOISON(Arg.VT));
4440 }
4441
4442 return Chain;
4443 }
4444
4445 if (IsVarArg) {
4446 return lowerUnhandledCall(CLI, InVals,
4447 "unsupported call to variadic function ");
4448 }
4449
4450 if (!CLI.CB)
4451 return lowerUnhandledCall(CLI, InVals, "unsupported libcall legalization");
4452
4453 if (IsTailCall && MF.getTarget().Options.GuaranteedTailCallOpt) {
4454 return lowerUnhandledCall(CLI, InVals,
4455 "unsupported required tail call to function ");
4456 }
4457
4458 if (IsTailCall) {
4459 IsTailCall = isEligibleForTailCallOptimization(Callee, CallConv, IsVarArg,
4460 Outs, OutVals, Ins, DAG);
4461 if (!IsTailCall &&
4462 ((CLI.CB && CLI.CB->isMustTailCall()) || IsChainCallConv)) {
4463 report_fatal_error("failed to perform tail call elimination on a call "
4464 "site marked musttail or on llvm.amdgcn.cs.chain");
4465 }
4466
4467 bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
4468
4469 // A sibling call is one where we're under the usual C ABI and not planning
4470 // to change that but can still do a tail call:
4471 if (!TailCallOpt && IsTailCall)
4472 IsSibCall = true;
4473
4474 if (IsTailCall)
4475 ++NumTailCalls;
4476 }
4477
4480 SmallVector<SDValue, 8> MemOpChains;
4481
4482 // Analyze operands of the call, assigning locations to each operand.
4484 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
4485 CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, IsVarArg);
4486
4487 if (CallConv != CallingConv::AMDGPU_Gfx && !AMDGPU::isChainCC(CallConv) &&
4489 // With a fixed ABI, allocate fixed registers before user arguments.
4490 passSpecialInputs(CLI, CCInfo, *Info, RegsToPass, MemOpChains, Chain);
4491 }
4492
4493 // Mark the scratch resource descriptor as allocated so the CC analysis
4494 // does not assign user arguments to these registers, matching the callee.
4495 if (!Subtarget->hasFlatScratchEnabled()) {
4496 CCInfo.AllocateReg(
4498 }
4499
4500 CCInfo.AnalyzeCallOperands(Outs, AssignFn);
4501
4502 // Get a count of how many bytes are to be pushed on the stack.
4503 unsigned NumBytes = CCInfo.getStackSize();
4504
4505 if (IsSibCall) {
4506 // Since we're not changing the ABI to make this a tail call, the memory
4507 // operands are already available in the caller's incoming argument space.
4508 NumBytes = 0;
4509 }
4510
4511 // FPDiff is the byte offset of the call's argument area from the callee's.
4512 // Stores to callee stack arguments will be placed in FixedStackSlots offset
4513 // by this amount for a tail call. In a sibling call it must be 0 because the
4514 // caller will deallocate the entire stack and the callee still expects its
4515 // arguments to begin at SP+0. Completely unused for non-tail calls.
4516 int32_t FPDiff = 0;
4517 MachineFrameInfo &MFI = MF.getFrameInfo();
4518 auto *TRI = Subtarget->getRegisterInfo();
4519
4520 // Adjust the stack pointer for the new arguments...
4521 // These operations are automatically eliminated by the prolog/epilog pass
4522 if (!IsSibCall)
4523 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, DL);
4524
4525 if (!IsSibCall || IsChainCallConv) {
4526 if (!Subtarget->hasFlatScratchEnabled()) {
4527 SmallVector<SDValue, 4> CopyFromChains;
4528
4529 // In the HSA case, this should be an identity copy.
4530 SDValue ScratchRSrcReg =
4531 DAG.getCopyFromReg(Chain, DL, Info->getScratchRSrcReg(), MVT::v4i32);
4532 RegsToPass.emplace_back(
4534 ScratchRSrcReg);
4535 CopyFromChains.push_back(ScratchRSrcReg.getValue(1));
4536 Chain = DAG.getTokenFactor(DL, CopyFromChains);
4537 }
4538 }
4539
4540 const unsigned NumSpecialInputs = RegsToPass.size();
4541
4542 MVT PtrVT = MVT::i32;
4543
4544 // Walk the register/memloc assignments, inserting copies/loads.
4545 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4546 CCValAssign &VA = ArgLocs[i];
4547 SDValue Arg = OutVals[i];
4548
4549 // Promote the value if needed.
4550 switch (VA.getLocInfo()) {
4551 case CCValAssign::Full:
4552 break;
4553 case CCValAssign::BCvt:
4554 Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg);
4555 break;
4556 case CCValAssign::ZExt:
4557 Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg);
4558 break;
4559 case CCValAssign::SExt:
4560 Arg = DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Arg);
4561 break;
4562 case CCValAssign::AExt:
4563 Arg = DAG.getBitcast(VA.getValVT().changeTypeToInteger(), Arg);
4564 Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg);
4565 break;
4566 case CCValAssign::FPExt:
4567 Arg = DAG.getNode(ISD::FP_EXTEND, DL, VA.getLocVT(), Arg);
4568 break;
4569 default:
4570 llvm_unreachable("Unknown loc info!");
4571 }
4572
4573 if (VA.isRegLoc()) {
4574 RegsToPass.push_back(std::pair(VA.getLocReg(), Arg));
4575 } else {
4576 assert(VA.isMemLoc());
4577
4578 SDValue DstAddr;
4579 MachinePointerInfo DstInfo;
4580
4581 unsigned LocMemOffset = VA.getLocMemOffset();
4582 int32_t Offset = LocMemOffset;
4583
4584 SDValue PtrOff = DAG.getConstant(Offset, DL, PtrVT);
4585 MaybeAlign Alignment;
4586
4587 if (IsTailCall) {
4588 ISD::ArgFlagsTy Flags = Outs[i].Flags;
4589 unsigned OpSize = Flags.isByVal() ? Flags.getByValSize()
4590 : VA.getValVT().getStoreSize();
4591
4592 // FIXME: We can have better than the minimum byval required alignment.
4593 Alignment =
4594 Flags.isByVal()
4595 ? Flags.getNonZeroByValAlign()
4596 : commonAlignment(Subtarget->getStackAlignment(), Offset);
4597
4598 Offset = Offset + FPDiff;
4599 int FI = MFI.CreateFixedObject(OpSize, Offset, true);
4600
4601 DstAddr = DAG.getFrameIndex(FI, PtrVT);
4602 DstInfo = MachinePointerInfo::getFixedStack(MF, FI);
4603
4604 // Make sure any stack arguments overlapping with where we're storing
4605 // are loaded before this eventual operation. Otherwise they'll be
4606 // clobbered.
4607
4608 // FIXME: Why is this really necessary? This seems to just result in a
4609 // lot of code to copy the stack and write them back to the same
4610 // locations, which are supposed to be immutable?
4611 Chain = addTokenForArgument(Chain, DAG, MFI, FI);
4612 } else {
4613 // Stores to the argument stack area are relative to the stack pointer.
4614 SDValue SP = DAG.getCopyFromReg(Chain, DL, Info->getStackPtrOffsetReg(),
4615 MVT::i32);
4616 DstAddr = DAG.getNode(ISD::ADD, DL, MVT::i32, SP, PtrOff);
4617 DstInfo = MachinePointerInfo::getStack(MF, LocMemOffset);
4618 Alignment =
4619 commonAlignment(Subtarget->getStackAlignment(), LocMemOffset);
4620 }
4621
4622 if (Outs[i].Flags.isByVal()) {
4623 SDValue SizeNode =
4624 DAG.getConstant(Outs[i].Flags.getByValSize(), DL, MVT::i32);
4625 SDValue Cpy =
4626 DAG.getMemcpy(Chain, DL, DstAddr, Arg, SizeNode,
4627 Outs[i].Flags.getNonZeroByValAlign(),
4628 Outs[i].Flags.getNonZeroByValAlign(),
4629 /*isVol = */ false, /*AlwaysInline = */ true,
4630 /*CI=*/nullptr, std::nullopt, DstInfo,
4632
4633 MemOpChains.push_back(Cpy);
4634 } else {
4635 SDValue Store =
4636 DAG.getStore(Chain, DL, Arg, DstAddr, DstInfo, Alignment);
4637 MemOpChains.push_back(Store);
4638 }
4639 }
4640 }
4641
4642 if (!MemOpChains.empty())
4643 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
4644
4645 SDValue ReadFirstLaneID =
4646 DAG.getTargetConstant(Intrinsic::amdgcn_readfirstlane, DL, MVT::i32);
4647
4648 SDValue TokenGlue;
4649 if (CLI.ConvergenceControlToken) {
4650 TokenGlue = DAG.getNode(ISD::CONVERGENCECTRL_GLUE, DL, MVT::Glue,
4652 }
4653
4654 // Build a sequence of copy-to-reg nodes chained together with token chain
4655 // and flag operands which copy the outgoing args into the appropriate regs.
4656 SDValue InGlue;
4657
4658 unsigned ArgIdx = 0;
4659 for (auto [Reg, Val] : RegsToPass) {
4660 if (ArgIdx++ >= NumSpecialInputs &&
4661 (IsChainCallConv || !Val->isDivergent()) && TRI->isSGPRPhysReg(Reg)) {
4662 // For chain calls, the inreg arguments are required to be
4663 // uniform. Speculatively Insert a readfirstlane in case we cannot prove
4664 // they are uniform.
4665 //
4666 // For other calls, if an inreg arguments is known to be uniform,
4667 // speculatively insert a readfirstlane in case it is in a VGPR.
4668 //
4669 // FIXME: We need to execute this in a waterfall loop if it is a divergent
4670 // value, so let that continue to produce invalid code.
4671
4672 SmallVector<SDValue, 3> ReadfirstlaneArgs({ReadFirstLaneID, Val});
4673 if (TokenGlue)
4674 ReadfirstlaneArgs.push_back(TokenGlue);
4676 ReadfirstlaneArgs);
4677 }
4678
4679 Chain = DAG.getCopyToReg(Chain, DL, Reg, Val, InGlue);
4680 InGlue = Chain.getValue(1);
4681 }
4682
4683 // We don't usually want to end the call-sequence here because we would tidy
4684 // the frame up *after* the call, however in the ABI-changing tail-call case
4685 // we've carefully laid out the parameters so that when sp is reset they'll be
4686 // in the correct location.
4687 if (IsTailCall && !IsSibCall) {
4688 Chain = DAG.getCALLSEQ_END(Chain, NumBytes, 0, InGlue, DL);
4689 InGlue = Chain.getValue(1);
4690 }
4691
4692 std::vector<SDValue> Ops({Chain});
4693
4694 // Add a redundant copy of the callee global which will not be legalized, as
4695 // we need direct access to the callee later.
4697 const GlobalValue *GV = GSD->getGlobal();
4698 Ops.push_back(Callee);
4699 Ops.push_back(DAG.getTargetGlobalAddress(GV, DL, MVT::i64));
4700 } else {
4701 if (IsTailCall) {
4702 // isEligibleForTailCallOptimization considered whether the call target is
4703 // divergent, but we may still end up with a uniform value in a VGPR.
4704 // Insert a readfirstlane just in case.
4705 SDValue ReadFirstLaneID =
4706 DAG.getTargetConstant(Intrinsic::amdgcn_readfirstlane, DL, MVT::i32);
4707
4708 SmallVector<SDValue, 3> ReadfirstlaneArgs({ReadFirstLaneID, Callee});
4709 if (TokenGlue)
4710 ReadfirstlaneArgs.push_back(TokenGlue); // Wire up convergence token.
4711 Callee = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Callee.getValueType(),
4712 ReadfirstlaneArgs);
4713 }
4714
4715 Ops.push_back(Callee);
4716 Ops.push_back(DAG.getTargetConstant(0, DL, MVT::i64));
4717 }
4718
4719 if (IsTailCall) {
4720 // Each tail call may have to adjust the stack by a different amount, so
4721 // this information must travel along with the operation for eventual
4722 // consumption by emitEpilogue.
4723 Ops.push_back(DAG.getTargetConstant(FPDiff, DL, MVT::i32));
4724 }
4725
4726 if (IsChainCallConv)
4727 llvm::append_range(Ops, ChainCallSpecialArgs);
4728
4729 // Add argument registers to the end of the list so that they are known live
4730 // into the call.
4731 for (auto &[Reg, Val] : RegsToPass)
4732 Ops.push_back(DAG.getRegister(Reg, Val.getValueType()));
4733
4734 // Add a register mask operand representing the call-preserved registers.
4735 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
4736 assert(Mask && "Missing call preserved mask for calling convention");
4737 Ops.push_back(DAG.getRegisterMask(Mask));
4738
4739 if (SDValue Token = CLI.ConvergenceControlToken) {
4741 GlueOps.push_back(Token);
4742 if (InGlue)
4743 GlueOps.push_back(InGlue);
4744
4745 InGlue = SDValue(DAG.getMachineNode(TargetOpcode::CONVERGENCECTRL_GLUE, DL,
4746 MVT::Glue, GlueOps),
4747 0);
4748 }
4749
4750 if (InGlue)
4751 Ops.push_back(InGlue);
4752
4753 // If we're doing a tall call, use a TC_RETURN here rather than an
4754 // actual call instruction.
4755 if (IsTailCall) {
4756 MFI.setHasTailCall();
4757 unsigned OPC = AMDGPUISD::TC_RETURN;
4758 switch (CallConv) {
4760 OPC = AMDGPUISD::TC_RETURN_GFX;
4761 break;
4764 OPC = UsesDynamicVGPRs ? AMDGPUISD::TC_RETURN_CHAIN_DVGPR
4765 : AMDGPUISD::TC_RETURN_CHAIN;
4766 break;
4767 }
4768
4769 // If the caller is a whole wave function, we need to use a special opcode
4770 // so we can patch up EXEC.
4771 if (Info->isWholeWaveFunction())
4772 OPC = AMDGPUISD::TC_RETURN_GFX_WholeWave;
4773
4774 SDValue Ret = DAG.getNode(OPC, DL, MVT::Other, Ops);
4775 DAG.addNoMergeSiteInfo(Ret.getNode(), CLI.NoMerge);
4776 return Ret;
4777 }
4778
4779 // Returns a chain and a flag for retval copy to use.
4780 SDValue Call = DAG.getNode(AMDGPUISD::CALL, DL, {MVT::Other, MVT::Glue}, Ops);
4781 DAG.addNoMergeSiteInfo(Call.getNode(), CLI.NoMerge);
4782 Chain = Call.getValue(0);
4783 InGlue = Call.getValue(1);
4784
4785 uint64_t CalleePopBytes = NumBytes;
4786 Chain = DAG.getCALLSEQ_END(Chain, 0, CalleePopBytes, InGlue, DL);
4787 if (!Ins.empty())
4788 InGlue = Chain.getValue(1);
4789
4790 // Handle result values, copying them out of physregs into vregs that we
4791 // return.
4792 return LowerCallResult(Chain, InGlue, CallConv, IsVarArg, Ins, DL, DAG,
4793 InVals, /*IsThisReturn=*/false, SDValue());
4794}
4795
4796// This is similar to the default implementation in ExpandDYNAMIC_STACKALLOC,
4797// except for:
4798// 1. Stack growth direction(default: downwards, AMDGPU: upwards), and
4799// 2. Scale size where, scale = wave-reduction(alloca-size) * wave-size
4801 SelectionDAG &DAG) const {
4802 const MachineFunction &MF = DAG.getMachineFunction();
4804
4805 SDLoc dl(Op);
4806 EVT VT = Op.getValueType();
4807 SDValue Chain = Op.getOperand(0);
4808 Register SPReg = Info->getStackPtrOffsetReg();
4809
4810 // Chain the dynamic stack allocation so that it doesn't modify the stack
4811 // pointer when other instructions are using the stack.
4812 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
4813
4814 SDValue Size = Op.getOperand(1);
4815 SDValue BaseAddr = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
4816 Align Alignment = cast<ConstantSDNode>(Op.getOperand(2))->getAlignValue();
4817
4818 const TargetFrameLowering *TFL = Subtarget->getFrameLowering();
4820 "Stack grows upwards for AMDGPU");
4821
4822 Chain = BaseAddr.getValue(1);
4823 // When using flat-scratch, the stack offset is unscaled.
4824 const bool HasFlatScratch = Subtarget->hasFlatScratchEnabled();
4825 const unsigned WavefrontSizeLog2 = Subtarget->getWavefrontSizeLog2();
4826
4827 Align StackAlign = TFL->getStackAlign();
4828 if (Alignment > StackAlign) {
4829 uint64_t ScaledAlignment = Alignment.value()
4830 << (HasFlatScratch ? 0 : WavefrontSizeLog2);
4831 uint64_t StackAlignMask = ScaledAlignment - 1;
4832 SDValue TmpAddr = DAG.getNode(ISD::ADD, dl, VT, BaseAddr,
4833 DAG.getConstant(StackAlignMask, dl, VT));
4834 BaseAddr = DAG.getNode(ISD::AND, dl, VT, TmpAddr,
4835 DAG.getSignedConstant(-ScaledAlignment, dl, VT));
4836 }
4837
4838 assert(Size.getValueType() == MVT::i32 && "Size must be 32-bit");
4839 SDValue NewSP;
4841 // Increase the stack pointer by the size of the alloca.
4842 // If not using flat-scratch, we have to scale the size by the wave-size.
4843 SDValue ScaledSize =
4844 HasFlatScratch
4845 ? Size
4846 : DAG.getNode(ISD::SHL, dl, VT, Size,
4847 DAG.getConstant(WavefrontSizeLog2, dl, MVT::i32));
4848 NewSP = DAG.getNode(ISD::ADD, dl, VT, BaseAddr, ScaledSize); // Value
4849 } else {
4850 // For dynamic sized alloca, perform wave-wide reduction to get max of
4851 // alloca size(divergent), and then scale it (when not using flat-scratch)
4852 // by wave-size.
4853 SDValue WaveReduction =
4854 DAG.getTargetConstant(Intrinsic::amdgcn_wave_reduce_umax, dl, MVT::i32);
4855 Size = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::i32, WaveReduction,
4856 Size, DAG.getTargetConstant(0, dl, MVT::i32));
4857 SDValue ScaledSize = Size;
4858 if (!HasFlatScratch) {
4859 ScaledSize =
4860 DAG.getNode(ISD::SHL, dl, VT, Size,
4861 DAG.getConstant(WavefrontSizeLog2, dl, MVT::i32));
4862 }
4863 NewSP =
4864 DAG.getNode(ISD::ADD, dl, VT, BaseAddr, ScaledSize); // Value in vgpr.
4865 SDValue ReadFirstLaneID =
4866 DAG.getTargetConstant(Intrinsic::amdgcn_readfirstlane, dl, MVT::i32);
4867 NewSP = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::i32, ReadFirstLaneID,
4868 NewSP);
4869 }
4870
4871 Chain = DAG.getCopyToReg(Chain, dl, SPReg, NewSP); // Output chain
4872 SDValue CallSeqEnd = DAG.getCALLSEQ_END(Chain, 0, 0, SDValue(), dl);
4873
4874 return DAG.getMergeValues({BaseAddr, CallSeqEnd}, dl);
4875}
4876
4878 if (Op.getValueType() != MVT::i32)
4879 return Op; // Defer to cannot select error.
4880
4882 SDLoc SL(Op);
4883
4884 SDValue CopyFromSP = DAG.getCopyFromReg(Op->getOperand(0), SL, SP, MVT::i32);
4885
4886 // Convert from wave uniform to swizzled vector address. This should protect
4887 // from any edge cases where the stacksave result isn't directly used with
4888 // stackrestore.
4889 SDValue VectorAddress =
4890 DAG.getNode(AMDGPUISD::WAVE_ADDRESS, SL, MVT::i32, CopyFromSP);
4891 return DAG.getMergeValues({VectorAddress, CopyFromSP.getValue(1)}, SL);
4892}
4893
4895 SelectionDAG &DAG) const {
4896 SDLoc SL(Op);
4897 assert(Op.getValueType() == MVT::i32);
4898
4899 uint32_t BothRoundHwReg =
4901 SDValue GetRoundBothImm = DAG.getTargetConstant(BothRoundHwReg, SL, MVT::i32);
4902
4903 SDValue IntrinID =
4904 DAG.getTargetConstant(Intrinsic::amdgcn_s_getreg, SL, MVT::i32);
4905 SDValue GetReg = DAG.getNode(ISD::INTRINSIC_W_CHAIN, SL, Op->getVTList(),
4906 Op.getOperand(0), IntrinID, GetRoundBothImm);
4907
4908 // There are two rounding modes, one for f32 and one for f64/f16. We only
4909 // report in the standard value range if both are the same.
4910 //
4911 // The raw values also differ from the expected FLT_ROUNDS values. Nearest
4912 // ties away from zero is not supported, and the other values are rotated by
4913 // 1.
4914 //
4915 // If the two rounding modes are not the same, report a target defined value.
4916
4917 // Mode register rounding mode fields:
4918 //
4919 // [1:0] Single-precision round mode.
4920 // [3:2] Double/Half-precision round mode.
4921 //
4922 // 0=nearest even; 1= +infinity; 2= -infinity, 3= toward zero.
4923 //
4924 // Hardware Spec
4925 // Toward-0 3 0
4926 // Nearest Even 0 1
4927 // +Inf 1 2
4928 // -Inf 2 3
4929 // NearestAway0 N/A 4
4930 //
4931 // We have to handle 16 permutations of a 4-bit value, so we create a 64-bit
4932 // table we can index by the raw hardware mode.
4933 //
4934 // (trunc (FltRoundConversionTable >> MODE.fp_round)) & 0xf
4935
4936 SDValue BitTable =
4938
4939 SDValue Two = DAG.getConstant(2, SL, MVT::i32);
4940 SDValue RoundModeTimesNumBits =
4941 DAG.getNode(ISD::SHL, SL, MVT::i32, GetReg, Two);
4942
4943 // TODO: We could possibly avoid a 64-bit shift and use a simpler table if we
4944 // knew only one mode was demanded.
4945 SDValue TableValue =
4946 DAG.getNode(ISD::SRL, SL, MVT::i64, BitTable, RoundModeTimesNumBits);
4947 SDValue TruncTable = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, TableValue);
4948
4949 SDValue EntryMask = DAG.getConstant(0xf, SL, MVT::i32);
4950 SDValue TableEntry =
4951 DAG.getNode(ISD::AND, SL, MVT::i32, TruncTable, EntryMask);
4952
4953 // There's a gap in the 4-bit encoded table and actual enum values, so offset
4954 // if it's an extended value.
4955 SDValue Four = DAG.getConstant(4, SL, MVT::i32);
4956 SDValue IsStandardValue =
4957 DAG.getSetCC(SL, MVT::i1, TableEntry, Four, ISD::SETULT);
4958 SDValue EnumOffset = DAG.getNode(ISD::ADD, SL, MVT::i32, TableEntry, Four);
4959 SDValue Result = DAG.getNode(ISD::SELECT, SL, MVT::i32, IsStandardValue,
4960 TableEntry, EnumOffset);
4961
4962 return DAG.getMergeValues({Result, GetReg.getValue(1)}, SL);
4963}
4964
4966 SelectionDAG &DAG) const {
4967 SDLoc SL(Op);
4968
4969 SDValue NewMode = Op.getOperand(1);
4970 assert(NewMode.getValueType() == MVT::i32);
4971
4972 // Index a table of 4-bit entries mapping from the C FLT_ROUNDS values to the
4973 // hardware MODE.fp_round values.
4974 if (auto *ConstMode = dyn_cast<ConstantSDNode>(NewMode)) {
4975 uint32_t ClampedVal = std::min(
4976 static_cast<uint32_t>(ConstMode->getZExtValue()),
4978 NewMode = DAG.getConstant(
4979 AMDGPU::decodeFltRoundToHWConversionTable(ClampedVal), SL, MVT::i32);
4980 } else {
4981 // If we know the input can only be one of the supported standard modes in
4982 // the range 0-3, we can use a simplified mapping to hardware values.
4983 KnownBits KB = DAG.computeKnownBits(NewMode);
4984 const bool UseReducedTable = KB.countMinLeadingZeros() >= 30;
4985 // The supported standard values are 0-3. The extended values start at 8. We
4986 // need to offset by 4 if the value is in the extended range.
4987
4988 if (UseReducedTable) {
4989 // Truncate to the low 32-bits.
4990 SDValue BitTable = DAG.getConstant(
4991 AMDGPU::FltRoundToHWConversionTable & 0xffff, SL, MVT::i32);
4992
4993 SDValue Two = DAG.getConstant(2, SL, MVT::i32);
4994 SDValue RoundModeTimesNumBits =
4995 DAG.getNode(ISD::SHL, SL, MVT::i32, NewMode, Two);
4996
4997 NewMode =
4998 DAG.getNode(ISD::SRL, SL, MVT::i32, BitTable, RoundModeTimesNumBits);
4999
5000 // TODO: SimplifyDemandedBits on the setreg source here can likely reduce
5001 // the table extracted bits into inline immediates.
5002 } else {
5003 // table_index = umin(value, value - 4)
5004 // MODE.fp_round = (bit_table >> (table_index << 2)) & 0xf
5005 SDValue BitTable =
5007
5008 SDValue Four = DAG.getConstant(4, SL, MVT::i32);
5009 SDValue OffsetEnum = DAG.getNode(ISD::SUB, SL, MVT::i32, NewMode, Four);
5010 SDValue IndexVal =
5011 DAG.getNode(ISD::UMIN, SL, MVT::i32, NewMode, OffsetEnum);
5012
5013 SDValue Two = DAG.getConstant(2, SL, MVT::i32);
5014 SDValue RoundModeTimesNumBits =
5015 DAG.getNode(ISD::SHL, SL, MVT::i32, IndexVal, Two);
5016
5017 SDValue TableValue =
5018 DAG.getNode(ISD::SRL, SL, MVT::i64, BitTable, RoundModeTimesNumBits);
5019 SDValue TruncTable = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, TableValue);
5020
5021 // No need to mask out the high bits since the setreg will ignore them
5022 // anyway.
5023 NewMode = TruncTable;
5024 }
5025
5026 // Insert a readfirstlane in case the value is a VGPR. We could do this
5027 // earlier and keep more operations scalar, but that interferes with
5028 // combining the source.
5029 SDValue ReadFirstLaneID =
5030 DAG.getTargetConstant(Intrinsic::amdgcn_readfirstlane, SL, MVT::i32);
5031 NewMode = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32,
5032 ReadFirstLaneID, NewMode);
5033 }
5034
5035 // N.B. The setreg will be later folded into s_round_mode on supported
5036 // targets.
5037 SDValue IntrinID =
5038 DAG.getTargetConstant(Intrinsic::amdgcn_s_setreg, SL, MVT::i32);
5039 uint32_t BothRoundHwReg =
5041 SDValue RoundBothImm = DAG.getTargetConstant(BothRoundHwReg, SL, MVT::i32);
5042
5043 SDValue SetReg =
5044 DAG.getNode(ISD::INTRINSIC_VOID, SL, Op->getVTList(), Op.getOperand(0),
5045 IntrinID, RoundBothImm, NewMode);
5046
5047 return SetReg;
5048}
5049
5051 unsigned AS = cast<MemSDNode>(Op)->getAddressSpace();
5052 if (AS == AMDGPUAS::BUFFER_RESOURCE)
5053 return Op.getOperand(0);
5055 return SDValue();
5056
5057 // I$ prefetch
5058 if (!Subtarget->hasSafeSmemPrefetch() && !Op.getConstantOperandVal(4))
5059 return SDValue();
5060
5061 // Prefetch the 64-bit address that a 32-bit constant pointer is cast to, as
5062 // a load does.
5063 SDValue Ptr = Op.getOperand(1);
5064 if (Ptr.getValueType() != MVT::i32)
5065 return Op;
5067 Ops[1] = DAG.getAddrSpaceCast(SDLoc(Op), MVT::i64, Ptr,
5070 return SDValue(DAG.UpdateNodeOperands(Op.getNode(), Ops), 0);
5071}
5072
5073// Work around DAG legality rules only based on the result type.
5075 bool IsStrict = Op.getOpcode() == ISD::STRICT_FP_EXTEND;
5076 SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
5077 EVT SrcVT = Src.getValueType();
5078
5079 if (SrcVT.getScalarType() != MVT::bf16)
5080 return Op;
5081
5082 SDLoc SL(Op);
5083 SDValue BitCast =
5084 DAG.getNode(ISD::BITCAST, SL, SrcVT.changeTypeToInteger(), Src);
5085
5086 EVT DstVT = Op.getValueType();
5087 if (IsStrict)
5088 llvm_unreachable("Need STRICT_BF16_TO_FP");
5089
5090 return DAG.getNode(ISD::BF16_TO_FP, SL, DstVT, BitCast);
5091}
5092
5094 SDLoc SL(Op);
5095 if (Op.getValueType() != MVT::i64)
5096 return Op;
5097
5098 uint32_t ModeHwReg =
5100 SDValue ModeHwRegImm = DAG.getTargetConstant(ModeHwReg, SL, MVT::i32);
5101 uint32_t TrapHwReg =
5103 SDValue TrapHwRegImm = DAG.getTargetConstant(TrapHwReg, SL, MVT::i32);
5104
5105 SDVTList VTList = DAG.getVTList(MVT::i32, MVT::Other);
5106 SDValue IntrinID =
5107 DAG.getTargetConstant(Intrinsic::amdgcn_s_getreg, SL, MVT::i32);
5108 SDValue GetModeReg = DAG.getNode(ISD::INTRINSIC_W_CHAIN, SL, VTList,
5109 Op.getOperand(0), IntrinID, ModeHwRegImm);
5110 SDValue GetTrapReg = DAG.getNode(ISD::INTRINSIC_W_CHAIN, SL, VTList,
5111 Op.getOperand(0), IntrinID, TrapHwRegImm);
5112 SDValue TokenReg =
5113 DAG.getNode(ISD::TokenFactor, SL, MVT::Other, GetModeReg.getValue(1),
5114 GetTrapReg.getValue(1));
5115
5116 SDValue CvtPtr =
5117 DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, GetModeReg, GetTrapReg);
5118 SDValue Result = DAG.getNode(ISD::BITCAST, SL, MVT::i64, CvtPtr);
5119
5120 return DAG.getMergeValues({Result, TokenReg}, SL);
5121}
5122
5124 SDLoc SL(Op);
5125 if (Op.getOperand(1).getValueType() != MVT::i64)
5126 return Op;
5127
5128 SDValue Input = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op.getOperand(1));
5129 SDValue NewModeReg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Input,
5130 DAG.getConstant(0, SL, MVT::i32));
5131 SDValue NewTrapReg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Input,
5132 DAG.getConstant(1, SL, MVT::i32));
5133
5134 SDValue ReadFirstLaneID =
5135 DAG.getTargetConstant(Intrinsic::amdgcn_readfirstlane, SL, MVT::i32);
5136 NewModeReg = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32,
5137 ReadFirstLaneID, NewModeReg);
5138 NewTrapReg = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32,
5139 ReadFirstLaneID, NewTrapReg);
5140
5141 unsigned ModeHwReg =
5143 SDValue ModeHwRegImm = DAG.getTargetConstant(ModeHwReg, SL, MVT::i32);
5144 unsigned TrapHwReg =
5146 SDValue TrapHwRegImm = DAG.getTargetConstant(TrapHwReg, SL, MVT::i32);
5147
5148 SDValue IntrinID =
5149 DAG.getTargetConstant(Intrinsic::amdgcn_s_setreg, SL, MVT::i32);
5150 SDValue SetModeReg =
5151 DAG.getNode(ISD::INTRINSIC_VOID, SL, MVT::Other, Op.getOperand(0),
5152 IntrinID, ModeHwRegImm, NewModeReg);
5153 SDValue SetTrapReg =
5154 DAG.getNode(ISD::INTRINSIC_VOID, SL, MVT::Other, Op.getOperand(0),
5155 IntrinID, TrapHwRegImm, NewTrapReg);
5156 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, SetTrapReg, SetModeReg);
5157}
5158
5160 const MachineFunction &MF) const {
5161 Register Reg =
5163 .Case("m0", AMDGPU::M0)
5164 .Case("exec", AMDGPU::EXEC)
5165 .Case("exec_lo", AMDGPU::EXEC_LO)
5166 .Case("exec_hi", AMDGPU::EXEC_HI)
5167 .Case("flat_scratch", AMDGPU::FLAT_SCR)
5168 .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
5169 .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
5170 .Case("src_flat_scratch_base", AMDGPU::SRC_FLAT_SCRATCH_BASE)
5171 .Case("src_flat_scratch_base_lo", AMDGPU::SRC_FLAT_SCRATCH_BASE_LO)
5172 .Case("src_flat_scratch_base_hi", AMDGPU::SRC_FLAT_SCRATCH_BASE_HI)
5173 .Default(Register());
5174 if (!Reg)
5175 return Reg;
5176
5177 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
5178 if (!Subtarget->hasFlatScrRegister() &&
5179 TRI->regsOverlap(Reg, AMDGPU::FLAT_SCR))
5180 return Register();
5181
5182 if (!Subtarget->hasGloballyAddressableScratch() &&
5183 TRI->regsOverlap(Reg, AMDGPU::SRC_FLAT_SCRATCH_BASE))
5184 return Register();
5185
5186 switch (Reg) {
5187 case AMDGPU::M0:
5188 case AMDGPU::EXEC_LO:
5189 case AMDGPU::EXEC_HI:
5190 case AMDGPU::FLAT_SCR_LO:
5191 case AMDGPU::FLAT_SCR_HI:
5192 case AMDGPU::SRC_FLAT_SCRATCH_BASE_LO:
5193 case AMDGPU::SRC_FLAT_SCRATCH_BASE_HI:
5194 if (VT.getSizeInBits() == 32)
5195 return Reg;
5196 break;
5197 case AMDGPU::EXEC:
5198 case AMDGPU::FLAT_SCR:
5199 case AMDGPU::SRC_FLAT_SCRATCH_BASE:
5200 if (VT.getSizeInBits() == 64)
5201 return Reg;
5202 break;
5203 default:
5204 llvm_unreachable("missing register type checking");
5205 }
5206
5208 Twine("invalid type for register \"" + StringRef(RegName) + "\"."));
5209}
5210
5211// If kill is not the last instruction, split the block so kill is always a
5212// proper terminator.
5215 MachineBasicBlock *BB) const {
5216 MachineBasicBlock *SplitBB = BB->splitAt(MI, /*UpdateLiveIns=*/true);
5218 MI.setDesc(TII->getKillTerminatorFromPseudo(MI.getOpcode()));
5219 return SplitBB;
5220}
5221
5222// Split block \p MBB at \p MI, as to insert a loop. If \p InstInLoop is true,
5223// \p MI will be the only instruction in the loop body block. Otherwise, it will
5224// be the first instruction in the remainder block.
5225//
5226/// \returns { LoopBody, Remainder }
5227static std::pair<MachineBasicBlock *, MachineBasicBlock *>
5229 MachineFunction *MF = MBB.getParent();
5231
5232 // To insert the loop we need to split the block. Move everything after this
5233 // point to a new block, and insert a new empty block between the two.
5235 MachineBasicBlock *RemainderBB = MF->CreateMachineBasicBlock();
5237 ++MBBI;
5238
5239 MF->insert(MBBI, LoopBB);
5240 MF->insert(MBBI, RemainderBB);
5241
5242 LoopBB->addSuccessor(LoopBB);
5243 LoopBB->addSuccessor(RemainderBB);
5244
5245 // Move the rest of the block into a new block.
5246 RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB);
5247
5248 if (InstInLoop) {
5249 auto Next = std::next(I);
5250
5251 // Move instruction to loop body.
5252 LoopBB->splice(LoopBB->begin(), &MBB, I, Next);
5253
5254 // Move the rest of the block.
5255 RemainderBB->splice(RemainderBB->begin(), &MBB, Next, MBB.end());
5256 } else {
5257 RemainderBB->splice(RemainderBB->begin(), &MBB, I, MBB.end());
5258 }
5259
5260 MBB.addSuccessor(LoopBB);
5261
5262 return std::pair(LoopBB, RemainderBB);
5263}
5264
5265/// Insert \p MI into a BUNDLE with an S_WAITCNT 0 immediately following it.
5267 MachineBasicBlock *MBB = MI.getParent();
5269 auto I = MI.getIterator();
5270 auto E = std::next(I);
5271
5272 // clang-format off
5273 BuildMI(*MBB, E, MI.getDebugLoc(), TII->get(AMDGPU::S_WAITCNT))
5274 .addImm(0);
5275 // clang-format on
5276
5277 MIBundleBuilder Bundler(*MBB, I, E);
5278 finalizeBundle(*MBB, Bundler.begin());
5279}
5280
5283 MachineBasicBlock *BB) const {
5284 const DebugLoc &DL = MI.getDebugLoc();
5285
5287
5289
5290 // Apparently kill flags are only valid if the def is in the same block?
5291 if (MachineOperand *Src = TII->getNamedOperand(MI, AMDGPU::OpName::data0))
5292 Src->setIsKill(false);
5293
5294 auto [LoopBB, RemainderBB] = splitBlockForLoop(MI, *BB, true);
5295
5296 MachineBasicBlock::iterator I = LoopBB->end();
5297
5298 const unsigned EncodedReg = AMDGPU::Hwreg::HwregEncoding::encode(
5300
5301 // Clear TRAP_STS.MEM_VIOL
5302 BuildMI(*LoopBB, LoopBB->begin(), DL, TII->get(AMDGPU::S_SETREG_IMM32_B32))
5303 .addImm(0)
5304 .addImm(EncodedReg);
5305
5307
5308 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5309
5310 // Load and check TRAP_STS.MEM_VIOL
5311 BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::S_GETREG_B32), Reg)
5312 .addImm(EncodedReg);
5313
5314 // clang-format off
5315 // FIXME: Do we need to use an isel pseudo that may clobber scc?
5316 BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::S_CMP_LG_U32))
5317 .addReg(Reg)
5318 .addImm(0);
5319 BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::S_CBRANCH_SCC1))
5320 .addMBB(LoopBB);
5321 // clang-format on
5322
5323 return RemainderBB;
5324}
5325
5326// Do a v_movrels_b32 or v_movreld_b32 for each unique value of \p IdxReg in the
5327// wavefront. If the value is uniform and just happens to be in a VGPR, this
5328// will only do one iteration. In the worst case, this will loop 64 times.
5329//
5330// TODO: Just use v_readlane_b32 if we know the VGPR has a uniform value.
5333 MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB,
5334 const DebugLoc &DL, const MachineOperand &Idx,
5335 unsigned InitReg, unsigned ResultReg, unsigned PhiReg,
5336 unsigned InitSaveExecReg, int Offset, bool UseGPRIdxMode,
5337 Register &SGPRIdxReg) {
5338
5339 MachineFunction *MF = OrigBB.getParent();
5340 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
5341 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5344
5345 const TargetRegisterClass *BoolRC = TRI->getBoolRC();
5346 Register PhiExec = MRI.createVirtualRegister(BoolRC);
5347 Register NewExec = MRI.createVirtualRegister(BoolRC);
5348 Register CurrentIdxReg =
5349 MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5350 Register CondReg = MRI.createVirtualRegister(BoolRC);
5351
5352 BuildMI(LoopBB, I, DL, TII->get(TargetOpcode::PHI), PhiReg)
5353 .addReg(InitReg)
5354 .addMBB(&OrigBB)
5355 .addReg(ResultReg)
5356 .addMBB(&LoopBB);
5357
5358 BuildMI(LoopBB, I, DL, TII->get(TargetOpcode::PHI), PhiExec)
5359 .addReg(InitSaveExecReg)
5360 .addMBB(&OrigBB)
5361 .addReg(NewExec)
5362 .addMBB(&LoopBB);
5363
5364 // Read the next variant <- also loop target.
5365 BuildMI(LoopBB, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), CurrentIdxReg)
5366 .addReg(Idx.getReg(), getUndefRegState(Idx.isUndef()));
5367
5368 // Compare the just read M0 value to all possible Idx values.
5369 BuildMI(LoopBB, I, DL, TII->get(AMDGPU::V_CMP_EQ_U32_e64), CondReg)
5370 .addReg(CurrentIdxReg)
5371 .addReg(Idx.getReg(), {}, Idx.getSubReg());
5372
5373 // Update EXEC, save the original EXEC value to VCC.
5374 BuildMI(LoopBB, I, DL, TII->get(LMC.AndSaveExecOpc), NewExec)
5375 .addReg(CondReg)
5376 .setOperandDead(3); // Dead scc
5377
5378 MRI.setSimpleHint(NewExec, CondReg);
5379
5380 if (UseGPRIdxMode) {
5381 if (Offset == 0) {
5382 SGPRIdxReg = CurrentIdxReg;
5383 } else {
5384 SGPRIdxReg = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
5385 BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_ADD_I32), SGPRIdxReg)
5386 .addReg(CurrentIdxReg)
5387 .addImm(Offset)
5388 .setOperandDead(3); // Dead scc
5389 }
5390 } else {
5391 // Move index from VCC into M0
5392 if (Offset == 0) {
5393 BuildMI(LoopBB, I, DL, TII->get(AMDGPU::COPY), AMDGPU::M0)
5394 .addReg(CurrentIdxReg);
5395 } else {
5396 BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_ADD_I32), AMDGPU::M0)
5397 .addReg(CurrentIdxReg)
5398 .addImm(Offset)
5399 .setOperandDead(3); // Dead scc
5400 }
5401 }
5402
5403 // Update EXEC, switch all done bits to 0 and all todo bits to 1.
5404 MachineInstr *InsertPt =
5405 BuildMI(LoopBB, I, DL, TII->get(LMC.XorTermOpc), LMC.ExecReg)
5406 .addReg(LMC.ExecReg)
5407 .addReg(NewExec)
5408 .setOperandDead(3); // Dead scc
5409
5410 // XXX - s_xor_b64 sets scc to 1 if the result is nonzero, so can we use
5411 // s_cbranch_scc0?
5412
5413 // Loop back to V_READFIRSTLANE_B32 if there are still variants to cover.
5414 // clang-format off
5415 BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_CBRANCH_EXECNZ))
5416 .addMBB(&LoopBB);
5417 // clang-format on
5418
5419 return InsertPt->getIterator();
5420}
5421
5422// This has slightly sub-optimal regalloc when the source vector is killed by
5423// the read. The register allocator does not understand that the kill is
5424// per-workitem, so is kept alive for the whole loop so we end up not re-using a
5425// subregister from it, using 1 more VGPR than necessary. This was saved when
5426// this was expanded after register allocation.
5429 unsigned InitResultReg, unsigned PhiReg, int Offset,
5430 bool UseGPRIdxMode, Register &SGPRIdxReg) {
5431 MachineFunction *MF = MBB.getParent();
5432 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
5433 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5435 const DebugLoc &DL = MI.getDebugLoc();
5437
5438 const auto *BoolXExecRC = TRI->getWaveMaskRegClass();
5439 Register DstReg = MI.getOperand(0).getReg();
5440 Register SaveExec = MRI.createVirtualRegister(BoolXExecRC);
5441 Register TmpExec = MRI.createVirtualRegister(BoolXExecRC);
5443
5444 BuildMI(MBB, I, DL, TII->get(TargetOpcode::IMPLICIT_DEF), TmpExec);
5445
5446 // Save the EXEC mask
5447 // clang-format off
5448 BuildMI(MBB, I, DL, TII->get(LMC.MovOpc), SaveExec)
5449 .addReg(LMC.ExecReg);
5450 // clang-format on
5451
5452 auto [LoopBB, RemainderBB] = splitBlockForLoop(MI, MBB, false);
5453
5454 const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
5455
5456 auto InsPt = emitLoadM0FromVGPRLoop(TII, MRI, MBB, *LoopBB, DL, *Idx,
5457 InitResultReg, DstReg, PhiReg, TmpExec,
5458 Offset, UseGPRIdxMode, SGPRIdxReg);
5459
5460 MachineBasicBlock *LandingPad = MF->CreateMachineBasicBlock();
5462 ++MBBI;
5463 MF->insert(MBBI, LandingPad);
5464 LoopBB->removeSuccessor(RemainderBB);
5465 LandingPad->addSuccessor(RemainderBB);
5466 LoopBB->addSuccessor(LandingPad);
5467 MachineBasicBlock::iterator First = LandingPad->begin();
5468 // clang-format off
5469 BuildMI(*LandingPad, First, DL, TII->get(LMC.MovOpc), LMC.ExecReg)
5470 .addReg(SaveExec);
5471 // clang-format on
5472
5473 return InsPt;
5474}
5475
5476// Returns subreg index, offset
5477static std::pair<unsigned, int>
5479 const TargetRegisterClass *SuperRC, unsigned VecReg,
5480 int Offset) {
5481 int NumElts = TRI.getRegSizeInBits(*SuperRC) / 32;
5482
5483 // Skip out of bounds offsets, or else we would end up using an undefined
5484 // register.
5485 if (Offset >= NumElts || Offset < 0)
5486 return std::pair(AMDGPU::sub0, Offset);
5487
5488 return std::pair(SIRegisterInfo::getSubRegFromChannel(Offset), 0);
5489}
5490
5493 int Offset) {
5494 MachineBasicBlock *MBB = MI.getParent();
5495 const DebugLoc &DL = MI.getDebugLoc();
5497
5498 const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
5499
5500 assert(Idx->getReg() != AMDGPU::NoRegister);
5501
5502 if (Offset == 0) {
5503 // clang-format off
5504 BuildMI(*MBB, I, DL, TII->get(AMDGPU::COPY), AMDGPU::M0)
5505 .add(*Idx);
5506 // clang-format on
5507 } else {
5508 BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_ADD_I32), AMDGPU::M0)
5509 .add(*Idx)
5510 .addImm(Offset)
5511 .setOperandDead(3); // Dead scc
5512 }
5513}
5514
5517 int Offset) {
5518 MachineBasicBlock *MBB = MI.getParent();
5519 const DebugLoc &DL = MI.getDebugLoc();
5521
5522 const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
5523
5524 if (Offset == 0)
5525 return Idx->getReg();
5526
5527 Register Tmp = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5528 BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_ADD_I32), Tmp)
5529 .add(*Idx)
5530 .addImm(Offset)
5531 .setOperandDead(3); // Dead scc
5532 return Tmp;
5533}
5534
5537 const GCNSubtarget &ST) {
5538 const SIInstrInfo *TII = ST.getInstrInfo();
5539 const SIRegisterInfo &TRI = TII->getRegisterInfo();
5540 MachineFunction *MF = MBB.getParent();
5542
5543 Register Dst = MI.getOperand(0).getReg();
5544 const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
5545 Register SrcReg = TII->getNamedOperand(MI, AMDGPU::OpName::src)->getReg();
5546 int Offset = TII->getNamedOperand(MI, AMDGPU::OpName::offset)->getImm();
5547
5548 const TargetRegisterClass *VecRC = MRI.getRegClass(SrcReg);
5549 const TargetRegisterClass *IdxRC = MRI.getRegClass(Idx->getReg());
5550
5551 unsigned SubReg;
5552 std::tie(SubReg, Offset) =
5553 computeIndirectRegAndOffset(TRI, VecRC, SrcReg, Offset);
5554
5555 const bool UseGPRIdxMode = ST.useVGPRIndexMode();
5556
5557 // Check for a SGPR index.
5558 if (TII->getRegisterInfo().isSGPRClass(IdxRC)) {
5560 const DebugLoc &DL = MI.getDebugLoc();
5561
5562 if (UseGPRIdxMode) {
5563 // TODO: Look at the uses to avoid the copy. This may require rescheduling
5564 // to avoid interfering with other uses, so probably requires a new
5565 // optimization pass.
5567
5568 const MCInstrDesc &GPRIDXDesc =
5569 TII->getIndirectGPRIDXPseudo(TRI.getRegSizeInBits(*VecRC), true);
5570 BuildMI(MBB, I, DL, GPRIDXDesc, Dst)
5571 .addReg(SrcReg)
5572 .addReg(Idx)
5573 .addImm(SubReg);
5574 } else {
5576
5577 BuildMI(MBB, I, DL, TII->get(AMDGPU::V_MOVRELS_B32_e32), Dst)
5578 .addReg(SrcReg, {}, SubReg)
5579 .addReg(SrcReg, RegState::Implicit);
5580 }
5581
5582 MI.eraseFromParent();
5583
5584 return &MBB;
5585 }
5586
5587 // Control flow needs to be inserted if indexing with a VGPR.
5588 const DebugLoc &DL = MI.getDebugLoc();
5590
5591 Register PhiReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5592 Register InitReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5593
5594 BuildMI(MBB, I, DL, TII->get(TargetOpcode::IMPLICIT_DEF), InitReg);
5595
5596 Register SGPRIdxReg;
5597 auto InsPt = loadM0FromVGPR(TII, MBB, MI, InitReg, PhiReg, Offset,
5598 UseGPRIdxMode, SGPRIdxReg);
5599
5600 MachineBasicBlock *LoopBB = InsPt->getParent();
5601
5602 if (UseGPRIdxMode) {
5603 const MCInstrDesc &GPRIDXDesc =
5604 TII->getIndirectGPRIDXPseudo(TRI.getRegSizeInBits(*VecRC), true);
5605
5606 BuildMI(*LoopBB, InsPt, DL, GPRIDXDesc, Dst)
5607 .addReg(SrcReg)
5608 .addReg(SGPRIdxReg)
5609 .addImm(SubReg);
5610 } else {
5611 BuildMI(*LoopBB, InsPt, DL, TII->get(AMDGPU::V_MOVRELS_B32_e32), Dst)
5612 .addReg(SrcReg, {}, SubReg)
5613 .addReg(SrcReg, RegState::Implicit);
5614 }
5615
5616 MI.eraseFromParent();
5617
5618 return LoopBB;
5619}
5620
5623 const GCNSubtarget &ST) {
5624 const SIInstrInfo *TII = ST.getInstrInfo();
5625 const SIRegisterInfo &TRI = TII->getRegisterInfo();
5626 MachineFunction *MF = MBB.getParent();
5628
5629 Register Dst = MI.getOperand(0).getReg();
5630 const MachineOperand *SrcVec = TII->getNamedOperand(MI, AMDGPU::OpName::src);
5631 const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
5632 const MachineOperand *Val = TII->getNamedOperand(MI, AMDGPU::OpName::val);
5633 int Offset = TII->getNamedOperand(MI, AMDGPU::OpName::offset)->getImm();
5634 const TargetRegisterClass *VecRC = MRI.getRegClass(SrcVec->getReg());
5635 const TargetRegisterClass *IdxRC = MRI.getRegClass(Idx->getReg());
5636
5637 // This can be an immediate, but will be folded later.
5638 assert(Val->getReg());
5639
5640 unsigned SubReg;
5641 std::tie(SubReg, Offset) =
5642 computeIndirectRegAndOffset(TRI, VecRC, SrcVec->getReg(), Offset);
5643 const bool UseGPRIdxMode = ST.useVGPRIndexMode();
5644
5645 if (Idx->getReg() == AMDGPU::NoRegister) {
5647 const DebugLoc &DL = MI.getDebugLoc();
5648
5649 assert(Offset == 0);
5650
5651 BuildMI(MBB, I, DL, TII->get(TargetOpcode::INSERT_SUBREG), Dst)
5652 .add(*SrcVec)
5653 .add(*Val)
5654 .addImm(SubReg);
5655
5656 MI.eraseFromParent();
5657 return &MBB;
5658 }
5659
5660 // Check for a SGPR index.
5661 if (TII->getRegisterInfo().isSGPRClass(IdxRC)) {
5663 const DebugLoc &DL = MI.getDebugLoc();
5664
5665 if (UseGPRIdxMode) {
5667
5668 const MCInstrDesc &GPRIDXDesc =
5669 TII->getIndirectGPRIDXPseudo(TRI.getRegSizeInBits(*VecRC), false);
5670 BuildMI(MBB, I, DL, GPRIDXDesc, Dst)
5671 .addReg(SrcVec->getReg())
5672 .add(*Val)
5673 .addReg(Idx)
5674 .addImm(SubReg);
5675 } else {
5677
5678 const MCInstrDesc &MovRelDesc = TII->getIndirectRegWriteMovRelPseudo(
5679 TRI.getRegSizeInBits(*VecRC), 32, false);
5680 BuildMI(MBB, I, DL, MovRelDesc, Dst)
5681 .addReg(SrcVec->getReg())
5682 .add(*Val)
5683 .addImm(SubReg);
5684 }
5685 MI.eraseFromParent();
5686 return &MBB;
5687 }
5688
5689 // Control flow needs to be inserted if indexing with a VGPR.
5690 const DebugLoc &DL = MI.getDebugLoc();
5691
5692 Register PhiReg = MRI.createVirtualRegister(VecRC);
5693
5694 Register SGPRIdxReg;
5695 auto InsPt = loadM0FromVGPR(TII, MBB, MI, SrcVec->getReg(), PhiReg, Offset,
5696 UseGPRIdxMode, SGPRIdxReg);
5697 MachineBasicBlock *LoopBB = InsPt->getParent();
5698
5699 if (UseGPRIdxMode) {
5700 const MCInstrDesc &GPRIDXDesc =
5701 TII->getIndirectGPRIDXPseudo(TRI.getRegSizeInBits(*VecRC), false);
5702
5703 BuildMI(*LoopBB, InsPt, DL, GPRIDXDesc, Dst)
5704 .addReg(PhiReg)
5705 .add(*Val)
5706 .addReg(SGPRIdxReg)
5707 .addImm(SubReg);
5708 } else {
5709 const MCInstrDesc &MovRelDesc = TII->getIndirectRegWriteMovRelPseudo(
5710 TRI.getRegSizeInBits(*VecRC), 32, false);
5711 BuildMI(*LoopBB, InsPt, DL, MovRelDesc, Dst)
5712 .addReg(PhiReg)
5713 .add(*Val)
5714 .addImm(SubReg);
5715 }
5716
5717 MI.eraseFromParent();
5718 return LoopBB;
5719}
5720
5722 MachineBasicBlock *BB) {
5723 // For targets older than GFX12, we emit a sequence of 32-bit operations.
5724 // For GFX12, we emit s_add_u64 and s_sub_u64.
5725 MachineFunction *MF = BB->getParent();
5726 const SIInstrInfo *TII = MF->getSubtarget<GCNSubtarget>().getInstrInfo();
5727 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
5729 const DebugLoc &DL = MI.getDebugLoc();
5730 MachineOperand &Dest = MI.getOperand(0);
5731 MachineOperand &Src0 = MI.getOperand(1);
5732 MachineOperand &Src1 = MI.getOperand(2);
5733 bool IsAdd = (MI.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO);
5734 if (ST.hasScalarAddSub64()) {
5735 // FIXME: If scc is used, this deletes the def
5736 unsigned Opc = IsAdd ? AMDGPU::S_ADD_U64 : AMDGPU::S_SUB_U64;
5737 // clang-format off
5738 BuildMI(*BB, MI, DL, TII->get(Opc), Dest.getReg())
5739 .add(Src0)
5740 .add(Src1);
5741 // clang-format on
5742 } else {
5743 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5744 const TargetRegisterClass *BoolRC = TRI->getBoolRC();
5745
5746 Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
5747 Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
5748
5749 MachineOperand Src0Sub0 = TII->buildExtractSubRegOrImm(
5750 MI, MRI, Src0, BoolRC, AMDGPU::sub0, &AMDGPU::SReg_32RegClass);
5751 MachineOperand Src0Sub1 = TII->buildExtractSubRegOrImm(
5752 MI, MRI, Src0, BoolRC, AMDGPU::sub1, &AMDGPU::SReg_32RegClass);
5753
5754 MachineOperand Src1Sub0 = TII->buildExtractSubRegOrImm(
5755 MI, MRI, Src1, BoolRC, AMDGPU::sub0, &AMDGPU::SReg_32RegClass);
5756 MachineOperand Src1Sub1 = TII->buildExtractSubRegOrImm(
5757 MI, MRI, Src1, BoolRC, AMDGPU::sub1, &AMDGPU::SReg_32RegClass);
5758
5759 const MachineOperand &ImpDefSCC = MI.getOperand(3);
5760 assert(ImpDefSCC.getReg() == AMDGPU::SCC && ImpDefSCC.isDef());
5761
5762 unsigned LoOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32;
5763 unsigned HiOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32;
5764 BuildMI(*BB, MI, DL, TII->get(LoOpc), DestSub0).add(Src0Sub0).add(Src1Sub0);
5765 auto Hi = BuildMI(*BB, MI, DL, TII->get(HiOpc), DestSub1)
5766 .add(Src0Sub1)
5767 .add(Src1Sub1);
5768 if (ImpDefSCC.isDead())
5769 Hi.setOperandDead(3);
5770 BuildMI(*BB, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), Dest.getReg())
5771 .addReg(DestSub0)
5772 .addImm(AMDGPU::sub0)
5773 .addReg(DestSub1)
5774 .addImm(AMDGPU::sub1);
5775 }
5776 MI.eraseFromParent();
5777 return BB;
5778}
5779
5781 MachineFunction *MF = BB->getParent();
5782 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
5783 const SIInstrInfo *TII = ST.getInstrInfo();
5784 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5786 const DebugLoc &DL = MI.getDebugLoc();
5787 Register Dst = MI.getOperand(0).getReg();
5788 const MachineOperand &Src0 = MI.getOperand(1);
5789 const MachineOperand &Src1 = MI.getOperand(2);
5790 Register SrcCond = MI.getOperand(3).getReg();
5791
5792 Register DstLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5793 Register DstHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5794 const TargetRegisterClass *CondRC = TRI->getWaveMaskRegClass();
5795 Register SrcCondCopy = MRI.createVirtualRegister(CondRC);
5796
5797 int Src0Idx =
5798 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0);
5799 int Src1Idx =
5800 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src1);
5801 const TargetRegisterClass *Src0RC =
5802 TRI->getAllocatableClass(TII->getRegClass(MI.getDesc(), Src0Idx));
5803 const TargetRegisterClass *Src1RC =
5804 TRI->getAllocatableClass(TII->getRegClass(MI.getDesc(), Src1Idx));
5805
5806 const TargetRegisterClass *Src0SubRC =
5807 TRI->getSubRegisterClass(Src0RC, AMDGPU::sub0);
5808 const TargetRegisterClass *Src1SubRC =
5809 TRI->getSubRegisterClass(Src1RC, AMDGPU::sub1);
5810
5811 MachineOperand Src0Sub0 = TII->buildExtractSubRegOrImm(
5812 MI, MRI, Src0, Src0RC, AMDGPU::sub0, Src0SubRC);
5813 MachineOperand Src1Sub0 = TII->buildExtractSubRegOrImm(
5814 MI, MRI, Src1, Src1RC, AMDGPU::sub0, Src1SubRC);
5815
5816 MachineOperand Src0Sub1 = TII->buildExtractSubRegOrImm(
5817 MI, MRI, Src0, Src0RC, AMDGPU::sub1, Src0SubRC);
5818 MachineOperand Src1Sub1 = TII->buildExtractSubRegOrImm(
5819 MI, MRI, Src1, Src1RC, AMDGPU::sub1, Src1SubRC);
5820
5821 BuildMI(*BB, MI, DL, TII->get(AMDGPU::COPY), SrcCondCopy).addReg(SrcCond);
5822 BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_CNDMASK_B32_e64), DstLo)
5823 .addImm(0)
5824 .add(Src0Sub0)
5825 .addImm(0)
5826 .add(Src1Sub0)
5827 .addReg(SrcCondCopy);
5828
5829 BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_CNDMASK_B32_e64), DstHi)
5830 .addImm(0)
5831 .add(Src0Sub1)
5832 .addImm(0)
5833 .add(Src1Sub1)
5834 .addReg(SrcCondCopy);
5835
5836 BuildMI(*BB, MI, DL, TII->get(AMDGPU::REG_SEQUENCE), Dst)
5837 .addReg(DstLo)
5838 .addImm(AMDGPU::sub0)
5839 .addReg(DstHi)
5840 .addImm(AMDGPU::sub1);
5841 MI.eraseFromParent();
5842}
5843
5845 switch (Opc) {
5846 case AMDGPU::S_MIN_U32:
5847 return std::numeric_limits<uint32_t>::max();
5848 case AMDGPU::S_MIN_I32:
5849 return std::numeric_limits<int32_t>::max();
5850 case AMDGPU::S_MAX_U32:
5851 return std::numeric_limits<uint32_t>::min();
5852 case AMDGPU::S_MAX_I32:
5853 return std::numeric_limits<int32_t>::min();
5854 case AMDGPU::V_ADD_F32_e64: // -0.0
5855 return 0x80000000;
5856 case AMDGPU::V_SUB_F32_e64: // +0.0
5857 return 0x0;
5858 case AMDGPU::S_ADD_I32:
5859 case AMDGPU::S_SUB_I32:
5860 case AMDGPU::S_OR_B32:
5861 case AMDGPU::S_XOR_B32:
5862 return std::numeric_limits<uint32_t>::min();
5863 case AMDGPU::S_AND_B32:
5864 return std::numeric_limits<uint32_t>::max();
5865 case AMDGPU::V_MIN_F32_e64:
5866 case AMDGPU::V_MAX_F32_e64:
5867 return 0x7fc00000; // qNAN
5868 case AMDGPU::V_CMP_LT_U64_e64: // umin.u64
5869 return std::numeric_limits<uint64_t>::max();
5870 case AMDGPU::V_CMP_LT_I64_e64: // min.i64
5871 return std::numeric_limits<int64_t>::max();
5872 case AMDGPU::V_CMP_GT_U64_e64: // umax.u64
5873 return std::numeric_limits<uint64_t>::min();
5874 case AMDGPU::V_CMP_GT_I64_e64: // max.i64
5875 return std::numeric_limits<int64_t>::min();
5876 case AMDGPU::V_MIN_F64_e64:
5877 case AMDGPU::V_MAX_F64_e64:
5878 case AMDGPU::V_MIN_NUM_F64_e64:
5879 case AMDGPU::V_MAX_NUM_F64_e64:
5880 return 0x7FF8000000000000; // qNAN
5881 case AMDGPU::S_ADD_U64_PSEUDO:
5882 case AMDGPU::S_SUB_U64_PSEUDO:
5883 case AMDGPU::S_OR_B64:
5884 case AMDGPU::S_XOR_B64:
5885 return std::numeric_limits<uint64_t>::min();
5886 case AMDGPU::S_AND_B64:
5887 return std::numeric_limits<uint64_t>::max();
5888 case AMDGPU::V_ADD_F64_e64:
5889 case AMDGPU::V_ADD_F64_pseudo_e64:
5890 return 0x8000000000000000; // -0.0
5891 default:
5892 llvm_unreachable("Unexpected opcode in getIdentityValueForWaveReduction");
5893 }
5894}
5895
5896static bool is32bitWaveReduceOperation(unsigned Opc) {
5897 return Opc == AMDGPU::S_MIN_U32 || Opc == AMDGPU::S_MIN_I32 ||
5898 Opc == AMDGPU::S_MAX_U32 || Opc == AMDGPU::S_MAX_I32 ||
5899 Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32 ||
5900 Opc == AMDGPU::S_AND_B32 || Opc == AMDGPU::S_OR_B32 ||
5901 Opc == AMDGPU::S_XOR_B32 || Opc == AMDGPU::V_MIN_F32_e64 ||
5902 Opc == AMDGPU::V_MAX_F32_e64 || Opc == AMDGPU::V_ADD_F32_e64 ||
5903 Opc == AMDGPU::V_SUB_F32_e64;
5904}
5905
5907 return Opc == AMDGPU::V_MIN_F32_e64 || Opc == AMDGPU::V_MAX_F32_e64 ||
5908 Opc == AMDGPU::V_ADD_F32_e64 || Opc == AMDGPU::V_SUB_F32_e64 ||
5909 Opc == AMDGPU::V_MIN_F64_e64 || Opc == AMDGPU::V_MAX_F64_e64 ||
5910 Opc == AMDGPU::V_MIN_NUM_F64_e64 || Opc == AMDGPU::V_MAX_NUM_F64_e64 ||
5911 Opc == AMDGPU::V_ADD_F64_e64 || Opc == AMDGPU::V_ADD_F64_pseudo_e64;
5912}
5913
5914static std::tuple<unsigned, unsigned>
5916 unsigned DPPOpc;
5917 switch (Opc) {
5918 case AMDGPU::S_MIN_U32:
5919 DPPOpc = AMDGPU::V_MIN_U32_dpp;
5920 break;
5921 case AMDGPU::S_MIN_I32:
5922 DPPOpc = AMDGPU::V_MIN_I32_dpp;
5923 break;
5924 case AMDGPU::S_MAX_U32:
5925 DPPOpc = AMDGPU::V_MAX_U32_dpp;
5926 break;
5927 case AMDGPU::S_MAX_I32:
5928 DPPOpc = AMDGPU::V_MAX_I32_dpp;
5929 break;
5930 case AMDGPU::S_ADD_I32:
5931 case AMDGPU::S_SUB_I32:
5932 DPPOpc = ST.hasAddNoCarryInsts() ? AMDGPU::V_ADD_U32_dpp
5933 : AMDGPU::V_ADD_CO_U32_dpp;
5934 break;
5935 case AMDGPU::S_AND_B32:
5936 DPPOpc = AMDGPU::V_AND_B32_dpp;
5937 break;
5938 case AMDGPU::S_OR_B32:
5939 DPPOpc = AMDGPU::V_OR_B32_dpp;
5940 break;
5941 case AMDGPU::S_XOR_B32:
5942 DPPOpc = AMDGPU::V_XOR_B32_dpp;
5943 break;
5944 case AMDGPU::V_ADD_F32_e64:
5945 case AMDGPU::V_SUB_F32_e64:
5946 DPPOpc = AMDGPU::V_ADD_F32_dpp;
5947 break;
5948 case AMDGPU::V_MIN_F32_e64:
5949 DPPOpc = AMDGPU::V_MIN_F32_dpp;
5950 break;
5951 case AMDGPU::V_MAX_F32_e64:
5952 DPPOpc = AMDGPU::V_MAX_F32_dpp;
5953 break;
5954 case AMDGPU::V_CMP_LT_U64_e64: // umin.u64
5955 case AMDGPU::V_CMP_LT_I64_e64: // min.i64
5956 case AMDGPU::V_CMP_GT_U64_e64: // umax.u64
5957 case AMDGPU::V_CMP_GT_I64_e64: // max.i64
5958 case AMDGPU::S_ADD_U64_PSEUDO:
5959 case AMDGPU::S_SUB_U64_PSEUDO:
5960 case AMDGPU::S_AND_B64:
5961 case AMDGPU::S_OR_B64:
5962 case AMDGPU::S_XOR_B64:
5963 case AMDGPU::V_MIN_NUM_F64_e64:
5964 case AMDGPU::V_MIN_F64_e64:
5965 case AMDGPU::V_MAX_NUM_F64_e64:
5966 case AMDGPU::V_MAX_F64_e64:
5967 case AMDGPU::V_ADD_F64_pseudo_e64:
5968 case AMDGPU::V_ADD_F64_e64:
5969 DPPOpc = AMDGPU::V_MOV_B64_DPP_PSEUDO;
5970 break;
5971 default:
5972 llvm_unreachable("unhandled lane op");
5973 }
5974 unsigned ClampOpc = Opc;
5975 if (Opc == AMDGPU::V_SUB_F32_e64)
5976 ClampOpc = AMDGPU::V_ADD_F32_e64;
5977 if (!ST.getInstrInfo()->isVALU(Opc, /*AllowLDSDMA=*/false)) {
5978 if (Opc == AMDGPU::S_SUB_I32)
5979 ClampOpc = AMDGPU::S_ADD_I32;
5980 if (Opc == AMDGPU::S_ADD_U64_PSEUDO || Opc == AMDGPU::S_SUB_U64_PSEUDO)
5981 ClampOpc = AMDGPU::V_ADD_CO_U32_e64;
5982 else if (Opc == AMDGPU::S_AND_B64)
5983 ClampOpc = AMDGPU::V_AND_B32_e64;
5984 else if (Opc == AMDGPU::S_OR_B64)
5985 ClampOpc = AMDGPU::V_OR_B32_e64;
5986 else if (Opc == AMDGPU::S_XOR_B64)
5987 ClampOpc = AMDGPU::V_XOR_B32_e64;
5988 else
5989 ClampOpc = ST.getInstrInfo()->getVALUOp(ClampOpc);
5990 }
5991 return {DPPOpc, ClampOpc};
5992}
5993
5994static std::pair<Register, Register>
5996 const TargetRegisterClass *SrcRC, const GCNSubtarget &ST,
5998 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5999 const SIInstrInfo *TII = ST.getInstrInfo();
6000 const TargetRegisterClass *SrcSubRC =
6001 TRI->getSubRegisterClass(SrcRC, AMDGPU::sub0);
6002 Register Op1L =
6003 TII->buildExtractSubReg(MI, MRI, Op, SrcRC, AMDGPU::sub0, SrcSubRC);
6004 Register Op1H =
6005 TII->buildExtractSubReg(MI, MRI, Op, SrcRC, AMDGPU::sub1, SrcSubRC);
6006 return {Op1L, Op1H};
6007}
6008
6011 const GCNSubtarget &ST,
6012 unsigned Opc) {
6014 const SIRegisterInfo *TRI = ST.getRegisterInfo();
6015 const DebugLoc &DL = MI.getDebugLoc();
6016 const SIInstrInfo *TII = ST.getInstrInfo();
6017
6018 // Reduction operations depend on whether the input operand is SGPR or VGPR.
6019 Register SrcReg = MI.getOperand(1).getReg();
6020 bool isSGPR = TRI->isSGPRClass(MRI.getRegClass(SrcReg));
6021 Register DstReg = MI.getOperand(0).getReg();
6022 unsigned Stratergy = static_cast<unsigned>(MI.getOperand(2).getImm());
6023 enum WAVE_REDUCE_STRATEGY : unsigned { DEFAULT = 0, ITERATIVE = 1, DPP = 2 };
6024 MachineBasicBlock *RetBB = nullptr;
6025 unsigned MIOpc = MI.getOpcode();
6026 auto BuildRegSequence = [&](MachineBasicBlock &BB,
6028 Register Src0, Register Src1) {
6029 auto RegSequence =
6030 BuildMI(BB, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), Dst)
6031 .addReg(Src0)
6032 .addImm(AMDGPU::sub0)
6033 .addReg(Src1)
6034 .addImm(AMDGPU::sub1);
6035 return RegSequence;
6036 };
6037 if (isSGPR) {
6038 switch (Opc) {
6039 case AMDGPU::S_MIN_U32:
6040 case AMDGPU::S_MIN_I32:
6041 case AMDGPU::V_MIN_F32_e64:
6042 case AMDGPU::S_MAX_U32:
6043 case AMDGPU::S_MAX_I32:
6044 case AMDGPU::V_MAX_F32_e64:
6045 case AMDGPU::S_AND_B32:
6046 case AMDGPU::S_OR_B32: {
6047 // Idempotent operations.
6048 BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MOV_B32), DstReg).addReg(SrcReg);
6049 RetBB = &BB;
6050 break;
6051 }
6052 case AMDGPU::V_CMP_LT_U64_e64: // umin
6053 case AMDGPU::V_CMP_LT_I64_e64: // min
6054 case AMDGPU::V_CMP_GT_U64_e64: // umax
6055 case AMDGPU::V_CMP_GT_I64_e64: // max
6056 case AMDGPU::V_MIN_F64_e64:
6057 case AMDGPU::V_MIN_NUM_F64_e64:
6058 case AMDGPU::V_MAX_F64_e64:
6059 case AMDGPU::V_MAX_NUM_F64_e64:
6060 case AMDGPU::S_AND_B64:
6061 case AMDGPU::S_OR_B64: {
6062 // Idempotent operations.
6063 BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MOV_B64), DstReg).addReg(SrcReg);
6064 RetBB = &BB;
6065 break;
6066 }
6067 case AMDGPU::S_XOR_B32:
6068 case AMDGPU::S_XOR_B64:
6069 case AMDGPU::S_ADD_I32:
6070 case AMDGPU::S_ADD_U64_PSEUDO:
6071 case AMDGPU::V_ADD_F32_e64:
6072 case AMDGPU::V_ADD_F64_e64:
6073 case AMDGPU::V_ADD_F64_pseudo_e64:
6074 case AMDGPU::S_SUB_I32:
6075 case AMDGPU::S_SUB_U64_PSEUDO:
6076 case AMDGPU::V_SUB_F32_e64: {
6077 const TargetRegisterClass *WaveMaskRegClass = TRI->getWaveMaskRegClass();
6078 const TargetRegisterClass *DstRegClass = MRI.getRegClass(DstReg);
6079 Register ExecMask = MRI.createVirtualRegister(WaveMaskRegClass);
6080 Register NumActiveLanes =
6081 MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6082
6083 bool IsWave32 = ST.isWave32();
6084 unsigned MovOpc = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
6085 MCRegister ExecReg = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
6086 unsigned BitCountOpc =
6087 IsWave32 ? AMDGPU::S_BCNT1_I32_B32 : AMDGPU::S_BCNT1_I32_B64;
6088
6089 BuildMI(BB, MI, DL, TII->get(MovOpc), ExecMask).addReg(ExecReg);
6090
6091 auto NewAccumulator =
6092 BuildMI(BB, MI, DL, TII->get(BitCountOpc), NumActiveLanes)
6093 .addReg(ExecMask)
6094 .setOperandDead(2); // Dead scc
6095
6096 switch (Opc) {
6097 case AMDGPU::S_XOR_B32:
6098 case AMDGPU::S_XOR_B64: {
6099 // Performing an XOR operation on a uniform value
6100 // depends on the parity of the number of active lanes.
6101 // For even parity, the result will be 0, for odd
6102 // parity the result will be the same as the input value.
6103 Register ParityRegister =
6104 MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6105 BuildMI(BB, MI, DL, TII->get(AMDGPU::S_AND_B32), ParityRegister)
6106 .addReg(NewAccumulator->getOperand(0).getReg())
6107 .addImm(1)
6108 .setOperandDead(3); // Dead scc
6109 // Check if Src is a known identity constant.
6110 if (TII->getImmOrMaterializedImm(MRI, SrcReg) == 1) {
6111 // 1 * parity(exec) = parity(exec)
6112 if (Opc == AMDGPU::S_XOR_B32) {
6113 BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MOV_B32), DstReg)
6114 .addReg(ParityRegister);
6115 } else {
6116 Register DstHi =
6117 MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6118 BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MOV_B32), DstHi).addImm(0);
6119 BuildRegSequence(BB, MI, DstReg, ParityRegister, DstHi);
6120 }
6121 break;
6122 }
6123 if (Opc == AMDGPU::S_XOR_B32) {
6124 BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DstReg)
6125 .addReg(SrcReg)
6126 .addReg(ParityRegister);
6127 } else {
6128 Register DestSub0 =
6129 MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6130 Register DestSub1 =
6131 MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6132 auto [Op1L, Op1H] = ExtractSubRegs(MI, MI.getOperand(1),
6133 MRI.getRegClass(SrcReg), ST, MRI);
6134 BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DestSub0)
6135 .addReg(Op1L)
6136 .addReg(ParityRegister);
6137 BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DestSub1)
6138 .addReg(Op1H)
6139 .addReg(ParityRegister);
6140 BuildRegSequence(BB, MI, DstReg, DestSub0, DestSub1);
6141 }
6142 break;
6143 }
6144 case AMDGPU::S_SUB_I32: {
6145 Register NegatedVal = MRI.createVirtualRegister(DstRegClass);
6146 // Take the negation of the source operand.
6147 BuildMI(BB, MI, DL, TII->get(AMDGPU::S_SUB_I32), NegatedVal)
6148 .addImm(0)
6149 .addReg(SrcReg)
6150 .setOperandDead(3); // Dead scc
6151 BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DstReg)
6152 .addReg(NegatedVal)
6153 .addReg(NewAccumulator->getOperand(0).getReg());
6154 break;
6155 }
6156 case AMDGPU::S_ADD_I32: {
6157 // Check if Src is a known identity constant.
6158 if (TII->getImmOrMaterializedImm(MRI, SrcReg) == 1) {
6159 // 1 * bitcount(exec) = bitcount(exec)
6160 BuildMI(BB, MI, DL, TII->get(AMDGPU::COPY), DstReg)
6161 .addReg(NewAccumulator->getOperand(0).getReg());
6162 break;
6163 }
6164 BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DstReg)
6165 .addReg(SrcReg)
6166 .addReg(NewAccumulator->getOperand(0).getReg());
6167 break;
6168 }
6169 case AMDGPU::S_ADD_U64_PSEUDO:
6170 case AMDGPU::S_SUB_U64_PSEUDO: {
6171 Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6172 Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6173 Register Op1H_Op0L_Reg =
6174 MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6175 Register Op1L_Op0H_Reg =
6176 MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6177 Register CarryReg =
6178 MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
6179 Register AddReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6180 Register NegatedValLo =
6181 MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6182 Register NegatedValHi =
6183 MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6184 auto [Op1L, Op1H] = ExtractSubRegs(MI, MI.getOperand(1),
6185 MRI.getRegClass(SrcReg), ST, MRI);
6186 // Check if Src is a known identity constant.
6187 if (Opc == AMDGPU::S_ADD_U64_PSEUDO &&
6188 TII->getImmOrMaterializedImm(MRI, SrcReg) == 1) {
6189 // 1 * bitcount(exec) = bitcount(exec)
6190 Register DstHi = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6191 BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MOV_B32), DstHi).addImm(0);
6192 BuildRegSequence(BB, MI, DstReg,
6193 NewAccumulator->getOperand(0).getReg(), DstHi);
6194 break;
6195 }
6196 if (Opc == AMDGPU::S_SUB_U64_PSEUDO) {
6197 BuildMI(BB, MI, DL, TII->get(AMDGPU::S_SUB_I32), NegatedValLo)
6198 .addImm(0)
6199 .addReg(NewAccumulator->getOperand(0).getReg())
6200 .setOperandDead(3); // Dead scc
6201 BuildMI(BB, MI, DL, TII->get(AMDGPU::S_ASHR_I32), NegatedValHi)
6202 .addReg(NegatedValLo)
6203 .addImm(31)
6204 .setOperandDead(3); // Dead scc
6205 BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), Op1L_Op0H_Reg)
6206 .addReg(Op1L)
6207 .addReg(NegatedValHi);
6208 }
6209 Register LowOpcode = Opc == AMDGPU::S_SUB_U64_PSEUDO
6210 ? NegatedValLo
6211 : NewAccumulator->getOperand(0).getReg();
6212 BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DestSub0)
6213 .addReg(Op1L)
6214 .addReg(LowOpcode);
6215 if (ST.hasScalarMulHiInsts()) {
6216 BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_HI_U32), CarryReg)
6217 .addReg(Op1L)
6218 .addReg(LowOpcode);
6219 } else {
6220 Register VCarryReg =
6221 MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6222 Register LowOpVGPR =
6223 MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6224 BuildMI(BB, MI, DL, TII->get(AMDGPU::COPY), LowOpVGPR)
6225 .addReg(LowOpcode);
6226 BuildMI(BB, MI, DL, TII->get(AMDGPU::V_MUL_HI_U32_e64), VCarryReg)
6227 .addReg(Op1L)
6228 .addReg(LowOpVGPR);
6229 BuildMI(BB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), CarryReg)
6230 .addReg(VCarryReg);
6231 }
6232 BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), Op1H_Op0L_Reg)
6233 .addReg(Op1H)
6234 .addReg(LowOpcode);
6235
6236 Register HiVal = Opc == AMDGPU::S_SUB_U64_PSEUDO ? AddReg : DestSub1;
6237 BuildMI(BB, MI, DL, TII->get(AMDGPU::S_ADD_U32), HiVal)
6238 .addReg(CarryReg)
6239 .addReg(Op1H_Op0L_Reg)
6240 .setOperandDead(3); // Dead scc
6241
6242 if (Opc == AMDGPU::S_SUB_U64_PSEUDO) {
6243 BuildMI(BB, MI, DL, TII->get(AMDGPU::S_ADD_U32), DestSub1)
6244 .addReg(HiVal)
6245 .addReg(Op1L_Op0H_Reg)
6246 .setOperandDead(3); // Dead scc
6247 }
6248 BuildRegSequence(BB, MI, DstReg, DestSub0, DestSub1);
6249 break;
6250 }
6251 case AMDGPU::V_ADD_F32_e64:
6252 case AMDGPU::V_ADD_F64_e64:
6253 case AMDGPU::V_ADD_F64_pseudo_e64:
6254 case AMDGPU::V_SUB_F32_e64: {
6255 bool is32BitOpc = is32bitWaveReduceOperation(Opc);
6256 const TargetRegisterClass *VregRC = TII->getRegClass(TII->get(Opc), 0);
6257 Register ActiveLanesVreg = MRI.createVirtualRegister(VregRC);
6258 Register DstVreg = MRI.createVirtualRegister(VregRC);
6259 // Get number of active lanes as a float val.
6260 BuildMI(BB, MI, DL,
6261 TII->get(is32BitOpc ? AMDGPU::V_CVT_F32_I32_e64
6262 : AMDGPU::V_CVT_F64_I32_e64),
6263 ActiveLanesVreg)
6264 .addReg(NewAccumulator->getOperand(0).getReg())
6265 .addImm(0) // clamp
6266 .addImm(0); // output-modifier
6267
6268 // Take negation of input for SUB reduction
6269 unsigned srcMod = (MIOpc == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F32 ||
6270 MIOpc == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64)
6273 unsigned MulOpc = is32BitOpc ? AMDGPU::V_MUL_F32_e64
6274 : ST.getGeneration() >= AMDGPUSubtarget::GFX12
6275 ? AMDGPU::V_MUL_F64_pseudo_e64
6276 : AMDGPU::V_MUL_F64_e64;
6277 auto DestVregInst = BuildMI(BB, MI, DL, TII->get(MulOpc),
6278 DstVreg)
6279 .addImm(srcMod) // src0 modifier
6280 .addReg(SrcReg)
6281 .addImm(SISrcMods::NONE) // src1 modifier
6282 .addReg(ActiveLanesVreg)
6283 .addImm(SISrcMods::NONE) // clamp
6284 .addImm(SISrcMods::NONE); // output-mod
6285 if (is32BitOpc) {
6286 BuildMI(BB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), DstReg)
6287 .addReg(DstVreg);
6288 } else {
6289 Register LaneValueLoReg =
6290 MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
6291 Register LaneValueHiReg =
6292 MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
6293 auto [Op1L, Op1H] =
6294 ExtractSubRegs(MI, DestVregInst->getOperand(0), VregRC, ST, MRI);
6295 // lane value input should be in an sgpr
6296 BuildMI(BB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32),
6297 LaneValueLoReg)
6298 .addReg(Op1L);
6299 BuildMI(BB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32),
6300 LaneValueHiReg)
6301 .addReg(Op1H);
6302 NewAccumulator =
6303 BuildRegSequence(BB, MI, DstReg, LaneValueLoReg, LaneValueHiReg);
6304 }
6305 }
6306 }
6307 RetBB = &BB;
6308 }
6309 }
6310 } else {
6312 Register SrcReg = MI.getOperand(1).getReg();
6313 bool is32BitOpc = is32bitWaveReduceOperation(Opc);
6315 bool NeedsMovDPP = !is32BitOpc;
6316 // Create virtual registers required for lowering.
6317 const TargetRegisterClass *WaveMaskRegClass = TRI->getWaveMaskRegClass();
6318 const TargetRegisterClass *DstRegClass = MRI.getRegClass(DstReg);
6319 const TargetRegisterClass *SrcRegClass = MRI.getRegClass(SrcReg);
6320 bool IsWave32 = ST.isWave32();
6321 unsigned MovOpcForExec = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
6322 unsigned ExecReg = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
6323 if (Stratergy == WAVE_REDUCE_STRATEGY::ITERATIVE ||
6324 !ST.hasDPP()) { // If target doesn't support DPP operations, default to
6325 // iterative stratergy
6326
6327 // To reduce the VGPR using iterative approach, we need to iterate
6328 // over all the active lanes. Lowering consists of ComputeLoop,
6329 // which iterate over only active lanes. We use copy of EXEC register
6330 // as induction variable and every active lane modifies it using bitset0
6331 // so that we will get the next active lane for next iteration.
6332
6333 // Create Control flow for loop
6334 // Split MI's Machine Basic block into For loop
6335 auto [ComputeLoop, ComputeEnd] = splitBlockForLoop(MI, BB, true);
6336
6337 Register LoopIterator = MRI.createVirtualRegister(WaveMaskRegClass);
6338 Register IdentityValReg = MRI.createVirtualRegister(DstRegClass);
6339 Register AccumulatorReg = MRI.createVirtualRegister(DstRegClass);
6340 Register ActiveBitsReg = MRI.createVirtualRegister(WaveMaskRegClass);
6341 Register NewActiveBitsReg = MRI.createVirtualRegister(WaveMaskRegClass);
6342 Register FF1Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6343 Register LaneValueReg = MRI.createVirtualRegister(DstRegClass);
6344
6345 // Create initial values of induction variable from Exec, Accumulator and
6346 // insert branch instr to newly created ComputeBlock
6347 BuildMI(BB, I, DL, TII->get(MovOpcForExec), LoopIterator).addReg(ExecReg);
6348 uint64_t IdentityValue =
6349 MI.getOpcode() == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64
6350 ? 0x0 // +0.0 for double sub reduction
6352 BuildMI(BB, I, DL,
6353 TII->get(is32BitOpc ? AMDGPU::S_MOV_B32
6354 : AMDGPU::S_MOV_B64_IMM_PSEUDO),
6355 IdentityValReg)
6356 .addImm(IdentityValue);
6357 // clang-format off
6358 BuildMI(BB, I, DL, TII->get(AMDGPU::S_BRANCH))
6359 .addMBB(ComputeLoop);
6360 // clang-format on
6361
6362 // Start constructing ComputeLoop
6363 I = ComputeLoop->begin();
6364 auto Accumulator =
6365 BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::PHI), AccumulatorReg)
6366 .addReg(IdentityValReg)
6367 .addMBB(&BB);
6368 auto ActiveBits =
6369 BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::PHI), ActiveBitsReg)
6370 .addReg(LoopIterator)
6371 .addMBB(&BB);
6372
6373 I = ComputeLoop->end();
6374 MachineInstr *NewAccumulator;
6375 // Perform the computations
6376 unsigned SFFOpc =
6377 IsWave32 ? AMDGPU::S_FF1_I32_B32 : AMDGPU::S_FF1_I32_B64;
6378 BuildMI(*ComputeLoop, I, DL, TII->get(SFFOpc), FF1Reg)
6379 .addReg(ActiveBitsReg);
6380 if (is32BitOpc) {
6381 Register OpDstReg = DstReg;
6382 bool hasSrc0Modifier = AMDGPU::getNamedOperandIdx(
6383 Opc, AMDGPU::OpName::src0_modifiers) != -1;
6384 bool hasSrc1Modifier = AMDGPU::getNamedOperandIdx(
6385 Opc, AMDGPU::OpName::src1_modifiers) != -1;
6386 bool hasClamp =
6387 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp) != -1;
6388 bool hasOpSel =
6389 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel) != -1;
6390 bool hasOMod =
6391 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod) != -1;
6392 BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::V_READLANE_B32),
6393 LaneValueReg)
6394 .addReg(SrcReg)
6395 .addReg(FF1Reg);
6396 if (ST.getInstrInfo()->isVALU(Opc, /*AllowLDSDMA=*/false)) {
6397 // Get the Lane Value in VGPR to avoid the Constant Bus Restriction
6398 Register LaneValVgpr = MRI.createVirtualRegister(SrcRegClass);
6399 Register VgprResultReg = MRI.createVirtualRegister(SrcRegClass);
6400 BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::COPY), LaneValVgpr)
6401 .addReg(LaneValueReg);
6402 OpDstReg = VgprResultReg;
6403 LaneValueReg = LaneValVgpr;
6404 }
6405 auto OpInstr = BuildMI(*ComputeLoop, I, DL, TII->get(Opc), OpDstReg);
6406 if (hasSrc0Modifier)
6407 OpInstr.addImm(SISrcMods::NONE); // src0 modifier
6408 OpInstr.addReg(AccumulatorReg); // src0
6409 if (hasSrc1Modifier)
6410 OpInstr.addImm(SISrcMods::NONE); // src1 modifier
6411 OpInstr.addReg(LaneValueReg); // src1
6412 if (hasClamp)
6413 OpInstr.addImm(0); // clamp
6414 if (hasOpSel)
6415 OpInstr.addImm(0); // opsel
6416 if (hasOMod)
6417 OpInstr.addImm(0); // omod
6418 if (TII->isSALU(Opc))
6419 OpInstr.setOperandDead(3); // Dead scc
6420 if (ST.getInstrInfo()->isVALU(Opc, /*AllowLDSDMA=*/false)) {
6421 BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32),
6422 DstReg)
6423 .addReg(OpDstReg);
6424 }
6425 } else {
6426 Register LaneValueLoReg =
6427 MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
6428 Register LaneValueHiReg =
6429 MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
6430 Register LaneValReg =
6431 MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
6432 auto [Op1L, Op1H] = ExtractSubRegs(MI, MI.getOperand(1),
6433 MRI.getRegClass(SrcReg), ST, MRI);
6434 // lane value input should be in an sgpr
6435 BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::V_READLANE_B32),
6436 LaneValueLoReg)
6437 .addReg(Op1L)
6438 .addReg(FF1Reg);
6439 BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::V_READLANE_B32),
6440 LaneValueHiReg)
6441 .addReg(Op1H)
6442 .addReg(FF1Reg);
6443 auto LaneValue = BuildRegSequence(*ComputeLoop, I, LaneValReg,
6444 LaneValueLoReg, LaneValueHiReg);
6445 switch (Opc) {
6446 case AMDGPU::S_OR_B64:
6447 case AMDGPU::S_AND_B64:
6448 case AMDGPU::S_XOR_B64: {
6449 NewAccumulator = BuildMI(*ComputeLoop, I, DL, TII->get(Opc), DstReg)
6450 .addReg(Accumulator->getOperand(0).getReg())
6451 .addReg(LaneValue->getOperand(0).getReg())
6452 .setOperandDead(3); // Dead scc
6453 break;
6454 }
6455 case AMDGPU::V_CMP_GT_I64_e64:
6456 case AMDGPU::V_CMP_GT_U64_e64:
6457 case AMDGPU::V_CMP_LT_I64_e64:
6458 case AMDGPU::V_CMP_LT_U64_e64: {
6459 Register LaneMaskReg = MRI.createVirtualRegister(WaveMaskRegClass);
6460 Register ComparisonResultReg =
6461 MRI.createVirtualRegister(WaveMaskRegClass);
6462 int SrcIdx =
6463 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src);
6464 const TargetRegisterClass *VregClass =
6465 TRI->getAllocatableClass(TII->getRegClass(MI.getDesc(), SrcIdx));
6466 Register AccumulatorVReg = MRI.createVirtualRegister(VregClass);
6467 auto [SrcReg0Sub0, SrcReg0Sub1] = ExtractSubRegs(
6468 MI, Accumulator->getOperand(0), VregClass, ST, MRI);
6469 BuildRegSequence(*ComputeLoop, I, AccumulatorVReg, SrcReg0Sub0,
6470 SrcReg0Sub1);
6471 BuildMI(*ComputeLoop, I, DL, TII->get(Opc), LaneMaskReg)
6472 .addReg(LaneValue->getOperand(0).getReg())
6473 .addReg(AccumulatorVReg);
6474
6475 unsigned AndOpc = IsWave32 ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64;
6476 BuildMI(*ComputeLoop, I, DL, TII->get(AndOpc), ComparisonResultReg)
6477 .addReg(LaneMaskReg)
6478 .addReg(ActiveBitsReg);
6479
6480 NewAccumulator = BuildMI(*ComputeLoop, I, DL,
6481 TII->get(AMDGPU::S_CSELECT_B64), DstReg)
6482 .addReg(LaneValue->getOperand(0).getReg())
6483 .addReg(Accumulator->getOperand(0).getReg());
6484 break;
6485 }
6486 case AMDGPU::V_MIN_F64_e64:
6487 case AMDGPU::V_MIN_NUM_F64_e64:
6488 case AMDGPU::V_MAX_F64_e64:
6489 case AMDGPU::V_MAX_NUM_F64_e64:
6490 case AMDGPU::V_ADD_F64_e64:
6491 case AMDGPU::V_ADD_F64_pseudo_e64: {
6492 int SrcIdx =
6493 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src);
6494 const TargetRegisterClass *VregRC =
6495 TRI->getAllocatableClass(TII->getRegClass(MI.getDesc(), SrcIdx));
6496 Register AccumulatorVReg = MRI.createVirtualRegister(VregRC);
6497 Register DstVreg = MRI.createVirtualRegister(VregRC);
6498 Register LaneValLo =
6499 MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
6500 Register LaneValHi =
6501 MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
6502 BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::COPY), AccumulatorVReg)
6503 .addReg(Accumulator->getOperand(0).getReg());
6504 unsigned Modifier =
6505 MI.getOpcode() == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64
6508 auto DstVregInst =
6509 BuildMI(*ComputeLoop, I, DL, TII->get(Opc), DstVreg)
6510 .addImm(Modifier) // src0 modifiers
6511 .addReg(LaneValue->getOperand(0).getReg())
6512 .addImm(SISrcMods::NONE) // src1 modifiers
6513 .addReg(AccumulatorVReg)
6514 .addImm(SISrcMods::NONE) // clamp
6515 .addImm(SISrcMods::NONE); // omod
6516 auto ReadLaneLo =
6517 BuildMI(*ComputeLoop, I, DL,
6518 TII->get(AMDGPU::V_READFIRSTLANE_B32), LaneValLo);
6519 auto ReadLaneHi =
6520 BuildMI(*ComputeLoop, I, DL,
6521 TII->get(AMDGPU::V_READFIRSTLANE_B32), LaneValHi);
6522 MachineBasicBlock::iterator Iters = *ReadLaneLo;
6523 auto [Op1L, Op1H] = ExtractSubRegs(*Iters, DstVregInst->getOperand(0),
6524 VregRC, ST, MRI);
6525 ReadLaneLo.addReg(Op1L);
6526 ReadLaneHi.addReg(Op1H);
6527 NewAccumulator =
6528 BuildRegSequence(*ComputeLoop, I, DstReg, LaneValLo, LaneValHi);
6529 break;
6530 }
6531 case AMDGPU::S_ADD_U64_PSEUDO:
6532 case AMDGPU::S_SUB_U64_PSEUDO: {
6533 NewAccumulator = BuildMI(*ComputeLoop, I, DL, TII->get(Opc), DstReg)
6534 .addReg(Accumulator->getOperand(0).getReg())
6535 .addReg(LaneValue->getOperand(0).getReg())
6536 .setOperandDead(3); // Dead scc
6537 ComputeLoop =
6538 expand64BitScalarArithmetic(*NewAccumulator, ComputeLoop);
6539 break;
6540 }
6541 }
6542 }
6543 // Manipulate the iterator to get the next active lane
6544 unsigned BITSETOpc =
6545 IsWave32 ? AMDGPU::S_BITSET0_B32 : AMDGPU::S_BITSET0_B64;
6546 BuildMI(*ComputeLoop, I, DL, TII->get(BITSETOpc), NewActiveBitsReg)
6547 .addReg(FF1Reg)
6548 .addReg(ActiveBitsReg);
6549
6550 // Add phi nodes
6551 Accumulator.addReg(DstReg).addMBB(ComputeLoop);
6552 ActiveBits.addReg(NewActiveBitsReg).addMBB(ComputeLoop);
6553
6554 // Creating branching
6555 MachineInstrBuilder SetSCCInstr;
6556 if (!ST.hasScalarCompareEq64()) {
6557 // For targets <= gfx7, use an S_OR_B32/B64 instruction to set SCC.
6558 Register LaneMaskReg = MRI.createVirtualRegister(WaveMaskRegClass);
6559 unsigned CMPOpc = IsWave32 ? AMDGPU::S_OR_B32 : AMDGPU::S_OR_B64;
6560 SetSCCInstr =
6561 BuildMI(*ComputeLoop, I, DL, TII->get(CMPOpc), LaneMaskReg);
6562 } else {
6563 unsigned CMPOpc =
6564 IsWave32 ? AMDGPU::S_CMP_LG_U32 : AMDGPU::S_CMP_LG_U64;
6565 SetSCCInstr = BuildMI(*ComputeLoop, I, DL, TII->get(CMPOpc));
6566 }
6567 SetSCCInstr.addReg(NewActiveBitsReg);
6568 if (ST.hasScalarCompareEq64())
6569 SetSCCInstr.addImm(0);
6570 else
6571 SetSCCInstr.addReg(NewActiveBitsReg);
6572 BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::S_CBRANCH_SCC1))
6573 .addMBB(ComputeLoop);
6574
6575 RetBB = ComputeEnd;
6576 } else {
6577 assert(ST.hasDPP() && "Sub Target does not support DPP Operations");
6578 MachineBasicBlock *CurrBB = &BB;
6579 Register SrcWithIdentity = MRI.createVirtualRegister(SrcRegClass);
6580 Register IdentityVGPR = MRI.createVirtualRegister(SrcRegClass);
6581 Register IdentitySGPR = MRI.createVirtualRegister(DstRegClass);
6582 Register DPPRowShr1 = MRI.createVirtualRegister(SrcRegClass);
6583 Register DPPRowShr2 = MRI.createVirtualRegister(SrcRegClass);
6584 Register DPPRowShr4 = MRI.createVirtualRegister(SrcRegClass);
6585 Register DPPRowShr8 = MRI.createVirtualRegister(SrcRegClass);
6586 Register RowBcast15 = MRI.createVirtualRegister(SrcRegClass);
6587 Register ReducedValSGPR = MRI.createVirtualRegister(DstRegClass);
6588 Register NegatedReducedVal = MRI.createVirtualRegister(DstRegClass);
6589 Register RowBcast31 = MRI.createVirtualRegister(SrcRegClass);
6590 Register UndefExec = MRI.createVirtualRegister(WaveMaskRegClass);
6591 Register FinalDPPResult;
6592 MachineInstr *SrcWithIdentityInstr;
6593 MachineInstr *LastBcastInstr;
6594 BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::IMPLICIT_DEF), UndefExec);
6595
6597 BuildMI(*CurrBB, MI, DL,
6598 TII->get(is32BitOpc ? AMDGPU::S_MOV_B32
6599 : AMDGPU::S_MOV_B64_IMM_PSEUDO),
6600 IdentitySGPR)
6601 .addImm(IdentityValue);
6602 auto IdentityCopyInstr =
6603 BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::COPY), IdentityVGPR)
6604 .addReg(IdentitySGPR);
6605 unsigned DPPOpc, ClampOpc;
6606 std::tie(DPPOpc, ClampOpc) = getDPPOpcForWaveReduction(Opc, ST);
6607 auto BuildSetInactiveInstr = [&](Register Dst, Register Src0,
6608 Register Src1) {
6609 return BuildMI(BB, MI, DL, TII->get(AMDGPU::V_SET_INACTIVE_B32),
6610 Dst)
6611 .addImm(0) // src0 modifiers
6612 .addReg(Src0) // src0
6613 .addImm(0) // src1 modifiers
6614 .addReg(Src1) // identity value for inactive lanes
6615 .addReg(UndefExec); // bool i1
6616 };
6617 auto BuildDPPMachineInstr = [&](Register Dst, Register Src,
6618 unsigned DPPCtrl) {
6619 auto DPPInstr =
6620 BuildMI(*CurrBB, MI, DL, TII->get(DPPOpc), Dst).addReg(Src); // old
6621 if (isFPOp && !NeedsMovDPP)
6622 DPPInstr.addImm(SISrcMods::NONE); // src0 modifier
6623 DPPInstr.addReg(Src); // src0
6624 if (isFPOp && !NeedsMovDPP)
6625 DPPInstr.addImm(SISrcMods::NONE); // src1 modifier
6626 if (!NeedsMovDPP)
6627 DPPInstr.addReg(Src); // src1
6628 if (AMDGPU::getNamedOperandIdx(DPPOpc, AMDGPU::OpName::clamp) >= 0)
6629 DPPInstr.addImm(0); // clamp
6630 DPPInstr
6631 .addImm(DPPCtrl) // dpp-ctrl
6632 .addImm(0xf) // row-mask
6633 .addImm(0xf) // bank-mask
6634 .addImm(0); // bound-control
6635 };
6636 auto BuildClampInstr = [&](Register Dst, Register Src0, Register Src1,
6637 bool isAddSub = false,
6638 bool needsCarryIn = false,
6639 Register CarryIn = Register()) {
6640 unsigned InstrOpc = ClampOpc;
6641 Register CarryOutReg = MRI.createVirtualRegister(WaveMaskRegClass);
6642 if (needsCarryIn)
6643 InstrOpc = AMDGPU::V_ADDC_U32_e64;
6644 auto ClampInstr = BuildMI(*CurrBB, MI, DL, TII->get(InstrOpc), Dst);
6645 if (isFPOp)
6646 ClampInstr.addImm(SISrcMods::NONE); // src0 mod
6647 if (isAddSub) {
6648 if (needsCarryIn)
6649 ClampInstr.addReg(CarryOutReg,
6651 RegState::Dead); // killed carry-out reg
6652 else
6653 ClampInstr.addReg(CarryOutReg, RegState::Define); // carry-out reg
6654 }
6655 ClampInstr.addReg(Src0); // src0
6656 if (isFPOp)
6657 ClampInstr.addImm(SISrcMods::NONE); // src1 mod
6658 ClampInstr.addReg(Src1); // src1
6659 if (needsCarryIn)
6660 ClampInstr.addReg(CarryIn); // carry-in reg
6661 if (AMDGPU::getNamedOperandIdx(InstrOpc, AMDGPU::OpName::clamp) >= 0)
6662 ClampInstr.addImm(0); // clamp
6663 if (isFPOp)
6664 ClampInstr.addImm(0); // omod
6665 LastBcastInstr = ClampInstr;
6666 return CarryOutReg;
6667 };
6668 auto BuildPostDPPInstr = [&](Register Src0, Register Src1) {
6669 bool isAddSubOpc =
6670 Opc == AMDGPU::S_ADD_U64_PSEUDO || Opc == AMDGPU::S_SUB_U64_PSEUDO;
6671 bool isBitWiseOpc = Opc == AMDGPU::S_AND_B64 ||
6672 Opc == AMDGPU::S_OR_B64 || Opc == AMDGPU::S_XOR_B64;
6673 Register ReturnReg = MRI.createVirtualRegister(SrcRegClass);
6674 if (isAddSubOpc || isBitWiseOpc) {
6675 Register ResLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6676 Register ResHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6677 MachineOperand Src0Operand =
6678 MachineOperand::CreateReg(Src0, /*isDef=*/false);
6679 MachineOperand Src1Operand =
6680 MachineOperand::CreateReg(Src1, /*isDef=*/false);
6681 auto [Src0Lo, Src0Hi] =
6682 ExtractSubRegs(MI, Src0Operand, SrcRegClass, ST, MRI);
6683 auto [Src1Lo, Src1Hi] =
6684 ExtractSubRegs(MI, Src1Operand, SrcRegClass, ST, MRI);
6685 Register CarryReg = BuildClampInstr(
6686 ResLo, Src0Lo, Src1Lo, isAddSubOpc, /*needsCarryIn*/ false);
6687 BuildClampInstr(ResHi, Src0Hi, Src1Hi, isAddSubOpc,
6688 /*needsCarryIn*/ isAddSubOpc, CarryReg);
6689 BuildRegSequence(*CurrBB, MI, ReturnReg, ResLo, ResHi);
6690 } else {
6691 if (isFPOp) {
6692 BuildMI(*CurrBB, MI, DL, TII->get(Opc), ReturnReg)
6693 .addImm(SISrcMods::NONE) // src0 modifiers
6694 .addReg(Src0)
6695 .addImm(SISrcMods::NONE) // src1 modifiers
6696 .addReg(Src1)
6697 .addImm(SISrcMods::NONE) // clamp
6698 .addImm(SISrcMods::NONE); // omod
6699 } else {
6700 Register CmpMaskReg = MRI.createVirtualRegister(WaveMaskRegClass);
6701 BuildMI(*CurrBB, MI, DL, TII->get(Opc), CmpMaskReg)
6702 .addReg(Src0) // src0
6703 .addReg(Src1); // src1
6704 LastBcastInstr =
6705 BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::V_CNDMASK_B64_PSEUDO),
6706 ReturnReg)
6707 .addReg(Src1) // src0
6708 .addReg(Src0) // src1
6709 .addReg(CmpMaskReg); // src2
6710 expand64BitV_CNDMASK(*LastBcastInstr, CurrBB);
6711 }
6712 }
6713 return ReturnReg;
6714 };
6715
6716 // Set inactive lanes to the identity value.
6717 if (is32BitOpc) {
6718 SrcWithIdentityInstr =
6719 BuildSetInactiveInstr(SrcWithIdentity, SrcReg, IdentityVGPR);
6720 } else {
6721 Register SrcWithIdentitylo =
6722 MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6723 Register SrcWithIdentityhi =
6724 MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6725 auto [Reg0Sub0, Reg0Sub1] = ExtractSubRegs(
6726 MI, IdentityCopyInstr->getOperand(0), SrcRegClass, ST, MRI);
6727 auto [SrcReg0Sub0, SrcReg0Sub1] =
6728 ExtractSubRegs(MI, MI.getOperand(1), SrcRegClass, ST, MRI);
6729 MachineInstr *SetInactiveLoInstr =
6730 BuildSetInactiveInstr(SrcWithIdentitylo, SrcReg0Sub0, Reg0Sub0);
6731 MachineInstr *SetInactiveHiInstr =
6732 BuildSetInactiveInstr(SrcWithIdentityhi, SrcReg0Sub1, Reg0Sub1);
6733 SrcWithIdentityInstr =
6734 BuildRegSequence(*CurrBB, MI, SrcWithIdentity,
6735 SetInactiveLoInstr->getOperand(0).getReg(),
6736 SetInactiveHiInstr->getOperand(0).getReg());
6737 }
6738 // DPP reduction
6739 Register SrcWithIdentityReg =
6740 SrcWithIdentityInstr->getOperand(0).getReg();
6741 BuildDPPMachineInstr(DPPRowShr1, SrcWithIdentityReg,
6743 if (NeedsMovDPP)
6744 DPPRowShr1 = BuildPostDPPInstr(SrcWithIdentityReg, DPPRowShr1);
6745
6746 BuildDPPMachineInstr(DPPRowShr2, DPPRowShr1,
6748 if (NeedsMovDPP)
6749 DPPRowShr2 = BuildPostDPPInstr(DPPRowShr1, DPPRowShr2);
6750
6751 BuildDPPMachineInstr(DPPRowShr4, DPPRowShr2,
6753 if (NeedsMovDPP)
6754 DPPRowShr4 = BuildPostDPPInstr(DPPRowShr2, DPPRowShr4);
6755
6756 BuildDPPMachineInstr(DPPRowShr8, DPPRowShr4,
6758 if (NeedsMovDPP)
6759 DPPRowShr8 = BuildPostDPPInstr(DPPRowShr4, DPPRowShr8);
6760
6761 if (ST.hasDPPBroadcasts()) {
6762 BuildDPPMachineInstr(RowBcast15, DPPRowShr8, AMDGPU::DPP::BCAST15);
6763 if (NeedsMovDPP)
6764 RowBcast15 = BuildPostDPPInstr(DPPRowShr8, RowBcast15);
6765 } else {
6766 // magic constant: 0x1E0
6767 // To Set BIT_MODE : bit 15 = 0
6768 // XOR mask : bit [14:10] = 0
6769 // OR mask : bit [9:5] = 15
6770 // AND mask : bit [4:0] = 0
6771 if (is32BitOpc) {
6772 Register SwizzledValue =
6773 MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6774 BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::DS_SWIZZLE_B32),
6775 SwizzledValue)
6776 .addReg(DPPRowShr8) // addr
6777 .addImm(0x1E0) // swizzle offset (i16)
6778 .addImm(0x0); // gds (i1)
6779 BuildClampInstr(RowBcast15, DPPRowShr8, SwizzledValue);
6780 } else {
6781 Register SwizzledValuelo =
6782 MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6783 Register SwizzledValuehi =
6784 MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6785 Register SwizzledValue64 = MRI.createVirtualRegister(SrcRegClass);
6786 MachineOperand DPPRowShr8Op =
6787 MachineOperand::CreateReg(DPPRowShr8, /*isDef=*/false);
6788 auto [Op1L, Op1H] =
6789 ExtractSubRegs(MI, DPPRowShr8Op, SrcRegClass, ST, MRI);
6790 BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::DS_SWIZZLE_B32),
6791 SwizzledValuelo)
6792 .addReg(Op1L) // addr
6793 .addImm(0x1E0) // swizzle offset (i16)
6794 .addImm(0x0); // gds (i1)
6795 BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::DS_SWIZZLE_B32),
6796 SwizzledValuehi)
6797 .addReg(Op1H) // addr
6798 .addImm(0x1E0) // swizzle offset (i16)
6799 .addImm(0x0); // gds (i1)
6800 BuildRegSequence(*CurrBB, MI, SwizzledValue64, SwizzledValuelo,
6801 SwizzledValuehi);
6802 if (NeedsMovDPP)
6803 RowBcast15 = BuildPostDPPInstr(DPPRowShr8, SwizzledValue64);
6804 else
6805 BuildClampInstr(RowBcast15, DPPRowShr8, SwizzledValue64);
6806 }
6807 }
6808 FinalDPPResult = RowBcast15;
6809 if (!IsWave32) {
6810 if (ST.hasDPPBroadcasts()) {
6811 BuildDPPMachineInstr(RowBcast31, RowBcast15, AMDGPU::DPP::BCAST31);
6812 if (NeedsMovDPP)
6813 RowBcast31 = BuildPostDPPInstr(RowBcast15, RowBcast31);
6814 } else {
6815 Register ShiftedThreadID =
6816 MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6817 Register PermuteByteOffset =
6818 MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6819 Register PermutedValue = MRI.createVirtualRegister(SrcRegClass);
6820 Register Lane32Offset =
6821 MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6822 Register WordSizeConst =
6823 MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6824 Register ThreadIDRegLo =
6825 MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6826 Register ThreadIDReg =
6827 MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6828 // Get the thread ID.
6829 BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::V_MBCNT_LO_U32_B32_e64),
6830 ThreadIDRegLo)
6831 .addImm(-1)
6832 .addImm(0);
6833 BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::V_MBCNT_HI_U32_B32_e64),
6834 ThreadIDReg)
6835 .addImm(-1)
6836 .addReg(ThreadIDRegLo);
6837 // shift each lane over by 32 positions, so value in 31st lane is
6838 // present in 63rd lane.
6839 BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::S_MOV_B32), Lane32Offset)
6840 .addImm(0x20);
6841 BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::V_ADD_U32_e64),
6842 ShiftedThreadID)
6843 .addReg(ThreadIDReg)
6844 .addReg(Lane32Offset)
6845 .addImm(0); // clamp
6846 // multiply by reg size.
6847 BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::S_MOV_B32), WordSizeConst)
6848 .addImm(0x4);
6849 BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::V_MUL_LO_U32_e64),
6850 PermuteByteOffset)
6851 .addReg(WordSizeConst)
6852 .addReg(ShiftedThreadID);
6853 // Permute the lanes
6854 if (is32BitOpc) {
6855 BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::DS_PERMUTE_B32),
6856 PermutedValue)
6857 .addReg(PermuteByteOffset) // addr
6858 .addReg(RowBcast15) // data
6859 .addImm(0); // offset
6860 } else {
6861 Register PermutedValuelo =
6862 MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6863 Register PermutedValuehi =
6864 MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6865 MachineOperand RowBcast15Op =
6866 MachineOperand::CreateReg(RowBcast15, /*isDef=*/false);
6867 auto [RowBcast15Lo, RowBcast15Hi] =
6868 ExtractSubRegs(MI, RowBcast15Op, SrcRegClass, ST, MRI);
6869 BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::DS_PERMUTE_B32),
6870 PermutedValuelo)
6871 .addReg(PermuteByteOffset) // addr
6872 .addReg(RowBcast15Lo) // swizzle offset (i16)
6873 .addImm(0x0); // gds (i1)
6874 BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::DS_PERMUTE_B32),
6875 PermutedValuehi)
6876 .addReg(PermuteByteOffset) // addr
6877 .addReg(RowBcast15Hi) // swizzle offset (i16)
6878 .addImm(0x0); // gds (i1)
6879 BuildRegSequence(*CurrBB, MI, PermutedValue, PermutedValuelo,
6880 PermutedValuehi);
6881 }
6882 if (NeedsMovDPP)
6883 RowBcast31 = BuildPostDPPInstr(RowBcast15, PermutedValue);
6884 else
6885 BuildClampInstr(RowBcast31, RowBcast15, PermutedValue);
6886 }
6887 FinalDPPResult = RowBcast31;
6888 }
6889 if (MIOpc == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F32 ||
6890 MIOpc == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64) {
6891 Register NegatedValVGPR = MRI.createVirtualRegister(SrcRegClass);
6892 // Opc for f32 reduction is V_SUB_F32.
6893 // For f64, there is no equivalent V_SUB_F64 opcode, so use
6894 // V_ADD_F64/V_ADD_F64_pseudo, and negate the second operand.
6895 BuildMI(*CurrBB, MI, DL, TII->get(Opc),
6896 NegatedValVGPR)
6897 .addImm(SISrcMods::NONE) // src0 mods
6898 .addReg(IdentityVGPR) // src0
6899 .addImm(is32BitOpc ? SISrcMods::NONE : SISrcMods::NEG) // src1 mods
6900 .addReg(IsWave32 ? RowBcast15 : RowBcast31) // src1
6901 .addImm(SISrcMods::NONE) // clamp
6902 .addImm(SISrcMods::NONE); // omod
6903 FinalDPPResult = NegatedValVGPR;
6904 }
6905 // The final reduced value is in the last lane.
6906 if (is32BitOpc) {
6907 BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::V_READLANE_B32),
6908 ReducedValSGPR)
6909 .addReg(FinalDPPResult)
6910 .addImm(ST.getWavefrontSize() - 1);
6911 } else {
6912 Register LaneValueLoReg =
6913 MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
6914 Register LaneValueHiReg =
6915 MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
6916 const TargetRegisterClass *SrcRC = MRI.getRegClass(SrcReg);
6917 MachineOperand FinalDPPResultOperand =
6918 MachineOperand::CreateReg(FinalDPPResult, /*isDef=*/false);
6919 auto [Op1L, Op1H] =
6920 ExtractSubRegs(MI, FinalDPPResultOperand, SrcRC, ST, MRI);
6921 // lane value input should be in an sgpr
6922 BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::V_READLANE_B32),
6923 LaneValueLoReg)
6924 .addReg(Op1L)
6925 .addImm(ST.getWavefrontSize() - 1);
6926 BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::V_READLANE_B32),
6927 LaneValueHiReg)
6928 .addReg(Op1H)
6929 .addImm(ST.getWavefrontSize() - 1);
6930 BuildRegSequence(*CurrBB, MI, ReducedValSGPR, LaneValueLoReg,
6931 LaneValueHiReg);
6932 }
6933 if (Opc == AMDGPU::S_SUB_I32) {
6934 BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::S_SUB_I32), NegatedReducedVal)
6935 .addImm(0)
6936 .addReg(ReducedValSGPR)
6937 .setOperandDead(3); // Dead scc
6938 } else if (Opc == AMDGPU::S_SUB_U64_PSEUDO) {
6939 auto NegatedValInstr =
6940 BuildMI(*CurrBB, MI, DL, TII->get(Opc), NegatedReducedVal)
6941 .addImm(0)
6942 .addReg(ReducedValSGPR)
6943 .setOperandDead(3); // Dead scc
6944 CurrBB = expand64BitScalarArithmetic(*NegatedValInstr, CurrBB);
6945 }
6946 // Mark the final result as a whole-wave-mode calculation.
6947 BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::STRICT_WWM), DstReg)
6948 .addReg(Opc == AMDGPU::S_SUB_I32 || Opc == AMDGPU::S_SUB_U64_PSEUDO
6949 ? NegatedReducedVal
6950 : ReducedValSGPR);
6951 RetBB = CurrBB;
6952 }
6953 }
6954 MI.eraseFromParent();
6955 return RetBB;
6956}
6957
6960 MachineBasicBlock *BB) const {
6961 MachineFunction *MF = BB->getParent();
6963 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
6965 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
6967 const DebugLoc &DL = MI.getDebugLoc();
6968
6969 switch (MI.getOpcode()) {
6970 case AMDGPU::WAVE_REDUCE_UMIN_PSEUDO_U32:
6971 return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_MIN_U32);
6972 case AMDGPU::WAVE_REDUCE_UMIN_PSEUDO_U64:
6973 return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::V_CMP_LT_U64_e64);
6974 case AMDGPU::WAVE_REDUCE_MIN_PSEUDO_I32:
6975 return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_MIN_I32);
6976 case AMDGPU::WAVE_REDUCE_MIN_PSEUDO_I64:
6977 return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::V_CMP_LT_I64_e64);
6978 case AMDGPU::WAVE_REDUCE_FMIN_PSEUDO_F32:
6979 return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::V_MIN_F32_e64);
6980 case AMDGPU::WAVE_REDUCE_FMIN_PSEUDO_F64:
6981 return lowerWaveReduce(MI, *BB, *getSubtarget(),
6982 ST.getGeneration() >= AMDGPUSubtarget::GFX12
6983 ? AMDGPU::V_MIN_NUM_F64_e64
6984 : AMDGPU::V_MIN_F64_e64);
6985 case AMDGPU::WAVE_REDUCE_UMAX_PSEUDO_U32:
6986 return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_MAX_U32);
6987 case AMDGPU::WAVE_REDUCE_UMAX_PSEUDO_U64:
6988 return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::V_CMP_GT_U64_e64);
6989 case AMDGPU::WAVE_REDUCE_MAX_PSEUDO_I32:
6990 return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_MAX_I32);
6991 case AMDGPU::WAVE_REDUCE_MAX_PSEUDO_I64:
6992 return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::V_CMP_GT_I64_e64);
6993 case AMDGPU::WAVE_REDUCE_FMAX_PSEUDO_F32:
6994 return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::V_MAX_F32_e64);
6995 case AMDGPU::WAVE_REDUCE_FMAX_PSEUDO_F64:
6996 return lowerWaveReduce(MI, *BB, *getSubtarget(),
6997 ST.getGeneration() >= AMDGPUSubtarget::GFX12
6998 ? AMDGPU::V_MAX_NUM_F64_e64
6999 : AMDGPU::V_MAX_F64_e64);
7000 case AMDGPU::WAVE_REDUCE_ADD_PSEUDO_I32:
7001 return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_ADD_I32);
7002 case AMDGPU::WAVE_REDUCE_ADD_PSEUDO_U64:
7003 return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_ADD_U64_PSEUDO);
7004 case AMDGPU::WAVE_REDUCE_FADD_PSEUDO_F32:
7005 return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::V_ADD_F32_e64);
7006 case AMDGPU::WAVE_REDUCE_FADD_PSEUDO_F64:
7007 return lowerWaveReduce(MI, *BB, *getSubtarget(),
7008 ST.getGeneration() >= AMDGPUSubtarget::GFX12
7009 ? AMDGPU::V_ADD_F64_pseudo_e64
7010 : AMDGPU::V_ADD_F64_e64);
7011 case AMDGPU::WAVE_REDUCE_SUB_PSEUDO_I32:
7012 return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_SUB_I32);
7013 case AMDGPU::WAVE_REDUCE_SUB_PSEUDO_U64:
7014 return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_SUB_U64_PSEUDO);
7015 case AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F32:
7016 return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::V_SUB_F32_e64);
7017 case AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64:
7018 // There is no S/V_SUB_F64 opcode. Double type subtraction is expanded as
7019 // fadd + neg, by setting the NEG bit in the instruction.
7020 return lowerWaveReduce(MI, *BB, *getSubtarget(),
7021 ST.getGeneration() >= AMDGPUSubtarget::GFX12
7022 ? AMDGPU::V_ADD_F64_pseudo_e64
7023 : AMDGPU::V_ADD_F64_e64);
7024 case AMDGPU::WAVE_REDUCE_AND_PSEUDO_B32:
7025 return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_AND_B32);
7026 case AMDGPU::WAVE_REDUCE_AND_PSEUDO_B64:
7027 return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_AND_B64);
7028 case AMDGPU::WAVE_REDUCE_OR_PSEUDO_B32:
7029 return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_OR_B32);
7030 case AMDGPU::WAVE_REDUCE_OR_PSEUDO_B64:
7031 return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_OR_B64);
7032 case AMDGPU::WAVE_REDUCE_XOR_PSEUDO_B32:
7033 return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_XOR_B32);
7034 case AMDGPU::WAVE_REDUCE_XOR_PSEUDO_B64:
7035 return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_XOR_B64);
7036 case AMDGPU::S_UADDO_PSEUDO:
7037 case AMDGPU::S_USUBO_PSEUDO: {
7038 MachineOperand &Dest0 = MI.getOperand(0);
7039 MachineOperand &Dest1 = MI.getOperand(1);
7040 MachineOperand &Src0 = MI.getOperand(2);
7041 MachineOperand &Src1 = MI.getOperand(3);
7042
7043 unsigned Opc = (MI.getOpcode() == AMDGPU::S_UADDO_PSEUDO)
7044 ? AMDGPU::S_ADD_U32
7045 : AMDGPU::S_SUB_U32;
7046 // clang-format off
7047 BuildMI(*BB, MI, DL, TII->get(Opc), Dest0.getReg())
7048 .add(Src0)
7049 .add(Src1);
7050 // clang-format on
7051
7052 unsigned SelOpc =
7053 Subtarget->isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
7054 BuildMI(*BB, MI, DL, TII->get(SelOpc), Dest1.getReg()).addImm(-1).addImm(0);
7055
7056 MI.eraseFromParent();
7057 return BB;
7058 }
7059 case AMDGPU::S_ADD_U64_PSEUDO:
7060 case AMDGPU::S_SUB_U64_PSEUDO: {
7061 return expand64BitScalarArithmetic(MI, BB);
7062 }
7063 case AMDGPU::V_ADD_U64_PSEUDO:
7064 case AMDGPU::V_SUB_U64_PSEUDO: {
7065 bool IsAdd = (MI.getOpcode() == AMDGPU::V_ADD_U64_PSEUDO);
7066
7067 MachineOperand &Dest = MI.getOperand(0);
7068 MachineOperand &Src0 = MI.getOperand(1);
7069 MachineOperand &Src1 = MI.getOperand(2);
7070
7071 if (ST.hasAddSubU64Insts()) {
7072 auto I = BuildMI(*BB, MI, DL,
7073 TII->get(IsAdd ? AMDGPU::V_ADD_U64_e64
7074 : AMDGPU::V_SUB_U64_e64),
7075 Dest.getReg())
7076 .add(Src0)
7077 .add(Src1)
7078 .addImm(0); // clamp
7079 TII->legalizeOperands(*I);
7080 MI.eraseFromParent();
7081 return BB;
7082 }
7083
7084 if (IsAdd && ST.hasLshlAddU64Inst()) {
7085 auto Add = BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_LSHL_ADD_U64_e64),
7086 Dest.getReg())
7087 .add(Src0)
7088 .addImm(0)
7089 .add(Src1);
7090 TII->legalizeOperands(*Add);
7091 MI.eraseFromParent();
7092 return BB;
7093 }
7094
7095 const auto *CarryRC = TRI->getWaveMaskRegClass();
7096
7097 Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
7098 Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
7099
7100 Register CarryReg = MRI.createVirtualRegister(CarryRC);
7101 Register DeadCarryReg = MRI.createVirtualRegister(CarryRC);
7102
7103 const TargetRegisterClass *Src0RC = Src0.isReg()
7104 ? MRI.getRegClass(Src0.getReg())
7105 : &AMDGPU::VReg_64RegClass;
7106 const TargetRegisterClass *Src1RC = Src1.isReg()
7107 ? MRI.getRegClass(Src1.getReg())
7108 : &AMDGPU::VReg_64RegClass;
7109
7110 const TargetRegisterClass *Src0SubRC =
7111 TRI->getSubRegisterClass(Src0RC, AMDGPU::sub0);
7112 const TargetRegisterClass *Src1SubRC =
7113 TRI->getSubRegisterClass(Src1RC, AMDGPU::sub1);
7114
7115 MachineOperand SrcReg0Sub0 = TII->buildExtractSubRegOrImm(
7116 MI, MRI, Src0, Src0RC, AMDGPU::sub0, Src0SubRC);
7117 MachineOperand SrcReg1Sub0 = TII->buildExtractSubRegOrImm(
7118 MI, MRI, Src1, Src1RC, AMDGPU::sub0, Src1SubRC);
7119
7120 MachineOperand SrcReg0Sub1 = TII->buildExtractSubRegOrImm(
7121 MI, MRI, Src0, Src0RC, AMDGPU::sub1, Src0SubRC);
7122 MachineOperand SrcReg1Sub1 = TII->buildExtractSubRegOrImm(
7123 MI, MRI, Src1, Src1RC, AMDGPU::sub1, Src1SubRC);
7124
7125 unsigned LoOpc =
7126 IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
7127 MachineInstr *LoHalf = BuildMI(*BB, MI, DL, TII->get(LoOpc), DestSub0)
7128 .addReg(CarryReg, RegState::Define)
7129 .add(SrcReg0Sub0)
7130 .add(SrcReg1Sub0)
7131 .addImm(0); // clamp bit
7132
7133 unsigned HiOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64;
7134 MachineInstr *HiHalf =
7135 BuildMI(*BB, MI, DL, TII->get(HiOpc), DestSub1)
7136 .addReg(DeadCarryReg, RegState::Define | RegState::Dead)
7137 .add(SrcReg0Sub1)
7138 .add(SrcReg1Sub1)
7139 .addReg(CarryReg)
7140 .addImm(0); // clamp bit
7141
7142 BuildMI(*BB, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), Dest.getReg())
7143 .addReg(DestSub0)
7144 .addImm(AMDGPU::sub0)
7145 .addReg(DestSub1)
7146 .addImm(AMDGPU::sub1);
7147 TII->legalizeOperands(*LoHalf);
7148 TII->legalizeOperands(*HiHalf);
7149 MI.eraseFromParent();
7150 return BB;
7151 }
7152 case AMDGPU::S_ADD_CO_PSEUDO:
7153 case AMDGPU::S_SUB_CO_PSEUDO: {
7154 // This pseudo has a chance to be selected
7155 // only from uniform add/subcarry node. All the VGPR operands
7156 // therefore assumed to be splat vectors.
7158 MachineOperand &Dest = MI.getOperand(0);
7159 MachineOperand &CarryDest = MI.getOperand(1);
7160 MachineOperand &Src0 = MI.getOperand(2);
7161 MachineOperand &Src1 = MI.getOperand(3);
7162 MachineOperand &Src2 = MI.getOperand(4);
7163 if (Src0.isReg() && TRI->isVectorRegister(MRI, Src0.getReg())) {
7164 Register RegOp0 = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
7165 BuildMI(*BB, MII, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), RegOp0)
7166 .addReg(Src0.getReg());
7167 Src0.setReg(RegOp0);
7168 }
7169 if (Src1.isReg() && TRI->isVectorRegister(MRI, Src1.getReg())) {
7170 Register RegOp1 = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
7171 BuildMI(*BB, MII, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), RegOp1)
7172 .addReg(Src1.getReg());
7173 Src1.setReg(RegOp1);
7174 }
7175 Register RegOp2 = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
7176 if (TRI->isVectorRegister(MRI, Src2.getReg())) {
7177 BuildMI(*BB, MII, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), RegOp2)
7178 .addReg(Src2.getReg());
7179 Src2.setReg(RegOp2);
7180 }
7181
7182 if (ST.isWave64()) {
7183 if (ST.hasScalarCompareEq64()) {
7184 BuildMI(*BB, MII, DL, TII->get(AMDGPU::S_CMP_LG_U64))
7185 .addReg(Src2.getReg())
7186 .addImm(0);
7187 } else {
7188 const TargetRegisterClass *Src2RC = MRI.getRegClass(Src2.getReg());
7189 const TargetRegisterClass *SubRC =
7190 TRI->getSubRegisterClass(Src2RC, AMDGPU::sub0);
7191 MachineOperand Src2Sub0 = TII->buildExtractSubRegOrImm(
7192 MII, MRI, Src2, Src2RC, AMDGPU::sub0, SubRC);
7193 MachineOperand Src2Sub1 = TII->buildExtractSubRegOrImm(
7194 MII, MRI, Src2, Src2RC, AMDGPU::sub1, SubRC);
7195 Register Src2_32 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
7196
7197 BuildMI(*BB, MII, DL, TII->get(AMDGPU::S_OR_B32), Src2_32)
7198 .add(Src2Sub0)
7199 .add(Src2Sub1);
7200
7201 BuildMI(*BB, MII, DL, TII->get(AMDGPU::S_CMP_LG_U32))
7202 .addReg(Src2_32)
7203 .addImm(0);
7204 }
7205 } else {
7206 BuildMI(*BB, MII, DL, TII->get(AMDGPU::S_CMP_LG_U32))
7207 .addReg(Src2.getReg())
7208 .addImm(0);
7209 }
7210
7211 unsigned Opc = MI.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO
7212 ? AMDGPU::S_ADDC_U32
7213 : AMDGPU::S_SUBB_U32;
7214
7215 BuildMI(*BB, MII, DL, TII->get(Opc), Dest.getReg()).add(Src0).add(Src1);
7216
7217 unsigned SelOpc =
7218 ST.isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
7219
7220 BuildMI(*BB, MII, DL, TII->get(SelOpc), CarryDest.getReg())
7221 .addImm(-1)
7222 .addImm(0);
7223
7224 MI.eraseFromParent();
7225 return BB;
7226 }
7227 case AMDGPU::SI_INIT_M0: {
7228 MachineOperand &M0Init = MI.getOperand(0);
7229 BuildMI(*BB, MI.getIterator(), MI.getDebugLoc(),
7230 TII->get(M0Init.isReg() ? AMDGPU::COPY : AMDGPU::S_MOV_B32),
7231 AMDGPU::M0)
7232 .add(M0Init);
7233 MI.eraseFromParent();
7234 return BB;
7235 }
7236 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM: {
7237 // Set SCC to true, in case the barrier instruction gets converted to a NOP.
7238 BuildMI(*BB, MI.getIterator(), MI.getDebugLoc(),
7239 TII->get(AMDGPU::S_CMP_EQ_U32))
7240 .addImm(0)
7241 .addImm(0);
7242 return BB;
7243 }
7244 case AMDGPU::GET_GROUPSTATICSIZE: {
7245 assert(getTargetMachine().getTargetTriple().getOS() == Triple::AMDHSA ||
7246 getTargetMachine().getTargetTriple().getOS() == Triple::AMDPAL);
7247 BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_MOV_B32))
7248 .add(MI.getOperand(0))
7249 .addImm(MFI->getLDSSize());
7250 MI.eraseFromParent();
7251 return BB;
7252 }
7253 case AMDGPU::GET_SHADERCYCLESHILO: {
7254 assert(MF->getSubtarget<GCNSubtarget>().hasShaderCyclesHiLoRegisters());
7255 // The algorithm is:
7256 //
7257 // hi1 = getreg(SHADER_CYCLES_HI)
7258 // lo1 = getreg(SHADER_CYCLES_LO)
7259 // hi2 = getreg(SHADER_CYCLES_HI)
7260 //
7261 // If hi1 == hi2 then there was no overflow and the result is hi2:lo1.
7262 // Otherwise there was overflow and the result is hi2:0. In both cases the
7263 // result should represent the actual time at some point during the sequence
7264 // of three getregs.
7265 using namespace AMDGPU::Hwreg;
7266 Register RegHi1 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
7267 BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_GETREG_B32), RegHi1)
7268 .addImm(HwregEncoding::encode(ID_SHADER_CYCLES_HI, 0, 32));
7269 Register RegLo1 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
7270 BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_GETREG_B32), RegLo1)
7271 .addImm(HwregEncoding::encode(ID_SHADER_CYCLES, 0, 32));
7272 Register RegHi2 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
7273 BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_GETREG_B32), RegHi2)
7274 .addImm(HwregEncoding::encode(ID_SHADER_CYCLES_HI, 0, 32));
7275 BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_CMP_EQ_U32))
7276 .addReg(RegHi1)
7277 .addReg(RegHi2);
7278 Register RegLo = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
7279 BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_CSELECT_B32), RegLo)
7280 .addReg(RegLo1)
7281 .addImm(0);
7282 BuildMI(*BB, MI, DL, TII->get(AMDGPU::REG_SEQUENCE))
7283 .add(MI.getOperand(0))
7284 .addReg(RegLo)
7285 .addImm(AMDGPU::sub0)
7286 .addReg(RegHi2)
7287 .addImm(AMDGPU::sub1);
7288 MI.eraseFromParent();
7289 return BB;
7290 }
7291 case AMDGPU::SI_INDIRECT_SRC_V1:
7292 case AMDGPU::SI_INDIRECT_SRC_V2:
7293 case AMDGPU::SI_INDIRECT_SRC_V3:
7294 case AMDGPU::SI_INDIRECT_SRC_V4:
7295 case AMDGPU::SI_INDIRECT_SRC_V5:
7296 case AMDGPU::SI_INDIRECT_SRC_V6:
7297 case AMDGPU::SI_INDIRECT_SRC_V7:
7298 case AMDGPU::SI_INDIRECT_SRC_V8:
7299 case AMDGPU::SI_INDIRECT_SRC_V9:
7300 case AMDGPU::SI_INDIRECT_SRC_V10:
7301 case AMDGPU::SI_INDIRECT_SRC_V11:
7302 case AMDGPU::SI_INDIRECT_SRC_V12:
7303 case AMDGPU::SI_INDIRECT_SRC_V16:
7304 case AMDGPU::SI_INDIRECT_SRC_V32:
7305 return emitIndirectSrc(MI, *BB, *getSubtarget());
7306 case AMDGPU::SI_INDIRECT_DST_V1:
7307 case AMDGPU::SI_INDIRECT_DST_V2:
7308 case AMDGPU::SI_INDIRECT_DST_V3:
7309 case AMDGPU::SI_INDIRECT_DST_V4:
7310 case AMDGPU::SI_INDIRECT_DST_V5:
7311 case AMDGPU::SI_INDIRECT_DST_V6:
7312 case AMDGPU::SI_INDIRECT_DST_V7:
7313 case AMDGPU::SI_INDIRECT_DST_V8:
7314 case AMDGPU::SI_INDIRECT_DST_V9:
7315 case AMDGPU::SI_INDIRECT_DST_V10:
7316 case AMDGPU::SI_INDIRECT_DST_V11:
7317 case AMDGPU::SI_INDIRECT_DST_V12:
7318 case AMDGPU::SI_INDIRECT_DST_V16:
7319 case AMDGPU::SI_INDIRECT_DST_V32:
7320 return emitIndirectDst(MI, *BB, *getSubtarget());
7321 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO:
7322 case AMDGPU::SI_KILL_I1_PSEUDO:
7323 return splitKillBlock(MI, BB);
7324 case AMDGPU::V_CNDMASK_B64_PSEUDO: {
7326 return BB;
7327 }
7328 case AMDGPU::SI_BR_UNDEF: {
7329 MachineInstr *Br = BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_CBRANCH_SCC1))
7330 .add(MI.getOperand(0));
7331 Br->getOperand(1).setIsUndef(); // read undef SCC
7332 MI.eraseFromParent();
7333 return BB;
7334 }
7335 case AMDGPU::ADJCALLSTACKUP:
7336 case AMDGPU::ADJCALLSTACKDOWN: {
7338 MachineInstrBuilder MIB(*MF, &MI);
7339 MIB.addReg(Info->getStackPtrOffsetReg(), RegState::ImplicitDefine)
7340 .addReg(Info->getStackPtrOffsetReg(), RegState::Implicit);
7341 return BB;
7342 }
7343 case AMDGPU::SI_CALL_ISEL: {
7344 unsigned ReturnAddrReg = TII->getRegisterInfo().getReturnAddressReg(*MF);
7345
7347 MIB = BuildMI(*BB, MI, DL, TII->get(AMDGPU::SI_CALL))
7348 .addDef(ReturnAddrReg, RegState::Dead);
7349
7350 for (const MachineOperand &MO : MI.operands())
7351 MIB.add(MO);
7352
7353 MIB.cloneMemRefs(MI);
7354 MIB.setMIFlags(MI.getFlags());
7355 MI.eraseFromParent();
7356 return BB;
7357 }
7358 case AMDGPU::V_ADDC_U32_e32:
7359 case AMDGPU::V_SUBB_U32_e32:
7360 case AMDGPU::V_SUBBREV_U32_e32:
7361 // These instructions have an implicit use of vcc which counts towards the
7362 // constant bus limit.
7363 TII->legalizeOperands(MI);
7364 return BB;
7365 case AMDGPU::DS_GWS_INIT:
7366 case AMDGPU::DS_GWS_SEMA_BR:
7367 case AMDGPU::DS_GWS_BARRIER:
7368 case AMDGPU::DS_GWS_SEMA_V:
7369 case AMDGPU::DS_GWS_SEMA_P:
7370 case AMDGPU::DS_GWS_SEMA_RELEASE_ALL:
7371 // A s_waitcnt 0 is required to be the instruction immediately following.
7372 if (getSubtarget()->hasGWSAutoReplay()) {
7374 return BB;
7375 }
7376
7377 return emitGWSMemViolTestLoop(MI, BB);
7378 case AMDGPU::S_SETREG_B32: {
7379 // Try to optimize cases that only set the denormal mode or rounding mode.
7380 //
7381 // If the s_setreg_b32 fully sets all of the bits in the rounding mode or
7382 // denormal mode to a constant, we can use s_round_mode or s_denorm_mode
7383 // instead.
7384 //
7385 // FIXME: This could be predicates on the immediate, but tablegen doesn't
7386 // allow you to have a no side effect instruction in the output of a
7387 // sideeffecting pattern.
7388 auto [ID, Offset, Width] =
7389 AMDGPU::Hwreg::HwregEncoding::decode(MI.getOperand(1).getImm());
7390 if (ID != AMDGPU::Hwreg::ID_MODE)
7391 return BB;
7392
7393 const unsigned WidthMask = maskTrailingOnes<unsigned>(Width);
7394 const unsigned SetMask = WidthMask << Offset;
7395
7396 if (getSubtarget()->hasDenormModeInst()) {
7397 unsigned SetDenormOp = 0;
7398 unsigned SetRoundOp = 0;
7399
7400 // The dedicated instructions can only set the whole denorm or round mode
7401 // at once, not a subset of bits in either.
7402 if (SetMask ==
7404 // If this fully sets both the round and denorm mode, emit the two
7405 // dedicated instructions for these.
7406 SetRoundOp = AMDGPU::S_ROUND_MODE;
7407 SetDenormOp = AMDGPU::S_DENORM_MODE;
7408 } else if (SetMask == AMDGPU::Hwreg::FP_ROUND_MASK) {
7409 SetRoundOp = AMDGPU::S_ROUND_MODE;
7410 } else if (SetMask == AMDGPU::Hwreg::FP_DENORM_MASK) {
7411 SetDenormOp = AMDGPU::S_DENORM_MODE;
7412 }
7413
7414 if (SetRoundOp || SetDenormOp) {
7415 MachineInstr *Def = MRI.getVRegDef(MI.getOperand(0).getReg());
7416 if (Def && Def->isMoveImmediate() && Def->getOperand(1).isImm()) {
7417 unsigned ImmVal = Def->getOperand(1).getImm();
7418 if (SetRoundOp) {
7419 BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(SetRoundOp))
7420 .addImm(ImmVal & 0xf);
7421
7422 // If we also have the denorm mode, get just the denorm mode bits.
7423 ImmVal >>= 4;
7424 }
7425
7426 if (SetDenormOp) {
7427 BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(SetDenormOp))
7428 .addImm(ImmVal & 0xf);
7429 }
7430
7431 MI.eraseFromParent();
7432 return BB;
7433 }
7434 }
7435 }
7436
7437 // If only FP bits are touched, used the no side effects pseudo.
7438 if ((SetMask & (AMDGPU::Hwreg::FP_ROUND_MASK |
7439 AMDGPU::Hwreg::FP_DENORM_MASK)) == SetMask)
7440 MI.setDesc(TII->get(AMDGPU::S_SETREG_B32_mode));
7441
7442 return BB;
7443 }
7444 case AMDGPU::S_INVERSE_BALLOT_U32:
7445 case AMDGPU::S_INVERSE_BALLOT_U64:
7446 // These opcodes only exist to let SIFixSGPRCopies insert a readfirstlane if
7447 // necessary. After that they are equivalent to a COPY.
7448 MI.setDesc(TII->get(AMDGPU::COPY));
7449 return BB;
7450 case AMDGPU::ENDPGM_TRAP: {
7451 if (BB->succ_empty() && std::next(MI.getIterator()) == BB->end()) {
7452 MI.setDesc(TII->get(AMDGPU::S_ENDPGM));
7453 MI.addOperand(MachineOperand::CreateImm(0));
7454 return BB;
7455 }
7456
7457 // We need a block split to make the real endpgm a terminator. We also don't
7458 // want to break phis in successor blocks, so we can't just delete to the
7459 // end of the block.
7460
7461 MachineBasicBlock *SplitBB = BB->splitAt(MI, false /*UpdateLiveIns*/);
7463 MF->push_back(TrapBB);
7464 // clang-format off
7465 BuildMI(*TrapBB, TrapBB->end(), DL, TII->get(AMDGPU::S_ENDPGM))
7466 .addImm(0);
7467 BuildMI(*BB, &MI, DL, TII->get(AMDGPU::S_CBRANCH_EXECNZ))
7468 .addMBB(TrapBB);
7469 // clang-format on
7470
7471 BB->addSuccessor(TrapBB);
7472 MI.eraseFromParent();
7473 return SplitBB;
7474 }
7475 case AMDGPU::SIMULATED_TRAP: {
7476 assert(Subtarget->hasPrivEnabledTrap2NopBug());
7477 MachineBasicBlock *SplitBB =
7478 TII->insertSimulatedTrap(MRI, *BB, MI, MI.getDebugLoc());
7479 MI.eraseFromParent();
7480 return SplitBB;
7481 }
7482 case AMDGPU::SI_TCRETURN_GFX_WholeWave:
7483 case AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN: {
7485
7486 // During ISel, it's difficult to propagate the original EXEC mask to use as
7487 // an input to SI_WHOLE_WAVE_FUNC_RETURN. Set it up here instead.
7488 MachineInstr *Setup = TII->getWholeWaveFunctionSetup(*BB->getParent());
7489 assert(Setup && "Couldn't find SI_SETUP_WHOLE_WAVE_FUNC");
7490 Register OriginalExec = Setup->getOperand(0).getReg();
7491 MI.getOperand(0).setReg(OriginalExec);
7492 return BB;
7493 }
7494 case AMDGPU::V_DOT2_F32_F16:
7495 case AMDGPU::V_DOT2_F32_BF16: {
7496 // Hint RA to assign dst and src2 the same physical register.
7497 // For targets without VOP2, but with VOPD, variant of the instruction this
7498 // is one of the conditions to attempt converting VOP3P to VOPD.
7499 MRI.setSimpleHint(MI.getOperand(0).getReg(), MI.getOperand(6).getReg());
7500 return BB;
7501 }
7502 case AMDGPU::SCHED_BARRIER:
7503 case AMDGPU::SCHED_GROUP_BARRIER:
7504 MI.getOperand(0).setImm(MI.getOperand(0).getImm() &
7505 static_cast<unsigned>(AMDGPU::SchedGroupMask::ALL));
7506 return BB;
7507 default:
7508 if (TII->isImage(MI) || TII->isMUBUF(MI)) {
7509 if (!MI.mayStore())
7511 return BB;
7512 }
7514 }
7515}
7516
7518 // This currently forces unfolding various combinations of fsub into fma with
7519 // free fneg'd operands. As long as we have fast FMA (controlled by
7520 // isFMAFasterThanFMulAndFAdd), we should perform these.
7521
7522 // When fma is quarter rate, for f64 where add / sub are at best half rate,
7523 // most of these combines appear to be cycle neutral but save on instruction
7524 // count / code size.
7525 return true;
7526}
7527
7529
7531 EVT VT) const {
7532 if (!VT.isVector()) {
7533 return MVT::i1;
7534 }
7535 return EVT::getVectorVT(Ctx, MVT::i1, VT.getVectorNumElements());
7536}
7537
7539 // TODO: Should i16 be used always if legal? For now it would force VALU
7540 // shifts.
7541 return (VT == MVT::i16) ? MVT::i16 : MVT::i32;
7542}
7543
7545 return (Ty.getScalarSizeInBits() <= 16 && Subtarget->has16BitInsts())
7546 ? Ty.changeElementSize(16)
7547 : Ty.changeElementSize(32);
7548}
7549
7550// Answering this is somewhat tricky and depends on the specific device which
7551// have different rates for fma or all f64 operations.
7552//
7553// v_fma_f64 and v_mul_f64 always take the same number of cycles as each other
7554// regardless of which device (although the number of cycles differs between
7555// devices), so it is always profitable for f64.
7556//
7557// v_fma_f32 takes 4 or 16 cycles depending on the device, so it is profitable
7558// only on full rate devices. Normally, we should prefer selecting v_mad_f32
7559// which we can always do even without fused FP ops since it returns the same
7560// result as the separate operations and since it is always full
7561// rate. Therefore, we lie and report that it is not faster for f32. v_mad_f32
7562// however does not support denormals, so we do report fma as faster if we have
7563// a fast fma device and require denormals.
7564//
7566 DenormalFPEnv FPEnv) const {
7567 VT = VT.getScalarType();
7568 if (!VT.isSimple())
7569 return false;
7570
7571 switch (VT.getSimpleVT().SimpleTy) {
7572 case MVT::f32: {
7573 // If mad is not available this depends only on if f32 fma is full rate.
7574 if (!Subtarget->hasMadMacF32Insts())
7575 return Subtarget->hasFastFMAF32();
7576
7577 // Otherwise f32 mad is always full rate and returns the same result as
7578 // the separate operations so should be preferred over fma.
7579 // However does not support denormals.
7581 return Subtarget->hasFastFMAF32() || Subtarget->hasDLInsts();
7582
7583 // If the subtarget has v_fmac_f32, that's just as good as v_mac_f32.
7584 return Subtarget->hasFastFMAF32() && Subtarget->hasDLInsts();
7585 }
7586 case MVT::f64:
7587 return true;
7588 case MVT::f16:
7589 case MVT::bf16:
7590 return Subtarget->has16BitInsts() &&
7592 default:
7593 break;
7594 }
7595
7596 return false;
7597}
7598
7603
7605 Type *Ty) const {
7607 getValueType(F.getDataLayout(), Ty, /*AllowUnknown=*/true),
7608 F.getDenormalFPEnv());
7609}
7610
7612 LLT Ty) const {
7613 switch (Ty.getScalarSizeInBits()) {
7614 case 16:
7615 return isFMAFasterThanFMulAndFAdd(MF, MVT::f16);
7616 case 32:
7617 return isFMAFasterThanFMulAndFAdd(MF, MVT::f32);
7618 case 64:
7619 return isFMAFasterThanFMulAndFAdd(MF, MVT::f64);
7620 default:
7621 break;
7622 }
7623
7624 return false;
7625}
7626
7628 // TODO: Check future ftz flag
7629 // v_mad_f32/v_mac_f32 do not support denormals.
7630 if (VT == MVT::f32)
7631 return Subtarget->hasMadMacF32Insts() &&
7633 if (VT == MVT::f16)
7634 return Subtarget->hasMadF16() &&
7636
7637 return false;
7638}
7639
7641 if (!Ty.isScalar())
7642 return false;
7643
7644 DenormalFPEnv FPEnv = getDenormalFPEnv(*MI.getMF());
7645 if (Ty.getScalarSizeInBits() == 16)
7646 return isFMADLegal(MVT::f16, FPEnv);
7647 if (Ty.getScalarSizeInBits() == 32)
7648 return isFMADLegal(MVT::f32, FPEnv);
7649
7650 return false;
7651}
7652
7654 const SDNode *N) const {
7655 return isFMADLegal(N->getValueType(0),
7657}
7658
7660 return isFMADLegal(getValueType(F.getDataLayout(), Ty->getScalarType(),
7661 /*AllowUnknown=*/true),
7662 F.getDenormalFPEnv());
7663}
7664
7665//===----------------------------------------------------------------------===//
7666// Custom DAG Lowering Operations
7667//===----------------------------------------------------------------------===//
7668
7669// Work around LegalizeDAG doing the wrong thing and fully scalarizing if the
7670// wider vector type is legal.
7672 SelectionDAG &DAG) const {
7673 unsigned Opc = Op.getOpcode();
7674 EVT VT = Op.getValueType();
7676
7677 auto [Lo, Hi] = DAG.SplitVectorOperand(Op.getNode(), 0);
7678 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT);
7679
7680 SDLoc SL(Op);
7681
7682 // Forward any trailing scalar operands unchanged to both halves.
7683 SmallVector<SDValue, 2> LoOps = {Lo};
7684 SmallVector<SDValue, 2> HiOps = {Hi};
7685 auto TrailingOps = drop_begin(Op->ops());
7686 LoOps.append(TrailingOps.begin(), TrailingOps.end());
7687 HiOps.append(TrailingOps.begin(), TrailingOps.end());
7688
7689 SDValue OpLo = DAG.getNode(Opc, SL, LoVT, LoOps, Op->getFlags());
7690 SDValue OpHi = DAG.getNode(Opc, SL, HiVT, HiOps, Op->getFlags());
7691
7692 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Op), VT, OpLo, OpHi);
7693}
7694
7695// Enable lowering of ROTR for vxi32 types. This is a workaround for a
7696// regression whereby extra unnecessary instructions were added to codegen
7697// for rotr operations, casued by legalising v2i32 or. This resulted in extra
7698// instructions to extract the result from the vector.
7700 [[maybe_unused]] EVT VT = Op.getValueType();
7701
7702 assert((VT == MVT::v2i32 || VT == MVT::v4i32 || VT == MVT::v8i32 ||
7703 VT == MVT::v16i32) &&
7704 "Unexpected ValueType.");
7705
7706 return DAG.UnrollVectorOp(Op.getNode());
7707}
7708
7709// Work around LegalizeDAG doing the wrong thing and fully scalarizing if the
7710// wider vector type is legal.
7712 SelectionDAG &DAG) const {
7713 unsigned Opc = Op.getOpcode();
7714 EVT VT = Op.getValueType();
7716
7717 auto [Lo0, Hi0] = DAG.SplitVectorOperand(Op.getNode(), 0);
7718 auto [Lo1, Hi1] = DAG.SplitVectorOperand(Op.getNode(), 1);
7719
7720 SDLoc SL(Op);
7721
7722 SDValue OpLo =
7723 DAG.getNode(Opc, SL, Lo0.getValueType(), Lo0, Lo1, Op->getFlags());
7724 SDValue OpHi =
7725 DAG.getNode(Opc, SL, Hi0.getValueType(), Hi0, Hi1, Op->getFlags());
7726
7727 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Op), VT, OpLo, OpHi);
7728}
7729
7731 SelectionDAG &DAG) const {
7732 unsigned Opc = Op.getOpcode();
7733 EVT VT = Op.getValueType();
7735
7736 SDValue Op0 = Op.getOperand(0);
7737 SDValue Lo0, Hi0;
7738 if (Op0.getValueType().isVector())
7739 std::tie(Lo0, Hi0) = DAG.SplitVectorOperand(Op.getNode(), 0);
7740 else
7741 Lo0 = Hi0 = DAG.getFreeze(Op0);
7742
7743 auto [Lo1, Hi1] = DAG.SplitVectorOperand(Op.getNode(), 1);
7744 auto [Lo2, Hi2] = DAG.SplitVectorOperand(Op.getNode(), 2);
7745
7746 SDLoc SL(Op);
7747 auto ResVT = DAG.GetSplitDestVTs(VT);
7748
7749 SDValue OpLo =
7750 DAG.getNode(Opc, SL, ResVT.first, Lo0, Lo1, Lo2, Op->getFlags());
7751 SDValue OpHi =
7752 DAG.getNode(Opc, SL, ResVT.second, Hi0, Hi1, Hi2, Op->getFlags());
7753
7754 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Op), VT, OpLo, OpHi);
7755}
7756
7758 switch (Op.getOpcode()) {
7759 default:
7761 case ISD::BRCOND:
7762 return LowerBRCOND(Op, DAG);
7763 case ISD::RETURNADDR:
7764 return LowerRETURNADDR(Op, DAG);
7765 case ISD::FrameIndex:
7766 return lowerFrameIndex(Op, DAG);
7767 case ISD::SPONENTRY:
7768 return LowerSPONENTRY(Op, DAG);
7769 case ISD::LOAD: {
7770 SDValue Result = LowerLOAD(Op, DAG);
7771 assert((!Result.getNode() || Result.getNode()->getNumValues() == 2) &&
7772 "Load should return a value and a chain");
7773 return Result;
7774 }
7775 case ISD::FSQRT: {
7776 EVT VT = Op.getValueType();
7777 if (VT == MVT::f32)
7778 return lowerFSQRTF32(Op, DAG);
7779 if (VT == MVT::f64)
7780 return lowerFSQRTF64(Op, DAG);
7781 return SDValue();
7782 }
7783 case ISD::FSIN:
7784 case ISD::FCOS:
7785 return LowerTrig(Op, DAG);
7786 case ISD::SELECT:
7787 return LowerSELECT(Op, DAG);
7788 case ISD::FDIV:
7789 return LowerFDIV(Op, DAG);
7790 case ISD::FFREXP:
7791 return LowerFFREXP(Op, DAG);
7793 return LowerATOMIC_CMP_SWAP(Op, DAG);
7794 case ISD::STORE:
7795 return LowerSTORE(Op, DAG);
7796 case ISD::GlobalAddress: {
7799 return LowerGlobalAddress(MFI, Op, DAG);
7800 }
7801 case ISD::BlockAddress:
7802 return LowerBlockAddress(Op, DAG);
7804 return LowerExternalSymbol(Op, DAG);
7806 return LowerINTRINSIC_WO_CHAIN(Op, DAG);
7808 return LowerCONVERT_FROM_ARBITRARY_FP(Op, DAG);
7810 return LowerCONVERT_TO_ARBITRARY_FP(Op, DAG);
7812 return LowerINTRINSIC_W_CHAIN(Op, DAG);
7814 return LowerINTRINSIC_VOID(Op, DAG);
7815 case ISD::ADDRSPACECAST:
7816 return lowerADDRSPACECAST(Op, DAG);
7818 return lowerINSERT_SUBVECTOR(Op, DAG);
7820 return lowerINSERT_VECTOR_ELT(Op, DAG);
7822 return lowerEXTRACT_VECTOR_ELT(Op, DAG);
7824 return lowerVECTOR_SHUFFLE(Op, DAG);
7826 return lowerSCALAR_TO_VECTOR(Op, DAG);
7827 case ISD::BUILD_VECTOR:
7828 return lowerBUILD_VECTOR(Op, DAG);
7829 case ISD::FP_ROUND:
7831 return lowerFP_ROUND(Op, DAG);
7832 case ISD::TRAP:
7833 return lowerTRAP(Op, DAG);
7834 case ISD::DEBUGTRAP:
7835 return lowerDEBUGTRAP(Op, DAG);
7836 case ISD::ABS:
7837 case ISD::FABS:
7838 case ISD::FNEG:
7839 case ISD::FCANONICALIZE:
7840 case ISD::BSWAP:
7841 return splitUnaryVectorOp(Op, DAG);
7844 if (Op.getValueType().isVector() && Op.getValueType() != MVT::v2i16 &&
7845 Op.getOperand(0).getValueType().getScalarType() == MVT::f32)
7846 return splitUnaryVectorOp(Op, DAG);
7847 return LowerFP_TO_INT_SAT(Op, DAG);
7848 case ISD::FSUB:
7849 if (Op.getValueType() == MVT::bf16) {
7850 // Custom expansion:
7851 // fsub bf16 %a, %b -> fadd v2bf16(widen %a), fneg v2bf16(widen %b)
7852 // Then extract back to bf16.
7853 //
7854 // We create fneg on v2bf16 (not bf16) so the instruction selector can
7855 // fold the negation into the packed add's neg_lo/neg_hi modifiers,
7856 // generating a single v_pk_add_bf16 instruction. If we negate bf16 first,
7857 // it becomes a separate v_xor instruction before widening.
7858 SDLoc DL(Op);
7859 SDValue Op0 = Op.getOperand(0);
7860 SDValue Op1 = Op.getOperand(1);
7861
7862 // Widen both operands to v2bf16
7863 SDValue Vec0 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2bf16, Op0);
7864 SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2bf16, Op1);
7865
7866 // Create FNEG v2bf16 for the second operand
7867 SDValue NegVec1 = DAG.getNode(ISD::FNEG, DL, MVT::v2bf16, Vec1);
7868
7869 // Perform FADD v2bf16
7870 SDValue Result = DAG.getNode(ISD::FADD, DL, MVT::v2bf16, Vec0, NegVec1);
7871
7872 // Extract element 0 back to bf16
7873 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::bf16, Result,
7874 DAG.getConstant(0, DL, MVT::i32));
7875 }
7876 return SDValue();
7877 case ISD::FMINNUM:
7878 case ISD::FMAXNUM:
7879 return lowerFMINNUM_FMAXNUM(Op, DAG);
7880 case ISD::FMINIMUMNUM:
7881 case ISD::FMAXIMUMNUM:
7882 return lowerFMINIMUMNUM_FMAXIMUMNUM(Op, DAG);
7883 case ISD::FLDEXP:
7884 case ISD::STRICT_FLDEXP:
7885 return lowerFLDEXP(Op, DAG);
7886 case ISD::FMA:
7887 return splitTernaryVectorOp(Op, DAG);
7888 case ISD::FP_TO_SINT:
7889 case ISD::FP_TO_UINT:
7890 if (Subtarget->hasVCvtPkIU16F32() && Op.getValueType() == MVT::i16 &&
7891 Op.getOperand(0).getValueType() == MVT::f32) {
7892 // Make f32->i16 legal so we can select V_CVT_PK_[IU]16_F32.
7893 return Op;
7894 }
7895 return LowerFP_TO_INT(Op, DAG);
7896 case ISD::SHL:
7897 case ISD::SRA:
7898 case ISD::SRL:
7899 case ISD::ADD:
7900 case ISD::SUB:
7901 case ISD::SMIN:
7902 case ISD::SMAX:
7903 case ISD::UMIN:
7904 case ISD::UMAX:
7905 case ISD::FMINNUM_IEEE:
7906 case ISD::FMAXNUM_IEEE:
7907 case ISD::FMINIMUM:
7908 case ISD::FMAXIMUM:
7909 case ISD::UADDSAT:
7910 case ISD::USUBSAT:
7911 case ISD::SADDSAT:
7912 case ISD::SSUBSAT:
7913 case ISD::FADD:
7914 case ISD::FMUL:
7915 return splitBinaryVectorOp(Op, DAG);
7916 case ISD::FCOPYSIGN:
7917 return lowerFCOPYSIGN(Op, DAG);
7918 case ISD::MUL:
7919 return lowerMUL(Op, DAG);
7920 case ISD::SMULO:
7921 case ISD::UMULO:
7922 return lowerXMULO(Op, DAG);
7923 case ISD::SMUL_LOHI:
7924 case ISD::UMUL_LOHI:
7925 return lowerXMUL_LOHI(Op, DAG);
7927 return LowerDYNAMIC_STACKALLOC(Op, DAG);
7928 case ISD::STACKSAVE:
7929 return LowerSTACKSAVE(Op, DAG);
7930 case ISD::GET_ROUNDING:
7931 return lowerGET_ROUNDING(Op, DAG);
7932 case ISD::SET_ROUNDING:
7933 return lowerSET_ROUNDING(Op, DAG);
7934 case ISD::PREFETCH:
7935 return lowerPREFETCH(Op, DAG);
7936 case ISD::FP_EXTEND:
7938 return lowerFP_EXTEND(Op, DAG);
7939 case ISD::GET_FPENV:
7940 return lowerGET_FPENV(Op, DAG);
7941 case ISD::SET_FPENV:
7942 return lowerSET_FPENV(Op, DAG);
7943 case ISD::ROTR:
7944 return lowerROTR(Op, DAG);
7945 case ISD::INLINEASM:
7946 return LowerINLINEASM(Op, DAG);
7947 }
7948 return SDValue();
7949}
7950
7951// TFE results are dword granular: value dwords followed by one status dword.
7952static std::pair<SDValue, SDValue>
7954 LLVMContext &C = *DAG.getContext();
7955 unsigned NumValueDWords = divideCeil(VT.getSizeInBits(), 32);
7957 DAG.getVectorIdxConstant(NumValueDWords, DL));
7958 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, DL);
7959 SDValue ValueDWords =
7960 NumValueDWords == 1
7961 ? DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Op, ZeroIdx)
7963 EVT::getVectorVT(C, MVT::i32, NumValueDWords), Op,
7964 ZeroIdx);
7965 if (!VT.isVector() && VT.getSizeInBits() < 32)
7966 ValueDWords =
7967 DAG.getNode(ISD::TRUNCATE, DL, VT.changeTypeToInteger(), ValueDWords);
7968 return {DAG.getNode(ISD::BITCAST, DL, VT, ValueDWords), Status};
7969}
7970
7971// Used for D16: Casts the result of an instruction into the right vector,
7972// packs values if loads return unpacked values.
7974 const SDLoc &DL, SelectionDAG &DAG,
7975 bool Unpacked) {
7976 if (!LoadVT.isVector())
7977 return Result;
7978
7979 // Cast back to the original packed type or to a larger type that is a
7980 // multiple of 32 bit for D16. Widening the return type is a required for
7981 // legalization.
7982 EVT FittingLoadVT = LoadVT;
7983 if ((LoadVT.getVectorNumElements() % 2) == 1) {
7984 FittingLoadVT =
7986 LoadVT.getVectorNumElements() + 1);
7987 }
7988
7989 if (Unpacked) { // From v2i32/v4i32 back to v2f16/v4f16.
7990 // Truncate to v2i16/v4i16.
7991 EVT IntLoadVT = FittingLoadVT.changeTypeToInteger();
7992
7993 // Workaround legalizer not scalarizing truncate after vector op
7994 // legalization but not creating intermediate vector trunc.
7996 DAG.ExtractVectorElements(Result, Elts);
7997 for (SDValue &Elt : Elts)
7998 Elt = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Elt);
7999
8000 // Pad illegal v1i16/v3fi6 to v4i16
8001 if ((LoadVT.getVectorNumElements() % 2) == 1)
8002 Elts.push_back(DAG.getPOISON(MVT::i16));
8003
8004 Result = DAG.getBuildVector(IntLoadVT, DL, Elts);
8005
8006 // Bitcast to original type (v2f16/v4f16).
8007 return DAG.getNode(ISD::BITCAST, DL, FittingLoadVT, Result);
8008 }
8009
8010 // Cast back to the original packed type.
8011 return DAG.getNode(ISD::BITCAST, DL, FittingLoadVT, Result);
8012}
8013
8014SDValue SITargetLowering::adjustLoadValueType(unsigned Opcode, MemSDNode *M,
8015 SelectionDAG &DAG,
8017 bool IsIntrinsic) const {
8018 SDLoc DL(M);
8019
8020 bool IsTFE = M->getNumValues() == 3;
8021 bool Unpacked = Subtarget->hasUnpackedD16VMem();
8022 EVT LoadVT = M->getValueType(0);
8023
8024 EVT EquivLoadVT = LoadVT;
8025 if (LoadVT.isVector()) {
8026 if (Unpacked) {
8027 EquivLoadVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32,
8028 LoadVT.getVectorNumElements());
8029 } else if ((LoadVT.getVectorNumElements() % 2) == 1) {
8030 // Widen v3f16 to legal type
8031 EquivLoadVT =
8033 LoadVT.getVectorNumElements() + 1);
8034 }
8035 }
8036
8037 if (IsTFE) {
8038 unsigned NumValueDWords = divideCeil(EquivLoadVT.getSizeInBits(), 32);
8039 EVT LoadDWordsVT =
8040 EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumValueDWords + 1);
8041 SDVTList VTList = DAG.getVTList(LoadDWordsVT, MVT::Other);
8043 Opcode, DL, VTList, Ops, M->getMemoryVT(), M->getMemOperand());
8044 auto [Value, Status] = splitTFEValueAndStatus(Load, EquivLoadVT, DL, DAG);
8045 SDValue Adjusted =
8046 adjustLoadValueTypeImpl(Value, LoadVT, DL, DAG, Unpacked);
8047 return DAG.getMergeValues({Adjusted, Status, Load.getValue(1)}, DL);
8048 }
8049
8050 // Change from v4f16/v2f16 to EquivLoadVT.
8051 SDVTList VTList = DAG.getVTList(EquivLoadVT, MVT::Other);
8052
8054 IsIntrinsic ? (unsigned)ISD::INTRINSIC_W_CHAIN : Opcode, DL, VTList, Ops,
8055 M->getMemoryVT(), M->getMemOperand());
8056
8057 SDValue Adjusted = adjustLoadValueTypeImpl(Load, LoadVT, DL, DAG, Unpacked);
8058
8059 return DAG.getMergeValues({Adjusted, Load.getValue(1)}, DL);
8060}
8061
8062SDValue SITargetLowering::lowerIntrinsicLoad(MemSDNode *M, bool IsFormat,
8063 SelectionDAG &DAG,
8064 ArrayRef<SDValue> Ops) const {
8065 SDLoc DL(M);
8066 EVT LoadVT = M->getValueType(0);
8067 EVT EltType = LoadVT.getScalarType();
8068 EVT IntVT = LoadVT.changeTypeToInteger();
8069
8070 bool IsD16 = IsFormat && (EltType.getSizeInBits() == 16);
8071
8072 if (IsFormat && !IsD16 && EltType.getSizeInBits() < 32) {
8073 DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
8075 "unsupported sub-dword format buffer load", DL.getDebugLoc()));
8076 return DAG.getMergeValues({DAG.getPOISON(LoadVT), M->getOperand(0)}, DL);
8077 }
8078
8079 assert(M->getNumValues() == 2 || M->getNumValues() == 3);
8080 bool IsTFE = M->getNumValues() == 3;
8081
8082 if (IsD16 && IsTFE && !Subtarget->hasBufferTFEFormatD16()) {
8083 DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
8085 "TFE D16 format buffer load is not supported on this GPU",
8086 DL.getDebugLoc()));
8087 return DAG.getErrorMergeValues({M->value_begin(), M->value_end()},
8088 M->getOperand(0), DL);
8089 }
8090
8091 unsigned Opc = IsD16 ? (IsTFE ? AMDGPUISD::BUFFER_LOAD_FORMAT_D16_TFE
8092 : AMDGPUISD::BUFFER_LOAD_FORMAT_D16)
8093 : IsFormat ? (IsTFE ? AMDGPUISD::BUFFER_LOAD_FORMAT_TFE
8094 : AMDGPUISD::BUFFER_LOAD_FORMAT)
8095 : IsTFE ? AMDGPUISD::BUFFER_LOAD_TFE
8096 : AMDGPUISD::BUFFER_LOAD;
8097
8098 if (IsD16)
8099 return adjustLoadValueType(Opc, M, DAG, Ops);
8100
8101 // Handle BUFFER_LOAD_BYTE/UBYTE/SHORT/USHORT overloaded intrinsics
8102 if (!IsD16 && !LoadVT.isVector() && EltType.getSizeInBits() < 32)
8103 return handleByteShortBufferLoads(DAG, LoadVT, DL, Ops, M->getMemOperand(),
8104 IsTFE);
8105
8106 if (isTypeLegal(LoadVT)) {
8107 return getMemIntrinsicNode(Opc, DL, M->getVTList(), Ops, IntVT,
8108 M->getMemOperand(), DAG);
8109 }
8110
8111 EVT CastVT = getEquivalentMemType(*DAG.getContext(), LoadVT);
8112 SDVTList VTList = IsTFE ? DAG.getVTList(CastVT, MVT::i32, MVT::Other)
8113 : DAG.getVTList(CastVT, MVT::Other);
8114 SDValue MemNode = getMemIntrinsicNode(Opc, DL, VTList, Ops, CastVT,
8115 M->getMemOperand(), DAG);
8116 SDValue Data = DAG.getNode(ISD::BITCAST, DL, LoadVT, MemNode);
8117 if (IsTFE)
8118 return DAG.getMergeValues({Data, MemNode.getValue(1), MemNode.getValue(2)},
8119 DL);
8120 return DAG.getMergeValues({Data, MemNode.getValue(1)}, DL);
8121}
8122
8124 SelectionDAG &DAG) {
8125 EVT VT = N->getValueType(0);
8126 SDValue Src = N->getOperand(1);
8127 SDLoc SL(N);
8128
8129 if (Src.getOpcode() == ISD::SETCC) {
8130 SDValue Op0 = Src.getOperand(0);
8131 SDValue Op1 = Src.getOperand(1);
8132 // Need to expand bfloat to float for comparison (setcc).
8133 if (Op0.getValueType() == MVT::bf16) {
8134 Op0 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Op0);
8135 Op1 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Op1);
8136 }
8137 // (ballot (ISD::SETCC ...)) -> (AMDGPUISD::SETCC ...)
8138 return DAG.getNode(AMDGPUISD::SETCC, SL, VT, Op0, Op1, Src.getOperand(2));
8139 }
8140 if (const ConstantSDNode *Arg = dyn_cast<ConstantSDNode>(Src)) {
8141 // (ballot 0) -> 0
8142 if (Arg->isZero())
8143 return DAG.getConstant(0, SL, VT);
8144
8145 // (ballot 1) -> EXEC/EXEC_LO
8146 if (Arg->isOne()) {
8147 Register Exec;
8148 if (VT.getScalarSizeInBits() == 32)
8149 Exec = AMDGPU::EXEC_LO;
8150 else if (VT.getScalarSizeInBits() == 64)
8151 Exec = AMDGPU::EXEC;
8152 else
8153 return SDValue();
8154
8155 return DAG.getCopyFromReg(DAG.getEntryNode(), SL, Exec, VT);
8156 }
8157 }
8158
8159 // (ballot (i1 $src)) -> (AMDGPUISD::SETCC (i32 (zext $src)) (i32 0)
8160 // ISD::SETNE)
8161 return DAG.getNode(
8162 AMDGPUISD::SETCC, SL, VT, DAG.getZExtOrTrunc(Src, SL, MVT::i32),
8163 DAG.getConstant(0, SL, MVT::i32), DAG.getCondCode(ISD::SETNE));
8164}
8165
8167 Intrinsic::ID IntrinsicID) {
8168 bool Signed = IntrinsicID == Intrinsic::amdgcn_sbfe;
8169 SDLoc DL(Op);
8170 EVT VT = Op.getValueType();
8171 SDValue Src = Op.getOperand(1);
8172 SDValue Offset = Op.getOperand(2);
8173 SDValue Width = Op.getOperand(3);
8174
8175 if (VT != MVT::i32) {
8178 Twine(Intrinsic::getBaseName(IntrinsicID)) + " only supports i32",
8179 DL.getDebugLoc()));
8180 return DAG.getPOISON(VT);
8181 }
8182
8183 return DAG.getNode(Signed ? AMDGPUISD::BFE_I32 : AMDGPUISD::BFE_U32, DL, VT,
8184 Src, Offset, Width);
8185}
8186
8188 EVT VT);
8189
8191 SelectionDAG &DAG) {
8192 EVT VT = N->getValueType(0);
8193 unsigned ValSize = VT.getSizeInBits();
8194 unsigned IID = N->getConstantOperandVal(0);
8195 bool IsPermLane16 = IID == Intrinsic::amdgcn_permlane16 ||
8196 IID == Intrinsic::amdgcn_permlanex16;
8197 bool IsSetInactive = IID == Intrinsic::amdgcn_set_inactive ||
8198 IID == Intrinsic::amdgcn_set_inactive_chain_arg;
8199 bool IsPermlaneShuffle = IID == Intrinsic::amdgcn_permlane_bcast ||
8200 IID == Intrinsic::amdgcn_permlane_up ||
8201 IID == Intrinsic::amdgcn_permlane_down ||
8202 IID == Intrinsic::amdgcn_permlane_xor;
8203 SDLoc SL(N);
8204 MVT IntVT = MVT::getIntegerVT(ValSize);
8205 const GCNSubtarget *ST = TLI.getSubtarget();
8206
8207 unsigned SplitSize = 32;
8208 if (IID == Intrinsic::amdgcn_update_dpp && (ValSize % 64 == 0) &&
8209 ST->hasDPALU_DPP() &&
8210 AMDGPU::isLegalDPALU_DPPControl(*ST, N->getConstantOperandVal(3)))
8211 SplitSize = 64;
8212
8213 auto createLaneOp = [&DAG, &SL, N, IID](SDValue Src0, SDValue Src1,
8214 SDValue Src2, MVT ValT) -> SDValue {
8216 switch (IID) {
8217 case Intrinsic::amdgcn_permlane16:
8218 case Intrinsic::amdgcn_permlanex16:
8219 case Intrinsic::amdgcn_update_dpp:
8220 Operands.push_back(N->getOperand(6));
8221 Operands.push_back(N->getOperand(5));
8222 Operands.push_back(N->getOperand(4));
8223 [[fallthrough]];
8224 case Intrinsic::amdgcn_writelane:
8225 case Intrinsic::amdgcn_permlane_bcast:
8226 case Intrinsic::amdgcn_permlane_up:
8227 case Intrinsic::amdgcn_permlane_down:
8228 case Intrinsic::amdgcn_permlane_xor:
8229 Operands.push_back(Src2);
8230 [[fallthrough]];
8231 case Intrinsic::amdgcn_readlane:
8232 case Intrinsic::amdgcn_set_inactive:
8233 case Intrinsic::amdgcn_set_inactive_chain_arg:
8234 case Intrinsic::amdgcn_mov_dpp8:
8235 Operands.push_back(Src1);
8236 [[fallthrough]];
8237 case Intrinsic::amdgcn_readfirstlane:
8238 case Intrinsic::amdgcn_permlane64:
8239 Operands.push_back(Src0);
8240 break;
8241 default:
8242 llvm_unreachable("unhandled lane op");
8243 }
8244
8245 Operands.push_back(DAG.getTargetConstant(IID, SL, MVT::i32));
8246 std::reverse(Operands.begin(), Operands.end());
8247
8248 if (SDNode *GL = N->getGluedNode()) {
8249 assert(GL->getOpcode() == ISD::CONVERGENCECTRL_GLUE);
8250 GL = GL->getOperand(0).getNode();
8251 Operands.push_back(DAG.getNode(ISD::CONVERGENCECTRL_GLUE, SL, MVT::Glue,
8252 SDValue(GL, 0)));
8253 }
8254
8255 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, ValT, Operands);
8256 };
8257
8258 SDValue Src0 = N->getOperand(1);
8259 SDValue Src1, Src2;
8260 if (IID == Intrinsic::amdgcn_readlane || IID == Intrinsic::amdgcn_writelane ||
8261 IID == Intrinsic::amdgcn_mov_dpp8 ||
8262 IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16 ||
8263 IsPermlaneShuffle) {
8264 Src1 = N->getOperand(2);
8265 if (IID == Intrinsic::amdgcn_writelane ||
8266 IID == Intrinsic::amdgcn_update_dpp || IsPermLane16 ||
8267 IsPermlaneShuffle)
8268 Src2 = N->getOperand(3);
8269 }
8270
8271 if (ValSize == SplitSize) {
8272 // Already legal
8273 return SDValue();
8274 }
8275
8276 if (ValSize < 32) {
8277 bool IsFloat = VT.isFloatingPoint();
8278 Src0 = DAG.getAnyExtOrTrunc(IsFloat ? DAG.getBitcast(IntVT, Src0) : Src0,
8279 SL, MVT::i32);
8280
8281 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16) {
8282 Src1 = DAG.getAnyExtOrTrunc(IsFloat ? DAG.getBitcast(IntVT, Src1) : Src1,
8283 SL, MVT::i32);
8284 }
8285
8286 if (IID == Intrinsic::amdgcn_writelane) {
8287 Src2 = DAG.getAnyExtOrTrunc(IsFloat ? DAG.getBitcast(IntVT, Src2) : Src2,
8288 SL, MVT::i32);
8289 }
8290
8291 SDValue LaneOp = createLaneOp(Src0, Src1, Src2, MVT::i32);
8292 SDValue Trunc = DAG.getAnyExtOrTrunc(LaneOp, SL, IntVT);
8293 return IsFloat ? DAG.getBitcast(VT, Trunc) : Trunc;
8294 }
8295
8296 if (ValSize % SplitSize != 0)
8297 return SDValue();
8298
8299 auto unrollLaneOp = [&DAG, &SL](SDNode *N) -> SDValue {
8300 EVT VT = N->getValueType(0);
8301 unsigned NE = VT.getVectorNumElements();
8302 EVT EltVT = VT.getVectorElementType();
8304 unsigned NumOperands = N->getNumOperands();
8305 SmallVector<SDValue, 4> Operands(NumOperands);
8306 SDNode *GL = N->getGluedNode();
8307
8308 // only handle convergencectrl_glue
8310
8311 for (unsigned i = 0; i != NE; ++i) {
8312 for (unsigned j = 0, e = GL ? NumOperands - 1 : NumOperands; j != e;
8313 ++j) {
8314 SDValue Operand = N->getOperand(j);
8315 EVT OperandVT = Operand.getValueType();
8316 if (OperandVT.isVector()) {
8317 // A vector operand; extract a single element.
8318 EVT OperandEltVT = OperandVT.getVectorElementType();
8319 Operands[j] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, OperandEltVT,
8320 Operand, DAG.getVectorIdxConstant(i, SL));
8321 } else {
8322 // A scalar operand; just use it as is.
8323 Operands[j] = Operand;
8324 }
8325 }
8326
8327 if (GL)
8328 Operands[NumOperands - 1] =
8329 DAG.getNode(ISD::CONVERGENCECTRL_GLUE, SL, MVT::Glue,
8330 SDValue(GL->getOperand(0).getNode(), 0));
8331
8332 Scalars.push_back(DAG.getNode(N->getOpcode(), SL, EltVT, Operands));
8333 }
8334
8335 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NE);
8336 return DAG.getBuildVector(VecVT, SL, Scalars);
8337 };
8338
8339 if (VT.isVector()) {
8340 switch (MVT::SimpleValueType EltTy =
8342 case MVT::i32:
8343 case MVT::f32:
8344 if (SplitSize == 32) {
8345 SDValue LaneOp = createLaneOp(Src0, Src1, Src2, VT.getSimpleVT());
8346 return unrollLaneOp(LaneOp.getNode());
8347 }
8348 [[fallthrough]];
8349 case MVT::i16:
8350 case MVT::f16:
8351 case MVT::bf16: {
8352 unsigned SubVecNumElt =
8353 SplitSize / VT.getVectorElementType().getSizeInBits();
8354 MVT SubVecVT = MVT::getVectorVT(EltTy, SubVecNumElt);
8356 SDValue Src0SubVec, Src1SubVec, Src2SubVec;
8357 for (unsigned i = 0, EltIdx = 0; i < ValSize / SplitSize; i++) {
8358 Src0SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, SubVecVT, Src0,
8359 DAG.getConstant(EltIdx, SL, MVT::i32));
8360
8361 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive ||
8362 IsPermLane16) {
8363 Src1SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, SubVecVT, Src1,
8364 DAG.getConstant(EltIdx, SL, MVT::i32));
8365
8366 Pieces.push_back(
8367 createLaneOp(Src0SubVec, Src1SubVec, Src2, SubVecVT));
8368 } else if (IID == Intrinsic::amdgcn_writelane) {
8369 Src2SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, SubVecVT, Src2,
8370 DAG.getConstant(EltIdx, SL, MVT::i32));
8371 Pieces.push_back(
8372 createLaneOp(Src0SubVec, Src1, Src2SubVec, SubVecVT));
8373 } else {
8374 Pieces.push_back(createLaneOp(Src0SubVec, Src1, Src2, SubVecVT));
8375 }
8376
8377 EltIdx += SubVecNumElt;
8378 }
8379 return DAG.getNode(ISD::CONCAT_VECTORS, SL, VT, Pieces);
8380 }
8381 default:
8382 // Handle all other cases by bitcasting to i32 vectors
8383 break;
8384 }
8385 }
8386
8387 MVT VecVT =
8388 MVT::getVectorVT(MVT::getIntegerVT(SplitSize), ValSize / SplitSize);
8389 Src0 = DAG.getBitcast(VecVT, Src0);
8390
8391 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16)
8392 Src1 = DAG.getBitcast(VecVT, Src1);
8393
8394 if (IID == Intrinsic::amdgcn_writelane)
8395 Src2 = DAG.getBitcast(VecVT, Src2);
8396
8397 SDValue LaneOp = createLaneOp(Src0, Src1, Src2, VecVT);
8398 SDValue UnrolledLaneOp = unrollLaneOp(LaneOp.getNode());
8399 return DAG.getBitcast(VT, UnrolledLaneOp);
8400}
8401
8403 SelectionDAG &DAG) {
8404 EVT VT = N->getValueType(0);
8405
8406 if (VT.getSizeInBits() != 32)
8407 return SDValue();
8408
8409 SDLoc SL(N);
8410
8411 SDValue Value = N->getOperand(1);
8412 SDValue Index = N->getOperand(2);
8413
8414 // ds_bpermute requires index to be multiplied by 4
8415 SDValue ShiftAmount = DAG.getShiftAmountConstant(2, MVT::i32, SL);
8416 SDValue ShiftedIndex =
8417 DAG.getNode(ISD::SHL, SL, Index.getValueType(), Index, ShiftAmount);
8418
8419 // Intrinsics will require i32 to operate on
8420 SDValue ValueI32 = DAG.getBitcast(MVT::i32, Value);
8421
8422 auto MakeIntrinsic = [&DAG, &SL](unsigned IID, MVT RetVT,
8423 SmallVector<SDValue> IntrinArgs) -> SDValue {
8425 Operands[0] = DAG.getTargetConstant(IID, SL, MVT::i32);
8426 Operands.append(IntrinArgs);
8427 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, RetVT, Operands);
8428 };
8429
8430 // If we can bpermute across the whole wave, then just do that
8431 if (TLI.getSubtarget()->supportsWaveWideBPermute()) {
8432 SDValue BPermute = MakeIntrinsic(Intrinsic::amdgcn_ds_bpermute, MVT::i32,
8433 {ShiftedIndex, ValueI32});
8434 return DAG.getBitcast(VT, BPermute);
8435 }
8436
8437 assert(TLI.getSubtarget()->isWave64());
8438
8439 // Otherwise, we need to make use of whole wave mode
8440 SDValue PoisonVal = DAG.getPOISON(ValueI32->getValueType(0));
8441
8442 // Set inactive lanes to poison
8443 SDValue WWMValue = MakeIntrinsic(Intrinsic::amdgcn_set_inactive, MVT::i32,
8444 {ValueI32, PoisonVal});
8445 SDValue WWMIndex = MakeIntrinsic(Intrinsic::amdgcn_set_inactive, MVT::i32,
8446 {ShiftedIndex, PoisonVal});
8447
8448 SDValue Swapped =
8449 MakeIntrinsic(Intrinsic::amdgcn_permlane64, MVT::i32, {WWMValue});
8450
8451 // Get permutation of each half, then we'll select which one to use
8452 SDValue BPermSameHalf = MakeIntrinsic(Intrinsic::amdgcn_ds_bpermute, MVT::i32,
8453 {WWMIndex, WWMValue});
8454 SDValue BPermOtherHalf = MakeIntrinsic(Intrinsic::amdgcn_ds_bpermute,
8455 MVT::i32, {WWMIndex, Swapped});
8456 SDValue BPermOtherHalfWWM =
8457 MakeIntrinsic(Intrinsic::amdgcn_wwm, MVT::i32, {BPermOtherHalf});
8458
8459 // Select which side to take the permute from
8460 SDValue ThreadIDMask = DAG.getAllOnesConstant(SL, MVT::i32);
8461 // We can get away with only using mbcnt_lo here since we're only
8462 // trying to detect which side of 32 each lane is on, and mbcnt_lo
8463 // returns 32 for lanes 32-63.
8464 SDValue ThreadID =
8465 MakeIntrinsic(Intrinsic::amdgcn_mbcnt_lo, MVT::i32,
8466 {ThreadIDMask, DAG.getTargetConstant(0, SL, MVT::i32)});
8467
8468 SDValue SameOrOtherHalf =
8469 DAG.getNode(ISD::AND, SL, MVT::i32,
8470 DAG.getNode(ISD::XOR, SL, MVT::i32, ThreadID, Index),
8471 DAG.getTargetConstant(32, SL, MVT::i32));
8472 SDValue UseSameHalf =
8473 DAG.getSetCC(SL, MVT::i1, SameOrOtherHalf,
8474 DAG.getConstant(0, SL, MVT::i32), ISD::SETEQ);
8475 SDValue Result = DAG.getSelect(SL, MVT::i32, UseSameHalf, BPermSameHalf,
8476 BPermOtherHalfWWM);
8477 return DAG.getBitcast(VT, Result);
8478}
8479
8482 SelectionDAG &DAG) const {
8483 switch (N->getOpcode()) {
8485 if (SDValue Res = lowerINSERT_VECTOR_ELT(SDValue(N, 0), DAG))
8486 Results.push_back(Res);
8487 return;
8488 }
8490 if (SDValue Res = lowerEXTRACT_VECTOR_ELT(SDValue(N, 0), DAG))
8491 Results.push_back(Res);
8492 return;
8493 }
8495 if (SDValue Res = LowerCONVERT_TO_ARBITRARY_FP(SDValue(N, 0), DAG))
8496 Results.push_back(Res);
8497 return;
8498 }
8500 unsigned IID = N->getConstantOperandVal(0);
8501 switch (IID) {
8502 case Intrinsic::amdgcn_wave_reduce_min:
8503 case Intrinsic::amdgcn_wave_reduce_umin:
8504 case Intrinsic::amdgcn_wave_reduce_max:
8505 case Intrinsic::amdgcn_wave_reduce_umax:
8506 case Intrinsic::amdgcn_wave_reduce_add:
8507 case Intrinsic::amdgcn_wave_reduce_sub:
8508 case Intrinsic::amdgcn_wave_reduce_and:
8509 case Intrinsic::amdgcn_wave_reduce_or:
8510 case Intrinsic::amdgcn_wave_reduce_xor: {
8511 EVT VT = N->getValueType(0);
8512 if (isTypeLegal(VT))
8513 return;
8514 SDLoc SL(N);
8515 bool NeedsSignExt = IID == Intrinsic::amdgcn_wave_reduce_min ||
8516 IID == Intrinsic::amdgcn_wave_reduce_max ||
8517 IID == Intrinsic::amdgcn_wave_reduce_add ||
8518 IID == Intrinsic::amdgcn_wave_reduce_sub;
8519 unsigned ExtOpc = NeedsSignExt ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
8520 SDValue ExtSrc = DAG.getNode(ExtOpc, SL, MVT::i32, N->getOperand(1));
8521 SDValue Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32,
8522 N->getOperand(0), ExtSrc, N->getOperand(2));
8523 Results.push_back(DAG.getNode(ISD::TRUNCATE, SL, VT, Result));
8524 return;
8525 }
8526 case Intrinsic::amdgcn_make_buffer_rsrc:
8527 Results.push_back(lowerPointerAsRsrcIntrin(N, DAG));
8528 return;
8529 case Intrinsic::amdgcn_cvt_pkrtz: {
8530 SDValue Src0 = N->getOperand(1);
8531 SDValue Src1 = N->getOperand(2);
8532 SDLoc SL(N);
8533 SDValue Cvt =
8534 DAG.getNode(AMDGPUISD::CVT_PKRTZ_F16_F32, SL, MVT::i32, Src0, Src1);
8535 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Cvt));
8536 return;
8537 }
8538 case Intrinsic::amdgcn_cvt_pknorm_i16:
8539 case Intrinsic::amdgcn_cvt_pknorm_u16:
8540 case Intrinsic::amdgcn_cvt_pk_i16:
8541 case Intrinsic::amdgcn_cvt_pk_u16: {
8542 SDValue Src0 = N->getOperand(1);
8543 SDValue Src1 = N->getOperand(2);
8544 SDLoc SL(N);
8545 unsigned Opcode;
8546
8547 if (IID == Intrinsic::amdgcn_cvt_pknorm_i16)
8548 Opcode = AMDGPUISD::CVT_PKNORM_I16_F32;
8549 else if (IID == Intrinsic::amdgcn_cvt_pknorm_u16)
8550 Opcode = AMDGPUISD::CVT_PKNORM_U16_F32;
8551 else if (IID == Intrinsic::amdgcn_cvt_pk_i16)
8552 Opcode = AMDGPUISD::CVT_PK_I16_I32;
8553 else
8554 Opcode = AMDGPUISD::CVT_PK_U16_U32;
8555
8556 EVT VT = N->getValueType(0);
8557 if (isTypeLegal(VT))
8558 Results.push_back(DAG.getNode(Opcode, SL, VT, Src0, Src1));
8559 else {
8560 SDValue Cvt = DAG.getNode(Opcode, SL, MVT::i32, Src0, Src1);
8561 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, Cvt));
8562 }
8563 return;
8564 }
8565 case Intrinsic::amdgcn_s_buffer_load: {
8566 SDValue Op = SDValue(N, 0);
8567 EVT VT = Op.getValueType();
8568 Results.push_back(lowerSBuffer(VT, VT, SDLoc(Op), DAG.getEntryNode(),
8569 Op.getOperand(1), Op.getOperand(2),
8570 Op.getOperand(3), DAG));
8571 return;
8572 }
8573 case Intrinsic::amdgcn_dead: {
8574 for (unsigned I = 0, E = N->getNumValues(); I < E; ++I)
8575 Results.push_back(DAG.getPOISON(N->getValueType(I)));
8576 return;
8577 }
8578 }
8579 break;
8580 }
8582 if (N->getConstantOperandVal(1) != Intrinsic::amdgcn_ptr_s_buffer_load &&
8583 N->getValueType(0).isSimple() &&
8584 SBufferLoadDiagnosticVTs[N->getSimpleValueType(0).SimpleTy])
8585 break;
8586 if (SDValue Res = LowerINTRINSIC_W_CHAIN(SDValue(N, 0), DAG)) {
8587 if (Res.getOpcode() == ISD::MERGE_VALUES) {
8588 // FIXME: Hacky
8589 for (unsigned I = 0; I < Res.getNumOperands(); I++) {
8590 Results.push_back(Res.getOperand(I));
8591 }
8592 } else {
8593 for (unsigned I = 0; I < N->getNumValues(); ++I)
8594 Results.push_back(Res.getValue(I));
8595 }
8596 return;
8597 }
8598
8599 break;
8600 }
8601 case ISD::SELECT: {
8602 SDLoc SL(N);
8603 EVT VT = N->getValueType(0);
8604 EVT NewVT = getEquivalentMemType(*DAG.getContext(), VT);
8605 SDValue LHS = DAG.getNode(ISD::BITCAST, SL, NewVT, N->getOperand(1));
8606 SDValue RHS = DAG.getNode(ISD::BITCAST, SL, NewVT, N->getOperand(2));
8607
8608 EVT SelectVT = NewVT;
8609 if (NewVT.bitsLT(MVT::i32)) {
8610 LHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, LHS);
8611 RHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, RHS);
8612 SelectVT = MVT::i32;
8613 }
8614
8615 SDValue NewSelect =
8616 DAG.getNode(ISD::SELECT, SL, SelectVT, N->getOperand(0), LHS, RHS);
8617
8618 if (NewVT != SelectVT)
8619 NewSelect = DAG.getNode(ISD::TRUNCATE, SL, NewVT, NewSelect);
8620 Results.push_back(DAG.getNode(ISD::BITCAST, SL, VT, NewSelect));
8621 return;
8622 }
8623 case ISD::FNEG: {
8624 if (N->getValueType(0) != MVT::v2f16)
8625 break;
8626
8627 SDLoc SL(N);
8628 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0));
8629
8630 SDValue Op = DAG.getNode(ISD::XOR, SL, MVT::i32, BC,
8631 DAG.getConstant(0x80008000, SL, MVT::i32));
8632 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Op));
8633 return;
8634 }
8635 case ISD::FABS: {
8636 if (N->getValueType(0) != MVT::v2f16)
8637 break;
8638
8639 SDLoc SL(N);
8640 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0));
8641
8642 SDValue Op = DAG.getNode(ISD::AND, SL, MVT::i32, BC,
8643 DAG.getConstant(0x7fff7fff, SL, MVT::i32));
8644 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Op));
8645 return;
8646 }
8647 case ISD::FSQRT: {
8648 if (N->getValueType(0) != MVT::f16)
8649 break;
8650 Results.push_back(lowerFSQRTF16(SDValue(N, 0), DAG));
8651 break;
8652 }
8653 default:
8655 break;
8656 }
8657}
8658
8659/// Helper function for LowerBRCOND
8660static SDNode *findUser(SDValue Value, unsigned Opcode) {
8661
8662 for (SDUse &U : Value->uses()) {
8663 if (U.get() != Value)
8664 continue;
8665
8666 if (U.getUser()->getOpcode() == Opcode)
8667 return U.getUser();
8668 }
8669 return nullptr;
8670}
8671
8672unsigned SITargetLowering::isCFIntrinsic(const SDNode *Intr) const {
8673 if (Intr->getOpcode() == ISD::INTRINSIC_W_CHAIN) {
8674 switch (Intr->getConstantOperandVal(1)) {
8675 case Intrinsic::amdgcn_if:
8676 return AMDGPUISD::IF;
8677 case Intrinsic::amdgcn_else:
8678 return AMDGPUISD::ELSE;
8679 case Intrinsic::amdgcn_loop:
8680 return AMDGPUISD::LOOP;
8681 case Intrinsic::amdgcn_end_cf:
8682 llvm_unreachable("should not occur");
8683 default:
8684 return 0;
8685 }
8686 }
8687
8688 // break, if_break, else_break are all only used as inputs to loop, not
8689 // directly as branch conditions.
8690 return 0;
8691}
8692
8699
8701 if (Subtarget->isAmdPalOS() || Subtarget->isMesa3DOS())
8702 return false;
8703
8704 // FIXME: Either avoid relying on address space here or change the default
8705 // address space for functions to avoid the explicit check.
8706 return (GV->getValueType()->isFunctionTy() ||
8709}
8710
8712 return !shouldEmitFixup(GV) && !shouldEmitGOTReloc(GV);
8713}
8714
8716 if (!GV->hasExternalLinkage())
8717 return true;
8718
8719 // With object linking, external LDS declarations need relocations so the
8720 // linker can assign their offsets.
8722 if (const auto *GVar = dyn_cast<GlobalVariable>(GV)) {
8723 if (GVar->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
8724 GVar->getAddressSpace() == AMDGPUAS::BARRIER) {
8725 assert(GVar->isDeclaration() &&
8726 "AS 3 & 13 GVs should be declaration here "
8727 "when object linking is enabled");
8728 return false;
8729 }
8730 }
8731 }
8732
8733 const auto OS = getTargetMachine().getTargetTriple().getOS();
8734 return OS == Triple::AMDHSA || OS == Triple::AMDPAL;
8735}
8736
8737/// This transforms the control flow intrinsics to get the branch destination as
8738/// last parameter, also switches branch target with BR if the need arise
8739SDValue SITargetLowering::LowerBRCOND(SDValue BRCOND, SelectionDAG &DAG) const {
8740 SDLoc DL(BRCOND);
8741
8742 SDNode *Intr = BRCOND.getOperand(1).getNode();
8743 SDValue Target = BRCOND.getOperand(2);
8744 SDNode *BR = nullptr;
8745 SDNode *SetCC = nullptr;
8746
8747 switch (Intr->getOpcode()) {
8748 case ISD::SETCC: {
8749 // As long as we negate the condition everything is fine
8750 SetCC = Intr;
8751 Intr = SetCC->getOperand(0).getNode();
8752 break;
8753 }
8754 case ISD::XOR: {
8755 // Similar to SETCC, if we have (xor c, -1), we will be fine.
8756 SDValue LHS = Intr->getOperand(0);
8757 SDValue RHS = Intr->getOperand(1);
8758 if (auto *C = dyn_cast<ConstantSDNode>(RHS); C && C->getZExtValue()) {
8759 Intr = LHS.getNode();
8760 break;
8761 }
8762 [[fallthrough]];
8763 }
8764 default: {
8765 // Get the target from BR if we don't negate the condition
8766 BR = findUser(BRCOND, ISD::BR);
8767 assert(BR && "brcond missing unconditional branch user");
8768 Target = BR->getOperand(1);
8769 }
8770 }
8771
8772 unsigned CFNode = isCFIntrinsic(Intr);
8773 if (CFNode == 0) {
8774 // This is a uniform branch so we don't need to legalize.
8775 return BRCOND;
8776 }
8777
8778 bool HaveChain = Intr->getOpcode() == ISD::INTRINSIC_VOID ||
8780
8781 assert(!SetCC || (SetCC->getConstantOperandVal(1) == 1 &&
8782 SetCC->getOperandCondCode(2) == ISD::SETNE));
8783
8784 // operands of the new intrinsic call
8786 if (HaveChain)
8787 Ops.push_back(BRCOND.getOperand(0));
8788
8789 Ops.append(Intr->op_begin() + (HaveChain ? 2 : 1), Intr->op_end());
8790 Ops.push_back(Target);
8791
8792 ArrayRef<EVT> Res(Intr->value_begin() + 1, Intr->value_end());
8793
8794 // build the new intrinsic call
8795 SDNode *Result = DAG.getNode(CFNode, DL, DAG.getVTList(Res), Ops).getNode();
8796
8797 if (!HaveChain) {
8798 SDValue Ops[] = {SDValue(Result, 0), BRCOND.getOperand(0)};
8799
8801 }
8802
8803 if (BR) {
8804 // Give the branch instruction our target
8805 SDValue Ops[] = {BR->getOperand(0), BRCOND.getOperand(2)};
8806 SDValue NewBR = DAG.getNode(ISD::BR, DL, BR->getVTList(), Ops);
8807 DAG.ReplaceAllUsesWith(BR, NewBR.getNode());
8808 }
8809
8810 SDValue Chain = SDValue(Result, Result->getNumValues() - 1);
8811
8812 // Copy the intrinsic results to registers
8813 for (unsigned i = 1, e = Intr->getNumValues() - 1; i != e; ++i) {
8814 SDNode *CopyToReg = findUser(SDValue(Intr, i), ISD::CopyToReg);
8815 if (!CopyToReg)
8816 continue;
8817
8818 Chain = DAG.getCopyToReg(Chain, DL, CopyToReg->getOperand(1),
8819 SDValue(Result, i - 1), SDValue());
8820
8821 DAG.ReplaceAllUsesWith(SDValue(CopyToReg, 0), CopyToReg->getOperand(0));
8822 }
8823
8824 // Remove the old intrinsic from the chain
8825 DAG.ReplaceAllUsesOfValueWith(SDValue(Intr, Intr->getNumValues() - 1),
8826 Intr->getOperand(0));
8827
8828 return Chain;
8829}
8830
8831SDValue SITargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const {
8832 MVT VT = Op.getSimpleValueType();
8833 SDLoc DL(Op);
8834 // Checking the depth
8835 if (Op.getConstantOperandVal(0) != 0)
8836 return DAG.getConstant(0, DL, VT);
8837
8839 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
8840 // Check for kernel and shader functions
8841 if (Info->isEntryFunction())
8842 return DAG.getConstant(0, DL, VT);
8843
8844 MachineFrameInfo &MFI = MF.getFrameInfo();
8845 // There is a call to @llvm.returnaddress in this function
8846 MFI.setReturnAddressIsTaken(true);
8847
8848 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
8849 // Get the return address reg and mark it as an implicit live-in
8850 Register Reg = MF.addLiveIn(TRI->getReturnAddressReg(MF),
8851 getRegClassFor(VT, Op.getNode()->isDivergent()));
8852
8853 return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, VT);
8854}
8855
8856SDValue SITargetLowering::LowerSPONENTRY(SDValue Op, SelectionDAG &DAG) const {
8858 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
8859
8860 // For functions that set up their own stack, select the GET_STACK_BASE
8861 // pseudo.
8862 if (MFI->isBottomOfStack())
8863 return Op;
8864
8865 // For everything else, create a dummy stack object.
8866 int FI = MF.getFrameInfo().CreateFixedObject(1, 0, /*IsImmutable=*/false);
8867 return DAG.getFrameIndex(FI, Op.getValueType());
8868}
8869
8870SDValue SITargetLowering::getFPExtOrFPRound(SelectionDAG &DAG, SDValue Op,
8871 const SDLoc &DL, EVT VT) const {
8872 return Op.getValueType().bitsLE(VT)
8873 ? DAG.getNode(ISD::FP_EXTEND, DL, VT, Op)
8874 : DAG.getNode(ISD::FP_ROUND, DL, VT, Op,
8875 DAG.getTargetConstant(0, DL, MVT::i32));
8876}
8877
8878SDValue SITargetLowering::splitFP_ROUNDVectorOp(SDValue Op,
8879 SelectionDAG &DAG) const {
8880 EVT DstVT = Op.getValueType();
8881 unsigned NumElts = DstVT.getVectorNumElements();
8882 assert(NumElts > 2 && isPowerOf2_32(NumElts));
8883
8884 auto [Lo, Hi] = DAG.SplitVectorOperand(Op.getNode(), 0);
8885
8886 SDLoc DL(Op);
8887 unsigned Opc = Op.getOpcode();
8888 SDValue Flags = Op.getOperand(1);
8889 EVT HalfDstVT =
8890 EVT::getVectorVT(*DAG.getContext(), DstVT.getScalarType(), NumElts / 2);
8891 SDValue OpLo = DAG.getNode(Opc, DL, HalfDstVT, Lo, Flags);
8892 SDValue OpHi = DAG.getNode(Opc, DL, HalfDstVT, Hi, Flags);
8893
8894 return DAG.getNode(ISD::CONCAT_VECTORS, DL, DstVT, OpLo, OpHi);
8895}
8896
8897SDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
8898 bool IsStrict = Op->isStrictFPOpcode();
8899 SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
8900 EVT SrcVT = Src.getValueType();
8901 EVT DstVT = Op.getValueType();
8902
8903 if (DstVT.isVectorOf(MVT::f16)) {
8904 assert(Subtarget->hasCvtPkF16F32Inst() && "support v_cvt_pk_f16_f32");
8905 if (SrcVT.getScalarType() != MVT::f32)
8906 return SDValue();
8907 return SrcVT == MVT::v2f32 ? Op : splitFP_ROUNDVectorOp(Op, DAG);
8908 }
8909
8910 if (SrcVT.getScalarType() != MVT::f64)
8911 return Op;
8912
8913 SDLoc DL(Op);
8914 if (DstVT == MVT::f16) {
8915 // TODO: Handle strictfp
8916 if (Op.getOpcode() != ISD::FP_ROUND)
8917 return Op;
8918
8919 if (!Subtarget->has16BitInsts()) {
8920 SDValue FpToFp16 = DAG.getNode(ISD::FP_TO_FP16, DL, MVT::i32, Src);
8921 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FpToFp16);
8922 return DAG.getNode(ISD::BITCAST, DL, MVT::f16, Trunc);
8923 }
8924 if (Op->getFlags().hasApproximateFuncs()) {
8925 SDValue Flags = Op.getOperand(1);
8926 SDValue Src32 = DAG.getNode(ISD::FP_ROUND, DL, MVT::f32, Src, Flags);
8927 return DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, Src32, Flags);
8928 }
8929 SDValue FpToFp16 = LowerF64ToF16Safe(Src, DL, DAG);
8930 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FpToFp16);
8931 return DAG.getNode(ISD::BITCAST, DL, MVT::f16, Trunc);
8932 }
8933
8934 assert(DstVT.getScalarType() == MVT::bf16 &&
8935 "custom lower FP_ROUND for f16 or bf16");
8936 assert(Subtarget->hasBF16ConversionInsts() && "f32 -> bf16 is legal");
8937
8938 // Round-inexact-to-odd f64 to f32, then do the final rounding using the
8939 // hardware f32 -> bf16 instruction.
8940 EVT F32VT = SrcVT.changeElementType(*DAG.getContext(), MVT::f32);
8941 SDValue Rod = expandRoundInexactToOdd(F32VT, Src, DL, DAG);
8942 if (IsStrict) {
8943 return DAG.getNode(
8944 ISD::STRICT_FP_ROUND, DL, {DstVT, MVT::Other},
8945 {Op.getOperand(0), Rod, DAG.getTargetConstant(0, DL, MVT::i32)});
8946 }
8947 return DAG.getNode(ISD::FP_ROUND, DL, DstVT, Rod,
8948 DAG.getTargetConstant(0, DL, MVT::i32));
8949}
8950
8951SDValue SITargetLowering::lowerFMINNUM_FMAXNUM(SDValue Op,
8952 SelectionDAG &DAG) const {
8953 EVT VT = Op.getValueType();
8954 const MachineFunction &MF = DAG.getMachineFunction();
8955 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
8956 bool IsIEEEMode = Info->getMode().IEEE;
8957
8958 // FIXME: Assert during selection that this is only selected for
8959 // ieee_mode. Currently a combine can produce the ieee version for non-ieee
8960 // mode functions, but this happens to be OK since it's only done in cases
8961 // where there is known no sNaN.
8962 if (IsIEEEMode && !Subtarget->hasIEEEMinimumMaximumInsts())
8963 return expandFMINNUM_FMAXNUM(Op.getNode(), DAG);
8964
8965 if (VT == MVT::v4f16 || VT == MVT::v8f16 || VT == MVT::v16f16 ||
8966 VT == MVT::v32f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16 ||
8967 VT == MVT::v16bf16 || VT == MVT::v32bf16 || VT == MVT::v4f64 ||
8968 VT == MVT::v8f64 || VT == MVT::v16f64 || VT == MVT::v32f64)
8969 return splitBinaryVectorOp(Op, DAG);
8970 return Op;
8971}
8972
8973SDValue
8974SITargetLowering::lowerFMINIMUMNUM_FMAXIMUMNUM(SDValue Op,
8975 SelectionDAG &DAG) const {
8976 EVT VT = Op.getValueType();
8977 const MachineFunction &MF = DAG.getMachineFunction();
8978 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
8979 bool IsIEEEMode = Info->getMode().IEEE;
8980
8981 if (IsIEEEMode && !Subtarget->hasIEEEMinimumMaximumInsts())
8982 return expandFMINIMUMNUM_FMAXIMUMNUM(Op.getNode(), DAG);
8983
8984 if (VT == MVT::v4f16 || VT == MVT::v8f16 || VT == MVT::v16f16 ||
8985 VT == MVT::v32f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16 ||
8986 VT == MVT::v16bf16 || VT == MVT::v32bf16 || VT == MVT::v4f64 ||
8987 VT == MVT::v8f64 || VT == MVT::v16f64 || VT == MVT::v32f64)
8988 return splitBinaryVectorOp(Op, DAG);
8989 return Op;
8990}
8991
8992SDValue SITargetLowering::lowerFLDEXP(SDValue Op, SelectionDAG &DAG) const {
8993 bool IsStrict = Op.getOpcode() == ISD::STRICT_FLDEXP;
8994 EVT VT = Op.getValueType();
8995 assert(VT == MVT::f16);
8996
8997 SDValue Exp = Op.getOperand(IsStrict ? 2 : 1);
8998 EVT ExpVT = Exp.getValueType();
8999 if (ExpVT == MVT::i16)
9000 return Op;
9001
9002 SDLoc DL(Op);
9003
9004 // Correct the exponent type for f16 to i16.
9005 // Clamp the range of the exponent to the instruction's range.
9006
9007 // TODO: This should be a generic narrowing legalization, and can easily be
9008 // for GlobalISel.
9009
9010 SDValue MinExp = DAG.getSignedConstant(minIntN(16), DL, ExpVT);
9011 SDValue ClampMin = DAG.getNode(ISD::SMAX, DL, ExpVT, Exp, MinExp);
9012
9013 SDValue MaxExp = DAG.getSignedConstant(maxIntN(16), DL, ExpVT);
9014 SDValue Clamp = DAG.getNode(ISD::SMIN, DL, ExpVT, ClampMin, MaxExp);
9015
9016 SDValue TruncExp = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Clamp);
9017
9018 if (IsStrict) {
9019 return DAG.getNode(ISD::STRICT_FLDEXP, DL, {VT, MVT::Other},
9020 {Op.getOperand(0), Op.getOperand(1), TruncExp});
9021 }
9022
9023 return DAG.getNode(ISD::FLDEXP, DL, VT, Op.getOperand(0), TruncExp);
9024}
9025
9027 switch (Op->getOpcode()) {
9028 case ISD::ABS:
9029 case ISD::SRA:
9030 case ISD::SMIN:
9031 case ISD::SMAX:
9032 return ISD::SIGN_EXTEND;
9033 case ISD::SRL:
9034 case ISD::UMIN:
9035 case ISD::UMAX:
9036 case ISD::USUBSAT:
9037 case ISD::UADDSAT:
9038 return ISD::ZERO_EXTEND;
9039 case ISD::ADD:
9040 case ISD::SUB:
9041 case ISD::AND:
9042 case ISD::OR:
9043 case ISD::XOR:
9044 case ISD::SHL:
9045 case ISD::SELECT:
9046 case ISD::MUL:
9047 // operation result won't be influenced by garbage high bits.
9048 // TODO: are all of those cases correct, and are there more?
9049 return ISD::ANY_EXTEND;
9050 case ISD::SETCC: {
9051 ISD::CondCode CC = Op.getOperandCondCode(2);
9053 }
9054 default:
9055 llvm_unreachable("unexpected opcode!");
9056 }
9057}
9058
9059SDValue
9060SITargetLowering::promoteUniformUnaryOpToI32(SDValue Op,
9061 DAGCombinerInfo &DCI) const {
9062 EVT OpTy = Op.getValueType();
9063 SelectionDAG &DAG = DCI.DAG;
9064 EVT ExtTy = OpTy.changeElementType(*DAG.getContext(), MVT::i32);
9065
9066 if (isNarrowingProfitable(Op.getNode(), ExtTy, OpTy))
9067 return SDValue();
9068
9069 SDLoc DL(Op);
9070 SDValue Input = Op.getOperand(0);
9071 const unsigned ExtOp = getExtOpcodeForPromotedOp(Op);
9072 Input = DAG.getNode(ExtOp, DL, ExtTy, Input);
9073
9074 SDValue NewVal = DAG.getNode(Op.getOpcode(), DL, ExtTy, Input);
9075
9076 return DAG.getNode(ISD::TRUNCATE, DL, OpTy, NewVal);
9077}
9078
9079SDValue SITargetLowering::promoteUniformOpToI32(SDValue Op,
9080 DAGCombinerInfo &DCI) const {
9081 const unsigned Opc = Op.getOpcode();
9082 assert(Opc == ISD::ADD || Opc == ISD::SUB || Opc == ISD::SHL ||
9083 Opc == ISD::SRL || Opc == ISD::SRA || Opc == ISD::AND ||
9084 Opc == ISD::OR || Opc == ISD::XOR || Opc == ISD::MUL ||
9085 Opc == ISD::SETCC || Opc == ISD::SELECT || Opc == ISD::SMIN ||
9086 Opc == ISD::SMAX || Opc == ISD::UMIN || Opc == ISD::UMAX ||
9087 Opc == ISD::USUBSAT || Opc == ISD::UADDSAT);
9088
9089 EVT OpTy = (Opc != ISD::SETCC) ? Op.getValueType()
9090 : Op->getOperand(0).getValueType();
9091 auto &DAG = DCI.DAG;
9092 auto ExtTy = OpTy.changeElementType(*DAG.getContext(), MVT::i32);
9093
9094 if (DCI.isBeforeLegalizeOps() ||
9095 isNarrowingProfitable(Op.getNode(), ExtTy, OpTy))
9096 return SDValue();
9097
9098 SDLoc DL(Op);
9099 SDValue LHS;
9100 SDValue RHS;
9101 if (Opc == ISD::SELECT) {
9102 LHS = Op->getOperand(1);
9103 RHS = Op->getOperand(2);
9104 } else {
9105 LHS = Op->getOperand(0);
9106 RHS = Op->getOperand(1);
9107 }
9108
9109 const unsigned ExtOp = getExtOpcodeForPromotedOp(Op);
9110 LHS = DAG.getNode(ExtOp, DL, ExtTy, {LHS});
9111
9112 // Special case: for shifts, the RHS always needs a zext.
9113 if (Opc == ISD::SHL || Opc == ISD::SRL || Opc == ISD::SRA)
9114 RHS = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtTy, {RHS});
9115 else
9116 RHS = DAG.getNode(ExtOp, DL, ExtTy, {RHS});
9117
9118 // setcc always return i1/i1 vec so no need to truncate after.
9119 if (Opc == ISD::SETCC) {
9120 ISD::CondCode CC = Op.getOperandCondCode(2);
9121 return DAG.getSetCC(DL, Op.getValueType(), LHS, RHS, CC);
9122 }
9123
9124 // For other ops, we extend the operation's return type as well so we need to
9125 // truncate back to the original type.
9126 SDValue NewVal;
9127 if (Opc == ISD::SELECT)
9128 NewVal = DAG.getNode(ISD::SELECT, DL, ExtTy, {Op->getOperand(0), LHS, RHS});
9129 else if (Opc == ISD::UADDSAT) {
9130 SDValue Sum = DAG.getNode(ISD::ADD, DL, ExtTy, LHS, RHS);
9131 SDValue MaxVal = DAG.getConstant(
9132 APInt::getMaxValue(OpTy.getScalarSizeInBits()).zext(32), DL, ExtTy);
9133 NewVal = DAG.getNode(ISD::UMIN, DL, ExtTy, Sum, MaxVal);
9134 } else
9135 NewVal = DAG.getNode(Opc, DL, ExtTy, {LHS, RHS});
9136
9137 return DAG.getZExtOrTrunc(NewVal, DL, OpTy);
9138}
9139
9140SDValue SITargetLowering::lowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const {
9141 SDValue Mag = Op.getOperand(0);
9142 EVT MagVT = Mag.getValueType();
9143
9144 if (MagVT.getVectorNumElements() > 2)
9145 return splitBinaryVectorOp(Op, DAG);
9146
9147 SDValue Sign = Op.getOperand(1);
9148 EVT SignVT = Sign.getValueType();
9149
9150 if (MagVT == SignVT)
9151 return Op;
9152
9153 // fcopysign v2f16:mag, v2f32:sign ->
9154 // fcopysign v2f16:mag,
9155 // bitcast (trunc (srl (bitcast sign to v2i32), 16) to v2i16)
9156
9157 SDLoc SL(Op);
9158 SDValue SignAsInt32 = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Sign);
9159 SDValue ShiftAmt = DAG.getShiftAmountConstant(16, MVT::v2i32, SL);
9160 SDValue SignShifted =
9161 DAG.getNode(ISD::SRL, SL, MVT::v2i32, SignAsInt32, ShiftAmt);
9162 SDValue SignAsInt16 = DAG.getNode(ISD::TRUNCATE, SL, MVT::v2i16, SignShifted);
9163
9164 SDValue SignAsHalf16 = DAG.getNode(ISD::BITCAST, SL, MagVT, SignAsInt16);
9165
9166 return DAG.getNode(ISD::FCOPYSIGN, SL, MagVT, Mag, SignAsHalf16);
9167}
9168
9169// Custom lowering for vector multiplications and s_mul_u64.
9170SDValue SITargetLowering::lowerMUL(SDValue Op, SelectionDAG &DAG) const {
9171 EVT VT = Op.getValueType();
9172
9173 // Split vector operands.
9174 if (VT.isVector())
9175 return splitBinaryVectorOp(Op, DAG);
9176
9177 assert(VT == MVT::i64 && "The following code is a special for s_mul_u64");
9178
9179 // There are four ways to lower s_mul_u64:
9180 //
9181 // 1. If all the operands are uniform, then we lower it as it is.
9182 //
9183 // 2. If the operands are divergent, then we have to split s_mul_u64 in 32-bit
9184 // multiplications because there is not a vector equivalent of s_mul_u64.
9185 //
9186 // 3. If the cost model decides that it is more efficient to use vector
9187 // registers, then we have to split s_mul_u64 in 32-bit multiplications.
9188 // This happens in splitScalarSMULU64() in SIInstrInfo.cpp .
9189 //
9190 // 4. If the cost model decides to use vector registers and both of the
9191 // operands are zero-extended/sign-extended from 32-bits, then we split the
9192 // s_mul_u64 in two 32-bit multiplications. The problem is that it is not
9193 // possible to check if the operands are zero-extended or sign-extended in
9194 // SIInstrInfo.cpp. For this reason, here, we replace s_mul_u64 with
9195 // s_mul_u64_u32_pseudo if both operands are zero-extended and we replace
9196 // s_mul_u64 with s_mul_i64_i32_pseudo if both operands are sign-extended.
9197 // If the cost model decides that we have to use vector registers, then
9198 // splitScalarSMulPseudo() (in SIInstrInfo.cpp) split s_mul_u64_u32/
9199 // s_mul_i64_i32_pseudo in two vector multiplications. If the cost model
9200 // decides that we should use scalar registers, then s_mul_u64_u32_pseudo/
9201 // s_mul_i64_i32_pseudo is lowered as s_mul_u64 in expandPostRAPseudo() in
9202 // SIInstrInfo.cpp .
9203
9204 if (Op->isDivergent())
9205 return SDValue();
9206
9207 SDValue Op0 = Op.getOperand(0);
9208 SDValue Op1 = Op.getOperand(1);
9209 // If all the operands are zero-enteted to 32-bits, then we replace s_mul_u64
9210 // with s_mul_u64_u32_pseudo. If all the operands are sign-extended to
9211 // 32-bits, then we replace s_mul_u64 with s_mul_i64_i32_pseudo.
9212 KnownBits Op0KnownBits = DAG.computeKnownBits(Op0);
9213 unsigned Op0LeadingZeros = Op0KnownBits.countMinLeadingZeros();
9214 KnownBits Op1KnownBits = DAG.computeKnownBits(Op1);
9215 unsigned Op1LeadingZeros = Op1KnownBits.countMinLeadingZeros();
9216 SDLoc SL(Op);
9217 if (Op0LeadingZeros >= 32 && Op1LeadingZeros >= 32)
9218 return SDValue(
9219 DAG.getMachineNode(AMDGPU::S_MUL_U64_U32_PSEUDO, SL, VT, Op0, Op1), 0);
9220 unsigned Op0SignBits = DAG.ComputeNumSignBits(Op0);
9221 unsigned Op1SignBits = DAG.ComputeNumSignBits(Op1);
9222 if (Op0SignBits >= 33 && Op1SignBits >= 33)
9223 return SDValue(
9224 DAG.getMachineNode(AMDGPU::S_MUL_I64_I32_PSEUDO, SL, VT, Op0, Op1), 0);
9225 // If all the operands are uniform, then we lower s_mul_u64 as it is.
9226 return Op;
9227}
9228
9229SDValue SITargetLowering::lowerXMULO(SDValue Op, SelectionDAG &DAG) const {
9230 EVT VT = Op.getValueType();
9231 SDLoc SL(Op);
9232 SDValue LHS = Op.getOperand(0);
9233 SDValue RHS = Op.getOperand(1);
9234 bool isSigned = Op.getOpcode() == ISD::SMULO;
9235
9236 if (ConstantSDNode *RHSC = isConstOrConstSplat(RHS)) {
9237 const APInt &C = RHSC->getAPIntValue();
9238 // mulo(X, 1 << S) -> { X << S, (X << S) >> S != X }
9239 if (C.isPowerOf2()) {
9240 // smulo(x, signed_min) is same as umulo(x, signed_min).
9241 bool UseArithShift = isSigned && !C.isMinSignedValue();
9242 SDValue ShiftAmt = DAG.getConstant(C.logBase2(), SL, MVT::i32);
9243 SDValue Result = DAG.getNode(ISD::SHL, SL, VT, LHS, ShiftAmt);
9244 SDValue Overflow =
9245 DAG.getSetCC(SL, MVT::i1,
9246 DAG.getNode(UseArithShift ? ISD::SRA : ISD::SRL, SL, VT,
9247 Result, ShiftAmt),
9248 LHS, ISD::SETNE);
9249 return DAG.getMergeValues({Result, Overflow}, SL);
9250 }
9251 }
9252
9253 SDValue Result = DAG.getNode(ISD::MUL, SL, VT, LHS, RHS);
9254 SDValue Top =
9255 DAG.getNode(isSigned ? ISD::MULHS : ISD::MULHU, SL, VT, LHS, RHS);
9256
9258 ? DAG.getNode(ISD::SRA, SL, VT, Result,
9259 DAG.getConstant(VT.getScalarSizeInBits() - 1,
9260 SL, MVT::i32))
9261 : DAG.getConstant(0, SL, VT);
9262 SDValue Overflow = DAG.getSetCC(SL, MVT::i1, Top, Sign, ISD::SETNE);
9263
9264 return DAG.getMergeValues({Result, Overflow}, SL);
9265}
9266
9267SDValue SITargetLowering::lowerXMUL_LOHI(SDValue Op, SelectionDAG &DAG) const {
9268 if (Op->isDivergent()) {
9269 // Select to V_MAD_[IU]64_[IU]32.
9270 return Op;
9271 }
9272 if (Subtarget->hasSMulHi()) {
9273 // Expand to S_MUL_I32 + S_MUL_HI_[IU]32.
9274 return SDValue();
9275 }
9276 // The multiply is uniform but we would have to use V_MUL_HI_[IU]32 to
9277 // calculate the high part, so we might as well do the whole thing with
9278 // V_MAD_[IU]64_[IU]32.
9279 return Op;
9280}
9281
9282SDValue SITargetLowering::lowerTRAP(SDValue Op, SelectionDAG &DAG) const {
9283 if (!Subtarget->hasTrapHandler() ||
9284 Subtarget->getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA)
9285 return lowerTrapEndpgm(Op, DAG);
9286
9287 return Subtarget->supportsGetDoorbellID() ? lowerTrapHsa(Op, DAG)
9288 : lowerTrapHsaQueuePtr(Op, DAG);
9289}
9290
9291SDValue SITargetLowering::lowerTrapEndpgm(SDValue Op, SelectionDAG &DAG) const {
9292 SDLoc SL(Op);
9293 SDValue Chain = Op.getOperand(0);
9294 return DAG.getNode(AMDGPUISD::ENDPGM_TRAP, SL, MVT::Other, Chain);
9295}
9296
9297SDValue
9298SITargetLowering::loadImplicitKernelArgument(SelectionDAG &DAG, MVT VT,
9299 const SDLoc &DL, Align Alignment,
9300 ImplicitParameter Param) const {
9303 SDValue Ptr = lowerKernArgParameterPtr(DAG, DL, DAG.getEntryNode(), Offset);
9304 MachinePointerInfo PtrInfo =
9306 return DAG.getLoad(
9307 VT, DL, DAG.getEntryNode(), Ptr, PtrInfo.getWithOffset(Offset), Alignment,
9309}
9310
9311SDValue SITargetLowering::lowerTrapHsaQueuePtr(SDValue Op,
9312 SelectionDAG &DAG) const {
9313 SDLoc SL(Op);
9314 SDValue Chain = Op.getOperand(0);
9315
9316 SDValue QueuePtr;
9317 // For code object version 5, QueuePtr is passed through implicit kernarg.
9318 const Module *M = DAG.getMachineFunction().getFunction().getParent();
9320 QueuePtr =
9321 loadImplicitKernelArgument(DAG, MVT::i64, SL, Align(8), QUEUE_PTR);
9322 } else {
9324 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
9325 Register UserSGPR = Info->getQueuePtrUserSGPR();
9326
9327 if (UserSGPR == AMDGPU::NoRegister) {
9328 // We probably are in a function incorrectly marked with
9329 // amdgpu-no-queue-ptr. This is undefined. We don't want to delete the
9330 // trap, so just use a null pointer.
9331 QueuePtr = DAG.getConstant(0, SL, MVT::i64);
9332 } else {
9333 QueuePtr = CreateLiveInRegister(DAG, &AMDGPU::SReg_64RegClass, UserSGPR,
9334 MVT::i64);
9335 }
9336 }
9337
9338 SDValue SGPR01 = DAG.getRegister(AMDGPU::SGPR0_SGPR1, MVT::i64);
9339 SDValue ToReg = DAG.getCopyToReg(Chain, SL, SGPR01, QueuePtr, SDValue());
9340
9342 SDValue Ops[] = {ToReg, DAG.getTargetConstant(TrapID, SL, MVT::i16), SGPR01,
9343 ToReg.getValue(1)};
9344 return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops);
9345}
9346
9347SDValue SITargetLowering::lowerTrapHsa(SDValue Op, SelectionDAG &DAG) const {
9348 SDLoc SL(Op);
9349 SDValue Chain = Op.getOperand(0);
9350
9351 // We need to simulate the 's_trap 2' instruction on targets that run in
9352 // PRIV=1 (where it is treated as a nop).
9353 if (Subtarget->hasPrivEnabledTrap2NopBug())
9354 return DAG.getNode(AMDGPUISD::SIMULATED_TRAP, SL, MVT::Other, Chain);
9355
9357 SDValue Ops[] = {Chain, DAG.getTargetConstant(TrapID, SL, MVT::i16)};
9358 return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops);
9359}
9360
9361SDValue SITargetLowering::lowerDEBUGTRAP(SDValue Op, SelectionDAG &DAG) const {
9362 SDLoc SL(Op);
9363 SDValue Chain = Op.getOperand(0);
9365
9366 if (!Subtarget->hasTrapHandler() ||
9367 Subtarget->getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA) {
9368 LLVMContext &Ctx = MF.getFunction().getContext();
9369 Ctx.diagnose(DiagnosticInfoUnsupported(MF.getFunction(),
9370 "debugtrap handler not supported",
9371 Op.getDebugLoc(), DS_Warning));
9372 return Chain;
9373 }
9374
9375 uint64_t TrapID =
9377 SDValue Ops[] = {Chain, DAG.getTargetConstant(TrapID, SL, MVT::i16)};
9378 return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops);
9379}
9380
9381/// When a divergent value (in VGPR) is passed to an inline asm with an SGPR
9382/// constraint ('s'), we need to insert v_readfirstlane to move the value from
9383/// VGPR to SGPR. This is done by modifying the CopyToReg nodes in the glue
9384/// chain that feed into the INLINEASM node.
9385SDValue SITargetLowering::LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const {
9386 unsigned NumOps = Op.getNumOperands();
9387
9388 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
9389 SmallSet<Register, 8> SGPRInputRegs;
9390
9391 unsigned NumVals = 0;
9392 for (unsigned I = InlineAsm::Op_FirstOperand; I < NumOps - 1;
9393 I += 1 + NumVals) {
9394 const InlineAsm::Flag Flags(Op.getConstantOperandVal(I));
9395 NumVals = Flags.getNumOperandRegisters();
9396
9397 unsigned RCID;
9398 bool IsSGPRInput = Flags.getKind() == InlineAsm::Kind::RegUse &&
9399 NumVals > 0 && Flags.hasRegClassConstraint(RCID) &&
9400 TRI->isSGPRClass(TRI->getRegClass(RCID));
9401
9402 for (unsigned J = 0; J < NumVals; ++J) {
9403 SDValue Val = Op.getOperand(I + 1 + J);
9404 if (const RegisterSDNode *RegNode =
9406 Register Reg = RegNode->getReg();
9407 if (IsSGPRInput || (Reg.isPhysical() && TRI->isSGPRPhysReg(Reg)))
9408 SGPRInputRegs.insert(Reg);
9409 }
9410 }
9411 }
9412
9413 if (SGPRInputRegs.empty())
9414 return Op;
9415
9416 // Walk the glue chain and insert readfirstlane for divergent SGPR inputs.
9417 SDLoc DL(Op);
9418 SDNode *N = Op.getOperand(NumOps - 1).getNode();
9419
9420 while (N && N->getOpcode() == ISD::CopyToReg) {
9421 Register Reg = cast<RegisterSDNode>(N->getOperand(1))->getReg();
9422 SDValue SrcVal = N->getOperand(2);
9423
9424 // Insert readfirstlane if copying a divergent value to an SGPR input.
9425 if (SrcVal->isDivergent() && SGPRInputRegs.count(Reg)) {
9426 SDValue ReadFirstLaneID =
9427 DAG.getTargetConstant(Intrinsic::amdgcn_readfirstlane, DL, MVT::i32);
9428 SDValue ReadFirstLane =
9430 ReadFirstLaneID, SrcVal);
9431
9432 SmallVector<SDValue, 4> Ops = {N->getOperand(0), N->getOperand(1),
9433 ReadFirstLane};
9434 if (N->getNumOperands() > 3)
9435 Ops.push_back(N->getOperand(3)); // Glue input
9436
9437 DAG.UpdateNodeOperands(N, Ops);
9438 }
9439
9440 // Follow glue chain to next CopyToReg.
9441 SDNode *Next = nullptr;
9442 for (unsigned I = 0, E = N->getNumOperands(); I != E; ++I) {
9443 if (N->getOperand(I).getValueType() == MVT::Glue) {
9444 Next = N->getOperand(I).getNode();
9445 break;
9446 }
9447 }
9448 N = Next;
9449 }
9450
9451 return Op;
9452}
9453
9454SDValue SITargetLowering::getSegmentAperture(unsigned AS, const SDLoc &DL,
9455 SelectionDAG &DAG) const {
9456 unsigned BaseAS = AS;
9457 unsigned SANum = AMDGPU::getSyntheticApertureNumber(AS);
9459 BaseAS = AMDGPUAS::LOCAL_ADDRESS;
9460
9461 SDValue Aperture = getBaseSegmentAperture(BaseAS, DL, DAG);
9462
9463 if (SANum != AMDGPU::SyntheticAperture::None) {
9464 SDValue Tag = DAG.getConstant(SANum, DL, MVT::i32);
9465 return DAG.getNode(ISD::OR, DL, MVT::i32, Aperture, Tag);
9466 }
9467
9468 return Aperture;
9469}
9470
9471SDValue SITargetLowering::getBaseSegmentAperture(unsigned AS, const SDLoc &DL,
9472 SelectionDAG &DAG) const {
9473 const bool IsLDS = (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::BARRIER);
9474
9475 if (Subtarget->hasApertureRegs()) {
9476 const unsigned ApertureRegNo =
9477 IsLDS ? AMDGPU::SRC_SHARED_BASE : AMDGPU::SRC_PRIVATE_BASE;
9478 assert((ApertureRegNo != AMDGPU::SRC_PRIVATE_BASE ||
9479 !Subtarget->hasGloballyAddressableScratch()) &&
9480 "Cannot use src_private_base with globally addressable scratch!");
9481 // Note: this feature (register) is broken. When used as a 32-bit operand,
9482 // it returns a wrong value (all zeroes?). The real value is in the upper 32
9483 // bits.
9484 //
9485 // To work around the issue, emit a 64 bit copy from this register
9486 // then extract the high bits. Note that this shouldn't even result in a
9487 // shift being emitted and simply become a pair of registers (e.g.):
9488 // s_mov_b64 s[6:7], src_shared_base
9489 // v_mov_b32_e32 v1, s7
9490 SDValue Copy =
9491 DAG.getCopyFromReg(DAG.getEntryNode(), DL, ApertureRegNo, MVT::v2i32);
9492 return DAG.getExtractVectorElt(DL, MVT::i32, Copy, 1);
9493 }
9494
9495 // For code object version 5, private_base and shared_base are passed through
9496 // implicit kernargs.
9497 const Module *M = DAG.getMachineFunction().getFunction().getParent();
9500 return loadImplicitKernelArgument(DAG, MVT::i32, DL, Align(4), Param);
9501 }
9502
9504 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
9505 Register UserSGPR = Info->getQueuePtrUserSGPR();
9506 if (UserSGPR == AMDGPU::NoRegister) {
9507 // We probably are in a function incorrectly marked with
9508 // amdgpu-no-queue-ptr. This is undefined.
9509 return DAG.getPOISON(MVT::i32);
9510 }
9511
9512 SDValue QueuePtr =
9513 CreateLiveInRegister(DAG, &AMDGPU::SReg_64RegClass, UserSGPR, MVT::i64);
9514
9515 // Offset into amd_queue_t for group_segment_aperture_base_hi /
9516 // private_segment_aperture_base_hi.
9517 uint32_t StructOffset = IsLDS ? 0x40 : 0x44;
9518
9519 SDValue Ptr =
9520 DAG.getObjectPtrOffset(DL, QueuePtr, TypeSize::getFixed(StructOffset));
9521
9522 // TODO: Use custom target PseudoSourceValue.
9523 // TODO: We should use the value from the IR intrinsic call, but it might not
9524 // be available and how do we get it?
9525 MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
9526 return DAG.getLoad(MVT::i32, DL, QueuePtr.getValue(1), Ptr, PtrInfo,
9527 commonAlignment(Align(64), StructOffset),
9530}
9531
9532/// Return true if the value is a known valid address, such that a null check is
9533/// not necessary.
9535 const AMDGPUTargetMachine &TM, unsigned AddrSpace) {
9537 return true;
9538
9539 if (auto *ConstVal = dyn_cast<ConstantSDNode>(Val))
9540 return ConstVal->getSExtValue() != AMDGPU::getNullPointerValue(AddrSpace);
9541
9542 // TODO: Search through arithmetic, handle arguments and loads
9543 // marked nonnull.
9544 return false;
9545}
9546
9547SDValue SITargetLowering::lowerADDRSPACECAST(SDValue Op,
9548 SelectionDAG &DAG) const {
9549 SDLoc SL(Op);
9550
9551 const AMDGPUTargetMachine &TM =
9552 static_cast<const AMDGPUTargetMachine &>(getTargetMachine());
9553
9554 const auto *ASC = cast<AddrSpaceCastSDNode>(Op);
9555 unsigned SrcAS = ASC->getSrcAddressSpace();
9556 SDValue Src = ASC->getOperand(0);
9557 unsigned DestAS = ASC->getDestAddressSpace();
9558 bool IsNonNull = ASC->getFlags().hasNonNull();
9559
9560 SDValue FlatNullPtr = DAG.getConstant(0, SL, MVT::i64);
9561
9562 // flat -> local/private/barrier/vgpr
9563 if (SrcAS == AMDGPUAS::FLAT_ADDRESS) {
9564 if (DestAS == AMDGPUAS::LOCAL_ADDRESS ||
9565 DestAS == AMDGPUAS::PRIVATE_ADDRESS || DestAS == AMDGPUAS::BARRIER ||
9566 DestAS == AMDGPUAS::VGPR) {
9567 SDValue Ptr = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Src);
9568
9569 if (DestAS == AMDGPUAS::PRIVATE_ADDRESS &&
9570 Subtarget->hasGloballyAddressableScratch()) {
9571 // flat -> private with globally addressable scratch: subtract
9572 // src_flat_scratch_base_lo.
9573 SDValue FlatScratchBaseLo(
9574 DAG.getMachineNode(
9575 AMDGPU::S_MOV_B32, SL, MVT::i32,
9576 DAG.getRegister(AMDGPU::SRC_FLAT_SCRATCH_BASE_LO, MVT::i32)),
9577 0);
9578 Ptr = DAG.getNode(ISD::SUB, SL, MVT::i32, Ptr, FlatScratchBaseLo);
9579 }
9580
9581 if (IsNonNull || isKnownNonNull(Op, DAG, TM, SrcAS))
9582 return Ptr;
9583
9584 unsigned NullVal = AMDGPU::getNullPointerValue(DestAS);
9585 SDValue SegmentNullPtr = DAG.getConstant(NullVal, SL, MVT::i32);
9586 SDValue NonNull = DAG.getSetCC(SL, MVT::i1, Src, FlatNullPtr, ISD::SETNE);
9587
9588 return DAG.getNode(ISD::SELECT, SL, MVT::i32, NonNull, Ptr,
9589 SegmentNullPtr);
9590 }
9591 }
9592
9593 // local/private/barrier/vgpr -> flat
9594 if (DestAS == AMDGPUAS::FLAT_ADDRESS) {
9595 if (SrcAS == AMDGPUAS::LOCAL_ADDRESS ||
9596 SrcAS == AMDGPUAS::PRIVATE_ADDRESS || SrcAS == AMDGPUAS::BARRIER ||
9597 SrcAS == AMDGPUAS::VGPR) {
9598 SDValue CvtPtr;
9599 if (SrcAS == AMDGPUAS::PRIVATE_ADDRESS &&
9600 Subtarget->hasGloballyAddressableScratch()) {
9601 // For wave32: Addr = (TID[4:0] << 52) + FLAT_SCRATCH_BASE + privateAddr
9602 // For wave64: Addr = (TID[5:0] << 51) + FLAT_SCRATCH_BASE + privateAddr
9603 SDValue AllOnes = DAG.getSignedTargetConstant(-1, SL, MVT::i32);
9604 SDValue ThreadID = DAG.getConstant(0, SL, MVT::i32);
9605 ThreadID = DAG.getNode(
9606 ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32,
9607 DAG.getTargetConstant(Intrinsic::amdgcn_mbcnt_lo, SL, MVT::i32),
9608 AllOnes, ThreadID);
9609 if (Subtarget->isWave64())
9610 ThreadID = DAG.getNode(
9611 ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32,
9612 DAG.getTargetConstant(Intrinsic::amdgcn_mbcnt_hi, SL, MVT::i32),
9613 AllOnes, ThreadID);
9614 SDValue ShAmt = DAG.getShiftAmountConstant(
9615 57 - 32 - Subtarget->getWavefrontSizeLog2(), MVT::i32, SL);
9616 SDValue SrcHi = DAG.getNode(ISD::SHL, SL, MVT::i32, ThreadID, ShAmt);
9617 CvtPtr = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, Src, SrcHi);
9618 CvtPtr = DAG.getNode(ISD::BITCAST, SL, MVT::i64, CvtPtr);
9619 // Accessing src_flat_scratch_base_lo as a 64-bit operand gives the full
9620 // 64-bit hi:lo value.
9621 SDValue FlatScratchBase = {
9622 DAG.getMachineNode(
9623 AMDGPU::S_MOV_B64, SL, MVT::i64,
9624 DAG.getRegister(AMDGPU::SRC_FLAT_SCRATCH_BASE, MVT::i64)),
9625 0};
9626 CvtPtr = DAG.getNode(ISD::ADD, SL, MVT::i64, CvtPtr, FlatScratchBase);
9627 } else {
9628 SDValue Aperture = getSegmentAperture(SrcAS, SL, DAG);
9629
9630 CvtPtr = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, Src, Aperture);
9631 CvtPtr = DAG.getNode(ISD::BITCAST, SL, MVT::i64, CvtPtr);
9632 }
9633
9634 if (IsNonNull || isKnownNonNull(Op, DAG, TM, SrcAS))
9635 return CvtPtr;
9636
9637 unsigned NullVal = AMDGPU::getNullPointerValue(SrcAS);
9638 SDValue SegmentNullPtr = DAG.getConstant(NullVal, SL, MVT::i32);
9639
9640 SDValue NonNull =
9641 DAG.getSetCC(SL, MVT::i1, Src, SegmentNullPtr, ISD::SETNE);
9642
9643 return DAG.getNode(ISD::SELECT, SL, MVT::i64, NonNull, CvtPtr,
9644 FlatNullPtr);
9645 }
9646 }
9647
9648 if (SrcAS == AMDGPUAS::CONSTANT_ADDRESS_32BIT &&
9649 Op.getValueType() == MVT::i64) {
9650 const SIMachineFunctionInfo *Info =
9651 DAG.getMachineFunction().getInfo<SIMachineFunctionInfo>();
9652 if (Info->get32BitAddressHighBits() == 0)
9653 return DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i64, Src);
9654
9655 SDValue Hi = DAG.getConstant(Info->get32BitAddressHighBits(), SL, MVT::i32);
9656 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, Src, Hi);
9657 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec);
9658 }
9659
9660 if (DestAS == AMDGPUAS::CONSTANT_ADDRESS_32BIT &&
9661 Src.getValueType() == MVT::i64)
9662 return DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Src);
9663
9664 // global <-> flat are no-ops and never emitted.
9665
9666 // Invalid casts are poison.
9667 return DAG.getPOISON(Op->getValueType(0));
9668}
9669
9670// This lowers an INSERT_SUBVECTOR by extracting the individual elements from
9671// the small vector and inserting them into the big vector. That is better than
9672// the default expansion of doing it via a stack slot. Even though the use of
9673// the stack slot would be optimized away afterwards, the stack slot itself
9674// remains.
9675SDValue SITargetLowering::lowerINSERT_SUBVECTOR(SDValue Op,
9676 SelectionDAG &DAG) const {
9677 SDValue Vec = Op.getOperand(0);
9678 SDValue Ins = Op.getOperand(1);
9679 SDValue Idx = Op.getOperand(2);
9680 EVT VecVT = Vec.getValueType();
9681 EVT InsVT = Ins.getValueType();
9682 EVT EltVT = VecVT.getVectorElementType();
9683 unsigned InsNumElts = InsVT.getVectorNumElements();
9684 unsigned IdxVal = Idx->getAsZExtVal();
9685 SDLoc SL(Op);
9686
9687 if (EltVT.getScalarSizeInBits() == 16 && IdxVal % 2 == 0) {
9688 // Insert 32-bit registers at a time.
9689 assert(InsNumElts % 2 == 0 && "expect legal vector types");
9690
9691 unsigned VecNumElts = VecVT.getVectorNumElements();
9692 EVT NewVecVT =
9693 EVT::getVectorVT(*DAG.getContext(), MVT::i32, VecNumElts / 2);
9694 EVT NewInsVT = InsNumElts == 2 ? MVT::i32
9696 MVT::i32, InsNumElts / 2);
9697
9698 Vec = DAG.getNode(ISD::BITCAST, SL, NewVecVT, Vec);
9699 Ins = DAG.getNode(ISD::BITCAST, SL, NewInsVT, Ins);
9700
9701 for (unsigned I = 0; I != InsNumElts / 2; ++I) {
9702 SDValue Elt;
9703 if (InsNumElts == 2) {
9704 Elt = Ins;
9705 } else {
9706 Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Ins,
9707 DAG.getConstant(I, SL, MVT::i32));
9708 }
9709 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NewVecVT, Vec, Elt,
9710 DAG.getConstant(IdxVal / 2 + I, SL, MVT::i32));
9711 }
9712
9713 return DAG.getNode(ISD::BITCAST, SL, VecVT, Vec);
9714 }
9715
9716 for (unsigned I = 0; I != InsNumElts; ++I) {
9717 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Ins,
9718 DAG.getConstant(I, SL, MVT::i32));
9719 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, VecVT, Vec, Elt,
9720 DAG.getConstant(IdxVal + I, SL, MVT::i32));
9721 }
9722 return Vec;
9723}
9724
9725SDValue SITargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
9726 SelectionDAG &DAG) const {
9727 SDValue Vec = Op.getOperand(0);
9728 SDValue InsVal = Op.getOperand(1);
9729 SDValue Idx = Op.getOperand(2);
9730 EVT VecVT = Vec.getValueType();
9731 EVT EltVT = VecVT.getVectorElementType();
9732 unsigned VecSize = VecVT.getSizeInBits();
9733 unsigned EltSize = EltVT.getSizeInBits();
9734 SDLoc SL(Op);
9735
9736 // Specially handle the case of v4i16 with static indexing.
9737 unsigned NumElts = VecVT.getVectorNumElements();
9738 auto *KIdx = dyn_cast<ConstantSDNode>(Idx);
9739 if (NumElts == 4 && EltSize == 16 && KIdx) {
9740 SDValue BCVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Vec);
9741
9742 SDValue LoHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BCVec,
9743 DAG.getConstant(0, SL, MVT::i32));
9744 SDValue HiHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BCVec,
9745 DAG.getConstant(1, SL, MVT::i32));
9746
9747 SDValue LoVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, LoHalf);
9748 SDValue HiVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, HiHalf);
9749
9750 unsigned Idx = KIdx->getZExtValue();
9751 bool InsertLo = Idx < 2;
9752 SDValue InsHalf = DAG.getNode(
9753 ISD::INSERT_VECTOR_ELT, SL, MVT::v2i16, InsertLo ? LoVec : HiVec,
9754 DAG.getNode(ISD::BITCAST, SL, MVT::i16, InsVal),
9755 DAG.getConstant(InsertLo ? Idx : (Idx - 2), SL, MVT::i32));
9756
9757 InsHalf = DAG.getNode(ISD::BITCAST, SL, MVT::i32, InsHalf);
9758
9759 SDValue Concat =
9760 InsertLo ? DAG.getBuildVector(MVT::v2i32, SL, {InsHalf, HiHalf})
9761 : DAG.getBuildVector(MVT::v2i32, SL, {LoHalf, InsHalf});
9762
9763 return DAG.getNode(ISD::BITCAST, SL, VecVT, Concat);
9764 }
9765
9766 // Static indexing does not lower to stack access, and hence there is no need
9767 // for special custom lowering to avoid stack access.
9768 if (isa<ConstantSDNode>(Idx))
9769 return SDValue();
9770
9771 // Avoid stack access for dynamic indexing by custom lowering to
9772 // v_bfi_b32 (v_bfm_b32 16, (shl idx, 16)), val, vec
9773 // Vectors wider than 64 bits do this on each 64-bit piece.
9774 assert(isPowerOf2_32(EltSize));
9775 unsigned PieceSize = std::min(VecSize, 64u);
9776 unsigned NumPieces = VecSize / PieceSize;
9777 unsigned EltsPerPiece = PieceSize / EltSize;
9778 MVT IntVT = MVT::getIntegerVT(PieceSize);
9779 EVT PieceVT = EVT::getVectorVT(*DAG.getContext(), EltVT, EltsPerPiece);
9780
9781 // Convert vector index to bit-index within its piece and get the required
9782 // bit mask. Using the index within the piece keeps the shift below 64.
9783 SDValue PieceIdx = Idx;
9784 if (NumPieces > 1)
9785 PieceIdx = DAG.getNode(ISD::AND, SL, MVT::i32, Idx,
9786 DAG.getConstant(EltsPerPiece - 1, SL, MVT::i32));
9787 const auto EltMask = maskTrailingOnes<uint64_t>(EltSize);
9788 SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32);
9789 SDValue ScaledIdx =
9790 DAG.getNode(ISD::SHL, SL, MVT::i32, PieceIdx, ScaleFactor);
9791 SDValue BFM = DAG.getNode(ISD::SHL, SL, IntVT,
9792 DAG.getConstant(EltMask, SL, IntVT), ScaledIdx);
9793
9794 // 1. Create a congruent vector with the target value in each element.
9795 SDValue ExtVal =
9796 DAG.getBitcast(IntVT, DAG.getSplatBuildVector(PieceVT, SL, InsVal));
9797
9798 auto InsertIntoPiece = [&](SDValue Piece, SDValue Mask) {
9799 // 2. Mask off all other indices except the required index within (1).
9800 SDValue LHS = DAG.getNode(ISD::AND, SL, IntVT, Mask, ExtVal);
9801
9802 // 3. Mask off the required index within the target vector.
9803 SDValue RHS =
9804 DAG.getNode(ISD::AND, SL, IntVT, DAG.getNOT(SL, Mask, IntVT), Piece);
9805
9806 // 4. Get (2) and (3) ORed into the target vector.
9807 return DAG.getNode(ISD::OR, SL, IntVT, LHS, RHS, SDNodeFlags::Disjoint);
9808 };
9809
9810 if (NumPieces == 1)
9811 return DAG.getBitcast(VecVT,
9812 InsertIntoPiece(DAG.getBitcast(IntVT, Vec), BFM));
9813
9814 // 5. Apply (2)-(4) to each 64-bit piece, with the mask cleared in every
9815 // piece except the one that holds the index, and rebuild the vector.
9816 MVT PiecesVT = MVT::getVectorVT(IntVT, NumPieces);
9817 SDValue BCVec = DAG.getBitcast(PiecesVT, Vec);
9818 SDValue PieceNo =
9819 DAG.getNode(ISD::SRL, SL, MVT::i32, Idx,
9820 DAG.getConstant(Log2_32(EltsPerPiece), SL, MVT::i32));
9821 SDValue Zero = DAG.getConstant(0, SL, IntVT);
9823 Pieces.reserve(NumPieces);
9824 for (unsigned I = 0; I != NumPieces; ++I) {
9825 SDValue CurPieceNo = DAG.getConstant(I, SL, MVT::i32);
9826 SDValue Piece =
9827 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, IntVT, BCVec, CurPieceNo);
9828 SDValue Mask =
9829 DAG.getSelectCC(SL, PieceNo, CurPieceNo, BFM, Zero, ISD::SETEQ);
9830 Pieces.push_back(InsertIntoPiece(Piece, Mask));
9831 }
9832
9833 return DAG.getBitcast(VecVT, DAG.getBuildVector(PiecesVT, SL, Pieces));
9834}
9835
9836SDValue SITargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
9837 SelectionDAG &DAG) const {
9838 SDLoc SL(Op);
9839
9840 EVT ResultVT = Op.getValueType();
9841 SDValue Vec = Op.getOperand(0);
9842 SDValue Idx = Op.getOperand(1);
9843 EVT VecVT = Vec.getValueType();
9844 unsigned VecSize = VecVT.getSizeInBits();
9845 EVT EltVT = VecVT.getVectorElementType();
9846
9847 DAGCombinerInfo DCI(DAG, AfterLegalizeVectorOps, true, nullptr);
9848
9849 // Make sure we do any optimizations that will make it easier to fold
9850 // source modifiers before obscuring it with bit operations.
9851
9852 // XXX - Why doesn't this get called when vector_shuffle is expanded?
9853 if (SDValue Combined = performExtractVectorEltCombine(Op.getNode(), DCI))
9854 return Combined;
9855
9856 if (VecSize == 128 || VecSize == 256 || VecSize == 512) {
9857 SDValue Lo, Hi;
9858 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VecVT);
9859
9860 if (VecSize == 128) {
9861 SDValue V2 = DAG.getBitcast(MVT::v2i64, Vec);
9862 Lo = DAG.getBitcast(LoVT,
9863 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2,
9864 DAG.getConstant(0, SL, MVT::i32)));
9865 Hi = DAG.getBitcast(HiVT,
9866 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2,
9867 DAG.getConstant(1, SL, MVT::i32)));
9868 } else if (VecSize == 256) {
9869 SDValue V2 = DAG.getBitcast(MVT::v4i64, Vec);
9870 SDValue Parts[4];
9871 for (unsigned P = 0; P < 4; ++P) {
9872 Parts[P] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2,
9873 DAG.getConstant(P, SL, MVT::i32));
9874 }
9875
9876 Lo = DAG.getBitcast(LoVT, DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i64,
9877 Parts[0], Parts[1]));
9878 Hi = DAG.getBitcast(HiVT, DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i64,
9879 Parts[2], Parts[3]));
9880 } else {
9881 assert(VecSize == 512);
9882
9883 SDValue V2 = DAG.getBitcast(MVT::v8i64, Vec);
9884 SDValue Parts[8];
9885 for (unsigned P = 0; P < 8; ++P) {
9886 Parts[P] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2,
9887 DAG.getConstant(P, SL, MVT::i32));
9888 }
9889
9890 Lo = DAG.getBitcast(LoVT,
9891 DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v4i64,
9892 Parts[0], Parts[1], Parts[2], Parts[3]));
9893 Hi = DAG.getBitcast(HiVT,
9894 DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v4i64,
9895 Parts[4], Parts[5], Parts[6], Parts[7]));
9896 }
9897
9898 EVT IdxVT = Idx.getValueType();
9899 unsigned NElem = VecVT.getVectorNumElements();
9900 assert(isPowerOf2_32(NElem));
9901 SDValue IdxMask = DAG.getConstant(NElem / 2 - 1, SL, IdxVT);
9902 SDValue NewIdx = DAG.getNode(ISD::AND, SL, IdxVT, Idx, IdxMask);
9903 SDValue Half = DAG.getSelectCC(SL, Idx, IdxMask, Hi, Lo, ISD::SETUGT);
9904 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Half, NewIdx);
9905 }
9906
9907 assert(VecSize <= 64);
9908
9909 MVT IntVT = MVT::getIntegerVT(VecSize);
9910
9911 // If Vec is just a SCALAR_TO_VECTOR, then use the scalar integer directly.
9912 SDValue VecBC = peekThroughBitcasts(Vec);
9913 if (VecBC.getOpcode() == ISD::SCALAR_TO_VECTOR) {
9914 SDValue Src = VecBC.getOperand(0);
9915 Src = DAG.getBitcast(Src.getValueType().changeTypeToInteger(), Src);
9916 Vec = DAG.getAnyExtOrTrunc(Src, SL, IntVT);
9917 }
9918
9919 unsigned EltSize = EltVT.getSizeInBits();
9920 assert(isPowerOf2_32(EltSize));
9921
9922 SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32);
9923
9924 // Convert vector index to bit-index (* EltSize)
9925 SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx, ScaleFactor);
9926
9927 SDValue BC = DAG.getNode(ISD::BITCAST, SL, IntVT, Vec);
9928 SDValue Elt = DAG.getNode(ISD::SRL, SL, IntVT, BC, ScaledIdx);
9929
9930 if (ResultVT == MVT::f16 || ResultVT == MVT::bf16) {
9931 SDValue Result = DAG.getNode(ISD::TRUNCATE, SL, MVT::i16, Elt);
9932 return DAG.getNode(ISD::BITCAST, SL, ResultVT, Result);
9933 }
9934
9935 return DAG.getAnyExtOrTrunc(Elt, SL, ResultVT);
9936}
9937
9938static bool elementPairIsContiguous(ArrayRef<int> Mask, int Elt) {
9939 assert(Elt % 2 == 0);
9940 return Mask[Elt + 1] == Mask[Elt] + 1 && (Mask[Elt] % 2 == 0);
9941}
9942
9943static bool elementPairIsOddToEven(ArrayRef<int> Mask, int Elt) {
9944 assert(Elt % 2 == 0);
9945 return Mask[Elt] >= 0 && Mask[Elt + 1] >= 0 && (Mask[Elt] & 1) &&
9946 !(Mask[Elt + 1] & 1);
9947}
9948
9949SDValue SITargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
9950 SelectionDAG &DAG) const {
9951 SDLoc SL(Op);
9952 EVT ResultVT = Op.getValueType();
9953 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
9954 MVT EltVT = ResultVT.getVectorElementType().getSimpleVT();
9955 const int NewSrcNumElts = 2;
9956 MVT PackVT = MVT::getVectorVT(EltVT, NewSrcNumElts);
9957 int SrcNumElts = Op.getOperand(0).getValueType().getVectorNumElements();
9958
9959 // Break up the shuffle into registers sized pieces.
9960 //
9961 // We're trying to form sub-shuffles that the register allocation pipeline
9962 // won't be able to figure out, like how to use v_pk_mov_b32 to do a register
9963 // blend or 16-bit op_sel. It should be able to figure out how to reassemble a
9964 // pair of copies into a consecutive register copy, so use the ordinary
9965 // extract_vector_elt lowering unless we can use the shuffle.
9966 //
9967 // TODO: This is a bit of hack, and we should probably always use
9968 // extract_subvector for the largest possible subvector we can (or at least
9969 // use it for PackVT aligned pieces). However we have worse support for
9970 // combines on them don't directly treat extract_subvector / insert_subvector
9971 // as legal. The DAG scheduler also ends up doing a worse job with the
9972 // extract_subvectors.
9973 const bool ShouldUseConsecutiveExtract = EltVT.getSizeInBits() == 16;
9974
9975 // vector_shuffle <0,1,6,7> lhs, rhs
9976 // -> concat_vectors (extract_subvector lhs, 0), (extract_subvector rhs, 2)
9977 //
9978 // vector_shuffle <6,7,2,3> lhs, rhs
9979 // -> concat_vectors (extract_subvector rhs, 2), (extract_subvector lhs, 2)
9980 //
9981 // vector_shuffle <6,7,0,1> lhs, rhs
9982 // -> concat_vectors (extract_subvector rhs, 2), (extract_subvector lhs, 0)
9983
9984 // Avoid scalarizing when both halves are reading from consecutive elements.
9985
9986 // If we're treating 2 element shuffles as legal, also create odd-to-even
9987 // shuffles of neighboring pairs.
9988 //
9989 // vector_shuffle <3,2,7,6> lhs, rhs
9990 // -> concat_vectors vector_shuffle <1, 0> (extract_subvector lhs, 0)
9991 // vector_shuffle <1, 0> (extract_subvector rhs, 2)
9992
9994 for (int I = 0, N = ResultVT.getVectorNumElements(); I != N; I += 2) {
9995 if (ShouldUseConsecutiveExtract &&
9997 const int Idx = SVN->getMaskElt(I);
9998 int VecIdx = Idx < SrcNumElts ? 0 : 1;
9999 int EltIdx = Idx < SrcNumElts ? Idx : Idx - SrcNumElts;
10000 SDValue SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, PackVT,
10001 SVN->getOperand(VecIdx),
10002 DAG.getConstant(EltIdx, SL, MVT::i32));
10003 Pieces.push_back(SubVec);
10004 } else if (elementPairIsOddToEven(SVN->getMask(), I) &&
10006 int Idx0 = SVN->getMaskElt(I);
10007 int Idx1 = SVN->getMaskElt(I + 1);
10008
10009 SDValue SrcOp0 = SVN->getOperand(0);
10010 SDValue SrcOp1 = SrcOp0;
10011 if (Idx0 >= SrcNumElts) {
10012 SrcOp0 = SVN->getOperand(1);
10013 Idx0 -= SrcNumElts;
10014 }
10015
10016 if (Idx1 >= SrcNumElts) {
10017 SrcOp1 = SVN->getOperand(1);
10018 Idx1 -= SrcNumElts;
10019 }
10020
10021 int AlignedIdx0 = Idx0 & ~(NewSrcNumElts - 1);
10022 int AlignedIdx1 = Idx1 & ~(NewSrcNumElts - 1);
10023
10024 // Extract nearest even aligned piece.
10025 SDValue SubVec0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, PackVT, SrcOp0,
10026 DAG.getConstant(AlignedIdx0, SL, MVT::i32));
10027 SDValue SubVec1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, PackVT, SrcOp1,
10028 DAG.getConstant(AlignedIdx1, SL, MVT::i32));
10029
10030 int NewMaskIdx0 = Idx0 - AlignedIdx0;
10031 int NewMaskIdx1 = Idx1 - AlignedIdx1;
10032
10033 SDValue Result0 = SubVec0;
10034 SDValue Result1 = SubVec0;
10035
10036 if (SubVec0 != SubVec1) {
10037 NewMaskIdx1 += NewSrcNumElts;
10038 Result1 = SubVec1;
10039 } else {
10040 Result1 = DAG.getPOISON(PackVT);
10041 }
10042
10043 SDValue Shuf = DAG.getVectorShuffle(PackVT, SL, Result0, Result1,
10044 {NewMaskIdx0, NewMaskIdx1});
10045 Pieces.push_back(Shuf);
10046 } else {
10047 const int Idx0 = SVN->getMaskElt(I);
10048 const int Idx1 = SVN->getMaskElt(I + 1);
10049 int VecIdx0 = Idx0 < SrcNumElts ? 0 : 1;
10050 int VecIdx1 = Idx1 < SrcNumElts ? 0 : 1;
10051 int EltIdx0 = Idx0 < SrcNumElts ? Idx0 : Idx0 - SrcNumElts;
10052 int EltIdx1 = Idx1 < SrcNumElts ? Idx1 : Idx1 - SrcNumElts;
10053
10054 SDValue Vec0 = SVN->getOperand(VecIdx0);
10055 SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Vec0,
10056 DAG.getSignedConstant(EltIdx0, SL, MVT::i32));
10057
10058 SDValue Vec1 = SVN->getOperand(VecIdx1);
10059 SDValue Elt1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Vec1,
10060 DAG.getSignedConstant(EltIdx1, SL, MVT::i32));
10061 Pieces.push_back(DAG.getBuildVector(PackVT, SL, {Elt0, Elt1}));
10062 }
10063 }
10064
10065 return DAG.getNode(ISD::CONCAT_VECTORS, SL, ResultVT, Pieces);
10066}
10067
10068SDValue SITargetLowering::lowerSCALAR_TO_VECTOR(SDValue Op,
10069 SelectionDAG &DAG) const {
10070 SDValue SVal = Op.getOperand(0);
10071 EVT ResultVT = Op.getValueType();
10072 EVT SValVT = SVal.getValueType();
10073 SDValue UndefVal = DAG.getPOISON(SValVT);
10074 SDLoc SL(Op);
10075
10077 VElts.push_back(SVal);
10078 for (int I = 1, E = ResultVT.getVectorNumElements(); I < E; ++I)
10079 VElts.push_back(UndefVal);
10080
10081 return DAG.getBuildVector(ResultVT, SL, VElts);
10082}
10083
10084SDValue SITargetLowering::lowerBUILD_VECTOR(SDValue Op,
10085 SelectionDAG &DAG) const {
10086 SDLoc SL(Op);
10087 EVT VT = Op.getValueType();
10088
10089 if (VT == MVT::v2f16 || VT == MVT::v2i16 || VT == MVT::v2bf16) {
10090 assert(!Subtarget->hasVOP3PInsts() && "this should be legal");
10091
10092 SDValue Lo = Op.getOperand(0);
10093 SDValue Hi = Op.getOperand(1);
10094
10095 // Avoid adding defined bits with the zero_extend.
10096 if (Hi.isUndef()) {
10097 Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Lo);
10098 SDValue ExtLo = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Lo);
10099 return DAG.getNode(ISD::BITCAST, SL, VT, ExtLo);
10100 }
10101
10102 Hi = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Hi);
10103 Hi = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Hi);
10104
10105 SDValue ShlHi = DAG.getNode(ISD::SHL, SL, MVT::i32, Hi,
10106 DAG.getConstant(16, SL, MVT::i32));
10107 if (Lo.isUndef())
10108 return DAG.getNode(ISD::BITCAST, SL, VT, ShlHi);
10109
10110 Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Lo);
10111 Lo = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Lo);
10112
10113 SDValue Or =
10114 DAG.getNode(ISD::OR, SL, MVT::i32, Lo, ShlHi, SDNodeFlags::Disjoint);
10115 return DAG.getNode(ISD::BITCAST, SL, VT, Or);
10116 }
10117
10118 // Split into 2-element chunks.
10119 const unsigned NumParts = VT.getVectorNumElements() / 2;
10120 EVT PartVT = MVT::getVectorVT(VT.getVectorElementType().getSimpleVT(), 2);
10121 MVT PartIntVT = MVT::getIntegerVT(PartVT.getSizeInBits());
10122
10124 for (unsigned P = 0; P < NumParts; ++P) {
10125 SDValue Vec = DAG.getBuildVector(
10126 PartVT, SL, {Op.getOperand(P * 2), Op.getOperand(P * 2 + 1)});
10127 Casts.push_back(DAG.getNode(ISD::BITCAST, SL, PartIntVT, Vec));
10128 }
10129
10130 SDValue Blend =
10131 DAG.getBuildVector(MVT::getVectorVT(PartIntVT, NumParts), SL, Casts);
10132 return DAG.getNode(ISD::BITCAST, SL, VT, Blend);
10133}
10134
10136 const GlobalAddressSDNode *GA) const {
10137 // Named barriers have fixed, non-relocated LDS addresses, so a constant
10138 // offset into an array of them can be folded into the address.
10140 const auto *GV = dyn_cast<GlobalVariable>(GA->getGlobal());
10141 return GV && AMDGPU::isNamedBarrier(*GV);
10142 }
10143
10144 // OSes that use ELF REL relocations (instead of RELA) can only store a
10145 // 32-bit addend in the instruction, so it is not safe to allow offset folding
10146 // which can create arbitrary 64-bit addends. (This is only a problem for
10147 // R_AMDGPU_*32_HI relocations since other relocation types are unaffected by
10148 // the high 32 bits of the addend.)
10149 //
10150 // This should be kept in sync with how HasRelocationAddend is initialized in
10151 // the constructor of ELFAMDGPUAsmBackend.
10152 if (!Subtarget->isAmdHsaOS())
10153 return false;
10154
10155 // We can fold offsets for anything that doesn't require a GOT relocation.
10156 return (GA->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS ||
10160}
10161
10162static SDValue
10164 const SDLoc &DL, int64_t Offset, EVT PtrVT,
10165 unsigned GAFlags = SIInstrInfo::MO_NONE) {
10166 assert(isInt<32>(Offset + 4) && "32-bit offset is expected!");
10167 // In order to support pc-relative addressing, the PC_ADD_REL_OFFSET SDNode is
10168 // lowered to the following code sequence:
10169 //
10170 // For constant address space:
10171 // s_getpc_b64 s[0:1]
10172 // s_add_u32 s0, s0, $symbol
10173 // s_addc_u32 s1, s1, 0
10174 //
10175 // s_getpc_b64 returns the address of the s_add_u32 instruction and then
10176 // a fixup or relocation is emitted to replace $symbol with a literal
10177 // constant, which is a pc-relative offset from the encoding of the $symbol
10178 // operand to the global variable.
10179 //
10180 // For global address space:
10181 // s_getpc_b64 s[0:1]
10182 // s_add_u32 s0, s0, $symbol@{gotpc}rel32@lo
10183 // s_addc_u32 s1, s1, $symbol@{gotpc}rel32@hi
10184 //
10185 // s_getpc_b64 returns the address of the s_add_u32 instruction and then
10186 // fixups or relocations are emitted to replace $symbol@*@lo and
10187 // $symbol@*@hi with lower 32 bits and higher 32 bits of a literal constant,
10188 // which is a 64-bit pc-relative offset from the encoding of the $symbol
10189 // operand to the global variable.
10190 if (((const GCNSubtarget &)DAG.getSubtarget()).has64BitLiterals()) {
10191 assert(GAFlags != SIInstrInfo::MO_NONE);
10192
10193 SDValue Ptr =
10194 DAG.getTargetGlobalAddress(GV, DL, MVT::i64, Offset, GAFlags + 2);
10195 return DAG.getNode(AMDGPUISD::PC_ADD_REL_OFFSET64, DL, PtrVT, Ptr);
10196 }
10197
10198 SDValue PtrLo = DAG.getTargetGlobalAddress(GV, DL, MVT::i32, Offset, GAFlags);
10199 SDValue PtrHi;
10200 if (GAFlags == SIInstrInfo::MO_NONE)
10201 PtrHi = DAG.getTargetConstant(0, DL, MVT::i32);
10202 else
10203 PtrHi = DAG.getTargetGlobalAddress(GV, DL, MVT::i32, Offset, GAFlags + 1);
10204 return DAG.getNode(AMDGPUISD::PC_ADD_REL_OFFSET, DL, PtrVT, PtrLo, PtrHi);
10205}
10206
10207SDValue SITargetLowering::LowerGlobalAddress(AMDGPUMachineFunctionInfo *MFI,
10208 SDValue Op,
10209 SelectionDAG &DAG) const {
10210 GlobalAddressSDNode *GSD = cast<GlobalAddressSDNode>(Op);
10211 SDLoc DL(GSD);
10212 EVT PtrVT = Op.getValueType();
10213
10214 const GlobalValue *GV = GSD->getGlobal();
10215 const unsigned AS = GSD->getAddressSpace();
10216 if (((AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::BARRIER) &&
10219 if (AS == AMDGPUAS::LOCAL_ADDRESS && GV->hasExternalLinkage()) {
10220 const GlobalVariable &GVar = *cast<GlobalVariable>(GV);
10221 // HIP uses an unsized array `extern __shared__ T s[]` or similar
10222 // zero-sized type in other languages to declare the dynamic shared
10223 // memory which size is not known at the compile time. They will be
10224 // allocated by the runtime and placed directly after the static
10225 // allocated ones. They all share the same offset.
10226 if (GVar.getGlobalSize(GVar.getDataLayout()) == 0) {
10227 assert(PtrVT == MVT::i32 && "32-bit pointer is expected.");
10228 // Adjust alignment for that dynamic shared memory array.
10230 MFI->setDynLDSAlign(F, GVar);
10231 MFI->setUsesDynamicLDS(true);
10232 return SDValue(
10233 DAG.getMachineNode(AMDGPU::GET_GROUPSTATICSIZE, DL, PtrVT), 0);
10234 }
10235 }
10237 }
10238
10239 if (AS == AMDGPUAS::BARRIER) {
10240 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, MVT::i32, GSD->getOffset(),
10242 return SDValue(DAG.getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, GA), 0);
10243 }
10244
10245 if (AS == AMDGPUAS::LOCAL_ADDRESS) {
10246 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, MVT::i32, GSD->getOffset(),
10248 return DAG.getNode(AMDGPUISD::LDS, DL, MVT::i32, GA);
10249 }
10250
10251 if (Subtarget->isAmdPalOS() || Subtarget->isMesa3DOS()) {
10252 if (Subtarget->has64BitLiterals()) {
10254 GV, DL, MVT::i64, GSD->getOffset(), SIInstrInfo::MO_ABS64);
10255 return SDValue(DAG.getMachineNode(AMDGPU::S_MOV_B64, DL, MVT::i64, Addr),
10256 0);
10257 }
10258
10259 SDValue AddrLo = DAG.getTargetGlobalAddress(
10260 GV, DL, MVT::i32, GSD->getOffset(), SIInstrInfo::MO_ABS32_LO);
10261 AddrLo = {DAG.getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, AddrLo), 0};
10262
10263 SDValue AddrHi = DAG.getTargetGlobalAddress(
10264 GV, DL, MVT::i32, GSD->getOffset(), SIInstrInfo::MO_ABS32_HI);
10265 AddrHi = {DAG.getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, AddrHi), 0};
10266
10267 return DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, AddrLo, AddrHi);
10268 }
10269
10270 if (shouldEmitFixup(GV))
10271 return buildPCRelGlobalAddress(DAG, GV, DL, GSD->getOffset(), PtrVT);
10272
10273 if (shouldEmitPCReloc(GV))
10274 return buildPCRelGlobalAddress(DAG, GV, DL, GSD->getOffset(), PtrVT,
10276
10277 SDValue GOTAddr = buildPCRelGlobalAddress(DAG, GV, DL, 0, PtrVT,
10279 PointerType *PtrTy =
10281 const DataLayout &DataLayout = DAG.getDataLayout();
10282 Align Alignment = DataLayout.getABITypeAlign(PtrTy);
10283 MachinePointerInfo PtrInfo =
10285
10286 return DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), GOTAddr, PtrInfo, Alignment,
10289}
10290
10291// An alloca in the VGPR ("as memory") address space has no registers to live
10292// in, so its stack offset must not be taken for a register address. Every
10293// other frame index keeps its default lowering.
10294SDValue SITargetLowering::lowerFrameIndex(SDValue Op, SelectionDAG &DAG) const {
10296 int FI = cast<FrameIndexSDNode>(Op)->getIndex();
10297 const AllocaInst *Alloca = MF.getFrameInfo().getObjectAllocation(FI);
10298 if (!Alloca || Alloca->getAddressSpace() != AMDGPUAS::VGPR)
10299 return SDValue();
10300
10301 DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
10302 MF.getFunction(),
10303 "unsupported alloca in VGPR 'as memory' address space (13)",
10304 Alloca->getDebugLoc()));
10305 return DAG.getPOISON(MVT::i32);
10306}
10307
10308SDValue SITargetLowering::LowerExternalSymbol(SDValue Op,
10309 SelectionDAG &DAG) const {
10310 // TODO: Handle this. It should be mostly the same as LowerGlobalAddress.
10311 const Function &Fn = DAG.getMachineFunction().getFunction();
10312 DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
10313 Fn, "unsupported external symbol", Op.getDebugLoc()));
10314 return DAG.getPOISON(Op.getValueType());
10315}
10316
10318 const SDLoc &DL, SDValue V) const {
10319 // We can't use S_MOV_B32 directly, because there is no way to specify m0 as
10320 // the destination register.
10321 //
10322 // We can't use CopyToReg, because MachineCSE won't combine COPY instructions,
10323 // so we will end up with redundant moves to m0.
10324 //
10325 // We use a pseudo to ensure we emit s_mov_b32 with m0 as the direct result.
10326
10327 // A Null SDValue creates a glue result.
10328 SDNode *M0 = DAG.getMachineNode(AMDGPU::SI_INIT_M0, DL, MVT::Other, MVT::Glue,
10329 V, Chain);
10330 return SDValue(M0, 0);
10331}
10332
10333SDValue SITargetLowering::lowerImplicitZextParam(SelectionDAG &DAG, SDValue Op,
10334 MVT VT,
10335 unsigned Offset) const {
10336 SDLoc SL(Op);
10337 SDValue Param = lowerKernargMemParameter(
10338 DAG, MVT::i32, MVT::i32, SL, DAG.getEntryNode(), Offset, Align(4), false);
10339 // The local size values will have the hi 16-bits as zero.
10340 return DAG.getNode(ISD::AssertZext, SL, MVT::i32, Param,
10341 DAG.getValueType(VT));
10342}
10343
10345 EVT VT) {
10348 "non-hsa intrinsic with hsa target", DL.getDebugLoc()));
10349 return DAG.getPOISON(VT);
10350}
10351
10353 EVT VT) {
10356 "intrinsic not supported on subtarget", DL.getDebugLoc()));
10357 return DAG.getPOISON(VT);
10358}
10359
10361 ArrayRef<SDValue> Elts) {
10362 assert(!Elts.empty());
10363 MVT Type;
10364 unsigned NumElts = Elts.size();
10365
10366 if (NumElts <= 12) {
10367 Type = MVT::getVectorVT(MVT::f32, NumElts);
10368 } else {
10369 assert(Elts.size() <= 16);
10370 Type = MVT::v16f32;
10371 NumElts = 16;
10372 }
10373
10374 SmallVector<SDValue, 16> VecElts(NumElts);
10375 for (unsigned i = 0; i < Elts.size(); ++i) {
10376 SDValue Elt = Elts[i];
10377 if (Elt.getValueType() != MVT::f32)
10378 Elt = DAG.getBitcast(MVT::f32, Elt);
10379 VecElts[i] = Elt;
10380 }
10381 for (unsigned i = Elts.size(); i < NumElts; ++i)
10382 VecElts[i] = DAG.getPOISON(MVT::f32);
10383
10384 if (NumElts == 1)
10385 return VecElts[0];
10386 return DAG.getBuildVector(Type, DL, VecElts);
10387}
10388
10389static SDValue padEltsToUndef(SelectionDAG &DAG, const SDLoc &DL, EVT CastVT,
10390 SDValue Src, int ExtraElts) {
10391 EVT SrcVT = Src.getValueType();
10392
10394
10395 if (SrcVT.isVector())
10396 DAG.ExtractVectorElements(Src, Elts);
10397 else
10398 Elts.push_back(Src);
10399
10400 SDValue Undef = DAG.getPOISON(SrcVT.getScalarType());
10401 while (ExtraElts--)
10402 Elts.push_back(Undef);
10403
10404 return DAG.getBuildVector(CastVT, DL, Elts);
10405}
10406
10407// Re-construct the required return value for a image load intrinsic.
10408// This is more complicated due to the optional use TexFailCtrl which means the
10409// required return type is an aggregate
10411 ArrayRef<EVT> ResultTypes, bool IsTexFail,
10412 bool Unpacked, bool IsD16, int DMaskPop,
10413 int NumVDataDwords, bool IsAtomicPacked16Bit,
10414 const SDLoc &DL) {
10415 // Determine the required return type. This is the same regardless of
10416 // IsTexFail flag
10417 EVT ReqRetVT = ResultTypes[0];
10418 int ReqRetNumElts = ReqRetVT.isVector() ? ReqRetVT.getVectorNumElements() : 1;
10419 int NumDataDwords = ((IsD16 && !Unpacked) || IsAtomicPacked16Bit)
10420 ? (ReqRetNumElts + 1) / 2
10421 : ReqRetNumElts;
10422
10423 int MaskPopDwords = (!IsD16 || Unpacked) ? DMaskPop : (DMaskPop + 1) / 2;
10424
10425 MVT DataDwordVT =
10426 NumDataDwords == 1 ? MVT::i32 : MVT::getVectorVT(MVT::i32, NumDataDwords);
10427
10428 MVT MaskPopVT =
10429 MaskPopDwords == 1 ? MVT::i32 : MVT::getVectorVT(MVT::i32, MaskPopDwords);
10430
10431 SDValue Data(Result, 0);
10432 SDValue TexFail;
10433
10434 if (DMaskPop > 0 && Data.getValueType() != MaskPopVT) {
10435 SDValue ZeroIdx = DAG.getConstant(0, DL, MVT::i32);
10436 if (MaskPopVT.isVector()) {
10437 Data = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MaskPopVT,
10438 SDValue(Result, 0), ZeroIdx);
10439 } else {
10440 Data = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MaskPopVT,
10441 SDValue(Result, 0), ZeroIdx);
10442 }
10443 }
10444
10445 if (DataDwordVT.isVector() && !IsAtomicPacked16Bit)
10446 Data = padEltsToUndef(DAG, DL, DataDwordVT, Data,
10447 NumDataDwords - MaskPopDwords);
10448
10449 if (IsD16)
10450 Data = adjustLoadValueTypeImpl(Data, ReqRetVT, DL, DAG, Unpacked);
10451
10452 EVT LegalReqRetVT = ReqRetVT;
10453 if (!ReqRetVT.isVector()) {
10454 if (!Data.getValueType().isInteger())
10455 Data = DAG.getNode(ISD::BITCAST, DL,
10456 Data.getValueType().changeTypeToInteger(), Data);
10457 Data = DAG.getNode(ISD::TRUNCATE, DL, ReqRetVT.changeTypeToInteger(), Data);
10458 } else {
10459 // We need to widen the return vector to a legal type
10460 if ((ReqRetVT.getVectorNumElements() % 2) == 1 &&
10461 ReqRetVT.getVectorElementType().getSizeInBits() == 16) {
10462 LegalReqRetVT =
10464 ReqRetVT.getVectorNumElements() + 1);
10465 }
10466 }
10467 Data = DAG.getNode(ISD::BITCAST, DL, LegalReqRetVT, Data);
10468
10469 if (IsTexFail) {
10470 TexFail =
10471 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, SDValue(Result, 0),
10472 DAG.getConstant(MaskPopDwords, DL, MVT::i32));
10473
10474 return DAG.getMergeValues({Data, TexFail, SDValue(Result, 1)}, DL);
10475 }
10476
10477 if (Result->getNumValues() == 1)
10478 return Data;
10479
10480 return DAG.getMergeValues({Data, SDValue(Result, 1)}, DL);
10481}
10482
10483static bool parseTexFail(SDValue TexFailCtrl, SelectionDAG &DAG, SDValue *TFE,
10484 SDValue *LWE, bool &IsTexFail) {
10485 auto *TexFailCtrlConst = cast<ConstantSDNode>(TexFailCtrl.getNode());
10486
10487 uint64_t Value = TexFailCtrlConst->getZExtValue();
10488 if (Value) {
10489 IsTexFail = true;
10490 }
10491
10492 SDLoc DL(TexFailCtrlConst);
10493 *TFE = DAG.getTargetConstant((Value & 0x1) ? 1 : 0, DL, MVT::i32);
10494 Value &= ~(uint64_t)0x1;
10495 *LWE = DAG.getTargetConstant((Value & 0x2) ? 1 : 0, DL, MVT::i32);
10496 Value &= ~(uint64_t)0x2;
10497
10498 return Value == 0;
10499}
10500
10502 MVT PackVectorVT,
10503 SmallVectorImpl<SDValue> &PackedAddrs,
10504 unsigned DimIdx, unsigned EndIdx,
10505 unsigned NumGradients) {
10506 SDLoc DL(Op);
10507 for (unsigned I = DimIdx; I < EndIdx; I++) {
10508 SDValue Addr = Op.getOperand(I);
10509
10510 // Gradients are packed with undef for each coordinate.
10511 // In <hi 16 bit>,<lo 16 bit> notation, the registers look like this:
10512 // 1D: undef,dx/dh; undef,dx/dv
10513 // 2D: dy/dh,dx/dh; dy/dv,dx/dv
10514 // 3D: dy/dh,dx/dh; undef,dz/dh; dy/dv,dx/dv; undef,dz/dv
10515 if (((I + 1) >= EndIdx) ||
10516 ((NumGradients / 2) % 2 == 1 && (I == DimIdx + (NumGradients / 2) - 1 ||
10517 I == DimIdx + NumGradients - 1))) {
10518 if (Addr.getValueType() != MVT::i16)
10519 Addr = DAG.getBitcast(MVT::i16, Addr);
10520 Addr = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Addr);
10521 } else {
10522 Addr = DAG.getBuildVector(PackVectorVT, DL, {Addr, Op.getOperand(I + 1)});
10523 I++;
10524 }
10525 Addr = DAG.getBitcast(MVT::f32, Addr);
10526 PackedAddrs.push_back(Addr);
10527 }
10528}
10529
10530/// Emit a DiagnosticInfoUnsupported for an unsupported image intrinsic and
10531/// return poison values of \p ResultTypes, preserving the chain if present.
10533 ArrayRef<EVT> ResultTypes,
10534 const SDLoc &DL, const Twine &Msg) {
10536 DAG.getMachineFunction().getFunction(), Msg, DL.getDebugLoc()));
10537 return DAG.getErrorMergeValues(ResultTypes, Op.getOperand(0), DL);
10538}
10539
10540SDValue SITargetLowering::lowerImage(SDValue Op,
10542 SelectionDAG &DAG, bool WithChain) const {
10543 SDLoc DL(Op);
10545 const GCNSubtarget *ST = &MF.getSubtarget<GCNSubtarget>();
10546 unsigned IntrOpcode = Intr->BaseOpcode;
10547 // For image atomic: use no-return opcode if result is unused.
10548 if (Intr->AtomicNoRetBaseOpcode != Intr->BaseOpcode &&
10549 !Op.getNode()->hasAnyUseOfValue(0))
10550 IntrOpcode = Intr->AtomicNoRetBaseOpcode;
10551 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
10553 const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfo(Intr->Dim);
10554 bool IsGFX10Plus = AMDGPU::isGFX10Plus(*Subtarget);
10555 bool IsGFX11Plus = AMDGPU::isGFX11Plus(*Subtarget);
10556 bool IsGFX12Plus = AMDGPU::isGFX12Plus(*Subtarget);
10557 bool IsGFX13 = AMDGPU::isGFX13(*Subtarget);
10558
10559 SmallVector<EVT, 3> ResultTypes(Op->values());
10560 SmallVector<EVT, 3> OrigResultTypes(Op->values());
10561 if (BaseOpcode->NoReturn && BaseOpcode->Atomic)
10562 ResultTypes.erase(&ResultTypes[0]);
10563
10564 bool IsD16 = false;
10565 bool IsG16 = false;
10566 bool IsA16 = false;
10567 SDValue VData;
10568 int NumVDataDwords = 0;
10569 bool AdjustRetType = false;
10570 bool IsAtomicPacked16Bit = false;
10571
10572 // Offset of intrinsic arguments
10573 const unsigned ArgOffset = WithChain ? 2 : 1;
10574
10575 unsigned DMask;
10576 unsigned DMaskLanes = 0;
10577
10578 if (BaseOpcode->Atomic) {
10579 VData = Op.getOperand(2);
10580
10581 IsAtomicPacked16Bit =
10582 (IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_F16 ||
10583 IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_F16_NORTN ||
10584 IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_BF16 ||
10585 IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_BF16_NORTN);
10586
10587 if (!IsAtomicPacked16Bit && VData.getValueSizeInBits() != 32 &&
10588 VData.getValueSizeInBits() != 64) {
10589 return diagnoseUnsupportedImage(DAG, Op, OrigResultTypes, DL,
10590 "unsupported image atomic data type");
10591 }
10592
10593 bool Is64Bit = VData.getValueSizeInBits() == 64;
10594 if (BaseOpcode->AtomicX2) {
10595 SDValue VData2 = Op.getOperand(3);
10596 VData = DAG.getBuildVector(Is64Bit ? MVT::v2i64 : MVT::v2i32, DL,
10597 {VData, VData2});
10598 if (Is64Bit)
10599 VData = DAG.getBitcast(MVT::v4i32, VData);
10600
10601 if (!BaseOpcode->NoReturn)
10602 ResultTypes[0] = Is64Bit ? MVT::v2i64 : MVT::v2i32;
10603
10604 DMask = Is64Bit ? 0xf : 0x3;
10605 NumVDataDwords = Is64Bit ? 4 : 2;
10606 } else {
10607 DMask = Is64Bit ? 0x3 : 0x1;
10608 NumVDataDwords = Is64Bit ? 2 : 1;
10609 }
10610 } else {
10611 DMask = Op->getConstantOperandVal(ArgOffset + Intr->DMaskIndex);
10612 DMaskLanes = BaseOpcode->Gather4 ? 4 : llvm::popcount(DMask);
10613
10614 if (BaseOpcode->Store) {
10615 VData = Op.getOperand(2);
10616
10617 MVT StoreVT = VData.getSimpleValueType();
10618 MVT StoreScalarVT = StoreVT.getScalarType();
10619 if (StoreScalarVT != MVT::f16 && StoreScalarVT.getSizeInBits() != 32 &&
10620 StoreScalarVT.getSizeInBits() != 64) {
10621 return diagnoseUnsupportedImage(DAG, Op, OrigResultTypes, DL,
10622 "unsupported image store data type");
10623 }
10624 if (StoreScalarVT == MVT::f16) {
10625 if (!Subtarget->hasD16Images() || !BaseOpcode->HasD16)
10626 return Op; // D16 is unsupported for this instruction
10627
10628 IsD16 = true;
10629 VData = handleD16VData(VData, DAG, true);
10630 }
10631
10632 NumVDataDwords = (VData.getValueType().getSizeInBits() + 31) / 32;
10633 } else if (!BaseOpcode->NoReturn) {
10634 // Work out the num dwords based on the dmask popcount and underlying type
10635 // and whether packing is supported.
10636 MVT LoadVT = ResultTypes[0].getSimpleVT();
10637 MVT LoadScalarVT = LoadVT.getScalarType();
10638 if (LoadScalarVT != MVT::f16 && LoadScalarVT.getSizeInBits() != 32 &&
10639 LoadScalarVT.getSizeInBits() != 64) {
10640 return diagnoseUnsupportedImage(DAG, Op, OrigResultTypes, DL,
10641 "unsupported image load data type");
10642 }
10643 if (LoadScalarVT == MVT::f16) {
10644 if (!Subtarget->hasD16Images() || !BaseOpcode->HasD16)
10645 return Op; // D16 is unsupported for this instruction
10646
10647 IsD16 = true;
10648 }
10649
10650 // Confirm that the return type is large enough for the dmask specified
10651 if ((LoadVT.isVector() && LoadVT.getVectorNumElements() < DMaskLanes) ||
10652 (!LoadVT.isVector() && DMaskLanes > 1))
10653 return Op;
10654
10655 // The sq block of gfx8 and gfx9 do not estimate register use correctly
10656 // for d16 image_gather4, image_gather4_l, and image_gather4_lz
10657 // instructions.
10658 if (IsD16 && !Subtarget->hasUnpackedD16VMem() &&
10659 !(BaseOpcode->Gather4 && Subtarget->hasImageGather4D16Bug()))
10660 NumVDataDwords = (DMaskLanes + 1) / 2;
10661 else
10662 NumVDataDwords = DMaskLanes;
10663
10664 AdjustRetType = true;
10665 }
10666 }
10667
10668 unsigned VAddrEnd = ArgOffset + Intr->VAddrEnd;
10670
10671 // Check for 16 bit addresses or derivatives and pack if true.
10672 MVT VAddrVT =
10673 Op.getOperand(ArgOffset + Intr->GradientStart).getSimpleValueType();
10674 MVT VAddrScalarVT = VAddrVT.getScalarType();
10675 MVT GradPackVectorVT = VAddrScalarVT == MVT::f16 ? MVT::v2f16 : MVT::v2i16;
10676 IsG16 = VAddrScalarVT == MVT::f16 || VAddrScalarVT == MVT::i16;
10677
10678 VAddrVT = Op.getOperand(ArgOffset + Intr->CoordStart).getSimpleValueType();
10679 VAddrScalarVT = VAddrVT.getScalarType();
10680 MVT AddrPackVectorVT = VAddrScalarVT == MVT::f16 ? MVT::v2f16 : MVT::v2i16;
10681 IsA16 = VAddrScalarVT == MVT::f16 || VAddrScalarVT == MVT::i16;
10682
10683 // Push back extra arguments.
10684 for (unsigned I = Intr->VAddrStart; I < Intr->GradientStart; I++) {
10685 if (IsA16 && (Op.getOperand(ArgOffset + I).getValueType() == MVT::f16)) {
10686 assert(I == Intr->BiasIndex && "Got unexpected 16-bit extra argument");
10687 // Special handling of bias when A16 is on. Bias is of type half but
10688 // occupies full 32-bit.
10689 SDValue Bias = DAG.getBuildVector(
10690 MVT::v2f16, DL,
10691 {Op.getOperand(ArgOffset + I), DAG.getPOISON(MVT::f16)});
10692 VAddrs.push_back(Bias);
10693 } else {
10694 assert((!IsA16 || Intr->NumBiasArgs == 0 || I != Intr->BiasIndex) &&
10695 "Bias needs to be converted to 16 bit in A16 mode");
10696 VAddrs.push_back(Op.getOperand(ArgOffset + I));
10697 }
10698 }
10699
10700 if (BaseOpcode->Gradients && !ST->hasG16() && (IsA16 != IsG16)) {
10701 // 16 bit gradients are supported, but are tied to the A16 control
10702 // so both gradients and addresses must be 16 bit
10703 LLVM_DEBUG(
10704 dbgs() << "Failed to lower image intrinsic: 16 bit addresses "
10705 "require 16 bit args for both gradients and addresses");
10706 return Op;
10707 }
10708
10709 if (IsA16) {
10710 if (!ST->hasA16()) {
10711 LLVM_DEBUG(dbgs() << "Failed to lower image intrinsic: Target does not "
10712 "support 16 bit addresses\n");
10713 return Op;
10714 }
10715 }
10716
10717 // We've dealt with incorrect input so we know that if IsA16, IsG16
10718 // are set then we have to compress/pack operands (either address,
10719 // gradient or both)
10720 // In the case where a16 and gradients are tied (no G16 support) then we
10721 // have already verified that both IsA16 and IsG16 are true
10722 if (BaseOpcode->Gradients && IsG16 && ST->hasG16()) {
10723 // Activate g16
10724 const AMDGPU::MIMGG16MappingInfo *G16MappingInfo =
10726 IntrOpcode = G16MappingInfo->G16; // set new opcode to variant with _g16
10727 }
10728
10729 // Add gradients (packed or unpacked)
10730 if (IsG16) {
10731 // Pack the gradients
10732 // const int PackEndIdx = IsA16 ? VAddrEnd : (ArgOffset + Intr->CoordStart);
10733 packImage16bitOpsToDwords(DAG, Op, GradPackVectorVT, VAddrs,
10734 ArgOffset + Intr->GradientStart,
10735 ArgOffset + Intr->CoordStart, Intr->NumGradients);
10736 } else {
10737 for (unsigned I = ArgOffset + Intr->GradientStart;
10738 I < ArgOffset + Intr->CoordStart; I++)
10739 VAddrs.push_back(Op.getOperand(I));
10740 }
10741
10742 // Add addresses (packed or unpacked)
10743 if (IsA16) {
10744 packImage16bitOpsToDwords(DAG, Op, AddrPackVectorVT, VAddrs,
10745 ArgOffset + Intr->CoordStart, VAddrEnd,
10746 0 /* No gradients */);
10747 } else {
10748 // Add uncompressed address
10749 for (unsigned I = ArgOffset + Intr->CoordStart; I < VAddrEnd; I++)
10750 VAddrs.push_back(Op.getOperand(I));
10751 }
10752
10753 // If the register allocator cannot place the address registers contiguously
10754 // without introducing moves, then using the non-sequential address encoding
10755 // is always preferable, since it saves VALU instructions and is usually a
10756 // wash in terms of code size or even better.
10757 //
10758 // However, we currently have no way of hinting to the register allocator that
10759 // MIMG addresses should be placed contiguously when it is possible to do so,
10760 // so force non-NSA for the common 2-address case as a heuristic.
10761 //
10762 // SIShrinkInstructions will convert NSA encodings to non-NSA after register
10763 // allocation when possible.
10764 //
10765 // Partial NSA is allowed on GFX11+ where the final register is a contiguous
10766 // set of the remaining addresses.
10767 const unsigned NSAMaxSize = ST->getNSAMaxSize(BaseOpcode->Sampler);
10768 const bool HasPartialNSAEncoding = ST->hasPartialNSAEncoding();
10769 const bool UseNSA = ST->hasNSAEncoding() &&
10770 VAddrs.size() >= ST->getNSAThreshold(MF) &&
10771 (VAddrs.size() <= NSAMaxSize || HasPartialNSAEncoding);
10772 const bool UsePartialNSA =
10773 UseNSA && HasPartialNSAEncoding && VAddrs.size() > NSAMaxSize;
10774
10775 SDValue VAddr;
10776 if (UsePartialNSA) {
10777 VAddr = getBuildDwordsVector(DAG, DL,
10778 ArrayRef(VAddrs).drop_front(NSAMaxSize - 1));
10779 } else if (!UseNSA) {
10780 VAddr = getBuildDwordsVector(DAG, DL, VAddrs);
10781 }
10782
10783 SDValue True = DAG.getTargetConstant(1, DL, MVT::i1);
10784 SDValue False = DAG.getTargetConstant(0, DL, MVT::i1);
10785 SDValue Unorm;
10786 if (!BaseOpcode->Sampler) {
10787 Unorm = True;
10788 } else {
10789 uint64_t UnormConst =
10790 Op.getConstantOperandVal(ArgOffset + Intr->UnormIndex);
10791
10792 Unorm = UnormConst ? True : False;
10793 }
10794
10795 SDValue TFE;
10796 SDValue LWE;
10797 SDValue TexFail = Op.getOperand(ArgOffset + Intr->TexFailCtrlIndex);
10798 bool IsTexFail = false;
10799 if (!parseTexFail(TexFail, DAG, &TFE, &LWE, IsTexFail))
10800 return Op;
10801
10802 if (IsTexFail) {
10803 if (!DMaskLanes) {
10804 // Expecting to get an error flag since TFC is on - and dmask is 0
10805 // Force dmask to be at least 1 otherwise the instruction will fail
10806 DMask = 0x1;
10807 DMaskLanes = 1;
10808 NumVDataDwords = 1;
10809 }
10810 NumVDataDwords += 1;
10811 AdjustRetType = true;
10812 }
10813
10814 // Has something earlier tagged that the return type needs adjusting
10815 // This happens if the instruction is a load or has set TexFailCtrl flags
10816 if (AdjustRetType) {
10817 // NumVDataDwords reflects the true number of dwords required in the return
10818 // type
10819 if (DMaskLanes == 0 && !BaseOpcode->Store) {
10820 // This is a no-op load. This can be eliminated
10821 SDValue Undef = DAG.getPOISON(Op.getValueType());
10822 if (isa<MemSDNode>(Op))
10823 return DAG.getMergeValues({Undef, Op.getOperand(0)}, DL);
10824 return Undef;
10825 }
10826
10827 EVT NewVT = NumVDataDwords > 1 ? EVT::getVectorVT(*DAG.getContext(),
10828 MVT::i32, NumVDataDwords)
10829 : MVT::i32;
10830
10831 ResultTypes[0] = NewVT;
10832 if (ResultTypes.size() == 3) {
10833 // Original result was aggregate type used for TexFailCtrl results
10834 // The actual instruction returns as a vector type which has now been
10835 // created. Remove the aggregate result.
10836 ResultTypes.erase(&ResultTypes[1]);
10837 }
10838 }
10839
10840 unsigned CPol = Op.getConstantOperandVal(ArgOffset + Intr->CachePolicyIndex);
10841 // Keep GLC only when the atomic's result is actually used.
10842 if (BaseOpcode->Atomic && !BaseOpcode->NoReturn)
10844 if (CPol & ~((IsGFX12Plus ? AMDGPU::CPol::ALL : AMDGPU::CPol::ALL_pregfx12) |
10846 return Op;
10847
10849 if (BaseOpcode->Store || BaseOpcode->Atomic)
10850 Ops.push_back(VData); // vdata
10851 if (UsePartialNSA) {
10852 append_range(Ops, ArrayRef(VAddrs).take_front(NSAMaxSize - 1));
10853 Ops.push_back(VAddr);
10854 } else if (UseNSA)
10855 append_range(Ops, VAddrs);
10856 else
10857 Ops.push_back(VAddr);
10858 SDValue Rsrc = Op.getOperand(ArgOffset + Intr->RsrcIndex);
10859 EVT RsrcVT = Rsrc.getValueType();
10860 bool IndexedRsrc = IsGFX13 && RsrcVT == MVT::i32;
10861 if (RsrcVT != MVT::v4i32 && RsrcVT != MVT::v8i32 && !IndexedRsrc)
10862 return Op;
10863 Ops.push_back(Rsrc);
10864 bool IsSampIndexed = false;
10865 if (BaseOpcode->Sampler) {
10866 SDValue Samp = Op.getOperand(ArgOffset + Intr->SampIndex);
10867 EVT SampVT = Samp.getValueType();
10868 IsSampIndexed = IsGFX13 && SampVT == MVT::i32;
10869 if (SampVT != MVT::v4i32 && !IsSampIndexed)
10870 return Op;
10871 Ops.push_back(Samp);
10872 }
10873 Ops.push_back(DAG.getTargetConstant(DMask, DL, MVT::i32));
10874 if (IsGFX10Plus)
10875 Ops.push_back(DAG.getTargetConstant(DimInfo->Encoding, DL, MVT::i32));
10876 if (!IsGFX12Plus || BaseOpcode->Sampler || BaseOpcode->MSAA)
10877 Ops.push_back(Unorm);
10878 Ops.push_back(DAG.getTargetConstant(CPol, DL, MVT::i32));
10879 Ops.push_back(IsA16 && // r128, a16 for gfx9
10880 ST->hasFeature(AMDGPU::FeatureR128A16)
10881 ? True
10882 : False);
10883 if (IsGFX10Plus)
10884 Ops.push_back(IsA16 ? True : False);
10885
10886 if (!Subtarget->hasGFX90AInsts())
10887 Ops.push_back(TFE); // tfe
10888 else if (TFE->getAsZExtVal()) {
10889 DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
10891 "TFE is not supported on this GPU", DL.getDebugLoc()));
10892 }
10893
10894 if (!IsGFX12Plus || BaseOpcode->Sampler || BaseOpcode->MSAA)
10895 Ops.push_back(LWE); // lwe
10896 if (!IsGFX10Plus)
10897 Ops.push_back(DimInfo->DA ? True : False);
10898 if (BaseOpcode->HasD16)
10899 Ops.push_back(IsD16 ? True : False);
10900 if (isa<MemSDNode>(Op))
10901 Ops.push_back(Op.getOperand(0)); // chain
10902
10903 int NumVAddrDwords =
10904 UseNSA ? VAddrs.size() : VAddr.getValueType().getSizeInBits() / 32;
10905 int Opcode = -1;
10906
10907 if (IsGFX13) {
10908 Opcode =
10909 AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx13, NumVDataDwords,
10910 NumVAddrDwords, IndexedRsrc, IsSampIndexed);
10911 } else if (IsGFX12Plus) {
10912 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx12,
10913 NumVDataDwords, NumVAddrDwords);
10914 } else if (IsGFX11Plus) {
10915 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode,
10916 UseNSA ? AMDGPU::MIMGEncGfx11NSA
10917 : AMDGPU::MIMGEncGfx11Default,
10918 NumVDataDwords, NumVAddrDwords);
10919 } else if (IsGFX10Plus) {
10920 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode,
10921 UseNSA ? AMDGPU::MIMGEncGfx10NSA
10922 : AMDGPU::MIMGEncGfx10Default,
10923 NumVDataDwords, NumVAddrDwords);
10924 } else {
10925 if (Subtarget->hasGFX90AInsts()) {
10926 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx90a,
10927 NumVDataDwords, NumVAddrDwords);
10928 if (Opcode == -1) {
10930 DAG, Op, OrigResultTypes, DL,
10931 "requested image instruction is not supported on this GPU");
10932 }
10933 }
10934 if (Opcode == -1 &&
10935 Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
10936 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx8,
10937 NumVDataDwords, NumVAddrDwords);
10938 if (Opcode == -1)
10939 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx6,
10940 NumVDataDwords, NumVAddrDwords);
10941 }
10942 if (Opcode == -1)
10943 return Op;
10944
10945 MachineSDNode *NewNode = DAG.getMachineNode(Opcode, DL, ResultTypes, Ops);
10946 if (auto *MemOp = dyn_cast<MemSDNode>(Op)) {
10947 MachineMemOperand *MemRef = MemOp->getMemOperand();
10948 DAG.setNodeMemRefs(NewNode, {MemRef});
10949 }
10950
10951 if (BaseOpcode->NoReturn) {
10952 if (BaseOpcode->Atomic)
10953 return DAG.getMergeValues(
10954 {DAG.getPOISON(OrigResultTypes[0]), SDValue(NewNode, 0)}, DL);
10955
10956 return SDValue(NewNode, 0);
10957 }
10958
10959 if (BaseOpcode->AtomicX2) {
10961 DAG.ExtractVectorElements(SDValue(NewNode, 0), Elt, 0, 1);
10962 return DAG.getMergeValues({Elt[0], SDValue(NewNode, 1)}, DL);
10963 }
10964
10965 return constructRetValue(DAG, NewNode, OrigResultTypes, IsTexFail,
10966 Subtarget->hasUnpackedD16VMem(), IsD16, DMaskLanes,
10967 NumVDataDwords, IsAtomicPacked16Bit, DL);
10968}
10969
10970SDValue SITargetLowering::lowerSBuffer(EVT VT, EVT MemVT, SDLoc DL,
10971 SDValue Chain, SDValue Rsrc,
10972 SDValue Offset, SDValue CachePolicy,
10973 SelectionDAG &DAG,
10974 MachineMemOperand *MMO) const {
10976 bool HasChainResult = MMO != nullptr;
10977
10978 // SBUFFER_LOAD only produces values that fill whole SGPRs, apart from the
10979 // subword loads below.
10980 bool IsSubwordLoad = (MemVT == MVT::i8 || MemVT == MVT::i16) &&
10981 Subtarget->hasScalarSubwordLoads();
10982 if ((!isTypeLegal(VT) || VT.getSizeInBits() % 32 != 0) && !IsSubwordLoad) {
10983 DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
10984 MF.getFunction(), "unsupported s_buffer_load result type",
10985 DL.getDebugLoc()));
10986 EVT ResultTypes[] = {VT, MVT::Other};
10987 return DAG.getErrorMergeValues(
10988 ArrayRef(ResultTypes, HasChainResult ? 2 : 1), Chain, DL);
10989 }
10990
10991 if (!HasChainResult) {
10992 const DataLayout &DataLayout = DAG.getDataLayout();
10994 DataLayout.getABITypeAlign(MemVT.getTypeForEVT(*DAG.getContext()));
10995
10996 MMO = MF.getMachineMemOperand(MachinePointerInfo(),
11000 MemVT.getStoreSize(), Alignment);
11001 }
11002
11003 if (!Offset->isDivergent()) {
11004 SDValue Ops[] = {Chain, Rsrc, Offset, CachePolicy};
11005
11006 // Lower llvm.amdgcn.*s.buffer.load.{i,u}N intrinsics. First, generate
11007 // s_buffer_load_u* for signed and unsigned load instructions. Next, DAG
11008 // combiner tries to merge the s_buffer_load_uN with a sext instruction
11009 // (performSignExtendInRegCombine()) and it replaces s_buffer_load_uN with
11010 // s_buffer_load_iN.
11011 auto HandleScalarSubwordLoads = [&](unsigned Opcode) -> SDValue {
11012 SDValue BufferLoad = DAG.getMemIntrinsicNode(
11013 Opcode, DL, DAG.getVTList(MVT::i32, MVT::Other), Ops, MemVT, MMO);
11014 SDValue LoadVal = DAG.getAnyExtOrTrunc(
11015 DAG.getNode(ISD::TRUNCATE, DL, MemVT, BufferLoad), DL, VT);
11016 if (HasChainResult)
11017 return DAG.getMergeValues({LoadVal, BufferLoad.getValue(1)}, DL);
11018 return LoadVal;
11019 };
11020 if (MemVT == MVT::i8 && Subtarget->hasScalarSubwordLoads())
11021 return HandleScalarSubwordLoads(AMDGPUISD::SBUFFER_LOAD_UBYTE);
11022
11023 if (MemVT == MVT::i16 && Subtarget->hasScalarSubwordLoads())
11024 return HandleScalarSubwordLoads(AMDGPUISD::SBUFFER_LOAD_USHORT);
11025
11026 // Widen vec3 load to vec4. Only 32-bit elements have a vec4 pattern.
11027 if (VT.isVector() && VT.getVectorNumElements() == 3 &&
11028 VT.getVectorElementType().getSizeInBits() == 32 &&
11029 !Subtarget->hasScalarDwordx3Loads()) {
11030 EVT WidenedVT =
11032 auto WidenedOp = DAG.getMemIntrinsicNode(
11033 AMDGPUISD::SBUFFER_LOAD, DL, DAG.getVTList(WidenedVT, MVT::Other),
11034 Ops, WidenedVT,
11035 MF.getMachineMemOperand(MMO, 0, WidenedVT.getStoreSize()));
11036 auto Subvector = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, WidenedOp,
11037 DAG.getVectorIdxConstant(0, DL));
11038 if (HasChainResult)
11039 return DAG.getMergeValues({Subvector, WidenedOp.getValue(1)}, DL);
11040 return Subvector;
11041 }
11042
11043 return DAG.getMemIntrinsicNode(AMDGPUISD::SBUFFER_LOAD, DL,
11044 DAG.getVTList(VT, MVT::Other), Ops, MemVT,
11045 MMO);
11046 }
11047
11048 // We have a divergent offset. Emit a MUBUF buffer load instead. We can
11049 // assume that the buffer is unswizzled.
11050 SDValue Ops[] = {
11051 Chain, // Chain
11052 Rsrc, // rsrc
11053 DAG.getConstant(0, DL, MVT::i32), // vindex
11054 {}, // voffset
11055 {}, // soffset
11056 {}, // offset
11057 CachePolicy, // cachepolicy
11058 DAG.getTargetConstant(0, DL, MVT::i1), // idxen
11059 };
11060 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
11061 Subtarget->hasScalarSubwordLoads()) {
11062 setBufferOffsets(Offset, DAG, &Ops[3], Align(4));
11063 SDValue Load = handleByteShortBufferLoads(DAG, MemVT, DL, Ops, MMO);
11064 SDValue LoadVal = DAG.getAnyExtOrTrunc(Load.getOperand(0), DL, VT);
11065 if (HasChainResult)
11066 return DAG.getMergeValues({LoadVal, Load.getOperand(1)}, DL);
11067 return LoadVal;
11068 }
11069
11071 unsigned NumLoads = 1;
11072 MVT LoadVT = VT.getSimpleVT();
11073 unsigned NumElts = LoadVT.isVector() ? LoadVT.getVectorNumElements() : 1;
11074 assert((LoadVT.getScalarType() == MVT::i32 ||
11075 LoadVT.getScalarType() == MVT::f32));
11076
11077 if (NumElts == 8 || NumElts == 16) {
11078 NumLoads = NumElts / 4;
11079 LoadVT = MVT::getVectorVT(LoadVT.getScalarType(), 4);
11080 }
11081
11082 SDVTList VTList = DAG.getVTList({LoadVT, MVT::Other});
11083
11084 // Use the alignment to ensure that the required offsets will fit into the
11085 // immediate offsets.
11086 setBufferOffsets(Offset, DAG, &Ops[3],
11087 NumLoads > 1 ? Align(16 * NumLoads) : Align(4));
11088
11089 uint64_t InstOffset = Ops[5]->getAsZExtVal();
11090 unsigned LoadSize = LoadVT.getStoreSize();
11091 for (unsigned i = 0; i < NumLoads; ++i) {
11092 Ops[5] = DAG.getTargetConstant(InstOffset + 16 * i, DL, MVT::i32);
11093 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(MMO, 16 * i, LoadSize);
11094 Loads.push_back(getMemIntrinsicNode(AMDGPUISD::BUFFER_LOAD, DL, VTList, Ops,
11095 LoadVT, LoadMMO, DAG));
11096 }
11097
11098 if (NumElts == 8 || NumElts == 16) {
11099 SDValue LoadVal = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Loads);
11100 if (HasChainResult) {
11101 SmallVector<SDValue, 4> LoadChains;
11102 for (SDValue Load : Loads)
11103 LoadChains.push_back(Load.getValue(1));
11104 SDValue Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, LoadChains);
11105 return DAG.getMergeValues({LoadVal, Chain}, DL);
11106 }
11107 return LoadVal;
11108 }
11109
11110 return Loads[0];
11111}
11112
11113SDValue SITargetLowering::lowerWaveID(SelectionDAG &DAG, SDValue Op) const {
11114 // With architected SGPRs, waveIDinGroup is in TTMP8[29:25].
11115 if (!Subtarget->hasArchitectedSGPRs())
11116 return {};
11117 SDLoc SL(Op);
11118 MVT VT = MVT::i32;
11119 SDValue TTMP8 = DAG.getCopyFromReg(DAG.getEntryNode(), SL, AMDGPU::TTMP8, VT);
11120 return DAG.getNode(AMDGPUISD::BFE_U32, SL, VT, TTMP8,
11121 DAG.getConstant(25, SL, VT), DAG.getConstant(5, SL, VT));
11122}
11123
11124SDValue SITargetLowering::lowerConstHwRegRead(SelectionDAG &DAG, SDValue Op,
11125 AMDGPU::Hwreg::Id HwReg,
11126 unsigned LowBit,
11127 unsigned Width) const {
11128 SDLoc SL(Op);
11129 using namespace AMDGPU::Hwreg;
11130 return {DAG.getMachineNode(
11131 AMDGPU::S_GETREG_B32_const, SL, MVT::i32,
11132 DAG.getTargetConstant(HwregEncoding::encode(HwReg, LowBit, Width),
11133 SL, MVT::i32)),
11134 0};
11135}
11136
11137SDValue SITargetLowering::lowerWorkitemID(SelectionDAG &DAG, SDValue Op,
11138 unsigned Dim,
11139 const ArgDescriptor &Arg) const {
11140 SDLoc SL(Op);
11142 unsigned MaxID = Subtarget->getMaxWorkitemID(MF.getFunction(), Dim);
11143 if (MaxID == 0)
11144 return DAG.getConstant(0, SL, MVT::i32);
11145
11146 // It's undefined behavior if a function marked with the amdgpu-no-*
11147 // attributes uses the corresponding intrinsic.
11148 if (!Arg)
11149 return DAG.getPOISON(Op->getValueType(0));
11150
11151 SDValue Val = loadInputValue(DAG, &AMDGPU::VGPR_32RegClass, MVT::i32,
11152 SDLoc(DAG.getEntryNode()), Arg);
11153
11154 // Don't bother inserting AssertZext for packed IDs since we're emitting the
11155 // masking operations anyway.
11156 //
11157 // TODO: We could assert the top bit is 0 for the source copy.
11158 if (Arg.isMasked())
11159 return Val;
11160
11161 // Preserve the known bits after expansion to a copy.
11162 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), llvm::bit_width(MaxID));
11163 return DAG.getNode(ISD::AssertZext, SL, MVT::i32, Val,
11164 DAG.getValueType(SmallVT));
11165}
11166
11167SDValue SITargetLowering::lowerFromFP8(SDValue Op, bool IsBF8,
11168 SelectionDAG &DAG) const {
11169 SDLoc SL(Op);
11170 SDValue Src = Op.getOperand(0);
11171 EVT DstVT = Op.getValueType();
11172 bool IsF16 = DstVT.getVectorElementType() == MVT::f16;
11173 assert((!IsF16 || Subtarget->hasFP8F16ConversionInsts()) &&
11174 "fp8/bf8 -> f16 conversion requires FP8F16ConversionInsts");
11175
11176 unsigned Opc;
11177 if (IsF16)
11178 Opc = IsBF8 ? AMDGPUISD::CVT_PK_F16_BF8 : AMDGPUISD::CVT_PK_F16_FP8;
11179 else
11180 Opc = IsBF8 ? AMDGPUISD::CVT_PK_F32_BF8 : AMDGPUISD::CVT_PK_F32_FP8;
11181
11182 // Pack the two i8 lanes into the integer type the packed HW node reads. The
11183 // f16 form takes i16 and the f32 form takes i32. v2i8 bitcasts to i16
11184 // directly and the f32 node reads the low half of an any-extended i32.
11185 EVT PackedVT =
11187 SDValue AsI16 = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Src);
11188 SDValue Packed = DAG.getAnyExtOrTrunc(AsI16, SL, PackedVT);
11189 return DAG.getNode(Opc, SL, DstVT, Packed);
11190}
11191
11192SDValue
11193SITargetLowering::LowerCONVERT_FROM_ARBITRARY_FP(SDValue Op,
11194 SelectionDAG &DAG) const {
11195 // Handle the OCP FP8 formats (E4M3FN, E5M2) and unsigned E5M3 on subtargets
11196 // with matching HW conversions. Other formats use the generic expansion.
11197 APFloatBase::Semantics FPSemantic =
11198 static_cast<APFloatBase::Semantics>(Op.getConstantOperandVal(1));
11199 const bool IsFP8 = FPSemantic == APFloatBase::S_Float8E4M3FN;
11200 const bool IsBF8 = FPSemantic == APFloatBase::S_Float8E5M2;
11201 const bool IsE5M3 = FPSemantic == APFloatBase::S_Float8E5M3FNU;
11202 const bool HasE5M3ConversionInsts =
11203 Subtarget->hasFP8ConversionInsts() && Subtarget->hasFP8E5M3Insts();
11204 const bool IsSupported = IsFP8 || IsBF8 || (IsE5M3 && HasE5M3ConversionInsts);
11205 if (!IsSupported)
11206 return SDValue();
11207
11208 EVT DstVT = Op.getValueType();
11209 // The custom action for a v2i8 source also reaches half conversions on
11210 // targets which only have FP8-to-f32 instructions.
11211 if (DstVT.getScalarType() == MVT::f16 &&
11212 !Subtarget->hasFP8F16ConversionInsts())
11213 return SDValue();
11214
11215 if (IsE5M3) {
11216 if (DstVT.getScalarType() != MVT::f32)
11217 return SDValue();
11218
11219 SDLoc SL(Op);
11220 SDValue Src = Op.getOperand(0);
11221 assert((!DstVT.isVector() || DstVT == MVT::v2f32) &&
11222 "only the v2f32 vector result is custom lowered");
11223
11224 if (DstVT.isVector())
11225 Src = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Src);
11226 Src = DAG.getAnyExtOrTrunc(Src, SL, MVT::i32);
11227
11228 auto ConvertByte = [&](unsigned ByteSel) {
11229 return DAG.getNode(AMDGPUISD::CVT_F32_FP8_E5M3, SL, MVT::f32, Src,
11230 DAG.getTargetConstant(ByteSel, SL, MVT::i32));
11231 };
11232
11233 if (!DstVT.isVector())
11234 return ConvertByte(0);
11235 return DAG.getBuildVector(DstVT, SL, {ConvertByte(0), ConvertByte(1)});
11236 }
11237
11238 if (!DstVT.isVector()) {
11239 SDValue Src = Op.getOperand(0);
11240 if (Src.getValueType() != MVT::i32) {
11241 SDLoc SL(Op);
11242 SDValue SrcI32 = DAG.getAnyExtOrTrunc(Src, SL, MVT::i32);
11243 return DAG.getNode(ISD::CONVERT_FROM_ARBITRARY_FP, SL, DstVT, SrcI32,
11244 Op.getOperand(1));
11245 }
11246 return Op;
11247 }
11248
11249 EVT EltVT = DstVT.getVectorElementType();
11250 if (EltVT == MVT::f16 || EltVT == MVT::f32)
11251 return lowerFromFP8(Op, IsBF8, DAG);
11252 return SDValue();
11253}
11254
11255SDValue SITargetLowering::lowerToFP8(SDValue Op, bool IsBF8, bool IsE5M3,
11256 SelectionDAG &DAG) const {
11257 SDLoc SL(Op);
11258 SDValue Src = Op.getOperand(0);
11259 EVT ResVT = Op.getValueType();
11260 bool IsF16 = Src.getValueType().getScalarType() == MVT::f16;
11261 assert((!IsF16 || Subtarget->hasF16FP8ConversionInsts()) &&
11262 "f16 -> fp8/bf8 conversion requires F16FP8ConversionInsts");
11263 assert((!ResVT.isVector() || ResVT == MVT::v2i8) &&
11264 "only the v2i8 vector result is custom lowered");
11265
11266 if (IsF16) {
11267 unsigned Opc =
11268 IsBF8 ? AMDGPUISD::CVT_PK_BF8_F16 : AMDGPUISD::CVT_PK_FP8_F16;
11269 SDValue Bytes = DAG.getNode(Opc, SL, MVT::i16, Src);
11270 return DAG.getNode(ISD::BITCAST, SL, ResVT, Bytes);
11271 }
11272
11273 unsigned Opc = IsBF8 ? AMDGPUISD::CVT_PK_BF8_F32
11274 : IsE5M3 ? AMDGPUISD::CVT_PK_FP8_F32_E5M3
11275 : AMDGPUISD::CVT_PK_FP8_F32;
11276 SDValue PoisonI32 = DAG.getPOISON(MVT::i32);
11277 SDValue WordSel = DAG.getTargetConstant(0, SL, MVT::i1);
11278
11279 if (!ResVT.isVector()) {
11280 // Convert one lane, the second is unused. Feed it the same source so the
11281 // instruction does not read an undefined register.
11282 SDValue Packed =
11283 DAG.getNode(Opc, SL, MVT::i32, Src, Src, PoisonI32, WordSel);
11284 return DAG.getAnyExtOrTrunc(Packed, SL, ResVT);
11285 }
11286
11287 SDValue A = DAG.getExtractVectorElt(SL, MVT::f32, Src, 0);
11288 SDValue B = DAG.getExtractVectorElt(SL, MVT::f32, Src, 1);
11289 SDValue Packed = DAG.getNode(Opc, SL, MVT::i32, A, B, PoisonI32, WordSel);
11290 SDValue Bytes = DAG.getNode(ISD::TRUNCATE, SL, MVT::i16, Packed);
11291 return DAG.getNode(ISD::BITCAST, SL, ResVT, Bytes);
11292}
11293
11294SDValue
11295SITargetLowering::LowerCONVERT_TO_ARBITRARY_FP(SDValue Op,
11296 SelectionDAG &DAG) const {
11297 // The OCP FP8 formats (E4M3FN, E5M2) and unsigned E5M3 map to HW conversions
11298 // on subtargets that support them. Everything else uses generic expansion.
11300 static_cast<APFloatBase::Semantics>(Op.getConstantOperandVal(1));
11301 const bool IsFP8 = Sem == APFloatBase::S_Float8E4M3FN;
11302 const bool IsBF8 = Sem == APFloatBase::S_Float8E5M2;
11303 const bool IsE5M3 = Sem == APFloatBase::S_Float8E5M3FNU;
11304 const bool HasE5M3ConversionInsts =
11305 Subtarget->hasFP8ConversionInsts() && Subtarget->hasFP8E5M3Insts();
11306 const bool IsSupported = IsFP8 || IsBF8 || (IsE5M3 && HasE5M3ConversionInsts);
11307 if (!IsSupported)
11308 return SDValue();
11309
11310 // The HW conversions only support nearest-even. The OCP conversions do not
11311 // saturate. The unsigned E5M3 conversion always clamps out-of-range inputs,
11312 // which also refines the non-saturating form where those inputs are poison.
11313 if (static_cast<RoundingMode>(Op.getConstantOperandVal(2)) !=
11315 return SDValue();
11316 if (!IsE5M3 && Op.getConstantOperandVal(3) != 0)
11317 return SDValue();
11318
11319 EVT SrcEltVT = Op.getOperand(0).getValueType().getScalarType();
11320 // The f32 form is built here rather than by a tablegen pattern because the
11321 // HW result is i32 while the node result is i16 after the i8 promotion.
11322 if (SrcEltVT == MVT::f32)
11323 return lowerToFP8(Op, IsBF8, IsE5M3, DAG);
11324 if (!IsE5M3 && SrcEltVT == MVT::f16 &&
11325 Subtarget->hasF16FP8ConversionInsts()) {
11326 // A scalar conversion is selected from the generic node by tablegen, only
11327 // the illegal v2i8 result type needs lowering here.
11328 if (!Op.getValueType().isVector())
11329 return Op;
11330 return lowerToFP8(Op, IsBF8, false, DAG);
11331 }
11332 return SDValue();
11333}
11334
11335SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
11336 SelectionDAG &DAG) const {
11338 auto *MFI = MF.getInfo<SIMachineFunctionInfo>();
11339
11340 EVT VT = Op.getValueType();
11341 SDLoc DL(Op);
11342 unsigned IntrinsicID = Op.getConstantOperandVal(0);
11343
11344 // TODO: Should this propagate fast-math-flags?
11345
11346 switch (IntrinsicID) {
11347 case Intrinsic::amdgcn_wave_reduce_min:
11348 case Intrinsic::amdgcn_wave_reduce_umin:
11349 case Intrinsic::amdgcn_wave_reduce_fmin:
11350 case Intrinsic::amdgcn_wave_reduce_max:
11351 case Intrinsic::amdgcn_wave_reduce_umax:
11352 case Intrinsic::amdgcn_wave_reduce_fmax:
11353 case Intrinsic::amdgcn_wave_reduce_add:
11354 case Intrinsic::amdgcn_wave_reduce_fadd:
11355 case Intrinsic::amdgcn_wave_reduce_sub:
11356 case Intrinsic::amdgcn_wave_reduce_fsub:
11357 case Intrinsic::amdgcn_wave_reduce_and:
11358 case Intrinsic::amdgcn_wave_reduce_or:
11359 case Intrinsic::amdgcn_wave_reduce_xor: {
11360 EVT SrcVT = Op.getOperand(1).getValueType();
11361 if (SrcVT.getFixedSizeInBits() == 16) {
11362 bool IsFPOp = SrcVT.isFloatingPoint();
11363 bool NeedsSignExt = IntrinsicID == Intrinsic::amdgcn_wave_reduce_min ||
11364 IntrinsicID == Intrinsic::amdgcn_wave_reduce_max ||
11365 IntrinsicID == Intrinsic::amdgcn_wave_reduce_add ||
11366 IntrinsicID == Intrinsic::amdgcn_wave_reduce_sub;
11367 unsigned ExtOpc = IsFPOp ? ISD::FP_EXTEND
11368 : NeedsSignExt ? ISD::SIGN_EXTEND
11370 auto SrcType = IsFPOp ? MVT::f16 : MVT::i16;
11371 auto ExtType = IsFPOp ? MVT::f32 : MVT::i32;
11372 SDValue ExtendedSrc = DAG.getNode(ExtOpc, DL, ExtType, Op.getOperand(1));
11373 SDValue Strategy = Op.getOperand(2);
11375 Op.getOperand(0), ExtendedSrc, Strategy);
11376 if (IsFPOp)
11377 return DAG.getNode(ISD::FP_ROUND, DL, SrcType, Result,
11378 DAG.getTargetConstant(1, DL, MVT::i32));
11379 else
11380 return DAG.getNode(ISD::TRUNCATE, DL, SrcType, Result);
11381 }
11382 return SDValue();
11383 }
11384 case Intrinsic::amdgcn_implicit_buffer_ptr: {
11385 if (!MFI->getUserSGPRInfo().hasImplicitBufferPtr()) {
11386 const char *Error =
11387 Subtarget->hasArchitectedFlatScratch()
11388 ? "intrinsic not supported on subtarget"
11389 : !Subtarget->isMesa3DOS()
11390 ? "intrinsic not supported for OS"
11391 : "intrinsic not supported for calling convention";
11392 DAG.getContext()->diagnose(
11393 DiagnosticInfoUnsupported(MF.getFunction(), Error, DL.getDebugLoc()));
11394 return DAG.getPOISON(VT);
11395 }
11396
11397 return getPreloadedValue(DAG, *MFI, VT,
11399 }
11400 case Intrinsic::amdgcn_dispatch_ptr:
11401 case Intrinsic::amdgcn_queue_ptr: {
11402 if (!Subtarget->isAmdHsaOrMesa(MF.getFunction())) {
11403 DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
11404 MF.getFunction(), "unsupported hsa intrinsic without hsa target",
11405 DL.getDebugLoc()));
11406 return DAG.getPOISON(VT);
11407 }
11408
11409 auto RegID = IntrinsicID == Intrinsic::amdgcn_dispatch_ptr
11412 return getPreloadedValue(DAG, *MFI, VT, RegID);
11413 }
11414 case Intrinsic::amdgcn_implicitarg_ptr: {
11415 if (MFI->isEntryFunction())
11416 return getImplicitArgPtr(DAG, DL);
11417 return getPreloadedValue(DAG, *MFI, VT,
11419 }
11420 case Intrinsic::amdgcn_kernarg_segment_ptr: {
11421 if (!AMDGPU::isKernel(MF.getFunction())) {
11422 // This only makes sense to call in a kernel, so just lower to null.
11423 return DAG.getConstant(0, DL, VT);
11424 }
11425
11426 return getPreloadedValue(DAG, *MFI, VT,
11428 }
11429 case Intrinsic::amdgcn_dispatch_id: {
11430 return getPreloadedValue(DAG, *MFI, VT, AMDGPUFunctionArgInfo::DISPATCH_ID);
11431 }
11432 case Intrinsic::amdgcn_rcp:
11433 return DAG.getNode(AMDGPUISD::RCP, DL, VT, Op.getOperand(1));
11434 case Intrinsic::amdgcn_rsq:
11435 return DAG.getNode(AMDGPUISD::RSQ, DL, VT, Op.getOperand(1));
11436 case Intrinsic::amdgcn_rsq_legacy:
11437 if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
11438 return emitRemovedIntrinsicError(DAG, DL, VT);
11439 return SDValue();
11440 case Intrinsic::amdgcn_rcp_legacy:
11441 if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
11442 return emitRemovedIntrinsicError(DAG, DL, VT);
11443 return DAG.getNode(AMDGPUISD::RCP_LEGACY, DL, VT, Op.getOperand(1));
11444 case Intrinsic::amdgcn_fma_legacy:
11445 case Intrinsic::amdgcn_sudot4:
11446 case Intrinsic::amdgcn_sudot8:
11447 case Intrinsic::amdgcn_tanh:
11448 return SDValue();
11449 case Intrinsic::amdgcn_rsq_clamp: {
11450 if (Subtarget->getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS)
11451 return DAG.getNode(AMDGPUISD::RSQ_CLAMP, DL, VT, Op.getOperand(1));
11452
11453 Type *Type = VT.getTypeForEVT(*DAG.getContext());
11454 APFloat Max = APFloat::getLargest(Type->getFltSemantics());
11455 APFloat Min = APFloat::getLargest(Type->getFltSemantics(), true);
11456
11457 SDValue Rsq = DAG.getNode(AMDGPUISD::RSQ, DL, VT, Op.getOperand(1));
11458 SDValue Tmp =
11459 DAG.getNode(ISD::FMINNUM, DL, VT, Rsq, DAG.getConstantFP(Max, DL, VT));
11460 return DAG.getNode(ISD::FMAXNUM, DL, VT, Tmp,
11461 DAG.getConstantFP(Min, DL, VT));
11462 }
11463 case Intrinsic::r600_read_ngroups_x:
11464 if (Subtarget->isAmdHsaOS())
11465 return emitNonHSAIntrinsicError(DAG, DL, VT);
11466
11467 return lowerKernargMemParameter(DAG, VT, VT, DL, DAG.getEntryNode(),
11469 false);
11470 case Intrinsic::r600_read_ngroups_y:
11471 if (Subtarget->isAmdHsaOS())
11472 return emitNonHSAIntrinsicError(DAG, DL, VT);
11473
11474 return lowerKernargMemParameter(DAG, VT, VT, DL, DAG.getEntryNode(),
11476 false);
11477 case Intrinsic::r600_read_ngroups_z:
11478 if (Subtarget->isAmdHsaOS())
11479 return emitNonHSAIntrinsicError(DAG, DL, VT);
11480
11481 return lowerKernargMemParameter(DAG, VT, VT, DL, DAG.getEntryNode(),
11483 false);
11484 case Intrinsic::r600_read_local_size_x:
11485 if (Subtarget->isAmdHsaOS())
11486 return emitNonHSAIntrinsicError(DAG, DL, VT);
11487
11488 return lowerImplicitZextParam(DAG, Op, MVT::i16,
11490 case Intrinsic::r600_read_local_size_y:
11491 if (Subtarget->isAmdHsaOS())
11492 return emitNonHSAIntrinsicError(DAG, DL, VT);
11493
11494 return lowerImplicitZextParam(DAG, Op, MVT::i16,
11496 case Intrinsic::r600_read_local_size_z:
11497 if (Subtarget->isAmdHsaOS())
11498 return emitNonHSAIntrinsicError(DAG, DL, VT);
11499
11500 return lowerImplicitZextParam(DAG, Op, MVT::i16,
11502 case Intrinsic::amdgcn_workgroup_id_x:
11503 return lowerWorkGroupId(DAG, *MFI, VT,
11507 case Intrinsic::amdgcn_workgroup_id_y:
11508 return lowerWorkGroupId(DAG, *MFI, VT,
11512 case Intrinsic::amdgcn_workgroup_id_z:
11513 return lowerWorkGroupId(DAG, *MFI, VT,
11517 case Intrinsic::amdgcn_cluster_id_x:
11518 return Subtarget->hasClusters()
11519 ? getPreloadedValue(DAG, *MFI, VT,
11521 : DAG.getPOISON(VT);
11522 case Intrinsic::amdgcn_cluster_id_y:
11523 return Subtarget->hasClusters()
11524 ? getPreloadedValue(DAG, *MFI, VT,
11526 : DAG.getPOISON(VT);
11527 case Intrinsic::amdgcn_cluster_id_z:
11528 return Subtarget->hasClusters()
11529 ? getPreloadedValue(DAG, *MFI, VT,
11531 : DAG.getPOISON(VT);
11532 case Intrinsic::amdgcn_cluster_workgroup_id_x:
11533 return Subtarget->hasClusters()
11534 ? getPreloadedValue(
11535 DAG, *MFI, VT,
11537 : DAG.getPOISON(VT);
11538 case Intrinsic::amdgcn_cluster_workgroup_id_y:
11539 return Subtarget->hasClusters()
11540 ? getPreloadedValue(
11541 DAG, *MFI, VT,
11543 : DAG.getPOISON(VT);
11544 case Intrinsic::amdgcn_cluster_workgroup_id_z:
11545 return Subtarget->hasClusters()
11546 ? getPreloadedValue(
11547 DAG, *MFI, VT,
11549 : DAG.getPOISON(VT);
11550 case Intrinsic::amdgcn_cluster_workgroup_flat_id:
11551 return Subtarget->hasClusters()
11552 ? lowerConstHwRegRead(DAG, Op, AMDGPU::Hwreg::ID_IB_STS2, 21, 4)
11553 : SDValue();
11554 case Intrinsic::amdgcn_cluster_workgroup_max_id_x:
11555 return Subtarget->hasClusters()
11556 ? getPreloadedValue(
11557 DAG, *MFI, VT,
11559 : DAG.getPOISON(VT);
11560 case Intrinsic::amdgcn_cluster_workgroup_max_id_y:
11561 return Subtarget->hasClusters()
11562 ? getPreloadedValue(
11563 DAG, *MFI, VT,
11565 : DAG.getPOISON(VT);
11566 case Intrinsic::amdgcn_cluster_workgroup_max_id_z:
11567 return Subtarget->hasClusters()
11568 ? getPreloadedValue(
11569 DAG, *MFI, VT,
11571 : DAG.getPOISON(VT);
11572 case Intrinsic::amdgcn_cluster_workgroup_max_flat_id:
11573 return Subtarget->hasClusters()
11574 ? getPreloadedValue(
11575 DAG, *MFI, VT,
11577 : DAG.getPOISON(VT);
11578 case Intrinsic::amdgcn_wave_id:
11579 return lowerWaveID(DAG, Op);
11580 case Intrinsic::amdgcn_lds_kernel_id: {
11581 if (MFI->isEntryFunction())
11582 return getLDSKernelId(DAG, DL);
11583 return getPreloadedValue(DAG, *MFI, VT,
11585 }
11586 case Intrinsic::amdgcn_workitem_id_x:
11587 return lowerWorkitemID(DAG, Op, 0, MFI->getArgInfo().WorkItemIDX);
11588 case Intrinsic::amdgcn_workitem_id_y:
11589 return lowerWorkitemID(DAG, Op, 1, MFI->getArgInfo().WorkItemIDY);
11590 case Intrinsic::amdgcn_workitem_id_z:
11591 return lowerWorkitemID(DAG, Op, 2, MFI->getArgInfo().WorkItemIDZ);
11592 case Intrinsic::amdgcn_wavefrontsize:
11593 return DAG.getConstant(MF.getSubtarget<GCNSubtarget>().getWavefrontSize(),
11594 SDLoc(Op), MVT::i32);
11595 case Intrinsic::amdgcn_s_buffer_load: {
11596 unsigned CPol = Op.getConstantOperandVal(3);
11597 // s_buffer_load, because of how it's optimized, can't be volatile
11598 // so reject ones with the volatile bit set.
11599 if (CPol & ~((Subtarget->getGeneration() >= AMDGPUSubtarget::GFX12)
11602 return Op;
11603 return lowerSBuffer(VT, VT, DL, DAG.getEntryNode(), Op.getOperand(1),
11604 Op.getOperand(2), Op.getOperand(3), DAG);
11605 }
11606 case Intrinsic::amdgcn_fdiv_fast:
11607 return lowerFDIV_FAST(Op, DAG);
11608 case Intrinsic::amdgcn_sin:
11609 return DAG.getNode(AMDGPUISD::SIN_HW, DL, VT, Op.getOperand(1));
11610
11611 case Intrinsic::amdgcn_cos:
11612 return DAG.getNode(AMDGPUISD::COS_HW, DL, VT, Op.getOperand(1));
11613
11614 case Intrinsic::amdgcn_mul_u24:
11615 return DAG.getNode(AMDGPUISD::MUL_U24, DL, VT, Op.getOperand(1),
11616 Op.getOperand(2));
11617 case Intrinsic::amdgcn_mul_i24:
11618 return DAG.getNode(AMDGPUISD::MUL_I24, DL, VT, Op.getOperand(1),
11619 Op.getOperand(2));
11620
11621 case Intrinsic::amdgcn_log_clamp: {
11622 if (Subtarget->getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS)
11623 return SDValue();
11624
11625 return emitRemovedIntrinsicError(DAG, DL, VT);
11626 }
11627 case Intrinsic::amdgcn_fract:
11628 return DAG.getNode(AMDGPUISD::FRACT, DL, VT, Op.getOperand(1));
11629
11630 case Intrinsic::amdgcn_class: {
11631 SDValue Src = Op.getOperand(1);
11632 EVT SrcVT = Src.getValueType();
11633 bool IsLegal = SrcVT == MVT::f32 || SrcVT == MVT::f64 ||
11634 (SrcVT == MVT::f16 && Subtarget->has16BitInsts());
11635 if (!IsLegal) {
11636 DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
11638 "llvm.amdgcn.class only supports f16, f32, and f64",
11639 DL.getDebugLoc()));
11640 return DAG.getPOISON(VT);
11641 }
11642 return DAG.getNode(AMDGPUISD::FP_CLASS, DL, VT, Src, Op.getOperand(2));
11643 }
11644 case Intrinsic::amdgcn_div_fmas:
11645 return DAG.getNode(AMDGPUISD::DIV_FMAS, DL, VT, Op.getOperand(1),
11646 Op.getOperand(2), Op.getOperand(3), Op.getOperand(4));
11647
11648 case Intrinsic::amdgcn_div_fixup:
11649 return DAG.getNode(AMDGPUISD::DIV_FIXUP, DL, VT, Op.getOperand(1),
11650 Op.getOperand(2), Op.getOperand(3));
11651
11652 case Intrinsic::amdgcn_div_scale: {
11653 const ConstantSDNode *Param = cast<ConstantSDNode>(Op.getOperand(3));
11654
11655 // Translate to the operands expected by the machine instruction. The
11656 // first parameter must be the same as the first instruction.
11657 SDValue Numerator = Op.getOperand(1);
11658 SDValue Denominator = Op.getOperand(2);
11659
11660 // Note this order is opposite of the machine instruction's operations,
11661 // which is s0.f = Quotient, s1.f = Denominator, s2.f = Numerator. The
11662 // intrinsic has the numerator as the first operand to match a normal
11663 // division operation.
11664
11665 SDValue Src0 = Param->isAllOnes() ? Numerator : Denominator;
11666
11667 return DAG.getNode(AMDGPUISD::DIV_SCALE, DL, Op->getVTList(), Src0,
11668 Denominator, Numerator);
11669 }
11670 case Intrinsic::amdgcn_ballot:
11671 return lowerBALLOTIntrinsic(*this, Op.getNode(), DAG);
11672 case Intrinsic::amdgcn_fmed3:
11673 return DAG.getNode(AMDGPUISD::FMED3, DL, VT, Op.getOperand(1),
11674 Op.getOperand(2), Op.getOperand(3), Op->getFlags());
11675 case Intrinsic::amdgcn_fdot2:
11676 return DAG.getNode(AMDGPUISD::FDOT2, DL, VT, Op.getOperand(1),
11677 Op.getOperand(2), Op.getOperand(3), Op.getOperand(4));
11678 case Intrinsic::amdgcn_fmul_legacy:
11679 return DAG.getNode(AMDGPUISD::FMUL_LEGACY, DL, VT, Op.getOperand(1),
11680 Op.getOperand(2));
11681 case Intrinsic::amdgcn_sbfe:
11682 case Intrinsic::amdgcn_ubfe:
11683 return lowerBFEIntrinsic(Op, DAG, IntrinsicID);
11684 case Intrinsic::amdgcn_cvt_pkrtz:
11685 case Intrinsic::amdgcn_cvt_pknorm_i16:
11686 case Intrinsic::amdgcn_cvt_pknorm_u16:
11687 case Intrinsic::amdgcn_cvt_pk_i16:
11688 case Intrinsic::amdgcn_cvt_pk_u16: {
11689 // FIXME: Stop adding cast if v2f16/v2i16 are legal.
11690 EVT VT = Op.getValueType();
11691 unsigned Opcode;
11692
11693 if (IntrinsicID == Intrinsic::amdgcn_cvt_pkrtz)
11694 Opcode = AMDGPUISD::CVT_PKRTZ_F16_F32;
11695 else if (IntrinsicID == Intrinsic::amdgcn_cvt_pknorm_i16)
11696 Opcode = AMDGPUISD::CVT_PKNORM_I16_F32;
11697 else if (IntrinsicID == Intrinsic::amdgcn_cvt_pknorm_u16)
11698 Opcode = AMDGPUISD::CVT_PKNORM_U16_F32;
11699 else if (IntrinsicID == Intrinsic::amdgcn_cvt_pk_i16)
11700 Opcode = AMDGPUISD::CVT_PK_I16_I32;
11701 else
11702 Opcode = AMDGPUISD::CVT_PK_U16_U32;
11703
11704 if (isTypeLegal(VT))
11705 return DAG.getNode(Opcode, DL, VT, Op.getOperand(1), Op.getOperand(2));
11706
11707 SDValue Node =
11708 DAG.getNode(Opcode, DL, MVT::i32, Op.getOperand(1), Op.getOperand(2));
11709 return DAG.getNode(ISD::BITCAST, DL, VT, Node);
11710 }
11711 case Intrinsic::amdgcn_fmad_ftz:
11712 return DAG.getNode(AMDGPUISD::FMAD_FTZ, DL, VT, Op.getOperand(1),
11713 Op.getOperand(2), Op.getOperand(3));
11714
11715 case Intrinsic::amdgcn_if_break:
11716 return SDValue(DAG.getMachineNode(AMDGPU::SI_IF_BREAK, DL, VT,
11717 Op->getOperand(1), Op->getOperand(2)),
11718 0);
11719
11720 case Intrinsic::amdgcn_groupstaticsize: {
11722 if (OS == Triple::AMDHSA || OS == Triple::AMDPAL)
11723 return Op;
11724
11725 const Module *M = MF.getFunction().getParent();
11726 const GlobalValue *GV =
11727 Intrinsic::getDeclarationIfExists(M, Intrinsic::amdgcn_groupstaticsize);
11728 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, MVT::i32, 0,
11730 return {DAG.getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, GA), 0};
11731 }
11732 case Intrinsic::amdgcn_is_shared:
11733 case Intrinsic::amdgcn_is_private: {
11734 SDLoc SL(Op);
11735 SDValue SrcVec =
11736 DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, Op.getOperand(1));
11737 SDValue SrcHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, SrcVec,
11738 DAG.getConstant(1, SL, MVT::i32));
11739
11740 unsigned AS = (IntrinsicID == Intrinsic::amdgcn_is_shared)
11742 : AMDGPUAS::PRIVATE_ADDRESS;
11743 if (AS == AMDGPUAS::PRIVATE_ADDRESS &&
11744 Subtarget->hasGloballyAddressableScratch()) {
11745 SDValue FlatScratchBaseHi(
11746 DAG.getMachineNode(
11747 AMDGPU::S_MOV_B32, DL, MVT::i32,
11748 DAG.getRegister(AMDGPU::SRC_FLAT_SCRATCH_BASE_HI, MVT::i32)),
11749 0);
11750 // Test bits 63..58 against the aperture address.
11751 return DAG.getSetCC(
11752 SL, MVT::i1,
11753 DAG.getNode(ISD::XOR, SL, MVT::i32, SrcHi, FlatScratchBaseHi),
11754 DAG.getConstant(1u << 26, SL, MVT::i32), ISD::SETULT);
11755 }
11756
11757 SDValue Aperture = getSegmentAperture(AS, SL, DAG);
11758 return DAG.getSetCC(SL, MVT::i1, SrcHi, Aperture, ISD::SETEQ);
11759 }
11760 case Intrinsic::amdgcn_perm:
11761 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, Op.getOperand(1),
11762 Op.getOperand(2), Op.getOperand(3));
11763 case Intrinsic::amdgcn_reloc_constant: {
11764 Module *M = MF.getFunction().getParent();
11765 const MDNode *Metadata = cast<MDNodeSDNode>(Op.getOperand(1))->getMD();
11766 auto SymbolName = cast<MDString>(Metadata->getOperand(0))->getString();
11767 auto *RelocSymbol = cast<GlobalVariable>(
11768 M->getOrInsertGlobal(SymbolName, Type::getInt32Ty(M->getContext())));
11769 SDValue GA = DAG.getTargetGlobalAddress(RelocSymbol, DL, MVT::i32, 0,
11771 return {DAG.getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, GA), 0};
11772 }
11773 case Intrinsic::amdgcn_swmmac_f16_16x16x32_f16:
11774 case Intrinsic::amdgcn_swmmac_bf16_16x16x32_bf16:
11775 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf16:
11776 case Intrinsic::amdgcn_swmmac_f32_16x16x32_f16:
11777 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_fp8:
11778 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_bf8:
11779 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_fp8:
11780 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_bf8: {
11781 if (Op.getOperand(4).getValueType() == MVT::i32)
11782 return SDValue();
11783
11784 SDLoc SL(Op);
11785 auto IndexKeyi32 = DAG.getAnyExtOrTrunc(Op.getOperand(4), SL, MVT::i32);
11786 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, Op.getValueType(),
11787 Op.getOperand(0), Op.getOperand(1), Op.getOperand(2),
11788 Op.getOperand(3), IndexKeyi32);
11789 }
11790 case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_fp8:
11791 case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_bf8:
11792 case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_fp8:
11793 case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_bf8:
11794 case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_fp8:
11795 case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_bf8:
11796 case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_fp8:
11797 case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_bf8: {
11798 if (Op.getOperand(4).getValueType() == MVT::i64)
11799 return SDValue();
11800
11801 SDLoc SL(Op);
11802 auto IndexKeyi64 =
11803 Op.getOperand(4).getValueType() == MVT::v2i32
11804 ? DAG.getBitcast(MVT::i64, Op.getOperand(4))
11805 : DAG.getAnyExtOrTrunc(Op.getOperand(4), SL, MVT::i64);
11806 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, Op.getValueType(),
11807 {Op.getOperand(0), Op.getOperand(1), Op.getOperand(2),
11808 Op.getOperand(3), IndexKeyi64, Op.getOperand(5),
11809 Op.getOperand(6)});
11810 }
11811 case Intrinsic::amdgcn_swmmac_f16_16x16x64_f16:
11812 case Intrinsic::amdgcn_swmmac_bf16_16x16x64_bf16:
11813 case Intrinsic::amdgcn_swmmac_f32_16x16x64_bf16:
11814 case Intrinsic::amdgcn_swmmac_bf16f32_16x16x64_bf16:
11815 case Intrinsic::amdgcn_swmmac_f32_16x16x64_f16:
11816 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8: {
11817 EVT IndexKeyTy = IntrinsicID == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8
11818 ? MVT::i64
11819 : MVT::i32;
11820 if (Op.getOperand(6).getValueType() == IndexKeyTy)
11821 return SDValue();
11822
11823 SDLoc SL(Op);
11824 auto IndexKey =
11825 Op.getOperand(6).getValueType().isVector()
11826 ? DAG.getBitcast(IndexKeyTy, Op.getOperand(6))
11827 : DAG.getAnyExtOrTrunc(Op.getOperand(6), SL, IndexKeyTy);
11829 Op.getOperand(0), Op.getOperand(1), Op.getOperand(2),
11830 Op.getOperand(3), Op.getOperand(4), Op.getOperand(5),
11831 IndexKey, Op.getOperand(7), Op.getOperand(8)};
11832 if (IntrinsicID == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8)
11833 Args.push_back(Op.getOperand(9));
11834 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, Op.getValueType(), Args);
11835 }
11836 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu4:
11837 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu8:
11838 case Intrinsic::amdgcn_swmmac_i32_16x16x64_iu4: {
11839 if (Op.getOperand(6).getValueType() == MVT::i32)
11840 return SDValue();
11841
11842 SDLoc SL(Op);
11843 auto IndexKeyi32 = DAG.getAnyExtOrTrunc(Op.getOperand(6), SL, MVT::i32);
11844 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, Op.getValueType(),
11845 {Op.getOperand(0), Op.getOperand(1), Op.getOperand(2),
11846 Op.getOperand(3), Op.getOperand(4), Op.getOperand(5),
11847 IndexKeyi32, Op.getOperand(7)});
11848 }
11849 case Intrinsic::amdgcn_wmma_scale_f32_16x16x128_f8f6f4:
11850 case Intrinsic::amdgcn_wmma_scale16_f32_16x16x128_f8f6f4: {
11851 unsigned AFmt = (unsigned)Op.getConstantOperandVal(1);
11852 unsigned BFmt = (unsigned)Op.getConstantOperandVal(3);
11853 unsigned AScaleFmt = (unsigned)Op.getConstantOperandVal(8);
11854 unsigned BScaleFmt = (unsigned)Op.getConstantOperandVal(11);
11855 if (!AMDGPU::isValidWMMAScaleFmtCombination(AFmt, AScaleFmt, BFmt,
11856 BScaleFmt)) {
11858 "invalid matrix and scale format combination in wmma call");
11859 Op->print(errs());
11860 errs() << '\n';
11861 }
11862 return SDValue();
11863 }
11864 case Intrinsic::amdgcn_readlane:
11865 case Intrinsic::amdgcn_readfirstlane:
11866 case Intrinsic::amdgcn_writelane:
11867 case Intrinsic::amdgcn_permlane16:
11868 case Intrinsic::amdgcn_permlanex16:
11869 case Intrinsic::amdgcn_permlane64:
11870 case Intrinsic::amdgcn_set_inactive:
11871 case Intrinsic::amdgcn_set_inactive_chain_arg:
11872 case Intrinsic::amdgcn_mov_dpp8:
11873 case Intrinsic::amdgcn_update_dpp:
11874 case Intrinsic::amdgcn_permlane_bcast:
11875 case Intrinsic::amdgcn_permlane_up:
11876 case Intrinsic::amdgcn_permlane_down:
11877 case Intrinsic::amdgcn_permlane_xor:
11878 return lowerLaneOp(*this, Op.getNode(), DAG);
11879 case Intrinsic::amdgcn_dead: {
11881 for (const EVT ValTy : Op.getNode()->values())
11882 Poisons.push_back(DAG.getPOISON(ValTy));
11883 return DAG.getMergeValues(Poisons, SDLoc(Op));
11884 }
11885 case Intrinsic::amdgcn_wave_shuffle:
11886 return lowerWaveShuffle(*this, Op.getNode(), DAG);
11887 default:
11888 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
11890 return lowerImage(Op, ImageDimIntr, DAG, false);
11891
11892 return Op;
11893 }
11894}
11895
11896// On targets not supporting constant in soffset field, turn zero to
11897// SGPR_NULL to avoid generating an extra s_mov with zero.
11899 const GCNSubtarget *Subtarget) {
11900 if (Subtarget->hasRestrictedSOffset() && isNullConstant(SOffset))
11901 return DAG.getRegister(AMDGPU::SGPR_NULL, MVT::i32);
11902 return SOffset;
11903}
11904
11905// Lower the arguments of the buffer intrinsic \p Op into the operands of the
11906// corresponding AMDGPUISD buffer node: the chain, \p VData and the cmp operand
11907// if present, then rsrc, vindex, voffset, soffset, offset, [format,]
11908// cachepolicy and idxen.
11909SmallVector<SDValue, 10> SITargetLowering::getBufferIntrinsicOps(
11911 SDValue VData) const {
11912 SDLoc DL(Op);
11913 auto GetArg = [&](unsigned Arg) {
11914 return Op.getOperand(AMDGPU::getSDOperandIdx(Arg));
11915 };
11916
11917 SDValue Rsrc = bufferRsrcPtrToVector(GetArg(Info.RsrcArg), DAG);
11918 auto [VOffset, Offset] =
11919 splitBufferOffsets(GetArg(*Info.getVOffsetArg()), DAG);
11920 SDValue SOffset =
11921 selectSOffset(GetArg(*Info.getSOffsetArg()), DAG, Subtarget);
11922
11923 SmallVector<SDValue, 10> Ops = {Op.getOperand(0)}; // Chain
11924 if (VData)
11925 Ops.push_back(VData);
11926 if (std::optional<unsigned> CmpArg = Info.getCmpArg())
11927 Ops.push_back(GetArg(*CmpArg));
11928 Ops.push_back(Rsrc);
11929 std::optional<unsigned> VIndexArg = Info.getVIndexArg();
11930 Ops.push_back(VIndexArg ? GetArg(*VIndexArg)
11931 : DAG.getConstant(0, DL, MVT::i32)); // vindex
11932 Ops.push_back(VOffset);
11933 Ops.push_back(SOffset);
11934 Ops.push_back(Offset);
11935 if (std::optional<unsigned> FormatArg = Info.getFormatArg())
11936 Ops.push_back(GetArg(*FormatArg));
11937 Ops.push_back(GetArg(*Info.getAuxArg())); // cachepolicy, swizzled buffer
11938 Ops.push_back(
11939 DAG.getTargetConstant(VIndexArg.has_value(), DL, MVT::i1)); // idxen
11940 return Ops;
11941}
11942
11943SDValue SITargetLowering::lowerBufferAtomicIntrin(SDValue Op, SelectionDAG &DAG,
11944 unsigned NewOpcode) const {
11945 SDLoc DL(Op);
11946 const AMDGPU::BufferIntrinsicInfo &Info =
11947 *AMDGPU::lookupBufferIntrinsic(Op.getConstantOperandVal(1));
11948
11949 SDValue VData = Op.getOperand(AMDGPU::getSDOperandIdx(*Info.getVDataArg()));
11950 if (VData.getValueSizeInBits() != 32 && VData.getValueSizeInBits() != 64) {
11951 SmallVector<EVT, 2> ResultTypes(Op->values());
11952 return diagnoseUnsupportedImage(DAG, Op, ResultTypes, DL,
11953 "unsupported buffer atomic data type");
11954 }
11955 SmallVector<SDValue, 10> Ops = getBufferIntrinsicOps(Op, Info, DAG, VData);
11956
11957 auto *M = cast<MemSDNode>(Op);
11958
11959 EVT MemVT = VData.getValueType();
11960 return DAG.getMemIntrinsicNode(NewOpcode, DL, Op->getVTList(), Ops, MemVT,
11961 M->getMemOperand());
11962}
11963
11964// Lower the buffer loads and the compare-and-swap atomics, which are all
11965// INTRINSIC_W_CHAIN nodes. Other atomics need an opcode for their operation,
11966// and so are handled by LowerINTRINSIC_W_CHAIN.
11967SDValue SITargetLowering::lowerBufferIntrinsicWithChain(
11969 SelectionDAG &DAG) const {
11970 SDLoc DL(Op);
11971 auto *M = cast<MemSDNode>(Op);
11972 auto GetArg = [&](unsigned Arg) {
11973 return Op.getOperand(AMDGPU::getSDOperandIdx(Arg));
11974 };
11975
11976 switch (Info.Kind) {
11977 case AMDGPU::BufferKindSLoad: {
11978 SDValue CachePolicy = GetArg(*Info.getAuxArg());
11979 if (CachePolicy->getAsZExtVal() &
11980 ~((Subtarget->getGeneration() >= AMDGPUSubtarget::GFX12)
11983 return Op;
11984
11985 return lowerSBuffer(
11986 Op.getValueType(), M->getMemoryVT(), DL, Op.getOperand(0),
11987 bufferRsrcPtrToVector(GetArg(Info.RsrcArg), DAG),
11988 GetArg(*Info.getSOffsetArg()), CachePolicy, DAG, M->getMemOperand());
11989 }
11990 case AMDGPU::BufferKindLoad:
11991 case AMDGPU::BufferKindAtomicLoad: {
11992 SmallVector<SDValue, 10> Ops = getBufferIntrinsicOps(Op, Info, DAG);
11993 if (!Info.isTyped())
11994 return lowerIntrinsicLoad(M, Info.isFormatFromRsrc(), DAG, Ops);
11995
11996 EVT LoadVT = Op.getValueType();
11997 if (LoadVT.getScalarSizeInBits() == 16)
11998 return adjustLoadValueType(AMDGPUISD::TBUFFER_LOAD_FORMAT_D16, M, DAG,
11999 Ops);
12000 return getMemIntrinsicNode(AMDGPUISD::TBUFFER_LOAD_FORMAT, DL,
12001 Op->getVTList(), Ops, LoadVT, M->getMemOperand(),
12002 DAG);
12003 }
12004 case AMDGPU::BufferKindAtomicCmpSwap: {
12005 SDValue Src = GetArg(*Info.getVDataArg());
12006 if (Src.getValueSizeInBits() != 32 && Src.getValueSizeInBits() != 64) {
12007 SmallVector<EVT, 2> ResultTypes(Op->values());
12008 return diagnoseUnsupportedImage(DAG, Op, ResultTypes, DL,
12009 "unsupported buffer atomic data type");
12010 }
12011 SmallVector<SDValue, 10> Ops = getBufferIntrinsicOps(Op, Info, DAG, Src);
12012 return DAG.getMemIntrinsicNode(AMDGPUISD::BUFFER_ATOMIC_CMPSWAP, DL,
12013 Op->getVTList(), Ops, Op.getValueType(),
12014 M->getMemOperand());
12015 }
12016 default:
12017 llvm_unreachable("not a buffer intrinsic with a chain and a result");
12018 }
12019}
12020
12022 SDLoc DL) {
12023 SDNode *N = Op.getNode();
12024 SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
12025 unsigned NumOperands = N->getNumOperands();
12026 if (N->getOperand(NumOperands - 1) == Zero)
12027 return;
12029 Ops[NumOperands - 1] = Zero; // M0 = 0
12030 DAG.UpdateNodeOperands(N, Ops);
12031}
12032
12033// Try to resolve a named barrier object to a constant barrier ID.
12034// Named barrier IDs are encoded at bits [5:0] of a BARRIER address space
12035// address - other bits of that address are ignored as reserved.
12036static std::optional<uint32_t>
12038 std::optional<uint64_t> BarVal;
12039 if (auto *C = dyn_cast<ConstantSDNode>(BarOp)) {
12040 BarVal = C->getZExtValue();
12041 } else if (BarOp->isUndef()) {
12042 // Refine poison to the 0 barrier to turn things into noops, thus reducing
12043 // the risk of hitting an unallocated barrier.
12044 BarVal = 0;
12045 } else if (auto *GA = dyn_cast<GlobalAddressSDNode>(BarOp)) {
12046 const auto *GV = cast<GlobalVariable>(GA->getGlobal());
12047 if (AMDGPU::isNamedBarrier(*GV) &&
12051 // Record the use of the manualy-specified barrier address to make sure
12052 // this function has the right number of resources. This function is named
12053 // weird, and we're calling it here since we're skipping the GlobalAddress
12054 // lowering with this pattern match.
12055 BarVal = MFI->allocateBarrierGlobal(DAG.getDataLayout(), *GV) +
12056 GA->getOffset();
12057 }
12058 }
12059 if (BarVal)
12060 return *BarVal & 0x3F;
12061 return std::nullopt;
12062}
12063
12064SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
12065 SelectionDAG &DAG) const {
12066 unsigned IntrID = Op.getConstantOperandVal(1);
12067 SDLoc DL(Op);
12068
12069 if (const AMDGPU::BufferIntrinsicInfo *BufInfo =
12071 BufInfo && BufInfo->Kind != AMDGPU::BufferKindAtomicRMW)
12072 return lowerBufferIntrinsicWithChain(Op, *BufInfo, DAG);
12073
12074 switch (IntrID) {
12075 case Intrinsic::amdgcn_cluster_load_b32:
12076 case Intrinsic::amdgcn_cluster_load_b64:
12077 case Intrinsic::amdgcn_cluster_load_b128: {
12078 if (Subtarget->hasGFX1250_STRICT())
12080 return SDValue();
12081 }
12082 case Intrinsic::amdgcn_ds_ordered_add:
12083 case Intrinsic::amdgcn_ds_ordered_swap: {
12084 MemSDNode *M = cast<MemSDNode>(Op);
12085 SDValue Chain = M->getOperand(0);
12086 SDValue M0 = M->getOperand(2);
12087 SDValue Value = M->getOperand(3);
12088 unsigned IndexOperand = M->getConstantOperandVal(7);
12089 unsigned WaveRelease = M->getConstantOperandVal(8);
12090 unsigned WaveDone = M->getConstantOperandVal(9);
12091
12092 unsigned OrderedCountIndex = IndexOperand & 0x3f;
12093 IndexOperand &= ~0x3f;
12094 unsigned CountDw = 0;
12095
12096 if (Subtarget->getGeneration() >= AMDGPUSubtarget::GFX10) {
12097 CountDw = (IndexOperand >> 24) & 0xf;
12098 IndexOperand &= ~(0xf << 24);
12099
12100 if (CountDw < 1 || CountDw > 4) {
12101 const Function &Fn = DAG.getMachineFunction().getFunction();
12102 DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
12103 Fn, "ds_ordered_count: dword count must be between 1 and 4",
12104 DL.getDebugLoc()));
12105 CountDw = 1;
12106 }
12107 }
12108
12109 if (IndexOperand) {
12110 const Function &Fn = DAG.getMachineFunction().getFunction();
12111 DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
12112 Fn, "ds_ordered_count: bad index operand", DL.getDebugLoc()));
12113 }
12114
12115 if (WaveDone && !WaveRelease) {
12116 // TODO: Move this to IR verifier
12117 const Function &Fn = DAG.getMachineFunction().getFunction();
12118 DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
12119 Fn, "ds_ordered_count: wave_done requires wave_release",
12120 DL.getDebugLoc()));
12121 }
12122
12123 unsigned Instruction = IntrID == Intrinsic::amdgcn_ds_ordered_add ? 0 : 1;
12124 unsigned ShaderType =
12126 unsigned Offset0 = OrderedCountIndex << 2;
12127 unsigned Offset1 = WaveRelease | (WaveDone << 1) | (Instruction << 4);
12128
12129 if (Subtarget->getGeneration() >= AMDGPUSubtarget::GFX10)
12130 Offset1 |= (CountDw - 1) << 6;
12131
12132 if (Subtarget->getGeneration() < AMDGPUSubtarget::GFX11)
12133 Offset1 |= ShaderType << 2;
12134
12135 unsigned Offset = Offset0 | (Offset1 << 8);
12136
12137 SDValue Ops[] = {
12138 Chain, Value, DAG.getTargetConstant(Offset, DL, MVT::i16),
12139 copyToM0(DAG, Chain, DL, M0).getValue(1), // Glue
12140 };
12141 return DAG.getMemIntrinsicNode(AMDGPUISD::DS_ORDERED_COUNT, DL,
12142 M->getVTList(), Ops, M->getMemoryVT(),
12143 M->getMemOperand());
12144 }
12145 case Intrinsic::amdgcn_raw_buffer_atomic_fadd:
12146 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd:
12147 case Intrinsic::amdgcn_struct_buffer_atomic_fadd:
12148 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fadd:
12149 return lowerBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_FADD);
12150 case Intrinsic::amdgcn_raw_buffer_atomic_fmin:
12151 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin:
12152 case Intrinsic::amdgcn_struct_buffer_atomic_fmin:
12153 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmin:
12154 return lowerBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_FMIN);
12155 case Intrinsic::amdgcn_raw_buffer_atomic_fmax:
12156 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax:
12157 case Intrinsic::amdgcn_struct_buffer_atomic_fmax:
12158 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmax:
12159 return lowerBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_FMAX);
12160 case Intrinsic::amdgcn_raw_buffer_atomic_swap:
12161 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_swap:
12162 case Intrinsic::amdgcn_struct_buffer_atomic_swap:
12163 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_swap:
12164 return lowerBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_SWAP);
12165 case Intrinsic::amdgcn_raw_buffer_atomic_add:
12166 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_add:
12167 case Intrinsic::amdgcn_struct_buffer_atomic_add:
12168 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_add:
12169 return lowerBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_ADD);
12170 case Intrinsic::amdgcn_raw_buffer_atomic_sub:
12171 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub:
12172 case Intrinsic::amdgcn_struct_buffer_atomic_sub:
12173 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub:
12174 return lowerBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_SUB);
12175 case Intrinsic::amdgcn_raw_buffer_atomic_smin:
12176 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin:
12177 case Intrinsic::amdgcn_struct_buffer_atomic_smin:
12178 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smin:
12179 return lowerBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_SMIN);
12180 case Intrinsic::amdgcn_raw_buffer_atomic_umin:
12181 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin:
12182 case Intrinsic::amdgcn_struct_buffer_atomic_umin:
12183 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umin:
12184 return lowerBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_UMIN);
12185 case Intrinsic::amdgcn_raw_buffer_atomic_smax:
12186 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax:
12187 case Intrinsic::amdgcn_struct_buffer_atomic_smax:
12188 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smax:
12189 return lowerBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_SMAX);
12190 case Intrinsic::amdgcn_raw_buffer_atomic_umax:
12191 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax:
12192 case Intrinsic::amdgcn_struct_buffer_atomic_umax:
12193 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umax:
12194 return lowerBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_UMAX);
12195 case Intrinsic::amdgcn_raw_buffer_atomic_and:
12196 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_and:
12197 case Intrinsic::amdgcn_struct_buffer_atomic_and:
12198 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_and:
12199 return lowerBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_AND);
12200 case Intrinsic::amdgcn_raw_buffer_atomic_or:
12201 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_or:
12202 case Intrinsic::amdgcn_struct_buffer_atomic_or:
12203 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_or:
12204 return lowerBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_OR);
12205 case Intrinsic::amdgcn_raw_buffer_atomic_xor:
12206 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor:
12207 case Intrinsic::amdgcn_struct_buffer_atomic_xor:
12208 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_xor:
12209 return lowerBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_XOR);
12210 case Intrinsic::amdgcn_raw_buffer_atomic_inc:
12211 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_inc:
12212 case Intrinsic::amdgcn_struct_buffer_atomic_inc:
12213 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_inc:
12214 return lowerBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_INC);
12215 case Intrinsic::amdgcn_raw_buffer_atomic_dec:
12216 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_dec:
12217 case Intrinsic::amdgcn_struct_buffer_atomic_dec:
12218 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_dec:
12219 return lowerBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_DEC);
12220 case Intrinsic::amdgcn_raw_buffer_atomic_sub_clamp_u32:
12221 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub_clamp_u32:
12222 case Intrinsic::amdgcn_struct_buffer_atomic_sub_clamp_u32:
12223 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub_clamp_u32:
12224 return lowerBufferAtomicIntrin(Op, DAG, AMDGPUISD::BUFFER_ATOMIC_CSUB);
12225 case Intrinsic::amdgcn_raw_buffer_atomic_cond_sub_u32:
12226 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cond_sub_u32:
12227 case Intrinsic::amdgcn_struct_buffer_atomic_cond_sub_u32:
12228 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cond_sub_u32:
12229 return lowerBufferAtomicIntrin(Op, DAG,
12230 AMDGPUISD::BUFFER_ATOMIC_COND_SUB_U32);
12231 case Intrinsic::amdgcn_image_bvh_dual_intersect_ray:
12232 case Intrinsic::amdgcn_image_bvh8_intersect_ray: {
12233 MemSDNode *M = cast<MemSDNode>(Op);
12234 SDValue NodePtr = M->getOperand(2);
12235 SDValue RayExtent = M->getOperand(3);
12236 SDValue InstanceMask = M->getOperand(4);
12237 SDValue RayOrigin = M->getOperand(5);
12238 SDValue RayDir = M->getOperand(6);
12239 SDValue Offsets = M->getOperand(7);
12240 SDValue TDescr = M->getOperand(8);
12241
12242 assert(NodePtr.getValueType() == MVT::i64);
12243 assert(RayDir.getValueType() == MVT::v3f32);
12244
12245 bool IsBVH8 = IntrID == Intrinsic::amdgcn_image_bvh8_intersect_ray;
12246 const unsigned NumVDataDwords = 10;
12247 const unsigned NumVAddrDwords = IsBVH8 ? 11 : 12;
12248 int Opcode = AMDGPU::getMIMGOpcode(
12249 IsBVH8 ? AMDGPU::IMAGE_BVH8_INTERSECT_RAY
12250 : AMDGPU::IMAGE_BVH_DUAL_INTERSECT_RAY,
12251 AMDGPU::MIMGEncGfx12, NumVDataDwords, NumVAddrDwords);
12252 assert(Opcode != -1);
12253
12255 Ops.push_back(NodePtr);
12256 Ops.push_back(DAG.getBuildVector(
12257 MVT::v2i32, DL,
12258 {DAG.getBitcast(MVT::i32, RayExtent),
12259 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, InstanceMask)}));
12260 Ops.push_back(RayOrigin);
12261 Ops.push_back(RayDir);
12262 Ops.push_back(Offsets);
12263 Ops.push_back(TDescr);
12264 Ops.push_back(M->getChain());
12265
12266 auto *NewNode = DAG.getMachineNode(Opcode, DL, M->getVTList(), Ops);
12267 MachineMemOperand *MemRef = M->getMemOperand();
12268 DAG.setNodeMemRefs(NewNode, {MemRef});
12269 return SDValue(NewNode, 0);
12270 }
12271 case Intrinsic::amdgcn_image_bvh_intersect_ray: {
12272 MemSDNode *M = cast<MemSDNode>(Op);
12273 SDValue NodePtr = M->getOperand(2);
12274 SDValue RayExtent = M->getOperand(3);
12275 SDValue RayOrigin = M->getOperand(4);
12276 SDValue RayDir = M->getOperand(5);
12277 SDValue RayInvDir = M->getOperand(6);
12278 SDValue TDescr = M->getOperand(7);
12279
12280 assert(NodePtr.getValueType() == MVT::i32 ||
12281 NodePtr.getValueType() == MVT::i64);
12282 assert(RayDir.getValueType() == MVT::v3f16 ||
12283 RayDir.getValueType() == MVT::v3f32);
12284
12285 const bool IsGFX11 = AMDGPU::isGFX11(*Subtarget);
12286 const bool IsGFX11Plus = AMDGPU::isGFX11Plus(*Subtarget);
12287 const bool IsGFX12Plus = AMDGPU::isGFX12Plus(*Subtarget);
12288 const bool IsA16 = RayDir.getValueType().getVectorElementType() == MVT::f16;
12289 const bool Is64 = NodePtr.getValueType() == MVT::i64;
12290 const unsigned NumVDataDwords = 4;
12291 const unsigned NumVAddrDwords = IsA16 ? (Is64 ? 9 : 8) : (Is64 ? 12 : 11);
12292 const unsigned NumVAddrs = IsGFX11Plus ? (IsA16 ? 4 : 5) : NumVAddrDwords;
12293 const bool UseNSA = (Subtarget->hasNSAEncoding() &&
12294 NumVAddrs <= Subtarget->getNSAMaxSize()) ||
12295 IsGFX12Plus;
12296 const unsigned BaseOpcodes[2][2] = {
12297 {AMDGPU::IMAGE_BVH_INTERSECT_RAY, AMDGPU::IMAGE_BVH_INTERSECT_RAY_a16},
12298 {AMDGPU::IMAGE_BVH64_INTERSECT_RAY,
12299 AMDGPU::IMAGE_BVH64_INTERSECT_RAY_a16}};
12300 int Opcode;
12301 if (UseNSA) {
12302 Opcode = AMDGPU::getMIMGOpcode(BaseOpcodes[Is64][IsA16],
12303 IsGFX12Plus ? AMDGPU::MIMGEncGfx12
12304 : IsGFX11 ? AMDGPU::MIMGEncGfx11NSA
12305 : AMDGPU::MIMGEncGfx10NSA,
12306 NumVDataDwords, NumVAddrDwords);
12307 } else {
12308 assert(!IsGFX12Plus);
12309 Opcode = AMDGPU::getMIMGOpcode(BaseOpcodes[Is64][IsA16],
12310 IsGFX11 ? AMDGPU::MIMGEncGfx11Default
12311 : AMDGPU::MIMGEncGfx10Default,
12312 NumVDataDwords, NumVAddrDwords);
12313 }
12314 assert(Opcode != -1);
12315
12317
12318 auto packLanes = [&DAG, &Ops, &DL](SDValue Op, bool IsAligned) {
12320 DAG.ExtractVectorElements(Op, Lanes, 0, 3);
12321 if (Lanes[0].getValueSizeInBits() == 32) {
12322 for (unsigned I = 0; I < 3; ++I)
12323 Ops.push_back(DAG.getBitcast(MVT::i32, Lanes[I]));
12324 } else {
12325 if (IsAligned) {
12326 Ops.push_back(DAG.getBitcast(
12327 MVT::i32,
12328 DAG.getBuildVector(MVT::v2f16, DL, {Lanes[0], Lanes[1]})));
12329 Ops.push_back(Lanes[2]);
12330 } else {
12331 SDValue Elt0 = Ops.pop_back_val();
12332 Ops.push_back(DAG.getBitcast(
12333 MVT::i32, DAG.getBuildVector(MVT::v2f16, DL, {Elt0, Lanes[0]})));
12334 Ops.push_back(DAG.getBitcast(
12335 MVT::i32,
12336 DAG.getBuildVector(MVT::v2f16, DL, {Lanes[1], Lanes[2]})));
12337 }
12338 }
12339 };
12340
12341 if (UseNSA && IsGFX11Plus) {
12342 Ops.push_back(NodePtr);
12343 Ops.push_back(DAG.getBitcast(MVT::i32, RayExtent));
12344 Ops.push_back(RayOrigin);
12345 if (IsA16) {
12346 SmallVector<SDValue, 3> DirLanes, InvDirLanes, MergedLanes;
12347 DAG.ExtractVectorElements(RayDir, DirLanes, 0, 3);
12348 DAG.ExtractVectorElements(RayInvDir, InvDirLanes, 0, 3);
12349 for (unsigned I = 0; I < 3; ++I) {
12350 MergedLanes.push_back(DAG.getBitcast(
12351 MVT::i32, DAG.getBuildVector(MVT::v2f16, DL,
12352 {DirLanes[I], InvDirLanes[I]})));
12353 }
12354 Ops.push_back(DAG.getBuildVector(MVT::v3i32, DL, MergedLanes));
12355 } else {
12356 Ops.push_back(RayDir);
12357 Ops.push_back(RayInvDir);
12358 }
12359 } else {
12360 if (Is64)
12361 DAG.ExtractVectorElements(DAG.getBitcast(MVT::v2i32, NodePtr), Ops, 0,
12362 2);
12363 else
12364 Ops.push_back(NodePtr);
12365
12366 Ops.push_back(DAG.getBitcast(MVT::i32, RayExtent));
12367 packLanes(RayOrigin, true);
12368 packLanes(RayDir, true);
12369 packLanes(RayInvDir, false);
12370 }
12371
12372 if (!UseNSA) {
12373 // Build a single vector containing all the operands so far prepared.
12374 if (NumVAddrDwords > 12) {
12375 SDValue Undef = DAG.getPOISON(MVT::i32);
12376 Ops.append(16 - Ops.size(), Undef);
12377 }
12378 assert(Ops.size() >= 8 && Ops.size() <= 12);
12379 SDValue MergedOps =
12380 DAG.getBuildVector(MVT::getVectorVT(MVT::i32, Ops.size()), DL, Ops);
12381 Ops.clear();
12382 Ops.push_back(MergedOps);
12383 }
12384
12385 Ops.push_back(TDescr);
12386 Ops.push_back(DAG.getTargetConstant(IsA16, DL, MVT::i1));
12387 Ops.push_back(M->getChain());
12388
12389 auto *NewNode = DAG.getMachineNode(Opcode, DL, M->getVTList(), Ops);
12390 MachineMemOperand *MemRef = M->getMemOperand();
12391 DAG.setNodeMemRefs(NewNode, {MemRef});
12392 return SDValue(NewNode, 0);
12393 }
12394 case Intrinsic::amdgcn_global_atomic_fmin_num:
12395 case Intrinsic::amdgcn_global_atomic_fmax_num:
12396 case Intrinsic::amdgcn_flat_atomic_fmin_num:
12397 case Intrinsic::amdgcn_flat_atomic_fmax_num: {
12398 MemSDNode *M = cast<MemSDNode>(Op);
12399 SDValue Ops[] = {
12400 M->getOperand(0), // Chain
12401 M->getOperand(2), // Ptr
12402 M->getOperand(3) // Value
12403 };
12404 unsigned Opcode = 0;
12405 switch (IntrID) {
12406 case Intrinsic::amdgcn_global_atomic_fmin_num:
12407 case Intrinsic::amdgcn_flat_atomic_fmin_num: {
12408 Opcode = ISD::ATOMIC_LOAD_FMIN;
12409 break;
12410 }
12411 case Intrinsic::amdgcn_global_atomic_fmax_num:
12412 case Intrinsic::amdgcn_flat_atomic_fmax_num: {
12413 Opcode = ISD::ATOMIC_LOAD_FMAX;
12414 break;
12415 }
12416 default:
12417 llvm_unreachable("unhandled atomic opcode");
12418 }
12419 return DAG.getAtomic(Opcode, SDLoc(Op), M->getMemoryVT(), M->getVTList(),
12420 Ops, M->getMemOperand());
12421 }
12422 case Intrinsic::amdgcn_s_alloc_vgpr: {
12423 SDValue NumVGPRs = Op.getOperand(2);
12424 if (!NumVGPRs->isDivergent())
12425 return Op;
12426
12427 SDValue ReadFirstLaneID =
12428 DAG.getTargetConstant(Intrinsic::amdgcn_readfirstlane, DL, MVT::i32);
12429 NumVGPRs = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
12430 ReadFirstLaneID, NumVGPRs);
12431
12432 return DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, Op->getVTList(),
12433 Op.getOperand(0), Op.getOperand(1), NumVGPRs);
12434 }
12435 case Intrinsic::amdgcn_s_get_barrier_state:
12436 case Intrinsic::amdgcn_s_get_named_barrier_state: {
12437 SDValue Chain = Op->getOperand(0);
12439 unsigned Opc;
12440
12441 std::optional<uint32_t> BarID;
12442 if (IntrID == Intrinsic::amdgcn_s_get_named_barrier_state)
12443 BarID = getConstantNamedBarrierNumber(Op->getOperand(2), DAG);
12444 else if (auto *C = dyn_cast<ConstantSDNode>(Op->getOperand(2)))
12445 BarID = C->getZExtValue();
12446
12447 if (BarID) {
12448 Opc = AMDGPU::S_GET_BARRIER_STATE_IMM;
12449 SDValue K = DAG.getTargetConstant(*BarID, DL, MVT::i32);
12450 Ops.push_back(K);
12451 Ops.push_back(Chain);
12452 } else {
12453 Opc = AMDGPU::S_GET_BARRIER_STATE_M0;
12454 if (IntrID == Intrinsic::amdgcn_s_get_named_barrier_state) {
12455 SDValue M0Val = DAG.getNode(ISD::AND, DL, MVT::i32, Op->getOperand(2),
12456 DAG.getConstant(0x3F, DL, MVT::i32));
12457 Ops.push_back(copyToM0(DAG, Chain, DL, M0Val).getValue(0));
12458 } else
12459 Ops.push_back(copyToM0(DAG, Chain, DL, Op->getOperand(2)).getValue(0));
12460 }
12461
12462 auto *NewMI = DAG.getMachineNode(Opc, DL, Op->getVTList(), Ops);
12463 return SDValue(NewMI, 0);
12464 }
12465 case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
12466 case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
12467 case Intrinsic::amdgcn_cooperative_atomic_load_8x16B: {
12468 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Op);
12469 SDValue Chain = Op->getOperand(0);
12470 SDValue Ptr = Op->getOperand(2);
12471 EVT VT = Op->getValueType(0);
12472 return DAG.getAtomicLoad(ISD::NON_EXTLOAD, DL, MII->getMemoryVT(), VT,
12473 Chain, Ptr, MII->getMemOperand());
12474 }
12475 case Intrinsic::amdgcn_av_load_b8:
12476 case Intrinsic::amdgcn_av_load_b16:
12477 case Intrinsic::amdgcn_av_load_b32:
12478 case Intrinsic::amdgcn_av_load_b64:
12479 case Intrinsic::amdgcn_av_load_b128: {
12480 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Op);
12481 SDValue Chain = Op->getOperand(0);
12482 SDValue Ptr = Op->getOperand(2);
12483 EVT VT = Op->getValueType(0);
12484 // Lower to a regular ISD::LOAD. The MachineMemOperand carries Monotonic
12485 // ordering and syncscope so that SIMemoryLegalizer sets cache policy bits.
12486 // Address space filtering in the load_global/load_flat PatFrags selects
12487 // the correct GLOBAL vs FLAT instruction.
12488 return DAG.getLoad(VT, DL, Chain, Ptr, MII->getMemOperand());
12489 }
12490 case Intrinsic::amdgcn_flat_load_monitor_b32:
12491 case Intrinsic::amdgcn_flat_load_monitor_b64:
12492 case Intrinsic::amdgcn_flat_load_monitor_b128: {
12493 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Op);
12494 SDValue Chain = Op->getOperand(0);
12495 SDValue Ptr = Op->getOperand(2);
12496 return DAG.getMemIntrinsicNode(AMDGPUISD::FLAT_LOAD_MONITOR, DL,
12497 Op->getVTList(), {Chain, Ptr},
12498 MII->getMemoryVT(), MII->getMemOperand());
12499 }
12500 case Intrinsic::amdgcn_global_load_monitor_b32:
12501 case Intrinsic::amdgcn_global_load_monitor_b64:
12502 case Intrinsic::amdgcn_global_load_monitor_b128: {
12503 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Op);
12504 SDValue Chain = Op->getOperand(0);
12505 SDValue Ptr = Op->getOperand(2);
12506 return DAG.getMemIntrinsicNode(AMDGPUISD::GLOBAL_LOAD_MONITOR, DL,
12507 Op->getVTList(), {Chain, Ptr},
12508 MII->getMemoryVT(), MII->getMemOperand());
12509 }
12510 default:
12511
12512 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
12514 return lowerImage(Op, ImageDimIntr, DAG, true);
12515
12516 return SDValue();
12517 }
12518}
12519
12520// Call DAG.getMemIntrinsicNode for a load, but first widen a dwordx3 type to
12521// dwordx4 if on SI and handle TFE loads.
12522SDValue SITargetLowering::getMemIntrinsicNode(unsigned Opcode, const SDLoc &DL,
12523 SDVTList VTList,
12524 ArrayRef<SDValue> Ops, EVT MemVT,
12525 MachineMemOperand *MMO,
12526 SelectionDAG &DAG) const {
12527 LLVMContext &C = *DAG.getContext();
12529 EVT VT = VTList.VTs[0];
12530
12531 assert(VTList.NumVTs == 2 || VTList.NumVTs == 3);
12532 bool IsTFE = VTList.NumVTs == 3;
12533 if (IsTFE) {
12534 unsigned NumValueDWords = divideCeil(VT.getSizeInBits(), 32);
12535 unsigned NumOpDWords = NumValueDWords + 1;
12536 EVT OpDWordsVT = EVT::getVectorVT(C, MVT::i32, NumOpDWords);
12537 SDVTList OpDWordsVTList = DAG.getVTList(OpDWordsVT, VTList.VTs[2]);
12538 MachineMemOperand *OpDWordsMMO =
12539 MF.getMachineMemOperand(MMO, 0, NumOpDWords * 4);
12540 SDValue Op = getMemIntrinsicNode(Opcode, DL, OpDWordsVTList, Ops,
12541 OpDWordsVT, OpDWordsMMO, DAG);
12542 auto [Value, Status] = splitTFEValueAndStatus(Op, VT, DL, DAG);
12543 return DAG.getMergeValues({Value, Status, SDValue(Op.getNode(), 1)}, DL);
12544 }
12545
12546 if (!Subtarget->hasDwordx3LoadStores() &&
12547 (VT == MVT::v3i32 || VT == MVT::v3f32)) {
12548 EVT WidenedVT = EVT::getVectorVT(C, VT.getVectorElementType(), 4);
12549 EVT WidenedMemVT = EVT::getVectorVT(C, MemVT.getVectorElementType(), 4);
12550 MachineMemOperand *WidenedMMO = MF.getMachineMemOperand(MMO, 0, 16);
12551 SDVTList WidenedVTList = DAG.getVTList(WidenedVT, VTList.VTs[1]);
12552 SDValue Op = DAG.getMemIntrinsicNode(Opcode, DL, WidenedVTList, Ops,
12553 WidenedMemVT, WidenedMMO);
12555 DAG.getVectorIdxConstant(0, DL));
12556 return DAG.getMergeValues({Value, SDValue(Op.getNode(), 1)}, DL);
12557 }
12558
12559 return DAG.getMemIntrinsicNode(Opcode, DL, VTList, Ops, MemVT, MMO);
12560}
12561
12562SDValue SITargetLowering::handleD16VData(SDValue VData, SelectionDAG &DAG,
12563 bool ImageStore) const {
12564 EVT StoreVT = VData.getValueType();
12565
12566 // No change for f16 and legal vector D16 types.
12567 if (!StoreVT.isVector())
12568 return VData;
12569
12570 SDLoc DL(VData);
12571 unsigned NumElements = StoreVT.getVectorNumElements();
12572
12573 if (Subtarget->hasUnpackedD16VMem()) {
12574 // We need to unpack the packed data to store.
12575 EVT IntStoreVT = StoreVT.changeTypeToInteger();
12576 SDValue IntVData = DAG.getNode(ISD::BITCAST, DL, IntStoreVT, VData);
12577
12578 EVT EquivStoreVT =
12579 EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumElements);
12580 SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, EquivStoreVT, IntVData);
12581 return DAG.UnrollVectorOp(ZExt.getNode());
12582 }
12583
12584 // The sq block of gfx8.1 does not estimate register use correctly for d16
12585 // image store instructions. The data operand is computed as if it were not a
12586 // d16 image instruction.
12587 if (ImageStore && Subtarget->hasImageStoreD16Bug()) {
12588 // Bitcast to i16
12589 EVT IntStoreVT = StoreVT.changeTypeToInteger();
12590 SDValue IntVData = DAG.getNode(ISD::BITCAST, DL, IntStoreVT, VData);
12591
12592 // Decompose into scalars
12594 DAG.ExtractVectorElements(IntVData, Elts);
12595
12596 // Group pairs of i16 into v2i16 and bitcast to i32
12597 SmallVector<SDValue, 4> PackedElts;
12598 for (unsigned I = 0; I < Elts.size() / 2; I += 1) {
12599 SDValue Pair =
12600 DAG.getBuildVector(MVT::v2i16, DL, {Elts[I * 2], Elts[I * 2 + 1]});
12601 SDValue IntPair = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Pair);
12602 PackedElts.push_back(IntPair);
12603 }
12604 if ((NumElements % 2) == 1) {
12605 // Handle v3i16
12606 unsigned I = Elts.size() / 2;
12607 SDValue Pair = DAG.getBuildVector(MVT::v2i16, DL,
12608 {Elts[I * 2], DAG.getPOISON(MVT::i16)});
12609 SDValue IntPair = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Pair);
12610 PackedElts.push_back(IntPair);
12611 }
12612
12613 // Pad using UNDEF
12614 PackedElts.resize(Elts.size(), DAG.getPOISON(MVT::i32));
12615
12616 // Build final vector
12617 EVT VecVT =
12618 EVT::getVectorVT(*DAG.getContext(), MVT::i32, PackedElts.size());
12619 return DAG.getBuildVector(VecVT, DL, PackedElts);
12620 }
12621
12622 if (NumElements == 3) {
12623 EVT IntStoreVT =
12625 SDValue IntVData = DAG.getNode(ISD::BITCAST, DL, IntStoreVT, VData);
12626
12627 EVT WidenedStoreVT = EVT::getVectorVT(
12628 *DAG.getContext(), StoreVT.getVectorElementType(), NumElements + 1);
12629 EVT WidenedIntVT = EVT::getIntegerVT(*DAG.getContext(),
12630 WidenedStoreVT.getStoreSizeInBits());
12631 SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenedIntVT, IntVData);
12632 return DAG.getNode(ISD::BITCAST, DL, WidenedStoreVT, ZExt);
12633 }
12634
12635 assert(isTypeLegal(StoreVT));
12636 return VData;
12637}
12638
12639static bool isAsyncLDSDMA(Intrinsic::ID Intr) {
12640 if (const AMDGPU::BufferIntrinsicInfo *BufInfo =
12642 return BufInfo->IsAsync;
12643 switch (Intr) {
12644 case Intrinsic::amdgcn_load_async_to_lds:
12645 case Intrinsic::amdgcn_global_load_async_lds:
12646 return true;
12647 }
12648 return false;
12649}
12650
12651// Lower the buffer stores, loads to LDS and prefetches, which are all
12652// INTRINSIC_VOID nodes.
12653SDValue SITargetLowering::lowerBufferIntrinsicVoid(
12655 SelectionDAG &DAG) const {
12656 SDLoc DL(Op);
12657 SDValue Chain = Op.getOperand(0);
12658 auto *M = cast<MemSDNode>(Op);
12659 auto GetArg = [&](unsigned Arg) {
12660 return Op.getOperand(AMDGPU::getSDOperandIdx(Arg));
12661 };
12662
12663 switch (Info.Kind) {
12664 case AMDGPU::BufferKindStore: {
12665 SDValue VData = GetArg(*Info.getVDataArg());
12666 EVT VDataVT = VData.getValueType();
12667 EVT EltType = VDataVT.getScalarType();
12668
12669 if (Info.isTyped()) {
12670 bool IsD16 = EltType.getSizeInBits() == 16;
12671 if (IsD16)
12672 VData = handleD16VData(VData, DAG);
12674 getBufferIntrinsicOps(Op, Info, DAG, VData);
12675 unsigned Opc = IsD16 ? AMDGPUISD::TBUFFER_STORE_FORMAT_D16
12676 : AMDGPUISD::TBUFFER_STORE_FORMAT;
12677 return DAG.getMemIntrinsicNode(Opc, DL, Op->getVTList(), Ops,
12678 M->getMemoryVT(), M->getMemOperand());
12679 }
12680
12681 bool IsFormat = Info.isFormatFromRsrc();
12682 bool IsD16 = IsFormat && (EltType.getSizeInBits() == 16);
12683
12684 if (IsFormat && !IsD16 && EltType.getSizeInBits() < 32) {
12685 DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
12687 "unsupported sub-dword format buffer store", DL.getDebugLoc()));
12688 return Chain;
12689 }
12690
12691 if (IsD16) {
12692 VData = handleD16VData(VData, DAG);
12693 VDataVT = VData.getValueType();
12694 }
12695
12696 if (!isTypeLegal(VDataVT)) {
12697 VData =
12698 DAG.getNode(ISD::BITCAST, DL,
12699 getEquivalentMemType(*DAG.getContext(), VDataVT), VData);
12700 }
12701
12702 SmallVector<SDValue, 10> Ops = getBufferIntrinsicOps(Op, Info, DAG, VData);
12703 unsigned Opc =
12704 IsFormat ? AMDGPUISD::BUFFER_STORE_FORMAT : AMDGPUISD::BUFFER_STORE;
12705 Opc = IsD16 ? AMDGPUISD::BUFFER_STORE_FORMAT_D16 : Opc;
12706
12707 // Handle BUFFER_STORE_BYTE/SHORT overloaded intrinsics
12708 if (!IsD16 && !VDataVT.isVector() && EltType.getSizeInBits() < 32)
12709 return handleByteShortBufferStores(DAG, VDataVT, DL, Ops.data(), M);
12710
12711 return DAG.getMemIntrinsicNode(Opc, DL, Op->getVTList(), Ops,
12712 M->getMemoryVT(), M->getMemOperand());
12713 }
12714 case AMDGPU::BufferKindLoadToLDS: {
12715 unsigned Opc;
12716 std::optional<unsigned> VIndexArg = Info.getVIndexArg();
12717 bool HasVIndex = VIndexArg.has_value();
12718 SDValue VOffset = GetArg(*Info.getVOffsetArg());
12719 bool HasVOffset = !isNullConstant(VOffset);
12720 unsigned Size = GetArg(*Info.getSizeArg())->getAsZExtVal();
12721
12722 switch (Size) {
12723 default:
12724 return SDValue();
12725 case 1:
12726 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_BOTHEN
12727 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_IDXEN
12728 : HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFEN
12729 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFSET;
12730 break;
12731 case 2:
12732 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_BOTHEN
12733 : AMDGPU::BUFFER_LOAD_USHORT_LDS_IDXEN
12734 : HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFEN
12735 : AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFSET;
12736 break;
12737 case 4:
12738 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_BOTHEN
12739 : AMDGPU::BUFFER_LOAD_DWORD_LDS_IDXEN
12740 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFEN
12741 : AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFSET;
12742 break;
12743 case 12:
12744 if (!Subtarget->hasLDSLoadB96_B128())
12745 return SDValue();
12746 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_BOTHEN
12747 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_IDXEN
12748 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFEN
12749 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFSET;
12750 break;
12751 case 16:
12752 if (!Subtarget->hasLDSLoadB96_B128())
12753 return SDValue();
12754 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_BOTHEN
12755 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_IDXEN
12756 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFEN
12757 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFSET;
12758 break;
12759 }
12760
12761 SDValue M0Val = copyToM0(DAG, Chain, DL, GetArg(*Info.getLDSPtrArg()));
12762
12764
12765 if (HasVIndex && HasVOffset)
12766 Ops.push_back(
12767 DAG.getBuildVector(MVT::v2i32, DL, {GetArg(*VIndexArg), VOffset}));
12768 else if (HasVIndex)
12769 Ops.push_back(GetArg(*VIndexArg));
12770 else if (HasVOffset)
12771 Ops.push_back(VOffset);
12772
12773 SDValue Rsrc = bufferRsrcPtrToVector(GetArg(Info.RsrcArg), DAG);
12774 Ops.push_back(Rsrc);
12775 Ops.push_back(GetArg(*Info.getSOffsetArg()));
12776 Ops.push_back(GetArg(*Info.getImmOffsetArg()));
12777 bool IsGFX12Plus = AMDGPU::isGFX12Plus(*Subtarget);
12778 unsigned Aux = GetArg(*Info.getAuxArg())->getAsZExtVal();
12779 Ops.push_back(DAG.getTargetConstant(
12780 Aux & (IsGFX12Plus ? AMDGPU::CPol::ALL : AMDGPU::CPol::ALL_pregfx12),
12781 DL, MVT::i16)); // cpol
12782 Ops.push_back(DAG.getTargetConstant(
12783 Aux & (IsGFX12Plus ? AMDGPU::CPol::SWZ : AMDGPU::CPol::SWZ_pregfx12)
12784 ? 1
12785 : 0,
12786 DL, MVT::i8)); // swz
12787 Ops.push_back(DAG.getTargetConstant(Info.IsAsync, DL, MVT::i8)); // async
12788 Ops.push_back(M0Val.getValue(0)); // Chain
12789 Ops.push_back(M0Val.getValue(1)); // Glue
12790
12791 auto *Load = DAG.getMachineNode(Opc, DL, M->getVTList(), Ops);
12792 DAG.setNodeMemRefs(Load, M->memoperands());
12793
12794 return SDValue(Load, 0);
12795 }
12796 case AMDGPU::BufferKindPrefetch: {
12797 SDValue Ops[] = {
12798 Chain,
12799 bufferRsrcPtrToVector(GetArg(Info.RsrcArg), DAG),
12800 GetArg(*Info.getSOffsetArg()), // soffset
12801 GetArg(*Info.getSizeArg()), // length
12802 };
12803
12804 return DAG.getMemIntrinsicNode(AMDGPUISD::SBUFFER_PREFETCH_DATA, DL,
12805 Op->getVTList(), Ops, M->getMemoryVT(),
12806 M->getMemOperand());
12807 }
12808 default:
12809 llvm_unreachable("not a buffer intrinsic without a result");
12810 }
12811}
12812
12813SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
12814 SelectionDAG &DAG) const {
12815 SDLoc DL(Op);
12816 SDValue Chain = Op.getOperand(0);
12817 unsigned IntrinsicID = Op.getConstantOperandVal(1);
12818
12819 if (const AMDGPU::BufferIntrinsicInfo *BufInfo =
12820 AMDGPU::lookupBufferIntrinsic(IntrinsicID))
12821 return lowerBufferIntrinsicVoid(Op, *BufInfo, DAG);
12822
12823 switch (IntrinsicID) {
12824 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
12825 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
12826 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
12827 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128: {
12828 if (Subtarget->hasGFX1250_STRICT())
12830 return SDValue();
12831 }
12832 case Intrinsic::amdgcn_exp_compr: {
12833 SDValue Src0 = Op.getOperand(4);
12834 SDValue Src1 = Op.getOperand(5);
12835 // Hack around illegal type on SI by directly selecting it.
12836 if (isTypeLegal(Src0.getValueType()))
12837 return SDValue();
12838
12839 const ConstantSDNode *Done = cast<ConstantSDNode>(Op.getOperand(6));
12840 SDValue Undef = DAG.getPOISON(MVT::f32);
12841 const SDValue Ops[] = {
12842 Op.getOperand(2), // tgt
12843 DAG.getNode(ISD::BITCAST, DL, MVT::f32, Src0), // src0
12844 DAG.getNode(ISD::BITCAST, DL, MVT::f32, Src1), // src1
12845 Undef, // src2
12846 Undef, // src3
12847 Op.getOperand(7), // vm
12848 DAG.getTargetConstant(1, DL, MVT::i1), // compr
12849 Op.getOperand(3), // en
12850 Op.getOperand(0) // Chain
12851 };
12852
12853 unsigned Opc = Done->isZero() ? AMDGPU::EXP : AMDGPU::EXP_DONE;
12854 return SDValue(DAG.getMachineNode(Opc, DL, Op->getVTList(), Ops), 0);
12855 }
12856
12857 // Buffers are handled by LowerBufferFatPointers, and we're going to go
12858 // for "trust me" that the remaining cases are global pointers until
12859 // such time as we can put two mem operands on an intrinsic.
12860 case Intrinsic::amdgcn_load_to_lds:
12861 case Intrinsic::amdgcn_load_async_to_lds:
12862 case Intrinsic::amdgcn_global_load_lds:
12863 case Intrinsic::amdgcn_global_load_async_lds: {
12864 if (!Subtarget->hasVMemToLDSLoad())
12865 return SDValue();
12866
12867 unsigned Opc;
12868 unsigned Size = Op->getConstantOperandVal(4);
12869 switch (Size) {
12870 default:
12871 return SDValue();
12872 case 1:
12873 Opc = AMDGPU::GLOBAL_LOAD_LDS_UBYTE;
12874 break;
12875 case 2:
12876 Opc = AMDGPU::GLOBAL_LOAD_LDS_USHORT;
12877 break;
12878 case 4:
12879 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORD;
12880 break;
12881 case 12:
12882 if (!Subtarget->hasLDSLoadB96_B128())
12883 return SDValue();
12884 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX3;
12885 break;
12886 case 16:
12887 if (!Subtarget->hasLDSLoadB96_B128())
12888 return SDValue();
12889 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX4;
12890 break;
12891 }
12892
12893 SDValue M0Val = copyToM0(DAG, Chain, DL, Op.getOperand(3));
12894
12896
12897 SDValue Addr = Op.getOperand(2); // Global ptr
12898 SDValue VOffset;
12899 // Try to split SAddr and VOffset. Global and LDS pointers share the same
12900 // immediate offset, so we cannot use a regular SelectGlobalSAddr().
12901 if (Addr->isDivergent() && Addr->isAnyAdd()) {
12902 SDValue LHS = Addr.getOperand(0);
12903 SDValue RHS = Addr.getOperand(1);
12904
12905 if (LHS->isDivergent())
12906 std::swap(LHS, RHS);
12907
12908 if (!LHS->isDivergent() && RHS.getOpcode() == ISD::ZERO_EXTEND &&
12909 RHS.getOperand(0).getValueType() == MVT::i32) {
12910 // add (i64 sgpr), (zero_extend (i32 vgpr))
12911 Addr = LHS;
12912 VOffset = RHS.getOperand(0);
12913 }
12914 }
12915
12916 Ops.push_back(Addr);
12917 if (!Addr->isDivergent()) {
12919 if (!VOffset)
12920 VOffset =
12921 SDValue(DAG.getMachineNode(AMDGPU::V_MOV_B32_e32, DL, MVT::i32,
12922 DAG.getTargetConstant(0, DL, MVT::i32)),
12923 0);
12924 Ops.push_back(VOffset);
12925 }
12926
12927 Ops.push_back(Op.getOperand(5)); // Offset
12928
12929 unsigned Aux = Op.getConstantOperandVal(6);
12930 Ops.push_back(DAG.getTargetConstant(Aux & ~AMDGPU::CPol::VIRTUAL_BITS, DL,
12931 MVT::i32)); // CPol
12932 Ops.push_back(
12933 DAG.getTargetConstant(isAsyncLDSDMA(IntrinsicID), DL, MVT::i8));
12934
12935 Ops.push_back(M0Val.getValue(0)); // Chain
12936 Ops.push_back(M0Val.getValue(1)); // Glue
12937
12938 auto *M = cast<MemSDNode>(Op);
12939 auto *Load = DAG.getMachineNode(Opc, DL, Op->getVTList(), Ops);
12940 DAG.setNodeMemRefs(Load, M->memoperands());
12941
12942 return SDValue(Load, 0);
12943 }
12944 case Intrinsic::amdgcn_end_cf:
12945 return SDValue(DAG.getMachineNode(AMDGPU::SI_END_CF, DL, MVT::Other,
12946 Op->getOperand(2), Chain),
12947 0);
12948 case Intrinsic::amdgcn_s_barrier_signal_var: {
12949 // Member count of 0 means to re-use a previous member count,
12950 // which, if the named barrier is statically chosen, means we can use
12951 // the immarg form. Otherwisee, fall through to constructiong M0 as for
12952 // s_barrier_init.
12953 SDValue CntOp = Op->getOperand(3);
12954 auto *CntC = dyn_cast<ConstantSDNode>(CntOp);
12955 if (CntC && CntC->isZero()) {
12956 SDValue Chain = Op->getOperand(0);
12957 SDValue BarOp = Op->getOperand(2);
12958
12959 if (auto BarID = getConstantNamedBarrierNumber(BarOp, DAG)) {
12960 std::array<SDValue, 2> Ops = {
12961 DAG.getTargetConstant(*BarID, DL, MVT::i32), Chain};
12962 auto *NewMI = DAG.getMachineNode(AMDGPU::S_BARRIER_SIGNAL_IMM, DL,
12963 Op->getVTList(), Ops);
12964 return SDValue(NewMI, 0);
12965 }
12966 }
12967 [[fallthrough]];
12968 }
12969 case Intrinsic::amdgcn_s_barrier_init: {
12970 // these two intrinsics have two operands: barrier pointer and member count
12971 SDValue Chain = Op->getOperand(0);
12973 SDValue BarOp = Op->getOperand(2);
12974 SDValue CntOp = Op->getOperand(3);
12975 SDValue M0Val;
12976 unsigned Opc = IntrinsicID == Intrinsic::amdgcn_s_barrier_init
12977 ? AMDGPU::S_BARRIER_INIT_M0
12978 : AMDGPU::S_BARRIER_SIGNAL_M0;
12979 // extract the BarrierID from bits 0-5 of BarOp
12980 SDValue BarID = DAG.getNode(ISD::AND, DL, MVT::i32, BarOp,
12981 DAG.getConstant(0x3F, DL, MVT::i32));
12982 // Member count should be put into M0[ShAmt:+CntMask width]
12983 // Barrier ID should be put into M0[5:0]
12984 unsigned CntMask = AMDGPU::getBarrierMemberCountBitMask(
12985 AMDGPU::getIsaVersion(Subtarget->getCPU()));
12986 SDValue MemberCnt = DAG.getNode(ISD::AND, DL, MVT::i32, CntOp,
12987 DAG.getConstant(CntMask, DL, MVT::i32));
12988 constexpr unsigned ShAmt = 16;
12989 M0Val = DAG.getNode(ISD::SHL, DL, MVT::i32, MemberCnt,
12990 DAG.getShiftAmountConstant(ShAmt, MVT::i32, DL));
12991
12992 M0Val = DAG.getNode(ISD::OR, DL, MVT::i32, M0Val, BarID);
12993
12994 Ops.push_back(copyToM0(DAG, Chain, DL, M0Val).getValue(0));
12995
12996 auto *NewMI = DAG.getMachineNode(Opc, DL, Op->getVTList(), Ops);
12997 return SDValue(NewMI, 0);
12998 }
12999 case Intrinsic::amdgcn_s_wakeup_barrier: {
13000 if (!Subtarget->hasSWakeupBarrier())
13001 return SDValue();
13002 [[fallthrough]];
13003 }
13004 case Intrinsic::amdgcn_s_barrier_join: {
13005 // these three intrinsics have one operand: barrier pointer
13006 SDValue Chain = Op->getOperand(0);
13008 SDValue BarOp = Op->getOperand(2);
13009 unsigned Opc;
13010
13011 if (auto BarID = getConstantNamedBarrierNumber(BarOp, DAG)) {
13012 switch (IntrinsicID) {
13013 default:
13014 return SDValue();
13015 case Intrinsic::amdgcn_s_barrier_join:
13016 Opc = AMDGPU::S_BARRIER_JOIN_IMM;
13017 break;
13018 case Intrinsic::amdgcn_s_wakeup_barrier:
13019 Opc = AMDGPU::S_WAKEUP_BARRIER_IMM;
13020 break;
13021 }
13022 SDValue K = DAG.getTargetConstant(*BarID, DL, MVT::i32);
13023 Ops.push_back(K);
13024 Ops.push_back(Chain);
13025 } else {
13026 switch (IntrinsicID) {
13027 default:
13028 return SDValue();
13029 case Intrinsic::amdgcn_s_barrier_join:
13030 Opc = AMDGPU::S_BARRIER_JOIN_M0;
13031 break;
13032 case Intrinsic::amdgcn_s_wakeup_barrier:
13033 Opc = AMDGPU::S_WAKEUP_BARRIER_M0;
13034 break;
13035 }
13036 // extract the BarrierID from bits 0-5 of BarOp, copy to M0[5:0]
13037 SDValue M0Val = DAG.getNode(ISD::AND, DL, MVT::i32, BarOp,
13038 DAG.getConstant(0x3F, DL, MVT::i32));
13039 Ops.push_back(copyToM0(DAG, Chain, DL, M0Val).getValue(0));
13040 }
13041
13042 auto *NewMI = DAG.getMachineNode(Opc, DL, Op->getVTList(), Ops);
13043 return SDValue(NewMI, 0);
13044 }
13045 case Intrinsic::amdgcn_s_prefetch_data:
13046 case Intrinsic::amdgcn_s_prefetch_inst: {
13047 // For non-global address space preserve the chain and remove the call.
13049 return Op.getOperand(0);
13050 return Op;
13051 }
13052 case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
13053 case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
13054 case Intrinsic::amdgcn_cooperative_atomic_store_8x16B: {
13055 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Op);
13056 SDValue Chain = Op->getOperand(0);
13057 SDValue Ptr = Op->getOperand(2);
13058 SDValue Val = Op->getOperand(3);
13059 return DAG.getAtomic(ISD::ATOMIC_STORE, DL, MII->getMemoryVT(), Chain, Val,
13060 Ptr, MII->getMemOperand());
13061 }
13062 case Intrinsic::amdgcn_av_store_b8:
13063 case Intrinsic::amdgcn_av_store_b16:
13064 case Intrinsic::amdgcn_av_store_b32:
13065 case Intrinsic::amdgcn_av_store_b64:
13066 case Intrinsic::amdgcn_av_store_b128: {
13067 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Op);
13068 SDValue Chain = Op->getOperand(0);
13069 SDValue Ptr = Op->getOperand(2);
13070 SDValue Val = Op->getOperand(3);
13071 return DAG.getStore(Chain, DL, Val, Ptr, MII->getMemOperand());
13072 }
13073 default: {
13074 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
13076 return lowerImage(Op, ImageDimIntr, DAG, true);
13077
13078 return Op;
13079 }
13080 }
13081}
13082
13083// Return whether the operation has NoUnsignedWrap property.
13084static bool isNoUnsignedWrap(SDValue Addr) {
13085 return (Addr.getOpcode() == ISD::ADD &&
13086 Addr->getFlags().hasNoUnsignedWrap()) ||
13087 Addr->getOpcode() == ISD::OR;
13088}
13089
13091 EVT PtrVT) const {
13092 return PtrVT == MVT::i64;
13093}
13094
13096 EVT PtrVT) const {
13097 return true;
13098}
13099
13100// The raw.(t)buffer and struct.(t)buffer intrinsics have two offset args:
13101// offset (the offset that is included in bounds checking and swizzling, to be
13102// split between the instruction's voffset and immoffset fields) and soffset
13103// (the offset that is excluded from bounds checking and swizzling, to go in
13104// the instruction's soffset field). This function takes the first kind of
13105// offset and figures out how to split it between voffset and immoffset.
13106std::pair<SDValue, SDValue>
13107SITargetLowering::splitBufferOffsets(SDValue Offset, SelectionDAG &DAG) const {
13108 SDLoc DL(Offset);
13109 const unsigned MaxImm = SIInstrInfo::getMaxMUBUFImmOffset(*Subtarget);
13110 SDValue N0 = Offset;
13111 ConstantSDNode *C1 = nullptr;
13112
13113 if ((C1 = dyn_cast<ConstantSDNode>(N0)))
13114 N0 = SDValue();
13115 else if (DAG.isBaseWithConstantOffset(N0)) {
13116 // On GFX1250+, voffset and immoffset are zero-extended from 32 bits before
13117 // being added, so we can only safely match a 32-bit addition with no
13118 // unsigned overflow.
13119 bool CheckNUW = Subtarget->hasGFX1250Insts();
13120 if (!CheckNUW || isNoUnsignedWrap(N0)) {
13122 N0 = N0.getOperand(0);
13123 }
13124 }
13125
13126 if (C1) {
13127 unsigned ImmOffset = C1->getZExtValue();
13128 // If the immediate value is too big for the immoffset field, put only bits
13129 // that would normally fit in the immoffset field. The remaining value that
13130 // is copied/added for the voffset field is a large power of 2, and it
13131 // stands more chance of being CSEd with the copy/add for another similar
13132 // load/store.
13133 // However, do not do that rounding down if that is a negative
13134 // number, as it appears to be illegal to have a negative offset in the
13135 // vgpr, even if adding the immediate offset makes it positive.
13136 unsigned Overflow = ImmOffset & ~MaxImm;
13137 ImmOffset -= Overflow;
13138 if ((int32_t)Overflow < 0) {
13139 Overflow += ImmOffset;
13140 ImmOffset = 0;
13141 }
13142 C1 = cast<ConstantSDNode>(DAG.getTargetConstant(ImmOffset, DL, MVT::i32));
13143 if (Overflow) {
13144 auto OverflowVal = DAG.getConstant(Overflow, DL, MVT::i32);
13145 if (!N0)
13146 N0 = OverflowVal;
13147 else {
13148 SDValue Ops[] = {N0, OverflowVal};
13149 N0 = DAG.getNode(ISD::ADD, DL, MVT::i32, Ops);
13150 }
13151 }
13152 }
13153 if (!N0)
13154 N0 = DAG.getConstant(0, DL, MVT::i32);
13155 if (!C1)
13156 C1 = cast<ConstantSDNode>(DAG.getTargetConstant(0, DL, MVT::i32));
13157 return {N0, SDValue(C1, 0)};
13158}
13159
13160// Analyze a combined offset from an amdgcn_s_buffer_load intrinsic and store
13161// the three offsets (voffset, soffset and instoffset) into the SDValue[3] array
13162// pointed to by Offsets.
13163void SITargetLowering::setBufferOffsets(SDValue CombinedOffset,
13164 SelectionDAG &DAG, SDValue *Offsets,
13165 Align Alignment) const {
13166 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
13167 SDLoc DL(CombinedOffset);
13168 if (auto *C = dyn_cast<ConstantSDNode>(CombinedOffset)) {
13169 uint32_t Imm = C->getZExtValue();
13170 uint32_t SOffset, ImmOffset;
13171 if (TII->splitMUBUFOffset(Imm, SOffset, ImmOffset, Alignment)) {
13172 Offsets[0] = DAG.getConstant(0, DL, MVT::i32);
13173 Offsets[1] = DAG.getConstant(SOffset, DL, MVT::i32);
13174 Offsets[2] = DAG.getTargetConstant(ImmOffset, DL, MVT::i32);
13175 return;
13176 }
13177 }
13178 if (DAG.isBaseWithConstantOffset(CombinedOffset)) {
13179 // On GFX1250+, voffset and immoffset are zero-extended from 32 bits before
13180 // being added, so we can only safely match a 32-bit addition with no
13181 // unsigned overflow.
13182 bool CheckNUW = Subtarget->hasGFX1250Insts();
13183 SDValue N0 = CombinedOffset.getOperand(0);
13184 SDValue N1 = CombinedOffset.getOperand(1);
13185 uint32_t SOffset, ImmOffset;
13186 int Offset = cast<ConstantSDNode>(N1)->getSExtValue();
13187 if (Offset >= 0 && (!CheckNUW || isNoUnsignedWrap(CombinedOffset)) &&
13188 TII->splitMUBUFOffset(Offset, SOffset, ImmOffset, Alignment)) {
13189 Offsets[0] = N0;
13190 Offsets[1] = DAG.getConstant(SOffset, DL, MVT::i32);
13191 Offsets[2] = DAG.getTargetConstant(ImmOffset, DL, MVT::i32);
13192 return;
13193 }
13194 }
13195
13196 SDValue SOffsetZero = Subtarget->hasRestrictedSOffset()
13197 ? DAG.getRegister(AMDGPU::SGPR_NULL, MVT::i32)
13198 : DAG.getConstant(0, DL, MVT::i32);
13199
13200 Offsets[0] = CombinedOffset;
13201 Offsets[1] = SOffsetZero;
13202 Offsets[2] = DAG.getTargetConstant(0, DL, MVT::i32);
13203}
13204
13205SDValue SITargetLowering::bufferRsrcPtrToVector(SDValue MaybePointer,
13206 SelectionDAG &DAG) const {
13207 if (!MaybePointer.getValueType().isScalarInteger())
13208 return MaybePointer;
13209
13210 SDValue Rsrc = DAG.getBitcast(MVT::v4i32, MaybePointer);
13211 return Rsrc;
13212}
13213
13214// Wrap a global or flat pointer into a buffer intrinsic using the flags
13215// specified in the intrinsic.
13216SDValue SITargetLowering::lowerPointerAsRsrcIntrin(SDNode *Op,
13217 SelectionDAG &DAG) const {
13218 SDLoc Loc(Op);
13219
13220 SDValue Pointer = Op->getOperand(1);
13221 SDValue Stride = Op->getOperand(2);
13222 SDValue NumRecords = Op->getOperand(3);
13223 SDValue Flags = Op->getOperand(4);
13224
13225 SDValue ExtStride = DAG.getAnyExtOrTrunc(Stride, Loc, MVT::i32);
13226 SDValue Rsrc;
13227
13228 if (Subtarget->getBufferResourceNumRecordsWidth() == 45) {
13229 NumRecords = DAG.getZExtOrTrunc(NumRecords, Loc, MVT::i64);
13230 NumRecords = DAG.getNode(ISD::AND, Loc, MVT::i64, NumRecords,
13231 DAG.getConstant((1ULL << 45) - 1, Loc, MVT::i64));
13232 SDValue Zero = DAG.getConstant(0, Loc, MVT::i32);
13233 // Build the lower 64-bit value, which has a 57-bit base and the lower 7-bit
13234 // num_records.
13235 SDValue ExtPointer = DAG.getAnyExtOrTrunc(Pointer, Loc, MVT::i64);
13236 SDValue NumRecordsLHS =
13237 DAG.getNode(ISD::SHL, Loc, MVT::i64, NumRecords,
13238 DAG.getShiftAmountConstant(57, MVT::i32, Loc));
13239 SDValue LowHalf =
13240 DAG.getNode(ISD::OR, Loc, MVT::i64, ExtPointer, NumRecordsLHS);
13241
13242 // Build the higher 64-bit value, which has the higher 38-bit num_records,
13243 // 6-bit zero (omit), 16-bit stride and scale and 4-bit flag.
13244 SDValue NumRecordsRHS =
13245 DAG.getNode(ISD::SRL, Loc, MVT::i64, NumRecords,
13246 DAG.getShiftAmountConstant(7, MVT::i32, Loc));
13247 SDValue ShiftedStride =
13248 DAG.getNode(ISD::SHL, Loc, MVT::i32, ExtStride,
13249 DAG.getShiftAmountConstant(12, MVT::i32, Loc));
13250 SDValue ExtShiftedStrideVec =
13251 DAG.getNode(ISD::BUILD_VECTOR, Loc, MVT::v2i32, Zero, ShiftedStride);
13252 SDValue ExtShiftedStride =
13253 DAG.getNode(ISD::BITCAST, Loc, MVT::i64, ExtShiftedStrideVec);
13254 SDValue ShiftedFlags =
13255 DAG.getNode(ISD::SHL, Loc, MVT::i32, Flags,
13256 DAG.getShiftAmountConstant(28, MVT::i32, Loc));
13257 SDValue ExtShiftedFlagsVec =
13258 DAG.getNode(ISD::BUILD_VECTOR, Loc, MVT::v2i32, Zero, ShiftedFlags);
13259 SDValue ExtShiftedFlags =
13260 DAG.getNode(ISD::BITCAST, Loc, MVT::i64, ExtShiftedFlagsVec);
13261 SDValue CombinedFields =
13262 DAG.getNode(ISD::OR, Loc, MVT::i64, NumRecordsRHS, ExtShiftedStride);
13263 SDValue HighHalf =
13264 DAG.getNode(ISD::OR, Loc, MVT::i64, CombinedFields, ExtShiftedFlags);
13265
13266 Rsrc = DAG.getNode(ISD::BUILD_VECTOR, Loc, MVT::v2i64, LowHalf, HighHalf);
13267 } else {
13268 NumRecords = DAG.getZExtOrTrunc(NumRecords, Loc, MVT::i32);
13269 auto [LowHalf, HighHalf] =
13270 DAG.SplitScalar(Pointer, Loc, MVT::i32, MVT::i32);
13271 SDValue Mask = DAG.getConstant(0x0000ffff, Loc, MVT::i32);
13272 SDValue Masked = DAG.getNode(ISD::AND, Loc, MVT::i32, HighHalf, Mask);
13273 SDValue ShiftedStride =
13274 DAG.getNode(ISD::SHL, Loc, MVT::i32, ExtStride,
13275 DAG.getShiftAmountConstant(16, MVT::i32, Loc));
13276 SDValue NewHighHalf =
13277 DAG.getNode(ISD::OR, Loc, MVT::i32, Masked, ShiftedStride);
13278
13279 Rsrc = DAG.getNode(ISD::BUILD_VECTOR, Loc, MVT::v4i32, LowHalf, NewHighHalf,
13280 NumRecords, Flags);
13281 }
13282
13283 SDValue RsrcPtr = DAG.getNode(ISD::BITCAST, Loc, MVT::i128, Rsrc);
13284 return RsrcPtr;
13285}
13286
13287// Handle 8 bit and 16 bit buffer loads
13288SDValue SITargetLowering::handleByteShortBufferLoads(SelectionDAG &DAG,
13289 EVT LoadVT, SDLoc DL,
13291 MachineMemOperand *MMO,
13292 bool IsTFE) const {
13293 EVT IntVT = LoadVT.changeTypeToInteger();
13294
13295 if (IsTFE) {
13296 unsigned Opc = (LoadVT.getScalarType() == MVT::i8)
13297 ? AMDGPUISD::BUFFER_LOAD_UBYTE_TFE
13298 : AMDGPUISD::BUFFER_LOAD_USHORT_TFE;
13300 MachineMemOperand *OpMMO = MF.getMachineMemOperand(MMO, 0, 8);
13301 SDVTList VTs = DAG.getVTList(MVT::v2i32, MVT::Other);
13302 SDValue Op = getMemIntrinsicNode(Opc, DL, VTs, Ops, MVT::v2i32, OpMMO, DAG);
13303 SDValue Status = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Op,
13304 DAG.getConstant(1, DL, MVT::i32));
13305 SDValue Data = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Op,
13306 DAG.getConstant(0, DL, MVT::i32));
13307 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, IntVT, Data);
13308 SDValue Value = DAG.getNode(ISD::BITCAST, DL, LoadVT, Trunc);
13309 return DAG.getMergeValues({Value, Status, SDValue(Op.getNode(), 1)}, DL);
13310 }
13311
13312 unsigned Opc = LoadVT.getScalarType() == MVT::i8
13313 ? AMDGPUISD::BUFFER_LOAD_UBYTE
13314 : AMDGPUISD::BUFFER_LOAD_USHORT;
13315
13316 SDVTList ResList = DAG.getVTList(MVT::i32, MVT::Other);
13317 SDValue BufferLoad =
13318 DAG.getMemIntrinsicNode(Opc, DL, ResList, Ops, IntVT, MMO);
13319 SDValue LoadVal = DAG.getNode(ISD::TRUNCATE, DL, IntVT, BufferLoad);
13320 LoadVal = DAG.getNode(ISD::BITCAST, DL, LoadVT, LoadVal);
13321
13322 return DAG.getMergeValues({LoadVal, BufferLoad.getValue(1)}, DL);
13323}
13324
13325// Handle 8 bit and 16 bit buffer stores
13326SDValue SITargetLowering::handleByteShortBufferStores(SelectionDAG &DAG,
13327 EVT VDataType, SDLoc DL,
13328 SDValue Ops[],
13329 MemSDNode *M) const {
13330 if (VDataType == MVT::f16 || VDataType == MVT::bf16)
13331 Ops[1] = DAG.getNode(ISD::BITCAST, DL, MVT::i16, Ops[1]);
13332
13333 SDValue BufferStoreExt = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Ops[1]);
13334 Ops[1] = BufferStoreExt;
13335 unsigned Opc = (VDataType == MVT::i8) ? AMDGPUISD::BUFFER_STORE_BYTE
13336 : AMDGPUISD::BUFFER_STORE_SHORT;
13337 ArrayRef<SDValue> OpsRef = ArrayRef(&Ops[0], 9);
13338 return DAG.getMemIntrinsicNode(Opc, DL, M->getVTList(), OpsRef, VDataType,
13339 M->getMemOperand());
13340}
13341
13343 SDValue Op, const SDLoc &SL, EVT VT) {
13344 if (VT.bitsLT(Op.getValueType()))
13345 return DAG.getNode(ISD::TRUNCATE, SL, VT, Op);
13346
13347 switch (ExtType) {
13348 case ISD::SEXTLOAD:
13349 return DAG.getNode(ISD::SIGN_EXTEND, SL, VT, Op);
13350 case ISD::ZEXTLOAD:
13351 return DAG.getNode(ISD::ZERO_EXTEND, SL, VT, Op);
13352 case ISD::EXTLOAD:
13353 return DAG.getNode(ISD::ANY_EXTEND, SL, VT, Op);
13354 case ISD::NON_EXTLOAD:
13355 return Op;
13356 }
13357
13358 llvm_unreachable("invalid ext type");
13359}
13360
13361// Try to turn 8 and 16-bit scalar loads into SMEM eligible 32-bit loads.
13362// TODO: Skip this on GFX12 which does have scalar sub-dword loads.
13363SDValue SITargetLowering::widenLoad(LoadSDNode *Ld,
13364 DAGCombinerInfo &DCI) const {
13365 SelectionDAG &DAG = DCI.DAG;
13366 if (Ld->getAlign() < Align(4) || Ld->isDivergent())
13367 return SDValue();
13368
13369 // FIXME: Constant loads should all be marked invariant.
13370 unsigned AS = Ld->getAddressSpace();
13371 if (AS != AMDGPUAS::CONSTANT_ADDRESS &&
13373 (AS != AMDGPUAS::GLOBAL_ADDRESS || !Ld->isInvariant()))
13374 return SDValue();
13375
13376 // Don't do this early, since it may interfere with adjacent load merging for
13377 // illegal types. We can avoid losing alignment information for exotic types
13378 // pre-legalize.
13379 EVT MemVT = Ld->getMemoryVT();
13380 if ((MemVT.isSimple() && !DCI.isAfterLegalizeDAG()) ||
13381 MemVT.getSizeInBits() >= 32)
13382 return SDValue();
13383
13384 SDLoc SL(Ld);
13385
13386 assert((!MemVT.isVector() || Ld->getExtensionType() == ISD::NON_EXTLOAD) &&
13387 "unexpected vector extload");
13388
13389 // TODO: Drop only high part of range.
13390 SDValue Ptr = Ld->getBasePtr();
13391 SDValue NewLoad = DAG.getLoad(
13392 ISD::UNINDEXED, ISD::NON_EXTLOAD, MVT::i32, SL, Ld->getChain(), Ptr,
13393 Ld->getOffset(), Ld->getPointerInfo(), MVT::i32, Ld->getAlign(),
13394 Ld->getMemOperand()->getFlags(), Ld->getAAInfo()); // Drop ranges
13395
13396 EVT TruncVT = EVT::getIntegerVT(*DAG.getContext(), MemVT.getSizeInBits());
13397 if (MemVT.isFloatingPoint()) {
13398 assert(Ld->getExtensionType() == ISD::NON_EXTLOAD &&
13399 "unexpected fp extload");
13400 TruncVT = MemVT.changeTypeToInteger();
13401 }
13402
13403 SDValue Cvt = NewLoad;
13404 if (Ld->getExtensionType() == ISD::SEXTLOAD) {
13405 Cvt = DAG.getNode(ISD::SIGN_EXTEND_INREG, SL, MVT::i32, NewLoad,
13406 DAG.getValueType(TruncVT));
13407 } else if (Ld->getExtensionType() == ISD::ZEXTLOAD ||
13408 Ld->getExtensionType() == ISD::NON_EXTLOAD) {
13409 Cvt = DAG.getZeroExtendInReg(NewLoad, SL, TruncVT);
13410 } else {
13411 assert(Ld->getExtensionType() == ISD::EXTLOAD);
13412 }
13413
13414 EVT VT = Ld->getValueType(0);
13415 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
13416
13417 DCI.AddToWorklist(Cvt.getNode());
13418
13419 // We may need to handle exotic cases, such as i16->i64 extloads, so insert
13420 // the appropriate extension from the 32-bit load.
13421 Cvt = getLoadExtOrTrunc(DAG, Ld->getExtensionType(), Cvt, SL, IntVT);
13422 DCI.AddToWorklist(Cvt.getNode());
13423
13424 // Handle conversion back to floating point if necessary.
13425 Cvt = DAG.getNode(ISD::BITCAST, SL, VT, Cvt);
13426
13427 return DAG.getMergeValues({Cvt, NewLoad.getValue(1)}, SL);
13428}
13429
13431 const SIMachineFunctionInfo &Info) {
13432 // TODO: Should check if the address can definitely not access stack.
13433 if (Info.isEntryFunction())
13434 return Info.getUserSGPRInfo().hasFlatScratchInit();
13435 return true;
13436}
13437
13438// Lower a VGPR ("as memory") load or store to a REG_LOAD / REG_STORE node
13439// carrying the dword index. TODO: sub-dword accesses.
13440SDValue SITargetLowering::lowerLoadStoreVGPR(SDValue Op,
13441 SelectionDAG &DAG) const {
13442 SDLoc DL(Op);
13443 MemSDNode *MemOp = cast<MemSDNode>(Op);
13444 EVT MemVT = MemOp->getMemoryVT();
13445 unsigned BitWidth = MemVT.getSizeInBits();
13446
13447 // Each caller replaces the node, so this is diagnosed once.
13448 auto reportUnsupported = [&]() -> SDValue {
13449 const Function &F = DAG.getMachineFunction().getFunction();
13450 DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
13451 F,
13452 "unsupported access of VGPR 'as memory' address space (13); only "
13453 "dword-aligned whole-dword loads and stores are implemented",
13454 DL.getDebugLoc()));
13455 SmallVector<EVT, 2> ResultTypes(Op->values());
13456 return DAG.getErrorMergeValues(ResultTypes, MemOp->getChain(), DL);
13457 };
13458
13459 if (BitWidth < 32)
13460 return reportUnsupported();
13461
13462 // The index is the pointer >> 2, so an under-aligned access would silently
13463 // reach the containing dword rather than the bytes asked for.
13464 if (MemOp->getAlign() < Align(4))
13465 return reportUnsupported();
13466 auto *Store = dyn_cast<StoreSDNode>(MemOp);
13467 if (Store && Store->isTruncatingStore())
13468 return reportUnsupported();
13469 // Only a width some VGPR tuple has is looked up. A whole-dword access of any
13470 // other width is done as one of <N x i32>, which type legalization splits.
13471 auto *Load = dyn_cast<LoadSDNode>(MemOp);
13472 if (!Subtarget->getRegisterInfo()->getVGPRClassForBitWidth(BitWidth) ||
13474 if (BitWidth % 32 != 0)
13475 return reportUnsupported();
13476 EVT ViewVT = getEquivalentMemType(*DAG.getContext(), MemVT);
13477 if (Store) {
13478 if (MemVT == ViewVT)
13479 return SDValue();
13480 return DAG.getStore(Store->getChain(), DL,
13481 DAG.getBitcast(ViewVT, Store->getValue()),
13482 Store->getBasePtr(), Store->getMemOperand());
13483 }
13484 if (MemVT == ViewVT && Load->getExtensionType() == ISD::NON_EXTLOAD)
13485 return SDValue();
13486 // The range is for the elements of MemVT, not for dwords.
13487 Load->getMemOperand()->clearRanges();
13488 SDValue NewLoad = DAG.getLoad(ViewVT, DL, Load->getChain(),
13489 Load->getBasePtr(), Load->getMemOperand());
13490 SDValue Value = getLoadExtOrTrunc(DAG, Load->getExtensionType(),
13491 DAG.getBitcast(MemVT, NewLoad), DL,
13492 Load->getValueType(0));
13493 return DAG.getMergeValues({Value, NewLoad.getValue(1)}, DL);
13494 }
13495
13496 SDValue Chain = MemOp->getChain();
13497 SDValue Index = DAG.getNode(ISD::SRL, DL, MVT::i32, MemOp->getBasePtr(),
13498 DAG.getConstant(2, DL, MVT::i32));
13499
13500 // View as i32 / <N x i32> when the memory type is not register legal.
13501 EVT RegVT = MemVT;
13502 if (!isTypeLegal(RegVT)) {
13503 unsigned NumDwords = BitWidth / 32;
13504 RegVT = NumDwords == 1
13505 ? EVT(MVT::i32)
13506 : EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumDwords);
13507 }
13508
13509 if (Store) {
13510 SDValue Value = DAG.getBitcast(RegVT, Store->getValue());
13511 return DAG.getMemIntrinsicNode(
13512 AMDGPUISD::REG_STORE, DL, DAG.getVTList(MVT::Other),
13513 {Chain, Value, Index}, MemVT, Store->getMemOperand());
13514 }
13515
13516 SDValue NewLoad = DAG.getMemIntrinsicNode(
13517 AMDGPUISD::REG_LOAD, DL, DAG.getVTList(RegVT, MVT::Other), {Chain, Index},
13518 MemVT, Load->getMemOperand());
13519 SDValue Value = DAG.getBitcast(MemVT, NewLoad);
13520 // The combiner folds an extension of a whole-dword load into the load.
13521 Value = getLoadExtOrTrunc(DAG, Load->getExtensionType(), Value, DL,
13522 Load->getValueType(0));
13523 if (Value == NewLoad)
13524 return NewLoad;
13525 return DAG.getMergeValues({Value, NewLoad.getValue(1)}, DL);
13526}
13527
13528SDValue SITargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
13529 SDLoc DL(Op);
13530 LoadSDNode *Load = cast<LoadSDNode>(Op);
13531 assert(Load->getExtensionType() == ISD::NON_EXTLOAD &&
13532 "Unexpected custom extending load!");
13533 EVT MemVT = Load->getMemoryVT();
13534 MachineMemOperand *MMO = Load->getMemOperand();
13535
13536 if (MemVT.getSizeInBits() < 32) {
13537 // Legalize uniform 16-bit loads to i16 = trunc (zextload i16->i32)
13538 // to match subword load patterns.
13539 // Only do this for loads that can use scalar subword load instructions.
13540 if (!MemVT.isVector() && MemVT.getSizeInBits() == 16 &&
13541 isTypeLegal(MemVT) && Subtarget->hasScalarSubwordLoads() &&
13543 SDValue Chain = Load->getChain();
13544 SDValue BasePtr = Load->getBasePtr();
13545
13546 // Load as i16 and zero-extend to i32 (matches S_LOAD_U16 behavior)
13547 SDValue NewLD = DAG.getExtLoad(ISD::ZEXTLOAD, DL, MVT::i32, Chain,
13548 BasePtr, MVT::i16, MMO);
13549
13550 // Truncate back to i16
13551 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, NewLD);
13552
13553 // For f16/bf16, bitcast from i16 to the original fp type
13554 SDValue Result = (MemVT == MVT::i16)
13555 ? Trunc
13556 : DAG.getNode(ISD::BITCAST, DL, MemVT, Trunc);
13557
13558 SDValue Ops[] = {Result, NewLD.getValue(1)};
13559 return DAG.getMergeValues(Ops, DL);
13560 }
13561
13562 if (MemVT == MVT::i16 && isTypeLegal(MVT::i16))
13563 return SDValue();
13564
13565 // FIXME: Copied from PPC
13566 // First, load into 32 bits, then truncate to 1 bit.
13567
13568 SDValue Chain = Load->getChain();
13569 SDValue BasePtr = Load->getBasePtr();
13570
13571 EVT RealMemVT = (MemVT == MVT::i1) ? MVT::i8 : MVT::i16;
13572
13573 SDValue NewLD = DAG.getExtLoad(ISD::EXTLOAD, DL, MVT::i32, Chain, BasePtr,
13574 RealMemVT, MMO);
13575
13576 if (!MemVT.isVector()) {
13577 SDValue Ops[] = {DAG.getNode(ISD::TRUNCATE, DL, MemVT, NewLD),
13578 NewLD.getValue(1)};
13579
13580 return DAG.getMergeValues(Ops, DL);
13581 }
13582
13584 for (unsigned I = 0, N = MemVT.getVectorNumElements(); I != N; ++I) {
13585 SDValue Elt = DAG.getNode(ISD::SRL, DL, MVT::i32, NewLD,
13586 DAG.getConstant(I, DL, MVT::i32));
13587
13588 Elts.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Elt));
13589 }
13590
13591 SDValue Ops[] = {DAG.getBuildVector(MemVT, DL, Elts), NewLD.getValue(1)};
13592
13593 return DAG.getMergeValues(Ops, DL);
13594 }
13595
13596 assert(MemVT.isVector() &&
13597 Op.getValueType().getVectorElementType() == MVT::i32 &&
13598 "Custom lowering for non-i32 vectors hasn't been implemented.");
13599
13600 Align Alignment = Load->getAlign();
13601 unsigned AS = Load->getAddressSpace();
13602 if (Subtarget->hasLDSMisalignedBugInWGPMode() &&
13603 AS == AMDGPUAS::FLAT_ADDRESS &&
13604 Alignment.value() < MemVT.getStoreSize() && MemVT.getSizeInBits() > 32) {
13605 return SplitVectorLoad(Op, DAG);
13606 }
13607
13609 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
13610 // If there is a possibility that flat instruction access scratch memory
13611 // then we need to use the same legalization rules we use for private.
13612 if (AS == AMDGPUAS::FLAT_ADDRESS &&
13613 !Subtarget->hasMultiDwordFlatScratchAddressing())
13614 AS = addressMayBeAccessedAsPrivate(Load->getMemOperand(), *MFI)
13617
13618 unsigned NumElements = MemVT.getVectorNumElements();
13619
13620 if (AS == AMDGPUAS::CONSTANT_ADDRESS ||
13622 (AS == AMDGPUAS::GLOBAL_ADDRESS && Load->isSimple() &&
13623 (Load->isInvariant() || isMemOpHasNoClobberedMemOperand(Load)))) {
13624 if ((!Op->isDivergent() || AMDGPU::isUniformMMO(MMO)) &&
13625 Alignment >= Align(4) && NumElements < 32) {
13626 if (MemVT.isPow2VectorType() ||
13627 (Subtarget->hasScalarDwordx3Loads() && NumElements == 3))
13628 return SDValue();
13629 return WidenOrSplitVectorLoad(Op, DAG);
13630 }
13631 // Non-uniform loads will be selected to MUBUF instructions, so they
13632 // have the same legalization requirements as global and private
13633 // loads.
13634 //
13635 }
13636 if (AS == AMDGPUAS::CONSTANT_ADDRESS ||
13639 if (NumElements > 4)
13640 return SplitVectorLoad(Op, DAG);
13641 // v3 loads not supported on SI.
13642 if (NumElements == 3 && !Subtarget->hasDwordx3LoadStores())
13643 return WidenOrSplitVectorLoad(Op, DAG);
13644
13645 // v3 and v4 loads are supported for private and global memory.
13646 return SDValue();
13647 }
13648 if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
13649 // Depending on the setting of the private_element_size field in the
13650 // resource descriptor, we can only make private accesses up to a certain
13651 // size.
13652 switch (Subtarget->getMaxPrivateElementSize()) {
13653 case 4: {
13654 auto [Op0, Op1] = scalarizeVectorLoad(Load, DAG);
13655 return DAG.getMergeValues({Op0, Op1}, DL);
13656 }
13657 case 8:
13658 if (NumElements > 2)
13659 return SplitVectorLoad(Op, DAG);
13660 return SDValue();
13661 case 16:
13662 // Same as global/flat
13663 if (NumElements > 4)
13664 return SplitVectorLoad(Op, DAG);
13665 // v3 loads not supported on SI.
13666 if (NumElements == 3 && !Subtarget->hasDwordx3LoadStores())
13667 return WidenOrSplitVectorLoad(Op, DAG);
13668
13669 return SDValue();
13670 default:
13671 llvm_unreachable("unsupported private_element_size");
13672 }
13673 } else if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) {
13674 unsigned Fast = 0;
13675 auto Flags = Load->getMemOperand()->getFlags();
13677 Load->getAlign(), Flags, &Fast) &&
13678 Fast > 1)
13679 return SDValue();
13680
13681 return SplitVectorLoad(Op, DAG);
13682 }
13683
13685 MemVT, *Load->getMemOperand())) {
13686 auto [Op0, Op1] = expandUnalignedLoad(Load, DAG);
13687 return DAG.getMergeValues({Op0, Op1}, DL);
13688 }
13689
13690 return SDValue();
13691}
13692
13693SDValue SITargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
13694 EVT VT = Op.getValueType();
13695 if (VT.getSizeInBits() == 128 || VT.getSizeInBits() == 256 ||
13696 VT.getSizeInBits() == 512)
13697 return splitTernaryVectorOp(Op, DAG);
13698
13699 assert(VT.getSizeInBits() == 64);
13700
13701 SDLoc DL(Op);
13702 SDValue Cond = DAG.getFreeze(Op.getOperand(0));
13703
13704 SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
13705 SDValue One = DAG.getConstant(1, DL, MVT::i32);
13706
13707 SDValue LHS = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, Op.getOperand(1));
13708 SDValue RHS = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, Op.getOperand(2));
13709
13710 SDValue Lo0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, LHS, Zero);
13711 SDValue Lo1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, RHS, Zero);
13712
13713 SDValue Lo = DAG.getSelect(DL, MVT::i32, Cond, Lo0, Lo1);
13714
13715 SDValue Hi0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, LHS, One);
13716 SDValue Hi1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, RHS, One);
13717
13718 SDValue Hi = DAG.getSelect(DL, MVT::i32, Cond, Hi0, Hi1);
13719
13720 SDValue Res = DAG.getBuildVector(MVT::v2i32, DL, {Lo, Hi});
13721 return DAG.getNode(ISD::BITCAST, DL, VT, Res);
13722}
13723
13724// Catch division cases where we can use shortcuts with rcp and rsq
13725// instructions.
13726SDValue SITargetLowering::lowerFastUnsafeFDIV(SDValue Op,
13727 SelectionDAG &DAG) const {
13728 SDLoc SL(Op);
13729 SDValue LHS = Op.getOperand(0);
13730 SDValue RHS = Op.getOperand(1);
13731 EVT VT = Op.getValueType();
13732 const SDNodeFlags Flags = Op->getFlags();
13733
13734 bool AllowInaccurateRcp = Flags.hasApproximateFuncs();
13735
13736 if (const ConstantFPSDNode *CLHS = dyn_cast<ConstantFPSDNode>(LHS)) {
13737 // Without !fpmath accuracy information, we can't do more because we don't
13738 // know exactly whether rcp is accurate enough to meet !fpmath requirement.
13739 // f16 is always accurate enough
13740 if (!AllowInaccurateRcp && VT != MVT::f16 && VT != MVT::bf16)
13741 return SDValue();
13742
13743 if (CLHS->isOne()) {
13744 // v_rcp_f32 and v_rsq_f32 do not support denormals, and according to
13745 // the CI documentation has a worst case error of 1 ulp.
13746 // OpenCL requires <= 2.5 ulp for 1.0 / x, so it should always be OK to
13747 // use it as long as we aren't trying to use denormals.
13748 //
13749 // v_rcp_f16 and v_rsq_f16 DO support denormals and 0.51ulp.
13750
13751 // 1.0 / sqrt(x) -> rsq(x)
13752
13753 // XXX - Is afn sufficient to do this for f64? The maximum ULP
13754 // error seems really high at 2^29 ULP.
13755 // 1.0 / x -> rcp(x)
13756 return DAG.getNode(AMDGPUISD::RCP, SL, VT, RHS);
13757 }
13758
13759 // Same as for 1.0, but expand the sign out of the constant.
13760 if (CLHS->isMinusOne()) {
13761 // -1.0 / x -> fneg (rcp x)
13762 SDValue Rcp = DAG.getNode(AMDGPUISD::RCP, SL, VT, RHS);
13763 return DAG.getNode(ISD::FNEG, SL, VT, Rcp);
13764 }
13765 }
13766
13767 // For f16 and bf16 require afn or arcp.
13768 // For f32 require afn.
13769 if (!AllowInaccurateRcp &&
13770 ((VT != MVT::f16 && VT != MVT::bf16) || !Flags.hasAllowReciprocal()))
13771 return SDValue();
13772
13773 // Turn into multiply by the reciprocal.
13774 // x / y -> x * (1.0 / y)
13775 SDValue Recip = DAG.getNode(AMDGPUISD::RCP, SL, VT, RHS);
13776 return DAG.getNode(ISD::FMUL, SL, VT, LHS, Recip, Flags);
13777}
13778
13779SDValue SITargetLowering::lowerFastUnsafeFDIV64(SDValue Op,
13780 SelectionDAG &DAG) const {
13781 SDLoc SL(Op);
13782 SDValue X = Op.getOperand(0);
13783 SDValue Y = Op.getOperand(1);
13784 EVT VT = Op.getValueType();
13785 const SDNodeFlags Flags = Op->getFlags();
13786
13787 bool AllowInaccurateDiv = Flags.hasApproximateFuncs();
13788 if (!AllowInaccurateDiv)
13789 return SDValue();
13790
13791 const ConstantFPSDNode *CLHS = dyn_cast<ConstantFPSDNode>(X);
13792 bool IsNegRcp = CLHS && CLHS->isMinusOne();
13793
13794 // Pull out the negation so it folds for free into the source modifiers.
13795 if (IsNegRcp)
13796 X = DAG.getConstantFP(1.0, SL, VT);
13797
13798 SDValue NegY = IsNegRcp ? Y : DAG.getNode(ISD::FNEG, SL, VT, Y);
13799 SDValue One = DAG.getConstantFP(1.0, SL, VT);
13800
13801 SDValue R = DAG.getNode(AMDGPUISD::RCP, SL, VT, Y);
13802 if (IsNegRcp)
13803 R = DAG.getNode(ISD::FNEG, SL, VT, R);
13804
13805 SDValue Tmp0 = DAG.getNode(ISD::FMA, SL, VT, NegY, R, One);
13806
13807 R = DAG.getNode(ISD::FMA, SL, VT, Tmp0, R, R);
13808 SDValue Tmp1 = DAG.getNode(ISD::FMA, SL, VT, NegY, R, One);
13809 R = DAG.getNode(ISD::FMA, SL, VT, Tmp1, R, R);
13810
13811 // Skip the last 2 correction terms for reciprocal.
13812 if (IsNegRcp || (CLHS && CLHS->isOne()))
13813 return R;
13814
13815 SDValue Ret = DAG.getNode(ISD::FMUL, SL, VT, X, R);
13816 SDValue Tmp2 = DAG.getNode(ISD::FMA, SL, VT, NegY, Ret, X);
13817 return DAG.getNode(ISD::FMA, SL, VT, Tmp2, R, Ret);
13818}
13819
13820static SDValue getFPBinOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL,
13821 EVT VT, SDValue A, SDValue B, SDValue GlueChain,
13822 SDNodeFlags Flags) {
13823 if (GlueChain->getNumValues() <= 1) {
13824 return DAG.getNode(Opcode, SL, VT, A, B, Flags);
13825 }
13826
13827 assert(GlueChain->getNumValues() == 3);
13828
13829 SDVTList VTList = DAG.getVTList(VT, MVT::Other, MVT::Glue);
13830 switch (Opcode) {
13831 default:
13832 llvm_unreachable("no chain equivalent for opcode");
13833 case ISD::FMUL:
13834 Opcode = AMDGPUISD::FMUL_W_CHAIN;
13835 break;
13836 }
13837
13838 return DAG.getNode(Opcode, SL, VTList,
13839 {GlueChain.getValue(1), A, B, GlueChain.getValue(2)},
13840 Flags);
13841}
13842
13843static SDValue getFPTernOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL,
13844 EVT VT, SDValue A, SDValue B, SDValue C,
13845 SDValue GlueChain, SDNodeFlags Flags) {
13846 if (GlueChain->getNumValues() <= 1) {
13847 return DAG.getNode(Opcode, SL, VT, {A, B, C}, Flags);
13848 }
13849
13850 assert(GlueChain->getNumValues() == 3);
13851
13852 SDVTList VTList = DAG.getVTList(VT, MVT::Other, MVT::Glue);
13853 switch (Opcode) {
13854 default:
13855 llvm_unreachable("no chain equivalent for opcode");
13856 case ISD::FMA:
13857 Opcode = AMDGPUISD::FMA_W_CHAIN;
13858 break;
13859 }
13860
13861 return DAG.getNode(Opcode, SL, VTList,
13862 {GlueChain.getValue(1), A, B, C, GlueChain.getValue(2)},
13863 Flags);
13864}
13865
13866SDValue SITargetLowering::LowerFDIV16(SDValue Op, SelectionDAG &DAG) const {
13867 if (SDValue FastLowered = lowerFastUnsafeFDIV(Op, DAG))
13868 return FastLowered;
13869
13870 SDLoc SL(Op);
13871 EVT VT = Op.getValueType();
13872 SDValue LHS = Op.getOperand(0);
13873 SDValue RHS = Op.getOperand(1);
13874
13875 SDValue LHSExt = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, LHS);
13876 SDValue RHSExt = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, RHS);
13877
13878 if (VT == MVT::bf16) {
13879 SDValue ExtDiv =
13880 DAG.getNode(ISD::FDIV, SL, MVT::f32, LHSExt, RHSExt, Op->getFlags());
13881 return DAG.getNode(ISD::FP_ROUND, SL, MVT::bf16, ExtDiv,
13882 DAG.getTargetConstant(0, SL, MVT::i32));
13883 }
13884
13885 assert(VT == MVT::f16);
13886
13887 // a32.u = opx(V_CVT_F32_F16, a.u); // CVT to F32
13888 // b32.u = opx(V_CVT_F32_F16, b.u); // CVT to F32
13889 // r32.u = opx(V_RCP_F32, b32.u); // rcp = 1 / d
13890 // q32.u = opx(V_MUL_F32, a32.u, r32.u); // q = n * rcp
13891 // e32.u = opx(V_MAD_F32, (b32.u^_neg32), q32.u, a32.u); // err = -d * q + n
13892 // q32.u = opx(V_MAD_F32, e32.u, r32.u, q32.u); // q = n * rcp
13893 // e32.u = opx(V_MAD_F32, (b32.u^_neg32), q32.u, a32.u); // err = -d * q + n
13894 // tmp.u = opx(V_MUL_F32, e32.u, r32.u);
13895 // tmp.u = opx(V_AND_B32, tmp.u, 0xff800000)
13896 // q32.u = opx(V_ADD_F32, tmp.u, q32.u);
13897 // q16.u = opx(V_CVT_F16_F32, q32.u);
13898 // q16.u = opx(V_DIV_FIXUP_F16, q16.u, b.u, a.u); // q = touchup(q, d, n)
13899
13900 // We will use ISD::FMA on targets that don't support ISD::FMAD.
13901 unsigned FMADOpCode =
13903 SDValue NegRHSExt = DAG.getNode(ISD::FNEG, SL, MVT::f32, RHSExt);
13904 SDValue Rcp =
13905 DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, RHSExt, Op->getFlags());
13906 SDValue Quot =
13907 DAG.getNode(ISD::FMUL, SL, MVT::f32, LHSExt, Rcp, Op->getFlags());
13908 SDValue Err = DAG.getNode(FMADOpCode, SL, MVT::f32, NegRHSExt, Quot, LHSExt,
13909 Op->getFlags());
13910 Quot = DAG.getNode(FMADOpCode, SL, MVT::f32, Err, Rcp, Quot, Op->getFlags());
13911 Err = DAG.getNode(FMADOpCode, SL, MVT::f32, NegRHSExt, Quot, LHSExt,
13912 Op->getFlags());
13913 SDValue Tmp = DAG.getNode(ISD::FMUL, SL, MVT::f32, Err, Rcp, Op->getFlags());
13914 SDValue TmpCast = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Tmp);
13915 TmpCast = DAG.getNode(ISD::AND, SL, MVT::i32, TmpCast,
13916 DAG.getConstant(0xff800000, SL, MVT::i32));
13917 Tmp = DAG.getNode(ISD::BITCAST, SL, MVT::f32, TmpCast);
13918 Quot = DAG.getNode(ISD::FADD, SL, MVT::f32, Tmp, Quot, Op->getFlags());
13919 SDValue RDst = DAG.getNode(ISD::FP_ROUND, SL, MVT::f16, Quot,
13920 DAG.getTargetConstant(0, SL, MVT::i32));
13921 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f16, RDst, RHS, LHS,
13922 Op->getFlags());
13923}
13924
13925// Faster 2.5 ULP division that does not support denormals.
13926SDValue SITargetLowering::lowerFDIV_FAST(SDValue Op, SelectionDAG &DAG) const {
13927 SDNodeFlags Flags = Op->getFlags();
13928 SDLoc SL(Op);
13929 SDValue LHS = Op.getOperand(1);
13930 SDValue RHS = Op.getOperand(2);
13931
13932 // TODO: The combiner should probably handle elimination of redundant fabs.
13934 ? RHS
13935 : DAG.getNode(ISD::FABS, SL, MVT::f32, RHS, Flags);
13936
13937 const APFloat K0Val(0x1p+96f);
13938 const SDValue K0 = DAG.getConstantFP(K0Val, SL, MVT::f32);
13939
13940 const APFloat K1Val(0x1p-32f);
13941 const SDValue K1 = DAG.getConstantFP(K1Val, SL, MVT::f32);
13942
13943 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f32);
13944
13945 EVT SetCCVT =
13946 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f32);
13947
13948 SDValue r2 = DAG.getSetCC(SL, SetCCVT, r1, K0, ISD::SETOGT);
13949
13950 SDValue r3 = DAG.getNode(ISD::SELECT, SL, MVT::f32, r2, K1, One, Flags);
13951
13952 r1 = DAG.getNode(ISD::FMUL, SL, MVT::f32, RHS, r3, Flags);
13953
13954 // rcp does not support denormals.
13955 SDValue r0 = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, r1, Flags);
13956
13957 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f32, LHS, r0, Flags);
13958
13959 return DAG.getNode(ISD::FMUL, SL, MVT::f32, r3, Mul, Flags);
13960}
13961
13962// Returns immediate value for setting the F32 denorm mode when using the
13963// S_DENORM_MODE instruction.
13965 const SIMachineFunctionInfo *Info,
13966 const GCNSubtarget *ST) {
13967 assert(ST->hasDenormModeInst() && "Requires S_DENORM_MODE");
13968 uint32_t DPDenormModeDefault = Info->getMode().fpDenormModeDPValue();
13969 uint32_t Mode = SPDenormMode | (DPDenormModeDefault << 2);
13970 return DAG.getTargetConstant(Mode, SDLoc(), MVT::i32);
13971}
13972
13973SDValue SITargetLowering::LowerFDIV32(SDValue Op, SelectionDAG &DAG) const {
13974 if (SDValue FastLowered = lowerFastUnsafeFDIV(Op, DAG))
13975 return FastLowered;
13976
13977 // The selection matcher assumes anything with a chain selecting to a
13978 // mayRaiseFPException machine instruction. Since we're introducing a chain
13979 // here, we need to explicitly report nofpexcept for the regular fdiv
13980 // lowering.
13981 SDNodeFlags Flags = Op->getFlags();
13982 Flags.setNoFPExcept(true);
13983
13984 SDLoc SL(Op);
13985 SDValue LHS = Op.getOperand(0);
13986 SDValue RHS = Op.getOperand(1);
13987
13988 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f32);
13989
13990 SDVTList ScaleVT = DAG.getVTList(MVT::f32, MVT::i1);
13991
13992 SDValue DenominatorScaled =
13993 DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT, {RHS, RHS, LHS}, Flags);
13994 SDValue NumeratorScaled =
13995 DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT, {LHS, RHS, LHS}, Flags);
13996
13997 // Denominator is scaled to not be denormal, so using rcp is ok.
13998 SDValue ApproxRcp =
13999 DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, DenominatorScaled, Flags);
14000 SDValue NegDivScale0 =
14001 DAG.getNode(ISD::FNEG, SL, MVT::f32, DenominatorScaled, Flags);
14002
14003 using namespace AMDGPU::Hwreg;
14004 const unsigned Denorm32Reg = HwregEncoding::encode(ID_MODE, 4, 2);
14005 const SDValue BitField = DAG.getTargetConstant(Denorm32Reg, SL, MVT::i32);
14006
14007 const MachineFunction &MF = DAG.getMachineFunction();
14008 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
14009 const DenormalMode DenormMode = Info->getMode().FP32Denormals;
14010
14011 const bool PreservesDenormals = DenormMode == DenormalMode::getIEEE();
14012 const bool HasDynamicDenormals =
14013 (DenormMode.Input == DenormalMode::Dynamic) ||
14014 (DenormMode.Output == DenormalMode::Dynamic);
14015
14016 SDValue SavedDenormMode;
14017
14018 if (!PreservesDenormals) {
14019 // Note we can't use the STRICT_FMA/STRICT_FMUL for the non-strict FDIV
14020 // lowering. The chain dependence is insufficient, and we need glue. We do
14021 // not need the glue variants in a strictfp function.
14022
14023 SDVTList BindParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
14024
14025 SDValue Glue = DAG.getEntryNode();
14026 if (HasDynamicDenormals) {
14027 SDNode *GetReg = DAG.getMachineNode(AMDGPU::S_GETREG_B32, SL,
14028 DAG.getVTList(MVT::i32, MVT::Glue),
14029 {BitField, Glue});
14030 SavedDenormMode = SDValue(GetReg, 0);
14031
14032 Glue = DAG.getMergeValues(
14033 {DAG.getEntryNode(), SDValue(GetReg, 0), SDValue(GetReg, 1)}, SL);
14034 }
14035
14036 SDNode *EnableDenorm;
14037 if (Subtarget->hasDenormModeInst()) {
14038 const SDValue EnableDenormValue =
14039 getSPDenormModeValue(FP_DENORM_FLUSH_NONE, DAG, Info, Subtarget);
14040
14041 EnableDenorm = DAG.getNode(AMDGPUISD::DENORM_MODE, SL, BindParamVTs, Glue,
14042 EnableDenormValue)
14043 .getNode();
14044 } else {
14045 const SDValue EnableDenormValue =
14046 DAG.getConstant(FP_DENORM_FLUSH_NONE, SL, MVT::i32);
14047 EnableDenorm = DAG.getMachineNode(AMDGPU::S_SETREG_B32, SL, BindParamVTs,
14048 {EnableDenormValue, BitField, Glue});
14049 }
14050
14051 SDValue Ops[3] = {NegDivScale0, SDValue(EnableDenorm, 0),
14052 SDValue(EnableDenorm, 1)};
14053
14054 NegDivScale0 = DAG.getMergeValues(Ops, SL);
14055 }
14056
14057 SDValue Fma0 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0,
14058 ApproxRcp, One, NegDivScale0, Flags);
14059
14060 SDValue Fma1 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, Fma0, ApproxRcp,
14061 ApproxRcp, Fma0, Flags);
14062
14063 SDValue Mul = getFPBinOp(DAG, ISD::FMUL, SL, MVT::f32, NumeratorScaled, Fma1,
14064 Fma1, Flags);
14065
14066 SDValue Fma2 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, Mul,
14067 NumeratorScaled, Mul, Flags);
14068
14069 SDValue Fma3 =
14070 getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, Fma2, Fma1, Mul, Fma2, Flags);
14071
14072 SDValue Fma4 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, Fma3,
14073 NumeratorScaled, Fma3, Flags);
14074
14075 if (!PreservesDenormals) {
14076 SDNode *DisableDenorm;
14077 if (!HasDynamicDenormals && Subtarget->hasDenormModeInst()) {
14078 const SDValue DisableDenormValue = getSPDenormModeValue(
14079 FP_DENORM_FLUSH_IN_FLUSH_OUT, DAG, Info, Subtarget);
14080
14081 SDVTList BindParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
14082 DisableDenorm =
14083 DAG.getNode(AMDGPUISD::DENORM_MODE, SL, BindParamVTs,
14084 Fma4.getValue(1), DisableDenormValue, Fma4.getValue(2))
14085 .getNode();
14086 } else {
14087 assert(HasDynamicDenormals == (bool)SavedDenormMode);
14088 const SDValue DisableDenormValue =
14089 HasDynamicDenormals
14090 ? SavedDenormMode
14091 : DAG.getConstant(FP_DENORM_FLUSH_IN_FLUSH_OUT, SL, MVT::i32);
14092
14093 DisableDenorm = DAG.getMachineNode(
14094 AMDGPU::S_SETREG_B32, SL, MVT::Other,
14095 {DisableDenormValue, BitField, Fma4.getValue(1), Fma4.getValue(2)});
14096 }
14097
14098 SDValue OutputChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other,
14099 SDValue(DisableDenorm, 0), DAG.getRoot());
14100 DAG.setRoot(OutputChain);
14101 }
14102
14103 SDValue Scale = NumeratorScaled.getValue(1);
14104 SDValue Fmas = DAG.getNode(AMDGPUISD::DIV_FMAS, SL, MVT::f32,
14105 {Fma4, Fma1, Fma3, Scale}, Flags);
14106
14107 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f32, Fmas, RHS, LHS, Flags);
14108}
14109
14110SDValue SITargetLowering::LowerFDIV64(SDValue Op, SelectionDAG &DAG) const {
14111 if (SDValue FastLowered = lowerFastUnsafeFDIV64(Op, DAG))
14112 return FastLowered;
14113
14114 SDLoc SL(Op);
14115 SDValue X = Op.getOperand(0);
14116 SDValue Y = Op.getOperand(1);
14117
14118 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64);
14119
14120 SDVTList ScaleVT = DAG.getVTList(MVT::f64, MVT::i1);
14121
14122 SDValue DivScale0 = DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT, Y, Y, X);
14123
14124 SDValue NegDivScale0 = DAG.getNode(ISD::FNEG, SL, MVT::f64, DivScale0);
14125
14126 SDValue Rcp = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f64, DivScale0);
14127
14128 SDValue Fma0 = DAG.getNode(ISD::FMA, SL, MVT::f64, NegDivScale0, Rcp, One);
14129
14130 SDValue Fma1 = DAG.getNode(ISD::FMA, SL, MVT::f64, Rcp, Fma0, Rcp);
14131
14132 SDValue Fma2 = DAG.getNode(ISD::FMA, SL, MVT::f64, NegDivScale0, Fma1, One);
14133
14134 SDValue DivScale1 = DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT, X, Y, X);
14135
14136 SDValue Fma3 = DAG.getNode(ISD::FMA, SL, MVT::f64, Fma1, Fma2, Fma1);
14137 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f64, DivScale1, Fma3);
14138
14139 SDValue Fma4 =
14140 DAG.getNode(ISD::FMA, SL, MVT::f64, NegDivScale0, Mul, DivScale1);
14141
14142 SDValue Scale;
14143
14144 if (!Subtarget->hasUsableDivScaleConditionOutput()) {
14145 // Workaround a hardware bug on SI where the condition output from div_scale
14146 // is not usable.
14147
14148 const SDValue Hi = DAG.getConstant(1, SL, MVT::i32);
14149
14150 // Figure out if the scale to use for div_fmas.
14151 SDValue NumBC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, X);
14152 SDValue DenBC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Y);
14153 SDValue Scale0BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, DivScale0);
14154 SDValue Scale1BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, DivScale1);
14155
14156 SDValue NumHi =
14157 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, NumBC, Hi);
14158 SDValue DenHi =
14159 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, DenBC, Hi);
14160
14161 SDValue Scale0Hi =
14162 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Scale0BC, Hi);
14163 SDValue Scale1Hi =
14164 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Scale1BC, Hi);
14165
14166 SDValue CmpDen = DAG.getSetCC(SL, MVT::i1, DenHi, Scale0Hi, ISD::SETEQ);
14167 SDValue CmpNum = DAG.getSetCC(SL, MVT::i1, NumHi, Scale1Hi, ISD::SETEQ);
14168 Scale = DAG.getNode(ISD::XOR, SL, MVT::i1, CmpNum, CmpDen);
14169 } else {
14170 Scale = DivScale1.getValue(1);
14171 }
14172
14173 SDValue Fmas =
14174 DAG.getNode(AMDGPUISD::DIV_FMAS, SL, MVT::f64, Fma4, Fma3, Mul, Scale);
14175
14176 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f64, Fmas, Y, X);
14177}
14178
14179SDValue SITargetLowering::LowerFDIV(SDValue Op, SelectionDAG &DAG) const {
14180 EVT VT = Op.getValueType();
14181
14182 if (VT == MVT::f32)
14183 return LowerFDIV32(Op, DAG);
14184
14185 if (VT == MVT::f64)
14186 return LowerFDIV64(Op, DAG);
14187
14188 if (VT == MVT::f16 || VT == MVT::bf16)
14189 return LowerFDIV16(Op, DAG);
14190
14191 llvm_unreachable("Unexpected type for fdiv");
14192}
14193
14194SDValue SITargetLowering::LowerFFREXP(SDValue Op, SelectionDAG &DAG) const {
14195 SDLoc dl(Op);
14196 SDValue Val = Op.getOperand(0);
14197 EVT VT = Val.getValueType();
14198 EVT ResultExpVT = Op->getValueType(1);
14199 EVT InstrExpVT = VT == MVT::f16 ? MVT::i16 : MVT::i32;
14200
14201 SDValue Mant = DAG.getNode(
14203 DAG.getTargetConstant(Intrinsic::amdgcn_frexp_mant, dl, MVT::i32), Val);
14204
14205 SDValue Exp = DAG.getNode(
14206 ISD::INTRINSIC_WO_CHAIN, dl, InstrExpVT,
14207 DAG.getTargetConstant(Intrinsic::amdgcn_frexp_exp, dl, MVT::i32), Val);
14208
14209 if (Subtarget->hasFractBug()) {
14210 SDValue Fabs = DAG.getNode(ISD::FABS, dl, VT, Val);
14211 SDValue Inf =
14213
14214 SDValue IsFinite = DAG.getSetCC(dl, MVT::i1, Fabs, Inf, ISD::SETOLT);
14215 SDValue Zero = DAG.getConstant(0, dl, InstrExpVT);
14216 Exp = DAG.getNode(ISD::SELECT, dl, InstrExpVT, IsFinite, Exp, Zero);
14217 Mant = DAG.getNode(ISD::SELECT, dl, VT, IsFinite, Mant, Val);
14218 }
14219
14220 SDValue CastExp = DAG.getSExtOrTrunc(Exp, dl, ResultExpVT);
14221 return DAG.getMergeValues({Mant, CastExp}, dl);
14222}
14223
14224SDValue SITargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
14225 SDLoc DL(Op);
14226 StoreSDNode *Store = cast<StoreSDNode>(Op);
14227 assert(!Store->isTruncatingStore() && "Unexpected custom truncating store!");
14228 EVT VT = Store->getMemoryVT();
14229
14230 if (VT == MVT::i1) {
14231 return DAG.getTruncStore(
14232 Store->getChain(), DL,
14233 DAG.getSExtOrTrunc(Store->getValue(), DL, MVT::i32),
14234 Store->getBasePtr(), MVT::i1, Store->getMemOperand());
14235 }
14236
14237 assert(VT.isVector() &&
14238 Store->getValue().getValueType().getScalarType() == MVT::i32);
14239
14240 unsigned AS = Store->getAddressSpace();
14241 if (Subtarget->hasLDSMisalignedBugInWGPMode() &&
14242 AS == AMDGPUAS::FLAT_ADDRESS &&
14243 Store->getAlign().value() < VT.getStoreSize() &&
14244 VT.getSizeInBits() > 32) {
14245 return SplitVectorStore(Op, DAG);
14246 }
14247
14249 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
14250 // If there is a possibility that flat instruction access scratch memory
14251 // then we need to use the same legalization rules we use for private.
14252 if (AS == AMDGPUAS::FLAT_ADDRESS &&
14253 !Subtarget->hasMultiDwordFlatScratchAddressing())
14254 AS = addressMayBeAccessedAsPrivate(Store->getMemOperand(), *MFI)
14257
14258 unsigned NumElements = VT.getVectorNumElements();
14260 if (NumElements > 4)
14261 return SplitVectorStore(Op, DAG);
14262 // v3 stores not supported on SI.
14263 if (NumElements == 3 && !Subtarget->hasDwordx3LoadStores())
14264 return SplitVectorStore(Op, DAG);
14265
14267 VT, *Store->getMemOperand()))
14268 return expandUnalignedStore(Store, DAG);
14269
14270 return SDValue();
14271 }
14272 if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
14273 switch (Subtarget->getMaxPrivateElementSize()) {
14274 case 4:
14275 return scalarizeVectorStore(Store, DAG);
14276 case 8:
14277 if (NumElements > 2)
14278 return SplitVectorStore(Op, DAG);
14279 return SDValue();
14280 case 16:
14281 if (NumElements > 4 ||
14282 (NumElements == 3 && !Subtarget->hasFlatScratchEnabled()))
14283 return SplitVectorStore(Op, DAG);
14284 return SDValue();
14285 default:
14286 llvm_unreachable("unsupported private_element_size");
14287 }
14288 } else if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) {
14289 unsigned Fast = 0;
14290 auto Flags = Store->getMemOperand()->getFlags();
14292 Store->getAlign(), Flags, &Fast) &&
14293 Fast > 1)
14294 return SDValue();
14295
14296 return SplitVectorStore(Op, DAG);
14297 }
14298
14299 // Probably an invalid store. If so we'll end up emitting a selection error.
14300 return SDValue();
14301}
14302
14303// Avoid the full correct expansion for f32 sqrt when promoting from f16.
14304SDValue SITargetLowering::lowerFSQRTF16(SDValue Op, SelectionDAG &DAG) const {
14305 SDLoc SL(Op);
14306 assert(!Subtarget->has16BitInsts());
14307 SDNodeFlags Flags = Op->getFlags();
14308 SDValue Ext =
14309 DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Op.getOperand(0), Flags);
14310
14311 SDValue SqrtID = DAG.getTargetConstant(Intrinsic::amdgcn_sqrt, SL, MVT::i32);
14312 SDValue Sqrt =
14313 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::f32, SqrtID, Ext, Flags);
14314
14315 return DAG.getNode(ISD::FP_ROUND, SL, MVT::f16, Sqrt,
14316 DAG.getTargetConstant(0, SL, MVT::i32), Flags);
14317}
14318
14319SDValue SITargetLowering::lowerFSQRTF32(SDValue Op, SelectionDAG &DAG) const {
14320 SDLoc DL(Op);
14321 SDNodeFlags Flags = Op->getFlags();
14322 MVT VT = Op.getValueType().getSimpleVT();
14323 const SDValue X = Op.getOperand(0);
14324
14325 if (allowApproxFunc(DAG, Flags)) {
14326 // Instruction is 1ulp but ignores denormals.
14327 return DAG.getNode(
14329 DAG.getTargetConstant(Intrinsic::amdgcn_sqrt, DL, MVT::i32), X, Flags);
14330 }
14331
14332 SDValue ScaleThreshold = DAG.getConstantFP(0x1.0p-96f, DL, VT);
14333 SDValue NeedScale = DAG.getSetCC(DL, MVT::i1, X, ScaleThreshold, ISD::SETOLT);
14334
14335 SDValue ScaleUpFactor = DAG.getConstantFP(0x1.0p+32f, DL, VT);
14336
14337 SDValue ScaledX = DAG.getNode(ISD::FMUL, DL, VT, X, ScaleUpFactor, Flags);
14338
14339 SDValue SqrtX =
14340 DAG.getNode(ISD::SELECT, DL, VT, NeedScale, ScaledX, X, Flags);
14341
14342 SDValue SqrtS;
14343 if (needsDenormHandlingF32(DAG, X, Flags)) {
14344 SDValue SqrtID =
14345 DAG.getTargetConstant(Intrinsic::amdgcn_sqrt, DL, MVT::i32);
14346 SqrtS = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, SqrtID, SqrtX, Flags);
14347
14348 SDValue SqrtSAsInt = DAG.getNode(ISD::BITCAST, DL, MVT::i32, SqrtS);
14349 SDValue SqrtSNextDownInt =
14350 DAG.getNode(ISD::ADD, DL, MVT::i32, SqrtSAsInt,
14351 DAG.getAllOnesConstant(DL, MVT::i32));
14352 SDValue SqrtSNextDown = DAG.getNode(ISD::BITCAST, DL, VT, SqrtSNextDownInt);
14353
14354 SDValue NegSqrtSNextDown =
14355 DAG.getNode(ISD::FNEG, DL, VT, SqrtSNextDown, Flags);
14356
14357 SDValue SqrtVP =
14358 DAG.getNode(ISD::FMA, DL, VT, NegSqrtSNextDown, SqrtS, SqrtX, Flags);
14359
14360 SDValue SqrtSNextUpInt = DAG.getNode(ISD::ADD, DL, MVT::i32, SqrtSAsInt,
14361 DAG.getConstant(1, DL, MVT::i32));
14362 SDValue SqrtSNextUp = DAG.getNode(ISD::BITCAST, DL, VT, SqrtSNextUpInt);
14363
14364 SDValue NegSqrtSNextUp = DAG.getNode(ISD::FNEG, DL, VT, SqrtSNextUp, Flags);
14365 SDValue SqrtVS =
14366 DAG.getNode(ISD::FMA, DL, VT, NegSqrtSNextUp, SqrtS, SqrtX, Flags);
14367
14368 SDValue Zero = DAG.getConstantFP(0.0f, DL, VT);
14369 SDValue SqrtVPLE0 = DAG.getSetCC(DL, MVT::i1, SqrtVP, Zero, ISD::SETOLE);
14370
14371 SqrtS = DAG.getNode(ISD::SELECT, DL, VT, SqrtVPLE0, SqrtSNextDown, SqrtS,
14372 Flags);
14373
14374 SDValue SqrtVPVSGT0 = DAG.getSetCC(DL, MVT::i1, SqrtVS, Zero, ISD::SETOGT);
14375 SqrtS = DAG.getNode(ISD::SELECT, DL, VT, SqrtVPVSGT0, SqrtSNextUp, SqrtS,
14376 Flags);
14377 } else {
14378 SDValue SqrtR = DAG.getNode(AMDGPUISD::RSQ, DL, VT, SqrtX, Flags);
14379
14380 SqrtS = DAG.getNode(ISD::FMUL, DL, VT, SqrtX, SqrtR, Flags);
14381
14382 SDValue Half = DAG.getConstantFP(0.5f, DL, VT);
14383 SDValue SqrtH = DAG.getNode(ISD::FMUL, DL, VT, SqrtR, Half, Flags);
14384 SDValue NegSqrtH = DAG.getNode(ISD::FNEG, DL, VT, SqrtH, Flags);
14385
14386 SDValue SqrtE = DAG.getNode(ISD::FMA, DL, VT, NegSqrtH, SqrtS, Half, Flags);
14387 SqrtH = DAG.getNode(ISD::FMA, DL, VT, SqrtH, SqrtE, SqrtH, Flags);
14388 SqrtS = DAG.getNode(ISD::FMA, DL, VT, SqrtS, SqrtE, SqrtS, Flags);
14389
14390 SDValue NegSqrtS = DAG.getNode(ISD::FNEG, DL, VT, SqrtS, Flags);
14391 SDValue SqrtD =
14392 DAG.getNode(ISD::FMA, DL, VT, NegSqrtS, SqrtS, SqrtX, Flags);
14393 SqrtS = DAG.getNode(ISD::FMA, DL, VT, SqrtD, SqrtH, SqrtS, Flags);
14394 }
14395
14396 SDValue ScaleDownFactor = DAG.getConstantFP(0x1.0p-16f, DL, VT);
14397
14398 SDValue ScaledDown =
14399 DAG.getNode(ISD::FMUL, DL, VT, SqrtS, ScaleDownFactor, Flags);
14400
14401 SqrtS = DAG.getNode(ISD::SELECT, DL, VT, NeedScale, ScaledDown, SqrtS, Flags);
14402 SDValue IsZeroOrInf =
14403 DAG.getNode(ISD::IS_FPCLASS, DL, MVT::i1, SqrtX,
14404 DAG.getTargetConstant(fcZero | fcPosInf, DL, MVT::i32));
14405
14406 return DAG.getNode(ISD::SELECT, DL, VT, IsZeroOrInf, SqrtX, SqrtS, Flags);
14407}
14408
14409SDValue SITargetLowering::lowerFSQRTF64(SDValue Op, SelectionDAG &DAG) const {
14410 // For double type, the SQRT and RSQ instructions don't have required
14411 // precision, we apply Goldschmidt's algorithm to improve the result:
14412 //
14413 // y0 = rsq(x)
14414 // g0 = x * y0
14415 // h0 = 0.5 * y0
14416 //
14417 // r0 = 0.5 - h0 * g0
14418 // g1 = g0 * r0 + g0
14419 // h1 = h0 * r0 + h0
14420 //
14421 // r1 = 0.5 - h1 * g1 => d0 = x - g1 * g1
14422 // g2 = g1 * r1 + g1 g2 = d0 * h1 + g1
14423 // h2 = h1 * r1 + h1
14424 //
14425 // r2 = 0.5 - h2 * g2 => d1 = x - g2 * g2
14426 // g3 = g2 * r2 + g2 g3 = d1 * h1 + g2
14427 //
14428 // sqrt(x) = g3
14429
14430 SDNodeFlags Flags = Op->getFlags();
14431
14432 SDLoc DL(Op);
14433
14434 SDValue X = Op.getOperand(0);
14435 SDValue ZeroInt = DAG.getConstant(0, DL, MVT::i32);
14436
14437 SDValue SqrtX = X;
14438 SDValue Scaling;
14439 if (!Flags.hasApproximateFuncs()) {
14440 SDValue ScaleConstant = DAG.getConstantFP(0x1.0p-767, DL, MVT::f64);
14441 Scaling = DAG.getSetCC(DL, MVT::i1, X, ScaleConstant, ISD::SETOLT);
14442
14443 // Scale up input if it is too small.
14444 SDValue ScaleUpFactor = DAG.getConstant(256, DL, MVT::i32);
14445 SDValue ScaleUp =
14446 DAG.getNode(ISD::SELECT, DL, MVT::i32, Scaling, ScaleUpFactor, ZeroInt);
14447 SqrtX = DAG.getNode(ISD::FLDEXP, DL, MVT::f64, X, ScaleUp, Flags);
14448 }
14449
14450 SDValue SqrtY = DAG.getNode(AMDGPUISD::RSQ, DL, MVT::f64, SqrtX);
14451
14452 SDValue SqrtS0 = DAG.getNode(ISD::FMUL, DL, MVT::f64, SqrtX, SqrtY);
14453
14454 SDValue Half = DAG.getConstantFP(0.5, DL, MVT::f64);
14455 SDValue SqrtH0 = DAG.getNode(ISD::FMUL, DL, MVT::f64, SqrtY, Half);
14456
14457 SDValue NegSqrtH0 = DAG.getNode(ISD::FNEG, DL, MVT::f64, SqrtH0);
14458 SDValue SqrtR0 = DAG.getNode(ISD::FMA, DL, MVT::f64, NegSqrtH0, SqrtS0, Half);
14459
14460 SDValue SqrtH1 = DAG.getNode(ISD::FMA, DL, MVT::f64, SqrtH0, SqrtR0, SqrtH0);
14461
14462 SDValue SqrtS1 = DAG.getNode(ISD::FMA, DL, MVT::f64, SqrtS0, SqrtR0, SqrtS0);
14463
14464 SDValue NegSqrtS1 = DAG.getNode(ISD::FNEG, DL, MVT::f64, SqrtS1);
14465 SDValue SqrtD0 =
14466 DAG.getNode(ISD::FMA, DL, MVT::f64, NegSqrtS1, SqrtS1, SqrtX);
14467
14468 SDValue SqrtS2 = DAG.getNode(ISD::FMA, DL, MVT::f64, SqrtD0, SqrtH1, SqrtS1);
14469
14470 SDValue SqrtRet = SqrtS2;
14471 if (!Flags.hasApproximateFuncs()) {
14472 SDValue NegSqrtS2 = DAG.getNode(ISD::FNEG, DL, MVT::f64, SqrtS2);
14473 SDValue SqrtD1 =
14474 DAG.getNode(ISD::FMA, DL, MVT::f64, NegSqrtS2, SqrtS2, SqrtX);
14475
14476 SqrtRet = DAG.getNode(ISD::FMA, DL, MVT::f64, SqrtD1, SqrtH1, SqrtS2);
14477
14478 SDValue ScaleDownFactor = DAG.getSignedConstant(-128, DL, MVT::i32);
14479 SDValue ScaleDown = DAG.getNode(ISD::SELECT, DL, MVT::i32, Scaling,
14480 ScaleDownFactor, ZeroInt);
14481 SqrtRet = DAG.getNode(ISD::FLDEXP, DL, MVT::f64, SqrtRet, ScaleDown, Flags);
14482 }
14483
14484 // TODO: Check for DAZ and expand to subnormals
14485
14486 SDValue IsZeroOrInf;
14487 if (Flags.hasNoInfs()) {
14488 SDValue Zero = DAG.getConstantFP(0.0, DL, MVT::f64);
14489 IsZeroOrInf = DAG.getSetCC(DL, MVT::i1, SqrtX, Zero, ISD::SETOEQ);
14490 } else {
14491 IsZeroOrInf =
14492 DAG.getNode(ISD::IS_FPCLASS, DL, MVT::i1, SqrtX,
14493 DAG.getTargetConstant(fcZero | fcPosInf, DL, MVT::i32));
14494 }
14495
14496 // If x is +INF, +0, or -0, use its original value
14497 return DAG.getNode(ISD::SELECT, DL, MVT::f64, IsZeroOrInf, SqrtX, SqrtRet,
14498 Flags);
14499}
14500
14501SDValue SITargetLowering::LowerTrig(SDValue Op, SelectionDAG &DAG) const {
14502 SDLoc DL(Op);
14503 EVT VT = Op.getValueType();
14504 SDValue Arg = Op.getOperand(0);
14505 SDValue TrigVal;
14506
14507 // Propagate fast-math flags so that the multiply we introduce can be folded
14508 // if Arg is already the result of a multiply by constant.
14509 auto Flags = Op->getFlags();
14510
14511 // AMDGPUISD nodes of vector type must be unrolled here since
14512 // they will not be expanded elsewhere.
14513 auto UnrollIfVec = [&DAG](SDValue V) -> SDValue {
14514 if (!V.getValueType().isVector())
14515 return V;
14516
14517 return DAG.UnrollVectorOp(cast<SDNode>(V));
14518 };
14519
14520 SDValue OneOver2Pi = DAG.getConstantFP(0.5 * numbers::inv_pi, DL, VT);
14521
14522 if (Subtarget->hasTrigReducedRange()) {
14523 SDValue MulVal = DAG.getNode(ISD::FMUL, DL, VT, Arg, OneOver2Pi, Flags);
14524 TrigVal = UnrollIfVec(DAG.getNode(AMDGPUISD::FRACT, DL, VT, MulVal, Flags));
14525 } else {
14526 TrigVal = DAG.getNode(ISD::FMUL, DL, VT, Arg, OneOver2Pi, Flags);
14527 }
14528
14529 switch (Op.getOpcode()) {
14530 case ISD::FCOS:
14531 TrigVal = DAG.getNode(AMDGPUISD::COS_HW, SDLoc(Op), VT, TrigVal, Flags);
14532 break;
14533 case ISD::FSIN:
14534 TrigVal = DAG.getNode(AMDGPUISD::SIN_HW, SDLoc(Op), VT, TrigVal, Flags);
14535 break;
14536 default:
14537 llvm_unreachable("Wrong trig opcode");
14538 }
14539
14540 return UnrollIfVec(TrigVal);
14541}
14542
14543SDValue SITargetLowering::LowerATOMIC_CMP_SWAP(SDValue Op,
14544 SelectionDAG &DAG) const {
14545 AtomicSDNode *AtomicNode = cast<AtomicSDNode>(Op);
14546 assert(AtomicNode->isCompareAndSwap());
14547 unsigned AS = AtomicNode->getAddressSpace();
14548
14549 // No custom lowering required for local address space
14551 return Op;
14552
14553 // Non-local address space requires custom lowering for atomic compare
14554 // and swap; cmp and swap should be in a v2i32 or v2i64 in case of _X2
14555 SDLoc DL(Op);
14556 SDValue ChainIn = Op.getOperand(0);
14557 SDValue Addr = Op.getOperand(1);
14558 SDValue Old = Op.getOperand(2);
14559 SDValue New = Op.getOperand(3);
14560 EVT VT = Op.getValueType();
14561 MVT SimpleVT = VT.getSimpleVT();
14562 MVT VecType = MVT::getVectorVT(SimpleVT, 2);
14563
14564 SDValue NewOld = DAG.getBuildVector(VecType, DL, {New, Old});
14565 SDValue Ops[] = {ChainIn, Addr, NewOld};
14566
14567 return DAG.getMemIntrinsicNode(AMDGPUISD::ATOMIC_CMP_SWAP, DL,
14568 Op->getVTList(), Ops, VT,
14569 AtomicNode->getMemOperand());
14570}
14571
14572//===----------------------------------------------------------------------===//
14573// Custom DAG optimizations
14574//===----------------------------------------------------------------------===//
14575
14576SDValue
14577SITargetLowering::performUCharToFloatCombine(SDNode *N,
14578 DAGCombinerInfo &DCI) const {
14579 EVT VT = N->getValueType(0);
14580 EVT ScalarVT = VT.getScalarType();
14581 if (ScalarVT != MVT::f32 && ScalarVT != MVT::f16)
14582 return SDValue();
14583
14584 SelectionDAG &DAG = DCI.DAG;
14585 SDLoc DL(N);
14586
14587 SDValue Src = N->getOperand(0);
14588 EVT SrcVT = Src.getValueType();
14589
14590 // TODO: We could try to match extracting the higher bytes, which would be
14591 // easier if i8 vectors weren't promoted to i32 vectors, particularly after
14592 // types are legalized. v4i8 -> v4f32 is probably the only case to worry
14593 // about in practice.
14594 if (DCI.isAfterLegalizeDAG() && SrcVT == MVT::i32) {
14595 if (DAG.MaskedValueIsZero(Src, APInt::getHighBitsSet(32, 24))) {
14596 SDValue Cvt = DAG.getNode(AMDGPUISD::CVT_F32_UBYTE0, DL, MVT::f32, Src);
14597 DCI.AddToWorklist(Cvt.getNode());
14598
14599 // For the f16 case, fold to a cast to f32 and then cast back to f16.
14600 if (ScalarVT != MVT::f32) {
14601 Cvt = DAG.getNode(ISD::FP_ROUND, DL, VT, Cvt,
14602 DAG.getTargetConstant(0, DL, MVT::i32));
14603 }
14604 return Cvt;
14605 }
14606 }
14607
14608 return SDValue();
14609}
14610
14611SDValue SITargetLowering::performFCopySignCombine(SDNode *N,
14612 DAGCombinerInfo &DCI) const {
14613 SDValue MagnitudeOp = N->getOperand(0);
14614 SDValue SignOp = N->getOperand(1);
14615
14616 // The generic combine for fcopysign + fp cast is too conservative with
14617 // vectors, and also gets confused by the splitting we will perform here, so
14618 // peek through FP casts.
14619 if (SignOp.getOpcode() == ISD::FP_EXTEND ||
14620 SignOp.getOpcode() == ISD::FP_ROUND)
14621 SignOp = SignOp.getOperand(0);
14622
14623 SelectionDAG &DAG = DCI.DAG;
14624 SDLoc DL(N);
14625 EVT SignVT = SignOp.getValueType();
14626
14627 // f64 fcopysign is really an f32 copysign on the high bits, so replace the
14628 // lower half with a copy.
14629 // fcopysign f64:x, _:y -> x.lo32, (fcopysign (f32 x.hi32), _:y)
14630 EVT MagVT = MagnitudeOp.getValueType();
14631
14632 unsigned NumElts = MagVT.isVector() ? MagVT.getVectorNumElements() : 1;
14633
14634 if (MagVT.getScalarType() == MVT::f64) {
14635 EVT F32VT = MagVT.isVector()
14636 ? EVT::getVectorVT(*DAG.getContext(), MVT::f32, 2 * NumElts)
14637 : MVT::v2f32;
14638
14639 SDValue MagAsVector = DAG.getNode(ISD::BITCAST, DL, F32VT, MagnitudeOp);
14640
14642 for (unsigned I = 0; I != NumElts; ++I) {
14643 SDValue MagLo =
14644 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, MagAsVector,
14645 DAG.getConstant(2 * I, DL, MVT::i32));
14646 SDValue MagHi =
14647 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, MagAsVector,
14648 DAG.getConstant(2 * I + 1, DL, MVT::i32));
14649
14650 SDValue SignOpElt =
14651 MagVT.isVector()
14653 SignOp, DAG.getConstant(I, DL, MVT::i32))
14654 : SignOp;
14655
14656 SDValue HiOp =
14657 DAG.getNode(ISD::FCOPYSIGN, DL, MVT::f32, MagHi, SignOpElt);
14658
14659 SDValue Vector =
14660 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v2f32, MagLo, HiOp);
14661
14662 SDValue NewElt = DAG.getNode(ISD::BITCAST, DL, MVT::f64, Vector);
14663 NewElts.push_back(NewElt);
14664 }
14665
14666 if (NewElts.size() == 1)
14667 return NewElts[0];
14668
14669 return DAG.getNode(ISD::BUILD_VECTOR, DL, MagVT, NewElts);
14670 }
14671
14672 if (SignVT.getScalarType() != MVT::f64)
14673 return SDValue();
14674
14675 // Reduce width of sign operand, we only need the highest bit.
14676 //
14677 // fcopysign f64:x, f64:y ->
14678 // fcopysign f64:x, (extract_vector_elt (bitcast f64:y to v2f32), 1)
14679 // TODO: In some cases it might make sense to go all the way to f16.
14680
14681 EVT F32VT = MagVT.isVector()
14682 ? EVT::getVectorVT(*DAG.getContext(), MVT::f32, 2 * NumElts)
14683 : MVT::v2f32;
14684
14685 SDValue SignAsVector = DAG.getNode(ISD::BITCAST, DL, F32VT, SignOp);
14686
14687 SmallVector<SDValue, 8> F32Signs;
14688 for (unsigned I = 0; I != NumElts; ++I) {
14689 // Take sign from odd elements of cast vector
14690 SDValue SignAsF32 =
14691 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, SignAsVector,
14692 DAG.getConstant(2 * I + 1, DL, MVT::i32));
14693 F32Signs.push_back(SignAsF32);
14694 }
14695
14696 SDValue NewSign =
14697 NumElts == 1
14698 ? F32Signs.back()
14700 EVT::getVectorVT(*DAG.getContext(), MVT::f32, NumElts),
14701 F32Signs);
14702
14703 return DAG.getNode(ISD::FCOPYSIGN, DL, N->getValueType(0), N->getOperand(0),
14704 NewSign);
14705}
14706
14707// (shl (add x, c1), c2) -> add (shl x, c2), (shl c1, c2)
14708// (shl (or x, c1), c2) -> add (shl x, c2), (shl c1, c2) iff x and c1 share no
14709// bits
14710
14711// This is a variant of
14712// (mul (add x, c1), c2) -> add (mul x, c2), (mul c1, c2),
14713//
14714// The normal DAG combiner will do this, but only if the add has one use since
14715// that would increase the number of instructions.
14716//
14717// This prevents us from seeing a constant offset that can be folded into a
14718// memory instruction's addressing mode. If we know the resulting add offset of
14719// a pointer can be folded into an addressing offset, we can replace the pointer
14720// operand with the add of new constant offset. This eliminates one of the uses,
14721// and may allow the remaining use to also be simplified.
14722//
14723SDValue SITargetLowering::performSHLPtrCombine(SDNode *N, unsigned AddrSpace,
14724 EVT MemVT,
14725 DAGCombinerInfo &DCI) const {
14726 SDValue N0 = N->getOperand(0);
14727 SDValue N1 = N->getOperand(1);
14728
14729 // We only do this to handle cases where it's profitable when there are
14730 // multiple uses of the add, so defer to the standard combine.
14731 if ((!N0->isAnyAdd() && N0.getOpcode() != ISD::OR) || N0->hasOneUse())
14732 return SDValue();
14733
14734 const ConstantSDNode *CN1 = dyn_cast<ConstantSDNode>(N1);
14735 if (!CN1)
14736 return SDValue();
14737
14738 const ConstantSDNode *CAdd = dyn_cast<ConstantSDNode>(N0.getOperand(1));
14739 if (!CAdd)
14740 return SDValue();
14741
14742 SelectionDAG &DAG = DCI.DAG;
14743
14744 if (N0->getOpcode() == ISD::OR &&
14745 !DAG.haveNoCommonBitsSet(N0.getOperand(0), N0.getOperand(1)))
14746 return SDValue();
14747
14748 // If the resulting offset is too large, we can't fold it into the
14749 // addressing mode offset.
14750 APInt Offset = CAdd->getAPIntValue() << CN1->getAPIntValue();
14751 Type *Ty = MemVT.getTypeForEVT(*DCI.DAG.getContext());
14752
14753 AddrMode AM;
14754 AM.HasBaseReg = true;
14755 AM.BaseOffs = Offset.getSExtValue();
14756 if (!isLegalAddressingMode(DCI.DAG.getDataLayout(), AM, Ty, AddrSpace))
14757 return SDValue();
14758
14759 SDLoc SL(N);
14760 EVT VT = N->getValueType(0);
14761
14762 SDValue ShlX = DAG.getNode(ISD::SHL, SL, VT, N0.getOperand(0), N1);
14763 SDValue COffset = DAG.getConstant(Offset, SL, VT);
14764
14765 SDNodeFlags Flags;
14766 Flags.setNoUnsignedWrap(
14767 N->getFlags().hasNoUnsignedWrap() &&
14768 (N0.getOpcode() == ISD::OR || N0->getFlags().hasNoUnsignedWrap()));
14769
14770 // Use ISD::ADD even if the original operation was ISD::PTRADD, since we can't
14771 // be sure that the new left operand is a proper base pointer.
14772 return DAG.getNode(ISD::ADD, SL, VT, ShlX, COffset, Flags);
14773}
14774
14775/// MemSDNode::getBasePtr() does not work for intrinsics, which needs to offset
14776/// by the chain and intrinsic ID. Theoretically we would also need to check the
14777/// specific intrinsic, but they all place the pointer operand first.
14778static unsigned getBasePtrIndex(const MemSDNode *N) {
14779 switch (N->getOpcode()) {
14780 case ISD::STORE:
14783 return 2;
14784 default:
14785 return 1;
14786 }
14787}
14788
14789SDValue SITargetLowering::performMemSDNodeCombine(MemSDNode *N,
14790 DAGCombinerInfo &DCI) const {
14791 SelectionDAG &DAG = DCI.DAG;
14792
14793 unsigned PtrIdx = getBasePtrIndex(N);
14794 SDValue Ptr = N->getOperand(PtrIdx);
14795
14796 // TODO: We could also do this for multiplies.
14797 if (Ptr.getOpcode() == ISD::SHL) {
14798 SDValue NewPtr = performSHLPtrCombine(Ptr.getNode(), N->getAddressSpace(),
14799 N->getMemoryVT(), DCI);
14800 if (NewPtr) {
14801 SmallVector<SDValue, 8> NewOps(N->ops());
14802
14803 NewOps[PtrIdx] = NewPtr;
14804 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
14805 }
14806 }
14807
14808 return SDValue();
14809}
14810
14811static bool bitOpWithConstantIsReducible(unsigned Opc, uint32_t Val) {
14812 return (Opc == ISD::AND && (Val == 0 || Val == 0xffffffff)) ||
14813 (Opc == ISD::OR && (Val == 0xffffffff || Val == 0)) ||
14814 (Opc == ISD::XOR && Val == 0);
14815}
14816
14817// Break up 64-bit bit operation of a constant into two 32-bit and/or/xor. This
14818// will typically happen anyway for a VALU 64-bit and. This exposes other 32-bit
14819// integer combine opportunities since most 64-bit operations are decomposed
14820// this way. TODO: We won't want this for SALU especially if it is an inline
14821// immediate.
14822SDValue SITargetLowering::splitBinaryBitConstantOp(
14823 DAGCombinerInfo &DCI, const SDLoc &SL, unsigned Opc, SDValue LHS,
14824 const ConstantSDNode *CRHS) const {
14825 uint64_t Val = CRHS->getZExtValue();
14826 uint32_t ValLo = Lo_32(Val);
14827 uint32_t ValHi = Hi_32(Val);
14828 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
14829
14830 if ((bitOpWithConstantIsReducible(Opc, ValLo) ||
14832 (CRHS->hasOneUse() && !TII->isInlineConstant(CRHS->getAPIntValue()))) {
14833 // We have 64-bit scalar and/or/xor, but do not have vector forms.
14834 if (Subtarget->has64BitLiterals() && CRHS->hasOneUse() &&
14835 !CRHS->user_begin()->isDivergent())
14836 return SDValue();
14837
14838 // If we need to materialize a 64-bit immediate, it will be split up later
14839 // anyway. Avoid creating the harder to understand 64-bit immediate
14840 // materialization.
14841 return splitBinaryBitConstantOpImpl(DCI, SL, Opc, LHS, ValLo, ValHi);
14842 }
14843
14844 return SDValue();
14845}
14846
14848 if (V.getValueType() != MVT::i1)
14849 return false;
14850 switch (V.getOpcode()) {
14851 default:
14852 break;
14853 case ISD::SETCC:
14854 case ISD::IS_FPCLASS:
14855 case AMDGPUISD::FP_CLASS:
14856 return true;
14857 case ISD::AND:
14858 case ISD::OR:
14859 case ISD::XOR:
14860 return isBoolSGPR(V.getOperand(0)) && isBoolSGPR(V.getOperand(1));
14861 case ISD::SADDO:
14862 case ISD::UADDO:
14863 case ISD::SSUBO:
14864 case ISD::USUBO:
14865 case ISD::SMULO:
14866 case ISD::UMULO:
14867 return V.getResNo() == 1;
14869 unsigned IntrinsicID = V.getConstantOperandVal(0);
14870 switch (IntrinsicID) {
14871 case Intrinsic::amdgcn_is_shared:
14872 case Intrinsic::amdgcn_is_private:
14873 return true;
14874 default:
14875 return false;
14876 }
14877
14878 return false;
14879 }
14880 }
14881 return false;
14882}
14883
14885 return (V.getOpcode() == ISD::SIGN_EXTEND ||
14886 V.getOpcode() == ISD::ANY_EXTEND) &&
14887 isBoolSGPR(V.getOperand(0));
14888}
14889
14890// If a constant has all zeroes or all ones within each byte return it.
14891// Otherwise return 0.
14893 // 0xff for any zero byte in the mask
14894 uint32_t ZeroByteMask = 0;
14895 if (!(C & 0x000000ff))
14896 ZeroByteMask |= 0x000000ff;
14897 if (!(C & 0x0000ff00))
14898 ZeroByteMask |= 0x0000ff00;
14899 if (!(C & 0x00ff0000))
14900 ZeroByteMask |= 0x00ff0000;
14901 if (!(C & 0xff000000))
14902 ZeroByteMask |= 0xff000000;
14903 uint32_t NonZeroByteMask = ~ZeroByteMask; // 0xff for any non-zero byte
14904 if ((NonZeroByteMask & C) != NonZeroByteMask)
14905 return 0; // Partial bytes selected.
14906 return C;
14907}
14908
14909// Check if a node selects whole bytes from its operand 0 starting at a byte
14910// boundary while masking the rest. Returns select mask as in the v_perm_b32
14911// or -1 if not succeeded.
14912// Note byte select encoding:
14913// value 0-3 selects corresponding source byte;
14914// value 0xc selects zero;
14915// value 0xff selects 0xff.
14917 assert(V.getValueSizeInBits() == 32);
14918
14919 if (V.getNumOperands() != 2)
14920 return ~0;
14921
14922 ConstantSDNode *N1 = dyn_cast<ConstantSDNode>(V.getOperand(1));
14923 if (!N1)
14924 return ~0;
14925
14926 uint32_t C = N1->getZExtValue();
14927
14928 switch (V.getOpcode()) {
14929 default:
14930 break;
14931 case ISD::AND:
14932 if (uint32_t ConstMask = getConstantPermuteMask(C))
14933 return (0x03020100 & ConstMask) | (0x0c0c0c0c & ~ConstMask);
14934 break;
14935
14936 case ISD::OR:
14937 if (uint32_t ConstMask = getConstantPermuteMask(C))
14938 return (0x03020100 & ~ConstMask) | ConstMask;
14939 break;
14940
14941 case ISD::SHL:
14942 if (C % 8)
14943 return ~0;
14944
14945 return uint32_t((0x030201000c0c0c0cull << C) >> 32);
14946
14947 case ISD::SRL:
14948 if (C % 8)
14949 return ~0;
14950
14951 return uint32_t(0x0c0c0c0c03020100ull >> C);
14952 }
14953
14954 return ~0;
14955}
14956
14957SDValue SITargetLowering::performAndCombine(SDNode *N,
14958 DAGCombinerInfo &DCI) const {
14959 if (DCI.isBeforeLegalize())
14960 return SDValue();
14961
14962 SelectionDAG &DAG = DCI.DAG;
14963 EVT VT = N->getValueType(0);
14964 SDValue LHS = N->getOperand(0);
14965 SDValue RHS = N->getOperand(1);
14966
14967 const ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(RHS);
14968 if (VT == MVT::i64 && CRHS) {
14969 if (SDValue Split =
14970 splitBinaryBitConstantOp(DCI, SDLoc(N), ISD::AND, LHS, CRHS))
14971 return Split;
14972 }
14973
14974 if (CRHS && VT == MVT::i32) {
14975 // and (srl x, c), mask => shl (bfe x, nb + c, mask >> nb), nb
14976 // nb = number of trailing zeroes in mask
14977 // It can be optimized out using SDWA for GFX8+ in the SDWA peephole pass,
14978 // given that we are selecting 8 or 16 bit fields starting at byte boundary.
14979 uint64_t Mask = CRHS->getZExtValue();
14980 unsigned Bits = llvm::popcount(Mask);
14981 if (getSubtarget()->hasSDWA() && LHS->getOpcode() == ISD::SRL &&
14982 (Bits == 8 || Bits == 16) && isShiftedMask_64(Mask) && !(Mask & 1)) {
14983 if (auto *CShift = dyn_cast<ConstantSDNode>(LHS->getOperand(1))) {
14984 unsigned Shift = CShift->getZExtValue();
14985 unsigned NB = CRHS->getAPIntValue().countr_zero();
14986 unsigned Offset = NB + Shift;
14987 if ((Offset & (Bits - 1)) == 0) { // Starts at a byte or word boundary.
14988 SDLoc SL(N);
14989 SDValue BFE =
14990 DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32, LHS->getOperand(0),
14991 DAG.getConstant(Offset, SL, MVT::i32),
14992 DAG.getConstant(Bits, SL, MVT::i32));
14993 EVT NarrowVT = EVT::getIntegerVT(*DAG.getContext(), Bits);
14994 SDValue Ext = DAG.getNode(ISD::AssertZext, SL, VT, BFE,
14995 DAG.getValueType(NarrowVT));
14996 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(LHS), VT, Ext,
14997 DAG.getConstant(NB, SDLoc(CRHS), MVT::i32));
14998 return Shl;
14999 }
15000 }
15001 }
15002
15003 // and (perm x, y, c1), c2 -> perm x, y, permute_mask(c1, c2)
15004 if (LHS.hasOneUse() && LHS.getOpcode() == AMDGPUISD::PERM &&
15005 isa<ConstantSDNode>(LHS.getOperand(2))) {
15006 uint32_t Sel = getConstantPermuteMask(Mask);
15007 if (!Sel)
15008 return SDValue();
15009
15010 // Select 0xc for all zero bytes
15011 Sel = (LHS.getConstantOperandVal(2) & Sel) | (~Sel & 0x0c0c0c0c);
15012 SDLoc DL(N);
15013 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, LHS.getOperand(0),
15014 LHS.getOperand(1), DAG.getConstant(Sel, DL, MVT::i32));
15015 }
15016 }
15017
15018 // and (trunc (srl x, c)), mask => srl (trunc x), c
15019 // if mask keeps exactly the bits that srl leaves in the low half
15020 if (CRHS && VT == MVT::i16 && LHS.getOpcode() == ISD::TRUNCATE &&
15021 LHS.hasOneUse() && LHS.getOperand(0).getOpcode() == ISD::SRL) {
15022 auto *Shift = dyn_cast<ConstantSDNode>(LHS.getOperand(0).getOperand(1));
15023 if (Shift && Shift->getZExtValue() < 16 &&
15024 CRHS->getAPIntValue().isMask(16 - Shift->getZExtValue())) {
15025 SDLoc SL(N);
15026 SDValue Trunc =
15027 DAG.getNode(ISD::TRUNCATE, SL, VT, LHS.getOperand(0).getOperand(0));
15028 return DAG.getNode(
15029 ISD::SRL, SL, VT, Trunc,
15030 DAG.getShiftAmountConstant(Shift->getZExtValue(), VT, SL));
15031 }
15032 }
15033
15034 // (and (fcmp ord x, x), (fcmp une (fabs x), inf)) ->
15035 // fp_class x, ~(s_nan | q_nan | n_infinity | p_infinity)
15036 if (LHS.getOpcode() == ISD::SETCC && RHS.getOpcode() == ISD::SETCC) {
15037 ISD::CondCode LCC = LHS.getOperandCondCode(2);
15038 ISD::CondCode RCC = RHS.getOperandCondCode(2);
15039
15040 SDValue X = LHS.getOperand(0);
15041 SDValue Y = RHS.getOperand(0);
15042 if (Y.getOpcode() != ISD::FABS || Y.getOperand(0) != X ||
15043 !isTypeLegal(X.getValueType()))
15044 return SDValue();
15045
15046 if (LCC == ISD::SETO) {
15047 if (X != LHS.getOperand(1))
15048 return SDValue();
15049
15050 if (RCC == ISD::SETUNE) {
15051 const ConstantFPSDNode *C1 =
15052 dyn_cast<ConstantFPSDNode>(RHS.getOperand(1));
15053 if (!C1 || !C1->isInfinity() || C1->isNegative())
15054 return SDValue();
15055
15056 const uint32_t Mask = SIInstrFlags::N_NORMAL |
15060
15061 static_assert(
15064 0x3ff) == Mask,
15065 "mask not equal");
15066
15067 SDLoc DL(N);
15068 return DAG.getNode(AMDGPUISD::FP_CLASS, DL, MVT::i1, X,
15069 DAG.getConstant(Mask, DL, MVT::i32));
15070 }
15071 }
15072 }
15073
15074 if (RHS.getOpcode() == ISD::SETCC && LHS.getOpcode() == AMDGPUISD::FP_CLASS)
15075 std::swap(LHS, RHS);
15076
15077 if (LHS.getOpcode() == ISD::SETCC && RHS.getOpcode() == AMDGPUISD::FP_CLASS &&
15078 RHS.hasOneUse()) {
15079 ISD::CondCode LCC = LHS.getOperandCondCode(2);
15080 // and (fcmp seto), (fp_class x, mask) -> fp_class x, mask & ~(p_nan |
15081 // n_nan) and (fcmp setuo), (fp_class x, mask) -> fp_class x, mask & (p_nan
15082 // | n_nan)
15083 const ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(RHS.getOperand(1));
15084 if ((LCC == ISD::SETO || LCC == ISD::SETUO) && Mask &&
15085 (RHS.getOperand(0) == LHS.getOperand(0) &&
15086 LHS.getOperand(0) == LHS.getOperand(1))) {
15087 const unsigned OrdMask = SIInstrFlags::S_NAN | SIInstrFlags::Q_NAN;
15088 unsigned NewMask = LCC == ISD::SETO ? Mask->getZExtValue() & ~OrdMask
15089 : Mask->getZExtValue() & OrdMask;
15090
15091 SDLoc DL(N);
15092 return DAG.getNode(AMDGPUISD::FP_CLASS, DL, MVT::i1, RHS.getOperand(0),
15093 DAG.getConstant(NewMask, DL, MVT::i32));
15094 }
15095 }
15096
15097 if (VT == MVT::i32 && (RHS.getOpcode() == ISD::SIGN_EXTEND ||
15098 LHS.getOpcode() == ISD::SIGN_EXTEND)) {
15099 // and x, (sext cc from i1) => select cc, x, 0
15100 if (RHS.getOpcode() != ISD::SIGN_EXTEND)
15101 std::swap(LHS, RHS);
15102 if (isBoolSGPR(RHS.getOperand(0)))
15103 return DAG.getSelect(SDLoc(N), MVT::i32, RHS.getOperand(0), LHS,
15104 DAG.getConstant(0, SDLoc(N), MVT::i32));
15105 }
15106
15107 // and (op x, c1), (op y, c2) -> perm x, y, permute_mask(c1, c2)
15108 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
15109 if (VT == MVT::i32 && LHS.hasOneUse() && RHS.hasOneUse() &&
15110 TII->pseudoToMCOpcode(AMDGPU::V_PERM_B32_e64) != -1) {
15111 uint32_t LHSMask = getPermuteMask(LHS);
15112 uint32_t RHSMask = getPermuteMask(RHS);
15113 if (LHSMask != ~0u && RHSMask != ~0u) {
15114 // Canonicalize the expression in an attempt to have fewer unique masks
15115 // and therefore fewer registers used to hold the masks.
15116 if (LHSMask > RHSMask) {
15117 std::swap(LHSMask, RHSMask);
15118 std::swap(LHS, RHS);
15119 }
15120
15121 // Select 0xc for each lane used from source operand. Zero has 0xc mask
15122 // set, 0xff have 0xff in the mask, actual lanes are in the 0-3 range.
15123 uint32_t LHSUsedLanes = ~(LHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
15124 uint32_t RHSUsedLanes = ~(RHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
15125
15126 // Check of we need to combine values from two sources within a byte.
15127 if (!(LHSUsedLanes & RHSUsedLanes) &&
15128 // If we select high and lower word keep it for SDWA.
15129 // TODO: teach SDWA to work with v_perm_b32 and remove the check.
15130 !(LHSUsedLanes == 0x0c0c0000 && RHSUsedLanes == 0x00000c0c)) {
15131 // Each byte in each mask is either selector mask 0-3, or has higher
15132 // bits set in either of masks, which can be 0xff for 0xff or 0x0c for
15133 // zero. If 0x0c is in either mask it shall always be 0x0c. Otherwise
15134 // mask which is not 0xff wins. By anding both masks we have a correct
15135 // result except that 0x0c shall be corrected to give 0x0c only.
15136 uint32_t Mask = LHSMask & RHSMask;
15137 for (unsigned I = 0; I < 32; I += 8) {
15138 uint32_t ByteSel = 0xff << I;
15139 if ((LHSMask & ByteSel) == 0x0c || (RHSMask & ByteSel) == 0x0c)
15140 Mask &= (0x0c << I) & 0xffffffff;
15141 }
15142
15143 // Add 4 to each active LHS lane. It will not affect any existing 0xff
15144 // or 0x0c.
15145 uint32_t Sel = Mask | (LHSUsedLanes & 0x04040404);
15146 SDLoc DL(N);
15147
15148 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, LHS.getOperand(0),
15149 RHS.getOperand(0),
15150 DAG.getConstant(Sel, DL, MVT::i32));
15151 }
15152 }
15153 }
15154
15155 return SDValue();
15156}
15157
15158// A key component of v_perm is a mapping between byte position of the src
15159// operands, and the byte position of the dest. To provide such, we need: 1. the
15160// node that provides x byte of the dest of the OR, and 2. the byte of the node
15161// used to provide that x byte. calculateByteProvider finds which node provides
15162// a certain byte of the dest of the OR, and calculateSrcByte takes that node,
15163// and finds an ultimate src and byte position For example: The supported
15164// LoadCombine pattern for vector loads is as follows
15165// t1
15166// or
15167// / \
15168// t2 t3
15169// zext shl
15170// | | \
15171// t4 t5 16
15172// or anyext
15173// / \ |
15174// t6 t7 t8
15175// srl shl or
15176// / | / \ / \
15177// t9 t10 t11 t12 t13 t14
15178// trunc* 8 trunc* 8 and and
15179// | | / | | \
15180// t15 t16 t17 t18 t19 t20
15181// trunc* 255 srl -256
15182// | / \
15183// t15 t15 16
15184//
15185// *In this example, the truncs are from i32->i16
15186//
15187// calculateByteProvider would find t6, t7, t13, and t14 for bytes 0-3
15188// respectively. calculateSrcByte would find (given node) -> ultimate src &
15189// byteposition: t6 -> t15 & 1, t7 -> t16 & 0, t13 -> t15 & 0, t14 -> t15 & 3.
15190// After finding the mapping, we can combine the tree into vperm t15, t16,
15191// 0x05000407
15192
15193// Find the source and byte position from a node.
15194// \p DestByte is the byte position of the dest of the or that the src
15195// ultimately provides. \p SrcIndex is the byte of the src that maps to this
15196// dest of the or byte. \p Depth tracks how many recursive iterations we have
15197// performed.
15198static const std::optional<ByteProvider<SDValue>>
15199calculateSrcByte(const SDValue Op, uint64_t DestByte, uint64_t SrcIndex = 0,
15200 unsigned Depth = 0) {
15201 // We may need to recursively traverse a series of SRLs
15202 if (Depth >= 6)
15203 return std::nullopt;
15204
15205 if (Op.getValueSizeInBits() < 8)
15206 return std::nullopt;
15207
15208 if (Op.getValueType().isVector())
15209 return ByteProvider<SDValue>::getSrc(Op, DestByte, SrcIndex);
15210
15211 switch (Op->getOpcode()) {
15212 case ISD::TRUNCATE: {
15213 return calculateSrcByte(Op->getOperand(0), DestByte, SrcIndex, Depth + 1);
15214 }
15215
15216 case ISD::ANY_EXTEND:
15217 case ISD::SIGN_EXTEND:
15218 case ISD::ZERO_EXTEND:
15220 SDValue NarrowOp = Op->getOperand(0);
15221 auto NarrowVT = NarrowOp.getValueType();
15222 if (Op->getOpcode() == ISD::SIGN_EXTEND_INREG) {
15223 auto *VTSign = cast<VTSDNode>(Op->getOperand(1));
15224 NarrowVT = VTSign->getVT();
15225 }
15226 if (!NarrowVT.isByteSized())
15227 return std::nullopt;
15228 uint64_t NarrowByteWidth = NarrowVT.getStoreSize();
15229
15230 if (SrcIndex >= NarrowByteWidth)
15231 return std::nullopt;
15232 return calculateSrcByte(Op->getOperand(0), DestByte, SrcIndex, Depth + 1);
15233 }
15234
15235 case ISD::SRA:
15236 case ISD::SRL: {
15237 auto *ShiftOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
15238 if (!ShiftOp)
15239 return std::nullopt;
15240
15241 uint64_t BitShift = ShiftOp->getZExtValue();
15242
15243 if (BitShift % 8 != 0)
15244 return std::nullopt;
15245
15246 uint64_t NewSrcIndex = SrcIndex + BitShift / 8;
15247 if (NewSrcIndex >= Op.getScalarValueSizeInBits() / 8)
15248 return std::nullopt;
15249
15250 return calculateSrcByte(Op->getOperand(0), DestByte, NewSrcIndex,
15251 Depth + 1);
15252 }
15253
15254 default: {
15255 return ByteProvider<SDValue>::getSrc(Op, DestByte, SrcIndex);
15256 }
15257 }
15258 llvm_unreachable("fully handled switch");
15259}
15260
15261// For a byte position in the result of an Or, traverse the tree and find the
15262// node (and the byte of the node) which ultimately provides this {Or,
15263// BytePosition}. \p Op is the operand we are currently examining. \p Index is
15264// the byte position of the Op that corresponds with the originally requested
15265// byte of the Or \p Depth tracks how many recursive iterations we have
15266// performed. \p StartingIndex is the originally requested byte of the Or
15267static const std::optional<ByteProvider<SDValue>>
15268calculateByteProvider(const SDValue &Op, unsigned Index, unsigned Depth,
15269 unsigned StartingIndex = 0) {
15270 // Finding Src tree of RHS of or typically requires at least 1 additional
15271 // depth
15272 if (Depth > 6)
15273 return std::nullopt;
15274
15275 unsigned BitWidth = Op.getScalarValueSizeInBits();
15276 if (BitWidth % 8 != 0)
15277 return std::nullopt;
15278 if (Index > BitWidth / 8 - 1)
15279 return std::nullopt;
15280
15281 bool IsVec = Op.getValueType().isVector();
15282 switch (Op.getOpcode()) {
15283 case ISD::OR: {
15284 if (IsVec)
15285 return std::nullopt;
15286
15287 auto RHS = calculateByteProvider(Op.getOperand(1), Index, Depth + 1,
15288 StartingIndex);
15289 if (!RHS)
15290 return std::nullopt;
15291 auto LHS = calculateByteProvider(Op.getOperand(0), Index, Depth + 1,
15292 StartingIndex);
15293 if (!LHS)
15294 return std::nullopt;
15295 // A well formed Or will have two ByteProviders for each byte, one of which
15296 // is constant zero
15297 if (!LHS->isConstantZero() && !RHS->isConstantZero())
15298 return std::nullopt;
15299 if (!LHS || LHS->isConstantZero())
15300 return RHS;
15301 if (!RHS || RHS->isConstantZero())
15302 return LHS;
15303 return std::nullopt;
15304 }
15305
15306 case ISD::AND: {
15307 if (IsVec)
15308 return std::nullopt;
15309
15310 auto *BitMaskOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
15311 if (!BitMaskOp)
15312 return std::nullopt;
15313
15314 uint32_t BitMask = BitMaskOp->getZExtValue();
15315 // Bits we expect for our StartingIndex
15316 uint32_t IndexMask = 0xFF << (Index * 8);
15317
15318 if ((IndexMask & BitMask) != IndexMask) {
15319 // If the result of the and partially provides the byte, then it
15320 // is not well formatted
15321 if (IndexMask & BitMask)
15322 return std::nullopt;
15324 }
15325
15326 return calculateSrcByte(Op->getOperand(0), StartingIndex, Index);
15327 }
15328
15329 case ISD::FSHR: {
15330 if (IsVec)
15331 return std::nullopt;
15332
15333 // fshr(X,Y,Z): (X << (BW - (Z % BW))) | (Y >> (Z % BW))
15334 auto *ShiftOp = dyn_cast<ConstantSDNode>(Op->getOperand(2));
15335 if (!ShiftOp || Op.getValueType().isVector())
15336 return std::nullopt;
15337
15338 uint64_t BitsProvided = Op.getValueSizeInBits();
15339 if (BitsProvided % 8 != 0)
15340 return std::nullopt;
15341
15342 uint64_t BitShift = ShiftOp->getAPIntValue().urem(BitsProvided);
15343 if (BitShift % 8)
15344 return std::nullopt;
15345
15346 uint64_t ConcatSizeInBytes = BitsProvided / 4;
15347 uint64_t ByteShift = BitShift / 8;
15348
15349 uint64_t NewIndex = (Index + ByteShift) % ConcatSizeInBytes;
15350 uint64_t BytesProvided = BitsProvided / 8;
15351 SDValue NextOp = Op.getOperand(NewIndex >= BytesProvided ? 0 : 1);
15352 NewIndex %= BytesProvided;
15353 return calculateByteProvider(NextOp, NewIndex, Depth + 1, StartingIndex);
15354 }
15355
15356 case ISD::SRA:
15357 case ISD::SRL: {
15358 if (IsVec)
15359 return std::nullopt;
15360
15361 auto *ShiftOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
15362 if (!ShiftOp)
15363 return std::nullopt;
15364
15365 uint64_t BitShift = ShiftOp->getZExtValue();
15366 if (BitShift % 8)
15367 return std::nullopt;
15368
15369 auto BitsProvided = Op.getScalarValueSizeInBits();
15370 if (BitsProvided % 8 != 0)
15371 return std::nullopt;
15372
15373 uint64_t BytesProvided = BitsProvided / 8;
15374 uint64_t ByteShift = BitShift / 8;
15375 if (Index + ByteShift < BytesProvided)
15376 return calculateSrcByte(Op->getOperand(0), StartingIndex,
15377 Index + ByteShift);
15378 // SRA's out-of-range bytes are sign bits, not constant zero.
15379 if (Op.getOpcode() == ISD::SRA)
15380 return std::nullopt;
15382 }
15383
15384 case ISD::SHL: {
15385 if (IsVec)
15386 return std::nullopt;
15387
15388 auto *ShiftOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
15389 if (!ShiftOp)
15390 return std::nullopt;
15391
15392 uint64_t BitShift = ShiftOp->getZExtValue();
15393 if (BitShift % 8 != 0)
15394 return std::nullopt;
15395 uint64_t ByteShift = BitShift / 8;
15396
15397 // If we are shifting by an amount greater than (or equal to)
15398 // the index we are trying to provide, then it provides 0s. If not,
15399 // then this bytes are not definitively 0s, and the corresponding byte
15400 // of interest is Index - ByteShift of the src
15401 return Index < ByteShift
15403 : calculateByteProvider(Op.getOperand(0), Index - ByteShift,
15404 Depth + 1, StartingIndex);
15405 }
15406 case ISD::ANY_EXTEND:
15407 case ISD::SIGN_EXTEND:
15408 case ISD::ZERO_EXTEND:
15410 case ISD::AssertZext:
15411 case ISD::AssertSext: {
15412 if (IsVec)
15413 return std::nullopt;
15414
15415 SDValue NarrowOp = Op->getOperand(0);
15416 unsigned NarrowBitWidth = NarrowOp.getValueSizeInBits();
15417 if (Op->getOpcode() == ISD::SIGN_EXTEND_INREG ||
15418 Op->getOpcode() == ISD::AssertZext ||
15419 Op->getOpcode() == ISD::AssertSext) {
15420 auto *VTSign = cast<VTSDNode>(Op->getOperand(1));
15421 NarrowBitWidth = VTSign->getVT().getSizeInBits();
15422 }
15423 if (NarrowBitWidth % 8 != 0)
15424 return std::nullopt;
15425 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
15426
15427 if (Index >= NarrowByteWidth)
15428 return Op.getOpcode() == ISD::ZERO_EXTEND
15429 ? std::optional<ByteProvider<SDValue>>(
15431 : std::nullopt;
15432 return calculateByteProvider(NarrowOp, Index, Depth + 1, StartingIndex);
15433 }
15434
15435 case ISD::TRUNCATE: {
15436 if (IsVec)
15437 return std::nullopt;
15438
15439 uint64_t NarrowByteWidth = BitWidth / 8;
15440
15441 if (NarrowByteWidth >= Index) {
15442 return calculateByteProvider(Op.getOperand(0), Index, Depth + 1,
15443 StartingIndex);
15444 }
15445
15446 return std::nullopt;
15447 }
15448
15449 case ISD::CopyFromReg: {
15450 if (BitWidth / 8 > Index)
15451 return calculateSrcByte(Op, StartingIndex, Index);
15452
15453 return std::nullopt;
15454 }
15455
15456 case ISD::LOAD: {
15457 auto *L = cast<LoadSDNode>(Op.getNode());
15458
15459 unsigned NarrowBitWidth = L->getMemoryVT().getSizeInBits();
15460 if (NarrowBitWidth % 8 != 0)
15461 return std::nullopt;
15462 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
15463
15464 // If the width of the load does not reach byte we are trying to provide for
15465 // and it is not a ZEXTLOAD, then the load does not provide for the byte in
15466 // question
15467 if (Index >= NarrowByteWidth) {
15468 return L->getExtensionType() == ISD::ZEXTLOAD
15469 ? std::optional<ByteProvider<SDValue>>(
15471 : std::nullopt;
15472 }
15473
15474 if (NarrowByteWidth > Index) {
15475 return calculateSrcByte(Op, StartingIndex, Index);
15476 }
15477
15478 return std::nullopt;
15479 }
15480
15481 case ISD::BSWAP: {
15482 if (IsVec)
15483 return std::nullopt;
15484
15485 return calculateByteProvider(Op->getOperand(0), BitWidth / 8 - Index - 1,
15486 Depth + 1, StartingIndex);
15487 }
15488
15490 auto *IdxOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
15491 if (!IdxOp)
15492 return std::nullopt;
15493 auto VecIdx = IdxOp->getZExtValue();
15494 auto ScalarSize = Op.getScalarValueSizeInBits();
15495 if (ScalarSize < 32)
15496 Index = ScalarSize == 8 ? VecIdx : VecIdx * 2 + Index;
15497 return calculateSrcByte(ScalarSize >= 32 ? Op : Op.getOperand(0),
15498 StartingIndex, Index);
15499 }
15500
15501 case AMDGPUISD::PERM: {
15502 if (IsVec)
15503 return std::nullopt;
15504
15505 auto *PermMask = dyn_cast<ConstantSDNode>(Op->getOperand(2));
15506 if (!PermMask)
15507 return std::nullopt;
15508
15509 auto IdxMask =
15510 (PermMask->getZExtValue() & (0xFF << (Index * 8))) >> (Index * 8);
15511 if (IdxMask > 0x07 && IdxMask != 0x0c)
15512 return std::nullopt;
15513
15514 auto NextOp = Op.getOperand(IdxMask > 0x03 ? 0 : 1);
15515 auto NextIndex = IdxMask > 0x03 ? IdxMask % 4 : IdxMask;
15516
15517 return IdxMask != 0x0c ? calculateSrcByte(NextOp, StartingIndex, NextIndex)
15520 }
15521
15522 default: {
15523 return std::nullopt;
15524 }
15525 }
15526
15527 llvm_unreachable("fully handled switch");
15528}
15529
15530// Returns true if the Operand is a scalar and is 16 bits
15531static bool isExtendedFrom16Bits(SDValue &Operand) {
15532
15533 switch (Operand.getOpcode()) {
15534 case ISD::ANY_EXTEND:
15535 case ISD::SIGN_EXTEND:
15536 case ISD::ZERO_EXTEND: {
15537 auto OpVT = Operand.getOperand(0).getValueType();
15538 return !OpVT.isVector() && OpVT.getSizeInBits() == 16;
15539 }
15540 case ISD::LOAD: {
15541 LoadSDNode *L = cast<LoadSDNode>(Operand.getNode());
15542 auto ExtType = cast<LoadSDNode>(L)->getExtensionType();
15543 if (ExtType == ISD::ZEXTLOAD || ExtType == ISD::SEXTLOAD ||
15544 ExtType == ISD::EXTLOAD) {
15545 auto MemVT = L->getMemoryVT();
15546 return !MemVT.isVector() && MemVT.getSizeInBits() == 16;
15547 }
15548 return L->getMemoryVT().getSizeInBits() == 16;
15549 }
15550 default:
15551 return false;
15552 }
15553}
15554
15555// Returns true if the mask matches consecutive bytes, and the first byte
15556// begins at a power of 2 byte offset from 0th byte
15557static bool addresses16Bits(int Mask) {
15558 int Low8 = Mask & 0xff;
15559 int Hi8 = (Mask & 0xff00) >> 8;
15560
15561 assert(Low8 < 8 && Hi8 < 8);
15562 // Are the bytes contiguous in the order of increasing addresses.
15563 bool IsConsecutive = (Hi8 - Low8 == 1);
15564 // Is the first byte at location that is aligned for 16 bit instructions.
15565 // A counter example is taking 2 consecutive bytes starting at the 8th bit.
15566 // In this case, we still need code to extract the 16 bit operand, so it
15567 // is better to use i8 v_perm
15568 bool Is16Aligned = !(Low8 % 2);
15569
15570 return IsConsecutive && Is16Aligned;
15571}
15572
15573// Do not lower into v_perm if the operands are actually 16 bit
15574// and the selected bits (based on PermMask) correspond with two
15575// easily addressable 16 bit operands.
15577 SDValue &OtherOp) {
15578 int Low16 = PermMask & 0xffff;
15579 int Hi16 = (PermMask & 0xffff0000) >> 16;
15580
15581 auto TempOp = peekThroughBitcasts(Op);
15582 auto TempOtherOp = peekThroughBitcasts(OtherOp);
15583
15584 auto OpIs16Bit =
15585 TempOp.getValueSizeInBits() == 16 || isExtendedFrom16Bits(TempOp);
15586 if (!OpIs16Bit)
15587 return true;
15588
15589 auto OtherOpIs16Bit = TempOtherOp.getValueSizeInBits() == 16 ||
15590 isExtendedFrom16Bits(TempOtherOp);
15591 if (!OtherOpIs16Bit)
15592 return true;
15593
15594 // Do we cleanly address both
15595 return !addresses16Bits(Low16) || !addresses16Bits(Hi16);
15596}
15597
15599 unsigned DWordOffset) {
15600 SDValue Ret;
15601
15602 auto TypeSize = Src.getValueSizeInBits().getFixedValue();
15603 // ByteProvider must be at least 8 bits
15604 assert(Src.getValueSizeInBits().isKnownMultipleOf(8));
15605
15606 if (TypeSize <= 32)
15607 return DAG.getBitcastedAnyExtOrTrunc(Src, SL, MVT::i32);
15608
15609 if (Src.getValueType().isVector()) {
15610 auto ScalarTySize = Src.getScalarValueSizeInBits();
15611 auto ScalarTy = Src.getValueType().getScalarType();
15612 if (ScalarTySize == 32) {
15613 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Src,
15614 DAG.getConstant(DWordOffset, SL, MVT::i32));
15615 }
15616 if (ScalarTySize > 32) {
15617 Ret = DAG.getNode(
15618 ISD::EXTRACT_VECTOR_ELT, SL, ScalarTy, Src,
15619 DAG.getConstant(DWordOffset / (ScalarTySize / 32), SL, MVT::i32));
15620 auto ShiftVal = 32 * (DWordOffset % (ScalarTySize / 32));
15621 if (ShiftVal)
15622 Ret = DAG.getNode(ISD::SRL, SL, Ret.getValueType(), Ret,
15623 DAG.getConstant(ShiftVal, SL, MVT::i32));
15624 return DAG.getBitcastedAnyExtOrTrunc(Ret, SL, MVT::i32);
15625 }
15626
15627 assert(ScalarTySize < 32);
15628 if (TypeSize % 32 == 0) {
15629 assert(DWordOffset < TypeSize / 32);
15630 SDValue Cast = DAG.getBitcast(
15631 EVT::getVectorVT(*DAG.getContext(), MVT::i32, TypeSize / 32), Src);
15632 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Cast,
15633 DAG.getConstant(DWordOffset, SL, MVT::i32));
15634 }
15635
15636 auto NumElements = TypeSize / ScalarTySize;
15637 auto Trunc32Elements = (ScalarTySize * NumElements) / 32;
15638 auto NormalizedTrunc = Trunc32Elements * 32 / ScalarTySize;
15639 auto NumElementsIn32 = 32 / ScalarTySize;
15640 auto NumAvailElements = DWordOffset < Trunc32Elements
15641 ? NumElementsIn32
15642 : NumElements - NormalizedTrunc;
15643
15645 DAG.ExtractVectorElements(Src, VecSrcs, DWordOffset * NumElementsIn32,
15646 NumAvailElements);
15647
15648 Ret = DAG.getBuildVector(
15649 MVT::getVectorVT(MVT::getIntegerVT(ScalarTySize), NumAvailElements), SL,
15650 VecSrcs);
15651 return Ret = DAG.getBitcastedAnyExtOrTrunc(Ret, SL, MVT::i32);
15652 }
15653
15654 /// Scalar Type
15655 auto ShiftVal = 32 * DWordOffset;
15656 Ret = DAG.getNode(ISD::SRL, SL, Src.getValueType(), Src,
15657 DAG.getConstant(ShiftVal, SL, MVT::i32));
15658 return DAG.getBitcastedAnyExtOrTrunc(Ret, SL, MVT::i32);
15659}
15660
15662 SelectionDAG &DAG = DCI.DAG;
15663 [[maybe_unused]] EVT VT = N->getValueType(0);
15665
15666 // VT is known to be MVT::i32, so we need to provide 4 bytes.
15667 assert(VT == MVT::i32);
15668 for (int i = 0; i < 4; i++) {
15669 // Find the ByteProvider that provides the ith byte of the result of OR
15670 std::optional<ByteProvider<SDValue>> P =
15671 calculateByteProvider(SDValue(N, 0), i, 0, /*StartingIndex = */ i);
15672 // TODO support constantZero
15673 if (!P || P->isConstantZero())
15674 return SDValue();
15675
15676 PermNodes.push_back(*P);
15677 }
15678 if (PermNodes.size() != 4)
15679 return SDValue();
15680
15681 std::pair<unsigned, unsigned> FirstSrc(0, PermNodes[0].SrcOffset / 4);
15682 std::optional<std::pair<unsigned, unsigned>> SecondSrc;
15683 uint64_t PermMask = 0x00000000;
15684 for (size_t i = 0; i < PermNodes.size(); i++) {
15685 auto PermOp = PermNodes[i];
15686 // Since the mask is applied to Src1:Src2, Src1 bytes must be offset
15687 // by sizeof(Src2) = 4
15688 int SrcByteAdjust = 4;
15689
15690 // If the Src uses a byte from a different DWORD, then it corresponds
15691 // with a difference source
15692 if (!PermOp.hasSameSrc(PermNodes[FirstSrc.first]) ||
15693 ((PermOp.SrcOffset / 4) != FirstSrc.second)) {
15694 if (SecondSrc)
15695 if (!PermOp.hasSameSrc(PermNodes[SecondSrc->first]) ||
15696 ((PermOp.SrcOffset / 4) != SecondSrc->second))
15697 return SDValue();
15698
15699 // Set the index of the second distinct Src node
15700 SecondSrc = {i, PermNodes[i].SrcOffset / 4};
15701 assert(!(PermNodes[SecondSrc->first].Src->getValueSizeInBits() % 8));
15702 SrcByteAdjust = 0;
15703 }
15704 assert((PermOp.SrcOffset % 4) + SrcByteAdjust < 8);
15706 PermMask |= ((PermOp.SrcOffset % 4) + SrcByteAdjust) << (i * 8);
15707 }
15708 SDLoc DL(N);
15709 SDValue Op = *PermNodes[FirstSrc.first].Src;
15710 Op = getDWordFromOffset(DAG, DL, Op, FirstSrc.second);
15711 assert(Op.getValueSizeInBits() == 32);
15712
15713 // Check that we are not just extracting the bytes in order from an op
15714 if (!SecondSrc) {
15715 int Low16 = PermMask & 0xffff;
15716 int Hi16 = (PermMask & 0xffff0000) >> 16;
15717
15718 bool WellFormedLow = (Low16 == 0x0504) || (Low16 == 0x0100);
15719 bool WellFormedHi = (Hi16 == 0x0706) || (Hi16 == 0x0302);
15720
15721 // The perm op would really just produce Op. So combine into Op
15722 if (WellFormedLow && WellFormedHi)
15723 return DAG.getBitcast(MVT::getIntegerVT(32), Op);
15724 }
15725
15726 SDValue OtherOp = SecondSrc ? *PermNodes[SecondSrc->first].Src : Op;
15727
15728 if (SecondSrc) {
15729 OtherOp = getDWordFromOffset(DAG, DL, OtherOp, SecondSrc->second);
15730 assert(OtherOp.getValueSizeInBits() == 32);
15731 }
15732
15733 // Check that we haven't just recreated the same FSHR node.
15734 if (N->getOpcode() == ISD::FSHR &&
15735 (N->getOperand(0) == Op || N->getOperand(0) == OtherOp) &&
15736 (N->getOperand(1) == Op || N->getOperand(1) == OtherOp))
15737 return SDValue();
15738
15739 if (hasNon16BitAccesses(PermMask, Op, OtherOp)) {
15740
15741 assert(Op.getValueType().isByteSized() &&
15742 OtherOp.getValueType().isByteSized());
15743
15744 // If the ultimate src is less than 32 bits, then we will only be
15745 // using bytes 0: Op.getValueSizeInBytes() - 1 in the or.
15746 // CalculateByteProvider would not have returned Op as source if we
15747 // used a byte that is outside its ValueType. Thus, we are free to
15748 // ANY_EXTEND as the extended bits are dont-cares.
15749 Op = DAG.getBitcastedAnyExtOrTrunc(Op, DL, MVT::i32);
15750 OtherOp = DAG.getBitcastedAnyExtOrTrunc(OtherOp, DL, MVT::i32);
15751
15752 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, Op, OtherOp,
15753 DAG.getConstant(PermMask, DL, MVT::i32));
15754 }
15755 return SDValue();
15756}
15757
15758SDValue SITargetLowering::performOrCombine(SDNode *N,
15759 DAGCombinerInfo &DCI) const {
15760 SelectionDAG &DAG = DCI.DAG;
15761 SDValue LHS = N->getOperand(0);
15762 SDValue RHS = N->getOperand(1);
15763
15764 EVT VT = N->getValueType(0);
15765 if (VT == MVT::i1) {
15766 // or (fp_class x, c1), (fp_class x, c2) -> fp_class x, (c1 | c2)
15767 if (LHS.getOpcode() == AMDGPUISD::FP_CLASS &&
15768 RHS.getOpcode() == AMDGPUISD::FP_CLASS) {
15769 SDValue Src = LHS.getOperand(0);
15770 if (Src != RHS.getOperand(0))
15771 return SDValue();
15772
15773 const ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(LHS.getOperand(1));
15774 const ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(RHS.getOperand(1));
15775 if (!CLHS || !CRHS)
15776 return SDValue();
15777
15778 // Only 10 bits are used.
15779 static const uint32_t MaxMask = 0x3ff;
15780
15781 uint32_t NewMask =
15782 (CLHS->getZExtValue() | CRHS->getZExtValue()) & MaxMask;
15783 SDLoc DL(N);
15784 return DAG.getNode(AMDGPUISD::FP_CLASS, DL, MVT::i1, Src,
15785 DAG.getConstant(NewMask, DL, MVT::i32));
15786 }
15787
15788 return SDValue();
15789 }
15790
15791 if (VT == MVT::i32 && !DCI.isBeforeLegalize()) {
15792 for (unsigned I : {1u, 0u}) {
15793 SDValue Ext = N->getOperand(I);
15794 // The select may need an e64 cndmask, so only fold when it eliminates
15795 // the extension.
15796 if (!Ext.hasOneUse() || !isBoolSGPRExt(Ext))
15797 continue;
15798 // A divergent select still needs to materialize a uniform condition.
15799 // Sharing that condition can also require converting it back to SCC.
15800 SDValue Cond = Ext.getOperand(0);
15801 if (N->isDivergent() && !Cond->isDivergent() && !Cond.hasOneUse())
15802 continue;
15803
15804 // or x, (sext/anyext cc from i1) => select cc, -1, x
15805 // Any-extended bits can be chosen to match sign extension.
15806 SDLoc DL(N);
15807 return DAG.getSelect(DL, MVT::i32, Cond,
15808 DAG.getAllOnesConstant(DL, MVT::i32),
15809 N->getOperand(1 - I));
15810 }
15811 }
15812
15813 // or (perm x, y, c1), c2 -> perm x, y, permute_mask(c1, c2)
15815 LHS.getOpcode() == AMDGPUISD::PERM &&
15816 isa<ConstantSDNode>(LHS.getOperand(2))) {
15817 uint32_t Sel = getConstantPermuteMask(N->getConstantOperandVal(1));
15818 if (!Sel)
15819 return SDValue();
15820
15821 Sel |= LHS.getConstantOperandVal(2);
15822 SDLoc DL(N);
15823 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, LHS.getOperand(0),
15824 LHS.getOperand(1), DAG.getConstant(Sel, DL, MVT::i32));
15825 }
15826
15827 // or (op x, c1), (op y, c2) -> perm x, y, permute_mask(c1, c2)
15828 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
15829 if (VT == MVT::i32 && LHS.hasOneUse() && RHS.hasOneUse() &&
15830 TII->pseudoToMCOpcode(AMDGPU::V_PERM_B32_e64) != -1) {
15831
15832 // If all the uses of an or need to extract the individual elements, do not
15833 // attempt to lower into v_perm
15834 auto usesCombinedOperand = [](SDNode *OrUse) {
15835 // If we have any non-vectorized use, then it is a candidate for v_perm
15836 if (OrUse->getOpcode() != ISD::BITCAST ||
15837 !OrUse->getValueType(0).isVector())
15838 return true;
15839
15840 // If we have any non-vectorized use, then it is a candidate for v_perm
15841 for (auto *VUser : OrUse->users()) {
15842 if (!VUser->getValueType(0).isVector())
15843 return true;
15844
15845 // If the use of a vector is a store, then combining via a v_perm
15846 // is beneficial.
15847 // TODO -- whitelist more uses
15848 for (auto VectorwiseOp : {ISD::STORE, ISD::CopyToReg, ISD::CopyFromReg})
15849 if (VUser->getOpcode() == VectorwiseOp)
15850 return true;
15851 }
15852 return false;
15853 };
15854
15855 if (!any_of(N->users(), usesCombinedOperand))
15856 return SDValue();
15857
15858 uint32_t LHSMask = getPermuteMask(LHS);
15859 uint32_t RHSMask = getPermuteMask(RHS);
15860
15861 if (LHSMask != ~0u && RHSMask != ~0u) {
15862 // Canonicalize the expression in an attempt to have fewer unique masks
15863 // and therefore fewer registers used to hold the masks.
15864 if (LHSMask > RHSMask) {
15865 std::swap(LHSMask, RHSMask);
15866 std::swap(LHS, RHS);
15867 }
15868
15869 // Select 0xc for each lane used from source operand. Zero has 0xc mask
15870 // set, 0xff have 0xff in the mask, actual lanes are in the 0-3 range.
15871 uint32_t LHSUsedLanes = ~(LHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
15872 uint32_t RHSUsedLanes = ~(RHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
15873
15874 // Check of we need to combine values from two sources within a byte.
15875 if (!(LHSUsedLanes & RHSUsedLanes) &&
15876 // If we select high and lower word keep it for SDWA.
15877 // TODO: teach SDWA to work with v_perm_b32 and remove the check.
15878 !(LHSUsedLanes == 0x0c0c0000 && RHSUsedLanes == 0x00000c0c)) {
15879 // Kill zero bytes selected by other mask. Zero value is 0xc.
15880 LHSMask &= ~RHSUsedLanes;
15881 RHSMask &= ~LHSUsedLanes;
15882 // Add 4 to each active LHS lane
15883 LHSMask |= LHSUsedLanes & 0x04040404;
15884 // Combine masks
15885 uint32_t Sel = LHSMask | RHSMask;
15886 SDLoc DL(N);
15887
15888 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, LHS.getOperand(0),
15889 RHS.getOperand(0),
15890 DAG.getConstant(Sel, DL, MVT::i32));
15891 }
15892 }
15893 if (LHSMask == ~0u || RHSMask == ~0u) {
15894 if (SDValue Perm = matchPERM(N, DCI))
15895 return Perm;
15896 }
15897 }
15898
15899 // Detect identity v2i32 OR and replace with identity source node.
15900 // Specifically an Or that has operands constructed from the same source node
15901 // via extract_vector_elt and build_vector. I.E.
15902 // v2i32 or(
15903 // v2i32 build_vector(
15904 // i32 extract_elt(%IdentitySrc, 0),
15905 // i32 0
15906 // ),
15907 // v2i32 build_vector(
15908 // i32 0,
15909 // i32 extract_elt(%IdentitySrc, 1)
15910 // ) )
15911 // =>
15912 // v2i32 %IdentitySrc
15913
15914 if (VT == MVT::v2i32 && LHS->getOpcode() == ISD::BUILD_VECTOR &&
15915 RHS->getOpcode() == ISD::BUILD_VECTOR) {
15916
15917 ConstantSDNode *LC = dyn_cast<ConstantSDNode>(LHS->getOperand(1));
15918 ConstantSDNode *RC = dyn_cast<ConstantSDNode>(RHS->getOperand(0));
15919
15920 // Test for and normalise build vectors.
15921 if (LC && RC && LC->getZExtValue() == 0 && RC->getZExtValue() == 0) {
15922
15923 // Get the extract_vector_element operands.
15924 SDValue LEVE = LHS->getOperand(0);
15925 SDValue REVE = RHS->getOperand(1);
15926
15927 if (LEVE->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15929 // Check that different elements from the same vector are
15930 // extracted.
15931 if (LEVE->getOperand(0) == REVE->getOperand(0) &&
15932 LEVE->getOperand(1) != REVE->getOperand(1)) {
15933 SDValue IdentitySrc = LEVE.getOperand(0);
15934 return IdentitySrc;
15935 }
15936 }
15937 }
15938 }
15939
15940 if (VT != MVT::i64 || DCI.isBeforeLegalizeOps())
15941 return SDValue();
15942
15943 // TODO: This could be a generic combine with a predicate for extracting the
15944 // high half of an integer being free.
15945
15946 // (or i64:x, (zero_extend i32:y)) ->
15947 // i64 (bitcast (v2i32 build_vector (or i32:y, lo_32(x)), hi_32(x)))
15948 if (LHS.getOpcode() == ISD::ZERO_EXTEND &&
15949 RHS.getOpcode() != ISD::ZERO_EXTEND)
15950 std::swap(LHS, RHS);
15951
15952 if (RHS.getOpcode() == ISD::ZERO_EXTEND) {
15953 SDValue ExtSrc = RHS.getOperand(0);
15954 EVT SrcVT = ExtSrc.getValueType();
15955 if (SrcVT == MVT::i32) {
15956 SDLoc SL(N);
15957 auto [LowLHS, HiBits] = split64BitValue(LHS, DAG);
15958 SDValue LowOr = DAG.getNode(ISD::OR, SL, MVT::i32, LowLHS, ExtSrc);
15959
15960 DCI.AddToWorklist(LowOr.getNode());
15961 DCI.AddToWorklist(HiBits.getNode());
15962
15963 SDValue Vec =
15964 DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, LowOr, HiBits);
15965 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec);
15966 }
15967 }
15968
15969 const ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(N->getOperand(1));
15970 if (CRHS) {
15971 if (SDValue Split = splitBinaryBitConstantOp(DCI, SDLoc(N), ISD::OR,
15972 N->getOperand(0), CRHS))
15973 return Split;
15974 }
15975
15976 return SDValue();
15977}
15978
15979SDValue SITargetLowering::performXorCombine(SDNode *N,
15980 DAGCombinerInfo &DCI) const {
15981 if (SDValue RV = reassociateScalarOps(N, DCI.DAG))
15982 return RV;
15983
15984 SDValue LHS = N->getOperand(0);
15985 SDValue RHS = N->getOperand(1);
15986
15987 const ConstantSDNode *CRHS = isConstOrConstSplat(RHS);
15988 SelectionDAG &DAG = DCI.DAG;
15989
15990 EVT VT = N->getValueType(0);
15991 if (CRHS && VT == MVT::i64) {
15992 if (SDValue Split =
15993 splitBinaryBitConstantOp(DCI, SDLoc(N), ISD::XOR, LHS, CRHS))
15994 return Split;
15995 }
15996
15997 // v2i32 (xor (vselect cc, x, y), K) ->
15998 // (v2i32 svelect cc, (xor x, K), (xor y, K)) This enables the xor to be
15999 // replaced with source modifiers when the select is lowered to CNDMASK.
16000 unsigned Opc = LHS.getOpcode();
16001 if (((Opc == ISD::VSELECT && VT == MVT::v2i32) ||
16002 (Opc == ISD::SELECT && VT == MVT::i64)) &&
16003 CRHS && CRHS->getAPIntValue().isSignMask()) {
16004 SDValue CC = LHS->getOperand(0);
16005 SDValue TRUE = LHS->getOperand(1);
16006 SDValue FALSE = LHS->getOperand(2);
16007 SDValue XTrue = DAG.getNode(ISD::XOR, SDLoc(N), VT, TRUE, RHS);
16008 SDValue XFalse = DAG.getNode(ISD::XOR, SDLoc(N), VT, FALSE, RHS);
16009 SDValue XSelect =
16010 DAG.getNode(ISD::VSELECT, SDLoc(N), VT, CC, XTrue, XFalse);
16011 return XSelect;
16012 }
16013
16014 // Make sure to apply the 64-bit constant splitting fold before trying to fold
16015 // fneg-like xors into 64-bit select.
16016 if (LHS.getOpcode() == ISD::SELECT && VT == MVT::i32) {
16017 // This looks like an fneg, try to fold as a source modifier.
16018 if (CRHS && CRHS->getAPIntValue().isSignMask() &&
16020 // xor (select c, a, b), 0x80000000 ->
16021 // bitcast (select c, (fneg (bitcast a)), (fneg (bitcast b)))
16022 SDLoc DL(N);
16023 SDValue CastLHS =
16024 DAG.getNode(ISD::BITCAST, DL, MVT::f32, LHS->getOperand(1));
16025 SDValue CastRHS =
16026 DAG.getNode(ISD::BITCAST, DL, MVT::f32, LHS->getOperand(2));
16027 SDValue FNegLHS = DAG.getNode(ISD::FNEG, DL, MVT::f32, CastLHS);
16028 SDValue FNegRHS = DAG.getNode(ISD::FNEG, DL, MVT::f32, CastRHS);
16029 SDValue NewSelect = DAG.getNode(ISD::SELECT, DL, MVT::f32,
16030 LHS->getOperand(0), FNegLHS, FNegRHS);
16031 return DAG.getNode(ISD::BITCAST, DL, VT, NewSelect);
16032 }
16033 }
16034
16035 return SDValue();
16036}
16037
16038SDValue
16039SITargetLowering::performZeroOrAnyExtendCombine(SDNode *N,
16040 DAGCombinerInfo &DCI) const {
16041 if (!Subtarget->has16BitInsts() ||
16042 DCI.getDAGCombineLevel() < AfterLegalizeTypes)
16043 return SDValue();
16044
16045 EVT VT = N->getValueType(0);
16046 if (VT != MVT::i32)
16047 return SDValue();
16048
16049 SDValue Src = N->getOperand(0);
16050 if (Src.getValueType() != MVT::i16)
16051 return SDValue();
16052
16053 if (!Src->hasOneUse())
16054 return SDValue();
16055
16056 if (SDValue Ext = performExtendTruncShiftCombine(N, DCI))
16057 return Ext;
16058
16059 // TODO: We bail out below if SrcOffset is not in the first dword (>= 4). It's
16060 // possible we're missing out on some combine opportunities, but we'd need to
16061 // weigh the cost of extracting the byte from the upper dwords.
16062
16063 std::optional<ByteProvider<SDValue>> BP0 =
16064 calculateByteProvider(SDValue(N, 0), 0, 0, 0);
16065 if (!BP0 || BP0->SrcOffset >= 4 || !BP0->Src)
16066 return SDValue();
16067 SDValue V0 = *BP0->Src;
16068
16069 std::optional<ByteProvider<SDValue>> BP1 =
16070 calculateByteProvider(SDValue(N, 0), 1, 0, 1);
16071 if (!BP1 || BP1->SrcOffset >= 4 || !BP1->Src)
16072 return SDValue();
16073
16074 SDValue V1 = *BP1->Src;
16075
16076 if (V0 == V1)
16077 return SDValue();
16078
16079 SelectionDAG &DAG = DCI.DAG;
16080 SDLoc DL(N);
16081 uint32_t PermMask = 0x0c0c0c0c;
16082 if (V0) {
16083 V0 = getDWordFromOffset(DAG, DL, V0, /*DWordOffset=*/0);
16084 PermMask = (PermMask & ~0xFF) | (BP0->SrcOffset + 4);
16085 }
16086
16087 if (V1) {
16088 V1 = getDWordFromOffset(DAG, DL, V1, /*DWordOffset=*/0);
16089 PermMask = (PermMask & ~(0xFF << 8)) | (BP1->SrcOffset << 8);
16090 }
16091
16092 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, V0, V1,
16093 DAG.getConstant(PermMask, DL, MVT::i32));
16094}
16095
16096// zext (srl (trunc x), c) => and (srl x, c), lowbits
16097// aext (srl (trunc x), c) => and (srl x, c), lowbits
16098// sext (sra (trunc x), 8) => sext_inreg (srl x, 8), i8
16099SDValue
16100SITargetLowering::performExtendTruncShiftCombine(SDNode *N,
16101 DAGCombinerInfo &DCI) const {
16102 if (DCI.getDAGCombineLevel() < AfterLegalizeTypes)
16103 return SDValue();
16104
16105 const unsigned Opc = N->getOpcode();
16106 const bool IsSigned = Opc == ISD::SIGN_EXTEND;
16107 SDValue Src = N->getOperand(0);
16108 if (N->getValueType(0) != MVT::i32 || Src.getValueType() != MVT::i16 ||
16109 Src.getOpcode() != (IsSigned ? ISD::SRA : ISD::SRL) || !Src.hasOneUse() ||
16110 Src.getOperand(0).getOpcode() != ISD::TRUNCATE)
16111 return SDValue();
16112
16113 // A uniform i16 srl is promoted to i32 later and yields the same nodes.
16114 // Skip it so that the node order, and with it the schedule, stays the same.
16115 if (!IsSigned && !Src->isDivergent())
16116 return SDValue();
16117
16118 SDValue X = Src.getOperand(0).getOperand(0);
16119 auto *Shift = dyn_cast<ConstantSDNode>(Src.getOperand(1));
16120 if (X.getValueType() != MVT::i32 || !Shift || Shift->getZExtValue() == 0 ||
16121 Shift->getZExtValue() >= 16 || (IsSigned && Shift->getZExtValue() != 8))
16122 return SDValue();
16123
16124 SelectionDAG &DAG = DCI.DAG;
16125 SDLoc SL(N);
16126 const unsigned Width = 16 - Shift->getZExtValue();
16127 SDValue Srl = DAG.getNode(
16128 ISD::SRL, SL, MVT::i32, X,
16129 DAG.getShiftAmountConstant(Shift->getZExtValue(), MVT::i32, SL));
16130 if (IsSigned)
16131 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SL, MVT::i32, Srl,
16132 DAG.getValueType(MVT::i8));
16133 return DAG.getNode(
16134 ISD::AND, SL, MVT::i32, Srl,
16135 DAG.getConstant(maskTrailingOnes<uint32_t>(Width), SL, MVT::i32));
16136}
16137
16138SDValue
16139SITargetLowering::performSignExtendInRegCombine(SDNode *N,
16140 DAGCombinerInfo &DCI) const {
16141 SDValue Src = N->getOperand(0);
16142 auto *VTSign = cast<VTSDNode>(N->getOperand(1));
16143
16144 // Combine s_buffer_load_u8 or s_buffer_load_u16 with sext and replace them
16145 // with s_buffer_load_i8 and s_buffer_load_i16 respectively.
16146 if (((Src.getOpcode() == AMDGPUISD::SBUFFER_LOAD_UBYTE &&
16147 VTSign->getVT() == MVT::i8) ||
16148 (Src.getOpcode() == AMDGPUISD::SBUFFER_LOAD_USHORT &&
16149 VTSign->getVT() == MVT::i16))) {
16150 assert(Subtarget->hasScalarSubwordLoads() &&
16151 "s_buffer_load_{u8, i8} are supported "
16152 "in GFX12 (or newer) architectures.");
16153 unsigned Opc = (Src.getOpcode() == AMDGPUISD::SBUFFER_LOAD_UBYTE)
16154 ? AMDGPUISD::SBUFFER_LOAD_BYTE
16155 : AMDGPUISD::SBUFFER_LOAD_SHORT;
16156 SDLoc DL(N);
16157 SDVTList ResList =
16158 DCI.DAG.getVTList(MVT::i32, Src.getOperand(0).getValueType());
16159 SDValue Ops[] = {
16160 Src.getOperand(0), // Chain
16161 Src.getOperand(1), // source register
16162 Src.getOperand(2), // offset
16163 Src.getOperand(3) // cachePolicy
16164 };
16165 auto *M = cast<MemSDNode>(Src);
16166 SDValue BufferLoad = DCI.DAG.getMemIntrinsicNode(
16167 Opc, DL, ResList, Ops, M->getMemoryVT(), M->getMemOperand());
16168 return DCI.DAG.getMergeValues({BufferLoad, BufferLoad.getValue(1)}, DL);
16169 }
16170 if (((Src.getOpcode() == AMDGPUISD::BUFFER_LOAD_UBYTE &&
16171 VTSign->getVT() == MVT::i8) ||
16172 (Src.getOpcode() == AMDGPUISD::BUFFER_LOAD_USHORT &&
16173 VTSign->getVT() == MVT::i16)) &&
16174 Src.hasOneUse()) {
16175 auto *M = cast<MemSDNode>(Src);
16176 SDValue Ops[] = {Src.getOperand(0), // Chain
16177 Src.getOperand(1), // rsrc
16178 Src.getOperand(2), // vindex
16179 Src.getOperand(3), // voffset
16180 Src.getOperand(4), // soffset
16181 Src.getOperand(5), // offset
16182 Src.getOperand(6), Src.getOperand(7)};
16183 // replace with BUFFER_LOAD_BYTE/SHORT
16184 SDVTList ResList =
16185 DCI.DAG.getVTList(MVT::i32, Src.getOperand(0).getValueType());
16186 unsigned Opc = (Src.getOpcode() == AMDGPUISD::BUFFER_LOAD_UBYTE)
16187 ? AMDGPUISD::BUFFER_LOAD_BYTE
16188 : AMDGPUISD::BUFFER_LOAD_SHORT;
16189 SDValue BufferLoadSignExt = DCI.DAG.getMemIntrinsicNode(
16190 Opc, SDLoc(N), ResList, Ops, M->getMemoryVT(), M->getMemOperand());
16191 return DCI.DAG.getMergeValues(
16192 {BufferLoadSignExt, BufferLoadSignExt.getValue(1)}, SDLoc(N));
16193 }
16194 return SDValue();
16195}
16196
16197SDValue SITargetLowering::performClassCombine(SDNode *N,
16198 DAGCombinerInfo &DCI) const {
16199 SelectionDAG &DAG = DCI.DAG;
16200 SDValue Mask = N->getOperand(1);
16201
16202 // fp_class x, 0 -> false
16203 if (isNullConstant(Mask))
16204 return DAG.getConstant(0, SDLoc(N), MVT::i1);
16205
16206 if (N->getOperand(0).isUndef())
16207 return DAG.getUNDEF(MVT::i1);
16208
16209 return SDValue();
16210}
16211
16212SDValue SITargetLowering::performRcpCombine(SDNode *N,
16213 DAGCombinerInfo &DCI) const {
16214 EVT VT = N->getValueType(0);
16215 SDValue N0 = N->getOperand(0);
16216
16217 if (N0.isUndef()) {
16218 return DCI.DAG.getConstantFP(APFloat::getQNaN(VT.getFltSemantics()),
16219 SDLoc(N), VT);
16220 }
16221
16222 // TODO: Could handle f32 + amdgcn.sqrt but probably never reaches here.
16223 if ((VT == MVT::f16 && N0.getOpcode() == ISD::FSQRT) &&
16224 N->getFlags().hasAllowContract() && N0->getFlags().hasAllowContract()) {
16225 return DCI.DAG.getNode(AMDGPUISD::RSQ, SDLoc(N), VT, N0.getOperand(0),
16226 N->getFlags());
16227 }
16228
16230}
16231
16233 SDNodeFlags UserFlags,
16234 unsigned MaxDepth) const {
16235 EVT VT = Op.getValueType();
16236 assert(VT.isFloatingPoint() &&
16237 "expected a floating-point value to query canonicality of");
16238 return isCanonicalized(DAG, Op, VT.getScalarType(), UserFlags, MaxDepth);
16239}
16240
16242 EVT QueryVT, SDNodeFlags UserFlags,
16243 unsigned MaxDepth) const {
16244 assert(QueryVT.isFloatingPoint() && !QueryVT.isVector() &&
16245 "QueryVT must be a floating-point scalar type");
16246 EVT VT = Op.getValueType();
16247 if (VT.isFloatingPoint() && VT.getScalarType() != QueryVT)
16248 return false;
16249
16250 unsigned Opcode = Op.getOpcode();
16251 if (Opcode == ISD::FCANONICALIZE)
16252 return true;
16253
16254 if (auto *CFP = dyn_cast<ConstantFPSDNode>(Op)) {
16255 const auto &F = CFP->getValueAPF();
16256 if (F.isNaN() && F.isSignaling())
16257 return false;
16258 if (!F.isDenormal())
16259 return true;
16260
16261 DenormalMode Mode =
16262 DAG.getMachineFunction().getDenormalMode(F.getSemantics());
16263 return Mode == DenormalMode::getIEEE();
16264 }
16265
16266 // If source is a result of another standard FP operation it is already in
16267 // canonical form.
16268 if (MaxDepth == 0)
16269 return false;
16270
16271 switch (Opcode) {
16272 // These will flush denorms if required.
16273 case ISD::FADD:
16274 case ISD::FSUB:
16275 case ISD::FMUL:
16276 case ISD::FCEIL:
16277 case ISD::FFLOOR:
16278 case ISD::FMA:
16279 case ISD::FMAD:
16280 case ISD::FSQRT:
16281 case ISD::FDIV:
16282 case ISD::FREM:
16283 case ISD::FP_ROUND:
16284 case ISD::FP_EXTEND:
16285 case ISD::FP16_TO_FP:
16286 case ISD::FP_TO_FP16:
16287 case ISD::BF16_TO_FP:
16288 case ISD::FP_TO_BF16:
16289 case ISD::FLDEXP:
16290 case AMDGPUISD::FMUL_LEGACY:
16291 case AMDGPUISD::FMAD_FTZ:
16292 case AMDGPUISD::RCP:
16293 case AMDGPUISD::RSQ:
16294 case AMDGPUISD::RSQ_CLAMP:
16295 case AMDGPUISD::RCP_LEGACY:
16296 case AMDGPUISD::RCP_IFLAG:
16297 case AMDGPUISD::LOG:
16298 case AMDGPUISD::EXP:
16299 case AMDGPUISD::DIV_SCALE:
16300 case AMDGPUISD::DIV_FMAS:
16301 case AMDGPUISD::DIV_FIXUP:
16302 case AMDGPUISD::FRACT:
16303 case AMDGPUISD::CVT_PKRTZ_F16_F32:
16304 case AMDGPUISD::CVT_F32_UBYTE0:
16305 case AMDGPUISD::CVT_F32_UBYTE1:
16306 case AMDGPUISD::CVT_F32_UBYTE2:
16307 case AMDGPUISD::CVT_F32_UBYTE3:
16308 case AMDGPUISD::FP_TO_FP16:
16309 case AMDGPUISD::SIN_HW:
16310 case AMDGPUISD::COS_HW:
16311 return true;
16312
16313 // It can/will be lowered or combined as a bit operation.
16314 // Need to check their input recursively to handle.
16315 case ISD::FNEG:
16316 case ISD::FABS:
16317 case ISD::FCOPYSIGN:
16318 return isCanonicalized(DAG, Op.getOperand(0), QueryVT, UserFlags,
16319 MaxDepth - 1);
16320
16321 case ISD::AND:
16322 if (Op.getValueType() == MVT::i32) {
16323 // Be careful as we only know it is a bitcast floating point type. It
16324 // could be f32, v2f16, we have no way of knowing. Luckily the constant
16325 // value that we optimize for, which comes up in fp32 to bf16 conversions,
16326 // is valid to optimize for all types.
16327 if (auto *RHS = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
16328 if (RHS->getZExtValue() == 0xffff0000) {
16329 return isCanonicalized(DAG, Op.getOperand(0), QueryVT, UserFlags,
16330 MaxDepth - 1);
16331 }
16332 }
16333 }
16334 break;
16335
16336 case ISD::FSIN:
16337 case ISD::FCOS:
16338 case ISD::FSINCOS:
16339 return Op.getValueType().getScalarType() != MVT::f16;
16340
16341 case ISD::FMINNUM:
16342 case ISD::FMAXNUM:
16343 case ISD::FMINNUM_IEEE:
16344 case ISD::FMAXNUM_IEEE:
16345 case ISD::FMINIMUM:
16346 case ISD::FMAXIMUM:
16347 case ISD::FMINIMUMNUM:
16348 case ISD::FMAXIMUMNUM:
16349 case AMDGPUISD::CLAMP:
16350 case AMDGPUISD::FMED3:
16351 case AMDGPUISD::FMAX3:
16352 case AMDGPUISD::FMIN3:
16353 case AMDGPUISD::FMAXIMUM3:
16354 case AMDGPUISD::FMINIMUM3: {
16355 // FIXME: Shouldn't treat the generic operations different based these.
16356 // However, we aren't really required to flush the result from
16357 // minnum/maxnum..
16358
16359 // snans will be quieted, so we only need to worry about denormals.
16360 if (Subtarget->supportsMinMaxDenormModes() ||
16361 // FIXME: denormalsEnabledForType is broken for dynamic
16362 denormalsEnabledForType(DAG, Op.getValueType()))
16363 return true;
16364
16365 // Flushing may be required.
16366 // In pre-GFX9 targets V_MIN_F32 and others do not flush denorms. For such
16367 // targets need to check their input recursively.
16368
16369 // FIXME: Does this apply with clamp? It's implemented with max.
16370 for (unsigned I = 0, E = Op.getNumOperands(); I != E; ++I) {
16371 if (!isCanonicalized(DAG, Op.getOperand(I), QueryVT, UserFlags,
16372 MaxDepth - 1))
16373 return false;
16374 }
16375
16376 return true;
16377 }
16378 case ISD::SELECT: {
16379 return isCanonicalized(DAG, Op.getOperand(1), QueryVT, UserFlags,
16380 MaxDepth - 1) &&
16381 isCanonicalized(DAG, Op.getOperand(2), QueryVT, UserFlags,
16382 MaxDepth - 1);
16383 }
16384 case ISD::BUILD_VECTOR: {
16385 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
16386 SDValue SrcOp = Op.getOperand(i);
16387 if (!isCanonicalized(DAG, SrcOp, QueryVT, UserFlags, MaxDepth - 1))
16388 return false;
16389 }
16390
16391 return true;
16392 }
16395 return isCanonicalized(DAG, Op.getOperand(0), QueryVT, UserFlags,
16396 MaxDepth - 1);
16397 }
16399 return isCanonicalized(DAG, Op.getOperand(0), QueryVT, UserFlags,
16400 MaxDepth - 1) &&
16401 isCanonicalized(DAG, Op.getOperand(1), QueryVT, UserFlags,
16402 MaxDepth - 1);
16403 }
16404 case ISD::POISON:
16405 return true;
16406 case ISD::UNDEF:
16407 // Could be anything.
16408 return false;
16409
16410 case ISD::BITCAST: {
16411 // Carry QueryVT through the bitcast unchanged. The top-of-function guard
16412 // rejects a source whose FP format differs from the consumed type, so a
16413 // value canonical in one FP format is not assumed canonical in another.
16414 SDValue Src = peekThroughBitcasts(Op.getOperand(0));
16415 return isCanonicalized(DAG, Src, QueryVT, UserFlags, MaxDepth - 1);
16416 }
16417 case ISD::TRUNCATE: {
16418 // Hack round the mess we make when legalizing extract_vector_elt
16419 if (Op.getValueType() == MVT::i16) {
16420 SDValue TruncSrc = Op.getOperand(0);
16421 if (TruncSrc.getValueType() == MVT::i32 &&
16422 TruncSrc.getOpcode() == ISD::BITCAST &&
16423 TruncSrc.getOperand(0).getValueType() == MVT::v2f16) {
16424 return isCanonicalized(DAG, TruncSrc.getOperand(0), QueryVT, UserFlags,
16425 MaxDepth - 1);
16426 }
16427 }
16428 return false;
16429 }
16431 unsigned IntrinsicID = Op.getConstantOperandVal(0);
16432 // TODO: Handle more intrinsics
16433 switch (IntrinsicID) {
16434 case Intrinsic::amdgcn_update_dpp: {
16435 enum UpdateDPPOperand {
16436 Src = 2,
16437 RowMask = 4,
16438 BankMask = 5,
16439 BoundCtrl = 6,
16440 };
16441
16442 if (QueryVT == MVT::f32 && (VT == MVT::i32 || VT == MVT::f32) &&
16443 Op.getConstantOperandVal(RowMask) == 0xf && // Enable all rows.
16444 Op.getConstantOperandVal(BankMask) == 0xf && // Enable all banks.
16445 Op.getConstantOperandVal(BoundCtrl) != 0) // Zero-fill OOB lanes.
16446 return isCanonicalized(DAG, Op.getOperand(2), QueryVT, UserFlags,
16447 MaxDepth - 1);
16448 return false;
16449 }
16450 case Intrinsic::amdgcn_cvt_pkrtz:
16451 case Intrinsic::amdgcn_cubeid:
16452 case Intrinsic::amdgcn_frexp_mant:
16453 case Intrinsic::amdgcn_fdot2:
16454 case Intrinsic::amdgcn_rcp:
16455 case Intrinsic::amdgcn_rsq:
16456 case Intrinsic::amdgcn_rsq_clamp:
16457 case Intrinsic::amdgcn_rcp_legacy:
16458 case Intrinsic::amdgcn_rsq_legacy:
16459 case Intrinsic::amdgcn_trig_preop:
16460 case Intrinsic::amdgcn_tanh:
16461 case Intrinsic::amdgcn_log:
16462 case Intrinsic::amdgcn_exp2:
16463 case Intrinsic::amdgcn_sqrt:
16464 return true;
16465 default:
16466 break;
16467 }
16468
16469 break;
16470 }
16471 default:
16472 break;
16473 }
16474
16475 // FIXME: denormalsEnabledForType is broken for dynamic
16476 return denormalsEnabledForType(DAG, Op.getValueType()) &&
16477 (UserFlags.hasNoNaNs() || DAG.isKnownNeverSNaN(Op));
16478}
16479
16481 unsigned MaxDepth) const {
16482 const MachineRegisterInfo &MRI = MF.getRegInfo();
16483 MachineInstr *MI = MRI.getVRegDef(Reg);
16484 unsigned Opcode = MI->getOpcode();
16485
16486 if (Opcode == AMDGPU::G_FCANONICALIZE)
16487 return true;
16488
16489 std::optional<FPValueAndVReg> FCR;
16490 // Constant splat (can be padded with undef) or scalar constant.
16492 if (FCR->Value.isSignaling())
16493 return false;
16494 if (!FCR->Value.isDenormal())
16495 return true;
16496
16497 DenormalMode Mode = MF.getDenormalMode(FCR->Value.getSemantics());
16498 return Mode == DenormalMode::getIEEE();
16499 }
16500
16501 if (MaxDepth == 0)
16502 return false;
16503
16504 switch (Opcode) {
16505 case AMDGPU::G_FADD:
16506 case AMDGPU::G_FSUB:
16507 case AMDGPU::G_FMUL:
16508 case AMDGPU::G_FCEIL:
16509 case AMDGPU::G_FFLOOR:
16510 case AMDGPU::G_FRINT:
16511 case AMDGPU::G_FNEARBYINT:
16512 case AMDGPU::G_INTRINSIC_FPTRUNC_ROUND:
16513 case AMDGPU::G_INTRINSIC_TRUNC:
16514 case AMDGPU::G_INTRINSIC_ROUNDEVEN:
16515 case AMDGPU::G_FMA:
16516 case AMDGPU::G_FMAD:
16517 case AMDGPU::G_FSQRT:
16518 case AMDGPU::G_FDIV:
16519 case AMDGPU::G_FREM:
16520 case AMDGPU::G_FPOW:
16521 case AMDGPU::G_FPEXT:
16522 case AMDGPU::G_FLOG:
16523 case AMDGPU::G_FLOG2:
16524 case AMDGPU::G_FLOG10:
16525 case AMDGPU::G_FPTRUNC:
16526 case AMDGPU::G_AMDGPU_RCP_IFLAG:
16527 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE0:
16528 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE1:
16529 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE2:
16530 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE3:
16531 return true;
16532 case AMDGPU::G_FNEG:
16533 case AMDGPU::G_FABS:
16534 case AMDGPU::G_FCOPYSIGN:
16535 return isCanonicalized(MI->getOperand(1).getReg(), MF, MaxDepth - 1);
16536 case AMDGPU::G_FMINNUM:
16537 case AMDGPU::G_FMAXNUM:
16538 case AMDGPU::G_FMINNUM_IEEE:
16539 case AMDGPU::G_FMAXNUM_IEEE:
16540 case AMDGPU::G_FMINIMUM:
16541 case AMDGPU::G_FMAXIMUM:
16542 case AMDGPU::G_FMINIMUMNUM:
16543 case AMDGPU::G_FMAXIMUMNUM: {
16544 if (Subtarget->supportsMinMaxDenormModes() ||
16545 // FIXME: denormalsEnabledForType is broken for dynamic
16546 denormalsEnabledForType(MRI.getType(Reg), MF))
16547 return true;
16548
16549 [[fallthrough]];
16550 }
16551 case AMDGPU::G_BUILD_VECTOR:
16552 for (const MachineOperand &MO : llvm::drop_begin(MI->operands()))
16553 if (!isCanonicalized(MO.getReg(), MF, MaxDepth - 1))
16554 return false;
16555 return true;
16556 case AMDGPU::G_INTRINSIC:
16557 case AMDGPU::G_INTRINSIC_CONVERGENT:
16558 switch (cast<GIntrinsic>(MI)->getIntrinsicID()) {
16559 case Intrinsic::amdgcn_fmul_legacy:
16560 case Intrinsic::amdgcn_fmad_ftz:
16561 case Intrinsic::amdgcn_sqrt:
16562 case Intrinsic::amdgcn_fmed3:
16563 case Intrinsic::amdgcn_sin:
16564 case Intrinsic::amdgcn_cos:
16565 case Intrinsic::amdgcn_log:
16566 case Intrinsic::amdgcn_exp2:
16567 case Intrinsic::amdgcn_log_clamp:
16568 case Intrinsic::amdgcn_rcp:
16569 case Intrinsic::amdgcn_rcp_legacy:
16570 case Intrinsic::amdgcn_rsq:
16571 case Intrinsic::amdgcn_rsq_clamp:
16572 case Intrinsic::amdgcn_rsq_legacy:
16573 case Intrinsic::amdgcn_div_scale:
16574 case Intrinsic::amdgcn_div_fmas:
16575 case Intrinsic::amdgcn_div_fixup:
16576 case Intrinsic::amdgcn_fract:
16577 case Intrinsic::amdgcn_cvt_pkrtz:
16578 case Intrinsic::amdgcn_cubeid:
16579 case Intrinsic::amdgcn_cubema:
16580 case Intrinsic::amdgcn_cubesc:
16581 case Intrinsic::amdgcn_cubetc:
16582 case Intrinsic::amdgcn_frexp_mant:
16583 case Intrinsic::amdgcn_fdot2:
16584 case Intrinsic::amdgcn_trig_preop:
16585 case Intrinsic::amdgcn_tanh:
16586 return true;
16587 default:
16588 break;
16589 }
16590
16591 [[fallthrough]];
16592 default:
16593 return false;
16594 }
16595
16596 llvm_unreachable("invalid operation");
16597}
16598
16599// Constant fold canonicalize.
16600SDValue SITargetLowering::getCanonicalConstantFP(SelectionDAG &DAG,
16601 const SDLoc &SL, EVT VT,
16602 const APFloat &C) const {
16603 // Flush denormals to 0 if not enabled.
16604 if (C.isDenormal()) {
16605 DenormalMode Mode =
16606 DAG.getMachineFunction().getDenormalMode(C.getSemantics());
16607 if (Mode == DenormalMode::getPreserveSign()) {
16608 return DAG.getConstantFP(
16609 APFloat::getZero(C.getSemantics(), C.isNegative()), SL, VT);
16610 }
16611
16612 if (Mode != DenormalMode::getIEEE())
16613 return SDValue();
16614 }
16615
16616 if (C.isNaN()) {
16617 if (C.isSignaling()) {
16618 // Quiet a signaling NaN.
16619 return DAG.getConstantFP(C.makeQuiet(), SL, VT);
16620 }
16621 }
16622
16623 // Already canonical.
16624 return DAG.getConstantFP(C, SL, VT);
16625}
16626
16628 return Op.isUndef() || isa<ConstantFPSDNode>(Op);
16629}
16630
16631SDValue
16632SITargetLowering::performFCanonicalizeCombine(SDNode *N,
16633 DAGCombinerInfo &DCI) const {
16634 SelectionDAG &DAG = DCI.DAG;
16635 SDValue N0 = N->getOperand(0);
16636 EVT VT = N->getValueType(0);
16637
16638 // fcanonicalize undef -> qnan
16639 if (N0.isUndef()) {
16641 return DAG.getConstantFP(QNaN, SDLoc(N), VT);
16642 }
16643
16644 if (ConstantFPSDNode *CFP = isConstOrConstSplatFP(N0))
16645 return getCanonicalConstantFP(DAG, SDLoc(N), VT, CFP->getValueAPF());
16646
16647 // fcanonicalize (build_vector x, k) -> build_vector (fcanonicalize x),
16648 // (fcanonicalize k)
16649 //
16650 // fcanonicalize (build_vector x, undef) -> build_vector (fcanonicalize x), 0
16651
16652 // TODO: This could be better with wider vectors that will be split to v2f16,
16653 // and to consider uses since there aren't that many packed operations.
16654 if (N0.getOpcode() == ISD::BUILD_VECTOR && N0.getNumOperands() == 2 &&
16655 isTypeLegal(VT)) {
16656 SDLoc SL(N);
16657 SDValue NewElts[2];
16658 SDValue Lo = N0.getOperand(0);
16659 SDValue Hi = N0.getOperand(1);
16660 EVT EltVT = Lo.getValueType();
16661
16662 // Only apply this optimization if scalar canonicalize is legal for the
16663 // element type. Otherwise, scalarizing may require widening the scalar back
16664 // to a vector, adding overhead (e.g., bf16 has no scalar instructions).
16666 return SDValue();
16667
16669 for (unsigned I = 0; I != 2; ++I) {
16670 SDValue Op = N0.getOperand(I);
16671 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Op)) {
16672 NewElts[I] =
16673 getCanonicalConstantFP(DAG, SL, EltVT, CFP->getValueAPF());
16674 } else if (Op.isUndef()) {
16675 // Handled below based on what the other operand is.
16676 NewElts[I] = Op;
16677 } else {
16678 NewElts[I] = DAG.getNode(ISD::FCANONICALIZE, SL, EltVT, Op);
16679 }
16680 }
16681
16682 // If one half is undef, and one is constant, prefer a splat vector.
16683 // Otherwise, convert the undef to 0.0 since that's cheaper to use and may
16684 // be free with a packed operation.
16685 if (NewElts[0].isUndef()) {
16686 NewElts[0] = isa<ConstantFPSDNode>(NewElts[1])
16687 ? NewElts[1]
16688 : DAG.getConstantFP(0.0f, SL, EltVT);
16689 }
16690
16691 if (NewElts[1].isUndef()) {
16692 NewElts[1] = isa<ConstantFPSDNode>(NewElts[0])
16693 ? NewElts[0]
16694 : DAG.getConstantFP(0.0f, SL, EltVT);
16695 }
16696
16697 return DAG.getBuildVector(VT, SL, NewElts);
16698 }
16699 }
16700
16701 return SDValue();
16702}
16703
16704static unsigned minMaxOpcToMin3Max3Opc(unsigned Opc) {
16705 switch (Opc) {
16706 case ISD::FMAXNUM:
16707 case ISD::FMAXNUM_IEEE:
16708 case ISD::FMAXIMUMNUM:
16709 return AMDGPUISD::FMAX3;
16710 case ISD::FMAXIMUM:
16711 return AMDGPUISD::FMAXIMUM3;
16712 case ISD::SMAX:
16713 return AMDGPUISD::SMAX3;
16714 case ISD::UMAX:
16715 return AMDGPUISD::UMAX3;
16716 case ISD::FMINNUM:
16717 case ISD::FMINNUM_IEEE:
16718 case ISD::FMINIMUMNUM:
16719 return AMDGPUISD::FMIN3;
16720 case ISD::FMINIMUM:
16721 return AMDGPUISD::FMINIMUM3;
16722 case ISD::SMIN:
16723 return AMDGPUISD::SMIN3;
16724 case ISD::UMIN:
16725 return AMDGPUISD::UMIN3;
16726 default:
16727 llvm_unreachable("Not a min/max opcode");
16728 }
16729}
16730
16731SDValue SITargetLowering::performIntMed3ImmCombine(SelectionDAG &DAG,
16732 const SDLoc &SL, SDValue Src,
16733 SDValue MinVal,
16734 SDValue MaxVal,
16735 bool Signed) const {
16736
16737 // med3 comes from
16738 // min(max(x, K0), K1), K0 < K1
16739 // max(min(x, K0), K1), K1 < K0
16740 //
16741 // "MinVal" and "MaxVal" respectively refer to the rhs of the
16742 // min/max op.
16743 ConstantSDNode *MinK = dyn_cast<ConstantSDNode>(MinVal);
16744 ConstantSDNode *MaxK = dyn_cast<ConstantSDNode>(MaxVal);
16745
16746 if (!MinK || !MaxK)
16747 return SDValue();
16748
16749 if (Signed) {
16750 if (MaxK->getAPIntValue().sge(MinK->getAPIntValue()))
16751 return SDValue();
16752 } else {
16753 if (MaxK->getAPIntValue().uge(MinK->getAPIntValue()))
16754 return SDValue();
16755 }
16756
16757 EVT VT = MinK->getValueType(0);
16758 unsigned Med3Opc = Signed ? AMDGPUISD::SMED3 : AMDGPUISD::UMED3;
16759 if (VT == MVT::i32 || (VT == MVT::i16 && Subtarget->hasMed3_16()))
16760 return DAG.getNode(Med3Opc, SL, VT, Src, MaxVal, MinVal);
16761
16762 // Note: we could also extend to i32 and use i32 med3 if i16 med3 is
16763 // not available, but this is unlikely to be profitable as constants
16764 // will often need to be materialized & extended, especially on
16765 // pre-GFX10 where VOP3 instructions couldn't take literal operands.
16766 return SDValue();
16767}
16768
16771 return C;
16772
16774 if (ConstantFPSDNode *C = BV->getConstantFPSplatNode())
16775 return C;
16776 }
16777
16778 return nullptr;
16779}
16780
16781SDValue SITargetLowering::performFPMed3ImmCombine(SelectionDAG &DAG,
16782 const SDLoc &SL, SDValue Op0,
16783 SDValue Op1,
16784 bool IsKnownNoNaNs) const {
16785 ConstantFPSDNode *K1 = getSplatConstantFP(Op1);
16786 if (!K1)
16787 return SDValue();
16788
16789 ConstantFPSDNode *K0 = getSplatConstantFP(Op0.getOperand(1));
16790 if (!K0)
16791 return SDValue();
16792
16793 // Ordered >= (although NaN inputs should have folded away by now).
16794 if (K0->getValueAPF() > K1->getValueAPF())
16795 return SDValue();
16796
16797 // med3 with a nan input acts like
16798 // v_min_f32(v_min_f32(S0.f32, S1.f32), S2.f32)
16799 //
16800 // So the result depends on whether the IEEE mode bit is enabled or not with a
16801 // signaling nan input.
16802 // ieee=1
16803 // s0 snan: yields s2
16804 // s1 snan: yields s2
16805 // s2 snan: qnan
16806
16807 // s0 qnan: min(s1, s2)
16808 // s1 qnan: min(s0, s2)
16809 // s2 qnan: min(s0, s1)
16810
16811 // ieee=0
16812 // s0 snan: min(s1, s2)
16813 // s1 snan: min(s0, s2)
16814 // s2 snan: qnan
16815
16816 // s0 qnan: min(s1, s2)
16817 // s1 qnan: min(s0, s2)
16818 // s2 qnan: min(s0, s1)
16819 const MachineFunction &MF = DAG.getMachineFunction();
16820 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
16821
16822 // TODO: Check IEEE bit enabled. We can form fmed3 with IEEE=0 regardless of
16823 // whether the input is a signaling nan if op0 is fmaximum or fmaximumnum. We
16824 // can only form if op0 is fmaxnum_ieee if IEEE=1.
16825 EVT VT = Op0.getValueType();
16826 if (Info->getMode().DX10Clamp) {
16827 // If dx10_clamp is enabled, NaNs clamp to 0.0. This is the same as the
16828 // hardware fmed3 behavior converting to a min.
16829 // FIXME: Should this be allowing -0.0?
16830 if (K1->isOne() && K0->isPosZero())
16831 return DAG.getNode(AMDGPUISD::CLAMP, SL, VT, Op0.getOperand(0));
16832 }
16833
16834 // med3 for f16 is only available on gfx9+, and not available for v2f16.
16835 if (VT == MVT::f32 || (VT == MVT::f16 && Subtarget->hasMed3_16())) {
16836 // This isn't safe with signaling NaNs because in IEEE mode, min/max on a
16837 // signaling NaN gives a quiet NaN. The quiet NaN input to the min would
16838 // then give the other result, which is different from med3 with a NaN
16839 // input.
16840 SDValue Var = Op0.getOperand(0);
16841 if (!IsKnownNoNaNs && !DAG.isKnownNeverSNaN(Var))
16842 return SDValue();
16843
16844 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
16845
16846 if ((!K0->hasOneUse() || TII->isInlineConstant(K0->getValueAPF())) &&
16847 (!K1->hasOneUse() || TII->isInlineConstant(K1->getValueAPF()))) {
16848 return DAG.getNode(AMDGPUISD::FMED3, SL, K0->getValueType(0), Var,
16849 SDValue(K0, 0), SDValue(K1, 0));
16850 }
16851 }
16852
16853 return SDValue();
16854}
16855
16856/// \return true if the subtarget supports minimum3 and maximum3 with the given
16857/// base min/max opcode \p Opc for type \p VT.
16858static bool supportsMin3Max3(const GCNSubtarget &Subtarget, unsigned Opc,
16859 EVT VT) {
16860 switch (Opc) {
16861 case ISD::FMINNUM:
16862 case ISD::FMAXNUM:
16863 case ISD::FMINNUM_IEEE:
16864 case ISD::FMAXNUM_IEEE:
16865 case ISD::FMINIMUMNUM:
16866 case ISD::FMAXIMUMNUM:
16867 return (VT == MVT::f32) || (VT == MVT::f16 && Subtarget.hasMin3Max3_16()) ||
16868 (VT == MVT::v2f16 && Subtarget.hasMin3Max3PKF16());
16869 case ISD::FMINIMUM:
16870 case ISD::FMAXIMUM:
16871 return (VT == MVT::f32 && Subtarget.hasMinimum3Maximum3F32()) ||
16872 (VT == MVT::f16 && Subtarget.hasMinimum3Maximum3F16()) ||
16873 (VT == MVT::v2f16 && Subtarget.hasMinimum3Maximum3PKF16());
16874 case ISD::SMAX:
16875 case ISD::SMIN:
16876 case ISD::UMAX:
16877 case ISD::UMIN:
16878 return (VT == MVT::i32) || (VT == MVT::i16 && Subtarget.hasMin3Max3_16());
16879 default:
16880 return false;
16881 }
16882
16883 llvm_unreachable("not a min/max opcode");
16884}
16885
16886SDValue SITargetLowering::performMinMaxCombine(SDNode *N,
16887 DAGCombinerInfo &DCI) const {
16888 SelectionDAG &DAG = DCI.DAG;
16889
16890 EVT VT = N->getValueType(0);
16891 unsigned Opc = N->getOpcode();
16892 SDValue Op0 = N->getOperand(0);
16893 SDValue Op1 = N->getOperand(1);
16894
16895 // Only do this if the inner op has one use since this will just increases
16896 // register pressure for no benefit.
16897
16898 if (supportsMin3Max3(*Subtarget, Opc, VT)) {
16899 auto IsTreeWithCombinableChildren = [Opc](SDValue Op) {
16900 return (Op.getOperand(0).getOpcode() == Opc &&
16901 Op.getOperand(0).hasOneUse()) ||
16902 (Op.getOperand(1).getOpcode() == Opc &&
16903 Op.getOperand(1).hasOneUse());
16904 };
16905
16906 bool CanTreeCombineApply = Op0.getOpcode() == Opc && Op0.hasOneUse() &&
16907 Op1.getOpcode() == Opc && Op1.hasOneUse();
16908 bool HasCombinableTreeChild =
16909 CanTreeCombineApply && (IsTreeWithCombinableChildren(Op0) ||
16910 IsTreeWithCombinableChildren(Op1));
16911
16912 // Tree reduction: when both operands are the same min/max op, restructure
16913 // to keep a 2-op node on top so higher tree levels can still combine.
16914 //
16915 // max(max(a, b), max(c, d)) -> max(max3(a, b, c), d)
16916 // min(min(a, b), min(c, d)) -> min(min3(a, b, c), d)
16917 //
16918 // Defer when either inner op is a tree node with combinable children.
16919 if (CanTreeCombineApply && !HasCombinableTreeChild) {
16920 SDLoc DL(N);
16921 SDValue Inner =
16923 Op0.getOperand(1), Op1.getOperand(0));
16924 return DAG.getNode(Opc, DL, VT, Inner, Op1.getOperand(1));
16925 }
16926
16927 // max(max(a, b), c) -> max3(a, b, c)
16928 // min(min(a, b), c) -> min3(a, b, c)
16929 // Deferred when Op0 is a tree node with combinable children.
16930 if (Op0.getOpcode() == Opc && Op0.hasOneUse() && !HasCombinableTreeChild) {
16931 SDLoc DL(N);
16932 return DAG.getNode(minMaxOpcToMin3Max3Opc(Opc), DL, N->getValueType(0),
16933 Op0.getOperand(0), Op0.getOperand(1), Op1);
16934 }
16935
16936 // Try commuted.
16937 // max(a, max(b, c)) -> max3(a, b, c)
16938 // min(a, min(b, c)) -> min3(a, b, c)
16939 // Deferred when Op1 is a tree node with combinable children.
16940 if (Op1.getOpcode() == Opc && Op1.hasOneUse() && !HasCombinableTreeChild) {
16941 SDLoc DL(N);
16942 return DAG.getNode(minMaxOpcToMin3Max3Opc(Opc), DL, N->getValueType(0),
16943 Op0, Op1.getOperand(0), Op1.getOperand(1));
16944 }
16945 }
16946
16947 // umin(sffbh(x), bitwidth) -> sffbh(x) if x is known to be not 0 or -1.
16948 SDValue FfbhSrc;
16949 uint64_t Clamp = 0;
16950 if (Opc == ISD::UMIN &&
16951 sd_match(Op0,
16953 sd_match(Op1, m_ConstInt(Clamp))) {
16954 unsigned BitWidth = FfbhSrc.getValueType().getScalarSizeInBits();
16955 if (Clamp >= BitWidth) {
16956 KnownBits Known = DAG.computeKnownBits(FfbhSrc);
16957 if (Known.isNonZero() && Known.Zero.getBoolValue())
16958 return Op0;
16959 }
16960 }
16961
16962 // min(max(x, K0), K1), K0 < K1 -> med3(x, K0, K1)
16963 // max(min(x, K0), K1), K1 < K0 -> med3(x, K1, K0)
16964 if (Opc == ISD::SMIN && Op0.getOpcode() == ISD::SMAX && Op0.hasOneUse()) {
16965 if (SDValue Med3 = performIntMed3ImmCombine(
16966 DAG, SDLoc(N), Op0->getOperand(0), Op1, Op0->getOperand(1), true))
16967 return Med3;
16968 }
16969 if (Opc == ISD::SMAX && Op0.getOpcode() == ISD::SMIN && Op0.hasOneUse()) {
16970 if (SDValue Med3 = performIntMed3ImmCombine(
16971 DAG, SDLoc(N), Op0->getOperand(0), Op0->getOperand(1), Op1, true))
16972 return Med3;
16973 }
16974
16975 if (Opc == ISD::UMIN && Op0.getOpcode() == ISD::UMAX && Op0.hasOneUse()) {
16976 if (SDValue Med3 = performIntMed3ImmCombine(
16977 DAG, SDLoc(N), Op0->getOperand(0), Op1, Op0->getOperand(1), false))
16978 return Med3;
16979 }
16980 if (Opc == ISD::UMAX && Op0.getOpcode() == ISD::UMIN && Op0.hasOneUse()) {
16981 if (SDValue Med3 = performIntMed3ImmCombine(
16982 DAG, SDLoc(N), Op0->getOperand(0), Op0->getOperand(1), Op1, false))
16983 return Med3;
16984 }
16985
16986 // if !is_snan(x):
16987 // fminnum(fmaxnum(x, K0), K1), K0 < K1 -> fmed3(x, K0, K1)
16988 // fminnum_ieee(fmaxnum_ieee(x, K0), K1), K0 < K1 -> fmed3(x, K0, K1)
16989 // fminnumnum(fmaxnumnum(x, K0), K1), K0 < K1 -> fmed3(x, K0, K1)
16990 // fmin_legacy(fmax_legacy(x, K0), K1), K0 < K1 -> fmed3(x, K0, K1)
16991 if (((Opc == ISD::FMINNUM && Op0.getOpcode() == ISD::FMAXNUM) ||
16994 (Opc == AMDGPUISD::FMIN_LEGACY &&
16995 Op0.getOpcode() == AMDGPUISD::FMAX_LEGACY)) &&
16996 (VT == MVT::f32 || VT == MVT::f64 ||
16997 (VT == MVT::f16 && Subtarget->has16BitInsts()) ||
16998 (VT == MVT::bf16 && Subtarget->hasBF16PackedInsts()) ||
16999 (VT == MVT::v2bf16 && Subtarget->hasBF16PackedInsts()) ||
17000 (VT == MVT::v2f16 && Subtarget->hasVOP3PInsts())) &&
17001 Op0.hasOneUse()) {
17002 if (SDValue Res = performFPMed3ImmCombine(DAG, SDLoc(N), Op0, Op1,
17003 N->getFlags().hasNoNaNs()))
17004 return Res;
17005 }
17006
17007 // Prefer fminnum_ieee over fminimum. For gfx950, minimum/maximum are legal
17008 // for some types, but at a higher cost since it's implemented with a 3
17009 // operand form.
17010 const SDNodeFlags Flags = N->getFlags();
17011 if ((Opc == ISD::FMINIMUM || Opc == ISD::FMAXIMUM) && Flags.hasNoNaNs() &&
17012 !Subtarget->hasIEEEMinimumMaximumInsts() &&
17014 unsigned NewOpc =
17016 return DAG.getNode(NewOpc, SDLoc(N), VT, Op0, Op1, Flags);
17017 }
17018
17019 return SDValue();
17020}
17021
17025 // FIXME: Should this be allowing -0.0?
17026 return (CA->isPosZero() && CB->isOne()) ||
17027 (CA->isOne() && CB->isPosZero());
17028 }
17029 }
17030
17031 return false;
17032}
17033
17034// FIXME: Should only worry about snans for version with chain.
17035SDValue SITargetLowering::performFMed3Combine(SDNode *N,
17036 DAGCombinerInfo &DCI) const {
17037 EVT VT = N->getValueType(0);
17038 // v_med3_f32 and v_max_f32 behave identically wrt denorms, exceptions and
17039 // NaNs. With a NaN input, the order of the operands may change the result.
17040
17041 SelectionDAG &DAG = DCI.DAG;
17042 SDLoc SL(N);
17043
17044 SDValue Src0 = N->getOperand(0);
17045 SDValue Src1 = N->getOperand(1);
17046 SDValue Src2 = N->getOperand(2);
17047
17048 if (isClampZeroToOne(Src0, Src1)) {
17049 // const_a, const_b, x -> clamp is safe in all cases including signaling
17050 // nans.
17051 // FIXME: Should this be allowing -0.0?
17052 return DAG.getNode(AMDGPUISD::CLAMP, SL, VT, Src2);
17053 }
17054
17055 const MachineFunction &MF = DAG.getMachineFunction();
17056 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
17057
17058 // FIXME: dx10_clamp behavior assumed in instcombine. Should we really bother
17059 // handling no dx10-clamp?
17060 if (Info->getMode().DX10Clamp) {
17061 // If NaNs is clamped to 0, we are free to reorder the inputs.
17062
17063 if (isa<ConstantFPSDNode>(Src0) && !isa<ConstantFPSDNode>(Src1))
17064 std::swap(Src0, Src1);
17065
17066 if (isa<ConstantFPSDNode>(Src1) && !isa<ConstantFPSDNode>(Src2))
17067 std::swap(Src1, Src2);
17068
17069 if (isa<ConstantFPSDNode>(Src0) && !isa<ConstantFPSDNode>(Src1))
17070 std::swap(Src0, Src1);
17071
17072 if (isClampZeroToOne(Src1, Src2))
17073 return DAG.getNode(AMDGPUISD::CLAMP, SL, VT, Src0);
17074 }
17075
17076 return SDValue();
17077}
17078
17079SDValue SITargetLowering::performCvtPkRTZCombine(SDNode *N,
17080 DAGCombinerInfo &DCI) const {
17081 SDValue Src0 = N->getOperand(0);
17082 SDValue Src1 = N->getOperand(1);
17083 if (Src0.isUndef() && Src1.isUndef())
17084 return DCI.DAG.getUNDEF(N->getValueType(0));
17085 return SDValue();
17086}
17087
17088/// V_CVT_SCALEF32_* conversions to a narrow format only read the exponent
17089/// field of their f32 scale operand.
17090SDValue
17091SITargetLowering::performExponentOnlyScaleCombine(SDNode *N,
17092 DAGCombinerInfo &DCI) const {
17093 std::optional<unsigned> ScaleArgIdx =
17094 AMDGPU::getExponentOnlyScaleArgIdx(N->getConstantOperandVal(0));
17095 if (!ScaleArgIdx)
17096 return SDValue();
17097
17098 // Operand 0 is the intrinsic ID.
17099 unsigned OpIdx = *ScaleArgIdx + 1;
17100 SelectionDAG &DAG = DCI.DAG;
17101 SDValue Scale = N->getOperand(OpIdx);
17103
17104 if (SDValue DemandedScale =
17105 SimplifyMultipleUseDemandedBits(Scale, Demanded, DAG)) {
17106 SmallVector<SDValue> Ops(N->ops());
17107 Ops[OpIdx] = DemandedScale;
17108 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SDLoc(N), N->getVTList(), Ops);
17109 }
17110
17111 if (SimplifyDemandedBits(Scale, Demanded, DCI))
17112 return SDValue(N, 0);
17113
17114 return SDValue();
17115}
17116
17117// Check if EXTRACT_VECTOR_ELT/INSERT_VECTOR_ELT (<n x e>, var-idx) should be
17118// expanded into a set of cmp/select instructions.
17120 unsigned NumElem,
17121 bool IsDivergentIdx,
17122 const GCNSubtarget *Subtarget) {
17123 if (Subtarget->getCLOpts().use_divergent_register_indexing)
17124 return false;
17125
17126 unsigned VecSize = EltSize * NumElem;
17127
17128 // Sub-dword vectors of size 2 dword or less have better implementation.
17129 if (VecSize <= 64 && EltSize < 32)
17130 return false;
17131
17132 // Always expand the rest of sub-dword instructions, otherwise it will be
17133 // lowered via memory.
17134 if (EltSize < 32)
17135 return true;
17136
17137 // Always do this if var-idx is divergent, otherwise it will become a loop.
17138 if (IsDivergentIdx)
17139 return true;
17140
17141 // Large vectors would yield too many compares and v_cndmask_b32 instructions.
17142 unsigned NumInsts = NumElem /* Number of compares */ +
17143 ((EltSize + 31) / 32) * NumElem /* Number of cndmasks */;
17144
17145 // On some architectures (GFX9) movrel is not available and it's better
17146 // to expand.
17147 if (Subtarget->useVGPRIndexMode())
17148 return NumInsts <= 16;
17149
17150 // If movrel is available, use it instead of expanding for vector of 8
17151 // elements.
17152 if (Subtarget->hasMovrel())
17153 return NumInsts <= 15;
17154
17155 return true;
17156}
17157
17159 SDValue Idx = N->getOperand(N->getNumOperands() - 1);
17160 if (isa<ConstantSDNode>(Idx))
17161 return false;
17162
17163 SDValue Vec = N->getOperand(0);
17164 EVT VecVT = Vec.getValueType();
17165 EVT EltVT = VecVT.getVectorElementType();
17166 unsigned EltSize = EltVT.getSizeInBits();
17167 unsigned NumElem = VecVT.getVectorNumElements();
17168
17170 EltSize, NumElem, Idx->isDivergent(), getSubtarget());
17171}
17172
17173SDValue
17174SITargetLowering::performExtractVectorEltCombine(SDNode *N,
17175 DAGCombinerInfo &DCI) const {
17176 SDValue Vec = N->getOperand(0);
17177 SelectionDAG &DAG = DCI.DAG;
17178
17179 EVT VecVT = Vec.getValueType();
17180 EVT VecEltVT = VecVT.getVectorElementType();
17181 EVT ResVT = N->getValueType(0);
17182
17183 unsigned VecSize = VecVT.getSizeInBits();
17184 unsigned VecEltSize = VecEltVT.getSizeInBits();
17185
17186 if ((Vec.getOpcode() == ISD::FNEG || Vec.getOpcode() == ISD::FABS) &&
17188 SDLoc SL(N);
17189 SDValue Idx = N->getOperand(1);
17190 SDValue Elt =
17191 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT, Vec.getOperand(0), Idx);
17192 return DAG.getNode(Vec.getOpcode(), SL, ResVT, Elt);
17193 }
17194
17195 // (extract_vector_element (and {y0, y1}, (build_vector 0x1f, 0x1f)), index)
17196 // -> (and (extract_vector_element {y0, y1}, index), 0x1f)
17197 // There are optimisations to transform 64-bit shifts into 32-bit shifts
17198 // depending on the shift operand. See e.g. performSraCombine().
17199 // This combine ensures that the optimisation is compatible with v2i32
17200 // legalised AND.
17201 if (VecVT == MVT::v2i32 && Vec->getOpcode() == ISD::AND &&
17202 Vec->getOperand(1)->getOpcode() == ISD::BUILD_VECTOR) {
17203
17205 if (!C || C->getZExtValue() != 0x1f)
17206 return SDValue();
17207
17208 SDLoc SL(N);
17209 SDValue AndMask = DAG.getConstant(0x1f, SL, MVT::i32);
17210 SDValue EVE = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32,
17211 Vec->getOperand(0), N->getOperand(1));
17212 SDValue A = DAG.getNode(ISD::AND, SL, MVT::i32, EVE, AndMask);
17213 DAG.ReplaceAllUsesWith(N, A.getNode());
17214 }
17215
17216 // ScalarRes = EXTRACT_VECTOR_ELT ((vector-BINOP Vec1, Vec2), Idx)
17217 // =>
17218 // Vec1Elt = EXTRACT_VECTOR_ELT(Vec1, Idx)
17219 // Vec2Elt = EXTRACT_VECTOR_ELT(Vec2, Idx)
17220 // ScalarRes = scalar-BINOP Vec1Elt, Vec2Elt
17221 if (Vec.hasOneUse() && DCI.isBeforeLegalize() && VecEltVT == ResVT) {
17222 SDLoc SL(N);
17223 SDValue Idx = N->getOperand(1);
17224 unsigned Opc = Vec.getOpcode();
17225
17226 switch (Opc) {
17227 default:
17228 break;
17229 // TODO: Support other binary operations.
17230 case ISD::FADD:
17231 case ISD::FSUB:
17232 case ISD::FMUL:
17233 case ISD::ADD:
17234 case ISD::UMIN:
17235 case ISD::UMAX:
17236 case ISD::SMIN:
17237 case ISD::SMAX:
17238 case ISD::FMAXNUM:
17239 case ISD::FMINNUM:
17240 case ISD::FMAXNUM_IEEE:
17241 case ISD::FMINNUM_IEEE:
17242 case ISD::FMAXIMUM:
17243 case ISD::FMINIMUM: {
17244 SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT,
17245 Vec.getOperand(0), Idx);
17246 SDValue Elt1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT,
17247 Vec.getOperand(1), Idx);
17248
17249 DCI.AddToWorklist(Elt0.getNode());
17250 DCI.AddToWorklist(Elt1.getNode());
17251 return DAG.getNode(Opc, SL, ResVT, Elt0, Elt1, Vec->getFlags());
17252 }
17253 }
17254 }
17255
17256 // EXTRACT_VECTOR_ELT (<n x e>, var-idx) => n x select (e, const-idx)
17258 SDLoc SL(N);
17259 SDValue Idx = N->getOperand(1);
17260 SDValue V;
17261 for (unsigned I = 0, E = VecVT.getVectorNumElements(); I < E; ++I) {
17262 SDValue IC = DAG.getVectorIdxConstant(I, SL);
17263 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT, Vec, IC);
17264 if (I == 0)
17265 V = Elt;
17266 else
17267 V = DAG.getSelectCC(SL, Idx, IC, Elt, V, ISD::SETEQ);
17268 }
17269 return V;
17270 }
17271
17272 // EXTRACT_VECTOR_ELT (v2i32 bitcast (i64/f64:k), Idx)
17273 // =>
17274 // i32:Lo(k) if Idx == 0, or
17275 // i32:Hi(k) if Idx == 1
17276 auto *Idx = dyn_cast<ConstantSDNode>(N->getOperand(1));
17277 if (Vec.getOpcode() == ISD::BITCAST && VecVT == MVT::v2i32 && Idx) {
17278 SDLoc SL(N);
17279 SDValue PeekThrough = Vec.getOperand(0);
17280 auto *KImm = dyn_cast<ConstantSDNode>(PeekThrough);
17281 if (KImm && KImm->getValueType(0).getSizeInBits() == 64) {
17282 uint64_t KImmValue = KImm->getZExtValue();
17283 return DAG.getConstant(
17284 (KImmValue >> (32 * Idx->getZExtValue())) & 0xffffffff, SL, MVT::i32);
17285 }
17286 auto *KFPImm = dyn_cast<ConstantFPSDNode>(PeekThrough);
17287 if (KFPImm && KFPImm->getValueType(0).getSizeInBits() == 64) {
17288 uint64_t KFPImmValue =
17289 KFPImm->getValueAPF().bitcastToAPInt().getZExtValue();
17290 return DAG.getConstant((KFPImmValue >> (32 * Idx->getZExtValue())) &
17291 0xffffffff,
17292 SL, MVT::i32);
17293 }
17294 }
17295
17296 if (!DCI.isBeforeLegalize())
17297 return SDValue();
17298
17299 // Try to turn sub-dword accesses of vectors into accesses of the same 32-bit
17300 // elements. This exposes more load reduction opportunities by replacing
17301 // multiple small extract_vector_elements with a single 32-bit extract.
17302 if (isa<MemSDNode>(Vec) && VecEltSize <= 16 && VecEltVT.isByteSized() &&
17303 VecSize > 32 && VecSize % 32 == 0 && Idx) {
17304 EVT NewVT = getEquivalentMemType(*DAG.getContext(), VecVT);
17305
17306 unsigned BitIndex = Idx->getZExtValue() * VecEltSize;
17307 unsigned EltIdx = BitIndex / 32;
17308 unsigned LeftoverBitIdx = BitIndex % 32;
17309 SDLoc SL(N);
17310
17311 SDValue Cast = DAG.getNode(ISD::BITCAST, SL, NewVT, Vec);
17312 DCI.AddToWorklist(Cast.getNode());
17313
17314 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Cast,
17315 DAG.getConstant(EltIdx, SL, MVT::i32));
17316 DCI.AddToWorklist(Elt.getNode());
17317 SDValue Srl = DAG.getNode(ISD::SRL, SL, MVT::i32, Elt,
17318 DAG.getConstant(LeftoverBitIdx, SL, MVT::i32));
17319 DCI.AddToWorklist(Srl.getNode());
17320
17321 EVT VecEltAsIntVT = VecEltVT.changeTypeToInteger();
17322 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, VecEltAsIntVT, Srl);
17323 DCI.AddToWorklist(Trunc.getNode());
17324
17325 if (VecEltVT == ResVT) {
17326 return DAG.getNode(ISD::BITCAST, SL, VecEltVT, Trunc);
17327 }
17328
17329 assert(ResVT.isScalarInteger());
17330 return DAG.getAnyExtOrTrunc(Trunc, SL, ResVT);
17331 }
17332
17333 return SDValue();
17334}
17335
17336SDValue
17337SITargetLowering::performInsertVectorEltCombine(SDNode *N,
17338 DAGCombinerInfo &DCI) const {
17339 SDValue Vec = N->getOperand(0);
17340 SDValue Idx = N->getOperand(2);
17341 EVT VecVT = Vec.getValueType();
17342 EVT EltVT = VecVT.getVectorElementType();
17343
17344 // INSERT_VECTOR_ELT (<n x e>, var-idx)
17345 // => BUILD_VECTOR n x select (e, const-idx)
17346 // 16-bit elements are left to lowerINSERT_VECTOR_ELT, which is cheaper.
17348 (EltVT.getSizeInBits() == 16 &&
17350 return SDValue();
17351
17352 SelectionDAG &DAG = DCI.DAG;
17353 SDLoc SL(N);
17354 SDValue Ins = N->getOperand(1);
17355 EVT IdxVT = Idx.getValueType();
17356
17358 for (unsigned I = 0, E = VecVT.getVectorNumElements(); I < E; ++I) {
17359 SDValue IC = DAG.getConstant(I, SL, IdxVT);
17360 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Vec, IC);
17361 SDValue V = DAG.getSelectCC(SL, Idx, IC, Ins, Elt, ISD::SETEQ);
17362 Ops.push_back(V);
17363 }
17364
17365 return DAG.getBuildVector(VecVT, SL, Ops);
17366}
17367
17368/// Return the source of an fp_extend from f16 to f32, or a converted FP
17369/// constant.
17371 if (Src.getOpcode() == ISD::FP_EXTEND &&
17372 Src.getOperand(0).getValueType() == MVT::f16) {
17373 return Src.getOperand(0);
17374 }
17375
17376 if (auto *CFP = dyn_cast<ConstantFPSDNode>(Src)) {
17377 APFloat Val = CFP->getValueAPF();
17378 bool LosesInfo = true;
17380 if (!LosesInfo)
17381 return DAG.getConstantFP(Val, SDLoc(Src), MVT::f16);
17382 }
17383
17384 return SDValue();
17385}
17386
17387SDValue SITargetLowering::performFPRoundCombine(SDNode *N,
17388 DAGCombinerInfo &DCI) const {
17389 assert(Subtarget->has16BitInsts() && !Subtarget->hasMed3_16() &&
17390 "combine only useful on gfx8");
17391
17392 SDValue TruncSrc = N->getOperand(0);
17393 EVT VT = N->getValueType(0);
17394 if (VT != MVT::f16)
17395 return SDValue();
17396
17397 if (TruncSrc.getOpcode() != AMDGPUISD::FMED3 ||
17398 TruncSrc.getValueType() != MVT::f32 || !TruncSrc.hasOneUse())
17399 return SDValue();
17400
17401 SelectionDAG &DAG = DCI.DAG;
17402 SDLoc SL(N);
17403
17404 // Optimize f16 fmed3 pattern performed on f32. On gfx8 there is no f16 fmed3,
17405 // and expanding it with min/max saves 1 instruction vs. casting to f32 and
17406 // casting back.
17407
17408 // fptrunc (f32 (fmed3 (fpext f16:a, fpext f16:b, fpext f16:c))) =>
17409 // fmin(fmax(a, b), fmax(fmin(a, b), c))
17410 SDValue A = strictFPExtFromF16(DAG, TruncSrc.getOperand(0));
17411 if (!A)
17412 return SDValue();
17413
17414 SDValue B = strictFPExtFromF16(DAG, TruncSrc.getOperand(1));
17415 if (!B)
17416 return SDValue();
17417
17418 SDValue C = strictFPExtFromF16(DAG, TruncSrc.getOperand(2));
17419 if (!C)
17420 return SDValue();
17421
17422 // This changes signaling nan behavior. If an input is a signaling nan, it
17423 // would have been quieted by the fpext originally. We don't care because
17424 // these are unconstrained ops. If we needed to insert quieting canonicalizes
17425 // we would be worse off than just doing the promotion.
17426 SDValue A1 = DAG.getNode(ISD::FMINNUM_IEEE, SL, VT, A, B);
17427 SDValue B1 = DAG.getNode(ISD::FMAXNUM_IEEE, SL, VT, A, B);
17428 SDValue C1 = DAG.getNode(ISD::FMAXNUM_IEEE, SL, VT, A1, C);
17429 return DAG.getNode(ISD::FMINNUM_IEEE, SL, VT, B1, C1);
17430}
17431
17432unsigned SITargetLowering::getFusedOpcode(const SelectionDAG &DAG,
17433 const SDNode *N0,
17434 const SDNode *N1) const {
17435 EVT VT = N0->getValueType(0);
17436
17437 // Only do this if we are not trying to support denormals. v_mad_f32 does not
17438 // support denormals ever.
17439 if (((VT == MVT::f32 &&
17441 (VT == MVT::f16 && Subtarget->hasMadF16() &&
17444 return ISD::FMAD;
17445
17446 if (N0->getFlags().hasAllowContract() && N1->getFlags().hasAllowContract() &&
17448 return ISD::FMA;
17449 }
17450
17451 return 0;
17452}
17453
17454// For a reassociatable opcode perform:
17455// op x, (op y, z) -> op (op x, z), y, if x and z are uniform
17456SDValue SITargetLowering::reassociateScalarOps(SDNode *N,
17457 SelectionDAG &DAG) const {
17458 EVT VT = N->getValueType(0);
17459 if (VT != MVT::i32 && VT != MVT::i64)
17460 return SDValue();
17461
17462 if (DAG.isBaseWithConstantOffset(SDValue(N, 0)))
17463 return SDValue();
17464
17465 unsigned Opc = N->getOpcode();
17466 SDValue Op0 = N->getOperand(0);
17467 SDValue Op1 = N->getOperand(1);
17468
17469 if (!(Op0->isDivergent() ^ Op1->isDivergent()))
17470 return SDValue();
17471
17472 if (Op0->isDivergent())
17473 std::swap(Op0, Op1);
17474
17475 if (Op1.getOpcode() != Opc || !Op1.hasOneUse())
17476 return SDValue();
17477
17478 SDValue Op2 = Op1.getOperand(1);
17479 Op1 = Op1.getOperand(0);
17480 if (!(Op1->isDivergent() ^ Op2->isDivergent()))
17481 return SDValue();
17482
17483 if (Op1->isDivergent())
17484 std::swap(Op1, Op2);
17485
17486 SDLoc SL(N);
17487 SDValue Add1 = DAG.getNode(Opc, SL, VT, Op0, Op1);
17488 return DAG.getNode(Opc, SL, VT, Add1, Op2);
17489}
17490
17491static SDValue getMad64_32(SelectionDAG &DAG, const SDLoc &SL, EVT VT,
17492 SDValue N0, SDValue N1, SDValue N2, bool Signed) {
17494 SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i1);
17495 SDValue Mad = DAG.getNode(MadOpc, SL, VTs, N0, N1, N2);
17496 return DAG.getNode(ISD::TRUNCATE, SL, VT, Mad);
17497}
17498
17499// Fold
17500// y = lshr i64 x, 32
17501// res = add (mul i64 y, Const), x where "Const" is a 64-bit constant
17502// with Const.hi == -1
17503// To
17504// res = mad_u64_u32 y.lo ,Const.lo, x.lo
17506 SDValue MulLHS, SDValue MulRHS,
17507 SDValue AddRHS) {
17508 if (MulRHS.getOpcode() == ISD::SRL)
17509 std::swap(MulLHS, MulRHS);
17510
17511 if (MulLHS.getValueType() != MVT::i64 || MulLHS.getOpcode() != ISD::SRL)
17512 return SDValue();
17513
17514 ConstantSDNode *ShiftVal = dyn_cast<ConstantSDNode>(MulLHS.getOperand(1));
17515 if (!ShiftVal || ShiftVal->getAsZExtVal() != 32 ||
17516 MulLHS.getOperand(0) != AddRHS)
17517 return SDValue();
17518
17520 if (!Const || Hi_32(Const->getZExtValue()) != uint32_t(-1))
17521 return SDValue();
17522
17523 SDValue ConstMul =
17524 DAG.getConstant(Lo_32(Const->getZExtValue()), SL, MVT::i32);
17525 return getMad64_32(DAG, SL, MVT::i64,
17526 DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, MulLHS), ConstMul,
17527 DAG.getZeroExtendInReg(AddRHS, SL, MVT::i32), false);
17528}
17529
17530// Fold (add (mul x, y), z) --> (mad_[iu]64_[iu]32 x, y, z) plus high
17531// multiplies, if any.
17532//
17533// Full 64-bit multiplies that feed into an addition are lowered here instead
17534// of using the generic expansion. The generic expansion ends up with
17535// a tree of ADD nodes that prevents us from using the "add" part of the
17536// MAD instruction. The expansion produced here results in a chain of ADDs
17537// instead of a tree.
17538SDValue SITargetLowering::tryFoldToMad64_32(SDNode *N,
17539 DAGCombinerInfo &DCI) const {
17540 assert(N->isAnyAdd());
17541
17542 SelectionDAG &DAG = DCI.DAG;
17543 EVT VT = N->getValueType(0);
17544 SDLoc SL(N);
17545 SDValue LHS = N->getOperand(0);
17546 SDValue RHS = N->getOperand(1);
17547
17548 if (VT.isVector())
17549 return SDValue();
17550
17551 // S_MUL_HI_[IU]32 was added in gfx9, which allows us to keep the overall
17552 // result in scalar registers for uniform values.
17553 if (!N->isDivergent() && Subtarget->hasSMulHi())
17554 return SDValue();
17555
17556 unsigned NumBits = VT.getScalarSizeInBits();
17557 if (NumBits <= 32 || NumBits > 64)
17558 return SDValue();
17559
17560 if (LHS.getOpcode() != ISD::MUL) {
17561 assert(RHS.getOpcode() == ISD::MUL);
17562 std::swap(LHS, RHS);
17563 }
17564
17565 // Avoid the fold if it would unduly increase the number of multiplies due to
17566 // multiple uses, except on hardware with full-rate multiply-add (which is
17567 // part of full-rate 64-bit ops).
17568 if (!Subtarget->hasFullRate64Ops()) {
17569 unsigned NumUsers = 0;
17570 for (SDNode *User : LHS->users()) {
17571 // There is a use that does not feed into addition, so the multiply can't
17572 // be removed. We prefer MUL + ADD + ADDC over MAD + MUL.
17573 if (!User->isAnyAdd())
17574 return SDValue();
17575
17576 // We prefer 2xMAD over MUL + 2xADD + 2xADDC (code density), and prefer
17577 // MUL + 3xADD + 3xADDC over 3xMAD.
17578 ++NumUsers;
17579 if (NumUsers >= 3)
17580 return SDValue();
17581 }
17582 }
17583
17584 SDValue MulLHS = LHS.getOperand(0);
17585 SDValue MulRHS = LHS.getOperand(1);
17586 SDValue AddRHS = RHS;
17587
17588 if (SDValue FoldedMAD = tryFoldMADwithSRL(DAG, SL, MulLHS, MulRHS, AddRHS))
17589 return FoldedMAD;
17590
17591 // Always check whether operands are small unsigned values, since that
17592 // knowledge is useful in more cases. Check for small signed values only if
17593 // doing so can unlock a shorter code sequence.
17594 bool MulLHSUnsigned32 = numBitsUnsigned(MulLHS, DAG) <= 32;
17595 bool MulRHSUnsigned32 = numBitsUnsigned(MulRHS, DAG) <= 32;
17596
17597 bool MulSignedLo = false;
17598 if (!MulLHSUnsigned32 || !MulRHSUnsigned32) {
17599 MulSignedLo =
17600 numBitsSigned(MulLHS, DAG) <= 32 && numBitsSigned(MulRHS, DAG) <= 32;
17601 }
17602
17603 // The operands and final result all have the same number of bits. If
17604 // operands need to be extended, they can be extended with garbage. The
17605 // resulting garbage in the high bits of the mad_[iu]64_[iu]32 result is
17606 // truncated away in the end.
17607 if (VT != MVT::i64) {
17608 MulLHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i64, MulLHS);
17609 MulRHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i64, MulRHS);
17610 AddRHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i64, AddRHS);
17611 }
17612
17613 // The basic code generated is conceptually straightforward. Pseudo code:
17614 //
17615 // accum = mad_64_32 lhs.lo, rhs.lo, accum
17616 // accum.hi = add (mul lhs.hi, rhs.lo), accum.hi
17617 // accum.hi = add (mul lhs.lo, rhs.hi), accum.hi
17618 //
17619 // The second and third lines are optional, depending on whether the factors
17620 // are {sign,zero}-extended or not.
17621 //
17622 // The actual DAG is noisier than the pseudo code, but only due to
17623 // instructions that disassemble values into low and high parts, and
17624 // assemble the final result.
17625 SDValue One = DAG.getConstant(1, SL, MVT::i32);
17626
17627 auto MulLHSLo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, MulLHS);
17628 auto MulRHSLo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, MulRHS);
17629 SDValue Accum =
17630 getMad64_32(DAG, SL, MVT::i64, MulLHSLo, MulRHSLo, AddRHS, MulSignedLo);
17631
17632 if (!MulSignedLo && (!MulLHSUnsigned32 || !MulRHSUnsigned32)) {
17633 auto [AccumLo, AccumHi] = DAG.SplitScalar(Accum, SL, MVT::i32, MVT::i32);
17634
17635 if (!MulLHSUnsigned32) {
17636 auto MulLHSHi =
17637 DAG.getNode(ISD::EXTRACT_ELEMENT, SL, MVT::i32, MulLHS, One);
17638 SDValue MulHi = DAG.getNode(ISD::MUL, SL, MVT::i32, MulLHSHi, MulRHSLo);
17639 AccumHi = DAG.getNode(ISD::ADD, SL, MVT::i32, MulHi, AccumHi);
17640 }
17641
17642 if (!MulRHSUnsigned32) {
17643 auto MulRHSHi =
17644 DAG.getNode(ISD::EXTRACT_ELEMENT, SL, MVT::i32, MulRHS, One);
17645 SDValue MulHi = DAG.getNode(ISD::MUL, SL, MVT::i32, MulLHSLo, MulRHSHi);
17646 AccumHi = DAG.getNode(ISD::ADD, SL, MVT::i32, MulHi, AccumHi);
17647 }
17648
17649 Accum = DAG.getBuildVector(MVT::v2i32, SL, {AccumLo, AccumHi});
17650 Accum = DAG.getBitcast(MVT::i64, Accum);
17651 }
17652
17653 if (VT != MVT::i64)
17654 Accum = DAG.getNode(ISD::TRUNCATE, SL, VT, Accum);
17655 return Accum;
17656}
17657
17658SDValue
17659SITargetLowering::foldAddSub64WithZeroLowBitsTo32(SDNode *N,
17660 DAGCombinerInfo &DCI) const {
17661 SDValue RHS = N->getOperand(1);
17662 auto *CRHS = dyn_cast<ConstantSDNode>(RHS);
17663 if (!CRHS)
17664 return SDValue();
17665
17666 // TODO: Worth using computeKnownBits? Maybe expensive since it's so
17667 // common.
17668 uint64_t Val = CRHS->getZExtValue();
17669 if (countr_zero(Val) >= 32) {
17670 SelectionDAG &DAG = DCI.DAG;
17671 SDLoc SL(N);
17672 SDValue LHS = N->getOperand(0);
17673
17674 // Avoid carry machinery if we know the low half of the add does not
17675 // contribute to the final result.
17676 //
17677 // add i64:x, K if computeTrailingZeros(K) >= 32
17678 // => build_pair (add x.hi, K.hi), x.lo
17679
17680 // Breaking the 64-bit add here with this strange constant is unlikely
17681 // to interfere with addressing mode patterns.
17682
17683 SDValue Hi = getHiHalf64(LHS, DAG);
17684 SDValue ConstHi32 = DAG.getConstant(Hi_32(Val), SL, MVT::i32);
17685 unsigned Opcode = N->getOpcode();
17686 if (Opcode == ISD::PTRADD)
17687 Opcode = ISD::ADD;
17688 SDValue AddHi =
17689 DAG.getNode(Opcode, SL, MVT::i32, Hi, ConstHi32, N->getFlags());
17690
17691 SDValue Lo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, LHS);
17692 return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i64, Lo, AddHi);
17693 }
17694
17695 return SDValue();
17696}
17697
17698// Collect the ultimate src of each of the mul node's operands, and confirm
17699// each operand is 8 bytes.
17700static std::optional<ByteProvider<SDValue>>
17701handleMulOperand(const SDValue &MulOperand) {
17702 auto Byte0 = calculateByteProvider(MulOperand, 0, 0);
17703 if (!Byte0 || Byte0->isConstantZero()) {
17704 return std::nullopt;
17705 }
17706 auto Byte1 = calculateByteProvider(MulOperand, 1, 0);
17707 if (Byte1 && !Byte1->isConstantZero()) {
17708 return std::nullopt;
17709 }
17710 return Byte0;
17711}
17712
17713static unsigned addPermMasks(unsigned First, unsigned Second) {
17714 unsigned FirstCs = First & 0x0c0c0c0c;
17715 unsigned SecondCs = Second & 0x0c0c0c0c;
17716 unsigned FirstNoCs = First & ~0x0c0c0c0c;
17717 unsigned SecondNoCs = Second & ~0x0c0c0c0c;
17718
17719 assert((FirstCs & 0xFF) | (SecondCs & 0xFF));
17720 assert((FirstCs & 0xFF00) | (SecondCs & 0xFF00));
17721 assert((FirstCs & 0xFF0000) | (SecondCs & 0xFF0000));
17722 assert((FirstCs & 0xFF000000) | (SecondCs & 0xFF000000));
17723
17724 return (FirstNoCs | SecondNoCs) | (FirstCs & SecondCs);
17725}
17726
17727struct DotSrc {
17729 int64_t PermMask;
17731};
17732
17736 SmallVectorImpl<DotSrc> &Src1s, int Step) {
17737
17738 assert(Src0.Src.has_value() && Src1.Src.has_value());
17739 // Src0s and Src1s are empty, just place arbitrarily.
17740 if (Step == 0) {
17741 Src0s.push_back({*Src0.Src, ((Src0.SrcOffset % 4) << 24) + 0x0c0c0c,
17742 Src0.SrcOffset / 4});
17743 Src1s.push_back({*Src1.Src, ((Src1.SrcOffset % 4) << 24) + 0x0c0c0c,
17744 Src1.SrcOffset / 4});
17745 return;
17746 }
17747
17748 for (int BPI = 0; BPI < 2; BPI++) {
17749 std::pair<ByteProvider<SDValue>, ByteProvider<SDValue>> BPP = {Src0, Src1};
17750 if (BPI == 1) {
17751 BPP = {Src1, Src0};
17752 }
17753 unsigned ZeroMask = 0x0c0c0c0c;
17754 unsigned FMask = 0xFF << (8 * (3 - Step));
17755
17756 unsigned FirstMask =
17757 (BPP.first.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask);
17758 unsigned SecondMask =
17759 (BPP.second.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask);
17760 // Attempt to find Src vector which contains our SDValue, if so, add our
17761 // perm mask to the existing one. If we are unable to find a match for the
17762 // first SDValue, attempt to find match for the second.
17763 int FirstGroup = -1;
17764 for (int I = 0; I < 2; I++) {
17765 SmallVectorImpl<DotSrc> &Srcs = I == 0 ? Src0s : Src1s;
17766 auto MatchesFirst = [&BPP](DotSrc &IterElt) {
17767 return IterElt.SrcOp == *BPP.first.Src &&
17768 (IterElt.DWordOffset == (BPP.first.SrcOffset / 4));
17769 };
17770
17771 auto *Match = llvm::find_if(Srcs, MatchesFirst);
17772 if (Match != Srcs.end()) {
17773 Match->PermMask = addPermMasks(FirstMask, Match->PermMask);
17774 FirstGroup = I;
17775 break;
17776 }
17777 }
17778 if (FirstGroup != -1) {
17779 SmallVectorImpl<DotSrc> &Srcs = FirstGroup == 1 ? Src0s : Src1s;
17780 auto MatchesSecond = [&BPP](DotSrc &IterElt) {
17781 return IterElt.SrcOp == *BPP.second.Src &&
17782 (IterElt.DWordOffset == (BPP.second.SrcOffset / 4));
17783 };
17784 auto *Match = llvm::find_if(Srcs, MatchesSecond);
17785 if (Match != Srcs.end()) {
17786 Match->PermMask = addPermMasks(SecondMask, Match->PermMask);
17787 } else
17788 Srcs.push_back({*BPP.second.Src, SecondMask, BPP.second.SrcOffset / 4});
17789 return;
17790 }
17791 }
17792
17793 // If we have made it here, then we could not find a match in Src0s or Src1s
17794 // for either Src0 or Src1, so just place them arbitrarily.
17795
17796 unsigned ZeroMask = 0x0c0c0c0c;
17797 unsigned FMask = 0xFF << (8 * (3 - Step));
17798
17799 Src0s.push_back(
17800 {*Src0.Src,
17801 ((Src0.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask)),
17802 Src0.SrcOffset / 4});
17803 Src1s.push_back(
17804 {*Src1.Src,
17805 ((Src1.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask)),
17806 Src1.SrcOffset / 4});
17807}
17808
17810 SmallVectorImpl<DotSrc> &Srcs, bool IsSigned,
17811 bool IsAny) {
17812
17813 // If we just have one source, just permute it accordingly.
17814 if (Srcs.size() == 1) {
17815 auto *Elt = Srcs.begin();
17816 auto EltOp = getDWordFromOffset(DAG, SL, Elt->SrcOp, Elt->DWordOffset);
17817
17818 // v_perm will produce the original value
17819 if (Elt->PermMask == 0x3020100)
17820 return EltOp;
17821
17822 return DAG.getNode(AMDGPUISD::PERM, SL, MVT::i32, EltOp, EltOp,
17823 DAG.getConstant(Elt->PermMask, SL, MVT::i32));
17824 }
17825
17826 auto *FirstElt = Srcs.begin();
17827 auto *SecondElt = std::next(FirstElt);
17828
17830
17831 // If we have multiple sources in the chain, combine them via perms (using
17832 // calculated perm mask) and Ors.
17833 while (true) {
17834 auto FirstMask = FirstElt->PermMask;
17835 auto SecondMask = SecondElt->PermMask;
17836
17837 unsigned FirstCs = FirstMask & 0x0c0c0c0c;
17838 unsigned FirstPlusFour = FirstMask | 0x04040404;
17839 // 0x0c + 0x04 = 0x10, so anding with 0x0F will produced 0x00 for any
17840 // original 0x0C.
17841 FirstMask = (FirstPlusFour & 0x0F0F0F0F) | FirstCs;
17842
17843 auto PermMask = addPermMasks(FirstMask, SecondMask);
17844 auto FirstVal =
17845 getDWordFromOffset(DAG, SL, FirstElt->SrcOp, FirstElt->DWordOffset);
17846 auto SecondVal =
17847 getDWordFromOffset(DAG, SL, SecondElt->SrcOp, SecondElt->DWordOffset);
17848
17849 Perms.push_back(DAG.getNode(AMDGPUISD::PERM, SL, MVT::i32, FirstVal,
17850 SecondVal,
17851 DAG.getConstant(PermMask, SL, MVT::i32)));
17852
17853 FirstElt = std::next(SecondElt);
17854 if (FirstElt == Srcs.end())
17855 break;
17856
17857 SecondElt = std::next(FirstElt);
17858 // If we only have a FirstElt, then just combine that into the cumulative
17859 // source node.
17860 if (SecondElt == Srcs.end()) {
17861 auto EltOp =
17862 getDWordFromOffset(DAG, SL, FirstElt->SrcOp, FirstElt->DWordOffset);
17863
17864 Perms.push_back(
17865 DAG.getNode(AMDGPUISD::PERM, SL, MVT::i32, EltOp, EltOp,
17866 DAG.getConstant(FirstElt->PermMask, SL, MVT::i32)));
17867 break;
17868 }
17869 }
17870
17871 assert(Perms.size() == 1 || Perms.size() == 2);
17872 return Perms.size() == 2
17873 ? DAG.getNode(ISD::OR, SL, MVT::i32, Perms[0], Perms[1])
17874 : Perms[0];
17875}
17876
17877static void fixMasks(SmallVectorImpl<DotSrc> &Srcs, unsigned ChainLength) {
17878 for (auto &[EntryVal, EntryMask, EntryOffset] : Srcs) {
17879 EntryMask = EntryMask >> ((4 - ChainLength) * 8);
17880 auto ZeroMask = ChainLength == 2 ? 0x0c0c0000 : 0x0c000000;
17881 EntryMask += ZeroMask;
17882 }
17883}
17884
17885static bool isMul(const SDValue Op) {
17886 auto Opcode = Op.getOpcode();
17887
17888 return (Opcode == ISD::MUL || Opcode == AMDGPUISD::MUL_U24 ||
17889 Opcode == AMDGPUISD::MUL_I24);
17890}
17891
17892static std::optional<bool>
17894 ByteProvider<SDValue> &Src1, const SDValue &S0Op,
17895 const SDValue &S1Op, const SelectionDAG &DAG) {
17896 // If we both ops are i8s (pre legalize-dag), then the signedness semantics
17897 // of the dot4 is irrelevant.
17898 if (S0Op.getValueSizeInBits() == 8 && S1Op.getValueSizeInBits() == 8)
17899 return false;
17900
17901 auto Known0 = DAG.computeKnownBits(S0Op, 0);
17902 bool S0IsUnsigned = Known0.countMinLeadingZeros() > 0;
17903 bool S0IsSigned = Known0.countMinLeadingOnes() > 0;
17904 auto Known1 = DAG.computeKnownBits(S1Op, 0);
17905 bool S1IsUnsigned = Known1.countMinLeadingZeros() > 0;
17906 bool S1IsSigned = Known1.countMinLeadingOnes() > 0;
17907
17908 assert(!(S0IsUnsigned && S0IsSigned));
17909 assert(!(S1IsUnsigned && S1IsSigned));
17910
17911 // There are 9 possible permutations of
17912 // {S0IsUnsigned, S0IsSigned, S1IsUnsigned, S1IsSigned}
17913
17914 // In two permutations, the sign bits are known to be the same for both Ops,
17915 // so simply return Signed / Unsigned corresponding to the MSB
17916
17917 if ((S0IsUnsigned && S1IsUnsigned) || (S0IsSigned && S1IsSigned))
17918 return S0IsSigned;
17919
17920 // In another two permutations, the sign bits are known to be opposite. In
17921 // this case return std::nullopt to indicate a bad match.
17922
17923 if ((S0IsUnsigned && S1IsSigned) || (S0IsSigned && S1IsUnsigned))
17924 return std::nullopt;
17925
17926 // In the remaining five permutations, we don't know the value of the sign
17927 // bit for at least one Op. Since we have a valid ByteProvider, we know that
17928 // the upper bits must be extension bits. Thus, the only ways for the sign
17929 // bit to be unknown is if it was sign extended from unknown value, or if it
17930 // was any extended. In either case, it is correct to use the signed
17931 // version of the signedness semantics of dot4
17932
17933 // In two of such permutations, we known the sign bit is set for
17934 // one op, and the other is unknown. It is okay to used signed version of
17935 // dot4.
17936 if ((S0IsSigned && !(S1IsSigned || S1IsUnsigned)) ||
17937 ((S1IsSigned && !(S0IsSigned || S0IsUnsigned))))
17938 return true;
17939
17940 // In one such permutation, we don't know either of the sign bits. It is okay
17941 // to used the signed version of dot4.
17942 if ((!(S1IsSigned || S1IsUnsigned) && !(S0IsSigned || S0IsUnsigned)))
17943 return true;
17944
17945 // In two of such permutations, we known the sign bit is unset for
17946 // one op, and the other is unknown. Return std::nullopt to indicate a
17947 // bad match.
17948 if ((S0IsUnsigned && !(S1IsSigned || S1IsUnsigned)) ||
17949 ((S1IsUnsigned && !(S0IsSigned || S0IsUnsigned))))
17950 return std::nullopt;
17951
17952 llvm_unreachable("Fully covered condition");
17953}
17954
17955SDValue SITargetLowering::performAddCombine(SDNode *N,
17956 DAGCombinerInfo &DCI) const {
17957 SelectionDAG &DAG = DCI.DAG;
17958 EVT VT = N->getValueType(0);
17959 SDLoc SL(N);
17960 SDValue LHS = N->getOperand(0);
17961 SDValue RHS = N->getOperand(1);
17962
17963 if (LHS.getOpcode() == ISD::MUL || RHS.getOpcode() == ISD::MUL) {
17964 if (Subtarget->hasMad64_32()) {
17965 if (SDValue Folded = tryFoldToMad64_32(N, DCI))
17966 return Folded;
17967 }
17968 }
17969
17970 if (SDValue V = reassociateScalarOps(N, DAG)) {
17971 return V;
17972 }
17973
17974 if (VT == MVT::i64) {
17975 if (SDValue Folded = foldAddSub64WithZeroLowBitsTo32(N, DCI))
17976 return Folded;
17977 }
17978
17979 // dot4 produces a 32-bit result, so a wider VT can't be folded.
17980 if (!VT.isVector() && VT.getSizeInBits() <= 32 &&
17981 (isMul(LHS) || isMul(RHS)) && Subtarget->hasDot7Insts() &&
17982 (Subtarget->hasDot1Insts() || Subtarget->hasDot8Insts())) {
17983 SDValue TempNode(N, 0);
17984 std::optional<bool> IsSigned;
17988
17989 // Match the v_dot4 tree, while collecting src nodes.
17990 int ChainLength = 0;
17991 for (int I = 0; I < 4; I++) {
17992 auto MulIdx = isMul(LHS) ? 0 : isMul(RHS) ? 1 : -1;
17993 if (MulIdx == -1)
17994 break;
17995 auto Src0 = handleMulOperand(TempNode->getOperand(MulIdx)->getOperand(0));
17996 if (!Src0)
17997 break;
17998 auto Src1 = handleMulOperand(TempNode->getOperand(MulIdx)->getOperand(1));
17999 if (!Src1)
18000 break;
18001
18002 auto IterIsSigned = checkDot4MulSignedness(
18003 TempNode->getOperand(MulIdx), *Src0, *Src1,
18004 TempNode->getOperand(MulIdx)->getOperand(0),
18005 TempNode->getOperand(MulIdx)->getOperand(1), DAG);
18006 if (!IterIsSigned)
18007 break;
18008 if (!IsSigned)
18009 IsSigned = *IterIsSigned;
18010 if (*IterIsSigned != *IsSigned)
18011 break;
18012 placeSources(*Src0, *Src1, Src0s, Src1s, I);
18013 auto AddIdx = 1 - MulIdx;
18014 // Allow the special case where add (add (mul24, 0), mul24) became ->
18015 // add (mul24, mul24).
18016 if (I == 2 && isMul(TempNode->getOperand(AddIdx))) {
18017 Src2s.push_back(TempNode->getOperand(AddIdx));
18018 auto Src0 =
18019 handleMulOperand(TempNode->getOperand(AddIdx)->getOperand(0));
18020 if (!Src0)
18021 break;
18022 auto Src1 =
18023 handleMulOperand(TempNode->getOperand(AddIdx)->getOperand(1));
18024 if (!Src1)
18025 break;
18026 auto IterIsSigned = checkDot4MulSignedness(
18027 TempNode->getOperand(AddIdx), *Src0, *Src1,
18028 TempNode->getOperand(AddIdx)->getOperand(0),
18029 TempNode->getOperand(AddIdx)->getOperand(1), DAG);
18030 if (!IterIsSigned)
18031 break;
18032 assert(IsSigned);
18033 if (*IterIsSigned != *IsSigned)
18034 break;
18035 placeSources(*Src0, *Src1, Src0s, Src1s, I + 1);
18036 Src2s.push_back(DAG.getConstant(0, SL, MVT::i32));
18037 ChainLength = I + 2;
18038 break;
18039 }
18040
18041 TempNode = TempNode->getOperand(AddIdx);
18042 Src2s.push_back(TempNode);
18043 ChainLength = I + 1;
18044 // The loop body treats TempNode's operands as addends.
18045 if (TempNode.getOpcode() != ISD::ADD)
18046 break;
18047 LHS = TempNode->getOperand(0);
18048 RHS = TempNode->getOperand(1);
18049 }
18050
18051 if (ChainLength < 2)
18052 return SDValue();
18053
18054 // Masks were constructed with assumption that we would find a chain of
18055 // length 4. If not, then we need to 0 out the MSB bits (via perm mask of
18056 // 0x0c) so they do not affect dot calculation.
18057 if (ChainLength < 4) {
18058 fixMasks(Src0s, ChainLength);
18059 fixMasks(Src1s, ChainLength);
18060 }
18061
18062 SDValue Src0, Src1;
18063
18064 // If we are just using a single source for both, and have permuted the
18065 // bytes consistently, we can just use the sources without permuting
18066 // (commutation).
18067 bool UseOriginalSrc = false;
18068 if (ChainLength == 4 && Src0s.size() == 1 && Src1s.size() == 1 &&
18069 Src0s.begin()->PermMask == Src1s.begin()->PermMask &&
18070 Src0s.begin()->SrcOp.getValueSizeInBits() >= 32 &&
18071 Src1s.begin()->SrcOp.getValueSizeInBits() >= 32) {
18072 SmallVector<unsigned, 4> SrcBytes;
18073 auto Src0Mask = Src0s.begin()->PermMask;
18074 SrcBytes.push_back((Src0Mask >> 24) & 0xFF);
18075 bool UniqueEntries = true;
18076 for (auto I = 1; I < 4; I++) {
18077 auto NextByte = (Src0Mask >> ((3 - I) * 8)) & 0xFF;
18078
18079 if (is_contained(SrcBytes, NextByte)) {
18080 UniqueEntries = false;
18081 break;
18082 }
18083 SrcBytes.push_back(NextByte);
18084 }
18085
18086 if (UniqueEntries) {
18087 UseOriginalSrc = true;
18088
18089 auto *FirstElt = Src0s.begin();
18090 auto FirstEltOp =
18091 getDWordFromOffset(DAG, SL, FirstElt->SrcOp, FirstElt->DWordOffset);
18092
18093 auto *SecondElt = Src1s.begin();
18094 auto SecondEltOp = getDWordFromOffset(DAG, SL, SecondElt->SrcOp,
18095 SecondElt->DWordOffset);
18096
18097 Src0 = DAG.getBitcastedAnyExtOrTrunc(FirstEltOp, SL,
18098 MVT::getIntegerVT(32));
18099 Src1 = DAG.getBitcastedAnyExtOrTrunc(SecondEltOp, SL,
18100 MVT::getIntegerVT(32));
18101 }
18102 }
18103
18104 if (!UseOriginalSrc) {
18105 Src0 = resolveSources(DAG, SL, Src0s, false, true);
18106 Src1 = resolveSources(DAG, SL, Src1s, false, true);
18107 }
18108
18109 assert(IsSigned);
18110 SDValue Src2 =
18111 DAG.getExtOrTrunc(*IsSigned, Src2s[ChainLength - 1], SL, MVT::i32);
18112
18113 SDValue IID = DAG.getTargetConstant(*IsSigned ? Intrinsic::amdgcn_sdot4
18114 : Intrinsic::amdgcn_udot4,
18115 SL, MVT::i64);
18116
18117 assert(!VT.isVector());
18118 auto Dot = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32, IID, Src0,
18119 Src1, Src2, DAG.getTargetConstant(0, SL, MVT::i1));
18120
18121 return DAG.getExtOrTrunc(*IsSigned, Dot, SL, VT);
18122 }
18123
18124 if (VT != MVT::i32 || !DCI.isAfterLegalizeDAG())
18125 return SDValue();
18126
18127 // add x, zext (setcc) => uaddo_carry x, 0, setcc
18128 // add x, sext (setcc) => usubo_carry x, 0, setcc
18129 unsigned Opc = LHS.getOpcode();
18132 std::swap(RHS, LHS);
18133
18134 Opc = RHS.getOpcode();
18135 switch (Opc) {
18136 default:
18137 break;
18138 case ISD::ZERO_EXTEND:
18139 case ISD::SIGN_EXTEND:
18140 case ISD::ANY_EXTEND: {
18141 auto Cond = RHS.getOperand(0);
18142 // If this won't be a real VOPC output, we would still need to insert an
18143 // extra instruction anyway.
18144 if (!isBoolSGPR(Cond))
18145 break;
18146 SDVTList VTList = DAG.getVTList(MVT::i32, MVT::i1);
18147 SDValue Args[] = {LHS, DAG.getConstant(0, SL, MVT::i32), Cond};
18149 return DAG.getNode(Opc, SL, VTList, Args);
18150 }
18151 case ISD::UADDO_CARRY: {
18152 // add x, (uaddo_carry y, 0, cc) => uaddo_carry x, y, cc
18153 if (!isNullConstant(RHS.getOperand(1)))
18154 break;
18155 SDValue Args[] = {LHS, RHS.getOperand(0), RHS.getOperand(2)};
18156 return DAG.getNode(ISD::UADDO_CARRY, SDLoc(N), RHS->getVTList(), Args);
18157 }
18158 }
18159 return SDValue();
18160}
18161
18162SDValue SITargetLowering::performPtrAddCombine(SDNode *N,
18163 DAGCombinerInfo &DCI) const {
18164 SelectionDAG &DAG = DCI.DAG;
18165 SDLoc DL(N);
18166 EVT VT = N->getValueType(0);
18167 SDValue N0 = N->getOperand(0);
18168 SDValue N1 = N->getOperand(1);
18169
18170 // The following folds transform PTRADDs into regular arithmetic in cases
18171 // where the PTRADD wouldn't be folded as an immediate offset into memory
18172 // instructions anyway. They are target-specific in that other targets might
18173 // prefer to not lose information about the pointer arithmetic.
18174
18175 // Fold (ptradd x, shl(0 - v, k)) -> sub(x, shl(v, k)).
18176 // Adapted from DAGCombiner::visitADDLikeCommutative.
18177 SDValue V, K;
18178 if (sd_match(N1, m_Shl(m_Neg(m_Value(V)), m_Value(K)))) {
18179 SDNodeFlags ShlFlags = N1->getFlags();
18180 // If the original shl is NUW and NSW, the first k+1 bits of 0-v are all 0,
18181 // so v is either 0 or the first k+1 bits of v are all 1 -> NSW can be
18182 // preserved.
18183 SDNodeFlags NewShlFlags =
18184 ShlFlags.hasNoUnsignedWrap() && ShlFlags.hasNoSignedWrap()
18186 : SDNodeFlags();
18187 SDValue Inner = DAG.getNode(ISD::SHL, DL, VT, V, K, NewShlFlags);
18188 DCI.AddToWorklist(Inner.getNode());
18189 return DAG.getNode(ISD::SUB, DL, VT, N0, Inner);
18190 }
18191
18192 // Fold into Mad64 if the right-hand side is a MUL. Analogous to a fold in
18193 // performAddCombine.
18194 if (N1.getOpcode() == ISD::MUL) {
18195 if (Subtarget->hasMad64_32()) {
18196 if (SDValue Folded = tryFoldToMad64_32(N, DCI))
18197 return Folded;
18198 }
18199 }
18200
18201 // If the 32 low bits of the constant are all zero, there is nothing to fold
18202 // into an immediate offset, so it's better to eliminate the unnecessary
18203 // addition for the lower 32 bits than to preserve the PTRADD.
18204 // Analogous to a fold in performAddCombine.
18205 if (VT == MVT::i64) {
18206 if (SDValue Folded = foldAddSub64WithZeroLowBitsTo32(N, DCI))
18207 return Folded;
18208 }
18209
18210 if (N1.getOpcode() != ISD::ADD || !N1.hasOneUse())
18211 return SDValue();
18212
18213 SDValue X = N0;
18214 SDValue Y = N1.getOperand(0);
18215 SDValue Z = N1.getOperand(1);
18216 bool YIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(Y);
18217 bool ZIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(Z);
18218
18219 if (!YIsConstant && !ZIsConstant && !X->isDivergent() &&
18220 Y->isDivergent() != Z->isDivergent()) {
18221 // Reassociate (ptradd x, (add y, z)) -> (ptradd (ptradd x, y), z) if x and
18222 // y are uniform and z isn't.
18223 // Reassociate (ptradd x, (add y, z)) -> (ptradd (ptradd x, z), y) if x and
18224 // z are uniform and y isn't.
18225 // The goal is to push uniform operands up in the computation, so that they
18226 // can be handled with scalar operations. We can't use reassociateScalarOps
18227 // for this since it requires two identical commutative operations to
18228 // reassociate.
18229 if (Y->isDivergent())
18230 std::swap(Y, Z);
18231 // If both additions in the original were NUW, reassociation preserves that.
18232 SDNodeFlags ReassocFlags =
18233 (N->getFlags() & N1->getFlags()) & SDNodeFlags::NoUnsignedWrap;
18234 SDValue UniformInner = DAG.getMemBasePlusOffset(X, Y, DL, ReassocFlags);
18235 DCI.AddToWorklist(UniformInner.getNode());
18236 return DAG.getMemBasePlusOffset(UniformInner, Z, DL, ReassocFlags);
18237 }
18238
18239 return SDValue();
18240}
18241
18242static bool isCtlzOpc(unsigned Opc) {
18243 return Opc == ISD::CTLZ || Opc == ISD::CTLZ_ZERO_POISON;
18244}
18245
18246SDValue SITargetLowering::performSubCombine(SDNode *N,
18247 DAGCombinerInfo &DCI) const {
18248 SelectionDAG &DAG = DCI.DAG;
18249 EVT VT = N->getValueType(0);
18250
18251 if (VT == MVT::i64) {
18252 if (SDValue Folded = foldAddSub64WithZeroLowBitsTo32(N, DCI))
18253 return Folded;
18254 }
18255
18256 if (VT != MVT::i32)
18257 return SDValue();
18258
18259 SDLoc SL(N);
18260 SDValue LHS = N->getOperand(0);
18261 SDValue RHS = N->getOperand(1);
18262
18263 // sub x, zext (setcc) => usubo_carry x, 0, setcc
18264 // sub x, sext (setcc) => uaddo_carry x, 0, setcc
18265 unsigned Opc = RHS.getOpcode();
18266 switch (Opc) {
18267 default:
18268 break;
18269 case ISD::ZERO_EXTEND:
18270 case ISD::SIGN_EXTEND:
18271 case ISD::ANY_EXTEND: {
18272 auto Cond = RHS.getOperand(0);
18273 // If this won't be a real VOPC output, we would still need to insert an
18274 // extra instruction anyway.
18275 if (!isBoolSGPR(Cond))
18276 break;
18277 SDVTList VTList = DAG.getVTList(MVT::i32, MVT::i1);
18278 SDValue Args[] = {LHS, DAG.getConstant(0, SL, MVT::i32), Cond};
18280 return DAG.getNode(Opc, SL, VTList, Args);
18281 }
18282 }
18283
18284 if (LHS.getOpcode() == ISD::USUBO_CARRY) {
18285 // sub (usubo_carry x, 0, cc), y => usubo_carry x, y, cc
18286 if (!isNullConstant(LHS.getOperand(1)))
18287 return SDValue();
18288 SDValue Args[] = {LHS.getOperand(0), RHS, LHS.getOperand(2)};
18289 return DAG.getNode(ISD::USUBO_CARRY, SDLoc(N), LHS->getVTList(), Args);
18290 }
18291
18292 // sub (ctlz (xor x, (sra x, 31))), 1 -> ctls x.
18293 if (isOneConstant(RHS) && isCtlzOpc(LHS.getOpcode())) {
18294 SDValue CtlzSrc = LHS.getOperand(0);
18295 // Check for xor x, (sra x, 31) pattern.
18296 if (CtlzSrc.getOpcode() == ISD::XOR) {
18297 SDValue X = CtlzSrc.getOperand(0);
18298 SDValue SignExt = CtlzSrc.getOperand(1);
18299 // Try both ordering of XOR operands.
18300 if (SignExt.getOpcode() != ISD::SRA)
18301 std::swap(X, SignExt);
18302 if (SignExt.getOpcode() == ISD::SRA && SignExt.getOperand(0) == X) {
18303 ConstantSDNode *ShiftAmt =
18305 unsigned BitWidth = X.getValueType().getScalarSizeInBits();
18306 if (ShiftAmt && ShiftAmt->getZExtValue() == BitWidth - 1)
18307 return DAG.getNode(ISD::CTLS, SL, VT, X);
18308 }
18309 }
18310 }
18311
18312 return SDValue();
18313}
18314
18315SDValue SITargetLowering::performFAddCombine(SDNode *N,
18316 DAGCombinerInfo &DCI) const {
18317 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
18318 return SDValue();
18319
18320 SelectionDAG &DAG = DCI.DAG;
18321 EVT VT = N->getValueType(0);
18322
18323 SDLoc SL(N);
18324 SDValue LHS = N->getOperand(0);
18325 SDValue RHS = N->getOperand(1);
18326
18327 // These should really be instruction patterns, but writing patterns with
18328 // source modifiers is a pain.
18329
18330 // fadd (fadd (a, a), b) -> mad 2.0, a, b
18331 if (LHS.getOpcode() == ISD::FADD) {
18332 SDValue A = LHS.getOperand(0);
18333 if (A == LHS.getOperand(1)) {
18334 unsigned FusedOp = getFusedOpcode(DAG, N, LHS.getNode());
18335 if (FusedOp != 0) {
18336 const SDValue Two = DAG.getConstantFP(2.0, SL, VT);
18337 return DAG.getNode(FusedOp, SL, VT, A, Two, RHS);
18338 }
18339 }
18340 }
18341
18342 // fadd (b, fadd (a, a)) -> mad 2.0, a, b
18343 if (RHS.getOpcode() == ISD::FADD) {
18344 SDValue A = RHS.getOperand(0);
18345 if (A == RHS.getOperand(1)) {
18346 unsigned FusedOp = getFusedOpcode(DAG, N, RHS.getNode());
18347 if (FusedOp != 0) {
18348 const SDValue Two = DAG.getConstantFP(2.0, SL, VT);
18349 return DAG.getNode(FusedOp, SL, VT, A, Two, LHS);
18350 }
18351 }
18352 }
18353
18354 return SDValue();
18355}
18356
18357SDValue SITargetLowering::performFSubCombine(SDNode *N,
18358 DAGCombinerInfo &DCI) const {
18359 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
18360 return SDValue();
18361
18362 SelectionDAG &DAG = DCI.DAG;
18363 SDLoc SL(N);
18364 EVT VT = N->getValueType(0);
18365 assert(!VT.isVector());
18366
18367 // Try to get the fneg to fold into the source modifier. This undoes generic
18368 // DAG combines and folds them into the mad.
18369 //
18370 // Only do this if we are not trying to support denormals. v_mad_f32 does
18371 // not support denormals ever.
18372 SDValue LHS = N->getOperand(0);
18373 SDValue RHS = N->getOperand(1);
18374 if (LHS.getOpcode() == ISD::FADD) {
18375 // (fsub (fadd a, a), c) -> mad 2.0, a, (fneg c)
18376 SDValue A = LHS.getOperand(0);
18377 if (A == LHS.getOperand(1)) {
18378 unsigned FusedOp = getFusedOpcode(DAG, N, LHS.getNode());
18379 if (FusedOp != 0) {
18380 const SDValue Two = DAG.getConstantFP(2.0, SL, VT);
18381 SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
18382
18383 return DAG.getNode(FusedOp, SL, VT, A, Two, NegRHS);
18384 }
18385 }
18386 }
18387
18388 if (RHS.getOpcode() == ISD::FADD) {
18389 // (fsub c, (fadd a, a)) -> mad -2.0, a, c
18390
18391 SDValue A = RHS.getOperand(0);
18392 if (A == RHS.getOperand(1)) {
18393 unsigned FusedOp = getFusedOpcode(DAG, N, RHS.getNode());
18394 if (FusedOp != 0) {
18395 const SDValue NegTwo = DAG.getConstantFP(-2.0, SL, VT);
18396 return DAG.getNode(FusedOp, SL, VT, A, NegTwo, LHS);
18397 }
18398 }
18399 }
18400
18401 return SDValue();
18402}
18403
18404SDValue SITargetLowering::performFDivCombine(SDNode *N,
18405 DAGCombinerInfo &DCI) const {
18406 SelectionDAG &DAG = DCI.DAG;
18407 SDLoc SL(N);
18408 EVT VT = N->getValueType(0);
18409
18410 if (VT != MVT::f16 && VT != MVT::bf16)
18411 return SDValue();
18412
18413 SDValue LHS = N->getOperand(0);
18414 SDValue RHS = N->getOperand(1);
18415
18416 SDNodeFlags Flags = N->getFlags();
18417 SDNodeFlags RHSFlags = RHS->getFlags();
18418 if (!Flags.hasAllowContract() || !RHSFlags.hasAllowContract() ||
18419 !RHS->hasOneUse())
18420 return SDValue();
18421
18422 if (const ConstantFPSDNode *CLHS = dyn_cast<ConstantFPSDNode>(LHS)) {
18423 bool IsNegative = false;
18424 if (CLHS->isOne() || (IsNegative = CLHS->isMinusOne())) {
18425 // fdiv contract 1.0, (sqrt contract x) -> rsq
18426 // fdiv contract -1.0, (sqrt contract x) -> fneg(rsq)
18427 if (RHS.getOpcode() == ISD::FSQRT) {
18428 // TODO: Or in RHS flags, somehow missing from SDNodeFlags
18429 SDValue SqrtOp = RHS.getOperand(0);
18430 SDValue Rsq;
18431 if (isOperationLegal(ISD::FSQRT, VT)) {
18432 // fsqrt legality correlates to rsq availability of the same type.
18433 Rsq = DAG.getNode(AMDGPUISD::RSQ, SL, VT, SqrtOp, Flags);
18434 } else if (VT == MVT::f16) {
18435 // Targets without 16-bit instructions (gfx6/gfx7) have no f16 rsq,
18436 // but v_rsq_f32 is more than accurate enough for f16. Unlike bf16,
18437 // every f16 value (including denormals) extends to a normal f32, and
18438 // an f16 rsq result is never denormal, so the f32 reciprocal square
18439 // root needs no denormal handling. Compute it in f32 and round back.
18440 SDValue Ext =
18441 DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, SqrtOp, Flags);
18442 SDValue F32Rsq =
18443 DAG.getNode(AMDGPUISD::RSQ, SL, MVT::f32, Ext, Flags);
18444 Rsq = DAG.getNode(ISD::FP_ROUND, SL, VT, F32Rsq,
18445 DAG.getTargetConstant(0, SL, MVT::i32), Flags);
18446 } else {
18447 // bf16 shares f32's exponent range, so bf16 denormals would extend to
18448 // f32 denormals that v_rsq_f32 does not handle. Leave it expanded.
18449 return SDValue();
18450 }
18451 return IsNegative ? DAG.getNode(ISD::FNEG, SL, VT, Rsq, Flags) : Rsq;
18452 }
18453 }
18454 }
18455
18456 return SDValue();
18457}
18458
18459SDValue SITargetLowering::performFMulCombine(SDNode *N,
18460 DAGCombinerInfo &DCI) const {
18461 SelectionDAG &DAG = DCI.DAG;
18462 EVT VT = N->getValueType(0);
18463 EVT ScalarVT = VT.getScalarType();
18464 EVT IntVT = VT.changeElementType(*DAG.getContext(), MVT::i32);
18465
18466 if (!N->isDivergent() && getSubtarget()->hasSALUFloatInsts() &&
18467 (ScalarVT == MVT::f32 || ScalarVT == MVT::f16)) {
18468 // Prefer to use s_mul_f16/f32 instead of v_ldexp_f16/f32.
18469 return SDValue();
18470 }
18471
18472 SDValue LHS = N->getOperand(0);
18473 SDValue RHS = N->getOperand(1);
18474
18475 // It is cheaper to realize i32 inline constants as compared against
18476 // materializing f16 or f64 (or even non-inline f32) values,
18477 // possible via ldexp usage, as shown below :
18478 //
18479 // Given : A = 2^a & B = 2^b ; where a and b are integers.
18480 // fmul x, (select y, A, B) -> ldexp( x, (select i32 y, a, b) )
18481 // fmul x, (select y, -A, -B) -> ldexp( (fneg x), (select i32 y, a, b) )
18482 if ((ScalarVT == MVT::f64 || ScalarVT == MVT::f32 || ScalarVT == MVT::f16) &&
18483 (RHS.hasOneUse() && RHS.getOpcode() == ISD::SELECT)) {
18484 const ConstantFPSDNode *TrueNode = isConstOrConstSplatFP(RHS.getOperand(1));
18485 if (!TrueNode)
18486 return SDValue();
18487 const ConstantFPSDNode *FalseNode =
18488 isConstOrConstSplatFP(RHS.getOperand(2));
18489 if (!FalseNode)
18490 return SDValue();
18491
18492 if (TrueNode->isNegative() != FalseNode->isNegative())
18493 return SDValue();
18494
18495 // For f32, only non-inline constants should be transformed.
18496 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
18497 if (ScalarVT == MVT::f32 &&
18498 TII->isInlineConstant(TrueNode->getValueAPF()) &&
18499 TII->isInlineConstant(FalseNode->getValueAPF()))
18500 return SDValue();
18501
18502 int TrueNodeExpVal = TrueNode->getValueAPF().getExactLog2Abs();
18503 if (TrueNodeExpVal == INT_MIN)
18504 return SDValue();
18505 int FalseNodeExpVal = FalseNode->getValueAPF().getExactLog2Abs();
18506 if (FalseNodeExpVal == INT_MIN)
18507 return SDValue();
18508
18509 SDLoc SL(N);
18510 SDValue SelectNode =
18511 DAG.getNode(ISD::SELECT, SL, IntVT, RHS.getOperand(0),
18512 DAG.getSignedConstant(TrueNodeExpVal, SL, IntVT),
18513 DAG.getSignedConstant(FalseNodeExpVal, SL, IntVT));
18514
18515 LHS = TrueNode->isNegative()
18516 ? DAG.getNode(ISD::FNEG, SL, VT, LHS, LHS->getFlags())
18517 : LHS;
18518
18519 return DAG.getNode(ISD::FLDEXP, SL, VT, LHS, SelectNode, N->getFlags());
18520 }
18521
18522 return SDValue();
18523}
18524
18525SDValue SITargetLowering::performFMACombine(SDNode *N,
18526 DAGCombinerInfo &DCI) const {
18527 SelectionDAG &DAG = DCI.DAG;
18528 EVT VT = N->getValueType(0);
18529 SDLoc SL(N);
18530
18531 if (!Subtarget->hasDot10Insts() || VT != MVT::f32)
18532 return SDValue();
18533
18534 // FMA((F32)S0.x, (F32)S1. x, FMA((F32)S0.y, (F32)S1.y, (F32)z)) ->
18535 // FDOT2((V2F16)S0, (V2F16)S1, (F32)z))
18536 SDValue Op1 = N->getOperand(0);
18537 SDValue Op2 = N->getOperand(1);
18538 SDValue FMA = N->getOperand(2);
18539
18540 if (FMA.getOpcode() != ISD::FMA || Op1.getOpcode() != ISD::FP_EXTEND ||
18541 Op2.getOpcode() != ISD::FP_EXTEND)
18542 return SDValue();
18543
18544 // The fdot2 fold (fma_mix -> dot2) is only safe when both instructions agree
18545 // on how f16 subnormal inputs are handled. However, if both FMAs carry afn
18546 // the caller accepts approximate results, so any subnormal flushing
18547 // introduced by dot2 is acceptable regardless of mode.
18548 //
18549 // gfx90a (CDNA2) is the sole exception (dot2UnconditionalFlush): v_dot2c
18550 // unconditionally flushes f16 subnormal inputs to zero regardless of MODE,
18551 // while v_fma_mix_f32 preserves them when ieee=1 (the default compute kernel
18552 // mode). The fold is safe only when f32 denorm = PreserveSign, which implies
18553 // ieee=0 so both flush.
18554 //
18555 // All other GPUs: v_dot2 does NOT flush f16 subnormal inputs. v_fma_mix_f32
18556 // flushes them only when f32 denorm = PreserveSign. The fold is safe only
18557 // when f32 denorm is IEEE (both preserve the subnormal). Dynamic mode is
18558 // also rejected since the runtime value is unknown.
18559 bool AllowInaccuracy = N->getFlags().hasApproximateFuncs() &&
18560 FMA->getFlags().hasApproximateFuncs();
18561 if (!AllowInaccuracy) {
18562 const MachineFunction &MF = DAG.getMachineFunction();
18563 DenormalMode Mode = MF.getDenormalMode(APFloat::IEEEsingle());
18564 if (Subtarget->dot2UnconditionalFlush()) {
18565 // gfx90a: fold safe only when f32 denorm flushes.
18567 return SDValue();
18568 } else {
18569 // All other GPUs: fold safe only when f32 denorm is IEEE.
18570 if (Mode != DenormalMode::getIEEE())
18571 return SDValue();
18572 }
18573 }
18574
18575 // fp-contract allows reassociating the fma tree into a dot product.
18576 if (N->getFlags().hasAllowContract() && FMA->getFlags().hasAllowContract()) {
18577 Op1 = Op1.getOperand(0);
18578 Op2 = Op2.getOperand(0);
18579 if (Op1.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
18581 return SDValue();
18582
18583 SDValue Vec1 = Op1.getOperand(0);
18584 SDValue Idx1 = Op1.getOperand(1);
18585 SDValue Vec2 = Op2.getOperand(0);
18586
18587 SDValue FMAOp1 = FMA.getOperand(0);
18588 SDValue FMAOp2 = FMA.getOperand(1);
18589 SDValue FMAAcc = FMA.getOperand(2);
18590
18591 if (FMAOp1.getOpcode() != ISD::FP_EXTEND ||
18592 FMAOp2.getOpcode() != ISD::FP_EXTEND)
18593 return SDValue();
18594
18595 FMAOp1 = FMAOp1.getOperand(0);
18596 FMAOp2 = FMAOp2.getOperand(0);
18597 if (FMAOp1.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
18599 return SDValue();
18600
18601 SDValue Vec3 = FMAOp1.getOperand(0);
18602 SDValue Vec4 = FMAOp2.getOperand(0);
18603 SDValue Idx2 = FMAOp1.getOperand(1);
18604
18605 if (Idx1 != Op2.getOperand(1) || Idx2 != FMAOp2.getOperand(1))
18606 return SDValue();
18607
18608 if (!isa<ConstantSDNode>(Idx1) || !isa<ConstantSDNode>(Idx2) ||
18609 Idx1 == Idx2)
18610 return SDValue();
18611
18612 if (Vec1 == Vec2 || Vec3 == Vec4)
18613 return SDValue();
18614
18615 if (Vec1.getValueType() != MVT::v2f16 || Vec2.getValueType() != MVT::v2f16)
18616 return SDValue();
18617
18618 if ((Vec1 == Vec3 && Vec2 == Vec4) || (Vec1 == Vec4 && Vec2 == Vec3)) {
18619 return DAG.getNode(AMDGPUISD::FDOT2, SL, MVT::f32, Vec1, Vec2, FMAAcc,
18620 DAG.getTargetConstant(0, SL, MVT::i1));
18621 }
18622 }
18623 return SDValue();
18624}
18625
18626// Given a double-precision ordered or unordered comparison, return the
18627// condition code for an equivalent integral comparison of the operands' upper
18628// 32 bits, or `SETCC_INVALID` if not possible.
18629// For simplicity, no simplification occurs if the operands are not both known
18630// to have sign bit zero.
18631//
18632// EQ/NE:
18633// If LHS.lo32 == RHS.lo32:
18634// setcc LHS, RHS, eq/ne => setcc LHS.hi32, RHS.hi32, eq/ne
18635// If LHS.lo32 != RHS.lo32:
18636// setcc LHS, RHS, eq/ne => setcc LHS.hi32, RHS.hi32, false/true
18637// The reduction is not possible if operands may be +0 and -0.
18638// For ordered eq / unordered ne, at most one operand may be NaN.
18639// For unordered eq / ordered ne, neither operand can be NaN.
18640//
18641// LT/GE:
18642// If LHS.lo32 >= RHS.lo32 (unsigned):
18643// setcc LHS, RHS, [u]lt/ge => LHS.hi32, RHS.hi32, [u]lt/ge
18644// If LHS.lo32 < RHS.lo32 (unsigned):
18645// setcc LHS, RHS, [u]lt/ge => LHS.hi32, RHS.hi32, [u]le/gt
18646// The reduction is only supported if both operands are nonnegative.
18647// For ordered lt / unordered ge, the RHS cannot be NaN.
18648// For unordered lt / ordered ge, neither operand can be NaN.
18649//
18650// LE/GT:
18651// If LHS.lo32 > RHS.lo32 (unsigned):
18652// setcc LHS, RHS, [u]le/gt => LHS.hi32, RHS.hi32, [u]lt/ge
18653// If LHS.lo32 <= RHS.lo32 (unsigned):
18654// setcc LHS, RHS, [u]le/gt => LHS.hi32, RHS.hi32, [u]le/gt
18655// The reduction is only supported if both operands are nonnegative.
18656// For unordered le / ordered gt, the LHS cannot be NaN.
18657// For ordered le / unordered gt, neither operand can be NaN.
18659 const SDValue LHS,
18660 const SDValue RHS,
18661 const SelectionDAG &DAG) {
18662 EVT VT = LHS.getValueType();
18663 assert(VT == MVT::f64 && "Incorrect operand type!");
18664
18665 const KnownBits RHSBits = DAG.computeKnownBits(RHS);
18666 // Bail if RHS sign bit is not known to be zero.
18667 if (!RHSBits.Zero.isSignBitSet())
18668 return ISD::SETCC_INVALID;
18669
18670 const KnownBits RHSKnownLo32 = RHSBits.trunc(32);
18671 const KnownFPClass RHSFPClass =
18673 const bool RHSMaybeNaN = !RHSFPClass.isKnownNeverNaN();
18674
18675 const KnownBits LHSBits = DAG.computeKnownBits(LHS);
18676 const KnownBits LHSKnownLo32 = LHSBits.trunc(32);
18677 const KnownFPClass LHSFPClass =
18679 const bool LHSMaybeNaN = !LHSFPClass.isKnownNeverNaN();
18680
18681 // Bail if LHS sign bit is not known to be zero.
18682 if (!LHSBits.Zero.isSignBitSet())
18683 return ISD::SETCC_INVALID;
18684
18685 switch (CC) {
18686 default:
18687 break;
18688 case ISD::SETEQ:
18689 case ISD::SETOEQ:
18690 case ISD::SETUEQ:
18691 case ISD::SETONE:
18692 case ISD::SETUNE: {
18693 // OEQ should be false if either operand is NaN, so it suffices that at
18694 // least one operand is not NaN.
18695 if (CC == ISD::SETOEQ && LHSMaybeNaN && RHSMaybeNaN)
18696 break;
18697 // UEQ should be true if either operand is NaN, but this cannot be checked
18698 // on underlying bits.
18699 if (CC == ISD::SETUEQ && (LHSMaybeNaN || RHSMaybeNaN))
18700 break;
18701 // ONE should be false if either operand is NaN, but this cannot be
18702 // checked on underlying bits.
18703 if (CC == ISD::SETONE && (LHSMaybeNaN || RHSMaybeNaN))
18704 break;
18705 // UNE should be true if either operand is NaN, so it suffices that they
18706 // are not both NaN.
18707 if (CC == ISD::SETUNE && LHSMaybeNaN && RHSMaybeNaN)
18708 break;
18709
18710 const std::optional<bool> KnownEq =
18711 KnownBits::eq(LHSKnownLo32, RHSKnownLo32);
18712
18713 if (!KnownEq)
18714 break;
18715
18716 if (*KnownEq)
18717 return (CC == ISD::SETEQ || CC == ISD::SETOEQ || CC == ISD::SETUEQ)
18718 ? ISD::SETEQ
18719 : ISD::SETNE;
18720
18721 return (CC == ISD::SETEQ || CC == ISD::SETOEQ || CC == ISD::SETUEQ)
18723 : ISD::SETTRUE;
18724 }
18725 case ISD::SETLT:
18726 case ISD::SETOLT:
18727 case ISD::SETULT:
18728 case ISD::SETGE:
18729 case ISD::SETOGE:
18730 case ISD::SETUGE: {
18731 // OLT should be false if either operand is NaN.
18732 // Since NaNs have maximum exponent and nonzero mantissa, false positives
18733 // are only possible if the RHS is NaN. (No issue with RHS == +inf since
18734 // the inequality is strict)
18735 if (CC == ISD::SETOLT && RHSMaybeNaN)
18736 break;
18737 // ULT should be true if either operand is NaN, but this cannot be ensured
18738 // with a truncated comparison.
18739 if (CC == ISD::SETULT && (LHSMaybeNaN || RHSMaybeNaN))
18740 break;
18741 // OGE should be false if either operand is NaN, but this cannot be
18742 // ensured with a truncated comparison.
18743 if (CC == ISD::SETOGE && (LHSMaybeNaN || RHSMaybeNaN))
18744 break;
18745 // UGE should be true if either operand is NaN.
18746 // False negatives are only possible if the RHS is NaN.
18747 // (No issue with RHS == +inf since the inequality is inclusive)
18748 if (CC == ISD::SETUGE && RHSMaybeNaN)
18749 break;
18750
18751 const std::optional<bool> KnownUge =
18752 KnownBits::uge(LHSKnownLo32, RHSKnownLo32);
18753
18754 if (!KnownUge)
18755 break;
18756
18757 if (*KnownUge) {
18758 // LHS.lo32 uge RHS.lo32, so LHS >= RHS iff LHS.hi32 >= RHS.hi32
18759 return (CC == ISD::SETLT || CC == ISD::SETOLT || CC == ISD::SETULT)
18760 ? ISD::SETLT
18761 : ISD::SETGE;
18762 }
18763 // LHS.lo32 ult RHS.lo32, so LHS >= RHS iff LHS.hi32 > RHS.hi32
18764 return (CC == ISD::SETLT || CC == ISD::SETOLT || CC == ISD::SETULT)
18765 ? ISD::SETLE
18766 : ISD::SETGT;
18767 }
18768 case ISD::SETLE:
18769 case ISD::SETOLE:
18770 case ISD::SETULE:
18771 case ISD::SETGT:
18772 case ISD::SETOGT:
18773 case ISD::SETUGT: {
18774 // OLE should be false if either operand is NaN, but this cannot be
18775 // ensured with a truncated comparison.
18776 if (CC == ISD::SETOLE && (LHSMaybeNaN || RHSMaybeNaN))
18777 break;
18778 // ULE should be true if either operand is NaN.
18779 // False negatives are only possible if the LHS is NaN.
18780 // (No issue with LHS == +inf since the inequality is inclusive)
18781 if (CC == ISD::SETULE && LHSMaybeNaN)
18782 break;
18783 // OGT should be false if either operand is NaN.
18784 // False positives are only possible if the LHS is NaN.
18785 // (No issue with LHS == +inf since the inequality is strict)
18786 if (CC == ISD::SETOGT && LHSMaybeNaN)
18787 break;
18788 // UGT should be true if either operand is NaN, but this cannot be ensured
18789 // with a truncated comparison.
18790 if (CC == ISD::SETUGT && (LHSMaybeNaN || RHSMaybeNaN))
18791 break;
18792
18793 const std::optional<bool> KnownUle =
18794 KnownBits::ule(LHSKnownLo32, RHSKnownLo32);
18795
18796 if (!KnownUle)
18797 break;
18798
18799 if (*KnownUle) {
18800 // LHS.lo32 ule RHS.lo32, so LHS <= RHS iff LHS.hi32 <= RHS.hi32
18801 return (CC == ISD::SETLE || CC == ISD::SETOLE || CC == ISD::SETULE)
18802 ? ISD::SETLE
18803 : ISD::SETGT;
18804 }
18805 // LHS.lo32 ugt RHS.lo32, so LHS <= RHS iff LHS.hi32 < RHS.hi32
18806 return (CC == ISD::SETLE || CC == ISD::SETOLE || CC == ISD::SETULE)
18807 ? ISD::SETLT
18808 : ISD::SETGE;
18809 }
18810 }
18811
18812 return ISD::SETCC_INVALID;
18813}
18814
18815SDValue SITargetLowering::performSetCCCombine(SDNode *N,
18816 DAGCombinerInfo &DCI) const {
18817 SelectionDAG &DAG = DCI.DAG;
18818 SDLoc SL(N);
18819
18820 SDValue LHS = N->getOperand(0);
18821 SDValue RHS = N->getOperand(1);
18822 EVT VT = LHS.getValueType();
18823 ISD::CondCode CC = N->getOperandCondCode(2);
18824
18825 auto *CRHS = dyn_cast<ConstantSDNode>(RHS);
18826 if (!CRHS) {
18828 if (CRHS) {
18829 std::swap(LHS, RHS);
18831 }
18832 }
18833
18834 if (CRHS) {
18835 if (VT == MVT::i32 && LHS.getOpcode() == ISD::SIGN_EXTEND &&
18836 isBoolSGPR(LHS.getOperand(0))) {
18837 // setcc (sext from i1 cc), -1, ne|sgt|ult) => not cc => xor cc, -1
18838 // setcc (sext from i1 cc), -1, eq|sle|uge) => cc
18839 // setcc (sext from i1 cc), 0, eq|sge|ule) => not cc => xor cc, -1
18840 // setcc (sext from i1 cc), 0, ne|ugt|slt) => cc
18841 if ((CRHS->isAllOnes() &&
18842 (CC == ISD::SETNE || CC == ISD::SETGT || CC == ISD::SETULT)) ||
18843 (CRHS->isZero() &&
18844 (CC == ISD::SETEQ || CC == ISD::SETGE || CC == ISD::SETULE)))
18845 return DAG.getNode(ISD::XOR, SL, MVT::i1, LHS.getOperand(0),
18846 DAG.getAllOnesConstant(SL, MVT::i1));
18847 if ((CRHS->isAllOnes() &&
18848 (CC == ISD::SETEQ || CC == ISD::SETLE || CC == ISD::SETUGE)) ||
18849 (CRHS->isZero() &&
18850 (CC == ISD::SETNE || CC == ISD::SETUGT || CC == ISD::SETLT)))
18851 return LHS.getOperand(0);
18852 }
18853
18854 const APInt &CRHSVal = CRHS->getAPIntValue();
18855 if ((CC == ISD::SETEQ || CC == ISD::SETNE) &&
18856 LHS.getOpcode() == ISD::SELECT &&
18857 isa<ConstantSDNode>(LHS.getOperand(1)) &&
18858 isa<ConstantSDNode>(LHS.getOperand(2)) &&
18859 isBoolSGPR(LHS.getOperand(0))) {
18860 // Given CT != FT:
18861 // setcc (select cc, CT, CF), CF, eq => xor cc, -1
18862 // setcc (select cc, CT, CF), CF, ne => cc
18863 // setcc (select cc, CT, CF), CT, ne => xor cc, -1
18864 // setcc (select cc, CT, CF), CT, eq => cc
18865 const APInt &CT = LHS.getConstantOperandAPInt(1);
18866 const APInt &CF = LHS.getConstantOperandAPInt(2);
18867
18868 if (CT != CF) {
18869 if ((CF == CRHSVal && CC == ISD::SETEQ) ||
18870 (CT == CRHSVal && CC == ISD::SETNE))
18871 return DAG.getNOT(SL, LHS.getOperand(0), MVT::i1);
18872 if ((CF == CRHSVal && CC == ISD::SETNE) ||
18873 (CT == CRHSVal && CC == ISD::SETEQ))
18874 return LHS.getOperand(0);
18875 }
18876 }
18877 }
18878
18879 // Truncate 64-bit setcc to only test upper or lower 32-bits.
18880 if (VT == MVT::i64) {
18881 const KnownBits LHSKnown = DAG.computeKnownBits(LHS);
18882 const KnownBits RHSKnown = DAG.computeKnownBits(RHS);
18883 const KnownBits LHSKnownLo32 = LHSKnown.trunc(32);
18884 const KnownBits RHSKnownLo32 = RHSKnown.trunc(32);
18885 // Truncate 64-bit setcc to test only upper 32-bits of its operands in the
18886 // following cases where information about the lower 32-bits of its operands
18887 // is known:
18888 //
18889 // If LHS.lo32 == RHS.lo32:
18890 // setcc LHS, RHS, eq/ne => setcc LHS.hi32, RHS.hi32, eq/ne
18891 // If LHS.lo32 != RHS.lo32:
18892 // setcc LHS, RHS, eq/ne => setcc LHS.hi32, RHS.hi32, false/true
18893 // If LHS.lo32 >= RHS.lo32 (unsigned):
18894 // setcc LHS, RHS, [u]lt/ge => LHS.hi32, RHS.hi32, [u]lt/ge
18895 // If LHS.lo32 > RHS.lo32 (unsigned):
18896 // setcc LHS, RHS, [u]le/gt => LHS.hi32, RHS.hi32, [u]lt/ge
18897 // If LHS.lo32 <= RHS.lo32 (unsigned):
18898 // setcc LHS, RHS, [u]le/gt => LHS.hi32, RHS.hi32, [u]le/gt
18899 // If LHS.lo32 < RHS.lo32 (unsigned):
18900 // setcc LHS, RHS, [u]lt/ge => LHS.hi32, RHS.hi32, [u]le/gt
18901
18902 // NewCC is valid iff we can truncate the setcc to only test the upper 32
18903 // bits
18905
18906 switch (CC) {
18907 default:
18908 llvm_unreachable("Unexpected condition code for integer setcc");
18909 break;
18910 case ISD::SETEQ: {
18911 const std::optional<bool> KnownEq =
18912 KnownBits::eq(LHSKnownLo32, RHSKnownLo32);
18913 if (KnownEq)
18914 NewCC = *KnownEq ? ISD::SETEQ : ISD::SETFALSE;
18915
18916 break;
18917 }
18918 case ISD::SETNE: {
18919 const std::optional<bool> KnownEq =
18920 KnownBits::eq(LHSKnownLo32, RHSKnownLo32);
18921 if (KnownEq)
18922 NewCC = *KnownEq ? ISD::SETNE : ISD::SETTRUE;
18923
18924 break;
18925 }
18926 case ISD::SETULT:
18927 case ISD::SETUGE:
18928 case ISD::SETLT:
18929 case ISD::SETGE: {
18930 const std::optional<bool> KnownUge =
18931 KnownBits::uge(LHSKnownLo32, RHSKnownLo32);
18932 if (KnownUge) {
18933 if (*KnownUge) {
18934 // LHS.lo32 uge RHS.lo32, so LHS >= RHS iff LHS.hi32 >= RHS.hi32
18935 NewCC = CC;
18936 } else {
18937 // LHS.lo32 ult RHS.lo32, so LHS >= RHS iff LHS.hi32 > RHS.hi32
18938 NewCC = CC == ISD::SETULT ? ISD::SETULE
18940 : CC == ISD::SETLT ? ISD::SETLE
18941 : ISD::SETGT;
18942 }
18943 }
18944 break;
18945 }
18946 case ISD::SETULE:
18947 case ISD::SETUGT:
18948 case ISD::SETLE:
18949 case ISD::SETGT: {
18950 const std::optional<bool> KnownUle =
18951 KnownBits::ule(LHSKnownLo32, RHSKnownLo32);
18952 if (KnownUle) {
18953 if (*KnownUle) {
18954 // LHS.lo32 ule RHS.lo32, so LHS <= RHS iff LHS.hi32 <= RHS.hi32
18955 NewCC = CC;
18956 } else {
18957 // LHS.lo32 ugt RHS.lo32, so LHS <= RHS iff LHS.hi32 < RHS.hi32
18958 NewCC = CC == ISD::SETULE ? ISD::SETULT
18960 : CC == ISD::SETLE ? ISD::SETLT
18961 : ISD::SETGE;
18962 }
18963 }
18964 break;
18965 }
18966 }
18967
18968 if (NewCC != ISD::SETCC_INVALID)
18969 return DAG.getSetCC(SL, N->getValueType(0), getHiHalf64(LHS, DAG),
18970 getHiHalf64(RHS, DAG), NewCC);
18971
18972 // Truncate 64-bit setcc to test only lower 32-bits when upper 32-bits are
18973 // known equal.
18974 const KnownBits LHSKnownHi32 = LHSKnown.extractBits(32, 32);
18975 const KnownBits RHSKnownHi32 = RHSKnown.extractBits(32, 32);
18976 const std::optional<bool> KnownEq =
18977 KnownBits::eq(LHSKnownHi32, RHSKnownHi32);
18978
18979 if (KnownEq == true) {
18980 // For signed comparisons, convert to unsigned since upper halves match
18982
18983 SDValue LHSLo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, LHS);
18984 SDValue RHSLo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, RHS);
18985 return DAG.getSetCC(SL, N->getValueType(0), LHSLo, RHSLo, NewCC);
18986 }
18987 }
18988
18989 // Eliminate setcc by using carryout from add/sub instruction
18990
18991 // LHS = ADD i64 RHS, Z LHSlo = UADDO i32 RHSlo, Zlo
18992 // setcc LHS ult RHS -> LHSHi = UADDO_CARRY i32 RHShi, Zhi
18993 // similarly for subtraction
18994
18995 // LHS = ADD i64 Y, 1 LHSlo = UADDO i32 Ylo, 1
18996 // setcc LHS eq 0 -> LHSHi = UADDO_CARRY i32 Yhi, 0
18997
18998 if (VT == MVT::i64 && ((CC == ISD::SETULT &&
19000 (CC == ISD::SETUGT &&
19002 (CC == ISD::SETEQ && CRHS && CRHS->isZero() &&
19003 sd_match(LHS, m_Add(m_Value(), m_One()))))) {
19004 bool IsAdd = LHS.getOpcode() == ISD::ADD;
19005
19006 SDValue Op0 = LHS.getOperand(0);
19007 SDValue Op1 = LHS.getOperand(1);
19008
19009 SDValue Op0Lo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Op0);
19010 SDValue Op1Lo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Op1);
19011
19012 SDValue Op0Hi = getHiHalf64(Op0, DAG);
19013 SDValue Op1Hi = getHiHalf64(Op1, DAG);
19014
19015 SDValue NodeLo =
19016 DAG.getNode(IsAdd ? ISD::UADDO : ISD::USUBO, SL,
19017 DAG.getVTList(MVT::i32, MVT::i1), {Op0Lo, Op1Lo});
19018
19019 SDValue CarryInHi = NodeLo.getValue(1);
19020 SDValue NodeHi = DAG.getNode(IsAdd ? ISD::UADDO_CARRY : ISD::USUBO_CARRY,
19021 SL, DAG.getVTList(MVT::i32, MVT::i1),
19022 {Op0Hi, Op1Hi, CarryInHi});
19023
19024 SDValue ResultLo = NodeLo.getValue(0);
19025 SDValue ResultHi = NodeHi.getValue(0);
19026
19027 SDValue JoinedResult =
19028 DAG.getBuildVector(MVT::v2i32, SL, {ResultLo, ResultHi});
19029
19030 SDValue Result = DAG.getNode(ISD::BITCAST, SL, VT, JoinedResult);
19031 SDValue Overflow = NodeHi.getValue(1);
19032 DCI.CombineTo(LHS.getNode(), Result);
19033 return Overflow;
19034 }
19035
19036 if (VT != MVT::f32 && VT != MVT::f64 &&
19037 (!Subtarget->has16BitInsts() || VT != MVT::f16))
19038 return SDValue();
19039
19040 // Match isinf/isfinite pattern
19041 // (fcmp oeq (fabs x), inf) -> (fp_class x, (p_infinity | n_infinity))
19042 // (fcmp one (fabs x), inf) -> (fp_class x,
19043 // (p_normal | n_normal | p_subnormal | n_subnormal | p_zero | n_zero)
19044 if ((CC == ISD::SETOEQ || CC == ISD::SETONE) &&
19045 LHS.getOpcode() == ISD::FABS) {
19046 const ConstantFPSDNode *CRHS = dyn_cast<ConstantFPSDNode>(RHS);
19047 if (!CRHS)
19048 return SDValue();
19049
19050 const APFloat &APF = CRHS->getValueAPF();
19051 if (APF.isInfinity() && !APF.isNegative()) {
19052 const unsigned IsInfMask =
19054 const unsigned IsFiniteMask =
19058 unsigned Mask = CC == ISD::SETOEQ ? IsInfMask : IsFiniteMask;
19059 return DAG.getNode(AMDGPUISD::FP_CLASS, SL, MVT::i1, LHS.getOperand(0),
19060 DAG.getConstant(Mask, SL, MVT::i32));
19061 }
19062 }
19063
19064 if (VT == MVT::f64) {
19066 if (HiHalfCC != ISD::SETCC_INVALID)
19067 return DAG.getSetCC(SL, N->getValueType(0), getHiHalf64(LHS, DAG),
19068 getHiHalf64(RHS, DAG), HiHalfCC);
19069 }
19070
19071 return SDValue();
19072}
19073
19074SDValue
19075SITargetLowering::performCvtF32UByteNCombine(SDNode *N,
19076 DAGCombinerInfo &DCI) const {
19077 SelectionDAG &DAG = DCI.DAG;
19078 SDLoc SL(N);
19079 unsigned Offset = N->getOpcode() - AMDGPUISD::CVT_F32_UBYTE0;
19080
19081 SDValue Src = N->getOperand(0);
19082 SDValue Shift = N->getOperand(0);
19083
19084 // TODO: Extend type shouldn't matter (assuming legal types).
19085 if (Shift.getOpcode() == ISD::ZERO_EXTEND)
19086 Shift = Shift.getOperand(0);
19087
19088 if (Shift.getOpcode() == ISD::SRL || Shift.getOpcode() == ISD::SHL) {
19089 // cvt_f32_ubyte1 (shl x, 8) -> cvt_f32_ubyte0 x
19090 // cvt_f32_ubyte3 (shl x, 16) -> cvt_f32_ubyte1 x
19091 // cvt_f32_ubyte0 (srl x, 16) -> cvt_f32_ubyte2 x
19092 // cvt_f32_ubyte1 (srl x, 16) -> cvt_f32_ubyte3 x
19093 // cvt_f32_ubyte0 (srl x, 8) -> cvt_f32_ubyte1 x
19094 if (auto *C = dyn_cast<ConstantSDNode>(Shift.getOperand(1))) {
19095 SDValue Shifted = DAG.getZExtOrTrunc(
19096 Shift.getOperand(0), SDLoc(Shift.getOperand(0)), MVT::i32);
19097
19098 unsigned ShiftOffset = 8 * Offset;
19099 if (Shift.getOpcode() == ISD::SHL)
19100 ShiftOffset -= C->getZExtValue();
19101 else
19102 ShiftOffset += C->getZExtValue();
19103
19104 if (ShiftOffset < 32 && (ShiftOffset % 8) == 0) {
19105 return DAG.getNode(AMDGPUISD::CVT_F32_UBYTE0 + ShiftOffset / 8, SL,
19106 MVT::f32, Shifted);
19107 }
19108 }
19109 }
19110
19112 APInt DemandedBits = APInt::getBitsSet(32, 8 * Offset, 8 * Offset + 8);
19113 if (TLI.SimplifyDemandedBits(Src, DemandedBits, DCI)) {
19114 // We simplified Src. If this node is not dead, visit it again so it is
19115 // folded properly.
19116 if (N->getOpcode() != ISD::DELETED_NODE)
19117 DCI.AddToWorklist(N);
19118 return SDValue(N, 0);
19119 }
19120
19121 // Handle (or x, (srl y, 8)) pattern when known bits are zero.
19122 if (SDValue DemandedSrc =
19123 TLI.SimplifyMultipleUseDemandedBits(Src, DemandedBits, DAG))
19124 return DAG.getNode(N->getOpcode(), SL, MVT::f32, DemandedSrc);
19125
19126 return SDValue();
19127}
19128
19129SDValue SITargetLowering::performClampCombine(SDNode *N,
19130 DAGCombinerInfo &DCI) const {
19131 ConstantFPSDNode *CSrc = dyn_cast<ConstantFPSDNode>(N->getOperand(0));
19132 if (!CSrc)
19133 return SDValue();
19134
19135 const MachineFunction &MF = DCI.DAG.getMachineFunction();
19136 const APFloat &F = CSrc->getValueAPF();
19137 APFloat Zero = APFloat::getZero(F.getSemantics());
19138 if (F < Zero ||
19139 (F.isNaN() && MF.getInfo<SIMachineFunctionInfo>()->getMode().DX10Clamp)) {
19140 return DCI.DAG.getConstantFP(Zero, SDLoc(N), N->getValueType(0));
19141 }
19142
19143 APFloat One = APFloat::getOne(F.getSemantics());
19144 if (F > One)
19145 return DCI.DAG.getConstantFP(One, SDLoc(N), N->getValueType(0));
19146
19147 return getCanonicalConstantFP(DCI.DAG, SDLoc(N), N->getValueType(0), F);
19148}
19149
19150// Check if V is the exponent result of a frexp operation. Returns the frexp
19151// input via FrexpInput if matched. We only match the exponent (not mantissa)
19152// because V_FREXP_MANT returns its input for Inf/NaN, not zero.
19153static bool isFrexpExp(SDValue V, SDValue &FrexpInput) {
19154 // ISD::FFREXP returns {mant, exp} - only match if using the exp result
19155 // (result number 1).
19156 if (V.getOpcode() == ISD::FFREXP && V.getResNo() == 1) {
19157 FrexpInput = V.getOperand(0);
19158 return true;
19159 }
19161 m_Value(FrexpInput))))
19162 return true;
19163 return false;
19164}
19165
19166SDValue
19167SITargetLowering::performFrexpSelectCombine(SDNode *N,
19168 DAGCombinerInfo &DCI) const {
19169 // This optimization only applies when the hardware handles inf/nan correctly.
19170 if (Subtarget->hasFractBug())
19171 return SDValue();
19172
19173 SDValue Cond = N->getOperand(0);
19174 SDValue TrueVal = N->getOperand(1);
19175 SDValue FalseVal = N->getOperand(2);
19176
19177 // Identify which operand is the frexp result and which is the zero constant.
19178 // Pattern 1: select cond, 0, frexp_result (cond true -> return 0)
19179 // Pattern 2: select cond, frexp_result, 0 (cond false -> return 0)
19180 SDValue FrexpVal;
19181 SDValue ZeroVal;
19182 bool CondSelectsZero; // If true, condition=true selects zero
19183
19184 // Check if FrexpVal comes from ISD::FFREXP (exponent result only) or
19185 // amdgcn_frexp_exp intrinsic.
19186 SDValue FrexpInput;
19187 if (isFrexpExp(FalseVal, FrexpInput)) {
19188 FrexpVal = FalseVal;
19189 ZeroVal = TrueVal;
19190 CondSelectsZero = true;
19191 } else if (isFrexpExp(TrueVal, FrexpInput)) {
19192 FrexpVal = TrueVal;
19193 ZeroVal = FalseVal;
19194 CondSelectsZero = false;
19195 } else {
19196 return SDValue();
19197 }
19198
19199 // frexp_exp returns integer, so check for integer zero.
19200 if (!isNullConstant(ZeroVal))
19201 return SDValue();
19202
19203 // The frexp intrinsics ignore sign, so we can strip sign ops when comparing.
19204 SDValue FrexpInputStripped = peekFPSignOps(FrexpInput);
19205
19206 bool IsNonFiniteTest = false;
19207
19208 // Handle SETCC conditions for inf/nan tests.
19209 // The canonical form of these checks is fcmp + fabs.
19210 if (Cond.getOpcode() == ISD::SETCC) {
19211 ISD::CondCode CC = Cond.getOperandCondCode(2);
19212 SDValue CondLHS = Cond.getOperand(0);
19213 SDValue CondRHS = Cond.getOperand(1);
19214
19215 // Check if LHS is fabs(FrexpInput) - required for infinity comparisons.
19216 SDValue FAbsInput;
19217 bool LHSIsFabs = sd_match(CondLHS, m_FAbs(m_Value(FAbsInput)));
19218 bool LHSMatchesFrexp =
19219 (CondLHS == FrexpInput) ||
19220 (LHSIsFabs && peekFPSignOps(FAbsInput) == FrexpInputStripped) ||
19221 (peekFPSignOps(CondLHS) == FrexpInputStripped);
19222 bool RHSMatchesFrexp = (CondRHS == FrexpInput) ||
19223 (peekFPSignOps(CondRHS) == FrexpInputStripped);
19224
19225 if (CC == ISD::SETUO) {
19226 // fcmp uno x, y - true if either x or y is NaN
19227 // We can only fold if the non-frexp operand is known to never be NaN,
19228 // otherwise the comparison could be true due to the other operand.
19229 // Special case: fcmp uno x, x (same operand) is a valid NaN test.
19230 SelectionDAG &DAG = DCI.DAG;
19231 if (LHSMatchesFrexp &&
19232 (CondLHS == CondRHS || DAG.isKnownNeverNaN(CondRHS)))
19233 IsNonFiniteTest = CondSelectsZero;
19234 else if (RHSMatchesFrexp && DAG.isKnownNeverNaN(CondLHS))
19235 IsNonFiniteTest = CondSelectsZero;
19236 } else if ((CC == ISD::SETOEQ || CC == ISD::SETUEQ) && LHSMatchesFrexp &&
19237 LHSIsFabs &&
19238 sd_match(CondRHS,
19240 CondRHS.getValueType().getFltSemantics())))) {
19241 // fcmp oeq/ueq fabs(x), +inf - true if x is inf (or inf/nan for ueq)
19242 IsNonFiniteTest = CondSelectsZero;
19243 } else if ((CC == ISD::SETONE || CC == ISD::SETUNE) && LHSMatchesFrexp &&
19244 LHSIsFabs &&
19245 sd_match(CondRHS,
19247 CondRHS.getValueType().getFltSemantics())))) {
19248 // fcmp one/une fabs(x), +inf - true if x is NOT inf
19249 IsNonFiniteTest = !CondSelectsZero;
19250 } else if (CC == ISD::SETO) {
19251 // fcmp ord x, y - true if both are NOT NaN
19252 // We can only fold if the non-frexp operand is known to never be NaN,
19253 // otherwise the comparison could be false due to the other operand.
19254 // Special case: fcmp ord x, x (same operand) is a valid not-NaN test.
19255 SelectionDAG &DAG = DCI.DAG;
19256 if (LHSMatchesFrexp &&
19257 (CondLHS == CondRHS || DAG.isKnownNeverNaN(CondRHS)))
19258 IsNonFiniteTest = !CondSelectsZero;
19259 else if (RHSMatchesFrexp && DAG.isKnownNeverNaN(CondLHS))
19260 IsNonFiniteTest = !CondSelectsZero;
19261 }
19262 }
19263
19264 if (!IsNonFiniteTest)
19265 return SDValue();
19266
19267 // The select can be eliminated - just return the frexp result directly.
19268 return FrexpVal;
19269}
19270
19271SDValue SITargetLowering::performSelectCombine(SDNode *N,
19272 DAGCombinerInfo &DCI) const {
19273
19274 // Try to fold CMP + SELECT patterns with shared constants (both FP and
19275 // integer).
19276 // Detect when CMP and SELECT use the same constant and fold them to avoid
19277 // loading the constant twice. Specifically handles patterns like:
19278 // %cmp = icmp eq i32 %val, 4242
19279 // %sel = select i1 %cmp, i32 4242, i32 %other
19280 // It can be optimized to reuse %val instead of 4242 in select.
19281 SDValue Cond = N->getOperand(0);
19282 SDValue TrueVal = N->getOperand(1);
19283 SDValue FalseVal = N->getOperand(2);
19284
19285 // Check if condition is a comparison.
19286 if (Cond.getOpcode() != ISD::SETCC)
19287 return SDValue();
19288
19289 SDValue LHS = Cond.getOperand(0);
19290 SDValue RHS = Cond.getOperand(1);
19291 ISD::CondCode CC = Cond.getOperandCondCode(2);
19292
19293 bool isFloatingPoint = LHS.getValueType().isFloatingPoint();
19294 bool isInteger = LHS.getValueType().isInteger();
19295
19296 // Handle simple floating-point and integer types only.
19297 if (!isFloatingPoint && !isInteger)
19298 return SDValue();
19299
19300 // Bare SETEQ/SETNE is the builder's NaN-impossible downgrade.
19301 bool isEquality = CC == ISD::SETEQ || (isFloatingPoint && CC == ISD::SETOEQ);
19302 bool isNonEquality =
19303 CC == ISD::SETNE || (isFloatingPoint && CC == ISD::SETONE);
19304 if (!isEquality && !isNonEquality)
19305 return SDValue();
19306
19307 SDValue ArgVal, ConstVal;
19308 if ((isFloatingPoint && isa<ConstantFPSDNode>(RHS)) ||
19309 (isInteger && isa<ConstantSDNode>(RHS))) {
19310 ConstVal = RHS;
19311 ArgVal = LHS;
19312 } else if ((isFloatingPoint && isa<ConstantFPSDNode>(LHS)) ||
19313 (isInteger && isa<ConstantSDNode>(LHS))) {
19314 ConstVal = LHS;
19315 ArgVal = RHS;
19316 } else {
19317 return SDValue();
19318 }
19319
19320 // Skip optimization for inlinable immediates.
19321 if (isFloatingPoint) {
19322 const APFloat &Val = cast<ConstantFPSDNode>(ConstVal)->getValueAPF();
19323 if (!Val.isNormal() || Subtarget->getInstrInfo()->isInlineConstant(Val))
19324 return SDValue();
19325 } else {
19326 const std::optional<int64_t> Val =
19327 cast<ConstantSDNode>(ConstVal)->getAPIntValue().trySExtValue();
19328 if (Val && AMDGPU::isInlinableIntLiteral(*Val))
19329 return SDValue();
19330 }
19331
19332 // For equality and non-equality comparisons, patterns:
19333 // select (setcc x, const), const, y -> select (setcc x, const), x, y
19334 // select (setccinv x, const), y, const -> select (setccinv x, const), y, x
19335 if (!(isEquality && TrueVal == ConstVal) &&
19336 !(isNonEquality && FalseVal == ConstVal))
19337 return SDValue();
19338
19339 // SETONE's false arm is also taken for NaN ArgVal, so require NaN excluded.
19340 if (isFloatingPoint && isNonEquality && FalseVal == ConstVal &&
19341 !Cond->getFlags().hasNoNaNs() && !DCI.DAG.isKnownNeverNaN(ArgVal))
19342 return SDValue();
19343
19344 SDValue SelectLHS = (isEquality && TrueVal == ConstVal) ? ArgVal : TrueVal;
19345 SDValue SelectRHS =
19346 (isNonEquality && FalseVal == ConstVal) ? ArgVal : FalseVal;
19347 return DCI.DAG.getNode(ISD::SELECT, SDLoc(N), N->getValueType(0), Cond,
19348 SelectLHS, SelectRHS);
19349}
19350
19352 DAGCombinerInfo &DCI) const {
19353 switch (N->getOpcode()) {
19354 case ISD::ABS:
19355 if (SDValue Res = promoteUniformUnaryOpToI32(SDValue(N, 0), DCI))
19356 return Res;
19357 break;
19358 case ISD::ADD:
19359 case ISD::SUB:
19360 case ISD::SHL:
19361 case ISD::SRL:
19362 case ISD::SRA:
19363 case ISD::AND:
19364 case ISD::OR:
19365 case ISD::XOR:
19366 case ISD::MUL:
19367 case ISD::SETCC:
19368 case ISD::SELECT:
19369 case ISD::SMIN:
19370 case ISD::SMAX:
19371 case ISD::UMIN:
19372 case ISD::UMAX:
19373 case ISD::USUBSAT:
19374 case ISD::UADDSAT:
19375 if (auto Res = promoteUniformOpToI32(SDValue(N, 0), DCI))
19376 return Res;
19377 break;
19378 case ISD::LOAD:
19379 case ISD::STORE:
19380 // TODO: Move this to LowerLOAD and LowerSTORE once the action of a
19381 // non-extending load or non-truncating store can depend on the address
19382 // space. Until then one whose action is Legal, such as i32, never reaches
19383 // them.
19385 if (SDValue V = lowerLoadStoreVGPR(SDValue(N, 0), DCI.DAG))
19386 return V;
19387 }
19388 break;
19389 default:
19390 break;
19391 }
19392
19393 if (getTargetMachine().getOptLevel() == CodeGenOptLevel::None)
19394 return SDValue();
19395
19396 switch (N->getOpcode()) {
19397 case ISD::ADD:
19398 return performAddCombine(N, DCI);
19399 case ISD::PTRADD:
19400 return performPtrAddCombine(N, DCI);
19401 case ISD::SUB:
19402 return performSubCombine(N, DCI);
19403 case ISD::FADD:
19404 return performFAddCombine(N, DCI);
19405 case ISD::FSUB:
19406 return performFSubCombine(N, DCI);
19407 case ISD::FDIV:
19408 return performFDivCombine(N, DCI);
19409 case ISD::FMUL:
19410 return performFMulCombine(N, DCI);
19411 case ISD::SETCC:
19412 return performSetCCCombine(N, DCI);
19413 case ISD::SELECT:
19414 if (auto Res = performFrexpSelectCombine(N, DCI))
19415 return Res;
19416 if (auto Res = performSelectCombine(N, DCI))
19417 return Res;
19418 break;
19419 case ISD::FMAXNUM:
19420 case ISD::FMINNUM:
19421 case ISD::FMAXNUM_IEEE:
19422 case ISD::FMINNUM_IEEE:
19423 case ISD::FMAXIMUM:
19424 case ISD::FMINIMUM:
19425 case ISD::FMAXIMUMNUM:
19426 case ISD::FMINIMUMNUM:
19427 case ISD::SMAX:
19428 case ISD::SMIN:
19429 case ISD::UMAX:
19430 case ISD::UMIN:
19431 case AMDGPUISD::FMIN_LEGACY:
19432 case AMDGPUISD::FMAX_LEGACY:
19433 return performMinMaxCombine(N, DCI);
19434 case ISD::FMA:
19435 return performFMACombine(N, DCI);
19436 case ISD::AND:
19437 return performAndCombine(N, DCI);
19438 case ISD::OR:
19439 return performOrCombine(N, DCI);
19440 case ISD::FSHR: {
19442 if (N->getValueType(0) == MVT::i32 && N->isDivergent() &&
19443 TII->pseudoToMCOpcode(AMDGPU::V_PERM_B32_e64) != -1) {
19444 return matchPERM(N, DCI);
19445 }
19446 break;
19447 }
19448 case ISD::XOR:
19449 return performXorCombine(N, DCI);
19450 case ISD::ANY_EXTEND:
19451 case ISD::ZERO_EXTEND:
19452 return performZeroOrAnyExtendCombine(N, DCI);
19453 case ISD::SIGN_EXTEND:
19454 return performExtendTruncShiftCombine(N, DCI);
19456 return performSignExtendInRegCombine(N, DCI);
19457 case AMDGPUISD::FP_CLASS:
19458 return performClassCombine(N, DCI);
19459 case ISD::FCANONICALIZE:
19460 return performFCanonicalizeCombine(N, DCI);
19461 case AMDGPUISD::RCP:
19462 return performRcpCombine(N, DCI);
19463 case ISD::FLDEXP:
19464 case AMDGPUISD::FRACT:
19465 case AMDGPUISD::RSQ:
19466 case AMDGPUISD::RCP_LEGACY:
19467 case AMDGPUISD::RCP_IFLAG:
19468 case AMDGPUISD::RSQ_CLAMP: {
19469 // FIXME: This is probably wrong. If src is an sNaN, it won't be quieted
19470 SDValue Src = N->getOperand(0);
19471 if (Src.isUndef())
19472 return Src;
19473 break;
19474 }
19475 case ISD::SINT_TO_FP:
19476 case ISD::UINT_TO_FP:
19477 return performUCharToFloatCombine(N, DCI);
19478 case ISD::FCOPYSIGN:
19479 return performFCopySignCombine(N, DCI);
19480 case AMDGPUISD::CVT_F32_UBYTE0:
19481 case AMDGPUISD::CVT_F32_UBYTE1:
19482 case AMDGPUISD::CVT_F32_UBYTE2:
19483 case AMDGPUISD::CVT_F32_UBYTE3:
19484 return performCvtF32UByteNCombine(N, DCI);
19485 case AMDGPUISD::FMED3:
19486 return performFMed3Combine(N, DCI);
19487 case AMDGPUISD::CVT_PKRTZ_F16_F32:
19488 return performCvtPkRTZCombine(N, DCI);
19490 if (SDValue Res = performExponentOnlyScaleCombine(N, DCI))
19491 return Res;
19492 break;
19493 case AMDGPUISD::CLAMP:
19494 return performClampCombine(N, DCI);
19495 case ISD::SCALAR_TO_VECTOR: {
19496 SelectionDAG &DAG = DCI.DAG;
19497 EVT VT = N->getValueType(0);
19498
19499 // v2i16 (scalar_to_vector i16:x) -> v2i16 (bitcast (any_extend i16:x))
19500 if (VT == MVT::v2i16 || VT == MVT::v2f16 || VT == MVT::v2bf16) {
19501 SDLoc SL(N);
19502 SDValue Src = N->getOperand(0);
19503 EVT EltVT = Src.getValueType();
19504 if (EltVT != MVT::i16)
19505 Src = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Src);
19506
19507 SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Src);
19508 return DAG.getNode(ISD::BITCAST, SL, VT, Ext);
19509 }
19510
19511 break;
19512 }
19514 return performExtractVectorEltCombine(N, DCI);
19516 return performInsertVectorEltCombine(N, DCI);
19517 case ISD::FP_ROUND:
19518 return performFPRoundCombine(N, DCI);
19519 case ISD::LOAD: {
19520 if (SDValue Widened = widenLoad(cast<LoadSDNode>(N), DCI))
19521 return Widened;
19522 [[fallthrough]];
19523 }
19524 default: {
19525 if (!DCI.isBeforeLegalize()) {
19526 if (MemSDNode *MemNode = dyn_cast<MemSDNode>(N))
19527 return performMemSDNodeCombine(MemNode, DCI);
19528 }
19529
19530 break;
19531 }
19532 }
19533
19535}
19536
19537/// Helper function for adjustWritemask
19538static unsigned SubIdx2Lane(unsigned Idx) {
19539 switch (Idx) {
19540 default:
19541 return ~0u;
19542 case AMDGPU::sub0:
19543 return 0;
19544 case AMDGPU::sub1:
19545 return 1;
19546 case AMDGPU::sub2:
19547 return 2;
19548 case AMDGPU::sub3:
19549 return 3;
19550 case AMDGPU::sub4:
19551 return 4; // Possible with TFE/LWE
19552 }
19553}
19554
19555/// Adjust the writemask of MIMG, VIMAGE or VSAMPLE instructions
19556SDNode *SITargetLowering::adjustWritemask(MachineSDNode *&Node,
19557 SelectionDAG &DAG) const {
19558 unsigned Opcode = Node->getMachineOpcode();
19559
19560 // Subtract 1 because the vdata output is not a MachineSDNode operand.
19561 int D16Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::d16) - 1;
19562 if (D16Idx >= 0 && Node->getConstantOperandVal(D16Idx))
19563 return Node; // not implemented for D16
19564
19565 SDNode *Users[5] = {nullptr};
19566 unsigned Lane = 0;
19567 unsigned DmaskIdx =
19568 AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::dmask) - 1;
19569 unsigned OldDmask = Node->getConstantOperandVal(DmaskIdx);
19570 unsigned NewDmask = 0;
19571 unsigned TFEIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::tfe) - 1;
19572 unsigned LWEIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::lwe) - 1;
19573 bool UsesTFC = (int(TFEIdx) >= 0 && Node->getConstantOperandVal(TFEIdx)) ||
19574 (int(LWEIdx) >= 0 && Node->getConstantOperandVal(LWEIdx));
19575 unsigned TFCLane = 0;
19576 bool HasChain = Node->getNumValues() > 1;
19577
19578 if (OldDmask == 0) {
19579 // These are folded out, but on the chance it happens don't assert.
19580 return Node;
19581 }
19582
19583 unsigned OldBitsSet = llvm::popcount(OldDmask);
19584 // Work out which is the TFE/LWE lane if that is enabled.
19585 if (UsesTFC) {
19586 TFCLane = OldBitsSet;
19587 }
19588
19589 // Try to figure out the used register components
19590 for (SDUse &Use : Node->uses()) {
19591
19592 // Don't look at users of the chain.
19593 if (Use.getResNo() != 0)
19594 continue;
19595
19596 SDNode *User = Use.getUser();
19597
19598 // Abort if we can't understand the usage
19599 if (!User->isMachineOpcode() ||
19600 User->getMachineOpcode() != TargetOpcode::EXTRACT_SUBREG)
19601 return Node;
19602
19603 // Lane means which subreg of %vgpra_vgprb_vgprc_vgprd is used.
19604 // Note that subregs are packed, i.e. Lane==0 is the first bit set
19605 // in OldDmask, so it can be any of X,Y,Z,W; Lane==1 is the second bit
19606 // set, etc.
19607 Lane = SubIdx2Lane(User->getConstantOperandVal(1));
19608 if (Lane == ~0u)
19609 return Node;
19610
19611 // Check if the use is for the TFE/LWE generated result at VGPRn+1.
19612 if (UsesTFC && Lane == TFCLane) {
19613 Users[Lane] = User;
19614 } else {
19615 // Set which texture component corresponds to the lane.
19616 unsigned Comp;
19617 for (unsigned i = 0, Dmask = OldDmask; (i <= Lane) && (Dmask != 0); i++) {
19618 Comp = llvm::countr_zero(Dmask);
19619 Dmask &= ~(1 << Comp);
19620 }
19621
19622 // Abort if we have more than one user per component.
19623 if (Users[Lane])
19624 return Node;
19625
19626 Users[Lane] = User;
19627 NewDmask |= 1 << Comp;
19628 }
19629 }
19630
19631 // Don't allow 0 dmask, as hardware assumes one channel enabled.
19632 bool NoChannels = !NewDmask;
19633 if (NoChannels) {
19634 if (!UsesTFC) {
19635 // No uses of the result and not using TFC. Then do nothing.
19636 return Node;
19637 }
19638 // If the original dmask has one channel - then nothing to do
19639 if (OldBitsSet == 1)
19640 return Node;
19641 // Use an arbitrary dmask - required for the instruction to work
19642 NewDmask = 1;
19643 }
19644 // Abort if there's no change
19645 if (NewDmask == OldDmask)
19646 return Node;
19647
19648 unsigned BitsSet = llvm::popcount(NewDmask);
19649
19650 // Check for TFE or LWE - increase the number of channels by one to account
19651 // for the extra return value
19652 // This will need adjustment for D16 if this is also included in
19653 // adjustWriteMask (this function) but at present D16 are excluded.
19654 unsigned NewChannels = BitsSet + UsesTFC;
19655
19656 int NewOpcode =
19657 AMDGPU::getMaskedMIMGOp(Node->getMachineOpcode(), NewChannels);
19658 assert(NewOpcode != -1 &&
19659 NewOpcode != static_cast<int>(Node->getMachineOpcode()) &&
19660 "failed to find equivalent MIMG op");
19661
19662 // Adjust the writemask in the node
19664 llvm::append_range(Ops, Node->ops().take_front(DmaskIdx));
19665 Ops.push_back(DAG.getTargetConstant(NewDmask, SDLoc(Node), MVT::i32));
19666 llvm::append_range(Ops, Node->ops().drop_front(DmaskIdx + 1));
19667
19668 MVT SVT = Node->getValueType(0).getVectorElementType().getSimpleVT();
19669
19670 MVT ResultVT = NewChannels == 1
19671 ? SVT
19672 : MVT::getVectorVT(SVT, NewChannels == 3 ? 4
19673 : NewChannels == 5 ? 8
19674 : NewChannels);
19675 SDVTList NewVTList =
19676 HasChain ? DAG.getVTList(ResultVT, MVT::Other) : DAG.getVTList(ResultVT);
19677
19678 MachineSDNode *NewNode =
19679 DAG.getMachineNode(NewOpcode, SDLoc(Node), NewVTList, Ops);
19680
19681 if (HasChain) {
19682 // Update chain.
19683 DAG.setNodeMemRefs(NewNode, Node->memoperands());
19684 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), SDValue(NewNode, 1));
19685 }
19686
19687 if (NewChannels == 1) {
19688 assert(Node->hasNUsesOfValue(1, 0));
19689 SDNode *Copy =
19690 DAG.getMachineNode(TargetOpcode::COPY, SDLoc(Node),
19691 Users[Lane]->getValueType(0), SDValue(NewNode, 0));
19692 DAG.ReplaceAllUsesWith(Users[Lane], Copy);
19693 return nullptr;
19694 }
19695
19696 // Update the users of the node with the new indices
19697 for (unsigned i = 0, Idx = AMDGPU::sub0; i < 5; ++i) {
19698 SDNode *User = Users[i];
19699 if (!User) {
19700 // Handle the special case of NoChannels. We set NewDmask to 1 above, but
19701 // Users[0] is still nullptr because channel 0 doesn't really have a use.
19702 if (i || !NoChannels)
19703 continue;
19704 } else {
19705 SDValue Op = DAG.getTargetConstant(Idx, SDLoc(User), MVT::i32);
19706 SDNode *NewUser = DAG.UpdateNodeOperands(User, SDValue(NewNode, 0), Op);
19707 if (NewUser != User) {
19708 DAG.ReplaceAllUsesWith(SDValue(User, 0), SDValue(NewUser, 0));
19709 DAG.RemoveDeadNode(User);
19710 }
19711 }
19712
19713 switch (Idx) {
19714 default:
19715 break;
19716 case AMDGPU::sub0:
19717 Idx = AMDGPU::sub1;
19718 break;
19719 case AMDGPU::sub1:
19720 Idx = AMDGPU::sub2;
19721 break;
19722 case AMDGPU::sub2:
19723 Idx = AMDGPU::sub3;
19724 break;
19725 case AMDGPU::sub3:
19726 Idx = AMDGPU::sub4;
19727 break;
19728 }
19729 }
19730
19731 DAG.RemoveDeadNode(Node);
19732 return nullptr;
19733}
19734
19736 if (Op.getOpcode() == ISD::AssertZext)
19737 Op = Op.getOperand(0);
19738
19739 return isa<FrameIndexSDNode>(Op);
19740}
19741
19742/// Legalize target independent instructions (e.g. INSERT_SUBREG)
19743/// with frame index operands.
19744/// LLVM assumes that inputs are to these instructions are registers.
19745SDNode *
19747 SelectionDAG &DAG) const {
19748 if (Node->getOpcode() == ISD::CopyToReg) {
19749 RegisterSDNode *DestReg = cast<RegisterSDNode>(Node->getOperand(1));
19750 SDValue SrcVal = Node->getOperand(2);
19751
19752 // Insert a copy to a VReg_1 virtual register so LowerI1Copies doesn't have
19753 // to try understanding copies to physical registers.
19754 if (SrcVal.getValueType() == MVT::i1 && DestReg->getReg().isPhysical()) {
19755 SDLoc SL(Node);
19757 SDValue VReg = DAG.getRegister(
19758 MRI.createVirtualRegister(&AMDGPU::VReg_1RegClass), MVT::i1);
19759
19760 SDNode *Glued = Node->getGluedNode();
19761 SDValue ToVReg = DAG.getCopyToReg(
19762 Node->getOperand(0), SL, VReg, SrcVal,
19763 SDValue(Glued, Glued ? Glued->getNumValues() - 1 : 0));
19764 SDValue ToResultReg = DAG.getCopyToReg(ToVReg, SL, SDValue(DestReg, 0),
19765 VReg, ToVReg.getValue(1));
19766 DAG.ReplaceAllUsesWith(Node, ToResultReg.getNode());
19767 DAG.RemoveDeadNode(Node);
19768 return ToResultReg.getNode();
19769 }
19770 }
19771
19773 for (unsigned i = 0; i < Node->getNumOperands(); ++i) {
19774 if (!isFrameIndexOp(Node->getOperand(i))) {
19775 Ops.push_back(Node->getOperand(i));
19776 continue;
19777 }
19778
19779 SDLoc DL(Node);
19780 Ops.push_back(SDValue(DAG.getMachineNode(AMDGPU::S_MOV_B32, DL,
19781 Node->getOperand(i).getValueType(),
19782 Node->getOperand(i)),
19783 0));
19784 }
19785
19786 return DAG.UpdateNodeOperands(Node, Ops);
19787}
19788
19789/// Fold the instructions after selecting them.
19790/// Returns null if users were already updated.
19792 SelectionDAG &DAG) const {
19794 unsigned Opcode = Node->getMachineOpcode();
19795
19796 if (TII->isImage(Opcode) && !TII->get(Opcode).mayStore() &&
19797 !TII->isGather4(Opcode) &&
19798 AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::dmask)) {
19799 return adjustWritemask(Node, DAG);
19800 }
19801
19802 if (Opcode == AMDGPU::INSERT_SUBREG || Opcode == AMDGPU::REG_SEQUENCE) {
19804 return Node;
19805 }
19806
19807 switch (Opcode) {
19808 case AMDGPU::V_DIV_SCALE_F32_e64:
19809 case AMDGPU::V_DIV_SCALE_F64_e64: {
19810 // Satisfy the operand register constraint when one of the inputs is
19811 // undefined. Ordinarily each undef value will have its own implicit_def of
19812 // a vreg, so force these to use a single register.
19813 SDValue Src0 = Node->getOperand(1);
19814 SDValue Src1 = Node->getOperand(3);
19815 SDValue Src2 = Node->getOperand(5);
19816
19817 if ((Src0.isMachineOpcode() &&
19818 Src0.getMachineOpcode() != AMDGPU::IMPLICIT_DEF) &&
19819 (Src0 == Src1 || Src0 == Src2))
19820 break;
19821
19822 MVT VT = Src0.getValueType().getSimpleVT();
19823 const TargetRegisterClass *RC =
19824 getRegClassFor(VT, Src0.getNode()->isDivergent());
19825
19827 SDValue UndefReg = DAG.getRegister(MRI.createVirtualRegister(RC), VT);
19828
19829 SDValue ImpDef = DAG.getCopyToReg(DAG.getEntryNode(), SDLoc(Node), UndefReg,
19830 Src0, SDValue());
19831
19832 // src0 must be the same register as src1 or src2, even if the value is
19833 // undefined, so make sure we don't violate this constraint.
19834 if (Src0.isMachineOpcode() &&
19835 Src0.getMachineOpcode() == AMDGPU::IMPLICIT_DEF) {
19836 if (Src1.isMachineOpcode() &&
19837 Src1.getMachineOpcode() != AMDGPU::IMPLICIT_DEF)
19838 Src0 = Src1;
19839 else if (Src2.isMachineOpcode() &&
19840 Src2.getMachineOpcode() != AMDGPU::IMPLICIT_DEF)
19841 Src0 = Src2;
19842 else {
19843 assert(Src1.getMachineOpcode() == AMDGPU::IMPLICIT_DEF);
19844 Src0 = UndefReg;
19845 Src1 = UndefReg;
19846 }
19847 } else
19848 break;
19849
19851 Ops[1] = Src0;
19852 Ops[3] = Src1;
19853 Ops[5] = Src2;
19854 Ops.push_back(ImpDef.getValue(1));
19855 return DAG.getMachineNode(Opcode, SDLoc(Node), Node->getVTList(), Ops);
19856 }
19857 default:
19858 break;
19859 }
19860
19861 return Node;
19862}
19863
19864// Any MIMG instructions that use tfe or lwe require an initialization of the
19865// result register that will be written in the case of a memory access failure.
19866// The required code is also added to tie this init code to the result of the
19867// img instruction.
19870 const SIRegisterInfo &TRI = TII->getRegisterInfo();
19871 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
19872 MachineBasicBlock &MBB = *MI.getParent();
19873
19874 int DstIdx =
19875 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata);
19876 unsigned InitIdx = 0;
19877
19878 if (TII->isImage(MI)) {
19879 MachineOperand *TFE = TII->getNamedOperand(MI, AMDGPU::OpName::tfe);
19880 MachineOperand *LWE = TII->getNamedOperand(MI, AMDGPU::OpName::lwe);
19881 MachineOperand *D16 = TII->getNamedOperand(MI, AMDGPU::OpName::d16);
19882
19883 if (!TFE && !LWE) // intersect_ray
19884 return;
19885
19886 unsigned TFEVal = TFE ? TFE->getImm() : 0;
19887 unsigned LWEVal = LWE ? LWE->getImm() : 0;
19888 unsigned D16Val = D16 ? D16->getImm() : 0;
19889
19890 if (!TFEVal && !LWEVal)
19891 return;
19892
19893 // At least one of TFE or LWE are non-zero
19894 // We have to insert a suitable initialization of the result value and
19895 // tie this to the dest of the image instruction.
19896
19897 // Calculate which dword we have to initialize to 0.
19898 MachineOperand *MO_Dmask = TII->getNamedOperand(MI, AMDGPU::OpName::dmask);
19899
19900 // check that dmask operand is found.
19901 assert(MO_Dmask && "Expected dmask operand in instruction");
19902
19903 unsigned dmask = MO_Dmask->getImm();
19904 // Determine the number of active lanes taking into account the
19905 // Gather4 special case
19906 unsigned ActiveLanes = TII->isGather4(MI) ? 4 : llvm::popcount(dmask);
19907
19908 bool Packed = !Subtarget->hasUnpackedD16VMem();
19909
19910 InitIdx = D16Val && Packed ? ((ActiveLanes + 1) >> 1) + 1 : ActiveLanes + 1;
19911
19912 // Abandon attempt if the dst size isn't large enough
19913 // - this is in fact an error but this is picked up elsewhere and
19914 // reported correctly.
19915 const TargetRegisterClass *DstRC = TII->getRegClass(MI.getDesc(), DstIdx);
19916
19917 uint32_t DstSize = TRI.getRegSizeInBits(*DstRC) / 32;
19918 if (DstSize < InitIdx)
19919 return;
19920 } else if (TII->isMUBUF(MI) && AMDGPU::getMUBUFTfe(MI.getOpcode())) {
19921 const TargetRegisterClass *DstRC = TII->getRegClass(MI.getDesc(), DstIdx);
19922 InitIdx = TRI.getRegSizeInBits(*DstRC) / 32;
19923 } else {
19924 return;
19925 }
19926
19927 const DebugLoc &DL = MI.getDebugLoc();
19928
19929 // Create a register for the initialization value.
19930 Register PrevDst = MRI.cloneVirtualRegister(MI.getOperand(DstIdx).getReg());
19931 unsigned NewDst = 0; // Final initialized value will be in here
19932
19933 // If PRTStrictNull feature is enabled (the default) then initialize
19934 // all the result registers to 0, otherwise just the error indication
19935 // register (VGPRn+1)
19936 unsigned SizeLeft = Subtarget->usePRTStrictNull() ? InitIdx : 1;
19937 unsigned CurrIdx = Subtarget->usePRTStrictNull() ? 0 : (InitIdx - 1);
19938
19939 BuildMI(MBB, MI, DL, TII->get(AMDGPU::IMPLICIT_DEF), PrevDst);
19940 for (; SizeLeft; SizeLeft--, CurrIdx++) {
19941 NewDst = MRI.createVirtualRegister(TII->getOpRegClass(MI, DstIdx));
19942 // Initialize dword
19943 Register SubReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
19944 // clang-format off
19945 BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), SubReg)
19946 .addImm(0);
19947 // clang-format on
19948 // Insert into the super-reg
19949 BuildMI(MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), NewDst)
19950 .addReg(PrevDst)
19951 .addReg(SubReg)
19953
19954 PrevDst = NewDst;
19955 }
19956
19957 // Add as an implicit operand
19958 MI.addOperand(MachineOperand::CreateReg(NewDst, false, true));
19959
19960 // Tie the just added implicit operand to the dst
19961 MI.tieOperands(DstIdx, MI.getNumOperands() - 1);
19962}
19963
19964/// Assign the register class depending on the number of
19965/// bits set in the writemask
19967 SDNode *Node) const {
19969
19970 MachineFunction *MF = MI.getMF();
19972
19973 if (TII->isVOP3(MI.getOpcode())) {
19974 // Make sure constant bus requirements are respected.
19975 TII->legalizeOperandsVOP3(MRI, MI);
19976
19977 if (TII->isMAI(MI)) {
19978 // The ordinary src0, src1, src2 were legalized above.
19979 //
19980 // We have to also legalize the appended v_mfma_ld_scale_b32 operands,
19981 // as a separate instruction.
19982 int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
19983 AMDGPU::OpName::scale_src0);
19984 if (Src0Idx != -1) {
19985 int Src1Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
19986 AMDGPU::OpName::scale_src1);
19987 if (TII->usesConstantBus(MRI, MI, Src0Idx) &&
19988 TII->usesConstantBus(MRI, MI, Src1Idx))
19989 TII->legalizeOpWithMove(MI, Src1Idx);
19990 }
19991 }
19992
19993 return;
19994 }
19995
19996 if (TII->isImage(MI))
19997 TII->enforceOperandRCAlignment(MI, AMDGPU::OpName::vaddr);
19998}
19999
20001 uint64_t Val) {
20002 SDValue K = DAG.getTargetConstant(Val, DL, MVT::i32);
20003 return SDValue(DAG.getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, K), 0);
20004}
20005
20007 const SDLoc &DL,
20008 SDValue Ptr) const {
20010
20011 // Build the half of the subregister with the constants before building the
20012 // full 128-bit register. If we are building multiple resource descriptors,
20013 // this will allow CSEing of the 2-component register.
20014 const SDValue Ops0[] = {
20015 DAG.getTargetConstant(AMDGPU::SGPR_64RegClassID, DL, MVT::i32),
20016 buildSMovImm32(DAG, DL, 0),
20017 DAG.getTargetConstant(AMDGPU::sub0, DL, MVT::i32),
20018 buildSMovImm32(DAG, DL, TII->getDefaultRsrcDataFormat() >> 32),
20019 DAG.getTargetConstant(AMDGPU::sub1, DL, MVT::i32)};
20020
20021 SDValue SubRegHi = SDValue(
20022 DAG.getMachineNode(AMDGPU::REG_SEQUENCE, DL, MVT::v2i32, Ops0), 0);
20023
20024 // Combine the constants and the pointer.
20025 const SDValue Ops1[] = {
20026 DAG.getTargetConstant(AMDGPU::SGPR_128RegClassID, DL, MVT::i32), Ptr,
20027 DAG.getTargetConstant(AMDGPU::sub0_sub1, DL, MVT::i32), SubRegHi,
20028 DAG.getTargetConstant(AMDGPU::sub2_sub3, DL, MVT::i32)};
20029
20030 return DAG.getMachineNode(AMDGPU::REG_SEQUENCE, DL, MVT::v4i32, Ops1);
20031}
20032
20033/// Return a resource descriptor with the 'Add TID' bit enabled
20034/// The TID (Thread ID) is multiplied by the stride value (bits [61:48]
20035/// of the resource descriptor) to create an offset, which is added to
20036/// the resource pointer.
20038 SDValue Ptr, uint32_t RsrcDword1,
20039 uint64_t RsrcDword2And3) const {
20040 SDValue PtrLo = DAG.getTargetExtractSubreg(AMDGPU::sub0, DL, MVT::i32, Ptr);
20041 SDValue PtrHi = DAG.getTargetExtractSubreg(AMDGPU::sub1, DL, MVT::i32, Ptr);
20042 if (RsrcDword1) {
20043 PtrHi = DAG.getNode(ISD::OR, DL, MVT::i32, PtrHi,
20044 DAG.getConstant(RsrcDword1, DL, MVT::i32));
20045 }
20046
20047 SDValue DataLo =
20048 buildSMovImm32(DAG, DL, RsrcDword2And3 & UINT64_C(0xFFFFFFFF));
20049 SDValue DataHi = buildSMovImm32(DAG, DL, RsrcDword2And3 >> 32);
20050
20051 const SDValue Ops[] = {
20052 DAG.getTargetConstant(AMDGPU::SGPR_128RegClassID, DL, MVT::i32),
20053 PtrLo,
20054 DAG.getTargetConstant(AMDGPU::sub0, DL, MVT::i32),
20055 PtrHi,
20056 DAG.getTargetConstant(AMDGPU::sub1, DL, MVT::i32),
20057 DataLo,
20058 DAG.getTargetConstant(AMDGPU::sub2, DL, MVT::i32),
20059 DataHi,
20060 DAG.getTargetConstant(AMDGPU::sub3, DL, MVT::i32)};
20061
20062 return DAG.getMachineNode(AMDGPU::REG_SEQUENCE, DL, MVT::v4i32, Ops);
20063}
20064
20065//===----------------------------------------------------------------------===//
20066// SI Inline Assembly Support
20067//===----------------------------------------------------------------------===//
20068
20069std::pair<unsigned, const TargetRegisterClass *>
20071 StringRef Constraint,
20072 MVT VT) const {
20073 const SIRegisterInfo *TRI = static_cast<const SIRegisterInfo *>(TRI_);
20074
20075 const TargetRegisterClass *RC = nullptr;
20076 if (Constraint.size() == 1) {
20077 // Check if we cannot determine the bit size of the given value type. This
20078 // can happen, for example, in this situation where we have an empty struct
20079 // (size 0): `call void asm "", "v"({} poison)`-
20080 if (VT == MVT::Other)
20081 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
20082 const unsigned BitWidth = VT.getSizeInBits();
20083 switch (Constraint[0]) {
20084 default:
20085 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
20086 case 's':
20087 case 'r':
20088 switch (BitWidth) {
20089 case 16:
20090 RC = &AMDGPU::SReg_32RegClass;
20091 break;
20092 case 64:
20093 RC = &AMDGPU::SGPR_64RegClass;
20094 break;
20095 default:
20097 if (!RC)
20098 return std::pair(0U, nullptr);
20099 break;
20100 }
20101 break;
20102 case 'v':
20103 switch (BitWidth) {
20104 case 1:
20105 return std::pair(0U, nullptr);
20106 case 16:
20107 RC = Subtarget->useRealTrue16Insts() ? &AMDGPU::VGPR_16RegClass
20108 : &AMDGPU::VGPR_32_Lo256RegClass;
20109 break;
20110 default:
20111 RC = Subtarget->hasVGPRMSBs()
20112 ? TRI->getAlignedLo256VGPRClassForBitWidth(BitWidth)
20113 : TRI->getVGPRClassForBitWidth(BitWidth);
20114 if (!RC)
20115 return std::pair(0U, nullptr);
20116 break;
20117 }
20118 break;
20119 case 'a':
20120 if (!Subtarget->hasMAIInsts())
20121 break;
20122 switch (BitWidth) {
20123 case 1:
20124 return std::pair(0U, nullptr);
20125 case 16:
20126 RC = &AMDGPU::AGPR_32RegClass;
20127 break;
20128 default:
20129 RC = TRI->getAGPRClassForBitWidth(BitWidth);
20130 if (!RC)
20131 return std::pair(0U, nullptr);
20132 break;
20133 }
20134 break;
20135 }
20136 } else if (Constraint == "VA" && Subtarget->hasGFX90AInsts()) {
20137 const unsigned BitWidth = VT.getSizeInBits();
20138 switch (BitWidth) {
20139 case 16:
20140 RC = &AMDGPU::AV_32RegClass;
20141 break;
20142 default:
20143 RC = TRI->getVectorSuperClassForBitWidth(BitWidth);
20144 if (!RC)
20145 return std::pair(0U, nullptr);
20146 break;
20147 }
20148 }
20149
20150 // We actually support i128, i16 and f16 as inline parameters
20151 // even if they are not reported as legal
20152 if (RC && (isTypeLegal(VT) || VT.SimpleTy == MVT::i128 ||
20153 VT.SimpleTy == MVT::i16 || VT.SimpleTy == MVT::f16))
20154 return std::pair(0U, RC);
20155
20156 auto [Kind, Idx, NumRegs] = AMDGPU::parseAsmConstraintPhysReg(Constraint);
20157 if (Kind != '\0') {
20158 if (Kind == 'v') {
20159 RC = &AMDGPU::VGPR_32_Lo256RegClass;
20160 } else if (Kind == 's') {
20161 RC = &AMDGPU::SGPR_32RegClass;
20162 } else if (Kind == 'a') {
20163 RC = &AMDGPU::AGPR_32RegClass;
20164 }
20165
20166 if (RC) {
20167 if (NumRegs > 1) {
20168 if (Idx >= RC->getNumRegs() || Idx + NumRegs - 1 >= RC->getNumRegs())
20169 return std::pair(0U, nullptr);
20170
20171 uint32_t Width = NumRegs * 32;
20172 // Prohibit constraints for register ranges with a width that does not
20173 // match the required type.
20174 if (VT.SimpleTy != MVT::Other && Width != VT.getSizeInBits())
20175 return std::pair(0U, nullptr);
20176
20177 MCRegister Reg = RC->getRegister(Idx);
20179 RC = TRI->getVGPRClassForBitWidth(Width);
20180 else if (SIRegisterInfo::isSGPRClass(RC))
20181 RC = TRI->getSGPRClassForBitWidth(Width);
20182 else if (SIRegisterInfo::isAGPRClass(RC))
20183 RC = TRI->getAGPRClassForBitWidth(Width);
20184 if (RC) {
20185 Reg = TRI->getMatchingSuperReg(Reg, AMDGPU::sub0, RC);
20186 if (!Reg) {
20187 // The register class does not contain the requested register,
20188 // e.g., because it is an SGPR pair that would violate alignment
20189 // requirements.
20190 return std::pair(0U, nullptr);
20191 }
20192 return std::pair(Reg, RC);
20193 }
20194 }
20195
20196 // Reject types that do not fit a single 32-bit register: any scalar wider
20197 // than 32 bits, or a vector that is not exactly 32 bits.
20198 if (VT.SimpleTy != MVT::Other &&
20199 (VT.getSizeInBits() > 32 ||
20200 (VT.isVector() && VT.getSizeInBits() != 32)))
20201 return std::pair(0U, nullptr);
20202 if (RC && Idx < RC->getNumRegs())
20203 return std::pair(RC->getRegister(Idx), RC);
20204 return std::pair(0U, nullptr);
20205 }
20206 }
20207
20208 auto Ret = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
20209 if (Ret.first) {
20210 Ret.second = TRI->getPhysRegBaseClass(Ret.first);
20211 if (!Subtarget->useRealTrue16Insts() &&
20212 TRI->getRegSizeInBits(*Ret.second) == 16)
20213 return std::pair(0U, nullptr);
20214 }
20215
20216 return Ret;
20217}
20218
20219static bool isImmConstraint(StringRef Constraint) {
20220 if (Constraint.size() == 1) {
20221 switch (Constraint[0]) {
20222 default:
20223 break;
20224 case 'I':
20225 case 'J':
20226 case 'A':
20227 case 'B':
20228 case 'C':
20229 return true;
20230 }
20231 } else if (Constraint == "DA" || Constraint == "DB") {
20232 return true;
20233 }
20234 return false;
20235}
20236
20239 if (Constraint.size() == 1) {
20240 switch (Constraint[0]) {
20241 default:
20242 break;
20243 case 's':
20244 case 'v':
20245 case 'a':
20246 return C_RegisterClass;
20247 case 'm':
20248 return C_Memory;
20249 }
20250 } else if (Constraint.size() == 2) {
20251 if (Constraint == "VA")
20252 return C_RegisterClass;
20253 if (Constraint == "RF")
20254 return C_Memory;
20255 }
20256 if (isImmConstraint(Constraint)) {
20257 return C_Other;
20258 }
20259 return TargetLowering::getConstraintType(Constraint);
20260}
20261
20264 if (ConstraintCode == "RF")
20266
20267 return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
20268}
20269
20270static uint64_t clearUnusedBits(uint64_t Val, unsigned Size) {
20272 Val = Val & maskTrailingOnes<uint64_t>(Size);
20273 }
20274 return Val;
20275}
20276
20278 StringRef Constraint,
20279 std::vector<SDValue> &Ops,
20280 SelectionDAG &DAG) const {
20281 if (isImmConstraint(Constraint)) {
20282 uint64_t Val;
20283 if (getAsmOperandConstVal(Op, Val) &&
20284 checkAsmConstraintVal(Op, Constraint, Val)) {
20285 Val = clearUnusedBits(Val, Op.getScalarValueSizeInBits());
20286 Ops.push_back(DAG.getTargetConstant(Val, SDLoc(Op), MVT::i64));
20287 }
20288 } else {
20290 }
20291}
20292
20294 unsigned Size = Op.getScalarValueSizeInBits();
20295 if (Size > 64)
20296 return false;
20297
20298 if (Size == 16 && !Subtarget->has16BitInsts())
20299 return false;
20300
20302 Val = C->getSExtValue();
20303 return true;
20304 }
20306 Val = C->getValueAPF().bitcastToAPInt().getSExtValue();
20307 return true;
20308 }
20310 if (Size != 16 || Op.getNumOperands() != 2)
20311 return false;
20312 if (Op.getOperand(0).isUndef() || Op.getOperand(1).isUndef())
20313 return false;
20314 if (ConstantSDNode *C = V->getConstantSplatNode()) {
20315 Val = C->getSExtValue();
20316 return true;
20317 }
20318 if (ConstantFPSDNode *C = V->getConstantFPSplatNode()) {
20319 Val = C->getValueAPF().bitcastToAPInt().getSExtValue();
20320 return true;
20321 }
20322 }
20323
20324 return false;
20325}
20326
20328 uint64_t Val) const {
20329 if (Constraint.size() == 1) {
20330 switch (Constraint[0]) {
20331 case 'I':
20333 case 'J':
20334 return isInt<16>(Val);
20335 case 'A':
20336 return checkAsmConstraintValA(Op, Val);
20337 case 'B':
20338 return isInt<32>(Val);
20339 case 'C':
20340 return isUInt<32>(clearUnusedBits(Val, Op.getScalarValueSizeInBits())) ||
20342 default:
20343 break;
20344 }
20345 } else if (Constraint.size() == 2) {
20346 if (Constraint == "DA") {
20347 int64_t HiBits = static_cast<int32_t>(Val >> 32);
20348 int64_t LoBits = static_cast<int32_t>(Val);
20349 return checkAsmConstraintValA(Op, HiBits, 32) &&
20350 checkAsmConstraintValA(Op, LoBits, 32);
20351 }
20352 if (Constraint == "DB") {
20353 return true;
20354 }
20355 }
20356 llvm_unreachable("Invalid asm constraint");
20357}
20358
20360 unsigned MaxSize) const {
20361 unsigned Size = std::min<unsigned>(Op.getScalarValueSizeInBits(), MaxSize);
20362 bool HasInv2Pi = Subtarget->hasInv2PiInlineImm();
20363 if (Size == 16) {
20364 MVT VT = Op.getSimpleValueType();
20365 switch (VT.SimpleTy) {
20366 default:
20367 return false;
20368 case MVT::i16:
20369 return AMDGPU::isInlinableLiteralI16(Val, HasInv2Pi);
20370 case MVT::f16:
20371 return AMDGPU::isInlinableLiteralFP16(Val, HasInv2Pi);
20372 case MVT::bf16:
20373 return AMDGPU::isInlinableLiteralBF16(Val, HasInv2Pi);
20374 case MVT::v2i16:
20375 return AMDGPU::getInlineEncodingV2I16(Val).has_value();
20376 case MVT::v2f16:
20377 return AMDGPU::getInlineEncodingV2F16(Val).has_value();
20378 case MVT::v2bf16:
20379 return AMDGPU::getInlineEncodingV2BF16(Val).has_value();
20380 }
20381 }
20382 if ((Size == 32 && AMDGPU::isInlinableLiteral32(Val, HasInv2Pi)) ||
20383 (Size == 64 && AMDGPU::isInlinableLiteral64(Val, HasInv2Pi)))
20384 return true;
20385 return false;
20386}
20387
20388static int getAlignedAGPRClassID(unsigned UnalignedClassID) {
20389 switch (UnalignedClassID) {
20390 case AMDGPU::VReg_64RegClassID:
20391 return AMDGPU::VReg_64_Align2RegClassID;
20392 case AMDGPU::VReg_96RegClassID:
20393 return AMDGPU::VReg_96_Align2RegClassID;
20394 case AMDGPU::VReg_128RegClassID:
20395 return AMDGPU::VReg_128_Align2RegClassID;
20396 case AMDGPU::VReg_160RegClassID:
20397 return AMDGPU::VReg_160_Align2RegClassID;
20398 case AMDGPU::VReg_192RegClassID:
20399 return AMDGPU::VReg_192_Align2RegClassID;
20400 case AMDGPU::VReg_224RegClassID:
20401 return AMDGPU::VReg_224_Align2RegClassID;
20402 case AMDGPU::VReg_256RegClassID:
20403 return AMDGPU::VReg_256_Align2RegClassID;
20404 case AMDGPU::VReg_288RegClassID:
20405 return AMDGPU::VReg_288_Align2RegClassID;
20406 case AMDGPU::VReg_320RegClassID:
20407 return AMDGPU::VReg_320_Align2RegClassID;
20408 case AMDGPU::VReg_352RegClassID:
20409 return AMDGPU::VReg_352_Align2RegClassID;
20410 case AMDGPU::VReg_384RegClassID:
20411 return AMDGPU::VReg_384_Align2RegClassID;
20412 case AMDGPU::VReg_512RegClassID:
20413 return AMDGPU::VReg_512_Align2RegClassID;
20414 case AMDGPU::VReg_1024RegClassID:
20415 return AMDGPU::VReg_1024_Align2RegClassID;
20416 case AMDGPU::AReg_64RegClassID:
20417 return AMDGPU::AReg_64_Align2RegClassID;
20418 case AMDGPU::AReg_96RegClassID:
20419 return AMDGPU::AReg_96_Align2RegClassID;
20420 case AMDGPU::AReg_128RegClassID:
20421 return AMDGPU::AReg_128_Align2RegClassID;
20422 case AMDGPU::AReg_160RegClassID:
20423 return AMDGPU::AReg_160_Align2RegClassID;
20424 case AMDGPU::AReg_192RegClassID:
20425 return AMDGPU::AReg_192_Align2RegClassID;
20426 case AMDGPU::AReg_256RegClassID:
20427 return AMDGPU::AReg_256_Align2RegClassID;
20428 case AMDGPU::AReg_512RegClassID:
20429 return AMDGPU::AReg_512_Align2RegClassID;
20430 case AMDGPU::AReg_1024RegClassID:
20431 return AMDGPU::AReg_1024_Align2RegClassID;
20432 default:
20433 return -1;
20434 }
20435}
20436
20437// Figure out which registers should be reserved for stack access. Only after
20438// the function is legalized do we know all of the non-spill stack objects or if
20439// calls are present.
20443 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
20444 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
20445 const SIInstrInfo *TII = ST.getInstrInfo();
20446
20447 if (Info->isEntryFunction()) {
20448 // Callable functions have fixed registers used for stack access.
20450 }
20451
20452 // TODO: Move this logic to getReservedRegs()
20453 // Reserve the SGPR(s) to save/restore EXEC for WWM spill/copy handling.
20454 unsigned MaxNumSGPRs = ST.getMaxNumSGPRs(MF);
20455 Register SReg = ST.isWave32()
20456 ? AMDGPU::SGPR_32RegClass.getRegister(MaxNumSGPRs - 1)
20457 : TRI->getAlignedHighSGPRForRC(MF, /*Align=*/2,
20458 &AMDGPU::SGPR_64RegClass);
20459 Info->setSGPRForEXECCopy(SReg);
20460
20461 assert(!TRI->isSubRegister(Info->getScratchRSrcReg(),
20462 Info->getStackPtrOffsetReg()));
20463 if (Info->getStackPtrOffsetReg() != AMDGPU::SP_REG)
20464 MRI.replaceRegWith(AMDGPU::SP_REG, Info->getStackPtrOffsetReg());
20465
20466 // We need to worry about replacing the default register with itself in case
20467 // of MIR testcases missing the MFI.
20468 if (Info->getScratchRSrcReg() != AMDGPU::PRIVATE_RSRC_REG)
20469 MRI.replaceRegWith(AMDGPU::PRIVATE_RSRC_REG, Info->getScratchRSrcReg());
20470
20471 if (Info->getFrameOffsetReg() != AMDGPU::FP_REG)
20472 MRI.replaceRegWith(AMDGPU::FP_REG, Info->getFrameOffsetReg());
20473
20474 Info->limitOccupancy(MF);
20475
20476 if (ST.isWave32() && !MF.empty()) {
20477 for (auto &MBB : MF) {
20478 for (auto &MI : MBB) {
20479 TII->fixImplicitOperands(MI);
20480 }
20481 }
20482 }
20483
20484 // FIXME: This is a hack to fixup AGPR classes to use the properly aligned
20485 // classes if required. Ideally the register class constraints would differ
20486 // per-subtarget, but there's no easy way to achieve that right now. This is
20487 // not a problem for VGPRs because the correctly aligned VGPR class is implied
20488 // from using them as the register class for legal types.
20489 if (ST.needsAlignedVGPRs()) {
20490 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
20491 const Register Reg = Register::index2VirtReg(I);
20492 const TargetRegisterClass *RC = MRI.getRegClassOrNull(Reg);
20493 if (!RC)
20494 continue;
20495 int NewClassID = getAlignedAGPRClassID(RC->getID());
20496 if (NewClassID != -1)
20497 MRI.setRegClass(Reg, TRI->getRegClass(NewClassID));
20498 }
20499 }
20500
20502}
20503
20506 const APInt &DemandedElts,
20507 const SelectionDAG &DAG,
20508 unsigned Depth) const {
20509 Known.resetAll();
20510 unsigned Opc = Op.getOpcode();
20511 switch (Opc) {
20513 unsigned IID = Op.getConstantOperandVal(0);
20514 switch (IID) {
20515 case Intrinsic::amdgcn_mbcnt_lo:
20516 case Intrinsic::amdgcn_mbcnt_hi: {
20517 const GCNSubtarget &ST =
20519 // Wave64 mbcnt_lo returns at most 32 + src1. Otherwise these return at
20520 // most 31 + src1.
20521 Known.Zero.setBitsFrom(
20522 IID == Intrinsic::amdgcn_mbcnt_lo ? ST.getWavefrontSizeLog2() : 5);
20523 KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(2), Depth + 1);
20524 Known = KnownBits::add(Known, Known2);
20525 return;
20526 }
20527 }
20528 break;
20529 }
20530 }
20532 Op, Known, DemandedElts, DAG, Depth);
20533}
20534
20536 KnownBits &Known, const MachineFunction &MF, Align Alignment) const {
20538
20539 // Set the high bits to zero based on the maximum allowed scratch size per
20540 // wave. We can't use vaddr in MUBUF instructions if we don't know the address
20541 // calculation won't overflow, so assume the sign bit is never set.
20542 Known.Zero.setHighBits(getSubtarget()->getKnownHighZeroBitsForFrameIndex());
20543}
20544
20547 unsigned Dim) {
20548 unsigned MaxValue =
20549 ST.getMaxWorkitemID(VT.getMachineFunction().getFunction(), Dim);
20550 Known.Zero.setHighBits(llvm::countl_zero(MaxValue));
20551}
20552
20554 KnownBits &Known, const APInt &DemandedElts,
20555 unsigned BFEWidth, bool SExt, unsigned Depth) {
20557 const MachineOperand &Src1 = MI.getOperand(2);
20558
20559 unsigned Src1Cst = 0;
20560 if (Src1.isImm()) {
20561 Src1Cst = Src1.getImm();
20562 } else if (Src1.isReg()) {
20563 auto Cst = getIConstantVRegValWithLookThrough(Src1.getReg(), MRI);
20564 if (!Cst)
20565 return;
20566 Src1Cst = Cst->Value.getZExtValue();
20567 } else {
20568 return;
20569 }
20570
20571 // Offset is at bits [4:0] for 32 bit, [5:0] for 64 bit.
20572 // Width is always [22:16].
20573 const unsigned Offset =
20574 Src1Cst & maskTrailingOnes<unsigned>((BFEWidth == 32) ? 5 : 6);
20575 const unsigned Width = (Src1Cst >> 16) & maskTrailingOnes<unsigned>(6);
20576
20577 if (Width >= BFEWidth) // Ill-formed.
20578 return;
20579
20580 VT.computeKnownBitsImpl(MI.getOperand(1).getReg(), Known, DemandedElts,
20581 Depth + 1);
20582
20583 Known = Known.extractBits(Width, Offset);
20584
20585 if (SExt)
20586 Known = Known.sext(BFEWidth);
20587 else
20588 Known = Known.zext(BFEWidth);
20589}
20590
20593 const APInt &DemandedElts, const MachineRegisterInfo &MRI,
20594 unsigned Depth) const {
20595 Known.resetAll();
20596 const MachineInstr *MI = MRI.getVRegDef(R);
20597 switch (MI->getOpcode()) {
20598 case AMDGPU::S_BFE_I32:
20599 return knownBitsForSBFE(*MI, VT, Known, DemandedElts, /*Width=*/32,
20600 /*SExt=*/true, Depth);
20601 case AMDGPU::S_BFE_U32:
20602 return knownBitsForSBFE(*MI, VT, Known, DemandedElts, /*Width=*/32,
20603 /*SExt=*/false, Depth);
20604 case AMDGPU::S_BFE_I64:
20605 return knownBitsForSBFE(*MI, VT, Known, DemandedElts, /*Width=*/64,
20606 /*SExt=*/true, Depth);
20607 case AMDGPU::S_BFE_U64:
20608 return knownBitsForSBFE(*MI, VT, Known, DemandedElts, /*Width=*/64,
20609 /*SExt=*/false, Depth);
20610 case AMDGPU::G_INTRINSIC:
20611 case AMDGPU::G_INTRINSIC_CONVERGENT: {
20612 Intrinsic::ID IID = cast<GIntrinsic>(MI)->getIntrinsicID();
20613 switch (IID) {
20614 case Intrinsic::amdgcn_workitem_id_x:
20616 break;
20617 case Intrinsic::amdgcn_workitem_id_y:
20619 break;
20620 case Intrinsic::amdgcn_workitem_id_z:
20622 break;
20623 case Intrinsic::amdgcn_mbcnt_lo:
20624 case Intrinsic::amdgcn_mbcnt_hi: {
20625 // Wave64 mbcnt_lo returns at most 32 + src1. Otherwise these return at
20626 // most 31 + src1.
20627 Known.Zero.setBitsFrom(IID == Intrinsic::amdgcn_mbcnt_lo
20628 ? getSubtarget()->getWavefrontSizeLog2()
20629 : 5);
20630 KnownBits Known2;
20631 VT.computeKnownBitsImpl(MI->getOperand(3).getReg(), Known2, DemandedElts,
20632 Depth + 1);
20633 Known = KnownBits::add(Known, Known2);
20634 break;
20635 }
20636 case Intrinsic::amdgcn_groupstaticsize: {
20637 // We can report everything over the maximum size as 0. We can't report
20638 // based on the actual size because we don't know if it's accurate or not
20639 // at any given point.
20640 Known.Zero.setHighBits(
20641 llvm::countl_zero(getSubtarget()->getAddressableLocalMemorySize()));
20642 break;
20643 }
20644 case Intrinsic::amdgcn_readfirstlane:
20645 case Intrinsic::amdgcn_readlane: {
20646 // Result is the data operand's value from some lane.
20647 VT.computeKnownBitsImpl(MI->getOperand(2).getReg(), Known, DemandedElts,
20648 Depth + 1);
20649 break;
20650 }
20651 case Intrinsic::amdgcn_mul_u24:
20652 case Intrinsic::amdgcn_mul_i24: {
20653 Register LHS = MI->getOperand(2).getReg();
20654 Register RHS = MI->getOperand(3).getReg();
20655
20656 KnownBits RHSKnown;
20657 VT.computeKnownBitsImpl(RHS, RHSKnown, DemandedElts, Depth + 1);
20658
20659 bool SelfMultiply = LHS == RHS;
20660
20661 // For a self-multiply both operands are the same value, so RHSKnown also
20662 // describes the LHS; avoid the duplicated query.
20663 KnownBits LHSKnown;
20664 if (SelfMultiply)
20665 LHSKnown = RHSKnown;
20666 else
20667 VT.computeKnownBitsImpl(LHS, LHSKnown, DemandedElts, Depth + 1);
20668
20669 if (SelfMultiply)
20670 SelfMultiply &= isGuaranteedNotToBeUndef(LHS, MRI, Depth + 1);
20671
20672 Known = computeKnownBitsForMul24(LHSKnown, RHSKnown, Known.getBitWidth(),
20673 IID == Intrinsic::amdgcn_mul_i24,
20674 SelfMultiply);
20675 break;
20676 }
20677 }
20678 break;
20679 }
20680 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
20681 Known.Zero.setHighBits(24);
20682 break;
20683 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
20684 Known.Zero.setHighBits(16);
20685 break;
20686 case AMDGPU::G_AMDGPU_COPY_SCC_VCC:
20687 // G_AMDGPU_COPY_SCC_VCC converts a uniform boolean in VCC to SGPR s32,
20688 // producing exactly 0 or 1.
20689 Known.Zero.setHighBits(Known.getBitWidth() - 1);
20690 break;
20691 case AMDGPU::G_AMDGPU_SMED3:
20692 case AMDGPU::G_AMDGPU_UMED3: {
20693 auto [Dst, Src0, Src1, Src2] = MI->getFirst4Regs();
20694
20695 KnownBits Known2;
20696 VT.computeKnownBitsImpl(Src2, Known2, DemandedElts, Depth + 1);
20697 if (Known2.isUnknown())
20698 break;
20699
20700 KnownBits Known1;
20701 VT.computeKnownBitsImpl(Src1, Known1, DemandedElts, Depth + 1);
20702 if (Known1.isUnknown())
20703 break;
20704
20705 KnownBits Known0;
20706 VT.computeKnownBitsImpl(Src0, Known0, DemandedElts, Depth + 1);
20707 if (Known0.isUnknown())
20708 break;
20709
20710 // TODO: Handle LeadZero/LeadOne from UMIN/UMAX handling.
20711 Known.Zero = Known0.Zero & Known1.Zero & Known2.Zero;
20712 Known.One = Known0.One & Known1.One & Known2.One;
20713 break;
20714 }
20715 }
20716}
20717
20720 unsigned Depth) const {
20721 const MachineInstr *MI = MRI.getVRegDef(R);
20722 if (auto *GI = dyn_cast<GIntrinsic>(MI)) {
20723 // FIXME: Can this move to generic code? What about the case where the call
20724 // site specifies a lower alignment?
20725 Intrinsic::ID IID = GI->getIntrinsicID();
20727 AttributeList Attrs =
20728 Intrinsic::getAttributes(Ctx, IID, Intrinsic::getType(Ctx, IID));
20729 if (MaybeAlign RetAlign = Attrs.getRetAlignment())
20730 return *RetAlign;
20731 }
20732 return Align(1);
20733}
20734
20736 MachineLoop *ML, const MachineBasicBlock *BlockToAlign) const {
20738 const Align CacheLineAlign = Align(64);
20739
20740 // GFX950: Prevent an 8-byte instruction at the block being aligned from being
20741 // split by the 32-byte instruction fetch window boundary. This avoids a
20742 // significant fetch delay after a backward branch. We use 32-byte alignment
20743 // with max padding of 4 bytes (one s_nop), see
20744 // getMaxPermittedBytesForAlignment().
20745 if (ML && getSubtarget()->getCLOpts().loop_alignment &&
20746 getSubtarget()->hasLoopHeadInstSplitSensitivity()) {
20747 // Loop rotation can make the backedge destination a block other than the
20748 // LoopInfo header, so prefer the block the caller is actually aligning.
20749 if (!BlockToAlign)
20750 BlockToAlign = ML->getHeader();
20751 // Respect user-specified or previously set alignment.
20752 if (BlockToAlign->getAlignment() != PrefAlign)
20753 return BlockToAlign->getAlignment();
20754 if (needsFetchWindowAlignment(*BlockToAlign))
20755 return Align(32);
20756 }
20757
20758 // Pre-GFX10 target did not benefit from loop alignment
20759 if (!ML || !getSubtarget()->getCLOpts().loop_alignment ||
20760 !getSubtarget()->hasInstPrefetch() ||
20761 getSubtarget()->hasInstFwdPrefetchBug())
20762 return PrefAlign;
20763
20764 // On GFX10 I$ is 4 x 64 bytes cache lines.
20765 // By default prefetcher keeps one cache line behind and reads two ahead.
20766 // We can modify it with S_INST_PREFETCH for larger loops to have two lines
20767 // behind and one ahead.
20768 // Therefor we can benefit from aligning loop headers if loop fits 192 bytes.
20769 // If loop fits 64 bytes it always spans no more than two cache lines and
20770 // does not need an alignment.
20771 // Else if loop is less or equal 128 bytes we do not need to modify prefetch,
20772 // Else if loop is less or equal 192 bytes we need two lines behind.
20773
20775 const MachineBasicBlock *Header = ML->getHeader();
20776 if (Header->getAlignment() != PrefAlign)
20777 return Header->getAlignment(); // Already processed.
20778
20779 unsigned LoopSize = 0;
20780 for (const MachineBasicBlock *MBB : ML->blocks()) {
20781 // If inner loop block is aligned assume in average half of the alignment
20782 // size to be added as nops.
20783 if (MBB != Header)
20784 LoopSize += MBB->getAlignment().value() / 2;
20785
20786 for (const MachineInstr &MI : *MBB) {
20787 LoopSize += TII->getInstSizeInBytes(MI);
20788 if (LoopSize > 192)
20789 return PrefAlign;
20790 }
20791 }
20792
20793 if (LoopSize <= 64)
20794 return PrefAlign;
20795
20796 if (LoopSize <= 128)
20797 return CacheLineAlign;
20798
20799 // If any of parent loops is surrounded by prefetch instructions do not
20800 // insert new for inner loop, which would reset parent's settings.
20801 for (MachineLoop *P = ML->getParentLoop(); P; P = P->getParentLoop()) {
20802 if (MachineBasicBlock *Exit = P->getExitBlock()) {
20803 auto I = Exit->getFirstNonDebugInstr();
20804 if (I != Exit->end() && I->getOpcode() == AMDGPU::S_INST_PREFETCH)
20805 return CacheLineAlign;
20806 }
20807 }
20808
20809 MachineBasicBlock *Pre = ML->getLoopPreheader();
20810 MachineBasicBlock *Exit = ML->getExitBlock();
20811
20812 if (Pre && Exit) {
20813 auto PreTerm = Pre->getFirstTerminator();
20814 if (PreTerm == Pre->begin() ||
20815 std::prev(PreTerm)->getOpcode() != AMDGPU::S_INST_PREFETCH)
20816 BuildMI(*Pre, PreTerm, DebugLoc(), TII->get(AMDGPU::S_INST_PREFETCH))
20817 .addImm(1); // prefetch 2 lines behind PC
20818
20819 auto ExitHead = Exit->getFirstNonDebugInstr();
20820 if (ExitHead == Exit->end() ||
20821 ExitHead->getOpcode() != AMDGPU::S_INST_PREFETCH)
20822 BuildMI(*Exit, ExitHead, DebugLoc(), TII->get(AMDGPU::S_INST_PREFETCH))
20823 .addImm(2); // prefetch 1 line behind PC
20824 }
20825
20826 return CacheLineAlign;
20827}
20828
20830 MachineBasicBlock *MBB) const {
20831 // GFX950: Limit padding to 4 bytes (one s_nop) for blocks where an 8-byte
20832 // instruction could be split by the 32-byte fetch window boundary.
20833 // See getPrefLoopAlignment() for context.
20834 if (needsFetchWindowAlignment(*MBB))
20835 return 4;
20837}
20838
20839bool SITargetLowering::needsFetchWindowAlignment(
20840 const MachineBasicBlock &MBB) const {
20841 if (!getSubtarget()->hasLoopHeadInstSplitSensitivity())
20842 return false;
20844 for (const MachineInstr &MI : MBB) {
20845 if (MI.isMetaInstruction())
20846 continue;
20847 // Instructions larger than 4 bytes can be split by a 32-byte boundary.
20848 return TII->getInstSizeInBytes(MI) > 4;
20849 }
20850 return false;
20851}
20852
20853[[maybe_unused]]
20854static bool isCopyFromRegOfInlineAsm(const SDNode *N) {
20855 assert(N->getOpcode() == ISD::CopyFromReg);
20856 do {
20857 // Follow the chain until we find an INLINEASM node.
20858 N = N->getOperand(0).getNode();
20859 if (N->getOpcode() == ISD::INLINEASM || N->getOpcode() == ISD::INLINEASM_BR)
20860 return true;
20861 } while (N->getOpcode() == ISD::CopyFromReg);
20862 return false;
20863}
20864
20867 UniformityInfo *UA) const {
20868 switch (N->getOpcode()) {
20869 case ISD::CopyFromReg: {
20870 const RegisterSDNode *R = cast<RegisterSDNode>(N->getOperand(1));
20871 const MachineRegisterInfo &MRI = FLI->MF->getRegInfo();
20872 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
20873 Register Reg = R->getReg();
20874
20875 // FIXME: Why does this need to consider isLiveIn?
20876 if (Reg.isPhysical() || MRI.isLiveIn(Reg))
20877 return !TRI->isSGPRReg(MRI, Reg);
20878
20879 if (const Value *V = FLI->getValueFromVirtualReg(R->getReg()))
20880 return UA->isDivergentAtDef(V);
20881
20883 return !TRI->isSGPRReg(MRI, Reg);
20884 }
20885 case ISD::LOAD: {
20886 const LoadSDNode *L = cast<LoadSDNode>(N);
20887 unsigned AS = L->getAddressSpace();
20888 // A flat load may access private memory, and a VGPR load reads each lane's
20889 // own registers.
20890 return AS == AMDGPUAS::PRIVATE_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS ||
20891 AS == AMDGPUAS::VGPR;
20892 }
20893 // As above, after the pre-ISel combine.
20894 case AMDGPUISD::REG_LOAD:
20895 return true;
20896 case ISD::CALLSEQ_END:
20897 return true;
20899 return AMDGPU::isIntrinsicSourceOfDivergence(N->getConstantOperandVal(0));
20901 return AMDGPU::isIntrinsicSourceOfDivergence(N->getConstantOperandVal(1));
20902 case AMDGPUISD::ATOMIC_CMP_SWAP:
20903 case AMDGPUISD::BUFFER_ATOMIC_SWAP:
20904 case AMDGPUISD::BUFFER_ATOMIC_ADD:
20905 case AMDGPUISD::BUFFER_ATOMIC_SUB:
20906 case AMDGPUISD::BUFFER_ATOMIC_SMIN:
20907 case AMDGPUISD::BUFFER_ATOMIC_UMIN:
20908 case AMDGPUISD::BUFFER_ATOMIC_SMAX:
20909 case AMDGPUISD::BUFFER_ATOMIC_UMAX:
20910 case AMDGPUISD::BUFFER_ATOMIC_AND:
20911 case AMDGPUISD::BUFFER_ATOMIC_OR:
20912 case AMDGPUISD::BUFFER_ATOMIC_XOR:
20913 case AMDGPUISD::BUFFER_ATOMIC_INC:
20914 case AMDGPUISD::BUFFER_ATOMIC_DEC:
20915 case AMDGPUISD::BUFFER_ATOMIC_CMPSWAP:
20916 case AMDGPUISD::BUFFER_ATOMIC_FADD:
20917 case AMDGPUISD::BUFFER_ATOMIC_FMIN:
20918 case AMDGPUISD::BUFFER_ATOMIC_FMAX:
20919 // Target-specific read-modify-write atomics are sources of divergence.
20920 return true;
20921 default:
20922 if (auto *A = dyn_cast<AtomicSDNode>(N)) {
20923 // Generic read-modify-write atomics are sources of divergence.
20924 return A->readMem() && A->writeMem();
20925 }
20926 return false;
20927 }
20928}
20929
20931 EVT VT) const {
20932 switch (VT.getScalarType().getSimpleVT().SimpleTy) {
20933 case MVT::f32:
20935 case MVT::f64:
20936 case MVT::f16:
20938 default:
20939 return false;
20940 }
20941}
20942
20944 LLT Ty, const MachineFunction &MF) const {
20945 switch (Ty.getScalarSizeInBits()) {
20946 case 32:
20947 return !denormalModeIsFlushAllF32(MF);
20948 case 64:
20949 case 16:
20950 return !denormalModeIsFlushAllF64F16(MF);
20951 default:
20952 return false;
20953 }
20954}
20955
20957 const APInt &DemandedElts,
20958 const SelectionDAG &DAG,
20959 bool SNaN,
20960 unsigned Depth) const {
20961 if (Op.getOpcode() == AMDGPUISD::CLAMP) {
20962 const MachineFunction &MF = DAG.getMachineFunction();
20964
20965 if (Info->getMode().DX10Clamp)
20966 return true; // Clamped to 0.
20967 return DAG.isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
20968 }
20969
20971 DAG, SNaN, Depth);
20972}
20973
20974namespace {
20975
20976/// Why a floating-point atomic instruction which may flush denormals is
20977/// acceptable.
20978enum class AtomicFlushDenormalReason {
20979 Native,
20980 IEEE,
20981 IgnoreDenormalMode,
20982 FunctionFlushesDenormals
20983};
20984
20985/// Why a native floating-point atomic instruction is acceptable for a global
20986/// memory address.
20987enum class GlobalFPAtomicLegality {
20988 Illegal,
20989 AgentScopeFineGrainedRemoteMemory,
20990 EmulatedSystemScope,
20991 NoRemoteMemory,
20992 NoFineGrainedMemory
20993};
20994
20995} // end anonymous namespace
20996
20997// On older subtargets, global FP atomic instructions have a hardcoded FP mode
20998// and do not support FP32 denormals, and only support v2f16/f64 denormals.
20999static AtomicFlushDenormalReason
21001 if (RMW->hasMetadata(LLVMContext::MD_atomic_ignore_denormal_mode))
21002 return AtomicFlushDenormalReason::IgnoreDenormalMode;
21003
21004 const fltSemantics &Flt = RMW->getType()->getScalarType()->getFltSemantics();
21005 auto DenormMode = RMW->getFunction()->getDenormalMode(Flt);
21006 return DenormMode == DenormalMode::getPreserveSign()
21007 ? AtomicFlushDenormalReason::FunctionFlushesDenormals
21008 : AtomicFlushDenormalReason::IEEE;
21009}
21010
21011static OptimizationRemark
21013 GlobalFPAtomicLegality MemLegality,
21014 AtomicFlushDenormalReason DenormReason) {
21015 LLVMContext &Ctx = RMW->getContext();
21016 StringRef MemScope = Ctx.getSyncScopeName(RMW->getSyncScopeID()).value_or("");
21017 if (MemScope.empty())
21018 MemScope = "system";
21019
21020 OptimizationRemark R(DEBUG_TYPE, "Passed", RMW);
21021 R << "hardware instruction generated for atomic "
21022 << ore::NV("Operation", RMW->getOperationName(RMW->getOperation()))
21023 << " at " << ore::NV("SyncScope", MemScope) << " scope since ";
21024
21025 switch (MemLegality) {
21026 case GlobalFPAtomicLegality::AgentScopeFineGrainedRemoteMemory:
21027 R << "fine-grained remote memory atomics work below system scope";
21028 break;
21029 case GlobalFPAtomicLegality::EmulatedSystemScope:
21030 R << "system scope atomics are emulated in hardware";
21031 break;
21032 case GlobalFPAtomicLegality::NoRemoteMemory:
21033 R << "memory is not remote (!amdgpu.no.remote.memory)";
21034 break;
21035 case GlobalFPAtomicLegality::NoFineGrainedMemory:
21036 R << "memory is not fine-grained (!amdgpu.no.fine.grained.memory)";
21037 break;
21038 case GlobalFPAtomicLegality::Illegal:
21039 llvm_unreachable("remark for illegal atomic");
21040 }
21041
21042 switch (DenormReason) {
21043 case AtomicFlushDenormalReason::Native:
21044 break;
21045 case AtomicFlushDenormalReason::IgnoreDenormalMode:
21046 R << ", and denormals may be flushed (!atomic.ignore.denormal.mode)";
21047 break;
21048 case AtomicFlushDenormalReason::FunctionFlushesDenormals:
21049 R << ", and the floating-point environment flushes denormals";
21050 break;
21051 case AtomicFlushDenormalReason::IEEE:
21052 llvm_unreachable("remark for illegal atomic");
21053 }
21054
21055 return R;
21056}
21057
21058static bool isV2F16OrV2BF16(Type *Ty) {
21059 if (auto *VT = dyn_cast<FixedVectorType>(Ty)) {
21060 Type *EltTy = VT->getElementType();
21061 return VT->getNumElements() == 2 &&
21062 (EltTy->isHalfTy() || EltTy->isBFloatTy());
21063 }
21064
21065 return false;
21066}
21067
21068static bool isV2F16(Type *Ty) {
21070 return VT && VT->getNumElements() == 2 && VT->getElementType()->isHalfTy();
21071}
21072
21073static bool isV2BF16(Type *Ty) {
21075 return VT && VT->getNumElements() == 2 && VT->getElementType()->isBFloatTy();
21076}
21077
21078/// \return true if atomicrmw integer ops work for the type.
21079static bool isAtomicRMWLegalIntTy(Type *Ty) {
21080 if (auto *IT = dyn_cast<IntegerType>(Ty)) {
21081 unsigned BW = IT->getBitWidth();
21082 return BW == 32 || BW == 64;
21083 }
21084
21085 return false;
21086}
21087
21088/// \return true if this atomicrmw xchg type can be selected.
21089static bool isAtomicRMWLegalXChgTy(const AtomicRMWInst *RMW) {
21090 Type *Ty = RMW->getType();
21091 if (isAtomicRMWLegalIntTy(Ty))
21092 return true;
21093
21094 if (PointerType *PT = dyn_cast<PointerType>(Ty)) {
21095 const DataLayout &DL = RMW->getFunction()->getDataLayout();
21096 unsigned BW = DL.getPointerSizeInBits(PT->getAddressSpace());
21097 return BW == 32 || BW == 64;
21098 }
21099
21100 if (Ty->isFloatTy() || Ty->isDoubleTy())
21101 return true;
21102
21104 return VT->getNumElements() == 2 &&
21105 VT->getElementType()->getPrimitiveSizeInBits() == 16;
21106 }
21107
21108 return false;
21109}
21110
21111/// \returns whether it's valid to emit a native instruction for \p RMW, and
21112/// why, based on the properties of the target memory.
21113static GlobalFPAtomicLegality
21115 const AtomicRMWInst *RMW, bool HasSystemScope) {
21116 // The remote/fine-grained access logic is different from the integer
21117 // atomics. Without AgentScopeFineGrainedRemoteMemoryAtomics support,
21118 // fine-grained access does not work, even for a device local allocation.
21119 //
21120 // With AgentScopeFineGrainedRemoteMemoryAtomics, system scoped device local
21121 // allocations work.
21122 if (HasSystemScope) {
21123 if (Subtarget.hasAgentScopeFineGrainedRemoteMemoryAtomics() &&
21124 RMW->hasMetadata("amdgpu.no.remote.memory"))
21125 return GlobalFPAtomicLegality::NoRemoteMemory;
21126 if (Subtarget.hasEmulatedSystemScopeAtomics())
21127 return GlobalFPAtomicLegality::EmulatedSystemScope;
21128 } else if (Subtarget.hasAgentScopeFineGrainedRemoteMemoryAtomics())
21129 return GlobalFPAtomicLegality::AgentScopeFineGrainedRemoteMemory;
21130
21131 return RMW->hasMetadata("amdgpu.no.fine.grained.memory")
21132 ? GlobalFPAtomicLegality::NoFineGrainedMemory
21133 : GlobalFPAtomicLegality::Illegal;
21134}
21135
21136/// \return Action to perform on AtomicRMWInsts for integer operations.
21143
21144/// Return if a flat address space atomicrmw can access private memory.
21146 const MDNode *MD = I->getMetadata(LLVMContext::MD_noalias_addrspace);
21147 return !MD ||
21149}
21150
21153 // For GAS, lower to flat atomic.
21154 return STI.hasGloballyAddressableScratch()
21157}
21158
21161 unsigned AS = RMW->getPointerAddressSpace();
21162 if (AS == AMDGPUAS::PRIVATE_ADDRESS)
21164
21165 // Only the executing lane can access its view of the VGPRs, so as with
21166 // private memory there is nothing to be atomic with respect to.
21167 if (AS == AMDGPUAS::VGPR)
21169
21170 // 64-bit flat atomics that dynamically reside in private memory will silently
21171 // be dropped.
21172 //
21173 // Note that we will emit a new copy of the original atomic in the expansion,
21174 // which will be incrementally relegalized.
21175 const DataLayout &DL = RMW->getFunction()->getDataLayout();
21176 if (AS == AMDGPUAS::FLAT_ADDRESS &&
21177 DL.getTypeSizeInBits(RMW->getType()) == 64 &&
21180
21181 GlobalFPAtomicLegality MemLegality = GlobalFPAtomicLegality::Illegal;
21182 AtomicFlushDenormalReason DenormReason = AtomicFlushDenormalReason::Native;
21183 auto ReportHWInst = [&](TargetLowering::AtomicExpansionKind Kind) {
21185 ORE.emit([&]() {
21186 return emitAtomicRMWLegalRemark(RMW, MemLegality, DenormReason);
21187 });
21188 return Kind;
21189 };
21190
21191 auto SSID = RMW->getSyncScopeID();
21192 bool HasSystemScope =
21193 SSID == SyncScope::System ||
21195 getTargetMachine().getTargetTriple(), AtomicScope::System,
21196 /*OneAddressSpace=*/true));
21197
21198 auto Op = RMW->getOperation();
21199 switch (Op) {
21201 // PCIe supports add and xchg for system atomics.
21202 return isAtomicRMWLegalXChgTy(RMW)
21205 case AtomicRMWInst::Add:
21206 // PCIe supports add and xchg for system atomics.
21208 case AtomicRMWInst::Sub:
21209 case AtomicRMWInst::And:
21210 case AtomicRMWInst::Or:
21211 case AtomicRMWInst::Xor:
21212 case AtomicRMWInst::Max:
21213 case AtomicRMWInst::Min:
21220 if (Op == AtomicRMWInst::USubCond && !Subtarget->hasCondSubInsts())
21222 if (Op == AtomicRMWInst::USubSat) {
21223 // The global and buffer forms predate the LDS and flat ones.
21224 if (!Subtarget->hasSubClampInsts() ||
21225 (AS == AMDGPUAS::LOCAL_ADDRESS &&
21226 !Subtarget->hasAtomicDsCondSubClampInsts()) ||
21227 (AS == AMDGPUAS::FLAT_ADDRESS &&
21228 !Subtarget->hasAtomicCondSubClampFlatInsts()))
21230 }
21232 auto *IT = dyn_cast<IntegerType>(RMW->getType());
21233 if (!IT || IT->getBitWidth() != 32)
21235 }
21236
21239 if (Subtarget->hasEmulatedSystemScopeAtomics())
21241
21242 // On most subtargets, for atomicrmw operations other than add/xchg,
21243 // whether or not the instructions will behave correctly depends on where
21244 // the address physically resides and what interconnect is used in the
21245 // system configuration. On some some targets the instruction will nop,
21246 // and in others synchronization will only occur at degraded device scope.
21247 //
21248 // If the allocation is known local to the device, the instructions should
21249 // work correctly.
21250 if (RMW->hasMetadata("amdgpu.no.remote.memory"))
21252
21253 // If fine-grained remote memory works at device scope, we don't need to
21254 // do anything.
21255 if (!HasSystemScope &&
21256 Subtarget->hasAgentScopeFineGrainedRemoteMemoryAtomics())
21258
21259 // If we are targeting a remote allocated address, it depends what kind of
21260 // allocation the address belongs to.
21261 //
21262 // If the allocation is fine-grained (in host memory, or in PCIe peer
21263 // device memory), the operation will fail depending on the target.
21264 //
21265 // Note fine-grained host memory access does work on APUs or if XGMI is
21266 // used, but we do not know if we are targeting an APU or the system
21267 // configuration from the ISA version/target-cpu.
21268 if (RMW->hasMetadata("amdgpu.no.fine.grained.memory"))
21270
21273 // Atomic sub/or/xor do not work over PCI express, but atomic add
21274 // does. InstCombine transforms these with 0 to or, so undo that.
21275 // Sub-word types are not selectable and take the cmpxchg expansion.
21276 if (const Constant *ConstVal = dyn_cast<Constant>(RMW->getValOperand());
21277 ConstVal && ConstVal->isNullValue() &&
21280 }
21281
21282 // If the allocation could be in remote, fine-grained memory, the rmw
21283 // instructions may fail. cmpxchg should work, so emit that. On some
21284 // system configurations, PCIe atomics aren't supported so cmpxchg won't
21285 // even work, so you're out of luck anyway.
21286
21287 // In summary:
21288 //
21289 // Cases that may fail:
21290 // - fine-grained pinned host memory
21291 // - fine-grained migratable host memory
21292 // - fine-grained PCIe peer device
21293 //
21294 // Cases that should work, but may be treated overly conservatively.
21295 // - fine-grained host memory on an APU
21296 // - fine-grained XGMI peer device
21298 }
21299
21301 }
21302 case AtomicRMWInst::FAdd: {
21303 Type *Ty = RMW->getType();
21304
21305 // TODO: Handle REGION_ADDRESS
21306 if (AS == AMDGPUAS::LOCAL_ADDRESS) {
21307 // DS F32 FP atomics do respect the denormal mode, but the rounding mode
21308 // is fixed to round-to-nearest-even.
21309 //
21310 // F64 / PK_F16 / PK_BF16 never flush and are also fixed to
21311 // round-to-nearest-even.
21312 //
21313 // We ignore the rounding mode problem, even in strictfp. The C++ standard
21314 // suggests it is OK if the floating-point mode may not match the calling
21315 // thread.
21316 if (Ty->isFloatTy()) {
21317 return Subtarget->hasLDSFPAtomicAddF32() ? AtomicExpansionKind::None
21319 }
21320
21321 if (Ty->isDoubleTy()) {
21322 // Ignores denormal mode, but we don't consider flushing mandatory.
21323 return Subtarget->hasLdsAtomicAddF64() ? AtomicExpansionKind::None
21325 }
21326
21327 if (Subtarget->hasAtomicDsPkAdd16Insts() && isV2F16OrV2BF16(Ty))
21329
21331 }
21332
21333 // LDS atomics respect the denormal mode from the mode register.
21334 //
21335 // Traditionally f32 global/buffer memory atomics would unconditionally
21336 // flush denormals, but newer targets do not flush. f64/f16/bf16 cases never
21337 // flush.
21338 //
21339 // On targets with flat atomic fadd, denormals would flush depending on
21340 // whether the target address resides in LDS or global memory. We consider
21341 // this flat-maybe-flush as will-flush.
21342 if (Ty->isFloatTy() &&
21343 !Subtarget->hasMemoryAtomicFaddF32DenormalSupport()) {
21344 DenormReason = getAtomicFlushDenormalReason(RMW);
21345 if (DenormReason == AtomicFlushDenormalReason::IEEE)
21347 }
21348
21349 MemLegality =
21350 getGlobalMemoryFPAtomicLegality(*Subtarget, RMW, HasSystemScope);
21351 if (MemLegality != GlobalFPAtomicLegality::Illegal) {
21352 if (AS == AMDGPUAS::FLAT_ADDRESS) {
21353 // gfx942, gfx12
21354 if (Subtarget->hasAtomicFlatPkAdd16Insts() && isV2F16OrV2BF16(Ty))
21355 return ReportHWInst(AtomicExpansionKind::None);
21356 } else if (AMDGPU::isExtendedGlobalAddrSpace(AS)) {
21357 // gfx90a, gfx942, gfx12
21358 if (Subtarget->hasAtomicBufferGlobalPkAddF16Insts() && isV2F16(Ty))
21359 return ReportHWInst(AtomicExpansionKind::None);
21360
21361 // gfx942, gfx12
21362 if (Subtarget->hasAtomicGlobalPkAddBF16Inst() && isV2BF16(Ty))
21363 return ReportHWInst(AtomicExpansionKind::None);
21364 } else if (AS == AMDGPUAS::BUFFER_FAT_POINTER) {
21365 // gfx90a, gfx942, gfx12
21366 if (Subtarget->hasAtomicBufferGlobalPkAddF16Insts() && isV2F16(Ty))
21367 return ReportHWInst(AtomicExpansionKind::None);
21368
21369 // While gfx90a/gfx942 supports v2bf16 for global/flat, it does not for
21370 // buffer. gfx12 does have the buffer version.
21371 if (Subtarget->hasAtomicBufferPkAddBF16Inst() && isV2BF16(Ty))
21372 return ReportHWInst(AtomicExpansionKind::None);
21373 }
21374
21375 // global and flat atomic fadd f64: gfx90a, gfx942.
21376 if (Subtarget->hasFlatBufferGlobalAtomicFaddF64Inst() && Ty->isDoubleTy())
21377 return ReportHWInst(AtomicExpansionKind::None);
21378
21379 if (AS != AMDGPUAS::FLAT_ADDRESS) {
21380 if (Ty->isFloatTy()) {
21381 // global/buffer atomic fadd f32 no-rtn: gfx908, gfx90a, gfx942,
21382 // gfx11+.
21383 if (RMW->use_empty() && Subtarget->hasAtomicFaddNoRtnInsts())
21384 return ReportHWInst(AtomicExpansionKind::None);
21385 // global/buffer atomic fadd f32 rtn: gfx90a, gfx942, gfx11+.
21386 if (!RMW->use_empty() && Subtarget->hasAtomicFaddRtnInsts())
21387 return ReportHWInst(AtomicExpansionKind::None);
21388 } else {
21389 // gfx908
21390 if (RMW->use_empty() &&
21391 Subtarget->hasAtomicBufferGlobalPkAddF16NoRtnInsts() &&
21392 isV2F16(Ty))
21393 return ReportHWInst(AtomicExpansionKind::None);
21394 }
21395 }
21396
21397 // flat atomic fadd f32: gfx942, gfx11+.
21398 if (AS == AMDGPUAS::FLAT_ADDRESS && Ty->isFloatTy()) {
21399 if (Subtarget->hasFlatAtomicFaddF32Inst())
21400 return ReportHWInst(AtomicExpansionKind::None);
21401
21402 // If it is in flat address space, and the type is float, we will try to
21403 // expand it, if the target supports global and lds atomic fadd. The
21404 // reason we need that is, in the expansion, we emit the check of
21405 // address space. If it is in global address space, we emit the global
21406 // atomic fadd; if it is in shared address space, we emit the LDS atomic
21407 // fadd.
21408 if (Subtarget->hasLDSFPAtomicAddF32()) {
21409 if (RMW->use_empty() && Subtarget->hasAtomicFaddNoRtnInsts())
21411 if (!RMW->use_empty() && Subtarget->hasAtomicFaddRtnInsts())
21413 }
21414 }
21415 }
21416
21418 }
21420 case AtomicRMWInst::FMax: {
21421 Type *Ty = RMW->getType();
21422
21423 // LDS float and double fmin/fmax were always supported.
21424 if (AS == AMDGPUAS::LOCAL_ADDRESS) {
21425 return Ty->isFloatTy() || Ty->isDoubleTy() ? AtomicExpansionKind::None
21427 }
21428
21429 MemLegality =
21430 getGlobalMemoryFPAtomicLegality(*Subtarget, RMW, HasSystemScope);
21431 if (MemLegality != GlobalFPAtomicLegality::Illegal) {
21432 // For flat and global cases:
21433 // float, double in gfx7. Manual claims denormal support.
21434 // Removed in gfx8.
21435 // float, double restored in gfx10.
21436 // double removed again in gfx11, so only f32 for gfx11/gfx12.
21437 //
21438 // For gfx9, gfx90a and gfx942 support f64 for global (same as fadd), but
21439 // no f32.
21440 if (AS == AMDGPUAS::FLAT_ADDRESS) {
21441 if (Subtarget->hasAtomicFMinFMaxF32FlatInsts() && Ty->isFloatTy())
21442 return ReportHWInst(AtomicExpansionKind::None);
21443 if (Subtarget->hasAtomicFMinFMaxF64FlatInsts() && Ty->isDoubleTy())
21444 return ReportHWInst(AtomicExpansionKind::None);
21445 } else if (AMDGPU::isExtendedGlobalAddrSpace(AS) ||
21447 if (Subtarget->hasAtomicFMinFMaxF32GlobalInsts() && Ty->isFloatTy())
21448 return ReportHWInst(AtomicExpansionKind::None);
21449 if (Subtarget->hasAtomicFMinFMaxF64GlobalInsts() && Ty->isDoubleTy())
21450 return ReportHWInst(AtomicExpansionKind::None);
21451 }
21452 }
21453
21455 }
21458 default:
21460 }
21461
21462 llvm_unreachable("covered atomicrmw op switch");
21463}
21464
21473
21476 if (SI->getPointerAddressSpace() == AMDGPUAS::VGPR)
21478 return SI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS
21481}
21482
21485 const AtomicCmpXchgInst *CmpX) const {
21486 unsigned AddrSpace = CmpX->getPointerAddressSpace();
21487 if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS)
21489
21490 if (AddrSpace == AMDGPUAS::VGPR)
21492
21493 if (AddrSpace != AMDGPUAS::FLAT_ADDRESS || !flatInstrMayAccessPrivate(CmpX))
21495
21496 const DataLayout &DL = CmpX->getDataLayout();
21497
21498 Type *ValTy = CmpX->getNewValOperand()->getType();
21499
21500 // If a 64-bit flat atomic may alias private, we need to avoid using the
21501 // atomic in the private case.
21502 return DL.getTypeSizeInBits(ValTy) == 64 ? AtomicExpansionKind::CustomExpand
21504}
21505
21506const TargetRegisterClass *
21507SITargetLowering::getRegClassFor(MVT VT, bool isDivergent) const {
21509 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
21510 if (RC == &AMDGPU::VReg_1RegClass && !isDivergent)
21511 return Subtarget->isWave64() ? &AMDGPU::SReg_64RegClass
21512 : &AMDGPU::SReg_32RegClass;
21513 if (!TRI->isSGPRClass(RC) && !isDivergent)
21514 return TRI->getEquivalentSGPRClass(RC);
21515 if (TRI->isSGPRClass(RC) && isDivergent) {
21516 if (Subtarget->hasGFX90AInsts())
21517 return TRI->getEquivalentAVClass(RC);
21518 return TRI->getEquivalentVGPRClass(RC);
21519 }
21520
21521 return RC;
21522}
21523
21524// FIXME: This is a workaround for DivergenceAnalysis not understanding always
21525// uniform values (as produced by the mask results of control flow intrinsics)
21526// used outside of divergent blocks. The phi users need to also be treated as
21527// always uniform.
21528//
21529// FIXME: DA is no longer in-use. Does this still apply to UniformityAnalysis?
21531 unsigned WaveSize) {
21532 // FIXME: We assume we never cast the mask results of a control flow
21533 // intrinsic.
21534 // Early exit if the type won't be consistent as a compile time hack.
21535 IntegerType *IT = dyn_cast<IntegerType>(V->getType());
21536 if (!IT || IT->getBitWidth() != WaveSize)
21537 return false;
21538
21539 if (!isa<Instruction>(V))
21540 return false;
21541 if (!Visited.insert(V).second)
21542 return false;
21543 bool Result = false;
21544 for (const auto *U : V->users()) {
21546 if (V == U->getOperand(1)) {
21547 switch (Intrinsic->getIntrinsicID()) {
21548 default:
21549 Result = false;
21550 break;
21551 case Intrinsic::amdgcn_if_break:
21552 case Intrinsic::amdgcn_if:
21553 case Intrinsic::amdgcn_else:
21554 Result = true;
21555 break;
21556 }
21557 }
21558 if (V == U->getOperand(0)) {
21559 switch (Intrinsic->getIntrinsicID()) {
21560 default:
21561 Result = false;
21562 break;
21563 case Intrinsic::amdgcn_end_cf:
21564 case Intrinsic::amdgcn_loop:
21565 Result = true;
21566 break;
21567 }
21568 }
21569 } else {
21570 Result = hasCFUser(U, Visited, WaveSize);
21571 }
21572 if (Result)
21573 break;
21574 }
21575 return Result;
21576}
21577
21579 const Value *V) const {
21580 if (const CallInst *CI = dyn_cast<CallInst>(V)) {
21581 if (CI->isInlineAsm()) {
21582 // FIXME: This cannot give a correct answer. This should only trigger in
21583 // the case where inline asm returns mixed SGPR and VGPR results, used
21584 // outside the defining block. We don't have a specific result to
21585 // consider, so this assumes if any value is SGPR, the overall register
21586 // also needs to be SGPR.
21587 const SIRegisterInfo *SIRI = Subtarget->getRegisterInfo();
21589 MF.getDataLayout(), Subtarget->getRegisterInfo(), *CI);
21590 for (auto &TC : TargetConstraints) {
21591 if (TC.Type == InlineAsm::isOutput) {
21593 const TargetRegisterClass *RC =
21594 getRegForInlineAsmConstraint(SIRI, TC.ConstraintCode,
21595 TC.ConstraintVT)
21596 .second;
21597 if (RC && SIRI->isSGPRClass(RC))
21598 return true;
21599 }
21600 }
21601 }
21602 }
21604 return hasCFUser(V, Visited, Subtarget->getWavefrontSize());
21605}
21606
21608 for (SDUse &Use : N->uses()) {
21610 if (getBasePtrIndex(M) == Use.getOperandNo())
21611 return true;
21612 }
21613 }
21614 return false;
21615}
21616
21618 SDValue N1) const {
21619 if (!N0.hasOneUse())
21620 return false;
21621 // Take care of the opportunity to keep N0 uniform
21622 if (N0->isDivergent() || !N1->isDivergent())
21623 return true;
21624 // Check if we have a good chance to form the memory access pattern with the
21625 // base and offset
21626 return (DAG.isBaseWithConstantOffset(N0) &&
21628}
21629
21631 Register N0, Register N1) const {
21632 return MRI.hasOneNonDBGUse(N0); // FIXME: handle regbanks
21633}
21634
21637 // Propagate metadata set by AMDGPUAnnotateUniformValues to the MMO of a load.
21639 if (I.getMetadata("amdgpu.noclobber"))
21640 Flags |= MONoClobber;
21641 if (I.getMetadata("amdgpu.last.use"))
21642 Flags |= MOLastUse;
21643 if (MDNode *N = I.getMetadata("amdgpu.cfs")) {
21644 const ConstantAsMetadata *CAM = cast<ConstantAsMetadata>(N->getOperand(0));
21646 unsigned CFS = CI->getZExtValue() & AMDGPU::CPol::CFS_MASK;
21647 static_assert((MOCFSB0 << 1) == MOCFSB1,
21648 "MOCFSB0 and MOCFSB1 are not adjacent bits");
21649 Flags |= static_cast<MachineMemOperand::Flags>(CFS * MOCFSB0);
21650 }
21651 return Flags;
21652}
21653
21655 Instruction *AI) const {
21656 // Given: atomicrmw fadd ptr %addr, float %val ordering
21657 //
21658 // With this expansion we produce the following code:
21659 // [...]
21660 // %is.shared = call i1 @llvm.amdgcn.is.shared(ptr %addr)
21661 // br i1 %is.shared, label %atomicrmw.shared, label %atomicrmw.check.private
21662 //
21663 // atomicrmw.shared:
21664 // %cast.shared = addrspacecast ptr %addr to ptr addrspace(3)
21665 // %loaded.shared = atomicrmw fadd ptr addrspace(3) %cast.shared,
21666 // float %val ordering
21667 // br label %atomicrmw.phi
21668 //
21669 // atomicrmw.check.private:
21670 // %is.private = call i1 @llvm.amdgcn.is.private(ptr %int8ptr)
21671 // br i1 %is.private, label %atomicrmw.private, label %atomicrmw.global
21672 //
21673 // atomicrmw.private:
21674 // %cast.private = addrspacecast ptr %addr to ptr addrspace(5)
21675 // %loaded.private = load float, ptr addrspace(5) %cast.private
21676 // %val.new = fadd float %loaded.private, %val
21677 // store float %val.new, ptr addrspace(5) %cast.private
21678 // br label %atomicrmw.phi
21679 //
21680 // atomicrmw.global:
21681 // %cast.global = addrspacecast ptr %addr to ptr addrspace(1)
21682 // %loaded.global = atomicrmw fadd ptr addrspace(1) %cast.global,
21683 // float %val ordering
21684 // br label %atomicrmw.phi
21685 //
21686 // atomicrmw.phi:
21687 // %loaded.phi = phi float [ %loaded.shared, %atomicrmw.shared ],
21688 // [ %loaded.private, %atomicrmw.private ],
21689 // [ %loaded.global, %atomicrmw.global ]
21690 // br label %atomicrmw.end
21691 //
21692 // atomicrmw.end:
21693 // [...]
21694 //
21695 //
21696 // For 64-bit atomics which may reside in private memory, we perform a simpler
21697 // version that only inserts the private check, and uses the flat operation.
21698
21699 IRBuilder<> Builder(AI);
21700 LLVMContext &Ctx = Builder.getContext();
21701
21702 auto *RMW = dyn_cast<AtomicRMWInst>(AI);
21703 const unsigned PtrOpIdx = RMW ? AtomicRMWInst::getPointerOperandIndex()
21705 Value *Addr = AI->getOperand(PtrOpIdx);
21706
21707 /// TODO: Only need to check private, then emit flat-known-not private (no
21708 /// need for shared block, or cast to global).
21710
21711 Align Alignment;
21712 if (RMW)
21713 Alignment = RMW->getAlign();
21714 else if (CX)
21715 Alignment = CX->getAlign();
21716 else
21717 llvm_unreachable("unhandled atomic operation");
21718
21719 // FullFlatEmulation is true if we need to issue the private, shared, and
21720 // global cases.
21721 //
21722 // If this is false, we are only dealing with the flat-targeting-private case,
21723 // where we only insert a check for private and still use the flat instruction
21724 // for global and shared.
21725
21726 bool FullFlatEmulation =
21727 RMW && RMW->getOperation() == AtomicRMWInst::FAdd &&
21728 ((Subtarget->hasAtomicFaddInsts() && RMW->getType()->isFloatTy()) ||
21729 (Subtarget->hasFlatBufferGlobalAtomicFaddF64Inst() &&
21730 RMW->getType()->isDoubleTy()));
21731
21732 // If the return value isn't used, do not introduce a false use in the phi.
21733 bool ReturnValueIsUsed = !AI->use_empty();
21734
21735 BasicBlock *BB = Builder.GetInsertBlock();
21736 Function *F = BB->getParent();
21737 BasicBlock *ExitBB =
21738 BB->splitBasicBlock(Builder.GetInsertPoint(), "atomicrmw.end");
21739 BasicBlock *SharedBB = nullptr;
21740
21741 BasicBlock *CheckPrivateBB = BB;
21742 if (FullFlatEmulation) {
21743 SharedBB = BasicBlock::Create(Ctx, "atomicrmw.shared", F, ExitBB);
21744 CheckPrivateBB =
21745 BasicBlock::Create(Ctx, "atomicrmw.check.private", F, ExitBB);
21746 }
21747
21748 BasicBlock *PrivateBB =
21749 BasicBlock::Create(Ctx, "atomicrmw.private", F, ExitBB);
21750 BasicBlock *GlobalBB = BasicBlock::Create(Ctx, "atomicrmw.global", F, ExitBB);
21751 BasicBlock *PhiBB = BasicBlock::Create(Ctx, "atomicrmw.phi", F, ExitBB);
21752
21753 std::prev(BB->end())->eraseFromParent();
21754 Builder.SetInsertPoint(BB);
21755
21756 Value *LoadedShared = nullptr;
21757 if (FullFlatEmulation) {
21758 Value *IsShared = Builder.CreateIntrinsic(Intrinsic::amdgcn_is_shared,
21759 {Addr}, nullptr, "is.shared");
21760 Builder.CreateCondBr(IsShared, SharedBB, CheckPrivateBB);
21761 Builder.SetInsertPoint(SharedBB);
21762 Value *CastToLocal = Builder.CreateAddrSpaceCast(
21764
21765 Instruction *Clone = AI->clone();
21766 Clone->insertInto(SharedBB, SharedBB->end());
21767 Clone->getOperandUse(PtrOpIdx).set(CastToLocal);
21768 LoadedShared = Clone;
21769
21770 Builder.CreateBr(PhiBB);
21771 Builder.SetInsertPoint(CheckPrivateBB);
21772 }
21773
21774 Value *IsPrivate = Builder.CreateIntrinsic(Intrinsic::amdgcn_is_private,
21775 {Addr}, nullptr, "is.private");
21776 Builder.CreateCondBr(IsPrivate, PrivateBB, GlobalBB);
21777
21778 Builder.SetInsertPoint(PrivateBB);
21779
21780 Value *CastToPrivate = Builder.CreateAddrSpaceCast(
21782
21783 Value *LoadedPrivate;
21784 if (RMW) {
21785 LoadedPrivate = Builder.CreateAlignedLoad(
21786 RMW->getType(), CastToPrivate, RMW->getAlign(), RMW->isVolatile(),
21787 "loaded.private");
21788
21789 Value *NewVal = buildAtomicRMWValue(RMW->getOperation(), Builder,
21790 LoadedPrivate, RMW->getValOperand());
21791
21792 Builder.CreateAlignedStore(NewVal, CastToPrivate, RMW->getAlign(),
21793 RMW->isVolatile());
21794 } else {
21795 auto [ResultLoad, Equal] = buildCmpXchgValue(
21796 Builder, CastToPrivate, CX->getCompareOperand(), CX->getNewValOperand(),
21797 CX->getAlign(), CX->isVolatile());
21798
21799 Value *Insert = Builder.CreateInsertValue(PoisonValue::get(CX->getType()),
21800 ResultLoad, 0);
21801 LoadedPrivate = Builder.CreateInsertValue(Insert, Equal, 1);
21802 }
21803
21804 Builder.CreateBr(PhiBB);
21805
21806 Builder.SetInsertPoint(GlobalBB);
21807
21808 // Continue using a flat instruction if we only emitted the check for private.
21809 Instruction *LoadedGlobal = AI;
21810 if (FullFlatEmulation) {
21811 Value *CastToGlobal = Builder.CreateAddrSpaceCast(
21813 AI->getOperandUse(PtrOpIdx).set(CastToGlobal);
21814 }
21815
21816 AI->removeFromParent();
21817 AI->insertInto(GlobalBB, GlobalBB->end());
21818
21819 // The new atomicrmw may go through another round of legalization later.
21820 if (!FullFlatEmulation) {
21821 // We inserted the runtime check already, make sure we do not try to
21822 // re-expand this.
21823 // TODO: Should union with any existing metadata.
21824 MDBuilder MDB(F->getContext());
21825 MDNode *RangeNotPrivate =
21828 LoadedGlobal->setMetadata(LLVMContext::MD_noalias_addrspace,
21829 RangeNotPrivate);
21830 }
21831
21832 Builder.CreateBr(PhiBB);
21833
21834 Builder.SetInsertPoint(PhiBB);
21835
21836 if (ReturnValueIsUsed) {
21837 PHINode *Loaded = Builder.CreatePHI(AI->getType(), 3);
21838 AI->replaceAllUsesWith(Loaded);
21839 if (FullFlatEmulation)
21840 Loaded->addIncoming(LoadedShared, SharedBB);
21841 Loaded->addIncoming(LoadedPrivate, PrivateBB);
21842 Loaded->addIncoming(LoadedGlobal, GlobalBB);
21843 Loaded->takeName(AI);
21844 }
21845
21846 Builder.CreateBr(ExitBB);
21847}
21848
21850 unsigned PtrOpIdx) {
21851 Value *PtrOp = I->getOperand(PtrOpIdx);
21854
21855 Type *FlatPtr = PointerType::get(I->getContext(), AMDGPUAS::FLAT_ADDRESS);
21856 Value *ASCast = CastInst::CreatePointerCast(PtrOp, FlatPtr, "scratch.ascast",
21857 I->getIterator());
21858 I->setOperand(PtrOpIdx, ASCast);
21859}
21860
21863
21866
21869 if (const auto *ConstVal = dyn_cast<Constant>(AI->getValOperand());
21870 ConstVal && ConstVal->isNullValue() &&
21872 // atomicrmw or %ptr, 0 -> atomicrmw add %ptr, 0
21874
21875 // We may still need the private-alias-flat handling below.
21876
21877 // TODO: Skip this for cases where we cannot access remote memory.
21878 }
21879 }
21880
21881 // The non-flat expansions should only perform the de-canonicalization of
21882 // identity values.
21884 return;
21885
21887}
21888
21895
21899
21901 "Expand Atomic Load only handles SCRATCH -> FLAT conversion");
21902}
21903
21905 if (SI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
21906 return convertScratchAtomicToFlatAtomic(SI, SI->getPointerOperandIndex());
21907
21909 "Expand Atomic Store only handles SCRATCH -> FLAT conversion");
21910}
static bool isMul(MachineInstr *MI)
unsigned NumParts
return SDValue()
static unsigned getIntrinsicID(const SDNode *N)
const AArch64Subtarget & Subtarget
const TargetLowering & TLI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPU address space definition.
static constexpr std::pair< ImplicitArgumentMask, StringLiteral > ImplicitAttrs[]
static bool allUsesHaveSourceMods(MachineInstr &MI, MachineRegisterInfo &MRI, unsigned CostThreshold=4)
static msgpack::DocNode getNode(msgpack::DocNode DN, msgpack::Type Type, MCValue Val)
uint64_t uint64_t
uint64_t Imm
static bool isCtlzOpc(unsigned Opc)
Contains the definition of a TargetInstrInfo class that is common to all AMD GPUs.
static bool isNoUnsignedWrap(MachineInstr *Addr)
static bool parseTexFail(uint64_t TexFailCtrl, bool &TFE, bool &LWE, bool &IsTexFail)
static bool isAsyncLDSDMA(Intrinsic::ID Intr)
static std::optional< uint32_t > getConstantNamedBarrierNumber(const MachineOperand &BarOp, const MachineRegisterInfo &MRI)
Return the barrier ID for BarOp if it's statically known, mapping poison to 0 and masking off the hig...
static void packImage16bitOpsToDwords(MachineIRBuilder &B, MachineInstr &MI, SmallVectorImpl< Register > &PackedAddrs, unsigned ArgOffset, const AMDGPU::ImageDimIntrinsicInfo *Intr, bool IsA16, bool IsG16)
Turn a set of f16 typed registers in AddrRegs into a dword sized vector with f16 typed elements.
static bool isKnownNonNull(Register Val, MachineRegisterInfo &MRI, const AMDGPUTargetMachine &TM, unsigned AddrSpace)
Return true if the value is a known valid address, such that a null check is not necessary.
Provides AMDGPU specific target descriptions.
The AMDGPU TargetMachine interface definition for hw codegen targets.
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static cl::opt< ITMode > IT(cl::desc("IT block support"), cl::Hidden, cl::init(DefaultIT), cl::values(clEnumValN(DefaultIT, "arm-default-it", "Generate any type of IT block"), clEnumValN(RestrictedIT, "arm-restrict-it", "Disallow complex IT blocks")))
Function Alias Analysis Results
static DenormalFPEnv getDenormalFPEnv(AttributeSet FnAttrs)
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
@ DEFAULT
Default weight is used in cases when there is no dedicated execution weight set.
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static std::optional< SDByteProvider > calculateByteProvider(SDValue Op, unsigned Index, unsigned Depth, std::optional< uint64_t > VectorIndex, unsigned StartingIndex=0, MutableArrayRef< uint8_t > ByteMask={})
dxil translate DXIL Translate Metadata
static bool isSigned(unsigned Opcode)
Utilities for dealing with flags related to floating point properties and mode controls.
AMD GCN specific subclass of TargetSubtarget.
Provides analysis for querying information about KnownBits during GISel passes.
#define DEBUG_TYPE
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
iv Induction Variable Users
Definition IVUsers.cpp:48
static constexpr Value * getValue(Ty &ValueOrUse)
const size_t AbstractManglingParser< Derived, Alloc >::NumOps
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
#define RegName(no)
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
Contains matchers for matching SSA Machine Instructions.
static bool isUndef(const MachineInstr &MI)
Register Reg
Register const TargetRegisterInfo * TRI
Promote Memory to Register
Definition Mem2Reg.cpp:110
static unsigned getAddressSpace(const Value *V, unsigned MaxLookup)
uint64_t IntrinsicInst * II
#define P(N)
if(PassOpts->AAPipeline)
static constexpr MCPhysReg SPReg
MachineBasicBlock * Folded
SmallVector< MachineOperand, 4 > Cond
int64_t C1
RISCVCC::CondCode CC
MachineRegisterInfo & MRI
static cl::opt< RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development, "development", "for training")))
Contains matchers for matching SelectionDAG nodes and values.
static void r0(uint32_t &A, uint32_t &B, uint32_t &C, uint32_t &D, uint32_t &E, int I, uint32_t *Buf)
Definition SHA1.cpp:39
static void r3(uint32_t &A, uint32_t &B, uint32_t &C, uint32_t &D, uint32_t &E, int I, uint32_t *Buf)
Definition SHA1.cpp:57
static void r2(uint32_t &A, uint32_t &B, uint32_t &C, uint32_t &D, uint32_t &E, int I, uint32_t *Buf)
Definition SHA1.cpp:51
static void r1(uint32_t &A, uint32_t &B, uint32_t &C, uint32_t &D, uint32_t &E, int I, uint32_t *Buf)
Definition SHA1.cpp:45
#define FP_DENORM_FLUSH_NONE
Definition SIDefines.h:1523
#define FP_DENORM_FLUSH_IN_FLUSH_OUT
Definition SIDefines.h:1520
SI Fold Operands
static void reservePrivateMemoryRegs(const TargetMachine &TM, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info)
static SDValue adjustLoadValueTypeImpl(SDValue Result, EVT LoadVT, const SDLoc &DL, SelectionDAG &DAG, bool Unpacked)
static MachineBasicBlock * emitIndirectSrc(MachineInstr &MI, MachineBasicBlock &MBB, const GCNSubtarget &ST)
static bool denormalModeIsFlushAllF64F16(const MachineFunction &MF)
static bool isAtomicRMWLegalIntTy(Type *Ty)
static void knownBitsForWorkitemID(const GCNSubtarget &ST, GISelValueTracking &VT, KnownBits &Known, unsigned Dim)
static bool flatInstrMayAccessPrivate(const Instruction *I)
Return if a flat address space atomicrmw can access private memory.
static std::pair< unsigned, int > computeIndirectRegAndOffset(const SIRegisterInfo &TRI, const TargetRegisterClass *SuperRC, unsigned VecReg, int Offset)
static bool denormalModeIsFlushAllF32(const MachineFunction &MF)
static bool addresses16Bits(int Mask)
static MachineBasicBlock * expand64BitScalarArithmetic(MachineInstr &MI, MachineBasicBlock *BB)
static bool isClampZeroToOne(SDValue A, SDValue B)
static bool isBoolSGPRExt(SDValue V)
static bool supportsMin3Max3(const GCNSubtarget &Subtarget, unsigned Opc, EVT VT)
static AtomicFlushDenormalReason getAtomicFlushDenormalReason(const AtomicRMWInst *RMW)
static unsigned findFirstFreeSGPR(CCState &CCInfo)
static uint32_t getPermuteMask(SDValue V)
static SDValue lowerLaneOp(const SITargetLowering &TLI, SDNode *N, SelectionDAG &DAG)
static int getAlignedAGPRClassID(unsigned UnalignedClassID)
static void processPSInputArgs(SmallVectorImpl< ISD::InputArg > &Splits, CallingConv::ID CallConv, ArrayRef< ISD::InputArg > Ins, BitVector &Skipped, FunctionType *FType, SIMachineFunctionInfo *Info)
static SDValue selectSOffset(SDValue SOffset, SelectionDAG &DAG, const GCNSubtarget *Subtarget)
static SDValue getLoadExtOrTrunc(SelectionDAG &DAG, ISD::LoadExtType ExtType, SDValue Op, const SDLoc &SL, EVT VT)
static std::tuple< unsigned, unsigned > getDPPOpcForWaveReduction(unsigned Opc, const GCNSubtarget &ST)
static void fixMasks(SmallVectorImpl< DotSrc > &Srcs, unsigned ChainLength)
static bool is32bitWaveReduceOperation(unsigned Opc)
static TargetLowering::AtomicExpansionKind atomicSupportedIfLegalIntType(const AtomicRMWInst *RMW)
static SDValue strictFPExtFromF16(SelectionDAG &DAG, SDValue Src)
Return the source of an fp_extend from f16 to f32, or a converted FP constant.
static bool isAtomicRMWLegalXChgTy(const AtomicRMWInst *RMW)
static bool bitOpWithConstantIsReducible(unsigned Opc, uint32_t Val)
static void convertScratchAtomicToFlatAtomic(Instruction *I, unsigned PtrOpIdx)
static bool isCopyFromRegOfInlineAsm(const SDNode *N)
static bool elementPairIsOddToEven(ArrayRef< int > Mask, int Elt)
static SDValue lowerBFEIntrinsic(SDValue Op, SelectionDAG &DAG, Intrinsic::ID IntrinsicID)
static SDValue getDWordFromOffset(SelectionDAG &DAG, SDLoc SL, SDValue Src, unsigned DWordOffset)
static MachineBasicBlock::iterator loadM0FromVGPR(const SIInstrInfo *TII, MachineBasicBlock &MBB, MachineInstr &MI, unsigned InitResultReg, unsigned PhiReg, int Offset, bool UseGPRIdxMode, Register &SGPRIdxReg)
static bool isFloatingPointWaveReduceOperation(unsigned Opc)
static bool isImmConstraint(StringRef Constraint)
static SDValue padEltsToUndef(SelectionDAG &DAG, const SDLoc &DL, EVT CastVT, SDValue Src, int ExtraElts)
static bool hasCFUser(const Value *V, SmallPtrSet< const Value *, 16 > &Visited, unsigned WaveSize)
static std::pair< Register, Register > ExtractSubRegs(MachineInstr &MI, MachineOperand &Op, const TargetRegisterClass *SrcRC, const GCNSubtarget &ST, MachineRegisterInfo &MRI)
static unsigned SubIdx2Lane(unsigned Idx)
Helper function for adjustWritemask.
static TargetLowering::AtomicExpansionKind getPrivateAtomicExpansionKind(const GCNSubtarget &STI)
static bool addressMayBeAccessedAsPrivate(const MachineMemOperand *MMO, const SIMachineFunctionInfo &Info)
static MachineBasicBlock * lowerWaveReduce(MachineInstr &MI, MachineBasicBlock &BB, const GCNSubtarget &ST, unsigned Opc)
static bool elementPairIsContiguous(ArrayRef< int > Mask, int Elt)
static bool isV2BF16(Type *Ty)
static bool isFrexpExp(SDValue V, SDValue &FrexpInput)
static ArgDescriptor allocateSGPR32InputImpl(CCState &CCInfo, const TargetRegisterClass *RC, unsigned NumArgRegs)
static SDValue getMad64_32(SelectionDAG &DAG, const SDLoc &SL, EVT VT, SDValue N0, SDValue N1, SDValue N2, bool Signed)
static SDValue resolveSources(SelectionDAG &DAG, SDLoc SL, SmallVectorImpl< DotSrc > &Srcs, bool IsSigned, bool IsAny)
static bool hasNon16BitAccesses(uint64_t PermMask, SDValue &Op, SDValue &OtherOp)
static SDValue lowerWaveShuffle(const SITargetLowering &TLI, SDNode *N, SelectionDAG &DAG)
static SDValue diagnoseUnsupportedImage(SelectionDAG &DAG, SDValue Op, ArrayRef< EVT > ResultTypes, const SDLoc &DL, const Twine &Msg)
Emit a DiagnosticInfoUnsupported for an unsupported image intrinsic and return poison values of Resul...
static void placeSources(ByteProvider< SDValue > &Src0, ByteProvider< SDValue > &Src1, SmallVectorImpl< DotSrc > &Src0s, SmallVectorImpl< DotSrc > &Src1s, int Step)
static unsigned parseSyncscopeMDArg(const CallBase &CI, unsigned ArgIdx)
static EVT memVTFromLoadIntrReturn(const SITargetLowering &TLI, const DataLayout &DL, Type *Ty, unsigned MaxNumLanes)
static MachineBasicBlock::iterator emitLoadM0FromVGPRLoop(const SIInstrInfo *TII, MachineRegisterInfo &MRI, MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB, const DebugLoc &DL, const MachineOperand &Idx, unsigned InitReg, unsigned ResultReg, unsigned PhiReg, unsigned InitSaveExecReg, int Offset, bool UseGPRIdxMode, Register &SGPRIdxReg)
static SDValue matchPERM(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool isFrameIndexOp(SDValue Op)
static ConstantFPSDNode * getSplatConstantFP(SDValue Op)
static void allocateSGPR32Input(CCState &CCInfo, ArgDescriptor &Arg)
static void knownBitsForSBFE(const MachineInstr &MI, GISelValueTracking &VT, KnownBits &Known, const APInt &DemandedElts, unsigned BFEWidth, bool SExt, unsigned Depth)
static bool isExtendedFrom16Bits(SDValue &Operand)
static std::optional< bool > checkDot4MulSignedness(const SDValue &N, ByteProvider< SDValue > &Src0, ByteProvider< SDValue > &Src1, const SDValue &S0Op, const SDValue &S1Op, const SelectionDAG &DAG)
static bool vectorEltWillFoldAway(SDValue Op)
static SDValue getSPDenormModeValue(uint32_t SPDenormMode, SelectionDAG &DAG, const SIMachineFunctionInfo *Info, const GCNSubtarget *ST)
static uint32_t getConstantPermuteMask(uint32_t C)
static AtomicOrdering parseAtomicOrderingCABIArg(const CallBase &CI, unsigned ArgIdx)
static MachineBasicBlock * emitIndirectDst(MachineInstr &MI, MachineBasicBlock &MBB, const GCNSubtarget &ST)
static void setM0ToIndexFromSGPR(const SIInstrInfo *TII, MachineRegisterInfo &MRI, MachineInstr &MI, int Offset)
static DenormalFPEnv getDenormalFPEnv(const MachineFunction &MF)
static std::pair< MachineBasicBlock *, MachineBasicBlock * > splitBlockForLoop(MachineInstr &MI, MachineBasicBlock &MBB, bool InstInLoop)
static unsigned getBasePtrIndex(const MemSDNode *N)
MemSDNode::getBasePtr() does not work for intrinsics, which needs to offset by the chain and intrinsi...
static void allocateFixedSGPRInputImpl(CCState &CCInfo, const TargetRegisterClass *RC, MCRegister Reg)
static SDValue constructRetValue(SelectionDAG &DAG, MachineSDNode *Result, ArrayRef< EVT > ResultTypes, bool IsTexFail, bool Unpacked, bool IsD16, int DMaskPop, int NumVDataDwords, bool IsAtomicPacked16Bit, const SDLoc &DL)
static std::pair< SDValue, SDValue > splitTFEValueAndStatus(SDValue Op, EVT VT, const SDLoc &DL, SelectionDAG &DAG)
static std::optional< ByteProvider< SDValue > > handleMulOperand(const SDValue &MulOperand)
static ISD::CondCode tryReduceF64CompareToHiHalf(const ISD::CondCode CC, const SDValue LHS, const SDValue RHS, const SelectionDAG &DAG)
static Register getIndirectSGPRIdx(const SIInstrInfo *TII, MachineRegisterInfo &MRI, MachineInstr &MI, int Offset)
static SDValue emitNonHSAIntrinsicError(SelectionDAG &DAG, const SDLoc &DL, EVT VT)
static EVT memVTFromLoadIntrData(const SITargetLowering &TLI, const DataLayout &DL, Type *Ty, unsigned MaxNumLanes)
static unsigned minMaxOpcToMin3Max3Opc(unsigned Opc)
static unsigned getExtOpcodeForPromotedOp(SDValue Op)
static void expand64BitV_CNDMASK(MachineInstr &MI, MachineBasicBlock *BB)
static GlobalFPAtomicLegality getGlobalMemoryFPAtomicLegality(const GCNSubtarget &Subtarget, const AtomicRMWInst *RMW, bool HasSystemScope)
static SDValue lowerBALLOTIntrinsic(const SITargetLowering &TLI, SDNode *N, SelectionDAG &DAG)
static SDValue buildSMovImm32(SelectionDAG &DAG, const SDLoc &DL, uint64_t Val)
static SDValue tryFoldMADwithSRL(SelectionDAG &DAG, const SDLoc &SL, SDValue MulLHS, SDValue MulRHS, SDValue AddRHS)
static unsigned getIntrMemWidth(unsigned IntrID)
static SDValue getBuildDwordsVector(SelectionDAG &DAG, SDLoc DL, ArrayRef< SDValue > Elts)
static SDNode * findUser(SDValue Value, unsigned Opcode)
Helper function for LowerBRCOND.
static unsigned addPermMasks(unsigned First, unsigned Second)
static uint64_t clearUnusedBits(uint64_t Val, unsigned Size)
static SDValue getFPTernOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL, EVT VT, SDValue A, SDValue B, SDValue C, SDValue GlueChain, SDNodeFlags Flags)
static bool isV2F16OrV2BF16(Type *Ty)
static OptimizationRemark emitAtomicRMWLegalRemark(const AtomicRMWInst *RMW, GlobalFPAtomicLegality MemLegality, AtomicFlushDenormalReason DenormReason)
static SDValue emitRemovedIntrinsicError(SelectionDAG &DAG, const SDLoc &DL, EVT VT)
static SDValue getFPBinOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL, EVT VT, SDValue A, SDValue B, SDValue GlueChain, SDNodeFlags Flags)
static SDValue buildPCRelGlobalAddress(SelectionDAG &DAG, const GlobalValue *GV, const SDLoc &DL, int64_t Offset, EVT PtrVT, unsigned GAFlags=SIInstrInfo::MO_NONE)
static const std::optional< ByteProvider< SDValue > > calculateSrcByte(const SDValue Op, uint64_t DestByte, uint64_t SrcIndex=0, unsigned Depth=0)
static bool isV2F16(Type *Ty)
static void initializeM0ToZeroForClusterLoad(SDValue Op, SelectionDAG &DAG, SDLoc DL)
static void allocateSGPR64Input(CCState &CCInfo, ArgDescriptor &Arg)
static uint64_t getIdentityValueForWaveReduction(unsigned Opc)
SI DAG Lowering interface definition.
Interface definition for SIRegisterInfo.
Func getContext().diagnose(DiagnosticInfoUnsupported(Func
const char * Msg
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition Value.cpp:479
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition Statistic.h:171
#define LLVM_DEBUG(...)
Definition Debug.h:119
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
LLVM IR instance of the generic uniformity analysis.
static constexpr int Concat[]
Value * RHS
Value * LHS
The Input class is used to parse a yaml document into in-memory structs and vectors.
static bool isSupportedBitWidth(unsigned Bits)
unsigned allocateBarrierGlobal(const DataLayout &DL, const GlobalVariable &GV)
static std::optional< uint32_t > getLDSKernelIdMetadata(const Function &F)
void setDynLDSAlign(const Function &F, const GlobalVariable &GV)
static std::optional< uint32_t > get32BitAbsoluteAddress(const GlobalValue &GV, unsigned AS)
static unsigned numBitsSigned(SDValue Op, SelectionDAG &DAG)
SDValue SplitVectorLoad(SDValue Op, SelectionDAG &DAG) const
Split a vector load into 2 loads of half the vector.
void analyzeFormalArgumentsCompute(CCState &State, const SmallVectorImpl< ISD::InputArg > &Ins) const
The SelectionDAGBuilder will automatically promote function arguments with illegal types.
SDValue LowerF64ToF16Safe(SDValue Src, const SDLoc &DL, SelectionDAG &DAG) const
SDValue storeStackInputValue(SelectionDAG &DAG, const SDLoc &SL, SDValue Chain, SDValue ArgVal, int64_t Offset) const
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
SDValue splitBinaryBitConstantOpImpl(DAGCombinerInfo &DCI, const SDLoc &SL, unsigned Opc, SDValue LHS, uint32_t ValLo, uint32_t ValHi) const
Split the 64-bit value LHS into two 32-bit components, and perform the binary operation Opc to it wit...
SDValue lowerUnhandledCall(CallLoweringInfo &CLI, SmallVectorImpl< SDValue > &InVals, StringRef Reason) const
virtual SDValue LowerGlobalAddress(AMDGPUMachineFunctionInfo *MFI, SDValue Op, SelectionDAG &DAG) const
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
SDValue addTokenForArgument(SDValue Chain, SelectionDAG &DAG, MachineFrameInfo &MFI, int ClobberedFI) const
bool isKnownNeverNaNForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool SNaN=false, unsigned Depth=0) const override
If SNaN is false,.
static bool needsDenormHandlingF32(const SelectionDAG &DAG, SDValue Src, SDNodeFlags Flags)
uint32_t getImplicitParameterOffset(const MachineFunction &MF, const ImplicitParameter Param) const
Helper function that returns the byte offset of the given type of implicit parameter.
SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const
static KnownBits computeKnownBitsForMul24(const KnownBits &LHSKnown, const KnownBits &RHSKnown, unsigned BitWidth, bool IsSigned, bool SelfMultiply)
SDValue loadInputValue(SelectionDAG &DAG, const TargetRegisterClass *RC, EVT VT, const SDLoc &SL, const ArgDescriptor &Arg) const
AMDGPUTargetLowering(const TargetMachine &TM, const TargetSubtargetInfo &STI, const AMDGPUSubtarget &AMDGPUSTI)
static EVT getEquivalentMemType(LLVMContext &Context, EVT VT)
SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const
SDValue CreateLiveInRegister(SelectionDAG &DAG, const TargetRegisterClass *RC, Register Reg, EVT VT, const SDLoc &SL, bool RawReg=false) const
Helper function that adds Reg to the LiveIn list of the DAG's MachineFunction.
SDValue SplitVectorStore(SDValue Op, SelectionDAG &DAG) const
Split a vector store into 2 stores of half the vector.
std::pair< SDValue, SDValue > split64BitValue(SDValue Op, SelectionDAG &DAG) const
Return 64-bit value Op as two 32-bit integers.
static CCAssignFn * CCAssignFnForReturn(CallingConv::ID CC, bool IsVarArg)
static CCAssignFn * CCAssignFnForCall(CallingConv::ID CC, bool IsVarArg)
Selects the correct CCAssignFn for a given CallingConvention value.
static unsigned numBitsUnsigned(SDValue Op, SelectionDAG &DAG)
static bool allowApproxFunc(const SelectionDAG &DAG, SDNodeFlags Flags)
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, const SmallVectorImpl< SDValue > &OutVals, const SDLoc &DL, SelectionDAG &DAG) const override
This hook must be implemented to lower outgoing return values, described by the Outs array,...
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
SDValue performRcpCombine(SDNode *N, DAGCombinerInfo &DCI) const
SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const
static bool shouldFoldFNegIntoSrc(SDNode *FNeg, SDValue FNegSrc)
bool isNarrowingProfitable(SDNode *N, EVT SrcVT, EVT DestVT) const override
Return true if it's profitable to narrow operations of type SrcVT to DestVT.
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
SDValue WidenOrSplitVectorLoad(SDValue Op, SelectionDAG &DAG) const
Widen a suitably aligned v3 load.
SDValue getHiHalf64(SDValue Op, SelectionDAG &DAG) const
bool isNoopAddrSpaceCast(const DataLayout &DL, unsigned SrcAS, unsigned DestAS) const override
Returns true if a cast between SrcAS and DestAS is a noop.
const std::array< unsigned, 3 > & getDims() const
static const LaneMaskConstants & get(const GCNSubtarget &ST)
static const fltSemantics & IEEEsingle()
Definition APFloat.h:304
static constexpr roundingMode rmNearestTiesToEven
Definition APFloat.h:361
static const fltSemantics & IEEEhalf()
Definition APFloat.h:302
static APFloat getQNaN(const fltSemantics &Sem, bool Negative=false, const APInt *payload=nullptr)
Factory for QNaN values.
Definition APFloat.h:1224
LLVM_ABI opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
Definition APFloat.cpp:6045
LLVM_READONLY int getExactLog2Abs() const
Definition APFloat.h:1654
bool isNegative() const
Definition APFloat.h:1583
bool isNormal() const
Definition APFloat.h:1587
static APFloat getOne(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative One.
Definition APFloat.h:1192
static APFloat getLargest(const fltSemantics &Sem, bool Negative=false)
Returns the largest finite number in the given semantics.
Definition APFloat.h:1242
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Definition APFloat.h:1202
static APFloat getZero(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Zero.
Definition APFloat.h:1183
bool isInfinity() const
Definition APFloat.h:1580
Class for arbitrary precision integers.
Definition APInt.h:78
LLVM_ABI APInt zext(unsigned width) const
Zero extend to a new width.
Definition APInt.cpp:1057
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
Definition APInt.h:202
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
Definition APInt.h:254
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
Definition APInt.h:376
bool isSignMask() const
Check if the APInt's value is returned by getSignMask.
Definition APInt.h:462
unsigned countr_zero() const
Count the number of trailing zero bits.
Definition APInt.h:1659
bool isOneBitSet(unsigned BitNo) const
Determine if this APInt Value only has the specified bit set.
Definition APInt.h:362
bool isMask(unsigned numBits) const
Definition APInt.h:484
bool isSignBitSet() const
Determine if sign bit of this APInt is set.
Definition APInt.h:337
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
Definition APInt.h:292
bool sge(const APInt &RHS) const
Signed greater or equal comparison.
Definition APInt.h:1241
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
Definition APInt.h:1225
unsigned getAddressSpace() const
Return the address space for the allocation.
This class represents an incoming formal argument to a Function.
Definition Argument.h:32
LLVM_ABI bool hasAttribute(Attribute::AttrKind Kind) const
Check if an argument has a given attribute.
Definition Function.cpp:332
const Function * getParent() const
Definition Argument.h:44
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
size_t size() const
Get the array size.
Definition ArrayRef.h:141
bool empty() const
Check if the array is empty.
Definition ArrayRef.h:136
An instruction that atomically checks whether a specified value is in a memory location,...
bool isVolatile() const
Return true if this is a cmpxchg from a volatile memory location.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
static unsigned getPointerOperandIndex()
an instruction that atomically reads a memory location, combines it with another value,...
static unsigned getPointerOperandIndex()
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ Add
*p = old + v
@ FAdd
*p = old + v
@ USubCond
Subtract only if no unsigned overflow.
@ Min
*p = old <signed v ? old : v
@ Sub
*p = old - v
@ And
*p = old & v
@ Xor
*p = old ^ v
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
@ FSub
*p = old - v
@ UIncWrap
Increment one up to a maximum value.
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
@ UMax
*p = old >unsigned v ? old : v
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
@ UDecWrap
Decrement one until a minimum value or zero.
@ Nand
*p = ~(old & v)
void setOperation(BinOp Operation)
BinOp getOperation() const
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this rmw instruction.
static LLVM_ABI StringRef getOperationName(BinOp Op)
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
bool isCompareAndSwap() const
Returns true if this SDNode represents cmpxchg atomic operation, false otherwise.
This class holds the attributes for a particular argument, parameter, function, or return value.
Definition Attributes.h:448
LLVM_ABI MemoryEffects getMemoryEffects() const
LLVM Basic Block Representation.
Definition BasicBlock.h:62
iterator end()
Definition BasicBlock.h:453
LLVM_ABI BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="")
Split the basic block into two basic blocks at the specified instruction.
const Function * getParent() const
Return the enclosing method, or null if none.
Definition BasicBlock.h:213
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition BasicBlock.h:206
A "pseudo-class" with methods for operating on BUILD_VECTORs.
Represents known origin of an individual byte in combine pattern.
static ByteProvider getConstantZero()
static ByteProvider getSrc(std::optional< ISelOp > Val, int64_t ByteOffset, int64_t VectorOffset)
CCState - This class holds information needed while lowering arguments and return values.
MachineFunction & getMachineFunction() const
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
static LLVM_ABI bool resultsCompatible(CallingConv::ID CalleeCC, CallingConv::ID CallerCC, MachineFunction &MF, LLVMContext &C, const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn CalleeFn, CCAssignFn CallerFn)
Returns true if the results of the two calling conventions are compatible.
LLVM_ABI void AnalyzeCallResult(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeCallResult - Analyze the return values of a call, incorporating info about the passed values i...
MCRegister AllocateReg(MCPhysReg Reg)
AllocateReg - Attempt to allocate one register.
LLVM_ABI bool CheckReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
CheckReturn - Analyze the return values of a function, returning true if the return can be performed ...
LLVM_ABI void AnalyzeReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeReturn - Analyze the returned values of a return, incorporating info about the result values i...
int64_t AllocateStack(unsigned Size, Align Alignment)
AllocateStack - Allocate a chunk of stack space with the specified size and alignment.
LLVM_ABI void AnalyzeCallOperands(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeCallOperands - Analyze the outgoing arguments to a call, incorporating info about the passed v...
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
bool isAllocated(MCRegister Reg) const
isAllocated - Return true if the specified register (or an alias) is allocated.
LLVM_ABI void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
CCValAssign - Represent assignment of one arg/retval to a location.
Register getLocReg() const
LocInfo getLocInfo() const
int64_t getLocMemOffset() const
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
bool hasFnAttr(Attribute::AttrKind Kind) const
Determine whether this call has the given attribute.
LLVM_ABI bool isMustTailCall() const
Tests if this call site must be tail call optimized.
Value * getArgOperand(unsigned i) const
unsigned arg_size() const
This class represents a function call, abstracting a target machine's calling convention.
bool isTailCall() const
static LLVM_ABI CastInst * CreatePointerCast(Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Create a BitCast, AddrSpaceCast or a PtrToInt cast instruction.
Constant * getValue() const
Definition Metadata.h:557
const APFloat & getValueAPF() const
bool isPosZero() const
Return true if the value is positive zero.
bool isOne() const
Returns true if this value is exactly +1.0.
bool isMinusOne() const
Returns true if this value is exactly -1.0.
bool isNegative() const
Return true if the value is negative.
This is the shared class of boolean and integer constants.
Definition Constants.h:87
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition Constants.h:219
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:168
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
This is an important base class in LLVM.
Definition Constant.h:44
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
LLVM_ABI Align getABITypeAlign(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
bool isBigEndian() const
Definition DataLayout.h:221
A debug info location.
Definition DebugLoc.h:126
Diagnostic information for unsupported feature in backend.
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition TypeSize.h:306
Class to represent fixed width SIMD vectors.
unsigned getNumElements() const
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
Register DemoteRegister
DemoteRegister - if CanLowerReturn is false, DemoteRegister is a vreg allocated to hold a pointer to ...
LLVM_ABI const Value * getValueFromVirtualReg(Register Vreg)
This method is called from TargetLowerinInfo::isSDNodeSourceOfDivergence to get the Value correspondi...
Class to represent function types.
Type * getParamType(unsigned i) const
Parameter type accessors.
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition Function.h:212
const DataLayout & getDataLayout() const
Get the data layout of the module this function belongs to.
Definition Function.cpp:356
iterator_range< arg_iterator > args()
Definition Function.h:886
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition Function.h:273
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition Function.cpp:352
DenormalMode getDenormalMode(const fltSemantics &FPType) const
Returns the denormal handling type for the default rounding mode of the function.
Definition Function.cpp:807
size_t arg_size() const
Definition Function.h:895
Argument * getArg(unsigned i) const
Definition Function.h:880
const SIInstrInfo * getInstrInfo() const override
bool hasMadF16() const
unsigned getInstCacheLineSize() const
Instruction cache line size in bytes (64 for pre-GFX11, 128 for GFX11+).
const SIRegisterInfo * getRegisterInfo() const override
unsigned getMaxPrivateElementSize(bool ForBufferRSrc=false) const
bool hasPrivateSegmentBuffer() const
const MachineFunction & getMachineFunction() const
void computeKnownBitsImpl(Register R, KnownBits &Known, const APInt &DemandedElts, unsigned Depth=0)
Evaluate a known-bits query with an explicit worklist instead of recursive descent.
bool isDivergentAtDef(ConstValueRefT V) const
Whether V is divergent at its definition.
LLVM_ABI unsigned getAddressSpace() const
const GlobalValue * getGlobal() const
bool hasExternalLinkage() const
unsigned getAddressSpace() const
Module * getParent()
Get the module that this global value is contained inside of...
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this global belongs to.
Definition Globals.cpp:205
Type * getValueType() const
LLVM_ABI uint64_t getGlobalSize(const DataLayout &DL) const
Get the size of this global variable in bytes.
Definition Globals.cpp:640
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2990
LLVM_ABI Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following:
LLVM_ABI void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
bool hasMetadata() const
Return true if this instruction has any metadata attached to it.
LLVM_ABI const Function * getFunction() const
Return the function this instruction belongs to.
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
LLVM_ABI InstListType::iterator insertInto(BasicBlock *ParentBB, InstListType::iterator It)
Inserts an unlinked instruction into ParentBB at position It and returns the iterator of the inserted...
Class to represent integer types.
A wrapper class for inspecting calls to intrinsic functions.
bool isKnownNeverNaN() const
Return true if it's known this can never be a nan.
static LLVM_ABI KnownFPClass bitcast(const fltSemantics &FltSemantics, const KnownBits &Bits)
Report known values for a bitcast into a float with provided semantics.
constexpr unsigned getScalarSizeInBits() const
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
static LLT integer(unsigned SizeInBits)
LLT changeElementSize(unsigned NewEltSize) const
If this type is a vector, return a vector with the same number of elements but the new element size.
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
LLVM_ABI void emitError(const Instruction *I, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
LLVM_ABI SyncScope::ID getOrInsertSyncScopeID(StringRef SSN)
getOrInsertSyncScopeID - Maps synchronization scope name to synchronization scope ID.
An instruction for reading from memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
static unsigned getPointerOperandIndex()
This class is used to represent ISD::LOAD nodes.
bool hasValue() const
TypeSize getValue() const
Describe properties that are true of each instruction in the target description file.
unsigned getID() const
getID() - Return the register class ID number.
MCRegister getRegister(unsigned i) const
getRegister - Return the specified register in the class.
unsigned getNumRegs() const
getNumRegs - Return the number of registers in this class.
iterator begin() const
begin/end - Return all of the registers in this class.
Wrapper class representing physical registers. Should be passed by value.
Definition MCRegister.h:41
LLVM_ABI MDNode * createRange(const APInt &Lo, const APInt &Hi)
Return metadata describing the range [Lo, Hi).
Definition MDBuilder.cpp:92
Metadata node.
Definition Metadata.h:1087
const MDOperand & getOperand(unsigned I) const
Definition Metadata.h:1443
Helper class for constructing bundles of MachineInstrs.
MachineBasicBlock::instr_iterator begin() const
Return an iterator to the first bundled instruction.
Machine Value Type.
SimpleValueType SimpleTy
uint64_t getScalarSizeInBits() const
bool bitsLE(MVT VT) const
Return true if this has no more bits than VT.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
MVT changeTypeToInteger()
Return the type converted to an equivalently sized integer or vector with integer element type.
static LLVM_ABI MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
static auto all_valuetypes()
SimpleValueType Iteration.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
static MVT getVectorVT(MVT VT, unsigned NumElements)
static MVT getIntegerVT(unsigned BitWidth)
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
LLVM_ABI void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
LLVM_ABI iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
LLVM_ABI void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
LLVM_ABI MachineBasicBlock * splitAt(MachineInstr &SplitInst, bool UpdateLiveIns=true, LiveIntervals *LIS=nullptr)
Split a basic block into 2 pieces at SplitPoint.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
Align getAlignment() const
Return alignment of the basic block.
MachineInstrBundleIterator< MachineInstr > iterator
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
LLVM_ABI int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
const AllocaInst * getObjectAllocation(int ObjectIdx) const
Return the underlying Alloca of the specified stack object if it exists.
bool hasCalls() const
Return true if the current function has any function calls.
void setHasTailCall(bool V=true)
void setReturnAddressIsTaken(bool s)
bool hasStackObjects() const
Return true if there are any stack objects in this function.
PseudoSourceValueManager & getPSVManager() const
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
DenormalMode getDenormalMode(const fltSemantics &FPType) const
Returns the denormal handling type for the default rounding mode of the function.
void push_back(MachineBasicBlock *MBB)
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
BasicBlockListType::iterator iterator
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineInstr - Allocate a new MachineInstr.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
const MachineInstrBuilder & setOperandDead(unsigned OpIdx) const
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addDef(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register definition operand.
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
Representation of each machine instruction.
const MachineOperand & getOperand(unsigned i) const
A description of a memory reference used in the backend.
LocationSize getSize() const
Return the size in bytes of the memory reference.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
Flags getFlags() const
Return the raw flags of the source value,.
MachineOperand class - Representation of each machine instruction operand.
int64_t getImm() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
static MachineOperand CreateImm(int64_t Val)
void setIsUndef(bool Val=true)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
An SDNode that represents everything that will be needed to construct a MachineInstr.
This is an abstract virtual class for memory operations.
unsigned getAddressSpace() const
Return the address space for the associated pointer.
MachineMemOperand * getMemOperand() const
Return the unique MachineMemOperand object describing the memory reference performed by operation.
EVT getMemoryVT() const
Return the type of the in-memory value.
bool onlyWritesMemory() const
Whether this function only (at most) writes memory.
Definition ModRef.h:252
bool doesNotAccessMemory() const
Whether this function accesses no memory.
Definition ModRef.h:246
bool onlyReadsMemory() const
Whether this function only (at most) reads memory.
Definition ModRef.h:249
const Triple & getTargetTriple() const
Get the target triple which is a string describing the target host.
Definition Module.h:328
The optimization diagnostic interface.
LLVM_ABI void emit(DiagnosticInfoOptimizationBase &OptDiag)
Output the remark via the diagnostic handler and to the optimization record file.
Diagnostic information for applied optimization remarks.
static LLVM_ABI PointerType * get(LLVMContext &C, unsigned AddressSpace)
This constructs an opaque pointer to an object in a numbered address space.
Definition Type.cpp:882
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
LLVM_ABI const PseudoSourceValue * getConstantPool()
Return a pseudo source value referencing the constant pool.
Wrapper class representing virtual and physical registers.
Definition Register.h:20
static Register index2VirtReg(unsigned Index)
Convert a 0-based index to a virtual register number.
Definition Register.h:72
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition Register.h:83
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool isDivergent() const
bool hasOneUse() const
Return true if there is exactly one use of this node.
value_iterator value_end() const
ISD::CondCode getOperandCondCode(unsigned Num) const
Helper method returns the CondCode of a CondCodeSDNode.
SDNodeFlags getFlags() const
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool isUndef() const
Returns true if the node type is UNDEF or POISON.
user_iterator user_begin() const
Provide iteration support to walk over all users of an SDNode.
op_iterator op_end() const
bool isAnyAdd() const
Returns true if the node type is ADD or PTRADD.
value_iterator value_begin() const
op_iterator op_begin() const
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
bool isUndef() const
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node, in exactly one operand.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
unsigned getNumOperands() const
static unsigned getMaxMUBUFImmOffset(const GCNSubtarget &ST)
static unsigned getDSShaderTypeValue(const MachineFunction &MF)
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
AMDGPU::ClusterDimsAttr getClusterDims() const
SIModeRegisterDefaults getMode() const
std::tuple< const ArgDescriptor *, const TargetRegisterClass *, LLT > getPreloadedValue(AMDGPUFunctionArgInfo::PreloadedValue Value) const
const AMDGPUGWSResourcePseudoSourceValue * getGWSPSV(const AMDGPUTargetMachine &TM)
static LLVM_READONLY const TargetRegisterClass * getSGPRClassForBitWidth(unsigned BitWidth)
static unsigned getSubRegFromChannel(uint64_t Channel, uint64_t NumRegs=1)
static bool isVGPRClass(const TargetRegisterClass *RC)
static bool isSGPRClass(const TargetRegisterClass *RC)
static bool isAGPRClass(const TargetRegisterClass *RC)
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
bool isTypeDesirableForOp(unsigned Op, EVT VT) const override
Return true if the target has native support for the specified value type and it is 'desirable' to us...
SDNode * PostISelFolding(MachineSDNode *N, SelectionDAG &DAG) const override
Fold the instructions after selecting them.
SDValue splitTernaryVectorOp(SDValue Op, SelectionDAG &DAG) const
MachineSDNode * wrapAddr64Rsrc(SelectionDAG &DAG, const SDLoc &DL, SDValue Ptr) const
bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT VT) const override
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
SDValue lowerGET_ROUNDING(SDValue Op, SelectionDAG &DAG) const
AtomicExpansionKind shouldExpandAtomicRMWInIR(const AtomicRMWInst *) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
bool requiresUniformRegister(MachineFunction &MF, const Value *V) const override
Allows target to decide about the register class of the specific value that is live outside the defin...
bool isFMADLegal(const SelectionDAG &DAG, const SDNode *N) const override
Returns true if be combined with to form an ISD::FMAD.
AtomicExpansionKind shouldExpandAtomicStoreInIR(StoreInst *SI) const override
Returns how the given (atomic) store should be expanded by the IR-level AtomicExpand pass into.
void bundleInstWithWaitcnt(MachineInstr &MI) const
Insert MI into a BUNDLE with an S_WAITCNT 0 immediately following it.
SDValue lowerROTR(SDValue Op, SelectionDAG &DAG) const
MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override
Return the type to use for a scalar shift opcode, given the shifted amount type.
SDValue LowerCall(CallLoweringInfo &CLI, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower calls into the specified DAG.
MVT getPointerTy(const DataLayout &DL, unsigned AS) const override
Map address space 7 to MVT::amdgpuBufferFatPointer because that's its in-memory representation.
bool denormalsEnabledForType(const SelectionDAG &DAG, EVT VT) const
void insertCopiesSplitCSR(MachineBasicBlock *Entry, const SmallVectorImpl< MachineBasicBlock * > &Exits) const override
Insert explicit copies in entry and exit blocks.
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the ValueType of the result of SETCC operations.
SDNode * legalizeTargetIndependentNode(SDNode *Node, SelectionDAG &DAG) const
Legalize target independent instructions (e.g.
bool allowsMisalignedMemoryAccessesImpl(unsigned Size, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *IsFast=nullptr) const
TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const override
Return the preferred vector type legalization action.
SDValue lowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const
const GCNSubtarget * getSubtarget() const
bool enableAggressiveFMAFusion(EVT VT) const override
Return true if target always benefits from combining into FMA for a given value type.
bool shouldEmitGOTReloc(const GlobalValue *GV) const
SDValue splitUnaryVectorOp(SDValue Op, SelectionDAG &DAG) const
SDValue lowerGET_FPENV(SDValue Op, SelectionDAG &DAG) const
bool isCanonicalized(SelectionDAG &DAG, SDValue Op, SDNodeFlags UserFlags={}, unsigned MaxDepth=5) const
void allocateSpecialInputSGPRs(CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const
void allocateLDSKernelId(CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const
SDValue LowerSTACKSAVE(SDValue Op, SelectionDAG &DAG) const
bool isReassocProfitable(SelectionDAG &DAG, SDValue N0, SDValue N1) const override
void allocateHSAUserSGPRs(CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const
ArrayRef< MCPhysReg > getRoundingControlRegisters() const override
Returns a 0 terminated array of rounding control registers that can be attached into strict FP call.
ConstraintType getConstraintType(StringRef Constraint) const override
Given a constraint, return the type of constraint it is for this target.
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, const SmallVectorImpl< SDValue > &OutVals, const SDLoc &DL, SelectionDAG &DAG) const override
This hook must be implemented to lower outgoing return values, described by the Outs array,...
const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent) const override
Return the register class that should be used for the specified value type.
void AddMemOpInit(MachineInstr &MI) const
MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const override
This callback is used to inspect load/store instructions and add target-specific MachineMemOperand fl...
bool isLegalGlobalAddressingMode(const AddrMode &AM) const
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Return true if it is beneficial to convert a load of a constant to just the constant itself.
InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const override
bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags, unsigned *IsFast) const override
Return true if the target supports a memory access of this type for the given address space and align...
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
void emitExpandAtomicStore(StoreInst *SI) const override
Perform a atomic store using a target-specific way.
AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const override
Returns how the given (atomic) load should be expanded by the IR-level AtomicExpand pass.
Align computeKnownAlignForTargetInstr(GISelValueTracking &Analysis, Register R, const MachineRegisterInfo &MRI, unsigned Depth=0) const override
Determine the known alignment for the pointer value R.
bool getAsmOperandConstVal(SDValue Op, uint64_t &Val) const
bool isShuffleMaskLegal(ArrayRef< int >, EVT) const override
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations,...
void emitExpandAtomicLoad(LoadInst *LI) const override
Perform a atomic load using a target-specific way.
EVT getOptimalMemOpType(LLVMContext &Context, const MemOp &Op, const AttributeList &FuncAttributes) const override
Returns the target specific optimal type for load and store operations as a result of memset,...
void computeKnownBitsForStackObjectPointer(KnownBits &Known, const MachineFunction &MF, Align Alignment) const override
Determine known bits of a pointer to a known valid stack object.
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
Register getRegisterByName(const char *RegName, LLT VT, const MachineFunction &MF) const override
Return the register ID of the name passed in.
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
Lower the specified operand into the Ops vector.
LLT getPreferredShiftAmountTy(LLT Ty) const override
Return the preferred type to use for a shift opcode, given the shifted amount type is ShiftValueTy.
ExtractSubvectorCost getExtractSubvectorCost(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return the cost of extracting a subvector of type ResVT from a vector of type SrcVT,...
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
Align getPrefLoopAlignment(MachineLoop *ML, const MachineBasicBlock *BlockToAlign) const override
Return the preferred loop alignment.
SDValue lowerSET_FPENV(SDValue Op, SelectionDAG &DAG) const
bool shouldPreservePtrArith(const Function &F, EVT PtrVT) const override
True if target has some particular form of dealing with pointer arithmetic semantics for pointers wit...
void getTgtMemIntrinsic(SmallVectorImpl< IntrinsicInfo > &, const CallBase &, MachineFunction &MF, unsigned IntrinsicID) const override
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
SDValue lowerSET_ROUNDING(SDValue Op, SelectionDAG &DAG) const
void allocateSpecialInputVGPRsFixed(CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const
Allocate implicit function VGPR arguments in fixed registers.
MachineBasicBlock * emitGWSMemViolTestLoop(MachineInstr &MI, MachineBasicBlock *BB) const
bool getAddrModeArguments(const IntrinsicInst *I, SmallVectorImpl< Value * > &Ops, Type *&AccessTy) const override
CodeGenPrepare sinks address calculations into the same BB as Load/Store instructions reading the add...
bool checkAsmConstraintValA(SDValue Op, uint64_t Val, unsigned MaxSize=64) const
bool shouldEmitFixup(const GlobalValue *GV) const
MachineBasicBlock * splitKillBlock(MachineInstr &MI, MachineBasicBlock *BB) const
void emitExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) const override
Perform a cmpxchg expansion using a target-specific method.
bool canTransformPtrArithOutOfBounds(const Function &F, EVT PtrVT) const override
True if the target allows transformations of in-bounds pointer arithmetic that cause out-of-bounds in...
bool hasMemSDNodeUser(SDNode *N) const
bool isSDNodeSourceOfDivergence(const SDNode *N, FunctionLoweringInfo *FLI, UniformityInfo *UA) const override
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
bool isEligibleForTailCallOptimization(SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, const SmallVectorImpl< SDValue > &OutVals, const SmallVectorImpl< ISD::InputArg > &Ins, SelectionDAG &DAG) const
bool isMemOpHasNoClobberedMemOperand(const SDNode *N) const
bool isLegalFlatAddressingMode(const AddrMode &AM, unsigned AddrSpace) const
SDValue LowerCallResult(SDValue Chain, SDValue InGlue, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl< ISD::InputArg > &Ins, const SDLoc &DL, SelectionDAG &DAG, SmallVectorImpl< SDValue > &InVals, bool isThisReturn, SDValue ThisVal) const
SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl< ISD::InputArg > &Ins, const SDLoc &DL, SelectionDAG &DAG, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array,...
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
bool isFPExtFoldable(const SelectionDAG &DAG, unsigned Opcode, EVT DestVT, EVT SrcVT) const override
Return true if an fpext operation input to an Opcode operation is free (for instance,...
void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const override
Assign the register class depending on the number of bits set in the writemask.
MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
void finalizeLowering(MachineFunction &MF) const override
Execute target specific actions to finalize target lowering.
static bool isNonGlobalAddrSpace(unsigned AS)
void emitExpandAtomicAddrSpacePredicate(Instruction *AI) const
MachineSDNode * buildRSRC(SelectionDAG &DAG, const SDLoc &DL, SDValue Ptr, uint32_t RsrcDword1, uint64_t RsrcDword2And3) const
Return a resource descriptor with the 'Add TID' bit enabled The TID (Thread ID) is multiplied by the ...
unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Certain targets require unusual breakdowns of certain types.
bool mayBeEmittedAsTailCall(const CallInst *) const override
Return true if the target may be able emit the call instruction as a tail call.
void passSpecialInputs(CallLoweringInfo &CLI, CCState &CCInfo, const SIMachineFunctionInfo &Info, SmallVectorImpl< std::pair< unsigned, SDValue > > &RegsToPass, SmallVectorImpl< SDValue > &MemOpChains, SDValue Chain) const
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
bool checkAsmConstraintVal(SDValue Op, StringRef Constraint, uint64_t Val) const
bool isKnownNeverNaNForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool SNaN=false, unsigned Depth=0) const override
If SNaN is false,.
void emitExpandAtomicRMW(AtomicRMWInst *AI) const override
Perform a atomicrmw expansion using a target-specific way.
static bool shouldExpandVectorDynExt(unsigned EltSize, unsigned NumElem, bool IsDivergentIdx, const GCNSubtarget *Subtarget)
Check if EXTRACT_VECTOR_ELT/INSERT_VECTOR_ELT (<n x e>, var-idx) should be expanded into a set of cmp...
bool shouldUseLDSConstAddress(const GlobalValue *GV) const
bool supportSplitCSR(MachineFunction *MF) const override
Return true if the target supports that a subset of CSRs for the given machine function is handled ex...
bool isExtractVecEltCheap(EVT VT, unsigned Index) const override
Return true if extraction of a scalar element from the given vector type at the given index is cheap.
SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const
bool allowsMisalignedMemoryAccesses(LLT Ty, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *IsFast=nullptr) const override
LLT handling variant.
bool canMergeStoresTo(unsigned AS, EVT MemVT, const MachineFunction &MF) const override
Returns if it's reasonable to merge stores to MemVT size.
SDValue lowerPREFETCH(SDValue Op, SelectionDAG &DAG) const
SITargetLowering(const TargetMachine &tm, const GCNSubtarget &STI)
void computeKnownBitsForTargetInstr(GISelValueTracking &Analysis, Register R, KnownBits &Known, const APInt &DemandedElts, const MachineRegisterInfo &MRI, unsigned Depth=0) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
bool isFreeAddrSpaceCast(const DataLayout &DL, unsigned SrcAS, unsigned DestAS) const override
Returns true if a cast from SrcAS to DestAS is "cheap", such that e.g.
bool shouldEmitPCReloc(const GlobalValue *GV) const
bool isUniformLoad(const LoadSDNode *Load) const
AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(const AtomicCmpXchgInst *AI) const override
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
void initializeSplitCSR(MachineBasicBlock *Entry) const override
Perform necessary initialization to handle a subset of CSRs explicitly via copies.
void allocateSpecialEntryInputVGPRs(CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const
void allocatePreloadKernArgSGPRs(CCState &CCInfo, SmallVectorImpl< CCValAssign > &ArgLocs, const SmallVectorImpl< ISD::InputArg > &Ins, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const
SDValue copyToM0(SelectionDAG &DAG, SDValue Chain, const SDLoc &DL, SDValue V) const
SDValue splitBinaryVectorOp(SDValue Op, SelectionDAG &DAG) const
MachinePointerInfo getKernargSegmentPtrInfo(MachineFunction &MF) const
unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const override
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
MVT getPointerMemTy(const DataLayout &DL, unsigned AS) const override
Similarly, the in-memory representation of a p7 is {p8, i32}, aka v8i32 when padding is added.
void allocateSystemSGPRs(CCState &CCInfo, MachineFunction &MF, SIMachineFunctionInfo &Info, CallingConv::ID CallConv, bool IsShader) const
bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF, bool isVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, LLVMContext &Context, const Type *RetTy) const override
This hook should be implemented to check whether the return values described by the Outs array can fi...
unsigned getMaxPermittedBytesForAlignment(MachineBasicBlock *MBB) const override
Return the maximum amount of bytes allowed to be emitted when padding for alignment.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
SDValue getExtractVectorElt(const SDLoc &DL, EVT VT, SDValue Vec, unsigned Idx)
Extract element at Idx from Vec.
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
bool isKnownNeverSNaN(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
const TargetSubtargetInfo & getSubtarget() const
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, Register Reg, SDValue N)
LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
LLVM_ABI SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
LLVM_ABI void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
LLVM_ABI SDValue getAtomicLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT MemVT, EVT VT, SDValue Chain, SDValue Ptr, MachineMemOperand *MMO)
LLVM_ABI SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
LLVM_ABI bool isConstantIntBuildVectorOrConstantInt(SDValue N, bool AllowOpaques=true) const
Test whether the given value is a constant int or similar node.
LLVM_ABI SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
LLVM_ABI SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
LLVM_ABI bool haveNoCommonBitsSet(SDValue A, SDValue B) const
Return true if A and B have no common bits set.
LLVM_ABI SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS, unsigned DestAS, const SDNodeFlags Flags=SDNodeFlags())
Return an AddrSpaceCastSDNode.
LLVM_ABI SDValue getRegister(Register Reg, EVT VT)
LLVM_ABI bool SignBitIsZeroFP(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero, for a floating-point value.
LLVM_ABI SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=LocationSize::precise(0), const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false, SDNodeFlags Flags={})
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
LLVM_ABI SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
LLVM_ABI SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
LLVM_ABI SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align DstAlign, Align SrcAlign, bool isVol, bool AlwaysInline, const CallInst *CI, std::optional< bool > OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), BatchAAResults *BatchAA=nullptr)
const TargetLowering & getTargetLoweringInfo() const
LLVM_ABI std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
LLVM_ABI SDValue getBitcastedAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by first bitcasting (from potentia...
LLVM_ABI SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, Register Reg, EVT VT)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
LLVM_ABI void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
LLVM_ABI SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
LLVM_ABI SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl< SDValue > &Vals)
Creates a new TokenFactor containing Vals.
LLVM_ABI SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Helper function to build ISD::STORE nodes.
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
LLVM_ABI SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getSignedTargetConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
LLVM_ABI SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
LLVM_ABI void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
LLVM_ABI SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
LLVM_ABI SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Loads are not normal binary operators: their result type is not determined by their operands,...
const TargetMachine & getTarget() const
LLVM_ABI SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
LLVM_ABI SDValue getValueType(EVT)
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
LLVM_ABI bool isKnownNeverNaN(SDValue Op, const APInt &DemandedElts, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue (or all elements of it, if it is a vector) is known to never be NaN in...
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
LLVM_ABI bool isBaseWithConstantOffset(SDValue Op) const
Return true if the specified operand is an ISD::ADD with a ConstantSDNode on the right-hand side,...
LLVM_ABI SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getPOISON(EVT VT)
Return a POISON node. POISON does not have a useful SDLoc.
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
LLVM_ABI SDValue getErrorMergeValues(ArrayRef< EVT > ResultTypes, SDValue Chain, const SDLoc &dl)
Return poison values for each of ResultTypes, substituting Chain for any result of type MVT::Other,...
LLVM_ABI SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
LLVM_ABI KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
LLVM_ABI SDValue getRegisterMask(const uint32_t *RegMask)
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVM_ABI SDValue getCondCode(ISD::CondCode Cond)
LLVM_ABI bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
LLVM_ABI SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
LLVM_ABI std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
LLVM_ABI SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
int getMaskElt(unsigned Idx) const
ArrayRef< int > getMask() const
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition SmallSet.h:176
bool empty() const
Definition SmallSet.h:169
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition SmallSet.h:184
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void resize(size_type N)
void push_back(const T &Elt)
LLVM_NO_SANITIZE_CFI_UNRELATED_CAST iterator begin()
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
constexpr bool empty() const
Check if the string is empty.
Definition StringRef.h:141
constexpr size_t size() const
Get the string size.
Definition StringRef.h:144
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
Information about stack frame layout on the target.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
StackDirection getStackGrowthDirection() const
getStackGrowthDirection - Return the direction the stack grows
TargetInstrInfo - Interface to description of machine instruction set.
Type * Ty
Same as OrigTy, or partially legalized for soft float libcalls.
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
virtual void finalizeLowering(MachineFunction &MF) const
Execute target specific actions to finalize target lowering.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual Align getPrefLoopAlignment(MachineLoop *ML=nullptr, const MachineBasicBlock *BlockToAlign=nullptr) const
Return the preferred loop alignment.
virtual unsigned getMaxPermittedBytesForAlignment(MachineBasicBlock *MBB) const
Return the maximum amount of bytes allowed to be emitted when padding for alignment.
const TargetMachine & getTargetMachine() const
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
void setOperationPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
Convenience method to set an operation to Promote and specify the type in a single call.
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
void setHasExtractBitsInsn(bool hasExtractInsn=true)
Tells the code generator that the target has BitExtract instructions.
virtual TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const
Return the preferred vector type legalization action.
virtual unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
Register getStackPointerRegisterToSaveRestore() const
If a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save...
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
bool isOperationCustom(unsigned Op, EVT VT) const
Return true if the operation uses custom lowering, regardless of whether the type is legal or not.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
ExtractSubvectorCost
Enum that specifies how expensive lowering an EXTRACT_SUBVECTOR is.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
Return true if the target supports a memory access of this type for the given address space and align...
virtual bool isNarrowingProfitable(SDNode *N, EVT SrcVT, EVT DestVT) const
Return true if it's profitable to narrow operations of type SrcVT to DestVT.
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
If Opc/OrigVT is specified as being promoted, the promotion code defaults to trying a larger integer/...
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
bool allowsMemoryAccessForAlignment(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
This function returns true if the memory access is aligned or if the target allows this specific unal...
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
SDValue scalarizeVectorStore(StoreSDNode *ST, SelectionDAG &DAG) const
virtual InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const
std::vector< AsmOperandInfo > AsmOperandInfoVector
SDValue SimplifyMultipleUseDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth=0) const
More limited version of SimplifyDemandedBits that can be used to "lookthrough" ops that don't contrib...
SDValue expandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG) const
Expands an unaligned store to 2 half-size stores for integer values, and possibly more for vectors.
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
bool parametersInCSRMatch(const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask, const SmallVectorImpl< CCValAssign > &ArgLocs, const SmallVectorImpl< SDValue > &OutVals) const
Check whether parameters to a call that are passed in callee saved registers are the same as from the...
std::pair< SDValue, SDValue > expandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG) const
Expands an unaligned load to 2 half-size loads for an integer, and possibly more for vectors.
SDValue expandFMINIMUMNUM_FMAXIMUMNUM(SDNode *N, SelectionDAG &DAG) const
Expand fminimumnum/fmaximumnum into multiple comparison with selects.
virtual bool isTypeDesirableForOp(unsigned, EVT VT) const
Return true if the target has native support for the specified value type and it is 'desirable' to us...
virtual void computeKnownBitsForStackObjectPointer(KnownBits &Known, const MachineFunction &MF, Align Alignment) const
Determine known bits of a pointer to a known valid stack object.
std::pair< SDValue, SDValue > scalarizeVectorLoad(LoadSDNode *LD, SelectionDAG &DAG) const
Turn load of vector type into a load of the individual elements.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
TargetLowering(const TargetLowering &)=delete
virtual MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, const CallBase &Call) const
Split up the constraint string from the inline assembly value into the specific constraints and their...
SDValue expandRoundInexactToOdd(EVT ResultVT, SDValue Op, const SDLoc &DL, SelectionDAG &DAG) const
Truncate Op to ResultVT.
virtual void ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG=nullptr) const
Determines the constraint code and constraint type to use for the specific AsmOperandInfo,...
SDValue annotateStackObjectPointer(SDValue Ptr, SelectionDAG &DAG, const SDLoc &DL, Align Alignment) const
Annotate a stack object pointer with known-bits assertions.
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
SDValue expandFMINNUM_FMAXNUM(SDNode *N, SelectionDAG &DAG) const
Expand fminnum/fmaxnum into fminnum_ieee/fmaxnum_ieee with quieted inputs.
Primary interface to the complete machine description for the target machine.
CodeGenOptLevel getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
const Triple & getTargetTriple() const
bool shouldAssumeDSOLocal(const GlobalValue *GV) const
TargetOptions Options
unsigned GuaranteedTailCallOpt
GuaranteedTailCallOpt - This flag is enabled when -tailcallopt is specified on the commandline.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
Definition Triple.h:48
OSType getOS() const
Get the parsed operating system type of this triple.
Definition Triple.h:526
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition TypeSize.h:352
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:294
bool isBFloatTy() const
Return true if this is 'bfloat', a 16-bit bfloat type.
Definition Type.h:147
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:370
bool isHalfTy() const
Return true if this is 'half', a 16-bit IEEE fp type.
Definition Type.h:144
bool isFunctionTy() const
True if this is an instance of FunctionType.
Definition Type.h:268
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:252
LLVM_ABI const fltSemantics & getFltSemantics() const
Definition Type.cpp:96
bool isVoidTy() const
Return true if this is 'void'.
Definition Type.h:141
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
LLVM_ABI unsigned getOperandNo() const
Return the operand # of this use in its User.
Definition Use.cpp:35
LLVM_ABI void set(Value *Val)
Definition Value.h:876
User * getUser() const
Returns the User that contains this Use.
Definition Use.h:61
const Use & getOperandUse(unsigned i) const
Definition User.h:220
Value * getOperand(unsigned i) const
Definition User.h:207
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition Value.h:441
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:548
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:260
iterator_range< user_iterator > users()
Definition Value.h:428
bool use_empty() const
Definition Value.h:348
iterator_range< use_iterator > uses()
Definition Value.h:382
Type * getElementType() const
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:201
constexpr bool isKnownEven() const
A return value of true indicates we know at compile time that the number of elements (vscale * Min) i...
Definition TypeSize.h:177
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition TypeSize.h:166
self_iterator getIterator()
Definition ilist_node.h:123
CallInst * Call
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ CONSTANT_ADDRESS_32BIT
Address space for 32-bit constant memory.
@ BUFFER_STRIDED_POINTER
Address space for 192-bit fat buffer pointers with an additional index.
@ BARRIER
Address space for modeling barrier IDs as addresses.
@ REGION_ADDRESS
Address space for region memory. (GDS)
@ LOCAL_ADDRESS
Address space for local memory.
@ STREAMOUT_REGISTER
Internal address spaces. Can be freely renumbered.
@ VGPR
Address space for VGPRs.
@ CONSTANT_ADDRESS
Address space for constant memory (VTX2).
@ FLAT_ADDRESS
Address space for flat memory.
@ GLOBAL_ADDRESS
Address space for global memory (RAT0, VTX0).
@ BUFFER_FAT_POINTER
Address space for 160-bit buffer fat pointers.
@ PRIVATE_ADDRESS
Address space for private memory.
@ BUFFER_RESOURCE
Address space for 128-bit buffer resources.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char NumVGPRs[]
Key for Kernel::CodeProps::Metadata::mNumVGPRs.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr char SymbolName[]
Key for Kernel::Metadata::mSymbolName.
APInt getExponentOnlyScaleDemandedBits()
LLVM_READONLY const MIMGG16MappingInfo * getMIMGG16MappingInfo(unsigned G)
bool isInlinableLiteralFP16(int16_t Literal, bool HasInv2Pi)
LLVM_READNONE constexpr bool isShader(CallingConv::ID CC)
bool shouldEmitConstantsToTextSection(const Triple &TT)
bool isFlatGlobalAddrSpace(unsigned AS)
const uint64_t FltRoundToHWConversionTable
bool isGFX12Plus(const MCSubtargetInfo &STI)
unsigned getNSAMaxSize(const MCSubtargetInfo &STI, bool HasSampler)
std::optional< unsigned > getExponentOnlyScaleArgIdx(unsigned IntrID)
constexpr int64_t getNullPointerValue(unsigned AS)
Get the null pointer value for the given address space.
bool isGFX11(const MCSubtargetInfo &STI)
bool isGFX13(const MCSubtargetInfo &STI)
bool hasValueInRangeLikeMetadata(const MDNode &MD, int64_t Val)
Checks if Val is inside MD, a !range-like metadata.
LLVM_ABI IsaVersion getIsaVersion(StringRef GPU)
LLVM_READNONE bool isLegalDPALU_DPPControl(const MCSubtargetInfo &ST, unsigned DC)
LLVM_READNONE constexpr bool mayTailCallThisCC(CallingConv::ID CC)
Return true if we might ever do TCO for calls with this calling convention.
unsigned getAMDHSACodeObjectVersion(const Module &M)
LLVM_READONLY bool hasNamedOperand(uint32_t Opcode, OpName NamedIdx)
int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding, unsigned VDataDwords, unsigned VAddrDwords, bool IndexedRsrc, bool IndexedSamp)
LLVM_READNONE constexpr bool isKernel(CallingConv::ID CC)
LLVM_READNONE constexpr bool isEntryFunctionCC(CallingConv::ID CC)
bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi)
LLVM_READNONE constexpr bool isCompute(CallingConv::ID CC)
const BufferIntrinsicInfo * lookupBufferIntrinsic(unsigned Intr)
bool isIntrinsicSourceOfDivergence(unsigned IntrID)
LLVM_READNONE bool isInlinableIntLiteral(int64_t Literal)
Is this literal inlinable, and not one of the values intended for floating point values.
bool getMUBUFTfe(unsigned Opc)
bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi, bool *IsBF16)
TargetExtType * isNamedBarrier(const GlobalVariable &GV)
LLVM_READONLY int32_t getGlobalSaddrOp(uint32_t Opcode)
bool isGFX11Plus(const MCSubtargetInfo &STI)
std::optional< unsigned > getInlineEncodingV2F16(uint32_t Literal)
std::tuple< char, unsigned, unsigned > parseAsmConstraintPhysReg(StringRef Constraint)
Returns a valid charcode or 0 in the first entry if this is a valid physical register constraint.
bool isGFX10Plus(const MCSubtargetInfo &STI)
bool isValidWMMAScaleFmtCombination(unsigned AFmt, unsigned AScale, unsigned BFmt, unsigned BScale)
bool isUniformMMO(const MachineMemOperand *MMO)
std::optional< unsigned > getInlineEncodingV2I16(uint32_t Literal)
uint32_t decodeFltRoundToHWConversionTable(uint32_t FltRounds)
Read the hardware rounding mode equivalent of a AMDGPUFltRounds value.
bool isExtendedGlobalAddrSpace(unsigned AS)
LLVM_READONLY const MIMGDimInfo * getMIMGDimInfo(unsigned DimEnum)
unsigned getBarrierMemberCountBitMask(const IsaVersion &Version)
std::optional< unsigned > getInlineEncodingV2BF16(uint32_t Literal)
LLVM_READONLY const MIMGBaseOpcodeInfo * getMIMGBaseOpcodeInfo(unsigned BaseOpcode)
unsigned getSyntheticApertureNumber(unsigned AS)
LLVM_READNONE constexpr bool isChainCC(CallingConv::ID CC)
constexpr unsigned getSDOperandIdx(unsigned Arg, bool HasChain=true)
Convert IR argument index Arg of an intrinsic into an operand index of the INTRINSIC_* SelectionDAG n...
int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels)
const ImageDimIntrinsicInfo * getImageDimIntrinsicInfo(unsigned Intr)
bool isInlinableLiteralI16(int32_t Literal, bool HasInv2Pi)
LLVM_READNONE constexpr bool canGuaranteeTCO(CallingConv::ID CC)
LLVM_READNONE constexpr bool isGraphics(CallingConv::ID CC)
bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi)
Is this literal inlinable.
const RsrcIntrinsic * lookupRsrcIntrinsic(unsigned Intr)
const uint64_t FltRoundConversionTable
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition CallingConv.h:24
@ AMDGPU_CS
Used for Mesa/AMDPAL compute shaders.
@ AMDGPU_KERNEL
Used for AMDGPU code object kernels.
@ MaxID
The highest possible ID. Must be some 2^k - 1.
@ AMDGPU_Gfx
Used for AMD graphics targets.
@ AMDGPU_CS_ChainPreserve
Used on AMDGPUs to give the middle-end more control over argument placement.
@ AMDGPU_CS_Chain
Used on AMDGPUs to give the middle-end more control over argument placement.
@ AMDGPU_PS
Used for Mesa/AMDPAL pixel shaders.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition ISDOpcodes.h:851
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition ISDOpcodes.h:263
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ PTRADD
PTRADD represents pointer arithmetic semantics, for targets that opt in using shouldPreservePtrArith(...
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
Definition ISDOpcodes.h:47
@ POISON
POISON - A poison node.
Definition ISDOpcodes.h:238
@ SET_FPENV
Sets the current floating-point environment.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition ISDOpcodes.h:277
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition ISDOpcodes.h:605
@ BSWAP
Byte Swap and Counting operators.
Definition ISDOpcodes.h:811
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
Definition ISDOpcodes.h:527
@ ADD
Simple integer binary arithmetic operators.
Definition ISDOpcodes.h:266
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition ISDOpcodes.h:885
@ ATOMIC_LOAD_USUB_COND
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition ISDOpcodes.h:523
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition ISDOpcodes.h:222
@ GlobalAddress
Definition ISDOpcodes.h:90
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition ISDOpcodes.h:912
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition ISDOpcodes.h:589
@ FADD
Simple binary floating point operators.
Definition ISDOpcodes.h:420
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition ISDOpcodes.h:771
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition ISDOpcodes.h:256
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ CONVERT_FROM_ARBITRARY_FP
CONVERT_FROM_ARBITRARY_FP - This operator converts from an arbitrary floating-point represented as an...
@ ATOMIC_LOAD_USUB_SAT
@ CTLZ_ZERO_POISON
Definition ISDOpcodes.h:820
@ SET_ROUNDING
Set rounding mode.
@ CONVERGENCECTRL_GLUE
This does not correspond to any convergence control intrinsic.
@ SIGN_EXTEND
Conversion operators.
Definition ISDOpcodes.h:876
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition ISDOpcodes.h:681
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ BR
Control flow instructions. These all have token chains.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
Definition ISDOpcodes.h:355
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition ISDOpcodes.h:546
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
Definition ISDOpcodes.h:553
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition ISDOpcodes.h:377
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition ISDOpcodes.h:828
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
Definition ISDOpcodes.h:235
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition ISDOpcodes.h:249
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition ISDOpcodes.h:232
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition ISDOpcodes.h:351
@ CTLS
Count leading redundant sign bits.
Definition ISDOpcodes.h:824
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition ISDOpcodes.h:720
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
@ GET_FPENV
Gets the current floating-point environment.
@ SHL
Shift and rotation operations.
Definition ISDOpcodes.h:793
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition ISDOpcodes.h:659
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition ISDOpcodes.h:619
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Same as FMINNUM/FMAXNUM, except that a signaling NaN operand deterministi...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition ISDOpcodes.h:581
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
Definition ISDOpcodes.h:226
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition ISDOpcodes.h:882
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition ISDOpcodes.h:843
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Same semantics as the llvm.minnum/maxnum intrinsics.
@ SMULO
Same for multiplication.
Definition ISDOpcodes.h:359
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition ISDOpcodes.h:920
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition ISDOpcodes.h:751
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition ISDOpcodes.h:837
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
Definition ISDOpcodes.h:331
@ INLINEASM_BR
INLINEASM_BR - Branching version of inline asm. Used by asm-goto.
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ ATOMIC_LOAD_UDEC_WRAP
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition ISDOpcodes.h:505
@ FMINIMUM
FMINIMUM/FMAXIMUM - Same semantics as the llvm.minimum/maximum intrinsics.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition ISDOpcodes.h:958
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition ISDOpcodes.h:510
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition ISDOpcodes.h:763
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition ISDOpcodes.h:207
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition ISDOpcodes.h:570
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition ISDOpcodes.h:55
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ CTTZ_ZERO_POISON
Bit counting operators with a poisoned result for zero inputs.
Definition ISDOpcodes.h:819
@ ExternalSymbol
Definition ISDOpcodes.h:95
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition ISDOpcodes.h:991
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
Definition ISDOpcodes.h:124
@ ADDRSPACECAST
ADDRSPACECAST - This operator converts between pointers of different address spaces.
@ INLINEASM
INLINEASM - Represents an inline asm block.
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition ISDOpcodes.h:977
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition ISDOpcodes.h:888
@ BRCOND
BRCOND - Conditional branch.
@ CONVERT_TO_ARBITRARY_FP
CONVERT_TO_ARBITRARY_FP - Converts a native FP value to an arbitrary floating-point format,...
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
Definition ISDOpcodes.h:865
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition ISDOpcodes.h:64
@ ATOMIC_LOAD_UINC_WRAP
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition ISDOpcodes.h:539
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition ISDOpcodes.h:368
@ FMINIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM - Same semantics as the llvm.minimumnum/maximumnum intrinsics.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition ISDOpcodes.h:215
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition ISDOpcodes.h:561
CondCode getUnsignedSetCC(CondCode Code)
Convert a signed integer condition code to unsigned.
LLVM_ABI CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
This namespace contains an enum with a value for every intrinsic/builtin function known by LLVM.
LLVM_ABI Function * getDeclarationIfExists(const Module *M, ID id)
Look up the Function declaration of the intrinsic id in the Module M and return it if it exists.
LLVM_ABI AttributeSet getFnAttributes(LLVMContext &C, ID id)
Return the function attributes for an intrinsic.
LLVM_ABI StringRef getBaseName(ID id)
Return the LLVM name for an intrinsic, without encoded types for overloading, such as "llvm....
LLVM_ABI AttributeList getAttributes(LLVMContext &C, ID id, FunctionType *FT)
Return the attributes for an intrinsic.
LLVM_ABI FunctionType * getType(LLVMContext &Context, ID id, ArrayRef< Type * > OverloadTys={})
Return the function type for an intrinsic.
BinaryOp_match< SpecificConstantMatch, SrcTy, TargetOpcode::G_SUB > m_Neg(const SrcTy &&Src)
Matches a register negated by a G_SUB.
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
GFCstOrSplatGFCstMatch m_GFCstOrSplat(std::optional< FPValueAndVReg > &FPValReg)
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
cst_pred_ty< is_one > m_One()
Match an integer 1 or a vector with all elements equal to 1.
specific_fpval m_SpecificFP(double V)
Match a specific floating point value or vector with all elements equal to the value.
auto m_Value()
Match an arbitrary value and ignore it.
auto m_FAbs(const Opnd0 &Op0)
BinaryOp_match< LHS, RHS, Instruction::Shl > m_Shl(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Sub > m_Sub(const LHS &L, const RHS &R)
auto m_IntrinsicWOChain(const OpndPreds &...Opnds)
bool sd_match(SDValue N, Pattern &&P)
ConstantInt_match m_ConstInt()
Match any integer constants or splat of an integer constant.
Offsets
Offsets in bytes from the start of the input buffer.
@ System
Synchronized with respect to all concurrently executing threads.
Definition LLVMContext.h:66
constexpr double inv_pi
@ User
could "use" a pointer
DiagnosticInfoOptimizationBase::Argument NV
NodeAddr< UseNode * > Use
Definition RDFGraph.h:385
NodeAddr< NodeBase * > Node
Definition RDFGraph.h:381
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Definition BasicBlock.h:73
unsigned getOpcode(const VPValue *V)
Return the instruction opcode for the recipe defining V or 0 for unsupported recipes and VPValues not...
This is an optimization pass for GlobalISel generic memory operations.
GenericUniformityInfo< SSAContext > UniformityInfo
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:316
@ Offset
Definition DWP.cpp:577
LLVM_ABI void finalizeBundle(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
finalizeBundle - Finalize a machine instruction bundle which includes a sequence of instructions star...
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
detail::zippy< detail::zip_first, T, U, Args... > zip_equal(T &&t, U &&u, Args &&...args)
zip iterator that assumes that all iteratees have the same length.
Definition STLExtras.h:856
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
Definition MathExtras.h:166
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
@ Known
Known to have no common set bits.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Dead
Unused definition.
@ Undef
Value of the register doesn't matter.
@ Define
Register definition.
LLVM_ABI SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
@ Done
Definition Threading.h:60
bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
CCAssignFn - This function assigns a location for Val, updating State to reflect the change.
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
constexpr int64_t minIntN(int64_t N)
Gets the minimum value for a N-bit signed integer.
Definition MathExtras.h:224
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
Definition bit.h:325
SDValue peekFPSignOps(SDValue Val)
Strip fabs/fneg/fcopysign from a value to get the underlying source.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition STLExtras.h:2224
static const MachineMemOperand::Flags MOCFSB1
Definition SIInstrInfo.h:75
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
Definition MathExtras.h:546
MemoryEffectsBase< IRMemLocation > MemoryEffects
Summary of how a function affects memory in the program.
Definition ModRef.h:356
static void reportUnsupported(MachineIRBuilder &MIRBuilder, const Twine &Msg)
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition bit.h:156
LLVM_ABI ConstantFPSDNode * isConstOrConstSplatFP(SDValue N, bool AllowUndefs=false)
Returns the SDNode if it is a constant splat BuildVector or constant float.
LLVM_ABI bool isGuaranteedNotToBeUndef(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Returns true if V cannot be undef, but may be poison.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
Definition MathExtras.h:380
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition bit.h:204
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
Definition MathExtras.h:274
constexpr T MinAlign(U A, V B)
A and B are either alignments or offsets.
Definition MathExtras.h:352
static const MachineMemOperand::Flags MONoClobber
Mark the MMO of a uniform load if there are no potentially clobbering stores on any path from the sta...
Definition SIInstrInfo.h:49
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition MathExtras.h:326
AtomicOrderingCABI
Atomic ordering for C11 / C++11's memory models.
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
Definition bit.h:263
bool isBoolSGPR(SDValue V)
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
constexpr uint32_t Hi_32(uint64_t Value)
Return the high 32 bits of a 64 bit value.
Definition MathExtras.h:151
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition Alignment.h:149
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
Definition MathExtras.h:190
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
constexpr uint32_t Lo_32(uint64_t Value)
Return the low 32 bits of a 64 bit value.
Definition MathExtras.h:156
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
static const MachineMemOperand::Flags MOCooperative
Mark the MMO of cooperative load/store atomics.
Definition SIInstrInfo.h:57
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
LLVM_ABI Value * buildAtomicRMWValue(AtomicRMWInst::BinOp Op, IRBuilderBase &Builder, Value *Loaded, Value *Val)
Emit IR to implement the given atomicrmw operation on values in registers, returning the new value.
AtomicOrdering
Atomic ordering for LLVM's memory model.
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
Definition MathExtras.h:389
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
Definition ModRef.h:74
@ AfterLegalizeDAG
Definition DAGCombine.h:19
@ AfterLegalizeVectorOps
Definition DAGCombine.h:18
@ AfterLegalizeTypes
Definition DAGCombine.h:17
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ Add
Sum of integers.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Definition MCRegister.h:21
@ Fast
Assign the register banks as fast as possible (default).
DWARFExpression::Operation Op
RoundingMode
Rounding mode.
@ NearestTiesToEven
roundTiesToEven.
unsigned M0(unsigned Val)
Definition VE.h:393
ArrayRef(const T &OneElt) -> ArrayRef< T >
LLVM_ABI ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
constexpr int64_t maxIntN(int64_t N)
Gets the maximum value for a N-bit signed integer.
Definition MathExtras.h:233
constexpr unsigned BitWidth
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
LLVM_ABI std::pair< Value *, Value * > buildCmpXchgValue(IRBuilderBase &Builder, Value *Ptr, Value *Cmp, Value *Val, Align Alignment, bool IsVolatile=false)
Emit IR to implement the given cmpxchg operation on values in registers, returning the new value.
LLVM_ABI std::optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...
Definition Utils.cpp:440
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1788
std::optional< StringRef > getAtomicScopeIRString(const Triple &T, AtomicScope S, bool IsSingleAddressSpace=false)
Returns the LLVM IR syncscope string that T uses to spell S.
Definition AtomicScope.h:34
LLVM_ABI bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
static const MachineMemOperand::Flags MOLastUse
Mark the MMO of a load as the last use.
Definition SIInstrInfo.h:53
static const MachineMemOperand::Flags MOCFSB0
Use two MMO flags to encode the 2-bit cache fill size field for GFX13+.
Definition SIInstrInfo.h:72
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1963
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition Alignment.h:206
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Next
Definition InstrProf.h:147
constexpr T maskTrailingOnes(unsigned N)
Create a bitmask with the N right-most bits set to 1, and all other bits set to 0.
Definition MathExtras.h:78
constexpr RegState getUndefRegState(bool B)
@ Custom
The result value requires a custom uniformity check.
Definition Uniformity.h:31
LLVM_ABI Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
#define N
int64_t DWordOffset
int64_t PermMask
std::tuple< const ArgDescriptor *, const TargetRegisterClass *, LLT > getPreloadedValue(PreloadedValue Value) const
static MCRegister getPrivateSegmentBufferReg(CallingConv::ID CC)
Returns the register a non-entry function with calling convention CC expects the private segment buff...
static const AMDGPUFunctionArgInfo FixedABIFunctionInfo
The argument layout of a buffer intrinsic.
std::optional< unsigned > getSizeArg() const
The access size of loads to LDS and prefetches.
std::optional< unsigned > getImmOffsetArg() const
The immediate offset of loads to LDS.
std::optional< unsigned > getAuxArg() const
The auxiliary (cache policy) argument.
std::optional< unsigned > getLDSPtrArg() const
The LDS destination of loads to LDS.
static constexpr uint64_t encode(Fields... Values)
static std::tuple< typename Fields::ValueType... > decode(uint64_t Encoded)
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
MCRegister getRegister() const
static ArgDescriptor createArg(const ArgDescriptor &Arg, unsigned Mask)
static ArgDescriptor createRegister(Register Reg, unsigned Mask=~0u)
Helper struct shared between Function Specialization and SCCP Solver.
Definition SCCPSolver.h:42
Represents the full denormal controls for a function, including the default mode and the f32 specific...
Represent subnormal handling kind for floating point instruction inputs and outputs.
@ Dynamic
Denormals have unknown treatment.
static constexpr DenormalMode getPreserveSign()
static constexpr DenormalMode getIEEE()
Extended Value Type.
Definition ValueTypes.h:35
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition ValueTypes.h:433
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition ValueTypes.h:160
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition ValueTypes.h:70
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
Definition ValueTypes.h:144
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition ValueTypes.h:338
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition ValueTypes.h:170
ElementCount getVectorElementCount() const
Definition ValueTypes.h:388
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition ValueTypes.h:411
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition ValueTypes.h:281
uint64_t getScalarSizeInBits() const
Definition ValueTypes.h:423
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
Definition ValueTypes.h:516
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
Definition ValueTypes.h:450
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition ValueTypes.h:354
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition ValueTypes.h:61
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
Definition ValueTypes.h:419
bool isVector() const
Return true if this is a vector value type.
Definition ValueTypes.h:191
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition ValueTypes.h:361
LLVM_ABI Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition ValueTypes.h:366
EVT changeElementType(LLVMContext &Context, EVT EltVT) const
Return a VT for a type whose attributes match ourselves with the exception of the element type that i...
Definition ValueTypes.h:136
bool isVectorOf(EVT EltVT) const
Return true if this is a vector with matching element type.
Definition ValueTypes.h:196
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition ValueTypes.h:180
LLVM_ABI const fltSemantics & getFltSemantics() const
Returns an APFloat semantics tag appropriate for the value type.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition ValueTypes.h:374
unsigned getPointerAddrSpace() const
unsigned getByValSize() const
Align getNonZeroMemAlign() const
InputArg - This struct carries flags and type information about a single incoming (formal) argument o...
MVT VT
Legalized type of this argument part.
unsigned getOrigArgIndex() const
OutputArg - This struct carries flags and a value for a single outgoing (actual) argument or outgoing...
static LLVM_ABI std::optional< bool > eq(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_EQ result.
bool isUnknown() const
Returns true if we don't know any bits.
Definition KnownBits.h:64
KnownBits trunc(unsigned BitWidth) const
Return known bits for a truncation of the value we're tracking.
Definition KnownBits.h:165
static KnownBits add(const KnownBits &LHS, const KnownBits &RHS, bool NSW=false, bool NUW=false, bool SelfAdd=false)
Compute knownbits resulting from addition of LHS and RHS.
Definition KnownBits.h:361
KnownBits extractBits(unsigned NumBits, unsigned BitPosition) const
Return a subset of the known bits from [bitPosition,bitPosition+numBits).
Definition KnownBits.h:239
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
Definition KnownBits.h:262
static LLVM_ABI std::optional< bool > ule(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_ULE result.
static LLVM_ABI std::optional< bool > uge(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_UGE result.
This class contains a discriminated union of information about pointers in memory operands,...
static LLVM_ABI MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID=0)
Stack pointer relative access.
MachinePointerInfo getWithOffset(int64_t O) const
static LLVM_ABI MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition Alignment.h:111
These are IR-level optimization flags that may be propagated to SDNodes.
bool hasNoUnsignedWrap() const
bool hasAllowContract() const
bool hasNoSignedWrap() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
unsigned int NumVTs
DenormalMode FP64FP16Denormals
If this is set, neither input or output denormals are flushed for both f64 and f16/v2f16 instructions...
DenormalMode FP32Denormals
If this is set, neither input or output denormals are flushed for most f32 instructions.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
std::optional< unsigned > fallbackAddressSpace
This structure contains all information that is necessary for lowering calls.
SmallVector< ISD::InputArg, 32 > Ins
SmallVector< ISD::OutputArg, 32 > Outs