/build/source/llvm/lib/Target/AMDGPU/SIISelLowering.cpp

Bug Summary

File:	build/source/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
Warning:	line 11962, column 52 Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

Show analyzer invocation

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name SIISelLowering.cpp -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/source/build-llvm/tools/clang/stage2-bins -resource-dir /usr/lib/llvm-17/lib/clang/17 -D _DEBUG -D _GLIBCXX_ASSERTIONS -D _GNU_SOURCE -D _LIBCPP_ENABLE_ASSERTIONS -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/Target/AMDGPU -I /build/source/llvm/lib/Target/AMDGPU -I include -I /build/source/llvm/include -D _FORTIFY_SOURCE=2 -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-17/lib/clang/17/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fmacro-prefix-map=/build/source/= -fcoverage-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fcoverage-prefix-map=/build/source/= -source-date-epoch 1679915782 -O2 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -Wno-misleading-indentation -std=c++17 -fdeprecated-macro -fdebug-compilation-dir=/build/source/build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/source/= -ferror-limit 19 -fvisibility=hidden -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2023-03-27-130437-16335-1 -x c++ /build/source/llvm/lib/Target/AMDGPU/SIISelLowering.cpp

1	//===-- SIISelLowering.cpp - SI DAG Lowering Implementation ---------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	/// \file
10	/// Custom DAG lowering for SI
11	//
12	//===----------------------------------------------------------------------===//
13
14	#include "SIISelLowering.h"
15	#include "AMDGPU.h"
16	#include "AMDGPUInstrInfo.h"
17	#include "AMDGPUTargetMachine.h"
18	#include "SIMachineFunctionInfo.h"
19	#include "SIRegisterInfo.h"
20	#include "llvm/ADT/FloatingPointMode.h"
21	#include "llvm/ADT/Statistic.h"
22	#include "llvm/Analysis/OptimizationRemarkEmitter.h"
23	#include "llvm/Analysis/UniformityAnalysis.h"
24	#include "llvm/BinaryFormat/ELF.h"
25	#include "llvm/CodeGen/Analysis.h"
26	#include "llvm/CodeGen/FunctionLoweringInfo.h"
27	#include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
28	#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
29	#include "llvm/CodeGen/MachineFrameInfo.h"
30	#include "llvm/CodeGen/MachineFunction.h"
31	#include "llvm/CodeGen/MachineLoopInfo.h"
32	#include "llvm/IR/DiagnosticInfo.h"
33	#include "llvm/IR/IRBuilder.h"
34	#include "llvm/IR/IntrinsicInst.h"
35	#include "llvm/IR/IntrinsicsAMDGPU.h"
36	#include "llvm/IR/IntrinsicsR600.h"
37	#include "llvm/Support/CommandLine.h"
38	#include "llvm/Support/KnownBits.h"
39	#include "llvm/Support/ModRef.h"
40
41	using namespace llvm;
42
43	#define DEBUG_TYPE"si-lower" "si-lower"
44
45	STATISTIC(NumTailCalls, "Number of tail calls")static llvm::Statistic NumTailCalls = {"si-lower", "NumTailCalls" , "Number of tail calls"};
46
47	static cl::opt<bool> DisableLoopAlignment(
48	"amdgpu-disable-loop-alignment",
49	cl::desc("Do not align and prefetch loops"),
50	cl::init(false));
51
52	static cl::opt<bool> UseDivergentRegisterIndexing(
53	"amdgpu-use-divergent-register-indexing",
54	cl::Hidden,
55	cl::desc("Use indirect register addressing for divergent indexes"),
56	cl::init(false));
57
58	static bool hasFP32Denormals(const MachineFunction &MF) {
59	const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
60	return Info->getMode().allFP32Denormals();
61	}
62
63	static bool hasFP64FP16Denormals(const MachineFunction &MF) {
64	const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
65	return Info->getMode().allFP64FP16Denormals();
66	}
67
68	static unsigned findFirstFreeSGPR(CCState &CCInfo) {
69	unsigned NumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs();
70	for (unsigned Reg = 0; Reg < NumSGPRs; ++Reg) {
71	if (!CCInfo.isAllocated(AMDGPU::SGPR0 + Reg)) {
72	return AMDGPU::SGPR0 + Reg;
73	}
74	}
75	llvm_unreachable("Cannot allocate sgpr")::llvm::llvm_unreachable_internal("Cannot allocate sgpr", "llvm/lib/Target/AMDGPU/SIISelLowering.cpp" , 75);
76	}
77
78	SITargetLowering::SITargetLowering(const TargetMachine &TM,
79	const GCNSubtarget &STI)
80	: AMDGPUTargetLowering(TM, STI),
81	Subtarget(&STI) {
82	addRegisterClass(MVT::i1, &AMDGPU::VReg_1RegClass);
83	addRegisterClass(MVT::i64, &AMDGPU::SReg_64RegClass);
84
85	addRegisterClass(MVT::i32, &AMDGPU::SReg_32RegClass);
86	addRegisterClass(MVT::f32, &AMDGPU::VGPR_32RegClass);
87
88	addRegisterClass(MVT::v2i32, &AMDGPU::SReg_64RegClass);
89
90	const SIRegisterInfo *TRI = STI.getRegisterInfo();
91	const TargetRegisterClass *V64RegClass = TRI->getVGPR64Class();
92
93	addRegisterClass(MVT::f64, V64RegClass);
94	addRegisterClass(MVT::v2f32, V64RegClass);
95
96	addRegisterClass(MVT::v3i32, &AMDGPU::SGPR_96RegClass);
97	addRegisterClass(MVT::v3f32, TRI->getVGPRClassForBitWidth(96));
98
99	addRegisterClass(MVT::v2i64, &AMDGPU::SGPR_128RegClass);
100	addRegisterClass(MVT::v2f64, &AMDGPU::SGPR_128RegClass);
101
102	addRegisterClass(MVT::v4i32, &AMDGPU::SGPR_128RegClass);
103	addRegisterClass(MVT::v4f32, TRI->getVGPRClassForBitWidth(128));
104
105	addRegisterClass(MVT::v5i32, &AMDGPU::SGPR_160RegClass);
106	addRegisterClass(MVT::v5f32, TRI->getVGPRClassForBitWidth(160));
107
108	addRegisterClass(MVT::v6i32, &AMDGPU::SGPR_192RegClass);
109	addRegisterClass(MVT::v6f32, TRI->getVGPRClassForBitWidth(192));
110
111	addRegisterClass(MVT::v3i64, &AMDGPU::SGPR_192RegClass);
112	addRegisterClass(MVT::v3f64, TRI->getVGPRClassForBitWidth(192));
113
114	addRegisterClass(MVT::v7i32, &AMDGPU::SGPR_224RegClass);
115	addRegisterClass(MVT::v7f32, TRI->getVGPRClassForBitWidth(224));
116
117	addRegisterClass(MVT::v8i32, &AMDGPU::SGPR_256RegClass);
118	addRegisterClass(MVT::v8f32, TRI->getVGPRClassForBitWidth(256));
119
120	addRegisterClass(MVT::v4i64, &AMDGPU::SGPR_256RegClass);
121	addRegisterClass(MVT::v4f64, TRI->getVGPRClassForBitWidth(256));
122
123	addRegisterClass(MVT::v9i32, &AMDGPU::SGPR_288RegClass);
124	addRegisterClass(MVT::v9f32, TRI->getVGPRClassForBitWidth(288));
125
126	addRegisterClass(MVT::v10i32, &AMDGPU::SGPR_320RegClass);
127	addRegisterClass(MVT::v10f32, TRI->getVGPRClassForBitWidth(320));
128
129	addRegisterClass(MVT::v11i32, &AMDGPU::SGPR_352RegClass);
130	addRegisterClass(MVT::v11f32, TRI->getVGPRClassForBitWidth(352));
131
132	addRegisterClass(MVT::v12i32, &AMDGPU::SGPR_384RegClass);
133	addRegisterClass(MVT::v12f32, TRI->getVGPRClassForBitWidth(384));
134
135	addRegisterClass(MVT::v16i32, &AMDGPU::SGPR_512RegClass);
136	addRegisterClass(MVT::v16f32, TRI->getVGPRClassForBitWidth(512));
137
138	addRegisterClass(MVT::v8i64, &AMDGPU::SGPR_512RegClass);
139	addRegisterClass(MVT::v8f64, TRI->getVGPRClassForBitWidth(512));
140
141	addRegisterClass(MVT::v16i64, &AMDGPU::SGPR_1024RegClass);
142	addRegisterClass(MVT::v16f64, TRI->getVGPRClassForBitWidth(1024));
143
144	if (Subtarget->has16BitInsts()) {
145	addRegisterClass(MVT::i16, &AMDGPU::SReg_32RegClass);
146	addRegisterClass(MVT::f16, &AMDGPU::SReg_32RegClass);
147
148	// Unless there are also VOP3P operations, not operations are really legal.
149	addRegisterClass(MVT::v2i16, &AMDGPU::SReg_32RegClass);
150	addRegisterClass(MVT::v2f16, &AMDGPU::SReg_32RegClass);
151	addRegisterClass(MVT::v4i16, &AMDGPU::SReg_64RegClass);
152	addRegisterClass(MVT::v4f16, &AMDGPU::SReg_64RegClass);
153	addRegisterClass(MVT::v8i16, &AMDGPU::SGPR_128RegClass);
154	addRegisterClass(MVT::v8f16, &AMDGPU::SGPR_128RegClass);
155	addRegisterClass(MVT::v16i16, &AMDGPU::SGPR_256RegClass);
156	addRegisterClass(MVT::v16f16, &AMDGPU::SGPR_256RegClass);
157	}
158
159	addRegisterClass(MVT::v32i32, &AMDGPU::VReg_1024RegClass);
160	addRegisterClass(MVT::v32f32, TRI->getVGPRClassForBitWidth(1024));
161
162	computeRegisterProperties(Subtarget->getRegisterInfo());
163
164	// The boolean content concept here is too inflexible. Compares only ever
165	// really produce a 1-bit result. Any copy/extend from these will turn into a
166	// select, and zext/1 or sext/-1 are equally cheap. Arbitrarily choose 0/1, as
167	// it's what most targets use.
168	setBooleanContents(ZeroOrOneBooleanContent);
169	setBooleanVectorContents(ZeroOrOneBooleanContent);
170
171	// We need to custom lower vector stores from local memory
172	setOperationAction(ISD::LOAD,
173	{MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
174	MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
175	MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32,
176	MVT::i1, MVT::v32i32},
177	Custom);
178
179	setOperationAction(ISD::STORE,
180	{MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
181	MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
182	MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32,
183	MVT::i1, MVT::v32i32},
184	Custom);
185
186	setTruncStoreAction(MVT::v2i32, MVT::v2i16, Expand);
187	setTruncStoreAction(MVT::v3i32, MVT::v3i16, Expand);
188	setTruncStoreAction(MVT::v4i32, MVT::v4i16, Expand);
189	setTruncStoreAction(MVT::v8i32, MVT::v8i16, Expand);
190	setTruncStoreAction(MVT::v16i32, MVT::v16i16, Expand);
191	setTruncStoreAction(MVT::v32i32, MVT::v32i16, Expand);
192	setTruncStoreAction(MVT::v2i32, MVT::v2i8, Expand);
193	setTruncStoreAction(MVT::v4i32, MVT::v4i8, Expand);
194	setTruncStoreAction(MVT::v8i32, MVT::v8i8, Expand);
195	setTruncStoreAction(MVT::v16i32, MVT::v16i8, Expand);
196	setTruncStoreAction(MVT::v32i32, MVT::v32i8, Expand);
197	setTruncStoreAction(MVT::v2i16, MVT::v2i8, Expand);
198	setTruncStoreAction(MVT::v4i16, MVT::v4i8, Expand);
199	setTruncStoreAction(MVT::v8i16, MVT::v8i8, Expand);
200	setTruncStoreAction(MVT::v16i16, MVT::v16i8, Expand);
201	setTruncStoreAction(MVT::v32i16, MVT::v32i8, Expand);
202
203	setTruncStoreAction(MVT::v3i64, MVT::v3i16, Expand);
204	setTruncStoreAction(MVT::v3i64, MVT::v3i32, Expand);
205	setTruncStoreAction(MVT::v4i64, MVT::v4i8, Expand);
206	setTruncStoreAction(MVT::v8i64, MVT::v8i8, Expand);
207	setTruncStoreAction(MVT::v8i64, MVT::v8i16, Expand);
208	setTruncStoreAction(MVT::v8i64, MVT::v8i32, Expand);
209	setTruncStoreAction(MVT::v16i64, MVT::v16i32, Expand);
210
211	setOperationAction(ISD::GlobalAddress, {MVT::i32, MVT::i64}, Custom);
212
213	setOperationAction(ISD::SELECT, MVT::i1, Promote);
214	setOperationAction(ISD::SELECT, MVT::i64, Custom);
215	setOperationAction(ISD::SELECT, MVT::f64, Promote);
216	AddPromotedToType(ISD::SELECT, MVT::f64, MVT::i64);
217
218	setOperationAction(ISD::SELECT_CC,
219	{MVT::f32, MVT::i32, MVT::i64, MVT::f64, MVT::i1}, Expand);
220
221	setOperationAction(ISD::SETCC, MVT::i1, Promote);
222	setOperationAction(ISD::SETCC, {MVT::v2i1, MVT::v4i1}, Expand);
223	AddPromotedToType(ISD::SETCC, MVT::i1, MVT::i32);
224
225	setOperationAction(ISD::TRUNCATE,
226	{MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
227	MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
228	MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32},
229	Expand);
230	setOperationAction(ISD::FP_ROUND,
231	{MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32,
232	MVT::v6f32, MVT::v7f32, MVT::v8f32, MVT::v9f32,
233	MVT::v10f32, MVT::v11f32, MVT::v12f32, MVT::v16f32},
234	Expand);
235
236	setOperationAction(ISD::SIGN_EXTEND_INREG,
237	{MVT::v2i1, MVT::v4i1, MVT::v2i8, MVT::v4i8, MVT::v2i16,
238	MVT::v3i16, MVT::v4i16, MVT::Other},
239	Custom);
240
241	setOperationAction(ISD::BRCOND, MVT::Other, Custom);
242	setOperationAction(ISD::BR_CC,
243	{MVT::i1, MVT::i32, MVT::i64, MVT::f32, MVT::f64}, Expand);
244
245	setOperationAction({ISD::UADDO, ISD::USUBO}, MVT::i32, Legal);
246
247	setOperationAction({ISD::ADDCARRY, ISD::SUBCARRY}, MVT::i32, Legal);
248
249	setOperationAction({ISD::SHL_PARTS, ISD::SRA_PARTS, ISD::SRL_PARTS}, MVT::i64,
250	Expand);
251
252	#if 0
253	setOperationAction({ISD::ADDCARRY, ISD::SUBCARRY}, MVT::i64, Legal);
254	#endif
255
256	// We only support LOAD/STORE and vector manipulation ops for vectors
257	// with > 4 elements.
258	for (MVT VT :
259	{MVT::v8i32, MVT::v8f32, MVT::v9i32, MVT::v9f32, MVT::v10i32,
260	MVT::v10f32, MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32,
261	MVT::v16i32, MVT::v16f32, MVT::v2i64, MVT::v2f64, MVT::v4i16,
262	MVT::v4f16, MVT::v3i64, MVT::v3f64, MVT::v6i32, MVT::v6f32,
263	MVT::v4i64, MVT::v4f64, MVT::v8i64, MVT::v8f64, MVT::v8i16,
264	MVT::v8f16, MVT::v16i16, MVT::v16f16, MVT::v16i64, MVT::v16f64,
265	MVT::v32i32, MVT::v32f32}) {
266	for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) {
267	switch (Op) {
268	case ISD::LOAD:
269	case ISD::STORE:
270	case ISD::BUILD_VECTOR:
271	case ISD::BITCAST:
272	case ISD::UNDEF:
273	case ISD::EXTRACT_VECTOR_ELT:
274	case ISD::INSERT_VECTOR_ELT:
275	case ISD::EXTRACT_SUBVECTOR:
276	case ISD::SCALAR_TO_VECTOR:
277	case ISD::IS_FPCLASS:
278	break;
279	case ISD::INSERT_SUBVECTOR:
280	case ISD::CONCAT_VECTORS:
281	setOperationAction(Op, VT, Custom);
282	break;
283	default:
284	setOperationAction(Op, VT, Expand);
285	break;
286	}
287	}
288	}
289
290	setOperationAction(ISD::FP_EXTEND, MVT::v4f32, Expand);
291
292	// TODO: For dynamic 64-bit vector inserts/extracts, should emit a pseudo that
293	// is expanded to avoid having two separate loops in case the index is a VGPR.
294
295	// Most operations are naturally 32-bit vector operations. We only support
296	// load and store of i64 vectors, so promote v2i64 vector operations to v4i32.
297	for (MVT Vec64 : { MVT::v2i64, MVT::v2f64 }) {
298	setOperationAction(ISD::BUILD_VECTOR, Vec64, Promote);
299	AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v4i32);
300
301	setOperationAction(ISD::EXTRACT_VECTOR_ELT, Vec64, Promote);
302	AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v4i32);
303
304	setOperationAction(ISD::INSERT_VECTOR_ELT, Vec64, Promote);
305	AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v4i32);
306
307	setOperationAction(ISD::SCALAR_TO_VECTOR, Vec64, Promote);
308	AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v4i32);
309	}
310
311	for (MVT Vec64 : { MVT::v3i64, MVT::v3f64 }) {
312	setOperationAction(ISD::BUILD_VECTOR, Vec64, Promote);
313	AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v6i32);
314
315	setOperationAction(ISD::EXTRACT_VECTOR_ELT, Vec64, Promote);
316	AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v6i32);
317
318	setOperationAction(ISD::INSERT_VECTOR_ELT, Vec64, Promote);
319	AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v6i32);
320
321	setOperationAction(ISD::SCALAR_TO_VECTOR, Vec64, Promote);
322	AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v6i32);
323	}
324
325	for (MVT Vec64 : { MVT::v4i64, MVT::v4f64 }) {
326	setOperationAction(ISD::BUILD_VECTOR, Vec64, Promote);
327	AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v8i32);
328
329	setOperationAction(ISD::EXTRACT_VECTOR_ELT, Vec64, Promote);
330	AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v8i32);
331
332	setOperationAction(ISD::INSERT_VECTOR_ELT, Vec64, Promote);
333	AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v8i32);
334
335	setOperationAction(ISD::SCALAR_TO_VECTOR, Vec64, Promote);
336	AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v8i32);
337	}
338
339	for (MVT Vec64 : { MVT::v8i64, MVT::v8f64 }) {
340	setOperationAction(ISD::BUILD_VECTOR, Vec64, Promote);
341	AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v16i32);
342
343	setOperationAction(ISD::EXTRACT_VECTOR_ELT, Vec64, Promote);
344	AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v16i32);
345
346	setOperationAction(ISD::INSERT_VECTOR_ELT, Vec64, Promote);
347	AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v16i32);
348
349	setOperationAction(ISD::SCALAR_TO_VECTOR, Vec64, Promote);
350	AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v16i32);
351	}
352
353	for (MVT Vec64 : { MVT::v16i64, MVT::v16f64 }) {
354	setOperationAction(ISD::BUILD_VECTOR, Vec64, Promote);
355	AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v32i32);
356
357	setOperationAction(ISD::EXTRACT_VECTOR_ELT, Vec64, Promote);
358	AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v32i32);
359
360	setOperationAction(ISD::INSERT_VECTOR_ELT, Vec64, Promote);
361	AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v32i32);
362
363	setOperationAction(ISD::SCALAR_TO_VECTOR, Vec64, Promote);
364	AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v32i32);
365	}
366
367	setOperationAction(ISD::VECTOR_SHUFFLE,
368	{MVT::v8i32, MVT::v8f32, MVT::v16i32, MVT::v16f32},
369	Expand);
370
371	setOperationAction(ISD::BUILD_VECTOR, {MVT::v4f16, MVT::v4i16}, Custom);
372
373	// Avoid stack access for these.
374	// TODO: Generalize to more vector types.
375	setOperationAction({ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT},
376	{MVT::v2i16, MVT::v2f16, MVT::v2i8, MVT::v4i8, MVT::v8i8,
377	MVT::v4i16, MVT::v4f16},
378	Custom);
379
380	// Deal with vec3 vector operations when widened to vec4.
381	setOperationAction(ISD::INSERT_SUBVECTOR,
382	{MVT::v3i32, MVT::v3f32, MVT::v4i32, MVT::v4f32}, Custom);
383
384	// Deal with vec5/6/7 vector operations when widened to vec8.
385	setOperationAction(ISD::INSERT_SUBVECTOR,
386	{MVT::v5i32, MVT::v5f32, MVT::v6i32, MVT::v6f32,
387	MVT::v7i32, MVT::v7f32, MVT::v8i32, MVT::v8f32,
388	MVT::v9i32, MVT::v9f32, MVT::v10i32, MVT::v10f32,
389	MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32},
390	Custom);
391
392	// BUFFER/FLAT_ATOMIC_CMP_SWAP on GCN GPUs needs input marshalling,
393	// and output demarshalling
394	setOperationAction(ISD::ATOMIC_CMP_SWAP, {MVT::i32, MVT::i64}, Custom);
395
396	// We can't return success/failure, only the old value,
397	// let LLVM add the comparison
398	setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, {MVT::i32, MVT::i64},
399	Expand);
400
401	setOperationAction(ISD::ADDRSPACECAST, {MVT::i32, MVT::i64}, Custom);
402
403	setOperationAction(ISD::BITREVERSE, {MVT::i32, MVT::i64}, Legal);
404
405	// FIXME: This should be narrowed to i32, but that only happens if i64 is
406	// illegal.
407	// FIXME: Should lower sub-i32 bswaps to bit-ops without v_perm_b32.
408	setOperationAction(ISD::BSWAP, {MVT::i64, MVT::i32}, Legal);
409
410	// On SI this is s_memtime and s_memrealtime on VI.
411	setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Legal);
412	setOperationAction({ISD::TRAP, ISD::DEBUGTRAP}, MVT::Other, Custom);
413
414	if (Subtarget->has16BitInsts()) {
415	setOperationAction({ISD::FPOW, ISD::FPOWI}, MVT::f16, Promote);
416	setOperationAction({ISD::FLOG, ISD::FEXP, ISD::FLOG10}, MVT::f16, Custom);
417	}
418
419	if (Subtarget->hasMadMacF32Insts())
420	setOperationAction(ISD::FMAD, MVT::f32, Legal);
421
422	if (!Subtarget->hasBFI())
423	// fcopysign can be done in a single instruction with BFI.
424	setOperationAction(ISD::FCOPYSIGN, {MVT::f32, MVT::f64}, Expand);
425
426	if (!Subtarget->hasBCNT(32))
427	setOperationAction(ISD::CTPOP, MVT::i32, Expand);
428
429	if (!Subtarget->hasBCNT(64))
430	setOperationAction(ISD::CTPOP, MVT::i64, Expand);
431
432	if (Subtarget->hasFFBH())
433	setOperationAction({ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF}, MVT::i32, Custom);
434
435	if (Subtarget->hasFFBL())
436	setOperationAction({ISD::CTTZ, ISD::CTTZ_ZERO_UNDEF}, MVT::i32, Custom);
437
438	// We only really have 32-bit BFE instructions (and 16-bit on VI).
439	//
440	// On SI+ there are 64-bit BFEs, but they are scalar only and there isn't any
441	// effort to match them now. We want this to be false for i64 cases when the
442	// extraction isn't restricted to the upper or lower half. Ideally we would
443	// have some pass reduce 64-bit extracts to 32-bit if possible. Extracts that
444	// span the midpoint are probably relatively rare, so don't worry about them
445	// for now.
446	if (Subtarget->hasBFE())
447	setHasExtractBitsInsn(true);
448
449	// Clamp modifier on add/sub
450	if (Subtarget->hasIntClamp())
451	setOperationAction({ISD::UADDSAT, ISD::USUBSAT}, MVT::i32, Legal);
452
453	if (Subtarget->hasAddNoCarry())
454	setOperationAction({ISD::SADDSAT, ISD::SSUBSAT}, {MVT::i16, MVT::i32},
455	Legal);
456
457	setOperationAction({ISD::FMINNUM, ISD::FMAXNUM}, {MVT::f32, MVT::f64},
458	Custom);
459
460	// These are really only legal for ieee_mode functions. We should be avoiding
461	// them for functions that don't have ieee_mode enabled, so just say they are
462	// legal.
463	setOperationAction({ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE},
464	{MVT::f32, MVT::f64}, Legal);
465
466	if (Subtarget->haveRoundOpsF64())
467	setOperationAction({ISD::FTRUNC, ISD::FCEIL, ISD::FRINT}, MVT::f64, Legal);
468	else
469	setOperationAction({ISD::FCEIL, ISD::FTRUNC, ISD::FRINT, ISD::FFLOOR},
470	MVT::f64, Custom);
471
472	setOperationAction(ISD::FFLOOR, MVT::f64, Legal);
473
474	setOperationAction({ISD::FSIN, ISD::FCOS, ISD::FDIV}, MVT::f32, Custom);
475	setOperationAction(ISD::FDIV, MVT::f64, Custom);
476
477	setOperationAction(ISD::BF16_TO_FP, {MVT::i16, MVT::f32, MVT::f64}, Expand);
478	setOperationAction(ISD::FP_TO_BF16, {MVT::i16, MVT::f32, MVT::f64}, Expand);
479
480	if (Subtarget->has16BitInsts()) {
481	setOperationAction({ISD::Constant, ISD::SMIN, ISD::SMAX, ISD::UMIN,
482	ISD::UMAX, ISD::UADDSAT, ISD::USUBSAT},
483	MVT::i16, Legal);
484
485	AddPromotedToType(ISD::SIGN_EXTEND, MVT::i16, MVT::i32);
486
487	setOperationAction({ISD::ROTR, ISD::ROTL, ISD::SELECT_CC, ISD::BR_CC},
488	MVT::i16, Expand);
489
490	setOperationAction({ISD::SIGN_EXTEND, ISD::SDIV, ISD::UDIV, ISD::SREM,
491	ISD::UREM, ISD::BITREVERSE, ISD::CTTZ,
492	ISD::CTTZ_ZERO_UNDEF, ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF,
493	ISD::CTPOP},
494	MVT::i16, Promote);
495
496	setOperationAction(ISD::LOAD, MVT::i16, Custom);
497
498	setTruncStoreAction(MVT::i64, MVT::i16, Expand);
499
500	setOperationAction(ISD::FP16_TO_FP, MVT::i16, Promote);
501	AddPromotedToType(ISD::FP16_TO_FP, MVT::i16, MVT::i32);
502	setOperationAction(ISD::FP_TO_FP16, MVT::i16, Promote);
503	AddPromotedToType(ISD::FP_TO_FP16, MVT::i16, MVT::i32);
504
505	setOperationAction({ISD::FP_TO_SINT, ISD::FP_TO_UINT}, MVT::i16, Custom);
506
507	// F16 - Constant Actions.
508	setOperationAction(ISD::ConstantFP, MVT::f16, Legal);
509
510	// F16 - Load/Store Actions.
511	setOperationAction(ISD::LOAD, MVT::f16, Promote);
512	AddPromotedToType(ISD::LOAD, MVT::f16, MVT::i16);
513	setOperationAction(ISD::STORE, MVT::f16, Promote);
514	AddPromotedToType(ISD::STORE, MVT::f16, MVT::i16);
515
516	// F16 - VOP1 Actions.
517	setOperationAction(
518	{ISD::FP_ROUND, ISD::FCOS, ISD::FSIN, ISD::FROUND, ISD::FPTRUNC_ROUND},
519	MVT::f16, Custom);
520
521	setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, MVT::i16, Custom);
522
523	setOperationAction(
524	{ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::SINT_TO_FP, ISD::UINT_TO_FP},
525	MVT::f16, Promote);
526
527	// F16 - VOP2 Actions.
528	setOperationAction({ISD::BR_CC, ISD::SELECT_CC}, MVT::f16, Expand);
529
530	setOperationAction(ISD::FDIV, MVT::f16, Custom);
531
532	// F16 - VOP3 Actions.
533	setOperationAction(ISD::FMA, MVT::f16, Legal);
534	if (STI.hasMadF16())
535	setOperationAction(ISD::FMAD, MVT::f16, Legal);
536
537	for (MVT VT : {MVT::v2i16, MVT::v2f16, MVT::v4i16, MVT::v4f16, MVT::v8i16,
538	MVT::v8f16, MVT::v16i16, MVT::v16f16}) {
539	for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) {
540	switch (Op) {
541	case ISD::LOAD:
542	case ISD::STORE:
543	case ISD::BUILD_VECTOR:
544	case ISD::BITCAST:
545	case ISD::UNDEF:
546	case ISD::EXTRACT_VECTOR_ELT:
547	case ISD::INSERT_VECTOR_ELT:
548	case ISD::INSERT_SUBVECTOR:
549	case ISD::EXTRACT_SUBVECTOR:
550	case ISD::SCALAR_TO_VECTOR:
551	case ISD::IS_FPCLASS:
552	break;
553	case ISD::CONCAT_VECTORS:
554	setOperationAction(Op, VT, Custom);
555	break;
556	default:
557	setOperationAction(Op, VT, Expand);
558	break;
559	}
560	}
561	}
562
563	// v_perm_b32 can handle either of these.
564	setOperationAction(ISD::BSWAP, {MVT::i16, MVT::v2i16}, Legal);
565	setOperationAction(ISD::BSWAP, MVT::v4i16, Custom);
566
567	// XXX - Do these do anything? Vector constants turn into build_vector.
568	setOperationAction(ISD::Constant, {MVT::v2i16, MVT::v2f16}, Legal);
569
570	setOperationAction(ISD::UNDEF, {MVT::v2i16, MVT::v2f16}, Legal);
571
572	setOperationAction(ISD::STORE, MVT::v2i16, Promote);
573	AddPromotedToType(ISD::STORE, MVT::v2i16, MVT::i32);
574	setOperationAction(ISD::STORE, MVT::v2f16, Promote);
575	AddPromotedToType(ISD::STORE, MVT::v2f16, MVT::i32);
576
577	setOperationAction(ISD::LOAD, MVT::v2i16, Promote);
578	AddPromotedToType(ISD::LOAD, MVT::v2i16, MVT::i32);
579	setOperationAction(ISD::LOAD, MVT::v2f16, Promote);
580	AddPromotedToType(ISD::LOAD, MVT::v2f16, MVT::i32);
581
582	setOperationAction(ISD::AND, MVT::v2i16, Promote);
583	AddPromotedToType(ISD::AND, MVT::v2i16, MVT::i32);
584	setOperationAction(ISD::OR, MVT::v2i16, Promote);
585	AddPromotedToType(ISD::OR, MVT::v2i16, MVT::i32);
586	setOperationAction(ISD::XOR, MVT::v2i16, Promote);
587	AddPromotedToType(ISD::XOR, MVT::v2i16, MVT::i32);
588
589	setOperationAction(ISD::LOAD, MVT::v4i16, Promote);
590	AddPromotedToType(ISD::LOAD, MVT::v4i16, MVT::v2i32);
591	setOperationAction(ISD::LOAD, MVT::v4f16, Promote);
592	AddPromotedToType(ISD::LOAD, MVT::v4f16, MVT::v2i32);
593
594	setOperationAction(ISD::STORE, MVT::v4i16, Promote);
595	AddPromotedToType(ISD::STORE, MVT::v4i16, MVT::v2i32);
596	setOperationAction(ISD::STORE, MVT::v4f16, Promote);
597	AddPromotedToType(ISD::STORE, MVT::v4f16, MVT::v2i32);
598
599	setOperationAction(ISD::LOAD, MVT::v8i16, Promote);
600	AddPromotedToType(ISD::LOAD, MVT::v8i16, MVT::v4i32);
601	setOperationAction(ISD::LOAD, MVT::v8f16, Promote);
602	AddPromotedToType(ISD::LOAD, MVT::v8f16, MVT::v4i32);
603
604	setOperationAction(ISD::STORE, MVT::v4i16, Promote);
605	AddPromotedToType(ISD::STORE, MVT::v4i16, MVT::v2i32);
606	setOperationAction(ISD::STORE, MVT::v4f16, Promote);
607	AddPromotedToType(ISD::STORE, MVT::v4f16, MVT::v2i32);
608
609	setOperationAction(ISD::STORE, MVT::v8i16, Promote);
610	AddPromotedToType(ISD::STORE, MVT::v8i16, MVT::v4i32);
611	setOperationAction(ISD::STORE, MVT::v8f16, Promote);
612	AddPromotedToType(ISD::STORE, MVT::v8f16, MVT::v4i32);
613
614	setOperationAction(ISD::LOAD, MVT::v16i16, Promote);
615	AddPromotedToType(ISD::LOAD, MVT::v16i16, MVT::v8i32);
616	setOperationAction(ISD::LOAD, MVT::v16f16, Promote);
617	AddPromotedToType(ISD::LOAD, MVT::v16f16, MVT::v8i32);
618
619	setOperationAction(ISD::STORE, MVT::v16i16, Promote);
620	AddPromotedToType(ISD::STORE, MVT::v16i16, MVT::v8i32);
621	setOperationAction(ISD::STORE, MVT::v16f16, Promote);
622	AddPromotedToType(ISD::STORE, MVT::v16f16, MVT::v8i32);
623
624	setOperationAction({ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
625	MVT::v2i32, Expand);
626	setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Expand);
627
628	setOperationAction({ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
629	MVT::v4i32, Expand);
630
631	setOperationAction({ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
632	MVT::v8i32, Expand);
633
634	if (!Subtarget->hasVOP3PInsts())
635	setOperationAction(ISD::BUILD_VECTOR, {MVT::v2i16, MVT::v2f16}, Custom);
636
637	setOperationAction(ISD::FNEG, MVT::v2f16, Legal);
638	// This isn't really legal, but this avoids the legalizer unrolling it (and
639	// allows matching fneg (fabs x) patterns)
640	setOperationAction(ISD::FABS, MVT::v2f16, Legal);
641
642	setOperationAction({ISD::FMAXNUM, ISD::FMINNUM}, MVT::f16, Custom);
643	setOperationAction({ISD::FMAXNUM_IEEE, ISD::FMINNUM_IEEE}, MVT::f16, Legal);
644
645	setOperationAction({ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE},
646	{MVT::v4f16, MVT::v8f16, MVT::v16f16}, Custom);
647
648	setOperationAction({ISD::FMINNUM, ISD::FMAXNUM},
649	{MVT::v4f16, MVT::v8f16, MVT::v16f16}, Expand);
650
651	for (MVT Vec16 : {MVT::v8i16, MVT::v8f16, MVT::v16i16, MVT::v16f16}) {
652	setOperationAction(
653	{ISD::BUILD_VECTOR, ISD::EXTRACT_VECTOR_ELT, ISD::SCALAR_TO_VECTOR},
654	Vec16, Custom);
655	setOperationAction(ISD::INSERT_VECTOR_ELT, Vec16, Expand);
656	}
657	}
658
659	if (Subtarget->hasVOP3PInsts()) {
660	setOperationAction({ISD::ADD, ISD::SUB, ISD::MUL, ISD::SHL, ISD::SRL,
661	ISD::SRA, ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX,
662	ISD::UADDSAT, ISD::USUBSAT, ISD::SADDSAT, ISD::SSUBSAT},
663	MVT::v2i16, Legal);
664
665	setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FMINNUM_IEEE,
666	ISD::FMAXNUM_IEEE, ISD::FCANONICALIZE},
667	MVT::v2f16, Legal);
668
669	setOperationAction(ISD::EXTRACT_VECTOR_ELT, {MVT::v2i16, MVT::v2f16},
670	Custom);
671
672	setOperationAction(ISD::VECTOR_SHUFFLE,
673	{MVT::v4f16, MVT::v4i16, MVT::v8f16, MVT::v8i16,
674	MVT::v16f16, MVT::v16i16},
675	Custom);
676
677	for (MVT VT : {MVT::v4i16, MVT::v8i16, MVT::v16i16})
678	// Split vector operations.
679	setOperationAction({ISD::SHL, ISD::SRA, ISD::SRL, ISD::ADD, ISD::SUB,
680	ISD::MUL, ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX,
681	ISD::UADDSAT, ISD::SADDSAT, ISD::USUBSAT,
682	ISD::SSUBSAT},
683	VT, Custom);
684
685	for (MVT VT : {MVT::v4f16, MVT::v8f16, MVT::v16f16})
686	// Split vector operations.
687	setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FCANONICALIZE},
688	VT, Custom);
689
690	setOperationAction({ISD::FMAXNUM, ISD::FMINNUM}, {MVT::v2f16, MVT::v4f16},
691	Custom);
692
693	setOperationAction(ISD::FEXP, MVT::v2f16, Custom);
694	setOperationAction(ISD::SELECT, {MVT::v4i16, MVT::v4f16}, Custom);
695
696	if (Subtarget->hasPackedFP32Ops()) {
697	setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG},
698	MVT::v2f32, Legal);
699	setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA},
700	{MVT::v4f32, MVT::v8f32, MVT::v16f32, MVT::v32f32},
701	Custom);
702	}
703	}
704
705	setOperationAction({ISD::FNEG, ISD::FABS}, MVT::v4f16, Custom);
706
707	if (Subtarget->has16BitInsts()) {
708	setOperationAction(ISD::SELECT, MVT::v2i16, Promote);
709	AddPromotedToType(ISD::SELECT, MVT::v2i16, MVT::i32);
710	setOperationAction(ISD::SELECT, MVT::v2f16, Promote);
711	AddPromotedToType(ISD::SELECT, MVT::v2f16, MVT::i32);
712	} else {
713	// Legalization hack.
714	setOperationAction(ISD::SELECT, {MVT::v2i16, MVT::v2f16}, Custom);
715
716	setOperationAction({ISD::FNEG, ISD::FABS}, MVT::v2f16, Custom);
717	}
718
719	setOperationAction(ISD::SELECT,
720	{MVT::v4i16, MVT::v4f16, MVT::v2i8, MVT::v4i8, MVT::v8i8,
721	MVT::v8i16, MVT::v8f16, MVT::v16i16, MVT::v16f16},
722	Custom);
723
724	setOperationAction({ISD::SMULO, ISD::UMULO}, MVT::i64, Custom);
725
726	if (Subtarget->hasMad64_32())
727	setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, MVT::i32, Custom);
728
729	setOperationAction(ISD::INTRINSIC_WO_CHAIN,
730	{MVT::Other, MVT::f32, MVT::v4f32, MVT::i16, MVT::f16,
731	MVT::v2i16, MVT::v2f16},
732	Custom);
733
734	setOperationAction(ISD::INTRINSIC_W_CHAIN,
735	{MVT::v2f16, MVT::v2i16, MVT::v3f16, MVT::v3i16,
736	MVT::v4f16, MVT::v4i16, MVT::v8f16, MVT::Other, MVT::f16,
737	MVT::i16, MVT::i8},
738	Custom);
739
740	setOperationAction(ISD::INTRINSIC_VOID,
741	{MVT::Other, MVT::v2i16, MVT::v2f16, MVT::v3i16,
742	MVT::v3f16, MVT::v4f16, MVT::v4i16, MVT::f16, MVT::i16,
743	MVT::i8},
744	Custom);
745
746	setTargetDAGCombine({ISD::ADD,
747	ISD::ADDCARRY,
748	ISD::SUB,
749	ISD::SUBCARRY,
750	ISD::FADD,
751	ISD::FSUB,
752	ISD::FMINNUM,
753	ISD::FMAXNUM,
754	ISD::FMINNUM_IEEE,
755	ISD::FMAXNUM_IEEE,
756	ISD::FMA,
757	ISD::SMIN,
758	ISD::SMAX,
759	ISD::UMIN,
760	ISD::UMAX,
761	ISD::SETCC,
762	ISD::AND,
763	ISD::OR,
764	ISD::XOR,
765	ISD::SINT_TO_FP,
766	ISD::UINT_TO_FP,
767	ISD::FCANONICALIZE,
768	ISD::SCALAR_TO_VECTOR,
769	ISD::ZERO_EXTEND,
770	ISD::SIGN_EXTEND_INREG,
771	ISD::EXTRACT_VECTOR_ELT,
772	ISD::INSERT_VECTOR_ELT,
773	ISD::FCOPYSIGN});
774
775	// All memory operations. Some folding on the pointer operand is done to help
776	// matching the constant offsets in the addressing modes.
777	setTargetDAGCombine({ISD::LOAD,
778	ISD::STORE,
779	ISD::ATOMIC_LOAD,
780	ISD::ATOMIC_STORE,
781	ISD::ATOMIC_CMP_SWAP,
782	ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
783	ISD::ATOMIC_SWAP,
784	ISD::ATOMIC_LOAD_ADD,
785	ISD::ATOMIC_LOAD_SUB,
786	ISD::ATOMIC_LOAD_AND,
787	ISD::ATOMIC_LOAD_OR,
788	ISD::ATOMIC_LOAD_XOR,
789	ISD::ATOMIC_LOAD_NAND,
790	ISD::ATOMIC_LOAD_MIN,
791	ISD::ATOMIC_LOAD_MAX,
792	ISD::ATOMIC_LOAD_UMIN,
793	ISD::ATOMIC_LOAD_UMAX,
794	ISD::ATOMIC_LOAD_FADD,
795	ISD::ATOMIC_LOAD_UINC_WRAP,
796	ISD::ATOMIC_LOAD_UDEC_WRAP,
797	ISD::INTRINSIC_VOID,
798	ISD::INTRINSIC_W_CHAIN});
799
800	// FIXME: In other contexts we pretend this is a per-function property.
801	setStackPointerRegisterToSaveRestore(AMDGPU::SGPR32);
802
803	setSchedulingPreference(Sched::RegPressure);
804	}
805
806	const GCNSubtarget *SITargetLowering::getSubtarget() const {
807	return Subtarget;
808	}
809
810	//===----------------------------------------------------------------------===//
811	// TargetLowering queries
812	//===----------------------------------------------------------------------===//
813
814	// v_mad_mix* support a conversion from f16 to f32.
815	//
816	// There is only one special case when denormals are enabled we don't currently,
817	// where this is OK to use.
818	bool SITargetLowering::isFPExtFoldable(const SelectionDAG &DAG, unsigned Opcode,
819	EVT DestVT, EVT SrcVT) const {
820	return ((Opcode == ISD::FMAD && Subtarget->hasMadMixInsts()) \|\|
821	(Opcode == ISD::FMA && Subtarget->hasFmaMixInsts())) &&
822	DestVT.getScalarType() == MVT::f32 &&
823	SrcVT.getScalarType() == MVT::f16 &&
824	// TODO: This probably only requires no input flushing?
825	!hasFP32Denormals(DAG.getMachineFunction());
826	}
827
828	bool SITargetLowering::isFPExtFoldable(const MachineInstr &MI, unsigned Opcode,
829	LLT DestTy, LLT SrcTy) const {
830	return ((Opcode == TargetOpcode::G_FMAD && Subtarget->hasMadMixInsts()) \|\|
831	(Opcode == TargetOpcode::G_FMA && Subtarget->hasFmaMixInsts())) &&
832	DestTy.getScalarSizeInBits() == 32 &&
833	SrcTy.getScalarSizeInBits() == 16 &&
834	// TODO: This probably only requires no input flushing?
835	!hasFP32Denormals(*MI.getMF());
836	}
837
838	bool SITargetLowering::isShuffleMaskLegal(ArrayRef<int>, EVT) const {
839	// SI has some legal vector types, but no legal vector operations. Say no
840	// shuffles are legal in order to prefer scalarizing some vector operations.
841	return false;
842	}
843
844	MVT SITargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
845	CallingConv::ID CC,
846	EVT VT) const {
847	if (CC == CallingConv::AMDGPU_KERNEL)
848	return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
849
850	if (VT.isVector()) {
851	EVT ScalarVT = VT.getScalarType();
852	unsigned Size = ScalarVT.getSizeInBits();
853	if (Size == 16) {
854	if (Subtarget->has16BitInsts()) {
855	if (VT.isInteger())
856	return MVT::v2i16;
857	return (ScalarVT == MVT::bf16 ? MVT::i32 : MVT::v2f16);
858	}
859	return VT.isInteger() ? MVT::i32 : MVT::f32;
860	}
861
862	if (Size < 16)
863	return Subtarget->has16BitInsts() ? MVT::i16 : MVT::i32;
864	return Size == 32 ? ScalarVT.getSimpleVT() : MVT::i32;
865	}
866
867	if (VT.getSizeInBits() > 32)
868	return MVT::i32;
869
870	return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
871	}
872
873	unsigned SITargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
874	CallingConv::ID CC,
875	EVT VT) const {
876	if (CC == CallingConv::AMDGPU_KERNEL)
877	return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
878
879	if (VT.isVector()) {
880	unsigned NumElts = VT.getVectorNumElements();
881	EVT ScalarVT = VT.getScalarType();
882	unsigned Size = ScalarVT.getSizeInBits();
883
884	// FIXME: Should probably promote 8-bit vectors to i16.
885	if (Size == 16 && Subtarget->has16BitInsts())
886	return (NumElts + 1) / 2;
887
888	if (Size <= 32)
889	return NumElts;
890
891	if (Size > 32)
892	return NumElts * ((Size + 31) / 32);
893	} else if (VT.getSizeInBits() > 32)
894	return (VT.getSizeInBits() + 31) / 32;
895
896	return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
897	}
898
899	unsigned SITargetLowering::getVectorTypeBreakdownForCallingConv(
900	LLVMContext &Context, CallingConv::ID CC,
901	EVT VT, EVT &IntermediateVT,
902	unsigned &NumIntermediates, MVT &RegisterVT) const {
903	if (CC != CallingConv::AMDGPU_KERNEL && VT.isVector()) {
904	unsigned NumElts = VT.getVectorNumElements();
905	EVT ScalarVT = VT.getScalarType();
906	unsigned Size = ScalarVT.getSizeInBits();
907	// FIXME: We should fix the ABI to be the same on targets without 16-bit
908	// support, but unless we can properly handle 3-vectors, it will be still be
909	// inconsistent.
910	if (Size == 16 && Subtarget->has16BitInsts()) {
911	if (ScalarVT == MVT::bf16) {
912	RegisterVT = MVT::i32;
913	IntermediateVT = MVT::v2bf16;
914	} else {
915	RegisterVT = VT.isInteger() ? MVT::v2i16 : MVT::v2f16;
916	IntermediateVT = RegisterVT;
917	}
918	NumIntermediates = (NumElts + 1) / 2;
919	return NumIntermediates;
920	}
921
922	if (Size == 32) {
923	RegisterVT = ScalarVT.getSimpleVT();
924	IntermediateVT = RegisterVT;
925	NumIntermediates = NumElts;
926	return NumIntermediates;
927	}
928
929	if (Size < 16 && Subtarget->has16BitInsts()) {
930	// FIXME: Should probably form v2i16 pieces
931	RegisterVT = MVT::i16;
932	IntermediateVT = ScalarVT;
933	NumIntermediates = NumElts;
934	return NumIntermediates;
935	}
936
937
938	if (Size != 16 && Size <= 32) {
939	RegisterVT = MVT::i32;
940	IntermediateVT = ScalarVT;
941	NumIntermediates = NumElts;
942	return NumIntermediates;
943	}
944
945	if (Size > 32) {
946	RegisterVT = MVT::i32;
947	IntermediateVT = RegisterVT;
948	NumIntermediates = NumElts * ((Size + 31) / 32);
949	return NumIntermediates;
950	}
951	}
952
953	return TargetLowering::getVectorTypeBreakdownForCallingConv(
954	Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
955	}
956
957	static EVT memVTFromLoadIntrData(Type *Ty, unsigned MaxNumLanes) {
958	assert(MaxNumLanes != 0)(static_cast <bool> (MaxNumLanes != 0) ? void (0) : __assert_fail ("MaxNumLanes != 0", "llvm/lib/Target/AMDGPU/SIISelLowering.cpp" , 958, __extension__ __PRETTY_FUNCTION__));
959
960	if (auto *VT = dyn_cast<FixedVectorType>(Ty)) {
961	unsigned NumElts = std::min(MaxNumLanes, VT->getNumElements());
962	return EVT::getVectorVT(Ty->getContext(),
963	EVT::getEVT(VT->getElementType()),
964	NumElts);
965	}
966
967	return EVT::getEVT(Ty);
968	}
969
970	// Peek through TFE struct returns to only use the data size.
971	static EVT memVTFromLoadIntrReturn(Type *Ty, unsigned MaxNumLanes) {
972	auto *ST = dyn_cast<StructType>(Ty);
973	if (!ST)
974	return memVTFromLoadIntrData(Ty, MaxNumLanes);
975
976	// TFE intrinsics return an aggregate type.
977	assert(ST->getNumContainedTypes() == 2 &&(static_cast <bool> (ST->getNumContainedTypes() == 2 && ST->getContainedType(1)->isIntegerTy(32)) ? void (0) : __assert_fail ("ST->getNumContainedTypes() == 2 && ST->getContainedType(1)->isIntegerTy(32)" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 978, __extension__ __PRETTY_FUNCTION__))
978	ST->getContainedType(1)->isIntegerTy(32))(static_cast <bool> (ST->getNumContainedTypes() == 2 && ST->getContainedType(1)->isIntegerTy(32)) ? void (0) : __assert_fail ("ST->getNumContainedTypes() == 2 && ST->getContainedType(1)->isIntegerTy(32)" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 978, __extension__ __PRETTY_FUNCTION__));
979	return memVTFromLoadIntrData(ST->getContainedType(0), MaxNumLanes);
980	}
981
982	bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
983	const CallInst &CI,
984	MachineFunction &MF,
985	unsigned IntrID) const {
986	Info.flags = MachineMemOperand::MONone;
987	if (CI.hasMetadata(LLVMContext::MD_invariant_load))
988	Info.flags \|= MachineMemOperand::MOInvariant;
989
990	if (const AMDGPU::RsrcIntrinsic *RsrcIntr =
991	AMDGPU::lookupRsrcIntrinsic(IntrID)) {
992	AttributeList Attr = Intrinsic::getAttributes(CI.getContext(),
993	(Intrinsic::ID)IntrID);
994	MemoryEffects ME = Attr.getMemoryEffects();
995	if (ME.doesNotAccessMemory())
996	return false;
997
998	// TODO: Should images get their own address space?
999	Info.fallbackAddressSpace = AMDGPUAS::BUFFER_FAT_POINTER;
1000
1001	if (RsrcIntr->IsImage)
1002	Info.align.reset();
1003
1004	Info.flags \|= MachineMemOperand::MODereferenceable;
1005	if (ME.onlyReadsMemory()) {
1006	unsigned MaxNumLanes = 4;
1007
1008	if (RsrcIntr->IsImage) {
1009	const AMDGPU::ImageDimIntrinsicInfo *Intr
1010	= AMDGPU::getImageDimIntrinsicInfo(IntrID);
1011	const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
1012	AMDGPU::getMIMGBaseOpcodeInfo(Intr->BaseOpcode);
1013
1014	if (!BaseOpcode->Gather4) {
1015	// If this isn't a gather, we may have excess loaded elements in the
1016	// IR type. Check the dmask for the real number of elements loaded.
1017	unsigned DMask
1018	= cast<ConstantInt>(CI.getArgOperand(0))->getZExtValue();
1019	MaxNumLanes = DMask == 0 ? 1 : llvm::popcount(DMask);
1020	}
1021	}
1022
1023	Info.memVT = memVTFromLoadIntrReturn(CI.getType(), MaxNumLanes);
1024
1025	// FIXME: What does alignment mean for an image?
1026	Info.opc = ISD::INTRINSIC_W_CHAIN;
1027	Info.flags \|= MachineMemOperand::MOLoad;
1028	} else if (ME.onlyWritesMemory()) {
1029	Info.opc = ISD::INTRINSIC_VOID;
1030
1031	Type *DataTy = CI.getArgOperand(0)->getType();
1032	if (RsrcIntr->IsImage) {
1033	unsigned DMask = cast<ConstantInt>(CI.getArgOperand(1))->getZExtValue();
1034	unsigned DMaskLanes = DMask == 0 ? 1 : llvm::popcount(DMask);
1035	Info.memVT = memVTFromLoadIntrData(DataTy, DMaskLanes);
1036	} else
1037	Info.memVT = EVT::getEVT(DataTy);
1038
1039	Info.flags \|= MachineMemOperand::MOStore;
1040	} else {
1041	// Atomic
1042	Info.opc = CI.getType()->isVoidTy() ? ISD::INTRINSIC_VOID :
1043	ISD::INTRINSIC_W_CHAIN;
1044	Info.memVT = MVT::getVT(CI.getArgOperand(0)->getType());
1045	Info.flags \|= MachineMemOperand::MOLoad \|
1046	MachineMemOperand::MOStore \|
1047	MachineMemOperand::MODereferenceable;
1048
1049	// XXX - Should this be volatile without known ordering?
1050	Info.flags \|= MachineMemOperand::MOVolatile;
1051
1052	switch (IntrID) {
1053	default:
1054	break;
1055	case Intrinsic::amdgcn_raw_buffer_load_lds:
1056	case Intrinsic::amdgcn_struct_buffer_load_lds: {
1057	unsigned Width = cast<ConstantInt>(CI.getArgOperand(2))->getZExtValue();
1058	Info.memVT = EVT::getIntegerVT(CI.getContext(), Width * 8);
1059	return true;
1060	}
1061	}
1062	}
1063	return true;
1064	}
1065
1066	switch (IntrID) {
1067	case Intrinsic::amdgcn_atomic_inc:
1068	case Intrinsic::amdgcn_atomic_dec:
1069	case Intrinsic::amdgcn_ds_ordered_add:
1070	case Intrinsic::amdgcn_ds_ordered_swap:
1071	case Intrinsic::amdgcn_ds_fadd:
1072	case Intrinsic::amdgcn_ds_fmin:
1073	case Intrinsic::amdgcn_ds_fmax: {
1074	Info.opc = ISD::INTRINSIC_W_CHAIN;
1075	Info.memVT = MVT::getVT(CI.getType());
1076	Info.ptrVal = CI.getOperand(0);
1077	Info.align.reset();
1078	Info.flags \|= MachineMemOperand::MOLoad \| MachineMemOperand::MOStore;
1079
1080	const ConstantInt *Vol = cast<ConstantInt>(CI.getOperand(4));
1081	if (!Vol->isZero())
1082	Info.flags \|= MachineMemOperand::MOVolatile;
1083
1084	return true;
1085	}
1086	case Intrinsic::amdgcn_buffer_atomic_fadd: {
1087	Info.opc = ISD::INTRINSIC_W_CHAIN;
1088	Info.memVT = MVT::getVT(CI.getOperand(0)->getType());
1089	Info.fallbackAddressSpace = AMDGPUAS::BUFFER_FAT_POINTER;
1090	Info.align.reset();
1091	Info.flags \|= MachineMemOperand::MOLoad \| MachineMemOperand::MOStore;
1092
1093	const ConstantInt *Vol = dyn_cast<ConstantInt>(CI.getOperand(4));
1094	if (!Vol \|\| !Vol->isZero())
1095	Info.flags \|= MachineMemOperand::MOVolatile;
1096
1097	return true;
1098	}
1099	case Intrinsic::amdgcn_ds_append:
1100	case Intrinsic::amdgcn_ds_consume: {
1101	Info.opc = ISD::INTRINSIC_W_CHAIN;
1102	Info.memVT = MVT::getVT(CI.getType());
1103	Info.ptrVal = CI.getOperand(0);
1104	Info.align.reset();
1105	Info.flags \|= MachineMemOperand::MOLoad \| MachineMemOperand::MOStore;
1106
1107	const ConstantInt *Vol = cast<ConstantInt>(CI.getOperand(1));
1108	if (!Vol->isZero())
1109	Info.flags \|= MachineMemOperand::MOVolatile;
1110
1111	return true;
1112	}
1113	case Intrinsic::amdgcn_global_atomic_csub: {
1114	Info.opc = ISD::INTRINSIC_W_CHAIN;
1115	Info.memVT = MVT::getVT(CI.getType());
1116	Info.ptrVal = CI.getOperand(0);
1117	Info.align.reset();
1118	Info.flags \|= MachineMemOperand::MOLoad \|
1119	MachineMemOperand::MOStore \|
1120	MachineMemOperand::MOVolatile;
1121	return true;
1122	}
1123	case Intrinsic::amdgcn_image_bvh_intersect_ray: {
1124	Info.opc = ISD::INTRINSIC_W_CHAIN;
1125	Info.memVT = MVT::getVT(CI.getType()); // XXX: what is correct VT?
1126
1127	Info.fallbackAddressSpace = AMDGPUAS::BUFFER_FAT_POINTER;
1128	Info.align.reset();
1129	Info.flags \|= MachineMemOperand::MOLoad \|
1130	MachineMemOperand::MODereferenceable;
1131	return true;
1132	}
1133	case Intrinsic::amdgcn_global_atomic_fadd:
1134	case Intrinsic::amdgcn_global_atomic_fmin:
1135	case Intrinsic::amdgcn_global_atomic_fmax:
1136	case Intrinsic::amdgcn_flat_atomic_fadd:
1137	case Intrinsic::amdgcn_flat_atomic_fmin:
1138	case Intrinsic::amdgcn_flat_atomic_fmax:
1139	case Intrinsic::amdgcn_global_atomic_fadd_v2bf16:
1140	case Intrinsic::amdgcn_flat_atomic_fadd_v2bf16: {
1141	Info.opc = ISD::INTRINSIC_W_CHAIN;
1142	Info.memVT = MVT::getVT(CI.getType());
1143	Info.ptrVal = CI.getOperand(0);
1144	Info.align.reset();
1145	Info.flags \|= MachineMemOperand::MOLoad \|
1146	MachineMemOperand::MOStore \|
1147	MachineMemOperand::MODereferenceable \|
1148	MachineMemOperand::MOVolatile;
1149	return true;
1150	}
1151	case Intrinsic::amdgcn_ds_gws_init:
1152	case Intrinsic::amdgcn_ds_gws_barrier:
1153	case Intrinsic::amdgcn_ds_gws_sema_v:
1154	case Intrinsic::amdgcn_ds_gws_sema_br:
1155	case Intrinsic::amdgcn_ds_gws_sema_p:
1156	case Intrinsic::amdgcn_ds_gws_sema_release_all: {
1157	Info.opc = ISD::INTRINSIC_VOID;
1158
1159	const GCNTargetMachine &TM =
1160	static_cast<const GCNTargetMachine &>(getTargetMachine());
1161
1162	SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1163	Info.ptrVal = MFI->getGWSPSV(TM);
1164
1165	// This is an abstract access, but we need to specify a type and size.
1166	Info.memVT = MVT::i32;
1167	Info.size = 4;
1168	Info.align = Align(4);
1169
1170	if (IntrID == Intrinsic::amdgcn_ds_gws_barrier)
1171	Info.flags \|= MachineMemOperand::MOLoad;
1172	else
1173	Info.flags \|= MachineMemOperand::MOStore;
1174	return true;
1175	}
1176	case Intrinsic::amdgcn_global_load_lds: {
1177	Info.opc = ISD::INTRINSIC_VOID;
1178	unsigned Width = cast<ConstantInt>(CI.getArgOperand(2))->getZExtValue();
1179	Info.memVT = EVT::getIntegerVT(CI.getContext(), Width * 8);
1180	Info.flags \|= MachineMemOperand::MOLoad \| MachineMemOperand::MOStore \|
1181	MachineMemOperand::MOVolatile;
1182	return true;
1183	}
1184	case Intrinsic::amdgcn_ds_bvh_stack_rtn: {
1185	Info.opc = ISD::INTRINSIC_W_CHAIN;
1186
1187	const GCNTargetMachine &TM =
1188	static_cast<const GCNTargetMachine &>(getTargetMachine());
1189
1190	SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1191	Info.ptrVal = MFI->getGWSPSV(TM);
1192
1193	// This is an abstract access, but we need to specify a type and size.
1194	Info.memVT = MVT::i32;
1195	Info.size = 4;
1196	Info.align = Align(4);
1197
1198	Info.flags = MachineMemOperand::MOLoad \| MachineMemOperand::MOStore;
1199	return true;
1200	}
1201	default:
1202	return false;
1203	}
1204	}
1205
1206	bool SITargetLowering::getAddrModeArguments(IntrinsicInst *II,
1207	SmallVectorImpl<Value*> &Ops,
1208	Type *&AccessTy) const {
1209	switch (II->getIntrinsicID()) {
1210	case Intrinsic::amdgcn_atomic_inc:
1211	case Intrinsic::amdgcn_atomic_dec:
1212	case Intrinsic::amdgcn_ds_ordered_add:
1213	case Intrinsic::amdgcn_ds_ordered_swap:
1214	case Intrinsic::amdgcn_ds_append:
1215	case Intrinsic::amdgcn_ds_consume:
1216	case Intrinsic::amdgcn_ds_fadd:
1217	case Intrinsic::amdgcn_ds_fmin:
1218	case Intrinsic::amdgcn_ds_fmax:
1219	case Intrinsic::amdgcn_global_atomic_fadd:
1220	case Intrinsic::amdgcn_flat_atomic_fadd:
1221	case Intrinsic::amdgcn_flat_atomic_fmin:
1222	case Intrinsic::amdgcn_flat_atomic_fmax:
1223	case Intrinsic::amdgcn_global_atomic_fadd_v2bf16:
1224	case Intrinsic::amdgcn_flat_atomic_fadd_v2bf16:
1225	case Intrinsic::amdgcn_global_atomic_csub: {
1226	Value *Ptr = II->getArgOperand(0);
1227	AccessTy = II->getType();
1228	Ops.push_back(Ptr);
1229	return true;
1230	}
1231	default:
1232	return false;
1233	}
1234	}
1235
1236	bool SITargetLowering::isLegalFlatAddressingMode(const AddrMode &AM) const {
1237	if (!Subtarget->hasFlatInstOffsets()) {
1238	// Flat instructions do not have offsets, and only have the register
1239	// address.
1240	return AM.BaseOffs == 0 && AM.Scale == 0;
1241	}
1242
1243	return AM.Scale == 0 &&
1244	(AM.BaseOffs == 0 \|\|
1245	Subtarget->getInstrInfo()->isLegalFLATOffset(
1246	AM.BaseOffs, AMDGPUAS::FLAT_ADDRESS, SIInstrFlags::FLAT));
1247	}
1248
1249	bool SITargetLowering::isLegalGlobalAddressingMode(const AddrMode &AM) const {
1250	if (Subtarget->hasFlatGlobalInsts())
1251	return AM.Scale == 0 &&
1252	(AM.BaseOffs == 0 \|\| Subtarget->getInstrInfo()->isLegalFLATOffset(
1253	AM.BaseOffs, AMDGPUAS::GLOBAL_ADDRESS,
1254	SIInstrFlags::FlatGlobal));
1255
1256	if (!Subtarget->hasAddr64() \|\| Subtarget->useFlatForGlobal()) {
1257	// Assume the we will use FLAT for all global memory accesses
1258	// on VI.
1259	// FIXME: This assumption is currently wrong. On VI we still use
1260	// MUBUF instructions for the r + i addressing mode. As currently
1261	// implemented, the MUBUF instructions only work on buffer < 4GB.
1262	// It may be possible to support > 4GB buffers with MUBUF instructions,
1263	// by setting the stride value in the resource descriptor which would
1264	// increase the size limit to (stride * 4GB). However, this is risky,
1265	// because it has never been validated.
1266	return isLegalFlatAddressingMode(AM);
1267	}
1268
1269	return isLegalMUBUFAddressingMode(AM);
1270	}
1271
1272	bool SITargetLowering::isLegalMUBUFAddressingMode(const AddrMode &AM) const {
1273	// MUBUF / MTBUF instructions have a 12-bit unsigned byte offset, and
1274	// additionally can do r + r + i with addr64. 32-bit has more addressing
1275	// mode options. Depending on the resource constant, it can also do
1276	// (i64 r0) + (i32 r1) * (i14 i).
1277	//
1278	// Private arrays end up using a scratch buffer most of the time, so also
1279	// assume those use MUBUF instructions. Scratch loads / stores are currently
1280	// implemented as mubuf instructions with offen bit set, so slightly
1281	// different than the normal addr64.
1282	if (!SIInstrInfo::isLegalMUBUFImmOffset(AM.BaseOffs))
1283	return false;
1284
1285	// FIXME: Since we can split immediate into soffset and immediate offset,
1286	// would it make sense to allow any immediate?
1287
1288	switch (AM.Scale) {
1289	case 0: // r + i or just i, depending on HasBaseReg.
1290	return true;
1291	case 1:
1292	return true; // We have r + r or r + i.
1293	case 2:
1294	if (AM.HasBaseReg) {
1295	// Reject 2 * r + r.
1296	return false;
1297	}
1298
1299	// Allow 2 * r as r + r
1300	// Or 2 * r + i is allowed as r + r + i.
1301	return true;
1302	default: // Don't allow n * r
1303	return false;
1304	}
1305	}
1306
1307	bool SITargetLowering::isLegalAddressingMode(const DataLayout &DL,
1308	const AddrMode &AM, Type *Ty,
1309	unsigned AS, Instruction *I) const {
1310	// No global is ever allowed as a base.
1311	if (AM.BaseGV)
1312	return false;
1313
1314	if (AS == AMDGPUAS::GLOBAL_ADDRESS)
1315	return isLegalGlobalAddressingMode(AM);
1316
1317	if (AS == AMDGPUAS::CONSTANT_ADDRESS \|\|
1318	AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT \|\|
1319	AS == AMDGPUAS::BUFFER_FAT_POINTER) {
1320	// If the offset isn't a multiple of 4, it probably isn't going to be
1321	// correctly aligned.
1322	// FIXME: Can we get the real alignment here?
1323	if (AM.BaseOffs % 4 != 0)
1324	return isLegalMUBUFAddressingMode(AM);
1325
1326	// There are no SMRD extloads, so if we have to do a small type access we
1327	// will use a MUBUF load.
1328	// FIXME?: We also need to do this if unaligned, but we don't know the
1329	// alignment here.
1330	if (Ty->isSized() && DL.getTypeStoreSize(Ty) < 4)
1331	return isLegalGlobalAddressingMode(AM);
1332
1333	if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS) {
1334	// SMRD instructions have an 8-bit, dword offset on SI.
1335	if (!isUInt<8>(AM.BaseOffs / 4))
1336	return false;
1337	} else if (Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS) {
1338	// On CI+, this can also be a 32-bit literal constant offset. If it fits
1339	// in 8-bits, it can use a smaller encoding.
1340	if (!isUInt<32>(AM.BaseOffs / 4))
1341	return false;
1342	} else if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
1343	// On VI, these use the SMEM format and the offset is 20-bit in bytes.
1344	if (!isUInt<20>(AM.BaseOffs))
1345	return false;
1346	} else
1347	llvm_unreachable("unhandled generation")::llvm::llvm_unreachable_internal("unhandled generation", "llvm/lib/Target/AMDGPU/SIISelLowering.cpp" , 1347);
1348
1349	if (AM.Scale == 0) // r + i or just i, depending on HasBaseReg.
1350	return true;
1351
1352	if (AM.Scale == 1 && AM.HasBaseReg)
1353	return true;
1354
1355	return false;
1356
1357	} else if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
1358	return isLegalMUBUFAddressingMode(AM);
1359	} else if (AS == AMDGPUAS::LOCAL_ADDRESS \|\|
1360	AS == AMDGPUAS::REGION_ADDRESS) {
1361	// Basic, single offset DS instructions allow a 16-bit unsigned immediate
1362	// field.
1363	// XXX - If doing a 4-byte aligned 8-byte type access, we effectively have
1364	// an 8-bit dword offset but we don't know the alignment here.
1365	if (!isUInt<16>(AM.BaseOffs))
1366	return false;
1367
1368	if (AM.Scale == 0) // r + i or just i, depending on HasBaseReg.
1369	return true;
1370
1371	if (AM.Scale == 1 && AM.HasBaseReg)
1372	return true;
1373
1374	return false;
1375	} else if (AS == AMDGPUAS::FLAT_ADDRESS \|\|
1376	AS == AMDGPUAS::UNKNOWN_ADDRESS_SPACE) {
1377	// For an unknown address space, this usually means that this is for some
1378	// reason being used for pure arithmetic, and not based on some addressing
1379	// computation. We don't have instructions that compute pointers with any
1380	// addressing modes, so treat them as having no offset like flat
1381	// instructions.
1382	return isLegalFlatAddressingMode(AM);
1383	}
1384
1385	// Assume a user alias of global for unknown address spaces.
1386	return isLegalGlobalAddressingMode(AM);
1387	}
1388
1389	bool SITargetLowering::canMergeStoresTo(unsigned AS, EVT MemVT,
1390	const MachineFunction &MF) const {
1391	if (AS == AMDGPUAS::GLOBAL_ADDRESS \|\| AS == AMDGPUAS::FLAT_ADDRESS) {
1392	return (MemVT.getSizeInBits() <= 4 * 32);
1393	} else if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
1394	unsigned MaxPrivateBits = 8 * getSubtarget()->getMaxPrivateElementSize();
1395	return (MemVT.getSizeInBits() <= MaxPrivateBits);
1396	} else if (AS == AMDGPUAS::LOCAL_ADDRESS \|\| AS == AMDGPUAS::REGION_ADDRESS) {
1397	return (MemVT.getSizeInBits() <= 2 * 32);
1398	}
1399	return true;
1400	}
1401
1402	bool SITargetLowering::allowsMisalignedMemoryAccessesImpl(
1403	unsigned Size, unsigned AddrSpace, Align Alignment,
1404	MachineMemOperand::Flags Flags, unsigned *IsFast) const {
1405	if (IsFast)
1406	*IsFast = 0;
1407
1408	if (AddrSpace == AMDGPUAS::LOCAL_ADDRESS \|\|
1409	AddrSpace == AMDGPUAS::REGION_ADDRESS) {
1410	// Check if alignment requirements for ds_read/write instructions are
1411	// disabled.
1412	if (!Subtarget->hasUnalignedDSAccessEnabled() && Alignment < Align(4))
1413	return false;
1414
1415	Align RequiredAlignment(PowerOf2Ceil(Size/8)); // Natural alignment.
1416	if (Subtarget->hasLDSMisalignedBug() && Size > 32 &&
1417	Alignment < RequiredAlignment)
1418	return false;
1419
1420	// Either, the alignment requirements are "enabled", or there is an
1421	// unaligned LDS access related hardware bug though alignment requirements
1422	// are "disabled". In either case, we need to check for proper alignment
1423	// requirements.
1424	//
1425	switch (Size) {
1426	case 64:
1427	// SI has a hardware bug in the LDS / GDS bounds checking: if the base
1428	// address is negative, then the instruction is incorrectly treated as
1429	// out-of-bounds even if base + offsets is in bounds. Split vectorized
1430	// loads here to avoid emitting ds_read2_b32. We may re-combine the
1431	// load later in the SILoadStoreOptimizer.
1432	if (!Subtarget->hasUsableDSOffset() && Alignment < Align(8))
1433	return false;
1434
1435	// 8 byte accessing via ds_read/write_b64 require 8-byte alignment, but we
1436	// can do a 4 byte aligned, 8 byte access in a single operation using
1437	// ds_read2/write2_b32 with adjacent offsets.
1438	RequiredAlignment = Align(4);
1439
1440	if (Subtarget->hasUnalignedDSAccessEnabled()) {
1441	// We will either select ds_read_b64/ds_write_b64 or ds_read2_b32/
1442	// ds_write2_b32 depending on the alignment. In either case with either
1443	// alignment there is no faster way of doing this.
1444
1445	// The numbers returned here and below are not additive, it is a 'speed
1446	// rank'. They are just meant to be compared to decide if a certain way
1447	// of lowering an operation is faster than another. For that purpose
1448	// naturally aligned operation gets it bitsize to indicate that "it
1449	// operates with a speed comparable to N-bit wide load". With the full
1450	// alignment ds128 is slower than ds96 for example. If underaligned it
1451	// is comparable to a speed of a single dword access, which would then
1452	// mean 32 < 128 and it is faster to issue a wide load regardless.
1453	// 1 is simply "slow, don't do it". I.e. comparing an aligned load to a
1454	// wider load which will not be aligned anymore the latter is slower.
1455	if (IsFast)
1456	*IsFast = (Alignment >= RequiredAlignment) ? 64
1457	: (Alignment < Align(4)) ? 32
1458	: 1;
1459	return true;
1460	}
1461
1462	break;
1463	case 96:
1464	if (!Subtarget->hasDS96AndDS128())
1465	return false;
1466
1467	// 12 byte accessing via ds_read/write_b96 require 16-byte alignment on
1468	// gfx8 and older.
1469
1470	if (Subtarget->hasUnalignedDSAccessEnabled()) {
1471	// Naturally aligned access is fastest. However, also report it is Fast
1472	// if memory is aligned less than DWORD. A narrow load or store will be
1473	// be equally slow as a single ds_read_b96/ds_write_b96, but there will
1474	// be more of them, so overall we will pay less penalty issuing a single
1475	// instruction.
1476
1477	// See comment on the values above.
1478	if (IsFast)
1479	*IsFast = (Alignment >= RequiredAlignment) ? 96
1480	: (Alignment < Align(4)) ? 32
1481	: 1;
1482	return true;
1483	}
1484
1485	break;
1486	case 128:
1487	if (!Subtarget->hasDS96AndDS128() \|\| !Subtarget->useDS128())
1488	return false;
1489
1490	// 16 byte accessing via ds_read/write_b128 require 16-byte alignment on
1491	// gfx8 and older, but we can do a 8 byte aligned, 16 byte access in a
1492	// single operation using ds_read2/write2_b64.
1493	RequiredAlignment = Align(8);
1494
1495	if (Subtarget->hasUnalignedDSAccessEnabled()) {
1496	// Naturally aligned access is fastest. However, also report it is Fast
1497	// if memory is aligned less than DWORD. A narrow load or store will be
1498	// be equally slow as a single ds_read_b128/ds_write_b128, but there
1499	// will be more of them, so overall we will pay less penalty issuing a
1500	// single instruction.
1501
1502	// See comment on the values above.
1503	if (IsFast)
1504	*IsFast = (Alignment >= RequiredAlignment) ? 128
1505	: (Alignment < Align(4)) ? 32
1506	: 1;
1507	return true;
1508	}
1509
1510	break;
1511	default:
1512	if (Size > 32)
1513	return false;
1514
1515	break;
1516	}
1517
1518	// See comment on the values above.
1519	// Note that we have a single-dword or sub-dword here, so if underaligned
1520	// it is a slowest possible access, hence returned value is 0.
1521	if (IsFast)
1522	*IsFast = (Alignment >= RequiredAlignment) ? Size : 0;
1523
1524	return Alignment >= RequiredAlignment \|\|
1525	Subtarget->hasUnalignedDSAccessEnabled();
1526	}
1527
1528	if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS) {
1529	bool AlignedBy4 = Alignment >= Align(4);
1530	if (IsFast)
1531	*IsFast = AlignedBy4;
1532
1533	return AlignedBy4 \|\|
1534	Subtarget->enableFlatScratch() \|\|
1535	Subtarget->hasUnalignedScratchAccess();
1536	}
1537
1538	// FIXME: We have to be conservative here and assume that flat operations
1539	// will access scratch. If we had access to the IR function, then we
1540	// could determine if any private memory was used in the function.
1541	if (AddrSpace == AMDGPUAS::FLAT_ADDRESS &&
1542	!Subtarget->hasUnalignedScratchAccess()) {
1543	bool AlignedBy4 = Alignment >= Align(4);
1544	if (IsFast)
1545	*IsFast = AlignedBy4;
1546
1547	return AlignedBy4;
1548	}
1549
1550	// So long as they are correct, wide global memory operations perform better
1551	// than multiple smaller memory ops -- even when misaligned
1552	if (AMDGPU::isExtendedGlobalAddrSpace(AddrSpace)) {
1553	if (IsFast)
1554	*IsFast = Size;
1555
1556	return Alignment >= Align(4) \|\|
1557	Subtarget->hasUnalignedBufferAccessEnabled();
1558	}
1559
1560	// Smaller than dword value must be aligned.
1561	if (Size < 32)
1562	return false;
1563
1564	// 8.1.6 - For Dword or larger reads or writes, the two LSBs of the
1565	// byte-address are ignored, thus forcing Dword alignment.
1566	// This applies to private, global, and constant memory.
1567	if (IsFast)
1568	*IsFast = 1;
1569
1570	return Size >= 32 && Alignment >= Align(4);
1571	}
1572
1573	bool SITargetLowering::allowsMisalignedMemoryAccesses(
1574	EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
1575	unsigned *IsFast) const {
1576	return allowsMisalignedMemoryAccessesImpl(VT.getSizeInBits(), AddrSpace,
1577	Alignment, Flags, IsFast);
1578	}
1579
1580	EVT SITargetLowering::getOptimalMemOpType(
1581	const MemOp &Op, const AttributeList &FuncAttributes) const {
1582	// FIXME: Should account for address space here.
1583
1584	// The default fallback uses the private pointer size as a guess for a type to
1585	// use. Make sure we switch these to 64-bit accesses.
1586
1587	if (Op.size() >= 16 &&
1588	Op.isDstAligned(Align(4))) // XXX: Should only do for global
1589	return MVT::v4i32;
1590
1591	if (Op.size() >= 8 && Op.isDstAligned(Align(4)))
1592	return MVT::v2i32;
1593
1594	// Use the default.
1595	return MVT::Other;
1596	}
1597
1598	bool SITargetLowering::isMemOpHasNoClobberedMemOperand(const SDNode *N) const {
1599	const MemSDNode *MemNode = cast<MemSDNode>(N);
1600	return MemNode->getMemOperand()->getFlags() & MONoClobber;
1601	}
1602
1603	bool SITargetLowering::isNonGlobalAddrSpace(unsigned AS) {
1604	return AS == AMDGPUAS::LOCAL_ADDRESS \|\| AS == AMDGPUAS::REGION_ADDRESS \|\|
1605	AS == AMDGPUAS::PRIVATE_ADDRESS;
1606	}
1607
1608	bool SITargetLowering::isFreeAddrSpaceCast(unsigned SrcAS,
1609	unsigned DestAS) const {
1610	// Flat -> private/local is a simple truncate.
1611	// Flat -> global is no-op
1612	if (SrcAS == AMDGPUAS::FLAT_ADDRESS)
1613	return true;
1614
1615	const GCNTargetMachine &TM =
1616	static_cast<const GCNTargetMachine &>(getTargetMachine());
1617	return TM.isNoopAddrSpaceCast(SrcAS, DestAS);
1618	}
1619
1620	bool SITargetLowering::isMemOpUniform(const SDNode *N) const {
1621	const MemSDNode *MemNode = cast<MemSDNode>(N);
1622
1623	return AMDGPUInstrInfo::isUniformMMO(MemNode->getMemOperand());
1624	}
1625
1626	TargetLoweringBase::LegalizeTypeAction
1627	SITargetLowering::getPreferredVectorAction(MVT VT) const {
1628	if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
1629	VT.getScalarType().bitsLE(MVT::i16))
1630	return VT.isPow2VectorType() ? TypeSplitVector : TypeWidenVector;
1631	return TargetLoweringBase::getPreferredVectorAction(VT);
1632	}
1633
1634	bool SITargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
1635	Type *Ty) const {
1636	// FIXME: Could be smarter if called for vector constants.
1637	return true;
1638	}
1639
1640	bool SITargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
1641	unsigned Index) const {
1642	if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
1643	return false;
1644
1645	// TODO: Add more cases that are cheap.
1646	return Index == 0;
1647	}
1648
1649	bool SITargetLowering::isTypeDesirableForOp(unsigned Op, EVT VT) const {
1650	if (Subtarget->has16BitInsts() && VT == MVT::i16) {
1651	switch (Op) {
1652	case ISD::LOAD:
1653	case ISD::STORE:
1654
1655	// These operations are done with 32-bit instructions anyway.
1656	case ISD::AND:
1657	case ISD::OR:
1658	case ISD::XOR:
1659	case ISD::SELECT:
1660	// TODO: Extensions?
1661	return true;
1662	default:
1663	return false;
1664	}
1665	}
1666
1667	// SimplifySetCC uses this function to determine whether or not it should
1668	// create setcc with i1 operands. We don't have instructions for i1 setcc.
1669	if (VT == MVT::i1 && Op == ISD::SETCC)
1670	return false;
1671
1672	return TargetLowering::isTypeDesirableForOp(Op, VT);
1673	}
1674
1675	SDValue SITargetLowering::lowerKernArgParameterPtr(SelectionDAG &DAG,
1676	const SDLoc &SL,
1677	SDValue Chain,
1678	uint64_t Offset) const {
1679	const DataLayout &DL = DAG.getDataLayout();
1680	MachineFunction &MF = DAG.getMachineFunction();
1681	const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1682
1683	const ArgDescriptor *InputPtrReg;
1684	const TargetRegisterClass *RC;
1685	LLT ArgTy;
1686	MVT PtrVT = getPointerTy(DL, AMDGPUAS::CONSTANT_ADDRESS);
1687
1688	std::tie(InputPtrReg, RC, ArgTy) =
1689	Info->getPreloadedValue(AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
1690
1691	// We may not have the kernarg segment argument if we have no kernel
1692	// arguments.
1693	if (!InputPtrReg)
1694	return DAG.getConstant(0, SL, PtrVT);
1695
1696	MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
1697	SDValue BasePtr = DAG.getCopyFromReg(Chain, SL,
1698	MRI.getLiveInVirtReg(InputPtrReg->getRegister()), PtrVT);
1699
1700	return DAG.getObjectPtrOffset(SL, BasePtr, TypeSize::Fixed(Offset));
1701	}
1702
1703	SDValue SITargetLowering::getImplicitArgPtr(SelectionDAG &DAG,
1704	const SDLoc &SL) const {
1705	uint64_t Offset = getImplicitParameterOffset(DAG.getMachineFunction(),
1706	FIRST_IMPLICIT);
1707	return lowerKernArgParameterPtr(DAG, SL, DAG.getEntryNode(), Offset);
1708	}
1709
1710	SDValue SITargetLowering::getLDSKernelId(SelectionDAG &DAG,
1711	const SDLoc &SL) const {
1712
1713	Function &F = DAG.getMachineFunction().getFunction();
1714	std::optional<uint32_t> KnownSize =
1715	AMDGPUMachineFunction::getLDSKernelIdMetadata(F);
1716	if (KnownSize.has_value())
1717	return DAG.getConstant(*KnownSize, SL, MVT::i32);
1718	return SDValue();
1719	}
1720
1721	SDValue SITargetLowering::convertArgType(SelectionDAG &DAG, EVT VT, EVT MemVT,
1722	const SDLoc &SL, SDValue Val,
1723	bool Signed,
1724	const ISD::InputArg *Arg) const {
1725	// First, if it is a widened vector, narrow it.
1726	if (VT.isVector() &&
1727	VT.getVectorNumElements() != MemVT.getVectorNumElements()) {
1728	EVT NarrowedVT =
1729	EVT::getVectorVT(*DAG.getContext(), MemVT.getVectorElementType(),
1730	VT.getVectorNumElements());
1731	Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, NarrowedVT, Val,
1732	DAG.getConstant(0, SL, MVT::i32));
1733	}
1734
1735	// Then convert the vector elements or scalar value.
1736	if (Arg && (Arg->Flags.isSExt() \|\| Arg->Flags.isZExt()) &&
1737	VT.bitsLT(MemVT)) {
1738	unsigned Opc = Arg->Flags.isZExt() ? ISD::AssertZext : ISD::AssertSext;
1739	Val = DAG.getNode(Opc, SL, MemVT, Val, DAG.getValueType(VT));
1740	}
1741
1742	if (MemVT.isFloatingPoint())
1743	Val = getFPExtOrFPRound(DAG, Val, SL, VT);
1744	else if (Signed)
1745	Val = DAG.getSExtOrTrunc(Val, SL, VT);
1746	else
1747	Val = DAG.getZExtOrTrunc(Val, SL, VT);
1748
1749	return Val;
1750	}
1751
1752	SDValue SITargetLowering::lowerKernargMemParameter(
1753	SelectionDAG &DAG, EVT VT, EVT MemVT, const SDLoc &SL, SDValue Chain,
1754	uint64_t Offset, Align Alignment, bool Signed,
1755	const ISD::InputArg *Arg) const {
1756	MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
1757
1758	// Try to avoid using an extload by loading earlier than the argument address,
1759	// and extracting the relevant bits. The load should hopefully be merged with
1760	// the previous argument.
1761	if (MemVT.getStoreSize() < 4 && Alignment < 4) {
1762	// TODO: Handle align < 4 and size >= 4 (can happen with packed structs).
1763	int64_t AlignDownOffset = alignDown(Offset, 4);
1764	int64_t OffsetDiff = Offset - AlignDownOffset;
1765
1766	EVT IntVT = MemVT.changeTypeToInteger();
1767
1768	// TODO: If we passed in the base kernel offset we could have a better
1769	// alignment than 4, but we don't really need it.
1770	SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, AlignDownOffset);
1771	SDValue Load = DAG.getLoad(MVT::i32, SL, Chain, Ptr, PtrInfo, Align(4),
1772	MachineMemOperand::MODereferenceable \|
1773	MachineMemOperand::MOInvariant);
1774
1775	SDValue ShiftAmt = DAG.getConstant(OffsetDiff * 8, SL, MVT::i32);
1776	SDValue Extract = DAG.getNode(ISD::SRL, SL, MVT::i32, Load, ShiftAmt);
1777
1778	SDValue ArgVal = DAG.getNode(ISD::TRUNCATE, SL, IntVT, Extract);
1779	ArgVal = DAG.getNode(ISD::BITCAST, SL, MemVT, ArgVal);
1780	ArgVal = convertArgType(DAG, VT, MemVT, SL, ArgVal, Signed, Arg);
1781
1782
1783	return DAG.getMergeValues({ ArgVal, Load.getValue(1) }, SL);
1784	}
1785
1786	SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, Offset);
1787	SDValue Load = DAG.getLoad(MemVT, SL, Chain, Ptr, PtrInfo, Alignment,
1788	MachineMemOperand::MODereferenceable \|
1789	MachineMemOperand::MOInvariant);
1790
1791	SDValue Val = convertArgType(DAG, VT, MemVT, SL, Load, Signed, Arg);
1792	return DAG.getMergeValues({ Val, Load.getValue(1) }, SL);
1793	}
1794
1795	SDValue SITargetLowering::lowerStackParameter(SelectionDAG &DAG, CCValAssign &VA,
1796	const SDLoc &SL, SDValue Chain,
1797	const ISD::InputArg &Arg) const {
1798	MachineFunction &MF = DAG.getMachineFunction();
1799	MachineFrameInfo &MFI = MF.getFrameInfo();
1800
1801	if (Arg.Flags.isByVal()) {
1802	unsigned Size = Arg.Flags.getByValSize();
1803	int FrameIdx = MFI.CreateFixedObject(Size, VA.getLocMemOffset(), false);
1804	return DAG.getFrameIndex(FrameIdx, MVT::i32);
1805	}
1806
1807	unsigned ArgOffset = VA.getLocMemOffset();
1808	unsigned ArgSize = VA.getValVT().getStoreSize();
1809
1810	int FI = MFI.CreateFixedObject(ArgSize, ArgOffset, true);
1811
1812	// Create load nodes to retrieve arguments from the stack.
1813	SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
1814	SDValue ArgValue;
1815
1816	// For NON_EXTLOAD, generic code in getLoad assert(ValVT == MemVT)
1817	ISD::LoadExtType ExtType = ISD::NON_EXTLOAD;
1818	MVT MemVT = VA.getValVT();
1819
1820	switch (VA.getLocInfo()) {
1821	default:
1822	break;
1823	case CCValAssign::BCvt:
1824	MemVT = VA.getLocVT();
1825	break;
1826	case CCValAssign::SExt:
1827	ExtType = ISD::SEXTLOAD;
1828	break;
1829	case CCValAssign::ZExt:
1830	ExtType = ISD::ZEXTLOAD;
1831	break;
1832	case CCValAssign::AExt:
1833	ExtType = ISD::EXTLOAD;
1834	break;
1835	}
1836
1837	ArgValue = DAG.getExtLoad(
1838	ExtType, SL, VA.getLocVT(), Chain, FIN,
1839	MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI),
1840	MemVT);
1841	return ArgValue;
1842	}
1843
1844	SDValue SITargetLowering::getPreloadedValue(SelectionDAG &DAG,
1845	const SIMachineFunctionInfo &MFI,
1846	EVT VT,
1847	AMDGPUFunctionArgInfo::PreloadedValue PVID) const {
1848	const ArgDescriptor *Reg;
1849	const TargetRegisterClass *RC;
1850	LLT Ty;
1851
1852	std::tie(Reg, RC, Ty) = MFI.getPreloadedValue(PVID);
1853	if (!Reg) {
1854	if (PVID == AMDGPUFunctionArgInfo::PreloadedValue::KERNARG_SEGMENT_PTR) {
1855	// It's possible for a kernarg intrinsic call to appear in a kernel with
1856	// no allocated segment, in which case we do not add the user sgpr
1857	// argument, so just return null.
1858	return DAG.getConstant(0, SDLoc(), VT);
1859	}
1860
1861	// It's undefined behavior if a function marked with the amdgpu-no-*
1862	// attributes uses the corresponding intrinsic.
1863	return DAG.getUNDEF(VT);
1864	}
1865
1866	return loadInputValue(DAG, RC, VT, SDLoc(DAG.getEntryNode()), *Reg);
1867	}
1868
1869	static void processPSInputArgs(SmallVectorImpl<ISD::InputArg> &Splits,
1870	CallingConv::ID CallConv,
1871	ArrayRef<ISD::InputArg> Ins, BitVector &Skipped,
1872	FunctionType *FType,
1873	SIMachineFunctionInfo *Info) {
1874	for (unsigned I = 0, E = Ins.size(), PSInputNum = 0; I != E; ++I) {
1875	const ISD::InputArg *Arg = &Ins[I];
1876
1877	assert((!Arg->VT.isVector() \|\| Arg->VT.getScalarSizeInBits() == 16) &&(static_cast <bool> ((!Arg->VT.isVector() \|\| Arg-> VT.getScalarSizeInBits() == 16) && "vector type argument should have been split" ) ? void (0) : __assert_fail ("(!Arg->VT.isVector() \|\| Arg->VT.getScalarSizeInBits() == 16) && \"vector type argument should have been split\"" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 1878, __extension__ __PRETTY_FUNCTION__))
1878	"vector type argument should have been split")(static_cast <bool> ((!Arg->VT.isVector() \|\| Arg-> VT.getScalarSizeInBits() == 16) && "vector type argument should have been split" ) ? void (0) : __assert_fail ("(!Arg->VT.isVector() \|\| Arg->VT.getScalarSizeInBits() == 16) && \"vector type argument should have been split\"" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 1878, __extension__ __PRETTY_FUNCTION__));
1879
1880	// First check if it's a PS input addr.
1881	if (CallConv == CallingConv::AMDGPU_PS &&
1882	!Arg->Flags.isInReg() && PSInputNum <= 15) {
1883	bool SkipArg = !Arg->Used && !Info->isPSInputAllocated(PSInputNum);
1884
1885	// Inconveniently only the first part of the split is marked as isSplit,
1886	// so skip to the end. We only want to increment PSInputNum once for the
1887	// entire split argument.
1888	if (Arg->Flags.isSplit()) {
1889	while (!Arg->Flags.isSplitEnd()) {
1890	assert((!Arg->VT.isVector() \|\|(static_cast <bool> ((!Arg->VT.isVector() \|\| Arg-> VT.getScalarSizeInBits() == 16) && "unexpected vector split in ps argument type" ) ? void (0) : __assert_fail ("(!Arg->VT.isVector() \|\| Arg->VT.getScalarSizeInBits() == 16) && \"unexpected vector split in ps argument type\"" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 1892, __extension__ __PRETTY_FUNCTION__))
1891	Arg->VT.getScalarSizeInBits() == 16) &&(static_cast <bool> ((!Arg->VT.isVector() \|\| Arg-> VT.getScalarSizeInBits() == 16) && "unexpected vector split in ps argument type" ) ? void (0) : __assert_fail ("(!Arg->VT.isVector() \|\| Arg->VT.getScalarSizeInBits() == 16) && \"unexpected vector split in ps argument type\"" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 1892, __extension__ __PRETTY_FUNCTION__))
1892	"unexpected vector split in ps argument type")(static_cast <bool> ((!Arg->VT.isVector() \|\| Arg-> VT.getScalarSizeInBits() == 16) && "unexpected vector split in ps argument type" ) ? void (0) : __assert_fail ("(!Arg->VT.isVector() \|\| Arg->VT.getScalarSizeInBits() == 16) && \"unexpected vector split in ps argument type\"" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 1892, __extension__ __PRETTY_FUNCTION__));
1893	if (!SkipArg)
1894	Splits.push_back(*Arg);
1895	Arg = &Ins[++I];
1896	}
1897	}
1898
1899	if (SkipArg) {
1900	// We can safely skip PS inputs.
1901	Skipped.set(Arg->getOrigArgIndex());
1902	++PSInputNum;
1903	continue;
1904	}
1905
1906	Info->markPSInputAllocated(PSInputNum);
1907	if (Arg->Used)
1908	Info->markPSInputEnabled(PSInputNum);
1909
1910	++PSInputNum;
1911	}
1912
1913	Splits.push_back(*Arg);
1914	}
1915	}
1916
1917	// Allocate special inputs passed in VGPRs.
1918	void SITargetLowering::allocateSpecialEntryInputVGPRs(CCState &CCInfo,
1919	MachineFunction &MF,
1920	const SIRegisterInfo &TRI,
1921	SIMachineFunctionInfo &Info) const {
1922	const LLT S32 = LLT::scalar(32);
1923	MachineRegisterInfo &MRI = MF.getRegInfo();
1924
1925	if (Info.hasWorkItemIDX()) {
1926	Register Reg = AMDGPU::VGPR0;
1927	MRI.setType(MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass), S32);
1928
1929	CCInfo.AllocateReg(Reg);
1930	unsigned Mask = (Subtarget->hasPackedTID() &&
1931	Info.hasWorkItemIDY()) ? 0x3ff : ~0u;
1932	Info.setWorkItemIDX(ArgDescriptor::createRegister(Reg, Mask));
1933	}
1934
1935	if (Info.hasWorkItemIDY()) {
1936	assert(Info.hasWorkItemIDX())(static_cast <bool> (Info.hasWorkItemIDX()) ? void (0) : __assert_fail ("Info.hasWorkItemIDX()", "llvm/lib/Target/AMDGPU/SIISelLowering.cpp" , 1936, __extension__ __PRETTY_FUNCTION__));
1937	if (Subtarget->hasPackedTID()) {
1938	Info.setWorkItemIDY(ArgDescriptor::createRegister(AMDGPU::VGPR0,
1939	0x3ff << 10));
1940	} else {
1941	unsigned Reg = AMDGPU::VGPR1;
1942	MRI.setType(MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass), S32);
1943
1944	CCInfo.AllocateReg(Reg);
1945	Info.setWorkItemIDY(ArgDescriptor::createRegister(Reg));
1946	}
1947	}
1948
1949	if (Info.hasWorkItemIDZ()) {
1950	assert(Info.hasWorkItemIDX() && Info.hasWorkItemIDY())(static_cast <bool> (Info.hasWorkItemIDX() && Info .hasWorkItemIDY()) ? void (0) : __assert_fail ("Info.hasWorkItemIDX() && Info.hasWorkItemIDY()" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 1950, __extension__ __PRETTY_FUNCTION__));
1951	if (Subtarget->hasPackedTID()) {
1952	Info.setWorkItemIDZ(ArgDescriptor::createRegister(AMDGPU::VGPR0,
1953	0x3ff << 20));
1954	} else {
1955	unsigned Reg = AMDGPU::VGPR2;
1956	MRI.setType(MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass), S32);
1957
1958	CCInfo.AllocateReg(Reg);
1959	Info.setWorkItemIDZ(ArgDescriptor::createRegister(Reg));
1960	}
1961	}
1962	}
1963
1964	// Try to allocate a VGPR at the end of the argument list, or if no argument
1965	// VGPRs are left allocating a stack slot.
1966	// If \p Mask is is given it indicates bitfield position in the register.
1967	// If \p Arg is given use it with new ]p Mask instead of allocating new.
1968	static ArgDescriptor allocateVGPR32Input(CCState &CCInfo, unsigned Mask = ~0u,
1969	ArgDescriptor Arg = ArgDescriptor()) {
1970	if (Arg.isSet())
1971	return ArgDescriptor::createArg(Arg, Mask);
1972
1973	ArrayRef<MCPhysReg> ArgVGPRs = ArrayRef(AMDGPU::VGPR_32RegClass.begin(), 32);
1974	unsigned RegIdx = CCInfo.getFirstUnallocated(ArgVGPRs);
1975	if (RegIdx == ArgVGPRs.size()) {
1976	// Spill to stack required.
1977	int64_t Offset = CCInfo.AllocateStack(4, Align(4));
1978
1979	return ArgDescriptor::createStack(Offset, Mask);
1980	}
1981
1982	unsigned Reg = ArgVGPRs[RegIdx];
1983	Reg = CCInfo.AllocateReg(Reg);
1984	assert(Reg != AMDGPU::NoRegister)(static_cast <bool> (Reg != AMDGPU::NoRegister) ? void ( 0) : __assert_fail ("Reg != AMDGPU::NoRegister", "llvm/lib/Target/AMDGPU/SIISelLowering.cpp" , 1984, __extension__ __PRETTY_FUNCTION__));
1985
1986	MachineFunction &MF = CCInfo.getMachineFunction();
1987	Register LiveInVReg = MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass);
1988	MF.getRegInfo().setType(LiveInVReg, LLT::scalar(32));
1989	return ArgDescriptor::createRegister(Reg, Mask);
1990	}
1991
1992	static ArgDescriptor allocateSGPR32InputImpl(CCState &CCInfo,
1993	const TargetRegisterClass *RC,
1994	unsigned NumArgRegs) {
1995	ArrayRef<MCPhysReg> ArgSGPRs = ArrayRef(RC->begin(), 32);
1996	unsigned RegIdx = CCInfo.getFirstUnallocated(ArgSGPRs);
1997	if (RegIdx == ArgSGPRs.size())
1998	report_fatal_error("ran out of SGPRs for arguments");
1999
2000	unsigned Reg = ArgSGPRs[RegIdx];
2001	Reg = CCInfo.AllocateReg(Reg);
2002	assert(Reg != AMDGPU::NoRegister)(static_cast <bool> (Reg != AMDGPU::NoRegister) ? void ( 0) : __assert_fail ("Reg != AMDGPU::NoRegister", "llvm/lib/Target/AMDGPU/SIISelLowering.cpp" , 2002, __extension__ __PRETTY_FUNCTION__));
2003
2004	MachineFunction &MF = CCInfo.getMachineFunction();
2005	MF.addLiveIn(Reg, RC);
2006	return ArgDescriptor::createRegister(Reg);
2007	}
2008
2009	// If this has a fixed position, we still should allocate the register in the
2010	// CCInfo state. Technically we could get away with this for values passed
2011	// outside of the normal argument range.
2012	static void allocateFixedSGPRInputImpl(CCState &CCInfo,
2013	const TargetRegisterClass *RC,
2014	MCRegister Reg) {
2015	Reg = CCInfo.AllocateReg(Reg);
2016	assert(Reg != AMDGPU::NoRegister)(static_cast <bool> (Reg != AMDGPU::NoRegister) ? void ( 0) : __assert_fail ("Reg != AMDGPU::NoRegister", "llvm/lib/Target/AMDGPU/SIISelLowering.cpp" , 2016, __extension__ __PRETTY_FUNCTION__));
2017	MachineFunction &MF = CCInfo.getMachineFunction();
2018	MF.addLiveIn(Reg, RC);
2019	}
2020
2021	static void allocateSGPR32Input(CCState &CCInfo, ArgDescriptor &Arg) {
2022	if (Arg) {
2023	allocateFixedSGPRInputImpl(CCInfo, &AMDGPU::SGPR_32RegClass,
2024	Arg.getRegister());
2025	} else
2026	Arg = allocateSGPR32InputImpl(CCInfo, &AMDGPU::SGPR_32RegClass, 32);
2027	}
2028
2029	static void allocateSGPR64Input(CCState &CCInfo, ArgDescriptor &Arg) {
2030	if (Arg) {
2031	allocateFixedSGPRInputImpl(CCInfo, &AMDGPU::SGPR_64RegClass,
2032	Arg.getRegister());
2033	} else
2034	Arg = allocateSGPR32InputImpl(CCInfo, &AMDGPU::SGPR_64RegClass, 16);
2035	}
2036
2037	/// Allocate implicit function VGPR arguments at the end of allocated user
2038	/// arguments.
2039	void SITargetLowering::allocateSpecialInputVGPRs(
2040	CCState &CCInfo, MachineFunction &MF,
2041	const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const {
2042	const unsigned Mask = 0x3ff;
2043	ArgDescriptor Arg;
2044
2045	if (Info.hasWorkItemIDX()) {
2046	Arg = allocateVGPR32Input(CCInfo, Mask);
2047	Info.setWorkItemIDX(Arg);
2048	}
2049
2050	if (Info.hasWorkItemIDY()) {
2051	Arg = allocateVGPR32Input(CCInfo, Mask << 10, Arg);
2052	Info.setWorkItemIDY(Arg);
2053	}
2054
2055	if (Info.hasWorkItemIDZ())
2056	Info.setWorkItemIDZ(allocateVGPR32Input(CCInfo, Mask << 20, Arg));
2057	}
2058
2059	/// Allocate implicit function VGPR arguments in fixed registers.
2060	void SITargetLowering::allocateSpecialInputVGPRsFixed(
2061	CCState &CCInfo, MachineFunction &MF,
2062	const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const {
2063	Register Reg = CCInfo.AllocateReg(AMDGPU::VGPR31);
2064	if (!Reg)
2065	report_fatal_error("failed to allocated VGPR for implicit arguments");
2066
2067	const unsigned Mask = 0x3ff;
2068	Info.setWorkItemIDX(ArgDescriptor::createRegister(Reg, Mask));
2069	Info.setWorkItemIDY(ArgDescriptor::createRegister(Reg, Mask << 10));
2070	Info.setWorkItemIDZ(ArgDescriptor::createRegister(Reg, Mask << 20));
2071	}
2072
2073	void SITargetLowering::allocateSpecialInputSGPRs(
2074	CCState &CCInfo,
2075	MachineFunction &MF,
2076	const SIRegisterInfo &TRI,
2077	SIMachineFunctionInfo &Info) const {
2078	auto &ArgInfo = Info.getArgInfo();
2079
2080	// TODO: Unify handling with private memory pointers.
2081	if (Info.hasDispatchPtr())
2082	allocateSGPR64Input(CCInfo, ArgInfo.DispatchPtr);
2083
2084	const Module *M = MF.getFunction().getParent();
2085	if (Info.hasQueuePtr() &&
2086	AMDGPU::getCodeObjectVersion(*M) < AMDGPU::AMDHSA_COV5)
2087	allocateSGPR64Input(CCInfo, ArgInfo.QueuePtr);
2088
2089	// Implicit arg ptr takes the place of the kernarg segment pointer. This is a
2090	// constant offset from the kernarg segment.
2091	if (Info.hasImplicitArgPtr())
2092	allocateSGPR64Input(CCInfo, ArgInfo.ImplicitArgPtr);
2093
2094	if (Info.hasDispatchID())
2095	allocateSGPR64Input(CCInfo, ArgInfo.DispatchID);
2096
2097	// flat_scratch_init is not applicable for non-kernel functions.
2098
2099	if (Info.hasWorkGroupIDX())
2100	allocateSGPR32Input(CCInfo, ArgInfo.WorkGroupIDX);
2101
2102	if (Info.hasWorkGroupIDY())
2103	allocateSGPR32Input(CCInfo, ArgInfo.WorkGroupIDY);
2104
2105	if (Info.hasWorkGroupIDZ())
2106	allocateSGPR32Input(CCInfo, ArgInfo.WorkGroupIDZ);
2107
2108	if (Info.hasLDSKernelId())
2109	allocateSGPR32Input(CCInfo, ArgInfo.LDSKernelId);
2110	}
2111
2112	// Allocate special inputs passed in user SGPRs.
2113	void SITargetLowering::allocateHSAUserSGPRs(CCState &CCInfo,
2114	MachineFunction &MF,
2115	const SIRegisterInfo &TRI,
2116	SIMachineFunctionInfo &Info) const {
2117	if (Info.hasImplicitBufferPtr()) {
2118	Register ImplicitBufferPtrReg = Info.addImplicitBufferPtr(TRI);
2119	MF.addLiveIn(ImplicitBufferPtrReg, &AMDGPU::SGPR_64RegClass);
2120	CCInfo.AllocateReg(ImplicitBufferPtrReg);
2121	}
2122
2123	// FIXME: How should these inputs interact with inreg / custom SGPR inputs?
2124	if (Info.hasPrivateSegmentBuffer()) {
2125	Register PrivateSegmentBufferReg = Info.addPrivateSegmentBuffer(TRI);
2126	MF.addLiveIn(PrivateSegmentBufferReg, &AMDGPU::SGPR_128RegClass);
2127	CCInfo.AllocateReg(PrivateSegmentBufferReg);
2128	}
2129
2130	if (Info.hasDispatchPtr()) {
2131	Register DispatchPtrReg = Info.addDispatchPtr(TRI);
2132	MF.addLiveIn(DispatchPtrReg, &AMDGPU::SGPR_64RegClass);
2133	CCInfo.AllocateReg(DispatchPtrReg);
2134	}
2135
2136	const Module *M = MF.getFunction().getParent();
2137	if (Info.hasQueuePtr() &&
2138	AMDGPU::getCodeObjectVersion(*M) < AMDGPU::AMDHSA_COV5) {
2139	Register QueuePtrReg = Info.addQueuePtr(TRI);
2140	MF.addLiveIn(QueuePtrReg, &AMDGPU::SGPR_64RegClass);
2141	CCInfo.AllocateReg(QueuePtrReg);
2142	}
2143
2144	if (Info.hasKernargSegmentPtr()) {
2145	MachineRegisterInfo &MRI = MF.getRegInfo();
2146	Register InputPtrReg = Info.addKernargSegmentPtr(TRI);
2147	CCInfo.AllocateReg(InputPtrReg);
2148
2149	Register VReg = MF.addLiveIn(InputPtrReg, &AMDGPU::SGPR_64RegClass);
2150	MRI.setType(VReg, LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));
2151	}
2152
2153	if (Info.hasDispatchID()) {
2154	Register DispatchIDReg = Info.addDispatchID(TRI);
2155	MF.addLiveIn(DispatchIDReg, &AMDGPU::SGPR_64RegClass);
2156	CCInfo.AllocateReg(DispatchIDReg);
2157	}
2158
2159	if (Info.hasFlatScratchInit() && !getSubtarget()->isAmdPalOS()) {
2160	Register FlatScratchInitReg = Info.addFlatScratchInit(TRI);
2161	MF.addLiveIn(FlatScratchInitReg, &AMDGPU::SGPR_64RegClass);
2162	CCInfo.AllocateReg(FlatScratchInitReg);
2163	}
2164
2165	if (Info.hasLDSKernelId()) {
2166	Register Reg = Info.addLDSKernelId();
2167	MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
2168	CCInfo.AllocateReg(Reg);
2169	}
2170
2171	// TODO: Add GridWorkGroupCount user SGPRs when used. For now with HSA we read
2172	// these from the dispatch pointer.
2173	}
2174
2175	// Allocate special input registers that are initialized per-wave.
2176	void SITargetLowering::allocateSystemSGPRs(CCState &CCInfo,
2177	MachineFunction &MF,
2178	SIMachineFunctionInfo &Info,
2179	CallingConv::ID CallConv,
2180	bool IsShader) const {
2181	bool HasArchitectedSGPRs = Subtarget->hasArchitectedSGPRs();
2182	if (Subtarget->hasUserSGPRInit16Bug() && !IsShader) {
2183	// Note: user SGPRs are handled by the front-end for graphics shaders
2184	// Pad up the used user SGPRs with dead inputs.
2185
2186	// TODO: NumRequiredSystemSGPRs computation should be adjusted appropriately
2187	// before enabling architected SGPRs for workgroup IDs.
2188	assert(!HasArchitectedSGPRs && "Unhandled feature for the subtarget")(static_cast <bool> (!HasArchitectedSGPRs && "Unhandled feature for the subtarget" ) ? void (0) : __assert_fail ("!HasArchitectedSGPRs && \"Unhandled feature for the subtarget\"" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 2188, __extension__ __PRETTY_FUNCTION__));
2189
2190	unsigned CurrentUserSGPRs = Info.getNumUserSGPRs();
2191	// Note we do not count the PrivateSegmentWaveByteOffset. We do not want to
2192	// rely on it to reach 16 since if we end up having no stack usage, it will
2193	// not really be added.
2194	unsigned NumRequiredSystemSGPRs = Info.hasWorkGroupIDX() +
2195	Info.hasWorkGroupIDY() +
2196	Info.hasWorkGroupIDZ() +
2197	Info.hasWorkGroupInfo();
2198	for (unsigned i = NumRequiredSystemSGPRs + CurrentUserSGPRs; i < 16; ++i) {
2199	Register Reg = Info.addReservedUserSGPR();
2200	MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
2201	CCInfo.AllocateReg(Reg);
2202	}
2203	}
2204
2205	if (Info.hasWorkGroupIDX()) {
2206	Register Reg = Info.addWorkGroupIDX(HasArchitectedSGPRs);
2207	if (!HasArchitectedSGPRs)
2208	MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
2209
2210	CCInfo.AllocateReg(Reg);
2211	}
2212
2213	if (Info.hasWorkGroupIDY()) {
2214	Register Reg = Info.addWorkGroupIDY(HasArchitectedSGPRs);
2215	if (!HasArchitectedSGPRs)
2216	MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
2217
2218	CCInfo.AllocateReg(Reg);
2219	}
2220
2221	if (Info.hasWorkGroupIDZ()) {
2222	Register Reg = Info.addWorkGroupIDZ(HasArchitectedSGPRs);
2223	if (!HasArchitectedSGPRs)
2224	MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
2225
2226	CCInfo.AllocateReg(Reg);
2227	}
2228
2229	if (Info.hasWorkGroupInfo()) {
2230	Register Reg = Info.addWorkGroupInfo();
2231	MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
2232	CCInfo.AllocateReg(Reg);
2233	}
2234
2235	if (Info.hasPrivateSegmentWaveByteOffset()) {
2236	// Scratch wave offset passed in system SGPR.
2237	unsigned PrivateSegmentWaveByteOffsetReg;
2238
2239	if (IsShader) {
2240	PrivateSegmentWaveByteOffsetReg =
2241	Info.getPrivateSegmentWaveByteOffsetSystemSGPR();
2242
2243	// This is true if the scratch wave byte offset doesn't have a fixed
2244	// location.
2245	if (PrivateSegmentWaveByteOffsetReg == AMDGPU::NoRegister) {
2246	PrivateSegmentWaveByteOffsetReg = findFirstFreeSGPR(CCInfo);
2247	Info.setPrivateSegmentWaveByteOffset(PrivateSegmentWaveByteOffsetReg);
2248	}
2249	} else
2250	PrivateSegmentWaveByteOffsetReg = Info.addPrivateSegmentWaveByteOffset();
2251
2252	MF.addLiveIn(PrivateSegmentWaveByteOffsetReg, &AMDGPU::SGPR_32RegClass);
2253	CCInfo.AllocateReg(PrivateSegmentWaveByteOffsetReg);
2254	}
2255
2256	assert(!Subtarget->hasUserSGPRInit16Bug() \|\| IsShader \|\|(static_cast <bool> (!Subtarget->hasUserSGPRInit16Bug () \|\| IsShader \|\| Info.getNumPreloadedSGPRs() >= 16) ? void (0) : __assert_fail ("!Subtarget->hasUserSGPRInit16Bug() \|\| IsShader \|\| Info.getNumPreloadedSGPRs() >= 16" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 2257, __extension__ __PRETTY_FUNCTION__))
2257	Info.getNumPreloadedSGPRs() >= 16)(static_cast <bool> (!Subtarget->hasUserSGPRInit16Bug () \|\| IsShader \|\| Info.getNumPreloadedSGPRs() >= 16) ? void (0) : __assert_fail ("!Subtarget->hasUserSGPRInit16Bug() \|\| IsShader \|\| Info.getNumPreloadedSGPRs() >= 16" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 2257, __extension__ __PRETTY_FUNCTION__));
2258	}
2259
2260	static void reservePrivateMemoryRegs(const TargetMachine &TM,
2261	MachineFunction &MF,
2262	const SIRegisterInfo &TRI,
2263	SIMachineFunctionInfo &Info) {
2264	// Now that we've figured out where the scratch register inputs are, see if
2265	// should reserve the arguments and use them directly.
2266	MachineFrameInfo &MFI = MF.getFrameInfo();
2267	bool HasStackObjects = MFI.hasStackObjects();
2268	const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2269
2270	// Record that we know we have non-spill stack objects so we don't need to
2271	// check all stack objects later.
2272	if (HasStackObjects)
2273	Info.setHasNonSpillStackObjects(true);
2274
2275	// Everything live out of a block is spilled with fast regalloc, so it's
2276	// almost certain that spilling will be required.
2277	if (TM.getOptLevel() == CodeGenOpt::None)
2278	HasStackObjects = true;
2279
2280	// For now assume stack access is needed in any callee functions, so we need
2281	// the scratch registers to pass in.
2282	bool RequiresStackAccess = HasStackObjects \|\| MFI.hasCalls();
2283
2284	if (!ST.enableFlatScratch()) {
2285	if (RequiresStackAccess && ST.isAmdHsaOrMesa(MF.getFunction())) {
2286	// If we have stack objects, we unquestionably need the private buffer
2287	// resource. For the Code Object V2 ABI, this will be the first 4 user
2288	// SGPR inputs. We can reserve those and use them directly.
2289
2290	Register PrivateSegmentBufferReg =
2291	Info.getPreloadedReg(AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_BUFFER);
2292	Info.setScratchRSrcReg(PrivateSegmentBufferReg);
2293	} else {
2294	unsigned ReservedBufferReg = TRI.reservedPrivateSegmentBufferReg(MF);
2295	// We tentatively reserve the last registers (skipping the last registers
2296	// which may contain VCC, FLAT_SCR, and XNACK). After register allocation,
2297	// we'll replace these with the ones immediately after those which were
2298	// really allocated. In the prologue copies will be inserted from the
2299	// argument to these reserved registers.
2300
2301	// Without HSA, relocations are used for the scratch pointer and the
2302	// buffer resource setup is always inserted in the prologue. Scratch wave
2303	// offset is still in an input SGPR.
2304	Info.setScratchRSrcReg(ReservedBufferReg);
2305	}
2306	}
2307
2308	MachineRegisterInfo &MRI = MF.getRegInfo();
2309
2310	// For entry functions we have to set up the stack pointer if we use it,
2311	// whereas non-entry functions get this "for free". This means there is no
2312	// intrinsic advantage to using S32 over S34 in cases where we do not have
2313	// calls but do need a frame pointer (i.e. if we are requested to have one
2314	// because frame pointer elimination is disabled). To keep things simple we
2315	// only ever use S32 as the call ABI stack pointer, and so using it does not
2316	// imply we need a separate frame pointer.
2317	//
2318	// Try to use s32 as the SP, but move it if it would interfere with input
2319	// arguments. This won't work with calls though.
2320	//
2321	// FIXME: Move SP to avoid any possible inputs, or find a way to spill input
2322	// registers.
2323	if (!MRI.isLiveIn(AMDGPU::SGPR32)) {
2324	Info.setStackPtrOffsetReg(AMDGPU::SGPR32);
2325	} else {
2326	assert(AMDGPU::isShader(MF.getFunction().getCallingConv()))(static_cast <bool> (AMDGPU::isShader(MF.getFunction(). getCallingConv())) ? void (0) : __assert_fail ("AMDGPU::isShader(MF.getFunction().getCallingConv())" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 2326, __extension__ __PRETTY_FUNCTION__));
2327
2328	if (MFI.hasCalls())
2329	report_fatal_error("call in graphics shader with too many input SGPRs");
2330
2331	for (unsigned Reg : AMDGPU::SGPR_32RegClass) {
2332	if (!MRI.isLiveIn(Reg)) {
2333	Info.setStackPtrOffsetReg(Reg);
2334	break;
2335	}
2336	}
2337
2338	if (Info.getStackPtrOffsetReg() == AMDGPU::SP_REG)
2339	report_fatal_error("failed to find register for SP");
2340	}
2341
2342	// hasFP should be accurate for entry functions even before the frame is
2343	// finalized, because it does not rely on the known stack size, only
2344	// properties like whether variable sized objects are present.
2345	if (ST.getFrameLowering()->hasFP(MF)) {
2346	Info.setFrameOffsetReg(AMDGPU::SGPR33);
2347	}
2348	}
2349
2350	bool SITargetLowering::supportSplitCSR(MachineFunction *MF) const {
2351	const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
2352	return !Info->isEntryFunction();
2353	}
2354
2355	void SITargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const {
2356
2357	}
2358
2359	void SITargetLowering::insertCopiesSplitCSR(
2360	MachineBasicBlock *Entry,
2361	const SmallVectorImpl<MachineBasicBlock *> &Exits) const {
2362	const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
2363
2364	const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(Entry->getParent());
2365	if (!IStart)
2366	return;
2367
2368	const TargetInstrInfo *TII = Subtarget->getInstrInfo();
2369	MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo();
2370	MachineBasicBlock::iterator MBBI = Entry->begin();
2371	for (const MCPhysReg I = IStart; I; ++I) {
2372	const TargetRegisterClass *RC = nullptr;
2373	if (AMDGPU::SReg_64RegClass.contains(*I))
2374	RC = &AMDGPU::SGPR_64RegClass;
2375	else if (AMDGPU::SReg_32RegClass.contains(*I))
2376	RC = &AMDGPU::SGPR_32RegClass;
2377	else
2378	llvm_unreachable("Unexpected register class in CSRsViaCopy!")::llvm::llvm_unreachable_internal("Unexpected register class in CSRsViaCopy!" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 2378);
2379
2380	Register NewVR = MRI->createVirtualRegister(RC);
2381	// Create copy from CSR to a virtual register.
2382	Entry->addLiveIn(*I);
2383	BuildMI(*Entry, MBBI, DebugLoc(), TII->get(TargetOpcode::COPY), NewVR)
2384	.addReg(*I);
2385
2386	// Insert the copy-back instructions right before the terminator.
2387	for (auto *Exit : Exits)
2388	BuildMI(*Exit, Exit->getFirstTerminator(), DebugLoc(),
2389	TII->get(TargetOpcode::COPY), *I)
2390	.addReg(NewVR);
2391	}
2392	}
2393
2394	SDValue SITargetLowering::LowerFormalArguments(
2395	SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
2396	const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
2397	SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
2398	const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
2399
2400	MachineFunction &MF = DAG.getMachineFunction();
2401	const Function &Fn = MF.getFunction();
2402	FunctionType *FType = MF.getFunction().getFunctionType();
2403	SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
2404
2405	if (Subtarget->isAmdHsaOS() && AMDGPU::isGraphics(CallConv)) {
2406	DiagnosticInfoUnsupported NoGraphicsHSA(
2407	Fn, "unsupported non-compute shaders with HSA", DL.getDebugLoc());
2408	DAG.getContext()->diagnose(NoGraphicsHSA);
2409	return DAG.getEntryNode();
2410	}
2411
2412	Info->allocateKnownAddressLDSGlobal(Fn);
2413
2414	SmallVector<ISD::InputArg, 16> Splits;
2415	SmallVector<CCValAssign, 16> ArgLocs;
2416	BitVector Skipped(Ins.size());
2417	CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
2418	*DAG.getContext());
2419
2420	bool IsGraphics = AMDGPU::isGraphics(CallConv);
2421	bool IsKernel = AMDGPU::isKernel(CallConv);
2422	bool IsEntryFunc = AMDGPU::isEntryFunctionCC(CallConv);
2423
2424	if (IsGraphics) {
2425	assert(!Info->hasDispatchPtr() && !Info->hasKernargSegmentPtr() &&(static_cast <bool> (!Info->hasDispatchPtr() && !Info->hasKernargSegmentPtr() && !Info->hasWorkGroupInfo () && !Info->hasLDSKernelId() && !Info-> hasWorkItemIDX() && !Info->hasWorkItemIDY() && !Info->hasWorkItemIDZ()) ? void (0) : __assert_fail ("!Info->hasDispatchPtr() && !Info->hasKernargSegmentPtr() && !Info->hasWorkGroupInfo() && !Info->hasLDSKernelId() && !Info->hasWorkItemIDX() && !Info->hasWorkItemIDY() && !Info->hasWorkItemIDZ()" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 2428, __extension__ __PRETTY_FUNCTION__))
2426	!Info->hasWorkGroupInfo() && !Info->hasLDSKernelId() &&(static_cast <bool> (!Info->hasDispatchPtr() && !Info->hasKernargSegmentPtr() && !Info->hasWorkGroupInfo () && !Info->hasLDSKernelId() && !Info-> hasWorkItemIDX() && !Info->hasWorkItemIDY() && !Info->hasWorkItemIDZ()) ? void (0) : __assert_fail ("!Info->hasDispatchPtr() && !Info->hasKernargSegmentPtr() && !Info->hasWorkGroupInfo() && !Info->hasLDSKernelId() && !Info->hasWorkItemIDX() && !Info->hasWorkItemIDY() && !Info->hasWorkItemIDZ()" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 2428, __extension__ __PRETTY_FUNCTION__))
2427	!Info->hasWorkItemIDX() && !Info->hasWorkItemIDY() &&(static_cast <bool> (!Info->hasDispatchPtr() && !Info->hasKernargSegmentPtr() && !Info->hasWorkGroupInfo () && !Info->hasLDSKernelId() && !Info-> hasWorkItemIDX() && !Info->hasWorkItemIDY() && !Info->hasWorkItemIDZ()) ? void (0) : __assert_fail ("!Info->hasDispatchPtr() && !Info->hasKernargSegmentPtr() && !Info->hasWorkGroupInfo() && !Info->hasLDSKernelId() && !Info->hasWorkItemIDX() && !Info->hasWorkItemIDY() && !Info->hasWorkItemIDZ()" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 2428, __extension__ __PRETTY_FUNCTION__))
2428	!Info->hasWorkItemIDZ())(static_cast <bool> (!Info->hasDispatchPtr() && !Info->hasKernargSegmentPtr() && !Info->hasWorkGroupInfo () && !Info->hasLDSKernelId() && !Info-> hasWorkItemIDX() && !Info->hasWorkItemIDY() && !Info->hasWorkItemIDZ()) ? void (0) : __assert_fail ("!Info->hasDispatchPtr() && !Info->hasKernargSegmentPtr() && !Info->hasWorkGroupInfo() && !Info->hasLDSKernelId() && !Info->hasWorkItemIDX() && !Info->hasWorkItemIDY() && !Info->hasWorkItemIDZ()" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 2428, __extension__ __PRETTY_FUNCTION__));
2429	if (!Subtarget->enableFlatScratch())
2430	assert(!Info->hasFlatScratchInit())(static_cast <bool> (!Info->hasFlatScratchInit()) ? void (0) : __assert_fail ("!Info->hasFlatScratchInit()", "llvm/lib/Target/AMDGPU/SIISelLowering.cpp" , 2430, __extension__ __PRETTY_FUNCTION__));
2431	if (CallConv != CallingConv::AMDGPU_CS \|\| !Subtarget->hasArchitectedSGPRs())
2432	assert(!Info->hasWorkGroupIDX() && !Info->hasWorkGroupIDY() &&(static_cast <bool> (!Info->hasWorkGroupIDX() && !Info->hasWorkGroupIDY() && !Info->hasWorkGroupIDZ ()) ? void (0) : __assert_fail ("!Info->hasWorkGroupIDX() && !Info->hasWorkGroupIDY() && !Info->hasWorkGroupIDZ()" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 2433, __extension__ __PRETTY_FUNCTION__))
2433	!Info->hasWorkGroupIDZ())(static_cast <bool> (!Info->hasWorkGroupIDX() && !Info->hasWorkGroupIDY() && !Info->hasWorkGroupIDZ ()) ? void (0) : __assert_fail ("!Info->hasWorkGroupIDX() && !Info->hasWorkGroupIDY() && !Info->hasWorkGroupIDZ()" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 2433, __extension__ __PRETTY_FUNCTION__));
2434	}
2435
2436	if (CallConv == CallingConv::AMDGPU_PS) {
2437	processPSInputArgs(Splits, CallConv, Ins, Skipped, FType, Info);
2438
2439	// At least one interpolation mode must be enabled or else the GPU will
2440	// hang.
2441	//
2442	// Check PSInputAddr instead of PSInputEnable. The idea is that if the user
2443	// set PSInputAddr, the user wants to enable some bits after the compilation
2444	// based on run-time states. Since we can't know what the final PSInputEna
2445	// will look like, so we shouldn't do anything here and the user should take
2446	// responsibility for the correct programming.
2447	//
2448	// Otherwise, the following restrictions apply:
2449	// - At least one of PERSP_* (0xF) or LINEAR_* (0x70) must be enabled.
2450	// - If POS_W_FLOAT (11) is enabled, at least one of PERSP_* must be
2451	// enabled too.
2452	if ((Info->getPSInputAddr() & 0x7F) == 0 \|\|
2453	((Info->getPSInputAddr() & 0xF) == 0 && Info->isPSInputAllocated(11))) {
2454	CCInfo.AllocateReg(AMDGPU::VGPR0);
2455	CCInfo.AllocateReg(AMDGPU::VGPR1);
2456	Info->markPSInputAllocated(0);
2457	Info->markPSInputEnabled(0);
2458	}
2459	if (Subtarget->isAmdPalOS()) {
2460	// For isAmdPalOS, the user does not enable some bits after compilation
2461	// based on run-time states; the register values being generated here are
2462	// the final ones set in hardware. Therefore we need to apply the
2463	// workaround to PSInputAddr and PSInputEnable together. (The case where
2464	// a bit is set in PSInputAddr but not PSInputEnable is where the
2465	// frontend set up an input arg for a particular interpolation mode, but
2466	// nothing uses that input arg. Really we should have an earlier pass
2467	// that removes such an arg.)
2468	unsigned PsInputBits = Info->getPSInputAddr() & Info->getPSInputEnable();
2469	if ((PsInputBits & 0x7F) == 0 \|\|
2470	((PsInputBits & 0xF) == 0 && (PsInputBits >> 11 & 1)))
2471	Info->markPSInputEnabled(llvm::countr_zero(Info->getPSInputAddr()));
2472	}
2473	} else if (IsKernel) {
2474	assert(Info->hasWorkGroupIDX() && Info->hasWorkItemIDX())(static_cast <bool> (Info->hasWorkGroupIDX() && Info->hasWorkItemIDX()) ? void (0) : __assert_fail ("Info->hasWorkGroupIDX() && Info->hasWorkItemIDX()" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 2474, __extension__ __PRETTY_FUNCTION__));
2475	} else {
2476	Splits.append(Ins.begin(), Ins.end());
2477	}
2478
2479	if (IsEntryFunc) {
2480	allocateSpecialEntryInputVGPRs(CCInfo, MF, TRI, Info);
2481	allocateHSAUserSGPRs(CCInfo, MF, TRI, Info);
2482	} else if (!IsGraphics) {
2483	// For the fixed ABI, pass workitem IDs in the last argument register.
2484	allocateSpecialInputVGPRsFixed(CCInfo, MF, TRI, Info);
2485	}
2486
2487	if (IsKernel) {
2488	analyzeFormalArgumentsCompute(CCInfo, Ins);
2489	} else {
2490	CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, isVarArg);
2491	CCInfo.AnalyzeFormalArguments(Splits, AssignFn);
2492	}
2493
2494	SmallVector<SDValue, 16> Chains;
2495
2496	// FIXME: This is the minimum kernel argument alignment. We should improve
2497	// this to the maximum alignment of the arguments.
2498	//
2499	// FIXME: Alignment of explicit arguments totally broken with non-0 explicit
2500	// kern arg offset.
2501	const Align KernelArgBaseAlign = Align(16);
2502
2503	for (unsigned i = 0, e = Ins.size(), ArgIdx = 0; i != e; ++i) {
2504	const ISD::InputArg &Arg = Ins[i];
2505	if (Arg.isOrigArg() && Skipped[Arg.getOrigArgIndex()]) {
2506	InVals.push_back(DAG.getUNDEF(Arg.VT));
2507	continue;
2508	}
2509
2510	CCValAssign &VA = ArgLocs[ArgIdx++];
2511	MVT VT = VA.getLocVT();
2512
2513	if (IsEntryFunc && VA.isMemLoc()) {
2514	VT = Ins[i].VT;
2515	EVT MemVT = VA.getLocVT();
2516
2517	const uint64_t Offset = VA.getLocMemOffset();
2518	Align Alignment = commonAlignment(KernelArgBaseAlign, Offset);
2519
2520	if (Arg.Flags.isByRef()) {
2521	SDValue Ptr = lowerKernArgParameterPtr(DAG, DL, Chain, Offset);
2522
2523	const GCNTargetMachine &TM =
2524	static_cast<const GCNTargetMachine &>(getTargetMachine());
2525	if (!TM.isNoopAddrSpaceCast(AMDGPUAS::CONSTANT_ADDRESS,
2526	Arg.Flags.getPointerAddrSpace())) {
2527	Ptr = DAG.getAddrSpaceCast(DL, VT, Ptr, AMDGPUAS::CONSTANT_ADDRESS,
2528	Arg.Flags.getPointerAddrSpace());
2529	}
2530
2531	InVals.push_back(Ptr);
2532	continue;
2533	}
2534
2535	SDValue Arg = lowerKernargMemParameter(
2536	DAG, VT, MemVT, DL, Chain, Offset, Alignment, Ins[i].Flags.isSExt(), &Ins[i]);
2537	Chains.push_back(Arg.getValue(1));
2538
2539	auto *ParamTy =
2540	dyn_cast<PointerType>(FType->getParamType(Ins[i].getOrigArgIndex()));
2541	if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS &&
2542	ParamTy && (ParamTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS \|\|
2543	ParamTy->getAddressSpace() == AMDGPUAS::REGION_ADDRESS)) {
2544	// On SI local pointers are just offsets into LDS, so they are always
2545	// less than 16-bits. On CI and newer they could potentially be
2546	// real pointers, so we can't guarantee their size.
2547	Arg = DAG.getNode(ISD::AssertZext, DL, Arg.getValueType(), Arg,
2548	DAG.getValueType(MVT::i16));
2549	}
2550
2551	InVals.push_back(Arg);
2552	continue;
2553	} else if (!IsEntryFunc && VA.isMemLoc()) {
2554	SDValue Val = lowerStackParameter(DAG, VA, DL, Chain, Arg);
2555	InVals.push_back(Val);
2556	if (!Arg.Flags.isByVal())
2557	Chains.push_back(Val.getValue(1));
2558	continue;
2559	}
2560
2561	assert(VA.isRegLoc() && "Parameter must be in a register!")(static_cast <bool> (VA.isRegLoc() && "Parameter must be in a register!" ) ? void (0) : __assert_fail ("VA.isRegLoc() && \"Parameter must be in a register!\"" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 2561, __extension__ __PRETTY_FUNCTION__));
2562
2563	Register Reg = VA.getLocReg();
2564	const TargetRegisterClass *RC = nullptr;
2565	if (AMDGPU::VGPR_32RegClass.contains(Reg))
2566	RC = &AMDGPU::VGPR_32RegClass;
2567	else if (AMDGPU::SGPR_32RegClass.contains(Reg))
2568	RC = &AMDGPU::SGPR_32RegClass;
2569	else
2570	llvm_unreachable("Unexpected register class in LowerFormalArguments!")::llvm::llvm_unreachable_internal("Unexpected register class in LowerFormalArguments!" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 2570);
2571	EVT ValVT = VA.getValVT();
2572
2573	Reg = MF.addLiveIn(Reg, RC);
2574	SDValue Val = DAG.getCopyFromReg(Chain, DL, Reg, VT);
2575
2576	if (Arg.Flags.isSRet()) {
2577	// The return object should be reasonably addressable.
2578
2579	// FIXME: This helps when the return is a real sret. If it is a
2580	// automatically inserted sret (i.e. CanLowerReturn returns false), an
2581	// extra copy is inserted in SelectionDAGBuilder which obscures this.
2582	unsigned NumBits
2583	= 32 - getSubtarget()->getKnownHighZeroBitsForFrameIndex();
2584	Val = DAG.getNode(ISD::AssertZext, DL, VT, Val,
2585	DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), NumBits)));
2586	}
2587
2588	// If this is an 8 or 16-bit value, it is really passed promoted
2589	// to 32 bits. Insert an assert[sz]ext to capture this, then
2590	// truncate to the right size.
2591	switch (VA.getLocInfo()) {
2592	case CCValAssign::Full:
2593	break;
2594	case CCValAssign::BCvt:
2595	Val = DAG.getNode(ISD::BITCAST, DL, ValVT, Val);
2596	break;
2597	case CCValAssign::SExt:
2598	Val = DAG.getNode(ISD::AssertSext, DL, VT, Val,
2599	DAG.getValueType(ValVT));
2600	Val = DAG.getNode(ISD::TRUNCATE, DL, ValVT, Val);
2601	break;
2602	case CCValAssign::ZExt:
2603	Val = DAG.getNode(ISD::AssertZext, DL, VT, Val,
2604	DAG.getValueType(ValVT));
2605	Val = DAG.getNode(ISD::TRUNCATE, DL, ValVT, Val);
2606	break;
2607	case CCValAssign::AExt:
2608	Val = DAG.getNode(ISD::TRUNCATE, DL, ValVT, Val);
2609	break;
2610	default:
2611	llvm_unreachable("Unknown loc info!")::llvm::llvm_unreachable_internal("Unknown loc info!", "llvm/lib/Target/AMDGPU/SIISelLowering.cpp" , 2611);
2612	}
2613
2614	InVals.push_back(Val);
2615	}
2616
2617	// Start adding system SGPRs.
2618	if (IsEntryFunc) {
2619	allocateSystemSGPRs(CCInfo, MF, *Info, CallConv, IsGraphics);
2620	} else {
2621	CCInfo.AllocateReg(Info->getScratchRSrcReg());
2622	if (!IsGraphics)
2623	allocateSpecialInputSGPRs(CCInfo, MF, TRI, Info);
2624	}
2625
2626	auto &ArgUsageInfo =
2627	DAG.getPass()->getAnalysis<AMDGPUArgumentUsageInfo>();
2628	ArgUsageInfo.setFuncArgInfo(Fn, Info->getArgInfo());
2629
2630	unsigned StackArgSize = CCInfo.getNextStackOffset();
2631	Info->setBytesInStackArgArea(StackArgSize);
2632
2633	return Chains.empty() ? Chain :
2634	DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
2635	}
2636
2637	// TODO: If return values can't fit in registers, we should return as many as
2638	// possible in registers before passing on stack.
2639	bool SITargetLowering::CanLowerReturn(
2640	CallingConv::ID CallConv,
2641	MachineFunction &MF, bool IsVarArg,
2642	const SmallVectorImpl<ISD::OutputArg> &Outs,
2643	LLVMContext &Context) const {
2644	// Replacing returns with sret/stack usage doesn't make sense for shaders.
2645	// FIXME: Also sort of a workaround for custom vector splitting in LowerReturn
2646	// for shaders. Vector types should be explicitly handled by CC.
2647	if (AMDGPU::isEntryFunctionCC(CallConv))
2648	return true;
2649
2650	SmallVector<CCValAssign, 16> RVLocs;
2651	CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
2652	return CCInfo.CheckReturn(Outs, CCAssignFnForReturn(CallConv, IsVarArg));
2653	}
2654
2655	SDValue
2656	SITargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
2657	bool isVarArg,
2658	const SmallVectorImpl<ISD::OutputArg> &Outs,
2659	const SmallVectorImpl<SDValue> &OutVals,
2660	const SDLoc &DL, SelectionDAG &DAG) const {
2661	MachineFunction &MF = DAG.getMachineFunction();
2662	SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
2663
2664	if (AMDGPU::isKernel(CallConv)) {
2665	return AMDGPUTargetLowering::LowerReturn(Chain, CallConv, isVarArg, Outs,
2666	OutVals, DL, DAG);
2667	}
2668
2669	bool IsShader = AMDGPU::isShader(CallConv);
2670
2671	Info->setIfReturnsVoid(Outs.empty());
2672	bool IsWaveEnd = Info->returnsVoid() && IsShader;
2673
2674	// CCValAssign - represent the assignment of the return value to a location.
2675	SmallVector<CCValAssign, 48> RVLocs;
2676	SmallVector<ISD::OutputArg, 48> Splits;
2677
2678	// CCState - Info about the registers and stack slots.
2679	CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
2680	*DAG.getContext());
2681
2682	// Analyze outgoing return values.
2683	CCInfo.AnalyzeReturn(Outs, CCAssignFnForReturn(CallConv, isVarArg));
2684
2685	SDValue Flag;
2686	SmallVector<SDValue, 48> RetOps;
2687	RetOps.push_back(Chain); // Operand #0 = Chain (updated below)
2688
2689	// Copy the result values into the output registers.
2690	for (unsigned I = 0, RealRVLocIdx = 0, E = RVLocs.size(); I != E;
2691	++I, ++RealRVLocIdx) {
2692	CCValAssign &VA = RVLocs[I];
2693	assert(VA.isRegLoc() && "Can only return in registers!")(static_cast <bool> (VA.isRegLoc() && "Can only return in registers!" ) ? void (0) : __assert_fail ("VA.isRegLoc() && \"Can only return in registers!\"" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 2693, __extension__ __PRETTY_FUNCTION__));
2694	// TODO: Partially return in registers if return values don't fit.
2695	SDValue Arg = OutVals[RealRVLocIdx];
2696
2697	// Copied from other backends.
2698	switch (VA.getLocInfo()) {
2699	case CCValAssign::Full:
2700	break;
2701	case CCValAssign::BCvt:
2702	Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg);
2703	break;
2704	case CCValAssign::SExt:
2705	Arg = DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Arg);
2706	break;
2707	case CCValAssign::ZExt:
2708	Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg);
2709	break;
2710	case CCValAssign::AExt:
2711	Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg);
2712	break;
2713	default:
2714	llvm_unreachable("Unknown loc info!")::llvm::llvm_unreachable_internal("Unknown loc info!", "llvm/lib/Target/AMDGPU/SIISelLowering.cpp" , 2714);
2715	}
2716
2717	Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Arg, Flag);
2718	Flag = Chain.getValue(1);
2719	RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
2720	}
2721
2722	// FIXME: Does sret work properly?
2723	if (!Info->isEntryFunction()) {
2724	const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
2725	const MCPhysReg *I =
2726	TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction());
2727	if (I) {
2728	for (; *I; ++I) {
2729	if (AMDGPU::SReg_64RegClass.contains(*I))
2730	RetOps.push_back(DAG.getRegister(*I, MVT::i64));
2731	else if (AMDGPU::SReg_32RegClass.contains(*I))
2732	RetOps.push_back(DAG.getRegister(*I, MVT::i32));
2733	else
2734	llvm_unreachable("Unexpected register class in CSRsViaCopy!")::llvm::llvm_unreachable_internal("Unexpected register class in CSRsViaCopy!" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 2734);
2735	}
2736	}
2737	}
2738
2739	// Update chain and glue.
2740	RetOps[0] = Chain;
2741	if (Flag.getNode())
2742	RetOps.push_back(Flag);
2743
2744	unsigned Opc = AMDGPUISD::ENDPGM;
2745	if (!IsWaveEnd)
2746	Opc = IsShader ? AMDGPUISD::RETURN_TO_EPILOG : AMDGPUISD::RET_FLAG;
2747	return DAG.getNode(Opc, DL, MVT::Other, RetOps);
2748	}
2749
2750	SDValue SITargetLowering::LowerCallResult(
2751	SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool IsVarArg,
2752	const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
2753	SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool IsThisReturn,
2754	SDValue ThisVal) const {
2755	CCAssignFn *RetCC = CCAssignFnForReturn(CallConv, IsVarArg);
2756
2757	// Assign locations to each value returned by this call.
2758	SmallVector<CCValAssign, 16> RVLocs;
2759	CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
2760	*DAG.getContext());
2761	CCInfo.AnalyzeCallResult(Ins, RetCC);
2762
2763	// Copy all of the result registers out of their specified physreg.
2764	for (unsigned i = 0; i != RVLocs.size(); ++i) {
2765	CCValAssign VA = RVLocs[i];
2766	SDValue Val;
2767
2768	if (VA.isRegLoc()) {
2769	Val = DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), InFlag);
2770	Chain = Val.getValue(1);
2771	InFlag = Val.getValue(2);
2772	} else if (VA.isMemLoc()) {
2773	report_fatal_error("TODO: return values in memory");
2774	} else
2775	llvm_unreachable("unknown argument location type")::llvm::llvm_unreachable_internal("unknown argument location type" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 2775);
2776
2777	switch (VA.getLocInfo()) {
2778	case CCValAssign::Full:
2779	break;
2780	case CCValAssign::BCvt:
2781	Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
2782	break;
2783	case CCValAssign::ZExt:
2784	Val = DAG.getNode(ISD::AssertZext, DL, VA.getLocVT(), Val,
2785	DAG.getValueType(VA.getValVT()));
2786	Val = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Val);
2787	break;
2788	case CCValAssign::SExt:
2789	Val = DAG.getNode(ISD::AssertSext, DL, VA.getLocVT(), Val,
2790	DAG.getValueType(VA.getValVT()));
2791	Val = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Val);
2792	break;
2793	case CCValAssign::AExt:
2794	Val = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Val);
2795	break;
2796	default:
2797	llvm_unreachable("Unknown loc info!")::llvm::llvm_unreachable_internal("Unknown loc info!", "llvm/lib/Target/AMDGPU/SIISelLowering.cpp" , 2797);
2798	}
2799
2800	InVals.push_back(Val);
2801	}
2802
2803	return Chain;
2804	}
2805
2806	// Add code to pass special inputs required depending on used features separate
2807	// from the explicit user arguments present in the IR.
2808	void SITargetLowering::passSpecialInputs(
2809	CallLoweringInfo &CLI,
2810	CCState &CCInfo,
2811	const SIMachineFunctionInfo &Info,
2812	SmallVectorImpl<std::pair<unsigned, SDValue>> &RegsToPass,
2813	SmallVectorImpl<SDValue> &MemOpChains,
2814	SDValue Chain) const {
2815	// If we don't have a call site, this was a call inserted by
2816	// legalization. These can never use special inputs.
2817	if (!CLI.CB)
2818	return;
2819
2820	SelectionDAG &DAG = CLI.DAG;
2821	const SDLoc &DL = CLI.DL;
2822	const Function &F = DAG.getMachineFunction().getFunction();
2823
2824	const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
2825	const AMDGPUFunctionArgInfo &CallerArgInfo = Info.getArgInfo();
2826
2827	const AMDGPUFunctionArgInfo *CalleeArgInfo
2828	= &AMDGPUArgumentUsageInfo::FixedABIFunctionInfo;
2829	if (const Function *CalleeFunc = CLI.CB->getCalledFunction()) {
2830	auto &ArgUsageInfo =
2831	DAG.getPass()->getAnalysis<AMDGPUArgumentUsageInfo>();
2832	CalleeArgInfo = &ArgUsageInfo.lookupFuncArgInfo(*CalleeFunc);
2833	}
2834
2835	// TODO: Unify with private memory register handling. This is complicated by
2836	// the fact that at least in kernels, the input argument is not necessarily
2837	// in the same location as the input.
2838	static constexpr std::pair<AMDGPUFunctionArgInfo::PreloadedValue,
2839	StringLiteral> ImplicitAttrs[] = {
2840	{AMDGPUFunctionArgInfo::DISPATCH_PTR, "amdgpu-no-dispatch-ptr"},
2841	{AMDGPUFunctionArgInfo::QUEUE_PTR, "amdgpu-no-queue-ptr" },
2842	{AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR, "amdgpu-no-implicitarg-ptr"},
2843	{AMDGPUFunctionArgInfo::DISPATCH_ID, "amdgpu-no-dispatch-id"},
2844	{AMDGPUFunctionArgInfo::WORKGROUP_ID_X, "amdgpu-no-workgroup-id-x"},
2845	{AMDGPUFunctionArgInfo::WORKGROUP_ID_Y,"amdgpu-no-workgroup-id-y"},
2846	{AMDGPUFunctionArgInfo::WORKGROUP_ID_Z,"amdgpu-no-workgroup-id-z"},
2847	{AMDGPUFunctionArgInfo::LDS_KERNEL_ID,"amdgpu-no-lds-kernel-id"},
2848	};
2849
2850	for (auto Attr : ImplicitAttrs) {
2851	const ArgDescriptor *OutgoingArg;
2852	const TargetRegisterClass *ArgRC;
2853	LLT ArgTy;
2854
2855	AMDGPUFunctionArgInfo::PreloadedValue InputID = Attr.first;
2856
2857	// If the callee does not use the attribute value, skip copying the value.
2858	if (CLI.CB->hasFnAttr(Attr.second))
2859	continue;
2860
2861	std::tie(OutgoingArg, ArgRC, ArgTy) =
2862	CalleeArgInfo->getPreloadedValue(InputID);
2863	if (!OutgoingArg)
2864	continue;
2865
2866	const ArgDescriptor *IncomingArg;
2867	const TargetRegisterClass *IncomingArgRC;
2868	LLT Ty;
2869	std::tie(IncomingArg, IncomingArgRC, Ty) =
2870	CallerArgInfo.getPreloadedValue(InputID);
2871	assert(IncomingArgRC == ArgRC)(static_cast <bool> (IncomingArgRC == ArgRC) ? void (0) : __assert_fail ("IncomingArgRC == ArgRC", "llvm/lib/Target/AMDGPU/SIISelLowering.cpp" , 2871, __extension__ __PRETTY_FUNCTION__));
2872
2873	// All special arguments are ints for now.
2874	EVT ArgVT = TRI->getSpillSize(*ArgRC) == 8 ? MVT::i64 : MVT::i32;
2875	SDValue InputReg;
2876
2877	if (IncomingArg) {
2878	InputReg = loadInputValue(DAG, ArgRC, ArgVT, DL, *IncomingArg);
2879	} else if (InputID == AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR) {
2880	// The implicit arg ptr is special because it doesn't have a corresponding
2881	// input for kernels, and is computed from the kernarg segment pointer.
2882	InputReg = getImplicitArgPtr(DAG, DL);
2883	} else if (InputID == AMDGPUFunctionArgInfo::LDS_KERNEL_ID) {
2884	std::optional<uint32_t> Id =
2885	AMDGPUMachineFunction::getLDSKernelIdMetadata(F);
2886	if (Id.has_value()) {
2887	InputReg = DAG.getConstant(*Id, DL, ArgVT);
2888	} else {
2889	InputReg = DAG.getUNDEF(ArgVT);
2890	}
2891	} else {
2892	// We may have proven the input wasn't needed, although the ABI is
2893	// requiring it. We just need to allocate the register appropriately.
2894	InputReg = DAG.getUNDEF(ArgVT);
2895	}
2896
2897	if (OutgoingArg->isRegister()) {
2898	RegsToPass.emplace_back(OutgoingArg->getRegister(), InputReg);
2899	if (!CCInfo.AllocateReg(OutgoingArg->getRegister()))
2900	report_fatal_error("failed to allocate implicit input argument");
2901	} else {
2902	unsigned SpecialArgOffset =
2903	CCInfo.AllocateStack(ArgVT.getStoreSize(), Align(4));
2904	SDValue ArgStore = storeStackInputValue(DAG, DL, Chain, InputReg,
2905	SpecialArgOffset);
2906	MemOpChains.push_back(ArgStore);
2907	}
2908	}
2909
2910	// Pack workitem IDs into a single register or pass it as is if already
2911	// packed.
2912	const ArgDescriptor *OutgoingArg;
2913	const TargetRegisterClass *ArgRC;
2914	LLT Ty;
2915
2916	std::tie(OutgoingArg, ArgRC, Ty) =
2917	CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_X);
2918	if (!OutgoingArg)
2919	std::tie(OutgoingArg, ArgRC, Ty) =
2920	CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Y);
2921	if (!OutgoingArg)
2922	std::tie(OutgoingArg, ArgRC, Ty) =
2923	CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Z);
2924	if (!OutgoingArg)
2925	return;
2926
2927	const ArgDescriptor *IncomingArgX = std::get<0>(
2928	CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_X));
2929	const ArgDescriptor *IncomingArgY = std::get<0>(
2930	CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Y));
2931	const ArgDescriptor *IncomingArgZ = std::get<0>(
2932	CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Z));
2933
2934	SDValue InputReg;
2935	SDLoc SL;
2936
2937	const bool NeedWorkItemIDX = !CLI.CB->hasFnAttr("amdgpu-no-workitem-id-x");
2938	const bool NeedWorkItemIDY = !CLI.CB->hasFnAttr("amdgpu-no-workitem-id-y");
2939	const bool NeedWorkItemIDZ = !CLI.CB->hasFnAttr("amdgpu-no-workitem-id-z");
2940
2941	// If incoming ids are not packed we need to pack them.
2942	if (IncomingArgX && !IncomingArgX->isMasked() && CalleeArgInfo->WorkItemIDX &&
2943	NeedWorkItemIDX) {
2944	if (Subtarget->getMaxWorkitemID(F, 0) != 0) {
2945	InputReg = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgX);
2946	} else {
2947	InputReg = DAG.getConstant(0, DL, MVT::i32);
2948	}
2949	}
2950
2951	if (IncomingArgY && !IncomingArgY->isMasked() && CalleeArgInfo->WorkItemIDY &&
2952	NeedWorkItemIDY && Subtarget->getMaxWorkitemID(F, 1) != 0) {
2953	SDValue Y = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgY);
2954	Y = DAG.getNode(ISD::SHL, SL, MVT::i32, Y,
2955	DAG.getShiftAmountConstant(10, MVT::i32, SL));
2956	InputReg = InputReg.getNode() ?
2957	DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Y) : Y;
2958	}
2959
2960	if (IncomingArgZ && !IncomingArgZ->isMasked() && CalleeArgInfo->WorkItemIDZ &&
2961	NeedWorkItemIDZ && Subtarget->getMaxWorkitemID(F, 2) != 0) {
2962	SDValue Z = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgZ);
2963	Z = DAG.getNode(ISD::SHL, SL, MVT::i32, Z,
2964	DAG.getShiftAmountConstant(20, MVT::i32, SL));
2965	InputReg = InputReg.getNode() ?
2966	DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Z) : Z;
2967	}
2968
2969	if (!InputReg && (NeedWorkItemIDX \|\| NeedWorkItemIDY \|\| NeedWorkItemIDZ)) {
2970	if (!IncomingArgX && !IncomingArgY && !IncomingArgZ) {
2971	// We're in a situation where the outgoing function requires the workitem
2972	// ID, but the calling function does not have it (e.g a graphics function
2973	// calling a C calling convention function). This is illegal, but we need
2974	// to produce something.
2975	InputReg = DAG.getUNDEF(MVT::i32);
2976	} else {
2977	// Workitem ids are already packed, any of present incoming arguments
2978	// will carry all required fields.
2979	ArgDescriptor IncomingArg = ArgDescriptor::createArg(
2980	IncomingArgX ? *IncomingArgX :
2981	IncomingArgY ? *IncomingArgY :
2982	*IncomingArgZ, ~0u);
2983	InputReg = loadInputValue(DAG, ArgRC, MVT::i32, DL, IncomingArg);
2984	}
2985	}
2986
2987	if (OutgoingArg->isRegister()) {
2988	if (InputReg)
2989	RegsToPass.emplace_back(OutgoingArg->getRegister(), InputReg);
2990
2991	CCInfo.AllocateReg(OutgoingArg->getRegister());
2992	} else {
2993	unsigned SpecialArgOffset = CCInfo.AllocateStack(4, Align(4));
2994	if (InputReg) {
2995	SDValue ArgStore = storeStackInputValue(DAG, DL, Chain, InputReg,
2996	SpecialArgOffset);
2997	MemOpChains.push_back(ArgStore);
2998	}
2999	}
3000	}
3001
3002	static bool canGuaranteeTCO(CallingConv::ID CC) {
3003	return CC == CallingConv::Fast;
3004	}
3005
3006	/// Return true if we might ever do TCO for calls with this calling convention.
3007	static bool mayTailCallThisCC(CallingConv::ID CC) {
3008	switch (CC) {
3009	case CallingConv::C:
3010	case CallingConv::AMDGPU_Gfx:
3011	return true;
3012	default:
3013	return canGuaranteeTCO(CC);
3014	}
3015	}
3016
3017	bool SITargetLowering::isEligibleForTailCallOptimization(
3018	SDValue Callee, CallingConv::ID CalleeCC, bool IsVarArg,
3019	const SmallVectorImpl<ISD::OutputArg> &Outs,
3020	const SmallVectorImpl<SDValue> &OutVals,
3021	const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const {
3022	if (!mayTailCallThisCC(CalleeCC))
3023	return false;
3024
3025	// For a divergent call target, we need to do a waterfall loop over the
3026	// possible callees which precludes us from using a simple jump.
3027	if (Callee->isDivergent())
3028	return false;
3029
3030	MachineFunction &MF = DAG.getMachineFunction();
3031	const Function &CallerF = MF.getFunction();
3032	CallingConv::ID CallerCC = CallerF.getCallingConv();
3033	const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
3034	const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
3035
3036	// Kernels aren't callable, and don't have a live in return address so it
3037	// doesn't make sense to do a tail call with entry functions.
3038	if (!CallerPreserved)
3039	return false;
3040
3041	bool CCMatch = CallerCC == CalleeCC;
3042
3043	if (DAG.getTarget().Options.GuaranteedTailCallOpt) {
3044	if (canGuaranteeTCO(CalleeCC) && CCMatch)
3045	return true;
3046	return false;
3047	}
3048
3049	// TODO: Can we handle var args?
3050	if (IsVarArg)
3051	return false;
3052
3053	for (const Argument &Arg : CallerF.args()) {
3054	if (Arg.hasByValAttr())
3055	return false;
3056	}
3057
3058	LLVMContext &Ctx = *DAG.getContext();
3059
3060	// Check that the call results are passed in the same way.
3061	if (!CCState::resultsCompatible(CalleeCC, CallerCC, MF, Ctx, Ins,
3062	CCAssignFnForCall(CalleeCC, IsVarArg),
3063	CCAssignFnForCall(CallerCC, IsVarArg)))
3064	return false;
3065
3066	// The callee has to preserve all registers the caller needs to preserve.
3067	if (!CCMatch) {
3068	const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
3069	if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
3070	return false;
3071	}
3072
3073	// Nothing more to check if the callee is taking no arguments.
3074	if (Outs.empty())
3075	return true;
3076
3077	SmallVector<CCValAssign, 16> ArgLocs;
3078	CCState CCInfo(CalleeCC, IsVarArg, MF, ArgLocs, Ctx);
3079
3080	CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CalleeCC, IsVarArg));
3081
3082	const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
3083	// If the stack arguments for this call do not fit into our own save area then
3084	// the call cannot be made tail.
3085	// TODO: Is this really necessary?
3086	if (CCInfo.getNextStackOffset() > FuncInfo->getBytesInStackArgArea())
3087	return false;
3088
3089	const MachineRegisterInfo &MRI = MF.getRegInfo();
3090	return parametersInCSRMatch(MRI, CallerPreserved, ArgLocs, OutVals);
3091	}
3092
3093	bool SITargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
3094	if (!CI->isTailCall())
3095	return false;
3096
3097	const Function *ParentFn = CI->getParent()->getParent();
3098	if (AMDGPU::isEntryFunctionCC(ParentFn->getCallingConv()))
3099	return false;
3100	return true;
3101	}
3102
3103	// The wave scratch offset register is used as the global base pointer.
3104	SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
3105	SmallVectorImpl<SDValue> &InVals) const {
3106	SelectionDAG &DAG = CLI.DAG;
3107	const SDLoc &DL = CLI.DL;
3108	SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs;
3109	SmallVector<SDValue, 32> &OutVals = CLI.OutVals;
3110	SmallVector<ISD::InputArg, 32> &Ins = CLI.Ins;
3111	SDValue Chain = CLI.Chain;
3112	SDValue Callee = CLI.Callee;
3113	bool &IsTailCall = CLI.IsTailCall;
3114	CallingConv::ID CallConv = CLI.CallConv;
3115	bool IsVarArg = CLI.IsVarArg;
3116	bool IsSibCall = false;
3117	bool IsThisReturn = false;
3118	MachineFunction &MF = DAG.getMachineFunction();
3119
3120	if (Callee.isUndef() \|\| isNullConstant(Callee)) {
3121	if (!CLI.IsTailCall) {
3122	for (unsigned I = 0, E = CLI.Ins.size(); I != E; ++I)
3123	InVals.push_back(DAG.getUNDEF(CLI.Ins[I].VT));
3124	}
3125
3126	return Chain;
3127	}
3128
3129	if (IsVarArg) {
3130	return lowerUnhandledCall(CLI, InVals,
3131	"unsupported call to variadic function ");
3132	}
3133
3134	if (!CLI.CB)
3135	report_fatal_error("unsupported libcall legalization");
3136
3137	if (IsTailCall && MF.getTarget().Options.GuaranteedTailCallOpt) {
3138	return lowerUnhandledCall(CLI, InVals,
3139	"unsupported required tail call to function ");
3140	}
3141
3142	if (AMDGPU::isShader(CallConv)) {
3143	// Note the issue is with the CC of the called function, not of the call
3144	// itself.
3145	return lowerUnhandledCall(CLI, InVals,
3146	"unsupported call to a shader function ");
3147	}
3148
3149	if (AMDGPU::isShader(MF.getFunction().getCallingConv()) &&
3150	CallConv != CallingConv::AMDGPU_Gfx) {
3151	// Only allow calls with specific calling conventions.
3152	return lowerUnhandledCall(CLI, InVals,
3153	"unsupported calling convention for call from "
3154	"graphics shader of function ");
3155	}
3156
3157	if (IsTailCall) {
3158	IsTailCall = isEligibleForTailCallOptimization(
3159	Callee, CallConv, IsVarArg, Outs, OutVals, Ins, DAG);
3160	if (!IsTailCall && CLI.CB && CLI.CB->isMustTailCall()) {
3161	report_fatal_error("failed to perform tail call elimination on a call "
3162	"site marked musttail");
3163	}
3164
3165	bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
3166
3167	// A sibling call is one where we're under the usual C ABI and not planning
3168	// to change that but can still do a tail call:
3169	if (!TailCallOpt && IsTailCall)
3170	IsSibCall = true;
3171
3172	if (IsTailCall)
3173	++NumTailCalls;
3174	}
3175
3176	const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
3177	SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
3178	SmallVector<SDValue, 8> MemOpChains;
3179
3180	// Analyze operands of the call, assigning locations to each operand.
3181	SmallVector<CCValAssign, 16> ArgLocs;
3182	CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
3183	CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, IsVarArg);
3184
3185	if (CallConv != CallingConv::AMDGPU_Gfx) {
3186	// With a fixed ABI, allocate fixed registers before user arguments.
3187	passSpecialInputs(CLI, CCInfo, *Info, RegsToPass, MemOpChains, Chain);
3188	}
3189
3190	CCInfo.AnalyzeCallOperands(Outs, AssignFn);
3191
3192	// Get a count of how many bytes are to be pushed on the stack.
3193	unsigned NumBytes = CCInfo.getNextStackOffset();
3194
3195	if (IsSibCall) {
3196	// Since we're not changing the ABI to make this a tail call, the memory
3197	// operands are already available in the caller's incoming argument space.
3198	NumBytes = 0;
3199	}
3200
3201	// FPDiff is the byte offset of the call's argument area from the callee's.
3202	// Stores to callee stack arguments will be placed in FixedStackSlots offset
3203	// by this amount for a tail call. In a sibling call it must be 0 because the
3204	// caller will deallocate the entire stack and the callee still expects its
3205	// arguments to begin at SP+0. Completely unused for non-tail calls.
3206	int32_t FPDiff = 0;
3207	MachineFrameInfo &MFI = MF.getFrameInfo();
3208
3209	// Adjust the stack pointer for the new arguments...
3210	// These operations are automatically eliminated by the prolog/epilog pass
3211	if (!IsSibCall) {
3212	Chain = DAG.getCALLSEQ_START(Chain, 0, 0, DL);
3213
3214	if (!Subtarget->enableFlatScratch()) {
3215	SmallVector<SDValue, 4> CopyFromChains;
3216
3217	// In the HSA case, this should be an identity copy.
3218	SDValue ScratchRSrcReg
3219	= DAG.getCopyFromReg(Chain, DL, Info->getScratchRSrcReg(), MVT::v4i32);
3220	RegsToPass.emplace_back(AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3, ScratchRSrcReg);
3221	CopyFromChains.push_back(ScratchRSrcReg.getValue(1));
3222	Chain = DAG.getTokenFactor(DL, CopyFromChains);
3223	}
3224	}
3225
3226	MVT PtrVT = MVT::i32;
3227
3228	// Walk the register/memloc assignments, inserting copies/loads.
3229	for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
3230	CCValAssign &VA = ArgLocs[i];
3231	SDValue Arg = OutVals[i];
3232
3233	// Promote the value if needed.
3234	switch (VA.getLocInfo()) {
3235	case CCValAssign::Full:
3236	break;
3237	case CCValAssign::BCvt:
3238	Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg);
3239	break;
3240	case CCValAssign::ZExt:
3241	Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg);
3242	break;
3243	case CCValAssign::SExt:
3244	Arg = DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Arg);
3245	break;
3246	case CCValAssign::AExt:
3247	Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg);
3248	break;
3249	case CCValAssign::FPExt:
3250	Arg = DAG.getNode(ISD::FP_EXTEND, DL, VA.getLocVT(), Arg);
3251	break;
3252	default:
3253	llvm_unreachable("Unknown loc info!")::llvm::llvm_unreachable_internal("Unknown loc info!", "llvm/lib/Target/AMDGPU/SIISelLowering.cpp" , 3253);
3254	}
3255
3256	if (VA.isRegLoc()) {
3257	RegsToPass.push_back(std::pair(VA.getLocReg(), Arg));
3258	} else {
3259	assert(VA.isMemLoc())(static_cast <bool> (VA.isMemLoc()) ? void (0) : __assert_fail ("VA.isMemLoc()", "llvm/lib/Target/AMDGPU/SIISelLowering.cpp" , 3259, __extension__ __PRETTY_FUNCTION__));
3260
3261	SDValue DstAddr;
3262	MachinePointerInfo DstInfo;
3263
3264	unsigned LocMemOffset = VA.getLocMemOffset();
3265	int32_t Offset = LocMemOffset;
3266
3267	SDValue PtrOff = DAG.getConstant(Offset, DL, PtrVT);
3268	MaybeAlign Alignment;
3269
3270	if (IsTailCall) {
3271	ISD::ArgFlagsTy Flags = Outs[i].Flags;
3272	unsigned OpSize = Flags.isByVal() ?
3273	Flags.getByValSize() : VA.getValVT().getStoreSize();
3274
3275	// FIXME: We can have better than the minimum byval required alignment.
3276	Alignment =
3277	Flags.isByVal()
3278	? Flags.getNonZeroByValAlign()
3279	: commonAlignment(Subtarget->getStackAlignment(), Offset);
3280
3281	Offset = Offset + FPDiff;
3282	int FI = MFI.CreateFixedObject(OpSize, Offset, true);
3283
3284	DstAddr = DAG.getFrameIndex(FI, PtrVT);
3285	DstInfo = MachinePointerInfo::getFixedStack(MF, FI);
3286
3287	// Make sure any stack arguments overlapping with where we're storing
3288	// are loaded before this eventual operation. Otherwise they'll be
3289	// clobbered.
3290
3291	// FIXME: Why is this really necessary? This seems to just result in a
3292	// lot of code to copy the stack and write them back to the same
3293	// locations, which are supposed to be immutable?
3294	Chain = addTokenForArgument(Chain, DAG, MFI, FI);
3295	} else {
3296	// Stores to the argument stack area are relative to the stack pointer.
3297	SDValue SP = DAG.getCopyFromReg(Chain, DL, Info->getStackPtrOffsetReg(),
3298	MVT::i32);
3299	DstAddr = DAG.getNode(ISD::ADD, DL, MVT::i32, SP, PtrOff);
3300	DstInfo = MachinePointerInfo::getStack(MF, LocMemOffset);
3301	Alignment =
3302	commonAlignment(Subtarget->getStackAlignment(), LocMemOffset);
3303	}
3304
3305	if (Outs[i].Flags.isByVal()) {
3306	SDValue SizeNode =
3307	DAG.getConstant(Outs[i].Flags.getByValSize(), DL, MVT::i32);
3308	SDValue Cpy =
3309	DAG.getMemcpy(Chain, DL, DstAddr, Arg, SizeNode,
3310	Outs[i].Flags.getNonZeroByValAlign(),
3311	/isVol = / false, /AlwaysInline = / true,
3312	/isTailCall = / false, DstInfo,
3313	MachinePointerInfo(AMDGPUAS::PRIVATE_ADDRESS));
3314
3315	MemOpChains.push_back(Cpy);
3316	} else {
3317	SDValue Store =
3318	DAG.getStore(Chain, DL, Arg, DstAddr, DstInfo, Alignment);
3319	MemOpChains.push_back(Store);
3320	}
3321	}
3322	}
3323
3324	if (!MemOpChains.empty())
3325	Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
3326
3327	// Build a sequence of copy-to-reg nodes chained together with token chain
3328	// and flag operands which copy the outgoing args into the appropriate regs.
3329	SDValue InFlag;
3330	for (auto &RegToPass : RegsToPass) {
3331	Chain = DAG.getCopyToReg(Chain, DL, RegToPass.first,
3332	RegToPass.second, InFlag);
3333	InFlag = Chain.getValue(1);
3334	}
3335
3336
3337	// We don't usually want to end the call-sequence here because we would tidy
3338	// the frame up after the call, however in the ABI-changing tail-call case
3339	// we've carefully laid out the parameters so that when sp is reset they'll be
3340	// in the correct location.
3341	if (IsTailCall && !IsSibCall) {
3342	Chain = DAG.getCALLSEQ_END(Chain, NumBytes, 0, InFlag, DL);
3343	InFlag = Chain.getValue(1);
3344	}
3345
3346	std::vector<SDValue> Ops;
3347	Ops.push_back(Chain);
3348	Ops.push_back(Callee);
3349	// Add a redundant copy of the callee global which will not be legalized, as
3350	// we need direct access to the callee later.
3351	if (GlobalAddressSDNode *GSD = dyn_cast<GlobalAddressSDNode>(Callee)) {
3352	const GlobalValue *GV = GSD->getGlobal();
3353	Ops.push_back(DAG.getTargetGlobalAddress(GV, DL, MVT::i64));
3354	} else {
3355	Ops.push_back(DAG.getTargetConstant(0, DL, MVT::i64));
3356	}
3357
3358	if (IsTailCall) {
3359	// Each tail call may have to adjust the stack by a different amount, so
3360	// this information must travel along with the operation for eventual
3361	// consumption by emitEpilogue.
3362	Ops.push_back(DAG.getTargetConstant(FPDiff, DL, MVT::i32));
3363	}
3364
3365	// Add argument registers to the end of the list so that they are known live
3366	// into the call.
3367	for (auto &RegToPass : RegsToPass) {
3368	Ops.push_back(DAG.getRegister(RegToPass.first,
3369	RegToPass.second.getValueType()));
3370	}
3371
3372	// Add a register mask operand representing the call-preserved registers.
3373
3374	auto TRI = static_cast<const SIRegisterInfo>(Subtarget->getRegisterInfo());
3375	const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
3376	assert(Mask && "Missing call preserved mask for calling convention")(static_cast <bool> (Mask && "Missing call preserved mask for calling convention" ) ? void (0) : __assert_fail ("Mask && \"Missing call preserved mask for calling convention\"" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 3376, __extension__ __PRETTY_FUNCTION__));
3377	Ops.push_back(DAG.getRegisterMask(Mask));
3378
3379	if (InFlag.getNode())
3380	Ops.push_back(InFlag);
3381
3382	SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
3383
3384	// If we're doing a tall call, use a TC_RETURN here rather than an
3385	// actual call instruction.
3386	if (IsTailCall) {
3387	MFI.setHasTailCall();
3388	return DAG.getNode(AMDGPUISD::TC_RETURN, DL, NodeTys, Ops);
3389	}
3390
3391	// Returns a chain and a flag for retval copy to use.
3392	SDValue Call = DAG.getNode(AMDGPUISD::CALL, DL, NodeTys, Ops);
3393	Chain = Call.getValue(0);
3394	InFlag = Call.getValue(1);
3395
3396	uint64_t CalleePopBytes = NumBytes;
3397	Chain = DAG.getCALLSEQ_END(Chain, 0, CalleePopBytes, InFlag, DL);
3398	if (!Ins.empty())
3399	InFlag = Chain.getValue(1);
3400
3401	// Handle result values, copying them out of physregs into vregs that we
3402	// return.
3403	return LowerCallResult(Chain, InFlag, CallConv, IsVarArg, Ins, DL, DAG,
3404	InVals, IsThisReturn,
3405	IsThisReturn ? OutVals[0] : SDValue());
3406	}
3407
3408	// This is identical to the default implementation in ExpandDYNAMIC_STACKALLOC,
3409	// except for applying the wave size scale to the increment amount.
3410	SDValue SITargetLowering::lowerDYNAMIC_STACKALLOCImpl(
3411	SDValue Op, SelectionDAG &DAG) const {
3412	const MachineFunction &MF = DAG.getMachineFunction();
3413	const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
3414
3415	SDLoc dl(Op);
3416	EVT VT = Op.getValueType();
3417	SDValue Tmp1 = Op;
3418	SDValue Tmp2 = Op.getValue(1);
3419	SDValue Tmp3 = Op.getOperand(2);
3420	SDValue Chain = Tmp1.getOperand(0);
3421
3422	Register SPReg = Info->getStackPtrOffsetReg();
3423
3424	// Chain the dynamic stack allocation so that it doesn't modify the stack
3425	// pointer when other instructions are using the stack.
3426	Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
3427
3428	SDValue Size = Tmp2.getOperand(1);
3429	SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
3430	Chain = SP.getValue(1);
3431	MaybeAlign Alignment = cast<ConstantSDNode>(Tmp3)->getMaybeAlignValue();
3432	const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
3433	const TargetFrameLowering *TFL = ST.getFrameLowering();
3434	unsigned Opc =
3435	TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp ?
3436	ISD::ADD : ISD::SUB;
3437
3438	SDValue ScaledSize = DAG.getNode(
3439	ISD::SHL, dl, VT, Size,
3440	DAG.getConstant(ST.getWavefrontSizeLog2(), dl, MVT::i32));
3441
3442	Align StackAlign = TFL->getStackAlign();
3443	Tmp1 = DAG.getNode(Opc, dl, VT, SP, ScaledSize); // Value
3444	if (Alignment && *Alignment > StackAlign) {
3445	Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
3446	DAG.getConstant(-(uint64_t)Alignment->value()
3447	<< ST.getWavefrontSizeLog2(),
3448	dl, VT));
3449	}
3450
3451	Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain
3452	Tmp2 = DAG.getCALLSEQ_END(Chain, 0, 0, SDValue(), dl);
3453
3454	return DAG.getMergeValues({Tmp1, Tmp2}, dl);
3455	}
3456
3457	SDValue SITargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
3458	SelectionDAG &DAG) const {
3459	// We only handle constant sizes here to allow non-entry block, static sized
3460	// allocas. A truly dynamic value is more difficult to support because we
3461	// don't know if the size value is uniform or not. If the size isn't uniform,
3462	// we would need to do a wave reduction to get the maximum size to know how
3463	// much to increment the uniform stack pointer.
3464	SDValue Size = Op.getOperand(1);
3465	if (isa<ConstantSDNode>(Size))
3466	return lowerDYNAMIC_STACKALLOCImpl(Op, DAG); // Use "generic" expansion.
3467
3468	return AMDGPUTargetLowering::LowerDYNAMIC_STACKALLOC(Op, DAG);
3469	}
3470
3471	Register SITargetLowering::getRegisterByName(const char* RegName, LLT VT,
3472	const MachineFunction &MF) const {
3473	Register Reg = StringSwitch<Register>(RegName)
3474	.Case("m0", AMDGPU::M0)
3475	.Case("exec", AMDGPU::EXEC)
3476	.Case("exec_lo", AMDGPU::EXEC_LO)
3477	.Case("exec_hi", AMDGPU::EXEC_HI)
3478	.Case("flat_scratch", AMDGPU::FLAT_SCR)
3479	.Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
3480	.Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
3481	.Default(Register());
3482
3483	if (Reg == AMDGPU::NoRegister) {
3484	report_fatal_error(Twine("invalid register name \""
3485	+ StringRef(RegName) + "\"."));
3486
3487	}
3488
3489	if (!Subtarget->hasFlatScrRegister() &&
3490	Subtarget->getRegisterInfo()->regsOverlap(Reg, AMDGPU::FLAT_SCR)) {
3491	report_fatal_error(Twine("invalid register \""
3492	+ StringRef(RegName) + "\" for subtarget."));
3493	}
3494
3495	switch (Reg) {
3496	case AMDGPU::M0:
3497	case AMDGPU::EXEC_LO:
3498	case AMDGPU::EXEC_HI:
3499	case AMDGPU::FLAT_SCR_LO:
3500	case AMDGPU::FLAT_SCR_HI:
3501	if (VT.getSizeInBits() == 32)
3502	return Reg;
3503	break;
3504	case AMDGPU::EXEC:
3505	case AMDGPU::FLAT_SCR:
3506	if (VT.getSizeInBits() == 64)
3507	return Reg;
3508	break;
3509	default:
3510	llvm_unreachable("missing register type checking")::llvm::llvm_unreachable_internal("missing register type checking" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 3510);
3511	}
3512
3513	report_fatal_error(Twine("invalid type for register \""
3514	+ StringRef(RegName) + "\"."));
3515	}
3516
3517	// If kill is not the last instruction, split the block so kill is always a
3518	// proper terminator.
3519	MachineBasicBlock *
3520	SITargetLowering::splitKillBlock(MachineInstr &MI,
3521	MachineBasicBlock *BB) const {
3522	MachineBasicBlock SplitBB = BB->splitAt(MI, false /UpdateLiveIns*/);
3523	const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
3524	MI.setDesc(TII->getKillTerminatorFromPseudo(MI.getOpcode()));
3525	return SplitBB;
3526	}
3527
3528	// Split block \p MBB at \p MI, as to insert a loop. If \p InstInLoop is true,
3529	// \p MI will be the only instruction in the loop body block. Otherwise, it will
3530	// be the first instruction in the remainder block.
3531	//
3532	/// \returns { LoopBody, Remainder }
3533	static std::pair<MachineBasicBlock , MachineBasicBlock >
3534	splitBlockForLoop(MachineInstr &MI, MachineBasicBlock &MBB, bool InstInLoop) {
3535	MachineFunction *MF = MBB.getParent();
3536	MachineBasicBlock::iterator I(&MI);
3537
3538	// To insert the loop we need to split the block. Move everything after this
3539	// point to a new block, and insert a new empty block between the two.
3540	MachineBasicBlock *LoopBB = MF->CreateMachineBasicBlock();
3541	MachineBasicBlock *RemainderBB = MF->CreateMachineBasicBlock();
3542	MachineFunction::iterator MBBI(MBB);
3543	++MBBI;
3544
3545	MF->insert(MBBI, LoopBB);
3546	MF->insert(MBBI, RemainderBB);
3547
3548	LoopBB->addSuccessor(LoopBB);
3549	LoopBB->addSuccessor(RemainderBB);
3550
3551	// Move the rest of the block into a new block.
3552	RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB);
3553
3554	if (InstInLoop) {
3555	auto Next = std::next(I);
3556
3557	// Move instruction to loop body.
3558	LoopBB->splice(LoopBB->begin(), &MBB, I, Next);
3559
3560	// Move the rest of the block.
3561	RemainderBB->splice(RemainderBB->begin(), &MBB, Next, MBB.end());
3562	} else {
3563	RemainderBB->splice(RemainderBB->begin(), &MBB, I, MBB.end());
3564	}
3565
3566	MBB.addSuccessor(LoopBB);
3567
3568	return std::pair(LoopBB, RemainderBB);
3569	}
3570
3571	/// Insert \p MI into a BUNDLE with an S_WAITCNT 0 immediately following it.
3572	void SITargetLowering::bundleInstWithWaitcnt(MachineInstr &MI) const {
3573	MachineBasicBlock *MBB = MI.getParent();
3574	const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
3575	auto I = MI.getIterator();
3576	auto E = std::next(I);
3577
3578	BuildMI(*MBB, E, MI.getDebugLoc(), TII->get(AMDGPU::S_WAITCNT))
3579	.addImm(0);
3580
3581	MIBundleBuilder Bundler(*MBB, I, E);
3582	finalizeBundle(*MBB, Bundler.begin());
3583	}
3584
3585	MachineBasicBlock *
3586	SITargetLowering::emitGWSMemViolTestLoop(MachineInstr &MI,
3587	MachineBasicBlock *BB) const {
3588	const DebugLoc &DL = MI.getDebugLoc();
3589
3590	MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
3591
3592	MachineBasicBlock *LoopBB;
3593	MachineBasicBlock *RemainderBB;
3594	const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
3595
3596	// Apparently kill flags are only valid if the def is in the same block?
3597	if (MachineOperand *Src = TII->getNamedOperand(MI, AMDGPU::OpName::data0))
3598	Src->setIsKill(false);
3599
3600	std::tie(LoopBB, RemainderBB) = splitBlockForLoop(MI, *BB, true);
3601
3602	MachineBasicBlock::iterator I = LoopBB->end();
3603
3604	const unsigned EncodedReg = AMDGPU::Hwreg::encodeHwreg(
3605	AMDGPU::Hwreg::ID_TRAPSTS, AMDGPU::Hwreg::OFFSET_MEM_VIOL, 1);
3606
3607	// Clear TRAP_STS.MEM_VIOL
3608	BuildMI(*LoopBB, LoopBB->begin(), DL, TII->get(AMDGPU::S_SETREG_IMM32_B32))
3609	.addImm(0)
3610	.addImm(EncodedReg);
3611
3612	bundleInstWithWaitcnt(MI);
3613
3614	Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3615
3616	// Load and check TRAP_STS.MEM_VIOL
3617	BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::S_GETREG_B32), Reg)
3618	.addImm(EncodedReg);
3619
3620	// FIXME: Do we need to use an isel pseudo that may clobber scc?
3621	BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::S_CMP_LG_U32))
3622	.addReg(Reg, RegState::Kill)
3623	.addImm(0);
3624	BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::S_CBRANCH_SCC1))
3625	.addMBB(LoopBB);
3626
3627	return RemainderBB;
3628	}
3629
3630	// Do a v_movrels_b32 or v_movreld_b32 for each unique value of \p IdxReg in the
3631	// wavefront. If the value is uniform and just happens to be in a VGPR, this
3632	// will only do one iteration. In the worst case, this will loop 64 times.
3633	//
3634	// TODO: Just use v_readlane_b32 if we know the VGPR has a uniform value.
3635	static MachineBasicBlock::iterator
3636	emitLoadM0FromVGPRLoop(const SIInstrInfo *TII, MachineRegisterInfo &MRI,
3637	MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB,
3638	const DebugLoc &DL, const MachineOperand &Idx,
3639	unsigned InitReg, unsigned ResultReg, unsigned PhiReg,
3640	unsigned InitSaveExecReg, int Offset, bool UseGPRIdxMode,
3641	Register &SGPRIdxReg) {
3642
3643	MachineFunction *MF = OrigBB.getParent();
3644	const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
3645	const SIRegisterInfo *TRI = ST.getRegisterInfo();
3646	MachineBasicBlock::iterator I = LoopBB.begin();
3647
3648	const TargetRegisterClass *BoolRC = TRI->getBoolRC();
3649	Register PhiExec = MRI.createVirtualRegister(BoolRC);
3650	Register NewExec = MRI.createVirtualRegister(BoolRC);
3651	Register CurrentIdxReg = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3652	Register CondReg = MRI.createVirtualRegister(BoolRC);
3653
3654	BuildMI(LoopBB, I, DL, TII->get(TargetOpcode::PHI), PhiReg)
3655	.addReg(InitReg)
3656	.addMBB(&OrigBB)
3657	.addReg(ResultReg)
3658	.addMBB(&LoopBB);
3659
3660	BuildMI(LoopBB, I, DL, TII->get(TargetOpcode::PHI), PhiExec)
3661	.addReg(InitSaveExecReg)
3662	.addMBB(&OrigBB)
3663	.addReg(NewExec)
3664	.addMBB(&LoopBB);
3665
3666	// Read the next variant <- also loop target.
3667	BuildMI(LoopBB, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), CurrentIdxReg)
3668	.addReg(Idx.getReg(), getUndefRegState(Idx.isUndef()));
3669
3670	// Compare the just read M0 value to all possible Idx values.
3671	BuildMI(LoopBB, I, DL, TII->get(AMDGPU::V_CMP_EQ_U32_e64), CondReg)
3672	.addReg(CurrentIdxReg)
3673	.addReg(Idx.getReg(), 0, Idx.getSubReg());
3674
3675	// Update EXEC, save the original EXEC value to VCC.
3676	BuildMI(LoopBB, I, DL, TII->get(ST.isWave32() ? AMDGPU::S_AND_SAVEEXEC_B32
3677	: AMDGPU::S_AND_SAVEEXEC_B64),
3678	NewExec)
3679	.addReg(CondReg, RegState::Kill);
3680
3681	MRI.setSimpleHint(NewExec, CondReg);
3682
3683	if (UseGPRIdxMode) {
3684	if (Offset == 0) {
3685	SGPRIdxReg = CurrentIdxReg;
3686	} else {
3687	SGPRIdxReg = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3688	BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_ADD_I32), SGPRIdxReg)
3689	.addReg(CurrentIdxReg, RegState::Kill)
3690	.addImm(Offset);
3691	}
3692	} else {
3693	// Move index from VCC into M0
3694	if (Offset == 0) {
3695	BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_MOV_B32), AMDGPU::M0)
3696	.addReg(CurrentIdxReg, RegState::Kill);
3697	} else {
3698	BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_ADD_I32), AMDGPU::M0)
3699	.addReg(CurrentIdxReg, RegState::Kill)
3700	.addImm(Offset);
3701	}
3702	}
3703
3704	// Update EXEC, switch all done bits to 0 and all todo bits to 1.
3705	unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
3706	MachineInstr *InsertPt =
3707	BuildMI(LoopBB, I, DL, TII->get(ST.isWave32() ? AMDGPU::S_XOR_B32_term
3708	: AMDGPU::S_XOR_B64_term), Exec)
3709	.addReg(Exec)
3710	.addReg(NewExec);
3711
3712	// XXX - s_xor_b64 sets scc to 1 if the result is nonzero, so can we use
3713	// s_cbranch_scc0?
3714
3715	// Loop back to V_READFIRSTLANE_B32 if there are still variants to cover.
3716	BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_CBRANCH_EXECNZ))
3717	.addMBB(&LoopBB);
3718
3719	return InsertPt->getIterator();
3720	}
3721
3722	// This has slightly sub-optimal regalloc when the source vector is killed by
3723	// the read. The register allocator does not understand that the kill is
3724	// per-workitem, so is kept alive for the whole loop so we end up not re-using a
3725	// subregister from it, using 1 more VGPR than necessary. This was saved when
3726	// this was expanded after register allocation.
3727	static MachineBasicBlock::iterator
3728	loadM0FromVGPR(const SIInstrInfo *TII, MachineBasicBlock &MBB, MachineInstr &MI,
3729	unsigned InitResultReg, unsigned PhiReg, int Offset,
3730	bool UseGPRIdxMode, Register &SGPRIdxReg) {
3731	MachineFunction *MF = MBB.getParent();
3732	const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
3733	const SIRegisterInfo *TRI = ST.getRegisterInfo();
3734	MachineRegisterInfo &MRI = MF->getRegInfo();
3735	const DebugLoc &DL = MI.getDebugLoc();
3736	MachineBasicBlock::iterator I(&MI);
3737
3738	const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
3739	Register DstReg = MI.getOperand(0).getReg();
3740	Register SaveExec = MRI.createVirtualRegister(BoolXExecRC);
3741	Register TmpExec = MRI.createVirtualRegister(BoolXExecRC);
3742	unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
3743	unsigned MovExecOpc = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
3744
3745	BuildMI(MBB, I, DL, TII->get(TargetOpcode::IMPLICIT_DEF), TmpExec);
3746
3747	// Save the EXEC mask
3748	BuildMI(MBB, I, DL, TII->get(MovExecOpc), SaveExec)
3749	.addReg(Exec);
3750
3751	MachineBasicBlock *LoopBB;
3752	MachineBasicBlock *RemainderBB;
3753	std::tie(LoopBB, RemainderBB) = splitBlockForLoop(MI, MBB, false);
3754
3755	const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
3756
3757	auto InsPt = emitLoadM0FromVGPRLoop(TII, MRI, MBB, LoopBB, DL, Idx,
3758	InitResultReg, DstReg, PhiReg, TmpExec,
3759	Offset, UseGPRIdxMode, SGPRIdxReg);
3760
3761	MachineBasicBlock* LandingPad = MF->CreateMachineBasicBlock();
3762	MachineFunction::iterator MBBI(LoopBB);
3763	++MBBI;
3764	MF->insert(MBBI, LandingPad);
3765	LoopBB->removeSuccessor(RemainderBB);
3766	LandingPad->addSuccessor(RemainderBB);
3767	LoopBB->addSuccessor(LandingPad);
3768	MachineBasicBlock::iterator First = LandingPad->begin();
3769	BuildMI(*LandingPad, First, DL, TII->get(MovExecOpc), Exec)
3770	.addReg(SaveExec);
3771
3772	return InsPt;
3773	}
3774
3775	// Returns subreg index, offset
3776	static std::pair<unsigned, int>
3777	computeIndirectRegAndOffset(const SIRegisterInfo &TRI,
3778	const TargetRegisterClass *SuperRC,
3779	unsigned VecReg,
3780	int Offset) {
3781	int NumElts = TRI.getRegSizeInBits(*SuperRC) / 32;
3782
3783	// Skip out of bounds offsets, or else we would end up using an undefined
3784	// register.
3785	if (Offset >= NumElts \|\| Offset < 0)
3786	return std::pair(AMDGPU::sub0, Offset);
3787
3788	return std::pair(SIRegisterInfo::getSubRegFromChannel(Offset), 0);
3789	}
3790
3791	static void setM0ToIndexFromSGPR(const SIInstrInfo *TII,
3792	MachineRegisterInfo &MRI, MachineInstr &MI,
3793	int Offset) {
3794	MachineBasicBlock *MBB = MI.getParent();
3795	const DebugLoc &DL = MI.getDebugLoc();
3796	MachineBasicBlock::iterator I(&MI);
3797
3798	const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
3799
3800	assert(Idx->getReg() != AMDGPU::NoRegister)(static_cast <bool> (Idx->getReg() != AMDGPU::NoRegister ) ? void (0) : __assert_fail ("Idx->getReg() != AMDGPU::NoRegister" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 3800, __extension__ __PRETTY_FUNCTION__));
3801
3802	if (Offset == 0) {
3803	BuildMI(MBB, I, DL, TII->get(AMDGPU::S_MOV_B32), AMDGPU::M0).add(Idx);
3804	} else {
3805	BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_ADD_I32), AMDGPU::M0)
3806	.add(*Idx)
3807	.addImm(Offset);
3808	}
3809	}
3810
3811	static Register getIndirectSGPRIdx(const SIInstrInfo *TII,
3812	MachineRegisterInfo &MRI, MachineInstr &MI,
3813	int Offset) {
3814	MachineBasicBlock *MBB = MI.getParent();
3815	const DebugLoc &DL = MI.getDebugLoc();
3816	MachineBasicBlock::iterator I(&MI);
3817
3818	const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
3819
3820	if (Offset == 0)
3821	return Idx->getReg();
3822
3823	Register Tmp = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3824	BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_ADD_I32), Tmp)
3825	.add(*Idx)
3826	.addImm(Offset);
3827	return Tmp;
3828	}
3829
3830	static MachineBasicBlock *emitIndirectSrc(MachineInstr &MI,
3831	MachineBasicBlock &MBB,
3832	const GCNSubtarget &ST) {
3833	const SIInstrInfo *TII = ST.getInstrInfo();
3834	const SIRegisterInfo &TRI = TII->getRegisterInfo();
3835	MachineFunction *MF = MBB.getParent();
3836	MachineRegisterInfo &MRI = MF->getRegInfo();
3837
3838	Register Dst = MI.getOperand(0).getReg();
3839	const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
3840	Register SrcReg = TII->getNamedOperand(MI, AMDGPU::OpName::src)->getReg();
3841	int Offset = TII->getNamedOperand(MI, AMDGPU::OpName::offset)->getImm();
3842
3843	const TargetRegisterClass *VecRC = MRI.getRegClass(SrcReg);
3844	const TargetRegisterClass *IdxRC = MRI.getRegClass(Idx->getReg());
3845
3846	unsigned SubReg;
3847	std::tie(SubReg, Offset)
3848	= computeIndirectRegAndOffset(TRI, VecRC, SrcReg, Offset);
3849
3850	const bool UseGPRIdxMode = ST.useVGPRIndexMode();
3851
3852	// Check for a SGPR index.
3853	if (TII->getRegisterInfo().isSGPRClass(IdxRC)) {
3854	MachineBasicBlock::iterator I(&MI);
3855	const DebugLoc &DL = MI.getDebugLoc();
3856
3857	if (UseGPRIdxMode) {
3858	// TODO: Look at the uses to avoid the copy. This may require rescheduling
3859	// to avoid interfering with other uses, so probably requires a new
3860	// optimization pass.
3861	Register Idx = getIndirectSGPRIdx(TII, MRI, MI, Offset);
3862
3863	const MCInstrDesc &GPRIDXDesc =
3864	TII->getIndirectGPRIDXPseudo(TRI.getRegSizeInBits(*VecRC), true);
3865	BuildMI(MBB, I, DL, GPRIDXDesc, Dst)
3866	.addReg(SrcReg)
3867	.addReg(Idx)
3868	.addImm(SubReg);
3869	} else {
3870	setM0ToIndexFromSGPR(TII, MRI, MI, Offset);
3871
3872	BuildMI(MBB, I, DL, TII->get(AMDGPU::V_MOVRELS_B32_e32), Dst)
3873	.addReg(SrcReg, 0, SubReg)
3874	.addReg(SrcReg, RegState::Implicit);
3875	}
3876
3877	MI.eraseFromParent();
3878
3879	return &MBB;
3880	}
3881
3882	// Control flow needs to be inserted if indexing with a VGPR.
3883	const DebugLoc &DL = MI.getDebugLoc();
3884	MachineBasicBlock::iterator I(&MI);
3885
3886	Register PhiReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
3887	Register InitReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
3888
3889	BuildMI(MBB, I, DL, TII->get(TargetOpcode::IMPLICIT_DEF), InitReg);
3890
3891	Register SGPRIdxReg;
3892	auto InsPt = loadM0FromVGPR(TII, MBB, MI, InitReg, PhiReg, Offset,
3893	UseGPRIdxMode, SGPRIdxReg);
3894
3895	MachineBasicBlock *LoopBB = InsPt->getParent();
3896
3897	if (UseGPRIdxMode) {
3898	const MCInstrDesc &GPRIDXDesc =
3899	TII->getIndirectGPRIDXPseudo(TRI.getRegSizeInBits(*VecRC), true);
3900
3901	BuildMI(*LoopBB, InsPt, DL, GPRIDXDesc, Dst)
3902	.addReg(SrcReg)
3903	.addReg(SGPRIdxReg)
3904	.addImm(SubReg);
3905	} else {
3906	BuildMI(*LoopBB, InsPt, DL, TII->get(AMDGPU::V_MOVRELS_B32_e32), Dst)
3907	.addReg(SrcReg, 0, SubReg)
3908	.addReg(SrcReg, RegState::Implicit);
3909	}
3910
3911	MI.eraseFromParent();
3912
3913	return LoopBB;
3914	}
3915
3916	static MachineBasicBlock *emitIndirectDst(MachineInstr &MI,
3917	MachineBasicBlock &MBB,
3918	const GCNSubtarget &ST) {
3919	const SIInstrInfo *TII = ST.getInstrInfo();
3920	const SIRegisterInfo &TRI = TII->getRegisterInfo();
3921	MachineFunction *MF = MBB.getParent();
3922	MachineRegisterInfo &MRI = MF->getRegInfo();
3923
3924	Register Dst = MI.getOperand(0).getReg();
3925	const MachineOperand *SrcVec = TII->getNamedOperand(MI, AMDGPU::OpName::src);
3926	const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
3927	const MachineOperand *Val = TII->getNamedOperand(MI, AMDGPU::OpName::val);
3928	int Offset = TII->getNamedOperand(MI, AMDGPU::OpName::offset)->getImm();
3929	const TargetRegisterClass *VecRC = MRI.getRegClass(SrcVec->getReg());
3930	const TargetRegisterClass *IdxRC = MRI.getRegClass(Idx->getReg());
3931
3932	// This can be an immediate, but will be folded later.
3933	assert(Val->getReg())(static_cast <bool> (Val->getReg()) ? void (0) : __assert_fail ("Val->getReg()", "llvm/lib/Target/AMDGPU/SIISelLowering.cpp" , 3933, __extension__ __PRETTY_FUNCTION__));
3934
3935	unsigned SubReg;
3936	std::tie(SubReg, Offset) = computeIndirectRegAndOffset(TRI, VecRC,
3937	SrcVec->getReg(),
3938	Offset);
3939	const bool UseGPRIdxMode = ST.useVGPRIndexMode();
3940
3941	if (Idx->getReg() == AMDGPU::NoRegister) {
3942	MachineBasicBlock::iterator I(&MI);
3943	const DebugLoc &DL = MI.getDebugLoc();
3944
3945	assert(Offset == 0)(static_cast <bool> (Offset == 0) ? void (0) : __assert_fail ("Offset == 0", "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 3945, __extension__ __PRETTY_FUNCTION__));
3946
3947	BuildMI(MBB, I, DL, TII->get(TargetOpcode::INSERT_SUBREG), Dst)
3948	.add(*SrcVec)
3949	.add(*Val)
3950	.addImm(SubReg);
3951
3952	MI.eraseFromParent();
3953	return &MBB;
3954	}
3955
3956	// Check for a SGPR index.
3957	if (TII->getRegisterInfo().isSGPRClass(IdxRC)) {
3958	MachineBasicBlock::iterator I(&MI);
3959	const DebugLoc &DL = MI.getDebugLoc();
3960
3961	if (UseGPRIdxMode) {
3962	Register Idx = getIndirectSGPRIdx(TII, MRI, MI, Offset);
3963
3964	const MCInstrDesc &GPRIDXDesc =
3965	TII->getIndirectGPRIDXPseudo(TRI.getRegSizeInBits(*VecRC), false);
3966	BuildMI(MBB, I, DL, GPRIDXDesc, Dst)
3967	.addReg(SrcVec->getReg())
3968	.add(*Val)
3969	.addReg(Idx)
3970	.addImm(SubReg);
3971	} else {
3972	setM0ToIndexFromSGPR(TII, MRI, MI, Offset);
3973
3974	const MCInstrDesc &MovRelDesc = TII->getIndirectRegWriteMovRelPseudo(
3975	TRI.getRegSizeInBits(*VecRC), 32, false);
3976	BuildMI(MBB, I, DL, MovRelDesc, Dst)
3977	.addReg(SrcVec->getReg())
3978	.add(*Val)
3979	.addImm(SubReg);
3980	}
3981	MI.eraseFromParent();
3982	return &MBB;
3983	}
3984
3985	// Control flow needs to be inserted if indexing with a VGPR.
3986	if (Val->isReg())
3987	MRI.clearKillFlags(Val->getReg());
3988
3989	const DebugLoc &DL = MI.getDebugLoc();
3990
3991	Register PhiReg = MRI.createVirtualRegister(VecRC);
3992
3993	Register SGPRIdxReg;
3994	auto InsPt = loadM0FromVGPR(TII, MBB, MI, SrcVec->getReg(), PhiReg, Offset,
3995	UseGPRIdxMode, SGPRIdxReg);
3996	MachineBasicBlock *LoopBB = InsPt->getParent();
3997
3998	if (UseGPRIdxMode) {
3999	const MCInstrDesc &GPRIDXDesc =
4000	TII->getIndirectGPRIDXPseudo(TRI.getRegSizeInBits(*VecRC), false);
4001
4002	BuildMI(*LoopBB, InsPt, DL, GPRIDXDesc, Dst)
4003	.addReg(PhiReg)
4004	.add(*Val)
4005	.addReg(SGPRIdxReg)
4006	.addImm(AMDGPU::sub0);
4007	} else {
4008	const MCInstrDesc &MovRelDesc = TII->getIndirectRegWriteMovRelPseudo(
4009	TRI.getRegSizeInBits(*VecRC), 32, false);
4010	BuildMI(*LoopBB, InsPt, DL, MovRelDesc, Dst)
4011	.addReg(PhiReg)
4012	.add(*Val)
4013	.addImm(AMDGPU::sub0);
4014	}
4015
4016	MI.eraseFromParent();
4017	return LoopBB;
4018	}
4019
4020	MachineBasicBlock *SITargetLowering::EmitInstrWithCustomInserter(
4021	MachineInstr &MI, MachineBasicBlock *BB) const {
4022
4023	const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
4024	MachineFunction *MF = BB->getParent();
4025	SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
4026
4027	switch (MI.getOpcode()) {
4028	case AMDGPU::S_UADDO_PSEUDO:
4029	case AMDGPU::S_USUBO_PSEUDO: {
4030	const DebugLoc &DL = MI.getDebugLoc();
4031	MachineOperand &Dest0 = MI.getOperand(0);
4032	MachineOperand &Dest1 = MI.getOperand(1);
4033	MachineOperand &Src0 = MI.getOperand(2);
4034	MachineOperand &Src1 = MI.getOperand(3);
4035
4036	unsigned Opc = (MI.getOpcode() == AMDGPU::S_UADDO_PSEUDO)
4037	? AMDGPU::S_ADD_I32
4038	: AMDGPU::S_SUB_I32;
4039	BuildMI(*BB, MI, DL, TII->get(Opc), Dest0.getReg()).add(Src0).add(Src1);
4040
4041	BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_CSELECT_B64), Dest1.getReg())
4042	.addImm(1)
4043	.addImm(0);
4044
4045	MI.eraseFromParent();
4046	return BB;
4047	}
4048	case AMDGPU::S_ADD_U64_PSEUDO:
4049	case AMDGPU::S_SUB_U64_PSEUDO: {
4050	MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
4051	const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
4052	const SIRegisterInfo *TRI = ST.getRegisterInfo();
4053	const TargetRegisterClass *BoolRC = TRI->getBoolRC();
4054	const DebugLoc &DL = MI.getDebugLoc();
4055
4056	MachineOperand &Dest = MI.getOperand(0);
4057	MachineOperand &Src0 = MI.getOperand(1);
4058	MachineOperand &Src1 = MI.getOperand(2);
4059
4060	Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
4061	Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
4062
4063	MachineOperand Src0Sub0 = TII->buildExtractSubRegOrImm(
4064	MI, MRI, Src0, BoolRC, AMDGPU::sub0, &AMDGPU::SReg_32RegClass);
4065	MachineOperand Src0Sub1 = TII->buildExtractSubRegOrImm(
4066	MI, MRI, Src0, BoolRC, AMDGPU::sub1, &AMDGPU::SReg_32RegClass);
4067
4068	MachineOperand Src1Sub0 = TII->buildExtractSubRegOrImm(
4069	MI, MRI, Src1, BoolRC, AMDGPU::sub0, &AMDGPU::SReg_32RegClass);
4070	MachineOperand Src1Sub1 = TII->buildExtractSubRegOrImm(
4071	MI, MRI, Src1, BoolRC, AMDGPU::sub1, &AMDGPU::SReg_32RegClass);
4072
4073	bool IsAdd = (MI.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO);
4074
4075	unsigned LoOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32;
4076	unsigned HiOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32;
4077	BuildMI(*BB, MI, DL, TII->get(LoOpc), DestSub0).add(Src0Sub0).add(Src1Sub0);
4078	BuildMI(*BB, MI, DL, TII->get(HiOpc), DestSub1).add(Src0Sub1).add(Src1Sub1);
4079	BuildMI(*BB, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), Dest.getReg())
4080	.addReg(DestSub0)
4081	.addImm(AMDGPU::sub0)
4082	.addReg(DestSub1)
4083	.addImm(AMDGPU::sub1);
4084	MI.eraseFromParent();
4085	return BB;
4086	}
4087	case AMDGPU::V_ADD_U64_PSEUDO:
4088	case AMDGPU::V_SUB_U64_PSEUDO: {
4089	MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
4090	const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
4091	const SIRegisterInfo *TRI = ST.getRegisterInfo();
4092	const DebugLoc &DL = MI.getDebugLoc();
4093
4094	bool IsAdd = (MI.getOpcode() == AMDGPU::V_ADD_U64_PSEUDO);
4095
4096	MachineOperand &Dest = MI.getOperand(0);
4097	MachineOperand &Src0 = MI.getOperand(1);
4098	MachineOperand &Src1 = MI.getOperand(2);
4099
4100	if (IsAdd && ST.hasLshlAddB64()) {
4101	auto Add = BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_LSHL_ADD_U64_e64),
4102	Dest.getReg())
4103	.add(Src0)
4104	.addImm(0)
4105	.add(Src1);
4106	TII->legalizeOperands(*Add);
4107	MI.eraseFromParent();
4108	return BB;
4109	}
4110
4111	const auto *CarryRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
4112
4113	Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4114	Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4115
4116	Register CarryReg = MRI.createVirtualRegister(CarryRC);
4117	Register DeadCarryReg = MRI.createVirtualRegister(CarryRC);
4118
4119	const TargetRegisterClass *Src0RC = Src0.isReg()
4120	? MRI.getRegClass(Src0.getReg())
4121	: &AMDGPU::VReg_64RegClass;
4122	const TargetRegisterClass *Src1RC = Src1.isReg()
4123	? MRI.getRegClass(Src1.getReg())
4124	: &AMDGPU::VReg_64RegClass;
4125
4126	const TargetRegisterClass *Src0SubRC =
4127	TRI->getSubRegisterClass(Src0RC, AMDGPU::sub0);
4128	const TargetRegisterClass *Src1SubRC =
4129	TRI->getSubRegisterClass(Src1RC, AMDGPU::sub1);
4130
4131	MachineOperand SrcReg0Sub0 = TII->buildExtractSubRegOrImm(
4132	MI, MRI, Src0, Src0RC, AMDGPU::sub0, Src0SubRC);
4133	MachineOperand SrcReg1Sub0 = TII->buildExtractSubRegOrImm(
4134	MI, MRI, Src1, Src1RC, AMDGPU::sub0, Src1SubRC);
4135
4136	MachineOperand SrcReg0Sub1 = TII->buildExtractSubRegOrImm(
4137	MI, MRI, Src0, Src0RC, AMDGPU::sub1, Src0SubRC);
4138	MachineOperand SrcReg1Sub1 = TII->buildExtractSubRegOrImm(
4139	MI, MRI, Src1, Src1RC, AMDGPU::sub1, Src1SubRC);
4140
4141	unsigned LoOpc = IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
4142	MachineInstr LoHalf = BuildMI(BB, MI, DL, TII->get(LoOpc), DestSub0)
4143	.addReg(CarryReg, RegState::Define)
4144	.add(SrcReg0Sub0)
4145	.add(SrcReg1Sub0)
4146	.addImm(0); // clamp bit
4147
4148	unsigned HiOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64;
4149	MachineInstr *HiHalf =
4150	BuildMI(*BB, MI, DL, TII->get(HiOpc), DestSub1)
4151	.addReg(DeadCarryReg, RegState::Define \| RegState::Dead)
4152	.add(SrcReg0Sub1)
4153	.add(SrcReg1Sub1)
4154	.addReg(CarryReg, RegState::Kill)
4155	.addImm(0); // clamp bit
4156
4157	BuildMI(*BB, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), Dest.getReg())
4158	.addReg(DestSub0)
4159	.addImm(AMDGPU::sub0)
4160	.addReg(DestSub1)
4161	.addImm(AMDGPU::sub1);
4162	TII->legalizeOperands(*LoHalf);
4163	TII->legalizeOperands(*HiHalf);
4164	MI.eraseFromParent();
4165	return BB;
4166	}
4167	case AMDGPU::S_ADD_CO_PSEUDO:
4168	case AMDGPU::S_SUB_CO_PSEUDO: {
4169	// This pseudo has a chance to be selected
4170	// only from uniform add/subcarry node. All the VGPR operands
4171	// therefore assumed to be splat vectors.
4172	MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
4173	const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
4174	const SIRegisterInfo *TRI = ST.getRegisterInfo();
4175	MachineBasicBlock::iterator MII = MI;
4176	const DebugLoc &DL = MI.getDebugLoc();
4177	MachineOperand &Dest = MI.getOperand(0);
4178	MachineOperand &CarryDest = MI.getOperand(1);
4179	MachineOperand &Src0 = MI.getOperand(2);
4180	MachineOperand &Src1 = MI.getOperand(3);
4181	MachineOperand &Src2 = MI.getOperand(4);
4182	unsigned Opc = (MI.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO)
4183	? AMDGPU::S_ADDC_U32
4184	: AMDGPU::S_SUBB_U32;
4185	if (Src0.isReg() && TRI->isVectorRegister(MRI, Src0.getReg())) {
4186	Register RegOp0 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
4187	BuildMI(*BB, MII, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), RegOp0)
4188	.addReg(Src0.getReg());
4189	Src0.setReg(RegOp0);
4190	}
4191	if (Src1.isReg() && TRI->isVectorRegister(MRI, Src1.getReg())) {
4192	Register RegOp1 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
4193	BuildMI(*BB, MII, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), RegOp1)
4194	.addReg(Src1.getReg());
4195	Src1.setReg(RegOp1);
4196	}
4197	Register RegOp2 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
4198	if (TRI->isVectorRegister(MRI, Src2.getReg())) {
4199	BuildMI(*BB, MII, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), RegOp2)
4200	.addReg(Src2.getReg());
4201	Src2.setReg(RegOp2);
4202	}
4203
4204	const TargetRegisterClass *Src2RC = MRI.getRegClass(Src2.getReg());
4205	unsigned WaveSize = TRI->getRegSizeInBits(*Src2RC);
4206	assert(WaveSize == 64 \|\| WaveSize == 32)(static_cast <bool> (WaveSize == 64 \|\| WaveSize == 32) ? void (0) : __assert_fail ("WaveSize == 64 \|\| WaveSize == 32" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 4206, __extension__ __PRETTY_FUNCTION__));
4207
4208	if (WaveSize == 64) {
4209	if (ST.hasScalarCompareEq64()) {
4210	BuildMI(*BB, MII, DL, TII->get(AMDGPU::S_CMP_LG_U64))
4211	.addReg(Src2.getReg())
4212	.addImm(0);
4213	} else {
4214	const TargetRegisterClass *SubRC =
4215	TRI->getSubRegisterClass(Src2RC, AMDGPU::sub0);
4216	MachineOperand Src2Sub0 = TII->buildExtractSubRegOrImm(
4217	MII, MRI, Src2, Src2RC, AMDGPU::sub0, SubRC);
4218	MachineOperand Src2Sub1 = TII->buildExtractSubRegOrImm(
4219	MII, MRI, Src2, Src2RC, AMDGPU::sub1, SubRC);
4220	Register Src2_32 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
4221
4222	BuildMI(*BB, MII, DL, TII->get(AMDGPU::S_OR_B32), Src2_32)
4223	.add(Src2Sub0)
4224	.add(Src2Sub1);
4225
4226	BuildMI(*BB, MII, DL, TII->get(AMDGPU::S_CMP_LG_U32))
4227	.addReg(Src2_32, RegState::Kill)
4228	.addImm(0);
4229	}
4230	} else {
4231	BuildMI(*BB, MII, DL, TII->get(AMDGPU::S_CMP_LG_U32))
4232	.addReg(Src2.getReg())
4233	.addImm(0);
4234	}
4235
4236	BuildMI(*BB, MII, DL, TII->get(Opc), Dest.getReg()).add(Src0).add(Src1);
4237
4238	unsigned SelOpc =
4239	(WaveSize == 64) ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
4240
4241	BuildMI(*BB, MII, DL, TII->get(SelOpc), CarryDest.getReg())
4242	.addImm(-1)
4243	.addImm(0);
4244
4245	MI.eraseFromParent();
4246	return BB;
4247	}
4248	case AMDGPU::SI_INIT_M0: {
4249	BuildMI(*BB, MI.getIterator(), MI.getDebugLoc(),
4250	TII->get(AMDGPU::S_MOV_B32), AMDGPU::M0)
4251	.add(MI.getOperand(0));
4252	MI.eraseFromParent();
4253	return BB;
4254	}
4255	case AMDGPU::GET_GROUPSTATICSIZE: {
4256	assert(getTargetMachine().getTargetTriple().getOS() == Triple::AMDHSA \|\|(static_cast <bool> (getTargetMachine().getTargetTriple ().getOS() == Triple::AMDHSA \|\| getTargetMachine().getTargetTriple ().getOS() == Triple::AMDPAL) ? void (0) : __assert_fail ("getTargetMachine().getTargetTriple().getOS() == Triple::AMDHSA \|\| getTargetMachine().getTargetTriple().getOS() == Triple::AMDPAL" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 4257, __extension__ __PRETTY_FUNCTION__))
4257	getTargetMachine().getTargetTriple().getOS() == Triple::AMDPAL)(static_cast <bool> (getTargetMachine().getTargetTriple ().getOS() == Triple::AMDHSA \|\| getTargetMachine().getTargetTriple ().getOS() == Triple::AMDPAL) ? void (0) : __assert_fail ("getTargetMachine().getTargetTriple().getOS() == Triple::AMDHSA \|\| getTargetMachine().getTargetTriple().getOS() == Triple::AMDPAL" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 4257, __extension__ __PRETTY_FUNCTION__));
4258	DebugLoc DL = MI.getDebugLoc();
4259	BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_MOV_B32))
4260	.add(MI.getOperand(0))
4261	.addImm(MFI->getLDSSize());
4262	MI.eraseFromParent();
4263	return BB;
4264	}
4265	case AMDGPU::SI_INDIRECT_SRC_V1:
4266	case AMDGPU::SI_INDIRECT_SRC_V2:
4267	case AMDGPU::SI_INDIRECT_SRC_V4:
4268	case AMDGPU::SI_INDIRECT_SRC_V8:
4269	case AMDGPU::SI_INDIRECT_SRC_V9:
4270	case AMDGPU::SI_INDIRECT_SRC_V10:
4271	case AMDGPU::SI_INDIRECT_SRC_V11:
4272	case AMDGPU::SI_INDIRECT_SRC_V12:
4273	case AMDGPU::SI_INDIRECT_SRC_V16:
4274	case AMDGPU::SI_INDIRECT_SRC_V32:
4275	return emitIndirectSrc(MI, BB, getSubtarget());
4276	case AMDGPU::SI_INDIRECT_DST_V1:
4277	case AMDGPU::SI_INDIRECT_DST_V2:
4278	case AMDGPU::SI_INDIRECT_DST_V4:
4279	case AMDGPU::SI_INDIRECT_DST_V8:
4280	case AMDGPU::SI_INDIRECT_DST_V9:
4281	case AMDGPU::SI_INDIRECT_DST_V10:
4282	case AMDGPU::SI_INDIRECT_DST_V11:
4283	case AMDGPU::SI_INDIRECT_DST_V12:
4284	case AMDGPU::SI_INDIRECT_DST_V16:
4285	case AMDGPU::SI_INDIRECT_DST_V32:
4286	return emitIndirectDst(MI, BB, getSubtarget());
4287	case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO:
4288	case AMDGPU::SI_KILL_I1_PSEUDO:
4289	return splitKillBlock(MI, BB);
4290	case AMDGPU::V_CNDMASK_B64_PSEUDO: {
4291	MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
4292	const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
4293	const SIRegisterInfo *TRI = ST.getRegisterInfo();
4294
4295	Register Dst = MI.getOperand(0).getReg();
4296	Register Src0 = MI.getOperand(1).getReg();
4297	Register Src1 = MI.getOperand(2).getReg();
4298	const DebugLoc &DL = MI.getDebugLoc();
4299	Register SrcCond = MI.getOperand(3).getReg();
4300
4301	Register DstLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4302	Register DstHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4303	const auto *CondRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
4304	Register SrcCondCopy = MRI.createVirtualRegister(CondRC);
4305
4306	BuildMI(*BB, MI, DL, TII->get(AMDGPU::COPY), SrcCondCopy)
4307	.addReg(SrcCond);
4308	BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_CNDMASK_B32_e64), DstLo)
4309	.addImm(0)
4310	.addReg(Src0, 0, AMDGPU::sub0)
4311	.addImm(0)
4312	.addReg(Src1, 0, AMDGPU::sub0)
4313	.addReg(SrcCondCopy);
4314	BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_CNDMASK_B32_e64), DstHi)
4315	.addImm(0)
4316	.addReg(Src0, 0, AMDGPU::sub1)
4317	.addImm(0)
4318	.addReg(Src1, 0, AMDGPU::sub1)
4319	.addReg(SrcCondCopy);
4320
4321	BuildMI(*BB, MI, DL, TII->get(AMDGPU::REG_SEQUENCE), Dst)
4322	.addReg(DstLo)
4323	.addImm(AMDGPU::sub0)
4324	.addReg(DstHi)
4325	.addImm(AMDGPU::sub1);
4326	MI.eraseFromParent();
4327	return BB;
4328	}
4329	case AMDGPU::SI_BR_UNDEF: {
4330	const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
4331	const DebugLoc &DL = MI.getDebugLoc();
4332	MachineInstr Br = BuildMI(BB, MI, DL, TII->get(AMDGPU::S_CBRANCH_SCC1))
4333	.add(MI.getOperand(0));
4334	Br->getOperand(1).setIsUndef(); // read undef SCC
4335	MI.eraseFromParent();
4336	return BB;
4337	}
4338	case AMDGPU::ADJCALLSTACKUP:
4339	case AMDGPU::ADJCALLSTACKDOWN: {
4340	const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
4341	MachineInstrBuilder MIB(*MF, &MI);
4342	MIB.addReg(Info->getStackPtrOffsetReg(), RegState::ImplicitDefine)
4343	.addReg(Info->getStackPtrOffsetReg(), RegState::Implicit);
4344	return BB;
4345	}
4346	case AMDGPU::SI_CALL_ISEL: {
4347	const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
4348	const DebugLoc &DL = MI.getDebugLoc();
4349
4350	unsigned ReturnAddrReg = TII->getRegisterInfo().getReturnAddressReg(*MF);
4351
4352	MachineInstrBuilder MIB;
4353	MIB = BuildMI(*BB, MI, DL, TII->get(AMDGPU::SI_CALL), ReturnAddrReg);
4354
4355	for (const MachineOperand &MO : MI.operands())
4356	MIB.add(MO);
4357
4358	MIB.cloneMemRefs(MI);
4359	MI.eraseFromParent();
4360	return BB;
4361	}
4362	case AMDGPU::V_ADD_CO_U32_e32:
4363	case AMDGPU::V_SUB_CO_U32_e32:
4364	case AMDGPU::V_SUBREV_CO_U32_e32: {
4365	// TODO: Define distinct V_*_I32_Pseudo instructions instead.
4366	const DebugLoc &DL = MI.getDebugLoc();
4367	unsigned Opc = MI.getOpcode();
4368
4369	bool NeedClampOperand = false;
4370	if (TII->pseudoToMCOpcode(Opc) == -1) {
4371	Opc = AMDGPU::getVOPe64(Opc);
4372	NeedClampOperand = true;
4373	}
4374
4375	auto I = BuildMI(*BB, MI, DL, TII->get(Opc), MI.getOperand(0).getReg());
4376	if (TII->isVOP3(*I)) {
4377	const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
4378	const SIRegisterInfo *TRI = ST.getRegisterInfo();
4379	I.addReg(TRI->getVCC(), RegState::Define);
4380	}
4381	I.add(MI.getOperand(1))
4382	.add(MI.getOperand(2));
4383	if (NeedClampOperand)
4384	I.addImm(0); // clamp bit for e64 encoding
4385
4386	TII->legalizeOperands(*I);
4387
4388	MI.eraseFromParent();
4389	return BB;
4390	}
4391	case AMDGPU::V_ADDC_U32_e32:
4392	case AMDGPU::V_SUBB_U32_e32:
4393	case AMDGPU::V_SUBBREV_U32_e32:
4394	// These instructions have an implicit use of vcc which counts towards the
4395	// constant bus limit.
4396	TII->legalizeOperands(MI);
4397	return BB;
4398	case AMDGPU::DS_GWS_INIT:
4399	case AMDGPU::DS_GWS_SEMA_BR:
4400	case AMDGPU::DS_GWS_BARRIER:
4401	TII->enforceOperandRCAlignment(MI, AMDGPU::OpName::data0);
4402	[[fallthrough]];
4403	case AMDGPU::DS_GWS_SEMA_V:
4404	case AMDGPU::DS_GWS_SEMA_P:
4405	case AMDGPU::DS_GWS_SEMA_RELEASE_ALL:
4406	// A s_waitcnt 0 is required to be the instruction immediately following.
4407	if (getSubtarget()->hasGWSAutoReplay()) {
4408	bundleInstWithWaitcnt(MI);
4409	return BB;
4410	}
4411
4412	return emitGWSMemViolTestLoop(MI, BB);
4413	case AMDGPU::S_SETREG_B32: {
4414	// Try to optimize cases that only set the denormal mode or rounding mode.
4415	//
4416	// If the s_setreg_b32 fully sets all of the bits in the rounding mode or
4417	// denormal mode to a constant, we can use s_round_mode or s_denorm_mode
4418	// instead.
4419	//
4420	// FIXME: This could be predicates on the immediate, but tablegen doesn't
4421	// allow you to have a no side effect instruction in the output of a
4422	// sideeffecting pattern.
4423	unsigned ID, Offset, Width;
4424	AMDGPU::Hwreg::decodeHwreg(MI.getOperand(1).getImm(), ID, Offset, Width);
4425	if (ID != AMDGPU::Hwreg::ID_MODE)
4426	return BB;
4427
4428	const unsigned WidthMask = maskTrailingOnes<unsigned>(Width);
4429	const unsigned SetMask = WidthMask << Offset;
4430
4431	if (getSubtarget()->hasDenormModeInst()) {
4432	unsigned SetDenormOp = 0;
4433	unsigned SetRoundOp = 0;
4434
4435	// The dedicated instructions can only set the whole denorm or round mode
4436	// at once, not a subset of bits in either.
4437	if (SetMask ==
4438	(AMDGPU::Hwreg::FP_ROUND_MASK \| AMDGPU::Hwreg::FP_DENORM_MASK)) {
4439	// If this fully sets both the round and denorm mode, emit the two
4440	// dedicated instructions for these.
4441	SetRoundOp = AMDGPU::S_ROUND_MODE;
4442	SetDenormOp = AMDGPU::S_DENORM_MODE;
4443	} else if (SetMask == AMDGPU::Hwreg::FP_ROUND_MASK) {
4444	SetRoundOp = AMDGPU::S_ROUND_MODE;
4445	} else if (SetMask == AMDGPU::Hwreg::FP_DENORM_MASK) {
4446	SetDenormOp = AMDGPU::S_DENORM_MODE;
4447	}
4448
4449	if (SetRoundOp \|\| SetDenormOp) {
4450	MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
4451	MachineInstr *Def = MRI.getVRegDef(MI.getOperand(0).getReg());
4452	if (Def && Def->isMoveImmediate() && Def->getOperand(1).isImm()) {
4453	unsigned ImmVal = Def->getOperand(1).getImm();
4454	if (SetRoundOp) {
4455	BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(SetRoundOp))
4456	.addImm(ImmVal & 0xf);
4457
4458	// If we also have the denorm mode, get just the denorm mode bits.
4459	ImmVal >>= 4;
4460	}
4461
4462	if (SetDenormOp) {
4463	BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(SetDenormOp))
4464	.addImm(ImmVal & 0xf);
4465	}
4466
4467	MI.eraseFromParent();
4468	return BB;
4469	}
4470	}
4471	}
4472
4473	// If only FP bits are touched, used the no side effects pseudo.
4474	if ((SetMask & (AMDGPU::Hwreg::FP_ROUND_MASK \|
4475	AMDGPU::Hwreg::FP_DENORM_MASK)) == SetMask)
4476	MI.setDesc(TII->get(AMDGPU::S_SETREG_B32_mode));
4477
4478	return BB;
4479	}
4480	default:
4481	return AMDGPUTargetLowering::EmitInstrWithCustomInserter(MI, BB);
4482	}
4483	}
4484
4485	bool SITargetLowering::hasAtomicFaddRtnForTy(SDValue &Op) const {
4486	switch (Op.getValue(0).getSimpleValueType().SimpleTy) {
4487	case MVT::f32:
4488	return Subtarget->hasAtomicFaddRtnInsts();
4489	case MVT::v2f16:
4490	case MVT::f64:
4491	return Subtarget->hasGFX90AInsts();
4492	default:
4493	return false;
4494	}
4495	}
4496
4497	bool SITargetLowering::enableAggressiveFMAFusion(EVT VT) const {
4498	// This currently forces unfolding various combinations of fsub into fma with
4499	// free fneg'd operands. As long as we have fast FMA (controlled by
4500	// isFMAFasterThanFMulAndFAdd), we should perform these.
4501
4502	// When fma is quarter rate, for f64 where add / sub are at best half rate,
4503	// most of these combines appear to be cycle neutral but save on instruction
4504	// count / code size.
4505	return true;
4506	}
4507
4508	bool SITargetLowering::enableAggressiveFMAFusion(LLT Ty) const { return true; }
4509
4510	EVT SITargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &Ctx,
4511	EVT VT) const {
4512	if (!VT.isVector()) {
4513	return MVT::i1;
4514	}
4515	return EVT::getVectorVT(Ctx, MVT::i1, VT.getVectorNumElements());
4516	}
4517
4518	MVT SITargetLowering::getScalarShiftAmountTy(const DataLayout &, EVT VT) const {
4519	// TODO: Should i16 be used always if legal? For now it would force VALU
4520	// shifts.
4521	return (VT == MVT::i16) ? MVT::i16 : MVT::i32;
4522	}
4523
4524	LLT SITargetLowering::getPreferredShiftAmountTy(LLT Ty) const {
4525	return (Ty.getScalarSizeInBits() <= 16 && Subtarget->has16BitInsts())
4526	? Ty.changeElementSize(16)
4527	: Ty.changeElementSize(32);
4528	}
4529
4530	// Answering this is somewhat tricky and depends on the specific device which
4531	// have different rates for fma or all f64 operations.
4532	//
4533	// v_fma_f64 and v_mul_f64 always take the same number of cycles as each other
4534	// regardless of which device (although the number of cycles differs between
4535	// devices), so it is always profitable for f64.
4536	//
4537	// v_fma_f32 takes 4 or 16 cycles depending on the device, so it is profitable
4538	// only on full rate devices. Normally, we should prefer selecting v_mad_f32
4539	// which we can always do even without fused FP ops since it returns the same
4540	// result as the separate operations and since it is always full
4541	// rate. Therefore, we lie and report that it is not faster for f32. v_mad_f32
4542	// however does not support denormals, so we do report fma as faster if we have
4543	// a fast fma device and require denormals.
4544	//
4545	bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
4546	EVT VT) const {
4547	VT = VT.getScalarType();
4548
4549	switch (VT.getSimpleVT().SimpleTy) {
4550	case MVT::f32: {
4551	// If mad is not available this depends only on if f32 fma is full rate.
4552	if (!Subtarget->hasMadMacF32Insts())
4553	return Subtarget->hasFastFMAF32();
4554
4555	// Otherwise f32 mad is always full rate and returns the same result as
4556	// the separate operations so should be preferred over fma.
4557	// However does not support denormals.
4558	if (hasFP32Denormals(MF))
4559	return Subtarget->hasFastFMAF32() \|\| Subtarget->hasDLInsts();
4560
4561	// If the subtarget has v_fmac_f32, that's just as good as v_mac_f32.
4562	return Subtarget->hasFastFMAF32() && Subtarget->hasDLInsts();
4563	}
4564	case MVT::f64:
4565	return true;
4566	case MVT::f16:
4567	return Subtarget->has16BitInsts() && hasFP64FP16Denormals(MF);
4568	default:
4569	break;
4570	}
4571
4572	return false;
4573	}
4574
4575	bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
4576	LLT Ty) const {
4577	switch (Ty.getScalarSizeInBits()) {
4578	case 16:
4579	return isFMAFasterThanFMulAndFAdd(MF, MVT::f16);
4580	case 32:
4581	return isFMAFasterThanFMulAndFAdd(MF, MVT::f32);
4582	case 64:
4583	return isFMAFasterThanFMulAndFAdd(MF, MVT::f64);
4584	default:
4585	break;
4586	}
4587
4588	return false;
4589	}
4590
4591	bool SITargetLowering::isFMADLegal(const MachineInstr &MI, LLT Ty) const {
4592	if (!Ty.isScalar())
4593	return false;
4594
4595	if (Ty.getScalarSizeInBits() == 16)
4596	return Subtarget->hasMadF16() && !hasFP64FP16Denormals(*MI.getMF());
4597	if (Ty.getScalarSizeInBits() == 32)
4598	return Subtarget->hasMadMacF32Insts() && !hasFP32Denormals(*MI.getMF());
4599
4600	return false;
4601	}
4602
4603	bool SITargetLowering::isFMADLegal(const SelectionDAG &DAG,
4604	const SDNode *N) const {
4605	// TODO: Check future ftz flag
4606	// v_mad_f32/v_mac_f32 do not support denormals.
4607	EVT VT = N->getValueType(0);
4608	if (VT == MVT::f32)
4609	return Subtarget->hasMadMacF32Insts() &&
4610	!hasFP32Denormals(DAG.getMachineFunction());
4611	if (VT == MVT::f16) {
4612	return Subtarget->hasMadF16() &&
4613	!hasFP64FP16Denormals(DAG.getMachineFunction());
4614	}
4615
4616	return false;
4617	}
4618
4619	//===----------------------------------------------------------------------===//
4620	// Custom DAG Lowering Operations
4621	//===----------------------------------------------------------------------===//
4622
4623	// Work around LegalizeDAG doing the wrong thing and fully scalarizing if the
4624	// wider vector type is legal.
4625	SDValue SITargetLowering::splitUnaryVectorOp(SDValue Op,
4626	SelectionDAG &DAG) const {
4627	unsigned Opc = Op.getOpcode();
4628	EVT VT = Op.getValueType();
4629	assert(VT == MVT::v4f16 \|\| VT == MVT::v4i16)(static_cast <bool> (VT == MVT::v4f16 \|\| VT == MVT::v4i16 ) ? void (0) : __assert_fail ("VT == MVT::v4f16 \|\| VT == MVT::v4i16" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 4629, __extension__ __PRETTY_FUNCTION__));
4630
4631	SDValue Lo, Hi;
4632	std::tie(Lo, Hi) = DAG.SplitVectorOperand(Op.getNode(), 0);
4633
4634	SDLoc SL(Op);
4635	SDValue OpLo = DAG.getNode(Opc, SL, Lo.getValueType(), Lo,
4636	Op->getFlags());
4637	SDValue OpHi = DAG.getNode(Opc, SL, Hi.getValueType(), Hi,
4638	Op->getFlags());
4639
4640	return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Op), VT, OpLo, OpHi);
4641	}
4642
4643	// Work around LegalizeDAG doing the wrong thing and fully scalarizing if the
4644	// wider vector type is legal.
4645	SDValue SITargetLowering::splitBinaryVectorOp(SDValue Op,
4646	SelectionDAG &DAG) const {
4647	unsigned Opc = Op.getOpcode();
4648	EVT VT = Op.getValueType();
4649	assert(VT == MVT::v4i16 \|\| VT == MVT::v4f16 \|\| VT == MVT::v4f32 \|\|(static_cast <bool> (VT == MVT::v4i16 \|\| VT == MVT::v4f16 \|\| VT == MVT::v4f32 \|\| VT == MVT::v8i16 \|\| VT == MVT::v8f16 \|\| VT == MVT::v16i16 \|\| VT == MVT::v16f16 \|\| VT == MVT::v8f32 \|\| VT == MVT::v16f32 \|\| VT == MVT::v32f32) ? void (0) : __assert_fail ("VT == MVT::v4i16 \|\| VT == MVT::v4f16 \|\| VT == MVT::v4f32 \|\| VT == MVT::v8i16 \|\| VT == MVT::v8f16 \|\| VT == MVT::v16i16 \|\| VT == MVT::v16f16 \|\| VT == MVT::v8f32 \|\| VT == MVT::v16f32 \|\| VT == MVT::v32f32" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 4652, __extension__ __PRETTY_FUNCTION__))
4650	VT == MVT::v8i16 \|\| VT == MVT::v8f16 \|\| VT == MVT::v16i16 \|\|(static_cast <bool> (VT == MVT::v4i16 \|\| VT == MVT::v4f16 \|\| VT == MVT::v4f32 \|\| VT == MVT::v8i16 \|\| VT == MVT::v8f16 \|\| VT == MVT::v16i16 \|\| VT == MVT::v16f16 \|\| VT == MVT::v8f32 \|\| VT == MVT::v16f32 \|\| VT == MVT::v32f32) ? void (0) : __assert_fail ("VT == MVT::v4i16 \|\| VT == MVT::v4f16 \|\| VT == MVT::v4f32 \|\| VT == MVT::v8i16 \|\| VT == MVT::v8f16 \|\| VT == MVT::v16i16 \|\| VT == MVT::v16f16 \|\| VT == MVT::v8f32 \|\| VT == MVT::v16f32 \|\| VT == MVT::v32f32" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 4652, __extension__ __PRETTY_FUNCTION__))
4651	VT == MVT::v16f16 \|\| VT == MVT::v8f32 \|\| VT == MVT::v16f32 \|\|(static_cast <bool> (VT == MVT::v4i16 \|\| VT == MVT::v4f16 \|\| VT == MVT::v4f32 \|\| VT == MVT::v8i16 \|\| VT == MVT::v8f16 \|\| VT == MVT::v16i16 \|\| VT == MVT::v16f16 \|\| VT == MVT::v8f32 \|\| VT == MVT::v16f32 \|\| VT == MVT::v32f32) ? void (0) : __assert_fail ("VT == MVT::v4i16 \|\| VT == MVT::v4f16 \|\| VT == MVT::v4f32 \|\| VT == MVT::v8i16 \|\| VT == MVT::v8f16 \|\| VT == MVT::v16i16 \|\| VT == MVT::v16f16 \|\| VT == MVT::v8f32 \|\| VT == MVT::v16f32 \|\| VT == MVT::v32f32" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 4652, __extension__ __PRETTY_FUNCTION__))
4652	VT == MVT::v32f32)(static_cast <bool> (VT == MVT::v4i16 \|\| VT == MVT::v4f16 \|\| VT == MVT::v4f32 \|\| VT == MVT::v8i16 \|\| VT == MVT::v8f16 \|\| VT == MVT::v16i16 \|\| VT == MVT::v16f16 \|\| VT == MVT::v8f32 \|\| VT == MVT::v16f32 \|\| VT == MVT::v32f32) ? void (0) : __assert_fail ("VT == MVT::v4i16 \|\| VT == MVT::v4f16 \|\| VT == MVT::v4f32 \|\| VT == MVT::v8i16 \|\| VT == MVT::v8f16 \|\| VT == MVT::v16i16 \|\| VT == MVT::v16f16 \|\| VT == MVT::v8f32 \|\| VT == MVT::v16f32 \|\| VT == MVT::v32f32" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 4652, __extension__ __PRETTY_FUNCTION__));
4653
4654	SDValue Lo0, Hi0;
4655	std::tie(Lo0, Hi0) = DAG.SplitVectorOperand(Op.getNode(), 0);
4656	SDValue Lo1, Hi1;
4657	std::tie(Lo1, Hi1) = DAG.SplitVectorOperand(Op.getNode(), 1);
4658
4659	SDLoc SL(Op);
4660
4661	SDValue OpLo = DAG.getNode(Opc, SL, Lo0.getValueType(), Lo0, Lo1,
4662	Op->getFlags());
4663	SDValue OpHi = DAG.getNode(Opc, SL, Hi0.getValueType(), Hi0, Hi1,
4664	Op->getFlags());
4665
4666	return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Op), VT, OpLo, OpHi);
4667	}
4668
4669	SDValue SITargetLowering::splitTernaryVectorOp(SDValue Op,
4670	SelectionDAG &DAG) const {
4671	unsigned Opc = Op.getOpcode();
4672	EVT VT = Op.getValueType();
4673	assert(VT == MVT::v4i16 \|\| VT == MVT::v4f16 \|\| VT == MVT::v8i16 \|\|(static_cast <bool> (VT == MVT::v4i16 \|\| VT == MVT::v4f16 \|\| VT == MVT::v8i16 \|\| VT == MVT::v8f16 \|\| VT == MVT::v4f32 \|\| VT == MVT::v16i16 \|\| VT == MVT::v16f16 \|\| VT == MVT::v8f32 \|\| VT == MVT::v16f32 \|\| VT == MVT::v32f32) ? void (0) : __assert_fail ("VT == MVT::v4i16 \|\| VT == MVT::v4f16 \|\| VT == MVT::v8i16 \|\| VT == MVT::v8f16 \|\| VT == MVT::v4f32 \|\| VT == MVT::v16i16 \|\| VT == MVT::v16f16 \|\| VT == MVT::v8f32 \|\| VT == MVT::v16f32 \|\| VT == MVT::v32f32" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 4676, __extension__ __PRETTY_FUNCTION__))
4674	VT == MVT::v8f16 \|\| VT == MVT::v4f32 \|\| VT == MVT::v16i16 \|\|(static_cast <bool> (VT == MVT::v4i16 \|\| VT == MVT::v4f16 \|\| VT == MVT::v8i16 \|\| VT == MVT::v8f16 \|\| VT == MVT::v4f32 \|\| VT == MVT::v16i16 \|\| VT == MVT::v16f16 \|\| VT == MVT::v8f32 \|\| VT == MVT::v16f32 \|\| VT == MVT::v32f32) ? void (0) : __assert_fail ("VT == MVT::v4i16 \|\| VT == MVT::v4f16 \|\| VT == MVT::v8i16 \|\| VT == MVT::v8f16 \|\| VT == MVT::v4f32 \|\| VT == MVT::v16i16 \|\| VT == MVT::v16f16 \|\| VT == MVT::v8f32 \|\| VT == MVT::v16f32 \|\| VT == MVT::v32f32" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 4676, __extension__ __PRETTY_FUNCTION__))
4675	VT == MVT::v16f16 \|\| VT == MVT::v8f32 \|\| VT == MVT::v16f32 \|\|(static_cast <bool> (VT == MVT::v4i16 \|\| VT == MVT::v4f16 \|\| VT == MVT::v8i16 \|\| VT == MVT::v8f16 \|\| VT == MVT::v4f32 \|\| VT == MVT::v16i16 \|\| VT == MVT::v16f16 \|\| VT == MVT::v8f32 \|\| VT == MVT::v16f32 \|\| VT == MVT::v32f32) ? void (0) : __assert_fail ("VT == MVT::v4i16 \|\| VT == MVT::v4f16 \|\| VT == MVT::v8i16 \|\| VT == MVT::v8f16 \|\| VT == MVT::v4f32 \|\| VT == MVT::v16i16 \|\| VT == MVT::v16f16 \|\| VT == MVT::v8f32 \|\| VT == MVT::v16f32 \|\| VT == MVT::v32f32" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 4676, __extension__ __PRETTY_FUNCTION__))
4676	VT == MVT::v32f32)(static_cast <bool> (VT == MVT::v4i16 \|\| VT == MVT::v4f16 \|\| VT == MVT::v8i16 \|\| VT == MVT::v8f16 \|\| VT == MVT::v4f32 \|\| VT == MVT::v16i16 \|\| VT == MVT::v16f16 \|\| VT == MVT::v8f32 \|\| VT == MVT::v16f32 \|\| VT == MVT::v32f32) ? void (0) : __assert_fail ("VT == MVT::v4i16 \|\| VT == MVT::v4f16 \|\| VT == MVT::v8i16 \|\| VT == MVT::v8f16 \|\| VT == MVT::v4f32 \|\| VT == MVT::v16i16 \|\| VT == MVT::v16f16 \|\| VT == MVT::v8f32 \|\| VT == MVT::v16f32 \|\| VT == MVT::v32f32" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 4676, __extension__ __PRETTY_FUNCTION__));
4677
4678	SDValue Lo0, Hi0;
4679	SDValue Op0 = Op.getOperand(0);
4680	std::tie(Lo0, Hi0) = Op0.getValueType().isVector()
4681	? DAG.SplitVectorOperand(Op.getNode(), 0)
4682	: std::pair(Op0, Op0);
4683	SDValue Lo1, Hi1;
4684	std::tie(Lo1, Hi1) = DAG.SplitVectorOperand(Op.getNode(), 1);
4685	SDValue Lo2, Hi2;
4686	std::tie(Lo2, Hi2) = DAG.SplitVectorOperand(Op.getNode(), 2);
4687
4688	SDLoc SL(Op);
4689	auto ResVT = DAG.GetSplitDestVTs(VT);
4690
4691	SDValue OpLo = DAG.getNode(Opc, SL, ResVT.first, Lo0, Lo1, Lo2,
4692	Op->getFlags());
4693	SDValue OpHi = DAG.getNode(Opc, SL, ResVT.second, Hi0, Hi1, Hi2,
4694	Op->getFlags());
4695
4696	return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Op), VT, OpLo, OpHi);
4697	}
4698
4699
4700	SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
4701	switch (Op.getOpcode()) {
4702	default: return AMDGPUTargetLowering::LowerOperation(Op, DAG);
4703	case ISD::BRCOND: return LowerBRCOND(Op, DAG);
4704	case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
4705	case ISD::LOAD: {
4706	SDValue Result = LowerLOAD(Op, DAG);
4707	assert((!Result.getNode() \|\|(static_cast <bool> ((!Result.getNode() \|\| Result.getNode ()->getNumValues() == 2) && "Load should return a value and a chain" ) ? void (0) : __assert_fail ("(!Result.getNode() \|\| Result.getNode()->getNumValues() == 2) && \"Load should return a value and a chain\"" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 4709, __extension__ __PRETTY_FUNCTION__))
4708	Result.getNode()->getNumValues() == 2) &&(static_cast <bool> ((!Result.getNode() \|\| Result.getNode ()->getNumValues() == 2) && "Load should return a value and a chain" ) ? void (0) : __assert_fail ("(!Result.getNode() \|\| Result.getNode()->getNumValues() == 2) && \"Load should return a value and a chain\"" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 4709, __extension__ __PRETTY_FUNCTION__))
4709	"Load should return a value and a chain")(static_cast <bool> ((!Result.getNode() \|\| Result.getNode ()->getNumValues() == 2) && "Load should return a value and a chain" ) ? void (0) : __assert_fail ("(!Result.getNode() \|\| Result.getNode()->getNumValues() == 2) && \"Load should return a value and a chain\"" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 4709, __extension__ __PRETTY_FUNCTION__));
4710	return Result;
4711	}
4712
4713	case ISD::FSIN:
4714	case ISD::FCOS:
4715	return LowerTrig(Op, DAG);
4716	case ISD::SELECT: return LowerSELECT(Op, DAG);
4717	case ISD::FDIV: return LowerFDIV(Op, DAG);
4718	case ISD::ATOMIC_CMP_SWAP: return LowerATOMIC_CMP_SWAP(Op, DAG);
4719	case ISD::STORE: return LowerSTORE(Op, DAG);
4720	case ISD::GlobalAddress: {
4721	MachineFunction &MF = DAG.getMachineFunction();
4722	SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
4723	return LowerGlobalAddress(MFI, Op, DAG);
4724	}
4725	case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
4726	case ISD::INTRINSIC_W_CHAIN: return LowerINTRINSIC_W_CHAIN(Op, DAG);
4727	case ISD::INTRINSIC_VOID: return LowerINTRINSIC_VOID(Op, DAG);
4728	case ISD::ADDRSPACECAST: return lowerADDRSPACECAST(Op, DAG);
4729	case ISD::INSERT_SUBVECTOR:
4730	return lowerINSERT_SUBVECTOR(Op, DAG);
4731	case ISD::INSERT_VECTOR_ELT:
4732	return lowerINSERT_VECTOR_ELT(Op, DAG);
4733	case ISD::EXTRACT_VECTOR_ELT:
4734	return lowerEXTRACT_VECTOR_ELT(Op, DAG);
4735	case ISD::VECTOR_SHUFFLE:
4736	return lowerVECTOR_SHUFFLE(Op, DAG);
4737	case ISD::SCALAR_TO_VECTOR:
4738	return lowerSCALAR_TO_VECTOR(Op, DAG);
4739	case ISD::BUILD_VECTOR:
4740	return lowerBUILD_VECTOR(Op, DAG);
4741	case ISD::FP_ROUND:
4742	return lowerFP_ROUND(Op, DAG);
4743	case ISD::FPTRUNC_ROUND: {
4744	unsigned Opc;
4745	SDLoc DL(Op);
4746
4747	if (Op.getOperand(0)->getValueType(0) != MVT::f32)
4748	return SDValue();
4749
4750	// Get the rounding mode from the last operand
4751	int RoundMode = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
4752	if (RoundMode == (int)RoundingMode::TowardPositive)
4753	Opc = AMDGPUISD::FPTRUNC_ROUND_UPWARD;
4754	else if (RoundMode == (int)RoundingMode::TowardNegative)
4755	Opc = AMDGPUISD::FPTRUNC_ROUND_DOWNWARD;
4756	else
4757	return SDValue();
4758
4759	return DAG.getNode(Opc, DL, Op.getNode()->getVTList(), Op->getOperand(0));
4760	}
4761	case ISD::TRAP:
4762	return lowerTRAP(Op, DAG);
4763	case ISD::DEBUGTRAP:
4764	return lowerDEBUGTRAP(Op, DAG);
4765	case ISD::FABS:
4766	case ISD::FNEG:
4767	case ISD::FCANONICALIZE:
4768	case ISD::BSWAP:
4769	return splitUnaryVectorOp(Op, DAG);
4770	case ISD::FMINNUM:
4771	case ISD::FMAXNUM:
4772	return lowerFMINNUM_FMAXNUM(Op, DAG);
4773	case ISD::FMA:
4774	return splitTernaryVectorOp(Op, DAG);
4775	case ISD::FP_TO_SINT:
4776	case ISD::FP_TO_UINT:
4777	return LowerFP_TO_INT(Op, DAG);
4778	case ISD::SHL:
4779	case ISD::SRA:
4780	case ISD::SRL:
4781	case ISD::ADD:
4782	case ISD::SUB:
4783	case ISD::MUL:
4784	case ISD::SMIN:
4785	case ISD::SMAX:
4786	case ISD::UMIN:
4787	case ISD::UMAX:
4788	case ISD::FADD:
4789	case ISD::FMUL:
4790	case ISD::FMINNUM_IEEE:
4791	case ISD::FMAXNUM_IEEE:
4792	case ISD::UADDSAT:
4793	case ISD::USUBSAT:
4794	case ISD::SADDSAT:
4795	case ISD::SSUBSAT:
4796	return splitBinaryVectorOp(Op, DAG);
4797	case ISD::SMULO:
4798	case ISD::UMULO:
4799	return lowerXMULO(Op, DAG);
4800	case ISD::SMUL_LOHI:
4801	case ISD::UMUL_LOHI:
4802	return lowerXMUL_LOHI(Op, DAG);
4803	case ISD::DYNAMIC_STACKALLOC:
4804	return LowerDYNAMIC_STACKALLOC(Op, DAG);
4805	}
4806	return SDValue();
4807	}
4808
4809	// Used for D16: Casts the result of an instruction into the right vector,
4810	// packs values if loads return unpacked values.
4811	static SDValue adjustLoadValueTypeImpl(SDValue Result, EVT LoadVT,
4812	const SDLoc &DL,
4813	SelectionDAG &DAG, bool Unpacked) {
4814	if (!LoadVT.isVector())
4815	return Result;
4816
4817	// Cast back to the original packed type or to a larger type that is a
4818	// multiple of 32 bit for D16. Widening the return type is a required for
4819	// legalization.
4820	EVT FittingLoadVT = LoadVT;
4821	if ((LoadVT.getVectorNumElements() % 2) == 1) {
4822	FittingLoadVT =
4823	EVT::getVectorVT(*DAG.getContext(), LoadVT.getVectorElementType(),
4824	LoadVT.getVectorNumElements() + 1);
4825	}
4826
4827	if (Unpacked) { // From v2i32/v4i32 back to v2f16/v4f16.
4828	// Truncate to v2i16/v4i16.
4829	EVT IntLoadVT = FittingLoadVT.changeTypeToInteger();
4830
4831	// Workaround legalizer not scalarizing truncate after vector op
4832	// legalization but not creating intermediate vector trunc.
4833	SmallVector<SDValue, 4> Elts;
4834	DAG.ExtractVectorElements(Result, Elts);
4835	for (SDValue &Elt : Elts)
4836	Elt = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Elt);
4837
4838	// Pad illegal v1i16/v3fi6 to v4i16
4839	if ((LoadVT.getVectorNumElements() % 2) == 1)
4840	Elts.push_back(DAG.getUNDEF(MVT::i16));
4841
4842	Result = DAG.getBuildVector(IntLoadVT, DL, Elts);
4843
4844	// Bitcast to original type (v2f16/v4f16).
4845	return DAG.getNode(ISD::BITCAST, DL, FittingLoadVT, Result);
4846	}
4847
4848	// Cast back to the original packed type.
4849	return DAG.getNode(ISD::BITCAST, DL, FittingLoadVT, Result);
4850	}
4851
4852	SDValue SITargetLowering::adjustLoadValueType(unsigned Opcode,
4853	MemSDNode *M,
4854	SelectionDAG &DAG,
4855	ArrayRef<SDValue> Ops,
4856	bool IsIntrinsic) const {
4857	SDLoc DL(M);
4858
4859	bool Unpacked = Subtarget->hasUnpackedD16VMem();
4860	EVT LoadVT = M->getValueType(0);
4861
4862	EVT EquivLoadVT = LoadVT;
4863	if (LoadVT.isVector()) {
4864	if (Unpacked) {
4865	EquivLoadVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32,
4866	LoadVT.getVectorNumElements());
4867	} else if ((LoadVT.getVectorNumElements() % 2) == 1) {
4868	// Widen v3f16 to legal type
4869	EquivLoadVT =
4870	EVT::getVectorVT(*DAG.getContext(), LoadVT.getVectorElementType(),
4871	LoadVT.getVectorNumElements() + 1);
4872	}
4873	}
4874
4875	// Change from v4f16/v2f16 to EquivLoadVT.
4876	SDVTList VTList = DAG.getVTList(EquivLoadVT, MVT::Other);
4877
4878	SDValue Load
4879	= DAG.getMemIntrinsicNode(
4880	IsIntrinsic ? (unsigned)ISD::INTRINSIC_W_CHAIN : Opcode, DL,
4881	VTList, Ops, M->getMemoryVT(),
4882	M->getMemOperand());
4883
4884	SDValue Adjusted = adjustLoadValueTypeImpl(Load, LoadVT, DL, DAG, Unpacked);
4885
4886	return DAG.getMergeValues({ Adjusted, Load.getValue(1) }, DL);
4887	}
4888
4889	SDValue SITargetLowering::lowerIntrinsicLoad(MemSDNode *M, bool IsFormat,
4890	SelectionDAG &DAG,
4891	ArrayRef<SDValue> Ops) const {
4892	SDLoc DL(M);
4893	EVT LoadVT = M->getValueType(0);
4894	EVT EltType = LoadVT.getScalarType();
4895	EVT IntVT = LoadVT.changeTypeToInteger();
4896
4897	bool IsD16 = IsFormat && (EltType.getSizeInBits() == 16);
4898
4899	assert(M->getNumValues() == 2 \|\| M->getNumValues() == 3)(static_cast <bool> (M->getNumValues() == 2 \|\| M-> getNumValues() == 3) ? void (0) : __assert_fail ("M->getNumValues() == 2 \|\| M->getNumValues() == 3" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 4899, __extension__ __PRETTY_FUNCTION__));
4900	bool IsTFE = M->getNumValues() == 3;
4901
4902	unsigned Opc;
4903	if (IsFormat) {
4904	Opc = IsTFE ? AMDGPUISD::BUFFER_LOAD_FORMAT_TFE
4905	: AMDGPUISD::BUFFER_LOAD_FORMAT;
4906	} else {
4907	// TODO: Support non-format TFE loads.
4908	if (IsTFE)
4909	return SDValue();
4910	Opc = AMDGPUISD::BUFFER_LOAD;
4911	}
4912
4913	if (IsD16) {
4914	return adjustLoadValueType(AMDGPUISD::BUFFER_LOAD_FORMAT_D16, M, DAG, Ops);
4915	}
4916
4917	// Handle BUFFER_LOAD_BYTE/UBYTE/SHORT/USHORT overloaded intrinsics
4918	if (!IsD16 && !LoadVT.isVector() && EltType.getSizeInBits() < 32)
4919	return handleByteShortBufferLoads(DAG, LoadVT, DL, Ops, M);
4920
4921	if (isTypeLegal(LoadVT)) {
4922	return getMemIntrinsicNode(Opc, DL, M->getVTList(), Ops, IntVT,
4923	M->getMemOperand(), DAG);
4924	}
4925
4926	EVT CastVT = getEquivalentMemType(*DAG.getContext(), LoadVT);
4927	SDVTList VTList = DAG.getVTList(CastVT, MVT::Other);
4928	SDValue MemNode = getMemIntrinsicNode(Opc, DL, VTList, Ops, CastVT,
4929	M->getMemOperand(), DAG);
4930	return DAG.getMergeValues(
4931	{DAG.getNode(ISD::BITCAST, DL, LoadVT, MemNode), MemNode.getValue(1)},
4932	DL);
4933	}
4934
4935	static SDValue lowerICMPIntrinsic(const SITargetLowering &TLI,
4936	SDNode *N, SelectionDAG &DAG) {
4937	EVT VT = N->getValueType(0);
4938	const auto *CD = cast<ConstantSDNode>(N->getOperand(3));
4939	unsigned CondCode = CD->getZExtValue();
4940	if (!ICmpInst::isIntPredicate(static_cast<ICmpInst::Predicate>(CondCode)))
4941	return DAG.getUNDEF(VT);
4942
4943	ICmpInst::Predicate IcInput = static_cast<ICmpInst::Predicate>(CondCode);
4944
4945	SDValue LHS = N->getOperand(1);
4946	SDValue RHS = N->getOperand(2);
4947
4948	SDLoc DL(N);
4949
4950	EVT CmpVT = LHS.getValueType();
4951	if (CmpVT == MVT::i16 && !TLI.isTypeLegal(MVT::i16)) {
4952	unsigned PromoteOp = ICmpInst::isSigned(IcInput) ?
4953	ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
4954	LHS = DAG.getNode(PromoteOp, DL, MVT::i32, LHS);
4955	RHS = DAG.getNode(PromoteOp, DL, MVT::i32, RHS);
4956	}
4957
4958	ISD::CondCode CCOpcode = getICmpCondCode(IcInput);
4959
4960	unsigned WavefrontSize = TLI.getSubtarget()->getWavefrontSize();
4961	EVT CCVT = EVT::getIntegerVT(*DAG.getContext(), WavefrontSize);
4962
4963	SDValue SetCC = DAG.getNode(AMDGPUISD::SETCC, DL, CCVT, LHS, RHS,
4964	DAG.getCondCode(CCOpcode));
4965	if (VT.bitsEq(CCVT))
4966	return SetCC;
4967	return DAG.getZExtOrTrunc(SetCC, DL, VT);
4968	}
4969
4970	static SDValue lowerFCMPIntrinsic(const SITargetLowering &TLI,
4971	SDNode *N, SelectionDAG &DAG) {
4972	EVT VT = N->getValueType(0);
4973	const auto *CD = cast<ConstantSDNode>(N->getOperand(3));
4974
4975	unsigned CondCode = CD->getZExtValue();
4976	if (!FCmpInst::isFPPredicate(static_cast<FCmpInst::Predicate>(CondCode)))
4977	return DAG.getUNDEF(VT);
4978
4979	SDValue Src0 = N->getOperand(1);
4980	SDValue Src1 = N->getOperand(2);
4981	EVT CmpVT = Src0.getValueType();
4982	SDLoc SL(N);
4983
4984	if (CmpVT == MVT::f16 && !TLI.isTypeLegal(CmpVT)) {
4985	Src0 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src0);
4986	Src1 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src1);
4987	}
4988
4989	FCmpInst::Predicate IcInput = static_cast<FCmpInst::Predicate>(CondCode);
4990	ISD::CondCode CCOpcode = getFCmpCondCode(IcInput);
4991	unsigned WavefrontSize = TLI.getSubtarget()->getWavefrontSize();
4992	EVT CCVT = EVT::getIntegerVT(*DAG.getContext(), WavefrontSize);
4993	SDValue SetCC = DAG.getNode(AMDGPUISD::SETCC, SL, CCVT, Src0,
4994	Src1, DAG.getCondCode(CCOpcode));
4995	if (VT.bitsEq(CCVT))
4996	return SetCC;
4997	return DAG.getZExtOrTrunc(SetCC, SL, VT);
4998	}
4999
5000	static SDValue lowerBALLOTIntrinsic(const SITargetLowering &TLI, SDNode *N,
5001	SelectionDAG &DAG) {
5002	EVT VT = N->getValueType(0);
5003	SDValue Src = N->getOperand(1);
5004	SDLoc SL(N);
5005
5006	if (Src.getOpcode() == ISD::SETCC) {
5007	// (ballot (ISD::SETCC ...)) -> (AMDGPUISD::SETCC ...)
5008	return DAG.getNode(AMDGPUISD::SETCC, SL, VT, Src.getOperand(0),
5009	Src.getOperand(1), Src.getOperand(2));
5010	}
5011	if (const ConstantSDNode *Arg = dyn_cast<ConstantSDNode>(Src)) {
5012	// (ballot 0) -> 0
5013	if (Arg->isZero())
5014	return DAG.getConstant(0, SL, VT);
5015
5016	// (ballot 1) -> EXEC/EXEC_LO
5017	if (Arg->isOne()) {
5018	Register Exec;
5019	if (VT.getScalarSizeInBits() == 32)
5020	Exec = AMDGPU::EXEC_LO;
5021	else if (VT.getScalarSizeInBits() == 64)
5022	Exec = AMDGPU::EXEC;
5023	else
5024	return SDValue();
5025
5026	return DAG.getCopyFromReg(DAG.getEntryNode(), SL, Exec, VT);
5027	}
5028	}
5029
5030	// (ballot (i1 $src)) -> (AMDGPUISD::SETCC (i32 (zext $src)) (i32 0)
5031	// ISD::SETNE)
5032	return DAG.getNode(
5033	AMDGPUISD::SETCC, SL, VT, DAG.getZExtOrTrunc(Src, SL, MVT::i32),
5034	DAG.getConstant(0, SL, MVT::i32), DAG.getCondCode(ISD::SETNE));
5035	}
5036
5037	void SITargetLowering::ReplaceNodeResults(SDNode *N,
5038	SmallVectorImpl<SDValue> &Results,
5039	SelectionDAG &DAG) const {
5040	switch (N->getOpcode()) {
5041	case ISD::INSERT_VECTOR_ELT: {
5042	if (SDValue Res = lowerINSERT_VECTOR_ELT(SDValue(N, 0), DAG))
5043	Results.push_back(Res);
5044	return;
5045	}
5046	case ISD::EXTRACT_VECTOR_ELT: {
5047	if (SDValue Res = lowerEXTRACT_VECTOR_ELT(SDValue(N, 0), DAG))
5048	Results.push_back(Res);
5049	return;
5050	}
5051	case ISD::INTRINSIC_WO_CHAIN: {
5052	unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
5053	switch (IID) {
5054	case Intrinsic::amdgcn_cvt_pkrtz: {
5055	SDValue Src0 = N->getOperand(1);
5056	SDValue Src1 = N->getOperand(2);
5057	SDLoc SL(N);
5058	SDValue Cvt = DAG.getNode(AMDGPUISD::CVT_PKRTZ_F16_F32, SL, MVT::i32,
5059	Src0, Src1);
5060	Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Cvt));
5061	return;
5062	}
5063	case Intrinsic::amdgcn_cvt_pknorm_i16:
5064	case Intrinsic::amdgcn_cvt_pknorm_u16:
5065	case Intrinsic::amdgcn_cvt_pk_i16:
5066	case Intrinsic::amdgcn_cvt_pk_u16: {
5067	SDValue Src0 = N->getOperand(1);
5068	SDValue Src1 = N->getOperand(2);
5069	SDLoc SL(N);
5070	unsigned Opcode;
5071
5072	if (IID == Intrinsic::amdgcn_cvt_pknorm_i16)
5073	Opcode = AMDGPUISD::CVT_PKNORM_I16_F32;
5074	else if (IID == Intrinsic::amdgcn_cvt_pknorm_u16)
5075	Opcode = AMDGPUISD::CVT_PKNORM_U16_F32;
5076	else if (IID == Intrinsic::amdgcn_cvt_pk_i16)
5077	Opcode = AMDGPUISD::CVT_PK_I16_I32;
5078	else
5079	Opcode = AMDGPUISD::CVT_PK_U16_U32;
5080
5081	EVT VT = N->getValueType(0);
5082	if (isTypeLegal(VT))
5083	Results.push_back(DAG.getNode(Opcode, SL, VT, Src0, Src1));
5084	else {
5085	SDValue Cvt = DAG.getNode(Opcode, SL, MVT::i32, Src0, Src1);
5086	Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, Cvt));
5087	}
5088	return;
5089	}
5090	}
5091	break;
5092	}
5093	case ISD::INTRINSIC_W_CHAIN: {
5094	if (SDValue Res = LowerINTRINSIC_W_CHAIN(SDValue(N, 0), DAG)) {
5095	if (Res.getOpcode() == ISD::MERGE_VALUES) {
5096	// FIXME: Hacky
5097	for (unsigned I = 0; I < Res.getNumOperands(); I++) {
5098	Results.push_back(Res.getOperand(I));
5099	}
5100	} else {
5101	Results.push_back(Res);
5102	Results.push_back(Res.getValue(1));
5103	}
5104	return;
5105	}
5106
5107	break;
5108	}
5109	case ISD::SELECT: {
5110	SDLoc SL(N);
5111	EVT VT = N->getValueType(0);
5112	EVT NewVT = getEquivalentMemType(*DAG.getContext(), VT);
5113	SDValue LHS = DAG.getNode(ISD::BITCAST, SL, NewVT, N->getOperand(1));
5114	SDValue RHS = DAG.getNode(ISD::BITCAST, SL, NewVT, N->getOperand(2));
5115
5116	EVT SelectVT = NewVT;
5117	if (NewVT.bitsLT(MVT::i32)) {
5118	LHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, LHS);
5119	RHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, RHS);
5120	SelectVT = MVT::i32;
5121	}
5122
5123	SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, SelectVT,
5124	N->getOperand(0), LHS, RHS);
5125
5126	if (NewVT != SelectVT)
5127	NewSelect = DAG.getNode(ISD::TRUNCATE, SL, NewVT, NewSelect);
5128	Results.push_back(DAG.getNode(ISD::BITCAST, SL, VT, NewSelect));
5129	return;
5130	}
5131	case ISD::FNEG: {
5132	if (N->getValueType(0) != MVT::v2f16)
5133	break;
5134
5135	SDLoc SL(N);
5136	SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0));
5137
5138	SDValue Op = DAG.getNode(ISD::XOR, SL, MVT::i32,
5139	BC,
5140	DAG.getConstant(0x80008000, SL, MVT::i32));
5141	Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Op));
5142	return;
5143	}
5144	case ISD::FABS: {
5145	if (N->getValueType(0) != MVT::v2f16)
5146	break;
5147
5148	SDLoc SL(N);
5149	SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0));
5150
5151	SDValue Op = DAG.getNode(ISD::AND, SL, MVT::i32,
5152	BC,
5153	DAG.getConstant(0x7fff7fff, SL, MVT::i32));
5154	Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Op));
5155	return;
5156	}
5157	default:
5158	break;
5159	}
5160	}
5161
5162	/// Helper function for LowerBRCOND
5163	static SDNode *findUser(SDValue Value, unsigned Opcode) {
5164
5165	SDNode *Parent = Value.getNode();
5166	for (SDNode::use_iterator I = Parent->use_begin(), E = Parent->use_end();
5167	I != E; ++I) {
5168
5169	if (I.getUse().get() != Value)
5170	continue;
5171
5172	if (I->getOpcode() == Opcode)
5173	return *I;
5174	}
5175	return nullptr;
5176	}
5177
5178	unsigned SITargetLowering::isCFIntrinsic(const SDNode *Intr) const {
5179	if (Intr->getOpcode() == ISD::INTRINSIC_W_CHAIN) {
5180	switch (cast<ConstantSDNode>(Intr->getOperand(1))->getZExtValue()) {
5181	case Intrinsic::amdgcn_if:
5182	return AMDGPUISD::IF;
5183	case Intrinsic::amdgcn_else:
5184	return AMDGPUISD::ELSE;
5185	case Intrinsic::amdgcn_loop:
5186	return AMDGPUISD::LOOP;
5187	case Intrinsic::amdgcn_end_cf:
5188	llvm_unreachable("should not occur")::llvm::llvm_unreachable_internal("should not occur", "llvm/lib/Target/AMDGPU/SIISelLowering.cpp" , 5188);
5189	default:
5190	return 0;
5191	}
5192	}
5193
5194	// break, if_break, else_break are all only used as inputs to loop, not
5195	// directly as branch conditions.
5196	return 0;
5197	}
5198
5199	bool SITargetLowering::shouldEmitFixup(const GlobalValue *GV) const {
5200	const Triple &TT = getTargetMachine().getTargetTriple();
5201	return (GV->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS \|\|
5202	GV->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) &&
5203	AMDGPU::shouldEmitConstantsToTextSection(TT);
5204	}
5205
5206	bool SITargetLowering::shouldEmitGOTReloc(const GlobalValue *GV) const {
5207	// FIXME: Either avoid relying on address space here or change the default
5208	// address space for functions to avoid the explicit check.
5209	return (GV->getValueType()->isFunctionTy() \|\|
5210	!isNonGlobalAddrSpace(GV->getAddressSpace())) &&
5211	!shouldEmitFixup(GV) &&
5212	!getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV);
5213	}
5214
5215	bool SITargetLowering::shouldEmitPCReloc(const GlobalValue *GV) const {
5216	return !shouldEmitFixup(GV) && !shouldEmitGOTReloc(GV);
5217	}
5218
5219	bool SITargetLowering::shouldUseLDSConstAddress(const GlobalValue *GV) const {
5220	if (!GV->hasExternalLinkage())
5221	return true;
5222
5223	const auto OS = getTargetMachine().getTargetTriple().getOS();
5224	return OS == Triple::AMDHSA \|\| OS == Triple::AMDPAL;
5225	}
5226
5227	/// This transforms the control flow intrinsics to get the branch destination as
5228	/// last parameter, also switches branch target with BR if the need arise
5229	SDValue SITargetLowering::LowerBRCOND(SDValue BRCOND,
5230	SelectionDAG &DAG) const {
5231	SDLoc DL(BRCOND);
5232
5233	SDNode *Intr = BRCOND.getOperand(1).getNode();
5234	SDValue Target = BRCOND.getOperand(2);
5235	SDNode *BR = nullptr;
5236	SDNode *SetCC = nullptr;
5237
5238	if (Intr->getOpcode() == ISD::SETCC) {
5239	// As long as we negate the condition everything is fine
5240	SetCC = Intr;
5241	Intr = SetCC->getOperand(0).getNode();
5242
5243	} else {
5244	// Get the target from BR if we don't negate the condition
5245	BR = findUser(BRCOND, ISD::BR);
5246	assert(BR && "brcond missing unconditional branch user")(static_cast <bool> (BR && "brcond missing unconditional branch user" ) ? void (0) : __assert_fail ("BR && \"brcond missing unconditional branch user\"" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 5246, __extension__ __PRETTY_FUNCTION__));
5247	Target = BR->getOperand(1);
5248	}
5249
5250	unsigned CFNode = isCFIntrinsic(Intr);
5251	if (CFNode == 0) {
5252	// This is a uniform branch so we don't need to legalize.
5253	return BRCOND;
5254	}
5255
5256	bool HaveChain = Intr->getOpcode() == ISD::INTRINSIC_VOID \|\|
5257	Intr->getOpcode() == ISD::INTRINSIC_W_CHAIN;
5258
5259	assert(!SetCC \|\|(static_cast <bool> (!SetCC \|\| (SetCC->getConstantOperandVal (1) == 1 && cast<CondCodeSDNode>(SetCC->getOperand (2).getNode())->get() == ISD::SETNE)) ? void (0) : __assert_fail ("!SetCC \|\| (SetCC->getConstantOperandVal(1) == 1 && cast<CondCodeSDNode>(SetCC->getOperand(2).getNode())->get() == ISD::SETNE)" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 5262, __extension__ __PRETTY_FUNCTION__))
5260	(SetCC->getConstantOperandVal(1) == 1 &&(static_cast <bool> (!SetCC \|\| (SetCC->getConstantOperandVal (1) == 1 && cast<CondCodeSDNode>(SetCC->getOperand (2).getNode())->get() == ISD::SETNE)) ? void (0) : __assert_fail ("!SetCC \|\| (SetCC->getConstantOperandVal(1) == 1 && cast<CondCodeSDNode>(SetCC->getOperand(2).getNode())->get() == ISD::SETNE)" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 5262, __extension__ __PRETTY_FUNCTION__))
5261	cast<CondCodeSDNode>(SetCC->getOperand(2).getNode())->get() ==(static_cast <bool> (!SetCC \|\| (SetCC->getConstantOperandVal (1) == 1 && cast<CondCodeSDNode>(SetCC->getOperand (2).getNode())->get() == ISD::SETNE)) ? void (0) : __assert_fail ("!SetCC \|\| (SetCC->getConstantOperandVal(1) == 1 && cast<CondCodeSDNode>(SetCC->getOperand(2).getNode())->get() == ISD::SETNE)" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 5262, __extension__ __PRETTY_FUNCTION__))
5262	ISD::SETNE))(static_cast <bool> (!SetCC \|\| (SetCC->getConstantOperandVal (1) == 1 && cast<CondCodeSDNode>(SetCC->getOperand (2).getNode())->get() == ISD::SETNE)) ? void (0) : __assert_fail ("!SetCC \|\| (SetCC->getConstantOperandVal(1) == 1 && cast<CondCodeSDNode>(SetCC->getOperand(2).getNode())->get() == ISD::SETNE)" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 5262, __extension__ __PRETTY_FUNCTION__));
5263
5264	// operands of the new intrinsic call
5265	SmallVector<SDValue, 4> Ops;
5266	if (HaveChain)
5267	Ops.push_back(BRCOND.getOperand(0));
5268
5269	Ops.append(Intr->op_begin() + (HaveChain ? 2 : 1), Intr->op_end());
5270	Ops.push_back(Target);
5271
5272	ArrayRef<EVT> Res(Intr->value_begin() + 1, Intr->value_end());
5273
5274	// build the new intrinsic call
5275	SDNode *Result = DAG.getNode(CFNode, DL, DAG.getVTList(Res), Ops).getNode();
5276
5277	if (!HaveChain) {
5278	SDValue Ops[] = {
5279	SDValue(Result, 0),
5280	BRCOND.getOperand(0)
5281	};
5282
5283	Result = DAG.getMergeValues(Ops, DL).getNode();
5284	}
5285
5286	if (BR) {
5287	// Give the branch instruction our target
5288	SDValue Ops[] = {
5289	BR->getOperand(0),
5290	BRCOND.getOperand(2)
5291	};
5292	SDValue NewBR = DAG.getNode(ISD::BR, DL, BR->getVTList(), Ops);
5293	DAG.ReplaceAllUsesWith(BR, NewBR.getNode());
5294	}
5295
5296	SDValue Chain = SDValue(Result, Result->getNumValues() - 1);
5297
5298	// Copy the intrinsic results to registers
5299	for (unsigned i = 1, e = Intr->getNumValues() - 1; i != e; ++i) {
5300	SDNode *CopyToReg = findUser(SDValue(Intr, i), ISD::CopyToReg);
5301	if (!CopyToReg)
5302	continue;
5303
5304	Chain = DAG.getCopyToReg(
5305	Chain, DL,
5306	CopyToReg->getOperand(1),
5307	SDValue(Result, i - 1),
5308	SDValue());
5309
5310	DAG.ReplaceAllUsesWith(SDValue(CopyToReg, 0), CopyToReg->getOperand(0));
5311	}
5312
5313	// Remove the old intrinsic from the chain
5314	DAG.ReplaceAllUsesOfValueWith(
5315	SDValue(Intr, Intr->getNumValues() - 1),
5316	Intr->getOperand(0));
5317
5318	return Chain;
5319	}
5320
5321	SDValue SITargetLowering::LowerRETURNADDR(SDValue Op,
5322	SelectionDAG &DAG) const {
5323	MVT VT = Op.getSimpleValueType();
5324	SDLoc DL(Op);
5325	// Checking the depth
5326	if (cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue() != 0)
5327	return DAG.getConstant(0, DL, VT);
5328
5329	MachineFunction &MF = DAG.getMachineFunction();
5330	const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
5331	// Check for kernel and shader functions
5332	if (Info->isEntryFunction())
5333	return DAG.getConstant(0, DL, VT);
5334
5335	MachineFrameInfo &MFI = MF.getFrameInfo();
5336	// There is a call to @llvm.returnaddress in this function
5337	MFI.setReturnAddressIsTaken(true);
5338
5339	const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
5340	// Get the return address reg and mark it as an implicit live-in
5341	Register Reg = MF.addLiveIn(TRI->getReturnAddressReg(MF), getRegClassFor(VT, Op.getNode()->isDivergent()));
5342
5343	return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, VT);
5344	}
5345
5346	SDValue SITargetLowering::getFPExtOrFPRound(SelectionDAG &DAG,
5347	SDValue Op,
5348	const SDLoc &DL,
5349	EVT VT) const {
5350	return Op.getValueType().bitsLE(VT) ?
5351	DAG.getNode(ISD::FP_EXTEND, DL, VT, Op) :
5352	DAG.getNode(ISD::FP_ROUND, DL, VT, Op,
5353	DAG.getTargetConstant(0, DL, MVT::i32));
5354	}
5355
5356	SDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
5357	assert(Op.getValueType() == MVT::f16 &&(static_cast <bool> (Op.getValueType() == MVT::f16 && "Do not know how to custom lower FP_ROUND for non-f16 type") ? void (0) : __assert_fail ("Op.getValueType() == MVT::f16 && \"Do not know how to custom lower FP_ROUND for non-f16 type\"" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 5358, __extension__ __PRETTY_FUNCTION__))
5358	"Do not know how to custom lower FP_ROUND for non-f16 type")(static_cast <bool> (Op.getValueType() == MVT::f16 && "Do not know how to custom lower FP_ROUND for non-f16 type") ? void (0) : __assert_fail ("Op.getValueType() == MVT::f16 && \"Do not know how to custom lower FP_ROUND for non-f16 type\"" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 5358, __extension__ __PRETTY_FUNCTION__));
5359
5360	SDValue Src = Op.getOperand(0);
5361	EVT SrcVT = Src.getValueType();
5362	if (SrcVT != MVT::f64)
5363	return Op;
5364
5365	SDLoc DL(Op);
5366
5367	SDValue FpToFp16 = DAG.getNode(ISD::FP_TO_FP16, DL, MVT::i32, Src);
5368	SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FpToFp16);
5369	return DAG.getNode(ISD::BITCAST, DL, MVT::f16, Trunc);
5370	}
5371
5372	SDValue SITargetLowering::lowerFMINNUM_FMAXNUM(SDValue Op,
5373	SelectionDAG &DAG) const {
5374	EVT VT = Op.getValueType();
5375	const MachineFunction &MF = DAG.getMachineFunction();
5376	const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
5377	bool IsIEEEMode = Info->getMode().IEEE;
5378
5379	// FIXME: Assert during selection that this is only selected for
5380	// ieee_mode. Currently a combine can produce the ieee version for non-ieee
5381	// mode functions, but this happens to be OK since it's only done in cases
5382	// where there is known no sNaN.
5383	if (IsIEEEMode)
5384	return expandFMINNUM_FMAXNUM(Op.getNode(), DAG);
5385
5386	if (VT == MVT::v4f16 \|\| VT == MVT::v8f16 \|\| VT == MVT::v16f16)
5387	return splitBinaryVectorOp(Op, DAG);
5388	return Op;
5389	}
5390
5391	SDValue SITargetLowering::lowerXMULO(SDValue Op, SelectionDAG &DAG) const {
5392	EVT VT = Op.getValueType();
5393	SDLoc SL(Op);
5394	SDValue LHS = Op.getOperand(0);
5395	SDValue RHS = Op.getOperand(1);
5396	bool isSigned = Op.getOpcode() == ISD::SMULO;
5397
5398	if (ConstantSDNode *RHSC = isConstOrConstSplat(RHS)) {
5399	const APInt &C = RHSC->getAPIntValue();
5400	// mulo(X, 1 << S) -> { X << S, (X << S) >> S != X }
5401	if (C.isPowerOf2()) {
5402	// smulo(x, signed_min) is same as umulo(x, signed_min).
5403	bool UseArithShift = isSigned && !C.isMinSignedValue();
5404	SDValue ShiftAmt = DAG.getConstant(C.logBase2(), SL, MVT::i32);
5405	SDValue Result = DAG.getNode(ISD::SHL, SL, VT, LHS, ShiftAmt);
5406	SDValue Overflow = DAG.getSetCC(SL, MVT::i1,
5407	DAG.getNode(UseArithShift ? ISD::SRA : ISD::SRL,
5408	SL, VT, Result, ShiftAmt),
5409	LHS, ISD::SETNE);
5410	return DAG.getMergeValues({ Result, Overflow }, SL);
5411	}
5412	}
5413
5414	SDValue Result = DAG.getNode(ISD::MUL, SL, VT, LHS, RHS);
5415	SDValue Top = DAG.getNode(isSigned ? ISD::MULHS : ISD::MULHU,
5416	SL, VT, LHS, RHS);
5417
5418	SDValue Sign = isSigned
5419	? DAG.getNode(ISD::SRA, SL, VT, Result,
5420	DAG.getConstant(VT.getScalarSizeInBits() - 1, SL, MVT::i32))
5421	: DAG.getConstant(0, SL, VT);
5422	SDValue Overflow = DAG.getSetCC(SL, MVT::i1, Top, Sign, ISD::SETNE);
5423
5424	return DAG.getMergeValues({ Result, Overflow }, SL);
5425	}
5426
5427	SDValue SITargetLowering::lowerXMUL_LOHI(SDValue Op, SelectionDAG &DAG) const {
5428	if (Op->isDivergent()) {
5429	// Select to V_MAD_[IU]64_[IU]32.
5430	return Op;
5431	}
5432	if (Subtarget->hasSMulHi()) {
5433	// Expand to S_MUL_I32 + S_MUL_HI_[IU]32.
5434	return SDValue();
5435	}
5436	// The multiply is uniform but we would have to use V_MUL_HI_[IU]32 to
5437	// calculate the high part, so we might as well do the whole thing with
5438	// V_MAD_[IU]64_[IU]32.
5439	return Op;
5440	}
5441
5442	SDValue SITargetLowering::lowerTRAP(SDValue Op, SelectionDAG &DAG) const {
5443	if (!Subtarget->isTrapHandlerEnabled() \|\|
5444	Subtarget->getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA)
5445	return lowerTrapEndpgm(Op, DAG);
5446
5447	const Module *M = DAG.getMachineFunction().getFunction().getParent();
5448	unsigned CodeObjectVersion = AMDGPU::getCodeObjectVersion(*M);
5449	if (CodeObjectVersion <= AMDGPU::AMDHSA_COV3)
5450	return lowerTrapHsaQueuePtr(Op, DAG);
5451
5452	return Subtarget->supportsGetDoorbellID() ? lowerTrapHsa(Op, DAG) :
5453	lowerTrapHsaQueuePtr(Op, DAG);
5454	}
5455
5456	SDValue SITargetLowering::lowerTrapEndpgm(
5457	SDValue Op, SelectionDAG &DAG) const {
5458	SDLoc SL(Op);
5459	SDValue Chain = Op.getOperand(0);
5460	return DAG.getNode(AMDGPUISD::ENDPGM, SL, MVT::Other, Chain);
5461	}
5462
5463	SDValue SITargetLowering::loadImplicitKernelArgument(SelectionDAG &DAG, MVT VT,
5464	const SDLoc &DL, Align Alignment, ImplicitParameter Param) const {
5465	MachineFunction &MF = DAG.getMachineFunction();
5466	uint64_t Offset = getImplicitParameterOffset(MF, Param);
5467	SDValue Ptr = lowerKernArgParameterPtr(DAG, DL, DAG.getEntryNode(), Offset);
5468	MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
5469	return DAG.getLoad(VT, DL, DAG.getEntryNode(), Ptr, PtrInfo, Alignment,
5470	MachineMemOperand::MODereferenceable \|
5471	MachineMemOperand::MOInvariant);
5472	}
5473
5474	SDValue SITargetLowering::lowerTrapHsaQueuePtr(
5475	SDValue Op, SelectionDAG &DAG) const {
5476	SDLoc SL(Op);
5477	SDValue Chain = Op.getOperand(0);
5478
5479	SDValue QueuePtr;
5480	// For code object version 5, QueuePtr is passed through implicit kernarg.
5481	const Module *M = DAG.getMachineFunction().getFunction().getParent();
5482	if (AMDGPU::getCodeObjectVersion(*M) >= AMDGPU::AMDHSA_COV5) {
5483	QueuePtr =
5484	loadImplicitKernelArgument(DAG, MVT::i64, SL, Align(8), QUEUE_PTR);
5485	} else {
5486	MachineFunction &MF = DAG.getMachineFunction();
5487	SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
5488	Register UserSGPR = Info->getQueuePtrUserSGPR();
5489
5490	if (UserSGPR == AMDGPU::NoRegister) {
5491	// We probably are in a function incorrectly marked with
5492	// amdgpu-no-queue-ptr. This is undefined. We don't want to delete the
5493	// trap, so just use a null pointer.
5494	QueuePtr = DAG.getConstant(0, SL, MVT::i64);
5495	} else {
5496	QueuePtr = CreateLiveInRegister(DAG, &AMDGPU::SReg_64RegClass, UserSGPR,
5497	MVT::i64);
5498	}
5499	}
5500
5501	SDValue SGPR01 = DAG.getRegister(AMDGPU::SGPR0_SGPR1, MVT::i64);
5502	SDValue ToReg = DAG.getCopyToReg(Chain, SL, SGPR01,
5503	QueuePtr, SDValue());
5504
5505	uint64_t TrapID = static_cast<uint64_t>(GCNSubtarget::TrapID::LLVMAMDHSATrap);
5506	SDValue Ops[] = {
5507	ToReg,
5508	DAG.getTargetConstant(TrapID, SL, MVT::i16),
5509	SGPR01,
5510	ToReg.getValue(1)
5511	};
5512	return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops);
5513	}
5514
5515	SDValue SITargetLowering::lowerTrapHsa(
5516	SDValue Op, SelectionDAG &DAG) const {
5517	SDLoc SL(Op);
5518	SDValue Chain = Op.getOperand(0);
5519
5520	uint64_t TrapID = static_cast<uint64_t>(GCNSubtarget::TrapID::LLVMAMDHSATrap);
5521	SDValue Ops[] = {
5522	Chain,
5523	DAG.getTargetConstant(TrapID, SL, MVT::i16)
5524	};
5525	return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops);
5526	}
5527
5528	SDValue SITargetLowering::lowerDEBUGTRAP(SDValue Op, SelectionDAG &DAG) const {
5529	SDLoc SL(Op);
5530	SDValue Chain = Op.getOperand(0);
5531	MachineFunction &MF = DAG.getMachineFunction();
5532
5533	if (!Subtarget->isTrapHandlerEnabled() \|\|
5534	Subtarget->getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA) {
5535	DiagnosticInfoUnsupported NoTrap(MF.getFunction(),
5536	"debugtrap handler not supported",
5537	Op.getDebugLoc(),
5538	DS_Warning);
5539	LLVMContext &Ctx = MF.getFunction().getContext();
5540	Ctx.diagnose(NoTrap);
5541	return Chain;
5542	}
5543
5544	uint64_t TrapID = static_cast<uint64_t>(GCNSubtarget::TrapID::LLVMAMDHSADebugTrap);
5545	SDValue Ops[] = {
5546	Chain,
5547	DAG.getTargetConstant(TrapID, SL, MVT::i16)
5548	};
5549	return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops);
5550	}
5551
5552	SDValue SITargetLowering::getSegmentAperture(unsigned AS, const SDLoc &DL,
5553	SelectionDAG &DAG) const {
5554	if (Subtarget->hasApertureRegs()) {
5555	const unsigned ApertureRegNo = (AS == AMDGPUAS::LOCAL_ADDRESS)
5556	? AMDGPU::SRC_SHARED_BASE
5557	: AMDGPU::SRC_PRIVATE_BASE;
5558	// Note: this feature (register) is broken. When used as a 32-bit operand,
5559	// it returns a wrong value (all zeroes?). The real value is in the upper 32
5560	// bits.
5561	//
5562	// To work around the issue, directly emit a 64 bit mov from this register
5563	// then extract the high bits. Note that this shouldn't even result in a
5564	// shift being emitted and simply become a pair of registers (e.g.):
5565	// s_mov_b64 s[6:7], src_shared_base
5566	// v_mov_b32_e32 v1, s7
5567	//
5568	// FIXME: It would be more natural to emit a CopyFromReg here, but then copy
5569	// coalescing would kick in and it would think it's okay to use the "HI"
5570	// subregister directly (instead of extracting the HI 32 bits) which is an
5571	// artificial (unusable) register.
5572	// Register TableGen definitions would need an overhaul to get rid of the
5573	// artificial "HI" aperture registers and prevent this kind of issue from
5574	// happening.
5575	SDNode *Mov = DAG.getMachineNode(AMDGPU::S_MOV_B64, DL, MVT::i64,
5576	DAG.getRegister(ApertureRegNo, MVT::i64));
5577	return DAG.getNode(
5578	ISD::TRUNCATE, DL, MVT::i32,
5579	DAG.getNode(ISD::SRL, DL, MVT::i64,
5580	{SDValue(Mov, 0), DAG.getConstant(32, DL, MVT::i64)}));
5581	}
5582
5583	// For code object version 5, private_base and shared_base are passed through
5584	// implicit kernargs.
5585	const Module *M = DAG.getMachineFunction().getFunction().getParent();
5586	if (AMDGPU::getCodeObjectVersion(*M) >= AMDGPU::AMDHSA_COV5) {
5587	ImplicitParameter Param =
5588	(AS == AMDGPUAS::LOCAL_ADDRESS) ? SHARED_BASE : PRIVATE_BASE;
5589	return loadImplicitKernelArgument(DAG, MVT::i32, DL, Align(4), Param);
5590	}
5591
5592	MachineFunction &MF = DAG.getMachineFunction();
5593	SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
5594	Register UserSGPR = Info->getQueuePtrUserSGPR();
5595	if (UserSGPR == AMDGPU::NoRegister) {
5596	// We probably are in a function incorrectly marked with
5597	// amdgpu-no-queue-ptr. This is undefined.
5598	return DAG.getUNDEF(MVT::i32);
5599	}
5600
5601	SDValue QueuePtr = CreateLiveInRegister(
5602	DAG, &AMDGPU::SReg_64RegClass, UserSGPR, MVT::i64);
5603
5604	// Offset into amd_queue_t for group_segment_aperture_base_hi /
5605	// private_segment_aperture_base_hi.
5606	uint32_t StructOffset = (AS == AMDGPUAS::LOCAL_ADDRESS) ? 0x40 : 0x44;
5607
5608	SDValue Ptr =
5609	DAG.getObjectPtrOffset(DL, QueuePtr, TypeSize::Fixed(StructOffset));
5610
5611	// TODO: Use custom target PseudoSourceValue.
5612	// TODO: We should use the value from the IR intrinsic call, but it might not
5613	// be available and how do we get it?
5614	MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
5615	return DAG.getLoad(MVT::i32, DL, QueuePtr.getValue(1), Ptr, PtrInfo,
5616	commonAlignment(Align(64), StructOffset),
5617	MachineMemOperand::MODereferenceable \|
5618	MachineMemOperand::MOInvariant);
5619	}
5620
5621	/// Return true if the value is a known valid address, such that a null check is
5622	/// not necessary.
5623	static bool isKnownNonNull(SDValue Val, SelectionDAG &DAG,
5624	const AMDGPUTargetMachine &TM, unsigned AddrSpace) {
5625	if (isa<FrameIndexSDNode>(Val) \|\| isa<GlobalAddressSDNode>(Val) \|\|
5626	isa<BasicBlockSDNode>(Val))
5627	return true;
5628
5629	if (auto *ConstVal = dyn_cast<ConstantSDNode>(Val))
5630	return ConstVal->getSExtValue() != TM.getNullPointerValue(AddrSpace);
5631
5632	// TODO: Search through arithmetic, handle arguments and loads
5633	// marked nonnull.
5634	return false;
5635	}
5636
5637	SDValue SITargetLowering::lowerADDRSPACECAST(SDValue Op,
5638	SelectionDAG &DAG) const {
5639	SDLoc SL(Op);
5640	const AddrSpaceCastSDNode *ASC = cast<AddrSpaceCastSDNode>(Op);
5641
5642	SDValue Src = ASC->getOperand(0);
5643	SDValue FlatNullPtr = DAG.getConstant(0, SL, MVT::i64);
5644	unsigned SrcAS = ASC->getSrcAddressSpace();
5645
5646	const AMDGPUTargetMachine &TM =
5647	static_cast<const AMDGPUTargetMachine &>(getTargetMachine());
5648
5649	// flat -> local/private
5650	if (SrcAS == AMDGPUAS::FLAT_ADDRESS) {
5651	unsigned DestAS = ASC->getDestAddressSpace();
5652
5653	if (DestAS == AMDGPUAS::LOCAL_ADDRESS \|\|
5654	DestAS == AMDGPUAS::PRIVATE_ADDRESS) {
5655	SDValue Ptr = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Src);
5656
5657	if (isKnownNonNull(Src, DAG, TM, SrcAS))
5658	return Ptr;
5659
5660	unsigned NullVal = TM.getNullPointerValue(DestAS);
5661	SDValue SegmentNullPtr = DAG.getConstant(NullVal, SL, MVT::i32);
5662	SDValue NonNull = DAG.getSetCC(SL, MVT::i1, Src, FlatNullPtr, ISD::SETNE);
5663
5664	return DAG.getNode(ISD::SELECT, SL, MVT::i32, NonNull, Ptr,
5665	SegmentNullPtr);
5666	}
5667	}
5668
5669	// local/private -> flat
5670	if (ASC->getDestAddressSpace() == AMDGPUAS::FLAT_ADDRESS) {
5671	if (SrcAS == AMDGPUAS::LOCAL_ADDRESS \|\|
5672	SrcAS == AMDGPUAS::PRIVATE_ADDRESS) {
5673
5674	SDValue Aperture = getSegmentAperture(ASC->getSrcAddressSpace(), SL, DAG);
5675	SDValue CvtPtr =
5676	DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, Src, Aperture);
5677	CvtPtr = DAG.getNode(ISD::BITCAST, SL, MVT::i64, CvtPtr);
5678
5679	if (isKnownNonNull(Src, DAG, TM, SrcAS))
5680	return CvtPtr;
5681
5682	unsigned NullVal = TM.getNullPointerValue(SrcAS);
5683	SDValue SegmentNullPtr = DAG.getConstant(NullVal, SL, MVT::i32);
5684
5685	SDValue NonNull
5686	= DAG.getSetCC(SL, MVT::i1, Src, SegmentNullPtr, ISD::SETNE);
5687
5688	return DAG.getNode(ISD::SELECT, SL, MVT::i64, NonNull, CvtPtr,
5689	FlatNullPtr);
5690	}
5691	}
5692
5693	if (SrcAS == AMDGPUAS::CONSTANT_ADDRESS_32BIT &&
5694	Op.getValueType() == MVT::i64) {
5695	const SIMachineFunctionInfo *Info =
5696	DAG.getMachineFunction().getInfo<SIMachineFunctionInfo>();
5697	SDValue Hi = DAG.getConstant(Info->get32BitAddressHighBits(), SL, MVT::i32);
5698	SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, Src, Hi);
5699	return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec);
5700	}
5701
5702	if (ASC->getDestAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT &&
5703	Src.getValueType() == MVT::i64)
5704	return DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Src);
5705
5706	// global <-> flat are no-ops and never emitted.
5707
5708	const MachineFunction &MF = DAG.getMachineFunction();
5709	DiagnosticInfoUnsupported InvalidAddrSpaceCast(
5710	MF.getFunction(), "invalid addrspacecast", SL.getDebugLoc());
5711	DAG.getContext()->diagnose(InvalidAddrSpaceCast);
5712
5713	return DAG.getUNDEF(ASC->getValueType(0));
5714	}
5715
5716	// This lowers an INSERT_SUBVECTOR by extracting the individual elements from
5717	// the small vector and inserting them into the big vector. That is better than
5718	// the default expansion of doing it via a stack slot. Even though the use of
5719	// the stack slot would be optimized away afterwards, the stack slot itself
5720	// remains.
5721	SDValue SITargetLowering::lowerINSERT_SUBVECTOR(SDValue Op,
5722	SelectionDAG &DAG) const {
5723	SDValue Vec = Op.getOperand(0);
5724	SDValue Ins = Op.getOperand(1);
5725	SDValue Idx = Op.getOperand(2);
5726	EVT VecVT = Vec.getValueType();
5727	EVT InsVT = Ins.getValueType();
5728	EVT EltVT = VecVT.getVectorElementType();
5729	unsigned InsNumElts = InsVT.getVectorNumElements();
5730	unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
5731	SDLoc SL(Op);
5732
5733	for (unsigned I = 0; I != InsNumElts; ++I) {
5734	SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Ins,
5735	DAG.getConstant(I, SL, MVT::i32));
5736	Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, VecVT, Vec, Elt,
5737	DAG.getConstant(IdxVal + I, SL, MVT::i32));
5738	}
5739	return Vec;
5740	}
5741
5742	SDValue SITargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
5743	SelectionDAG &DAG) const {
5744	SDValue Vec = Op.getOperand(0);
5745	SDValue InsVal = Op.getOperand(1);
5746	SDValue Idx = Op.getOperand(2);
5747	EVT VecVT = Vec.getValueType();
5748	EVT EltVT = VecVT.getVectorElementType();
5749	unsigned VecSize = VecVT.getSizeInBits();
5750	unsigned EltSize = EltVT.getSizeInBits();
5751	SDLoc SL(Op);
5752
5753	// Specially handle the case of v4i16 with static indexing.
5754	unsigned NumElts = VecVT.getVectorNumElements();
5755	auto KIdx = dyn_cast<ConstantSDNode>(Idx);
5756	if (NumElts == 4 && EltSize == 16 && KIdx) {
5757	SDValue BCVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Vec);
5758
5759	SDValue LoHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BCVec,
5760	DAG.getConstant(0, SL, MVT::i32));
5761	SDValue HiHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BCVec,
5762	DAG.getConstant(1, SL, MVT::i32));
5763
5764	SDValue LoVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, LoHalf);
5765	SDValue HiVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, HiHalf);
5766
5767	unsigned Idx = KIdx->getZExtValue();
5768	bool InsertLo = Idx < 2;
5769	SDValue InsHalf = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, MVT::v2i16,
5770	InsertLo ? LoVec : HiVec,
5771	DAG.getNode(ISD::BITCAST, SL, MVT::i16, InsVal),
5772	DAG.getConstant(InsertLo ? Idx : (Idx - 2), SL, MVT::i32));
5773
5774	InsHalf = DAG.getNode(ISD::BITCAST, SL, MVT::i32, InsHalf);
5775
5776	SDValue Concat = InsertLo ?
5777	DAG.getBuildVector(MVT::v2i32, SL, { InsHalf, HiHalf }) :
5778	DAG.getBuildVector(MVT::v2i32, SL, { LoHalf, InsHalf });
5779
5780	return DAG.getNode(ISD::BITCAST, SL, VecVT, Concat);
5781	}
5782
5783	// Static indexing does not lower to stack access, and hence there is no need
5784	// for special custom lowering to avoid stack access.
5785	if (isa<ConstantSDNode>(Idx))
5786	return SDValue();
5787
5788	// Avoid stack access for dynamic indexing by custom lowering to
5789	// v_bfi_b32 (v_bfm_b32 16, (shl idx, 16)), val, vec
5790
5791	assert(VecSize <= 64 && "Expected target vector size to be <= 64 bits")(static_cast <bool> (VecSize <= 64 && "Expected target vector size to be <= 64 bits" ) ? void (0) : __assert_fail ("VecSize <= 64 && \"Expected target vector size to be <= 64 bits\"" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 5791, __extension__ __PRETTY_FUNCTION__));
5792
5793	MVT IntVT = MVT::getIntegerVT(VecSize);
5794
5795	// Convert vector index to bit-index and get the required bit mask.
5796	assert(isPowerOf2_32(EltSize))(static_cast <bool> (isPowerOf2_32(EltSize)) ? void (0) : __assert_fail ("isPowerOf2_32(EltSize)", "llvm/lib/Target/AMDGPU/SIISelLowering.cpp" , 5796, __extension__ __PRETTY_FUNCTION__));
5797	const auto EltMask = maskTrailingOnes<uint64_t>(EltSize);
5798	SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32);
5799	SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx, ScaleFactor);
5800	SDValue BFM = DAG.getNode(ISD::SHL, SL, IntVT,
5801	DAG.getConstant(EltMask, SL, IntVT), ScaledIdx);
5802
5803	// 1. Create a congruent vector with the target value in each element.
5804	SDValue ExtVal = DAG.getNode(ISD::BITCAST, SL, IntVT,
5805	DAG.getSplatBuildVector(VecVT, SL, InsVal));
5806
5807	// 2. Mask off all other indicies except the required index within (1).
5808	SDValue LHS = DAG.getNode(ISD::AND, SL, IntVT, BFM, ExtVal);
5809
5810	// 3. Mask off the required index within the target vector.
5811	SDValue BCVec = DAG.getNode(ISD::BITCAST, SL, IntVT, Vec);
5812	SDValue RHS = DAG.getNode(ISD::AND, SL, IntVT,
5813	DAG.getNOT(SL, BFM, IntVT), BCVec);
5814
5815	// 4. Get (2) and (3) ORed into the target vector.
5816	SDValue BFI = DAG.getNode(ISD::OR, SL, IntVT, LHS, RHS);
5817
5818	return DAG.getNode(ISD::BITCAST, SL, VecVT, BFI);
5819	}
5820
5821	SDValue SITargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
5822	SelectionDAG &DAG) const {
5823	SDLoc SL(Op);
5824
5825	EVT ResultVT = Op.getValueType();
5826	SDValue Vec = Op.getOperand(0);
5827	SDValue Idx = Op.getOperand(1);
5828	EVT VecVT = Vec.getValueType();
5829	unsigned VecSize = VecVT.getSizeInBits();
5830	EVT EltVT = VecVT.getVectorElementType();
5831
5832	DAGCombinerInfo DCI(DAG, AfterLegalizeVectorOps, true, nullptr);
5833
5834	// Make sure we do any optimizations that will make it easier to fold
5835	// source modifiers before obscuring it with bit operations.
5836
5837	// XXX - Why doesn't this get called when vector_shuffle is expanded?
5838	if (SDValue Combined = performExtractVectorEltCombine(Op.getNode(), DCI))
5839	return Combined;
5840
5841	if (VecSize == 128 \|\| VecSize == 256) {
5842	SDValue Lo, Hi;
5843	EVT LoVT, HiVT;
5844	std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
5845
5846	if (VecSize == 128) {
5847	SDValue V2 = DAG.getBitcast(MVT::v2i64, Vec);
5848	Lo = DAG.getBitcast(LoVT,
5849	DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2,
5850	DAG.getConstant(0, SL, MVT::i32)));
5851	Hi = DAG.getBitcast(HiVT,
5852	DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2,
5853	DAG.getConstant(1, SL, MVT::i32)));
5854	} else {
5855	assert(VecSize == 256)(static_cast <bool> (VecSize == 256) ? void (0) : __assert_fail ("VecSize == 256", "llvm/lib/Target/AMDGPU/SIISelLowering.cpp" , 5855, __extension__ __PRETTY_FUNCTION__));
5856
5857	SDValue V2 = DAG.getBitcast(MVT::v4i64, Vec);
5858	SDValue Parts[4];
5859	for (unsigned P = 0; P < 4; ++P) {
5860	Parts[P] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2,
5861	DAG.getConstant(P, SL, MVT::i32));
5862	}
5863
5864	Lo = DAG.getBitcast(LoVT, DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i64,
5865	Parts[0], Parts[1]));
5866	Hi = DAG.getBitcast(HiVT, DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i64,
5867	Parts[2], Parts[3]));
5868	}
5869
5870	EVT IdxVT = Idx.getValueType();
5871	unsigned NElem = VecVT.getVectorNumElements();
5872	assert(isPowerOf2_32(NElem))(static_cast <bool> (isPowerOf2_32(NElem)) ? void (0) : __assert_fail ("isPowerOf2_32(NElem)", "llvm/lib/Target/AMDGPU/SIISelLowering.cpp" , 5872, __extension__ __PRETTY_FUNCTION__));
5873	SDValue IdxMask = DAG.getConstant(NElem / 2 - 1, SL, IdxVT);
5874	SDValue NewIdx = DAG.getNode(ISD::AND, SL, IdxVT, Idx, IdxMask);
5875	SDValue Half = DAG.getSelectCC(SL, Idx, IdxMask, Hi, Lo, ISD::SETUGT);
5876	return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Half, NewIdx);
5877	}
5878
5879	assert(VecSize <= 64)(static_cast <bool> (VecSize <= 64) ? void (0) : __assert_fail ("VecSize <= 64", "llvm/lib/Target/AMDGPU/SIISelLowering.cpp" , 5879, __extension__ __PRETTY_FUNCTION__));
5880
5881	MVT IntVT = MVT::getIntegerVT(VecSize);
5882
5883	// If Vec is just a SCALAR_TO_VECTOR, then use the scalar integer directly.
5884	SDValue VecBC = peekThroughBitcasts(Vec);
5885	if (VecBC.getOpcode() == ISD::SCALAR_TO_VECTOR) {
5886	SDValue Src = VecBC.getOperand(0);
5887	Src = DAG.getBitcast(Src.getValueType().changeTypeToInteger(), Src);
5888	Vec = DAG.getAnyExtOrTrunc(Src, SL, IntVT);
5889	}
5890
5891	unsigned EltSize = EltVT.getSizeInBits();
5892	assert(isPowerOf2_32(EltSize))(static_cast <bool> (isPowerOf2_32(EltSize)) ? void (0) : __assert_fail ("isPowerOf2_32(EltSize)", "llvm/lib/Target/AMDGPU/SIISelLowering.cpp" , 5892, __extension__ __PRETTY_FUNCTION__));
5893
5894	SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32);
5895
5896	// Convert vector index to bit-index (* EltSize)
5897	SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx, ScaleFactor);
5898
5899	SDValue BC = DAG.getNode(ISD::BITCAST, SL, IntVT, Vec);
5900	SDValue Elt = DAG.getNode(ISD::SRL, SL, IntVT, BC, ScaledIdx);
5901
5902	if (ResultVT == MVT::f16) {
5903	SDValue Result = DAG.getNode(ISD::TRUNCATE, SL, MVT::i16, Elt);
5904	return DAG.getNode(ISD::BITCAST, SL, ResultVT, Result);
5905	}
5906
5907	return DAG.getAnyExtOrTrunc(Elt, SL, ResultVT);
5908	}
5909
5910	static bool elementPairIsContiguous(ArrayRef<int> Mask, int Elt) {
5911	assert(Elt % 2 == 0)(static_cast <bool> (Elt % 2 == 0) ? void (0) : __assert_fail ("Elt % 2 == 0", "llvm/lib/Target/AMDGPU/SIISelLowering.cpp" , 5911, __extension__ __PRETTY_FUNCTION__));
5912	return Mask[Elt + 1] == Mask[Elt] + 1 && (Mask[Elt] % 2 == 0);
5913	}
5914
5915	SDValue SITargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
5916	SelectionDAG &DAG) const {
5917	SDLoc SL(Op);
5918	EVT ResultVT = Op.getValueType();
5919	ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
5920
5921	EVT PackVT = ResultVT.isInteger() ? MVT::v2i16 : MVT::v2f16;
5922	EVT EltVT = PackVT.getVectorElementType();
5923	int SrcNumElts = Op.getOperand(0).getValueType().getVectorNumElements();
5924
5925	// vector_shuffle <0,1,6,7> lhs, rhs
5926	// -> concat_vectors (extract_subvector lhs, 0), (extract_subvector rhs, 2)
5927	//
5928	// vector_shuffle <6,7,2,3> lhs, rhs
5929	// -> concat_vectors (extract_subvector rhs, 2), (extract_subvector lhs, 2)
5930	//
5931	// vector_shuffle <6,7,0,1> lhs, rhs
5932	// -> concat_vectors (extract_subvector rhs, 2), (extract_subvector lhs, 0)
5933
5934	// Avoid scalarizing when both halves are reading from consecutive elements.
5935	SmallVector<SDValue, 4> Pieces;
5936	for (int I = 0, N = ResultVT.getVectorNumElements(); I != N; I += 2) {
5937	if (elementPairIsContiguous(SVN->getMask(), I)) {
5938	const int Idx = SVN->getMaskElt(I);
5939	int VecIdx = Idx < SrcNumElts ? 0 : 1;
5940	int EltIdx = Idx < SrcNumElts ? Idx : Idx - SrcNumElts;
5941	SDValue SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL,
5942	PackVT, SVN->getOperand(VecIdx),
5943	DAG.getConstant(EltIdx, SL, MVT::i32));
5944	Pieces.push_back(SubVec);
5945	} else {
5946	const int Idx0 = SVN->getMaskElt(I);
5947	const int Idx1 = SVN->getMaskElt(I + 1);
5948	int VecIdx0 = Idx0 < SrcNumElts ? 0 : 1;
5949	int VecIdx1 = Idx1 < SrcNumElts ? 0 : 1;
5950	int EltIdx0 = Idx0 < SrcNumElts ? Idx0 : Idx0 - SrcNumElts;
5951	int EltIdx1 = Idx1 < SrcNumElts ? Idx1 : Idx1 - SrcNumElts;
5952
5953	SDValue Vec0 = SVN->getOperand(VecIdx0);
5954	SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT,
5955	Vec0, DAG.getConstant(EltIdx0, SL, MVT::i32));
5956
5957	SDValue Vec1 = SVN->getOperand(VecIdx1);
5958	SDValue Elt1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT,
5959	Vec1, DAG.getConstant(EltIdx1, SL, MVT::i32));
5960	Pieces.push_back(DAG.getBuildVector(PackVT, SL, { Elt0, Elt1 }));
5961	}
5962	}
5963
5964	return DAG.getNode(ISD::CONCAT_VECTORS, SL, ResultVT, Pieces);
5965	}
5966
5967	SDValue SITargetLowering::lowerSCALAR_TO_VECTOR(SDValue Op,
5968	SelectionDAG &DAG) const {
5969	SDValue SVal = Op.getOperand(0);
5970	EVT ResultVT = Op.getValueType();
5971	EVT SValVT = SVal.getValueType();
5972	SDValue UndefVal = DAG.getUNDEF(SValVT);
5973	SDLoc SL(Op);
5974
5975	SmallVector<SDValue, 8> VElts;
5976	VElts.push_back(SVal);
5977	for (int I = 1, E = ResultVT.getVectorNumElements(); I < E; ++I)
5978	VElts.push_back(UndefVal);
5979
5980	return DAG.getBuildVector(ResultVT, SL, VElts);
5981	}
5982
5983	SDValue SITargetLowering::lowerBUILD_VECTOR(SDValue Op,
5984	SelectionDAG &DAG) const {
5985	SDLoc SL(Op);
5986	EVT VT = Op.getValueType();
5987
5988	if (VT == MVT::v4i16 \|\| VT == MVT::v4f16 \|\|
5989	VT == MVT::v8i16 \|\| VT == MVT::v8f16) {
5990	EVT HalfVT = MVT::getVectorVT(VT.getVectorElementType().getSimpleVT(),
5991	VT.getVectorNumElements() / 2);
5992	MVT HalfIntVT = MVT::getIntegerVT(HalfVT.getSizeInBits());
5993
5994	// Turn into pair of packed build_vectors.
5995	// TODO: Special case for constants that can be materialized with s_mov_b64.
5996	SmallVector<SDValue, 4> LoOps, HiOps;
5997	for (unsigned I = 0, E = VT.getVectorNumElements() / 2; I != E; ++I) {
5998	LoOps.push_back(Op.getOperand(I));
5999	HiOps.push_back(Op.getOperand(I + E));
6000	}
6001	SDValue Lo = DAG.getBuildVector(HalfVT, SL, LoOps);
6002	SDValue Hi = DAG.getBuildVector(HalfVT, SL, HiOps);
6003
6004	SDValue CastLo = DAG.getNode(ISD::BITCAST, SL, HalfIntVT, Lo);
6005	SDValue CastHi = DAG.getNode(ISD::BITCAST, SL, HalfIntVT, Hi);
6006
6007	SDValue Blend = DAG.getBuildVector(MVT::getVectorVT(HalfIntVT, 2), SL,
6008	{ CastLo, CastHi });
6009	return DAG.getNode(ISD::BITCAST, SL, VT, Blend);
6010	}
6011
6012	if (VT == MVT::v16i16 \|\| VT == MVT::v16f16) {
6013	EVT QuarterVT = MVT::getVectorVT(VT.getVectorElementType().getSimpleVT(),
6014	VT.getVectorNumElements() / 4);
6015	MVT QuarterIntVT = MVT::getIntegerVT(QuarterVT.getSizeInBits());
6016
6017	SmallVector<SDValue, 4> Parts[4];
6018	for (unsigned I = 0, E = VT.getVectorNumElements() / 4; I != E; ++I) {
6019	for (unsigned P = 0; P < 4; ++P)
6020	Parts[P].push_back(Op.getOperand(I + P * E));
6021	}
6022	SDValue Casts[4];
6023	for (unsigned P = 0; P < 4; ++P) {
6024	SDValue Vec = DAG.getBuildVector(QuarterVT, SL, Parts[P]);
6025	Casts[P] = DAG.getNode(ISD::BITCAST, SL, QuarterIntVT, Vec);
6026	}
6027
6028	SDValue Blend =
6029	DAG.getBuildVector(MVT::getVectorVT(QuarterIntVT, 4), SL, Casts);
6030	return DAG.getNode(ISD::BITCAST, SL, VT, Blend);
6031	}
6032
6033	assert(VT == MVT::v2f16 \|\| VT == MVT::v2i16)(static_cast <bool> (VT == MVT::v2f16 \|\| VT == MVT::v2i16 ) ? void (0) : __assert_fail ("VT == MVT::v2f16 \|\| VT == MVT::v2i16" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 6033, __extension__ __PRETTY_FUNCTION__));
6034	assert(!Subtarget->hasVOP3PInsts() && "this should be legal")(static_cast <bool> (!Subtarget->hasVOP3PInsts() && "this should be legal") ? void (0) : __assert_fail ("!Subtarget->hasVOP3PInsts() && \"this should be legal\"" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 6034, __extension__ __PRETTY_FUNCTION__));
6035
6036	SDValue Lo = Op.getOperand(0);
6037	SDValue Hi = Op.getOperand(1);
6038
6039	// Avoid adding defined bits with the zero_extend.
6040	if (Hi.isUndef()) {
6041	Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Lo);
6042	SDValue ExtLo = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Lo);
6043	return DAG.getNode(ISD::BITCAST, SL, VT, ExtLo);
6044	}
6045
6046	Hi = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Hi);
6047	Hi = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Hi);
6048
6049	SDValue ShlHi = DAG.getNode(ISD::SHL, SL, MVT::i32, Hi,
6050	DAG.getConstant(16, SL, MVT::i32));
6051	if (Lo.isUndef())
6052	return DAG.getNode(ISD::BITCAST, SL, VT, ShlHi);
6053
6054	Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Lo);
6055	Lo = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Lo);
6056
6057	SDValue Or = DAG.getNode(ISD::OR, SL, MVT::i32, Lo, ShlHi);
6058	return DAG.getNode(ISD::BITCAST, SL, VT, Or);
6059	}
6060
6061	bool
6062	SITargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
6063	// We can fold offsets for anything that doesn't require a GOT relocation.
6064	return (GA->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS \|\|
6065	GA->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS \|\|
6066	GA->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) &&
6067	!shouldEmitGOTReloc(GA->getGlobal());
6068	}
6069
6070	static SDValue
6071	buildPCRelGlobalAddress(SelectionDAG &DAG, const GlobalValue *GV,
6072	const SDLoc &DL, int64_t Offset, EVT PtrVT,
6073	unsigned GAFlags = SIInstrInfo::MO_NONE) {
6074	assert(isInt<32>(Offset + 4) && "32-bit offset is expected!")(static_cast <bool> (isInt<32>(Offset + 4) && "32-bit offset is expected!") ? void (0) : __assert_fail ("isInt<32>(Offset + 4) && \"32-bit offset is expected!\"" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 6074, __extension__ __PRETTY_FUNCTION__));
6075	// In order to support pc-relative addressing, the PC_ADD_REL_OFFSET SDNode is
6076	// lowered to the following code sequence:
6077	//
6078	// For constant address space:
6079	// s_getpc_b64 s[0:1]
6080	// s_add_u32 s0, s0, $symbol
6081	// s_addc_u32 s1, s1, 0
6082	//
6083	// s_getpc_b64 returns the address of the s_add_u32 instruction and then
6084	// a fixup or relocation is emitted to replace $symbol with a literal
6085	// constant, which is a pc-relative offset from the encoding of the $symbol
6086	// operand to the global variable.
6087	//
6088	// For global address space:
6089	// s_getpc_b64 s[0:1]
6090	// s_add_u32 s0, s0, $symbol@{gotpc}rel32@lo
6091	// s_addc_u32 s1, s1, $symbol@{gotpc}rel32@hi
6092	//
6093	// s_getpc_b64 returns the address of the s_add_u32 instruction and then
6094	// fixups or relocations are emitted to replace $symbol@*@lo and
6095	// $symbol@*@hi with lower 32 bits and higher 32 bits of a literal constant,
6096	// which is a 64-bit pc-relative offset from the encoding of the $symbol
6097	// operand to the global variable.
6098	//
6099	// What we want here is an offset from the value returned by s_getpc
6100	// (which is the address of the s_add_u32 instruction) to the global
6101	// variable, but since the encoding of $symbol starts 4 bytes after the start
6102	// of the s_add_u32 instruction, we end up with an offset that is 4 bytes too
6103	// small. This requires us to add 4 to the global variable offset in order to
6104	// compute the correct address. Similarly for the s_addc_u32 instruction, the
6105	// encoding of $symbol starts 12 bytes after the start of the s_add_u32
6106	// instruction.
6107	SDValue PtrLo =
6108	DAG.getTargetGlobalAddress(GV, DL, MVT::i32, Offset + 4, GAFlags);
6109	SDValue PtrHi;
6110	if (GAFlags == SIInstrInfo::MO_NONE) {
6111	PtrHi = DAG.getTargetConstant(0, DL, MVT::i32);
6112	} else {
6113	PtrHi =
6114	DAG.getTargetGlobalAddress(GV, DL, MVT::i32, Offset + 12, GAFlags + 1);
6115	}
6116	return DAG.getNode(AMDGPUISD::PC_ADD_REL_OFFSET, DL, PtrVT, PtrLo, PtrHi);
6117	}
6118
6119	SDValue SITargetLowering::LowerGlobalAddress(AMDGPUMachineFunction *MFI,
6120	SDValue Op,
6121	SelectionDAG &DAG) const {
6122	GlobalAddressSDNode *GSD = cast<GlobalAddressSDNode>(Op);
6123	SDLoc DL(GSD);
6124	EVT PtrVT = Op.getValueType();
6125
6126	const GlobalValue *GV = GSD->getGlobal();
6127	if ((GSD->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS &&
6128	shouldUseLDSConstAddress(GV)) \|\|
6129	GSD->getAddressSpace() == AMDGPUAS::REGION_ADDRESS \|\|
6130	GSD->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS) {
6131	if (GSD->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS &&
6132	GV->hasExternalLinkage()) {
6133	Type *Ty = GV->getValueType();
6134	// HIP uses an unsized array `extern __shared__ T s[]` or similar
6135	// zero-sized type in other languages to declare the dynamic shared
6136	// memory which size is not known at the compile time. They will be
6137	// allocated by the runtime and placed directly after the static
6138	// allocated ones. They all share the same offset.
6139	if (DAG.getDataLayout().getTypeAllocSize(Ty).isZero()) {
6140	assert(PtrVT == MVT::i32 && "32-bit pointer is expected.")(static_cast <bool> (PtrVT == MVT::i32 && "32-bit pointer is expected." ) ? void (0) : __assert_fail ("PtrVT == MVT::i32 && \"32-bit pointer is expected.\"" , "llvm/lib/Target/AMDGPU/SIISelLowering.cpp", 6140, __extension__ __PRETTY_FUNCTION__));
6141	// Adjust alignment for that dynamic shared memory array.
6142	MFI->setDynLDSAlign(DAG.getDataLayout(), *cast<GlobalVariable>(GV));
6143	return SDValue(
6144	DAG.getMachineNode(AMDGPU::GET_GROUPSTATICSIZE, DL, PtrVT), 0);
6145	}
6146	}
6147	return AMDGPUTargetLowering::LowerGlobalAddress(MFI, Op, DAG);
6148	}
6149
6150	if (GSD->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {
6151	SDValue GA = DAG.getTargetGlobalAddress(GV, DL, MVT::i32, GSD->getOffset(),
6152	SIInstrInfo::MO_ABS32_LO);
6153	return DAG.getNode(AMDGPUISD::LDS, DL, MVT::i32, GA);
6154	}
6155
6156	if (shouldEmitFixup(GV))
6157	return buildPCRelGlobalAddress(DAG, GV, DL, GSD->getOffset(), PtrVT);
6158	else if (shouldEmitPCReloc(GV))
6159	return buildPCRelGlobalAddress(DAG, GV, DL, GSD->getOffset(), PtrVT,
6160	SIInstrInfo::MO_REL32);
6161
6162	SDValue GOTAddr = buildPCRelGlobalAddress(DAG, GV, DL, 0, PtrVT,
6163	SIInstrInfo::MO_GOTPCREL32);
6164
6165	Type Ty = PtrVT.getTypeForEVT(DAG.getContext());
6166	PointerType *PtrTy = PointerType::get(Ty, AMDGPUAS::CONSTANT_ADDRESS);
6167	const DataLayout &DataLayout = DAG.getDataLayout();
6168	Align Alignment = DataLayout.getABITypeAlign(PtrTy);
6169	MachinePointerInfo PtrInfo
6170	= MachinePointerInfo::getGOT(DAG.getMachineFunction());
6171
6172	return DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), GOTAddr, PtrInfo, Alignment,
6173	MachineMemOperand::MODereferenceable \|
6174	MachineMemOperand::MOInvariant);
6175	}
6176
6177	SDValue SITargetLowering::copyToM0(SelectionDAG &DAG, SDValue Chain,
6178	const SDLoc &DL, SDValue V) const {
6179	// We can't use S_MOV_B32 directly, because there is no way to specify m0 as
6180	// the destination register.
6181	//
6182	<