/build/source/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp

Bug Summary

File:	build/source/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
Warning:	line 3614, column 5 Value stored to 'MI' is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

Show analyzer invocation

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name AMDGPUInstructionSelector.cpp -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/source/build-llvm/tools/clang/stage2-bins -resource-dir /usr/lib/llvm-17/lib/clang/17 -D _DEBUG -D _GLIBCXX_ASSERTIONS -D _GNU_SOURCE -D _LIBCPP_ENABLE_ASSERTIONS -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/Target/AMDGPU -I /build/source/llvm/lib/Target/AMDGPU -I include -I /build/source/llvm/include -D _FORTIFY_SOURCE=2 -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-17/lib/clang/17/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fmacro-prefix-map=/build/source/= -fcoverage-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fcoverage-prefix-map=/build/source/= -source-date-epoch 1683717183 -O2 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -Wno-misleading-indentation -std=c++17 -fdeprecated-macro -fdebug-compilation-dir=/build/source/build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/source/= -ferror-limit 19 -fvisibility=hidden -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2023-05-10-133810-16478-1 -x c++ /build/source/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp

1	//===- AMDGPUInstructionSelector.cpp ----------------------------- C++ --==//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	/// \file
9	/// This file implements the targeting of the InstructionSelector class for
10	/// AMDGPU.
11	/// \todo This should be generated by TableGen.
12	//===----------------------------------------------------------------------===//
13
14	#include "AMDGPUInstructionSelector.h"
15	#include "AMDGPU.h"
16	#include "AMDGPUGlobalISelUtils.h"
17	#include "AMDGPUInstrInfo.h"
18	#include "AMDGPURegisterBankInfo.h"
19	#include "AMDGPUTargetMachine.h"
20	#include "SIMachineFunctionInfo.h"
21	#include "Utils/AMDGPUBaseInfo.h"
22	#include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
23	#include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h"
24	#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
25	#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
26	#include "llvm/CodeGen/MachineFrameInfo.h"
27	#include "llvm/IR/DiagnosticInfo.h"
28	#include "llvm/IR/IntrinsicsAMDGPU.h"
29	#include <optional>
30
31	#define DEBUG_TYPE"amdgpu-isel" "amdgpu-isel"
32
33	using namespace llvm;
34	using namespace MIPatternMatch;
35
36	static cl::opt<bool> AllowRiskySelect(
37	"amdgpu-global-isel-risky-select",
38	cl::desc("Allow GlobalISel to select cases that are likely to not work yet"),
39	cl::init(false),
40	cl::ReallyHidden);
41
42	#define GET_GLOBALISEL_IMPL
43	#define AMDGPUSubtarget GCNSubtarget
44	#include "AMDGPUGenGlobalISel.inc"
45	#undef GET_GLOBALISEL_IMPL
46	#undef AMDGPUSubtarget
47
48	AMDGPUInstructionSelector::AMDGPUInstructionSelector(
49	const GCNSubtarget &STI, const AMDGPURegisterBankInfo &RBI,
50	const AMDGPUTargetMachine &TM)
51	: TII(STI.getInstrInfo()), TRI(STI.getRegisterInfo()), RBI(RBI), TM(TM),
52	STI(STI),
53	EnableLateStructurizeCFG(AMDGPUTargetMachine::EnableLateStructurizeCFG),
54	#define GET_GLOBALISEL_PREDICATES_INIT
55	#include "AMDGPUGenGlobalISel.inc"
56	#undef GET_GLOBALISEL_PREDICATES_INIT
57	#define GET_GLOBALISEL_TEMPORARIES_INIT
58	#include "AMDGPUGenGlobalISel.inc"
59	#undef GET_GLOBALISEL_TEMPORARIES_INIT
60	{
61	}
62
63	const char *AMDGPUInstructionSelector::getName() { return DEBUG_TYPE"amdgpu-isel"; }
64
65	void AMDGPUInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB,
66	CodeGenCoverage &CoverageInfo,
67	ProfileSummaryInfo *PSI,
68	BlockFrequencyInfo *BFI) {
69	MRI = &MF.getRegInfo();
70	Subtarget = &MF.getSubtarget<GCNSubtarget>();
71	InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI);
72	}
73
74	bool AMDGPUInstructionSelector::isVCC(Register Reg,
75	const MachineRegisterInfo &MRI) const {
76	// The verifier is oblivious to s1 being a valid value for wavesize registers.
77	if (Reg.isPhysical())
78	return false;
79
80	auto &RegClassOrBank = MRI.getRegClassOrRegBank(Reg);
81	const TargetRegisterClass *RC =
82	RegClassOrBank.dyn_cast<const TargetRegisterClass*>();
83	if (RC) {
84	const LLT Ty = MRI.getType(Reg);
85	if (!Ty.isValid() \|\| Ty.getSizeInBits() != 1)
86	return false;
87	// G_TRUNC s1 result is never vcc.
88	return MRI.getVRegDef(Reg)->getOpcode() != AMDGPU::G_TRUNC &&
89	RC->hasSuperClassEq(TRI.getBoolRC());
90	}
91
92	const RegisterBank RB = RegClassOrBank.get<const RegisterBank >();
93	return RB->getID() == AMDGPU::VCCRegBankID;
94	}
95
96	bool AMDGPUInstructionSelector::constrainCopyLikeIntrin(MachineInstr &MI,
97	unsigned NewOpc) const {
98	MI.setDesc(TII.get(NewOpc));
99	MI.removeOperand(1); // Remove intrinsic ID.
100	MI.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true));
101
102	MachineOperand &Dst = MI.getOperand(0);
103	MachineOperand &Src = MI.getOperand(1);
104
105	// TODO: This should be legalized to s32 if needed
106	if (MRI->getType(Dst.getReg()) == LLT::scalar(1))
107	return false;
108
109	const TargetRegisterClass *DstRC
110	= TRI.getConstrainedRegClassForOperand(Dst, *MRI);
111	const TargetRegisterClass *SrcRC
112	= TRI.getConstrainedRegClassForOperand(Src, *MRI);
113	if (!DstRC \|\| DstRC != SrcRC)
114	return false;
115
116	return RBI.constrainGenericRegister(Dst.getReg(), DstRC, MRI) &&
117	RBI.constrainGenericRegister(Src.getReg(), SrcRC, MRI);
118	}
119
120	bool AMDGPUInstructionSelector::selectCOPY(MachineInstr &I) const {
121	const DebugLoc &DL = I.getDebugLoc();
122	MachineBasicBlock *BB = I.getParent();
123	I.setDesc(TII.get(TargetOpcode::COPY));
124
125	const MachineOperand &Src = I.getOperand(1);
126	MachineOperand &Dst = I.getOperand(0);
127	Register DstReg = Dst.getReg();
128	Register SrcReg = Src.getReg();
129
130	if (isVCC(DstReg, *MRI)) {
131	if (SrcReg == AMDGPU::SCC) {
132	const TargetRegisterClass *RC
133	= TRI.getConstrainedRegClassForOperand(Dst, *MRI);
134	if (!RC)
135	return true;
136	return RBI.constrainGenericRegister(DstReg, RC, MRI);
137	}
138
139	if (!isVCC(SrcReg, *MRI)) {
140	// TODO: Should probably leave the copy and let copyPhysReg expand it.
141	if (!RBI.constrainGenericRegister(DstReg, TRI.getBoolRC(), MRI))
142	return false;
143
144	const TargetRegisterClass *SrcRC
145	= TRI.getConstrainedRegClassForOperand(Src, *MRI);
146
147	std::optional<ValueAndVReg> ConstVal =
148	getIConstantVRegValWithLookThrough(SrcReg, *MRI, true);
149	if (ConstVal) {
150	unsigned MovOpc =
151	STI.isWave64() ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
152	BuildMI(*BB, &I, DL, TII.get(MovOpc), DstReg)
153	.addImm(ConstVal->Value.getBoolValue() ? -1 : 0);
154	} else {
155	Register MaskedReg = MRI->createVirtualRegister(SrcRC);
156
157	// We can't trust the high bits at this point, so clear them.
158
159	// TODO: Skip masking high bits if def is known boolean.
160
161	unsigned AndOpc =
162	TRI.isSGPRClass(SrcRC) ? AMDGPU::S_AND_B32 : AMDGPU::V_AND_B32_e32;
163	BuildMI(*BB, &I, DL, TII.get(AndOpc), MaskedReg)
164	.addImm(1)
165	.addReg(SrcReg);
166	BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_CMP_NE_U32_e64), DstReg)
167	.addImm(0)
168	.addReg(MaskedReg);
169	}
170
171	if (!MRI->getRegClassOrNull(SrcReg))
172	MRI->setRegClass(SrcReg, SrcRC);
173	I.eraseFromParent();
174	return true;
175	}
176
177	const TargetRegisterClass *RC =
178	TRI.getConstrainedRegClassForOperand(Dst, *MRI);
179	if (RC && !RBI.constrainGenericRegister(DstReg, RC, MRI))
180	return false;
181
182	return true;
183	}
184
185	for (const MachineOperand &MO : I.operands()) {
186	if (MO.getReg().isPhysical())
187	continue;
188
189	const TargetRegisterClass *RC =
190	TRI.getConstrainedRegClassForOperand(MO, *MRI);
191	if (!RC)
192	continue;
193	RBI.constrainGenericRegister(MO.getReg(), RC, MRI);
194	}
195	return true;
196	}
197
198	bool AMDGPUInstructionSelector::selectPHI(MachineInstr &I) const {
199	const Register DefReg = I.getOperand(0).getReg();
200	const LLT DefTy = MRI->getType(DefReg);
201	if (DefTy == LLT::scalar(1)) {
202	if (!AllowRiskySelect) {
203	LLVM_DEBUG(dbgs() << "Skipping risky boolean phi\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("amdgpu-isel")) { dbgs() << "Skipping risky boolean phi\n" ; } } while (false);
204	return false;
205	}
206
207	LLVM_DEBUG(dbgs() << "Selecting risky boolean phi\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("amdgpu-isel")) { dbgs() << "Selecting risky boolean phi\n" ; } } while (false);
208	}
209
210	// TODO: Verify this doesn't have insane operands (i.e. VGPR to SGPR copy)
211
212	const RegClassOrRegBank &RegClassOrBank =
213	MRI->getRegClassOrRegBank(DefReg);
214
215	const TargetRegisterClass *DefRC
216	= RegClassOrBank.dyn_cast<const TargetRegisterClass *>();
217	if (!DefRC) {
218	if (!DefTy.isValid()) {
219	LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("amdgpu-isel")) { dbgs() << "PHI operand has no type, not a gvreg?\n" ; } } while (false);
220	return false;
221	}
222
223	const RegisterBank &RB = RegClassOrBank.get<const RegisterBank >();
224	DefRC = TRI.getRegClassForTypeOnBank(DefTy, RB);
225	if (!DefRC) {
226	LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("amdgpu-isel")) { dbgs() << "PHI operand has unexpected size/bank\n" ; } } while (false);
227	return false;
228	}
229	}
230
231	// TODO: Verify that all registers have the same bank
232	I.setDesc(TII.get(TargetOpcode::PHI));
233	return RBI.constrainGenericRegister(DefReg, DefRC, MRI);
234	}
235
236	MachineOperand
237	AMDGPUInstructionSelector::getSubOperand64(MachineOperand &MO,
238	const TargetRegisterClass &SubRC,
239	unsigned SubIdx) const {
240
241	MachineInstr *MI = MO.getParent();
242	MachineBasicBlock *BB = MO.getParent()->getParent();
243	Register DstReg = MRI->createVirtualRegister(&SubRC);
244
245	if (MO.isReg()) {
246	unsigned ComposedSubIdx = TRI.composeSubRegIndices(MO.getSubReg(), SubIdx);
247	Register Reg = MO.getReg();
248	BuildMI(*BB, MI, MI->getDebugLoc(), TII.get(AMDGPU::COPY), DstReg)
249	.addReg(Reg, 0, ComposedSubIdx);
250
251	return MachineOperand::CreateReg(DstReg, MO.isDef(), MO.isImplicit(),
252	MO.isKill(), MO.isDead(), MO.isUndef(),
253	MO.isEarlyClobber(), 0, MO.isDebug(),
254	MO.isInternalRead());
255	}
256
257	assert(MO.isImm())(static_cast <bool> (MO.isImm()) ? void (0) : __assert_fail ("MO.isImm()", "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp" , 257, __extension__ __PRETTY_FUNCTION__));
258
259	APInt Imm(64, MO.getImm());
260
261	switch (SubIdx) {
262	default:
263	llvm_unreachable("do not know to split immediate with this sub index.")::llvm::llvm_unreachable_internal("do not know to split immediate with this sub index." , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 263 );
264	case AMDGPU::sub0:
265	return MachineOperand::CreateImm(Imm.getLoBits(32).getSExtValue());
266	case AMDGPU::sub1:
267	return MachineOperand::CreateImm(Imm.getHiBits(32).getSExtValue());
268	}
269	}
270
271	static unsigned getLogicalBitOpcode(unsigned Opc, bool Is64) {
272	switch (Opc) {
273	case AMDGPU::G_AND:
274	return Is64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32;
275	case AMDGPU::G_OR:
276	return Is64 ? AMDGPU::S_OR_B64 : AMDGPU::S_OR_B32;
277	case AMDGPU::G_XOR:
278	return Is64 ? AMDGPU::S_XOR_B64 : AMDGPU::S_XOR_B32;
279	default:
280	llvm_unreachable("not a bit op")::llvm::llvm_unreachable_internal("not a bit op", "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp" , 280);
281	}
282	}
283
284	bool AMDGPUInstructionSelector::selectG_AND_OR_XOR(MachineInstr &I) const {
285	Register DstReg = I.getOperand(0).getReg();
286	unsigned Size = RBI.getSizeInBits(DstReg, *MRI, TRI);
287
288	const RegisterBank DstRB = RBI.getRegBank(DstReg, MRI, TRI);
289	if (DstRB->getID() != AMDGPU::SGPRRegBankID &&
290	DstRB->getID() != AMDGPU::VCCRegBankID)
291	return false;
292
293	bool Is64 = Size > 32 \|\| (DstRB->getID() == AMDGPU::VCCRegBankID &&
294	STI.isWave64());
295	I.setDesc(TII.get(getLogicalBitOpcode(I.getOpcode(), Is64)));
296
297	// Dead implicit-def of scc
298	I.addOperand(MachineOperand::CreateReg(AMDGPU::SCC, true, // isDef
299	true, // isImp
300	false, // isKill
301	true)); // isDead
302	return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
303	}
304
305	bool AMDGPUInstructionSelector::selectG_ADD_SUB(MachineInstr &I) const {
306	MachineBasicBlock *BB = I.getParent();
307	MachineFunction *MF = BB->getParent();
308	Register DstReg = I.getOperand(0).getReg();
309	const DebugLoc &DL = I.getDebugLoc();
310	LLT Ty = MRI->getType(DstReg);
311	if (Ty.isVector())
312	return false;
313
314	unsigned Size = Ty.getSizeInBits();
315	const RegisterBank DstRB = RBI.getRegBank(DstReg, MRI, TRI);
316	const bool IsSALU = DstRB->getID() == AMDGPU::SGPRRegBankID;
317	const bool Sub = I.getOpcode() == TargetOpcode::G_SUB;
318
319	if (Size == 32) {
320	if (IsSALU) {
321	const unsigned Opc = Sub ? AMDGPU::S_SUB_U32 : AMDGPU::S_ADD_U32;
322	MachineInstr *Add =
323	BuildMI(*BB, &I, DL, TII.get(Opc), DstReg)
324	.add(I.getOperand(1))
325	.add(I.getOperand(2));
326	I.eraseFromParent();
327	return constrainSelectedInstRegOperands(*Add, TII, TRI, RBI);
328	}
329
330	if (STI.hasAddNoCarry()) {
331	const unsigned Opc = Sub ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_ADD_U32_e64;
332	I.setDesc(TII.get(Opc));
333	I.addOperand(*MF, MachineOperand::CreateImm(0));
334	I.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true));
335	return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
336	}
337
338	const unsigned Opc = Sub ? AMDGPU::V_SUB_CO_U32_e64 : AMDGPU::V_ADD_CO_U32_e64;
339
340	Register UnusedCarry = MRI->createVirtualRegister(TRI.getWaveMaskRegClass());
341	MachineInstr *Add
342	= BuildMI(*BB, &I, DL, TII.get(Opc), DstReg)
343	.addDef(UnusedCarry, RegState::Dead)
344	.add(I.getOperand(1))
345	.add(I.getOperand(2))
346	.addImm(0);
347	I.eraseFromParent();
348	return constrainSelectedInstRegOperands(*Add, TII, TRI, RBI);
349	}
350
351	assert(!Sub && "illegal sub should not reach here")(static_cast <bool> (!Sub && "illegal sub should not reach here" ) ? void (0) : __assert_fail ("!Sub && \"illegal sub should not reach here\"" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 351 , __extension__ __PRETTY_FUNCTION__));
352
353	const TargetRegisterClass &RC
354	= IsSALU ? AMDGPU::SReg_64_XEXECRegClass : AMDGPU::VReg_64RegClass;
355	const TargetRegisterClass &HalfRC
356	= IsSALU ? AMDGPU::SReg_32RegClass : AMDGPU::VGPR_32RegClass;
357
358	MachineOperand Lo1(getSubOperand64(I.getOperand(1), HalfRC, AMDGPU::sub0));
359	MachineOperand Lo2(getSubOperand64(I.getOperand(2), HalfRC, AMDGPU::sub0));
360	MachineOperand Hi1(getSubOperand64(I.getOperand(1), HalfRC, AMDGPU::sub1));
361	MachineOperand Hi2(getSubOperand64(I.getOperand(2), HalfRC, AMDGPU::sub1));
362
363	Register DstLo = MRI->createVirtualRegister(&HalfRC);
364	Register DstHi = MRI->createVirtualRegister(&HalfRC);
365
366	if (IsSALU) {
367	BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_ADD_U32), DstLo)
368	.add(Lo1)
369	.add(Lo2);
370	BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_ADDC_U32), DstHi)
371	.add(Hi1)
372	.add(Hi2);
373	} else {
374	const TargetRegisterClass *CarryRC = TRI.getWaveMaskRegClass();
375	Register CarryReg = MRI->createVirtualRegister(CarryRC);
376	BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_ADD_CO_U32_e64), DstLo)
377	.addDef(CarryReg)
378	.add(Lo1)
379	.add(Lo2)
380	.addImm(0);
381	MachineInstr Addc = BuildMI(BB, &I, DL, TII.get(AMDGPU::V_ADDC_U32_e64), DstHi)
382	.addDef(MRI->createVirtualRegister(CarryRC), RegState::Dead)
383	.add(Hi1)
384	.add(Hi2)
385	.addReg(CarryReg, RegState::Kill)
386	.addImm(0);
387
388	if (!constrainSelectedInstRegOperands(*Addc, TII, TRI, RBI))
389	return false;
390	}
391
392	BuildMI(*BB, &I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg)
393	.addReg(DstLo)
394	.addImm(AMDGPU::sub0)
395	.addReg(DstHi)
396	.addImm(AMDGPU::sub1);
397
398
399	if (!RBI.constrainGenericRegister(DstReg, RC, *MRI))
400	return false;
401
402	I.eraseFromParent();
403	return true;
404	}
405
406	bool AMDGPUInstructionSelector::selectG_UADDO_USUBO_UADDE_USUBE(
407	MachineInstr &I) const {
408	MachineBasicBlock *BB = I.getParent();
409	MachineFunction *MF = BB->getParent();
410	const DebugLoc &DL = I.getDebugLoc();
411	Register Dst0Reg = I.getOperand(0).getReg();
412	Register Dst1Reg = I.getOperand(1).getReg();
413	const bool IsAdd = I.getOpcode() == AMDGPU::G_UADDO \|\|
414	I.getOpcode() == AMDGPU::G_UADDE;
415	const bool HasCarryIn = I.getOpcode() == AMDGPU::G_UADDE \|\|
416	I.getOpcode() == AMDGPU::G_USUBE;
417
418	if (isVCC(Dst1Reg, *MRI)) {
419	unsigned NoCarryOpc =
420	IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
421	unsigned CarryOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64;
422	I.setDesc(TII.get(HasCarryIn ? CarryOpc : NoCarryOpc));
423	I.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true));
424	I.addOperand(*MF, MachineOperand::CreateImm(0));
425	return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
426	}
427
428	Register Src0Reg = I.getOperand(2).getReg();
429	Register Src1Reg = I.getOperand(3).getReg();
430
431	if (HasCarryIn) {
432	BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), AMDGPU::SCC)
433	.addReg(I.getOperand(4).getReg());
434	}
435
436	unsigned NoCarryOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32;
437	unsigned CarryOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32;
438
439	BuildMI(*BB, &I, DL, TII.get(HasCarryIn ? CarryOpc : NoCarryOpc), Dst0Reg)
440	.add(I.getOperand(2))
441	.add(I.getOperand(3));
442	BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), Dst1Reg)
443	.addReg(AMDGPU::SCC);
444
445	if (!MRI->getRegClassOrNull(Dst1Reg))
446	MRI->setRegClass(Dst1Reg, &AMDGPU::SReg_32RegClass);
447
448	if (!RBI.constrainGenericRegister(Dst0Reg, AMDGPU::SReg_32RegClass, *MRI) \|\|
449	!RBI.constrainGenericRegister(Src0Reg, AMDGPU::SReg_32RegClass, *MRI) \|\|
450	!RBI.constrainGenericRegister(Src1Reg, AMDGPU::SReg_32RegClass, *MRI))
451	return false;
452
453	if (HasCarryIn &&
454	!RBI.constrainGenericRegister(I.getOperand(4).getReg(),
455	AMDGPU::SReg_32RegClass, *MRI))
456	return false;
457
458	I.eraseFromParent();
459	return true;
460	}
461
462	bool AMDGPUInstructionSelector::selectG_AMDGPU_MAD_64_32(
463	MachineInstr &I) const {
464	MachineBasicBlock *BB = I.getParent();
465	MachineFunction *MF = BB->getParent();
466	const bool IsUnsigned = I.getOpcode() == AMDGPU::G_AMDGPU_MAD_U64_U32;
467
468	unsigned Opc;
469	if (Subtarget->hasMADIntraFwdBug())
470	Opc = IsUnsigned ? AMDGPU::V_MAD_U64_U32_gfx11_e64
471	: AMDGPU::V_MAD_I64_I32_gfx11_e64;
472	else
473	Opc = IsUnsigned ? AMDGPU::V_MAD_U64_U32_e64 : AMDGPU::V_MAD_I64_I32_e64;
474	I.setDesc(TII.get(Opc));
475	I.addOperand(*MF, MachineOperand::CreateImm(0));
476	I.addImplicitDefUseOperands(*MF);
477	return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
478	}
479
480	// TODO: We should probably legalize these to only using 32-bit results.
481	bool AMDGPUInstructionSelector::selectG_EXTRACT(MachineInstr &I) const {
482	MachineBasicBlock *BB = I.getParent();
483	Register DstReg = I.getOperand(0).getReg();
484	Register SrcReg = I.getOperand(1).getReg();
485	LLT DstTy = MRI->getType(DstReg);
486	LLT SrcTy = MRI->getType(SrcReg);
487	const unsigned SrcSize = SrcTy.getSizeInBits();
488	unsigned DstSize = DstTy.getSizeInBits();
489
490	// TODO: Should handle any multiple of 32 offset.
491	unsigned Offset = I.getOperand(2).getImm();
492	if (Offset % 32 != 0 \|\| DstSize > 128)
493	return false;
494
495	// 16-bit operations really use 32-bit registers.
496	// FIXME: Probably should not allow 16-bit G_EXTRACT results.
497	if (DstSize == 16)
498	DstSize = 32;
499
500	const TargetRegisterClass *DstRC =
501	TRI.getConstrainedRegClassForOperand(I.getOperand(0), *MRI);
502	if (!DstRC \|\| !RBI.constrainGenericRegister(DstReg, DstRC, MRI))
503	return false;
504
505	const RegisterBank SrcBank = RBI.getRegBank(SrcReg, MRI, TRI);
506	const TargetRegisterClass *SrcRC =
507	TRI.getRegClassForSizeOnBank(SrcSize, *SrcBank);
508	if (!SrcRC)
509	return false;
510	unsigned SubReg = SIRegisterInfo::getSubRegFromChannel(Offset / 32,
511	DstSize / 32);
512	SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubReg);
513	if (!SrcRC)
514	return false;
515
516	SrcReg = constrainOperandRegClass(MF, TRI, MRI, TII, RBI, I,
517	*SrcRC, I.getOperand(1));
518	const DebugLoc &DL = I.getDebugLoc();
519	BuildMI(*BB, &I, DL, TII.get(TargetOpcode::COPY), DstReg)
520	.addReg(SrcReg, 0, SubReg);
521
522	I.eraseFromParent();
523	return true;
524	}
525
526	bool AMDGPUInstructionSelector::selectG_MERGE_VALUES(MachineInstr &MI) const {
527	MachineBasicBlock *BB = MI.getParent();
528	Register DstReg = MI.getOperand(0).getReg();
529	LLT DstTy = MRI->getType(DstReg);
530	LLT SrcTy = MRI->getType(MI.getOperand(1).getReg());
531
532	const unsigned SrcSize = SrcTy.getSizeInBits();
533	if (SrcSize < 32)
534	return selectImpl(MI, *CoverageInfo);
535
536	const DebugLoc &DL = MI.getDebugLoc();
537	const RegisterBank DstBank = RBI.getRegBank(DstReg, MRI, TRI);
538	const unsigned DstSize = DstTy.getSizeInBits();
539	const TargetRegisterClass *DstRC =
540	TRI.getRegClassForSizeOnBank(DstSize, *DstBank);
541	if (!DstRC)
542	return false;
543
544	ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(DstRC, SrcSize / 8);
545	MachineInstrBuilder MIB =
546	BuildMI(*BB, &MI, DL, TII.get(TargetOpcode::REG_SEQUENCE), DstReg);
547	for (int I = 0, E = MI.getNumOperands() - 1; I != E; ++I) {
548	MachineOperand &Src = MI.getOperand(I + 1);
549	MIB.addReg(Src.getReg(), getUndefRegState(Src.isUndef()));
550	MIB.addImm(SubRegs[I]);
551
552	const TargetRegisterClass *SrcRC
553	= TRI.getConstrainedRegClassForOperand(Src, *MRI);
554	if (SrcRC && !RBI.constrainGenericRegister(Src.getReg(), SrcRC, MRI))
555	return false;
556	}
557
558	if (!RBI.constrainGenericRegister(DstReg, DstRC, MRI))
559	return false;
560
561	MI.eraseFromParent();
562	return true;
563	}
564
565	bool AMDGPUInstructionSelector::selectG_UNMERGE_VALUES(MachineInstr &MI) const {
566	MachineBasicBlock *BB = MI.getParent();
567	const int NumDst = MI.getNumOperands() - 1;
568
569	MachineOperand &Src = MI.getOperand(NumDst);
570
571	Register SrcReg = Src.getReg();
572	Register DstReg0 = MI.getOperand(0).getReg();
573	LLT DstTy = MRI->getType(DstReg0);
574	LLT SrcTy = MRI->getType(SrcReg);
575
576	const unsigned DstSize = DstTy.getSizeInBits();
577	const unsigned SrcSize = SrcTy.getSizeInBits();
578	const DebugLoc &DL = MI.getDebugLoc();
579	const RegisterBank SrcBank = RBI.getRegBank(SrcReg, MRI, TRI);
580
581	const TargetRegisterClass *SrcRC =
582	TRI.getRegClassForSizeOnBank(SrcSize, *SrcBank);
583	if (!SrcRC \|\| !RBI.constrainGenericRegister(SrcReg, SrcRC, MRI))
584	return false;
585
586	// Note we could have mixed SGPR and VGPR destination banks for an SGPR
587	// source, and this relies on the fact that the same subregister indices are
588	// used for both.
589	ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(SrcRC, DstSize / 8);
590	for (int I = 0, E = NumDst; I != E; ++I) {
591	MachineOperand &Dst = MI.getOperand(I);
592	BuildMI(*BB, &MI, DL, TII.get(TargetOpcode::COPY), Dst.getReg())
593	.addReg(SrcReg, 0, SubRegs[I]);
594
595	// Make sure the subregister index is valid for the source register.
596	SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubRegs[I]);
597	if (!SrcRC \|\| !RBI.constrainGenericRegister(SrcReg, SrcRC, MRI))
598	return false;
599
600	const TargetRegisterClass *DstRC =
601	TRI.getConstrainedRegClassForOperand(Dst, *MRI);
602	if (DstRC && !RBI.constrainGenericRegister(Dst.getReg(), DstRC, MRI))
603	return false;
604	}
605
606	MI.eraseFromParent();
607	return true;
608	}
609
610	bool AMDGPUInstructionSelector::selectG_BUILD_VECTOR(MachineInstr &MI) const {
611	assert(MI.getOpcode() == AMDGPU::G_BUILD_VECTOR_TRUNC \|\|(static_cast <bool> (MI.getOpcode() == AMDGPU::G_BUILD_VECTOR_TRUNC \|\| MI.getOpcode() == AMDGPU::G_BUILD_VECTOR) ? void (0) : __assert_fail ("MI.getOpcode() == AMDGPU::G_BUILD_VECTOR_TRUNC \|\| MI.getOpcode() == AMDGPU::G_BUILD_VECTOR" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 612 , __extension__ __PRETTY_FUNCTION__))
612	MI.getOpcode() == AMDGPU::G_BUILD_VECTOR)(static_cast <bool> (MI.getOpcode() == AMDGPU::G_BUILD_VECTOR_TRUNC \|\| MI.getOpcode() == AMDGPU::G_BUILD_VECTOR) ? void (0) : __assert_fail ("MI.getOpcode() == AMDGPU::G_BUILD_VECTOR_TRUNC \|\| MI.getOpcode() == AMDGPU::G_BUILD_VECTOR" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 612 , __extension__ __PRETTY_FUNCTION__));
613
614	Register Src0 = MI.getOperand(1).getReg();
615	Register Src1 = MI.getOperand(2).getReg();
616	LLT SrcTy = MRI->getType(Src0);
617	const unsigned SrcSize = SrcTy.getSizeInBits();
618
619	// BUILD_VECTOR with >=32 bits source is handled by MERGE_VALUE.
620	if (MI.getOpcode() == AMDGPU::G_BUILD_VECTOR && SrcSize >= 32) {
621	return selectG_MERGE_VALUES(MI);
622	}
623
624	// Selection logic below is for V2S16 only.
625	// For G_BUILD_VECTOR_TRUNC, additionally check that the operands are s32.
626	Register Dst = MI.getOperand(0).getReg();
627	if (MRI->getType(Dst) != LLT::fixed_vector(2, 16) \|\|
628	(MI.getOpcode() == AMDGPU::G_BUILD_VECTOR_TRUNC &&
629	SrcTy != LLT::scalar(32)))
630	return selectImpl(MI, *CoverageInfo);
631
632	const RegisterBank DstBank = RBI.getRegBank(Dst, MRI, TRI);
633	if (DstBank->getID() == AMDGPU::AGPRRegBankID)
634	return false;
635
636	assert(DstBank->getID() == AMDGPU::SGPRRegBankID \|\|(static_cast <bool> (DstBank->getID() == AMDGPU::SGPRRegBankID \|\| DstBank->getID() == AMDGPU::VGPRRegBankID) ? void (0) : __assert_fail ("DstBank->getID() == AMDGPU::SGPRRegBankID \|\| DstBank->getID() == AMDGPU::VGPRRegBankID" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 637 , __extension__ __PRETTY_FUNCTION__))
637	DstBank->getID() == AMDGPU::VGPRRegBankID)(static_cast <bool> (DstBank->getID() == AMDGPU::SGPRRegBankID \|\| DstBank->getID() == AMDGPU::VGPRRegBankID) ? void (0) : __assert_fail ("DstBank->getID() == AMDGPU::SGPRRegBankID \|\| DstBank->getID() == AMDGPU::VGPRRegBankID" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 637 , __extension__ __PRETTY_FUNCTION__));
638	const bool IsVector = DstBank->getID() == AMDGPU::VGPRRegBankID;
639
640	const DebugLoc &DL = MI.getDebugLoc();
641	MachineBasicBlock *BB = MI.getParent();
642
643	// First, before trying TableGen patterns, check if both sources are
644	// constants. In those cases, we can trivially compute the final constant
645	// and emit a simple move.
646	auto ConstSrc1 = getAnyConstantVRegValWithLookThrough(Src1, *MRI, true, true);
647	if (ConstSrc1) {
648	auto ConstSrc0 =
649	getAnyConstantVRegValWithLookThrough(Src0, *MRI, true, true);
650	if (ConstSrc0) {
651	const int64_t K0 = ConstSrc0->Value.getSExtValue();
652	const int64_t K1 = ConstSrc1->Value.getSExtValue();
653	uint32_t Lo16 = static_cast<uint32_t>(K0) & 0xffff;
654	uint32_t Hi16 = static_cast<uint32_t>(K1) & 0xffff;
655	uint32_t Imm = Lo16 \| (Hi16 << 16);
656
657	// VALU
658	if (IsVector) {
659	BuildMI(*BB, &MI, DL, TII.get(AMDGPU::V_MOV_B32_e32), Dst).addImm(Imm);
660	MI.eraseFromParent();
661	return RBI.constrainGenericRegister(Dst, AMDGPU::VGPR_32RegClass, *MRI);
662	}
663
664	// SALU
665	BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_MOV_B32), Dst).addImm(Imm);
666	MI.eraseFromParent();
667	return RBI.constrainGenericRegister(Dst, AMDGPU::SReg_32RegClass, *MRI);
668	}
669	}
670
671	// Now try TableGen patterns.
672	if (selectImpl(MI, *CoverageInfo))
673	return true;
674
675	// TODO: This should probably be a combine somewhere
676	// (build_vector $src0, undef) -> copy $src0
677	MachineInstr Src1Def = getDefIgnoringCopies(Src1, MRI);
678	if (Src1Def->getOpcode() == AMDGPU::G_IMPLICIT_DEF) {
679	MI.setDesc(TII.get(AMDGPU::COPY));
680	MI.removeOperand(2);
681	const auto &RC =
682	IsVector ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass;
683	return RBI.constrainGenericRegister(Dst, RC, *MRI) &&
684	RBI.constrainGenericRegister(Src0, RC, *MRI);
685	}
686
687	// TODO: Can be improved?
688	if (IsVector) {
689	Register TmpReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
690	auto MIB = BuildMI(*BB, MI, DL, TII.get(AMDGPU::V_AND_B32_e32), TmpReg)
691	.addImm(0xFFFF)
692	.addReg(Src0);
693	if (!constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI))
694	return false;
695
696	MIB = BuildMI(*BB, MI, DL, TII.get(AMDGPU::V_LSHL_OR_B32_e64), Dst)
697	.addReg(Src1)
698	.addImm(16)
699	.addReg(TmpReg);
700	if (!constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI))
701	return false;
702
703	MI.eraseFromParent();
704	return true;
705	}
706
707	Register ShiftSrc0;
708	Register ShiftSrc1;
709
710	// With multiple uses of the shift, this will duplicate the shift and
711	// increase register pressure.
712	//
713	// (build_vector (lshr_oneuse $src0, 16), (lshr_oneuse $src1, 16)
714	// => (S_PACK_HH_B32_B16 $src0, $src1)
715	// (build_vector (lshr_oneuse SReg_32:$src0, 16), $src1)
716	// => (S_PACK_HL_B32_B16 $src0, $src1)
717	// (build_vector $src0, (lshr_oneuse SReg_32:$src1, 16))
718	// => (S_PACK_LH_B32_B16 $src0, $src1)
719	// (build_vector $src0, $src1)
720	// => (S_PACK_LL_B32_B16 $src0, $src1)
721
722	bool Shift0 = mi_match(
723	Src0, *MRI, m_OneUse(m_GLShr(m_Reg(ShiftSrc0), m_SpecificICst(16))));
724
725	bool Shift1 = mi_match(
726	Src1, *MRI, m_OneUse(m_GLShr(m_Reg(ShiftSrc1), m_SpecificICst(16))));
727
728	unsigned Opc = AMDGPU::S_PACK_LL_B32_B16;
729	if (Shift0 && Shift1) {
730	Opc = AMDGPU::S_PACK_HH_B32_B16;
731	MI.getOperand(1).setReg(ShiftSrc0);
732	MI.getOperand(2).setReg(ShiftSrc1);
733	} else if (Shift1) {
734	Opc = AMDGPU::S_PACK_LH_B32_B16;
735	MI.getOperand(2).setReg(ShiftSrc1);
736	} else if (Shift0) {
737	auto ConstSrc1 =
738	getAnyConstantVRegValWithLookThrough(Src1, *MRI, true, true);
739	if (ConstSrc1 && ConstSrc1->Value == 0) {
740	// build_vector_trunc (lshr $src0, 16), 0 -> s_lshr_b32 $src0, 16
741	auto MIB = BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_LSHR_B32), Dst)
742	.addReg(ShiftSrc0)
743	.addImm(16);
744
745	MI.eraseFromParent();
746	return constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
747	}
748	if (STI.hasSPackHL()) {
749	Opc = AMDGPU::S_PACK_HL_B32_B16;
750	MI.getOperand(1).setReg(ShiftSrc0);
751	}
752	}
753
754	MI.setDesc(TII.get(Opc));
755	return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
756	}
757
758	bool AMDGPUInstructionSelector::selectG_PTR_ADD(MachineInstr &I) const {
759	return selectG_ADD_SUB(I);
760	}
761
762	bool AMDGPUInstructionSelector::selectG_IMPLICIT_DEF(MachineInstr &I) const {
763	const MachineOperand &MO = I.getOperand(0);
764
765	// FIXME: Interface for getConstrainedRegClassForOperand needs work. The
766	// regbank check here is to know why getConstrainedRegClassForOperand failed.
767	const TargetRegisterClass RC = TRI.getConstrainedRegClassForOperand(MO, MRI);
768	if ((!RC && !MRI->getRegBankOrNull(MO.getReg())) \|\|
769	(RC && RBI.constrainGenericRegister(MO.getReg(), RC, MRI))) {
770	I.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF));
771	return true;
772	}
773
774	return false;
775	}
776
777	bool AMDGPUInstructionSelector::selectG_INSERT(MachineInstr &I) const {
778	MachineBasicBlock *BB = I.getParent();
779
780	Register DstReg = I.getOperand(0).getReg();
781	Register Src0Reg = I.getOperand(1).getReg();
782	Register Src1Reg = I.getOperand(2).getReg();
783	LLT Src1Ty = MRI->getType(Src1Reg);
784
785	unsigned DstSize = MRI->getType(DstReg).getSizeInBits();
786	unsigned InsSize = Src1Ty.getSizeInBits();
787
788	int64_t Offset = I.getOperand(3).getImm();
789
790	// FIXME: These cases should have been illegal and unnecessary to check here.
791	if (Offset % 32 != 0 \|\| InsSize % 32 != 0)
792	return false;
793
794	// Currently not handled by getSubRegFromChannel.
795	if (InsSize > 128)
796	return false;
797
798	unsigned SubReg = TRI.getSubRegFromChannel(Offset / 32, InsSize / 32);
799	if (SubReg == AMDGPU::NoSubRegister)
800	return false;
801
802	const RegisterBank DstBank = RBI.getRegBank(DstReg, MRI, TRI);
803	const TargetRegisterClass *DstRC =
804	TRI.getRegClassForSizeOnBank(DstSize, *DstBank);
805	if (!DstRC)
806	return false;
807
808	const RegisterBank Src0Bank = RBI.getRegBank(Src0Reg, MRI, TRI);
809	const RegisterBank Src1Bank = RBI.getRegBank(Src1Reg, MRI, TRI);
810	const TargetRegisterClass *Src0RC =
811	TRI.getRegClassForSizeOnBank(DstSize, *Src0Bank);
812	const TargetRegisterClass *Src1RC =
813	TRI.getRegClassForSizeOnBank(InsSize, *Src1Bank);
814
815	// Deal with weird cases where the class only partially supports the subreg
816	// index.
817	Src0RC = TRI.getSubClassWithSubReg(Src0RC, SubReg);
818	if (!Src0RC \|\| !Src1RC)
819	return false;
820
821	if (!RBI.constrainGenericRegister(DstReg, DstRC, MRI) \|\|
822	!RBI.constrainGenericRegister(Src0Reg, Src0RC, MRI) \|\|
823	!RBI.constrainGenericRegister(Src1Reg, Src1RC, MRI))
824	return false;
825
826	const DebugLoc &DL = I.getDebugLoc();
827	BuildMI(*BB, &I, DL, TII.get(TargetOpcode::INSERT_SUBREG), DstReg)
828	.addReg(Src0Reg)
829	.addReg(Src1Reg)
830	.addImm(SubReg);
831
832	I.eraseFromParent();
833	return true;
834	}
835
836	bool AMDGPUInstructionSelector::selectG_SBFX_UBFX(MachineInstr &MI) const {
837	Register DstReg = MI.getOperand(0).getReg();
838	Register SrcReg = MI.getOperand(1).getReg();
839	Register OffsetReg = MI.getOperand(2).getReg();
840	Register WidthReg = MI.getOperand(3).getReg();
841
842	assert(RBI.getRegBank(DstReg, MRI, TRI)->getID() == AMDGPU::VGPRRegBankID &&(static_cast <bool> (RBI.getRegBank(DstReg, MRI, TRI)-> getID() == AMDGPU::VGPRRegBankID && "scalar BFX instructions are expanded in regbankselect" ) ? void (0) : __assert_fail ("RBI.getRegBank(DstReg, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID && \"scalar BFX instructions are expanded in regbankselect\"" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 843 , __extension__ __PRETTY_FUNCTION__))
843	"scalar BFX instructions are expanded in regbankselect")(static_cast <bool> (RBI.getRegBank(DstReg, MRI, TRI)-> getID() == AMDGPU::VGPRRegBankID && "scalar BFX instructions are expanded in regbankselect" ) ? void (0) : __assert_fail ("RBI.getRegBank(DstReg, MRI, TRI)->getID() == AMDGPU::VGPRRegBankID && \"scalar BFX instructions are expanded in regbankselect\"" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 843 , __extension__ __PRETTY_FUNCTION__));
844	assert(MRI->getType(MI.getOperand(0).getReg()).getSizeInBits() == 32 &&(static_cast <bool> (MRI->getType(MI.getOperand(0).getReg ()).getSizeInBits() == 32 && "64-bit vector BFX instructions are expanded in regbankselect" ) ? void (0) : __assert_fail ("MRI->getType(MI.getOperand(0).getReg()).getSizeInBits() == 32 && \"64-bit vector BFX instructions are expanded in regbankselect\"" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 845 , __extension__ __PRETTY_FUNCTION__))
845	"64-bit vector BFX instructions are expanded in regbankselect")(static_cast <bool> (MRI->getType(MI.getOperand(0).getReg ()).getSizeInBits() == 32 && "64-bit vector BFX instructions are expanded in regbankselect" ) ? void (0) : __assert_fail ("MRI->getType(MI.getOperand(0).getReg()).getSizeInBits() == 32 && \"64-bit vector BFX instructions are expanded in regbankselect\"" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 845 , __extension__ __PRETTY_FUNCTION__));
846
847	const DebugLoc &DL = MI.getDebugLoc();
848	MachineBasicBlock *MBB = MI.getParent();
849
850	bool IsSigned = MI.getOpcode() == TargetOpcode::G_SBFX;
851	unsigned Opc = IsSigned ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64;
852	auto MIB = BuildMI(*MBB, &MI, DL, TII.get(Opc), DstReg)
853	.addReg(SrcReg)
854	.addReg(OffsetReg)
855	.addReg(WidthReg);
856	MI.eraseFromParent();
857	return constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
858	}
859
860	bool AMDGPUInstructionSelector::selectInterpP1F16(MachineInstr &MI) const {
861	if (STI.getLDSBankCount() != 16)
862	return selectImpl(MI, *CoverageInfo);
863
864	Register Dst = MI.getOperand(0).getReg();
865	Register Src0 = MI.getOperand(2).getReg();
866	Register M0Val = MI.getOperand(6).getReg();
867	if (!RBI.constrainGenericRegister(M0Val, AMDGPU::SReg_32RegClass, *MRI) \|\|
868	!RBI.constrainGenericRegister(Dst, AMDGPU::VGPR_32RegClass, *MRI) \|\|
869	!RBI.constrainGenericRegister(Src0, AMDGPU::VGPR_32RegClass, *MRI))
870	return false;
871
872	// This requires 2 instructions. It is possible to write a pattern to support
873	// this, but the generated isel emitter doesn't correctly deal with multiple
874	// output instructions using the same physical register input. The copy to m0
875	// is incorrectly placed before the second instruction.
876	//
877	// TODO: Match source modifiers.
878
879	Register InterpMov = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
880	const DebugLoc &DL = MI.getDebugLoc();
881	MachineBasicBlock *MBB = MI.getParent();
882
883	BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0)
884	.addReg(M0Val);
885	BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::V_INTERP_MOV_F32), InterpMov)
886	.addImm(2)
887	.addImm(MI.getOperand(4).getImm()) // $attr
888	.addImm(MI.getOperand(3).getImm()); // $attrchan
889
890	BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::V_INTERP_P1LV_F16), Dst)
891	.addImm(0) // $src0_modifiers
892	.addReg(Src0) // $src0
893	.addImm(MI.getOperand(4).getImm()) // $attr
894	.addImm(MI.getOperand(3).getImm()) // $attrchan
895	.addImm(0) // $src2_modifiers
896	.addReg(InterpMov) // $src2 - 2 f16 values selected by high
897	.addImm(MI.getOperand(5).getImm()) // $high
898	.addImm(0) // $clamp
899	.addImm(0); // $omod
900
901	MI.eraseFromParent();
902	return true;
903	}
904
905	// Writelane is special in that it can use SGPR and M0 (which would normally
906	// count as using the constant bus twice - but in this case it is allowed since
907	// the lane selector doesn't count as a use of the constant bus). However, it is
908	// still required to abide by the 1 SGPR rule. Fix this up if we might have
909	// multiple SGPRs.
910	bool AMDGPUInstructionSelector::selectWritelane(MachineInstr &MI) const {
911	// With a constant bus limit of at least 2, there's no issue.
912	if (STI.getConstantBusLimit(AMDGPU::V_WRITELANE_B32) > 1)
913	return selectImpl(MI, *CoverageInfo);
914
915	MachineBasicBlock *MBB = MI.getParent();
916	const DebugLoc &DL = MI.getDebugLoc();
917	Register VDst = MI.getOperand(0).getReg();
918	Register Val = MI.getOperand(2).getReg();
919	Register LaneSelect = MI.getOperand(3).getReg();
920	Register VDstIn = MI.getOperand(4).getReg();
921
922	auto MIB = BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::V_WRITELANE_B32), VDst);
923
924	std::optional<ValueAndVReg> ConstSelect =
925	getIConstantVRegValWithLookThrough(LaneSelect, *MRI);
926	if (ConstSelect) {
927	// The selector has to be an inline immediate, so we can use whatever for
928	// the other operands.
929	MIB.addReg(Val);
930	MIB.addImm(ConstSelect->Value.getSExtValue() &
931	maskTrailingOnes<uint64_t>(STI.getWavefrontSizeLog2()));
932	} else {
933	std::optional<ValueAndVReg> ConstVal =
934	getIConstantVRegValWithLookThrough(Val, *MRI);
935
936	// If the value written is an inline immediate, we can get away without a
937	// copy to m0.
938	if (ConstVal && AMDGPU::isInlinableLiteral32(ConstVal->Value.getSExtValue(),
939	STI.hasInv2PiInlineImm())) {
940	MIB.addImm(ConstVal->Value.getSExtValue());
941	MIB.addReg(LaneSelect);
942	} else {
943	MIB.addReg(Val);
944
945	// If the lane selector was originally in a VGPR and copied with
946	// readfirstlane, there's a hazard to read the same SGPR from the
947	// VALU. Constrain to a different SGPR to help avoid needing a nop later.
948	RBI.constrainGenericRegister(LaneSelect, AMDGPU::SReg_32_XM0RegClass, *MRI);
949
950	BuildMI(MBB, MIB, DL, TII.get(AMDGPU::COPY), AMDGPU::M0)
951	.addReg(LaneSelect);
952	MIB.addReg(AMDGPU::M0);
953	}
954	}
955
956	MIB.addReg(VDstIn);
957
958	MI.eraseFromParent();
959	return constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
960	}
961
962	// We need to handle this here because tablegen doesn't support matching
963	// instructions with multiple outputs.
964	bool AMDGPUInstructionSelector::selectDivScale(MachineInstr &MI) const {
965	Register Dst0 = MI.getOperand(0).getReg();
966	Register Dst1 = MI.getOperand(1).getReg();
967
968	LLT Ty = MRI->getType(Dst0);
969	unsigned Opc;
970	if (Ty == LLT::scalar(32))
971	Opc = AMDGPU::V_DIV_SCALE_F32_e64;
972	else if (Ty == LLT::scalar(64))
973	Opc = AMDGPU::V_DIV_SCALE_F64_e64;
974	else
975	return false;
976
977	// TODO: Match source modifiers.
978
979	const DebugLoc &DL = MI.getDebugLoc();
980	MachineBasicBlock *MBB = MI.getParent();
981
982	Register Numer = MI.getOperand(3).getReg();
983	Register Denom = MI.getOperand(4).getReg();
984	unsigned ChooseDenom = MI.getOperand(5).getImm();
985
986	Register Src0 = ChooseDenom != 0 ? Numer : Denom;
987
988	auto MIB = BuildMI(*MBB, &MI, DL, TII.get(Opc), Dst0)
989	.addDef(Dst1)
990	.addImm(0) // $src0_modifiers
991	.addUse(Src0) // $src0
992	.addImm(0) // $src1_modifiers
993	.addUse(Denom) // $src1
994	.addImm(0) // $src2_modifiers
995	.addUse(Numer) // $src2
996	.addImm(0) // $clamp
997	.addImm(0); // $omod
998
999	MI.eraseFromParent();
1000	return constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
1001	}
1002
1003	bool AMDGPUInstructionSelector::selectG_INTRINSIC(MachineInstr &I) const {
1004	unsigned IntrinsicID = I.getIntrinsicID();
1005	switch (IntrinsicID) {
1006	case Intrinsic::amdgcn_if_break: {
1007	MachineBasicBlock *BB = I.getParent();
1008
1009	// FIXME: Manually selecting to avoid dealing with the SReg_1 trick
1010	// SelectionDAG uses for wave32 vs wave64.
1011	BuildMI(*BB, &I, I.getDebugLoc(), TII.get(AMDGPU::SI_IF_BREAK))
1012	.add(I.getOperand(0))
1013	.add(I.getOperand(2))
1014	.add(I.getOperand(3));
1015
1016	Register DstReg = I.getOperand(0).getReg();
1017	Register Src0Reg = I.getOperand(2).getReg();
1018	Register Src1Reg = I.getOperand(3).getReg();
1019
1020	I.eraseFromParent();
1021
1022	for (Register Reg : { DstReg, Src0Reg, Src1Reg })
1023	MRI->setRegClass(Reg, TRI.getWaveMaskRegClass());
1024
1025	return true;
1026	}
1027	case Intrinsic::amdgcn_interp_p1_f16:
1028	return selectInterpP1F16(I);
1029	case Intrinsic::amdgcn_wqm:
1030	return constrainCopyLikeIntrin(I, AMDGPU::WQM);
1031	case Intrinsic::amdgcn_softwqm:
1032	return constrainCopyLikeIntrin(I, AMDGPU::SOFT_WQM);
1033	case Intrinsic::amdgcn_strict_wwm:
1034	case Intrinsic::amdgcn_wwm:
1035	return constrainCopyLikeIntrin(I, AMDGPU::STRICT_WWM);
1036	case Intrinsic::amdgcn_strict_wqm:
1037	return constrainCopyLikeIntrin(I, AMDGPU::STRICT_WQM);
1038	case Intrinsic::amdgcn_writelane:
1039	return selectWritelane(I);
1040	case Intrinsic::amdgcn_div_scale:
1041	return selectDivScale(I);
1042	case Intrinsic::amdgcn_icmp:
1043	case Intrinsic::amdgcn_fcmp:
1044	if (selectImpl(I, *CoverageInfo))
1045	return true;
1046	return selectIntrinsicCmp(I);
1047	case Intrinsic::amdgcn_ballot:
1048	return selectBallot(I);
1049	case Intrinsic::amdgcn_inverse_ballot:
1050	return selectInverseBallot(I);
1051	case Intrinsic::amdgcn_reloc_constant:
1052	return selectRelocConstant(I);
1053	case Intrinsic::amdgcn_groupstaticsize:
1054	return selectGroupStaticSize(I);
1055	case Intrinsic::returnaddress:
1056	return selectReturnAddress(I);
1057	case Intrinsic::amdgcn_smfmac_f32_16x16x32_f16:
1058	case Intrinsic::amdgcn_smfmac_f32_32x32x16_f16:
1059	case Intrinsic::amdgcn_smfmac_f32_16x16x32_bf16:
1060	case Intrinsic::amdgcn_smfmac_f32_32x32x16_bf16:
1061	case Intrinsic::amdgcn_smfmac_i32_16x16x64_i8:
1062	case Intrinsic::amdgcn_smfmac_i32_32x32x32_i8:
1063	case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_bf8:
1064	case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_fp8:
1065	case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_bf8:
1066	case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_fp8:
1067	case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_bf8:
1068	case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_fp8:
1069	case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_bf8:
1070	case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_fp8:
1071	return selectSMFMACIntrin(I);
1072	default:
1073	return selectImpl(I, *CoverageInfo);
1074	}
1075	}
1076
1077	static int getV_CMPOpcode(CmpInst::Predicate P, unsigned Size,
1078	const GCNSubtarget &ST) {
1079	if (Size != 16 && Size != 32 && Size != 64)
1080	return -1;
1081
1082	if (Size == 16 && !ST.has16BitInsts())
1083	return -1;
1084
1085	const auto Select = [&](unsigned S16Opc, unsigned TrueS16Opc, unsigned S32Opc,
1086	unsigned S64Opc) {
1087	if (Size == 16)
1088	return ST.hasTrue16BitInsts() ? TrueS16Opc : S16Opc;
1089	if (Size == 32)
1090	return S32Opc;
1091	return S64Opc;
1092	};
1093
1094	switch (P) {
1095	default:
1096	llvm_unreachable("Unknown condition code!")::llvm::llvm_unreachable_internal("Unknown condition code!", "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp" , 1096);
1097	case CmpInst::ICMP_NE:
1098	return Select(AMDGPU::V_CMP_NE_U16_e64, AMDGPU::V_CMP_NE_U16_t16_e64,
1099	AMDGPU::V_CMP_NE_U32_e64, AMDGPU::V_CMP_NE_U64_e64);
1100	case CmpInst::ICMP_EQ:
1101	return Select(AMDGPU::V_CMP_EQ_U16_e64, AMDGPU::V_CMP_EQ_U16_t16_e64,
1102	AMDGPU::V_CMP_EQ_U32_e64, AMDGPU::V_CMP_EQ_U64_e64);
1103	case CmpInst::ICMP_SGT:
1104	return Select(AMDGPU::V_CMP_GT_I16_e64, AMDGPU::V_CMP_GT_I16_t16_e64,
1105	AMDGPU::V_CMP_GT_I32_e64, AMDGPU::V_CMP_GT_I64_e64);
1106	case CmpInst::ICMP_SGE:
1107	return Select(AMDGPU::V_CMP_GE_I16_e64, AMDGPU::V_CMP_GE_I16_t16_e64,
1108	AMDGPU::V_CMP_GE_I32_e64, AMDGPU::V_CMP_GE_I64_e64);
1109	case CmpInst::ICMP_SLT:
1110	return Select(AMDGPU::V_CMP_LT_I16_e64, AMDGPU::V_CMP_LT_I16_t16_e64,
1111	AMDGPU::V_CMP_LT_I32_e64, AMDGPU::V_CMP_LT_I64_e64);
1112	case CmpInst::ICMP_SLE:
1113	return Select(AMDGPU::V_CMP_LE_I16_e64, AMDGPU::V_CMP_LE_I16_t16_e64,
1114	AMDGPU::V_CMP_LE_I32_e64, AMDGPU::V_CMP_LE_I64_e64);
1115	case CmpInst::ICMP_UGT:
1116	return Select(AMDGPU::V_CMP_GT_U16_e64, AMDGPU::V_CMP_GT_U16_t16_e64,
1117	AMDGPU::V_CMP_GT_U32_e64, AMDGPU::V_CMP_GT_U64_e64);
1118	case CmpInst::ICMP_UGE:
1119	return Select(AMDGPU::V_CMP_GE_U16_e64, AMDGPU::V_CMP_GE_U16_t16_e64,
1120	AMDGPU::V_CMP_GE_U32_e64, AMDGPU::V_CMP_GE_U64_e64);
1121	case CmpInst::ICMP_ULT:
1122	return Select(AMDGPU::V_CMP_LT_U16_e64, AMDGPU::V_CMP_LT_U16_t16_e64,
1123	AMDGPU::V_CMP_LT_U32_e64, AMDGPU::V_CMP_LT_U64_e64);
1124	case CmpInst::ICMP_ULE:
1125	return Select(AMDGPU::V_CMP_LE_U16_e64, AMDGPU::V_CMP_LE_U16_t16_e64,
1126	AMDGPU::V_CMP_LE_U32_e64, AMDGPU::V_CMP_LE_U64_e64);
1127
1128	case CmpInst::FCMP_OEQ:
1129	return Select(AMDGPU::V_CMP_EQ_F16_e64, AMDGPU::V_CMP_EQ_F16_t16_e64,
1130	AMDGPU::V_CMP_EQ_F32_e64, AMDGPU::V_CMP_EQ_F64_e64);
1131	case CmpInst::FCMP_OGT:
1132	return Select(AMDGPU::V_CMP_GT_F16_e64, AMDGPU::V_CMP_GT_F16_t16_e64,
1133	AMDGPU::V_CMP_GT_F32_e64, AMDGPU::V_CMP_GT_F64_e64);
1134	case CmpInst::FCMP_OGE:
1135	return Select(AMDGPU::V_CMP_GE_F16_e64, AMDGPU::V_CMP_GE_F16_t16_e64,
1136	AMDGPU::V_CMP_GE_F32_e64, AMDGPU::V_CMP_GE_F64_e64);
1137	case CmpInst::FCMP_OLT:
1138	return Select(AMDGPU::V_CMP_LT_F16_e64, AMDGPU::V_CMP_LT_F16_t16_e64,
1139	AMDGPU::V_CMP_LT_F32_e64, AMDGPU::V_CMP_LT_F64_e64);
1140	case CmpInst::FCMP_OLE:
1141	return Select(AMDGPU::V_CMP_LE_F16_e64, AMDGPU::V_CMP_LE_F16_t16_e64,
1142	AMDGPU::V_CMP_LE_F32_e64, AMDGPU::V_CMP_LE_F64_e64);
1143	case CmpInst::FCMP_ONE:
1144	return Select(AMDGPU::V_CMP_NEQ_F16_e64, AMDGPU::V_CMP_NEQ_F16_t16_e64,
1145	AMDGPU::V_CMP_NEQ_F32_e64, AMDGPU::V_CMP_NEQ_F64_e64);
1146	case CmpInst::FCMP_ORD:
1147	return Select(AMDGPU::V_CMP_O_F16_e64, AMDGPU::V_CMP_O_F16_t16_e64,
1148	AMDGPU::V_CMP_O_F32_e64, AMDGPU::V_CMP_O_F64_e64);
1149	case CmpInst::FCMP_UNO:
1150	return Select(AMDGPU::V_CMP_U_F16_e64, AMDGPU::V_CMP_U_F16_t16_e64,
1151	AMDGPU::V_CMP_U_F32_e64, AMDGPU::V_CMP_U_F64_e64);
1152	case CmpInst::FCMP_UEQ:
1153	return Select(AMDGPU::V_CMP_NLG_F16_e64, AMDGPU::V_CMP_NLG_F16_t16_e64,
1154	AMDGPU::V_CMP_NLG_F32_e64, AMDGPU::V_CMP_NLG_F64_e64);
1155	case CmpInst::FCMP_UGT:
1156	return Select(AMDGPU::V_CMP_NLE_F16_e64, AMDGPU::V_CMP_NLE_F16_t16_e64,
1157	AMDGPU::V_CMP_NLE_F32_e64, AMDGPU::V_CMP_NLE_F64_e64);
1158	case CmpInst::FCMP_UGE:
1159	return Select(AMDGPU::V_CMP_NLT_F16_e64, AMDGPU::V_CMP_NLT_F16_t16_e64,
1160	AMDGPU::V_CMP_NLT_F32_e64, AMDGPU::V_CMP_NLT_F64_e64);
1161	case CmpInst::FCMP_ULT:
1162	return Select(AMDGPU::V_CMP_NGE_F16_e64, AMDGPU::V_CMP_NGE_F16_t16_e64,
1163	AMDGPU::V_CMP_NGE_F32_e64, AMDGPU::V_CMP_NGE_F64_e64);
1164	case CmpInst::FCMP_ULE:
1165	return Select(AMDGPU::V_CMP_NGT_F16_e64, AMDGPU::V_CMP_NGT_F16_t16_e64,
1166	AMDGPU::V_CMP_NGT_F32_e64, AMDGPU::V_CMP_NGT_F64_e64);
1167	case CmpInst::FCMP_UNE:
1168	return Select(AMDGPU::V_CMP_NEQ_F16_e64, AMDGPU::V_CMP_NEQ_F16_t16_e64,
1169	AMDGPU::V_CMP_NEQ_F32_e64, AMDGPU::V_CMP_NEQ_F64_e64);
1170	case CmpInst::FCMP_TRUE:
1171	return Select(AMDGPU::V_CMP_TRU_F16_e64, AMDGPU::V_CMP_TRU_F16_t16_e64,
1172	AMDGPU::V_CMP_TRU_F32_e64, AMDGPU::V_CMP_TRU_F64_e64);
1173	case CmpInst::FCMP_FALSE:
1174	return Select(AMDGPU::V_CMP_F_F16_e64, AMDGPU::V_CMP_F_F16_t16_e64,
1175	AMDGPU::V_CMP_F_F32_e64, AMDGPU::V_CMP_F_F64_e64);
1176	}
1177	}
1178
1179	int AMDGPUInstructionSelector::getS_CMPOpcode(CmpInst::Predicate P,
1180	unsigned Size) const {
1181	if (Size == 64) {
1182	if (!STI.hasScalarCompareEq64())
1183	return -1;
1184
1185	switch (P) {
1186	case CmpInst::ICMP_NE:
1187	return AMDGPU::S_CMP_LG_U64;
1188	case CmpInst::ICMP_EQ:
1189	return AMDGPU::S_CMP_EQ_U64;
1190	default:
1191	return -1;
1192	}
1193	}
1194
1195	if (Size != 32)
1196	return -1;
1197
1198	switch (P) {
1199	case CmpInst::ICMP_NE:
1200	return AMDGPU::S_CMP_LG_U32;
1201	case CmpInst::ICMP_EQ:
1202	return AMDGPU::S_CMP_EQ_U32;
1203	case CmpInst::ICMP_SGT:
1204	return AMDGPU::S_CMP_GT_I32;
1205	case CmpInst::ICMP_SGE:
1206	return AMDGPU::S_CMP_GE_I32;
1207	case CmpInst::ICMP_SLT:
1208	return AMDGPU::S_CMP_LT_I32;
1209	case CmpInst::ICMP_SLE:
1210	return AMDGPU::S_CMP_LE_I32;
1211	case CmpInst::ICMP_UGT:
1212	return AMDGPU::S_CMP_GT_U32;
1213	case CmpInst::ICMP_UGE:
1214	return AMDGPU::S_CMP_GE_U32;
1215	case CmpInst::ICMP_ULT:
1216	return AMDGPU::S_CMP_LT_U32;
1217	case CmpInst::ICMP_ULE:
1218	return AMDGPU::S_CMP_LE_U32;
1219	default:
1220	llvm_unreachable("Unknown condition code!")::llvm::llvm_unreachable_internal("Unknown condition code!", "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp" , 1220);
1221	}
1222	}
1223
1224	bool AMDGPUInstructionSelector::selectG_ICMP(MachineInstr &I) const {
1225	MachineBasicBlock *BB = I.getParent();
1226	const DebugLoc &DL = I.getDebugLoc();
1227
1228	Register SrcReg = I.getOperand(2).getReg();
1229	unsigned Size = RBI.getSizeInBits(SrcReg, *MRI, TRI);
1230
1231	auto Pred = (CmpInst::Predicate)I.getOperand(1).getPredicate();
1232
1233	Register CCReg = I.getOperand(0).getReg();
1234	if (!isVCC(CCReg, *MRI)) {
1235	int Opcode = getS_CMPOpcode(Pred, Size);
1236	if (Opcode == -1)
1237	return false;
1238	MachineInstr ICmp = BuildMI(BB, &I, DL, TII.get(Opcode))
1239	.add(I.getOperand(2))
1240	.add(I.getOperand(3));
1241	BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), CCReg)
1242	.addReg(AMDGPU::SCC);
1243	bool Ret =
1244	constrainSelectedInstRegOperands(*ICmp, TII, TRI, RBI) &&
1245	RBI.constrainGenericRegister(CCReg, AMDGPU::SReg_32RegClass, *MRI);
1246	I.eraseFromParent();
1247	return Ret;
1248	}
1249
1250	int Opcode = getV_CMPOpcode(Pred, Size, *Subtarget);
1251	if (Opcode == -1)
1252	return false;
1253
1254	MachineInstr ICmp = BuildMI(BB, &I, DL, TII.get(Opcode),
1255	I.getOperand(0).getReg())
1256	.add(I.getOperand(2))
1257	.add(I.getOperand(3));
1258	RBI.constrainGenericRegister(ICmp->getOperand(0).getReg(),
1259	TRI.getBoolRC(), MRI);
1260	bool Ret = constrainSelectedInstRegOperands(*ICmp, TII, TRI, RBI);
1261	I.eraseFromParent();
1262	return Ret;
1263	}
1264
1265	bool AMDGPUInstructionSelector::selectIntrinsicCmp(MachineInstr &I) const {
1266	Register Dst = I.getOperand(0).getReg();
1267	if (isVCC(Dst, *MRI))
1268	return false;
1269
1270	LLT DstTy = MRI->getType(Dst);
1271	if (DstTy.getSizeInBits() != STI.getWavefrontSize())
1272	return false;
1273
1274	MachineBasicBlock *BB = I.getParent();
1275	const DebugLoc &DL = I.getDebugLoc();
1276	Register SrcReg = I.getOperand(2).getReg();
1277	unsigned Size = RBI.getSizeInBits(SrcReg, *MRI, TRI);
1278
1279	// i1 inputs are not supported in GlobalISel.
1280	if (Size == 1)
1281	return false;
1282
1283	auto Pred = static_cast<CmpInst::Predicate>(I.getOperand(4).getImm());
1284	if (!CmpInst::isIntPredicate(Pred) && !CmpInst::isFPPredicate(Pred)) {
1285	BuildMI(*BB, &I, DL, TII.get(AMDGPU::IMPLICIT_DEF), Dst);
1286	I.eraseFromParent();
1287	return RBI.constrainGenericRegister(Dst, TRI.getBoolRC(), MRI);
1288	}
1289
1290	const int Opcode = getV_CMPOpcode(Pred, Size, *Subtarget);
1291	if (Opcode == -1)
1292	return false;
1293
1294	MachineInstrBuilder SelectedMI;
1295	MachineOperand &LHS = I.getOperand(2);
1296	MachineOperand &RHS = I.getOperand(3);
1297	auto [Src0, Src0Mods] = selectVOP3ModsImpl(LHS);
1298	auto [Src1, Src1Mods] = selectVOP3ModsImpl(RHS);
1299	Register Src0Reg =
1300	copyToVGPRIfSrcFolded(Src0, Src0Mods, LHS, &I, /ForceVGPR/ true);
1301	Register Src1Reg =
1302	copyToVGPRIfSrcFolded(Src1, Src1Mods, RHS, &I, /ForceVGPR/ true);
1303	SelectedMI = BuildMI(*BB, &I, DL, TII.get(Opcode), Dst);
1304	if (AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::src0_modifiers))
1305	SelectedMI.addImm(Src0Mods);
1306	SelectedMI.addReg(Src0Reg);
1307	if (AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::src1_modifiers))
1308	SelectedMI.addImm(Src1Mods);
1309	SelectedMI.addReg(Src1Reg);
1310	if (AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::clamp))
1311	SelectedMI.addImm(0); // clamp
1312	if (AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::op_sel))
1313	SelectedMI.addImm(0); // op_sel
1314
1315	RBI.constrainGenericRegister(Dst, TRI.getBoolRC(), MRI);
1316	if (!constrainSelectedInstRegOperands(*SelectedMI, TII, TRI, RBI))
1317	return false;
1318
1319	I.eraseFromParent();
1320	return true;
1321	}
1322
1323	bool AMDGPUInstructionSelector::selectBallot(MachineInstr &I) const {
1324	MachineBasicBlock *BB = I.getParent();
1325	const DebugLoc &DL = I.getDebugLoc();
1326	Register DstReg = I.getOperand(0).getReg();
1327	const unsigned Size = MRI->getType(DstReg).getSizeInBits();
1328	const bool Is64 = Size == 64;
1329
1330	if (Size != STI.getWavefrontSize())
1331	return false;
1332
1333	std::optional<ValueAndVReg> Arg =
1334	getIConstantVRegValWithLookThrough(I.getOperand(2).getReg(), *MRI);
1335
1336	if (Arg) {
1337	const int64_t Value = Arg->Value.getSExtValue();
1338	if (Value == 0) {
1339	unsigned Opcode = Is64 ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
1340	BuildMI(*BB, &I, DL, TII.get(Opcode), DstReg).addImm(0);
1341	} else if (Value == -1) { // all ones
1342	Register SrcReg = Is64 ? AMDGPU::EXEC : AMDGPU::EXEC_LO;
1343	BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), DstReg).addReg(SrcReg);
1344	} else
1345	return false;
1346	} else {
1347	Register SrcReg = I.getOperand(2).getReg();
1348	BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), DstReg).addReg(SrcReg);
1349	}
1350
1351	I.eraseFromParent();
1352	return true;
1353	}
1354
1355	bool AMDGPUInstructionSelector::selectInverseBallot(MachineInstr &I) const {
1356	MachineBasicBlock *BB = I.getParent();
1357	const DebugLoc &DL = I.getDebugLoc();
1358	const Register DstReg = I.getOperand(0).getReg();
1359	const Register MaskReg = I.getOperand(2).getReg();
1360
1361	BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), DstReg).addReg(MaskReg);
1362	I.eraseFromParent();
1363	return true;
1364	}
1365
1366	bool AMDGPUInstructionSelector::selectRelocConstant(MachineInstr &I) const {
1367	Register DstReg = I.getOperand(0).getReg();
1368	const RegisterBank DstBank = RBI.getRegBank(DstReg, MRI, TRI);
1369	const TargetRegisterClass DstRC = TRI.getRegClassForSizeOnBank(32, DstBank);
1370	if (!DstRC \|\| !RBI.constrainGenericRegister(DstReg, DstRC, MRI))
1371	return false;
1372
1373	const bool IsVALU = DstBank->getID() == AMDGPU::VGPRRegBankID;
1374
1375	Module *M = MF->getFunction().getParent();
1376	const MDNode *Metadata = I.getOperand(2).getMetadata();
1377	auto SymbolName = cast<MDString>(Metadata->getOperand(0))->getString();
1378	auto RelocSymbol = cast<GlobalVariable>(
1379	M->getOrInsertGlobal(SymbolName, Type::getInt32Ty(M->getContext())));
1380
1381	MachineBasicBlock *BB = I.getParent();
1382	BuildMI(*BB, &I, I.getDebugLoc(),
1383	TII.get(IsVALU ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32), DstReg)
1384	.addGlobalAddress(RelocSymbol, 0, SIInstrInfo::MO_ABS32_LO);
1385
1386	I.eraseFromParent();
1387	return true;
1388	}
1389
1390	bool AMDGPUInstructionSelector::selectGroupStaticSize(MachineInstr &I) const {
1391	Triple::OSType OS = MF->getTarget().getTargetTriple().getOS();
1392
1393	Register DstReg = I.getOperand(0).getReg();
1394	const RegisterBank DstRB = RBI.getRegBank(DstReg, MRI, TRI);
1395	unsigned Mov = DstRB->getID() == AMDGPU::SGPRRegBankID ?
1396	AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
1397
1398	MachineBasicBlock *MBB = I.getParent();
1399	const DebugLoc &DL = I.getDebugLoc();
1400
1401	auto MIB = BuildMI(*MBB, &I, DL, TII.get(Mov), DstReg);
1402
1403	if (OS == Triple::AMDHSA \|\| OS == Triple::AMDPAL) {
1404	const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1405	MIB.addImm(MFI->getLDSSize());
1406	} else {
1407	Module *M = MF->getFunction().getParent();
1408	const GlobalValue *GV
1409	= Intrinsic::getDeclaration(M, Intrinsic::amdgcn_groupstaticsize);
1410	MIB.addGlobalAddress(GV, 0, SIInstrInfo::MO_ABS32_LO);
1411	}
1412
1413	I.eraseFromParent();
1414	return constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
1415	}
1416
1417	bool AMDGPUInstructionSelector::selectReturnAddress(MachineInstr &I) const {
1418	MachineBasicBlock *MBB = I.getParent();
1419	MachineFunction &MF = *MBB->getParent();
1420	const DebugLoc &DL = I.getDebugLoc();
1421
1422	MachineOperand &Dst = I.getOperand(0);
1423	Register DstReg = Dst.getReg();
1424	unsigned Depth = I.getOperand(2).getImm();
1425
1426	const TargetRegisterClass *RC
1427	= TRI.getConstrainedRegClassForOperand(Dst, *MRI);
1428	if (!RC->hasSubClassEq(&AMDGPU::SGPR_64RegClass) \|\|
1429	!RBI.constrainGenericRegister(DstReg, RC, MRI))
1430	return false;
1431
1432	// Check for kernel and shader functions
1433	if (Depth != 0 \|\|
1434	MF.getInfo<SIMachineFunctionInfo>()->isEntryFunction()) {
1435	BuildMI(*MBB, &I, DL, TII.get(AMDGPU::S_MOV_B64), DstReg)
1436	.addImm(0);
1437	I.eraseFromParent();
1438	return true;
1439	}
1440
1441	MachineFrameInfo &MFI = MF.getFrameInfo();
1442	// There is a call to @llvm.returnaddress in this function
1443	MFI.setReturnAddressIsTaken(true);
1444
1445	// Get the return address reg and mark it as an implicit live-in
1446	Register ReturnAddrReg = TRI.getReturnAddressReg(MF);
1447	Register LiveIn = getFunctionLiveInPhysReg(MF, TII, ReturnAddrReg,
1448	AMDGPU::SReg_64RegClass, DL);
1449	BuildMI(*MBB, &I, DL, TII.get(AMDGPU::COPY), DstReg)
1450	.addReg(LiveIn);
1451	I.eraseFromParent();
1452	return true;
1453	}
1454
1455	bool AMDGPUInstructionSelector::selectEndCfIntrinsic(MachineInstr &MI) const {
1456	// FIXME: Manually selecting to avoid dealing with the SReg_1 trick
1457	// SelectionDAG uses for wave32 vs wave64.
1458	MachineBasicBlock *BB = MI.getParent();
1459	BuildMI(*BB, &MI, MI.getDebugLoc(), TII.get(AMDGPU::SI_END_CF))
1460	.add(MI.getOperand(1));
1461
1462	Register Reg = MI.getOperand(1).getReg();
1463	MI.eraseFromParent();
1464
1465	if (!MRI->getRegClassOrNull(Reg))
1466	MRI->setRegClass(Reg, TRI.getWaveMaskRegClass());
1467	return true;
1468	}
1469
1470	bool AMDGPUInstructionSelector::selectDSOrderedIntrinsic(
1471	MachineInstr &MI, Intrinsic::ID IntrID) const {
1472	MachineBasicBlock *MBB = MI.getParent();
1473	MachineFunction *MF = MBB->getParent();
1474	const DebugLoc &DL = MI.getDebugLoc();
1475
1476	unsigned IndexOperand = MI.getOperand(7).getImm();
1477	bool WaveRelease = MI.getOperand(8).getImm() != 0;
1478	bool WaveDone = MI.getOperand(9).getImm() != 0;
1479
1480	if (WaveDone && !WaveRelease)
1481	report_fatal_error("ds_ordered_count: wave_done requires wave_release");
1482
1483	unsigned OrderedCountIndex = IndexOperand & 0x3f;
1484	IndexOperand &= ~0x3f;
1485	unsigned CountDw = 0;
1486
1487	if (STI.getGeneration() >= AMDGPUSubtarget::GFX10) {
1488	CountDw = (IndexOperand >> 24) & 0xf;
1489	IndexOperand &= ~(0xf << 24);
1490
1491	if (CountDw < 1 \|\| CountDw > 4) {
1492	report_fatal_error(
1493	"ds_ordered_count: dword count must be between 1 and 4");
1494	}
1495	}
1496
1497	if (IndexOperand)
1498	report_fatal_error("ds_ordered_count: bad index operand");
1499
1500	unsigned Instruction = IntrID == Intrinsic::amdgcn_ds_ordered_add ? 0 : 1;
1501	unsigned ShaderType = SIInstrInfo::getDSShaderTypeValue(*MF);
1502
1503	unsigned Offset0 = OrderedCountIndex << 2;
1504	unsigned Offset1 = WaveRelease \| (WaveDone << 1) \| (Instruction << 4);
1505
1506	if (STI.getGeneration() >= AMDGPUSubtarget::GFX10)
1507	Offset1 \|= (CountDw - 1) << 6;
1508
1509	if (STI.getGeneration() < AMDGPUSubtarget::GFX11)
1510	Offset1 \|= ShaderType << 2;
1511
1512	unsigned Offset = Offset0 \| (Offset1 << 8);
1513
1514	Register M0Val = MI.getOperand(2).getReg();
1515	BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0)
1516	.addReg(M0Val);
1517
1518	Register DstReg = MI.getOperand(0).getReg();
1519	Register ValReg = MI.getOperand(3).getReg();
1520	MachineInstrBuilder DS =
1521	BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::DS_ORDERED_COUNT), DstReg)
1522	.addReg(ValReg)
1523	.addImm(Offset)
1524	.cloneMemRefs(MI);
1525
1526	if (!RBI.constrainGenericRegister(M0Val, AMDGPU::SReg_32RegClass, *MRI))
1527	return false;
1528
1529	bool Ret = constrainSelectedInstRegOperands(*DS, TII, TRI, RBI);
1530	MI.eraseFromParent();
1531	return Ret;
1532	}
1533
1534	static unsigned gwsIntrinToOpcode(unsigned IntrID) {
1535	switch (IntrID) {
1536	case Intrinsic::amdgcn_ds_gws_init:
1537	return AMDGPU::DS_GWS_INIT;
1538	case Intrinsic::amdgcn_ds_gws_barrier:
1539	return AMDGPU::DS_GWS_BARRIER;
1540	case Intrinsic::amdgcn_ds_gws_sema_v:
1541	return AMDGPU::DS_GWS_SEMA_V;
1542	case Intrinsic::amdgcn_ds_gws_sema_br:
1543	return AMDGPU::DS_GWS_SEMA_BR;
1544	case Intrinsic::amdgcn_ds_gws_sema_p:
1545	return AMDGPU::DS_GWS_SEMA_P;
1546	case Intrinsic::amdgcn_ds_gws_sema_release_all:
1547	return AMDGPU::DS_GWS_SEMA_RELEASE_ALL;
1548	default:
1549	llvm_unreachable("not a gws intrinsic")::llvm::llvm_unreachable_internal("not a gws intrinsic", "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp" , 1549);
1550	}
1551	}
1552
1553	bool AMDGPUInstructionSelector::selectDSGWSIntrinsic(MachineInstr &MI,
1554	Intrinsic::ID IID) const {
1555	if (IID == Intrinsic::amdgcn_ds_gws_sema_release_all &&
1556	!STI.hasGWSSemaReleaseAll())
1557	return false;
1558
1559	// intrinsic ID, vsrc, offset
1560	const bool HasVSrc = MI.getNumOperands() == 3;
1561	assert(HasVSrc \|\| MI.getNumOperands() == 2)(static_cast <bool> (HasVSrc \|\| MI.getNumOperands() == 2 ) ? void (0) : __assert_fail ("HasVSrc \|\| MI.getNumOperands() == 2" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 1561 , __extension__ __PRETTY_FUNCTION__));
1562
1563	Register BaseOffset = MI.getOperand(HasVSrc ? 2 : 1).getReg();
1564	const RegisterBank OffsetRB = RBI.getRegBank(BaseOffset, MRI, TRI);
1565	if (OffsetRB->getID() != AMDGPU::SGPRRegBankID)
1566	return false;
1567
1568	MachineInstr OffsetDef = getDefIgnoringCopies(BaseOffset, MRI);
1569	unsigned ImmOffset;
1570
1571	MachineBasicBlock *MBB = MI.getParent();
1572	const DebugLoc &DL = MI.getDebugLoc();
1573
1574	MachineInstr *Readfirstlane = nullptr;
1575
1576	// If we legalized the VGPR input, strip out the readfirstlane to analyze the
1577	// incoming offset, in case there's an add of a constant. We'll have to put it
1578	// back later.
1579	if (OffsetDef->getOpcode() == AMDGPU::V_READFIRSTLANE_B32) {
1580	Readfirstlane = OffsetDef;
1581	BaseOffset = OffsetDef->getOperand(1).getReg();
1582	OffsetDef = getDefIgnoringCopies(BaseOffset, *MRI);
1583	}
1584
1585	if (OffsetDef->getOpcode() == AMDGPU::G_CONSTANT) {
1586	// If we have a constant offset, try to use the 0 in m0 as the base.
1587	// TODO: Look into changing the default m0 initialization value. If the
1588	// default -1 only set the low 16-bits, we could leave it as-is and add 1 to
1589	// the immediate offset.
1590
1591	ImmOffset = OffsetDef->getOperand(1).getCImm()->getZExtValue();
1592	BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::S_MOV_B32), AMDGPU::M0)
1593	.addImm(0);
1594	} else {
1595	std::tie(BaseOffset, ImmOffset) =
1596	AMDGPU::getBaseWithConstantOffset(*MRI, BaseOffset, KnownBits);
1597
1598	if (Readfirstlane) {
1599	// We have the constant offset now, so put the readfirstlane back on the
1600	// variable component.
1601	if (!RBI.constrainGenericRegister(BaseOffset, AMDGPU::VGPR_32RegClass, *MRI))
1602	return false;
1603
1604	Readfirstlane->getOperand(1).setReg(BaseOffset);
1605	BaseOffset = Readfirstlane->getOperand(0).getReg();
1606	} else {
1607	if (!RBI.constrainGenericRegister(BaseOffset,
1608	AMDGPU::SReg_32RegClass, *MRI))
1609	return false;
1610	}
1611
1612	Register M0Base = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
1613	BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::S_LSHL_B32), M0Base)
1614	.addReg(BaseOffset)
1615	.addImm(16);
1616
1617	BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0)
1618	.addReg(M0Base);
1619	}
1620
1621	// The resource id offset is computed as (<isa opaque base> + M0[21:16] +
1622	// offset field) % 64. Some versions of the programming guide omit the m0
1623	// part, or claim it's from offset 0.
1624	auto MIB = BuildMI(*MBB, &MI, DL, TII.get(gwsIntrinToOpcode(IID)));
1625
1626	if (HasVSrc) {
1627	Register VSrc = MI.getOperand(1).getReg();
1628	MIB.addReg(VSrc);
1629
1630	if (!RBI.constrainGenericRegister(VSrc, AMDGPU::VGPR_32RegClass, *MRI))
1631	return false;
1632	}
1633
1634	MIB.addImm(ImmOffset)
1635	.cloneMemRefs(MI);
1636
1637	TII.enforceOperandRCAlignment(*MIB, AMDGPU::OpName::data0);
1638
1639	MI.eraseFromParent();
1640	return true;
1641	}
1642
1643	bool AMDGPUInstructionSelector::selectDSAppendConsume(MachineInstr &MI,
1644	bool IsAppend) const {
1645	Register PtrBase = MI.getOperand(2).getReg();
1646	LLT PtrTy = MRI->getType(PtrBase);
1647	bool IsGDS = PtrTy.getAddressSpace() == AMDGPUAS::REGION_ADDRESS;
1648
1649	unsigned Offset;
1650	std::tie(PtrBase, Offset) = selectDS1Addr1OffsetImpl(MI.getOperand(2));
1651
1652	// TODO: Should this try to look through readfirstlane like GWS?
1653	if (!isDSOffsetLegal(PtrBase, Offset)) {
1654	PtrBase = MI.getOperand(2).getReg();
1655	Offset = 0;
1656	}
1657
1658	MachineBasicBlock *MBB = MI.getParent();
1659	const DebugLoc &DL = MI.getDebugLoc();
1660	const unsigned Opc = IsAppend ? AMDGPU::DS_APPEND : AMDGPU::DS_CONSUME;
1661
1662	BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0)
1663	.addReg(PtrBase);
1664	if (!RBI.constrainGenericRegister(PtrBase, AMDGPU::SReg_32RegClass, *MRI))
1665	return false;
1666
1667	auto MIB = BuildMI(*MBB, &MI, DL, TII.get(Opc), MI.getOperand(0).getReg())
1668	.addImm(Offset)
1669	.addImm(IsGDS ? -1 : 0)
1670	.cloneMemRefs(MI);
1671	MI.eraseFromParent();
1672	return constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
1673	}
1674
1675	bool AMDGPUInstructionSelector::selectSBarrier(MachineInstr &MI) const {
1676	if (TM.getOptLevel() > CodeGenOpt::None) {
1677	unsigned WGSize = STI.getFlatWorkGroupSizes(MF->getFunction()).second;
1678	if (WGSize <= STI.getWavefrontSize()) {
1679	MachineBasicBlock *MBB = MI.getParent();
1680	const DebugLoc &DL = MI.getDebugLoc();
1681	BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::WAVE_BARRIER));
1682	MI.eraseFromParent();
1683	return true;
1684	}
1685	}
1686	return selectImpl(MI, *CoverageInfo);
1687	}
1688
1689	static bool parseTexFail(uint64_t TexFailCtrl, bool &TFE, bool &LWE,
1690	bool &IsTexFail) {
1691	if (TexFailCtrl)
1692	IsTexFail = true;
1693
1694	TFE = (TexFailCtrl & 0x1) ? true : false;
1695	TexFailCtrl &= ~(uint64_t)0x1;
1696	LWE = (TexFailCtrl & 0x2) ? true : false;
1697	TexFailCtrl &= ~(uint64_t)0x2;
1698
1699	return TexFailCtrl == 0;
1700	}
1701
1702	bool AMDGPUInstructionSelector::selectImageIntrinsic(
1703	MachineInstr &MI, const AMDGPU::ImageDimIntrinsicInfo *Intr) const {
1704	MachineBasicBlock *MBB = MI.getParent();
1705	const DebugLoc &DL = MI.getDebugLoc();
1706
1707	const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
1708	AMDGPU::getMIMGBaseOpcodeInfo(Intr->BaseOpcode);
1709
1710	const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfo(Intr->Dim);
1711	unsigned IntrOpcode = Intr->BaseOpcode;
1712	const bool IsGFX10Plus = AMDGPU::isGFX10Plus(STI);
1713	const bool IsGFX11Plus = AMDGPU::isGFX11Plus(STI);
1714
1715	const unsigned ArgOffset = MI.getNumExplicitDefs() + 1;
1716
1717	Register VDataIn, VDataOut;
1718	LLT VDataTy;
1719	int NumVDataDwords = -1;
1720	bool IsD16 = MI.getOpcode() == AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16 \|\|
1721	MI.getOpcode() == AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16;
1722
1723	bool Unorm;
1724	if (!BaseOpcode->Sampler)
1725	Unorm = true;
1726	else
1727	Unorm = MI.getOperand(ArgOffset + Intr->UnormIndex).getImm() != 0;
1728
1729	bool TFE;
1730	bool LWE;
1731	bool IsTexFail = false;
1732	if (!parseTexFail(MI.getOperand(ArgOffset + Intr->TexFailCtrlIndex).getImm(),
1733	TFE, LWE, IsTexFail))
1734	return false;
1735
1736	const int Flags = MI.getOperand(ArgOffset + Intr->NumArgs).getImm();
1737	const bool IsA16 = (Flags & 1) != 0;
1738	const bool IsG16 = (Flags & 2) != 0;
1739
1740	// A16 implies 16 bit gradients if subtarget doesn't support G16
1741	if (IsA16 && !STI.hasG16() && !IsG16)
1742	return false;
1743
1744	unsigned DMask = 0;
1745	unsigned DMaskLanes = 0;
1746
1747	if (BaseOpcode->Atomic) {
1748	VDataOut = MI.getOperand(0).getReg();
1749	VDataIn = MI.getOperand(2).getReg();
1750	LLT Ty = MRI->getType(VDataIn);
1751
1752	// Be careful to allow atomic swap on 16-bit element vectors.
1753	const bool Is64Bit = BaseOpcode->AtomicX2 ?
1754	Ty.getSizeInBits() == 128 :
1755	Ty.getSizeInBits() == 64;
1756
1757	if (BaseOpcode->AtomicX2) {
1758	assert(MI.getOperand(3).getReg() == AMDGPU::NoRegister)(static_cast <bool> (MI.getOperand(3).getReg() == AMDGPU ::NoRegister) ? void (0) : __assert_fail ("MI.getOperand(3).getReg() == AMDGPU::NoRegister" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 1758 , __extension__ __PRETTY_FUNCTION__));
1759
1760	DMask = Is64Bit ? 0xf : 0x3;
1761	NumVDataDwords = Is64Bit ? 4 : 2;
1762	} else {
1763	DMask = Is64Bit ? 0x3 : 0x1;
1764	NumVDataDwords = Is64Bit ? 2 : 1;
1765	}
1766	} else {
1767	DMask = MI.getOperand(ArgOffset + Intr->DMaskIndex).getImm();
1768	DMaskLanes = BaseOpcode->Gather4 ? 4 : llvm::popcount(DMask);
1769
1770	if (BaseOpcode->Store) {
1771	VDataIn = MI.getOperand(1).getReg();
1772	VDataTy = MRI->getType(VDataIn);
1773	NumVDataDwords = (VDataTy.getSizeInBits() + 31) / 32;
1774	} else {
1775	VDataOut = MI.getOperand(0).getReg();
1776	VDataTy = MRI->getType(VDataOut);
1777	NumVDataDwords = DMaskLanes;
1778
1779	if (IsD16 && !STI.hasUnpackedD16VMem())
1780	NumVDataDwords = (DMaskLanes + 1) / 2;
1781	}
1782	}
1783
1784	// Set G16 opcode
1785	if (Subtarget->hasG16() && IsG16) {
1786	const AMDGPU::MIMGG16MappingInfo *G16MappingInfo =
1787	AMDGPU::getMIMGG16MappingInfo(Intr->BaseOpcode);
1788	assert(G16MappingInfo)(static_cast <bool> (G16MappingInfo) ? void (0) : __assert_fail ("G16MappingInfo", "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp" , 1788, __extension__ __PRETTY_FUNCTION__));
1789	IntrOpcode = G16MappingInfo->G16; // set opcode to variant with _g16
1790	}
1791
1792	// TODO: Check this in verifier.
1793	assert((!IsTexFail \|\| DMaskLanes >= 1) && "should have legalized this")(static_cast <bool> ((!IsTexFail \|\| DMaskLanes >= 1) && "should have legalized this") ? void (0) : __assert_fail ("(!IsTexFail \|\| DMaskLanes >= 1) && \"should have legalized this\"" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 1793 , __extension__ __PRETTY_FUNCTION__));
1794
1795	unsigned CPol = MI.getOperand(ArgOffset + Intr->CachePolicyIndex).getImm();
1796	if (BaseOpcode->Atomic)
1797	CPol \|= AMDGPU::CPol::GLC; // TODO no-return optimization
1798	if (CPol & ~AMDGPU::CPol::ALL)
1799	return false;
1800
1801	int NumVAddrRegs = 0;
1802	int NumVAddrDwords = 0;
1803	for (unsigned I = Intr->VAddrStart; I < Intr->VAddrEnd; I++) {
1804	// Skip the $noregs and 0s inserted during legalization.
1805	MachineOperand &AddrOp = MI.getOperand(ArgOffset + I);
1806	if (!AddrOp.isReg())
1807	continue; // XXX - Break?
1808
1809	Register Addr = AddrOp.getReg();
1810	if (!Addr)
1811	break;
1812
1813	++NumVAddrRegs;
1814	NumVAddrDwords += (MRI->getType(Addr).getSizeInBits() + 31) / 32;
1815	}
1816
1817	// The legalizer preprocessed the intrinsic arguments. If we aren't using
1818	// NSA, these should have been packed into a single value in the first
1819	// address register
1820	const bool UseNSA =
1821	NumVAddrRegs != 1 &&
1822	(STI.hasPartialNSAEncoding() ? NumVAddrDwords >= NumVAddrRegs
1823	: NumVAddrDwords == NumVAddrRegs);
1824	if (UseNSA && !STI.hasFeature(AMDGPU::FeatureNSAEncoding)) {
1825	LLVM_DEBUG(dbgs() << "Trying to use NSA on non-NSA target\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("amdgpu-isel")) { dbgs() << "Trying to use NSA on non-NSA target\n" ; } } while (false);
1826	return false;
1827	}
1828
1829	if (IsTexFail)
1830	++NumVDataDwords;
1831
1832	int Opcode = -1;
1833	if (IsGFX11Plus) {
1834	Opcode = AMDGPU::getMIMGOpcode(IntrOpcode,
1835	UseNSA ? AMDGPU::MIMGEncGfx11NSA
1836	: AMDGPU::MIMGEncGfx11Default,
1837	NumVDataDwords, NumVAddrDwords);
1838	} else if (IsGFX10Plus) {
1839	Opcode = AMDGPU::getMIMGOpcode(IntrOpcode,
1840	UseNSA ? AMDGPU::MIMGEncGfx10NSA
1841	: AMDGPU::MIMGEncGfx10Default,
1842	NumVDataDwords, NumVAddrDwords);
1843	} else {
1844	if (Subtarget->hasGFX90AInsts()) {
1845	Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx90a,
1846	NumVDataDwords, NumVAddrDwords);
1847	if (Opcode == -1) {
1848	LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("amdgpu-isel")) { dbgs() << "requested image instruction is not supported on this GPU\n" ; } } while (false)
1849	dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("amdgpu-isel")) { dbgs() << "requested image instruction is not supported on this GPU\n" ; } } while (false)
1850	<< "requested image instruction is not supported on this GPU\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("amdgpu-isel")) { dbgs() << "requested image instruction is not supported on this GPU\n" ; } } while (false);
1851	return false;
1852	}
1853	}
1854	if (Opcode == -1 &&
1855	STI.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
1856	Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx8,
1857	NumVDataDwords, NumVAddrDwords);
1858	if (Opcode == -1)
1859	Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx6,
1860	NumVDataDwords, NumVAddrDwords);
1861	}
1862	if (Opcode == -1)
1863	return false;
1864
1865	auto MIB = BuildMI(*MBB, &MI, DL, TII.get(Opcode))
1866	.cloneMemRefs(MI);
1867
1868	if (VDataOut) {
1869	if (BaseOpcode->AtomicX2) {
1870	const bool Is64 = MRI->getType(VDataOut).getSizeInBits() == 64;
1871
1872	Register TmpReg = MRI->createVirtualRegister(
1873	Is64 ? &AMDGPU::VReg_128RegClass : &AMDGPU::VReg_64RegClass);
1874	unsigned SubReg = Is64 ? AMDGPU::sub0_sub1 : AMDGPU::sub0;
1875
1876	MIB.addDef(TmpReg);
1877	if (!MRI->use_empty(VDataOut)) {
1878	BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), VDataOut)
1879	.addReg(TmpReg, RegState::Kill, SubReg);
1880	}
1881
1882	} else {
1883	MIB.addDef(VDataOut); // vdata output
1884	}
1885	}
1886
1887	if (VDataIn)
1888	MIB.addReg(VDataIn); // vdata input
1889
1890	for (int I = 0; I != NumVAddrRegs; ++I) {
1891	MachineOperand &SrcOp = MI.getOperand(ArgOffset + Intr->VAddrStart + I);
1892	if (SrcOp.isReg()) {
1893	assert(SrcOp.getReg() != 0)(static_cast <bool> (SrcOp.getReg() != 0) ? void (0) : __assert_fail ("SrcOp.getReg() != 0", "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp" , 1893, __extension__ __PRETTY_FUNCTION__));
1894	MIB.addReg(SrcOp.getReg());
1895	}
1896	}
1897
1898	MIB.addReg(MI.getOperand(ArgOffset + Intr->RsrcIndex).getReg());
1899	if (BaseOpcode->Sampler)
1900	MIB.addReg(MI.getOperand(ArgOffset + Intr->SampIndex).getReg());
1901
1902	MIB.addImm(DMask); // dmask
1903
1904	if (IsGFX10Plus)
1905	MIB.addImm(DimInfo->Encoding);
1906	MIB.addImm(Unorm);
1907
1908	MIB.addImm(CPol);
1909	MIB.addImm(IsA16 && // a16 or r128
1910	STI.hasFeature(AMDGPU::FeatureR128A16) ? -1 : 0);
1911	if (IsGFX10Plus)
1912	MIB.addImm(IsA16 ? -1 : 0);
1913
1914	if (!Subtarget->hasGFX90AInsts()) {
1915	MIB.addImm(TFE); // tfe
1916	} else if (TFE) {
1917	LLVM_DEBUG(dbgs() << "TFE is not supported on this GPU\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("amdgpu-isel")) { dbgs() << "TFE is not supported on this GPU\n" ; } } while (false);
1918	return false;
1919	}
1920
1921	MIB.addImm(LWE); // lwe
1922	if (!IsGFX10Plus)
1923	MIB.addImm(DimInfo->DA ? -1 : 0);
1924	if (BaseOpcode->HasD16)
1925	MIB.addImm(IsD16 ? -1 : 0);
1926
1927	if (IsTexFail) {
1928	// An image load instruction with TFE/LWE only conditionally writes to its
1929	// result registers. Initialize them to zero so that we always get well
1930	// defined result values.
1931	assert(VDataOut && !VDataIn)(static_cast <bool> (VDataOut && !VDataIn) ? void (0) : __assert_fail ("VDataOut && !VDataIn", "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp" , 1931, __extension__ __PRETTY_FUNCTION__));
1932	Register Tied = MRI->cloneVirtualRegister(VDataOut);
1933	Register Zero = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1934	BuildMI(MBB, MIB, DL, TII.get(AMDGPU::V_MOV_B32_e32), Zero)
1935	.addImm(0);
1936	auto Parts = TRI.getRegSplitParts(MRI->getRegClass(Tied), 4);
1937	if (STI.usePRTStrictNull()) {
1938	// With enable-prt-strict-null enabled, initialize all result registers to
1939	// zero.
1940	auto RegSeq =
1941	BuildMI(MBB, MIB, DL, TII.get(AMDGPU::REG_SEQUENCE), Tied);
1942	for (auto Sub : Parts)
1943	RegSeq.addReg(Zero).addImm(Sub);
1944	} else {
1945	// With enable-prt-strict-null disabled, only initialize the extra TFE/LWE
1946	// result register.
1947	Register Undef = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1948	BuildMI(MBB, MIB, DL, TII.get(AMDGPU::IMPLICIT_DEF), Undef);
1949	auto RegSeq =
1950	BuildMI(MBB, MIB, DL, TII.get(AMDGPU::REG_SEQUENCE), Tied);
1951	for (auto Sub : Parts.drop_back(1))
1952	RegSeq.addReg(Undef).addImm(Sub);
1953	RegSeq.addReg(Zero).addImm(Parts.back());
1954	}
1955	MIB.addReg(Tied, RegState::Implicit);
1956	MIB->tieOperands(0, MIB->getNumOperands() - 1);
1957	}
1958
1959	MI.eraseFromParent();
1960	constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
1961	TII.enforceOperandRCAlignment(*MIB, AMDGPU::OpName::vaddr);
1962	return true;
1963	}
1964
1965	// We need to handle this here because tablegen doesn't support matching
1966	// instructions with multiple outputs.
1967	bool AMDGPUInstructionSelector::selectDSBvhStackIntrinsic(
1968	MachineInstr &MI) const {
1969	Register Dst0 = MI.getOperand(0).getReg();
1970	Register Dst1 = MI.getOperand(1).getReg();
1971
1972	const DebugLoc &DL = MI.getDebugLoc();
1973	MachineBasicBlock *MBB = MI.getParent();
1974
1975	Register Addr = MI.getOperand(3).getReg();
1976	Register Data0 = MI.getOperand(4).getReg();
1977	Register Data1 = MI.getOperand(5).getReg();
1978	unsigned Offset = MI.getOperand(6).getImm();
1979
1980	auto MIB = BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::DS_BVH_STACK_RTN_B32), Dst0)
1981	.addDef(Dst1)
1982	.addUse(Addr)
1983	.addUse(Data0)
1984	.addUse(Data1)
1985	.addImm(Offset)
1986	.cloneMemRefs(MI);
1987
1988	MI.eraseFromParent();
1989	return constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
1990	}
1991
1992	bool AMDGPUInstructionSelector::selectG_INTRINSIC_W_SIDE_EFFECTS(
1993	MachineInstr &I) const {
1994	unsigned IntrinsicID = I.getIntrinsicID();
1995	switch (IntrinsicID) {
1996	case Intrinsic::amdgcn_end_cf:
1997	return selectEndCfIntrinsic(I);
1998	case Intrinsic::amdgcn_ds_ordered_add:
1999	case Intrinsic::amdgcn_ds_ordered_swap:
2000	return selectDSOrderedIntrinsic(I, IntrinsicID);
2001	case Intrinsic::amdgcn_ds_gws_init:
2002	case Intrinsic::amdgcn_ds_gws_barrier:
2003	case Intrinsic::amdgcn_ds_gws_sema_v:
2004	case Intrinsic::amdgcn_ds_gws_sema_br:
2005	case Intrinsic::amdgcn_ds_gws_sema_p:
2006	case Intrinsic::amdgcn_ds_gws_sema_release_all:
2007	return selectDSGWSIntrinsic(I, IntrinsicID);
2008	case Intrinsic::amdgcn_ds_append:
2009	return selectDSAppendConsume(I, true);
2010	case Intrinsic::amdgcn_ds_consume:
2011	return selectDSAppendConsume(I, false);
2012	case Intrinsic::amdgcn_s_barrier:
2013	return selectSBarrier(I);
2014	case Intrinsic::amdgcn_raw_buffer_load_lds:
2015	case Intrinsic::amdgcn_struct_buffer_load_lds:
2016	return selectBufferLoadLds(I);
2017	case Intrinsic::amdgcn_global_load_lds:
2018	return selectGlobalLoadLds(I);
2019	case Intrinsic::amdgcn_exp_compr:
2020	if (!STI.hasCompressedExport()) {
2021	Function &F = I.getMF()->getFunction();
2022	DiagnosticInfoUnsupported NoFpRet(
2023	F, "intrinsic not supported on subtarget", I.getDebugLoc(), DS_Error);
2024	F.getContext().diagnose(NoFpRet);
2025	return false;
2026	}
2027	break;
2028	case Intrinsic::amdgcn_ds_bvh_stack_rtn:
2029	return selectDSBvhStackIntrinsic(I);
2030	}
2031	return selectImpl(I, *CoverageInfo);
2032	}
2033
2034	bool AMDGPUInstructionSelector::selectG_SELECT(MachineInstr &I) const {
2035	if (selectImpl(I, *CoverageInfo))
2036	return true;
2037
2038	MachineBasicBlock *BB = I.getParent();
2039	const DebugLoc &DL = I.getDebugLoc();
2040
2041	Register DstReg = I.getOperand(0).getReg();
2042	unsigned Size = RBI.getSizeInBits(DstReg, *MRI, TRI);
2043	assert(Size <= 32 \|\| Size == 64)(static_cast <bool> (Size <= 32 \|\| Size == 64) ? void (0) : __assert_fail ("Size <= 32 \|\| Size == 64", "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp" , 2043, __extension__ __PRETTY_FUNCTION__));
2044	const MachineOperand &CCOp = I.getOperand(1);
2045	Register CCReg = CCOp.getReg();
2046	if (!isVCC(CCReg, *MRI)) {
2047	unsigned SelectOpcode = Size == 64 ? AMDGPU::S_CSELECT_B64 :
2048	AMDGPU::S_CSELECT_B32;
2049	MachineInstr CopySCC = BuildMI(BB, &I, DL, TII.get(AMDGPU::COPY), AMDGPU::SCC)
2050	.addReg(CCReg);
2051
2052	// The generic constrainSelectedInstRegOperands doesn't work for the scc register
2053	// bank, because it does not cover the register class that we used to represent
2054	// for it. So we need to manually set the register class here.
2055	if (!MRI->getRegClassOrNull(CCReg))
2056	MRI->setRegClass(CCReg, TRI.getConstrainedRegClassForOperand(CCOp, *MRI));
2057	MachineInstr Select = BuildMI(BB, &I, DL, TII.get(SelectOpcode), DstReg)
2058	.add(I.getOperand(2))
2059	.add(I.getOperand(3));
2060
2061	bool Ret = false;
2062	Ret \|= constrainSelectedInstRegOperands(*Select, TII, TRI, RBI);
2063	Ret \|= constrainSelectedInstRegOperands(*CopySCC, TII, TRI, RBI);
2064	I.eraseFromParent();
2065	return Ret;
2066	}
2067
2068	// Wide VGPR select should have been split in RegBankSelect.
2069	if (Size > 32)
2070	return false;
2071
2072	MachineInstr *Select =
2073	BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
2074	.addImm(0)
2075	.add(I.getOperand(3))
2076	.addImm(0)
2077	.add(I.getOperand(2))
2078	.add(I.getOperand(1));
2079
2080	bool Ret = constrainSelectedInstRegOperands(*Select, TII, TRI, RBI);
2081	I.eraseFromParent();
2082	return Ret;
2083	}
2084
2085	static int sizeToSubRegIndex(unsigned Size) {
2086	switch (Size) {
2087	case 32:
2088	return AMDGPU::sub0;
2089	case 64:
2090	return AMDGPU::sub0_sub1;
2091	case 96:
2092	return AMDGPU::sub0_sub1_sub2;
2093	case 128:
2094	return AMDGPU::sub0_sub1_sub2_sub3;
2095	case 256:
2096	return AMDGPU::sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7;
2097	default:
2098	if (Size < 32)
2099	return AMDGPU::sub0;
2100	if (Size > 256)
2101	return -1;
2102	return sizeToSubRegIndex(llvm::bit_ceil(Size));
2103	}
2104	}
2105
2106	bool AMDGPUInstructionSelector::selectG_TRUNC(MachineInstr &I) const {
2107	Register DstReg = I.getOperand(0).getReg();
2108	Register SrcReg = I.getOperand(1).getReg();
2109	const LLT DstTy = MRI->getType(DstReg);
2110	const LLT SrcTy = MRI->getType(SrcReg);
2111	const LLT S1 = LLT::scalar(1);
2112
2113	const RegisterBank SrcRB = RBI.getRegBank(SrcReg, MRI, TRI);
2114	const RegisterBank *DstRB;
2115	if (DstTy == S1) {
2116	// This is a special case. We don't treat s1 for legalization artifacts as
2117	// vcc booleans.
2118	DstRB = SrcRB;
2119	} else {
2120	DstRB = RBI.getRegBank(DstReg, *MRI, TRI);
2121	if (SrcRB != DstRB)
2122	return false;
2123	}
2124
2125	const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID;
2126
2127	unsigned DstSize = DstTy.getSizeInBits();
2128	unsigned SrcSize = SrcTy.getSizeInBits();
2129
2130	const TargetRegisterClass *SrcRC =
2131	TRI.getRegClassForSizeOnBank(SrcSize, *SrcRB);
2132	const TargetRegisterClass *DstRC =
2133	TRI.getRegClassForSizeOnBank(DstSize, *DstRB);
2134	if (!SrcRC \|\| !DstRC)
2135	return false;
2136
2137	if (!RBI.constrainGenericRegister(SrcReg, SrcRC, MRI) \|\|
2138	!RBI.constrainGenericRegister(DstReg, DstRC, MRI)) {
2139	LLVM_DEBUG(dbgs() << "Failed to constrain G_TRUNC\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("amdgpu-isel")) { dbgs() << "Failed to constrain G_TRUNC\n" ; } } while (false);
2140	return false;
2141	}
2142
2143	if (DstTy == LLT::fixed_vector(2, 16) && SrcTy == LLT::fixed_vector(2, 32)) {
2144	MachineBasicBlock *MBB = I.getParent();
2145	const DebugLoc &DL = I.getDebugLoc();
2146
2147	Register LoReg = MRI->createVirtualRegister(DstRC);
2148	Register HiReg = MRI->createVirtualRegister(DstRC);
2149	BuildMI(*MBB, I, DL, TII.get(AMDGPU::COPY), LoReg)
2150	.addReg(SrcReg, 0, AMDGPU::sub0);
2151	BuildMI(*MBB, I, DL, TII.get(AMDGPU::COPY), HiReg)
2152	.addReg(SrcReg, 0, AMDGPU::sub1);
2153
2154	if (IsVALU && STI.hasSDWA()) {
2155	// Write the low 16-bits of the high element into the high 16-bits of the
2156	// low element.
2157	MachineInstr *MovSDWA =
2158	BuildMI(*MBB, I, DL, TII.get(AMDGPU::V_MOV_B32_sdwa), DstReg)
2159	.addImm(0) // $src0_modifiers
2160	.addReg(HiReg) // $src0
2161	.addImm(0) // $clamp
2162	.addImm(AMDGPU::SDWA::WORD_1) // $dst_sel
2163	.addImm(AMDGPU::SDWA::UNUSED_PRESERVE) // $dst_unused
2164	.addImm(AMDGPU::SDWA::WORD_0) // $src0_sel
2165	.addReg(LoReg, RegState::Implicit);
2166	MovSDWA->tieOperands(0, MovSDWA->getNumOperands() - 1);
2167	} else {
2168	Register TmpReg0 = MRI->createVirtualRegister(DstRC);
2169	Register TmpReg1 = MRI->createVirtualRegister(DstRC);
2170	Register ImmReg = MRI->createVirtualRegister(DstRC);
2171	if (IsVALU) {
2172	BuildMI(*MBB, I, DL, TII.get(AMDGPU::V_LSHLREV_B32_e64), TmpReg0)
2173	.addImm(16)
2174	.addReg(HiReg);
2175	} else {
2176	BuildMI(*MBB, I, DL, TII.get(AMDGPU::S_LSHL_B32), TmpReg0)
2177	.addReg(HiReg)
2178	.addImm(16);
2179	}
2180
2181	unsigned MovOpc = IsVALU ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32;
2182	unsigned AndOpc = IsVALU ? AMDGPU::V_AND_B32_e64 : AMDGPU::S_AND_B32;
2183	unsigned OrOpc = IsVALU ? AMDGPU::V_OR_B32_e64 : AMDGPU::S_OR_B32;
2184
2185	BuildMI(*MBB, I, DL, TII.get(MovOpc), ImmReg)
2186	.addImm(0xffff);
2187	BuildMI(*MBB, I, DL, TII.get(AndOpc), TmpReg1)
2188	.addReg(LoReg)
2189	.addReg(ImmReg);
2190	BuildMI(*MBB, I, DL, TII.get(OrOpc), DstReg)
2191	.addReg(TmpReg0)
2192	.addReg(TmpReg1);
2193	}
2194
2195	I.eraseFromParent();
2196	return true;
2197	}
2198
2199	if (!DstTy.isScalar())
2200	return false;
2201
2202	if (SrcSize > 32) {
2203	int SubRegIdx = sizeToSubRegIndex(DstSize);
2204	if (SubRegIdx == -1)
2205	return false;
2206
2207	// Deal with weird cases where the class only partially supports the subreg
2208	// index.
2209	const TargetRegisterClass *SrcWithSubRC
2210	= TRI.getSubClassWithSubReg(SrcRC, SubRegIdx);
2211	if (!SrcWithSubRC)
2212	return false;
2213
2214	if (SrcWithSubRC != SrcRC) {
2215	if (!RBI.constrainGenericRegister(SrcReg, SrcWithSubRC, MRI))
2216	return false;
2217	}
2218
2219	I.getOperand(1).setSubReg(SubRegIdx);
2220	}
2221
2222	I.setDesc(TII.get(TargetOpcode::COPY));
2223	return true;
2224	}
2225
2226	/// \returns true if a bitmask for \p Size bits will be an inline immediate.
2227	static bool shouldUseAndMask(unsigned Size, unsigned &Mask) {
2228	Mask = maskTrailingOnes<unsigned>(Size);
2229	int SignedMask = static_cast<int>(Mask);
2230	return SignedMask >= -16 && SignedMask <= 64;
2231	}
2232
2233	// Like RegisterBankInfo::getRegBank, but don't assume vcc for s1.
2234	const RegisterBank *AMDGPUInstructionSelector::getArtifactRegBank(
2235	Register Reg, const MachineRegisterInfo &MRI,
2236	const TargetRegisterInfo &TRI) const {
2237	const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Reg);
2238	if (auto RB = RegClassOrBank.dyn_cast<const RegisterBank >())
2239	return RB;
2240
2241	// Ignore the type, since we don't use vcc in artifacts.
2242	if (auto RC = RegClassOrBank.dyn_cast<const TargetRegisterClass >())
2243	return &RBI.getRegBankFromRegClass(*RC, LLT());
2244	return nullptr;
2245	}
2246
2247	bool AMDGPUInstructionSelector::selectG_SZA_EXT(MachineInstr &I) const {
2248	bool InReg = I.getOpcode() == AMDGPU::G_SEXT_INREG;
2249	bool Signed = I.getOpcode() == AMDGPU::G_SEXT \|\| InReg;
2250	const DebugLoc &DL = I.getDebugLoc();
2251	MachineBasicBlock &MBB = *I.getParent();
2252	const Register DstReg = I.getOperand(0).getReg();
2253	const Register SrcReg = I.getOperand(1).getReg();
2254
2255	const LLT DstTy = MRI->getType(DstReg);
2256	const LLT SrcTy = MRI->getType(SrcReg);
2257	const unsigned SrcSize = I.getOpcode() == AMDGPU::G_SEXT_INREG ?
2258	I.getOperand(2).getImm() : SrcTy.getSizeInBits();
2259	const unsigned DstSize = DstTy.getSizeInBits();
2260	if (!DstTy.isScalar())
2261	return false;
2262
2263	// Artifact casts should never use vcc.
2264	const RegisterBank SrcBank = getArtifactRegBank(SrcReg, MRI, TRI);
2265
2266	// FIXME: This should probably be illegal and split earlier.
2267	if (I.getOpcode() == AMDGPU::G_ANYEXT) {
2268	if (DstSize <= 32)
2269	return selectCOPY(I);
2270
2271	const TargetRegisterClass *SrcRC =
2272	TRI.getRegClassForTypeOnBank(SrcTy, *SrcBank);
2273	const RegisterBank DstBank = RBI.getRegBank(DstReg, MRI, TRI);
2274	const TargetRegisterClass *DstRC =
2275	TRI.getRegClassForSizeOnBank(DstSize, *DstBank);
2276
2277	Register UndefReg = MRI->createVirtualRegister(SrcRC);
2278	BuildMI(MBB, I, DL, TII.get(AMDGPU::IMPLICIT_DEF), UndefReg);
2279	BuildMI(MBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg)
2280	.addReg(SrcReg)
2281	.addImm(AMDGPU::sub0)
2282	.addReg(UndefReg)
2283	.addImm(AMDGPU::sub1);
2284	I.eraseFromParent();
2285
2286	return RBI.constrainGenericRegister(DstReg, DstRC, MRI) &&
2287	RBI.constrainGenericRegister(SrcReg, SrcRC, MRI);
2288	}
2289
2290	if (SrcBank->getID() == AMDGPU::VGPRRegBankID && DstSize <= 32) {
2291	// 64-bit should have been split up in RegBankSelect
2292
2293	// Try to use an and with a mask if it will save code size.
2294	unsigned Mask;
2295	if (!Signed && shouldUseAndMask(SrcSize, Mask)) {
2296	MachineInstr *ExtI =
2297	BuildMI(MBB, I, DL, TII.get(AMDGPU::V_AND_B32_e32), DstReg)
2298	.addImm(Mask)
2299	.addReg(SrcReg);
2300	I.eraseFromParent();
2301	return constrainSelectedInstRegOperands(*ExtI, TII, TRI, RBI);
2302	}
2303
2304	const unsigned BFE = Signed ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64;
2305	MachineInstr *ExtI =
2306	BuildMI(MBB, I, DL, TII.get(BFE), DstReg)
2307	.addReg(SrcReg)
2308	.addImm(0) // Offset
2309	.addImm(SrcSize); // Width
2310	I.eraseFromParent();
2311	return constrainSelectedInstRegOperands(*ExtI, TII, TRI, RBI);
2312	}
2313
2314	if (SrcBank->getID() == AMDGPU::SGPRRegBankID && DstSize <= 64) {
2315	const TargetRegisterClass &SrcRC = InReg && DstSize > 32 ?
2316	AMDGPU::SReg_64RegClass : AMDGPU::SReg_32RegClass;
2317	if (!RBI.constrainGenericRegister(SrcReg, SrcRC, *MRI))
2318	return false;
2319
2320	if (Signed && DstSize == 32 && (SrcSize == 8 \|\| SrcSize == 16)) {
2321	const unsigned SextOpc = SrcSize == 8 ?
2322	AMDGPU::S_SEXT_I32_I8 : AMDGPU::S_SEXT_I32_I16;
2323	BuildMI(MBB, I, DL, TII.get(SextOpc), DstReg)
2324	.addReg(SrcReg);
2325	I.eraseFromParent();
2326	return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, *MRI);
2327	}
2328
2329	// Using a single 32-bit SALU to calculate the high half is smaller than
2330	// S_BFE with a literal constant operand.
2331	if (DstSize > 32 && SrcSize == 32) {
2332	Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
2333	unsigned SubReg = InReg ? AMDGPU::sub0 : AMDGPU::NoSubRegister;
2334	if (Signed) {
2335	BuildMI(MBB, I, DL, TII.get(AMDGPU::S_ASHR_I32), HiReg)
2336	.addReg(SrcReg, 0, SubReg)
2337	.addImm(31);
2338	} else {
2339	BuildMI(MBB, I, DL, TII.get(AMDGPU::S_MOV_B32), HiReg)
2340	.addImm(0);
2341	}
2342	BuildMI(MBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg)
2343	.addReg(SrcReg, 0, SubReg)
2344	.addImm(AMDGPU::sub0)
2345	.addReg(HiReg)
2346	.addImm(AMDGPU::sub1);
2347	I.eraseFromParent();
2348	return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_64RegClass,
2349	*MRI);
2350	}
2351
2352	const unsigned BFE64 = Signed ? AMDGPU::S_BFE_I64 : AMDGPU::S_BFE_U64;
2353	const unsigned BFE32 = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
2354
2355	// Scalar BFE is encoded as S1[5:0] = offset, S1[22:16]= width.
2356	if (DstSize > 32 && (SrcSize <= 32 \|\| InReg)) {
2357	// We need a 64-bit register source, but the high bits don't matter.
2358	Register ExtReg = MRI->createVirtualRegister(&AMDGPU::SReg_64RegClass);
2359	Register UndefReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
2360	unsigned SubReg = InReg ? AMDGPU::sub0 : AMDGPU::NoSubRegister;
2361
2362	BuildMI(MBB, I, DL, TII.get(AMDGPU::IMPLICIT_DEF), UndefReg);
2363	BuildMI(MBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), ExtReg)
2364	.addReg(SrcReg, 0, SubReg)
2365	.addImm(AMDGPU::sub0)
2366	.addReg(UndefReg)
2367	.addImm(AMDGPU::sub1);
2368
2369	BuildMI(MBB, I, DL, TII.get(BFE64), DstReg)
2370	.addReg(ExtReg)
2371	.addImm(SrcSize << 16);
2372
2373	I.eraseFromParent();
2374	return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_64RegClass, *MRI);
2375	}
2376
2377	unsigned Mask;
2378	if (!Signed && shouldUseAndMask(SrcSize, Mask)) {
2379	BuildMI(MBB, I, DL, TII.get(AMDGPU::S_AND_B32), DstReg)
2380	.addReg(SrcReg)
2381	.addImm(Mask);
2382	} else {
2383	BuildMI(MBB, I, DL, TII.get(BFE32), DstReg)
2384	.addReg(SrcReg)
2385	.addImm(SrcSize << 16);
2386	}
2387
2388	I.eraseFromParent();
2389	return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, *MRI);
2390	}
2391
2392	return false;
2393	}
2394
2395	bool AMDGPUInstructionSelector::selectG_CONSTANT(MachineInstr &I) const {
2396	MachineBasicBlock *BB = I.getParent();
2397	MachineOperand &ImmOp = I.getOperand(1);
2398	Register DstReg = I.getOperand(0).getReg();
2399	unsigned Size = MRI->getType(DstReg).getSizeInBits();
2400
2401	// The AMDGPU backend only supports Imm operands and not CImm or FPImm.
2402	if (ImmOp.isFPImm()) {
2403	const APInt &Imm = ImmOp.getFPImm()->getValueAPF().bitcastToAPInt();
2404	ImmOp.ChangeToImmediate(Imm.getZExtValue());
2405	} else if (ImmOp.isCImm()) {
2406	ImmOp.ChangeToImmediate(ImmOp.getCImm()->getSExtValue());
2407	} else {
2408	llvm_unreachable("Not supported by g_constants")::llvm::llvm_unreachable_internal("Not supported by g_constants" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 2408 );
2409	}
2410
2411	const RegisterBank DstRB = RBI.getRegBank(DstReg, MRI, TRI);
2412	const bool IsSgpr = DstRB->getID() == AMDGPU::SGPRRegBankID;
2413
2414	unsigned Opcode;
2415	if (DstRB->getID() == AMDGPU::VCCRegBankID) {
2416	Opcode = STI.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
2417	} else {
2418	Opcode = IsSgpr ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
2419
2420	// We should never produce s1 values on banks other than VCC. If the user of
2421	// this already constrained the register, we may incorrectly think it's VCC
2422	// if it wasn't originally.
2423	if (Size == 1)
2424	return false;
2425	}
2426
2427	if (Size != 64) {
2428	I.setDesc(TII.get(Opcode));
2429	I.addImplicitDefUseOperands(*MF);
2430	return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2431	}
2432
2433	const DebugLoc &DL = I.getDebugLoc();
2434
2435	APInt Imm(Size, I.getOperand(1).getImm());
2436
2437	MachineInstr *ResInst;
2438	if (IsSgpr && TII.isInlineConstant(Imm)) {
2439	ResInst = BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_MOV_B64), DstReg)
2440	.addImm(I.getOperand(1).getImm());
2441	} else {
2442	const TargetRegisterClass *RC = IsSgpr ?
2443	&AMDGPU::SReg_32RegClass : &AMDGPU::VGPR_32RegClass;
2444	Register LoReg = MRI->createVirtualRegister(RC);
2445	Register HiReg = MRI->createVirtualRegister(RC);
2446
2447	BuildMI(*BB, &I, DL, TII.get(Opcode), LoReg)
2448	.addImm(Imm.trunc(32).getZExtValue());
2449
2450	BuildMI(*BB, &I, DL, TII.get(Opcode), HiReg)
2451	.addImm(Imm.ashr(32).getZExtValue());
2452
2453	ResInst = BuildMI(*BB, &I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg)
2454	.addReg(LoReg)
2455	.addImm(AMDGPU::sub0)
2456	.addReg(HiReg)
2457	.addImm(AMDGPU::sub1);
2458	}
2459
2460	// We can't call constrainSelectedInstRegOperands here, because it doesn't
2461	// work for target independent opcodes
2462	I.eraseFromParent();
2463	const TargetRegisterClass *DstRC =
2464	TRI.getConstrainedRegClassForOperand(ResInst->getOperand(0), *MRI);
2465	if (!DstRC)
2466	return true;
2467	return RBI.constrainGenericRegister(DstReg, DstRC, MRI);
2468	}
2469
2470	bool AMDGPUInstructionSelector::selectG_FNEG(MachineInstr &MI) const {
2471	// Only manually handle the f64 SGPR case.
2472	//
2473	// FIXME: This is a workaround for 2.5 different tablegen problems. Because
2474	// the bit ops theoretically have a second result due to the implicit def of
2475	// SCC, the GlobalISelEmitter is overly conservative and rejects it. Fixing
2476	// that is easy by disabling the check. The result works, but uses a
2477	// nonsensical sreg32orlds_and_sreg_1 regclass.
2478	//
2479	// The DAG emitter is more problematic, and incorrectly adds both S_XOR_B32 to
2480	// the variadic REG_SEQUENCE operands.
2481
2482	Register Dst = MI.getOperand(0).getReg();
2483	const RegisterBank DstRB = RBI.getRegBank(Dst, MRI, TRI);
2484	if (DstRB->getID() != AMDGPU::SGPRRegBankID \|\|
2485	MRI->getType(Dst) != LLT::scalar(64))
2486	return false;
2487
2488	Register Src = MI.getOperand(1).getReg();
2489	MachineInstr Fabs = getOpcodeDef(TargetOpcode::G_FABS, Src, MRI);
2490	if (Fabs)
2491	Src = Fabs->getOperand(1).getReg();
2492
2493	if (!RBI.constrainGenericRegister(Src, AMDGPU::SReg_64RegClass, *MRI) \|\|
2494	!RBI.constrainGenericRegister(Dst, AMDGPU::SReg_64RegClass, *MRI))
2495	return false;
2496
2497	MachineBasicBlock *BB = MI.getParent();
2498	const DebugLoc &DL = MI.getDebugLoc();
2499	Register LoReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
2500	Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
2501	Register ConstReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
2502	Register OpReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
2503
2504	BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), LoReg)
2505	.addReg(Src, 0, AMDGPU::sub0);
2506	BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), HiReg)
2507	.addReg(Src, 0, AMDGPU::sub1);
2508	BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_MOV_B32), ConstReg)
2509	.addImm(0x80000000);
2510
2511	// Set or toggle sign bit.
2512	unsigned Opc = Fabs ? AMDGPU::S_OR_B32 : AMDGPU::S_XOR_B32;
2513	BuildMI(*BB, &MI, DL, TII.get(Opc), OpReg)
2514	.addReg(HiReg)
2515	.addReg(ConstReg);
2516	BuildMI(*BB, &MI, DL, TII.get(AMDGPU::REG_SEQUENCE), Dst)
2517	.addReg(LoReg)
2518	.addImm(AMDGPU::sub0)
2519	.addReg(OpReg)
2520	.addImm(AMDGPU::sub1);
2521	MI.eraseFromParent();
2522	return true;
2523	}
2524
2525	// FIXME: This is a workaround for the same tablegen problems as G_FNEG
2526	bool AMDGPUInstructionSelector::selectG_FABS(MachineInstr &MI) const {
2527	Register Dst = MI.getOperand(0).getReg();
2528	const RegisterBank DstRB = RBI.getRegBank(Dst, MRI, TRI);
2529	if (DstRB->getID() != AMDGPU::SGPRRegBankID \|\|
2530	MRI->getType(Dst) != LLT::scalar(64))
2531	return false;
2532
2533	Register Src = MI.getOperand(1).getReg();
2534	MachineBasicBlock *BB = MI.getParent();
2535	const DebugLoc &DL = MI.getDebugLoc();
2536	Register LoReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
2537	Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
2538	Register ConstReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
2539	Register OpReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
2540
2541	if (!RBI.constrainGenericRegister(Src, AMDGPU::SReg_64RegClass, *MRI) \|\|
2542	!RBI.constrainGenericRegister(Dst, AMDGPU::SReg_64RegClass, *MRI))
2543	return false;
2544
2545	BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), LoReg)
2546	.addReg(Src, 0, AMDGPU::sub0);
2547	BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), HiReg)
2548	.addReg(Src, 0, AMDGPU::sub1);
2549	BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_MOV_B32), ConstReg)
2550	.addImm(0x7fffffff);
2551
2552	// Clear sign bit.
2553	// TODO: Should this used S_BITSET0_*?
2554	BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_AND_B32), OpReg)
2555	.addReg(HiReg)
2556	.addReg(ConstReg);
2557	BuildMI(*BB, &MI, DL, TII.get(AMDGPU::REG_SEQUENCE), Dst)
2558	.addReg(LoReg)
2559	.addImm(AMDGPU::sub0)
2560	.addReg(OpReg)
2561	.addImm(AMDGPU::sub1);
2562
2563	MI.eraseFromParent();
2564	return true;
2565	}
2566
2567	static bool isConstant(const MachineInstr &MI) {
2568	return MI.getOpcode() == TargetOpcode::G_CONSTANT;
2569	}
2570
2571	void AMDGPUInstructionSelector::getAddrModeInfo(const MachineInstr &Load,
2572	const MachineRegisterInfo &MRI, SmallVectorImpl<GEPInfo> &AddrInfo) const {
2573
2574	const MachineInstr *PtrMI = MRI.getUniqueVRegDef(Load.getOperand(1).getReg());
2575
2576	assert(PtrMI)(static_cast <bool> (PtrMI) ? void (0) : __assert_fail ( "PtrMI", "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp" , 2576, __extension__ __PRETTY_FUNCTION__));
2577
2578	if (PtrMI->getOpcode() != TargetOpcode::G_PTR_ADD)
2579	return;
2580
2581	GEPInfo GEPInfo;
2582
2583	for (unsigned i = 1; i != 3; ++i) {
2584	const MachineOperand &GEPOp = PtrMI->getOperand(i);
2585	const MachineInstr *OpDef = MRI.getUniqueVRegDef(GEPOp.getReg());
2586	assert(OpDef)(static_cast <bool> (OpDef) ? void (0) : __assert_fail ( "OpDef", "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp" , 2586, __extension__ __PRETTY_FUNCTION__));
2587	if (i == 2 && isConstant(*OpDef)) {
2588	// TODO: Could handle constant base + variable offset, but a combine
2589	// probably should have commuted it.
2590	assert(GEPInfo.Imm == 0)(static_cast <bool> (GEPInfo.Imm == 0) ? void (0) : __assert_fail ("GEPInfo.Imm == 0", "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp" , 2590, __extension__ __PRETTY_FUNCTION__));
2591	GEPInfo.Imm = OpDef->getOperand(1).getCImm()->getSExtValue();
2592	continue;
2593	}
2594	const RegisterBank *OpBank = RBI.getRegBank(GEPOp.getReg(), MRI, TRI);
2595	if (OpBank->getID() == AMDGPU::SGPRRegBankID)
2596	GEPInfo.SgprParts.push_back(GEPOp.getReg());
2597	else
2598	GEPInfo.VgprParts.push_back(GEPOp.getReg());
2599	}
2600
2601	AddrInfo.push_back(GEPInfo);
2602	getAddrModeInfo(*PtrMI, MRI, AddrInfo);
2603	}
2604
2605	bool AMDGPUInstructionSelector::isSGPR(Register Reg) const {
2606	return RBI.getRegBank(Reg, *MRI, TRI)->getID() == AMDGPU::SGPRRegBankID;
2607	}
2608
2609	bool AMDGPUInstructionSelector::isInstrUniform(const MachineInstr &MI) const {
2610	if (!MI.hasOneMemOperand())
2611	return false;
2612
2613	const MachineMemOperand MMO = MI.memoperands_begin();
2614	const Value *Ptr = MMO->getValue();
2615
2616	// UndefValue means this is a load of a kernel input. These are uniform.
2617	// Sometimes LDS instructions have constant pointers.
2618	// If Ptr is null, then that means this mem operand contains a
2619	// PseudoSourceValue like GOT.
2620	if (!Ptr \|\| isa<UndefValue>(Ptr) \|\| isa<Argument>(Ptr) \|\|
2621	isa<Constant>(Ptr) \|\| isa<GlobalValue>(Ptr))
2622	return true;
2623
2624	if (MMO->getAddrSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT)
2625	return true;
2626
2627	const Instruction *I = dyn_cast<Instruction>(Ptr);
2628	return I && I->getMetadata("amdgpu.uniform");
2629	}
2630
2631	bool AMDGPUInstructionSelector::hasVgprParts(ArrayRef<GEPInfo> AddrInfo) const {
2632	for (const GEPInfo &GEPInfo : AddrInfo) {
2633	if (!GEPInfo.VgprParts.empty())
2634	return true;
2635	}
2636	return false;
2637	}
2638
2639	void AMDGPUInstructionSelector::initM0(MachineInstr &I) const {
2640	const LLT PtrTy = MRI->getType(I.getOperand(1).getReg());
2641	unsigned AS = PtrTy.getAddressSpace();
2642	if ((AS == AMDGPUAS::LOCAL_ADDRESS \|\| AS == AMDGPUAS::REGION_ADDRESS) &&
2643	STI.ldsRequiresM0Init()) {
2644	MachineBasicBlock *BB = I.getParent();
2645
2646	// If DS instructions require M0 initialization, insert it before selecting.
2647	BuildMI(*BB, &I, I.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), AMDGPU::M0)
2648	.addImm(-1);
2649	}
2650	}
2651
2652	bool AMDGPUInstructionSelector::selectG_LOAD_STORE_ATOMICRMW(
2653	MachineInstr &I) const {
2654	initM0(I);
2655	return selectImpl(I, *CoverageInfo);
2656	}
2657
2658	static bool isVCmpResult(Register Reg, MachineRegisterInfo &MRI) {
2659	if (Reg.isPhysical())
2660	return false;
2661
2662	MachineInstr &MI = *MRI.getUniqueVRegDef(Reg);
2663	const unsigned Opcode = MI.getOpcode();
2664
2665	if (Opcode == AMDGPU::COPY)
2666	return isVCmpResult(MI.getOperand(1).getReg(), MRI);
2667
2668	if (Opcode == AMDGPU::G_AND \|\| Opcode == AMDGPU::G_OR \|\|
2669	Opcode == AMDGPU::G_XOR)
2670	return isVCmpResult(MI.getOperand(1).getReg(), MRI) &&
2671	isVCmpResult(MI.getOperand(2).getReg(), MRI);
2672
2673	if (Opcode == TargetOpcode::G_INTRINSIC)
2674	return MI.getIntrinsicID() == Intrinsic::amdgcn_class;
2675
2676	return Opcode == AMDGPU::G_ICMP \|\| Opcode == AMDGPU::G_FCMP;
2677	}
2678
2679	bool AMDGPUInstructionSelector::selectG_BRCOND(MachineInstr &I) const {
2680	MachineBasicBlock *BB = I.getParent();
2681	MachineOperand &CondOp = I.getOperand(0);
2682	Register CondReg = CondOp.getReg();
2683	const DebugLoc &DL = I.getDebugLoc();
2684
2685	unsigned BrOpcode;
2686	Register CondPhysReg;
2687	const TargetRegisterClass *ConstrainRC;
2688
2689	// In SelectionDAG, we inspect the IR block for uniformity metadata to decide
2690	// whether the branch is uniform when selecting the instruction. In
2691	// GlobalISel, we should push that decision into RegBankSelect. Assume for now
2692	// RegBankSelect knows what it's doing if the branch condition is scc, even
2693	// though it currently does not.
2694	if (!isVCC(CondReg, *MRI)) {
2695	if (MRI->getType(CondReg) != LLT::scalar(32))
2696	return false;
2697
2698	CondPhysReg = AMDGPU::SCC;
2699	BrOpcode = AMDGPU::S_CBRANCH_SCC1;
2700	ConstrainRC = &AMDGPU::SReg_32RegClass;
2701	} else {
2702	// FIXME: Should scc->vcc copies and with exec?
2703
2704	// Unless the value of CondReg is a result of a V_CMP* instruction then we
2705	// need to insert an and with exec.
2706	if (!isVCmpResult(CondReg, *MRI)) {
2707	const bool Is64 = STI.isWave64();
2708	const unsigned Opcode = Is64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32;
2709	const Register Exec = Is64 ? AMDGPU::EXEC : AMDGPU::EXEC_LO;
2710
2711	Register TmpReg = MRI->createVirtualRegister(TRI.getBoolRC());
2712	BuildMI(*BB, &I, DL, TII.get(Opcode), TmpReg)
2713	.addReg(CondReg)
2714	.addReg(Exec);
2715	CondReg = TmpReg;
2716	}
2717
2718	CondPhysReg = TRI.getVCC();
2719	BrOpcode = AMDGPU::S_CBRANCH_VCCNZ;
2720	ConstrainRC = TRI.getBoolRC();
2721	}
2722
2723	if (!MRI->getRegClassOrNull(CondReg))
2724	MRI->setRegClass(CondReg, ConstrainRC);
2725
2726	BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), CondPhysReg)
2727	.addReg(CondReg);
2728	BuildMI(*BB, &I, DL, TII.get(BrOpcode))
2729	.addMBB(I.getOperand(1).getMBB());
2730
2731	I.eraseFromParent();
2732	return true;
2733	}
2734
2735	bool AMDGPUInstructionSelector::selectG_GLOBAL_VALUE(
2736	MachineInstr &I) const {
2737	Register DstReg = I.getOperand(0).getReg();
2738	const RegisterBank DstRB = RBI.getRegBank(DstReg, MRI, TRI);
2739	const bool IsVGPR = DstRB->getID() == AMDGPU::VGPRRegBankID;
2740	I.setDesc(TII.get(IsVGPR ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32));
2741	if (IsVGPR)
2742	I.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true));
2743
2744	return RBI.constrainGenericRegister(
2745	DstReg, IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass, *MRI);
2746	}
2747
2748	bool AMDGPUInstructionSelector::selectG_PTRMASK(MachineInstr &I) const {
2749	Register DstReg = I.getOperand(0).getReg();
2750	Register SrcReg = I.getOperand(1).getReg();
2751	Register MaskReg = I.getOperand(2).getReg();
2752	LLT Ty = MRI->getType(DstReg);
2753	LLT MaskTy = MRI->getType(MaskReg);
2754	MachineBasicBlock *BB = I.getParent();
2755	const DebugLoc &DL = I.getDebugLoc();
2756
2757	const RegisterBank DstRB = RBI.getRegBank(DstReg, MRI, TRI);
2758	const RegisterBank SrcRB = RBI.getRegBank(SrcReg, MRI, TRI);
2759	const RegisterBank MaskRB = RBI.getRegBank(MaskReg, MRI, TRI);
2760	const bool IsVGPR = DstRB->getID() == AMDGPU::VGPRRegBankID;
2761	if (DstRB != SrcRB) // Should only happen for hand written MIR.
2762	return false;
2763
2764	// Try to avoid emitting a bit operation when we only need to touch half of
2765	// the 64-bit pointer.
2766	APInt MaskOnes = KnownBits->getKnownOnes(MaskReg).zext(64);
2767	const APInt MaskHi32 = APInt::getHighBitsSet(64, 32);
2768	const APInt MaskLo32 = APInt::getLowBitsSet(64, 32);
2769
2770	const bool CanCopyLow32 = (MaskOnes & MaskLo32) == MaskLo32;
2771	const bool CanCopyHi32 = (MaskOnes & MaskHi32) == MaskHi32;
2772
2773	if (!IsVGPR && Ty.getSizeInBits() == 64 &&
2774	!CanCopyLow32 && !CanCopyHi32) {
2775	auto MIB = BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_AND_B64), DstReg)
2776	.addReg(SrcReg)
2777	.addReg(MaskReg);
2778	I.eraseFromParent();
2779	return constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
2780	}
2781
2782	unsigned NewOpc = IsVGPR ? AMDGPU::V_AND_B32_e64 : AMDGPU::S_AND_B32;
2783	const TargetRegisterClass &RegRC
2784	= IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass;
2785
2786	const TargetRegisterClass DstRC = TRI.getRegClassForTypeOnBank(Ty, DstRB);
2787	const TargetRegisterClass SrcRC = TRI.getRegClassForTypeOnBank(Ty, SrcRB);
2788	const TargetRegisterClass *MaskRC =
2789	TRI.getRegClassForTypeOnBank(MaskTy, *MaskRB);
2790
2791	if (!RBI.constrainGenericRegister(DstReg, DstRC, MRI) \|\|
2792	!RBI.constrainGenericRegister(SrcReg, SrcRC, MRI) \|\|
2793	!RBI.constrainGenericRegister(MaskReg, MaskRC, MRI))
2794	return false;
2795
2796	if (Ty.getSizeInBits() == 32) {
2797	assert(MaskTy.getSizeInBits() == 32 &&(static_cast <bool> (MaskTy.getSizeInBits() == 32 && "ptrmask should have been narrowed during legalize") ? void ( 0) : __assert_fail ("MaskTy.getSizeInBits() == 32 && \"ptrmask should have been narrowed during legalize\"" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 2798 , __extension__ __PRETTY_FUNCTION__))
2798	"ptrmask should have been narrowed during legalize")(static_cast <bool> (MaskTy.getSizeInBits() == 32 && "ptrmask should have been narrowed during legalize") ? void ( 0) : __assert_fail ("MaskTy.getSizeInBits() == 32 && \"ptrmask should have been narrowed during legalize\"" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 2798 , __extension__ __PRETTY_FUNCTION__));
2799
2800	BuildMI(*BB, &I, DL, TII.get(NewOpc), DstReg)
2801	.addReg(SrcReg)
2802	.addReg(MaskReg);
2803	I.eraseFromParent();
2804	return true;
2805	}
2806
2807	Register HiReg = MRI->createVirtualRegister(&RegRC);
2808	Register LoReg = MRI->createVirtualRegister(&RegRC);
2809
2810	// Extract the subregisters from the source pointer.
2811	BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), LoReg)
2812	.addReg(SrcReg, 0, AMDGPU::sub0);
2813	BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), HiReg)
2814	.addReg(SrcReg, 0, AMDGPU::sub1);
2815
2816	Register MaskedLo, MaskedHi;
2817
2818	if (CanCopyLow32) {
2819	// If all the bits in the low half are 1, we only need a copy for it.
2820	MaskedLo = LoReg;
2821	} else {
2822	// Extract the mask subregister and apply the and.
2823	Register MaskLo = MRI->createVirtualRegister(&RegRC);
2824	MaskedLo = MRI->createVirtualRegister(&RegRC);
2825
2826	BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), MaskLo)
2827	.addReg(MaskReg, 0, AMDGPU::sub0);
2828	BuildMI(*BB, &I, DL, TII.get(NewOpc), MaskedLo)
2829	.addReg(LoReg)
2830	.addReg(MaskLo);
2831	}
2832
2833	if (CanCopyHi32) {
2834	// If all the bits in the high half are 1, we only need a copy for it.
2835	MaskedHi = HiReg;
2836	} else {
2837	Register MaskHi = MRI->createVirtualRegister(&RegRC);
2838	MaskedHi = MRI->createVirtualRegister(&RegRC);
2839
2840	BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), MaskHi)
2841	.addReg(MaskReg, 0, AMDGPU::sub1);
2842	BuildMI(*BB, &I, DL, TII.get(NewOpc), MaskedHi)
2843	.addReg(HiReg)
2844	.addReg(MaskHi);
2845	}
2846
2847	BuildMI(*BB, &I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg)
2848	.addReg(MaskedLo)
2849	.addImm(AMDGPU::sub0)
2850	.addReg(MaskedHi)
2851	.addImm(AMDGPU::sub1);
2852	I.eraseFromParent();
2853	return true;
2854	}
2855
2856	/// Return the register to use for the index value, and the subregister to use
2857	/// for the indirectly accessed register.
2858	static std::pair<Register, unsigned>
2859	computeIndirectRegIndex(MachineRegisterInfo &MRI, const SIRegisterInfo &TRI,
2860	const TargetRegisterClass *SuperRC, Register IdxReg,
2861	unsigned EltSize, GISelKnownBits &KnownBits) {
2862	Register IdxBaseReg;
2863	int Offset;
2864
2865	std::tie(IdxBaseReg, Offset) =
2866	AMDGPU::getBaseWithConstantOffset(MRI, IdxReg, &KnownBits);
2867	if (IdxBaseReg == AMDGPU::NoRegister) {
2868	// This will happen if the index is a known constant. This should ordinarily
2869	// be legalized out, but handle it as a register just in case.
2870	assert(Offset == 0)(static_cast <bool> (Offset == 0) ? void (0) : __assert_fail ("Offset == 0", "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp" , 2870, __extension__ __PRETTY_FUNCTION__));
2871	IdxBaseReg = IdxReg;
2872	}
2873
2874	ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(SuperRC, EltSize);
2875
2876	// Skip out of bounds offsets, or else we would end up using an undefined
2877	// register.
2878	if (static_cast<unsigned>(Offset) >= SubRegs.size())
2879	return std::pair(IdxReg, SubRegs[0]);
2880	return std::pair(IdxBaseReg, SubRegs[Offset]);
2881	}
2882
2883	bool AMDGPUInstructionSelector::selectG_EXTRACT_VECTOR_ELT(
2884	MachineInstr &MI) const {
2885	Register DstReg = MI.getOperand(0).getReg();
2886	Register SrcReg = MI.getOperand(1).getReg();
2887	Register IdxReg = MI.getOperand(2).getReg();
2888
2889	LLT DstTy = MRI->getType(DstReg);
2890	LLT SrcTy = MRI->getType(SrcReg);
2891
2892	const RegisterBank DstRB = RBI.getRegBank(DstReg, MRI, TRI);
2893	const RegisterBank SrcRB = RBI.getRegBank(SrcReg, MRI, TRI);
2894	const RegisterBank IdxRB = RBI.getRegBank(IdxReg, MRI, TRI);
2895
2896	// The index must be scalar. If it wasn't RegBankSelect should have moved this
2897	// into a waterfall loop.
2898	if (IdxRB->getID() != AMDGPU::SGPRRegBankID)
2899	return false;
2900
2901	const TargetRegisterClass *SrcRC =
2902	TRI.getRegClassForTypeOnBank(SrcTy, *SrcRB);
2903	const TargetRegisterClass *DstRC =
2904	TRI.getRegClassForTypeOnBank(DstTy, *DstRB);
2905	if (!SrcRC \|\| !DstRC)
2906	return false;
2907	if (!RBI.constrainGenericRegister(SrcReg, SrcRC, MRI) \|\|
2908	!RBI.constrainGenericRegister(DstReg, DstRC, MRI) \|\|
2909	!RBI.constrainGenericRegister(IdxReg, AMDGPU::SReg_32RegClass, *MRI))
2910	return false;
2911
2912	MachineBasicBlock *BB = MI.getParent();
2913	const DebugLoc &DL = MI.getDebugLoc();
2914	const bool Is64 = DstTy.getSizeInBits() == 64;
2915
2916	unsigned SubReg;
2917	std::tie(IdxReg, SubReg) = computeIndirectRegIndex(
2918	MRI, TRI, SrcRC, IdxReg, DstTy.getSizeInBits() / 8, KnownBits);
2919
2920	if (SrcRB->getID() == AMDGPU::SGPRRegBankID) {
2921	if (DstTy.getSizeInBits() != 32 && !Is64)
2922	return false;
2923
2924	BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0)
2925	.addReg(IdxReg);
2926
2927	unsigned Opc = Is64 ? AMDGPU::S_MOVRELS_B64 : AMDGPU::S_MOVRELS_B32;
2928	BuildMI(*BB, &MI, DL, TII.get(Opc), DstReg)
2929	.addReg(SrcReg, 0, SubReg)
2930	.addReg(SrcReg, RegState::Implicit);
2931	MI.eraseFromParent();
2932	return true;
2933	}
2934
2935	if (SrcRB->getID() != AMDGPU::VGPRRegBankID \|\| DstTy.getSizeInBits() != 32)
2936	return false;
2937
2938	if (!STI.useVGPRIndexMode()) {
2939	BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0)
2940	.addReg(IdxReg);
2941	BuildMI(*BB, &MI, DL, TII.get(AMDGPU::V_MOVRELS_B32_e32), DstReg)
2942	.addReg(SrcReg, 0, SubReg)
2943	.addReg(SrcReg, RegState::Implicit);
2944	MI.eraseFromParent();
2945	return true;
2946	}
2947
2948	const MCInstrDesc &GPRIDXDesc =
2949	TII.getIndirectGPRIDXPseudo(TRI.getRegSizeInBits(*SrcRC), true);
2950	BuildMI(*BB, MI, DL, GPRIDXDesc, DstReg)
2951	.addReg(SrcReg)
2952	.addReg(IdxReg)
2953	.addImm(SubReg);
2954
2955	MI.eraseFromParent();
2956	return true;
2957	}
2958
2959	// TODO: Fold insert_vector_elt (extract_vector_elt) into movrelsd
2960	bool AMDGPUInstructionSelector::selectG_INSERT_VECTOR_ELT(
2961	MachineInstr &MI) const {
2962	Register DstReg = MI.getOperand(0).getReg();
2963	Register VecReg = MI.getOperand(1).getReg();
2964	Register ValReg = MI.getOperand(2).getReg();
2965	Register IdxReg = MI.getOperand(3).getReg();
2966
2967	LLT VecTy = MRI->getType(DstReg);
2968	LLT ValTy = MRI->getType(ValReg);
2969	unsigned VecSize = VecTy.getSizeInBits();
2970	unsigned ValSize = ValTy.getSizeInBits();
2971
2972	const RegisterBank VecRB = RBI.getRegBank(VecReg, MRI, TRI);
2973	const RegisterBank ValRB = RBI.getRegBank(ValReg, MRI, TRI);
2974	const RegisterBank IdxRB = RBI.getRegBank(IdxReg, MRI, TRI);
2975
2976	assert(VecTy.getElementType() == ValTy)(static_cast <bool> (VecTy.getElementType() == ValTy) ? void (0) : __assert_fail ("VecTy.getElementType() == ValTy", "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 2976 , __extension__ __PRETTY_FUNCTION__));
2977
2978	// The index must be scalar. If it wasn't RegBankSelect should have moved this
2979	// into a waterfall loop.
2980	if (IdxRB->getID() != AMDGPU::SGPRRegBankID)
2981	return false;
2982
2983	const TargetRegisterClass *VecRC =
2984	TRI.getRegClassForTypeOnBank(VecTy, *VecRB);
2985	const TargetRegisterClass *ValRC =
2986	TRI.getRegClassForTypeOnBank(ValTy, *ValRB);
2987
2988	if (!RBI.constrainGenericRegister(VecReg, VecRC, MRI) \|\|
2989	!RBI.constrainGenericRegister(DstReg, VecRC, MRI) \|\|
2990	!RBI.constrainGenericRegister(ValReg, ValRC, MRI) \|\|
2991	!RBI.constrainGenericRegister(IdxReg, AMDGPU::SReg_32RegClass, *MRI))
2992	return false;
2993
2994	if (VecRB->getID() == AMDGPU::VGPRRegBankID && ValSize != 32)
2995	return false;
2996
2997	unsigned SubReg;
2998	std::tie(IdxReg, SubReg) = computeIndirectRegIndex(*MRI, TRI, VecRC, IdxReg,
2999	ValSize / 8, *KnownBits);
3000
3001	const bool IndexMode = VecRB->getID() == AMDGPU::VGPRRegBankID &&
3002	STI.useVGPRIndexMode();
3003
3004	MachineBasicBlock *BB = MI.getParent();
3005	const DebugLoc &DL = MI.getDebugLoc();
3006
3007	if (!IndexMode) {
3008	BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0)
3009	.addReg(IdxReg);
3010
3011	const MCInstrDesc &RegWriteOp = TII.getIndirectRegWriteMovRelPseudo(
3012	VecSize, ValSize, VecRB->getID() == AMDGPU::SGPRRegBankID);
3013	BuildMI(*BB, MI, DL, RegWriteOp, DstReg)
3014	.addReg(VecReg)
3015	.addReg(ValReg)
3016	.addImm(SubReg);
3017	MI.eraseFromParent();
3018	return true;
3019	}
3020
3021	const MCInstrDesc &GPRIDXDesc =
3022	TII.getIndirectGPRIDXPseudo(TRI.getRegSizeInBits(*VecRC), false);
3023	BuildMI(*BB, MI, DL, GPRIDXDesc, DstReg)
3024	.addReg(VecReg)
3025	.addReg(ValReg)
3026	.addReg(IdxReg)
3027	.addImm(SubReg);
3028
3029	MI.eraseFromParent();
3030	return true;
3031	}
3032
3033	bool AMDGPUInstructionSelector::selectBufferLoadLds(MachineInstr &MI) const {
3034	unsigned Opc;
3035	unsigned Size = MI.getOperand(3).getImm();
3036
3037	// The struct intrinsic variants add one additional operand over raw.
3038	const bool HasVIndex = MI.getNumOperands() == 9;
3039	Register VIndex;
3040	int OpOffset = 0;
3041	if (HasVIndex) {
3042	VIndex = MI.getOperand(4).getReg();
3043	OpOffset = 1;
3044	}
3045
3046	Register VOffset = MI.getOperand(4 + OpOffset).getReg();
3047	std::optional<ValueAndVReg> MaybeVOffset =
3048	getIConstantVRegValWithLookThrough(VOffset, *MRI);
3049	const bool HasVOffset = !MaybeVOffset \|\| MaybeVOffset->Value.getZExtValue();
3050
3051	switch (Size) {
3052	default:
3053	return false;
3054	case 1:
3055	Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_BOTHEN
3056	: AMDGPU::BUFFER_LOAD_UBYTE_LDS_IDXEN
3057	: HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFEN
3058	: AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFSET;
3059	break;
3060	case 2:
3061	Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_BOTHEN
3062	: AMDGPU::BUFFER_LOAD_USHORT_LDS_IDXEN
3063	: HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFEN
3064	: AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFSET;
3065	break;
3066	case 4:
3067	Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_BOTHEN
3068	: AMDGPU::BUFFER_LOAD_DWORD_LDS_IDXEN
3069	: HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFEN
3070	: AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFSET;
3071	break;
3072	}
3073
3074	MachineBasicBlock *MBB = MI.getParent();
3075	const DebugLoc &DL = MI.getDebugLoc();
3076	BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0)
3077	.add(MI.getOperand(2));
3078
3079	auto MIB = BuildMI(*MBB, &MI, DL, TII.get(Opc));
3080
3081	if (HasVIndex && HasVOffset) {
3082	Register IdxReg = MRI->createVirtualRegister(TRI.getVGPR64Class());
3083	BuildMI(MBB, &MIB, DL, TII.get(AMDGPU::REG_SEQUENCE), IdxReg)
3084	.addReg(VIndex)
3085	.addImm(AMDGPU::sub0)
3086	.addReg(VOffset)
3087	.addImm(AMDGPU::sub1);
3088
3089	MIB.addReg(IdxReg);
3090	} else if (HasVIndex) {
3091	MIB.addReg(VIndex);
3092	} else if (HasVOffset) {
3093	MIB.addReg(VOffset);
3094	}
3095
3096	MIB.add(MI.getOperand(1)); // rsrc
3097	MIB.add(MI.getOperand(5 + OpOffset)); // soffset
3098	MIB.add(MI.getOperand(6 + OpOffset)); // imm offset
3099	unsigned Aux = MI.getOperand(7 + OpOffset).getImm();
3100	MIB.addImm(Aux & AMDGPU::CPol::ALL); // cpol
3101	MIB.addImm((Aux >> 3) & 1); // swz
3102
3103	MachineMemOperand LoadMMO = MI.memoperands_begin();
3104	MachinePointerInfo LoadPtrI = LoadMMO->getPointerInfo();
3105	LoadPtrI.Offset = MI.getOperand(6 + OpOffset).getImm();
3106	MachinePointerInfo StorePtrI = LoadPtrI;
3107	StorePtrI.V = nullptr;
3108	StorePtrI.AddrSpace = AMDGPUAS::LOCAL_ADDRESS;
3109
3110	auto F = LoadMMO->getFlags() &
3111	~(MachineMemOperand::MOStore \| MachineMemOperand::MOLoad);
3112	LoadMMO = MF->getMachineMemOperand(LoadPtrI, F \| MachineMemOperand::MOLoad,
3113	Size, LoadMMO->getBaseAlign());
3114
3115	MachineMemOperand *StoreMMO =
3116	MF->getMachineMemOperand(StorePtrI, F \| MachineMemOperand::MOStore,
3117	sizeof(int32_t), LoadMMO->getBaseAlign());
3118
3119	MIB.setMemRefs({LoadMMO, StoreMMO});
3120
3121	MI.eraseFromParent();
3122	return constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
3123	}
3124
3125	/// Match a zero extend from a 32-bit value to 64-bits.
3126	static Register matchZeroExtendFromS32(MachineRegisterInfo &MRI, Register Reg) {
3127	Register ZExtSrc;
3128	if (mi_match(Reg, MRI, m_GZExt(m_Reg(ZExtSrc))))
3129	return MRI.getType(ZExtSrc) == LLT::scalar(32) ? ZExtSrc : Register();
3130
3131	// Match legalized form %zext = G_MERGE_VALUES (s32 %x), (s32 0)
3132	const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI);
3133	if (Def->getOpcode() != AMDGPU::G_MERGE_VALUES)
3134	return Register();
3135
3136	assert(Def->getNumOperands() == 3 &&(static_cast <bool> (Def->getNumOperands() == 3 && MRI.getType(Def->getOperand(0).getReg()) == LLT::scalar(64 )) ? void (0) : __assert_fail ("Def->getNumOperands() == 3 && MRI.getType(Def->getOperand(0).getReg()) == LLT::scalar(64)" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 3137 , __extension__ __PRETTY_FUNCTION__))
3137	MRI.getType(Def->getOperand(0).getReg()) == LLT::scalar(64))(static_cast <bool> (Def->getNumOperands() == 3 && MRI.getType(Def->getOperand(0).getReg()) == LLT::scalar(64 )) ? void (0) : __assert_fail ("Def->getNumOperands() == 3 && MRI.getType(Def->getOperand(0).getReg()) == LLT::scalar(64)" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 3137 , __extension__ __PRETTY_FUNCTION__));
3138	if (mi_match(Def->getOperand(2).getReg(), MRI, m_ZeroInt())) {
3139	return Def->getOperand(1).getReg();
3140	}
3141
3142	return Register();
3143	}
3144
3145	bool AMDGPUInstructionSelector::selectGlobalLoadLds(MachineInstr &MI) const{
3146	unsigned Opc;
3147	unsigned Size = MI.getOperand(3).getImm();
3148
3149	switch (Size) {
3150	default:
3151	return false;
3152	case 1:
3153	Opc = AMDGPU::GLOBAL_LOAD_LDS_UBYTE;
3154	break;
3155	case 2:
3156	Opc = AMDGPU::GLOBAL_LOAD_LDS_USHORT;
3157	break;
3158	case 4:
3159	Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORD;
3160	break;
3161	}
3162
3163	MachineBasicBlock *MBB = MI.getParent();
3164	const DebugLoc &DL = MI.getDebugLoc();
3165	BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0)
3166	.add(MI.getOperand(2));
3167
3168	Register Addr = MI.getOperand(1).getReg();
3169	Register VOffset;
3170	// Try to split SAddr and VOffset. Global and LDS pointers share the same
3171	// immediate offset, so we cannot use a regular SelectGlobalSAddr().
3172	if (!isSGPR(Addr)) {
3173	auto AddrDef = getDefSrcRegIgnoringCopies(Addr, *MRI);
3174	if (isSGPR(AddrDef->Reg)) {
3175	Addr = AddrDef->Reg;
3176	} else if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) {
3177	Register SAddr =
3178	getSrcRegIgnoringCopies(AddrDef->MI->getOperand(1).getReg(), *MRI);
3179	if (isSGPR(SAddr)) {
3180	Register PtrBaseOffset = AddrDef->MI->getOperand(2).getReg();
3181	if (Register Off = matchZeroExtendFromS32(*MRI, PtrBaseOffset)) {
3182	Addr = SAddr;
3183	VOffset = Off;
3184	}
3185	}
3186	}
3187	}
3188
3189	if (isSGPR(Addr)) {
3190	Opc = AMDGPU::getGlobalSaddrOp(Opc);
3191	if (!VOffset) {
3192	VOffset = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
3193	BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::V_MOV_B32_e32), VOffset)
3194	.addImm(0);
3195	}
3196	}
3197
3198	auto MIB = BuildMI(*MBB, &MI, DL, TII.get(Opc))
3199	.addReg(Addr);
3200
3201	if (isSGPR(Addr))
3202	MIB.addReg(VOffset);
3203
3204	MIB.add(MI.getOperand(4)) // offset
3205	.add(MI.getOperand(5)); // cpol
3206
3207	MachineMemOperand LoadMMO = MI.memoperands_begin();
3208	MachinePointerInfo LoadPtrI = LoadMMO->getPointerInfo();
3209	LoadPtrI.Offset = MI.getOperand(4).getImm();
3210	MachinePointerInfo StorePtrI = LoadPtrI;
3211	LoadPtrI.AddrSpace = AMDGPUAS::GLOBAL_ADDRESS;
3212	StorePtrI.AddrSpace = AMDGPUAS::LOCAL_ADDRESS;
3213	auto F = LoadMMO->getFlags() &
3214	~(MachineMemOperand::MOStore \| MachineMemOperand::MOLoad);
3215	LoadMMO = MF->getMachineMemOperand(LoadPtrI, F \| MachineMemOperand::MOLoad,
3216	Size, LoadMMO->getBaseAlign());
3217	MachineMemOperand *StoreMMO =
3218	MF->getMachineMemOperand(StorePtrI, F \| MachineMemOperand::MOStore,
3219	sizeof(int32_t), Align(4));
3220
3221	MIB.setMemRefs({LoadMMO, StoreMMO});
3222
3223	MI.eraseFromParent();
3224	return constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
3225	}
3226
3227	bool AMDGPUInstructionSelector::selectBVHIntrinsic(MachineInstr &MI) const{
3228	MI.setDesc(TII.get(MI.getOperand(1).getImm()));
3229	MI.removeOperand(1);
3230	MI.addImplicitDefUseOperands(*MI.getParent()->getParent());
3231	return true;
3232	}
3233
3234	bool AMDGPUInstructionSelector::selectSMFMACIntrin(MachineInstr &MI) const {
3235	unsigned Opc;
3236	switch (MI.getIntrinsicID()) {
3237	case Intrinsic::amdgcn_smfmac_f32_16x16x32_f16:
3238	Opc = AMDGPU::V_SMFMAC_F32_16X16X32_F16_e64;
3239	break;
3240	case Intrinsic::amdgcn_smfmac_f32_32x32x16_f16:
3241	Opc = AMDGPU::V_SMFMAC_F32_32X32X16_F16_e64;
3242	break;
3243	case Intrinsic::amdgcn_smfmac_f32_16x16x32_bf16:
3244	Opc = AMDGPU::V_SMFMAC_F32_16X16X32_BF16_e64;
3245	break;
3246	case Intrinsic::amdgcn_smfmac_f32_32x32x16_bf16:
3247	Opc = AMDGPU::V_SMFMAC_F32_32X32X16_BF16_e64;
3248	break;
3249	case Intrinsic::amdgcn_smfmac_i32_16x16x64_i8:
3250	Opc = AMDGPU::V_SMFMAC_I32_16X16X64_I8_e64;
3251	break;
3252	case Intrinsic::amdgcn_smfmac_i32_32x32x32_i8:
3253	Opc = AMDGPU::V_SMFMAC_I32_32X32X32_I8_e64;
3254	break;
3255	case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_bf8:
3256	Opc = AMDGPU::V_SMFMAC_F32_16X16X64_BF8_BF8_e64;
3257	break;
3258	case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_fp8:
3259	Opc = AMDGPU::V_SMFMAC_F32_16X16X64_BF8_FP8_e64;
3260	break;
3261	case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_bf8:
3262	Opc = AMDGPU::V_SMFMAC_F32_16X16X64_FP8_BF8_e64;
3263	break;
3264	case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_fp8:
3265	Opc = AMDGPU::V_SMFMAC_F32_16X16X64_FP8_FP8_e64;
3266	break;
3267	case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_bf8:
3268	Opc = AMDGPU::V_SMFMAC_F32_32X32X32_BF8_BF8_e64;
3269	break;
3270	case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_fp8:
3271	Opc = AMDGPU::V_SMFMAC_F32_32X32X32_BF8_FP8_e64;
3272	break;
3273	case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_bf8:
3274	Opc = AMDGPU::V_SMFMAC_F32_32X32X32_FP8_BF8_e64;
3275	break;
3276	case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_fp8:
3277	Opc = AMDGPU::V_SMFMAC_F32_32X32X32_FP8_FP8_e64;
3278	break;
3279	default:
3280	llvm_unreachable("unhandled smfmac intrinsic")::llvm::llvm_unreachable_internal("unhandled smfmac intrinsic" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 3280 );
3281	}
3282
3283	auto VDst_In = MI.getOperand(4);
3284
3285	MI.setDesc(TII.get(Opc));
3286	MI.removeOperand(4); // VDst_In
3287	MI.removeOperand(1); // Intrinsic ID
3288	MI.addOperand(VDst_In); // Readd VDst_In to the end
3289	MI.addImplicitDefUseOperands(*MI.getParent()->getParent());
3290	return true;
3291	}
3292
3293	bool AMDGPUInstructionSelector::selectWaveAddress(MachineInstr &MI) const {
3294	Register DstReg = MI.getOperand(0).getReg();
3295	Register SrcReg = MI.getOperand(1).getReg();
3296	const RegisterBank DstRB = RBI.getRegBank(DstReg, MRI, TRI);
3297	const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID;
3298	MachineBasicBlock *MBB = MI.getParent();
3299	const DebugLoc &DL = MI.getDebugLoc();
3300
3301	if (IsVALU) {
3302	BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_LSHRREV_B32_e64), DstReg)
3303	.addImm(Subtarget->getWavefrontSizeLog2())
3304	.addReg(SrcReg);
3305	} else {
3306	BuildMI(*MBB, MI, DL, TII.get(AMDGPU::S_LSHR_B32), DstReg)
3307	.addReg(SrcReg)
3308	.addImm(Subtarget->getWavefrontSizeLog2());
3309	}
3310
3311	const TargetRegisterClass &RC =
3312	IsVALU ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass;
3313	if (!RBI.constrainGenericRegister(DstReg, RC, *MRI))
3314	return false;
3315
3316	MI.eraseFromParent();
3317	return true;
3318	}
3319
3320	bool AMDGPUInstructionSelector::select(MachineInstr &I) {
3321	if (I.isPHI())
3322	return selectPHI(I);
3323
3324	if (!I.isPreISelOpcode()) {
3325	if (I.isCopy())
3326	return selectCOPY(I);
3327	return true;
3328	}
3329
3330	switch (I.getOpcode()) {
3331	case TargetOpcode::G_AND:
3332	case TargetOpcode::G_OR:
3333	case TargetOpcode::G_XOR:
3334	if (selectImpl(I, *CoverageInfo))
3335	return true;
3336	return selectG_AND_OR_XOR(I);
3337	case TargetOpcode::G_ADD:
3338	case TargetOpcode::G_SUB:
3339	if (selectImpl(I, *CoverageInfo))
3340	return true;
3341	return selectG_ADD_SUB(I);
3342	case TargetOpcode::G_UADDO:
3343	case TargetOpcode::G_USUBO:
3344	case TargetOpcode::G_UADDE:
3345	case TargetOpcode::G_USUBE:
3346	return selectG_UADDO_USUBO_UADDE_USUBE(I);
3347	case AMDGPU::G_AMDGPU_MAD_U64_U32:
3348	case AMDGPU::G_AMDGPU_MAD_I64_I32:
3349	return selectG_AMDGPU_MAD_64_32(I);
3350	case TargetOpcode::G_INTTOPTR:
3351	case TargetOpcode::G_BITCAST:
3352	case TargetOpcode::G_PTRTOINT:
3353	return selectCOPY(I);
3354	case TargetOpcode::G_CONSTANT:
3355	case TargetOpcode::G_FCONSTANT:
3356	return selectG_CONSTANT(I);
3357	case TargetOpcode::G_FNEG:
3358	if (selectImpl(I, *CoverageInfo))
3359	return true;
3360	return selectG_FNEG(I);
3361	case TargetOpcode::G_FABS:
3362	if (selectImpl(I, *CoverageInfo))
3363	return true;
3364	return selectG_FABS(I);
3365	case TargetOpcode::G_EXTRACT:
3366	return selectG_EXTRACT(I);
3367	case TargetOpcode::G_MERGE_VALUES:
3368	case TargetOpcode::G_CONCAT_VECTORS:
3369	return selectG_MERGE_VALUES(I);
3370	case TargetOpcode::G_UNMERGE_VALUES:
3371	return selectG_UNMERGE_VALUES(I);
3372	case TargetOpcode::G_BUILD_VECTOR:
3373	case TargetOpcode::G_BUILD_VECTOR_TRUNC:
3374	return selectG_BUILD_VECTOR(I);
3375	case TargetOpcode::G_PTR_ADD:
3376	if (selectImpl(I, *CoverageInfo))
3377	return true;
3378	return selectG_PTR_ADD(I);
3379	case TargetOpcode::G_IMPLICIT_DEF:
3380	return selectG_IMPLICIT_DEF(I);
3381	case TargetOpcode::G_FREEZE:
3382	return selectCOPY(I);
3383	case TargetOpcode::G_INSERT:
3384	return selectG_INSERT(I);
3385	case TargetOpcode::G_INTRINSIC:
3386	return selectG_INTRINSIC(I);
3387	case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
3388	return selectG_INTRINSIC_W_SIDE_EFFECTS(I);
3389	case TargetOpcode::G_ICMP:
3390	if (selectG_ICMP(I))
3391	return true;
3392	return selectImpl(I, *CoverageInfo);
3393	case TargetOpcode::G_LOAD:
3394	case TargetOpcode::G_STORE:
3395	case TargetOpcode::G_ATOMIC_CMPXCHG:
3396	case TargetOpcode::G_ATOMICRMW_XCHG:
3397	case TargetOpcode::G_ATOMICRMW_ADD:
3398	case TargetOpcode::G_ATOMICRMW_SUB:
3399	case TargetOpcode::G_ATOMICRMW_AND:
3400	case TargetOpcode::G_ATOMICRMW_OR:
3401	case TargetOpcode::G_ATOMICRMW_XOR:
3402	case TargetOpcode::G_ATOMICRMW_MIN:
3403	case TargetOpcode::G_ATOMICRMW_MAX:
3404	case TargetOpcode::G_ATOMICRMW_UMIN:
3405	case TargetOpcode::G_ATOMICRMW_UMAX:
3406	case TargetOpcode::G_ATOMICRMW_UINC_WRAP:
3407	case TargetOpcode::G_ATOMICRMW_UDEC_WRAP:
3408	case TargetOpcode::G_ATOMICRMW_FADD:
3409	case AMDGPU::G_AMDGPU_ATOMIC_FMIN:
3410	case AMDGPU::G_AMDGPU_ATOMIC_FMAX:
3411	return selectG_LOAD_STORE_ATOMICRMW(I);
3412	case TargetOpcode::G_SELECT:
3413	return selectG_SELECT(I);
3414	case TargetOpcode::G_TRUNC:
3415	return selectG_TRUNC(I);
3416	case TargetOpcode::G_SEXT:
3417	case TargetOpcode::G_ZEXT:
3418	case TargetOpcode::G_ANYEXT:
3419	case TargetOpcode::G_SEXT_INREG:
3420	// This is a workaround. For extension from type i1, `selectImpl()` uses
3421	// patterns from TD file and generates an illegal VGPR to SGPR COPY as type
3422	// i1 can only be hold in a SGPR class.
3423	if (MRI->getType(I.getOperand(1).getReg()) != LLT::scalar(1) &&
3424	selectImpl(I, *CoverageInfo))
3425	return true;
3426	return selectG_SZA_EXT(I);
3427	case TargetOpcode::G_BRCOND:
3428	return selectG_BRCOND(I);
3429	case TargetOpcode::G_GLOBAL_VALUE:
3430	return selectG_GLOBAL_VALUE(I);
3431	case TargetOpcode::G_PTRMASK:
3432	return selectG_PTRMASK(I);
3433	case TargetOpcode::G_EXTRACT_VECTOR_ELT:
3434	return selectG_EXTRACT_VECTOR_ELT(I);
3435	case TargetOpcode::G_INSERT_VECTOR_ELT:
3436	return selectG_INSERT_VECTOR_ELT(I);
3437	case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD:
3438	case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16:
3439	case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE:
3440	case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16: {
3441	const AMDGPU::ImageDimIntrinsicInfo *Intr
3442	= AMDGPU::getImageDimIntrinsicInfo(I.getIntrinsicID());
3443	assert(Intr && "not an image intrinsic with image pseudo")(static_cast <bool> (Intr && "not an image intrinsic with image pseudo" ) ? void (0) : __assert_fail ("Intr && \"not an image intrinsic with image pseudo\"" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 3443 , __extension__ __PRETTY_FUNCTION__));
3444	return selectImageIntrinsic(I, Intr);
3445	}
3446	case AMDGPU::G_AMDGPU_INTRIN_BVH_INTERSECT_RAY:
3447	return selectBVHIntrinsic(I);
3448	case AMDGPU::G_SBFX:
3449	case AMDGPU::G_UBFX:
3450	return selectG_SBFX_UBFX(I);
3451	case AMDGPU::G_SI_CALL:
3452	I.setDesc(TII.get(AMDGPU::SI_CALL));
3453	return true;
3454	case AMDGPU::G_AMDGPU_WAVE_ADDRESS:
3455	return selectWaveAddress(I);
3456	default:
3457	return selectImpl(I, *CoverageInfo);
3458	}
3459	return false;
3460	}
3461
3462	InstructionSelector::ComplexRendererFns
3463	AMDGPUInstructionSelector::selectVCSRC(MachineOperand &Root) const {
3464	return {{
3465	[=](MachineInstrBuilder &MIB) { MIB.add(Root); }
3466	}};
3467
3468	}
3469
3470	std::pair<Register, unsigned> AMDGPUInstructionSelector::selectVOP3ModsImpl(
3471	MachineOperand &Root, bool AllowAbs, bool OpSel) const {
3472	Register Src = Root.getReg();
3473	unsigned Mods = 0;
3474	MachineInstr MI = getDefIgnoringCopies(Src, MRI);
3475
3476	if (MI->getOpcode() == AMDGPU::G_FNEG) {
3477	Src = MI->getOperand(1).getReg();
3478	Mods \|= SISrcMods::NEG;
3479	MI = getDefIgnoringCopies(Src, *MRI);
3480	}
3481
3482	if (AllowAbs && MI->getOpcode() == AMDGPU::G_FABS) {
3483	Src = MI->getOperand(1).getReg();
3484	Mods \|= SISrcMods::ABS;
3485	}
3486
3487	if (OpSel)
3488	Mods \|= SISrcMods::OP_SEL_0;
3489
3490	return std::pair(Src, Mods);
3491	}
3492
3493	Register AMDGPUInstructionSelector::copyToVGPRIfSrcFolded(
3494	Register Src, unsigned Mods, MachineOperand Root, MachineInstr *InsertPt,
3495	bool ForceVGPR) const {
3496	if ((Mods != 0 \|\| ForceVGPR) &&
3497	RBI.getRegBank(Src, *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID) {
3498
3499	// If we looked through copies to find source modifiers on an SGPR operand,
3500	// we now have an SGPR register source. To avoid potentially violating the
3501	// constant bus restriction, we need to insert a copy to a VGPR.
3502	Register VGPRSrc = MRI->cloneVirtualRegister(Root.getReg());
3503	BuildMI(*InsertPt->getParent(), InsertPt, InsertPt->getDebugLoc(),
3504	TII.get(AMDGPU::COPY), VGPRSrc)
3505	.addReg(Src);
3506	Src = VGPRSrc;
3507	}
3508
3509	return Src;
3510	}
3511
3512	///
3513	/// This will select either an SGPR or VGPR operand and will save us from
3514	/// having to write an extra tablegen pattern.
3515	InstructionSelector::ComplexRendererFns
3516	AMDGPUInstructionSelector::selectVSRC0(MachineOperand &Root) const {
3517	return {{
3518	[=](MachineInstrBuilder &MIB) { MIB.add(Root); }
3519	}};
3520	}
3521
3522	InstructionSelector::ComplexRendererFns
3523	AMDGPUInstructionSelector::selectVOP3Mods0(MachineOperand &Root) const {
3524	Register Src;
3525	unsigned Mods;
3526	std::tie(Src, Mods) = selectVOP3ModsImpl(Root);
3527
3528	return {{
3529	[=](MachineInstrBuilder &MIB) {
3530	MIB.addReg(copyToVGPRIfSrcFolded(Src, Mods, Root, MIB));
3531	},
3532	[=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); }, // src0_mods
3533	[=](MachineInstrBuilder &MIB) { MIB.addImm(0); }, // clamp
3534	[=](MachineInstrBuilder &MIB) { MIB.addImm(0); } // omod
3535	}};
3536	}
3537
3538	InstructionSelector::ComplexRendererFns
3539	AMDGPUInstructionSelector::selectVOP3BMods0(MachineOperand &Root) const {
3540	Register Src;
3541	unsigned Mods;
3542	std::tie(Src, Mods) = selectVOP3ModsImpl(Root, /* AllowAbs */ false);
3543
3544	return {{
3545	[=](MachineInstrBuilder &MIB) {
3546	MIB.addReg(copyToVGPRIfSrcFolded(Src, Mods, Root, MIB));
3547	},
3548	[=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); }, // src0_mods
3549	[=](MachineInstrBuilder &MIB) { MIB.addImm(0); }, // clamp
3550	[=](MachineInstrBuilder &MIB) { MIB.addImm(0); } // omod
3551	}};
3552	}
3553
3554	InstructionSelector::ComplexRendererFns
3555	AMDGPUInstructionSelector::selectVOP3OMods(MachineOperand &Root) const {
3556	return {{
3557	[=](MachineInstrBuilder &MIB) { MIB.add(Root); },
3558	[=](MachineInstrBuilder &MIB) { MIB.addImm(0); }, // clamp
3559	[=](MachineInstrBuilder &MIB) { MIB.addImm(0); } // omod
3560	}};
3561	}
3562
3563	InstructionSelector::ComplexRendererFns
3564	AMDGPUInstructionSelector::selectVOP3Mods(MachineOperand &Root) const {
3565	Register Src;
3566	unsigned Mods;
3567	std::tie(Src, Mods) = selectVOP3ModsImpl(Root);
3568
3569	return {{
3570	[=](MachineInstrBuilder &MIB) {
3571	MIB.addReg(copyToVGPRIfSrcFolded(Src, Mods, Root, MIB));
3572	},
3573	[=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); } // src_mods
3574	}};
3575	}
3576
3577	InstructionSelector::ComplexRendererFns
3578	AMDGPUInstructionSelector::selectVOP3BMods(MachineOperand &Root) const {
3579	Register Src;
3580	unsigned Mods;
3581	std::tie(Src, Mods) = selectVOP3ModsImpl(Root, /* AllowAbs */ false);
3582
3583	return {{
3584	[=](MachineInstrBuilder &MIB) {
3585	MIB.addReg(copyToVGPRIfSrcFolded(Src, Mods, Root, MIB));
3586	},
3587	[=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); } // src_mods
3588	}};
3589	}
3590
3591	InstructionSelector::ComplexRendererFns
3592	AMDGPUInstructionSelector::selectVOP3NoMods(MachineOperand &Root) const {
3593	Register Reg = Root.getReg();
3594	const MachineInstr Def = getDefIgnoringCopies(Reg, MRI);
3595	if (Def->getOpcode() == AMDGPU::G_FNEG \|\| Def->getOpcode() == AMDGPU::G_FABS)
3596	return {};
3597	return {{
3598	[=](MachineInstrBuilder &MIB) { MIB.addReg(Reg); },
3599	}};
3600	}
3601
3602	std::pair<Register, unsigned>
3603	AMDGPUInstructionSelector::selectVOP3PModsImpl(
3604	Register Src, const MachineRegisterInfo &MRI, bool IsDOT) const {
3605	unsigned Mods = 0;
3606	MachineInstr *MI = MRI.getVRegDef(Src);
3607
3608	if (MI && MI->getOpcode() == AMDGPU::G_FNEG &&
3609	// It's possible to see an f32 fneg here, but unlikely.
3610	// TODO: Treat f32 fneg as only high bit.
3611	MRI.getType(Src) == LLT::fixed_vector(2, 16)) {
3612	Mods ^= (SISrcMods::NEG \| SISrcMods::NEG_HI);
3613	Src = MI->getOperand(1).getReg();
3614	MI = MRI.getVRegDef(Src);
	Value stored to 'MI' is never read
3615	}
3616
3617	// TODO: Match op_sel through g_build_vector_trunc and g_shuffle_vector.
3618	(void)IsDOT; // DOTs do not use OPSEL on gfx940+, check ST.hasDOTOpSelHazard()
3619
3620	// Packed instructions do not have abs modifiers.
3621	Mods \|= SISrcMods::OP_SEL_1;
3622
3623	return std::pair(Src, Mods);
3624	}
3625
3626	InstructionSelector::ComplexRendererFns
3627	AMDGPUInstructionSelector::selectVOP3PMods(MachineOperand &Root) const {
3628	MachineRegisterInfo &MRI
3629	= Root.getParent()->getParent()->getParent()->getRegInfo();
3630
3631	Register Src;
3632	unsigned Mods;
3633	std::tie(Src, Mods) = selectVOP3PModsImpl(Root.getReg(), MRI);
3634
3635	return {{
3636	[=](MachineInstrBuilder &MIB) { MIB.addReg(Src); },
3637	[=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); } // src_mods
3638	}};
3639	}
3640
3641	InstructionSelector::ComplexRendererFns
3642	AMDGPUInstructionSelector::selectVOP3PModsDOT(MachineOperand &Root) const {
3643	MachineRegisterInfo &MRI
3644	= Root.getParent()->getParent()->getParent()->getRegInfo();
3645
3646	Register Src;
3647	unsigned Mods;
3648	std::tie(Src, Mods) = selectVOP3PModsImpl(Root.getReg(), MRI, true);
3649
3650	return {{
3651	[=](MachineInstrBuilder &MIB) { MIB.addReg(Src); },
3652	[=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); } // src_mods
3653	}};
3654	}
3655
3656	InstructionSelector::ComplexRendererFns
3657	AMDGPUInstructionSelector::selectDotIUVOP3PMods(MachineOperand &Root) const {
3658	// Literal i1 value set in intrinsic, represents SrcMods for the next operand.
3659	// Value is in Imm operand as i1 sign extended to int64_t.
3660	// 1(-1) promotes packed values to signed, 0 treats them as unsigned.
3661	assert((Root.isImm() && (Root.getImm() == -1 \|\| Root.getImm() == 0)) &&(static_cast <bool> ((Root.isImm() && (Root.getImm () == -1 \|\| Root.getImm() == 0)) && "expected i1 value" ) ? void (0) : __assert_fail ("(Root.isImm() && (Root.getImm() == -1 \|\| Root.getImm() == 0)) && \"expected i1 value\"" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 3662 , __extension__ __PRETTY_FUNCTION__))
3662	"expected i1 value")(static_cast <bool> ((Root.isImm() && (Root.getImm () == -1 \|\| Root.getImm() == 0)) && "expected i1 value" ) ? void (0) : __assert_fail ("(Root.isImm() && (Root.getImm() == -1 \|\| Root.getImm() == 0)) && \"expected i1 value\"" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 3662 , __extension__ __PRETTY_FUNCTION__));
3663	unsigned Mods = SISrcMods::OP_SEL_1;
3664	if (Root.getImm() == -1)
3665	Mods ^= SISrcMods::NEG;
3666	return {{
3667	[=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); } // src_mods
3668	}};
3669	}
3670
3671	InstructionSelector::ComplexRendererFns
3672	AMDGPUInstructionSelector::selectWMMAOpSelVOP3PMods(
3673	MachineOperand &Root) const {
3674	assert((Root.isImm() && (Root.getImm() == -1 \|\| Root.getImm() == 0)) &&(static_cast <bool> ((Root.isImm() && (Root.getImm () == -1 \|\| Root.getImm() == 0)) && "expected i1 value" ) ? void (0) : __assert_fail ("(Root.isImm() && (Root.getImm() == -1 \|\| Root.getImm() == 0)) && \"expected i1 value\"" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 3675 , __extension__ __PRETTY_FUNCTION__))
3675	"expected i1 value")(static_cast <bool> ((Root.isImm() && (Root.getImm () == -1 \|\| Root.getImm() == 0)) && "expected i1 value" ) ? void (0) : __assert_fail ("(Root.isImm() && (Root.getImm() == -1 \|\| Root.getImm() == 0)) && \"expected i1 value\"" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 3675 , __extension__ __PRETTY_FUNCTION__));
3676	unsigned Mods = SISrcMods::OP_SEL_1;
3677	if (Root.getImm() != 0)
3678	Mods \|= SISrcMods::OP_SEL_0;
3679
3680	return {{
3681	[=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); } // src_mods
3682	}};
3683	}
3684
3685	InstructionSelector::ComplexRendererFns
3686	AMDGPUInstructionSelector::selectVOP3OpSelMods(MachineOperand &Root) const {
3687	Register Src;
3688	unsigned Mods;
3689	std::tie(Src, Mods) = selectVOP3ModsImpl(Root);
3690
3691	// FIXME: Handle op_sel
3692	return {{
3693	[=](MachineInstrBuilder &MIB) { MIB.addReg(Src); },
3694	[=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); } // src_mods
3695	}};
3696	}
3697
3698	InstructionSelector::ComplexRendererFns
3699	AMDGPUInstructionSelector::selectVINTERPMods(MachineOperand &Root) const {
3700	Register Src;
3701	unsigned Mods;
3702	std::tie(Src, Mods) = selectVOP3ModsImpl(Root,
3703	/* AllowAbs */ false,
3704	/* OpSel */ false);
3705
3706	return {{
3707	[=](MachineInstrBuilder &MIB) {
3708	MIB.addReg(
3709	copyToVGPRIfSrcFolded(Src, Mods, Root, MIB, /* ForceVGPR */ true));
3710	},
3711	[=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); }, // src0_mods
3712	}};
3713	}
3714
3715	InstructionSelector::ComplexRendererFns
3716	AMDGPUInstructionSelector::selectVINTERPModsHi(MachineOperand &Root) const {
3717	Register Src;
3718	unsigned Mods;
3719	std::tie(Src, Mods) = selectVOP3ModsImpl(Root,
3720	/* AllowAbs */ false,
3721	/* OpSel */ true);
3722
3723	return {{
3724	[=](MachineInstrBuilder &MIB) {
3725	MIB.addReg(
3726	copyToVGPRIfSrcFolded(Src, Mods, Root, MIB, /* ForceVGPR */ true));
3727	},
3728	[=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); }, // src0_mods
3729	}};
3730	}
3731
3732	bool AMDGPUInstructionSelector::selectSmrdOffset(MachineOperand &Root,
3733	Register &Base,
3734	Register *SOffset,
3735	int64_t *Offset) const {
3736	MachineInstr *MI = Root.getParent();
3737	MachineBasicBlock *MBB = MI->getParent();
3738
3739	// FIXME: We should shrink the GEP if the offset is known to be <= 32-bits,
3740	// then we can select all ptr + 32-bit offsets.
3741	SmallVector<GEPInfo, 4> AddrInfo;
3742	getAddrModeInfo(MI, MRI, AddrInfo);
3743
3744	if (AddrInfo.empty())
3745	return false;
3746
3747	const GEPInfo &GEPI = AddrInfo[0];
3748	std::optional<int64_t> EncodedImm =
3749	AMDGPU::getSMRDEncodedOffset(STI, GEPI.Imm, false);
3750
3751	if (SOffset && Offset) {
3752	if (GEPI.SgprParts.size() == 1 && GEPI.Imm != 0 && EncodedImm &&
3753	AddrInfo.size() > 1) {
3754	const GEPInfo &GEPI2 = AddrInfo[1];
3755	if (GEPI2.SgprParts.size() == 2 && GEPI2.Imm == 0) {
3756	if (Register OffsetReg =
3757	matchZeroExtendFromS32(*MRI, GEPI2.SgprParts[1])) {
3758	Base = GEPI2.SgprParts[0];
3759	*SOffset = OffsetReg;
3760	Offset = EncodedImm;
3761	return true;
3762	}
3763	}
3764	}
3765	return false;
3766	}
3767
3768	if (Offset && GEPI.SgprParts.size() == 1 && EncodedImm) {
3769	Base = GEPI.SgprParts[0];
3770	Offset = EncodedImm;
3771	return true;
3772	}
3773
3774	// SGPR offset is unsigned.
3775	if (SOffset && GEPI.SgprParts.size() == 1 && isUInt<32>(GEPI.Imm) &&
3776	GEPI.Imm != 0) {
3777	// If we make it this far we have a load with an 32-bit immediate offset.
3778	// It is OK to select this using a sgpr offset, because we have already
3779	// failed trying to select this load into one of the _IMM variants since
3780	// the _IMM Patterns are considered before the _SGPR patterns.
3781	Base = GEPI.SgprParts[0];
3782	*SOffset = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
3783	BuildMI(MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SOffset)
3784	.addImm(GEPI.Imm);
3785	return true;
3786	}
3787
3788	if (SOffset && GEPI.SgprParts.size() && GEPI.Imm == 0) {
3789	if (Register OffsetReg = matchZeroExtendFromS32(*MRI, GEPI.SgprParts[1])) {
3790	Base = GEPI.SgprParts[0];
3791	*SOffset = OffsetReg;
3792	return true;
3793	}
3794	}
3795
3796	return false;
3797	}
3798
3799	InstructionSelector::ComplexRendererFns
3800	AMDGPUInstructionSelector::selectSmrdImm(MachineOperand &Root) const {
3801	Register Base;
3802	int64_t Offset;
3803	if (!selectSmrdOffset(Root, Base, /* SOffset= */ nullptr, &Offset))
3804	return std::nullopt;
3805
3806	return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(Base); },
3807	[=](MachineInstrBuilder &MIB) { MIB.addImm(Offset); }}};
3808	}
3809
3810	InstructionSelector::ComplexRendererFns
3811	AMDGPUInstructionSelector::selectSmrdImm32(MachineOperand &Root) const {
3812	SmallVector<GEPInfo, 4> AddrInfo;
3813	getAddrModeInfo(Root.getParent(), MRI, AddrInfo);
3814
3815	if (AddrInfo.empty() \|\| AddrInfo[0].SgprParts.size() != 1)
3816	return std::nullopt;
3817
3818	const GEPInfo &GEPInfo = AddrInfo[0];
3819	Register PtrReg = GEPInfo.SgprParts[0];
3820	std::optional<int64_t> EncodedImm =
3821	AMDGPU::getSMRDEncodedLiteralOffset32(STI, GEPInfo.Imm);
3822	if (!EncodedImm)
3823	return std::nullopt;
3824
3825	return {{
3826	[=](MachineInstrBuilder &MIB) { MIB.addReg(PtrReg); },
3827	[=](MachineInstrBuilder &MIB) { MIB.addImm(*EncodedImm); }
3828	}};
3829	}
3830
3831	InstructionSelector::ComplexRendererFns
3832	AMDGPUInstructionSelector::selectSmrdSgpr(MachineOperand &Root) const {
3833	Register Base, SOffset;
3834	if (!selectSmrdOffset(Root, Base, &SOffset, /* Offset= */ nullptr))
3835	return std::nullopt;
3836
3837	return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(Base); },
3838	[=](MachineInstrBuilder &MIB) { MIB.addReg(SOffset); }}};
3839	}
3840
3841	InstructionSelector::ComplexRendererFns
3842	AMDGPUInstructionSelector::selectSmrdSgprImm(MachineOperand &Root) const {
3843	Register Base, SOffset;
3844	int64_t Offset;
3845	if (!selectSmrdOffset(Root, Base, &SOffset, &Offset))
3846	return std::nullopt;
3847
3848	return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(Base); },
3849	[=](MachineInstrBuilder &MIB) { MIB.addReg(SOffset); },
3850	[=](MachineInstrBuilder &MIB) { MIB.addImm(Offset); }}};
3851	}
3852
3853	std::pair<Register, int>
3854	AMDGPUInstructionSelector::selectFlatOffsetImpl(MachineOperand &Root,
3855	uint64_t FlatVariant) const {
3856	MachineInstr *MI = Root.getParent();
3857
3858	auto Default = std::pair(Root.getReg(), 0);
3859
3860	if (!STI.hasFlatInstOffsets())
3861	return Default;
3862
3863	Register PtrBase;
3864	int64_t ConstOffset;
3865	std::tie(PtrBase, ConstOffset) =
3866	getPtrBaseWithConstantOffset(Root.getReg(), *MRI);
3867	if (ConstOffset == 0 \|\| !isFlatScratchBaseLegal(PtrBase, FlatVariant))
3868	return Default;
3869
3870	unsigned AddrSpace = (*MI->memoperands_begin())->getAddrSpace();
3871	if (!TII.isLegalFLATOffset(ConstOffset, AddrSpace, FlatVariant))
3872	return Default;
3873
3874	return std::pair(PtrBase, ConstOffset);
3875	}
3876
3877	InstructionSelector::ComplexRendererFns
3878	AMDGPUInstructionSelector::selectFlatOffset(MachineOperand &Root) const {
3879	auto PtrWithOffset = selectFlatOffsetImpl(Root, SIInstrFlags::FLAT);
3880
3881	return {{
3882	[=](MachineInstrBuilder &MIB) { MIB.addReg(PtrWithOffset.first); },
3883	[=](MachineInstrBuilder &MIB) { MIB.addImm(PtrWithOffset.second); },
3884	}};
3885	}
3886
3887	InstructionSelector::ComplexRendererFns
3888	AMDGPUInstructionSelector::selectGlobalOffset(MachineOperand &Root) const {
3889	auto PtrWithOffset = selectFlatOffsetImpl(Root, SIInstrFlags::FlatGlobal);
3890
3891	return {{
3892	[=](MachineInstrBuilder &MIB) { MIB.addReg(PtrWithOffset.first); },
3893	[=](MachineInstrBuilder &MIB) { MIB.addImm(PtrWithOffset.second); },
3894	}};
3895	}
3896
3897	InstructionSelector::ComplexRendererFns
3898	AMDGPUInstructionSelector::selectScratchOffset(MachineOperand &Root) const {
3899	auto PtrWithOffset = selectFlatOffsetImpl(Root, SIInstrFlags::FlatScratch);
3900
3901	return {{
3902	[=](MachineInstrBuilder &MIB) { MIB.addReg(PtrWithOffset.first); },
3903	[=](MachineInstrBuilder &MIB) { MIB.addImm(PtrWithOffset.second); },
3904	}};
3905	}
3906
3907	// Match (64-bit SGPR base) + (zext vgpr offset) + sext(imm offset)
3908	InstructionSelector::ComplexRendererFns
3909	AMDGPUInstructionSelector::selectGlobalSAddr(MachineOperand &Root) const {
3910	Register Addr = Root.getReg();
3911	Register PtrBase;
3912	int64_t ConstOffset;
3913	int64_t ImmOffset = 0;
3914
3915	// Match the immediate offset first, which canonically is moved as low as
3916	// possible.
3917	std::tie(PtrBase, ConstOffset) = getPtrBaseWithConstantOffset(Addr, *MRI);
3918
3919	if (ConstOffset != 0) {
3920	if (TII.isLegalFLATOffset(ConstOffset, AMDGPUAS::GLOBAL_ADDRESS,
3921	SIInstrFlags::FlatGlobal)) {
3922	Addr = PtrBase;
3923	ImmOffset = ConstOffset;
3924	} else {
3925	auto PtrBaseDef = getDefSrcRegIgnoringCopies(PtrBase, *MRI);
3926	if (isSGPR(PtrBaseDef->Reg)) {
3927	if (ConstOffset > 0) {
3928	// Offset is too large.
3929	//
3930	// saddr + large_offset -> saddr +
3931	// (voffset = large_offset & ~MaxOffset) +
3932	// (large_offset & MaxOffset);
3933	int64_t SplitImmOffset, RemainderOffset;
3934	std::tie(SplitImmOffset, RemainderOffset) = TII.splitFlatOffset(
3935	ConstOffset, AMDGPUAS::GLOBAL_ADDRESS, SIInstrFlags::FlatGlobal);
3936
3937	if (isUInt<32>(RemainderOffset)) {
3938	MachineInstr *MI = Root.getParent();
3939	MachineBasicBlock *MBB = MI->getParent();
3940	Register HighBits =
3941	MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
3942
3943	BuildMI(*MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::V_MOV_B32_e32),
3944	HighBits)
3945	.addImm(RemainderOffset);
3946
3947	return {{
3948	[=](MachineInstrBuilder &MIB) { MIB.addReg(PtrBase); }, // saddr
3949	[=](MachineInstrBuilder &MIB) {
3950	MIB.addReg(HighBits);
3951	}, // voffset
3952	[=](MachineInstrBuilder &MIB) { MIB.addImm(SplitImmOffset); },
3953	}};
3954	}
3955	}
3956
3957	// We are adding a 64 bit SGPR and a constant. If constant bus limit
3958	// is 1 we would need to perform 1 or 2 extra moves for each half of
3959	// the constant and it is better to do a scalar add and then issue a
3960	// single VALU instruction to materialize zero. Otherwise it is less
3961	// instructions to perform VALU adds with immediates or inline literals.
3962	unsigned NumLiterals =
3963	!TII.isInlineConstant(APInt(32, ConstOffset & 0xffffffff)) +
3964	!TII.isInlineConstant(APInt(32, ConstOffset >> 32));
3965	if (STI.getConstantBusLimit(AMDGPU::V_ADD_U32_e64) > NumLiterals)
3966	return std::nullopt;
3967	}
3968	}
3969	}
3970
3971	// Match the variable offset.
3972	auto AddrDef = getDefSrcRegIgnoringCopies(Addr, *MRI);
3973	if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) {
3974	// Look through the SGPR->VGPR copy.
3975	Register SAddr =
3976	getSrcRegIgnoringCopies(AddrDef->MI->getOperand(1).getReg(), *MRI);
3977
3978	if (isSGPR(SAddr)) {
3979	Register PtrBaseOffset = AddrDef->MI->getOperand(2).getReg();
3980
3981	// It's possible voffset is an SGPR here, but the copy to VGPR will be
3982	// inserted later.
3983	if (Register VOffset = matchZeroExtendFromS32(*MRI, PtrBaseOffset)) {
3984	return {{[=](MachineInstrBuilder &MIB) { // saddr
3985	MIB.addReg(SAddr);
3986	},
3987	[=](MachineInstrBuilder &MIB) { // voffset
3988	MIB.addReg(VOffset);
3989	},
3990	[=](MachineInstrBuilder &MIB) { // offset
3991	MIB.addImm(ImmOffset);
3992	}}};
3993	}
3994	}
3995	}
3996
3997	// FIXME: We should probably have folded COPY (G_IMPLICIT_DEF) earlier, and
3998	// drop this.
3999	if (AddrDef->MI->getOpcode() == AMDGPU::G_IMPLICIT_DEF \|\|
4000	AddrDef->MI->getOpcode() == AMDGPU::G_CONSTANT \|\| !isSGPR(AddrDef->Reg))
4001	return std::nullopt;
4002
4003	// It's cheaper to materialize a single 32-bit zero for vaddr than the two
4004	// moves required to copy a 64-bit SGPR to VGPR.
4005	MachineInstr *MI = Root.getParent();
4006	MachineBasicBlock *MBB = MI->getParent();
4007	Register VOffset = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4008
4009	BuildMI(*MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::V_MOV_B32_e32), VOffset)
4010	.addImm(0);
4011
4012	return {{
4013	[=](MachineInstrBuilder &MIB) { MIB.addReg(AddrDef->Reg); }, // saddr
4014	[=](MachineInstrBuilder &MIB) { MIB.addReg(VOffset); }, // voffset
4015	[=](MachineInstrBuilder &MIB) { MIB.addImm(ImmOffset); } // offset
4016	}};
4017	}
4018
4019	InstructionSelector::ComplexRendererFns
4020	AMDGPUInstructionSelector::selectScratchSAddr(MachineOperand &Root) const {
4021	Register Addr = Root.getReg();
4022	Register PtrBase;
4023	int64_t ConstOffset;
4024	int64_t ImmOffset = 0;
4025
4026	// Match the immediate offset first, which canonically is moved as low as
4027	// possible.
4028	std::tie(PtrBase, ConstOffset) = getPtrBaseWithConstantOffset(Addr, *MRI);
4029
4030	if (ConstOffset != 0 && isFlatScratchBaseLegal(PtrBase) &&
4031	TII.isLegalFLATOffset(ConstOffset, AMDGPUAS::PRIVATE_ADDRESS,
4032	SIInstrFlags::FlatScratch)) {
4033	Addr = PtrBase;
4034	ImmOffset = ConstOffset;
4035	}
4036
4037	auto AddrDef = getDefSrcRegIgnoringCopies(Addr, *MRI);
4038	if (AddrDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX) {
4039	int FI = AddrDef->MI->getOperand(1).getIndex();
4040	return {{
4041	[=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(FI); }, // saddr
4042	[=](MachineInstrBuilder &MIB) { MIB.addImm(ImmOffset); } // offset
4043	}};
4044	}
4045
4046	Register SAddr = AddrDef->Reg;
4047
4048	if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) {
4049	Register LHS = AddrDef->MI->getOperand(1).getReg();
4050	Register RHS = AddrDef->MI->getOperand(2).getReg();
4051	auto LHSDef = getDefSrcRegIgnoringCopies(LHS, *MRI);
4052	auto RHSDef = getDefSrcRegIgnoringCopies(RHS, *MRI);
4053
4054	if (LHSDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX &&
4055	isSGPR(RHSDef->Reg)) {
4056	int FI = LHSDef->MI->getOperand(1).getIndex();
4057	MachineInstr &I = *Root.getParent();
4058	MachineBasicBlock *BB = I.getParent();
4059	const DebugLoc &DL = I.getDebugLoc();
4060	SAddr = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
4061
4062	BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_ADD_I32), SAddr)
4063	.addFrameIndex(FI)
4064	.addReg(RHSDef->Reg);
4065	}
4066	}
4067
4068	if (!isSGPR(SAddr))
4069	return std::nullopt;
4070
4071	return {{
4072	[=](MachineInstrBuilder &MIB) { MIB.addReg(SAddr); }, // saddr
4073	[=](MachineInstrBuilder &MIB) { MIB.addImm(ImmOffset); } // offset
4074	}};
4075	}
4076
4077	// Check whether the flat scratch SVS swizzle bug affects this access.
4078	bool AMDGPUInstructionSelector::checkFlatScratchSVSSwizzleBug(
4079	Register VAddr, Register SAddr, uint64_t ImmOffset) const {
4080	if (!Subtarget->hasFlatScratchSVSSwizzleBug())
4081	return false;
4082
4083	// The bug affects the swizzling of SVS accesses if there is any carry out
4084	// from the two low order bits (i.e. from bit 1 into bit 2) when adding
4085	// voffset to (soffset + inst_offset).
4086	auto VKnown = KnownBits->getKnownBits(VAddr);
4087	auto SKnown = KnownBits::computeForAddSub(
4088	true, false, KnownBits->getKnownBits(SAddr),
4089	KnownBits::makeConstant(APInt(32, ImmOffset)));
4090	uint64_t VMax = VKnown.getMaxValue().getZExtValue();
4091	uint64_t SMax = SKnown.getMaxValue().getZExtValue();
4092	return (VMax & 3) + (SMax & 3) >= 4;
4093	}
4094
4095	InstructionSelector::ComplexRendererFns
4096	AMDGPUInstructionSelector::selectScratchSVAddr(MachineOperand &Root) const {
4097	Register Addr = Root.getReg();
4098	Register PtrBase;
4099	int64_t ConstOffset;
4100	int64_t ImmOffset = 0;
4101
4102	// Match the immediate offset first, which canonically is moved as low as
4103	// possible.
4104	std::tie(PtrBase, ConstOffset) = getPtrBaseWithConstantOffset(Addr, *MRI);
4105
4106	if (ConstOffset != 0 &&
4107	TII.isLegalFLATOffset(ConstOffset, AMDGPUAS::PRIVATE_ADDRESS, true)) {
4108	Addr = PtrBase;
4109	ImmOffset = ConstOffset;
4110	}
4111
4112	auto AddrDef = getDefSrcRegIgnoringCopies(Addr, *MRI);
4113	if (AddrDef->MI->getOpcode() != AMDGPU::G_PTR_ADD)
4114	return std::nullopt;
4115
4116	Register RHS = AddrDef->MI->getOperand(2).getReg();
4117	if (RBI.getRegBank(RHS, *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID)
4118	return std::nullopt;
4119
4120	Register LHS = AddrDef->MI->getOperand(1).getReg();
4121	auto LHSDef = getDefSrcRegIgnoringCopies(LHS, *MRI);
4122
4123	if (!isFlatScratchBaseLegal(LHS) \|\| !isFlatScratchBaseLegal(RHS))
4124	return std::nullopt;
4125
4126	if (checkFlatScratchSVSSwizzleBug(RHS, LHS, ImmOffset))
4127	return std::nullopt;
4128
4129	if (LHSDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX) {
4130	int FI = LHSDef->MI->getOperand(1).getIndex();
4131	return {{
4132	[=](MachineInstrBuilder &MIB) { MIB.addReg(RHS); }, // vaddr
4133	[=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(FI); }, // saddr
4134	[=](MachineInstrBuilder &MIB) { MIB.addImm(ImmOffset); } // offset
4135	}};
4136	}
4137
4138	if (!isSGPR(LHS))
4139	return std::nullopt;
4140
4141	return {{
4142	[=](MachineInstrBuilder &MIB) { MIB.addReg(RHS); }, // vaddr
4143	[=](MachineInstrBuilder &MIB) { MIB.addReg(LHS); }, // saddr
4144	[=](MachineInstrBuilder &MIB) { MIB.addImm(ImmOffset); } // offset
4145	}};
4146	}
4147
4148	InstructionSelector::ComplexRendererFns
4149	AMDGPUInstructionSelector::selectMUBUFScratchOffen(MachineOperand &Root) const {
4150	MachineInstr *MI = Root.getParent();
4151	MachineBasicBlock *MBB = MI->getParent();
4152	MachineFunction *MF = MBB->getParent();
4153	const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
4154
4155	int64_t Offset = 0;
4156	if (mi_match(Root.getReg(), *MRI, m_ICst(Offset)) &&
4157	Offset != TM.getNullPointerValue(AMDGPUAS::PRIVATE_ADDRESS)) {
4158	Register HighBits = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4159
4160	// TODO: Should this be inside the render function? The iterator seems to
4161	// move.
4162	const uint32_t MaxOffset = SIInstrInfo::getMaxMUBUFImmOffset();
4163	BuildMI(*MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::V_MOV_B32_e32),
4164	HighBits)
4165	.addImm(Offset & ~MaxOffset);
4166
4167	return {{[=](MachineInstrBuilder &MIB) { // rsrc
4168	MIB.addReg(Info->getScratchRSrcReg());
4169	},
4170	[=](MachineInstrBuilder &MIB) { // vaddr
4171	MIB.addReg(HighBits);
4172	},
4173	[=](MachineInstrBuilder &MIB) { // soffset
4174	// Use constant zero for soffset and rely on eliminateFrameIndex
4175	// to choose the appropriate frame register if need be.
4176	MIB.addImm(0);
4177	},
4178	[=](MachineInstrBuilder &MIB) { // offset
4179	MIB.addImm(Offset & MaxOffset);
4180	}}};
4181	}
4182
4183	assert(Offset == 0 \|\| Offset == -1)(static_cast <bool> (Offset == 0 \|\| Offset == -1) ? void (0) : __assert_fail ("Offset == 0 \|\| Offset == -1", "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp" , 4183, __extension__ __PRETTY_FUNCTION__));
4184
4185	// Try to fold a frame index directly into the MUBUF vaddr field, and any
4186	// offsets.
4187	std::optional<int> FI;
4188	Register VAddr = Root.getReg();
4189	if (const MachineInstr *RootDef = MRI->getVRegDef(Root.getReg())) {
4190	Register PtrBase;
4191	int64_t ConstOffset;
4192	std::tie(PtrBase, ConstOffset) = getPtrBaseWithConstantOffset(VAddr, *MRI);
4193	if (ConstOffset != 0) {
4194	if (SIInstrInfo::isLegalMUBUFImmOffset(ConstOffset) &&
4195	(!STI.privateMemoryResourceIsRangeChecked() \|\|
4196	KnownBits->signBitIsZero(PtrBase))) {
4197	const MachineInstr *PtrBaseDef = MRI->getVRegDef(PtrBase);
4198	if (PtrBaseDef->getOpcode() == AMDGPU::G_FRAME_INDEX)
4199	FI = PtrBaseDef->getOperand(1).getIndex();
4200	else
4201	VAddr = PtrBase;
4202	Offset = ConstOffset;
4203	}
4204	} else if (RootDef->getOpcode() == AMDGPU::G_FRAME_INDEX) {
4205	FI = RootDef->getOperand(1).getIndex();
4206	}
4207	}
4208
4209	return {{[=](MachineInstrBuilder &MIB) { // rsrc
4210	MIB.addReg(Info->getScratchRSrcReg());
4211	},
4212	[=](MachineInstrBuilder &MIB) { // vaddr
4213	if (FI)
4214	MIB.addFrameIndex(*FI);
4215	else
4216	MIB.addReg(VAddr);
4217	},
4218	[=](MachineInstrBuilder &MIB) { // soffset
4219	// Use constant zero for soffset and rely on eliminateFrameIndex
4220	// to choose the appropriate frame register if need be.
4221	MIB.addImm(0);
4222	},
4223	[=](MachineInstrBuilder &MIB) { // offset
4224	MIB.addImm(Offset);
4225	}}};
4226	}
4227
4228	bool AMDGPUInstructionSelector::isDSOffsetLegal(Register Base,
4229	int64_t Offset) const {
4230	if (!isUInt<16>(Offset))
4231	return false;
4232
4233	if (STI.hasUsableDSOffset() \|\| STI.unsafeDSOffsetFoldingEnabled())
4234	return true;
4235
4236	// On Southern Islands instruction with a negative base value and an offset
4237	// don't seem to work.
4238	return KnownBits->signBitIsZero(Base);
4239	}
4240
4241	bool AMDGPUInstructionSelector::isDSOffset2Legal(Register Base, int64_t Offset0,
4242	int64_t Offset1,
4243	unsigned Size) const {
4244	if (Offset0 % Size != 0 \|\| Offset1 % Size != 0)
4245	return false;
4246	if (!isUInt<8>(Offset0 / Size) \|\| !isUInt<8>(Offset1 / Size))
4247	return false;
4248
4249	if (STI.hasUsableDSOffset() \|\| STI.unsafeDSOffsetFoldingEnabled())
4250	return true;
4251
4252	// On Southern Islands instruction with a negative base value and an offset
4253	// don't seem to work.
4254	return KnownBits->signBitIsZero(Base);
4255	}
4256
4257	bool AMDGPUInstructionSelector::isFlatScratchBaseLegal(
4258	Register Base, uint64_t FlatVariant) const {
4259	if (FlatVariant != SIInstrFlags::FlatScratch)
4260	return true;
4261
4262	// When value in 32-bit Base can be negative calculate scratch offset using
4263	// 32-bit add instruction, otherwise use Base(unsigned) + offset.
4264	return KnownBits->signBitIsZero(Base);
4265	}
4266
4267	bool AMDGPUInstructionSelector::isUnneededShiftMask(const MachineInstr &MI,
4268	unsigned ShAmtBits) const {
4269	assert(MI.getOpcode() == TargetOpcode::G_AND)(static_cast <bool> (MI.getOpcode() == TargetOpcode::G_AND ) ? void (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_AND" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 4269 , __extension__ __PRETTY_FUNCTION__));
4270
4271	std::optional<APInt> RHS =
4272	getIConstantVRegVal(MI.getOperand(2).getReg(), *MRI);
4273	if (!RHS)
4274	return false;
4275
4276	if (RHS->countr_one() >= ShAmtBits)
4277	return true;
4278
4279	const APInt &LHSKnownZeros =
4280	KnownBits->getKnownZeroes(MI.getOperand(1).getReg());
4281	return (LHSKnownZeros \| *RHS).countr_one() >= ShAmtBits;
4282	}
4283
4284	// Return the wave level SGPR base address if this is a wave address.
4285	static Register getWaveAddress(const MachineInstr *Def) {
4286	return Def->getOpcode() == AMDGPU::G_AMDGPU_WAVE_ADDRESS
4287	? Def->getOperand(1).getReg()
4288	: Register();
4289	}
4290
4291	InstructionSelector::ComplexRendererFns
4292	AMDGPUInstructionSelector::selectMUBUFScratchOffset(
4293	MachineOperand &Root) const {
4294	Register Reg = Root.getReg();
4295	const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
4296
4297	const MachineInstr *Def = MRI->getVRegDef(Reg);
4298	if (Register WaveBase = getWaveAddress(Def)) {
4299	return {{
4300	[=](MachineInstrBuilder &MIB) { // rsrc
4301	MIB.addReg(Info->getScratchRSrcReg());
4302	},
4303	[=](MachineInstrBuilder &MIB) { // soffset
4304	MIB.addReg(WaveBase);
4305	},
4306	[=](MachineInstrBuilder &MIB) { MIB.addImm(0); } // offset
4307	}};
4308	}
4309
4310	int64_t Offset = 0;
4311
4312	// FIXME: Copy check is a hack
4313	Register BasePtr;
4314	if (mi_match(Reg, *MRI, m_GPtrAdd(m_Reg(BasePtr), m_Copy(m_ICst(Offset))))) {
4315	if (!SIInstrInfo::isLegalMUBUFImmOffset(Offset))
4316	return {};
4317	const MachineInstr *BasePtrDef = MRI->getVRegDef(BasePtr);
4318	Register WaveBase = getWaveAddress(BasePtrDef);
4319	if (!WaveBase)
4320	return {};
4321
4322	return {{
4323	[=](MachineInstrBuilder &MIB) { // rsrc
4324	MIB.addReg(Info->getScratchRSrcReg());
4325	},
4326	[=](MachineInstrBuilder &MIB) { // soffset
4327	MIB.addReg(WaveBase);
4328	},
4329	[=](MachineInstrBuilder &MIB) { MIB.addImm(Offset); } // offset
4330	}};
4331	}
4332
4333	if (!mi_match(Root.getReg(), *MRI, m_ICst(Offset)) \|\|
4334	!SIInstrInfo::isLegalMUBUFImmOffset(Offset))
4335	return {};
4336
4337	return {{
4338	[=](MachineInstrBuilder &MIB) { // rsrc
4339	MIB.addReg(Info->getScratchRSrcReg());
4340	},
4341	[=](MachineInstrBuilder &MIB) { // soffset
4342	MIB.addImm(0);
4343	},
4344	[=](MachineInstrBuilder &MIB) { MIB.addImm(Offset); } // offset
4345	}};
4346	}
4347
4348	std::pair<Register, unsigned>
4349	AMDGPUInstructionSelector::selectDS1Addr1OffsetImpl(MachineOperand &Root) const {
4350	const MachineInstr *RootDef = MRI->getVRegDef(Root.getReg());
4351	if (!RootDef)
4352	return std::pair(Root.getReg(), 0);
4353
4354	int64_t ConstAddr = 0;
4355
4356	Register PtrBase;
4357	int64_t Offset;
4358	std::tie(PtrBase, Offset) =
4359	getPtrBaseWithConstantOffset(Root.getReg(), *MRI);
4360
4361	if (Offset) {
4362	if (isDSOffsetLegal(PtrBase, Offset)) {
4363	// (add n0, c0)
4364	return std::pair(PtrBase, Offset);
4365	}
4366	} else if (RootDef->getOpcode() == AMDGPU::G_SUB) {
4367	// TODO
4368
4369
4370	} else if (mi_match(Root.getReg(), *MRI, m_ICst(ConstAddr))) {
4371	// TODO
4372
4373	}
4374
4375	return std::pair(Root.getReg(), 0);
4376	}
4377
4378	InstructionSelector::ComplexRendererFns
4379	AMDGPUInstructionSelector::selectDS1Addr1Offset(MachineOperand &Root) const {
4380	Register Reg;
4381	unsigned Offset;
4382	std::tie(Reg, Offset) = selectDS1Addr1OffsetImpl(Root);
4383	return {{
4384	[=](MachineInstrBuilder &MIB) { MIB.addReg(Reg); },
4385	[=](MachineInstrBuilder &MIB) { MIB.addImm(Offset); }
4386	}};
4387	}
4388
4389	InstructionSelector::ComplexRendererFns
4390	AMDGPUInstructionSelector::selectDS64Bit4ByteAligned(MachineOperand &Root) const {
4391	return selectDSReadWrite2(Root, 4);
4392	}
4393
4394	InstructionSelector::ComplexRendererFns
4395	AMDGPUInstructionSelector::selectDS128Bit8ByteAligned(MachineOperand &Root) const {
4396	return selectDSReadWrite2(Root, 8);
4397	}
4398
4399	InstructionSelector::ComplexRendererFns
4400	AMDGPUInstructionSelector::selectDSReadWrite2(MachineOperand &Root,
4401	unsigned Size) const {
4402	Register Reg;
4403	unsigned Offset;
4404	std::tie(Reg, Offset) = selectDSReadWrite2Impl(Root, Size);
4405	return {{
4406	[=](MachineInstrBuilder &MIB) { MIB.addReg(Reg); },
4407	[=](MachineInstrBuilder &MIB) { MIB.addImm(Offset); },
4408	[=](MachineInstrBuilder &MIB) { MIB.addImm(Offset+1); }
4409	}};
4410	}
4411
4412	std::pair<Register, unsigned>
4413	AMDGPUInstructionSelector::selectDSReadWrite2Impl(MachineOperand &Root,
4414	unsigned Size) const {
4415	const MachineInstr *RootDef = MRI->getVRegDef(Root.getReg());
4416	if (!RootDef)
4417	return std::pair(Root.getReg(), 0);
4418
4419	int64_t ConstAddr = 0;
4420
4421	Register PtrBase;
4422	int64_t Offset;
4423	std::tie(PtrBase, Offset) =
4424	getPtrBaseWithConstantOffset(Root.getReg(), *MRI);
4425
4426	if (Offset) {
4427	int64_t OffsetValue0 = Offset;
4428	int64_t OffsetValue1 = Offset + Size;
4429	if (isDSOffset2Legal(PtrBase, OffsetValue0, OffsetValue1, Size)) {
4430	// (add n0, c0)
4431	return std::pair(PtrBase, OffsetValue0 / Size);
4432	}
4433	} else if (RootDef->getOpcode() == AMDGPU::G_SUB) {
4434	// TODO
4435
4436	} else if (mi_match(Root.getReg(), *MRI, m_ICst(ConstAddr))) {
4437	// TODO
4438
4439	}
4440
4441	return std::pair(Root.getReg(), 0);
4442	}
4443
4444	/// If \p Root is a G_PTR_ADD with a G_CONSTANT on the right hand side, return
4445	/// the base value with the constant offset. There may be intervening copies
4446	/// between \p Root and the identified constant. Returns \p Root, 0 if this does
4447	/// not match the pattern.
4448	std::pair<Register, int64_t>
4449	AMDGPUInstructionSelector::getPtrBaseWithConstantOffset(
4450	Register Root, const MachineRegisterInfo &MRI) const {
4451	MachineInstr *RootI = getDefIgnoringCopies(Root, MRI);
4452	if (RootI->getOpcode() != TargetOpcode::G_PTR_ADD)
4453	return {Root, 0};
4454
4455	MachineOperand &RHS = RootI->getOperand(2);
4456	std::optional<ValueAndVReg> MaybeOffset =
4457	getIConstantVRegValWithLookThrough(RHS.getReg(), MRI);
4458	if (!MaybeOffset)
4459	return {Root, 0};
4460	return {RootI->getOperand(1).getReg(), MaybeOffset->Value.getSExtValue()};
4461	}
4462
4463	static void addZeroImm(MachineInstrBuilder &MIB) {
4464	MIB.addImm(0);
4465	}
4466
4467	/// Return a resource descriptor for use with an arbitrary 64-bit pointer. If \p
4468	/// BasePtr is not valid, a null base pointer will be used.
4469	static Register buildRSRC(MachineIRBuilder &B, MachineRegisterInfo &MRI,
4470	uint32_t FormatLo, uint32_t FormatHi,
4471	Register BasePtr) {
4472	Register RSrc2 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
4473	Register RSrc3 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
4474	Register RSrcHi = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
4475	Register RSrc = MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass);
4476
4477	B.buildInstr(AMDGPU::S_MOV_B32)
4478	.addDef(RSrc2)
4479	.addImm(FormatLo);
4480	B.buildInstr(AMDGPU::S_MOV_B32)
4481	.addDef(RSrc3)
4482	.addImm(FormatHi);
4483
4484	// Build the half of the subregister with the constants before building the
4485	// full 128-bit register. If we are building multiple resource descriptors,
4486	// this will allow CSEing of the 2-component register.
4487	B.buildInstr(AMDGPU::REG_SEQUENCE)
4488	.addDef(RSrcHi)
4489	.addReg(RSrc2)
4490	.addImm(AMDGPU::sub0)
4491	.addReg(RSrc3)
4492	.addImm(AMDGPU::sub1);
4493
4494	Register RSrcLo = BasePtr;
4495	if (!BasePtr) {
4496	RSrcLo = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
4497	B.buildInstr(AMDGPU::S_MOV_B64)
4498	.addDef(RSrcLo)
4499	.addImm(0);
4500	}
4501
4502	B.buildInstr(AMDGPU::REG_SEQUENCE)
4503	.addDef(RSrc)
4504	.addReg(RSrcLo)
4505	.addImm(AMDGPU::sub0_sub1)
4506	.addReg(RSrcHi)
4507	.addImm(AMDGPU::sub2_sub3);
4508
4509	return RSrc;
4510	}
4511
4512	static Register buildAddr64RSrc(MachineIRBuilder &B, MachineRegisterInfo &MRI,
4513	const SIInstrInfo &TII, Register BasePtr) {
4514	uint64_t DefaultFormat = TII.getDefaultRsrcDataFormat();
4515
4516	// FIXME: Why are half the "default" bits ignored based on the addressing
4517	// mode?
4518	return buildRSRC(B, MRI, 0, Hi_32(DefaultFormat), BasePtr);
4519	}
4520
4521	static Register buildOffsetSrc(MachineIRBuilder &B, MachineRegisterInfo &MRI,
4522	const SIInstrInfo &TII, Register BasePtr) {
4523	uint64_t DefaultFormat = TII.getDefaultRsrcDataFormat();
4524
4525	// FIXME: Why are half the "default" bits ignored based on the addressing
4526	// mode?
4527	return buildRSRC(B, MRI, -1, Hi_32(DefaultFormat), BasePtr);
4528	}
4529
4530	AMDGPUInstructionSelector::MUBUFAddressData
4531	AMDGPUInstructionSelector::parseMUBUFAddress(Register Src) const {
4532	MUBUFAddressData Data;
4533	Data.N0 = Src;
4534
4535	Register PtrBase;
4536	int64_t Offset;
4537
4538	std::tie(PtrBase, Offset) = getPtrBaseWithConstantOffset(Src, *MRI);
4539	if (isUInt<32>(Offset)) {
4540	Data.N0 = PtrBase;
4541	Data.Offset = Offset;
4542	}
4543
4544	if (MachineInstr *InputAdd
4545	= getOpcodeDef(TargetOpcode::G_PTR_ADD, Data.N0, *MRI)) {
4546	Data.N2 = InputAdd->getOperand(1).getReg();
4547	Data.N3 = InputAdd->getOperand(2).getReg();
4548
4549	// FIXME: Need to fix extra SGPR->VGPRcopies inserted
4550	// FIXME: Don't know this was defined by operand 0
4551	//
4552	// TODO: Remove this when we have copy folding optimizations after
4553	// RegBankSelect.
4554	Data.N2 = getDefIgnoringCopies(Data.N2, *MRI)->getOperand(0).getReg();
4555	Data.N3 = getDefIgnoringCopies(Data.N3, *MRI)->getOperand(0).getReg();
4556	}
4557
4558	return Data;
4559	}
4560
4561	/// Return if the addr64 mubuf mode should be used for the given address.
4562	bool AMDGPUInstructionSelector::shouldUseAddr64(MUBUFAddressData Addr) const {
4563	// (ptr_add N2, N3) -> addr64, or
4564	// (ptr_add (ptr_add N2, N3), C1) -> addr64
4565	if (Addr.N2)
4566	return true;
4567
4568	const RegisterBank N0Bank = RBI.getRegBank(Addr.N0, MRI, TRI);
4569	return N0Bank->getID() == AMDGPU::VGPRRegBankID;
4570	}
4571
4572	/// Split an immediate offset \p ImmOffset depending on whether it fits in the
4573	/// immediate field. Modifies \p ImmOffset and sets \p SOffset to the variable
4574	/// component.
4575	void AMDGPUInstructionSelector::splitIllegalMUBUFOffset(
4576	MachineIRBuilder &B, Register &SOffset, int64_t &ImmOffset) const {
4577	if (SIInstrInfo::isLegalMUBUFImmOffset(ImmOffset))
4578	return;
4579
4580	// Illegal offset, store it in soffset.
4581	SOffset = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
4582	B.buildInstr(AMDGPU::S_MOV_B32)
4583	.addDef(SOffset)
4584	.addImm(ImmOffset);
4585	ImmOffset = 0;
4586	}
4587
4588	bool AMDGPUInstructionSelector::selectMUBUFAddr64Impl(
4589	MachineOperand &Root, Register &VAddr, Register &RSrcReg,
4590	Register &SOffset, int64_t &Offset) const {
4591	// FIXME: Predicates should stop this from reaching here.
4592	// addr64 bit was removed for volcanic islands.
4593	if (!STI.hasAddr64() \|\| STI.useFlatForGlobal())
4594	return false;
4595
4596	MUBUFAddressData AddrData = parseMUBUFAddress(Root.getReg());
4597	if (!shouldUseAddr64(AddrData))
4598	return false;
4599
4600	Register N0 = AddrData.N0;
4601	Register N2 = AddrData.N2;
4602	Register N3 = AddrData.N3;
4603	Offset = AddrData.Offset;
4604
4605	// Base pointer for the SRD.
4606	Register SRDPtr;
4607
4608	if (N2) {
4609	if (RBI.getRegBank(N2, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) {
4610	assert(N3)(static_cast <bool> (N3) ? void (0) : __assert_fail ("N3" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 4610 , __extension__ __PRETTY_FUNCTION__));
4611	if (RBI.getRegBank(N3, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) {
4612	// Both N2 and N3 are divergent. Use N0 (the result of the add) as the
4613	// addr64, and construct the default resource from a 0 address.
4614	VAddr = N0;
4615	} else {
4616	SRDPtr = N3;
4617	VAddr = N2;
4618	}
4619	} else {
4620	// N2 is not divergent.
4621	SRDPtr = N2;
4622	VAddr = N3;
4623	}
4624	} else if (RBI.getRegBank(N0, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) {
4625	// Use the default null pointer in the resource
4626	VAddr = N0;
4627	} else {
4628	// N0 -> offset, or
4629	// (N0 + C1) -> offset
4630	SRDPtr = N0;
4631	}
4632
4633	MachineIRBuilder B(*Root.getParent());
4634	RSrcReg = buildAddr64RSrc(B, *MRI, TII, SRDPtr);
4635	splitIllegalMUBUFOffset(B, SOffset, Offset);
4636	return true;
4637	}
4638
4639	bool AMDGPUInstructionSelector::selectMUBUFOffsetImpl(
4640	MachineOperand &Root, Register &RSrcReg, Register &SOffset,
4641	int64_t &Offset) const {
4642
4643	// FIXME: Pattern should not reach here.
4644	if (STI.useFlatForGlobal())
4645	return false;
4646
4647	MUBUFAddressData AddrData = parseMUBUFAddress(Root.getReg());
4648	if (shouldUseAddr64(AddrData))
4649	return false;
4650
4651	// N0 -> offset, or
4652	// (N0 + C1) -> offset
4653	Register SRDPtr = AddrData.N0;
4654	Offset = AddrData.Offset;
4655
4656	// TODO: Look through extensions for 32-bit soffset.
4657	MachineIRBuilder B(*Root.getParent());
4658
4659	RSrcReg = buildOffsetSrc(B, *MRI, TII, SRDPtr);
4660	splitIllegalMUBUFOffset(B, SOffset, Offset);
4661	return true;
4662	}
4663
4664	InstructionSelector::ComplexRendererFns
4665	AMDGPUInstructionSelector::selectMUBUFAddr64(MachineOperand &Root) const {
4666	Register VAddr;
4667	Register RSrcReg;
4668	Register SOffset;
4669	int64_t Offset = 0;
4670
4671	if (!selectMUBUFAddr64Impl(Root, VAddr, RSrcReg, SOffset, Offset))
4672	return {};
4673
4674	// FIXME: Use defaulted operands for trailing 0s and remove from the complex
4675	// pattern.
4676	return {{
4677	[=](MachineInstrBuilder &MIB) { // rsrc
4678	MIB.addReg(RSrcReg);
4679	},
4680	[=](MachineInstrBuilder &MIB) { // vaddr
4681	MIB.addReg(VAddr);
4682	},
4683	[=](MachineInstrBuilder &MIB) { // soffset
4684	if (SOffset)
4685	MIB.addReg(SOffset);
4686	else
4687	MIB.addImm(0);
4688	},
4689	[=](MachineInstrBuilder &MIB) { // offset
4690	MIB.addImm(Offset);
4691	},
4692	addZeroImm, // cpol
4693	addZeroImm, // tfe
4694	addZeroImm // swz
4695	}};
4696	}
4697
4698	InstructionSelector::ComplexRendererFns
4699	AMDGPUInstructionSelector::selectMUBUFOffset(MachineOperand &Root) const {
4700	Register RSrcReg;
4701	Register SOffset;
4702	int64_t Offset = 0;
4703
4704	if (!selectMUBUFOffsetImpl(Root, RSrcReg, SOffset, Offset))
4705	return {};
4706
4707	return {{
4708	[=](MachineInstrBuilder &MIB) { // rsrc
4709	MIB.addReg(RSrcReg);
4710	},
4711	[=](MachineInstrBuilder &MIB) { // soffset
4712	if (SOffset)
4713	MIB.addReg(SOffset);
4714	else
4715	MIB.addImm(0);
4716	},
4717	[=](MachineInstrBuilder &MIB) { MIB.addImm(Offset); }, // offset
4718	addZeroImm, // cpol
4719	addZeroImm, // tfe
4720	addZeroImm, // swz
4721	}};
4722	}
4723
4724	/// Get an immediate that must be 32-bits, and treated as zero extended.
4725	static std::optional<uint64_t>
4726	getConstantZext32Val(Register Reg, const MachineRegisterInfo &MRI) {
4727	// getIConstantVRegVal sexts any values, so see if that matters.
4728	std::optional<int64_t> OffsetVal = getIConstantVRegSExtVal(Reg, MRI);
4729	if (!OffsetVal \|\| !isInt<32>(*OffsetVal))
4730	return std::nullopt;
4731	return Lo_32(*OffsetVal);
4732	}
4733
4734	InstructionSelector::ComplexRendererFns
4735	AMDGPUInstructionSelector::selectSMRDBufferImm(MachineOperand &Root) const {
4736	std::optional<uint64_t> OffsetVal = getConstantZext32Val(Root.getReg(), *MRI);
4737	if (!OffsetVal)
4738	return {};
4739
4740	std::optional<int64_t> EncodedImm =
4741	AMDGPU::getSMRDEncodedOffset(STI, *OffsetVal, true);
4742	if (!EncodedImm)
4743	return {};
4744
4745	return {{ [=](MachineInstrBuilder &MIB) { MIB.addImm(*EncodedImm); } }};
4746	}
4747
4748	InstructionSelector::ComplexRendererFns
4749	AMDGPUInstructionSelector::selectSMRDBufferImm32(MachineOperand &Root) const {
4750	assert(STI.getGeneration() == AMDGPUSubtarget::SEA_ISLANDS)(static_cast <bool> (STI.getGeneration() == AMDGPUSubtarget ::SEA_ISLANDS) ? void (0) : __assert_fail ("STI.getGeneration() == AMDGPUSubtarget::SEA_ISLANDS" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 4750 , __extension__ __PRETTY_FUNCTION__));
4751
4752	std::optional<uint64_t> OffsetVal = getConstantZext32Val(Root.getReg(), *MRI);
4753	if (!OffsetVal)
4754	return {};
4755
4756	std::optional<int64_t> EncodedImm =
4757	AMDGPU::getSMRDEncodedLiteralOffset32(STI, *OffsetVal);
4758	if (!EncodedImm)
4759	return {};
4760
4761	return {{ [=](MachineInstrBuilder &MIB) { MIB.addImm(*EncodedImm); } }};
4762	}
4763
4764	InstructionSelector::ComplexRendererFns
4765	AMDGPUInstructionSelector::selectSMRDBufferSgprImm(MachineOperand &Root) const {
4766	// Match the (soffset + offset) pair as a 32-bit register base and
4767	// an immediate offset.
4768	Register SOffset;
4769	unsigned Offset;
4770	std::tie(SOffset, Offset) =
4771	AMDGPU::getBaseWithConstantOffset(*MRI, Root.getReg(), KnownBits);
4772	if (!SOffset)
4773	return std::nullopt;
4774
4775	std::optional<int64_t> EncodedOffset =
4776	AMDGPU::getSMRDEncodedOffset(STI, Offset, /* IsBuffer */ true);
4777	if (!EncodedOffset)
4778	return std::nullopt;
4779
4780	assert(MRI->getType(SOffset) == LLT::scalar(32))(static_cast <bool> (MRI->getType(SOffset) == LLT::scalar (32)) ? void (0) : __assert_fail ("MRI->getType(SOffset) == LLT::scalar(32)" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 4780 , __extension__ __PRETTY_FUNCTION__));
4781	return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(SOffset); },
4782	[=](MachineInstrBuilder &MIB) { MIB.addImm(*EncodedOffset); }}};
4783	}
4784
4785	// Variant of stripBitCast that returns the instruction instead of a
4786	// MachineOperand.
4787	static MachineInstr stripBitCast(MachineInstr MI, MachineRegisterInfo &MRI) {
4788	if (MI->getOpcode() == AMDGPU::G_BITCAST)
4789	return getDefIgnoringCopies(MI->getOperand(1).getReg(), MRI);
4790	return MI;
4791	}
4792
4793	// Figure out if this is really an extract of the high 16-bits of a dword,
4794	// returns nullptr if it isn't.
4795	static MachineInstr isExtractHiElt(MachineInstr Inst,
4796	MachineRegisterInfo &MRI) {
4797	Inst = stripBitCast(Inst, MRI);
4798
4799	if (Inst->getOpcode() != AMDGPU::G_TRUNC)
4800	return nullptr;
4801
4802	MachineInstr *TruncOp =
4803	getDefIgnoringCopies(Inst->getOperand(1).getReg(), MRI);
4804	TruncOp = stripBitCast(TruncOp, MRI);
4805
4806	// G_LSHR x, (G_CONSTANT i32 16)
4807	if (TruncOp->getOpcode() == AMDGPU::G_LSHR) {
4808	auto SrlAmount = getIConstantVRegValWithLookThrough(
4809	TruncOp->getOperand(2).getReg(), MRI);
4810	if (SrlAmount && SrlAmount->Value.getZExtValue() == 16) {
4811	MachineInstr *SrlOp =
4812	getDefIgnoringCopies(TruncOp->getOperand(1).getReg(), MRI);
4813	return stripBitCast(SrlOp, MRI);
4814	}
4815	}
4816
4817	// G_SHUFFLE_VECTOR x, y, shufflemask(1, 1\|0)
4818	// 1, 0 swaps the low/high 16 bits.
4819	// 1, 1 sets the high 16 bits to be the same as the low 16.
4820	// in any case, it selects the high elts.
4821	if (TruncOp->getOpcode() == AMDGPU::G_SHUFFLE_VECTOR) {
4822	assert(MRI.getType(TruncOp->getOperand(0).getReg()) ==(static_cast <bool> (MRI.getType(TruncOp->getOperand (0).getReg()) == LLT::fixed_vector(2, 16)) ? void (0) : __assert_fail ("MRI.getType(TruncOp->getOperand(0).getReg()) == LLT::fixed_vector(2, 16)" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 4823 , __extension__ __PRETTY_FUNCTION__))
4823	LLT::fixed_vector(2, 16))(static_cast <bool> (MRI.getType(TruncOp->getOperand (0).getReg()) == LLT::fixed_vector(2, 16)) ? void (0) : __assert_fail ("MRI.getType(TruncOp->getOperand(0).getReg()) == LLT::fixed_vector(2, 16)" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 4823 , __extension__ __PRETTY_FUNCTION__));
4824
4825	ArrayRef<int> Mask = TruncOp->getOperand(3).getShuffleMask();
4826	assert(Mask.size() == 2)(static_cast <bool> (Mask.size() == 2) ? void (0) : __assert_fail ("Mask.size() == 2", "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp" , 4826, __extension__ __PRETTY_FUNCTION__));
4827
4828	if (Mask[0] == 1 && Mask[1] <= 1) {
4829	MachineInstr *LHS =
4830	getDefIgnoringCopies(TruncOp->getOperand(1).getReg(), MRI);
4831	return stripBitCast(LHS, MRI);
4832	}
4833	}
4834
4835	return nullptr;
4836	}
4837
4838	std::pair<Register, unsigned>
4839	AMDGPUInstructionSelector::selectVOP3PMadMixModsImpl(MachineOperand &Root,
4840	bool &Matched) const {
4841	Matched = false;
4842
4843	Register Src;
4844	unsigned Mods;
4845	std::tie(Src, Mods) = selectVOP3ModsImpl(Root);
4846
4847	MachineInstr MI = getDefIgnoringCopies(Src, MRI);
4848	if (MI->getOpcode() == AMDGPU::G_FPEXT) {
4849	MachineOperand *MO = &MI->getOperand(1);
4850	Src = MO->getReg();
4851	MI = getDefIgnoringCopies(Src, *MRI);
4852
4853	assert(MRI->getType(Src) == LLT::scalar(16))(static_cast <bool> (MRI->getType(Src) == LLT::scalar (16)) ? void (0) : __assert_fail ("MRI->getType(Src) == LLT::scalar(16)" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 4853 , __extension__ __PRETTY_FUNCTION__));
4854
4855	// See through bitcasts.
4856	// FIXME: Would be nice to use stripBitCast here.
4857	if (MI->getOpcode() == AMDGPU::G_BITCAST) {
4858	MO = &MI->getOperand(1);
4859	Src = MO->getReg();
4860	MI = getDefIgnoringCopies(Src, *MRI);
4861	}
4862
4863	const auto CheckAbsNeg = [&]() {
4864	// Be careful about folding modifiers if we already have an abs. fneg is
4865	// applied last, so we don't want to apply an earlier fneg.
4866	if ((Mods & SISrcMods::ABS) == 0) {
4867	unsigned ModsTmp;
4868	std::tie(Src, ModsTmp) = selectVOP3ModsImpl(*MO);
4869	MI = getDefIgnoringCopies(Src, *MRI);
4870
4871	if ((ModsTmp & SISrcMods::NEG) != 0)
4872	Mods ^= SISrcMods::NEG;
4873
4874	if ((ModsTmp & SISrcMods::ABS) != 0)
4875	Mods \|= SISrcMods::ABS;
4876	}
4877	};
4878
4879	CheckAbsNeg();
4880
4881	// op_sel/op_sel_hi decide the source type and source.
4882	// If the source's op_sel_hi is set, it indicates to do a conversion from
4883	// fp16. If the sources's op_sel is set, it picks the high half of the
4884	// source register.
4885
4886	Mods \|= SISrcMods::OP_SEL_1;
4887
4888	if (MachineInstr ExtractHiEltMI = isExtractHiElt(MI, MRI)) {
4889	Mods \|= SISrcMods::OP_SEL_0;
4890	MI = ExtractHiEltMI;
4891	MO = &MI->getOperand(0);
4892	Src = MO->getReg();
4893
4894	CheckAbsNeg();
4895	}
4896
4897	Matched = true;
4898	}
4899
4900	return {Src, Mods};
4901	}
4902
4903	InstructionSelector::ComplexRendererFns
4904	AMDGPUInstructionSelector::selectVOP3PMadMixModsExt(
4905	MachineOperand &Root) const {
4906	Register Src;
4907	unsigned Mods;
4908	bool Matched;
4909	std::tie(Src, Mods) = selectVOP3PMadMixModsImpl(Root, Matched);
4910	if (!Matched)
4911	return {};
4912
4913	return {{
4914	[=](MachineInstrBuilder &MIB) { MIB.addReg(Src); },
4915	[=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); } // src_mods
4916	}};
4917	}
4918
4919	InstructionSelector::ComplexRendererFns
4920	AMDGPUInstructionSelector::selectVOP3PMadMixMods(MachineOperand &Root) const {
4921	Register Src;
4922	unsigned Mods;
4923	bool Matched;
4924	std::tie(Src, Mods) = selectVOP3PMadMixModsImpl(Root, Matched);
4925
4926	return {{
4927	[=](MachineInstrBuilder &MIB) { MIB.addReg(Src); },
4928	[=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); } // src_mods
4929	}};
4930	}
4931
4932	void AMDGPUInstructionSelector::renderTruncImm32(MachineInstrBuilder &MIB,
4933	const MachineInstr &MI,
4934	int OpIdx) const {
4935	assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&(static_cast <bool> (MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && "Expected G_CONSTANT") ? void (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && \"Expected G_CONSTANT\"" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 4936 , __extension__ __PRETTY_FUNCTION__))
4936	"Expected G_CONSTANT")(static_cast <bool> (MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && "Expected G_CONSTANT") ? void (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && \"Expected G_CONSTANT\"" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 4936 , __extension__ __PRETTY_FUNCTION__));
4937	MIB.addImm(MI.getOperand(1).getCImm()->getSExtValue());
4938	}
4939
4940	void AMDGPUInstructionSelector::renderNegateImm(MachineInstrBuilder &MIB,
4941	const MachineInstr &MI,
4942	int OpIdx) const {
4943	assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&(static_cast <bool> (MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && "Expected G_CONSTANT") ? void (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && \"Expected G_CONSTANT\"" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 4944 , __extension__ __PRETTY_FUNCTION__))
4944	"Expected G_CONSTANT")(static_cast <bool> (MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && "Expected G_CONSTANT") ? void (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && \"Expected G_CONSTANT\"" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 4944 , __extension__ __PRETTY_FUNCTION__));
4945	MIB.addImm(-MI.getOperand(1).getCImm()->getSExtValue());
4946	}
4947
4948	void AMDGPUInstructionSelector::renderBitcastImm(MachineInstrBuilder &MIB,
4949	const MachineInstr &MI,
4950	int OpIdx) const {
4951	assert(OpIdx == -1)(static_cast <bool> (OpIdx == -1) ? void (0) : __assert_fail ("OpIdx == -1", "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp" , 4951, __extension__ __PRETTY_FUNCTION__));
4952
4953	const MachineOperand &Op = MI.getOperand(1);
4954	if (MI.getOpcode() == TargetOpcode::G_FCONSTANT)
4955	MIB.addImm(Op.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue());
4956	else {
4957	assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT")(static_cast <bool> (MI.getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT") ? void (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_CONSTANT && \"Expected G_CONSTANT\"" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 4957 , __extension__ __PRETTY_FUNCTION__));
4958	MIB.addImm(Op.getCImm()->getSExtValue());
4959	}
4960	}
4961
4962	void AMDGPUInstructionSelector::renderPopcntImm(MachineInstrBuilder &MIB,
4963	const MachineInstr &MI,
4964	int OpIdx) const {
4965	assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&(static_cast <bool> (MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && "Expected G_CONSTANT") ? void (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && \"Expected G_CONSTANT\"" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 4966 , __extension__ __PRETTY_FUNCTION__))
4966	"Expected G_CONSTANT")(static_cast <bool> (MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && "Expected G_CONSTANT") ? void (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && \"Expected G_CONSTANT\"" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 4966 , __extension__ __PRETTY_FUNCTION__));
4967	MIB.addImm(MI.getOperand(1).getCImm()->getValue().popcount());
4968	}
4969
4970	/// This only really exists to satisfy DAG type checking machinery, so is a
4971	/// no-op here.
4972	void AMDGPUInstructionSelector::renderTruncTImm(MachineInstrBuilder &MIB,
4973	const MachineInstr &MI,
4974	int OpIdx) const {
4975	MIB.addImm(MI.getOperand(OpIdx).getImm());
4976	}
4977
4978	void AMDGPUInstructionSelector::renderOpSelTImm(MachineInstrBuilder &MIB,
4979	const MachineInstr &MI,
4980	int OpIdx) const {
4981	assert(OpIdx >= 0 && "expected to match an immediate operand")(static_cast <bool> (OpIdx >= 0 && "expected to match an immediate operand" ) ? void (0) : __assert_fail ("OpIdx >= 0 && \"expected to match an immediate operand\"" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 4981 , __extension__ __PRETTY_FUNCTION__));
4982	MIB.addImm(MI.getOperand(OpIdx).getImm() ? SISrcMods::OP_SEL_0 : 0);
4983	}
4984
4985	void AMDGPUInstructionSelector::renderExtractCPol(MachineInstrBuilder &MIB,
4986	const MachineInstr &MI,
4987	int OpIdx) const {
4988	assert(OpIdx >= 0 && "expected to match an immediate operand")(static_cast <bool> (OpIdx >= 0 && "expected to match an immediate operand" ) ? void (0) : __assert_fail ("OpIdx >= 0 && \"expected to match an immediate operand\"" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 4988 , __extension__ __PRETTY_FUNCTION__));
4989	MIB.addImm(MI.getOperand(OpIdx).getImm() & AMDGPU::CPol::ALL);
4990	}
4991
4992	void AMDGPUInstructionSelector::renderExtractSWZ(MachineInstrBuilder &MIB,
4993	const MachineInstr &MI,
4994	int OpIdx) const {
4995	assert(OpIdx >= 0 && "expected to match an immediate operand")(static_cast <bool> (OpIdx >= 0 && "expected to match an immediate operand" ) ? void (0) : __assert_fail ("OpIdx >= 0 && \"expected to match an immediate operand\"" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 4995 , __extension__ __PRETTY_FUNCTION__));
4996	MIB.addImm((MI.getOperand(OpIdx).getImm() >> 3) & 1);
4997	}
4998
4999	void AMDGPUInstructionSelector::renderSetGLC(MachineInstrBuilder &MIB,
5000	const MachineInstr &MI,
5001	int OpIdx) const {
5002	assert(OpIdx >= 0 && "expected to match an immediate operand")(static_cast <bool> (OpIdx >= 0 && "expected to match an immediate operand" ) ? void (0) : __assert_fail ("OpIdx >= 0 && \"expected to match an immediate operand\"" , "llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp", 5002 , __extension__ __PRETTY_FUNCTION__));
5003	MIB.addImm(MI.getOperand(OpIdx).getImm() \| AMDGPU::CPol::GLC);
5004	}
5005
5006	void AMDGPUInstructionSelector::renderFrameIndex(MachineInstrBuilder &MIB,
5007	const MachineInstr &MI,
5008	int OpIdx) const {
5009	MIB.addFrameIndex((MI.getOperand(1).getIndex()));
5010	}
5011
5012	bool AMDGPUInstructionSelector::isInlineImmediate16(int64_t Imm) const {
5013	return AMDGPU::isInlinableLiteral16(Imm, STI.hasInv2PiInlineImm());
5014	}
5015
5016	bool AMDGPUInstructionSelector::isInlineImmediate32(int64_t Imm) const {
5017	return AMDGPU::isInlinableLiteral32(Imm, STI.hasInv2PiInlineImm());
5018	}
5019
5020	bool AMDGPUInstructionSelector::isInlineImmediate64(int64_t Imm) const {
5021	return AMDGPU::isInlinableLiteral64(Imm, STI.hasInv2PiInlineImm());
5022	}
5023
5024	bool AMDGPUInstructionSelector::isInlineImmediate(const APFloat &Imm) const {
5025	return TII.isInlineConstant(Imm);
5026	}