/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp

Bug Summary

File:	llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
Warning:	line 4160, column 7 Value stored to 'NumLeftover' is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

Show analyzer invocation

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name LegalizerHelper.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/build-llvm/lib/CodeGen/GlobalISel -resource-dir /usr/lib/llvm-14/lib/clang/14.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/build-llvm/lib/CodeGen/GlobalISel -I /build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel -I /build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/build-llvm/include -I /build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-14/lib/clang/14.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/build-llvm/lib/CodeGen/GlobalISel -fdebug-prefix-map=/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0=. -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-08-28-193554-24367-1 -x c++ /build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp

1	//===-- llvm/CodeGen/GlobalISel/LegalizerHelper.cpp -----------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	/// \file This file implements the LegalizerHelper class to legalize
10	/// individual instructions and the LegalizeMachineIR wrapper pass for the
11	/// primary legalization.
12	//
13	//===----------------------------------------------------------------------===//
14
15	#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
16	#include "llvm/CodeGen/GlobalISel/CallLowering.h"
17	#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
18	#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
19	#include "llvm/CodeGen/GlobalISel/LostDebugLocObserver.h"
20	#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
21	#include "llvm/CodeGen/GlobalISel/Utils.h"
22	#include "llvm/CodeGen/LowLevelType.h"
23	#include "llvm/CodeGen/MachineRegisterInfo.h"
24	#include "llvm/CodeGen/TargetFrameLowering.h"
25	#include "llvm/CodeGen/TargetInstrInfo.h"
26	#include "llvm/CodeGen/TargetLowering.h"
27	#include "llvm/CodeGen/TargetOpcodes.h"
28	#include "llvm/CodeGen/TargetSubtargetInfo.h"
29	#include "llvm/IR/Instructions.h"
30	#include "llvm/Support/Debug.h"
31	#include "llvm/Support/MathExtras.h"
32	#include "llvm/Support/raw_ostream.h"
33
34	#define DEBUG_TYPE"legalizer" "legalizer"
35
36	using namespace llvm;
37	using namespace LegalizeActions;
38	using namespace MIPatternMatch;
39
40	/// Try to break down \p OrigTy into \p NarrowTy sized pieces.
41	///
42	/// Returns the number of \p NarrowTy elements needed to reconstruct \p OrigTy,
43	/// with any leftover piece as type \p LeftoverTy
44	///
45	/// Returns -1 in the first element of the pair if the breakdown is not
46	/// satisfiable.
47	static std::pair<int, int>
48	getNarrowTypeBreakDown(LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy) {
49	assert(!LeftoverTy.isValid() && "this is an out argument")(static_cast <bool> (!LeftoverTy.isValid() && "this is an out argument" ) ? void (0) : __assert_fail ("!LeftoverTy.isValid() && \"this is an out argument\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 49, __extension__ __PRETTY_FUNCTION__));
50
51	unsigned Size = OrigTy.getSizeInBits();
52	unsigned NarrowSize = NarrowTy.getSizeInBits();
53	unsigned NumParts = Size / NarrowSize;
54	unsigned LeftoverSize = Size - NumParts * NarrowSize;
55	assert(Size > NarrowSize)(static_cast <bool> (Size > NarrowSize) ? void (0) : __assert_fail ("Size > NarrowSize", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 55, __extension__ __PRETTY_FUNCTION__));
56
57	if (LeftoverSize == 0)
58	return {NumParts, 0};
59
60	if (NarrowTy.isVector()) {
61	unsigned EltSize = OrigTy.getScalarSizeInBits();
62	if (LeftoverSize % EltSize != 0)
63	return {-1, -1};
64	LeftoverTy = LLT::scalarOrVector(
65	ElementCount::getFixed(LeftoverSize / EltSize), EltSize);
66	} else {
67	LeftoverTy = LLT::scalar(LeftoverSize);
68	}
69
70	int NumLeftover = LeftoverSize / LeftoverTy.getSizeInBits();
71	return std::make_pair(NumParts, NumLeftover);
72	}
73
74	static Type *getFloatTypeForLLT(LLVMContext &Ctx, LLT Ty) {
75
76	if (!Ty.isScalar())
77	return nullptr;
78
79	switch (Ty.getSizeInBits()) {
80	case 16:
81	return Type::getHalfTy(Ctx);
82	case 32:
83	return Type::getFloatTy(Ctx);
84	case 64:
85	return Type::getDoubleTy(Ctx);
86	case 80:
87	return Type::getX86_FP80Ty(Ctx);
88	case 128:
89	return Type::getFP128Ty(Ctx);
90	default:
91	return nullptr;
92	}
93	}
94
95	LegalizerHelper::LegalizerHelper(MachineFunction &MF,
96	GISelChangeObserver &Observer,
97	MachineIRBuilder &Builder)
98	: MIRBuilder(Builder), Observer(Observer), MRI(MF.getRegInfo()),
99	LI(*MF.getSubtarget().getLegalizerInfo()),
100	TLI(*MF.getSubtarget().getTargetLowering()) { }
101
102	LegalizerHelper::LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI,
103	GISelChangeObserver &Observer,
104	MachineIRBuilder &B)
105	: MIRBuilder(B), Observer(Observer), MRI(MF.getRegInfo()), LI(LI),
106	TLI(*MF.getSubtarget().getTargetLowering()) { }
107
108	LegalizerHelper::LegalizeResult
109	LegalizerHelper::legalizeInstrStep(MachineInstr &MI,
110	LostDebugLocObserver &LocObserver) {
111	LLVM_DEBUG(dbgs() << "Legalizing: " << MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizer")) { dbgs() << "Legalizing: " << MI; } } while (false);
112
113	MIRBuilder.setInstrAndDebugLoc(MI);
114
115	if (MI.getOpcode() == TargetOpcode::G_INTRINSIC \|\|
116	MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS)
117	return LI.legalizeIntrinsic(*this, MI) ? Legalized : UnableToLegalize;
118	auto Step = LI.getAction(MI, MRI);
119	switch (Step.Action) {
120	case Legal:
121	LLVM_DEBUG(dbgs() << ".. Already legal\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizer")) { dbgs() << ".. Already legal\n"; } } while (false);
122	return AlreadyLegal;
123	case Libcall:
124	LLVM_DEBUG(dbgs() << ".. Convert to libcall\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizer")) { dbgs() << ".. Convert to libcall\n"; } } while (false);
125	return libcall(MI, LocObserver);
126	case NarrowScalar:
127	LLVM_DEBUG(dbgs() << ".. Narrow scalar\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizer")) { dbgs() << ".. Narrow scalar\n"; } } while (false);
128	return narrowScalar(MI, Step.TypeIdx, Step.NewType);
129	case WidenScalar:
130	LLVM_DEBUG(dbgs() << ".. Widen scalar\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizer")) { dbgs() << ".. Widen scalar\n"; } } while (false);
131	return widenScalar(MI, Step.TypeIdx, Step.NewType);
132	case Bitcast:
133	LLVM_DEBUG(dbgs() << ".. Bitcast type\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizer")) { dbgs() << ".. Bitcast type\n"; } } while (false);
134	return bitcast(MI, Step.TypeIdx, Step.NewType);
135	case Lower:
136	LLVM_DEBUG(dbgs() << ".. Lower\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizer")) { dbgs() << ".. Lower\n"; } } while (false );
137	return lower(MI, Step.TypeIdx, Step.NewType);
138	case FewerElements:
139	LLVM_DEBUG(dbgs() << ".. Reduce number of elements\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizer")) { dbgs() << ".. Reduce number of elements\n" ; } } while (false);
140	return fewerElementsVector(MI, Step.TypeIdx, Step.NewType);
141	case MoreElements:
142	LLVM_DEBUG(dbgs() << ".. Increase number of elements\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizer")) { dbgs() << ".. Increase number of elements\n" ; } } while (false);
143	return moreElementsVector(MI, Step.TypeIdx, Step.NewType);
144	case Custom:
145	LLVM_DEBUG(dbgs() << ".. Custom legalization\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizer")) { dbgs() << ".. Custom legalization\n"; } } while (false);
146	return LI.legalizeCustom(*this, MI) ? Legalized : UnableToLegalize;
147	default:
148	LLVM_DEBUG(dbgs() << ".. Unable to legalize\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizer")) { dbgs() << ".. Unable to legalize\n"; } } while (false);
149	return UnableToLegalize;
150	}
151	}
152
153	void LegalizerHelper::extractParts(Register Reg, LLT Ty, int NumParts,
154	SmallVectorImpl<Register> &VRegs) {
155	for (int i = 0; i < NumParts; ++i)
156	VRegs.push_back(MRI.createGenericVirtualRegister(Ty));
157	MIRBuilder.buildUnmerge(VRegs, Reg);
158	}
159
160	bool LegalizerHelper::extractParts(Register Reg, LLT RegTy,
161	LLT MainTy, LLT &LeftoverTy,
162	SmallVectorImpl<Register> &VRegs,
163	SmallVectorImpl<Register> &LeftoverRegs) {
164	assert(!LeftoverTy.isValid() && "this is an out argument")(static_cast <bool> (!LeftoverTy.isValid() && "this is an out argument" ) ? void (0) : __assert_fail ("!LeftoverTy.isValid() && \"this is an out argument\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 164, __extension__ __PRETTY_FUNCTION__));
165
166	unsigned RegSize = RegTy.getSizeInBits();
167	unsigned MainSize = MainTy.getSizeInBits();
168	unsigned NumParts = RegSize / MainSize;
169	unsigned LeftoverSize = RegSize - NumParts * MainSize;
170
171	// Use an unmerge when possible.
172	if (LeftoverSize == 0) {
173	for (unsigned I = 0; I < NumParts; ++I)
174	VRegs.push_back(MRI.createGenericVirtualRegister(MainTy));
175	MIRBuilder.buildUnmerge(VRegs, Reg);
176	return true;
177	}
178
179	if (MainTy.isVector()) {
180	unsigned EltSize = MainTy.getScalarSizeInBits();
181	if (LeftoverSize % EltSize != 0)
182	return false;
183	LeftoverTy = LLT::scalarOrVector(
184	ElementCount::getFixed(LeftoverSize / EltSize), EltSize);
185	} else {
186	LeftoverTy = LLT::scalar(LeftoverSize);
187	}
188
189	// For irregular sizes, extract the individual parts.
190	for (unsigned I = 0; I != NumParts; ++I) {
191	Register NewReg = MRI.createGenericVirtualRegister(MainTy);
192	VRegs.push_back(NewReg);
193	MIRBuilder.buildExtract(NewReg, Reg, MainSize * I);
194	}
195
196	for (unsigned Offset = MainSize * NumParts; Offset < RegSize;
197	Offset += LeftoverSize) {
198	Register NewReg = MRI.createGenericVirtualRegister(LeftoverTy);
199	LeftoverRegs.push_back(NewReg);
200	MIRBuilder.buildExtract(NewReg, Reg, Offset);
201	}
202
203	return true;
204	}
205
206	void LegalizerHelper::insertParts(Register DstReg,
207	LLT ResultTy, LLT PartTy,
208	ArrayRef<Register> PartRegs,
209	LLT LeftoverTy,
210	ArrayRef<Register> LeftoverRegs) {
211	if (!LeftoverTy.isValid()) {
212	assert(LeftoverRegs.empty())(static_cast <bool> (LeftoverRegs.empty()) ? void (0) : __assert_fail ("LeftoverRegs.empty()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 212, __extension__ __PRETTY_FUNCTION__));
213
214	if (!ResultTy.isVector()) {
215	MIRBuilder.buildMerge(DstReg, PartRegs);
216	return;
217	}
218
219	if (PartTy.isVector())
220	MIRBuilder.buildConcatVectors(DstReg, PartRegs);
221	else
222	MIRBuilder.buildBuildVector(DstReg, PartRegs);
223	return;
224	}
225
226	SmallVector<Register> GCDRegs;
227	LLT GCDTy = getGCDType(getGCDType(ResultTy, LeftoverTy), PartTy);
228	for (auto PartReg : concat<const Register>(PartRegs, LeftoverRegs))
229	extractGCDType(GCDRegs, GCDTy, PartReg);
230	LLT ResultLCMTy = buildLCMMergePieces(ResultTy, LeftoverTy, GCDTy, GCDRegs);
231	buildWidenedRemergeToDst(DstReg, ResultLCMTy, GCDRegs);
232	}
233
234	/// Append the result registers of G_UNMERGE_VALUES \p MI to \p Regs.
235	static void getUnmergeResults(SmallVectorImpl<Register> &Regs,
236	const MachineInstr &MI) {
237	assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES)(static_cast <bool> (MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES ) ? void (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 237, __extension__ __PRETTY_FUNCTION__));
238
239	const int StartIdx = Regs.size();
240	const int NumResults = MI.getNumOperands() - 1;
241	Regs.resize(Regs.size() + NumResults);
242	for (int I = 0; I != NumResults; ++I)
243	Regs[StartIdx + I] = MI.getOperand(I).getReg();
244	}
245
246	void LegalizerHelper::extractGCDType(SmallVectorImpl<Register> &Parts,
247	LLT GCDTy, Register SrcReg) {
248	LLT SrcTy = MRI.getType(SrcReg);
249	if (SrcTy == GCDTy) {
250	// If the source already evenly divides the result type, we don't need to do
251	// anything.
252	Parts.push_back(SrcReg);
253	} else {
254	// Need to split into common type sized pieces.
255	auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg);
256	getUnmergeResults(Parts, *Unmerge);
257	}
258	}
259
260	LLT LegalizerHelper::extractGCDType(SmallVectorImpl<Register> &Parts, LLT DstTy,
261	LLT NarrowTy, Register SrcReg) {
262	LLT SrcTy = MRI.getType(SrcReg);
263	LLT GCDTy = getGCDType(getGCDType(SrcTy, NarrowTy), DstTy);
264	extractGCDType(Parts, GCDTy, SrcReg);
265	return GCDTy;
266	}
267
268	LLT LegalizerHelper::buildLCMMergePieces(LLT DstTy, LLT NarrowTy, LLT GCDTy,
269	SmallVectorImpl<Register> &VRegs,
270	unsigned PadStrategy) {
271	LLT LCMTy = getLCMType(DstTy, NarrowTy);
272
273	int NumParts = LCMTy.getSizeInBits() / NarrowTy.getSizeInBits();
274	int NumSubParts = NarrowTy.getSizeInBits() / GCDTy.getSizeInBits();
275	int NumOrigSrc = VRegs.size();
276
277	Register PadReg;
278
279	// Get a value we can use to pad the source value if the sources won't evenly
280	// cover the result type.
281	if (NumOrigSrc < NumParts * NumSubParts) {
282	if (PadStrategy == TargetOpcode::G_ZEXT)
283	PadReg = MIRBuilder.buildConstant(GCDTy, 0).getReg(0);
284	else if (PadStrategy == TargetOpcode::G_ANYEXT)
285	PadReg = MIRBuilder.buildUndef(GCDTy).getReg(0);
286	else {
287	assert(PadStrategy == TargetOpcode::G_SEXT)(static_cast <bool> (PadStrategy == TargetOpcode::G_SEXT ) ? void (0) : __assert_fail ("PadStrategy == TargetOpcode::G_SEXT" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 287, __extension__ __PRETTY_FUNCTION__));
288
289	// Shift the sign bit of the low register through the high register.
290	auto ShiftAmt =
291	MIRBuilder.buildConstant(LLT::scalar(64), GCDTy.getSizeInBits() - 1);
292	PadReg = MIRBuilder.buildAShr(GCDTy, VRegs.back(), ShiftAmt).getReg(0);
293	}
294	}
295
296	// Registers for the final merge to be produced.
297	SmallVector<Register, 4> Remerge(NumParts);
298
299	// Registers needed for intermediate merges, which will be merged into a
300	// source for Remerge.
301	SmallVector<Register, 4> SubMerge(NumSubParts);
302
303	// Once we've fully read off the end of the original source bits, we can reuse
304	// the same high bits for remaining padding elements.
305	Register AllPadReg;
306
307	// Build merges to the LCM type to cover the original result type.
308	for (int I = 0; I != NumParts; ++I) {
309	bool AllMergePartsArePadding = true;
310
311	// Build the requested merges to the requested type.
312	for (int J = 0; J != NumSubParts; ++J) {
313	int Idx = I * NumSubParts + J;
314	if (Idx >= NumOrigSrc) {
315	SubMerge[J] = PadReg;
316	continue;
317	}
318
319	SubMerge[J] = VRegs[Idx];
320
321	// There are meaningful bits here we can't reuse later.
322	AllMergePartsArePadding = false;
323	}
324
325	// If we've filled up a complete piece with padding bits, we can directly
326	// emit the natural sized constant if applicable, rather than a merge of
327	// smaller constants.
328	if (AllMergePartsArePadding && !AllPadReg) {
329	if (PadStrategy == TargetOpcode::G_ANYEXT)
330	AllPadReg = MIRBuilder.buildUndef(NarrowTy).getReg(0);
331	else if (PadStrategy == TargetOpcode::G_ZEXT)
332	AllPadReg = MIRBuilder.buildConstant(NarrowTy, 0).getReg(0);
333
334	// If this is a sign extension, we can't materialize a trivial constant
335	// with the right type and have to produce a merge.
336	}
337
338	if (AllPadReg) {
339	// Avoid creating additional instructions if we're just adding additional
340	// copies of padding bits.
341	Remerge[I] = AllPadReg;
342	continue;
343	}
344
345	if (NumSubParts == 1)
346	Remerge[I] = SubMerge[0];
347	else
348	Remerge[I] = MIRBuilder.buildMerge(NarrowTy, SubMerge).getReg(0);
349
350	// In the sign extend padding case, re-use the first all-signbit merge.
351	if (AllMergePartsArePadding && !AllPadReg)
352	AllPadReg = Remerge[I];
353	}
354
355	VRegs = std::move(Remerge);
356	return LCMTy;
357	}
358
359	void LegalizerHelper::buildWidenedRemergeToDst(Register DstReg, LLT LCMTy,
360	ArrayRef<Register> RemergeRegs) {
361	LLT DstTy = MRI.getType(DstReg);
362
363	// Create the merge to the widened source, and extract the relevant bits into
364	// the result.
365
366	if (DstTy == LCMTy) {
367	MIRBuilder.buildMerge(DstReg, RemergeRegs);
368	return;
369	}
370
371	auto Remerge = MIRBuilder.buildMerge(LCMTy, RemergeRegs);
372	if (DstTy.isScalar() && LCMTy.isScalar()) {
373	MIRBuilder.buildTrunc(DstReg, Remerge);
374	return;
375	}
376
377	if (LCMTy.isVector()) {
378	unsigned NumDefs = LCMTy.getSizeInBits() / DstTy.getSizeInBits();
379	SmallVector<Register, 8> UnmergeDefs(NumDefs);
380	UnmergeDefs[0] = DstReg;
381	for (unsigned I = 1; I != NumDefs; ++I)
382	UnmergeDefs[I] = MRI.createGenericVirtualRegister(DstTy);
383
384	MIRBuilder.buildUnmerge(UnmergeDefs,
385	MIRBuilder.buildMerge(LCMTy, RemergeRegs));
386	return;
387	}
388
389	llvm_unreachable("unhandled case")::llvm::llvm_unreachable_internal("unhandled case", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 389);
390	}
391
392	static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
393	#define RTLIBCASE_INT(LibcallPrefix)do { switch (Size) { case 32: return RTLIB::LibcallPrefix32; case 64: return RTLIB::LibcallPrefix64; case 128: return RTLIB::LibcallPrefix128 ; default: ::llvm::llvm_unreachable_internal("unexpected size" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 393); } } while (0) \
394	do { \
395	switch (Size) { \
396	case 32: \
397	return RTLIB::LibcallPrefix##32; \
398	case 64: \
399	return RTLIB::LibcallPrefix##64; \
400	case 128: \
401	return RTLIB::LibcallPrefix##128; \
402	default: \
403	llvm_unreachable("unexpected size")::llvm::llvm_unreachable_internal("unexpected size", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 403); \
404	} \
405	} while (0)
406
407	#define RTLIBCASE(LibcallPrefix)do { switch (Size) { case 32: return RTLIB::LibcallPrefix32; case 64: return RTLIB::LibcallPrefix64; case 80: return RTLIB::LibcallPrefix80 ; case 128: return RTLIB::LibcallPrefix128; default: ::llvm:: llvm_unreachable_internal("unexpected size", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 407); } } while (0) \
408	do { \
409	switch (Size) { \
410	case 32: \
411	return RTLIB::LibcallPrefix##32; \
412	case 64: \
413	return RTLIB::LibcallPrefix##64; \
414	case 80: \
415	return RTLIB::LibcallPrefix##80; \
416	case 128: \
417	return RTLIB::LibcallPrefix##128; \
418	default: \
419	llvm_unreachable("unexpected size")::llvm::llvm_unreachable_internal("unexpected size", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 419); \
420	} \
421	} while (0)
422
423	switch (Opcode) {
424	case TargetOpcode::G_SDIV:
425	RTLIBCASE_INT(SDIV_I)do { switch (Size) { case 32: return RTLIB::SDIV_I32; case 64 : return RTLIB::SDIV_I64; case 128: return RTLIB::SDIV_I128; default : ::llvm::llvm_unreachable_internal("unexpected size", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 425); } } while (0);
426	case TargetOpcode::G_UDIV:
427	RTLIBCASE_INT(UDIV_I)do { switch (Size) { case 32: return RTLIB::UDIV_I32; case 64 : return RTLIB::UDIV_I64; case 128: return RTLIB::UDIV_I128; default : ::llvm::llvm_unreachable_internal("unexpected size", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 427); } } while (0);
428	case TargetOpcode::G_SREM:
429	RTLIBCASE_INT(SREM_I)do { switch (Size) { case 32: return RTLIB::SREM_I32; case 64 : return RTLIB::SREM_I64; case 128: return RTLIB::SREM_I128; default : ::llvm::llvm_unreachable_internal("unexpected size", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 429); } } while (0);
430	case TargetOpcode::G_UREM:
431	RTLIBCASE_INT(UREM_I)do { switch (Size) { case 32: return RTLIB::UREM_I32; case 64 : return RTLIB::UREM_I64; case 128: return RTLIB::UREM_I128; default : ::llvm::llvm_unreachable_internal("unexpected size", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 431); } } while (0);
432	case TargetOpcode::G_CTLZ_ZERO_UNDEF:
433	RTLIBCASE_INT(CTLZ_I)do { switch (Size) { case 32: return RTLIB::CTLZ_I32; case 64 : return RTLIB::CTLZ_I64; case 128: return RTLIB::CTLZ_I128; default : ::llvm::llvm_unreachable_internal("unexpected size", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 433); } } while (0);
434	case TargetOpcode::G_FADD:
435	RTLIBCASE(ADD_F)do { switch (Size) { case 32: return RTLIB::ADD_F32; case 64: return RTLIB::ADD_F64; case 80: return RTLIB::ADD_F80; case 128 : return RTLIB::ADD_F128; default: ::llvm::llvm_unreachable_internal ("unexpected size", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 435); } } while (0);
436	case TargetOpcode::G_FSUB:
437	RTLIBCASE(SUB_F)do { switch (Size) { case 32: return RTLIB::SUB_F32; case 64: return RTLIB::SUB_F64; case 80: return RTLIB::SUB_F80; case 128 : return RTLIB::SUB_F128; default: ::llvm::llvm_unreachable_internal ("unexpected size", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 437); } } while (0);
438	case TargetOpcode::G_FMUL:
439	RTLIBCASE(MUL_F)do { switch (Size) { case 32: return RTLIB::MUL_F32; case 64: return RTLIB::MUL_F64; case 80: return RTLIB::MUL_F80; case 128 : return RTLIB::MUL_F128; default: ::llvm::llvm_unreachable_internal ("unexpected size", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 439); } } while (0);
440	case TargetOpcode::G_FDIV:
441	RTLIBCASE(DIV_F)do { switch (Size) { case 32: return RTLIB::DIV_F32; case 64: return RTLIB::DIV_F64; case 80: return RTLIB::DIV_F80; case 128 : return RTLIB::DIV_F128; default: ::llvm::llvm_unreachable_internal ("unexpected size", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 441); } } while (0);
442	case TargetOpcode::G_FEXP:
443	RTLIBCASE(EXP_F)do { switch (Size) { case 32: return RTLIB::EXP_F32; case 64: return RTLIB::EXP_F64; case 80: return RTLIB::EXP_F80; case 128 : return RTLIB::EXP_F128; default: ::llvm::llvm_unreachable_internal ("unexpected size", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 443); } } while (0);
444	case TargetOpcode::G_FEXP2:
445	RTLIBCASE(EXP2_F)do { switch (Size) { case 32: return RTLIB::EXP2_F32; case 64 : return RTLIB::EXP2_F64; case 80: return RTLIB::EXP2_F80; case 128: return RTLIB::EXP2_F128; default: ::llvm::llvm_unreachable_internal ("unexpected size", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 445); } } while (0);
446	case TargetOpcode::G_FREM:
447	RTLIBCASE(REM_F)do { switch (Size) { case 32: return RTLIB::REM_F32; case 64: return RTLIB::REM_F64; case 80: return RTLIB::REM_F80; case 128 : return RTLIB::REM_F128; default: ::llvm::llvm_unreachable_internal ("unexpected size", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 447); } } while (0);
448	case TargetOpcode::G_FPOW:
449	RTLIBCASE(POW_F)do { switch (Size) { case 32: return RTLIB::POW_F32; case 64: return RTLIB::POW_F64; case 80: return RTLIB::POW_F80; case 128 : return RTLIB::POW_F128; default: ::llvm::llvm_unreachable_internal ("unexpected size", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 449); } } while (0);
450	case TargetOpcode::G_FMA:
451	RTLIBCASE(FMA_F)do { switch (Size) { case 32: return RTLIB::FMA_F32; case 64: return RTLIB::FMA_F64; case 80: return RTLIB::FMA_F80; case 128 : return RTLIB::FMA_F128; default: ::llvm::llvm_unreachable_internal ("unexpected size", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 451); } } while (0);
452	case TargetOpcode::G_FSIN:
453	RTLIBCASE(SIN_F)do { switch (Size) { case 32: return RTLIB::SIN_F32; case 64: return RTLIB::SIN_F64; case 80: return RTLIB::SIN_F80; case 128 : return RTLIB::SIN_F128; default: ::llvm::llvm_unreachable_internal ("unexpected size", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 453); } } while (0);
454	case TargetOpcode::G_FCOS:
455	RTLIBCASE(COS_F)do { switch (Size) { case 32: return RTLIB::COS_F32; case 64: return RTLIB::COS_F64; case 80: return RTLIB::COS_F80; case 128 : return RTLIB::COS_F128; default: ::llvm::llvm_unreachable_internal ("unexpected size", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 455); } } while (0);
456	case TargetOpcode::G_FLOG10:
457	RTLIBCASE(LOG10_F)do { switch (Size) { case 32: return RTLIB::LOG10_F32; case 64 : return RTLIB::LOG10_F64; case 80: return RTLIB::LOG10_F80; case 128: return RTLIB::LOG10_F128; default: ::llvm::llvm_unreachable_internal ("unexpected size", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 457); } } while (0);
458	case TargetOpcode::G_FLOG:
459	RTLIBCASE(LOG_F)do { switch (Size) { case 32: return RTLIB::LOG_F32; case 64: return RTLIB::LOG_F64; case 80: return RTLIB::LOG_F80; case 128 : return RTLIB::LOG_F128; default: ::llvm::llvm_unreachable_internal ("unexpected size", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 459); } } while (0);
460	case TargetOpcode::G_FLOG2:
461	RTLIBCASE(LOG2_F)do { switch (Size) { case 32: return RTLIB::LOG2_F32; case 64 : return RTLIB::LOG2_F64; case 80: return RTLIB::LOG2_F80; case 128: return RTLIB::LOG2_F128; default: ::llvm::llvm_unreachable_internal ("unexpected size", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 461); } } while (0);
462	case TargetOpcode::G_FCEIL:
463	RTLIBCASE(CEIL_F)do { switch (Size) { case 32: return RTLIB::CEIL_F32; case 64 : return RTLIB::CEIL_F64; case 80: return RTLIB::CEIL_F80; case 128: return RTLIB::CEIL_F128; default: ::llvm::llvm_unreachable_internal ("unexpected size", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 463); } } while (0);
464	case TargetOpcode::G_FFLOOR:
465	RTLIBCASE(FLOOR_F)do { switch (Size) { case 32: return RTLIB::FLOOR_F32; case 64 : return RTLIB::FLOOR_F64; case 80: return RTLIB::FLOOR_F80; case 128: return RTLIB::FLOOR_F128; default: ::llvm::llvm_unreachable_internal ("unexpected size", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 465); } } while (0);
466	case TargetOpcode::G_FMINNUM:
467	RTLIBCASE(FMIN_F)do { switch (Size) { case 32: return RTLIB::FMIN_F32; case 64 : return RTLIB::FMIN_F64; case 80: return RTLIB::FMIN_F80; case 128: return RTLIB::FMIN_F128; default: ::llvm::llvm_unreachable_internal ("unexpected size", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 467); } } while (0);
468	case TargetOpcode::G_FMAXNUM:
469	RTLIBCASE(FMAX_F)do { switch (Size) { case 32: return RTLIB::FMAX_F32; case 64 : return RTLIB::FMAX_F64; case 80: return RTLIB::FMAX_F80; case 128: return RTLIB::FMAX_F128; default: ::llvm::llvm_unreachable_internal ("unexpected size", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 469); } } while (0);
470	case TargetOpcode::G_FSQRT:
471	RTLIBCASE(SQRT_F)do { switch (Size) { case 32: return RTLIB::SQRT_F32; case 64 : return RTLIB::SQRT_F64; case 80: return RTLIB::SQRT_F80; case 128: return RTLIB::SQRT_F128; default: ::llvm::llvm_unreachable_internal ("unexpected size", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 471); } } while (0);
472	case TargetOpcode::G_FRINT:
473	RTLIBCASE(RINT_F)do { switch (Size) { case 32: return RTLIB::RINT_F32; case 64 : return RTLIB::RINT_F64; case 80: return RTLIB::RINT_F80; case 128: return RTLIB::RINT_F128; default: ::llvm::llvm_unreachable_internal ("unexpected size", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 473); } } while (0);
474	case TargetOpcode::G_FNEARBYINT:
475	RTLIBCASE(NEARBYINT_F)do { switch (Size) { case 32: return RTLIB::NEARBYINT_F32; case 64: return RTLIB::NEARBYINT_F64; case 80: return RTLIB::NEARBYINT_F80 ; case 128: return RTLIB::NEARBYINT_F128; default: ::llvm::llvm_unreachable_internal ("unexpected size", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 475); } } while (0);
476	case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
477	RTLIBCASE(ROUNDEVEN_F)do { switch (Size) { case 32: return RTLIB::ROUNDEVEN_F32; case 64: return RTLIB::ROUNDEVEN_F64; case 80: return RTLIB::ROUNDEVEN_F80 ; case 128: return RTLIB::ROUNDEVEN_F128; default: ::llvm::llvm_unreachable_internal ("unexpected size", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 477); } } while (0);
478	}
479	llvm_unreachable("Unknown libcall function")::llvm::llvm_unreachable_internal("Unknown libcall function", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 479);
480	}
481
482	/// True if an instruction is in tail position in its caller. Intended for
483	/// legalizing libcalls as tail calls when possible.
484	static bool isLibCallInTailPosition(MachineInstr &MI,
485	const TargetInstrInfo &TII,
486	MachineRegisterInfo &MRI) {
487	MachineBasicBlock &MBB = *MI.getParent();
488	const Function &F = MBB.getParent()->getFunction();
489
490	// Conservatively require the attributes of the call to match those of
491	// the return. Ignore NoAlias and NonNull because they don't affect the
492	// call sequence.
493	AttributeList CallerAttrs = F.getAttributes();
494	if (AttrBuilder(CallerAttrs, AttributeList::ReturnIndex)
495	.removeAttribute(Attribute::NoAlias)
496	.removeAttribute(Attribute::NonNull)
497	.hasAttributes())
498	return false;
499
500	// It's not safe to eliminate the sign / zero extension of the return value.
501	if (CallerAttrs.hasRetAttr(Attribute::ZExt) \|\|
502	CallerAttrs.hasRetAttr(Attribute::SExt))
503	return false;
504
505	// Only tail call if the following instruction is a standard return or if we
506	// have a `thisreturn` callee, and a sequence like:
507	//
508	// G_MEMCPY %0, %1, %2
509	// $x0 = COPY %0
510	// RET_ReallyLR implicit $x0
511	auto Next = next_nodbg(MI.getIterator(), MBB.instr_end());
512	if (Next != MBB.instr_end() && Next->isCopy()) {
513	switch (MI.getOpcode()) {
514	default:
515	llvm_unreachable("unsupported opcode")::llvm::llvm_unreachable_internal("unsupported opcode", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 515);
516	case TargetOpcode::G_BZERO:
517	return false;
518	case TargetOpcode::G_MEMCPY:
519	case TargetOpcode::G_MEMMOVE:
520	case TargetOpcode::G_MEMSET:
521	break;
522	}
523
524	Register VReg = MI.getOperand(0).getReg();
525	if (!VReg.isVirtual() \|\| VReg != Next->getOperand(1).getReg())
526	return false;
527
528	Register PReg = Next->getOperand(0).getReg();
529	if (!PReg.isPhysical())
530	return false;
531
532	auto Ret = next_nodbg(Next, MBB.instr_end());
533	if (Ret == MBB.instr_end() \|\| !Ret->isReturn())
534	return false;
535
536	if (Ret->getNumImplicitOperands() != 1)
537	return false;
538
539	if (PReg != Ret->getOperand(0).getReg())
540	return false;
541
542	// Skip over the COPY that we just validated.
543	Next = Ret;
544	}
545
546	if (Next == MBB.instr_end() \|\| TII.isTailCall(*Next) \|\| !Next->isReturn())
547	return false;
548
549	return true;
550	}
551
552	LegalizerHelper::LegalizeResult
553	llvm::createLibcall(MachineIRBuilder &MIRBuilder, const char *Name,
554	const CallLowering::ArgInfo &Result,
555	ArrayRef<CallLowering::ArgInfo> Args,
556	const CallingConv::ID CC) {
557	auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
558
559	CallLowering::CallLoweringInfo Info;
560	Info.CallConv = CC;
561	Info.Callee = MachineOperand::CreateES(Name);
562	Info.OrigRet = Result;
563	std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs));
564	if (!CLI.lowerCall(MIRBuilder, Info))
565	return LegalizerHelper::UnableToLegalize;
566
567	return LegalizerHelper::Legalized;
568	}
569
570	LegalizerHelper::LegalizeResult
571	llvm::createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall,
572	const CallLowering::ArgInfo &Result,
573	ArrayRef<CallLowering::ArgInfo> Args) {
574	auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
575	const char *Name = TLI.getLibcallName(Libcall);
576	const CallingConv::ID CC = TLI.getLibcallCallingConv(Libcall);
577	return createLibcall(MIRBuilder, Name, Result, Args, CC);
578	}
579
580	// Useful for libcalls where all operands have the same type.
581	static LegalizerHelper::LegalizeResult
582	simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size,
583	Type *OpType) {
584	auto Libcall = getRTLibDesc(MI.getOpcode(), Size);
585
586	// FIXME: What does the original arg index mean here?
587	SmallVector<CallLowering::ArgInfo, 3> Args;
588	for (unsigned i = 1; i < MI.getNumOperands(); i++)
589	Args.push_back({MI.getOperand(i).getReg(), OpType, 0});
590	return createLibcall(MIRBuilder, Libcall,
591	{MI.getOperand(0).getReg(), OpType, 0}, Args);
592	}
593
594	LegalizerHelper::LegalizeResult
595	llvm::createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
596	MachineInstr &MI, LostDebugLocObserver &LocObserver) {
597	auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
598
599	SmallVector<CallLowering::ArgInfo, 3> Args;
600	// Add all the args, except for the last which is an imm denoting 'tail'.
601	for (unsigned i = 0; i < MI.getNumOperands() - 1; ++i) {
602	Register Reg = MI.getOperand(i).getReg();
603
604	// Need derive an IR type for call lowering.
605	LLT OpLLT = MRI.getType(Reg);
606	Type *OpTy = nullptr;
607	if (OpLLT.isPointer())
608	OpTy = Type::getInt8PtrTy(Ctx, OpLLT.getAddressSpace());
609	else
610	OpTy = IntegerType::get(Ctx, OpLLT.getSizeInBits());
611	Args.push_back({Reg, OpTy, 0});
612	}
613
614	auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
615	auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
616	RTLIB::Libcall RTLibcall;
617	unsigned Opc = MI.getOpcode();
618	switch (Opc) {
619	case TargetOpcode::G_BZERO:
620	RTLibcall = RTLIB::BZERO;
621	break;
622	case TargetOpcode::G_MEMCPY:
623	RTLibcall = RTLIB::MEMCPY;
624	Args[0].Flags[0].setReturned();
625	break;
626	case TargetOpcode::G_MEMMOVE:
627	RTLibcall = RTLIB::MEMMOVE;
628	Args[0].Flags[0].setReturned();
629	break;
630	case TargetOpcode::G_MEMSET:
631	RTLibcall = RTLIB::MEMSET;
632	Args[0].Flags[0].setReturned();
633	break;
634	default:
635	llvm_unreachable("unsupported opcode")::llvm::llvm_unreachable_internal("unsupported opcode", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 635);
636	}
637	const char *Name = TLI.getLibcallName(RTLibcall);
638
639	// Unsupported libcall on the target.
640	if (!Name) {
641	LLVM_DEBUG(dbgs() << ".. .. Could not find libcall name for "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizer")) { dbgs() << ".. .. Could not find libcall name for " << MIRBuilder.getTII().getName(Opc) << "\n"; } } while (false)
642	<< MIRBuilder.getTII().getName(Opc) << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizer")) { dbgs() << ".. .. Could not find libcall name for " << MIRBuilder.getTII().getName(Opc) << "\n"; } } while (false);
643	return LegalizerHelper::UnableToLegalize;
644	}
645
646	CallLowering::CallLoweringInfo Info;
647	Info.CallConv = TLI.getLibcallCallingConv(RTLibcall);
648	Info.Callee = MachineOperand::CreateES(Name);
649	Info.OrigRet = CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx), 0);
650	Info.IsTailCall = MI.getOperand(MI.getNumOperands() - 1).getImm() &&
651	isLibCallInTailPosition(MI, MIRBuilder.getTII(), MRI);
652
653	std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs));
654	if (!CLI.lowerCall(MIRBuilder, Info))
655	return LegalizerHelper::UnableToLegalize;
656
657	if (Info.LoweredTailCall) {
658	assert(Info.IsTailCall && "Lowered tail call when it wasn't a tail call?")(static_cast <bool> (Info.IsTailCall && "Lowered tail call when it wasn't a tail call?" ) ? void (0) : __assert_fail ("Info.IsTailCall && \"Lowered tail call when it wasn't a tail call?\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 658, __extension__ __PRETTY_FUNCTION__));
659
660	// Check debug locations before removing the return.
661	LocObserver.checkpoint(true);
662
663	// We must have a return following the call (or debug insts) to get past
664	// isLibCallInTailPosition.
665	do {
666	MachineInstr *Next = MI.getNextNode();
667	assert(Next &&(static_cast <bool> (Next && (Next->isCopy() \|\| Next->isReturn() \|\| Next->isDebugInstr()) && "Expected instr following MI to be return or debug inst?") ? void (0) : __assert_fail ("Next && (Next->isCopy() \|\| Next->isReturn() \|\| Next->isDebugInstr()) && \"Expected instr following MI to be return or debug inst?\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 669, __extension__ __PRETTY_FUNCTION__))
668	(Next->isCopy() \|\| Next->isReturn() \|\| Next->isDebugInstr()) &&(static_cast <bool> (Next && (Next->isCopy() \|\| Next->isReturn() \|\| Next->isDebugInstr()) && "Expected instr following MI to be return or debug inst?") ? void (0) : __assert_fail ("Next && (Next->isCopy() \|\| Next->isReturn() \|\| Next->isDebugInstr()) && \"Expected instr following MI to be return or debug inst?\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 669, __extension__ __PRETTY_FUNCTION__))
669	"Expected instr following MI to be return or debug inst?")(static_cast <bool> (Next && (Next->isCopy() \|\| Next->isReturn() \|\| Next->isDebugInstr()) && "Expected instr following MI to be return or debug inst?") ? void (0) : __assert_fail ("Next && (Next->isCopy() \|\| Next->isReturn() \|\| Next->isDebugInstr()) && \"Expected instr following MI to be return or debug inst?\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 669, __extension__ __PRETTY_FUNCTION__));
670	// We lowered a tail call, so the call is now the return from the block.
671	// Delete the old return.
672	Next->eraseFromParent();
673	} while (MI.getNextNode());
674
675	// We expect to lose the debug location from the return.
676	LocObserver.checkpoint(false);
677	}
678
679	return LegalizerHelper::Legalized;
680	}
681
682	static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType,
683	Type *FromType) {
684	auto ToMVT = MVT::getVT(ToType);
685	auto FromMVT = MVT::getVT(FromType);
686
687	switch (Opcode) {
688	case TargetOpcode::G_FPEXT:
689	return RTLIB::getFPEXT(FromMVT, ToMVT);
690	case TargetOpcode::G_FPTRUNC:
691	return RTLIB::getFPROUND(FromMVT, ToMVT);
692	case TargetOpcode::G_FPTOSI:
693	return RTLIB::getFPTOSINT(FromMVT, ToMVT);
694	case TargetOpcode::G_FPTOUI:
695	return RTLIB::getFPTOUINT(FromMVT, ToMVT);
696	case TargetOpcode::G_SITOFP:
697	return RTLIB::getSINTTOFP(FromMVT, ToMVT);
698	case TargetOpcode::G_UITOFP:
699	return RTLIB::getUINTTOFP(FromMVT, ToMVT);
700	}
701	llvm_unreachable("Unsupported libcall function")::llvm::llvm_unreachable_internal("Unsupported libcall function" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 701);
702	}
703
704	static LegalizerHelper::LegalizeResult
705	conversionLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, Type *ToType,
706	Type *FromType) {
707	RTLIB::Libcall Libcall = getConvRTLibDesc(MI.getOpcode(), ToType, FromType);
708	return createLibcall(MIRBuilder, Libcall,
709	{MI.getOperand(0).getReg(), ToType, 0},
710	{{MI.getOperand(1).getReg(), FromType, 0}});
711	}
712
713	LegalizerHelper::LegalizeResult
714	LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
715	LLT LLTy = MRI.getType(MI.getOperand(0).getReg());
716	unsigned Size = LLTy.getSizeInBits();
717	auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
718
719	switch (MI.getOpcode()) {
720	default:
721	return UnableToLegalize;
722	case TargetOpcode::G_SDIV:
723	case TargetOpcode::G_UDIV:
724	case TargetOpcode::G_SREM:
725	case TargetOpcode::G_UREM:
726	case TargetOpcode::G_CTLZ_ZERO_UNDEF: {
727	Type *HLTy = IntegerType::get(Ctx, Size);
728	auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
729	if (Status != Legalized)
730	return Status;
731	break;
732	}
733	case TargetOpcode::G_FADD:
734	case TargetOpcode::G_FSUB:
735	case TargetOpcode::G_FMUL:
736	case TargetOpcode::G_FDIV:
737	case TargetOpcode::G_FMA:
738	case TargetOpcode::G_FPOW:
739	case TargetOpcode::G_FREM:
740	case TargetOpcode::G_FCOS:
741	case TargetOpcode::G_FSIN:
742	case TargetOpcode::G_FLOG10:
743	case TargetOpcode::G_FLOG:
744	case TargetOpcode::G_FLOG2:
745	case TargetOpcode::G_FEXP:
746	case TargetOpcode::G_FEXP2:
747	case TargetOpcode::G_FCEIL:
748	case TargetOpcode::G_FFLOOR:
749	case TargetOpcode::G_FMINNUM:
750	case TargetOpcode::G_FMAXNUM:
751	case TargetOpcode::G_FSQRT:
752	case TargetOpcode::G_FRINT:
753	case TargetOpcode::G_FNEARBYINT:
754	case TargetOpcode::G_INTRINSIC_ROUNDEVEN: {
755	Type *HLTy = getFloatTypeForLLT(Ctx, LLTy);
756	if (!HLTy \|\| (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
757	LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizer")) { dbgs() << "No libcall available for type " << LLTy << ".\n"; } } while (false);
758	return UnableToLegalize;
759	}
760	auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
761	if (Status != Legalized)
762	return Status;
763	break;
764	}
765	case TargetOpcode::G_FPEXT:
766	case TargetOpcode::G_FPTRUNC: {
767	Type *FromTy = getFloatTypeForLLT(Ctx, MRI.getType(MI.getOperand(1).getReg()));
768	Type *ToTy = getFloatTypeForLLT(Ctx, MRI.getType(MI.getOperand(0).getReg()));
769	if (!FromTy \|\| !ToTy)
770	return UnableToLegalize;
771	LegalizeResult Status = conversionLibcall(MI, MIRBuilder, ToTy, FromTy );
772	if (Status != Legalized)
773	return Status;
774	break;
775	}
776	case TargetOpcode::G_FPTOSI:
777	case TargetOpcode::G_FPTOUI: {
778	// FIXME: Support other types
779	unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
780	unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
781	if ((ToSize != 32 && ToSize != 64) \|\| (FromSize != 32 && FromSize != 64))
782	return UnableToLegalize;
783	LegalizeResult Status = conversionLibcall(
784	MI, MIRBuilder,
785	ToSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx),
786	FromSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx));
787	if (Status != Legalized)
788	return Status;
789	break;
790	}
791	case TargetOpcode::G_SITOFP:
792	case TargetOpcode::G_UITOFP: {
793	// FIXME: Support other types
794	unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
795	unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
796	if ((FromSize != 32 && FromSize != 64) \|\| (ToSize != 32 && ToSize != 64))
797	return UnableToLegalize;
798	LegalizeResult Status = conversionLibcall(
799	MI, MIRBuilder,
800	ToSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx),
801	FromSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx));
802	if (Status != Legalized)
803	return Status;
804	break;
805	}
806	case TargetOpcode::G_BZERO:
807	case TargetOpcode::G_MEMCPY:
808	case TargetOpcode::G_MEMMOVE:
809	case TargetOpcode::G_MEMSET: {
810	LegalizeResult Result =
811	createMemLibcall(MIRBuilder, *MIRBuilder.getMRI(), MI, LocObserver);
812	if (Result != Legalized)
813	return Result;
814	MI.eraseFromParent();
815	return Result;
816	}
817	}
818
819	MI.eraseFromParent();
820	return Legalized;
821	}
822
823	LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
824	unsigned TypeIdx,
825	LLT NarrowTy) {
826	uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
827	uint64_t NarrowSize = NarrowTy.getSizeInBits();
828
829	switch (MI.getOpcode()) {
830	default:
831	return UnableToLegalize;
832	case TargetOpcode::G_IMPLICIT_DEF: {
833	Register DstReg = MI.getOperand(0).getReg();
834	LLT DstTy = MRI.getType(DstReg);
835
836	// If SizeOp0 is not an exact multiple of NarrowSize, emit
837	// G_ANYEXT(G_IMPLICIT_DEF). Cast result to vector if needed.
838	// FIXME: Although this would also be legal for the general case, it causes
839	// a lot of regressions in the emitted code (superfluous COPYs, artifact
840	// combines not being hit). This seems to be a problem related to the
841	// artifact combiner.
842	if (SizeOp0 % NarrowSize != 0) {
843	LLT ImplicitTy = NarrowTy;
844	if (DstTy.isVector())
845	ImplicitTy = LLT::vector(DstTy.getElementCount(), ImplicitTy);
846
847	Register ImplicitReg = MIRBuilder.buildUndef(ImplicitTy).getReg(0);
848	MIRBuilder.buildAnyExt(DstReg, ImplicitReg);
849
850	MI.eraseFromParent();
851	return Legalized;
852	}
853
854	int NumParts = SizeOp0 / NarrowSize;
855
856	SmallVector<Register, 2> DstRegs;
857	for (int i = 0; i < NumParts; ++i)
858	DstRegs.push_back(MIRBuilder.buildUndef(NarrowTy).getReg(0));
859
860	if (DstTy.isVector())
861	MIRBuilder.buildBuildVector(DstReg, DstRegs);
862	else
863	MIRBuilder.buildMerge(DstReg, DstRegs);
864	MI.eraseFromParent();
865	return Legalized;
866	}
867	case TargetOpcode::G_CONSTANT: {
868	LLT Ty = MRI.getType(MI.getOperand(0).getReg());
869	const APInt &Val = MI.getOperand(1).getCImm()->getValue();
870	unsigned TotalSize = Ty.getSizeInBits();
871	unsigned NarrowSize = NarrowTy.getSizeInBits();
872	int NumParts = TotalSize / NarrowSize;
873
874	SmallVector<Register, 4> PartRegs;
875	for (int I = 0; I != NumParts; ++I) {
876	unsigned Offset = I * NarrowSize;
877	auto K = MIRBuilder.buildConstant(NarrowTy,
878	Val.lshr(Offset).trunc(NarrowSize));
879	PartRegs.push_back(K.getReg(0));
880	}
881
882	LLT LeftoverTy;
883	unsigned LeftoverBits = TotalSize - NumParts * NarrowSize;
884	SmallVector<Register, 1> LeftoverRegs;
885	if (LeftoverBits != 0) {
886	LeftoverTy = LLT::scalar(LeftoverBits);
887	auto K = MIRBuilder.buildConstant(
888	LeftoverTy,
889	Val.lshr(NumParts * NarrowSize).trunc(LeftoverBits));
890	LeftoverRegs.push_back(K.getReg(0));
891	}
892
893	insertParts(MI.getOperand(0).getReg(),
894	Ty, NarrowTy, PartRegs, LeftoverTy, LeftoverRegs);
895
896	MI.eraseFromParent();
897	return Legalized;
898	}
899	case TargetOpcode::G_SEXT:
900	case TargetOpcode::G_ZEXT:
901	case TargetOpcode::G_ANYEXT:
902	return narrowScalarExt(MI, TypeIdx, NarrowTy);
903	case TargetOpcode::G_TRUNC: {
904	if (TypeIdx != 1)
905	return UnableToLegalize;
906
907	uint64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
908	if (NarrowTy.getSizeInBits() * 2 != SizeOp1) {
909	LLVM_DEBUG(dbgs() << "Can't narrow trunc to type " << NarrowTy << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizer")) { dbgs() << "Can't narrow trunc to type " << NarrowTy << "\n"; } } while (false);
910	return UnableToLegalize;
911	}
912
913	auto Unmerge = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1));
914	MIRBuilder.buildCopy(MI.getOperand(0), Unmerge.getReg(0));
915	MI.eraseFromParent();
916	return Legalized;
917	}
918
919	case TargetOpcode::G_FREEZE:
920	return reduceOperationWidth(MI, TypeIdx, NarrowTy);
921	case TargetOpcode::G_ADD:
922	case TargetOpcode::G_SUB:
923	case TargetOpcode::G_SADDO:
924	case TargetOpcode::G_SSUBO:
925	case TargetOpcode::G_SADDE:
926	case TargetOpcode::G_SSUBE:
927	case TargetOpcode::G_UADDO:
928	case TargetOpcode::G_USUBO:
929	case TargetOpcode::G_UADDE:
930	case TargetOpcode::G_USUBE:
931	return narrowScalarAddSub(MI, TypeIdx, NarrowTy);
932	case TargetOpcode::G_MUL:
933	case TargetOpcode::G_UMULH:
934	return narrowScalarMul(MI, NarrowTy);
935	case TargetOpcode::G_EXTRACT:
936	return narrowScalarExtract(MI, TypeIdx, NarrowTy);
937	case TargetOpcode::G_INSERT:
938	return narrowScalarInsert(MI, TypeIdx, NarrowTy);
939	case TargetOpcode::G_LOAD: {
940	auto &LoadMI = cast<GLoad>(MI);
941	Register DstReg = LoadMI.getDstReg();
942	LLT DstTy = MRI.getType(DstReg);
943	if (DstTy.isVector())
944	return UnableToLegalize;
945
946	if (8 * LoadMI.getMemSize() != DstTy.getSizeInBits()) {
947	Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
948	MIRBuilder.buildLoad(TmpReg, LoadMI.getPointerReg(), LoadMI.getMMO());
949	MIRBuilder.buildAnyExt(DstReg, TmpReg);
950	LoadMI.eraseFromParent();
951	return Legalized;
952	}
953
954	return reduceLoadStoreWidth(LoadMI, TypeIdx, NarrowTy);
955	}
956	case TargetOpcode::G_ZEXTLOAD:
957	case TargetOpcode::G_SEXTLOAD: {
958	auto &LoadMI = cast<GExtLoad>(MI);
959	Register DstReg = LoadMI.getDstReg();
960	Register PtrReg = LoadMI.getPointerReg();
961
962	Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
963	auto &MMO = LoadMI.getMMO();
964	unsigned MemSize = MMO.getSizeInBits();
965
966	if (MemSize == NarrowSize) {
967	MIRBuilder.buildLoad(TmpReg, PtrReg, MMO);
968	} else if (MemSize < NarrowSize) {
969	MIRBuilder.buildLoadInstr(LoadMI.getOpcode(), TmpReg, PtrReg, MMO);
970	} else if (MemSize > NarrowSize) {
971	// FIXME: Need to split the load.
972	return UnableToLegalize;
973	}
974
975	if (isa<GZExtLoad>(LoadMI))
976	MIRBuilder.buildZExt(DstReg, TmpReg);
977	else
978	MIRBuilder.buildSExt(DstReg, TmpReg);
979
980	LoadMI.eraseFromParent();
981	return Legalized;
982	}
983	case TargetOpcode::G_STORE: {
984	auto &StoreMI = cast<GStore>(MI);
985
986	Register SrcReg = StoreMI.getValueReg();
987	LLT SrcTy = MRI.getType(SrcReg);
988	if (SrcTy.isVector())
989	return UnableToLegalize;
990
991	int NumParts = SizeOp0 / NarrowSize;
992	unsigned HandledSize = NumParts * NarrowTy.getSizeInBits();
993	unsigned LeftoverBits = SrcTy.getSizeInBits() - HandledSize;
994	if (SrcTy.isVector() && LeftoverBits != 0)
995	return UnableToLegalize;
996
997	if (8 * StoreMI.getMemSize() != SrcTy.getSizeInBits()) {
998	Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
999	MIRBuilder.buildTrunc(TmpReg, SrcReg);
1000	MIRBuilder.buildStore(TmpReg, StoreMI.getPointerReg(), StoreMI.getMMO());
1001	StoreMI.eraseFromParent();
1002	return Legalized;
1003	}
1004
1005	return reduceLoadStoreWidth(StoreMI, 0, NarrowTy);
1006	}
1007	case TargetOpcode::G_SELECT:
1008	return narrowScalarSelect(MI, TypeIdx, NarrowTy);
1009	case TargetOpcode::G_AND:
1010	case TargetOpcode::G_OR:
1011	case TargetOpcode::G_XOR: {
1012	// Legalize bitwise operation:
1013	// A = BinOp<Ty> B, C
1014	// into:
1015	// B1, ..., BN = G_UNMERGE_VALUES B
1016	// C1, ..., CN = G_UNMERGE_VALUES C
1017	// A1 = BinOp<Ty/N> B1, C2
1018	// ...
1019	// AN = BinOp<Ty/N> BN, CN
1020	// A = G_MERGE_VALUES A1, ..., AN
1021	return narrowScalarBasic(MI, TypeIdx, NarrowTy);
1022	}
1023	case TargetOpcode::G_SHL:
1024	case TargetOpcode::G_LSHR:
1025	case TargetOpcode::G_ASHR:
1026	return narrowScalarShift(MI, TypeIdx, NarrowTy);
1027	case TargetOpcode::G_CTLZ:
1028	case TargetOpcode::G_CTLZ_ZERO_UNDEF:
1029	case TargetOpcode::G_CTTZ:
1030	case TargetOpcode::G_CTTZ_ZERO_UNDEF:
1031	case TargetOpcode::G_CTPOP:
1032	if (TypeIdx == 1)
1033	switch (MI.getOpcode()) {
1034	case TargetOpcode::G_CTLZ:
1035	case TargetOpcode::G_CTLZ_ZERO_UNDEF:
1036	return narrowScalarCTLZ(MI, TypeIdx, NarrowTy);
1037	case TargetOpcode::G_CTTZ:
1038	case TargetOpcode::G_CTTZ_ZERO_UNDEF:
1039	return narrowScalarCTTZ(MI, TypeIdx, NarrowTy);
1040	case TargetOpcode::G_CTPOP:
1041	return narrowScalarCTPOP(MI, TypeIdx, NarrowTy);
1042	default:
1043	return UnableToLegalize;
1044	}
1045
1046	Observer.changingInstr(MI);
1047	narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT);
1048	Observer.changedInstr(MI);
1049	return Legalized;
1050	case TargetOpcode::G_INTTOPTR:
1051	if (TypeIdx != 1)
1052	return UnableToLegalize;
1053
1054	Observer.changingInstr(MI);
1055	narrowScalarSrc(MI, NarrowTy, 1);
1056	Observer.changedInstr(MI);
1057	return Legalized;
1058	case TargetOpcode::G_PTRTOINT:
1059	if (TypeIdx != 0)
1060	return UnableToLegalize;
1061
1062	Observer.changingInstr(MI);
1063	narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT);
1064	Observer.changedInstr(MI);
1065	return Legalized;
1066	case TargetOpcode::G_PHI: {
1067	// FIXME: add support for when SizeOp0 isn't an exact multiple of
1068	// NarrowSize.
1069	if (SizeOp0 % NarrowSize != 0)
1070	return UnableToLegalize;
1071
1072	unsigned NumParts = SizeOp0 / NarrowSize;
1073	SmallVector<Register, 2> DstRegs(NumParts);
1074	SmallVector<SmallVector<Register, 2>, 2> SrcRegs(MI.getNumOperands() / 2);
1075	Observer.changingInstr(MI);
1076	for (unsigned i = 1; i < MI.getNumOperands(); i += 2) {
1077	MachineBasicBlock &OpMBB = *MI.getOperand(i + 1).getMBB();
1078	MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
1079	extractParts(MI.getOperand(i).getReg(), NarrowTy, NumParts,
1080	SrcRegs[i / 2]);
1081	}
1082	MachineBasicBlock &MBB = *MI.getParent();
1083	MIRBuilder.setInsertPt(MBB, MI);
1084	for (unsigned i = 0; i < NumParts; ++i) {
1085	DstRegs[i] = MRI.createGenericVirtualRegister(NarrowTy);
1086	MachineInstrBuilder MIB =
1087	MIRBuilder.buildInstr(TargetOpcode::G_PHI).addDef(DstRegs[i]);
1088	for (unsigned j = 1; j < MI.getNumOperands(); j += 2)
1089	MIB.addUse(SrcRegs[j / 2][i]).add(MI.getOperand(j + 1));
1090	}
1091	MIRBuilder.setInsertPt(MBB, MBB.getFirstNonPHI());
1092	MIRBuilder.buildMerge(MI.getOperand(0), DstRegs);
1093	Observer.changedInstr(MI);
1094	MI.eraseFromParent();
1095	return Legalized;
1096	}
1097	case TargetOpcode::G_EXTRACT_VECTOR_ELT:
1098	case TargetOpcode::G_INSERT_VECTOR_ELT: {
1099	if (TypeIdx != 2)
1100	return UnableToLegalize;
1101
1102	int OpIdx = MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3;
1103	Observer.changingInstr(MI);
1104	narrowScalarSrc(MI, NarrowTy, OpIdx);
1105	Observer.changedInstr(MI);
1106	return Legalized;
1107	}
1108	case TargetOpcode::G_ICMP: {
1109	Register LHS = MI.getOperand(2).getReg();
1110	LLT SrcTy = MRI.getType(LHS);
1111	uint64_t SrcSize = SrcTy.getSizeInBits();
1112	CmpInst::Predicate Pred =
1113	static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
1114
1115	// TODO: Handle the non-equality case for weird sizes.
1116	if (NarrowSize * 2 != SrcSize && !ICmpInst::isEquality(Pred))
1117	return UnableToLegalize;
1118
1119	LLT LeftoverTy; // Example: s88 -> s64 (NarrowTy) + s24 (leftover)
1120	SmallVector<Register, 4> LHSPartRegs, LHSLeftoverRegs;
1121	if (!extractParts(LHS, SrcTy, NarrowTy, LeftoverTy, LHSPartRegs,
1122	LHSLeftoverRegs))
1123	return UnableToLegalize;
1124
1125	LLT Unused; // Matches LeftoverTy; G_ICMP LHS and RHS are the same type.
1126	SmallVector<Register, 4> RHSPartRegs, RHSLeftoverRegs;
1127	if (!extractParts(MI.getOperand(3).getReg(), SrcTy, NarrowTy, Unused,
1128	RHSPartRegs, RHSLeftoverRegs))
1129	return UnableToLegalize;
1130
1131	// We now have the LHS and RHS of the compare split into narrow-type
1132	// registers, plus potentially some leftover type.
1133	Register Dst = MI.getOperand(0).getReg();
1134	LLT ResTy = MRI.getType(Dst);
1135	if (ICmpInst::isEquality(Pred)) {
1136	// For each part on the LHS and RHS, keep track of the result of XOR-ing
1137	// them together. For each equal part, the result should be all 0s. For
1138	// each non-equal part, we'll get at least one 1.
1139	auto Zero = MIRBuilder.buildConstant(NarrowTy, 0);
1140	SmallVector<Register, 4> Xors;
1141	for (auto LHSAndRHS : zip(LHSPartRegs, RHSPartRegs)) {
1142	auto LHS = std::get<0>(LHSAndRHS);
1143	auto RHS = std::get<1>(LHSAndRHS);
1144	auto Xor = MIRBuilder.buildXor(NarrowTy, LHS, RHS).getReg(0);
1145	Xors.push_back(Xor);
1146	}
1147
1148	// Build a G_XOR for each leftover register. Each G_XOR must be widened
1149	// to the desired narrow type so that we can OR them together later.
1150	SmallVector<Register, 4> WidenedXors;
1151	for (auto LHSAndRHS : zip(LHSLeftoverRegs, RHSLeftoverRegs)) {
1152	auto LHS = std::get<0>(LHSAndRHS);
1153	auto RHS = std::get<1>(LHSAndRHS);
1154	auto Xor = MIRBuilder.buildXor(LeftoverTy, LHS, RHS).getReg(0);
1155	LLT GCDTy = extractGCDType(WidenedXors, NarrowTy, LeftoverTy, Xor);
1156	buildLCMMergePieces(LeftoverTy, NarrowTy, GCDTy, WidenedXors,
1157	/* PadStrategy = */ TargetOpcode::G_ZEXT);
1158	Xors.insert(Xors.end(), WidenedXors.begin(), WidenedXors.end());
1159	}
1160
1161	// Now, for each part we broke up, we know if they are equal/not equal
1162	// based off the G_XOR. We can OR these all together and compare against
1163	// 0 to get the result.
1164	assert(Xors.size() >= 2 && "Should have gotten at least two Xors?")(static_cast <bool> (Xors.size() >= 2 && "Should have gotten at least two Xors?" ) ? void (0) : __assert_fail ("Xors.size() >= 2 && \"Should have gotten at least two Xors?\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 1164, __extension__ __PRETTY_FUNCTION__));
1165	auto Or = MIRBuilder.buildOr(NarrowTy, Xors[0], Xors[1]);
1166	for (unsigned I = 2, E = Xors.size(); I < E; ++I)
1167	Or = MIRBuilder.buildOr(NarrowTy, Or, Xors[I]);
1168	MIRBuilder.buildICmp(Pred, Dst, Or, Zero);
1169	} else {
1170	// TODO: Handle non-power-of-two types.
1171	assert(LHSPartRegs.size() == 2 && "Expected exactly 2 LHS part regs?")(static_cast <bool> (LHSPartRegs.size() == 2 && "Expected exactly 2 LHS part regs?") ? void (0) : __assert_fail ("LHSPartRegs.size() == 2 && \"Expected exactly 2 LHS part regs?\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 1171, __extension__ __PRETTY_FUNCTION__));
1172	assert(RHSPartRegs.size() == 2 && "Expected exactly 2 RHS part regs?")(static_cast <bool> (RHSPartRegs.size() == 2 && "Expected exactly 2 RHS part regs?") ? void (0) : __assert_fail ("RHSPartRegs.size() == 2 && \"Expected exactly 2 RHS part regs?\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 1172, __extension__ __PRETTY_FUNCTION__));
1173	Register LHSL = LHSPartRegs[0];
1174	Register LHSH = LHSPartRegs[1];
1175	Register RHSL = RHSPartRegs[0];
1176	Register RHSH = RHSPartRegs[1];
1177	MachineInstrBuilder CmpH = MIRBuilder.buildICmp(Pred, ResTy, LHSH, RHSH);
1178	MachineInstrBuilder CmpHEQ =
1179	MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, ResTy, LHSH, RHSH);
1180	MachineInstrBuilder CmpLU = MIRBuilder.buildICmp(
1181	ICmpInst::getUnsignedPredicate(Pred), ResTy, LHSL, RHSL);
1182	MIRBuilder.buildSelect(Dst, CmpHEQ, CmpLU, CmpH);
1183	}
1184	MI.eraseFromParent();
1185	return Legalized;
1186	}
1187	case TargetOpcode::G_SEXT_INREG: {
1188	if (TypeIdx != 0)
1189	return UnableToLegalize;
1190
1191	int64_t SizeInBits = MI.getOperand(2).getImm();
1192
1193	// So long as the new type has more bits than the bits we're extending we
1194	// don't need to break it apart.
1195	if (NarrowTy.getScalarSizeInBits() >= SizeInBits) {
1196	Observer.changingInstr(MI);
1197	// We don't lose any non-extension bits by truncating the src and
1198	// sign-extending the dst.
1199	MachineOperand &MO1 = MI.getOperand(1);
1200	auto TruncMIB = MIRBuilder.buildTrunc(NarrowTy, MO1);
1201	MO1.setReg(TruncMIB.getReg(0));
1202
1203	MachineOperand &MO2 = MI.getOperand(0);
1204	Register DstExt = MRI.createGenericVirtualRegister(NarrowTy);
1205	MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1206	MIRBuilder.buildSExt(MO2, DstExt);
1207	MO2.setReg(DstExt);
1208	Observer.changedInstr(MI);
1209	return Legalized;
1210	}
1211
1212	// Break it apart. Components below the extension point are unmodified. The
1213	// component containing the extension point becomes a narrower SEXT_INREG.
1214	// Components above it are ashr'd from the component containing the
1215	// extension point.
1216	if (SizeOp0 % NarrowSize != 0)
1217	return UnableToLegalize;
1218	int NumParts = SizeOp0 / NarrowSize;
1219
1220	// List the registers where the destination will be scattered.
1221	SmallVector<Register, 2> DstRegs;
1222	// List the registers where the source will be split.
1223	SmallVector<Register, 2> SrcRegs;
1224
1225	// Create all the temporary registers.
1226	for (int i = 0; i < NumParts; ++i) {
1227	Register SrcReg = MRI.createGenericVirtualRegister(NarrowTy);
1228
1229	SrcRegs.push_back(SrcReg);
1230	}
1231
1232	// Explode the big arguments into smaller chunks.
1233	MIRBuilder.buildUnmerge(SrcRegs, MI.getOperand(1));
1234
1235	Register AshrCstReg =
1236	MIRBuilder.buildConstant(NarrowTy, NarrowTy.getScalarSizeInBits() - 1)
1237	.getReg(0);
1238	Register FullExtensionReg = 0;
1239	Register PartialExtensionReg = 0;
1240
1241	// Do the operation on each small part.
1242	for (int i = 0; i < NumParts; ++i) {
1243	if ((i + 1) * NarrowTy.getScalarSizeInBits() < SizeInBits)
1244	DstRegs.push_back(SrcRegs[i]);
1245	else if (i * NarrowTy.getScalarSizeInBits() > SizeInBits) {
1246	assert(PartialExtensionReg &&(static_cast <bool> (PartialExtensionReg && "Expected to visit partial extension before full" ) ? void (0) : __assert_fail ("PartialExtensionReg && \"Expected to visit partial extension before full\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 1247, __extension__ __PRETTY_FUNCTION__))
1247	"Expected to visit partial extension before full")(static_cast <bool> (PartialExtensionReg && "Expected to visit partial extension before full" ) ? void (0) : __assert_fail ("PartialExtensionReg && \"Expected to visit partial extension before full\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 1247, __extension__ __PRETTY_FUNCTION__));
1248	if (FullExtensionReg) {
1249	DstRegs.push_back(FullExtensionReg);
1250	continue;
1251	}
1252	DstRegs.push_back(
1253	MIRBuilder.buildAShr(NarrowTy, PartialExtensionReg, AshrCstReg)
1254	.getReg(0));
1255	FullExtensionReg = DstRegs.back();
1256	} else {
1257	DstRegs.push_back(
1258	MIRBuilder
1259	.buildInstr(
1260	TargetOpcode::G_SEXT_INREG, {NarrowTy},
1261	{SrcRegs[i], SizeInBits % NarrowTy.getScalarSizeInBits()})
1262	.getReg(0));
1263	PartialExtensionReg = DstRegs.back();
1264	}
1265	}
1266
1267	// Gather the destination registers into the final destination.
1268	Register DstReg = MI.getOperand(0).getReg();
1269	MIRBuilder.buildMerge(DstReg, DstRegs);
1270	MI.eraseFromParent();
1271	return Legalized;
1272	}
1273	case TargetOpcode::G_BSWAP:
1274	case TargetOpcode::G_BITREVERSE: {
1275	if (SizeOp0 % NarrowSize != 0)
1276	return UnableToLegalize;
1277
1278	Observer.changingInstr(MI);
1279	SmallVector<Register, 2> SrcRegs, DstRegs;
1280	unsigned NumParts = SizeOp0 / NarrowSize;
1281	extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
1282
1283	for (unsigned i = 0; i < NumParts; ++i) {
1284	auto DstPart = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy},
1285	{SrcRegs[NumParts - 1 - i]});
1286	DstRegs.push_back(DstPart.getReg(0));
1287	}
1288
1289	MIRBuilder.buildMerge(MI.getOperand(0), DstRegs);
1290
1291	Observer.changedInstr(MI);
1292	MI.eraseFromParent();
1293	return Legalized;
1294	}
1295	case TargetOpcode::G_PTR_ADD:
1296	case TargetOpcode::G_PTRMASK: {
1297	if (TypeIdx != 1)
1298	return UnableToLegalize;
1299	Observer.changingInstr(MI);
1300	narrowScalarSrc(MI, NarrowTy, 2);
1301	Observer.changedInstr(MI);
1302	return Legalized;
1303	}
1304	case TargetOpcode::G_FPTOUI:
1305	case TargetOpcode::G_FPTOSI:
1306	return narrowScalarFPTOI(MI, TypeIdx, NarrowTy);
1307	case TargetOpcode::G_FPEXT:
1308	if (TypeIdx != 0)
1309	return UnableToLegalize;
1310	Observer.changingInstr(MI);
1311	narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_FPEXT);
1312	Observer.changedInstr(MI);
1313	return Legalized;
1314	}
1315	}
1316
1317	Register LegalizerHelper::coerceToScalar(Register Val) {
1318	LLT Ty = MRI.getType(Val);
1319	if (Ty.isScalar())
1320	return Val;
1321
1322	const DataLayout &DL = MIRBuilder.getDataLayout();
1323	LLT NewTy = LLT::scalar(Ty.getSizeInBits());
1324	if (Ty.isPointer()) {
1325	if (DL.isNonIntegralAddressSpace(Ty.getAddressSpace()))
1326	return Register();
1327	return MIRBuilder.buildPtrToInt(NewTy, Val).getReg(0);
1328	}
1329
1330	Register NewVal = Val;
1331
1332	assert(Ty.isVector())(static_cast <bool> (Ty.isVector()) ? void (0) : __assert_fail ("Ty.isVector()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 1332, __extension__ __PRETTY_FUNCTION__));
1333	LLT EltTy = Ty.getElementType();
1334	if (EltTy.isPointer())
1335	NewVal = MIRBuilder.buildPtrToInt(NewTy, NewVal).getReg(0);
1336	return MIRBuilder.buildBitcast(NewTy, NewVal).getReg(0);
1337	}
1338
1339	void LegalizerHelper::widenScalarSrc(MachineInstr &MI, LLT WideTy,
1340	unsigned OpIdx, unsigned ExtOpcode) {
1341	MachineOperand &MO = MI.getOperand(OpIdx);
1342	auto ExtB = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MO});
1343	MO.setReg(ExtB.getReg(0));
1344	}
1345
1346	void LegalizerHelper::narrowScalarSrc(MachineInstr &MI, LLT NarrowTy,
1347	unsigned OpIdx) {
1348	MachineOperand &MO = MI.getOperand(OpIdx);
1349	auto ExtB = MIRBuilder.buildTrunc(NarrowTy, MO);
1350	MO.setReg(ExtB.getReg(0));
1351	}
1352
1353	void LegalizerHelper::widenScalarDst(MachineInstr &MI, LLT WideTy,
1354	unsigned OpIdx, unsigned TruncOpcode) {
1355	MachineOperand &MO = MI.getOperand(OpIdx);
1356	Register DstExt = MRI.createGenericVirtualRegister(WideTy);
1357	MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1358	MIRBuilder.buildInstr(TruncOpcode, {MO}, {DstExt});
1359	MO.setReg(DstExt);
1360	}
1361
1362	void LegalizerHelper::narrowScalarDst(MachineInstr &MI, LLT NarrowTy,
1363	unsigned OpIdx, unsigned ExtOpcode) {
1364	MachineOperand &MO = MI.getOperand(OpIdx);
1365	Register DstTrunc = MRI.createGenericVirtualRegister(NarrowTy);
1366	MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1367	MIRBuilder.buildInstr(ExtOpcode, {MO}, {DstTrunc});
1368	MO.setReg(DstTrunc);
1369	}
1370
1371	void LegalizerHelper::moreElementsVectorDst(MachineInstr &MI, LLT WideTy,
1372	unsigned OpIdx) {
1373	MachineOperand &MO = MI.getOperand(OpIdx);
1374	MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1375	MO.setReg(widenWithUnmerge(WideTy, MO.getReg()));
1376	}
1377
1378	void LegalizerHelper::moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy,
1379	unsigned OpIdx) {
1380	MachineOperand &MO = MI.getOperand(OpIdx);
1381
1382	LLT OldTy = MRI.getType(MO.getReg());
1383	unsigned OldElts = OldTy.getNumElements();
1384	unsigned NewElts = MoreTy.getNumElements();
1385
1386	unsigned NumParts = NewElts / OldElts;
1387
1388	// Use concat_vectors if the result is a multiple of the number of elements.
1389	if (NumParts * OldElts == NewElts) {
1390	SmallVector<Register, 8> Parts;
1391	Parts.push_back(MO.getReg());
1392
1393	Register ImpDef = MIRBuilder.buildUndef(OldTy).getReg(0);
1394	for (unsigned I = 1; I != NumParts; ++I)
1395	Parts.push_back(ImpDef);
1396
1397	auto Concat = MIRBuilder.buildConcatVectors(MoreTy, Parts);
1398	MO.setReg(Concat.getReg(0));
1399	return;
1400	}
1401
1402	Register MoreReg = MRI.createGenericVirtualRegister(MoreTy);
1403	Register ImpDef = MIRBuilder.buildUndef(MoreTy).getReg(0);
1404	MIRBuilder.buildInsert(MoreReg, ImpDef, MO.getReg(), 0);
1405	MO.setReg(MoreReg);
1406	}
1407
1408	void LegalizerHelper::bitcastSrc(MachineInstr &MI, LLT CastTy, unsigned OpIdx) {
1409	MachineOperand &Op = MI.getOperand(OpIdx);
1410	Op.setReg(MIRBuilder.buildBitcast(CastTy, Op).getReg(0));
1411	}
1412
1413	void LegalizerHelper::bitcastDst(MachineInstr &MI, LLT CastTy, unsigned OpIdx) {
1414	MachineOperand &MO = MI.getOperand(OpIdx);
1415	Register CastDst = MRI.createGenericVirtualRegister(CastTy);
1416	MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1417	MIRBuilder.buildBitcast(MO, CastDst);
1418	MO.setReg(CastDst);
1419	}
1420
1421	LegalizerHelper::LegalizeResult
1422	LegalizerHelper::widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx,
1423	LLT WideTy) {
1424	if (TypeIdx != 1)
1425	return UnableToLegalize;
1426
1427	Register DstReg = MI.getOperand(0).getReg();
1428	LLT DstTy = MRI.getType(DstReg);
1429	if (DstTy.isVector())
1430	return UnableToLegalize;
1431
1432	Register Src1 = MI.getOperand(1).getReg();
1433	LLT SrcTy = MRI.getType(Src1);
1434	const int DstSize = DstTy.getSizeInBits();
1435	const int SrcSize = SrcTy.getSizeInBits();
1436	const int WideSize = WideTy.getSizeInBits();
1437	const int NumMerge = (DstSize + WideSize - 1) / WideSize;
1438
1439	unsigned NumOps = MI.getNumOperands();
1440	unsigned NumSrc = MI.getNumOperands() - 1;
1441	unsigned PartSize = DstTy.getSizeInBits() / NumSrc;
1442
1443	if (WideSize >= DstSize) {
1444	// Directly pack the bits in the target type.
1445	Register ResultReg = MIRBuilder.buildZExt(WideTy, Src1).getReg(0);
1446
1447	for (unsigned I = 2; I != NumOps; ++I) {
1448	const unsigned Offset = (I - 1) * PartSize;
1449
1450	Register SrcReg = MI.getOperand(I).getReg();
1451	assert(MRI.getType(SrcReg) == LLT::scalar(PartSize))(static_cast <bool> (MRI.getType(SrcReg) == LLT::scalar (PartSize)) ? void (0) : __assert_fail ("MRI.getType(SrcReg) == LLT::scalar(PartSize)" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 1451, __extension__ __PRETTY_FUNCTION__));
1452
1453	auto ZextInput = MIRBuilder.buildZExt(WideTy, SrcReg);
1454
1455	Register NextResult = I + 1 == NumOps && WideTy == DstTy ? DstReg :
1456	MRI.createGenericVirtualRegister(WideTy);
1457
1458	auto ShiftAmt = MIRBuilder.buildConstant(WideTy, Offset);
1459	auto Shl = MIRBuilder.buildShl(WideTy, ZextInput, ShiftAmt);
1460	MIRBuilder.buildOr(NextResult, ResultReg, Shl);
1461	ResultReg = NextResult;
1462	}
1463
1464	if (WideSize > DstSize)
1465	MIRBuilder.buildTrunc(DstReg, ResultReg);
1466	else if (DstTy.isPointer())
1467	MIRBuilder.buildIntToPtr(DstReg, ResultReg);
1468
1469	MI.eraseFromParent();
1470	return Legalized;
1471	}
1472
1473	// Unmerge the original values to the GCD type, and recombine to the next
1474	// multiple greater than the original type.
1475	//
1476	// %3:_(s12) = G_MERGE_VALUES %0:_(s4), %1:_(s4), %2:_(s4) -> s6
1477	// %4:_(s2), %5:_(s2) = G_UNMERGE_VALUES %0
1478	// %6:_(s2), %7:_(s2) = G_UNMERGE_VALUES %1
1479	// %8:_(s2), %9:_(s2) = G_UNMERGE_VALUES %2
1480	// %10:_(s6) = G_MERGE_VALUES %4, %5, %6
1481	// %11:_(s6) = G_MERGE_VALUES %7, %8, %9
1482	// %12:_(s12) = G_MERGE_VALUES %10, %11
1483	//
1484	// Padding with undef if necessary:
1485	//
1486	// %2:_(s8) = G_MERGE_VALUES %0:_(s4), %1:_(s4) -> s6
1487	// %3:_(s2), %4:_(s2) = G_UNMERGE_VALUES %0
1488	// %5:_(s2), %6:_(s2) = G_UNMERGE_VALUES %1
1489	// %7:_(s2) = G_IMPLICIT_DEF
1490	// %8:_(s6) = G_MERGE_VALUES %3, %4, %5
1491	// %9:_(s6) = G_MERGE_VALUES %6, %7, %7
1492	// %10:_(s12) = G_MERGE_VALUES %8, %9
1493
1494	const int GCD = greatestCommonDivisor(SrcSize, WideSize);
1495	LLT GCDTy = LLT::scalar(GCD);
1496
1497	SmallVector<Register, 8> Parts;
1498	SmallVector<Register, 8> NewMergeRegs;
1499	SmallVector<Register, 8> Unmerges;
1500	LLT WideDstTy = LLT::scalar(NumMerge * WideSize);
1501
1502	// Decompose the original operands if they don't evenly divide.
1503	for (int I = 1, E = MI.getNumOperands(); I != E; ++I) {
1504	Register SrcReg = MI.getOperand(I).getReg();
1505	if (GCD == SrcSize) {
1506	Unmerges.push_back(SrcReg);
1507	} else {
1508	auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg);
1509	for (int J = 0, JE = Unmerge->getNumOperands() - 1; J != JE; ++J)
1510	Unmerges.push_back(Unmerge.getReg(J));
1511	}
1512	}
1513
1514	// Pad with undef to the next size that is a multiple of the requested size.
1515	if (static_cast<int>(Unmerges.size()) != NumMerge * WideSize) {
1516	Register UndefReg = MIRBuilder.buildUndef(GCDTy).getReg(0);
1517	for (int I = Unmerges.size(); I != NumMerge * WideSize; ++I)
1518	Unmerges.push_back(UndefReg);
1519	}
1520
1521	const int PartsPerGCD = WideSize / GCD;
1522
1523	// Build merges of each piece.
1524	ArrayRef<Register> Slicer(Unmerges);
1525	for (int I = 0; I != NumMerge; ++I, Slicer = Slicer.drop_front(PartsPerGCD)) {
1526	auto Merge = MIRBuilder.buildMerge(WideTy, Slicer.take_front(PartsPerGCD));
1527	NewMergeRegs.push_back(Merge.getReg(0));
1528	}
1529
1530	// A truncate may be necessary if the requested type doesn't evenly divide the
1531	// original result type.
1532	if (DstTy.getSizeInBits() == WideDstTy.getSizeInBits()) {
1533	MIRBuilder.buildMerge(DstReg, NewMergeRegs);
1534	} else {
1535	auto FinalMerge = MIRBuilder.buildMerge(WideDstTy, NewMergeRegs);
1536	MIRBuilder.buildTrunc(DstReg, FinalMerge.getReg(0));
1537	}
1538
1539	MI.eraseFromParent();
1540	return Legalized;
1541	}
1542
1543	Register LegalizerHelper::widenWithUnmerge(LLT WideTy, Register OrigReg) {
1544	Register WideReg = MRI.createGenericVirtualRegister(WideTy);
1545	LLT OrigTy = MRI.getType(OrigReg);
1546	LLT LCMTy = getLCMType(WideTy, OrigTy);
1547
1548	const int NumMergeParts = LCMTy.getSizeInBits() / WideTy.getSizeInBits();
1549	const int NumUnmergeParts = LCMTy.getSizeInBits() / OrigTy.getSizeInBits();
1550
1551	Register UnmergeSrc = WideReg;
1552
1553	// Create a merge to the LCM type, padding with undef
1554	// %0:_(<3 x s32>) = G_FOO => <4 x s32>
1555	// =>
1556	// %1:_(<4 x s32>) = G_FOO
1557	// %2:_(<4 x s32>) = G_IMPLICIT_DEF
1558	// %3:_(<12 x s32>) = G_CONCAT_VECTORS %1, %2, %2
1559	// %0:_(<3 x s32>), %4:_, %5:_, %6:_ = G_UNMERGE_VALUES %3
1560	if (NumMergeParts > 1) {
1561	Register Undef = MIRBuilder.buildUndef(WideTy).getReg(0);
1562	SmallVector<Register, 8> MergeParts(NumMergeParts, Undef);
1563	MergeParts[0] = WideReg;
1564	UnmergeSrc = MIRBuilder.buildMerge(LCMTy, MergeParts).getReg(0);
1565	}
1566
1567	// Unmerge to the original register and pad with dead defs.
1568	SmallVector<Register, 8> UnmergeResults(NumUnmergeParts);
1569	UnmergeResults[0] = OrigReg;
1570	for (int I = 1; I != NumUnmergeParts; ++I)
1571	UnmergeResults[I] = MRI.createGenericVirtualRegister(OrigTy);
1572
1573	MIRBuilder.buildUnmerge(UnmergeResults, UnmergeSrc);
1574	return WideReg;
1575	}
1576
1577	LegalizerHelper::LegalizeResult
1578	LegalizerHelper::widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx,
1579	LLT WideTy) {
1580	if (TypeIdx != 0)
1581	return UnableToLegalize;
1582
1583	int NumDst = MI.getNumOperands() - 1;
1584	Register SrcReg = MI.getOperand(NumDst).getReg();
1585	LLT SrcTy = MRI.getType(SrcReg);
1586	if (SrcTy.isVector())
1587	return UnableToLegalize;
1588
1589	Register Dst0Reg = MI.getOperand(0).getReg();
1590	LLT DstTy = MRI.getType(Dst0Reg);
1591	if (!DstTy.isScalar())
1592	return UnableToLegalize;
1593
1594	if (WideTy.getSizeInBits() >= SrcTy.getSizeInBits()) {
1595	if (SrcTy.isPointer()) {
1596	const DataLayout &DL = MIRBuilder.getDataLayout();
1597	if (DL.isNonIntegralAddressSpace(SrcTy.getAddressSpace())) {
1598	LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizer")) { dbgs() << "Not casting non-integral address space integer\n" ; } } while (false)
1599	dbgs() << "Not casting non-integral address space integer\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizer")) { dbgs() << "Not casting non-integral address space integer\n" ; } } while (false);
1600	return UnableToLegalize;
1601	}
1602
1603	SrcTy = LLT::scalar(SrcTy.getSizeInBits());
1604	SrcReg = MIRBuilder.buildPtrToInt(SrcTy, SrcReg).getReg(0);
1605	}
1606
1607	// Widen SrcTy to WideTy. This does not affect the result, but since the
1608	// user requested this size, it is probably better handled than SrcTy and
1609	// should reduce the total number of legalization artifacts
1610	if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) {
1611	SrcTy = WideTy;
1612	SrcReg = MIRBuilder.buildAnyExt(WideTy, SrcReg).getReg(0);
1613	}
1614
1615	// Theres no unmerge type to target. Directly extract the bits from the
1616	// source type
1617	unsigned DstSize = DstTy.getSizeInBits();
1618
1619	MIRBuilder.buildTrunc(Dst0Reg, SrcReg);
1620	for (int I = 1; I != NumDst; ++I) {
1621	auto ShiftAmt = MIRBuilder.buildConstant(SrcTy, DstSize * I);
1622	auto Shr = MIRBuilder.buildLShr(SrcTy, SrcReg, ShiftAmt);
1623	MIRBuilder.buildTrunc(MI.getOperand(I), Shr);
1624	}
1625
1626	MI.eraseFromParent();
1627	return Legalized;
1628	}
1629
1630	// Extend the source to a wider type.
1631	LLT LCMTy = getLCMType(SrcTy, WideTy);
1632
1633	Register WideSrc = SrcReg;
1634	if (LCMTy.getSizeInBits() != SrcTy.getSizeInBits()) {
1635	// TODO: If this is an integral address space, cast to integer and anyext.
1636	if (SrcTy.isPointer()) {
1637	LLVM_DEBUG(dbgs() << "Widening pointer source types not implemented\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizer")) { dbgs() << "Widening pointer source types not implemented\n" ; } } while (false);
1638	return UnableToLegalize;
1639	}
1640
1641	WideSrc = MIRBuilder.buildAnyExt(LCMTy, WideSrc).getReg(0);
1642	}
1643
1644	auto Unmerge = MIRBuilder.buildUnmerge(WideTy, WideSrc);
1645
1646	// Create a sequence of unmerges and merges to the original results. Since we
1647	// may have widened the source, we will need to pad the results with dead defs
1648	// to cover the source register.
1649	// e.g. widen s48 to s64:
1650	// %1:_(s48), %2:_(s48) = G_UNMERGE_VALUES %0:_(s96)
1651	//
1652	// =>
1653	// %4:_(s192) = G_ANYEXT %0:_(s96)
1654	// %5:_(s64), %6, %7 = G_UNMERGE_VALUES %4 ; Requested unmerge
1655	// ; unpack to GCD type, with extra dead defs
1656	// %8:_(s16), %9, %10, %11 = G_UNMERGE_VALUES %5:_(s64)
1657	// %12:_(s16), %13, dead %14, dead %15 = G_UNMERGE_VALUES %6:_(s64)
1658	// dead %16:_(s16), dead %17, dead %18, dead %18 = G_UNMERGE_VALUES %7:_(s64)
1659	// %1:_(s48) = G_MERGE_VALUES %8:_(s16), %9, %10 ; Remerge to destination
1660	// %2:_(s48) = G_MERGE_VALUES %11:_(s16), %12, %13 ; Remerge to destination
1661	const LLT GCDTy = getGCDType(WideTy, DstTy);
1662	const int NumUnmerge = Unmerge->getNumOperands() - 1;
1663	const int PartsPerRemerge = DstTy.getSizeInBits() / GCDTy.getSizeInBits();
1664
1665	// Directly unmerge to the destination without going through a GCD type
1666	// if possible
1667	if (PartsPerRemerge == 1) {
1668	const int PartsPerUnmerge = WideTy.getSizeInBits() / DstTy.getSizeInBits();
1669
1670	for (int I = 0; I != NumUnmerge; ++I) {
1671	auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES);
1672
1673	for (int J = 0; J != PartsPerUnmerge; ++J) {
1674	int Idx = I * PartsPerUnmerge + J;
1675	if (Idx < NumDst)
1676	MIB.addDef(MI.getOperand(Idx).getReg());
1677	else {
1678	// Create dead def for excess components.
1679	MIB.addDef(MRI.createGenericVirtualRegister(DstTy));
1680	}
1681	}
1682
1683	MIB.addUse(Unmerge.getReg(I));
1684	}
1685	} else {
1686	SmallVector<Register, 16> Parts;
1687	for (int J = 0; J != NumUnmerge; ++J)
1688	extractGCDType(Parts, GCDTy, Unmerge.getReg(J));
1689
1690	SmallVector<Register, 8> RemergeParts;
1691	for (int I = 0; I != NumDst; ++I) {
1692	for (int J = 0; J < PartsPerRemerge; ++J) {
1693	const int Idx = I * PartsPerRemerge + J;
1694	RemergeParts.emplace_back(Parts[Idx]);
1695	}
1696
1697	MIRBuilder.buildMerge(MI.getOperand(I).getReg(), RemergeParts);
1698	RemergeParts.clear();
1699	}
1700	}
1701
1702	MI.eraseFromParent();
1703	return Legalized;
1704	}
1705
1706	LegalizerHelper::LegalizeResult
1707	LegalizerHelper::widenScalarExtract(MachineInstr &MI, unsigned TypeIdx,
1708	LLT WideTy) {
1709	Register DstReg = MI.getOperand(0).getReg();
1710	Register SrcReg = MI.getOperand(1).getReg();
1711	LLT SrcTy = MRI.getType(SrcReg);
1712
1713	LLT DstTy = MRI.getType(DstReg);
1714	unsigned Offset = MI.getOperand(2).getImm();
1715
1716	if (TypeIdx == 0) {
1717	if (SrcTy.isVector() \|\| DstTy.isVector())
1718	return UnableToLegalize;
1719
1720	SrcOp Src(SrcReg);
1721	if (SrcTy.isPointer()) {
1722	// Extracts from pointers can be handled only if they are really just
1723	// simple integers.
1724	const DataLayout &DL = MIRBuilder.getDataLayout();
1725	if (DL.isNonIntegralAddressSpace(SrcTy.getAddressSpace()))
1726	return UnableToLegalize;
1727
1728	LLT SrcAsIntTy = LLT::scalar(SrcTy.getSizeInBits());
1729	Src = MIRBuilder.buildPtrToInt(SrcAsIntTy, Src);
1730	SrcTy = SrcAsIntTy;
1731	}
1732
1733	if (DstTy.isPointer())
1734	return UnableToLegalize;
1735
1736	if (Offset == 0) {
1737	// Avoid a shift in the degenerate case.
1738	MIRBuilder.buildTrunc(DstReg,
1739	MIRBuilder.buildAnyExtOrTrunc(WideTy, Src));
1740	MI.eraseFromParent();
1741	return Legalized;
1742	}
1743
1744	// Do a shift in the source type.
1745	LLT ShiftTy = SrcTy;
1746	if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) {
1747	Src = MIRBuilder.buildAnyExt(WideTy, Src);
1748	ShiftTy = WideTy;
1749	}
1750
1751	auto LShr = MIRBuilder.buildLShr(
1752	ShiftTy, Src, MIRBuilder.buildConstant(ShiftTy, Offset));
1753	MIRBuilder.buildTrunc(DstReg, LShr);
1754	MI.eraseFromParent();
1755	return Legalized;
1756	}
1757
1758	if (SrcTy.isScalar()) {
1759	Observer.changingInstr(MI);
1760	widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1761	Observer.changedInstr(MI);
1762	return Legalized;
1763	}
1764
1765	if (!SrcTy.isVector())
1766	return UnableToLegalize;
1767
1768	if (DstTy != SrcTy.getElementType())
1769	return UnableToLegalize;
1770
1771	if (Offset % SrcTy.getScalarSizeInBits() != 0)
1772	return UnableToLegalize;
1773
1774	Observer.changingInstr(MI);
1775	widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1776
1777	MI.getOperand(2).setImm((WideTy.getSizeInBits() / SrcTy.getSizeInBits()) *
1778	Offset);
1779	widenScalarDst(MI, WideTy.getScalarType(), 0);
1780	Observer.changedInstr(MI);
1781	return Legalized;
1782	}
1783
1784	LegalizerHelper::LegalizeResult
1785	LegalizerHelper::widenScalarInsert(MachineInstr &MI, unsigned TypeIdx,
1786	LLT WideTy) {
1787	if (TypeIdx != 0 \|\| WideTy.isVector())
1788	return UnableToLegalize;
1789	Observer.changingInstr(MI);
1790	widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1791	widenScalarDst(MI, WideTy);
1792	Observer.changedInstr(MI);
1793	return Legalized;
1794	}
1795
1796	LegalizerHelper::LegalizeResult
1797	LegalizerHelper::widenScalarAddSubOverflow(MachineInstr &MI, unsigned TypeIdx,
1798	LLT WideTy) {
1799	if (TypeIdx == 1)
1800	return UnableToLegalize; // TODO
1801
1802	unsigned Opcode;
1803	unsigned ExtOpcode;
1804	Optional<Register> CarryIn = None;
1805	switch (MI.getOpcode()) {
1806	default:
1807	llvm_unreachable("Unexpected opcode!")::llvm::llvm_unreachable_internal("Unexpected opcode!", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 1807);
1808	case TargetOpcode::G_SADDO:
1809	Opcode = TargetOpcode::G_ADD;
1810	ExtOpcode = TargetOpcode::G_SEXT;
1811	break;
1812	case TargetOpcode::G_SSUBO:
1813	Opcode = TargetOpcode::G_SUB;
1814	ExtOpcode = TargetOpcode::G_SEXT;
1815	break;
1816	case TargetOpcode::G_UADDO:
1817	Opcode = TargetOpcode::G_ADD;
1818	ExtOpcode = TargetOpcode::G_ZEXT;
1819	break;
1820	case TargetOpcode::G_USUBO:
1821	Opcode = TargetOpcode::G_SUB;
1822	ExtOpcode = TargetOpcode::G_ZEXT;
1823	break;
1824	case TargetOpcode::G_SADDE:
1825	Opcode = TargetOpcode::G_UADDE;
1826	ExtOpcode = TargetOpcode::G_SEXT;
1827	CarryIn = MI.getOperand(4).getReg();
1828	break;
1829	case TargetOpcode::G_SSUBE:
1830	Opcode = TargetOpcode::G_USUBE;
1831	ExtOpcode = TargetOpcode::G_SEXT;
1832	CarryIn = MI.getOperand(4).getReg();
1833	break;
1834	case TargetOpcode::G_UADDE:
1835	Opcode = TargetOpcode::G_UADDE;
1836	ExtOpcode = TargetOpcode::G_ZEXT;
1837	CarryIn = MI.getOperand(4).getReg();
1838	break;
1839	case TargetOpcode::G_USUBE:
1840	Opcode = TargetOpcode::G_USUBE;
1841	ExtOpcode = TargetOpcode::G_ZEXT;
1842	CarryIn = MI.getOperand(4).getReg();
1843	break;
1844	}
1845
1846	auto LHSExt = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MI.getOperand(2)});
1847	auto RHSExt = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MI.getOperand(3)});
1848	// Do the arithmetic in the larger type.
1849	Register NewOp;
1850	if (CarryIn) {
1851	LLT CarryOutTy = MRI.getType(MI.getOperand(1).getReg());
1852	NewOp = MIRBuilder
1853	.buildInstr(Opcode, {WideTy, CarryOutTy},
1854	{LHSExt, RHSExt, *CarryIn})
1855	.getReg(0);
1856	} else {
1857	NewOp = MIRBuilder.buildInstr(Opcode, {WideTy}, {LHSExt, RHSExt}).getReg(0);
1858	}
1859	LLT OrigTy = MRI.getType(MI.getOperand(0).getReg());
1860	auto TruncOp = MIRBuilder.buildTrunc(OrigTy, NewOp);
1861	auto ExtOp = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {TruncOp});
1862	// There is no overflow if the ExtOp is the same as NewOp.
1863	MIRBuilder.buildICmp(CmpInst::ICMP_NE, MI.getOperand(1), NewOp, ExtOp);
1864	// Now trunc the NewOp to the original result.
1865	MIRBuilder.buildTrunc(MI.getOperand(0), NewOp);
1866	MI.eraseFromParent();
1867	return Legalized;
1868	}
1869
1870	LegalizerHelper::LegalizeResult
1871	LegalizerHelper::widenScalarAddSubShlSat(MachineInstr &MI, unsigned TypeIdx,
1872	LLT WideTy) {
1873	bool IsSigned = MI.getOpcode() == TargetOpcode::G_SADDSAT \|\|
1874	MI.getOpcode() == TargetOpcode::G_SSUBSAT \|\|
1875	MI.getOpcode() == TargetOpcode::G_SSHLSAT;
1876	bool IsShift = MI.getOpcode() == TargetOpcode::G_SSHLSAT \|\|
1877	MI.getOpcode() == TargetOpcode::G_USHLSAT;
1878	// We can convert this to:
1879	// 1. Any extend iN to iM
1880	// 2. SHL by M-N
1881	// 3. [US][ADD\|SUB\|SHL]SAT
1882	// 4. L/ASHR by M-N
1883	//
1884	// It may be more efficient to lower this to a min and a max operation in
1885	// the higher precision arithmetic if the promoted operation isn't legal,
1886	// but this decision is up to the target's lowering request.
1887	Register DstReg = MI.getOperand(0).getReg();
1888
1889	unsigned NewBits = WideTy.getScalarSizeInBits();
1890	unsigned SHLAmount = NewBits - MRI.getType(DstReg).getScalarSizeInBits();
1891
1892	// Shifts must zero-extend the RHS to preserve the unsigned quantity, and
1893	// must not left shift the RHS to preserve the shift amount.
1894	auto LHS = MIRBuilder.buildAnyExt(WideTy, MI.getOperand(1));
1895	auto RHS = IsShift ? MIRBuilder.buildZExt(WideTy, MI.getOperand(2))
1896	: MIRBuilder.buildAnyExt(WideTy, MI.getOperand(2));
1897	auto ShiftK = MIRBuilder.buildConstant(WideTy, SHLAmount);
1898	auto ShiftL = MIRBuilder.buildShl(WideTy, LHS, ShiftK);
1899	auto ShiftR = IsShift ? RHS : MIRBuilder.buildShl(WideTy, RHS, ShiftK);
1900
1901	auto WideInst = MIRBuilder.buildInstr(MI.getOpcode(), {WideTy},
1902	{ShiftL, ShiftR}, MI.getFlags());
1903
1904	// Use a shift that will preserve the number of sign bits when the trunc is
1905	// folded away.
1906	auto Result = IsSigned ? MIRBuilder.buildAShr(WideTy, WideInst, ShiftK)
1907	: MIRBuilder.buildLShr(WideTy, WideInst, ShiftK);
1908
1909	MIRBuilder.buildTrunc(DstReg, Result);
1910	MI.eraseFromParent();
1911	return Legalized;
1912	}
1913
1914	LegalizerHelper::LegalizeResult
1915	LegalizerHelper::widenScalarMulo(MachineInstr &MI, unsigned TypeIdx,
1916	LLT WideTy) {
1917	if (TypeIdx == 1)
1918	return UnableToLegalize;
1919
1920	bool IsSigned = MI.getOpcode() == TargetOpcode::G_SMULO;
1921	Register Result = MI.getOperand(0).getReg();
1922	Register OriginalOverflow = MI.getOperand(1).getReg();
1923	Register LHS = MI.getOperand(2).getReg();
1924	Register RHS = MI.getOperand(3).getReg();
1925	LLT SrcTy = MRI.getType(LHS);
1926	LLT OverflowTy = MRI.getType(OriginalOverflow);
1927	unsigned SrcBitWidth = SrcTy.getScalarSizeInBits();
1928
1929	// To determine if the result overflowed in the larger type, we extend the
1930	// input to the larger type, do the multiply (checking if it overflows),
1931	// then also check the high bits of the result to see if overflow happened
1932	// there.
1933	unsigned ExtOp = IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
1934	auto LeftOperand = MIRBuilder.buildInstr(ExtOp, {WideTy}, {LHS});
1935	auto RightOperand = MIRBuilder.buildInstr(ExtOp, {WideTy}, {RHS});
1936
1937	auto Mulo = MIRBuilder.buildInstr(MI.getOpcode(), {WideTy, OverflowTy},
1938	{LeftOperand, RightOperand});
1939	auto Mul = Mulo->getOperand(0);
1940	MIRBuilder.buildTrunc(Result, Mul);
1941
1942	MachineInstrBuilder ExtResult;
1943	// Overflow occurred if it occurred in the larger type, or if the high part
1944	// of the result does not zero/sign-extend the low part. Check this second
1945	// possibility first.
1946	if (IsSigned) {
1947	// For signed, overflow occurred when the high part does not sign-extend
1948	// the low part.
1949	ExtResult = MIRBuilder.buildSExtInReg(WideTy, Mul, SrcBitWidth);
1950	} else {
1951	// Unsigned overflow occurred when the high part does not zero-extend the
1952	// low part.
1953	ExtResult = MIRBuilder.buildZExtInReg(WideTy, Mul, SrcBitWidth);
1954	}
1955
1956	// Multiplication cannot overflow if the WideTy is >= 2 * original width,
1957	// so we don't need to check the overflow result of larger type Mulo.
1958	if (WideTy.getScalarSizeInBits() < 2 * SrcBitWidth) {
1959	auto Overflow =
1960	MIRBuilder.buildICmp(CmpInst::ICMP_NE, OverflowTy, Mul, ExtResult);
1961	// Finally check if the multiplication in the larger type itself overflowed.
1962	MIRBuilder.buildOr(OriginalOverflow, Mulo->getOperand(1), Overflow);
1963	} else {
1964	MIRBuilder.buildICmp(CmpInst::ICMP_NE, OriginalOverflow, Mul, ExtResult);
1965	}
1966	MI.eraseFromParent();
1967	return Legalized;
1968	}
1969
1970	LegalizerHelper::LegalizeResult
1971	LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
1972	switch (MI.getOpcode()) {
1973	default:
1974	return UnableToLegalize;
1975	case TargetOpcode::G_ATOMICRMW_XCHG:
1976	case TargetOpcode::G_ATOMICRMW_ADD:
1977	case TargetOpcode::G_ATOMICRMW_SUB:
1978	case TargetOpcode::G_ATOMICRMW_AND:
1979	case TargetOpcode::G_ATOMICRMW_OR:
1980	case TargetOpcode::G_ATOMICRMW_XOR:
1981	case TargetOpcode::G_ATOMICRMW_MIN:
1982	case TargetOpcode::G_ATOMICRMW_MAX:
1983	case TargetOpcode::G_ATOMICRMW_UMIN:
1984	case TargetOpcode::G_ATOMICRMW_UMAX:
1985	assert(TypeIdx == 0 && "atomicrmw with second scalar type")(static_cast <bool> (TypeIdx == 0 && "atomicrmw with second scalar type" ) ? void (0) : __assert_fail ("TypeIdx == 0 && \"atomicrmw with second scalar type\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 1985, __extension__ __PRETTY_FUNCTION__));
1986	Observer.changingInstr(MI);
1987	widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
1988	widenScalarDst(MI, WideTy, 0);
1989	Observer.changedInstr(MI);
1990	return Legalized;
1991	case TargetOpcode::G_ATOMIC_CMPXCHG:
1992	assert(TypeIdx == 0 && "G_ATOMIC_CMPXCHG with second scalar type")(static_cast <bool> (TypeIdx == 0 && "G_ATOMIC_CMPXCHG with second scalar type" ) ? void (0) : __assert_fail ("TypeIdx == 0 && \"G_ATOMIC_CMPXCHG with second scalar type\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 1992, __extension__ __PRETTY_FUNCTION__));
1993	Observer.changingInstr(MI);
1994	widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
1995	widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ANYEXT);
1996	widenScalarDst(MI, WideTy, 0);
1997	Observer.changedInstr(MI);
1998	return Legalized;
1999	case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS:
2000	if (TypeIdx == 0) {
2001	Observer.changingInstr(MI);
2002	widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ANYEXT);
2003	widenScalarSrc(MI, WideTy, 4, TargetOpcode::G_ANYEXT);
2004	widenScalarDst(MI, WideTy, 0);
2005	Observer.changedInstr(MI);
2006	return Legalized;
2007	}
2008	assert(TypeIdx == 1 &&(static_cast <bool> (TypeIdx == 1 && "G_ATOMIC_CMPXCHG_WITH_SUCCESS with third scalar type" ) ? void (0) : __assert_fail ("TypeIdx == 1 && \"G_ATOMIC_CMPXCHG_WITH_SUCCESS with third scalar type\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 2009, __extension__ __PRETTY_FUNCTION__))
2009	"G_ATOMIC_CMPXCHG_WITH_SUCCESS with third scalar type")(static_cast <bool> (TypeIdx == 1 && "G_ATOMIC_CMPXCHG_WITH_SUCCESS with third scalar type" ) ? void (0) : __assert_fail ("TypeIdx == 1 && \"G_ATOMIC_CMPXCHG_WITH_SUCCESS with third scalar type\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 2009, __extension__ __PRETTY_FUNCTION__));
2010	Observer.changingInstr(MI);
2011	widenScalarDst(MI, WideTy, 1);
2012	Observer.changedInstr(MI);
2013	return Legalized;
2014	case TargetOpcode::G_EXTRACT:
2015	return widenScalarExtract(MI, TypeIdx, WideTy);
2016	case TargetOpcode::G_INSERT:
2017	return widenScalarInsert(MI, TypeIdx, WideTy);
2018	case TargetOpcode::G_MERGE_VALUES:
2019	return widenScalarMergeValues(MI, TypeIdx, WideTy);
2020	case TargetOpcode::G_UNMERGE_VALUES:
2021	return widenScalarUnmergeValues(MI, TypeIdx, WideTy);
2022	case TargetOpcode::G_SADDO:
2023	case TargetOpcode::G_SSUBO:
2024	case TargetOpcode::G_UADDO:
2025	case TargetOpcode::G_USUBO:
2026	case TargetOpcode::G_SADDE:
2027	case TargetOpcode::G_SSUBE:
2028	case TargetOpcode::G_UADDE:
2029	case TargetOpcode::G_USUBE:
2030	return widenScalarAddSubOverflow(MI, TypeIdx, WideTy);
2031	case TargetOpcode::G_UMULO:
2032	case TargetOpcode::G_SMULO:
2033	return widenScalarMulo(MI, TypeIdx, WideTy);
2034	case TargetOpcode::G_SADDSAT:
2035	case TargetOpcode::G_SSUBSAT:
2036	case TargetOpcode::G_SSHLSAT:
2037	case TargetOpcode::G_UADDSAT:
2038	case TargetOpcode::G_USUBSAT:
2039	case TargetOpcode::G_USHLSAT:
2040	return widenScalarAddSubShlSat(MI, TypeIdx, WideTy);
2041	case TargetOpcode::G_CTTZ:
2042	case TargetOpcode::G_CTTZ_ZERO_UNDEF:
2043	case TargetOpcode::G_CTLZ:
2044	case TargetOpcode::G_CTLZ_ZERO_UNDEF:
2045	case TargetOpcode::G_CTPOP: {
2046	if (TypeIdx == 0) {
2047	Observer.changingInstr(MI);
2048	widenScalarDst(MI, WideTy, 0);
2049	Observer.changedInstr(MI);
2050	return Legalized;
2051	}
2052
2053	Register SrcReg = MI.getOperand(1).getReg();
2054
2055	// First extend the input.
2056	unsigned ExtOpc = MI.getOpcode() == TargetOpcode::G_CTTZ \|\|
2057	MI.getOpcode() == TargetOpcode::G_CTTZ_ZERO_UNDEF
2058	? TargetOpcode::G_ANYEXT
2059	: TargetOpcode::G_ZEXT;
2060	auto MIBSrc = MIRBuilder.buildInstr(ExtOpc, {WideTy}, {SrcReg});
2061	LLT CurTy = MRI.getType(SrcReg);
2062	unsigned NewOpc = MI.getOpcode();
2063	if (NewOpc == TargetOpcode::G_CTTZ) {
2064	// The count is the same in the larger type except if the original
2065	// value was zero. This can be handled by setting the bit just off
2066	// the top of the original type.
2067	auto TopBit =
2068	APInt::getOneBitSet(WideTy.getSizeInBits(), CurTy.getSizeInBits());
2069	MIBSrc = MIRBuilder.buildOr(
2070	WideTy, MIBSrc, MIRBuilder.buildConstant(WideTy, TopBit));
2071	// Now we know the operand is non-zero, use the more relaxed opcode.
2072	NewOpc = TargetOpcode::G_CTTZ_ZERO_UNDEF;
2073	}
2074
2075	// Perform the operation at the larger size.
2076	auto MIBNewOp = MIRBuilder.buildInstr(NewOpc, {WideTy}, {MIBSrc});
2077	// This is already the correct result for CTPOP and CTTZs
2078	if (MI.getOpcode() == TargetOpcode::G_CTLZ \|\|
2079	MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF) {
2080	// The correct result is NewOp - (Difference in widety and current ty).
2081	unsigned SizeDiff = WideTy.getSizeInBits() - CurTy.getSizeInBits();
2082	MIBNewOp = MIRBuilder.buildSub(
2083	WideTy, MIBNewOp, MIRBuilder.buildConstant(WideTy, SizeDiff));
2084	}
2085
2086	MIRBuilder.buildZExtOrTrunc(MI.getOperand(0), MIBNewOp);
2087	MI.eraseFromParent();
2088	return Legalized;
2089	}
2090	case TargetOpcode::G_BSWAP: {
2091	Observer.changingInstr(MI);
2092	Register DstReg = MI.getOperand(0).getReg();
2093
2094	Register ShrReg = MRI.createGenericVirtualRegister(WideTy);
2095	Register DstExt = MRI.createGenericVirtualRegister(WideTy);
2096	Register ShiftAmtReg = MRI.createGenericVirtualRegister(WideTy);
2097	widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
2098
2099	MI.getOperand(0).setReg(DstExt);
2100
2101	MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
2102
2103	LLT Ty = MRI.getType(DstReg);
2104	unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits();
2105	MIRBuilder.buildConstant(ShiftAmtReg, DiffBits);
2106	MIRBuilder.buildLShr(ShrReg, DstExt, ShiftAmtReg);
2107
2108	MIRBuilder.buildTrunc(DstReg, ShrReg);
2109	Observer.changedInstr(MI);
2110	return Legalized;
2111	}
2112	case TargetOpcode::G_BITREVERSE: {
2113	Observer.changingInstr(MI);
2114
2115	Register DstReg = MI.getOperand(0).getReg();
2116	LLT Ty = MRI.getType(DstReg);
2117	unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits();
2118
2119	Register DstExt = MRI.createGenericVirtualRegister(WideTy);
2120	widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
2121	MI.getOperand(0).setReg(DstExt);
2122	MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
2123
2124	auto ShiftAmt = MIRBuilder.buildConstant(WideTy, DiffBits);
2125	auto Shift = MIRBuilder.buildLShr(WideTy, DstExt, ShiftAmt);
2126	MIRBuilder.buildTrunc(DstReg, Shift);
2127	Observer.changedInstr(MI);
2128	return Legalized;
2129	}
2130	case TargetOpcode::G_FREEZE:
2131	Observer.changingInstr(MI);
2132	widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
2133	widenScalarDst(MI, WideTy);
2134	Observer.changedInstr(MI);
2135	return Legalized;
2136
2137	case TargetOpcode::G_ABS:
2138	Observer.changingInstr(MI);
2139	widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
2140	widenScalarDst(MI, WideTy);
2141	Observer.changedInstr(MI);
2142	return Legalized;
2143
2144	case TargetOpcode::G_ADD:
2145	case TargetOpcode::G_AND:
2146	case TargetOpcode::G_MUL:
2147	case TargetOpcode::G_OR:
2148	case TargetOpcode::G_XOR:
2149	case TargetOpcode::G_SUB:
2150	// Perform operation at larger width (any extension is fines here, high bits
2151	// don't affect the result) and then truncate the result back to the
2152	// original type.
2153	Observer.changingInstr(MI);
2154	widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
2155	widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
2156	widenScalarDst(MI, WideTy);
2157	Observer.changedInstr(MI);
2158	return Legalized;
2159
2160	case TargetOpcode::G_SBFX:
2161	case TargetOpcode::G_UBFX:
2162	Observer.changingInstr(MI);
2163
2164	if (TypeIdx == 0) {
2165	widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
2166	widenScalarDst(MI, WideTy);
2167	} else {
2168	widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
2169	widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ZEXT);
2170	}
2171
2172	Observer.changedInstr(MI);
2173	return Legalized;
2174
2175	case TargetOpcode::G_SHL:
2176	Observer.changingInstr(MI);
2177
2178	if (TypeIdx == 0) {
2179	widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
2180	widenScalarDst(MI, WideTy);
2181	} else {
2182	assert(TypeIdx == 1)(static_cast <bool> (TypeIdx == 1) ? void (0) : __assert_fail ("TypeIdx == 1", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 2182, __extension__ __PRETTY_FUNCTION__));
2183	// The "number of bits to shift" operand must preserve its value as an
2184	// unsigned integer:
2185	widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
2186	}
2187
2188	Observer.changedInstr(MI);
2189	return Legalized;
2190
2191	case TargetOpcode::G_SDIV:
2192	case TargetOpcode::G_SREM:
2193	case TargetOpcode::G_SMIN:
2194	case TargetOpcode::G_SMAX:
2195	Observer.changingInstr(MI);
2196	widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
2197	widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
2198	widenScalarDst(MI, WideTy);
2199	Observer.changedInstr(MI);
2200	return Legalized;
2201
2202	case TargetOpcode::G_SDIVREM:
2203	Observer.changingInstr(MI);
2204	widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
2205	widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_SEXT);
2206	widenScalarDst(MI, WideTy);
2207	widenScalarDst(MI, WideTy, 1);
2208	Observer.changedInstr(MI);
2209	return Legalized;
2210
2211	case TargetOpcode::G_ASHR:
2212	case TargetOpcode::G_LSHR:
2213	Observer.changingInstr(MI);
2214
2215	if (TypeIdx == 0) {
2216	unsigned CvtOp = MI.getOpcode() == TargetOpcode::G_ASHR ?
2217	TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
2218
2219	widenScalarSrc(MI, WideTy, 1, CvtOp);
2220	widenScalarDst(MI, WideTy);
2221	} else {
2222	assert(TypeIdx == 1)(static_cast <bool> (TypeIdx == 1) ? void (0) : __assert_fail ("TypeIdx == 1", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 2222, __extension__ __PRETTY_FUNCTION__));
2223	// The "number of bits to shift" operand must preserve its value as an
2224	// unsigned integer:
2225	widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
2226	}
2227
2228	Observer.changedInstr(MI);
2229	return Legalized;
2230	case TargetOpcode::G_UDIV:
2231	case TargetOpcode::G_UREM:
2232	case TargetOpcode::G_UMIN:
2233	case TargetOpcode::G_UMAX:
2234	Observer.changingInstr(MI);
2235	widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
2236	widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
2237	widenScalarDst(MI, WideTy);
2238	Observer.changedInstr(MI);
2239	return Legalized;
2240
2241	case TargetOpcode::G_UDIVREM:
2242	Observer.changingInstr(MI);
2243	widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
2244	widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ZEXT);
2245	widenScalarDst(MI, WideTy);
2246	widenScalarDst(MI, WideTy, 1);
2247	Observer.changedInstr(MI);
2248	return Legalized;
2249
2250	case TargetOpcode::G_SELECT:
2251	Observer.changingInstr(MI);
2252	if (TypeIdx == 0) {
2253	// Perform operation at larger width (any extension is fine here, high
2254	// bits don't affect the result) and then truncate the result back to the
2255	// original type.
2256	widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
2257	widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ANYEXT);
2258	widenScalarDst(MI, WideTy);
2259	} else {
2260	bool IsVec = MRI.getType(MI.getOperand(1).getReg()).isVector();
2261	// Explicit extension is required here since high bits affect the result.
2262	widenScalarSrc(MI, WideTy, 1, MIRBuilder.getBoolExtOp(IsVec, false));
2263	}
2264	Observer.changedInstr(MI);
2265	return Legalized;
2266
2267	case TargetOpcode::G_FPTOSI:
2268	case TargetOpcode::G_FPTOUI:
2269	Observer.changingInstr(MI);
2270
2271	if (TypeIdx == 0)
2272	widenScalarDst(MI, WideTy);
2273	else
2274	widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT);
2275
2276	Observer.changedInstr(MI);
2277	return Legalized;
2278	case TargetOpcode::G_SITOFP:
2279	Observer.changingInstr(MI);
2280
2281	if (TypeIdx == 0)
2282	widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
2283	else
2284	widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
2285
2286	Observer.changedInstr(MI);
2287	return Legalized;
2288	case TargetOpcode::G_UITOFP:
2289	Observer.changingInstr(MI);
2290
2291	if (TypeIdx == 0)
2292	widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
2293	else
2294	widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
2295
2296	Observer.changedInstr(MI);
2297	return Legalized;
2298	case TargetOpcode::G_LOAD:
2299	case TargetOpcode::G_SEXTLOAD:
2300	case TargetOpcode::G_ZEXTLOAD:
2301	Observer.changingInstr(MI);
2302	widenScalarDst(MI, WideTy);
2303	Observer.changedInstr(MI);
2304	return Legalized;
2305
2306	case TargetOpcode::G_STORE: {
2307	if (TypeIdx != 0)
2308	return UnableToLegalize;
2309
2310	LLT Ty = MRI.getType(MI.getOperand(0).getReg());
2311	if (!Ty.isScalar())
2312	return UnableToLegalize;
2313
2314	Observer.changingInstr(MI);
2315
2316	unsigned ExtType = Ty.getScalarSizeInBits() == 1 ?
2317	TargetOpcode::G_ZEXT : TargetOpcode::G_ANYEXT;
2318	widenScalarSrc(MI, WideTy, 0, ExtType);
2319
2320	Observer.changedInstr(MI);
2321	return Legalized;
2322	}
2323	case TargetOpcode::G_CONSTANT: {
2324	MachineOperand &SrcMO = MI.getOperand(1);
2325	LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
2326	unsigned ExtOpc = LI.getExtOpcodeForWideningConstant(
2327	MRI.getType(MI.getOperand(0).getReg()));
2328	assert((ExtOpc == TargetOpcode::G_ZEXT \|\| ExtOpc == TargetOpcode::G_SEXT \|\|(static_cast <bool> ((ExtOpc == TargetOpcode::G_ZEXT \|\| ExtOpc == TargetOpcode::G_SEXT \|\| ExtOpc == TargetOpcode::G_ANYEXT ) && "Illegal Extend") ? void (0) : __assert_fail ("(ExtOpc == TargetOpcode::G_ZEXT \|\| ExtOpc == TargetOpcode::G_SEXT \|\| ExtOpc == TargetOpcode::G_ANYEXT) && \"Illegal Extend\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 2330, __extension__ __PRETTY_FUNCTION__))
2329	ExtOpc == TargetOpcode::G_ANYEXT) &&(static_cast <bool> ((ExtOpc == TargetOpcode::G_ZEXT \|\| ExtOpc == TargetOpcode::G_SEXT \|\| ExtOpc == TargetOpcode::G_ANYEXT ) && "Illegal Extend") ? void (0) : __assert_fail ("(ExtOpc == TargetOpcode::G_ZEXT \|\| ExtOpc == TargetOpcode::G_SEXT \|\| ExtOpc == TargetOpcode::G_ANYEXT) && \"Illegal Extend\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 2330, __extension__ __PRETTY_FUNCTION__))
2330	"Illegal Extend")(static_cast <bool> ((ExtOpc == TargetOpcode::G_ZEXT \|\| ExtOpc == TargetOpcode::G_SEXT \|\| ExtOpc == TargetOpcode::G_ANYEXT ) && "Illegal Extend") ? void (0) : __assert_fail ("(ExtOpc == TargetOpcode::G_ZEXT \|\| ExtOpc == TargetOpcode::G_SEXT \|\| ExtOpc == TargetOpcode::G_ANYEXT) && \"Illegal Extend\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 2330, __extension__ __PRETTY_FUNCTION__));
2331	const APInt &SrcVal = SrcMO.getCImm()->getValue();
2332	const APInt &Val = (ExtOpc == TargetOpcode::G_SEXT)
2333	? SrcVal.sext(WideTy.getSizeInBits())
2334	: SrcVal.zext(WideTy.getSizeInBits());
2335	Observer.changingInstr(MI);
2336	SrcMO.setCImm(ConstantInt::get(Ctx, Val));
2337
2338	widenScalarDst(MI, WideTy);
2339	Observer.changedInstr(MI);
2340	return Legalized;
2341	}
2342	case TargetOpcode::G_FCONSTANT: {
2343	MachineOperand &SrcMO = MI.getOperand(1);
2344	LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
2345	APFloat Val = SrcMO.getFPImm()->getValueAPF();
2346	bool LosesInfo;
2347	switch (WideTy.getSizeInBits()) {
2348	case 32:
2349	Val.convert(APFloat::IEEEsingle(), APFloat::rmNearestTiesToEven,
2350	&LosesInfo);
2351	break;
2352	case 64:
2353	Val.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
2354	&LosesInfo);
2355	break;
2356	default:
2357	return UnableToLegalize;
2358	}
2359
2360	assert(!LosesInfo && "extend should always be lossless")(static_cast <bool> (!LosesInfo && "extend should always be lossless" ) ? void (0) : __assert_fail ("!LosesInfo && \"extend should always be lossless\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 2360, __extension__ __PRETTY_FUNCTION__));
2361
2362	Observer.changingInstr(MI);
2363	SrcMO.setFPImm(ConstantFP::get(Ctx, Val));
2364
2365	widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
2366	Observer.changedInstr(MI);
2367	return Legalized;
2368	}
2369	case TargetOpcode::G_IMPLICIT_DEF: {
2370	Observer.changingInstr(MI);
2371	widenScalarDst(MI, WideTy);
2372	Observer.changedInstr(MI);
2373	return Legalized;
2374	}
2375	case TargetOpcode::G_BRCOND:
2376	Observer.changingInstr(MI);
2377	widenScalarSrc(MI, WideTy, 0, MIRBuilder.getBoolExtOp(false, false));
2378	Observer.changedInstr(MI);
2379	return Legalized;
2380
2381	case TargetOpcode::G_FCMP:
2382	Observer.changingInstr(MI);
2383	if (TypeIdx == 0)
2384	widenScalarDst(MI, WideTy);
2385	else {
2386	widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_FPEXT);
2387	widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_FPEXT);
2388	}
2389	Observer.changedInstr(MI);
2390	return Legalized;
2391
2392	case TargetOpcode::G_ICMP:
2393	Observer.changingInstr(MI);
2394	if (TypeIdx == 0)
2395	widenScalarDst(MI, WideTy);
2396	else {
2397	unsigned ExtOpcode = CmpInst::isSigned(static_cast<CmpInst::Predicate>(
2398	MI.getOperand(1).getPredicate()))
2399	? TargetOpcode::G_SEXT
2400	: TargetOpcode::G_ZEXT;
2401	widenScalarSrc(MI, WideTy, 2, ExtOpcode);
2402	widenScalarSrc(MI, WideTy, 3, ExtOpcode);
2403	}
2404	Observer.changedInstr(MI);
2405	return Legalized;
2406
2407	case TargetOpcode::G_PTR_ADD:
2408	assert(TypeIdx == 1 && "unable to legalize pointer of G_PTR_ADD")(static_cast <bool> (TypeIdx == 1 && "unable to legalize pointer of G_PTR_ADD" ) ? void (0) : __assert_fail ("TypeIdx == 1 && \"unable to legalize pointer of G_PTR_ADD\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 2408, __extension__ __PRETTY_FUNCTION__));
2409	Observer.changingInstr(MI);
2410	widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
2411	Observer.changedInstr(MI);
2412	return Legalized;
2413
2414	case TargetOpcode::G_PHI: {
2415	assert(TypeIdx == 0 && "Expecting only Idx 0")(static_cast <bool> (TypeIdx == 0 && "Expecting only Idx 0" ) ? void (0) : __assert_fail ("TypeIdx == 0 && \"Expecting only Idx 0\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 2415, __extension__ __PRETTY_FUNCTION__));
2416
2417	Observer.changingInstr(MI);
2418	for (unsigned I = 1; I < MI.getNumOperands(); I += 2) {
2419	MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
2420	MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
2421	widenScalarSrc(MI, WideTy, I, TargetOpcode::G_ANYEXT);
2422	}
2423
2424	MachineBasicBlock &MBB = *MI.getParent();
2425	MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI());
2426	widenScalarDst(MI, WideTy);
2427	Observer.changedInstr(MI);
2428	return Legalized;
2429	}
2430	case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
2431	if (TypeIdx == 0) {
2432	Register VecReg = MI.getOperand(1).getReg();
2433	LLT VecTy = MRI.getType(VecReg);
2434	Observer.changingInstr(MI);
2435
2436	widenScalarSrc(
2437	MI, LLT::vector(VecTy.getElementCount(), WideTy.getSizeInBits()), 1,
2438	TargetOpcode::G_SEXT);
2439
2440	widenScalarDst(MI, WideTy, 0);
2441	Observer.changedInstr(MI);
2442	return Legalized;
2443	}
2444
2445	if (TypeIdx != 2)
2446	return UnableToLegalize;
2447	Observer.changingInstr(MI);
2448	// TODO: Probably should be zext
2449	widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
2450	Observer.changedInstr(MI);
2451	return Legalized;
2452	}
2453	case TargetOpcode::G_INSERT_VECTOR_ELT: {
2454	if (TypeIdx == 1) {
2455	Observer.changingInstr(MI);
2456
2457	Register VecReg = MI.getOperand(1).getReg();
2458	LLT VecTy = MRI.getType(VecReg);
2459	LLT WideVecTy = LLT::vector(VecTy.getElementCount(), WideTy);
2460
2461	widenScalarSrc(MI, WideVecTy, 1, TargetOpcode::G_ANYEXT);
2462	widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
2463	widenScalarDst(MI, WideVecTy, 0);
2464	Observer.changedInstr(MI);
2465	return Legalized;
2466	}
2467
2468	if (TypeIdx == 2) {
2469	Observer.changingInstr(MI);
2470	// TODO: Probably should be zext
2471	widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_SEXT);
2472	Observer.changedInstr(MI);
2473	return Legalized;
2474	}
2475
2476	return UnableToLegalize;
2477	}
2478	case TargetOpcode::G_FADD:
2479	case TargetOpcode::G_FMUL:
2480	case TargetOpcode::G_FSUB:
2481	case TargetOpcode::G_FMA:
2482	case TargetOpcode::G_FMAD:
2483	case TargetOpcode::G_FNEG:
2484	case TargetOpcode::G_FABS:
2485	case TargetOpcode::G_FCANONICALIZE:
2486	case TargetOpcode::G_FMINNUM:
2487	case TargetOpcode::G_FMAXNUM:
2488	case TargetOpcode::G_FMINNUM_IEEE:
2489	case TargetOpcode::G_FMAXNUM_IEEE:
2490	case TargetOpcode::G_FMINIMUM:
2491	case TargetOpcode::G_FMAXIMUM:
2492	case TargetOpcode::G_FDIV:
2493	case TargetOpcode::G_FREM:
2494	case TargetOpcode::G_FCEIL:
2495	case TargetOpcode::G_FFLOOR:
2496	case TargetOpcode::G_FCOS:
2497	case TargetOpcode::G_FSIN:
2498	case TargetOpcode::G_FLOG10:
2499	case TargetOpcode::G_FLOG:
2500	case TargetOpcode::G_FLOG2:
2501	case TargetOpcode::G_FRINT:
2502	case TargetOpcode::G_FNEARBYINT:
2503	case TargetOpcode::G_FSQRT:
2504	case TargetOpcode::G_FEXP:
2505	case TargetOpcode::G_FEXP2:
2506	case TargetOpcode::G_FPOW:
2507	case TargetOpcode::G_INTRINSIC_TRUNC:
2508	case TargetOpcode::G_INTRINSIC_ROUND:
2509	case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
2510	assert(TypeIdx == 0)(static_cast <bool> (TypeIdx == 0) ? void (0) : __assert_fail ("TypeIdx == 0", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 2510, __extension__ __PRETTY_FUNCTION__));
2511	Observer.changingInstr(MI);
2512
2513	for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I)
2514	widenScalarSrc(MI, WideTy, I, TargetOpcode::G_FPEXT);
2515
2516	widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
2517	Observer.changedInstr(MI);
2518	return Legalized;
2519	case TargetOpcode::G_FPOWI: {
2520	if (TypeIdx != 0)
2521	return UnableToLegalize;
2522	Observer.changingInstr(MI);
2523	widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT);
2524	widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
2525	Observer.changedInstr(MI);
2526	return Legalized;
2527	}
2528	case TargetOpcode::G_INTTOPTR:
2529	if (TypeIdx != 1)
2530	return UnableToLegalize;
2531
2532	Observer.changingInstr(MI);
2533	widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
2534	Observer.changedInstr(MI);
2535	return Legalized;
2536	case TargetOpcode::G_PTRTOINT:
2537	if (TypeIdx != 0)
2538	return UnableToLegalize;
2539
2540	Observer.changingInstr(MI);
2541	widenScalarDst(MI, WideTy, 0);
2542	Observer.changedInstr(MI);
2543	return Legalized;
2544	case TargetOpcode::G_BUILD_VECTOR: {
2545	Observer.changingInstr(MI);
2546
2547	const LLT WideEltTy = TypeIdx == 1 ? WideTy : WideTy.getElementType();
2548	for (int I = 1, E = MI.getNumOperands(); I != E; ++I)
2549	widenScalarSrc(MI, WideEltTy, I, TargetOpcode::G_ANYEXT);
2550
2551	// Avoid changing the result vector type if the source element type was
2552	// requested.
2553	if (TypeIdx == 1) {
2554	MI.setDesc(MIRBuilder.getTII().get(TargetOpcode::G_BUILD_VECTOR_TRUNC));
2555	} else {
2556	widenScalarDst(MI, WideTy, 0);
2557	}
2558
2559	Observer.changedInstr(MI);
2560	return Legalized;
2561	}
2562	case TargetOpcode::G_SEXT_INREG:
2563	if (TypeIdx != 0)
2564	return UnableToLegalize;
2565
2566	Observer.changingInstr(MI);
2567	widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
2568	widenScalarDst(MI, WideTy, 0, TargetOpcode::G_TRUNC);
2569	Observer.changedInstr(MI);
2570	return Legalized;
2571	case TargetOpcode::G_PTRMASK: {
2572	if (TypeIdx != 1)
2573	return UnableToLegalize;
2574	Observer.changingInstr(MI);
2575	widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
2576	Observer.changedInstr(MI);
2577	return Legalized;
2578	}
2579	}
2580	}
2581
2582	static void getUnmergePieces(SmallVectorImpl<Register> &Pieces,
2583	MachineIRBuilder &B, Register Src, LLT Ty) {
2584	auto Unmerge = B.buildUnmerge(Ty, Src);
2585	for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
2586	Pieces.push_back(Unmerge.getReg(I));
2587	}
2588
2589	LegalizerHelper::LegalizeResult
2590	LegalizerHelper::lowerBitcast(MachineInstr &MI) {
2591	Register Dst = MI.getOperand(0).getReg();
2592	Register Src = MI.getOperand(1).getReg();
2593	LLT DstTy = MRI.getType(Dst);
2594	LLT SrcTy = MRI.getType(Src);
2595
2596	if (SrcTy.isVector()) {
2597	LLT SrcEltTy = SrcTy.getElementType();
2598	SmallVector<Register, 8> SrcRegs;
2599
2600	if (DstTy.isVector()) {
2601	int NumDstElt = DstTy.getNumElements();
2602	int NumSrcElt = SrcTy.getNumElements();
2603
2604	LLT DstEltTy = DstTy.getElementType();
2605	LLT DstCastTy = DstEltTy; // Intermediate bitcast result type
2606	LLT SrcPartTy = SrcEltTy; // Original unmerge result type.
2607
2608	// If there's an element size mismatch, insert intermediate casts to match
2609	// the result element type.
2610	if (NumSrcElt < NumDstElt) { // Source element type is larger.
2611	// %1:_(<4 x s8>) = G_BITCAST %0:_(<2 x s16>)
2612	//
2613	// =>
2614	//
2615	// %2:_(s16), %3:_(s16) = G_UNMERGE_VALUES %0
2616	// %3:_(<2 x s8>) = G_BITCAST %2
2617	// %4:_(<2 x s8>) = G_BITCAST %3
2618	// %1:_(<4 x s16>) = G_CONCAT_VECTORS %3, %4
2619	DstCastTy = LLT::fixed_vector(NumDstElt / NumSrcElt, DstEltTy);
2620	SrcPartTy = SrcEltTy;
2621	} else if (NumSrcElt > NumDstElt) { // Source element type is smaller.
2622	//
2623	// %1:_(<2 x s16>) = G_BITCAST %0:_(<4 x s8>)
2624	//
2625	// =>
2626	//
2627	// %2:_(<2 x s8>), %3:_(<2 x s8>) = G_UNMERGE_VALUES %0
2628	// %3:_(s16) = G_BITCAST %2
2629	// %4:_(s16) = G_BITCAST %3
2630	// %1:_(<2 x s16>) = G_BUILD_VECTOR %3, %4
2631	SrcPartTy = LLT::fixed_vector(NumSrcElt / NumDstElt, SrcEltTy);
2632	DstCastTy = DstEltTy;
2633	}
2634
2635	getUnmergePieces(SrcRegs, MIRBuilder, Src, SrcPartTy);
2636	for (Register &SrcReg : SrcRegs)
2637	SrcReg = MIRBuilder.buildBitcast(DstCastTy, SrcReg).getReg(0);
2638	} else
2639	getUnmergePieces(SrcRegs, MIRBuilder, Src, SrcEltTy);
2640
2641	MIRBuilder.buildMerge(Dst, SrcRegs);
2642	MI.eraseFromParent();
2643	return Legalized;
2644	}
2645
2646	if (DstTy.isVector()) {
2647	SmallVector<Register, 8> SrcRegs;
2648	getUnmergePieces(SrcRegs, MIRBuilder, Src, DstTy.getElementType());
2649	MIRBuilder.buildMerge(Dst, SrcRegs);
2650	MI.eraseFromParent();
2651	return Legalized;
2652	}
2653
2654	return UnableToLegalize;
2655	}
2656
2657	/// Figure out the bit offset into a register when coercing a vector index for
2658	/// the wide element type. This is only for the case when promoting vector to
2659	/// one with larger elements.
2660	//
2661	///
2662	/// %offset_idx = G_AND %idx, ~(-1 << Log2(DstEltSize / SrcEltSize))
2663	/// %offset_bits = G_SHL %offset_idx, Log2(SrcEltSize)
2664	static Register getBitcastWiderVectorElementOffset(MachineIRBuilder &B,
2665	Register Idx,
2666	unsigned NewEltSize,
2667	unsigned OldEltSize) {
2668	const unsigned Log2EltRatio = Log2_32(NewEltSize / OldEltSize);
2669	LLT IdxTy = B.getMRI()->getType(Idx);
2670
2671	// Now figure out the amount we need to shift to get the target bits.
2672	auto OffsetMask = B.buildConstant(
2673	IdxTy, ~(APInt::getAllOnesValue(IdxTy.getSizeInBits()) << Log2EltRatio));
2674	auto OffsetIdx = B.buildAnd(IdxTy, Idx, OffsetMask);
2675	return B.buildShl(IdxTy, OffsetIdx,
2676	B.buildConstant(IdxTy, Log2_32(OldEltSize))).getReg(0);
2677	}
2678
2679	/// Perform a G_EXTRACT_VECTOR_ELT in a different sized vector element. If this
2680	/// is casting to a vector with a smaller element size, perform multiple element
2681	/// extracts and merge the results. If this is coercing to a vector with larger
2682	/// elements, index the bitcasted vector and extract the target element with bit
2683	/// operations. This is intended to force the indexing in the native register
2684	/// size for architectures that can dynamically index the register file.
2685	LegalizerHelper::LegalizeResult
2686	LegalizerHelper::bitcastExtractVectorElt(MachineInstr &MI, unsigned TypeIdx,
2687	LLT CastTy) {
2688	if (TypeIdx != 1)
2689	return UnableToLegalize;
2690
2691	Register Dst = MI.getOperand(0).getReg();
2692	Register SrcVec = MI.getOperand(1).getReg();
2693	Register Idx = MI.getOperand(2).getReg();
2694	LLT SrcVecTy = MRI.getType(SrcVec);
2695	LLT IdxTy = MRI.getType(Idx);
2696
2697	LLT SrcEltTy = SrcVecTy.getElementType();
2698	unsigned NewNumElts = CastTy.isVector() ? CastTy.getNumElements() : 1;
2699	unsigned OldNumElts = SrcVecTy.getNumElements();
2700
2701	LLT NewEltTy = CastTy.isVector() ? CastTy.getElementType() : CastTy;
2702	Register CastVec = MIRBuilder.buildBitcast(CastTy, SrcVec).getReg(0);
2703
2704	const unsigned NewEltSize = NewEltTy.getSizeInBits();
2705	const unsigned OldEltSize = SrcEltTy.getSizeInBits();
2706	if (NewNumElts > OldNumElts) {
2707	// Decreasing the vector element size
2708	//
2709	// e.g. i64 = extract_vector_elt x:v2i64, y:i32
2710	// =>
2711	// v4i32:castx = bitcast x:v2i64
2712	//
2713	// i64 = bitcast
2714	// (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
2715	// (i32 (extract_vector_elt castx, (2 * y + 1)))
2716	//
2717	if (NewNumElts % OldNumElts != 0)
2718	return UnableToLegalize;
2719
2720	// Type of the intermediate result vector.
2721	const unsigned NewEltsPerOldElt = NewNumElts / OldNumElts;
2722	LLT MidTy =
2723	LLT::scalarOrVector(ElementCount::getFixed(NewEltsPerOldElt), NewEltTy);
2724
2725	auto NewEltsPerOldEltK = MIRBuilder.buildConstant(IdxTy, NewEltsPerOldElt);
2726
2727	SmallVector<Register, 8> NewOps(NewEltsPerOldElt);
2728	auto NewBaseIdx = MIRBuilder.buildMul(IdxTy, Idx, NewEltsPerOldEltK);
2729
2730	for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
2731	auto IdxOffset = MIRBuilder.buildConstant(IdxTy, I);
2732	auto TmpIdx = MIRBuilder.buildAdd(IdxTy, NewBaseIdx, IdxOffset);
2733	auto Elt = MIRBuilder.buildExtractVectorElement(NewEltTy, CastVec, TmpIdx);
2734	NewOps[I] = Elt.getReg(0);
2735	}
2736
2737	auto NewVec = MIRBuilder.buildBuildVector(MidTy, NewOps);
2738	MIRBuilder.buildBitcast(Dst, NewVec);
2739	MI.eraseFromParent();
2740	return Legalized;
2741	}
2742
2743	if (NewNumElts < OldNumElts) {
2744	if (NewEltSize % OldEltSize != 0)
2745	return UnableToLegalize;
2746
2747	// This only depends on powers of 2 because we use bit tricks to figure out
2748	// the bit offset we need to shift to get the target element. A general
2749	// expansion could emit division/multiply.
2750	if (!isPowerOf2_32(NewEltSize / OldEltSize))
2751	return UnableToLegalize;
2752
2753	// Increasing the vector element size.
2754	// %elt:_(small_elt) = G_EXTRACT_VECTOR_ELT %vec:_(<N x small_elt>), %idx
2755	//
2756	// =>
2757	//
2758	// %cast = G_BITCAST %vec
2759	// %scaled_idx = G_LSHR %idx, Log2(DstEltSize / SrcEltSize)
2760	// %wide_elt = G_EXTRACT_VECTOR_ELT %cast, %scaled_idx
2761	// %offset_idx = G_AND %idx, ~(-1 << Log2(DstEltSize / SrcEltSize))
2762	// %offset_bits = G_SHL %offset_idx, Log2(SrcEltSize)
2763	// %elt_bits = G_LSHR %wide_elt, %offset_bits
2764	// %elt = G_TRUNC %elt_bits
2765
2766	const unsigned Log2EltRatio = Log2_32(NewEltSize / OldEltSize);
2767	auto Log2Ratio = MIRBuilder.buildConstant(IdxTy, Log2EltRatio);
2768
2769	// Divide to get the index in the wider element type.
2770	auto ScaledIdx = MIRBuilder.buildLShr(IdxTy, Idx, Log2Ratio);
2771
2772	Register WideElt = CastVec;
2773	if (CastTy.isVector()) {
2774	WideElt = MIRBuilder.buildExtractVectorElement(NewEltTy, CastVec,
2775	ScaledIdx).getReg(0);
2776	}
2777
2778	// Compute the bit offset into the register of the target element.
2779	Register OffsetBits = getBitcastWiderVectorElementOffset(
2780	MIRBuilder, Idx, NewEltSize, OldEltSize);
2781
2782	// Shift the wide element to get the target element.
2783	auto ExtractedBits = MIRBuilder.buildLShr(NewEltTy, WideElt, OffsetBits);
2784	MIRBuilder.buildTrunc(Dst, ExtractedBits);
2785	MI.eraseFromParent();
2786	return Legalized;
2787	}
2788
2789	return UnableToLegalize;
2790	}
2791
2792	/// Emit code to insert \p InsertReg into \p TargetRet at \p OffsetBits in \p
2793	/// TargetReg, while preserving other bits in \p TargetReg.
2794	///
2795	/// (InsertReg << Offset) \| (TargetReg & ~(-1 >> InsertReg.size()) << Offset)
2796	static Register buildBitFieldInsert(MachineIRBuilder &B,
2797	Register TargetReg, Register InsertReg,
2798	Register OffsetBits) {
2799	LLT TargetTy = B.getMRI()->getType(TargetReg);
2800	LLT InsertTy = B.getMRI()->getType(InsertReg);
2801	auto ZextVal = B.buildZExt(TargetTy, InsertReg);
2802	auto ShiftedInsertVal = B.buildShl(TargetTy, ZextVal, OffsetBits);
2803
2804	// Produce a bitmask of the value to insert
2805	auto EltMask = B.buildConstant(
2806	TargetTy, APInt::getLowBitsSet(TargetTy.getSizeInBits(),
2807	InsertTy.getSizeInBits()));
2808	// Shift it into position
2809	auto ShiftedMask = B.buildShl(TargetTy, EltMask, OffsetBits);
2810	auto InvShiftedMask = B.buildNot(TargetTy, ShiftedMask);
2811
2812	// Clear out the bits in the wide element
2813	auto MaskedOldElt = B.buildAnd(TargetTy, TargetReg, InvShiftedMask);
2814
2815	// The value to insert has all zeros already, so stick it into the masked
2816	// wide element.
2817	return B.buildOr(TargetTy, MaskedOldElt, ShiftedInsertVal).getReg(0);
2818	}
2819
2820	/// Perform a G_INSERT_VECTOR_ELT in a different sized vector element. If this
2821	/// is increasing the element size, perform the indexing in the target element
2822	/// type, and use bit operations to insert at the element position. This is
2823	/// intended for architectures that can dynamically index the register file and
2824	/// want to force indexing in the native register size.
2825	LegalizerHelper::LegalizeResult
2826	LegalizerHelper::bitcastInsertVectorElt(MachineInstr &MI, unsigned TypeIdx,
2827	LLT CastTy) {
2828	if (TypeIdx != 0)
2829	return UnableToLegalize;
2830
2831	Register Dst = MI.getOperand(0).getReg();
2832	Register SrcVec = MI.getOperand(1).getReg();
2833	Register Val = MI.getOperand(2).getReg();
2834	Register Idx = MI.getOperand(3).getReg();
2835
2836	LLT VecTy = MRI.getType(Dst);
2837	LLT IdxTy = MRI.getType(Idx);
2838
2839	LLT VecEltTy = VecTy.getElementType();
2840	LLT NewEltTy = CastTy.isVector() ? CastTy.getElementType() : CastTy;
2841	const unsigned NewEltSize = NewEltTy.getSizeInBits();
2842	const unsigned OldEltSize = VecEltTy.getSizeInBits();
2843
2844	unsigned NewNumElts = CastTy.isVector() ? CastTy.getNumElements() : 1;
2845	unsigned OldNumElts = VecTy.getNumElements();
2846
2847	Register CastVec = MIRBuilder.buildBitcast(CastTy, SrcVec).getReg(0);
2848	if (NewNumElts < OldNumElts) {
2849	if (NewEltSize % OldEltSize != 0)
2850	return UnableToLegalize;
2851
2852	// This only depends on powers of 2 because we use bit tricks to figure out
2853	// the bit offset we need to shift to get the target element. A general
2854	// expansion could emit division/multiply.
2855	if (!isPowerOf2_32(NewEltSize / OldEltSize))
2856	return UnableToLegalize;
2857
2858	const unsigned Log2EltRatio = Log2_32(NewEltSize / OldEltSize);
2859	auto Log2Ratio = MIRBuilder.buildConstant(IdxTy, Log2EltRatio);
2860
2861	// Divide to get the index in the wider element type.
2862	auto ScaledIdx = MIRBuilder.buildLShr(IdxTy, Idx, Log2Ratio);
2863
2864	Register ExtractedElt = CastVec;
2865	if (CastTy.isVector()) {
2866	ExtractedElt = MIRBuilder.buildExtractVectorElement(NewEltTy, CastVec,
2867	ScaledIdx).getReg(0);
2868	}
2869
2870	// Compute the bit offset into the register of the target element.
2871	Register OffsetBits = getBitcastWiderVectorElementOffset(
2872	MIRBuilder, Idx, NewEltSize, OldEltSize);
2873
2874	Register InsertedElt = buildBitFieldInsert(MIRBuilder, ExtractedElt,
2875	Val, OffsetBits);
2876	if (CastTy.isVector()) {
2877	InsertedElt = MIRBuilder.buildInsertVectorElement(
2878	CastTy, CastVec, InsertedElt, ScaledIdx).getReg(0);
2879	}
2880
2881	MIRBuilder.buildBitcast(Dst, InsertedElt);
2882	MI.eraseFromParent();
2883	return Legalized;
2884	}
2885
2886	return UnableToLegalize;
2887	}
2888
2889	LegalizerHelper::LegalizeResult LegalizerHelper::lowerLoad(GAnyLoad &LoadMI) {
2890	// Lower to a memory-width G_LOAD and a G_SEXT/G_ZEXT/G_ANYEXT
2891	Register DstReg = LoadMI.getDstReg();
2892	Register PtrReg = LoadMI.getPointerReg();
2893	LLT DstTy = MRI.getType(DstReg);
2894	MachineMemOperand &MMO = LoadMI.getMMO();
2895	LLT MemTy = MMO.getMemoryType();
2896	MachineFunction &MF = MIRBuilder.getMF();
2897
2898	unsigned MemSizeInBits = MemTy.getSizeInBits();
2899	unsigned MemStoreSizeInBits = 8 * MemTy.getSizeInBytes();
2900
2901	if (MemSizeInBits != MemStoreSizeInBits) {
2902	if (MemTy.isVector())
2903	return UnableToLegalize;
2904
2905	// Promote to a byte-sized load if not loading an integral number of
2906	// bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
2907	LLT WideMemTy = LLT::scalar(MemStoreSizeInBits);
2908	MachineMemOperand *NewMMO =
2909	MF.getMachineMemOperand(&MMO, MMO.getPointerInfo(), WideMemTy);
2910
2911	Register LoadReg = DstReg;
2912	LLT LoadTy = DstTy;
2913
2914	// If this wasn't already an extending load, we need to widen the result
2915	// register to avoid creating a load with a narrower result than the source.
2916	if (MemStoreSizeInBits > DstTy.getSizeInBits()) {
2917	LoadTy = WideMemTy;
2918	LoadReg = MRI.createGenericVirtualRegister(WideMemTy);
2919	}
2920
2921	if (isa<GSExtLoad>(LoadMI)) {
2922	auto NewLoad = MIRBuilder.buildLoad(LoadTy, PtrReg, *NewMMO);
2923	MIRBuilder.buildSExtInReg(LoadReg, NewLoad, MemSizeInBits);
2924	} else if (isa<GZExtLoad>(LoadMI) \|\| WideMemTy == DstTy) {
2925	auto NewLoad = MIRBuilder.buildLoad(LoadTy, PtrReg, *NewMMO);
2926	// The extra bits are guaranteed to be zero, since we stored them that
2927	// way. A zext load from Wide thus automatically gives zext from MemVT.
2928	MIRBuilder.buildAssertZExt(LoadReg, NewLoad, MemSizeInBits);
2929	} else {
2930	MIRBuilder.buildLoad(LoadReg, PtrReg, *NewMMO);
2931	}
2932
2933	if (DstTy != LoadTy)
2934	MIRBuilder.buildTrunc(DstReg, LoadReg);
2935
2936	LoadMI.eraseFromParent();
2937	return Legalized;
2938	}
2939
2940	// Big endian lowering not implemented.
2941	if (MIRBuilder.getDataLayout().isBigEndian())
2942	return UnableToLegalize;
2943
2944	// This load needs splitting into power of 2 sized loads.
2945	//
2946	// Our strategy here is to generate anyextending loads for the smaller
2947	// types up to next power-2 result type, and then combine the two larger
2948	// result values together, before truncating back down to the non-pow-2
2949	// type.
2950	// E.g. v1 = i24 load =>
2951	// v2 = i32 zextload (2 byte)
2952	// v3 = i32 load (1 byte)
2953	// v4 = i32 shl v3, 16
2954	// v5 = i32 or v4, v2
2955	// v1 = i24 trunc v5
2956	// By doing this we generate the correct truncate which should get
2957	// combined away as an artifact with a matching extend.
2958
2959	uint64_t LargeSplitSize, SmallSplitSize;
2960
2961	if (!isPowerOf2_32(MemSizeInBits)) {
2962	// This load needs splitting into power of 2 sized loads.
2963	LargeSplitSize = PowerOf2Floor(MemSizeInBits);
2964	SmallSplitSize = MemSizeInBits - LargeSplitSize;
2965	} else {
2966	// This is already a power of 2, but we still need to split this in half.
2967	//
2968	// Assume we're being asked to decompose an unaligned load.
2969	// TODO: If this requires multiple splits, handle them all at once.
2970	auto &Ctx = MF.getFunction().getContext();
2971	if (TLI.allowsMemoryAccess(Ctx, MIRBuilder.getDataLayout(), MemTy, MMO))
2972	return UnableToLegalize;
2973
2974	SmallSplitSize = LargeSplitSize = MemSizeInBits / 2;
2975	}
2976
2977	if (MemTy.isVector()) {
2978	// TODO: Handle vector extloads
2979	if (MemTy != DstTy)
2980	return UnableToLegalize;
2981
2982	// TODO: We can do better than scalarizing the vector and at least split it
2983	// in half.
2984	return reduceLoadStoreWidth(LoadMI, 0, DstTy.getElementType());
2985	}
2986
2987	MachineMemOperand *LargeMMO =
2988	MF.getMachineMemOperand(&MMO, 0, LargeSplitSize / 8);
2989	MachineMemOperand *SmallMMO =
2990	MF.getMachineMemOperand(&MMO, LargeSplitSize / 8, SmallSplitSize / 8);
2991
2992	LLT PtrTy = MRI.getType(PtrReg);
2993	unsigned AnyExtSize = PowerOf2Ceil(DstTy.getSizeInBits());
2994	LLT AnyExtTy = LLT::scalar(AnyExtSize);
2995	auto LargeLoad = MIRBuilder.buildLoadInstr(TargetOpcode::G_ZEXTLOAD, AnyExtTy,
2996	PtrReg, *LargeMMO);
2997
2998	auto OffsetCst = MIRBuilder.buildConstant(LLT::scalar(PtrTy.getSizeInBits()),
2999	LargeSplitSize / 8);
3000	Register PtrAddReg = MRI.createGenericVirtualRegister(PtrTy);
3001	auto SmallPtr = MIRBuilder.buildPtrAdd(PtrAddReg, PtrReg, OffsetCst);
3002	auto SmallLoad = MIRBuilder.buildLoadInstr(LoadMI.getOpcode(), AnyExtTy,
3003	SmallPtr, *SmallMMO);
3004
3005	auto ShiftAmt = MIRBuilder.buildConstant(AnyExtTy, LargeSplitSize);
3006	auto Shift = MIRBuilder.buildShl(AnyExtTy, SmallLoad, ShiftAmt);
3007
3008	if (AnyExtTy == DstTy)
3009	MIRBuilder.buildOr(DstReg, Shift, LargeLoad);
3010	else if (AnyExtTy.getSizeInBits() != DstTy.getSizeInBits()) {
3011	auto Or = MIRBuilder.buildOr(AnyExtTy, Shift, LargeLoad);
3012	MIRBuilder.buildTrunc(DstReg, {Or});
3013	} else {
3014	assert(DstTy.isPointer() && "expected pointer")(static_cast <bool> (DstTy.isPointer() && "expected pointer" ) ? void (0) : __assert_fail ("DstTy.isPointer() && \"expected pointer\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 3014, __extension__ __PRETTY_FUNCTION__));
3015	auto Or = MIRBuilder.buildOr(AnyExtTy, Shift, LargeLoad);
3016
3017	// FIXME: We currently consider this to be illegal for non-integral address
3018	// spaces, but we need still need a way to reinterpret the bits.
3019	MIRBuilder.buildIntToPtr(DstReg, Or);
3020	}
3021
3022	LoadMI.eraseFromParent();
3023	return Legalized;
3024	}
3025
3026	LegalizerHelper::LegalizeResult LegalizerHelper::lowerStore(GStore &StoreMI) {
3027	// Lower a non-power of 2 store into multiple pow-2 stores.
3028	// E.g. split an i24 store into an i16 store + i8 store.
3029	// We do this by first extending the stored value to the next largest power
3030	// of 2 type, and then using truncating stores to store the components.
3031	// By doing this, likewise with G_LOAD, generate an extend that can be
3032	// artifact-combined away instead of leaving behind extracts.
3033	Register SrcReg = StoreMI.getValueReg();
3034	Register PtrReg = StoreMI.getPointerReg();
3035	LLT SrcTy = MRI.getType(SrcReg);
3036	MachineFunction &MF = MIRBuilder.getMF();
3037	MachineMemOperand &MMO = **StoreMI.memoperands_begin();
3038	LLT MemTy = MMO.getMemoryType();
3039
3040	unsigned StoreWidth = MemTy.getSizeInBits();
3041	unsigned StoreSizeInBits = 8 * MemTy.getSizeInBytes();
3042
3043	if (StoreWidth != StoreSizeInBits) {
3044	if (SrcTy.isVector())
3045	return UnableToLegalize;
3046
3047	// Promote to a byte-sized store with upper bits zero if not
3048	// storing an integral number of bytes. For example, promote
3049	// TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
3050	LLT WideTy = LLT::scalar(StoreSizeInBits);
3051
3052	if (StoreSizeInBits > SrcTy.getSizeInBits()) {
3053	// Avoid creating a store with a narrower source than result.
3054	SrcReg = MIRBuilder.buildAnyExt(WideTy, SrcReg).getReg(0);
3055	SrcTy = WideTy;
3056	}
3057
3058	auto ZextInReg = MIRBuilder.buildZExtInReg(SrcTy, SrcReg, StoreWidth);
3059
3060	MachineMemOperand *NewMMO =
3061	MF.getMachineMemOperand(&MMO, MMO.getPointerInfo(), WideTy);
3062	MIRBuilder.buildStore(ZextInReg, PtrReg, *NewMMO);
3063	StoreMI.eraseFromParent();
3064	return Legalized;
3065	}
3066
3067	if (MemTy.isVector()) {
3068	// TODO: Handle vector trunc stores
3069	if (MemTy != SrcTy)
3070	return UnableToLegalize;
3071
3072	// TODO: We can do better than scalarizing the vector and at least split it
3073	// in half.
3074	return reduceLoadStoreWidth(StoreMI, 0, SrcTy.getElementType());
3075	}
3076
3077	unsigned MemSizeInBits = MemTy.getSizeInBits();
3078	uint64_t LargeSplitSize, SmallSplitSize;
3079
3080	if (!isPowerOf2_32(MemSizeInBits)) {
3081	LargeSplitSize = PowerOf2Floor(MemTy.getSizeInBits());
3082	SmallSplitSize = MemTy.getSizeInBits() - LargeSplitSize;
3083	} else {
3084	auto &Ctx = MF.getFunction().getContext();
3085	if (TLI.allowsMemoryAccess(Ctx, MIRBuilder.getDataLayout(), MemTy, MMO))
3086	return UnableToLegalize; // Don't know what we're being asked to do.
3087
3088	SmallSplitSize = LargeSplitSize = MemSizeInBits / 2;
3089	}
3090
3091	// Extend to the next pow-2. If this store was itself the result of lowering,
3092	// e.g. an s56 store being broken into s32 + s24, we might have a stored type
3093	// that's wider than the stored size.
3094	unsigned AnyExtSize = PowerOf2Ceil(MemTy.getSizeInBits());
3095	const LLT NewSrcTy = LLT::scalar(AnyExtSize);
3096
3097	if (SrcTy.isPointer()) {
3098	const LLT IntPtrTy = LLT::scalar(SrcTy.getSizeInBits());
3099	SrcReg = MIRBuilder.buildPtrToInt(IntPtrTy, SrcReg).getReg(0);
3100	}
3101
3102	auto ExtVal = MIRBuilder.buildAnyExtOrTrunc(NewSrcTy, SrcReg);
3103
3104	// Obtain the smaller value by shifting away the larger value.
3105	auto ShiftAmt = MIRBuilder.buildConstant(NewSrcTy, LargeSplitSize);
3106	auto SmallVal = MIRBuilder.buildLShr(NewSrcTy, ExtVal, ShiftAmt);
3107
3108	// Generate the PtrAdd and truncating stores.
3109	LLT PtrTy = MRI.getType(PtrReg);
3110	auto OffsetCst = MIRBuilder.buildConstant(
3111	LLT::scalar(PtrTy.getSizeInBits()), LargeSplitSize / 8);
3112	auto SmallPtr =
3113	MIRBuilder.buildPtrAdd(PtrTy, PtrReg, OffsetCst);
3114
3115	MachineMemOperand *LargeMMO =
3116	MF.getMachineMemOperand(&MMO, 0, LargeSplitSize / 8);
3117	MachineMemOperand *SmallMMO =
3118	MF.getMachineMemOperand(&MMO, LargeSplitSize / 8, SmallSplitSize / 8);
3119	MIRBuilder.buildStore(ExtVal, PtrReg, *LargeMMO);
3120	MIRBuilder.buildStore(SmallVal, SmallPtr, *SmallMMO);
3121	StoreMI.eraseFromParent();
3122	return Legalized;
3123	}
3124
3125	LegalizerHelper::LegalizeResult
3126	LegalizerHelper::bitcast(MachineInstr &MI, unsigned TypeIdx, LLT CastTy) {
3127	switch (MI.getOpcode()) {
3128	case TargetOpcode::G_LOAD: {
3129	if (TypeIdx != 0)
3130	return UnableToLegalize;
3131	MachineMemOperand &MMO = **MI.memoperands_begin();
3132
3133	// Not sure how to interpret a bitcast of an extending load.
3134	if (MMO.getMemoryType().getSizeInBits() != CastTy.getSizeInBits())
3135	return UnableToLegalize;
3136
3137	Observer.changingInstr(MI);
3138	bitcastDst(MI, CastTy, 0);
3139	MMO.setType(CastTy);
3140	Observer.changedInstr(MI);
3141	return Legalized;
3142	}
3143	case TargetOpcode::G_STORE: {
3144	if (TypeIdx != 0)
3145	return UnableToLegalize;
3146
3147	MachineMemOperand &MMO = **MI.memoperands_begin();
3148
3149	// Not sure how to interpret a bitcast of a truncating store.
3150	if (MMO.getMemoryType().getSizeInBits() != CastTy.getSizeInBits())
3151	return UnableToLegalize;
3152
3153	Observer.changingInstr(MI);
3154	bitcastSrc(MI, CastTy, 0);
3155	MMO.setType(CastTy);
3156	Observer.changedInstr(MI);
3157	return Legalized;
3158	}
3159	case TargetOpcode::G_SELECT: {
3160	if (TypeIdx != 0)
3161	return UnableToLegalize;
3162
3163	if (MRI.getType(MI.getOperand(1).getReg()).isVector()) {
3164	LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizer")) { dbgs() << "bitcast action not implemented for vector select\n" ; } } while (false)
3165	dbgs() << "bitcast action not implemented for vector select\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizer")) { dbgs() << "bitcast action not implemented for vector select\n" ; } } while (false);
3166	return UnableToLegalize;
3167	}
3168
3169	Observer.changingInstr(MI);
3170	bitcastSrc(MI, CastTy, 2);
3171	bitcastSrc(MI, CastTy, 3);
3172	bitcastDst(MI, CastTy, 0);
3173	Observer.changedInstr(MI);
3174	return Legalized;
3175	}
3176	case TargetOpcode::G_AND:
3177	case TargetOpcode::G_OR:
3178	case TargetOpcode::G_XOR: {
3179	Observer.changingInstr(MI);
3180	bitcastSrc(MI, CastTy, 1);
3181	bitcastSrc(MI, CastTy, 2);
3182	bitcastDst(MI, CastTy, 0);
3183	Observer.changedInstr(MI);
3184	return Legalized;
3185	}
3186	case TargetOpcode::G_EXTRACT_VECTOR_ELT:
3187	return bitcastExtractVectorElt(MI, TypeIdx, CastTy);
3188	case TargetOpcode::G_INSERT_VECTOR_ELT:
3189	return bitcastInsertVectorElt(MI, TypeIdx, CastTy);
3190	default:
3191	return UnableToLegalize;
3192	}
3193	}
3194
3195	// Legalize an instruction by changing the opcode in place.
3196	void LegalizerHelper::changeOpcode(MachineInstr &MI, unsigned NewOpcode) {
3197	Observer.changingInstr(MI);
3198	MI.setDesc(MIRBuilder.getTII().get(NewOpcode));
3199	Observer.changedInstr(MI);
3200	}
3201
3202	LegalizerHelper::LegalizeResult
3203	LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) {
3204	using namespace TargetOpcode;
3205
3206	switch(MI.getOpcode()) {
3207	default:
3208	return UnableToLegalize;
3209	case TargetOpcode::G_BITCAST:
3210	return lowerBitcast(MI);
3211	case TargetOpcode::G_SREM:
3212	case TargetOpcode::G_UREM: {
3213	LLT Ty = MRI.getType(MI.getOperand(0).getReg());
3214	auto Quot =
3215	MIRBuilder.buildInstr(MI.getOpcode() == G_SREM ? G_SDIV : G_UDIV, {Ty},
3216	{MI.getOperand(1), MI.getOperand(2)});
3217
3218	auto Prod = MIRBuilder.buildMul(Ty, Quot, MI.getOperand(2));
3219	MIRBuilder.buildSub(MI.getOperand(0), MI.getOperand(1), Prod);
3220	MI.eraseFromParent();
3221	return Legalized;
3222	}
3223	case TargetOpcode::G_SADDO:
3224	case TargetOpcode::G_SSUBO:
3225	return lowerSADDO_SSUBO(MI);
3226	case TargetOpcode::G_UMULH:
3227	case TargetOpcode::G_SMULH:
3228	return lowerSMULH_UMULH(MI);
3229	case TargetOpcode::G_SMULO:
3230	case TargetOpcode::G_UMULO: {
3231	// Generate G_UMULH/G_SMULH to check for overflow and a normal G_MUL for the
3232	// result.
3233	Register Res = MI.getOperand(0).getReg();
3234	Register Overflow = MI.getOperand(1).getReg();
3235	Register LHS = MI.getOperand(2).getReg();
3236	Register RHS = MI.getOperand(3).getReg();
3237	LLT Ty = MRI.getType(Res);
3238
3239	unsigned Opcode = MI.getOpcode() == TargetOpcode::G_SMULO
3240	? TargetOpcode::G_SMULH
3241	: TargetOpcode::G_UMULH;
3242
3243	Observer.changingInstr(MI);
3244	const auto &TII = MIRBuilder.getTII();
3245	MI.setDesc(TII.get(TargetOpcode::G_MUL));
3246	MI.RemoveOperand(1);
3247	Observer.changedInstr(MI);
3248
3249	auto HiPart = MIRBuilder.buildInstr(Opcode, {Ty}, {LHS, RHS});
3250	auto Zero = MIRBuilder.buildConstant(Ty, 0);
3251
3252	// Move insert point forward so we can use the Res register if needed.
3253	MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
3254
3255	// For signed multiply, overflow is detected by checking:
3256	// (hi != (lo >> bitwidth-1))
3257	if (Opcode == TargetOpcode::G_SMULH) {
3258	auto ShiftAmt = MIRBuilder.buildConstant(Ty, Ty.getSizeInBits() - 1);
3259	auto Shifted = MIRBuilder.buildAShr(Ty, Res, ShiftAmt);
3260	MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Shifted);
3261	} else {
3262	MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Zero);
3263	}
3264	return Legalized;
3265	}
3266	case TargetOpcode::G_FNEG: {
3267	Register Res = MI.getOperand(0).getReg();
3268	LLT Ty = MRI.getType(Res);
3269
3270	// TODO: Handle vector types once we are able to
3271	// represent them.
3272	if (Ty.isVector())
3273	return UnableToLegalize;
3274	auto SignMask =
3275	MIRBuilder.buildConstant(Ty, APInt::getSignMask(Ty.getSizeInBits()));
3276	Register SubByReg = MI.getOperand(1).getReg();
3277	MIRBuilder.buildXor(Res, SubByReg, SignMask);
3278	MI.eraseFromParent();
3279	return Legalized;
3280	}
3281	case TargetOpcode::G_FSUB: {
3282	Register Res = MI.getOperand(0).getReg();
3283	LLT Ty = MRI.getType(Res);
3284
3285	// Lower (G_FSUB LHS, RHS) to (G_FADD LHS, (G_FNEG RHS)).
3286	// First, check if G_FNEG is marked as Lower. If so, we may
3287	// end up with an infinite loop as G_FSUB is used to legalize G_FNEG.
3288	if (LI.getAction({G_FNEG, {Ty}}).Action == Lower)
3289	return UnableToLegalize;
3290	Register LHS = MI.getOperand(1).getReg();
3291	Register RHS = MI.getOperand(2).getReg();
3292	Register Neg = MRI.createGenericVirtualRegister(Ty);
3293	MIRBuilder.buildFNeg(Neg, RHS);
3294	MIRBuilder.buildFAdd(Res, LHS, Neg, MI.getFlags());
3295	MI.eraseFromParent();
3296	return Legalized;
3297	}
3298	case TargetOpcode::G_FMAD:
3299	return lowerFMad(MI);
3300	case TargetOpcode::G_FFLOOR:
3301	return lowerFFloor(MI);
3302	case TargetOpcode::G_INTRINSIC_ROUND:
3303	return lowerIntrinsicRound(MI);
3304	case TargetOpcode::G_INTRINSIC_ROUNDEVEN: {
3305	// Since round even is the assumed rounding mode for unconstrained FP
3306	// operations, rint and roundeven are the same operation.
3307	changeOpcode(MI, TargetOpcode::G_FRINT);
3308	return Legalized;
3309	}
3310	case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
3311	Register OldValRes = MI.getOperand(0).getReg();
3312	Register SuccessRes = MI.getOperand(1).getReg();
3313	Register Addr = MI.getOperand(2).getReg();
3314	Register CmpVal = MI.getOperand(3).getReg();
3315	Register NewVal = MI.getOperand(4).getReg();
3316	MIRBuilder.buildAtomicCmpXchg(OldValRes, Addr, CmpVal, NewVal,
3317	**MI.memoperands_begin());
3318	MIRBuilder.buildICmp(CmpInst::ICMP_EQ, SuccessRes, OldValRes, CmpVal);
3319	MI.eraseFromParent();
3320	return Legalized;
3321	}
3322	case TargetOpcode::G_LOAD:
3323	case TargetOpcode::G_SEXTLOAD:
3324	case TargetOpcode::G_ZEXTLOAD:
3325	return lowerLoad(cast<GAnyLoad>(MI));
3326	case TargetOpcode::G_STORE:
3327	return lowerStore(cast<GStore>(MI));
3328	case TargetOpcode::G_CTLZ_ZERO_UNDEF:
3329	case TargetOpcode::G_CTTZ_ZERO_UNDEF:
3330	case TargetOpcode::G_CTLZ:
3331	case TargetOpcode::G_CTTZ:
3332	case TargetOpcode::G_CTPOP:
3333	return lowerBitCount(MI);
3334	case G_UADDO: {
3335	Register Res = MI.getOperand(0).getReg();
3336	Register CarryOut = MI.getOperand(1).getReg();
3337	Register LHS = MI.getOperand(2).getReg();
3338	Register RHS = MI.getOperand(3).getReg();
3339
3340	MIRBuilder.buildAdd(Res, LHS, RHS);
3341	MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, RHS);
3342
3343	MI.eraseFromParent();
3344	return Legalized;
3345	}
3346	case G_UADDE: {
3347	Register Res = MI.getOperand(0).getReg();
3348	Register CarryOut = MI.getOperand(1).getReg();
3349	Register LHS = MI.getOperand(2).getReg();
3350	Register RHS = MI.getOperand(3).getReg();
3351	Register CarryIn = MI.getOperand(4).getReg();
3352	LLT Ty = MRI.getType(Res);
3353
3354	auto TmpRes = MIRBuilder.buildAdd(Ty, LHS, RHS);
3355	auto ZExtCarryIn = MIRBuilder.buildZExt(Ty, CarryIn);
3356	MIRBuilder.buildAdd(Res, TmpRes, ZExtCarryIn);
3357	MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, LHS);
3358
3359	MI.eraseFromParent();
3360	return Legalized;
3361	}
3362	case G_USUBO: {
3363	Register Res = MI.getOperand(0).getReg();
3364	Register BorrowOut = MI.getOperand(1).getReg();
3365	Register LHS = MI.getOperand(2).getReg();
3366	Register RHS = MI.getOperand(3).getReg();
3367
3368	MIRBuilder.buildSub(Res, LHS, RHS);
3369	MIRBuilder.buildICmp(CmpInst::ICMP_ULT, BorrowOut, LHS, RHS);
3370
3371	MI.eraseFromParent();
3372	return Legalized;
3373	}
3374	case G_USUBE: {
3375	Register Res = MI.getOperand(0).getReg();
3376	Register BorrowOut = MI.getOperand(1).getReg();
3377	Register LHS = MI.getOperand(2).getReg();
3378	Register RHS = MI.getOperand(3).getReg();
3379	Register BorrowIn = MI.getOperand(4).getReg();
3380	const LLT CondTy = MRI.getType(BorrowOut);
3381	const LLT Ty = MRI.getType(Res);
3382
3383	auto TmpRes = MIRBuilder.buildSub(Ty, LHS, RHS);
3384	auto ZExtBorrowIn = MIRBuilder.buildZExt(Ty, BorrowIn);
3385	MIRBuilder.buildSub(Res, TmpRes, ZExtBorrowIn);
3386
3387	auto LHS_EQ_RHS = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, CondTy, LHS, RHS);
3388	auto LHS_ULT_RHS = MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CondTy, LHS, RHS);
3389	MIRBuilder.buildSelect(BorrowOut, LHS_EQ_RHS, BorrowIn, LHS_ULT_RHS);
3390
3391	MI.eraseFromParent();
3392	return Legalized;
3393	}
3394	case G_UITOFP:
3395	return lowerUITOFP(MI);
3396	case G_SITOFP:
3397	return lowerSITOFP(MI);
3398	case G_FPTOUI:
3399	return lowerFPTOUI(MI);
3400	case G_FPTOSI:
3401	return lowerFPTOSI(MI);
3402	case G_FPTRUNC:
3403	return lowerFPTRUNC(MI);
3404	case G_FPOWI:
3405	return lowerFPOWI(MI);
3406	case G_SMIN:
3407	case G_SMAX:
3408	case G_UMIN:
3409	case G_UMAX:
3410	return lowerMinMax(MI);
3411	case G_FCOPYSIGN:
3412	return lowerFCopySign(MI);
3413	case G_FMINNUM:
3414	case G_FMAXNUM:
3415	return lowerFMinNumMaxNum(MI);
3416	case G_MERGE_VALUES:
3417	return lowerMergeValues(MI);
3418	case G_UNMERGE_VALUES:
3419	return lowerUnmergeValues(MI);
3420	case TargetOpcode::G_SEXT_INREG: {
3421	assert(MI.getOperand(2).isImm() && "Expected immediate")(static_cast <bool> (MI.getOperand(2).isImm() && "Expected immediate") ? void (0) : __assert_fail ("MI.getOperand(2).isImm() && \"Expected immediate\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 3421, __extension__ __PRETTY_FUNCTION__));
3422	int64_t SizeInBits = MI.getOperand(2).getImm();
3423
3424	Register DstReg = MI.getOperand(0).getReg();
3425	Register SrcReg = MI.getOperand(1).getReg();
3426	LLT DstTy = MRI.getType(DstReg);
3427	Register TmpRes = MRI.createGenericVirtualRegister(DstTy);
3428
3429	auto MIBSz = MIRBuilder.buildConstant(DstTy, DstTy.getScalarSizeInBits() - SizeInBits);
3430	MIRBuilder.buildShl(TmpRes, SrcReg, MIBSz->getOperand(0));
3431	MIRBuilder.buildAShr(DstReg, TmpRes, MIBSz->getOperand(0));
3432	MI.eraseFromParent();
3433	return Legalized;
3434	}
3435	case G_EXTRACT_VECTOR_ELT:
3436	case G_INSERT_VECTOR_ELT:
3437	return lowerExtractInsertVectorElt(MI);
3438	case G_SHUFFLE_VECTOR:
3439	return lowerShuffleVector(MI);
3440	case G_DYN_STACKALLOC:
3441	return lowerDynStackAlloc(MI);
3442	case G_EXTRACT:
3443	return lowerExtract(MI);
3444	case G_INSERT:
3445	return lowerInsert(MI);
3446	case G_BSWAP:
3447	return lowerBswap(MI);
3448	case G_BITREVERSE:
3449	return lowerBitreverse(MI);
3450	case G_READ_REGISTER:
3451	case G_WRITE_REGISTER:
3452	return lowerReadWriteRegister(MI);
3453	case G_UADDSAT:
3454	case G_USUBSAT: {
3455	// Try to make a reasonable guess about which lowering strategy to use. The
3456	// target can override this with custom lowering and calling the
3457	// implementation functions.
3458	LLT Ty = MRI.getType(MI.getOperand(0).getReg());
3459	if (LI.isLegalOrCustom({G_UMIN, Ty}))
3460	return lowerAddSubSatToMinMax(MI);
3461	return lowerAddSubSatToAddoSubo(MI);
3462	}
3463	case G_SADDSAT:
3464	case G_SSUBSAT: {
3465	LLT Ty = MRI.getType(MI.getOperand(0).getReg());
3466
3467	// FIXME: It would probably make more sense to see if G_SADDO is preferred,
3468	// since it's a shorter expansion. However, we would need to figure out the
3469	// preferred boolean type for the carry out for the query.
3470	if (LI.isLegalOrCustom({G_SMIN, Ty}) && LI.isLegalOrCustom({G_SMAX, Ty}))
3471	return lowerAddSubSatToMinMax(MI);
3472	return lowerAddSubSatToAddoSubo(MI);
3473	}
3474	case G_SSHLSAT:
3475	case G_USHLSAT:
3476	return lowerShlSat(MI);
3477	case G_ABS:
3478	return lowerAbsToAddXor(MI);
3479	case G_SELECT:
3480	return lowerSelect(MI);
3481	case G_SDIVREM:
3482	case G_UDIVREM:
3483	return lowerDIVREM(MI);
3484	case G_FSHL:
3485	case G_FSHR:
3486	return lowerFunnelShift(MI);
3487	case G_ROTL:
3488	case G_ROTR:
3489	return lowerRotate(MI);
3490	case G_ISNAN:
3491	return lowerIsNaN(MI);
3492	GISEL_VECREDUCE_CASES_NONSEQcase TargetOpcode::G_VECREDUCE_FADD: case TargetOpcode::G_VECREDUCE_FMUL : case TargetOpcode::G_VECREDUCE_FMAX: case TargetOpcode::G_VECREDUCE_FMIN : case TargetOpcode::G_VECREDUCE_ADD: case TargetOpcode::G_VECREDUCE_MUL : case TargetOpcode::G_VECREDUCE_AND: case TargetOpcode::G_VECREDUCE_OR : case TargetOpcode::G_VECREDUCE_XOR: case TargetOpcode::G_VECREDUCE_SMAX : case TargetOpcode::G_VECREDUCE_SMIN: case TargetOpcode::G_VECREDUCE_UMAX : case TargetOpcode::G_VECREDUCE_UMIN:
3493	return lowerVectorReduction(MI);
3494	}
3495	}
3496
3497	Align LegalizerHelper::getStackTemporaryAlignment(LLT Ty,
3498	Align MinAlign) const {
3499	// FIXME: We're missing a way to go back from LLT to llvm::Type to query the
3500	// datalayout for the preferred alignment. Also there should be a target hook
3501	// for this to allow targets to reduce the alignment and ignore the
3502	// datalayout. e.g. AMDGPU should always use a 4-byte alignment, regardless of
3503	// the type.
3504	return std::max(Align(PowerOf2Ceil(Ty.getSizeInBytes())), MinAlign);
3505	}
3506
3507	MachineInstrBuilder
3508	LegalizerHelper::createStackTemporary(TypeSize Bytes, Align Alignment,
3509	MachinePointerInfo &PtrInfo) {
3510	MachineFunction &MF = MIRBuilder.getMF();
3511	const DataLayout &DL = MIRBuilder.getDataLayout();
3512	int FrameIdx = MF.getFrameInfo().CreateStackObject(Bytes, Alignment, false);
3513
3514	unsigned AddrSpace = DL.getAllocaAddrSpace();
3515	LLT FramePtrTy = LLT::pointer(AddrSpace, DL.getPointerSizeInBits(AddrSpace));
3516
3517	PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIdx);
3518	return MIRBuilder.buildFrameIndex(FramePtrTy, FrameIdx);
3519	}
3520
3521	static Register clampDynamicVectorIndex(MachineIRBuilder &B, Register IdxReg,
3522	LLT VecTy) {
3523	int64_t IdxVal;
3524	if (mi_match(IdxReg, *B.getMRI(), m_ICst(IdxVal)))
3525	return IdxReg;
3526
3527	LLT IdxTy = B.getMRI()->getType(IdxReg);
3528	unsigned NElts = VecTy.getNumElements();
3529	if (isPowerOf2_32(NElts)) {
3530	APInt Imm = APInt::getLowBitsSet(IdxTy.getSizeInBits(), Log2_32(NElts));
3531	return B.buildAnd(IdxTy, IdxReg, B.buildConstant(IdxTy, Imm)).getReg(0);
3532	}
3533
3534	return B.buildUMin(IdxTy, IdxReg, B.buildConstant(IdxTy, NElts - 1))
3535	.getReg(0);
3536	}
3537
3538	Register LegalizerHelper::getVectorElementPointer(Register VecPtr, LLT VecTy,
3539	Register Index) {
3540	LLT EltTy = VecTy.getElementType();
3541
3542	// Calculate the element offset and add it to the pointer.
3543	unsigned EltSize = EltTy.getSizeInBits() / 8; // FIXME: should be ABI size.
3544	assert(EltSize * 8 == EltTy.getSizeInBits() &&(static_cast <bool> (EltSize * 8 == EltTy.getSizeInBits () && "Converting bits to bytes lost precision") ? void (0) : __assert_fail ("EltSize * 8 == EltTy.getSizeInBits() && \"Converting bits to bytes lost precision\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 3545, __extension__ __PRETTY_FUNCTION__))
3545	"Converting bits to bytes lost precision")(static_cast <bool> (EltSize * 8 == EltTy.getSizeInBits () && "Converting bits to bytes lost precision") ? void (0) : __assert_fail ("EltSize * 8 == EltTy.getSizeInBits() && \"Converting bits to bytes lost precision\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 3545, __extension__ __PRETTY_FUNCTION__));
3546
3547	Index = clampDynamicVectorIndex(MIRBuilder, Index, VecTy);
3548
3549	LLT IdxTy = MRI.getType(Index);
3550	auto Mul = MIRBuilder.buildMul(IdxTy, Index,
3551	MIRBuilder.buildConstant(IdxTy, EltSize));
3552
3553	LLT PtrTy = MRI.getType(VecPtr);
3554	return MIRBuilder.buildPtrAdd(PtrTy, VecPtr, Mul).getReg(0);
3555	}
3556
3557	LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorImplicitDef(
3558	MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) {
3559	Register DstReg = MI.getOperand(0).getReg();
3560	LLT DstTy = MRI.getType(DstReg);
3561	LLT LCMTy = getLCMType(DstTy, NarrowTy);
3562
3563	unsigned NumParts = LCMTy.getSizeInBits() / NarrowTy.getSizeInBits();
3564
3565	auto NewUndef = MIRBuilder.buildUndef(NarrowTy);
3566	SmallVector<Register, 8> Parts(NumParts, NewUndef.getReg(0));
3567
3568	buildWidenedRemergeToDst(DstReg, LCMTy, Parts);
3569	MI.eraseFromParent();
3570	return Legalized;
3571	}
3572
3573	// Handle splitting vector operations which need to have the same number of
3574	// elements in each type index, but each type index may have a different element
3575	// type.
3576	//
3577	// e.g. <4 x s64> = G_SHL <4 x s64>, <4 x s32> ->
3578	// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
3579	// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
3580	//
3581	// Also handles some irregular breakdown cases, e.g.
3582	// e.g. <3 x s64> = G_SHL <3 x s64>, <3 x s32> ->
3583	// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
3584	// s64 = G_SHL s64, s32
3585	LegalizerHelper::LegalizeResult
3586	LegalizerHelper::fewerElementsVectorMultiEltType(
3587	MachineInstr &MI, unsigned TypeIdx, LLT NarrowTyArg) {
3588	if (TypeIdx != 0)
3589	return UnableToLegalize;
3590
3591	const LLT NarrowTy0 = NarrowTyArg;
3592	const Register DstReg = MI.getOperand(0).getReg();
3593	LLT DstTy = MRI.getType(DstReg);
3594	LLT LeftoverTy0;
3595
3596	// All of the operands need to have the same number of elements, so if we can
3597	// determine a type breakdown for the result type, we can for all of the
3598	// source types.
3599	int NumParts = getNarrowTypeBreakDown(DstTy, NarrowTy0, LeftoverTy0).first;
3600	if (NumParts < 0)
3601	return UnableToLegalize;
3602
3603	SmallVector<MachineInstrBuilder, 4> NewInsts;
3604
3605	SmallVector<Register, 4> DstRegs, LeftoverDstRegs;
3606	SmallVector<Register, 4> PartRegs, LeftoverRegs;
3607
3608	for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
3609	Register SrcReg = MI.getOperand(I).getReg();
3610	LLT SrcTyI = MRI.getType(SrcReg);
3611	const auto NewEC = NarrowTy0.isVector() ? NarrowTy0.getElementCount()
3612	: ElementCount::getFixed(1);
3613	LLT NarrowTyI = LLT::scalarOrVector(NewEC, SrcTyI.getScalarType());
3614	LLT LeftoverTyI;
3615
3616	// Split this operand into the requested typed registers, and any leftover
3617	// required to reproduce the original type.
3618	if (!extractParts(SrcReg, SrcTyI, NarrowTyI, LeftoverTyI, PartRegs,
3619	LeftoverRegs))
3620	return UnableToLegalize;
3621
3622	if (I == 1) {
3623	// For the first operand, create an instruction for each part and setup
3624	// the result.
3625	for (Register PartReg : PartRegs) {
3626	Register PartDstReg = MRI.createGenericVirtualRegister(NarrowTy0);
3627	NewInsts.push_back(MIRBuilder.buildInstrNoInsert(MI.getOpcode())
3628	.addDef(PartDstReg)
3629	.addUse(PartReg));
3630	DstRegs.push_back(PartDstReg);
3631	}
3632
3633	for (Register LeftoverReg : LeftoverRegs) {
3634	Register PartDstReg = MRI.createGenericVirtualRegister(LeftoverTy0);
3635	NewInsts.push_back(MIRBuilder.buildInstrNoInsert(MI.getOpcode())
3636	.addDef(PartDstReg)
3637	.addUse(LeftoverReg));
3638	LeftoverDstRegs.push_back(PartDstReg);
3639	}
3640	} else {
3641	assert(NewInsts.size() == PartRegs.size() + LeftoverRegs.size())(static_cast <bool> (NewInsts.size() == PartRegs.size() + LeftoverRegs.size()) ? void (0) : __assert_fail ("NewInsts.size() == PartRegs.size() + LeftoverRegs.size()" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 3641, __extension__ __PRETTY_FUNCTION__));
3642
3643	// Add the newly created operand splits to the existing instructions. The
3644	// odd-sized pieces are ordered after the requested NarrowTyArg sized
3645	// pieces.
3646	unsigned InstCount = 0;
3647	for (unsigned J = 0, JE = PartRegs.size(); J != JE; ++J)
3648	NewInsts[InstCount++].addUse(PartRegs[J]);
3649	for (unsigned J = 0, JE = LeftoverRegs.size(); J != JE; ++J)
3650	NewInsts[InstCount++].addUse(LeftoverRegs[J]);
3651	}
3652
3653	PartRegs.clear();
3654	LeftoverRegs.clear();
3655	}
3656
3657	// Insert the newly built operations and rebuild the result register.
3658	for (auto &MIB : NewInsts)
3659	MIRBuilder.insertInstr(MIB);
3660
3661	insertParts(DstReg, DstTy, NarrowTy0, DstRegs, LeftoverTy0, LeftoverDstRegs);
3662
3663	MI.eraseFromParent();
3664	return Legalized;
3665	}
3666
3667	LegalizerHelper::LegalizeResult
3668	LegalizerHelper::fewerElementsVectorCasts(MachineInstr &MI, unsigned TypeIdx,
3669	LLT NarrowTy) {
3670	if (TypeIdx != 0)
3671	return UnableToLegalize;
3672
3673	Register DstReg = MI.getOperand(0).getReg();
3674	Register SrcReg = MI.getOperand(1).getReg();
3675	LLT DstTy = MRI.getType(DstReg);
3676	LLT SrcTy = MRI.getType(SrcReg);
3677
3678	LLT NarrowTy0 = NarrowTy;
3679	LLT NarrowTy1;
3680	unsigned NumParts;
3681
3682	if (NarrowTy.isVector()) {
3683	// Uneven breakdown not handled.
3684	NumParts = DstTy.getNumElements() / NarrowTy.getNumElements();
3685	if (NumParts * NarrowTy.getNumElements() != DstTy.getNumElements())
3686	return UnableToLegalize;
3687
3688	NarrowTy1 = LLT::vector(NarrowTy.getElementCount(), SrcTy.getElementType());
3689	} else {
3690	NumParts = DstTy.getNumElements();
3691	NarrowTy1 = SrcTy.getElementType();
3692	}
3693
3694	SmallVector<Register, 4> SrcRegs, DstRegs;
3695	extractParts(SrcReg, NarrowTy1, NumParts, SrcRegs);
3696
3697	for (unsigned I = 0; I < NumParts; ++I) {
3698	Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
3699	MachineInstr *NewInst =
3700	MIRBuilder.buildInstr(MI.getOpcode(), {DstReg}, {SrcRegs[I]});
3701
3702	NewInst->setFlags(MI.getFlags());
3703	DstRegs.push_back(DstReg);
3704	}
3705
3706	if (NarrowTy.isVector())
3707	MIRBuilder.buildConcatVectors(DstReg, DstRegs);
3708	else
3709	MIRBuilder.buildBuildVector(DstReg, DstRegs);
3710
3711	MI.eraseFromParent();
3712	return Legalized;
3713	}
3714
3715	LegalizerHelper::LegalizeResult
3716	LegalizerHelper::fewerElementsVectorCmp(MachineInstr &MI, unsigned TypeIdx,
3717	LLT NarrowTy) {
3718	Register DstReg = MI.getOperand(0).getReg();
3719	Register Src0Reg = MI.getOperand(2).getReg();
3720	LLT DstTy = MRI.getType(DstReg);
3721	LLT SrcTy = MRI.getType(Src0Reg);
3722
3723	unsigned NumParts;
3724	LLT NarrowTy0, NarrowTy1;
3725
3726	if (TypeIdx == 0) {
3727	unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
3728	unsigned OldElts = DstTy.getNumElements();
3729
3730	NarrowTy0 = NarrowTy;
3731	NumParts = NarrowTy.isVector() ? (OldElts / NewElts) : DstTy.getNumElements();
3732	NarrowTy1 = NarrowTy.isVector() ? LLT::vector(NarrowTy.getElementCount(),
3733	SrcTy.getScalarSizeInBits())
3734	: SrcTy.getElementType();
3735
3736	} else {
3737	unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
3738	unsigned OldElts = SrcTy.getNumElements();
3739
3740	NumParts = NarrowTy.isVector() ? (OldElts / NewElts) :
3741	NarrowTy.getNumElements();
3742	NarrowTy0 =
3743	LLT::vector(NarrowTy.getElementCount(), DstTy.getScalarSizeInBits());
3744	NarrowTy1 = NarrowTy;
3745	}
3746
3747	// FIXME: Don't know how to handle the situation where the small vectors
3748	// aren't all the same size yet.
3749	if (NarrowTy1.isVector() &&
3750	NarrowTy1.getNumElements() * NumParts != DstTy.getNumElements())
3751	return UnableToLegalize;
3752
3753	CmpInst::Predicate Pred
3754	= static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
3755
3756	SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs;
3757	extractParts(MI.getOperand(2).getReg(), NarrowTy1, NumParts, Src1Regs);
3758	extractParts(MI.getOperand(3).getReg(), NarrowTy1, NumParts, Src2Regs);
3759
3760	for (unsigned I = 0; I < NumParts; ++I) {
3761	Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
3762	DstRegs.push_back(DstReg);
3763
3764	if (MI.getOpcode() == TargetOpcode::G_ICMP)
3765	MIRBuilder.buildICmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]);
3766	else {
3767	MachineInstr *NewCmp
3768	= MIRBuilder.buildFCmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]);
3769	NewCmp->setFlags(MI.getFlags());
3770	}
3771	}
3772
3773	if (NarrowTy1.isVector())
3774	MIRBuilder.buildConcatVectors(DstReg, DstRegs);
3775	else
3776	MIRBuilder.buildBuildVector(DstReg, DstRegs);
3777
3778	MI.eraseFromParent();
3779	return Legalized;
3780	}
3781
3782	LegalizerHelper::LegalizeResult
3783	LegalizerHelper::fewerElementsVectorSelect(MachineInstr &MI, unsigned TypeIdx,
3784	LLT NarrowTy) {
3785	Register DstReg = MI.getOperand(0).getReg();
3786	Register CondReg = MI.getOperand(1).getReg();
3787
3788	unsigned NumParts = 0;
3789	LLT NarrowTy0, NarrowTy1;
3790
3791	LLT DstTy = MRI.getType(DstReg);
3792	LLT CondTy = MRI.getType(CondReg);
3793	unsigned Size = DstTy.getSizeInBits();
3794
3795	assert(TypeIdx == 0 \|\| CondTy.isVector())(static_cast <bool> (TypeIdx == 0 \|\| CondTy.isVector()) ? void (0) : __assert_fail ("TypeIdx == 0 \|\| CondTy.isVector()" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 3795, __extension__ __PRETTY_FUNCTION__));
3796
3797	if (TypeIdx == 0) {
3798	NarrowTy0 = NarrowTy;
3799	NarrowTy1 = CondTy;
3800
3801	unsigned NarrowSize = NarrowTy0.getSizeInBits();
3802	// FIXME: Don't know how to handle the situation where the small vectors
3803	// aren't all the same size yet.
3804	if (Size % NarrowSize != 0)
3805	return UnableToLegalize;
3806
3807	NumParts = Size / NarrowSize;
3808
3809	// Need to break down the condition type
3810	if (CondTy.isVector()) {
3811	if (CondTy.getNumElements() == NumParts)
3812	NarrowTy1 = CondTy.getElementType();
3813	else
3814	NarrowTy1 =
3815	LLT::vector(CondTy.getElementCount().divideCoefficientBy(NumParts),
3816	CondTy.getScalarSizeInBits());
3817	}
3818	} else {
3819	NumParts = CondTy.getNumElements();
3820	if (NarrowTy.isVector()) {
3821	// TODO: Handle uneven breakdown.
3822	if (NumParts * NarrowTy.getNumElements() != CondTy.getNumElements())
3823	return UnableToLegalize;
3824
3825	return UnableToLegalize;
3826	} else {
3827	NarrowTy0 = DstTy.getElementType();
3828	NarrowTy1 = NarrowTy;
3829	}
3830	}
3831
3832	SmallVector<Register, 2> DstRegs, Src0Regs, Src1Regs, Src2Regs;
3833	if (CondTy.isVector())
3834	extractParts(MI.getOperand(1).getReg(), NarrowTy1, NumParts, Src0Regs);
3835
3836	extractParts(MI.getOperand(2).getReg(), NarrowTy0, NumParts, Src1Regs);
3837	extractParts(MI.getOperand(3).getReg(), NarrowTy0, NumParts, Src2Regs);
3838
3839	for (unsigned i = 0; i < NumParts; ++i) {
3840	Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
3841	MIRBuilder.buildSelect(DstReg, CondTy.isVector() ? Src0Regs[i] : CondReg,
3842	Src1Regs[i], Src2Regs[i]);
3843	DstRegs.push_back(DstReg);
3844	}
3845
3846	if (NarrowTy0.isVector())
3847	MIRBuilder.buildConcatVectors(DstReg, DstRegs);
3848	else
3849	MIRBuilder.buildBuildVector(DstReg, DstRegs);
3850
3851	MI.eraseFromParent();
3852	return Legalized;
3853	}
3854
3855	LegalizerHelper::LegalizeResult
3856	LegalizerHelper::fewerElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx,
3857	LLT NarrowTy) {
3858	const Register DstReg = MI.getOperand(0).getReg();
3859	LLT PhiTy = MRI.getType(DstReg);
3860	LLT LeftoverTy;
3861
3862	// All of the operands need to have the same number of elements, so if we can
3863	// determine a type breakdown for the result type, we can for all of the
3864	// source types.
3865	int NumParts, NumLeftover;
3866	std::tie(NumParts, NumLeftover)
3867	= getNarrowTypeBreakDown(PhiTy, NarrowTy, LeftoverTy);
3868	if (NumParts < 0)
3869	return UnableToLegalize;
3870
3871	SmallVector<Register, 4> DstRegs, LeftoverDstRegs;
3872	SmallVector<MachineInstrBuilder, 4> NewInsts;
3873
3874	const int TotalNumParts = NumParts + NumLeftover;
3875
3876	// Insert the new phis in the result block first.
3877	for (int I = 0; I != TotalNumParts; ++I) {
3878	LLT Ty = I < NumParts ? NarrowTy : LeftoverTy;
3879	Register PartDstReg = MRI.createGenericVirtualRegister(Ty);
3880	NewInsts.push_back(MIRBuilder.buildInstr(TargetOpcode::G_PHI)
3881	.addDef(PartDstReg));
3882	if (I < NumParts)
3883	DstRegs.push_back(PartDstReg);
3884	else
3885	LeftoverDstRegs.push_back(PartDstReg);
3886	}
3887
3888	MachineBasicBlock *MBB = MI.getParent();
3889	MIRBuilder.setInsertPt(*MBB, MBB->getFirstNonPHI());
3890	insertParts(DstReg, PhiTy, NarrowTy, DstRegs, LeftoverTy, LeftoverDstRegs);
3891
3892	SmallVector<Register, 4> PartRegs, LeftoverRegs;
3893
3894	// Insert code to extract the incoming values in each predecessor block.
3895	for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
3896	PartRegs.clear();
3897	LeftoverRegs.clear();
3898
3899	Register SrcReg = MI.getOperand(I).getReg();
3900	MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
3901	MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
3902
3903	LLT Unused;
3904	if (!extractParts(SrcReg, PhiTy, NarrowTy, Unused, PartRegs,
3905	LeftoverRegs))
3906	return UnableToLegalize;
3907
3908	// Add the newly created operand splits to the existing instructions. The
3909	// odd-sized pieces are ordered after the requested NarrowTyArg sized
3910	// pieces.
3911	for (int J = 0; J != TotalNumParts; ++J) {
3912	MachineInstrBuilder MIB = NewInsts[J];
3913	MIB.addUse(J < NumParts ? PartRegs[J] : LeftoverRegs[J - NumParts]);
3914	MIB.addMBB(&OpMBB);
3915	}
3916	}
3917
3918	MI.eraseFromParent();
3919	return Legalized;
3920	}
3921
3922	LegalizerHelper::LegalizeResult
3923	LegalizerHelper::fewerElementsVectorUnmergeValues(MachineInstr &MI,
3924	unsigned TypeIdx,
3925	LLT NarrowTy) {
3926	if (TypeIdx != 1)
3927	return UnableToLegalize;
3928
3929	const int NumDst = MI.getNumOperands() - 1;
3930	const Register SrcReg = MI.getOperand(NumDst).getReg();
3931	LLT SrcTy = MRI.getType(SrcReg);
3932
3933	LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
3934
3935	// TODO: Create sequence of extracts.
3936	if (DstTy == NarrowTy)
3937	return UnableToLegalize;
3938
3939	LLT GCDTy = getGCDType(SrcTy, NarrowTy);
3940	if (DstTy == GCDTy) {
3941	// This would just be a copy of the same unmerge.
3942	// TODO: Create extracts, pad with undef and create intermediate merges.
3943	return UnableToLegalize;
3944	}
3945
3946	auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg);
3947	const int NumUnmerge = Unmerge->getNumOperands() - 1;
3948	const int PartsPerUnmerge = NumDst / NumUnmerge;
3949
3950	for (int I = 0; I != NumUnmerge; ++I) {
3951	auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES);
3952
3953	for (int J = 0; J != PartsPerUnmerge; ++J)
3954	MIB.addDef(MI.getOperand(I * PartsPerUnmerge + J).getReg());
3955	MIB.addUse(Unmerge.getReg(I));
3956	}
3957
3958	MI.eraseFromParent();
3959	return Legalized;
3960	}
3961
3962	LegalizerHelper::LegalizeResult
3963	LegalizerHelper::fewerElementsVectorMulo(MachineInstr &MI, unsigned TypeIdx,
3964	LLT NarrowTy) {
3965	Register Result = MI.getOperand(0).getReg();
3966	Register Overflow = MI.getOperand(1).getReg();
3967	Register LHS = MI.getOperand(2).getReg();
3968	Register RHS = MI.getOperand(3).getReg();
3969
3970	LLT SrcTy = MRI.getType(LHS);
3971	if (!SrcTy.isVector())
3972	return UnableToLegalize;
3973
3974	LLT ElementType = SrcTy.getElementType();
3975	LLT OverflowElementTy = MRI.getType(Overflow).getElementType();
3976	const ElementCount NumResult = SrcTy.getElementCount();
3977	LLT GCDTy = getGCDType(SrcTy, NarrowTy);
3978
3979	// Unmerge the operands to smaller parts of GCD type.
3980	auto UnmergeLHS = MIRBuilder.buildUnmerge(GCDTy, LHS);
3981	auto UnmergeRHS = MIRBuilder.buildUnmerge(GCDTy, RHS);
3982
3983	const int NumOps = UnmergeLHS->getNumOperands() - 1;
3984	const ElementCount PartsPerUnmerge = NumResult.divideCoefficientBy(NumOps);
3985	LLT OverflowTy = LLT::scalarOrVector(PartsPerUnmerge, OverflowElementTy);
3986	LLT ResultTy = LLT::scalarOrVector(PartsPerUnmerge, ElementType);
3987
3988	// Perform the operation over unmerged parts.
3989	SmallVector<Register, 8> ResultParts;
3990	SmallVector<Register, 8> OverflowParts;
3991	for (int I = 0; I != NumOps; ++I) {
3992	Register Operand1 = UnmergeLHS->getOperand(I).getReg();
3993	Register Operand2 = UnmergeRHS->getOperand(I).getReg();
3994	auto PartMul = MIRBuilder.buildInstr(MI.getOpcode(), {ResultTy, OverflowTy},
3995	{Operand1, Operand2});
3996	ResultParts.push_back(PartMul->getOperand(0).getReg());
3997	OverflowParts.push_back(PartMul->getOperand(1).getReg());
3998	}
3999
4000	LLT ResultLCMTy = buildLCMMergePieces(SrcTy, NarrowTy, GCDTy, ResultParts);
4001	LLT OverflowLCMTy =
4002	LLT::scalarOrVector(ResultLCMTy.getElementCount(), OverflowElementTy);
4003
4004	// Recombine the pieces to the original result and overflow registers.
4005	buildWidenedRemergeToDst(Result, ResultLCMTy, ResultParts);
4006	buildWidenedRemergeToDst(Overflow, OverflowLCMTy, OverflowParts);
4007	MI.eraseFromParent();
4008	return Legalized;
4009	}
4010
4011	// Handle FewerElementsVector a G_BUILD_VECTOR or G_CONCAT_VECTORS that produces
4012	// a vector
4013	//
4014	// Create a G_BUILD_VECTOR or G_CONCAT_VECTORS of NarrowTy pieces, padding with
4015	// undef as necessary.
4016	//
4017	// %3:_(<3 x s16>) = G_BUILD_VECTOR %0, %1, %2
4018	// -> <2 x s16>
4019	//
4020	// %4:_(s16) = G_IMPLICIT_DEF
4021	// %5:_(<2 x s16>) = G_BUILD_VECTOR %0, %1
4022	// %6:_(<2 x s16>) = G_BUILD_VECTOR %2, %4
4023	// %7:_(<2 x s16>) = G_IMPLICIT_DEF
4024	// %8:_(<6 x s16>) = G_CONCAT_VECTORS %5, %6, %7
4025	// %3:_(<3 x s16>), %8:_(<3 x s16>) = G_UNMERGE_VALUES %8
4026	LegalizerHelper::LegalizeResult
4027	LegalizerHelper::fewerElementsVectorMerge(MachineInstr &MI, unsigned TypeIdx,
4028	LLT NarrowTy) {
4029	Register DstReg = MI.getOperand(0).getReg();
4030	LLT DstTy = MRI.getType(DstReg);
4031	LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
4032	LLT GCDTy = getGCDType(getGCDType(SrcTy, NarrowTy), DstTy);
4033
4034	// Break into a common type
4035	SmallVector<Register, 16> Parts;
4036	for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I)
4037	extractGCDType(Parts, GCDTy, MI.getOperand(I).getReg());
4038
4039	// Build the requested new merge, padding with undef.
4040	LLT LCMTy = buildLCMMergePieces(DstTy, NarrowTy, GCDTy, Parts,
4041	TargetOpcode::G_ANYEXT);
4042
4043	// Pack into the original result register.
4044	buildWidenedRemergeToDst(DstReg, LCMTy, Parts);
4045
4046	MI.eraseFromParent();
4047	return Legalized;
4048	}
4049
4050	LegalizerHelper::LegalizeResult
4051	LegalizerHelper::fewerElementsVectorExtractInsertVectorElt(MachineInstr &MI,
4052	unsigned TypeIdx,
4053	LLT NarrowVecTy) {
4054	Register DstReg = MI.getOperand(0).getReg();
4055	Register SrcVec = MI.getOperand(1).getReg();
4056	Register InsertVal;
4057	bool IsInsert = MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT;
4058
4059	assert((IsInsert ? TypeIdx == 0 : TypeIdx == 1) && "not a vector type index")(static_cast <bool> ((IsInsert ? TypeIdx == 0 : TypeIdx == 1) && "not a vector type index") ? void (0) : __assert_fail ("(IsInsert ? TypeIdx == 0 : TypeIdx == 1) && \"not a vector type index\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 4059, __extension__ __PRETTY_FUNCTION__));
4060	if (IsInsert)
4061	InsertVal = MI.getOperand(2).getReg();
4062
4063	Register Idx = MI.getOperand(MI.getNumOperands() - 1).getReg();
4064
4065	// TODO: Handle total scalarization case.
4066	if (!NarrowVecTy.isVector())
4067	return UnableToLegalize;
4068
4069	LLT VecTy = MRI.getType(SrcVec);
4070
4071	// If the index is a constant, we can really break this down as you would
4072	// expect, and index into the target size pieces.
4073	int64_t IdxVal;
4074	auto MaybeCst =
4075	getConstantVRegValWithLookThrough(Idx, MRI, /LookThroughInstrs/ true,
4076	/HandleFConstants/ false);
4077	if (MaybeCst) {
4078	IdxVal = MaybeCst->Value.getSExtValue();
4079	// Avoid out of bounds indexing the pieces.
4080	if (IdxVal >= VecTy.getNumElements()) {
4081	MIRBuilder.buildUndef(DstReg);
4082	MI.eraseFromParent();
4083	return Legalized;
4084	}
4085
4086	SmallVector<Register, 8> VecParts;
4087	LLT GCDTy = extractGCDType(VecParts, VecTy, NarrowVecTy, SrcVec);
4088
4089	// Build a sequence of NarrowTy pieces in VecParts for this operand.
4090	LLT LCMTy = buildLCMMergePieces(VecTy, NarrowVecTy, GCDTy, VecParts,
4091	TargetOpcode::G_ANYEXT);
4092
4093	unsigned NewNumElts = NarrowVecTy.getNumElements();
4094
4095	LLT IdxTy = MRI.getType(Idx);
4096	int64_t PartIdx = IdxVal / NewNumElts;
4097	auto NewIdx =
4098	MIRBuilder.buildConstant(IdxTy, IdxVal - NewNumElts * PartIdx);
4099
4100	if (IsInsert) {
4101	LLT PartTy = MRI.getType(VecParts[PartIdx]);
4102
4103	// Use the adjusted index to insert into one of the subvectors.
4104	auto InsertPart = MIRBuilder.buildInsertVectorElement(
4105	PartTy, VecParts[PartIdx], InsertVal, NewIdx);
4106	VecParts[PartIdx] = InsertPart.getReg(0);
4107
4108	// Recombine the inserted subvector with the others to reform the result
4109	// vector.
4110	buildWidenedRemergeToDst(DstReg, LCMTy, VecParts);
4111	} else {
4112	MIRBuilder.buildExtractVectorElement(DstReg, VecParts[PartIdx], NewIdx);
4113	}
4114
4115	MI.eraseFromParent();
4116	return Legalized;
4117	}
4118
4119	// With a variable index, we can't perform the operation in a smaller type, so
4120	// we're forced to expand this.
4121	//
4122	// TODO: We could emit a chain of compare/select to figure out which piece to
4123	// index.
4124	return lowerExtractInsertVectorElt(MI);
4125	}
4126
4127	LegalizerHelper::LegalizeResult
4128	LegalizerHelper::reduceLoadStoreWidth(GLoadStore &LdStMI, unsigned TypeIdx,
4129	LLT NarrowTy) {
4130	// FIXME: Don't know how to handle secondary types yet.
4131	if (TypeIdx != 0)
4132	return UnableToLegalize;
4133
4134	// This implementation doesn't work for atomics. Give up instead of doing
4135	// something invalid.
4136	if (LdStMI.isAtomic())
4137	return UnableToLegalize;
4138
4139	bool IsLoad = isa<GLoad>(LdStMI);
4140	Register ValReg = LdStMI.getReg(0);
4141	Register AddrReg = LdStMI.getPointerReg();
4142	LLT ValTy = MRI.getType(ValReg);
4143
4144	// FIXME: Do we need a distinct NarrowMemory legalize action?
4145	if (ValTy.getSizeInBits() != 8 * LdStMI.getMemSize()) {
4146	LLVM_DEBUG(dbgs() << "Can't narrow extload/truncstore\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizer")) { dbgs() << "Can't narrow extload/truncstore\n" ; } } while (false);
4147	return UnableToLegalize;
4148	}
4149
4150	int NumParts = -1;
4151	int NumLeftover = -1;
4152	LLT LeftoverTy;
4153	SmallVector<Register, 8> NarrowRegs, NarrowLeftoverRegs;
4154	if (IsLoad) {
4155	std::tie(NumParts, NumLeftover) = getNarrowTypeBreakDown(ValTy, NarrowTy, LeftoverTy);
4156	} else {
4157	if (extractParts(ValReg, ValTy, NarrowTy, LeftoverTy, NarrowRegs,
4158	NarrowLeftoverRegs)) {
4159	NumParts = NarrowRegs.size();
4160	NumLeftover = NarrowLeftoverRegs.size();
	Value stored to 'NumLeftover' is never read
4161	}
4162	}
4163
4164	if (NumParts == -1)
4165	return UnableToLegalize;
4166
4167	LLT PtrTy = MRI.getType(AddrReg);
4168	const LLT OffsetTy = LLT::scalar(PtrTy.getSizeInBits());
4169
4170	unsigned TotalSize = ValTy.getSizeInBits();
4171
4172	// Split the load/store into PartTy sized pieces starting at Offset. If this
4173	// is a load, return the new registers in ValRegs. For a store, each elements
4174	// of ValRegs should be PartTy. Returns the next offset that needs to be
4175	// handled.
4176	auto MMO = LdStMI.getMMO();
4177	auto splitTypePieces = [=](LLT PartTy, SmallVectorImpl<Register> &ValRegs,
4178	unsigned Offset) -> unsigned {
4179	MachineFunction &MF = MIRBuilder.getMF();
4180	unsigned PartSize = PartTy.getSizeInBits();
4181	for (unsigned Idx = 0, E = NumParts; Idx != E && Offset < TotalSize;
4182	Offset += PartSize, ++Idx) {
4183	unsigned ByteOffset = Offset / 8;
4184	Register NewAddrReg;
4185
4186	MIRBuilder.materializePtrAdd(NewAddrReg, AddrReg, OffsetTy, ByteOffset);
4187
4188	MachineMemOperand *NewMMO =
4189	MF.getMachineMemOperand(&MMO, ByteOffset, PartTy);
4190
4191	if (IsLoad) {
4192	Register Dst = MRI.createGenericVirtualRegister(PartTy);
4193	ValRegs.push_back(Dst);
4194	MIRBuilder.buildLoad(Dst, NewAddrReg, *NewMMO);
4195	} else {
4196	MIRBuilder.buildStore(ValRegs[Idx], NewAddrReg, *NewMMO);
4197	}
4198	}
4199
4200	return Offset;
4201	};
4202
4203	unsigned HandledOffset = splitTypePieces(NarrowTy, NarrowRegs, 0);
4204
4205	// Handle the rest of the register if this isn't an even type breakdown.
4206	if (LeftoverTy.isValid())
4207	splitTypePieces(LeftoverTy, NarrowLeftoverRegs, HandledOffset);
4208
4209	if (IsLoad) {
4210	insertParts(ValReg, ValTy, NarrowTy, NarrowRegs,
4211	LeftoverTy, NarrowLeftoverRegs);
4212	}
4213
4214	LdStMI.eraseFromParent();
4215	return Legalized;
4216	}
4217
4218	LegalizerHelper::LegalizeResult
4219	LegalizerHelper::reduceOperationWidth(MachineInstr &MI, unsigned int TypeIdx,
4220	LLT NarrowTy) {
4221	assert(TypeIdx == 0 && "only one type index expected")(static_cast <bool> (TypeIdx == 0 && "only one type index expected" ) ? void (0) : __assert_fail ("TypeIdx == 0 && \"only one type index expected\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 4221, __extension__ __PRETTY_FUNCTION__));
4222
4223	const unsigned Opc = MI.getOpcode();
4224	const int NumDefOps = MI.getNumExplicitDefs();
4225	const int NumSrcOps = MI.getNumOperands() - NumDefOps;
4226	const unsigned Flags = MI.getFlags();
4227	const unsigned NarrowSize = NarrowTy.getSizeInBits();
4228	const LLT NarrowScalarTy = LLT::scalar(NarrowSize);
4229
4230	assert(MI.getNumOperands() <= 4 && "expected instruction with either 1 "(static_cast <bool> (MI.getNumOperands() <= 4 && "expected instruction with either 1 " "result and 1-3 sources or 2 results and " "1-2 sources") ? void (0) : __assert_fail ("MI.getNumOperands() <= 4 && \"expected instruction with either 1 \" \"result and 1-3 sources or 2 results and \" \"1-2 sources\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 4232, __extension__ __PRETTY_FUNCTION__))
4231	"result and 1-3 sources or 2 results and "(static_cast <bool> (MI.getNumOperands() <= 4 && "expected instruction with either 1 " "result and 1-3 sources or 2 results and " "1-2 sources") ? void (0) : __assert_fail ("MI.getNumOperands() <= 4 && \"expected instruction with either 1 \" \"result and 1-3 sources or 2 results and \" \"1-2 sources\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 4232, __extension__ __PRETTY_FUNCTION__))
4232	"1-2 sources")(static_cast <bool> (MI.getNumOperands() <= 4 && "expected instruction with either 1 " "result and 1-3 sources or 2 results and " "1-2 sources") ? void (0) : __assert_fail ("MI.getNumOperands() <= 4 && \"expected instruction with either 1 \" \"result and 1-3 sources or 2 results and \" \"1-2 sources\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 4232, __extension__ __PRETTY_FUNCTION__));
4233
4234	SmallVector<Register, 2> DstRegs;
4235	for (int I = 0; I < NumDefOps; ++I)
4236	DstRegs.push_back(MI.getOperand(I).getReg());
4237
4238	// First of all check whether we are narrowing (changing the element type)
4239	// or reducing the vector elements
4240	const LLT DstTy = MRI.getType(DstRegs[0]);
4241	const bool IsNarrow = NarrowTy.getScalarType() != DstTy.getScalarType();
4242
4243	SmallVector<Register, 8> ExtractedRegs[3];
4244	SmallVector<Register, 8> Parts;
4245
4246	// Break down all the sources into NarrowTy pieces we can operate on. This may
4247	// involve creating merges to a wider type, padded with undef.
4248	for (int I = 0; I != NumSrcOps; ++I) {
4249	Register SrcReg = MI.getOperand(I + NumDefOps).getReg();
4250	LLT SrcTy = MRI.getType(SrcReg);
4251
4252	// The type to narrow SrcReg to. For narrowing, this is a smaller scalar.
4253	// For fewerElements, this is a smaller vector with the same element type.
4254	LLT OpNarrowTy;
4255	if (IsNarrow) {
4256	OpNarrowTy = NarrowScalarTy;
4257
4258	// In case of narrowing, we need to cast vectors to scalars for this to
4259	// work properly
4260	// FIXME: Can we do without the bitcast here if we're narrowing?
4261	if (SrcTy.isVector()) {
4262	SrcTy = LLT::scalar(SrcTy.getSizeInBits());
4263	SrcReg = MIRBuilder.buildBitcast(SrcTy, SrcReg).getReg(0);
4264	}
4265	} else {
4266	auto NarrowEC = NarrowTy.isVector() ? NarrowTy.getElementCount()
4267	: ElementCount::getFixed(1);
4268	OpNarrowTy = LLT::scalarOrVector(NarrowEC, SrcTy.getScalarType());
4269	}
4270
4271	LLT GCDTy = extractGCDType(ExtractedRegs[I], SrcTy, OpNarrowTy, SrcReg);
4272
4273	// Build a sequence of NarrowTy pieces in ExtractedRegs for this operand.
4274	buildLCMMergePieces(SrcTy, OpNarrowTy, GCDTy, ExtractedRegs[I],
4275	TargetOpcode::G_ANYEXT);
4276	}
4277
4278	SmallVector<Register, 8> ResultRegs[2];
4279
4280	// Input operands for each sub-instruction.
4281	SmallVector<SrcOp, 4> InputRegs(NumSrcOps, Register());
4282
4283	int NumParts = ExtractedRegs[0].size();
4284	const unsigned DstSize = DstTy.getSizeInBits();
4285	const LLT DstScalarTy = LLT::scalar(DstSize);
4286
4287	// Narrowing needs to use scalar types
4288	LLT DstLCMTy, NarrowDstTy;
4289	if (IsNarrow) {
4290	DstLCMTy = getLCMType(DstScalarTy, NarrowScalarTy);
4291	NarrowDstTy = NarrowScalarTy;
4292	} else {
4293	DstLCMTy = getLCMType(DstTy, NarrowTy);
4294	NarrowDstTy = NarrowTy;
4295	}
4296
4297	// We widened the source registers to satisfy merge/unmerge size
4298	// constraints. We'll have some extra fully undef parts.
4299	const int NumRealParts = (DstSize + NarrowSize - 1) / NarrowSize;
4300
4301	for (int I = 0; I != NumRealParts; ++I) {
4302	// Emit this instruction on each of the split pieces.
4303	for (int J = 0; J != NumSrcOps; ++J)
4304	InputRegs[J] = ExtractedRegs[J][I];
4305
4306	MachineInstrBuilder Inst;
4307	if (NumDefOps == 1)
4308	Inst = MIRBuilder.buildInstr(Opc, {NarrowDstTy}, InputRegs, Flags);
4309	else
4310	Inst = MIRBuilder.buildInstr(Opc, {NarrowDstTy, NarrowDstTy}, InputRegs,
4311	Flags);
4312
4313	for (int J = 0; J != NumDefOps; ++J)
4314	ResultRegs[J].push_back(Inst.getReg(J));
4315	}
4316
4317	// Fill out the widened result with undef instead of creating instructions
4318	// with undef inputs.
4319	int NumUndefParts = NumParts - NumRealParts;
4320	if (NumUndefParts != 0) {
4321	Register Undef = MIRBuilder.buildUndef(NarrowDstTy).getReg(0);
4322	for (int I = 0; I != NumDefOps; ++I)
4323	ResultRegs[I].append(NumUndefParts, Undef);
4324	}
4325
4326	// Extract the possibly padded result. Use a scratch register if we need to do
4327	// a final bitcast, otherwise use the original result register.
4328	Register MergeDstReg;
4329	for (int I = 0; I != NumDefOps; ++I) {
4330	if (IsNarrow && DstTy.isVector())
4331	MergeDstReg = MRI.createGenericVirtualRegister(DstScalarTy);
4332	else
4333	MergeDstReg = DstRegs[I];
4334
4335	buildWidenedRemergeToDst(MergeDstReg, DstLCMTy, ResultRegs[I]);
4336
4337	// Recast to vector if we narrowed a vector
4338	if (IsNarrow && DstTy.isVector())
4339	MIRBuilder.buildBitcast(DstRegs[I], MergeDstReg);
4340	}
4341
4342	MI.eraseFromParent();
4343	return Legalized;
4344	}
4345
4346	LegalizerHelper::LegalizeResult
4347	LegalizerHelper::fewerElementsVectorSextInReg(MachineInstr &MI, unsigned TypeIdx,
4348	LLT NarrowTy) {
4349	Register DstReg = MI.getOperand(0).getReg();
4350	Register SrcReg = MI.getOperand(1).getReg();
4351	int64_t Imm = MI.getOperand(2).getImm();
4352
4353	LLT DstTy = MRI.getType(DstReg);
4354
4355	SmallVector<Register, 8> Parts;
4356	LLT GCDTy = extractGCDType(Parts, DstTy, NarrowTy, SrcReg);
4357	LLT LCMTy = buildLCMMergePieces(DstTy, NarrowTy, GCDTy, Parts);
4358
4359	for (Register &R : Parts)
4360	R = MIRBuilder.buildSExtInReg(NarrowTy, R, Imm).getReg(0);
4361
4362	buildWidenedRemergeToDst(DstReg, LCMTy, Parts);
4363
4364	MI.eraseFromParent();
4365	return Legalized;
4366	}
4367
4368	LegalizerHelper::LegalizeResult
4369	LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
4370	LLT NarrowTy) {
4371	using namespace TargetOpcode;
4372
4373	switch (MI.getOpcode()) {
4374	case G_IMPLICIT_DEF:
4375	return fewerElementsVectorImplicitDef(MI, TypeIdx, NarrowTy);
4376	case G_TRUNC:
4377	case G_AND:
4378	case G_OR:
4379	case G_XOR:
4380	case G_ADD:
4381	case G_SUB:
4382	case G_MUL:
4383	case G_PTR_ADD:
4384	case G_SMULH:
4385	case G_UMULH:
4386	case G_FADD:
4387	case G_FMUL:
4388	case G_FSUB:
4389	case G_FNEG:
4390	case G_FABS:
4391	case G_FCANONICALIZE:
4392	case G_FDIV:
4393	case G_FREM:
4394	case G_FMA:
4395	case G_FMAD:
4396	case G_FPOW:
4397	case G_FEXP:
4398	case G_FEXP2:
4399	case G_FLOG:
4400	case G_FLOG2:
4401	case G_FLOG10:
4402	case G_FNEARBYINT:
4403	case G_FCEIL:
4404	case G_FFLOOR:
4405	case G_FRINT:
4406	case G_INTRINSIC_ROUND:
4407	case G_INTRINSIC_ROUNDEVEN:
4408	case G_INTRINSIC_TRUNC:
4409	case G_FCOS:
4410	case G_FSIN:
4411	case G_FSQRT:
4412	case G_BSWAP:
4413	case G_BITREVERSE:
4414	case G_SDIV:
4415	case G_UDIV:
4416	case G_SREM:
4417	case G_UREM:
4418	case G_SDIVREM:
4419	case G_UDIVREM:
4420	case G_SMIN:
4421	case G_SMAX:
4422	case G_UMIN:
4423	case G_UMAX:
4424	case G_ABS:
4425	case G_FMINNUM:
4426	case G_FMAXNUM:
4427	case G_FMINNUM_IEEE:
4428	case G_FMAXNUM_IEEE:
4429	case G_FMINIMUM:
4430	case G_FMAXIMUM:
4431	case G_FSHL:
4432	case G_FSHR:
4433	case G_FREEZE:
4434	case G_SADDSAT:
4435	case G_SSUBSAT:
4436	case G_UADDSAT:
4437	case G_USUBSAT:
4438	return reduceOperationWidth(MI, TypeIdx, NarrowTy);
4439	case G_UMULO:
4440	case G_SMULO:
4441	return fewerElementsVectorMulo(MI, TypeIdx, NarrowTy);
4442	case G_SHL:
4443	case G_LSHR:
4444	case G_ASHR:
4445	case G_SSHLSAT:
4446	case G_USHLSAT:
4447	case G_CTLZ:
4448	case G_CTLZ_ZERO_UNDEF:
4449	case G_CTTZ:
4450	case G_CTTZ_ZERO_UNDEF:
4451	case G_CTPOP:
4452	case G_FCOPYSIGN:
4453	return fewerElementsVectorMultiEltType(MI, TypeIdx, NarrowTy);
4454	case G_ZEXT:
4455	case G_SEXT:
4456	case G_ANYEXT:
4457	case G_FPEXT:
4458	case G_FPTRUNC:
4459	case G_SITOFP:
4460	case G_UITOFP:
4461	case G_FPTOSI:
4462	case G_FPTOUI:
4463	case G_INTTOPTR:
4464	case G_PTRTOINT:
4465	case G_ADDRSPACE_CAST:
4466	return fewerElementsVectorCasts(MI, TypeIdx, NarrowTy);
4467	case G_ICMP:
4468	case G_FCMP:
4469	return fewerElementsVectorCmp(MI, TypeIdx, NarrowTy);
4470	case G_SELECT:
4471	return fewerElementsVectorSelect(MI, TypeIdx, NarrowTy);
4472	case G_PHI:
4473	return fewerElementsVectorPhi(MI, TypeIdx, NarrowTy);
4474	case G_UNMERGE_VALUES:
4475	return fewerElementsVectorUnmergeValues(MI, TypeIdx, NarrowTy);
4476	case G_BUILD_VECTOR:
4477	assert(TypeIdx == 0 && "not a vector type index")(static_cast <bool> (TypeIdx == 0 && "not a vector type index" ) ? void (0) : __assert_fail ("TypeIdx == 0 && \"not a vector type index\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 4477, __extension__ __PRETTY_FUNCTION__));
4478	return fewerElementsVectorMerge(MI, TypeIdx, NarrowTy);
4479	case G_CONCAT_VECTORS:
4480	if (TypeIdx != 1) // TODO: This probably does work as expected already.
4481	return UnableToLegalize;
4482	return fewerElementsVectorMerge(MI, TypeIdx, NarrowTy);
4483	case G_EXTRACT_VECTOR_ELT:
4484	case G_INSERT_VECTOR_ELT:
4485	return fewerElementsVectorExtractInsertVectorElt(MI, TypeIdx, NarrowTy);
4486	case G_LOAD:
4487	case G_STORE:
4488	return reduceLoadStoreWidth(cast<GLoadStore>(MI), TypeIdx, NarrowTy);
4489	case G_SEXT_INREG:
4490	return fewerElementsVectorSextInReg(MI, TypeIdx, NarrowTy);
4491	GISEL_VECREDUCE_CASES_NONSEQcase TargetOpcode::G_VECREDUCE_FADD: case TargetOpcode::G_VECREDUCE_FMUL : case TargetOpcode::G_VECREDUCE_FMAX: case TargetOpcode::G_VECREDUCE_FMIN : case TargetOpcode::G_VECREDUCE_ADD: case TargetOpcode::G_VECREDUCE_MUL : case TargetOpcode::G_VECREDUCE_AND: case TargetOpcode::G_VECREDUCE_OR : case TargetOpcode::G_VECREDUCE_XOR: case TargetOpcode::G_VECREDUCE_SMAX : case TargetOpcode::G_VECREDUCE_SMIN: case TargetOpcode::G_VECREDUCE_UMAX : case TargetOpcode::G_VECREDUCE_UMIN:
4492	return fewerElementsVectorReductions(MI, TypeIdx, NarrowTy);
4493	case G_SHUFFLE_VECTOR:
4494	return fewerElementsVectorShuffle(MI, TypeIdx, NarrowTy);
4495	default:
4496	return UnableToLegalize;
4497	}
4498	}
4499
4500	LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorShuffle(
4501	MachineInstr &MI, unsigned int TypeIdx, LLT NarrowTy) {
4502	assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR)(static_cast <bool> (MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR ) ? void (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 4502, __extension__ __PRETTY_FUNCTION__));
4503	if (TypeIdx != 0)
4504	return UnableToLegalize;
4505
4506	Register DstReg = MI.getOperand(0).getReg();
4507	Register Src1Reg = MI.getOperand(1).getReg();
4508	Register Src2Reg = MI.getOperand(2).getReg();
4509	ArrayRef<int> Mask = MI.getOperand(3).getShuffleMask();
4510	LLT DstTy = MRI.getType(DstReg);
4511	LLT Src1Ty = MRI.getType(Src1Reg);
4512	LLT Src2Ty = MRI.getType(Src2Reg);
4513	// The shuffle should be canonicalized by now.
4514	if (DstTy != Src1Ty)
4515	return UnableToLegalize;
4516	if (DstTy != Src2Ty)
4517	return UnableToLegalize;
4518
4519	if (!isPowerOf2_32(DstTy.getNumElements()))
4520	return UnableToLegalize;
4521
4522	// We only support splitting a shuffle into 2, so adjust NarrowTy accordingly.
4523	// Further legalization attempts will be needed to do split further.
4524	NarrowTy =
4525	DstTy.changeElementCount(DstTy.getElementCount().divideCoefficientBy(2));
4526	unsigned NewElts = NarrowTy.getNumElements();
4527
4528	SmallVector<Register> SplitSrc1Regs, SplitSrc2Regs;
4529	extractParts(Src1Reg, NarrowTy, 2, SplitSrc1Regs);
4530	extractParts(Src2Reg, NarrowTy, 2, SplitSrc2Regs);
4531	Register Inputs[4] = {SplitSrc1Regs[0], SplitSrc1Regs[1], SplitSrc2Regs[0],
4532	SplitSrc2Regs[1]};
4533
4534	Register Hi, Lo;
4535
4536	// If Lo or Hi uses elements from at most two of the four input vectors, then
4537	// express it as a vector shuffle of those two inputs. Otherwise extract the
4538	// input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
4539	SmallVector<int, 16> Ops;
4540	for (unsigned High = 0; High < 2; ++High) {
4541	Register &Output = High ? Hi : Lo;
4542
4543	// Build a shuffle mask for the output, discovering on the fly which
4544	// input vectors to use as shuffle operands (recorded in InputUsed).
4545	// If building a suitable shuffle vector proves too hard, then bail
4546	// out with useBuildVector set.
4547	unsigned InputUsed[2] = {-1U, -1U}; // Not yet discovered.
4548	unsigned FirstMaskIdx = High * NewElts;
4549	bool UseBuildVector = false;
4550	for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
4551	// The mask element. This indexes into the input.
4552	int Idx = Mask[FirstMaskIdx + MaskOffset];
4553
4554	// The input vector this mask element indexes into.
4555	unsigned Input = (unsigned)Idx / NewElts;
4556
4557	if (Input >= array_lengthof(Inputs)) {
4558	// The mask element does not index into any input vector.
4559	Ops.push_back(-1);
4560	continue;
4561	}
4562
4563	// Turn the index into an offset from the start of the input vector.
4564	Idx -= Input * NewElts;
4565
4566	// Find or create a shuffle vector operand to hold this input.
4567	unsigned OpNo;
4568	for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) {
4569	if (InputUsed[OpNo] == Input) {
4570	// This input vector is already an operand.
4571	break;
4572	} else if (InputUsed[OpNo] == -1U) {
4573	// Create a new operand for this input vector.
4574	InputUsed[OpNo] = Input;
4575	break;
4576	}
4577	}
4578
4579	if (OpNo >= array_lengthof(InputUsed)) {
4580	// More than two input vectors used! Give up on trying to create a
4581	// shuffle vector. Insert all elements into a BUILD_VECTOR instead.
4582	UseBuildVector = true;
4583	break;
4584	}
4585
4586	// Add the mask index for the new shuffle vector.
4587	Ops.push_back(Idx + OpNo * NewElts);
4588	}
4589
4590	if (UseBuildVector) {
4591	LLT EltTy = NarrowTy.getElementType();
4592	SmallVector<Register, 16> SVOps;
4593
4594	// Extract the input elements by hand.
4595	for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
4596	// The mask element. This indexes into the input.
4597	int Idx = Mask[FirstMaskIdx + MaskOffset];
4598
4599	// The input vector this mask element indexes into.
4600	unsigned Input = (unsigned)Idx / NewElts;
4601
4602	if (Input >= array_lengthof(Inputs)) {
4603	// The mask element is "undef" or indexes off the end of the input.
4604	SVOps.push_back(MIRBuilder.buildUndef(EltTy).getReg(0));
4605	continue;
4606	}
4607
4608	// Turn the index into an offset from the start of the input vector.
4609	Idx -= Input * NewElts;
4610
4611	// Extract the vector element by hand.
4612	SVOps.push_back(MIRBuilder
4613	.buildExtractVectorElement(
4614	EltTy, Inputs[Input],
4615	MIRBuilder.buildConstant(LLT::scalar(32), Idx))
4616	.getReg(0));
4617	}
4618
4619	// Construct the Lo/Hi output using a G_BUILD_VECTOR.
4620	Output = MIRBuilder.buildBuildVector(NarrowTy, SVOps).getReg(0);
4621	} else if (InputUsed[0] == -1U) {
4622	// No input vectors were used! The result is undefined.
4623	Output = MIRBuilder.buildUndef(NarrowTy).getReg(0);
4624	} else {
4625	Register Op0 = Inputs[InputUsed[0]];
4626	// If only one input was used, use an undefined vector for the other.
4627	Register Op1 = InputUsed[1] == -1U
4628	? MIRBuilder.buildUndef(NarrowTy).getReg(0)
4629	: Inputs[InputUsed[1]];
4630	// At least one input vector was used. Create a new shuffle vector.
4631	Output = MIRBuilder.buildShuffleVector(NarrowTy, Op0, Op1, Ops).getReg(0);
4632	}
4633
4634	Ops.clear();
4635	}
4636
4637	MIRBuilder.buildConcatVectors(DstReg, {Lo, Hi});
4638	MI.eraseFromParent();
4639	return Legalized;
4640	}
4641
4642	static unsigned getScalarOpcForReduction(unsigned Opc) {
4643	unsigned ScalarOpc;
4644	switch (Opc) {
4645	case TargetOpcode::G_VECREDUCE_FADD:
4646	ScalarOpc = TargetOpcode::G_FADD;
4647	break;
4648	case TargetOpcode::G_VECREDUCE_FMUL:
4649	ScalarOpc = TargetOpcode::G_FMUL;
4650	break;
4651	case TargetOpcode::G_VECREDUCE_FMAX:
4652	ScalarOpc = TargetOpcode::G_FMAXNUM;
4653	break;
4654	case TargetOpcode::G_VECREDUCE_FMIN:
4655	ScalarOpc = TargetOpcode::G_FMINNUM;
4656	break;
4657	case TargetOpcode::G_VECREDUCE_ADD:
4658	ScalarOpc = TargetOpcode::G_ADD;
4659	break;
4660	case TargetOpcode::G_VECREDUCE_MUL:
4661	ScalarOpc = TargetOpcode::G_MUL;
4662	break;
4663	case TargetOpcode::G_VECREDUCE_AND:
4664	ScalarOpc = TargetOpcode::G_AND;
4665	break;
4666	case TargetOpcode::G_VECREDUCE_OR:
4667	ScalarOpc = TargetOpcode::G_OR;
4668	break;
4669	case TargetOpcode::G_VECREDUCE_XOR:
4670	ScalarOpc = TargetOpcode::G_XOR;
4671	break;
4672	case TargetOpcode::G_VECREDUCE_SMAX:
4673	ScalarOpc = TargetOpcode::G_SMAX;
4674	break;
4675	case TargetOpcode::G_VECREDUCE_SMIN:
4676	ScalarOpc = TargetOpcode::G_SMIN;
4677	break;
4678	case TargetOpcode::G_VECREDUCE_UMAX:
4679	ScalarOpc = TargetOpcode::G_UMAX;
4680	break;
4681	case TargetOpcode::G_VECREDUCE_UMIN:
4682	ScalarOpc = TargetOpcode::G_UMIN;
4683	break;
4684	default:
4685	llvm_unreachable("Unhandled reduction")::llvm::llvm_unreachable_internal("Unhandled reduction", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 4685);
4686	}
4687	return ScalarOpc;
4688	}
4689
4690	LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorReductions(
4691	MachineInstr &MI, unsigned int TypeIdx, LLT NarrowTy) {
4692	unsigned Opc = MI.getOpcode();
4693	assert(Opc != TargetOpcode::G_VECREDUCE_SEQ_FADD &&(static_cast <bool> (Opc != TargetOpcode::G_VECREDUCE_SEQ_FADD && Opc != TargetOpcode::G_VECREDUCE_SEQ_FMUL && "Sequential reductions not expected") ? void (0) : __assert_fail ("Opc != TargetOpcode::G_VECREDUCE_SEQ_FADD && Opc != TargetOpcode::G_VECREDUCE_SEQ_FMUL && \"Sequential reductions not expected\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 4695, __extension__ __PRETTY_FUNCTION__))
4694	Opc != TargetOpcode::G_VECREDUCE_SEQ_FMUL &&(static_cast <bool> (Opc != TargetOpcode::G_VECREDUCE_SEQ_FADD && Opc != TargetOpcode::G_VECREDUCE_SEQ_FMUL && "Sequential reductions not expected") ? void (0) : __assert_fail ("Opc != TargetOpcode::G_VECREDUCE_SEQ_FADD && Opc != TargetOpcode::G_VECREDUCE_SEQ_FMUL && \"Sequential reductions not expected\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 4695, __extension__ __PRETTY_FUNCTION__))
4695	"Sequential reductions not expected")(static_cast <bool> (Opc != TargetOpcode::G_VECREDUCE_SEQ_FADD && Opc != TargetOpcode::G_VECREDUCE_SEQ_FMUL && "Sequential reductions not expected") ? void (0) : __assert_fail ("Opc != TargetOpcode::G_VECREDUCE_SEQ_FADD && Opc != TargetOpcode::G_VECREDUCE_SEQ_FMUL && \"Sequential reductions not expected\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 4695, __extension__ __PRETTY_FUNCTION__));
4696
4697	if (TypeIdx != 1)
4698	return UnableToLegalize;
4699
4700	// The semantics of the normal non-sequential reductions allow us to freely
4701	// re-associate the operation.
4702	Register SrcReg = MI.getOperand(1).getReg();
4703	LLT SrcTy = MRI.getType(SrcReg);
4704	Register DstReg = MI.getOperand(0).getReg();
4705	LLT DstTy = MRI.getType(DstReg);
4706
4707	if (NarrowTy.isVector() &&
4708	(SrcTy.getNumElements() % NarrowTy.getNumElements() != 0))
4709	return UnableToLegalize;
4710
4711	unsigned ScalarOpc = getScalarOpcForReduction(Opc);
4712	SmallVector<Register> SplitSrcs;
4713	// If NarrowTy is a scalar then we're being asked to scalarize.
4714	const unsigned NumParts =
4715	NarrowTy.isVector() ? SrcTy.getNumElements() / NarrowTy.getNumElements()
4716	: SrcTy.getNumElements();
4717
4718	extractParts(SrcReg, NarrowTy, NumParts, SplitSrcs);
4719	if (NarrowTy.isScalar()) {
4720	if (DstTy != NarrowTy)
4721	return UnableToLegalize; // FIXME: handle implicit extensions.
4722
4723	if (isPowerOf2_32(NumParts)) {
4724	// Generate a tree of scalar operations to reduce the critical path.
4725	SmallVector<Register> PartialResults;
4726	unsigned NumPartsLeft = NumParts;
4727	while (NumPartsLeft > 1) {
4728	for (unsigned Idx = 0; Idx < NumPartsLeft - 1; Idx += 2) {
4729	PartialResults.emplace_back(
4730	MIRBuilder
4731	.buildInstr(ScalarOpc, {NarrowTy},
4732	{SplitSrcs[Idx], SplitSrcs[Idx + 1]})
4733	.getReg(0));
4734	}
4735	SplitSrcs = PartialResults;
4736	PartialResults.clear();
4737	NumPartsLeft = SplitSrcs.size();
4738	}
4739	assert(SplitSrcs.size() == 1)(static_cast <bool> (SplitSrcs.size() == 1) ? void (0) : __assert_fail ("SplitSrcs.size() == 1", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 4739, __extension__ __PRETTY_FUNCTION__));
4740	MIRBuilder.buildCopy(DstReg, SplitSrcs[0]);
4741	MI.eraseFromParent();
4742	return Legalized;
4743	}
4744	// If we can't generate a tree, then just do sequential operations.
4745	Register Acc = SplitSrcs[0];
4746	for (unsigned Idx = 1; Idx < NumParts; ++Idx)
4747	Acc = MIRBuilder.buildInstr(ScalarOpc, {NarrowTy}, {Acc, SplitSrcs[Idx]})
4748	.getReg(0);
4749	MIRBuilder.buildCopy(DstReg, Acc);
4750	MI.eraseFromParent();
4751	return Legalized;
4752	}
4753	SmallVector<Register> PartialReductions;
4754	for (unsigned Part = 0; Part < NumParts; ++Part) {
4755	PartialReductions.push_back(
4756	MIRBuilder.buildInstr(Opc, {DstTy}, {SplitSrcs[Part]}).getReg(0));
4757	}
4758
4759
4760	// If the types involved are powers of 2, we can generate intermediate vector
4761	// ops, before generating a final reduction operation.
4762	if (isPowerOf2_32(SrcTy.getNumElements()) &&
4763	isPowerOf2_32(NarrowTy.getNumElements())) {
4764	return tryNarrowPow2Reduction(MI, SrcReg, SrcTy, NarrowTy, ScalarOpc);
4765	}
4766
4767	Register Acc = PartialReductions[0];
4768	for (unsigned Part = 1; Part < NumParts; ++Part) {
4769	if (Part == NumParts - 1) {
4770	MIRBuilder.buildInstr(ScalarOpc, {DstReg},
4771	{Acc, PartialReductions[Part]});
4772	} else {
4773	Acc = MIRBuilder
4774	.buildInstr(ScalarOpc, {DstTy}, {Acc, PartialReductions[Part]})
4775	.getReg(0);
4776	}
4777	}
4778	MI.eraseFromParent();
4779	return Legalized;
4780	}
4781
4782	LegalizerHelper::LegalizeResult
4783	LegalizerHelper::tryNarrowPow2Reduction(MachineInstr &MI, Register SrcReg,
4784	LLT SrcTy, LLT NarrowTy,
4785	unsigned ScalarOpc) {
4786	SmallVector<Register> SplitSrcs;
4787	// Split the sources into NarrowTy size pieces.
4788	extractParts(SrcReg, NarrowTy,
4789	SrcTy.getNumElements() / NarrowTy.getNumElements(), SplitSrcs);
4790	// We're going to do a tree reduction using vector operations until we have
4791	// one NarrowTy size value left.
4792	while (SplitSrcs.size() > 1) {
4793	SmallVector<Register> PartialRdxs;
4794	for (unsigned Idx = 0; Idx < SplitSrcs.size()-1; Idx += 2) {
4795	Register LHS = SplitSrcs[Idx];
4796	Register RHS = SplitSrcs[Idx + 1];
4797	// Create the intermediate vector op.
4798	Register Res =
4799	MIRBuilder.buildInstr(ScalarOpc, {NarrowTy}, {LHS, RHS}).getReg(0);
4800	PartialRdxs.push_back(Res);
4801	}
4802	SplitSrcs = std::move(PartialRdxs);
4803	}
4804	// Finally generate the requested NarrowTy based reduction.
4805	Observer.changingInstr(MI);
4806	MI.getOperand(1).setReg(SplitSrcs[0]);
4807	Observer.changedInstr(MI);
4808	return Legalized;
4809	}
4810
4811	LegalizerHelper::LegalizeResult
4812	LegalizerHelper::narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt,
4813	const LLT HalfTy, const LLT AmtTy) {
4814
4815	Register InL = MRI.createGenericVirtualRegister(HalfTy);
4816	Register InH = MRI.createGenericVirtualRegister(HalfTy);
4817	MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1));
4818
4819	if (Amt.isNullValue()) {
4820	MIRBuilder.buildMerge(MI.getOperand(0), {InL, InH});
4821	MI.eraseFromParent();
4822	return Legalized;
4823	}
4824
4825	LLT NVT = HalfTy;
4826	unsigned NVTBits = HalfTy.getSizeInBits();
4827	unsigned VTBits = 2 * NVTBits;
4828
4829	SrcOp Lo(Register(0)), Hi(Register(0));
4830	if (MI.getOpcode() == TargetOpcode::G_SHL) {
4831	if (Amt.ugt(VTBits)) {
4832	Lo = Hi = MIRBuilder.buildConstant(NVT, 0);
4833	} else if (Amt.ugt(NVTBits)) {
4834	Lo = MIRBuilder.buildConstant(NVT, 0);
4835	Hi = MIRBuilder.buildShl(NVT, InL,
4836	MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
4837	} else if (Amt == NVTBits) {
4838	Lo = MIRBuilder.buildConstant(NVT, 0);
4839	Hi = InL;
4840	} else {
4841	Lo = MIRBuilder.buildShl(NVT, InL, MIRBuilder.buildConstant(AmtTy, Amt));
4842	auto OrLHS =
4843	MIRBuilder.buildShl(NVT, InH, MIRBuilder.buildConstant(AmtTy, Amt));
4844	auto OrRHS = MIRBuilder.buildLShr(
4845	NVT, InL, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
4846	Hi = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
4847	}
4848	} else if (MI.getOpcode() == TargetOpcode::G_LSHR) {
4849	if (Amt.ugt(VTBits)) {
4850	Lo = Hi = MIRBuilder.buildConstant(NVT, 0);
4851	} else if (Amt.ugt(NVTBits)) {
4852	Lo = MIRBuilder.buildLShr(NVT, InH,
4853	MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
4854	Hi = MIRBuilder.buildConstant(NVT, 0);
4855	} else if (Amt == NVTBits) {
4856	Lo = InH;
4857	Hi = MIRBuilder.buildConstant(NVT, 0);
4858	} else {
4859	auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt);
4860
4861	auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst);
4862	auto OrRHS = MIRBuilder.buildShl(
4863	NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
4864
4865	Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
4866	Hi = MIRBuilder.buildLShr(NVT, InH, ShiftAmtConst);
4867	}
4868	} else {
4869	if (Amt.ugt(VTBits)) {
4870	Hi = Lo = MIRBuilder.buildAShr(
4871	NVT, InH, MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
4872	} else if (Amt.ugt(NVTBits)) {
4873	Lo = MIRBuilder.buildAShr(NVT, InH,
4874	MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
4875	Hi = MIRBuilder.buildAShr(NVT, InH,
4876	MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
4877	} else if (Amt == NVTBits) {
4878	Lo = InH;
4879	Hi = MIRBuilder.buildAShr(NVT, InH,
4880	MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
4881	} else {
4882	auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt);
4883
4884	auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst);
4885	auto OrRHS = MIRBuilder.buildShl(
4886	NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
4887
4888	Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
4889	Hi = MIRBuilder.buildAShr(NVT, InH, ShiftAmtConst);
4890	}
4891	}
4892
4893	MIRBuilder.buildMerge(MI.getOperand(0), {Lo, Hi});
4894	MI.eraseFromParent();
4895
4896	return Legalized;
4897	}
4898
4899	// TODO: Optimize if constant shift amount.
4900	LegalizerHelper::LegalizeResult
4901	LegalizerHelper::narrowScalarShift(MachineInstr &MI, unsigned TypeIdx,
4902	LLT RequestedTy) {
4903	if (TypeIdx == 1) {
4904	Observer.changingInstr(MI);
4905	narrowScalarSrc(MI, RequestedTy, 2);
4906	Observer.changedInstr(MI);
4907	return Legalized;
4908	}
4909
4910	Register DstReg = MI.getOperand(0).getReg();
4911	LLT DstTy = MRI.getType(DstReg);
4912	if (DstTy.isVector())
4913	return UnableToLegalize;
4914
4915	Register Amt = MI.getOperand(2).getReg();
4916	LLT ShiftAmtTy = MRI.getType(Amt);
4917	const unsigned DstEltSize = DstTy.getScalarSizeInBits();
4918	if (DstEltSize % 2 != 0)
4919	return UnableToLegalize;
4920
4921	// Ignore the input type. We can only go to exactly half the size of the
4922	// input. If that isn't small enough, the resulting pieces will be further
4923	// legalized.
4924	const unsigned NewBitSize = DstEltSize / 2;
4925	const LLT HalfTy = LLT::scalar(NewBitSize);
4926	const LLT CondTy = LLT::scalar(1);
4927
4928	if (auto VRegAndVal =
4929	getConstantVRegValWithLookThrough(Amt, MRI, true, false)) {
4930	return narrowScalarShiftByConstant(MI, VRegAndVal->Value, HalfTy,
4931	ShiftAmtTy);
4932	}
4933
4934	// TODO: Expand with known bits.
4935
4936	// Handle the fully general expansion by an unknown amount.
4937	auto NewBits = MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize);
4938
4939	Register InL = MRI.createGenericVirtualRegister(HalfTy);
4940	Register InH = MRI.createGenericVirtualRegister(HalfTy);
4941	MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1));
4942
4943	auto AmtExcess = MIRBuilder.buildSub(ShiftAmtTy, Amt, NewBits);
4944	auto AmtLack = MIRBuilder.buildSub(ShiftAmtTy, NewBits, Amt);
4945
4946	auto Zero = MIRBuilder.buildConstant(ShiftAmtTy, 0);
4947	auto IsShort = MIRBuilder.buildICmp(ICmpInst::ICMP_ULT, CondTy, Amt, NewBits);
4948	auto IsZero = MIRBuilder.buildICmp(ICmpInst::ICMP_EQ, CondTy, Amt, Zero);
4949
4950	Register ResultRegs[2];
4951	switch (MI.getOpcode()) {
4952	case TargetOpcode::G_SHL: {
4953	// Short: ShAmt < NewBitSize
4954	auto LoS = MIRBuilder.buildShl(HalfTy, InL, Amt);
4955
4956	auto LoOr = MIRBuilder.buildLShr(HalfTy, InL, AmtLack);
4957	auto HiOr = MIRBuilder.buildShl(HalfTy, InH, Amt);
4958	auto HiS = MIRBuilder.buildOr(HalfTy, LoOr, HiOr);
4959
4960	// Long: ShAmt >= NewBitSize
4961	auto LoL = MIRBuilder.buildConstant(HalfTy, 0); // Lo part is zero.
4962	auto HiL = MIRBuilder.buildShl(HalfTy, InL, AmtExcess); // Hi from Lo part.
4963
4964	auto Lo = MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL);
4965	auto Hi = MIRBuilder.buildSelect(
4966	HalfTy, IsZero, InH, MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL));
4967
4968	ResultRegs[0] = Lo.getReg(0);
4969	ResultRegs[1] = Hi.getReg(0);
4970	break;
4971	}
4972	case TargetOpcode::G_LSHR:
4973	case TargetOpcode::G_ASHR: {
4974	// Short: ShAmt < NewBitSize
4975	auto HiS = MIRBuilder.buildInstr(MI.getOpcode(), {HalfTy}, {InH, Amt});
4976
4977	auto LoOr = MIRBuilder.buildLShr(HalfTy, InL, Amt);
4978	auto HiOr = MIRBuilder.buildShl(HalfTy, InH, AmtLack);
4979	auto LoS = MIRBuilder.buildOr(HalfTy, LoOr, HiOr);
4980
4981	// Long: ShAmt >= NewBitSize
4982	MachineInstrBuilder HiL;
4983	if (MI.getOpcode() == TargetOpcode::G_LSHR) {
4984	HiL = MIRBuilder.buildConstant(HalfTy, 0); // Hi part is zero.
4985	} else {
4986	auto ShiftAmt = MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize - 1);
4987	HiL = MIRBuilder.buildAShr(HalfTy, InH, ShiftAmt); // Sign of Hi part.
4988	}
4989	auto LoL = MIRBuilder.buildInstr(MI.getOpcode(), {HalfTy},
4990	{InH, AmtExcess}); // Lo from Hi part.
4991
4992	auto Lo = MIRBuilder.buildSelect(
4993	HalfTy, IsZero, InL, MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL));
4994
4995	auto Hi = MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL);
4996
4997	ResultRegs[0] = Lo.getReg(0);
4998	ResultRegs[1] = Hi.getReg(0);
4999	break;
5000	}
5001	default:
5002	llvm_unreachable("not a shift")::llvm::llvm_unreachable_internal("not a shift", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 5002);
5003	}
5004
5005	MIRBuilder.buildMerge(DstReg, ResultRegs);
5006	MI.eraseFromParent();
5007	return Legalized;
5008	}
5009
5010	LegalizerHelper::LegalizeResult
5011	LegalizerHelper::moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx,
5012	LLT MoreTy) {
5013	assert(TypeIdx == 0 && "Expecting only Idx 0")(static_cast <bool> (TypeIdx == 0 && "Expecting only Idx 0" ) ? void (0) : __assert_fail ("TypeIdx == 0 && \"Expecting only Idx 0\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 5013, __extension__ __PRETTY_FUNCTION__));
5014
5015	Observer.changingInstr(MI);
5016	for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
5017	MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
5018	MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
5019	moreElementsVectorSrc(MI, MoreTy, I);
5020	}
5021
5022	MachineBasicBlock &MBB = *MI.getParent();
5023	MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI());
5024	moreElementsVectorDst(MI, MoreTy, 0);
5025	Observer.changedInstr(MI);
5026	return Legalized;
5027	}
5028
5029	LegalizerHelper::LegalizeResult
5030	LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
5031	LLT MoreTy) {
5032	unsigned Opc = MI.getOpcode();
5033	switch (Opc) {
5034	case TargetOpcode::G_IMPLICIT_DEF:
5035	case TargetOpcode::G_LOAD: {
5036	if (TypeIdx != 0)
5037	return UnableToLegalize;
5038	Observer.changingInstr(MI);
5039	moreElementsVectorDst(MI, MoreTy, 0);
5040	Observer.changedInstr(MI);
5041	return Legalized;
5042	}
5043	case TargetOpcode::G_STORE:
5044	if (TypeIdx != 0)
5045	return UnableToLegalize;
5046	Observer.changingInstr(MI);
5047	moreElementsVectorSrc(MI, MoreTy, 0);
5048	Observer.changedInstr(MI);
5049	return Legalized;
5050	case TargetOpcode::G_AND:
5051	case TargetOpcode::G_OR:
5052	case TargetOpcode::G_XOR:
5053	case TargetOpcode::G_SMIN:
5054	case TargetOpcode::G_SMAX:
5055	case TargetOpcode::G_UMIN:
5056	case TargetOpcode::G_UMAX:
5057	case TargetOpcode::G_FMINNUM:
5058	case TargetOpcode::G_FMAXNUM:
5059	case TargetOpcode::G_FMINNUM_IEEE:
5060	case TargetOpcode::G_FMAXNUM_IEEE:
5061	case TargetOpcode::G_FMINIMUM:
5062	case TargetOpcode::G_FMAXIMUM: {
5063	Observer.changingInstr(MI);
5064	moreElementsVectorSrc(MI, MoreTy, 1);
5065	moreElementsVectorSrc(MI, MoreTy, 2);
5066	moreElementsVectorDst(MI, MoreTy, 0);
5067	Observer.changedInstr(MI);
5068	return Legalized;
5069	}
5070	case TargetOpcode::G_EXTRACT:
5071	if (TypeIdx != 1)
5072	return UnableToLegalize;
5073	Observer.changingInstr(MI);
5074	moreElementsVectorSrc(MI, MoreTy, 1);
5075	Observer.changedInstr(MI);
5076	return Legalized;
5077	case TargetOpcode::G_INSERT:
5078	case TargetOpcode::G_FREEZE:
5079	if (TypeIdx != 0)
5080	return UnableToLegalize;
5081	Observer.changingInstr(MI);
5082	moreElementsVectorSrc(MI, MoreTy, 1);
5083	moreElementsVectorDst(MI, MoreTy, 0);
5084	Observer.changedInstr(MI);
5085	return Legalized;
5086	case TargetOpcode::G_SELECT:
5087	if (TypeIdx != 0)
5088	return UnableToLegalize;
5089	if (MRI.getType(MI.getOperand(1).getReg()).isVector())
5090	return UnableToLegalize;
5091
5092	Observer.changingInstr(MI);
5093	moreElementsVectorSrc(MI, MoreTy, 2);
5094	moreElementsVectorSrc(MI, MoreTy, 3);
5095	moreElementsVectorDst(MI, MoreTy, 0);
5096	Observer.changedInstr(MI);
5097	return Legalized;
5098	case TargetOpcode::G_UNMERGE_VALUES: {
5099	if (TypeIdx != 1)
5100	return UnableToLegalize;
5101
5102	LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
5103	int NumDst = MI.getNumOperands() - 1;
5104	moreElementsVectorSrc(MI, MoreTy, NumDst);
5105
5106	auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES);
5107	for (int I = 0; I != NumDst; ++I)
5108	MIB.addDef(MI.getOperand(I).getReg());
5109
5110	int NewNumDst = MoreTy.getSizeInBits() / DstTy.getSizeInBits();
5111	for (int I = NumDst; I != NewNumDst; ++I)
5112	MIB.addDef(MRI.createGenericVirtualRegister(DstTy));
5113
5114	MIB.addUse(MI.getOperand(NumDst).getReg());
5115	MI.eraseFromParent();
5116	return Legalized;
5117	}
5118	case TargetOpcode::G_PHI:
5119	return moreElementsVectorPhi(MI, TypeIdx, MoreTy);
5120	case TargetOpcode::G_SHUFFLE_VECTOR:
5121	return moreElementsVectorShuffle(MI, TypeIdx, MoreTy);
5122	default:
5123	return UnableToLegalize;
5124	}
5125	}
5126
5127	LegalizerHelper::LegalizeResult
5128	LegalizerHelper::moreElementsVectorShuffle(MachineInstr &MI,
5129	unsigned int TypeIdx, LLT MoreTy) {
5130	if (TypeIdx != 0)
5131	return UnableToLegalize;
5132
5133	Register DstReg = MI.getOperand(0).getReg();
5134	Register Src1Reg = MI.getOperand(1).getReg();
5135	Register Src2Reg = MI.getOperand(2).getReg();
5136	ArrayRef<int> Mask = MI.getOperand(3).getShuffleMask();
5137	LLT DstTy = MRI.getType(DstReg);
5138	LLT Src1Ty = MRI.getType(Src1Reg);
5139	LLT Src2Ty = MRI.getType(Src2Reg);
5140	unsigned NumElts = DstTy.getNumElements();
5141	unsigned WidenNumElts = MoreTy.getNumElements();
5142
5143	// Expect a canonicalized shuffle.
5144	if (DstTy != Src1Ty \|\| DstTy != Src2Ty)
5145	return UnableToLegalize;
5146
5147	moreElementsVectorSrc(MI, MoreTy, 1);
5148	moreElementsVectorSrc(MI, MoreTy, 2);
5149
5150	// Adjust mask based on new input vector length.
5151	SmallVector<int, 16> NewMask;
5152	for (unsigned I = 0; I != NumElts; ++I) {
5153	int Idx = Mask[I];
5154	if (Idx < static_cast<int>(NumElts))
5155	NewMask.push_back(Idx);
5156	else
5157	NewMask.push_back(Idx - NumElts + WidenNumElts);
5158	}
5159	for (unsigned I = NumElts; I != WidenNumElts; ++I)
5160	NewMask.push_back(-1);
5161	moreElementsVectorDst(MI, MoreTy, 0);
5162	MIRBuilder.setInstrAndDebugLoc(MI);
5163	MIRBuilder.buildShuffleVector(MI.getOperand(0).getReg(),
5164	MI.getOperand(1).getReg(),
5165	MI.getOperand(2).getReg(), NewMask);
5166	MI.eraseFromParent();
5167	return Legalized;
5168	}
5169
5170	void LegalizerHelper::multiplyRegisters(SmallVectorImpl<Register> &DstRegs,
5171	ArrayRef<Register> Src1Regs,
5172	ArrayRef<Register> Src2Regs,
5173	LLT NarrowTy) {
5174	MachineIRBuilder &B = MIRBuilder;
5175	unsigned SrcParts = Src1Regs.size();
5176	unsigned DstParts = DstRegs.size();
5177
5178	unsigned DstIdx = 0; // Low bits of the result.
5179	Register FactorSum =
5180	B.buildMul(NarrowTy, Src1Regs[DstIdx], Src2Regs[DstIdx]).getReg(0);
5181	DstRegs[DstIdx] = FactorSum;
5182
5183	unsigned CarrySumPrevDstIdx;
5184	SmallVector<Register, 4> Factors;
5185
5186	for (DstIdx = 1; DstIdx < DstParts; DstIdx++) {
5187	// Collect low parts of muls for DstIdx.
5188	for (unsigned i = DstIdx + 1 < SrcParts ? 0 : DstIdx - SrcParts + 1;
5189	i <= std::min(DstIdx, SrcParts - 1); ++i) {
5190	MachineInstrBuilder Mul =
5191	B.buildMul(NarrowTy, Src1Regs[DstIdx - i], Src2Regs[i]);
5192	Factors.push_back(Mul.getReg(0));
5193	}
5194	// Collect high parts of muls from previous DstIdx.
5195	for (unsigned i = DstIdx < SrcParts ? 0 : DstIdx - SrcParts;
5196	i <= std::min(DstIdx - 1, SrcParts - 1); ++i) {
5197	MachineInstrBuilder Umulh =
5198	B.buildUMulH(NarrowTy, Src1Regs[DstIdx - 1 - i], Src2Regs[i]);
5199	Factors.push_back(Umulh.getReg(0));
5200	}
5201	// Add CarrySum from additions calculated for previous DstIdx.
5202	if (DstIdx != 1) {
5203	Factors.push_back(CarrySumPrevDstIdx);
5204	}
5205
5206	Register CarrySum;
5207	// Add all factors and accumulate all carries into CarrySum.
5208	if (DstIdx != DstParts - 1) {
5209	MachineInstrBuilder Uaddo =
5210	B.buildUAddo(NarrowTy, LLT::scalar(1), Factors[0], Factors[1]);
5211	FactorSum = Uaddo.getReg(0);
5212	CarrySum = B.buildZExt(NarrowTy, Uaddo.getReg(1)).getReg(0);
5213	for (unsigned i = 2; i < Factors.size(); ++i) {
5214	MachineInstrBuilder Uaddo =
5215	B.buildUAddo(NarrowTy, LLT::scalar(1), FactorSum, Factors[i]);
5216	FactorSum = Uaddo.getReg(0);
5217	MachineInstrBuilder Carry = B.buildZExt(NarrowTy, Uaddo.getReg(1));
5218	CarrySum = B.buildAdd(NarrowTy, CarrySum, Carry).getReg(0);
5219	}
5220	} else {
5221	// Since value for the next index is not calculated, neither is CarrySum.
5222	FactorSum = B.buildAdd(NarrowTy, Factors[0], Factors[1]).getReg(0);
5223	for (unsigned i = 2; i < Factors.size(); ++i)
5224	FactorSum = B.buildAdd(NarrowTy, FactorSum, Factors[i]).getReg(0);
5225	}
5226
5227	CarrySumPrevDstIdx = CarrySum;
5228	DstRegs[DstIdx] = FactorSum;
5229	Factors.clear();
5230	}
5231	}
5232
5233	LegalizerHelper::LegalizeResult
5234	LegalizerHelper::narrowScalarAddSub(MachineInstr &MI, unsigned TypeIdx,
5235	LLT NarrowTy) {
5236	if (TypeIdx != 0)
5237	return UnableToLegalize;
5238
5239	Register DstReg = MI.getOperand(0).getReg();
5240	LLT DstType = MRI.getType(DstReg);
5241	// FIXME: add support for vector types
5242	if (DstType.isVector())
5243	return UnableToLegalize;
5244
5245	unsigned Opcode = MI.getOpcode();
5246	unsigned OpO, OpE, OpF;
5247	switch (Opcode) {
5248	case TargetOpcode::G_SADDO:
5249	case TargetOpcode::G_SADDE:
5250	case TargetOpcode::G_UADDO:
5251	case TargetOpcode::G_UADDE:
5252	case TargetOpcode::G_ADD:
5253	OpO = TargetOpcode::G_UADDO;
5254	OpE = TargetOpcode::G_UADDE;
5255	OpF = TargetOpcode::G_UADDE;
5256	if (Opcode == TargetOpcode::G_SADDO \|\| Opcode == TargetOpcode::G_SADDE)
5257	OpF = TargetOpcode::G_SADDE;
5258	break;
5259	case TargetOpcode::G_SSUBO:
5260	case TargetOpcode::G_SSUBE:
5261	case TargetOpcode::G_USUBO:
5262	case TargetOpcode::G_USUBE:
5263	case TargetOpcode::G_SUB:
5264	OpO = TargetOpcode::G_USUBO;
5265	OpE = TargetOpcode::G_USUBE;
5266	OpF = TargetOpcode::G_USUBE;
5267	if (Opcode == TargetOpcode::G_SSUBO \|\| Opcode == TargetOpcode::G_SSUBE)
5268	OpF = TargetOpcode::G_SSUBE;
5269	break;
5270	default:
5271	llvm_unreachable("Unexpected add/sub opcode!")::llvm::llvm_unreachable_internal("Unexpected add/sub opcode!" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 5271);
5272	}
5273
5274	// 1 for a plain add/sub, 2 if this is an operation with a carry-out.
5275	unsigned NumDefs = MI.getNumExplicitDefs();
5276	Register Src1 = MI.getOperand(NumDefs).getReg();
5277	Register Src2 = MI.getOperand(NumDefs + 1).getReg();
5278	Register CarryDst, CarryIn;
5279	if (NumDefs == 2)
5280	CarryDst = MI.getOperand(1).getReg();
5281	if (MI.getNumOperands() == NumDefs + 3)
5282	CarryIn = MI.getOperand(NumDefs + 2).getReg();
5283
5284	LLT RegTy = MRI.getType(MI.getOperand(0).getReg());
5285	LLT LeftoverTy, DummyTy;
5286	SmallVector<Register, 2> Src1Regs, Src2Regs, Src1Left, Src2Left, DstRegs;
5287	extractParts(Src1, RegTy, NarrowTy, LeftoverTy, Src1Regs, Src1Left);
5288	extractParts(Src2, RegTy, NarrowTy, DummyTy, Src2Regs, Src2Left);
5289
5290	int NarrowParts = Src1Regs.size();
5291	for (int I = 0, E = Src1Left.size(); I != E; ++I) {
5292	Src1Regs.push_back(Src1Left[I]);
5293	Src2Regs.push_back(Src2Left[I]);
5294	}
5295	DstRegs.reserve(Src1Regs.size());
5296
5297	for (int i = 0, e = Src1Regs.size(); i != e; ++i) {
5298	Register DstReg =
5299	MRI.createGenericVirtualRegister(MRI.getType(Src1Regs[i]));
5300	Register CarryOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
5301	// Forward the final carry-out to the destination register
5302	if (i == e - 1 && CarryDst)
5303	CarryOut = CarryDst;
5304
5305	if (!CarryIn) {
5306	MIRBuilder.buildInstr(OpO, {DstReg, CarryOut},
5307	{Src1Regs[i], Src2Regs[i]});
5308	} else if (i == e - 1) {
5309	MIRBuilder.buildInstr(OpF, {DstReg, CarryOut},
5310	{Src1Regs[i], Src2Regs[i], CarryIn});
5311	} else {
5312	MIRBuilder.buildInstr(OpE, {DstReg, CarryOut},
5313	{Src1Regs[i], Src2Regs[i], CarryIn});
5314	}
5315
5316	DstRegs.push_back(DstReg);
5317	CarryIn = CarryOut;
5318	}
5319	insertParts(MI.getOperand(0).getReg(), RegTy, NarrowTy,
5320	makeArrayRef(DstRegs).take_front(NarrowParts), LeftoverTy,
5321	makeArrayRef(DstRegs).drop_front(NarrowParts));
5322
5323	MI.eraseFromParent();
5324	return Legalized;
5325	}
5326
5327	LegalizerHelper::LegalizeResult
5328	LegalizerHelper::narrowScalarMul(MachineInstr &MI, LLT NarrowTy) {
5329	Register DstReg = MI.getOperand(0).getReg();
5330	Register Src1 = MI.getOperand(1).getReg();
5331	Register Src2 = MI.getOperand(2).getReg();
5332
5333	LLT Ty = MRI.getType(DstReg);
5334	if (Ty.isVector())
5335	return UnableToLegalize;
5336
5337	unsigned SrcSize = MRI.getType(Src1).getSizeInBits();
5338	unsigned DstSize = Ty.getSizeInBits();
5339	unsigned NarrowSize = NarrowTy.getSizeInBits();
5340	if (DstSize % NarrowSize != 0 \|\| SrcSize % NarrowSize != 0)
5341	return UnableToLegalize;
5342
5343	unsigned NumDstParts = DstSize / NarrowSize;
5344	unsigned NumSrcParts = SrcSize / NarrowSize;
5345	bool IsMulHigh = MI.getOpcode() == TargetOpcode::G_UMULH;
5346	unsigned DstTmpParts = NumDstParts * (IsMulHigh ? 2 : 1);
5347
5348	SmallVector<Register, 2> Src1Parts, Src2Parts;
5349	SmallVector<Register, 2> DstTmpRegs(DstTmpParts);
5350	extractParts(Src1, NarrowTy, NumSrcParts, Src1Parts);
5351	extractParts(Src2, NarrowTy, NumSrcParts, Src2Parts);
5352	multiplyRegisters(DstTmpRegs, Src1Parts, Src2Parts, NarrowTy);
5353
5354	// Take only high half of registers if this is high mul.
5355	ArrayRef<Register> DstRegs(
5356	IsMulHigh ? &DstTmpRegs[DstTmpParts / 2] : &DstTmpRegs[0], NumDstParts);
5357	MIRBuilder.buildMerge(DstReg, DstRegs);
5358	MI.eraseFromParent();
5359	return Legalized;
5360	}
5361
5362	LegalizerHelper::LegalizeResult
5363	LegalizerHelper::narrowScalarFPTOI(MachineInstr &MI, unsigned TypeIdx,
5364	LLT NarrowTy) {
5365	if (TypeIdx != 0)
5366	return UnableToLegalize;
5367
5368	bool IsSigned = MI.getOpcode() == TargetOpcode::G_FPTOSI;
5369
5370	Register Src = MI.getOperand(1).getReg();
5371	LLT SrcTy = MRI.getType(Src);
5372
5373	// If all finite floats fit into the narrowed integer type, we can just swap
5374	// out the result type. This is practically only useful for conversions from
5375	// half to at least 16-bits, so just handle the one case.
5376	if (SrcTy.getScalarType() != LLT::scalar(16) \|\|
5377	NarrowTy.getScalarSizeInBits() < (IsSigned ? 17u : 16u))
5378	return UnableToLegalize;
5379
5380	Observer.changingInstr(MI);
5381	narrowScalarDst(MI, NarrowTy, 0,
5382	IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT);
5383	Observer.changedInstr(MI);
5384	return Legalized;
5385	}
5386
5387	LegalizerHelper::LegalizeResult
5388	LegalizerHelper::narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx,
5389	LLT NarrowTy) {
5390	if (TypeIdx != 1)
5391	return UnableToLegalize;
5392
5393	uint64_t NarrowSize = NarrowTy.getSizeInBits();
5394
5395	int64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
5396	// FIXME: add support for when SizeOp1 isn't an exact multiple of
5397	// NarrowSize.
5398	if (SizeOp1 % NarrowSize != 0)
5399	return UnableToLegalize;
5400	int NumParts = SizeOp1 / NarrowSize;
5401
5402	SmallVector<Register, 2> SrcRegs, DstRegs;
5403	SmallVector<uint64_t, 2> Indexes;
5404	extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
5405
5406	Register OpReg = MI.getOperand(0).getReg();
5407	uint64_t OpStart = MI.getOperand(2).getImm();
5408	uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
5409	for (int i = 0; i < NumParts; ++i) {
5410	unsigned SrcStart = i * NarrowSize;
5411
5412	if (SrcStart + NarrowSize <= OpStart \|\| SrcStart >= OpStart + OpSize) {
5413	// No part of the extract uses this subregister, ignore it.
5414	continue;
5415	} else if (SrcStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
5416	// The entire subregister is extracted, forward the value.
5417	DstRegs.push_back(SrcRegs[i]);
5418	continue;
5419	}
5420
5421	// OpSegStart is where this destination segment would start in OpReg if it
5422	// extended infinitely in both directions.
5423	int64_t ExtractOffset;
5424	uint64_t SegSize;
5425	if (OpStart < SrcStart) {
5426	ExtractOffset = 0;
5427	SegSize = std::min(NarrowSize, OpStart + OpSize - SrcStart);
5428	} else {
5429	ExtractOffset = OpStart - SrcStart;
5430	SegSize = std::min(SrcStart + NarrowSize - OpStart, OpSize);
5431	}
5432
5433	Register SegReg = SrcRegs[i];
5434	if (ExtractOffset != 0 \|\| SegSize != NarrowSize) {
5435	// A genuine extract is needed.
5436	SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
5437	MIRBuilder.buildExtract(SegReg, SrcRegs[i], ExtractOffset);
5438	}
5439
5440	DstRegs.push_back(SegReg);
5441	}
5442
5443	Register DstReg = MI.getOperand(0).getReg();
5444	if (MRI.getType(DstReg).isVector())
5445	MIRBuilder.buildBuildVector(DstReg, DstRegs);
5446	else if (DstRegs.size() > 1)
5447	MIRBuilder.buildMerge(DstReg, DstRegs);
5448	else
5449	MIRBuilder.buildCopy(DstReg, DstRegs[0]);
5450	MI.eraseFromParent();
5451	return Legalized;
5452	}
5453
5454	LegalizerHelper::LegalizeResult
5455	LegalizerHelper::narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx,
5456	LLT NarrowTy) {
5457	// FIXME: Don't know how to handle secondary types yet.
5458	if (TypeIdx != 0)
5459	return UnableToLegalize;
5460
5461	SmallVector<Register, 2> SrcRegs, LeftoverRegs, DstRegs;
5462	SmallVector<uint64_t, 2> Indexes;
5463	LLT RegTy = MRI.getType(MI.getOperand(0).getReg());
5464	LLT LeftoverTy;
5465	extractParts(MI.getOperand(1).getReg(), RegTy, NarrowTy, LeftoverTy, SrcRegs,
5466	LeftoverRegs);
5467
5468	for (Register Reg : LeftoverRegs)
5469	SrcRegs.push_back(Reg);
5470
5471	uint64_t NarrowSize = NarrowTy.getSizeInBits();
5472	Register OpReg = MI.getOperand(2).getReg();
5473	uint64_t OpStart = MI.getOperand(3).getImm();
5474	uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
5475	for (int I = 0, E = SrcRegs.size(); I != E; ++I) {
5476	unsigned DstStart = I * NarrowSize;
5477
5478	if (DstStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
5479	// The entire subregister is defined by this insert, forward the new
5480	// value.
5481	DstRegs.push_back(OpReg);
5482	continue;
5483	}
5484
5485	Register SrcReg = SrcRegs[I];
5486	if (MRI.getType(SrcRegs[I]) == LeftoverTy) {
5487	// The leftover reg is smaller than NarrowTy, so we need to extend it.
5488	SrcReg = MRI.createGenericVirtualRegister(NarrowTy);
5489	MIRBuilder.buildAnyExt(SrcReg, SrcRegs[I]);
5490	}
5491
5492	if (DstStart + NarrowSize <= OpStart \|\| DstStart >= OpStart + OpSize) {
5493	// No part of the insert affects this subregister, forward the original.
5494	DstRegs.push_back(SrcReg);
5495	continue;
5496	}
5497
5498	// OpSegStart is where this destination segment would start in OpReg if it
5499	// extended infinitely in both directions.
5500	int64_t ExtractOffset, InsertOffset;
5501	uint64_t SegSize;
5502	if (OpStart < DstStart) {
5503	InsertOffset = 0;
5504	ExtractOffset = DstStart - OpStart;
5505	SegSize = std::min(NarrowSize, OpStart + OpSize - DstStart);
5506	} else {
5507	InsertOffset = OpStart - DstStart;
5508	ExtractOffset = 0;
5509	SegSize =
5510	std::min(NarrowSize - InsertOffset, OpStart + OpSize - DstStart);
5511	}
5512
5513	Register SegReg = OpReg;
5514	if (ExtractOffset != 0 \|\| SegSize != OpSize) {
5515	// A genuine extract is needed.
5516	SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
5517	MIRBuilder.buildExtract(SegReg, OpReg, ExtractOffset);
5518	}
5519
5520	Register DstReg = MRI.createGenericVirtualRegister(NarrowTy);
5521	MIRBuilder.buildInsert(DstReg, SrcReg, SegReg, InsertOffset);
5522	DstRegs.push_back(DstReg);
5523	}
5524
5525	uint64_t WideSize = DstRegs.size() * NarrowSize;
5526	Register DstReg = MI.getOperand(0).getReg();
5527	if (WideSize > RegTy.getSizeInBits()) {
5528	Register MergeReg = MRI.createGenericVirtualRegister(LLT::scalar(WideSize));
5529	MIRBuilder.buildMerge(MergeReg, DstRegs);
5530	MIRBuilder.buildTrunc(DstReg, MergeReg);
5531	} else
5532	MIRBuilder.buildMerge(DstReg, DstRegs);
5533
5534	MI.eraseFromParent();
5535	return Legalized;
5536	}
5537
5538	LegalizerHelper::LegalizeResult
5539	LegalizerHelper::narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx,
5540	LLT NarrowTy) {
5541	Register DstReg = MI.getOperand(0).getReg();
5542	LLT DstTy = MRI.getType(DstReg);
5543
5544	assert(MI.getNumOperands() == 3 && TypeIdx == 0)(static_cast <bool> (MI.getNumOperands() == 3 && TypeIdx == 0) ? void (0) : __assert_fail ("MI.getNumOperands() == 3 && TypeIdx == 0" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 5544, __extension__ __PRETTY_FUNCTION__));
5545
5546	SmallVector<Register, 4> DstRegs, DstLeftoverRegs;
5547	SmallVector<Register, 4> Src0Regs, Src0LeftoverRegs;
5548	SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs;
5549	LLT LeftoverTy;
5550	if (!extractParts(MI.getOperand(1).getReg(), DstTy, NarrowTy, LeftoverTy,
5551	Src0Regs, Src0LeftoverRegs))
5552	return UnableToLegalize;
5553
5554	LLT Unused;
5555	if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, Unused,
5556	Src1Regs, Src1LeftoverRegs))
5557	llvm_unreachable("inconsistent extractParts result")::llvm::llvm_unreachable_internal("inconsistent extractParts result" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 5557);
5558
5559	for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) {
5560	auto Inst = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy},
5561	{Src0Regs[I], Src1Regs[I]});
5562	DstRegs.push_back(Inst.getReg(0));
5563	}
5564
5565	for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) {
5566	auto Inst = MIRBuilder.buildInstr(
5567	MI.getOpcode(),
5568	{LeftoverTy}, {Src0LeftoverRegs[I], Src1LeftoverRegs[I]});
5569	DstLeftoverRegs.push_back(Inst.getReg(0));
5570	}
5571
5572	insertParts(DstReg, DstTy, NarrowTy, DstRegs,
5573	LeftoverTy, DstLeftoverRegs);
5574
5575	MI.eraseFromParent();
5576	return Legalized;
5577	}
5578
5579	LegalizerHelper::LegalizeResult
5580	LegalizerHelper::narrowScalarExt(MachineInstr &MI, unsigned TypeIdx,
5581	LLT NarrowTy) {
5582	if (TypeIdx != 0)
5583	return UnableToLegalize;
5584
5585	Register DstReg = MI.getOperand(0).getReg();
5586	Register SrcReg = MI.getOperand(1).getReg();
5587
5588	LLT DstTy = MRI.getType(DstReg);
5589	if (DstTy.isVector())
5590	return UnableToLegalize;
5591
5592	SmallVector<Register, 8> Parts;
5593	LLT GCDTy = extractGCDType(Parts, DstTy, NarrowTy, SrcReg);
5594	LLT LCMTy = buildLCMMergePieces(DstTy, NarrowTy, GCDTy, Parts, MI.getOpcode());
5595	buildWidenedRemergeToDst(DstReg, LCMTy, Parts);
5596
5597	MI.eraseFromParent();
5598	return Legalized;
5599	}
5600
5601	LegalizerHelper::LegalizeResult
5602	LegalizerHelper::narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx,
5603	LLT NarrowTy) {
5604	if (TypeIdx != 0)
5605	return UnableToLegalize;
5606
5607	Register CondReg = MI.getOperand(1).getReg();
5608	LLT CondTy = MRI.getType(CondReg);
5609	if (CondTy.isVector()) // TODO: Handle vselect
5610	return UnableToLegalize;
5611
5612	Register DstReg = MI.getOperand(0).getReg();
5613	LLT DstTy = MRI.getType(DstReg);
5614
5615	SmallVector<Register, 4> DstRegs, DstLeftoverRegs;
5616	SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs;
5617	SmallVector<Register, 4> Src2Regs, Src2LeftoverRegs;
5618	LLT LeftoverTy;
5619	if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, LeftoverTy,
5620	Src1Regs, Src1LeftoverRegs))
5621	return UnableToLegalize;
5622
5623	LLT Unused;
5624	if (!extractParts(MI.getOperand(3).getReg(), DstTy, NarrowTy, Unused,
5625	Src2Regs, Src2LeftoverRegs))
5626	llvm_unreachable("inconsistent extractParts result")::llvm::llvm_unreachable_internal("inconsistent extractParts result" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 5626);
5627
5628	for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) {
5629	auto Select = MIRBuilder.buildSelect(NarrowTy,
5630	CondReg, Src1Regs[I], Src2Regs[I]);
5631	DstRegs.push_back(Select.getReg(0));
5632	}
5633
5634	for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) {
5635	auto Select = MIRBuilder.buildSelect(
5636	LeftoverTy, CondReg, Src1LeftoverRegs[I], Src2LeftoverRegs[I]);
5637	DstLeftoverRegs.push_back(Select.getReg(0));
5638	}
5639
5640	insertParts(DstReg, DstTy, NarrowTy, DstRegs,
5641	LeftoverTy, DstLeftoverRegs);
5642
5643	MI.eraseFromParent();
5644	return Legalized;
5645	}
5646
5647	LegalizerHelper::LegalizeResult
5648	LegalizerHelper::narrowScalarCTLZ(MachineInstr &MI, unsigned TypeIdx,
5649	LLT NarrowTy) {
5650	if (TypeIdx != 1)
5651	return UnableToLegalize;
5652
5653	Register DstReg = MI.getOperand(0).getReg();
5654	Register SrcReg = MI.getOperand(1).getReg();
5655	LLT DstTy = MRI.getType(DstReg);
5656	LLT SrcTy = MRI.getType(SrcReg);
5657	unsigned NarrowSize = NarrowTy.getSizeInBits();
5658
5659	if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) {
5660	const bool IsUndef = MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF;
5661
5662	MachineIRBuilder &B = MIRBuilder;
5663	auto UnmergeSrc = B.buildUnmerge(NarrowTy, SrcReg);
5664	// ctlz(Hi:Lo) -> Hi == 0 ? (NarrowSize + ctlz(Lo)) : ctlz(Hi)
5665	auto C_0 = B.buildConstant(NarrowTy, 0);
5666	auto HiIsZero = B.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1),
5667	UnmergeSrc.getReg(1), C_0);
5668	auto LoCTLZ = IsUndef ?
5669	B.buildCTLZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(0)) :
5670	B.buildCTLZ(DstTy, UnmergeSrc.getReg(0));
5671	auto C_NarrowSize = B.buildConstant(DstTy, NarrowSize);
5672	auto HiIsZeroCTLZ = B.buildAdd(DstTy, LoCTLZ, C_NarrowSize);
5673	auto HiCTLZ = B.buildCTLZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(1));
5674	B.buildSelect(DstReg, HiIsZero, HiIsZeroCTLZ, HiCTLZ);
5675
5676	MI.eraseFromParent();
5677	return Legalized;
5678	}
5679
5680	return UnableToLegalize;
5681	}
5682
5683	LegalizerHelper::LegalizeResult
5684	LegalizerHelper::narrowScalarCTTZ(MachineInstr &MI, unsigned TypeIdx,
5685	LLT NarrowTy) {
5686	if (TypeIdx != 1)
5687	return UnableToLegalize;
5688
5689	Register DstReg = MI.getOperand(0).getReg();
5690	Register SrcReg = MI.getOperand(1).getReg();
5691	LLT DstTy = MRI.getType(DstReg);
5692	LLT SrcTy = MRI.getType(SrcReg);
5693	unsigned NarrowSize = NarrowTy.getSizeInBits();
5694
5695	if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) {
5696	const bool IsUndef = MI.getOpcode() == TargetOpcode::G_CTTZ_ZERO_UNDEF;
5697
5698	MachineIRBuilder &B = MIRBuilder;
5699	auto UnmergeSrc = B.buildUnmerge(NarrowTy, SrcReg);
5700	// cttz(Hi:Lo) -> Lo == 0 ? (cttz(Hi) + NarrowSize) : cttz(Lo)
5701	auto C_0 = B.buildConstant(NarrowTy, 0);
5702	auto LoIsZero = B.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1),
5703	UnmergeSrc.getReg(0), C_0);
5704	auto HiCTTZ = IsUndef ?
5705	B.buildCTTZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(1)) :
5706	B.buildCTTZ(DstTy, UnmergeSrc.getReg(1));
5707	auto C_NarrowSize = B.buildConstant(DstTy, NarrowSize);
5708	auto LoIsZeroCTTZ = B.buildAdd(DstTy, HiCTTZ, C_NarrowSize);
5709	auto LoCTTZ = B.buildCTTZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(0));
5710	B.buildSelect(DstReg, LoIsZero, LoIsZeroCTTZ, LoCTTZ);
5711
5712	MI.eraseFromParent();
5713	return Legalized;
5714	}
5715
5716	return UnableToLegalize;
5717	}
5718
5719	LegalizerHelper::LegalizeResult
5720	LegalizerHelper::narrowScalarCTPOP(MachineInstr &MI, unsigned TypeIdx,
5721	LLT NarrowTy) {
5722	if (TypeIdx != 1)
5723	return UnableToLegalize;
5724
5725	Register DstReg = MI.getOperand(0).getReg();
5726	LLT DstTy = MRI.getType(DstReg);
5727	LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
5728	unsigned NarrowSize = NarrowTy.getSizeInBits();
5729
5730	if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) {
5731	auto UnmergeSrc = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1));
5732
5733	auto LoCTPOP = MIRBuilder.buildCTPOP(DstTy, UnmergeSrc.getReg(0));
5734	auto HiCTPOP = MIRBuilder.buildCTPOP(DstTy, UnmergeSrc.getReg(1));
5735	MIRBuilder.buildAdd(DstReg, HiCTPOP, LoCTPOP);
5736
5737	MI.eraseFromParent();
5738	return Legalized;
5739	}
5740
5741	return UnableToLegalize;
5742	}
5743
5744	LegalizerHelper::LegalizeResult
5745	LegalizerHelper::lowerBitCount(MachineInstr &MI) {
5746	unsigned Opc = MI.getOpcode();
5747	const auto &TII = MIRBuilder.getTII();
5748	auto isSupported = [this](const LegalityQuery &Q) {
5749	auto QAction = LI.getAction(Q).Action;
5750	return QAction == Legal \|\| QAction == Libcall \|\| QAction == Custom;
5751	};
5752	switch (Opc) {
5753	default:
5754	return UnableToLegalize;
5755	case TargetOpcode::G_CTLZ_ZERO_UNDEF: {
5756	// This trivially expands to CTLZ.
5757	Observer.changingInstr(MI);
5758	MI.setDesc(TII.get(TargetOpcode::G_CTLZ));
5759	Observer.changedInstr(MI);
5760	return Legalized;
5761	}
5762	case TargetOpcode::G_CTLZ: {
5763	Register DstReg = MI.getOperand(0).getReg();
5764	Register SrcReg = MI.getOperand(1).getReg();
5765	LLT DstTy = MRI.getType(DstReg);
5766	LLT SrcTy = MRI.getType(SrcReg);
5767	unsigned Len = SrcTy.getSizeInBits();
5768
5769	if (isSupported({TargetOpcode::G_CTLZ_ZERO_UNDEF, {DstTy, SrcTy}})) {
5770	// If CTLZ_ZERO_UNDEF is supported, emit that and a select for zero.
5771	auto CtlzZU = MIRBuilder.buildCTLZ_ZERO_UNDEF(DstTy, SrcReg);
5772	auto ZeroSrc = MIRBuilder.buildConstant(SrcTy, 0);
5773	auto ICmp = MIRBuilder.buildICmp(
5774	CmpInst::ICMP_EQ, SrcTy.changeElementSize(1), SrcReg, ZeroSrc);
5775	auto LenConst = MIRBuilder.buildConstant(DstTy, Len);
5776	MIRBuilder.buildSelect(DstReg, ICmp, LenConst, CtlzZU);
5777	MI.eraseFromParent();
5778	return Legalized;
5779	}
5780	// for now, we do this:
5781	// NewLen = NextPowerOf2(Len);
5782	// x = x \| (x >> 1);
5783	// x = x \| (x >> 2);
5784	// ...
5785	// x = x \| (x >>16);
5786	// x = x \| (x >>32); // for 64-bit input
5787	// Upto NewLen/2
5788	// return Len - popcount(x);
5789	//
5790	// Ref: "Hacker's Delight" by Henry Warren
5791	Register Op = SrcReg;
5792	unsigned NewLen = PowerOf2Ceil(Len);
5793	for (unsigned i = 0; (1U << i) <= (NewLen / 2); ++i) {
5794	auto MIBShiftAmt = MIRBuilder.buildConstant(SrcTy, 1ULL << i);
5795	auto MIBOp = MIRBuilder.buildOr(
5796	SrcTy, Op, MIRBuilder.buildLShr(SrcTy, Op, MIBShiftAmt));
5797	Op = MIBOp.getReg(0);
5798	}
5799	auto MIBPop = MIRBuilder.buildCTPOP(DstTy, Op);
5800	MIRBuilder.buildSub(MI.getOperand(0), MIRBuilder.buildConstant(DstTy, Len),
5801	MIBPop);
5802	MI.eraseFromParent();
5803	return Legalized;
5804	}
5805	case TargetOpcode::G_CTTZ_ZERO_UNDEF: {
5806	// This trivially expands to CTTZ.
5807	Observer.changingInstr(MI);
5808	MI.setDesc(TII.get(TargetOpcode::G_CTTZ));
5809	Observer.changedInstr(MI);
5810	return Legalized;
5811	}
5812	case TargetOpcode::G_CTTZ: {
5813	Register DstReg = MI.getOperand(0).getReg();
5814	Register SrcReg = MI.getOperand(1).getReg();
5815	LLT DstTy = MRI.getType(DstReg);
5816	LLT SrcTy = MRI.getType(SrcReg);
5817
5818	unsigned Len = SrcTy.getSizeInBits();
5819	if (isSupported({TargetOpcode::G_CTTZ_ZERO_UNDEF, {DstTy, SrcTy}})) {
5820	// If CTTZ_ZERO_UNDEF is legal or custom, emit that and a select with
5821	// zero.
5822	auto CttzZU = MIRBuilder.buildCTTZ_ZERO_UNDEF(DstTy, SrcReg);
5823	auto Zero = MIRBuilder.buildConstant(SrcTy, 0);
5824	auto ICmp = MIRBuilder.buildICmp(
5825	CmpInst::ICMP_EQ, DstTy.changeElementSize(1), SrcReg, Zero);
5826	auto LenConst = MIRBuilder.buildConstant(DstTy, Len);
5827	MIRBuilder.buildSelect(DstReg, ICmp, LenConst, CttzZU);
5828	MI.eraseFromParent();
5829	return Legalized;
5830	}
5831	// for now, we use: { return popcount(~x & (x - 1)); }
5832	// unless the target has ctlz but not ctpop, in which case we use:
5833	// { return 32 - nlz(~x & (x-1)); }
5834	// Ref: "Hacker's Delight" by Henry Warren
5835	auto MIBCstNeg1 = MIRBuilder.buildConstant(SrcTy, -1);
5836	auto MIBNot = MIRBuilder.buildXor(SrcTy, SrcReg, MIBCstNeg1);
5837	auto MIBTmp = MIRBuilder.buildAnd(
5838	SrcTy, MIBNot, MIRBuilder.buildAdd(SrcTy, SrcReg, MIBCstNeg1));
5839	if (!isSupported({TargetOpcode::G_CTPOP, {SrcTy, SrcTy}}) &&
5840	isSupported({TargetOpcode::G_CTLZ, {SrcTy, SrcTy}})) {
5841	auto MIBCstLen = MIRBuilder.buildConstant(SrcTy, Len);
5842	MIRBuilder.buildSub(MI.getOperand(0), MIBCstLen,
5843	MIRBuilder.buildCTLZ(SrcTy, MIBTmp));
5844	MI.eraseFromParent();
5845	return Legalized;
5846	}
5847	MI.setDesc(TII.get(TargetOpcode::G_CTPOP));
5848	MI.getOperand(1).setReg(MIBTmp.getReg(0));
5849	return Legalized;
5850	}
5851	case TargetOpcode::G_CTPOP: {
5852	Register SrcReg = MI.getOperand(1).getReg();
5853	LLT Ty = MRI.getType(SrcReg);
5854	unsigned Size = Ty.getSizeInBits();
5855	MachineIRBuilder &B = MIRBuilder;
5856
5857	// Count set bits in blocks of 2 bits. Default approach would be
5858	// B2Count = { val & 0x55555555 } + { (val >> 1) & 0x55555555 }
5859	// We use following formula instead:
5860	// B2Count = val - { (val >> 1) & 0x55555555 }
5861	// since it gives same result in blocks of 2 with one instruction less.
5862	auto C_1 = B.buildConstant(Ty, 1);
5863	auto B2Set1LoTo1Hi = B.buildLShr(Ty, SrcReg, C_1);
5864	APInt B2Mask1HiTo0 = APInt::getSplat(Size, APInt(8, 0x55));
5865	auto C_B2Mask1HiTo0 = B.buildConstant(Ty, B2Mask1HiTo0);
5866	auto B2Count1Hi = B.buildAnd(Ty, B2Set1LoTo1Hi, C_B2Mask1HiTo0);
5867	auto B2Count = B.buildSub(Ty, SrcReg, B2Count1Hi);
5868
5869	// In order to get count in blocks of 4 add values from adjacent block of 2.
5870	// B4Count = { B2Count & 0x33333333 } + { (B2Count >> 2) & 0x33333333 }
5871	auto C_2 = B.buildConstant(Ty, 2);
5872	auto B4Set2LoTo2Hi = B.buildLShr(Ty, B2Count, C_2);
5873	APInt B4Mask2HiTo0 = APInt::getSplat(Size, APInt(8, 0x33));
5874	auto C_B4Mask2HiTo0 = B.buildConstant(Ty, B4Mask2HiTo0);
5875	auto B4HiB2Count = B.buildAnd(Ty, B4Set2LoTo2Hi, C_B4Mask2HiTo0);
5876	auto B4LoB2Count = B.buildAnd(Ty, B2Count, C_B4Mask2HiTo0);
5877	auto B4Count = B.buildAdd(Ty, B4HiB2Count, B4LoB2Count);
5878
5879	// For count in blocks of 8 bits we don't have to mask high 4 bits before
5880	// addition since count value sits in range {0,...,8} and 4 bits are enough
5881	// to hold such binary values. After addition high 4 bits still hold count
5882	// of set bits in high 4 bit block, set them to zero and get 8 bit result.
5883	// B8Count = { B4Count + (B4Count >> 4) } & 0x0F0F0F0F
5884	auto C_4 = B.buildConstant(Ty, 4);
5885	auto B8HiB4Count = B.buildLShr(Ty, B4Count, C_4);
5886	auto B8CountDirty4Hi = B.buildAdd(Ty, B8HiB4Count, B4Count);
5887	APInt B8Mask4HiTo0 = APInt::getSplat(Size, APInt(8, 0x0F));
5888	auto C_B8Mask4HiTo0 = B.buildConstant(Ty, B8Mask4HiTo0);
5889	auto B8Count = B.buildAnd(Ty, B8CountDirty4Hi, C_B8Mask4HiTo0);
5890
5891	assert(Size<=128 && "Scalar size is too large for CTPOP lower algorithm")(static_cast <bool> (Size<=128 && "Scalar size is too large for CTPOP lower algorithm" ) ? void (0) : __assert_fail ("Size<=128 && \"Scalar size is too large for CTPOP lower algorithm\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 5891, __extension__ __PRETTY_FUNCTION__));
5892	// 8 bits can hold CTPOP result of 128 bit int or smaller. Mul with this
5893	// bitmask will set 8 msb in ResTmp to sum of all B8Counts in 8 bit blocks.
5894	auto MulMask = B.buildConstant(Ty, APInt::getSplat(Size, APInt(8, 0x01)));
5895	auto ResTmp = B.buildMul(Ty, B8Count, MulMask);
5896
5897	// Shift count result from 8 high bits to low bits.
5898	auto C_SizeM8 = B.buildConstant(Ty, Size - 8);
5899	B.buildLShr(MI.getOperand(0).getReg(), ResTmp, C_SizeM8);
5900
5901	MI.eraseFromParent();
5902	return Legalized;
5903	}
5904	}
5905	}
5906
5907	// Check that (every element of) Reg is undef or not an exact multiple of BW.
5908	static bool isNonZeroModBitWidthOrUndef(const MachineRegisterInfo &MRI,
5909	Register Reg, unsigned BW) {
5910	return matchUnaryPredicate(
5911	MRI, Reg,
5912	[=](const Constant *C) {
5913	// Null constant here means an undef.
5914	const ConstantInt *CI = dyn_cast_or_null<ConstantInt>(C);
5915	return !CI \|\| CI->getValue().urem(BW) != 0;
5916	},
5917	/AllowUndefs/ true);
5918	}
5919
5920	LegalizerHelper::LegalizeResult
5921	LegalizerHelper::lowerFunnelShiftWithInverse(MachineInstr &MI) {
5922	Register Dst = MI.getOperand(0).getReg();
5923	Register X = MI.getOperand(1).getReg();
5924	Register Y = MI.getOperand(2).getReg();
5925	Register Z = MI.getOperand(3).getReg();
5926	LLT Ty = MRI.getType(Dst);
5927	LLT ShTy = MRI.getType(Z);
5928
5929	unsigned BW = Ty.getScalarSizeInBits();
5930
5931	if (!isPowerOf2_32(BW))
5932	return UnableToLegalize;
5933
5934	const bool IsFSHL = MI.getOpcode() == TargetOpcode::G_FSHL;
5935	unsigned RevOpcode = IsFSHL ? TargetOpcode::G_FSHR : TargetOpcode::G_FSHL;
5936
5937	if (isNonZeroModBitWidthOrUndef(MRI, Z, BW)) {
5938	// fshl X, Y, Z -> fshr X, Y, -Z
5939	// fshr X, Y, Z -> fshl X, Y, -Z
5940	auto Zero = MIRBuilder.buildConstant(ShTy, 0);
5941	Z = MIRBuilder.buildSub(Ty, Zero, Z).getReg(0);
5942	} else {
5943	// fshl X, Y, Z -> fshr (srl X, 1), (fshr X, Y, 1), ~Z
5944	// fshr X, Y, Z -> fshl (fshl X, Y, 1), (shl Y, 1), ~Z
5945	auto One = MIRBuilder.buildConstant(ShTy, 1);
5946	if (IsFSHL) {
5947	Y = MIRBuilder.buildInstr(RevOpcode, {Ty}, {X, Y, One}).getReg(0);
5948	X = MIRBuilder.buildLShr(Ty, X, One).getReg(0);
5949	} else {
5950	X = MIRBuilder.buildInstr(RevOpcode, {Ty}, {X, Y, One}).getReg(0);
5951	Y = MIRBuilder.buildShl(Ty, Y, One).getReg(0);
5952	}
5953
5954	Z = MIRBuilder.buildNot(ShTy, Z).getReg(0);
5955	}
5956
5957	MIRBuilder.buildInstr(RevOpcode, {Dst}, {X, Y, Z});
5958	MI.eraseFromParent();
5959	return Legalized;
5960	}
5961
5962	LegalizerHelper::LegalizeResult
5963	LegalizerHelper::lowerFunnelShiftAsShifts(MachineInstr &MI) {
5964	Register Dst = MI.getOperand(0).getReg();
5965	Register X = MI.getOperand(1).getReg();
5966	Register Y = MI.getOperand(2).getReg();
5967	Register Z = MI.getOperand(3).getReg();
5968	LLT Ty = MRI.getType(Dst);
5969	LLT ShTy = MRI.getType(Z);
5970
5971	const unsigned BW = Ty.getScalarSizeInBits();
5972	const bool IsFSHL = MI.getOpcode() == TargetOpcode::G_FSHL;
5973
5974	Register ShX, ShY;
5975	Register ShAmt, InvShAmt;
5976
5977	// FIXME: Emit optimized urem by constant instead of letting it expand later.
5978	if (isNonZeroModBitWidthOrUndef(MRI, Z, BW)) {
5979	// fshl: X << C \| Y >> (BW - C)
5980	// fshr: X << (BW - C) \| Y >> C
5981	// where C = Z % BW is not zero
5982	auto BitWidthC = MIRBuilder.buildConstant(ShTy, BW);
5983	ShAmt = MIRBuilder.buildURem(ShTy, Z, BitWidthC).getReg(0);
5984	InvShAmt = MIRBuilder.buildSub(ShTy, BitWidthC, ShAmt).getReg(0);
5985	ShX = MIRBuilder.buildShl(Ty, X, IsFSHL ? ShAmt : InvShAmt).getReg(0);
5986	ShY = MIRBuilder.buildLShr(Ty, Y, IsFSHL ? InvShAmt : ShAmt).getReg(0);
5987	} else {
5988	// fshl: X << (Z % BW) \| Y >> 1 >> (BW - 1 - (Z % BW))
5989	// fshr: X << 1 << (BW - 1 - (Z % BW)) \| Y >> (Z % BW)
5990	auto Mask = MIRBuilder.buildConstant(ShTy, BW - 1);
5991	if (isPowerOf2_32(BW)) {
5992	// Z % BW -> Z & (BW - 1)
5993	ShAmt = MIRBuilder.buildAnd(ShTy, Z, Mask).getReg(0);
5994	// (BW - 1) - (Z % BW) -> ~Z & (BW - 1)
5995	auto NotZ = MIRBuilder.buildNot(ShTy, Z);
5996	InvShAmt = MIRBuilder.buildAnd(ShTy, NotZ, Mask).getReg(0);
5997	} else {
5998	auto BitWidthC = MIRBuilder.buildConstant(ShTy, BW);
5999	ShAmt = MIRBuilder.buildURem(ShTy, Z, BitWidthC).getReg(0);
6000	InvShAmt = MIRBuilder.buildSub(ShTy, Mask, ShAmt).getReg(0);
6001	}
6002
6003	auto One = MIRBuilder.buildConstant(ShTy, 1);
6004	if (IsFSHL) {
6005	ShX = MIRBuilder.buildShl(Ty, X, ShAmt).getReg(0);
6006	auto ShY1 = MIRBuilder.buildLShr(Ty, Y, One);
6007	ShY = MIRBuilder.buildLShr(Ty, ShY1, InvShAmt).getReg(0);
6008	} else {
6009	auto ShX1 = MIRBuilder.buildShl(Ty, X, One);
6010	ShX = MIRBuilder.buildShl(Ty, ShX1, InvShAmt).getReg(0);
6011	ShY = MIRBuilder.buildLShr(Ty, Y, ShAmt).getReg(0);
6012	}
6013	}
6014
6015	MIRBuilder.buildOr(Dst, ShX, ShY);
6016	MI.eraseFromParent();
6017	return Legalized;
6018	}
6019
6020	LegalizerHelper::LegalizeResult
6021	LegalizerHelper::lowerFunnelShift(MachineInstr &MI) {
6022	// These operations approximately do the following (while avoiding undefined
6023	// shifts by BW):
6024	// G_FSHL: (X << (Z % BW)) \| (Y >> (BW - (Z % BW)))
6025	// G_FSHR: (X << (BW - (Z % BW))) \| (Y >> (Z % BW))
6026	Register Dst = MI.getOperand(0).getReg();
6027	LLT Ty = MRI.getType(Dst);
6028	LLT ShTy = MRI.getType(MI.getOperand(3).getReg());
6029
6030	bool IsFSHL = MI.getOpcode() == TargetOpcode::G_FSHL;
6031	unsigned RevOpcode = IsFSHL ? TargetOpcode::G_FSHR : TargetOpcode::G_FSHL;
6032
6033	// TODO: Use smarter heuristic that accounts for vector legalization.
6034	if (LI.getAction({RevOpcode, {Ty, ShTy}}).Action == Lower)
6035	return lowerFunnelShiftAsShifts(MI);
6036
6037	// This only works for powers of 2, fallback to shifts if it fails.
6038	LegalizerHelper::LegalizeResult Result = lowerFunnelShiftWithInverse(MI);
6039	if (Result == UnableToLegalize)
6040	return lowerFunnelShiftAsShifts(MI);
6041	return Result;
6042	}
6043
6044	LegalizerHelper::LegalizeResult
6045	LegalizerHelper::lowerRotateWithReverseRotate(MachineInstr &MI) {
6046	Register Dst = MI.getOperand(0).getReg();
6047	Register Src = MI.getOperand(1).getReg();
6048	Register Amt = MI.getOperand(2).getReg();
6049	LLT AmtTy = MRI.getType(Amt);
6050	auto Zero = MIRBuilder.buildConstant(AmtTy, 0);
6051	bool IsLeft = MI.getOpcode() == TargetOpcode::G_ROTL;
6052	unsigned RevRot = IsLeft ? TargetOpcode::G_ROTR : TargetOpcode::G_ROTL;
6053	auto Neg = MIRBuilder.buildSub(AmtTy, Zero, Amt);
6054	MIRBuilder.buildInstr(RevRot, {Dst}, {Src, Neg});
6055	MI.eraseFromParent();
6056	return Legalized;
6057	}
6058
6059	LegalizerHelper::LegalizeResult LegalizerHelper::lowerRotate(MachineInstr &MI) {
6060	Register Dst = MI.getOperand(0).getReg();
6061	Register Src = MI.getOperand(1).getReg();
6062	Register Amt = MI.getOperand(2).getReg();
6063	LLT DstTy = MRI.getType(Dst);
6064	LLT SrcTy = MRI.getType(Dst);
6065	LLT AmtTy = MRI.getType(Amt);
6066
6067	unsigned EltSizeInBits = DstTy.getScalarSizeInBits();
6068	bool IsLeft = MI.getOpcode() == TargetOpcode::G_ROTL;
6069
6070	MIRBuilder.setInstrAndDebugLoc(MI);
6071
6072	// If a rotate in the other direction is supported, use it.
6073	unsigned RevRot = IsLeft ? TargetOpcode::G_ROTR : TargetOpcode::G_ROTL;
6074	if (LI.isLegalOrCustom({RevRot, {DstTy, SrcTy}}) &&
6075	isPowerOf2_32(EltSizeInBits))
6076	return lowerRotateWithReverseRotate(MI);
6077
6078	auto Zero = MIRBuilder.buildConstant(AmtTy, 0);
6079	unsigned ShOpc = IsLeft ? TargetOpcode::G_SHL : TargetOpcode::G_LSHR;
6080	unsigned RevShiftOpc = IsLeft ? TargetOpcode::G_LSHR : TargetOpcode::G_SHL;
6081	auto BitWidthMinusOneC = MIRBuilder.buildConstant(AmtTy, EltSizeInBits - 1);
6082	Register ShVal;
6083	Register RevShiftVal;
6084	if (isPowerOf2_32(EltSizeInBits)) {
6085	// (rotl x, c) -> x << (c & (w - 1)) \| x >> (-c & (w - 1))
6086	// (rotr x, c) -> x >> (c & (w - 1)) \| x << (-c & (w - 1))
6087	auto NegAmt = MIRBuilder.buildSub(AmtTy, Zero, Amt);
6088	auto ShAmt = MIRBuilder.buildAnd(AmtTy, Amt, BitWidthMinusOneC);
6089	ShVal = MIRBuilder.buildInstr(ShOpc, {DstTy}, {Src, ShAmt}).getReg(0);
6090	auto RevAmt = MIRBuilder.buildAnd(AmtTy, NegAmt, BitWidthMinusOneC);
6091	RevShiftVal =
6092	MIRBuilder.buildInstr(RevShiftOpc, {DstTy}, {Src, RevAmt}).getReg(0);
6093	} else {
6094	// (rotl x, c) -> x << (c % w) \| x >> 1 >> (w - 1 - (c % w))
6095	// (rotr x, c) -> x >> (c % w) \| x << 1 << (w - 1 - (c % w))
6096	auto BitWidthC = MIRBuilder.buildConstant(AmtTy, EltSizeInBits);
6097	auto ShAmt = MIRBuilder.buildURem(AmtTy, Amt, BitWidthC);
6098	ShVal = MIRBuilder.buildInstr(ShOpc, {DstTy}, {Src, ShAmt}).getReg(0);
6099	auto RevAmt = MIRBuilder.buildSub(AmtTy, BitWidthMinusOneC, ShAmt);
6100	auto One = MIRBuilder.buildConstant(AmtTy, 1);
6101	auto Inner = MIRBuilder.buildInstr(RevShiftOpc, {DstTy}, {Src, One});
6102	RevShiftVal =
6103	MIRBuilder.buildInstr(RevShiftOpc, {DstTy}, {Inner, RevAmt}).getReg(0);
6104	}
6105	MIRBuilder.buildOr(Dst, ShVal, RevShiftVal);
6106	MI.eraseFromParent();
6107	return Legalized;
6108	}
6109
6110	// Expand s32 = G_UITOFP s64 using bit operations to an IEEE float
6111	// representation.
6112	LegalizerHelper::LegalizeResult
6113	LegalizerHelper::lowerU64ToF32BitOps(MachineInstr &MI) {
6114	Register Dst = MI.getOperand(0).getReg();
6115	Register Src = MI.getOperand(1).getReg();
6116	const LLT S64 = LLT::scalar(64);
6117	const LLT S32 = LLT::scalar(32);
6118	const LLT S1 = LLT::scalar(1);
6119
6120	assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S32)(static_cast <bool> (MRI.getType(Src) == S64 && MRI.getType(Dst) == S32) ? void (0) : __assert_fail ("MRI.getType(Src) == S64 && MRI.getType(Dst) == S32" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 6120, __extension__ __PRETTY_FUNCTION__));
6121
6122	// unsigned cul2f(ulong u) {
6123	// uint lz = clz(u);
6124	// uint e = (u != 0) ? 127U + 63U - lz : 0;
6125	// u = (u << lz) & 0x7fffffffffffffffUL;
6126	// ulong t = u & 0xffffffffffUL;
6127	// uint v = (e << 23) \| (uint)(u >> 40);
6128	// uint r = t > 0x8000000000UL ? 1U : (t == 0x8000000000UL ? v & 1U : 0U);
6129	// return as_float(v + r);
6130	// }
6131
6132	auto Zero32 = MIRBuilder.buildConstant(S32, 0);
6133	auto Zero64 = MIRBuilder.buildConstant(S64, 0);
6134
6135	auto LZ = MIRBuilder.buildCTLZ_ZERO_UNDEF(S32, Src);
6136
6137	auto K = MIRBuilder.buildConstant(S32, 127U + 63U);
6138	auto Sub = MIRBuilder.buildSub(S32, K, LZ);
6139
6140	auto NotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, Src, Zero64);
6141	auto E = MIRBuilder.buildSelect(S32, NotZero, Sub, Zero32);
6142
6143	auto Mask0 = MIRBuilder.buildConstant(S64, (-1ULL) >> 1);
6144	auto ShlLZ = MIRBuilder.buildShl(S64, Src, LZ);
6145
6146	auto U = MIRBuilder.buildAnd(S64, ShlLZ, Mask0);
6147
6148	auto Mask1 = MIRBuilder.buildConstant(S64, 0xffffffffffULL);
6149	auto T = MIRBuilder.buildAnd(S64, U, Mask1);
6150
6151	auto UShl = MIRBuilder.buildLShr(S64, U, MIRBuilder.buildConstant(S64, 40));
6152	auto ShlE = MIRBuilder.buildShl(S32, E, MIRBuilder.buildConstant(S32, 23));
6153	auto V = MIRBuilder.buildOr(S32, ShlE, MIRBuilder.buildTrunc(S32, UShl));
6154
6155	auto C = MIRBuilder.buildConstant(S64, 0x8000000000ULL);
6156	auto RCmp = MIRBuilder.buildICmp(CmpInst::ICMP_UGT, S1, T, C);
6157	auto TCmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1, T, C);
6158	auto One = MIRBuilder.buildConstant(S32, 1);
6159
6160	auto VTrunc1 = MIRBuilder.buildAnd(S32, V, One);
6161	auto Select0 = MIRBuilder.buildSelect(S32, TCmp, VTrunc1, Zero32);
6162	auto R = MIRBuilder.buildSelect(S32, RCmp, One, Select0);
6163	MIRBuilder.buildAdd(Dst, V, R);
6164
6165	MI.eraseFromParent();
6166	return Legalized;
6167	}
6168
6169	LegalizerHelper::LegalizeResult LegalizerHelper::lowerUITOFP(MachineInstr &MI) {
6170	Register Dst = MI.getOperand(0).getReg();
6171	Register Src = MI.getOperand(1).getReg();
6172	LLT DstTy = MRI.getType(Dst);
6173	LLT SrcTy = MRI.getType(Src);
6174
6175	if (SrcTy == LLT::scalar(1)) {
6176	auto True = MIRBuilder.buildFConstant(DstTy, 1.0);
6177	auto False = MIRBuilder.buildFConstant(DstTy, 0.0);
6178	MIRBuilder.buildSelect(Dst, Src, True, False);
6179	MI.eraseFromParent();
6180	return Legalized;
6181	}
6182
6183	if (SrcTy != LLT::scalar(64))
6184	return UnableToLegalize;
6185
6186	if (DstTy == LLT::scalar(32)) {
6187	// TODO: SelectionDAG has several alternative expansions to port which may
6188	// be more reasonble depending on the available instructions. If a target
6189	// has sitofp, does not have CTLZ, or can efficiently use f64 as an
6190	// intermediate type, this is probably worse.
6191	return lowerU64ToF32BitOps(MI);
6192	}
6193
6194	return UnableToLegalize;
6195	}
6196
6197	LegalizerHelper::LegalizeResult LegalizerHelper::lowerSITOFP(MachineInstr &MI) {
6198	Register Dst = MI.getOperand(0).getReg();
6199	Register Src = MI.getOperand(1).getReg();
6200	LLT DstTy = MRI.getType(Dst);
6201	LLT SrcTy = MRI.getType(Src);
6202
6203	const LLT S64 = LLT::scalar(64);
6204	const LLT S32 = LLT::scalar(32);
6205	const LLT S1 = LLT::scalar(1);
6206
6207	if (SrcTy == S1) {
6208	auto True = MIRBuilder.buildFConstant(DstTy, -1.0);
6209	auto False = MIRBuilder.buildFConstant(DstTy, 0.0);
6210	MIRBuilder.buildSelect(Dst, Src, True, False);
6211	MI.eraseFromParent();
6212	return Legalized;
6213	}
6214
6215	if (SrcTy != S64)
6216	return UnableToLegalize;
6217
6218	if (DstTy == S32) {
6219	// signed cl2f(long l) {
6220	// long s = l >> 63;
6221	// float r = cul2f((l + s) ^ s);
6222	// return s ? -r : r;
6223	// }
6224	Register L = Src;
6225	auto SignBit = MIRBuilder.buildConstant(S64, 63);
6226	auto S = MIRBuilder.buildAShr(S64, L, SignBit);
6227
6228	auto LPlusS = MIRBuilder.buildAdd(S64, L, S);
6229	auto Xor = MIRBuilder.buildXor(S64, LPlusS, S);
6230	auto R = MIRBuilder.buildUITOFP(S32, Xor);
6231
6232	auto RNeg = MIRBuilder.buildFNeg(S32, R);
6233	auto SignNotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, S,
6234	MIRBuilder.buildConstant(S64, 0));
6235	MIRBuilder.buildSelect(Dst, SignNotZero, RNeg, R);
6236	MI.eraseFromParent();
6237	return Legalized;
6238	}
6239
6240	return UnableToLegalize;
6241	}
6242
6243	LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPTOUI(MachineInstr &MI) {
6244	Register Dst = MI.getOperand(0).getReg();
6245	Register Src = MI.getOperand(1).getReg();
6246	LLT DstTy = MRI.getType(Dst);
6247	LLT SrcTy = MRI.getType(Src);
6248	const LLT S64 = LLT::scalar(64);
6249	const LLT S32 = LLT::scalar(32);
6250
6251	if (SrcTy != S64 && SrcTy != S32)
6252	return UnableToLegalize;
6253	if (DstTy != S32 && DstTy != S64)
6254	return UnableToLegalize;
6255
6256	// FPTOSI gives same result as FPTOUI for positive signed integers.
6257	// FPTOUI needs to deal with fp values that convert to unsigned integers
6258	// greater or equal to 2^31 for float or 2^63 for double. For brevity 2^Exp.
6259
6260	APInt TwoPExpInt = APInt::getSignMask(DstTy.getSizeInBits());
6261	APFloat TwoPExpFP(SrcTy.getSizeInBits() == 32 ? APFloat::IEEEsingle()
6262	: APFloat::IEEEdouble(),
6263	APInt::getNullValue(SrcTy.getSizeInBits()));
6264	TwoPExpFP.convertFromAPInt(TwoPExpInt, false, APFloat::rmNearestTiesToEven);
6265
6266	MachineInstrBuilder FPTOSI = MIRBuilder.buildFPTOSI(DstTy, Src);
6267
6268	MachineInstrBuilder Threshold = MIRBuilder.buildFConstant(SrcTy, TwoPExpFP);
6269	// For fp Value greater or equal to Threshold(2^Exp), we use FPTOSI on
6270	// (Value - 2^Exp) and add 2^Exp by setting highest bit in result to 1.
6271	MachineInstrBuilder FSub = MIRBuilder.buildFSub(SrcTy, Src, Threshold);
6272	MachineInstrBuilder ResLowBits = MIRBuilder.buildFPTOSI(DstTy, FSub);
6273	MachineInstrBuilder ResHighBit = MIRBuilder.buildConstant(DstTy, TwoPExpInt);
6274	MachineInstrBuilder Res = MIRBuilder.buildXor(DstTy, ResLowBits, ResHighBit);
6275
6276	const LLT S1 = LLT::scalar(1);
6277
6278	MachineInstrBuilder FCMP =
6279	MIRBuilder.buildFCmp(CmpInst::FCMP_ULT, S1, Src, Threshold);
6280	MIRBuilder.buildSelect(Dst, FCMP, FPTOSI, Res);
6281
6282	MI.eraseFromParent();
6283	return Legalized;
6284	}
6285
6286	LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPTOSI(MachineInstr &MI) {
6287	Register Dst = MI.getOperand(0).getReg();
6288	Register Src = MI.getOperand(1).getReg();
6289	LLT DstTy = MRI.getType(Dst);
6290	LLT SrcTy = MRI.getType(Src);
6291	const LLT S64 = LLT::scalar(64);
6292	const LLT S32 = LLT::scalar(32);
6293
6294	// FIXME: Only f32 to i64 conversions are supported.
6295	if (SrcTy.getScalarType() != S32 \|\| DstTy.getScalarType() != S64)
6296	return UnableToLegalize;
6297
6298	// Expand f32 -> i64 conversion
6299	// This algorithm comes from compiler-rt's implementation of fixsfdi:
6300	// https://github.com/llvm/llvm-project/blob/main/compiler-rt/lib/builtins/fixsfdi.c
6301
6302	unsigned SrcEltBits = SrcTy.getScalarSizeInBits();
6303
6304	auto ExponentMask = MIRBuilder.buildConstant(SrcTy, 0x7F800000);
6305	auto ExponentLoBit = MIRBuilder.buildConstant(SrcTy, 23);
6306
6307	auto AndExpMask = MIRBuilder.buildAnd(SrcTy, Src, ExponentMask);
6308	auto ExponentBits = MIRBuilder.buildLShr(SrcTy, AndExpMask, ExponentLoBit);
6309
6310	auto SignMask = MIRBuilder.buildConstant(SrcTy,
6311	APInt::getSignMask(SrcEltBits));
6312	auto AndSignMask = MIRBuilder.buildAnd(SrcTy, Src, SignMask);
6313	auto SignLowBit = MIRBuilder.buildConstant(SrcTy, SrcEltBits - 1);
6314	auto Sign = MIRBuilder.buildAShr(SrcTy, AndSignMask, SignLowBit);
6315	Sign = MIRBuilder.buildSExt(DstTy, Sign);
6316
6317	auto MantissaMask = MIRBuilder.buildConstant(SrcTy, 0x007FFFFF);
6318	auto AndMantissaMask = MIRBuilder.buildAnd(SrcTy, Src, MantissaMask);
6319	auto K = MIRBuilder.buildConstant(SrcTy, 0x00800000);
6320
6321	auto R = MIRBuilder.buildOr(SrcTy, AndMantissaMask, K);
6322	R = MIRBuilder.buildZExt(DstTy, R);
6323
6324	auto Bias = MIRBuilder.buildConstant(SrcTy, 127);
6325	auto Exponent = MIRBuilder.buildSub(SrcTy, ExponentBits, Bias);
6326	auto SubExponent = MIRBuilder.buildSub(SrcTy, Exponent, ExponentLoBit);
6327	auto ExponentSub = MIRBuilder.buildSub(SrcTy, ExponentLoBit, Exponent);
6328
6329	auto Shl = MIRBuilder.buildShl(DstTy, R, SubExponent);
6330	auto Srl = MIRBuilder.buildLShr(DstTy, R, ExponentSub);
6331
6332	const LLT S1 = LLT::scalar(1);
6333	auto CmpGt = MIRBuilder.buildICmp(CmpInst::ICMP_SGT,
6334	S1, Exponent, ExponentLoBit);
6335
6336	R = MIRBuilder.buildSelect(DstTy, CmpGt, Shl, Srl);
6337
6338	auto XorSign = MIRBuilder.buildXor(DstTy, R, Sign);
6339	auto Ret = MIRBuilder.buildSub(DstTy, XorSign, Sign);
6340
6341	auto ZeroSrcTy = MIRBuilder.buildConstant(SrcTy, 0);
6342
6343	auto ExponentLt0 = MIRBuilder.buildICmp(CmpInst::ICMP_SLT,
6344	S1, Exponent, ZeroSrcTy);
6345
6346	auto ZeroDstTy = MIRBuilder.buildConstant(DstTy, 0);
6347	MIRBuilder.buildSelect(Dst, ExponentLt0, ZeroDstTy, Ret);
6348
6349	MI.eraseFromParent();
6350	return Legalized;
6351	}
6352
6353	// f64 -> f16 conversion using round-to-nearest-even rounding mode.
6354	LegalizerHelper::LegalizeResult
6355	LegalizerHelper::lowerFPTRUNC_F64_TO_F16(MachineInstr &MI) {
6356	Register Dst = MI.getOperand(0).getReg();
6357	Register Src = MI.getOperand(1).getReg();
6358
6359	if (MRI.getType(Src).isVector()) // TODO: Handle vectors directly.
6360	return UnableToLegalize;
6361
6362	const unsigned ExpMask = 0x7ff;
6363	const unsigned ExpBiasf64 = 1023;
6364	const unsigned ExpBiasf16 = 15;
6365	const LLT S32 = LLT::scalar(32);
6366	const LLT S1 = LLT::scalar(1);
6367
6368	auto Unmerge = MIRBuilder.buildUnmerge(S32, Src);
6369	Register U = Unmerge.getReg(0);
6370	Register UH = Unmerge.getReg(1);
6371
6372	auto E = MIRBuilder.buildLShr(S32, UH, MIRBuilder.buildConstant(S32, 20));
6373	E = MIRBuilder.buildAnd(S32, E, MIRBuilder.buildConstant(S32, ExpMask));
6374
6375	// Subtract the fp64 exponent bias (1023) to get the real exponent and
6376	// add the f16 bias (15) to get the biased exponent for the f16 format.
6377	E = MIRBuilder.buildAdd(
6378	S32, E, MIRBuilder.buildConstant(S32, -ExpBiasf64 + ExpBiasf16));
6379
6380	auto M = MIRBuilder.buildLShr(S32, UH, MIRBuilder.buildConstant(S32, 8));
6381	M = MIRBuilder.buildAnd(S32, M, MIRBuilder.buildConstant(S32, 0xffe));
6382
6383	auto MaskedSig = MIRBuilder.buildAnd(S32, UH,
6384	MIRBuilder.buildConstant(S32, 0x1ff));
6385	MaskedSig = MIRBuilder.buildOr(S32, MaskedSig, U);
6386
6387	auto Zero = MIRBuilder.buildConstant(S32, 0);
6388	auto SigCmpNE0 = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, MaskedSig, Zero);
6389	auto Lo40Set = MIRBuilder.buildZExt(S32, SigCmpNE0);
6390	M = MIRBuilder.buildOr(S32, M, Lo40Set);
6391
6392	// (M != 0 ? 0x0200 : 0) \| 0x7c00;
6393	auto Bits0x200 = MIRBuilder.buildConstant(S32, 0x0200);
6394	auto CmpM_NE0 = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, M, Zero);
6395	auto SelectCC = MIRBuilder.buildSelect(S32, CmpM_NE0, Bits0x200, Zero);
6396
6397	auto Bits0x7c00 = MIRBuilder.buildConstant(S32, 0x7c00);
6398	auto I = MIRBuilder.buildOr(S32, SelectCC, Bits0x7c00);
6399
6400	// N = M \| (E << 12);
6401	auto EShl12 = MIRBuilder.buildShl(S32, E, MIRBuilder.buildConstant(S32, 12));
6402	auto N = MIRBuilder.buildOr(S32, M, EShl12);
6403
6404	// B = clamp(1-E, 0, 13);
6405	auto One = MIRBuilder.buildConstant(S32, 1);
6406	auto OneSubExp = MIRBuilder.buildSub(S32, One, E);
6407	auto B = MIRBuilder.buildSMax(S32, OneSubExp, Zero);
6408	B = MIRBuilder.buildSMin(S32, B, MIRBuilder.buildConstant(S32, 13));
6409
6410	auto SigSetHigh = MIRBuilder.buildOr(S32, M,
6411	MIRBuilder.buildConstant(S32, 0x1000));
6412
6413	auto D = MIRBuilder.buildLShr(S32, SigSetHigh, B);
6414	auto D0 = MIRBuilder.buildShl(S32, D, B);
6415
6416	auto D0_NE_SigSetHigh = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1,
6417	D0, SigSetHigh);
6418	auto D1 = MIRBuilder.buildZExt(S32, D0_NE_SigSetHigh);
6419	D = MIRBuilder.buildOr(S32, D, D1);
6420
6421	auto CmpELtOne = MIRBuilder.buildICmp(CmpInst::ICMP_SLT, S1, E, One);
6422	auto V = MIRBuilder.buildSelect(S32, CmpELtOne, D, N);
6423
6424	auto VLow3 = MIRBuilder.buildAnd(S32, V, MIRBuilder.buildConstant(S32, 7));
6425	V = MIRBuilder.buildLShr(S32, V, MIRBuilder.buildConstant(S32, 2));
6426
6427	auto VLow3Eq3 = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1, VLow3,
6428	MIRBuilder.buildConstant(S32, 3));
6429	auto V0 = MIRBuilder.buildZExt(S32, VLow3Eq3);
6430
6431	auto VLow3Gt5 = MIRBuilder.buildICmp(CmpInst::ICMP_SGT, S1, VLow3,
6432	MIRBuilder.buildConstant(S32, 5));
6433	auto V1 = MIRBuilder.buildZExt(S32, VLow3Gt5);
6434
6435	V1 = MIRBuilder.buildOr(S32, V0, V1);
6436	V = MIRBuilder.buildAdd(S32, V, V1);
6437
6438	auto CmpEGt30 = MIRBuilder.buildICmp(CmpInst::ICMP_SGT, S1,
6439	E, MIRBuilder.buildConstant(S32, 30));
6440	V = MIRBuilder.buildSelect(S32, CmpEGt30,
6441	MIRBuilder.buildConstant(S32, 0x7c00), V);
6442
6443	auto CmpEGt1039 = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1,
6444	E, MIRBuilder.buildConstant(S32, 1039));
6445	V = MIRBuilder.buildSelect(S32, CmpEGt1039, I, V);
6446
6447	// Extract the sign bit.
6448	auto Sign = MIRBuilder.buildLShr(S32, UH, MIRBuilder.buildConstant(S32, 16));
6449	Sign = MIRBuilder.buildAnd(S32, Sign, MIRBuilder.buildConstant(S32, 0x8000));
6450
6451	// Insert the sign bit
6452	V = MIRBuilder.buildOr(S32, Sign, V);
6453
6454	MIRBuilder.buildTrunc(Dst, V);
6455	MI.eraseFromParent();
6456	return Legalized;
6457	}
6458
6459	LegalizerHelper::LegalizeResult
6460	LegalizerHelper::lowerFPTRUNC(MachineInstr &MI) {
6461	Register Dst = MI.getOperand(0).getReg();
6462	Register Src = MI.getOperand(1).getReg();
6463
6464	LLT DstTy = MRI.getType(Dst);
6465	LLT SrcTy = MRI.getType(Src);
6466	const LLT S64 = LLT::scalar(64);
6467	const LLT S16 = LLT::scalar(16);
6468
6469	if (DstTy.getScalarType() == S16 && SrcTy.getScalarType() == S64)
6470	return lowerFPTRUNC_F64_TO_F16(MI);
6471
6472	return UnableToLegalize;
6473	}
6474
6475	// TODO: If RHS is a constant SelectionDAGBuilder expands this into a
6476	// multiplication tree.
6477	LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPOWI(MachineInstr &MI) {
6478	Register Dst = MI.getOperand(0).getReg();
6479	Register Src0 = MI.getOperand(1).getReg();
6480	Register Src1 = MI.getOperand(2).getReg();
6481	LLT Ty = MRI.getType(Dst);
6482
6483	auto CvtSrc1 = MIRBuilder.buildSITOFP(Ty, Src1);
6484	MIRBuilder.buildFPow(Dst, Src0, CvtSrc1, MI.getFlags());
6485	MI.eraseFromParent();
6486	return Legalized;
6487	}
6488
6489	static CmpInst::Predicate minMaxToCompare(unsigned Opc) {
6490	switch (Opc) {
6491	case TargetOpcode::G_SMIN:
6492	return CmpInst::ICMP_SLT;
6493	case TargetOpcode::G_SMAX:
6494	return CmpInst::ICMP_SGT;
6495	case TargetOpcode::G_UMIN:
6496	return CmpInst::ICMP_ULT;
6497	case TargetOpcode::G_UMAX:
6498	return CmpInst::ICMP_UGT;
6499	default:
6500	llvm_unreachable("not in integer min/max")::llvm::llvm_unreachable_internal("not in integer min/max", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 6500);
6501	}
6502	}
6503
6504	LegalizerHelper::LegalizeResult LegalizerHelper::lowerMinMax(MachineInstr &MI) {
6505	Register Dst = MI.getOperand(0).getReg();
6506	Register Src0 = MI.getOperand(1).getReg();
6507	Register Src1 = MI.getOperand(2).getReg();
6508
6509	const CmpInst::Predicate Pred = minMaxToCompare(MI.getOpcode());
6510	LLT CmpType = MRI.getType(Dst).changeElementSize(1);
6511
6512	auto Cmp = MIRBuilder.buildICmp(Pred, CmpType, Src0, Src1);
6513	MIRBuilder.buildSelect(Dst, Cmp, Src0, Src1);
6514
6515	MI.eraseFromParent();
6516	return Legalized;
6517	}
6518
6519	LegalizerHelper::LegalizeResult
6520	LegalizerHelper::lowerFCopySign(MachineInstr &MI) {
6521	Register Dst = MI.getOperand(0).getReg();
6522	Register Src0 = MI.getOperand(1).getReg();
6523	Register Src1 = MI.getOperand(2).getReg();
6524
6525	const LLT Src0Ty = MRI.getType(Src0);
6526	const LLT Src1Ty = MRI.getType(Src1);
6527
6528	const int Src0Size = Src0Ty.getScalarSizeInBits();
6529	const int Src1Size = Src1Ty.getScalarSizeInBits();
6530
6531	auto SignBitMask = MIRBuilder.buildConstant(
6532	Src0Ty, APInt::getSignMask(Src0Size));
6533
6534	auto NotSignBitMask = MIRBuilder.buildConstant(
6535	Src0Ty, APInt::getLowBitsSet(Src0Size, Src0Size - 1));
6536
6537	Register And0 = MIRBuilder.buildAnd(Src0Ty, Src0, NotSignBitMask).getReg(0);
6538	Register And1;
6539	if (Src0Ty == Src1Ty) {
6540	And1 = MIRBuilder.buildAnd(Src1Ty, Src1, SignBitMask).getReg(0);
6541	} else if (Src0Size > Src1Size) {
6542	auto ShiftAmt = MIRBuilder.buildConstant(Src0Ty, Src0Size - Src1Size);
6543	auto Zext = MIRBuilder.buildZExt(Src0Ty, Src1);
6544	auto Shift = MIRBuilder.buildShl(Src0Ty, Zext, ShiftAmt);
6545	And1 = MIRBuilder.buildAnd(Src0Ty, Shift, SignBitMask).getReg(0);
6546	} else {
6547	auto ShiftAmt = MIRBuilder.buildConstant(Src1Ty, Src1Size - Src0Size);
6548	auto Shift = MIRBuilder.buildLShr(Src1Ty, Src1, ShiftAmt);
6549	auto Trunc = MIRBuilder.buildTrunc(Src0Ty, Shift);
6550	And1 = MIRBuilder.buildAnd(Src0Ty, Trunc, SignBitMask).getReg(0);
6551	}
6552
6553	// Be careful about setting nsz/nnan/ninf on every instruction, since the
6554	// constants are a nan and -0.0, but the final result should preserve
6555	// everything.
6556	unsigned Flags = MI.getFlags();
6557	MIRBuilder.buildOr(Dst, And0, And1, Flags);
6558
6559	MI.eraseFromParent();
6560	return Legalized;
6561	}
6562
6563	LegalizerHelper::LegalizeResult
6564	LegalizerHelper::lowerFMinNumMaxNum(MachineInstr &MI) {
6565	unsigned NewOp = MI.getOpcode() == TargetOpcode::G_FMINNUM ?
6566	TargetOpcode::G_FMINNUM_IEEE : TargetOpcode::G_FMAXNUM_IEEE;
6567
6568	Register Dst = MI.getOperand(0).getReg();
6569	Register Src0 = MI.getOperand(1).getReg();
6570	Register Src1 = MI.getOperand(2).getReg();
6571	LLT Ty = MRI.getType(Dst);
6572
6573	if (!MI.getFlag(MachineInstr::FmNoNans)) {
6574	// Insert canonicalizes if it's possible we need to quiet to get correct
6575	// sNaN behavior.
6576
6577	// Note this must be done here, and not as an optimization combine in the
6578	// absence of a dedicate quiet-snan instruction as we're using an
6579	// omni-purpose G_FCANONICALIZE.
6580	if (!isKnownNeverSNaN(Src0, MRI))
6581	Src0 = MIRBuilder.buildFCanonicalize(Ty, Src0, MI.getFlags()).getReg(0);
6582
6583	if (!isKnownNeverSNaN(Src1, MRI))
6584	Src1 = MIRBuilder.buildFCanonicalize(Ty, Src1, MI.getFlags()).getReg(0);
6585	}
6586
6587	// If there are no nans, it's safe to simply replace this with the non-IEEE
6588	// version.
6589	MIRBuilder.buildInstr(NewOp, {Dst}, {Src0, Src1}, MI.getFlags());
6590	MI.eraseFromParent();
6591	return Legalized;
6592	}
6593
6594	LegalizerHelper::LegalizeResult LegalizerHelper::lowerFMad(MachineInstr &MI) {
6595	// Expand G_FMAD a, b, c -> G_FADD (G_FMUL a, b), c
6596	Register DstReg = MI.getOperand(0).getReg();
6597	LLT Ty = MRI.getType(DstReg);
6598	unsigned Flags = MI.getFlags();
6599
6600	auto Mul = MIRBuilder.buildFMul(Ty, MI.getOperand(1), MI.getOperand(2),
6601	Flags);
6602	MIRBuilder.buildFAdd(DstReg, Mul, MI.getOperand(3), Flags);
6603	MI.eraseFromParent();
6604	return Legalized;
6605	}
6606
6607	LegalizerHelper::LegalizeResult
6608	LegalizerHelper::lowerIntrinsicRound(MachineInstr &MI) {
6609	Register DstReg = MI.getOperand(0).getReg();
6610	Register X = MI.getOperand(1).getReg();
6611	const unsigned Flags = MI.getFlags();
6612	const LLT Ty = MRI.getType(DstReg);
6613	const LLT CondTy = Ty.changeElementSize(1);
6614
6615	// round(x) =>
6616	// t = trunc(x);
6617	// d = fabs(x - t);
6618	// o = copysign(1.0f, x);
6619	// return t + (d >= 0.5 ? o : 0.0);
6620
6621	auto T = MIRBuilder.buildIntrinsicTrunc(Ty, X, Flags);
6622
6623	auto Diff = MIRBuilder.buildFSub(Ty, X, T, Flags);
6624	auto AbsDiff = MIRBuilder.buildFAbs(Ty, Diff, Flags);
6625	auto Zero = MIRBuilder.buildFConstant(Ty, 0.0);
6626	auto One = MIRBuilder.buildFConstant(Ty, 1.0);
6627	auto Half = MIRBuilder.buildFConstant(Ty, 0.5);
6628	auto SignOne = MIRBuilder.buildFCopysign(Ty, One, X);
6629
6630	auto Cmp = MIRBuilder.buildFCmp(CmpInst::FCMP_OGE, CondTy, AbsDiff, Half,
6631	Flags);
6632	auto Sel = MIRBuilder.buildSelect(Ty, Cmp, SignOne, Zero, Flags);
6633
6634	MIRBuilder.buildFAdd(DstReg, T, Sel, Flags);
6635
6636	MI.eraseFromParent();
6637	return Legalized;
6638	}
6639
6640	LegalizerHelper::LegalizeResult
6641	LegalizerHelper::lowerFFloor(MachineInstr &MI) {
6642	Register DstReg = MI.getOperand(0).getReg();
6643	Register SrcReg = MI.getOperand(1).getReg();
6644	unsigned Flags = MI.getFlags();
6645	LLT Ty = MRI.getType(DstReg);
6646	const LLT CondTy = Ty.changeElementSize(1);
6647
6648	// result = trunc(src);
6649	// if (src < 0.0 && src != result)
6650	// result += -1.0.
6651
6652	auto Trunc = MIRBuilder.buildIntrinsicTrunc(Ty, SrcReg, Flags);
6653	auto Zero = MIRBuilder.buildFConstant(Ty, 0.0);
6654
6655	auto Lt0 = MIRBuilder.buildFCmp(CmpInst::FCMP_OLT, CondTy,
6656	SrcReg, Zero, Flags);
6657	auto NeTrunc = MIRBuilder.buildFCmp(CmpInst::FCMP_ONE, CondTy,
6658	SrcReg, Trunc, Flags);
6659	auto And = MIRBuilder.buildAnd(CondTy, Lt0, NeTrunc);
6660	auto AddVal = MIRBuilder.buildSITOFP(Ty, And);
6661
6662	MIRBuilder.buildFAdd(DstReg, Trunc, AddVal, Flags);
6663	MI.eraseFromParent();
6664	return Legalized;
6665	}
6666
6667	LegalizerHelper::LegalizeResult
6668	LegalizerHelper::lowerMergeValues(MachineInstr &MI) {
6669	const unsigned NumOps = MI.getNumOperands();
6670	Register DstReg = MI.getOperand(0).getReg();
6671	Register Src0Reg = MI.getOperand(1).getReg();
6672	LLT DstTy = MRI.getType(DstReg);
6673	LLT SrcTy = MRI.getType(Src0Reg);
6674	unsigned PartSize = SrcTy.getSizeInBits();
6675
6676	LLT WideTy = LLT::scalar(DstTy.getSizeInBits());
6677	Register ResultReg = MIRBuilder.buildZExt(WideTy, Src0Reg).getReg(0);
6678
6679	for (unsigned I = 2; I != NumOps; ++I) {
6680	const unsigned Offset = (I - 1) * PartSize;
6681
6682	Register SrcReg = MI.getOperand(I).getReg();
6683	auto ZextInput = MIRBuilder.buildZExt(WideTy, SrcReg);
6684
6685	Register NextResult = I + 1 == NumOps && WideTy == DstTy ? DstReg :
6686	MRI.createGenericVirtualRegister(WideTy);
6687
6688	auto ShiftAmt = MIRBuilder.buildConstant(WideTy, Offset);
6689	auto Shl = MIRBuilder.buildShl(WideTy, ZextInput, ShiftAmt);
6690	MIRBuilder.buildOr(NextResult, ResultReg, Shl);
6691	ResultReg = NextResult;
6692	}
6693
6694	if (DstTy.isPointer()) {
6695	if (MIRBuilder.getDataLayout().isNonIntegralAddressSpace(
6696	DstTy.getAddressSpace())) {
6697	LLVM_DEBUG(dbgs() << "Not casting nonintegral address space\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizer")) { dbgs() << "Not casting nonintegral address space\n" ; } } while (false);
6698	return UnableToLegalize;
6699	}
6700
6701	MIRBuilder.buildIntToPtr(DstReg, ResultReg);
6702	}
6703
6704	MI.eraseFromParent();
6705	return Legalized;
6706	}
6707
6708	LegalizerHelper::LegalizeResult
6709	LegalizerHelper::lowerUnmergeValues(MachineInstr &MI) {
6710	const unsigned NumDst = MI.getNumOperands() - 1;
6711	Register SrcReg = MI.getOperand(NumDst).getReg();
6712	Register Dst0Reg = MI.getOperand(0).getReg();
6713	LLT DstTy = MRI.getType(Dst0Reg);
6714	if (DstTy.isPointer())
6715	return UnableToLegalize; // TODO
6716
6717	SrcReg = coerceToScalar(SrcReg);
6718	if (!SrcReg)
6719	return UnableToLegalize;
6720
6721	// Expand scalarizing unmerge as bitcast to integer and shift.
6722	LLT IntTy = MRI.getType(SrcReg);
6723
6724	MIRBuilder.buildTrunc(Dst0Reg, SrcReg);
6725
6726	const unsigned DstSize = DstTy.getSizeInBits();
6727	unsigned Offset = DstSize;
6728	for (unsigned I = 1; I != NumDst; ++I, Offset += DstSize) {
6729	auto ShiftAmt = MIRBuilder.buildConstant(IntTy, Offset);
6730	auto Shift = MIRBuilder.buildLShr(IntTy, SrcReg, ShiftAmt);
6731	MIRBuilder.buildTrunc(MI.getOperand(I), Shift);
6732	}
6733
6734	MI.eraseFromParent();
6735	return Legalized;
6736	}
6737
6738	/// Lower a vector extract or insert by writing the vector to a stack temporary
6739	/// and reloading the element or vector.
6740	///
6741	/// %dst = G_EXTRACT_VECTOR_ELT %vec, %idx
6742	/// =>
6743	/// %stack_temp = G_FRAME_INDEX
6744	/// G_STORE %vec, %stack_temp
6745	/// %idx = clamp(%idx, %vec.getNumElements())
6746	/// %element_ptr = G_PTR_ADD %stack_temp, %idx
6747	/// %dst = G_LOAD %element_ptr
6748	LegalizerHelper::LegalizeResult
6749	LegalizerHelper::lowerExtractInsertVectorElt(MachineInstr &MI) {
6750	Register DstReg = MI.getOperand(0).getReg();
6751	Register SrcVec = MI.getOperand(1).getReg();
6752	Register InsertVal;
6753	if (MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT)
6754	InsertVal = MI.getOperand(2).getReg();
6755
6756	Register Idx = MI.getOperand(MI.getNumOperands() - 1).getReg();
6757
6758	LLT VecTy = MRI.getType(SrcVec);
6759	LLT EltTy = VecTy.getElementType();
6760	if (!EltTy.isByteSized()) { // Not implemented.
6761	LLVM_DEBUG(dbgs() << "Can't handle non-byte element vectors yet\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizer")) { dbgs() << "Can't handle non-byte element vectors yet\n" ; } } while (false);
6762	return UnableToLegalize;
6763	}
6764
6765	unsigned EltBytes = EltTy.getSizeInBytes();
6766	Align VecAlign = getStackTemporaryAlignment(VecTy);
6767	Align EltAlign;
6768
6769	MachinePointerInfo PtrInfo;
6770	auto StackTemp = createStackTemporary(TypeSize::Fixed(VecTy.getSizeInBytes()),
6771	VecAlign, PtrInfo);
6772	MIRBuilder.buildStore(SrcVec, StackTemp, PtrInfo, VecAlign);
6773
6774	// Get the pointer to the element, and be sure not to hit undefined behavior
6775	// if the index is out of bounds.
6776	Register EltPtr = getVectorElementPointer(StackTemp.getReg(0), VecTy, Idx);
6777
6778	int64_t IdxVal;
6779	if (mi_match(Idx, MRI, m_ICst(IdxVal))) {
6780	int64_t Offset = IdxVal * EltBytes;
6781	PtrInfo = PtrInfo.getWithOffset(Offset);
6782	EltAlign = commonAlignment(VecAlign, Offset);
6783	} else {
6784	// We lose information with a variable offset.
6785	EltAlign = getStackTemporaryAlignment(EltTy);
6786	PtrInfo = MachinePointerInfo(MRI.getType(EltPtr).getAddressSpace());
6787	}
6788
6789	if (InsertVal) {
6790	// Write the inserted element
6791	MIRBuilder.buildStore(InsertVal, EltPtr, PtrInfo, EltAlign);
6792
6793	// Reload the whole vector.
6794	MIRBuilder.buildLoad(DstReg, StackTemp, PtrInfo, VecAlign);
6795	} else {
6796	MIRBuilder.buildLoad(DstReg, EltPtr, PtrInfo, EltAlign);
6797	}
6798
6799	MI.eraseFromParent();
6800	return Legalized;
6801	}
6802
6803	LegalizerHelper::LegalizeResult
6804	LegalizerHelper::lowerShuffleVector(MachineInstr &MI) {
6805	Register DstReg = MI.getOperand(0).getReg();
6806	Register Src0Reg = MI.getOperand(1).getReg();
6807	Register Src1Reg = MI.getOperand(2).getReg();
6808	LLT Src0Ty = MRI.getType(Src0Reg);
6809	LLT DstTy = MRI.getType(DstReg);
6810	LLT IdxTy = LLT::scalar(32);
6811
6812	ArrayRef<int> Mask = MI.getOperand(3).getShuffleMask();
6813
6814	if (DstTy.isScalar()) {
6815	if (Src0Ty.isVector())
6816	return UnableToLegalize;
6817
6818	// This is just a SELECT.
6819	assert(Mask.size() == 1 && "Expected a single mask element")(static_cast <bool> (Mask.size() == 1 && "Expected a single mask element" ) ? void (0) : __assert_fail ("Mask.size() == 1 && \"Expected a single mask element\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 6819, __extension__ __PRETTY_FUNCTION__));
6820	Register Val;
6821	if (Mask[0] < 0 \|\| Mask[0] > 1)
6822	Val = MIRBuilder.buildUndef(DstTy).getReg(0);
6823	else
6824	Val = Mask[0] == 0 ? Src0Reg : Src1Reg;
6825	MIRBuilder.buildCopy(DstReg, Val);
6826	MI.eraseFromParent();
6827	return Legalized;
6828	}
6829
6830	Register Undef;
6831	SmallVector<Register, 32> BuildVec;
6832	LLT EltTy = DstTy.getElementType();
6833
6834	for (int Idx : Mask) {
6835	if (Idx < 0) {
6836	if (!Undef.isValid())
6837	Undef = MIRBuilder.buildUndef(EltTy).getReg(0);
6838	BuildVec.push_back(Undef);
6839	continue;
6840	}
6841
6842	if (Src0Ty.isScalar()) {
6843	BuildVec.push_back(Idx == 0 ? Src0Reg : Src1Reg);
6844	} else {
6845	int NumElts = Src0Ty.getNumElements();
6846	Register SrcVec = Idx < NumElts ? Src0Reg : Src1Reg;
6847	int ExtractIdx = Idx < NumElts ? Idx : Idx - NumElts;
6848	auto IdxK = MIRBuilder.buildConstant(IdxTy, ExtractIdx);
6849	auto Extract = MIRBuilder.buildExtractVectorElement(EltTy, SrcVec, IdxK);
6850	BuildVec.push_back(Extract.getReg(0));
6851	}
6852	}
6853
6854	MIRBuilder.buildBuildVector(DstReg, BuildVec);
6855	MI.eraseFromParent();
6856	return Legalized;
6857	}
6858
6859	LegalizerHelper::LegalizeResult
6860	LegalizerHelper::lowerDynStackAlloc(MachineInstr &MI) {
6861	const auto &MF = *MI.getMF();
6862	const auto &TFI = *MF.getSubtarget().getFrameLowering();
6863	if (TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp)
6864	return UnableToLegalize;
6865
6866	Register Dst = MI.getOperand(0).getReg();
6867	Register AllocSize = MI.getOperand(1).getReg();
6868	Align Alignment = assumeAligned(MI.getOperand(2).getImm());
6869
6870	LLT PtrTy = MRI.getType(Dst);
6871	LLT IntPtrTy = LLT::scalar(PtrTy.getSizeInBits());
6872
6873	Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
6874	auto SPTmp = MIRBuilder.buildCopy(PtrTy, SPReg);
6875	SPTmp = MIRBuilder.buildCast(IntPtrTy, SPTmp);
6876
6877	// Subtract the final alloc from the SP. We use G_PTRTOINT here so we don't
6878	// have to generate an extra instruction to negate the alloc and then use
6879	// G_PTR_ADD to add the negative offset.
6880	auto Alloc = MIRBuilder.buildSub(IntPtrTy, SPTmp, AllocSize);
6881	if (Alignment > Align(1)) {
6882	APInt AlignMask(IntPtrTy.getSizeInBits(), Alignment.value(), true);
6883	AlignMask.negate();
6884	auto AlignCst = MIRBuilder.buildConstant(IntPtrTy, AlignMask);
6885	Alloc = MIRBuilder.buildAnd(IntPtrTy, Alloc, AlignCst);
6886	}
6887
6888	SPTmp = MIRBuilder.buildCast(PtrTy, Alloc);
6889	MIRBuilder.buildCopy(SPReg, SPTmp);
6890	MIRBuilder.buildCopy(Dst, SPTmp);
6891
6892	MI.eraseFromParent();
6893	return Legalized;
6894	}
6895
6896	LegalizerHelper::LegalizeResult
6897	LegalizerHelper::lowerExtract(MachineInstr &MI) {
6898	Register Dst = MI.getOperand(0).getReg();
6899	Register Src = MI.getOperand(1).getReg();
6900	unsigned Offset = MI.getOperand(2).getImm();
6901
6902	LLT DstTy = MRI.getType(Dst);
6903	LLT SrcTy = MRI.getType(Src);
6904
6905	if (DstTy.isScalar() &&
6906	(SrcTy.isScalar() \|\|
6907	(SrcTy.isVector() && DstTy == SrcTy.getElementType()))) {
6908	LLT SrcIntTy = SrcTy;
6909	if (!SrcTy.isScalar()) {
6910	SrcIntTy = LLT::scalar(SrcTy.getSizeInBits());
6911	Src = MIRBuilder.buildBitcast(SrcIntTy, Src).getReg(0);
6912	}
6913
6914	if (Offset == 0)
6915	MIRBuilder.buildTrunc(Dst, Src);
6916	else {
6917	auto ShiftAmt = MIRBuilder.buildConstant(SrcIntTy, Offset);
6918	auto Shr = MIRBuilder.buildLShr(SrcIntTy, Src, ShiftAmt);
6919	MIRBuilder.buildTrunc(Dst, Shr);
6920	}
6921
6922	MI.eraseFromParent();
6923	return Legalized;
6924	}
6925
6926	return UnableToLegalize;
6927	}
6928
6929	LegalizerHelper::LegalizeResult LegalizerHelper::lowerInsert(MachineInstr &MI) {
6930	Register Dst = MI.getOperand(0).getReg();
6931	Register Src = MI.getOperand(1).getReg();
6932	Register InsertSrc = MI.getOperand(2).getReg();
6933	uint64_t Offset = MI.getOperand(3).getImm();
6934
6935	LLT DstTy = MRI.getType(Src);
6936	LLT InsertTy = MRI.getType(InsertSrc);
6937
6938	if (InsertTy.isVector() \|\|
6939	(DstTy.isVector() && DstTy.getElementType() != InsertTy))
6940	return UnableToLegalize;
6941
6942	const DataLayout &DL = MIRBuilder.getDataLayout();
6943	if ((DstTy.isPointer() &&
6944	DL.isNonIntegralAddressSpace(DstTy.getAddressSpace())) \|\|
6945	(InsertTy.isPointer() &&
6946	DL.isNonIntegralAddressSpace(InsertTy.getAddressSpace()))) {
6947	LLVM_DEBUG(dbgs() << "Not casting non-integral address space integer\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizer")) { dbgs() << "Not casting non-integral address space integer\n" ; } } while (false);
6948	return UnableToLegalize;
6949	}
6950
6951	LLT IntDstTy = DstTy;
6952
6953	if (!DstTy.isScalar()) {
6954	IntDstTy = LLT::scalar(DstTy.getSizeInBits());
6955	Src = MIRBuilder.buildCast(IntDstTy, Src).getReg(0);
6956	}
6957
6958	if (!InsertTy.isScalar()) {
6959	const LLT IntInsertTy = LLT::scalar(InsertTy.getSizeInBits());
6960	InsertSrc = MIRBuilder.buildPtrToInt(IntInsertTy, InsertSrc).getReg(0);
6961	}
6962
6963	Register ExtInsSrc = MIRBuilder.buildZExt(IntDstTy, InsertSrc).getReg(0);
6964	if (Offset != 0) {
6965	auto ShiftAmt = MIRBuilder.buildConstant(IntDstTy, Offset);
6966	ExtInsSrc = MIRBuilder.buildShl(IntDstTy, ExtInsSrc, ShiftAmt).getReg(0);
6967	}
6968
6969	APInt MaskVal = APInt::getBitsSetWithWrap(
6970	DstTy.getSizeInBits(), Offset + InsertTy.getSizeInBits(), Offset);
6971
6972	auto Mask = MIRBuilder.buildConstant(IntDstTy, MaskVal);
6973	auto MaskedSrc = MIRBuilder.buildAnd(IntDstTy, Src, Mask);
6974	auto Or = MIRBuilder.buildOr(IntDstTy, MaskedSrc, ExtInsSrc);
6975
6976	MIRBuilder.buildCast(Dst, Or);
6977	MI.eraseFromParent();
6978	return Legalized;
6979	}
6980
6981	LegalizerHelper::LegalizeResult
6982	LegalizerHelper::lowerSADDO_SSUBO(MachineInstr &MI) {
6983	Register Dst0 = MI.getOperand(0).getReg();
6984	Register Dst1 = MI.getOperand(1).getReg();
6985	Register LHS = MI.getOperand(2).getReg();
6986	Register RHS = MI.getOperand(3).getReg();
6987	const bool IsAdd = MI.getOpcode() == TargetOpcode::G_SADDO;
6988
6989	LLT Ty = MRI.getType(Dst0);
6990	LLT BoolTy = MRI.getType(Dst1);
6991
6992	if (IsAdd)
6993	MIRBuilder.buildAdd(Dst0, LHS, RHS);
6994	else
6995	MIRBuilder.buildSub(Dst0, LHS, RHS);
6996
6997	// TODO: If SADDSAT/SSUBSAT is legal, compare results to detect overflow.
6998
6999	auto Zero = MIRBuilder.buildConstant(Ty, 0);
7000
7001	// For an addition, the result should be less than one of the operands (LHS)
7002	// if and only if the other operand (RHS) is negative, otherwise there will
7003	// be overflow.
7004	// For a subtraction, the result should be less than one of the operands
7005	// (LHS) if and only if the other operand (RHS) is (non-zero) positive,
7006	// otherwise there will be overflow.
7007	auto ResultLowerThanLHS =
7008	MIRBuilder.buildICmp(CmpInst::ICMP_SLT, BoolTy, Dst0, LHS);
7009	auto ConditionRHS = MIRBuilder.buildICmp(
7010	IsAdd ? CmpInst::ICMP_SLT : CmpInst::ICMP_SGT, BoolTy, RHS, Zero);
7011
7012	MIRBuilder.buildXor(Dst1, ConditionRHS, ResultLowerThanLHS);
7013	MI.eraseFromParent();
7014	return Legalized;
7015	}
7016
7017	LegalizerHelper::LegalizeResult
7018	LegalizerHelper::lowerAddSubSatToMinMax(MachineInstr &MI) {
7019	Register Res = MI.getOperand(0).getReg();
7020	Register LHS = MI.getOperand(1).getReg();
7021	Register RHS = MI.getOperand(2).getReg();
7022	LLT Ty = MRI.getType(Res);
7023	bool IsSigned;
7024	bool IsAdd;
7025	unsigned BaseOp;
7026	switch (MI.getOpcode()) {
7027	default:
7028	llvm_unreachable("unexpected addsat/subsat opcode")::llvm::llvm_unreachable_internal("unexpected addsat/subsat opcode" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 7028);
7029	case TargetOpcode::G_UADDSAT:
7030	IsSigned = false;
7031	IsAdd = true;
7032	BaseOp = TargetOpcode::G_ADD;
7033	break;
7034	case TargetOpcode::G_SADDSAT:
7035	IsSigned = true;
7036	IsAdd = true;
7037	BaseOp = TargetOpcode::G_ADD;
7038	break;
7039	case TargetOpcode::G_USUBSAT:
7040	IsSigned = false;
7041	IsAdd = false;
7042	BaseOp = TargetOpcode::G_SUB;
7043	break;
7044	case TargetOpcode::G_SSUBSAT:
7045	IsSigned = true;
7046	IsAdd = false;
7047	BaseOp = TargetOpcode::G_SUB;
7048	break;
7049	}
7050
7051	if (IsSigned) {
7052	// sadd.sat(a, b) ->
7053	// hi = 0x7fffffff - smax(a, 0)
7054	// lo = 0x80000000 - smin(a, 0)
7055	// a + smin(smax(lo, b), hi)
7056	// ssub.sat(a, b) ->
7057	// lo = smax(a, -1) - 0x7fffffff
7058	// hi = smin(a, -1) - 0x80000000
7059	// a - smin(smax(lo, b), hi)
7060	// TODO: AMDGPU can use a "median of 3" instruction here:
7061	// a +/- med3(lo, b, hi)
7062	uint64_t NumBits = Ty.getScalarSizeInBits();
7063	auto MaxVal =
7064	MIRBuilder.buildConstant(Ty, APInt::getSignedMaxValue(NumBits));
7065	auto MinVal =
7066	MIRBuilder.buildConstant(Ty, APInt::getSignedMinValue(NumBits));
7067	MachineInstrBuilder Hi, Lo;
7068	if (IsAdd) {
7069	auto Zero = MIRBuilder.buildConstant(Ty, 0);
7070	Hi = MIRBuilder.buildSub(Ty, MaxVal, MIRBuilder.buildSMax(Ty, LHS, Zero));
7071	Lo = MIRBuilder.buildSub(Ty, MinVal, MIRBuilder.buildSMin(Ty, LHS, Zero));
7072	} else {
7073	auto NegOne = MIRBuilder.buildConstant(Ty, -1);
7074	Lo = MIRBuilder.buildSub(Ty, MIRBuilder.buildSMax(Ty, LHS, NegOne),
7075	MaxVal);
7076	Hi = MIRBuilder.buildSub(Ty, MIRBuilder.buildSMin(Ty, LHS, NegOne),
7077	MinVal);
7078	}
7079	auto RHSClamped =
7080	MIRBuilder.buildSMin(Ty, MIRBuilder.buildSMax(Ty, Lo, RHS), Hi);
7081	MIRBuilder.buildInstr(BaseOp, {Res}, {LHS, RHSClamped});
7082	} else {
7083	// uadd.sat(a, b) -> a + umin(~a, b)
7084	// usub.sat(a, b) -> a - umin(a, b)
7085	Register Not = IsAdd ? MIRBuilder.buildNot(Ty, LHS).getReg(0) : LHS;
7086	auto Min = MIRBuilder.buildUMin(Ty, Not, RHS);
7087	MIRBuilder.buildInstr(BaseOp, {Res}, {LHS, Min});
7088	}
7089
7090	MI.eraseFromParent();
7091	return Legalized;
7092	}
7093
7094	LegalizerHelper::LegalizeResult
7095	LegalizerHelper::lowerAddSubSatToAddoSubo(MachineInstr &MI) {
7096	Register Res = MI.getOperand(0).getReg();
7097	Register LHS = MI.getOperand(1).getReg();
7098	Register RHS = MI.getOperand(2).getReg();
7099	LLT Ty = MRI.getType(Res);
7100	LLT BoolTy = Ty.changeElementSize(1);
7101	bool IsSigned;
7102	bool IsAdd;
7103	unsigned OverflowOp;
7104	switch (MI.getOpcode()) {
7105	default:
7106	llvm_unreachable("unexpected addsat/subsat opcode")::llvm::llvm_unreachable_internal("unexpected addsat/subsat opcode" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 7106);
7107	case TargetOpcode::G_UADDSAT:
7108	IsSigned = false;
7109	IsAdd = true;
7110	OverflowOp = TargetOpcode::G_UADDO;
7111	break;
7112	case TargetOpcode::G_SADDSAT:
7113	IsSigned = true;
7114	IsAdd = true;
7115	OverflowOp = TargetOpcode::G_SADDO;
7116	break;
7117	case TargetOpcode::G_USUBSAT:
7118	IsSigned = false;
7119	IsAdd = false;
7120	OverflowOp = TargetOpcode::G_USUBO;
7121	break;
7122	case TargetOpcode::G_SSUBSAT:
7123	IsSigned = true;
7124	IsAdd = false;
7125	OverflowOp = TargetOpcode::G_SSUBO;
7126	break;
7127	}
7128
7129	auto OverflowRes =
7130	MIRBuilder.buildInstr(OverflowOp, {Ty, BoolTy}, {LHS, RHS});
7131	Register Tmp = OverflowRes.getReg(0);
7132	Register Ov = OverflowRes.getReg(1);
7133	MachineInstrBuilder Clamp;
7134	if (IsSigned) {
7135	// sadd.sat(a, b) ->
7136	// {tmp, ov} = saddo(a, b)
7137	// ov ? (tmp >>s 31) + 0x80000000 : r
7138	// ssub.sat(a, b) ->
7139	// {tmp, ov} = ssubo(a, b)
7140	// ov ? (tmp >>s 31) + 0x80000000 : r
7141	uint64_t NumBits = Ty.getScalarSizeInBits();
7142	auto ShiftAmount = MIRBuilder.buildConstant(Ty, NumBits - 1);
7143	auto Sign = MIRBuilder.buildAShr(Ty, Tmp, ShiftAmount);
7144	auto MinVal =
7145	MIRBuilder.buildConstant(Ty, APInt::getSignedMinValue(NumBits));
7146	Clamp = MIRBuilder.buildAdd(Ty, Sign, MinVal);
7147	} else {
7148	// uadd.sat(a, b) ->
7149	// {tmp, ov} = uaddo(a, b)
7150	// ov ? 0xffffffff : tmp
7151	// usub.sat(a, b) ->
7152	// {tmp, ov} = usubo(a, b)
7153	// ov ? 0 : tmp
7154	Clamp = MIRBuilder.buildConstant(Ty, IsAdd ? -1 : 0);
7155	}
7156	MIRBuilder.buildSelect(Res, Ov, Clamp, Tmp);
7157
7158	MI.eraseFromParent();
7159	return Legalized;
7160	}
7161
7162	LegalizerHelper::LegalizeResult
7163	LegalizerHelper::lowerShlSat(MachineInstr &MI) {
7164	assert((MI.getOpcode() == TargetOpcode::G_SSHLSAT \|\|(static_cast <bool> ((MI.getOpcode() == TargetOpcode::G_SSHLSAT \|\| MI.getOpcode() == TargetOpcode::G_USHLSAT) && "Expected shlsat opcode!" ) ? void (0) : __assert_fail ("(MI.getOpcode() == TargetOpcode::G_SSHLSAT \|\| MI.getOpcode() == TargetOpcode::G_USHLSAT) && \"Expected shlsat opcode!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 7166, __extension__ __PRETTY_FUNCTION__))
7165	MI.getOpcode() == TargetOpcode::G_USHLSAT) &&(static_cast <bool> ((MI.getOpcode() == TargetOpcode::G_SSHLSAT \|\| MI.getOpcode() == TargetOpcode::G_USHLSAT) && "Expected shlsat opcode!" ) ? void (0) : __assert_fail ("(MI.getOpcode() == TargetOpcode::G_SSHLSAT \|\| MI.getOpcode() == TargetOpcode::G_USHLSAT) && \"Expected shlsat opcode!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 7166, __extension__ __PRETTY_FUNCTION__))
7166	"Expected shlsat opcode!")(static_cast <bool> ((MI.getOpcode() == TargetOpcode::G_SSHLSAT \|\| MI.getOpcode() == TargetOpcode::G_USHLSAT) && "Expected shlsat opcode!" ) ? void (0) : __assert_fail ("(MI.getOpcode() == TargetOpcode::G_SSHLSAT \|\| MI.getOpcode() == TargetOpcode::G_USHLSAT) && \"Expected shlsat opcode!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp" , 7166, __extension__ __PRETTY_FUNCTION__));
7167	bool IsSigned = MI.getOpcode() == TargetOpcode::G_SSHLSAT;
7168	Register Res = MI.getOperand(0).getReg();
7169	Register LHS = MI.getOperand(1).getReg();
7170	Register RHS = MI.getOperand(2).getReg();
7171	LLT Ty = MRI.getType(Res);
7172	LLT BoolTy = Ty.changeElementSize(1);
7173
7174	unsigned BW = Ty.getScalarSizeInBits();
7175	auto Result = MIRBuilder.buildShl(Ty, LHS, RHS);
7176	auto Orig = IsSigned ? MIRBuilder.buildAShr(Ty, Result, RHS)
7177	: MIRBuilder.buildLShr(Ty, Result, RHS);
7178
7179	MachineInstrBuilder SatVal;
7180	if (IsSigned) {
7181	auto SatMin = MIRBuilder.buildConstant(Ty, APInt::getSignedMinValue(BW));
7182	auto SatMax = MIRBuilder.buildConstant(Ty, APInt::getSignedMaxValue(BW));
7183	auto Cmp = MIRBuilder.buildICmp(CmpInst::ICMP_SLT, BoolTy, LHS,
7184	MIRBuilder.buildConstant(Ty, 0));
7185	SatVal = MIRBuilder.buildSelect(Ty, Cmp, SatMin, SatMax);
7186	} else {
7187	SatVal = MIRBuilder.buildConstant(Ty, APInt::getMaxValue(BW));
7188	}
7189	auto Ov = MIRBuilder.buildICmp(CmpInst::ICMP_NE, BoolTy, LHS, Orig);
7190	MIRBuilder.buildSelect(Res, Ov, SatVal, Result);
7191
7192	MI.eraseFromParent();
7193	return Legalized;
7194	}
7195
7196	LegalizerHelper::LegalizeResult
7197	LegalizerHelper::lowerBswap(MachineInstr &MI) {
7198	Register Dst = MI.getOperand(0).getReg();
7199	Register Src = MI.getOperand(1).getReg();
7200	const LLT Ty = MRI.getType(Src);
7201	unsigned SizeInBytes = (Ty.getScalarSizeInBits() + 7) / 8;
7202	unsigned BaseShiftAmt = (SizeInBytes - 1) * 8;
7203
7204	// Swap most and least significant byte, set remaining bytes in Res to zero.
7205	auto ShiftAmt = MIRBuilder.buildConstant(Ty, BaseShiftAmt);
7206	auto LSByteShiftedLeft = MIRBuilder.buildShl(Ty, Src, ShiftAmt);
7207	auto MSByteShiftedRight = MIRBuilder.buildLShr(Ty, Src, ShiftAmt);
7208	auto Res = MIRBuilder.buildOr(Ty, MSByteShiftedRight, LSByteShiftedLeft);
7209
7210	// Set i-th high/low byte in Res to i-th low/high byte from Src.
7211	for (unsigned i = 1; i < SizeInBytes / 2; ++i) {
7212	// AND with Mask leaves byte i unchanged and sets remaining bytes to 0.
7213	APInt APMask(SizeInBytes * 8, 0xFF << (i * 8));
7214	auto Mask = MIRBuilder.buildConstant(Ty, APMask);
7215	auto ShiftAmt = MIRBuilder.buildConstant(Ty, BaseShiftAmt - 16 * i);
7216	// Low byte shifted left to place of high byte: (Src & Mask) << ShiftAmt.
7217	auto LoByte = MIRBuilder.buildAnd(Ty, Src, Mask);
7218	auto LoShiftedLeft = MIRBuilder.buildShl(Ty, LoByte, ShiftAmt);
7219	Res = MIRBuilder.buildOr(Ty, Res, LoShiftedLeft);
7220	// High byte shifted right to place of low byte: (Src >> ShiftAmt) & Mask.
7221	auto SrcShiftedRight = MIRBuilder.buildLShr(Ty, Src, ShiftAmt);
7222	auto HiShiftedRight = MIRBuilder.buildAnd(Ty, SrcShiftedRight, Mask);
7223	Res = MIRBuilder.buildOr(Ty, Res, HiShiftedRight);
7224	}
7225	Res.getInstr()->getOperand(0).setReg(Dst);
7226
7227	MI.eraseFromParent();
7228	return Legalized;
7229	}
7230
7231	//{ (Src & Mask) >> N } \| { (Src << N) & Mask }
7232	static MachineInstrBuilder SwapN(unsigned N, DstOp Dst, MachineIRBuilder &B,
7233	MachineInstrBuilder Src, APInt Mask) {
7234	const LLT Ty = Dst.getLLTTy(*B.getMRI());
7235	MachineInstrBuilder C_N = B.buildConstant(Ty, N);
7236	MachineInstrBuilder MaskLoNTo0 = B.buildConstant(Ty, Mask);
7237	auto LHS = B.buildLShr(Ty, B.buildAnd(Ty, Src, MaskLoNTo0), C_N);
7238	auto RHS = B.buildAnd(Ty, B.buildShl(Ty, Src, C_N), MaskLoNTo0);
7239	return B.buildOr(Dst, LHS, RHS);
7240	}
7241
7242	LegalizerHelper::LegalizeResult
7243	LegalizerHelper::lowerBitreverse(MachineInstr &MI) {
7244	Register Dst = MI.getOperand(0).getReg();
7245	Register Src = MI.getOperand(1).getReg();
7246	const LLT Ty = MRI.getType(Src);
7247	unsigned Size = Ty.getSizeInBits();
7248
7249	MachineInstrBuilder BSWAP =
7250	MIRBuilder.buildInstr(TargetOpcode::G_BSWAP, {Ty}, {Src});
7251
7252	// swap high and low 4 bits in 8 bit blocks 7654\|3210 -> 3210\|7654
7253	// [(val & 0xF0F0F0F0) >> 4] \| [(val & 0x0F0F0F0F) << 4]
7254	// -> [(val & 0xF0F0F0F0) >> 4] \| [(val << 4) & 0xF0F0F0F0]
7255	MachineInstrBuilder Swap4 =
7256	SwapN(4, Ty, MIRBuilder, BSWAP, APInt::getSplat(Size, APInt(8, 0xF0)));
7257
7258	// swap high and low 2 bits in 4 bit blocks 32\|10 76\|54 -> 10\|32 54\|76
7259	// [(val & 0xCCCCCCCC) >> 2] & [(val & 0x33333333) << 2]
7260	// -> [(val & 0xCCCCCCCC) >> 2] & [(val << 2) & 0xCCCCCCCC]
7261	MachineInstrBuilder Swap2 =
7262	SwapN(2, Ty, MIRBuilder, Swap4, APInt::getSplat(Size, APInt(8, 0xCC)));
7263
7264	// swap high and low 1 bit in 2 bit blocks 1\|0 3\|2 5\|4 7\|6 -> 0\|1 2\|3 4\|5 6\|7
7265	// [(val & 0xAAAAAAAA) >> 1] & [(val & 0x55555555) << 1]
7266	// -> [(val & 0xAAAAAAAA) >> 1] & [(val << 1) & 0xAAAAAAAA]
7267	SwapN(1, Dst, MIRBuilder, Swap2, APInt::getSplat(Size, APInt(8, 0xAA)));
7268
7269	MI.eraseFromParent();
7270	return Legalized;
7271	}
7272
7273	LegalizerHelper::LegalizeResult
7274	LegalizerHelper::lowerReadWriteRegister(MachineInstr &MI) {
7275	MachineFunction &MF = MIRBuilder.getMF();
7276
7277	bool IsRead = MI.getOpcode() == TargetOpcode::G_READ_REGISTER;
7278	int NameOpIdx = IsRead ? 1 : 0;
7279	int ValRegIndex = IsRead ? 0 : 1;
7280
7281	Register ValReg = MI.getOperand(ValRegIndex).getReg();
7282	const LLT Ty = MRI.getType(ValReg);
7283	const MDString *RegStr = cast<MDString>(
7284	cast<MDNode>(MI.getOperand(NameOpIdx).getMetadata())->getOperand(0));
7285
7286	Register PhysReg = TLI.getRegisterByName(RegStr->getString().data(), Ty, MF);
7287	if (!PhysReg.isValid())
7288	return UnableToLegalize;
7289
7290	if (IsRead)
7291	MIRBuilder.buildCopy(ValReg, PhysReg);
7292	else
7293	MIRBuilder.buildCopy(PhysReg, ValReg);
7294
7295	MI.eraseFromParent();
7296	return Legalized;
7297	}
7298
7299	LegalizerHelper::LegalizeResult
7300	LegalizerHelper::lowerSMULH_UMULH(MachineInstr &MI) {
7301	bool IsSigned = MI.getOpcode() == TargetOpcode::G_SMULH;
7302	unsigned ExtOp = IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
7303	Register Result = MI.getOperand(0).getReg();
7304	LLT OrigTy = MRI.getType(Result);
7305	auto SizeInBits = OrigTy.getScalarSizeInBits();
7306	LLT WideTy = OrigTy.changeElementSize(SizeInBits * 2);
7307
7308	auto LHS = MIRBuilder.buildInstr(ExtOp, {WideTy}, {MI.getOperand(1)});
7309	auto RHS = MIRBuilder.buildInstr(ExtOp, {WideTy}, {MI.getOperand(2)});
7310	auto Mul = MIRBuilder.buildMul(WideTy, LHS, RHS);
7311	unsigned ShiftOp = IsSigned ? TargetOpcode::G_ASHR : TargetOpcode::G_LSHR;
7312
7313	auto ShiftAmt = MIRBuilder.buildConstant(WideTy, SizeInBits);
7314	auto Shifted = MIRBuilder.buildInstr(ShiftOp, {WideTy}, {Mul, ShiftAmt});
7315	MIRBuilder.buildTrunc(Result, Shifted);
7316
7317	MI.eraseFromParent();
7318	return Legalized;
7319	}
7320
7321	LegalizerHelper::LegalizeResult LegalizerHelper::lowerSelect(MachineInstr &MI) {
7322	// Implement vector G_SELECT in terms of XOR, AND, OR.
7323	Register DstReg = MI.getOperand(0).getReg();
7324	Register MaskReg = MI.getOperand(1).getReg();
7325	Register Op1Reg = MI.getOperand(2).getReg();
7326	Register Op2Reg = MI.getOperand(3).getReg();
7327	LLT DstTy = MRI.getType(DstReg);
7328	LLT MaskTy = MRI.getType(MaskReg);
7329	LLT Op1Ty = MRI.getType(Op1Reg);
7330	if (!DstTy.isVector())
7331	return UnableToLegalize;
7332
7333	// Vector selects can have a scalar predicate. If so, splat into a vector and
7334	// finish for later legalization attempts to try again.
7335	if (MaskTy.isScalar()) {
7336	Register MaskElt = MaskReg;
7337	if (MaskTy.getSizeInBits() < DstTy.getScalarSizeInBits())
7338	MaskElt = MIRBuilder.buildSExt(DstTy.getElementType(), MaskElt).getReg(0);
7339	// Generate a vector splat idiom to be pattern matched later.
7340	auto ShufSplat = MIRBuilder.buildShuffleSplat(DstTy, MaskElt);
7341	Observer.changingInstr(MI);
7342	MI.getOperand(1).setReg(ShufSplat.getReg(0));
7343	Observer.changedInstr(MI);
7344	return Legalized;
7345	}
7346
7347	if (MaskTy.getSizeInBits() != Op1Ty.getSizeInBits()) {
7348	return UnableToLegalize;
7349	}
7350
7351	auto NotMask = MIRBuilder.buildNot(MaskTy, MaskReg);
7352	auto NewOp1 = MIRBuilder.buildAnd(MaskTy, Op1Reg, MaskReg);
7353	auto NewOp2 = MIRBuilder.buildAnd(MaskTy, Op2Reg, NotMask);
7354	MIRBuilder.buildOr(DstReg, NewOp1, NewOp2);
7355	MI.eraseFromParent();
7356	return Legalized;
7357	}
7358
7359	LegalizerHelper::LegalizeResult LegalizerHelper::lowerDIVREM(MachineInstr &MI) {
7360	// Split DIVREM into individual instructions.
7361	unsigned Opcode = MI.getOpcode();
7362
7363	MIRBuilder.buildInstr(
7364	Opcode == TargetOpcode::G_SDIVREM ? TargetOpcode::G_SDIV
7365	: TargetOpcode::G_UDIV,
7366	{MI.getOperand(0).getReg()}, {MI.getOperand(2), MI.getOperand(3)});
7367	MIRBuilder.buildInstr(
7368	Opcode == TargetOpcode::G_SDIVREM ? TargetOpcode::G_SREM
7369	: TargetOpcode::G_UREM,
7370	{MI.getOperand(1).getReg()}, {MI.getOperand(2), MI.getOperand(3)});
7371	MI.eraseFromParent();
7372	return Legalized;
7373	}
7374
7375	LegalizerHelper::LegalizeResult
7376	LegalizerHelper::lowerAbsToAddXor(MachineInstr &MI) {
7377	// Expand %res = G_ABS %a into:
7378	// %v1 = G_ASHR %a, scalar_size-1
7379	// %v2 = G_ADD %a, %v1
7380	// %res = G_XOR %v2, %v1
7381	LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
7382	Register OpReg = MI.getOperand(1).getReg();
7383	auto ShiftAmt =
7384	MIRBuilder.buildConstant(DstTy, DstTy.getScalarSizeInBits() - 1);
7385	auto Shift = MIRBuilder.buildAShr(DstTy, OpReg, ShiftAmt);
7386	auto Add = MIRBuilder.buildAdd(DstTy, OpReg, Shift);
7387	MIRBuilder.buildXor(MI.getOperand(0).getReg(), Add, Shift);
7388	MI.eraseFromParent();
7389	return Legalized;
7390	}
7391
7392	LegalizerHelper::LegalizeResult
7393	LegalizerHelper::lowerAbsToMaxNeg(MachineInstr &MI) {
7394	// Expand %res = G_ABS %a into:
7395	// %v1 = G_CONSTANT 0
7396	// %v2 = G_SUB %v1, %a
7397	// %res = G_SMAX %a, %v2
7398	Register SrcReg = MI.getOperand(1).getReg();
7399	LLT Ty = MRI.getType(SrcReg);
7400	auto Zero = MIRBuilder.buildConstant(Ty, 0).getReg(0);
7401	auto Sub = MIRBuilder.buildSub(Ty, Zero, SrcReg).getReg(0);
7402	MIRBuilder.buildSMax(MI.getOperand(0), SrcReg, Sub);
7403	MI.eraseFromParent();
7404	return Legalized;
7405	}
7406
7407	LegalizerHelper::LegalizeResult LegalizerHelper::lowerIsNaN(MachineInstr &MI) {
7408	Register Dst = MI.getOperand(0).getReg();
7409	Register Src = MI.getOperand(1).getReg();
7410	LLT SrcTy = MRI.getType(Src);
7411	if (MI.getFlags() & MachineInstr::NoFPExcept) {
7412	// Lower to an unordered comparison.
7413	auto Zero = MIRBuilder.buildFConstant(SrcTy, 0.0);
7414	MIRBuilder.buildFCmp(CmpInst::Predicate::FCMP_UNO, Dst, Src, Zero);
7415	MI.eraseFromParent();
7416	return Legalized;
7417	}
7418
7419	// Use integer operations to avoid traps if the argument is SNaN.
7420
7421	// NaN has all exp bits set and a non zero significand. Therefore:
7422	// isnan(V) == exp mask < abs(V)
7423	auto FPToSI = MIRBuilder.buildFPTOSI(SrcTy, Src);
7424	auto Mask = APInt::getSignedMaxValue(SrcTy.getScalarSizeInBits());
7425	auto MaskCst = MIRBuilder.buildConstant(SrcTy, Mask);
7426	auto AbsV = MIRBuilder.buildAnd(SrcTy, FPToSI, MaskCst);
7427	auto *FloatTy = getFloatTypeForLLT(MI.getMF()->getFunction().getContext(),
7428	SrcTy.getScalarType());
7429	if (!FloatTy)
7430	return UnableToLegalize;
7431	auto ExpMask = APFloat::getInf(FloatTy->getFltSemantics()).bitcastToAPInt();
7432	auto ExpMaskCst = MIRBuilder.buildConstant(SrcTy, ExpMask);
7433	MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_SLT, Dst, ExpMaskCst, AbsV);
7434	MI.eraseFromParent();
7435	return Legalized;
7436	}
7437
7438	LegalizerHelper::LegalizeResult
7439	LegalizerHelper::lowerVectorReduction(MachineInstr &MI) {
7440	Register SrcReg = MI.getOperand(1).getReg();
7441	LLT SrcTy = MRI.getType(SrcReg);
7442	LLT DstTy = MRI.getType(SrcReg);
7443
7444	// The source could be a scalar if the IR type was <1 x sN>.
7445	if (SrcTy.isScalar()) {
7446	if (DstTy.getSizeInBits() > SrcTy.getSizeInBits())
7447	return UnableToLegalize; // FIXME: handle extension.
7448	// This can be just a plain copy.
7449	Observer.changingInstr(MI);
7450	MI.setDesc(MIRBuilder.getTII().get(TargetOpcode::COPY));
7451	Observer.changedInstr(MI);
7452	return Legalized;
7453	}
7454	return UnableToLegalize;;
7455	}