Bug Summary

File:llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
Warning:line 3729, column 7
Value stored to 'NumLeftover' is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name LegalizerHelper.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -fno-split-dwarf-inlining -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-12/lib/clang/12.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/build-llvm/lib/CodeGen/GlobalISel -I /build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel -I /build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/build-llvm/include -I /build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-12/lib/clang/12.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/build-llvm/lib/CodeGen/GlobalISel -fdebug-prefix-map=/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb=. -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2020-09-26-161721-17566-1 -x c++ /build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
1//===-- llvm/CodeGen/GlobalISel/LegalizerHelper.cpp -----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This file implements the LegalizerHelper class to legalize
10/// individual instructions and the LegalizeMachineIR wrapper pass for the
11/// primary legalization.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
16#include "llvm/CodeGen/GlobalISel/CallLowering.h"
17#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
18#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
19#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
20#include "llvm/CodeGen/MachineRegisterInfo.h"
21#include "llvm/CodeGen/TargetFrameLowering.h"
22#include "llvm/CodeGen/TargetInstrInfo.h"
23#include "llvm/CodeGen/TargetLowering.h"
24#include "llvm/CodeGen/TargetSubtargetInfo.h"
25#include "llvm/Support/Debug.h"
26#include "llvm/Support/MathExtras.h"
27#include "llvm/Support/raw_ostream.h"
28
29#define DEBUG_TYPE"legalizer" "legalizer"
30
31using namespace llvm;
32using namespace LegalizeActions;
33using namespace MIPatternMatch;
34
35/// Try to break down \p OrigTy into \p NarrowTy sized pieces.
36///
37/// Returns the number of \p NarrowTy elements needed to reconstruct \p OrigTy,
38/// with any leftover piece as type \p LeftoverTy
39///
40/// Returns -1 in the first element of the pair if the breakdown is not
41/// satisfiable.
42static std::pair<int, int>
43getNarrowTypeBreakDown(LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy) {
44 assert(!LeftoverTy.isValid() && "this is an out argument")((!LeftoverTy.isValid() && "this is an out argument")
? static_cast<void> (0) : __assert_fail ("!LeftoverTy.isValid() && \"this is an out argument\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 44, __PRETTY_FUNCTION__))
;
45
46 unsigned Size = OrigTy.getSizeInBits();
47 unsigned NarrowSize = NarrowTy.getSizeInBits();
48 unsigned NumParts = Size / NarrowSize;
49 unsigned LeftoverSize = Size - NumParts * NarrowSize;
50 assert(Size > NarrowSize)((Size > NarrowSize) ? static_cast<void> (0) : __assert_fail
("Size > NarrowSize", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 50, __PRETTY_FUNCTION__))
;
51
52 if (LeftoverSize == 0)
53 return {NumParts, 0};
54
55 if (NarrowTy.isVector()) {
56 unsigned EltSize = OrigTy.getScalarSizeInBits();
57 if (LeftoverSize % EltSize != 0)
58 return {-1, -1};
59 LeftoverTy = LLT::scalarOrVector(LeftoverSize / EltSize, EltSize);
60 } else {
61 LeftoverTy = LLT::scalar(LeftoverSize);
62 }
63
64 int NumLeftover = LeftoverSize / LeftoverTy.getSizeInBits();
65 return std::make_pair(NumParts, NumLeftover);
66}
67
68static Type *getFloatTypeForLLT(LLVMContext &Ctx, LLT Ty) {
69
70 if (!Ty.isScalar())
71 return nullptr;
72
73 switch (Ty.getSizeInBits()) {
74 case 16:
75 return Type::getHalfTy(Ctx);
76 case 32:
77 return Type::getFloatTy(Ctx);
78 case 64:
79 return Type::getDoubleTy(Ctx);
80 case 80:
81 return Type::getX86_FP80Ty(Ctx);
82 case 128:
83 return Type::getFP128Ty(Ctx);
84 default:
85 return nullptr;
86 }
87}
88
89LegalizerHelper::LegalizerHelper(MachineFunction &MF,
90 GISelChangeObserver &Observer,
91 MachineIRBuilder &Builder)
92 : MIRBuilder(Builder), Observer(Observer), MRI(MF.getRegInfo()),
93 LI(*MF.getSubtarget().getLegalizerInfo()),
94 TLI(*MF.getSubtarget().getTargetLowering()) {
95 MIRBuilder.setChangeObserver(Observer);
96}
97
98LegalizerHelper::LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI,
99 GISelChangeObserver &Observer,
100 MachineIRBuilder &B)
101 : MIRBuilder(B), Observer(Observer), MRI(MF.getRegInfo()), LI(LI),
102 TLI(*MF.getSubtarget().getTargetLowering()) {
103 MIRBuilder.setChangeObserver(Observer);
104}
105LegalizerHelper::LegalizeResult
106LegalizerHelper::legalizeInstrStep(MachineInstr &MI) {
107 LLVM_DEBUG(dbgs() << "Legalizing: " << MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << "Legalizing: " << MI; }
} while (false)
;
108
109 MIRBuilder.setInstrAndDebugLoc(MI);
110
111 if (MI.getOpcode() == TargetOpcode::G_INTRINSIC ||
112 MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS)
113 return LI.legalizeIntrinsic(*this, MI) ? Legalized : UnableToLegalize;
114 auto Step = LI.getAction(MI, MRI);
115 switch (Step.Action) {
116 case Legal:
117 LLVM_DEBUG(dbgs() << ".. Already legal\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Already legal\n"; } } while
(false)
;
118 return AlreadyLegal;
119 case Libcall:
120 LLVM_DEBUG(dbgs() << ".. Convert to libcall\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Convert to libcall\n"; }
} while (false)
;
121 return libcall(MI);
122 case NarrowScalar:
123 LLVM_DEBUG(dbgs() << ".. Narrow scalar\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Narrow scalar\n"; } } while
(false)
;
124 return narrowScalar(MI, Step.TypeIdx, Step.NewType);
125 case WidenScalar:
126 LLVM_DEBUG(dbgs() << ".. Widen scalar\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Widen scalar\n"; } } while
(false)
;
127 return widenScalar(MI, Step.TypeIdx, Step.NewType);
128 case Bitcast:
129 LLVM_DEBUG(dbgs() << ".. Bitcast type\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Bitcast type\n"; } } while
(false)
;
130 return bitcast(MI, Step.TypeIdx, Step.NewType);
131 case Lower:
132 LLVM_DEBUG(dbgs() << ".. Lower\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Lower\n"; } } while (false
)
;
133 return lower(MI, Step.TypeIdx, Step.NewType);
134 case FewerElements:
135 LLVM_DEBUG(dbgs() << ".. Reduce number of elements\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Reduce number of elements\n"
; } } while (false)
;
136 return fewerElementsVector(MI, Step.TypeIdx, Step.NewType);
137 case MoreElements:
138 LLVM_DEBUG(dbgs() << ".. Increase number of elements\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Increase number of elements\n"
; } } while (false)
;
139 return moreElementsVector(MI, Step.TypeIdx, Step.NewType);
140 case Custom:
141 LLVM_DEBUG(dbgs() << ".. Custom legalization\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Custom legalization\n"; }
} while (false)
;
142 return LI.legalizeCustom(*this, MI) ? Legalized : UnableToLegalize;
143 default:
144 LLVM_DEBUG(dbgs() << ".. Unable to legalize\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Unable to legalize\n"; }
} while (false)
;
145 return UnableToLegalize;
146 }
147}
148
149void LegalizerHelper::extractParts(Register Reg, LLT Ty, int NumParts,
150 SmallVectorImpl<Register> &VRegs) {
151 for (int i = 0; i < NumParts; ++i)
152 VRegs.push_back(MRI.createGenericVirtualRegister(Ty));
153 MIRBuilder.buildUnmerge(VRegs, Reg);
154}
155
156bool LegalizerHelper::extractParts(Register Reg, LLT RegTy,
157 LLT MainTy, LLT &LeftoverTy,
158 SmallVectorImpl<Register> &VRegs,
159 SmallVectorImpl<Register> &LeftoverRegs) {
160 assert(!LeftoverTy.isValid() && "this is an out argument")((!LeftoverTy.isValid() && "this is an out argument")
? static_cast<void> (0) : __assert_fail ("!LeftoverTy.isValid() && \"this is an out argument\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 160, __PRETTY_FUNCTION__))
;
161
162 unsigned RegSize = RegTy.getSizeInBits();
163 unsigned MainSize = MainTy.getSizeInBits();
164 unsigned NumParts = RegSize / MainSize;
165 unsigned LeftoverSize = RegSize - NumParts * MainSize;
166
167 // Use an unmerge when possible.
168 if (LeftoverSize == 0) {
169 for (unsigned I = 0; I < NumParts; ++I)
170 VRegs.push_back(MRI.createGenericVirtualRegister(MainTy));
171 MIRBuilder.buildUnmerge(VRegs, Reg);
172 return true;
173 }
174
175 if (MainTy.isVector()) {
176 unsigned EltSize = MainTy.getScalarSizeInBits();
177 if (LeftoverSize % EltSize != 0)
178 return false;
179 LeftoverTy = LLT::scalarOrVector(LeftoverSize / EltSize, EltSize);
180 } else {
181 LeftoverTy = LLT::scalar(LeftoverSize);
182 }
183
184 // For irregular sizes, extract the individual parts.
185 for (unsigned I = 0; I != NumParts; ++I) {
186 Register NewReg = MRI.createGenericVirtualRegister(MainTy);
187 VRegs.push_back(NewReg);
188 MIRBuilder.buildExtract(NewReg, Reg, MainSize * I);
189 }
190
191 for (unsigned Offset = MainSize * NumParts; Offset < RegSize;
192 Offset += LeftoverSize) {
193 Register NewReg = MRI.createGenericVirtualRegister(LeftoverTy);
194 LeftoverRegs.push_back(NewReg);
195 MIRBuilder.buildExtract(NewReg, Reg, Offset);
196 }
197
198 return true;
199}
200
201void LegalizerHelper::insertParts(Register DstReg,
202 LLT ResultTy, LLT PartTy,
203 ArrayRef<Register> PartRegs,
204 LLT LeftoverTy,
205 ArrayRef<Register> LeftoverRegs) {
206 if (!LeftoverTy.isValid()) {
207 assert(LeftoverRegs.empty())((LeftoverRegs.empty()) ? static_cast<void> (0) : __assert_fail
("LeftoverRegs.empty()", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 207, __PRETTY_FUNCTION__))
;
208
209 if (!ResultTy.isVector()) {
210 MIRBuilder.buildMerge(DstReg, PartRegs);
211 return;
212 }
213
214 if (PartTy.isVector())
215 MIRBuilder.buildConcatVectors(DstReg, PartRegs);
216 else
217 MIRBuilder.buildBuildVector(DstReg, PartRegs);
218 return;
219 }
220
221 unsigned PartSize = PartTy.getSizeInBits();
222 unsigned LeftoverPartSize = LeftoverTy.getSizeInBits();
223
224 Register CurResultReg = MRI.createGenericVirtualRegister(ResultTy);
225 MIRBuilder.buildUndef(CurResultReg);
226
227 unsigned Offset = 0;
228 for (Register PartReg : PartRegs) {
229 Register NewResultReg = MRI.createGenericVirtualRegister(ResultTy);
230 MIRBuilder.buildInsert(NewResultReg, CurResultReg, PartReg, Offset);
231 CurResultReg = NewResultReg;
232 Offset += PartSize;
233 }
234
235 for (unsigned I = 0, E = LeftoverRegs.size(); I != E; ++I) {
236 // Use the original output register for the final insert to avoid a copy.
237 Register NewResultReg = (I + 1 == E) ?
238 DstReg : MRI.createGenericVirtualRegister(ResultTy);
239
240 MIRBuilder.buildInsert(NewResultReg, CurResultReg, LeftoverRegs[I], Offset);
241 CurResultReg = NewResultReg;
242 Offset += LeftoverPartSize;
243 }
244}
245
246/// Append the result registers of G_UNMERGE_VALUES \p MI to \p Regs.
247static void getUnmergeResults(SmallVectorImpl<Register> &Regs,
248 const MachineInstr &MI) {
249 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES)((MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES) ? static_cast
<void> (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 249, __PRETTY_FUNCTION__))
;
250
251 const int StartIdx = Regs.size();
252 const int NumResults = MI.getNumOperands() - 1;
253 Regs.resize(Regs.size() + NumResults);
254 for (int I = 0; I != NumResults; ++I)
255 Regs[StartIdx + I] = MI.getOperand(I).getReg();
256}
257
258void LegalizerHelper::extractGCDType(SmallVectorImpl<Register> &Parts,
259 LLT GCDTy, Register SrcReg) {
260 LLT SrcTy = MRI.getType(SrcReg);
261 if (SrcTy == GCDTy) {
262 // If the source already evenly divides the result type, we don't need to do
263 // anything.
264 Parts.push_back(SrcReg);
265 } else {
266 // Need to split into common type sized pieces.
267 auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg);
268 getUnmergeResults(Parts, *Unmerge);
269 }
270}
271
272LLT LegalizerHelper::extractGCDType(SmallVectorImpl<Register> &Parts, LLT DstTy,
273 LLT NarrowTy, Register SrcReg) {
274 LLT SrcTy = MRI.getType(SrcReg);
275 LLT GCDTy = getGCDType(getGCDType(SrcTy, NarrowTy), DstTy);
276 extractGCDType(Parts, GCDTy, SrcReg);
277 return GCDTy;
278}
279
280LLT LegalizerHelper::buildLCMMergePieces(LLT DstTy, LLT NarrowTy, LLT GCDTy,
281 SmallVectorImpl<Register> &VRegs,
282 unsigned PadStrategy) {
283 LLT LCMTy = getLCMType(DstTy, NarrowTy);
284
285 int NumParts = LCMTy.getSizeInBits() / NarrowTy.getSizeInBits();
286 int NumSubParts = NarrowTy.getSizeInBits() / GCDTy.getSizeInBits();
287 int NumOrigSrc = VRegs.size();
288
289 Register PadReg;
290
291 // Get a value we can use to pad the source value if the sources won't evenly
292 // cover the result type.
293 if (NumOrigSrc < NumParts * NumSubParts) {
294 if (PadStrategy == TargetOpcode::G_ZEXT)
295 PadReg = MIRBuilder.buildConstant(GCDTy, 0).getReg(0);
296 else if (PadStrategy == TargetOpcode::G_ANYEXT)
297 PadReg = MIRBuilder.buildUndef(GCDTy).getReg(0);
298 else {
299 assert(PadStrategy == TargetOpcode::G_SEXT)((PadStrategy == TargetOpcode::G_SEXT) ? static_cast<void>
(0) : __assert_fail ("PadStrategy == TargetOpcode::G_SEXT", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 299, __PRETTY_FUNCTION__))
;
300
301 // Shift the sign bit of the low register through the high register.
302 auto ShiftAmt =
303 MIRBuilder.buildConstant(LLT::scalar(64), GCDTy.getSizeInBits() - 1);
304 PadReg = MIRBuilder.buildAShr(GCDTy, VRegs.back(), ShiftAmt).getReg(0);
305 }
306 }
307
308 // Registers for the final merge to be produced.
309 SmallVector<Register, 4> Remerge(NumParts);
310
311 // Registers needed for intermediate merges, which will be merged into a
312 // source for Remerge.
313 SmallVector<Register, 4> SubMerge(NumSubParts);
314
315 // Once we've fully read off the end of the original source bits, we can reuse
316 // the same high bits for remaining padding elements.
317 Register AllPadReg;
318
319 // Build merges to the LCM type to cover the original result type.
320 for (int I = 0; I != NumParts; ++I) {
321 bool AllMergePartsArePadding = true;
322
323 // Build the requested merges to the requested type.
324 for (int J = 0; J != NumSubParts; ++J) {
325 int Idx = I * NumSubParts + J;
326 if (Idx >= NumOrigSrc) {
327 SubMerge[J] = PadReg;
328 continue;
329 }
330
331 SubMerge[J] = VRegs[Idx];
332
333 // There are meaningful bits here we can't reuse later.
334 AllMergePartsArePadding = false;
335 }
336
337 // If we've filled up a complete piece with padding bits, we can directly
338 // emit the natural sized constant if applicable, rather than a merge of
339 // smaller constants.
340 if (AllMergePartsArePadding && !AllPadReg) {
341 if (PadStrategy == TargetOpcode::G_ANYEXT)
342 AllPadReg = MIRBuilder.buildUndef(NarrowTy).getReg(0);
343 else if (PadStrategy == TargetOpcode::G_ZEXT)
344 AllPadReg = MIRBuilder.buildConstant(NarrowTy, 0).getReg(0);
345
346 // If this is a sign extension, we can't materialize a trivial constant
347 // with the right type and have to produce a merge.
348 }
349
350 if (AllPadReg) {
351 // Avoid creating additional instructions if we're just adding additional
352 // copies of padding bits.
353 Remerge[I] = AllPadReg;
354 continue;
355 }
356
357 if (NumSubParts == 1)
358 Remerge[I] = SubMerge[0];
359 else
360 Remerge[I] = MIRBuilder.buildMerge(NarrowTy, SubMerge).getReg(0);
361
362 // In the sign extend padding case, re-use the first all-signbit merge.
363 if (AllMergePartsArePadding && !AllPadReg)
364 AllPadReg = Remerge[I];
365 }
366
367 VRegs = std::move(Remerge);
368 return LCMTy;
369}
370
371void LegalizerHelper::buildWidenedRemergeToDst(Register DstReg, LLT LCMTy,
372 ArrayRef<Register> RemergeRegs) {
373 LLT DstTy = MRI.getType(DstReg);
374
375 // Create the merge to the widened source, and extract the relevant bits into
376 // the result.
377
378 if (DstTy == LCMTy) {
379 MIRBuilder.buildMerge(DstReg, RemergeRegs);
380 return;
381 }
382
383 auto Remerge = MIRBuilder.buildMerge(LCMTy, RemergeRegs);
384 if (DstTy.isScalar() && LCMTy.isScalar()) {
385 MIRBuilder.buildTrunc(DstReg, Remerge);
386 return;
387 }
388
389 if (LCMTy.isVector()) {
390 unsigned NumDefs = LCMTy.getSizeInBits() / DstTy.getSizeInBits();
391 SmallVector<Register, 8> UnmergeDefs(NumDefs);
392 UnmergeDefs[0] = DstReg;
393 for (unsigned I = 1; I != NumDefs; ++I)
394 UnmergeDefs[I] = MRI.createGenericVirtualRegister(DstTy);
395
396 MIRBuilder.buildUnmerge(UnmergeDefs,
397 MIRBuilder.buildMerge(LCMTy, RemergeRegs));
398 return;
399 }
400
401 llvm_unreachable("unhandled case")::llvm::llvm_unreachable_internal("unhandled case", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 401)
;
402}
403
404static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
405#define RTLIBCASE_INT(LibcallPrefix)do { switch (Size) { case 32: return RTLIB::LibcallPrefix32; case
64: return RTLIB::LibcallPrefix64; case 128: return RTLIB::LibcallPrefix128
; default: ::llvm::llvm_unreachable_internal("unexpected size"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 405); } } while (0)
\
406 do { \
407 switch (Size) { \
408 case 32: \
409 return RTLIB::LibcallPrefix##32; \
410 case 64: \
411 return RTLIB::LibcallPrefix##64; \
412 case 128: \
413 return RTLIB::LibcallPrefix##128; \
414 default: \
415 llvm_unreachable("unexpected size")::llvm::llvm_unreachable_internal("unexpected size", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 415)
; \
416 } \
417 } while (0)
418
419#define RTLIBCASE(LibcallPrefix)do { switch (Size) { case 32: return RTLIB::LibcallPrefix32; case
64: return RTLIB::LibcallPrefix64; case 80: return RTLIB::LibcallPrefix80
; case 128: return RTLIB::LibcallPrefix128; default: ::llvm::
llvm_unreachable_internal("unexpected size", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 419); } } while (0)
\
420 do { \
421 switch (Size) { \
422 case 32: \
423 return RTLIB::LibcallPrefix##32; \
424 case 64: \
425 return RTLIB::LibcallPrefix##64; \
426 case 80: \
427 return RTLIB::LibcallPrefix##80; \
428 case 128: \
429 return RTLIB::LibcallPrefix##128; \
430 default: \
431 llvm_unreachable("unexpected size")::llvm::llvm_unreachable_internal("unexpected size", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 431)
; \
432 } \
433 } while (0)
434
435 switch (Opcode) {
436 case TargetOpcode::G_SDIV:
437 RTLIBCASE_INT(SDIV_I)do { switch (Size) { case 32: return RTLIB::SDIV_I32; case 64
: return RTLIB::SDIV_I64; case 128: return RTLIB::SDIV_I128; default
: ::llvm::llvm_unreachable_internal("unexpected size", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 437); } } while (0)
;
438 case TargetOpcode::G_UDIV:
439 RTLIBCASE_INT(UDIV_I)do { switch (Size) { case 32: return RTLIB::UDIV_I32; case 64
: return RTLIB::UDIV_I64; case 128: return RTLIB::UDIV_I128; default
: ::llvm::llvm_unreachable_internal("unexpected size", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 439); } } while (0)
;
440 case TargetOpcode::G_SREM:
441 RTLIBCASE_INT(SREM_I)do { switch (Size) { case 32: return RTLIB::SREM_I32; case 64
: return RTLIB::SREM_I64; case 128: return RTLIB::SREM_I128; default
: ::llvm::llvm_unreachable_internal("unexpected size", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 441); } } while (0)
;
442 case TargetOpcode::G_UREM:
443 RTLIBCASE_INT(UREM_I)do { switch (Size) { case 32: return RTLIB::UREM_I32; case 64
: return RTLIB::UREM_I64; case 128: return RTLIB::UREM_I128; default
: ::llvm::llvm_unreachable_internal("unexpected size", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 443); } } while (0)
;
444 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
445 RTLIBCASE_INT(CTLZ_I)do { switch (Size) { case 32: return RTLIB::CTLZ_I32; case 64
: return RTLIB::CTLZ_I64; case 128: return RTLIB::CTLZ_I128; default
: ::llvm::llvm_unreachable_internal("unexpected size", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 445); } } while (0)
;
446 case TargetOpcode::G_FADD:
447 RTLIBCASE(ADD_F)do { switch (Size) { case 32: return RTLIB::ADD_F32; case 64:
return RTLIB::ADD_F64; case 80: return RTLIB::ADD_F80; case 128
: return RTLIB::ADD_F128; default: ::llvm::llvm_unreachable_internal
("unexpected size", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 447); } } while (0)
;
448 case TargetOpcode::G_FSUB:
449 RTLIBCASE(SUB_F)do { switch (Size) { case 32: return RTLIB::SUB_F32; case 64:
return RTLIB::SUB_F64; case 80: return RTLIB::SUB_F80; case 128
: return RTLIB::SUB_F128; default: ::llvm::llvm_unreachable_internal
("unexpected size", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 449); } } while (0)
;
450 case TargetOpcode::G_FMUL:
451 RTLIBCASE(MUL_F)do { switch (Size) { case 32: return RTLIB::MUL_F32; case 64:
return RTLIB::MUL_F64; case 80: return RTLIB::MUL_F80; case 128
: return RTLIB::MUL_F128; default: ::llvm::llvm_unreachable_internal
("unexpected size", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 451); } } while (0)
;
452 case TargetOpcode::G_FDIV:
453 RTLIBCASE(DIV_F)do { switch (Size) { case 32: return RTLIB::DIV_F32; case 64:
return RTLIB::DIV_F64; case 80: return RTLIB::DIV_F80; case 128
: return RTLIB::DIV_F128; default: ::llvm::llvm_unreachable_internal
("unexpected size", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 453); } } while (0)
;
454 case TargetOpcode::G_FEXP:
455 RTLIBCASE(EXP_F)do { switch (Size) { case 32: return RTLIB::EXP_F32; case 64:
return RTLIB::EXP_F64; case 80: return RTLIB::EXP_F80; case 128
: return RTLIB::EXP_F128; default: ::llvm::llvm_unreachable_internal
("unexpected size", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 455); } } while (0)
;
456 case TargetOpcode::G_FEXP2:
457 RTLIBCASE(EXP2_F)do { switch (Size) { case 32: return RTLIB::EXP2_F32; case 64
: return RTLIB::EXP2_F64; case 80: return RTLIB::EXP2_F80; case
128: return RTLIB::EXP2_F128; default: ::llvm::llvm_unreachable_internal
("unexpected size", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 457); } } while (0)
;
458 case TargetOpcode::G_FREM:
459 RTLIBCASE(REM_F)do { switch (Size) { case 32: return RTLIB::REM_F32; case 64:
return RTLIB::REM_F64; case 80: return RTLIB::REM_F80; case 128
: return RTLIB::REM_F128; default: ::llvm::llvm_unreachable_internal
("unexpected size", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 459); } } while (0)
;
460 case TargetOpcode::G_FPOW:
461 RTLIBCASE(POW_F)do { switch (Size) { case 32: return RTLIB::POW_F32; case 64:
return RTLIB::POW_F64; case 80: return RTLIB::POW_F80; case 128
: return RTLIB::POW_F128; default: ::llvm::llvm_unreachable_internal
("unexpected size", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 461); } } while (0)
;
462 case TargetOpcode::G_FMA:
463 RTLIBCASE(FMA_F)do { switch (Size) { case 32: return RTLIB::FMA_F32; case 64:
return RTLIB::FMA_F64; case 80: return RTLIB::FMA_F80; case 128
: return RTLIB::FMA_F128; default: ::llvm::llvm_unreachable_internal
("unexpected size", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 463); } } while (0)
;
464 case TargetOpcode::G_FSIN:
465 RTLIBCASE(SIN_F)do { switch (Size) { case 32: return RTLIB::SIN_F32; case 64:
return RTLIB::SIN_F64; case 80: return RTLIB::SIN_F80; case 128
: return RTLIB::SIN_F128; default: ::llvm::llvm_unreachable_internal
("unexpected size", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 465); } } while (0)
;
466 case TargetOpcode::G_FCOS:
467 RTLIBCASE(COS_F)do { switch (Size) { case 32: return RTLIB::COS_F32; case 64:
return RTLIB::COS_F64; case 80: return RTLIB::COS_F80; case 128
: return RTLIB::COS_F128; default: ::llvm::llvm_unreachable_internal
("unexpected size", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 467); } } while (0)
;
468 case TargetOpcode::G_FLOG10:
469 RTLIBCASE(LOG10_F)do { switch (Size) { case 32: return RTLIB::LOG10_F32; case 64
: return RTLIB::LOG10_F64; case 80: return RTLIB::LOG10_F80; case
128: return RTLIB::LOG10_F128; default: ::llvm::llvm_unreachable_internal
("unexpected size", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 469); } } while (0)
;
470 case TargetOpcode::G_FLOG:
471 RTLIBCASE(LOG_F)do { switch (Size) { case 32: return RTLIB::LOG_F32; case 64:
return RTLIB::LOG_F64; case 80: return RTLIB::LOG_F80; case 128
: return RTLIB::LOG_F128; default: ::llvm::llvm_unreachable_internal
("unexpected size", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 471); } } while (0)
;
472 case TargetOpcode::G_FLOG2:
473 RTLIBCASE(LOG2_F)do { switch (Size) { case 32: return RTLIB::LOG2_F32; case 64
: return RTLIB::LOG2_F64; case 80: return RTLIB::LOG2_F80; case
128: return RTLIB::LOG2_F128; default: ::llvm::llvm_unreachable_internal
("unexpected size", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 473); } } while (0)
;
474 case TargetOpcode::G_FCEIL:
475 RTLIBCASE(CEIL_F)do { switch (Size) { case 32: return RTLIB::CEIL_F32; case 64
: return RTLIB::CEIL_F64; case 80: return RTLIB::CEIL_F80; case
128: return RTLIB::CEIL_F128; default: ::llvm::llvm_unreachable_internal
("unexpected size", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 475); } } while (0)
;
476 case TargetOpcode::G_FFLOOR:
477 RTLIBCASE(FLOOR_F)do { switch (Size) { case 32: return RTLIB::FLOOR_F32; case 64
: return RTLIB::FLOOR_F64; case 80: return RTLIB::FLOOR_F80; case
128: return RTLIB::FLOOR_F128; default: ::llvm::llvm_unreachable_internal
("unexpected size", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 477); } } while (0)
;
478 case TargetOpcode::G_FMINNUM:
479 RTLIBCASE(FMIN_F)do { switch (Size) { case 32: return RTLIB::FMIN_F32; case 64
: return RTLIB::FMIN_F64; case 80: return RTLIB::FMIN_F80; case
128: return RTLIB::FMIN_F128; default: ::llvm::llvm_unreachable_internal
("unexpected size", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 479); } } while (0)
;
480 case TargetOpcode::G_FMAXNUM:
481 RTLIBCASE(FMAX_F)do { switch (Size) { case 32: return RTLIB::FMAX_F32; case 64
: return RTLIB::FMAX_F64; case 80: return RTLIB::FMAX_F80; case
128: return RTLIB::FMAX_F128; default: ::llvm::llvm_unreachable_internal
("unexpected size", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 481); } } while (0)
;
482 case TargetOpcode::G_FSQRT:
483 RTLIBCASE(SQRT_F)do { switch (Size) { case 32: return RTLIB::SQRT_F32; case 64
: return RTLIB::SQRT_F64; case 80: return RTLIB::SQRT_F80; case
128: return RTLIB::SQRT_F128; default: ::llvm::llvm_unreachable_internal
("unexpected size", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 483); } } while (0)
;
484 case TargetOpcode::G_FRINT:
485 RTLIBCASE(RINT_F)do { switch (Size) { case 32: return RTLIB::RINT_F32; case 64
: return RTLIB::RINT_F64; case 80: return RTLIB::RINT_F80; case
128: return RTLIB::RINT_F128; default: ::llvm::llvm_unreachable_internal
("unexpected size", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 485); } } while (0)
;
486 case TargetOpcode::G_FNEARBYINT:
487 RTLIBCASE(NEARBYINT_F)do { switch (Size) { case 32: return RTLIB::NEARBYINT_F32; case
64: return RTLIB::NEARBYINT_F64; case 80: return RTLIB::NEARBYINT_F80
; case 128: return RTLIB::NEARBYINT_F128; default: ::llvm::llvm_unreachable_internal
("unexpected size", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 487); } } while (0)
;
488 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
489 RTLIBCASE(ROUNDEVEN_F)do { switch (Size) { case 32: return RTLIB::ROUNDEVEN_F32; case
64: return RTLIB::ROUNDEVEN_F64; case 80: return RTLIB::ROUNDEVEN_F80
; case 128: return RTLIB::ROUNDEVEN_F128; default: ::llvm::llvm_unreachable_internal
("unexpected size", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 489); } } while (0)
;
490 }
491 llvm_unreachable("Unknown libcall function")::llvm::llvm_unreachable_internal("Unknown libcall function",
"/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 491)
;
492}
493
494/// True if an instruction is in tail position in its caller. Intended for
495/// legalizing libcalls as tail calls when possible.
496static bool isLibCallInTailPosition(const TargetInstrInfo &TII,
497 MachineInstr &MI) {
498 MachineBasicBlock &MBB = *MI.getParent();
499 const Function &F = MBB.getParent()->getFunction();
500
501 // Conservatively require the attributes of the call to match those of
502 // the return. Ignore NoAlias and NonNull because they don't affect the
503 // call sequence.
504 AttributeList CallerAttrs = F.getAttributes();
505 if (AttrBuilder(CallerAttrs, AttributeList::ReturnIndex)
506 .removeAttribute(Attribute::NoAlias)
507 .removeAttribute(Attribute::NonNull)
508 .hasAttributes())
509 return false;
510
511 // It's not safe to eliminate the sign / zero extension of the return value.
512 if (CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::ZExt) ||
513 CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::SExt))
514 return false;
515
516 // Only tail call if the following instruction is a standard return.
517 auto Next = next_nodbg(MI.getIterator(), MBB.instr_end());
518 if (Next == MBB.instr_end() || TII.isTailCall(*Next) || !Next->isReturn())
519 return false;
520
521 return true;
522}
523
524LegalizerHelper::LegalizeResult
525llvm::createLibcall(MachineIRBuilder &MIRBuilder, const char *Name,
526 const CallLowering::ArgInfo &Result,
527 ArrayRef<CallLowering::ArgInfo> Args,
528 const CallingConv::ID CC) {
529 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
530
531 CallLowering::CallLoweringInfo Info;
532 Info.CallConv = CC;
533 Info.Callee = MachineOperand::CreateES(Name);
534 Info.OrigRet = Result;
535 std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs));
536 if (!CLI.lowerCall(MIRBuilder, Info))
537 return LegalizerHelper::UnableToLegalize;
538
539 return LegalizerHelper::Legalized;
540}
541
542LegalizerHelper::LegalizeResult
543llvm::createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall,
544 const CallLowering::ArgInfo &Result,
545 ArrayRef<CallLowering::ArgInfo> Args) {
546 auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
547 const char *Name = TLI.getLibcallName(Libcall);
548 const CallingConv::ID CC = TLI.getLibcallCallingConv(Libcall);
549 return createLibcall(MIRBuilder, Name, Result, Args, CC);
550}
551
552// Useful for libcalls where all operands have the same type.
553static LegalizerHelper::LegalizeResult
554simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size,
555 Type *OpType) {
556 auto Libcall = getRTLibDesc(MI.getOpcode(), Size);
557
558 SmallVector<CallLowering::ArgInfo, 3> Args;
559 for (unsigned i = 1; i < MI.getNumOperands(); i++)
560 Args.push_back({MI.getOperand(i).getReg(), OpType});
561 return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), OpType},
562 Args);
563}
564
565LegalizerHelper::LegalizeResult
566llvm::createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
567 MachineInstr &MI) {
568 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
569
570 SmallVector<CallLowering::ArgInfo, 3> Args;
571 // Add all the args, except for the last which is an imm denoting 'tail'.
572 for (unsigned i = 0; i < MI.getNumOperands() - 1; ++i) {
573 Register Reg = MI.getOperand(i).getReg();
574
575 // Need derive an IR type for call lowering.
576 LLT OpLLT = MRI.getType(Reg);
577 Type *OpTy = nullptr;
578 if (OpLLT.isPointer())
579 OpTy = Type::getInt8PtrTy(Ctx, OpLLT.getAddressSpace());
580 else
581 OpTy = IntegerType::get(Ctx, OpLLT.getSizeInBits());
582 Args.push_back({Reg, OpTy});
583 }
584
585 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
586 auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
587 RTLIB::Libcall RTLibcall;
588 switch (MI.getOpcode()) {
589 case TargetOpcode::G_MEMCPY:
590 RTLibcall = RTLIB::MEMCPY;
591 break;
592 case TargetOpcode::G_MEMMOVE:
593 RTLibcall = RTLIB::MEMMOVE;
594 break;
595 case TargetOpcode::G_MEMSET:
596 RTLibcall = RTLIB::MEMSET;
597 break;
598 default:
599 return LegalizerHelper::UnableToLegalize;
600 }
601 const char *Name = TLI.getLibcallName(RTLibcall);
602
603 CallLowering::CallLoweringInfo Info;
604 Info.CallConv = TLI.getLibcallCallingConv(RTLibcall);
605 Info.Callee = MachineOperand::CreateES(Name);
606 Info.OrigRet = CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx));
607 Info.IsTailCall = MI.getOperand(MI.getNumOperands() - 1).getImm() &&
608 isLibCallInTailPosition(MIRBuilder.getTII(), MI);
609
610 std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs));
611 if (!CLI.lowerCall(MIRBuilder, Info))
612 return LegalizerHelper::UnableToLegalize;
613
614 if (Info.LoweredTailCall) {
615 assert(Info.IsTailCall && "Lowered tail call when it wasn't a tail call?")((Info.IsTailCall && "Lowered tail call when it wasn't a tail call?"
) ? static_cast<void> (0) : __assert_fail ("Info.IsTailCall && \"Lowered tail call when it wasn't a tail call?\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 615, __PRETTY_FUNCTION__))
;
616 // We must have a return following the call (or debug insts) to get past
617 // isLibCallInTailPosition.
618 do {
619 MachineInstr *Next = MI.getNextNode();
620 assert(Next && (Next->isReturn() || Next->isDebugInstr()) &&((Next && (Next->isReturn() || Next->isDebugInstr
()) && "Expected instr following MI to be return or debug inst?"
) ? static_cast<void> (0) : __assert_fail ("Next && (Next->isReturn() || Next->isDebugInstr()) && \"Expected instr following MI to be return or debug inst?\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 621, __PRETTY_FUNCTION__))
621 "Expected instr following MI to be return or debug inst?")((Next && (Next->isReturn() || Next->isDebugInstr
()) && "Expected instr following MI to be return or debug inst?"
) ? static_cast<void> (0) : __assert_fail ("Next && (Next->isReturn() || Next->isDebugInstr()) && \"Expected instr following MI to be return or debug inst?\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 621, __PRETTY_FUNCTION__))
;
622 // We lowered a tail call, so the call is now the return from the block.
623 // Delete the old return.
624 Next->eraseFromParent();
625 } while (MI.getNextNode());
626 }
627
628 return LegalizerHelper::Legalized;
629}
630
631static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType,
632 Type *FromType) {
633 auto ToMVT = MVT::getVT(ToType);
634 auto FromMVT = MVT::getVT(FromType);
635
636 switch (Opcode) {
637 case TargetOpcode::G_FPEXT:
638 return RTLIB::getFPEXT(FromMVT, ToMVT);
639 case TargetOpcode::G_FPTRUNC:
640 return RTLIB::getFPROUND(FromMVT, ToMVT);
641 case TargetOpcode::G_FPTOSI:
642 return RTLIB::getFPTOSINT(FromMVT, ToMVT);
643 case TargetOpcode::G_FPTOUI:
644 return RTLIB::getFPTOUINT(FromMVT, ToMVT);
645 case TargetOpcode::G_SITOFP:
646 return RTLIB::getSINTTOFP(FromMVT, ToMVT);
647 case TargetOpcode::G_UITOFP:
648 return RTLIB::getUINTTOFP(FromMVT, ToMVT);
649 }
650 llvm_unreachable("Unsupported libcall function")::llvm::llvm_unreachable_internal("Unsupported libcall function"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 650)
;
651}
652
653static LegalizerHelper::LegalizeResult
654conversionLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, Type *ToType,
655 Type *FromType) {
656 RTLIB::Libcall Libcall = getConvRTLibDesc(MI.getOpcode(), ToType, FromType);
657 return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), ToType},
658 {{MI.getOperand(1).getReg(), FromType}});
659}
660
661LegalizerHelper::LegalizeResult
662LegalizerHelper::libcall(MachineInstr &MI) {
663 LLT LLTy = MRI.getType(MI.getOperand(0).getReg());
664 unsigned Size = LLTy.getSizeInBits();
665 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
666
667 switch (MI.getOpcode()) {
668 default:
669 return UnableToLegalize;
670 case TargetOpcode::G_SDIV:
671 case TargetOpcode::G_UDIV:
672 case TargetOpcode::G_SREM:
673 case TargetOpcode::G_UREM:
674 case TargetOpcode::G_CTLZ_ZERO_UNDEF: {
675 Type *HLTy = IntegerType::get(Ctx, Size);
676 auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
677 if (Status != Legalized)
678 return Status;
679 break;
680 }
681 case TargetOpcode::G_FADD:
682 case TargetOpcode::G_FSUB:
683 case TargetOpcode::G_FMUL:
684 case TargetOpcode::G_FDIV:
685 case TargetOpcode::G_FMA:
686 case TargetOpcode::G_FPOW:
687 case TargetOpcode::G_FREM:
688 case TargetOpcode::G_FCOS:
689 case TargetOpcode::G_FSIN:
690 case TargetOpcode::G_FLOG10:
691 case TargetOpcode::G_FLOG:
692 case TargetOpcode::G_FLOG2:
693 case TargetOpcode::G_FEXP:
694 case TargetOpcode::G_FEXP2:
695 case TargetOpcode::G_FCEIL:
696 case TargetOpcode::G_FFLOOR:
697 case TargetOpcode::G_FMINNUM:
698 case TargetOpcode::G_FMAXNUM:
699 case TargetOpcode::G_FSQRT:
700 case TargetOpcode::G_FRINT:
701 case TargetOpcode::G_FNEARBYINT:
702 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: {
703 Type *HLTy = getFloatTypeForLLT(Ctx, LLTy);
704 if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
705 LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << "No libcall available for type "
<< LLTy << ".\n"; } } while (false)
;
706 return UnableToLegalize;
707 }
708 auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
709 if (Status != Legalized)
710 return Status;
711 break;
712 }
713 case TargetOpcode::G_FPEXT:
714 case TargetOpcode::G_FPTRUNC: {
715 Type *FromTy = getFloatTypeForLLT(Ctx, MRI.getType(MI.getOperand(1).getReg()));
716 Type *ToTy = getFloatTypeForLLT(Ctx, MRI.getType(MI.getOperand(0).getReg()));
717 if (!FromTy || !ToTy)
718 return UnableToLegalize;
719 LegalizeResult Status = conversionLibcall(MI, MIRBuilder, ToTy, FromTy );
720 if (Status != Legalized)
721 return Status;
722 break;
723 }
724 case TargetOpcode::G_FPTOSI:
725 case TargetOpcode::G_FPTOUI: {
726 // FIXME: Support other types
727 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
728 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
729 if ((ToSize != 32 && ToSize != 64) || (FromSize != 32 && FromSize != 64))
730 return UnableToLegalize;
731 LegalizeResult Status = conversionLibcall(
732 MI, MIRBuilder,
733 ToSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx),
734 FromSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx));
735 if (Status != Legalized)
736 return Status;
737 break;
738 }
739 case TargetOpcode::G_SITOFP:
740 case TargetOpcode::G_UITOFP: {
741 // FIXME: Support other types
742 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
743 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
744 if ((FromSize != 32 && FromSize != 64) || (ToSize != 32 && ToSize != 64))
745 return UnableToLegalize;
746 LegalizeResult Status = conversionLibcall(
747 MI, MIRBuilder,
748 ToSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx),
749 FromSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx));
750 if (Status != Legalized)
751 return Status;
752 break;
753 }
754 case TargetOpcode::G_MEMCPY:
755 case TargetOpcode::G_MEMMOVE:
756 case TargetOpcode::G_MEMSET: {
757 LegalizeResult Result = createMemLibcall(MIRBuilder, *MIRBuilder.getMRI(), MI);
758 MI.eraseFromParent();
759 return Result;
760 }
761 }
762
763 MI.eraseFromParent();
764 return Legalized;
765}
766
767LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
768 unsigned TypeIdx,
769 LLT NarrowTy) {
770 uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
771 uint64_t NarrowSize = NarrowTy.getSizeInBits();
772
773 switch (MI.getOpcode()) {
774 default:
775 return UnableToLegalize;
776 case TargetOpcode::G_IMPLICIT_DEF: {
777 Register DstReg = MI.getOperand(0).getReg();
778 LLT DstTy = MRI.getType(DstReg);
779
780 // If SizeOp0 is not an exact multiple of NarrowSize, emit
781 // G_ANYEXT(G_IMPLICIT_DEF). Cast result to vector if needed.
782 // FIXME: Although this would also be legal for the general case, it causes
783 // a lot of regressions in the emitted code (superfluous COPYs, artifact
784 // combines not being hit). This seems to be a problem related to the
785 // artifact combiner.
786 if (SizeOp0 % NarrowSize != 0) {
787 LLT ImplicitTy = NarrowTy;
788 if (DstTy.isVector())
789 ImplicitTy = LLT::vector(DstTy.getNumElements(), ImplicitTy);
790
791 Register ImplicitReg = MIRBuilder.buildUndef(ImplicitTy).getReg(0);
792 MIRBuilder.buildAnyExt(DstReg, ImplicitReg);
793
794 MI.eraseFromParent();
795 return Legalized;
796 }
797
798 int NumParts = SizeOp0 / NarrowSize;
799
800 SmallVector<Register, 2> DstRegs;
801 for (int i = 0; i < NumParts; ++i)
802 DstRegs.push_back(MIRBuilder.buildUndef(NarrowTy).getReg(0));
803
804 if (DstTy.isVector())
805 MIRBuilder.buildBuildVector(DstReg, DstRegs);
806 else
807 MIRBuilder.buildMerge(DstReg, DstRegs);
808 MI.eraseFromParent();
809 return Legalized;
810 }
811 case TargetOpcode::G_CONSTANT: {
812 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
813 const APInt &Val = MI.getOperand(1).getCImm()->getValue();
814 unsigned TotalSize = Ty.getSizeInBits();
815 unsigned NarrowSize = NarrowTy.getSizeInBits();
816 int NumParts = TotalSize / NarrowSize;
817
818 SmallVector<Register, 4> PartRegs;
819 for (int I = 0; I != NumParts; ++I) {
820 unsigned Offset = I * NarrowSize;
821 auto K = MIRBuilder.buildConstant(NarrowTy,
822 Val.lshr(Offset).trunc(NarrowSize));
823 PartRegs.push_back(K.getReg(0));
824 }
825
826 LLT LeftoverTy;
827 unsigned LeftoverBits = TotalSize - NumParts * NarrowSize;
828 SmallVector<Register, 1> LeftoverRegs;
829 if (LeftoverBits != 0) {
830 LeftoverTy = LLT::scalar(LeftoverBits);
831 auto K = MIRBuilder.buildConstant(
832 LeftoverTy,
833 Val.lshr(NumParts * NarrowSize).trunc(LeftoverBits));
834 LeftoverRegs.push_back(K.getReg(0));
835 }
836
837 insertParts(MI.getOperand(0).getReg(),
838 Ty, NarrowTy, PartRegs, LeftoverTy, LeftoverRegs);
839
840 MI.eraseFromParent();
841 return Legalized;
842 }
843 case TargetOpcode::G_SEXT:
844 case TargetOpcode::G_ZEXT:
845 case TargetOpcode::G_ANYEXT:
846 return narrowScalarExt(MI, TypeIdx, NarrowTy);
847 case TargetOpcode::G_TRUNC: {
848 if (TypeIdx != 1)
849 return UnableToLegalize;
850
851 uint64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
852 if (NarrowTy.getSizeInBits() * 2 != SizeOp1) {
853 LLVM_DEBUG(dbgs() << "Can't narrow trunc to type " << NarrowTy << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << "Can't narrow trunc to type "
<< NarrowTy << "\n"; } } while (false)
;
854 return UnableToLegalize;
855 }
856
857 auto Unmerge = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1));
858 MIRBuilder.buildCopy(MI.getOperand(0), Unmerge.getReg(0));
859 MI.eraseFromParent();
860 return Legalized;
861 }
862
863 case TargetOpcode::G_FREEZE:
864 return reduceOperationWidth(MI, TypeIdx, NarrowTy);
865
866 case TargetOpcode::G_ADD: {
867 // FIXME: add support for when SizeOp0 isn't an exact multiple of
868 // NarrowSize.
869 if (SizeOp0 % NarrowSize != 0)
870 return UnableToLegalize;
871 // Expand in terms of carry-setting/consuming G_ADDE instructions.
872 int NumParts = SizeOp0 / NarrowTy.getSizeInBits();
873
874 SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs;
875 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
876 extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
877
878 Register CarryIn;
879 for (int i = 0; i < NumParts; ++i) {
880 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy);
881 Register CarryOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
882
883 if (i == 0)
884 MIRBuilder.buildUAddo(DstReg, CarryOut, Src1Regs[i], Src2Regs[i]);
885 else {
886 MIRBuilder.buildUAdde(DstReg, CarryOut, Src1Regs[i],
887 Src2Regs[i], CarryIn);
888 }
889
890 DstRegs.push_back(DstReg);
891 CarryIn = CarryOut;
892 }
893 Register DstReg = MI.getOperand(0).getReg();
894 if(MRI.getType(DstReg).isVector())
895 MIRBuilder.buildBuildVector(DstReg, DstRegs);
896 else
897 MIRBuilder.buildMerge(DstReg, DstRegs);
898 MI.eraseFromParent();
899 return Legalized;
900 }
901 case TargetOpcode::G_SUB: {
902 // FIXME: add support for when SizeOp0 isn't an exact multiple of
903 // NarrowSize.
904 if (SizeOp0 % NarrowSize != 0)
905 return UnableToLegalize;
906
907 int NumParts = SizeOp0 / NarrowTy.getSizeInBits();
908
909 SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs;
910 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
911 extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
912
913 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy);
914 Register BorrowOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
915 MIRBuilder.buildInstr(TargetOpcode::G_USUBO, {DstReg, BorrowOut},
916 {Src1Regs[0], Src2Regs[0]});
917 DstRegs.push_back(DstReg);
918 Register BorrowIn = BorrowOut;
919 for (int i = 1; i < NumParts; ++i) {
920 DstReg = MRI.createGenericVirtualRegister(NarrowTy);
921 BorrowOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
922
923 MIRBuilder.buildInstr(TargetOpcode::G_USUBE, {DstReg, BorrowOut},
924 {Src1Regs[i], Src2Regs[i], BorrowIn});
925
926 DstRegs.push_back(DstReg);
927 BorrowIn = BorrowOut;
928 }
929 MIRBuilder.buildMerge(MI.getOperand(0), DstRegs);
930 MI.eraseFromParent();
931 return Legalized;
932 }
933 case TargetOpcode::G_MUL:
934 case TargetOpcode::G_UMULH:
935 return narrowScalarMul(MI, NarrowTy);
936 case TargetOpcode::G_EXTRACT:
937 return narrowScalarExtract(MI, TypeIdx, NarrowTy);
938 case TargetOpcode::G_INSERT:
939 return narrowScalarInsert(MI, TypeIdx, NarrowTy);
940 case TargetOpcode::G_LOAD: {
941 auto &MMO = **MI.memoperands_begin();
942 Register DstReg = MI.getOperand(0).getReg();
943 LLT DstTy = MRI.getType(DstReg);
944 if (DstTy.isVector())
945 return UnableToLegalize;
946
947 if (8 * MMO.getSize() != DstTy.getSizeInBits()) {
948 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
949 MIRBuilder.buildLoad(TmpReg, MI.getOperand(1), MMO);
950 MIRBuilder.buildAnyExt(DstReg, TmpReg);
951 MI.eraseFromParent();
952 return Legalized;
953 }
954
955 return reduceLoadStoreWidth(MI, TypeIdx, NarrowTy);
956 }
957 case TargetOpcode::G_ZEXTLOAD:
958 case TargetOpcode::G_SEXTLOAD: {
959 bool ZExt = MI.getOpcode() == TargetOpcode::G_ZEXTLOAD;
960 Register DstReg = MI.getOperand(0).getReg();
961 Register PtrReg = MI.getOperand(1).getReg();
962
963 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
964 auto &MMO = **MI.memoperands_begin();
965 if (MMO.getSizeInBits() == NarrowSize) {
966 MIRBuilder.buildLoad(TmpReg, PtrReg, MMO);
967 } else {
968 MIRBuilder.buildLoadInstr(MI.getOpcode(), TmpReg, PtrReg, MMO);
969 }
970
971 if (ZExt)
972 MIRBuilder.buildZExt(DstReg, TmpReg);
973 else
974 MIRBuilder.buildSExt(DstReg, TmpReg);
975
976 MI.eraseFromParent();
977 return Legalized;
978 }
979 case TargetOpcode::G_STORE: {
980 const auto &MMO = **MI.memoperands_begin();
981
982 Register SrcReg = MI.getOperand(0).getReg();
983 LLT SrcTy = MRI.getType(SrcReg);
984 if (SrcTy.isVector())
985 return UnableToLegalize;
986
987 int NumParts = SizeOp0 / NarrowSize;
988 unsigned HandledSize = NumParts * NarrowTy.getSizeInBits();
989 unsigned LeftoverBits = SrcTy.getSizeInBits() - HandledSize;
990 if (SrcTy.isVector() && LeftoverBits != 0)
991 return UnableToLegalize;
992
993 if (8 * MMO.getSize() != SrcTy.getSizeInBits()) {
994 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
995 auto &MMO = **MI.memoperands_begin();
996 MIRBuilder.buildTrunc(TmpReg, SrcReg);
997 MIRBuilder.buildStore(TmpReg, MI.getOperand(1), MMO);
998 MI.eraseFromParent();
999 return Legalized;
1000 }
1001
1002 return reduceLoadStoreWidth(MI, 0, NarrowTy);
1003 }
1004 case TargetOpcode::G_SELECT:
1005 return narrowScalarSelect(MI, TypeIdx, NarrowTy);
1006 case TargetOpcode::G_AND:
1007 case TargetOpcode::G_OR:
1008 case TargetOpcode::G_XOR: {
1009 // Legalize bitwise operation:
1010 // A = BinOp<Ty> B, C
1011 // into:
1012 // B1, ..., BN = G_UNMERGE_VALUES B
1013 // C1, ..., CN = G_UNMERGE_VALUES C
1014 // A1 = BinOp<Ty/N> B1, C2
1015 // ...
1016 // AN = BinOp<Ty/N> BN, CN
1017 // A = G_MERGE_VALUES A1, ..., AN
1018 return narrowScalarBasic(MI, TypeIdx, NarrowTy);
1019 }
1020 case TargetOpcode::G_SHL:
1021 case TargetOpcode::G_LSHR:
1022 case TargetOpcode::G_ASHR:
1023 return narrowScalarShift(MI, TypeIdx, NarrowTy);
1024 case TargetOpcode::G_CTLZ:
1025 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
1026 case TargetOpcode::G_CTTZ:
1027 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
1028 case TargetOpcode::G_CTPOP:
1029 if (TypeIdx == 1)
1030 switch (MI.getOpcode()) {
1031 case TargetOpcode::G_CTLZ:
1032 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
1033 return narrowScalarCTLZ(MI, TypeIdx, NarrowTy);
1034 case TargetOpcode::G_CTTZ:
1035 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
1036 return narrowScalarCTTZ(MI, TypeIdx, NarrowTy);
1037 case TargetOpcode::G_CTPOP:
1038 return narrowScalarCTPOP(MI, TypeIdx, NarrowTy);
1039 default:
1040 return UnableToLegalize;
1041 }
1042
1043 Observer.changingInstr(MI);
1044 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT);
1045 Observer.changedInstr(MI);
1046 return Legalized;
1047 case TargetOpcode::G_INTTOPTR:
1048 if (TypeIdx != 1)
1049 return UnableToLegalize;
1050
1051 Observer.changingInstr(MI);
1052 narrowScalarSrc(MI, NarrowTy, 1);
1053 Observer.changedInstr(MI);
1054 return Legalized;
1055 case TargetOpcode::G_PTRTOINT:
1056 if (TypeIdx != 0)
1057 return UnableToLegalize;
1058
1059 Observer.changingInstr(MI);
1060 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT);
1061 Observer.changedInstr(MI);
1062 return Legalized;
1063 case TargetOpcode::G_PHI: {
1064 unsigned NumParts = SizeOp0 / NarrowSize;
1065 SmallVector<Register, 2> DstRegs(NumParts);
1066 SmallVector<SmallVector<Register, 2>, 2> SrcRegs(MI.getNumOperands() / 2);
1067 Observer.changingInstr(MI);
1068 for (unsigned i = 1; i < MI.getNumOperands(); i += 2) {
1069 MachineBasicBlock &OpMBB = *MI.getOperand(i + 1).getMBB();
1070 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
1071 extractParts(MI.getOperand(i).getReg(), NarrowTy, NumParts,
1072 SrcRegs[i / 2]);
1073 }
1074 MachineBasicBlock &MBB = *MI.getParent();
1075 MIRBuilder.setInsertPt(MBB, MI);
1076 for (unsigned i = 0; i < NumParts; ++i) {
1077 DstRegs[i] = MRI.createGenericVirtualRegister(NarrowTy);
1078 MachineInstrBuilder MIB =
1079 MIRBuilder.buildInstr(TargetOpcode::G_PHI).addDef(DstRegs[i]);
1080 for (unsigned j = 1; j < MI.getNumOperands(); j += 2)
1081 MIB.addUse(SrcRegs[j / 2][i]).add(MI.getOperand(j + 1));
1082 }
1083 MIRBuilder.setInsertPt(MBB, MBB.getFirstNonPHI());
1084 MIRBuilder.buildMerge(MI.getOperand(0), DstRegs);
1085 Observer.changedInstr(MI);
1086 MI.eraseFromParent();
1087 return Legalized;
1088 }
1089 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
1090 case TargetOpcode::G_INSERT_VECTOR_ELT: {
1091 if (TypeIdx != 2)
1092 return UnableToLegalize;
1093
1094 int OpIdx = MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3;
1095 Observer.changingInstr(MI);
1096 narrowScalarSrc(MI, NarrowTy, OpIdx);
1097 Observer.changedInstr(MI);
1098 return Legalized;
1099 }
1100 case TargetOpcode::G_ICMP: {
1101 uint64_t SrcSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
1102 if (NarrowSize * 2 != SrcSize)
1103 return UnableToLegalize;
1104
1105 Observer.changingInstr(MI);
1106 Register LHSL = MRI.createGenericVirtualRegister(NarrowTy);
1107 Register LHSH = MRI.createGenericVirtualRegister(NarrowTy);
1108 MIRBuilder.buildUnmerge({LHSL, LHSH}, MI.getOperand(2));
1109
1110 Register RHSL = MRI.createGenericVirtualRegister(NarrowTy);
1111 Register RHSH = MRI.createGenericVirtualRegister(NarrowTy);
1112 MIRBuilder.buildUnmerge({RHSL, RHSH}, MI.getOperand(3));
1113
1114 CmpInst::Predicate Pred =
1115 static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
1116 LLT ResTy = MRI.getType(MI.getOperand(0).getReg());
1117
1118 if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE) {
1119 MachineInstrBuilder XorL = MIRBuilder.buildXor(NarrowTy, LHSL, RHSL);
1120 MachineInstrBuilder XorH = MIRBuilder.buildXor(NarrowTy, LHSH, RHSH);
1121 MachineInstrBuilder Or = MIRBuilder.buildOr(NarrowTy, XorL, XorH);
1122 MachineInstrBuilder Zero = MIRBuilder.buildConstant(NarrowTy, 0);
1123 MIRBuilder.buildICmp(Pred, MI.getOperand(0), Or, Zero);
1124 } else {
1125 MachineInstrBuilder CmpH = MIRBuilder.buildICmp(Pred, ResTy, LHSH, RHSH);
1126 MachineInstrBuilder CmpHEQ =
1127 MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, ResTy, LHSH, RHSH);
1128 MachineInstrBuilder CmpLU = MIRBuilder.buildICmp(
1129 ICmpInst::getUnsignedPredicate(Pred), ResTy, LHSL, RHSL);
1130 MIRBuilder.buildSelect(MI.getOperand(0), CmpHEQ, CmpLU, CmpH);
1131 }
1132 Observer.changedInstr(MI);
1133 MI.eraseFromParent();
1134 return Legalized;
1135 }
1136 case TargetOpcode::G_SEXT_INREG: {
1137 if (TypeIdx != 0)
1138 return UnableToLegalize;
1139
1140 int64_t SizeInBits = MI.getOperand(2).getImm();
1141
1142 // So long as the new type has more bits than the bits we're extending we
1143 // don't need to break it apart.
1144 if (NarrowTy.getScalarSizeInBits() >= SizeInBits) {
1145 Observer.changingInstr(MI);
1146 // We don't lose any non-extension bits by truncating the src and
1147 // sign-extending the dst.
1148 MachineOperand &MO1 = MI.getOperand(1);
1149 auto TruncMIB = MIRBuilder.buildTrunc(NarrowTy, MO1);
1150 MO1.setReg(TruncMIB.getReg(0));
1151
1152 MachineOperand &MO2 = MI.getOperand(0);
1153 Register DstExt = MRI.createGenericVirtualRegister(NarrowTy);
1154 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1155 MIRBuilder.buildSExt(MO2, DstExt);
1156 MO2.setReg(DstExt);
1157 Observer.changedInstr(MI);
1158 return Legalized;
1159 }
1160
1161 // Break it apart. Components below the extension point are unmodified. The
1162 // component containing the extension point becomes a narrower SEXT_INREG.
1163 // Components above it are ashr'd from the component containing the
1164 // extension point.
1165 if (SizeOp0 % NarrowSize != 0)
1166 return UnableToLegalize;
1167 int NumParts = SizeOp0 / NarrowSize;
1168
1169 // List the registers where the destination will be scattered.
1170 SmallVector<Register, 2> DstRegs;
1171 // List the registers where the source will be split.
1172 SmallVector<Register, 2> SrcRegs;
1173
1174 // Create all the temporary registers.
1175 for (int i = 0; i < NumParts; ++i) {
1176 Register SrcReg = MRI.createGenericVirtualRegister(NarrowTy);
1177
1178 SrcRegs.push_back(SrcReg);
1179 }
1180
1181 // Explode the big arguments into smaller chunks.
1182 MIRBuilder.buildUnmerge(SrcRegs, MI.getOperand(1));
1183
1184 Register AshrCstReg =
1185 MIRBuilder.buildConstant(NarrowTy, NarrowTy.getScalarSizeInBits() - 1)
1186 .getReg(0);
1187 Register FullExtensionReg = 0;
1188 Register PartialExtensionReg = 0;
1189
1190 // Do the operation on each small part.
1191 for (int i = 0; i < NumParts; ++i) {
1192 if ((i + 1) * NarrowTy.getScalarSizeInBits() < SizeInBits)
1193 DstRegs.push_back(SrcRegs[i]);
1194 else if (i * NarrowTy.getScalarSizeInBits() > SizeInBits) {
1195 assert(PartialExtensionReg &&((PartialExtensionReg && "Expected to visit partial extension before full"
) ? static_cast<void> (0) : __assert_fail ("PartialExtensionReg && \"Expected to visit partial extension before full\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1196, __PRETTY_FUNCTION__))
1196 "Expected to visit partial extension before full")((PartialExtensionReg && "Expected to visit partial extension before full"
) ? static_cast<void> (0) : __assert_fail ("PartialExtensionReg && \"Expected to visit partial extension before full\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1196, __PRETTY_FUNCTION__))
;
1197 if (FullExtensionReg) {
1198 DstRegs.push_back(FullExtensionReg);
1199 continue;
1200 }
1201 DstRegs.push_back(
1202 MIRBuilder.buildAShr(NarrowTy, PartialExtensionReg, AshrCstReg)
1203 .getReg(0));
1204 FullExtensionReg = DstRegs.back();
1205 } else {
1206 DstRegs.push_back(
1207 MIRBuilder
1208 .buildInstr(
1209 TargetOpcode::G_SEXT_INREG, {NarrowTy},
1210 {SrcRegs[i], SizeInBits % NarrowTy.getScalarSizeInBits()})
1211 .getReg(0));
1212 PartialExtensionReg = DstRegs.back();
1213 }
1214 }
1215
1216 // Gather the destination registers into the final destination.
1217 Register DstReg = MI.getOperand(0).getReg();
1218 MIRBuilder.buildMerge(DstReg, DstRegs);
1219 MI.eraseFromParent();
1220 return Legalized;
1221 }
1222 case TargetOpcode::G_BSWAP:
1223 case TargetOpcode::G_BITREVERSE: {
1224 if (SizeOp0 % NarrowSize != 0)
1225 return UnableToLegalize;
1226
1227 Observer.changingInstr(MI);
1228 SmallVector<Register, 2> SrcRegs, DstRegs;
1229 unsigned NumParts = SizeOp0 / NarrowSize;
1230 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
1231
1232 for (unsigned i = 0; i < NumParts; ++i) {
1233 auto DstPart = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy},
1234 {SrcRegs[NumParts - 1 - i]});
1235 DstRegs.push_back(DstPart.getReg(0));
1236 }
1237
1238 MIRBuilder.buildMerge(MI.getOperand(0), DstRegs);
1239
1240 Observer.changedInstr(MI);
1241 MI.eraseFromParent();
1242 return Legalized;
1243 }
1244 case TargetOpcode::G_PTR_ADD:
1245 case TargetOpcode::G_PTRMASK: {
1246 if (TypeIdx != 1)
1247 return UnableToLegalize;
1248 Observer.changingInstr(MI);
1249 narrowScalarSrc(MI, NarrowTy, 2);
1250 Observer.changedInstr(MI);
1251 return Legalized;
1252 }
1253 case TargetOpcode::G_FPTOUI: {
1254 if (TypeIdx != 0)
1255 return UnableToLegalize;
1256 Observer.changingInstr(MI);
1257 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT);
1258 Observer.changedInstr(MI);
1259 return Legalized;
1260 }
1261 case TargetOpcode::G_FPTOSI: {
1262 if (TypeIdx != 0)
1263 return UnableToLegalize;
1264 Observer.changingInstr(MI);
1265 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_SEXT);
1266 Observer.changedInstr(MI);
1267 return Legalized;
1268 }
1269 case TargetOpcode::G_FPEXT:
1270 if (TypeIdx != 0)
1271 return UnableToLegalize;
1272 Observer.changingInstr(MI);
1273 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_FPEXT);
1274 Observer.changedInstr(MI);
1275 return Legalized;
1276 }
1277}
1278
1279Register LegalizerHelper::coerceToScalar(Register Val) {
1280 LLT Ty = MRI.getType(Val);
1281 if (Ty.isScalar())
1282 return Val;
1283
1284 const DataLayout &DL = MIRBuilder.getDataLayout();
1285 LLT NewTy = LLT::scalar(Ty.getSizeInBits());
1286 if (Ty.isPointer()) {
1287 if (DL.isNonIntegralAddressSpace(Ty.getAddressSpace()))
1288 return Register();
1289 return MIRBuilder.buildPtrToInt(NewTy, Val).getReg(0);
1290 }
1291
1292 Register NewVal = Val;
1293
1294 assert(Ty.isVector())((Ty.isVector()) ? static_cast<void> (0) : __assert_fail
("Ty.isVector()", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1294, __PRETTY_FUNCTION__))
;
1295 LLT EltTy = Ty.getElementType();
1296 if (EltTy.isPointer())
1297 NewVal = MIRBuilder.buildPtrToInt(NewTy, NewVal).getReg(0);
1298 return MIRBuilder.buildBitcast(NewTy, NewVal).getReg(0);
1299}
1300
1301void LegalizerHelper::widenScalarSrc(MachineInstr &MI, LLT WideTy,
1302 unsigned OpIdx, unsigned ExtOpcode) {
1303 MachineOperand &MO = MI.getOperand(OpIdx);
1304 auto ExtB = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MO});
1305 MO.setReg(ExtB.getReg(0));
1306}
1307
1308void LegalizerHelper::narrowScalarSrc(MachineInstr &MI, LLT NarrowTy,
1309 unsigned OpIdx) {
1310 MachineOperand &MO = MI.getOperand(OpIdx);
1311 auto ExtB = MIRBuilder.buildTrunc(NarrowTy, MO);
1312 MO.setReg(ExtB.getReg(0));
1313}
1314
1315void LegalizerHelper::widenScalarDst(MachineInstr &MI, LLT WideTy,
1316 unsigned OpIdx, unsigned TruncOpcode) {
1317 MachineOperand &MO = MI.getOperand(OpIdx);
1318 Register DstExt = MRI.createGenericVirtualRegister(WideTy);
1319 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1320 MIRBuilder.buildInstr(TruncOpcode, {MO}, {DstExt});
1321 MO.setReg(DstExt);
1322}
1323
1324void LegalizerHelper::narrowScalarDst(MachineInstr &MI, LLT NarrowTy,
1325 unsigned OpIdx, unsigned ExtOpcode) {
1326 MachineOperand &MO = MI.getOperand(OpIdx);
1327 Register DstTrunc = MRI.createGenericVirtualRegister(NarrowTy);
1328 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1329 MIRBuilder.buildInstr(ExtOpcode, {MO}, {DstTrunc});
1330 MO.setReg(DstTrunc);
1331}
1332
1333void LegalizerHelper::moreElementsVectorDst(MachineInstr &MI, LLT WideTy,
1334 unsigned OpIdx) {
1335 MachineOperand &MO = MI.getOperand(OpIdx);
1336 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1337 MO.setReg(widenWithUnmerge(WideTy, MO.getReg()));
1338}
1339
1340void LegalizerHelper::moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy,
1341 unsigned OpIdx) {
1342 MachineOperand &MO = MI.getOperand(OpIdx);
1343
1344 LLT OldTy = MRI.getType(MO.getReg());
1345 unsigned OldElts = OldTy.getNumElements();
1346 unsigned NewElts = MoreTy.getNumElements();
1347
1348 unsigned NumParts = NewElts / OldElts;
1349
1350 // Use concat_vectors if the result is a multiple of the number of elements.
1351 if (NumParts * OldElts == NewElts) {
1352 SmallVector<Register, 8> Parts;
1353 Parts.push_back(MO.getReg());
1354
1355 Register ImpDef = MIRBuilder.buildUndef(OldTy).getReg(0);
1356 for (unsigned I = 1; I != NumParts; ++I)
1357 Parts.push_back(ImpDef);
1358
1359 auto Concat = MIRBuilder.buildConcatVectors(MoreTy, Parts);
1360 MO.setReg(Concat.getReg(0));
1361 return;
1362 }
1363
1364 Register MoreReg = MRI.createGenericVirtualRegister(MoreTy);
1365 Register ImpDef = MIRBuilder.buildUndef(MoreTy).getReg(0);
1366 MIRBuilder.buildInsert(MoreReg, ImpDef, MO.getReg(), 0);
1367 MO.setReg(MoreReg);
1368}
1369
1370void LegalizerHelper::bitcastSrc(MachineInstr &MI, LLT CastTy, unsigned OpIdx) {
1371 MachineOperand &Op = MI.getOperand(OpIdx);
1372 Op.setReg(MIRBuilder.buildBitcast(CastTy, Op).getReg(0));
1373}
1374
1375void LegalizerHelper::bitcastDst(MachineInstr &MI, LLT CastTy, unsigned OpIdx) {
1376 MachineOperand &MO = MI.getOperand(OpIdx);
1377 Register CastDst = MRI.createGenericVirtualRegister(CastTy);
1378 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1379 MIRBuilder.buildBitcast(MO, CastDst);
1380 MO.setReg(CastDst);
1381}
1382
1383LegalizerHelper::LegalizeResult
1384LegalizerHelper::widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx,
1385 LLT WideTy) {
1386 if (TypeIdx != 1)
1387 return UnableToLegalize;
1388
1389 Register DstReg = MI.getOperand(0).getReg();
1390 LLT DstTy = MRI.getType(DstReg);
1391 if (DstTy.isVector())
1392 return UnableToLegalize;
1393
1394 Register Src1 = MI.getOperand(1).getReg();
1395 LLT SrcTy = MRI.getType(Src1);
1396 const int DstSize = DstTy.getSizeInBits();
1397 const int SrcSize = SrcTy.getSizeInBits();
1398 const int WideSize = WideTy.getSizeInBits();
1399 const int NumMerge = (DstSize + WideSize - 1) / WideSize;
1400
1401 unsigned NumOps = MI.getNumOperands();
1402 unsigned NumSrc = MI.getNumOperands() - 1;
1403 unsigned PartSize = DstTy.getSizeInBits() / NumSrc;
1404
1405 if (WideSize >= DstSize) {
1406 // Directly pack the bits in the target type.
1407 Register ResultReg = MIRBuilder.buildZExt(WideTy, Src1).getReg(0);
1408
1409 for (unsigned I = 2; I != NumOps; ++I) {
1410 const unsigned Offset = (I - 1) * PartSize;
1411
1412 Register SrcReg = MI.getOperand(I).getReg();
1413 assert(MRI.getType(SrcReg) == LLT::scalar(PartSize))((MRI.getType(SrcReg) == LLT::scalar(PartSize)) ? static_cast
<void> (0) : __assert_fail ("MRI.getType(SrcReg) == LLT::scalar(PartSize)"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1413, __PRETTY_FUNCTION__))
;
1414
1415 auto ZextInput = MIRBuilder.buildZExt(WideTy, SrcReg);
1416
1417 Register NextResult = I + 1 == NumOps && WideTy == DstTy ? DstReg :
1418 MRI.createGenericVirtualRegister(WideTy);
1419
1420 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, Offset);
1421 auto Shl = MIRBuilder.buildShl(WideTy, ZextInput, ShiftAmt);
1422 MIRBuilder.buildOr(NextResult, ResultReg, Shl);
1423 ResultReg = NextResult;
1424 }
1425
1426 if (WideSize > DstSize)
1427 MIRBuilder.buildTrunc(DstReg, ResultReg);
1428 else if (DstTy.isPointer())
1429 MIRBuilder.buildIntToPtr(DstReg, ResultReg);
1430
1431 MI.eraseFromParent();
1432 return Legalized;
1433 }
1434
1435 // Unmerge the original values to the GCD type, and recombine to the next
1436 // multiple greater than the original type.
1437 //
1438 // %3:_(s12) = G_MERGE_VALUES %0:_(s4), %1:_(s4), %2:_(s4) -> s6
1439 // %4:_(s2), %5:_(s2) = G_UNMERGE_VALUES %0
1440 // %6:_(s2), %7:_(s2) = G_UNMERGE_VALUES %1
1441 // %8:_(s2), %9:_(s2) = G_UNMERGE_VALUES %2
1442 // %10:_(s6) = G_MERGE_VALUES %4, %5, %6
1443 // %11:_(s6) = G_MERGE_VALUES %7, %8, %9
1444 // %12:_(s12) = G_MERGE_VALUES %10, %11
1445 //
1446 // Padding with undef if necessary:
1447 //
1448 // %2:_(s8) = G_MERGE_VALUES %0:_(s4), %1:_(s4) -> s6
1449 // %3:_(s2), %4:_(s2) = G_UNMERGE_VALUES %0
1450 // %5:_(s2), %6:_(s2) = G_UNMERGE_VALUES %1
1451 // %7:_(s2) = G_IMPLICIT_DEF
1452 // %8:_(s6) = G_MERGE_VALUES %3, %4, %5
1453 // %9:_(s6) = G_MERGE_VALUES %6, %7, %7
1454 // %10:_(s12) = G_MERGE_VALUES %8, %9
1455
1456 const int GCD = greatestCommonDivisor(SrcSize, WideSize);
1457 LLT GCDTy = LLT::scalar(GCD);
1458
1459 SmallVector<Register, 8> Parts;
1460 SmallVector<Register, 8> NewMergeRegs;
1461 SmallVector<Register, 8> Unmerges;
1462 LLT WideDstTy = LLT::scalar(NumMerge * WideSize);
1463
1464 // Decompose the original operands if they don't evenly divide.
1465 for (int I = 1, E = MI.getNumOperands(); I != E; ++I) {
1466 Register SrcReg = MI.getOperand(I).getReg();
1467 if (GCD == SrcSize) {
1468 Unmerges.push_back(SrcReg);
1469 } else {
1470 auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg);
1471 for (int J = 0, JE = Unmerge->getNumOperands() - 1; J != JE; ++J)
1472 Unmerges.push_back(Unmerge.getReg(J));
1473 }
1474 }
1475
1476 // Pad with undef to the next size that is a multiple of the requested size.
1477 if (static_cast<int>(Unmerges.size()) != NumMerge * WideSize) {
1478 Register UndefReg = MIRBuilder.buildUndef(GCDTy).getReg(0);
1479 for (int I = Unmerges.size(); I != NumMerge * WideSize; ++I)
1480 Unmerges.push_back(UndefReg);
1481 }
1482
1483 const int PartsPerGCD = WideSize / GCD;
1484
1485 // Build merges of each piece.
1486 ArrayRef<Register> Slicer(Unmerges);
1487 for (int I = 0; I != NumMerge; ++I, Slicer = Slicer.drop_front(PartsPerGCD)) {
1488 auto Merge = MIRBuilder.buildMerge(WideTy, Slicer.take_front(PartsPerGCD));
1489 NewMergeRegs.push_back(Merge.getReg(0));
1490 }
1491
1492 // A truncate may be necessary if the requested type doesn't evenly divide the
1493 // original result type.
1494 if (DstTy.getSizeInBits() == WideDstTy.getSizeInBits()) {
1495 MIRBuilder.buildMerge(DstReg, NewMergeRegs);
1496 } else {
1497 auto FinalMerge = MIRBuilder.buildMerge(WideDstTy, NewMergeRegs);
1498 MIRBuilder.buildTrunc(DstReg, FinalMerge.getReg(0));
1499 }
1500
1501 MI.eraseFromParent();
1502 return Legalized;
1503}
1504
1505Register LegalizerHelper::widenWithUnmerge(LLT WideTy, Register OrigReg) {
1506 Register WideReg = MRI.createGenericVirtualRegister(WideTy);
1507 LLT OrigTy = MRI.getType(OrigReg);
1508 LLT LCMTy = getLCMType(WideTy, OrigTy);
1509
1510 const int NumMergeParts = LCMTy.getSizeInBits() / WideTy.getSizeInBits();
1511 const int NumUnmergeParts = LCMTy.getSizeInBits() / OrigTy.getSizeInBits();
1512
1513 Register UnmergeSrc = WideReg;
1514
1515 // Create a merge to the LCM type, padding with undef
1516 // %0:_(<3 x s32>) = G_FOO => <4 x s32>
1517 // =>
1518 // %1:_(<4 x s32>) = G_FOO
1519 // %2:_(<4 x s32>) = G_IMPLICIT_DEF
1520 // %3:_(<12 x s32>) = G_CONCAT_VECTORS %1, %2, %2
1521 // %0:_(<3 x s32>), %4:_, %5:_, %6:_ = G_UNMERGE_VALUES %3
1522 if (NumMergeParts > 1) {
1523 Register Undef = MIRBuilder.buildUndef(WideTy).getReg(0);
1524 SmallVector<Register, 8> MergeParts(NumMergeParts, Undef);
1525 MergeParts[0] = WideReg;
1526 UnmergeSrc = MIRBuilder.buildMerge(LCMTy, MergeParts).getReg(0);
1527 }
1528
1529 // Unmerge to the original register and pad with dead defs.
1530 SmallVector<Register, 8> UnmergeResults(NumUnmergeParts);
1531 UnmergeResults[0] = OrigReg;
1532 for (int I = 1; I != NumUnmergeParts; ++I)
1533 UnmergeResults[I] = MRI.createGenericVirtualRegister(OrigTy);
1534
1535 MIRBuilder.buildUnmerge(UnmergeResults, UnmergeSrc);
1536 return WideReg;
1537}
1538
1539LegalizerHelper::LegalizeResult
1540LegalizerHelper::widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx,
1541 LLT WideTy) {
1542 if (TypeIdx != 0)
1543 return UnableToLegalize;
1544
1545 int NumDst = MI.getNumOperands() - 1;
1546 Register SrcReg = MI.getOperand(NumDst).getReg();
1547 LLT SrcTy = MRI.getType(SrcReg);
1548 if (SrcTy.isVector())
1549 return UnableToLegalize;
1550
1551 Register Dst0Reg = MI.getOperand(0).getReg();
1552 LLT DstTy = MRI.getType(Dst0Reg);
1553 if (!DstTy.isScalar())
1554 return UnableToLegalize;
1555
1556 if (WideTy.getSizeInBits() >= SrcTy.getSizeInBits()) {
1557 if (SrcTy.isPointer()) {
1558 const DataLayout &DL = MIRBuilder.getDataLayout();
1559 if (DL.isNonIntegralAddressSpace(SrcTy.getAddressSpace())) {
1560 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << "Not casting non-integral address space integer\n"
; } } while (false)
1561 dbgs() << "Not casting non-integral address space integer\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << "Not casting non-integral address space integer\n"
; } } while (false)
;
1562 return UnableToLegalize;
1563 }
1564
1565 SrcTy = LLT::scalar(SrcTy.getSizeInBits());
1566 SrcReg = MIRBuilder.buildPtrToInt(SrcTy, SrcReg).getReg(0);
1567 }
1568
1569 // Widen SrcTy to WideTy. This does not affect the result, but since the
1570 // user requested this size, it is probably better handled than SrcTy and
1571 // should reduce the total number of legalization artifacts
1572 if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) {
1573 SrcTy = WideTy;
1574 SrcReg = MIRBuilder.buildAnyExt(WideTy, SrcReg).getReg(0);
1575 }
1576
1577 // Theres no unmerge type to target. Directly extract the bits from the
1578 // source type
1579 unsigned DstSize = DstTy.getSizeInBits();
1580
1581 MIRBuilder.buildTrunc(Dst0Reg, SrcReg);
1582 for (int I = 1; I != NumDst; ++I) {
1583 auto ShiftAmt = MIRBuilder.buildConstant(SrcTy, DstSize * I);
1584 auto Shr = MIRBuilder.buildLShr(SrcTy, SrcReg, ShiftAmt);
1585 MIRBuilder.buildTrunc(MI.getOperand(I), Shr);
1586 }
1587
1588 MI.eraseFromParent();
1589 return Legalized;
1590 }
1591
1592 // Extend the source to a wider type.
1593 LLT LCMTy = getLCMType(SrcTy, WideTy);
1594
1595 Register WideSrc = SrcReg;
1596 if (LCMTy.getSizeInBits() != SrcTy.getSizeInBits()) {
1597 // TODO: If this is an integral address space, cast to integer and anyext.
1598 if (SrcTy.isPointer()) {
1599 LLVM_DEBUG(dbgs() << "Widening pointer source types not implemented\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << "Widening pointer source types not implemented\n"
; } } while (false)
;
1600 return UnableToLegalize;
1601 }
1602
1603 WideSrc = MIRBuilder.buildAnyExt(LCMTy, WideSrc).getReg(0);
1604 }
1605
1606 auto Unmerge = MIRBuilder.buildUnmerge(WideTy, WideSrc);
1607
1608 // Create a sequence of unmerges to the original results. since we may have
1609 // widened the source, we will need to pad the results with dead defs to cover
1610 // the source register.
1611 // e.g. widen s16 to s32:
1612 // %1:_(s16), %2:_(s16), %3:_(s16) = G_UNMERGE_VALUES %0:_(s48)
1613 //
1614 // =>
1615 // %4:_(s64) = G_ANYEXT %0:_(s48)
1616 // %5:_(s32), %6:_(s32) = G_UNMERGE_VALUES %4 ; Requested unmerge
1617 // %1:_(s16), %2:_(s16) = G_UNMERGE_VALUES %5 ; unpack to original regs
1618 // %3:_(s16), dead %7 = G_UNMERGE_VALUES %6 ; original reg + extra dead def
1619
1620 const int NumUnmerge = Unmerge->getNumOperands() - 1;
1621 const int PartsPerUnmerge = WideTy.getSizeInBits() / DstTy.getSizeInBits();
1622
1623 for (int I = 0; I != NumUnmerge; ++I) {
1624 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES);
1625
1626 for (int J = 0; J != PartsPerUnmerge; ++J) {
1627 int Idx = I * PartsPerUnmerge + J;
1628 if (Idx < NumDst)
1629 MIB.addDef(MI.getOperand(Idx).getReg());
1630 else {
1631 // Create dead def for excess components.
1632 MIB.addDef(MRI.createGenericVirtualRegister(DstTy));
1633 }
1634 }
1635
1636 MIB.addUse(Unmerge.getReg(I));
1637 }
1638
1639 MI.eraseFromParent();
1640 return Legalized;
1641}
1642
1643LegalizerHelper::LegalizeResult
1644LegalizerHelper::widenScalarExtract(MachineInstr &MI, unsigned TypeIdx,
1645 LLT WideTy) {
1646 Register DstReg = MI.getOperand(0).getReg();
1647 Register SrcReg = MI.getOperand(1).getReg();
1648 LLT SrcTy = MRI.getType(SrcReg);
1649
1650 LLT DstTy = MRI.getType(DstReg);
1651 unsigned Offset = MI.getOperand(2).getImm();
1652
1653 if (TypeIdx == 0) {
1654 if (SrcTy.isVector() || DstTy.isVector())
1655 return UnableToLegalize;
1656
1657 SrcOp Src(SrcReg);
1658 if (SrcTy.isPointer()) {
1659 // Extracts from pointers can be handled only if they are really just
1660 // simple integers.
1661 const DataLayout &DL = MIRBuilder.getDataLayout();
1662 if (DL.isNonIntegralAddressSpace(SrcTy.getAddressSpace()))
1663 return UnableToLegalize;
1664
1665 LLT SrcAsIntTy = LLT::scalar(SrcTy.getSizeInBits());
1666 Src = MIRBuilder.buildPtrToInt(SrcAsIntTy, Src);
1667 SrcTy = SrcAsIntTy;
1668 }
1669
1670 if (DstTy.isPointer())
1671 return UnableToLegalize;
1672
1673 if (Offset == 0) {
1674 // Avoid a shift in the degenerate case.
1675 MIRBuilder.buildTrunc(DstReg,
1676 MIRBuilder.buildAnyExtOrTrunc(WideTy, Src));
1677 MI.eraseFromParent();
1678 return Legalized;
1679 }
1680
1681 // Do a shift in the source type.
1682 LLT ShiftTy = SrcTy;
1683 if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) {
1684 Src = MIRBuilder.buildAnyExt(WideTy, Src);
1685 ShiftTy = WideTy;
1686 }
1687
1688 auto LShr = MIRBuilder.buildLShr(
1689 ShiftTy, Src, MIRBuilder.buildConstant(ShiftTy, Offset));
1690 MIRBuilder.buildTrunc(DstReg, LShr);
1691 MI.eraseFromParent();
1692 return Legalized;
1693 }
1694
1695 if (SrcTy.isScalar()) {
1696 Observer.changingInstr(MI);
1697 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1698 Observer.changedInstr(MI);
1699 return Legalized;
1700 }
1701
1702 if (!SrcTy.isVector())
1703 return UnableToLegalize;
1704
1705 if (DstTy != SrcTy.getElementType())
1706 return UnableToLegalize;
1707
1708 if (Offset % SrcTy.getScalarSizeInBits() != 0)
1709 return UnableToLegalize;
1710
1711 Observer.changingInstr(MI);
1712 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1713
1714 MI.getOperand(2).setImm((WideTy.getSizeInBits() / SrcTy.getSizeInBits()) *
1715 Offset);
1716 widenScalarDst(MI, WideTy.getScalarType(), 0);
1717 Observer.changedInstr(MI);
1718 return Legalized;
1719}
1720
1721LegalizerHelper::LegalizeResult
1722LegalizerHelper::widenScalarInsert(MachineInstr &MI, unsigned TypeIdx,
1723 LLT WideTy) {
1724 if (TypeIdx != 0 || WideTy.isVector())
1725 return UnableToLegalize;
1726 Observer.changingInstr(MI);
1727 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1728 widenScalarDst(MI, WideTy);
1729 Observer.changedInstr(MI);
1730 return Legalized;
1731}
1732
1733LegalizerHelper::LegalizeResult
1734LegalizerHelper::widenScalarAddSubShlSat(MachineInstr &MI, unsigned TypeIdx,
1735 LLT WideTy) {
1736 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SADDSAT ||
1737 MI.getOpcode() == TargetOpcode::G_SSUBSAT ||
1738 MI.getOpcode() == TargetOpcode::G_SSHLSAT;
1739 bool IsShift = MI.getOpcode() == TargetOpcode::G_SSHLSAT ||
1740 MI.getOpcode() == TargetOpcode::G_USHLSAT;
1741 // We can convert this to:
1742 // 1. Any extend iN to iM
1743 // 2. SHL by M-N
1744 // 3. [US][ADD|SUB|SHL]SAT
1745 // 4. L/ASHR by M-N
1746 //
1747 // It may be more efficient to lower this to a min and a max operation in
1748 // the higher precision arithmetic if the promoted operation isn't legal,
1749 // but this decision is up to the target's lowering request.
1750 Register DstReg = MI.getOperand(0).getReg();
1751
1752 unsigned NewBits = WideTy.getScalarSizeInBits();
1753 unsigned SHLAmount = NewBits - MRI.getType(DstReg).getScalarSizeInBits();
1754
1755 // Shifts must zero-extend the RHS to preserve the unsigned quantity, and
1756 // must not left shift the RHS to preserve the shift amount.
1757 auto LHS = MIRBuilder.buildAnyExt(WideTy, MI.getOperand(1));
1758 auto RHS = IsShift ? MIRBuilder.buildZExt(WideTy, MI.getOperand(2))
1759 : MIRBuilder.buildAnyExt(WideTy, MI.getOperand(2));
1760 auto ShiftK = MIRBuilder.buildConstant(WideTy, SHLAmount);
1761 auto ShiftL = MIRBuilder.buildShl(WideTy, LHS, ShiftK);
1762 auto ShiftR = IsShift ? RHS : MIRBuilder.buildShl(WideTy, RHS, ShiftK);
1763
1764 auto WideInst = MIRBuilder.buildInstr(MI.getOpcode(), {WideTy},
1765 {ShiftL, ShiftR}, MI.getFlags());
1766
1767 // Use a shift that will preserve the number of sign bits when the trunc is
1768 // folded away.
1769 auto Result = IsSigned ? MIRBuilder.buildAShr(WideTy, WideInst, ShiftK)
1770 : MIRBuilder.buildLShr(WideTy, WideInst, ShiftK);
1771
1772 MIRBuilder.buildTrunc(DstReg, Result);
1773 MI.eraseFromParent();
1774 return Legalized;
1775}
1776
1777LegalizerHelper::LegalizeResult
1778LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
1779 switch (MI.getOpcode()) {
1780 default:
1781 return UnableToLegalize;
1782 case TargetOpcode::G_EXTRACT:
1783 return widenScalarExtract(MI, TypeIdx, WideTy);
1784 case TargetOpcode::G_INSERT:
1785 return widenScalarInsert(MI, TypeIdx, WideTy);
1786 case TargetOpcode::G_MERGE_VALUES:
1787 return widenScalarMergeValues(MI, TypeIdx, WideTy);
1788 case TargetOpcode::G_UNMERGE_VALUES:
1789 return widenScalarUnmergeValues(MI, TypeIdx, WideTy);
1790 case TargetOpcode::G_UADDO:
1791 case TargetOpcode::G_USUBO: {
1792 if (TypeIdx == 1)
1793 return UnableToLegalize; // TODO
1794 auto LHSZext = MIRBuilder.buildZExt(WideTy, MI.getOperand(2));
1795 auto RHSZext = MIRBuilder.buildZExt(WideTy, MI.getOperand(3));
1796 unsigned Opcode = MI.getOpcode() == TargetOpcode::G_UADDO
1797 ? TargetOpcode::G_ADD
1798 : TargetOpcode::G_SUB;
1799 // Do the arithmetic in the larger type.
1800 auto NewOp = MIRBuilder.buildInstr(Opcode, {WideTy}, {LHSZext, RHSZext});
1801 LLT OrigTy = MRI.getType(MI.getOperand(0).getReg());
1802 APInt Mask =
1803 APInt::getLowBitsSet(WideTy.getSizeInBits(), OrigTy.getSizeInBits());
1804 auto AndOp = MIRBuilder.buildAnd(
1805 WideTy, NewOp, MIRBuilder.buildConstant(WideTy, Mask));
1806 // There is no overflow if the AndOp is the same as NewOp.
1807 MIRBuilder.buildICmp(CmpInst::ICMP_NE, MI.getOperand(1), NewOp, AndOp);
1808 // Now trunc the NewOp to the original result.
1809 MIRBuilder.buildTrunc(MI.getOperand(0), NewOp);
1810 MI.eraseFromParent();
1811 return Legalized;
1812 }
1813 case TargetOpcode::G_SADDSAT:
1814 case TargetOpcode::G_SSUBSAT:
1815 case TargetOpcode::G_SSHLSAT:
1816 case TargetOpcode::G_UADDSAT:
1817 case TargetOpcode::G_USUBSAT:
1818 case TargetOpcode::G_USHLSAT:
1819 return widenScalarAddSubShlSat(MI, TypeIdx, WideTy);
1820 case TargetOpcode::G_CTTZ:
1821 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
1822 case TargetOpcode::G_CTLZ:
1823 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
1824 case TargetOpcode::G_CTPOP: {
1825 if (TypeIdx == 0) {
1826 Observer.changingInstr(MI);
1827 widenScalarDst(MI, WideTy, 0);
1828 Observer.changedInstr(MI);
1829 return Legalized;
1830 }
1831
1832 Register SrcReg = MI.getOperand(1).getReg();
1833
1834 // First ZEXT the input.
1835 auto MIBSrc = MIRBuilder.buildZExt(WideTy, SrcReg);
1836 LLT CurTy = MRI.getType(SrcReg);
1837 if (MI.getOpcode() == TargetOpcode::G_CTTZ) {
1838 // The count is the same in the larger type except if the original
1839 // value was zero. This can be handled by setting the bit just off
1840 // the top of the original type.
1841 auto TopBit =
1842 APInt::getOneBitSet(WideTy.getSizeInBits(), CurTy.getSizeInBits());
1843 MIBSrc = MIRBuilder.buildOr(
1844 WideTy, MIBSrc, MIRBuilder.buildConstant(WideTy, TopBit));
1845 }
1846
1847 // Perform the operation at the larger size.
1848 auto MIBNewOp = MIRBuilder.buildInstr(MI.getOpcode(), {WideTy}, {MIBSrc});
1849 // This is already the correct result for CTPOP and CTTZs
1850 if (MI.getOpcode() == TargetOpcode::G_CTLZ ||
1851 MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF) {
1852 // The correct result is NewOp - (Difference in widety and current ty).
1853 unsigned SizeDiff = WideTy.getSizeInBits() - CurTy.getSizeInBits();
1854 MIBNewOp = MIRBuilder.buildSub(
1855 WideTy, MIBNewOp, MIRBuilder.buildConstant(WideTy, SizeDiff));
1856 }
1857
1858 MIRBuilder.buildZExtOrTrunc(MI.getOperand(0), MIBNewOp);
1859 MI.eraseFromParent();
1860 return Legalized;
1861 }
1862 case TargetOpcode::G_BSWAP: {
1863 Observer.changingInstr(MI);
1864 Register DstReg = MI.getOperand(0).getReg();
1865
1866 Register ShrReg = MRI.createGenericVirtualRegister(WideTy);
1867 Register DstExt = MRI.createGenericVirtualRegister(WideTy);
1868 Register ShiftAmtReg = MRI.createGenericVirtualRegister(WideTy);
1869 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1870
1871 MI.getOperand(0).setReg(DstExt);
1872
1873 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1874
1875 LLT Ty = MRI.getType(DstReg);
1876 unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits();
1877 MIRBuilder.buildConstant(ShiftAmtReg, DiffBits);
1878 MIRBuilder.buildLShr(ShrReg, DstExt, ShiftAmtReg);
1879
1880 MIRBuilder.buildTrunc(DstReg, ShrReg);
1881 Observer.changedInstr(MI);
1882 return Legalized;
1883 }
1884 case TargetOpcode::G_BITREVERSE: {
1885 Observer.changingInstr(MI);
1886
1887 Register DstReg = MI.getOperand(0).getReg();
1888 LLT Ty = MRI.getType(DstReg);
1889 unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits();
1890
1891 Register DstExt = MRI.createGenericVirtualRegister(WideTy);
1892 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1893 MI.getOperand(0).setReg(DstExt);
1894 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1895
1896 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, DiffBits);
1897 auto Shift = MIRBuilder.buildLShr(WideTy, DstExt, ShiftAmt);
1898 MIRBuilder.buildTrunc(DstReg, Shift);
1899 Observer.changedInstr(MI);
1900 return Legalized;
1901 }
1902 case TargetOpcode::G_FREEZE:
1903 Observer.changingInstr(MI);
1904 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1905 widenScalarDst(MI, WideTy);
1906 Observer.changedInstr(MI);
1907 return Legalized;
1908
1909 case TargetOpcode::G_ADD:
1910 case TargetOpcode::G_AND:
1911 case TargetOpcode::G_MUL:
1912 case TargetOpcode::G_OR:
1913 case TargetOpcode::G_XOR:
1914 case TargetOpcode::G_SUB:
1915 // Perform operation at larger width (any extension is fines here, high bits
1916 // don't affect the result) and then truncate the result back to the
1917 // original type.
1918 Observer.changingInstr(MI);
1919 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1920 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
1921 widenScalarDst(MI, WideTy);
1922 Observer.changedInstr(MI);
1923 return Legalized;
1924
1925 case TargetOpcode::G_SHL:
1926 Observer.changingInstr(MI);
1927
1928 if (TypeIdx == 0) {
1929 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1930 widenScalarDst(MI, WideTy);
1931 } else {
1932 assert(TypeIdx == 1)((TypeIdx == 1) ? static_cast<void> (0) : __assert_fail
("TypeIdx == 1", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1932, __PRETTY_FUNCTION__))
;
1933 // The "number of bits to shift" operand must preserve its value as an
1934 // unsigned integer:
1935 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
1936 }
1937
1938 Observer.changedInstr(MI);
1939 return Legalized;
1940
1941 case TargetOpcode::G_SDIV:
1942 case TargetOpcode::G_SREM:
1943 case TargetOpcode::G_SMIN:
1944 case TargetOpcode::G_SMAX:
1945 Observer.changingInstr(MI);
1946 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
1947 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
1948 widenScalarDst(MI, WideTy);
1949 Observer.changedInstr(MI);
1950 return Legalized;
1951
1952 case TargetOpcode::G_ASHR:
1953 case TargetOpcode::G_LSHR:
1954 Observer.changingInstr(MI);
1955
1956 if (TypeIdx == 0) {
1957 unsigned CvtOp = MI.getOpcode() == TargetOpcode::G_ASHR ?
1958 TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
1959
1960 widenScalarSrc(MI, WideTy, 1, CvtOp);
1961 widenScalarDst(MI, WideTy);
1962 } else {
1963 assert(TypeIdx == 1)((TypeIdx == 1) ? static_cast<void> (0) : __assert_fail
("TypeIdx == 1", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1963, __PRETTY_FUNCTION__))
;
1964 // The "number of bits to shift" operand must preserve its value as an
1965 // unsigned integer:
1966 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
1967 }
1968
1969 Observer.changedInstr(MI);
1970 return Legalized;
1971 case TargetOpcode::G_UDIV:
1972 case TargetOpcode::G_UREM:
1973 case TargetOpcode::G_UMIN:
1974 case TargetOpcode::G_UMAX:
1975 Observer.changingInstr(MI);
1976 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
1977 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
1978 widenScalarDst(MI, WideTy);
1979 Observer.changedInstr(MI);
1980 return Legalized;
1981
1982 case TargetOpcode::G_SELECT:
1983 Observer.changingInstr(MI);
1984 if (TypeIdx == 0) {
1985 // Perform operation at larger width (any extension is fine here, high
1986 // bits don't affect the result) and then truncate the result back to the
1987 // original type.
1988 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
1989 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ANYEXT);
1990 widenScalarDst(MI, WideTy);
1991 } else {
1992 bool IsVec = MRI.getType(MI.getOperand(1).getReg()).isVector();
1993 // Explicit extension is required here since high bits affect the result.
1994 widenScalarSrc(MI, WideTy, 1, MIRBuilder.getBoolExtOp(IsVec, false));
1995 }
1996 Observer.changedInstr(MI);
1997 return Legalized;
1998
1999 case TargetOpcode::G_FPTOSI:
2000 case TargetOpcode::G_FPTOUI:
2001 Observer.changingInstr(MI);
2002
2003 if (TypeIdx == 0)
2004 widenScalarDst(MI, WideTy);
2005 else
2006 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT);
2007
2008 Observer.changedInstr(MI);
2009 return Legalized;
2010 case TargetOpcode::G_SITOFP:
2011 Observer.changingInstr(MI);
2012
2013 if (TypeIdx == 0)
2014 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
2015 else
2016 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
2017
2018 Observer.changedInstr(MI);
2019 return Legalized;
2020 case TargetOpcode::G_UITOFP:
2021 Observer.changingInstr(MI);
2022
2023 if (TypeIdx == 0)
2024 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
2025 else
2026 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
2027
2028 Observer.changedInstr(MI);
2029 return Legalized;
2030 case TargetOpcode::G_LOAD:
2031 case TargetOpcode::G_SEXTLOAD:
2032 case TargetOpcode::G_ZEXTLOAD:
2033 Observer.changingInstr(MI);
2034 widenScalarDst(MI, WideTy);
2035 Observer.changedInstr(MI);
2036 return Legalized;
2037
2038 case TargetOpcode::G_STORE: {
2039 if (TypeIdx != 0)
2040 return UnableToLegalize;
2041
2042 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
2043 if (!Ty.isScalar())
2044 return UnableToLegalize;
2045
2046 Observer.changingInstr(MI);
2047
2048 unsigned ExtType = Ty.getScalarSizeInBits() == 1 ?
2049 TargetOpcode::G_ZEXT : TargetOpcode::G_ANYEXT;
2050 widenScalarSrc(MI, WideTy, 0, ExtType);
2051
2052 Observer.changedInstr(MI);
2053 return Legalized;
2054 }
2055 case TargetOpcode::G_CONSTANT: {
2056 MachineOperand &SrcMO = MI.getOperand(1);
2057 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
2058 unsigned ExtOpc = LI.getExtOpcodeForWideningConstant(
2059 MRI.getType(MI.getOperand(0).getReg()));
2060 assert((ExtOpc == TargetOpcode::G_ZEXT || ExtOpc == TargetOpcode::G_SEXT ||(((ExtOpc == TargetOpcode::G_ZEXT || ExtOpc == TargetOpcode::
G_SEXT || ExtOpc == TargetOpcode::G_ANYEXT) && "Illegal Extend"
) ? static_cast<void> (0) : __assert_fail ("(ExtOpc == TargetOpcode::G_ZEXT || ExtOpc == TargetOpcode::G_SEXT || ExtOpc == TargetOpcode::G_ANYEXT) && \"Illegal Extend\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 2062, __PRETTY_FUNCTION__))
2061 ExtOpc == TargetOpcode::G_ANYEXT) &&(((ExtOpc == TargetOpcode::G_ZEXT || ExtOpc == TargetOpcode::
G_SEXT || ExtOpc == TargetOpcode::G_ANYEXT) && "Illegal Extend"
) ? static_cast<void> (0) : __assert_fail ("(ExtOpc == TargetOpcode::G_ZEXT || ExtOpc == TargetOpcode::G_SEXT || ExtOpc == TargetOpcode::G_ANYEXT) && \"Illegal Extend\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 2062, __PRETTY_FUNCTION__))
2062 "Illegal Extend")(((ExtOpc == TargetOpcode::G_ZEXT || ExtOpc == TargetOpcode::
G_SEXT || ExtOpc == TargetOpcode::G_ANYEXT) && "Illegal Extend"
) ? static_cast<void> (0) : __assert_fail ("(ExtOpc == TargetOpcode::G_ZEXT || ExtOpc == TargetOpcode::G_SEXT || ExtOpc == TargetOpcode::G_ANYEXT) && \"Illegal Extend\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 2062, __PRETTY_FUNCTION__))
;
2063 const APInt &SrcVal = SrcMO.getCImm()->getValue();
2064 const APInt &Val = (ExtOpc == TargetOpcode::G_SEXT)
2065 ? SrcVal.sext(WideTy.getSizeInBits())
2066 : SrcVal.zext(WideTy.getSizeInBits());
2067 Observer.changingInstr(MI);
2068 SrcMO.setCImm(ConstantInt::get(Ctx, Val));
2069
2070 widenScalarDst(MI, WideTy);
2071 Observer.changedInstr(MI);
2072 return Legalized;
2073 }
2074 case TargetOpcode::G_FCONSTANT: {
2075 MachineOperand &SrcMO = MI.getOperand(1);
2076 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
2077 APFloat Val = SrcMO.getFPImm()->getValueAPF();
2078 bool LosesInfo;
2079 switch (WideTy.getSizeInBits()) {
2080 case 32:
2081 Val.convert(APFloat::IEEEsingle(), APFloat::rmNearestTiesToEven,
2082 &LosesInfo);
2083 break;
2084 case 64:
2085 Val.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
2086 &LosesInfo);
2087 break;
2088 default:
2089 return UnableToLegalize;
2090 }
2091
2092 assert(!LosesInfo && "extend should always be lossless")((!LosesInfo && "extend should always be lossless") ?
static_cast<void> (0) : __assert_fail ("!LosesInfo && \"extend should always be lossless\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 2092, __PRETTY_FUNCTION__))
;
2093
2094 Observer.changingInstr(MI);
2095 SrcMO.setFPImm(ConstantFP::get(Ctx, Val));
2096
2097 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
2098 Observer.changedInstr(MI);
2099 return Legalized;
2100 }
2101 case TargetOpcode::G_IMPLICIT_DEF: {
2102 Observer.changingInstr(MI);
2103 widenScalarDst(MI, WideTy);
2104 Observer.changedInstr(MI);
2105 return Legalized;
2106 }
2107 case TargetOpcode::G_BRCOND:
2108 Observer.changingInstr(MI);
2109 widenScalarSrc(MI, WideTy, 0, MIRBuilder.getBoolExtOp(false, false));
2110 Observer.changedInstr(MI);
2111 return Legalized;
2112
2113 case TargetOpcode::G_FCMP:
2114 Observer.changingInstr(MI);
2115 if (TypeIdx == 0)
2116 widenScalarDst(MI, WideTy);
2117 else {
2118 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_FPEXT);
2119 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_FPEXT);
2120 }
2121 Observer.changedInstr(MI);
2122 return Legalized;
2123
2124 case TargetOpcode::G_ICMP:
2125 Observer.changingInstr(MI);
2126 if (TypeIdx == 0)
2127 widenScalarDst(MI, WideTy);
2128 else {
2129 unsigned ExtOpcode = CmpInst::isSigned(static_cast<CmpInst::Predicate>(
2130 MI.getOperand(1).getPredicate()))
2131 ? TargetOpcode::G_SEXT
2132 : TargetOpcode::G_ZEXT;
2133 widenScalarSrc(MI, WideTy, 2, ExtOpcode);
2134 widenScalarSrc(MI, WideTy, 3, ExtOpcode);
2135 }
2136 Observer.changedInstr(MI);
2137 return Legalized;
2138
2139 case TargetOpcode::G_PTR_ADD:
2140 assert(TypeIdx == 1 && "unable to legalize pointer of G_PTR_ADD")((TypeIdx == 1 && "unable to legalize pointer of G_PTR_ADD"
) ? static_cast<void> (0) : __assert_fail ("TypeIdx == 1 && \"unable to legalize pointer of G_PTR_ADD\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 2140, __PRETTY_FUNCTION__))
;
2141 Observer.changingInstr(MI);
2142 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
2143 Observer.changedInstr(MI);
2144 return Legalized;
2145
2146 case TargetOpcode::G_PHI: {
2147 assert(TypeIdx == 0 && "Expecting only Idx 0")((TypeIdx == 0 && "Expecting only Idx 0") ? static_cast
<void> (0) : __assert_fail ("TypeIdx == 0 && \"Expecting only Idx 0\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 2147, __PRETTY_FUNCTION__))
;
2148
2149 Observer.changingInstr(MI);
2150 for (unsigned I = 1; I < MI.getNumOperands(); I += 2) {
2151 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
2152 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
2153 widenScalarSrc(MI, WideTy, I, TargetOpcode::G_ANYEXT);
2154 }
2155
2156 MachineBasicBlock &MBB = *MI.getParent();
2157 MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI());
2158 widenScalarDst(MI, WideTy);
2159 Observer.changedInstr(MI);
2160 return Legalized;
2161 }
2162 case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
2163 if (TypeIdx == 0) {
2164 Register VecReg = MI.getOperand(1).getReg();
2165 LLT VecTy = MRI.getType(VecReg);
2166 Observer.changingInstr(MI);
2167
2168 widenScalarSrc(MI, LLT::vector(VecTy.getNumElements(),
2169 WideTy.getSizeInBits()),
2170 1, TargetOpcode::G_SEXT);
2171
2172 widenScalarDst(MI, WideTy, 0);
2173 Observer.changedInstr(MI);
2174 return Legalized;
2175 }
2176
2177 if (TypeIdx != 2)
2178 return UnableToLegalize;
2179 Observer.changingInstr(MI);
2180 // TODO: Probably should be zext
2181 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
2182 Observer.changedInstr(MI);
2183 return Legalized;
2184 }
2185 case TargetOpcode::G_INSERT_VECTOR_ELT: {
2186 if (TypeIdx == 1) {
2187 Observer.changingInstr(MI);
2188
2189 Register VecReg = MI.getOperand(1).getReg();
2190 LLT VecTy = MRI.getType(VecReg);
2191 LLT WideVecTy = LLT::vector(VecTy.getNumElements(), WideTy);
2192
2193 widenScalarSrc(MI, WideVecTy, 1, TargetOpcode::G_ANYEXT);
2194 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
2195 widenScalarDst(MI, WideVecTy, 0);
2196 Observer.changedInstr(MI);
2197 return Legalized;
2198 }
2199
2200 if (TypeIdx == 2) {
2201 Observer.changingInstr(MI);
2202 // TODO: Probably should be zext
2203 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_SEXT);
2204 Observer.changedInstr(MI);
2205 return Legalized;
2206 }
2207
2208 return UnableToLegalize;
2209 }
2210 case TargetOpcode::G_FADD:
2211 case TargetOpcode::G_FMUL:
2212 case TargetOpcode::G_FSUB:
2213 case TargetOpcode::G_FMA:
2214 case TargetOpcode::G_FMAD:
2215 case TargetOpcode::G_FNEG:
2216 case TargetOpcode::G_FABS:
2217 case TargetOpcode::G_FCANONICALIZE:
2218 case TargetOpcode::G_FMINNUM:
2219 case TargetOpcode::G_FMAXNUM:
2220 case TargetOpcode::G_FMINNUM_IEEE:
2221 case TargetOpcode::G_FMAXNUM_IEEE:
2222 case TargetOpcode::G_FMINIMUM:
2223 case TargetOpcode::G_FMAXIMUM:
2224 case TargetOpcode::G_FDIV:
2225 case TargetOpcode::G_FREM:
2226 case TargetOpcode::G_FCEIL:
2227 case TargetOpcode::G_FFLOOR:
2228 case TargetOpcode::G_FCOS:
2229 case TargetOpcode::G_FSIN:
2230 case TargetOpcode::G_FLOG10:
2231 case TargetOpcode::G_FLOG:
2232 case TargetOpcode::G_FLOG2:
2233 case TargetOpcode::G_FRINT:
2234 case TargetOpcode::G_FNEARBYINT:
2235 case TargetOpcode::G_FSQRT:
2236 case TargetOpcode::G_FEXP:
2237 case TargetOpcode::G_FEXP2:
2238 case TargetOpcode::G_FPOW:
2239 case TargetOpcode::G_INTRINSIC_TRUNC:
2240 case TargetOpcode::G_INTRINSIC_ROUND:
2241 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
2242 assert(TypeIdx == 0)((TypeIdx == 0) ? static_cast<void> (0) : __assert_fail
("TypeIdx == 0", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 2242, __PRETTY_FUNCTION__))
;
2243 Observer.changingInstr(MI);
2244
2245 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I)
2246 widenScalarSrc(MI, WideTy, I, TargetOpcode::G_FPEXT);
2247
2248 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
2249 Observer.changedInstr(MI);
2250 return Legalized;
2251 case TargetOpcode::G_FPOWI: {
2252 if (TypeIdx != 0)
2253 return UnableToLegalize;
2254 Observer.changingInstr(MI);
2255 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT);
2256 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
2257 Observer.changedInstr(MI);
2258 return Legalized;
2259 }
2260 case TargetOpcode::G_INTTOPTR:
2261 if (TypeIdx != 1)
2262 return UnableToLegalize;
2263
2264 Observer.changingInstr(MI);
2265 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
2266 Observer.changedInstr(MI);
2267 return Legalized;
2268 case TargetOpcode::G_PTRTOINT:
2269 if (TypeIdx != 0)
2270 return UnableToLegalize;
2271
2272 Observer.changingInstr(MI);
2273 widenScalarDst(MI, WideTy, 0);
2274 Observer.changedInstr(MI);
2275 return Legalized;
2276 case TargetOpcode::G_BUILD_VECTOR: {
2277 Observer.changingInstr(MI);
2278
2279 const LLT WideEltTy = TypeIdx == 1 ? WideTy : WideTy.getElementType();
2280 for (int I = 1, E = MI.getNumOperands(); I != E; ++I)
2281 widenScalarSrc(MI, WideEltTy, I, TargetOpcode::G_ANYEXT);
2282
2283 // Avoid changing the result vector type if the source element type was
2284 // requested.
2285 if (TypeIdx == 1) {
2286 MI.setDesc(MIRBuilder.getTII().get(TargetOpcode::G_BUILD_VECTOR_TRUNC));
2287 } else {
2288 widenScalarDst(MI, WideTy, 0);
2289 }
2290
2291 Observer.changedInstr(MI);
2292 return Legalized;
2293 }
2294 case TargetOpcode::G_SEXT_INREG:
2295 if (TypeIdx != 0)
2296 return UnableToLegalize;
2297
2298 Observer.changingInstr(MI);
2299 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
2300 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_TRUNC);
2301 Observer.changedInstr(MI);
2302 return Legalized;
2303 case TargetOpcode::G_PTRMASK: {
2304 if (TypeIdx != 1)
2305 return UnableToLegalize;
2306 Observer.changingInstr(MI);
2307 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
2308 Observer.changedInstr(MI);
2309 return Legalized;
2310 }
2311 }
2312}
2313
2314static void getUnmergePieces(SmallVectorImpl<Register> &Pieces,
2315 MachineIRBuilder &B, Register Src, LLT Ty) {
2316 auto Unmerge = B.buildUnmerge(Ty, Src);
2317 for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
2318 Pieces.push_back(Unmerge.getReg(I));
2319}
2320
2321LegalizerHelper::LegalizeResult
2322LegalizerHelper::lowerBitcast(MachineInstr &MI) {
2323 Register Dst = MI.getOperand(0).getReg();
2324 Register Src = MI.getOperand(1).getReg();
2325 LLT DstTy = MRI.getType(Dst);
2326 LLT SrcTy = MRI.getType(Src);
2327
2328 if (SrcTy.isVector()) {
2329 LLT SrcEltTy = SrcTy.getElementType();
2330 SmallVector<Register, 8> SrcRegs;
2331
2332 if (DstTy.isVector()) {
2333 int NumDstElt = DstTy.getNumElements();
2334 int NumSrcElt = SrcTy.getNumElements();
2335
2336 LLT DstEltTy = DstTy.getElementType();
2337 LLT DstCastTy = DstEltTy; // Intermediate bitcast result type
2338 LLT SrcPartTy = SrcEltTy; // Original unmerge result type.
2339
2340 // If there's an element size mismatch, insert intermediate casts to match
2341 // the result element type.
2342 if (NumSrcElt < NumDstElt) { // Source element type is larger.
2343 // %1:_(<4 x s8>) = G_BITCAST %0:_(<2 x s16>)
2344 //
2345 // =>
2346 //
2347 // %2:_(s16), %3:_(s16) = G_UNMERGE_VALUES %0
2348 // %3:_(<2 x s8>) = G_BITCAST %2
2349 // %4:_(<2 x s8>) = G_BITCAST %3
2350 // %1:_(<4 x s16>) = G_CONCAT_VECTORS %3, %4
2351 DstCastTy = LLT::vector(NumDstElt / NumSrcElt, DstEltTy);
2352 SrcPartTy = SrcEltTy;
2353 } else if (NumSrcElt > NumDstElt) { // Source element type is smaller.
2354 //
2355 // %1:_(<2 x s16>) = G_BITCAST %0:_(<4 x s8>)
2356 //
2357 // =>
2358 //
2359 // %2:_(<2 x s8>), %3:_(<2 x s8>) = G_UNMERGE_VALUES %0
2360 // %3:_(s16) = G_BITCAST %2
2361 // %4:_(s16) = G_BITCAST %3
2362 // %1:_(<2 x s16>) = G_BUILD_VECTOR %3, %4
2363 SrcPartTy = LLT::vector(NumSrcElt / NumDstElt, SrcEltTy);
2364 DstCastTy = DstEltTy;
2365 }
2366
2367 getUnmergePieces(SrcRegs, MIRBuilder, Src, SrcPartTy);
2368 for (Register &SrcReg : SrcRegs)
2369 SrcReg = MIRBuilder.buildBitcast(DstCastTy, SrcReg).getReg(0);
2370 } else
2371 getUnmergePieces(SrcRegs, MIRBuilder, Src, SrcEltTy);
2372
2373 MIRBuilder.buildMerge(Dst, SrcRegs);
2374 MI.eraseFromParent();
2375 return Legalized;
2376 }
2377
2378 if (DstTy.isVector()) {
2379 SmallVector<Register, 8> SrcRegs;
2380 getUnmergePieces(SrcRegs, MIRBuilder, Src, DstTy.getElementType());
2381 MIRBuilder.buildMerge(Dst, SrcRegs);
2382 MI.eraseFromParent();
2383 return Legalized;
2384 }
2385
2386 return UnableToLegalize;
2387}
2388
2389/// Figure out the bit offset into a register when coercing a vector index for
2390/// the wide element type. This is only for the case when promoting vector to
2391/// one with larger elements.
2392//
2393///
2394/// %offset_idx = G_AND %idx, ~(-1 << Log2(DstEltSize / SrcEltSize))
2395/// %offset_bits = G_SHL %offset_idx, Log2(SrcEltSize)
2396static Register getBitcastWiderVectorElementOffset(MachineIRBuilder &B,
2397 Register Idx,
2398 unsigned NewEltSize,
2399 unsigned OldEltSize) {
2400 const unsigned Log2EltRatio = Log2_32(NewEltSize / OldEltSize);
2401 LLT IdxTy = B.getMRI()->getType(Idx);
2402
2403 // Now figure out the amount we need to shift to get the target bits.
2404 auto OffsetMask = B.buildConstant(
2405 IdxTy, ~(APInt::getAllOnesValue(IdxTy.getSizeInBits()) << Log2EltRatio));
2406 auto OffsetIdx = B.buildAnd(IdxTy, Idx, OffsetMask);
2407 return B.buildShl(IdxTy, OffsetIdx,
2408 B.buildConstant(IdxTy, Log2_32(OldEltSize))).getReg(0);
2409}
2410
2411/// Perform a G_EXTRACT_VECTOR_ELT in a different sized vector element. If this
2412/// is casting to a vector with a smaller element size, perform multiple element
2413/// extracts and merge the results. If this is coercing to a vector with larger
2414/// elements, index the bitcasted vector and extract the target element with bit
2415/// operations. This is intended to force the indexing in the native register
2416/// size for architectures that can dynamically index the register file.
2417LegalizerHelper::LegalizeResult
2418LegalizerHelper::bitcastExtractVectorElt(MachineInstr &MI, unsigned TypeIdx,
2419 LLT CastTy) {
2420 if (TypeIdx != 1)
2421 return UnableToLegalize;
2422
2423 Register Dst = MI.getOperand(0).getReg();
2424 Register SrcVec = MI.getOperand(1).getReg();
2425 Register Idx = MI.getOperand(2).getReg();
2426 LLT SrcVecTy = MRI.getType(SrcVec);
2427 LLT IdxTy = MRI.getType(Idx);
2428
2429 LLT SrcEltTy = SrcVecTy.getElementType();
2430 unsigned NewNumElts = CastTy.isVector() ? CastTy.getNumElements() : 1;
2431 unsigned OldNumElts = SrcVecTy.getNumElements();
2432
2433 LLT NewEltTy = CastTy.isVector() ? CastTy.getElementType() : CastTy;
2434 Register CastVec = MIRBuilder.buildBitcast(CastTy, SrcVec).getReg(0);
2435
2436 const unsigned NewEltSize = NewEltTy.getSizeInBits();
2437 const unsigned OldEltSize = SrcEltTy.getSizeInBits();
2438 if (NewNumElts > OldNumElts) {
2439 // Decreasing the vector element size
2440 //
2441 // e.g. i64 = extract_vector_elt x:v2i64, y:i32
2442 // =>
2443 // v4i32:castx = bitcast x:v2i64
2444 //
2445 // i64 = bitcast
2446 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
2447 // (i32 (extract_vector_elt castx, (2 * y + 1)))
2448 //
2449 if (NewNumElts % OldNumElts != 0)
2450 return UnableToLegalize;
2451
2452 // Type of the intermediate result vector.
2453 const unsigned NewEltsPerOldElt = NewNumElts / OldNumElts;
2454 LLT MidTy = LLT::scalarOrVector(NewEltsPerOldElt, NewEltTy);
2455
2456 auto NewEltsPerOldEltK = MIRBuilder.buildConstant(IdxTy, NewEltsPerOldElt);
2457
2458 SmallVector<Register, 8> NewOps(NewEltsPerOldElt);
2459 auto NewBaseIdx = MIRBuilder.buildMul(IdxTy, Idx, NewEltsPerOldEltK);
2460
2461 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
2462 auto IdxOffset = MIRBuilder.buildConstant(IdxTy, I);
2463 auto TmpIdx = MIRBuilder.buildAdd(IdxTy, NewBaseIdx, IdxOffset);
2464 auto Elt = MIRBuilder.buildExtractVectorElement(NewEltTy, CastVec, TmpIdx);
2465 NewOps[I] = Elt.getReg(0);
2466 }
2467
2468 auto NewVec = MIRBuilder.buildBuildVector(MidTy, NewOps);
2469 MIRBuilder.buildBitcast(Dst, NewVec);
2470 MI.eraseFromParent();
2471 return Legalized;
2472 }
2473
2474 if (NewNumElts < OldNumElts) {
2475 if (NewEltSize % OldEltSize != 0)
2476 return UnableToLegalize;
2477
2478 // This only depends on powers of 2 because we use bit tricks to figure out
2479 // the bit offset we need to shift to get the target element. A general
2480 // expansion could emit division/multiply.
2481 if (!isPowerOf2_32(NewEltSize / OldEltSize))
2482 return UnableToLegalize;
2483
2484 // Increasing the vector element size.
2485 // %elt:_(small_elt) = G_EXTRACT_VECTOR_ELT %vec:_(<N x small_elt>), %idx
2486 //
2487 // =>
2488 //
2489 // %cast = G_BITCAST %vec
2490 // %scaled_idx = G_LSHR %idx, Log2(DstEltSize / SrcEltSize)
2491 // %wide_elt = G_EXTRACT_VECTOR_ELT %cast, %scaled_idx
2492 // %offset_idx = G_AND %idx, ~(-1 << Log2(DstEltSize / SrcEltSize))
2493 // %offset_bits = G_SHL %offset_idx, Log2(SrcEltSize)
2494 // %elt_bits = G_LSHR %wide_elt, %offset_bits
2495 // %elt = G_TRUNC %elt_bits
2496
2497 const unsigned Log2EltRatio = Log2_32(NewEltSize / OldEltSize);
2498 auto Log2Ratio = MIRBuilder.buildConstant(IdxTy, Log2EltRatio);
2499
2500 // Divide to get the index in the wider element type.
2501 auto ScaledIdx = MIRBuilder.buildLShr(IdxTy, Idx, Log2Ratio);
2502
2503 Register WideElt = CastVec;
2504 if (CastTy.isVector()) {
2505 WideElt = MIRBuilder.buildExtractVectorElement(NewEltTy, CastVec,
2506 ScaledIdx).getReg(0);
2507 }
2508
2509 // Compute the bit offset into the register of the target element.
2510 Register OffsetBits = getBitcastWiderVectorElementOffset(
2511 MIRBuilder, Idx, NewEltSize, OldEltSize);
2512
2513 // Shift the wide element to get the target element.
2514 auto ExtractedBits = MIRBuilder.buildLShr(NewEltTy, WideElt, OffsetBits);
2515 MIRBuilder.buildTrunc(Dst, ExtractedBits);
2516 MI.eraseFromParent();
2517 return Legalized;
2518 }
2519
2520 return UnableToLegalize;
2521}
2522
2523/// Emit code to insert \p InsertReg into \p TargetRet at \p OffsetBits in \p
2524/// TargetReg, while preserving other bits in \p TargetReg.
2525///
2526/// (InsertReg << Offset) | (TargetReg & ~(-1 >> InsertReg.size()) << Offset)
2527static Register buildBitFieldInsert(MachineIRBuilder &B,
2528 Register TargetReg, Register InsertReg,
2529 Register OffsetBits) {
2530 LLT TargetTy = B.getMRI()->getType(TargetReg);
2531 LLT InsertTy = B.getMRI()->getType(InsertReg);
2532 auto ZextVal = B.buildZExt(TargetTy, InsertReg);
2533 auto ShiftedInsertVal = B.buildShl(TargetTy, ZextVal, OffsetBits);
2534
2535 // Produce a bitmask of the value to insert
2536 auto EltMask = B.buildConstant(
2537 TargetTy, APInt::getLowBitsSet(TargetTy.getSizeInBits(),
2538 InsertTy.getSizeInBits()));
2539 // Shift it into position
2540 auto ShiftedMask = B.buildShl(TargetTy, EltMask, OffsetBits);
2541 auto InvShiftedMask = B.buildNot(TargetTy, ShiftedMask);
2542
2543 // Clear out the bits in the wide element
2544 auto MaskedOldElt = B.buildAnd(TargetTy, TargetReg, InvShiftedMask);
2545
2546 // The value to insert has all zeros already, so stick it into the masked
2547 // wide element.
2548 return B.buildOr(TargetTy, MaskedOldElt, ShiftedInsertVal).getReg(0);
2549}
2550
2551/// Perform a G_INSERT_VECTOR_ELT in a different sized vector element. If this
2552/// is increasing the element size, perform the indexing in the target element
2553/// type, and use bit operations to insert at the element position. This is
2554/// intended for architectures that can dynamically index the register file and
2555/// want to force indexing in the native register size.
2556LegalizerHelper::LegalizeResult
2557LegalizerHelper::bitcastInsertVectorElt(MachineInstr &MI, unsigned TypeIdx,
2558 LLT CastTy) {
2559 if (TypeIdx != 0)
2560 return UnableToLegalize;
2561
2562 Register Dst = MI.getOperand(0).getReg();
2563 Register SrcVec = MI.getOperand(1).getReg();
2564 Register Val = MI.getOperand(2).getReg();
2565 Register Idx = MI.getOperand(3).getReg();
2566
2567 LLT VecTy = MRI.getType(Dst);
2568 LLT IdxTy = MRI.getType(Idx);
2569
2570 LLT VecEltTy = VecTy.getElementType();
2571 LLT NewEltTy = CastTy.isVector() ? CastTy.getElementType() : CastTy;
2572 const unsigned NewEltSize = NewEltTy.getSizeInBits();
2573 const unsigned OldEltSize = VecEltTy.getSizeInBits();
2574
2575 unsigned NewNumElts = CastTy.isVector() ? CastTy.getNumElements() : 1;
2576 unsigned OldNumElts = VecTy.getNumElements();
2577
2578 Register CastVec = MIRBuilder.buildBitcast(CastTy, SrcVec).getReg(0);
2579 if (NewNumElts < OldNumElts) {
2580 if (NewEltSize % OldEltSize != 0)
2581 return UnableToLegalize;
2582
2583 // This only depends on powers of 2 because we use bit tricks to figure out
2584 // the bit offset we need to shift to get the target element. A general
2585 // expansion could emit division/multiply.
2586 if (!isPowerOf2_32(NewEltSize / OldEltSize))
2587 return UnableToLegalize;
2588
2589 const unsigned Log2EltRatio = Log2_32(NewEltSize / OldEltSize);
2590 auto Log2Ratio = MIRBuilder.buildConstant(IdxTy, Log2EltRatio);
2591
2592 // Divide to get the index in the wider element type.
2593 auto ScaledIdx = MIRBuilder.buildLShr(IdxTy, Idx, Log2Ratio);
2594
2595 Register ExtractedElt = CastVec;
2596 if (CastTy.isVector()) {
2597 ExtractedElt = MIRBuilder.buildExtractVectorElement(NewEltTy, CastVec,
2598 ScaledIdx).getReg(0);
2599 }
2600
2601 // Compute the bit offset into the register of the target element.
2602 Register OffsetBits = getBitcastWiderVectorElementOffset(
2603 MIRBuilder, Idx, NewEltSize, OldEltSize);
2604
2605 Register InsertedElt = buildBitFieldInsert(MIRBuilder, ExtractedElt,
2606 Val, OffsetBits);
2607 if (CastTy.isVector()) {
2608 InsertedElt = MIRBuilder.buildInsertVectorElement(
2609 CastTy, CastVec, InsertedElt, ScaledIdx).getReg(0);
2610 }
2611
2612 MIRBuilder.buildBitcast(Dst, InsertedElt);
2613 MI.eraseFromParent();
2614 return Legalized;
2615 }
2616
2617 return UnableToLegalize;
2618}
2619
2620LegalizerHelper::LegalizeResult
2621LegalizerHelper::lowerLoad(MachineInstr &MI) {
2622 // Lower to a memory-width G_LOAD and a G_SEXT/G_ZEXT/G_ANYEXT
2623 Register DstReg = MI.getOperand(0).getReg();
2624 Register PtrReg = MI.getOperand(1).getReg();
2625 LLT DstTy = MRI.getType(DstReg);
2626 auto &MMO = **MI.memoperands_begin();
2627
2628 if (DstTy.getSizeInBits() == MMO.getSizeInBits()) {
2629 if (MI.getOpcode() == TargetOpcode::G_LOAD) {
2630 // This load needs splitting into power of 2 sized loads.
2631 if (DstTy.isVector())
2632 return UnableToLegalize;
2633 if (isPowerOf2_32(DstTy.getSizeInBits()))
2634 return UnableToLegalize; // Don't know what we're being asked to do.
2635
2636 // Our strategy here is to generate anyextending loads for the smaller
2637 // types up to next power-2 result type, and then combine the two larger
2638 // result values together, before truncating back down to the non-pow-2
2639 // type.
2640 // E.g. v1 = i24 load =>
2641 // v2 = i32 zextload (2 byte)
2642 // v3 = i32 load (1 byte)
2643 // v4 = i32 shl v3, 16
2644 // v5 = i32 or v4, v2
2645 // v1 = i24 trunc v5
2646 // By doing this we generate the correct truncate which should get
2647 // combined away as an artifact with a matching extend.
2648 uint64_t LargeSplitSize = PowerOf2Floor(DstTy.getSizeInBits());
2649 uint64_t SmallSplitSize = DstTy.getSizeInBits() - LargeSplitSize;
2650
2651 MachineFunction &MF = MIRBuilder.getMF();
2652 MachineMemOperand *LargeMMO =
2653 MF.getMachineMemOperand(&MMO, 0, LargeSplitSize / 8);
2654 MachineMemOperand *SmallMMO = MF.getMachineMemOperand(
2655 &MMO, LargeSplitSize / 8, SmallSplitSize / 8);
2656
2657 LLT PtrTy = MRI.getType(PtrReg);
2658 unsigned AnyExtSize = NextPowerOf2(DstTy.getSizeInBits());
2659 LLT AnyExtTy = LLT::scalar(AnyExtSize);
2660 Register LargeLdReg = MRI.createGenericVirtualRegister(AnyExtTy);
2661 Register SmallLdReg = MRI.createGenericVirtualRegister(AnyExtTy);
2662 auto LargeLoad = MIRBuilder.buildLoadInstr(
2663 TargetOpcode::G_ZEXTLOAD, LargeLdReg, PtrReg, *LargeMMO);
2664
2665 auto OffsetCst = MIRBuilder.buildConstant(
2666 LLT::scalar(PtrTy.getSizeInBits()), LargeSplitSize / 8);
2667 Register PtrAddReg = MRI.createGenericVirtualRegister(PtrTy);
2668 auto SmallPtr =
2669 MIRBuilder.buildPtrAdd(PtrAddReg, PtrReg, OffsetCst.getReg(0));
2670 auto SmallLoad = MIRBuilder.buildLoad(SmallLdReg, SmallPtr.getReg(0),
2671 *SmallMMO);
2672
2673 auto ShiftAmt = MIRBuilder.buildConstant(AnyExtTy, LargeSplitSize);
2674 auto Shift = MIRBuilder.buildShl(AnyExtTy, SmallLoad, ShiftAmt);
2675 auto Or = MIRBuilder.buildOr(AnyExtTy, Shift, LargeLoad);
2676 MIRBuilder.buildTrunc(DstReg, {Or.getReg(0)});
2677 MI.eraseFromParent();
2678 return Legalized;
2679 }
2680
2681 MIRBuilder.buildLoad(DstReg, PtrReg, MMO);
2682 MI.eraseFromParent();
2683 return Legalized;
2684 }
2685
2686 if (DstTy.isScalar()) {
2687 Register TmpReg =
2688 MRI.createGenericVirtualRegister(LLT::scalar(MMO.getSizeInBits()));
2689 MIRBuilder.buildLoad(TmpReg, PtrReg, MMO);
2690 switch (MI.getOpcode()) {
2691 default:
2692 llvm_unreachable("Unexpected opcode")::llvm::llvm_unreachable_internal("Unexpected opcode", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 2692)
;
2693 case TargetOpcode::G_LOAD:
2694 MIRBuilder.buildAnyExtOrTrunc(DstReg, TmpReg);
2695 break;
2696 case TargetOpcode::G_SEXTLOAD:
2697 MIRBuilder.buildSExt(DstReg, TmpReg);
2698 break;
2699 case TargetOpcode::G_ZEXTLOAD:
2700 MIRBuilder.buildZExt(DstReg, TmpReg);
2701 break;
2702 }
2703
2704 MI.eraseFromParent();
2705 return Legalized;
2706 }
2707
2708 return UnableToLegalize;
2709}
2710
2711LegalizerHelper::LegalizeResult
2712LegalizerHelper::lowerStore(MachineInstr &MI) {
2713 // Lower a non-power of 2 store into multiple pow-2 stores.
2714 // E.g. split an i24 store into an i16 store + i8 store.
2715 // We do this by first extending the stored value to the next largest power
2716 // of 2 type, and then using truncating stores to store the components.
2717 // By doing this, likewise with G_LOAD, generate an extend that can be
2718 // artifact-combined away instead of leaving behind extracts.
2719 Register SrcReg = MI.getOperand(0).getReg();
2720 Register PtrReg = MI.getOperand(1).getReg();
2721 LLT SrcTy = MRI.getType(SrcReg);
2722 MachineMemOperand &MMO = **MI.memoperands_begin();
2723 if (SrcTy.getSizeInBits() != MMO.getSizeInBits())
2724 return UnableToLegalize;
2725 if (SrcTy.isVector())
2726 return UnableToLegalize;
2727 if (isPowerOf2_32(SrcTy.getSizeInBits()))
2728 return UnableToLegalize; // Don't know what we're being asked to do.
2729
2730 // Extend to the next pow-2.
2731 const LLT ExtendTy = LLT::scalar(NextPowerOf2(SrcTy.getSizeInBits()));
2732 auto ExtVal = MIRBuilder.buildAnyExt(ExtendTy, SrcReg);
2733
2734 // Obtain the smaller value by shifting away the larger value.
2735 uint64_t LargeSplitSize = PowerOf2Floor(SrcTy.getSizeInBits());
2736 uint64_t SmallSplitSize = SrcTy.getSizeInBits() - LargeSplitSize;
2737 auto ShiftAmt = MIRBuilder.buildConstant(ExtendTy, LargeSplitSize);
2738 auto SmallVal = MIRBuilder.buildLShr(ExtendTy, ExtVal, ShiftAmt);
2739
2740 // Generate the PtrAdd and truncating stores.
2741 LLT PtrTy = MRI.getType(PtrReg);
2742 auto OffsetCst = MIRBuilder.buildConstant(
2743 LLT::scalar(PtrTy.getSizeInBits()), LargeSplitSize / 8);
2744 Register PtrAddReg = MRI.createGenericVirtualRegister(PtrTy);
2745 auto SmallPtr =
2746 MIRBuilder.buildPtrAdd(PtrAddReg, PtrReg, OffsetCst.getReg(0));
2747
2748 MachineFunction &MF = MIRBuilder.getMF();
2749 MachineMemOperand *LargeMMO =
2750 MF.getMachineMemOperand(&MMO, 0, LargeSplitSize / 8);
2751 MachineMemOperand *SmallMMO =
2752 MF.getMachineMemOperand(&MMO, LargeSplitSize / 8, SmallSplitSize / 8);
2753 MIRBuilder.buildStore(ExtVal.getReg(0), PtrReg, *LargeMMO);
2754 MIRBuilder.buildStore(SmallVal.getReg(0), SmallPtr.getReg(0), *SmallMMO);
2755 MI.eraseFromParent();
2756 return Legalized;
2757}
2758
2759LegalizerHelper::LegalizeResult
2760LegalizerHelper::bitcast(MachineInstr &MI, unsigned TypeIdx, LLT CastTy) {
2761 switch (MI.getOpcode()) {
2762 case TargetOpcode::G_LOAD: {
2763 if (TypeIdx != 0)
2764 return UnableToLegalize;
2765
2766 Observer.changingInstr(MI);
2767 bitcastDst(MI, CastTy, 0);
2768 Observer.changedInstr(MI);
2769 return Legalized;
2770 }
2771 case TargetOpcode::G_STORE: {
2772 if (TypeIdx != 0)
2773 return UnableToLegalize;
2774
2775 Observer.changingInstr(MI);
2776 bitcastSrc(MI, CastTy, 0);
2777 Observer.changedInstr(MI);
2778 return Legalized;
2779 }
2780 case TargetOpcode::G_SELECT: {
2781 if (TypeIdx != 0)
2782 return UnableToLegalize;
2783
2784 if (MRI.getType(MI.getOperand(1).getReg()).isVector()) {
2785 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << "bitcast action not implemented for vector select\n"
; } } while (false)
2786 dbgs() << "bitcast action not implemented for vector select\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << "bitcast action not implemented for vector select\n"
; } } while (false)
;
2787 return UnableToLegalize;
2788 }
2789
2790 Observer.changingInstr(MI);
2791 bitcastSrc(MI, CastTy, 2);
2792 bitcastSrc(MI, CastTy, 3);
2793 bitcastDst(MI, CastTy, 0);
2794 Observer.changedInstr(MI);
2795 return Legalized;
2796 }
2797 case TargetOpcode::G_AND:
2798 case TargetOpcode::G_OR:
2799 case TargetOpcode::G_XOR: {
2800 Observer.changingInstr(MI);
2801 bitcastSrc(MI, CastTy, 1);
2802 bitcastSrc(MI, CastTy, 2);
2803 bitcastDst(MI, CastTy, 0);
2804 Observer.changedInstr(MI);
2805 return Legalized;
2806 }
2807 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
2808 return bitcastExtractVectorElt(MI, TypeIdx, CastTy);
2809 case TargetOpcode::G_INSERT_VECTOR_ELT:
2810 return bitcastInsertVectorElt(MI, TypeIdx, CastTy);
2811 default:
2812 return UnableToLegalize;
2813 }
2814}
2815
2816// Legalize an instruction by changing the opcode in place.
2817void LegalizerHelper::changeOpcode(MachineInstr &MI, unsigned NewOpcode) {
2818 Observer.changingInstr(MI);
2819 MI.setDesc(MIRBuilder.getTII().get(NewOpcode));
2820 Observer.changedInstr(MI);
2821}
2822
2823LegalizerHelper::LegalizeResult
2824LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) {
2825 using namespace TargetOpcode;
2826
2827 switch(MI.getOpcode()) {
2828 default:
2829 return UnableToLegalize;
2830 case TargetOpcode::G_BITCAST:
2831 return lowerBitcast(MI);
2832 case TargetOpcode::G_SREM:
2833 case TargetOpcode::G_UREM: {
2834 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
2835 auto Quot =
2836 MIRBuilder.buildInstr(MI.getOpcode() == G_SREM ? G_SDIV : G_UDIV, {Ty},
2837 {MI.getOperand(1), MI.getOperand(2)});
2838
2839 auto Prod = MIRBuilder.buildMul(Ty, Quot, MI.getOperand(2));
2840 MIRBuilder.buildSub(MI.getOperand(0), MI.getOperand(1), Prod);
2841 MI.eraseFromParent();
2842 return Legalized;
2843 }
2844 case TargetOpcode::G_SADDO:
2845 case TargetOpcode::G_SSUBO:
2846 return lowerSADDO_SSUBO(MI);
2847 case TargetOpcode::G_UMULH:
2848 case TargetOpcode::G_SMULH:
2849 return lowerSMULH_UMULH(MI);
2850 case TargetOpcode::G_SMULO:
2851 case TargetOpcode::G_UMULO: {
2852 // Generate G_UMULH/G_SMULH to check for overflow and a normal G_MUL for the
2853 // result.
2854 Register Res = MI.getOperand(0).getReg();
2855 Register Overflow = MI.getOperand(1).getReg();
2856 Register LHS = MI.getOperand(2).getReg();
2857 Register RHS = MI.getOperand(3).getReg();
2858 LLT Ty = MRI.getType(Res);
2859
2860 unsigned Opcode = MI.getOpcode() == TargetOpcode::G_SMULO
2861 ? TargetOpcode::G_SMULH
2862 : TargetOpcode::G_UMULH;
2863
2864 Observer.changingInstr(MI);
2865 const auto &TII = MIRBuilder.getTII();
2866 MI.setDesc(TII.get(TargetOpcode::G_MUL));
2867 MI.RemoveOperand(1);
2868 Observer.changedInstr(MI);
2869
2870 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
2871
2872 auto HiPart = MIRBuilder.buildInstr(Opcode, {Ty}, {LHS, RHS});
2873 auto Zero = MIRBuilder.buildConstant(Ty, 0);
2874
2875 // For *signed* multiply, overflow is detected by checking:
2876 // (hi != (lo >> bitwidth-1))
2877 if (Opcode == TargetOpcode::G_SMULH) {
2878 auto ShiftAmt = MIRBuilder.buildConstant(Ty, Ty.getSizeInBits() - 1);
2879 auto Shifted = MIRBuilder.buildAShr(Ty, Res, ShiftAmt);
2880 MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Shifted);
2881 } else {
2882 MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Zero);
2883 }
2884 return Legalized;
2885 }
2886 case TargetOpcode::G_FNEG: {
2887 Register Res = MI.getOperand(0).getReg();
2888 LLT Ty = MRI.getType(Res);
2889
2890 // TODO: Handle vector types once we are able to
2891 // represent them.
2892 if (Ty.isVector())
2893 return UnableToLegalize;
2894 auto SignMask =
2895 MIRBuilder.buildConstant(Ty, APInt::getSignMask(Ty.getSizeInBits()));
2896 Register SubByReg = MI.getOperand(1).getReg();
2897 MIRBuilder.buildXor(Res, SubByReg, SignMask);
2898 MI.eraseFromParent();
2899 return Legalized;
2900 }
2901 case TargetOpcode::G_FSUB: {
2902 Register Res = MI.getOperand(0).getReg();
2903 LLT Ty = MRI.getType(Res);
2904
2905 // Lower (G_FSUB LHS, RHS) to (G_FADD LHS, (G_FNEG RHS)).
2906 // First, check if G_FNEG is marked as Lower. If so, we may
2907 // end up with an infinite loop as G_FSUB is used to legalize G_FNEG.
2908 if (LI.getAction({G_FNEG, {Ty}}).Action == Lower)
2909 return UnableToLegalize;
2910 Register LHS = MI.getOperand(1).getReg();
2911 Register RHS = MI.getOperand(2).getReg();
2912 Register Neg = MRI.createGenericVirtualRegister(Ty);
2913 MIRBuilder.buildFNeg(Neg, RHS);
2914 MIRBuilder.buildFAdd(Res, LHS, Neg, MI.getFlags());
2915 MI.eraseFromParent();
2916 return Legalized;
2917 }
2918 case TargetOpcode::G_FMAD:
2919 return lowerFMad(MI);
2920 case TargetOpcode::G_FFLOOR:
2921 return lowerFFloor(MI);
2922 case TargetOpcode::G_INTRINSIC_ROUND:
2923 return lowerIntrinsicRound(MI);
2924 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: {
2925 // Since round even is the assumed rounding mode for unconstrained FP
2926 // operations, rint and roundeven are the same operation.
2927 changeOpcode(MI, TargetOpcode::G_FRINT);
2928 return Legalized;
2929 }
2930 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
2931 Register OldValRes = MI.getOperand(0).getReg();
2932 Register SuccessRes = MI.getOperand(1).getReg();
2933 Register Addr = MI.getOperand(2).getReg();
2934 Register CmpVal = MI.getOperand(3).getReg();
2935 Register NewVal = MI.getOperand(4).getReg();
2936 MIRBuilder.buildAtomicCmpXchg(OldValRes, Addr, CmpVal, NewVal,
2937 **MI.memoperands_begin());
2938 MIRBuilder.buildICmp(CmpInst::ICMP_EQ, SuccessRes, OldValRes, CmpVal);
2939 MI.eraseFromParent();
2940 return Legalized;
2941 }
2942 case TargetOpcode::G_LOAD:
2943 case TargetOpcode::G_SEXTLOAD:
2944 case TargetOpcode::G_ZEXTLOAD:
2945 return lowerLoad(MI);
2946 case TargetOpcode::G_STORE:
2947 return lowerStore(MI);
2948 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
2949 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
2950 case TargetOpcode::G_CTLZ:
2951 case TargetOpcode::G_CTTZ:
2952 case TargetOpcode::G_CTPOP:
2953 return lowerBitCount(MI);
2954 case G_UADDO: {
2955 Register Res = MI.getOperand(0).getReg();
2956 Register CarryOut = MI.getOperand(1).getReg();
2957 Register LHS = MI.getOperand(2).getReg();
2958 Register RHS = MI.getOperand(3).getReg();
2959
2960 MIRBuilder.buildAdd(Res, LHS, RHS);
2961 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, RHS);
2962
2963 MI.eraseFromParent();
2964 return Legalized;
2965 }
2966 case G_UADDE: {
2967 Register Res = MI.getOperand(0).getReg();
2968 Register CarryOut = MI.getOperand(1).getReg();
2969 Register LHS = MI.getOperand(2).getReg();
2970 Register RHS = MI.getOperand(3).getReg();
2971 Register CarryIn = MI.getOperand(4).getReg();
2972 LLT Ty = MRI.getType(Res);
2973
2974 auto TmpRes = MIRBuilder.buildAdd(Ty, LHS, RHS);
2975 auto ZExtCarryIn = MIRBuilder.buildZExt(Ty, CarryIn);
2976 MIRBuilder.buildAdd(Res, TmpRes, ZExtCarryIn);
2977 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, LHS);
2978
2979 MI.eraseFromParent();
2980 return Legalized;
2981 }
2982 case G_USUBO: {
2983 Register Res = MI.getOperand(0).getReg();
2984 Register BorrowOut = MI.getOperand(1).getReg();
2985 Register LHS = MI.getOperand(2).getReg();
2986 Register RHS = MI.getOperand(3).getReg();
2987
2988 MIRBuilder.buildSub(Res, LHS, RHS);
2989 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, BorrowOut, LHS, RHS);
2990
2991 MI.eraseFromParent();
2992 return Legalized;
2993 }
2994 case G_USUBE: {
2995 Register Res = MI.getOperand(0).getReg();
2996 Register BorrowOut = MI.getOperand(1).getReg();
2997 Register LHS = MI.getOperand(2).getReg();
2998 Register RHS = MI.getOperand(3).getReg();
2999 Register BorrowIn = MI.getOperand(4).getReg();
3000 const LLT CondTy = MRI.getType(BorrowOut);
3001 const LLT Ty = MRI.getType(Res);
3002
3003 auto TmpRes = MIRBuilder.buildSub(Ty, LHS, RHS);
3004 auto ZExtBorrowIn = MIRBuilder.buildZExt(Ty, BorrowIn);
3005 MIRBuilder.buildSub(Res, TmpRes, ZExtBorrowIn);
3006
3007 auto LHS_EQ_RHS = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, CondTy, LHS, RHS);
3008 auto LHS_ULT_RHS = MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CondTy, LHS, RHS);
3009 MIRBuilder.buildSelect(BorrowOut, LHS_EQ_RHS, BorrowIn, LHS_ULT_RHS);
3010
3011 MI.eraseFromParent();
3012 return Legalized;
3013 }
3014 case G_UITOFP:
3015 return lowerUITOFP(MI);
3016 case G_SITOFP:
3017 return lowerSITOFP(MI);
3018 case G_FPTOUI:
3019 return lowerFPTOUI(MI);
3020 case G_FPTOSI:
3021 return lowerFPTOSI(MI);
3022 case G_FPTRUNC:
3023 return lowerFPTRUNC(MI);
3024 case G_FPOWI:
3025 return lowerFPOWI(MI);
3026 case G_SMIN:
3027 case G_SMAX:
3028 case G_UMIN:
3029 case G_UMAX:
3030 return lowerMinMax(MI);
3031 case G_FCOPYSIGN:
3032 return lowerFCopySign(MI);
3033 case G_FMINNUM:
3034 case G_FMAXNUM:
3035 return lowerFMinNumMaxNum(MI);
3036 case G_MERGE_VALUES:
3037 return lowerMergeValues(MI);
3038 case G_UNMERGE_VALUES:
3039 return lowerUnmergeValues(MI);
3040 case TargetOpcode::G_SEXT_INREG: {
3041 assert(MI.getOperand(2).isImm() && "Expected immediate")((MI.getOperand(2).isImm() && "Expected immediate") ?
static_cast<void> (0) : __assert_fail ("MI.getOperand(2).isImm() && \"Expected immediate\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 3041, __PRETTY_FUNCTION__))
;
3042 int64_t SizeInBits = MI.getOperand(2).getImm();
3043
3044 Register DstReg = MI.getOperand(0).getReg();
3045 Register SrcReg = MI.getOperand(1).getReg();
3046 LLT DstTy = MRI.getType(DstReg);
3047 Register TmpRes = MRI.createGenericVirtualRegister(DstTy);
3048
3049 auto MIBSz = MIRBuilder.buildConstant(DstTy, DstTy.getScalarSizeInBits() - SizeInBits);
3050 MIRBuilder.buildShl(TmpRes, SrcReg, MIBSz->getOperand(0));
3051 MIRBuilder.buildAShr(DstReg, TmpRes, MIBSz->getOperand(0));
3052 MI.eraseFromParent();
3053 return Legalized;
3054 }
3055 case G_EXTRACT_VECTOR_ELT:
3056 case G_INSERT_VECTOR_ELT:
3057 return lowerExtractInsertVectorElt(MI);
3058 case G_SHUFFLE_VECTOR:
3059 return lowerShuffleVector(MI);
3060 case G_DYN_STACKALLOC:
3061 return lowerDynStackAlloc(MI);
3062 case G_EXTRACT:
3063 return lowerExtract(MI);
3064 case G_INSERT:
3065 return lowerInsert(MI);
3066 case G_BSWAP:
3067 return lowerBswap(MI);
3068 case G_BITREVERSE:
3069 return lowerBitreverse(MI);
3070 case G_READ_REGISTER:
3071 case G_WRITE_REGISTER:
3072 return lowerReadWriteRegister(MI);
3073 case G_UADDSAT:
3074 case G_USUBSAT: {
3075 // Try to make a reasonable guess about which lowering strategy to use. The
3076 // target can override this with custom lowering and calling the
3077 // implementation functions.
3078 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
3079 if (LI.isLegalOrCustom({G_UMIN, Ty}))
3080 return lowerAddSubSatToMinMax(MI);
3081 return lowerAddSubSatToAddoSubo(MI);
3082 }
3083 case G_SADDSAT:
3084 case G_SSUBSAT: {
3085 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
3086
3087 // FIXME: It would probably make more sense to see if G_SADDO is preferred,
3088 // since it's a shorter expansion. However, we would need to figure out the
3089 // preferred boolean type for the carry out for the query.
3090 if (LI.isLegalOrCustom({G_SMIN, Ty}) && LI.isLegalOrCustom({G_SMAX, Ty}))
3091 return lowerAddSubSatToMinMax(MI);
3092 return lowerAddSubSatToAddoSubo(MI);
3093 }
3094 case G_SSHLSAT:
3095 case G_USHLSAT:
3096 return lowerShlSat(MI);
3097 case G_ABS: {
3098 // Expand %res = G_ABS %a into:
3099 // %v1 = G_ASHR %a, scalar_size-1
3100 // %v2 = G_ADD %a, %v1
3101 // %res = G_XOR %v2, %v1
3102 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
3103 Register OpReg = MI.getOperand(1).getReg();
3104 auto ShiftAmt =
3105 MIRBuilder.buildConstant(DstTy, DstTy.getScalarSizeInBits() - 1);
3106 auto Shift =
3107 MIRBuilder.buildAShr(DstTy, OpReg, ShiftAmt);
3108 auto Add = MIRBuilder.buildAdd(DstTy, OpReg, Shift);
3109 MIRBuilder.buildXor(MI.getOperand(0).getReg(), Add, Shift);
3110 MI.eraseFromParent();
3111 return Legalized;
3112 }
3113 }
3114}
3115
3116Align LegalizerHelper::getStackTemporaryAlignment(LLT Ty,
3117 Align MinAlign) const {
3118 // FIXME: We're missing a way to go back from LLT to llvm::Type to query the
3119 // datalayout for the preferred alignment. Also there should be a target hook
3120 // for this to allow targets to reduce the alignment and ignore the
3121 // datalayout. e.g. AMDGPU should always use a 4-byte alignment, regardless of
3122 // the type.
3123 return std::max(Align(PowerOf2Ceil(Ty.getSizeInBytes())), MinAlign);
3124}
3125
3126MachineInstrBuilder
3127LegalizerHelper::createStackTemporary(TypeSize Bytes, Align Alignment,
3128 MachinePointerInfo &PtrInfo) {
3129 MachineFunction &MF = MIRBuilder.getMF();
3130 const DataLayout &DL = MIRBuilder.getDataLayout();
3131 int FrameIdx = MF.getFrameInfo().CreateStackObject(Bytes, Alignment, false);
3132
3133 unsigned AddrSpace = DL.getAllocaAddrSpace();
3134 LLT FramePtrTy = LLT::pointer(AddrSpace, DL.getPointerSizeInBits(AddrSpace));
3135
3136 PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIdx);
3137 return MIRBuilder.buildFrameIndex(FramePtrTy, FrameIdx);
3138}
3139
3140static Register clampDynamicVectorIndex(MachineIRBuilder &B, Register IdxReg,
3141 LLT VecTy) {
3142 int64_t IdxVal;
3143 if (mi_match(IdxReg, *B.getMRI(), m_ICst(IdxVal)))
3144 return IdxReg;
3145
3146 LLT IdxTy = B.getMRI()->getType(IdxReg);
3147 unsigned NElts = VecTy.getNumElements();
3148 if (isPowerOf2_32(NElts)) {
3149 APInt Imm = APInt::getLowBitsSet(IdxTy.getSizeInBits(), Log2_32(NElts));
3150 return B.buildAnd(IdxTy, IdxReg, B.buildConstant(IdxTy, Imm)).getReg(0);
3151 }
3152
3153 return B.buildUMin(IdxTy, IdxReg, B.buildConstant(IdxTy, NElts - 1))
3154 .getReg(0);
3155}
3156
3157Register LegalizerHelper::getVectorElementPointer(Register VecPtr, LLT VecTy,
3158 Register Index) {
3159 LLT EltTy = VecTy.getElementType();
3160
3161 // Calculate the element offset and add it to the pointer.
3162 unsigned EltSize = EltTy.getSizeInBits() / 8; // FIXME: should be ABI size.
3163 assert(EltSize * 8 == EltTy.getSizeInBits() &&((EltSize * 8 == EltTy.getSizeInBits() && "Converting bits to bytes lost precision"
) ? static_cast<void> (0) : __assert_fail ("EltSize * 8 == EltTy.getSizeInBits() && \"Converting bits to bytes lost precision\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 3164, __PRETTY_FUNCTION__))
3164 "Converting bits to bytes lost precision")((EltSize * 8 == EltTy.getSizeInBits() && "Converting bits to bytes lost precision"
) ? static_cast<void> (0) : __assert_fail ("EltSize * 8 == EltTy.getSizeInBits() && \"Converting bits to bytes lost precision\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 3164, __PRETTY_FUNCTION__))
;
3165
3166 Index = clampDynamicVectorIndex(MIRBuilder, Index, VecTy);
3167
3168 LLT IdxTy = MRI.getType(Index);
3169 auto Mul = MIRBuilder.buildMul(IdxTy, Index,
3170 MIRBuilder.buildConstant(IdxTy, EltSize));
3171
3172 LLT PtrTy = MRI.getType(VecPtr);
3173 return MIRBuilder.buildPtrAdd(PtrTy, VecPtr, Mul).getReg(0);
3174}
3175
3176LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorImplicitDef(
3177 MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) {
3178 Register DstReg = MI.getOperand(0).getReg();
3179 LLT DstTy = MRI.getType(DstReg);
3180 LLT LCMTy = getLCMType(DstTy, NarrowTy);
3181
3182 unsigned NumParts = LCMTy.getSizeInBits() / NarrowTy.getSizeInBits();
3183
3184 auto NewUndef = MIRBuilder.buildUndef(NarrowTy);
3185 SmallVector<Register, 8> Parts(NumParts, NewUndef.getReg(0));
3186
3187 buildWidenedRemergeToDst(DstReg, LCMTy, Parts);
3188 MI.eraseFromParent();
3189 return Legalized;
3190}
3191
3192// Handle splitting vector operations which need to have the same number of
3193// elements in each type index, but each type index may have a different element
3194// type.
3195//
3196// e.g. <4 x s64> = G_SHL <4 x s64>, <4 x s32> ->
3197// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
3198// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
3199//
3200// Also handles some irregular breakdown cases, e.g.
3201// e.g. <3 x s64> = G_SHL <3 x s64>, <3 x s32> ->
3202// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
3203// s64 = G_SHL s64, s32
3204LegalizerHelper::LegalizeResult
3205LegalizerHelper::fewerElementsVectorMultiEltType(
3206 MachineInstr &MI, unsigned TypeIdx, LLT NarrowTyArg) {
3207 if (TypeIdx != 0)
3208 return UnableToLegalize;
3209
3210 const LLT NarrowTy0 = NarrowTyArg;
3211 const unsigned NewNumElts =
3212 NarrowTy0.isVector() ? NarrowTy0.getNumElements() : 1;
3213
3214 const Register DstReg = MI.getOperand(0).getReg();
3215 LLT DstTy = MRI.getType(DstReg);
3216 LLT LeftoverTy0;
3217
3218 // All of the operands need to have the same number of elements, so if we can
3219 // determine a type breakdown for the result type, we can for all of the
3220 // source types.
3221 int NumParts = getNarrowTypeBreakDown(DstTy, NarrowTy0, LeftoverTy0).first;
3222 if (NumParts < 0)
3223 return UnableToLegalize;
3224
3225 SmallVector<MachineInstrBuilder, 4> NewInsts;
3226
3227 SmallVector<Register, 4> DstRegs, LeftoverDstRegs;
3228 SmallVector<Register, 4> PartRegs, LeftoverRegs;
3229
3230 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
3231 Register SrcReg = MI.getOperand(I).getReg();
3232 LLT SrcTyI = MRI.getType(SrcReg);
3233 LLT NarrowTyI = LLT::scalarOrVector(NewNumElts, SrcTyI.getScalarType());
3234 LLT LeftoverTyI;
3235
3236 // Split this operand into the requested typed registers, and any leftover
3237 // required to reproduce the original type.
3238 if (!extractParts(SrcReg, SrcTyI, NarrowTyI, LeftoverTyI, PartRegs,
3239 LeftoverRegs))
3240 return UnableToLegalize;
3241
3242 if (I == 1) {
3243 // For the first operand, create an instruction for each part and setup
3244 // the result.
3245 for (Register PartReg : PartRegs) {
3246 Register PartDstReg = MRI.createGenericVirtualRegister(NarrowTy0);
3247 NewInsts.push_back(MIRBuilder.buildInstrNoInsert(MI.getOpcode())
3248 .addDef(PartDstReg)
3249 .addUse(PartReg));
3250 DstRegs.push_back(PartDstReg);
3251 }
3252
3253 for (Register LeftoverReg : LeftoverRegs) {
3254 Register PartDstReg = MRI.createGenericVirtualRegister(LeftoverTy0);
3255 NewInsts.push_back(MIRBuilder.buildInstrNoInsert(MI.getOpcode())
3256 .addDef(PartDstReg)
3257 .addUse(LeftoverReg));
3258 LeftoverDstRegs.push_back(PartDstReg);
3259 }
3260 } else {
3261 assert(NewInsts.size() == PartRegs.size() + LeftoverRegs.size())((NewInsts.size() == PartRegs.size() + LeftoverRegs.size()) ?
static_cast<void> (0) : __assert_fail ("NewInsts.size() == PartRegs.size() + LeftoverRegs.size()"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 3261, __PRETTY_FUNCTION__))
;
3262
3263 // Add the newly created operand splits to the existing instructions. The
3264 // odd-sized pieces are ordered after the requested NarrowTyArg sized
3265 // pieces.
3266 unsigned InstCount = 0;
3267 for (unsigned J = 0, JE = PartRegs.size(); J != JE; ++J)
3268 NewInsts[InstCount++].addUse(PartRegs[J]);
3269 for (unsigned J = 0, JE = LeftoverRegs.size(); J != JE; ++J)
3270 NewInsts[InstCount++].addUse(LeftoverRegs[J]);
3271 }
3272
3273 PartRegs.clear();
3274 LeftoverRegs.clear();
3275 }
3276
3277 // Insert the newly built operations and rebuild the result register.
3278 for (auto &MIB : NewInsts)
3279 MIRBuilder.insertInstr(MIB);
3280
3281 insertParts(DstReg, DstTy, NarrowTy0, DstRegs, LeftoverTy0, LeftoverDstRegs);
3282
3283 MI.eraseFromParent();
3284 return Legalized;
3285}
3286
3287LegalizerHelper::LegalizeResult
3288LegalizerHelper::fewerElementsVectorCasts(MachineInstr &MI, unsigned TypeIdx,
3289 LLT NarrowTy) {
3290 if (TypeIdx != 0)
3291 return UnableToLegalize;
3292
3293 Register DstReg = MI.getOperand(0).getReg();
3294 Register SrcReg = MI.getOperand(1).getReg();
3295 LLT DstTy = MRI.getType(DstReg);
3296 LLT SrcTy = MRI.getType(SrcReg);
3297
3298 LLT NarrowTy0 = NarrowTy;
3299 LLT NarrowTy1;
3300 unsigned NumParts;
3301
3302 if (NarrowTy.isVector()) {
3303 // Uneven breakdown not handled.
3304 NumParts = DstTy.getNumElements() / NarrowTy.getNumElements();
3305 if (NumParts * NarrowTy.getNumElements() != DstTy.getNumElements())
3306 return UnableToLegalize;
3307
3308 NarrowTy1 = LLT::vector(NarrowTy.getNumElements(), SrcTy.getElementType());
3309 } else {
3310 NumParts = DstTy.getNumElements();
3311 NarrowTy1 = SrcTy.getElementType();
3312 }
3313
3314 SmallVector<Register, 4> SrcRegs, DstRegs;
3315 extractParts(SrcReg, NarrowTy1, NumParts, SrcRegs);
3316
3317 for (unsigned I = 0; I < NumParts; ++I) {
3318 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
3319 MachineInstr *NewInst =
3320 MIRBuilder.buildInstr(MI.getOpcode(), {DstReg}, {SrcRegs[I]});
3321
3322 NewInst->setFlags(MI.getFlags());
3323 DstRegs.push_back(DstReg);
3324 }
3325
3326 if (NarrowTy.isVector())
3327 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
3328 else
3329 MIRBuilder.buildBuildVector(DstReg, DstRegs);
3330
3331 MI.eraseFromParent();
3332 return Legalized;
3333}
3334
3335LegalizerHelper::LegalizeResult
3336LegalizerHelper::fewerElementsVectorCmp(MachineInstr &MI, unsigned TypeIdx,
3337 LLT NarrowTy) {
3338 Register DstReg = MI.getOperand(0).getReg();
3339 Register Src0Reg = MI.getOperand(2).getReg();
3340 LLT DstTy = MRI.getType(DstReg);
3341 LLT SrcTy = MRI.getType(Src0Reg);
3342
3343 unsigned NumParts;
3344 LLT NarrowTy0, NarrowTy1;
3345
3346 if (TypeIdx == 0) {
3347 unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
3348 unsigned OldElts = DstTy.getNumElements();
3349
3350 NarrowTy0 = NarrowTy;
3351 NumParts = NarrowTy.isVector() ? (OldElts / NewElts) : DstTy.getNumElements();
3352 NarrowTy1 = NarrowTy.isVector() ?
3353 LLT::vector(NarrowTy.getNumElements(), SrcTy.getScalarSizeInBits()) :
3354 SrcTy.getElementType();
3355
3356 } else {
3357 unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
3358 unsigned OldElts = SrcTy.getNumElements();
3359
3360 NumParts = NarrowTy.isVector() ? (OldElts / NewElts) :
3361 NarrowTy.getNumElements();
3362 NarrowTy0 = LLT::vector(NarrowTy.getNumElements(),
3363 DstTy.getScalarSizeInBits());
3364 NarrowTy1 = NarrowTy;
3365 }
3366
3367 // FIXME: Don't know how to handle the situation where the small vectors
3368 // aren't all the same size yet.
3369 if (NarrowTy1.isVector() &&
3370 NarrowTy1.getNumElements() * NumParts != DstTy.getNumElements())
3371 return UnableToLegalize;
3372
3373 CmpInst::Predicate Pred
3374 = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
3375
3376 SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs;
3377 extractParts(MI.getOperand(2).getReg(), NarrowTy1, NumParts, Src1Regs);
3378 extractParts(MI.getOperand(3).getReg(), NarrowTy1, NumParts, Src2Regs);
3379
3380 for (unsigned I = 0; I < NumParts; ++I) {
3381 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
3382 DstRegs.push_back(DstReg);
3383
3384 if (MI.getOpcode() == TargetOpcode::G_ICMP)
3385 MIRBuilder.buildICmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]);
3386 else {
3387 MachineInstr *NewCmp
3388 = MIRBuilder.buildFCmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]);
3389 NewCmp->setFlags(MI.getFlags());
3390 }
3391 }
3392
3393 if (NarrowTy1.isVector())
3394 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
3395 else
3396 MIRBuilder.buildBuildVector(DstReg, DstRegs);
3397
3398 MI.eraseFromParent();
3399 return Legalized;
3400}
3401
3402LegalizerHelper::LegalizeResult
3403LegalizerHelper::fewerElementsVectorSelect(MachineInstr &MI, unsigned TypeIdx,
3404 LLT NarrowTy) {
3405 Register DstReg = MI.getOperand(0).getReg();
3406 Register CondReg = MI.getOperand(1).getReg();
3407
3408 unsigned NumParts = 0;
3409 LLT NarrowTy0, NarrowTy1;
3410
3411 LLT DstTy = MRI.getType(DstReg);
3412 LLT CondTy = MRI.getType(CondReg);
3413 unsigned Size = DstTy.getSizeInBits();
3414
3415 assert(TypeIdx == 0 || CondTy.isVector())((TypeIdx == 0 || CondTy.isVector()) ? static_cast<void>
(0) : __assert_fail ("TypeIdx == 0 || CondTy.isVector()", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 3415, __PRETTY_FUNCTION__))
;
3416
3417 if (TypeIdx == 0) {
3418 NarrowTy0 = NarrowTy;
3419 NarrowTy1 = CondTy;
3420
3421 unsigned NarrowSize = NarrowTy0.getSizeInBits();
3422 // FIXME: Don't know how to handle the situation where the small vectors
3423 // aren't all the same size yet.
3424 if (Size % NarrowSize != 0)
3425 return UnableToLegalize;
3426
3427 NumParts = Size / NarrowSize;
3428
3429 // Need to break down the condition type
3430 if (CondTy.isVector()) {
3431 if (CondTy.getNumElements() == NumParts)
3432 NarrowTy1 = CondTy.getElementType();
3433 else
3434 NarrowTy1 = LLT::vector(CondTy.getNumElements() / NumParts,
3435 CondTy.getScalarSizeInBits());
3436 }
3437 } else {
3438 NumParts = CondTy.getNumElements();
3439 if (NarrowTy.isVector()) {
3440 // TODO: Handle uneven breakdown.
3441 if (NumParts * NarrowTy.getNumElements() != CondTy.getNumElements())
3442 return UnableToLegalize;
3443
3444 return UnableToLegalize;
3445 } else {
3446 NarrowTy0 = DstTy.getElementType();
3447 NarrowTy1 = NarrowTy;
3448 }
3449 }
3450
3451 SmallVector<Register, 2> DstRegs, Src0Regs, Src1Regs, Src2Regs;
3452 if (CondTy.isVector())
3453 extractParts(MI.getOperand(1).getReg(), NarrowTy1, NumParts, Src0Regs);
3454
3455 extractParts(MI.getOperand(2).getReg(), NarrowTy0, NumParts, Src1Regs);
3456 extractParts(MI.getOperand(3).getReg(), NarrowTy0, NumParts, Src2Regs);
3457
3458 for (unsigned i = 0; i < NumParts; ++i) {
3459 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
3460 MIRBuilder.buildSelect(DstReg, CondTy.isVector() ? Src0Regs[i] : CondReg,
3461 Src1Regs[i], Src2Regs[i]);
3462 DstRegs.push_back(DstReg);
3463 }
3464
3465 if (NarrowTy0.isVector())
3466 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
3467 else
3468 MIRBuilder.buildBuildVector(DstReg, DstRegs);
3469
3470 MI.eraseFromParent();
3471 return Legalized;
3472}
3473
3474LegalizerHelper::LegalizeResult
3475LegalizerHelper::fewerElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx,
3476 LLT NarrowTy) {
3477 const Register DstReg = MI.getOperand(0).getReg();
3478 LLT PhiTy = MRI.getType(DstReg);
3479 LLT LeftoverTy;
3480
3481 // All of the operands need to have the same number of elements, so if we can
3482 // determine a type breakdown for the result type, we can for all of the
3483 // source types.
3484 int NumParts, NumLeftover;
3485 std::tie(NumParts, NumLeftover)
3486 = getNarrowTypeBreakDown(PhiTy, NarrowTy, LeftoverTy);
3487 if (NumParts < 0)
3488 return UnableToLegalize;
3489
3490 SmallVector<Register, 4> DstRegs, LeftoverDstRegs;
3491 SmallVector<MachineInstrBuilder, 4> NewInsts;
3492
3493 const int TotalNumParts = NumParts + NumLeftover;
3494
3495 // Insert the new phis in the result block first.
3496 for (int I = 0; I != TotalNumParts; ++I) {
3497 LLT Ty = I < NumParts ? NarrowTy : LeftoverTy;
3498 Register PartDstReg = MRI.createGenericVirtualRegister(Ty);
3499 NewInsts.push_back(MIRBuilder.buildInstr(TargetOpcode::G_PHI)
3500 .addDef(PartDstReg));
3501 if (I < NumParts)
3502 DstRegs.push_back(PartDstReg);
3503 else
3504 LeftoverDstRegs.push_back(PartDstReg);
3505 }
3506
3507 MachineBasicBlock *MBB = MI.getParent();
3508 MIRBuilder.setInsertPt(*MBB, MBB->getFirstNonPHI());
3509 insertParts(DstReg, PhiTy, NarrowTy, DstRegs, LeftoverTy, LeftoverDstRegs);
3510
3511 SmallVector<Register, 4> PartRegs, LeftoverRegs;
3512
3513 // Insert code to extract the incoming values in each predecessor block.
3514 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
3515 PartRegs.clear();
3516 LeftoverRegs.clear();
3517
3518 Register SrcReg = MI.getOperand(I).getReg();
3519 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
3520 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
3521
3522 LLT Unused;
3523 if (!extractParts(SrcReg, PhiTy, NarrowTy, Unused, PartRegs,
3524 LeftoverRegs))
3525 return UnableToLegalize;
3526
3527 // Add the newly created operand splits to the existing instructions. The
3528 // odd-sized pieces are ordered after the requested NarrowTyArg sized
3529 // pieces.
3530 for (int J = 0; J != TotalNumParts; ++J) {
3531 MachineInstrBuilder MIB = NewInsts[J];
3532 MIB.addUse(J < NumParts ? PartRegs[J] : LeftoverRegs[J - NumParts]);
3533 MIB.addMBB(&OpMBB);
3534 }
3535 }
3536
3537 MI.eraseFromParent();
3538 return Legalized;
3539}
3540
3541LegalizerHelper::LegalizeResult
3542LegalizerHelper::fewerElementsVectorUnmergeValues(MachineInstr &MI,
3543 unsigned TypeIdx,
3544 LLT NarrowTy) {
3545 if (TypeIdx != 1)
3546 return UnableToLegalize;
3547
3548 const int NumDst = MI.getNumOperands() - 1;
3549 const Register SrcReg = MI.getOperand(NumDst).getReg();
3550 LLT SrcTy = MRI.getType(SrcReg);
3551
3552 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
3553
3554 // TODO: Create sequence of extracts.
3555 if (DstTy == NarrowTy)
3556 return UnableToLegalize;
3557
3558 LLT GCDTy = getGCDType(SrcTy, NarrowTy);
3559 if (DstTy == GCDTy) {
3560 // This would just be a copy of the same unmerge.
3561 // TODO: Create extracts, pad with undef and create intermediate merges.
3562 return UnableToLegalize;
3563 }
3564
3565 auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg);
3566 const int NumUnmerge = Unmerge->getNumOperands() - 1;
3567 const int PartsPerUnmerge = NumDst / NumUnmerge;
3568
3569 for (int I = 0; I != NumUnmerge; ++I) {
3570 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES);
3571
3572 for (int J = 0; J != PartsPerUnmerge; ++J)
3573 MIB.addDef(MI.getOperand(I * PartsPerUnmerge + J).getReg());
3574 MIB.addUse(Unmerge.getReg(I));
3575 }
3576
3577 MI.eraseFromParent();
3578 return Legalized;
3579}
3580
3581// Handle FewerElementsVector a G_BUILD_VECTOR or G_CONCAT_VECTORS that produces
3582// a vector
3583//
3584// Create a G_BUILD_VECTOR or G_CONCAT_VECTORS of NarrowTy pieces, padding with
3585// undef as necessary.
3586//
3587// %3:_(<3 x s16>) = G_BUILD_VECTOR %0, %1, %2
3588// -> <2 x s16>
3589//
3590// %4:_(s16) = G_IMPLICIT_DEF
3591// %5:_(<2 x s16>) = G_BUILD_VECTOR %0, %1
3592// %6:_(<2 x s16>) = G_BUILD_VECTOR %2, %4
3593// %7:_(<2 x s16>) = G_IMPLICIT_DEF
3594// %8:_(<6 x s16>) = G_CONCAT_VECTORS %5, %6, %7
3595// %3:_(<3 x s16>), %8:_(<3 x s16>) = G_UNMERGE_VALUES %8
3596LegalizerHelper::LegalizeResult
3597LegalizerHelper::fewerElementsVectorMerge(MachineInstr &MI, unsigned TypeIdx,
3598 LLT NarrowTy) {
3599 Register DstReg = MI.getOperand(0).getReg();
3600 LLT DstTy = MRI.getType(DstReg);
3601 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
3602 LLT GCDTy = getGCDType(getGCDType(SrcTy, NarrowTy), DstTy);
3603
3604 // Break into a common type
3605 SmallVector<Register, 16> Parts;
3606 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I)
3607 extractGCDType(Parts, GCDTy, MI.getOperand(I).getReg());
3608
3609 // Build the requested new merge, padding with undef.
3610 LLT LCMTy = buildLCMMergePieces(DstTy, NarrowTy, GCDTy, Parts,
3611 TargetOpcode::G_ANYEXT);
3612
3613 // Pack into the original result register.
3614 buildWidenedRemergeToDst(DstReg, LCMTy, Parts);
3615
3616 MI.eraseFromParent();
3617 return Legalized;
3618}
3619
3620LegalizerHelper::LegalizeResult
3621LegalizerHelper::fewerElementsVectorExtractInsertVectorElt(MachineInstr &MI,
3622 unsigned TypeIdx,
3623 LLT NarrowVecTy) {
3624 Register DstReg = MI.getOperand(0).getReg();
3625 Register SrcVec = MI.getOperand(1).getReg();
3626 Register InsertVal;
3627 bool IsInsert = MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT;
3628
3629 assert((IsInsert ? TypeIdx == 0 : TypeIdx == 1) && "not a vector type index")(((IsInsert ? TypeIdx == 0 : TypeIdx == 1) && "not a vector type index"
) ? static_cast<void> (0) : __assert_fail ("(IsInsert ? TypeIdx == 0 : TypeIdx == 1) && \"not a vector type index\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 3629, __PRETTY_FUNCTION__))
;
3630 if (IsInsert)
3631 InsertVal = MI.getOperand(2).getReg();
3632
3633 Register Idx = MI.getOperand(MI.getNumOperands() - 1).getReg();
3634
3635 // TODO: Handle total scalarization case.
3636 if (!NarrowVecTy.isVector())
3637 return UnableToLegalize;
3638
3639 LLT VecTy = MRI.getType(SrcVec);
3640
3641 // If the index is a constant, we can really break this down as you would
3642 // expect, and index into the target size pieces.
3643 int64_t IdxVal;
3644 if (mi_match(Idx, MRI, m_ICst(IdxVal))) {
3645 // Avoid out of bounds indexing the pieces.
3646 if (IdxVal >= VecTy.getNumElements()) {
3647 MIRBuilder.buildUndef(DstReg);
3648 MI.eraseFromParent();
3649 return Legalized;
3650 }
3651
3652 SmallVector<Register, 8> VecParts;
3653 LLT GCDTy = extractGCDType(VecParts, VecTy, NarrowVecTy, SrcVec);
3654
3655 // Build a sequence of NarrowTy pieces in VecParts for this operand.
3656 LLT LCMTy = buildLCMMergePieces(VecTy, NarrowVecTy, GCDTy, VecParts,
3657 TargetOpcode::G_ANYEXT);
3658
3659 unsigned NewNumElts = NarrowVecTy.getNumElements();
3660
3661 LLT IdxTy = MRI.getType(Idx);
3662 int64_t PartIdx = IdxVal / NewNumElts;
3663 auto NewIdx =
3664 MIRBuilder.buildConstant(IdxTy, IdxVal - NewNumElts * PartIdx);
3665
3666 if (IsInsert) {
3667 LLT PartTy = MRI.getType(VecParts[PartIdx]);
3668
3669 // Use the adjusted index to insert into one of the subvectors.
3670 auto InsertPart = MIRBuilder.buildInsertVectorElement(
3671 PartTy, VecParts[PartIdx], InsertVal, NewIdx);
3672 VecParts[PartIdx] = InsertPart.getReg(0);
3673
3674 // Recombine the inserted subvector with the others to reform the result
3675 // vector.
3676 buildWidenedRemergeToDst(DstReg, LCMTy, VecParts);
3677 } else {
3678 MIRBuilder.buildExtractVectorElement(DstReg, VecParts[PartIdx], NewIdx);
3679 }
3680
3681 MI.eraseFromParent();
3682 return Legalized;
3683 }
3684
3685 // With a variable index, we can't perform the operation in a smaller type, so
3686 // we're forced to expand this.
3687 //
3688 // TODO: We could emit a chain of compare/select to figure out which piece to
3689 // index.
3690 return lowerExtractInsertVectorElt(MI);
3691}
3692
3693LegalizerHelper::LegalizeResult
3694LegalizerHelper::reduceLoadStoreWidth(MachineInstr &MI, unsigned TypeIdx,
3695 LLT NarrowTy) {
3696 // FIXME: Don't know how to handle secondary types yet.
3697 if (TypeIdx != 0)
3698 return UnableToLegalize;
3699
3700 MachineMemOperand *MMO = *MI.memoperands_begin();
3701
3702 // This implementation doesn't work for atomics. Give up instead of doing
3703 // something invalid.
3704 if (MMO->getOrdering() != AtomicOrdering::NotAtomic ||
3705 MMO->getFailureOrdering() != AtomicOrdering::NotAtomic)
3706 return UnableToLegalize;
3707
3708 bool IsLoad = MI.getOpcode() == TargetOpcode::G_LOAD;
3709 Register ValReg = MI.getOperand(0).getReg();
3710 Register AddrReg = MI.getOperand(1).getReg();
3711 LLT ValTy = MRI.getType(ValReg);
3712
3713 // FIXME: Do we need a distinct NarrowMemory legalize action?
3714 if (ValTy.getSizeInBits() != 8 * MMO->getSize()) {
3715 LLVM_DEBUG(dbgs() << "Can't narrow extload/truncstore\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << "Can't narrow extload/truncstore\n"
; } } while (false)
;
3716 return UnableToLegalize;
3717 }
3718
3719 int NumParts = -1;
3720 int NumLeftover = -1;
3721 LLT LeftoverTy;
3722 SmallVector<Register, 8> NarrowRegs, NarrowLeftoverRegs;
3723 if (IsLoad) {
3724 std::tie(NumParts, NumLeftover) = getNarrowTypeBreakDown(ValTy, NarrowTy, LeftoverTy);
3725 } else {
3726 if (extractParts(ValReg, ValTy, NarrowTy, LeftoverTy, NarrowRegs,
3727 NarrowLeftoverRegs)) {
3728 NumParts = NarrowRegs.size();
3729 NumLeftover = NarrowLeftoverRegs.size();
Value stored to 'NumLeftover' is never read
3730 }
3731 }
3732
3733 if (NumParts == -1)
3734 return UnableToLegalize;
3735
3736 LLT PtrTy = MRI.getType(AddrReg);
3737 const LLT OffsetTy = LLT::scalar(PtrTy.getSizeInBits());
3738
3739 unsigned TotalSize = ValTy.getSizeInBits();
3740
3741 // Split the load/store into PartTy sized pieces starting at Offset. If this
3742 // is a load, return the new registers in ValRegs. For a store, each elements
3743 // of ValRegs should be PartTy. Returns the next offset that needs to be
3744 // handled.
3745 auto splitTypePieces = [=](LLT PartTy, SmallVectorImpl<Register> &ValRegs,
3746 unsigned Offset) -> unsigned {
3747 MachineFunction &MF = MIRBuilder.getMF();
3748 unsigned PartSize = PartTy.getSizeInBits();
3749 for (unsigned Idx = 0, E = NumParts; Idx != E && Offset < TotalSize;
3750 Offset += PartSize, ++Idx) {
3751 unsigned ByteSize = PartSize / 8;
3752 unsigned ByteOffset = Offset / 8;
3753 Register NewAddrReg;
3754
3755 MIRBuilder.materializePtrAdd(NewAddrReg, AddrReg, OffsetTy, ByteOffset);
3756
3757 MachineMemOperand *NewMMO =
3758 MF.getMachineMemOperand(MMO, ByteOffset, ByteSize);
3759
3760 if (IsLoad) {
3761 Register Dst = MRI.createGenericVirtualRegister(PartTy);
3762 ValRegs.push_back(Dst);
3763 MIRBuilder.buildLoad(Dst, NewAddrReg, *NewMMO);
3764 } else {
3765 MIRBuilder.buildStore(ValRegs[Idx], NewAddrReg, *NewMMO);
3766 }
3767 }
3768
3769 return Offset;
3770 };
3771
3772 unsigned HandledOffset = splitTypePieces(NarrowTy, NarrowRegs, 0);
3773
3774 // Handle the rest of the register if this isn't an even type breakdown.
3775 if (LeftoverTy.isValid())
3776 splitTypePieces(LeftoverTy, NarrowLeftoverRegs, HandledOffset);
3777
3778 if (IsLoad) {
3779 insertParts(ValReg, ValTy, NarrowTy, NarrowRegs,
3780 LeftoverTy, NarrowLeftoverRegs);
3781 }
3782
3783 MI.eraseFromParent();
3784 return Legalized;
3785}
3786
3787LegalizerHelper::LegalizeResult
3788LegalizerHelper::reduceOperationWidth(MachineInstr &MI, unsigned int TypeIdx,
3789 LLT NarrowTy) {
3790 assert(TypeIdx == 0 && "only one type index expected")((TypeIdx == 0 && "only one type index expected") ? static_cast
<void> (0) : __assert_fail ("TypeIdx == 0 && \"only one type index expected\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 3790, __PRETTY_FUNCTION__))
;
3791
3792 const unsigned Opc = MI.getOpcode();
3793 const int NumOps = MI.getNumOperands() - 1;
3794 const Register DstReg = MI.getOperand(0).getReg();
3795 const unsigned Flags = MI.getFlags();
3796 const unsigned NarrowSize = NarrowTy.getSizeInBits();
3797 const LLT NarrowScalarTy = LLT::scalar(NarrowSize);
3798
3799 assert(NumOps <= 3 && "expected instruction with 1 result and 1-3 sources")((NumOps <= 3 && "expected instruction with 1 result and 1-3 sources"
) ? static_cast<void> (0) : __assert_fail ("NumOps <= 3 && \"expected instruction with 1 result and 1-3 sources\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 3799, __PRETTY_FUNCTION__))
;
3800
3801 // First of all check whether we are narrowing (changing the element type)
3802 // or reducing the vector elements
3803 const LLT DstTy = MRI.getType(DstReg);
3804 const bool IsNarrow = NarrowTy.getScalarType() != DstTy.getScalarType();
3805
3806 SmallVector<Register, 8> ExtractedRegs[3];
3807 SmallVector<Register, 8> Parts;
3808
3809 unsigned NarrowElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
3810
3811 // Break down all the sources into NarrowTy pieces we can operate on. This may
3812 // involve creating merges to a wider type, padded with undef.
3813 for (int I = 0; I != NumOps; ++I) {
3814 Register SrcReg = MI.getOperand(I + 1).getReg();
3815 LLT SrcTy = MRI.getType(SrcReg);
3816
3817 // The type to narrow SrcReg to. For narrowing, this is a smaller scalar.
3818 // For fewerElements, this is a smaller vector with the same element type.
3819 LLT OpNarrowTy;
3820 if (IsNarrow) {
3821 OpNarrowTy = NarrowScalarTy;
3822
3823 // In case of narrowing, we need to cast vectors to scalars for this to
3824 // work properly
3825 // FIXME: Can we do without the bitcast here if we're narrowing?
3826 if (SrcTy.isVector()) {
3827 SrcTy = LLT::scalar(SrcTy.getSizeInBits());
3828 SrcReg = MIRBuilder.buildBitcast(SrcTy, SrcReg).getReg(0);
3829 }
3830 } else {
3831 OpNarrowTy = LLT::scalarOrVector(NarrowElts, SrcTy.getScalarType());
3832 }
3833
3834 LLT GCDTy = extractGCDType(ExtractedRegs[I], SrcTy, OpNarrowTy, SrcReg);
3835
3836 // Build a sequence of NarrowTy pieces in ExtractedRegs for this operand.
3837 buildLCMMergePieces(SrcTy, OpNarrowTy, GCDTy, ExtractedRegs[I],
3838 TargetOpcode::G_ANYEXT);
3839 }
3840
3841 SmallVector<Register, 8> ResultRegs;
3842
3843 // Input operands for each sub-instruction.
3844 SmallVector<SrcOp, 4> InputRegs(NumOps, Register());
3845
3846 int NumParts = ExtractedRegs[0].size();
3847 const unsigned DstSize = DstTy.getSizeInBits();
3848 const LLT DstScalarTy = LLT::scalar(DstSize);
3849
3850 // Narrowing needs to use scalar types
3851 LLT DstLCMTy, NarrowDstTy;
3852 if (IsNarrow) {
3853 DstLCMTy = getLCMType(DstScalarTy, NarrowScalarTy);
3854 NarrowDstTy = NarrowScalarTy;
3855 } else {
3856 DstLCMTy = getLCMType(DstTy, NarrowTy);
3857 NarrowDstTy = NarrowTy;
3858 }
3859
3860 // We widened the source registers to satisfy merge/unmerge size
3861 // constraints. We'll have some extra fully undef parts.
3862 const int NumRealParts = (DstSize + NarrowSize - 1) / NarrowSize;
3863
3864 for (int I = 0; I != NumRealParts; ++I) {
3865 // Emit this instruction on each of the split pieces.
3866 for (int J = 0; J != NumOps; ++J)
3867 InputRegs[J] = ExtractedRegs[J][I];
3868
3869 auto Inst = MIRBuilder.buildInstr(Opc, {NarrowDstTy}, InputRegs, Flags);
3870 ResultRegs.push_back(Inst.getReg(0));
3871 }
3872
3873 // Fill out the widened result with undef instead of creating instructions
3874 // with undef inputs.
3875 int NumUndefParts = NumParts - NumRealParts;
3876 if (NumUndefParts != 0)
3877 ResultRegs.append(NumUndefParts,
3878 MIRBuilder.buildUndef(NarrowDstTy).getReg(0));
3879
3880 // Extract the possibly padded result. Use a scratch register if we need to do
3881 // a final bitcast, otherwise use the original result register.
3882 Register MergeDstReg;
3883 if (IsNarrow && DstTy.isVector())
3884 MergeDstReg = MRI.createGenericVirtualRegister(DstScalarTy);
3885 else
3886 MergeDstReg = DstReg;
3887
3888 buildWidenedRemergeToDst(MergeDstReg, DstLCMTy, ResultRegs);
3889
3890 // Recast to vector if we narrowed a vector
3891 if (IsNarrow && DstTy.isVector())
3892 MIRBuilder.buildBitcast(DstReg, MergeDstReg);
3893
3894 MI.eraseFromParent();
3895 return Legalized;
3896}
3897
3898LegalizerHelper::LegalizeResult
3899LegalizerHelper::fewerElementsVectorSextInReg(MachineInstr &MI, unsigned TypeIdx,
3900 LLT NarrowTy) {
3901 Register DstReg = MI.getOperand(0).getReg();
3902 Register SrcReg = MI.getOperand(1).getReg();
3903 int64_t Imm = MI.getOperand(2).getImm();
3904
3905 LLT DstTy = MRI.getType(DstReg);
3906
3907 SmallVector<Register, 8> Parts;
3908 LLT GCDTy = extractGCDType(Parts, DstTy, NarrowTy, SrcReg);
3909 LLT LCMTy = buildLCMMergePieces(DstTy, NarrowTy, GCDTy, Parts);
3910
3911 for (Register &R : Parts)
3912 R = MIRBuilder.buildSExtInReg(NarrowTy, R, Imm).getReg(0);
3913
3914 buildWidenedRemergeToDst(DstReg, LCMTy, Parts);
3915
3916 MI.eraseFromParent();
3917 return Legalized;
3918}
3919
3920LegalizerHelper::LegalizeResult
3921LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
3922 LLT NarrowTy) {
3923 using namespace TargetOpcode;
3924
3925 switch (MI.getOpcode()) {
3926 case G_IMPLICIT_DEF:
3927 return fewerElementsVectorImplicitDef(MI, TypeIdx, NarrowTy);
3928 case G_TRUNC:
3929 case G_AND:
3930 case G_OR:
3931 case G_XOR:
3932 case G_ADD:
3933 case G_SUB:
3934 case G_MUL:
3935 case G_PTR_ADD:
3936 case G_SMULH:
3937 case G_UMULH:
3938 case G_FADD:
3939 case G_FMUL:
3940 case G_FSUB:
3941 case G_FNEG:
3942 case G_FABS:
3943 case G_FCANONICALIZE:
3944 case G_FDIV:
3945 case G_FREM:
3946 case G_FMA:
3947 case G_FMAD:
3948 case G_FPOW:
3949 case G_FEXP:
3950 case G_FEXP2:
3951 case G_FLOG:
3952 case G_FLOG2:
3953 case G_FLOG10:
3954 case G_FNEARBYINT:
3955 case G_FCEIL:
3956 case G_FFLOOR:
3957 case G_FRINT:
3958 case G_INTRINSIC_ROUND:
3959 case G_INTRINSIC_ROUNDEVEN:
3960 case G_INTRINSIC_TRUNC:
3961 case G_FCOS:
3962 case G_FSIN:
3963 case G_FSQRT:
3964 case G_BSWAP:
3965 case G_BITREVERSE:
3966 case G_SDIV:
3967 case G_UDIV:
3968 case G_SREM:
3969 case G_UREM:
3970 case G_SMIN:
3971 case G_SMAX:
3972 case G_UMIN:
3973 case G_UMAX:
3974 case G_FMINNUM:
3975 case G_FMAXNUM:
3976 case G_FMINNUM_IEEE:
3977 case G_FMAXNUM_IEEE:
3978 case G_FMINIMUM:
3979 case G_FMAXIMUM:
3980 case G_FSHL:
3981 case G_FSHR:
3982 case G_FREEZE:
3983 case G_SADDSAT:
3984 case G_SSUBSAT:
3985 case G_UADDSAT:
3986 case G_USUBSAT:
3987 return reduceOperationWidth(MI, TypeIdx, NarrowTy);
3988 case G_SHL:
3989 case G_LSHR:
3990 case G_ASHR:
3991 case G_SSHLSAT:
3992 case G_USHLSAT:
3993 case G_CTLZ:
3994 case G_CTLZ_ZERO_UNDEF:
3995 case G_CTTZ:
3996 case G_CTTZ_ZERO_UNDEF:
3997 case G_CTPOP:
3998 case G_FCOPYSIGN:
3999 return fewerElementsVectorMultiEltType(MI, TypeIdx, NarrowTy);
4000 case G_ZEXT:
4001 case G_SEXT:
4002 case G_ANYEXT:
4003 case G_FPEXT:
4004 case G_FPTRUNC:
4005 case G_SITOFP:
4006 case G_UITOFP:
4007 case G_FPTOSI:
4008 case G_FPTOUI:
4009 case G_INTTOPTR:
4010 case G_PTRTOINT:
4011 case G_ADDRSPACE_CAST:
4012 return fewerElementsVectorCasts(MI, TypeIdx, NarrowTy);
4013 case G_ICMP:
4014 case G_FCMP:
4015 return fewerElementsVectorCmp(MI, TypeIdx, NarrowTy);
4016 case G_SELECT:
4017 return fewerElementsVectorSelect(MI, TypeIdx, NarrowTy);
4018 case G_PHI:
4019 return fewerElementsVectorPhi(MI, TypeIdx, NarrowTy);
4020 case G_UNMERGE_VALUES:
4021 return fewerElementsVectorUnmergeValues(MI, TypeIdx, NarrowTy);
4022 case G_BUILD_VECTOR:
4023 assert(TypeIdx == 0 && "not a vector type index")((TypeIdx == 0 && "not a vector type index") ? static_cast
<void> (0) : __assert_fail ("TypeIdx == 0 && \"not a vector type index\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 4023, __PRETTY_FUNCTION__))
;
4024 return fewerElementsVectorMerge(MI, TypeIdx, NarrowTy);
4025 case G_CONCAT_VECTORS:
4026 if (TypeIdx != 1) // TODO: This probably does work as expected already.
4027 return UnableToLegalize;
4028 return fewerElementsVectorMerge(MI, TypeIdx, NarrowTy);
4029 case G_EXTRACT_VECTOR_ELT:
4030 case G_INSERT_VECTOR_ELT:
4031 return fewerElementsVectorExtractInsertVectorElt(MI, TypeIdx, NarrowTy);
4032 case G_LOAD:
4033 case G_STORE:
4034 return reduceLoadStoreWidth(MI, TypeIdx, NarrowTy);
4035 case G_SEXT_INREG:
4036 return fewerElementsVectorSextInReg(MI, TypeIdx, NarrowTy);
4037 default:
4038 return UnableToLegalize;
4039 }
4040}
4041
4042LegalizerHelper::LegalizeResult
4043LegalizerHelper::narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt,
4044 const LLT HalfTy, const LLT AmtTy) {
4045
4046 Register InL = MRI.createGenericVirtualRegister(HalfTy);
4047 Register InH = MRI.createGenericVirtualRegister(HalfTy);
4048 MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1));
4049
4050 if (Amt.isNullValue()) {
4051 MIRBuilder.buildMerge(MI.getOperand(0), {InL, InH});
4052 MI.eraseFromParent();
4053 return Legalized;
4054 }
4055
4056 LLT NVT = HalfTy;
4057 unsigned NVTBits = HalfTy.getSizeInBits();
4058 unsigned VTBits = 2 * NVTBits;
4059
4060 SrcOp Lo(Register(0)), Hi(Register(0));
4061 if (MI.getOpcode() == TargetOpcode::G_SHL) {
4062 if (Amt.ugt(VTBits)) {
4063 Lo = Hi = MIRBuilder.buildConstant(NVT, 0);
4064 } else if (Amt.ugt(NVTBits)) {
4065 Lo = MIRBuilder.buildConstant(NVT, 0);
4066 Hi = MIRBuilder.buildShl(NVT, InL,
4067 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
4068 } else if (Amt == NVTBits) {
4069 Lo = MIRBuilder.buildConstant(NVT, 0);
4070 Hi = InL;
4071 } else {
4072 Lo = MIRBuilder.buildShl(NVT, InL, MIRBuilder.buildConstant(AmtTy, Amt));
4073 auto OrLHS =
4074 MIRBuilder.buildShl(NVT, InH, MIRBuilder.buildConstant(AmtTy, Amt));
4075 auto OrRHS = MIRBuilder.buildLShr(
4076 NVT, InL, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
4077 Hi = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
4078 }
4079 } else if (MI.getOpcode() == TargetOpcode::G_LSHR) {
4080 if (Amt.ugt(VTBits)) {
4081 Lo = Hi = MIRBuilder.buildConstant(NVT, 0);
4082 } else if (Amt.ugt(NVTBits)) {
4083 Lo = MIRBuilder.buildLShr(NVT, InH,
4084 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
4085 Hi = MIRBuilder.buildConstant(NVT, 0);
4086 } else if (Amt == NVTBits) {
4087 Lo = InH;
4088 Hi = MIRBuilder.buildConstant(NVT, 0);
4089 } else {
4090 auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt);
4091
4092 auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst);
4093 auto OrRHS = MIRBuilder.buildShl(
4094 NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
4095
4096 Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
4097 Hi = MIRBuilder.buildLShr(NVT, InH, ShiftAmtConst);
4098 }
4099 } else {
4100 if (Amt.ugt(VTBits)) {
4101 Hi = Lo = MIRBuilder.buildAShr(
4102 NVT, InH, MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
4103 } else if (Amt.ugt(NVTBits)) {
4104 Lo = MIRBuilder.buildAShr(NVT, InH,
4105 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
4106 Hi = MIRBuilder.buildAShr(NVT, InH,
4107 MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
4108 } else if (Amt == NVTBits) {
4109 Lo = InH;
4110 Hi = MIRBuilder.buildAShr(NVT, InH,
4111 MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
4112 } else {
4113 auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt);
4114
4115 auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst);
4116 auto OrRHS = MIRBuilder.buildShl(
4117 NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
4118
4119 Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
4120 Hi = MIRBuilder.buildAShr(NVT, InH, ShiftAmtConst);
4121 }
4122 }
4123
4124 MIRBuilder.buildMerge(MI.getOperand(0), {Lo, Hi});
4125 MI.eraseFromParent();
4126
4127 return Legalized;
4128}
4129
4130// TODO: Optimize if constant shift amount.
4131LegalizerHelper::LegalizeResult
4132LegalizerHelper::narrowScalarShift(MachineInstr &MI, unsigned TypeIdx,
4133 LLT RequestedTy) {
4134 if (TypeIdx == 1) {
4135 Observer.changingInstr(MI);
4136 narrowScalarSrc(MI, RequestedTy, 2);
4137 Observer.changedInstr(MI);
4138 return Legalized;
4139 }
4140
4141 Register DstReg = MI.getOperand(0).getReg();
4142 LLT DstTy = MRI.getType(DstReg);
4143 if (DstTy.isVector())
4144 return UnableToLegalize;
4145
4146 Register Amt = MI.getOperand(2).getReg();
4147 LLT ShiftAmtTy = MRI.getType(Amt);
4148 const unsigned DstEltSize = DstTy.getScalarSizeInBits();
4149 if (DstEltSize % 2 != 0)
4150 return UnableToLegalize;
4151
4152 // Ignore the input type. We can only go to exactly half the size of the
4153 // input. If that isn't small enough, the resulting pieces will be further
4154 // legalized.
4155 const unsigned NewBitSize = DstEltSize / 2;
4156 const LLT HalfTy = LLT::scalar(NewBitSize);
4157 const LLT CondTy = LLT::scalar(1);
4158
4159 if (const MachineInstr *KShiftAmt =
4160 getOpcodeDef(TargetOpcode::G_CONSTANT, Amt, MRI)) {
4161 return narrowScalarShiftByConstant(
4162 MI, KShiftAmt->getOperand(1).getCImm()->getValue(), HalfTy, ShiftAmtTy);
4163 }
4164
4165 // TODO: Expand with known bits.
4166
4167 // Handle the fully general expansion by an unknown amount.
4168 auto NewBits = MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize);
4169
4170 Register InL = MRI.createGenericVirtualRegister(HalfTy);
4171 Register InH = MRI.createGenericVirtualRegister(HalfTy);
4172 MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1));
4173
4174 auto AmtExcess = MIRBuilder.buildSub(ShiftAmtTy, Amt, NewBits);
4175 auto AmtLack = MIRBuilder.buildSub(ShiftAmtTy, NewBits, Amt);
4176
4177 auto Zero = MIRBuilder.buildConstant(ShiftAmtTy, 0);
4178 auto IsShort = MIRBuilder.buildICmp(ICmpInst::ICMP_ULT, CondTy, Amt, NewBits);
4179 auto IsZero = MIRBuilder.buildICmp(ICmpInst::ICMP_EQ, CondTy, Amt, Zero);
4180
4181 Register ResultRegs[2];
4182 switch (MI.getOpcode()) {
4183 case TargetOpcode::G_SHL: {
4184 // Short: ShAmt < NewBitSize
4185 auto LoS = MIRBuilder.buildShl(HalfTy, InL, Amt);
4186
4187 auto LoOr = MIRBuilder.buildLShr(HalfTy, InL, AmtLack);
4188 auto HiOr = MIRBuilder.buildShl(HalfTy, InH, Amt);
4189 auto HiS = MIRBuilder.buildOr(HalfTy, LoOr, HiOr);
4190
4191 // Long: ShAmt >= NewBitSize
4192 auto LoL = MIRBuilder.buildConstant(HalfTy, 0); // Lo part is zero.
4193 auto HiL = MIRBuilder.buildShl(HalfTy, InL, AmtExcess); // Hi from Lo part.
4194
4195 auto Lo = MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL);
4196 auto Hi = MIRBuilder.buildSelect(
4197 HalfTy, IsZero, InH, MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL));
4198
4199 ResultRegs[0] = Lo.getReg(0);
4200 ResultRegs[1] = Hi.getReg(0);
4201 break;
4202 }
4203 case TargetOpcode::G_LSHR:
4204 case TargetOpcode::G_ASHR: {
4205 // Short: ShAmt < NewBitSize
4206 auto HiS = MIRBuilder.buildInstr(MI.getOpcode(), {HalfTy}, {InH, Amt});
4207
4208 auto LoOr = MIRBuilder.buildLShr(HalfTy, InL, Amt);
4209 auto HiOr = MIRBuilder.buildShl(HalfTy, InH, AmtLack);
4210 auto LoS = MIRBuilder.buildOr(HalfTy, LoOr, HiOr);
4211
4212 // Long: ShAmt >= NewBitSize
4213 MachineInstrBuilder HiL;
4214 if (MI.getOpcode() == TargetOpcode::G_LSHR) {
4215 HiL = MIRBuilder.buildConstant(HalfTy, 0); // Hi part is zero.
4216 } else {
4217 auto ShiftAmt = MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize - 1);
4218 HiL = MIRBuilder.buildAShr(HalfTy, InH, ShiftAmt); // Sign of Hi part.
4219 }
4220 auto LoL = MIRBuilder.buildInstr(MI.getOpcode(), {HalfTy},
4221 {InH, AmtExcess}); // Lo from Hi part.
4222
4223 auto Lo = MIRBuilder.buildSelect(
4224 HalfTy, IsZero, InL, MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL));
4225
4226 auto Hi = MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL);
4227
4228 ResultRegs[0] = Lo.getReg(0);
4229 ResultRegs[1] = Hi.getReg(0);
4230 break;
4231 }
4232 default:
4233 llvm_unreachable("not a shift")::llvm::llvm_unreachable_internal("not a shift", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 4233)
;
4234 }
4235
4236 MIRBuilder.buildMerge(DstReg, ResultRegs);
4237 MI.eraseFromParent();
4238 return Legalized;
4239}
4240
4241LegalizerHelper::LegalizeResult
4242LegalizerHelper::moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx,
4243 LLT MoreTy) {
4244 assert(TypeIdx == 0 && "Expecting only Idx 0")((TypeIdx == 0 && "Expecting only Idx 0") ? static_cast
<void> (0) : __assert_fail ("TypeIdx == 0 && \"Expecting only Idx 0\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 4244, __PRETTY_FUNCTION__))
;
4245
4246 Observer.changingInstr(MI);
4247 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
4248 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
4249 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
4250 moreElementsVectorSrc(MI, MoreTy, I);
4251 }
4252
4253 MachineBasicBlock &MBB = *MI.getParent();
4254 MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI());
4255 moreElementsVectorDst(MI, MoreTy, 0);
4256 Observer.changedInstr(MI);
4257 return Legalized;
4258}
4259
4260LegalizerHelper::LegalizeResult
4261LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
4262 LLT MoreTy) {
4263 unsigned Opc = MI.getOpcode();
4264 switch (Opc) {
4265 case TargetOpcode::G_IMPLICIT_DEF:
4266 case TargetOpcode::G_LOAD: {
4267 if (TypeIdx != 0)
4268 return UnableToLegalize;
4269 Observer.changingInstr(MI);
4270 moreElementsVectorDst(MI, MoreTy, 0);
4271 Observer.changedInstr(MI);
4272 return Legalized;
4273 }
4274 case TargetOpcode::G_STORE:
4275 if (TypeIdx != 0)
4276 return UnableToLegalize;
4277 Observer.changingInstr(MI);
4278 moreElementsVectorSrc(MI, MoreTy, 0);
4279 Observer.changedInstr(MI);
4280 return Legalized;
4281 case TargetOpcode::G_AND:
4282 case TargetOpcode::G_OR:
4283 case TargetOpcode::G_XOR:
4284 case TargetOpcode::G_SMIN:
4285 case TargetOpcode::G_SMAX:
4286 case TargetOpcode::G_UMIN:
4287 case TargetOpcode::G_UMAX:
4288 case TargetOpcode::G_FMINNUM:
4289 case TargetOpcode::G_FMAXNUM:
4290 case TargetOpcode::G_FMINNUM_IEEE:
4291 case TargetOpcode::G_FMAXNUM_IEEE:
4292 case TargetOpcode::G_FMINIMUM:
4293 case TargetOpcode::G_FMAXIMUM: {
4294 Observer.changingInstr(MI);
4295 moreElementsVectorSrc(MI, MoreTy, 1);
4296 moreElementsVectorSrc(MI, MoreTy, 2);
4297 moreElementsVectorDst(MI, MoreTy, 0);
4298 Observer.changedInstr(MI);
4299 return Legalized;
4300 }
4301 case TargetOpcode::G_EXTRACT:
4302 if (TypeIdx != 1)
4303 return UnableToLegalize;
4304 Observer.changingInstr(MI);
4305 moreElementsVectorSrc(MI, MoreTy, 1);
4306 Observer.changedInstr(MI);
4307 return Legalized;
4308 case TargetOpcode::G_INSERT:
4309 case TargetOpcode::G_FREEZE:
4310 if (TypeIdx != 0)
4311 return UnableToLegalize;
4312 Observer.changingInstr(MI);
4313 moreElementsVectorSrc(MI, MoreTy, 1);
4314 moreElementsVectorDst(MI, MoreTy, 0);
4315 Observer.changedInstr(MI);
4316 return Legalized;
4317 case TargetOpcode::G_SELECT:
4318 if (TypeIdx != 0)
4319 return UnableToLegalize;
4320 if (MRI.getType(MI.getOperand(1).getReg()).isVector())
4321 return UnableToLegalize;
4322
4323 Observer.changingInstr(MI);
4324 moreElementsVectorSrc(MI, MoreTy, 2);
4325 moreElementsVectorSrc(MI, MoreTy, 3);
4326 moreElementsVectorDst(MI, MoreTy, 0);
4327 Observer.changedInstr(MI);
4328 return Legalized;
4329 case TargetOpcode::G_UNMERGE_VALUES: {
4330 if (TypeIdx != 1)
4331 return UnableToLegalize;
4332
4333 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
4334 int NumDst = MI.getNumOperands() - 1;
4335 moreElementsVectorSrc(MI, MoreTy, NumDst);
4336
4337 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES);
4338 for (int I = 0; I != NumDst; ++I)
4339 MIB.addDef(MI.getOperand(I).getReg());
4340
4341 int NewNumDst = MoreTy.getSizeInBits() / DstTy.getSizeInBits();
4342 for (int I = NumDst; I != NewNumDst; ++I)
4343 MIB.addDef(MRI.createGenericVirtualRegister(DstTy));
4344
4345 MIB.addUse(MI.getOperand(NumDst).getReg());
4346 MI.eraseFromParent();
4347 return Legalized;
4348 }
4349 case TargetOpcode::G_PHI:
4350 return moreElementsVectorPhi(MI, TypeIdx, MoreTy);
4351 default:
4352 return UnableToLegalize;
4353 }
4354}
4355
4356void LegalizerHelper::multiplyRegisters(SmallVectorImpl<Register> &DstRegs,
4357 ArrayRef<Register> Src1Regs,
4358 ArrayRef<Register> Src2Regs,
4359 LLT NarrowTy) {
4360 MachineIRBuilder &B = MIRBuilder;
4361 unsigned SrcParts = Src1Regs.size();
4362 unsigned DstParts = DstRegs.size();
4363
4364 unsigned DstIdx = 0; // Low bits of the result.
4365 Register FactorSum =
4366 B.buildMul(NarrowTy, Src1Regs[DstIdx], Src2Regs[DstIdx]).getReg(0);
4367 DstRegs[DstIdx] = FactorSum;
4368
4369 unsigned CarrySumPrevDstIdx;
4370 SmallVector<Register, 4> Factors;
4371
4372 for (DstIdx = 1; DstIdx < DstParts; DstIdx++) {
4373 // Collect low parts of muls for DstIdx.
4374 for (unsigned i = DstIdx + 1 < SrcParts ? 0 : DstIdx - SrcParts + 1;
4375 i <= std::min(DstIdx, SrcParts - 1); ++i) {
4376 MachineInstrBuilder Mul =
4377 B.buildMul(NarrowTy, Src1Regs[DstIdx - i], Src2Regs[i]);
4378 Factors.push_back(Mul.getReg(0));
4379 }
4380 // Collect high parts of muls from previous DstIdx.
4381 for (unsigned i = DstIdx < SrcParts ? 0 : DstIdx - SrcParts;
4382 i <= std::min(DstIdx - 1, SrcParts - 1); ++i) {
4383 MachineInstrBuilder Umulh =
4384 B.buildUMulH(NarrowTy, Src1Regs[DstIdx - 1 - i], Src2Regs[i]);
4385 Factors.push_back(Umulh.getReg(0));
4386 }
4387 // Add CarrySum from additions calculated for previous DstIdx.
4388 if (DstIdx != 1) {
4389 Factors.push_back(CarrySumPrevDstIdx);
4390 }
4391
4392 Register CarrySum;
4393 // Add all factors and accumulate all carries into CarrySum.
4394 if (DstIdx != DstParts - 1) {
4395 MachineInstrBuilder Uaddo =
4396 B.buildUAddo(NarrowTy, LLT::scalar(1), Factors[0], Factors[1]);
4397 FactorSum = Uaddo.getReg(0);
4398 CarrySum = B.buildZExt(NarrowTy, Uaddo.getReg(1)).getReg(0);
4399 for (unsigned i = 2; i < Factors.size(); ++i) {
4400 MachineInstrBuilder Uaddo =
4401 B.buildUAddo(NarrowTy, LLT::scalar(1), FactorSum, Factors[i]);
4402 FactorSum = Uaddo.getReg(0);
4403 MachineInstrBuilder Carry = B.buildZExt(NarrowTy, Uaddo.getReg(1));
4404 CarrySum = B.buildAdd(NarrowTy, CarrySum, Carry).getReg(0);
4405 }
4406 } else {
4407 // Since value for the next index is not calculated, neither is CarrySum.
4408 FactorSum = B.buildAdd(NarrowTy, Factors[0], Factors[1]).getReg(0);
4409 for (unsigned i = 2; i < Factors.size(); ++i)
4410 FactorSum = B.buildAdd(NarrowTy, FactorSum, Factors[i]).getReg(0);
4411 }
4412
4413 CarrySumPrevDstIdx = CarrySum;
4414 DstRegs[DstIdx] = FactorSum;
4415 Factors.clear();
4416 }
4417}
4418
4419LegalizerHelper::LegalizeResult
4420LegalizerHelper::narrowScalarMul(MachineInstr &MI, LLT NarrowTy) {
4421 Register DstReg = MI.getOperand(0).getReg();
4422 Register Src1 = MI.getOperand(1).getReg();
4423 Register Src2 = MI.getOperand(2).getReg();
4424
4425 LLT Ty = MRI.getType(DstReg);
4426 if (Ty.isVector())
4427 return UnableToLegalize;
4428
4429 unsigned SrcSize = MRI.getType(Src1).getSizeInBits();
4430 unsigned DstSize = Ty.getSizeInBits();
4431 unsigned NarrowSize = NarrowTy.getSizeInBits();
4432 if (DstSize % NarrowSize != 0 || SrcSize % NarrowSize != 0)
4433 return UnableToLegalize;
4434
4435 unsigned NumDstParts = DstSize / NarrowSize;
4436 unsigned NumSrcParts = SrcSize / NarrowSize;
4437 bool IsMulHigh = MI.getOpcode() == TargetOpcode::G_UMULH;
4438 unsigned DstTmpParts = NumDstParts * (IsMulHigh ? 2 : 1);
4439
4440 SmallVector<Register, 2> Src1Parts, Src2Parts;
4441 SmallVector<Register, 2> DstTmpRegs(DstTmpParts);
4442 extractParts(Src1, NarrowTy, NumSrcParts, Src1Parts);
4443 extractParts(Src2, NarrowTy, NumSrcParts, Src2Parts);
4444 multiplyRegisters(DstTmpRegs, Src1Parts, Src2Parts, NarrowTy);
4445
4446 // Take only high half of registers if this is high mul.
4447 ArrayRef<Register> DstRegs(
4448 IsMulHigh ? &DstTmpRegs[DstTmpParts / 2] : &DstTmpRegs[0], NumDstParts);
4449 MIRBuilder.buildMerge(DstReg, DstRegs);
4450 MI.eraseFromParent();
4451 return Legalized;
4452}
4453
4454LegalizerHelper::LegalizeResult
4455LegalizerHelper::narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx,
4456 LLT NarrowTy) {
4457 if (TypeIdx != 1)
4458 return UnableToLegalize;
4459
4460 uint64_t NarrowSize = NarrowTy.getSizeInBits();
4461
4462 int64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
4463 // FIXME: add support for when SizeOp1 isn't an exact multiple of
4464 // NarrowSize.
4465 if (SizeOp1 % NarrowSize != 0)
4466 return UnableToLegalize;
4467 int NumParts = SizeOp1 / NarrowSize;
4468
4469 SmallVector<Register, 2> SrcRegs, DstRegs;
4470 SmallVector<uint64_t, 2> Indexes;
4471 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
4472
4473 Register OpReg = MI.getOperand(0).getReg();
4474 uint64_t OpStart = MI.getOperand(2).getImm();
4475 uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
4476 for (int i = 0; i < NumParts; ++i) {
4477 unsigned SrcStart = i * NarrowSize;
4478
4479 if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) {
4480 // No part of the extract uses this subregister, ignore it.
4481 continue;
4482 } else if (SrcStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
4483 // The entire subregister is extracted, forward the value.
4484 DstRegs.push_back(SrcRegs[i]);
4485 continue;
4486 }
4487
4488 // OpSegStart is where this destination segment would start in OpReg if it
4489 // extended infinitely in both directions.
4490 int64_t ExtractOffset;
4491 uint64_t SegSize;
4492 if (OpStart < SrcStart) {
4493 ExtractOffset = 0;
4494 SegSize = std::min(NarrowSize, OpStart + OpSize - SrcStart);
4495 } else {
4496 ExtractOffset = OpStart - SrcStart;
4497 SegSize = std::min(SrcStart + NarrowSize - OpStart, OpSize);
4498 }
4499
4500 Register SegReg = SrcRegs[i];
4501 if (ExtractOffset != 0 || SegSize != NarrowSize) {
4502 // A genuine extract is needed.
4503 SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
4504 MIRBuilder.buildExtract(SegReg, SrcRegs[i], ExtractOffset);
4505 }
4506
4507 DstRegs.push_back(SegReg);
4508 }
4509
4510 Register DstReg = MI.getOperand(0).getReg();
4511 if (MRI.getType(DstReg).isVector())
4512 MIRBuilder.buildBuildVector(DstReg, DstRegs);
4513 else if (DstRegs.size() > 1)
4514 MIRBuilder.buildMerge(DstReg, DstRegs);
4515 else
4516 MIRBuilder.buildCopy(DstReg, DstRegs[0]);
4517 MI.eraseFromParent();
4518 return Legalized;
4519}
4520
4521LegalizerHelper::LegalizeResult
4522LegalizerHelper::narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx,
4523 LLT NarrowTy) {
4524 // FIXME: Don't know how to handle secondary types yet.
4525 if (TypeIdx != 0)
4526 return UnableToLegalize;
4527
4528 uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
4529 uint64_t NarrowSize = NarrowTy.getSizeInBits();
4530
4531 // FIXME: add support for when SizeOp0 isn't an exact multiple of
4532 // NarrowSize.
4533 if (SizeOp0 % NarrowSize != 0)
4534 return UnableToLegalize;
4535
4536 int NumParts = SizeOp0 / NarrowSize;
4537
4538 SmallVector<Register, 2> SrcRegs, DstRegs;
4539 SmallVector<uint64_t, 2> Indexes;
4540 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
4541
4542 Register OpReg = MI.getOperand(2).getReg();
4543 uint64_t OpStart = MI.getOperand(3).getImm();
4544 uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
4545 for (int i = 0; i < NumParts; ++i) {
4546 unsigned DstStart = i * NarrowSize;
4547
4548 if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) {
4549 // No part of the insert affects this subregister, forward the original.
4550 DstRegs.push_back(SrcRegs[i]);
4551 continue;
4552 } else if (DstStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
4553 // The entire subregister is defined by this insert, forward the new
4554 // value.
4555 DstRegs.push_back(OpReg);
4556 continue;
4557 }
4558
4559 // OpSegStart is where this destination segment would start in OpReg if it
4560 // extended infinitely in both directions.
4561 int64_t ExtractOffset, InsertOffset;
4562 uint64_t SegSize;
4563 if (OpStart < DstStart) {
4564 InsertOffset = 0;
4565 ExtractOffset = DstStart - OpStart;
4566 SegSize = std::min(NarrowSize, OpStart + OpSize - DstStart);
4567 } else {
4568 InsertOffset = OpStart - DstStart;
4569 ExtractOffset = 0;
4570 SegSize =
4571 std::min(NarrowSize - InsertOffset, OpStart + OpSize - DstStart);
4572 }
4573
4574 Register SegReg = OpReg;
4575 if (ExtractOffset != 0 || SegSize != OpSize) {
4576 // A genuine extract is needed.
4577 SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
4578 MIRBuilder.buildExtract(SegReg, OpReg, ExtractOffset);
4579 }
4580
4581 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy);
4582 MIRBuilder.buildInsert(DstReg, SrcRegs[i], SegReg, InsertOffset);
4583 DstRegs.push_back(DstReg);
4584 }
4585
4586 assert(DstRegs.size() == (unsigned)NumParts && "not all parts covered")((DstRegs.size() == (unsigned)NumParts && "not all parts covered"
) ? static_cast<void> (0) : __assert_fail ("DstRegs.size() == (unsigned)NumParts && \"not all parts covered\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 4586, __PRETTY_FUNCTION__))
;
4587 Register DstReg = MI.getOperand(0).getReg();
4588 if(MRI.getType(DstReg).isVector())
4589 MIRBuilder.buildBuildVector(DstReg, DstRegs);
4590 else
4591 MIRBuilder.buildMerge(DstReg, DstRegs);
4592 MI.eraseFromParent();
4593 return Legalized;
4594}
4595
4596LegalizerHelper::LegalizeResult
4597LegalizerHelper::narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx,
4598 LLT NarrowTy) {
4599 Register DstReg = MI.getOperand(0).getReg();
4600 LLT DstTy = MRI.getType(DstReg);
4601
4602 assert(MI.getNumOperands() == 3 && TypeIdx == 0)((MI.getNumOperands() == 3 && TypeIdx == 0) ? static_cast
<void> (0) : __assert_fail ("MI.getNumOperands() == 3 && TypeIdx == 0"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 4602, __PRETTY_FUNCTION__))
;
4603
4604 SmallVector<Register, 4> DstRegs, DstLeftoverRegs;
4605 SmallVector<Register, 4> Src0Regs, Src0LeftoverRegs;
4606 SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs;
4607 LLT LeftoverTy;
4608 if (!extractParts(MI.getOperand(1).getReg(), DstTy, NarrowTy, LeftoverTy,
4609 Src0Regs, Src0LeftoverRegs))
4610 return UnableToLegalize;
4611
4612 LLT Unused;
4613 if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, Unused,
4614 Src1Regs, Src1LeftoverRegs))
4615 llvm_unreachable("inconsistent extractParts result")::llvm::llvm_unreachable_internal("inconsistent extractParts result"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 4615)
;
4616
4617 for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) {
4618 auto Inst = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy},
4619 {Src0Regs[I], Src1Regs[I]});
4620 DstRegs.push_back(Inst.getReg(0));
4621 }
4622
4623 for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) {
4624 auto Inst = MIRBuilder.buildInstr(
4625 MI.getOpcode(),
4626 {LeftoverTy}, {Src0LeftoverRegs[I], Src1LeftoverRegs[I]});
4627 DstLeftoverRegs.push_back(Inst.getReg(0));
4628 }
4629
4630 insertParts(DstReg, DstTy, NarrowTy, DstRegs,
4631 LeftoverTy, DstLeftoverRegs);
4632
4633 MI.eraseFromParent();
4634 return Legalized;
4635}
4636
4637LegalizerHelper::LegalizeResult
4638LegalizerHelper::narrowScalarExt(MachineInstr &MI, unsigned TypeIdx,
4639 LLT NarrowTy) {
4640 if (TypeIdx != 0)
4641 return UnableToLegalize;
4642
4643 Register DstReg = MI.getOperand(0).getReg();
4644 Register SrcReg = MI.getOperand(1).getReg();
4645
4646 LLT DstTy = MRI.getType(DstReg);
4647 if (DstTy.isVector())
4648 return UnableToLegalize;
4649
4650 SmallVector<Register, 8> Parts;
4651 LLT GCDTy = extractGCDType(Parts, DstTy, NarrowTy, SrcReg);
4652 LLT LCMTy = buildLCMMergePieces(DstTy, NarrowTy, GCDTy, Parts, MI.getOpcode());
4653 buildWidenedRemergeToDst(DstReg, LCMTy, Parts);
4654
4655 MI.eraseFromParent();
4656 return Legalized;
4657}
4658
4659LegalizerHelper::LegalizeResult
4660LegalizerHelper::narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx,
4661 LLT NarrowTy) {
4662 if (TypeIdx != 0)
4663 return UnableToLegalize;
4664
4665 Register CondReg = MI.getOperand(1).getReg();
4666 LLT CondTy = MRI.getType(CondReg);
4667 if (CondTy.isVector()) // TODO: Handle vselect
4668 return UnableToLegalize;
4669
4670 Register DstReg = MI.getOperand(0).getReg();
4671 LLT DstTy = MRI.getType(DstReg);
4672
4673 SmallVector<Register, 4> DstRegs, DstLeftoverRegs;
4674 SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs;
4675 SmallVector<Register, 4> Src2Regs, Src2LeftoverRegs;
4676 LLT LeftoverTy;
4677 if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, LeftoverTy,
4678 Src1Regs, Src1LeftoverRegs))
4679 return UnableToLegalize;
4680
4681 LLT Unused;
4682 if (!extractParts(MI.getOperand(3).getReg(), DstTy, NarrowTy, Unused,
4683 Src2Regs, Src2LeftoverRegs))
4684 llvm_unreachable("inconsistent extractParts result")::llvm::llvm_unreachable_internal("inconsistent extractParts result"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 4684)
;
4685
4686 for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) {
4687 auto Select = MIRBuilder.buildSelect(NarrowTy,
4688 CondReg, Src1Regs[I], Src2Regs[I]);
4689 DstRegs.push_back(Select.getReg(0));
4690 }
4691
4692 for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) {
4693 auto Select = MIRBuilder.buildSelect(
4694 LeftoverTy, CondReg, Src1LeftoverRegs[I], Src2LeftoverRegs[I]);
4695 DstLeftoverRegs.push_back(Select.getReg(0));
4696 }
4697
4698 insertParts(DstReg, DstTy, NarrowTy, DstRegs,
4699 LeftoverTy, DstLeftoverRegs);
4700
4701 MI.eraseFromParent();
4702 return Legalized;
4703}
4704
4705LegalizerHelper::LegalizeResult
4706LegalizerHelper::narrowScalarCTLZ(MachineInstr &MI, unsigned TypeIdx,
4707 LLT NarrowTy) {
4708 if (TypeIdx != 1)
4709 return UnableToLegalize;
4710
4711 Register DstReg = MI.getOperand(0).getReg();
4712 Register SrcReg = MI.getOperand(1).getReg();
4713 LLT DstTy = MRI.getType(DstReg);
4714 LLT SrcTy = MRI.getType(SrcReg);
4715 unsigned NarrowSize = NarrowTy.getSizeInBits();
4716
4717 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) {
4718 const bool IsUndef = MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF;
4719
4720 MachineIRBuilder &B = MIRBuilder;
4721 auto UnmergeSrc = B.buildUnmerge(NarrowTy, SrcReg);
4722 // ctlz(Hi:Lo) -> Hi == 0 ? (NarrowSize + ctlz(Lo)) : ctlz(Hi)
4723 auto C_0 = B.buildConstant(NarrowTy, 0);
4724 auto HiIsZero = B.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1),
4725 UnmergeSrc.getReg(1), C_0);
4726 auto LoCTLZ = IsUndef ?
4727 B.buildCTLZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(0)) :
4728 B.buildCTLZ(DstTy, UnmergeSrc.getReg(0));
4729 auto C_NarrowSize = B.buildConstant(DstTy, NarrowSize);
4730 auto HiIsZeroCTLZ = B.buildAdd(DstTy, LoCTLZ, C_NarrowSize);
4731 auto HiCTLZ = B.buildCTLZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(1));
4732 B.buildSelect(DstReg, HiIsZero, HiIsZeroCTLZ, HiCTLZ);
4733
4734 MI.eraseFromParent();
4735 return Legalized;
4736 }
4737
4738 return UnableToLegalize;
4739}
4740
4741LegalizerHelper::LegalizeResult
4742LegalizerHelper::narrowScalarCTTZ(MachineInstr &MI, unsigned TypeIdx,
4743 LLT NarrowTy) {
4744 if (TypeIdx != 1)
4745 return UnableToLegalize;
4746
4747 Register DstReg = MI.getOperand(0).getReg();
4748 Register SrcReg = MI.getOperand(1).getReg();
4749 LLT DstTy = MRI.getType(DstReg);
4750 LLT SrcTy = MRI.getType(SrcReg);
4751 unsigned NarrowSize = NarrowTy.getSizeInBits();
4752
4753 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) {
4754 const bool IsUndef = MI.getOpcode() == TargetOpcode::G_CTTZ_ZERO_UNDEF;
4755
4756 MachineIRBuilder &B = MIRBuilder;
4757 auto UnmergeSrc = B.buildUnmerge(NarrowTy, SrcReg);
4758 // cttz(Hi:Lo) -> Lo == 0 ? (cttz(Hi) + NarrowSize) : cttz(Lo)
4759 auto C_0 = B.buildConstant(NarrowTy, 0);
4760 auto LoIsZero = B.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1),
4761 UnmergeSrc.getReg(0), C_0);
4762 auto HiCTTZ = IsUndef ?
4763 B.buildCTTZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(1)) :
4764 B.buildCTTZ(DstTy, UnmergeSrc.getReg(1));
4765 auto C_NarrowSize = B.buildConstant(DstTy, NarrowSize);
4766 auto LoIsZeroCTTZ = B.buildAdd(DstTy, HiCTTZ, C_NarrowSize);
4767 auto LoCTTZ = B.buildCTTZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(0));
4768 B.buildSelect(DstReg, LoIsZero, LoIsZeroCTTZ, LoCTTZ);
4769
4770 MI.eraseFromParent();
4771 return Legalized;
4772 }
4773
4774 return UnableToLegalize;
4775}
4776
4777LegalizerHelper::LegalizeResult
4778LegalizerHelper::narrowScalarCTPOP(MachineInstr &MI, unsigned TypeIdx,
4779 LLT NarrowTy) {
4780 if (TypeIdx != 1)
4781 return UnableToLegalize;
4782
4783 Register DstReg = MI.getOperand(0).getReg();
4784 LLT DstTy = MRI.getType(DstReg);
4785 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
4786 unsigned NarrowSize = NarrowTy.getSizeInBits();
4787
4788 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) {
4789 auto UnmergeSrc = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1));
4790
4791 auto LoCTPOP = MIRBuilder.buildCTPOP(DstTy, UnmergeSrc.getReg(0));
4792 auto HiCTPOP = MIRBuilder.buildCTPOP(DstTy, UnmergeSrc.getReg(1));
4793 MIRBuilder.buildAdd(DstReg, HiCTPOP, LoCTPOP);
4794
4795 MI.eraseFromParent();
4796 return Legalized;
4797 }
4798
4799 return UnableToLegalize;
4800}
4801
4802LegalizerHelper::LegalizeResult
4803LegalizerHelper::lowerBitCount(MachineInstr &MI) {
4804 unsigned Opc = MI.getOpcode();
4805 const auto &TII = MIRBuilder.getTII();
4806 auto isSupported = [this](const LegalityQuery &Q) {
4807 auto QAction = LI.getAction(Q).Action;
4808 return QAction == Legal || QAction == Libcall || QAction == Custom;
4809 };
4810 switch (Opc) {
4811 default:
4812 return UnableToLegalize;
4813 case TargetOpcode::G_CTLZ_ZERO_UNDEF: {
4814 // This trivially expands to CTLZ.
4815 Observer.changingInstr(MI);
4816 MI.setDesc(TII.get(TargetOpcode::G_CTLZ));
4817 Observer.changedInstr(MI);
4818 return Legalized;
4819 }
4820 case TargetOpcode::G_CTLZ: {
4821 Register DstReg = MI.getOperand(0).getReg();
4822 Register SrcReg = MI.getOperand(1).getReg();
4823 LLT DstTy = MRI.getType(DstReg);
4824 LLT SrcTy = MRI.getType(SrcReg);
4825 unsigned Len = SrcTy.getSizeInBits();
4826
4827 if (isSupported({TargetOpcode::G_CTLZ_ZERO_UNDEF, {DstTy, SrcTy}})) {
4828 // If CTLZ_ZERO_UNDEF is supported, emit that and a select for zero.
4829 auto CtlzZU = MIRBuilder.buildCTLZ_ZERO_UNDEF(DstTy, SrcReg);
4830 auto ZeroSrc = MIRBuilder.buildConstant(SrcTy, 0);
4831 auto ICmp = MIRBuilder.buildICmp(
4832 CmpInst::ICMP_EQ, SrcTy.changeElementSize(1), SrcReg, ZeroSrc);
4833 auto LenConst = MIRBuilder.buildConstant(DstTy, Len);
4834 MIRBuilder.buildSelect(DstReg, ICmp, LenConst, CtlzZU);
4835 MI.eraseFromParent();
4836 return Legalized;
4837 }
4838 // for now, we do this:
4839 // NewLen = NextPowerOf2(Len);
4840 // x = x | (x >> 1);
4841 // x = x | (x >> 2);
4842 // ...
4843 // x = x | (x >>16);
4844 // x = x | (x >>32); // for 64-bit input
4845 // Upto NewLen/2
4846 // return Len - popcount(x);
4847 //
4848 // Ref: "Hacker's Delight" by Henry Warren
4849 Register Op = SrcReg;
4850 unsigned NewLen = PowerOf2Ceil(Len);
4851 for (unsigned i = 0; (1U << i) <= (NewLen / 2); ++i) {
4852 auto MIBShiftAmt = MIRBuilder.buildConstant(SrcTy, 1ULL << i);
4853 auto MIBOp = MIRBuilder.buildOr(
4854 SrcTy, Op, MIRBuilder.buildLShr(SrcTy, Op, MIBShiftAmt));
4855 Op = MIBOp.getReg(0);
4856 }
4857 auto MIBPop = MIRBuilder.buildCTPOP(DstTy, Op);
4858 MIRBuilder.buildSub(MI.getOperand(0), MIRBuilder.buildConstant(DstTy, Len),
4859 MIBPop);
4860 MI.eraseFromParent();
4861 return Legalized;
4862 }
4863 case TargetOpcode::G_CTTZ_ZERO_UNDEF: {
4864 // This trivially expands to CTTZ.
4865 Observer.changingInstr(MI);
4866 MI.setDesc(TII.get(TargetOpcode::G_CTTZ));
4867 Observer.changedInstr(MI);
4868 return Legalized;
4869 }
4870 case TargetOpcode::G_CTTZ: {
4871 Register DstReg = MI.getOperand(0).getReg();
4872 Register SrcReg = MI.getOperand(1).getReg();
4873 LLT DstTy = MRI.getType(DstReg);
4874 LLT SrcTy = MRI.getType(SrcReg);
4875
4876 unsigned Len = SrcTy.getSizeInBits();
4877 if (isSupported({TargetOpcode::G_CTTZ_ZERO_UNDEF, {DstTy, SrcTy}})) {
4878 // If CTTZ_ZERO_UNDEF is legal or custom, emit that and a select with
4879 // zero.
4880 auto CttzZU = MIRBuilder.buildCTTZ_ZERO_UNDEF(DstTy, SrcReg);
4881 auto Zero = MIRBuilder.buildConstant(SrcTy, 0);
4882 auto ICmp = MIRBuilder.buildICmp(
4883 CmpInst::ICMP_EQ, DstTy.changeElementSize(1), SrcReg, Zero);
4884 auto LenConst = MIRBuilder.buildConstant(DstTy, Len);
4885 MIRBuilder.buildSelect(DstReg, ICmp, LenConst, CttzZU);
4886 MI.eraseFromParent();
4887 return Legalized;
4888 }
4889 // for now, we use: { return popcount(~x & (x - 1)); }
4890 // unless the target has ctlz but not ctpop, in which case we use:
4891 // { return 32 - nlz(~x & (x-1)); }
4892 // Ref: "Hacker's Delight" by Henry Warren
4893 auto MIBCstNeg1 = MIRBuilder.buildConstant(SrcTy, -1);
4894 auto MIBNot = MIRBuilder.buildXor(SrcTy, SrcReg, MIBCstNeg1);
4895 auto MIBTmp = MIRBuilder.buildAnd(
4896 SrcTy, MIBNot, MIRBuilder.buildAdd(SrcTy, SrcReg, MIBCstNeg1));
4897 if (!isSupported({TargetOpcode::G_CTPOP, {SrcTy, SrcTy}}) &&
4898 isSupported({TargetOpcode::G_CTLZ, {SrcTy, SrcTy}})) {
4899 auto MIBCstLen = MIRBuilder.buildConstant(SrcTy, Len);
4900 MIRBuilder.buildSub(MI.getOperand(0), MIBCstLen,
4901 MIRBuilder.buildCTLZ(SrcTy, MIBTmp));
4902 MI.eraseFromParent();
4903 return Legalized;
4904 }
4905 MI.setDesc(TII.get(TargetOpcode::G_CTPOP));
4906 MI.getOperand(1).setReg(MIBTmp.getReg(0));
4907 return Legalized;
4908 }
4909 case TargetOpcode::G_CTPOP: {
4910 Register SrcReg = MI.getOperand(1).getReg();
4911 LLT Ty = MRI.getType(SrcReg);
4912 unsigned Size = Ty.getSizeInBits();
4913 MachineIRBuilder &B = MIRBuilder;
4914
4915 // Count set bits in blocks of 2 bits. Default approach would be
4916 // B2Count = { val & 0x55555555 } + { (val >> 1) & 0x55555555 }
4917 // We use following formula instead:
4918 // B2Count = val - { (val >> 1) & 0x55555555 }
4919 // since it gives same result in blocks of 2 with one instruction less.
4920 auto C_1 = B.buildConstant(Ty, 1);
4921 auto B2Set1LoTo1Hi = B.buildLShr(Ty, SrcReg, C_1);
4922 APInt B2Mask1HiTo0 = APInt::getSplat(Size, APInt(8, 0x55));
4923 auto C_B2Mask1HiTo0 = B.buildConstant(Ty, B2Mask1HiTo0);
4924 auto B2Count1Hi = B.buildAnd(Ty, B2Set1LoTo1Hi, C_B2Mask1HiTo0);
4925 auto B2Count = B.buildSub(Ty, SrcReg, B2Count1Hi);
4926
4927 // In order to get count in blocks of 4 add values from adjacent block of 2.
4928 // B4Count = { B2Count & 0x33333333 } + { (B2Count >> 2) & 0x33333333 }
4929 auto C_2 = B.buildConstant(Ty, 2);
4930 auto B4Set2LoTo2Hi = B.buildLShr(Ty, B2Count, C_2);
4931 APInt B4Mask2HiTo0 = APInt::getSplat(Size, APInt(8, 0x33));
4932 auto C_B4Mask2HiTo0 = B.buildConstant(Ty, B4Mask2HiTo0);
4933 auto B4HiB2Count = B.buildAnd(Ty, B4Set2LoTo2Hi, C_B4Mask2HiTo0);
4934 auto B4LoB2Count = B.buildAnd(Ty, B2Count, C_B4Mask2HiTo0);
4935 auto B4Count = B.buildAdd(Ty, B4HiB2Count, B4LoB2Count);
4936
4937 // For count in blocks of 8 bits we don't have to mask high 4 bits before
4938 // addition since count value sits in range {0,...,8} and 4 bits are enough
4939 // to hold such binary values. After addition high 4 bits still hold count
4940 // of set bits in high 4 bit block, set them to zero and get 8 bit result.
4941 // B8Count = { B4Count + (B4Count >> 4) } & 0x0F0F0F0F
4942 auto C_4 = B.buildConstant(Ty, 4);
4943 auto B8HiB4Count = B.buildLShr(Ty, B4Count, C_4);
4944 auto B8CountDirty4Hi = B.buildAdd(Ty, B8HiB4Count, B4Count);
4945 APInt B8Mask4HiTo0 = APInt::getSplat(Size, APInt(8, 0x0F));
4946 auto C_B8Mask4HiTo0 = B.buildConstant(Ty, B8Mask4HiTo0);
4947 auto B8Count = B.buildAnd(Ty, B8CountDirty4Hi, C_B8Mask4HiTo0);
4948
4949 assert(Size<=128 && "Scalar size is too large for CTPOP lower algorithm")((Size<=128 && "Scalar size is too large for CTPOP lower algorithm"
) ? static_cast<void> (0) : __assert_fail ("Size<=128 && \"Scalar size is too large for CTPOP lower algorithm\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 4949, __PRETTY_FUNCTION__))
;
4950 // 8 bits can hold CTPOP result of 128 bit int or smaller. Mul with this
4951 // bitmask will set 8 msb in ResTmp to sum of all B8Counts in 8 bit blocks.
4952 auto MulMask = B.buildConstant(Ty, APInt::getSplat(Size, APInt(8, 0x01)));
4953 auto ResTmp = B.buildMul(Ty, B8Count, MulMask);
4954
4955 // Shift count result from 8 high bits to low bits.
4956 auto C_SizeM8 = B.buildConstant(Ty, Size - 8);
4957 B.buildLShr(MI.getOperand(0).getReg(), ResTmp, C_SizeM8);
4958
4959 MI.eraseFromParent();
4960 return Legalized;
4961 }
4962 }
4963}
4964
4965// Expand s32 = G_UITOFP s64 using bit operations to an IEEE float
4966// representation.
4967LegalizerHelper::LegalizeResult
4968LegalizerHelper::lowerU64ToF32BitOps(MachineInstr &MI) {
4969 Register Dst = MI.getOperand(0).getReg();
4970 Register Src = MI.getOperand(1).getReg();
4971 const LLT S64 = LLT::scalar(64);
4972 const LLT S32 = LLT::scalar(32);
4973 const LLT S1 = LLT::scalar(1);
4974
4975 assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S32)((MRI.getType(Src) == S64 && MRI.getType(Dst) == S32)
? static_cast<void> (0) : __assert_fail ("MRI.getType(Src) == S64 && MRI.getType(Dst) == S32"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 4975, __PRETTY_FUNCTION__))
;
4976
4977 // unsigned cul2f(ulong u) {
4978 // uint lz = clz(u);
4979 // uint e = (u != 0) ? 127U + 63U - lz : 0;
4980 // u = (u << lz) & 0x7fffffffffffffffUL;
4981 // ulong t = u & 0xffffffffffUL;
4982 // uint v = (e << 23) | (uint)(u >> 40);
4983 // uint r = t > 0x8000000000UL ? 1U : (t == 0x8000000000UL ? v & 1U : 0U);
4984 // return as_float(v + r);
4985 // }
4986
4987 auto Zero32 = MIRBuilder.buildConstant(S32, 0);
4988 auto Zero64 = MIRBuilder.buildConstant(S64, 0);
4989
4990 auto LZ = MIRBuilder.buildCTLZ_ZERO_UNDEF(S32, Src);
4991
4992 auto K = MIRBuilder.buildConstant(S32, 127U + 63U);
4993 auto Sub = MIRBuilder.buildSub(S32, K, LZ);
4994
4995 auto NotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, Src, Zero64);
4996 auto E = MIRBuilder.buildSelect(S32, NotZero, Sub, Zero32);
4997
4998 auto Mask0 = MIRBuilder.buildConstant(S64, (-1ULL) >> 1);
4999 auto ShlLZ = MIRBuilder.buildShl(S64, Src, LZ);
5000
5001 auto U = MIRBuilder.buildAnd(S64, ShlLZ, Mask0);
5002
5003 auto Mask1 = MIRBuilder.buildConstant(S64, 0xffffffffffULL);
5004 auto T = MIRBuilder.buildAnd(S64, U, Mask1);
5005
5006 auto UShl = MIRBuilder.buildLShr(S64, U, MIRBuilder.buildConstant(S64, 40));
5007 auto ShlE = MIRBuilder.buildShl(S32, E, MIRBuilder.buildConstant(S32, 23));
5008 auto V = MIRBuilder.buildOr(S32, ShlE, MIRBuilder.buildTrunc(S32, UShl));
5009
5010 auto C = MIRBuilder.buildConstant(S64, 0x8000000000ULL);
5011 auto RCmp = MIRBuilder.buildICmp(CmpInst::ICMP_UGT, S1, T, C);
5012 auto TCmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1, T, C);
5013 auto One = MIRBuilder.buildConstant(S32, 1);
5014
5015 auto VTrunc1 = MIRBuilder.buildAnd(S32, V, One);
5016 auto Select0 = MIRBuilder.buildSelect(S32, TCmp, VTrunc1, Zero32);
5017 auto R = MIRBuilder.buildSelect(S32, RCmp, One, Select0);
5018 MIRBuilder.buildAdd(Dst, V, R);
5019
5020 MI.eraseFromParent();
5021 return Legalized;
5022}
5023
5024LegalizerHelper::LegalizeResult LegalizerHelper::lowerUITOFP(MachineInstr &MI) {
5025 Register Dst = MI.getOperand(0).getReg();
5026 Register Src = MI.getOperand(1).getReg();
5027 LLT DstTy = MRI.getType(Dst);
5028 LLT SrcTy = MRI.getType(Src);
5029
5030 if (SrcTy == LLT::scalar(1)) {
5031 auto True = MIRBuilder.buildFConstant(DstTy, 1.0);
5032 auto False = MIRBuilder.buildFConstant(DstTy, 0.0);
5033 MIRBuilder.buildSelect(Dst, Src, True, False);
5034 MI.eraseFromParent();
5035 return Legalized;
5036 }
5037
5038 if (SrcTy != LLT::scalar(64))
5039 return UnableToLegalize;
5040
5041 if (DstTy == LLT::scalar(32)) {
5042 // TODO: SelectionDAG has several alternative expansions to port which may
5043 // be more reasonble depending on the available instructions. If a target
5044 // has sitofp, does not have CTLZ, or can efficiently use f64 as an
5045 // intermediate type, this is probably worse.
5046 return lowerU64ToF32BitOps(MI);
5047 }
5048
5049 return UnableToLegalize;
5050}
5051
5052LegalizerHelper::LegalizeResult LegalizerHelper::lowerSITOFP(MachineInstr &MI) {
5053 Register Dst = MI.getOperand(0).getReg();
5054 Register Src = MI.getOperand(1).getReg();
5055 LLT DstTy = MRI.getType(Dst);
5056 LLT SrcTy = MRI.getType(Src);
5057
5058 const LLT S64 = LLT::scalar(64);
5059 const LLT S32 = LLT::scalar(32);
5060 const LLT S1 = LLT::scalar(1);
5061
5062 if (SrcTy == S1) {
5063 auto True = MIRBuilder.buildFConstant(DstTy, -1.0);
5064 auto False = MIRBuilder.buildFConstant(DstTy, 0.0);
5065 MIRBuilder.buildSelect(Dst, Src, True, False);
5066 MI.eraseFromParent();
5067 return Legalized;
5068 }
5069
5070 if (SrcTy != S64)
5071 return UnableToLegalize;
5072
5073 if (DstTy == S32) {
5074 // signed cl2f(long l) {
5075 // long s = l >> 63;
5076 // float r = cul2f((l + s) ^ s);
5077 // return s ? -r : r;
5078 // }
5079 Register L = Src;
5080 auto SignBit = MIRBuilder.buildConstant(S64, 63);
5081 auto S = MIRBuilder.buildAShr(S64, L, SignBit);
5082
5083 auto LPlusS = MIRBuilder.buildAdd(S64, L, S);
5084 auto Xor = MIRBuilder.buildXor(S64, LPlusS, S);
5085 auto R = MIRBuilder.buildUITOFP(S32, Xor);
5086
5087 auto RNeg = MIRBuilder.buildFNeg(S32, R);
5088 auto SignNotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, S,
5089 MIRBuilder.buildConstant(S64, 0));
5090 MIRBuilder.buildSelect(Dst, SignNotZero, RNeg, R);
5091 MI.eraseFromParent();
5092 return Legalized;
5093 }
5094
5095 return UnableToLegalize;
5096}
5097
5098LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPTOUI(MachineInstr &MI) {
5099 Register Dst = MI.getOperand(0).getReg();
5100 Register Src = MI.getOperand(1).getReg();
5101 LLT DstTy = MRI.getType(Dst);
5102 LLT SrcTy = MRI.getType(Src);
5103 const LLT S64 = LLT::scalar(64);
5104 const LLT S32 = LLT::scalar(32);
5105
5106 if (SrcTy != S64 && SrcTy != S32)
5107 return UnableToLegalize;
5108 if (DstTy != S32 && DstTy != S64)
5109 return UnableToLegalize;
5110
5111 // FPTOSI gives same result as FPTOUI for positive signed integers.
5112 // FPTOUI needs to deal with fp values that convert to unsigned integers
5113 // greater or equal to 2^31 for float or 2^63 for double. For brevity 2^Exp.
5114
5115 APInt TwoPExpInt = APInt::getSignMask(DstTy.getSizeInBits());
5116 APFloat TwoPExpFP(SrcTy.getSizeInBits() == 32 ? APFloat::IEEEsingle()
5117 : APFloat::IEEEdouble(),
5118 APInt::getNullValue(SrcTy.getSizeInBits()));
5119 TwoPExpFP.convertFromAPInt(TwoPExpInt, false, APFloat::rmNearestTiesToEven);
5120
5121 MachineInstrBuilder FPTOSI = MIRBuilder.buildFPTOSI(DstTy, Src);
5122
5123 MachineInstrBuilder Threshold = MIRBuilder.buildFConstant(SrcTy, TwoPExpFP);
5124 // For fp Value greater or equal to Threshold(2^Exp), we use FPTOSI on
5125 // (Value - 2^Exp) and add 2^Exp by setting highest bit in result to 1.
5126 MachineInstrBuilder FSub = MIRBuilder.buildFSub(SrcTy, Src, Threshold);
5127 MachineInstrBuilder ResLowBits = MIRBuilder.buildFPTOSI(DstTy, FSub);
5128 MachineInstrBuilder ResHighBit = MIRBuilder.buildConstant(DstTy, TwoPExpInt);
5129 MachineInstrBuilder Res = MIRBuilder.buildXor(DstTy, ResLowBits, ResHighBit);
5130
5131 const LLT S1 = LLT::scalar(1);
5132
5133 MachineInstrBuilder FCMP =
5134 MIRBuilder.buildFCmp(CmpInst::FCMP_ULT, S1, Src, Threshold);
5135 MIRBuilder.buildSelect(Dst, FCMP, FPTOSI, Res);
5136
5137 MI.eraseFromParent();
5138 return Legalized;
5139}
5140
5141LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPTOSI(MachineInstr &MI) {
5142 Register Dst = MI.getOperand(0).getReg();
5143 Register Src = MI.getOperand(1).getReg();
5144 LLT DstTy = MRI.getType(Dst);
5145 LLT SrcTy = MRI.getType(Src);
5146 const LLT S64 = LLT::scalar(64);
5147 const LLT S32 = LLT::scalar(32);
5148
5149 // FIXME: Only f32 to i64 conversions are supported.
5150 if (SrcTy.getScalarType() != S32 || DstTy.getScalarType() != S64)
5151 return UnableToLegalize;
5152
5153 // Expand f32 -> i64 conversion
5154 // This algorithm comes from compiler-rt's implementation of fixsfdi:
5155 // https://github.com/llvm/llvm-project/blob/master/compiler-rt/lib/builtins/fixsfdi.c
5156
5157 unsigned SrcEltBits = SrcTy.getScalarSizeInBits();
5158
5159 auto ExponentMask = MIRBuilder.buildConstant(SrcTy, 0x7F800000);
5160 auto ExponentLoBit = MIRBuilder.buildConstant(SrcTy, 23);
5161
5162 auto AndExpMask = MIRBuilder.buildAnd(SrcTy, Src, ExponentMask);
5163 auto ExponentBits = MIRBuilder.buildLShr(SrcTy, AndExpMask, ExponentLoBit);
5164
5165 auto SignMask = MIRBuilder.buildConstant(SrcTy,
5166 APInt::getSignMask(SrcEltBits));
5167 auto AndSignMask = MIRBuilder.buildAnd(SrcTy, Src, SignMask);
5168 auto SignLowBit = MIRBuilder.buildConstant(SrcTy, SrcEltBits - 1);
5169 auto Sign = MIRBuilder.buildAShr(SrcTy, AndSignMask, SignLowBit);
5170 Sign = MIRBuilder.buildSExt(DstTy, Sign);
5171
5172 auto MantissaMask = MIRBuilder.buildConstant(SrcTy, 0x007FFFFF);
5173 auto AndMantissaMask = MIRBuilder.buildAnd(SrcTy, Src, MantissaMask);
5174 auto K = MIRBuilder.buildConstant(SrcTy, 0x00800000);
5175
5176 auto R = MIRBuilder.buildOr(SrcTy, AndMantissaMask, K);
5177 R = MIRBuilder.buildZExt(DstTy, R);
5178
5179 auto Bias = MIRBuilder.buildConstant(SrcTy, 127);
5180 auto Exponent = MIRBuilder.buildSub(SrcTy, ExponentBits, Bias);
5181 auto SubExponent = MIRBuilder.buildSub(SrcTy, Exponent, ExponentLoBit);
5182 auto ExponentSub = MIRBuilder.buildSub(SrcTy, ExponentLoBit, Exponent);
5183
5184 auto Shl = MIRBuilder.buildShl(DstTy, R, SubExponent);
5185 auto Srl = MIRBuilder.buildLShr(DstTy, R, ExponentSub);
5186
5187 const LLT S1 = LLT::scalar(1);
5188 auto CmpGt = MIRBuilder.buildICmp(CmpInst::ICMP_SGT,
5189 S1, Exponent, ExponentLoBit);
5190
5191 R = MIRBuilder.buildSelect(DstTy, CmpGt, Shl, Srl);
5192
5193 auto XorSign = MIRBuilder.buildXor(DstTy, R, Sign);
5194 auto Ret = MIRBuilder.buildSub(DstTy, XorSign, Sign);
5195
5196 auto ZeroSrcTy = MIRBuilder.buildConstant(SrcTy, 0);
5197
5198 auto ExponentLt0 = MIRBuilder.buildICmp(CmpInst::ICMP_SLT,
5199 S1, Exponent, ZeroSrcTy);
5200
5201 auto ZeroDstTy = MIRBuilder.buildConstant(DstTy, 0);
5202 MIRBuilder.buildSelect(Dst, ExponentLt0, ZeroDstTy, Ret);
5203
5204 MI.eraseFromParent();
5205 return Legalized;
5206}
5207
5208// f64 -> f16 conversion using round-to-nearest-even rounding mode.
5209LegalizerHelper::LegalizeResult
5210LegalizerHelper::lowerFPTRUNC_F64_TO_F16(MachineInstr &MI) {
5211 Register Dst = MI.getOperand(0).getReg();
5212 Register Src = MI.getOperand(1).getReg();
5213
5214 if (MRI.getType(Src).isVector()) // TODO: Handle vectors directly.
5215 return UnableToLegalize;
5216
5217 const unsigned ExpMask = 0x7ff;
5218 const unsigned ExpBiasf64 = 1023;
5219 const unsigned ExpBiasf16 = 15;
5220 const LLT S32 = LLT::scalar(32);
5221 const LLT S1 = LLT::scalar(1);
5222
5223 auto Unmerge = MIRBuilder.buildUnmerge(S32, Src);
5224 Register U = Unmerge.getReg(0);
5225 Register UH = Unmerge.getReg(1);
5226
5227 auto E = MIRBuilder.buildLShr(S32, UH, MIRBuilder.buildConstant(S32, 20));
5228 E = MIRBuilder.buildAnd(S32, E, MIRBuilder.buildConstant(S32, ExpMask));
5229
5230 // Subtract the fp64 exponent bias (1023) to get the real exponent and
5231 // add the f16 bias (15) to get the biased exponent for the f16 format.
5232 E = MIRBuilder.buildAdd(
5233 S32, E, MIRBuilder.buildConstant(S32, -ExpBiasf64 + ExpBiasf16));
5234
5235 auto M = MIRBuilder.buildLShr(S32, UH, MIRBuilder.buildConstant(S32, 8));
5236 M = MIRBuilder.buildAnd(S32, M, MIRBuilder.buildConstant(S32, 0xffe));
5237
5238 auto MaskedSig = MIRBuilder.buildAnd(S32, UH,
5239 MIRBuilder.buildConstant(S32, 0x1ff));
5240 MaskedSig = MIRBuilder.buildOr(S32, MaskedSig, U);
5241
5242 auto Zero = MIRBuilder.buildConstant(S32, 0);
5243 auto SigCmpNE0 = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, MaskedSig, Zero);
5244 auto Lo40Set = MIRBuilder.buildZExt(S32, SigCmpNE0);
5245 M = MIRBuilder.buildOr(S32, M, Lo40Set);
5246
5247 // (M != 0 ? 0x0200 : 0) | 0x7c00;
5248 auto Bits0x200 = MIRBuilder.buildConstant(S32, 0x0200);
5249 auto CmpM_NE0 = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, M, Zero);
5250 auto SelectCC = MIRBuilder.buildSelect(S32, CmpM_NE0, Bits0x200, Zero);
5251
5252 auto Bits0x7c00 = MIRBuilder.buildConstant(S32, 0x7c00);
5253 auto I = MIRBuilder.buildOr(S32, SelectCC, Bits0x7c00);
5254
5255 // N = M | (E << 12);
5256 auto EShl12 = MIRBuilder.buildShl(S32, E, MIRBuilder.buildConstant(S32, 12));
5257 auto N = MIRBuilder.buildOr(S32, M, EShl12);
5258
5259 // B = clamp(1-E, 0, 13);
5260 auto One = MIRBuilder.buildConstant(S32, 1);
5261 auto OneSubExp = MIRBuilder.buildSub(S32, One, E);
5262 auto B = MIRBuilder.buildSMax(S32, OneSubExp, Zero);
5263 B = MIRBuilder.buildSMin(S32, B, MIRBuilder.buildConstant(S32, 13));
5264
5265 auto SigSetHigh = MIRBuilder.buildOr(S32, M,
5266 MIRBuilder.buildConstant(S32, 0x1000));
5267
5268 auto D = MIRBuilder.buildLShr(S32, SigSetHigh, B);
5269 auto D0 = MIRBuilder.buildShl(S32, D, B);
5270
5271 auto D0_NE_SigSetHigh = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1,
5272 D0, SigSetHigh);
5273 auto D1 = MIRBuilder.buildZExt(S32, D0_NE_SigSetHigh);
5274 D = MIRBuilder.buildOr(S32, D, D1);
5275
5276 auto CmpELtOne = MIRBuilder.buildICmp(CmpInst::ICMP_SLT, S1, E, One);
5277 auto V = MIRBuilder.buildSelect(S32, CmpELtOne, D, N);
5278
5279 auto VLow3 = MIRBuilder.buildAnd(S32, V, MIRBuilder.buildConstant(S32, 7));
5280 V = MIRBuilder.buildLShr(S32, V, MIRBuilder.buildConstant(S32, 2));
5281
5282 auto VLow3Eq3 = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1, VLow3,
5283 MIRBuilder.buildConstant(S32, 3));
5284 auto V0 = MIRBuilder.buildZExt(S32, VLow3Eq3);
5285
5286 auto VLow3Gt5 = MIRBuilder.buildICmp(CmpInst::ICMP_SGT, S1, VLow3,
5287 MIRBuilder.buildConstant(S32, 5));
5288 auto V1 = MIRBuilder.buildZExt(S32, VLow3Gt5);
5289
5290 V1 = MIRBuilder.buildOr(S32, V0, V1);
5291 V = MIRBuilder.buildAdd(S32, V, V1);
5292
5293 auto CmpEGt30 = MIRBuilder.buildICmp(CmpInst::ICMP_SGT, S1,
5294 E, MIRBuilder.buildConstant(S32, 30));
5295 V = MIRBuilder.buildSelect(S32, CmpEGt30,
5296 MIRBuilder.buildConstant(S32, 0x7c00), V);
5297
5298 auto CmpEGt1039 = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1,
5299 E, MIRBuilder.buildConstant(S32, 1039));
5300 V = MIRBuilder.buildSelect(S32, CmpEGt1039, I, V);
5301
5302 // Extract the sign bit.
5303 auto Sign = MIRBuilder.buildLShr(S32, UH, MIRBuilder.buildConstant(S32, 16));
5304 Sign = MIRBuilder.buildAnd(S32, Sign, MIRBuilder.buildConstant(S32, 0x8000));
5305
5306 // Insert the sign bit
5307 V = MIRBuilder.buildOr(S32, Sign, V);
5308
5309 MIRBuilder.buildTrunc(Dst, V);
5310 MI.eraseFromParent();
5311 return Legalized;
5312}
5313
5314LegalizerHelper::LegalizeResult
5315LegalizerHelper::lowerFPTRUNC(MachineInstr &MI) {
5316 Register Dst = MI.getOperand(0).getReg();
5317 Register Src = MI.getOperand(1).getReg();
5318
5319 LLT DstTy = MRI.getType(Dst);
5320 LLT SrcTy = MRI.getType(Src);
5321 const LLT S64 = LLT::scalar(64);
5322 const LLT S16 = LLT::scalar(16);
5323
5324 if (DstTy.getScalarType() == S16 && SrcTy.getScalarType() == S64)
5325 return lowerFPTRUNC_F64_TO_F16(MI);
5326
5327 return UnableToLegalize;
5328}
5329
5330// TODO: If RHS is a constant SelectionDAGBuilder expands this into a
5331// multiplication tree.
5332LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPOWI(MachineInstr &MI) {
5333 Register Dst = MI.getOperand(0).getReg();
5334 Register Src0 = MI.getOperand(1).getReg();
5335 Register Src1 = MI.getOperand(2).getReg();
5336 LLT Ty = MRI.getType(Dst);
5337
5338 auto CvtSrc1 = MIRBuilder.buildSITOFP(Ty, Src1);
5339 MIRBuilder.buildFPow(Dst, Src0, CvtSrc1, MI.getFlags());
5340 MI.eraseFromParent();
5341 return Legalized;
5342}
5343
5344static CmpInst::Predicate minMaxToCompare(unsigned Opc) {
5345 switch (Opc) {
5346 case TargetOpcode::G_SMIN:
5347 return CmpInst::ICMP_SLT;
5348 case TargetOpcode::G_SMAX:
5349 return CmpInst::ICMP_SGT;
5350 case TargetOpcode::G_UMIN:
5351 return CmpInst::ICMP_ULT;
5352 case TargetOpcode::G_UMAX:
5353 return CmpInst::ICMP_UGT;
5354 default:
5355 llvm_unreachable("not in integer min/max")::llvm::llvm_unreachable_internal("not in integer min/max", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 5355)
;
5356 }
5357}
5358
5359LegalizerHelper::LegalizeResult LegalizerHelper::lowerMinMax(MachineInstr &MI) {
5360 Register Dst = MI.getOperand(0).getReg();
5361 Register Src0 = MI.getOperand(1).getReg();
5362 Register Src1 = MI.getOperand(2).getReg();
5363
5364 const CmpInst::Predicate Pred = minMaxToCompare(MI.getOpcode());
5365 LLT CmpType = MRI.getType(Dst).changeElementSize(1);
5366
5367 auto Cmp = MIRBuilder.buildICmp(Pred, CmpType, Src0, Src1);
5368 MIRBuilder.buildSelect(Dst, Cmp, Src0, Src1);
5369
5370 MI.eraseFromParent();
5371 return Legalized;
5372}
5373
5374LegalizerHelper::LegalizeResult
5375LegalizerHelper::lowerFCopySign(MachineInstr &MI) {
5376 Register Dst = MI.getOperand(0).getReg();
5377 Register Src0 = MI.getOperand(1).getReg();
5378 Register Src1 = MI.getOperand(2).getReg();
5379
5380 const LLT Src0Ty = MRI.getType(Src0);
5381 const LLT Src1Ty = MRI.getType(Src1);
5382
5383 const int Src0Size = Src0Ty.getScalarSizeInBits();
5384 const int Src1Size = Src1Ty.getScalarSizeInBits();
5385
5386 auto SignBitMask = MIRBuilder.buildConstant(
5387 Src0Ty, APInt::getSignMask(Src0Size));
5388
5389 auto NotSignBitMask = MIRBuilder.buildConstant(
5390 Src0Ty, APInt::getLowBitsSet(Src0Size, Src0Size - 1));
5391
5392 auto And0 = MIRBuilder.buildAnd(Src0Ty, Src0, NotSignBitMask);
5393 MachineInstr *Or;
5394
5395 if (Src0Ty == Src1Ty) {
5396 auto And1 = MIRBuilder.buildAnd(Src1Ty, Src1, SignBitMask);
5397 Or = MIRBuilder.buildOr(Dst, And0, And1);
5398 } else if (Src0Size > Src1Size) {
5399 auto ShiftAmt = MIRBuilder.buildConstant(Src0Ty, Src0Size - Src1Size);
5400 auto Zext = MIRBuilder.buildZExt(Src0Ty, Src1);
5401 auto Shift = MIRBuilder.buildShl(Src0Ty, Zext, ShiftAmt);
5402 auto And1 = MIRBuilder.buildAnd(Src0Ty, Shift, SignBitMask);
5403 Or = MIRBuilder.buildOr(Dst, And0, And1);
5404 } else {
5405 auto ShiftAmt = MIRBuilder.buildConstant(Src1Ty, Src1Size - Src0Size);
5406 auto Shift = MIRBuilder.buildLShr(Src1Ty, Src1, ShiftAmt);
5407 auto Trunc = MIRBuilder.buildTrunc(Src0Ty, Shift);
5408 auto And1 = MIRBuilder.buildAnd(Src0Ty, Trunc, SignBitMask);
5409 Or = MIRBuilder.buildOr(Dst, And0, And1);
5410 }
5411
5412 // Be careful about setting nsz/nnan/ninf on every instruction, since the
5413 // constants are a nan and -0.0, but the final result should preserve
5414 // everything.
5415 if (unsigned Flags = MI.getFlags())
5416 Or->setFlags(Flags);
5417
5418 MI.eraseFromParent();
5419 return Legalized;
5420}
5421
5422LegalizerHelper::LegalizeResult
5423LegalizerHelper::lowerFMinNumMaxNum(MachineInstr &MI) {
5424 unsigned NewOp = MI.getOpcode() == TargetOpcode::G_FMINNUM ?
5425 TargetOpcode::G_FMINNUM_IEEE : TargetOpcode::G_FMAXNUM_IEEE;
5426
5427 Register Dst = MI.getOperand(0).getReg();
5428 Register Src0 = MI.getOperand(1).getReg();
5429 Register Src1 = MI.getOperand(2).getReg();
5430 LLT Ty = MRI.getType(Dst);
5431
5432 if (!MI.getFlag(MachineInstr::FmNoNans)) {
5433 // Insert canonicalizes if it's possible we need to quiet to get correct
5434 // sNaN behavior.
5435
5436 // Note this must be done here, and not as an optimization combine in the
5437 // absence of a dedicate quiet-snan instruction as we're using an
5438 // omni-purpose G_FCANONICALIZE.
5439 if (!isKnownNeverSNaN(Src0, MRI))
5440 Src0 = MIRBuilder.buildFCanonicalize(Ty, Src0, MI.getFlags()).getReg(0);
5441
5442 if (!isKnownNeverSNaN(Src1, MRI))
5443 Src1 = MIRBuilder.buildFCanonicalize(Ty, Src1, MI.getFlags()).getReg(0);
5444 }
5445
5446 // If there are no nans, it's safe to simply replace this with the non-IEEE
5447 // version.
5448 MIRBuilder.buildInstr(NewOp, {Dst}, {Src0, Src1}, MI.getFlags());
5449 MI.eraseFromParent();
5450 return Legalized;
5451}
5452
5453LegalizerHelper::LegalizeResult LegalizerHelper::lowerFMad(MachineInstr &MI) {
5454 // Expand G_FMAD a, b, c -> G_FADD (G_FMUL a, b), c
5455 Register DstReg = MI.getOperand(0).getReg();
5456 LLT Ty = MRI.getType(DstReg);
5457 unsigned Flags = MI.getFlags();
5458
5459 auto Mul = MIRBuilder.buildFMul(Ty, MI.getOperand(1), MI.getOperand(2),
5460 Flags);
5461 MIRBuilder.buildFAdd(DstReg, Mul, MI.getOperand(3), Flags);
5462 MI.eraseFromParent();
5463 return Legalized;
5464}
5465
5466LegalizerHelper::LegalizeResult
5467LegalizerHelper::lowerIntrinsicRound(MachineInstr &MI) {
5468 Register DstReg = MI.getOperand(0).getReg();
5469 Register X = MI.getOperand(1).getReg();
5470 const unsigned Flags = MI.getFlags();
5471 const LLT Ty = MRI.getType(DstReg);
5472 const LLT CondTy = Ty.changeElementSize(1);
5473
5474 // round(x) =>
5475 // t = trunc(x);
5476 // d = fabs(x - t);
5477 // o = copysign(1.0f, x);
5478 // return t + (d >= 0.5 ? o : 0.0);
5479
5480 auto T = MIRBuilder.buildIntrinsicTrunc(Ty, X, Flags);
5481
5482 auto Diff = MIRBuilder.buildFSub(Ty, X, T, Flags);
5483 auto AbsDiff = MIRBuilder.buildFAbs(Ty, Diff, Flags);
5484 auto Zero = MIRBuilder.buildFConstant(Ty, 0.0);
5485 auto One = MIRBuilder.buildFConstant(Ty, 1.0);
5486 auto Half = MIRBuilder.buildFConstant(Ty, 0.5);
5487 auto SignOne = MIRBuilder.buildFCopysign(Ty, One, X);
5488
5489 auto Cmp = MIRBuilder.buildFCmp(CmpInst::FCMP_OGE, CondTy, AbsDiff, Half,
5490 Flags);
5491 auto Sel = MIRBuilder.buildSelect(Ty, Cmp, SignOne, Zero, Flags);
5492
5493 MIRBuilder.buildFAdd(DstReg, T, Sel, Flags);
5494
5495 MI.eraseFromParent();
5496 return Legalized;
5497}
5498
5499LegalizerHelper::LegalizeResult
5500LegalizerHelper::lowerFFloor(MachineInstr &MI) {
5501 Register DstReg = MI.getOperand(0).getReg();
5502 Register SrcReg = MI.getOperand(1).getReg();
5503 unsigned Flags = MI.getFlags();
5504 LLT Ty = MRI.getType(DstReg);
5505 const LLT CondTy = Ty.changeElementSize(1);
5506
5507 // result = trunc(src);
5508 // if (src < 0.0 && src != result)
5509 // result += -1.0.
5510
5511 auto Trunc = MIRBuilder.buildIntrinsicTrunc(Ty, SrcReg, Flags);
5512 auto Zero = MIRBuilder.buildFConstant(Ty, 0.0);
5513
5514 auto Lt0 = MIRBuilder.buildFCmp(CmpInst::FCMP_OLT, CondTy,
5515 SrcReg, Zero, Flags);
5516 auto NeTrunc = MIRBuilder.buildFCmp(CmpInst::FCMP_ONE, CondTy,
5517 SrcReg, Trunc, Flags);
5518 auto And = MIRBuilder.buildAnd(CondTy, Lt0, NeTrunc);
5519 auto AddVal = MIRBuilder.buildSITOFP(Ty, And);
5520
5521 MIRBuilder.buildFAdd(DstReg, Trunc, AddVal, Flags);
5522 MI.eraseFromParent();
5523 return Legalized;
5524}
5525
5526LegalizerHelper::LegalizeResult
5527LegalizerHelper::lowerMergeValues(MachineInstr &MI) {
5528 const unsigned NumOps = MI.getNumOperands();
5529 Register DstReg = MI.getOperand(0).getReg();
5530 Register Src0Reg = MI.getOperand(1).getReg();
5531 LLT DstTy = MRI.getType(DstReg);
5532 LLT SrcTy = MRI.getType(Src0Reg);
5533 unsigned PartSize = SrcTy.getSizeInBits();
5534
5535 LLT WideTy = LLT::scalar(DstTy.getSizeInBits());
5536 Register ResultReg = MIRBuilder.buildZExt(WideTy, Src0Reg).getReg(0);
5537
5538 for (unsigned I = 2; I != NumOps; ++I) {
5539 const unsigned Offset = (I - 1) * PartSize;
5540
5541 Register SrcReg = MI.getOperand(I).getReg();
5542 auto ZextInput = MIRBuilder.buildZExt(WideTy, SrcReg);
5543
5544 Register NextResult = I + 1 == NumOps && WideTy == DstTy ? DstReg :
5545 MRI.createGenericVirtualRegister(WideTy);
5546
5547 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, Offset);
5548 auto Shl = MIRBuilder.buildShl(WideTy, ZextInput, ShiftAmt);
5549 MIRBuilder.buildOr(NextResult, ResultReg, Shl);
5550 ResultReg = NextResult;
5551 }
5552
5553 if (DstTy.isPointer()) {
5554 if (MIRBuilder.getDataLayout().isNonIntegralAddressSpace(
5555 DstTy.getAddressSpace())) {
5556 LLVM_DEBUG(dbgs() << "Not casting nonintegral address space\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << "Not casting nonintegral address space\n"
; } } while (false)
;
5557 return UnableToLegalize;
5558 }
5559
5560 MIRBuilder.buildIntToPtr(DstReg, ResultReg);
5561 }
5562
5563 MI.eraseFromParent();
5564 return Legalized;
5565}
5566
5567LegalizerHelper::LegalizeResult
5568LegalizerHelper::lowerUnmergeValues(MachineInstr &MI) {
5569 const unsigned NumDst = MI.getNumOperands() - 1;
5570 Register SrcReg = MI.getOperand(NumDst).getReg();
5571 Register Dst0Reg = MI.getOperand(0).getReg();
5572 LLT DstTy = MRI.getType(Dst0Reg);
5573 if (DstTy.isPointer())
5574 return UnableToLegalize; // TODO
5575
5576 SrcReg = coerceToScalar(SrcReg);
5577 if (!SrcReg)
5578 return UnableToLegalize;
5579
5580 // Expand scalarizing unmerge as bitcast to integer and shift.
5581 LLT IntTy = MRI.getType(SrcReg);
5582
5583 MIRBuilder.buildTrunc(Dst0Reg, SrcReg);
5584
5585 const unsigned DstSize = DstTy.getSizeInBits();
5586 unsigned Offset = DstSize;
5587 for (unsigned I = 1; I != NumDst; ++I, Offset += DstSize) {
5588 auto ShiftAmt = MIRBuilder.buildConstant(IntTy, Offset);
5589 auto Shift = MIRBuilder.buildLShr(IntTy, SrcReg, ShiftAmt);
5590 MIRBuilder.buildTrunc(MI.getOperand(I), Shift);
5591 }
5592
5593 MI.eraseFromParent();
5594 return Legalized;
5595}
5596
5597/// Lower a vector extract or insert by writing the vector to a stack temporary
5598/// and reloading the element or vector.
5599///
5600/// %dst = G_EXTRACT_VECTOR_ELT %vec, %idx
5601/// =>
5602/// %stack_temp = G_FRAME_INDEX
5603/// G_STORE %vec, %stack_temp
5604/// %idx = clamp(%idx, %vec.getNumElements())
5605/// %element_ptr = G_PTR_ADD %stack_temp, %idx
5606/// %dst = G_LOAD %element_ptr
5607LegalizerHelper::LegalizeResult
5608LegalizerHelper::lowerExtractInsertVectorElt(MachineInstr &MI) {
5609 Register DstReg = MI.getOperand(0).getReg();
5610 Register SrcVec = MI.getOperand(1).getReg();
5611 Register InsertVal;
5612 if (MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT)
5613 InsertVal = MI.getOperand(2).getReg();
5614
5615 Register Idx = MI.getOperand(MI.getNumOperands() - 1).getReg();
5616
5617 LLT VecTy = MRI.getType(SrcVec);
5618 LLT EltTy = VecTy.getElementType();
5619 if (!EltTy.isByteSized()) { // Not implemented.
5620 LLVM_DEBUG(dbgs() << "Can't handle non-byte element vectors yet\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << "Can't handle non-byte element vectors yet\n"
; } } while (false)
;
5621 return UnableToLegalize;
5622 }
5623
5624 unsigned EltBytes = EltTy.getSizeInBytes();
5625 Align VecAlign = getStackTemporaryAlignment(VecTy);
5626 Align EltAlign;
5627
5628 MachinePointerInfo PtrInfo;
5629 auto StackTemp = createStackTemporary(TypeSize::Fixed(VecTy.getSizeInBytes()),
5630 VecAlign, PtrInfo);
5631 MIRBuilder.buildStore(SrcVec, StackTemp, PtrInfo, VecAlign);
5632
5633 // Get the pointer to the element, and be sure not to hit undefined behavior
5634 // if the index is out of bounds.
5635 Register EltPtr = getVectorElementPointer(StackTemp.getReg(0), VecTy, Idx);
5636
5637 int64_t IdxVal;
5638 if (mi_match(Idx, MRI, m_ICst(IdxVal))) {
5639 int64_t Offset = IdxVal * EltBytes;
5640 PtrInfo = PtrInfo.getWithOffset(Offset);
5641 EltAlign = commonAlignment(VecAlign, Offset);
5642 } else {
5643 // We lose information with a variable offset.
5644 EltAlign = getStackTemporaryAlignment(EltTy);
5645 PtrInfo = MachinePointerInfo(MRI.getType(EltPtr).getAddressSpace());
5646 }
5647
5648 if (InsertVal) {
5649 // Write the inserted element
5650 MIRBuilder.buildStore(InsertVal, EltPtr, PtrInfo, EltAlign);
5651
5652 // Reload the whole vector.
5653 MIRBuilder.buildLoad(DstReg, StackTemp, PtrInfo, VecAlign);
5654 } else {
5655 MIRBuilder.buildLoad(DstReg, EltPtr, PtrInfo, EltAlign);
5656 }
5657
5658 MI.eraseFromParent();
5659 return Legalized;
5660}
5661
5662LegalizerHelper::LegalizeResult
5663LegalizerHelper::lowerShuffleVector(MachineInstr &MI) {
5664 Register DstReg = MI.getOperand(0).getReg();
5665 Register Src0Reg = MI.getOperand(1).getReg();
5666 Register Src1Reg = MI.getOperand(2).getReg();
5667 LLT Src0Ty = MRI.getType(Src0Reg);
5668 LLT DstTy = MRI.getType(DstReg);
5669 LLT IdxTy = LLT::scalar(32);
5670
5671 ArrayRef<int> Mask = MI.getOperand(3).getShuffleMask();
5672
5673 if (DstTy.isScalar()) {
5674 if (Src0Ty.isVector())
5675 return UnableToLegalize;
5676
5677 // This is just a SELECT.
5678 assert(Mask.size() == 1 && "Expected a single mask element")((Mask.size() == 1 && "Expected a single mask element"
) ? static_cast<void> (0) : __assert_fail ("Mask.size() == 1 && \"Expected a single mask element\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 5678, __PRETTY_FUNCTION__))
;
5679 Register Val;
5680 if (Mask[0] < 0 || Mask[0] > 1)
5681 Val = MIRBuilder.buildUndef(DstTy).getReg(0);
5682 else
5683 Val = Mask[0] == 0 ? Src0Reg : Src1Reg;
5684 MIRBuilder.buildCopy(DstReg, Val);
5685 MI.eraseFromParent();
5686 return Legalized;
5687 }
5688
5689 Register Undef;
5690 SmallVector<Register, 32> BuildVec;
5691 LLT EltTy = DstTy.getElementType();
5692
5693 for (int Idx : Mask) {
5694 if (Idx < 0) {
5695 if (!Undef.isValid())
5696 Undef = MIRBuilder.buildUndef(EltTy).getReg(0);
5697 BuildVec.push_back(Undef);
5698 continue;
5699 }
5700
5701 if (Src0Ty.isScalar()) {
5702 BuildVec.push_back(Idx == 0 ? Src0Reg : Src1Reg);
5703 } else {
5704 int NumElts = Src0Ty.getNumElements();
5705 Register SrcVec = Idx < NumElts ? Src0Reg : Src1Reg;
5706 int ExtractIdx = Idx < NumElts ? Idx : Idx - NumElts;
5707 auto IdxK = MIRBuilder.buildConstant(IdxTy, ExtractIdx);
5708 auto Extract = MIRBuilder.buildExtractVectorElement(EltTy, SrcVec, IdxK);
5709 BuildVec.push_back(Extract.getReg(0));
5710 }
5711 }
5712
5713 MIRBuilder.buildBuildVector(DstReg, BuildVec);
5714 MI.eraseFromParent();
5715 return Legalized;
5716}
5717
5718LegalizerHelper::LegalizeResult
5719LegalizerHelper::lowerDynStackAlloc(MachineInstr &MI) {
5720 const auto &MF = *MI.getMF();
5721 const auto &TFI = *MF.getSubtarget().getFrameLowering();
5722 if (TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp)
5723 return UnableToLegalize;
5724
5725 Register Dst = MI.getOperand(0).getReg();
5726 Register AllocSize = MI.getOperand(1).getReg();
5727 Align Alignment = assumeAligned(MI.getOperand(2).getImm());
5728
5729 LLT PtrTy = MRI.getType(Dst);
5730 LLT IntPtrTy = LLT::scalar(PtrTy.getSizeInBits());
5731
5732 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
5733 auto SPTmp = MIRBuilder.buildCopy(PtrTy, SPReg);
5734 SPTmp = MIRBuilder.buildCast(IntPtrTy, SPTmp);
5735
5736 // Subtract the final alloc from the SP. We use G_PTRTOINT here so we don't
5737 // have to generate an extra instruction to negate the alloc and then use
5738 // G_PTR_ADD to add the negative offset.
5739 auto Alloc = MIRBuilder.buildSub(IntPtrTy, SPTmp, AllocSize);
5740 if (Alignment > Align(1)) {
5741 APInt AlignMask(IntPtrTy.getSizeInBits(), Alignment.value(), true);
5742 AlignMask.negate();
5743 auto AlignCst = MIRBuilder.buildConstant(IntPtrTy, AlignMask);
5744 Alloc = MIRBuilder.buildAnd(IntPtrTy, Alloc, AlignCst);
5745 }
5746
5747 SPTmp = MIRBuilder.buildCast(PtrTy, Alloc);
5748 MIRBuilder.buildCopy(SPReg, SPTmp);
5749 MIRBuilder.buildCopy(Dst, SPTmp);
5750
5751 MI.eraseFromParent();
5752 return Legalized;
5753}
5754
5755LegalizerHelper::LegalizeResult
5756LegalizerHelper::lowerExtract(MachineInstr &MI) {
5757 Register Dst = MI.getOperand(0).getReg();
5758 Register Src = MI.getOperand(1).getReg();
5759 unsigned Offset = MI.getOperand(2).getImm();
5760
5761 LLT DstTy = MRI.getType(Dst);
5762 LLT SrcTy = MRI.getType(Src);
5763
5764 if (DstTy.isScalar() &&
5765 (SrcTy.isScalar() ||
5766 (SrcTy.isVector() && DstTy == SrcTy.getElementType()))) {
5767 LLT SrcIntTy = SrcTy;
5768 if (!SrcTy.isScalar()) {
5769 SrcIntTy = LLT::scalar(SrcTy.getSizeInBits());
5770 Src = MIRBuilder.buildBitcast(SrcIntTy, Src).getReg(0);
5771 }
5772
5773 if (Offset == 0)
5774 MIRBuilder.buildTrunc(Dst, Src);
5775 else {
5776 auto ShiftAmt = MIRBuilder.buildConstant(SrcIntTy, Offset);
5777 auto Shr = MIRBuilder.buildLShr(SrcIntTy, Src, ShiftAmt);
5778 MIRBuilder.buildTrunc(Dst, Shr);
5779 }
5780
5781 MI.eraseFromParent();
5782 return Legalized;
5783 }
5784
5785 return UnableToLegalize;
5786}
5787
5788LegalizerHelper::LegalizeResult LegalizerHelper::lowerInsert(MachineInstr &MI) {
5789 Register Dst = MI.getOperand(0).getReg();
5790 Register Src = MI.getOperand(1).getReg();
5791 Register InsertSrc = MI.getOperand(2).getReg();
5792 uint64_t Offset = MI.getOperand(3).getImm();
5793
5794 LLT DstTy = MRI.getType(Src);
5795 LLT InsertTy = MRI.getType(InsertSrc);
5796
5797 if (InsertTy.isVector() ||
5798 (DstTy.isVector() && DstTy.getElementType() != InsertTy))
5799 return UnableToLegalize;
5800
5801 const DataLayout &DL = MIRBuilder.getDataLayout();
5802 if ((DstTy.isPointer() &&
5803 DL.isNonIntegralAddressSpace(DstTy.getAddressSpace())) ||
5804 (InsertTy.isPointer() &&
5805 DL.isNonIntegralAddressSpace(InsertTy.getAddressSpace()))) {
5806 LLVM_DEBUG(dbgs() << "Not casting non-integral address space integer\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << "Not casting non-integral address space integer\n"
; } } while (false)
;
5807 return UnableToLegalize;
5808 }
5809
5810 LLT IntDstTy = DstTy;
5811
5812 if (!DstTy.isScalar()) {
5813 IntDstTy = LLT::scalar(DstTy.getSizeInBits());
5814 Src = MIRBuilder.buildCast(IntDstTy, Src).getReg(0);
5815 }
5816
5817 if (!InsertTy.isScalar()) {
5818 const LLT IntInsertTy = LLT::scalar(InsertTy.getSizeInBits());
5819 InsertSrc = MIRBuilder.buildPtrToInt(IntInsertTy, InsertSrc).getReg(0);
5820 }
5821
5822 Register ExtInsSrc = MIRBuilder.buildZExt(IntDstTy, InsertSrc).getReg(0);
5823 if (Offset != 0) {
5824 auto ShiftAmt = MIRBuilder.buildConstant(IntDstTy, Offset);
5825 ExtInsSrc = MIRBuilder.buildShl(IntDstTy, ExtInsSrc, ShiftAmt).getReg(0);
5826 }
5827
5828 APInt MaskVal = APInt::getBitsSetWithWrap(
5829 DstTy.getSizeInBits(), Offset + InsertTy.getSizeInBits(), Offset);
5830
5831 auto Mask = MIRBuilder.buildConstant(IntDstTy, MaskVal);
5832 auto MaskedSrc = MIRBuilder.buildAnd(IntDstTy, Src, Mask);
5833 auto Or = MIRBuilder.buildOr(IntDstTy, MaskedSrc, ExtInsSrc);
5834
5835 MIRBuilder.buildCast(Dst, Or);
5836 MI.eraseFromParent();
5837 return Legalized;
5838}
5839
5840LegalizerHelper::LegalizeResult
5841LegalizerHelper::lowerSADDO_SSUBO(MachineInstr &MI) {
5842 Register Dst0 = MI.getOperand(0).getReg();
5843 Register Dst1 = MI.getOperand(1).getReg();
5844 Register LHS = MI.getOperand(2).getReg();
5845 Register RHS = MI.getOperand(3).getReg();
5846 const bool IsAdd = MI.getOpcode() == TargetOpcode::G_SADDO;
5847
5848 LLT Ty = MRI.getType(Dst0);
5849 LLT BoolTy = MRI.getType(Dst1);
5850
5851 if (IsAdd)
5852 MIRBuilder.buildAdd(Dst0, LHS, RHS);
5853 else
5854 MIRBuilder.buildSub(Dst0, LHS, RHS);
5855
5856 // TODO: If SADDSAT/SSUBSAT is legal, compare results to detect overflow.
5857
5858 auto Zero = MIRBuilder.buildConstant(Ty, 0);
5859
5860 // For an addition, the result should be less than one of the operands (LHS)
5861 // if and only if the other operand (RHS) is negative, otherwise there will
5862 // be overflow.
5863 // For a subtraction, the result should be less than one of the operands
5864 // (LHS) if and only if the other operand (RHS) is (non-zero) positive,
5865 // otherwise there will be overflow.
5866 auto ResultLowerThanLHS =
5867 MIRBuilder.buildICmp(CmpInst::ICMP_SLT, BoolTy, Dst0, LHS);
5868 auto ConditionRHS = MIRBuilder.buildICmp(
5869 IsAdd ? CmpInst::ICMP_SLT : CmpInst::ICMP_SGT, BoolTy, RHS, Zero);
5870
5871 MIRBuilder.buildXor(Dst1, ConditionRHS, ResultLowerThanLHS);
5872 MI.eraseFromParent();
5873 return Legalized;
5874}
5875
5876LegalizerHelper::LegalizeResult
5877LegalizerHelper::lowerAddSubSatToMinMax(MachineInstr &MI) {
5878 Register Res = MI.getOperand(0).getReg();
5879 Register LHS = MI.getOperand(1).getReg();
5880 Register RHS = MI.getOperand(2).getReg();
5881 LLT Ty = MRI.getType(Res);
5882 bool IsSigned;
5883 bool IsAdd;
5884 unsigned BaseOp;
5885 switch (MI.getOpcode()) {
5886 default:
5887 llvm_unreachable("unexpected addsat/subsat opcode")::llvm::llvm_unreachable_internal("unexpected addsat/subsat opcode"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 5887)
;
5888 case TargetOpcode::G_UADDSAT:
5889 IsSigned = false;
5890 IsAdd = true;
5891 BaseOp = TargetOpcode::G_ADD;
5892 break;
5893 case TargetOpcode::G_SADDSAT:
5894 IsSigned = true;
5895 IsAdd = true;
5896 BaseOp = TargetOpcode::G_ADD;
5897 break;
5898 case TargetOpcode::G_USUBSAT:
5899 IsSigned = false;
5900 IsAdd = false;
5901 BaseOp = TargetOpcode::G_SUB;
5902 break;
5903 case TargetOpcode::G_SSUBSAT:
5904 IsSigned = true;
5905 IsAdd = false;
5906 BaseOp = TargetOpcode::G_SUB;
5907 break;
5908 }
5909
5910 if (IsSigned) {
5911 // sadd.sat(a, b) ->
5912 // hi = 0x7fffffff - smax(a, 0)
5913 // lo = 0x80000000 - smin(a, 0)
5914 // a + smin(smax(lo, b), hi)
5915 // ssub.sat(a, b) ->
5916 // lo = smax(a, -1) - 0x7fffffff
5917 // hi = smin(a, -1) - 0x80000000
5918 // a - smin(smax(lo, b), hi)
5919 // TODO: AMDGPU can use a "median of 3" instruction here:
5920 // a +/- med3(lo, b, hi)
5921 uint64_t NumBits = Ty.getScalarSizeInBits();
5922 auto MaxVal =
5923 MIRBuilder.buildConstant(Ty, APInt::getSignedMaxValue(NumBits));
5924 auto MinVal =
5925 MIRBuilder.buildConstant(Ty, APInt::getSignedMinValue(NumBits));
5926 MachineInstrBuilder Hi, Lo;
5927 if (IsAdd) {
5928 auto Zero = MIRBuilder.buildConstant(Ty, 0);
5929 Hi = MIRBuilder.buildSub(Ty, MaxVal, MIRBuilder.buildSMax(Ty, LHS, Zero));
5930 Lo = MIRBuilder.buildSub(Ty, MinVal, MIRBuilder.buildSMin(Ty, LHS, Zero));
5931 } else {
5932 auto NegOne = MIRBuilder.buildConstant(Ty, -1);
5933 Lo = MIRBuilder.buildSub(Ty, MIRBuilder.buildSMax(Ty, LHS, NegOne),
5934 MaxVal);
5935 Hi = MIRBuilder.buildSub(Ty, MIRBuilder.buildSMin(Ty, LHS, NegOne),
5936 MinVal);
5937 }
5938 auto RHSClamped =
5939 MIRBuilder.buildSMin(Ty, MIRBuilder.buildSMax(Ty, Lo, RHS), Hi);
5940 MIRBuilder.buildInstr(BaseOp, {Res}, {LHS, RHSClamped});
5941 } else {
5942 // uadd.sat(a, b) -> a + umin(~a, b)
5943 // usub.sat(a, b) -> a - umin(a, b)
5944 Register Not = IsAdd ? MIRBuilder.buildNot(Ty, LHS).getReg(0) : LHS;
5945 auto Min = MIRBuilder.buildUMin(Ty, Not, RHS);
5946 MIRBuilder.buildInstr(BaseOp, {Res}, {LHS, Min});
5947 }
5948
5949 MI.eraseFromParent();
5950 return Legalized;
5951}
5952
5953LegalizerHelper::LegalizeResult
5954LegalizerHelper::lowerAddSubSatToAddoSubo(MachineInstr &MI) {
5955 Register Res = MI.getOperand(0).getReg();
5956 Register LHS = MI.getOperand(1).getReg();
5957 Register RHS = MI.getOperand(2).getReg();
5958 LLT Ty = MRI.getType(Res);
5959 LLT BoolTy = Ty.changeElementSize(1);
5960 bool IsSigned;
5961 bool IsAdd;
5962 unsigned OverflowOp;
5963 switch (MI.getOpcode()) {
5964 default:
5965 llvm_unreachable("unexpected addsat/subsat opcode")::llvm::llvm_unreachable_internal("unexpected addsat/subsat opcode"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 5965)
;
5966 case TargetOpcode::G_UADDSAT:
5967 IsSigned = false;
5968 IsAdd = true;
5969 OverflowOp = TargetOpcode::G_UADDO;
5970 break;
5971 case TargetOpcode::G_SADDSAT:
5972 IsSigned = true;
5973 IsAdd = true;
5974 OverflowOp = TargetOpcode::G_SADDO;
5975 break;
5976 case TargetOpcode::G_USUBSAT:
5977 IsSigned = false;
5978 IsAdd = false;
5979 OverflowOp = TargetOpcode::G_USUBO;
5980 break;
5981 case TargetOpcode::G_SSUBSAT:
5982 IsSigned = true;
5983 IsAdd = false;
5984 OverflowOp = TargetOpcode::G_SSUBO;
5985 break;
5986 }
5987
5988 auto OverflowRes =
5989 MIRBuilder.buildInstr(OverflowOp, {Ty, BoolTy}, {LHS, RHS});
5990 Register Tmp = OverflowRes.getReg(0);
5991 Register Ov = OverflowRes.getReg(1);
5992 MachineInstrBuilder Clamp;
5993 if (IsSigned) {
5994 // sadd.sat(a, b) ->
5995 // {tmp, ov} = saddo(a, b)
5996 // ov ? (tmp >>s 31) + 0x80000000 : r
5997 // ssub.sat(a, b) ->
5998 // {tmp, ov} = ssubo(a, b)
5999 // ov ? (tmp >>s 31) + 0x80000000 : r
6000 uint64_t NumBits = Ty.getScalarSizeInBits();
6001 auto ShiftAmount = MIRBuilder.buildConstant(Ty, NumBits - 1);
6002 auto Sign = MIRBuilder.buildAShr(Ty, Tmp, ShiftAmount);
6003 auto MinVal =
6004 MIRBuilder.buildConstant(Ty, APInt::getSignedMinValue(NumBits));
6005 Clamp = MIRBuilder.buildAdd(Ty, Sign, MinVal);
6006 } else {
6007 // uadd.sat(a, b) ->
6008 // {tmp, ov} = uaddo(a, b)
6009 // ov ? 0xffffffff : tmp
6010 // usub.sat(a, b) ->
6011 // {tmp, ov} = usubo(a, b)
6012 // ov ? 0 : tmp
6013 Clamp = MIRBuilder.buildConstant(Ty, IsAdd ? -1 : 0);
6014 }
6015 MIRBuilder.buildSelect(Res, Ov, Clamp, Tmp);
6016
6017 MI.eraseFromParent();
6018 return Legalized;
6019}
6020
6021LegalizerHelper::LegalizeResult
6022LegalizerHelper::lowerShlSat(MachineInstr &MI) {
6023 assert((MI.getOpcode() == TargetOpcode::G_SSHLSAT ||(((MI.getOpcode() == TargetOpcode::G_SSHLSAT || MI.getOpcode(
) == TargetOpcode::G_USHLSAT) && "Expected shlsat opcode!"
) ? static_cast<void> (0) : __assert_fail ("(MI.getOpcode() == TargetOpcode::G_SSHLSAT || MI.getOpcode() == TargetOpcode::G_USHLSAT) && \"Expected shlsat opcode!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 6025, __PRETTY_FUNCTION__))
6024 MI.getOpcode() == TargetOpcode::G_USHLSAT) &&(((MI.getOpcode() == TargetOpcode::G_SSHLSAT || MI.getOpcode(
) == TargetOpcode::G_USHLSAT) && "Expected shlsat opcode!"
) ? static_cast<void> (0) : __assert_fail ("(MI.getOpcode() == TargetOpcode::G_SSHLSAT || MI.getOpcode() == TargetOpcode::G_USHLSAT) && \"Expected shlsat opcode!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 6025, __PRETTY_FUNCTION__))
6025 "Expected shlsat opcode!")(((MI.getOpcode() == TargetOpcode::G_SSHLSAT || MI.getOpcode(
) == TargetOpcode::G_USHLSAT) && "Expected shlsat opcode!"
) ? static_cast<void> (0) : __assert_fail ("(MI.getOpcode() == TargetOpcode::G_SSHLSAT || MI.getOpcode() == TargetOpcode::G_USHLSAT) && \"Expected shlsat opcode!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 6025, __PRETTY_FUNCTION__))
;
6026 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SSHLSAT;
6027 Register Res = MI.getOperand(0).getReg();
6028 Register LHS = MI.getOperand(1).getReg();
6029 Register RHS = MI.getOperand(2).getReg();
6030 LLT Ty = MRI.getType(Res);
6031 LLT BoolTy = Ty.changeElementSize(1);
6032
6033 unsigned BW = Ty.getScalarSizeInBits();
6034 auto Result = MIRBuilder.buildShl(Ty, LHS, RHS);
6035 auto Orig = IsSigned ? MIRBuilder.buildAShr(Ty, Result, RHS)
6036 : MIRBuilder.buildLShr(Ty, Result, RHS);
6037
6038 MachineInstrBuilder SatVal;
6039 if (IsSigned) {
6040 auto SatMin = MIRBuilder.buildConstant(Ty, APInt::getSignedMinValue(BW));
6041 auto SatMax = MIRBuilder.buildConstant(Ty, APInt::getSignedMaxValue(BW));
6042 auto Cmp = MIRBuilder.buildICmp(CmpInst::ICMP_SLT, BoolTy, LHS,
6043 MIRBuilder.buildConstant(Ty, 0));
6044 SatVal = MIRBuilder.buildSelect(Ty, Cmp, SatMin, SatMax);
6045 } else {
6046 SatVal = MIRBuilder.buildConstant(Ty, APInt::getMaxValue(BW));
6047 }
6048 auto Ov = MIRBuilder.buildICmp(CmpInst::ICMP_NE, Ty, LHS, Orig);
6049 MIRBuilder.buildSelect(Res, Ov, SatVal, Result);
6050
6051 MI.eraseFromParent();
6052 return Legalized;
6053}
6054
6055LegalizerHelper::LegalizeResult
6056LegalizerHelper::lowerBswap(MachineInstr &MI) {
6057 Register Dst = MI.getOperand(0).getReg();
6058 Register Src = MI.getOperand(1).getReg();
6059 const LLT Ty = MRI.getType(Src);
6060 unsigned SizeInBytes = (Ty.getScalarSizeInBits() + 7) / 8;
6061 unsigned BaseShiftAmt = (SizeInBytes - 1) * 8;
6062
6063 // Swap most and least significant byte, set remaining bytes in Res to zero.
6064 auto ShiftAmt = MIRBuilder.buildConstant(Ty, BaseShiftAmt);
6065 auto LSByteShiftedLeft = MIRBuilder.buildShl(Ty, Src, ShiftAmt);
6066 auto MSByteShiftedRight = MIRBuilder.buildLShr(Ty, Src, ShiftAmt);
6067 auto Res = MIRBuilder.buildOr(Ty, MSByteShiftedRight, LSByteShiftedLeft);
6068
6069 // Set i-th high/low byte in Res to i-th low/high byte from Src.
6070 for (unsigned i = 1; i < SizeInBytes / 2; ++i) {
6071 // AND with Mask leaves byte i unchanged and sets remaining bytes to 0.
6072 APInt APMask(SizeInBytes * 8, 0xFF << (i * 8));
6073 auto Mask = MIRBuilder.buildConstant(Ty, APMask);
6074 auto ShiftAmt = MIRBuilder.buildConstant(Ty, BaseShiftAmt - 16 * i);
6075 // Low byte shifted left to place of high byte: (Src & Mask) << ShiftAmt.
6076 auto LoByte = MIRBuilder.buildAnd(Ty, Src, Mask);
6077 auto LoShiftedLeft = MIRBuilder.buildShl(Ty, LoByte, ShiftAmt);
6078 Res = MIRBuilder.buildOr(Ty, Res, LoShiftedLeft);
6079 // High byte shifted right to place of low byte: (Src >> ShiftAmt) & Mask.
6080 auto SrcShiftedRight = MIRBuilder.buildLShr(Ty, Src, ShiftAmt);
6081 auto HiShiftedRight = MIRBuilder.buildAnd(Ty, SrcShiftedRight, Mask);
6082 Res = MIRBuilder.buildOr(Ty, Res, HiShiftedRight);
6083 }
6084 Res.getInstr()->getOperand(0).setReg(Dst);
6085
6086 MI.eraseFromParent();
6087 return Legalized;
6088}
6089
6090//{ (Src & Mask) >> N } | { (Src << N) & Mask }
6091static MachineInstrBuilder SwapN(unsigned N, DstOp Dst, MachineIRBuilder &B,
6092 MachineInstrBuilder Src, APInt Mask) {
6093 const LLT Ty = Dst.getLLTTy(*B.getMRI());
6094 MachineInstrBuilder C_N = B.buildConstant(Ty, N);
6095 MachineInstrBuilder MaskLoNTo0 = B.buildConstant(Ty, Mask);
6096 auto LHS = B.buildLShr(Ty, B.buildAnd(Ty, Src, MaskLoNTo0), C_N);
6097 auto RHS = B.buildAnd(Ty, B.buildShl(Ty, Src, C_N), MaskLoNTo0);
6098 return B.buildOr(Dst, LHS, RHS);
6099}
6100
6101LegalizerHelper::LegalizeResult
6102LegalizerHelper::lowerBitreverse(MachineInstr &MI) {
6103 Register Dst = MI.getOperand(0).getReg();
6104 Register Src = MI.getOperand(1).getReg();
6105 const LLT Ty = MRI.getType(Src);
6106 unsigned Size = Ty.getSizeInBits();
6107
6108 MachineInstrBuilder BSWAP =
6109 MIRBuilder.buildInstr(TargetOpcode::G_BSWAP, {Ty}, {Src});
6110
6111 // swap high and low 4 bits in 8 bit blocks 7654|3210 -> 3210|7654
6112 // [(val & 0xF0F0F0F0) >> 4] | [(val & 0x0F0F0F0F) << 4]
6113 // -> [(val & 0xF0F0F0F0) >> 4] | [(val << 4) & 0xF0F0F0F0]
6114 MachineInstrBuilder Swap4 =
6115 SwapN(4, Ty, MIRBuilder, BSWAP, APInt::getSplat(Size, APInt(8, 0xF0)));
6116
6117 // swap high and low 2 bits in 4 bit blocks 32|10 76|54 -> 10|32 54|76
6118 // [(val & 0xCCCCCCCC) >> 2] & [(val & 0x33333333) << 2]
6119 // -> [(val & 0xCCCCCCCC) >> 2] & [(val << 2) & 0xCCCCCCCC]
6120 MachineInstrBuilder Swap2 =
6121 SwapN(2, Ty, MIRBuilder, Swap4, APInt::getSplat(Size, APInt(8, 0xCC)));
6122
6123 // swap high and low 1 bit in 2 bit blocks 1|0 3|2 5|4 7|6 -> 0|1 2|3 4|5 6|7
6124 // [(val & 0xAAAAAAAA) >> 1] & [(val & 0x55555555) << 1]
6125 // -> [(val & 0xAAAAAAAA) >> 1] & [(val << 1) & 0xAAAAAAAA]
6126 SwapN(1, Dst, MIRBuilder, Swap2, APInt::getSplat(Size, APInt(8, 0xAA)));
6127
6128 MI.eraseFromParent();
6129 return Legalized;
6130}
6131
6132LegalizerHelper::LegalizeResult
6133LegalizerHelper::lowerReadWriteRegister(MachineInstr &MI) {
6134 MachineFunction &MF = MIRBuilder.getMF();
6135
6136 bool IsRead = MI.getOpcode() == TargetOpcode::G_READ_REGISTER;
6137 int NameOpIdx = IsRead ? 1 : 0;
6138 int ValRegIndex = IsRead ? 0 : 1;
6139
6140 Register ValReg = MI.getOperand(ValRegIndex).getReg();
6141 const LLT Ty = MRI.getType(ValReg);
6142 const MDString *RegStr = cast<MDString>(
6143 cast<MDNode>(MI.getOperand(NameOpIdx).getMetadata())->getOperand(0));
6144
6145 Register PhysReg = TLI.getRegisterByName(RegStr->getString().data(), Ty, MF);
6146 if (!PhysReg.isValid())
6147 return UnableToLegalize;
6148
6149 if (IsRead)
6150 MIRBuilder.buildCopy(ValReg, PhysReg);
6151 else
6152 MIRBuilder.buildCopy(PhysReg, ValReg);
6153
6154 MI.eraseFromParent();
6155 return Legalized;
6156}
6157
6158LegalizerHelper::LegalizeResult
6159LegalizerHelper::lowerSMULH_UMULH(MachineInstr &MI) {
6160 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SMULH;
6161 unsigned ExtOp = IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
6162 Register Result = MI.getOperand(0).getReg();
6163 LLT OrigTy = MRI.getType(Result);
6164 auto SizeInBits = OrigTy.getScalarSizeInBits();
6165 LLT WideTy = OrigTy.changeElementSize(SizeInBits * 2);
6166
6167 auto LHS = MIRBuilder.buildInstr(ExtOp, {WideTy}, {MI.getOperand(1)});
6168 auto RHS = MIRBuilder.buildInstr(ExtOp, {WideTy}, {MI.getOperand(2)});
6169 auto Mul = MIRBuilder.buildMul(WideTy, LHS, RHS);
6170 unsigned ShiftOp = IsSigned ? TargetOpcode::G_ASHR : TargetOpcode::G_LSHR;
6171
6172 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, SizeInBits);
6173 auto Shifted = MIRBuilder.buildInstr(ShiftOp, {WideTy}, {Mul, ShiftAmt});
6174 MIRBuilder.buildTrunc(Result, Shifted);
6175
6176 MI.eraseFromParent();
6177 return Legalized;
6178}