Bug Summary

File:llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
Warning:line 2888, column 7
Value stored to 'NumLeftover' is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name LegalizerHelper.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -mframe-pointer=none -fmath-errno -fno-rounding-math -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-10/lib/clang/10.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/build-llvm/lib/CodeGen/GlobalISel -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/build-llvm/include -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-10/lib/clang/10.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/build-llvm/lib/CodeGen/GlobalISel -fdebug-prefix-map=/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2019-12-07-102640-14763-1 -x c++ /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
1//===-- llvm/CodeGen/GlobalISel/LegalizerHelper.cpp -----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This file implements the LegalizerHelper class to legalize
10/// individual instructions and the LegalizeMachineIR wrapper pass for the
11/// primary legalization.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
16#include "llvm/CodeGen/GlobalISel/CallLowering.h"
17#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
18#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
19#include "llvm/CodeGen/MachineRegisterInfo.h"
20#include "llvm/CodeGen/TargetFrameLowering.h"
21#include "llvm/CodeGen/TargetInstrInfo.h"
22#include "llvm/CodeGen/TargetLowering.h"
23#include "llvm/CodeGen/TargetSubtargetInfo.h"
24#include "llvm/Support/Debug.h"
25#include "llvm/Support/MathExtras.h"
26#include "llvm/Support/raw_ostream.h"
27
28#define DEBUG_TYPE"legalizer" "legalizer"
29
30using namespace llvm;
31using namespace LegalizeActions;
32
33/// Try to break down \p OrigTy into \p NarrowTy sized pieces.
34///
35/// Returns the number of \p NarrowTy elements needed to reconstruct \p OrigTy,
36/// with any leftover piece as type \p LeftoverTy
37///
38/// Returns -1 in the first element of the pair if the breakdown is not
39/// satisfiable.
40static std::pair<int, int>
41getNarrowTypeBreakDown(LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy) {
42 assert(!LeftoverTy.isValid() && "this is an out argument")((!LeftoverTy.isValid() && "this is an out argument")
? static_cast<void> (0) : __assert_fail ("!LeftoverTy.isValid() && \"this is an out argument\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 42, __PRETTY_FUNCTION__))
;
43
44 unsigned Size = OrigTy.getSizeInBits();
45 unsigned NarrowSize = NarrowTy.getSizeInBits();
46 unsigned NumParts = Size / NarrowSize;
47 unsigned LeftoverSize = Size - NumParts * NarrowSize;
48 assert(Size > NarrowSize)((Size > NarrowSize) ? static_cast<void> (0) : __assert_fail
("Size > NarrowSize", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 48, __PRETTY_FUNCTION__))
;
49
50 if (LeftoverSize == 0)
51 return {NumParts, 0};
52
53 if (NarrowTy.isVector()) {
54 unsigned EltSize = OrigTy.getScalarSizeInBits();
55 if (LeftoverSize % EltSize != 0)
56 return {-1, -1};
57 LeftoverTy = LLT::scalarOrVector(LeftoverSize / EltSize, EltSize);
58 } else {
59 LeftoverTy = LLT::scalar(LeftoverSize);
60 }
61
62 int NumLeftover = LeftoverSize / LeftoverTy.getSizeInBits();
63 return std::make_pair(NumParts, NumLeftover);
64}
65
66LegalizerHelper::LegalizerHelper(MachineFunction &MF,
67 GISelChangeObserver &Observer,
68 MachineIRBuilder &Builder)
69 : MIRBuilder(Builder), MRI(MF.getRegInfo()),
70 LI(*MF.getSubtarget().getLegalizerInfo()), Observer(Observer) {
71 MIRBuilder.setMF(MF);
72 MIRBuilder.setChangeObserver(Observer);
73}
74
75LegalizerHelper::LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI,
76 GISelChangeObserver &Observer,
77 MachineIRBuilder &B)
78 : MIRBuilder(B), MRI(MF.getRegInfo()), LI(LI), Observer(Observer) {
79 MIRBuilder.setMF(MF);
80 MIRBuilder.setChangeObserver(Observer);
81}
82LegalizerHelper::LegalizeResult
83LegalizerHelper::legalizeInstrStep(MachineInstr &MI) {
84 LLVM_DEBUG(dbgs() << "Legalizing: "; MI.print(dbgs()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << "Legalizing: "; MI.print(dbgs
()); } } while (false)
;
85
86 if (MI.getOpcode() == TargetOpcode::G_INTRINSIC ||
87 MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS)
88 return LI.legalizeIntrinsic(MI, MRI, MIRBuilder) ? Legalized
89 : UnableToLegalize;
90 auto Step = LI.getAction(MI, MRI);
91 switch (Step.Action) {
92 case Legal:
93 LLVM_DEBUG(dbgs() << ".. Already legal\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Already legal\n"; } } while
(false)
;
94 return AlreadyLegal;
95 case Libcall:
96 LLVM_DEBUG(dbgs() << ".. Convert to libcall\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Convert to libcall\n"; }
} while (false)
;
97 return libcall(MI);
98 case NarrowScalar:
99 LLVM_DEBUG(dbgs() << ".. Narrow scalar\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Narrow scalar\n"; } } while
(false)
;
100 return narrowScalar(MI, Step.TypeIdx, Step.NewType);
101 case WidenScalar:
102 LLVM_DEBUG(dbgs() << ".. Widen scalar\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Widen scalar\n"; } } while
(false)
;
103 return widenScalar(MI, Step.TypeIdx, Step.NewType);
104 case Lower:
105 LLVM_DEBUG(dbgs() << ".. Lower\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Lower\n"; } } while (false
)
;
106 return lower(MI, Step.TypeIdx, Step.NewType);
107 case FewerElements:
108 LLVM_DEBUG(dbgs() << ".. Reduce number of elements\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Reduce number of elements\n"
; } } while (false)
;
109 return fewerElementsVector(MI, Step.TypeIdx, Step.NewType);
110 case MoreElements:
111 LLVM_DEBUG(dbgs() << ".. Increase number of elements\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Increase number of elements\n"
; } } while (false)
;
112 return moreElementsVector(MI, Step.TypeIdx, Step.NewType);
113 case Custom:
114 LLVM_DEBUG(dbgs() << ".. Custom legalization\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Custom legalization\n"; }
} while (false)
;
115 return LI.legalizeCustom(MI, MRI, MIRBuilder, Observer) ? Legalized
116 : UnableToLegalize;
117 default:
118 LLVM_DEBUG(dbgs() << ".. Unable to legalize\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Unable to legalize\n"; }
} while (false)
;
119 return UnableToLegalize;
120 }
121}
122
123void LegalizerHelper::extractParts(Register Reg, LLT Ty, int NumParts,
124 SmallVectorImpl<Register> &VRegs) {
125 for (int i = 0; i < NumParts; ++i)
126 VRegs.push_back(MRI.createGenericVirtualRegister(Ty));
127 MIRBuilder.buildUnmerge(VRegs, Reg);
128}
129
130bool LegalizerHelper::extractParts(Register Reg, LLT RegTy,
131 LLT MainTy, LLT &LeftoverTy,
132 SmallVectorImpl<Register> &VRegs,
133 SmallVectorImpl<Register> &LeftoverRegs) {
134 assert(!LeftoverTy.isValid() && "this is an out argument")((!LeftoverTy.isValid() && "this is an out argument")
? static_cast<void> (0) : __assert_fail ("!LeftoverTy.isValid() && \"this is an out argument\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 134, __PRETTY_FUNCTION__))
;
135
136 unsigned RegSize = RegTy.getSizeInBits();
137 unsigned MainSize = MainTy.getSizeInBits();
138 unsigned NumParts = RegSize / MainSize;
139 unsigned LeftoverSize = RegSize - NumParts * MainSize;
140
141 // Use an unmerge when possible.
142 if (LeftoverSize == 0) {
143 for (unsigned I = 0; I < NumParts; ++I)
144 VRegs.push_back(MRI.createGenericVirtualRegister(MainTy));
145 MIRBuilder.buildUnmerge(VRegs, Reg);
146 return true;
147 }
148
149 if (MainTy.isVector()) {
150 unsigned EltSize = MainTy.getScalarSizeInBits();
151 if (LeftoverSize % EltSize != 0)
152 return false;
153 LeftoverTy = LLT::scalarOrVector(LeftoverSize / EltSize, EltSize);
154 } else {
155 LeftoverTy = LLT::scalar(LeftoverSize);
156 }
157
158 // For irregular sizes, extract the individual parts.
159 for (unsigned I = 0; I != NumParts; ++I) {
160 Register NewReg = MRI.createGenericVirtualRegister(MainTy);
161 VRegs.push_back(NewReg);
162 MIRBuilder.buildExtract(NewReg, Reg, MainSize * I);
163 }
164
165 for (unsigned Offset = MainSize * NumParts; Offset < RegSize;
166 Offset += LeftoverSize) {
167 Register NewReg = MRI.createGenericVirtualRegister(LeftoverTy);
168 LeftoverRegs.push_back(NewReg);
169 MIRBuilder.buildExtract(NewReg, Reg, Offset);
170 }
171
172 return true;
173}
174
175static LLT getGCDType(LLT OrigTy, LLT TargetTy) {
176 if (OrigTy.isVector() && TargetTy.isVector()) {
177 assert(OrigTy.getElementType() == TargetTy.getElementType())((OrigTy.getElementType() == TargetTy.getElementType()) ? static_cast
<void> (0) : __assert_fail ("OrigTy.getElementType() == TargetTy.getElementType()"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 177, __PRETTY_FUNCTION__))
;
178 int GCD = greatestCommonDivisor(OrigTy.getNumElements(),
179 TargetTy.getNumElements());
180 return LLT::scalarOrVector(GCD, OrigTy.getElementType());
181 }
182
183 if (OrigTy.isVector() && !TargetTy.isVector()) {
184 assert(OrigTy.getElementType() == TargetTy)((OrigTy.getElementType() == TargetTy) ? static_cast<void>
(0) : __assert_fail ("OrigTy.getElementType() == TargetTy", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 184, __PRETTY_FUNCTION__))
;
185 return TargetTy;
186 }
187
188 assert(!OrigTy.isVector() && !TargetTy.isVector())((!OrigTy.isVector() && !TargetTy.isVector()) ? static_cast
<void> (0) : __assert_fail ("!OrigTy.isVector() && !TargetTy.isVector()"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 188, __PRETTY_FUNCTION__))
;
189
190 int GCD = greatestCommonDivisor(OrigTy.getSizeInBits(),
191 TargetTy.getSizeInBits());
192 return LLT::scalar(GCD);
193}
194
195void LegalizerHelper::insertParts(Register DstReg,
196 LLT ResultTy, LLT PartTy,
197 ArrayRef<Register> PartRegs,
198 LLT LeftoverTy,
199 ArrayRef<Register> LeftoverRegs) {
200 if (!LeftoverTy.isValid()) {
201 assert(LeftoverRegs.empty())((LeftoverRegs.empty()) ? static_cast<void> (0) : __assert_fail
("LeftoverRegs.empty()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 201, __PRETTY_FUNCTION__))
;
202
203 if (!ResultTy.isVector()) {
204 MIRBuilder.buildMerge(DstReg, PartRegs);
205 return;
206 }
207
208 if (PartTy.isVector())
209 MIRBuilder.buildConcatVectors(DstReg, PartRegs);
210 else
211 MIRBuilder.buildBuildVector(DstReg, PartRegs);
212 return;
213 }
214
215 unsigned PartSize = PartTy.getSizeInBits();
216 unsigned LeftoverPartSize = LeftoverTy.getSizeInBits();
217
218 Register CurResultReg = MRI.createGenericVirtualRegister(ResultTy);
219 MIRBuilder.buildUndef(CurResultReg);
220
221 unsigned Offset = 0;
222 for (Register PartReg : PartRegs) {
223 Register NewResultReg = MRI.createGenericVirtualRegister(ResultTy);
224 MIRBuilder.buildInsert(NewResultReg, CurResultReg, PartReg, Offset);
225 CurResultReg = NewResultReg;
226 Offset += PartSize;
227 }
228
229 for (unsigned I = 0, E = LeftoverRegs.size(); I != E; ++I) {
230 // Use the original output register for the final insert to avoid a copy.
231 Register NewResultReg = (I + 1 == E) ?
232 DstReg : MRI.createGenericVirtualRegister(ResultTy);
233
234 MIRBuilder.buildInsert(NewResultReg, CurResultReg, LeftoverRegs[I], Offset);
235 CurResultReg = NewResultReg;
236 Offset += LeftoverPartSize;
237 }
238}
239
240static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
241 switch (Opcode) {
242 case TargetOpcode::G_SDIV:
243 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size")(((Size == 32 || Size == 64 || Size == 128) && "Unsupported size"
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64 || Size == 128) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 243, __PRETTY_FUNCTION__))
;
244 switch (Size) {
245 case 32:
246 return RTLIB::SDIV_I32;
247 case 64:
248 return RTLIB::SDIV_I64;
249 case 128:
250 return RTLIB::SDIV_I128;
251 default:
252 llvm_unreachable("unexpected size")::llvm::llvm_unreachable_internal("unexpected size", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 252)
;
253 }
254 case TargetOpcode::G_UDIV:
255 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size")(((Size == 32 || Size == 64 || Size == 128) && "Unsupported size"
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64 || Size == 128) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 255, __PRETTY_FUNCTION__))
;
256 switch (Size) {
257 case 32:
258 return RTLIB::UDIV_I32;
259 case 64:
260 return RTLIB::UDIV_I64;
261 case 128:
262 return RTLIB::UDIV_I128;
263 default:
264 llvm_unreachable("unexpected size")::llvm::llvm_unreachable_internal("unexpected size", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 264)
;
265 }
266 case TargetOpcode::G_SREM:
267 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 267, __PRETTY_FUNCTION__))
;
268 return Size == 64 ? RTLIB::SREM_I64 : RTLIB::SREM_I32;
269 case TargetOpcode::G_UREM:
270 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 270, __PRETTY_FUNCTION__))
;
271 return Size == 64 ? RTLIB::UREM_I64 : RTLIB::UREM_I32;
272 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
273 assert(Size == 32 && "Unsupported size")((Size == 32 && "Unsupported size") ? static_cast<
void> (0) : __assert_fail ("Size == 32 && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 273, __PRETTY_FUNCTION__))
;
274 return RTLIB::CTLZ_I32;
275 case TargetOpcode::G_FADD:
276 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 276, __PRETTY_FUNCTION__))
;
277 return Size == 64 ? RTLIB::ADD_F64 : RTLIB::ADD_F32;
278 case TargetOpcode::G_FSUB:
279 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 279, __PRETTY_FUNCTION__))
;
280 return Size == 64 ? RTLIB::SUB_F64 : RTLIB::SUB_F32;
281 case TargetOpcode::G_FMUL:
282 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 282, __PRETTY_FUNCTION__))
;
283 return Size == 64 ? RTLIB::MUL_F64 : RTLIB::MUL_F32;
284 case TargetOpcode::G_FDIV:
285 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 285, __PRETTY_FUNCTION__))
;
286 return Size == 64 ? RTLIB::DIV_F64 : RTLIB::DIV_F32;
287 case TargetOpcode::G_FEXP:
288 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 288, __PRETTY_FUNCTION__))
;
289 return Size == 64 ? RTLIB::EXP_F64 : RTLIB::EXP_F32;
290 case TargetOpcode::G_FEXP2:
291 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 291, __PRETTY_FUNCTION__))
;
292 return Size == 64 ? RTLIB::EXP2_F64 : RTLIB::EXP2_F32;
293 case TargetOpcode::G_FREM:
294 return Size == 64 ? RTLIB::REM_F64 : RTLIB::REM_F32;
295 case TargetOpcode::G_FPOW:
296 return Size == 64 ? RTLIB::POW_F64 : RTLIB::POW_F32;
297 case TargetOpcode::G_FMA:
298 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 298, __PRETTY_FUNCTION__))
;
299 return Size == 64 ? RTLIB::FMA_F64 : RTLIB::FMA_F32;
300 case TargetOpcode::G_FSIN:
301 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size")(((Size == 32 || Size == 64 || Size == 128) && "Unsupported size"
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64 || Size == 128) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 301, __PRETTY_FUNCTION__))
;
302 return Size == 128 ? RTLIB::SIN_F128
303 : Size == 64 ? RTLIB::SIN_F64 : RTLIB::SIN_F32;
304 case TargetOpcode::G_FCOS:
305 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size")(((Size == 32 || Size == 64 || Size == 128) && "Unsupported size"
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64 || Size == 128) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 305, __PRETTY_FUNCTION__))
;
306 return Size == 128 ? RTLIB::COS_F128
307 : Size == 64 ? RTLIB::COS_F64 : RTLIB::COS_F32;
308 case TargetOpcode::G_FLOG10:
309 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size")(((Size == 32 || Size == 64 || Size == 128) && "Unsupported size"
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64 || Size == 128) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 309, __PRETTY_FUNCTION__))
;
310 return Size == 128 ? RTLIB::LOG10_F128
311 : Size == 64 ? RTLIB::LOG10_F64 : RTLIB::LOG10_F32;
312 case TargetOpcode::G_FLOG:
313 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size")(((Size == 32 || Size == 64 || Size == 128) && "Unsupported size"
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64 || Size == 128) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 313, __PRETTY_FUNCTION__))
;
314 return Size == 128 ? RTLIB::LOG_F128
315 : Size == 64 ? RTLIB::LOG_F64 : RTLIB::LOG_F32;
316 case TargetOpcode::G_FLOG2:
317 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size")(((Size == 32 || Size == 64 || Size == 128) && "Unsupported size"
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64 || Size == 128) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 317, __PRETTY_FUNCTION__))
;
318 return Size == 128 ? RTLIB::LOG2_F128
319 : Size == 64 ? RTLIB::LOG2_F64 : RTLIB::LOG2_F32;
320 case TargetOpcode::G_FCEIL:
321 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 321, __PRETTY_FUNCTION__))
;
322 return Size == 64 ? RTLIB::CEIL_F64 : RTLIB::CEIL_F32;
323 case TargetOpcode::G_FFLOOR:
324 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 324, __PRETTY_FUNCTION__))
;
325 return Size == 64 ? RTLIB::FLOOR_F64 : RTLIB::FLOOR_F32;
326 }
327 llvm_unreachable("Unknown libcall function")::llvm::llvm_unreachable_internal("Unknown libcall function",
"/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 327)
;
328}
329
330/// True if an instruction is in tail position in its caller. Intended for
331/// legalizing libcalls as tail calls when possible.
332static bool isLibCallInTailPosition(MachineInstr &MI) {
333 const Function &F = MI.getParent()->getParent()->getFunction();
334
335 // Conservatively require the attributes of the call to match those of
336 // the return. Ignore NoAlias and NonNull because they don't affect the
337 // call sequence.
338 AttributeList CallerAttrs = F.getAttributes();
339 if (AttrBuilder(CallerAttrs, AttributeList::ReturnIndex)
340 .removeAttribute(Attribute::NoAlias)
341 .removeAttribute(Attribute::NonNull)
342 .hasAttributes())
343 return false;
344
345 // It's not safe to eliminate the sign / zero extension of the return value.
346 if (CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::ZExt) ||
347 CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::SExt))
348 return false;
349
350 // Only tail call if the following instruction is a standard return.
351 auto &TII = *MI.getMF()->getSubtarget().getInstrInfo();
352 MachineInstr *Next = MI.getNextNode();
353 if (!Next || TII.isTailCall(*Next) || !Next->isReturn())
354 return false;
355
356 return true;
357}
358
359LegalizerHelper::LegalizeResult
360llvm::createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall,
361 const CallLowering::ArgInfo &Result,
362 ArrayRef<CallLowering::ArgInfo> Args) {
363 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
364 auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
365 const char *Name = TLI.getLibcallName(Libcall);
366
367 CallLowering::CallLoweringInfo Info;
368 Info.CallConv = TLI.getLibcallCallingConv(Libcall);
369 Info.Callee = MachineOperand::CreateES(Name);
370 Info.OrigRet = Result;
371 std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs));
372 if (!CLI.lowerCall(MIRBuilder, Info))
373 return LegalizerHelper::UnableToLegalize;
374
375 return LegalizerHelper::Legalized;
376}
377
378// Useful for libcalls where all operands have the same type.
379static LegalizerHelper::LegalizeResult
380simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size,
381 Type *OpType) {
382 auto Libcall = getRTLibDesc(MI.getOpcode(), Size);
383
384 SmallVector<CallLowering::ArgInfo, 3> Args;
385 for (unsigned i = 1; i < MI.getNumOperands(); i++)
386 Args.push_back({MI.getOperand(i).getReg(), OpType});
387 return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), OpType},
388 Args);
389}
390
391LegalizerHelper::LegalizeResult
392llvm::createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
393 MachineInstr &MI) {
394 assert(MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS)((MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS)
? static_cast<void> (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 394, __PRETTY_FUNCTION__))
;
395 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
396
397 SmallVector<CallLowering::ArgInfo, 3> Args;
398 // Add all the args, except for the last which is an imm denoting 'tail'.
399 for (unsigned i = 1; i < MI.getNumOperands() - 1; i++) {
400 Register Reg = MI.getOperand(i).getReg();
401
402 // Need derive an IR type for call lowering.
403 LLT OpLLT = MRI.getType(Reg);
404 Type *OpTy = nullptr;
405 if (OpLLT.isPointer())
406 OpTy = Type::getInt8PtrTy(Ctx, OpLLT.getAddressSpace());
407 else
408 OpTy = IntegerType::get(Ctx, OpLLT.getSizeInBits());
409 Args.push_back({Reg, OpTy});
410 }
411
412 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
413 auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
414 Intrinsic::ID ID = MI.getOperand(0).getIntrinsicID();
415 RTLIB::Libcall RTLibcall;
416 switch (ID) {
417 case Intrinsic::memcpy:
418 RTLibcall = RTLIB::MEMCPY;
419 break;
420 case Intrinsic::memset:
421 RTLibcall = RTLIB::MEMSET;
422 break;
423 case Intrinsic::memmove:
424 RTLibcall = RTLIB::MEMMOVE;
425 break;
426 default:
427 return LegalizerHelper::UnableToLegalize;
428 }
429 const char *Name = TLI.getLibcallName(RTLibcall);
430
431 MIRBuilder.setInstr(MI);
432
433 CallLowering::CallLoweringInfo Info;
434 Info.CallConv = TLI.getLibcallCallingConv(RTLibcall);
435 Info.Callee = MachineOperand::CreateES(Name);
436 Info.OrigRet = CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx));
437 Info.IsTailCall = MI.getOperand(MI.getNumOperands() - 1).getImm() == 1 &&
438 isLibCallInTailPosition(MI);
439
440 std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs));
441 if (!CLI.lowerCall(MIRBuilder, Info))
442 return LegalizerHelper::UnableToLegalize;
443
444 if (Info.LoweredTailCall) {
445 assert(Info.IsTailCall && "Lowered tail call when it wasn't a tail call?")((Info.IsTailCall && "Lowered tail call when it wasn't a tail call?"
) ? static_cast<void> (0) : __assert_fail ("Info.IsTailCall && \"Lowered tail call when it wasn't a tail call?\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 445, __PRETTY_FUNCTION__))
;
446 // We must have a return following the call to get past
447 // isLibCallInTailPosition.
448 assert(MI.getNextNode() && MI.getNextNode()->isReturn() &&((MI.getNextNode() && MI.getNextNode()->isReturn()
&& "Expected instr following MI to be a return?") ? static_cast
<void> (0) : __assert_fail ("MI.getNextNode() && MI.getNextNode()->isReturn() && \"Expected instr following MI to be a return?\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 449, __PRETTY_FUNCTION__))
449 "Expected instr following MI to be a return?")((MI.getNextNode() && MI.getNextNode()->isReturn()
&& "Expected instr following MI to be a return?") ? static_cast
<void> (0) : __assert_fail ("MI.getNextNode() && MI.getNextNode()->isReturn() && \"Expected instr following MI to be a return?\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 449, __PRETTY_FUNCTION__))
;
450
451 // We lowered a tail call, so the call is now the return from the block.
452 // Delete the old return.
453 MI.getNextNode()->eraseFromParent();
454 }
455
456 return LegalizerHelper::Legalized;
457}
458
459static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType,
460 Type *FromType) {
461 auto ToMVT = MVT::getVT(ToType);
462 auto FromMVT = MVT::getVT(FromType);
463
464 switch (Opcode) {
465 case TargetOpcode::G_FPEXT:
466 return RTLIB::getFPEXT(FromMVT, ToMVT);
467 case TargetOpcode::G_FPTRUNC:
468 return RTLIB::getFPROUND(FromMVT, ToMVT);
469 case TargetOpcode::G_FPTOSI:
470 return RTLIB::getFPTOSINT(FromMVT, ToMVT);
471 case TargetOpcode::G_FPTOUI:
472 return RTLIB::getFPTOUINT(FromMVT, ToMVT);
473 case TargetOpcode::G_SITOFP:
474 return RTLIB::getSINTTOFP(FromMVT, ToMVT);
475 case TargetOpcode::G_UITOFP:
476 return RTLIB::getUINTTOFP(FromMVT, ToMVT);
477 }
478 llvm_unreachable("Unsupported libcall function")::llvm::llvm_unreachable_internal("Unsupported libcall function"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 478)
;
479}
480
481static LegalizerHelper::LegalizeResult
482conversionLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, Type *ToType,
483 Type *FromType) {
484 RTLIB::Libcall Libcall = getConvRTLibDesc(MI.getOpcode(), ToType, FromType);
485 return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), ToType},
486 {{MI.getOperand(1).getReg(), FromType}});
487}
488
489LegalizerHelper::LegalizeResult
490LegalizerHelper::libcall(MachineInstr &MI) {
491 LLT LLTy = MRI.getType(MI.getOperand(0).getReg());
492 unsigned Size = LLTy.getSizeInBits();
493 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
494
495 MIRBuilder.setInstr(MI);
496
497 switch (MI.getOpcode()) {
498 default:
499 return UnableToLegalize;
500 case TargetOpcode::G_SDIV:
501 case TargetOpcode::G_UDIV:
502 case TargetOpcode::G_SREM:
503 case TargetOpcode::G_UREM:
504 case TargetOpcode::G_CTLZ_ZERO_UNDEF: {
505 Type *HLTy = IntegerType::get(Ctx, Size);
506 auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
507 if (Status != Legalized)
508 return Status;
509 break;
510 }
511 case TargetOpcode::G_FADD:
512 case TargetOpcode::G_FSUB:
513 case TargetOpcode::G_FMUL:
514 case TargetOpcode::G_FDIV:
515 case TargetOpcode::G_FMA:
516 case TargetOpcode::G_FPOW:
517 case TargetOpcode::G_FREM:
518 case TargetOpcode::G_FCOS:
519 case TargetOpcode::G_FSIN:
520 case TargetOpcode::G_FLOG10:
521 case TargetOpcode::G_FLOG:
522 case TargetOpcode::G_FLOG2:
523 case TargetOpcode::G_FEXP:
524 case TargetOpcode::G_FEXP2:
525 case TargetOpcode::G_FCEIL:
526 case TargetOpcode::G_FFLOOR: {
527 if (Size > 64) {
528 LLVM_DEBUG(dbgs() << "Size " << Size << " too large to legalize.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << "Size " << Size <<
" too large to legalize.\n"; } } while (false)
;
529 return UnableToLegalize;
530 }
531 Type *HLTy = Size == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx);
532 auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
533 if (Status != Legalized)
534 return Status;
535 break;
536 }
537 case TargetOpcode::G_FPEXT: {
538 // FIXME: Support other floating point types (half, fp128 etc)
539 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
540 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
541 if (ToSize != 64 || FromSize != 32)
542 return UnableToLegalize;
543 LegalizeResult Status = conversionLibcall(
544 MI, MIRBuilder, Type::getDoubleTy(Ctx), Type::getFloatTy(Ctx));
545 if (Status != Legalized)
546 return Status;
547 break;
548 }
549 case TargetOpcode::G_FPTRUNC: {
550 // FIXME: Support other floating point types (half, fp128 etc)
551 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
552 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
553 if (ToSize != 32 || FromSize != 64)
554 return UnableToLegalize;
555 LegalizeResult Status = conversionLibcall(
556 MI, MIRBuilder, Type::getFloatTy(Ctx), Type::getDoubleTy(Ctx));
557 if (Status != Legalized)
558 return Status;
559 break;
560 }
561 case TargetOpcode::G_FPTOSI:
562 case TargetOpcode::G_FPTOUI: {
563 // FIXME: Support other types
564 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
565 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
566 if ((ToSize != 32 && ToSize != 64) || (FromSize != 32 && FromSize != 64))
567 return UnableToLegalize;
568 LegalizeResult Status = conversionLibcall(
569 MI, MIRBuilder,
570 ToSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx),
571 FromSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx));
572 if (Status != Legalized)
573 return Status;
574 break;
575 }
576 case TargetOpcode::G_SITOFP:
577 case TargetOpcode::G_UITOFP: {
578 // FIXME: Support other types
579 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
580 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
581 if ((FromSize != 32 && FromSize != 64) || (ToSize != 32 && ToSize != 64))
582 return UnableToLegalize;
583 LegalizeResult Status = conversionLibcall(
584 MI, MIRBuilder,
585 ToSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx),
586 FromSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx));
587 if (Status != Legalized)
588 return Status;
589 break;
590 }
591 }
592
593 MI.eraseFromParent();
594 return Legalized;
595}
596
597LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
598 unsigned TypeIdx,
599 LLT NarrowTy) {
600 MIRBuilder.setInstr(MI);
601
602 uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
603 uint64_t NarrowSize = NarrowTy.getSizeInBits();
604
605 switch (MI.getOpcode()) {
606 default:
607 return UnableToLegalize;
608 case TargetOpcode::G_IMPLICIT_DEF: {
609 // FIXME: add support for when SizeOp0 isn't an exact multiple of
610 // NarrowSize.
611 if (SizeOp0 % NarrowSize != 0)
612 return UnableToLegalize;
613 int NumParts = SizeOp0 / NarrowSize;
614
615 SmallVector<Register, 2> DstRegs;
616 for (int i = 0; i < NumParts; ++i)
617 DstRegs.push_back(
618 MIRBuilder.buildUndef(NarrowTy)->getOperand(0).getReg());
619
620 Register DstReg = MI.getOperand(0).getReg();
621 if(MRI.getType(DstReg).isVector())
622 MIRBuilder.buildBuildVector(DstReg, DstRegs);
623 else
624 MIRBuilder.buildMerge(DstReg, DstRegs);
625 MI.eraseFromParent();
626 return Legalized;
627 }
628 case TargetOpcode::G_CONSTANT: {
629 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
630 const APInt &Val = MI.getOperand(1).getCImm()->getValue();
631 unsigned TotalSize = Ty.getSizeInBits();
632 unsigned NarrowSize = NarrowTy.getSizeInBits();
633 int NumParts = TotalSize / NarrowSize;
634
635 SmallVector<Register, 4> PartRegs;
636 for (int I = 0; I != NumParts; ++I) {
637 unsigned Offset = I * NarrowSize;
638 auto K = MIRBuilder.buildConstant(NarrowTy,
639 Val.lshr(Offset).trunc(NarrowSize));
640 PartRegs.push_back(K.getReg(0));
641 }
642
643 LLT LeftoverTy;
644 unsigned LeftoverBits = TotalSize - NumParts * NarrowSize;
645 SmallVector<Register, 1> LeftoverRegs;
646 if (LeftoverBits != 0) {
647 LeftoverTy = LLT::scalar(LeftoverBits);
648 auto K = MIRBuilder.buildConstant(
649 LeftoverTy,
650 Val.lshr(NumParts * NarrowSize).trunc(LeftoverBits));
651 LeftoverRegs.push_back(K.getReg(0));
652 }
653
654 insertParts(MI.getOperand(0).getReg(),
655 Ty, NarrowTy, PartRegs, LeftoverTy, LeftoverRegs);
656
657 MI.eraseFromParent();
658 return Legalized;
659 }
660 case TargetOpcode::G_SEXT: {
661 if (TypeIdx != 0)
662 return UnableToLegalize;
663
664 Register SrcReg = MI.getOperand(1).getReg();
665 LLT SrcTy = MRI.getType(SrcReg);
666
667 // FIXME: support the general case where the requested NarrowTy may not be
668 // the same as the source type. E.g. s128 = sext(s32)
669 if ((SrcTy.getSizeInBits() != SizeOp0 / 2) ||
670 SrcTy.getSizeInBits() != NarrowTy.getSizeInBits()) {
671 LLVM_DEBUG(dbgs() << "Can't narrow sext to type " << NarrowTy << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << "Can't narrow sext to type "
<< NarrowTy << "\n"; } } while (false)
;
672 return UnableToLegalize;
673 }
674
675 // Shift the sign bit of the low register through the high register.
676 auto ShiftAmt =
677 MIRBuilder.buildConstant(LLT::scalar(64), NarrowTy.getSizeInBits() - 1);
678 auto Shift = MIRBuilder.buildAShr(NarrowTy, SrcReg, ShiftAmt);
679 MIRBuilder.buildMerge(MI.getOperand(0).getReg(), {SrcReg, Shift.getReg(0)});
680 MI.eraseFromParent();
681 return Legalized;
682 }
683 case TargetOpcode::G_ZEXT: {
684 if (TypeIdx != 0)
685 return UnableToLegalize;
686
687 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
688 uint64_t SizeOp1 = SrcTy.getSizeInBits();
689 if (SizeOp0 % SizeOp1 != 0)
690 return UnableToLegalize;
691
692 // Generate a merge where the bottom bits are taken from the source, and
693 // zero everything else.
694 Register ZeroReg = MIRBuilder.buildConstant(SrcTy, 0).getReg(0);
695 unsigned NumParts = SizeOp0 / SizeOp1;
696 SmallVector<Register, 4> Srcs = {MI.getOperand(1).getReg()};
697 for (unsigned Part = 1; Part < NumParts; ++Part)
698 Srcs.push_back(ZeroReg);
699 MIRBuilder.buildMerge(MI.getOperand(0).getReg(), Srcs);
700 MI.eraseFromParent();
701 return Legalized;
702 }
703 case TargetOpcode::G_TRUNC: {
704 if (TypeIdx != 1)
705 return UnableToLegalize;
706
707 uint64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
708 if (NarrowTy.getSizeInBits() * 2 != SizeOp1) {
709 LLVM_DEBUG(dbgs() << "Can't narrow trunc to type " << NarrowTy << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << "Can't narrow trunc to type "
<< NarrowTy << "\n"; } } while (false)
;
710 return UnableToLegalize;
711 }
712
713 auto Unmerge = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1).getReg());
714 MIRBuilder.buildCopy(MI.getOperand(0).getReg(), Unmerge.getReg(0));
715 MI.eraseFromParent();
716 return Legalized;
717 }
718
719 case TargetOpcode::G_ADD: {
720 // FIXME: add support for when SizeOp0 isn't an exact multiple of
721 // NarrowSize.
722 if (SizeOp0 % NarrowSize != 0)
723 return UnableToLegalize;
724 // Expand in terms of carry-setting/consuming G_ADDE instructions.
725 int NumParts = SizeOp0 / NarrowTy.getSizeInBits();
726
727 SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs;
728 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
729 extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
730
731 Register CarryIn;
732 for (int i = 0; i < NumParts; ++i) {
733 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy);
734 Register CarryOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
735
736 if (i == 0)
737 MIRBuilder.buildUAddo(DstReg, CarryOut, Src1Regs[i], Src2Regs[i]);
738 else {
739 MIRBuilder.buildUAdde(DstReg, CarryOut, Src1Regs[i],
740 Src2Regs[i], CarryIn);
741 }
742
743 DstRegs.push_back(DstReg);
744 CarryIn = CarryOut;
745 }
746 Register DstReg = MI.getOperand(0).getReg();
747 if(MRI.getType(DstReg).isVector())
748 MIRBuilder.buildBuildVector(DstReg, DstRegs);
749 else
750 MIRBuilder.buildMerge(DstReg, DstRegs);
751 MI.eraseFromParent();
752 return Legalized;
753 }
754 case TargetOpcode::G_SUB: {
755 // FIXME: add support for when SizeOp0 isn't an exact multiple of
756 // NarrowSize.
757 if (SizeOp0 % NarrowSize != 0)
758 return UnableToLegalize;
759
760 int NumParts = SizeOp0 / NarrowTy.getSizeInBits();
761
762 SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs;
763 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
764 extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
765
766 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy);
767 Register BorrowOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
768 MIRBuilder.buildInstr(TargetOpcode::G_USUBO, {DstReg, BorrowOut},
769 {Src1Regs[0], Src2Regs[0]});
770 DstRegs.push_back(DstReg);
771 Register BorrowIn = BorrowOut;
772 for (int i = 1; i < NumParts; ++i) {
773 DstReg = MRI.createGenericVirtualRegister(NarrowTy);
774 BorrowOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
775
776 MIRBuilder.buildInstr(TargetOpcode::G_USUBE, {DstReg, BorrowOut},
777 {Src1Regs[i], Src2Regs[i], BorrowIn});
778
779 DstRegs.push_back(DstReg);
780 BorrowIn = BorrowOut;
781 }
782 MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
783 MI.eraseFromParent();
784 return Legalized;
785 }
786 case TargetOpcode::G_MUL:
787 case TargetOpcode::G_UMULH:
788 return narrowScalarMul(MI, NarrowTy);
789 case TargetOpcode::G_EXTRACT:
790 return narrowScalarExtract(MI, TypeIdx, NarrowTy);
791 case TargetOpcode::G_INSERT:
792 return narrowScalarInsert(MI, TypeIdx, NarrowTy);
793 case TargetOpcode::G_LOAD: {
794 const auto &MMO = **MI.memoperands_begin();
795 Register DstReg = MI.getOperand(0).getReg();
796 LLT DstTy = MRI.getType(DstReg);
797 if (DstTy.isVector())
798 return UnableToLegalize;
799
800 if (8 * MMO.getSize() != DstTy.getSizeInBits()) {
801 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
802 auto &MMO = **MI.memoperands_begin();
803 MIRBuilder.buildLoad(TmpReg, MI.getOperand(1).getReg(), MMO);
804 MIRBuilder.buildAnyExt(DstReg, TmpReg);
805 MI.eraseFromParent();
806 return Legalized;
807 }
808
809 return reduceLoadStoreWidth(MI, TypeIdx, NarrowTy);
810 }
811 case TargetOpcode::G_ZEXTLOAD:
812 case TargetOpcode::G_SEXTLOAD: {
813 bool ZExt = MI.getOpcode() == TargetOpcode::G_ZEXTLOAD;
814 Register DstReg = MI.getOperand(0).getReg();
815 Register PtrReg = MI.getOperand(1).getReg();
816
817 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
818 auto &MMO = **MI.memoperands_begin();
819 if (MMO.getSizeInBits() == NarrowSize) {
820 MIRBuilder.buildLoad(TmpReg, PtrReg, MMO);
821 } else {
822 unsigned ExtLoad = ZExt ? TargetOpcode::G_ZEXTLOAD
823 : TargetOpcode::G_SEXTLOAD;
824 MIRBuilder.buildInstr(ExtLoad)
825 .addDef(TmpReg)
826 .addUse(PtrReg)
827 .addMemOperand(&MMO);
828 }
829
830 if (ZExt)
831 MIRBuilder.buildZExt(DstReg, TmpReg);
832 else
833 MIRBuilder.buildSExt(DstReg, TmpReg);
834
835 MI.eraseFromParent();
836 return Legalized;
837 }
838 case TargetOpcode::G_STORE: {
839 const auto &MMO = **MI.memoperands_begin();
840
841 Register SrcReg = MI.getOperand(0).getReg();
842 LLT SrcTy = MRI.getType(SrcReg);
843 if (SrcTy.isVector())
844 return UnableToLegalize;
845
846 int NumParts = SizeOp0 / NarrowSize;
847 unsigned HandledSize = NumParts * NarrowTy.getSizeInBits();
848 unsigned LeftoverBits = SrcTy.getSizeInBits() - HandledSize;
849 if (SrcTy.isVector() && LeftoverBits != 0)
850 return UnableToLegalize;
851
852 if (8 * MMO.getSize() != SrcTy.getSizeInBits()) {
853 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
854 auto &MMO = **MI.memoperands_begin();
855 MIRBuilder.buildTrunc(TmpReg, SrcReg);
856 MIRBuilder.buildStore(TmpReg, MI.getOperand(1).getReg(), MMO);
857 MI.eraseFromParent();
858 return Legalized;
859 }
860
861 return reduceLoadStoreWidth(MI, 0, NarrowTy);
862 }
863 case TargetOpcode::G_SELECT:
864 return narrowScalarSelect(MI, TypeIdx, NarrowTy);
865 case TargetOpcode::G_AND:
866 case TargetOpcode::G_OR:
867 case TargetOpcode::G_XOR: {
868 // Legalize bitwise operation:
869 // A = BinOp<Ty> B, C
870 // into:
871 // B1, ..., BN = G_UNMERGE_VALUES B
872 // C1, ..., CN = G_UNMERGE_VALUES C
873 // A1 = BinOp<Ty/N> B1, C2
874 // ...
875 // AN = BinOp<Ty/N> BN, CN
876 // A = G_MERGE_VALUES A1, ..., AN
877 return narrowScalarBasic(MI, TypeIdx, NarrowTy);
878 }
879 case TargetOpcode::G_SHL:
880 case TargetOpcode::G_LSHR:
881 case TargetOpcode::G_ASHR:
882 return narrowScalarShift(MI, TypeIdx, NarrowTy);
883 case TargetOpcode::G_CTLZ:
884 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
885 case TargetOpcode::G_CTTZ:
886 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
887 case TargetOpcode::G_CTPOP:
888 if (TypeIdx != 0)
889 return UnableToLegalize; // TODO
890
891 Observer.changingInstr(MI);
892 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT);
893 Observer.changedInstr(MI);
894 return Legalized;
895 case TargetOpcode::G_INTTOPTR:
896 if (TypeIdx != 1)
897 return UnableToLegalize;
898
899 Observer.changingInstr(MI);
900 narrowScalarSrc(MI, NarrowTy, 1);
901 Observer.changedInstr(MI);
902 return Legalized;
903 case TargetOpcode::G_PTRTOINT:
904 if (TypeIdx != 0)
905 return UnableToLegalize;
906
907 Observer.changingInstr(MI);
908 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT);
909 Observer.changedInstr(MI);
910 return Legalized;
911 case TargetOpcode::G_PHI: {
912 unsigned NumParts = SizeOp0 / NarrowSize;
913 SmallVector<Register, 2> DstRegs;
914 SmallVector<SmallVector<Register, 2>, 2> SrcRegs;
915 DstRegs.resize(NumParts);
916 SrcRegs.resize(MI.getNumOperands() / 2);
917 Observer.changingInstr(MI);
918 for (unsigned i = 1; i < MI.getNumOperands(); i += 2) {
919 MachineBasicBlock &OpMBB = *MI.getOperand(i + 1).getMBB();
920 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
921 extractParts(MI.getOperand(i).getReg(), NarrowTy, NumParts,
922 SrcRegs[i / 2]);
923 }
924 MachineBasicBlock &MBB = *MI.getParent();
925 MIRBuilder.setInsertPt(MBB, MI);
926 for (unsigned i = 0; i < NumParts; ++i) {
927 DstRegs[i] = MRI.createGenericVirtualRegister(NarrowTy);
928 MachineInstrBuilder MIB =
929 MIRBuilder.buildInstr(TargetOpcode::G_PHI).addDef(DstRegs[i]);
930 for (unsigned j = 1; j < MI.getNumOperands(); j += 2)
931 MIB.addUse(SrcRegs[j / 2][i]).add(MI.getOperand(j + 1));
932 }
933 MIRBuilder.setInsertPt(MBB, MBB.getFirstNonPHI());
934 MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
935 Observer.changedInstr(MI);
936 MI.eraseFromParent();
937 return Legalized;
938 }
939 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
940 case TargetOpcode::G_INSERT_VECTOR_ELT: {
941 if (TypeIdx != 2)
942 return UnableToLegalize;
943
944 int OpIdx = MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3;
945 Observer.changingInstr(MI);
946 narrowScalarSrc(MI, NarrowTy, OpIdx);
947 Observer.changedInstr(MI);
948 return Legalized;
949 }
950 case TargetOpcode::G_ICMP: {
951 uint64_t SrcSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
952 if (NarrowSize * 2 != SrcSize)
953 return UnableToLegalize;
954
955 Observer.changingInstr(MI);
956 Register LHSL = MRI.createGenericVirtualRegister(NarrowTy);
957 Register LHSH = MRI.createGenericVirtualRegister(NarrowTy);
958 MIRBuilder.buildUnmerge({LHSL, LHSH}, MI.getOperand(2).getReg());
959
960 Register RHSL = MRI.createGenericVirtualRegister(NarrowTy);
961 Register RHSH = MRI.createGenericVirtualRegister(NarrowTy);
962 MIRBuilder.buildUnmerge({RHSL, RHSH}, MI.getOperand(3).getReg());
963
964 CmpInst::Predicate Pred =
965 static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
966 LLT ResTy = MRI.getType(MI.getOperand(0).getReg());
967
968 if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE) {
969 MachineInstrBuilder XorL = MIRBuilder.buildXor(NarrowTy, LHSL, RHSL);
970 MachineInstrBuilder XorH = MIRBuilder.buildXor(NarrowTy, LHSH, RHSH);
971 MachineInstrBuilder Or = MIRBuilder.buildOr(NarrowTy, XorL, XorH);
972 MachineInstrBuilder Zero = MIRBuilder.buildConstant(NarrowTy, 0);
973 MIRBuilder.buildICmp(Pred, MI.getOperand(0).getReg(), Or, Zero);
974 } else {
975 MachineInstrBuilder CmpH = MIRBuilder.buildICmp(Pred, ResTy, LHSH, RHSH);
976 MachineInstrBuilder CmpHEQ =
977 MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, ResTy, LHSH, RHSH);
978 MachineInstrBuilder CmpLU = MIRBuilder.buildICmp(
979 ICmpInst::getUnsignedPredicate(Pred), ResTy, LHSL, RHSL);
980 MIRBuilder.buildSelect(MI.getOperand(0).getReg(), CmpHEQ, CmpLU, CmpH);
981 }
982 Observer.changedInstr(MI);
983 MI.eraseFromParent();
984 return Legalized;
985 }
986 case TargetOpcode::G_SEXT_INREG: {
987 if (TypeIdx != 0)
988 return UnableToLegalize;
989
990 if (!MI.getOperand(2).isImm())
991 return UnableToLegalize;
992 int64_t SizeInBits = MI.getOperand(2).getImm();
993
994 // So long as the new type has more bits than the bits we're extending we
995 // don't need to break it apart.
996 if (NarrowTy.getScalarSizeInBits() >= SizeInBits) {
997 Observer.changingInstr(MI);
998 // We don't lose any non-extension bits by truncating the src and
999 // sign-extending the dst.
1000 MachineOperand &MO1 = MI.getOperand(1);
1001 auto TruncMIB = MIRBuilder.buildTrunc(NarrowTy, MO1.getReg());
1002 MO1.setReg(TruncMIB->getOperand(0).getReg());
1003
1004 MachineOperand &MO2 = MI.getOperand(0);
1005 Register DstExt = MRI.createGenericVirtualRegister(NarrowTy);
1006 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1007 MIRBuilder.buildInstr(TargetOpcode::G_SEXT, {MO2.getReg()}, {DstExt});
1008 MO2.setReg(DstExt);
1009 Observer.changedInstr(MI);
1010 return Legalized;
1011 }
1012
1013 // Break it apart. Components below the extension point are unmodified. The
1014 // component containing the extension point becomes a narrower SEXT_INREG.
1015 // Components above it are ashr'd from the component containing the
1016 // extension point.
1017 if (SizeOp0 % NarrowSize != 0)
1018 return UnableToLegalize;
1019 int NumParts = SizeOp0 / NarrowSize;
1020
1021 // List the registers where the destination will be scattered.
1022 SmallVector<Register, 2> DstRegs;
1023 // List the registers where the source will be split.
1024 SmallVector<Register, 2> SrcRegs;
1025
1026 // Create all the temporary registers.
1027 for (int i = 0; i < NumParts; ++i) {
1028 Register SrcReg = MRI.createGenericVirtualRegister(NarrowTy);
1029
1030 SrcRegs.push_back(SrcReg);
1031 }
1032
1033 // Explode the big arguments into smaller chunks.
1034 MIRBuilder.buildUnmerge(SrcRegs, MI.getOperand(1).getReg());
1035
1036 Register AshrCstReg =
1037 MIRBuilder.buildConstant(NarrowTy, NarrowTy.getScalarSizeInBits() - 1)
1038 ->getOperand(0)
1039 .getReg();
1040 Register FullExtensionReg = 0;
1041 Register PartialExtensionReg = 0;
1042
1043 // Do the operation on each small part.
1044 for (int i = 0; i < NumParts; ++i) {
1045 if ((i + 1) * NarrowTy.getScalarSizeInBits() < SizeInBits)
1046 DstRegs.push_back(SrcRegs[i]);
1047 else if (i * NarrowTy.getScalarSizeInBits() > SizeInBits) {
1048 assert(PartialExtensionReg &&((PartialExtensionReg && "Expected to visit partial extension before full"
) ? static_cast<void> (0) : __assert_fail ("PartialExtensionReg && \"Expected to visit partial extension before full\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1049, __PRETTY_FUNCTION__))
1049 "Expected to visit partial extension before full")((PartialExtensionReg && "Expected to visit partial extension before full"
) ? static_cast<void> (0) : __assert_fail ("PartialExtensionReg && \"Expected to visit partial extension before full\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1049, __PRETTY_FUNCTION__))
;
1050 if (FullExtensionReg) {
1051 DstRegs.push_back(FullExtensionReg);
1052 continue;
1053 }
1054 DstRegs.push_back(MIRBuilder
1055 .buildInstr(TargetOpcode::G_ASHR, {NarrowTy},
1056 {PartialExtensionReg, AshrCstReg})
1057 ->getOperand(0)
1058 .getReg());
1059 FullExtensionReg = DstRegs.back();
1060 } else {
1061 DstRegs.push_back(
1062 MIRBuilder
1063 .buildInstr(
1064 TargetOpcode::G_SEXT_INREG, {NarrowTy},
1065 {SrcRegs[i], SizeInBits % NarrowTy.getScalarSizeInBits()})
1066 ->getOperand(0)
1067 .getReg());
1068 PartialExtensionReg = DstRegs.back();
1069 }
1070 }
1071
1072 // Gather the destination registers into the final destination.
1073 Register DstReg = MI.getOperand(0).getReg();
1074 MIRBuilder.buildMerge(DstReg, DstRegs);
1075 MI.eraseFromParent();
1076 return Legalized;
1077 }
1078 }
1079}
1080
1081void LegalizerHelper::widenScalarSrc(MachineInstr &MI, LLT WideTy,
1082 unsigned OpIdx, unsigned ExtOpcode) {
1083 MachineOperand &MO = MI.getOperand(OpIdx);
1084 auto ExtB = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MO.getReg()});
1085 MO.setReg(ExtB->getOperand(0).getReg());
1086}
1087
1088void LegalizerHelper::narrowScalarSrc(MachineInstr &MI, LLT NarrowTy,
1089 unsigned OpIdx) {
1090 MachineOperand &MO = MI.getOperand(OpIdx);
1091 auto ExtB = MIRBuilder.buildInstr(TargetOpcode::G_TRUNC, {NarrowTy},
1092 {MO.getReg()});
1093 MO.setReg(ExtB->getOperand(0).getReg());
1094}
1095
1096void LegalizerHelper::widenScalarDst(MachineInstr &MI, LLT WideTy,
1097 unsigned OpIdx, unsigned TruncOpcode) {
1098 MachineOperand &MO = MI.getOperand(OpIdx);
1099 Register DstExt = MRI.createGenericVirtualRegister(WideTy);
1100 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1101 MIRBuilder.buildInstr(TruncOpcode, {MO.getReg()}, {DstExt});
1102 MO.setReg(DstExt);
1103}
1104
1105void LegalizerHelper::narrowScalarDst(MachineInstr &MI, LLT NarrowTy,
1106 unsigned OpIdx, unsigned ExtOpcode) {
1107 MachineOperand &MO = MI.getOperand(OpIdx);
1108 Register DstTrunc = MRI.createGenericVirtualRegister(NarrowTy);
1109 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1110 MIRBuilder.buildInstr(ExtOpcode, {MO.getReg()}, {DstTrunc});
1111 MO.setReg(DstTrunc);
1112}
1113
1114void LegalizerHelper::moreElementsVectorDst(MachineInstr &MI, LLT WideTy,
1115 unsigned OpIdx) {
1116 MachineOperand &MO = MI.getOperand(OpIdx);
1117 Register DstExt = MRI.createGenericVirtualRegister(WideTy);
1118 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1119 MIRBuilder.buildExtract(MO.getReg(), DstExt, 0);
1120 MO.setReg(DstExt);
1121}
1122
1123void LegalizerHelper::moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy,
1124 unsigned OpIdx) {
1125 MachineOperand &MO = MI.getOperand(OpIdx);
1126
1127 LLT OldTy = MRI.getType(MO.getReg());
1128 unsigned OldElts = OldTy.getNumElements();
1129 unsigned NewElts = MoreTy.getNumElements();
1130
1131 unsigned NumParts = NewElts / OldElts;
1132
1133 // Use concat_vectors if the result is a multiple of the number of elements.
1134 if (NumParts * OldElts == NewElts) {
1135 SmallVector<Register, 8> Parts;
1136 Parts.push_back(MO.getReg());
1137
1138 Register ImpDef = MIRBuilder.buildUndef(OldTy).getReg(0);
1139 for (unsigned I = 1; I != NumParts; ++I)
1140 Parts.push_back(ImpDef);
1141
1142 auto Concat = MIRBuilder.buildConcatVectors(MoreTy, Parts);
1143 MO.setReg(Concat.getReg(0));
1144 return;
1145 }
1146
1147 Register MoreReg = MRI.createGenericVirtualRegister(MoreTy);
1148 Register ImpDef = MIRBuilder.buildUndef(MoreTy).getReg(0);
1149 MIRBuilder.buildInsert(MoreReg, ImpDef, MO.getReg(), 0);
1150 MO.setReg(MoreReg);
1151}
1152
1153LegalizerHelper::LegalizeResult
1154LegalizerHelper::widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx,
1155 LLT WideTy) {
1156 if (TypeIdx != 1)
1157 return UnableToLegalize;
1158
1159 Register DstReg = MI.getOperand(0).getReg();
1160 LLT DstTy = MRI.getType(DstReg);
1161 if (DstTy.isVector())
1162 return UnableToLegalize;
1163
1164 Register Src1 = MI.getOperand(1).getReg();
1165 LLT SrcTy = MRI.getType(Src1);
1166 const int DstSize = DstTy.getSizeInBits();
1167 const int SrcSize = SrcTy.getSizeInBits();
1168 const int WideSize = WideTy.getSizeInBits();
1169 const int NumMerge = (DstSize + WideSize - 1) / WideSize;
1170
1171 unsigned NumOps = MI.getNumOperands();
1172 unsigned NumSrc = MI.getNumOperands() - 1;
1173 unsigned PartSize = DstTy.getSizeInBits() / NumSrc;
1174
1175 if (WideSize >= DstSize) {
1176 // Directly pack the bits in the target type.
1177 Register ResultReg = MIRBuilder.buildZExt(WideTy, Src1).getReg(0);
1178
1179 for (unsigned I = 2; I != NumOps; ++I) {
1180 const unsigned Offset = (I - 1) * PartSize;
1181
1182 Register SrcReg = MI.getOperand(I).getReg();
1183 assert(MRI.getType(SrcReg) == LLT::scalar(PartSize))((MRI.getType(SrcReg) == LLT::scalar(PartSize)) ? static_cast
<void> (0) : __assert_fail ("MRI.getType(SrcReg) == LLT::scalar(PartSize)"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1183, __PRETTY_FUNCTION__))
;
1184
1185 auto ZextInput = MIRBuilder.buildZExt(WideTy, SrcReg);
1186
1187 Register NextResult = I + 1 == NumOps && WideTy == DstTy ? DstReg :
1188 MRI.createGenericVirtualRegister(WideTy);
1189
1190 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, Offset);
1191 auto Shl = MIRBuilder.buildShl(WideTy, ZextInput, ShiftAmt);
1192 MIRBuilder.buildOr(NextResult, ResultReg, Shl);
1193 ResultReg = NextResult;
1194 }
1195
1196 if (WideSize > DstSize)
1197 MIRBuilder.buildTrunc(DstReg, ResultReg);
1198 else if (DstTy.isPointer())
1199 MIRBuilder.buildIntToPtr(DstReg, ResultReg);
1200
1201 MI.eraseFromParent();
1202 return Legalized;
1203 }
1204
1205 // Unmerge the original values to the GCD type, and recombine to the next
1206 // multiple greater than the original type.
1207 //
1208 // %3:_(s12) = G_MERGE_VALUES %0:_(s4), %1:_(s4), %2:_(s4) -> s6
1209 // %4:_(s2), %5:_(s2) = G_UNMERGE_VALUES %0
1210 // %6:_(s2), %7:_(s2) = G_UNMERGE_VALUES %1
1211 // %8:_(s2), %9:_(s2) = G_UNMERGE_VALUES %2
1212 // %10:_(s6) = G_MERGE_VALUES %4, %5, %6
1213 // %11:_(s6) = G_MERGE_VALUES %7, %8, %9
1214 // %12:_(s12) = G_MERGE_VALUES %10, %11
1215 //
1216 // Padding with undef if necessary:
1217 //
1218 // %2:_(s8) = G_MERGE_VALUES %0:_(s4), %1:_(s4) -> s6
1219 // %3:_(s2), %4:_(s2) = G_UNMERGE_VALUES %0
1220 // %5:_(s2), %6:_(s2) = G_UNMERGE_VALUES %1
1221 // %7:_(s2) = G_IMPLICIT_DEF
1222 // %8:_(s6) = G_MERGE_VALUES %3, %4, %5
1223 // %9:_(s6) = G_MERGE_VALUES %6, %7, %7
1224 // %10:_(s12) = G_MERGE_VALUES %8, %9
1225
1226 const int GCD = greatestCommonDivisor(SrcSize, WideSize);
1227 LLT GCDTy = LLT::scalar(GCD);
1228
1229 SmallVector<Register, 8> Parts;
1230 SmallVector<Register, 8> NewMergeRegs;
1231 SmallVector<Register, 8> Unmerges;
1232 LLT WideDstTy = LLT::scalar(NumMerge * WideSize);
1233
1234 // Decompose the original operands if they don't evenly divide.
1235 for (int I = 1, E = MI.getNumOperands(); I != E; ++I) {
1236 Register SrcReg = MI.getOperand(I).getReg();
1237 if (GCD == SrcSize) {
1238 Unmerges.push_back(SrcReg);
1239 } else {
1240 auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg);
1241 for (int J = 0, JE = Unmerge->getNumOperands() - 1; J != JE; ++J)
1242 Unmerges.push_back(Unmerge.getReg(J));
1243 }
1244 }
1245
1246 // Pad with undef to the next size that is a multiple of the requested size.
1247 if (static_cast<int>(Unmerges.size()) != NumMerge * WideSize) {
1248 Register UndefReg = MIRBuilder.buildUndef(GCDTy).getReg(0);
1249 for (int I = Unmerges.size(); I != NumMerge * WideSize; ++I)
1250 Unmerges.push_back(UndefReg);
1251 }
1252
1253 const int PartsPerGCD = WideSize / GCD;
1254
1255 // Build merges of each piece.
1256 ArrayRef<Register> Slicer(Unmerges);
1257 for (int I = 0; I != NumMerge; ++I, Slicer = Slicer.drop_front(PartsPerGCD)) {
1258 auto Merge = MIRBuilder.buildMerge(WideTy, Slicer.take_front(PartsPerGCD));
1259 NewMergeRegs.push_back(Merge.getReg(0));
1260 }
1261
1262 // A truncate may be necessary if the requested type doesn't evenly divide the
1263 // original result type.
1264 if (DstTy.getSizeInBits() == WideDstTy.getSizeInBits()) {
1265 MIRBuilder.buildMerge(DstReg, NewMergeRegs);
1266 } else {
1267 auto FinalMerge = MIRBuilder.buildMerge(WideDstTy, NewMergeRegs);
1268 MIRBuilder.buildTrunc(DstReg, FinalMerge.getReg(0));
1269 }
1270
1271 MI.eraseFromParent();
1272 return Legalized;
1273}
1274
1275LegalizerHelper::LegalizeResult
1276LegalizerHelper::widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx,
1277 LLT WideTy) {
1278 if (TypeIdx != 0)
1279 return UnableToLegalize;
1280
1281 unsigned NumDst = MI.getNumOperands() - 1;
1282 Register SrcReg = MI.getOperand(NumDst).getReg();
1283 LLT SrcTy = MRI.getType(SrcReg);
1284 if (!SrcTy.isScalar())
1285 return UnableToLegalize;
1286
1287 Register Dst0Reg = MI.getOperand(0).getReg();
1288 LLT DstTy = MRI.getType(Dst0Reg);
1289 if (!DstTy.isScalar())
1290 return UnableToLegalize;
1291
1292 unsigned NewSrcSize = NumDst * WideTy.getSizeInBits();
1293 LLT NewSrcTy = LLT::scalar(NewSrcSize);
1294 unsigned SizeDiff = WideTy.getSizeInBits() - DstTy.getSizeInBits();
1295
1296 auto WideSrc = MIRBuilder.buildZExt(NewSrcTy, SrcReg);
1297
1298 for (unsigned I = 1; I != NumDst; ++I) {
1299 auto ShiftAmt = MIRBuilder.buildConstant(NewSrcTy, SizeDiff * I);
1300 auto Shl = MIRBuilder.buildShl(NewSrcTy, WideSrc, ShiftAmt);
1301 WideSrc = MIRBuilder.buildOr(NewSrcTy, WideSrc, Shl);
1302 }
1303
1304 Observer.changingInstr(MI);
1305
1306 MI.getOperand(NumDst).setReg(WideSrc->getOperand(0).getReg());
1307 for (unsigned I = 0; I != NumDst; ++I)
1308 widenScalarDst(MI, WideTy, I);
1309
1310 Observer.changedInstr(MI);
1311
1312 return Legalized;
1313}
1314
1315LegalizerHelper::LegalizeResult
1316LegalizerHelper::widenScalarExtract(MachineInstr &MI, unsigned TypeIdx,
1317 LLT WideTy) {
1318 Register DstReg = MI.getOperand(0).getReg();
1319 Register SrcReg = MI.getOperand(1).getReg();
1320 LLT SrcTy = MRI.getType(SrcReg);
1321
1322 LLT DstTy = MRI.getType(DstReg);
1323 unsigned Offset = MI.getOperand(2).getImm();
1324
1325 if (TypeIdx == 0) {
1326 if (SrcTy.isVector() || DstTy.isVector())
1327 return UnableToLegalize;
1328
1329 SrcOp Src(SrcReg);
1330 if (SrcTy.isPointer()) {
1331 // Extracts from pointers can be handled only if they are really just
1332 // simple integers.
1333 const DataLayout &DL = MIRBuilder.getDataLayout();
1334 if (DL.isNonIntegralAddressSpace(SrcTy.getAddressSpace()))
1335 return UnableToLegalize;
1336
1337 LLT SrcAsIntTy = LLT::scalar(SrcTy.getSizeInBits());
1338 Src = MIRBuilder.buildPtrToInt(SrcAsIntTy, Src);
1339 SrcTy = SrcAsIntTy;
1340 }
1341
1342 if (DstTy.isPointer())
1343 return UnableToLegalize;
1344
1345 if (Offset == 0) {
1346 // Avoid a shift in the degenerate case.
1347 MIRBuilder.buildTrunc(DstReg,
1348 MIRBuilder.buildAnyExtOrTrunc(WideTy, Src));
1349 MI.eraseFromParent();
1350 return Legalized;
1351 }
1352
1353 // Do a shift in the source type.
1354 LLT ShiftTy = SrcTy;
1355 if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) {
1356 Src = MIRBuilder.buildAnyExt(WideTy, Src);
1357 ShiftTy = WideTy;
1358 } else if (WideTy.getSizeInBits() > SrcTy.getSizeInBits())
1359 return UnableToLegalize;
1360
1361 auto LShr = MIRBuilder.buildLShr(
1362 ShiftTy, Src, MIRBuilder.buildConstant(ShiftTy, Offset));
1363 MIRBuilder.buildTrunc(DstReg, LShr);
1364 MI.eraseFromParent();
1365 return Legalized;
1366 }
1367
1368 if (SrcTy.isScalar()) {
1369 Observer.changingInstr(MI);
1370 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1371 Observer.changedInstr(MI);
1372 return Legalized;
1373 }
1374
1375 if (!SrcTy.isVector())
1376 return UnableToLegalize;
1377
1378 if (DstTy != SrcTy.getElementType())
1379 return UnableToLegalize;
1380
1381 if (Offset % SrcTy.getScalarSizeInBits() != 0)
1382 return UnableToLegalize;
1383
1384 Observer.changingInstr(MI);
1385 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1386
1387 MI.getOperand(2).setImm((WideTy.getSizeInBits() / SrcTy.getSizeInBits()) *
1388 Offset);
1389 widenScalarDst(MI, WideTy.getScalarType(), 0);
1390 Observer.changedInstr(MI);
1391 return Legalized;
1392}
1393
1394LegalizerHelper::LegalizeResult
1395LegalizerHelper::widenScalarInsert(MachineInstr &MI, unsigned TypeIdx,
1396 LLT WideTy) {
1397 if (TypeIdx != 0)
1398 return UnableToLegalize;
1399 Observer.changingInstr(MI);
1400 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1401 widenScalarDst(MI, WideTy);
1402 Observer.changedInstr(MI);
1403 return Legalized;
1404}
1405
1406LegalizerHelper::LegalizeResult
1407LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
1408 MIRBuilder.setInstr(MI);
1409
1410 switch (MI.getOpcode()) {
1411 default:
1412 return UnableToLegalize;
1413 case TargetOpcode::G_EXTRACT:
1414 return widenScalarExtract(MI, TypeIdx, WideTy);
1415 case TargetOpcode::G_INSERT:
1416 return widenScalarInsert(MI, TypeIdx, WideTy);
1417 case TargetOpcode::G_MERGE_VALUES:
1418 return widenScalarMergeValues(MI, TypeIdx, WideTy);
1419 case TargetOpcode::G_UNMERGE_VALUES:
1420 return widenScalarUnmergeValues(MI, TypeIdx, WideTy);
1421 case TargetOpcode::G_UADDO:
1422 case TargetOpcode::G_USUBO: {
1423 if (TypeIdx == 1)
1424 return UnableToLegalize; // TODO
1425 auto LHSZext = MIRBuilder.buildInstr(TargetOpcode::G_ZEXT, {WideTy},
1426 {MI.getOperand(2).getReg()});
1427 auto RHSZext = MIRBuilder.buildInstr(TargetOpcode::G_ZEXT, {WideTy},
1428 {MI.getOperand(3).getReg()});
1429 unsigned Opcode = MI.getOpcode() == TargetOpcode::G_UADDO
1430 ? TargetOpcode::G_ADD
1431 : TargetOpcode::G_SUB;
1432 // Do the arithmetic in the larger type.
1433 auto NewOp = MIRBuilder.buildInstr(Opcode, {WideTy}, {LHSZext, RHSZext});
1434 LLT OrigTy = MRI.getType(MI.getOperand(0).getReg());
1435 APInt Mask = APInt::getAllOnesValue(OrigTy.getSizeInBits());
1436 auto AndOp = MIRBuilder.buildInstr(
1437 TargetOpcode::G_AND, {WideTy},
1438 {NewOp, MIRBuilder.buildConstant(WideTy, Mask.getZExtValue())});
1439 // There is no overflow if the AndOp is the same as NewOp.
1440 MIRBuilder.buildICmp(CmpInst::ICMP_NE, MI.getOperand(1).getReg(), NewOp,
1441 AndOp);
1442 // Now trunc the NewOp to the original result.
1443 MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), NewOp);
1444 MI.eraseFromParent();
1445 return Legalized;
1446 }
1447 case TargetOpcode::G_CTTZ:
1448 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
1449 case TargetOpcode::G_CTLZ:
1450 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
1451 case TargetOpcode::G_CTPOP: {
1452 if (TypeIdx == 0) {
1453 Observer.changingInstr(MI);
1454 widenScalarDst(MI, WideTy, 0);
1455 Observer.changedInstr(MI);
1456 return Legalized;
1457 }
1458
1459 Register SrcReg = MI.getOperand(1).getReg();
1460
1461 // First ZEXT the input.
1462 auto MIBSrc = MIRBuilder.buildZExt(WideTy, SrcReg);
1463 LLT CurTy = MRI.getType(SrcReg);
1464 if (MI.getOpcode() == TargetOpcode::G_CTTZ) {
1465 // The count is the same in the larger type except if the original
1466 // value was zero. This can be handled by setting the bit just off
1467 // the top of the original type.
1468 auto TopBit =
1469 APInt::getOneBitSet(WideTy.getSizeInBits(), CurTy.getSizeInBits());
1470 MIBSrc = MIRBuilder.buildOr(
1471 WideTy, MIBSrc, MIRBuilder.buildConstant(WideTy, TopBit));
1472 }
1473
1474 // Perform the operation at the larger size.
1475 auto MIBNewOp = MIRBuilder.buildInstr(MI.getOpcode(), {WideTy}, {MIBSrc});
1476 // This is already the correct result for CTPOP and CTTZs
1477 if (MI.getOpcode() == TargetOpcode::G_CTLZ ||
1478 MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF) {
1479 // The correct result is NewOp - (Difference in widety and current ty).
1480 unsigned SizeDiff = WideTy.getSizeInBits() - CurTy.getSizeInBits();
1481 MIBNewOp = MIRBuilder.buildInstr(
1482 TargetOpcode::G_SUB, {WideTy},
1483 {MIBNewOp, MIRBuilder.buildConstant(WideTy, SizeDiff)});
1484 }
1485
1486 MIRBuilder.buildZExtOrTrunc(MI.getOperand(0), MIBNewOp);
1487 MI.eraseFromParent();
1488 return Legalized;
1489 }
1490 case TargetOpcode::G_BSWAP: {
1491 Observer.changingInstr(MI);
1492 Register DstReg = MI.getOperand(0).getReg();
1493
1494 Register ShrReg = MRI.createGenericVirtualRegister(WideTy);
1495 Register DstExt = MRI.createGenericVirtualRegister(WideTy);
1496 Register ShiftAmtReg = MRI.createGenericVirtualRegister(WideTy);
1497 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1498
1499 MI.getOperand(0).setReg(DstExt);
1500
1501 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1502
1503 LLT Ty = MRI.getType(DstReg);
1504 unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits();
1505 MIRBuilder.buildConstant(ShiftAmtReg, DiffBits);
1506 MIRBuilder.buildInstr(TargetOpcode::G_LSHR)
1507 .addDef(ShrReg)
1508 .addUse(DstExt)
1509 .addUse(ShiftAmtReg);
1510
1511 MIRBuilder.buildTrunc(DstReg, ShrReg);
1512 Observer.changedInstr(MI);
1513 return Legalized;
1514 }
1515 case TargetOpcode::G_BITREVERSE: {
1516 Observer.changingInstr(MI);
1517
1518 Register DstReg = MI.getOperand(0).getReg();
1519 LLT Ty = MRI.getType(DstReg);
1520 unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits();
1521
1522 Register DstExt = MRI.createGenericVirtualRegister(WideTy);
1523 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1524 MI.getOperand(0).setReg(DstExt);
1525 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1526
1527 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, DiffBits);
1528 auto Shift = MIRBuilder.buildLShr(WideTy, DstExt, ShiftAmt);
1529 MIRBuilder.buildTrunc(DstReg, Shift);
1530 Observer.changedInstr(MI);
1531 return Legalized;
1532 }
1533 case TargetOpcode::G_ADD:
1534 case TargetOpcode::G_AND:
1535 case TargetOpcode::G_MUL:
1536 case TargetOpcode::G_OR:
1537 case TargetOpcode::G_XOR:
1538 case TargetOpcode::G_SUB:
1539 // Perform operation at larger width (any extension is fines here, high bits
1540 // don't affect the result) and then truncate the result back to the
1541 // original type.
1542 Observer.changingInstr(MI);
1543 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1544 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
1545 widenScalarDst(MI, WideTy);
1546 Observer.changedInstr(MI);
1547 return Legalized;
1548
1549 case TargetOpcode::G_SHL:
1550 Observer.changingInstr(MI);
1551
1552 if (TypeIdx == 0) {
1553 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1554 widenScalarDst(MI, WideTy);
1555 } else {
1556 assert(TypeIdx == 1)((TypeIdx == 1) ? static_cast<void> (0) : __assert_fail
("TypeIdx == 1", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1556, __PRETTY_FUNCTION__))
;
1557 // The "number of bits to shift" operand must preserve its value as an
1558 // unsigned integer:
1559 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
1560 }
1561
1562 Observer.changedInstr(MI);
1563 return Legalized;
1564
1565 case TargetOpcode::G_SDIV:
1566 case TargetOpcode::G_SREM:
1567 case TargetOpcode::G_SMIN:
1568 case TargetOpcode::G_SMAX:
1569 Observer.changingInstr(MI);
1570 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
1571 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
1572 widenScalarDst(MI, WideTy);
1573 Observer.changedInstr(MI);
1574 return Legalized;
1575
1576 case TargetOpcode::G_ASHR:
1577 case TargetOpcode::G_LSHR:
1578 Observer.changingInstr(MI);
1579
1580 if (TypeIdx == 0) {
1581 unsigned CvtOp = MI.getOpcode() == TargetOpcode::G_ASHR ?
1582 TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
1583
1584 widenScalarSrc(MI, WideTy, 1, CvtOp);
1585 widenScalarDst(MI, WideTy);
1586 } else {
1587 assert(TypeIdx == 1)((TypeIdx == 1) ? static_cast<void> (0) : __assert_fail
("TypeIdx == 1", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1587, __PRETTY_FUNCTION__))
;
1588 // The "number of bits to shift" operand must preserve its value as an
1589 // unsigned integer:
1590 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
1591 }
1592
1593 Observer.changedInstr(MI);
1594 return Legalized;
1595 case TargetOpcode::G_UDIV:
1596 case TargetOpcode::G_UREM:
1597 case TargetOpcode::G_UMIN:
1598 case TargetOpcode::G_UMAX:
1599 Observer.changingInstr(MI);
1600 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
1601 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
1602 widenScalarDst(MI, WideTy);
1603 Observer.changedInstr(MI);
1604 return Legalized;
1605
1606 case TargetOpcode::G_SELECT:
1607 Observer.changingInstr(MI);
1608 if (TypeIdx == 0) {
1609 // Perform operation at larger width (any extension is fine here, high
1610 // bits don't affect the result) and then truncate the result back to the
1611 // original type.
1612 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
1613 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ANYEXT);
1614 widenScalarDst(MI, WideTy);
1615 } else {
1616 bool IsVec = MRI.getType(MI.getOperand(1).getReg()).isVector();
1617 // Explicit extension is required here since high bits affect the result.
1618 widenScalarSrc(MI, WideTy, 1, MIRBuilder.getBoolExtOp(IsVec, false));
1619 }
1620 Observer.changedInstr(MI);
1621 return Legalized;
1622
1623 case TargetOpcode::G_FPTOSI:
1624 case TargetOpcode::G_FPTOUI:
1625 Observer.changingInstr(MI);
1626
1627 if (TypeIdx == 0)
1628 widenScalarDst(MI, WideTy);
1629 else
1630 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT);
1631
1632 Observer.changedInstr(MI);
1633 return Legalized;
1634 case TargetOpcode::G_SITOFP:
1635 if (TypeIdx != 1)
1636 return UnableToLegalize;
1637 Observer.changingInstr(MI);
1638 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
1639 Observer.changedInstr(MI);
1640 return Legalized;
1641
1642 case TargetOpcode::G_UITOFP:
1643 if (TypeIdx != 1)
1644 return UnableToLegalize;
1645 Observer.changingInstr(MI);
1646 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
1647 Observer.changedInstr(MI);
1648 return Legalized;
1649
1650 case TargetOpcode::G_LOAD:
1651 case TargetOpcode::G_SEXTLOAD:
1652 case TargetOpcode::G_ZEXTLOAD:
1653 Observer.changingInstr(MI);
1654 widenScalarDst(MI, WideTy);
1655 Observer.changedInstr(MI);
1656 return Legalized;
1657
1658 case TargetOpcode::G_STORE: {
1659 if (TypeIdx != 0)
1660 return UnableToLegalize;
1661
1662 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
1663 if (!isPowerOf2_32(Ty.getSizeInBits()))
1664 return UnableToLegalize;
1665
1666 Observer.changingInstr(MI);
1667
1668 unsigned ExtType = Ty.getScalarSizeInBits() == 1 ?
1669 TargetOpcode::G_ZEXT : TargetOpcode::G_ANYEXT;
1670 widenScalarSrc(MI, WideTy, 0, ExtType);
1671
1672 Observer.changedInstr(MI);
1673 return Legalized;
1674 }
1675 case TargetOpcode::G_CONSTANT: {
1676 MachineOperand &SrcMO = MI.getOperand(1);
1677 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
1678 const APInt &Val = SrcMO.getCImm()->getValue().sext(WideTy.getSizeInBits());
1679 Observer.changingInstr(MI);
1680 SrcMO.setCImm(ConstantInt::get(Ctx, Val));
1681
1682 widenScalarDst(MI, WideTy);
1683 Observer.changedInstr(MI);
1684 return Legalized;
1685 }
1686 case TargetOpcode::G_FCONSTANT: {
1687 MachineOperand &SrcMO = MI.getOperand(1);
1688 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
1689 APFloat Val = SrcMO.getFPImm()->getValueAPF();
1690 bool LosesInfo;
1691 switch (WideTy.getSizeInBits()) {
1692 case 32:
1693 Val.convert(APFloat::IEEEsingle(), APFloat::rmNearestTiesToEven,
1694 &LosesInfo);
1695 break;
1696 case 64:
1697 Val.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
1698 &LosesInfo);
1699 break;
1700 default:
1701 return UnableToLegalize;
1702 }
1703
1704 assert(!LosesInfo && "extend should always be lossless")((!LosesInfo && "extend should always be lossless") ?
static_cast<void> (0) : __assert_fail ("!LosesInfo && \"extend should always be lossless\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1704, __PRETTY_FUNCTION__))
;
1705
1706 Observer.changingInstr(MI);
1707 SrcMO.setFPImm(ConstantFP::get(Ctx, Val));
1708
1709 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
1710 Observer.changedInstr(MI);
1711 return Legalized;
1712 }
1713 case TargetOpcode::G_IMPLICIT_DEF: {
1714 Observer.changingInstr(MI);
1715 widenScalarDst(MI, WideTy);
1716 Observer.changedInstr(MI);
1717 return Legalized;
1718 }
1719 case TargetOpcode::G_BRCOND:
1720 Observer.changingInstr(MI);
1721 widenScalarSrc(MI, WideTy, 0, MIRBuilder.getBoolExtOp(false, false));
1722 Observer.changedInstr(MI);
1723 return Legalized;
1724
1725 case TargetOpcode::G_FCMP:
1726 Observer.changingInstr(MI);
1727 if (TypeIdx == 0)
1728 widenScalarDst(MI, WideTy);
1729 else {
1730 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_FPEXT);
1731 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_FPEXT);
1732 }
1733 Observer.changedInstr(MI);
1734 return Legalized;
1735
1736 case TargetOpcode::G_ICMP:
1737 Observer.changingInstr(MI);
1738 if (TypeIdx == 0)
1739 widenScalarDst(MI, WideTy);
1740 else {
1741 unsigned ExtOpcode = CmpInst::isSigned(static_cast<CmpInst::Predicate>(
1742 MI.getOperand(1).getPredicate()))
1743 ? TargetOpcode::G_SEXT
1744 : TargetOpcode::G_ZEXT;
1745 widenScalarSrc(MI, WideTy, 2, ExtOpcode);
1746 widenScalarSrc(MI, WideTy, 3, ExtOpcode);
1747 }
1748 Observer.changedInstr(MI);
1749 return Legalized;
1750
1751 case TargetOpcode::G_PTR_ADD:
1752 assert(TypeIdx == 1 && "unable to legalize pointer of G_PTR_ADD")((TypeIdx == 1 && "unable to legalize pointer of G_PTR_ADD"
) ? static_cast<void> (0) : __assert_fail ("TypeIdx == 1 && \"unable to legalize pointer of G_PTR_ADD\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1752, __PRETTY_FUNCTION__))
;
1753 Observer.changingInstr(MI);
1754 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
1755 Observer.changedInstr(MI);
1756 return Legalized;
1757
1758 case TargetOpcode::G_PHI: {
1759 assert(TypeIdx == 0 && "Expecting only Idx 0")((TypeIdx == 0 && "Expecting only Idx 0") ? static_cast
<void> (0) : __assert_fail ("TypeIdx == 0 && \"Expecting only Idx 0\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1759, __PRETTY_FUNCTION__))
;
1760
1761 Observer.changingInstr(MI);
1762 for (unsigned I = 1; I < MI.getNumOperands(); I += 2) {
1763 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
1764 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
1765 widenScalarSrc(MI, WideTy, I, TargetOpcode::G_ANYEXT);
1766 }
1767
1768 MachineBasicBlock &MBB = *MI.getParent();
1769 MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI());
1770 widenScalarDst(MI, WideTy);
1771 Observer.changedInstr(MI);
1772 return Legalized;
1773 }
1774 case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
1775 if (TypeIdx == 0) {
1776 Register VecReg = MI.getOperand(1).getReg();
1777 LLT VecTy = MRI.getType(VecReg);
1778 Observer.changingInstr(MI);
1779
1780 widenScalarSrc(MI, LLT::vector(VecTy.getNumElements(),
1781 WideTy.getSizeInBits()),
1782 1, TargetOpcode::G_SEXT);
1783
1784 widenScalarDst(MI, WideTy, 0);
1785 Observer.changedInstr(MI);
1786 return Legalized;
1787 }
1788
1789 if (TypeIdx != 2)
1790 return UnableToLegalize;
1791 Observer.changingInstr(MI);
1792 // TODO: Probably should be zext
1793 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
1794 Observer.changedInstr(MI);
1795 return Legalized;
1796 }
1797 case TargetOpcode::G_INSERT_VECTOR_ELT: {
1798 if (TypeIdx == 1) {
1799 Observer.changingInstr(MI);
1800
1801 Register VecReg = MI.getOperand(1).getReg();
1802 LLT VecTy = MRI.getType(VecReg);
1803 LLT WideVecTy = LLT::vector(VecTy.getNumElements(), WideTy);
1804
1805 widenScalarSrc(MI, WideVecTy, 1, TargetOpcode::G_ANYEXT);
1806 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
1807 widenScalarDst(MI, WideVecTy, 0);
1808 Observer.changedInstr(MI);
1809 return Legalized;
1810 }
1811
1812 if (TypeIdx == 2) {
1813 Observer.changingInstr(MI);
1814 // TODO: Probably should be zext
1815 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_SEXT);
1816 Observer.changedInstr(MI);
1817 }
1818
1819 return Legalized;
1820 }
1821 case TargetOpcode::G_FADD:
1822 case TargetOpcode::G_FMUL:
1823 case TargetOpcode::G_FSUB:
1824 case TargetOpcode::G_FMA:
1825 case TargetOpcode::G_FMAD:
1826 case TargetOpcode::G_FNEG:
1827 case TargetOpcode::G_FABS:
1828 case TargetOpcode::G_FCANONICALIZE:
1829 case TargetOpcode::G_FMINNUM:
1830 case TargetOpcode::G_FMAXNUM:
1831 case TargetOpcode::G_FMINNUM_IEEE:
1832 case TargetOpcode::G_FMAXNUM_IEEE:
1833 case TargetOpcode::G_FMINIMUM:
1834 case TargetOpcode::G_FMAXIMUM:
1835 case TargetOpcode::G_FDIV:
1836 case TargetOpcode::G_FREM:
1837 case TargetOpcode::G_FCEIL:
1838 case TargetOpcode::G_FFLOOR:
1839 case TargetOpcode::G_FCOS:
1840 case TargetOpcode::G_FSIN:
1841 case TargetOpcode::G_FLOG10:
1842 case TargetOpcode::G_FLOG:
1843 case TargetOpcode::G_FLOG2:
1844 case TargetOpcode::G_FRINT:
1845 case TargetOpcode::G_FNEARBYINT:
1846 case TargetOpcode::G_FSQRT:
1847 case TargetOpcode::G_FEXP:
1848 case TargetOpcode::G_FEXP2:
1849 case TargetOpcode::G_FPOW:
1850 case TargetOpcode::G_INTRINSIC_TRUNC:
1851 case TargetOpcode::G_INTRINSIC_ROUND:
1852 assert(TypeIdx == 0)((TypeIdx == 0) ? static_cast<void> (0) : __assert_fail
("TypeIdx == 0", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1852, __PRETTY_FUNCTION__))
;
1853 Observer.changingInstr(MI);
1854
1855 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I)
1856 widenScalarSrc(MI, WideTy, I, TargetOpcode::G_FPEXT);
1857
1858 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
1859 Observer.changedInstr(MI);
1860 return Legalized;
1861 case TargetOpcode::G_INTTOPTR:
1862 if (TypeIdx != 1)
1863 return UnableToLegalize;
1864
1865 Observer.changingInstr(MI);
1866 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
1867 Observer.changedInstr(MI);
1868 return Legalized;
1869 case TargetOpcode::G_PTRTOINT:
1870 if (TypeIdx != 0)
1871 return UnableToLegalize;
1872
1873 Observer.changingInstr(MI);
1874 widenScalarDst(MI, WideTy, 0);
1875 Observer.changedInstr(MI);
1876 return Legalized;
1877 case TargetOpcode::G_BUILD_VECTOR: {
1878 Observer.changingInstr(MI);
1879
1880 const LLT WideEltTy = TypeIdx == 1 ? WideTy : WideTy.getElementType();
1881 for (int I = 1, E = MI.getNumOperands(); I != E; ++I)
1882 widenScalarSrc(MI, WideEltTy, I, TargetOpcode::G_ANYEXT);
1883
1884 // Avoid changing the result vector type if the source element type was
1885 // requested.
1886 if (TypeIdx == 1) {
1887 auto &TII = *MI.getMF()->getSubtarget().getInstrInfo();
1888 MI.setDesc(TII.get(TargetOpcode::G_BUILD_VECTOR_TRUNC));
1889 } else {
1890 widenScalarDst(MI, WideTy, 0);
1891 }
1892
1893 Observer.changedInstr(MI);
1894 return Legalized;
1895 }
1896 case TargetOpcode::G_SEXT_INREG:
1897 if (TypeIdx != 0)
1898 return UnableToLegalize;
1899
1900 Observer.changingInstr(MI);
1901 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1902 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_TRUNC);
1903 Observer.changedInstr(MI);
1904 return Legalized;
1905 }
1906}
1907
1908LegalizerHelper::LegalizeResult
1909LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
1910 using namespace TargetOpcode;
1911 MIRBuilder.setInstr(MI);
1912
1913 switch(MI.getOpcode()) {
1914 default:
1915 return UnableToLegalize;
1916 case TargetOpcode::G_SREM:
1917 case TargetOpcode::G_UREM: {
1918 Register QuotReg = MRI.createGenericVirtualRegister(Ty);
1919 MIRBuilder.buildInstr(MI.getOpcode() == G_SREM ? G_SDIV : G_UDIV)
1920 .addDef(QuotReg)
1921 .addUse(MI.getOperand(1).getReg())
1922 .addUse(MI.getOperand(2).getReg());
1923
1924 Register ProdReg = MRI.createGenericVirtualRegister(Ty);
1925 MIRBuilder.buildMul(ProdReg, QuotReg, MI.getOperand(2).getReg());
1926 MIRBuilder.buildSub(MI.getOperand(0).getReg(), MI.getOperand(1).getReg(),
1927 ProdReg);
1928 MI.eraseFromParent();
1929 return Legalized;
1930 }
1931 case TargetOpcode::G_SADDO:
1932 case TargetOpcode::G_SSUBO:
1933 return lowerSADDO_SSUBO(MI);
1934 case TargetOpcode::G_SMULO:
1935 case TargetOpcode::G_UMULO: {
1936 // Generate G_UMULH/G_SMULH to check for overflow and a normal G_MUL for the
1937 // result.
1938 Register Res = MI.getOperand(0).getReg();
1939 Register Overflow = MI.getOperand(1).getReg();
1940 Register LHS = MI.getOperand(2).getReg();
1941 Register RHS = MI.getOperand(3).getReg();
1942
1943 MIRBuilder.buildMul(Res, LHS, RHS);
1944
1945 unsigned Opcode = MI.getOpcode() == TargetOpcode::G_SMULO
1946 ? TargetOpcode::G_SMULH
1947 : TargetOpcode::G_UMULH;
1948
1949 Register HiPart = MRI.createGenericVirtualRegister(Ty);
1950 MIRBuilder.buildInstr(Opcode)
1951 .addDef(HiPart)
1952 .addUse(LHS)
1953 .addUse(RHS);
1954
1955 Register Zero = MRI.createGenericVirtualRegister(Ty);
1956 MIRBuilder.buildConstant(Zero, 0);
1957
1958 // For *signed* multiply, overflow is detected by checking:
1959 // (hi != (lo >> bitwidth-1))
1960 if (Opcode == TargetOpcode::G_SMULH) {
1961 Register Shifted = MRI.createGenericVirtualRegister(Ty);
1962 Register ShiftAmt = MRI.createGenericVirtualRegister(Ty);
1963 MIRBuilder.buildConstant(ShiftAmt, Ty.getSizeInBits() - 1);
1964 MIRBuilder.buildInstr(TargetOpcode::G_ASHR)
1965 .addDef(Shifted)
1966 .addUse(Res)
1967 .addUse(ShiftAmt);
1968 MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Shifted);
1969 } else {
1970 MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Zero);
1971 }
1972 MI.eraseFromParent();
1973 return Legalized;
1974 }
1975 case TargetOpcode::G_FNEG: {
1976 // TODO: Handle vector types once we are able to
1977 // represent them.
1978 if (Ty.isVector())
1979 return UnableToLegalize;
1980 Register Res = MI.getOperand(0).getReg();
1981 Type *ZeroTy;
1982 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
1983 switch (Ty.getSizeInBits()) {
1984 case 16:
1985 ZeroTy = Type::getHalfTy(Ctx);
1986 break;
1987 case 32:
1988 ZeroTy = Type::getFloatTy(Ctx);
1989 break;
1990 case 64:
1991 ZeroTy = Type::getDoubleTy(Ctx);
1992 break;
1993 case 128:
1994 ZeroTy = Type::getFP128Ty(Ctx);
1995 break;
1996 default:
1997 llvm_unreachable("unexpected floating-point type")::llvm::llvm_unreachable_internal("unexpected floating-point type"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1997)
;
1998 }
1999 ConstantFP &ZeroForNegation =
2000 *cast<ConstantFP>(ConstantFP::getZeroValueForNegation(ZeroTy));
2001 auto Zero = MIRBuilder.buildFConstant(Ty, ZeroForNegation);
2002 Register SubByReg = MI.getOperand(1).getReg();
2003 Register ZeroReg = Zero->getOperand(0).getReg();
2004 MIRBuilder.buildInstr(TargetOpcode::G_FSUB, {Res}, {ZeroReg, SubByReg},
2005 MI.getFlags());
2006 MI.eraseFromParent();
2007 return Legalized;
2008 }
2009 case TargetOpcode::G_FSUB: {
2010 // Lower (G_FSUB LHS, RHS) to (G_FADD LHS, (G_FNEG RHS)).
2011 // First, check if G_FNEG is marked as Lower. If so, we may
2012 // end up with an infinite loop as G_FSUB is used to legalize G_FNEG.
2013 if (LI.getAction({G_FNEG, {Ty}}).Action == Lower)
2014 return UnableToLegalize;
2015 Register Res = MI.getOperand(0).getReg();
2016 Register LHS = MI.getOperand(1).getReg();
2017 Register RHS = MI.getOperand(2).getReg();
2018 Register Neg = MRI.createGenericVirtualRegister(Ty);
2019 MIRBuilder.buildInstr(TargetOpcode::G_FNEG).addDef(Neg).addUse(RHS);
2020 MIRBuilder.buildInstr(TargetOpcode::G_FADD, {Res}, {LHS, Neg}, MI.getFlags());
2021 MI.eraseFromParent();
2022 return Legalized;
2023 }
2024 case TargetOpcode::G_FMAD:
2025 return lowerFMad(MI);
2026 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
2027 Register OldValRes = MI.getOperand(0).getReg();
2028 Register SuccessRes = MI.getOperand(1).getReg();
2029 Register Addr = MI.getOperand(2).getReg();
2030 Register CmpVal = MI.getOperand(3).getReg();
2031 Register NewVal = MI.getOperand(4).getReg();
2032 MIRBuilder.buildAtomicCmpXchg(OldValRes, Addr, CmpVal, NewVal,
2033 **MI.memoperands_begin());
2034 MIRBuilder.buildICmp(CmpInst::ICMP_EQ, SuccessRes, OldValRes, CmpVal);
2035 MI.eraseFromParent();
2036 return Legalized;
2037 }
2038 case TargetOpcode::G_LOAD:
2039 case TargetOpcode::G_SEXTLOAD:
2040 case TargetOpcode::G_ZEXTLOAD: {
2041 // Lower to a memory-width G_LOAD and a G_SEXT/G_ZEXT/G_ANYEXT
2042 Register DstReg = MI.getOperand(0).getReg();
2043 Register PtrReg = MI.getOperand(1).getReg();
2044 LLT DstTy = MRI.getType(DstReg);
2045 auto &MMO = **MI.memoperands_begin();
2046
2047 if (DstTy.getSizeInBits() == MMO.getSizeInBits()) {
2048 if (MI.getOpcode() == TargetOpcode::G_LOAD) {
2049 // This load needs splitting into power of 2 sized loads.
2050 if (DstTy.isVector())
2051 return UnableToLegalize;
2052 if (isPowerOf2_32(DstTy.getSizeInBits()))
2053 return UnableToLegalize; // Don't know what we're being asked to do.
2054
2055 // Our strategy here is to generate anyextending loads for the smaller
2056 // types up to next power-2 result type, and then combine the two larger
2057 // result values together, before truncating back down to the non-pow-2
2058 // type.
2059 // E.g. v1 = i24 load =>
2060 // v2 = i32 load (2 byte)
2061 // v3 = i32 load (1 byte)
2062 // v4 = i32 shl v3, 16
2063 // v5 = i32 or v4, v2
2064 // v1 = i24 trunc v5
2065 // By doing this we generate the correct truncate which should get
2066 // combined away as an artifact with a matching extend.
2067 uint64_t LargeSplitSize = PowerOf2Floor(DstTy.getSizeInBits());
2068 uint64_t SmallSplitSize = DstTy.getSizeInBits() - LargeSplitSize;
2069
2070 MachineFunction &MF = MIRBuilder.getMF();
2071 MachineMemOperand *LargeMMO =
2072 MF.getMachineMemOperand(&MMO, 0, LargeSplitSize / 8);
2073 MachineMemOperand *SmallMMO = MF.getMachineMemOperand(
2074 &MMO, LargeSplitSize / 8, SmallSplitSize / 8);
2075
2076 LLT PtrTy = MRI.getType(PtrReg);
2077 unsigned AnyExtSize = NextPowerOf2(DstTy.getSizeInBits());
2078 LLT AnyExtTy = LLT::scalar(AnyExtSize);
2079 Register LargeLdReg = MRI.createGenericVirtualRegister(AnyExtTy);
2080 Register SmallLdReg = MRI.createGenericVirtualRegister(AnyExtTy);
2081 auto LargeLoad =
2082 MIRBuilder.buildLoad(LargeLdReg, PtrReg, *LargeMMO);
2083
2084 auto OffsetCst =
2085 MIRBuilder.buildConstant(LLT::scalar(64), LargeSplitSize / 8);
2086 Register PtrAddReg = MRI.createGenericVirtualRegister(PtrTy);
2087 auto SmallPtr =
2088 MIRBuilder.buildPtrAdd(PtrAddReg, PtrReg, OffsetCst.getReg(0));
2089 auto SmallLoad = MIRBuilder.buildLoad(SmallLdReg, SmallPtr.getReg(0),
2090 *SmallMMO);
2091
2092 auto ShiftAmt = MIRBuilder.buildConstant(AnyExtTy, LargeSplitSize);
2093 auto Shift = MIRBuilder.buildShl(AnyExtTy, SmallLoad, ShiftAmt);
2094 auto Or = MIRBuilder.buildOr(AnyExtTy, Shift, LargeLoad);
2095 MIRBuilder.buildTrunc(DstReg, {Or.getReg(0)});
2096 MI.eraseFromParent();
2097 return Legalized;
2098 }
2099 MIRBuilder.buildLoad(DstReg, PtrReg, MMO);
2100 MI.eraseFromParent();
2101 return Legalized;
2102 }
2103
2104 if (DstTy.isScalar()) {
2105 Register TmpReg =
2106 MRI.createGenericVirtualRegister(LLT::scalar(MMO.getSizeInBits()));
2107 MIRBuilder.buildLoad(TmpReg, PtrReg, MMO);
2108 switch (MI.getOpcode()) {
2109 default:
2110 llvm_unreachable("Unexpected opcode")::llvm::llvm_unreachable_internal("Unexpected opcode", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 2110)
;
2111 case TargetOpcode::G_LOAD:
2112 MIRBuilder.buildAnyExt(DstReg, TmpReg);
2113 break;
2114 case TargetOpcode::G_SEXTLOAD:
2115 MIRBuilder.buildSExt(DstReg, TmpReg);
2116 break;
2117 case TargetOpcode::G_ZEXTLOAD:
2118 MIRBuilder.buildZExt(DstReg, TmpReg);
2119 break;
2120 }
2121 MI.eraseFromParent();
2122 return Legalized;
2123 }
2124
2125 return UnableToLegalize;
2126 }
2127 case TargetOpcode::G_STORE: {
2128 // Lower a non-power of 2 store into multiple pow-2 stores.
2129 // E.g. split an i24 store into an i16 store + i8 store.
2130 // We do this by first extending the stored value to the next largest power
2131 // of 2 type, and then using truncating stores to store the components.
2132 // By doing this, likewise with G_LOAD, generate an extend that can be
2133 // artifact-combined away instead of leaving behind extracts.
2134 Register SrcReg = MI.getOperand(0).getReg();
2135 Register PtrReg = MI.getOperand(1).getReg();
2136 LLT SrcTy = MRI.getType(SrcReg);
2137 MachineMemOperand &MMO = **MI.memoperands_begin();
2138 if (SrcTy.getSizeInBits() != MMO.getSizeInBits())
2139 return UnableToLegalize;
2140 if (SrcTy.isVector())
2141 return UnableToLegalize;
2142 if (isPowerOf2_32(SrcTy.getSizeInBits()))
2143 return UnableToLegalize; // Don't know what we're being asked to do.
2144
2145 // Extend to the next pow-2.
2146 const LLT ExtendTy = LLT::scalar(NextPowerOf2(SrcTy.getSizeInBits()));
2147 auto ExtVal = MIRBuilder.buildAnyExt(ExtendTy, SrcReg);
2148
2149 // Obtain the smaller value by shifting away the larger value.
2150 uint64_t LargeSplitSize = PowerOf2Floor(SrcTy.getSizeInBits());
2151 uint64_t SmallSplitSize = SrcTy.getSizeInBits() - LargeSplitSize;
2152 auto ShiftAmt = MIRBuilder.buildConstant(ExtendTy, LargeSplitSize);
2153 auto SmallVal = MIRBuilder.buildLShr(ExtendTy, ExtVal, ShiftAmt);
2154
2155 // Generate the PtrAdd and truncating stores.
2156 LLT PtrTy = MRI.getType(PtrReg);
2157 auto OffsetCst =
2158 MIRBuilder.buildConstant(LLT::scalar(64), LargeSplitSize / 8);
2159 Register PtrAddReg = MRI.createGenericVirtualRegister(PtrTy);
2160 auto SmallPtr =
2161 MIRBuilder.buildPtrAdd(PtrAddReg, PtrReg, OffsetCst.getReg(0));
2162
2163 MachineFunction &MF = MIRBuilder.getMF();
2164 MachineMemOperand *LargeMMO =
2165 MF.getMachineMemOperand(&MMO, 0, LargeSplitSize / 8);
2166 MachineMemOperand *SmallMMO =
2167 MF.getMachineMemOperand(&MMO, LargeSplitSize / 8, SmallSplitSize / 8);
2168 MIRBuilder.buildStore(ExtVal.getReg(0), PtrReg, *LargeMMO);
2169 MIRBuilder.buildStore(SmallVal.getReg(0), SmallPtr.getReg(0), *SmallMMO);
2170 MI.eraseFromParent();
2171 return Legalized;
2172 }
2173 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
2174 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
2175 case TargetOpcode::G_CTLZ:
2176 case TargetOpcode::G_CTTZ:
2177 case TargetOpcode::G_CTPOP:
2178 return lowerBitCount(MI, TypeIdx, Ty);
2179 case G_UADDO: {
2180 Register Res = MI.getOperand(0).getReg();
2181 Register CarryOut = MI.getOperand(1).getReg();
2182 Register LHS = MI.getOperand(2).getReg();
2183 Register RHS = MI.getOperand(3).getReg();
2184
2185 MIRBuilder.buildAdd(Res, LHS, RHS);
2186 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, RHS);
2187
2188 MI.eraseFromParent();
2189 return Legalized;
2190 }
2191 case G_UADDE: {
2192 Register Res = MI.getOperand(0).getReg();
2193 Register CarryOut = MI.getOperand(1).getReg();
2194 Register LHS = MI.getOperand(2).getReg();
2195 Register RHS = MI.getOperand(3).getReg();
2196 Register CarryIn = MI.getOperand(4).getReg();
2197
2198 Register TmpRes = MRI.createGenericVirtualRegister(Ty);
2199 Register ZExtCarryIn = MRI.createGenericVirtualRegister(Ty);
2200
2201 MIRBuilder.buildAdd(TmpRes, LHS, RHS);
2202 MIRBuilder.buildZExt(ZExtCarryIn, CarryIn);
2203 MIRBuilder.buildAdd(Res, TmpRes, ZExtCarryIn);
2204 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, LHS);
2205
2206 MI.eraseFromParent();
2207 return Legalized;
2208 }
2209 case G_USUBO: {
2210 Register Res = MI.getOperand(0).getReg();
2211 Register BorrowOut = MI.getOperand(1).getReg();
2212 Register LHS = MI.getOperand(2).getReg();
2213 Register RHS = MI.getOperand(3).getReg();
2214
2215 MIRBuilder.buildSub(Res, LHS, RHS);
2216 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, BorrowOut, LHS, RHS);
2217
2218 MI.eraseFromParent();
2219 return Legalized;
2220 }
2221 case G_USUBE: {
2222 Register Res = MI.getOperand(0).getReg();
2223 Register BorrowOut = MI.getOperand(1).getReg();
2224 Register LHS = MI.getOperand(2).getReg();
2225 Register RHS = MI.getOperand(3).getReg();
2226 Register BorrowIn = MI.getOperand(4).getReg();
2227
2228 Register TmpRes = MRI.createGenericVirtualRegister(Ty);
2229 Register ZExtBorrowIn = MRI.createGenericVirtualRegister(Ty);
2230 Register LHS_EQ_RHS = MRI.createGenericVirtualRegister(LLT::scalar(1));
2231 Register LHS_ULT_RHS = MRI.createGenericVirtualRegister(LLT::scalar(1));
2232
2233 MIRBuilder.buildSub(TmpRes, LHS, RHS);
2234 MIRBuilder.buildZExt(ZExtBorrowIn, BorrowIn);
2235 MIRBuilder.buildSub(Res, TmpRes, ZExtBorrowIn);
2236 MIRBuilder.buildICmp(CmpInst::ICMP_EQ, LHS_EQ_RHS, LHS, RHS);
2237 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, LHS_ULT_RHS, LHS, RHS);
2238 MIRBuilder.buildSelect(BorrowOut, LHS_EQ_RHS, BorrowIn, LHS_ULT_RHS);
2239
2240 MI.eraseFromParent();
2241 return Legalized;
2242 }
2243 case G_UITOFP:
2244 return lowerUITOFP(MI, TypeIdx, Ty);
2245 case G_SITOFP:
2246 return lowerSITOFP(MI, TypeIdx, Ty);
2247 case G_FPTOUI:
2248 return lowerFPTOUI(MI, TypeIdx, Ty);
2249 case G_SMIN:
2250 case G_SMAX:
2251 case G_UMIN:
2252 case G_UMAX:
2253 return lowerMinMax(MI, TypeIdx, Ty);
2254 case G_FCOPYSIGN:
2255 return lowerFCopySign(MI, TypeIdx, Ty);
2256 case G_FMINNUM:
2257 case G_FMAXNUM:
2258 return lowerFMinNumMaxNum(MI);
2259 case G_UNMERGE_VALUES:
2260 return lowerUnmergeValues(MI);
2261 case TargetOpcode::G_SEXT_INREG: {
2262 assert(MI.getOperand(2).isImm() && "Expected immediate")((MI.getOperand(2).isImm() && "Expected immediate") ?
static_cast<void> (0) : __assert_fail ("MI.getOperand(2).isImm() && \"Expected immediate\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 2262, __PRETTY_FUNCTION__))
;
2263 int64_t SizeInBits = MI.getOperand(2).getImm();
2264
2265 Register DstReg = MI.getOperand(0).getReg();
2266 Register SrcReg = MI.getOperand(1).getReg();
2267 LLT DstTy = MRI.getType(DstReg);
2268 Register TmpRes = MRI.createGenericVirtualRegister(DstTy);
2269
2270 auto MIBSz = MIRBuilder.buildConstant(DstTy, DstTy.getScalarSizeInBits() - SizeInBits);
2271 MIRBuilder.buildInstr(TargetOpcode::G_SHL, {TmpRes}, {SrcReg, MIBSz->getOperand(0).getReg()});
2272 MIRBuilder.buildInstr(TargetOpcode::G_ASHR, {DstReg}, {TmpRes, MIBSz->getOperand(0).getReg()});
2273 MI.eraseFromParent();
2274 return Legalized;
2275 }
2276 case G_SHUFFLE_VECTOR:
2277 return lowerShuffleVector(MI);
2278 case G_DYN_STACKALLOC:
2279 return lowerDynStackAlloc(MI);
2280 case G_EXTRACT:
2281 return lowerExtract(MI);
2282 case G_INSERT:
2283 return lowerInsert(MI);
2284 }
2285}
2286
2287LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorImplicitDef(
2288 MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) {
2289 SmallVector<Register, 2> DstRegs;
2290
2291 unsigned NarrowSize = NarrowTy.getSizeInBits();
2292 Register DstReg = MI.getOperand(0).getReg();
2293 unsigned Size = MRI.getType(DstReg).getSizeInBits();
2294 int NumParts = Size / NarrowSize;
2295 // FIXME: Don't know how to handle the situation where the small vectors
2296 // aren't all the same size yet.
2297 if (Size % NarrowSize != 0)
2298 return UnableToLegalize;
2299
2300 for (int i = 0; i < NumParts; ++i) {
2301 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
2302 MIRBuilder.buildUndef(TmpReg);
2303 DstRegs.push_back(TmpReg);
2304 }
2305
2306 if (NarrowTy.isVector())
2307 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
2308 else
2309 MIRBuilder.buildBuildVector(DstReg, DstRegs);
2310
2311 MI.eraseFromParent();
2312 return Legalized;
2313}
2314
2315LegalizerHelper::LegalizeResult
2316LegalizerHelper::fewerElementsVectorBasic(MachineInstr &MI, unsigned TypeIdx,
2317 LLT NarrowTy) {
2318 const unsigned Opc = MI.getOpcode();
2319 const unsigned NumOps = MI.getNumOperands() - 1;
2320 const unsigned NarrowSize = NarrowTy.getSizeInBits();
2321 const Register DstReg = MI.getOperand(0).getReg();
2322 const unsigned Flags = MI.getFlags();
2323 const LLT DstTy = MRI.getType(DstReg);
2324 const unsigned Size = DstTy.getSizeInBits();
2325 const int NumParts = Size / NarrowSize;
2326 const LLT EltTy = DstTy.getElementType();
2327 const unsigned EltSize = EltTy.getSizeInBits();
2328 const unsigned BitsForNumParts = NarrowSize * NumParts;
2329
2330 // Check if we have any leftovers. If we do, then only handle the case where
2331 // the leftover is one element.
2332 if (BitsForNumParts != Size && BitsForNumParts + EltSize != Size)
2333 return UnableToLegalize;
2334
2335 if (BitsForNumParts != Size) {
2336 Register AccumDstReg = MRI.createGenericVirtualRegister(DstTy);
2337 MIRBuilder.buildUndef(AccumDstReg);
2338
2339 // Handle the pieces which evenly divide into the requested type with
2340 // extract/op/insert sequence.
2341 for (unsigned Offset = 0; Offset < BitsForNumParts; Offset += NarrowSize) {
2342 SmallVector<SrcOp, 4> SrcOps;
2343 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
2344 Register PartOpReg = MRI.createGenericVirtualRegister(NarrowTy);
2345 MIRBuilder.buildExtract(PartOpReg, MI.getOperand(I).getReg(), Offset);
2346 SrcOps.push_back(PartOpReg);
2347 }
2348
2349 Register PartDstReg = MRI.createGenericVirtualRegister(NarrowTy);
2350 MIRBuilder.buildInstr(Opc, {PartDstReg}, SrcOps, Flags);
2351
2352 Register PartInsertReg = MRI.createGenericVirtualRegister(DstTy);
2353 MIRBuilder.buildInsert(PartInsertReg, AccumDstReg, PartDstReg, Offset);
2354 AccumDstReg = PartInsertReg;
2355 }
2356
2357 // Handle the remaining element sized leftover piece.
2358 SmallVector<SrcOp, 4> SrcOps;
2359 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
2360 Register PartOpReg = MRI.createGenericVirtualRegister(EltTy);
2361 MIRBuilder.buildExtract(PartOpReg, MI.getOperand(I).getReg(),
2362 BitsForNumParts);
2363 SrcOps.push_back(PartOpReg);
2364 }
2365
2366 Register PartDstReg = MRI.createGenericVirtualRegister(EltTy);
2367 MIRBuilder.buildInstr(Opc, {PartDstReg}, SrcOps, Flags);
2368 MIRBuilder.buildInsert(DstReg, AccumDstReg, PartDstReg, BitsForNumParts);
2369 MI.eraseFromParent();
2370
2371 return Legalized;
2372 }
2373
2374 SmallVector<Register, 2> DstRegs, Src0Regs, Src1Regs, Src2Regs;
2375
2376 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src0Regs);
2377
2378 if (NumOps >= 2)
2379 extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src1Regs);
2380
2381 if (NumOps >= 3)
2382 extractParts(MI.getOperand(3).getReg(), NarrowTy, NumParts, Src2Regs);
2383
2384 for (int i = 0; i < NumParts; ++i) {
2385 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy);
2386
2387 if (NumOps == 1)
2388 MIRBuilder.buildInstr(Opc, {DstReg}, {Src0Regs[i]}, Flags);
2389 else if (NumOps == 2) {
2390 MIRBuilder.buildInstr(Opc, {DstReg}, {Src0Regs[i], Src1Regs[i]}, Flags);
2391 } else if (NumOps == 3) {
2392 MIRBuilder.buildInstr(Opc, {DstReg},
2393 {Src0Regs[i], Src1Regs[i], Src2Regs[i]}, Flags);
2394 }
2395
2396 DstRegs.push_back(DstReg);
2397 }
2398
2399 if (NarrowTy.isVector())
2400 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
2401 else
2402 MIRBuilder.buildBuildVector(DstReg, DstRegs);
2403
2404 MI.eraseFromParent();
2405 return Legalized;
2406}
2407
2408// Handle splitting vector operations which need to have the same number of
2409// elements in each type index, but each type index may have a different element
2410// type.
2411//
2412// e.g. <4 x s64> = G_SHL <4 x s64>, <4 x s32> ->
2413// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
2414// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
2415//
2416// Also handles some irregular breakdown cases, e.g.
2417// e.g. <3 x s64> = G_SHL <3 x s64>, <3 x s32> ->
2418// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
2419// s64 = G_SHL s64, s32
2420LegalizerHelper::LegalizeResult
2421LegalizerHelper::fewerElementsVectorMultiEltType(
2422 MachineInstr &MI, unsigned TypeIdx, LLT NarrowTyArg) {
2423 if (TypeIdx != 0)
2424 return UnableToLegalize;
2425
2426 const LLT NarrowTy0 = NarrowTyArg;
2427 const unsigned NewNumElts =
2428 NarrowTy0.isVector() ? NarrowTy0.getNumElements() : 1;
2429
2430 const Register DstReg = MI.getOperand(0).getReg();
2431 LLT DstTy = MRI.getType(DstReg);
2432 LLT LeftoverTy0;
2433
2434 // All of the operands need to have the same number of elements, so if we can
2435 // determine a type breakdown for the result type, we can for all of the
2436 // source types.
2437 int NumParts = getNarrowTypeBreakDown(DstTy, NarrowTy0, LeftoverTy0).first;
2438 if (NumParts < 0)
2439 return UnableToLegalize;
2440
2441 SmallVector<MachineInstrBuilder, 4> NewInsts;
2442
2443 SmallVector<Register, 4> DstRegs, LeftoverDstRegs;
2444 SmallVector<Register, 4> PartRegs, LeftoverRegs;
2445
2446 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
2447 LLT LeftoverTy;
2448 Register SrcReg = MI.getOperand(I).getReg();
2449 LLT SrcTyI = MRI.getType(SrcReg);
2450 LLT NarrowTyI = LLT::scalarOrVector(NewNumElts, SrcTyI.getScalarType());
2451 LLT LeftoverTyI;
2452
2453 // Split this operand into the requested typed registers, and any leftover
2454 // required to reproduce the original type.
2455 if (!extractParts(SrcReg, SrcTyI, NarrowTyI, LeftoverTyI, PartRegs,
2456 LeftoverRegs))
2457 return UnableToLegalize;
2458
2459 if (I == 1) {
2460 // For the first operand, create an instruction for each part and setup
2461 // the result.
2462 for (Register PartReg : PartRegs) {
2463 Register PartDstReg = MRI.createGenericVirtualRegister(NarrowTy0);
2464 NewInsts.push_back(MIRBuilder.buildInstrNoInsert(MI.getOpcode())
2465 .addDef(PartDstReg)
2466 .addUse(PartReg));
2467 DstRegs.push_back(PartDstReg);
2468 }
2469
2470 for (Register LeftoverReg : LeftoverRegs) {
2471 Register PartDstReg = MRI.createGenericVirtualRegister(LeftoverTy0);
2472 NewInsts.push_back(MIRBuilder.buildInstrNoInsert(MI.getOpcode())
2473 .addDef(PartDstReg)
2474 .addUse(LeftoverReg));
2475 LeftoverDstRegs.push_back(PartDstReg);
2476 }
2477 } else {
2478 assert(NewInsts.size() == PartRegs.size() + LeftoverRegs.size())((NewInsts.size() == PartRegs.size() + LeftoverRegs.size()) ?
static_cast<void> (0) : __assert_fail ("NewInsts.size() == PartRegs.size() + LeftoverRegs.size()"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 2478, __PRETTY_FUNCTION__))
;
2479
2480 // Add the newly created operand splits to the existing instructions. The
2481 // odd-sized pieces are ordered after the requested NarrowTyArg sized
2482 // pieces.
2483 unsigned InstCount = 0;
2484 for (unsigned J = 0, JE = PartRegs.size(); J != JE; ++J)
2485 NewInsts[InstCount++].addUse(PartRegs[J]);
2486 for (unsigned J = 0, JE = LeftoverRegs.size(); J != JE; ++J)
2487 NewInsts[InstCount++].addUse(LeftoverRegs[J]);
2488 }
2489
2490 PartRegs.clear();
2491 LeftoverRegs.clear();
2492 }
2493
2494 // Insert the newly built operations and rebuild the result register.
2495 for (auto &MIB : NewInsts)
2496 MIRBuilder.insertInstr(MIB);
2497
2498 insertParts(DstReg, DstTy, NarrowTy0, DstRegs, LeftoverTy0, LeftoverDstRegs);
2499
2500 MI.eraseFromParent();
2501 return Legalized;
2502}
2503
2504LegalizerHelper::LegalizeResult
2505LegalizerHelper::fewerElementsVectorCasts(MachineInstr &MI, unsigned TypeIdx,
2506 LLT NarrowTy) {
2507 if (TypeIdx != 0)
2508 return UnableToLegalize;
2509
2510 Register DstReg = MI.getOperand(0).getReg();
2511 Register SrcReg = MI.getOperand(1).getReg();
2512 LLT DstTy = MRI.getType(DstReg);
2513 LLT SrcTy = MRI.getType(SrcReg);
2514
2515 LLT NarrowTy0 = NarrowTy;
2516 LLT NarrowTy1;
2517 unsigned NumParts;
2518
2519 if (NarrowTy.isVector()) {
2520 // Uneven breakdown not handled.
2521 NumParts = DstTy.getNumElements() / NarrowTy.getNumElements();
2522 if (NumParts * NarrowTy.getNumElements() != DstTy.getNumElements())
2523 return UnableToLegalize;
2524
2525 NarrowTy1 = LLT::vector(NumParts, SrcTy.getElementType().getSizeInBits());
2526 } else {
2527 NumParts = DstTy.getNumElements();
2528 NarrowTy1 = SrcTy.getElementType();
2529 }
2530
2531 SmallVector<Register, 4> SrcRegs, DstRegs;
2532 extractParts(SrcReg, NarrowTy1, NumParts, SrcRegs);
2533
2534 for (unsigned I = 0; I < NumParts; ++I) {
2535 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
2536 MachineInstr *NewInst = MIRBuilder.buildInstr(MI.getOpcode())
2537 .addDef(DstReg)
2538 .addUse(SrcRegs[I]);
2539
2540 NewInst->setFlags(MI.getFlags());
2541 DstRegs.push_back(DstReg);
2542 }
2543
2544 if (NarrowTy.isVector())
2545 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
2546 else
2547 MIRBuilder.buildBuildVector(DstReg, DstRegs);
2548
2549 MI.eraseFromParent();
2550 return Legalized;
2551}
2552
2553LegalizerHelper::LegalizeResult
2554LegalizerHelper::fewerElementsVectorCmp(MachineInstr &MI, unsigned TypeIdx,
2555 LLT NarrowTy) {
2556 Register DstReg = MI.getOperand(0).getReg();
2557 Register Src0Reg = MI.getOperand(2).getReg();
2558 LLT DstTy = MRI.getType(DstReg);
2559 LLT SrcTy = MRI.getType(Src0Reg);
2560
2561 unsigned NumParts;
2562 LLT NarrowTy0, NarrowTy1;
2563
2564 if (TypeIdx == 0) {
2565 unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
2566 unsigned OldElts = DstTy.getNumElements();
2567
2568 NarrowTy0 = NarrowTy;
2569 NumParts = NarrowTy.isVector() ? (OldElts / NewElts) : DstTy.getNumElements();
2570 NarrowTy1 = NarrowTy.isVector() ?
2571 LLT::vector(NarrowTy.getNumElements(), SrcTy.getScalarSizeInBits()) :
2572 SrcTy.getElementType();
2573
2574 } else {
2575 unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
2576 unsigned OldElts = SrcTy.getNumElements();
2577
2578 NumParts = NarrowTy.isVector() ? (OldElts / NewElts) :
2579 NarrowTy.getNumElements();
2580 NarrowTy0 = LLT::vector(NarrowTy.getNumElements(),
2581 DstTy.getScalarSizeInBits());
2582 NarrowTy1 = NarrowTy;
2583 }
2584
2585 // FIXME: Don't know how to handle the situation where the small vectors
2586 // aren't all the same size yet.
2587 if (NarrowTy1.isVector() &&
2588 NarrowTy1.getNumElements() * NumParts != DstTy.getNumElements())
2589 return UnableToLegalize;
2590
2591 CmpInst::Predicate Pred
2592 = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
2593
2594 SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs;
2595 extractParts(MI.getOperand(2).getReg(), NarrowTy1, NumParts, Src1Regs);
2596 extractParts(MI.getOperand(3).getReg(), NarrowTy1, NumParts, Src2Regs);
2597
2598 for (unsigned I = 0; I < NumParts; ++I) {
2599 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
2600 DstRegs.push_back(DstReg);
2601
2602 if (MI.getOpcode() == TargetOpcode::G_ICMP)
2603 MIRBuilder.buildICmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]);
2604 else {
2605 MachineInstr *NewCmp
2606 = MIRBuilder.buildFCmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]);
2607 NewCmp->setFlags(MI.getFlags());
2608 }
2609 }
2610
2611 if (NarrowTy1.isVector())
2612 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
2613 else
2614 MIRBuilder.buildBuildVector(DstReg, DstRegs);
2615
2616 MI.eraseFromParent();
2617 return Legalized;
2618}
2619
2620LegalizerHelper::LegalizeResult
2621LegalizerHelper::fewerElementsVectorSelect(MachineInstr &MI, unsigned TypeIdx,
2622 LLT NarrowTy) {
2623 Register DstReg = MI.getOperand(0).getReg();
2624 Register CondReg = MI.getOperand(1).getReg();
2625
2626 unsigned NumParts = 0;
2627 LLT NarrowTy0, NarrowTy1;
2628
2629 LLT DstTy = MRI.getType(DstReg);
2630 LLT CondTy = MRI.getType(CondReg);
2631 unsigned Size = DstTy.getSizeInBits();
2632
2633 assert(TypeIdx == 0 || CondTy.isVector())((TypeIdx == 0 || CondTy.isVector()) ? static_cast<void>
(0) : __assert_fail ("TypeIdx == 0 || CondTy.isVector()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 2633, __PRETTY_FUNCTION__))
;
2634
2635 if (TypeIdx == 0) {
2636 NarrowTy0 = NarrowTy;
2637 NarrowTy1 = CondTy;
2638
2639 unsigned NarrowSize = NarrowTy0.getSizeInBits();
2640 // FIXME: Don't know how to handle the situation where the small vectors
2641 // aren't all the same size yet.
2642 if (Size % NarrowSize != 0)
2643 return UnableToLegalize;
2644
2645 NumParts = Size / NarrowSize;
2646
2647 // Need to break down the condition type
2648 if (CondTy.isVector()) {
2649 if (CondTy.getNumElements() == NumParts)
2650 NarrowTy1 = CondTy.getElementType();
2651 else
2652 NarrowTy1 = LLT::vector(CondTy.getNumElements() / NumParts,
2653 CondTy.getScalarSizeInBits());
2654 }
2655 } else {
2656 NumParts = CondTy.getNumElements();
2657 if (NarrowTy.isVector()) {
2658 // TODO: Handle uneven breakdown.
2659 if (NumParts * NarrowTy.getNumElements() != CondTy.getNumElements())
2660 return UnableToLegalize;
2661
2662 return UnableToLegalize;
2663 } else {
2664 NarrowTy0 = DstTy.getElementType();
2665 NarrowTy1 = NarrowTy;
2666 }
2667 }
2668
2669 SmallVector<Register, 2> DstRegs, Src0Regs, Src1Regs, Src2Regs;
2670 if (CondTy.isVector())
2671 extractParts(MI.getOperand(1).getReg(), NarrowTy1, NumParts, Src0Regs);
2672
2673 extractParts(MI.getOperand(2).getReg(), NarrowTy0, NumParts, Src1Regs);
2674 extractParts(MI.getOperand(3).getReg(), NarrowTy0, NumParts, Src2Regs);
2675
2676 for (unsigned i = 0; i < NumParts; ++i) {
2677 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
2678 MIRBuilder.buildSelect(DstReg, CondTy.isVector() ? Src0Regs[i] : CondReg,
2679 Src1Regs[i], Src2Regs[i]);
2680 DstRegs.push_back(DstReg);
2681 }
2682
2683 if (NarrowTy0.isVector())
2684 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
2685 else
2686 MIRBuilder.buildBuildVector(DstReg, DstRegs);
2687
2688 MI.eraseFromParent();
2689 return Legalized;
2690}
2691
2692LegalizerHelper::LegalizeResult
2693LegalizerHelper::fewerElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx,
2694 LLT NarrowTy) {
2695 const Register DstReg = MI.getOperand(0).getReg();
2696 LLT PhiTy = MRI.getType(DstReg);
2697 LLT LeftoverTy;
2698
2699 // All of the operands need to have the same number of elements, so if we can
2700 // determine a type breakdown for the result type, we can for all of the
2701 // source types.
2702 int NumParts, NumLeftover;
2703 std::tie(NumParts, NumLeftover)
2704 = getNarrowTypeBreakDown(PhiTy, NarrowTy, LeftoverTy);
2705 if (NumParts < 0)
2706 return UnableToLegalize;
2707
2708 SmallVector<Register, 4> DstRegs, LeftoverDstRegs;
2709 SmallVector<MachineInstrBuilder, 4> NewInsts;
2710
2711 const int TotalNumParts = NumParts + NumLeftover;
2712
2713 // Insert the new phis in the result block first.
2714 for (int I = 0; I != TotalNumParts; ++I) {
2715 LLT Ty = I < NumParts ? NarrowTy : LeftoverTy;
2716 Register PartDstReg = MRI.createGenericVirtualRegister(Ty);
2717 NewInsts.push_back(MIRBuilder.buildInstr(TargetOpcode::G_PHI)
2718 .addDef(PartDstReg));
2719 if (I < NumParts)
2720 DstRegs.push_back(PartDstReg);
2721 else
2722 LeftoverDstRegs.push_back(PartDstReg);
2723 }
2724
2725 MachineBasicBlock *MBB = MI.getParent();
2726 MIRBuilder.setInsertPt(*MBB, MBB->getFirstNonPHI());
2727 insertParts(DstReg, PhiTy, NarrowTy, DstRegs, LeftoverTy, LeftoverDstRegs);
2728
2729 SmallVector<Register, 4> PartRegs, LeftoverRegs;
2730
2731 // Insert code to extract the incoming values in each predecessor block.
2732 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
2733 PartRegs.clear();
2734 LeftoverRegs.clear();
2735
2736 Register SrcReg = MI.getOperand(I).getReg();
2737 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
2738 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
2739
2740 LLT Unused;
2741 if (!extractParts(SrcReg, PhiTy, NarrowTy, Unused, PartRegs,
2742 LeftoverRegs))
2743 return UnableToLegalize;
2744
2745 // Add the newly created operand splits to the existing instructions. The
2746 // odd-sized pieces are ordered after the requested NarrowTyArg sized
2747 // pieces.
2748 for (int J = 0; J != TotalNumParts; ++J) {
2749 MachineInstrBuilder MIB = NewInsts[J];
2750 MIB.addUse(J < NumParts ? PartRegs[J] : LeftoverRegs[J - NumParts]);
2751 MIB.addMBB(&OpMBB);
2752 }
2753 }
2754
2755 MI.eraseFromParent();
2756 return Legalized;
2757}
2758
2759LegalizerHelper::LegalizeResult
2760LegalizerHelper::fewerElementsVectorUnmergeValues(MachineInstr &MI,
2761 unsigned TypeIdx,
2762 LLT NarrowTy) {
2763 if (TypeIdx != 1)
2764 return UnableToLegalize;
2765
2766 const int NumDst = MI.getNumOperands() - 1;
2767 const Register SrcReg = MI.getOperand(NumDst).getReg();
2768 LLT SrcTy = MRI.getType(SrcReg);
2769
2770 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
2771
2772 // TODO: Create sequence of extracts.
2773 if (DstTy == NarrowTy)
2774 return UnableToLegalize;
2775
2776 LLT GCDTy = getGCDType(SrcTy, NarrowTy);
2777 if (DstTy == GCDTy) {
2778 // This would just be a copy of the same unmerge.
2779 // TODO: Create extracts, pad with undef and create intermediate merges.
2780 return UnableToLegalize;
2781 }
2782
2783 auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg);
2784 const int NumUnmerge = Unmerge->getNumOperands() - 1;
2785 const int PartsPerUnmerge = NumDst / NumUnmerge;
2786
2787 for (int I = 0; I != NumUnmerge; ++I) {
2788 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES);
2789
2790 for (int J = 0; J != PartsPerUnmerge; ++J)
2791 MIB.addDef(MI.getOperand(I * PartsPerUnmerge + J).getReg());
2792 MIB.addUse(Unmerge.getReg(I));
2793 }
2794
2795 MI.eraseFromParent();
2796 return Legalized;
2797}
2798
2799LegalizerHelper::LegalizeResult
2800LegalizerHelper::fewerElementsVectorBuildVector(MachineInstr &MI,
2801 unsigned TypeIdx,
2802 LLT NarrowTy) {
2803 assert(TypeIdx == 0 && "not a vector type index")((TypeIdx == 0 && "not a vector type index") ? static_cast
<void> (0) : __assert_fail ("TypeIdx == 0 && \"not a vector type index\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 2803, __PRETTY_FUNCTION__))
;
2804 Register DstReg = MI.getOperand(0).getReg();
2805 LLT DstTy = MRI.getType(DstReg);
2806 LLT SrcTy = DstTy.getElementType();
2807
2808 int DstNumElts = DstTy.getNumElements();
2809 int NarrowNumElts = NarrowTy.getNumElements();
2810 int NumConcat = (DstNumElts + NarrowNumElts - 1) / NarrowNumElts;
2811 LLT WidenedDstTy = LLT::vector(NarrowNumElts * NumConcat, SrcTy);
2812
2813 SmallVector<Register, 8> ConcatOps;
2814 SmallVector<Register, 8> SubBuildVector;
2815
2816 Register UndefReg;
2817 if (WidenedDstTy != DstTy)
2818 UndefReg = MIRBuilder.buildUndef(SrcTy).getReg(0);
2819
2820 // Create a G_CONCAT_VECTORS of NarrowTy pieces, padding with undef as
2821 // necessary.
2822 //
2823 // %3:_(<3 x s16>) = G_BUILD_VECTOR %0, %1, %2
2824 // -> <2 x s16>
2825 //
2826 // %4:_(s16) = G_IMPLICIT_DEF
2827 // %5:_(<2 x s16>) = G_BUILD_VECTOR %0, %1
2828 // %6:_(<2 x s16>) = G_BUILD_VECTOR %2, %4
2829 // %7:_(<4 x s16>) = G_CONCAT_VECTORS %5, %6
2830 // %3:_(<3 x s16>) = G_EXTRACT %7, 0
2831 for (int I = 0; I != NumConcat; ++I) {
2832 for (int J = 0; J != NarrowNumElts; ++J) {
2833 int SrcIdx = NarrowNumElts * I + J;
2834
2835 if (SrcIdx < DstNumElts) {
2836 Register SrcReg = MI.getOperand(SrcIdx + 1).getReg();
2837 SubBuildVector.push_back(SrcReg);
2838 } else
2839 SubBuildVector.push_back(UndefReg);
2840 }
2841
2842 auto BuildVec = MIRBuilder.buildBuildVector(NarrowTy, SubBuildVector);
2843 ConcatOps.push_back(BuildVec.getReg(0));
2844 SubBuildVector.clear();
2845 }
2846
2847 if (DstTy == WidenedDstTy)
2848 MIRBuilder.buildConcatVectors(DstReg, ConcatOps);
2849 else {
2850 auto Concat = MIRBuilder.buildConcatVectors(WidenedDstTy, ConcatOps);
2851 MIRBuilder.buildExtract(DstReg, Concat, 0);
2852 }
2853
2854 MI.eraseFromParent();
2855 return Legalized;
2856}
2857
2858LegalizerHelper::LegalizeResult
2859LegalizerHelper::reduceLoadStoreWidth(MachineInstr &MI, unsigned TypeIdx,
2860 LLT NarrowTy) {
2861 // FIXME: Don't know how to handle secondary types yet.
2862 if (TypeIdx != 0)
2863 return UnableToLegalize;
2864
2865 MachineMemOperand *MMO = *MI.memoperands_begin();
2866
2867 // This implementation doesn't work for atomics. Give up instead of doing
2868 // something invalid.
2869 if (MMO->getOrdering() != AtomicOrdering::NotAtomic ||
2870 MMO->getFailureOrdering() != AtomicOrdering::NotAtomic)
2871 return UnableToLegalize;
2872
2873 bool IsLoad = MI.getOpcode() == TargetOpcode::G_LOAD;
2874 Register ValReg = MI.getOperand(0).getReg();
2875 Register AddrReg = MI.getOperand(1).getReg();
2876 LLT ValTy = MRI.getType(ValReg);
2877
2878 int NumParts = -1;
2879 int NumLeftover = -1;
2880 LLT LeftoverTy;
2881 SmallVector<Register, 8> NarrowRegs, NarrowLeftoverRegs;
2882 if (IsLoad) {
2883 std::tie(NumParts, NumLeftover) = getNarrowTypeBreakDown(ValTy, NarrowTy, LeftoverTy);
2884 } else {
2885 if (extractParts(ValReg, ValTy, NarrowTy, LeftoverTy, NarrowRegs,
2886 NarrowLeftoverRegs)) {
2887 NumParts = NarrowRegs.size();
2888 NumLeftover = NarrowLeftoverRegs.size();
Value stored to 'NumLeftover' is never read
2889 }
2890 }
2891
2892 if (NumParts == -1)
2893 return UnableToLegalize;
2894
2895 const LLT OffsetTy = LLT::scalar(MRI.getType(AddrReg).getScalarSizeInBits());
2896
2897 unsigned TotalSize = ValTy.getSizeInBits();
2898
2899 // Split the load/store into PartTy sized pieces starting at Offset. If this
2900 // is a load, return the new registers in ValRegs. For a store, each elements
2901 // of ValRegs should be PartTy. Returns the next offset that needs to be
2902 // handled.
2903 auto splitTypePieces = [=](LLT PartTy, SmallVectorImpl<Register> &ValRegs,
2904 unsigned Offset) -> unsigned {
2905 MachineFunction &MF = MIRBuilder.getMF();
2906 unsigned PartSize = PartTy.getSizeInBits();
2907 for (unsigned Idx = 0, E = NumParts; Idx != E && Offset < TotalSize;
2908 Offset += PartSize, ++Idx) {
2909 unsigned ByteSize = PartSize / 8;
2910 unsigned ByteOffset = Offset / 8;
2911 Register NewAddrReg;
2912
2913 MIRBuilder.materializePtrAdd(NewAddrReg, AddrReg, OffsetTy, ByteOffset);
2914
2915 MachineMemOperand *NewMMO =
2916 MF.getMachineMemOperand(MMO, ByteOffset, ByteSize);
2917
2918 if (IsLoad) {
2919 Register Dst = MRI.createGenericVirtualRegister(PartTy);
2920 ValRegs.push_back(Dst);
2921 MIRBuilder.buildLoad(Dst, NewAddrReg, *NewMMO);
2922 } else {
2923 MIRBuilder.buildStore(ValRegs[Idx], NewAddrReg, *NewMMO);
2924 }
2925 }
2926
2927 return Offset;
2928 };
2929
2930 unsigned HandledOffset = splitTypePieces(NarrowTy, NarrowRegs, 0);
2931
2932 // Handle the rest of the register if this isn't an even type breakdown.
2933 if (LeftoverTy.isValid())
2934 splitTypePieces(LeftoverTy, NarrowLeftoverRegs, HandledOffset);
2935
2936 if (IsLoad) {
2937 insertParts(ValReg, ValTy, NarrowTy, NarrowRegs,
2938 LeftoverTy, NarrowLeftoverRegs);
2939 }
2940
2941 MI.eraseFromParent();
2942 return Legalized;
2943}
2944
2945LegalizerHelper::LegalizeResult
2946LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
2947 LLT NarrowTy) {
2948 using namespace TargetOpcode;
2949
2950 MIRBuilder.setInstr(MI);
2951 switch (MI.getOpcode()) {
2952 case G_IMPLICIT_DEF:
2953 return fewerElementsVectorImplicitDef(MI, TypeIdx, NarrowTy);
2954 case G_AND:
2955 case G_OR:
2956 case G_XOR:
2957 case G_ADD:
2958 case G_SUB:
2959 case G_MUL:
2960 case G_SMULH:
2961 case G_UMULH:
2962 case G_FADD:
2963 case G_FMUL:
2964 case G_FSUB:
2965 case G_FNEG:
2966 case G_FABS:
2967 case G_FCANONICALIZE:
2968 case G_FDIV:
2969 case G_FREM:
2970 case G_FMA:
2971 case G_FMAD:
2972 case G_FPOW:
2973 case G_FEXP:
2974 case G_FEXP2:
2975 case G_FLOG:
2976 case G_FLOG2:
2977 case G_FLOG10:
2978 case G_FNEARBYINT:
2979 case G_FCEIL:
2980 case G_FFLOOR:
2981 case G_FRINT:
2982 case G_INTRINSIC_ROUND:
2983 case G_INTRINSIC_TRUNC:
2984 case G_FCOS:
2985 case G_FSIN:
2986 case G_FSQRT:
2987 case G_BSWAP:
2988 case G_BITREVERSE:
2989 case G_SDIV:
2990 case G_SMIN:
2991 case G_SMAX:
2992 case G_UMIN:
2993 case G_UMAX:
2994 case G_FMINNUM:
2995 case G_FMAXNUM:
2996 case G_FMINNUM_IEEE:
2997 case G_FMAXNUM_IEEE:
2998 case G_FMINIMUM:
2999 case G_FMAXIMUM:
3000 return fewerElementsVectorBasic(MI, TypeIdx, NarrowTy);
3001 case G_SHL:
3002 case G_LSHR:
3003 case G_ASHR:
3004 case G_CTLZ:
3005 case G_CTLZ_ZERO_UNDEF:
3006 case G_CTTZ:
3007 case G_CTTZ_ZERO_UNDEF:
3008 case G_CTPOP:
3009 case G_FCOPYSIGN:
3010 return fewerElementsVectorMultiEltType(MI, TypeIdx, NarrowTy);
3011 case G_ZEXT:
3012 case G_SEXT:
3013 case G_ANYEXT:
3014 case G_FPEXT:
3015 case G_FPTRUNC:
3016 case G_SITOFP:
3017 case G_UITOFP:
3018 case G_FPTOSI:
3019 case G_FPTOUI:
3020 case G_INTTOPTR:
3021 case G_PTRTOINT:
3022 case G_ADDRSPACE_CAST:
3023 return fewerElementsVectorCasts(MI, TypeIdx, NarrowTy);
3024 case G_ICMP:
3025 case G_FCMP:
3026 return fewerElementsVectorCmp(MI, TypeIdx, NarrowTy);
3027 case G_SELECT:
3028 return fewerElementsVectorSelect(MI, TypeIdx, NarrowTy);
3029 case G_PHI:
3030 return fewerElementsVectorPhi(MI, TypeIdx, NarrowTy);
3031 case G_UNMERGE_VALUES:
3032 return fewerElementsVectorUnmergeValues(MI, TypeIdx, NarrowTy);
3033 case G_BUILD_VECTOR:
3034 return fewerElementsVectorBuildVector(MI, TypeIdx, NarrowTy);
3035 case G_LOAD:
3036 case G_STORE:
3037 return reduceLoadStoreWidth(MI, TypeIdx, NarrowTy);
3038 default:
3039 return UnableToLegalize;
3040 }
3041}
3042
3043LegalizerHelper::LegalizeResult
3044LegalizerHelper::narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt,
3045 const LLT HalfTy, const LLT AmtTy) {
3046
3047 Register InL = MRI.createGenericVirtualRegister(HalfTy);
3048 Register InH = MRI.createGenericVirtualRegister(HalfTy);
3049 MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1).getReg());
3050
3051 if (Amt.isNullValue()) {
3052 MIRBuilder.buildMerge(MI.getOperand(0).getReg(), {InL, InH});
3053 MI.eraseFromParent();
3054 return Legalized;
3055 }
3056
3057 LLT NVT = HalfTy;
3058 unsigned NVTBits = HalfTy.getSizeInBits();
3059 unsigned VTBits = 2 * NVTBits;
3060
3061 SrcOp Lo(Register(0)), Hi(Register(0));
3062 if (MI.getOpcode() == TargetOpcode::G_SHL) {
3063 if (Amt.ugt(VTBits)) {
3064 Lo = Hi = MIRBuilder.buildConstant(NVT, 0);
3065 } else if (Amt.ugt(NVTBits)) {
3066 Lo = MIRBuilder.buildConstant(NVT, 0);
3067 Hi = MIRBuilder.buildShl(NVT, InL,
3068 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
3069 } else if (Amt == NVTBits) {
3070 Lo = MIRBuilder.buildConstant(NVT, 0);
3071 Hi = InL;
3072 } else {
3073 Lo = MIRBuilder.buildShl(NVT, InL, MIRBuilder.buildConstant(AmtTy, Amt));
3074 auto OrLHS =
3075 MIRBuilder.buildShl(NVT, InH, MIRBuilder.buildConstant(AmtTy, Amt));
3076 auto OrRHS = MIRBuilder.buildLShr(
3077 NVT, InL, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
3078 Hi = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
3079 }
3080 } else if (MI.getOpcode() == TargetOpcode::G_LSHR) {
3081 if (Amt.ugt(VTBits)) {
3082 Lo = Hi = MIRBuilder.buildConstant(NVT, 0);
3083 } else if (Amt.ugt(NVTBits)) {
3084 Lo = MIRBuilder.buildLShr(NVT, InH,
3085 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
3086 Hi = MIRBuilder.buildConstant(NVT, 0);
3087 } else if (Amt == NVTBits) {
3088 Lo = InH;
3089 Hi = MIRBuilder.buildConstant(NVT, 0);
3090 } else {
3091 auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt);
3092
3093 auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst);
3094 auto OrRHS = MIRBuilder.buildShl(
3095 NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
3096
3097 Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
3098 Hi = MIRBuilder.buildLShr(NVT, InH, ShiftAmtConst);
3099 }
3100 } else {
3101 if (Amt.ugt(VTBits)) {
3102 Hi = Lo = MIRBuilder.buildAShr(
3103 NVT, InH, MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
3104 } else if (Amt.ugt(NVTBits)) {
3105 Lo = MIRBuilder.buildAShr(NVT, InH,
3106 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
3107 Hi = MIRBuilder.buildAShr(NVT, InH,
3108 MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
3109 } else if (Amt == NVTBits) {
3110 Lo = InH;
3111 Hi = MIRBuilder.buildAShr(NVT, InH,
3112 MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
3113 } else {
3114 auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt);
3115
3116 auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst);
3117 auto OrRHS = MIRBuilder.buildShl(
3118 NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
3119
3120 Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
3121 Hi = MIRBuilder.buildAShr(NVT, InH, ShiftAmtConst);
3122 }
3123 }
3124
3125 MIRBuilder.buildMerge(MI.getOperand(0).getReg(), {Lo.getReg(), Hi.getReg()});
3126 MI.eraseFromParent();
3127
3128 return Legalized;
3129}
3130
3131// TODO: Optimize if constant shift amount.
3132LegalizerHelper::LegalizeResult
3133LegalizerHelper::narrowScalarShift(MachineInstr &MI, unsigned TypeIdx,
3134 LLT RequestedTy) {
3135 if (TypeIdx == 1) {
3136 Observer.changingInstr(MI);
3137 narrowScalarSrc(MI, RequestedTy, 2);
3138 Observer.changedInstr(MI);
3139 return Legalized;
3140 }
3141
3142 Register DstReg = MI.getOperand(0).getReg();
3143 LLT DstTy = MRI.getType(DstReg);
3144 if (DstTy.isVector())
3145 return UnableToLegalize;
3146
3147 Register Amt = MI.getOperand(2).getReg();
3148 LLT ShiftAmtTy = MRI.getType(Amt);
3149 const unsigned DstEltSize = DstTy.getScalarSizeInBits();
3150 if (DstEltSize % 2 != 0)
3151 return UnableToLegalize;
3152
3153 // Ignore the input type. We can only go to exactly half the size of the
3154 // input. If that isn't small enough, the resulting pieces will be further
3155 // legalized.
3156 const unsigned NewBitSize = DstEltSize / 2;
3157 const LLT HalfTy = LLT::scalar(NewBitSize);
3158 const LLT CondTy = LLT::scalar(1);
3159
3160 if (const MachineInstr *KShiftAmt =
3161 getOpcodeDef(TargetOpcode::G_CONSTANT, Amt, MRI)) {
3162 return narrowScalarShiftByConstant(
3163 MI, KShiftAmt->getOperand(1).getCImm()->getValue(), HalfTy, ShiftAmtTy);
3164 }
3165
3166 // TODO: Expand with known bits.
3167
3168 // Handle the fully general expansion by an unknown amount.
3169 auto NewBits = MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize);
3170
3171 Register InL = MRI.createGenericVirtualRegister(HalfTy);
3172 Register InH = MRI.createGenericVirtualRegister(HalfTy);
3173 MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1).getReg());
3174
3175 auto AmtExcess = MIRBuilder.buildSub(ShiftAmtTy, Amt, NewBits);
3176 auto AmtLack = MIRBuilder.buildSub(ShiftAmtTy, NewBits, Amt);
3177
3178 auto Zero = MIRBuilder.buildConstant(ShiftAmtTy, 0);
3179 auto IsShort = MIRBuilder.buildICmp(ICmpInst::ICMP_ULT, CondTy, Amt, NewBits);
3180 auto IsZero = MIRBuilder.buildICmp(ICmpInst::ICMP_EQ, CondTy, Amt, Zero);
3181
3182 Register ResultRegs[2];
3183 switch (MI.getOpcode()) {
3184 case TargetOpcode::G_SHL: {
3185 // Short: ShAmt < NewBitSize
3186 auto LoS = MIRBuilder.buildShl(HalfTy, InL, Amt);
3187
3188 auto LoOr = MIRBuilder.buildLShr(HalfTy, InL, AmtLack);
3189 auto HiOr = MIRBuilder.buildShl(HalfTy, InH, Amt);
3190 auto HiS = MIRBuilder.buildOr(HalfTy, LoOr, HiOr);
3191
3192 // Long: ShAmt >= NewBitSize
3193 auto LoL = MIRBuilder.buildConstant(HalfTy, 0); // Lo part is zero.
3194 auto HiL = MIRBuilder.buildShl(HalfTy, InL, AmtExcess); // Hi from Lo part.
3195
3196 auto Lo = MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL);
3197 auto Hi = MIRBuilder.buildSelect(
3198 HalfTy, IsZero, InH, MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL));
3199
3200 ResultRegs[0] = Lo.getReg(0);
3201 ResultRegs[1] = Hi.getReg(0);
3202 break;
3203 }
3204 case TargetOpcode::G_LSHR:
3205 case TargetOpcode::G_ASHR: {
3206 // Short: ShAmt < NewBitSize
3207 auto HiS = MIRBuilder.buildInstr(MI.getOpcode(), {HalfTy}, {InH, Amt});
3208
3209 auto LoOr = MIRBuilder.buildLShr(HalfTy, InL, Amt);
3210 auto HiOr = MIRBuilder.buildShl(HalfTy, InH, AmtLack);
3211 auto LoS = MIRBuilder.buildOr(HalfTy, LoOr, HiOr);
3212
3213 // Long: ShAmt >= NewBitSize
3214 MachineInstrBuilder HiL;
3215 if (MI.getOpcode() == TargetOpcode::G_LSHR) {
3216 HiL = MIRBuilder.buildConstant(HalfTy, 0); // Hi part is zero.
3217 } else {
3218 auto ShiftAmt = MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize - 1);
3219 HiL = MIRBuilder.buildAShr(HalfTy, InH, ShiftAmt); // Sign of Hi part.
3220 }
3221 auto LoL = MIRBuilder.buildInstr(MI.getOpcode(), {HalfTy},
3222 {InH, AmtExcess}); // Lo from Hi part.
3223
3224 auto Lo = MIRBuilder.buildSelect(
3225 HalfTy, IsZero, InL, MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL));
3226
3227 auto Hi = MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL);
3228
3229 ResultRegs[0] = Lo.getReg(0);
3230 ResultRegs[1] = Hi.getReg(0);
3231 break;
3232 }
3233 default:
3234 llvm_unreachable("not a shift")::llvm::llvm_unreachable_internal("not a shift", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 3234)
;
3235 }
3236
3237 MIRBuilder.buildMerge(DstReg, ResultRegs);
3238 MI.eraseFromParent();
3239 return Legalized;
3240}
3241
3242LegalizerHelper::LegalizeResult
3243LegalizerHelper::moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx,
3244 LLT MoreTy) {
3245 assert(TypeIdx == 0 && "Expecting only Idx 0")((TypeIdx == 0 && "Expecting only Idx 0") ? static_cast
<void> (0) : __assert_fail ("TypeIdx == 0 && \"Expecting only Idx 0\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 3245, __PRETTY_FUNCTION__))
;
3246
3247 Observer.changingInstr(MI);
3248 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
3249 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
3250 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
3251 moreElementsVectorSrc(MI, MoreTy, I);
3252 }
3253
3254 MachineBasicBlock &MBB = *MI.getParent();
3255 MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI());
3256 moreElementsVectorDst(MI, MoreTy, 0);
3257 Observer.changedInstr(MI);
3258 return Legalized;
3259}
3260
3261LegalizerHelper::LegalizeResult
3262LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
3263 LLT MoreTy) {
3264 MIRBuilder.setInstr(MI);
3265 unsigned Opc = MI.getOpcode();
3266 switch (Opc) {
3267 case TargetOpcode::G_IMPLICIT_DEF:
3268 case TargetOpcode::G_LOAD: {
3269 if (TypeIdx != 0)
3270 return UnableToLegalize;
3271 Observer.changingInstr(MI);
3272 moreElementsVectorDst(MI, MoreTy, 0);
3273 Observer.changedInstr(MI);
3274 return Legalized;
3275 }
3276 case TargetOpcode::G_STORE:
3277 if (TypeIdx != 0)
3278 return UnableToLegalize;
3279 Observer.changingInstr(MI);
3280 moreElementsVectorSrc(MI, MoreTy, 0);
3281 Observer.changedInstr(MI);
3282 return Legalized;
3283 case TargetOpcode::G_AND:
3284 case TargetOpcode::G_OR:
3285 case TargetOpcode::G_XOR:
3286 case TargetOpcode::G_SMIN:
3287 case TargetOpcode::G_SMAX:
3288 case TargetOpcode::G_UMIN:
3289 case TargetOpcode::G_UMAX: {
3290 Observer.changingInstr(MI);
3291 moreElementsVectorSrc(MI, MoreTy, 1);
3292 moreElementsVectorSrc(MI, MoreTy, 2);
3293 moreElementsVectorDst(MI, MoreTy, 0);
3294 Observer.changedInstr(MI);
3295 return Legalized;
3296 }
3297 case TargetOpcode::G_EXTRACT:
3298 if (TypeIdx != 1)
3299 return UnableToLegalize;
3300 Observer.changingInstr(MI);
3301 moreElementsVectorSrc(MI, MoreTy, 1);
3302 Observer.changedInstr(MI);
3303 return Legalized;
3304 case TargetOpcode::G_INSERT:
3305 if (TypeIdx != 0)
3306 return UnableToLegalize;
3307 Observer.changingInstr(MI);
3308 moreElementsVectorSrc(MI, MoreTy, 1);
3309 moreElementsVectorDst(MI, MoreTy, 0);
3310 Observer.changedInstr(MI);
3311 return Legalized;
3312 case TargetOpcode::G_SELECT:
3313 if (TypeIdx != 0)
3314 return UnableToLegalize;
3315 if (MRI.getType(MI.getOperand(1).getReg()).isVector())
3316 return UnableToLegalize;
3317
3318 Observer.changingInstr(MI);
3319 moreElementsVectorSrc(MI, MoreTy, 2);
3320 moreElementsVectorSrc(MI, MoreTy, 3);
3321 moreElementsVectorDst(MI, MoreTy, 0);
3322 Observer.changedInstr(MI);
3323 return Legalized;
3324 case TargetOpcode::G_UNMERGE_VALUES: {
3325 if (TypeIdx != 1)
3326 return UnableToLegalize;
3327
3328 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
3329 int NumDst = MI.getNumOperands() - 1;
3330 moreElementsVectorSrc(MI, MoreTy, NumDst);
3331
3332 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES);
3333 for (int I = 0; I != NumDst; ++I)
3334 MIB.addDef(MI.getOperand(I).getReg());
3335
3336 int NewNumDst = MoreTy.getSizeInBits() / DstTy.getSizeInBits();
3337 for (int I = NumDst; I != NewNumDst; ++I)
3338 MIB.addDef(MRI.createGenericVirtualRegister(DstTy));
3339
3340 MIB.addUse(MI.getOperand(NumDst).getReg());
3341 MI.eraseFromParent();
3342 return Legalized;
3343 }
3344 case TargetOpcode::G_PHI:
3345 return moreElementsVectorPhi(MI, TypeIdx, MoreTy);
3346 default:
3347 return UnableToLegalize;
3348 }
3349}
3350
3351void LegalizerHelper::multiplyRegisters(SmallVectorImpl<Register> &DstRegs,
3352 ArrayRef<Register> Src1Regs,
3353 ArrayRef<Register> Src2Regs,
3354 LLT NarrowTy) {
3355 MachineIRBuilder &B = MIRBuilder;
3356 unsigned SrcParts = Src1Regs.size();
3357 unsigned DstParts = DstRegs.size();
3358
3359 unsigned DstIdx = 0; // Low bits of the result.
3360 Register FactorSum =
3361 B.buildMul(NarrowTy, Src1Regs[DstIdx], Src2Regs[DstIdx]).getReg(0);
3362 DstRegs[DstIdx] = FactorSum;
3363
3364 unsigned CarrySumPrevDstIdx;
3365 SmallVector<Register, 4> Factors;
3366
3367 for (DstIdx = 1; DstIdx < DstParts; DstIdx++) {
3368 // Collect low parts of muls for DstIdx.
3369 for (unsigned i = DstIdx + 1 < SrcParts ? 0 : DstIdx - SrcParts + 1;
3370 i <= std::min(DstIdx, SrcParts - 1); ++i) {
3371 MachineInstrBuilder Mul =
3372 B.buildMul(NarrowTy, Src1Regs[DstIdx - i], Src2Regs[i]);
3373 Factors.push_back(Mul.getReg(0));
3374 }
3375 // Collect high parts of muls from previous DstIdx.
3376 for (unsigned i = DstIdx < SrcParts ? 0 : DstIdx - SrcParts;
3377 i <= std::min(DstIdx - 1, SrcParts - 1); ++i) {
3378 MachineInstrBuilder Umulh =
3379 B.buildUMulH(NarrowTy, Src1Regs[DstIdx - 1 - i], Src2Regs[i]);
3380 Factors.push_back(Umulh.getReg(0));
3381 }
3382 // Add CarrySum from additions calculated for previous DstIdx.
3383 if (DstIdx != 1) {
3384 Factors.push_back(CarrySumPrevDstIdx);
3385 }
3386
3387 Register CarrySum;
3388 // Add all factors and accumulate all carries into CarrySum.
3389 if (DstIdx != DstParts - 1) {
3390 MachineInstrBuilder Uaddo =
3391 B.buildUAddo(NarrowTy, LLT::scalar(1), Factors[0], Factors[1]);
3392 FactorSum = Uaddo.getReg(0);
3393 CarrySum = B.buildZExt(NarrowTy, Uaddo.getReg(1)).getReg(0);
3394 for (unsigned i = 2; i < Factors.size(); ++i) {
3395 MachineInstrBuilder Uaddo =
3396 B.buildUAddo(NarrowTy, LLT::scalar(1), FactorSum, Factors[i]);
3397 FactorSum = Uaddo.getReg(0);
3398 MachineInstrBuilder Carry = B.buildZExt(NarrowTy, Uaddo.getReg(1));
3399 CarrySum = B.buildAdd(NarrowTy, CarrySum, Carry).getReg(0);
3400 }
3401 } else {
3402 // Since value for the next index is not calculated, neither is CarrySum.
3403 FactorSum = B.buildAdd(NarrowTy, Factors[0], Factors[1]).getReg(0);
3404 for (unsigned i = 2; i < Factors.size(); ++i)
3405 FactorSum = B.buildAdd(NarrowTy, FactorSum, Factors[i]).getReg(0);
3406 }
3407
3408 CarrySumPrevDstIdx = CarrySum;
3409 DstRegs[DstIdx] = FactorSum;
3410 Factors.clear();
3411 }
3412}
3413
3414LegalizerHelper::LegalizeResult
3415LegalizerHelper::narrowScalarMul(MachineInstr &MI, LLT NarrowTy) {
3416 Register DstReg = MI.getOperand(0).getReg();
3417 Register Src1 = MI.getOperand(1).getReg();
3418 Register Src2 = MI.getOperand(2).getReg();
3419
3420 LLT Ty = MRI.getType(DstReg);
3421 if (Ty.isVector())
3422 return UnableToLegalize;
3423
3424 unsigned SrcSize = MRI.getType(Src1).getSizeInBits();
3425 unsigned DstSize = Ty.getSizeInBits();
3426 unsigned NarrowSize = NarrowTy.getSizeInBits();
3427 if (DstSize % NarrowSize != 0 || SrcSize % NarrowSize != 0)
3428 return UnableToLegalize;
3429
3430 unsigned NumDstParts = DstSize / NarrowSize;
3431 unsigned NumSrcParts = SrcSize / NarrowSize;
3432 bool IsMulHigh = MI.getOpcode() == TargetOpcode::G_UMULH;
3433 unsigned DstTmpParts = NumDstParts * (IsMulHigh ? 2 : 1);
3434
3435 SmallVector<Register, 2> Src1Parts, Src2Parts, DstTmpRegs;
3436 extractParts(Src1, NarrowTy, NumSrcParts, Src1Parts);
3437 extractParts(Src2, NarrowTy, NumSrcParts, Src2Parts);
3438 DstTmpRegs.resize(DstTmpParts);
3439 multiplyRegisters(DstTmpRegs, Src1Parts, Src2Parts, NarrowTy);
3440
3441 // Take only high half of registers if this is high mul.
3442 ArrayRef<Register> DstRegs(
3443 IsMulHigh ? &DstTmpRegs[DstTmpParts / 2] : &DstTmpRegs[0], NumDstParts);
3444 MIRBuilder.buildMerge(DstReg, DstRegs);
3445 MI.eraseFromParent();
3446 return Legalized;
3447}
3448
3449LegalizerHelper::LegalizeResult
3450LegalizerHelper::narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx,
3451 LLT NarrowTy) {
3452 if (TypeIdx != 1)
3453 return UnableToLegalize;
3454
3455 uint64_t NarrowSize = NarrowTy.getSizeInBits();
3456
3457 int64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
3458 // FIXME: add support for when SizeOp1 isn't an exact multiple of
3459 // NarrowSize.
3460 if (SizeOp1 % NarrowSize != 0)
3461 return UnableToLegalize;
3462 int NumParts = SizeOp1 / NarrowSize;
3463
3464 SmallVector<Register, 2> SrcRegs, DstRegs;
3465 SmallVector<uint64_t, 2> Indexes;
3466 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
3467
3468 Register OpReg = MI.getOperand(0).getReg();
3469 uint64_t OpStart = MI.getOperand(2).getImm();
3470 uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
3471 for (int i = 0; i < NumParts; ++i) {
3472 unsigned SrcStart = i * NarrowSize;
3473
3474 if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) {
3475 // No part of the extract uses this subregister, ignore it.
3476 continue;
3477 } else if (SrcStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
3478 // The entire subregister is extracted, forward the value.
3479 DstRegs.push_back(SrcRegs[i]);
3480 continue;
3481 }
3482
3483 // OpSegStart is where this destination segment would start in OpReg if it
3484 // extended infinitely in both directions.
3485 int64_t ExtractOffset;
3486 uint64_t SegSize;
3487 if (OpStart < SrcStart) {
3488 ExtractOffset = 0;
3489 SegSize = std::min(NarrowSize, OpStart + OpSize - SrcStart);
3490 } else {
3491 ExtractOffset = OpStart - SrcStart;
3492 SegSize = std::min(SrcStart + NarrowSize - OpStart, OpSize);
3493 }
3494
3495 Register SegReg = SrcRegs[i];
3496 if (ExtractOffset != 0 || SegSize != NarrowSize) {
3497 // A genuine extract is needed.
3498 SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
3499 MIRBuilder.buildExtract(SegReg, SrcRegs[i], ExtractOffset);
3500 }
3501
3502 DstRegs.push_back(SegReg);
3503 }
3504
3505 Register DstReg = MI.getOperand(0).getReg();
3506 if(MRI.getType(DstReg).isVector())
3507 MIRBuilder.buildBuildVector(DstReg, DstRegs);
3508 else
3509 MIRBuilder.buildMerge(DstReg, DstRegs);
3510 MI.eraseFromParent();
3511 return Legalized;
3512}
3513
3514LegalizerHelper::LegalizeResult
3515LegalizerHelper::narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx,
3516 LLT NarrowTy) {
3517 // FIXME: Don't know how to handle secondary types yet.
3518 if (TypeIdx != 0)
3519 return UnableToLegalize;
3520
3521 uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
3522 uint64_t NarrowSize = NarrowTy.getSizeInBits();
3523
3524 // FIXME: add support for when SizeOp0 isn't an exact multiple of
3525 // NarrowSize.
3526 if (SizeOp0 % NarrowSize != 0)
3527 return UnableToLegalize;
3528
3529 int NumParts = SizeOp0 / NarrowSize;
3530
3531 SmallVector<Register, 2> SrcRegs, DstRegs;
3532 SmallVector<uint64_t, 2> Indexes;
3533 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
3534
3535 Register OpReg = MI.getOperand(2).getReg();
3536 uint64_t OpStart = MI.getOperand(3).getImm();
3537 uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
3538 for (int i = 0; i < NumParts; ++i) {
3539 unsigned DstStart = i * NarrowSize;
3540
3541 if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) {
3542 // No part of the insert affects this subregister, forward the original.
3543 DstRegs.push_back(SrcRegs[i]);
3544 continue;
3545 } else if (DstStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
3546 // The entire subregister is defined by this insert, forward the new
3547 // value.
3548 DstRegs.push_back(OpReg);
3549 continue;
3550 }
3551
3552 // OpSegStart is where this destination segment would start in OpReg if it
3553 // extended infinitely in both directions.
3554 int64_t ExtractOffset, InsertOffset;
3555 uint64_t SegSize;
3556 if (OpStart < DstStart) {
3557 InsertOffset = 0;
3558 ExtractOffset = DstStart - OpStart;
3559 SegSize = std::min(NarrowSize, OpStart + OpSize - DstStart);
3560 } else {
3561 InsertOffset = OpStart - DstStart;
3562 ExtractOffset = 0;
3563 SegSize =
3564 std::min(NarrowSize - InsertOffset, OpStart + OpSize - DstStart);
3565 }
3566
3567 Register SegReg = OpReg;
3568 if (ExtractOffset != 0 || SegSize != OpSize) {
3569 // A genuine extract is needed.
3570 SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
3571 MIRBuilder.buildExtract(SegReg, OpReg, ExtractOffset);
3572 }
3573
3574 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy);
3575 MIRBuilder.buildInsert(DstReg, SrcRegs[i], SegReg, InsertOffset);
3576 DstRegs.push_back(DstReg);
3577 }
3578
3579 assert(DstRegs.size() == (unsigned)NumParts && "not all parts covered")((DstRegs.size() == (unsigned)NumParts && "not all parts covered"
) ? static_cast<void> (0) : __assert_fail ("DstRegs.size() == (unsigned)NumParts && \"not all parts covered\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 3579, __PRETTY_FUNCTION__))
;
3580 Register DstReg = MI.getOperand(0).getReg();
3581 if(MRI.getType(DstReg).isVector())
3582 MIRBuilder.buildBuildVector(DstReg, DstRegs);
3583 else
3584 MIRBuilder.buildMerge(DstReg, DstRegs);
3585 MI.eraseFromParent();
3586 return Legalized;
3587}
3588
3589LegalizerHelper::LegalizeResult
3590LegalizerHelper::narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx,
3591 LLT NarrowTy) {
3592 Register DstReg = MI.getOperand(0).getReg();
3593 LLT DstTy = MRI.getType(DstReg);
3594
3595 assert(MI.getNumOperands() == 3 && TypeIdx == 0)((MI.getNumOperands() == 3 && TypeIdx == 0) ? static_cast
<void> (0) : __assert_fail ("MI.getNumOperands() == 3 && TypeIdx == 0"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 3595, __PRETTY_FUNCTION__))
;
3596
3597 SmallVector<Register, 4> DstRegs, DstLeftoverRegs;
3598 SmallVector<Register, 4> Src0Regs, Src0LeftoverRegs;
3599 SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs;
3600 LLT LeftoverTy;
3601 if (!extractParts(MI.getOperand(1).getReg(), DstTy, NarrowTy, LeftoverTy,
3602 Src0Regs, Src0LeftoverRegs))
3603 return UnableToLegalize;
3604
3605 LLT Unused;
3606 if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, Unused,
3607 Src1Regs, Src1LeftoverRegs))
3608 llvm_unreachable("inconsistent extractParts result")::llvm::llvm_unreachable_internal("inconsistent extractParts result"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 3608)
;
3609
3610 for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) {
3611 auto Inst = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy},
3612 {Src0Regs[I], Src1Regs[I]});
3613 DstRegs.push_back(Inst->getOperand(0).getReg());
3614 }
3615
3616 for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) {
3617 auto Inst = MIRBuilder.buildInstr(
3618 MI.getOpcode(),
3619 {LeftoverTy}, {Src0LeftoverRegs[I], Src1LeftoverRegs[I]});
3620 DstLeftoverRegs.push_back(Inst->getOperand(0).getReg());
3621 }
3622
3623 insertParts(DstReg, DstTy, NarrowTy, DstRegs,
3624 LeftoverTy, DstLeftoverRegs);
3625
3626 MI.eraseFromParent();
3627 return Legalized;
3628}
3629
3630LegalizerHelper::LegalizeResult
3631LegalizerHelper::narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx,
3632 LLT NarrowTy) {
3633 if (TypeIdx != 0)
3634 return UnableToLegalize;
3635
3636 Register CondReg = MI.getOperand(1).getReg();
3637 LLT CondTy = MRI.getType(CondReg);
3638 if (CondTy.isVector()) // TODO: Handle vselect
3639 return UnableToLegalize;
3640
3641 Register DstReg = MI.getOperand(0).getReg();
3642 LLT DstTy = MRI.getType(DstReg);
3643
3644 SmallVector<Register, 4> DstRegs, DstLeftoverRegs;
3645 SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs;
3646 SmallVector<Register, 4> Src2Regs, Src2LeftoverRegs;
3647 LLT LeftoverTy;
3648 if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, LeftoverTy,
3649 Src1Regs, Src1LeftoverRegs))
3650 return UnableToLegalize;
3651
3652 LLT Unused;
3653 if (!extractParts(MI.getOperand(3).getReg(), DstTy, NarrowTy, Unused,
3654 Src2Regs, Src2LeftoverRegs))
3655 llvm_unreachable("inconsistent extractParts result")::llvm::llvm_unreachable_internal("inconsistent extractParts result"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 3655)
;
3656
3657 for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) {
3658 auto Select = MIRBuilder.buildSelect(NarrowTy,
3659 CondReg, Src1Regs[I], Src2Regs[I]);
3660 DstRegs.push_back(Select->getOperand(0).getReg());
3661 }
3662
3663 for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) {
3664 auto Select = MIRBuilder.buildSelect(
3665 LeftoverTy, CondReg, Src1LeftoverRegs[I], Src2LeftoverRegs[I]);
3666 DstLeftoverRegs.push_back(Select->getOperand(0).getReg());
3667 }
3668
3669 insertParts(DstReg, DstTy, NarrowTy, DstRegs,
3670 LeftoverTy, DstLeftoverRegs);
3671
3672 MI.eraseFromParent();
3673 return Legalized;
3674}
3675
3676LegalizerHelper::LegalizeResult
3677LegalizerHelper::lowerBitCount(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
3678 unsigned Opc = MI.getOpcode();
3679 auto &TII = *MI.getMF()->getSubtarget().getInstrInfo();
3680 auto isSupported = [this](const LegalityQuery &Q) {
3681 auto QAction = LI.getAction(Q).Action;
3682 return QAction == Legal || QAction == Libcall || QAction == Custom;
3683 };
3684 switch (Opc) {
3685 default:
3686 return UnableToLegalize;
3687 case TargetOpcode::G_CTLZ_ZERO_UNDEF: {
3688 // This trivially expands to CTLZ.
3689 Observer.changingInstr(MI);
3690 MI.setDesc(TII.get(TargetOpcode::G_CTLZ));
3691 Observer.changedInstr(MI);
3692 return Legalized;
3693 }
3694 case TargetOpcode::G_CTLZ: {
3695 Register SrcReg = MI.getOperand(1).getReg();
3696 unsigned Len = Ty.getSizeInBits();
3697 if (isSupported({TargetOpcode::G_CTLZ_ZERO_UNDEF, {Ty, Ty}})) {
3698 // If CTLZ_ZERO_UNDEF is supported, emit that and a select for zero.
3699 auto MIBCtlzZU = MIRBuilder.buildInstr(TargetOpcode::G_CTLZ_ZERO_UNDEF,
3700 {Ty}, {SrcReg});
3701 auto MIBZero = MIRBuilder.buildConstant(Ty, 0);
3702 auto MIBLen = MIRBuilder.buildConstant(Ty, Len);
3703 auto MIBICmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1),
3704 SrcReg, MIBZero);
3705 MIRBuilder.buildSelect(MI.getOperand(0).getReg(), MIBICmp, MIBLen,
3706 MIBCtlzZU);
3707 MI.eraseFromParent();
3708 return Legalized;
3709 }
3710 // for now, we do this:
3711 // NewLen = NextPowerOf2(Len);
3712 // x = x | (x >> 1);
3713 // x = x | (x >> 2);
3714 // ...
3715 // x = x | (x >>16);
3716 // x = x | (x >>32); // for 64-bit input
3717 // Upto NewLen/2
3718 // return Len - popcount(x);
3719 //
3720 // Ref: "Hacker's Delight" by Henry Warren
3721 Register Op = SrcReg;
3722 unsigned NewLen = PowerOf2Ceil(Len);
3723 for (unsigned i = 0; (1U << i) <= (NewLen / 2); ++i) {
3724 auto MIBShiftAmt = MIRBuilder.buildConstant(Ty, 1ULL << i);
3725 auto MIBOp = MIRBuilder.buildInstr(
3726 TargetOpcode::G_OR, {Ty},
3727 {Op, MIRBuilder.buildInstr(TargetOpcode::G_LSHR, {Ty},
3728 {Op, MIBShiftAmt})});
3729 Op = MIBOp->getOperand(0).getReg();
3730 }
3731 auto MIBPop = MIRBuilder.buildInstr(TargetOpcode::G_CTPOP, {Ty}, {Op});
3732 MIRBuilder.buildInstr(TargetOpcode::G_SUB, {MI.getOperand(0).getReg()},
3733 {MIRBuilder.buildConstant(Ty, Len), MIBPop});
3734 MI.eraseFromParent();
3735 return Legalized;
3736 }
3737 case TargetOpcode::G_CTTZ_ZERO_UNDEF: {
3738 // This trivially expands to CTTZ.
3739 Observer.changingInstr(MI);
3740 MI.setDesc(TII.get(TargetOpcode::G_CTTZ));
3741 Observer.changedInstr(MI);
3742 return Legalized;
3743 }
3744 case TargetOpcode::G_CTTZ: {
3745 Register SrcReg = MI.getOperand(1).getReg();
3746 unsigned Len = Ty.getSizeInBits();
3747 if (isSupported({TargetOpcode::G_CTTZ_ZERO_UNDEF, {Ty, Ty}})) {
3748 // If CTTZ_ZERO_UNDEF is legal or custom, emit that and a select with
3749 // zero.
3750 auto MIBCttzZU = MIRBuilder.buildInstr(TargetOpcode::G_CTTZ_ZERO_UNDEF,
3751 {Ty}, {SrcReg});
3752 auto MIBZero = MIRBuilder.buildConstant(Ty, 0);
3753 auto MIBLen = MIRBuilder.buildConstant(Ty, Len);
3754 auto MIBICmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1),
3755 SrcReg, MIBZero);
3756 MIRBuilder.buildSelect(MI.getOperand(0).getReg(), MIBICmp, MIBLen,
3757 MIBCttzZU);
3758 MI.eraseFromParent();
3759 return Legalized;
3760 }
3761 // for now, we use: { return popcount(~x & (x - 1)); }
3762 // unless the target has ctlz but not ctpop, in which case we use:
3763 // { return 32 - nlz(~x & (x-1)); }
3764 // Ref: "Hacker's Delight" by Henry Warren
3765 auto MIBCstNeg1 = MIRBuilder.buildConstant(Ty, -1);
3766 auto MIBNot =
3767 MIRBuilder.buildInstr(TargetOpcode::G_XOR, {Ty}, {SrcReg, MIBCstNeg1});
3768 auto MIBTmp = MIRBuilder.buildInstr(
3769 TargetOpcode::G_AND, {Ty},
3770 {MIBNot, MIRBuilder.buildInstr(TargetOpcode::G_ADD, {Ty},
3771 {SrcReg, MIBCstNeg1})});
3772 if (!isSupported({TargetOpcode::G_CTPOP, {Ty, Ty}}) &&
3773 isSupported({TargetOpcode::G_CTLZ, {Ty, Ty}})) {
3774 auto MIBCstLen = MIRBuilder.buildConstant(Ty, Len);
3775 MIRBuilder.buildInstr(
3776 TargetOpcode::G_SUB, {MI.getOperand(0).getReg()},
3777 {MIBCstLen,
3778 MIRBuilder.buildInstr(TargetOpcode::G_CTLZ, {Ty}, {MIBTmp})});
3779 MI.eraseFromParent();
3780 return Legalized;
3781 }
3782 MI.setDesc(TII.get(TargetOpcode::G_CTPOP));
3783 MI.getOperand(1).setReg(MIBTmp->getOperand(0).getReg());
3784 return Legalized;
3785 }
3786 }
3787}
3788
3789// Expand s32 = G_UITOFP s64 using bit operations to an IEEE float
3790// representation.
3791LegalizerHelper::LegalizeResult
3792LegalizerHelper::lowerU64ToF32BitOps(MachineInstr &MI) {
3793 Register Dst = MI.getOperand(0).getReg();
3794 Register Src = MI.getOperand(1).getReg();
3795 const LLT S64 = LLT::scalar(64);
3796 const LLT S32 = LLT::scalar(32);
3797 const LLT S1 = LLT::scalar(1);
3798
3799 assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S32)((MRI.getType(Src) == S64 && MRI.getType(Dst) == S32)
? static_cast<void> (0) : __assert_fail ("MRI.getType(Src) == S64 && MRI.getType(Dst) == S32"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 3799, __PRETTY_FUNCTION__))
;
3800
3801 // unsigned cul2f(ulong u) {
3802 // uint lz = clz(u);
3803 // uint e = (u != 0) ? 127U + 63U - lz : 0;
3804 // u = (u << lz) & 0x7fffffffffffffffUL;
3805 // ulong t = u & 0xffffffffffUL;
3806 // uint v = (e << 23) | (uint)(u >> 40);
3807 // uint r = t > 0x8000000000UL ? 1U : (t == 0x8000000000UL ? v & 1U : 0U);
3808 // return as_float(v + r);
3809 // }
3810
3811 auto Zero32 = MIRBuilder.buildConstant(S32, 0);
3812 auto Zero64 = MIRBuilder.buildConstant(S64, 0);
3813
3814 auto LZ = MIRBuilder.buildCTLZ_ZERO_UNDEF(S32, Src);
3815
3816 auto K = MIRBuilder.buildConstant(S32, 127U + 63U);
3817 auto Sub = MIRBuilder.buildSub(S32, K, LZ);
3818
3819 auto NotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, Src, Zero64);
3820 auto E = MIRBuilder.buildSelect(S32, NotZero, Sub, Zero32);
3821
3822 auto Mask0 = MIRBuilder.buildConstant(S64, (-1ULL) >> 1);
3823 auto ShlLZ = MIRBuilder.buildShl(S64, Src, LZ);
3824
3825 auto U = MIRBuilder.buildAnd(S64, ShlLZ, Mask0);
3826
3827 auto Mask1 = MIRBuilder.buildConstant(S64, 0xffffffffffULL);
3828 auto T = MIRBuilder.buildAnd(S64, U, Mask1);
3829
3830 auto UShl = MIRBuilder.buildLShr(S64, U, MIRBuilder.buildConstant(S64, 40));
3831 auto ShlE = MIRBuilder.buildShl(S32, E, MIRBuilder.buildConstant(S32, 23));
3832 auto V = MIRBuilder.buildOr(S32, ShlE, MIRBuilder.buildTrunc(S32, UShl));
3833
3834 auto C = MIRBuilder.buildConstant(S64, 0x8000000000ULL);
3835 auto RCmp = MIRBuilder.buildICmp(CmpInst::ICMP_UGT, S1, T, C);
3836 auto TCmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1, T, C);
3837 auto One = MIRBuilder.buildConstant(S32, 1);
3838
3839 auto VTrunc1 = MIRBuilder.buildAnd(S32, V, One);
3840 auto Select0 = MIRBuilder.buildSelect(S32, TCmp, VTrunc1, Zero32);
3841 auto R = MIRBuilder.buildSelect(S32, RCmp, One, Select0);
3842 MIRBuilder.buildAdd(Dst, V, R);
3843
3844 return Legalized;
3845}
3846
3847LegalizerHelper::LegalizeResult
3848LegalizerHelper::lowerUITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
3849 Register Dst = MI.getOperand(0).getReg();
3850 Register Src = MI.getOperand(1).getReg();
3851 LLT DstTy = MRI.getType(Dst);
3852 LLT SrcTy = MRI.getType(Src);
3853
3854 if (SrcTy != LLT::scalar(64))
3855 return UnableToLegalize;
3856
3857 if (DstTy == LLT::scalar(32)) {
3858 // TODO: SelectionDAG has several alternative expansions to port which may
3859 // be more reasonble depending on the available instructions. If a target
3860 // has sitofp, does not have CTLZ, or can efficiently use f64 as an
3861 // intermediate type, this is probably worse.
3862 return lowerU64ToF32BitOps(MI);
3863 }
3864
3865 return UnableToLegalize;
3866}
3867
3868LegalizerHelper::LegalizeResult
3869LegalizerHelper::lowerSITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
3870 Register Dst = MI.getOperand(0).getReg();
3871 Register Src = MI.getOperand(1).getReg();
3872 LLT DstTy = MRI.getType(Dst);
3873 LLT SrcTy = MRI.getType(Src);
3874
3875 const LLT S64 = LLT::scalar(64);
3876 const LLT S32 = LLT::scalar(32);
3877 const LLT S1 = LLT::scalar(1);
3878
3879 if (SrcTy != S64)
3880 return UnableToLegalize;
3881
3882 if (DstTy == S32) {
3883 // signed cl2f(long l) {
3884 // long s = l >> 63;
3885 // float r = cul2f((l + s) ^ s);
3886 // return s ? -r : r;
3887 // }
3888 Register L = Src;
3889 auto SignBit = MIRBuilder.buildConstant(S64, 63);
3890 auto S = MIRBuilder.buildAShr(S64, L, SignBit);
3891
3892 auto LPlusS = MIRBuilder.buildAdd(S64, L, S);
3893 auto Xor = MIRBuilder.buildXor(S64, LPlusS, S);
3894 auto R = MIRBuilder.buildUITOFP(S32, Xor);
3895
3896 auto RNeg = MIRBuilder.buildFNeg(S32, R);
3897 auto SignNotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, S,
3898 MIRBuilder.buildConstant(S64, 0));
3899 MIRBuilder.buildSelect(Dst, SignNotZero, RNeg, R);
3900 return Legalized;
3901 }
3902
3903 return UnableToLegalize;
3904}
3905
3906LegalizerHelper::LegalizeResult
3907LegalizerHelper::lowerFPTOUI(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
3908 Register Dst = MI.getOperand(0).getReg();
3909 Register Src = MI.getOperand(1).getReg();
3910 LLT DstTy = MRI.getType(Dst);
3911 LLT SrcTy = MRI.getType(Src);
3912 const LLT S64 = LLT::scalar(64);
3913 const LLT S32 = LLT::scalar(32);
3914
3915 if (SrcTy != S64 && SrcTy != S32)
3916 return UnableToLegalize;
3917 if (DstTy != S32 && DstTy != S64)
3918 return UnableToLegalize;
3919
3920 // FPTOSI gives same result as FPTOUI for positive signed integers.
3921 // FPTOUI needs to deal with fp values that convert to unsigned integers
3922 // greater or equal to 2^31 for float or 2^63 for double. For brevity 2^Exp.
3923
3924 APInt TwoPExpInt = APInt::getSignMask(DstTy.getSizeInBits());
3925 APFloat TwoPExpFP(SrcTy.getSizeInBits() == 32 ? APFloat::IEEEsingle()
3926 : APFloat::IEEEdouble(),
3927 APInt::getNullValue(SrcTy.getSizeInBits()));
3928 TwoPExpFP.convertFromAPInt(TwoPExpInt, false, APFloat::rmNearestTiesToEven);
3929
3930 MachineInstrBuilder FPTOSI = MIRBuilder.buildFPTOSI(DstTy, Src);
3931
3932 MachineInstrBuilder Threshold = MIRBuilder.buildFConstant(SrcTy, TwoPExpFP);
3933 // For fp Value greater or equal to Threshold(2^Exp), we use FPTOSI on
3934 // (Value - 2^Exp) and add 2^Exp by setting highest bit in result to 1.
3935 MachineInstrBuilder FSub = MIRBuilder.buildFSub(SrcTy, Src, Threshold);
3936 MachineInstrBuilder ResLowBits = MIRBuilder.buildFPTOSI(DstTy, FSub);
3937 MachineInstrBuilder ResHighBit = MIRBuilder.buildConstant(DstTy, TwoPExpInt);
3938 MachineInstrBuilder Res = MIRBuilder.buildXor(DstTy, ResLowBits, ResHighBit);
3939
3940 MachineInstrBuilder FCMP =
3941 MIRBuilder.buildFCmp(CmpInst::FCMP_ULT, DstTy, Src, Threshold);
3942 MIRBuilder.buildSelect(Dst, FCMP, FPTOSI, Res);
3943
3944 MI.eraseFromParent();
3945 return Legalized;
3946}
3947
3948static CmpInst::Predicate minMaxToCompare(unsigned Opc) {
3949 switch (Opc) {
3950 case TargetOpcode::G_SMIN:
3951 return CmpInst::ICMP_SLT;
3952 case TargetOpcode::G_SMAX:
3953 return CmpInst::ICMP_SGT;
3954 case TargetOpcode::G_UMIN:
3955 return CmpInst::ICMP_ULT;
3956 case TargetOpcode::G_UMAX:
3957 return CmpInst::ICMP_UGT;
3958 default:
3959 llvm_unreachable("not in integer min/max")::llvm::llvm_unreachable_internal("not in integer min/max", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 3959)
;
3960 }
3961}
3962
3963LegalizerHelper::LegalizeResult
3964LegalizerHelper::lowerMinMax(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
3965 Register Dst = MI.getOperand(0).getReg();
3966 Register Src0 = MI.getOperand(1).getReg();
3967 Register Src1 = MI.getOperand(2).getReg();
3968
3969 const CmpInst::Predicate Pred = minMaxToCompare(MI.getOpcode());
3970 LLT CmpType = MRI.getType(Dst).changeElementSize(1);
3971
3972 auto Cmp = MIRBuilder.buildICmp(Pred, CmpType, Src0, Src1);
3973 MIRBuilder.buildSelect(Dst, Cmp, Src0, Src1);
3974
3975 MI.eraseFromParent();
3976 return Legalized;
3977}
3978
3979LegalizerHelper::LegalizeResult
3980LegalizerHelper::lowerFCopySign(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
3981 Register Dst = MI.getOperand(0).getReg();
3982 Register Src0 = MI.getOperand(1).getReg();
3983 Register Src1 = MI.getOperand(2).getReg();
3984
3985 const LLT Src0Ty = MRI.getType(Src0);
3986 const LLT Src1Ty = MRI.getType(Src1);
3987
3988 const int Src0Size = Src0Ty.getScalarSizeInBits();
3989 const int Src1Size = Src1Ty.getScalarSizeInBits();
3990
3991 auto SignBitMask = MIRBuilder.buildConstant(
3992 Src0Ty, APInt::getSignMask(Src0Size));
3993
3994 auto NotSignBitMask = MIRBuilder.buildConstant(
3995 Src0Ty, APInt::getLowBitsSet(Src0Size, Src0Size - 1));
3996
3997 auto And0 = MIRBuilder.buildAnd(Src0Ty, Src0, NotSignBitMask);
3998 MachineInstr *Or;
3999
4000 if (Src0Ty == Src1Ty) {
4001 auto And1 = MIRBuilder.buildAnd(Src1Ty, Src0, SignBitMask);
4002 Or = MIRBuilder.buildOr(Dst, And0, And1);
4003 } else if (Src0Size > Src1Size) {
4004 auto ShiftAmt = MIRBuilder.buildConstant(Src0Ty, Src0Size - Src1Size);
4005 auto Zext = MIRBuilder.buildZExt(Src0Ty, Src1);
4006 auto Shift = MIRBuilder.buildShl(Src0Ty, Zext, ShiftAmt);
4007 auto And1 = MIRBuilder.buildAnd(Src0Ty, Shift, SignBitMask);
4008 Or = MIRBuilder.buildOr(Dst, And0, And1);
4009 } else {
4010 auto ShiftAmt = MIRBuilder.buildConstant(Src1Ty, Src1Size - Src0Size);
4011 auto Shift = MIRBuilder.buildLShr(Src1Ty, Src1, ShiftAmt);
4012 auto Trunc = MIRBuilder.buildTrunc(Src0Ty, Shift);
4013 auto And1 = MIRBuilder.buildAnd(Src0Ty, Trunc, SignBitMask);
4014 Or = MIRBuilder.buildOr(Dst, And0, And1);
4015 }
4016
4017 // Be careful about setting nsz/nnan/ninf on every instruction, since the
4018 // constants are a nan and -0.0, but the final result should preserve
4019 // everything.
4020 if (unsigned Flags = MI.getFlags())
4021 Or->setFlags(Flags);
4022
4023 MI.eraseFromParent();
4024 return Legalized;
4025}
4026
4027LegalizerHelper::LegalizeResult
4028LegalizerHelper::lowerFMinNumMaxNum(MachineInstr &MI) {
4029 unsigned NewOp = MI.getOpcode() == TargetOpcode::G_FMINNUM ?
4030 TargetOpcode::G_FMINNUM_IEEE : TargetOpcode::G_FMAXNUM_IEEE;
4031
4032 Register Dst = MI.getOperand(0).getReg();
4033 Register Src0 = MI.getOperand(1).getReg();
4034 Register Src1 = MI.getOperand(2).getReg();
4035 LLT Ty = MRI.getType(Dst);
4036
4037 if (!MI.getFlag(MachineInstr::FmNoNans)) {
4038 // Insert canonicalizes if it's possible we need to quiet to get correct
4039 // sNaN behavior.
4040
4041 // Note this must be done here, and not as an optimization combine in the
4042 // absence of a dedicate quiet-snan instruction as we're using an
4043 // omni-purpose G_FCANONICALIZE.
4044 if (!isKnownNeverSNaN(Src0, MRI))
4045 Src0 = MIRBuilder.buildFCanonicalize(Ty, Src0, MI.getFlags()).getReg(0);
4046
4047 if (!isKnownNeverSNaN(Src1, MRI))
4048 Src1 = MIRBuilder.buildFCanonicalize(Ty, Src1, MI.getFlags()).getReg(0);
4049 }
4050
4051 // If there are no nans, it's safe to simply replace this with the non-IEEE
4052 // version.
4053 MIRBuilder.buildInstr(NewOp, {Dst}, {Src0, Src1}, MI.getFlags());
4054 MI.eraseFromParent();
4055 return Legalized;
4056}
4057
4058LegalizerHelper::LegalizeResult LegalizerHelper::lowerFMad(MachineInstr &MI) {
4059 // Expand G_FMAD a, b, c -> G_FADD (G_FMUL a, b), c
4060 Register DstReg = MI.getOperand(0).getReg();
4061 LLT Ty = MRI.getType(DstReg);
4062 unsigned Flags = MI.getFlags();
4063
4064 auto Mul = MIRBuilder.buildFMul(Ty, MI.getOperand(1), MI.getOperand(2),
4065 Flags);
4066 MIRBuilder.buildFAdd(DstReg, Mul, MI.getOperand(3), Flags);
4067 MI.eraseFromParent();
4068 return Legalized;
4069}
4070
4071LegalizerHelper::LegalizeResult
4072LegalizerHelper::lowerUnmergeValues(MachineInstr &MI) {
4073 const unsigned NumDst = MI.getNumOperands() - 1;
4074 const Register SrcReg = MI.getOperand(NumDst).getReg();
4075 LLT SrcTy = MRI.getType(SrcReg);
4076
4077 Register Dst0Reg = MI.getOperand(0).getReg();
4078 LLT DstTy = MRI.getType(Dst0Reg);
4079
4080
4081 // Expand scalarizing unmerge as bitcast to integer and shift.
4082 if (!DstTy.isVector() && SrcTy.isVector() &&
4083 SrcTy.getElementType() == DstTy) {
4084 LLT IntTy = LLT::scalar(SrcTy.getSizeInBits());
4085 Register Cast = MIRBuilder.buildBitcast(IntTy, SrcReg).getReg(0);
4086
4087 MIRBuilder.buildTrunc(Dst0Reg, Cast);
4088
4089 const unsigned DstSize = DstTy.getSizeInBits();
4090 unsigned Offset = DstSize;
4091 for (unsigned I = 1; I != NumDst; ++I, Offset += DstSize) {
4092 auto ShiftAmt = MIRBuilder.buildConstant(IntTy, Offset);
4093 auto Shift = MIRBuilder.buildLShr(IntTy, Cast, ShiftAmt);
4094 MIRBuilder.buildTrunc(MI.getOperand(I), Shift);
4095 }
4096
4097 MI.eraseFromParent();
4098 return Legalized;
4099 }
4100
4101 return UnableToLegalize;
4102}
4103
4104LegalizerHelper::LegalizeResult
4105LegalizerHelper::lowerShuffleVector(MachineInstr &MI) {
4106 Register DstReg = MI.getOperand(0).getReg();
4107 Register Src0Reg = MI.getOperand(1).getReg();
4108 Register Src1Reg = MI.getOperand(2).getReg();
4109 LLT Src0Ty = MRI.getType(Src0Reg);
4110 LLT DstTy = MRI.getType(DstReg);
4111 LLT IdxTy = LLT::scalar(32);
4112
4113 const Constant *ShufMask = MI.getOperand(3).getShuffleMask();
4114
4115 SmallVector<int, 32> Mask;
4116 ShuffleVectorInst::getShuffleMask(ShufMask, Mask);
4117
4118 if (DstTy.isScalar()) {
4119 if (Src0Ty.isVector())
4120 return UnableToLegalize;
4121
4122 // This is just a SELECT.
4123 assert(Mask.size() == 1 && "Expected a single mask element")((Mask.size() == 1 && "Expected a single mask element"
) ? static_cast<void> (0) : __assert_fail ("Mask.size() == 1 && \"Expected a single mask element\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 4123, __PRETTY_FUNCTION__))
;
4124 Register Val;
4125 if (Mask[0] < 0 || Mask[0] > 1)
4126 Val = MIRBuilder.buildUndef(DstTy).getReg(0);
4127 else
4128 Val = Mask[0] == 0 ? Src0Reg : Src1Reg;
4129 MIRBuilder.buildCopy(DstReg, Val);
4130 MI.eraseFromParent();
4131 return Legalized;
4132 }
4133
4134 Register Undef;
4135 SmallVector<Register, 32> BuildVec;
4136 LLT EltTy = DstTy.getElementType();
4137
4138 for (int Idx : Mask) {
4139 if (Idx < 0) {
4140 if (!Undef.isValid())
4141 Undef = MIRBuilder.buildUndef(EltTy).getReg(0);
4142 BuildVec.push_back(Undef);
4143 continue;
4144 }
4145
4146 if (Src0Ty.isScalar()) {
4147 BuildVec.push_back(Idx == 0 ? Src0Reg : Src1Reg);
4148 } else {
4149 int NumElts = Src0Ty.getNumElements();
4150 Register SrcVec = Idx < NumElts ? Src0Reg : Src1Reg;
4151 int ExtractIdx = Idx < NumElts ? Idx : Idx - NumElts;
4152 auto IdxK = MIRBuilder.buildConstant(IdxTy, ExtractIdx);
4153 auto Extract = MIRBuilder.buildExtractVectorElement(EltTy, SrcVec, IdxK);
4154 BuildVec.push_back(Extract.getReg(0));
4155 }
4156 }
4157
4158 MIRBuilder.buildBuildVector(DstReg, BuildVec);
4159 MI.eraseFromParent();
4160 return Legalized;
4161}
4162
4163LegalizerHelper::LegalizeResult
4164LegalizerHelper::lowerDynStackAlloc(MachineInstr &MI) {
4165 Register Dst = MI.getOperand(0).getReg();
4166 Register AllocSize = MI.getOperand(1).getReg();
4167 unsigned Align = MI.getOperand(2).getImm();
4168
4169 const auto &MF = *MI.getMF();
4170 const auto &TLI = *MF.getSubtarget().getTargetLowering();
4171
4172 LLT PtrTy = MRI.getType(Dst);
4173 LLT IntPtrTy = LLT::scalar(PtrTy.getSizeInBits());
4174
4175 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
4176 auto SPTmp = MIRBuilder.buildCopy(PtrTy, SPReg);
4177 SPTmp = MIRBuilder.buildCast(IntPtrTy, SPTmp);
4178
4179 // Subtract the final alloc from the SP. We use G_PTRTOINT here so we don't
4180 // have to generate an extra instruction to negate the alloc and then use
4181 // G_PTR_ADD to add the negative offset.
4182 auto Alloc = MIRBuilder.buildSub(IntPtrTy, SPTmp, AllocSize);
4183 if (Align) {
4184 APInt AlignMask(IntPtrTy.getSizeInBits(), Align, true);
4185 AlignMask.negate();
4186 auto AlignCst = MIRBuilder.buildConstant(IntPtrTy, AlignMask);
4187 Alloc = MIRBuilder.buildAnd(IntPtrTy, Alloc, AlignCst);
4188 }
4189
4190 SPTmp = MIRBuilder.buildCast(PtrTy, Alloc);
4191 MIRBuilder.buildCopy(SPReg, SPTmp);
4192 MIRBuilder.buildCopy(Dst, SPTmp);
4193
4194 MI.eraseFromParent();
4195 return Legalized;
4196}
4197
4198LegalizerHelper::LegalizeResult
4199LegalizerHelper::lowerExtract(MachineInstr &MI) {
4200 Register Dst = MI.getOperand(0).getReg();
4201 Register Src = MI.getOperand(1).getReg();
4202 unsigned Offset = MI.getOperand(2).getImm();
4203
4204 LLT DstTy = MRI.getType(Dst);
4205 LLT SrcTy = MRI.getType(Src);
4206
4207 if (DstTy.isScalar() &&
4208 (SrcTy.isScalar() ||
4209 (SrcTy.isVector() && DstTy == SrcTy.getElementType()))) {
4210 LLT SrcIntTy = SrcTy;
4211 if (!SrcTy.isScalar()) {
4212 SrcIntTy = LLT::scalar(SrcTy.getSizeInBits());
4213 Src = MIRBuilder.buildBitcast(SrcIntTy, Src).getReg(0);
4214 }
4215
4216 if (Offset == 0)
4217 MIRBuilder.buildTrunc(Dst, Src);
4218 else {
4219 auto ShiftAmt = MIRBuilder.buildConstant(SrcIntTy, Offset);
4220 auto Shr = MIRBuilder.buildLShr(SrcIntTy, Src, ShiftAmt);
4221 MIRBuilder.buildTrunc(Dst, Shr);
4222 }
4223
4224 MI.eraseFromParent();
4225 return Legalized;
4226 }
4227
4228 return UnableToLegalize;
4229}
4230
4231LegalizerHelper::LegalizeResult LegalizerHelper::lowerInsert(MachineInstr &MI) {
4232 Register Dst = MI.getOperand(0).getReg();
4233 Register Src = MI.getOperand(1).getReg();
4234 Register InsertSrc = MI.getOperand(2).getReg();
4235 uint64_t Offset = MI.getOperand(3).getImm();
4236
4237 LLT DstTy = MRI.getType(Src);
4238 LLT InsertTy = MRI.getType(InsertSrc);
4239
4240 if (InsertTy.isScalar() &&
4241 (DstTy.isScalar() ||
4242 (DstTy.isVector() && DstTy.getElementType() == InsertTy))) {
4243 LLT IntDstTy = DstTy;
4244 if (!DstTy.isScalar()) {
4245 IntDstTy = LLT::scalar(DstTy.getSizeInBits());
4246 Src = MIRBuilder.buildBitcast(IntDstTy, Src).getReg(0);
4247 }
4248
4249 Register ExtInsSrc = MIRBuilder.buildZExt(IntDstTy, InsertSrc).getReg(0);
4250 if (Offset != 0) {
4251 auto ShiftAmt = MIRBuilder.buildConstant(IntDstTy, Offset);
4252 ExtInsSrc = MIRBuilder.buildShl(IntDstTy, ExtInsSrc, ShiftAmt).getReg(0);
4253 }
4254
4255 APInt MaskVal = ~APInt::getBitsSet(DstTy.getSizeInBits(), Offset,
4256 InsertTy.getSizeInBits());
4257
4258 auto Mask = MIRBuilder.buildConstant(IntDstTy, MaskVal);
4259 auto MaskedSrc = MIRBuilder.buildAnd(IntDstTy, Src, Mask);
4260 auto Or = MIRBuilder.buildOr(IntDstTy, MaskedSrc, ExtInsSrc);
4261
4262 MIRBuilder.buildBitcast(Dst, Or);
4263 MI.eraseFromParent();
4264 return Legalized;
4265 }
4266
4267 return UnableToLegalize;
4268}
4269
4270LegalizerHelper::LegalizeResult
4271LegalizerHelper::lowerSADDO_SSUBO(MachineInstr &MI) {
4272 Register Dst0 = MI.getOperand(0).getReg();
4273 Register Dst1 = MI.getOperand(1).getReg();
4274 Register LHS = MI.getOperand(2).getReg();
4275 Register RHS = MI.getOperand(3).getReg();
4276 const bool IsAdd = MI.getOpcode() == TargetOpcode::G_SADDO;
4277
4278 LLT Ty = MRI.getType(Dst0);
4279 LLT BoolTy = MRI.getType(Dst1);
4280
4281 if (IsAdd)
4282 MIRBuilder.buildAdd(Dst0, LHS, RHS);
4283 else
4284 MIRBuilder.buildSub(Dst0, LHS, RHS);
4285
4286 // TODO: If SADDSAT/SSUBSAT is legal, compare results to detect overflow.
4287
4288 auto Zero = MIRBuilder.buildConstant(Ty, 0);
4289
4290 // For an addition, the result should be less than one of the operands (LHS)
4291 // if and only if the other operand (RHS) is negative, otherwise there will
4292 // be overflow.
4293 // For a subtraction, the result should be less than one of the operands
4294 // (LHS) if and only if the other operand (RHS) is (non-zero) positive,
4295 // otherwise there will be overflow.
4296 auto ResultLowerThanLHS =
4297 MIRBuilder.buildICmp(CmpInst::ICMP_SLT, BoolTy, Dst0, LHS);
4298 auto ConditionRHS = MIRBuilder.buildICmp(
4299 IsAdd ? CmpInst::ICMP_SLT : CmpInst::ICMP_SGT, BoolTy, RHS, Zero);
4300
4301 MIRBuilder.buildXor(Dst1, ConditionRHS, ResultLowerThanLHS);
4302 MI.eraseFromParent();
4303 return Legalized;
4304}