Bug Summary

File:lib/CodeGen/GlobalISel/LegalizerHelper.cpp
Warning:line 2858, column 7
Value stored to 'NumLeftover' is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name LegalizerHelper.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -mframe-pointer=none -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-10/lib/clang/10.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-10~svn374877/build-llvm/lib/CodeGen/GlobalISel -I /build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel -I /build/llvm-toolchain-snapshot-10~svn374877/build-llvm/include -I /build/llvm-toolchain-snapshot-10~svn374877/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-10/lib/clang/10.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-10~svn374877/build-llvm/lib/CodeGen/GlobalISel -fdebug-prefix-map=/build/llvm-toolchain-snapshot-10~svn374877=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2019-10-15-233810-7101-1 -x c++ /build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
1//===-- llvm/CodeGen/GlobalISel/LegalizerHelper.cpp -----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This file implements the LegalizerHelper class to legalize
10/// individual instructions and the LegalizeMachineIR wrapper pass for the
11/// primary legalization.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
16#include "llvm/CodeGen/GlobalISel/CallLowering.h"
17#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
18#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
19#include "llvm/CodeGen/MachineRegisterInfo.h"
20#include "llvm/CodeGen/TargetFrameLowering.h"
21#include "llvm/CodeGen/TargetInstrInfo.h"
22#include "llvm/CodeGen/TargetLowering.h"
23#include "llvm/CodeGen/TargetSubtargetInfo.h"
24#include "llvm/Support/Debug.h"
25#include "llvm/Support/MathExtras.h"
26#include "llvm/Support/raw_ostream.h"
27
28#define DEBUG_TYPE"legalizer" "legalizer"
29
30using namespace llvm;
31using namespace LegalizeActions;
32
33/// Try to break down \p OrigTy into \p NarrowTy sized pieces.
34///
35/// Returns the number of \p NarrowTy elements needed to reconstruct \p OrigTy,
36/// with any leftover piece as type \p LeftoverTy
37///
38/// Returns -1 in the first element of the pair if the breakdown is not
39/// satisfiable.
40static std::pair<int, int>
41getNarrowTypeBreakDown(LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy) {
42 assert(!LeftoverTy.isValid() && "this is an out argument")((!LeftoverTy.isValid() && "this is an out argument")
? static_cast<void> (0) : __assert_fail ("!LeftoverTy.isValid() && \"this is an out argument\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 42, __PRETTY_FUNCTION__))
;
43
44 unsigned Size = OrigTy.getSizeInBits();
45 unsigned NarrowSize = NarrowTy.getSizeInBits();
46 unsigned NumParts = Size / NarrowSize;
47 unsigned LeftoverSize = Size - NumParts * NarrowSize;
48 assert(Size > NarrowSize)((Size > NarrowSize) ? static_cast<void> (0) : __assert_fail
("Size > NarrowSize", "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 48, __PRETTY_FUNCTION__))
;
49
50 if (LeftoverSize == 0)
51 return {NumParts, 0};
52
53 if (NarrowTy.isVector()) {
54 unsigned EltSize = OrigTy.getScalarSizeInBits();
55 if (LeftoverSize % EltSize != 0)
56 return {-1, -1};
57 LeftoverTy = LLT::scalarOrVector(LeftoverSize / EltSize, EltSize);
58 } else {
59 LeftoverTy = LLT::scalar(LeftoverSize);
60 }
61
62 int NumLeftover = LeftoverSize / LeftoverTy.getSizeInBits();
63 return std::make_pair(NumParts, NumLeftover);
64}
65
66LegalizerHelper::LegalizerHelper(MachineFunction &MF,
67 GISelChangeObserver &Observer,
68 MachineIRBuilder &Builder)
69 : MIRBuilder(Builder), MRI(MF.getRegInfo()),
70 LI(*MF.getSubtarget().getLegalizerInfo()), Observer(Observer) {
71 MIRBuilder.setMF(MF);
72 MIRBuilder.setChangeObserver(Observer);
73}
74
75LegalizerHelper::LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI,
76 GISelChangeObserver &Observer,
77 MachineIRBuilder &B)
78 : MIRBuilder(B), MRI(MF.getRegInfo()), LI(LI), Observer(Observer) {
79 MIRBuilder.setMF(MF);
80 MIRBuilder.setChangeObserver(Observer);
81}
82LegalizerHelper::LegalizeResult
83LegalizerHelper::legalizeInstrStep(MachineInstr &MI) {
84 LLVM_DEBUG(dbgs() << "Legalizing: "; MI.print(dbgs()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << "Legalizing: "; MI.print(dbgs
()); } } while (false)
;
85
86 if (MI.getOpcode() == TargetOpcode::G_INTRINSIC ||
87 MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS)
88 return LI.legalizeIntrinsic(MI, MRI, MIRBuilder) ? Legalized
89 : UnableToLegalize;
90 auto Step = LI.getAction(MI, MRI);
91 switch (Step.Action) {
92 case Legal:
93 LLVM_DEBUG(dbgs() << ".. Already legal\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Already legal\n"; } } while
(false)
;
94 return AlreadyLegal;
95 case Libcall:
96 LLVM_DEBUG(dbgs() << ".. Convert to libcall\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Convert to libcall\n"; }
} while (false)
;
97 return libcall(MI);
98 case NarrowScalar:
99 LLVM_DEBUG(dbgs() << ".. Narrow scalar\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Narrow scalar\n"; } } while
(false)
;
100 return narrowScalar(MI, Step.TypeIdx, Step.NewType);
101 case WidenScalar:
102 LLVM_DEBUG(dbgs() << ".. Widen scalar\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Widen scalar\n"; } } while
(false)
;
103 return widenScalar(MI, Step.TypeIdx, Step.NewType);
104 case Lower:
105 LLVM_DEBUG(dbgs() << ".. Lower\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Lower\n"; } } while (false
)
;
106 return lower(MI, Step.TypeIdx, Step.NewType);
107 case FewerElements:
108 LLVM_DEBUG(dbgs() << ".. Reduce number of elements\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Reduce number of elements\n"
; } } while (false)
;
109 return fewerElementsVector(MI, Step.TypeIdx, Step.NewType);
110 case MoreElements:
111 LLVM_DEBUG(dbgs() << ".. Increase number of elements\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Increase number of elements\n"
; } } while (false)
;
112 return moreElementsVector(MI, Step.TypeIdx, Step.NewType);
113 case Custom:
114 LLVM_DEBUG(dbgs() << ".. Custom legalization\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Custom legalization\n"; }
} while (false)
;
115 return LI.legalizeCustom(MI, MRI, MIRBuilder, Observer) ? Legalized
116 : UnableToLegalize;
117 default:
118 LLVM_DEBUG(dbgs() << ".. Unable to legalize\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Unable to legalize\n"; }
} while (false)
;
119 return UnableToLegalize;
120 }
121}
122
123void LegalizerHelper::extractParts(Register Reg, LLT Ty, int NumParts,
124 SmallVectorImpl<Register> &VRegs) {
125 for (int i = 0; i < NumParts; ++i)
126 VRegs.push_back(MRI.createGenericVirtualRegister(Ty));
127 MIRBuilder.buildUnmerge(VRegs, Reg);
128}
129
130bool LegalizerHelper::extractParts(Register Reg, LLT RegTy,
131 LLT MainTy, LLT &LeftoverTy,
132 SmallVectorImpl<Register> &VRegs,
133 SmallVectorImpl<Register> &LeftoverRegs) {
134 assert(!LeftoverTy.isValid() && "this is an out argument")((!LeftoverTy.isValid() && "this is an out argument")
? static_cast<void> (0) : __assert_fail ("!LeftoverTy.isValid() && \"this is an out argument\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 134, __PRETTY_FUNCTION__))
;
135
136 unsigned RegSize = RegTy.getSizeInBits();
137 unsigned MainSize = MainTy.getSizeInBits();
138 unsigned NumParts = RegSize / MainSize;
139 unsigned LeftoverSize = RegSize - NumParts * MainSize;
140
141 // Use an unmerge when possible.
142 if (LeftoverSize == 0) {
143 for (unsigned I = 0; I < NumParts; ++I)
144 VRegs.push_back(MRI.createGenericVirtualRegister(MainTy));
145 MIRBuilder.buildUnmerge(VRegs, Reg);
146 return true;
147 }
148
149 if (MainTy.isVector()) {
150 unsigned EltSize = MainTy.getScalarSizeInBits();
151 if (LeftoverSize % EltSize != 0)
152 return false;
153 LeftoverTy = LLT::scalarOrVector(LeftoverSize / EltSize, EltSize);
154 } else {
155 LeftoverTy = LLT::scalar(LeftoverSize);
156 }
157
158 // For irregular sizes, extract the individual parts.
159 for (unsigned I = 0; I != NumParts; ++I) {
160 Register NewReg = MRI.createGenericVirtualRegister(MainTy);
161 VRegs.push_back(NewReg);
162 MIRBuilder.buildExtract(NewReg, Reg, MainSize * I);
163 }
164
165 for (unsigned Offset = MainSize * NumParts; Offset < RegSize;
166 Offset += LeftoverSize) {
167 Register NewReg = MRI.createGenericVirtualRegister(LeftoverTy);
168 LeftoverRegs.push_back(NewReg);
169 MIRBuilder.buildExtract(NewReg, Reg, Offset);
170 }
171
172 return true;
173}
174
175static LLT getGCDType(LLT OrigTy, LLT TargetTy) {
176 if (OrigTy.isVector() && TargetTy.isVector()) {
177 assert(OrigTy.getElementType() == TargetTy.getElementType())((OrigTy.getElementType() == TargetTy.getElementType()) ? static_cast
<void> (0) : __assert_fail ("OrigTy.getElementType() == TargetTy.getElementType()"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 177, __PRETTY_FUNCTION__))
;
178 int GCD = greatestCommonDivisor(OrigTy.getNumElements(),
179 TargetTy.getNumElements());
180 return LLT::scalarOrVector(GCD, OrigTy.getElementType());
181 }
182
183 if (OrigTy.isVector() && !TargetTy.isVector()) {
184 assert(OrigTy.getElementType() == TargetTy)((OrigTy.getElementType() == TargetTy) ? static_cast<void>
(0) : __assert_fail ("OrigTy.getElementType() == TargetTy", "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 184, __PRETTY_FUNCTION__))
;
185 return TargetTy;
186 }
187
188 assert(!OrigTy.isVector() && !TargetTy.isVector())((!OrigTy.isVector() && !TargetTy.isVector()) ? static_cast
<void> (0) : __assert_fail ("!OrigTy.isVector() && !TargetTy.isVector()"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 188, __PRETTY_FUNCTION__))
;
189
190 int GCD = greatestCommonDivisor(OrigTy.getSizeInBits(),
191 TargetTy.getSizeInBits());
192 return LLT::scalar(GCD);
193}
194
195void LegalizerHelper::insertParts(Register DstReg,
196 LLT ResultTy, LLT PartTy,
197 ArrayRef<Register> PartRegs,
198 LLT LeftoverTy,
199 ArrayRef<Register> LeftoverRegs) {
200 if (!LeftoverTy.isValid()) {
201 assert(LeftoverRegs.empty())((LeftoverRegs.empty()) ? static_cast<void> (0) : __assert_fail
("LeftoverRegs.empty()", "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 201, __PRETTY_FUNCTION__))
;
202
203 if (!ResultTy.isVector()) {
204 MIRBuilder.buildMerge(DstReg, PartRegs);
205 return;
206 }
207
208 if (PartTy.isVector())
209 MIRBuilder.buildConcatVectors(DstReg, PartRegs);
210 else
211 MIRBuilder.buildBuildVector(DstReg, PartRegs);
212 return;
213 }
214
215 unsigned PartSize = PartTy.getSizeInBits();
216 unsigned LeftoverPartSize = LeftoverTy.getSizeInBits();
217
218 Register CurResultReg = MRI.createGenericVirtualRegister(ResultTy);
219 MIRBuilder.buildUndef(CurResultReg);
220
221 unsigned Offset = 0;
222 for (Register PartReg : PartRegs) {
223 Register NewResultReg = MRI.createGenericVirtualRegister(ResultTy);
224 MIRBuilder.buildInsert(NewResultReg, CurResultReg, PartReg, Offset);
225 CurResultReg = NewResultReg;
226 Offset += PartSize;
227 }
228
229 for (unsigned I = 0, E = LeftoverRegs.size(); I != E; ++I) {
230 // Use the original output register for the final insert to avoid a copy.
231 Register NewResultReg = (I + 1 == E) ?
232 DstReg : MRI.createGenericVirtualRegister(ResultTy);
233
234 MIRBuilder.buildInsert(NewResultReg, CurResultReg, LeftoverRegs[I], Offset);
235 CurResultReg = NewResultReg;
236 Offset += LeftoverPartSize;
237 }
238}
239
240static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
241 switch (Opcode) {
242 case TargetOpcode::G_SDIV:
243 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size")(((Size == 32 || Size == 64 || Size == 128) && "Unsupported size"
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64 || Size == 128) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 243, __PRETTY_FUNCTION__))
;
244 switch (Size) {
245 case 32:
246 return RTLIB::SDIV_I32;
247 case 64:
248 return RTLIB::SDIV_I64;
249 case 128:
250 return RTLIB::SDIV_I128;
251 default:
252 llvm_unreachable("unexpected size")::llvm::llvm_unreachable_internal("unexpected size", "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 252)
;
253 }
254 case TargetOpcode::G_UDIV:
255 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size")(((Size == 32 || Size == 64 || Size == 128) && "Unsupported size"
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64 || Size == 128) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 255, __PRETTY_FUNCTION__))
;
256 switch (Size) {
257 case 32:
258 return RTLIB::UDIV_I32;
259 case 64:
260 return RTLIB::UDIV_I64;
261 case 128:
262 return RTLIB::UDIV_I128;
263 default:
264 llvm_unreachable("unexpected size")::llvm::llvm_unreachable_internal("unexpected size", "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 264)
;
265 }
266 case TargetOpcode::G_SREM:
267 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 267, __PRETTY_FUNCTION__))
;
268 return Size == 64 ? RTLIB::SREM_I64 : RTLIB::SREM_I32;
269 case TargetOpcode::G_UREM:
270 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 270, __PRETTY_FUNCTION__))
;
271 return Size == 64 ? RTLIB::UREM_I64 : RTLIB::UREM_I32;
272 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
273 assert(Size == 32 && "Unsupported size")((Size == 32 && "Unsupported size") ? static_cast<
void> (0) : __assert_fail ("Size == 32 && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 273, __PRETTY_FUNCTION__))
;
274 return RTLIB::CTLZ_I32;
275 case TargetOpcode::G_FADD:
276 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 276, __PRETTY_FUNCTION__))
;
277 return Size == 64 ? RTLIB::ADD_F64 : RTLIB::ADD_F32;
278 case TargetOpcode::G_FSUB:
279 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 279, __PRETTY_FUNCTION__))
;
280 return Size == 64 ? RTLIB::SUB_F64 : RTLIB::SUB_F32;
281 case TargetOpcode::G_FMUL:
282 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 282, __PRETTY_FUNCTION__))
;
283 return Size == 64 ? RTLIB::MUL_F64 : RTLIB::MUL_F32;
284 case TargetOpcode::G_FDIV:
285 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 285, __PRETTY_FUNCTION__))
;
286 return Size == 64 ? RTLIB::DIV_F64 : RTLIB::DIV_F32;
287 case TargetOpcode::G_FEXP:
288 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 288, __PRETTY_FUNCTION__))
;
289 return Size == 64 ? RTLIB::EXP_F64 : RTLIB::EXP_F32;
290 case TargetOpcode::G_FEXP2:
291 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 291, __PRETTY_FUNCTION__))
;
292 return Size == 64 ? RTLIB::EXP2_F64 : RTLIB::EXP2_F32;
293 case TargetOpcode::G_FREM:
294 return Size == 64 ? RTLIB::REM_F64 : RTLIB::REM_F32;
295 case TargetOpcode::G_FPOW:
296 return Size == 64 ? RTLIB::POW_F64 : RTLIB::POW_F32;
297 case TargetOpcode::G_FMA:
298 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 298, __PRETTY_FUNCTION__))
;
299 return Size == 64 ? RTLIB::FMA_F64 : RTLIB::FMA_F32;
300 case TargetOpcode::G_FSIN:
301 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size")(((Size == 32 || Size == 64 || Size == 128) && "Unsupported size"
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64 || Size == 128) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 301, __PRETTY_FUNCTION__))
;
302 return Size == 128 ? RTLIB::SIN_F128
303 : Size == 64 ? RTLIB::SIN_F64 : RTLIB::SIN_F32;
304 case TargetOpcode::G_FCOS:
305 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size")(((Size == 32 || Size == 64 || Size == 128) && "Unsupported size"
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64 || Size == 128) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 305, __PRETTY_FUNCTION__))
;
306 return Size == 128 ? RTLIB::COS_F128
307 : Size == 64 ? RTLIB::COS_F64 : RTLIB::COS_F32;
308 case TargetOpcode::G_FLOG10:
309 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size")(((Size == 32 || Size == 64 || Size == 128) && "Unsupported size"
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64 || Size == 128) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 309, __PRETTY_FUNCTION__))
;
310 return Size == 128 ? RTLIB::LOG10_F128
311 : Size == 64 ? RTLIB::LOG10_F64 : RTLIB::LOG10_F32;
312 case TargetOpcode::G_FLOG:
313 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size")(((Size == 32 || Size == 64 || Size == 128) && "Unsupported size"
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64 || Size == 128) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 313, __PRETTY_FUNCTION__))
;
314 return Size == 128 ? RTLIB::LOG_F128
315 : Size == 64 ? RTLIB::LOG_F64 : RTLIB::LOG_F32;
316 case TargetOpcode::G_FLOG2:
317 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size")(((Size == 32 || Size == 64 || Size == 128) && "Unsupported size"
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64 || Size == 128) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 317, __PRETTY_FUNCTION__))
;
318 return Size == 128 ? RTLIB::LOG2_F128
319 : Size == 64 ? RTLIB::LOG2_F64 : RTLIB::LOG2_F32;
320 case TargetOpcode::G_FCEIL:
321 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 321, __PRETTY_FUNCTION__))
;
322 return Size == 64 ? RTLIB::CEIL_F64 : RTLIB::CEIL_F32;
323 case TargetOpcode::G_FFLOOR:
324 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 324, __PRETTY_FUNCTION__))
;
325 return Size == 64 ? RTLIB::FLOOR_F64 : RTLIB::FLOOR_F32;
326 }
327 llvm_unreachable("Unknown libcall function")::llvm::llvm_unreachable_internal("Unknown libcall function",
"/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 327)
;
328}
329
330/// True if an instruction is in tail position in its caller. Intended for
331/// legalizing libcalls as tail calls when possible.
332static bool isLibCallInTailPosition(MachineInstr &MI) {
333 const Function &F = MI.getParent()->getParent()->getFunction();
334
335 // Conservatively require the attributes of the call to match those of
336 // the return. Ignore NoAlias and NonNull because they don't affect the
337 // call sequence.
338 AttributeList CallerAttrs = F.getAttributes();
339 if (AttrBuilder(CallerAttrs, AttributeList::ReturnIndex)
340 .removeAttribute(Attribute::NoAlias)
341 .removeAttribute(Attribute::NonNull)
342 .hasAttributes())
343 return false;
344
345 // It's not safe to eliminate the sign / zero extension of the return value.
346 if (CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::ZExt) ||
347 CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::SExt))
348 return false;
349
350 // Only tail call if the following instruction is a standard return.
351 auto &TII = *MI.getMF()->getSubtarget().getInstrInfo();
352 MachineInstr *Next = MI.getNextNode();
353 if (!Next || TII.isTailCall(*Next) || !Next->isReturn())
354 return false;
355
356 return true;
357}
358
359LegalizerHelper::LegalizeResult
360llvm::createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall,
361 const CallLowering::ArgInfo &Result,
362 ArrayRef<CallLowering::ArgInfo> Args) {
363 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
364 auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
365 const char *Name = TLI.getLibcallName(Libcall);
366
367 CallLowering::CallLoweringInfo Info;
368 Info.CallConv = TLI.getLibcallCallingConv(Libcall);
369 Info.Callee = MachineOperand::CreateES(Name);
370 Info.OrigRet = Result;
371 std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs));
372 if (!CLI.lowerCall(MIRBuilder, Info))
373 return LegalizerHelper::UnableToLegalize;
374
375 return LegalizerHelper::Legalized;
376}
377
378// Useful for libcalls where all operands have the same type.
379static LegalizerHelper::LegalizeResult
380simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size,
381 Type *OpType) {
382 auto Libcall = getRTLibDesc(MI.getOpcode(), Size);
383
384 SmallVector<CallLowering::ArgInfo, 3> Args;
385 for (unsigned i = 1; i < MI.getNumOperands(); i++)
386 Args.push_back({MI.getOperand(i).getReg(), OpType});
387 return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), OpType},
388 Args);
389}
390
391LegalizerHelper::LegalizeResult
392llvm::createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
393 MachineInstr &MI) {
394 assert(MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS)((MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS)
? static_cast<void> (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 394, __PRETTY_FUNCTION__))
;
395 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
396
397 SmallVector<CallLowering::ArgInfo, 3> Args;
398 // Add all the args, except for the last which is an imm denoting 'tail'.
399 for (unsigned i = 1; i < MI.getNumOperands() - 1; i++) {
400 Register Reg = MI.getOperand(i).getReg();
401
402 // Need derive an IR type for call lowering.
403 LLT OpLLT = MRI.getType(Reg);
404 Type *OpTy = nullptr;
405 if (OpLLT.isPointer())
406 OpTy = Type::getInt8PtrTy(Ctx, OpLLT.getAddressSpace());
407 else
408 OpTy = IntegerType::get(Ctx, OpLLT.getSizeInBits());
409 Args.push_back({Reg, OpTy});
410 }
411
412 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
413 auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
414 Intrinsic::ID ID = MI.getOperand(0).getIntrinsicID();
415 RTLIB::Libcall RTLibcall;
416 switch (ID) {
417 case Intrinsic::memcpy:
418 RTLibcall = RTLIB::MEMCPY;
419 break;
420 case Intrinsic::memset:
421 RTLibcall = RTLIB::MEMSET;
422 break;
423 case Intrinsic::memmove:
424 RTLibcall = RTLIB::MEMMOVE;
425 break;
426 default:
427 return LegalizerHelper::UnableToLegalize;
428 }
429 const char *Name = TLI.getLibcallName(RTLibcall);
430
431 MIRBuilder.setInstr(MI);
432
433 CallLowering::CallLoweringInfo Info;
434 Info.CallConv = TLI.getLibcallCallingConv(RTLibcall);
435 Info.Callee = MachineOperand::CreateES(Name);
436 Info.OrigRet = CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx));
437 Info.IsTailCall = MI.getOperand(MI.getNumOperands() - 1).getImm() == 1 &&
438 isLibCallInTailPosition(MI);
439
440 std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs));
441 if (!CLI.lowerCall(MIRBuilder, Info))
442 return LegalizerHelper::UnableToLegalize;
443
444 if (Info.LoweredTailCall) {
445 assert(Info.IsTailCall && "Lowered tail call when it wasn't a tail call?")((Info.IsTailCall && "Lowered tail call when it wasn't a tail call?"
) ? static_cast<void> (0) : __assert_fail ("Info.IsTailCall && \"Lowered tail call when it wasn't a tail call?\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 445, __PRETTY_FUNCTION__))
;
446 // We must have a return following the call to get past
447 // isLibCallInTailPosition.
448 assert(MI.getNextNode() && MI.getNextNode()->isReturn() &&((MI.getNextNode() && MI.getNextNode()->isReturn()
&& "Expected instr following MI to be a return?") ? static_cast
<void> (0) : __assert_fail ("MI.getNextNode() && MI.getNextNode()->isReturn() && \"Expected instr following MI to be a return?\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 449, __PRETTY_FUNCTION__))
449 "Expected instr following MI to be a return?")((MI.getNextNode() && MI.getNextNode()->isReturn()
&& "Expected instr following MI to be a return?") ? static_cast
<void> (0) : __assert_fail ("MI.getNextNode() && MI.getNextNode()->isReturn() && \"Expected instr following MI to be a return?\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 449, __PRETTY_FUNCTION__))
;
450
451 // We lowered a tail call, so the call is now the return from the block.
452 // Delete the old return.
453 MI.getNextNode()->eraseFromParent();
454 }
455
456 return LegalizerHelper::Legalized;
457}
458
459static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType,
460 Type *FromType) {
461 auto ToMVT = MVT::getVT(ToType);
462 auto FromMVT = MVT::getVT(FromType);
463
464 switch (Opcode) {
465 case TargetOpcode::G_FPEXT:
466 return RTLIB::getFPEXT(FromMVT, ToMVT);
467 case TargetOpcode::G_FPTRUNC:
468 return RTLIB::getFPROUND(FromMVT, ToMVT);
469 case TargetOpcode::G_FPTOSI:
470 return RTLIB::getFPTOSINT(FromMVT, ToMVT);
471 case TargetOpcode::G_FPTOUI:
472 return RTLIB::getFPTOUINT(FromMVT, ToMVT);
473 case TargetOpcode::G_SITOFP:
474 return RTLIB::getSINTTOFP(FromMVT, ToMVT);
475 case TargetOpcode::G_UITOFP:
476 return RTLIB::getUINTTOFP(FromMVT, ToMVT);
477 }
478 llvm_unreachable("Unsupported libcall function")::llvm::llvm_unreachable_internal("Unsupported libcall function"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 478)
;
479}
480
481static LegalizerHelper::LegalizeResult
482conversionLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, Type *ToType,
483 Type *FromType) {
484 RTLIB::Libcall Libcall = getConvRTLibDesc(MI.getOpcode(), ToType, FromType);
485 return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), ToType},
486 {{MI.getOperand(1).getReg(), FromType}});
487}
488
489LegalizerHelper::LegalizeResult
490LegalizerHelper::libcall(MachineInstr &MI) {
491 LLT LLTy = MRI.getType(MI.getOperand(0).getReg());
492 unsigned Size = LLTy.getSizeInBits();
493 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
494
495 MIRBuilder.setInstr(MI);
496
497 switch (MI.getOpcode()) {
498 default:
499 return UnableToLegalize;
500 case TargetOpcode::G_SDIV:
501 case TargetOpcode::G_UDIV:
502 case TargetOpcode::G_SREM:
503 case TargetOpcode::G_UREM:
504 case TargetOpcode::G_CTLZ_ZERO_UNDEF: {
505 Type *HLTy = IntegerType::get(Ctx, Size);
506 auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
507 if (Status != Legalized)
508 return Status;
509 break;
510 }
511 case TargetOpcode::G_FADD:
512 case TargetOpcode::G_FSUB:
513 case TargetOpcode::G_FMUL:
514 case TargetOpcode::G_FDIV:
515 case TargetOpcode::G_FMA:
516 case TargetOpcode::G_FPOW:
517 case TargetOpcode::G_FREM:
518 case TargetOpcode::G_FCOS:
519 case TargetOpcode::G_FSIN:
520 case TargetOpcode::G_FLOG10:
521 case TargetOpcode::G_FLOG:
522 case TargetOpcode::G_FLOG2:
523 case TargetOpcode::G_FEXP:
524 case TargetOpcode::G_FEXP2:
525 case TargetOpcode::G_FCEIL:
526 case TargetOpcode::G_FFLOOR: {
527 if (Size > 64) {
528 LLVM_DEBUG(dbgs() << "Size " << Size << " too large to legalize.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << "Size " << Size <<
" too large to legalize.\n"; } } while (false)
;
529 return UnableToLegalize;
530 }
531 Type *HLTy = Size == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx);
532 auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
533 if (Status != Legalized)
534 return Status;
535 break;
536 }
537 case TargetOpcode::G_FPEXT: {
538 // FIXME: Support other floating point types (half, fp128 etc)
539 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
540 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
541 if (ToSize != 64 || FromSize != 32)
542 return UnableToLegalize;
543 LegalizeResult Status = conversionLibcall(
544 MI, MIRBuilder, Type::getDoubleTy(Ctx), Type::getFloatTy(Ctx));
545 if (Status != Legalized)
546 return Status;
547 break;
548 }
549 case TargetOpcode::G_FPTRUNC: {
550 // FIXME: Support other floating point types (half, fp128 etc)
551 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
552 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
553 if (ToSize != 32 || FromSize != 64)
554 return UnableToLegalize;
555 LegalizeResult Status = conversionLibcall(
556 MI, MIRBuilder, Type::getFloatTy(Ctx), Type::getDoubleTy(Ctx));
557 if (Status != Legalized)
558 return Status;
559 break;
560 }
561 case TargetOpcode::G_FPTOSI:
562 case TargetOpcode::G_FPTOUI: {
563 // FIXME: Support other types
564 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
565 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
566 if ((ToSize != 32 && ToSize != 64) || (FromSize != 32 && FromSize != 64))
567 return UnableToLegalize;
568 LegalizeResult Status = conversionLibcall(
569 MI, MIRBuilder,
570 ToSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx),
571 FromSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx));
572 if (Status != Legalized)
573 return Status;
574 break;
575 }
576 case TargetOpcode::G_SITOFP:
577 case TargetOpcode::G_UITOFP: {
578 // FIXME: Support other types
579 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
580 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
581 if ((FromSize != 32 && FromSize != 64) || (ToSize != 32 && ToSize != 64))
582 return UnableToLegalize;
583 LegalizeResult Status = conversionLibcall(
584 MI, MIRBuilder,
585 ToSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx),
586 FromSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx));
587 if (Status != Legalized)
588 return Status;
589 break;
590 }
591 }
592
593 MI.eraseFromParent();
594 return Legalized;
595}
596
597LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
598 unsigned TypeIdx,
599 LLT NarrowTy) {
600 MIRBuilder.setInstr(MI);
601
602 uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
603 uint64_t NarrowSize = NarrowTy.getSizeInBits();
604
605 switch (MI.getOpcode()) {
606 default:
607 return UnableToLegalize;
608 case TargetOpcode::G_IMPLICIT_DEF: {
609 // FIXME: add support for when SizeOp0 isn't an exact multiple of
610 // NarrowSize.
611 if (SizeOp0 % NarrowSize != 0)
612 return UnableToLegalize;
613 int NumParts = SizeOp0 / NarrowSize;
614
615 SmallVector<Register, 2> DstRegs;
616 for (int i = 0; i < NumParts; ++i)
617 DstRegs.push_back(
618 MIRBuilder.buildUndef(NarrowTy)->getOperand(0).getReg());
619
620 Register DstReg = MI.getOperand(0).getReg();
621 if(MRI.getType(DstReg).isVector())
622 MIRBuilder.buildBuildVector(DstReg, DstRegs);
623 else
624 MIRBuilder.buildMerge(DstReg, DstRegs);
625 MI.eraseFromParent();
626 return Legalized;
627 }
628 case TargetOpcode::G_CONSTANT: {
629 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
630 const APInt &Val = MI.getOperand(1).getCImm()->getValue();
631 unsigned TotalSize = Ty.getSizeInBits();
632 unsigned NarrowSize = NarrowTy.getSizeInBits();
633 int NumParts = TotalSize / NarrowSize;
634
635 SmallVector<Register, 4> PartRegs;
636 for (int I = 0; I != NumParts; ++I) {
637 unsigned Offset = I * NarrowSize;
638 auto K = MIRBuilder.buildConstant(NarrowTy,
639 Val.lshr(Offset).trunc(NarrowSize));
640 PartRegs.push_back(K.getReg(0));
641 }
642
643 LLT LeftoverTy;
644 unsigned LeftoverBits = TotalSize - NumParts * NarrowSize;
645 SmallVector<Register, 1> LeftoverRegs;
646 if (LeftoverBits != 0) {
647 LeftoverTy = LLT::scalar(LeftoverBits);
648 auto K = MIRBuilder.buildConstant(
649 LeftoverTy,
650 Val.lshr(NumParts * NarrowSize).trunc(LeftoverBits));
651 LeftoverRegs.push_back(K.getReg(0));
652 }
653
654 insertParts(MI.getOperand(0).getReg(),
655 Ty, NarrowTy, PartRegs, LeftoverTy, LeftoverRegs);
656
657 MI.eraseFromParent();
658 return Legalized;
659 }
660 case TargetOpcode::G_SEXT: {
661 if (TypeIdx != 0)
662 return UnableToLegalize;
663
664 Register SrcReg = MI.getOperand(1).getReg();
665 LLT SrcTy = MRI.getType(SrcReg);
666
667 // FIXME: support the general case where the requested NarrowTy may not be
668 // the same as the source type. E.g. s128 = sext(s32)
669 if ((SrcTy.getSizeInBits() != SizeOp0 / 2) ||
670 SrcTy.getSizeInBits() != NarrowTy.getSizeInBits()) {
671 LLVM_DEBUG(dbgs() << "Can't narrow sext to type " << NarrowTy << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << "Can't narrow sext to type "
<< NarrowTy << "\n"; } } while (false)
;
672 return UnableToLegalize;
673 }
674
675 // Shift the sign bit of the low register through the high register.
676 auto ShiftAmt =
677 MIRBuilder.buildConstant(LLT::scalar(64), NarrowTy.getSizeInBits() - 1);
678 auto Shift = MIRBuilder.buildAShr(NarrowTy, SrcReg, ShiftAmt);
679 MIRBuilder.buildMerge(MI.getOperand(0).getReg(), {SrcReg, Shift.getReg(0)});
680 MI.eraseFromParent();
681 return Legalized;
682 }
683 case TargetOpcode::G_ZEXT: {
684 if (TypeIdx != 0)
685 return UnableToLegalize;
686
687 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
688 uint64_t SizeOp1 = SrcTy.getSizeInBits();
689 if (SizeOp0 % SizeOp1 != 0)
690 return UnableToLegalize;
691
692 // Generate a merge where the bottom bits are taken from the source, and
693 // zero everything else.
694 Register ZeroReg = MIRBuilder.buildConstant(SrcTy, 0).getReg(0);
695 unsigned NumParts = SizeOp0 / SizeOp1;
696 SmallVector<Register, 4> Srcs = {MI.getOperand(1).getReg()};
697 for (unsigned Part = 1; Part < NumParts; ++Part)
698 Srcs.push_back(ZeroReg);
699 MIRBuilder.buildMerge(MI.getOperand(0).getReg(), Srcs);
700 MI.eraseFromParent();
701 return Legalized;
702 }
703 case TargetOpcode::G_TRUNC: {
704 if (TypeIdx != 1)
705 return UnableToLegalize;
706
707 uint64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
708 if (NarrowTy.getSizeInBits() * 2 != SizeOp1) {
709 LLVM_DEBUG(dbgs() << "Can't narrow trunc to type " << NarrowTy << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << "Can't narrow trunc to type "
<< NarrowTy << "\n"; } } while (false)
;
710 return UnableToLegalize;
711 }
712
713 auto Unmerge = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1).getReg());
714 MIRBuilder.buildCopy(MI.getOperand(0).getReg(), Unmerge.getReg(0));
715 MI.eraseFromParent();
716 return Legalized;
717 }
718
719 case TargetOpcode::G_ADD: {
720 // FIXME: add support for when SizeOp0 isn't an exact multiple of
721 // NarrowSize.
722 if (SizeOp0 % NarrowSize != 0)
723 return UnableToLegalize;
724 // Expand in terms of carry-setting/consuming G_ADDE instructions.
725 int NumParts = SizeOp0 / NarrowTy.getSizeInBits();
726
727 SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs;
728 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
729 extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
730
731 Register CarryIn;
732 for (int i = 0; i < NumParts; ++i) {
733 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy);
734 Register CarryOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
735
736 if (i == 0)
737 MIRBuilder.buildUAddo(DstReg, CarryOut, Src1Regs[i], Src2Regs[i]);
738 else {
739 MIRBuilder.buildUAdde(DstReg, CarryOut, Src1Regs[i],
740 Src2Regs[i], CarryIn);
741 }
742
743 DstRegs.push_back(DstReg);
744 CarryIn = CarryOut;
745 }
746 Register DstReg = MI.getOperand(0).getReg();
747 if(MRI.getType(DstReg).isVector())
748 MIRBuilder.buildBuildVector(DstReg, DstRegs);
749 else
750 MIRBuilder.buildMerge(DstReg, DstRegs);
751 MI.eraseFromParent();
752 return Legalized;
753 }
754 case TargetOpcode::G_SUB: {
755 // FIXME: add support for when SizeOp0 isn't an exact multiple of
756 // NarrowSize.
757 if (SizeOp0 % NarrowSize != 0)
758 return UnableToLegalize;
759
760 int NumParts = SizeOp0 / NarrowTy.getSizeInBits();
761
762 SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs;
763 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
764 extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
765
766 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy);
767 Register BorrowOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
768 MIRBuilder.buildInstr(TargetOpcode::G_USUBO, {DstReg, BorrowOut},
769 {Src1Regs[0], Src2Regs[0]});
770 DstRegs.push_back(DstReg);
771 Register BorrowIn = BorrowOut;
772 for (int i = 1; i < NumParts; ++i) {
773 DstReg = MRI.createGenericVirtualRegister(NarrowTy);
774 BorrowOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
775
776 MIRBuilder.buildInstr(TargetOpcode::G_USUBE, {DstReg, BorrowOut},
777 {Src1Regs[i], Src2Regs[i], BorrowIn});
778
779 DstRegs.push_back(DstReg);
780 BorrowIn = BorrowOut;
781 }
782 MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
783 MI.eraseFromParent();
784 return Legalized;
785 }
786 case TargetOpcode::G_MUL:
787 case TargetOpcode::G_UMULH:
788 return narrowScalarMul(MI, NarrowTy);
789 case TargetOpcode::G_EXTRACT:
790 return narrowScalarExtract(MI, TypeIdx, NarrowTy);
791 case TargetOpcode::G_INSERT:
792 return narrowScalarInsert(MI, TypeIdx, NarrowTy);
793 case TargetOpcode::G_LOAD: {
794 const auto &MMO = **MI.memoperands_begin();
795 Register DstReg = MI.getOperand(0).getReg();
796 LLT DstTy = MRI.getType(DstReg);
797 if (DstTy.isVector())
798 return UnableToLegalize;
799
800 if (8 * MMO.getSize() != DstTy.getSizeInBits()) {
801 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
802 auto &MMO = **MI.memoperands_begin();
803 MIRBuilder.buildLoad(TmpReg, MI.getOperand(1).getReg(), MMO);
804 MIRBuilder.buildAnyExt(DstReg, TmpReg);
805 MI.eraseFromParent();
806 return Legalized;
807 }
808
809 return reduceLoadStoreWidth(MI, TypeIdx, NarrowTy);
810 }
811 case TargetOpcode::G_ZEXTLOAD:
812 case TargetOpcode::G_SEXTLOAD: {
813 bool ZExt = MI.getOpcode() == TargetOpcode::G_ZEXTLOAD;
814 Register DstReg = MI.getOperand(0).getReg();
815 Register PtrReg = MI.getOperand(1).getReg();
816
817 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
818 auto &MMO = **MI.memoperands_begin();
819 if (MMO.getSizeInBits() == NarrowSize) {
820 MIRBuilder.buildLoad(TmpReg, PtrReg, MMO);
821 } else {
822 unsigned ExtLoad = ZExt ? TargetOpcode::G_ZEXTLOAD
823 : TargetOpcode::G_SEXTLOAD;
824 MIRBuilder.buildInstr(ExtLoad)
825 .addDef(TmpReg)
826 .addUse(PtrReg)
827 .addMemOperand(&MMO);
828 }
829
830 if (ZExt)
831 MIRBuilder.buildZExt(DstReg, TmpReg);
832 else
833 MIRBuilder.buildSExt(DstReg, TmpReg);
834
835 MI.eraseFromParent();
836 return Legalized;
837 }
838 case TargetOpcode::G_STORE: {
839 const auto &MMO = **MI.memoperands_begin();
840
841 Register SrcReg = MI.getOperand(0).getReg();
842 LLT SrcTy = MRI.getType(SrcReg);
843 if (SrcTy.isVector())
844 return UnableToLegalize;
845
846 int NumParts = SizeOp0 / NarrowSize;
847 unsigned HandledSize = NumParts * NarrowTy.getSizeInBits();
848 unsigned LeftoverBits = SrcTy.getSizeInBits() - HandledSize;
849 if (SrcTy.isVector() && LeftoverBits != 0)
850 return UnableToLegalize;
851
852 if (8 * MMO.getSize() != SrcTy.getSizeInBits()) {
853 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
854 auto &MMO = **MI.memoperands_begin();
855 MIRBuilder.buildTrunc(TmpReg, SrcReg);
856 MIRBuilder.buildStore(TmpReg, MI.getOperand(1).getReg(), MMO);
857 MI.eraseFromParent();
858 return Legalized;
859 }
860
861 return reduceLoadStoreWidth(MI, 0, NarrowTy);
862 }
863 case TargetOpcode::G_SELECT:
864 return narrowScalarSelect(MI, TypeIdx, NarrowTy);
865 case TargetOpcode::G_AND:
866 case TargetOpcode::G_OR:
867 case TargetOpcode::G_XOR: {
868 // Legalize bitwise operation:
869 // A = BinOp<Ty> B, C
870 // into:
871 // B1, ..., BN = G_UNMERGE_VALUES B
872 // C1, ..., CN = G_UNMERGE_VALUES C
873 // A1 = BinOp<Ty/N> B1, C2
874 // ...
875 // AN = BinOp<Ty/N> BN, CN
876 // A = G_MERGE_VALUES A1, ..., AN
877 return narrowScalarBasic(MI, TypeIdx, NarrowTy);
878 }
879 case TargetOpcode::G_SHL:
880 case TargetOpcode::G_LSHR:
881 case TargetOpcode::G_ASHR:
882 return narrowScalarShift(MI, TypeIdx, NarrowTy);
883 case TargetOpcode::G_CTLZ:
884 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
885 case TargetOpcode::G_CTTZ:
886 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
887 case TargetOpcode::G_CTPOP:
888 if (TypeIdx != 0)
889 return UnableToLegalize; // TODO
890
891 Observer.changingInstr(MI);
892 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT);
893 Observer.changedInstr(MI);
894 return Legalized;
895 case TargetOpcode::G_INTTOPTR:
896 if (TypeIdx != 1)
897 return UnableToLegalize;
898
899 Observer.changingInstr(MI);
900 narrowScalarSrc(MI, NarrowTy, 1);
901 Observer.changedInstr(MI);
902 return Legalized;
903 case TargetOpcode::G_PTRTOINT:
904 if (TypeIdx != 0)
905 return UnableToLegalize;
906
907 Observer.changingInstr(MI);
908 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT);
909 Observer.changedInstr(MI);
910 return Legalized;
911 case TargetOpcode::G_PHI: {
912 unsigned NumParts = SizeOp0 / NarrowSize;
913 SmallVector<Register, 2> DstRegs;
914 SmallVector<SmallVector<Register, 2>, 2> SrcRegs;
915 DstRegs.resize(NumParts);
916 SrcRegs.resize(MI.getNumOperands() / 2);
917 Observer.changingInstr(MI);
918 for (unsigned i = 1; i < MI.getNumOperands(); i += 2) {
919 MachineBasicBlock &OpMBB = *MI.getOperand(i + 1).getMBB();
920 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
921 extractParts(MI.getOperand(i).getReg(), NarrowTy, NumParts,
922 SrcRegs[i / 2]);
923 }
924 MachineBasicBlock &MBB = *MI.getParent();
925 MIRBuilder.setInsertPt(MBB, MI);
926 for (unsigned i = 0; i < NumParts; ++i) {
927 DstRegs[i] = MRI.createGenericVirtualRegister(NarrowTy);
928 MachineInstrBuilder MIB =
929 MIRBuilder.buildInstr(TargetOpcode::G_PHI).addDef(DstRegs[i]);
930 for (unsigned j = 1; j < MI.getNumOperands(); j += 2)
931 MIB.addUse(SrcRegs[j / 2][i]).add(MI.getOperand(j + 1));
932 }
933 MIRBuilder.setInsertPt(MBB, MBB.getFirstNonPHI());
934 MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
935 Observer.changedInstr(MI);
936 MI.eraseFromParent();
937 return Legalized;
938 }
939 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
940 case TargetOpcode::G_INSERT_VECTOR_ELT: {
941 if (TypeIdx != 2)
942 return UnableToLegalize;
943
944 int OpIdx = MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3;
945 Observer.changingInstr(MI);
946 narrowScalarSrc(MI, NarrowTy, OpIdx);
947 Observer.changedInstr(MI);
948 return Legalized;
949 }
950 case TargetOpcode::G_ICMP: {
951 uint64_t SrcSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
952 if (NarrowSize * 2 != SrcSize)
953 return UnableToLegalize;
954
955 Observer.changingInstr(MI);
956 Register LHSL = MRI.createGenericVirtualRegister(NarrowTy);
957 Register LHSH = MRI.createGenericVirtualRegister(NarrowTy);
958 MIRBuilder.buildUnmerge({LHSL, LHSH}, MI.getOperand(2).getReg());
959
960 Register RHSL = MRI.createGenericVirtualRegister(NarrowTy);
961 Register RHSH = MRI.createGenericVirtualRegister(NarrowTy);
962 MIRBuilder.buildUnmerge({RHSL, RHSH}, MI.getOperand(3).getReg());
963
964 CmpInst::Predicate Pred =
965 static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
966 LLT ResTy = MRI.getType(MI.getOperand(0).getReg());
967
968 if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE) {
969 MachineInstrBuilder XorL = MIRBuilder.buildXor(NarrowTy, LHSL, RHSL);
970 MachineInstrBuilder XorH = MIRBuilder.buildXor(NarrowTy, LHSH, RHSH);
971 MachineInstrBuilder Or = MIRBuilder.buildOr(NarrowTy, XorL, XorH);
972 MachineInstrBuilder Zero = MIRBuilder.buildConstant(NarrowTy, 0);
973 MIRBuilder.buildICmp(Pred, MI.getOperand(0).getReg(), Or, Zero);
974 } else {
975 MachineInstrBuilder CmpH = MIRBuilder.buildICmp(Pred, ResTy, LHSH, RHSH);
976 MachineInstrBuilder CmpHEQ =
977 MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, ResTy, LHSH, RHSH);
978 MachineInstrBuilder CmpLU = MIRBuilder.buildICmp(
979 ICmpInst::getUnsignedPredicate(Pred), ResTy, LHSL, RHSL);
980 MIRBuilder.buildSelect(MI.getOperand(0).getReg(), CmpHEQ, CmpLU, CmpH);
981 }
982 Observer.changedInstr(MI);
983 MI.eraseFromParent();
984 return Legalized;
985 }
986 case TargetOpcode::G_SEXT_INREG: {
987 if (TypeIdx != 0)
988 return UnableToLegalize;
989
990 if (!MI.getOperand(2).isImm())
991 return UnableToLegalize;
992 int64_t SizeInBits = MI.getOperand(2).getImm();
993
994 // So long as the new type has more bits than the bits we're extending we
995 // don't need to break it apart.
996 if (NarrowTy.getScalarSizeInBits() >= SizeInBits) {
997 Observer.changingInstr(MI);
998 // We don't lose any non-extension bits by truncating the src and
999 // sign-extending the dst.
1000 MachineOperand &MO1 = MI.getOperand(1);
1001 auto TruncMIB = MIRBuilder.buildTrunc(NarrowTy, MO1.getReg());
1002 MO1.setReg(TruncMIB->getOperand(0).getReg());
1003
1004 MachineOperand &MO2 = MI.getOperand(0);
1005 Register DstExt = MRI.createGenericVirtualRegister(NarrowTy);
1006 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1007 MIRBuilder.buildInstr(TargetOpcode::G_SEXT, {MO2.getReg()}, {DstExt});
1008 MO2.setReg(DstExt);
1009 Observer.changedInstr(MI);
1010 return Legalized;
1011 }
1012
1013 // Break it apart. Components below the extension point are unmodified. The
1014 // component containing the extension point becomes a narrower SEXT_INREG.
1015 // Components above it are ashr'd from the component containing the
1016 // extension point.
1017 if (SizeOp0 % NarrowSize != 0)
1018 return UnableToLegalize;
1019 int NumParts = SizeOp0 / NarrowSize;
1020
1021 // List the registers where the destination will be scattered.
1022 SmallVector<Register, 2> DstRegs;
1023 // List the registers where the source will be split.
1024 SmallVector<Register, 2> SrcRegs;
1025
1026 // Create all the temporary registers.
1027 for (int i = 0; i < NumParts; ++i) {
1028 Register SrcReg = MRI.createGenericVirtualRegister(NarrowTy);
1029
1030 SrcRegs.push_back(SrcReg);
1031 }
1032
1033 // Explode the big arguments into smaller chunks.
1034 MIRBuilder.buildUnmerge(SrcRegs, MI.getOperand(1).getReg());
1035
1036 Register AshrCstReg =
1037 MIRBuilder.buildConstant(NarrowTy, NarrowTy.getScalarSizeInBits() - 1)
1038 ->getOperand(0)
1039 .getReg();
1040 Register FullExtensionReg = 0;
1041 Register PartialExtensionReg = 0;
1042
1043 // Do the operation on each small part.
1044 for (int i = 0; i < NumParts; ++i) {
1045 if ((i + 1) * NarrowTy.getScalarSizeInBits() < SizeInBits)
1046 DstRegs.push_back(SrcRegs[i]);
1047 else if (i * NarrowTy.getScalarSizeInBits() > SizeInBits) {
1048 assert(PartialExtensionReg &&((PartialExtensionReg && "Expected to visit partial extension before full"
) ? static_cast<void> (0) : __assert_fail ("PartialExtensionReg && \"Expected to visit partial extension before full\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1049, __PRETTY_FUNCTION__))
1049 "Expected to visit partial extension before full")((PartialExtensionReg && "Expected to visit partial extension before full"
) ? static_cast<void> (0) : __assert_fail ("PartialExtensionReg && \"Expected to visit partial extension before full\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1049, __PRETTY_FUNCTION__))
;
1050 if (FullExtensionReg) {
1051 DstRegs.push_back(FullExtensionReg);
1052 continue;
1053 }
1054 DstRegs.push_back(MIRBuilder
1055 .buildInstr(TargetOpcode::G_ASHR, {NarrowTy},
1056 {PartialExtensionReg, AshrCstReg})
1057 ->getOperand(0)
1058 .getReg());
1059 FullExtensionReg = DstRegs.back();
1060 } else {
1061 DstRegs.push_back(
1062 MIRBuilder
1063 .buildInstr(
1064 TargetOpcode::G_SEXT_INREG, {NarrowTy},
1065 {SrcRegs[i], SizeInBits % NarrowTy.getScalarSizeInBits()})
1066 ->getOperand(0)
1067 .getReg());
1068 PartialExtensionReg = DstRegs.back();
1069 }
1070 }
1071
1072 // Gather the destination registers into the final destination.
1073 Register DstReg = MI.getOperand(0).getReg();
1074 MIRBuilder.buildMerge(DstReg, DstRegs);
1075 MI.eraseFromParent();
1076 return Legalized;
1077 }
1078 }
1079}
1080
1081void LegalizerHelper::widenScalarSrc(MachineInstr &MI, LLT WideTy,
1082 unsigned OpIdx, unsigned ExtOpcode) {
1083 MachineOperand &MO = MI.getOperand(OpIdx);
1084 auto ExtB = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MO.getReg()});
1085 MO.setReg(ExtB->getOperand(0).getReg());
1086}
1087
1088void LegalizerHelper::narrowScalarSrc(MachineInstr &MI, LLT NarrowTy,
1089 unsigned OpIdx) {
1090 MachineOperand &MO = MI.getOperand(OpIdx);
1091 auto ExtB = MIRBuilder.buildInstr(TargetOpcode::G_TRUNC, {NarrowTy},
1092 {MO.getReg()});
1093 MO.setReg(ExtB->getOperand(0).getReg());
1094}
1095
1096void LegalizerHelper::widenScalarDst(MachineInstr &MI, LLT WideTy,
1097 unsigned OpIdx, unsigned TruncOpcode) {
1098 MachineOperand &MO = MI.getOperand(OpIdx);
1099 Register DstExt = MRI.createGenericVirtualRegister(WideTy);
1100 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1101 MIRBuilder.buildInstr(TruncOpcode, {MO.getReg()}, {DstExt});
1102 MO.setReg(DstExt);
1103}
1104
1105void LegalizerHelper::narrowScalarDst(MachineInstr &MI, LLT NarrowTy,
1106 unsigned OpIdx, unsigned ExtOpcode) {
1107 MachineOperand &MO = MI.getOperand(OpIdx);
1108 Register DstTrunc = MRI.createGenericVirtualRegister(NarrowTy);
1109 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1110 MIRBuilder.buildInstr(ExtOpcode, {MO.getReg()}, {DstTrunc});
1111 MO.setReg(DstTrunc);
1112}
1113
1114void LegalizerHelper::moreElementsVectorDst(MachineInstr &MI, LLT WideTy,
1115 unsigned OpIdx) {
1116 MachineOperand &MO = MI.getOperand(OpIdx);
1117 Register DstExt = MRI.createGenericVirtualRegister(WideTy);
1118 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1119 MIRBuilder.buildExtract(MO.getReg(), DstExt, 0);
1120 MO.setReg(DstExt);
1121}
1122
1123void LegalizerHelper::moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy,
1124 unsigned OpIdx) {
1125 MachineOperand &MO = MI.getOperand(OpIdx);
1126
1127 LLT OldTy = MRI.getType(MO.getReg());
1128 unsigned OldElts = OldTy.getNumElements();
1129 unsigned NewElts = MoreTy.getNumElements();
1130
1131 unsigned NumParts = NewElts / OldElts;
1132
1133 // Use concat_vectors if the result is a multiple of the number of elements.
1134 if (NumParts * OldElts == NewElts) {
1135 SmallVector<Register, 8> Parts;
1136 Parts.push_back(MO.getReg());
1137
1138 Register ImpDef = MIRBuilder.buildUndef(OldTy).getReg(0);
1139 for (unsigned I = 1; I != NumParts; ++I)
1140 Parts.push_back(ImpDef);
1141
1142 auto Concat = MIRBuilder.buildConcatVectors(MoreTy, Parts);
1143 MO.setReg(Concat.getReg(0));
1144 return;
1145 }
1146
1147 Register MoreReg = MRI.createGenericVirtualRegister(MoreTy);
1148 Register ImpDef = MIRBuilder.buildUndef(MoreTy).getReg(0);
1149 MIRBuilder.buildInsert(MoreReg, ImpDef, MO.getReg(), 0);
1150 MO.setReg(MoreReg);
1151}
1152
1153LegalizerHelper::LegalizeResult
1154LegalizerHelper::widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx,
1155 LLT WideTy) {
1156 if (TypeIdx != 1)
1157 return UnableToLegalize;
1158
1159 Register DstReg = MI.getOperand(0).getReg();
1160 LLT DstTy = MRI.getType(DstReg);
1161 if (DstTy.isVector())
1162 return UnableToLegalize;
1163
1164 Register Src1 = MI.getOperand(1).getReg();
1165 LLT SrcTy = MRI.getType(Src1);
1166 const int DstSize = DstTy.getSizeInBits();
1167 const int SrcSize = SrcTy.getSizeInBits();
1168 const int WideSize = WideTy.getSizeInBits();
1169 const int NumMerge = (DstSize + WideSize - 1) / WideSize;
1170
1171 unsigned NumOps = MI.getNumOperands();
1172 unsigned NumSrc = MI.getNumOperands() - 1;
1173 unsigned PartSize = DstTy.getSizeInBits() / NumSrc;
1174
1175 if (WideSize >= DstSize) {
1176 // Directly pack the bits in the target type.
1177 Register ResultReg = MIRBuilder.buildZExt(WideTy, Src1).getReg(0);
1178
1179 for (unsigned I = 2; I != NumOps; ++I) {
1180 const unsigned Offset = (I - 1) * PartSize;
1181
1182 Register SrcReg = MI.getOperand(I).getReg();
1183 assert(MRI.getType(SrcReg) == LLT::scalar(PartSize))((MRI.getType(SrcReg) == LLT::scalar(PartSize)) ? static_cast
<void> (0) : __assert_fail ("MRI.getType(SrcReg) == LLT::scalar(PartSize)"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1183, __PRETTY_FUNCTION__))
;
1184
1185 auto ZextInput = MIRBuilder.buildZExt(WideTy, SrcReg);
1186
1187 Register NextResult = I + 1 == NumOps && WideTy == DstTy ? DstReg :
1188 MRI.createGenericVirtualRegister(WideTy);
1189
1190 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, Offset);
1191 auto Shl = MIRBuilder.buildShl(WideTy, ZextInput, ShiftAmt);
1192 MIRBuilder.buildOr(NextResult, ResultReg, Shl);
1193 ResultReg = NextResult;
1194 }
1195
1196 if (WideSize > DstSize)
1197 MIRBuilder.buildTrunc(DstReg, ResultReg);
1198 else if (DstTy.isPointer())
1199 MIRBuilder.buildIntToPtr(DstReg, ResultReg);
1200
1201 MI.eraseFromParent();
1202 return Legalized;
1203 }
1204
1205 // Unmerge the original values to the GCD type, and recombine to the next
1206 // multiple greater than the original type.
1207 //
1208 // %3:_(s12) = G_MERGE_VALUES %0:_(s4), %1:_(s4), %2:_(s4) -> s6
1209 // %4:_(s2), %5:_(s2) = G_UNMERGE_VALUES %0
1210 // %6:_(s2), %7:_(s2) = G_UNMERGE_VALUES %1
1211 // %8:_(s2), %9:_(s2) = G_UNMERGE_VALUES %2
1212 // %10:_(s6) = G_MERGE_VALUES %4, %5, %6
1213 // %11:_(s6) = G_MERGE_VALUES %7, %8, %9
1214 // %12:_(s12) = G_MERGE_VALUES %10, %11
1215 //
1216 // Padding with undef if necessary:
1217 //
1218 // %2:_(s8) = G_MERGE_VALUES %0:_(s4), %1:_(s4) -> s6
1219 // %3:_(s2), %4:_(s2) = G_UNMERGE_VALUES %0
1220 // %5:_(s2), %6:_(s2) = G_UNMERGE_VALUES %1
1221 // %7:_(s2) = G_IMPLICIT_DEF
1222 // %8:_(s6) = G_MERGE_VALUES %3, %4, %5
1223 // %9:_(s6) = G_MERGE_VALUES %6, %7, %7
1224 // %10:_(s12) = G_MERGE_VALUES %8, %9
1225
1226 const int GCD = greatestCommonDivisor(SrcSize, WideSize);
1227 LLT GCDTy = LLT::scalar(GCD);
1228
1229 SmallVector<Register, 8> Parts;
1230 SmallVector<Register, 8> NewMergeRegs;
1231 SmallVector<Register, 8> Unmerges;
1232 LLT WideDstTy = LLT::scalar(NumMerge * WideSize);
1233
1234 // Decompose the original operands if they don't evenly divide.
1235 for (int I = 1, E = MI.getNumOperands(); I != E; ++I) {
1236 Register SrcReg = MI.getOperand(I).getReg();
1237 if (GCD == SrcSize) {
1238 Unmerges.push_back(SrcReg);
1239 } else {
1240 auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg);
1241 for (int J = 0, JE = Unmerge->getNumOperands() - 1; J != JE; ++J)
1242 Unmerges.push_back(Unmerge.getReg(J));
1243 }
1244 }
1245
1246 // Pad with undef to the next size that is a multiple of the requested size.
1247 if (static_cast<int>(Unmerges.size()) != NumMerge * WideSize) {
1248 Register UndefReg = MIRBuilder.buildUndef(GCDTy).getReg(0);
1249 for (int I = Unmerges.size(); I != NumMerge * WideSize; ++I)
1250 Unmerges.push_back(UndefReg);
1251 }
1252
1253 const int PartsPerGCD = WideSize / GCD;
1254
1255 // Build merges of each piece.
1256 ArrayRef<Register> Slicer(Unmerges);
1257 for (int I = 0; I != NumMerge; ++I, Slicer = Slicer.drop_front(PartsPerGCD)) {
1258 auto Merge = MIRBuilder.buildMerge(WideTy, Slicer.take_front(PartsPerGCD));
1259 NewMergeRegs.push_back(Merge.getReg(0));
1260 }
1261
1262 // A truncate may be necessary if the requested type doesn't evenly divide the
1263 // original result type.
1264 if (DstTy.getSizeInBits() == WideDstTy.getSizeInBits()) {
1265 MIRBuilder.buildMerge(DstReg, NewMergeRegs);
1266 } else {
1267 auto FinalMerge = MIRBuilder.buildMerge(WideDstTy, NewMergeRegs);
1268 MIRBuilder.buildTrunc(DstReg, FinalMerge.getReg(0));
1269 }
1270
1271 MI.eraseFromParent();
1272 return Legalized;
1273}
1274
1275LegalizerHelper::LegalizeResult
1276LegalizerHelper::widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx,
1277 LLT WideTy) {
1278 if (TypeIdx != 0)
1279 return UnableToLegalize;
1280
1281 unsigned NumDst = MI.getNumOperands() - 1;
1282 Register SrcReg = MI.getOperand(NumDst).getReg();
1283 LLT SrcTy = MRI.getType(SrcReg);
1284 if (!SrcTy.isScalar())
1285 return UnableToLegalize;
1286
1287 Register Dst0Reg = MI.getOperand(0).getReg();
1288 LLT DstTy = MRI.getType(Dst0Reg);
1289 if (!DstTy.isScalar())
1290 return UnableToLegalize;
1291
1292 unsigned NewSrcSize = NumDst * WideTy.getSizeInBits();
1293 LLT NewSrcTy = LLT::scalar(NewSrcSize);
1294 unsigned SizeDiff = WideTy.getSizeInBits() - DstTy.getSizeInBits();
1295
1296 auto WideSrc = MIRBuilder.buildZExt(NewSrcTy, SrcReg);
1297
1298 for (unsigned I = 1; I != NumDst; ++I) {
1299 auto ShiftAmt = MIRBuilder.buildConstant(NewSrcTy, SizeDiff * I);
1300 auto Shl = MIRBuilder.buildShl(NewSrcTy, WideSrc, ShiftAmt);
1301 WideSrc = MIRBuilder.buildOr(NewSrcTy, WideSrc, Shl);
1302 }
1303
1304 Observer.changingInstr(MI);
1305
1306 MI.getOperand(NumDst).setReg(WideSrc->getOperand(0).getReg());
1307 for (unsigned I = 0; I != NumDst; ++I)
1308 widenScalarDst(MI, WideTy, I);
1309
1310 Observer.changedInstr(MI);
1311
1312 return Legalized;
1313}
1314
1315LegalizerHelper::LegalizeResult
1316LegalizerHelper::widenScalarExtract(MachineInstr &MI, unsigned TypeIdx,
1317 LLT WideTy) {
1318 Register DstReg = MI.getOperand(0).getReg();
1319 Register SrcReg = MI.getOperand(1).getReg();
1320 LLT SrcTy = MRI.getType(SrcReg);
1321
1322 LLT DstTy = MRI.getType(DstReg);
1323 unsigned Offset = MI.getOperand(2).getImm();
1324
1325 if (TypeIdx == 0) {
1326 if (SrcTy.isVector() || DstTy.isVector())
1327 return UnableToLegalize;
1328
1329 SrcOp Src(SrcReg);
1330 if (SrcTy.isPointer()) {
1331 // Extracts from pointers can be handled only if they are really just
1332 // simple integers.
1333 const DataLayout &DL = MIRBuilder.getDataLayout();
1334 if (DL.isNonIntegralAddressSpace(SrcTy.getAddressSpace()))
1335 return UnableToLegalize;
1336
1337 LLT SrcAsIntTy = LLT::scalar(SrcTy.getSizeInBits());
1338 Src = MIRBuilder.buildPtrToInt(SrcAsIntTy, Src);
1339 SrcTy = SrcAsIntTy;
1340 }
1341
1342 if (DstTy.isPointer())
1343 return UnableToLegalize;
1344
1345 if (Offset == 0) {
1346 // Avoid a shift in the degenerate case.
1347 MIRBuilder.buildTrunc(DstReg,
1348 MIRBuilder.buildAnyExtOrTrunc(WideTy, Src));
1349 MI.eraseFromParent();
1350 return Legalized;
1351 }
1352
1353 // Do a shift in the source type.
1354 LLT ShiftTy = SrcTy;
1355 if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) {
1356 Src = MIRBuilder.buildAnyExt(WideTy, Src);
1357 ShiftTy = WideTy;
1358 } else if (WideTy.getSizeInBits() > SrcTy.getSizeInBits())
1359 return UnableToLegalize;
1360
1361 auto LShr = MIRBuilder.buildLShr(
1362 ShiftTy, Src, MIRBuilder.buildConstant(ShiftTy, Offset));
1363 MIRBuilder.buildTrunc(DstReg, LShr);
1364 MI.eraseFromParent();
1365 return Legalized;
1366 }
1367
1368 if (SrcTy.isScalar()) {
1369 Observer.changingInstr(MI);
1370 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1371 Observer.changedInstr(MI);
1372 return Legalized;
1373 }
1374
1375 if (!SrcTy.isVector())
1376 return UnableToLegalize;
1377
1378 if (DstTy != SrcTy.getElementType())
1379 return UnableToLegalize;
1380
1381 if (Offset % SrcTy.getScalarSizeInBits() != 0)
1382 return UnableToLegalize;
1383
1384 Observer.changingInstr(MI);
1385 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1386
1387 MI.getOperand(2).setImm((WideTy.getSizeInBits() / SrcTy.getSizeInBits()) *
1388 Offset);
1389 widenScalarDst(MI, WideTy.getScalarType(), 0);
1390 Observer.changedInstr(MI);
1391 return Legalized;
1392}
1393
1394LegalizerHelper::LegalizeResult
1395LegalizerHelper::widenScalarInsert(MachineInstr &MI, unsigned TypeIdx,
1396 LLT WideTy) {
1397 if (TypeIdx != 0)
1398 return UnableToLegalize;
1399 Observer.changingInstr(MI);
1400 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1401 widenScalarDst(MI, WideTy);
1402 Observer.changedInstr(MI);
1403 return Legalized;
1404}
1405
1406LegalizerHelper::LegalizeResult
1407LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
1408 MIRBuilder.setInstr(MI);
1409
1410 switch (MI.getOpcode()) {
1411 default:
1412 return UnableToLegalize;
1413 case TargetOpcode::G_EXTRACT:
1414 return widenScalarExtract(MI, TypeIdx, WideTy);
1415 case TargetOpcode::G_INSERT:
1416 return widenScalarInsert(MI, TypeIdx, WideTy);
1417 case TargetOpcode::G_MERGE_VALUES:
1418 return widenScalarMergeValues(MI, TypeIdx, WideTy);
1419 case TargetOpcode::G_UNMERGE_VALUES:
1420 return widenScalarUnmergeValues(MI, TypeIdx, WideTy);
1421 case TargetOpcode::G_UADDO:
1422 case TargetOpcode::G_USUBO: {
1423 if (TypeIdx == 1)
1424 return UnableToLegalize; // TODO
1425 auto LHSZext = MIRBuilder.buildInstr(TargetOpcode::G_ZEXT, {WideTy},
1426 {MI.getOperand(2).getReg()});
1427 auto RHSZext = MIRBuilder.buildInstr(TargetOpcode::G_ZEXT, {WideTy},
1428 {MI.getOperand(3).getReg()});
1429 unsigned Opcode = MI.getOpcode() == TargetOpcode::G_UADDO
1430 ? TargetOpcode::G_ADD
1431 : TargetOpcode::G_SUB;
1432 // Do the arithmetic in the larger type.
1433 auto NewOp = MIRBuilder.buildInstr(Opcode, {WideTy}, {LHSZext, RHSZext});
1434 LLT OrigTy = MRI.getType(MI.getOperand(0).getReg());
1435 APInt Mask = APInt::getAllOnesValue(OrigTy.getSizeInBits());
1436 auto AndOp = MIRBuilder.buildInstr(
1437 TargetOpcode::G_AND, {WideTy},
1438 {NewOp, MIRBuilder.buildConstant(WideTy, Mask.getZExtValue())});
1439 // There is no overflow if the AndOp is the same as NewOp.
1440 MIRBuilder.buildICmp(CmpInst::ICMP_NE, MI.getOperand(1).getReg(), NewOp,
1441 AndOp);
1442 // Now trunc the NewOp to the original result.
1443 MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), NewOp);
1444 MI.eraseFromParent();
1445 return Legalized;
1446 }
1447 case TargetOpcode::G_CTTZ:
1448 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
1449 case TargetOpcode::G_CTLZ:
1450 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
1451 case TargetOpcode::G_CTPOP: {
1452 if (TypeIdx == 0) {
1453 Observer.changingInstr(MI);
1454 widenScalarDst(MI, WideTy, 0);
1455 Observer.changedInstr(MI);
1456 return Legalized;
1457 }
1458
1459 Register SrcReg = MI.getOperand(1).getReg();
1460
1461 // First ZEXT the input.
1462 auto MIBSrc = MIRBuilder.buildZExt(WideTy, SrcReg);
1463 LLT CurTy = MRI.getType(SrcReg);
1464 if (MI.getOpcode() == TargetOpcode::G_CTTZ) {
1465 // The count is the same in the larger type except if the original
1466 // value was zero. This can be handled by setting the bit just off
1467 // the top of the original type.
1468 auto TopBit =
1469 APInt::getOneBitSet(WideTy.getSizeInBits(), CurTy.getSizeInBits());
1470 MIBSrc = MIRBuilder.buildOr(
1471 WideTy, MIBSrc, MIRBuilder.buildConstant(WideTy, TopBit));
1472 }
1473
1474 // Perform the operation at the larger size.
1475 auto MIBNewOp = MIRBuilder.buildInstr(MI.getOpcode(), {WideTy}, {MIBSrc});
1476 // This is already the correct result for CTPOP and CTTZs
1477 if (MI.getOpcode() == TargetOpcode::G_CTLZ ||
1478 MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF) {
1479 // The correct result is NewOp - (Difference in widety and current ty).
1480 unsigned SizeDiff = WideTy.getSizeInBits() - CurTy.getSizeInBits();
1481 MIBNewOp = MIRBuilder.buildInstr(
1482 TargetOpcode::G_SUB, {WideTy},
1483 {MIBNewOp, MIRBuilder.buildConstant(WideTy, SizeDiff)});
1484 }
1485
1486 MIRBuilder.buildZExtOrTrunc(MI.getOperand(0), MIBNewOp);
1487 MI.eraseFromParent();
1488 return Legalized;
1489 }
1490 case TargetOpcode::G_BSWAP: {
1491 Observer.changingInstr(MI);
1492 Register DstReg = MI.getOperand(0).getReg();
1493
1494 Register ShrReg = MRI.createGenericVirtualRegister(WideTy);
1495 Register DstExt = MRI.createGenericVirtualRegister(WideTy);
1496 Register ShiftAmtReg = MRI.createGenericVirtualRegister(WideTy);
1497 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1498
1499 MI.getOperand(0).setReg(DstExt);
1500
1501 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1502
1503 LLT Ty = MRI.getType(DstReg);
1504 unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits();
1505 MIRBuilder.buildConstant(ShiftAmtReg, DiffBits);
1506 MIRBuilder.buildInstr(TargetOpcode::G_LSHR)
1507 .addDef(ShrReg)
1508 .addUse(DstExt)
1509 .addUse(ShiftAmtReg);
1510
1511 MIRBuilder.buildTrunc(DstReg, ShrReg);
1512 Observer.changedInstr(MI);
1513 return Legalized;
1514 }
1515 case TargetOpcode::G_BITREVERSE: {
1516 Observer.changingInstr(MI);
1517
1518 Register DstReg = MI.getOperand(0).getReg();
1519 LLT Ty = MRI.getType(DstReg);
1520 unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits();
1521
1522 Register DstExt = MRI.createGenericVirtualRegister(WideTy);
1523 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1524 MI.getOperand(0).setReg(DstExt);
1525 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1526
1527 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, DiffBits);
1528 auto Shift = MIRBuilder.buildLShr(WideTy, DstExt, ShiftAmt);
1529 MIRBuilder.buildTrunc(DstReg, Shift);
1530 Observer.changedInstr(MI);
1531 return Legalized;
1532 }
1533 case TargetOpcode::G_ADD:
1534 case TargetOpcode::G_AND:
1535 case TargetOpcode::G_MUL:
1536 case TargetOpcode::G_OR:
1537 case TargetOpcode::G_XOR:
1538 case TargetOpcode::G_SUB:
1539 // Perform operation at larger width (any extension is fines here, high bits
1540 // don't affect the result) and then truncate the result back to the
1541 // original type.
1542 Observer.changingInstr(MI);
1543 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1544 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
1545 widenScalarDst(MI, WideTy);
1546 Observer.changedInstr(MI);
1547 return Legalized;
1548
1549 case TargetOpcode::G_SHL:
1550 Observer.changingInstr(MI);
1551
1552 if (TypeIdx == 0) {
1553 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1554 widenScalarDst(MI, WideTy);
1555 } else {
1556 assert(TypeIdx == 1)((TypeIdx == 1) ? static_cast<void> (0) : __assert_fail
("TypeIdx == 1", "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1556, __PRETTY_FUNCTION__))
;
1557 // The "number of bits to shift" operand must preserve its value as an
1558 // unsigned integer:
1559 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
1560 }
1561
1562 Observer.changedInstr(MI);
1563 return Legalized;
1564
1565 case TargetOpcode::G_SDIV:
1566 case TargetOpcode::G_SREM:
1567 case TargetOpcode::G_SMIN:
1568 case TargetOpcode::G_SMAX:
1569 Observer.changingInstr(MI);
1570 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
1571 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
1572 widenScalarDst(MI, WideTy);
1573 Observer.changedInstr(MI);
1574 return Legalized;
1575
1576 case TargetOpcode::G_ASHR:
1577 case TargetOpcode::G_LSHR:
1578 Observer.changingInstr(MI);
1579
1580 if (TypeIdx == 0) {
1581 unsigned CvtOp = MI.getOpcode() == TargetOpcode::G_ASHR ?
1582 TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
1583
1584 widenScalarSrc(MI, WideTy, 1, CvtOp);
1585 widenScalarDst(MI, WideTy);
1586 } else {
1587 assert(TypeIdx == 1)((TypeIdx == 1) ? static_cast<void> (0) : __assert_fail
("TypeIdx == 1", "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1587, __PRETTY_FUNCTION__))
;
1588 // The "number of bits to shift" operand must preserve its value as an
1589 // unsigned integer:
1590 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
1591 }
1592
1593 Observer.changedInstr(MI);
1594 return Legalized;
1595 case TargetOpcode::G_UDIV:
1596 case TargetOpcode::G_UREM:
1597 case TargetOpcode::G_UMIN:
1598 case TargetOpcode::G_UMAX:
1599 Observer.changingInstr(MI);
1600 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
1601 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
1602 widenScalarDst(MI, WideTy);
1603 Observer.changedInstr(MI);
1604 return Legalized;
1605
1606 case TargetOpcode::G_SELECT:
1607 Observer.changingInstr(MI);
1608 if (TypeIdx == 0) {
1609 // Perform operation at larger width (any extension is fine here, high
1610 // bits don't affect the result) and then truncate the result back to the
1611 // original type.
1612 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
1613 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ANYEXT);
1614 widenScalarDst(MI, WideTy);
1615 } else {
1616 bool IsVec = MRI.getType(MI.getOperand(1).getReg()).isVector();
1617 // Explicit extension is required here since high bits affect the result.
1618 widenScalarSrc(MI, WideTy, 1, MIRBuilder.getBoolExtOp(IsVec, false));
1619 }
1620 Observer.changedInstr(MI);
1621 return Legalized;
1622
1623 case TargetOpcode::G_FPTOSI:
1624 case TargetOpcode::G_FPTOUI:
1625 Observer.changingInstr(MI);
1626
1627 if (TypeIdx == 0)
1628 widenScalarDst(MI, WideTy);
1629 else
1630 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT);
1631
1632 Observer.changedInstr(MI);
1633 return Legalized;
1634 case TargetOpcode::G_SITOFP:
1635 if (TypeIdx != 1)
1636 return UnableToLegalize;
1637 Observer.changingInstr(MI);
1638 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
1639 Observer.changedInstr(MI);
1640 return Legalized;
1641
1642 case TargetOpcode::G_UITOFP:
1643 if (TypeIdx != 1)
1644 return UnableToLegalize;
1645 Observer.changingInstr(MI);
1646 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
1647 Observer.changedInstr(MI);
1648 return Legalized;
1649
1650 case TargetOpcode::G_LOAD:
1651 case TargetOpcode::G_SEXTLOAD:
1652 case TargetOpcode::G_ZEXTLOAD:
1653 Observer.changingInstr(MI);
1654 widenScalarDst(MI, WideTy);
1655 Observer.changedInstr(MI);
1656 return Legalized;
1657
1658 case TargetOpcode::G_STORE: {
1659 if (TypeIdx != 0)
1660 return UnableToLegalize;
1661
1662 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
1663 if (!isPowerOf2_32(Ty.getSizeInBits()))
1664 return UnableToLegalize;
1665
1666 Observer.changingInstr(MI);
1667
1668 unsigned ExtType = Ty.getScalarSizeInBits() == 1 ?
1669 TargetOpcode::G_ZEXT : TargetOpcode::G_ANYEXT;
1670 widenScalarSrc(MI, WideTy, 0, ExtType);
1671
1672 Observer.changedInstr(MI);
1673 return Legalized;
1674 }
1675 case TargetOpcode::G_CONSTANT: {
1676 MachineOperand &SrcMO = MI.getOperand(1);
1677 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
1678 const APInt &Val = SrcMO.getCImm()->getValue().sext(WideTy.getSizeInBits());
1679 Observer.changingInstr(MI);
1680 SrcMO.setCImm(ConstantInt::get(Ctx, Val));
1681
1682 widenScalarDst(MI, WideTy);
1683 Observer.changedInstr(MI);
1684 return Legalized;
1685 }
1686 case TargetOpcode::G_FCONSTANT: {
1687 MachineOperand &SrcMO = MI.getOperand(1);
1688 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
1689 APFloat Val = SrcMO.getFPImm()->getValueAPF();
1690 bool LosesInfo;
1691 switch (WideTy.getSizeInBits()) {
1692 case 32:
1693 Val.convert(APFloat::IEEEsingle(), APFloat::rmNearestTiesToEven,
1694 &LosesInfo);
1695 break;
1696 case 64:
1697 Val.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
1698 &LosesInfo);
1699 break;
1700 default:
1701 return UnableToLegalize;
1702 }
1703
1704 assert(!LosesInfo && "extend should always be lossless")((!LosesInfo && "extend should always be lossless") ?
static_cast<void> (0) : __assert_fail ("!LosesInfo && \"extend should always be lossless\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1704, __PRETTY_FUNCTION__))
;
1705
1706 Observer.changingInstr(MI);
1707 SrcMO.setFPImm(ConstantFP::get(Ctx, Val));
1708
1709 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
1710 Observer.changedInstr(MI);
1711 return Legalized;
1712 }
1713 case TargetOpcode::G_IMPLICIT_DEF: {
1714 Observer.changingInstr(MI);
1715 widenScalarDst(MI, WideTy);
1716 Observer.changedInstr(MI);
1717 return Legalized;
1718 }
1719 case TargetOpcode::G_BRCOND:
1720 Observer.changingInstr(MI);
1721 widenScalarSrc(MI, WideTy, 0, MIRBuilder.getBoolExtOp(false, false));
1722 Observer.changedInstr(MI);
1723 return Legalized;
1724
1725 case TargetOpcode::G_FCMP:
1726 Observer.changingInstr(MI);
1727 if (TypeIdx == 0)
1728 widenScalarDst(MI, WideTy);
1729 else {
1730 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_FPEXT);
1731 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_FPEXT);
1732 }
1733 Observer.changedInstr(MI);
1734 return Legalized;
1735
1736 case TargetOpcode::G_ICMP:
1737 Observer.changingInstr(MI);
1738 if (TypeIdx == 0)
1739 widenScalarDst(MI, WideTy);
1740 else {
1741 unsigned ExtOpcode = CmpInst::isSigned(static_cast<CmpInst::Predicate>(
1742 MI.getOperand(1).getPredicate()))
1743 ? TargetOpcode::G_SEXT
1744 : TargetOpcode::G_ZEXT;
1745 widenScalarSrc(MI, WideTy, 2, ExtOpcode);
1746 widenScalarSrc(MI, WideTy, 3, ExtOpcode);
1747 }
1748 Observer.changedInstr(MI);
1749 return Legalized;
1750
1751 case TargetOpcode::G_GEP:
1752 assert(TypeIdx == 1 && "unable to legalize pointer of GEP")((TypeIdx == 1 && "unable to legalize pointer of GEP"
) ? static_cast<void> (0) : __assert_fail ("TypeIdx == 1 && \"unable to legalize pointer of GEP\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1752, __PRETTY_FUNCTION__))
;
1753 Observer.changingInstr(MI);
1754 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
1755 Observer.changedInstr(MI);
1756 return Legalized;
1757
1758 case TargetOpcode::G_PHI: {
1759 assert(TypeIdx == 0 && "Expecting only Idx 0")((TypeIdx == 0 && "Expecting only Idx 0") ? static_cast
<void> (0) : __assert_fail ("TypeIdx == 0 && \"Expecting only Idx 0\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1759, __PRETTY_FUNCTION__))
;
1760
1761 Observer.changingInstr(MI);
1762 for (unsigned I = 1; I < MI.getNumOperands(); I += 2) {
1763 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
1764 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
1765 widenScalarSrc(MI, WideTy, I, TargetOpcode::G_ANYEXT);
1766 }
1767
1768 MachineBasicBlock &MBB = *MI.getParent();
1769 MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI());
1770 widenScalarDst(MI, WideTy);
1771 Observer.changedInstr(MI);
1772 return Legalized;
1773 }
1774 case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
1775 if (TypeIdx == 0) {
1776 Register VecReg = MI.getOperand(1).getReg();
1777 LLT VecTy = MRI.getType(VecReg);
1778 Observer.changingInstr(MI);
1779
1780 widenScalarSrc(MI, LLT::vector(VecTy.getNumElements(),
1781 WideTy.getSizeInBits()),
1782 1, TargetOpcode::G_SEXT);
1783
1784 widenScalarDst(MI, WideTy, 0);
1785 Observer.changedInstr(MI);
1786 return Legalized;
1787 }
1788
1789 if (TypeIdx != 2)
1790 return UnableToLegalize;
1791 Observer.changingInstr(MI);
1792 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
1793 Observer.changedInstr(MI);
1794 return Legalized;
1795 }
1796 case TargetOpcode::G_FADD:
1797 case TargetOpcode::G_FMUL:
1798 case TargetOpcode::G_FSUB:
1799 case TargetOpcode::G_FMA:
1800 case TargetOpcode::G_FMAD:
1801 case TargetOpcode::G_FNEG:
1802 case TargetOpcode::G_FABS:
1803 case TargetOpcode::G_FCANONICALIZE:
1804 case TargetOpcode::G_FMINNUM:
1805 case TargetOpcode::G_FMAXNUM:
1806 case TargetOpcode::G_FMINNUM_IEEE:
1807 case TargetOpcode::G_FMAXNUM_IEEE:
1808 case TargetOpcode::G_FMINIMUM:
1809 case TargetOpcode::G_FMAXIMUM:
1810 case TargetOpcode::G_FDIV:
1811 case TargetOpcode::G_FREM:
1812 case TargetOpcode::G_FCEIL:
1813 case TargetOpcode::G_FFLOOR:
1814 case TargetOpcode::G_FCOS:
1815 case TargetOpcode::G_FSIN:
1816 case TargetOpcode::G_FLOG10:
1817 case TargetOpcode::G_FLOG:
1818 case TargetOpcode::G_FLOG2:
1819 case TargetOpcode::G_FRINT:
1820 case TargetOpcode::G_FNEARBYINT:
1821 case TargetOpcode::G_FSQRT:
1822 case TargetOpcode::G_FEXP:
1823 case TargetOpcode::G_FEXP2:
1824 case TargetOpcode::G_FPOW:
1825 case TargetOpcode::G_INTRINSIC_TRUNC:
1826 case TargetOpcode::G_INTRINSIC_ROUND:
1827 assert(TypeIdx == 0)((TypeIdx == 0) ? static_cast<void> (0) : __assert_fail
("TypeIdx == 0", "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1827, __PRETTY_FUNCTION__))
;
1828 Observer.changingInstr(MI);
1829
1830 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I)
1831 widenScalarSrc(MI, WideTy, I, TargetOpcode::G_FPEXT);
1832
1833 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
1834 Observer.changedInstr(MI);
1835 return Legalized;
1836 case TargetOpcode::G_INTTOPTR:
1837 if (TypeIdx != 1)
1838 return UnableToLegalize;
1839
1840 Observer.changingInstr(MI);
1841 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
1842 Observer.changedInstr(MI);
1843 return Legalized;
1844 case TargetOpcode::G_PTRTOINT:
1845 if (TypeIdx != 0)
1846 return UnableToLegalize;
1847
1848 Observer.changingInstr(MI);
1849 widenScalarDst(MI, WideTy, 0);
1850 Observer.changedInstr(MI);
1851 return Legalized;
1852 case TargetOpcode::G_BUILD_VECTOR: {
1853 Observer.changingInstr(MI);
1854
1855 const LLT WideEltTy = TypeIdx == 1 ? WideTy : WideTy.getElementType();
1856 for (int I = 1, E = MI.getNumOperands(); I != E; ++I)
1857 widenScalarSrc(MI, WideEltTy, I, TargetOpcode::G_ANYEXT);
1858
1859 // Avoid changing the result vector type if the source element type was
1860 // requested.
1861 if (TypeIdx == 1) {
1862 auto &TII = *MI.getMF()->getSubtarget().getInstrInfo();
1863 MI.setDesc(TII.get(TargetOpcode::G_BUILD_VECTOR_TRUNC));
1864 } else {
1865 widenScalarDst(MI, WideTy, 0);
1866 }
1867
1868 Observer.changedInstr(MI);
1869 return Legalized;
1870 }
1871 case TargetOpcode::G_SEXT_INREG:
1872 if (TypeIdx != 0)
1873 return UnableToLegalize;
1874
1875 Observer.changingInstr(MI);
1876 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1877 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_TRUNC);
1878 Observer.changedInstr(MI);
1879 return Legalized;
1880 }
1881}
1882
1883LegalizerHelper::LegalizeResult
1884LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
1885 using namespace TargetOpcode;
1886 MIRBuilder.setInstr(MI);
1887
1888 switch(MI.getOpcode()) {
1889 default:
1890 return UnableToLegalize;
1891 case TargetOpcode::G_SREM:
1892 case TargetOpcode::G_UREM: {
1893 Register QuotReg = MRI.createGenericVirtualRegister(Ty);
1894 MIRBuilder.buildInstr(MI.getOpcode() == G_SREM ? G_SDIV : G_UDIV)
1895 .addDef(QuotReg)
1896 .addUse(MI.getOperand(1).getReg())
1897 .addUse(MI.getOperand(2).getReg());
1898
1899 Register ProdReg = MRI.createGenericVirtualRegister(Ty);
1900 MIRBuilder.buildMul(ProdReg, QuotReg, MI.getOperand(2).getReg());
1901 MIRBuilder.buildSub(MI.getOperand(0).getReg(), MI.getOperand(1).getReg(),
1902 ProdReg);
1903 MI.eraseFromParent();
1904 return Legalized;
1905 }
1906 case TargetOpcode::G_SMULO:
1907 case TargetOpcode::G_UMULO: {
1908 // Generate G_UMULH/G_SMULH to check for overflow and a normal G_MUL for the
1909 // result.
1910 Register Res = MI.getOperand(0).getReg();
1911 Register Overflow = MI.getOperand(1).getReg();
1912 Register LHS = MI.getOperand(2).getReg();
1913 Register RHS = MI.getOperand(3).getReg();
1914
1915 MIRBuilder.buildMul(Res, LHS, RHS);
1916
1917 unsigned Opcode = MI.getOpcode() == TargetOpcode::G_SMULO
1918 ? TargetOpcode::G_SMULH
1919 : TargetOpcode::G_UMULH;
1920
1921 Register HiPart = MRI.createGenericVirtualRegister(Ty);
1922 MIRBuilder.buildInstr(Opcode)
1923 .addDef(HiPart)
1924 .addUse(LHS)
1925 .addUse(RHS);
1926
1927 Register Zero = MRI.createGenericVirtualRegister(Ty);
1928 MIRBuilder.buildConstant(Zero, 0);
1929
1930 // For *signed* multiply, overflow is detected by checking:
1931 // (hi != (lo >> bitwidth-1))
1932 if (Opcode == TargetOpcode::G_SMULH) {
1933 Register Shifted = MRI.createGenericVirtualRegister(Ty);
1934 Register ShiftAmt = MRI.createGenericVirtualRegister(Ty);
1935 MIRBuilder.buildConstant(ShiftAmt, Ty.getSizeInBits() - 1);
1936 MIRBuilder.buildInstr(TargetOpcode::G_ASHR)
1937 .addDef(Shifted)
1938 .addUse(Res)
1939 .addUse(ShiftAmt);
1940 MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Shifted);
1941 } else {
1942 MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Zero);
1943 }
1944 MI.eraseFromParent();
1945 return Legalized;
1946 }
1947 case TargetOpcode::G_FNEG: {
1948 // TODO: Handle vector types once we are able to
1949 // represent them.
1950 if (Ty.isVector())
1951 return UnableToLegalize;
1952 Register Res = MI.getOperand(0).getReg();
1953 Type *ZeroTy;
1954 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
1955 switch (Ty.getSizeInBits()) {
1956 case 16:
1957 ZeroTy = Type::getHalfTy(Ctx);
1958 break;
1959 case 32:
1960 ZeroTy = Type::getFloatTy(Ctx);
1961 break;
1962 case 64:
1963 ZeroTy = Type::getDoubleTy(Ctx);
1964 break;
1965 case 128:
1966 ZeroTy = Type::getFP128Ty(Ctx);
1967 break;
1968 default:
1969 llvm_unreachable("unexpected floating-point type")::llvm::llvm_unreachable_internal("unexpected floating-point type"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1969)
;
1970 }
1971 ConstantFP &ZeroForNegation =
1972 *cast<ConstantFP>(ConstantFP::getZeroValueForNegation(ZeroTy));
1973 auto Zero = MIRBuilder.buildFConstant(Ty, ZeroForNegation);
1974 Register SubByReg = MI.getOperand(1).getReg();
1975 Register ZeroReg = Zero->getOperand(0).getReg();
1976 MIRBuilder.buildInstr(TargetOpcode::G_FSUB, {Res}, {ZeroReg, SubByReg},
1977 MI.getFlags());
1978 MI.eraseFromParent();
1979 return Legalized;
1980 }
1981 case TargetOpcode::G_FSUB: {
1982 // Lower (G_FSUB LHS, RHS) to (G_FADD LHS, (G_FNEG RHS)).
1983 // First, check if G_FNEG is marked as Lower. If so, we may
1984 // end up with an infinite loop as G_FSUB is used to legalize G_FNEG.
1985 if (LI.getAction({G_FNEG, {Ty}}).Action == Lower)
1986 return UnableToLegalize;
1987 Register Res = MI.getOperand(0).getReg();
1988 Register LHS = MI.getOperand(1).getReg();
1989 Register RHS = MI.getOperand(2).getReg();
1990 Register Neg = MRI.createGenericVirtualRegister(Ty);
1991 MIRBuilder.buildInstr(TargetOpcode::G_FNEG).addDef(Neg).addUse(RHS);
1992 MIRBuilder.buildInstr(TargetOpcode::G_FADD, {Res}, {LHS, Neg}, MI.getFlags());
1993 MI.eraseFromParent();
1994 return Legalized;
1995 }
1996 case TargetOpcode::G_FMAD:
1997 return lowerFMad(MI);
1998 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
1999 Register OldValRes = MI.getOperand(0).getReg();
2000 Register SuccessRes = MI.getOperand(1).getReg();
2001 Register Addr = MI.getOperand(2).getReg();
2002 Register CmpVal = MI.getOperand(3).getReg();
2003 Register NewVal = MI.getOperand(4).getReg();
2004 MIRBuilder.buildAtomicCmpXchg(OldValRes, Addr, CmpVal, NewVal,
2005 **MI.memoperands_begin());
2006 MIRBuilder.buildICmp(CmpInst::ICMP_EQ, SuccessRes, OldValRes, CmpVal);
2007 MI.eraseFromParent();
2008 return Legalized;
2009 }
2010 case TargetOpcode::G_LOAD:
2011 case TargetOpcode::G_SEXTLOAD:
2012 case TargetOpcode::G_ZEXTLOAD: {
2013 // Lower to a memory-width G_LOAD and a G_SEXT/G_ZEXT/G_ANYEXT
2014 Register DstReg = MI.getOperand(0).getReg();
2015 Register PtrReg = MI.getOperand(1).getReg();
2016 LLT DstTy = MRI.getType(DstReg);
2017 auto &MMO = **MI.memoperands_begin();
2018
2019 if (DstTy.getSizeInBits() == MMO.getSizeInBits()) {
2020 if (MI.getOpcode() == TargetOpcode::G_LOAD) {
2021 // This load needs splitting into power of 2 sized loads.
2022 if (DstTy.isVector())
2023 return UnableToLegalize;
2024 if (isPowerOf2_32(DstTy.getSizeInBits()))
2025 return UnableToLegalize; // Don't know what we're being asked to do.
2026
2027 // Our strategy here is to generate anyextending loads for the smaller
2028 // types up to next power-2 result type, and then combine the two larger
2029 // result values together, before truncating back down to the non-pow-2
2030 // type.
2031 // E.g. v1 = i24 load =>
2032 // v2 = i32 load (2 byte)
2033 // v3 = i32 load (1 byte)
2034 // v4 = i32 shl v3, 16
2035 // v5 = i32 or v4, v2
2036 // v1 = i24 trunc v5
2037 // By doing this we generate the correct truncate which should get
2038 // combined away as an artifact with a matching extend.
2039 uint64_t LargeSplitSize = PowerOf2Floor(DstTy.getSizeInBits());
2040 uint64_t SmallSplitSize = DstTy.getSizeInBits() - LargeSplitSize;
2041
2042 MachineFunction &MF = MIRBuilder.getMF();
2043 MachineMemOperand *LargeMMO =
2044 MF.getMachineMemOperand(&MMO, 0, LargeSplitSize / 8);
2045 MachineMemOperand *SmallMMO = MF.getMachineMemOperand(
2046 &MMO, LargeSplitSize / 8, SmallSplitSize / 8);
2047
2048 LLT PtrTy = MRI.getType(PtrReg);
2049 unsigned AnyExtSize = NextPowerOf2(DstTy.getSizeInBits());
2050 LLT AnyExtTy = LLT::scalar(AnyExtSize);
2051 Register LargeLdReg = MRI.createGenericVirtualRegister(AnyExtTy);
2052 Register SmallLdReg = MRI.createGenericVirtualRegister(AnyExtTy);
2053 auto LargeLoad =
2054 MIRBuilder.buildLoad(LargeLdReg, PtrReg, *LargeMMO);
2055
2056 auto OffsetCst =
2057 MIRBuilder.buildConstant(LLT::scalar(64), LargeSplitSize / 8);
2058 Register GEPReg = MRI.createGenericVirtualRegister(PtrTy);
2059 auto SmallPtr = MIRBuilder.buildGEP(GEPReg, PtrReg, OffsetCst.getReg(0));
2060 auto SmallLoad = MIRBuilder.buildLoad(SmallLdReg, SmallPtr.getReg(0),
2061 *SmallMMO);
2062
2063 auto ShiftAmt = MIRBuilder.buildConstant(AnyExtTy, LargeSplitSize);
2064 auto Shift = MIRBuilder.buildShl(AnyExtTy, SmallLoad, ShiftAmt);
2065 auto Or = MIRBuilder.buildOr(AnyExtTy, Shift, LargeLoad);
2066 MIRBuilder.buildTrunc(DstReg, {Or.getReg(0)});
2067 MI.eraseFromParent();
2068 return Legalized;
2069 }
2070 MIRBuilder.buildLoad(DstReg, PtrReg, MMO);
2071 MI.eraseFromParent();
2072 return Legalized;
2073 }
2074
2075 if (DstTy.isScalar()) {
2076 Register TmpReg =
2077 MRI.createGenericVirtualRegister(LLT::scalar(MMO.getSizeInBits()));
2078 MIRBuilder.buildLoad(TmpReg, PtrReg, MMO);
2079 switch (MI.getOpcode()) {
2080 default:
2081 llvm_unreachable("Unexpected opcode")::llvm::llvm_unreachable_internal("Unexpected opcode", "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 2081)
;
2082 case TargetOpcode::G_LOAD:
2083 MIRBuilder.buildAnyExt(DstReg, TmpReg);
2084 break;
2085 case TargetOpcode::G_SEXTLOAD:
2086 MIRBuilder.buildSExt(DstReg, TmpReg);
2087 break;
2088 case TargetOpcode::G_ZEXTLOAD:
2089 MIRBuilder.buildZExt(DstReg, TmpReg);
2090 break;
2091 }
2092 MI.eraseFromParent();
2093 return Legalized;
2094 }
2095
2096 return UnableToLegalize;
2097 }
2098 case TargetOpcode::G_STORE: {
2099 // Lower a non-power of 2 store into multiple pow-2 stores.
2100 // E.g. split an i24 store into an i16 store + i8 store.
2101 // We do this by first extending the stored value to the next largest power
2102 // of 2 type, and then using truncating stores to store the components.
2103 // By doing this, likewise with G_LOAD, generate an extend that can be
2104 // artifact-combined away instead of leaving behind extracts.
2105 Register SrcReg = MI.getOperand(0).getReg();
2106 Register PtrReg = MI.getOperand(1).getReg();
2107 LLT SrcTy = MRI.getType(SrcReg);
2108 MachineMemOperand &MMO = **MI.memoperands_begin();
2109 if (SrcTy.getSizeInBits() != MMO.getSizeInBits())
2110 return UnableToLegalize;
2111 if (SrcTy.isVector())
2112 return UnableToLegalize;
2113 if (isPowerOf2_32(SrcTy.getSizeInBits()))
2114 return UnableToLegalize; // Don't know what we're being asked to do.
2115
2116 // Extend to the next pow-2.
2117 const LLT ExtendTy = LLT::scalar(NextPowerOf2(SrcTy.getSizeInBits()));
2118 auto ExtVal = MIRBuilder.buildAnyExt(ExtendTy, SrcReg);
2119
2120 // Obtain the smaller value by shifting away the larger value.
2121 uint64_t LargeSplitSize = PowerOf2Floor(SrcTy.getSizeInBits());
2122 uint64_t SmallSplitSize = SrcTy.getSizeInBits() - LargeSplitSize;
2123 auto ShiftAmt = MIRBuilder.buildConstant(ExtendTy, LargeSplitSize);
2124 auto SmallVal = MIRBuilder.buildLShr(ExtendTy, ExtVal, ShiftAmt);
2125
2126 // Generate the GEP and truncating stores.
2127 LLT PtrTy = MRI.getType(PtrReg);
2128 auto OffsetCst =
2129 MIRBuilder.buildConstant(LLT::scalar(64), LargeSplitSize / 8);
2130 Register GEPReg = MRI.createGenericVirtualRegister(PtrTy);
2131 auto SmallPtr = MIRBuilder.buildGEP(GEPReg, PtrReg, OffsetCst.getReg(0));
2132
2133 MachineFunction &MF = MIRBuilder.getMF();
2134 MachineMemOperand *LargeMMO =
2135 MF.getMachineMemOperand(&MMO, 0, LargeSplitSize / 8);
2136 MachineMemOperand *SmallMMO =
2137 MF.getMachineMemOperand(&MMO, LargeSplitSize / 8, SmallSplitSize / 8);
2138 MIRBuilder.buildStore(ExtVal.getReg(0), PtrReg, *LargeMMO);
2139 MIRBuilder.buildStore(SmallVal.getReg(0), SmallPtr.getReg(0), *SmallMMO);
2140 MI.eraseFromParent();
2141 return Legalized;
2142 }
2143 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
2144 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
2145 case TargetOpcode::G_CTLZ:
2146 case TargetOpcode::G_CTTZ:
2147 case TargetOpcode::G_CTPOP:
2148 return lowerBitCount(MI, TypeIdx, Ty);
2149 case G_UADDO: {
2150 Register Res = MI.getOperand(0).getReg();
2151 Register CarryOut = MI.getOperand(1).getReg();
2152 Register LHS = MI.getOperand(2).getReg();
2153 Register RHS = MI.getOperand(3).getReg();
2154
2155 MIRBuilder.buildAdd(Res, LHS, RHS);
2156 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, RHS);
2157
2158 MI.eraseFromParent();
2159 return Legalized;
2160 }
2161 case G_UADDE: {
2162 Register Res = MI.getOperand(0).getReg();
2163 Register CarryOut = MI.getOperand(1).getReg();
2164 Register LHS = MI.getOperand(2).getReg();
2165 Register RHS = MI.getOperand(3).getReg();
2166 Register CarryIn = MI.getOperand(4).getReg();
2167
2168 Register TmpRes = MRI.createGenericVirtualRegister(Ty);
2169 Register ZExtCarryIn = MRI.createGenericVirtualRegister(Ty);
2170
2171 MIRBuilder.buildAdd(TmpRes, LHS, RHS);
2172 MIRBuilder.buildZExt(ZExtCarryIn, CarryIn);
2173 MIRBuilder.buildAdd(Res, TmpRes, ZExtCarryIn);
2174 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, LHS);
2175
2176 MI.eraseFromParent();
2177 return Legalized;
2178 }
2179 case G_USUBO: {
2180 Register Res = MI.getOperand(0).getReg();
2181 Register BorrowOut = MI.getOperand(1).getReg();
2182 Register LHS = MI.getOperand(2).getReg();
2183 Register RHS = MI.getOperand(3).getReg();
2184
2185 MIRBuilder.buildSub(Res, LHS, RHS);
2186 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, BorrowOut, LHS, RHS);
2187
2188 MI.eraseFromParent();
2189 return Legalized;
2190 }
2191 case G_USUBE: {
2192 Register Res = MI.getOperand(0).getReg();
2193 Register BorrowOut = MI.getOperand(1).getReg();
2194 Register LHS = MI.getOperand(2).getReg();
2195 Register RHS = MI.getOperand(3).getReg();
2196 Register BorrowIn = MI.getOperand(4).getReg();
2197
2198 Register TmpRes = MRI.createGenericVirtualRegister(Ty);
2199 Register ZExtBorrowIn = MRI.createGenericVirtualRegister(Ty);
2200 Register LHS_EQ_RHS = MRI.createGenericVirtualRegister(LLT::scalar(1));
2201 Register LHS_ULT_RHS = MRI.createGenericVirtualRegister(LLT::scalar(1));
2202
2203 MIRBuilder.buildSub(TmpRes, LHS, RHS);
2204 MIRBuilder.buildZExt(ZExtBorrowIn, BorrowIn);
2205 MIRBuilder.buildSub(Res, TmpRes, ZExtBorrowIn);
2206 MIRBuilder.buildICmp(CmpInst::ICMP_EQ, LHS_EQ_RHS, LHS, RHS);
2207 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, LHS_ULT_RHS, LHS, RHS);
2208 MIRBuilder.buildSelect(BorrowOut, LHS_EQ_RHS, BorrowIn, LHS_ULT_RHS);
2209
2210 MI.eraseFromParent();
2211 return Legalized;
2212 }
2213 case G_UITOFP:
2214 return lowerUITOFP(MI, TypeIdx, Ty);
2215 case G_SITOFP:
2216 return lowerSITOFP(MI, TypeIdx, Ty);
2217 case G_FPTOUI:
2218 return lowerFPTOUI(MI, TypeIdx, Ty);
2219 case G_SMIN:
2220 case G_SMAX:
2221 case G_UMIN:
2222 case G_UMAX:
2223 return lowerMinMax(MI, TypeIdx, Ty);
2224 case G_FCOPYSIGN:
2225 return lowerFCopySign(MI, TypeIdx, Ty);
2226 case G_FMINNUM:
2227 case G_FMAXNUM:
2228 return lowerFMinNumMaxNum(MI);
2229 case G_UNMERGE_VALUES:
2230 return lowerUnmergeValues(MI);
2231 case TargetOpcode::G_SEXT_INREG: {
2232 assert(MI.getOperand(2).isImm() && "Expected immediate")((MI.getOperand(2).isImm() && "Expected immediate") ?
static_cast<void> (0) : __assert_fail ("MI.getOperand(2).isImm() && \"Expected immediate\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 2232, __PRETTY_FUNCTION__))
;
2233 int64_t SizeInBits = MI.getOperand(2).getImm();
2234
2235 Register DstReg = MI.getOperand(0).getReg();
2236 Register SrcReg = MI.getOperand(1).getReg();
2237 LLT DstTy = MRI.getType(DstReg);
2238 Register TmpRes = MRI.createGenericVirtualRegister(DstTy);
2239
2240 auto MIBSz = MIRBuilder.buildConstant(DstTy, DstTy.getScalarSizeInBits() - SizeInBits);
2241 MIRBuilder.buildInstr(TargetOpcode::G_SHL, {TmpRes}, {SrcReg, MIBSz->getOperand(0).getReg()});
2242 MIRBuilder.buildInstr(TargetOpcode::G_ASHR, {DstReg}, {TmpRes, MIBSz->getOperand(0).getReg()});
2243 MI.eraseFromParent();
2244 return Legalized;
2245 }
2246 case G_SHUFFLE_VECTOR:
2247 return lowerShuffleVector(MI);
2248 case G_DYN_STACKALLOC:
2249 return lowerDynStackAlloc(MI);
2250 case G_EXTRACT:
2251 return lowerExtract(MI);
2252 case G_INSERT:
2253 return lowerInsert(MI);
2254 }
2255}
2256
2257LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorImplicitDef(
2258 MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) {
2259 SmallVector<Register, 2> DstRegs;
2260
2261 unsigned NarrowSize = NarrowTy.getSizeInBits();
2262 Register DstReg = MI.getOperand(0).getReg();
2263 unsigned Size = MRI.getType(DstReg).getSizeInBits();
2264 int NumParts = Size / NarrowSize;
2265 // FIXME: Don't know how to handle the situation where the small vectors
2266 // aren't all the same size yet.
2267 if (Size % NarrowSize != 0)
2268 return UnableToLegalize;
2269
2270 for (int i = 0; i < NumParts; ++i) {
2271 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
2272 MIRBuilder.buildUndef(TmpReg);
2273 DstRegs.push_back(TmpReg);
2274 }
2275
2276 if (NarrowTy.isVector())
2277 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
2278 else
2279 MIRBuilder.buildBuildVector(DstReg, DstRegs);
2280
2281 MI.eraseFromParent();
2282 return Legalized;
2283}
2284
2285LegalizerHelper::LegalizeResult
2286LegalizerHelper::fewerElementsVectorBasic(MachineInstr &MI, unsigned TypeIdx,
2287 LLT NarrowTy) {
2288 const unsigned Opc = MI.getOpcode();
2289 const unsigned NumOps = MI.getNumOperands() - 1;
2290 const unsigned NarrowSize = NarrowTy.getSizeInBits();
2291 const Register DstReg = MI.getOperand(0).getReg();
2292 const unsigned Flags = MI.getFlags();
2293 const LLT DstTy = MRI.getType(DstReg);
2294 const unsigned Size = DstTy.getSizeInBits();
2295 const int NumParts = Size / NarrowSize;
2296 const LLT EltTy = DstTy.getElementType();
2297 const unsigned EltSize = EltTy.getSizeInBits();
2298 const unsigned BitsForNumParts = NarrowSize * NumParts;
2299
2300 // Check if we have any leftovers. If we do, then only handle the case where
2301 // the leftover is one element.
2302 if (BitsForNumParts != Size && BitsForNumParts + EltSize != Size)
2303 return UnableToLegalize;
2304
2305 if (BitsForNumParts != Size) {
2306 Register AccumDstReg = MRI.createGenericVirtualRegister(DstTy);
2307 MIRBuilder.buildUndef(AccumDstReg);
2308
2309 // Handle the pieces which evenly divide into the requested type with
2310 // extract/op/insert sequence.
2311 for (unsigned Offset = 0; Offset < BitsForNumParts; Offset += NarrowSize) {
2312 SmallVector<SrcOp, 4> SrcOps;
2313 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
2314 Register PartOpReg = MRI.createGenericVirtualRegister(NarrowTy);
2315 MIRBuilder.buildExtract(PartOpReg, MI.getOperand(I).getReg(), Offset);
2316 SrcOps.push_back(PartOpReg);
2317 }
2318
2319 Register PartDstReg = MRI.createGenericVirtualRegister(NarrowTy);
2320 MIRBuilder.buildInstr(Opc, {PartDstReg}, SrcOps, Flags);
2321
2322 Register PartInsertReg = MRI.createGenericVirtualRegister(DstTy);
2323 MIRBuilder.buildInsert(PartInsertReg, AccumDstReg, PartDstReg, Offset);
2324 AccumDstReg = PartInsertReg;
2325 }
2326
2327 // Handle the remaining element sized leftover piece.
2328 SmallVector<SrcOp, 4> SrcOps;
2329 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
2330 Register PartOpReg = MRI.createGenericVirtualRegister(EltTy);
2331 MIRBuilder.buildExtract(PartOpReg, MI.getOperand(I).getReg(),
2332 BitsForNumParts);
2333 SrcOps.push_back(PartOpReg);
2334 }
2335
2336 Register PartDstReg = MRI.createGenericVirtualRegister(EltTy);
2337 MIRBuilder.buildInstr(Opc, {PartDstReg}, SrcOps, Flags);
2338 MIRBuilder.buildInsert(DstReg, AccumDstReg, PartDstReg, BitsForNumParts);
2339 MI.eraseFromParent();
2340
2341 return Legalized;
2342 }
2343
2344 SmallVector<Register, 2> DstRegs, Src0Regs, Src1Regs, Src2Regs;
2345
2346 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src0Regs);
2347
2348 if (NumOps >= 2)
2349 extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src1Regs);
2350
2351 if (NumOps >= 3)
2352 extractParts(MI.getOperand(3).getReg(), NarrowTy, NumParts, Src2Regs);
2353
2354 for (int i = 0; i < NumParts; ++i) {
2355 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy);
2356
2357 if (NumOps == 1)
2358 MIRBuilder.buildInstr(Opc, {DstReg}, {Src0Regs[i]}, Flags);
2359 else if (NumOps == 2) {
2360 MIRBuilder.buildInstr(Opc, {DstReg}, {Src0Regs[i], Src1Regs[i]}, Flags);
2361 } else if (NumOps == 3) {
2362 MIRBuilder.buildInstr(Opc, {DstReg},
2363 {Src0Regs[i], Src1Regs[i], Src2Regs[i]}, Flags);
2364 }
2365
2366 DstRegs.push_back(DstReg);
2367 }
2368
2369 if (NarrowTy.isVector())
2370 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
2371 else
2372 MIRBuilder.buildBuildVector(DstReg, DstRegs);
2373
2374 MI.eraseFromParent();
2375 return Legalized;
2376}
2377
2378// Handle splitting vector operations which need to have the same number of
2379// elements in each type index, but each type index may have a different element
2380// type.
2381//
2382// e.g. <4 x s64> = G_SHL <4 x s64>, <4 x s32> ->
2383// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
2384// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
2385//
2386// Also handles some irregular breakdown cases, e.g.
2387// e.g. <3 x s64> = G_SHL <3 x s64>, <3 x s32> ->
2388// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
2389// s64 = G_SHL s64, s32
2390LegalizerHelper::LegalizeResult
2391LegalizerHelper::fewerElementsVectorMultiEltType(
2392 MachineInstr &MI, unsigned TypeIdx, LLT NarrowTyArg) {
2393 if (TypeIdx != 0)
2394 return UnableToLegalize;
2395
2396 const LLT NarrowTy0 = NarrowTyArg;
2397 const unsigned NewNumElts =
2398 NarrowTy0.isVector() ? NarrowTy0.getNumElements() : 1;
2399
2400 const Register DstReg = MI.getOperand(0).getReg();
2401 LLT DstTy = MRI.getType(DstReg);
2402 LLT LeftoverTy0;
2403
2404 // All of the operands need to have the same number of elements, so if we can
2405 // determine a type breakdown for the result type, we can for all of the
2406 // source types.
2407 int NumParts = getNarrowTypeBreakDown(DstTy, NarrowTy0, LeftoverTy0).first;
2408 if (NumParts < 0)
2409 return UnableToLegalize;
2410
2411 SmallVector<MachineInstrBuilder, 4> NewInsts;
2412
2413 SmallVector<Register, 4> DstRegs, LeftoverDstRegs;
2414 SmallVector<Register, 4> PartRegs, LeftoverRegs;
2415
2416 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
2417 LLT LeftoverTy;
2418 Register SrcReg = MI.getOperand(I).getReg();
2419 LLT SrcTyI = MRI.getType(SrcReg);
2420 LLT NarrowTyI = LLT::scalarOrVector(NewNumElts, SrcTyI.getScalarType());
2421 LLT LeftoverTyI;
2422
2423 // Split this operand into the requested typed registers, and any leftover
2424 // required to reproduce the original type.
2425 if (!extractParts(SrcReg, SrcTyI, NarrowTyI, LeftoverTyI, PartRegs,
2426 LeftoverRegs))
2427 return UnableToLegalize;
2428
2429 if (I == 1) {
2430 // For the first operand, create an instruction for each part and setup
2431 // the result.
2432 for (Register PartReg : PartRegs) {
2433 Register PartDstReg = MRI.createGenericVirtualRegister(NarrowTy0);
2434 NewInsts.push_back(MIRBuilder.buildInstrNoInsert(MI.getOpcode())
2435 .addDef(PartDstReg)
2436 .addUse(PartReg));
2437 DstRegs.push_back(PartDstReg);
2438 }
2439
2440 for (Register LeftoverReg : LeftoverRegs) {
2441 Register PartDstReg = MRI.createGenericVirtualRegister(LeftoverTy0);
2442 NewInsts.push_back(MIRBuilder.buildInstrNoInsert(MI.getOpcode())
2443 .addDef(PartDstReg)
2444 .addUse(LeftoverReg));
2445 LeftoverDstRegs.push_back(PartDstReg);
2446 }
2447 } else {
2448 assert(NewInsts.size() == PartRegs.size() + LeftoverRegs.size())((NewInsts.size() == PartRegs.size() + LeftoverRegs.size()) ?
static_cast<void> (0) : __assert_fail ("NewInsts.size() == PartRegs.size() + LeftoverRegs.size()"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 2448, __PRETTY_FUNCTION__))
;
2449
2450 // Add the newly created operand splits to the existing instructions. The
2451 // odd-sized pieces are ordered after the requested NarrowTyArg sized
2452 // pieces.
2453 unsigned InstCount = 0;
2454 for (unsigned J = 0, JE = PartRegs.size(); J != JE; ++J)
2455 NewInsts[InstCount++].addUse(PartRegs[J]);
2456 for (unsigned J = 0, JE = LeftoverRegs.size(); J != JE; ++J)
2457 NewInsts[InstCount++].addUse(LeftoverRegs[J]);
2458 }
2459
2460 PartRegs.clear();
2461 LeftoverRegs.clear();
2462 }
2463
2464 // Insert the newly built operations and rebuild the result register.
2465 for (auto &MIB : NewInsts)
2466 MIRBuilder.insertInstr(MIB);
2467
2468 insertParts(DstReg, DstTy, NarrowTy0, DstRegs, LeftoverTy0, LeftoverDstRegs);
2469
2470 MI.eraseFromParent();
2471 return Legalized;
2472}
2473
2474LegalizerHelper::LegalizeResult
2475LegalizerHelper::fewerElementsVectorCasts(MachineInstr &MI, unsigned TypeIdx,
2476 LLT NarrowTy) {
2477 if (TypeIdx != 0)
2478 return UnableToLegalize;
2479
2480 Register DstReg = MI.getOperand(0).getReg();
2481 Register SrcReg = MI.getOperand(1).getReg();
2482 LLT DstTy = MRI.getType(DstReg);
2483 LLT SrcTy = MRI.getType(SrcReg);
2484
2485 LLT NarrowTy0 = NarrowTy;
2486 LLT NarrowTy1;
2487 unsigned NumParts;
2488
2489 if (NarrowTy.isVector()) {
2490 // Uneven breakdown not handled.
2491 NumParts = DstTy.getNumElements() / NarrowTy.getNumElements();
2492 if (NumParts * NarrowTy.getNumElements() != DstTy.getNumElements())
2493 return UnableToLegalize;
2494
2495 NarrowTy1 = LLT::vector(NumParts, SrcTy.getElementType().getSizeInBits());
2496 } else {
2497 NumParts = DstTy.getNumElements();
2498 NarrowTy1 = SrcTy.getElementType();
2499 }
2500
2501 SmallVector<Register, 4> SrcRegs, DstRegs;
2502 extractParts(SrcReg, NarrowTy1, NumParts, SrcRegs);
2503
2504 for (unsigned I = 0; I < NumParts; ++I) {
2505 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
2506 MachineInstr *NewInst = MIRBuilder.buildInstr(MI.getOpcode())
2507 .addDef(DstReg)
2508 .addUse(SrcRegs[I]);
2509
2510 NewInst->setFlags(MI.getFlags());
2511 DstRegs.push_back(DstReg);
2512 }
2513
2514 if (NarrowTy.isVector())
2515 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
2516 else
2517 MIRBuilder.buildBuildVector(DstReg, DstRegs);
2518
2519 MI.eraseFromParent();
2520 return Legalized;
2521}
2522
2523LegalizerHelper::LegalizeResult
2524LegalizerHelper::fewerElementsVectorCmp(MachineInstr &MI, unsigned TypeIdx,
2525 LLT NarrowTy) {
2526 Register DstReg = MI.getOperand(0).getReg();
2527 Register Src0Reg = MI.getOperand(2).getReg();
2528 LLT DstTy = MRI.getType(DstReg);
2529 LLT SrcTy = MRI.getType(Src0Reg);
2530
2531 unsigned NumParts;
2532 LLT NarrowTy0, NarrowTy1;
2533
2534 if (TypeIdx == 0) {
2535 unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
2536 unsigned OldElts = DstTy.getNumElements();
2537
2538 NarrowTy0 = NarrowTy;
2539 NumParts = NarrowTy.isVector() ? (OldElts / NewElts) : DstTy.getNumElements();
2540 NarrowTy1 = NarrowTy.isVector() ?
2541 LLT::vector(NarrowTy.getNumElements(), SrcTy.getScalarSizeInBits()) :
2542 SrcTy.getElementType();
2543
2544 } else {
2545 unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
2546 unsigned OldElts = SrcTy.getNumElements();
2547
2548 NumParts = NarrowTy.isVector() ? (OldElts / NewElts) :
2549 NarrowTy.getNumElements();
2550 NarrowTy0 = LLT::vector(NarrowTy.getNumElements(),
2551 DstTy.getScalarSizeInBits());
2552 NarrowTy1 = NarrowTy;
2553 }
2554
2555 // FIXME: Don't know how to handle the situation where the small vectors
2556 // aren't all the same size yet.
2557 if (NarrowTy1.isVector() &&
2558 NarrowTy1.getNumElements() * NumParts != DstTy.getNumElements())
2559 return UnableToLegalize;
2560
2561 CmpInst::Predicate Pred
2562 = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
2563
2564 SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs;
2565 extractParts(MI.getOperand(2).getReg(), NarrowTy1, NumParts, Src1Regs);
2566 extractParts(MI.getOperand(3).getReg(), NarrowTy1, NumParts, Src2Regs);
2567
2568 for (unsigned I = 0; I < NumParts; ++I) {
2569 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
2570 DstRegs.push_back(DstReg);
2571
2572 if (MI.getOpcode() == TargetOpcode::G_ICMP)
2573 MIRBuilder.buildICmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]);
2574 else {
2575 MachineInstr *NewCmp
2576 = MIRBuilder.buildFCmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]);
2577 NewCmp->setFlags(MI.getFlags());
2578 }
2579 }
2580
2581 if (NarrowTy1.isVector())
2582 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
2583 else
2584 MIRBuilder.buildBuildVector(DstReg, DstRegs);
2585
2586 MI.eraseFromParent();
2587 return Legalized;
2588}
2589
2590LegalizerHelper::LegalizeResult
2591LegalizerHelper::fewerElementsVectorSelect(MachineInstr &MI, unsigned TypeIdx,
2592 LLT NarrowTy) {
2593 Register DstReg = MI.getOperand(0).getReg();
2594 Register CondReg = MI.getOperand(1).getReg();
2595
2596 unsigned NumParts = 0;
2597 LLT NarrowTy0, NarrowTy1;
2598
2599 LLT DstTy = MRI.getType(DstReg);
2600 LLT CondTy = MRI.getType(CondReg);
2601 unsigned Size = DstTy.getSizeInBits();
2602
2603 assert(TypeIdx == 0 || CondTy.isVector())((TypeIdx == 0 || CondTy.isVector()) ? static_cast<void>
(0) : __assert_fail ("TypeIdx == 0 || CondTy.isVector()", "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 2603, __PRETTY_FUNCTION__))
;
2604
2605 if (TypeIdx == 0) {
2606 NarrowTy0 = NarrowTy;
2607 NarrowTy1 = CondTy;
2608
2609 unsigned NarrowSize = NarrowTy0.getSizeInBits();
2610 // FIXME: Don't know how to handle the situation where the small vectors
2611 // aren't all the same size yet.
2612 if (Size % NarrowSize != 0)
2613 return UnableToLegalize;
2614
2615 NumParts = Size / NarrowSize;
2616
2617 // Need to break down the condition type
2618 if (CondTy.isVector()) {
2619 if (CondTy.getNumElements() == NumParts)
2620 NarrowTy1 = CondTy.getElementType();
2621 else
2622 NarrowTy1 = LLT::vector(CondTy.getNumElements() / NumParts,
2623 CondTy.getScalarSizeInBits());
2624 }
2625 } else {
2626 NumParts = CondTy.getNumElements();
2627 if (NarrowTy.isVector()) {
2628 // TODO: Handle uneven breakdown.
2629 if (NumParts * NarrowTy.getNumElements() != CondTy.getNumElements())
2630 return UnableToLegalize;
2631
2632 return UnableToLegalize;
2633 } else {
2634 NarrowTy0 = DstTy.getElementType();
2635 NarrowTy1 = NarrowTy;
2636 }
2637 }
2638
2639 SmallVector<Register, 2> DstRegs, Src0Regs, Src1Regs, Src2Regs;
2640 if (CondTy.isVector())
2641 extractParts(MI.getOperand(1).getReg(), NarrowTy1, NumParts, Src0Regs);
2642
2643 extractParts(MI.getOperand(2).getReg(), NarrowTy0, NumParts, Src1Regs);
2644 extractParts(MI.getOperand(3).getReg(), NarrowTy0, NumParts, Src2Regs);
2645
2646 for (unsigned i = 0; i < NumParts; ++i) {
2647 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
2648 MIRBuilder.buildSelect(DstReg, CondTy.isVector() ? Src0Regs[i] : CondReg,
2649 Src1Regs[i], Src2Regs[i]);
2650 DstRegs.push_back(DstReg);
2651 }
2652
2653 if (NarrowTy0.isVector())
2654 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
2655 else
2656 MIRBuilder.buildBuildVector(DstReg, DstRegs);
2657
2658 MI.eraseFromParent();
2659 return Legalized;
2660}
2661
2662LegalizerHelper::LegalizeResult
2663LegalizerHelper::fewerElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx,
2664 LLT NarrowTy) {
2665 const Register DstReg = MI.getOperand(0).getReg();
2666 LLT PhiTy = MRI.getType(DstReg);
2667 LLT LeftoverTy;
2668
2669 // All of the operands need to have the same number of elements, so if we can
2670 // determine a type breakdown for the result type, we can for all of the
2671 // source types.
2672 int NumParts, NumLeftover;
2673 std::tie(NumParts, NumLeftover)
2674 = getNarrowTypeBreakDown(PhiTy, NarrowTy, LeftoverTy);
2675 if (NumParts < 0)
2676 return UnableToLegalize;
2677
2678 SmallVector<Register, 4> DstRegs, LeftoverDstRegs;
2679 SmallVector<MachineInstrBuilder, 4> NewInsts;
2680
2681 const int TotalNumParts = NumParts + NumLeftover;
2682
2683 // Insert the new phis in the result block first.
2684 for (int I = 0; I != TotalNumParts; ++I) {
2685 LLT Ty = I < NumParts ? NarrowTy : LeftoverTy;
2686 Register PartDstReg = MRI.createGenericVirtualRegister(Ty);
2687 NewInsts.push_back(MIRBuilder.buildInstr(TargetOpcode::G_PHI)
2688 .addDef(PartDstReg));
2689 if (I < NumParts)
2690 DstRegs.push_back(PartDstReg);
2691 else
2692 LeftoverDstRegs.push_back(PartDstReg);
2693 }
2694
2695 MachineBasicBlock *MBB = MI.getParent();
2696 MIRBuilder.setInsertPt(*MBB, MBB->getFirstNonPHI());
2697 insertParts(DstReg, PhiTy, NarrowTy, DstRegs, LeftoverTy, LeftoverDstRegs);
2698
2699 SmallVector<Register, 4> PartRegs, LeftoverRegs;
2700
2701 // Insert code to extract the incoming values in each predecessor block.
2702 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
2703 PartRegs.clear();
2704 LeftoverRegs.clear();
2705
2706 Register SrcReg = MI.getOperand(I).getReg();
2707 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
2708 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
2709
2710 LLT Unused;
2711 if (!extractParts(SrcReg, PhiTy, NarrowTy, Unused, PartRegs,
2712 LeftoverRegs))
2713 return UnableToLegalize;
2714
2715 // Add the newly created operand splits to the existing instructions. The
2716 // odd-sized pieces are ordered after the requested NarrowTyArg sized
2717 // pieces.
2718 for (int J = 0; J != TotalNumParts; ++J) {
2719 MachineInstrBuilder MIB = NewInsts[J];
2720 MIB.addUse(J < NumParts ? PartRegs[J] : LeftoverRegs[J - NumParts]);
2721 MIB.addMBB(&OpMBB);
2722 }
2723 }
2724
2725 MI.eraseFromParent();
2726 return Legalized;
2727}
2728
2729LegalizerHelper::LegalizeResult
2730LegalizerHelper::fewerElementsVectorUnmergeValues(MachineInstr &MI,
2731 unsigned TypeIdx,
2732 LLT NarrowTy) {
2733 if (TypeIdx != 1)
2734 return UnableToLegalize;
2735
2736 const int NumDst = MI.getNumOperands() - 1;
2737 const Register SrcReg = MI.getOperand(NumDst).getReg();
2738 LLT SrcTy = MRI.getType(SrcReg);
2739
2740 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
2741
2742 // TODO: Create sequence of extracts.
2743 if (DstTy == NarrowTy)
2744 return UnableToLegalize;
2745
2746 LLT GCDTy = getGCDType(SrcTy, NarrowTy);
2747 if (DstTy == GCDTy) {
2748 // This would just be a copy of the same unmerge.
2749 // TODO: Create extracts, pad with undef and create intermediate merges.
2750 return UnableToLegalize;
2751 }
2752
2753 auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg);
2754 const int NumUnmerge = Unmerge->getNumOperands() - 1;
2755 const int PartsPerUnmerge = NumDst / NumUnmerge;
2756
2757 for (int I = 0; I != NumUnmerge; ++I) {
2758 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES);
2759
2760 for (int J = 0; J != PartsPerUnmerge; ++J)
2761 MIB.addDef(MI.getOperand(I * PartsPerUnmerge + J).getReg());
2762 MIB.addUse(Unmerge.getReg(I));
2763 }
2764
2765 MI.eraseFromParent();
2766 return Legalized;
2767}
2768
2769LegalizerHelper::LegalizeResult
2770LegalizerHelper::fewerElementsVectorBuildVector(MachineInstr &MI,
2771 unsigned TypeIdx,
2772 LLT NarrowTy) {
2773 assert(TypeIdx == 0 && "not a vector type index")((TypeIdx == 0 && "not a vector type index") ? static_cast
<void> (0) : __assert_fail ("TypeIdx == 0 && \"not a vector type index\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 2773, __PRETTY_FUNCTION__))
;
2774 Register DstReg = MI.getOperand(0).getReg();
2775 LLT DstTy = MRI.getType(DstReg);
2776 LLT SrcTy = DstTy.getElementType();
2777
2778 int DstNumElts = DstTy.getNumElements();
2779 int NarrowNumElts = NarrowTy.getNumElements();
2780 int NumConcat = (DstNumElts + NarrowNumElts - 1) / NarrowNumElts;
2781 LLT WidenedDstTy = LLT::vector(NarrowNumElts * NumConcat, SrcTy);
2782
2783 SmallVector<Register, 8> ConcatOps;
2784 SmallVector<Register, 8> SubBuildVector;
2785
2786 Register UndefReg;
2787 if (WidenedDstTy != DstTy)
2788 UndefReg = MIRBuilder.buildUndef(SrcTy).getReg(0);
2789
2790 // Create a G_CONCAT_VECTORS of NarrowTy pieces, padding with undef as
2791 // necessary.
2792 //
2793 // %3:_(<3 x s16>) = G_BUILD_VECTOR %0, %1, %2
2794 // -> <2 x s16>
2795 //
2796 // %4:_(s16) = G_IMPLICIT_DEF
2797 // %5:_(<2 x s16>) = G_BUILD_VECTOR %0, %1
2798 // %6:_(<2 x s16>) = G_BUILD_VECTOR %2, %4
2799 // %7:_(<4 x s16>) = G_CONCAT_VECTORS %5, %6
2800 // %3:_(<3 x s16>) = G_EXTRACT %7, 0
2801 for (int I = 0; I != NumConcat; ++I) {
2802 for (int J = 0; J != NarrowNumElts; ++J) {
2803 int SrcIdx = NarrowNumElts * I + J;
2804
2805 if (SrcIdx < DstNumElts) {
2806 Register SrcReg = MI.getOperand(SrcIdx + 1).getReg();
2807 SubBuildVector.push_back(SrcReg);
2808 } else
2809 SubBuildVector.push_back(UndefReg);
2810 }
2811
2812 auto BuildVec = MIRBuilder.buildBuildVector(NarrowTy, SubBuildVector);
2813 ConcatOps.push_back(BuildVec.getReg(0));
2814 SubBuildVector.clear();
2815 }
2816
2817 if (DstTy == WidenedDstTy)
2818 MIRBuilder.buildConcatVectors(DstReg, ConcatOps);
2819 else {
2820 auto Concat = MIRBuilder.buildConcatVectors(WidenedDstTy, ConcatOps);
2821 MIRBuilder.buildExtract(DstReg, Concat, 0);
2822 }
2823
2824 MI.eraseFromParent();
2825 return Legalized;
2826}
2827
2828LegalizerHelper::LegalizeResult
2829LegalizerHelper::reduceLoadStoreWidth(MachineInstr &MI, unsigned TypeIdx,
2830 LLT NarrowTy) {
2831 // FIXME: Don't know how to handle secondary types yet.
2832 if (TypeIdx != 0)
2833 return UnableToLegalize;
2834
2835 MachineMemOperand *MMO = *MI.memoperands_begin();
2836
2837 // This implementation doesn't work for atomics. Give up instead of doing
2838 // something invalid.
2839 if (MMO->getOrdering() != AtomicOrdering::NotAtomic ||
2840 MMO->getFailureOrdering() != AtomicOrdering::NotAtomic)
2841 return UnableToLegalize;
2842
2843 bool IsLoad = MI.getOpcode() == TargetOpcode::G_LOAD;
2844 Register ValReg = MI.getOperand(0).getReg();
2845 Register AddrReg = MI.getOperand(1).getReg();
2846 LLT ValTy = MRI.getType(ValReg);
2847
2848 int NumParts = -1;
2849 int NumLeftover = -1;
2850 LLT LeftoverTy;
2851 SmallVector<Register, 8> NarrowRegs, NarrowLeftoverRegs;
2852 if (IsLoad) {
2853 std::tie(NumParts, NumLeftover) = getNarrowTypeBreakDown(ValTy, NarrowTy, LeftoverTy);
2854 } else {
2855 if (extractParts(ValReg, ValTy, NarrowTy, LeftoverTy, NarrowRegs,
2856 NarrowLeftoverRegs)) {
2857 NumParts = NarrowRegs.size();
2858 NumLeftover = NarrowLeftoverRegs.size();
Value stored to 'NumLeftover' is never read
2859 }
2860 }
2861
2862 if (NumParts == -1)
2863 return UnableToLegalize;
2864
2865 const LLT OffsetTy = LLT::scalar(MRI.getType(AddrReg).getScalarSizeInBits());
2866
2867 unsigned TotalSize = ValTy.getSizeInBits();
2868
2869 // Split the load/store into PartTy sized pieces starting at Offset. If this
2870 // is a load, return the new registers in ValRegs. For a store, each elements
2871 // of ValRegs should be PartTy. Returns the next offset that needs to be
2872 // handled.
2873 auto splitTypePieces = [=](LLT PartTy, SmallVectorImpl<Register> &ValRegs,
2874 unsigned Offset) -> unsigned {
2875 MachineFunction &MF = MIRBuilder.getMF();
2876 unsigned PartSize = PartTy.getSizeInBits();
2877 for (unsigned Idx = 0, E = NumParts; Idx != E && Offset < TotalSize;
2878 Offset += PartSize, ++Idx) {
2879 unsigned ByteSize = PartSize / 8;
2880 unsigned ByteOffset = Offset / 8;
2881 Register NewAddrReg;
2882
2883 MIRBuilder.materializeGEP(NewAddrReg, AddrReg, OffsetTy, ByteOffset);
2884
2885 MachineMemOperand *NewMMO =
2886 MF.getMachineMemOperand(MMO, ByteOffset, ByteSize);
2887
2888 if (IsLoad) {
2889 Register Dst = MRI.createGenericVirtualRegister(PartTy);
2890 ValRegs.push_back(Dst);
2891 MIRBuilder.buildLoad(Dst, NewAddrReg, *NewMMO);
2892 } else {
2893 MIRBuilder.buildStore(ValRegs[Idx], NewAddrReg, *NewMMO);
2894 }
2895 }
2896
2897 return Offset;
2898 };
2899
2900 unsigned HandledOffset = splitTypePieces(NarrowTy, NarrowRegs, 0);
2901
2902 // Handle the rest of the register if this isn't an even type breakdown.
2903 if (LeftoverTy.isValid())
2904 splitTypePieces(LeftoverTy, NarrowLeftoverRegs, HandledOffset);
2905
2906 if (IsLoad) {
2907 insertParts(ValReg, ValTy, NarrowTy, NarrowRegs,
2908 LeftoverTy, NarrowLeftoverRegs);
2909 }
2910
2911 MI.eraseFromParent();
2912 return Legalized;
2913}
2914
2915LegalizerHelper::LegalizeResult
2916LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
2917 LLT NarrowTy) {
2918 using namespace TargetOpcode;
2919
2920 MIRBuilder.setInstr(MI);
2921 switch (MI.getOpcode()) {
2922 case G_IMPLICIT_DEF:
2923 return fewerElementsVectorImplicitDef(MI, TypeIdx, NarrowTy);
2924 case G_AND:
2925 case G_OR:
2926 case G_XOR:
2927 case G_ADD:
2928 case G_SUB:
2929 case G_MUL:
2930 case G_SMULH:
2931 case G_UMULH:
2932 case G_FADD:
2933 case G_FMUL:
2934 case G_FSUB:
2935 case G_FNEG:
2936 case G_FABS:
2937 case G_FCANONICALIZE:
2938 case G_FDIV:
2939 case G_FREM:
2940 case G_FMA:
2941 case G_FMAD:
2942 case G_FPOW:
2943 case G_FEXP:
2944 case G_FEXP2:
2945 case G_FLOG:
2946 case G_FLOG2:
2947 case G_FLOG10:
2948 case G_FNEARBYINT:
2949 case G_FCEIL:
2950 case G_FFLOOR:
2951 case G_FRINT:
2952 case G_INTRINSIC_ROUND:
2953 case G_INTRINSIC_TRUNC:
2954 case G_FCOS:
2955 case G_FSIN:
2956 case G_FSQRT:
2957 case G_BSWAP:
2958 case G_BITREVERSE:
2959 case G_SDIV:
2960 case G_SMIN:
2961 case G_SMAX:
2962 case G_UMIN:
2963 case G_UMAX:
2964 case G_FMINNUM:
2965 case G_FMAXNUM:
2966 case G_FMINNUM_IEEE:
2967 case G_FMAXNUM_IEEE:
2968 case G_FMINIMUM:
2969 case G_FMAXIMUM:
2970 return fewerElementsVectorBasic(MI, TypeIdx, NarrowTy);
2971 case G_SHL:
2972 case G_LSHR:
2973 case G_ASHR:
2974 case G_CTLZ:
2975 case G_CTLZ_ZERO_UNDEF:
2976 case G_CTTZ:
2977 case G_CTTZ_ZERO_UNDEF:
2978 case G_CTPOP:
2979 case G_FCOPYSIGN:
2980 return fewerElementsVectorMultiEltType(MI, TypeIdx, NarrowTy);
2981 case G_ZEXT:
2982 case G_SEXT:
2983 case G_ANYEXT:
2984 case G_FPEXT:
2985 case G_FPTRUNC:
2986 case G_SITOFP:
2987 case G_UITOFP:
2988 case G_FPTOSI:
2989 case G_FPTOUI:
2990 case G_INTTOPTR:
2991 case G_PTRTOINT:
2992 case G_ADDRSPACE_CAST:
2993 return fewerElementsVectorCasts(MI, TypeIdx, NarrowTy);
2994 case G_ICMP:
2995 case G_FCMP:
2996 return fewerElementsVectorCmp(MI, TypeIdx, NarrowTy);
2997 case G_SELECT:
2998 return fewerElementsVectorSelect(MI, TypeIdx, NarrowTy);
2999 case G_PHI:
3000 return fewerElementsVectorPhi(MI, TypeIdx, NarrowTy);
3001 case G_UNMERGE_VALUES:
3002 return fewerElementsVectorUnmergeValues(MI, TypeIdx, NarrowTy);
3003 case G_BUILD_VECTOR:
3004 return fewerElementsVectorBuildVector(MI, TypeIdx, NarrowTy);
3005 case G_LOAD:
3006 case G_STORE:
3007 return reduceLoadStoreWidth(MI, TypeIdx, NarrowTy);
3008 default:
3009 return UnableToLegalize;
3010 }
3011}
3012
3013LegalizerHelper::LegalizeResult
3014LegalizerHelper::narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt,
3015 const LLT HalfTy, const LLT AmtTy) {
3016
3017 Register InL = MRI.createGenericVirtualRegister(HalfTy);
3018 Register InH = MRI.createGenericVirtualRegister(HalfTy);
3019 MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1).getReg());
3020
3021 if (Amt.isNullValue()) {
3022 MIRBuilder.buildMerge(MI.getOperand(0).getReg(), {InL, InH});
3023 MI.eraseFromParent();
3024 return Legalized;
3025 }
3026
3027 LLT NVT = HalfTy;
3028 unsigned NVTBits = HalfTy.getSizeInBits();
3029 unsigned VTBits = 2 * NVTBits;
3030
3031 SrcOp Lo(Register(0)), Hi(Register(0));
3032 if (MI.getOpcode() == TargetOpcode::G_SHL) {
3033 if (Amt.ugt(VTBits)) {
3034 Lo = Hi = MIRBuilder.buildConstant(NVT, 0);
3035 } else if (Amt.ugt(NVTBits)) {
3036 Lo = MIRBuilder.buildConstant(NVT, 0);
3037 Hi = MIRBuilder.buildShl(NVT, InL,
3038 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
3039 } else if (Amt == NVTBits) {
3040 Lo = MIRBuilder.buildConstant(NVT, 0);
3041 Hi = InL;
3042 } else {
3043 Lo = MIRBuilder.buildShl(NVT, InL, MIRBuilder.buildConstant(AmtTy, Amt));
3044 auto OrLHS =
3045 MIRBuilder.buildShl(NVT, InH, MIRBuilder.buildConstant(AmtTy, Amt));
3046 auto OrRHS = MIRBuilder.buildLShr(
3047 NVT, InL, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
3048 Hi = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
3049 }
3050 } else if (MI.getOpcode() == TargetOpcode::G_LSHR) {
3051 if (Amt.ugt(VTBits)) {
3052 Lo = Hi = MIRBuilder.buildConstant(NVT, 0);
3053 } else if (Amt.ugt(NVTBits)) {
3054 Lo = MIRBuilder.buildLShr(NVT, InH,
3055 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
3056 Hi = MIRBuilder.buildConstant(NVT, 0);
3057 } else if (Amt == NVTBits) {
3058 Lo = InH;
3059 Hi = MIRBuilder.buildConstant(NVT, 0);
3060 } else {
3061 auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt);
3062
3063 auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst);
3064 auto OrRHS = MIRBuilder.buildShl(
3065 NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
3066
3067 Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
3068 Hi = MIRBuilder.buildLShr(NVT, InH, ShiftAmtConst);
3069 }
3070 } else {
3071 if (Amt.ugt(VTBits)) {
3072 Hi = Lo = MIRBuilder.buildAShr(
3073 NVT, InH, MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
3074 } else if (Amt.ugt(NVTBits)) {
3075 Lo = MIRBuilder.buildAShr(NVT, InH,
3076 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
3077 Hi = MIRBuilder.buildAShr(NVT, InH,
3078 MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
3079 } else if (Amt == NVTBits) {
3080 Lo = InH;
3081 Hi = MIRBuilder.buildAShr(NVT, InH,
3082 MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
3083 } else {
3084 auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt);
3085
3086 auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst);
3087 auto OrRHS = MIRBuilder.buildShl(
3088 NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
3089
3090 Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
3091 Hi = MIRBuilder.buildAShr(NVT, InH, ShiftAmtConst);
3092 }
3093 }
3094
3095 MIRBuilder.buildMerge(MI.getOperand(0).getReg(), {Lo.getReg(), Hi.getReg()});
3096 MI.eraseFromParent();
3097
3098 return Legalized;
3099}
3100
3101// TODO: Optimize if constant shift amount.
3102LegalizerHelper::LegalizeResult
3103LegalizerHelper::narrowScalarShift(MachineInstr &MI, unsigned TypeIdx,
3104 LLT RequestedTy) {
3105 if (TypeIdx == 1) {
3106 Observer.changingInstr(MI);
3107 narrowScalarSrc(MI, RequestedTy, 2);
3108 Observer.changedInstr(MI);
3109 return Legalized;
3110 }
3111
3112 Register DstReg = MI.getOperand(0).getReg();
3113 LLT DstTy = MRI.getType(DstReg);
3114 if (DstTy.isVector())
3115 return UnableToLegalize;
3116
3117 Register Amt = MI.getOperand(2).getReg();
3118 LLT ShiftAmtTy = MRI.getType(Amt);
3119 const unsigned DstEltSize = DstTy.getScalarSizeInBits();
3120 if (DstEltSize % 2 != 0)
3121 return UnableToLegalize;
3122
3123 // Ignore the input type. We can only go to exactly half the size of the
3124 // input. If that isn't small enough, the resulting pieces will be further
3125 // legalized.
3126 const unsigned NewBitSize = DstEltSize / 2;
3127 const LLT HalfTy = LLT::scalar(NewBitSize);
3128 const LLT CondTy = LLT::scalar(1);
3129
3130 if (const MachineInstr *KShiftAmt =
3131 getOpcodeDef(TargetOpcode::G_CONSTANT, Amt, MRI)) {
3132 return narrowScalarShiftByConstant(
3133 MI, KShiftAmt->getOperand(1).getCImm()->getValue(), HalfTy, ShiftAmtTy);
3134 }
3135
3136 // TODO: Expand with known bits.
3137
3138 // Handle the fully general expansion by an unknown amount.
3139 auto NewBits = MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize);
3140
3141 Register InL = MRI.createGenericVirtualRegister(HalfTy);
3142 Register InH = MRI.createGenericVirtualRegister(HalfTy);
3143 MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1).getReg());
3144
3145 auto AmtExcess = MIRBuilder.buildSub(ShiftAmtTy, Amt, NewBits);
3146 auto AmtLack = MIRBuilder.buildSub(ShiftAmtTy, NewBits, Amt);
3147
3148 auto Zero = MIRBuilder.buildConstant(ShiftAmtTy, 0);
3149 auto IsShort = MIRBuilder.buildICmp(ICmpInst::ICMP_ULT, CondTy, Amt, NewBits);
3150 auto IsZero = MIRBuilder.buildICmp(ICmpInst::ICMP_EQ, CondTy, Amt, Zero);
3151
3152 Register ResultRegs[2];
3153 switch (MI.getOpcode()) {
3154 case TargetOpcode::G_SHL: {
3155 // Short: ShAmt < NewBitSize
3156 auto LoS = MIRBuilder.buildShl(HalfTy, InL, Amt);
3157
3158 auto LoOr = MIRBuilder.buildLShr(HalfTy, InL, AmtLack);
3159 auto HiOr = MIRBuilder.buildShl(HalfTy, InH, Amt);
3160 auto HiS = MIRBuilder.buildOr(HalfTy, LoOr, HiOr);
3161
3162 // Long: ShAmt >= NewBitSize
3163 auto LoL = MIRBuilder.buildConstant(HalfTy, 0); // Lo part is zero.
3164 auto HiL = MIRBuilder.buildShl(HalfTy, InL, AmtExcess); // Hi from Lo part.
3165
3166 auto Lo = MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL);
3167 auto Hi = MIRBuilder.buildSelect(
3168 HalfTy, IsZero, InH, MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL));
3169
3170 ResultRegs[0] = Lo.getReg(0);
3171 ResultRegs[1] = Hi.getReg(0);
3172 break;
3173 }
3174 case TargetOpcode::G_LSHR:
3175 case TargetOpcode::G_ASHR: {
3176 // Short: ShAmt < NewBitSize
3177 auto HiS = MIRBuilder.buildInstr(MI.getOpcode(), {HalfTy}, {InH, Amt});
3178
3179 auto LoOr = MIRBuilder.buildLShr(HalfTy, InL, Amt);
3180 auto HiOr = MIRBuilder.buildShl(HalfTy, InH, AmtLack);
3181 auto LoS = MIRBuilder.buildOr(HalfTy, LoOr, HiOr);
3182
3183 // Long: ShAmt >= NewBitSize
3184 MachineInstrBuilder HiL;
3185 if (MI.getOpcode() == TargetOpcode::G_LSHR) {
3186 HiL = MIRBuilder.buildConstant(HalfTy, 0); // Hi part is zero.
3187 } else {
3188 auto ShiftAmt = MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize - 1);
3189 HiL = MIRBuilder.buildAShr(HalfTy, InH, ShiftAmt); // Sign of Hi part.
3190 }
3191 auto LoL = MIRBuilder.buildInstr(MI.getOpcode(), {HalfTy},
3192 {InH, AmtExcess}); // Lo from Hi part.
3193
3194 auto Lo = MIRBuilder.buildSelect(
3195 HalfTy, IsZero, InL, MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL));
3196
3197 auto Hi = MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL);
3198
3199 ResultRegs[0] = Lo.getReg(0);
3200 ResultRegs[1] = Hi.getReg(0);
3201 break;
3202 }
3203 default:
3204 llvm_unreachable("not a shift")::llvm::llvm_unreachable_internal("not a shift", "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 3204)
;
3205 }
3206
3207 MIRBuilder.buildMerge(DstReg, ResultRegs);
3208 MI.eraseFromParent();
3209 return Legalized;
3210}
3211
3212LegalizerHelper::LegalizeResult
3213LegalizerHelper::moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx,
3214 LLT MoreTy) {
3215 assert(TypeIdx == 0 && "Expecting only Idx 0")((TypeIdx == 0 && "Expecting only Idx 0") ? static_cast
<void> (0) : __assert_fail ("TypeIdx == 0 && \"Expecting only Idx 0\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 3215, __PRETTY_FUNCTION__))
;
3216
3217 Observer.changingInstr(MI);
3218 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
3219 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
3220 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
3221 moreElementsVectorSrc(MI, MoreTy, I);
3222 }
3223
3224 MachineBasicBlock &MBB = *MI.getParent();
3225 MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI());
3226 moreElementsVectorDst(MI, MoreTy, 0);
3227 Observer.changedInstr(MI);
3228 return Legalized;
3229}
3230
3231LegalizerHelper::LegalizeResult
3232LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
3233 LLT MoreTy) {
3234 MIRBuilder.setInstr(MI);
3235 unsigned Opc = MI.getOpcode();
3236 switch (Opc) {
3237 case TargetOpcode::G_IMPLICIT_DEF:
3238 case TargetOpcode::G_LOAD: {
3239 if (TypeIdx != 0)
3240 return UnableToLegalize;
3241 Observer.changingInstr(MI);
3242 moreElementsVectorDst(MI, MoreTy, 0);
3243 Observer.changedInstr(MI);
3244 return Legalized;
3245 }
3246 case TargetOpcode::G_STORE:
3247 if (TypeIdx != 0)
3248 return UnableToLegalize;
3249 Observer.changingInstr(MI);
3250 moreElementsVectorSrc(MI, MoreTy, 0);
3251 Observer.changedInstr(MI);
3252 return Legalized;
3253 case TargetOpcode::G_AND:
3254 case TargetOpcode::G_OR:
3255 case TargetOpcode::G_XOR:
3256 case TargetOpcode::G_SMIN:
3257 case TargetOpcode::G_SMAX:
3258 case TargetOpcode::G_UMIN:
3259 case TargetOpcode::G_UMAX: {
3260 Observer.changingInstr(MI);
3261 moreElementsVectorSrc(MI, MoreTy, 1);
3262 moreElementsVectorSrc(MI, MoreTy, 2);
3263 moreElementsVectorDst(MI, MoreTy, 0);
3264 Observer.changedInstr(MI);
3265 return Legalized;
3266 }
3267 case TargetOpcode::G_EXTRACT:
3268 if (TypeIdx != 1)
3269 return UnableToLegalize;
3270 Observer.changingInstr(MI);
3271 moreElementsVectorSrc(MI, MoreTy, 1);
3272 Observer.changedInstr(MI);
3273 return Legalized;
3274 case TargetOpcode::G_INSERT:
3275 if (TypeIdx != 0)
3276 return UnableToLegalize;
3277 Observer.changingInstr(MI);
3278 moreElementsVectorSrc(MI, MoreTy, 1);
3279 moreElementsVectorDst(MI, MoreTy, 0);
3280 Observer.changedInstr(MI);
3281 return Legalized;
3282 case TargetOpcode::G_SELECT:
3283 if (TypeIdx != 0)
3284 return UnableToLegalize;
3285 if (MRI.getType(MI.getOperand(1).getReg()).isVector())
3286 return UnableToLegalize;
3287
3288 Observer.changingInstr(MI);
3289 moreElementsVectorSrc(MI, MoreTy, 2);
3290 moreElementsVectorSrc(MI, MoreTy, 3);
3291 moreElementsVectorDst(MI, MoreTy, 0);
3292 Observer.changedInstr(MI);
3293 return Legalized;
3294 case TargetOpcode::G_UNMERGE_VALUES: {
3295 if (TypeIdx != 1)
3296 return UnableToLegalize;
3297
3298 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
3299 int NumDst = MI.getNumOperands() - 1;
3300 moreElementsVectorSrc(MI, MoreTy, NumDst);
3301
3302 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES);
3303 for (int I = 0; I != NumDst; ++I)
3304 MIB.addDef(MI.getOperand(I).getReg());
3305
3306 int NewNumDst = MoreTy.getSizeInBits() / DstTy.getSizeInBits();
3307 for (int I = NumDst; I != NewNumDst; ++I)
3308 MIB.addDef(MRI.createGenericVirtualRegister(DstTy));
3309
3310 MIB.addUse(MI.getOperand(NumDst).getReg());
3311 MI.eraseFromParent();
3312 return Legalized;
3313 }
3314 case TargetOpcode::G_PHI:
3315 return moreElementsVectorPhi(MI, TypeIdx, MoreTy);
3316 default:
3317 return UnableToLegalize;
3318 }
3319}
3320
3321void LegalizerHelper::multiplyRegisters(SmallVectorImpl<Register> &DstRegs,
3322 ArrayRef<Register> Src1Regs,
3323 ArrayRef<Register> Src2Regs,
3324 LLT NarrowTy) {
3325 MachineIRBuilder &B = MIRBuilder;
3326 unsigned SrcParts = Src1Regs.size();
3327 unsigned DstParts = DstRegs.size();
3328
3329 unsigned DstIdx = 0; // Low bits of the result.
3330 Register FactorSum =
3331 B.buildMul(NarrowTy, Src1Regs[DstIdx], Src2Regs[DstIdx]).getReg(0);
3332 DstRegs[DstIdx] = FactorSum;
3333
3334 unsigned CarrySumPrevDstIdx;
3335 SmallVector<Register, 4> Factors;
3336
3337 for (DstIdx = 1; DstIdx < DstParts; DstIdx++) {
3338 // Collect low parts of muls for DstIdx.
3339 for (unsigned i = DstIdx + 1 < SrcParts ? 0 : DstIdx - SrcParts + 1;
3340 i <= std::min(DstIdx, SrcParts - 1); ++i) {
3341 MachineInstrBuilder Mul =
3342 B.buildMul(NarrowTy, Src1Regs[DstIdx - i], Src2Regs[i]);
3343 Factors.push_back(Mul.getReg(0));
3344 }
3345 // Collect high parts of muls from previous DstIdx.
3346 for (unsigned i = DstIdx < SrcParts ? 0 : DstIdx - SrcParts;
3347 i <= std::min(DstIdx - 1, SrcParts - 1); ++i) {
3348 MachineInstrBuilder Umulh =
3349 B.buildUMulH(NarrowTy, Src1Regs[DstIdx - 1 - i], Src2Regs[i]);
3350 Factors.push_back(Umulh.getReg(0));
3351 }
3352 // Add CarrySum from additons calculated for previous DstIdx.
3353 if (DstIdx != 1) {
3354 Factors.push_back(CarrySumPrevDstIdx);
3355 }
3356
3357 Register CarrySum;
3358 // Add all factors and accumulate all carries into CarrySum.
3359 if (DstIdx != DstParts - 1) {
3360 MachineInstrBuilder Uaddo =
3361 B.buildUAddo(NarrowTy, LLT::scalar(1), Factors[0], Factors[1]);
3362 FactorSum = Uaddo.getReg(0);
3363 CarrySum = B.buildZExt(NarrowTy, Uaddo.getReg(1)).getReg(0);
3364 for (unsigned i = 2; i < Factors.size(); ++i) {
3365 MachineInstrBuilder Uaddo =
3366 B.buildUAddo(NarrowTy, LLT::scalar(1), FactorSum, Factors[i]);
3367 FactorSum = Uaddo.getReg(0);
3368 MachineInstrBuilder Carry = B.buildZExt(NarrowTy, Uaddo.getReg(1));
3369 CarrySum = B.buildAdd(NarrowTy, CarrySum, Carry).getReg(0);
3370 }
3371 } else {
3372 // Since value for the next index is not calculated, neither is CarrySum.
3373 FactorSum = B.buildAdd(NarrowTy, Factors[0], Factors[1]).getReg(0);
3374 for (unsigned i = 2; i < Factors.size(); ++i)
3375 FactorSum = B.buildAdd(NarrowTy, FactorSum, Factors[i]).getReg(0);
3376 }
3377
3378 CarrySumPrevDstIdx = CarrySum;
3379 DstRegs[DstIdx] = FactorSum;
3380 Factors.clear();
3381 }
3382}
3383
3384LegalizerHelper::LegalizeResult
3385LegalizerHelper::narrowScalarMul(MachineInstr &MI, LLT NarrowTy) {
3386 Register DstReg = MI.getOperand(0).getReg();
3387 Register Src1 = MI.getOperand(1).getReg();
3388 Register Src2 = MI.getOperand(2).getReg();
3389
3390 LLT Ty = MRI.getType(DstReg);
3391 if (Ty.isVector())
3392 return UnableToLegalize;
3393
3394 unsigned SrcSize = MRI.getType(Src1).getSizeInBits();
3395 unsigned DstSize = Ty.getSizeInBits();
3396 unsigned NarrowSize = NarrowTy.getSizeInBits();
3397 if (DstSize % NarrowSize != 0 || SrcSize % NarrowSize != 0)
3398 return UnableToLegalize;
3399
3400 unsigned NumDstParts = DstSize / NarrowSize;
3401 unsigned NumSrcParts = SrcSize / NarrowSize;
3402 bool IsMulHigh = MI.getOpcode() == TargetOpcode::G_UMULH;
3403 unsigned DstTmpParts = NumDstParts * (IsMulHigh ? 2 : 1);
3404
3405 SmallVector<Register, 2> Src1Parts, Src2Parts, DstTmpRegs;
3406 extractParts(Src1, NarrowTy, NumSrcParts, Src1Parts);
3407 extractParts(Src2, NarrowTy, NumSrcParts, Src2Parts);
3408 DstTmpRegs.resize(DstTmpParts);
3409 multiplyRegisters(DstTmpRegs, Src1Parts, Src2Parts, NarrowTy);
3410
3411 // Take only high half of registers if this is high mul.
3412 ArrayRef<Register> DstRegs(
3413 IsMulHigh ? &DstTmpRegs[DstTmpParts / 2] : &DstTmpRegs[0], NumDstParts);
3414 MIRBuilder.buildMerge(DstReg, DstRegs);
3415 MI.eraseFromParent();
3416 return Legalized;
3417}
3418
3419LegalizerHelper::LegalizeResult
3420LegalizerHelper::narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx,
3421 LLT NarrowTy) {
3422 if (TypeIdx != 1)
3423 return UnableToLegalize;
3424
3425 uint64_t NarrowSize = NarrowTy.getSizeInBits();
3426
3427 int64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
3428 // FIXME: add support for when SizeOp1 isn't an exact multiple of
3429 // NarrowSize.
3430 if (SizeOp1 % NarrowSize != 0)
3431 return UnableToLegalize;
3432 int NumParts = SizeOp1 / NarrowSize;
3433
3434 SmallVector<Register, 2> SrcRegs, DstRegs;
3435 SmallVector<uint64_t, 2> Indexes;
3436 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
3437
3438 Register OpReg = MI.getOperand(0).getReg();
3439 uint64_t OpStart = MI.getOperand(2).getImm();
3440 uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
3441 for (int i = 0; i < NumParts; ++i) {
3442 unsigned SrcStart = i * NarrowSize;
3443
3444 if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) {
3445 // No part of the extract uses this subregister, ignore it.
3446 continue;
3447 } else if (SrcStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
3448 // The entire subregister is extracted, forward the value.
3449 DstRegs.push_back(SrcRegs[i]);
3450 continue;
3451 }
3452
3453 // OpSegStart is where this destination segment would start in OpReg if it
3454 // extended infinitely in both directions.
3455 int64_t ExtractOffset;
3456 uint64_t SegSize;
3457 if (OpStart < SrcStart) {
3458 ExtractOffset = 0;
3459 SegSize = std::min(NarrowSize, OpStart + OpSize - SrcStart);
3460 } else {
3461 ExtractOffset = OpStart - SrcStart;
3462 SegSize = std::min(SrcStart + NarrowSize - OpStart, OpSize);
3463 }
3464
3465 Register SegReg = SrcRegs[i];
3466 if (ExtractOffset != 0 || SegSize != NarrowSize) {
3467 // A genuine extract is needed.
3468 SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
3469 MIRBuilder.buildExtract(SegReg, SrcRegs[i], ExtractOffset);
3470 }
3471
3472 DstRegs.push_back(SegReg);
3473 }
3474
3475 Register DstReg = MI.getOperand(0).getReg();
3476 if(MRI.getType(DstReg).isVector())
3477 MIRBuilder.buildBuildVector(DstReg, DstRegs);
3478 else
3479 MIRBuilder.buildMerge(DstReg, DstRegs);
3480 MI.eraseFromParent();
3481 return Legalized;
3482}
3483
3484LegalizerHelper::LegalizeResult
3485LegalizerHelper::narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx,
3486 LLT NarrowTy) {
3487 // FIXME: Don't know how to handle secondary types yet.
3488 if (TypeIdx != 0)
3489 return UnableToLegalize;
3490
3491 uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
3492 uint64_t NarrowSize = NarrowTy.getSizeInBits();
3493
3494 // FIXME: add support for when SizeOp0 isn't an exact multiple of
3495 // NarrowSize.
3496 if (SizeOp0 % NarrowSize != 0)
3497 return UnableToLegalize;
3498
3499 int NumParts = SizeOp0 / NarrowSize;
3500
3501 SmallVector<Register, 2> SrcRegs, DstRegs;
3502 SmallVector<uint64_t, 2> Indexes;
3503 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
3504
3505 Register OpReg = MI.getOperand(2).getReg();
3506 uint64_t OpStart = MI.getOperand(3).getImm();
3507 uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
3508 for (int i = 0; i < NumParts; ++i) {
3509 unsigned DstStart = i * NarrowSize;
3510
3511 if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) {
3512 // No part of the insert affects this subregister, forward the original.
3513 DstRegs.push_back(SrcRegs[i]);
3514 continue;
3515 } else if (DstStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
3516 // The entire subregister is defined by this insert, forward the new
3517 // value.
3518 DstRegs.push_back(OpReg);
3519 continue;
3520 }
3521
3522 // OpSegStart is where this destination segment would start in OpReg if it
3523 // extended infinitely in both directions.
3524 int64_t ExtractOffset, InsertOffset;
3525 uint64_t SegSize;
3526 if (OpStart < DstStart) {
3527 InsertOffset = 0;
3528 ExtractOffset = DstStart - OpStart;
3529 SegSize = std::min(NarrowSize, OpStart + OpSize - DstStart);
3530 } else {
3531 InsertOffset = OpStart - DstStart;
3532 ExtractOffset = 0;
3533 SegSize =
3534 std::min(NarrowSize - InsertOffset, OpStart + OpSize - DstStart);
3535 }
3536
3537 Register SegReg = OpReg;
3538 if (ExtractOffset != 0 || SegSize != OpSize) {
3539 // A genuine extract is needed.
3540 SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
3541 MIRBuilder.buildExtract(SegReg, OpReg, ExtractOffset);
3542 }
3543
3544 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy);
3545 MIRBuilder.buildInsert(DstReg, SrcRegs[i], SegReg, InsertOffset);
3546 DstRegs.push_back(DstReg);
3547 }
3548
3549 assert(DstRegs.size() == (unsigned)NumParts && "not all parts covered")((DstRegs.size() == (unsigned)NumParts && "not all parts covered"
) ? static_cast<void> (0) : __assert_fail ("DstRegs.size() == (unsigned)NumParts && \"not all parts covered\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 3549, __PRETTY_FUNCTION__))
;
3550 Register DstReg = MI.getOperand(0).getReg();
3551 if(MRI.getType(DstReg).isVector())
3552 MIRBuilder.buildBuildVector(DstReg, DstRegs);
3553 else
3554 MIRBuilder.buildMerge(DstReg, DstRegs);
3555 MI.eraseFromParent();
3556 return Legalized;
3557}
3558
3559LegalizerHelper::LegalizeResult
3560LegalizerHelper::narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx,
3561 LLT NarrowTy) {
3562 Register DstReg = MI.getOperand(0).getReg();
3563 LLT DstTy = MRI.getType(DstReg);
3564
3565 assert(MI.getNumOperands() == 3 && TypeIdx == 0)((MI.getNumOperands() == 3 && TypeIdx == 0) ? static_cast
<void> (0) : __assert_fail ("MI.getNumOperands() == 3 && TypeIdx == 0"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 3565, __PRETTY_FUNCTION__))
;
3566
3567 SmallVector<Register, 4> DstRegs, DstLeftoverRegs;
3568 SmallVector<Register, 4> Src0Regs, Src0LeftoverRegs;
3569 SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs;
3570 LLT LeftoverTy;
3571 if (!extractParts(MI.getOperand(1).getReg(), DstTy, NarrowTy, LeftoverTy,
3572 Src0Regs, Src0LeftoverRegs))
3573 return UnableToLegalize;
3574
3575 LLT Unused;
3576 if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, Unused,
3577 Src1Regs, Src1LeftoverRegs))
3578 llvm_unreachable("inconsistent extractParts result")::llvm::llvm_unreachable_internal("inconsistent extractParts result"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 3578)
;
3579
3580 for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) {
3581 auto Inst = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy},
3582 {Src0Regs[I], Src1Regs[I]});
3583 DstRegs.push_back(Inst->getOperand(0).getReg());
3584 }
3585
3586 for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) {
3587 auto Inst = MIRBuilder.buildInstr(
3588 MI.getOpcode(),
3589 {LeftoverTy}, {Src0LeftoverRegs[I], Src1LeftoverRegs[I]});
3590 DstLeftoverRegs.push_back(Inst->getOperand(0).getReg());
3591 }
3592
3593 insertParts(DstReg, DstTy, NarrowTy, DstRegs,
3594 LeftoverTy, DstLeftoverRegs);
3595
3596 MI.eraseFromParent();
3597 return Legalized;
3598}
3599
3600LegalizerHelper::LegalizeResult
3601LegalizerHelper::narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx,
3602 LLT NarrowTy) {
3603 if (TypeIdx != 0)
3604 return UnableToLegalize;
3605
3606 Register CondReg = MI.getOperand(1).getReg();
3607 LLT CondTy = MRI.getType(CondReg);
3608 if (CondTy.isVector()) // TODO: Handle vselect
3609 return UnableToLegalize;
3610
3611 Register DstReg = MI.getOperand(0).getReg();
3612 LLT DstTy = MRI.getType(DstReg);
3613
3614 SmallVector<Register, 4> DstRegs, DstLeftoverRegs;
3615 SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs;
3616 SmallVector<Register, 4> Src2Regs, Src2LeftoverRegs;
3617 LLT LeftoverTy;
3618 if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, LeftoverTy,
3619 Src1Regs, Src1LeftoverRegs))
3620 return UnableToLegalize;
3621
3622 LLT Unused;
3623 if (!extractParts(MI.getOperand(3).getReg(), DstTy, NarrowTy, Unused,
3624 Src2Regs, Src2LeftoverRegs))
3625 llvm_unreachable("inconsistent extractParts result")::llvm::llvm_unreachable_internal("inconsistent extractParts result"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 3625)
;
3626
3627 for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) {
3628 auto Select = MIRBuilder.buildSelect(NarrowTy,
3629 CondReg, Src1Regs[I], Src2Regs[I]);
3630 DstRegs.push_back(Select->getOperand(0).getReg());
3631 }
3632
3633 for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) {
3634 auto Select = MIRBuilder.buildSelect(
3635 LeftoverTy, CondReg, Src1LeftoverRegs[I], Src2LeftoverRegs[I]);
3636 DstLeftoverRegs.push_back(Select->getOperand(0).getReg());
3637 }
3638
3639 insertParts(DstReg, DstTy, NarrowTy, DstRegs,
3640 LeftoverTy, DstLeftoverRegs);
3641
3642 MI.eraseFromParent();
3643 return Legalized;
3644}
3645
3646LegalizerHelper::LegalizeResult
3647LegalizerHelper::lowerBitCount(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
3648 unsigned Opc = MI.getOpcode();
3649 auto &TII = *MI.getMF()->getSubtarget().getInstrInfo();
3650 auto isSupported = [this](const LegalityQuery &Q) {
3651 auto QAction = LI.getAction(Q).Action;
3652 return QAction == Legal || QAction == Libcall || QAction == Custom;
3653 };
3654 switch (Opc) {
3655 default:
3656 return UnableToLegalize;
3657 case TargetOpcode::G_CTLZ_ZERO_UNDEF: {
3658 // This trivially expands to CTLZ.
3659 Observer.changingInstr(MI);
3660 MI.setDesc(TII.get(TargetOpcode::G_CTLZ));
3661 Observer.changedInstr(MI);
3662 return Legalized;
3663 }
3664 case TargetOpcode::G_CTLZ: {
3665 Register SrcReg = MI.getOperand(1).getReg();
3666 unsigned Len = Ty.getSizeInBits();
3667 if (isSupported({TargetOpcode::G_CTLZ_ZERO_UNDEF, {Ty, Ty}})) {
3668 // If CTLZ_ZERO_UNDEF is supported, emit that and a select for zero.
3669 auto MIBCtlzZU = MIRBuilder.buildInstr(TargetOpcode::G_CTLZ_ZERO_UNDEF,
3670 {Ty}, {SrcReg});
3671 auto MIBZero = MIRBuilder.buildConstant(Ty, 0);
3672 auto MIBLen = MIRBuilder.buildConstant(Ty, Len);
3673 auto MIBICmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1),
3674 SrcReg, MIBZero);
3675 MIRBuilder.buildSelect(MI.getOperand(0).getReg(), MIBICmp, MIBLen,
3676 MIBCtlzZU);
3677 MI.eraseFromParent();
3678 return Legalized;
3679 }
3680 // for now, we do this:
3681 // NewLen = NextPowerOf2(Len);
3682 // x = x | (x >> 1);
3683 // x = x | (x >> 2);
3684 // ...
3685 // x = x | (x >>16);
3686 // x = x | (x >>32); // for 64-bit input
3687 // Upto NewLen/2
3688 // return Len - popcount(x);
3689 //
3690 // Ref: "Hacker's Delight" by Henry Warren
3691 Register Op = SrcReg;
3692 unsigned NewLen = PowerOf2Ceil(Len);
3693 for (unsigned i = 0; (1U << i) <= (NewLen / 2); ++i) {
3694 auto MIBShiftAmt = MIRBuilder.buildConstant(Ty, 1ULL << i);
3695 auto MIBOp = MIRBuilder.buildInstr(
3696 TargetOpcode::G_OR, {Ty},
3697 {Op, MIRBuilder.buildInstr(TargetOpcode::G_LSHR, {Ty},
3698 {Op, MIBShiftAmt})});
3699 Op = MIBOp->getOperand(0).getReg();
3700 }
3701 auto MIBPop = MIRBuilder.buildInstr(TargetOpcode::G_CTPOP, {Ty}, {Op});
3702 MIRBuilder.buildInstr(TargetOpcode::G_SUB, {MI.getOperand(0).getReg()},
3703 {MIRBuilder.buildConstant(Ty, Len), MIBPop});
3704 MI.eraseFromParent();
3705 return Legalized;
3706 }
3707 case TargetOpcode::G_CTTZ_ZERO_UNDEF: {
3708 // This trivially expands to CTTZ.
3709 Observer.changingInstr(MI);
3710 MI.setDesc(TII.get(TargetOpcode::G_CTTZ));
3711 Observer.changedInstr(MI);
3712 return Legalized;
3713 }
3714 case TargetOpcode::G_CTTZ: {
3715 Register SrcReg = MI.getOperand(1).getReg();
3716 unsigned Len = Ty.getSizeInBits();
3717 if (isSupported({TargetOpcode::G_CTTZ_ZERO_UNDEF, {Ty, Ty}})) {
3718 // If CTTZ_ZERO_UNDEF is legal or custom, emit that and a select with
3719 // zero.
3720 auto MIBCttzZU = MIRBuilder.buildInstr(TargetOpcode::G_CTTZ_ZERO_UNDEF,
3721 {Ty}, {SrcReg});
3722 auto MIBZero = MIRBuilder.buildConstant(Ty, 0);
3723 auto MIBLen = MIRBuilder.buildConstant(Ty, Len);
3724 auto MIBICmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1),
3725 SrcReg, MIBZero);
3726 MIRBuilder.buildSelect(MI.getOperand(0).getReg(), MIBICmp, MIBLen,
3727 MIBCttzZU);
3728 MI.eraseFromParent();
3729 return Legalized;
3730 }
3731 // for now, we use: { return popcount(~x & (x - 1)); }
3732 // unless the target has ctlz but not ctpop, in which case we use:
3733 // { return 32 - nlz(~x & (x-1)); }
3734 // Ref: "Hacker's Delight" by Henry Warren
3735 auto MIBCstNeg1 = MIRBuilder.buildConstant(Ty, -1);
3736 auto MIBNot =
3737 MIRBuilder.buildInstr(TargetOpcode::G_XOR, {Ty}, {SrcReg, MIBCstNeg1});
3738 auto MIBTmp = MIRBuilder.buildInstr(
3739 TargetOpcode::G_AND, {Ty},
3740 {MIBNot, MIRBuilder.buildInstr(TargetOpcode::G_ADD, {Ty},
3741 {SrcReg, MIBCstNeg1})});
3742 if (!isSupported({TargetOpcode::G_CTPOP, {Ty, Ty}}) &&
3743 isSupported({TargetOpcode::G_CTLZ, {Ty, Ty}})) {
3744 auto MIBCstLen = MIRBuilder.buildConstant(Ty, Len);
3745 MIRBuilder.buildInstr(
3746 TargetOpcode::G_SUB, {MI.getOperand(0).getReg()},
3747 {MIBCstLen,
3748 MIRBuilder.buildInstr(TargetOpcode::G_CTLZ, {Ty}, {MIBTmp})});
3749 MI.eraseFromParent();
3750 return Legalized;
3751 }
3752 MI.setDesc(TII.get(TargetOpcode::G_CTPOP));
3753 MI.getOperand(1).setReg(MIBTmp->getOperand(0).getReg());
3754 return Legalized;
3755 }
3756 }
3757}
3758
3759// Expand s32 = G_UITOFP s64 using bit operations to an IEEE float
3760// representation.
3761LegalizerHelper::LegalizeResult
3762LegalizerHelper::lowerU64ToF32BitOps(MachineInstr &MI) {
3763 Register Dst = MI.getOperand(0).getReg();
3764 Register Src = MI.getOperand(1).getReg();
3765 const LLT S64 = LLT::scalar(64);
3766 const LLT S32 = LLT::scalar(32);
3767 const LLT S1 = LLT::scalar(1);
3768
3769 assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S32)((MRI.getType(Src) == S64 && MRI.getType(Dst) == S32)
? static_cast<void> (0) : __assert_fail ("MRI.getType(Src) == S64 && MRI.getType(Dst) == S32"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 3769, __PRETTY_FUNCTION__))
;
3770
3771 // unsigned cul2f(ulong u) {
3772 // uint lz = clz(u);
3773 // uint e = (u != 0) ? 127U + 63U - lz : 0;
3774 // u = (u << lz) & 0x7fffffffffffffffUL;
3775 // ulong t = u & 0xffffffffffUL;
3776 // uint v = (e << 23) | (uint)(u >> 40);
3777 // uint r = t > 0x8000000000UL ? 1U : (t == 0x8000000000UL ? v & 1U : 0U);
3778 // return as_float(v + r);
3779 // }
3780
3781 auto Zero32 = MIRBuilder.buildConstant(S32, 0);
3782 auto Zero64 = MIRBuilder.buildConstant(S64, 0);
3783
3784 auto LZ = MIRBuilder.buildCTLZ_ZERO_UNDEF(S32, Src);
3785
3786 auto K = MIRBuilder.buildConstant(S32, 127U + 63U);
3787 auto Sub = MIRBuilder.buildSub(S32, K, LZ);
3788
3789 auto NotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, Src, Zero64);
3790 auto E = MIRBuilder.buildSelect(S32, NotZero, Sub, Zero32);
3791
3792 auto Mask0 = MIRBuilder.buildConstant(S64, (-1ULL) >> 1);
3793 auto ShlLZ = MIRBuilder.buildShl(S64, Src, LZ);
3794
3795 auto U = MIRBuilder.buildAnd(S64, ShlLZ, Mask0);
3796
3797 auto Mask1 = MIRBuilder.buildConstant(S64, 0xffffffffffULL);
3798 auto T = MIRBuilder.buildAnd(S64, U, Mask1);
3799
3800 auto UShl = MIRBuilder.buildLShr(S64, U, MIRBuilder.buildConstant(S64, 40));
3801 auto ShlE = MIRBuilder.buildShl(S32, E, MIRBuilder.buildConstant(S32, 23));
3802 auto V = MIRBuilder.buildOr(S32, ShlE, MIRBuilder.buildTrunc(S32, UShl));
3803
3804 auto C = MIRBuilder.buildConstant(S64, 0x8000000000ULL);
3805 auto RCmp = MIRBuilder.buildICmp(CmpInst::ICMP_UGT, S1, T, C);
3806 auto TCmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1, T, C);
3807 auto One = MIRBuilder.buildConstant(S32, 1);
3808
3809 auto VTrunc1 = MIRBuilder.buildAnd(S32, V, One);
3810 auto Select0 = MIRBuilder.buildSelect(S32, TCmp, VTrunc1, Zero32);
3811 auto R = MIRBuilder.buildSelect(S32, RCmp, One, Select0);
3812 MIRBuilder.buildAdd(Dst, V, R);
3813
3814 return Legalized;
3815}
3816
3817LegalizerHelper::LegalizeResult
3818LegalizerHelper::lowerUITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
3819 Register Dst = MI.getOperand(0).getReg();
3820 Register Src = MI.getOperand(1).getReg();
3821 LLT DstTy = MRI.getType(Dst);
3822 LLT SrcTy = MRI.getType(Src);
3823
3824 if (SrcTy != LLT::scalar(64))
3825 return UnableToLegalize;
3826
3827 if (DstTy == LLT::scalar(32)) {
3828 // TODO: SelectionDAG has several alternative expansions to port which may
3829 // be more reasonble depending on the available instructions. If a target
3830 // has sitofp, does not have CTLZ, or can efficiently use f64 as an
3831 // intermediate type, this is probably worse.
3832 return lowerU64ToF32BitOps(MI);
3833 }
3834
3835 return UnableToLegalize;
3836}
3837
3838LegalizerHelper::LegalizeResult
3839LegalizerHelper::lowerSITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
3840 Register Dst = MI.getOperand(0).getReg();
3841 Register Src = MI.getOperand(1).getReg();
3842 LLT DstTy = MRI.getType(Dst);
3843 LLT SrcTy = MRI.getType(Src);
3844
3845 const LLT S64 = LLT::scalar(64);
3846 const LLT S32 = LLT::scalar(32);
3847 const LLT S1 = LLT::scalar(1);
3848
3849 if (SrcTy != S64)
3850 return UnableToLegalize;
3851
3852 if (DstTy == S32) {
3853 // signed cl2f(long l) {
3854 // long s = l >> 63;
3855 // float r = cul2f((l + s) ^ s);
3856 // return s ? -r : r;
3857 // }
3858 Register L = Src;
3859 auto SignBit = MIRBuilder.buildConstant(S64, 63);
3860 auto S = MIRBuilder.buildAShr(S64, L, SignBit);
3861
3862 auto LPlusS = MIRBuilder.buildAdd(S64, L, S);
3863 auto Xor = MIRBuilder.buildXor(S64, LPlusS, S);
3864 auto R = MIRBuilder.buildUITOFP(S32, Xor);
3865
3866 auto RNeg = MIRBuilder.buildFNeg(S32, R);
3867 auto SignNotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, S,
3868 MIRBuilder.buildConstant(S64, 0));
3869 MIRBuilder.buildSelect(Dst, SignNotZero, RNeg, R);
3870 return Legalized;
3871 }
3872
3873 return UnableToLegalize;
3874}
3875
3876LegalizerHelper::LegalizeResult
3877LegalizerHelper::lowerFPTOUI(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
3878 Register Dst = MI.getOperand(0).getReg();
3879 Register Src = MI.getOperand(1).getReg();
3880 LLT DstTy = MRI.getType(Dst);
3881 LLT SrcTy = MRI.getType(Src);
3882 const LLT S64 = LLT::scalar(64);
3883 const LLT S32 = LLT::scalar(32);
3884
3885 if (SrcTy != S64 && SrcTy != S32)
3886 return UnableToLegalize;
3887 if (DstTy != S32 && DstTy != S64)
3888 return UnableToLegalize;
3889
3890 // FPTOSI gives same result as FPTOUI for positive signed integers.
3891 // FPTOUI needs to deal with fp values that convert to unsigned integers
3892 // greater or equal to 2^31 for float or 2^63 for double. For brevity 2^Exp.
3893
3894 APInt TwoPExpInt = APInt::getSignMask(DstTy.getSizeInBits());
3895 APFloat TwoPExpFP(SrcTy.getSizeInBits() == 32 ? APFloat::IEEEsingle()
3896 : APFloat::IEEEdouble(),
3897 APInt::getNullValue(SrcTy.getSizeInBits()));
3898 TwoPExpFP.convertFromAPInt(TwoPExpInt, false, APFloat::rmNearestTiesToEven);
3899
3900 MachineInstrBuilder FPTOSI = MIRBuilder.buildFPTOSI(DstTy, Src);
3901
3902 MachineInstrBuilder Threshold = MIRBuilder.buildFConstant(SrcTy, TwoPExpFP);
3903 // For fp Value greater or equal to Threshold(2^Exp), we use FPTOSI on
3904 // (Value - 2^Exp) and add 2^Exp by setting highest bit in result to 1.
3905 MachineInstrBuilder FSub = MIRBuilder.buildFSub(SrcTy, Src, Threshold);
3906 MachineInstrBuilder ResLowBits = MIRBuilder.buildFPTOSI(DstTy, FSub);
3907 MachineInstrBuilder ResHighBit = MIRBuilder.buildConstant(DstTy, TwoPExpInt);
3908 MachineInstrBuilder Res = MIRBuilder.buildXor(DstTy, ResLowBits, ResHighBit);
3909
3910 MachineInstrBuilder FCMP =
3911 MIRBuilder.buildFCmp(CmpInst::FCMP_ULT, DstTy, Src, Threshold);
3912 MIRBuilder.buildSelect(Dst, FCMP, FPTOSI, Res);
3913
3914 MI.eraseFromParent();
3915 return Legalized;
3916}
3917
3918static CmpInst::Predicate minMaxToCompare(unsigned Opc) {
3919 switch (Opc) {
3920 case TargetOpcode::G_SMIN:
3921 return CmpInst::ICMP_SLT;
3922 case TargetOpcode::G_SMAX:
3923 return CmpInst::ICMP_SGT;
3924 case TargetOpcode::G_UMIN:
3925 return CmpInst::ICMP_ULT;
3926 case TargetOpcode::G_UMAX:
3927 return CmpInst::ICMP_UGT;
3928 default:
3929 llvm_unreachable("not in integer min/max")::llvm::llvm_unreachable_internal("not in integer min/max", "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 3929)
;
3930 }
3931}
3932
3933LegalizerHelper::LegalizeResult
3934LegalizerHelper::lowerMinMax(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
3935 Register Dst = MI.getOperand(0).getReg();
3936 Register Src0 = MI.getOperand(1).getReg();
3937 Register Src1 = MI.getOperand(2).getReg();
3938
3939 const CmpInst::Predicate Pred = minMaxToCompare(MI.getOpcode());
3940 LLT CmpType = MRI.getType(Dst).changeElementSize(1);
3941
3942 auto Cmp = MIRBuilder.buildICmp(Pred, CmpType, Src0, Src1);
3943 MIRBuilder.buildSelect(Dst, Cmp, Src0, Src1);
3944
3945 MI.eraseFromParent();
3946 return Legalized;
3947}
3948
3949LegalizerHelper::LegalizeResult
3950LegalizerHelper::lowerFCopySign(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
3951 Register Dst = MI.getOperand(0).getReg();
3952 Register Src0 = MI.getOperand(1).getReg();
3953 Register Src1 = MI.getOperand(2).getReg();
3954
3955 const LLT Src0Ty = MRI.getType(Src0);
3956 const LLT Src1Ty = MRI.getType(Src1);
3957
3958 const int Src0Size = Src0Ty.getScalarSizeInBits();
3959 const int Src1Size = Src1Ty.getScalarSizeInBits();
3960
3961 auto SignBitMask = MIRBuilder.buildConstant(
3962 Src0Ty, APInt::getSignMask(Src0Size));
3963
3964 auto NotSignBitMask = MIRBuilder.buildConstant(
3965 Src0Ty, APInt::getLowBitsSet(Src0Size, Src0Size - 1));
3966
3967 auto And0 = MIRBuilder.buildAnd(Src0Ty, Src0, NotSignBitMask);
3968 MachineInstr *Or;
3969
3970 if (Src0Ty == Src1Ty) {
3971 auto And1 = MIRBuilder.buildAnd(Src1Ty, Src0, SignBitMask);
3972 Or = MIRBuilder.buildOr(Dst, And0, And1);
3973 } else if (Src0Size > Src1Size) {
3974 auto ShiftAmt = MIRBuilder.buildConstant(Src0Ty, Src0Size - Src1Size);
3975 auto Zext = MIRBuilder.buildZExt(Src0Ty, Src1);
3976 auto Shift = MIRBuilder.buildShl(Src0Ty, Zext, ShiftAmt);
3977 auto And1 = MIRBuilder.buildAnd(Src0Ty, Shift, SignBitMask);
3978 Or = MIRBuilder.buildOr(Dst, And0, And1);
3979 } else {
3980 auto ShiftAmt = MIRBuilder.buildConstant(Src1Ty, Src1Size - Src0Size);
3981 auto Shift = MIRBuilder.buildLShr(Src1Ty, Src1, ShiftAmt);
3982 auto Trunc = MIRBuilder.buildTrunc(Src0Ty, Shift);
3983 auto And1 = MIRBuilder.buildAnd(Src0Ty, Trunc, SignBitMask);
3984 Or = MIRBuilder.buildOr(Dst, And0, And1);
3985 }
3986
3987 // Be careful about setting nsz/nnan/ninf on every instruction, since the
3988 // constants are a nan and -0.0, but the final result should preserve
3989 // everything.
3990 if (unsigned Flags = MI.getFlags())
3991 Or->setFlags(Flags);
3992
3993 MI.eraseFromParent();
3994 return Legalized;
3995}
3996
3997LegalizerHelper::LegalizeResult
3998LegalizerHelper::lowerFMinNumMaxNum(MachineInstr &MI) {
3999 unsigned NewOp = MI.getOpcode() == TargetOpcode::G_FMINNUM ?
4000 TargetOpcode::G_FMINNUM_IEEE : TargetOpcode::G_FMAXNUM_IEEE;
4001
4002 Register Dst = MI.getOperand(0).getReg();
4003 Register Src0 = MI.getOperand(1).getReg();
4004 Register Src1 = MI.getOperand(2).getReg();
4005 LLT Ty = MRI.getType(Dst);
4006
4007 if (!MI.getFlag(MachineInstr::FmNoNans)) {
4008 // Insert canonicalizes if it's possible we need to quiet to get correct
4009 // sNaN behavior.
4010
4011 // Note this must be done here, and not as an optimization combine in the
4012 // absence of a dedicate quiet-snan instruction as we're using an
4013 // omni-purpose G_FCANONICALIZE.
4014 if (!isKnownNeverSNaN(Src0, MRI))
4015 Src0 = MIRBuilder.buildFCanonicalize(Ty, Src0, MI.getFlags()).getReg(0);
4016
4017 if (!isKnownNeverSNaN(Src1, MRI))
4018 Src1 = MIRBuilder.buildFCanonicalize(Ty, Src1, MI.getFlags()).getReg(0);
4019 }
4020
4021 // If there are no nans, it's safe to simply replace this with the non-IEEE
4022 // version.
4023 MIRBuilder.buildInstr(NewOp, {Dst}, {Src0, Src1}, MI.getFlags());
4024 MI.eraseFromParent();
4025 return Legalized;
4026}
4027
4028LegalizerHelper::LegalizeResult LegalizerHelper::lowerFMad(MachineInstr &MI) {
4029 // Expand G_FMAD a, b, c -> G_FADD (G_FMUL a, b), c
4030 Register DstReg = MI.getOperand(0).getReg();
4031 LLT Ty = MRI.getType(DstReg);
4032 unsigned Flags = MI.getFlags();
4033
4034 auto Mul = MIRBuilder.buildFMul(Ty, MI.getOperand(1), MI.getOperand(2),
4035 Flags);
4036 MIRBuilder.buildFAdd(DstReg, Mul, MI.getOperand(3), Flags);
4037 MI.eraseFromParent();
4038 return Legalized;
4039}
4040
4041LegalizerHelper::LegalizeResult
4042LegalizerHelper::lowerUnmergeValues(MachineInstr &MI) {
4043 const unsigned NumDst = MI.getNumOperands() - 1;
4044 const Register SrcReg = MI.getOperand(NumDst).getReg();
4045 LLT SrcTy = MRI.getType(SrcReg);
4046
4047 Register Dst0Reg = MI.getOperand(0).getReg();
4048 LLT DstTy = MRI.getType(Dst0Reg);
4049
4050
4051 // Expand scalarizing unmerge as bitcast to integer and shift.
4052 if (!DstTy.isVector() && SrcTy.isVector() &&
4053 SrcTy.getElementType() == DstTy) {
4054 LLT IntTy = LLT::scalar(SrcTy.getSizeInBits());
4055 Register Cast = MIRBuilder.buildBitcast(IntTy, SrcReg).getReg(0);
4056
4057 MIRBuilder.buildTrunc(Dst0Reg, Cast);
4058
4059 const unsigned DstSize = DstTy.getSizeInBits();
4060 unsigned Offset = DstSize;
4061 for (unsigned I = 1; I != NumDst; ++I, Offset += DstSize) {
4062 auto ShiftAmt = MIRBuilder.buildConstant(IntTy, Offset);
4063 auto Shift = MIRBuilder.buildLShr(IntTy, Cast, ShiftAmt);
4064 MIRBuilder.buildTrunc(MI.getOperand(I), Shift);
4065 }
4066
4067 MI.eraseFromParent();
4068 return Legalized;
4069 }
4070
4071 return UnableToLegalize;
4072}
4073
4074LegalizerHelper::LegalizeResult
4075LegalizerHelper::lowerShuffleVector(MachineInstr &MI) {
4076 Register DstReg = MI.getOperand(0).getReg();
4077 Register Src0Reg = MI.getOperand(1).getReg();
4078 Register Src1Reg = MI.getOperand(2).getReg();
4079 LLT Src0Ty = MRI.getType(Src0Reg);
4080 LLT DstTy = MRI.getType(DstReg);
4081 LLT IdxTy = LLT::scalar(32);
4082
4083 const Constant *ShufMask = MI.getOperand(3).getShuffleMask();
4084
4085 SmallVector<int, 32> Mask;
4086 ShuffleVectorInst::getShuffleMask(ShufMask, Mask);
4087
4088 if (DstTy.isScalar()) {
4089 if (Src0Ty.isVector())
4090 return UnableToLegalize;
4091
4092 // This is just a SELECT.
4093 assert(Mask.size() == 1 && "Expected a single mask element")((Mask.size() == 1 && "Expected a single mask element"
) ? static_cast<void> (0) : __assert_fail ("Mask.size() == 1 && \"Expected a single mask element\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 4093, __PRETTY_FUNCTION__))
;
4094 Register Val;
4095 if (Mask[0] < 0 || Mask[0] > 1)
4096 Val = MIRBuilder.buildUndef(DstTy).getReg(0);
4097 else
4098 Val = Mask[0] == 0 ? Src0Reg : Src1Reg;
4099 MIRBuilder.buildCopy(DstReg, Val);
4100 MI.eraseFromParent();
4101 return Legalized;
4102 }
4103
4104 Register Undef;
4105 SmallVector<Register, 32> BuildVec;
4106 LLT EltTy = DstTy.getElementType();
4107
4108 for (int Idx : Mask) {
4109 if (Idx < 0) {
4110 if (!Undef.isValid())
4111 Undef = MIRBuilder.buildUndef(EltTy).getReg(0);
4112 BuildVec.push_back(Undef);
4113 continue;
4114 }
4115
4116 if (Src0Ty.isScalar()) {
4117 BuildVec.push_back(Idx == 0 ? Src0Reg : Src1Reg);
4118 } else {
4119 int NumElts = Src0Ty.getNumElements();
4120 Register SrcVec = Idx < NumElts ? Src0Reg : Src1Reg;
4121 int ExtractIdx = Idx < NumElts ? Idx : Idx - NumElts;
4122 auto IdxK = MIRBuilder.buildConstant(IdxTy, ExtractIdx);
4123 auto Extract = MIRBuilder.buildExtractVectorElement(EltTy, SrcVec, IdxK);
4124 BuildVec.push_back(Extract.getReg(0));
4125 }
4126 }
4127
4128 MIRBuilder.buildBuildVector(DstReg, BuildVec);
4129 MI.eraseFromParent();
4130 return Legalized;
4131}
4132
4133LegalizerHelper::LegalizeResult
4134LegalizerHelper::lowerDynStackAlloc(MachineInstr &MI) {
4135 Register Dst = MI.getOperand(0).getReg();
4136 Register AllocSize = MI.getOperand(1).getReg();
4137 unsigned Align = MI.getOperand(2).getImm();
4138
4139 const auto &MF = *MI.getMF();
4140 const auto &TLI = *MF.getSubtarget().getTargetLowering();
4141
4142 LLT PtrTy = MRI.getType(Dst);
4143 LLT IntPtrTy = LLT::scalar(PtrTy.getSizeInBits());
4144
4145 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
4146 auto SPTmp = MIRBuilder.buildCopy(PtrTy, SPReg);
4147 SPTmp = MIRBuilder.buildCast(IntPtrTy, SPTmp);
4148
4149 // Subtract the final alloc from the SP. We use G_PTRTOINT here so we don't
4150 // have to generate an extra instruction to negate the alloc and then use
4151 // G_GEP to add the negative offset.
4152 auto Alloc = MIRBuilder.buildSub(IntPtrTy, SPTmp, AllocSize);
4153 if (Align) {
4154 APInt AlignMask(IntPtrTy.getSizeInBits(), Align, true);
4155 AlignMask.negate();
4156 auto AlignCst = MIRBuilder.buildConstant(IntPtrTy, AlignMask);
4157 Alloc = MIRBuilder.buildAnd(IntPtrTy, Alloc, AlignCst);
4158 }
4159
4160 SPTmp = MIRBuilder.buildCast(PtrTy, Alloc);
4161 MIRBuilder.buildCopy(SPReg, SPTmp);
4162 MIRBuilder.buildCopy(Dst, SPTmp);
4163
4164 MI.eraseFromParent();
4165 return Legalized;
4166}
4167
4168LegalizerHelper::LegalizeResult
4169LegalizerHelper::lowerExtract(MachineInstr &MI) {
4170 Register Dst = MI.getOperand(0).getReg();
4171 Register Src = MI.getOperand(1).getReg();
4172 unsigned Offset = MI.getOperand(2).getImm();
4173
4174 LLT DstTy = MRI.getType(Dst);
4175 LLT SrcTy = MRI.getType(Src);
4176
4177 if (DstTy.isScalar() &&
4178 (SrcTy.isScalar() ||
4179 (SrcTy.isVector() && DstTy == SrcTy.getElementType()))) {
4180 LLT SrcIntTy = SrcTy;
4181 if (!SrcTy.isScalar()) {
4182 SrcIntTy = LLT::scalar(SrcTy.getSizeInBits());
4183 Src = MIRBuilder.buildBitcast(SrcIntTy, Src).getReg(0);
4184 }
4185
4186 if (Offset == 0)
4187 MIRBuilder.buildTrunc(Dst, Src);
4188 else {
4189 auto ShiftAmt = MIRBuilder.buildConstant(SrcIntTy, Offset);
4190 auto Shr = MIRBuilder.buildLShr(SrcIntTy, Src, ShiftAmt);
4191 MIRBuilder.buildTrunc(Dst, Shr);
4192 }
4193
4194 MI.eraseFromParent();
4195 return Legalized;
4196 }
4197
4198 return UnableToLegalize;
4199}
4200
4201LegalizerHelper::LegalizeResult LegalizerHelper::lowerInsert(MachineInstr &MI) {
4202 Register Dst = MI.getOperand(0).getReg();
4203 Register Src = MI.getOperand(1).getReg();
4204 Register InsertSrc = MI.getOperand(2).getReg();
4205 uint64_t Offset = MI.getOperand(3).getImm();
4206
4207 LLT DstTy = MRI.getType(Src);
4208 LLT InsertTy = MRI.getType(InsertSrc);
4209
4210 if (InsertTy.isScalar() &&
4211 (DstTy.isScalar() ||
4212 (DstTy.isVector() && DstTy.getElementType() == InsertTy))) {
4213 LLT IntDstTy = DstTy;
4214 if (!DstTy.isScalar()) {
4215 IntDstTy = LLT::scalar(DstTy.getSizeInBits());
4216 Src = MIRBuilder.buildBitcast(IntDstTy, Src).getReg(0);
4217 }
4218
4219 Register ExtInsSrc = MIRBuilder.buildZExt(IntDstTy, InsertSrc).getReg(0);
4220 if (Offset != 0) {
4221 auto ShiftAmt = MIRBuilder.buildConstant(IntDstTy, Offset);
4222 ExtInsSrc = MIRBuilder.buildShl(IntDstTy, ExtInsSrc, ShiftAmt).getReg(0);
4223 }
4224
4225 APInt MaskVal = ~APInt::getBitsSet(DstTy.getSizeInBits(), Offset,
4226 InsertTy.getSizeInBits());
4227
4228 auto Mask = MIRBuilder.buildConstant(IntDstTy, MaskVal);
4229 auto MaskedSrc = MIRBuilder.buildAnd(IntDstTy, Src, Mask);
4230 auto Or = MIRBuilder.buildOr(IntDstTy, MaskedSrc, ExtInsSrc);
4231
4232 MIRBuilder.buildBitcast(Dst, Or);
4233 MI.eraseFromParent();
4234 return Legalized;
4235 }
4236
4237 return UnableToLegalize;
4238}