Bug Summary

File:lib/CodeGen/GlobalISel/LegalizerHelper.cpp
Warning:line 2114, column 7
Value stored to 'NumLeftover' is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name LegalizerHelper.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-9/lib/clang/9.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-9~svn362543/build-llvm/lib/CodeGen/GlobalISel -I /build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel -I /build/llvm-toolchain-snapshot-9~svn362543/build-llvm/include -I /build/llvm-toolchain-snapshot-9~svn362543/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/include/clang/9.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-9/lib/clang/9.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-9~svn362543/build-llvm/lib/CodeGen/GlobalISel -fdebug-prefix-map=/build/llvm-toolchain-snapshot-9~svn362543=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2019-06-05-060531-1271-1 -x c++ /build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp -faddrsig
1//===-- llvm/CodeGen/GlobalISel/LegalizerHelper.cpp -----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This file implements the LegalizerHelper class to legalize
10/// individual instructions and the LegalizeMachineIR wrapper pass for the
11/// primary legalization.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
16#include "llvm/CodeGen/GlobalISel/CallLowering.h"
17#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
18#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
19#include "llvm/CodeGen/MachineRegisterInfo.h"
20#include "llvm/CodeGen/TargetInstrInfo.h"
21#include "llvm/CodeGen/TargetLowering.h"
22#include "llvm/CodeGen/TargetSubtargetInfo.h"
23#include "llvm/Support/Debug.h"
24#include "llvm/Support/MathExtras.h"
25#include "llvm/Support/raw_ostream.h"
26
27#define DEBUG_TYPE"legalizer" "legalizer"
28
29using namespace llvm;
30using namespace LegalizeActions;
31
32/// Try to break down \p OrigTy into \p NarrowTy sized pieces.
33///
34/// Returns the number of \p NarrowTy elements needed to reconstruct \p OrigTy,
35/// with any leftover piece as type \p LeftoverTy
36///
37/// Returns -1 in the first element of the pair if the breakdown is not
38/// satisfiable.
39static std::pair<int, int>
40getNarrowTypeBreakDown(LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy) {
41 assert(!LeftoverTy.isValid() && "this is an out argument")((!LeftoverTy.isValid() && "this is an out argument")
? static_cast<void> (0) : __assert_fail ("!LeftoverTy.isValid() && \"this is an out argument\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 41, __PRETTY_FUNCTION__))
;
42
43 unsigned Size = OrigTy.getSizeInBits();
44 unsigned NarrowSize = NarrowTy.getSizeInBits();
45 unsigned NumParts = Size / NarrowSize;
46 unsigned LeftoverSize = Size - NumParts * NarrowSize;
47 assert(Size > NarrowSize)((Size > NarrowSize) ? static_cast<void> (0) : __assert_fail
("Size > NarrowSize", "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 47, __PRETTY_FUNCTION__))
;
48
49 if (LeftoverSize == 0)
50 return {NumParts, 0};
51
52 if (NarrowTy.isVector()) {
53 unsigned EltSize = OrigTy.getScalarSizeInBits();
54 if (LeftoverSize % EltSize != 0)
55 return {-1, -1};
56 LeftoverTy = LLT::scalarOrVector(LeftoverSize / EltSize, EltSize);
57 } else {
58 LeftoverTy = LLT::scalar(LeftoverSize);
59 }
60
61 int NumLeftover = LeftoverSize / LeftoverTy.getSizeInBits();
62 return std::make_pair(NumParts, NumLeftover);
63}
64
65LegalizerHelper::LegalizerHelper(MachineFunction &MF,
66 GISelChangeObserver &Observer,
67 MachineIRBuilder &Builder)
68 : MIRBuilder(Builder), MRI(MF.getRegInfo()),
69 LI(*MF.getSubtarget().getLegalizerInfo()), Observer(Observer) {
70 MIRBuilder.setMF(MF);
71 MIRBuilder.setChangeObserver(Observer);
72}
73
74LegalizerHelper::LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI,
75 GISelChangeObserver &Observer,
76 MachineIRBuilder &B)
77 : MIRBuilder(B), MRI(MF.getRegInfo()), LI(LI), Observer(Observer) {
78 MIRBuilder.setMF(MF);
79 MIRBuilder.setChangeObserver(Observer);
80}
81LegalizerHelper::LegalizeResult
82LegalizerHelper::legalizeInstrStep(MachineInstr &MI) {
83 LLVM_DEBUG(dbgs() << "Legalizing: "; MI.print(dbgs()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << "Legalizing: "; MI.print(dbgs
()); } } while (false)
;
84
85 auto Step = LI.getAction(MI, MRI);
86 switch (Step.Action) {
87 case Legal:
88 LLVM_DEBUG(dbgs() << ".. Already legal\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Already legal\n"; } } while
(false)
;
89 return AlreadyLegal;
90 case Libcall:
91 LLVM_DEBUG(dbgs() << ".. Convert to libcall\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Convert to libcall\n"; }
} while (false)
;
92 return libcall(MI);
93 case NarrowScalar:
94 LLVM_DEBUG(dbgs() << ".. Narrow scalar\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Narrow scalar\n"; } } while
(false)
;
95 return narrowScalar(MI, Step.TypeIdx, Step.NewType);
96 case WidenScalar:
97 LLVM_DEBUG(dbgs() << ".. Widen scalar\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Widen scalar\n"; } } while
(false)
;
98 return widenScalar(MI, Step.TypeIdx, Step.NewType);
99 case Lower:
100 LLVM_DEBUG(dbgs() << ".. Lower\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Lower\n"; } } while (false
)
;
101 return lower(MI, Step.TypeIdx, Step.NewType);
102 case FewerElements:
103 LLVM_DEBUG(dbgs() << ".. Reduce number of elements\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Reduce number of elements\n"
; } } while (false)
;
104 return fewerElementsVector(MI, Step.TypeIdx, Step.NewType);
105 case MoreElements:
106 LLVM_DEBUG(dbgs() << ".. Increase number of elements\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Increase number of elements\n"
; } } while (false)
;
107 return moreElementsVector(MI, Step.TypeIdx, Step.NewType);
108 case Custom:
109 LLVM_DEBUG(dbgs() << ".. Custom legalization\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Custom legalization\n"; }
} while (false)
;
110 return LI.legalizeCustom(MI, MRI, MIRBuilder, Observer) ? Legalized
111 : UnableToLegalize;
112 default:
113 LLVM_DEBUG(dbgs() << ".. Unable to legalize\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << ".. Unable to legalize\n"; }
} while (false)
;
114 return UnableToLegalize;
115 }
116}
117
118void LegalizerHelper::extractParts(unsigned Reg, LLT Ty, int NumParts,
119 SmallVectorImpl<unsigned> &VRegs) {
120 for (int i = 0; i < NumParts; ++i)
121 VRegs.push_back(MRI.createGenericVirtualRegister(Ty));
122 MIRBuilder.buildUnmerge(VRegs, Reg);
123}
124
125bool LegalizerHelper::extractParts(unsigned Reg, LLT RegTy,
126 LLT MainTy, LLT &LeftoverTy,
127 SmallVectorImpl<unsigned> &VRegs,
128 SmallVectorImpl<unsigned> &LeftoverRegs) {
129 assert(!LeftoverTy.isValid() && "this is an out argument")((!LeftoverTy.isValid() && "this is an out argument")
? static_cast<void> (0) : __assert_fail ("!LeftoverTy.isValid() && \"this is an out argument\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 129, __PRETTY_FUNCTION__))
;
130
131 unsigned RegSize = RegTy.getSizeInBits();
132 unsigned MainSize = MainTy.getSizeInBits();
133 unsigned NumParts = RegSize / MainSize;
134 unsigned LeftoverSize = RegSize - NumParts * MainSize;
135
136 // Use an unmerge when possible.
137 if (LeftoverSize == 0) {
138 for (unsigned I = 0; I < NumParts; ++I)
139 VRegs.push_back(MRI.createGenericVirtualRegister(MainTy));
140 MIRBuilder.buildUnmerge(VRegs, Reg);
141 return true;
142 }
143
144 if (MainTy.isVector()) {
145 unsigned EltSize = MainTy.getScalarSizeInBits();
146 if (LeftoverSize % EltSize != 0)
147 return false;
148 LeftoverTy = LLT::scalarOrVector(LeftoverSize / EltSize, EltSize);
149 } else {
150 LeftoverTy = LLT::scalar(LeftoverSize);
151 }
152
153 // For irregular sizes, extract the individual parts.
154 for (unsigned I = 0; I != NumParts; ++I) {
155 unsigned NewReg = MRI.createGenericVirtualRegister(MainTy);
156 VRegs.push_back(NewReg);
157 MIRBuilder.buildExtract(NewReg, Reg, MainSize * I);
158 }
159
160 for (unsigned Offset = MainSize * NumParts; Offset < RegSize;
161 Offset += LeftoverSize) {
162 unsigned NewReg = MRI.createGenericVirtualRegister(LeftoverTy);
163 LeftoverRegs.push_back(NewReg);
164 MIRBuilder.buildExtract(NewReg, Reg, Offset);
165 }
166
167 return true;
168}
169
170void LegalizerHelper::insertParts(unsigned DstReg,
171 LLT ResultTy, LLT PartTy,
172 ArrayRef<unsigned> PartRegs,
173 LLT LeftoverTy,
174 ArrayRef<unsigned> LeftoverRegs) {
175 if (!LeftoverTy.isValid()) {
176 assert(LeftoverRegs.empty())((LeftoverRegs.empty()) ? static_cast<void> (0) : __assert_fail
("LeftoverRegs.empty()", "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 176, __PRETTY_FUNCTION__))
;
177
178 if (!ResultTy.isVector()) {
179 MIRBuilder.buildMerge(DstReg, PartRegs);
180 return;
181 }
182
183 if (PartTy.isVector())
184 MIRBuilder.buildConcatVectors(DstReg, PartRegs);
185 else
186 MIRBuilder.buildBuildVector(DstReg, PartRegs);
187 return;
188 }
189
190 unsigned PartSize = PartTy.getSizeInBits();
191 unsigned LeftoverPartSize = LeftoverTy.getSizeInBits();
192
193 unsigned CurResultReg = MRI.createGenericVirtualRegister(ResultTy);
194 MIRBuilder.buildUndef(CurResultReg);
195
196 unsigned Offset = 0;
197 for (unsigned PartReg : PartRegs) {
198 unsigned NewResultReg = MRI.createGenericVirtualRegister(ResultTy);
199 MIRBuilder.buildInsert(NewResultReg, CurResultReg, PartReg, Offset);
200 CurResultReg = NewResultReg;
201 Offset += PartSize;
202 }
203
204 for (unsigned I = 0, E = LeftoverRegs.size(); I != E; ++I) {
205 // Use the original output register for the final insert to avoid a copy.
206 unsigned NewResultReg = (I + 1 == E) ?
207 DstReg : MRI.createGenericVirtualRegister(ResultTy);
208
209 MIRBuilder.buildInsert(NewResultReg, CurResultReg, LeftoverRegs[I], Offset);
210 CurResultReg = NewResultReg;
211 Offset += LeftoverPartSize;
212 }
213}
214
215static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
216 switch (Opcode) {
217 case TargetOpcode::G_SDIV:
218 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 218, __PRETTY_FUNCTION__))
;
219 return Size == 64 ? RTLIB::SDIV_I64 : RTLIB::SDIV_I32;
220 case TargetOpcode::G_UDIV:
221 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 221, __PRETTY_FUNCTION__))
;
222 return Size == 64 ? RTLIB::UDIV_I64 : RTLIB::UDIV_I32;
223 case TargetOpcode::G_SREM:
224 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 224, __PRETTY_FUNCTION__))
;
225 return Size == 64 ? RTLIB::SREM_I64 : RTLIB::SREM_I32;
226 case TargetOpcode::G_UREM:
227 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 227, __PRETTY_FUNCTION__))
;
228 return Size == 64 ? RTLIB::UREM_I64 : RTLIB::UREM_I32;
229 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
230 assert(Size == 32 && "Unsupported size")((Size == 32 && "Unsupported size") ? static_cast<
void> (0) : __assert_fail ("Size == 32 && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 230, __PRETTY_FUNCTION__))
;
231 return RTLIB::CTLZ_I32;
232 case TargetOpcode::G_FADD:
233 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 233, __PRETTY_FUNCTION__))
;
234 return Size == 64 ? RTLIB::ADD_F64 : RTLIB::ADD_F32;
235 case TargetOpcode::G_FSUB:
236 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 236, __PRETTY_FUNCTION__))
;
237 return Size == 64 ? RTLIB::SUB_F64 : RTLIB::SUB_F32;
238 case TargetOpcode::G_FMUL:
239 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 239, __PRETTY_FUNCTION__))
;
240 return Size == 64 ? RTLIB::MUL_F64 : RTLIB::MUL_F32;
241 case TargetOpcode::G_FDIV:
242 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 242, __PRETTY_FUNCTION__))
;
243 return Size == 64 ? RTLIB::DIV_F64 : RTLIB::DIV_F32;
244 case TargetOpcode::G_FEXP:
245 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 245, __PRETTY_FUNCTION__))
;
246 return Size == 64 ? RTLIB::EXP_F64 : RTLIB::EXP_F32;
247 case TargetOpcode::G_FEXP2:
248 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 248, __PRETTY_FUNCTION__))
;
249 return Size == 64 ? RTLIB::EXP2_F64 : RTLIB::EXP2_F32;
250 case TargetOpcode::G_FREM:
251 return Size == 64 ? RTLIB::REM_F64 : RTLIB::REM_F32;
252 case TargetOpcode::G_FPOW:
253 return Size == 64 ? RTLIB::POW_F64 : RTLIB::POW_F32;
254 case TargetOpcode::G_FMA:
255 assert((Size == 32 || Size == 64) && "Unsupported size")(((Size == 32 || Size == 64) && "Unsupported size") ?
static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 255, __PRETTY_FUNCTION__))
;
256 return Size == 64 ? RTLIB::FMA_F64 : RTLIB::FMA_F32;
257 case TargetOpcode::G_FSIN:
258 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size")(((Size == 32 || Size == 64 || Size == 128) && "Unsupported size"
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64 || Size == 128) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 258, __PRETTY_FUNCTION__))
;
259 return Size == 128 ? RTLIB::SIN_F128
260 : Size == 64 ? RTLIB::SIN_F64 : RTLIB::SIN_F32;
261 case TargetOpcode::G_FCOS:
262 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size")(((Size == 32 || Size == 64 || Size == 128) && "Unsupported size"
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64 || Size == 128) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 262, __PRETTY_FUNCTION__))
;
263 return Size == 128 ? RTLIB::COS_F128
264 : Size == 64 ? RTLIB::COS_F64 : RTLIB::COS_F32;
265 case TargetOpcode::G_FLOG10:
266 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size")(((Size == 32 || Size == 64 || Size == 128) && "Unsupported size"
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64 || Size == 128) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 266, __PRETTY_FUNCTION__))
;
267 return Size == 128 ? RTLIB::LOG10_F128
268 : Size == 64 ? RTLIB::LOG10_F64 : RTLIB::LOG10_F32;
269 case TargetOpcode::G_FLOG:
270 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size")(((Size == 32 || Size == 64 || Size == 128) && "Unsupported size"
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64 || Size == 128) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 270, __PRETTY_FUNCTION__))
;
271 return Size == 128 ? RTLIB::LOG_F128
272 : Size == 64 ? RTLIB::LOG_F64 : RTLIB::LOG_F32;
273 case TargetOpcode::G_FLOG2:
274 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size")(((Size == 32 || Size == 64 || Size == 128) && "Unsupported size"
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64 || Size == 128) && \"Unsupported size\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 274, __PRETTY_FUNCTION__))
;
275 return Size == 128 ? RTLIB::LOG2_F128
276 : Size == 64 ? RTLIB::LOG2_F64 : RTLIB::LOG2_F32;
277 }
278 llvm_unreachable("Unknown libcall function")::llvm::llvm_unreachable_internal("Unknown libcall function",
"/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 278)
;
279}
280
281LegalizerHelper::LegalizeResult
282llvm::createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall,
283 const CallLowering::ArgInfo &Result,
284 ArrayRef<CallLowering::ArgInfo> Args) {
285 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
286 auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
287 const char *Name = TLI.getLibcallName(Libcall);
288
289 MIRBuilder.getMF().getFrameInfo().setHasCalls(true);
290 if (!CLI.lowerCall(MIRBuilder, TLI.getLibcallCallingConv(Libcall),
291 MachineOperand::CreateES(Name), Result, Args))
292 return LegalizerHelper::UnableToLegalize;
293
294 return LegalizerHelper::Legalized;
295}
296
297// Useful for libcalls where all operands have the same type.
298static LegalizerHelper::LegalizeResult
299simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size,
300 Type *OpType) {
301 auto Libcall = getRTLibDesc(MI.getOpcode(), Size);
302
303 SmallVector<CallLowering::ArgInfo, 3> Args;
304 for (unsigned i = 1; i < MI.getNumOperands(); i++)
305 Args.push_back({MI.getOperand(i).getReg(), OpType});
306 return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), OpType},
307 Args);
308}
309
310static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType,
311 Type *FromType) {
312 auto ToMVT = MVT::getVT(ToType);
313 auto FromMVT = MVT::getVT(FromType);
314
315 switch (Opcode) {
316 case TargetOpcode::G_FPEXT:
317 return RTLIB::getFPEXT(FromMVT, ToMVT);
318 case TargetOpcode::G_FPTRUNC:
319 return RTLIB::getFPROUND(FromMVT, ToMVT);
320 case TargetOpcode::G_FPTOSI:
321 return RTLIB::getFPTOSINT(FromMVT, ToMVT);
322 case TargetOpcode::G_FPTOUI:
323 return RTLIB::getFPTOUINT(FromMVT, ToMVT);
324 case TargetOpcode::G_SITOFP:
325 return RTLIB::getSINTTOFP(FromMVT, ToMVT);
326 case TargetOpcode::G_UITOFP:
327 return RTLIB::getUINTTOFP(FromMVT, ToMVT);
328 }
329 llvm_unreachable("Unsupported libcall function")::llvm::llvm_unreachable_internal("Unsupported libcall function"
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 329)
;
330}
331
332static LegalizerHelper::LegalizeResult
333conversionLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, Type *ToType,
334 Type *FromType) {
335 RTLIB::Libcall Libcall = getConvRTLibDesc(MI.getOpcode(), ToType, FromType);
336 return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), ToType},
337 {{MI.getOperand(1).getReg(), FromType}});
338}
339
340LegalizerHelper::LegalizeResult
341LegalizerHelper::libcall(MachineInstr &MI) {
342 LLT LLTy = MRI.getType(MI.getOperand(0).getReg());
343 unsigned Size = LLTy.getSizeInBits();
344 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
345
346 MIRBuilder.setInstr(MI);
347
348 switch (MI.getOpcode()) {
349 default:
350 return UnableToLegalize;
351 case TargetOpcode::G_SDIV:
352 case TargetOpcode::G_UDIV:
353 case TargetOpcode::G_SREM:
354 case TargetOpcode::G_UREM:
355 case TargetOpcode::G_CTLZ_ZERO_UNDEF: {
356 Type *HLTy = IntegerType::get(Ctx, Size);
357 auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
358 if (Status != Legalized)
359 return Status;
360 break;
361 }
362 case TargetOpcode::G_FADD:
363 case TargetOpcode::G_FSUB:
364 case TargetOpcode::G_FMUL:
365 case TargetOpcode::G_FDIV:
366 case TargetOpcode::G_FMA:
367 case TargetOpcode::G_FPOW:
368 case TargetOpcode::G_FREM:
369 case TargetOpcode::G_FCOS:
370 case TargetOpcode::G_FSIN:
371 case TargetOpcode::G_FLOG10:
372 case TargetOpcode::G_FLOG:
373 case TargetOpcode::G_FLOG2:
374 case TargetOpcode::G_FEXP:
375 case TargetOpcode::G_FEXP2: {
376 if (Size > 64) {
377 LLVM_DEBUG(dbgs() << "Size " << Size << " too large to legalize.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizer")) { dbgs() << "Size " << Size <<
" too large to legalize.\n"; } } while (false)
;
378 return UnableToLegalize;
379 }
380 Type *HLTy = Size == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx);
381 auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
382 if (Status != Legalized)
383 return Status;
384 break;
385 }
386 case TargetOpcode::G_FPEXT: {
387 // FIXME: Support other floating point types (half, fp128 etc)
388 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
389 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
390 if (ToSize != 64 || FromSize != 32)
391 return UnableToLegalize;
392 LegalizeResult Status = conversionLibcall(
393 MI, MIRBuilder, Type::getDoubleTy(Ctx), Type::getFloatTy(Ctx));
394 if (Status != Legalized)
395 return Status;
396 break;
397 }
398 case TargetOpcode::G_FPTRUNC: {
399 // FIXME: Support other floating point types (half, fp128 etc)
400 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
401 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
402 if (ToSize != 32 || FromSize != 64)
403 return UnableToLegalize;
404 LegalizeResult Status = conversionLibcall(
405 MI, MIRBuilder, Type::getFloatTy(Ctx), Type::getDoubleTy(Ctx));
406 if (Status != Legalized)
407 return Status;
408 break;
409 }
410 case TargetOpcode::G_FPTOSI:
411 case TargetOpcode::G_FPTOUI: {
412 // FIXME: Support other types
413 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
414 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
415 if (ToSize != 32 || (FromSize != 32 && FromSize != 64))
416 return UnableToLegalize;
417 LegalizeResult Status = conversionLibcall(
418 MI, MIRBuilder, Type::getInt32Ty(Ctx),
419 FromSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx));
420 if (Status != Legalized)
421 return Status;
422 break;
423 }
424 case TargetOpcode::G_SITOFP:
425 case TargetOpcode::G_UITOFP: {
426 // FIXME: Support other types
427 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
428 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
429 if (FromSize != 32 || (ToSize != 32 && ToSize != 64))
430 return UnableToLegalize;
431 LegalizeResult Status = conversionLibcall(
432 MI, MIRBuilder,
433 ToSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx),
434 Type::getInt32Ty(Ctx));
435 if (Status != Legalized)
436 return Status;
437 break;
438 }
439 }
440
441 MI.eraseFromParent();
442 return Legalized;
443}
444
445LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
446 unsigned TypeIdx,
447 LLT NarrowTy) {
448 MIRBuilder.setInstr(MI);
449
450 uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
451 uint64_t NarrowSize = NarrowTy.getSizeInBits();
452
453 switch (MI.getOpcode()) {
454 default:
455 return UnableToLegalize;
456 case TargetOpcode::G_IMPLICIT_DEF: {
457 // FIXME: add support for when SizeOp0 isn't an exact multiple of
458 // NarrowSize.
459 if (SizeOp0 % NarrowSize != 0)
460 return UnableToLegalize;
461 int NumParts = SizeOp0 / NarrowSize;
462
463 SmallVector<unsigned, 2> DstRegs;
464 for (int i = 0; i < NumParts; ++i)
465 DstRegs.push_back(
466 MIRBuilder.buildUndef(NarrowTy)->getOperand(0).getReg());
467
468 unsigned DstReg = MI.getOperand(0).getReg();
469 if(MRI.getType(DstReg).isVector())
470 MIRBuilder.buildBuildVector(DstReg, DstRegs);
471 else
472 MIRBuilder.buildMerge(DstReg, DstRegs);
473 MI.eraseFromParent();
474 return Legalized;
475 }
476 case TargetOpcode::G_CONSTANT: {
477 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
478 const APInt &Val = MI.getOperand(1).getCImm()->getValue();
479 unsigned TotalSize = Ty.getSizeInBits();
480 unsigned NarrowSize = NarrowTy.getSizeInBits();
481 int NumParts = TotalSize / NarrowSize;
482
483 SmallVector<unsigned, 4> PartRegs;
484 for (int I = 0; I != NumParts; ++I) {
485 unsigned Offset = I * NarrowSize;
486 auto K = MIRBuilder.buildConstant(NarrowTy,
487 Val.lshr(Offset).trunc(NarrowSize));
488 PartRegs.push_back(K.getReg(0));
489 }
490
491 LLT LeftoverTy;
492 unsigned LeftoverBits = TotalSize - NumParts * NarrowSize;
493 SmallVector<unsigned, 1> LeftoverRegs;
494 if (LeftoverBits != 0) {
495 LeftoverTy = LLT::scalar(LeftoverBits);
496 auto K = MIRBuilder.buildConstant(
497 LeftoverTy,
498 Val.lshr(NumParts * NarrowSize).trunc(LeftoverBits));
499 LeftoverRegs.push_back(K.getReg(0));
500 }
501
502 insertParts(MI.getOperand(0).getReg(),
503 Ty, NarrowTy, PartRegs, LeftoverTy, LeftoverRegs);
504
505 MI.eraseFromParent();
506 return Legalized;
507 }
508 case TargetOpcode::G_ADD: {
509 // FIXME: add support for when SizeOp0 isn't an exact multiple of
510 // NarrowSize.
511 if (SizeOp0 % NarrowSize != 0)
512 return UnableToLegalize;
513 // Expand in terms of carry-setting/consuming G_ADDE instructions.
514 int NumParts = SizeOp0 / NarrowTy.getSizeInBits();
515
516 SmallVector<unsigned, 2> Src1Regs, Src2Regs, DstRegs;
517 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
518 extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
519
520 unsigned CarryIn = MRI.createGenericVirtualRegister(LLT::scalar(1));
521 MIRBuilder.buildConstant(CarryIn, 0);
522
523 for (int i = 0; i < NumParts; ++i) {
524 unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
525 unsigned CarryOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
526
527 MIRBuilder.buildUAdde(DstReg, CarryOut, Src1Regs[i],
528 Src2Regs[i], CarryIn);
529
530 DstRegs.push_back(DstReg);
531 CarryIn = CarryOut;
532 }
533 unsigned DstReg = MI.getOperand(0).getReg();
534 if(MRI.getType(DstReg).isVector())
535 MIRBuilder.buildBuildVector(DstReg, DstRegs);
536 else
537 MIRBuilder.buildMerge(DstReg, DstRegs);
538 MI.eraseFromParent();
539 return Legalized;
540 }
541 case TargetOpcode::G_SUB: {
542 // FIXME: add support for when SizeOp0 isn't an exact multiple of
543 // NarrowSize.
544 if (SizeOp0 % NarrowSize != 0)
545 return UnableToLegalize;
546
547 int NumParts = SizeOp0 / NarrowTy.getSizeInBits();
548
549 SmallVector<unsigned, 2> Src1Regs, Src2Regs, DstRegs;
550 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
551 extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
552
553 unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
554 unsigned BorrowOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
555 MIRBuilder.buildInstr(TargetOpcode::G_USUBO, {DstReg, BorrowOut},
556 {Src1Regs[0], Src2Regs[0]});
557 DstRegs.push_back(DstReg);
558 unsigned BorrowIn = BorrowOut;
559 for (int i = 1; i < NumParts; ++i) {
560 DstReg = MRI.createGenericVirtualRegister(NarrowTy);
561 BorrowOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
562
563 MIRBuilder.buildInstr(TargetOpcode::G_USUBE, {DstReg, BorrowOut},
564 {Src1Regs[i], Src2Regs[i], BorrowIn});
565
566 DstRegs.push_back(DstReg);
567 BorrowIn = BorrowOut;
568 }
569 MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
570 MI.eraseFromParent();
571 return Legalized;
572 }
573 case TargetOpcode::G_MUL:
574 case TargetOpcode::G_UMULH:
575 return narrowScalarMul(MI, NarrowTy);
576 case TargetOpcode::G_EXTRACT:
577 return narrowScalarExtract(MI, TypeIdx, NarrowTy);
578 case TargetOpcode::G_INSERT:
579 return narrowScalarInsert(MI, TypeIdx, NarrowTy);
580 case TargetOpcode::G_LOAD: {
581 const auto &MMO = **MI.memoperands_begin();
582 unsigned DstReg = MI.getOperand(0).getReg();
583 LLT DstTy = MRI.getType(DstReg);
584 if (DstTy.isVector())
585 return UnableToLegalize;
586
587 if (8 * MMO.getSize() != DstTy.getSizeInBits()) {
588 unsigned TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
589 auto &MMO = **MI.memoperands_begin();
590 MIRBuilder.buildLoad(TmpReg, MI.getOperand(1).getReg(), MMO);
591 MIRBuilder.buildAnyExt(DstReg, TmpReg);
592 MI.eraseFromParent();
593 return Legalized;
594 }
595
596 return reduceLoadStoreWidth(MI, TypeIdx, NarrowTy);
597 }
598 case TargetOpcode::G_ZEXTLOAD:
599 case TargetOpcode::G_SEXTLOAD: {
600 bool ZExt = MI.getOpcode() == TargetOpcode::G_ZEXTLOAD;
601 unsigned DstReg = MI.getOperand(0).getReg();
602 unsigned PtrReg = MI.getOperand(1).getReg();
603
604 unsigned TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
605 auto &MMO = **MI.memoperands_begin();
606 if (MMO.getSizeInBits() == NarrowSize) {
607 MIRBuilder.buildLoad(TmpReg, PtrReg, MMO);
608 } else {
609 unsigned ExtLoad = ZExt ? TargetOpcode::G_ZEXTLOAD
610 : TargetOpcode::G_SEXTLOAD;
611 MIRBuilder.buildInstr(ExtLoad)
612 .addDef(TmpReg)
613 .addUse(PtrReg)
614 .addMemOperand(&MMO);
615 }
616
617 if (ZExt)
618 MIRBuilder.buildZExt(DstReg, TmpReg);
619 else
620 MIRBuilder.buildSExt(DstReg, TmpReg);
621
622 MI.eraseFromParent();
623 return Legalized;
624 }
625 case TargetOpcode::G_STORE: {
626 const auto &MMO = **MI.memoperands_begin();
627
628 unsigned SrcReg = MI.getOperand(0).getReg();
629 LLT SrcTy = MRI.getType(SrcReg);
630 if (SrcTy.isVector())
631 return UnableToLegalize;
632
633 int NumParts = SizeOp0 / NarrowSize;
634 unsigned HandledSize = NumParts * NarrowTy.getSizeInBits();
635 unsigned LeftoverBits = SrcTy.getSizeInBits() - HandledSize;
636 if (SrcTy.isVector() && LeftoverBits != 0)
637 return UnableToLegalize;
638
639 if (8 * MMO.getSize() != SrcTy.getSizeInBits()) {
640 unsigned TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
641 auto &MMO = **MI.memoperands_begin();
642 MIRBuilder.buildTrunc(TmpReg, SrcReg);
643 MIRBuilder.buildStore(TmpReg, MI.getOperand(1).getReg(), MMO);
644 MI.eraseFromParent();
645 return Legalized;
646 }
647
648 return reduceLoadStoreWidth(MI, 0, NarrowTy);
649 }
650 case TargetOpcode::G_SELECT:
651 return narrowScalarSelect(MI, TypeIdx, NarrowTy);
652 case TargetOpcode::G_AND:
653 case TargetOpcode::G_OR:
654 case TargetOpcode::G_XOR: {
655 // Legalize bitwise operation:
656 // A = BinOp<Ty> B, C
657 // into:
658 // B1, ..., BN = G_UNMERGE_VALUES B
659 // C1, ..., CN = G_UNMERGE_VALUES C
660 // A1 = BinOp<Ty/N> B1, C2
661 // ...
662 // AN = BinOp<Ty/N> BN, CN
663 // A = G_MERGE_VALUES A1, ..., AN
664 return narrowScalarBasic(MI, TypeIdx, NarrowTy);
665 }
666 case TargetOpcode::G_SHL:
667 case TargetOpcode::G_LSHR:
668 case TargetOpcode::G_ASHR:
669 return narrowScalarShift(MI, TypeIdx, NarrowTy);
670 case TargetOpcode::G_CTLZ:
671 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
672 case TargetOpcode::G_CTTZ:
673 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
674 case TargetOpcode::G_CTPOP:
675 if (TypeIdx != 0)
676 return UnableToLegalize; // TODO
677
678 Observer.changingInstr(MI);
679 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT);
680 Observer.changedInstr(MI);
681 return Legalized;
682 case TargetOpcode::G_INTTOPTR:
683 if (TypeIdx != 1)
684 return UnableToLegalize;
685
686 Observer.changingInstr(MI);
687 narrowScalarSrc(MI, NarrowTy, 1);
688 Observer.changedInstr(MI);
689 return Legalized;
690 case TargetOpcode::G_PTRTOINT:
691 if (TypeIdx != 0)
692 return UnableToLegalize;
693
694 Observer.changingInstr(MI);
695 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT);
696 Observer.changedInstr(MI);
697 return Legalized;
698 }
699}
700
701void LegalizerHelper::widenScalarSrc(MachineInstr &MI, LLT WideTy,
702 unsigned OpIdx, unsigned ExtOpcode) {
703 MachineOperand &MO = MI.getOperand(OpIdx);
704 auto ExtB = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MO.getReg()});
705 MO.setReg(ExtB->getOperand(0).getReg());
706}
707
708void LegalizerHelper::narrowScalarSrc(MachineInstr &MI, LLT NarrowTy,
709 unsigned OpIdx) {
710 MachineOperand &MO = MI.getOperand(OpIdx);
711 auto ExtB = MIRBuilder.buildInstr(TargetOpcode::G_TRUNC, {NarrowTy},
712 {MO.getReg()});
713 MO.setReg(ExtB->getOperand(0).getReg());
714}
715
716void LegalizerHelper::widenScalarDst(MachineInstr &MI, LLT WideTy,
717 unsigned OpIdx, unsigned TruncOpcode) {
718 MachineOperand &MO = MI.getOperand(OpIdx);
719 unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
720 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
721 MIRBuilder.buildInstr(TruncOpcode, {MO.getReg()}, {DstExt});
722 MO.setReg(DstExt);
723}
724
725void LegalizerHelper::narrowScalarDst(MachineInstr &MI, LLT NarrowTy,
726 unsigned OpIdx, unsigned ExtOpcode) {
727 MachineOperand &MO = MI.getOperand(OpIdx);
728 unsigned DstTrunc = MRI.createGenericVirtualRegister(NarrowTy);
729 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
730 MIRBuilder.buildInstr(ExtOpcode, {MO.getReg()}, {DstTrunc});
731 MO.setReg(DstTrunc);
732}
733
734void LegalizerHelper::moreElementsVectorDst(MachineInstr &MI, LLT WideTy,
735 unsigned OpIdx) {
736 MachineOperand &MO = MI.getOperand(OpIdx);
737 unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
738 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
739 MIRBuilder.buildExtract(MO.getReg(), DstExt, 0);
740 MO.setReg(DstExt);
741}
742
743void LegalizerHelper::moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy,
744 unsigned OpIdx) {
745 MachineOperand &MO = MI.getOperand(OpIdx);
746
747 LLT OldTy = MRI.getType(MO.getReg());
748 unsigned OldElts = OldTy.getNumElements();
749 unsigned NewElts = MoreTy.getNumElements();
750
751 unsigned NumParts = NewElts / OldElts;
752
753 // Use concat_vectors if the result is a multiple of the number of elements.
754 if (NumParts * OldElts == NewElts) {
755 SmallVector<unsigned, 8> Parts;
756 Parts.push_back(MO.getReg());
757
758 unsigned ImpDef = MIRBuilder.buildUndef(OldTy).getReg(0);
759 for (unsigned I = 1; I != NumParts; ++I)
760 Parts.push_back(ImpDef);
761
762 auto Concat = MIRBuilder.buildConcatVectors(MoreTy, Parts);
763 MO.setReg(Concat.getReg(0));
764 return;
765 }
766
767 unsigned MoreReg = MRI.createGenericVirtualRegister(MoreTy);
768 unsigned ImpDef = MIRBuilder.buildUndef(MoreTy).getReg(0);
769 MIRBuilder.buildInsert(MoreReg, ImpDef, MO.getReg(), 0);
770 MO.setReg(MoreReg);
771}
772
773LegalizerHelper::LegalizeResult
774LegalizerHelper::widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx,
775 LLT WideTy) {
776 if (TypeIdx != 1)
777 return UnableToLegalize;
778
779 unsigned DstReg = MI.getOperand(0).getReg();
780 LLT DstTy = MRI.getType(DstReg);
781 if (!DstTy.isScalar())
782 return UnableToLegalize;
783
784 unsigned NumOps = MI.getNumOperands();
785 unsigned NumSrc = MI.getNumOperands() - 1;
786 unsigned PartSize = DstTy.getSizeInBits() / NumSrc;
787
788 unsigned Src1 = MI.getOperand(1).getReg();
789 unsigned ResultReg = MIRBuilder.buildZExt(DstTy, Src1)->getOperand(0).getReg();
790
791 for (unsigned I = 2; I != NumOps; ++I) {
792 const unsigned Offset = (I - 1) * PartSize;
793
794 unsigned SrcReg = MI.getOperand(I).getReg();
795 assert(MRI.getType(SrcReg) == LLT::scalar(PartSize))((MRI.getType(SrcReg) == LLT::scalar(PartSize)) ? static_cast
<void> (0) : __assert_fail ("MRI.getType(SrcReg) == LLT::scalar(PartSize)"
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 795, __PRETTY_FUNCTION__))
;
796
797 auto ZextInput = MIRBuilder.buildZExt(DstTy, SrcReg);
798
799 unsigned NextResult = I + 1 == NumOps ? DstReg :
800 MRI.createGenericVirtualRegister(DstTy);
801
802 auto ShiftAmt = MIRBuilder.buildConstant(DstTy, Offset);
803 auto Shl = MIRBuilder.buildShl(DstTy, ZextInput, ShiftAmt);
804 MIRBuilder.buildOr(NextResult, ResultReg, Shl);
805 ResultReg = NextResult;
806 }
807
808 MI.eraseFromParent();
809 return Legalized;
810}
811
812LegalizerHelper::LegalizeResult
813LegalizerHelper::widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx,
814 LLT WideTy) {
815 if (TypeIdx != 0)
816 return UnableToLegalize;
817
818 unsigned NumDst = MI.getNumOperands() - 1;
819 unsigned SrcReg = MI.getOperand(NumDst).getReg();
820 LLT SrcTy = MRI.getType(SrcReg);
821 if (!SrcTy.isScalar())
822 return UnableToLegalize;
823
824 unsigned Dst0Reg = MI.getOperand(0).getReg();
825 LLT DstTy = MRI.getType(Dst0Reg);
826 if (!DstTy.isScalar())
827 return UnableToLegalize;
828
829 unsigned NewSrcSize = NumDst * WideTy.getSizeInBits();
830 LLT NewSrcTy = LLT::scalar(NewSrcSize);
831 unsigned SizeDiff = WideTy.getSizeInBits() - DstTy.getSizeInBits();
832
833 auto WideSrc = MIRBuilder.buildZExt(NewSrcTy, SrcReg);
834
835 for (unsigned I = 1; I != NumDst; ++I) {
836 auto ShiftAmt = MIRBuilder.buildConstant(NewSrcTy, SizeDiff * I);
837 auto Shl = MIRBuilder.buildShl(NewSrcTy, WideSrc, ShiftAmt);
838 WideSrc = MIRBuilder.buildOr(NewSrcTy, WideSrc, Shl);
839 }
840
841 Observer.changingInstr(MI);
842
843 MI.getOperand(NumDst).setReg(WideSrc->getOperand(0).getReg());
844 for (unsigned I = 0; I != NumDst; ++I)
845 widenScalarDst(MI, WideTy, I);
846
847 Observer.changedInstr(MI);
848
849 return Legalized;
850}
851
852LegalizerHelper::LegalizeResult
853LegalizerHelper::widenScalarExtract(MachineInstr &MI, unsigned TypeIdx,
854 LLT WideTy) {
855 unsigned DstReg = MI.getOperand(0).getReg();
856 unsigned SrcReg = MI.getOperand(1).getReg();
857 LLT SrcTy = MRI.getType(SrcReg);
858
859 LLT DstTy = MRI.getType(DstReg);
860 unsigned Offset = MI.getOperand(2).getImm();
861
862 if (TypeIdx == 0) {
863 if (SrcTy.isVector() || DstTy.isVector())
864 return UnableToLegalize;
865
866 SrcOp Src(SrcReg);
867 if (SrcTy.isPointer()) {
868 // Extracts from pointers can be handled only if they are really just
869 // simple integers.
870 const DataLayout &DL = MIRBuilder.getDataLayout();
871 if (DL.isNonIntegralAddressSpace(SrcTy.getAddressSpace()))
872 return UnableToLegalize;
873
874 LLT SrcAsIntTy = LLT::scalar(SrcTy.getSizeInBits());
875 Src = MIRBuilder.buildPtrToInt(SrcAsIntTy, Src);
876 SrcTy = SrcAsIntTy;
877 }
878
879 if (DstTy.isPointer())
880 return UnableToLegalize;
881
882 if (Offset == 0) {
883 // Avoid a shift in the degenerate case.
884 MIRBuilder.buildTrunc(DstReg,
885 MIRBuilder.buildAnyExtOrTrunc(WideTy, Src));
886 MI.eraseFromParent();
887 return Legalized;
888 }
889
890 // Do a shift in the source type.
891 LLT ShiftTy = SrcTy;
892 if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) {
893 Src = MIRBuilder.buildAnyExt(WideTy, Src);
894 ShiftTy = WideTy;
895 } else if (WideTy.getSizeInBits() > SrcTy.getSizeInBits())
896 return UnableToLegalize;
897
898 auto LShr = MIRBuilder.buildLShr(
899 ShiftTy, Src, MIRBuilder.buildConstant(ShiftTy, Offset));
900 MIRBuilder.buildTrunc(DstReg, LShr);
901 MI.eraseFromParent();
902 return Legalized;
903 }
904
905 if (SrcTy.isScalar()) {
906 Observer.changingInstr(MI);
907 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
908 Observer.changedInstr(MI);
909 return Legalized;
910 }
911
912 if (!SrcTy.isVector())
913 return UnableToLegalize;
914
915 if (DstTy != SrcTy.getElementType())
916 return UnableToLegalize;
917
918 if (Offset % SrcTy.getScalarSizeInBits() != 0)
919 return UnableToLegalize;
920
921 Observer.changingInstr(MI);
922 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
923
924 MI.getOperand(2).setImm((WideTy.getSizeInBits() / SrcTy.getSizeInBits()) *
925 Offset);
926 widenScalarDst(MI, WideTy.getScalarType(), 0);
927 Observer.changedInstr(MI);
928 return Legalized;
929}
930
931LegalizerHelper::LegalizeResult
932LegalizerHelper::widenScalarInsert(MachineInstr &MI, unsigned TypeIdx,
933 LLT WideTy) {
934 if (TypeIdx != 0)
935 return UnableToLegalize;
936 Observer.changingInstr(MI);
937 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
938 widenScalarDst(MI, WideTy);
939 Observer.changedInstr(MI);
940 return Legalized;
941}
942
943LegalizerHelper::LegalizeResult
944LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
945 MIRBuilder.setInstr(MI);
946
947 switch (MI.getOpcode()) {
948 default:
949 return UnableToLegalize;
950 case TargetOpcode::G_EXTRACT:
951 return widenScalarExtract(MI, TypeIdx, WideTy);
952 case TargetOpcode::G_INSERT:
953 return widenScalarInsert(MI, TypeIdx, WideTy);
954 case TargetOpcode::G_MERGE_VALUES:
955 return widenScalarMergeValues(MI, TypeIdx, WideTy);
956 case TargetOpcode::G_UNMERGE_VALUES:
957 return widenScalarUnmergeValues(MI, TypeIdx, WideTy);
958 case TargetOpcode::G_UADDO:
959 case TargetOpcode::G_USUBO: {
960 if (TypeIdx == 1)
961 return UnableToLegalize; // TODO
962 auto LHSZext = MIRBuilder.buildInstr(TargetOpcode::G_ZEXT, {WideTy},
963 {MI.getOperand(2).getReg()});
964 auto RHSZext = MIRBuilder.buildInstr(TargetOpcode::G_ZEXT, {WideTy},
965 {MI.getOperand(3).getReg()});
966 unsigned Opcode = MI.getOpcode() == TargetOpcode::G_UADDO
967 ? TargetOpcode::G_ADD
968 : TargetOpcode::G_SUB;
969 // Do the arithmetic in the larger type.
970 auto NewOp = MIRBuilder.buildInstr(Opcode, {WideTy}, {LHSZext, RHSZext});
971 LLT OrigTy = MRI.getType(MI.getOperand(0).getReg());
972 APInt Mask = APInt::getAllOnesValue(OrigTy.getSizeInBits());
973 auto AndOp = MIRBuilder.buildInstr(
974 TargetOpcode::G_AND, {WideTy},
975 {NewOp, MIRBuilder.buildConstant(WideTy, Mask.getZExtValue())});
976 // There is no overflow if the AndOp is the same as NewOp.
977 MIRBuilder.buildICmp(CmpInst::ICMP_NE, MI.getOperand(1).getReg(), NewOp,
978 AndOp);
979 // Now trunc the NewOp to the original result.
980 MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), NewOp);
981 MI.eraseFromParent();
982 return Legalized;
983 }
984 case TargetOpcode::G_CTTZ:
985 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
986 case TargetOpcode::G_CTLZ:
987 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
988 case TargetOpcode::G_CTPOP: {
989 if (TypeIdx == 0) {
990 Observer.changingInstr(MI);
991 widenScalarDst(MI, WideTy, 0);
992 Observer.changedInstr(MI);
993 return Legalized;
994 }
995
996 unsigned SrcReg = MI.getOperand(1).getReg();
997
998 // First ZEXT the input.
999 auto MIBSrc = MIRBuilder.buildZExt(WideTy, SrcReg);
1000 LLT CurTy = MRI.getType(SrcReg);
1001 if (MI.getOpcode() == TargetOpcode::G_CTTZ) {
1002 // The count is the same in the larger type except if the original
1003 // value was zero. This can be handled by setting the bit just off
1004 // the top of the original type.
1005 auto TopBit =
1006 APInt::getOneBitSet(WideTy.getSizeInBits(), CurTy.getSizeInBits());
1007 MIBSrc = MIRBuilder.buildOr(
1008 WideTy, MIBSrc, MIRBuilder.buildConstant(WideTy, TopBit));
1009 }
1010
1011 // Perform the operation at the larger size.
1012 auto MIBNewOp = MIRBuilder.buildInstr(MI.getOpcode(), {WideTy}, {MIBSrc});
1013 // This is already the correct result for CTPOP and CTTZs
1014 if (MI.getOpcode() == TargetOpcode::G_CTLZ ||
1015 MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF) {
1016 // The correct result is NewOp - (Difference in widety and current ty).
1017 unsigned SizeDiff = WideTy.getSizeInBits() - CurTy.getSizeInBits();
1018 MIBNewOp = MIRBuilder.buildInstr(
1019 TargetOpcode::G_SUB, {WideTy},
1020 {MIBNewOp, MIRBuilder.buildConstant(WideTy, SizeDiff)});
1021 }
1022
1023 MIRBuilder.buildZExtOrTrunc(MI.getOperand(0), MIBNewOp);
1024 MI.eraseFromParent();
1025 return Legalized;
1026 }
1027 case TargetOpcode::G_BSWAP: {
1028 Observer.changingInstr(MI);
1029 unsigned DstReg = MI.getOperand(0).getReg();
1030
1031 unsigned ShrReg = MRI.createGenericVirtualRegister(WideTy);
1032 unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
1033 unsigned ShiftAmtReg = MRI.createGenericVirtualRegister(WideTy);
1034 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1035
1036 MI.getOperand(0).setReg(DstExt);
1037
1038 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1039
1040 LLT Ty = MRI.getType(DstReg);
1041 unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits();
1042 MIRBuilder.buildConstant(ShiftAmtReg, DiffBits);
1043 MIRBuilder.buildInstr(TargetOpcode::G_LSHR)
1044 .addDef(ShrReg)
1045 .addUse(DstExt)
1046 .addUse(ShiftAmtReg);
1047
1048 MIRBuilder.buildTrunc(DstReg, ShrReg);
1049 Observer.changedInstr(MI);
1050 return Legalized;
1051 }
1052 case TargetOpcode::G_ADD:
1053 case TargetOpcode::G_AND:
1054 case TargetOpcode::G_MUL:
1055 case TargetOpcode::G_OR:
1056 case TargetOpcode::G_XOR:
1057 case TargetOpcode::G_SUB:
1058 // Perform operation at larger width (any extension is fines here, high bits
1059 // don't affect the result) and then truncate the result back to the
1060 // original type.
1061 Observer.changingInstr(MI);
1062 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1063 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
1064 widenScalarDst(MI, WideTy);
1065 Observer.changedInstr(MI);
1066 return Legalized;
1067
1068 case TargetOpcode::G_SHL:
1069 Observer.changingInstr(MI);
1070
1071 if (TypeIdx == 0) {
1072 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1073 widenScalarDst(MI, WideTy);
1074 } else {
1075 assert(TypeIdx == 1)((TypeIdx == 1) ? static_cast<void> (0) : __assert_fail
("TypeIdx == 1", "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1075, __PRETTY_FUNCTION__))
;
1076 // The "number of bits to shift" operand must preserve its value as an
1077 // unsigned integer:
1078 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
1079 }
1080
1081 Observer.changedInstr(MI);
1082 return Legalized;
1083
1084 case TargetOpcode::G_SDIV:
1085 case TargetOpcode::G_SREM:
1086 case TargetOpcode::G_SMIN:
1087 case TargetOpcode::G_SMAX:
1088 Observer.changingInstr(MI);
1089 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
1090 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
1091 widenScalarDst(MI, WideTy);
1092 Observer.changedInstr(MI);
1093 return Legalized;
1094
1095 case TargetOpcode::G_ASHR:
1096 case TargetOpcode::G_LSHR:
1097 Observer.changingInstr(MI);
1098
1099 if (TypeIdx == 0) {
1100 unsigned CvtOp = MI.getOpcode() == TargetOpcode::G_ASHR ?
1101 TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
1102
1103 widenScalarSrc(MI, WideTy, 1, CvtOp);
1104 widenScalarDst(MI, WideTy);
1105 } else {
1106 assert(TypeIdx == 1)((TypeIdx == 1) ? static_cast<void> (0) : __assert_fail
("TypeIdx == 1", "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1106, __PRETTY_FUNCTION__))
;
1107 // The "number of bits to shift" operand must preserve its value as an
1108 // unsigned integer:
1109 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
1110 }
1111
1112 Observer.changedInstr(MI);
1113 return Legalized;
1114 case TargetOpcode::G_UDIV:
1115 case TargetOpcode::G_UREM:
1116 case TargetOpcode::G_UMIN:
1117 case TargetOpcode::G_UMAX:
1118 Observer.changingInstr(MI);
1119 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
1120 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
1121 widenScalarDst(MI, WideTy);
1122 Observer.changedInstr(MI);
1123 return Legalized;
1124
1125 case TargetOpcode::G_SELECT:
1126 Observer.changingInstr(MI);
1127 if (TypeIdx == 0) {
1128 // Perform operation at larger width (any extension is fine here, high
1129 // bits don't affect the result) and then truncate the result back to the
1130 // original type.
1131 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
1132 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ANYEXT);
1133 widenScalarDst(MI, WideTy);
1134 } else {
1135 bool IsVec = MRI.getType(MI.getOperand(1).getReg()).isVector();
1136 // Explicit extension is required here since high bits affect the result.
1137 widenScalarSrc(MI, WideTy, 1, MIRBuilder.getBoolExtOp(IsVec, false));
1138 }
1139 Observer.changedInstr(MI);
1140 return Legalized;
1141
1142 case TargetOpcode::G_FPTOSI:
1143 case TargetOpcode::G_FPTOUI:
1144 if (TypeIdx != 0)
1145 return UnableToLegalize;
1146 Observer.changingInstr(MI);
1147 widenScalarDst(MI, WideTy);
1148 Observer.changedInstr(MI);
1149 return Legalized;
1150
1151 case TargetOpcode::G_SITOFP:
1152 if (TypeIdx != 1)
1153 return UnableToLegalize;
1154 Observer.changingInstr(MI);
1155 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
1156 Observer.changedInstr(MI);
1157 return Legalized;
1158
1159 case TargetOpcode::G_UITOFP:
1160 if (TypeIdx != 1)
1161 return UnableToLegalize;
1162 Observer.changingInstr(MI);
1163 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
1164 Observer.changedInstr(MI);
1165 return Legalized;
1166
1167 case TargetOpcode::G_LOAD:
1168 case TargetOpcode::G_SEXTLOAD:
1169 case TargetOpcode::G_ZEXTLOAD:
1170 Observer.changingInstr(MI);
1171 widenScalarDst(MI, WideTy);
1172 Observer.changedInstr(MI);
1173 return Legalized;
1174
1175 case TargetOpcode::G_STORE: {
1176 if (TypeIdx != 0)
1177 return UnableToLegalize;
1178
1179 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
1180 if (!isPowerOf2_32(Ty.getSizeInBits()))
1181 return UnableToLegalize;
1182
1183 Observer.changingInstr(MI);
1184
1185 unsigned ExtType = Ty.getScalarSizeInBits() == 1 ?
1186 TargetOpcode::G_ZEXT : TargetOpcode::G_ANYEXT;
1187 widenScalarSrc(MI, WideTy, 0, ExtType);
1188
1189 Observer.changedInstr(MI);
1190 return Legalized;
1191 }
1192 case TargetOpcode::G_CONSTANT: {
1193 MachineOperand &SrcMO = MI.getOperand(1);
1194 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
1195 const APInt &Val = SrcMO.getCImm()->getValue().sext(WideTy.getSizeInBits());
1196 Observer.changingInstr(MI);
1197 SrcMO.setCImm(ConstantInt::get(Ctx, Val));
1198
1199 widenScalarDst(MI, WideTy);
1200 Observer.changedInstr(MI);
1201 return Legalized;
1202 }
1203 case TargetOpcode::G_FCONSTANT: {
1204 MachineOperand &SrcMO = MI.getOperand(1);
1205 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
1206 APFloat Val = SrcMO.getFPImm()->getValueAPF();
1207 bool LosesInfo;
1208 switch (WideTy.getSizeInBits()) {
1209 case 32:
1210 Val.convert(APFloat::IEEEsingle(), APFloat::rmNearestTiesToEven,
1211 &LosesInfo);
1212 break;
1213 case 64:
1214 Val.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
1215 &LosesInfo);
1216 break;
1217 default:
1218 return UnableToLegalize;
1219 }
1220
1221 assert(!LosesInfo && "extend should always be lossless")((!LosesInfo && "extend should always be lossless") ?
static_cast<void> (0) : __assert_fail ("!LosesInfo && \"extend should always be lossless\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1221, __PRETTY_FUNCTION__))
;
1222
1223 Observer.changingInstr(MI);
1224 SrcMO.setFPImm(ConstantFP::get(Ctx, Val));
1225
1226 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
1227 Observer.changedInstr(MI);
1228 return Legalized;
1229 }
1230 case TargetOpcode::G_IMPLICIT_DEF: {
1231 Observer.changingInstr(MI);
1232 widenScalarDst(MI, WideTy);
1233 Observer.changedInstr(MI);
1234 return Legalized;
1235 }
1236 case TargetOpcode::G_BRCOND:
1237 Observer.changingInstr(MI);
1238 widenScalarSrc(MI, WideTy, 0, MIRBuilder.getBoolExtOp(false, false));
1239 Observer.changedInstr(MI);
1240 return Legalized;
1241
1242 case TargetOpcode::G_FCMP:
1243 Observer.changingInstr(MI);
1244 if (TypeIdx == 0)
1245 widenScalarDst(MI, WideTy);
1246 else {
1247 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_FPEXT);
1248 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_FPEXT);
1249 }
1250 Observer.changedInstr(MI);
1251 return Legalized;
1252
1253 case TargetOpcode::G_ICMP:
1254 Observer.changingInstr(MI);
1255 if (TypeIdx == 0)
1256 widenScalarDst(MI, WideTy);
1257 else {
1258 unsigned ExtOpcode = CmpInst::isSigned(static_cast<CmpInst::Predicate>(
1259 MI.getOperand(1).getPredicate()))
1260 ? TargetOpcode::G_SEXT
1261 : TargetOpcode::G_ZEXT;
1262 widenScalarSrc(MI, WideTy, 2, ExtOpcode);
1263 widenScalarSrc(MI, WideTy, 3, ExtOpcode);
1264 }
1265 Observer.changedInstr(MI);
1266 return Legalized;
1267
1268 case TargetOpcode::G_GEP:
1269 assert(TypeIdx == 1 && "unable to legalize pointer of GEP")((TypeIdx == 1 && "unable to legalize pointer of GEP"
) ? static_cast<void> (0) : __assert_fail ("TypeIdx == 1 && \"unable to legalize pointer of GEP\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1269, __PRETTY_FUNCTION__))
;
1270 Observer.changingInstr(MI);
1271 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
1272 Observer.changedInstr(MI);
1273 return Legalized;
1274
1275 case TargetOpcode::G_PHI: {
1276 assert(TypeIdx == 0 && "Expecting only Idx 0")((TypeIdx == 0 && "Expecting only Idx 0") ? static_cast
<void> (0) : __assert_fail ("TypeIdx == 0 && \"Expecting only Idx 0\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1276, __PRETTY_FUNCTION__))
;
1277
1278 Observer.changingInstr(MI);
1279 for (unsigned I = 1; I < MI.getNumOperands(); I += 2) {
1280 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
1281 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
1282 widenScalarSrc(MI, WideTy, I, TargetOpcode::G_ANYEXT);
1283 }
1284
1285 MachineBasicBlock &MBB = *MI.getParent();
1286 MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI());
1287 widenScalarDst(MI, WideTy);
1288 Observer.changedInstr(MI);
1289 return Legalized;
1290 }
1291 case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
1292 if (TypeIdx == 0) {
1293 unsigned VecReg = MI.getOperand(1).getReg();
1294 LLT VecTy = MRI.getType(VecReg);
1295 Observer.changingInstr(MI);
1296
1297 widenScalarSrc(MI, LLT::vector(VecTy.getNumElements(),
1298 WideTy.getSizeInBits()),
1299 1, TargetOpcode::G_SEXT);
1300
1301 widenScalarDst(MI, WideTy, 0);
1302 Observer.changedInstr(MI);
1303 return Legalized;
1304 }
1305
1306 if (TypeIdx != 2)
1307 return UnableToLegalize;
1308 Observer.changingInstr(MI);
1309 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
1310 Observer.changedInstr(MI);
1311 return Legalized;
1312 }
1313 case TargetOpcode::G_FADD:
1314 case TargetOpcode::G_FMUL:
1315 case TargetOpcode::G_FSUB:
1316 case TargetOpcode::G_FMA:
1317 case TargetOpcode::G_FNEG:
1318 case TargetOpcode::G_FABS:
1319 case TargetOpcode::G_FCANONICALIZE:
1320 case TargetOpcode::G_FDIV:
1321 case TargetOpcode::G_FREM:
1322 case TargetOpcode::G_FCEIL:
1323 case TargetOpcode::G_FFLOOR:
1324 case TargetOpcode::G_FCOS:
1325 case TargetOpcode::G_FSIN:
1326 case TargetOpcode::G_FLOG10:
1327 case TargetOpcode::G_FLOG:
1328 case TargetOpcode::G_FLOG2:
1329 case TargetOpcode::G_FRINT:
1330 case TargetOpcode::G_FNEARBYINT:
1331 case TargetOpcode::G_FSQRT:
1332 case TargetOpcode::G_FEXP:
1333 case TargetOpcode::G_FEXP2:
1334 case TargetOpcode::G_FPOW:
1335 case TargetOpcode::G_INTRINSIC_TRUNC:
1336 case TargetOpcode::G_INTRINSIC_ROUND:
1337 assert(TypeIdx == 0)((TypeIdx == 0) ? static_cast<void> (0) : __assert_fail
("TypeIdx == 0", "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1337, __PRETTY_FUNCTION__))
;
1338 Observer.changingInstr(MI);
1339
1340 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I)
1341 widenScalarSrc(MI, WideTy, I, TargetOpcode::G_FPEXT);
1342
1343 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
1344 Observer.changedInstr(MI);
1345 return Legalized;
1346 case TargetOpcode::G_INTTOPTR:
1347 if (TypeIdx != 1)
1348 return UnableToLegalize;
1349
1350 Observer.changingInstr(MI);
1351 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
1352 Observer.changedInstr(MI);
1353 return Legalized;
1354 case TargetOpcode::G_PTRTOINT:
1355 if (TypeIdx != 0)
1356 return UnableToLegalize;
1357
1358 Observer.changingInstr(MI);
1359 widenScalarDst(MI, WideTy, 0);
1360 Observer.changedInstr(MI);
1361 return Legalized;
1362 }
1363}
1364
1365LegalizerHelper::LegalizeResult
1366LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
1367 using namespace TargetOpcode;
1368 MIRBuilder.setInstr(MI);
1369
1370 switch(MI.getOpcode()) {
1371 default:
1372 return UnableToLegalize;
1373 case TargetOpcode::G_SREM:
1374 case TargetOpcode::G_UREM: {
1375 unsigned QuotReg = MRI.createGenericVirtualRegister(Ty);
1376 MIRBuilder.buildInstr(MI.getOpcode() == G_SREM ? G_SDIV : G_UDIV)
1377 .addDef(QuotReg)
1378 .addUse(MI.getOperand(1).getReg())
1379 .addUse(MI.getOperand(2).getReg());
1380
1381 unsigned ProdReg = MRI.createGenericVirtualRegister(Ty);
1382 MIRBuilder.buildMul(ProdReg, QuotReg, MI.getOperand(2).getReg());
1383 MIRBuilder.buildSub(MI.getOperand(0).getReg(), MI.getOperand(1).getReg(),
1384 ProdReg);
1385 MI.eraseFromParent();
1386 return Legalized;
1387 }
1388 case TargetOpcode::G_SMULO:
1389 case TargetOpcode::G_UMULO: {
1390 // Generate G_UMULH/G_SMULH to check for overflow and a normal G_MUL for the
1391 // result.
1392 unsigned Res = MI.getOperand(0).getReg();
1393 unsigned Overflow = MI.getOperand(1).getReg();
1394 unsigned LHS = MI.getOperand(2).getReg();
1395 unsigned RHS = MI.getOperand(3).getReg();
1396
1397 MIRBuilder.buildMul(Res, LHS, RHS);
1398
1399 unsigned Opcode = MI.getOpcode() == TargetOpcode::G_SMULO
1400 ? TargetOpcode::G_SMULH
1401 : TargetOpcode::G_UMULH;
1402
1403 unsigned HiPart = MRI.createGenericVirtualRegister(Ty);
1404 MIRBuilder.buildInstr(Opcode)
1405 .addDef(HiPart)
1406 .addUse(LHS)
1407 .addUse(RHS);
1408
1409 unsigned Zero = MRI.createGenericVirtualRegister(Ty);
1410 MIRBuilder.buildConstant(Zero, 0);
1411
1412 // For *signed* multiply, overflow is detected by checking:
1413 // (hi != (lo >> bitwidth-1))
1414 if (Opcode == TargetOpcode::G_SMULH) {
1415 unsigned Shifted = MRI.createGenericVirtualRegister(Ty);
1416 unsigned ShiftAmt = MRI.createGenericVirtualRegister(Ty);
1417 MIRBuilder.buildConstant(ShiftAmt, Ty.getSizeInBits() - 1);
1418 MIRBuilder.buildInstr(TargetOpcode::G_ASHR)
1419 .addDef(Shifted)
1420 .addUse(Res)
1421 .addUse(ShiftAmt);
1422 MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Shifted);
1423 } else {
1424 MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Zero);
1425 }
1426 MI.eraseFromParent();
1427 return Legalized;
1428 }
1429 case TargetOpcode::G_FNEG: {
1430 // TODO: Handle vector types once we are able to
1431 // represent them.
1432 if (Ty.isVector())
1433 return UnableToLegalize;
1434 unsigned Res = MI.getOperand(0).getReg();
1435 Type *ZeroTy;
1436 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
1437 switch (Ty.getSizeInBits()) {
1438 case 16:
1439 ZeroTy = Type::getHalfTy(Ctx);
1440 break;
1441 case 32:
1442 ZeroTy = Type::getFloatTy(Ctx);
1443 break;
1444 case 64:
1445 ZeroTy = Type::getDoubleTy(Ctx);
1446 break;
1447 case 128:
1448 ZeroTy = Type::getFP128Ty(Ctx);
1449 break;
1450 default:
1451 llvm_unreachable("unexpected floating-point type")::llvm::llvm_unreachable_internal("unexpected floating-point type"
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1451)
;
1452 }
1453 ConstantFP &ZeroForNegation =
1454 *cast<ConstantFP>(ConstantFP::getZeroValueForNegation(ZeroTy));
1455 auto Zero = MIRBuilder.buildFConstant(Ty, ZeroForNegation);
1456 unsigned SubByReg = MI.getOperand(1).getReg();
1457 unsigned ZeroReg = Zero->getOperand(0).getReg();
1458 MachineInstr *SrcMI = MRI.getVRegDef(SubByReg);
1459 MIRBuilder.buildInstr(TargetOpcode::G_FSUB, {Res}, {ZeroReg, SubByReg},
1460 SrcMI->getFlags());
1461 MI.eraseFromParent();
1462 return Legalized;
1463 }
1464 case TargetOpcode::G_FSUB: {
1465 // Lower (G_FSUB LHS, RHS) to (G_FADD LHS, (G_FNEG RHS)).
1466 // First, check if G_FNEG is marked as Lower. If so, we may
1467 // end up with an infinite loop as G_FSUB is used to legalize G_FNEG.
1468 if (LI.getAction({G_FNEG, {Ty}}).Action == Lower)
1469 return UnableToLegalize;
1470 unsigned Res = MI.getOperand(0).getReg();
1471 unsigned LHS = MI.getOperand(1).getReg();
1472 unsigned RHS = MI.getOperand(2).getReg();
1473 unsigned Neg = MRI.createGenericVirtualRegister(Ty);
1474 MIRBuilder.buildInstr(TargetOpcode::G_FNEG).addDef(Neg).addUse(RHS);
1475 MIRBuilder.buildInstr(TargetOpcode::G_FADD, {Res}, {LHS, Neg}, MI.getFlags());
1476 MI.eraseFromParent();
1477 return Legalized;
1478 }
1479 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
1480 unsigned OldValRes = MI.getOperand(0).getReg();
1481 unsigned SuccessRes = MI.getOperand(1).getReg();
1482 unsigned Addr = MI.getOperand(2).getReg();
1483 unsigned CmpVal = MI.getOperand(3).getReg();
1484 unsigned NewVal = MI.getOperand(4).getReg();
1485 MIRBuilder.buildAtomicCmpXchg(OldValRes, Addr, CmpVal, NewVal,
1486 **MI.memoperands_begin());
1487 MIRBuilder.buildICmp(CmpInst::ICMP_EQ, SuccessRes, OldValRes, CmpVal);
1488 MI.eraseFromParent();
1489 return Legalized;
1490 }
1491 case TargetOpcode::G_LOAD:
1492 case TargetOpcode::G_SEXTLOAD:
1493 case TargetOpcode::G_ZEXTLOAD: {
1494 // Lower to a memory-width G_LOAD and a G_SEXT/G_ZEXT/G_ANYEXT
1495 unsigned DstReg = MI.getOperand(0).getReg();
1496 unsigned PtrReg = MI.getOperand(1).getReg();
1497 LLT DstTy = MRI.getType(DstReg);
1498 auto &MMO = **MI.memoperands_begin();
1499
1500 if (DstTy.getSizeInBits() == MMO.getSize() /* in bytes */ * 8) {
1501 // In the case of G_LOAD, this was a non-extending load already and we're
1502 // about to lower to the same instruction.
1503 if (MI.getOpcode() == TargetOpcode::G_LOAD)
1504 return UnableToLegalize;
1505 MIRBuilder.buildLoad(DstReg, PtrReg, MMO);
1506 MI.eraseFromParent();
1507 return Legalized;
1508 }
1509
1510 if (DstTy.isScalar()) {
1511 unsigned TmpReg =
1512 MRI.createGenericVirtualRegister(LLT::scalar(MMO.getSizeInBits()));
1513 MIRBuilder.buildLoad(TmpReg, PtrReg, MMO);
1514 switch (MI.getOpcode()) {
1515 default:
1516 llvm_unreachable("Unexpected opcode")::llvm::llvm_unreachable_internal("Unexpected opcode", "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1516)
;
1517 case TargetOpcode::G_LOAD:
1518 MIRBuilder.buildAnyExt(DstReg, TmpReg);
1519 break;
1520 case TargetOpcode::G_SEXTLOAD:
1521 MIRBuilder.buildSExt(DstReg, TmpReg);
1522 break;
1523 case TargetOpcode::G_ZEXTLOAD:
1524 MIRBuilder.buildZExt(DstReg, TmpReg);
1525 break;
1526 }
1527 MI.eraseFromParent();
1528 return Legalized;
1529 }
1530
1531 return UnableToLegalize;
1532 }
1533 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
1534 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
1535 case TargetOpcode::G_CTLZ:
1536 case TargetOpcode::G_CTTZ:
1537 case TargetOpcode::G_CTPOP:
1538 return lowerBitCount(MI, TypeIdx, Ty);
1539 case G_UADDO: {
1540 unsigned Res = MI.getOperand(0).getReg();
1541 unsigned CarryOut = MI.getOperand(1).getReg();
1542 unsigned LHS = MI.getOperand(2).getReg();
1543 unsigned RHS = MI.getOperand(3).getReg();
1544
1545 MIRBuilder.buildAdd(Res, LHS, RHS);
1546 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, RHS);
1547
1548 MI.eraseFromParent();
1549 return Legalized;
1550 }
1551 case G_UADDE: {
1552 unsigned Res = MI.getOperand(0).getReg();
1553 unsigned CarryOut = MI.getOperand(1).getReg();
1554 unsigned LHS = MI.getOperand(2).getReg();
1555 unsigned RHS = MI.getOperand(3).getReg();
1556 unsigned CarryIn = MI.getOperand(4).getReg();
1557
1558 unsigned TmpRes = MRI.createGenericVirtualRegister(Ty);
1559 unsigned ZExtCarryIn = MRI.createGenericVirtualRegister(Ty);
1560
1561 MIRBuilder.buildAdd(TmpRes, LHS, RHS);
1562 MIRBuilder.buildZExt(ZExtCarryIn, CarryIn);
1563 MIRBuilder.buildAdd(Res, TmpRes, ZExtCarryIn);
1564 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, LHS);
1565
1566 MI.eraseFromParent();
1567 return Legalized;
1568 }
1569 case G_USUBO: {
1570 unsigned Res = MI.getOperand(0).getReg();
1571 unsigned BorrowOut = MI.getOperand(1).getReg();
1572 unsigned LHS = MI.getOperand(2).getReg();
1573 unsigned RHS = MI.getOperand(3).getReg();
1574
1575 MIRBuilder.buildSub(Res, LHS, RHS);
1576 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, BorrowOut, LHS, RHS);
1577
1578 MI.eraseFromParent();
1579 return Legalized;
1580 }
1581 case G_USUBE: {
1582 unsigned Res = MI.getOperand(0).getReg();
1583 unsigned BorrowOut = MI.getOperand(1).getReg();
1584 unsigned LHS = MI.getOperand(2).getReg();
1585 unsigned RHS = MI.getOperand(3).getReg();
1586 unsigned BorrowIn = MI.getOperand(4).getReg();
1587
1588 unsigned TmpRes = MRI.createGenericVirtualRegister(Ty);
1589 unsigned ZExtBorrowIn = MRI.createGenericVirtualRegister(Ty);
1590 unsigned LHS_EQ_RHS = MRI.createGenericVirtualRegister(LLT::scalar(1));
1591 unsigned LHS_ULT_RHS = MRI.createGenericVirtualRegister(LLT::scalar(1));
1592
1593 MIRBuilder.buildSub(TmpRes, LHS, RHS);
1594 MIRBuilder.buildZExt(ZExtBorrowIn, BorrowIn);
1595 MIRBuilder.buildSub(Res, TmpRes, ZExtBorrowIn);
1596 MIRBuilder.buildICmp(CmpInst::ICMP_EQ, LHS_EQ_RHS, LHS, RHS);
1597 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, LHS_ULT_RHS, LHS, RHS);
1598 MIRBuilder.buildSelect(BorrowOut, LHS_EQ_RHS, BorrowIn, LHS_ULT_RHS);
1599
1600 MI.eraseFromParent();
1601 return Legalized;
1602 }
1603 case G_UITOFP:
1604 return lowerUITOFP(MI, TypeIdx, Ty);
1605 case G_SITOFP:
1606 return lowerSITOFP(MI, TypeIdx, Ty);
1607 }
1608}
1609
1610LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorImplicitDef(
1611 MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) {
1612 SmallVector<unsigned, 2> DstRegs;
1613
1614 unsigned NarrowSize = NarrowTy.getSizeInBits();
1615 unsigned DstReg = MI.getOperand(0).getReg();
1616 unsigned Size = MRI.getType(DstReg).getSizeInBits();
1617 int NumParts = Size / NarrowSize;
1618 // FIXME: Don't know how to handle the situation where the small vectors
1619 // aren't all the same size yet.
1620 if (Size % NarrowSize != 0)
1621 return UnableToLegalize;
1622
1623 for (int i = 0; i < NumParts; ++i) {
1624 unsigned TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
1625 MIRBuilder.buildUndef(TmpReg);
1626 DstRegs.push_back(TmpReg);
1627 }
1628
1629 if (NarrowTy.isVector())
1630 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
1631 else
1632 MIRBuilder.buildBuildVector(DstReg, DstRegs);
1633
1634 MI.eraseFromParent();
1635 return Legalized;
1636}
1637
1638LegalizerHelper::LegalizeResult
1639LegalizerHelper::fewerElementsVectorBasic(MachineInstr &MI, unsigned TypeIdx,
1640 LLT NarrowTy) {
1641 const unsigned Opc = MI.getOpcode();
1642 const unsigned NumOps = MI.getNumOperands() - 1;
1643 const unsigned NarrowSize = NarrowTy.getSizeInBits();
1644 const unsigned DstReg = MI.getOperand(0).getReg();
1645 const unsigned Flags = MI.getFlags();
1646 const LLT DstTy = MRI.getType(DstReg);
1647 const unsigned Size = DstTy.getSizeInBits();
1648 const int NumParts = Size / NarrowSize;
1649 const LLT EltTy = DstTy.getElementType();
1650 const unsigned EltSize = EltTy.getSizeInBits();
1651 const unsigned BitsForNumParts = NarrowSize * NumParts;
1652
1653 // Check if we have any leftovers. If we do, then only handle the case where
1654 // the leftover is one element.
1655 if (BitsForNumParts != Size && BitsForNumParts + EltSize != Size)
1656 return UnableToLegalize;
1657
1658 if (BitsForNumParts != Size) {
1659 unsigned AccumDstReg = MRI.createGenericVirtualRegister(DstTy);
1660 MIRBuilder.buildUndef(AccumDstReg);
1661
1662 // Handle the pieces which evenly divide into the requested type with
1663 // extract/op/insert sequence.
1664 for (unsigned Offset = 0; Offset < BitsForNumParts; Offset += NarrowSize) {
1665 SmallVector<SrcOp, 4> SrcOps;
1666 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
1667 unsigned PartOpReg = MRI.createGenericVirtualRegister(NarrowTy);
1668 MIRBuilder.buildExtract(PartOpReg, MI.getOperand(I).getReg(), Offset);
1669 SrcOps.push_back(PartOpReg);
1670 }
1671
1672 unsigned PartDstReg = MRI.createGenericVirtualRegister(NarrowTy);
1673 MIRBuilder.buildInstr(Opc, {PartDstReg}, SrcOps, Flags);
1674
1675 unsigned PartInsertReg = MRI.createGenericVirtualRegister(DstTy);
1676 MIRBuilder.buildInsert(PartInsertReg, AccumDstReg, PartDstReg, Offset);
1677 AccumDstReg = PartInsertReg;
1678 }
1679
1680 // Handle the remaining element sized leftover piece.
1681 SmallVector<SrcOp, 4> SrcOps;
1682 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
1683 unsigned PartOpReg = MRI.createGenericVirtualRegister(EltTy);
1684 MIRBuilder.buildExtract(PartOpReg, MI.getOperand(I).getReg(),
1685 BitsForNumParts);
1686 SrcOps.push_back(PartOpReg);
1687 }
1688
1689 unsigned PartDstReg = MRI.createGenericVirtualRegister(EltTy);
1690 MIRBuilder.buildInstr(Opc, {PartDstReg}, SrcOps, Flags);
1691 MIRBuilder.buildInsert(DstReg, AccumDstReg, PartDstReg, BitsForNumParts);
1692 MI.eraseFromParent();
1693
1694 return Legalized;
1695 }
1696
1697 SmallVector<unsigned, 2> DstRegs, Src0Regs, Src1Regs, Src2Regs;
1698
1699 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src0Regs);
1700
1701 if (NumOps >= 2)
1702 extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src1Regs);
1703
1704 if (NumOps >= 3)
1705 extractParts(MI.getOperand(3).getReg(), NarrowTy, NumParts, Src2Regs);
1706
1707 for (int i = 0; i < NumParts; ++i) {
1708 unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
1709
1710 if (NumOps == 1)
1711 MIRBuilder.buildInstr(Opc, {DstReg}, {Src0Regs[i]}, Flags);
1712 else if (NumOps == 2) {
1713 MIRBuilder.buildInstr(Opc, {DstReg}, {Src0Regs[i], Src1Regs[i]}, Flags);
1714 } else if (NumOps == 3) {
1715 MIRBuilder.buildInstr(Opc, {DstReg},
1716 {Src0Regs[i], Src1Regs[i], Src2Regs[i]}, Flags);
1717 }
1718
1719 DstRegs.push_back(DstReg);
1720 }
1721
1722 if (NarrowTy.isVector())
1723 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
1724 else
1725 MIRBuilder.buildBuildVector(DstReg, DstRegs);
1726
1727 MI.eraseFromParent();
1728 return Legalized;
1729}
1730
1731// Handle splitting vector operations which need to have the same number of
1732// elements in each type index, but each type index may have a different element
1733// type.
1734//
1735// e.g. <4 x s64> = G_SHL <4 x s64>, <4 x s32> ->
1736// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
1737// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
1738//
1739// Also handles some irregular breakdown cases, e.g.
1740// e.g. <3 x s64> = G_SHL <3 x s64>, <3 x s32> ->
1741// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
1742// s64 = G_SHL s64, s32
1743LegalizerHelper::LegalizeResult
1744LegalizerHelper::fewerElementsVectorMultiEltType(
1745 MachineInstr &MI, unsigned TypeIdx, LLT NarrowTyArg) {
1746 if (TypeIdx != 0)
1747 return UnableToLegalize;
1748
1749 const LLT NarrowTy0 = NarrowTyArg;
1750 const unsigned NewNumElts =
1751 NarrowTy0.isVector() ? NarrowTy0.getNumElements() : 1;
1752
1753 const unsigned DstReg = MI.getOperand(0).getReg();
1754 LLT DstTy = MRI.getType(DstReg);
1755 LLT LeftoverTy0;
1756
1757 int NumParts, NumLeftover;
1758 // All of the operands need to have the same number of elements, so if we can
1759 // determine a type breakdown for the result type, we can for all of the
1760 // source types.
1761 std::tie(NumParts, NumLeftover)
1762 = getNarrowTypeBreakDown(DstTy, NarrowTy0, LeftoverTy0);
1763 if (NumParts < 0)
1764 return UnableToLegalize;
1765
1766 SmallVector<MachineInstrBuilder, 4> NewInsts;
1767
1768 SmallVector<unsigned, 4> DstRegs, LeftoverDstRegs;
1769 SmallVector<unsigned, 4> PartRegs, LeftoverRegs;
1770
1771 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
1772 LLT LeftoverTy;
1773 unsigned SrcReg = MI.getOperand(I).getReg();
1774 LLT SrcTyI = MRI.getType(SrcReg);
1775 LLT NarrowTyI = LLT::scalarOrVector(NewNumElts, SrcTyI.getScalarType());
1776 LLT LeftoverTyI;
1777
1778 // Split this operand into the requested typed registers, and any leftover
1779 // required to reproduce the original type.
1780 if (!extractParts(SrcReg, SrcTyI, NarrowTyI, LeftoverTyI, PartRegs,
1781 LeftoverRegs))
1782 return UnableToLegalize;
1783
1784 if (I == 1) {
1785 // For the first operand, create an instruction for each part and setup
1786 // the result.
1787 for (unsigned PartReg : PartRegs) {
1788 unsigned PartDstReg = MRI.createGenericVirtualRegister(NarrowTy0);
1789 NewInsts.push_back(MIRBuilder.buildInstrNoInsert(MI.getOpcode())
1790 .addDef(PartDstReg)
1791 .addUse(PartReg));
1792 DstRegs.push_back(PartDstReg);
1793 }
1794
1795 for (unsigned LeftoverReg : LeftoverRegs) {
1796 unsigned PartDstReg = MRI.createGenericVirtualRegister(LeftoverTy0);
1797 NewInsts.push_back(MIRBuilder.buildInstrNoInsert(MI.getOpcode())
1798 .addDef(PartDstReg)
1799 .addUse(LeftoverReg));
1800 LeftoverDstRegs.push_back(PartDstReg);
1801 }
1802 } else {
1803 assert(NewInsts.size() == PartRegs.size() + LeftoverRegs.size())((NewInsts.size() == PartRegs.size() + LeftoverRegs.size()) ?
static_cast<void> (0) : __assert_fail ("NewInsts.size() == PartRegs.size() + LeftoverRegs.size()"
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1803, __PRETTY_FUNCTION__))
;
1804
1805 // Add the newly created operand splits to the existing instructions. The
1806 // odd-sized pieces are ordered after the requested NarrowTyArg sized
1807 // pieces.
1808 unsigned InstCount = 0;
1809 for (unsigned J = 0, JE = PartRegs.size(); J != JE; ++J)
1810 NewInsts[InstCount++].addUse(PartRegs[J]);
1811 for (unsigned J = 0, JE = LeftoverRegs.size(); J != JE; ++J)
1812 NewInsts[InstCount++].addUse(LeftoverRegs[J]);
1813 }
1814
1815 PartRegs.clear();
1816 LeftoverRegs.clear();
1817 }
1818
1819 // Insert the newly built operations and rebuild the result register.
1820 for (auto &MIB : NewInsts)
1821 MIRBuilder.insertInstr(MIB);
1822
1823 insertParts(DstReg, DstTy, NarrowTy0, DstRegs, LeftoverTy0, LeftoverDstRegs);
1824
1825 MI.eraseFromParent();
1826 return Legalized;
1827}
1828
1829LegalizerHelper::LegalizeResult
1830LegalizerHelper::fewerElementsVectorCasts(MachineInstr &MI, unsigned TypeIdx,
1831 LLT NarrowTy) {
1832 if (TypeIdx != 0)
1833 return UnableToLegalize;
1834
1835 unsigned DstReg = MI.getOperand(0).getReg();
1836 unsigned SrcReg = MI.getOperand(1).getReg();
1837 LLT DstTy = MRI.getType(DstReg);
1838 LLT SrcTy = MRI.getType(SrcReg);
1839
1840 LLT NarrowTy0 = NarrowTy;
1841 LLT NarrowTy1;
1842 unsigned NumParts;
1843
1844 if (NarrowTy.isVector()) {
1845 // Uneven breakdown not handled.
1846 NumParts = DstTy.getNumElements() / NarrowTy.getNumElements();
1847 if (NumParts * NarrowTy.getNumElements() != DstTy.getNumElements())
1848 return UnableToLegalize;
1849
1850 NarrowTy1 = LLT::vector(NumParts, SrcTy.getElementType().getSizeInBits());
1851 } else {
1852 NumParts = DstTy.getNumElements();
1853 NarrowTy1 = SrcTy.getElementType();
1854 }
1855
1856 SmallVector<unsigned, 4> SrcRegs, DstRegs;
1857 extractParts(SrcReg, NarrowTy1, NumParts, SrcRegs);
1858
1859 for (unsigned I = 0; I < NumParts; ++I) {
1860 unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
1861 MachineInstr *NewInst = MIRBuilder.buildInstr(MI.getOpcode())
1862 .addDef(DstReg)
1863 .addUse(SrcRegs[I]);
1864
1865 NewInst->setFlags(MI.getFlags());
1866 DstRegs.push_back(DstReg);
1867 }
1868
1869 if (NarrowTy.isVector())
1870 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
1871 else
1872 MIRBuilder.buildBuildVector(DstReg, DstRegs);
1873
1874 MI.eraseFromParent();
1875 return Legalized;
1876}
1877
1878LegalizerHelper::LegalizeResult
1879LegalizerHelper::fewerElementsVectorCmp(MachineInstr &MI, unsigned TypeIdx,
1880 LLT NarrowTy) {
1881 unsigned DstReg = MI.getOperand(0).getReg();
1882 unsigned Src0Reg = MI.getOperand(2).getReg();
1883 LLT DstTy = MRI.getType(DstReg);
1884 LLT SrcTy = MRI.getType(Src0Reg);
1885
1886 unsigned NumParts;
1887 LLT NarrowTy0, NarrowTy1;
1888
1889 if (TypeIdx == 0) {
1890 unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
1891 unsigned OldElts = DstTy.getNumElements();
1892
1893 NarrowTy0 = NarrowTy;
1894 NumParts = NarrowTy.isVector() ? (OldElts / NewElts) : DstTy.getNumElements();
1895 NarrowTy1 = NarrowTy.isVector() ?
1896 LLT::vector(NarrowTy.getNumElements(), SrcTy.getScalarSizeInBits()) :
1897 SrcTy.getElementType();
1898
1899 } else {
1900 unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
1901 unsigned OldElts = SrcTy.getNumElements();
1902
1903 NumParts = NarrowTy.isVector() ? (OldElts / NewElts) :
1904 NarrowTy.getNumElements();
1905 NarrowTy0 = LLT::vector(NarrowTy.getNumElements(),
1906 DstTy.getScalarSizeInBits());
1907 NarrowTy1 = NarrowTy;
1908 }
1909
1910 // FIXME: Don't know how to handle the situation where the small vectors
1911 // aren't all the same size yet.
1912 if (NarrowTy1.isVector() &&
1913 NarrowTy1.getNumElements() * NumParts != DstTy.getNumElements())
1914 return UnableToLegalize;
1915
1916 CmpInst::Predicate Pred
1917 = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
1918
1919 SmallVector<unsigned, 2> Src1Regs, Src2Regs, DstRegs;
1920 extractParts(MI.getOperand(2).getReg(), NarrowTy1, NumParts, Src1Regs);
1921 extractParts(MI.getOperand(3).getReg(), NarrowTy1, NumParts, Src2Regs);
1922
1923 for (unsigned I = 0; I < NumParts; ++I) {
1924 unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
1925 DstRegs.push_back(DstReg);
1926
1927 if (MI.getOpcode() == TargetOpcode::G_ICMP)
1928 MIRBuilder.buildICmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]);
1929 else {
1930 MachineInstr *NewCmp
1931 = MIRBuilder.buildFCmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]);
1932 NewCmp->setFlags(MI.getFlags());
1933 }
1934 }
1935
1936 if (NarrowTy1.isVector())
1937 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
1938 else
1939 MIRBuilder.buildBuildVector(DstReg, DstRegs);
1940
1941 MI.eraseFromParent();
1942 return Legalized;
1943}
1944
1945LegalizerHelper::LegalizeResult
1946LegalizerHelper::fewerElementsVectorSelect(MachineInstr &MI, unsigned TypeIdx,
1947 LLT NarrowTy) {
1948 unsigned DstReg = MI.getOperand(0).getReg();
1949 unsigned CondReg = MI.getOperand(1).getReg();
1950
1951 unsigned NumParts = 0;
1952 LLT NarrowTy0, NarrowTy1;
1953
1954 LLT DstTy = MRI.getType(DstReg);
1955 LLT CondTy = MRI.getType(CondReg);
1956 unsigned Size = DstTy.getSizeInBits();
1957
1958 assert(TypeIdx == 0 || CondTy.isVector())((TypeIdx == 0 || CondTy.isVector()) ? static_cast<void>
(0) : __assert_fail ("TypeIdx == 0 || CondTy.isVector()", "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 1958, __PRETTY_FUNCTION__))
;
1959
1960 if (TypeIdx == 0) {
1961 NarrowTy0 = NarrowTy;
1962 NarrowTy1 = CondTy;
1963
1964 unsigned NarrowSize = NarrowTy0.getSizeInBits();
1965 // FIXME: Don't know how to handle the situation where the small vectors
1966 // aren't all the same size yet.
1967 if (Size % NarrowSize != 0)
1968 return UnableToLegalize;
1969
1970 NumParts = Size / NarrowSize;
1971
1972 // Need to break down the condition type
1973 if (CondTy.isVector()) {
1974 if (CondTy.getNumElements() == NumParts)
1975 NarrowTy1 = CondTy.getElementType();
1976 else
1977 NarrowTy1 = LLT::vector(CondTy.getNumElements() / NumParts,
1978 CondTy.getScalarSizeInBits());
1979 }
1980 } else {
1981 NumParts = CondTy.getNumElements();
1982 if (NarrowTy.isVector()) {
1983 // TODO: Handle uneven breakdown.
1984 if (NumParts * NarrowTy.getNumElements() != CondTy.getNumElements())
1985 return UnableToLegalize;
1986
1987 return UnableToLegalize;
1988 } else {
1989 NarrowTy0 = DstTy.getElementType();
1990 NarrowTy1 = NarrowTy;
1991 }
1992 }
1993
1994 SmallVector<unsigned, 2> DstRegs, Src0Regs, Src1Regs, Src2Regs;
1995 if (CondTy.isVector())
1996 extractParts(MI.getOperand(1).getReg(), NarrowTy1, NumParts, Src0Regs);
1997
1998 extractParts(MI.getOperand(2).getReg(), NarrowTy0, NumParts, Src1Regs);
1999 extractParts(MI.getOperand(3).getReg(), NarrowTy0, NumParts, Src2Regs);
2000
2001 for (unsigned i = 0; i < NumParts; ++i) {
2002 unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
2003 MIRBuilder.buildSelect(DstReg, CondTy.isVector() ? Src0Regs[i] : CondReg,
2004 Src1Regs[i], Src2Regs[i]);
2005 DstRegs.push_back(DstReg);
2006 }
2007
2008 if (NarrowTy0.isVector())
2009 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
2010 else
2011 MIRBuilder.buildBuildVector(DstReg, DstRegs);
2012
2013 MI.eraseFromParent();
2014 return Legalized;
2015}
2016
2017LegalizerHelper::LegalizeResult
2018LegalizerHelper::fewerElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx,
2019 LLT NarrowTy) {
2020 const unsigned DstReg = MI.getOperand(0).getReg();
2021 LLT PhiTy = MRI.getType(DstReg);
2022 LLT LeftoverTy;
2023
2024 // All of the operands need to have the same number of elements, so if we can
2025 // determine a type breakdown for the result type, we can for all of the
2026 // source types.
2027 int NumParts, NumLeftover;
2028 std::tie(NumParts, NumLeftover)
2029 = getNarrowTypeBreakDown(PhiTy, NarrowTy, LeftoverTy);
2030 if (NumParts < 0)
2031 return UnableToLegalize;
2032
2033 SmallVector<unsigned, 4> DstRegs, LeftoverDstRegs;
2034 SmallVector<MachineInstrBuilder, 4> NewInsts;
2035
2036 const int TotalNumParts = NumParts + NumLeftover;
2037
2038 // Insert the new phis in the result block first.
2039 for (int I = 0; I != TotalNumParts; ++I) {
2040 LLT Ty = I < NumParts ? NarrowTy : LeftoverTy;
2041 unsigned PartDstReg = MRI.createGenericVirtualRegister(Ty);
2042 NewInsts.push_back(MIRBuilder.buildInstr(TargetOpcode::G_PHI)
2043 .addDef(PartDstReg));
2044 if (I < NumParts)
2045 DstRegs.push_back(PartDstReg);
2046 else
2047 LeftoverDstRegs.push_back(PartDstReg);
2048 }
2049
2050 MachineBasicBlock *MBB = MI.getParent();
2051 MIRBuilder.setInsertPt(*MBB, MBB->getFirstNonPHI());
2052 insertParts(DstReg, PhiTy, NarrowTy, DstRegs, LeftoverTy, LeftoverDstRegs);
2053
2054 SmallVector<unsigned, 4> PartRegs, LeftoverRegs;
2055
2056 // Insert code to extract the incoming values in each predecessor block.
2057 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
2058 PartRegs.clear();
2059 LeftoverRegs.clear();
2060
2061 unsigned SrcReg = MI.getOperand(I).getReg();
2062 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
2063 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
2064
2065 LLT Unused;
2066 if (!extractParts(SrcReg, PhiTy, NarrowTy, Unused, PartRegs,
2067 LeftoverRegs))
2068 return UnableToLegalize;
2069
2070 // Add the newly created operand splits to the existing instructions. The
2071 // odd-sized pieces are ordered after the requested NarrowTyArg sized
2072 // pieces.
2073 for (int J = 0; J != TotalNumParts; ++J) {
2074 MachineInstrBuilder MIB = NewInsts[J];
2075 MIB.addUse(J < NumParts ? PartRegs[J] : LeftoverRegs[J - NumParts]);
2076 MIB.addMBB(&OpMBB);
2077 }
2078 }
2079
2080 MI.eraseFromParent();
2081 return Legalized;
2082}
2083
2084LegalizerHelper::LegalizeResult
2085LegalizerHelper::reduceLoadStoreWidth(MachineInstr &MI, unsigned TypeIdx,
2086 LLT NarrowTy) {
2087 // FIXME: Don't know how to handle secondary types yet.
2088 if (TypeIdx != 0)
2089 return UnableToLegalize;
2090
2091 MachineMemOperand *MMO = *MI.memoperands_begin();
2092
2093 // This implementation doesn't work for atomics. Give up instead of doing
2094 // something invalid.
2095 if (MMO->getOrdering() != AtomicOrdering::NotAtomic ||
2096 MMO->getFailureOrdering() != AtomicOrdering::NotAtomic)
2097 return UnableToLegalize;
2098
2099 bool IsLoad = MI.getOpcode() == TargetOpcode::G_LOAD;
2100 unsigned ValReg = MI.getOperand(0).getReg();
2101 unsigned AddrReg = MI.getOperand(1).getReg();
2102 LLT ValTy = MRI.getType(ValReg);
2103
2104 int NumParts = -1;
2105 int NumLeftover = -1;
2106 LLT LeftoverTy;
2107 SmallVector<unsigned, 8> NarrowRegs, NarrowLeftoverRegs;
2108 if (IsLoad) {
2109 std::tie(NumParts, NumLeftover) = getNarrowTypeBreakDown(ValTy, NarrowTy, LeftoverTy);
2110 } else {
2111 if (extractParts(ValReg, ValTy, NarrowTy, LeftoverTy, NarrowRegs,
2112 NarrowLeftoverRegs)) {
2113 NumParts = NarrowRegs.size();
2114 NumLeftover = NarrowLeftoverRegs.size();
Value stored to 'NumLeftover' is never read
2115 }
2116 }
2117
2118 if (NumParts == -1)
2119 return UnableToLegalize;
2120
2121 const LLT OffsetTy = LLT::scalar(MRI.getType(AddrReg).getScalarSizeInBits());
2122
2123 unsigned TotalSize = ValTy.getSizeInBits();
2124
2125 // Split the load/store into PartTy sized pieces starting at Offset. If this
2126 // is a load, return the new registers in ValRegs. For a store, each elements
2127 // of ValRegs should be PartTy. Returns the next offset that needs to be
2128 // handled.
2129 auto splitTypePieces = [=](LLT PartTy, SmallVectorImpl<unsigned> &ValRegs,
2130 unsigned Offset) -> unsigned {
2131 MachineFunction &MF = MIRBuilder.getMF();
2132 unsigned PartSize = PartTy.getSizeInBits();
2133 for (unsigned Idx = 0, E = NumParts; Idx != E && Offset < TotalSize;
2134 Offset += PartSize, ++Idx) {
2135 unsigned ByteSize = PartSize / 8;
2136 unsigned ByteOffset = Offset / 8;
2137 unsigned NewAddrReg = 0;
2138
2139 MIRBuilder.materializeGEP(NewAddrReg, AddrReg, OffsetTy, ByteOffset);
2140
2141 MachineMemOperand *NewMMO =
2142 MF.getMachineMemOperand(MMO, ByteOffset, ByteSize);
2143
2144 if (IsLoad) {
2145 unsigned Dst = MRI.createGenericVirtualRegister(PartTy);
2146 ValRegs.push_back(Dst);
2147 MIRBuilder.buildLoad(Dst, NewAddrReg, *NewMMO);
2148 } else {
2149 MIRBuilder.buildStore(ValRegs[Idx], NewAddrReg, *NewMMO);
2150 }
2151 }
2152
2153 return Offset;
2154 };
2155
2156 unsigned HandledOffset = splitTypePieces(NarrowTy, NarrowRegs, 0);
2157
2158 // Handle the rest of the register if this isn't an even type breakdown.
2159 if (LeftoverTy.isValid())
2160 splitTypePieces(LeftoverTy, NarrowLeftoverRegs, HandledOffset);
2161
2162 if (IsLoad) {
2163 insertParts(ValReg, ValTy, NarrowTy, NarrowRegs,
2164 LeftoverTy, NarrowLeftoverRegs);
2165 }
2166
2167 MI.eraseFromParent();
2168 return Legalized;
2169}
2170
2171LegalizerHelper::LegalizeResult
2172LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
2173 LLT NarrowTy) {
2174 using namespace TargetOpcode;
2175
2176 MIRBuilder.setInstr(MI);
2177 switch (MI.getOpcode()) {
2178 case G_IMPLICIT_DEF:
2179 return fewerElementsVectorImplicitDef(MI, TypeIdx, NarrowTy);
2180 case G_AND:
2181 case G_OR:
2182 case G_XOR:
2183 case G_ADD:
2184 case G_SUB:
2185 case G_MUL:
2186 case G_SMULH:
2187 case G_UMULH:
2188 case G_FADD:
2189 case G_FMUL:
2190 case G_FSUB:
2191 case G_FNEG:
2192 case G_FABS:
2193 case G_FCANONICALIZE:
2194 case G_FDIV:
2195 case G_FREM:
2196 case G_FMA:
2197 case G_FPOW:
2198 case G_FEXP:
2199 case G_FEXP2:
2200 case G_FLOG:
2201 case G_FLOG2:
2202 case G_FLOG10:
2203 case G_FNEARBYINT:
2204 case G_FCEIL:
2205 case G_FFLOOR:
2206 case G_FRINT:
2207 case G_INTRINSIC_ROUND:
2208 case G_INTRINSIC_TRUNC:
2209 case G_FCOS:
2210 case G_FSIN:
2211 case G_FSQRT:
2212 case G_BSWAP:
2213 case G_SDIV:
2214 case G_SMIN:
2215 case G_SMAX:
2216 case G_UMIN:
2217 case G_UMAX:
2218 return fewerElementsVectorBasic(MI, TypeIdx, NarrowTy);
2219 case G_SHL:
2220 case G_LSHR:
2221 case G_ASHR:
2222 case G_CTLZ:
2223 case G_CTLZ_ZERO_UNDEF:
2224 case G_CTTZ:
2225 case G_CTTZ_ZERO_UNDEF:
2226 case G_CTPOP:
2227 case G_FCOPYSIGN:
2228 return fewerElementsVectorMultiEltType(MI, TypeIdx, NarrowTy);
2229 case G_ZEXT:
2230 case G_SEXT:
2231 case G_ANYEXT:
2232 case G_FPEXT:
2233 case G_FPTRUNC:
2234 case G_SITOFP:
2235 case G_UITOFP:
2236 case G_FPTOSI:
2237 case G_FPTOUI:
2238 case G_INTTOPTR:
2239 case G_PTRTOINT:
2240 case G_ADDRSPACE_CAST:
2241 return fewerElementsVectorCasts(MI, TypeIdx, NarrowTy);
2242 case G_ICMP:
2243 case G_FCMP:
2244 return fewerElementsVectorCmp(MI, TypeIdx, NarrowTy);
2245 case G_SELECT:
2246 return fewerElementsVectorSelect(MI, TypeIdx, NarrowTy);
2247 case G_PHI:
2248 return fewerElementsVectorPhi(MI, TypeIdx, NarrowTy);
2249 case G_LOAD:
2250 case G_STORE:
2251 return reduceLoadStoreWidth(MI, TypeIdx, NarrowTy);
2252 default:
2253 return UnableToLegalize;
2254 }
2255}
2256
2257LegalizerHelper::LegalizeResult
2258LegalizerHelper::narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt,
2259 const LLT HalfTy, const LLT AmtTy) {
2260
2261 unsigned InL = MRI.createGenericVirtualRegister(HalfTy);
2262 unsigned InH = MRI.createGenericVirtualRegister(HalfTy);
2263 MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1).getReg());
2264
2265 if (Amt.isNullValue()) {
2266 MIRBuilder.buildMerge(MI.getOperand(0).getReg(), {InL, InH});
2267 MI.eraseFromParent();
2268 return Legalized;
2269 }
2270
2271 LLT NVT = HalfTy;
2272 unsigned NVTBits = HalfTy.getSizeInBits();
2273 unsigned VTBits = 2 * NVTBits;
2274
2275 SrcOp Lo(0), Hi(0);
2276 if (MI.getOpcode() == TargetOpcode::G_SHL) {
2277 if (Amt.ugt(VTBits)) {
2278 Lo = Hi = MIRBuilder.buildConstant(NVT, 0);
2279 } else if (Amt.ugt(NVTBits)) {
2280 Lo = MIRBuilder.buildConstant(NVT, 0);
2281 Hi = MIRBuilder.buildShl(NVT, InL,
2282 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
2283 } else if (Amt == NVTBits) {
2284 Lo = MIRBuilder.buildConstant(NVT, 0);
2285 Hi = InL;
2286 } else {
2287 Lo = MIRBuilder.buildShl(NVT, InL, MIRBuilder.buildConstant(AmtTy, Amt));
2288 auto OrLHS =
2289 MIRBuilder.buildShl(NVT, InH, MIRBuilder.buildConstant(AmtTy, Amt));
2290 auto OrRHS = MIRBuilder.buildLShr(
2291 NVT, InL, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
2292 Hi = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
2293 }
2294 } else if (MI.getOpcode() == TargetOpcode::G_LSHR) {
2295 if (Amt.ugt(VTBits)) {
2296 Lo = Hi = MIRBuilder.buildConstant(NVT, 0);
2297 } else if (Amt.ugt(NVTBits)) {
2298 Lo = MIRBuilder.buildLShr(NVT, InH,
2299 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
2300 Hi = MIRBuilder.buildConstant(NVT, 0);
2301 } else if (Amt == NVTBits) {
2302 Lo = InH;
2303 Hi = MIRBuilder.buildConstant(NVT, 0);
2304 } else {
2305 auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt);
2306
2307 auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst);
2308 auto OrRHS = MIRBuilder.buildShl(
2309 NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
2310
2311 Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
2312 Hi = MIRBuilder.buildLShr(NVT, InH, ShiftAmtConst);
2313 }
2314 } else {
2315 if (Amt.ugt(VTBits)) {
2316 Hi = Lo = MIRBuilder.buildAShr(
2317 NVT, InH, MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
2318 } else if (Amt.ugt(NVTBits)) {
2319 Lo = MIRBuilder.buildAShr(NVT, InH,
2320 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
2321 Hi = MIRBuilder.buildAShr(NVT, InH,
2322 MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
2323 } else if (Amt == NVTBits) {
2324 Lo = InH;
2325 Hi = MIRBuilder.buildAShr(NVT, InH,
2326 MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
2327 } else {
2328 auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt);
2329
2330 auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst);
2331 auto OrRHS = MIRBuilder.buildShl(
2332 NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
2333
2334 Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
2335 Hi = MIRBuilder.buildAShr(NVT, InH, ShiftAmtConst);
2336 }
2337 }
2338
2339 MIRBuilder.buildMerge(MI.getOperand(0).getReg(), {Lo.getReg(), Hi.getReg()});
2340 MI.eraseFromParent();
2341
2342 return Legalized;
2343}
2344
2345// TODO: Optimize if constant shift amount.
2346LegalizerHelper::LegalizeResult
2347LegalizerHelper::narrowScalarShift(MachineInstr &MI, unsigned TypeIdx,
2348 LLT RequestedTy) {
2349 if (TypeIdx == 1) {
2350 Observer.changingInstr(MI);
2351 narrowScalarSrc(MI, RequestedTy, 2);
2352 Observer.changedInstr(MI);
2353 return Legalized;
2354 }
2355
2356 unsigned DstReg = MI.getOperand(0).getReg();
2357 LLT DstTy = MRI.getType(DstReg);
2358 if (DstTy.isVector())
2359 return UnableToLegalize;
2360
2361 unsigned Amt = MI.getOperand(2).getReg();
2362 LLT ShiftAmtTy = MRI.getType(Amt);
2363 const unsigned DstEltSize = DstTy.getScalarSizeInBits();
2364 if (DstEltSize % 2 != 0)
2365 return UnableToLegalize;
2366
2367 // Ignore the input type. We can only go to exactly half the size of the
2368 // input. If that isn't small enough, the resulting pieces will be further
2369 // legalized.
2370 const unsigned NewBitSize = DstEltSize / 2;
2371 const LLT HalfTy = LLT::scalar(NewBitSize);
2372 const LLT CondTy = LLT::scalar(1);
2373
2374 if (const MachineInstr *KShiftAmt =
2375 getOpcodeDef(TargetOpcode::G_CONSTANT, Amt, MRI)) {
2376 return narrowScalarShiftByConstant(
2377 MI, KShiftAmt->getOperand(1).getCImm()->getValue(), HalfTy, ShiftAmtTy);
2378 }
2379
2380 // TODO: Expand with known bits.
2381
2382 // Handle the fully general expansion by an unknown amount.
2383 auto NewBits = MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize);
2384
2385 unsigned InL = MRI.createGenericVirtualRegister(HalfTy);
2386 unsigned InH = MRI.createGenericVirtualRegister(HalfTy);
2387 MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1).getReg());
2388
2389 auto AmtExcess = MIRBuilder.buildSub(ShiftAmtTy, Amt, NewBits);
2390 auto AmtLack = MIRBuilder.buildSub(ShiftAmtTy, NewBits, Amt);
2391
2392 auto Zero = MIRBuilder.buildConstant(ShiftAmtTy, 0);
2393 auto IsShort = MIRBuilder.buildICmp(ICmpInst::ICMP_ULT, CondTy, Amt, NewBits);
2394 auto IsZero = MIRBuilder.buildICmp(ICmpInst::ICMP_EQ, CondTy, Amt, Zero);
2395
2396 unsigned ResultRegs[2];
2397 switch (MI.getOpcode()) {
2398 case TargetOpcode::G_SHL: {
2399 // Short: ShAmt < NewBitSize
2400 auto LoS = MIRBuilder.buildShl(HalfTy, InH, Amt);
2401
2402 auto OrLHS = MIRBuilder.buildShl(HalfTy, InH, Amt);
2403 auto OrRHS = MIRBuilder.buildLShr(HalfTy, InL, AmtLack);
2404 auto HiS = MIRBuilder.buildOr(HalfTy, OrLHS, OrRHS);
2405
2406 // Long: ShAmt >= NewBitSize
2407 auto LoL = MIRBuilder.buildConstant(HalfTy, 0); // Lo part is zero.
2408 auto HiL = MIRBuilder.buildShl(HalfTy, InL, AmtExcess); // Hi from Lo part.
2409
2410 auto Lo = MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL);
2411 auto Hi = MIRBuilder.buildSelect(
2412 HalfTy, IsZero, InH, MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL));
2413
2414 ResultRegs[0] = Lo.getReg(0);
2415 ResultRegs[1] = Hi.getReg(0);
2416 break;
2417 }
2418 case TargetOpcode::G_LSHR: {
2419 // Short: ShAmt < NewBitSize
2420 auto HiS = MIRBuilder.buildLShr(HalfTy, InH, Amt);
2421
2422 auto OrLHS = MIRBuilder.buildLShr(HalfTy, InL, Amt);
2423 auto OrRHS = MIRBuilder.buildShl(HalfTy, InH, AmtLack);
2424 auto LoS = MIRBuilder.buildOr(HalfTy, OrLHS, OrRHS);
2425
2426 // Long: ShAmt >= NewBitSize
2427 auto HiL = MIRBuilder.buildConstant(HalfTy, 0); // Hi part is zero.
2428 auto LoL = MIRBuilder.buildLShr(HalfTy, InH, AmtExcess); // Lo from Hi part.
2429
2430 auto Lo = MIRBuilder.buildSelect(
2431 HalfTy, IsZero, InL, MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL));
2432 auto Hi = MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL);
2433
2434 ResultRegs[0] = Lo.getReg(0);
2435 ResultRegs[1] = Hi.getReg(0);
2436 break;
2437 }
2438 case TargetOpcode::G_ASHR: {
2439 // Short: ShAmt < NewBitSize
2440 auto HiS = MIRBuilder.buildAShr(HalfTy, InH, Amt);
2441
2442 auto OrLHS = MIRBuilder.buildLShr(HalfTy, InL, Amt);
2443 auto OrRHS = MIRBuilder.buildLShr(HalfTy, InH, AmtLack);
2444 auto LoS = MIRBuilder.buildOr(HalfTy, OrLHS, OrRHS);
2445
2446 // Long: ShAmt >= NewBitSize
2447
2448 // Sign of Hi part.
2449 auto HiL = MIRBuilder.buildAShr(
2450 HalfTy, InH, MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize - 1));
2451
2452 auto LoL = MIRBuilder.buildAShr(HalfTy, InH, AmtExcess); // Lo from Hi part.
2453
2454 auto Lo = MIRBuilder.buildSelect(
2455 HalfTy, IsZero, InL, MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL));
2456
2457 auto Hi = MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL);
2458
2459 ResultRegs[0] = Lo.getReg(0);
2460 ResultRegs[1] = Hi.getReg(0);
2461 break;
2462 }
2463 default:
2464 llvm_unreachable("not a shift")::llvm::llvm_unreachable_internal("not a shift", "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 2464)
;
2465 }
2466
2467 MIRBuilder.buildMerge(DstReg, ResultRegs);
2468 MI.eraseFromParent();
2469 return Legalized;
2470}
2471
2472LegalizerHelper::LegalizeResult
2473LegalizerHelper::moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx,
2474 LLT MoreTy) {
2475 assert(TypeIdx == 0 && "Expecting only Idx 0")((TypeIdx == 0 && "Expecting only Idx 0") ? static_cast
<void> (0) : __assert_fail ("TypeIdx == 0 && \"Expecting only Idx 0\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 2475, __PRETTY_FUNCTION__))
;
2476
2477 Observer.changingInstr(MI);
2478 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
2479 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
2480 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
2481 moreElementsVectorSrc(MI, MoreTy, I);
2482 }
2483
2484 MachineBasicBlock &MBB = *MI.getParent();
2485 MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI());
2486 moreElementsVectorDst(MI, MoreTy, 0);
2487 Observer.changedInstr(MI);
2488 return Legalized;
2489}
2490
2491LegalizerHelper::LegalizeResult
2492LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
2493 LLT MoreTy) {
2494 MIRBuilder.setInstr(MI);
2495 unsigned Opc = MI.getOpcode();
2496 switch (Opc) {
2497 case TargetOpcode::G_IMPLICIT_DEF: {
2498 Observer.changingInstr(MI);
2499 moreElementsVectorDst(MI, MoreTy, 0);
2500 Observer.changedInstr(MI);
2501 return Legalized;
2502 }
2503 case TargetOpcode::G_AND:
2504 case TargetOpcode::G_OR:
2505 case TargetOpcode::G_XOR:
2506 case TargetOpcode::G_SMIN:
2507 case TargetOpcode::G_SMAX:
2508 case TargetOpcode::G_UMIN:
2509 case TargetOpcode::G_UMAX: {
2510 Observer.changingInstr(MI);
2511 moreElementsVectorSrc(MI, MoreTy, 1);
2512 moreElementsVectorSrc(MI, MoreTy, 2);
2513 moreElementsVectorDst(MI, MoreTy, 0);
2514 Observer.changedInstr(MI);
2515 return Legalized;
2516 }
2517 case TargetOpcode::G_EXTRACT:
2518 if (TypeIdx != 1)
2519 return UnableToLegalize;
2520 Observer.changingInstr(MI);
2521 moreElementsVectorSrc(MI, MoreTy, 1);
2522 Observer.changedInstr(MI);
2523 return Legalized;
2524 case TargetOpcode::G_INSERT:
2525 if (TypeIdx != 0)
2526 return UnableToLegalize;
2527 Observer.changingInstr(MI);
2528 moreElementsVectorSrc(MI, MoreTy, 1);
2529 moreElementsVectorDst(MI, MoreTy, 0);
2530 Observer.changedInstr(MI);
2531 return Legalized;
2532 case TargetOpcode::G_SELECT:
2533 if (TypeIdx != 0)
2534 return UnableToLegalize;
2535 if (MRI.getType(MI.getOperand(1).getReg()).isVector())
2536 return UnableToLegalize;
2537
2538 Observer.changingInstr(MI);
2539 moreElementsVectorSrc(MI, MoreTy, 2);
2540 moreElementsVectorSrc(MI, MoreTy, 3);
2541 moreElementsVectorDst(MI, MoreTy, 0);
2542 Observer.changedInstr(MI);
2543 return Legalized;
2544 case TargetOpcode::G_PHI:
2545 return moreElementsVectorPhi(MI, TypeIdx, MoreTy);
2546 default:
2547 return UnableToLegalize;
2548 }
2549}
2550
2551void LegalizerHelper::multiplyRegisters(SmallVectorImpl<unsigned> &DstRegs,
2552 ArrayRef<unsigned> Src1Regs,
2553 ArrayRef<unsigned> Src2Regs,
2554 LLT NarrowTy) {
2555 MachineIRBuilder &B = MIRBuilder;
2556 unsigned SrcParts = Src1Regs.size();
2557 unsigned DstParts = DstRegs.size();
2558
2559 unsigned DstIdx = 0; // Low bits of the result.
2560 unsigned FactorSum =
2561 B.buildMul(NarrowTy, Src1Regs[DstIdx], Src2Regs[DstIdx]).getReg(0);
2562 DstRegs[DstIdx] = FactorSum;
2563
2564 unsigned CarrySumPrevDstIdx;
2565 SmallVector<unsigned, 4> Factors;
2566
2567 for (DstIdx = 1; DstIdx < DstParts; DstIdx++) {
2568 // Collect low parts of muls for DstIdx.
2569 for (unsigned i = DstIdx + 1 < SrcParts ? 0 : DstIdx - SrcParts + 1;
2570 i <= std::min(DstIdx, SrcParts - 1); ++i) {
2571 MachineInstrBuilder Mul =
2572 B.buildMul(NarrowTy, Src1Regs[DstIdx - i], Src2Regs[i]);
2573 Factors.push_back(Mul.getReg(0));
2574 }
2575 // Collect high parts of muls from previous DstIdx.
2576 for (unsigned i = DstIdx < SrcParts ? 0 : DstIdx - SrcParts;
2577 i <= std::min(DstIdx - 1, SrcParts - 1); ++i) {
2578 MachineInstrBuilder Umulh =
2579 B.buildUMulH(NarrowTy, Src1Regs[DstIdx - 1 - i], Src2Regs[i]);
2580 Factors.push_back(Umulh.getReg(0));
2581 }
2582 // Add CarrySum from additons calculated for previous DstIdx.
2583 if (DstIdx != 1) {
2584 Factors.push_back(CarrySumPrevDstIdx);
2585 }
2586
2587 unsigned CarrySum = 0;
2588 // Add all factors and accumulate all carries into CarrySum.
2589 if (DstIdx != DstParts - 1) {
2590 MachineInstrBuilder Uaddo =
2591 B.buildUAddo(NarrowTy, LLT::scalar(1), Factors[0], Factors[1]);
2592 FactorSum = Uaddo.getReg(0);
2593 CarrySum = B.buildZExt(NarrowTy, Uaddo.getReg(1)).getReg(0);
2594 for (unsigned i = 2; i < Factors.size(); ++i) {
2595 MachineInstrBuilder Uaddo =
2596 B.buildUAddo(NarrowTy, LLT::scalar(1), FactorSum, Factors[i]);
2597 FactorSum = Uaddo.getReg(0);
2598 MachineInstrBuilder Carry = B.buildZExt(NarrowTy, Uaddo.getReg(1));
2599 CarrySum = B.buildAdd(NarrowTy, CarrySum, Carry).getReg(0);
2600 }
2601 } else {
2602 // Since value for the next index is not calculated, neither is CarrySum.
2603 FactorSum = B.buildAdd(NarrowTy, Factors[0], Factors[1]).getReg(0);
2604 for (unsigned i = 2; i < Factors.size(); ++i)
2605 FactorSum = B.buildAdd(NarrowTy, FactorSum, Factors[i]).getReg(0);
2606 }
2607
2608 CarrySumPrevDstIdx = CarrySum;
2609 DstRegs[DstIdx] = FactorSum;
2610 Factors.clear();
2611 }
2612}
2613
2614LegalizerHelper::LegalizeResult
2615LegalizerHelper::narrowScalarMul(MachineInstr &MI, LLT NarrowTy) {
2616 unsigned DstReg = MI.getOperand(0).getReg();
2617 unsigned Src1 = MI.getOperand(1).getReg();
2618 unsigned Src2 = MI.getOperand(2).getReg();
2619
2620 LLT Ty = MRI.getType(DstReg);
2621 if (Ty.isVector())
2622 return UnableToLegalize;
2623
2624 unsigned SrcSize = MRI.getType(Src1).getSizeInBits();
2625 unsigned DstSize = Ty.getSizeInBits();
2626 unsigned NarrowSize = NarrowTy.getSizeInBits();
2627 if (DstSize % NarrowSize != 0 || SrcSize % NarrowSize != 0)
2628 return UnableToLegalize;
2629
2630 unsigned NumDstParts = DstSize / NarrowSize;
2631 unsigned NumSrcParts = SrcSize / NarrowSize;
2632 bool IsMulHigh = MI.getOpcode() == TargetOpcode::G_UMULH;
2633 unsigned DstTmpParts = NumDstParts * (IsMulHigh ? 2 : 1);
2634
2635 SmallVector<unsigned, 2> Src1Parts, Src2Parts, DstTmpRegs;
2636 extractParts(Src1, NarrowTy, NumSrcParts, Src1Parts);
2637 extractParts(Src2, NarrowTy, NumSrcParts, Src2Parts);
2638 DstTmpRegs.resize(DstTmpParts);
2639 multiplyRegisters(DstTmpRegs, Src1Parts, Src2Parts, NarrowTy);
2640
2641 // Take only high half of registers if this is high mul.
2642 ArrayRef<unsigned> DstRegs(
2643 IsMulHigh ? &DstTmpRegs[DstTmpParts / 2] : &DstTmpRegs[0], NumDstParts);
2644 MIRBuilder.buildMerge(DstReg, DstRegs);
2645 MI.eraseFromParent();
2646 return Legalized;
2647}
2648
2649LegalizerHelper::LegalizeResult
2650LegalizerHelper::narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx,
2651 LLT NarrowTy) {
2652 if (TypeIdx != 1)
2653 return UnableToLegalize;
2654
2655 uint64_t NarrowSize = NarrowTy.getSizeInBits();
2656
2657 int64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
2658 // FIXME: add support for when SizeOp1 isn't an exact multiple of
2659 // NarrowSize.
2660 if (SizeOp1 % NarrowSize != 0)
2661 return UnableToLegalize;
2662 int NumParts = SizeOp1 / NarrowSize;
2663
2664 SmallVector<unsigned, 2> SrcRegs, DstRegs;
2665 SmallVector<uint64_t, 2> Indexes;
2666 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
2667
2668 unsigned OpReg = MI.getOperand(0).getReg();
2669 uint64_t OpStart = MI.getOperand(2).getImm();
2670 uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
2671 for (int i = 0; i < NumParts; ++i) {
2672 unsigned SrcStart = i * NarrowSize;
2673
2674 if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) {
2675 // No part of the extract uses this subregister, ignore it.
2676 continue;
2677 } else if (SrcStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
2678 // The entire subregister is extracted, forward the value.
2679 DstRegs.push_back(SrcRegs[i]);
2680 continue;
2681 }
2682
2683 // OpSegStart is where this destination segment would start in OpReg if it
2684 // extended infinitely in both directions.
2685 int64_t ExtractOffset;
2686 uint64_t SegSize;
2687 if (OpStart < SrcStart) {
2688 ExtractOffset = 0;
2689 SegSize = std::min(NarrowSize, OpStart + OpSize - SrcStart);
2690 } else {
2691 ExtractOffset = OpStart - SrcStart;
2692 SegSize = std::min(SrcStart + NarrowSize - OpStart, OpSize);
2693 }
2694
2695 unsigned SegReg = SrcRegs[i];
2696 if (ExtractOffset != 0 || SegSize != NarrowSize) {
2697 // A genuine extract is needed.
2698 SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
2699 MIRBuilder.buildExtract(SegReg, SrcRegs[i], ExtractOffset);
2700 }
2701
2702 DstRegs.push_back(SegReg);
2703 }
2704
2705 unsigned DstReg = MI.getOperand(0).getReg();
2706 if(MRI.getType(DstReg).isVector())
2707 MIRBuilder.buildBuildVector(DstReg, DstRegs);
2708 else
2709 MIRBuilder.buildMerge(DstReg, DstRegs);
2710 MI.eraseFromParent();
2711 return Legalized;
2712}
2713
2714LegalizerHelper::LegalizeResult
2715LegalizerHelper::narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx,
2716 LLT NarrowTy) {
2717 // FIXME: Don't know how to handle secondary types yet.
2718 if (TypeIdx != 0)
2719 return UnableToLegalize;
2720
2721 uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
2722 uint64_t NarrowSize = NarrowTy.getSizeInBits();
2723
2724 // FIXME: add support for when SizeOp0 isn't an exact multiple of
2725 // NarrowSize.
2726 if (SizeOp0 % NarrowSize != 0)
2727 return UnableToLegalize;
2728
2729 int NumParts = SizeOp0 / NarrowSize;
2730
2731 SmallVector<unsigned, 2> SrcRegs, DstRegs;
2732 SmallVector<uint64_t, 2> Indexes;
2733 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
2734
2735 unsigned OpReg = MI.getOperand(2).getReg();
2736 uint64_t OpStart = MI.getOperand(3).getImm();
2737 uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
2738 for (int i = 0; i < NumParts; ++i) {
2739 unsigned DstStart = i * NarrowSize;
2740
2741 if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) {
2742 // No part of the insert affects this subregister, forward the original.
2743 DstRegs.push_back(SrcRegs[i]);
2744 continue;
2745 } else if (DstStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
2746 // The entire subregister is defined by this insert, forward the new
2747 // value.
2748 DstRegs.push_back(OpReg);
2749 continue;
2750 }
2751
2752 // OpSegStart is where this destination segment would start in OpReg if it
2753 // extended infinitely in both directions.
2754 int64_t ExtractOffset, InsertOffset;
2755 uint64_t SegSize;
2756 if (OpStart < DstStart) {
2757 InsertOffset = 0;
2758 ExtractOffset = DstStart - OpStart;
2759 SegSize = std::min(NarrowSize, OpStart + OpSize - DstStart);
2760 } else {
2761 InsertOffset = OpStart - DstStart;
2762 ExtractOffset = 0;
2763 SegSize =
2764 std::min(NarrowSize - InsertOffset, OpStart + OpSize - DstStart);
2765 }
2766
2767 unsigned SegReg = OpReg;
2768 if (ExtractOffset != 0 || SegSize != OpSize) {
2769 // A genuine extract is needed.
2770 SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
2771 MIRBuilder.buildExtract(SegReg, OpReg, ExtractOffset);
2772 }
2773
2774 unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
2775 MIRBuilder.buildInsert(DstReg, SrcRegs[i], SegReg, InsertOffset);
2776 DstRegs.push_back(DstReg);
2777 }
2778
2779 assert(DstRegs.size() == (unsigned)NumParts && "not all parts covered")((DstRegs.size() == (unsigned)NumParts && "not all parts covered"
) ? static_cast<void> (0) : __assert_fail ("DstRegs.size() == (unsigned)NumParts && \"not all parts covered\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 2779, __PRETTY_FUNCTION__))
;
2780 unsigned DstReg = MI.getOperand(0).getReg();
2781 if(MRI.getType(DstReg).isVector())
2782 MIRBuilder.buildBuildVector(DstReg, DstRegs);
2783 else
2784 MIRBuilder.buildMerge(DstReg, DstRegs);
2785 MI.eraseFromParent();
2786 return Legalized;
2787}
2788
2789LegalizerHelper::LegalizeResult
2790LegalizerHelper::narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx,
2791 LLT NarrowTy) {
2792 unsigned DstReg = MI.getOperand(0).getReg();
2793 LLT DstTy = MRI.getType(DstReg);
2794
2795 assert(MI.getNumOperands() == 3 && TypeIdx == 0)((MI.getNumOperands() == 3 && TypeIdx == 0) ? static_cast
<void> (0) : __assert_fail ("MI.getNumOperands() == 3 && TypeIdx == 0"
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 2795, __PRETTY_FUNCTION__))
;
2796
2797 SmallVector<unsigned, 4> DstRegs, DstLeftoverRegs;
2798 SmallVector<unsigned, 4> Src0Regs, Src0LeftoverRegs;
2799 SmallVector<unsigned, 4> Src1Regs, Src1LeftoverRegs;
2800 LLT LeftoverTy;
2801 if (!extractParts(MI.getOperand(1).getReg(), DstTy, NarrowTy, LeftoverTy,
2802 Src0Regs, Src0LeftoverRegs))
2803 return UnableToLegalize;
2804
2805 LLT Unused;
2806 if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, Unused,
2807 Src1Regs, Src1LeftoverRegs))
2808 llvm_unreachable("inconsistent extractParts result")::llvm::llvm_unreachable_internal("inconsistent extractParts result"
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 2808)
;
2809
2810 for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) {
2811 auto Inst = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy},
2812 {Src0Regs[I], Src1Regs[I]});
2813 DstRegs.push_back(Inst->getOperand(0).getReg());
2814 }
2815
2816 for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) {
2817 auto Inst = MIRBuilder.buildInstr(
2818 MI.getOpcode(),
2819 {LeftoverTy}, {Src0LeftoverRegs[I], Src1LeftoverRegs[I]});
2820 DstLeftoverRegs.push_back(Inst->getOperand(0).getReg());
2821 }
2822
2823 insertParts(DstReg, DstTy, NarrowTy, DstRegs,
2824 LeftoverTy, DstLeftoverRegs);
2825
2826 MI.eraseFromParent();
2827 return Legalized;
2828}
2829
2830LegalizerHelper::LegalizeResult
2831LegalizerHelper::narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx,
2832 LLT NarrowTy) {
2833 if (TypeIdx != 0)
2834 return UnableToLegalize;
2835
2836 unsigned CondReg = MI.getOperand(1).getReg();
2837 LLT CondTy = MRI.getType(CondReg);
2838 if (CondTy.isVector()) // TODO: Handle vselect
2839 return UnableToLegalize;
2840
2841 unsigned DstReg = MI.getOperand(0).getReg();
2842 LLT DstTy = MRI.getType(DstReg);
2843
2844 SmallVector<unsigned, 4> DstRegs, DstLeftoverRegs;
2845 SmallVector<unsigned, 4> Src1Regs, Src1LeftoverRegs;
2846 SmallVector<unsigned, 4> Src2Regs, Src2LeftoverRegs;
2847 LLT LeftoverTy;
2848 if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, LeftoverTy,
2849 Src1Regs, Src1LeftoverRegs))
2850 return UnableToLegalize;
2851
2852 LLT Unused;
2853 if (!extractParts(MI.getOperand(3).getReg(), DstTy, NarrowTy, Unused,
2854 Src2Regs, Src2LeftoverRegs))
2855 llvm_unreachable("inconsistent extractParts result")::llvm::llvm_unreachable_internal("inconsistent extractParts result"
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 2855)
;
2856
2857 for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) {
2858 auto Select = MIRBuilder.buildSelect(NarrowTy,
2859 CondReg, Src1Regs[I], Src2Regs[I]);
2860 DstRegs.push_back(Select->getOperand(0).getReg());
2861 }
2862
2863 for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) {
2864 auto Select = MIRBuilder.buildSelect(
2865 LeftoverTy, CondReg, Src1LeftoverRegs[I], Src2LeftoverRegs[I]);
2866 DstLeftoverRegs.push_back(Select->getOperand(0).getReg());
2867 }
2868
2869 insertParts(DstReg, DstTy, NarrowTy, DstRegs,
2870 LeftoverTy, DstLeftoverRegs);
2871
2872 MI.eraseFromParent();
2873 return Legalized;
2874}
2875
2876LegalizerHelper::LegalizeResult
2877LegalizerHelper::lowerBitCount(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
2878 unsigned Opc = MI.getOpcode();
2879 auto &TII = *MI.getMF()->getSubtarget().getInstrInfo();
2880 auto isSupported = [this](const LegalityQuery &Q) {
2881 auto QAction = LI.getAction(Q).Action;
2882 return QAction == Legal || QAction == Libcall || QAction == Custom;
2883 };
2884 switch (Opc) {
2885 default:
2886 return UnableToLegalize;
2887 case TargetOpcode::G_CTLZ_ZERO_UNDEF: {
2888 // This trivially expands to CTLZ.
2889 Observer.changingInstr(MI);
2890 MI.setDesc(TII.get(TargetOpcode::G_CTLZ));
2891 Observer.changedInstr(MI);
2892 return Legalized;
2893 }
2894 case TargetOpcode::G_CTLZ: {
2895 unsigned SrcReg = MI.getOperand(1).getReg();
2896 unsigned Len = Ty.getSizeInBits();
2897 if (isSupported({TargetOpcode::G_CTLZ_ZERO_UNDEF, {Ty, Ty}})) {
2898 // If CTLZ_ZERO_UNDEF is supported, emit that and a select for zero.
2899 auto MIBCtlzZU = MIRBuilder.buildInstr(TargetOpcode::G_CTLZ_ZERO_UNDEF,
2900 {Ty}, {SrcReg});
2901 auto MIBZero = MIRBuilder.buildConstant(Ty, 0);
2902 auto MIBLen = MIRBuilder.buildConstant(Ty, Len);
2903 auto MIBICmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1),
2904 SrcReg, MIBZero);
2905 MIRBuilder.buildSelect(MI.getOperand(0).getReg(), MIBICmp, MIBLen,
2906 MIBCtlzZU);
2907 MI.eraseFromParent();
2908 return Legalized;
2909 }
2910 // for now, we do this:
2911 // NewLen = NextPowerOf2(Len);
2912 // x = x | (x >> 1);
2913 // x = x | (x >> 2);
2914 // ...
2915 // x = x | (x >>16);
2916 // x = x | (x >>32); // for 64-bit input
2917 // Upto NewLen/2
2918 // return Len - popcount(x);
2919 //
2920 // Ref: "Hacker's Delight" by Henry Warren
2921 unsigned Op = SrcReg;
2922 unsigned NewLen = PowerOf2Ceil(Len);
2923 for (unsigned i = 0; (1U << i) <= (NewLen / 2); ++i) {
2924 auto MIBShiftAmt = MIRBuilder.buildConstant(Ty, 1ULL << i);
2925 auto MIBOp = MIRBuilder.buildInstr(
2926 TargetOpcode::G_OR, {Ty},
2927 {Op, MIRBuilder.buildInstr(TargetOpcode::G_LSHR, {Ty},
2928 {Op, MIBShiftAmt})});
2929 Op = MIBOp->getOperand(0).getReg();
2930 }
2931 auto MIBPop = MIRBuilder.buildInstr(TargetOpcode::G_CTPOP, {Ty}, {Op});
2932 MIRBuilder.buildInstr(TargetOpcode::G_SUB, {MI.getOperand(0).getReg()},
2933 {MIRBuilder.buildConstant(Ty, Len), MIBPop});
2934 MI.eraseFromParent();
2935 return Legalized;
2936 }
2937 case TargetOpcode::G_CTTZ_ZERO_UNDEF: {
2938 // This trivially expands to CTTZ.
2939 Observer.changingInstr(MI);
2940 MI.setDesc(TII.get(TargetOpcode::G_CTTZ));
2941 Observer.changedInstr(MI);
2942 return Legalized;
2943 }
2944 case TargetOpcode::G_CTTZ: {
2945 unsigned SrcReg = MI.getOperand(1).getReg();
2946 unsigned Len = Ty.getSizeInBits();
2947 if (isSupported({TargetOpcode::G_CTTZ_ZERO_UNDEF, {Ty, Ty}})) {
2948 // If CTTZ_ZERO_UNDEF is legal or custom, emit that and a select with
2949 // zero.
2950 auto MIBCttzZU = MIRBuilder.buildInstr(TargetOpcode::G_CTTZ_ZERO_UNDEF,
2951 {Ty}, {SrcReg});
2952 auto MIBZero = MIRBuilder.buildConstant(Ty, 0);
2953 auto MIBLen = MIRBuilder.buildConstant(Ty, Len);
2954 auto MIBICmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1),
2955 SrcReg, MIBZero);
2956 MIRBuilder.buildSelect(MI.getOperand(0).getReg(), MIBICmp, MIBLen,
2957 MIBCttzZU);
2958 MI.eraseFromParent();
2959 return Legalized;
2960 }
2961 // for now, we use: { return popcount(~x & (x - 1)); }
2962 // unless the target has ctlz but not ctpop, in which case we use:
2963 // { return 32 - nlz(~x & (x-1)); }
2964 // Ref: "Hacker's Delight" by Henry Warren
2965 auto MIBCstNeg1 = MIRBuilder.buildConstant(Ty, -1);
2966 auto MIBNot =
2967 MIRBuilder.buildInstr(TargetOpcode::G_XOR, {Ty}, {SrcReg, MIBCstNeg1});
2968 auto MIBTmp = MIRBuilder.buildInstr(
2969 TargetOpcode::G_AND, {Ty},
2970 {MIBNot, MIRBuilder.buildInstr(TargetOpcode::G_ADD, {Ty},
2971 {SrcReg, MIBCstNeg1})});
2972 if (!isSupported({TargetOpcode::G_CTPOP, {Ty, Ty}}) &&
2973 isSupported({TargetOpcode::G_CTLZ, {Ty, Ty}})) {
2974 auto MIBCstLen = MIRBuilder.buildConstant(Ty, Len);
2975 MIRBuilder.buildInstr(
2976 TargetOpcode::G_SUB, {MI.getOperand(0).getReg()},
2977 {MIBCstLen,
2978 MIRBuilder.buildInstr(TargetOpcode::G_CTLZ, {Ty}, {MIBTmp})});
2979 MI.eraseFromParent();
2980 return Legalized;
2981 }
2982 MI.setDesc(TII.get(TargetOpcode::G_CTPOP));
2983 MI.getOperand(1).setReg(MIBTmp->getOperand(0).getReg());
2984 return Legalized;
2985 }
2986 }
2987}
2988
2989// Expand s32 = G_UITOFP s64 using bit operations to an IEEE float
2990// representation.
2991LegalizerHelper::LegalizeResult
2992LegalizerHelper::lowerU64ToF32BitOps(MachineInstr &MI) {
2993 unsigned Dst = MI.getOperand(0).getReg();
2994 unsigned Src = MI.getOperand(1).getReg();
2995 const LLT S64 = LLT::scalar(64);
2996 const LLT S32 = LLT::scalar(32);
2997 const LLT S1 = LLT::scalar(1);
2998
2999 assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S32)((MRI.getType(Src) == S64 && MRI.getType(Dst) == S32)
? static_cast<void> (0) : __assert_fail ("MRI.getType(Src) == S64 && MRI.getType(Dst) == S32"
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/CodeGen/GlobalISel/LegalizerHelper.cpp"
, 2999, __PRETTY_FUNCTION__))
;
3000
3001 // unsigned cul2f(ulong u) {
3002 // uint lz = clz(u);
3003 // uint e = (u != 0) ? 127U + 63U - lz : 0;
3004 // u = (u << lz) & 0x7fffffffffffffffUL;
3005 // ulong t = u & 0xffffffffffUL;
3006 // uint v = (e << 23) | (uint)(u >> 40);
3007 // uint r = t > 0x8000000000UL ? 1U : (t == 0x8000000000UL ? v & 1U : 0U);
3008 // return as_float(v + r);
3009 // }
3010
3011 auto Zero32 = MIRBuilder.buildConstant(S32, 0);
3012 auto Zero64 = MIRBuilder.buildConstant(S64, 0);
3013
3014 auto LZ = MIRBuilder.buildCTLZ_ZERO_UNDEF(S32, Src);
3015
3016 auto K = MIRBuilder.buildConstant(S32, 127U + 63U);
3017 auto Sub = MIRBuilder.buildSub(S32, K, LZ);
3018
3019 auto NotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, Src, Zero64);
3020 auto E = MIRBuilder.buildSelect(S32, NotZero, Sub, Zero32);
3021
3022 auto Mask0 = MIRBuilder.buildConstant(S64, (-1ULL) >> 1);
3023 auto ShlLZ = MIRBuilder.buildShl(S64, Src, LZ);
3024
3025 auto U = MIRBuilder.buildAnd(S64, ShlLZ, Mask0);
3026
3027 auto Mask1 = MIRBuilder.buildConstant(S64, 0xffffffffffULL);
3028 auto T = MIRBuilder.buildAnd(S64, U, Mask1);
3029
3030 auto UShl = MIRBuilder.buildLShr(S64, U, MIRBuilder.buildConstant(S64, 40));
3031 auto ShlE = MIRBuilder.buildShl(S32, E, MIRBuilder.buildConstant(S32, 23));
3032 auto V = MIRBuilder.buildOr(S32, ShlE, MIRBuilder.buildTrunc(S32, UShl));
3033
3034 auto C = MIRBuilder.buildConstant(S64, 0x8000000000ULL);
3035 auto RCmp = MIRBuilder.buildICmp(CmpInst::ICMP_UGT, S1, T, C);
3036 auto TCmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1, T, C);
3037 auto One = MIRBuilder.buildConstant(S32, 1);
3038
3039 auto VTrunc1 = MIRBuilder.buildAnd(S32, V, One);
3040 auto Select0 = MIRBuilder.buildSelect(S32, TCmp, VTrunc1, Zero32);
3041 auto R = MIRBuilder.buildSelect(S32, RCmp, One, Select0);
3042 MIRBuilder.buildAdd(Dst, V, R);
3043
3044 return Legalized;
3045}
3046
3047LegalizerHelper::LegalizeResult
3048LegalizerHelper::lowerUITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
3049 unsigned Dst = MI.getOperand(0).getReg();
3050 unsigned Src = MI.getOperand(1).getReg();
3051 LLT DstTy = MRI.getType(Dst);
3052 LLT SrcTy = MRI.getType(Src);
3053
3054 if (SrcTy != LLT::scalar(64))
3055 return UnableToLegalize;
3056
3057 if (DstTy == LLT::scalar(32)) {
3058 // TODO: SelectionDAG has several alternative expansions to port which may
3059 // be more reasonble depending on the available instructions. If a target
3060 // has sitofp, does not have CTLZ, or can efficiently use f64 as an
3061 // intermediate type, this is probably worse.
3062 return lowerU64ToF32BitOps(MI);
3063 }
3064
3065 return UnableToLegalize;
3066}
3067
3068LegalizerHelper::LegalizeResult
3069LegalizerHelper::lowerSITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
3070 unsigned Dst = MI.getOperand(0).getReg();
3071 unsigned Src = MI.getOperand(1).getReg();
3072 LLT DstTy = MRI.getType(Dst);
3073 LLT SrcTy = MRI.getType(Src);
3074
3075 const LLT S64 = LLT::scalar(64);
3076 const LLT S32 = LLT::scalar(32);
3077 const LLT S1 = LLT::scalar(1);
3078
3079 if (SrcTy != S64)
3080 return UnableToLegalize;
3081
3082 if (DstTy == S32) {
3083 // signed cl2f(long l) {
3084 // long s = l >> 63;
3085 // float r = cul2f((l + s) ^ s);
3086 // return s ? -r : r;
3087 // }
3088 unsigned L = Src;
3089 auto SignBit = MIRBuilder.buildConstant(S64, 63);
3090 auto S = MIRBuilder.buildAShr(S64, L, SignBit);
3091
3092 auto LPlusS = MIRBuilder.buildAdd(S64, L, S);
3093 auto Xor = MIRBuilder.buildXor(S64, LPlusS, S);
3094 auto R = MIRBuilder.buildUITOFP(S32, Xor);
3095
3096 auto RNeg = MIRBuilder.buildFNeg(S32, R);
3097 auto SignNotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, S,
3098 MIRBuilder.buildConstant(S64, 0));
3099 MIRBuilder.buildSelect(Dst, SignNotZero, RNeg, R);
3100 return Legalized;
3101 }
3102
3103 return UnableToLegalize;
3104}