Bug Summary

File:llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
Warning:line 882, column 7
1st function call argument is an uninitialized value

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name AArch64InstructionSelector.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -fno-split-dwarf-inlining -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-12/lib/clang/12.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/build-llvm/lib/Target/AArch64 -I /build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64 -I /build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/build-llvm/include -I /build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-12/lib/clang/12.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/build-llvm/lib/Target/AArch64 -fdebug-prefix-map=/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b=. -ferror-limit 19 -fvisibility hidden -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2020-09-17-195756-12974-1 -x c++ /build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp

/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp

1//===- AArch64InstructionSelector.cpp ----------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// AArch64.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "AArch64InstrInfo.h"
15#include "AArch64MachineFunctionInfo.h"
16#include "AArch64RegisterBankInfo.h"
17#include "AArch64RegisterInfo.h"
18#include "AArch64Subtarget.h"
19#include "AArch64TargetMachine.h"
20#include "MCTargetDesc/AArch64AddressingModes.h"
21#include "llvm/ADT/Optional.h"
22#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
23#include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h"
24#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
25#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
26#include "llvm/CodeGen/GlobalISel/Utils.h"
27#include "llvm/CodeGen/MachineBasicBlock.h"
28#include "llvm/CodeGen/MachineConstantPool.h"
29#include "llvm/CodeGen/MachineFunction.h"
30#include "llvm/CodeGen/MachineInstr.h"
31#include "llvm/CodeGen/MachineInstrBuilder.h"
32#include "llvm/CodeGen/MachineOperand.h"
33#include "llvm/CodeGen/MachineRegisterInfo.h"
34#include "llvm/CodeGen/TargetOpcodes.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/Type.h"
37#include "llvm/IR/IntrinsicsAArch64.h"
38#include "llvm/Pass.h"
39#include "llvm/Support/Debug.h"
40#include "llvm/Support/raw_ostream.h"
41
42#define DEBUG_TYPE"aarch64-isel" "aarch64-isel"
43
44using namespace llvm;
45using namespace MIPatternMatch;
46
47namespace {
48
49#define GET_GLOBALISEL_PREDICATE_BITSET
50#include "AArch64GenGlobalISel.inc"
51#undef GET_GLOBALISEL_PREDICATE_BITSET
52
53class AArch64InstructionSelector : public InstructionSelector {
54public:
55 AArch64InstructionSelector(const AArch64TargetMachine &TM,
56 const AArch64Subtarget &STI,
57 const AArch64RegisterBankInfo &RBI);
58
59 bool select(MachineInstr &I) override;
60 static const char *getName() { return DEBUG_TYPE"aarch64-isel"; }
61
62 void setupMF(MachineFunction &MF, GISelKnownBits &KB,
63 CodeGenCoverage &CoverageInfo) override {
64 InstructionSelector::setupMF(MF, KB, CoverageInfo);
65
66 // hasFnAttribute() is expensive to call on every BRCOND selection, so
67 // cache it here for each run of the selector.
68 ProduceNonFlagSettingCondBr =
69 !MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening);
70 MFReturnAddr = Register();
71
72 processPHIs(MF);
73 }
74
75private:
76 /// tblgen-erated 'select' implementation, used as the initial selector for
77 /// the patterns that don't require complex C++.
78 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
79
80 // A lowering phase that runs before any selection attempts.
81 // Returns true if the instruction was modified.
82 bool preISelLower(MachineInstr &I);
83
84 // An early selection function that runs before the selectImpl() call.
85 bool earlySelect(MachineInstr &I) const;
86
87 // Do some preprocessing of G_PHIs before we begin selection.
88 void processPHIs(MachineFunction &MF);
89
90 bool earlySelectSHL(MachineInstr &I, MachineRegisterInfo &MRI) const;
91
92 /// Eliminate same-sized cross-bank copies into stores before selectImpl().
93 bool contractCrossBankCopyIntoStore(MachineInstr &I,
94 MachineRegisterInfo &MRI);
95
96 bool convertPtrAddToAdd(MachineInstr &I, MachineRegisterInfo &MRI);
97
98 bool selectVaStartAAPCS(MachineInstr &I, MachineFunction &MF,
99 MachineRegisterInfo &MRI) const;
100 bool selectVaStartDarwin(MachineInstr &I, MachineFunction &MF,
101 MachineRegisterInfo &MRI) const;
102
103 bool tryOptAndIntoCompareBranch(MachineInstr *LHS,
104 int64_t CmpConstant,
105 const CmpInst::Predicate &Pred,
106 MachineBasicBlock *DstMBB,
107 MachineIRBuilder &MIB) const;
108 bool selectCompareBranch(MachineInstr &I, MachineFunction &MF,
109 MachineRegisterInfo &MRI) const;
110
111 bool selectVectorASHR(MachineInstr &I, MachineRegisterInfo &MRI) const;
112 bool selectVectorSHL(MachineInstr &I, MachineRegisterInfo &MRI) const;
113
114 // Helper to generate an equivalent of scalar_to_vector into a new register,
115 // returned via 'Dst'.
116 MachineInstr *emitScalarToVector(unsigned EltSize,
117 const TargetRegisterClass *DstRC,
118 Register Scalar,
119 MachineIRBuilder &MIRBuilder) const;
120
121 /// Emit a lane insert into \p DstReg, or a new vector register if None is
122 /// provided.
123 ///
124 /// The lane inserted into is defined by \p LaneIdx. The vector source
125 /// register is given by \p SrcReg. The register containing the element is
126 /// given by \p EltReg.
127 MachineInstr *emitLaneInsert(Optional<Register> DstReg, Register SrcReg,
128 Register EltReg, unsigned LaneIdx,
129 const RegisterBank &RB,
130 MachineIRBuilder &MIRBuilder) const;
131 bool selectInsertElt(MachineInstr &I, MachineRegisterInfo &MRI) const;
132 bool tryOptConstantBuildVec(MachineInstr &MI, LLT DstTy,
133 MachineRegisterInfo &MRI) const;
134 bool selectBuildVector(MachineInstr &I, MachineRegisterInfo &MRI) const;
135 bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI) const;
136 bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI) const;
137
138 bool tryOptShuffleDupLane(MachineInstr &I, LLT DstTy, LLT SrcTy,
139 ArrayRef<int> Mask, MachineRegisterInfo &MRI) const;
140 bool selectShuffleVector(MachineInstr &I, MachineRegisterInfo &MRI) const;
141 bool selectExtractElt(MachineInstr &I, MachineRegisterInfo &MRI) const;
142 bool selectConcatVectors(MachineInstr &I, MachineRegisterInfo &MRI) const;
143 bool selectSplitVectorUnmerge(MachineInstr &I,
144 MachineRegisterInfo &MRI) const;
145 bool selectIntrinsicWithSideEffects(MachineInstr &I,
146 MachineRegisterInfo &MRI) const;
147 bool selectIntrinsic(MachineInstr &I, MachineRegisterInfo &MRI);
148 bool selectVectorICmp(MachineInstr &I, MachineRegisterInfo &MRI) const;
149 bool selectIntrinsicTrunc(MachineInstr &I, MachineRegisterInfo &MRI) const;
150 bool selectIntrinsicRound(MachineInstr &I, MachineRegisterInfo &MRI) const;
151 bool selectJumpTable(MachineInstr &I, MachineRegisterInfo &MRI) const;
152 bool selectBrJT(MachineInstr &I, MachineRegisterInfo &MRI) const;
153 bool selectTLSGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI) const;
154
155 unsigned emitConstantPoolEntry(const Constant *CPVal,
156 MachineFunction &MF) const;
157 MachineInstr *emitLoadFromConstantPool(const Constant *CPVal,
158 MachineIRBuilder &MIRBuilder) const;
159
160 // Emit a vector concat operation.
161 MachineInstr *emitVectorConcat(Optional<Register> Dst, Register Op1,
162 Register Op2,
163 MachineIRBuilder &MIRBuilder) const;
164
165 // Emit an integer compare between LHS and RHS, which checks for Predicate.
166 //
167 // This returns the produced compare instruction, and the predicate which
168 // was ultimately used in the compare. The predicate may differ from what
169 // is passed in \p Predicate due to optimization.
170 std::pair<MachineInstr *, CmpInst::Predicate>
171 emitIntegerCompare(MachineOperand &LHS, MachineOperand &RHS,
172 MachineOperand &Predicate,
173 MachineIRBuilder &MIRBuilder) const;
174 MachineInstr *emitInstr(unsigned Opcode,
175 std::initializer_list<llvm::DstOp> DstOps,
176 std::initializer_list<llvm::SrcOp> SrcOps,
177 MachineIRBuilder &MIRBuilder,
178 const ComplexRendererFns &RenderFns = None) const;
179 /// Helper function to emit a binary operation such as an ADD, ADDS, etc.
180 ///
181 /// This is intended for instructions with the following opcode variants:
182 ///
183 /// - Xri, Wri (arithmetic immediate form)
184 /// - Xrs, Wrs (shifted register form)
185 /// - Xrr, Wrr (register form)
186 ///
187 /// For example, for ADD, we have ADDXri, ADDWri, ADDXrs, etc.
188 ///
189 /// \p AddrModeAndSizeToOpcode must contain each of the opcode variants above
190 /// in a specific order.
191 ///
192 /// Below is an example of the expected input to \p AddrModeAndSizeToOpcode.
193 ///
194 /// \code
195 /// const std::array<std::array<unsigned, 2>, 3> Table {
196 /// {{AArch64::ADDXri, AArch64::ADDWri},
197 /// {AArch64::ADDXrs, AArch64::ADDWrs},
198 /// {AArch64::ADDXrr, AArch64::ADDWrr}}};
199 /// \endcode
200 ///
201 /// Each row in the table corresponds to a different addressing mode. Each
202 /// column corresponds to a different register size.
203 ///
204 /// \attention Rows must be structured as follows:
205 /// - Row 0: The ri opcode variants
206 /// - Row 1: The rs opcode variants
207 /// - Row 2: The rr opcode variants
208 ///
209 /// \attention Columns must be structured as follows:
210 /// - Column 0: The 64-bit opcode variants
211 /// - Column 1: The 32-bit opcode variants
212 ///
213 /// \p Dst is the destination register of the binop to emit.
214 /// \p LHS is the left-hand operand of the binop to emit.
215 /// \p RHS is the right-hand operand of the binop to emit.
216 MachineInstr *emitBinOp(
217 const std::array<std::array<unsigned, 2>, 3> &AddrModeAndSizeToOpcode,
218 Register Dst, MachineOperand &LHS, MachineOperand &RHS,
219 MachineIRBuilder &MIRBuilder) const;
220 MachineInstr *emitADD(Register DefReg, MachineOperand &LHS,
221 MachineOperand &RHS,
222 MachineIRBuilder &MIRBuilder) const;
223 MachineInstr *emitADDS(Register Dst, MachineOperand &LHS, MachineOperand &RHS,
224 MachineIRBuilder &MIRBuilder) const;
225 MachineInstr *emitCMN(MachineOperand &LHS, MachineOperand &RHS,
226 MachineIRBuilder &MIRBuilder) const;
227 MachineInstr *emitTST(const Register &LHS, const Register &RHS,
228 MachineIRBuilder &MIRBuilder) const;
229 MachineInstr *emitExtractVectorElt(Optional<Register> DstReg,
230 const RegisterBank &DstRB, LLT ScalarTy,
231 Register VecReg, unsigned LaneIdx,
232 MachineIRBuilder &MIRBuilder) const;
233
234 /// Helper function for selecting G_FCONSTANT. If the G_FCONSTANT can be
235 /// materialized using a FMOV instruction, then update MI and return it.
236 /// Otherwise, do nothing and return a nullptr.
237 MachineInstr *emitFMovForFConstant(MachineInstr &MI,
238 MachineRegisterInfo &MRI) const;
239
240 /// Emit a CSet for a compare.
241 MachineInstr *emitCSetForICMP(Register DefReg, unsigned Pred,
242 MachineIRBuilder &MIRBuilder) const;
243
244 /// Emit a TB(N)Z instruction which tests \p Bit in \p TestReg.
245 /// \p IsNegative is true if the test should be "not zero".
246 /// This will also optimize the test bit instruction when possible.
247 MachineInstr *emitTestBit(Register TestReg, uint64_t Bit, bool IsNegative,
248 MachineBasicBlock *DstMBB,
249 MachineIRBuilder &MIB) const;
250
251 // Equivalent to the i32shift_a and friends from AArch64InstrInfo.td.
252 // We use these manually instead of using the importer since it doesn't
253 // support SDNodeXForm.
254 ComplexRendererFns selectShiftA_32(const MachineOperand &Root) const;
255 ComplexRendererFns selectShiftB_32(const MachineOperand &Root) const;
256 ComplexRendererFns selectShiftA_64(const MachineOperand &Root) const;
257 ComplexRendererFns selectShiftB_64(const MachineOperand &Root) const;
258
259 ComplexRendererFns select12BitValueWithLeftShift(uint64_t Immed) const;
260 ComplexRendererFns selectArithImmed(MachineOperand &Root) const;
261 ComplexRendererFns selectNegArithImmed(MachineOperand &Root) const;
262
263 ComplexRendererFns selectAddrModeUnscaled(MachineOperand &Root,
264 unsigned Size) const;
265
266 ComplexRendererFns selectAddrModeUnscaled8(MachineOperand &Root) const {
267 return selectAddrModeUnscaled(Root, 1);
268 }
269 ComplexRendererFns selectAddrModeUnscaled16(MachineOperand &Root) const {
270 return selectAddrModeUnscaled(Root, 2);
271 }
272 ComplexRendererFns selectAddrModeUnscaled32(MachineOperand &Root) const {
273 return selectAddrModeUnscaled(Root, 4);
274 }
275 ComplexRendererFns selectAddrModeUnscaled64(MachineOperand &Root) const {
276 return selectAddrModeUnscaled(Root, 8);
277 }
278 ComplexRendererFns selectAddrModeUnscaled128(MachineOperand &Root) const {
279 return selectAddrModeUnscaled(Root, 16);
280 }
281
282 /// Helper to try to fold in a GISEL_ADD_LOW into an immediate, to be used
283 /// from complex pattern matchers like selectAddrModeIndexed().
284 ComplexRendererFns tryFoldAddLowIntoImm(MachineInstr &RootDef, unsigned Size,
285 MachineRegisterInfo &MRI) const;
286
287 ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root,
288 unsigned Size) const;
289 template <int Width>
290 ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root) const {
291 return selectAddrModeIndexed(Root, Width / 8);
292 }
293
294 bool isWorthFoldingIntoExtendedReg(MachineInstr &MI,
295 const MachineRegisterInfo &MRI) const;
296 ComplexRendererFns
297 selectAddrModeShiftedExtendXReg(MachineOperand &Root,
298 unsigned SizeInBytes) const;
299
300 /// Returns a \p ComplexRendererFns which contains a base, offset, and whether
301 /// or not a shift + extend should be folded into an addressing mode. Returns
302 /// None when this is not profitable or possible.
303 ComplexRendererFns
304 selectExtendedSHL(MachineOperand &Root, MachineOperand &Base,
305 MachineOperand &Offset, unsigned SizeInBytes,
306 bool WantsExt) const;
307 ComplexRendererFns selectAddrModeRegisterOffset(MachineOperand &Root) const;
308 ComplexRendererFns selectAddrModeXRO(MachineOperand &Root,
309 unsigned SizeInBytes) const;
310 template <int Width>
311 ComplexRendererFns selectAddrModeXRO(MachineOperand &Root) const {
312 return selectAddrModeXRO(Root, Width / 8);
313 }
314
315 ComplexRendererFns selectAddrModeWRO(MachineOperand &Root,
316 unsigned SizeInBytes) const;
317 template <int Width>
318 ComplexRendererFns selectAddrModeWRO(MachineOperand &Root) const {
319 return selectAddrModeWRO(Root, Width / 8);
320 }
321
322 ComplexRendererFns selectShiftedRegister(MachineOperand &Root) const;
323
324 ComplexRendererFns selectArithShiftedRegister(MachineOperand &Root) const {
325 return selectShiftedRegister(Root);
326 }
327
328 ComplexRendererFns selectLogicalShiftedRegister(MachineOperand &Root) const {
329 // TODO: selectShiftedRegister should allow for rotates on logical shifts.
330 // For now, make them the same. The only difference between the two is that
331 // logical shifts are allowed to fold in rotates. Otherwise, these are
332 // functionally the same.
333 return selectShiftedRegister(Root);
334 }
335
336 /// Given an extend instruction, determine the correct shift-extend type for
337 /// that instruction.
338 ///
339 /// If the instruction is going to be used in a load or store, pass
340 /// \p IsLoadStore = true.
341 AArch64_AM::ShiftExtendType
342 getExtendTypeForInst(MachineInstr &MI, MachineRegisterInfo &MRI,
343 bool IsLoadStore = false) const;
344
345 /// Instructions that accept extend modifiers like UXTW expect the register
346 /// being extended to be a GPR32. Narrow ExtReg to a 32-bit register using a
347 /// subregister copy if necessary. Return either ExtReg, or the result of the
348 /// new copy.
349 Register narrowExtendRegIfNeeded(Register ExtReg,
350 MachineIRBuilder &MIB) const;
351 Register widenGPRBankRegIfNeeded(Register Reg, unsigned Size,
352 MachineIRBuilder &MIB) const;
353 ComplexRendererFns selectArithExtendedRegister(MachineOperand &Root) const;
354
355 void renderTruncImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
356 int OpIdx = -1) const;
357 void renderLogicalImm32(MachineInstrBuilder &MIB, const MachineInstr &I,
358 int OpIdx = -1) const;
359 void renderLogicalImm64(MachineInstrBuilder &MIB, const MachineInstr &I,
360 int OpIdx = -1) const;
361
362 // Materialize a GlobalValue or BlockAddress using a movz+movk sequence.
363 void materializeLargeCMVal(MachineInstr &I, const Value *V,
364 unsigned OpFlags) const;
365
366 // Optimization methods.
367 bool tryOptSelect(MachineInstr &MI) const;
368 MachineInstr *tryFoldIntegerCompare(MachineOperand &LHS, MachineOperand &RHS,
369 MachineOperand &Predicate,
370 MachineIRBuilder &MIRBuilder) const;
371 MachineInstr *tryOptArithImmedIntegerCompare(MachineOperand &LHS,
372 MachineOperand &RHS,
373 CmpInst::Predicate &Predicate,
374 MachineIRBuilder &MIB) const;
375 MachineInstr *tryOptArithShiftedCompare(MachineOperand &LHS,
376 MachineOperand &RHS,
377 MachineIRBuilder &MIB) const;
378
379 /// Return true if \p MI is a load or store of \p NumBytes bytes.
380 bool isLoadStoreOfNumBytes(const MachineInstr &MI, unsigned NumBytes) const;
381
382 /// Returns true if \p MI is guaranteed to have the high-half of a 64-bit
383 /// register zeroed out. In other words, the result of MI has been explicitly
384 /// zero extended.
385 bool isDef32(const MachineInstr &MI) const;
386
387 const AArch64TargetMachine &TM;
388 const AArch64Subtarget &STI;
389 const AArch64InstrInfo &TII;
390 const AArch64RegisterInfo &TRI;
391 const AArch64RegisterBankInfo &RBI;
392
393 bool ProduceNonFlagSettingCondBr = false;
394
395 // Some cached values used during selection.
396 // We use LR as a live-in register, and we keep track of it here as it can be
397 // clobbered by calls.
398 Register MFReturnAddr;
399
400#define GET_GLOBALISEL_PREDICATES_DECL
401#include "AArch64GenGlobalISel.inc"
402#undef GET_GLOBALISEL_PREDICATES_DECL
403
404// We declare the temporaries used by selectImpl() in the class to minimize the
405// cost of constructing placeholder values.
406#define GET_GLOBALISEL_TEMPORARIES_DECL
407#include "AArch64GenGlobalISel.inc"
408#undef GET_GLOBALISEL_TEMPORARIES_DECL
409};
410
411} // end anonymous namespace
412
413#define GET_GLOBALISEL_IMPL
414#include "AArch64GenGlobalISel.inc"
415#undef GET_GLOBALISEL_IMPL
416
417AArch64InstructionSelector::AArch64InstructionSelector(
418 const AArch64TargetMachine &TM, const AArch64Subtarget &STI,
419 const AArch64RegisterBankInfo &RBI)
420 : InstructionSelector(), TM(TM), STI(STI), TII(*STI.getInstrInfo()),
421 TRI(*STI.getRegisterInfo()), RBI(RBI),
422#define GET_GLOBALISEL_PREDICATES_INIT
423#include "AArch64GenGlobalISel.inc"
424#undef GET_GLOBALISEL_PREDICATES_INIT
425#define GET_GLOBALISEL_TEMPORARIES_INIT
426#include "AArch64GenGlobalISel.inc"
427#undef GET_GLOBALISEL_TEMPORARIES_INIT
428{
429}
430
431// FIXME: This should be target-independent, inferred from the types declared
432// for each class in the bank.
433static const TargetRegisterClass *
434getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB,
435 const RegisterBankInfo &RBI,
436 bool GetAllRegSet = false) {
437 if (RB.getID() == AArch64::GPRRegBankID) {
438 if (Ty.getSizeInBits() <= 32)
439 return GetAllRegSet ? &AArch64::GPR32allRegClass
440 : &AArch64::GPR32RegClass;
441 if (Ty.getSizeInBits() == 64)
442 return GetAllRegSet ? &AArch64::GPR64allRegClass
443 : &AArch64::GPR64RegClass;
444 return nullptr;
445 }
446
447 if (RB.getID() == AArch64::FPRRegBankID) {
448 if (Ty.getSizeInBits() <= 16)
449 return &AArch64::FPR16RegClass;
450 if (Ty.getSizeInBits() == 32)
451 return &AArch64::FPR32RegClass;
452 if (Ty.getSizeInBits() == 64)
453 return &AArch64::FPR64RegClass;
454 if (Ty.getSizeInBits() == 128)
455 return &AArch64::FPR128RegClass;
456 return nullptr;
457 }
458
459 return nullptr;
460}
461
462/// Given a register bank, and size in bits, return the smallest register class
463/// that can represent that combination.
464static const TargetRegisterClass *
465getMinClassForRegBank(const RegisterBank &RB, unsigned SizeInBits,
466 bool GetAllRegSet = false) {
467 unsigned RegBankID = RB.getID();
468
469 if (RegBankID == AArch64::GPRRegBankID) {
28
Assuming 'RegBankID' is not equal to GPRRegBankID
29
Taking false branch
36
Assuming 'RegBankID' is equal to GPRRegBankID
37
Taking true branch
470 if (SizeInBits <= 32)
38
Assuming 'SizeInBits' is <= 32
39
Taking true branch
471 return GetAllRegSet
39.1
'GetAllRegSet' is true
39.1
'GetAllRegSet' is true
39.1
'GetAllRegSet' is true
? &AArch64::GPR32allRegClass
40
'?' condition is true
41
Returning pointer, which participates in a condition later
472 : &AArch64::GPR32RegClass;
473 if (SizeInBits == 64)
474 return GetAllRegSet ? &AArch64::GPR64allRegClass
475 : &AArch64::GPR64RegClass;
476 }
477
478 if (RegBankID == AArch64::FPRRegBankID) {
30
Assuming 'RegBankID' is equal to FPRRegBankID
31
Taking true branch
479 switch (SizeInBits) {
32
Control jumps to 'case 128:' at line 490
480 default:
481 return nullptr;
482 case 8:
483 return &AArch64::FPR8RegClass;
484 case 16:
485 return &AArch64::FPR16RegClass;
486 case 32:
487 return &AArch64::FPR32RegClass;
488 case 64:
489 return &AArch64::FPR64RegClass;
490 case 128:
491 return &AArch64::FPR128RegClass;
33
Returning pointer, which participates in a condition later
492 }
493 }
494
495 return nullptr;
496}
497
498/// Returns the correct subregister to use for a given register class.
499static bool getSubRegForClass(const TargetRegisterClass *RC,
500 const TargetRegisterInfo &TRI, unsigned &SubReg) {
501 switch (TRI.getRegSizeInBits(*RC)) {
63
Control jumps to the 'default' case at line 517
502 case 8:
503 SubReg = AArch64::bsub;
504 break;
505 case 16:
506 SubReg = AArch64::hsub;
507 break;
508 case 32:
509 if (RC != &AArch64::FPR32RegClass)
510 SubReg = AArch64::sub_32;
511 else
512 SubReg = AArch64::ssub;
513 break;
514 case 64:
515 SubReg = AArch64::dsub;
516 break;
517 default:
518 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't find appropriate subregister for register class."
; } } while (false)
64
Assuming 'DebugFlag' is false
65
Loop condition is false. Exiting loop
519 dbgs() << "Couldn't find appropriate subregister for register class.")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't find appropriate subregister for register class."
; } } while (false)
;
520 return false;
66
Returning without writing to 'SubReg'
521 }
522
523 return true;
524}
525
526/// Returns the minimum size the given register bank can hold.
527static unsigned getMinSizeForRegBank(const RegisterBank &RB) {
528 switch (RB.getID()) {
529 case AArch64::GPRRegBankID:
530 return 32;
531 case AArch64::FPRRegBankID:
532 return 8;
533 default:
534 llvm_unreachable("Tried to get minimum size for unknown register bank.")::llvm::llvm_unreachable_internal("Tried to get minimum size for unknown register bank."
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 534)
;
535 }
536}
537
538static Optional<uint64_t> getImmedFromMO(const MachineOperand &Root) {
539 auto &MI = *Root.getParent();
540 auto &MBB = *MI.getParent();
541 auto &MF = *MBB.getParent();
542 auto &MRI = MF.getRegInfo();
543 uint64_t Immed;
544 if (Root.isImm())
545 Immed = Root.getImm();
546 else if (Root.isCImm())
547 Immed = Root.getCImm()->getZExtValue();
548 else if (Root.isReg()) {
549 auto ValAndVReg =
550 getConstantVRegValWithLookThrough(Root.getReg(), MRI, true);
551 if (!ValAndVReg)
552 return None;
553 Immed = ValAndVReg->Value;
554 } else
555 return None;
556 return Immed;
557}
558
559/// Check whether \p I is a currently unsupported binary operation:
560/// - it has an unsized type
561/// - an operand is not a vreg
562/// - all operands are not in the same bank
563/// These are checks that should someday live in the verifier, but right now,
564/// these are mostly limitations of the aarch64 selector.
565static bool unsupportedBinOp(const MachineInstr &I,
566 const AArch64RegisterBankInfo &RBI,
567 const MachineRegisterInfo &MRI,
568 const AArch64RegisterInfo &TRI) {
569 LLT Ty = MRI.getType(I.getOperand(0).getReg());
570 if (!Ty.isValid()) {
571 LLVM_DEBUG(dbgs() << "Generic binop register should be typed\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic binop register should be typed\n"
; } } while (false)
;
572 return true;
573 }
574
575 const RegisterBank *PrevOpBank = nullptr;
576 for (auto &MO : I.operands()) {
577 // FIXME: Support non-register operands.
578 if (!MO.isReg()) {
579 LLVM_DEBUG(dbgs() << "Generic inst non-reg operands are unsupported\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic inst non-reg operands are unsupported\n"
; } } while (false)
;
580 return true;
581 }
582
583 // FIXME: Can generic operations have physical registers operands? If
584 // so, this will need to be taught about that, and we'll need to get the
585 // bank out of the minimal class for the register.
586 // Either way, this needs to be documented (and possibly verified).
587 if (!Register::isVirtualRegister(MO.getReg())) {
588 LLVM_DEBUG(dbgs() << "Generic inst has physical register operand\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic inst has physical register operand\n"
; } } while (false)
;
589 return true;
590 }
591
592 const RegisterBank *OpBank = RBI.getRegBank(MO.getReg(), MRI, TRI);
593 if (!OpBank) {
594 LLVM_DEBUG(dbgs() << "Generic register has no bank or class\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic register has no bank or class\n"
; } } while (false)
;
595 return true;
596 }
597
598 if (PrevOpBank && OpBank != PrevOpBank) {
599 LLVM_DEBUG(dbgs() << "Generic inst operands have different banks\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic inst operands have different banks\n"
; } } while (false)
;
600 return true;
601 }
602 PrevOpBank = OpBank;
603 }
604 return false;
605}
606
607/// Select the AArch64 opcode for the basic binary operation \p GenericOpc
608/// (such as G_OR or G_SDIV), appropriate for the register bank \p RegBankID
609/// and of size \p OpSize.
610/// \returns \p GenericOpc if the combination is unsupported.
611static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID,
612 unsigned OpSize) {
613 switch (RegBankID) {
614 case AArch64::GPRRegBankID:
615 if (OpSize == 32) {
616 switch (GenericOpc) {
617 case TargetOpcode::G_SHL:
618 return AArch64::LSLVWr;
619 case TargetOpcode::G_LSHR:
620 return AArch64::LSRVWr;
621 case TargetOpcode::G_ASHR:
622 return AArch64::ASRVWr;
623 default:
624 return GenericOpc;
625 }
626 } else if (OpSize == 64) {
627 switch (GenericOpc) {
628 case TargetOpcode::G_PTR_ADD:
629 return AArch64::ADDXrr;
630 case TargetOpcode::G_SHL:
631 return AArch64::LSLVXr;
632 case TargetOpcode::G_LSHR:
633 return AArch64::LSRVXr;
634 case TargetOpcode::G_ASHR:
635 return AArch64::ASRVXr;
636 default:
637 return GenericOpc;
638 }
639 }
640 break;
641 case AArch64::FPRRegBankID:
642 switch (OpSize) {
643 case 32:
644 switch (GenericOpc) {
645 case TargetOpcode::G_FADD:
646 return AArch64::FADDSrr;
647 case TargetOpcode::G_FSUB:
648 return AArch64::FSUBSrr;
649 case TargetOpcode::G_FMUL:
650 return AArch64::FMULSrr;
651 case TargetOpcode::G_FDIV:
652 return AArch64::FDIVSrr;
653 default:
654 return GenericOpc;
655 }
656 case 64:
657 switch (GenericOpc) {
658 case TargetOpcode::G_FADD:
659 return AArch64::FADDDrr;
660 case TargetOpcode::G_FSUB:
661 return AArch64::FSUBDrr;
662 case TargetOpcode::G_FMUL:
663 return AArch64::FMULDrr;
664 case TargetOpcode::G_FDIV:
665 return AArch64::FDIVDrr;
666 case TargetOpcode::G_OR:
667 return AArch64::ORRv8i8;
668 default:
669 return GenericOpc;
670 }
671 }
672 break;
673 }
674 return GenericOpc;
675}
676
677/// Select the AArch64 opcode for the G_LOAD or G_STORE operation \p GenericOpc,
678/// appropriate for the (value) register bank \p RegBankID and of memory access
679/// size \p OpSize. This returns the variant with the base+unsigned-immediate
680/// addressing mode (e.g., LDRXui).
681/// \returns \p GenericOpc if the combination is unsupported.
682static unsigned selectLoadStoreUIOp(unsigned GenericOpc, unsigned RegBankID,
683 unsigned OpSize) {
684 const bool isStore = GenericOpc == TargetOpcode::G_STORE;
685 switch (RegBankID) {
686 case AArch64::GPRRegBankID:
687 switch (OpSize) {
688 case 8:
689 return isStore ? AArch64::STRBBui : AArch64::LDRBBui;
690 case 16:
691 return isStore ? AArch64::STRHHui : AArch64::LDRHHui;
692 case 32:
693 return isStore ? AArch64::STRWui : AArch64::LDRWui;
694 case 64:
695 return isStore ? AArch64::STRXui : AArch64::LDRXui;
696 }
697 break;
698 case AArch64::FPRRegBankID:
699 switch (OpSize) {
700 case 8:
701 return isStore ? AArch64::STRBui : AArch64::LDRBui;
702 case 16:
703 return isStore ? AArch64::STRHui : AArch64::LDRHui;
704 case 32:
705 return isStore ? AArch64::STRSui : AArch64::LDRSui;
706 case 64:
707 return isStore ? AArch64::STRDui : AArch64::LDRDui;
708 }
709 break;
710 }
711 return GenericOpc;
712}
713
714#ifndef NDEBUG
715/// Helper function that verifies that we have a valid copy at the end of
716/// selectCopy. Verifies that the source and dest have the expected sizes and
717/// then returns true.
718static bool isValidCopy(const MachineInstr &I, const RegisterBank &DstBank,
719 const MachineRegisterInfo &MRI,
720 const TargetRegisterInfo &TRI,
721 const RegisterBankInfo &RBI) {
722 const Register DstReg = I.getOperand(0).getReg();
723 const Register SrcReg = I.getOperand(1).getReg();
724 const unsigned DstSize = RBI.getSizeInBits(DstReg, MRI, TRI);
725 const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI);
726
727 // Make sure the size of the source and dest line up.
728 assert((((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg
) && DstSize <= SrcSize) || (((DstSize + 31) / 32 ==
(SrcSize + 31) / 32) && DstSize > SrcSize)) &&
"Copy with different width?!") ? static_cast<void> (0)
: __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 737, __PRETTY_FUNCTION__))
729 (DstSize == SrcSize ||(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg
) && DstSize <= SrcSize) || (((DstSize + 31) / 32 ==
(SrcSize + 31) / 32) && DstSize > SrcSize)) &&
"Copy with different width?!") ? static_cast<void> (0)
: __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 737, __PRETTY_FUNCTION__))
730 // Copies are a mean to setup initial types, the number of(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg
) && DstSize <= SrcSize) || (((DstSize + 31) / 32 ==
(SrcSize + 31) / 32) && DstSize > SrcSize)) &&
"Copy with different width?!") ? static_cast<void> (0)
: __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 737, __PRETTY_FUNCTION__))
731 // bits may not exactly match.(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg
) && DstSize <= SrcSize) || (((DstSize + 31) / 32 ==
(SrcSize + 31) / 32) && DstSize > SrcSize)) &&
"Copy with different width?!") ? static_cast<void> (0)
: __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 737, __PRETTY_FUNCTION__))
732 (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) ||(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg
) && DstSize <= SrcSize) || (((DstSize + 31) / 32 ==
(SrcSize + 31) / 32) && DstSize > SrcSize)) &&
"Copy with different width?!") ? static_cast<void> (0)
: __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 737, __PRETTY_FUNCTION__))
733 // Copies are a mean to copy bits around, as long as we are(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg
) && DstSize <= SrcSize) || (((DstSize + 31) / 32 ==
(SrcSize + 31) / 32) && DstSize > SrcSize)) &&
"Copy with different width?!") ? static_cast<void> (0)
: __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 737, __PRETTY_FUNCTION__))
734 // on the same register class, that's fine. Otherwise, that(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg
) && DstSize <= SrcSize) || (((DstSize + 31) / 32 ==
(SrcSize + 31) / 32) && DstSize > SrcSize)) &&
"Copy with different width?!") ? static_cast<void> (0)
: __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 737, __PRETTY_FUNCTION__))
735 // means we need some SUBREG_TO_REG or AND & co.(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg
) && DstSize <= SrcSize) || (((DstSize + 31) / 32 ==
(SrcSize + 31) / 32) && DstSize > SrcSize)) &&
"Copy with different width?!") ? static_cast<void> (0)
: __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 737, __PRETTY_FUNCTION__))
736 (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) &&(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg
) && DstSize <= SrcSize) || (((DstSize + 31) / 32 ==
(SrcSize + 31) / 32) && DstSize > SrcSize)) &&
"Copy with different width?!") ? static_cast<void> (0)
: __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 737, __PRETTY_FUNCTION__))
737 "Copy with different width?!")(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg
) && DstSize <= SrcSize) || (((DstSize + 31) / 32 ==
(SrcSize + 31) / 32) && DstSize > SrcSize)) &&
"Copy with different width?!") ? static_cast<void> (0)
: __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 737, __PRETTY_FUNCTION__))
;
738
739 // Check the size of the destination.
740 assert((DstSize <= 64 || DstBank.getID() == AArch64::FPRRegBankID) &&(((DstSize <= 64 || DstBank.getID() == AArch64::FPRRegBankID
) && "GPRs cannot get more than 64-bit width values")
? static_cast<void> (0) : __assert_fail ("(DstSize <= 64 || DstBank.getID() == AArch64::FPRRegBankID) && \"GPRs cannot get more than 64-bit width values\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 741, __PRETTY_FUNCTION__))
741 "GPRs cannot get more than 64-bit width values")(((DstSize <= 64 || DstBank.getID() == AArch64::FPRRegBankID
) && "GPRs cannot get more than 64-bit width values")
? static_cast<void> (0) : __assert_fail ("(DstSize <= 64 || DstBank.getID() == AArch64::FPRRegBankID) && \"GPRs cannot get more than 64-bit width values\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 741, __PRETTY_FUNCTION__))
;
742
743 return true;
744}
745#endif
746
747/// Helper function for selectCopy. Inserts a subregister copy from \p SrcReg
748/// to \p *To.
749///
750/// E.g "To = COPY SrcReg:SubReg"
751static bool copySubReg(MachineInstr &I, MachineRegisterInfo &MRI,
752 const RegisterBankInfo &RBI, Register SrcReg,
753 const TargetRegisterClass *To, unsigned SubReg) {
754 assert(SrcReg.isValid() && "Expected a valid source register?")((SrcReg.isValid() && "Expected a valid source register?"
) ? static_cast<void> (0) : __assert_fail ("SrcReg.isValid() && \"Expected a valid source register?\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 754, __PRETTY_FUNCTION__))
;
755 assert(To && "Destination register class cannot be null")((To && "Destination register class cannot be null") ?
static_cast<void> (0) : __assert_fail ("To && \"Destination register class cannot be null\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 755, __PRETTY_FUNCTION__))
;
756 assert(SubReg && "Expected a valid subregister")((SubReg && "Expected a valid subregister") ? static_cast
<void> (0) : __assert_fail ("SubReg && \"Expected a valid subregister\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 756, __PRETTY_FUNCTION__))
;
757
758 MachineIRBuilder MIB(I);
759 auto SubRegCopy =
760 MIB.buildInstr(TargetOpcode::COPY, {To}, {}).addReg(SrcReg, 0, SubReg);
761 MachineOperand &RegOp = I.getOperand(1);
762 RegOp.setReg(SubRegCopy.getReg(0));
763
764 // It's possible that the destination register won't be constrained. Make
765 // sure that happens.
766 if (!Register::isPhysicalRegister(I.getOperand(0).getReg()))
767 RBI.constrainGenericRegister(I.getOperand(0).getReg(), *To, MRI);
768
769 return true;
770}
771
772/// Helper function to get the source and destination register classes for a
773/// copy. Returns a std::pair containing the source register class for the
774/// copy, and the destination register class for the copy. If a register class
775/// cannot be determined, then it will be nullptr.
776static std::pair<const TargetRegisterClass *, const TargetRegisterClass *>
777getRegClassesForCopy(MachineInstr &I, const TargetInstrInfo &TII,
778 MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
779 const RegisterBankInfo &RBI) {
780 Register DstReg = I.getOperand(0).getReg();
781 Register SrcReg = I.getOperand(1).getReg();
782 const RegisterBank &DstRegBank = *RBI.getRegBank(DstReg, MRI, TRI);
783 const RegisterBank &SrcRegBank = *RBI.getRegBank(SrcReg, MRI, TRI);
784 unsigned DstSize = RBI.getSizeInBits(DstReg, MRI, TRI);
785 unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI);
786
787 // Special casing for cross-bank copies of s1s. We can technically represent
788 // a 1-bit value with any size of register. The minimum size for a GPR is 32
789 // bits. So, we need to put the FPR on 32 bits as well.
790 //
791 // FIXME: I'm not sure if this case holds true outside of copies. If it does,
792 // then we can pull it into the helpers that get the appropriate class for a
793 // register bank. Or make a new helper that carries along some constraint
794 // information.
795 if (SrcRegBank != DstRegBank && (DstSize == 1 && SrcSize == 1))
796 SrcSize = DstSize = 32;
797
798 return {getMinClassForRegBank(SrcRegBank, SrcSize, true),
27
Calling 'getMinClassForRegBank'
34
Returning from 'getMinClassForRegBank'
799 getMinClassForRegBank(DstRegBank, DstSize, true)};
35
Calling 'getMinClassForRegBank'
42
Returning from 'getMinClassForRegBank'
800}
801
802static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII,
803 MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
804 const RegisterBankInfo &RBI) {
805 Register DstReg = I.getOperand(0).getReg();
806 Register SrcReg = I.getOperand(1).getReg();
807 const RegisterBank &DstRegBank = *RBI.getRegBank(DstReg, MRI, TRI);
808 const RegisterBank &SrcRegBank = *RBI.getRegBank(SrcReg, MRI, TRI);
809
810 // Find the correct register classes for the source and destination registers.
811 const TargetRegisterClass *SrcRC;
812 const TargetRegisterClass *DstRC;
813 std::tie(SrcRC, DstRC) = getRegClassesForCopy(I, TII, MRI, TRI, RBI);
14
Calling 'tie<const llvm::TargetRegisterClass *, const llvm::TargetRegisterClass *>'
25
Returning from 'tie<const llvm::TargetRegisterClass *, const llvm::TargetRegisterClass *>'
26
Calling 'getRegClassesForCopy'
43
Returning from 'getRegClassesForCopy'
44
Calling 'tuple::operator='
47
Returning from 'tuple::operator='
814
815 if (!DstRC
47.1
'DstRC' is non-null
47.1
'DstRC' is non-null
47.1
'DstRC' is non-null
) {
48
Taking false branch
816 LLVM_DEBUG(dbgs() << "Unexpected dest size "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unexpected dest size " <<
RBI.getSizeInBits(DstReg, MRI, TRI) << '\n'; } } while
(false)
817 << RBI.getSizeInBits(DstReg, MRI, TRI) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unexpected dest size " <<
RBI.getSizeInBits(DstReg, MRI, TRI) << '\n'; } } while
(false)
;
818 return false;
819 }
820
821 // A couple helpers below, for making sure that the copy we produce is valid.
822
823 // Set to true if we insert a SUBREG_TO_REG. If we do this, then we don't want
824 // to verify that the src and dst are the same size, since that's handled by
825 // the SUBREG_TO_REG.
826 bool KnownValid = false;
827
828 // Returns true, or asserts if something we don't expect happens. Instead of
829 // returning true, we return isValidCopy() to ensure that we verify the
830 // result.
831 auto CheckCopy = [&]() {
832 // If we have a bitcast or something, we can't have physical registers.
833 assert((I.isCopy() ||(((I.isCopy() || (!Register::isPhysicalRegister(I.getOperand(
0).getReg()) && !Register::isPhysicalRegister(I.getOperand
(1).getReg()))) && "No phys reg on generic operator!"
) ? static_cast<void> (0) : __assert_fail ("(I.isCopy() || (!Register::isPhysicalRegister(I.getOperand(0).getReg()) && !Register::isPhysicalRegister(I.getOperand(1).getReg()))) && \"No phys reg on generic operator!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 836, __PRETTY_FUNCTION__))
834 (!Register::isPhysicalRegister(I.getOperand(0).getReg()) &&(((I.isCopy() || (!Register::isPhysicalRegister(I.getOperand(
0).getReg()) && !Register::isPhysicalRegister(I.getOperand
(1).getReg()))) && "No phys reg on generic operator!"
) ? static_cast<void> (0) : __assert_fail ("(I.isCopy() || (!Register::isPhysicalRegister(I.getOperand(0).getReg()) && !Register::isPhysicalRegister(I.getOperand(1).getReg()))) && \"No phys reg on generic operator!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 836, __PRETTY_FUNCTION__))
835 !Register::isPhysicalRegister(I.getOperand(1).getReg()))) &&(((I.isCopy() || (!Register::isPhysicalRegister(I.getOperand(
0).getReg()) && !Register::isPhysicalRegister(I.getOperand
(1).getReg()))) && "No phys reg on generic operator!"
) ? static_cast<void> (0) : __assert_fail ("(I.isCopy() || (!Register::isPhysicalRegister(I.getOperand(0).getReg()) && !Register::isPhysicalRegister(I.getOperand(1).getReg()))) && \"No phys reg on generic operator!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 836, __PRETTY_FUNCTION__))
836 "No phys reg on generic operator!")(((I.isCopy() || (!Register::isPhysicalRegister(I.getOperand(
0).getReg()) && !Register::isPhysicalRegister(I.getOperand
(1).getReg()))) && "No phys reg on generic operator!"
) ? static_cast<void> (0) : __assert_fail ("(I.isCopy() || (!Register::isPhysicalRegister(I.getOperand(0).getReg()) && !Register::isPhysicalRegister(I.getOperand(1).getReg()))) && \"No phys reg on generic operator!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 836, __PRETTY_FUNCTION__))
;
837 bool ValidCopy = true;
838#ifndef NDEBUG
839 ValidCopy = KnownValid || isValidCopy(I, DstRegBank, MRI, TRI, RBI);
840 assert(ValidCopy && "Invalid copy.")((ValidCopy && "Invalid copy.") ? static_cast<void
> (0) : __assert_fail ("ValidCopy && \"Invalid copy.\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 840, __PRETTY_FUNCTION__))
;
841#endif
842 return ValidCopy;
843 };
844
845 // Is this a copy? If so, then we may need to insert a subregister copy.
846 if (I.isCopy()) {
49
Calling 'MachineInstr::isCopy'
52
Returning from 'MachineInstr::isCopy'
53
Taking true branch
847 // Yes. Check if there's anything to fix up.
848 if (!SrcRC
53.1
'SrcRC' is non-null
53.1
'SrcRC' is non-null
53.1
'SrcRC' is non-null
) {
54
Taking false branch
849 LLVM_DEBUG(dbgs() << "Couldn't determine source register class\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't determine source register class\n"
; } } while (false)
;
850 return false;
851 }
852
853 unsigned SrcSize = TRI.getRegSizeInBits(*SrcRC);
854 unsigned DstSize = TRI.getRegSizeInBits(*DstRC);
855 unsigned SubReg;
55
'SubReg' declared without an initial value
856
857 // If the source bank doesn't support a subregister copy small enough,
858 // then we first need to copy to the destination bank.
859 if (getMinSizeForRegBank(SrcRegBank) > DstSize) {
56
Assuming the condition is false
57
Taking false branch
860 const TargetRegisterClass *DstTempRC =
861 getMinClassForRegBank(DstRegBank, SrcSize, /* GetAllRegSet */ true);
862 getSubRegForClass(DstRC, TRI, SubReg);
863
864 MachineIRBuilder MIB(I);
865 auto Copy = MIB.buildCopy({DstTempRC}, {SrcReg});
866 copySubReg(I, MRI, RBI, Copy.getReg(0), DstRC, SubReg);
867 } else if (SrcSize > DstSize) {
58
Assuming 'SrcSize' is <= 'DstSize'
59
Taking false branch
868 // If the source register is bigger than the destination we need to
869 // perform a subregister copy.
870 const TargetRegisterClass *SubRegRC =
871 getMinClassForRegBank(SrcRegBank, DstSize, /* GetAllRegSet */ true);
872 getSubRegForClass(SubRegRC, TRI, SubReg);
873 copySubReg(I, MRI, RBI, SrcReg, DstRC, SubReg);
874 } else if (DstSize > SrcSize) {
60
Assuming 'DstSize' is > 'SrcSize'
61
Taking true branch
875 // If the destination register is bigger than the source we need to do
876 // a promotion using SUBREG_TO_REG.
877 const TargetRegisterClass *PromotionRC =
878 getMinClassForRegBank(SrcRegBank, DstSize, /* GetAllRegSet */ true);
879 getSubRegForClass(SrcRC, TRI, SubReg);
62
Calling 'getSubRegForClass'
67
Returning from 'getSubRegForClass'
880
881 Register PromoteReg = MRI.createVirtualRegister(PromotionRC);
882 BuildMI(*I.getParent(), I, I.getDebugLoc(),
68
1st function call argument is an uninitialized value
883 TII.get(AArch64::SUBREG_TO_REG), PromoteReg)
884 .addImm(0)
885 .addUse(SrcReg)
886 .addImm(SubReg);
887 MachineOperand &RegOp = I.getOperand(1);
888 RegOp.setReg(PromoteReg);
889
890 // Promise that the copy is implicitly validated by the SUBREG_TO_REG.
891 KnownValid = true;
892 }
893
894 // If the destination is a physical register, then there's nothing to
895 // change, so we're done.
896 if (Register::isPhysicalRegister(DstReg))
897 return CheckCopy();
898 }
899
900 // No need to constrain SrcReg. It will get constrained when we hit another
901 // of its use or its defs. Copies do not have constraints.
902 if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
903 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain " <<
TII.getName(I.getOpcode()) << " operand\n"; } } while (
false)
904 << " operand\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain " <<
TII.getName(I.getOpcode()) << " operand\n"; } } while (
false)
;
905 return false;
906 }
907 I.setDesc(TII.get(AArch64::COPY));
908 return CheckCopy();
909}
910
911static unsigned selectFPConvOpc(unsigned GenericOpc, LLT DstTy, LLT SrcTy) {
912 if (!DstTy.isScalar() || !SrcTy.isScalar())
913 return GenericOpc;
914
915 const unsigned DstSize = DstTy.getSizeInBits();
916 const unsigned SrcSize = SrcTy.getSizeInBits();
917
918 switch (DstSize) {
919 case 32:
920 switch (SrcSize) {
921 case 32:
922 switch (GenericOpc) {
923 case TargetOpcode::G_SITOFP:
924 return AArch64::SCVTFUWSri;
925 case TargetOpcode::G_UITOFP:
926 return AArch64::UCVTFUWSri;
927 case TargetOpcode::G_FPTOSI:
928 return AArch64::FCVTZSUWSr;
929 case TargetOpcode::G_FPTOUI:
930 return AArch64::FCVTZUUWSr;
931 default:
932 return GenericOpc;
933 }
934 case 64:
935 switch (GenericOpc) {
936 case TargetOpcode::G_SITOFP:
937 return AArch64::SCVTFUXSri;
938 case TargetOpcode::G_UITOFP:
939 return AArch64::UCVTFUXSri;
940 case TargetOpcode::G_FPTOSI:
941 return AArch64::FCVTZSUWDr;
942 case TargetOpcode::G_FPTOUI:
943 return AArch64::FCVTZUUWDr;
944 default:
945 return GenericOpc;
946 }
947 default:
948 return GenericOpc;
949 }
950 case 64:
951 switch (SrcSize) {
952 case 32:
953 switch (GenericOpc) {
954 case TargetOpcode::G_SITOFP:
955 return AArch64::SCVTFUWDri;
956 case TargetOpcode::G_UITOFP:
957 return AArch64::UCVTFUWDri;
958 case TargetOpcode::G_FPTOSI:
959 return AArch64::FCVTZSUXSr;
960 case TargetOpcode::G_FPTOUI:
961 return AArch64::FCVTZUUXSr;
962 default:
963 return GenericOpc;
964 }
965 case 64:
966 switch (GenericOpc) {
967 case TargetOpcode::G_SITOFP:
968 return AArch64::SCVTFUXDri;
969 case TargetOpcode::G_UITOFP:
970 return AArch64::UCVTFUXDri;
971 case TargetOpcode::G_FPTOSI:
972 return AArch64::FCVTZSUXDr;
973 case TargetOpcode::G_FPTOUI:
974 return AArch64::FCVTZUUXDr;
975 default:
976 return GenericOpc;
977 }
978 default:
979 return GenericOpc;
980 }
981 default:
982 return GenericOpc;
983 };
984 return GenericOpc;
985}
986
987static unsigned selectSelectOpc(MachineInstr &I, MachineRegisterInfo &MRI,
988 const RegisterBankInfo &RBI) {
989 const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo();
990 bool IsFP = (RBI.getRegBank(I.getOperand(0).getReg(), MRI, TRI)->getID() !=
991 AArch64::GPRRegBankID);
992 LLT Ty = MRI.getType(I.getOperand(0).getReg());
993 if (Ty == LLT::scalar(32))
994 return IsFP ? AArch64::FCSELSrrr : AArch64::CSELWr;
995 else if (Ty == LLT::scalar(64) || Ty == LLT::pointer(0, 64))
996 return IsFP ? AArch64::FCSELDrrr : AArch64::CSELXr;
997 return 0;
998}
999
1000/// Helper function to select the opcode for a G_FCMP.
1001static unsigned selectFCMPOpc(MachineInstr &I, MachineRegisterInfo &MRI) {
1002 // If this is a compare against +0.0, then we don't have to explicitly
1003 // materialize a constant.
1004 const ConstantFP *FPImm = getConstantFPVRegVal(I.getOperand(3).getReg(), MRI);
1005 bool ShouldUseImm = FPImm && (FPImm->isZero() && !FPImm->isNegative());
1006 unsigned OpSize = MRI.getType(I.getOperand(2).getReg()).getSizeInBits();
1007 if (OpSize != 32 && OpSize != 64)
1008 return 0;
1009 unsigned CmpOpcTbl[2][2] = {{AArch64::FCMPSrr, AArch64::FCMPDrr},
1010 {AArch64::FCMPSri, AArch64::FCMPDri}};
1011 return CmpOpcTbl[ShouldUseImm][OpSize == 64];
1012}
1013
1014/// Returns true if \p P is an unsigned integer comparison predicate.
1015static bool isUnsignedICMPPred(const CmpInst::Predicate P) {
1016 switch (P) {
1017 default:
1018 return false;
1019 case CmpInst::ICMP_UGT:
1020 case CmpInst::ICMP_UGE:
1021 case CmpInst::ICMP_ULT:
1022 case CmpInst::ICMP_ULE:
1023 return true;
1024 }
1025}
1026
1027static AArch64CC::CondCode changeICMPPredToAArch64CC(CmpInst::Predicate P) {
1028 switch (P) {
1029 default:
1030 llvm_unreachable("Unknown condition code!")::llvm::llvm_unreachable_internal("Unknown condition code!", "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1030)
;
1031 case CmpInst::ICMP_NE:
1032 return AArch64CC::NE;
1033 case CmpInst::ICMP_EQ:
1034 return AArch64CC::EQ;
1035 case CmpInst::ICMP_SGT:
1036 return AArch64CC::GT;
1037 case CmpInst::ICMP_SGE:
1038 return AArch64CC::GE;
1039 case CmpInst::ICMP_SLT:
1040 return AArch64CC::LT;
1041 case CmpInst::ICMP_SLE:
1042 return AArch64CC::LE;
1043 case CmpInst::ICMP_UGT:
1044 return AArch64CC::HI;
1045 case CmpInst::ICMP_UGE:
1046 return AArch64CC::HS;
1047 case CmpInst::ICMP_ULT:
1048 return AArch64CC::LO;
1049 case CmpInst::ICMP_ULE:
1050 return AArch64CC::LS;
1051 }
1052}
1053
1054static void changeFCMPPredToAArch64CC(CmpInst::Predicate P,
1055 AArch64CC::CondCode &CondCode,
1056 AArch64CC::CondCode &CondCode2) {
1057 CondCode2 = AArch64CC::AL;
1058 switch (P) {
1059 default:
1060 llvm_unreachable("Unknown FP condition!")::llvm::llvm_unreachable_internal("Unknown FP condition!", "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1060)
;
1061 case CmpInst::FCMP_OEQ:
1062 CondCode = AArch64CC::EQ;
1063 break;
1064 case CmpInst::FCMP_OGT:
1065 CondCode = AArch64CC::GT;
1066 break;
1067 case CmpInst::FCMP_OGE:
1068 CondCode = AArch64CC::GE;
1069 break;
1070 case CmpInst::FCMP_OLT:
1071 CondCode = AArch64CC::MI;
1072 break;
1073 case CmpInst::FCMP_OLE:
1074 CondCode = AArch64CC::LS;
1075 break;
1076 case CmpInst::FCMP_ONE:
1077 CondCode = AArch64CC::MI;
1078 CondCode2 = AArch64CC::GT;
1079 break;
1080 case CmpInst::FCMP_ORD:
1081 CondCode = AArch64CC::VC;
1082 break;
1083 case CmpInst::FCMP_UNO:
1084 CondCode = AArch64CC::VS;
1085 break;
1086 case CmpInst::FCMP_UEQ:
1087 CondCode = AArch64CC::EQ;
1088 CondCode2 = AArch64CC::VS;
1089 break;
1090 case CmpInst::FCMP_UGT:
1091 CondCode = AArch64CC::HI;
1092 break;
1093 case CmpInst::FCMP_UGE:
1094 CondCode = AArch64CC::PL;
1095 break;
1096 case CmpInst::FCMP_ULT:
1097 CondCode = AArch64CC::LT;
1098 break;
1099 case CmpInst::FCMP_ULE:
1100 CondCode = AArch64CC::LE;
1101 break;
1102 case CmpInst::FCMP_UNE:
1103 CondCode = AArch64CC::NE;
1104 break;
1105 }
1106}
1107
1108/// Return a register which can be used as a bit to test in a TB(N)Z.
1109static Register getTestBitReg(Register Reg, uint64_t &Bit, bool &Invert,
1110 MachineRegisterInfo &MRI) {
1111 assert(Reg.isValid() && "Expected valid register!")((Reg.isValid() && "Expected valid register!") ? static_cast
<void> (0) : __assert_fail ("Reg.isValid() && \"Expected valid register!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1111, __PRETTY_FUNCTION__))
;
1112 while (MachineInstr *MI = getDefIgnoringCopies(Reg, MRI)) {
1113 unsigned Opc = MI->getOpcode();
1114
1115 if (!MI->getOperand(0).isReg() ||
1116 !MRI.hasOneNonDBGUse(MI->getOperand(0).getReg()))
1117 break;
1118
1119 // (tbz (any_ext x), b) -> (tbz x, b) if we don't use the extended bits.
1120 //
1121 // (tbz (trunc x), b) -> (tbz x, b) is always safe, because the bit number
1122 // on the truncated x is the same as the bit number on x.
1123 if (Opc == TargetOpcode::G_ANYEXT || Opc == TargetOpcode::G_ZEXT ||
1124 Opc == TargetOpcode::G_TRUNC) {
1125 Register NextReg = MI->getOperand(1).getReg();
1126 // Did we find something worth folding?
1127 if (!NextReg.isValid() || !MRI.hasOneNonDBGUse(NextReg))
1128 break;
1129
1130 // NextReg is worth folding. Keep looking.
1131 Reg = NextReg;
1132 continue;
1133 }
1134
1135 // Attempt to find a suitable operation with a constant on one side.
1136 Optional<uint64_t> C;
1137 Register TestReg;
1138 switch (Opc) {
1139 default:
1140 break;
1141 case TargetOpcode::G_AND:
1142 case TargetOpcode::G_XOR: {
1143 TestReg = MI->getOperand(1).getReg();
1144 Register ConstantReg = MI->getOperand(2).getReg();
1145 auto VRegAndVal = getConstantVRegValWithLookThrough(ConstantReg, MRI);
1146 if (!VRegAndVal) {
1147 // AND commutes, check the other side for a constant.
1148 // FIXME: Can we canonicalize the constant so that it's always on the
1149 // same side at some point earlier?
1150 std::swap(ConstantReg, TestReg);
1151 VRegAndVal = getConstantVRegValWithLookThrough(ConstantReg, MRI);
1152 }
1153 if (VRegAndVal)
1154 C = VRegAndVal->Value;
1155 break;
1156 }
1157 case TargetOpcode::G_ASHR:
1158 case TargetOpcode::G_LSHR:
1159 case TargetOpcode::G_SHL: {
1160 TestReg = MI->getOperand(1).getReg();
1161 auto VRegAndVal =
1162 getConstantVRegValWithLookThrough(MI->getOperand(2).getReg(), MRI);
1163 if (VRegAndVal)
1164 C = VRegAndVal->Value;
1165 break;
1166 }
1167 }
1168
1169 // Didn't find a constant or viable register. Bail out of the loop.
1170 if (!C || !TestReg.isValid())
1171 break;
1172
1173 // We found a suitable instruction with a constant. Check to see if we can
1174 // walk through the instruction.
1175 Register NextReg;
1176 unsigned TestRegSize = MRI.getType(TestReg).getSizeInBits();
1177 switch (Opc) {
1178 default:
1179 break;
1180 case TargetOpcode::G_AND:
1181 // (tbz (and x, m), b) -> (tbz x, b) when the b-th bit of m is set.
1182 if ((*C >> Bit) & 1)
1183 NextReg = TestReg;
1184 break;
1185 case TargetOpcode::G_SHL:
1186 // (tbz (shl x, c), b) -> (tbz x, b-c) when b-c is positive and fits in
1187 // the type of the register.
1188 if (*C <= Bit && (Bit - *C) < TestRegSize) {
1189 NextReg = TestReg;
1190 Bit = Bit - *C;
1191 }
1192 break;
1193 case TargetOpcode::G_ASHR:
1194 // (tbz (ashr x, c), b) -> (tbz x, b+c) or (tbz x, msb) if b+c is > # bits
1195 // in x
1196 NextReg = TestReg;
1197 Bit = Bit + *C;
1198 if (Bit >= TestRegSize)
1199 Bit = TestRegSize - 1;
1200 break;
1201 case TargetOpcode::G_LSHR:
1202 // (tbz (lshr x, c), b) -> (tbz x, b+c) when b + c is < # bits in x
1203 if ((Bit + *C) < TestRegSize) {
1204 NextReg = TestReg;
1205 Bit = Bit + *C;
1206 }
1207 break;
1208 case TargetOpcode::G_XOR:
1209 // We can walk through a G_XOR by inverting whether we use tbz/tbnz when
1210 // appropriate.
1211 //
1212 // e.g. If x' = xor x, c, and the b-th bit is set in c then
1213 //
1214 // tbz x', b -> tbnz x, b
1215 //
1216 // Because x' only has the b-th bit set if x does not.
1217 if ((*C >> Bit) & 1)
1218 Invert = !Invert;
1219 NextReg = TestReg;
1220 break;
1221 }
1222
1223 // Check if we found anything worth folding.
1224 if (!NextReg.isValid())
1225 return Reg;
1226 Reg = NextReg;
1227 }
1228
1229 return Reg;
1230}
1231
1232MachineInstr *AArch64InstructionSelector::emitTestBit(
1233 Register TestReg, uint64_t Bit, bool IsNegative, MachineBasicBlock *DstMBB,
1234 MachineIRBuilder &MIB) const {
1235 assert(TestReg.isValid())((TestReg.isValid()) ? static_cast<void> (0) : __assert_fail
("TestReg.isValid()", "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1235, __PRETTY_FUNCTION__))
;
1236 assert(ProduceNonFlagSettingCondBr &&((ProduceNonFlagSettingCondBr && "Cannot emit TB(N)Z with speculation tracking!"
) ? static_cast<void> (0) : __assert_fail ("ProduceNonFlagSettingCondBr && \"Cannot emit TB(N)Z with speculation tracking!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1237, __PRETTY_FUNCTION__))
1237 "Cannot emit TB(N)Z with speculation tracking!")((ProduceNonFlagSettingCondBr && "Cannot emit TB(N)Z with speculation tracking!"
) ? static_cast<void> (0) : __assert_fail ("ProduceNonFlagSettingCondBr && \"Cannot emit TB(N)Z with speculation tracking!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1237, __PRETTY_FUNCTION__))
;
1238 MachineRegisterInfo &MRI = *MIB.getMRI();
1239
1240 // Attempt to optimize the test bit by walking over instructions.
1241 TestReg = getTestBitReg(TestReg, Bit, IsNegative, MRI);
1242 LLT Ty = MRI.getType(TestReg);
1243 unsigned Size = Ty.getSizeInBits();
1244 assert(!Ty.isVector() && "Expected a scalar!")((!Ty.isVector() && "Expected a scalar!") ? static_cast
<void> (0) : __assert_fail ("!Ty.isVector() && \"Expected a scalar!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1244, __PRETTY_FUNCTION__))
;
1245 assert(Bit < 64 && "Bit is too large!")((Bit < 64 && "Bit is too large!") ? static_cast<
void> (0) : __assert_fail ("Bit < 64 && \"Bit is too large!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1245, __PRETTY_FUNCTION__))
;
1246
1247 // When the test register is a 64-bit register, we have to narrow to make
1248 // TBNZW work.
1249 bool UseWReg = Bit < 32;
1250 unsigned NecessarySize = UseWReg ? 32 : 64;
1251 if (Size < NecessarySize)
1252 TestReg = widenGPRBankRegIfNeeded(TestReg, NecessarySize, MIB);
1253 else if (Size > NecessarySize)
1254 TestReg = narrowExtendRegIfNeeded(TestReg, MIB);
1255
1256 static const unsigned OpcTable[2][2] = {{AArch64::TBZX, AArch64::TBNZX},
1257 {AArch64::TBZW, AArch64::TBNZW}};
1258 unsigned Opc = OpcTable[UseWReg][IsNegative];
1259 auto TestBitMI =
1260 MIB.buildInstr(Opc).addReg(TestReg).addImm(Bit).addMBB(DstMBB);
1261 constrainSelectedInstRegOperands(*TestBitMI, TII, TRI, RBI);
1262 return &*TestBitMI;
1263}
1264
1265bool AArch64InstructionSelector::tryOptAndIntoCompareBranch(
1266 MachineInstr *AndInst, int64_t CmpConstant, const CmpInst::Predicate &Pred,
1267 MachineBasicBlock *DstMBB, MachineIRBuilder &MIB) const {
1268 // Given something like this:
1269 //
1270 // %x = ...Something...
1271 // %one = G_CONSTANT i64 1
1272 // %zero = G_CONSTANT i64 0
1273 // %and = G_AND %x, %one
1274 // %cmp = G_ICMP intpred(ne), %and, %zero
1275 // %cmp_trunc = G_TRUNC %cmp
1276 // G_BRCOND %cmp_trunc, %bb.3
1277 //
1278 // We want to try and fold the AND into the G_BRCOND and produce either a
1279 // TBNZ (when we have intpred(ne)) or a TBZ (when we have intpred(eq)).
1280 //
1281 // In this case, we'd get
1282 //
1283 // TBNZ %x %bb.3
1284 //
1285 if (!AndInst || AndInst->getOpcode() != TargetOpcode::G_AND)
1286 return false;
1287
1288 // Need to be comparing against 0 to fold.
1289 if (CmpConstant != 0)
1290 return false;
1291
1292 MachineRegisterInfo &MRI = *MIB.getMRI();
1293
1294 // Only support EQ and NE. If we have LT, then it *is* possible to fold, but
1295 // we don't want to do this. When we have an AND and LT, we need a TST/ANDS,
1296 // so folding would be redundant.
1297 if (Pred != CmpInst::Predicate::ICMP_EQ &&
1298 Pred != CmpInst::Predicate::ICMP_NE)
1299 return false;
1300
1301 // Check if the AND has a constant on its RHS which we can use as a mask.
1302 // If it's a power of 2, then it's the same as checking a specific bit.
1303 // (e.g, ANDing with 8 == ANDing with 000...100 == testing if bit 3 is set)
1304 auto MaybeBit =
1305 getConstantVRegValWithLookThrough(AndInst->getOperand(2).getReg(), MRI);
1306 if (!MaybeBit || !isPowerOf2_64(MaybeBit->Value))
1307 return false;
1308
1309 uint64_t Bit = Log2_64(static_cast<uint64_t>(MaybeBit->Value));
1310 Register TestReg = AndInst->getOperand(1).getReg();
1311 bool Invert = Pred == CmpInst::Predicate::ICMP_NE;
1312
1313 // Emit a TB(N)Z.
1314 emitTestBit(TestReg, Bit, Invert, DstMBB, MIB);
1315 return true;
1316}
1317
1318bool AArch64InstructionSelector::selectCompareBranch(
1319 MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
1320
1321 const Register CondReg = I.getOperand(0).getReg();
1322 MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
1323 MachineInstr *CCMI = MRI.getVRegDef(CondReg);
1324 if (CCMI->getOpcode() == TargetOpcode::G_TRUNC)
1325 CCMI = MRI.getVRegDef(CCMI->getOperand(1).getReg());
1326 if (CCMI->getOpcode() != TargetOpcode::G_ICMP)
1327 return false;
1328
1329 Register LHS = CCMI->getOperand(2).getReg();
1330 Register RHS = CCMI->getOperand(3).getReg();
1331 auto VRegAndVal = getConstantVRegValWithLookThrough(RHS, MRI);
1332 MachineIRBuilder MIB(I);
1333 CmpInst::Predicate Pred =
1334 (CmpInst::Predicate)CCMI->getOperand(1).getPredicate();
1335 MachineInstr *LHSMI = getDefIgnoringCopies(LHS, MRI);
1336
1337 // When we can emit a TB(N)Z, prefer that.
1338 //
1339 // Handle non-commutative condition codes first.
1340 // Note that we don't want to do this when we have a G_AND because it can
1341 // become a tst. The tst will make the test bit in the TB(N)Z redundant.
1342 if (VRegAndVal && LHSMI->getOpcode() != TargetOpcode::G_AND) {
1343 int64_t C = VRegAndVal->Value;
1344
1345 // When we have a greater-than comparison, we can just test if the msb is
1346 // zero.
1347 if (C == -1 && Pred == CmpInst::ICMP_SGT) {
1348 uint64_t Bit = MRI.getType(LHS).getSizeInBits() - 1;
1349 emitTestBit(LHS, Bit, /*IsNegative = */ false, DestMBB, MIB);
1350 I.eraseFromParent();
1351 return true;
1352 }
1353
1354 // When we have a less than comparison, we can just test if the msb is not
1355 // zero.
1356 if (C == 0 && Pred == CmpInst::ICMP_SLT) {
1357 uint64_t Bit = MRI.getType(LHS).getSizeInBits() - 1;
1358 emitTestBit(LHS, Bit, /*IsNegative = */ true, DestMBB, MIB);
1359 I.eraseFromParent();
1360 return true;
1361 }
1362 }
1363
1364 if (!VRegAndVal) {
1365 std::swap(RHS, LHS);
1366 VRegAndVal = getConstantVRegValWithLookThrough(RHS, MRI);
1367 LHSMI = getDefIgnoringCopies(LHS, MRI);
1368 }
1369
1370 if (!VRegAndVal || VRegAndVal->Value != 0) {
1371 // If we can't select a CBZ then emit a cmp + Bcc.
1372 MachineInstr *Cmp;
1373 std::tie(Cmp, Pred) = emitIntegerCompare(
1374 CCMI->getOperand(2), CCMI->getOperand(3), CCMI->getOperand(1), MIB);
1375 if (!Cmp)
1376 return false;
1377 const AArch64CC::CondCode CC = changeICMPPredToAArch64CC(Pred);
1378 MIB.buildInstr(AArch64::Bcc, {}, {}).addImm(CC).addMBB(DestMBB);
1379 I.eraseFromParent();
1380 return true;
1381 }
1382
1383 // Try to emit a TB(N)Z for an eq or ne condition.
1384 if (tryOptAndIntoCompareBranch(LHSMI, VRegAndVal->Value, Pred, DestMBB,
1385 MIB)) {
1386 I.eraseFromParent();
1387 return true;
1388 }
1389
1390 const RegisterBank &RB = *RBI.getRegBank(LHS, MRI, TRI);
1391 if (RB.getID() != AArch64::GPRRegBankID)
1392 return false;
1393 if (Pred != CmpInst::ICMP_NE && Pred != CmpInst::ICMP_EQ)
1394 return false;
1395
1396 const unsigned CmpWidth = MRI.getType(LHS).getSizeInBits();
1397 unsigned CBOpc = 0;
1398 if (CmpWidth <= 32)
1399 CBOpc = (Pred == CmpInst::ICMP_EQ ? AArch64::CBZW : AArch64::CBNZW);
1400 else if (CmpWidth == 64)
1401 CBOpc = (Pred == CmpInst::ICMP_EQ ? AArch64::CBZX : AArch64::CBNZX);
1402 else
1403 return false;
1404
1405 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CBOpc))
1406 .addUse(LHS)
1407 .addMBB(DestMBB)
1408 .constrainAllUses(TII, TRI, RBI);
1409
1410 I.eraseFromParent();
1411 return true;
1412}
1413
1414/// Returns the element immediate value of a vector shift operand if found.
1415/// This needs to detect a splat-like operation, e.g. a G_BUILD_VECTOR.
1416static Optional<int64_t> getVectorShiftImm(Register Reg,
1417 MachineRegisterInfo &MRI) {
1418 assert(MRI.getType(Reg).isVector() && "Expected a *vector* shift operand")((MRI.getType(Reg).isVector() && "Expected a *vector* shift operand"
) ? static_cast<void> (0) : __assert_fail ("MRI.getType(Reg).isVector() && \"Expected a *vector* shift operand\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1418, __PRETTY_FUNCTION__))
;
1419 MachineInstr *OpMI = MRI.getVRegDef(Reg);
1420 assert(OpMI && "Expected to find a vreg def for vector shift operand")((OpMI && "Expected to find a vreg def for vector shift operand"
) ? static_cast<void> (0) : __assert_fail ("OpMI && \"Expected to find a vreg def for vector shift operand\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1420, __PRETTY_FUNCTION__))
;
1421 if (OpMI->getOpcode() != TargetOpcode::G_BUILD_VECTOR)
1422 return None;
1423
1424 // Check all operands are identical immediates.
1425 int64_t ImmVal = 0;
1426 for (unsigned Idx = 1; Idx < OpMI->getNumOperands(); ++Idx) {
1427 auto VRegAndVal = getConstantVRegValWithLookThrough(OpMI->getOperand(Idx).getReg(), MRI);
1428 if (!VRegAndVal)
1429 return None;
1430
1431 if (Idx == 1)
1432 ImmVal = VRegAndVal->Value;
1433 if (ImmVal != VRegAndVal->Value)
1434 return None;
1435 }
1436
1437 return ImmVal;
1438}
1439
1440/// Matches and returns the shift immediate value for a SHL instruction given
1441/// a shift operand.
1442static Optional<int64_t> getVectorSHLImm(LLT SrcTy, Register Reg, MachineRegisterInfo &MRI) {
1443 Optional<int64_t> ShiftImm = getVectorShiftImm(Reg, MRI);
1444 if (!ShiftImm)
1445 return None;
1446 // Check the immediate is in range for a SHL.
1447 int64_t Imm = *ShiftImm;
1448 if (Imm < 0)
1449 return None;
1450 switch (SrcTy.getElementType().getSizeInBits()) {
1451 default:
1452 LLVM_DEBUG(dbgs() << "Unhandled element type for vector shift")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled element type for vector shift"
; } } while (false)
;
1453 return None;
1454 case 8:
1455 if (Imm > 7)
1456 return None;
1457 break;
1458 case 16:
1459 if (Imm > 15)
1460 return None;
1461 break;
1462 case 32:
1463 if (Imm > 31)
1464 return None;
1465 break;
1466 case 64:
1467 if (Imm > 63)
1468 return None;
1469 break;
1470 }
1471 return Imm;
1472}
1473
1474bool AArch64InstructionSelector::selectVectorSHL(
1475 MachineInstr &I, MachineRegisterInfo &MRI) const {
1476 assert(I.getOpcode() == TargetOpcode::G_SHL)((I.getOpcode() == TargetOpcode::G_SHL) ? static_cast<void
> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_SHL"
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1476, __PRETTY_FUNCTION__))
;
1477 Register DstReg = I.getOperand(0).getReg();
1478 const LLT Ty = MRI.getType(DstReg);
1479 Register Src1Reg = I.getOperand(1).getReg();
1480 Register Src2Reg = I.getOperand(2).getReg();
1481
1482 if (!Ty.isVector())
1483 return false;
1484
1485 // Check if we have a vector of constants on RHS that we can select as the
1486 // immediate form.
1487 Optional<int64_t> ImmVal = getVectorSHLImm(Ty, Src2Reg, MRI);
1488
1489 unsigned Opc = 0;
1490 if (Ty == LLT::vector(2, 64)) {
1491 Opc = ImmVal ? AArch64::SHLv2i64_shift : AArch64::USHLv2i64;
1492 } else if (Ty == LLT::vector(4, 32)) {
1493 Opc = ImmVal ? AArch64::SHLv4i32_shift : AArch64::USHLv4i32;
1494 } else if (Ty == LLT::vector(2, 32)) {
1495 Opc = ImmVal ? AArch64::SHLv2i32_shift : AArch64::USHLv2i32;
1496 } else {
1497 LLVM_DEBUG(dbgs() << "Unhandled G_SHL type")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled G_SHL type"; }
} while (false)
;
1498 return false;
1499 }
1500
1501 MachineIRBuilder MIB(I);
1502 auto Shl = MIB.buildInstr(Opc, {DstReg}, {Src1Reg});
1503 if (ImmVal)
1504 Shl.addImm(*ImmVal);
1505 else
1506 Shl.addUse(Src2Reg);
1507 constrainSelectedInstRegOperands(*Shl, TII, TRI, RBI);
1508 I.eraseFromParent();
1509 return true;
1510}
1511
1512bool AArch64InstructionSelector::selectVectorASHR(
1513 MachineInstr &I, MachineRegisterInfo &MRI) const {
1514 assert(I.getOpcode() == TargetOpcode::G_ASHR)((I.getOpcode() == TargetOpcode::G_ASHR) ? static_cast<void
> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_ASHR"
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1514, __PRETTY_FUNCTION__))
;
1515 Register DstReg = I.getOperand(0).getReg();
1516 const LLT Ty = MRI.getType(DstReg);
1517 Register Src1Reg = I.getOperand(1).getReg();
1518 Register Src2Reg = I.getOperand(2).getReg();
1519
1520 if (!Ty.isVector())
1521 return false;
1522
1523 // There is not a shift right register instruction, but the shift left
1524 // register instruction takes a signed value, where negative numbers specify a
1525 // right shift.
1526
1527 unsigned Opc = 0;
1528 unsigned NegOpc = 0;
1529 const TargetRegisterClass *RC = nullptr;
1530 if (Ty == LLT::vector(2, 64)) {
1531 Opc = AArch64::SSHLv2i64;
1532 NegOpc = AArch64::NEGv2i64;
1533 RC = &AArch64::FPR128RegClass;
1534 } else if (Ty == LLT::vector(4, 32)) {
1535 Opc = AArch64::SSHLv4i32;
1536 NegOpc = AArch64::NEGv4i32;
1537 RC = &AArch64::FPR128RegClass;
1538 } else if (Ty == LLT::vector(2, 32)) {
1539 Opc = AArch64::SSHLv2i32;
1540 NegOpc = AArch64::NEGv2i32;
1541 RC = &AArch64::FPR64RegClass;
1542 } else {
1543 LLVM_DEBUG(dbgs() << "Unhandled G_ASHR type")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled G_ASHR type"; }
} while (false)
;
1544 return false;
1545 }
1546
1547 MachineIRBuilder MIB(I);
1548 auto Neg = MIB.buildInstr(NegOpc, {RC}, {Src2Reg});
1549 constrainSelectedInstRegOperands(*Neg, TII, TRI, RBI);
1550 auto SShl = MIB.buildInstr(Opc, {DstReg}, {Src1Reg, Neg});
1551 constrainSelectedInstRegOperands(*SShl, TII, TRI, RBI);
1552 I.eraseFromParent();
1553 return true;
1554}
1555
1556bool AArch64InstructionSelector::selectVaStartAAPCS(
1557 MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
1558 return false;
1559}
1560
1561bool AArch64InstructionSelector::selectVaStartDarwin(
1562 MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
1563 AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
1564 Register ListReg = I.getOperand(0).getReg();
1565
1566 Register ArgsAddrReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
1567
1568 auto MIB =
1569 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::ADDXri))
1570 .addDef(ArgsAddrReg)
1571 .addFrameIndex(FuncInfo->getVarArgsStackIndex())
1572 .addImm(0)
1573 .addImm(0);
1574
1575 constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
1576
1577 MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::STRXui))
1578 .addUse(ArgsAddrReg)
1579 .addUse(ListReg)
1580 .addImm(0)
1581 .addMemOperand(*I.memoperands_begin());
1582
1583 constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
1584 I.eraseFromParent();
1585 return true;
1586}
1587
1588void AArch64InstructionSelector::materializeLargeCMVal(
1589 MachineInstr &I, const Value *V, unsigned OpFlags) const {
1590 MachineBasicBlock &MBB = *I.getParent();
1591 MachineFunction &MF = *MBB.getParent();
1592 MachineRegisterInfo &MRI = MF.getRegInfo();
1593 MachineIRBuilder MIB(I);
1594
1595 auto MovZ = MIB.buildInstr(AArch64::MOVZXi, {&AArch64::GPR64RegClass}, {});
1596 MovZ->addOperand(MF, I.getOperand(1));
1597 MovZ->getOperand(1).setTargetFlags(OpFlags | AArch64II::MO_G0 |
1598 AArch64II::MO_NC);
1599 MovZ->addOperand(MF, MachineOperand::CreateImm(0));
1600 constrainSelectedInstRegOperands(*MovZ, TII, TRI, RBI);
1601
1602 auto BuildMovK = [&](Register SrcReg, unsigned char Flags, unsigned Offset,
1603 Register ForceDstReg) {
1604 Register DstReg = ForceDstReg
1605 ? ForceDstReg
1606 : MRI.createVirtualRegister(&AArch64::GPR64RegClass);
1607 auto MovI = MIB.buildInstr(AArch64::MOVKXi).addDef(DstReg).addUse(SrcReg);
1608 if (auto *GV = dyn_cast<GlobalValue>(V)) {
1609 MovI->addOperand(MF, MachineOperand::CreateGA(
1610 GV, MovZ->getOperand(1).getOffset(), Flags));
1611 } else {
1612 MovI->addOperand(
1613 MF, MachineOperand::CreateBA(cast<BlockAddress>(V),
1614 MovZ->getOperand(1).getOffset(), Flags));
1615 }
1616 MovI->addOperand(MF, MachineOperand::CreateImm(Offset));
1617 constrainSelectedInstRegOperands(*MovI, TII, TRI, RBI);
1618 return DstReg;
1619 };
1620 Register DstReg = BuildMovK(MovZ.getReg(0),
1621 AArch64II::MO_G1 | AArch64II::MO_NC, 16, 0);
1622 DstReg = BuildMovK(DstReg, AArch64II::MO_G2 | AArch64II::MO_NC, 32, 0);
1623 BuildMovK(DstReg, AArch64II::MO_G3, 48, I.getOperand(0).getReg());
1624 return;
1625}
1626
1627bool AArch64InstructionSelector::preISelLower(MachineInstr &I) {
1628 MachineBasicBlock &MBB = *I.getParent();
1629 MachineFunction &MF = *MBB.getParent();
1630 MachineRegisterInfo &MRI = MF.getRegInfo();
1631
1632 switch (I.getOpcode()) {
1633 case TargetOpcode::G_SHL:
1634 case TargetOpcode::G_ASHR:
1635 case TargetOpcode::G_LSHR: {
1636 // These shifts are legalized to have 64 bit shift amounts because we want
1637 // to take advantage of the existing imported selection patterns that assume
1638 // the immediates are s64s. However, if the shifted type is 32 bits and for
1639 // some reason we receive input GMIR that has an s64 shift amount that's not
1640 // a G_CONSTANT, insert a truncate so that we can still select the s32
1641 // register-register variant.
1642 Register SrcReg = I.getOperand(1).getReg();
1643 Register ShiftReg = I.getOperand(2).getReg();
1644 const LLT ShiftTy = MRI.getType(ShiftReg);
1645 const LLT SrcTy = MRI.getType(SrcReg);
1646 if (SrcTy.isVector())
1647 return false;
1648 assert(!ShiftTy.isVector() && "unexpected vector shift ty")((!ShiftTy.isVector() && "unexpected vector shift ty"
) ? static_cast<void> (0) : __assert_fail ("!ShiftTy.isVector() && \"unexpected vector shift ty\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1648, __PRETTY_FUNCTION__))
;
1649 if (SrcTy.getSizeInBits() != 32 || ShiftTy.getSizeInBits() != 64)
1650 return false;
1651 auto *AmtMI = MRI.getVRegDef(ShiftReg);
1652 assert(AmtMI && "could not find a vreg definition for shift amount")((AmtMI && "could not find a vreg definition for shift amount"
) ? static_cast<void> (0) : __assert_fail ("AmtMI && \"could not find a vreg definition for shift amount\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1652, __PRETTY_FUNCTION__))
;
1653 if (AmtMI->getOpcode() != TargetOpcode::G_CONSTANT) {
1654 // Insert a subregister copy to implement a 64->32 trunc
1655 MachineIRBuilder MIB(I);
1656 auto Trunc = MIB.buildInstr(TargetOpcode::COPY, {SrcTy}, {})
1657 .addReg(ShiftReg, 0, AArch64::sub_32);
1658 MRI.setRegBank(Trunc.getReg(0), RBI.getRegBank(AArch64::GPRRegBankID));
1659 I.getOperand(2).setReg(Trunc.getReg(0));
1660 }
1661 return true;
1662 }
1663 case TargetOpcode::G_STORE:
1664 return contractCrossBankCopyIntoStore(I, MRI);
1665 case TargetOpcode::G_PTR_ADD:
1666 return convertPtrAddToAdd(I, MRI);
1667 case TargetOpcode::G_LOAD: {
1668 // For scalar loads of pointers, we try to convert the dest type from p0
1669 // to s64 so that our imported patterns can match. Like with the G_PTR_ADD
1670 // conversion, this should be ok because all users should have been
1671 // selected already, so the type doesn't matter for them.
1672 Register DstReg = I.getOperand(0).getReg();
1673 const LLT DstTy = MRI.getType(DstReg);
1674 if (!DstTy.isPointer())
1675 return false;
1676 MRI.setType(DstReg, LLT::scalar(64));
1677 return true;
1678 }
1679 default:
1680 return false;
1681 }
1682}
1683
1684/// This lowering tries to look for G_PTR_ADD instructions and then converts
1685/// them to a standard G_ADD with a COPY on the source.
1686///
1687/// The motivation behind this is to expose the add semantics to the imported
1688/// tablegen patterns. We shouldn't need to check for uses being loads/stores,
1689/// because the selector works bottom up, uses before defs. By the time we
1690/// end up trying to select a G_PTR_ADD, we should have already attempted to
1691/// fold this into addressing modes and were therefore unsuccessful.
1692bool AArch64InstructionSelector::convertPtrAddToAdd(
1693 MachineInstr &I, MachineRegisterInfo &MRI) {
1694 assert(I.getOpcode() == TargetOpcode::G_PTR_ADD && "Expected G_PTR_ADD")((I.getOpcode() == TargetOpcode::G_PTR_ADD && "Expected G_PTR_ADD"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_PTR_ADD && \"Expected G_PTR_ADD\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1694, __PRETTY_FUNCTION__))
;
1695 Register DstReg = I.getOperand(0).getReg();
1696 Register AddOp1Reg = I.getOperand(1).getReg();
1697 const LLT PtrTy = MRI.getType(DstReg);
1698 if (PtrTy.getAddressSpace() != 0)
1699 return false;
1700
1701 MachineIRBuilder MIB(I);
1702 const LLT CastPtrTy = PtrTy.isVector() ? LLT::vector(2, 64) : LLT::scalar(64);
1703 auto PtrToInt = MIB.buildPtrToInt(CastPtrTy, AddOp1Reg);
1704 // Set regbanks on the registers.
1705 if (PtrTy.isVector())
1706 MRI.setRegBank(PtrToInt.getReg(0), RBI.getRegBank(AArch64::FPRRegBankID));
1707 else
1708 MRI.setRegBank(PtrToInt.getReg(0), RBI.getRegBank(AArch64::GPRRegBankID));
1709
1710 // Now turn the %dst(p0) = G_PTR_ADD %base, off into:
1711 // %dst(intty) = G_ADD %intbase, off
1712 I.setDesc(TII.get(TargetOpcode::G_ADD));
1713 MRI.setType(DstReg, CastPtrTy);
1714 I.getOperand(1).setReg(PtrToInt.getReg(0));
1715 if (!select(*PtrToInt)) {
1716 LLVM_DEBUG(dbgs() << "Failed to select G_PTRTOINT in convertPtrAddToAdd")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to select G_PTRTOINT in convertPtrAddToAdd"
; } } while (false)
;
1717 return false;
1718 }
1719 return true;
1720}
1721
1722bool AArch64InstructionSelector::earlySelectSHL(
1723 MachineInstr &I, MachineRegisterInfo &MRI) const {
1724 // We try to match the immediate variant of LSL, which is actually an alias
1725 // for a special case of UBFM. Otherwise, we fall back to the imported
1726 // selector which will match the register variant.
1727 assert(I.getOpcode() == TargetOpcode::G_SHL && "unexpected op")((I.getOpcode() == TargetOpcode::G_SHL && "unexpected op"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_SHL && \"unexpected op\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1727, __PRETTY_FUNCTION__))
;
1728 const auto &MO = I.getOperand(2);
1729 auto VRegAndVal = getConstantVRegVal(MO.getReg(), MRI);
1730 if (!VRegAndVal)
1731 return false;
1732
1733 const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
1734 if (DstTy.isVector())
1735 return false;
1736 bool Is64Bit = DstTy.getSizeInBits() == 64;
1737 auto Imm1Fn = Is64Bit ? selectShiftA_64(MO) : selectShiftA_32(MO);
1738 auto Imm2Fn = Is64Bit ? selectShiftB_64(MO) : selectShiftB_32(MO);
1739 MachineIRBuilder MIB(I);
1740
1741 if (!Imm1Fn || !Imm2Fn)
1742 return false;
1743
1744 auto NewI =
1745 MIB.buildInstr(Is64Bit ? AArch64::UBFMXri : AArch64::UBFMWri,
1746 {I.getOperand(0).getReg()}, {I.getOperand(1).getReg()});
1747
1748 for (auto &RenderFn : *Imm1Fn)
1749 RenderFn(NewI);
1750 for (auto &RenderFn : *Imm2Fn)
1751 RenderFn(NewI);
1752
1753 I.eraseFromParent();
1754 return constrainSelectedInstRegOperands(*NewI, TII, TRI, RBI);
1755}
1756
1757bool AArch64InstructionSelector::contractCrossBankCopyIntoStore(
1758 MachineInstr &I, MachineRegisterInfo &MRI) {
1759 assert(I.getOpcode() == TargetOpcode::G_STORE && "Expected G_STORE")((I.getOpcode() == TargetOpcode::G_STORE && "Expected G_STORE"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_STORE && \"Expected G_STORE\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1759, __PRETTY_FUNCTION__))
;
1760 // If we're storing a scalar, it doesn't matter what register bank that
1761 // scalar is on. All that matters is the size.
1762 //
1763 // So, if we see something like this (with a 32-bit scalar as an example):
1764 //
1765 // %x:gpr(s32) = ... something ...
1766 // %y:fpr(s32) = COPY %x:gpr(s32)
1767 // G_STORE %y:fpr(s32)
1768 //
1769 // We can fix this up into something like this:
1770 //
1771 // G_STORE %x:gpr(s32)
1772 //
1773 // And then continue the selection process normally.
1774 Register DefDstReg = getSrcRegIgnoringCopies(I.getOperand(0).getReg(), MRI);
1775 if (!DefDstReg.isValid())
1776 return false;
1777 LLT DefDstTy = MRI.getType(DefDstReg);
1778 Register StoreSrcReg = I.getOperand(0).getReg();
1779 LLT StoreSrcTy = MRI.getType(StoreSrcReg);
1780
1781 // If we get something strange like a physical register, then we shouldn't
1782 // go any further.
1783 if (!DefDstTy.isValid())
1784 return false;
1785
1786 // Are the source and dst types the same size?
1787 if (DefDstTy.getSizeInBits() != StoreSrcTy.getSizeInBits())
1788 return false;
1789
1790 if (RBI.getRegBank(StoreSrcReg, MRI, TRI) ==
1791 RBI.getRegBank(DefDstReg, MRI, TRI))
1792 return false;
1793
1794 // We have a cross-bank copy, which is entering a store. Let's fold it.
1795 I.getOperand(0).setReg(DefDstReg);
1796 return true;
1797}
1798
1799bool AArch64InstructionSelector::earlySelect(MachineInstr &I) const {
1800 assert(I.getParent() && "Instruction should be in a basic block!")((I.getParent() && "Instruction should be in a basic block!"
) ? static_cast<void> (0) : __assert_fail ("I.getParent() && \"Instruction should be in a basic block!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1800, __PRETTY_FUNCTION__))
;
1801 assert(I.getParent()->getParent() && "Instruction should be in a function!")((I.getParent()->getParent() && "Instruction should be in a function!"
) ? static_cast<void> (0) : __assert_fail ("I.getParent()->getParent() && \"Instruction should be in a function!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1801, __PRETTY_FUNCTION__))
;
1802
1803 MachineBasicBlock &MBB = *I.getParent();
1804 MachineFunction &MF = *MBB.getParent();
1805 MachineRegisterInfo &MRI = MF.getRegInfo();
1806
1807 switch (I.getOpcode()) {
1808 case TargetOpcode::G_BR: {
1809 // If the branch jumps to the fallthrough block, don't bother emitting it.
1810 // Only do this for -O0 for a good code size improvement, because when
1811 // optimizations are enabled we want to leave this choice to
1812 // MachineBlockPlacement.
1813 bool EnableOpt = MF.getTarget().getOptLevel() != CodeGenOpt::None;
1814 if (EnableOpt || !MBB.isLayoutSuccessor(I.getOperand(0).getMBB()))
1815 return false;
1816 I.eraseFromParent();
1817 return true;
1818 }
1819 case TargetOpcode::G_SHL:
1820 return earlySelectSHL(I, MRI);
1821 case TargetOpcode::G_CONSTANT: {
1822 bool IsZero = false;
1823 if (I.getOperand(1).isCImm())
1824 IsZero = I.getOperand(1).getCImm()->getZExtValue() == 0;
1825 else if (I.getOperand(1).isImm())
1826 IsZero = I.getOperand(1).getImm() == 0;
1827
1828 if (!IsZero)
1829 return false;
1830
1831 Register DefReg = I.getOperand(0).getReg();
1832 LLT Ty = MRI.getType(DefReg);
1833 if (Ty.getSizeInBits() == 64) {
1834 I.getOperand(1).ChangeToRegister(AArch64::XZR, false);
1835 RBI.constrainGenericRegister(DefReg, AArch64::GPR64RegClass, MRI);
1836 } else if (Ty.getSizeInBits() == 32) {
1837 I.getOperand(1).ChangeToRegister(AArch64::WZR, false);
1838 RBI.constrainGenericRegister(DefReg, AArch64::GPR32RegClass, MRI);
1839 } else
1840 return false;
1841
1842 I.setDesc(TII.get(TargetOpcode::COPY));
1843 return true;
1844 }
1845 default:
1846 return false;
1847 }
1848}
1849
1850bool AArch64InstructionSelector::select(MachineInstr &I) {
1851 assert(I.getParent() && "Instruction should be in a basic block!")((I.getParent() && "Instruction should be in a basic block!"
) ? static_cast<void> (0) : __assert_fail ("I.getParent() && \"Instruction should be in a basic block!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1851, __PRETTY_FUNCTION__))
;
1852 assert(I.getParent()->getParent() && "Instruction should be in a function!")((I.getParent()->getParent() && "Instruction should be in a function!"
) ? static_cast<void> (0) : __assert_fail ("I.getParent()->getParent() && \"Instruction should be in a function!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1852, __PRETTY_FUNCTION__))
;
1853
1854 MachineBasicBlock &MBB = *I.getParent();
1855 MachineFunction &MF = *MBB.getParent();
1856 MachineRegisterInfo &MRI = MF.getRegInfo();
1857
1858 const AArch64Subtarget *Subtarget =
1859 &static_cast<const AArch64Subtarget &>(MF.getSubtarget());
1860 if (Subtarget->requiresStrictAlign()) {
1861 // We don't support this feature yet.
1862 LLVM_DEBUG(dbgs() << "AArch64 GISel does not support strict-align yet\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "AArch64 GISel does not support strict-align yet\n"
; } } while (false)
;
1863 return false;
1864 }
1865
1866 unsigned Opcode = I.getOpcode();
1867 // G_PHI requires same handling as PHI
1868 if (!I.isPreISelOpcode() || Opcode == TargetOpcode::G_PHI) {
1869 // Certain non-generic instructions also need some special handling.
1870
1871 if (Opcode == TargetOpcode::LOAD_STACK_GUARD)
1872 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1873
1874 if (Opcode == TargetOpcode::PHI || Opcode == TargetOpcode::G_PHI) {
1875 const Register DefReg = I.getOperand(0).getReg();
1876 const LLT DefTy = MRI.getType(DefReg);
1877
1878 const RegClassOrRegBank &RegClassOrBank =
1879 MRI.getRegClassOrRegBank(DefReg);
1880
1881 const TargetRegisterClass *DefRC
1882 = RegClassOrBank.dyn_cast<const TargetRegisterClass *>();
1883 if (!DefRC) {
1884 if (!DefTy.isValid()) {
1885 LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "PHI operand has no type, not a gvreg?\n"
; } } while (false)
;
1886 return false;
1887 }
1888 const RegisterBank &RB = *RegClassOrBank.get<const RegisterBank *>();
1889 DefRC = getRegClassForTypeOnBank(DefTy, RB, RBI);
1890 if (!DefRC) {
1891 LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "PHI operand has unexpected size/bank\n"
; } } while (false)
;
1892 return false;
1893 }
1894 }
1895
1896 I.setDesc(TII.get(TargetOpcode::PHI));
1897
1898 return RBI.constrainGenericRegister(DefReg, *DefRC, MRI);
1899 }
1900
1901 if (I.isCopy())
1902 return selectCopy(I, TII, MRI, TRI, RBI);
1903
1904 return true;
1905 }
1906
1907
1908 if (I.getNumOperands() != I.getNumExplicitOperands()) {
1909 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic instruction has unexpected implicit operands\n"
; } } while (false)
1910 dbgs() << "Generic instruction has unexpected implicit operands\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic instruction has unexpected implicit operands\n"
; } } while (false)
;
1911 return false;
1912 }
1913
1914 // Try to do some lowering before we start instruction selecting. These
1915 // lowerings are purely transformations on the input G_MIR and so selection
1916 // must continue after any modification of the instruction.
1917 if (preISelLower(I)) {
1918 Opcode = I.getOpcode(); // The opcode may have been modified, refresh it.
1919 }
1920
1921 // There may be patterns where the importer can't deal with them optimally,
1922 // but does select it to a suboptimal sequence so our custom C++ selection
1923 // code later never has a chance to work on it. Therefore, we have an early
1924 // selection attempt here to give priority to certain selection routines
1925 // over the imported ones.
1926 if (earlySelect(I))
1927 return true;
1928
1929 if (selectImpl(I, *CoverageInfo))
1930 return true;
1931
1932 LLT Ty =
1933 I.getOperand(0).isReg() ? MRI.getType(I.getOperand(0).getReg()) : LLT{};
1934
1935 MachineIRBuilder MIB(I);
1936
1937 switch (Opcode) {
1938 case TargetOpcode::G_BRCOND: {
1939 if (Ty.getSizeInBits() > 32) {
1940 // We shouldn't need this on AArch64, but it would be implemented as an
1941 // EXTRACT_SUBREG followed by a TBNZW because TBNZX has no encoding if the
1942 // bit being tested is < 32.
1943 LLVM_DEBUG(dbgs() << "G_BRCOND has type: " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_BRCOND has type: " <<
Ty << ", expected at most 32-bits"; } } while (false)
1944 << ", expected at most 32-bits")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_BRCOND has type: " <<
Ty << ", expected at most 32-bits"; } } while (false)
;
1945 return false;
1946 }
1947
1948 Register CondReg = I.getOperand(0).getReg();
1949 MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
1950
1951 // Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z
1952 // instructions will not be produced, as they are conditional branch
1953 // instructions that do not set flags.
1954 if (ProduceNonFlagSettingCondBr && selectCompareBranch(I, MF, MRI))
1955 return true;
1956
1957 if (ProduceNonFlagSettingCondBr) {
1958 unsigned BOpc = AArch64::TBNZW;
1959 // Try to fold a not, i.e. a xor, cond, 1.
1960 Register XorSrc;
1961 int64_t Cst;
1962 if (mi_match(CondReg, MRI,
1963 m_GTrunc(m_GXor(m_Reg(XorSrc), m_ICst(Cst)))) &&
1964 Cst == 1) {
1965 CondReg = XorSrc;
1966 BOpc = AArch64::TBZW;
1967 if (MRI.getType(XorSrc).getSizeInBits() > 32)
1968 BOpc = AArch64::TBZX;
1969 }
1970 auto MIB = BuildMI(MBB, I, I.getDebugLoc(), TII.get(BOpc))
1971 .addUse(CondReg)
1972 .addImm(/*bit offset=*/0)
1973 .addMBB(DestMBB);
1974
1975 I.eraseFromParent();
1976 return constrainSelectedInstRegOperands(*MIB.getInstr(), TII, TRI, RBI);
1977 } else {
1978 auto CMP = BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::ANDSWri))
1979 .addDef(AArch64::WZR)
1980 .addUse(CondReg)
1981 .addImm(1);
1982 constrainSelectedInstRegOperands(*CMP.getInstr(), TII, TRI, RBI);
1983 auto Bcc =
1984 BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::Bcc))
1985 .addImm(AArch64CC::EQ)
1986 .addMBB(DestMBB);
1987
1988 I.eraseFromParent();
1989 return constrainSelectedInstRegOperands(*Bcc.getInstr(), TII, TRI, RBI);
1990 }
1991 }
1992
1993 case TargetOpcode::G_BRINDIRECT: {
1994 I.setDesc(TII.get(AArch64::BR));
1995 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1996 }
1997
1998 case TargetOpcode::G_BRJT:
1999 return selectBrJT(I, MRI);
2000
2001 case AArch64::G_ADD_LOW: {
2002 // This op may have been separated from it's ADRP companion by the localizer
2003 // or some other code motion pass. Given that many CPUs will try to
2004 // macro fuse these operations anyway, select this into a MOVaddr pseudo
2005 // which will later be expanded into an ADRP+ADD pair after scheduling.
2006 MachineInstr *BaseMI = MRI.getVRegDef(I.getOperand(1).getReg());
2007 if (BaseMI->getOpcode() != AArch64::ADRP) {
2008 I.setDesc(TII.get(AArch64::ADDXri));
2009 I.addOperand(MachineOperand::CreateImm(0));
2010 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2011 }
2012 assert(TM.getCodeModel() == CodeModel::Small &&((TM.getCodeModel() == CodeModel::Small && "Expected small code model"
) ? static_cast<void> (0) : __assert_fail ("TM.getCodeModel() == CodeModel::Small && \"Expected small code model\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2013, __PRETTY_FUNCTION__))
2013 "Expected small code model")((TM.getCodeModel() == CodeModel::Small && "Expected small code model"
) ? static_cast<void> (0) : __assert_fail ("TM.getCodeModel() == CodeModel::Small && \"Expected small code model\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2013, __PRETTY_FUNCTION__))
;
2014 MachineIRBuilder MIB(I);
2015 auto Op1 = BaseMI->getOperand(1);
2016 auto Op2 = I.getOperand(2);
2017 auto MovAddr = MIB.buildInstr(AArch64::MOVaddr, {I.getOperand(0)}, {})
2018 .addGlobalAddress(Op1.getGlobal(), Op1.getOffset(),
2019 Op1.getTargetFlags())
2020 .addGlobalAddress(Op2.getGlobal(), Op2.getOffset(),
2021 Op2.getTargetFlags());
2022 I.eraseFromParent();
2023 return constrainSelectedInstRegOperands(*MovAddr, TII, TRI, RBI);
2024 }
2025
2026 case TargetOpcode::G_BSWAP: {
2027 // Handle vector types for G_BSWAP directly.
2028 Register DstReg = I.getOperand(0).getReg();
2029 LLT DstTy = MRI.getType(DstReg);
2030
2031 // We should only get vector types here; everything else is handled by the
2032 // importer right now.
2033 if (!DstTy.isVector() || DstTy.getSizeInBits() > 128) {
2034 LLVM_DEBUG(dbgs() << "Dst type for G_BSWAP currently unsupported.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Dst type for G_BSWAP currently unsupported.\n"
; } } while (false)
;
2035 return false;
2036 }
2037
2038 // Only handle 4 and 2 element vectors for now.
2039 // TODO: 16-bit elements.
2040 unsigned NumElts = DstTy.getNumElements();
2041 if (NumElts != 4 && NumElts != 2) {
2042 LLVM_DEBUG(dbgs() << "Unsupported number of elements for G_BSWAP.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported number of elements for G_BSWAP.\n"
; } } while (false)
;
2043 return false;
2044 }
2045
2046 // Choose the correct opcode for the supported types. Right now, that's
2047 // v2s32, v4s32, and v2s64.
2048 unsigned Opc = 0;
2049 unsigned EltSize = DstTy.getElementType().getSizeInBits();
2050 if (EltSize == 32)
2051 Opc = (DstTy.getNumElements() == 2) ? AArch64::REV32v8i8
2052 : AArch64::REV32v16i8;
2053 else if (EltSize == 64)
2054 Opc = AArch64::REV64v16i8;
2055
2056 // We should always get something by the time we get here...
2057 assert(Opc != 0 && "Didn't get an opcode for G_BSWAP?")((Opc != 0 && "Didn't get an opcode for G_BSWAP?") ? static_cast
<void> (0) : __assert_fail ("Opc != 0 && \"Didn't get an opcode for G_BSWAP?\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2057, __PRETTY_FUNCTION__))
;
2058
2059 I.setDesc(TII.get(Opc));
2060 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2061 }
2062
2063 case TargetOpcode::G_FCONSTANT:
2064 case TargetOpcode::G_CONSTANT: {
2065 const bool isFP = Opcode == TargetOpcode::G_FCONSTANT;
2066
2067 const LLT s8 = LLT::scalar(8);
2068 const LLT s16 = LLT::scalar(16);
2069 const LLT s32 = LLT::scalar(32);
2070 const LLT s64 = LLT::scalar(64);
2071 const LLT p0 = LLT::pointer(0, 64);
2072
2073 const Register DefReg = I.getOperand(0).getReg();
2074 const LLT DefTy = MRI.getType(DefReg);
2075 const unsigned DefSize = DefTy.getSizeInBits();
2076 const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
2077
2078 // FIXME: Redundant check, but even less readable when factored out.
2079 if (isFP) {
2080 if (Ty != s32 && Ty != s64) {
2081 LLVM_DEBUG(dbgs() << "Unable to materialize FP " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize FP "
<< Ty << " constant, expected: " << s32 <<
" or " << s64 << '\n'; } } while (false)
2082 << " constant, expected: " << s32 << " or " << s64do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize FP "
<< Ty << " constant, expected: " << s32 <<
" or " << s64 << '\n'; } } while (false)
2083 << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize FP "
<< Ty << " constant, expected: " << s32 <<
" or " << s64 << '\n'; } } while (false)
;
2084 return false;
2085 }
2086
2087 if (RB.getID() != AArch64::FPRRegBankID) {
2088 LLVM_DEBUG(dbgs() << "Unable to materialize FP " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize FP "
<< Ty << " constant on bank: " << RB <<
", expected: FPR\n"; } } while (false)
2089 << " constant on bank: " << RBdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize FP "
<< Ty << " constant on bank: " << RB <<
", expected: FPR\n"; } } while (false)
2090 << ", expected: FPR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize FP "
<< Ty << " constant on bank: " << RB <<
", expected: FPR\n"; } } while (false)
;
2091 return false;
2092 }
2093
2094 // The case when we have 0.0 is covered by tablegen. Reject it here so we
2095 // can be sure tablegen works correctly and isn't rescued by this code.
2096 if (I.getOperand(1).getFPImm()->getValueAPF().isExactlyValue(0.0))
2097 return false;
2098 } else {
2099 // s32 and s64 are covered by tablegen.
2100 if (Ty != p0 && Ty != s8 && Ty != s16) {
2101 LLVM_DEBUG(dbgs() << "Unable to materialize integer " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize integer "
<< Ty << " constant, expected: " << s32 <<
", " << s64 << ", or " << p0 << '\n'
; } } while (false)
2102 << " constant, expected: " << s32 << ", " << s64do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize integer "
<< Ty << " constant, expected: " << s32 <<
", " << s64 << ", or " << p0 << '\n'
; } } while (false)
2103 << ", or " << p0 << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize integer "
<< Ty << " constant, expected: " << s32 <<
", " << s64 << ", or " << p0 << '\n'
; } } while (false)
;
2104 return false;
2105 }
2106
2107 if (RB.getID() != AArch64::GPRRegBankID) {
2108 LLVM_DEBUG(dbgs() << "Unable to materialize integer " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize integer "
<< Ty << " constant on bank: " << RB <<
", expected: GPR\n"; } } while (false)
2109 << " constant on bank: " << RBdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize integer "
<< Ty << " constant on bank: " << RB <<
", expected: GPR\n"; } } while (false)
2110 << ", expected: GPR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize integer "
<< Ty << " constant on bank: " << RB <<
", expected: GPR\n"; } } while (false)
;
2111 return false;
2112 }
2113 }
2114
2115 // We allow G_CONSTANT of types < 32b.
2116 const unsigned MovOpc =
2117 DefSize == 64 ? AArch64::MOVi64imm : AArch64::MOVi32imm;
2118
2119 if (isFP) {
2120 // Either emit a FMOV, or emit a copy to emit a normal mov.
2121 const TargetRegisterClass &GPRRC =
2122 DefSize == 32 ? AArch64::GPR32RegClass : AArch64::GPR64RegClass;
2123 const TargetRegisterClass &FPRRC =
2124 DefSize == 32 ? AArch64::FPR32RegClass : AArch64::FPR64RegClass;
2125
2126 // Can we use a FMOV instruction to represent the immediate?
2127 if (emitFMovForFConstant(I, MRI))
2128 return true;
2129
2130 // For 64b values, emit a constant pool load instead.
2131 if (DefSize == 64) {
2132 auto *FPImm = I.getOperand(1).getFPImm();
2133 MachineIRBuilder MIB(I);
2134 auto *LoadMI = emitLoadFromConstantPool(FPImm, MIB);
2135 if (!LoadMI) {
2136 LLVM_DEBUG(dbgs() << "Failed to load double constant pool entry\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to load double constant pool entry\n"
; } } while (false)
;
2137 return false;
2138 }
2139 MIB.buildCopy({DefReg}, {LoadMI->getOperand(0).getReg()});
2140 I.eraseFromParent();
2141 return RBI.constrainGenericRegister(DefReg, FPRRC, MRI);
2142 }
2143
2144 // Nope. Emit a copy and use a normal mov instead.
2145 const Register DefGPRReg = MRI.createVirtualRegister(&GPRRC);
2146 MachineOperand &RegOp = I.getOperand(0);
2147 RegOp.setReg(DefGPRReg);
2148 MIB.setInsertPt(MIB.getMBB(), std::next(I.getIterator()));
2149 MIB.buildCopy({DefReg}, {DefGPRReg});
2150
2151 if (!RBI.constrainGenericRegister(DefReg, FPRRC, MRI)) {
2152 LLVM_DEBUG(dbgs() << "Failed to constrain G_FCONSTANT def operand\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain G_FCONSTANT def operand\n"
; } } while (false)
;
2153 return false;
2154 }
2155
2156 MachineOperand &ImmOp = I.getOperand(1);
2157 // FIXME: Is going through int64_t always correct?
2158 ImmOp.ChangeToImmediate(
2159 ImmOp.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue());
2160 } else if (I.getOperand(1).isCImm()) {
2161 uint64_t Val = I.getOperand(1).getCImm()->getZExtValue();
2162 I.getOperand(1).ChangeToImmediate(Val);
2163 } else if (I.getOperand(1).isImm()) {
2164 uint64_t Val = I.getOperand(1).getImm();
2165 I.getOperand(1).ChangeToImmediate(Val);
2166 }
2167
2168 I.setDesc(TII.get(MovOpc));
2169 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2170 return true;
2171 }
2172 case TargetOpcode::G_EXTRACT: {
2173 Register DstReg = I.getOperand(0).getReg();
2174 Register SrcReg = I.getOperand(1).getReg();
2175 LLT SrcTy = MRI.getType(SrcReg);
2176 LLT DstTy = MRI.getType(DstReg);
2177 (void)DstTy;
2178 unsigned SrcSize = SrcTy.getSizeInBits();
2179
2180 if (SrcTy.getSizeInBits() > 64) {
2181 // This should be an extract of an s128, which is like a vector extract.
2182 if (SrcTy.getSizeInBits() != 128)
2183 return false;
2184 // Only support extracting 64 bits from an s128 at the moment.
2185 if (DstTy.getSizeInBits() != 64)
2186 return false;
2187
2188 const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
2189 const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
2190 // Check we have the right regbank always.
2191 assert(SrcRB.getID() == AArch64::FPRRegBankID &&((SrcRB.getID() == AArch64::FPRRegBankID && DstRB.getID
() == AArch64::FPRRegBankID && "Wrong extract regbank!"
) ? static_cast<void> (0) : __assert_fail ("SrcRB.getID() == AArch64::FPRRegBankID && DstRB.getID() == AArch64::FPRRegBankID && \"Wrong extract regbank!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2193, __PRETTY_FUNCTION__))
2192 DstRB.getID() == AArch64::FPRRegBankID &&((SrcRB.getID() == AArch64::FPRRegBankID && DstRB.getID
() == AArch64::FPRRegBankID && "Wrong extract regbank!"
) ? static_cast<void> (0) : __assert_fail ("SrcRB.getID() == AArch64::FPRRegBankID && DstRB.getID() == AArch64::FPRRegBankID && \"Wrong extract regbank!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2193, __PRETTY_FUNCTION__))
2193 "Wrong extract regbank!")((SrcRB.getID() == AArch64::FPRRegBankID && DstRB.getID
() == AArch64::FPRRegBankID && "Wrong extract regbank!"
) ? static_cast<void> (0) : __assert_fail ("SrcRB.getID() == AArch64::FPRRegBankID && DstRB.getID() == AArch64::FPRRegBankID && \"Wrong extract regbank!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2193, __PRETTY_FUNCTION__))
;
2194 (void)SrcRB;
2195
2196 // Emit the same code as a vector extract.
2197 // Offset must be a multiple of 64.
2198 unsigned Offset = I.getOperand(2).getImm();
2199 if (Offset % 64 != 0)
2200 return false;
2201 unsigned LaneIdx = Offset / 64;
2202 MachineIRBuilder MIB(I);
2203 MachineInstr *Extract = emitExtractVectorElt(
2204 DstReg, DstRB, LLT::scalar(64), SrcReg, LaneIdx, MIB);
2205 if (!Extract)
2206 return false;
2207 I.eraseFromParent();
2208 return true;
2209 }
2210
2211 I.setDesc(TII.get(SrcSize == 64 ? AArch64::UBFMXri : AArch64::UBFMWri));
2212 MachineInstrBuilder(MF, I).addImm(I.getOperand(2).getImm() +
2213 Ty.getSizeInBits() - 1);
2214
2215 if (SrcSize < 64) {
2216 assert(SrcSize == 32 && DstTy.getSizeInBits() == 16 &&((SrcSize == 32 && DstTy.getSizeInBits() == 16 &&
"unexpected G_EXTRACT types") ? static_cast<void> (0) :
__assert_fail ("SrcSize == 32 && DstTy.getSizeInBits() == 16 && \"unexpected G_EXTRACT types\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2217, __PRETTY_FUNCTION__))
2217 "unexpected G_EXTRACT types")((SrcSize == 32 && DstTy.getSizeInBits() == 16 &&
"unexpected G_EXTRACT types") ? static_cast<void> (0) :
__assert_fail ("SrcSize == 32 && DstTy.getSizeInBits() == 16 && \"unexpected G_EXTRACT types\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2217, __PRETTY_FUNCTION__))
;
2218 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2219 }
2220
2221 DstReg = MRI.createGenericVirtualRegister(LLT::scalar(64));
2222 MIB.setInsertPt(MIB.getMBB(), std::next(I.getIterator()));
2223 MIB.buildInstr(TargetOpcode::COPY, {I.getOperand(0).getReg()}, {})
2224 .addReg(DstReg, 0, AArch64::sub_32);
2225 RBI.constrainGenericRegister(I.getOperand(0).getReg(),
2226 AArch64::GPR32RegClass, MRI);
2227 I.getOperand(0).setReg(DstReg);
2228
2229 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2230 }
2231
2232 case TargetOpcode::G_INSERT: {
2233 LLT SrcTy = MRI.getType(I.getOperand(2).getReg());
2234 LLT DstTy = MRI.getType(I.getOperand(0).getReg());
2235 unsigned DstSize = DstTy.getSizeInBits();
2236 // Larger inserts are vectors, same-size ones should be something else by
2237 // now (split up or turned into COPYs).
2238 if (Ty.getSizeInBits() > 64 || SrcTy.getSizeInBits() > 32)
2239 return false;
2240
2241 I.setDesc(TII.get(DstSize == 64 ? AArch64::BFMXri : AArch64::BFMWri));
2242 unsigned LSB = I.getOperand(3).getImm();
2243 unsigned Width = MRI.getType(I.getOperand(2).getReg()).getSizeInBits();
2244 I.getOperand(3).setImm((DstSize - LSB) % DstSize);
2245 MachineInstrBuilder(MF, I).addImm(Width - 1);
2246
2247 if (DstSize < 64) {
2248 assert(DstSize == 32 && SrcTy.getSizeInBits() == 16 &&((DstSize == 32 && SrcTy.getSizeInBits() == 16 &&
"unexpected G_INSERT types") ? static_cast<void> (0) :
__assert_fail ("DstSize == 32 && SrcTy.getSizeInBits() == 16 && \"unexpected G_INSERT types\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2249, __PRETTY_FUNCTION__))
2249 "unexpected G_INSERT types")((DstSize == 32 && SrcTy.getSizeInBits() == 16 &&
"unexpected G_INSERT types") ? static_cast<void> (0) :
__assert_fail ("DstSize == 32 && SrcTy.getSizeInBits() == 16 && \"unexpected G_INSERT types\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2249, __PRETTY_FUNCTION__))
;
2250 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2251 }
2252
2253 Register SrcReg = MRI.createGenericVirtualRegister(LLT::scalar(64));
2254 BuildMI(MBB, I.getIterator(), I.getDebugLoc(),
2255 TII.get(AArch64::SUBREG_TO_REG))
2256 .addDef(SrcReg)
2257 .addImm(0)
2258 .addUse(I.getOperand(2).getReg())
2259 .addImm(AArch64::sub_32);
2260 RBI.constrainGenericRegister(I.getOperand(2).getReg(),
2261 AArch64::GPR32RegClass, MRI);
2262 I.getOperand(2).setReg(SrcReg);
2263
2264 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2265 }
2266 case TargetOpcode::G_FRAME_INDEX: {
2267 // allocas and G_FRAME_INDEX are only supported in addrspace(0).
2268 if (Ty != LLT::pointer(0, 64)) {
2269 LLVM_DEBUG(dbgs() << "G_FRAME_INDEX pointer has type: " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_FRAME_INDEX pointer has type: "
<< Ty << ", expected: " << LLT::pointer(0,
64) << '\n'; } } while (false)
2270 << ", expected: " << LLT::pointer(0, 64) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_FRAME_INDEX pointer has type: "
<< Ty << ", expected: " << LLT::pointer(0,
64) << '\n'; } } while (false)
;
2271 return false;
2272 }
2273 I.setDesc(TII.get(AArch64::ADDXri));
2274
2275 // MOs for a #0 shifted immediate.
2276 I.addOperand(MachineOperand::CreateImm(0));
2277 I.addOperand(MachineOperand::CreateImm(0));
2278
2279 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2280 }
2281
2282 case TargetOpcode::G_GLOBAL_VALUE: {
2283 auto GV = I.getOperand(1).getGlobal();
2284 if (GV->isThreadLocal())
2285 return selectTLSGlobalValue(I, MRI);
2286
2287 unsigned OpFlags = STI.ClassifyGlobalReference(GV, TM);
2288 if (OpFlags & AArch64II::MO_GOT) {
2289 I.setDesc(TII.get(AArch64::LOADgot));
2290 I.getOperand(1).setTargetFlags(OpFlags);
2291 } else if (TM.getCodeModel() == CodeModel::Large) {
2292 // Materialize the global using movz/movk instructions.
2293 materializeLargeCMVal(I, GV, OpFlags);
2294 I.eraseFromParent();
2295 return true;
2296 } else if (TM.getCodeModel() == CodeModel::Tiny) {
2297 I.setDesc(TII.get(AArch64::ADR));
2298 I.getOperand(1).setTargetFlags(OpFlags);
2299 } else {
2300 I.setDesc(TII.get(AArch64::MOVaddr));
2301 I.getOperand(1).setTargetFlags(OpFlags | AArch64II::MO_PAGE);
2302 MachineInstrBuilder MIB(MF, I);
2303 MIB.addGlobalAddress(GV, I.getOperand(1).getOffset(),
2304 OpFlags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
2305 }
2306 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2307 }
2308
2309 case TargetOpcode::G_ZEXTLOAD:
2310 case TargetOpcode::G_LOAD:
2311 case TargetOpcode::G_STORE: {
2312 bool IsZExtLoad = I.getOpcode() == TargetOpcode::G_ZEXTLOAD;
2313 MachineIRBuilder MIB(I);
2314
2315 LLT PtrTy = MRI.getType(I.getOperand(1).getReg());
2316
2317 if (PtrTy != LLT::pointer(0, 64)) {
2318 LLVM_DEBUG(dbgs() << "Load/Store pointer has type: " << PtrTydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Load/Store pointer has type: "
<< PtrTy << ", expected: " << LLT::pointer
(0, 64) << '\n'; } } while (false)
2319 << ", expected: " << LLT::pointer(0, 64) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Load/Store pointer has type: "
<< PtrTy << ", expected: " << LLT::pointer
(0, 64) << '\n'; } } while (false)
;
2320 return false;
2321 }
2322
2323 auto &MemOp = **I.memoperands_begin();
2324 uint64_t MemSizeInBytes = MemOp.getSize();
2325 if (MemOp.isAtomic()) {
2326 // For now we just support s8 acquire loads to be able to compile stack
2327 // protector code.
2328 if (MemOp.getOrdering() == AtomicOrdering::Acquire &&
2329 MemSizeInBytes == 1) {
2330 I.setDesc(TII.get(AArch64::LDARB));
2331 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2332 }
2333 LLVM_DEBUG(dbgs() << "Atomic load/store not fully supported yet\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Atomic load/store not fully supported yet\n"
; } } while (false)
;
2334 return false;
2335 }
2336 unsigned MemSizeInBits = MemSizeInBytes * 8;
2337
2338#ifndef NDEBUG
2339 const Register PtrReg = I.getOperand(1).getReg();
2340 const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, MRI, TRI);
2341 // Sanity-check the pointer register.
2342 assert(PtrRB.getID() == AArch64::GPRRegBankID &&((PtrRB.getID() == AArch64::GPRRegBankID && "Load/Store pointer operand isn't a GPR"
) ? static_cast<void> (0) : __assert_fail ("PtrRB.getID() == AArch64::GPRRegBankID && \"Load/Store pointer operand isn't a GPR\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2343, __PRETTY_FUNCTION__))
2343 "Load/Store pointer operand isn't a GPR")((PtrRB.getID() == AArch64::GPRRegBankID && "Load/Store pointer operand isn't a GPR"
) ? static_cast<void> (0) : __assert_fail ("PtrRB.getID() == AArch64::GPRRegBankID && \"Load/Store pointer operand isn't a GPR\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2343, __PRETTY_FUNCTION__))
;
2344 assert(MRI.getType(PtrReg).isPointer() &&((MRI.getType(PtrReg).isPointer() && "Load/Store pointer operand isn't a pointer"
) ? static_cast<void> (0) : __assert_fail ("MRI.getType(PtrReg).isPointer() && \"Load/Store pointer operand isn't a pointer\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2345, __PRETTY_FUNCTION__))
2345 "Load/Store pointer operand isn't a pointer")((MRI.getType(PtrReg).isPointer() && "Load/Store pointer operand isn't a pointer"
) ? static_cast<void> (0) : __assert_fail ("MRI.getType(PtrReg).isPointer() && \"Load/Store pointer operand isn't a pointer\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2345, __PRETTY_FUNCTION__))
;
2346#endif
2347
2348 const Register ValReg = I.getOperand(0).getReg();
2349 const RegisterBank &RB = *RBI.getRegBank(ValReg, MRI, TRI);
2350
2351 // Helper lambda for partially selecting I. Either returns the original
2352 // instruction with an updated opcode, or a new instruction.
2353 auto SelectLoadStoreAddressingMode = [&]() -> MachineInstr * {
2354 bool IsStore = I.getOpcode() == TargetOpcode::G_STORE;
2355 const unsigned NewOpc =
2356 selectLoadStoreUIOp(I.getOpcode(), RB.getID(), MemSizeInBits);
2357 if (NewOpc == I.getOpcode())
2358 return nullptr;
2359 // Check if we can fold anything into the addressing mode.
2360 auto AddrModeFns =
2361 selectAddrModeIndexed(I.getOperand(1), MemSizeInBytes);
2362 if (!AddrModeFns) {
2363 // Can't fold anything. Use the original instruction.
2364 I.setDesc(TII.get(NewOpc));
2365 I.addOperand(MachineOperand::CreateImm(0));
2366 return &I;
2367 }
2368
2369 // Folded something. Create a new instruction and return it.
2370 auto NewInst = MIB.buildInstr(NewOpc, {}, {}, I.getFlags());
2371 IsStore ? NewInst.addUse(ValReg) : NewInst.addDef(ValReg);
2372 NewInst.cloneMemRefs(I);
2373 for (auto &Fn : *AddrModeFns)
2374 Fn(NewInst);
2375 I.eraseFromParent();
2376 return &*NewInst;
2377 };
2378
2379 MachineInstr *LoadStore = SelectLoadStoreAddressingMode();
2380 if (!LoadStore)
2381 return false;
2382
2383 // If we're storing a 0, use WZR/XZR.
2384 if (Opcode == TargetOpcode::G_STORE) {
2385 auto CVal = getConstantVRegValWithLookThrough(
2386 LoadStore->getOperand(0).getReg(), MRI, /*LookThroughInstrs = */ true,
2387 /*HandleFConstants = */ false);
2388 if (CVal && CVal->Value == 0) {
2389 switch (LoadStore->getOpcode()) {
2390 case AArch64::STRWui:
2391 case AArch64::STRHHui:
2392 case AArch64::STRBBui:
2393 LoadStore->getOperand(0).setReg(AArch64::WZR);
2394 break;
2395 case AArch64::STRXui:
2396 LoadStore->getOperand(0).setReg(AArch64::XZR);
2397 break;
2398 }
2399 }
2400 }
2401
2402 if (IsZExtLoad) {
2403 // The zextload from a smaller type to i32 should be handled by the
2404 // importer.
2405 if (MRI.getType(LoadStore->getOperand(0).getReg()).getSizeInBits() != 64)
2406 return false;
2407 // If we have a ZEXTLOAD then change the load's type to be a narrower reg
2408 // and zero_extend with SUBREG_TO_REG.
2409 Register LdReg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
2410 Register DstReg = LoadStore->getOperand(0).getReg();
2411 LoadStore->getOperand(0).setReg(LdReg);
2412
2413 MIB.setInsertPt(MIB.getMBB(), std::next(LoadStore->getIterator()));
2414 MIB.buildInstr(AArch64::SUBREG_TO_REG, {DstReg}, {})
2415 .addImm(0)
2416 .addUse(LdReg)
2417 .addImm(AArch64::sub_32);
2418 constrainSelectedInstRegOperands(*LoadStore, TII, TRI, RBI);
2419 return RBI.constrainGenericRegister(DstReg, AArch64::GPR64allRegClass,
2420 MRI);
2421 }
2422 return constrainSelectedInstRegOperands(*LoadStore, TII, TRI, RBI);
2423 }
2424
2425 case TargetOpcode::G_SMULH:
2426 case TargetOpcode::G_UMULH: {
2427 // Reject the various things we don't support yet.
2428 if (unsupportedBinOp(I, RBI, MRI, TRI))
2429 return false;
2430
2431 const Register DefReg = I.getOperand(0).getReg();
2432 const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
2433
2434 if (RB.getID() != AArch64::GPRRegBankID) {
2435 LLVM_DEBUG(dbgs() << "G_[SU]MULH on bank: " << RB << ", expected: GPR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_[SU]MULH on bank: " <<
RB << ", expected: GPR\n"; } } while (false)
;
2436 return false;
2437 }
2438
2439 if (Ty != LLT::scalar(64)) {
2440 LLVM_DEBUG(dbgs() << "G_[SU]MULH has type: " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_[SU]MULH has type: " <<
Ty << ", expected: " << LLT::scalar(64) <<
'\n'; } } while (false)
2441 << ", expected: " << LLT::scalar(64) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_[SU]MULH has type: " <<
Ty << ", expected: " << LLT::scalar(64) <<
'\n'; } } while (false)
;
2442 return false;
2443 }
2444
2445 unsigned NewOpc = I.getOpcode() == TargetOpcode::G_SMULH ? AArch64::SMULHrr
2446 : AArch64::UMULHrr;
2447 I.setDesc(TII.get(NewOpc));
2448
2449 // Now that we selected an opcode, we need to constrain the register
2450 // operands to use appropriate classes.
2451 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2452 }
2453 case TargetOpcode::G_FADD:
2454 case TargetOpcode::G_FSUB:
2455 case TargetOpcode::G_FMUL:
2456 case TargetOpcode::G_FDIV:
2457
2458 case TargetOpcode::G_ASHR:
2459 if (MRI.getType(I.getOperand(0).getReg()).isVector())
2460 return selectVectorASHR(I, MRI);
2461 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2462 case TargetOpcode::G_SHL:
2463 if (Opcode == TargetOpcode::G_SHL &&
2464 MRI.getType(I.getOperand(0).getReg()).isVector())
2465 return selectVectorSHL(I, MRI);
2466 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2467 case TargetOpcode::G_OR:
2468 case TargetOpcode::G_LSHR: {
2469 // Reject the various things we don't support yet.
2470 if (unsupportedBinOp(I, RBI, MRI, TRI))
2471 return false;
2472
2473 const unsigned OpSize = Ty.getSizeInBits();
2474
2475 const Register DefReg = I.getOperand(0).getReg();
2476 const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
2477
2478 const unsigned NewOpc = selectBinaryOp(I.getOpcode(), RB.getID(), OpSize);
2479 if (NewOpc == I.getOpcode())
2480 return false;
2481
2482 I.setDesc(TII.get(NewOpc));
2483 // FIXME: Should the type be always reset in setDesc?
2484
2485 // Now that we selected an opcode, we need to constrain the register
2486 // operands to use appropriate classes.
2487 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2488 }
2489
2490 case TargetOpcode::G_PTR_ADD: {
2491 MachineIRBuilder MIRBuilder(I);
2492 emitADD(I.getOperand(0).getReg(), I.getOperand(1), I.getOperand(2),
2493 MIRBuilder);
2494 I.eraseFromParent();
2495 return true;
2496 }
2497 case TargetOpcode::G_UADDO: {
2498 // TODO: Support other types.
2499 unsigned OpSize = Ty.getSizeInBits();
2500 if (OpSize != 32 && OpSize != 64) {
2501 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_UADDO currently only supported for 32 and 64 b types.\n"
; } } while (false)
2502 dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_UADDO currently only supported for 32 and 64 b types.\n"
; } } while (false)
2503 << "G_UADDO currently only supported for 32 and 64 b types.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_UADDO currently only supported for 32 and 64 b types.\n"
; } } while (false)
;
2504 return false;
2505 }
2506
2507 // TODO: Support vectors.
2508 if (Ty.isVector()) {
2509 LLVM_DEBUG(dbgs() << "G_UADDO currently only supported for scalars.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_UADDO currently only supported for scalars.\n"
; } } while (false)
;
2510 return false;
2511 }
2512
2513 // Add and set the set condition flag.
2514 MachineIRBuilder MIRBuilder(I);
2515 emitADDS(I.getOperand(0).getReg(), I.getOperand(2), I.getOperand(3),
2516 MIRBuilder);
2517
2518 // Now, put the overflow result in the register given by the first operand
2519 // to the G_UADDO. CSINC increments the result when the predicate is false,
2520 // so to get the increment when it's true, we need to use the inverse. In
2521 // this case, we want to increment when carry is set.
2522 auto CsetMI = MIRBuilder
2523 .buildInstr(AArch64::CSINCWr, {I.getOperand(1).getReg()},
2524 {Register(AArch64::WZR), Register(AArch64::WZR)})
2525 .addImm(getInvertedCondCode(AArch64CC::HS));
2526 constrainSelectedInstRegOperands(*CsetMI, TII, TRI, RBI);
2527 I.eraseFromParent();
2528 return true;
2529 }
2530
2531 case TargetOpcode::G_PTRMASK: {
2532 Register MaskReg = I.getOperand(2).getReg();
2533 Optional<int64_t> MaskVal = getConstantVRegVal(MaskReg, MRI);
2534 // TODO: Implement arbitrary cases
2535 if (!MaskVal || !isShiftedMask_64(*MaskVal))
2536 return false;
2537
2538 uint64_t Mask = *MaskVal;
2539 I.setDesc(TII.get(AArch64::ANDXri));
2540 I.getOperand(2).ChangeToImmediate(
2541 AArch64_AM::encodeLogicalImmediate(Mask, 64));
2542
2543 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2544 }
2545 case TargetOpcode::G_PTRTOINT:
2546 case TargetOpcode::G_TRUNC: {
2547 const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
2548 const LLT SrcTy = MRI.getType(I.getOperand(1).getReg());
2549
2550 const Register DstReg = I.getOperand(0).getReg();
2551 const Register SrcReg = I.getOperand(1).getReg();
2552
2553 const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
2554 const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
2555
2556 if (DstRB.getID() != SrcRB.getID()) {
2557 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_TRUNC/G_PTRTOINT input/output on different banks\n"
; } } while (false)
2558 dbgs() << "G_TRUNC/G_PTRTOINT input/output on different banks\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_TRUNC/G_PTRTOINT input/output on different banks\n"
; } } while (false)
;
2559 return false;
2560 }
2561
2562 if (DstRB.getID() == AArch64::GPRRegBankID) {
2563 const TargetRegisterClass *DstRC =
2564 getRegClassForTypeOnBank(DstTy, DstRB, RBI);
2565 if (!DstRC)
2566 return false;
2567
2568 const TargetRegisterClass *SrcRC =
2569 getRegClassForTypeOnBank(SrcTy, SrcRB, RBI);
2570 if (!SrcRC)
2571 return false;
2572
2573 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
2574 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
2575 LLVM_DEBUG(dbgs() << "Failed to constrain G_TRUNC/G_PTRTOINT\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain G_TRUNC/G_PTRTOINT\n"
; } } while (false)
;
2576 return false;
2577 }
2578
2579 if (DstRC == SrcRC) {
2580 // Nothing to be done
2581 } else if (Opcode == TargetOpcode::G_TRUNC && DstTy == LLT::scalar(32) &&
2582 SrcTy == LLT::scalar(64)) {
2583 llvm_unreachable("TableGen can import this case")::llvm::llvm_unreachable_internal("TableGen can import this case"
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2583)
;
2584 return false;
2585 } else if (DstRC == &AArch64::GPR32RegClass &&
2586 SrcRC == &AArch64::GPR64RegClass) {
2587 I.getOperand(1).setSubReg(AArch64::sub_32);
2588 } else {
2589 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n"
; } } while (false)
2590 dbgs() << "Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n"
; } } while (false)
;
2591 return false;
2592 }
2593
2594 I.setDesc(TII.get(TargetOpcode::COPY));
2595 return true;
2596 } else if (DstRB.getID() == AArch64::FPRRegBankID) {
2597 if (DstTy == LLT::vector(4, 16) && SrcTy == LLT::vector(4, 32)) {
2598 I.setDesc(TII.get(AArch64::XTNv4i16));
2599 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2600 return true;
2601 }
2602
2603 if (!SrcTy.isVector() && SrcTy.getSizeInBits() == 128) {
2604 MachineIRBuilder MIB(I);
2605 MachineInstr *Extract = emitExtractVectorElt(
2606 DstReg, DstRB, LLT::scalar(DstTy.getSizeInBits()), SrcReg, 0, MIB);
2607 if (!Extract)
2608 return false;
2609 I.eraseFromParent();
2610 return true;
2611 }
2612
2613 // We might have a vector G_PTRTOINT, in which case just emit a COPY.
2614 if (Opcode == TargetOpcode::G_PTRTOINT) {
2615 assert(DstTy.isVector() && "Expected an FPR ptrtoint to be a vector")((DstTy.isVector() && "Expected an FPR ptrtoint to be a vector"
) ? static_cast<void> (0) : __assert_fail ("DstTy.isVector() && \"Expected an FPR ptrtoint to be a vector\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2615, __PRETTY_FUNCTION__))
;
2616 I.setDesc(TII.get(TargetOpcode::COPY));
2617 return true;
2618 }
2619 }
2620
2621 return false;
2622 }
2623
2624 case TargetOpcode::G_ANYEXT: {
2625 const Register DstReg = I.getOperand(0).getReg();
2626 const Register SrcReg = I.getOperand(1).getReg();
2627
2628 const RegisterBank &RBDst = *RBI.getRegBank(DstReg, MRI, TRI);
2629 if (RBDst.getID() != AArch64::GPRRegBankID) {
2630 LLVM_DEBUG(dbgs() << "G_ANYEXT on bank: " << RBDstdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT on bank: " <<
RBDst << ", expected: GPR\n"; } } while (false)
2631 << ", expected: GPR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT on bank: " <<
RBDst << ", expected: GPR\n"; } } while (false)
;
2632 return false;
2633 }
2634
2635 const RegisterBank &RBSrc = *RBI.getRegBank(SrcReg, MRI, TRI);
2636 if (RBSrc.getID() != AArch64::GPRRegBankID) {
2637 LLVM_DEBUG(dbgs() << "G_ANYEXT on bank: " << RBSrcdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT on bank: " <<
RBSrc << ", expected: GPR\n"; } } while (false)
2638 << ", expected: GPR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT on bank: " <<
RBSrc << ", expected: GPR\n"; } } while (false)
;
2639 return false;
2640 }
2641
2642 const unsigned DstSize = MRI.getType(DstReg).getSizeInBits();
2643
2644 if (DstSize == 0) {
2645 LLVM_DEBUG(dbgs() << "G_ANYEXT operand has no size, not a gvreg?\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT operand has no size, not a gvreg?\n"
; } } while (false)
;
2646 return false;
2647 }
2648
2649 if (DstSize != 64 && DstSize > 32) {
2650 LLVM_DEBUG(dbgs() << "G_ANYEXT to size: " << DstSizedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT to size: " <<
DstSize << ", expected: 32 or 64\n"; } } while (false)
2651 << ", expected: 32 or 64\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT to size: " <<
DstSize << ", expected: 32 or 64\n"; } } while (false)
;
2652 return false;
2653 }
2654 // At this point G_ANYEXT is just like a plain COPY, but we need
2655 // to explicitly form the 64-bit value if any.
2656 if (DstSize > 32) {
2657 Register ExtSrc = MRI.createVirtualRegister(&AArch64::GPR64allRegClass);
2658 BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::SUBREG_TO_REG))
2659 .addDef(ExtSrc)
2660 .addImm(0)
2661 .addUse(SrcReg)
2662 .addImm(AArch64::sub_32);
2663 I.getOperand(1).setReg(ExtSrc);
2664 }
2665 return selectCopy(I, TII, MRI, TRI, RBI);
2666 }
2667
2668 case TargetOpcode::G_ZEXT:
2669 case TargetOpcode::G_SEXT_INREG:
2670 case TargetOpcode::G_SEXT: {
2671 unsigned Opcode = I.getOpcode();
2672 const bool IsSigned = Opcode != TargetOpcode::G_ZEXT;
2673 const Register DefReg = I.getOperand(0).getReg();
2674 Register SrcReg = I.getOperand(1).getReg();
2675 const LLT DstTy = MRI.getType(DefReg);
2676 const LLT SrcTy = MRI.getType(SrcReg);
2677 unsigned DstSize = DstTy.getSizeInBits();
2678 unsigned SrcSize = SrcTy.getSizeInBits();
2679
2680 // SEXT_INREG has the same src reg size as dst, the size of the value to be
2681 // extended is encoded in the imm.
2682 if (Opcode == TargetOpcode::G_SEXT_INREG)
2683 SrcSize = I.getOperand(2).getImm();
2684
2685 if (DstTy.isVector())
2686 return false; // Should be handled by imported patterns.
2687
2688 assert((*RBI.getRegBank(DefReg, MRI, TRI)).getID() ==(((*RBI.getRegBank(DefReg, MRI, TRI)).getID() == AArch64::GPRRegBankID
&& "Unexpected ext regbank") ? static_cast<void>
(0) : __assert_fail ("(*RBI.getRegBank(DefReg, MRI, TRI)).getID() == AArch64::GPRRegBankID && \"Unexpected ext regbank\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2690, __PRETTY_FUNCTION__))
2689 AArch64::GPRRegBankID &&(((*RBI.getRegBank(DefReg, MRI, TRI)).getID() == AArch64::GPRRegBankID
&& "Unexpected ext regbank") ? static_cast<void>
(0) : __assert_fail ("(*RBI.getRegBank(DefReg, MRI, TRI)).getID() == AArch64::GPRRegBankID && \"Unexpected ext regbank\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2690, __PRETTY_FUNCTION__))
2690 "Unexpected ext regbank")(((*RBI.getRegBank(DefReg, MRI, TRI)).getID() == AArch64::GPRRegBankID
&& "Unexpected ext regbank") ? static_cast<void>
(0) : __assert_fail ("(*RBI.getRegBank(DefReg, MRI, TRI)).getID() == AArch64::GPRRegBankID && \"Unexpected ext regbank\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2690, __PRETTY_FUNCTION__))
;
2691
2692 MachineIRBuilder MIB(I);
2693 MachineInstr *ExtI;
2694
2695 // First check if we're extending the result of a load which has a dest type
2696 // smaller than 32 bits, then this zext is redundant. GPR32 is the smallest
2697 // GPR register on AArch64 and all loads which are smaller automatically
2698 // zero-extend the upper bits. E.g.
2699 // %v(s8) = G_LOAD %p, :: (load 1)
2700 // %v2(s32) = G_ZEXT %v(s8)
2701 if (!IsSigned) {
2702 auto *LoadMI = getOpcodeDef(TargetOpcode::G_LOAD, SrcReg, MRI);
2703 bool IsGPR =
2704 RBI.getRegBank(SrcReg, MRI, TRI)->getID() == AArch64::GPRRegBankID;
2705 if (LoadMI && IsGPR) {
2706 const MachineMemOperand *MemOp = *LoadMI->memoperands_begin();
2707 unsigned BytesLoaded = MemOp->getSize();
2708 if (BytesLoaded < 4 && SrcTy.getSizeInBytes() == BytesLoaded)
2709 return selectCopy(I, TII, MRI, TRI, RBI);
2710 }
2711
2712 // If we are zero extending from 32 bits to 64 bits, it's possible that
2713 // the instruction implicitly does the zero extend for us. In that case,
2714 // we can just emit a SUBREG_TO_REG.
2715 if (IsGPR && SrcSize == 32 && DstSize == 64) {
2716 // Unlike with the G_LOAD case, we don't want to look through copies
2717 // here.
2718 MachineInstr *Def = MRI.getVRegDef(SrcReg);
2719 if (Def && isDef32(*Def)) {
2720 MIB.buildInstr(AArch64::SUBREG_TO_REG, {DefReg}, {})
2721 .addImm(0)
2722 .addUse(SrcReg)
2723 .addImm(AArch64::sub_32);
2724
2725 if (!RBI.constrainGenericRegister(DefReg, AArch64::GPR64RegClass,
2726 MRI)) {
2727 LLVM_DEBUG(dbgs() << "Failed to constrain G_ZEXT destination\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain G_ZEXT destination\n"
; } } while (false)
;
2728 return false;
2729 }
2730
2731 if (!RBI.constrainGenericRegister(SrcReg, AArch64::GPR32RegClass,
2732 MRI)) {
2733 LLVM_DEBUG(dbgs() << "Failed to constrain G_ZEXT source\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain G_ZEXT source\n"
; } } while (false)
;
2734 return false;
2735 }
2736
2737 I.eraseFromParent();
2738 return true;
2739 }
2740 }
2741 }
2742
2743 if (DstSize == 64) {
2744 if (Opcode != TargetOpcode::G_SEXT_INREG) {
2745 // FIXME: Can we avoid manually doing this?
2746 if (!RBI.constrainGenericRegister(SrcReg, AArch64::GPR32RegClass,
2747 MRI)) {
2748 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(Opcode)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain " <<
TII.getName(Opcode) << " operand\n"; } } while (false)
2749 << " operand\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain " <<
TII.getName(Opcode) << " operand\n"; } } while (false)
;
2750 return false;
2751 }
2752 SrcReg = MIB.buildInstr(AArch64::SUBREG_TO_REG,
2753 {&AArch64::GPR64RegClass}, {})
2754 .addImm(0)
2755 .addUse(SrcReg)
2756 .addImm(AArch64::sub_32)
2757 .getReg(0);
2758 }
2759
2760 ExtI = MIB.buildInstr(IsSigned ? AArch64::SBFMXri : AArch64::UBFMXri,
2761 {DefReg}, {SrcReg})
2762 .addImm(0)
2763 .addImm(SrcSize - 1);
2764 } else if (DstSize <= 32) {
2765 ExtI = MIB.buildInstr(IsSigned ? AArch64::SBFMWri : AArch64::UBFMWri,
2766 {DefReg}, {SrcReg})
2767 .addImm(0)
2768 .addImm(SrcSize - 1);
2769 } else {
2770 return false;
2771 }
2772
2773 constrainSelectedInstRegOperands(*ExtI, TII, TRI, RBI);
2774 I.eraseFromParent();
2775 return true;
2776 }
2777
2778 case TargetOpcode::G_SITOFP:
2779 case TargetOpcode::G_UITOFP:
2780 case TargetOpcode::G_FPTOSI:
2781 case TargetOpcode::G_FPTOUI: {
2782 const LLT DstTy = MRI.getType(I.getOperand(0).getReg()),
2783 SrcTy = MRI.getType(I.getOperand(1).getReg());
2784 const unsigned NewOpc = selectFPConvOpc(Opcode, DstTy, SrcTy);
2785 if (NewOpc == Opcode)
2786 return false;
2787
2788 I.setDesc(TII.get(NewOpc));
2789 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2790
2791 return true;
2792 }
2793
2794 case TargetOpcode::G_FREEZE:
2795 return selectCopy(I, TII, MRI, TRI, RBI);
2796
2797 case TargetOpcode::G_INTTOPTR:
2798 // The importer is currently unable to import pointer types since they
2799 // didn't exist in SelectionDAG.
2800 return selectCopy(I, TII, MRI, TRI, RBI);
2801
2802 case TargetOpcode::G_BITCAST:
2803 // Imported SelectionDAG rules can handle every bitcast except those that
2804 // bitcast from a type to the same type. Ideally, these shouldn't occur
2805 // but we might not run an optimizer that deletes them. The other exception
2806 // is bitcasts involving pointer types, as SelectionDAG has no knowledge
2807 // of them.
2808 return selectCopy(I, TII, MRI, TRI, RBI);
2809
2810 case TargetOpcode::G_SELECT: {
2811 if (MRI.getType(I.getOperand(1).getReg()) != LLT::scalar(1)) {
2812 LLVM_DEBUG(dbgs() << "G_SELECT cond has type: " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_SELECT cond has type: "
<< Ty << ", expected: " << LLT::scalar(1) <<
'\n'; } } while (false)
2813 << ", expected: " << LLT::scalar(1) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_SELECT cond has type: "
<< Ty << ", expected: " << LLT::scalar(1) <<
'\n'; } } while (false)
;
2814 return false;
2815 }
2816
2817 const Register CondReg = I.getOperand(1).getReg();
2818 const Register TReg = I.getOperand(2).getReg();
2819 const Register FReg = I.getOperand(3).getReg();
2820
2821 if (tryOptSelect(I))
2822 return true;
2823
2824 Register CSelOpc = selectSelectOpc(I, MRI, RBI);
2825 MachineInstr &TstMI =
2826 *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::ANDSWri))
2827 .addDef(AArch64::WZR)
2828 .addUse(CondReg)
2829 .addImm(AArch64_AM::encodeLogicalImmediate(1, 32));
2830
2831 MachineInstr &CSelMI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CSelOpc))
2832 .addDef(I.getOperand(0).getReg())
2833 .addUse(TReg)
2834 .addUse(FReg)
2835 .addImm(AArch64CC::NE);
2836
2837 constrainSelectedInstRegOperands(TstMI, TII, TRI, RBI);
2838 constrainSelectedInstRegOperands(CSelMI, TII, TRI, RBI);
2839
2840 I.eraseFromParent();
2841 return true;
2842 }
2843 case TargetOpcode::G_ICMP: {
2844 if (Ty.isVector())
2845 return selectVectorICmp(I, MRI);
2846
2847 if (Ty != LLT::scalar(32)) {
2848 LLVM_DEBUG(dbgs() << "G_ICMP result has type: " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ICMP result has type: "
<< Ty << ", expected: " << LLT::scalar(32)
<< '\n'; } } while (false)
2849 << ", expected: " << LLT::scalar(32) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ICMP result has type: "
<< Ty << ", expected: " << LLT::scalar(32)
<< '\n'; } } while (false)
;
2850 return false;
2851 }
2852
2853 MachineIRBuilder MIRBuilder(I);
2854 MachineInstr *Cmp;
2855 CmpInst::Predicate Pred;
2856 std::tie(Cmp, Pred) = emitIntegerCompare(I.getOperand(2), I.getOperand(3),
2857 I.getOperand(1), MIRBuilder);
2858 if (!Cmp)
2859 return false;
2860 emitCSetForICMP(I.getOperand(0).getReg(), Pred, MIRBuilder);
2861 I.eraseFromParent();
2862 return true;
2863 }
2864
2865 case TargetOpcode::G_FCMP: {
2866 if (Ty != LLT::scalar(32)) {
2867 LLVM_DEBUG(dbgs() << "G_FCMP result has type: " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_FCMP result has type: "
<< Ty << ", expected: " << LLT::scalar(32)
<< '\n'; } } while (false)
2868 << ", expected: " << LLT::scalar(32) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_FCMP result has type: "
<< Ty << ", expected: " << LLT::scalar(32)
<< '\n'; } } while (false)
;
2869 return false;
2870 }
2871
2872 unsigned CmpOpc = selectFCMPOpc(I, MRI);
2873 if (!CmpOpc)
2874 return false;
2875
2876 // FIXME: regbank
2877
2878 AArch64CC::CondCode CC1, CC2;
2879 changeFCMPPredToAArch64CC(
2880 (CmpInst::Predicate)I.getOperand(1).getPredicate(), CC1, CC2);
2881
2882 // Partially build the compare. Decide if we need to add a use for the
2883 // third operand based off whether or not we're comparing against 0.0.
2884 auto CmpMI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(CmpOpc))
2885 .addUse(I.getOperand(2).getReg());
2886
2887 // If we don't have an immediate compare, then we need to add a use of the
2888 // register which wasn't used for the immediate.
2889 // Note that the immediate will always be the last operand.
2890 if (CmpOpc != AArch64::FCMPSri && CmpOpc != AArch64::FCMPDri)
2891 CmpMI = CmpMI.addUse(I.getOperand(3).getReg());
2892
2893 const Register DefReg = I.getOperand(0).getReg();
2894 Register Def1Reg = DefReg;
2895 if (CC2 != AArch64CC::AL)
2896 Def1Reg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
2897
2898 MachineInstr &CSetMI =
2899 *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr))
2900 .addDef(Def1Reg)
2901 .addUse(AArch64::WZR)
2902 .addUse(AArch64::WZR)
2903 .addImm(getInvertedCondCode(CC1));
2904
2905 if (CC2 != AArch64CC::AL) {
2906 Register Def2Reg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
2907 MachineInstr &CSet2MI =
2908 *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr))
2909 .addDef(Def2Reg)
2910 .addUse(AArch64::WZR)
2911 .addUse(AArch64::WZR)
2912 .addImm(getInvertedCondCode(CC2));
2913 MachineInstr &OrMI =
2914 *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::ORRWrr))
2915 .addDef(DefReg)
2916 .addUse(Def1Reg)
2917 .addUse(Def2Reg);
2918 constrainSelectedInstRegOperands(OrMI, TII, TRI, RBI);
2919 constrainSelectedInstRegOperands(CSet2MI, TII, TRI, RBI);
2920 }
2921 constrainSelectedInstRegOperands(*CmpMI, TII, TRI, RBI);
2922 constrainSelectedInstRegOperands(CSetMI, TII, TRI, RBI);
2923
2924 I.eraseFromParent();
2925 return true;
2926 }
2927 case TargetOpcode::G_VASTART:
2928 return STI.isTargetDarwin() ? selectVaStartDarwin(I, MF, MRI)
2929 : selectVaStartAAPCS(I, MF, MRI);
2930 case TargetOpcode::G_INTRINSIC:
2931 return selectIntrinsic(I, MRI);
2932 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
2933 return selectIntrinsicWithSideEffects(I, MRI);
2934 case TargetOpcode::G_IMPLICIT_DEF: {
2935 I.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF));
2936 const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
2937 const Register DstReg = I.getOperand(0).getReg();
2938 const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
2939 const TargetRegisterClass *DstRC =
2940 getRegClassForTypeOnBank(DstTy, DstRB, RBI);
2941 RBI.constrainGenericRegister(DstReg, *DstRC, MRI);
2942 return true;
2943 }
2944 case TargetOpcode::G_BLOCK_ADDR: {
2945 if (TM.getCodeModel() == CodeModel::Large) {
2946 materializeLargeCMVal(I, I.getOperand(1).getBlockAddress(), 0);
2947 I.eraseFromParent();
2948 return true;
2949 } else {
2950 I.setDesc(TII.get(AArch64::MOVaddrBA));
2951 auto MovMI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::MOVaddrBA),
2952 I.getOperand(0).getReg())
2953 .addBlockAddress(I.getOperand(1).getBlockAddress(),
2954 /* Offset */ 0, AArch64II::MO_PAGE)
2955 .addBlockAddress(
2956 I.getOperand(1).getBlockAddress(), /* Offset */ 0,
2957 AArch64II::MO_NC | AArch64II::MO_PAGEOFF);
2958 I.eraseFromParent();
2959 return constrainSelectedInstRegOperands(*MovMI, TII, TRI, RBI);
2960 }
2961 }
2962 case TargetOpcode::G_INTRINSIC_TRUNC:
2963 return selectIntrinsicTrunc(I, MRI);
2964 case TargetOpcode::G_INTRINSIC_ROUND:
2965 return selectIntrinsicRound(I, MRI);
2966 case TargetOpcode::G_BUILD_VECTOR:
2967 return selectBuildVector(I, MRI);
2968 case TargetOpcode::G_MERGE_VALUES:
2969 return selectMergeValues(I, MRI);
2970 case TargetOpcode::G_UNMERGE_VALUES:
2971 return selectUnmergeValues(I, MRI);
2972 case TargetOpcode::G_SHUFFLE_VECTOR:
2973 return selectShuffleVector(I, MRI);
2974 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
2975 return selectExtractElt(I, MRI);
2976 case TargetOpcode::G_INSERT_VECTOR_ELT:
2977 return selectInsertElt(I, MRI);
2978 case TargetOpcode::G_CONCAT_VECTORS:
2979 return selectConcatVectors(I, MRI);
2980 case TargetOpcode::G_JUMP_TABLE:
2981 return selectJumpTable(I, MRI);
2982 }
2983
2984 return false;
2985}
2986
2987bool AArch64InstructionSelector::selectBrJT(MachineInstr &I,
2988 MachineRegisterInfo &MRI) const {
2989 assert(I.getOpcode() == TargetOpcode::G_BRJT && "Expected G_BRJT")((I.getOpcode() == TargetOpcode::G_BRJT && "Expected G_BRJT"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BRJT && \"Expected G_BRJT\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2989, __PRETTY_FUNCTION__))
;
2990 Register JTAddr = I.getOperand(0).getReg();
2991 unsigned JTI = I.getOperand(1).getIndex();
2992 Register Index = I.getOperand(2).getReg();
2993 MachineIRBuilder MIB(I);
2994
2995 Register TargetReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
2996 Register ScratchReg = MRI.createVirtualRegister(&AArch64::GPR64spRegClass);
2997 auto JumpTableInst = MIB.buildInstr(AArch64::JumpTableDest32,
2998 {TargetReg, ScratchReg}, {JTAddr, Index})
2999 .addJumpTableIndex(JTI);
3000 // Build the indirect branch.
3001 MIB.buildInstr(AArch64::BR, {}, {TargetReg});
3002 I.eraseFromParent();
3003 return constrainSelectedInstRegOperands(*JumpTableInst, TII, TRI, RBI);
3004}
3005
3006bool AArch64InstructionSelector::selectJumpTable(
3007 MachineInstr &I, MachineRegisterInfo &MRI) const {
3008 assert(I.getOpcode() == TargetOpcode::G_JUMP_TABLE && "Expected jump table")((I.getOpcode() == TargetOpcode::G_JUMP_TABLE && "Expected jump table"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_JUMP_TABLE && \"Expected jump table\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3008, __PRETTY_FUNCTION__))
;
3009 assert(I.getOperand(1).isJTI() && "Jump table op should have a JTI!")((I.getOperand(1).isJTI() && "Jump table op should have a JTI!"
) ? static_cast<void> (0) : __assert_fail ("I.getOperand(1).isJTI() && \"Jump table op should have a JTI!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3009, __PRETTY_FUNCTION__))
;
3010
3011 Register DstReg = I.getOperand(0).getReg();
3012 unsigned JTI = I.getOperand(1).getIndex();
3013 // We generate a MOVaddrJT which will get expanded to an ADRP + ADD later.
3014 MachineIRBuilder MIB(I);
3015 auto MovMI =
3016 MIB.buildInstr(AArch64::MOVaddrJT, {DstReg}, {})
3017 .addJumpTableIndex(JTI, AArch64II::MO_PAGE)
3018 .addJumpTableIndex(JTI, AArch64II::MO_NC | AArch64II::MO_PAGEOFF);
3019 I.eraseFromParent();
3020 return constrainSelectedInstRegOperands(*MovMI, TII, TRI, RBI);
3021}
3022
3023bool AArch64InstructionSelector::selectTLSGlobalValue(
3024 MachineInstr &I, MachineRegisterInfo &MRI) const {
3025 if (!STI.isTargetMachO())
3026 return false;
3027 MachineFunction &MF = *I.getParent()->getParent();
3028 MF.getFrameInfo().setAdjustsStack(true);
3029
3030 const GlobalValue &GV = *I.getOperand(1).getGlobal();
3031 MachineIRBuilder MIB(I);
3032
3033 auto LoadGOT =
3034 MIB.buildInstr(AArch64::LOADgot, {&AArch64::GPR64commonRegClass}, {})
3035 .addGlobalAddress(&GV, 0, AArch64II::MO_TLS);
3036
3037 auto Load = MIB.buildInstr(AArch64::LDRXui, {&AArch64::GPR64commonRegClass},
3038 {LoadGOT.getReg(0)})
3039 .addImm(0);
3040
3041 MIB.buildCopy(Register(AArch64::X0), LoadGOT.getReg(0));
3042 // TLS calls preserve all registers except those that absolutely must be
3043 // trashed: X0 (it takes an argument), LR (it's a call) and NZCV (let's not be
3044 // silly).
3045 MIB.buildInstr(getBLRCallOpcode(MF), {}, {Load})
3046 .addUse(AArch64::X0, RegState::Implicit)
3047 .addDef(AArch64::X0, RegState::Implicit)
3048 .addRegMask(TRI.getTLSCallPreservedMask());
3049
3050 MIB.buildCopy(I.getOperand(0).getReg(), Register(AArch64::X0));
3051 RBI.constrainGenericRegister(I.getOperand(0).getReg(), AArch64::GPR64RegClass,
3052 MRI);
3053 I.eraseFromParent();
3054 return true;
3055}
3056
3057bool AArch64InstructionSelector::selectIntrinsicTrunc(
3058 MachineInstr &I, MachineRegisterInfo &MRI) const {
3059 const LLT SrcTy = MRI.getType(I.getOperand(0).getReg());
3060
3061 // Select the correct opcode.
3062 unsigned Opc = 0;
3063 if (!SrcTy.isVector()) {
3064 switch (SrcTy.getSizeInBits()) {
3065 default:
3066 case 16:
3067 Opc = AArch64::FRINTZHr;
3068 break;
3069 case 32:
3070 Opc = AArch64::FRINTZSr;
3071 break;
3072 case 64:
3073 Opc = AArch64::FRINTZDr;
3074 break;
3075 }
3076 } else {
3077 unsigned NumElts = SrcTy.getNumElements();
3078 switch (SrcTy.getElementType().getSizeInBits()) {
3079 default:
3080 break;
3081 case 16:
3082 if (NumElts == 4)
3083 Opc = AArch64::FRINTZv4f16;
3084 else if (NumElts == 8)
3085 Opc = AArch64::FRINTZv8f16;
3086 break;
3087 case 32:
3088 if (NumElts == 2)
3089 Opc = AArch64::FRINTZv2f32;
3090 else if (NumElts == 4)
3091 Opc = AArch64::FRINTZv4f32;
3092 break;
3093 case 64:
3094 if (NumElts == 2)
3095 Opc = AArch64::FRINTZv2f64;
3096 break;
3097 }
3098 }
3099
3100 if (!Opc) {
3101 // Didn't get an opcode above, bail.
3102 LLVM_DEBUG(dbgs() << "Unsupported type for G_INTRINSIC_TRUNC!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported type for G_INTRINSIC_TRUNC!\n"
; } } while (false)
;
3103 return false;
3104 }
3105
3106 // Legalization would have set us up perfectly for this; we just need to
3107 // set the opcode and move on.
3108 I.setDesc(TII.get(Opc));
3109 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
3110}
3111
3112bool AArch64InstructionSelector::selectIntrinsicRound(
3113 MachineInstr &I, MachineRegisterInfo &MRI) const {
3114 const LLT SrcTy = MRI.getType(I.getOperand(0).getReg());
3115
3116 // Select the correct opcode.
3117 unsigned Opc = 0;
3118 if (!SrcTy.isVector()) {
3119 switch (SrcTy.getSizeInBits()) {
3120 default:
3121 case 16:
3122 Opc = AArch64::FRINTAHr;
3123 break;
3124 case 32:
3125 Opc = AArch64::FRINTASr;
3126 break;
3127 case 64:
3128 Opc = AArch64::FRINTADr;
3129 break;
3130 }
3131 } else {
3132 unsigned NumElts = SrcTy.getNumElements();
3133 switch (SrcTy.getElementType().getSizeInBits()) {
3134 default:
3135 break;
3136 case 16:
3137 if (NumElts == 4)
3138 Opc = AArch64::FRINTAv4f16;
3139 else if (NumElts == 8)
3140 Opc = AArch64::FRINTAv8f16;
3141 break;
3142 case 32:
3143 if (NumElts == 2)
3144 Opc = AArch64::FRINTAv2f32;
3145 else if (NumElts == 4)
3146 Opc = AArch64::FRINTAv4f32;
3147 break;
3148 case 64:
3149 if (NumElts == 2)
3150 Opc = AArch64::FRINTAv2f64;
3151 break;
3152 }
3153 }
3154
3155 if (!Opc) {
3156 // Didn't get an opcode above, bail.
3157 LLVM_DEBUG(dbgs() << "Unsupported type for G_INTRINSIC_ROUND!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported type for G_INTRINSIC_ROUND!\n"
; } } while (false)
;
3158 return false;
3159 }
3160
3161 // Legalization would have set us up perfectly for this; we just need to
3162 // set the opcode and move on.
3163 I.setDesc(TII.get(Opc));
3164 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
3165}
3166
3167bool AArch64InstructionSelector::selectVectorICmp(
3168 MachineInstr &I, MachineRegisterInfo &MRI) const {
3169 Register DstReg = I.getOperand(0).getReg();
3170 LLT DstTy = MRI.getType(DstReg);
3171 Register SrcReg = I.getOperand(2).getReg();
3172 Register Src2Reg = I.getOperand(3).getReg();
3173 LLT SrcTy = MRI.getType(SrcReg);
3174
3175 unsigned SrcEltSize = SrcTy.getElementType().getSizeInBits();
3176 unsigned NumElts = DstTy.getNumElements();
3177
3178 // First index is element size, 0 == 8b, 1 == 16b, 2 == 32b, 3 == 64b
3179 // Second index is num elts, 0 == v2, 1 == v4, 2 == v8, 3 == v16
3180 // Third index is cc opcode:
3181 // 0 == eq
3182 // 1 == ugt
3183 // 2 == uge
3184 // 3 == ult
3185 // 4 == ule
3186 // 5 == sgt
3187 // 6 == sge
3188 // 7 == slt
3189 // 8 == sle
3190 // ne is done by negating 'eq' result.
3191
3192 // This table below assumes that for some comparisons the operands will be
3193 // commuted.
3194 // ult op == commute + ugt op
3195 // ule op == commute + uge op
3196 // slt op == commute + sgt op
3197 // sle op == commute + sge op
3198 unsigned PredIdx = 0;
3199 bool SwapOperands = false;
3200 CmpInst::Predicate Pred = (CmpInst::Predicate)I.getOperand(1).getPredicate();
3201 switch (Pred) {
3202 case CmpInst::ICMP_NE:
3203 case CmpInst::ICMP_EQ:
3204 PredIdx = 0;
3205 break;
3206 case CmpInst::ICMP_UGT:
3207 PredIdx = 1;
3208 break;
3209 case CmpInst::ICMP_UGE:
3210 PredIdx = 2;
3211 break;
3212 case CmpInst::ICMP_ULT:
3213 PredIdx = 3;
3214 SwapOperands = true;
3215 break;
3216 case CmpInst::ICMP_ULE:
3217 PredIdx = 4;
3218 SwapOperands = true;
3219 break;
3220 case CmpInst::ICMP_SGT:
3221 PredIdx = 5;
3222 break;
3223 case CmpInst::ICMP_SGE:
3224 PredIdx = 6;
3225 break;
3226 case CmpInst::ICMP_SLT:
3227 PredIdx = 7;
3228 SwapOperands = true;
3229 break;
3230 case CmpInst::ICMP_SLE:
3231 PredIdx = 8;
3232 SwapOperands = true;
3233 break;
3234 default:
3235 llvm_unreachable("Unhandled icmp predicate")::llvm::llvm_unreachable_internal("Unhandled icmp predicate",
"/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3235)
;
3236 return false;
3237 }
3238
3239 // This table obviously should be tablegen'd when we have our GISel native
3240 // tablegen selector.
3241
3242 static const unsigned OpcTable[4][4][9] = {
3243 {
3244 {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3245 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3246 0 /* invalid */},
3247 {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3248 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3249 0 /* invalid */},
3250 {AArch64::CMEQv8i8, AArch64::CMHIv8i8, AArch64::CMHSv8i8,
3251 AArch64::CMHIv8i8, AArch64::CMHSv8i8, AArch64::CMGTv8i8,
3252 AArch64::CMGEv8i8, AArch64::CMGTv8i8, AArch64::CMGEv8i8},
3253 {AArch64::CMEQv16i8, AArch64::CMHIv16i8, AArch64::CMHSv16i8,
3254 AArch64::CMHIv16i8, AArch64::CMHSv16i8, AArch64::CMGTv16i8,
3255 AArch64::CMGEv16i8, AArch64::CMGTv16i8, AArch64::CMGEv16i8}
3256 },
3257 {
3258 {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3259 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3260 0 /* invalid */},
3261 {AArch64::CMEQv4i16, AArch64::CMHIv4i16, AArch64::CMHSv4i16,
3262 AArch64::CMHIv4i16, AArch64::CMHSv4i16, AArch64::CMGTv4i16,
3263 AArch64::CMGEv4i16, AArch64::CMGTv4i16, AArch64::CMGEv4i16},
3264 {AArch64::CMEQv8i16, AArch64::CMHIv8i16, AArch64::CMHSv8i16,
3265 AArch64::CMHIv8i16, AArch64::CMHSv8i16, AArch64::CMGTv8i16,
3266 AArch64::CMGEv8i16, AArch64::CMGTv8i16, AArch64::CMGEv8i16},
3267 {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3268 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3269 0 /* invalid */}
3270 },
3271 {
3272 {AArch64::CMEQv2i32, AArch64::CMHIv2i32, AArch64::CMHSv2i32,
3273 AArch64::CMHIv2i32, AArch64::CMHSv2i32, AArch64::CMGTv2i32,
3274 AArch64::CMGEv2i32, AArch64::CMGTv2i32, AArch64::CMGEv2i32},
3275 {AArch64::CMEQv4i32, AArch64::CMHIv4i32, AArch64::CMHSv4i32,
3276 AArch64::CMHIv4i32, AArch64::CMHSv4i32, AArch64::CMGTv4i32,
3277 AArch64::CMGEv4i32, AArch64::CMGTv4i32, AArch64::CMGEv4i32},
3278 {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3279 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3280 0 /* invalid */},
3281 {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3282 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3283 0 /* invalid */}
3284 },
3285 {
3286 {AArch64::CMEQv2i64, AArch64::CMHIv2i64, AArch64::CMHSv2i64,
3287 AArch64::CMHIv2i64, AArch64::CMHSv2i64, AArch64::CMGTv2i64,
3288 AArch64::CMGEv2i64, AArch64::CMGTv2i64, AArch64::CMGEv2i64},
3289 {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3290 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3291 0 /* invalid */},
3292 {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3293 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3294 0 /* invalid */},
3295 {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3296 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3297 0 /* invalid */}
3298 },
3299 };
3300 unsigned EltIdx = Log2_32(SrcEltSize / 8);
3301 unsigned NumEltsIdx = Log2_32(NumElts / 2);
3302 unsigned Opc = OpcTable[EltIdx][NumEltsIdx][PredIdx];
3303 if (!Opc) {
3304 LLVM_DEBUG(dbgs() << "Could not map G_ICMP to cmp opcode")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not map G_ICMP to cmp opcode"
; } } while (false)
;
3305 return false;
3306 }
3307
3308 const RegisterBank &VecRB = *RBI.getRegBank(SrcReg, MRI, TRI);
3309 const TargetRegisterClass *SrcRC =
3310 getRegClassForTypeOnBank(SrcTy, VecRB, RBI, true);
3311 if (!SrcRC) {
3312 LLVM_DEBUG(dbgs() << "Could not determine source register class.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not determine source register class.\n"
; } } while (false)
;
3313 return false;
3314 }
3315
3316 unsigned NotOpc = Pred == ICmpInst::ICMP_NE ? AArch64::NOTv8i8 : 0;
3317 if (SrcTy.getSizeInBits() == 128)
3318 NotOpc = NotOpc ? AArch64::NOTv16i8 : 0;
3319
3320 if (SwapOperands)
3321 std::swap(SrcReg, Src2Reg);
3322
3323 MachineIRBuilder MIB(I);
3324 auto Cmp = MIB.buildInstr(Opc, {SrcRC}, {SrcReg, Src2Reg});
3325 constrainSelectedInstRegOperands(*Cmp, TII, TRI, RBI);
3326
3327 // Invert if we had a 'ne' cc.
3328 if (NotOpc) {
3329 Cmp = MIB.buildInstr(NotOpc, {DstReg}, {Cmp});
3330 constrainSelectedInstRegOperands(*Cmp, TII, TRI, RBI);
3331 } else {
3332 MIB.buildCopy(DstReg, Cmp.getReg(0));
3333 }
3334 RBI.constrainGenericRegister(DstReg, *SrcRC, MRI);
3335 I.eraseFromParent();
3336 return true;
3337}
3338
3339MachineInstr *AArch64InstructionSelector::emitScalarToVector(
3340 unsigned EltSize, const TargetRegisterClass *DstRC, Register Scalar,
3341 MachineIRBuilder &MIRBuilder) const {
3342 auto Undef = MIRBuilder.buildInstr(TargetOpcode::IMPLICIT_DEF, {DstRC}, {});
3343
3344 auto BuildFn = [&](unsigned SubregIndex) {
3345 auto Ins =
3346 MIRBuilder
3347 .buildInstr(TargetOpcode::INSERT_SUBREG, {DstRC}, {Undef, Scalar})
3348 .addImm(SubregIndex);
3349 constrainSelectedInstRegOperands(*Undef, TII, TRI, RBI);
3350 constrainSelectedInstRegOperands(*Ins, TII, TRI, RBI);
3351 return &*Ins;
3352 };
3353
3354 switch (EltSize) {
3355 case 16:
3356 return BuildFn(AArch64::hsub);
3357 case 32:
3358 return BuildFn(AArch64::ssub);
3359 case 64:
3360 return BuildFn(AArch64::dsub);
3361 default:
3362 return nullptr;
3363 }
3364}
3365
3366bool AArch64InstructionSelector::selectMergeValues(
3367 MachineInstr &I, MachineRegisterInfo &MRI) const {
3368 assert(I.getOpcode() == TargetOpcode::G_MERGE_VALUES && "unexpected opcode")((I.getOpcode() == TargetOpcode::G_MERGE_VALUES && "unexpected opcode"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_MERGE_VALUES && \"unexpected opcode\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3368, __PRETTY_FUNCTION__))
;
3369 const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
3370 const LLT SrcTy = MRI.getType(I.getOperand(1).getReg());
3371 assert(!DstTy.isVector() && !SrcTy.isVector() && "invalid merge operation")((!DstTy.isVector() && !SrcTy.isVector() && "invalid merge operation"
) ? static_cast<void> (0) : __assert_fail ("!DstTy.isVector() && !SrcTy.isVector() && \"invalid merge operation\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3371, __PRETTY_FUNCTION__))
;
3372 const RegisterBank &RB = *RBI.getRegBank(I.getOperand(1).getReg(), MRI, TRI);
3373
3374 if (I.getNumOperands() != 3)
3375 return false;
3376
3377 // Merging 2 s64s into an s128.
3378 if (DstTy == LLT::scalar(128)) {
3379 if (SrcTy.getSizeInBits() != 64)
3380 return false;
3381 MachineIRBuilder MIB(I);
3382 Register DstReg = I.getOperand(0).getReg();
3383 Register Src1Reg = I.getOperand(1).getReg();
3384 Register Src2Reg = I.getOperand(2).getReg();
3385 auto Tmp = MIB.buildInstr(TargetOpcode::IMPLICIT_DEF, {DstTy}, {});
3386 MachineInstr *InsMI =
3387 emitLaneInsert(None, Tmp.getReg(0), Src1Reg, /* LaneIdx */ 0, RB, MIB);
3388 if (!InsMI)
3389 return false;
3390 MachineInstr *Ins2MI = emitLaneInsert(DstReg, InsMI->getOperand(0).getReg(),
3391 Src2Reg, /* LaneIdx */ 1, RB, MIB);
3392 if (!Ins2MI)
3393 return false;
3394 constrainSelectedInstRegOperands(*InsMI, TII, TRI, RBI);
3395 constrainSelectedInstRegOperands(*Ins2MI, TII, TRI, RBI);
3396 I.eraseFromParent();
3397 return true;
3398 }
3399
3400 if (RB.getID() != AArch64::GPRRegBankID)
3401 return false;
3402
3403 if (DstTy.getSizeInBits() != 64 || SrcTy.getSizeInBits() != 32)
3404 return false;
3405
3406 auto *DstRC = &AArch64::GPR64RegClass;
3407 Register SubToRegDef = MRI.createVirtualRegister(DstRC);
3408 MachineInstr &SubRegMI = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
3409 TII.get(TargetOpcode::SUBREG_TO_REG))
3410 .addDef(SubToRegDef)
3411 .addImm(0)
3412 .addUse(I.getOperand(1).getReg())
3413 .addImm(AArch64::sub_32);
3414 Register SubToRegDef2 = MRI.createVirtualRegister(DstRC);
3415 // Need to anyext the second scalar before we can use bfm
3416 MachineInstr &SubRegMI2 = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
3417 TII.get(TargetOpcode::SUBREG_TO_REG))
3418 .addDef(SubToRegDef2)
3419 .addImm(0)
3420 .addUse(I.getOperand(2).getReg())
3421 .addImm(AArch64::sub_32);
3422 MachineInstr &BFM =
3423 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::BFMXri))
3424 .addDef(I.getOperand(0).getReg())
3425 .addUse(SubToRegDef)
3426 .addUse(SubToRegDef2)
3427 .addImm(32)
3428 .addImm(31);
3429 constrainSelectedInstRegOperands(SubRegMI, TII, TRI, RBI);
3430 constrainSelectedInstRegOperands(SubRegMI2, TII, TRI, RBI);
3431 constrainSelectedInstRegOperands(BFM, TII, TRI, RBI);
3432 I.eraseFromParent();
3433 return true;
3434}
3435
3436static bool getLaneCopyOpcode(unsigned &CopyOpc, unsigned &ExtractSubReg,
3437 const unsigned EltSize) {
3438 // Choose a lane copy opcode and subregister based off of the size of the
3439 // vector's elements.
3440 switch (EltSize) {
3441 case 16:
3442 CopyOpc = AArch64::CPYi16;
3443 ExtractSubReg = AArch64::hsub;
3444 break;
3445 case 32:
3446 CopyOpc = AArch64::CPYi32;
3447 ExtractSubReg = AArch64::ssub;
3448 break;
3449 case 64:
3450 CopyOpc = AArch64::CPYi64;
3451 ExtractSubReg = AArch64::dsub;
3452 break;
3453 default:
3454 // Unknown size, bail out.
3455 LLVM_DEBUG(dbgs() << "Elt size '" << EltSize << "' unsupported.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Elt size '" << EltSize
<< "' unsupported.\n"; } } while (false)
;
3456 return false;
3457 }
3458 return true;
3459}
3460
3461MachineInstr *AArch64InstructionSelector::emitExtractVectorElt(
3462 Optional<Register> DstReg, const RegisterBank &DstRB, LLT ScalarTy,
3463 Register VecReg, unsigned LaneIdx, MachineIRBuilder &MIRBuilder) const {
3464 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
3465 unsigned CopyOpc = 0;
3466 unsigned ExtractSubReg = 0;
3467 if (!getLaneCopyOpcode(CopyOpc, ExtractSubReg, ScalarTy.getSizeInBits())) {
3468 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't determine lane copy opcode for instruction.\n"
; } } while (false)
3469 dbgs() << "Couldn't determine lane copy opcode for instruction.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't determine lane copy opcode for instruction.\n"
; } } while (false)
;
3470 return nullptr;
3471 }
3472
3473 const TargetRegisterClass *DstRC =
3474 getRegClassForTypeOnBank(ScalarTy, DstRB, RBI, true);
3475 if (!DstRC) {
3476 LLVM_DEBUG(dbgs() << "Could not determine destination register class.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not determine destination register class.\n"
; } } while (false)
;
3477 return nullptr;
3478 }
3479
3480 const RegisterBank &VecRB = *RBI.getRegBank(VecReg, MRI, TRI);
3481 const LLT &VecTy = MRI.getType(VecReg);
3482 const TargetRegisterClass *VecRC =
3483 getRegClassForTypeOnBank(VecTy, VecRB, RBI, true);
3484 if (!VecRC) {
3485 LLVM_DEBUG(dbgs() << "Could not determine source register class.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not determine source register class.\n"
; } } while (false)
;
3486 return nullptr;
3487 }
3488
3489 // The register that we're going to copy into.
3490 Register InsertReg = VecReg;
3491 if (!DstReg)
3492 DstReg = MRI.createVirtualRegister(DstRC);
3493 // If the lane index is 0, we just use a subregister COPY.
3494 if (LaneIdx == 0) {
3495 auto Copy = MIRBuilder.buildInstr(TargetOpcode::COPY, {*DstReg}, {})
3496 .addReg(VecReg, 0, ExtractSubReg);
3497 RBI.constrainGenericRegister(*DstReg, *DstRC, MRI);
3498 return &*Copy;
3499 }
3500
3501 // Lane copies require 128-bit wide registers. If we're dealing with an
3502 // unpacked vector, then we need to move up to that width. Insert an implicit
3503 // def and a subregister insert to get us there.
3504 if (VecTy.getSizeInBits() != 128) {
3505 MachineInstr *ScalarToVector = emitScalarToVector(
3506 VecTy.getSizeInBits(), &AArch64::FPR128RegClass, VecReg, MIRBuilder);
3507 if (!ScalarToVector)
3508 return nullptr;
3509 InsertReg = ScalarToVector->getOperand(0).getReg();
3510 }
3511
3512 MachineInstr *LaneCopyMI =
3513 MIRBuilder.buildInstr(CopyOpc, {*DstReg}, {InsertReg}).addImm(LaneIdx);
3514 constrainSelectedInstRegOperands(*LaneCopyMI, TII, TRI, RBI);
3515
3516 // Make sure that we actually constrain the initial copy.
3517 RBI.constrainGenericRegister(*DstReg, *DstRC, MRI);
3518 return LaneCopyMI;
3519}
3520
3521bool AArch64InstructionSelector::selectExtractElt(
3522 MachineInstr &I, MachineRegisterInfo &MRI) const {
3523 assert(I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT &&((I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT &&
"unexpected opcode!") ? static_cast<void> (0) : __assert_fail
("I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT && \"unexpected opcode!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3524, __PRETTY_FUNCTION__))
3524 "unexpected opcode!")((I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT &&
"unexpected opcode!") ? static_cast<void> (0) : __assert_fail
("I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT && \"unexpected opcode!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3524, __PRETTY_FUNCTION__))
;
3525 Register DstReg = I.getOperand(0).getReg();
3526 const LLT NarrowTy = MRI.getType(DstReg);
3527 const Register SrcReg = I.getOperand(1).getReg();
3528 const LLT WideTy = MRI.getType(SrcReg);
3529 (void)WideTy;
3530 assert(WideTy.getSizeInBits() >= NarrowTy.getSizeInBits() &&((WideTy.getSizeInBits() >= NarrowTy.getSizeInBits() &&
"source register size too small!") ? static_cast<void>
(0) : __assert_fail ("WideTy.getSizeInBits() >= NarrowTy.getSizeInBits() && \"source register size too small!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3531, __PRETTY_FUNCTION__))
3531 "source register size too small!")((WideTy.getSizeInBits() >= NarrowTy.getSizeInBits() &&
"source register size too small!") ? static_cast<void>
(0) : __assert_fail ("WideTy.getSizeInBits() >= NarrowTy.getSizeInBits() && \"source register size too small!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3531, __PRETTY_FUNCTION__))
;
3532 assert(NarrowTy.isScalar() && "cannot extract vector into vector!")((NarrowTy.isScalar() && "cannot extract vector into vector!"
) ? static_cast<void> (0) : __assert_fail ("NarrowTy.isScalar() && \"cannot extract vector into vector!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3532, __PRETTY_FUNCTION__))
;
3533
3534 // Need the lane index to determine the correct copy opcode.
3535 MachineOperand &LaneIdxOp = I.getOperand(2);
3536 assert(LaneIdxOp.isReg() && "Lane index operand was not a register?")((LaneIdxOp.isReg() && "Lane index operand was not a register?"
) ? static_cast<void> (0) : __assert_fail ("LaneIdxOp.isReg() && \"Lane index operand was not a register?\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3536, __PRETTY_FUNCTION__))
;
3537
3538 if (RBI.getRegBank(DstReg, MRI, TRI)->getID() != AArch64::FPRRegBankID) {
3539 LLVM_DEBUG(dbgs() << "Cannot extract into GPR.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Cannot extract into GPR.\n"
; } } while (false)
;
3540 return false;
3541 }
3542
3543 // Find the index to extract from.
3544 auto VRegAndVal = getConstantVRegValWithLookThrough(LaneIdxOp.getReg(), MRI);
3545 if (!VRegAndVal)
3546 return false;
3547 unsigned LaneIdx = VRegAndVal->Value;
3548
3549 MachineIRBuilder MIRBuilder(I);
3550
3551 const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
3552 MachineInstr *Extract = emitExtractVectorElt(DstReg, DstRB, NarrowTy, SrcReg,
3553 LaneIdx, MIRBuilder);
3554 if (!Extract)
3555 return false;
3556
3557 I.eraseFromParent();
3558 return true;
3559}
3560
3561bool AArch64InstructionSelector::selectSplitVectorUnmerge(
3562 MachineInstr &I, MachineRegisterInfo &MRI) const {
3563 unsigned NumElts = I.getNumOperands() - 1;
3564 Register SrcReg = I.getOperand(NumElts).getReg();
3565 const LLT NarrowTy = MRI.getType(I.getOperand(0).getReg());
3566 const LLT SrcTy = MRI.getType(SrcReg);
3567
3568 assert(NarrowTy.isVector() && "Expected an unmerge into vectors")((NarrowTy.isVector() && "Expected an unmerge into vectors"
) ? static_cast<void> (0) : __assert_fail ("NarrowTy.isVector() && \"Expected an unmerge into vectors\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3568, __PRETTY_FUNCTION__))
;
3569 if (SrcTy.getSizeInBits() > 128) {
3570 LLVM_DEBUG(dbgs() << "Unexpected vector type for vec split unmerge")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unexpected vector type for vec split unmerge"
; } } while (false)
;
3571 return false;
3572 }
3573
3574 MachineIRBuilder MIB(I);
3575
3576 // We implement a split vector operation by treating the sub-vectors as
3577 // scalars and extracting them.
3578 const RegisterBank &DstRB =
3579 *RBI.getRegBank(I.getOperand(0).getReg(), MRI, TRI);
3580 for (unsigned OpIdx = 0; OpIdx < NumElts; ++OpIdx) {
3581 Register Dst = I.getOperand(OpIdx).getReg();
3582 MachineInstr *Extract =
3583 emitExtractVectorElt(Dst, DstRB, NarrowTy, SrcReg, OpIdx, MIB);
3584 if (!Extract)
3585 return false;
3586 }
3587 I.eraseFromParent();
3588 return true;
3589}
3590
3591bool AArch64InstructionSelector::selectUnmergeValues(
3592 MachineInstr &I, MachineRegisterInfo &MRI) const {
3593 assert(I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&((I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES && "unexpected opcode"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES && \"unexpected opcode\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3594, __PRETTY_FUNCTION__))
3594 "unexpected opcode")((I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES && "unexpected opcode"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES && \"unexpected opcode\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3594, __PRETTY_FUNCTION__))
;
3595
3596 // TODO: Handle unmerging into GPRs and from scalars to scalars.
3597 if (RBI.getRegBank(I.getOperand(0).getReg(), MRI, TRI)->getID() !=
3598 AArch64::FPRRegBankID ||
3599 RBI.getRegBank(I.getOperand(1).getReg(), MRI, TRI)->getID() !=
3600 AArch64::FPRRegBankID) {
3601 LLVM_DEBUG(dbgs() << "Unmerging vector-to-gpr and scalar-to-scalar "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unmerging vector-to-gpr and scalar-to-scalar "
"currently unsupported.\n"; } } while (false)
3602 "currently unsupported.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unmerging vector-to-gpr and scalar-to-scalar "
"currently unsupported.\n"; } } while (false)
;
3603 return false;
3604 }
3605
3606 // The last operand is the vector source register, and every other operand is
3607 // a register to unpack into.
3608 unsigned NumElts = I.getNumOperands() - 1;
3609 Register SrcReg = I.getOperand(NumElts).getReg();
3610 const LLT NarrowTy = MRI.getType(I.getOperand(0).getReg());
3611 const LLT WideTy = MRI.getType(SrcReg);
3612 (void)WideTy;
3613 assert((WideTy.isVector() || WideTy.getSizeInBits() == 128) &&(((WideTy.isVector() || WideTy.getSizeInBits() == 128) &&
"can only unmerge from vector or s128 types!") ? static_cast
<void> (0) : __assert_fail ("(WideTy.isVector() || WideTy.getSizeInBits() == 128) && \"can only unmerge from vector or s128 types!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3614, __PRETTY_FUNCTION__))
3614 "can only unmerge from vector or s128 types!")(((WideTy.isVector() || WideTy.getSizeInBits() == 128) &&
"can only unmerge from vector or s128 types!") ? static_cast
<void> (0) : __assert_fail ("(WideTy.isVector() || WideTy.getSizeInBits() == 128) && \"can only unmerge from vector or s128 types!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3614, __PRETTY_FUNCTION__))
;
3615 assert(WideTy.getSizeInBits() > NarrowTy.getSizeInBits() &&((WideTy.getSizeInBits() > NarrowTy.getSizeInBits() &&
"source register size too small!") ? static_cast<void>
(0) : __assert_fail ("WideTy.getSizeInBits() > NarrowTy.getSizeInBits() && \"source register size too small!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3616, __PRETTY_FUNCTION__))
3616 "source register size too small!")((WideTy.getSizeInBits() > NarrowTy.getSizeInBits() &&
"source register size too small!") ? static_cast<void>
(0) : __assert_fail ("WideTy.getSizeInBits() > NarrowTy.getSizeInBits() && \"source register size too small!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3616, __PRETTY_FUNCTION__))
;
3617
3618 if (!NarrowTy.isScalar())
3619 return selectSplitVectorUnmerge(I, MRI);
3620
3621 MachineIRBuilder MIB(I);
3622
3623 // Choose a lane copy opcode and subregister based off of the size of the
3624 // vector's elements.
3625 unsigned CopyOpc = 0;
3626 unsigned ExtractSubReg = 0;
3627 if (!getLaneCopyOpcode(CopyOpc, ExtractSubReg, NarrowTy.getSizeInBits()))
3628 return false;
3629
3630 // Set up for the lane copies.
3631 MachineBasicBlock &MBB = *I.getParent();
3632
3633 // Stores the registers we'll be copying from.
3634 SmallVector<Register, 4> InsertRegs;
3635
3636 // We'll use the first register twice, so we only need NumElts-1 registers.
3637 unsigned NumInsertRegs = NumElts - 1;
3638
3639 // If our elements fit into exactly 128 bits, then we can copy from the source
3640 // directly. Otherwise, we need to do a bit of setup with some subregister
3641 // inserts.
3642 if (NarrowTy.getSizeInBits() * NumElts == 128) {
3643 InsertRegs = SmallVector<Register, 4>(NumInsertRegs, SrcReg);
3644 } else {
3645 // No. We have to perform subregister inserts. For each insert, create an
3646 // implicit def and a subregister insert, and save the register we create.
3647 for (unsigned Idx = 0; Idx < NumInsertRegs; ++Idx) {
3648 Register ImpDefReg = MRI.createVirtualRegister(&AArch64::FPR128RegClass);
3649 MachineInstr &ImpDefMI =
3650 *BuildMI(MBB, I, I.getDebugLoc(), TII.get(TargetOpcode::IMPLICIT_DEF),
3651 ImpDefReg);
3652
3653 // Now, create the subregister insert from SrcReg.
3654 Register InsertReg = MRI.createVirtualRegister(&AArch64::FPR128RegClass);
3655 MachineInstr &InsMI =
3656 *BuildMI(MBB, I, I.getDebugLoc(),
3657 TII.get(TargetOpcode::INSERT_SUBREG), InsertReg)
3658 .addUse(ImpDefReg)
3659 .addUse(SrcReg)
3660 .addImm(AArch64::dsub);
3661
3662 constrainSelectedInstRegOperands(ImpDefMI, TII, TRI, RBI);
3663 constrainSelectedInstRegOperands(InsMI, TII, TRI, RBI);
3664
3665 // Save the register so that we can copy from it after.
3666 InsertRegs.push_back(InsertReg);
3667 }
3668 }
3669
3670 // Now that we've created any necessary subregister inserts, we can
3671 // create the copies.
3672 //
3673 // Perform the first copy separately as a subregister copy.
3674 Register CopyTo = I.getOperand(0).getReg();
3675 auto FirstCopy = MIB.buildInstr(TargetOpcode::COPY, {CopyTo}, {})
3676 .addReg(InsertRegs[0], 0, ExtractSubReg);
3677 constrainSelectedInstRegOperands(*FirstCopy, TII, TRI, RBI);
3678
3679 // Now, perform the remaining copies as vector lane copies.
3680 unsigned LaneIdx = 1;
3681 for (Register InsReg : InsertRegs) {
3682 Register CopyTo = I.getOperand(LaneIdx).getReg();
3683 MachineInstr &CopyInst =
3684 *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CopyOpc), CopyTo)
3685 .addUse(InsReg)
3686 .addImm(LaneIdx);
3687 constrainSelectedInstRegOperands(CopyInst, TII, TRI, RBI);
3688 ++LaneIdx;
3689 }
3690
3691 // Separately constrain the first copy's destination. Because of the
3692 // limitation in constrainOperandRegClass, we can't guarantee that this will
3693 // actually be constrained. So, do it ourselves using the second operand.
3694 const TargetRegisterClass *RC =
3695 MRI.getRegClassOrNull(I.getOperand(1).getReg());
3696 if (!RC) {
3697 LLVM_DEBUG(dbgs() << "Couldn't constrain copy destination.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't constrain copy destination.\n"
; } } while (false)
;
3698 return false;
3699 }
3700
3701 RBI.constrainGenericRegister(CopyTo, *RC, MRI);
3702 I.eraseFromParent();
3703 return true;
3704}
3705
3706bool AArch64InstructionSelector::selectConcatVectors(
3707 MachineInstr &I, MachineRegisterInfo &MRI) const {
3708 assert(I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS &&((I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS && "Unexpected opcode"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS && \"Unexpected opcode\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3709, __PRETTY_FUNCTION__))
3709 "Unexpected opcode")((I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS && "Unexpected opcode"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS && \"Unexpected opcode\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3709, __PRETTY_FUNCTION__))
;
3710 Register Dst = I.getOperand(0).getReg();
3711 Register Op1 = I.getOperand(1).getReg();
3712 Register Op2 = I.getOperand(2).getReg();
3713 MachineIRBuilder MIRBuilder(I);
3714 MachineInstr *ConcatMI = emitVectorConcat(Dst, Op1, Op2, MIRBuilder);
3715 if (!ConcatMI)
3716 return false;
3717 I.eraseFromParent();
3718 return true;
3719}
3720
3721unsigned
3722AArch64InstructionSelector::emitConstantPoolEntry(const Constant *CPVal,
3723 MachineFunction &MF) const {
3724 Type *CPTy = CPVal->getType();
3725 Align Alignment = MF.getDataLayout().getPrefTypeAlign(CPTy);
3726
3727 MachineConstantPool *MCP = MF.getConstantPool();
3728 return MCP->getConstantPoolIndex(CPVal, Alignment);
3729}
3730
3731MachineInstr *AArch64InstructionSelector::emitLoadFromConstantPool(
3732 const Constant *CPVal, MachineIRBuilder &MIRBuilder) const {
3733 unsigned CPIdx = emitConstantPoolEntry(CPVal, MIRBuilder.getMF());
3734
3735 auto Adrp =
3736 MIRBuilder.buildInstr(AArch64::ADRP, {&AArch64::GPR64RegClass}, {})
3737 .addConstantPoolIndex(CPIdx, 0, AArch64II::MO_PAGE);
3738
3739 MachineInstr *LoadMI = nullptr;
3740 switch (MIRBuilder.getDataLayout().getTypeStoreSize(CPVal->getType())) {
3741 case 16:
3742 LoadMI =
3743 &*MIRBuilder
3744 .buildInstr(AArch64::LDRQui, {&AArch64::FPR128RegClass}, {Adrp})
3745 .addConstantPoolIndex(CPIdx, 0,
3746 AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
3747 break;
3748 case 8:
3749 LoadMI = &*MIRBuilder
3750 .buildInstr(AArch64::LDRDui, {&AArch64::FPR64RegClass}, {Adrp})
3751 .addConstantPoolIndex(
3752 CPIdx, 0, AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
3753 break;
3754 default:
3755 LLVM_DEBUG(dbgs() << "Could not load from constant pool of type "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not load from constant pool of type "
<< *CPVal->getType(); } } while (false)
3756 << *CPVal->getType())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not load from constant pool of type "
<< *CPVal->getType(); } } while (false)
;
3757 return nullptr;
3758 }
3759 constrainSelectedInstRegOperands(*Adrp, TII, TRI, RBI);
3760 constrainSelectedInstRegOperands(*LoadMI, TII, TRI, RBI);
3761 return LoadMI;
3762}
3763
3764/// Return an <Opcode, SubregIndex> pair to do an vector elt insert of a given
3765/// size and RB.
3766static std::pair<unsigned, unsigned>
3767getInsertVecEltOpInfo(const RegisterBank &RB, unsigned EltSize) {
3768 unsigned Opc, SubregIdx;
3769 if (RB.getID() == AArch64::GPRRegBankID) {
3770 if (EltSize == 32) {
3771 Opc = AArch64::INSvi32gpr;
3772 SubregIdx = AArch64::ssub;
3773 } else if (EltSize == 64) {
3774 Opc = AArch64::INSvi64gpr;
3775 SubregIdx = AArch64::dsub;
3776 } else {
3777 llvm_unreachable("invalid elt size!")::llvm::llvm_unreachable_internal("invalid elt size!", "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3777)
;
3778 }
3779 } else {
3780 if (EltSize == 8) {
3781 Opc = AArch64::INSvi8lane;
3782 SubregIdx = AArch64::bsub;
3783 } else if (EltSize == 16) {
3784 Opc = AArch64::INSvi16lane;
3785 SubregIdx = AArch64::hsub;
3786 } else if (EltSize == 32) {
3787 Opc = AArch64::INSvi32lane;
3788 SubregIdx = AArch64::ssub;
3789 } else if (EltSize == 64) {
3790 Opc = AArch64::INSvi64lane;
3791 SubregIdx = AArch64::dsub;
3792 } else {
3793 llvm_unreachable("invalid elt size!")::llvm::llvm_unreachable_internal("invalid elt size!", "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3793)
;
3794 }
3795 }
3796 return std::make_pair(Opc, SubregIdx);
3797}
3798
3799MachineInstr *AArch64InstructionSelector::emitInstr(
3800 unsigned Opcode, std::initializer_list<llvm::DstOp> DstOps,
3801 std::initializer_list<llvm::SrcOp> SrcOps, MachineIRBuilder &MIRBuilder,
3802 const ComplexRendererFns &RenderFns) const {
3803 assert(Opcode && "Expected an opcode?")((Opcode && "Expected an opcode?") ? static_cast<void
> (0) : __assert_fail ("Opcode && \"Expected an opcode?\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3803, __PRETTY_FUNCTION__))
;
3804 assert(!isPreISelGenericOpcode(Opcode) &&((!isPreISelGenericOpcode(Opcode) && "Function should only be used to produce selected instructions!"
) ? static_cast<void> (0) : __assert_fail ("!isPreISelGenericOpcode(Opcode) && \"Function should only be used to produce selected instructions!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3805, __PRETTY_FUNCTION__))
3805 "Function should only be used to produce selected instructions!")((!isPreISelGenericOpcode(Opcode) && "Function should only be used to produce selected instructions!"
) ? static_cast<void> (0) : __assert_fail ("!isPreISelGenericOpcode(Opcode) && \"Function should only be used to produce selected instructions!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3805, __PRETTY_FUNCTION__))
;
3806 auto MI = MIRBuilder.buildInstr(Opcode, DstOps, SrcOps);
3807 if (RenderFns)
3808 for (auto &Fn : *RenderFns)
3809 Fn(MI);
3810 constrainSelectedInstRegOperands(*MI, TII, TRI, RBI);
3811 return &*MI;
3812}
3813
3814MachineInstr *AArch64InstructionSelector::emitBinOp(
3815 const std::array<std::array<unsigned, 2>, 3> &AddrModeAndSizeToOpcode,
3816 Register Dst, MachineOperand &LHS, MachineOperand &RHS,
3817 MachineIRBuilder &MIRBuilder) const {
3818 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
3819 assert(LHS.isReg() && RHS.isReg() && "Expected register operands?")((LHS.isReg() && RHS.isReg() && "Expected register operands?"
) ? static_cast<void> (0) : __assert_fail ("LHS.isReg() && RHS.isReg() && \"Expected register operands?\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3819, __PRETTY_FUNCTION__))
;
3820 auto Ty = MRI.getType(LHS.getReg());
3821 assert(Ty.isScalar() && "Expected a scalar?")((Ty.isScalar() && "Expected a scalar?") ? static_cast
<void> (0) : __assert_fail ("Ty.isScalar() && \"Expected a scalar?\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3821, __PRETTY_FUNCTION__))
;
3822 unsigned Size = Ty.getSizeInBits();
3823 assert((Size == 32 || Size == 64) && "Expected a 32-bit or 64-bit type only")(((Size == 32 || Size == 64) && "Expected a 32-bit or 64-bit type only"
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Expected a 32-bit or 64-bit type only\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3823, __PRETTY_FUNCTION__))
;
3824 bool Is32Bit = Size == 32;
3825 if (auto Fns = selectArithImmed(RHS))
3826 return emitInstr(AddrModeAndSizeToOpcode[0][Is32Bit], {Dst}, {LHS},
3827 MIRBuilder, Fns);
3828 if (auto Fns = selectShiftedRegister(RHS))
3829 return emitInstr(AddrModeAndSizeToOpcode[1][Is32Bit], {Dst}, {LHS},
3830 MIRBuilder, Fns);
3831 return emitInstr(AddrModeAndSizeToOpcode[2][Is32Bit], {Dst}, {LHS, RHS},
3832 MIRBuilder);
3833}
3834
3835MachineInstr *
3836AArch64InstructionSelector::emitADD(Register DefReg, MachineOperand &LHS,
3837 MachineOperand &RHS,
3838 MachineIRBuilder &MIRBuilder) const {
3839 const std::array<std::array<unsigned, 2>, 3> OpcTable{
3840 {{AArch64::ADDXri, AArch64::ADDWri},
3841 {AArch64::ADDXrs, AArch64::ADDWrs},
3842 {AArch64::ADDXrr, AArch64::ADDWrr}}};
3843 return emitBinOp(OpcTable, DefReg, LHS, RHS, MIRBuilder);
3844}
3845
3846MachineInstr *
3847AArch64InstructionSelector::emitADDS(Register Dst, MachineOperand &LHS,
3848 MachineOperand &RHS,
3849 MachineIRBuilder &MIRBuilder) const {
3850 const std::array<std::array<unsigned, 2>, 3> OpcTable{
3851 {{AArch64::ADDSXri, AArch64::ADDSWri},
3852 {AArch64::ADDSXrs, AArch64::ADDSWrs},
3853 {AArch64::ADDSXrr, AArch64::ADDSWrr}}};
3854 return emitBinOp(OpcTable, Dst, LHS, RHS, MIRBuilder);
3855}
3856
3857MachineInstr *
3858AArch64InstructionSelector::emitCMN(MachineOperand &LHS, MachineOperand &RHS,
3859 MachineIRBuilder &MIRBuilder) const {
3860 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
3861 bool Is32Bit = (MRI.getType(LHS.getReg()).getSizeInBits() == 32);
3862 return emitADDS(Is32Bit ? AArch64::WZR : AArch64::XZR, LHS, RHS, MIRBuilder);
3863}
3864
3865MachineInstr *
3866AArch64InstructionSelector::emitTST(const Register &LHS, const Register &RHS,
3867 MachineIRBuilder &MIRBuilder) const {
3868 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
3869 unsigned RegSize = MRI.getType(LHS).getSizeInBits();
3870 bool Is32Bit = (RegSize == 32);
3871 static const unsigned OpcTable[2][2]{{AArch64::ANDSXrr, AArch64::ANDSXri},
3872 {AArch64::ANDSWrr, AArch64::ANDSWri}};
3873 Register ZReg = Is32Bit ? AArch64::WZR : AArch64::XZR;
3874
3875 // We might be able to fold in an immediate into the TST. We need to make sure
3876 // it's a logical immediate though, since ANDS requires that.
3877 auto ValAndVReg = getConstantVRegValWithLookThrough(RHS, MRI);
3878 bool IsImmForm = ValAndVReg.hasValue() &&
3879 AArch64_AM::isLogicalImmediate(ValAndVReg->Value, RegSize);
3880 unsigned Opc = OpcTable[Is32Bit][IsImmForm];
3881 auto TstMI = MIRBuilder.buildInstr(Opc, {ZReg}, {LHS});
3882
3883 if (IsImmForm)
3884 TstMI.addImm(
3885 AArch64_AM::encodeLogicalImmediate(ValAndVReg->Value, RegSize));
3886 else
3887 TstMI.addUse(RHS);
3888
3889 constrainSelectedInstRegOperands(*TstMI, TII, TRI, RBI);
3890 return &*TstMI;
3891}
3892
3893std::pair<MachineInstr *, CmpInst::Predicate>
3894AArch64InstructionSelector::emitIntegerCompare(
3895 MachineOperand &LHS, MachineOperand &RHS, MachineOperand &Predicate,
3896 MachineIRBuilder &MIRBuilder) const {
3897 assert(LHS.isReg() && RHS.isReg() && "Expected LHS and RHS to be registers!")((LHS.isReg() && RHS.isReg() && "Expected LHS and RHS to be registers!"
) ? static_cast<void> (0) : __assert_fail ("LHS.isReg() && RHS.isReg() && \"Expected LHS and RHS to be registers!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3897, __PRETTY_FUNCTION__))
;
3898 assert(Predicate.isPredicate() && "Expected predicate?")((Predicate.isPredicate() && "Expected predicate?") ?
static_cast<void> (0) : __assert_fail ("Predicate.isPredicate() && \"Expected predicate?\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3898, __PRETTY_FUNCTION__))
;
3899 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
3900
3901 CmpInst::Predicate P = (CmpInst::Predicate)Predicate.getPredicate();
3902
3903 // Fold the compare if possible.
3904 MachineInstr *FoldCmp =
3905 tryFoldIntegerCompare(LHS, RHS, Predicate, MIRBuilder);
3906 if (FoldCmp)
3907 return {FoldCmp, P};
3908
3909 // Can't fold into a CMN. Just emit a normal compare.
3910 unsigned CmpOpc = 0;
3911 Register ZReg;
3912
3913 LLT CmpTy = MRI.getType(LHS.getReg());
3914 assert((CmpTy.isScalar() || CmpTy.isPointer()) &&(((CmpTy.isScalar() || CmpTy.isPointer()) && "Expected scalar or pointer"
) ? static_cast<void> (0) : __assert_fail ("(CmpTy.isScalar() || CmpTy.isPointer()) && \"Expected scalar or pointer\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3915, __PRETTY_FUNCTION__))
3915 "Expected scalar or pointer")(((CmpTy.isScalar() || CmpTy.isPointer()) && "Expected scalar or pointer"
) ? static_cast<void> (0) : __assert_fail ("(CmpTy.isScalar() || CmpTy.isPointer()) && \"Expected scalar or pointer\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3915, __PRETTY_FUNCTION__))
;
3916 if (CmpTy == LLT::scalar(32)) {
3917 CmpOpc = AArch64::SUBSWrr;
3918 ZReg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
3919 } else if (CmpTy == LLT::scalar(64) || CmpTy.isPointer()) {
3920 CmpOpc = AArch64::SUBSXrr;
3921 ZReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
3922 } else {
3923 return {nullptr, CmpInst::Predicate::BAD_ICMP_PREDICATE};
3924 }
3925
3926 // Try to match immediate forms.
3927 MachineInstr *ImmedCmp =
3928 tryOptArithImmedIntegerCompare(LHS, RHS, P, MIRBuilder);
3929 if (ImmedCmp)
3930 return {ImmedCmp, P};
3931
3932 // If we don't have an immediate, we may have a shift which can be folded
3933 // into the compare.
3934 MachineInstr *ShiftedCmp = tryOptArithShiftedCompare(LHS, RHS, MIRBuilder);
3935 if (ShiftedCmp)
3936 return {ShiftedCmp, P};
3937
3938 auto CmpMI =
3939 MIRBuilder.buildInstr(CmpOpc, {ZReg}, {LHS.getReg(), RHS.getReg()});
3940 // Make sure that we can constrain the compare that we emitted.
3941 constrainSelectedInstRegOperands(*CmpMI, TII, TRI, RBI);
3942 return {&*CmpMI, P};
3943}
3944
3945MachineInstr *AArch64InstructionSelector::emitVectorConcat(
3946 Optional<Register> Dst, Register Op1, Register Op2,
3947 MachineIRBuilder &MIRBuilder) const {
3948 // We implement a vector concat by:
3949 // 1. Use scalar_to_vector to insert the lower vector into the larger dest
3950 // 2. Insert the upper vector into the destination's upper element
3951 // TODO: some of this code is common with G_BUILD_VECTOR handling.
3952 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
3953
3954 const LLT Op1Ty = MRI.getType(Op1);
3955 const LLT Op2Ty = MRI.getType(Op2);
3956
3957 if (Op1Ty != Op2Ty) {
3958 LLVM_DEBUG(dbgs() << "Could not do vector concat of differing vector tys")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not do vector concat of differing vector tys"
; } } while (false)
;
3959 return nullptr;
3960 }
3961 assert(Op1Ty.isVector() && "Expected a vector for vector concat")((Op1Ty.isVector() && "Expected a vector for vector concat"
) ? static_cast<void> (0) : __assert_fail ("Op1Ty.isVector() && \"Expected a vector for vector concat\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3961, __PRETTY_FUNCTION__))
;
3962
3963 if (Op1Ty.getSizeInBits() >= 128) {
3964 LLVM_DEBUG(dbgs() << "Vector concat not supported for full size vectors")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Vector concat not supported for full size vectors"
; } } while (false)
;
3965 return nullptr;
3966 }
3967
3968 // At the moment we just support 64 bit vector concats.
3969 if (Op1Ty.getSizeInBits() != 64) {
3970 LLVM_DEBUG(dbgs() << "Vector concat supported for 64b vectors")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Vector concat supported for 64b vectors"
; } } while (false)
;
3971 return nullptr;
3972 }
3973
3974 const LLT ScalarTy = LLT::scalar(Op1Ty.getSizeInBits());
3975 const RegisterBank &FPRBank = *RBI.getRegBank(Op1, MRI, TRI);
3976 const TargetRegisterClass *DstRC =
3977 getMinClassForRegBank(FPRBank, Op1Ty.getSizeInBits() * 2);
3978
3979 MachineInstr *WidenedOp1 =
3980 emitScalarToVector(ScalarTy.getSizeInBits(), DstRC, Op1, MIRBuilder);
3981 MachineInstr *WidenedOp2 =
3982 emitScalarToVector(ScalarTy.getSizeInBits(), DstRC, Op2, MIRBuilder);
3983 if (!WidenedOp1 || !WidenedOp2) {
3984 LLVM_DEBUG(dbgs() << "Could not emit a vector from scalar value")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not emit a vector from scalar value"
; } } while (false)
;
3985 return nullptr;
3986 }
3987
3988 // Now do the insert of the upper element.
3989 unsigned InsertOpc, InsSubRegIdx;
3990 std::tie(InsertOpc, InsSubRegIdx) =
3991 getInsertVecEltOpInfo(FPRBank, ScalarTy.getSizeInBits());
3992
3993 if (!Dst)
3994 Dst = MRI.createVirtualRegister(DstRC);
3995 auto InsElt =
3996 MIRBuilder
3997 .buildInstr(InsertOpc, {*Dst}, {WidenedOp1->getOperand(0).getReg()})
3998 .addImm(1) /* Lane index */
3999 .addUse(WidenedOp2->getOperand(0).getReg())
4000 .addImm(0);
4001 constrainSelectedInstRegOperands(*InsElt, TII, TRI, RBI);
4002 return &*InsElt;
4003}
4004
4005MachineInstr *AArch64InstructionSelector::emitFMovForFConstant(
4006 MachineInstr &I, MachineRegisterInfo &MRI) const {
4007 assert(I.getOpcode() == TargetOpcode::G_FCONSTANT &&((I.getOpcode() == TargetOpcode::G_FCONSTANT && "Expected a G_FCONSTANT!"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_FCONSTANT && \"Expected a G_FCONSTANT!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4008, __PRETTY_FUNCTION__))
4008 "Expected a G_FCONSTANT!")((I.getOpcode() == TargetOpcode::G_FCONSTANT && "Expected a G_FCONSTANT!"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_FCONSTANT && \"Expected a G_FCONSTANT!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4008, __PRETTY_FUNCTION__))
;
4009 MachineOperand &ImmOp = I.getOperand(1);
4010 unsigned DefSize = MRI.getType(I.getOperand(0).getReg()).getSizeInBits();
4011
4012 // Only handle 32 and 64 bit defs for now.
4013 if (DefSize != 32 && DefSize != 64)
4014 return nullptr;
4015
4016 // Don't handle null values using FMOV.
4017 if (ImmOp.getFPImm()->isNullValue())
4018 return nullptr;
4019
4020 // Get the immediate representation for the FMOV.
4021 const APFloat &ImmValAPF = ImmOp.getFPImm()->getValueAPF();
4022 int Imm = DefSize == 32 ? AArch64_AM::getFP32Imm(ImmValAPF)
4023 : AArch64_AM::getFP64Imm(ImmValAPF);
4024
4025 // If this is -1, it means the immediate can't be represented as the requested
4026 // floating point value. Bail.
4027 if (Imm == -1)
4028 return nullptr;
4029
4030 // Update MI to represent the new FMOV instruction, constrain it, and return.
4031 ImmOp.ChangeToImmediate(Imm);
4032 unsigned MovOpc = DefSize == 32 ? AArch64::FMOVSi : AArch64::FMOVDi;
4033 I.setDesc(TII.get(MovOpc));
4034 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
4035 return &I;
4036}
4037
4038MachineInstr *
4039AArch64InstructionSelector::emitCSetForICMP(Register DefReg, unsigned Pred,
4040 MachineIRBuilder &MIRBuilder) const {
4041 // CSINC increments the result when the predicate is false. Invert it.
4042 const AArch64CC::CondCode InvCC = changeICMPPredToAArch64CC(
4043 CmpInst::getInversePredicate((CmpInst::Predicate)Pred));
4044 auto I =
4045 MIRBuilder
4046 .buildInstr(AArch64::CSINCWr, {DefReg}, {Register(AArch64::WZR), Register(AArch64::WZR)})
4047 .addImm(InvCC);
4048 constrainSelectedInstRegOperands(*I, TII, TRI, RBI);
4049 return &*I;
4050}
4051
4052bool AArch64InstructionSelector::tryOptSelect(MachineInstr &I) const {
4053 MachineIRBuilder MIB(I);
4054 MachineRegisterInfo &MRI = *MIB.getMRI();
4055 const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo();
4056
4057 // We want to recognize this pattern:
4058 //
4059 // $z = G_FCMP pred, $x, $y
4060 // ...
4061 // $w = G_SELECT $z, $a, $b
4062 //
4063 // Where the value of $z is *only* ever used by the G_SELECT (possibly with
4064 // some copies/truncs in between.)
4065 //
4066 // If we see this, then we can emit something like this:
4067 //
4068 // fcmp $x, $y
4069 // fcsel $w, $a, $b, pred
4070 //
4071 // Rather than emitting both of the rather long sequences in the standard
4072 // G_FCMP/G_SELECT select methods.
4073
4074 // First, check if the condition is defined by a compare.
4075 MachineInstr *CondDef = MRI.getVRegDef(I.getOperand(1).getReg());
4076 while (CondDef) {
4077 // We can only fold if all of the defs have one use.
4078 Register CondDefReg = CondDef->getOperand(0).getReg();
4079 if (!MRI.hasOneNonDBGUse(CondDefReg)) {
4080 // Unless it's another select.
4081 for (const MachineInstr &UI : MRI.use_nodbg_instructions(CondDefReg)) {
4082 if (CondDef == &UI)
4083 continue;
4084 if (UI.getOpcode() != TargetOpcode::G_SELECT)
4085 return false;
4086 }
4087 }
4088
4089 // We can skip over G_TRUNC since the condition is 1-bit.
4090 // Truncating/extending can have no impact on the value.
4091 unsigned Opc = CondDef->getOpcode();
4092 if (Opc != TargetOpcode::COPY && Opc != TargetOpcode::G_TRUNC)
4093 break;
4094
4095 // Can't see past copies from physregs.
4096 if (Opc == TargetOpcode::COPY &&
4097 Register::isPhysicalRegister(CondDef->getOperand(1).getReg()))
4098 return false;
4099
4100 CondDef = MRI.getVRegDef(CondDef->getOperand(1).getReg());
4101 }
4102
4103 // Is the condition defined by a compare?
4104 if (!CondDef)
4105 return false;
4106
4107 unsigned CondOpc = CondDef->getOpcode();
4108 if (CondOpc != TargetOpcode::G_ICMP && CondOpc != TargetOpcode::G_FCMP)
4109 return false;
4110
4111 AArch64CC::CondCode CondCode;
4112 if (CondOpc == TargetOpcode::G_ICMP) {
4113 MachineInstr *Cmp;
4114 CmpInst::Predicate Pred;
4115
4116 std::tie(Cmp, Pred) =
4117 emitIntegerCompare(CondDef->getOperand(2), CondDef->getOperand(3),
4118 CondDef->getOperand(1), MIB);
4119
4120 if (!Cmp) {
4121 LLVM_DEBUG(dbgs() << "Couldn't emit compare for select!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't emit compare for select!\n"
; } } while (false)
;
4122 return false;
4123 }
4124
4125 // Have to collect the CondCode after emitIntegerCompare, since it can
4126 // update the predicate.
4127 CondCode = changeICMPPredToAArch64CC(Pred);
4128 } else {
4129 // Get the condition code for the select.
4130 AArch64CC::CondCode CondCode2;
4131 changeFCMPPredToAArch64CC(
4132 (CmpInst::Predicate)CondDef->getOperand(1).getPredicate(), CondCode,
4133 CondCode2);
4134
4135 // changeFCMPPredToAArch64CC sets CondCode2 to AL when we require two
4136 // instructions to emit the comparison.
4137 // TODO: Handle FCMP_UEQ and FCMP_ONE. After that, this check will be
4138 // unnecessary.
4139 if (CondCode2 != AArch64CC::AL)
4140 return false;
4141
4142 // Make sure we'll be able to select the compare.
4143 unsigned CmpOpc = selectFCMPOpc(*CondDef, MRI);
4144 if (!CmpOpc)
4145 return false;
4146
4147 // Emit a new compare.
4148 auto Cmp = MIB.buildInstr(CmpOpc, {}, {CondDef->getOperand(2).getReg()});
4149 if (CmpOpc != AArch64::FCMPSri && CmpOpc != AArch64::FCMPDri)
4150 Cmp.addUse(CondDef->getOperand(3).getReg());
4151 constrainSelectedInstRegOperands(*Cmp, TII, TRI, RBI);
4152 }
4153
4154 // Emit the select.
4155 unsigned CSelOpc = selectSelectOpc(I, MRI, RBI);
4156 auto CSel =
4157 MIB.buildInstr(CSelOpc, {I.getOperand(0).getReg()},
4158 {I.getOperand(2).getReg(), I.getOperand(3).getReg()})
4159 .addImm(CondCode);
4160 constrainSelectedInstRegOperands(*CSel, TII, TRI, RBI);
4161 I.eraseFromParent();
4162 return true;
4163}
4164
4165MachineInstr *AArch64InstructionSelector::tryFoldIntegerCompare(
4166 MachineOperand &LHS, MachineOperand &RHS, MachineOperand &Predicate,
4167 MachineIRBuilder &MIRBuilder) const {
4168 assert(LHS.isReg() && RHS.isReg() && Predicate.isPredicate() &&((LHS.isReg() && RHS.isReg() && Predicate.isPredicate
() && "Unexpected MachineOperand") ? static_cast<void
> (0) : __assert_fail ("LHS.isReg() && RHS.isReg() && Predicate.isPredicate() && \"Unexpected MachineOperand\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4169, __PRETTY_FUNCTION__))
4169 "Unexpected MachineOperand")((LHS.isReg() && RHS.isReg() && Predicate.isPredicate
() && "Unexpected MachineOperand") ? static_cast<void
> (0) : __assert_fail ("LHS.isReg() && RHS.isReg() && Predicate.isPredicate() && \"Unexpected MachineOperand\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4169, __PRETTY_FUNCTION__))
;
4170 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4171 // We want to find this sort of thing:
4172 // x = G_SUB 0, y
4173 // G_ICMP z, x
4174 //
4175 // In this case, we can fold the G_SUB into the G_ICMP using a CMN instead.
4176 // e.g:
4177 //
4178 // cmn z, y
4179
4180 // Helper lambda to detect the subtract followed by the compare.
4181 // Takes in the def of the LHS or RHS, and checks if it's a subtract from 0.
4182 auto IsCMN = [&](MachineInstr *DefMI, const AArch64CC::CondCode &CC) {
4183 if (!DefMI || DefMI->getOpcode() != TargetOpcode::G_SUB)
4184 return false;
4185
4186 // Need to make sure NZCV is the same at the end of the transformation.
4187 if (CC != AArch64CC::EQ && CC != AArch64CC::NE)
4188 return false;
4189
4190 // We want to match against SUBs.
4191 if (DefMI->getOpcode() != TargetOpcode::G_SUB)
4192 return false;
4193
4194 // Make sure that we're getting
4195 // x = G_SUB 0, y
4196 auto ValAndVReg =
4197 getConstantVRegValWithLookThrough(DefMI->getOperand(1).getReg(), MRI);
4198 if (!ValAndVReg || ValAndVReg->Value != 0)
4199 return false;
4200
4201 // This can safely be represented as a CMN.
4202 return true;
4203 };
4204
4205 // Check if the RHS or LHS of the G_ICMP is defined by a SUB
4206 MachineInstr *LHSDef = getDefIgnoringCopies(LHS.getReg(), MRI);
4207 MachineInstr *RHSDef = getDefIgnoringCopies(RHS.getReg(), MRI);
4208 CmpInst::Predicate P = (CmpInst::Predicate)Predicate.getPredicate();
4209 const AArch64CC::CondCode CC = changeICMPPredToAArch64CC(P);
4210
4211 // Given this:
4212 //
4213 // x = G_SUB 0, y
4214 // G_ICMP x, z
4215 //
4216 // Produce this:
4217 //
4218 // cmn y, z
4219 if (IsCMN(LHSDef, CC))
4220 return emitCMN(LHSDef->getOperand(2), RHS, MIRBuilder);
4221
4222 // Same idea here, but with the RHS of the compare instead:
4223 //
4224 // Given this:
4225 //
4226 // x = G_SUB 0, y
4227 // G_ICMP z, x
4228 //
4229 // Produce this:
4230 //
4231 // cmn z, y
4232 if (IsCMN(RHSDef, CC))
4233 return emitCMN(LHS, RHSDef->getOperand(2), MIRBuilder);
4234
4235 // Given this:
4236 //
4237 // z = G_AND x, y
4238 // G_ICMP z, 0
4239 //
4240 // Produce this if the compare is signed:
4241 //
4242 // tst x, y
4243 if (!isUnsignedICMPPred(P) && LHSDef &&
4244 LHSDef->getOpcode() == TargetOpcode::G_AND) {
4245 // Make sure that the RHS is 0.
4246 auto ValAndVReg = getConstantVRegValWithLookThrough(RHS.getReg(), MRI);
4247 if (!ValAndVReg || ValAndVReg->Value != 0)
4248 return nullptr;
4249
4250 return emitTST(LHSDef->getOperand(1).getReg(),
4251 LHSDef->getOperand(2).getReg(), MIRBuilder);
4252 }
4253
4254 return nullptr;
4255}
4256
4257MachineInstr *AArch64InstructionSelector::tryOptArithImmedIntegerCompare(
4258 MachineOperand &LHS, MachineOperand &RHS, CmpInst::Predicate &P,
4259 MachineIRBuilder &MIB) const {
4260 // Attempt to select the immediate form of an integer compare.
4261 MachineRegisterInfo &MRI = *MIB.getMRI();
4262 auto Ty = MRI.getType(LHS.getReg());
4263 assert(!Ty.isVector() && "Expected scalar or pointer only?")((!Ty.isVector() && "Expected scalar or pointer only?"
) ? static_cast<void> (0) : __assert_fail ("!Ty.isVector() && \"Expected scalar or pointer only?\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4263, __PRETTY_FUNCTION__))
;
4264 unsigned Size = Ty.getSizeInBits();
4265 assert((Size == 32 || Size == 64) &&(((Size == 32 || Size == 64) && "Expected 32 bit or 64 bit compare only?"
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Expected 32 bit or 64 bit compare only?\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4266, __PRETTY_FUNCTION__))
4266 "Expected 32 bit or 64 bit compare only?")(((Size == 32 || Size == 64) && "Expected 32 bit or 64 bit compare only?"
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Expected 32 bit or 64 bit compare only?\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4266, __PRETTY_FUNCTION__))
;
4267
4268 // Check if this is a case we can already handle.
4269 InstructionSelector::ComplexRendererFns ImmFns;
4270 ImmFns = selectArithImmed(RHS);
4271
4272 if (!ImmFns) {
4273 // We didn't get a rendering function, but we may still have a constant.
4274 auto MaybeImmed = getImmedFromMO(RHS);
4275 if (!MaybeImmed)
4276 return nullptr;
4277
4278 // We have a constant, but it doesn't fit. Try adjusting it by one and
4279 // updating the predicate if possible.
4280 uint64_t C = *MaybeImmed;
4281 CmpInst::Predicate NewP;
4282 switch (P) {
4283 default:
4284 return nullptr;
4285 case CmpInst::ICMP_SLT:
4286 case CmpInst::ICMP_SGE:
4287 // Check for
4288 //
4289 // x slt c => x sle c - 1
4290 // x sge c => x sgt c - 1
4291 //
4292 // When c is not the smallest possible negative number.
4293 if ((Size == 64 && static_cast<int64_t>(C) == INT64_MIN(-9223372036854775807L -1)) ||
4294 (Size == 32 && static_cast<int32_t>(C) == INT32_MIN(-2147483647-1)))
4295 return nullptr;
4296 NewP = (P == CmpInst::ICMP_SLT) ? CmpInst::ICMP_SLE : CmpInst::ICMP_SGT;
4297 C -= 1;
4298 break;
4299 case CmpInst::ICMP_ULT:
4300 case CmpInst::ICMP_UGE:
4301 // Check for
4302 //
4303 // x ult c => x ule c - 1
4304 // x uge c => x ugt c - 1
4305 //
4306 // When c is not zero.
4307 if (C == 0)
4308 return nullptr;
4309 NewP = (P == CmpInst::ICMP_ULT) ? CmpInst::ICMP_ULE : CmpInst::ICMP_UGT;
4310 C -= 1;
4311 break;
4312 case CmpInst::ICMP_SLE:
4313 case CmpInst::ICMP_SGT:
4314 // Check for
4315 //
4316 // x sle c => x slt c + 1
4317 // x sgt c => s sge c + 1
4318 //
4319 // When c is not the largest possible signed integer.
4320 if ((Size == 32 && static_cast<int32_t>(C) == INT32_MAX(2147483647)) ||
4321 (Size == 64 && static_cast<int64_t>(C) == INT64_MAX(9223372036854775807L)))
4322 return nullptr;
4323 NewP = (P == CmpInst::ICMP_SLE) ? CmpInst::ICMP_SLT : CmpInst::ICMP_SGE;
4324 C += 1;
4325 break;
4326 case CmpInst::ICMP_ULE:
4327 case CmpInst::ICMP_UGT:
4328 // Check for
4329 //
4330 // x ule c => x ult c + 1
4331 // x ugt c => s uge c + 1
4332 //
4333 // When c is not the largest possible unsigned integer.
4334 if ((Size == 32 && static_cast<uint32_t>(C) == UINT32_MAX(4294967295U)) ||
4335 (Size == 64 && C == UINT64_MAX(18446744073709551615UL)))
4336 return nullptr;
4337 NewP = (P == CmpInst::ICMP_ULE) ? CmpInst::ICMP_ULT : CmpInst::ICMP_UGE;
4338 C += 1;
4339 break;
4340 }
4341
4342 // Check if the new constant is valid.
4343 if (Size == 32)
4344 C = static_cast<uint32_t>(C);
4345 ImmFns = select12BitValueWithLeftShift(C);
4346 if (!ImmFns)
4347 return nullptr;
4348 P = NewP;
4349 }
4350
4351 // At this point, we know we can select an immediate form. Go ahead and do
4352 // that.
4353 Register ZReg;
4354 unsigned Opc;
4355 if (Size == 32) {
4356 ZReg = AArch64::WZR;
4357 Opc = AArch64::SUBSWri;
4358 } else {
4359 ZReg = AArch64::XZR;
4360 Opc = AArch64::SUBSXri;
4361 }
4362
4363 auto CmpMI = MIB.buildInstr(Opc, {ZReg}, {LHS.getReg()});
4364 for (auto &RenderFn : *ImmFns)
4365 RenderFn(CmpMI);
4366 constrainSelectedInstRegOperands(*CmpMI, TII, TRI, RBI);
4367 return &*CmpMI;
4368}
4369
4370MachineInstr *AArch64InstructionSelector::tryOptArithShiftedCompare(
4371 MachineOperand &LHS, MachineOperand &RHS, MachineIRBuilder &MIB) const {
4372 // We are looking for the following pattern:
4373 //
4374 // shift = G_SHL/ASHR/LHSR y, c
4375 // ...
4376 // cmp = G_ICMP pred, something, shift
4377 //
4378 // Since we will select the G_ICMP to a SUBS, we can potentially fold the
4379 // shift into the subtract.
4380 static const unsigned OpcTable[2] = {AArch64::SUBSWrs, AArch64::SUBSXrs};
4381 static const Register ZRegTable[2] = {AArch64::WZR, AArch64::XZR};
4382 auto ImmFns = selectShiftedRegister(RHS);
4383 if (!ImmFns)
4384 return nullptr;
4385 MachineRegisterInfo &MRI = *MIB.getMRI();
4386 auto Ty = MRI.getType(LHS.getReg());
4387 assert(!Ty.isVector() && "Expected scalar or pointer only?")((!Ty.isVector() && "Expected scalar or pointer only?"
) ? static_cast<void> (0) : __assert_fail ("!Ty.isVector() && \"Expected scalar or pointer only?\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4387, __PRETTY_FUNCTION__))
;
4388 unsigned Size = Ty.getSizeInBits();
4389 bool Idx = (Size == 64);
4390 Register ZReg = ZRegTable[Idx];
4391 unsigned Opc = OpcTable[Idx];
4392 auto CmpMI = MIB.buildInstr(Opc, {ZReg}, {LHS.getReg()});
4393 for (auto &RenderFn : *ImmFns)
4394 RenderFn(CmpMI);
4395 constrainSelectedInstRegOperands(*CmpMI, TII, TRI, RBI);
4396 return &*CmpMI;
4397}
4398
4399bool AArch64InstructionSelector::tryOptShuffleDupLane(
4400 MachineInstr &I, LLT DstTy, LLT SrcTy, ArrayRef<int> Mask,
4401 MachineRegisterInfo &MRI) const {
4402 assert(I.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR)((I.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR) ? static_cast
<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR"
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4402, __PRETTY_FUNCTION__))
;
4403
4404 // We assume that scalar->vector splats have been been handled in the
4405 // post-legalizer combiner to G_DUP. However splats of a source vector's
4406 // lane don't fit that pattern, detect it here:
4407 // %res = G_SHUFFLE_VECTOR %src:<n x ty>, undef, <n x i32> splat(lane-idx)
4408 // =>
4409 // %res = DUPv[N][Ty]lane %src, lane-idx
4410 // FIXME: this case should be covered by re-implementing the perfect shuffle
4411 // codegen mechanism.
4412
4413 auto LaneIdx = getSplatIndex(I);
4414 if (!LaneIdx)
4415 return false;
4416
4417 // The lane idx should be within the first source vector.
4418 if (*LaneIdx >= SrcTy.getNumElements())
4419 return false;
4420
4421 if (DstTy != SrcTy)
4422 return false;
4423
4424 LLT ScalarTy = SrcTy.getElementType();
4425 unsigned ScalarSize = ScalarTy.getSizeInBits();
4426
4427 unsigned Opc = 0;
4428 switch (SrcTy.getNumElements()) {
4429 case 2:
4430 if (ScalarSize == 64)
4431 Opc = AArch64::DUPv2i64lane;
4432 break;
4433 case 4:
4434 if (ScalarSize == 32)
4435 Opc = AArch64::DUPv4i32lane;
4436 break;
4437 case 8:
4438 if (ScalarSize == 16)
4439 Opc = AArch64::DUPv8i16lane;
4440 break;
4441 case 16:
4442 if (ScalarSize == 8)
4443 Opc = AArch64::DUPv16i8lane;
4444 break;
4445 default:
4446 break;
4447 }
4448 if (!Opc)
4449 return false;
4450
4451 MachineIRBuilder MIB(I);
4452 auto Dup = MIB.buildInstr(Opc, {I.getOperand(0).getReg()},
4453 {I.getOperand(1).getReg()})
4454 .addImm(*LaneIdx);
4455 constrainSelectedInstRegOperands(*Dup, TII, TRI, RBI);
4456 I.eraseFromParent();
4457 return true;
4458}
4459
4460bool AArch64InstructionSelector::selectShuffleVector(
4461 MachineInstr &I, MachineRegisterInfo &MRI) const {
4462 const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
4463 Register Src1Reg = I.getOperand(1).getReg();
4464 const LLT Src1Ty = MRI.getType(Src1Reg);
4465 Register Src2Reg = I.getOperand(2).getReg();
4466 const LLT Src2Ty = MRI.getType(Src2Reg);
4467 ArrayRef<int> Mask = I.getOperand(3).getShuffleMask();
4468
4469 MachineBasicBlock &MBB = *I.getParent();
4470 MachineFunction &MF = *MBB.getParent();
4471 LLVMContext &Ctx = MF.getFunction().getContext();
4472
4473 // G_SHUFFLE_VECTOR is weird in that the source operands can be scalars, if
4474 // it's originated from a <1 x T> type. Those should have been lowered into
4475 // G_BUILD_VECTOR earlier.
4476 if (!Src1Ty.isVector() || !Src2Ty.isVector()) {
4477 LLVM_DEBUG(dbgs() << "Could not select a \"scalar\" G_SHUFFLE_VECTOR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not select a \"scalar\" G_SHUFFLE_VECTOR\n"
; } } while (false)
;
4478 return false;
4479 }
4480
4481 if (tryOptShuffleDupLane(I, DstTy, Src1Ty, Mask, MRI))
4482 return true;
4483
4484 unsigned BytesPerElt = DstTy.getElementType().getSizeInBits() / 8;
4485
4486 SmallVector<Constant *, 64> CstIdxs;
4487 for (int Val : Mask) {
4488 // For now, any undef indexes we'll just assume to be 0. This should be
4489 // optimized in future, e.g. to select DUP etc.
4490 Val = Val < 0 ? 0 : Val;
4491 for (unsigned Byte = 0; Byte < BytesPerElt; ++Byte) {
4492 unsigned Offset = Byte + Val * BytesPerElt;
4493 CstIdxs.emplace_back(ConstantInt::get(Type::getInt8Ty(Ctx), Offset));
4494 }
4495 }
4496
4497 MachineIRBuilder MIRBuilder(I);
4498
4499 // Use a constant pool to load the index vector for TBL.
4500 Constant *CPVal = ConstantVector::get(CstIdxs);
4501 MachineInstr *IndexLoad = emitLoadFromConstantPool(CPVal, MIRBuilder);
4502 if (!IndexLoad) {
4503 LLVM_DEBUG(dbgs() << "Could not load from a constant pool")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not load from a constant pool"
; } } while (false)
;
4504 return false;
4505 }
4506
4507 if (DstTy.getSizeInBits() != 128) {
4508 assert(DstTy.getSizeInBits() == 64 && "Unexpected shuffle result ty")((DstTy.getSizeInBits() == 64 && "Unexpected shuffle result ty"
) ? static_cast<void> (0) : __assert_fail ("DstTy.getSizeInBits() == 64 && \"Unexpected shuffle result ty\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4508, __PRETTY_FUNCTION__))
;
4509 // This case can be done with TBL1.
4510 MachineInstr *Concat = emitVectorConcat(None, Src1Reg, Src2Reg, MIRBuilder);
4511 if (!Concat) {
4512 LLVM_DEBUG(dbgs() << "Could not do vector concat for tbl1")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not do vector concat for tbl1"
; } } while (false)
;
4513 return false;
4514 }
4515
4516 // The constant pool load will be 64 bits, so need to convert to FPR128 reg.
4517 IndexLoad =
4518 emitScalarToVector(64, &AArch64::FPR128RegClass,
4519 IndexLoad->getOperand(0).getReg(), MIRBuilder);
4520
4521 auto TBL1 = MIRBuilder.buildInstr(
4522 AArch64::TBLv16i8One, {&AArch64::FPR128RegClass},
4523 {Concat->getOperand(0).getReg(), IndexLoad->getOperand(0).getReg()});
4524 constrainSelectedInstRegOperands(*TBL1, TII, TRI, RBI);
4525
4526 auto Copy =
4527 MIRBuilder
4528 .buildInstr(TargetOpcode::COPY, {I.getOperand(0).getReg()}, {})
4529 .addReg(TBL1.getReg(0), 0, AArch64::dsub);
4530 RBI.constrainGenericRegister(Copy.getReg(0), AArch64::FPR64RegClass, MRI);
4531 I.eraseFromParent();
4532 return true;
4533 }
4534
4535 // For TBL2 we need to emit a REG_SEQUENCE to tie together two consecutive
4536 // Q registers for regalloc.
4537 auto RegSeq = MIRBuilder
4538 .buildInstr(TargetOpcode::REG_SEQUENCE,
4539 {&AArch64::QQRegClass}, {Src1Reg})
4540 .addImm(AArch64::qsub0)
4541 .addUse(Src2Reg)
4542 .addImm(AArch64::qsub1);
4543
4544 auto TBL2 = MIRBuilder.buildInstr(AArch64::TBLv16i8Two, {I.getOperand(0)},
4545 {RegSeq, IndexLoad->getOperand(0)});
4546 constrainSelectedInstRegOperands(*RegSeq, TII, TRI, RBI);
4547 constrainSelectedInstRegOperands(*TBL2, TII, TRI, RBI);
4548 I.eraseFromParent();
4549 return true;
4550}
4551
4552MachineInstr *AArch64InstructionSelector::emitLaneInsert(
4553 Optional<Register> DstReg, Register SrcReg, Register EltReg,
4554 unsigned LaneIdx, const RegisterBank &RB,
4555 MachineIRBuilder &MIRBuilder) const {
4556 MachineInstr *InsElt = nullptr;
4557 const TargetRegisterClass *DstRC = &AArch64::FPR128RegClass;
4558 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4559
4560 // Create a register to define with the insert if one wasn't passed in.
4561 if (!DstReg)
4562 DstReg = MRI.createVirtualRegister(DstRC);
4563
4564 unsigned EltSize = MRI.getType(EltReg).getSizeInBits();
4565 unsigned Opc = getInsertVecEltOpInfo(RB, EltSize).first;
4566
4567 if (RB.getID() == AArch64::FPRRegBankID) {
4568 auto InsSub = emitScalarToVector(EltSize, DstRC, EltReg, MIRBuilder);
4569 InsElt = MIRBuilder.buildInstr(Opc, {*DstReg}, {SrcReg})
4570 .addImm(LaneIdx)
4571 .addUse(InsSub->getOperand(0).getReg())
4572 .addImm(0);
4573 } else {
4574 InsElt = MIRBuilder.buildInstr(Opc, {*DstReg}, {SrcReg})
4575 .addImm(LaneIdx)
4576 .addUse(EltReg);
4577 }
4578
4579 constrainSelectedInstRegOperands(*InsElt, TII, TRI, RBI);
4580 return InsElt;
4581}
4582
4583bool AArch64InstructionSelector::selectInsertElt(
4584 MachineInstr &I, MachineRegisterInfo &MRI) const {
4585 assert(I.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT)((I.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT) ? static_cast
<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT"
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4585, __PRETTY_FUNCTION__))
;
4586
4587 // Get information on the destination.
4588 Register DstReg = I.getOperand(0).getReg();
4589 const LLT DstTy = MRI.getType(DstReg);
4590 unsigned VecSize = DstTy.getSizeInBits();
4591
4592 // Get information on the element we want to insert into the destination.
4593 Register EltReg = I.getOperand(2).getReg();
4594 const LLT EltTy = MRI.getType(EltReg);
4595 unsigned EltSize = EltTy.getSizeInBits();
4596 if (EltSize < 16 || EltSize > 64)
4597 return false; // Don't support all element types yet.
4598
4599 // Find the definition of the index. Bail out if it's not defined by a
4600 // G_CONSTANT.
4601 Register IdxReg = I.getOperand(3).getReg();
4602 auto VRegAndVal = getConstantVRegValWithLookThrough(IdxReg, MRI);
4603 if (!VRegAndVal)
4604 return false;
4605 unsigned LaneIdx = VRegAndVal->Value;
4606
4607 // Perform the lane insert.
4608 Register SrcReg = I.getOperand(1).getReg();
4609 const RegisterBank &EltRB = *RBI.getRegBank(EltReg, MRI, TRI);
4610 MachineIRBuilder MIRBuilder(I);
4611
4612 if (VecSize < 128) {
4613 // If the vector we're inserting into is smaller than 128 bits, widen it
4614 // to 128 to do the insert.
4615 MachineInstr *ScalarToVec = emitScalarToVector(
4616 VecSize, &AArch64::FPR128RegClass, SrcReg, MIRBuilder);
4617 if (!ScalarToVec)
4618 return false;
4619 SrcReg = ScalarToVec->getOperand(0).getReg();
4620 }
4621
4622 // Create an insert into a new FPR128 register.
4623 // Note that if our vector is already 128 bits, we end up emitting an extra
4624 // register.
4625 MachineInstr *InsMI =
4626 emitLaneInsert(None, SrcReg, EltReg, LaneIdx, EltRB, MIRBuilder);
4627
4628 if (VecSize < 128) {
4629 // If we had to widen to perform the insert, then we have to demote back to
4630 // the original size to get the result we want.
4631 Register DemoteVec = InsMI->getOperand(0).getReg();
4632 const TargetRegisterClass *RC =
4633 getMinClassForRegBank(*RBI.getRegBank(DemoteVec, MRI, TRI), VecSize);
4634 if (RC != &AArch64::FPR32RegClass && RC != &AArch64::FPR64RegClass) {
4635 LLVM_DEBUG(dbgs() << "Unsupported register class!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported register class!\n"
; } } while (false)
;
4636 return false;
4637 }
4638 unsigned SubReg = 0;
4639 if (!getSubRegForClass(RC, TRI, SubReg))
4640 return false;
4641 if (SubReg != AArch64::ssub && SubReg != AArch64::dsub) {
4642 LLVM_DEBUG(dbgs() << "Unsupported destination size! (" << VecSizedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported destination size! ("
<< VecSize << "\n"; } } while (false)
4643 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported destination size! ("
<< VecSize << "\n"; } } while (false)
;
4644 return false;
4645 }
4646 MIRBuilder.buildInstr(TargetOpcode::COPY, {DstReg}, {})
4647 .addReg(DemoteVec, 0, SubReg);
4648 RBI.constrainGenericRegister(DstReg, *RC, MRI);
4649 } else {
4650 // No widening needed.
4651 InsMI->getOperand(0).setReg(DstReg);
4652 constrainSelectedInstRegOperands(*InsMI, TII, TRI, RBI);
4653 }
4654
4655 I.eraseFromParent();
4656 return true;
4657}
4658
4659bool AArch64InstructionSelector::tryOptConstantBuildVec(
4660 MachineInstr &I, LLT DstTy, MachineRegisterInfo &MRI) const {
4661 assert(I.getOpcode() == TargetOpcode::G_BUILD_VECTOR)((I.getOpcode() == TargetOpcode::G_BUILD_VECTOR) ? static_cast
<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BUILD_VECTOR"
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4661, __PRETTY_FUNCTION__))
;
4662 assert(DstTy.getSizeInBits() <= 128 && "Unexpected build_vec type!")((DstTy.getSizeInBits() <= 128 && "Unexpected build_vec type!"
) ? static_cast<void> (0) : __assert_fail ("DstTy.getSizeInBits() <= 128 && \"Unexpected build_vec type!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4662, __PRETTY_FUNCTION__))
;
4663 if (DstTy.getSizeInBits() < 32)
4664 return false;
4665 // Check if we're building a constant vector, in which case we want to
4666 // generate a constant pool load instead of a vector insert sequence.
4667 SmallVector<Constant *, 16> Csts;
4668 for (unsigned Idx = 1; Idx < I.getNumOperands(); ++Idx) {
4669 // Try to find G_CONSTANT or G_FCONSTANT
4670 auto *OpMI =
4671 getOpcodeDef(TargetOpcode::G_CONSTANT, I.getOperand(Idx).getReg(), MRI);
4672 if (OpMI)
4673 Csts.emplace_back(
4674 const_cast<ConstantInt *>(OpMI->getOperand(1).getCImm()));
4675 else if ((OpMI = getOpcodeDef(TargetOpcode::G_FCONSTANT,
4676 I.getOperand(Idx).getReg(), MRI)))
4677 Csts.emplace_back(
4678 const_cast<ConstantFP *>(OpMI->getOperand(1).getFPImm()));
4679 else
4680 return false;
4681 }
4682 Constant *CV = ConstantVector::get(Csts);
4683 MachineIRBuilder MIB(I);
4684 auto *CPLoad = emitLoadFromConstantPool(CV, MIB);
4685 if (!CPLoad) {
4686 LLVM_DEBUG(dbgs() << "Could not generate cp load for build_vector")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not generate cp load for build_vector"
; } } while (false)
;
4687 return false;
4688 }
4689 MIB.buildCopy(I.getOperand(0), CPLoad->getOperand(0));
4690 RBI.constrainGenericRegister(I.getOperand(0).getReg(),
4691 *MRI.getRegClass(CPLoad->getOperand(0).getReg()),
4692 MRI);
4693 I.eraseFromParent();
4694 return true;
4695}
4696
4697bool AArch64InstructionSelector::selectBuildVector(
4698 MachineInstr &I, MachineRegisterInfo &MRI) const {
4699 assert(I.getOpcode() == TargetOpcode::G_BUILD_VECTOR)((I.getOpcode() == TargetOpcode::G_BUILD_VECTOR) ? static_cast
<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BUILD_VECTOR"
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4699, __PRETTY_FUNCTION__))
;
4700 // Until we port more of the optimized selections, for now just use a vector
4701 // insert sequence.
4702 const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
4703 const LLT EltTy = MRI.getType(I.getOperand(1).getReg());
4704 unsigned EltSize = EltTy.getSizeInBits();
4705
4706 if (tryOptConstantBuildVec(I, DstTy, MRI))
4707 return true;
4708 if (EltSize < 16 || EltSize > 64)
4709 return false; // Don't support all element types yet.
4710 const RegisterBank &RB = *RBI.getRegBank(I.getOperand(1).getReg(), MRI, TRI);
4711 MachineIRBuilder MIRBuilder(I);
4712
4713 const TargetRegisterClass *DstRC = &AArch64::FPR128RegClass;
4714 MachineInstr *ScalarToVec =
4715 emitScalarToVector(DstTy.getElementType().getSizeInBits(), DstRC,
4716 I.getOperand(1).getReg(), MIRBuilder);
4717 if (!ScalarToVec)
4718 return false;
4719
4720 Register DstVec = ScalarToVec->getOperand(0).getReg();
4721 unsigned DstSize = DstTy.getSizeInBits();
4722
4723 // Keep track of the last MI we inserted. Later on, we might be able to save
4724 // a copy using it.
4725 MachineInstr *PrevMI = nullptr;
4726 for (unsigned i = 2, e = DstSize / EltSize + 1; i < e; ++i) {
4727 // Note that if we don't do a subregister copy, we can end up making an
4728 // extra register.
4729 PrevMI = &*emitLaneInsert(None, DstVec, I.getOperand(i).getReg(), i - 1, RB,
4730 MIRBuilder);
4731 DstVec = PrevMI->getOperand(0).getReg();
4732 }
4733
4734 // If DstTy's size in bits is less than 128, then emit a subregister copy
4735 // from DstVec to the last register we've defined.
4736 if (DstSize < 128) {
4737 // Force this to be FPR using the destination vector.
4738 const TargetRegisterClass *RC =
4739 getMinClassForRegBank(*RBI.getRegBank(DstVec, MRI, TRI), DstSize);
4740 if (!RC)
4741 return false;
4742 if (RC != &AArch64::FPR32RegClass && RC != &AArch64::FPR64RegClass) {
4743 LLVM_DEBUG(dbgs() << "Unsupported register class!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported register class!\n"
; } } while (false)
;
4744 return false;
4745 }
4746
4747 unsigned SubReg = 0;
4748 if (!getSubRegForClass(RC, TRI, SubReg))
4749 return false;
4750 if (SubReg != AArch64::ssub && SubReg != AArch64::dsub) {
4751 LLVM_DEBUG(dbgs() << "Unsupported destination size! (" << DstSizedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported destination size! ("
<< DstSize << "\n"; } } while (false)
4752 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported destination size! ("
<< DstSize << "\n"; } } while (false)
;
4753 return false;
4754 }
4755
4756 Register Reg = MRI.createVirtualRegister(RC);
4757 Register DstReg = I.getOperand(0).getReg();
4758
4759 MIRBuilder.buildInstr(TargetOpcode::COPY, {DstReg}, {})
4760 .addReg(DstVec, 0, SubReg);
4761 MachineOperand &RegOp = I.getOperand(1);
4762 RegOp.setReg(Reg);
4763 RBI.constrainGenericRegister(DstReg, *RC, MRI);
4764 } else {
4765 // We don't need a subregister copy. Save a copy by re-using the
4766 // destination register on the final insert.
4767 assert(PrevMI && "PrevMI was null?")((PrevMI && "PrevMI was null?") ? static_cast<void
> (0) : __assert_fail ("PrevMI && \"PrevMI was null?\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4767, __PRETTY_FUNCTION__))
;
4768 PrevMI->getOperand(0).setReg(I.getOperand(0).getReg());
4769 constrainSelectedInstRegOperands(*PrevMI, TII, TRI, RBI);
4770 }
4771
4772 I.eraseFromParent();
4773 return true;
4774}
4775
4776/// Helper function to find an intrinsic ID on an a MachineInstr. Returns the
4777/// ID if it exists, and 0 otherwise.
4778static unsigned findIntrinsicID(MachineInstr &I) {
4779 auto IntrinOp = find_if(I.operands(), [&](const MachineOperand &Op) {
4780 return Op.isIntrinsicID();
4781 });
4782 if (IntrinOp == I.operands_end())
4783 return 0;
4784 return IntrinOp->getIntrinsicID();
4785}
4786
4787bool AArch64InstructionSelector::selectIntrinsicWithSideEffects(
4788 MachineInstr &I, MachineRegisterInfo &MRI) const {
4789 // Find the intrinsic ID.
4790 unsigned IntrinID = findIntrinsicID(I);
4791 if (!IntrinID)
4792 return false;
4793 MachineIRBuilder MIRBuilder(I);
4794
4795 // Select the instruction.
4796 switch (IntrinID) {
4797 default:
4798 return false;
4799 case Intrinsic::trap:
4800 MIRBuilder.buildInstr(AArch64::BRK, {}, {}).addImm(1);
4801 break;
4802 case Intrinsic::debugtrap:
4803 MIRBuilder.buildInstr(AArch64::BRK, {}, {}).addImm(0xF000);
4804 break;
4805 }
4806
4807 I.eraseFromParent();
4808 return true;
4809}
4810
4811bool AArch64InstructionSelector::selectIntrinsic(MachineInstr &I,
4812 MachineRegisterInfo &MRI) {
4813 unsigned IntrinID = findIntrinsicID(I);
4814 if (!IntrinID)
4815 return false;
4816 MachineIRBuilder MIRBuilder(I);
4817
4818 switch (IntrinID) {
4819 default:
4820 break;
4821 case Intrinsic::aarch64_crypto_sha1h: {
4822 Register DstReg = I.getOperand(0).getReg();
4823 Register SrcReg = I.getOperand(2).getReg();
4824
4825 // FIXME: Should this be an assert?
4826 if (MRI.getType(DstReg).getSizeInBits() != 32 ||
4827 MRI.getType(SrcReg).getSizeInBits() != 32)
4828 return false;
4829
4830 // The operation has to happen on FPRs. Set up some new FPR registers for
4831 // the source and destination if they are on GPRs.
4832 if (RBI.getRegBank(SrcReg, MRI, TRI)->getID() != AArch64::FPRRegBankID) {
4833 SrcReg = MRI.createVirtualRegister(&AArch64::FPR32RegClass);
4834 MIRBuilder.buildCopy({SrcReg}, {I.getOperand(2)});
4835
4836 // Make sure the copy ends up getting constrained properly.
4837 RBI.constrainGenericRegister(I.getOperand(2).getReg(),
4838 AArch64::GPR32RegClass, MRI);
4839 }
4840
4841 if (RBI.getRegBank(DstReg, MRI, TRI)->getID() != AArch64::FPRRegBankID)
4842 DstReg = MRI.createVirtualRegister(&AArch64::FPR32RegClass);
4843
4844 // Actually insert the instruction.
4845 auto SHA1Inst = MIRBuilder.buildInstr(AArch64::SHA1Hrr, {DstReg}, {SrcReg});
4846 constrainSelectedInstRegOperands(*SHA1Inst, TII, TRI, RBI);
4847
4848 // Did we create a new register for the destination?
4849 if (DstReg != I.getOperand(0).getReg()) {
4850 // Yep. Copy the result of the instruction back into the original
4851 // destination.
4852 MIRBuilder.buildCopy({I.getOperand(0)}, {DstReg});
4853 RBI.constrainGenericRegister(I.getOperand(0).getReg(),
4854 AArch64::GPR32RegClass, MRI);
4855 }
4856
4857 I.eraseFromParent();
4858 return true;
4859 }
4860 case Intrinsic::frameaddress:
4861 case Intrinsic::returnaddress: {
4862 MachineFunction &MF = *I.getParent()->getParent();
4863 MachineFrameInfo &MFI = MF.getFrameInfo();
4864
4865 unsigned Depth = I.getOperand(2).getImm();
4866 Register DstReg = I.getOperand(0).getReg();
4867 RBI.constrainGenericRegister(DstReg, AArch64::GPR64RegClass, MRI);
4868
4869 if (Depth == 0 && IntrinID == Intrinsic::returnaddress) {
4870 if (MFReturnAddr) {
4871 MIRBuilder.buildCopy({DstReg}, MFReturnAddr);
4872 I.eraseFromParent();
4873 return true;
4874 }
4875
4876 MFI.setReturnAddressIsTaken(true);
4877
4878 // Insert the copy from LR/X30 into the entry block, before it can be
4879 // clobbered by anything.
4880 Register LiveInLR = getFunctionLiveInPhysReg(MF, TII, AArch64::LR,
4881 AArch64::GPR64spRegClass);
4882 MIRBuilder.buildCopy(DstReg, LiveInLR);
4883
4884 MFReturnAddr = LiveInLR;
4885 I.eraseFromParent();
4886 return true;
4887 }
4888
4889 MFI.setFrameAddressIsTaken(true);
4890 Register FrameAddr(AArch64::FP);
4891 while (Depth--) {
4892 Register NextFrame = MRI.createVirtualRegister(&AArch64::GPR64spRegClass);
4893 auto Ldr =
4894 MIRBuilder.buildInstr(AArch64::LDRXui, {NextFrame}, {FrameAddr})
4895 .addImm(0);
4896 constrainSelectedInstRegOperands(*Ldr, TII, TRI, RBI);
4897 FrameAddr = NextFrame;
4898 }
4899
4900 if (IntrinID == Intrinsic::frameaddress)
4901 MIRBuilder.buildCopy({DstReg}, {FrameAddr});
4902 else {
4903 MFI.setReturnAddressIsTaken(true);
4904 MIRBuilder.buildInstr(AArch64::LDRXui, {DstReg}, {FrameAddr}).addImm(1);
4905 }
4906
4907 I.eraseFromParent();
4908 return true;
4909 }
4910 }
4911 return false;
4912}
4913
4914InstructionSelector::ComplexRendererFns
4915AArch64InstructionSelector::selectShiftA_32(const MachineOperand &Root) const {
4916 auto MaybeImmed = getImmedFromMO(Root);
4917 if (MaybeImmed == None || *MaybeImmed > 31)
4918 return None;
4919 uint64_t Enc = (32 - *MaybeImmed) & 0x1f;
4920 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}};
4921}
4922
4923InstructionSelector::ComplexRendererFns
4924AArch64InstructionSelector::selectShiftB_32(const MachineOperand &Root) const {
4925 auto MaybeImmed = getImmedFromMO(Root);
4926 if (MaybeImmed == None || *MaybeImmed > 31)
4927 return None;
4928 uint64_t Enc = 31 - *MaybeImmed;
4929 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}};
4930}
4931
4932InstructionSelector::ComplexRendererFns
4933AArch64InstructionSelector::selectShiftA_64(const MachineOperand &Root) const {
4934 auto MaybeImmed = getImmedFromMO(Root);
4935 if (MaybeImmed == None || *MaybeImmed > 63)
4936 return None;
4937 uint64_t Enc = (64 - *MaybeImmed) & 0x3f;
4938 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}};
4939}
4940
4941InstructionSelector::ComplexRendererFns
4942AArch64InstructionSelector::selectShiftB_64(const MachineOperand &Root) const {
4943 auto MaybeImmed = getImmedFromMO(Root);
4944 if (MaybeImmed == None || *MaybeImmed > 63)
4945 return None;
4946 uint64_t Enc = 63 - *MaybeImmed;
4947 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}};
4948}
4949
4950/// Helper to select an immediate value that can be represented as a 12-bit
4951/// value shifted left by either 0 or 12. If it is possible to do so, return
4952/// the immediate and shift value. If not, return None.
4953///
4954/// Used by selectArithImmed and selectNegArithImmed.
4955InstructionSelector::ComplexRendererFns
4956AArch64InstructionSelector::select12BitValueWithLeftShift(
4957 uint64_t Immed) const {
4958 unsigned ShiftAmt;
4959 if (Immed >> 12 == 0) {
4960 ShiftAmt = 0;
4961 } else if ((Immed & 0xfff) == 0 && Immed >> 24 == 0) {
4962 ShiftAmt = 12;
4963 Immed = Immed >> 12;
4964 } else
4965 return None;
4966
4967 unsigned ShVal = AArch64_AM::getShifterImm(AArch64_AM::LSL, ShiftAmt);
4968 return {{
4969 [=](MachineInstrBuilder &MIB) { MIB.addImm(Immed); },
4970 [=](MachineInstrBuilder &MIB) { MIB.addImm(ShVal); },
4971 }};
4972}
4973
4974/// SelectArithImmed - Select an immediate value that can be represented as
4975/// a 12-bit value shifted left by either 0 or 12. If so, return true with
4976/// Val set to the 12-bit value and Shift set to the shifter operand.
4977InstructionSelector::ComplexRendererFns
4978AArch64InstructionSelector::selectArithImmed(MachineOperand &Root) const {
4979 // This function is called from the addsub_shifted_imm ComplexPattern,
4980 // which lists [imm] as the list of opcode it's interested in, however
4981 // we still need to check whether the operand is actually an immediate
4982 // here because the ComplexPattern opcode list is only used in
4983 // root-level opcode matching.
4984 auto MaybeImmed = getImmedFromMO(Root);
4985 if (MaybeImmed == None)
4986 return None;
4987 return select12BitValueWithLeftShift(*MaybeImmed);
4988}
4989
4990/// SelectNegArithImmed - As above, but negates the value before trying to
4991/// select it.
4992InstructionSelector::ComplexRendererFns
4993AArch64InstructionSelector::selectNegArithImmed(MachineOperand &Root) const {
4994 // We need a register here, because we need to know if we have a 64 or 32
4995 // bit immediate.
4996 if (!Root.isReg())
4997 return None;
4998 auto MaybeImmed = getImmedFromMO(Root);
4999 if (MaybeImmed == None)
5000 return None;
5001 uint64_t Immed = *MaybeImmed;
5002
5003 // This negation is almost always valid, but "cmp wN, #0" and "cmn wN, #0"
5004 // have the opposite effect on the C flag, so this pattern mustn't match under
5005 // those circumstances.
5006 if (Immed == 0)
5007 return None;
5008
5009 // Check if we're dealing with a 32-bit type on the root or a 64-bit type on
5010 // the root.
5011 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
5012 if (MRI.getType(Root.getReg()).getSizeInBits() == 32)
5013 Immed = ~((uint32_t)Immed) + 1;
5014 else
5015 Immed = ~Immed + 1ULL;
5016
5017 if (Immed & 0xFFFFFFFFFF000000ULL)
5018 return None;
5019
5020 Immed &= 0xFFFFFFULL;
5021 return select12BitValueWithLeftShift(Immed);
5022}
5023
5024/// Return true if it is worth folding MI into an extended register. That is,
5025/// if it's safe to pull it into the addressing mode of a load or store as a
5026/// shift.
5027bool AArch64InstructionSelector::isWorthFoldingIntoExtendedReg(
5028 MachineInstr &MI, const MachineRegisterInfo &MRI) const {
5029 // Always fold if there is one use, or if we're optimizing for size.
5030 Register DefReg = MI.getOperand(0).getReg();
5031 if (MRI.hasOneNonDBGUse(DefReg) ||
5032 MI.getParent()->getParent()->getFunction().hasMinSize())
5033 return true;
5034
5035 // It's better to avoid folding and recomputing shifts when we don't have a
5036 // fastpath.
5037 if (!STI.hasLSLFast())
5038 return false;
5039
5040 // We have a fastpath, so folding a shift in and potentially computing it
5041 // many times may be beneficial. Check if this is only used in memory ops.
5042 // If it is, then we should fold.
5043 return all_of(MRI.use_nodbg_instructions(DefReg),
5044 [](MachineInstr &Use) { return Use.mayLoadOrStore(); });
5045}
5046
5047static bool isSignExtendShiftType(AArch64_AM::ShiftExtendType Type) {
5048 switch (Type) {
5049 case AArch64_AM::SXTB:
5050 case AArch64_AM::SXTH:
5051 case AArch64_AM::SXTW:
5052 return true;
5053 default:
5054 return false;
5055 }
5056}
5057
5058InstructionSelector::ComplexRendererFns
5059AArch64InstructionSelector::selectExtendedSHL(
5060 MachineOperand &Root, MachineOperand &Base, MachineOperand &Offset,
5061 unsigned SizeInBytes, bool WantsExt) const {
5062 assert(Base.isReg() && "Expected base to be a register operand")((Base.isReg() && "Expected base to be a register operand"
) ? static_cast<void> (0) : __assert_fail ("Base.isReg() && \"Expected base to be a register operand\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5062, __PRETTY_FUNCTION__))
;
5063 assert(Offset.isReg() && "Expected offset to be a register operand")((Offset.isReg() && "Expected offset to be a register operand"
) ? static_cast<void> (0) : __assert_fail ("Offset.isReg() && \"Expected offset to be a register operand\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5063, __PRETTY_FUNCTION__))
;
5064
5065 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
5066 MachineInstr *OffsetInst = MRI.getVRegDef(Offset.getReg());
5067 if (!OffsetInst)
5068 return None;
5069
5070 unsigned OffsetOpc = OffsetInst->getOpcode();
5071 if (OffsetOpc != TargetOpcode::G_SHL && OffsetOpc != TargetOpcode::G_MUL)
5072 return None;
5073
5074 // Make sure that the memory op is a valid size.
5075 int64_t LegalShiftVal = Log2_32(SizeInBytes);
5076 if (LegalShiftVal == 0)
5077 return None;
5078 if (!isWorthFoldingIntoExtendedReg(*OffsetInst, MRI))
5079 return None;
5080
5081 // Now, try to find the specific G_CONSTANT. Start by assuming that the
5082 // register we will offset is the LHS, and the register containing the
5083 // constant is the RHS.
5084 Register OffsetReg = OffsetInst->getOperand(1).getReg();
5085 Register ConstantReg = OffsetInst->getOperand(2).getReg();
5086 auto ValAndVReg = getConstantVRegValWithLookThrough(ConstantReg, MRI);
5087 if (!ValAndVReg) {
5088 // We didn't get a constant on the RHS. If the opcode is a shift, then
5089 // we're done.
5090 if (OffsetOpc == TargetOpcode::G_SHL)
5091 return None;
5092
5093 // If we have a G_MUL, we can use either register. Try looking at the RHS.
5094 std::swap(OffsetReg, ConstantReg);
5095 ValAndVReg = getConstantVRegValWithLookThrough(ConstantReg, MRI);
5096 if (!ValAndVReg)
5097 return None;
5098 }
5099
5100 // The value must fit into 3 bits, and must be positive. Make sure that is
5101 // true.
5102 int64_t ImmVal = ValAndVReg->Value;
5103
5104 // Since we're going to pull this into a shift, the constant value must be
5105 // a power of 2. If we got a multiply, then we need to check this.
5106 if (OffsetOpc == TargetOpcode::G_MUL) {
5107 if (!isPowerOf2_32(ImmVal))
5108 return None;
5109
5110 // Got a power of 2. So, the amount we'll shift is the log base-2 of that.
5111 ImmVal = Log2_32(ImmVal);
5112 }
5113
5114 if ((ImmVal & 0x7) != ImmVal)
5115 return None;
5116
5117 // We are only allowed to shift by LegalShiftVal. This shift value is built
5118 // into the instruction, so we can't just use whatever we want.
5119 if (ImmVal != LegalShiftVal)
5120 return None;
5121
5122 unsigned SignExtend = 0;
5123 if (WantsExt) {
5124 // Check if the offset is defined by an extend.
5125 MachineInstr *ExtInst = getDefIgnoringCopies(OffsetReg, MRI);
5126 auto Ext = getExtendTypeForInst(*ExtInst, MRI, true);
5127 if (Ext == AArch64_AM::InvalidShiftExtend)
5128 return None;
5129
5130 SignExtend = isSignExtendShiftType(Ext) ? 1 : 0;
5131 // We only support SXTW for signed extension here.
5132 if (SignExtend && Ext != AArch64_AM::SXTW)
5133 return None;
5134
5135 // Need a 32-bit wide register here.
5136 MachineIRBuilder MIB(*MRI.getVRegDef(Root.getReg()));
5137 OffsetReg = ExtInst->getOperand(1).getReg();
5138 OffsetReg = narrowExtendRegIfNeeded(OffsetReg, MIB);
5139 }
5140
5141 // We can use the LHS of the GEP as the base, and the LHS of the shift as an
5142 // offset. Signify that we are shifting by setting the shift flag to 1.
5143 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(Base.getReg()); },
5144 [=](MachineInstrBuilder &MIB) { MIB.addUse(OffsetReg); },
5145 [=](MachineInstrBuilder &MIB) {
5146 // Need to add both immediates here to make sure that they are both
5147 // added to the instruction.
5148 MIB.addImm(SignExtend);
5149 MIB.addImm(1);
5150 }}};
5151}
5152
5153/// This is used for computing addresses like this:
5154///
5155/// ldr x1, [x2, x3, lsl #3]
5156///
5157/// Where x2 is the base register, and x3 is an offset register. The shift-left
5158/// is a constant value specific to this load instruction. That is, we'll never
5159/// see anything other than a 3 here (which corresponds to the size of the
5160/// element being loaded.)
5161InstructionSelector::ComplexRendererFns
5162AArch64InstructionSelector::selectAddrModeShiftedExtendXReg(
5163 MachineOperand &Root, unsigned SizeInBytes) const {
5164 if (!Root.isReg())
5165 return None;
5166 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
5167
5168 // We want to find something like this:
5169 //
5170 // val = G_CONSTANT LegalShiftVal
5171 // shift = G_SHL off_reg val
5172 // ptr = G_PTR_ADD base_reg shift
5173 // x = G_LOAD ptr
5174 //
5175 // And fold it into this addressing mode:
5176 //
5177 // ldr x, [base_reg, off_reg, lsl #LegalShiftVal]
5178
5179 // Check if we can find the G_PTR_ADD.
5180 MachineInstr *PtrAdd =
5181 getOpcodeDef(TargetOpcode::G_PTR_ADD, Root.getReg(), MRI);
5182 if (!PtrAdd || !isWorthFoldingIntoExtendedReg(*PtrAdd, MRI))
5183 return None;
5184
5185 // Now, try to match an opcode which will match our specific offset.
5186 // We want a G_SHL or a G_MUL.
5187 MachineInstr *OffsetInst =
5188 getDefIgnoringCopies(PtrAdd->getOperand(2).getReg(), MRI);
5189 return selectExtendedSHL(Root, PtrAdd->getOperand(1),
5190 OffsetInst->getOperand(0), SizeInBytes,
5191 /*WantsExt=*/false);
5192}
5193
5194/// This is used for computing addresses like this:
5195///
5196/// ldr x1, [x2, x3]
5197///
5198/// Where x2 is the base register, and x3 is an offset register.
5199///
5200/// When possible (or profitable) to fold a G_PTR_ADD into the address calculation,
5201/// this will do so. Otherwise, it will return None.
5202InstructionSelector::ComplexRendererFns
5203AArch64InstructionSelector::selectAddrModeRegisterOffset(
5204 MachineOperand &Root) const {
5205 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
5206
5207 // We need a GEP.
5208 MachineInstr *Gep = MRI.getVRegDef(Root.getReg());
5209 if (!Gep || Gep->getOpcode() != TargetOpcode::G_PTR_ADD)
5210 return None;
5211
5212 // If this is used more than once, let's not bother folding.
5213 // TODO: Check if they are memory ops. If they are, then we can still fold
5214 // without having to recompute anything.
5215 if (!MRI.hasOneNonDBGUse(Gep->getOperand(0).getReg()))
5216 return None;
5217
5218 // Base is the GEP's LHS, offset is its RHS.
5219 return {{[=](MachineInstrBuilder &MIB) {
5220 MIB.addUse(Gep->getOperand(1).getReg());
5221 },
5222 [=](MachineInstrBuilder &MIB) {
5223 MIB.addUse(Gep->getOperand(2).getReg());
5224 },
5225 [=](MachineInstrBuilder &MIB) {
5226 // Need to add both immediates here to make sure that they are both
5227 // added to the instruction.
5228 MIB.addImm(0);
5229 MIB.addImm(0);
5230 }}};
5231}
5232
5233/// This is intended to be equivalent to selectAddrModeXRO in
5234/// AArch64ISelDAGtoDAG. It's used for selecting X register offset loads.
5235InstructionSelector::ComplexRendererFns
5236AArch64InstructionSelector::selectAddrModeXRO(MachineOperand &Root,
5237 unsigned SizeInBytes) const {
5238 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
5239 if (!Root.isReg())
5240 return None;
5241 MachineInstr *PtrAdd =
5242 getOpcodeDef(TargetOpcode::G_PTR_ADD, Root.getReg(), MRI);
5243 if (!PtrAdd)
5244 return None;
5245
5246 // Check for an immediates which cannot be encoded in the [base + imm]
5247 // addressing mode, and can't be encoded in an add/sub. If this happens, we'll
5248 // end up with code like:
5249 //
5250 // mov x0, wide
5251 // add x1 base, x0
5252 // ldr x2, [x1, x0]
5253 //
5254 // In this situation, we can use the [base, xreg] addressing mode to save an
5255 // add/sub:
5256 //
5257 // mov x0, wide
5258 // ldr x2, [base, x0]
5259 auto ValAndVReg =
5260 getConstantVRegValWithLookThrough(PtrAdd->getOperand(2).getReg(), MRI);
5261 if (ValAndVReg) {
5262 unsigned Scale = Log2_32(SizeInBytes);
5263 int64_t ImmOff = ValAndVReg->Value;
5264
5265 // Skip immediates that can be selected in the load/store addresing
5266 // mode.
5267 if (ImmOff % SizeInBytes == 0 && ImmOff >= 0 &&
5268 ImmOff < (0x1000 << Scale))
5269 return None;
5270
5271 // Helper lambda to decide whether or not it is preferable to emit an add.
5272 auto isPreferredADD = [](int64_t ImmOff) {
5273 // Constants in [0x0, 0xfff] can be encoded in an add.
5274 if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL)
5275 return true;
5276
5277 // Can it be encoded in an add lsl #12?
5278 if ((ImmOff & 0xffffffffff000fffLL) != 0x0LL)
5279 return false;
5280
5281 // It can be encoded in an add lsl #12, but we may not want to. If it is
5282 // possible to select this as a single movz, then prefer that. A single
5283 // movz is faster than an add with a shift.
5284 return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL &&
5285 (ImmOff & 0xffffffffffff0fffLL) != 0x0LL;
5286 };
5287
5288 // If the immediate can be encoded in a single add/sub, then bail out.
5289 if (isPreferredADD(ImmOff) || isPreferredADD(-ImmOff))
5290 return None;
5291 }
5292
5293 // Try to fold shifts into the addressing mode.
5294 auto AddrModeFns = selectAddrModeShiftedExtendXReg(Root, SizeInBytes);
5295 if (AddrModeFns)
5296 return AddrModeFns;
5297
5298 // If that doesn't work, see if it's possible to fold in registers from
5299 // a GEP.
5300 return selectAddrModeRegisterOffset(Root);
5301}
5302
5303/// This is used for computing addresses like this:
5304///
5305/// ldr x0, [xBase, wOffset, sxtw #LegalShiftVal]
5306///
5307/// Where we have a 64-bit base register, a 32-bit offset register, and an
5308/// extend (which may or may not be signed).
5309InstructionSelector::ComplexRendererFns
5310AArch64InstructionSelector::selectAddrModeWRO(MachineOperand &Root,
5311 unsigned SizeInBytes) const {
5312 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
5313
5314 MachineInstr *PtrAdd =
5315 getOpcodeDef(TargetOpcode::G_PTR_ADD, Root.getReg(), MRI);
5316 if (!PtrAdd || !isWorthFoldingIntoExtendedReg(*PtrAdd, MRI))
5317 return None;
5318
5319 MachineOperand &LHS = PtrAdd->getOperand(1);
5320 MachineOperand &RHS = PtrAdd->getOperand(2);
5321 MachineInstr *OffsetInst = getDefIgnoringCopies(RHS.getReg(), MRI);
5322
5323 // The first case is the same as selectAddrModeXRO, except we need an extend.
5324 // In this case, we try to find a shift and extend, and fold them into the
5325 // addressing mode.
5326 //
5327 // E.g.
5328 //
5329 // off_reg = G_Z/S/ANYEXT ext_reg
5330 // val = G_CONSTANT LegalShiftVal
5331 // shift = G_SHL off_reg val
5332 // ptr = G_PTR_ADD base_reg shift
5333 // x = G_LOAD ptr
5334 //
5335 // In this case we can get a load like this:
5336 //
5337 // ldr x0, [base_reg, ext_reg, sxtw #LegalShiftVal]
5338 auto ExtendedShl = selectExtendedSHL(Root, LHS, OffsetInst->getOperand(0),
5339 SizeInBytes, /*WantsExt=*/true);
5340 if (ExtendedShl)
5341 return ExtendedShl;
5342
5343 // There was no shift. We can try and fold a G_Z/S/ANYEXT in alone though.
5344 //
5345 // e.g.
5346 // ldr something, [base_reg, ext_reg, sxtw]
5347 if (!isWorthFoldingIntoExtendedReg(*OffsetInst, MRI))
5348 return None;
5349
5350 // Check if this is an extend. We'll get an extend type if it is.
5351 AArch64_AM::ShiftExtendType Ext =
5352 getExtendTypeForInst(*OffsetInst, MRI, /*IsLoadStore=*/true);
5353 if (Ext == AArch64_AM::InvalidShiftExtend)
5354 return None;
5355
5356 // Need a 32-bit wide register.
5357 MachineIRBuilder MIB(*PtrAdd);
5358 Register ExtReg =
5359 narrowExtendRegIfNeeded(OffsetInst->getOperand(1).getReg(), MIB);
5360 unsigned SignExtend = Ext == AArch64_AM::SXTW;
5361
5362 // Base is LHS, offset is ExtReg.
5363 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(LHS.getReg()); },
5364 [=](MachineInstrBuilder &MIB) { MIB.addUse(ExtReg); },
5365 [=](MachineInstrBuilder &MIB) {
5366 MIB.addImm(SignExtend);
5367 MIB.addImm(0);
5368 }}};
5369}
5370
5371/// Select a "register plus unscaled signed 9-bit immediate" address. This
5372/// should only match when there is an offset that is not valid for a scaled
5373/// immediate addressing mode. The "Size" argument is the size in bytes of the
5374/// memory reference, which is needed here to know what is valid for a scaled
5375/// immediate.
5376InstructionSelector::ComplexRendererFns
5377AArch64InstructionSelector::selectAddrModeUnscaled(MachineOperand &Root,
5378 unsigned Size) const {
5379 MachineRegisterInfo &MRI =
5380 Root.getParent()->getParent()->getParent()->getRegInfo();
5381
5382 if (!Root.isReg())
5383 return None;
5384
5385 if (!isBaseWithConstantOffset(Root, MRI))
5386 return None;
5387
5388 MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());
5389 if (!RootDef)
5390 return None;
5391
5392 MachineOperand &OffImm = RootDef->getOperand(2);
5393 if (!OffImm.isReg())
5394 return None;
5395 MachineInstr *RHS = MRI.getVRegDef(OffImm.getReg());
5396 if (!RHS || RHS->getOpcode() != TargetOpcode::G_CONSTANT)
5397 return None;
5398 int64_t RHSC;
5399 MachineOperand &RHSOp1 = RHS->getOperand(1);
5400 if (!RHSOp1.isCImm() || RHSOp1.getCImm()->getBitWidth() > 64)
5401 return None;
5402 RHSC = RHSOp1.getCImm()->getSExtValue();
5403
5404 // If the offset is valid as a scaled immediate, don't match here.
5405 if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Log2_32(Size)))
5406 return None;
5407 if (RHSC >= -256 && RHSC < 256) {
5408 MachineOperand &Base = RootDef->getOperand(1);
5409 return {{
5410 [=](MachineInstrBuilder &MIB) { MIB.add(Base); },
5411 [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); },
5412 }};
5413 }
5414 return None;
5415}
5416
5417InstructionSelector::ComplexRendererFns
5418AArch64InstructionSelector::tryFoldAddLowIntoImm(MachineInstr &RootDef,
5419 unsigned Size,
5420 MachineRegisterInfo &MRI) const {
5421 if (RootDef.getOpcode() != AArch64::G_ADD_LOW)
5422 return None;
5423 MachineInstr &Adrp = *MRI.getVRegDef(RootDef.getOperand(1).getReg());
5424 if (Adrp.getOpcode() != AArch64::ADRP)
5425 return None;
5426
5427 // TODO: add heuristics like isWorthFoldingADDlow() from SelectionDAG.
5428 // TODO: Need to check GV's offset % size if doing offset folding into globals.
5429 assert(Adrp.getOperand(1).getOffset() == 0 && "Unexpected offset in global")((Adrp.getOperand(1).getOffset() == 0 && "Unexpected offset in global"
) ? static_cast<void> (0) : __assert_fail ("Adrp.getOperand(1).getOffset() == 0 && \"Unexpected offset in global\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5429, __PRETTY_FUNCTION__))
;
5430 auto GV = Adrp.getOperand(1).getGlobal();
5431 if (GV->isThreadLocal())
5432 return None;
5433
5434 auto &MF = *RootDef.getParent()->getParent();
5435 if (GV->getPointerAlignment(MF.getDataLayout()) < Size)
5436 return None;
5437
5438 unsigned OpFlags = STI.ClassifyGlobalReference(GV, MF.getTarget());
5439 MachineIRBuilder MIRBuilder(RootDef);
5440 Register AdrpReg = Adrp.getOperand(0).getReg();
5441 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(AdrpReg); },
5442 [=](MachineInstrBuilder &MIB) {
5443 MIB.addGlobalAddress(GV, /* Offset */ 0,
5444 OpFlags | AArch64II::MO_PAGEOFF |
5445 AArch64II::MO_NC);
5446 }}};
5447}
5448
5449/// Select a "register plus scaled unsigned 12-bit immediate" address. The
5450/// "Size" argument is the size in bytes of the memory reference, which
5451/// determines the scale.
5452InstructionSelector::ComplexRendererFns
5453AArch64InstructionSelector::selectAddrModeIndexed(MachineOperand &Root,
5454 unsigned Size) const {
5455 MachineFunction &MF = *Root.getParent()->getParent()->getParent();
5456 MachineRegisterInfo &MRI = MF.getRegInfo();
5457
5458 if (!Root.isReg())
5459 return None;
5460
5461 MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());
5462 if (!RootDef)
5463 return None;
5464
5465 if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
5466 return {{
5467 [=](MachineInstrBuilder &MIB) { MIB.add(RootDef->getOperand(1)); },
5468 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
5469 }};
5470 }
5471
5472 CodeModel::Model CM = MF.getTarget().getCodeModel();
5473 // Check if we can fold in the ADD of small code model ADRP + ADD address.
5474 if (CM == CodeModel::Small) {
5475 auto OpFns = tryFoldAddLowIntoImm(*RootDef, Size, MRI);
5476 if (OpFns)
5477 return OpFns;
5478 }
5479
5480 if (isBaseWithConstantOffset(Root, MRI)) {
5481 MachineOperand &LHS = RootDef->getOperand(1);
5482 MachineOperand &RHS = RootDef->getOperand(2);
5483 MachineInstr *LHSDef = MRI.getVRegDef(LHS.getReg());
5484 MachineInstr *RHSDef = MRI.getVRegDef(RHS.getReg());
5485 if (LHSDef && RHSDef) {
5486 int64_t RHSC = (int64_t)RHSDef->getOperand(1).getCImm()->getZExtValue();
5487 unsigned Scale = Log2_32(Size);
5488 if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Scale)) {
5489 if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
5490 return {{
5491 [=](MachineInstrBuilder &MIB) { MIB.add(LHSDef->getOperand(1)); },
5492 [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC >> Scale); },
5493 }};
5494
5495 return {{
5496 [=](MachineInstrBuilder &MIB) { MIB.add(LHS); },
5497 [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC >> Scale); },
5498 }};
5499 }
5500 }
5501 }
5502
5503 // Before falling back to our general case, check if the unscaled
5504 // instructions can handle this. If so, that's preferable.
5505 if (selectAddrModeUnscaled(Root, Size).hasValue())
5506 return None;
5507
5508 return {{
5509 [=](MachineInstrBuilder &MIB) { MIB.add(Root); },
5510 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
5511 }};
5512}
5513
5514/// Given a shift instruction, return the correct shift type for that
5515/// instruction.
5516static AArch64_AM::ShiftExtendType getShiftTypeForInst(MachineInstr &MI) {
5517 // TODO: Handle AArch64_AM::ROR
5518 switch (MI.getOpcode()) {
5519 default:
5520 return AArch64_AM::InvalidShiftExtend;
5521 case TargetOpcode::G_SHL:
5522 return AArch64_AM::LSL;
5523 case TargetOpcode::G_LSHR:
5524 return AArch64_AM::LSR;
5525 case TargetOpcode::G_ASHR:
5526 return AArch64_AM::ASR;
5527 }
5528}
5529
5530/// Select a "shifted register" operand. If the value is not shifted, set the
5531/// shift operand to a default value of "lsl 0".
5532///
5533/// TODO: Allow shifted register to be rotated in logical instructions.
5534InstructionSelector::ComplexRendererFns
5535AArch64InstructionSelector::selectShiftedRegister(MachineOperand &Root) const {
5536 if (!Root.isReg())
5537 return None;
5538 MachineRegisterInfo &MRI =
5539 Root.getParent()->getParent()->getParent()->getRegInfo();
5540
5541 // Check if the operand is defined by an instruction which corresponds to
5542 // a ShiftExtendType. E.g. a G_SHL, G_LSHR, etc.
5543 //
5544 // TODO: Handle AArch64_AM::ROR for logical instructions.
5545 MachineInstr *ShiftInst = MRI.getVRegDef(Root.getReg());
5546 if (!ShiftInst)
5547 return None;
5548 AArch64_AM::ShiftExtendType ShType = getShiftTypeForInst(*ShiftInst);
5549 if (ShType == AArch64_AM::InvalidShiftExtend)
5550 return None;
5551 if (!isWorthFoldingIntoExtendedReg(*ShiftInst, MRI))
5552 return None;
5553
5554 // Need an immediate on the RHS.
5555 MachineOperand &ShiftRHS = ShiftInst->getOperand(2);
5556 auto Immed = getImmedFromMO(ShiftRHS);
5557 if (!Immed)
5558 return None;
5559
5560 // We have something that we can fold. Fold in the shift's LHS and RHS into
5561 // the instruction.
5562 MachineOperand &ShiftLHS = ShiftInst->getOperand(1);
5563 Register ShiftReg = ShiftLHS.getReg();
5564
5565 unsigned NumBits = MRI.getType(ShiftReg).getSizeInBits();
5566 unsigned Val = *Immed & (NumBits - 1);
5567 unsigned ShiftVal = AArch64_AM::getShifterImm(ShType, Val);
5568
5569 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(ShiftReg); },
5570 [=](MachineInstrBuilder &MIB) { MIB.addImm(ShiftVal); }}};
5571}
5572
5573AArch64_AM::ShiftExtendType AArch64InstructionSelector::getExtendTypeForInst(
5574 MachineInstr &MI, MachineRegisterInfo &MRI, bool IsLoadStore) const {
5575 unsigned Opc = MI.getOpcode();
5576
5577 // Handle explicit extend instructions first.
5578 if (Opc == TargetOpcode::G_SEXT || Opc == TargetOpcode::G_SEXT_INREG) {
5579 unsigned Size;
5580 if (Opc == TargetOpcode::G_SEXT)
5581 Size = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
5582 else
5583 Size = MI.getOperand(2).getImm();
5584 assert(Size != 64 && "Extend from 64 bits?")((Size != 64 && "Extend from 64 bits?") ? static_cast
<void> (0) : __assert_fail ("Size != 64 && \"Extend from 64 bits?\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5584, __PRETTY_FUNCTION__))
;
5585 switch (Size) {
5586 case 8:
5587 return AArch64_AM::SXTB;
5588 case 16:
5589 return AArch64_AM::SXTH;
5590 case 32:
5591 return AArch64_AM::SXTW;
5592 default:
5593 return AArch64_AM::InvalidShiftExtend;
5594 }
5595 }
5596
5597 if (Opc == TargetOpcode::G_ZEXT || Opc == TargetOpcode::G_ANYEXT) {
5598 unsigned Size = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
5599 assert(Size != 64 && "Extend from 64 bits?")((Size != 64 && "Extend from 64 bits?") ? static_cast
<void> (0) : __assert_fail ("Size != 64 && \"Extend from 64 bits?\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5599, __PRETTY_FUNCTION__))
;
5600 switch (Size) {
5601 case 8:
5602 return AArch64_AM::UXTB;
5603 case 16:
5604 return AArch64_AM::UXTH;
5605 case 32:
5606 return AArch64_AM::UXTW;
5607 default:
5608 return AArch64_AM::InvalidShiftExtend;
5609 }
5610 }
5611
5612 // Don't have an explicit extend. Try to handle a G_AND with a constant mask
5613 // on the RHS.
5614 if (Opc != TargetOpcode::G_AND)
5615 return AArch64_AM::InvalidShiftExtend;
5616
5617 Optional<uint64_t> MaybeAndMask = getImmedFromMO(MI.getOperand(2));
5618 if (!MaybeAndMask)
5619 return AArch64_AM::InvalidShiftExtend;
5620 uint64_t AndMask = *MaybeAndMask;
5621 switch (AndMask) {
5622 default:
5623 return AArch64_AM::InvalidShiftExtend;
5624 case 0xFF:
5625 return !IsLoadStore ? AArch64_AM::UXTB : AArch64_AM::InvalidShiftExtend;
5626 case 0xFFFF:
5627 return !IsLoadStore ? AArch64_AM::UXTH : AArch64_AM::InvalidShiftExtend;
5628 case 0xFFFFFFFF:
5629 return AArch64_AM::UXTW;
5630 }
5631}
5632
5633Register AArch64InstructionSelector::narrowExtendRegIfNeeded(
5634 Register ExtReg, MachineIRBuilder &MIB) const {
5635 MachineRegisterInfo &MRI = *MIB.getMRI();
5636 if (MRI.getType(ExtReg).getSiz