Bug Summary

File:llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
Warning:line 908, column 7
6th function call argument is an uninitialized value

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name AArch64InstructionSelector.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/build-llvm/lib/Target/AArch64 -resource-dir /usr/lib/llvm-13/lib/clang/13.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/build-llvm/lib/Target/AArch64 -I /build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64 -I /build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/build-llvm/include -I /build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-13/lib/clang/13.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/build-llvm/lib/Target/AArch64 -fdebug-prefix-map=/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c=. -ferror-limit 19 -fvisibility hidden -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2021-03-08-182450-10039-1 -x c++ /build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp

/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp

1//===- AArch64InstructionSelector.cpp ----------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// AArch64.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "AArch64InstrInfo.h"
15#include "AArch64MachineFunctionInfo.h"
16#include "AArch64RegisterBankInfo.h"
17#include "AArch64RegisterInfo.h"
18#include "AArch64Subtarget.h"
19#include "AArch64TargetMachine.h"
20#include "MCTargetDesc/AArch64AddressingModes.h"
21#include "MCTargetDesc/AArch64MCTargetDesc.h"
22#include "llvm/ADT/Optional.h"
23#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
24#include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h"
25#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
26#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
27#include "llvm/CodeGen/GlobalISel/Utils.h"
28#include "llvm/CodeGen/MachineBasicBlock.h"
29#include "llvm/CodeGen/MachineConstantPool.h"
30#include "llvm/CodeGen/MachineFunction.h"
31#include "llvm/CodeGen/MachineInstr.h"
32#include "llvm/CodeGen/MachineInstrBuilder.h"
33#include "llvm/CodeGen/MachineOperand.h"
34#include "llvm/CodeGen/MachineRegisterInfo.h"
35#include "llvm/CodeGen/TargetOpcodes.h"
36#include "llvm/IR/Constants.h"
37#include "llvm/IR/Instructions.h"
38#include "llvm/IR/PatternMatch.h"
39#include "llvm/IR/Type.h"
40#include "llvm/IR/IntrinsicsAArch64.h"
41#include "llvm/Pass.h"
42#include "llvm/Support/Debug.h"
43#include "llvm/Support/raw_ostream.h"
44
45#define DEBUG_TYPE"aarch64-isel" "aarch64-isel"
46
47using namespace llvm;
48using namespace MIPatternMatch;
49
50namespace llvm {
51class BlockFrequencyInfo;
52class ProfileSummaryInfo;
53}
54
55namespace {
56
57#define GET_GLOBALISEL_PREDICATE_BITSET
58#include "AArch64GenGlobalISel.inc"
59#undef GET_GLOBALISEL_PREDICATE_BITSET
60
61class AArch64InstructionSelector : public InstructionSelector {
62public:
63 AArch64InstructionSelector(const AArch64TargetMachine &TM,
64 const AArch64Subtarget &STI,
65 const AArch64RegisterBankInfo &RBI);
66
67 bool select(MachineInstr &I) override;
68 static const char *getName() { return DEBUG_TYPE"aarch64-isel"; }
69
70 void setupMF(MachineFunction &MF, GISelKnownBits *KB,
71 CodeGenCoverage &CoverageInfo, ProfileSummaryInfo *PSI,
72 BlockFrequencyInfo *BFI) override {
73 InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI);
74
75 // hasFnAttribute() is expensive to call on every BRCOND selection, so
76 // cache it here for each run of the selector.
77 ProduceNonFlagSettingCondBr =
78 !MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening);
79 MFReturnAddr = Register();
80
81 processPHIs(MF);
82 }
83
84private:
85 /// tblgen-erated 'select' implementation, used as the initial selector for
86 /// the patterns that don't require complex C++.
87 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
88
89 // A lowering phase that runs before any selection attempts.
90 // Returns true if the instruction was modified.
91 bool preISelLower(MachineInstr &I);
92
93 // An early selection function that runs before the selectImpl() call.
94 bool earlySelect(MachineInstr &I) const;
95
96 // Do some preprocessing of G_PHIs before we begin selection.
97 void processPHIs(MachineFunction &MF);
98
99 bool earlySelectSHL(MachineInstr &I, MachineRegisterInfo &MRI) const;
100
101 /// Eliminate same-sized cross-bank copies into stores before selectImpl().
102 bool contractCrossBankCopyIntoStore(MachineInstr &I,
103 MachineRegisterInfo &MRI);
104
105 bool convertPtrAddToAdd(MachineInstr &I, MachineRegisterInfo &MRI);
106
107 bool selectVaStartAAPCS(MachineInstr &I, MachineFunction &MF,
108 MachineRegisterInfo &MRI) const;
109 bool selectVaStartDarwin(MachineInstr &I, MachineFunction &MF,
110 MachineRegisterInfo &MRI) const;
111
112 ///@{
113 /// Helper functions for selectCompareBranch.
114 bool selectCompareBranchFedByFCmp(MachineInstr &I, MachineInstr &FCmp,
115 MachineIRBuilder &MIB) const;
116 bool selectCompareBranchFedByICmp(MachineInstr &I, MachineInstr &ICmp,
117 MachineIRBuilder &MIB) const;
118 bool tryOptCompareBranchFedByICmp(MachineInstr &I, MachineInstr &ICmp,
119 MachineIRBuilder &MIB) const;
120 bool tryOptAndIntoCompareBranch(MachineInstr &AndInst, bool Invert,
121 MachineBasicBlock *DstMBB,
122 MachineIRBuilder &MIB) const;
123 ///@}
124
125 bool selectCompareBranch(MachineInstr &I, MachineFunction &MF,
126 MachineRegisterInfo &MRI) const;
127
128 bool selectVectorAshrLshr(MachineInstr &I, MachineRegisterInfo &MRI) const;
129 bool selectVectorSHL(MachineInstr &I, MachineRegisterInfo &MRI) const;
130
131 // Helper to generate an equivalent of scalar_to_vector into a new register,
132 // returned via 'Dst'.
133 MachineInstr *emitScalarToVector(unsigned EltSize,
134 const TargetRegisterClass *DstRC,
135 Register Scalar,
136 MachineIRBuilder &MIRBuilder) const;
137
138 /// Emit a lane insert into \p DstReg, or a new vector register if None is
139 /// provided.
140 ///
141 /// The lane inserted into is defined by \p LaneIdx. The vector source
142 /// register is given by \p SrcReg. The register containing the element is
143 /// given by \p EltReg.
144 MachineInstr *emitLaneInsert(Optional<Register> DstReg, Register SrcReg,
145 Register EltReg, unsigned LaneIdx,
146 const RegisterBank &RB,
147 MachineIRBuilder &MIRBuilder) const;
148 bool selectInsertElt(MachineInstr &I, MachineRegisterInfo &MRI) const;
149 bool tryOptConstantBuildVec(MachineInstr &MI, LLT DstTy,
150 MachineRegisterInfo &MRI) const;
151 bool selectBuildVector(MachineInstr &I, MachineRegisterInfo &MRI) const;
152 bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI) const;
153 bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI) const;
154
155 bool selectShuffleVector(MachineInstr &I, MachineRegisterInfo &MRI) const;
156 bool selectExtractElt(MachineInstr &I, MachineRegisterInfo &MRI) const;
157 bool selectConcatVectors(MachineInstr &I, MachineRegisterInfo &MRI) const;
158 bool selectSplitVectorUnmerge(MachineInstr &I,
159 MachineRegisterInfo &MRI) const;
160 bool selectIntrinsicWithSideEffects(MachineInstr &I,
161 MachineRegisterInfo &MRI) const;
162 bool selectIntrinsic(MachineInstr &I, MachineRegisterInfo &MRI);
163 bool selectVectorICmp(MachineInstr &I, MachineRegisterInfo &MRI) const;
164 bool selectIntrinsicTrunc(MachineInstr &I, MachineRegisterInfo &MRI) const;
165 bool selectIntrinsicRound(MachineInstr &I, MachineRegisterInfo &MRI) const;
166 bool selectJumpTable(MachineInstr &I, MachineRegisterInfo &MRI) const;
167 bool selectBrJT(MachineInstr &I, MachineRegisterInfo &MRI) const;
168 bool selectTLSGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI) const;
169 bool selectReduction(MachineInstr &I, MachineRegisterInfo &MRI) const;
170
171 unsigned emitConstantPoolEntry(const Constant *CPVal,
172 MachineFunction &MF) const;
173 MachineInstr *emitLoadFromConstantPool(const Constant *CPVal,
174 MachineIRBuilder &MIRBuilder) const;
175
176 // Emit a vector concat operation.
177 MachineInstr *emitVectorConcat(Optional<Register> Dst, Register Op1,
178 Register Op2,
179 MachineIRBuilder &MIRBuilder) const;
180
181 // Emit an integer compare between LHS and RHS, which checks for Predicate.
182 MachineInstr *emitIntegerCompare(MachineOperand &LHS, MachineOperand &RHS,
183 MachineOperand &Predicate,
184 MachineIRBuilder &MIRBuilder) const;
185
186 /// Emit a floating point comparison between \p LHS and \p RHS.
187 /// \p Pred if given is the intended predicate to use.
188 MachineInstr *emitFPCompare(Register LHS, Register RHS,
189 MachineIRBuilder &MIRBuilder,
190 Optional<CmpInst::Predicate> = None) const;
191
192 MachineInstr *emitInstr(unsigned Opcode,
193 std::initializer_list<llvm::DstOp> DstOps,
194 std::initializer_list<llvm::SrcOp> SrcOps,
195 MachineIRBuilder &MIRBuilder,
196 const ComplexRendererFns &RenderFns = None) const;
197 /// Helper function to emit an add or sub instruction.
198 ///
199 /// \p AddrModeAndSizeToOpcode must contain each of the opcode variants above
200 /// in a specific order.
201 ///
202 /// Below is an example of the expected input to \p AddrModeAndSizeToOpcode.
203 ///
204 /// \code
205 /// const std::array<std::array<unsigned, 2>, 4> Table {
206 /// {{AArch64::ADDXri, AArch64::ADDWri},
207 /// {AArch64::ADDXrs, AArch64::ADDWrs},
208 /// {AArch64::ADDXrr, AArch64::ADDWrr},
209 /// {AArch64::SUBXri, AArch64::SUBWri},
210 /// {AArch64::ADDXrx, AArch64::ADDWrx}}};
211 /// \endcode
212 ///
213 /// Each row in the table corresponds to a different addressing mode. Each
214 /// column corresponds to a different register size.
215 ///
216 /// \attention Rows must be structured as follows:
217 /// - Row 0: The ri opcode variants
218 /// - Row 1: The rs opcode variants
219 /// - Row 2: The rr opcode variants
220 /// - Row 3: The ri opcode variants for negative immediates
221 /// - Row 4: The rx opcode variants
222 ///
223 /// \attention Columns must be structured as follows:
224 /// - Column 0: The 64-bit opcode variants
225 /// - Column 1: The 32-bit opcode variants
226 ///
227 /// \p Dst is the destination register of the binop to emit.
228 /// \p LHS is the left-hand operand of the binop to emit.
229 /// \p RHS is the right-hand operand of the binop to emit.
230 MachineInstr *emitAddSub(
231 const std::array<std::array<unsigned, 2>, 5> &AddrModeAndSizeToOpcode,
232 Register Dst, MachineOperand &LHS, MachineOperand &RHS,
233 MachineIRBuilder &MIRBuilder) const;
234 MachineInstr *emitADD(Register DefReg, MachineOperand &LHS,
235 MachineOperand &RHS,
236 MachineIRBuilder &MIRBuilder) const;
237 MachineInstr *emitADDS(Register Dst, MachineOperand &LHS, MachineOperand &RHS,
238 MachineIRBuilder &MIRBuilder) const;
239 MachineInstr *emitSUBS(Register Dst, MachineOperand &LHS, MachineOperand &RHS,
240 MachineIRBuilder &MIRBuilder) const;
241 MachineInstr *emitCMN(MachineOperand &LHS, MachineOperand &RHS,
242 MachineIRBuilder &MIRBuilder) const;
243 MachineInstr *emitTST(MachineOperand &LHS, MachineOperand &RHS,
244 MachineIRBuilder &MIRBuilder) const;
245 MachineInstr *emitSelect(Register Dst, Register LHS, Register RHS,
246 AArch64CC::CondCode CC,
247 MachineIRBuilder &MIRBuilder) const;
248 MachineInstr *emitExtractVectorElt(Optional<Register> DstReg,
249 const RegisterBank &DstRB, LLT ScalarTy,
250 Register VecReg, unsigned LaneIdx,
251 MachineIRBuilder &MIRBuilder) const;
252
253 /// Emit a CSet for an integer compare.
254 ///
255 /// \p DefReg and \p SrcReg are expected to be 32-bit scalar registers.
256 MachineInstr *emitCSetForICMP(Register DefReg, unsigned Pred,
257 MachineIRBuilder &MIRBuilder,
258 Register SrcReg = AArch64::WZR) const;
259 /// Emit a CSet for a FP compare.
260 ///
261 /// \p Dst is expected to be a 32-bit scalar register.
262 MachineInstr *emitCSetForFCmp(Register Dst, CmpInst::Predicate Pred,
263 MachineIRBuilder &MIRBuilder) const;
264
265 /// Emit the overflow op for \p Opcode.
266 ///
267 /// \p Opcode is expected to be an overflow op's opcode, e.g. G_UADDO,
268 /// G_USUBO, etc.
269 std::pair<MachineInstr *, AArch64CC::CondCode>
270 emitOverflowOp(unsigned Opcode, Register Dst, MachineOperand &LHS,
271 MachineOperand &RHS, MachineIRBuilder &MIRBuilder) const;
272
273 /// Emit a TB(N)Z instruction which tests \p Bit in \p TestReg.
274 /// \p IsNegative is true if the test should be "not zero".
275 /// This will also optimize the test bit instruction when possible.
276 MachineInstr *emitTestBit(Register TestReg, uint64_t Bit, bool IsNegative,
277 MachineBasicBlock *DstMBB,
278 MachineIRBuilder &MIB) const;
279
280 /// Emit a CB(N)Z instruction which branches to \p DestMBB.
281 MachineInstr *emitCBZ(Register CompareReg, bool IsNegative,
282 MachineBasicBlock *DestMBB,
283 MachineIRBuilder &MIB) const;
284
285 // Equivalent to the i32shift_a and friends from AArch64InstrInfo.td.
286 // We use these manually instead of using the importer since it doesn't
287 // support SDNodeXForm.
288 ComplexRendererFns selectShiftA_32(const MachineOperand &Root) const;
289 ComplexRendererFns selectShiftB_32(const MachineOperand &Root) const;
290 ComplexRendererFns selectShiftA_64(const MachineOperand &Root) const;
291 ComplexRendererFns selectShiftB_64(const MachineOperand &Root) const;
292
293 ComplexRendererFns select12BitValueWithLeftShift(uint64_t Immed) const;
294 ComplexRendererFns selectArithImmed(MachineOperand &Root) const;
295 ComplexRendererFns selectNegArithImmed(MachineOperand &Root) const;
296
297 ComplexRendererFns selectAddrModeUnscaled(MachineOperand &Root,
298 unsigned Size) const;
299
300 ComplexRendererFns selectAddrModeUnscaled8(MachineOperand &Root) const {
301 return selectAddrModeUnscaled(Root, 1);
302 }
303 ComplexRendererFns selectAddrModeUnscaled16(MachineOperand &Root) const {
304 return selectAddrModeUnscaled(Root, 2);
305 }
306 ComplexRendererFns selectAddrModeUnscaled32(MachineOperand &Root) const {
307 return selectAddrModeUnscaled(Root, 4);
308 }
309 ComplexRendererFns selectAddrModeUnscaled64(MachineOperand &Root) const {
310 return selectAddrModeUnscaled(Root, 8);
311 }
312 ComplexRendererFns selectAddrModeUnscaled128(MachineOperand &Root) const {
313 return selectAddrModeUnscaled(Root, 16);
314 }
315
316 /// Helper to try to fold in a GISEL_ADD_LOW into an immediate, to be used
317 /// from complex pattern matchers like selectAddrModeIndexed().
318 ComplexRendererFns tryFoldAddLowIntoImm(MachineInstr &RootDef, unsigned Size,
319 MachineRegisterInfo &MRI) const;
320
321 ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root,
322 unsigned Size) const;
323 template <int Width>
324 ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root) const {
325 return selectAddrModeIndexed(Root, Width / 8);
326 }
327
328 bool isWorthFoldingIntoExtendedReg(MachineInstr &MI,
329 const MachineRegisterInfo &MRI) const;
330 ComplexRendererFns
331 selectAddrModeShiftedExtendXReg(MachineOperand &Root,
332 unsigned SizeInBytes) const;
333
334 /// Returns a \p ComplexRendererFns which contains a base, offset, and whether
335 /// or not a shift + extend should be folded into an addressing mode. Returns
336 /// None when this is not profitable or possible.
337 ComplexRendererFns
338 selectExtendedSHL(MachineOperand &Root, MachineOperand &Base,
339 MachineOperand &Offset, unsigned SizeInBytes,
340 bool WantsExt) const;
341 ComplexRendererFns selectAddrModeRegisterOffset(MachineOperand &Root) const;
342 ComplexRendererFns selectAddrModeXRO(MachineOperand &Root,
343 unsigned SizeInBytes) const;
344 template <int Width>
345 ComplexRendererFns selectAddrModeXRO(MachineOperand &Root) const {
346 return selectAddrModeXRO(Root, Width / 8);
347 }
348
349 ComplexRendererFns selectAddrModeWRO(MachineOperand &Root,
350 unsigned SizeInBytes) const;
351 template <int Width>
352 ComplexRendererFns selectAddrModeWRO(MachineOperand &Root) const {
353 return selectAddrModeWRO(Root, Width / 8);
354 }
355
356 ComplexRendererFns selectShiftedRegister(MachineOperand &Root) const;
357
358 ComplexRendererFns selectArithShiftedRegister(MachineOperand &Root) const {
359 return selectShiftedRegister(Root);
360 }
361
362 ComplexRendererFns selectLogicalShiftedRegister(MachineOperand &Root) const {
363 // TODO: selectShiftedRegister should allow for rotates on logical shifts.
364 // For now, make them the same. The only difference between the two is that
365 // logical shifts are allowed to fold in rotates. Otherwise, these are
366 // functionally the same.
367 return selectShiftedRegister(Root);
368 }
369
370 /// Given an extend instruction, determine the correct shift-extend type for
371 /// that instruction.
372 ///
373 /// If the instruction is going to be used in a load or store, pass
374 /// \p IsLoadStore = true.
375 AArch64_AM::ShiftExtendType
376 getExtendTypeForInst(MachineInstr &MI, MachineRegisterInfo &MRI,
377 bool IsLoadStore = false) const;
378
379 /// Move \p Reg to \p RC if \p Reg is not already on \p RC.
380 ///
381 /// \returns Either \p Reg if no change was necessary, or the new register
382 /// created by moving \p Reg.
383 ///
384 /// Note: This uses emitCopy right now.
385 Register moveScalarRegClass(Register Reg, const TargetRegisterClass &RC,
386 MachineIRBuilder &MIB) const;
387
388 ComplexRendererFns selectArithExtendedRegister(MachineOperand &Root) const;
389
390 void renderTruncImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
391 int OpIdx = -1) const;
392 void renderLogicalImm32(MachineInstrBuilder &MIB, const MachineInstr &I,
393 int OpIdx = -1) const;
394 void renderLogicalImm64(MachineInstrBuilder &MIB, const MachineInstr &I,
395 int OpIdx = -1) const;
396 void renderFPImm16(MachineInstrBuilder &MIB, const MachineInstr &MI,
397 int OpIdx = -1) const;
398 void renderFPImm32(MachineInstrBuilder &MIB, const MachineInstr &MI,
399 int OpIdx = -1) const;
400 void renderFPImm64(MachineInstrBuilder &MIB, const MachineInstr &MI,
401 int OpIdx = -1) const;
402
403 // Materialize a GlobalValue or BlockAddress using a movz+movk sequence.
404 void materializeLargeCMVal(MachineInstr &I, const Value *V,
405 unsigned OpFlags) const;
406
407 // Optimization methods.
408 bool tryOptSelect(MachineInstr &MI) const;
409 MachineInstr *tryFoldIntegerCompare(MachineOperand &LHS, MachineOperand &RHS,
410 MachineOperand &Predicate,
411 MachineIRBuilder &MIRBuilder) const;
412
413 /// Return true if \p MI is a load or store of \p NumBytes bytes.
414 bool isLoadStoreOfNumBytes(const MachineInstr &MI, unsigned NumBytes) const;
415
416 /// Returns true if \p MI is guaranteed to have the high-half of a 64-bit
417 /// register zeroed out. In other words, the result of MI has been explicitly
418 /// zero extended.
419 bool isDef32(const MachineInstr &MI) const;
420
421 const AArch64TargetMachine &TM;
422 const AArch64Subtarget &STI;
423 const AArch64InstrInfo &TII;
424 const AArch64RegisterInfo &TRI;
425 const AArch64RegisterBankInfo &RBI;
426
427 bool ProduceNonFlagSettingCondBr = false;
428
429 // Some cached values used during selection.
430 // We use LR as a live-in register, and we keep track of it here as it can be
431 // clobbered by calls.
432 Register MFReturnAddr;
433
434#define GET_GLOBALISEL_PREDICATES_DECL
435#include "AArch64GenGlobalISel.inc"
436#undef GET_GLOBALISEL_PREDICATES_DECL
437
438// We declare the temporaries used by selectImpl() in the class to minimize the
439// cost of constructing placeholder values.
440#define GET_GLOBALISEL_TEMPORARIES_DECL
441#include "AArch64GenGlobalISel.inc"
442#undef GET_GLOBALISEL_TEMPORARIES_DECL
443};
444
445} // end anonymous namespace
446
447#define GET_GLOBALISEL_IMPL
448#include "AArch64GenGlobalISel.inc"
449#undef GET_GLOBALISEL_IMPL
450
451AArch64InstructionSelector::AArch64InstructionSelector(
452 const AArch64TargetMachine &TM, const AArch64Subtarget &STI,
453 const AArch64RegisterBankInfo &RBI)
454 : InstructionSelector(), TM(TM), STI(STI), TII(*STI.getInstrInfo()),
455 TRI(*STI.getRegisterInfo()), RBI(RBI),
456#define GET_GLOBALISEL_PREDICATES_INIT
457#include "AArch64GenGlobalISel.inc"
458#undef GET_GLOBALISEL_PREDICATES_INIT
459#define GET_GLOBALISEL_TEMPORARIES_INIT
460#include "AArch64GenGlobalISel.inc"
461#undef GET_GLOBALISEL_TEMPORARIES_INIT
462{
463}
464
465// FIXME: This should be target-independent, inferred from the types declared
466// for each class in the bank.
467static const TargetRegisterClass *
468getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB,
469 const RegisterBankInfo &RBI,
470 bool GetAllRegSet = false) {
471 if (RB.getID() == AArch64::GPRRegBankID) {
472 if (Ty.getSizeInBits() <= 32)
473 return GetAllRegSet ? &AArch64::GPR32allRegClass
474 : &AArch64::GPR32RegClass;
475 if (Ty.getSizeInBits() == 64)
476 return GetAllRegSet ? &AArch64::GPR64allRegClass
477 : &AArch64::GPR64RegClass;
478 return nullptr;
479 }
480
481 if (RB.getID() == AArch64::FPRRegBankID) {
482 if (Ty.getSizeInBits() <= 16)
483 return &AArch64::FPR16RegClass;
484 if (Ty.getSizeInBits() == 32)
485 return &AArch64::FPR32RegClass;
486 if (Ty.getSizeInBits() == 64)
487 return &AArch64::FPR64RegClass;
488 if (Ty.getSizeInBits() == 128)
489 return &AArch64::FPR128RegClass;
490 return nullptr;
491 }
492
493 return nullptr;
494}
495
496/// Given a register bank, and size in bits, return the smallest register class
497/// that can represent that combination.
498static const TargetRegisterClass *
499getMinClassForRegBank(const RegisterBank &RB, unsigned SizeInBits,
500 bool GetAllRegSet = false) {
501 unsigned RegBankID = RB.getID();
502
503 if (RegBankID == AArch64::GPRRegBankID) {
504 if (SizeInBits <= 32)
505 return GetAllRegSet ? &AArch64::GPR32allRegClass
506 : &AArch64::GPR32RegClass;
507 if (SizeInBits == 64)
508 return GetAllRegSet ? &AArch64::GPR64allRegClass
509 : &AArch64::GPR64RegClass;
510 }
511
512 if (RegBankID == AArch64::FPRRegBankID) {
513 switch (SizeInBits) {
514 default:
515 return nullptr;
516 case 8:
517 return &AArch64::FPR8RegClass;
518 case 16:
519 return &AArch64::FPR16RegClass;
520 case 32:
521 return &AArch64::FPR32RegClass;
522 case 64:
523 return &AArch64::FPR64RegClass;
524 case 128:
525 return &AArch64::FPR128RegClass;
526 }
527 }
528
529 return nullptr;
530}
531
532/// Returns the correct subregister to use for a given register class.
533static bool getSubRegForClass(const TargetRegisterClass *RC,
534 const TargetRegisterInfo &TRI, unsigned &SubReg) {
535 switch (TRI.getRegSizeInBits(*RC)) {
55
Control jumps to the 'default' case at line 551
536 case 8:
537 SubReg = AArch64::bsub;
538 break;
539 case 16:
540 SubReg = AArch64::hsub;
541 break;
542 case 32:
543 if (RC != &AArch64::FPR32RegClass)
544 SubReg = AArch64::sub_32;
545 else
546 SubReg = AArch64::ssub;
547 break;
548 case 64:
549 SubReg = AArch64::dsub;
550 break;
551 default:
552 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't find appropriate subregister for register class."
; } } while (false)
56
Assuming 'DebugFlag' is false
57
Loop condition is false. Exiting loop
553 dbgs() << "Couldn't find appropriate subregister for register class.")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't find appropriate subregister for register class."
; } } while (false)
;
554 return false;
58
Returning without writing to 'SubReg'
555 }
556
557 return true;
558}
559
560/// Returns the minimum size the given register bank can hold.
561static unsigned getMinSizeForRegBank(const RegisterBank &RB) {
562 switch (RB.getID()) {
563 case AArch64::GPRRegBankID:
564 return 32;
565 case AArch64::FPRRegBankID:
566 return 8;
567 default:
568 llvm_unreachable("Tried to get minimum size for unknown register bank.")::llvm::llvm_unreachable_internal("Tried to get minimum size for unknown register bank."
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 568)
;
569 }
570}
571
572static Optional<uint64_t> getImmedFromMO(const MachineOperand &Root) {
573 auto &MI = *Root.getParent();
574 auto &MBB = *MI.getParent();
575 auto &MF = *MBB.getParent();
576 auto &MRI = MF.getRegInfo();
577 uint64_t Immed;
578 if (Root.isImm())
579 Immed = Root.getImm();
580 else if (Root.isCImm())
581 Immed = Root.getCImm()->getZExtValue();
582 else if (Root.isReg()) {
583 auto ValAndVReg =
584 getConstantVRegValWithLookThrough(Root.getReg(), MRI, true);
585 if (!ValAndVReg)
586 return None;
587 Immed = ValAndVReg->Value.getSExtValue();
588 } else
589 return None;
590 return Immed;
591}
592
593/// Check whether \p I is a currently unsupported binary operation:
594/// - it has an unsized type
595/// - an operand is not a vreg
596/// - all operands are not in the same bank
597/// These are checks that should someday live in the verifier, but right now,
598/// these are mostly limitations of the aarch64 selector.
599static bool unsupportedBinOp(const MachineInstr &I,
600 const AArch64RegisterBankInfo &RBI,
601 const MachineRegisterInfo &MRI,
602 const AArch64RegisterInfo &TRI) {
603 LLT Ty = MRI.getType(I.getOperand(0).getReg());
604 if (!Ty.isValid()) {
605 LLVM_DEBUG(dbgs() << "Generic binop register should be typed\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic binop register should be typed\n"
; } } while (false)
;
606 return true;
607 }
608
609 const RegisterBank *PrevOpBank = nullptr;
610 for (auto &MO : I.operands()) {
611 // FIXME: Support non-register operands.
612 if (!MO.isReg()) {
613 LLVM_DEBUG(dbgs() << "Generic inst non-reg operands are unsupported\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic inst non-reg operands are unsupported\n"
; } } while (false)
;
614 return true;
615 }
616
617 // FIXME: Can generic operations have physical registers operands? If
618 // so, this will need to be taught about that, and we'll need to get the
619 // bank out of the minimal class for the register.
620 // Either way, this needs to be documented (and possibly verified).
621 if (!Register::isVirtualRegister(MO.getReg())) {
622 LLVM_DEBUG(dbgs() << "Generic inst has physical register operand\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic inst has physical register operand\n"
; } } while (false)
;
623 return true;
624 }
625
626 const RegisterBank *OpBank = RBI.getRegBank(MO.getReg(), MRI, TRI);
627 if (!OpBank) {
628 LLVM_DEBUG(dbgs() << "Generic register has no bank or class\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic register has no bank or class\n"
; } } while (false)
;
629 return true;
630 }
631
632 if (PrevOpBank && OpBank != PrevOpBank) {
633 LLVM_DEBUG(dbgs() << "Generic inst operands have different banks\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic inst operands have different banks\n"
; } } while (false)
;
634 return true;
635 }
636 PrevOpBank = OpBank;
637 }
638 return false;
639}
640
641/// Select the AArch64 opcode for the basic binary operation \p GenericOpc
642/// (such as G_OR or G_SDIV), appropriate for the register bank \p RegBankID
643/// and of size \p OpSize.
644/// \returns \p GenericOpc if the combination is unsupported.
645static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID,
646 unsigned OpSize) {
647 switch (RegBankID) {
648 case AArch64::GPRRegBankID:
649 if (OpSize == 32) {
650 switch (GenericOpc) {
651 case TargetOpcode::G_SHL:
652 return AArch64::LSLVWr;
653 case TargetOpcode::G_LSHR:
654 return AArch64::LSRVWr;
655 case TargetOpcode::G_ASHR:
656 return AArch64::ASRVWr;
657 default:
658 return GenericOpc;
659 }
660 } else if (OpSize == 64) {
661 switch (GenericOpc) {
662 case TargetOpcode::G_PTR_ADD:
663 return AArch64::ADDXrr;
664 case TargetOpcode::G_SHL:
665 return AArch64::LSLVXr;
666 case TargetOpcode::G_LSHR:
667 return AArch64::LSRVXr;
668 case TargetOpcode::G_ASHR:
669 return AArch64::ASRVXr;
670 default:
671 return GenericOpc;
672 }
673 }
674 break;
675 case AArch64::FPRRegBankID:
676 switch (OpSize) {
677 case 32:
678 switch (GenericOpc) {
679 case TargetOpcode::G_FADD:
680 return AArch64::FADDSrr;
681 case TargetOpcode::G_FSUB:
682 return AArch64::FSUBSrr;
683 case TargetOpcode::G_FMUL:
684 return AArch64::FMULSrr;
685 case TargetOpcode::G_FDIV:
686 return AArch64::FDIVSrr;
687 default:
688 return GenericOpc;
689 }
690 case 64:
691 switch (GenericOpc) {
692 case TargetOpcode::G_FADD:
693 return AArch64::FADDDrr;
694 case TargetOpcode::G_FSUB:
695 return AArch64::FSUBDrr;
696 case TargetOpcode::G_FMUL:
697 return AArch64::FMULDrr;
698 case TargetOpcode::G_FDIV:
699 return AArch64::FDIVDrr;
700 case TargetOpcode::G_OR:
701 return AArch64::ORRv8i8;
702 default:
703 return GenericOpc;
704 }
705 }
706 break;
707 }
708 return GenericOpc;
709}
710
711/// Select the AArch64 opcode for the G_LOAD or G_STORE operation \p GenericOpc,
712/// appropriate for the (value) register bank \p RegBankID and of memory access
713/// size \p OpSize. This returns the variant with the base+unsigned-immediate
714/// addressing mode (e.g., LDRXui).
715/// \returns \p GenericOpc if the combination is unsupported.
716static unsigned selectLoadStoreUIOp(unsigned GenericOpc, unsigned RegBankID,
717 unsigned OpSize) {
718 const bool isStore = GenericOpc == TargetOpcode::G_STORE;
719 switch (RegBankID) {
720 case AArch64::GPRRegBankID:
721 switch (OpSize) {
722 case 8:
723 return isStore ? AArch64::STRBBui : AArch64::LDRBBui;
724 case 16:
725 return isStore ? AArch64::STRHHui : AArch64::LDRHHui;
726 case 32:
727 return isStore ? AArch64::STRWui : AArch64::LDRWui;
728 case 64:
729 return isStore ? AArch64::STRXui : AArch64::LDRXui;
730 }
731 break;
732 case AArch64::FPRRegBankID:
733 switch (OpSize) {
734 case 8:
735 return isStore ? AArch64::STRBui : AArch64::LDRBui;
736 case 16:
737 return isStore ? AArch64::STRHui : AArch64::LDRHui;
738 case 32:
739 return isStore ? AArch64::STRSui : AArch64::LDRSui;
740 case 64:
741 return isStore ? AArch64::STRDui : AArch64::LDRDui;
742 }
743 break;
744 }
745 return GenericOpc;
746}
747
748#ifndef NDEBUG
749/// Helper function that verifies that we have a valid copy at the end of
750/// selectCopy. Verifies that the source and dest have the expected sizes and
751/// then returns true.
752static bool isValidCopy(const MachineInstr &I, const RegisterBank &DstBank,
753 const MachineRegisterInfo &MRI,
754 const TargetRegisterInfo &TRI,
755 const RegisterBankInfo &RBI) {
756 const Register DstReg = I.getOperand(0).getReg();
757 const Register SrcReg = I.getOperand(1).getReg();
758 const unsigned DstSize = RBI.getSizeInBits(DstReg, MRI, TRI);
759 const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI);
760
761 // Make sure the size of the source and dest line up.
762 assert((((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg
) && DstSize <= SrcSize) || (((DstSize + 31) / 32 ==
(SrcSize + 31) / 32) && DstSize > SrcSize)) &&
"Copy with different width?!") ? static_cast<void> (0)
: __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 771, __PRETTY_FUNCTION__))
763 (DstSize == SrcSize ||(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg
) && DstSize <= SrcSize) || (((DstSize + 31) / 32 ==
(SrcSize + 31) / 32) && DstSize > SrcSize)) &&
"Copy with different width?!") ? static_cast<void> (0)
: __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 771, __PRETTY_FUNCTION__))
764 // Copies are a mean to setup initial types, the number of(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg
) && DstSize <= SrcSize) || (((DstSize + 31) / 32 ==
(SrcSize + 31) / 32) && DstSize > SrcSize)) &&
"Copy with different width?!") ? static_cast<void> (0)
: __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 771, __PRETTY_FUNCTION__))
765 // bits may not exactly match.(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg
) && DstSize <= SrcSize) || (((DstSize + 31) / 32 ==
(SrcSize + 31) / 32) && DstSize > SrcSize)) &&
"Copy with different width?!") ? static_cast<void> (0)
: __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 771, __PRETTY_FUNCTION__))
766 (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) ||(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg
) && DstSize <= SrcSize) || (((DstSize + 31) / 32 ==
(SrcSize + 31) / 32) && DstSize > SrcSize)) &&
"Copy with different width?!") ? static_cast<void> (0)
: __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 771, __PRETTY_FUNCTION__))
767 // Copies are a mean to copy bits around, as long as we are(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg
) && DstSize <= SrcSize) || (((DstSize + 31) / 32 ==
(SrcSize + 31) / 32) && DstSize > SrcSize)) &&
"Copy with different width?!") ? static_cast<void> (0)
: __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 771, __PRETTY_FUNCTION__))
768 // on the same register class, that's fine. Otherwise, that(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg
) && DstSize <= SrcSize) || (((DstSize + 31) / 32 ==
(SrcSize + 31) / 32) && DstSize > SrcSize)) &&
"Copy with different width?!") ? static_cast<void> (0)
: __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 771, __PRETTY_FUNCTION__))
769 // means we need some SUBREG_TO_REG or AND & co.(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg
) && DstSize <= SrcSize) || (((DstSize + 31) / 32 ==
(SrcSize + 31) / 32) && DstSize > SrcSize)) &&
"Copy with different width?!") ? static_cast<void> (0)
: __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 771, __PRETTY_FUNCTION__))
770 (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) &&(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg
) && DstSize <= SrcSize) || (((DstSize + 31) / 32 ==
(SrcSize + 31) / 32) && DstSize > SrcSize)) &&
"Copy with different width?!") ? static_cast<void> (0)
: __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 771, __PRETTY_FUNCTION__))
771 "Copy with different width?!")(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg
) && DstSize <= SrcSize) || (((DstSize + 31) / 32 ==
(SrcSize + 31) / 32) && DstSize > SrcSize)) &&
"Copy with different width?!") ? static_cast<void> (0)
: __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 771, __PRETTY_FUNCTION__))
;
772
773 // Check the size of the destination.
774 assert((DstSize <= 64 || DstBank.getID() == AArch64::FPRRegBankID) &&(((DstSize <= 64 || DstBank.getID() == AArch64::FPRRegBankID
) && "GPRs cannot get more than 64-bit width values")
? static_cast<void> (0) : __assert_fail ("(DstSize <= 64 || DstBank.getID() == AArch64::FPRRegBankID) && \"GPRs cannot get more than 64-bit width values\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 775, __PRETTY_FUNCTION__))
775 "GPRs cannot get more than 64-bit width values")(((DstSize <= 64 || DstBank.getID() == AArch64::FPRRegBankID
) && "GPRs cannot get more than 64-bit width values")
? static_cast<void> (0) : __assert_fail ("(DstSize <= 64 || DstBank.getID() == AArch64::FPRRegBankID) && \"GPRs cannot get more than 64-bit width values\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 775, __PRETTY_FUNCTION__))
;
776
777 return true;
778}
779#endif
780
781/// Helper function for selectCopy. Inserts a subregister copy from \p SrcReg
782/// to \p *To.
783///
784/// E.g "To = COPY SrcReg:SubReg"
785static bool copySubReg(MachineInstr &I, MachineRegisterInfo &MRI,
786 const RegisterBankInfo &RBI, Register SrcReg,
787 const TargetRegisterClass *To, unsigned SubReg) {
788 assert(SrcReg.isValid() && "Expected a valid source register?")((SrcReg.isValid() && "Expected a valid source register?"
) ? static_cast<void> (0) : __assert_fail ("SrcReg.isValid() && \"Expected a valid source register?\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 788, __PRETTY_FUNCTION__))
;
789 assert(To && "Destination register class cannot be null")((To && "Destination register class cannot be null") ?
static_cast<void> (0) : __assert_fail ("To && \"Destination register class cannot be null\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 789, __PRETTY_FUNCTION__))
;
790 assert(SubReg && "Expected a valid subregister")((SubReg && "Expected a valid subregister") ? static_cast
<void> (0) : __assert_fail ("SubReg && \"Expected a valid subregister\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 790, __PRETTY_FUNCTION__))
;
791
792 MachineIRBuilder MIB(I);
793 auto SubRegCopy =
794 MIB.buildInstr(TargetOpcode::COPY, {To}, {}).addReg(SrcReg, 0, SubReg);
795 MachineOperand &RegOp = I.getOperand(1);
796 RegOp.setReg(SubRegCopy.getReg(0));
797
798 // It's possible that the destination register won't be constrained. Make
799 // sure that happens.
800 if (!Register::isPhysicalRegister(I.getOperand(0).getReg()))
801 RBI.constrainGenericRegister(I.getOperand(0).getReg(), *To, MRI);
802
803 return true;
804}
805
806/// Helper function to get the source and destination register classes for a
807/// copy. Returns a std::pair containing the source register class for the
808/// copy, and the destination register class for the copy. If a register class
809/// cannot be determined, then it will be nullptr.
810static std::pair<const TargetRegisterClass *, const TargetRegisterClass *>
811getRegClassesForCopy(MachineInstr &I, const TargetInstrInfo &TII,
812 MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
813 const RegisterBankInfo &RBI) {
814 Register DstReg = I.getOperand(0).getReg();
815 Register SrcReg = I.getOperand(1).getReg();
816 const RegisterBank &DstRegBank = *RBI.getRegBank(DstReg, MRI, TRI);
817 const RegisterBank &SrcRegBank = *RBI.getRegBank(SrcReg, MRI, TRI);
818 unsigned DstSize = RBI.getSizeInBits(DstReg, MRI, TRI);
819 unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI);
820
821 // Special casing for cross-bank copies of s1s. We can technically represent
822 // a 1-bit value with any size of register. The minimum size for a GPR is 32
823 // bits. So, we need to put the FPR on 32 bits as well.
824 //
825 // FIXME: I'm not sure if this case holds true outside of copies. If it does,
826 // then we can pull it into the helpers that get the appropriate class for a
827 // register bank. Or make a new helper that carries along some constraint
828 // information.
829 if (SrcRegBank != DstRegBank && (DstSize == 1 && SrcSize == 1))
830 SrcSize = DstSize = 32;
831
832 return {getMinClassForRegBank(SrcRegBank, SrcSize, true),
833 getMinClassForRegBank(DstRegBank, DstSize, true)};
834}
835
836static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII,
837 MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
838 const RegisterBankInfo &RBI) {
839 Register DstReg = I.getOperand(0).getReg();
840 Register SrcReg = I.getOperand(1).getReg();
841 const RegisterBank &DstRegBank = *RBI.getRegBank(DstReg, MRI, TRI);
842 const RegisterBank &SrcRegBank = *RBI.getRegBank(SrcReg, MRI, TRI);
843
844 // Find the correct register classes for the source and destination registers.
845 const TargetRegisterClass *SrcRC;
846 const TargetRegisterClass *DstRC;
847 std::tie(SrcRC, DstRC) = getRegClassesForCopy(I, TII, MRI, TRI, RBI);
24
Calling 'tie<const llvm::TargetRegisterClass *, const llvm::TargetRegisterClass *>'
35
Returning from 'tie<const llvm::TargetRegisterClass *, const llvm::TargetRegisterClass *>'
36
Calling 'tuple::operator='
39
Returning from 'tuple::operator='
848
849 if (!DstRC) {
40
Assuming 'DstRC' is non-null
41
Taking false branch
850 LLVM_DEBUG(dbgs() << "Unexpected dest size "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unexpected dest size " <<
RBI.getSizeInBits(DstReg, MRI, TRI) << '\n'; } } while
(false)
851 << RBI.getSizeInBits(DstReg, MRI, TRI) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unexpected dest size " <<
RBI.getSizeInBits(DstReg, MRI, TRI) << '\n'; } } while
(false)
;
852 return false;
853 }
854
855 // A couple helpers below, for making sure that the copy we produce is valid.
856
857 // Set to true if we insert a SUBREG_TO_REG. If we do this, then we don't want
858 // to verify that the src and dst are the same size, since that's handled by
859 // the SUBREG_TO_REG.
860 bool KnownValid = false;
861
862 // Returns true, or asserts if something we don't expect happens. Instead of
863 // returning true, we return isValidCopy() to ensure that we verify the
864 // result.
865 auto CheckCopy = [&]() {
866 // If we have a bitcast or something, we can't have physical registers.
867 assert((I.isCopy() ||(((I.isCopy() || (!Register::isPhysicalRegister(I.getOperand(
0).getReg()) && !Register::isPhysicalRegister(I.getOperand
(1).getReg()))) && "No phys reg on generic operator!"
) ? static_cast<void> (0) : __assert_fail ("(I.isCopy() || (!Register::isPhysicalRegister(I.getOperand(0).getReg()) && !Register::isPhysicalRegister(I.getOperand(1).getReg()))) && \"No phys reg on generic operator!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 870, __PRETTY_FUNCTION__))
868 (!Register::isPhysicalRegister(I.getOperand(0).getReg()) &&(((I.isCopy() || (!Register::isPhysicalRegister(I.getOperand(
0).getReg()) && !Register::isPhysicalRegister(I.getOperand
(1).getReg()))) && "No phys reg on generic operator!"
) ? static_cast<void> (0) : __assert_fail ("(I.isCopy() || (!Register::isPhysicalRegister(I.getOperand(0).getReg()) && !Register::isPhysicalRegister(I.getOperand(1).getReg()))) && \"No phys reg on generic operator!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 870, __PRETTY_FUNCTION__))
869 !Register::isPhysicalRegister(I.getOperand(1).getReg()))) &&(((I.isCopy() || (!Register::isPhysicalRegister(I.getOperand(
0).getReg()) && !Register::isPhysicalRegister(I.getOperand
(1).getReg()))) && "No phys reg on generic operator!"
) ? static_cast<void> (0) : __assert_fail ("(I.isCopy() || (!Register::isPhysicalRegister(I.getOperand(0).getReg()) && !Register::isPhysicalRegister(I.getOperand(1).getReg()))) && \"No phys reg on generic operator!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 870, __PRETTY_FUNCTION__))
870 "No phys reg on generic operator!")(((I.isCopy() || (!Register::isPhysicalRegister(I.getOperand(
0).getReg()) && !Register::isPhysicalRegister(I.getOperand
(1).getReg()))) && "No phys reg on generic operator!"
) ? static_cast<void> (0) : __assert_fail ("(I.isCopy() || (!Register::isPhysicalRegister(I.getOperand(0).getReg()) && !Register::isPhysicalRegister(I.getOperand(1).getReg()))) && \"No phys reg on generic operator!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 870, __PRETTY_FUNCTION__))
;
871 bool ValidCopy = true;
872#ifndef NDEBUG
873 ValidCopy = KnownValid || isValidCopy(I, DstRegBank, MRI, TRI, RBI);
874 assert(ValidCopy && "Invalid copy.")((ValidCopy && "Invalid copy.") ? static_cast<void
> (0) : __assert_fail ("ValidCopy && \"Invalid copy.\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 874, __PRETTY_FUNCTION__))
;
875 (void)KnownValid;
876#endif
877 return ValidCopy;
878 };
879
880 // Is this a copy? If so, then we may need to insert a subregister copy.
881 if (I.isCopy()) {
42
Calling 'MachineInstr::isCopy'
45
Returning from 'MachineInstr::isCopy'
46
Taking true branch
882 // Yes. Check if there's anything to fix up.
883 if (!SrcRC) {
47
Assuming 'SrcRC' is non-null
48
Taking false branch
884 LLVM_DEBUG(dbgs() << "Couldn't determine source register class\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't determine source register class\n"
; } } while (false)
;
885 return false;
886 }
887
888 unsigned SrcSize = TRI.getRegSizeInBits(*SrcRC);
889 unsigned DstSize = TRI.getRegSizeInBits(*DstRC);
890 unsigned SubReg;
49
'SubReg' declared without an initial value
891
892 // If the source bank doesn't support a subregister copy small enough,
893 // then we first need to copy to the destination bank.
894 if (getMinSizeForRegBank(SrcRegBank) > DstSize) {
50
Assuming the condition is false
51
Taking false branch
895 const TargetRegisterClass *DstTempRC =
896 getMinClassForRegBank(DstRegBank, SrcSize, /* GetAllRegSet */ true);
897 getSubRegForClass(DstRC, TRI, SubReg);
898
899 MachineIRBuilder MIB(I);
900 auto Copy = MIB.buildCopy({DstTempRC}, {SrcReg});
901 copySubReg(I, MRI, RBI, Copy.getReg(0), DstRC, SubReg);
902 } else if (SrcSize > DstSize) {
52
Assuming 'SrcSize' is > 'DstSize'
53
Taking true branch
903 // If the source register is bigger than the destination we need to
904 // perform a subregister copy.
905 const TargetRegisterClass *SubRegRC =
906 getMinClassForRegBank(SrcRegBank, DstSize, /* GetAllRegSet */ true);
907 getSubRegForClass(SubRegRC, TRI, SubReg);
54
Calling 'getSubRegForClass'
59
Returning from 'getSubRegForClass'
908 copySubReg(I, MRI, RBI, SrcReg, DstRC, SubReg);
60
6th function call argument is an uninitialized value
909 } else if (DstSize > SrcSize) {
910 // If the destination register is bigger than the source we need to do
911 // a promotion using SUBREG_TO_REG.
912 const TargetRegisterClass *PromotionRC =
913 getMinClassForRegBank(SrcRegBank, DstSize, /* GetAllRegSet */ true);
914 getSubRegForClass(SrcRC, TRI, SubReg);
915
916 Register PromoteReg = MRI.createVirtualRegister(PromotionRC);
917 BuildMI(*I.getParent(), I, I.getDebugLoc(),
918 TII.get(AArch64::SUBREG_TO_REG), PromoteReg)
919 .addImm(0)
920 .addUse(SrcReg)
921 .addImm(SubReg);
922 MachineOperand &RegOp = I.getOperand(1);
923 RegOp.setReg(PromoteReg);
924
925 // Promise that the copy is implicitly validated by the SUBREG_TO_REG.
926 KnownValid = true;
927 }
928
929 // If the destination is a physical register, then there's nothing to
930 // change, so we're done.
931 if (Register::isPhysicalRegister(DstReg))
932 return CheckCopy();
933 }
934
935 // No need to constrain SrcReg. It will get constrained when we hit another
936 // of its use or its defs. Copies do not have constraints.
937 if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
938 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain " <<
TII.getName(I.getOpcode()) << " operand\n"; } } while (
false)
939 << " operand\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain " <<
TII.getName(I.getOpcode()) << " operand\n"; } } while (
false)
;
940 return false;
941 }
942 I.setDesc(TII.get(AArch64::COPY));
943 return CheckCopy();
944}
945
946static unsigned selectFPConvOpc(unsigned GenericOpc, LLT DstTy, LLT SrcTy) {
947 if (!DstTy.isScalar() || !SrcTy.isScalar())
948 return GenericOpc;
949
950 const unsigned DstSize = DstTy.getSizeInBits();
951 const unsigned SrcSize = SrcTy.getSizeInBits();
952
953 switch (DstSize) {
954 case 32:
955 switch (SrcSize) {
956 case 32:
957 switch (GenericOpc) {
958 case TargetOpcode::G_SITOFP:
959 return AArch64::SCVTFUWSri;
960 case TargetOpcode::G_UITOFP:
961 return AArch64::UCVTFUWSri;
962 case TargetOpcode::G_FPTOSI:
963 return AArch64::FCVTZSUWSr;
964 case TargetOpcode::G_FPTOUI:
965 return AArch64::FCVTZUUWSr;
966 default:
967 return GenericOpc;
968 }
969 case 64:
970 switch (GenericOpc) {
971 case TargetOpcode::G_SITOFP:
972 return AArch64::SCVTFUXSri;
973 case TargetOpcode::G_UITOFP:
974 return AArch64::UCVTFUXSri;
975 case TargetOpcode::G_FPTOSI:
976 return AArch64::FCVTZSUWDr;
977 case TargetOpcode::G_FPTOUI:
978 return AArch64::FCVTZUUWDr;
979 default:
980 return GenericOpc;
981 }
982 default:
983 return GenericOpc;
984 }
985 case 64:
986 switch (SrcSize) {
987 case 32:
988 switch (GenericOpc) {
989 case TargetOpcode::G_SITOFP:
990 return AArch64::SCVTFUWDri;
991 case TargetOpcode::G_UITOFP:
992 return AArch64::UCVTFUWDri;
993 case TargetOpcode::G_FPTOSI:
994 return AArch64::FCVTZSUXSr;
995 case TargetOpcode::G_FPTOUI:
996 return AArch64::FCVTZUUXSr;
997 default:
998 return GenericOpc;
999 }
1000 case 64:
1001 switch (GenericOpc) {
1002 case TargetOpcode::G_SITOFP:
1003 return AArch64::SCVTFUXDri;
1004 case TargetOpcode::G_UITOFP:
1005 return AArch64::UCVTFUXDri;
1006 case TargetOpcode::G_FPTOSI:
1007 return AArch64::FCVTZSUXDr;
1008 case TargetOpcode::G_FPTOUI:
1009 return AArch64::FCVTZUUXDr;
1010 default:
1011 return GenericOpc;
1012 }
1013 default:
1014 return GenericOpc;
1015 }
1016 default:
1017 return GenericOpc;
1018 };
1019 return GenericOpc;
1020}
1021
1022MachineInstr *
1023AArch64InstructionSelector::emitSelect(Register Dst, Register True,
1024 Register False, AArch64CC::CondCode CC,
1025 MachineIRBuilder &MIB) const {
1026 MachineRegisterInfo &MRI = *MIB.getMRI();
1027 assert(RBI.getRegBank(False, MRI, TRI)->getID() ==((RBI.getRegBank(False, MRI, TRI)->getID() == RBI.getRegBank
(True, MRI, TRI)->getID() && "Expected both select operands to have the same regbank?"
) ? static_cast<void> (0) : __assert_fail ("RBI.getRegBank(False, MRI, TRI)->getID() == RBI.getRegBank(True, MRI, TRI)->getID() && \"Expected both select operands to have the same regbank?\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1029, __PRETTY_FUNCTION__))
1028 RBI.getRegBank(True, MRI, TRI)->getID() &&((RBI.getRegBank(False, MRI, TRI)->getID() == RBI.getRegBank
(True, MRI, TRI)->getID() && "Expected both select operands to have the same regbank?"
) ? static_cast<void> (0) : __assert_fail ("RBI.getRegBank(False, MRI, TRI)->getID() == RBI.getRegBank(True, MRI, TRI)->getID() && \"Expected both select operands to have the same regbank?\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1029, __PRETTY_FUNCTION__))
1029 "Expected both select operands to have the same regbank?")((RBI.getRegBank(False, MRI, TRI)->getID() == RBI.getRegBank
(True, MRI, TRI)->getID() && "Expected both select operands to have the same regbank?"
) ? static_cast<void> (0) : __assert_fail ("RBI.getRegBank(False, MRI, TRI)->getID() == RBI.getRegBank(True, MRI, TRI)->getID() && \"Expected both select operands to have the same regbank?\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1029, __PRETTY_FUNCTION__))
;
1030 LLT Ty = MRI.getType(True);
1031 if (Ty.isVector())
1032 return nullptr;
1033 const unsigned Size = Ty.getSizeInBits();
1034 assert((Size == 32 || Size == 64) &&(((Size == 32 || Size == 64) && "Expected 32 bit or 64 bit select only?"
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Expected 32 bit or 64 bit select only?\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1035, __PRETTY_FUNCTION__))
1035 "Expected 32 bit or 64 bit select only?")(((Size == 32 || Size == 64) && "Expected 32 bit or 64 bit select only?"
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Expected 32 bit or 64 bit select only?\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1035, __PRETTY_FUNCTION__))
;
1036 const bool Is32Bit = Size == 32;
1037 if (RBI.getRegBank(True, MRI, TRI)->getID() != AArch64::GPRRegBankID) {
1038 unsigned Opc = Is32Bit ? AArch64::FCSELSrrr : AArch64::FCSELDrrr;
1039 auto FCSel = MIB.buildInstr(Opc, {Dst}, {True, False}).addImm(CC);
1040 constrainSelectedInstRegOperands(*FCSel, TII, TRI, RBI);
1041 return &*FCSel;
1042 }
1043
1044 // By default, we'll try and emit a CSEL.
1045 unsigned Opc = Is32Bit ? AArch64::CSELWr : AArch64::CSELXr;
1046 bool Optimized = false;
1047 auto TryFoldBinOpIntoSelect = [&Opc, Is32Bit, &CC, &MRI,
1048 &Optimized](Register &Reg, Register &OtherReg,
1049 bool Invert) {
1050 if (Optimized)
1051 return false;
1052
1053 // Attempt to fold:
1054 //
1055 // %sub = G_SUB 0, %x
1056 // %select = G_SELECT cc, %reg, %sub
1057 //
1058 // Into:
1059 // %select = CSNEG %reg, %x, cc
1060 Register MatchReg;
1061 if (mi_match(Reg, MRI, m_Neg(m_Reg(MatchReg)))) {
1062 Opc = Is32Bit ? AArch64::CSNEGWr : AArch64::CSNEGXr;
1063 Reg = MatchReg;
1064 if (Invert) {
1065 CC = AArch64CC::getInvertedCondCode(CC);
1066 std::swap(Reg, OtherReg);
1067 }
1068 return true;
1069 }
1070
1071 // Attempt to fold:
1072 //
1073 // %xor = G_XOR %x, -1
1074 // %select = G_SELECT cc, %reg, %xor
1075 //
1076 // Into:
1077 // %select = CSINV %reg, %x, cc
1078 if (mi_match(Reg, MRI, m_Not(m_Reg(MatchReg)))) {
1079 Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
1080 Reg = MatchReg;
1081 if (Invert) {
1082 CC = AArch64CC::getInvertedCondCode(CC);
1083 std::swap(Reg, OtherReg);
1084 }
1085 return true;
1086 }
1087
1088 // Attempt to fold:
1089 //
1090 // %add = G_ADD %x, 1
1091 // %select = G_SELECT cc, %reg, %add
1092 //
1093 // Into:
1094 // %select = CSINC %reg, %x, cc
1095 if (mi_match(Reg, MRI,
1096 m_any_of(m_GAdd(m_Reg(MatchReg), m_SpecificICst(1)),
1097 m_GPtrAdd(m_Reg(MatchReg), m_SpecificICst(1))))) {
1098 Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
1099 Reg = MatchReg;
1100 if (Invert) {
1101 CC = AArch64CC::getInvertedCondCode(CC);
1102 std::swap(Reg, OtherReg);
1103 }
1104 return true;
1105 }
1106
1107 return false;
1108 };
1109
1110 // Helper lambda which tries to use CSINC/CSINV for the instruction when its
1111 // true/false values are constants.
1112 // FIXME: All of these patterns already exist in tablegen. We should be
1113 // able to import these.
1114 auto TryOptSelectCst = [&Opc, &True, &False, &CC, Is32Bit, &MRI,
1115 &Optimized]() {
1116 if (Optimized)
1117 return false;
1118 auto TrueCst = getConstantVRegValWithLookThrough(True, MRI);
1119 auto FalseCst = getConstantVRegValWithLookThrough(False, MRI);
1120 if (!TrueCst && !FalseCst)
1121 return false;
1122
1123 Register ZReg = Is32Bit ? AArch64::WZR : AArch64::XZR;
1124 if (TrueCst && FalseCst) {
1125 int64_t T = TrueCst->Value.getSExtValue();
1126 int64_t F = FalseCst->Value.getSExtValue();
1127
1128 if (T == 0 && F == 1) {
1129 // G_SELECT cc, 0, 1 -> CSINC zreg, zreg, cc
1130 Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
1131 True = ZReg;
1132 False = ZReg;
1133 return true;
1134 }
1135
1136 if (T == 0 && F == -1) {
1137 // G_SELECT cc 0, -1 -> CSINV zreg, zreg cc
1138 Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
1139 True = ZReg;
1140 False = ZReg;
1141 return true;
1142 }
1143 }
1144
1145 if (TrueCst) {
1146 int64_t T = TrueCst->Value.getSExtValue();
1147 if (T == 1) {
1148 // G_SELECT cc, 1, f -> CSINC f, zreg, inv_cc
1149 Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
1150 True = False;
1151 False = ZReg;
1152 CC = AArch64CC::getInvertedCondCode(CC);
1153 return true;
1154 }
1155
1156 if (T == -1) {
1157 // G_SELECT cc, -1, f -> CSINV f, zreg, inv_cc
1158 Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
1159 True = False;
1160 False = ZReg;
1161 CC = AArch64CC::getInvertedCondCode(CC);
1162 return true;
1163 }
1164 }
1165
1166 if (FalseCst) {
1167 int64_t F = FalseCst->Value.getSExtValue();
1168 if (F == 1) {
1169 // G_SELECT cc, t, 1 -> CSINC t, zreg, cc
1170 Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
1171 False = ZReg;
1172 return true;
1173 }
1174
1175 if (F == -1) {
1176 // G_SELECT cc, t, -1 -> CSINC t, zreg, cc
1177 Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
1178 False = ZReg;
1179 return true;
1180 }
1181 }
1182 return false;
1183 };
1184
1185 Optimized |= TryFoldBinOpIntoSelect(False, True, /*Invert = */ false);
1186 Optimized |= TryFoldBinOpIntoSelect(True, False, /*Invert = */ true);
1187 Optimized |= TryOptSelectCst();
1188 auto SelectInst = MIB.buildInstr(Opc, {Dst}, {True, False}).addImm(CC);
1189 constrainSelectedInstRegOperands(*SelectInst, TII, TRI, RBI);
1190 return &*SelectInst;
1191}
1192
1193static AArch64CC::CondCode changeICMPPredToAArch64CC(CmpInst::Predicate P) {
1194 switch (P) {
1195 default:
1196 llvm_unreachable("Unknown condition code!")::llvm::llvm_unreachable_internal("Unknown condition code!", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1196)
;
1197 case CmpInst::ICMP_NE:
1198 return AArch64CC::NE;
1199 case CmpInst::ICMP_EQ:
1200 return AArch64CC::EQ;
1201 case CmpInst::ICMP_SGT:
1202 return AArch64CC::GT;
1203 case CmpInst::ICMP_SGE:
1204 return AArch64CC::GE;
1205 case CmpInst::ICMP_SLT:
1206 return AArch64CC::LT;
1207 case CmpInst::ICMP_SLE:
1208 return AArch64CC::LE;
1209 case CmpInst::ICMP_UGT:
1210 return AArch64CC::HI;
1211 case CmpInst::ICMP_UGE:
1212 return AArch64CC::HS;
1213 case CmpInst::ICMP_ULT:
1214 return AArch64CC::LO;
1215 case CmpInst::ICMP_ULE:
1216 return AArch64CC::LS;
1217 }
1218}
1219
1220static void changeFCMPPredToAArch64CC(CmpInst::Predicate P,
1221 AArch64CC::CondCode &CondCode,
1222 AArch64CC::CondCode &CondCode2) {
1223 CondCode2 = AArch64CC::AL;
1224 switch (P) {
1225 default:
1226 llvm_unreachable("Unknown FP condition!")::llvm::llvm_unreachable_internal("Unknown FP condition!", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1226)
;
1227 case CmpInst::FCMP_OEQ:
1228 CondCode = AArch64CC::EQ;
1229 break;
1230 case CmpInst::FCMP_OGT:
1231 CondCode = AArch64CC::GT;
1232 break;
1233 case CmpInst::FCMP_OGE:
1234 CondCode = AArch64CC::GE;
1235 break;
1236 case CmpInst::FCMP_OLT:
1237 CondCode = AArch64CC::MI;
1238 break;
1239 case CmpInst::FCMP_OLE:
1240 CondCode = AArch64CC::LS;
1241 break;
1242 case CmpInst::FCMP_ONE:
1243 CondCode = AArch64CC::MI;
1244 CondCode2 = AArch64CC::GT;
1245 break;
1246 case CmpInst::FCMP_ORD:
1247 CondCode = AArch64CC::VC;
1248 break;
1249 case CmpInst::FCMP_UNO:
1250 CondCode = AArch64CC::VS;
1251 break;
1252 case CmpInst::FCMP_UEQ:
1253 CondCode = AArch64CC::EQ;
1254 CondCode2 = AArch64CC::VS;
1255 break;
1256 case CmpInst::FCMP_UGT:
1257 CondCode = AArch64CC::HI;
1258 break;
1259 case CmpInst::FCMP_UGE:
1260 CondCode = AArch64CC::PL;
1261 break;
1262 case CmpInst::FCMP_ULT:
1263 CondCode = AArch64CC::LT;
1264 break;
1265 case CmpInst::FCMP_ULE:
1266 CondCode = AArch64CC::LE;
1267 break;
1268 case CmpInst::FCMP_UNE:
1269 CondCode = AArch64CC::NE;
1270 break;
1271 }
1272}
1273
1274/// Return a register which can be used as a bit to test in a TB(N)Z.
1275static Register getTestBitReg(Register Reg, uint64_t &Bit, bool &Invert,
1276 MachineRegisterInfo &MRI) {
1277 assert(Reg.isValid() && "Expected valid register!")((Reg.isValid() && "Expected valid register!") ? static_cast
<void> (0) : __assert_fail ("Reg.isValid() && \"Expected valid register!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1277, __PRETTY_FUNCTION__))
;
1278 while (MachineInstr *MI = getDefIgnoringCopies(Reg, MRI)) {
1279 unsigned Opc = MI->getOpcode();
1280
1281 if (!MI->getOperand(0).isReg() ||
1282 !MRI.hasOneNonDBGUse(MI->getOperand(0).getReg()))
1283 break;
1284
1285 // (tbz (any_ext x), b) -> (tbz x, b) if we don't use the extended bits.
1286 //
1287 // (tbz (trunc x), b) -> (tbz x, b) is always safe, because the bit number
1288 // on the truncated x is the same as the bit number on x.
1289 if (Opc == TargetOpcode::G_ANYEXT || Opc == TargetOpcode::G_ZEXT ||
1290 Opc == TargetOpcode::G_TRUNC) {
1291 Register NextReg = MI->getOperand(1).getReg();
1292 // Did we find something worth folding?
1293 if (!NextReg.isValid() || !MRI.hasOneNonDBGUse(NextReg))
1294 break;
1295
1296 // NextReg is worth folding. Keep looking.
1297 Reg = NextReg;
1298 continue;
1299 }
1300
1301 // Attempt to find a suitable operation with a constant on one side.
1302 Optional<uint64_t> C;
1303 Register TestReg;
1304 switch (Opc) {
1305 default:
1306 break;
1307 case TargetOpcode::G_AND:
1308 case TargetOpcode::G_XOR: {
1309 TestReg = MI->getOperand(1).getReg();
1310 Register ConstantReg = MI->getOperand(2).getReg();
1311 auto VRegAndVal = getConstantVRegValWithLookThrough(ConstantReg, MRI);
1312 if (!VRegAndVal) {
1313 // AND commutes, check the other side for a constant.
1314 // FIXME: Can we canonicalize the constant so that it's always on the
1315 // same side at some point earlier?
1316 std::swap(ConstantReg, TestReg);
1317 VRegAndVal = getConstantVRegValWithLookThrough(ConstantReg, MRI);
1318 }
1319 if (VRegAndVal)
1320 C = VRegAndVal->Value.getSExtValue();
1321 break;
1322 }
1323 case TargetOpcode::G_ASHR:
1324 case TargetOpcode::G_LSHR:
1325 case TargetOpcode::G_SHL: {
1326 TestReg = MI->getOperand(1).getReg();
1327 auto VRegAndVal =
1328 getConstantVRegValWithLookThrough(MI->getOperand(2).getReg(), MRI);
1329 if (VRegAndVal)
1330 C = VRegAndVal->Value.getSExtValue();
1331 break;
1332 }
1333 }
1334
1335 // Didn't find a constant or viable register. Bail out of the loop.
1336 if (!C || !TestReg.isValid())
1337 break;
1338
1339 // We found a suitable instruction with a constant. Check to see if we can
1340 // walk through the instruction.
1341 Register NextReg;
1342 unsigned TestRegSize = MRI.getType(TestReg).getSizeInBits();
1343 switch (Opc) {
1344 default:
1345 break;
1346 case TargetOpcode::G_AND:
1347 // (tbz (and x, m), b) -> (tbz x, b) when the b-th bit of m is set.
1348 if ((*C >> Bit) & 1)
1349 NextReg = TestReg;
1350 break;
1351 case TargetOpcode::G_SHL:
1352 // (tbz (shl x, c), b) -> (tbz x, b-c) when b-c is positive and fits in
1353 // the type of the register.
1354 if (*C <= Bit && (Bit - *C) < TestRegSize) {
1355 NextReg = TestReg;
1356 Bit = Bit - *C;
1357 }
1358 break;
1359 case TargetOpcode::G_ASHR:
1360 // (tbz (ashr x, c), b) -> (tbz x, b+c) or (tbz x, msb) if b+c is > # bits
1361 // in x
1362 NextReg = TestReg;
1363 Bit = Bit + *C;
1364 if (Bit >= TestRegSize)
1365 Bit = TestRegSize - 1;
1366 break;
1367 case TargetOpcode::G_LSHR:
1368 // (tbz (lshr x, c), b) -> (tbz x, b+c) when b + c is < # bits in x
1369 if ((Bit + *C) < TestRegSize) {
1370 NextReg = TestReg;
1371 Bit = Bit + *C;
1372 }
1373 break;
1374 case TargetOpcode::G_XOR:
1375 // We can walk through a G_XOR by inverting whether we use tbz/tbnz when
1376 // appropriate.
1377 //
1378 // e.g. If x' = xor x, c, and the b-th bit is set in c then
1379 //
1380 // tbz x', b -> tbnz x, b
1381 //
1382 // Because x' only has the b-th bit set if x does not.
1383 if ((*C >> Bit) & 1)
1384 Invert = !Invert;
1385 NextReg = TestReg;
1386 break;
1387 }
1388
1389 // Check if we found anything worth folding.
1390 if (!NextReg.isValid())
1391 return Reg;
1392 Reg = NextReg;
1393 }
1394
1395 return Reg;
1396}
1397
1398MachineInstr *AArch64InstructionSelector::emitTestBit(
1399 Register TestReg, uint64_t Bit, bool IsNegative, MachineBasicBlock *DstMBB,
1400 MachineIRBuilder &MIB) const {
1401 assert(TestReg.isValid())((TestReg.isValid()) ? static_cast<void> (0) : __assert_fail
("TestReg.isValid()", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1401, __PRETTY_FUNCTION__))
;
1402 assert(ProduceNonFlagSettingCondBr &&((ProduceNonFlagSettingCondBr && "Cannot emit TB(N)Z with speculation tracking!"
) ? static_cast<void> (0) : __assert_fail ("ProduceNonFlagSettingCondBr && \"Cannot emit TB(N)Z with speculation tracking!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1403, __PRETTY_FUNCTION__))
1403 "Cannot emit TB(N)Z with speculation tracking!")((ProduceNonFlagSettingCondBr && "Cannot emit TB(N)Z with speculation tracking!"
) ? static_cast<void> (0) : __assert_fail ("ProduceNonFlagSettingCondBr && \"Cannot emit TB(N)Z with speculation tracking!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1403, __PRETTY_FUNCTION__))
;
1404 MachineRegisterInfo &MRI = *MIB.getMRI();
1405
1406 // Attempt to optimize the test bit by walking over instructions.
1407 TestReg = getTestBitReg(TestReg, Bit, IsNegative, MRI);
1408 LLT Ty = MRI.getType(TestReg);
1409 unsigned Size = Ty.getSizeInBits();
1410 assert(!Ty.isVector() && "Expected a scalar!")((!Ty.isVector() && "Expected a scalar!") ? static_cast
<void> (0) : __assert_fail ("!Ty.isVector() && \"Expected a scalar!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1410, __PRETTY_FUNCTION__))
;
1411 assert(Bit < 64 && "Bit is too large!")((Bit < 64 && "Bit is too large!") ? static_cast<
void> (0) : __assert_fail ("Bit < 64 && \"Bit is too large!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1411, __PRETTY_FUNCTION__))
;
1412
1413 // When the test register is a 64-bit register, we have to narrow to make
1414 // TBNZW work.
1415 bool UseWReg = Bit < 32;
1416 unsigned NecessarySize = UseWReg ? 32 : 64;
1417 if (Size != NecessarySize)
1418 TestReg = moveScalarRegClass(
1419 TestReg, UseWReg ? AArch64::GPR32RegClass : AArch64::GPR64RegClass,
1420 MIB);
1421
1422 static const unsigned OpcTable[2][2] = {{AArch64::TBZX, AArch64::TBNZX},
1423 {AArch64::TBZW, AArch64::TBNZW}};
1424 unsigned Opc = OpcTable[UseWReg][IsNegative];
1425 auto TestBitMI =
1426 MIB.buildInstr(Opc).addReg(TestReg).addImm(Bit).addMBB(DstMBB);
1427 constrainSelectedInstRegOperands(*TestBitMI, TII, TRI, RBI);
1428 return &*TestBitMI;
1429}
1430
1431bool AArch64InstructionSelector::tryOptAndIntoCompareBranch(
1432 MachineInstr &AndInst, bool Invert, MachineBasicBlock *DstMBB,
1433 MachineIRBuilder &MIB) const {
1434 assert(AndInst.getOpcode() == TargetOpcode::G_AND && "Expected G_AND only?")((AndInst.getOpcode() == TargetOpcode::G_AND && "Expected G_AND only?"
) ? static_cast<void> (0) : __assert_fail ("AndInst.getOpcode() == TargetOpcode::G_AND && \"Expected G_AND only?\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1434, __PRETTY_FUNCTION__))
;
1435 // Given something like this:
1436 //
1437 // %x = ...Something...
1438 // %one = G_CONSTANT i64 1
1439 // %zero = G_CONSTANT i64 0
1440 // %and = G_AND %x, %one
1441 // %cmp = G_ICMP intpred(ne), %and, %zero
1442 // %cmp_trunc = G_TRUNC %cmp
1443 // G_BRCOND %cmp_trunc, %bb.3
1444 //
1445 // We want to try and fold the AND into the G_BRCOND and produce either a
1446 // TBNZ (when we have intpred(ne)) or a TBZ (when we have intpred(eq)).
1447 //
1448 // In this case, we'd get
1449 //
1450 // TBNZ %x %bb.3
1451 //
1452
1453 // Check if the AND has a constant on its RHS which we can use as a mask.
1454 // If it's a power of 2, then it's the same as checking a specific bit.
1455 // (e.g, ANDing with 8 == ANDing with 000...100 == testing if bit 3 is set)
1456 auto MaybeBit = getConstantVRegValWithLookThrough(
1457 AndInst.getOperand(2).getReg(), *MIB.getMRI());
1458 if (!MaybeBit)
1459 return false;
1460
1461 int32_t Bit = MaybeBit->Value.exactLogBase2();
1462 if (Bit < 0)
1463 return false;
1464
1465 Register TestReg = AndInst.getOperand(1).getReg();
1466
1467 // Emit a TB(N)Z.
1468 emitTestBit(TestReg, Bit, Invert, DstMBB, MIB);
1469 return true;
1470}
1471
1472MachineInstr *AArch64InstructionSelector::emitCBZ(Register CompareReg,
1473 bool IsNegative,
1474 MachineBasicBlock *DestMBB,
1475 MachineIRBuilder &MIB) const {
1476 assert(ProduceNonFlagSettingCondBr && "CBZ does not set flags!")((ProduceNonFlagSettingCondBr && "CBZ does not set flags!"
) ? static_cast<void> (0) : __assert_fail ("ProduceNonFlagSettingCondBr && \"CBZ does not set flags!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1476, __PRETTY_FUNCTION__))
;
1477 MachineRegisterInfo &MRI = *MIB.getMRI();
1478 assert(RBI.getRegBank(CompareReg, MRI, TRI)->getID() ==((RBI.getRegBank(CompareReg, MRI, TRI)->getID() == AArch64
::GPRRegBankID && "Expected GPRs only?") ? static_cast
<void> (0) : __assert_fail ("RBI.getRegBank(CompareReg, MRI, TRI)->getID() == AArch64::GPRRegBankID && \"Expected GPRs only?\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1480, __PRETTY_FUNCTION__))
1479 AArch64::GPRRegBankID &&((RBI.getRegBank(CompareReg, MRI, TRI)->getID() == AArch64
::GPRRegBankID && "Expected GPRs only?") ? static_cast
<void> (0) : __assert_fail ("RBI.getRegBank(CompareReg, MRI, TRI)->getID() == AArch64::GPRRegBankID && \"Expected GPRs only?\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1480, __PRETTY_FUNCTION__))
1480 "Expected GPRs only?")((RBI.getRegBank(CompareReg, MRI, TRI)->getID() == AArch64
::GPRRegBankID && "Expected GPRs only?") ? static_cast
<void> (0) : __assert_fail ("RBI.getRegBank(CompareReg, MRI, TRI)->getID() == AArch64::GPRRegBankID && \"Expected GPRs only?\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1480, __PRETTY_FUNCTION__))
;
1481 auto Ty = MRI.getType(CompareReg);
1482 unsigned Width = Ty.getSizeInBits();
1483 assert(!Ty.isVector() && "Expected scalar only?")((!Ty.isVector() && "Expected scalar only?") ? static_cast
<void> (0) : __assert_fail ("!Ty.isVector() && \"Expected scalar only?\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1483, __PRETTY_FUNCTION__))
;
1484 assert(Width <= 64 && "Expected width to be at most 64?")((Width <= 64 && "Expected width to be at most 64?"
) ? static_cast<void> (0) : __assert_fail ("Width <= 64 && \"Expected width to be at most 64?\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1484, __PRETTY_FUNCTION__))
;
1485 static const unsigned OpcTable[2][2] = {{AArch64::CBZW, AArch64::CBZX},
1486 {AArch64::CBNZW, AArch64::CBNZX}};
1487 unsigned Opc = OpcTable[IsNegative][Width == 64];
1488 auto BranchMI = MIB.buildInstr(Opc, {}, {CompareReg}).addMBB(DestMBB);
1489 constrainSelectedInstRegOperands(*BranchMI, TII, TRI, RBI);
1490 return &*BranchMI;
1491}
1492
1493bool AArch64InstructionSelector::selectCompareBranchFedByFCmp(
1494 MachineInstr &I, MachineInstr &FCmp, MachineIRBuilder &MIB) const {
1495 assert(FCmp.getOpcode() == TargetOpcode::G_FCMP)((FCmp.getOpcode() == TargetOpcode::G_FCMP) ? static_cast<
void> (0) : __assert_fail ("FCmp.getOpcode() == TargetOpcode::G_FCMP"
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1495, __PRETTY_FUNCTION__))
;
1496 assert(I.getOpcode() == TargetOpcode::G_BRCOND)((I.getOpcode() == TargetOpcode::G_BRCOND) ? static_cast<void
> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BRCOND"
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1496, __PRETTY_FUNCTION__))
;
1497 // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't
1498 // totally clean. Some of them require two branches to implement.
1499 auto Pred = (CmpInst::Predicate)FCmp.getOperand(1).getPredicate();
1500 emitFPCompare(FCmp.getOperand(2).getReg(), FCmp.getOperand(3).getReg(), MIB,
1501 Pred);
1502 AArch64CC::CondCode CC1, CC2;
1503 changeFCMPPredToAArch64CC(static_cast<CmpInst::Predicate>(Pred), CC1, CC2);
1504 MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
1505 MIB.buildInstr(AArch64::Bcc, {}, {}).addImm(CC1).addMBB(DestMBB);
1506 if (CC2 != AArch64CC::AL)
1507 MIB.buildInstr(AArch64::Bcc, {}, {}).addImm(CC2).addMBB(DestMBB);
1508 I.eraseFromParent();
1509 return true;
1510}
1511
1512bool AArch64InstructionSelector::tryOptCompareBranchFedByICmp(
1513 MachineInstr &I, MachineInstr &ICmp, MachineIRBuilder &MIB) const {
1514 assert(ICmp.getOpcode() == TargetOpcode::G_ICMP)((ICmp.getOpcode() == TargetOpcode::G_ICMP) ? static_cast<
void> (0) : __assert_fail ("ICmp.getOpcode() == TargetOpcode::G_ICMP"
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1514, __PRETTY_FUNCTION__))
;
1515 assert(I.getOpcode() == TargetOpcode::G_BRCOND)((I.getOpcode() == TargetOpcode::G_BRCOND) ? static_cast<void
> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BRCOND"
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1515, __PRETTY_FUNCTION__))
;
1516 // Attempt to optimize the G_BRCOND + G_ICMP into a TB(N)Z/CB(N)Z.
1517 //
1518 // Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z
1519 // instructions will not be produced, as they are conditional branch
1520 // instructions that do not set flags.
1521 if (!ProduceNonFlagSettingCondBr)
1522 return false;
1523
1524 MachineRegisterInfo &MRI = *MIB.getMRI();
1525 MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
1526 auto Pred =
1527 static_cast<CmpInst::Predicate>(ICmp.getOperand(1).getPredicate());
1528 Register LHS = ICmp.getOperand(2).getReg();
1529 Register RHS = ICmp.getOperand(3).getReg();
1530
1531 // We're allowed to emit a TB(N)Z/CB(N)Z. Try to do that.
1532 auto VRegAndVal = getConstantVRegValWithLookThrough(RHS, MRI);
1533 MachineInstr *AndInst = getOpcodeDef(TargetOpcode::G_AND, LHS, MRI);
1534
1535 // When we can emit a TB(N)Z, prefer that.
1536 //
1537 // Handle non-commutative condition codes first.
1538 // Note that we don't want to do this when we have a G_AND because it can
1539 // become a tst. The tst will make the test bit in the TB(N)Z redundant.
1540 if (VRegAndVal && !AndInst) {
1541 int64_t C = VRegAndVal->Value.getSExtValue();
1542
1543 // When we have a greater-than comparison, we can just test if the msb is
1544 // zero.
1545 if (C == -1 && Pred == CmpInst::ICMP_SGT) {
1546 uint64_t Bit = MRI.getType(LHS).getSizeInBits() - 1;
1547 emitTestBit(LHS, Bit, /*IsNegative = */ false, DestMBB, MIB);
1548 I.eraseFromParent();
1549 return true;
1550 }
1551
1552 // When we have a less than comparison, we can just test if the msb is not
1553 // zero.
1554 if (C == 0 && Pred == CmpInst::ICMP_SLT) {
1555 uint64_t Bit = MRI.getType(LHS).getSizeInBits() - 1;
1556 emitTestBit(LHS, Bit, /*IsNegative = */ true, DestMBB, MIB);
1557 I.eraseFromParent();
1558 return true;
1559 }
1560 }
1561
1562 // Attempt to handle commutative condition codes. Right now, that's only
1563 // eq/ne.
1564 if (ICmpInst::isEquality(Pred)) {
1565 if (!VRegAndVal) {
1566 std::swap(RHS, LHS);
1567 VRegAndVal = getConstantVRegValWithLookThrough(RHS, MRI);
1568 AndInst = getOpcodeDef(TargetOpcode::G_AND, LHS, MRI);
1569 }
1570
1571 if (VRegAndVal && VRegAndVal->Value == 0) {
1572 // If there's a G_AND feeding into this branch, try to fold it away by
1573 // emitting a TB(N)Z instead.
1574 //
1575 // Note: If we have LT, then it *is* possible to fold, but it wouldn't be
1576 // beneficial. When we have an AND and LT, we need a TST/ANDS, so folding
1577 // would be redundant.
1578 if (AndInst &&
1579 tryOptAndIntoCompareBranch(
1580 *AndInst, /*Invert = */ Pred == CmpInst::ICMP_NE, DestMBB, MIB)) {
1581 I.eraseFromParent();
1582 return true;
1583 }
1584
1585 // Otherwise, try to emit a CB(N)Z instead.
1586 auto LHSTy = MRI.getType(LHS);
1587 if (!LHSTy.isVector() && LHSTy.getSizeInBits() <= 64) {
1588 emitCBZ(LHS, /*IsNegative = */ Pred == CmpInst::ICMP_NE, DestMBB, MIB);
1589 I.eraseFromParent();
1590 return true;
1591 }
1592 }
1593 }
1594
1595 return false;
1596}
1597
1598bool AArch64InstructionSelector::selectCompareBranchFedByICmp(
1599 MachineInstr &I, MachineInstr &ICmp, MachineIRBuilder &MIB) const {
1600 assert(ICmp.getOpcode() == TargetOpcode::G_ICMP)((ICmp.getOpcode() == TargetOpcode::G_ICMP) ? static_cast<
void> (0) : __assert_fail ("ICmp.getOpcode() == TargetOpcode::G_ICMP"
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1600, __PRETTY_FUNCTION__))
;
1601 assert(I.getOpcode() == TargetOpcode::G_BRCOND)((I.getOpcode() == TargetOpcode::G_BRCOND) ? static_cast<void
> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BRCOND"
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1601, __PRETTY_FUNCTION__))
;
1602 if (tryOptCompareBranchFedByICmp(I, ICmp, MIB))
1603 return true;
1604
1605 // Couldn't optimize. Emit a compare + a Bcc.
1606 MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
1607 auto PredOp = ICmp.getOperand(1);
1608 emitIntegerCompare(ICmp.getOperand(2), ICmp.getOperand(3), PredOp, MIB);
1609 const AArch64CC::CondCode CC = changeICMPPredToAArch64CC(
1610 static_cast<CmpInst::Predicate>(PredOp.getPredicate()));
1611 MIB.buildInstr(AArch64::Bcc, {}, {}).addImm(CC).addMBB(DestMBB);
1612 I.eraseFromParent();
1613 return true;
1614}
1615
1616bool AArch64InstructionSelector::selectCompareBranch(
1617 MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
1618 Register CondReg = I.getOperand(0).getReg();
1619 MachineInstr *CCMI = MRI.getVRegDef(CondReg);
1620 if (CCMI->getOpcode() == TargetOpcode::G_TRUNC) {
1621 CondReg = CCMI->getOperand(1).getReg();
1622 CCMI = MRI.getVRegDef(CondReg);
1623 }
1624
1625 // Try to select the G_BRCOND using whatever is feeding the condition if
1626 // possible.
1627 MachineIRBuilder MIB(I);
1628 unsigned CCMIOpc = CCMI->getOpcode();
1629 if (CCMIOpc == TargetOpcode::G_FCMP)
1630 return selectCompareBranchFedByFCmp(I, *CCMI, MIB);
1631 if (CCMIOpc == TargetOpcode::G_ICMP)
1632 return selectCompareBranchFedByICmp(I, *CCMI, MIB);
1633
1634 // Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z
1635 // instructions will not be produced, as they are conditional branch
1636 // instructions that do not set flags.
1637 if (ProduceNonFlagSettingCondBr) {
1638 emitTestBit(CondReg, /*Bit = */ 0, /*IsNegative = */ true,
1639 I.getOperand(1).getMBB(), MIB);
1640 I.eraseFromParent();
1641 return true;
1642 }
1643
1644 // Can't emit TB(N)Z/CB(N)Z. Emit a tst + bcc instead.
1645 auto TstMI =
1646 MIB.buildInstr(AArch64::ANDSWri, {LLT::scalar(32)}, {CondReg}).addImm(1);
1647 constrainSelectedInstRegOperands(*TstMI, TII, TRI, RBI);
1648 auto Bcc = MIB.buildInstr(AArch64::Bcc)
1649 .addImm(AArch64CC::EQ)
1650 .addMBB(I.getOperand(1).getMBB());
1651 I.eraseFromParent();
1652 return constrainSelectedInstRegOperands(*Bcc, TII, TRI, RBI);
1653}
1654
1655/// Returns the element immediate value of a vector shift operand if found.
1656/// This needs to detect a splat-like operation, e.g. a G_BUILD_VECTOR.
1657static Optional<int64_t> getVectorShiftImm(Register Reg,
1658 MachineRegisterInfo &MRI) {
1659 assert(MRI.getType(Reg).isVector() && "Expected a *vector* shift operand")((MRI.getType(Reg).isVector() && "Expected a *vector* shift operand"
) ? static_cast<void> (0) : __assert_fail ("MRI.getType(Reg).isVector() && \"Expected a *vector* shift operand\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1659, __PRETTY_FUNCTION__))
;
1660 MachineInstr *OpMI = MRI.getVRegDef(Reg);
1661 assert(OpMI && "Expected to find a vreg def for vector shift operand")((OpMI && "Expected to find a vreg def for vector shift operand"
) ? static_cast<void> (0) : __assert_fail ("OpMI && \"Expected to find a vreg def for vector shift operand\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1661, __PRETTY_FUNCTION__))
;
1662 if (OpMI->getOpcode() != TargetOpcode::G_BUILD_VECTOR)
1663 return None;
1664
1665 // Check all operands are identical immediates.
1666 int64_t ImmVal = 0;
1667 for (unsigned Idx = 1; Idx < OpMI->getNumOperands(); ++Idx) {
1668 auto VRegAndVal = getConstantVRegValWithLookThrough(OpMI->getOperand(Idx).getReg(), MRI);
1669 if (!VRegAndVal)
1670 return None;
1671
1672 if (Idx == 1)
1673 ImmVal = VRegAndVal->Value.getSExtValue();
1674 if (ImmVal != VRegAndVal->Value.getSExtValue())
1675 return None;
1676 }
1677
1678 return ImmVal;
1679}
1680
1681/// Matches and returns the shift immediate value for a SHL instruction given
1682/// a shift operand.
1683static Optional<int64_t> getVectorSHLImm(LLT SrcTy, Register Reg, MachineRegisterInfo &MRI) {
1684 Optional<int64_t> ShiftImm = getVectorShiftImm(Reg, MRI);
1685 if (!ShiftImm)
1686 return None;
1687 // Check the immediate is in range for a SHL.
1688 int64_t Imm = *ShiftImm;
1689 if (Imm < 0)
1690 return None;
1691 switch (SrcTy.getElementType().getSizeInBits()) {
1692 default:
1693 LLVM_DEBUG(dbgs() << "Unhandled element type for vector shift")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled element type for vector shift"
; } } while (false)
;
1694 return None;
1695 case 8:
1696 if (Imm > 7)
1697 return None;
1698 break;
1699 case 16:
1700 if (Imm > 15)
1701 return None;
1702 break;
1703 case 32:
1704 if (Imm > 31)
1705 return None;
1706 break;
1707 case 64:
1708 if (Imm > 63)
1709 return None;
1710 break;
1711 }
1712 return Imm;
1713}
1714
1715bool AArch64InstructionSelector::selectVectorSHL(
1716 MachineInstr &I, MachineRegisterInfo &MRI) const {
1717 assert(I.getOpcode() == TargetOpcode::G_SHL)((I.getOpcode() == TargetOpcode::G_SHL) ? static_cast<void
> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_SHL"
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1717, __PRETTY_FUNCTION__))
;
1718 Register DstReg = I.getOperand(0).getReg();
1719 const LLT Ty = MRI.getType(DstReg);
1720 Register Src1Reg = I.getOperand(1).getReg();
1721 Register Src2Reg = I.getOperand(2).getReg();
1722
1723 if (!Ty.isVector())
1724 return false;
1725
1726 // Check if we have a vector of constants on RHS that we can select as the
1727 // immediate form.
1728 Optional<int64_t> ImmVal = getVectorSHLImm(Ty, Src2Reg, MRI);
1729
1730 unsigned Opc = 0;
1731 if (Ty == LLT::vector(2, 64)) {
1732 Opc = ImmVal ? AArch64::SHLv2i64_shift : AArch64::USHLv2i64;
1733 } else if (Ty == LLT::vector(4, 32)) {
1734 Opc = ImmVal ? AArch64::SHLv4i32_shift : AArch64::USHLv4i32;
1735 } else if (Ty == LLT::vector(2, 32)) {
1736 Opc = ImmVal ? AArch64::SHLv2i32_shift : AArch64::USHLv2i32;
1737 } else if (Ty == LLT::vector(4, 16)) {
1738 Opc = ImmVal ? AArch64::SHLv4i16_shift : AArch64::USHLv4i16;
1739 } else if (Ty == LLT::vector(8, 16)) {
1740 Opc = ImmVal ? AArch64::SHLv8i16_shift : AArch64::USHLv8i16;
1741 } else if (Ty == LLT::vector(16, 8)) {
1742 Opc = ImmVal ? AArch64::SHLv16i8_shift : AArch64::USHLv16i8;
1743 } else if (Ty == LLT::vector(8, 8)) {
1744 Opc = ImmVal ? AArch64::SHLv8i8_shift : AArch64::USHLv8i8;
1745 } else {
1746 LLVM_DEBUG(dbgs() << "Unhandled G_SHL type")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled G_SHL type"; }
} while (false)
;
1747 return false;
1748 }
1749
1750 MachineIRBuilder MIB(I);
1751 auto Shl = MIB.buildInstr(Opc, {DstReg}, {Src1Reg});
1752 if (ImmVal)
1753 Shl.addImm(*ImmVal);
1754 else
1755 Shl.addUse(Src2Reg);
1756 constrainSelectedInstRegOperands(*Shl, TII, TRI, RBI);
1757 I.eraseFromParent();
1758 return true;
1759}
1760
1761bool AArch64InstructionSelector::selectVectorAshrLshr(
1762 MachineInstr &I, MachineRegisterInfo &MRI) const {
1763 assert(I.getOpcode() == TargetOpcode::G_ASHR ||((I.getOpcode() == TargetOpcode::G_ASHR || I.getOpcode() == TargetOpcode
::G_LSHR) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_ASHR || I.getOpcode() == TargetOpcode::G_LSHR"
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1764, __PRETTY_FUNCTION__))
1764 I.getOpcode() == TargetOpcode::G_LSHR)((I.getOpcode() == TargetOpcode::G_ASHR || I.getOpcode() == TargetOpcode
::G_LSHR) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_ASHR || I.getOpcode() == TargetOpcode::G_LSHR"
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1764, __PRETTY_FUNCTION__))
;
1765 Register DstReg = I.getOperand(0).getReg();
1766 const LLT Ty = MRI.getType(DstReg);
1767 Register Src1Reg = I.getOperand(1).getReg();
1768 Register Src2Reg = I.getOperand(2).getReg();
1769
1770 if (!Ty.isVector())
1771 return false;
1772
1773 bool IsASHR = I.getOpcode() == TargetOpcode::G_ASHR;
1774
1775 // We expect the immediate case to be lowered in the PostLegalCombiner to
1776 // AArch64ISD::VASHR or AArch64ISD::VLSHR equivalents.
1777
1778 // There is not a shift right register instruction, but the shift left
1779 // register instruction takes a signed value, where negative numbers specify a
1780 // right shift.
1781
1782 unsigned Opc = 0;
1783 unsigned NegOpc = 0;
1784 const TargetRegisterClass *RC =
1785 getRegClassForTypeOnBank(Ty, RBI.getRegBank(AArch64::FPRRegBankID), RBI);
1786 if (Ty == LLT::vector(2, 64)) {
1787 Opc = IsASHR ? AArch64::SSHLv2i64 : AArch64::USHLv2i64;
1788 NegOpc = AArch64::NEGv2i64;
1789 } else if (Ty == LLT::vector(4, 32)) {
1790 Opc = IsASHR ? AArch64::SSHLv4i32 : AArch64::USHLv4i32;
1791 NegOpc = AArch64::NEGv4i32;
1792 } else if (Ty == LLT::vector(2, 32)) {
1793 Opc = IsASHR ? AArch64::SSHLv2i32 : AArch64::USHLv2i32;
1794 NegOpc = AArch64::NEGv2i32;
1795 } else if (Ty == LLT::vector(4, 16)) {
1796 Opc = IsASHR ? AArch64::SSHLv4i16 : AArch64::USHLv4i16;
1797 NegOpc = AArch64::NEGv4i16;
1798 } else if (Ty == LLT::vector(8, 16)) {
1799 Opc = IsASHR ? AArch64::SSHLv8i16 : AArch64::USHLv8i16;
1800 NegOpc = AArch64::NEGv8i16;
1801 } else if (Ty == LLT::vector(16, 8)) {
1802 Opc = IsASHR ? AArch64::SSHLv16i8 : AArch64::USHLv16i8;
1803 NegOpc = AArch64::NEGv8i16;
1804 } else if (Ty == LLT::vector(8, 8)) {
1805 Opc = IsASHR ? AArch64::SSHLv8i8 : AArch64::USHLv8i8;
1806 NegOpc = AArch64::NEGv8i8;
1807 } else {
1808 LLVM_DEBUG(dbgs() << "Unhandled G_ASHR type")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled G_ASHR type"; }
} while (false)
;
1809 return false;
1810 }
1811
1812 MachineIRBuilder MIB(I);
1813 auto Neg = MIB.buildInstr(NegOpc, {RC}, {Src2Reg});
1814 constrainSelectedInstRegOperands(*Neg, TII, TRI, RBI);
1815 auto SShl = MIB.buildInstr(Opc, {DstReg}, {Src1Reg, Neg});
1816 constrainSelectedInstRegOperands(*SShl, TII, TRI, RBI);
1817 I.eraseFromParent();
1818 return true;
1819}
1820
1821bool AArch64InstructionSelector::selectVaStartAAPCS(
1822 MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
1823 return false;
1824}
1825
1826bool AArch64InstructionSelector::selectVaStartDarwin(
1827 MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
1828 AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
1829 Register ListReg = I.getOperand(0).getReg();
1830
1831 Register ArgsAddrReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
1832
1833 auto MIB =
1834 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::ADDXri))
1835 .addDef(ArgsAddrReg)
1836 .addFrameIndex(FuncInfo->getVarArgsStackIndex())
1837 .addImm(0)
1838 .addImm(0);
1839
1840 constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
1841
1842 MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::STRXui))
1843 .addUse(ArgsAddrReg)
1844 .addUse(ListReg)
1845 .addImm(0)
1846 .addMemOperand(*I.memoperands_begin());
1847
1848 constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
1849 I.eraseFromParent();
1850 return true;
1851}
1852
1853void AArch64InstructionSelector::materializeLargeCMVal(
1854 MachineInstr &I, const Value *V, unsigned OpFlags) const {
1855 MachineBasicBlock &MBB = *I.getParent();
1856 MachineFunction &MF = *MBB.getParent();
1857 MachineRegisterInfo &MRI = MF.getRegInfo();
1858 MachineIRBuilder MIB(I);
1859
1860 auto MovZ = MIB.buildInstr(AArch64::MOVZXi, {&AArch64::GPR64RegClass}, {});
1861 MovZ->addOperand(MF, I.getOperand(1));
1862 MovZ->getOperand(1).setTargetFlags(OpFlags | AArch64II::MO_G0 |
1863 AArch64II::MO_NC);
1864 MovZ->addOperand(MF, MachineOperand::CreateImm(0));
1865 constrainSelectedInstRegOperands(*MovZ, TII, TRI, RBI);
1866
1867 auto BuildMovK = [&](Register SrcReg, unsigned char Flags, unsigned Offset,
1868 Register ForceDstReg) {
1869 Register DstReg = ForceDstReg
1870 ? ForceDstReg
1871 : MRI.createVirtualRegister(&AArch64::GPR64RegClass);
1872 auto MovI = MIB.buildInstr(AArch64::MOVKXi).addDef(DstReg).addUse(SrcReg);
1873 if (auto *GV = dyn_cast<GlobalValue>(V)) {
1874 MovI->addOperand(MF, MachineOperand::CreateGA(
1875 GV, MovZ->getOperand(1).getOffset(), Flags));
1876 } else {
1877 MovI->addOperand(
1878 MF, MachineOperand::CreateBA(cast<BlockAddress>(V),
1879 MovZ->getOperand(1).getOffset(), Flags));
1880 }
1881 MovI->addOperand(MF, MachineOperand::CreateImm(Offset));
1882 constrainSelectedInstRegOperands(*MovI, TII, TRI, RBI);
1883 return DstReg;
1884 };
1885 Register DstReg = BuildMovK(MovZ.getReg(0),
1886 AArch64II::MO_G1 | AArch64II::MO_NC, 16, 0);
1887 DstReg = BuildMovK(DstReg, AArch64II::MO_G2 | AArch64II::MO_NC, 32, 0);
1888 BuildMovK(DstReg, AArch64II::MO_G3, 48, I.getOperand(0).getReg());
1889}
1890
1891bool AArch64InstructionSelector::preISelLower(MachineInstr &I) {
1892 MachineBasicBlock &MBB = *I.getParent();
1893 MachineFunction &MF = *MBB.getParent();
1894 MachineRegisterInfo &MRI = MF.getRegInfo();
1895
1896 switch (I.getOpcode()) {
1897 case TargetOpcode::G_SHL:
1898 case TargetOpcode::G_ASHR:
1899 case TargetOpcode::G_LSHR: {
1900 // These shifts are legalized to have 64 bit shift amounts because we want
1901 // to take advantage of the existing imported selection patterns that assume
1902 // the immediates are s64s. However, if the shifted type is 32 bits and for
1903 // some reason we receive input GMIR that has an s64 shift amount that's not
1904 // a G_CONSTANT, insert a truncate so that we can still select the s32
1905 // register-register variant.
1906 Register SrcReg = I.getOperand(1).getReg();
1907 Register ShiftReg = I.getOperand(2).getReg();
1908 const LLT ShiftTy = MRI.getType(ShiftReg);
1909 const LLT SrcTy = MRI.getType(SrcReg);
1910 if (SrcTy.isVector())
1911 return false;
1912 assert(!ShiftTy.isVector() && "unexpected vector shift ty")((!ShiftTy.isVector() && "unexpected vector shift ty"
) ? static_cast<void> (0) : __assert_fail ("!ShiftTy.isVector() && \"unexpected vector shift ty\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1912, __PRETTY_FUNCTION__))
;
1913 if (SrcTy.getSizeInBits() != 32 || ShiftTy.getSizeInBits() != 64)
1914 return false;
1915 auto *AmtMI = MRI.getVRegDef(ShiftReg);
1916 assert(AmtMI && "could not find a vreg definition for shift amount")((AmtMI && "could not find a vreg definition for shift amount"
) ? static_cast<void> (0) : __assert_fail ("AmtMI && \"could not find a vreg definition for shift amount\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1916, __PRETTY_FUNCTION__))
;
1917 if (AmtMI->getOpcode() != TargetOpcode::G_CONSTANT) {
1918 // Insert a subregister copy to implement a 64->32 trunc
1919 MachineIRBuilder MIB(I);
1920 auto Trunc = MIB.buildInstr(TargetOpcode::COPY, {SrcTy}, {})
1921 .addReg(ShiftReg, 0, AArch64::sub_32);
1922 MRI.setRegBank(Trunc.getReg(0), RBI.getRegBank(AArch64::GPRRegBankID));
1923 I.getOperand(2).setReg(Trunc.getReg(0));
1924 }
1925 return true;
1926 }
1927 case TargetOpcode::G_STORE: {
1928 bool Changed = contractCrossBankCopyIntoStore(I, MRI);
1929 MachineOperand &SrcOp = I.getOperand(0);
1930 if (MRI.getType(SrcOp.getReg()).isPointer()) {
1931 // Allow matching with imported patterns for stores of pointers. Unlike
1932 // G_LOAD/G_PTR_ADD, we may not have selected all users. So, emit a copy
1933 // and constrain.
1934 MachineIRBuilder MIB(I);
1935 auto Copy = MIB.buildCopy(LLT::scalar(64), SrcOp);
1936 Register NewSrc = Copy.getReg(0);
1937 SrcOp.setReg(NewSrc);
1938 RBI.constrainGenericRegister(NewSrc, AArch64::GPR64RegClass, MRI);
1939 Changed = true;
1940 }
1941 return Changed;
1942 }
1943 case TargetOpcode::G_PTR_ADD:
1944 return convertPtrAddToAdd(I, MRI);
1945 case TargetOpcode::G_LOAD: {
1946 // For scalar loads of pointers, we try to convert the dest type from p0
1947 // to s64 so that our imported patterns can match. Like with the G_PTR_ADD
1948 // conversion, this should be ok because all users should have been
1949 // selected already, so the type doesn't matter for them.
1950 Register DstReg = I.getOperand(0).getReg();
1951 const LLT DstTy = MRI.getType(DstReg);
1952 if (!DstTy.isPointer())
1953 return false;
1954 MRI.setType(DstReg, LLT::scalar(64));
1955 return true;
1956 }
1957 case AArch64::G_DUP: {
1958 // Convert the type from p0 to s64 to help selection.
1959 LLT DstTy = MRI.getType(I.getOperand(0).getReg());
1960 if (!DstTy.getElementType().isPointer())
1961 return false;
1962 MachineIRBuilder MIB(I);
1963 auto NewSrc = MIB.buildCopy(LLT::scalar(64), I.getOperand(1).getReg());
1964 MRI.setType(I.getOperand(0).getReg(),
1965 DstTy.changeElementType(LLT::scalar(64)));
1966 MRI.setRegBank(NewSrc.getReg(0), RBI.getRegBank(AArch64::GPRRegBankID));
1967 I.getOperand(1).setReg(NewSrc.getReg(0));
1968 return true;
1969 }
1970 case TargetOpcode::G_UITOFP:
1971 case TargetOpcode::G_SITOFP: {
1972 // If both source and destination regbanks are FPR, then convert the opcode
1973 // to G_SITOF so that the importer can select it to an fpr variant.
1974 // Otherwise, it ends up matching an fpr/gpr variant and adding a cross-bank
1975 // copy.
1976 Register SrcReg = I.getOperand(1).getReg();
1977 LLT SrcTy = MRI.getType(SrcReg);
1978 LLT DstTy = MRI.getType(I.getOperand(0).getReg());
1979 if (SrcTy.isVector() || SrcTy.getSizeInBits() != DstTy.getSizeInBits())
1980 return false;
1981
1982 if (RBI.getRegBank(SrcReg, MRI, TRI)->getID() == AArch64::FPRRegBankID) {
1983 if (I.getOpcode() == TargetOpcode::G_SITOFP)
1984 I.setDesc(TII.get(AArch64::G_SITOF));
1985 else
1986 I.setDesc(TII.get(AArch64::G_UITOF));
1987 return true;
1988 }
1989 return false;
1990 }
1991 default:
1992 return false;
1993 }
1994}
1995
1996/// This lowering tries to look for G_PTR_ADD instructions and then converts
1997/// them to a standard G_ADD with a COPY on the source.
1998///
1999/// The motivation behind this is to expose the add semantics to the imported
2000/// tablegen patterns. We shouldn't need to check for uses being loads/stores,
2001/// because the selector works bottom up, uses before defs. By the time we
2002/// end up trying to select a G_PTR_ADD, we should have already attempted to
2003/// fold this into addressing modes and were therefore unsuccessful.
2004bool AArch64InstructionSelector::convertPtrAddToAdd(
2005 MachineInstr &I, MachineRegisterInfo &MRI) {
2006 assert(I.getOpcode() == TargetOpcode::G_PTR_ADD && "Expected G_PTR_ADD")((I.getOpcode() == TargetOpcode::G_PTR_ADD && "Expected G_PTR_ADD"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_PTR_ADD && \"Expected G_PTR_ADD\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2006, __PRETTY_FUNCTION__))
;
2007 Register DstReg = I.getOperand(0).getReg();
2008 Register AddOp1Reg = I.getOperand(1).getReg();
2009 const LLT PtrTy = MRI.getType(DstReg);
2010 if (PtrTy.getAddressSpace() != 0)
2011 return false;
2012
2013 MachineIRBuilder MIB(I);
2014 const LLT CastPtrTy = PtrTy.isVector() ? LLT::vector(2, 64) : LLT::scalar(64);
2015 auto PtrToInt = MIB.buildPtrToInt(CastPtrTy, AddOp1Reg);
2016 // Set regbanks on the registers.
2017 if (PtrTy.isVector())
2018 MRI.setRegBank(PtrToInt.getReg(0), RBI.getRegBank(AArch64::FPRRegBankID));
2019 else
2020 MRI.setRegBank(PtrToInt.getReg(0), RBI.getRegBank(AArch64::GPRRegBankID));
2021
2022 // Now turn the %dst(p0) = G_PTR_ADD %base, off into:
2023 // %dst(intty) = G_ADD %intbase, off
2024 I.setDesc(TII.get(TargetOpcode::G_ADD));
2025 MRI.setType(DstReg, CastPtrTy);
2026 I.getOperand(1).setReg(PtrToInt.getReg(0));
2027 if (!select(*PtrToInt)) {
2028 LLVM_DEBUG(dbgs() << "Failed to select G_PTRTOINT in convertPtrAddToAdd")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to select G_PTRTOINT in convertPtrAddToAdd"
; } } while (false)
;
2029 return false;
2030 }
2031
2032 // Also take the opportunity here to try to do some optimization.
2033 // Try to convert this into a G_SUB if the offset is a 0-x negate idiom.
2034 Register NegatedReg;
2035 if (!mi_match(I.getOperand(2).getReg(), MRI, m_Neg(m_Reg(NegatedReg))))
2036 return true;
2037 I.getOperand(2).setReg(NegatedReg);
2038 I.setDesc(TII.get(TargetOpcode::G_SUB));
2039 return true;
2040}
2041
2042bool AArch64InstructionSelector::earlySelectSHL(
2043 MachineInstr &I, MachineRegisterInfo &MRI) const {
2044 // We try to match the immediate variant of LSL, which is actually an alias
2045 // for a special case of UBFM. Otherwise, we fall back to the imported
2046 // selector which will match the register variant.
2047 assert(I.getOpcode() == TargetOpcode::G_SHL && "unexpected op")((I.getOpcode() == TargetOpcode::G_SHL && "unexpected op"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_SHL && \"unexpected op\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2047, __PRETTY_FUNCTION__))
;
2048 const auto &MO = I.getOperand(2);
2049 auto VRegAndVal = getConstantVRegVal(MO.getReg(), MRI);
2050 if (!VRegAndVal)
2051 return false;
2052
2053 const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
2054 if (DstTy.isVector())
2055 return false;
2056 bool Is64Bit = DstTy.getSizeInBits() == 64;
2057 auto Imm1Fn = Is64Bit ? selectShiftA_64(MO) : selectShiftA_32(MO);
2058 auto Imm2Fn = Is64Bit ? selectShiftB_64(MO) : selectShiftB_32(MO);
2059 MachineIRBuilder MIB(I);
2060
2061 if (!Imm1Fn || !Imm2Fn)
2062 return false;
2063
2064 auto NewI =
2065 MIB.buildInstr(Is64Bit ? AArch64::UBFMXri : AArch64::UBFMWri,
2066 {I.getOperand(0).getReg()}, {I.getOperand(1).getReg()});
2067
2068 for (auto &RenderFn : *Imm1Fn)
2069 RenderFn(NewI);
2070 for (auto &RenderFn : *Imm2Fn)
2071 RenderFn(NewI);
2072
2073 I.eraseFromParent();
2074 return constrainSelectedInstRegOperands(*NewI, TII, TRI, RBI);
2075}
2076
2077bool AArch64InstructionSelector::contractCrossBankCopyIntoStore(
2078 MachineInstr &I, MachineRegisterInfo &MRI) {
2079 assert(I.getOpcode() == TargetOpcode::G_STORE && "Expected G_STORE")((I.getOpcode() == TargetOpcode::G_STORE && "Expected G_STORE"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_STORE && \"Expected G_STORE\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2079, __PRETTY_FUNCTION__))
;
2080 // If we're storing a scalar, it doesn't matter what register bank that
2081 // scalar is on. All that matters is the size.
2082 //
2083 // So, if we see something like this (with a 32-bit scalar as an example):
2084 //
2085 // %x:gpr(s32) = ... something ...
2086 // %y:fpr(s32) = COPY %x:gpr(s32)
2087 // G_STORE %y:fpr(s32)
2088 //
2089 // We can fix this up into something like this:
2090 //
2091 // G_STORE %x:gpr(s32)
2092 //
2093 // And then continue the selection process normally.
2094 Register DefDstReg = getSrcRegIgnoringCopies(I.getOperand(0).getReg(), MRI);
2095 if (!DefDstReg.isValid())
2096 return false;
2097 LLT DefDstTy = MRI.getType(DefDstReg);
2098 Register StoreSrcReg = I.getOperand(0).getReg();
2099 LLT StoreSrcTy = MRI.getType(StoreSrcReg);
2100
2101 // If we get something strange like a physical register, then we shouldn't
2102 // go any further.
2103 if (!DefDstTy.isValid())
2104 return false;
2105
2106 // Are the source and dst types the same size?
2107 if (DefDstTy.getSizeInBits() != StoreSrcTy.getSizeInBits())
2108 return false;
2109
2110 if (RBI.getRegBank(StoreSrcReg, MRI, TRI) ==
2111 RBI.getRegBank(DefDstReg, MRI, TRI))
2112 return false;
2113
2114 // We have a cross-bank copy, which is entering a store. Let's fold it.
2115 I.getOperand(0).setReg(DefDstReg);
2116 return true;
2117}
2118
2119bool AArch64InstructionSelector::earlySelect(MachineInstr &I) const {
2120 assert(I.getParent() && "Instruction should be in a basic block!")((I.getParent() && "Instruction should be in a basic block!"
) ? static_cast<void> (0) : __assert_fail ("I.getParent() && \"Instruction should be in a basic block!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2120, __PRETTY_FUNCTION__))
;
2121 assert(I.getParent()->getParent() && "Instruction should be in a function!")((I.getParent()->getParent() && "Instruction should be in a function!"
) ? static_cast<void> (0) : __assert_fail ("I.getParent()->getParent() && \"Instruction should be in a function!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2121, __PRETTY_FUNCTION__))
;
2122
2123 MachineBasicBlock &MBB = *I.getParent();
2124 MachineFunction &MF = *MBB.getParent();
2125 MachineRegisterInfo &MRI = MF.getRegInfo();
2126
2127 switch (I.getOpcode()) {
2128 case TargetOpcode::G_BR: {
2129 // If the branch jumps to the fallthrough block, don't bother emitting it.
2130 // Only do this for -O0 for a good code size improvement, because when
2131 // optimizations are enabled we want to leave this choice to
2132 // MachineBlockPlacement.
2133 bool EnableOpt = MF.getTarget().getOptLevel() != CodeGenOpt::None;
2134 if (EnableOpt || !MBB.isLayoutSuccessor(I.getOperand(0).getMBB()))
2135 return false;
2136 I.eraseFromParent();
2137 return true;
2138 }
2139 case TargetOpcode::G_SHL:
2140 return earlySelectSHL(I, MRI);
2141 case TargetOpcode::G_CONSTANT: {
2142 bool IsZero = false;
2143 if (I.getOperand(1).isCImm())
2144 IsZero = I.getOperand(1).getCImm()->getZExtValue() == 0;
2145 else if (I.getOperand(1).isImm())
2146 IsZero = I.getOperand(1).getImm() == 0;
2147
2148 if (!IsZero)
2149 return false;
2150
2151 Register DefReg = I.getOperand(0).getReg();
2152 LLT Ty = MRI.getType(DefReg);
2153 if (Ty.getSizeInBits() == 64) {
2154 I.getOperand(1).ChangeToRegister(AArch64::XZR, false);
2155 RBI.constrainGenericRegister(DefReg, AArch64::GPR64RegClass, MRI);
2156 } else if (Ty.getSizeInBits() == 32) {
2157 I.getOperand(1).ChangeToRegister(AArch64::WZR, false);
2158 RBI.constrainGenericRegister(DefReg, AArch64::GPR32RegClass, MRI);
2159 } else
2160 return false;
2161
2162 I.setDesc(TII.get(TargetOpcode::COPY));
2163 return true;
2164 }
2165
2166 case TargetOpcode::G_ADD: {
2167 // Check if this is being fed by a G_ICMP on either side.
2168 //
2169 // (cmp pred, x, y) + z
2170 //
2171 // In the above case, when the cmp is true, we increment z by 1. So, we can
2172 // fold the add into the cset for the cmp by using cinc.
2173 //
2174 // FIXME: This would probably be a lot nicer in PostLegalizerLowering.
2175 Register X = I.getOperand(1).getReg();
2176
2177 // Only handle scalars. Scalar G_ICMP is only legal for s32, so bail out
2178 // early if we see it.
2179 LLT Ty = MRI.getType(X);
2180 if (Ty.isVector() || Ty.getSizeInBits() != 32)
2181 return false;
2182
2183 Register CmpReg = I.getOperand(2).getReg();
2184 MachineInstr *Cmp = getOpcodeDef(TargetOpcode::G_ICMP, CmpReg, MRI);
2185 if (!Cmp) {
2186 std::swap(X, CmpReg);
2187 Cmp = getOpcodeDef(TargetOpcode::G_ICMP, CmpReg, MRI);
2188 if (!Cmp)
2189 return false;
2190 }
2191 MachineIRBuilder MIRBuilder(I);
2192 auto Pred =
2193 static_cast<CmpInst::Predicate>(Cmp->getOperand(1).getPredicate());
2194 emitIntegerCompare(Cmp->getOperand(2), Cmp->getOperand(3),
2195 Cmp->getOperand(1), MIRBuilder);
2196 emitCSetForICMP(I.getOperand(0).getReg(), Pred, MIRBuilder, X);
2197 I.eraseFromParent();
2198 return true;
2199 }
2200 default:
2201 return false;
2202 }
2203}
2204
2205bool AArch64InstructionSelector::select(MachineInstr &I) {
2206 assert(I.getParent() && "Instruction should be in a basic block!")((I.getParent() && "Instruction should be in a basic block!"
) ? static_cast<void> (0) : __assert_fail ("I.getParent() && \"Instruction should be in a basic block!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2206, __PRETTY_FUNCTION__))
;
2207 assert(I.getParent()->getParent() && "Instruction should be in a function!")((I.getParent()->getParent() && "Instruction should be in a function!"
) ? static_cast<void> (0) : __assert_fail ("I.getParent()->getParent() && \"Instruction should be in a function!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2207, __PRETTY_FUNCTION__))
;
2208
2209 MachineBasicBlock &MBB = *I.getParent();
2210 MachineFunction &MF = *MBB.getParent();
2211 MachineRegisterInfo &MRI = MF.getRegInfo();
2212
2213 const AArch64Subtarget *Subtarget =
2214 &static_cast<const AArch64Subtarget &>(MF.getSubtarget());
2215 if (Subtarget->requiresStrictAlign()) {
2216 // We don't support this feature yet.
2217 LLVM_DEBUG(dbgs() << "AArch64 GISel does not support strict-align yet\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "AArch64 GISel does not support strict-align yet\n"
; } } while (false)
;
2218 return false;
2219 }
2220
2221 unsigned Opcode = I.getOpcode();
2222 // G_PHI requires same handling as PHI
2223 if (!I.isPreISelOpcode() || Opcode == TargetOpcode::G_PHI) {
2224 // Certain non-generic instructions also need some special handling.
2225
2226 if (Opcode == TargetOpcode::LOAD_STACK_GUARD)
2227 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2228
2229 if (Opcode == TargetOpcode::PHI || Opcode == TargetOpcode::G_PHI) {
2230 const Register DefReg = I.getOperand(0).getReg();
2231 const LLT DefTy = MRI.getType(DefReg);
2232
2233 const RegClassOrRegBank &RegClassOrBank =
2234 MRI.getRegClassOrRegBank(DefReg);
2235
2236 const TargetRegisterClass *DefRC
2237 = RegClassOrBank.dyn_cast<const TargetRegisterClass *>();
2238 if (!DefRC) {
2239 if (!DefTy.isValid()) {
2240 LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "PHI operand has no type, not a gvreg?\n"
; } } while (false)
;
2241 return false;
2242 }
2243 const RegisterBank &RB = *RegClassOrBank.get<const RegisterBank *>();
2244 DefRC = getRegClassForTypeOnBank(DefTy, RB, RBI);
2245 if (!DefRC) {
2246 LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "PHI operand has unexpected size/bank\n"
; } } while (false)
;
2247 return false;
2248 }
2249 }
2250
2251 I.setDesc(TII.get(TargetOpcode::PHI));
2252
2253 return RBI.constrainGenericRegister(DefReg, *DefRC, MRI);
2254 }
2255
2256 if (I.isCopy())
2257 return selectCopy(I, TII, MRI, TRI, RBI);
2258
2259 return true;
2260 }
2261
2262
2263 if (I.getNumOperands() != I.getNumExplicitOperands()) {
2264 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic instruction has unexpected implicit operands\n"
; } } while (false)
2265 dbgs() << "Generic instruction has unexpected implicit operands\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic instruction has unexpected implicit operands\n"
; } } while (false)
;
2266 return false;
2267 }
2268
2269 // Try to do some lowering before we start instruction selecting. These
2270 // lowerings are purely transformations on the input G_MIR and so selection
2271 // must continue after any modification of the instruction.
2272 if (preISelLower(I)) {
2273 Opcode = I.getOpcode(); // The opcode may have been modified, refresh it.
2274 }
2275
2276 // There may be patterns where the importer can't deal with them optimally,
2277 // but does select it to a suboptimal sequence so our custom C++ selection
2278 // code later never has a chance to work on it. Therefore, we have an early
2279 // selection attempt here to give priority to certain selection routines
2280 // over the imported ones.
2281 if (earlySelect(I))
2282 return true;
2283
2284 if (selectImpl(I, *CoverageInfo))
2285 return true;
2286
2287 LLT Ty =
2288 I.getOperand(0).isReg() ? MRI.getType(I.getOperand(0).getReg()) : LLT{};
2289
2290 MachineIRBuilder MIB(I);
2291
2292 switch (Opcode) {
2293 case TargetOpcode::G_BRCOND:
2294 return selectCompareBranch(I, MF, MRI);
2295
2296 case TargetOpcode::G_BRINDIRECT: {
2297 I.setDesc(TII.get(AArch64::BR));
2298 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2299 }
2300
2301 case TargetOpcode::G_BRJT:
2302 return selectBrJT(I, MRI);
2303
2304 case AArch64::G_ADD_LOW: {
2305 // This op may have been separated from it's ADRP companion by the localizer
2306 // or some other code motion pass. Given that many CPUs will try to
2307 // macro fuse these operations anyway, select this into a MOVaddr pseudo
2308 // which will later be expanded into an ADRP+ADD pair after scheduling.
2309 MachineInstr *BaseMI = MRI.getVRegDef(I.getOperand(1).getReg());
2310 if (BaseMI->getOpcode() != AArch64::ADRP) {
2311 I.setDesc(TII.get(AArch64::ADDXri));
2312 I.addOperand(MachineOperand::CreateImm(0));
2313 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2314 }
2315 assert(TM.getCodeModel() == CodeModel::Small &&((TM.getCodeModel() == CodeModel::Small && "Expected small code model"
) ? static_cast<void> (0) : __assert_fail ("TM.getCodeModel() == CodeModel::Small && \"Expected small code model\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2316, __PRETTY_FUNCTION__))
2316 "Expected small code model")((TM.getCodeModel() == CodeModel::Small && "Expected small code model"
) ? static_cast<void> (0) : __assert_fail ("TM.getCodeModel() == CodeModel::Small && \"Expected small code model\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2316, __PRETTY_FUNCTION__))
;
2317 MachineIRBuilder MIB(I);
2318 auto Op1 = BaseMI->getOperand(1);
2319 auto Op2 = I.getOperand(2);
2320 auto MovAddr = MIB.buildInstr(AArch64::MOVaddr, {I.getOperand(0)}, {})
2321 .addGlobalAddress(Op1.getGlobal(), Op1.getOffset(),
2322 Op1.getTargetFlags())
2323 .addGlobalAddress(Op2.getGlobal(), Op2.getOffset(),
2324 Op2.getTargetFlags());
2325 I.eraseFromParent();
2326 return constrainSelectedInstRegOperands(*MovAddr, TII, TRI, RBI);
2327 }
2328
2329 case TargetOpcode::G_BSWAP: {
2330 // Handle vector types for G_BSWAP directly.
2331 Register DstReg = I.getOperand(0).getReg();
2332 LLT DstTy = MRI.getType(DstReg);
2333
2334 // We should only get vector types here; everything else is handled by the
2335 // importer right now.
2336 if (!DstTy.isVector() || DstTy.getSizeInBits() > 128) {
2337 LLVM_DEBUG(dbgs() << "Dst type for G_BSWAP currently unsupported.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Dst type for G_BSWAP currently unsupported.\n"
; } } while (false)
;
2338 return false;
2339 }
2340
2341 // Only handle 4 and 2 element vectors for now.
2342 // TODO: 16-bit elements.
2343 unsigned NumElts = DstTy.getNumElements();
2344 if (NumElts != 4 && NumElts != 2) {
2345 LLVM_DEBUG(dbgs() << "Unsupported number of elements for G_BSWAP.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported number of elements for G_BSWAP.\n"
; } } while (false)
;
2346 return false;
2347 }
2348
2349 // Choose the correct opcode for the supported types. Right now, that's
2350 // v2s32, v4s32, and v2s64.
2351 unsigned Opc = 0;
2352 unsigned EltSize = DstTy.getElementType().getSizeInBits();
2353 if (EltSize == 32)
2354 Opc = (DstTy.getNumElements() == 2) ? AArch64::REV32v8i8
2355 : AArch64::REV32v16i8;
2356 else if (EltSize == 64)
2357 Opc = AArch64::REV64v16i8;
2358
2359 // We should always get something by the time we get here...
2360 assert(Opc != 0 && "Didn't get an opcode for G_BSWAP?")((Opc != 0 && "Didn't get an opcode for G_BSWAP?") ? static_cast
<void> (0) : __assert_fail ("Opc != 0 && \"Didn't get an opcode for G_BSWAP?\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2360, __PRETTY_FUNCTION__))
;
2361
2362 I.setDesc(TII.get(Opc));
2363 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2364 }
2365
2366 case TargetOpcode::G_FCONSTANT:
2367 case TargetOpcode::G_CONSTANT: {
2368 const bool isFP = Opcode == TargetOpcode::G_FCONSTANT;
2369
2370 const LLT s8 = LLT::scalar(8);
2371 const LLT s16 = LLT::scalar(16);
2372 const LLT s32 = LLT::scalar(32);
2373 const LLT s64 = LLT::scalar(64);
2374 const LLT s128 = LLT::scalar(128);
2375 const LLT p0 = LLT::pointer(0, 64);
2376
2377 const Register DefReg = I.getOperand(0).getReg();
2378 const LLT DefTy = MRI.getType(DefReg);
2379 const unsigned DefSize = DefTy.getSizeInBits();
2380 const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
2381
2382 // FIXME: Redundant check, but even less readable when factored out.
2383 if (isFP) {
2384 if (Ty != s32 && Ty != s64 && Ty != s128) {
2385 LLVM_DEBUG(dbgs() << "Unable to materialize FP " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize FP "
<< Ty << " constant, expected: " << s32 <<
" or " << s64 << " or " << s128 << '\n'
; } } while (false)
2386 << " constant, expected: " << s32 << " or " << s64do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize FP "
<< Ty << " constant, expected: " << s32 <<
" or " << s64 << " or " << s128 << '\n'
; } } while (false)
2387 << " or " << s128 << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize FP "
<< Ty << " constant, expected: " << s32 <<
" or " << s64 << " or " << s128 << '\n'
; } } while (false)
;
2388 return false;
2389 }
2390
2391 if (RB.getID() != AArch64::FPRRegBankID) {
2392 LLVM_DEBUG(dbgs() << "Unable to materialize FP " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize FP "
<< Ty << " constant on bank: " << RB <<
", expected: FPR\n"; } } while (false)
2393 << " constant on bank: " << RBdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize FP "
<< Ty << " constant on bank: " << RB <<
", expected: FPR\n"; } } while (false)
2394 << ", expected: FPR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize FP "
<< Ty << " constant on bank: " << RB <<
", expected: FPR\n"; } } while (false)
;
2395 return false;
2396 }
2397
2398 // The case when we have 0.0 is covered by tablegen. Reject it here so we
2399 // can be sure tablegen works correctly and isn't rescued by this code.
2400 // 0.0 is not covered by tablegen for FP128. So we will handle this
2401 // scenario in the code here.
2402 if (DefSize != 128 && I.getOperand(1).getFPImm()->isExactlyValue(0.0))
2403 return false;
2404 } else {
2405 // s32 and s64 are covered by tablegen.
2406 if (Ty != p0 && Ty != s8 && Ty != s16) {
2407 LLVM_DEBUG(dbgs() << "Unable to materialize integer " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize integer "
<< Ty << " constant, expected: " << s32 <<
", " << s64 << ", or " << p0 << '\n'
; } } while (false)
2408 << " constant, expected: " << s32 << ", " << s64do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize integer "
<< Ty << " constant, expected: " << s32 <<
", " << s64 << ", or " << p0 << '\n'
; } } while (false)
2409 << ", or " << p0 << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize integer "
<< Ty << " constant, expected: " << s32 <<
", " << s64 << ", or " << p0 << '\n'
; } } while (false)
;
2410 return false;
2411 }
2412
2413 if (RB.getID() != AArch64::GPRRegBankID) {
2414 LLVM_DEBUG(dbgs() << "Unable to materialize integer " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize integer "
<< Ty << " constant on bank: " << RB <<
", expected: GPR\n"; } } while (false)
2415 << " constant on bank: " << RBdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize integer "
<< Ty << " constant on bank: " << RB <<
", expected: GPR\n"; } } while (false)
2416 << ", expected: GPR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize integer "
<< Ty << " constant on bank: " << RB <<
", expected: GPR\n"; } } while (false)
;
2417 return false;
2418 }
2419 }
2420
2421 // We allow G_CONSTANT of types < 32b.
2422 const unsigned MovOpc =
2423 DefSize == 64 ? AArch64::MOVi64imm : AArch64::MOVi32imm;
2424
2425 if (isFP) {
2426 // Either emit a FMOV, or emit a copy to emit a normal mov.
2427 const TargetRegisterClass &GPRRC =
2428 DefSize == 32 ? AArch64::GPR32RegClass : AArch64::GPR64RegClass;
2429 const TargetRegisterClass &FPRRC =
2430 DefSize == 32 ? AArch64::FPR32RegClass
2431 : (DefSize == 64 ? AArch64::FPR64RegClass
2432 : AArch64::FPR128RegClass);
2433
2434 // For 64b values, emit a constant pool load instead.
2435 // For s32, use a cp load if we have optsize/minsize.
2436 if (DefSize == 64 || DefSize == 128 ||
2437 (DefSize == 32 && shouldOptForSize(&MF))) {
2438 auto *FPImm = I.getOperand(1).getFPImm();
2439 MachineIRBuilder MIB(I);
2440 auto *LoadMI = emitLoadFromConstantPool(FPImm, MIB);
2441 if (!LoadMI) {
2442 LLVM_DEBUG(dbgs() << "Failed to load double constant pool entry\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to load double constant pool entry\n"
; } } while (false)
;
2443 return false;
2444 }
2445 MIB.buildCopy({DefReg}, {LoadMI->getOperand(0).getReg()});
2446 I.eraseFromParent();
2447 return RBI.constrainGenericRegister(DefReg, FPRRC, MRI);
2448 }
2449
2450 // Nope. Emit a copy and use a normal mov instead.
2451 const Register DefGPRReg = MRI.createVirtualRegister(&GPRRC);
2452 MachineOperand &RegOp = I.getOperand(0);
2453 RegOp.setReg(DefGPRReg);
2454 MIB.setInsertPt(MIB.getMBB(), std::next(I.getIterator()));
2455 MIB.buildCopy({DefReg}, {DefGPRReg});
2456
2457 if (!RBI.constrainGenericRegister(DefReg, FPRRC, MRI)) {
2458 LLVM_DEBUG(dbgs() << "Failed to constrain G_FCONSTANT def operand\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain G_FCONSTANT def operand\n"
; } } while (false)
;
2459 return false;
2460 }
2461
2462 MachineOperand &ImmOp = I.getOperand(1);
2463 // FIXME: Is going through int64_t always correct?
2464 ImmOp.ChangeToImmediate(
2465 ImmOp.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue());
2466 } else if (I.getOperand(1).isCImm()) {
2467 uint64_t Val = I.getOperand(1).getCImm()->getZExtValue();
2468 I.getOperand(1).ChangeToImmediate(Val);
2469 } else if (I.getOperand(1).isImm()) {
2470 uint64_t Val = I.getOperand(1).getImm();
2471 I.getOperand(1).ChangeToImmediate(Val);
2472 }
2473
2474 I.setDesc(TII.get(MovOpc));
2475 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2476 return true;
2477 }
2478 case TargetOpcode::G_EXTRACT: {
2479 Register DstReg = I.getOperand(0).getReg();
2480 Register SrcReg = I.getOperand(1).getReg();
2481 LLT SrcTy = MRI.getType(SrcReg);
2482 LLT DstTy = MRI.getType(DstReg);
2483 (void)DstTy;
2484 unsigned SrcSize = SrcTy.getSizeInBits();
2485
2486 if (SrcTy.getSizeInBits() > 64) {
2487 // This should be an extract of an s128, which is like a vector extract.
2488 if (SrcTy.getSizeInBits() != 128)
2489 return false;
2490 // Only support extracting 64 bits from an s128 at the moment.
2491 if (DstTy.getSizeInBits() != 64)
2492 return false;
2493
2494 const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
2495 const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
2496 // Check we have the right regbank always.
2497 assert(SrcRB.getID() == AArch64::FPRRegBankID &&((SrcRB.getID() == AArch64::FPRRegBankID && DstRB.getID
() == AArch64::FPRRegBankID && "Wrong extract regbank!"
) ? static_cast<void> (0) : __assert_fail ("SrcRB.getID() == AArch64::FPRRegBankID && DstRB.getID() == AArch64::FPRRegBankID && \"Wrong extract regbank!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2499, __PRETTY_FUNCTION__))
2498 DstRB.getID() == AArch64::FPRRegBankID &&((SrcRB.getID() == AArch64::FPRRegBankID && DstRB.getID
() == AArch64::FPRRegBankID && "Wrong extract regbank!"
) ? static_cast<void> (0) : __assert_fail ("SrcRB.getID() == AArch64::FPRRegBankID && DstRB.getID() == AArch64::FPRRegBankID && \"Wrong extract regbank!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2499, __PRETTY_FUNCTION__))
2499 "Wrong extract regbank!")((SrcRB.getID() == AArch64::FPRRegBankID && DstRB.getID
() == AArch64::FPRRegBankID && "Wrong extract regbank!"
) ? static_cast<void> (0) : __assert_fail ("SrcRB.getID() == AArch64::FPRRegBankID && DstRB.getID() == AArch64::FPRRegBankID && \"Wrong extract regbank!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2499, __PRETTY_FUNCTION__))
;
2500 (void)SrcRB;
2501
2502 // Emit the same code as a vector extract.
2503 // Offset must be a multiple of 64.
2504 unsigned Offset = I.getOperand(2).getImm();
2505 if (Offset % 64 != 0)
2506 return false;
2507 unsigned LaneIdx = Offset / 64;
2508 MachineIRBuilder MIB(I);
2509 MachineInstr *Extract = emitExtractVectorElt(
2510 DstReg, DstRB, LLT::scalar(64), SrcReg, LaneIdx, MIB);
2511 if (!Extract)
2512 return false;
2513 I.eraseFromParent();
2514 return true;
2515 }
2516
2517 I.setDesc(TII.get(SrcSize == 64 ? AArch64::UBFMXri : AArch64::UBFMWri));
2518 MachineInstrBuilder(MF, I).addImm(I.getOperand(2).getImm() +
2519 Ty.getSizeInBits() - 1);
2520
2521 if (SrcSize < 64) {
2522 assert(SrcSize == 32 && DstTy.getSizeInBits() == 16 &&((SrcSize == 32 && DstTy.getSizeInBits() == 16 &&
"unexpected G_EXTRACT types") ? static_cast<void> (0) :
__assert_fail ("SrcSize == 32 && DstTy.getSizeInBits() == 16 && \"unexpected G_EXTRACT types\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2523, __PRETTY_FUNCTION__))
2523 "unexpected G_EXTRACT types")((SrcSize == 32 && DstTy.getSizeInBits() == 16 &&
"unexpected G_EXTRACT types") ? static_cast<void> (0) :
__assert_fail ("SrcSize == 32 && DstTy.getSizeInBits() == 16 && \"unexpected G_EXTRACT types\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2523, __PRETTY_FUNCTION__))
;
2524 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2525 }
2526
2527 DstReg = MRI.createGenericVirtualRegister(LLT::scalar(64));
2528 MIB.setInsertPt(MIB.getMBB(), std::next(I.getIterator()));
2529 MIB.buildInstr(TargetOpcode::COPY, {I.getOperand(0).getReg()}, {})
2530 .addReg(DstReg, 0, AArch64::sub_32);
2531 RBI.constrainGenericRegister(I.getOperand(0).getReg(),
2532 AArch64::GPR32RegClass, MRI);
2533 I.getOperand(0).setReg(DstReg);
2534
2535 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2536 }
2537
2538 case TargetOpcode::G_INSERT: {
2539 LLT SrcTy = MRI.getType(I.getOperand(2).getReg());
2540 LLT DstTy = MRI.getType(I.getOperand(0).getReg());
2541 unsigned DstSize = DstTy.getSizeInBits();
2542 // Larger inserts are vectors, same-size ones should be something else by
2543 // now (split up or turned into COPYs).
2544 if (Ty.getSizeInBits() > 64 || SrcTy.getSizeInBits() > 32)
2545 return false;
2546
2547 I.setDesc(TII.get(DstSize == 64 ? AArch64::BFMXri : AArch64::BFMWri));
2548 unsigned LSB = I.getOperand(3).getImm();
2549 unsigned Width = MRI.getType(I.getOperand(2).getReg()).getSizeInBits();
2550 I.getOperand(3).setImm((DstSize - LSB) % DstSize);
2551 MachineInstrBuilder(MF, I).addImm(Width - 1);
2552
2553 if (DstSize < 64) {
2554 assert(DstSize == 32 && SrcTy.getSizeInBits() == 16 &&((DstSize == 32 && SrcTy.getSizeInBits() == 16 &&
"unexpected G_INSERT types") ? static_cast<void> (0) :
__assert_fail ("DstSize == 32 && SrcTy.getSizeInBits() == 16 && \"unexpected G_INSERT types\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2555, __PRETTY_FUNCTION__))
2555 "unexpected G_INSERT types")((DstSize == 32 && SrcTy.getSizeInBits() == 16 &&
"unexpected G_INSERT types") ? static_cast<void> (0) :
__assert_fail ("DstSize == 32 && SrcTy.getSizeInBits() == 16 && \"unexpected G_INSERT types\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2555, __PRETTY_FUNCTION__))
;
2556 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2557 }
2558
2559 Register SrcReg = MRI.createGenericVirtualRegister(LLT::scalar(64));
2560 BuildMI(MBB, I.getIterator(), I.getDebugLoc(),
2561 TII.get(AArch64::SUBREG_TO_REG))
2562 .addDef(SrcReg)
2563 .addImm(0)
2564 .addUse(I.getOperand(2).getReg())
2565 .addImm(AArch64::sub_32);
2566 RBI.constrainGenericRegister(I.getOperand(2).getReg(),
2567 AArch64::GPR32RegClass, MRI);
2568 I.getOperand(2).setReg(SrcReg);
2569
2570 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2571 }
2572 case TargetOpcode::G_FRAME_INDEX: {
2573 // allocas and G_FRAME_INDEX are only supported in addrspace(0).
2574 if (Ty != LLT::pointer(0, 64)) {
2575 LLVM_DEBUG(dbgs() << "G_FRAME_INDEX pointer has type: " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_FRAME_INDEX pointer has type: "
<< Ty << ", expected: " << LLT::pointer(0,
64) << '\n'; } } while (false)
2576 << ", expected: " << LLT::pointer(0, 64) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_FRAME_INDEX pointer has type: "
<< Ty << ", expected: " << LLT::pointer(0,
64) << '\n'; } } while (false)
;
2577 return false;
2578 }
2579 I.setDesc(TII.get(AArch64::ADDXri));
2580
2581 // MOs for a #0 shifted immediate.
2582 I.addOperand(MachineOperand::CreateImm(0));
2583 I.addOperand(MachineOperand::CreateImm(0));
2584
2585 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2586 }
2587
2588 case TargetOpcode::G_GLOBAL_VALUE: {
2589 auto GV = I.getOperand(1).getGlobal();
2590 if (GV->isThreadLocal())
2591 return selectTLSGlobalValue(I, MRI);
2592
2593 unsigned OpFlags = STI.ClassifyGlobalReference(GV, TM);
2594 if (OpFlags & AArch64II::MO_GOT) {
2595 I.setDesc(TII.get(AArch64::LOADgot));
2596 I.getOperand(1).setTargetFlags(OpFlags);
2597 } else if (TM.getCodeModel() == CodeModel::Large) {
2598 // Materialize the global using movz/movk instructions.
2599 materializeLargeCMVal(I, GV, OpFlags);
2600 I.eraseFromParent();
2601 return true;
2602 } else if (TM.getCodeModel() == CodeModel::Tiny) {
2603 I.setDesc(TII.get(AArch64::ADR));
2604 I.getOperand(1).setTargetFlags(OpFlags);
2605 } else {
2606 I.setDesc(TII.get(AArch64::MOVaddr));
2607 I.getOperand(1).setTargetFlags(OpFlags | AArch64II::MO_PAGE);
2608 MachineInstrBuilder MIB(MF, I);
2609 MIB.addGlobalAddress(GV, I.getOperand(1).getOffset(),
2610 OpFlags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
2611 }
2612 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2613 }
2614
2615 case TargetOpcode::G_ZEXTLOAD:
2616 case TargetOpcode::G_LOAD:
2617 case TargetOpcode::G_STORE: {
2618 bool IsZExtLoad = I.getOpcode() == TargetOpcode::G_ZEXTLOAD;
2619 MachineIRBuilder MIB(I);
2620
2621 LLT PtrTy = MRI.getType(I.getOperand(1).getReg());
2622
2623 if (PtrTy != LLT::pointer(0, 64)) {
2624 LLVM_DEBUG(dbgs() << "Load/Store pointer has type: " << PtrTydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Load/Store pointer has type: "
<< PtrTy << ", expected: " << LLT::pointer
(0, 64) << '\n'; } } while (false)
2625 << ", expected: " << LLT::pointer(0, 64) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Load/Store pointer has type: "
<< PtrTy << ", expected: " << LLT::pointer
(0, 64) << '\n'; } } while (false)
;
2626 return false;
2627 }
2628
2629 auto &MemOp = **I.memoperands_begin();
2630 uint64_t MemSizeInBytes = MemOp.getSize();
2631 if (MemOp.isAtomic()) {
2632 // For now we just support s8 acquire loads to be able to compile stack
2633 // protector code.
2634 if (MemOp.getOrdering() == AtomicOrdering::Acquire &&
2635 MemSizeInBytes == 1) {
2636 I.setDesc(TII.get(AArch64::LDARB));
2637 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2638 }
2639 LLVM_DEBUG(dbgs() << "Atomic load/store not fully supported yet\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Atomic load/store not fully supported yet\n"
; } } while (false)
;
2640 return false;
2641 }
2642 unsigned MemSizeInBits = MemSizeInBytes * 8;
2643
2644#ifndef NDEBUG
2645 const Register PtrReg = I.getOperand(1).getReg();
2646 const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, MRI, TRI);
2647 // Sanity-check the pointer register.
2648 assert(PtrRB.getID() == AArch64::GPRRegBankID &&((PtrRB.getID() == AArch64::GPRRegBankID && "Load/Store pointer operand isn't a GPR"
) ? static_cast<void> (0) : __assert_fail ("PtrRB.getID() == AArch64::GPRRegBankID && \"Load/Store pointer operand isn't a GPR\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2649, __PRETTY_FUNCTION__))
2649 "Load/Store pointer operand isn't a GPR")((PtrRB.getID() == AArch64::GPRRegBankID && "Load/Store pointer operand isn't a GPR"
) ? static_cast<void> (0) : __assert_fail ("PtrRB.getID() == AArch64::GPRRegBankID && \"Load/Store pointer operand isn't a GPR\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2649, __PRETTY_FUNCTION__))
;
2650 assert(MRI.getType(PtrReg).isPointer() &&((MRI.getType(PtrReg).isPointer() && "Load/Store pointer operand isn't a pointer"
) ? static_cast<void> (0) : __assert_fail ("MRI.getType(PtrReg).isPointer() && \"Load/Store pointer operand isn't a pointer\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2651, __PRETTY_FUNCTION__))
2651 "Load/Store pointer operand isn't a pointer")((MRI.getType(PtrReg).isPointer() && "Load/Store pointer operand isn't a pointer"
) ? static_cast<void> (0) : __assert_fail ("MRI.getType(PtrReg).isPointer() && \"Load/Store pointer operand isn't a pointer\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2651, __PRETTY_FUNCTION__))
;
2652#endif
2653
2654 const Register ValReg = I.getOperand(0).getReg();
2655 const RegisterBank &RB = *RBI.getRegBank(ValReg, MRI, TRI);
2656
2657 // Helper lambda for partially selecting I. Either returns the original
2658 // instruction with an updated opcode, or a new instruction.
2659 auto SelectLoadStoreAddressingMode = [&]() -> MachineInstr * {
2660 bool IsStore = I.getOpcode() == TargetOpcode::G_STORE;
2661 const unsigned NewOpc =
2662 selectLoadStoreUIOp(I.getOpcode(), RB.getID(), MemSizeInBits);
2663 if (NewOpc == I.getOpcode())
2664 return nullptr;
2665 // Check if we can fold anything into the addressing mode.
2666 auto AddrModeFns =
2667 selectAddrModeIndexed(I.getOperand(1), MemSizeInBytes);
2668 if (!AddrModeFns) {
2669 // Can't fold anything. Use the original instruction.
2670 I.setDesc(TII.get(NewOpc));
2671 I.addOperand(MachineOperand::CreateImm(0));
2672 return &I;
2673 }
2674
2675 // Folded something. Create a new instruction and return it.
2676 auto NewInst = MIB.buildInstr(NewOpc, {}, {}, I.getFlags());
2677 IsStore ? NewInst.addUse(ValReg) : NewInst.addDef(ValReg);
2678 NewInst.cloneMemRefs(I);
2679 for (auto &Fn : *AddrModeFns)
2680 Fn(NewInst);
2681 I.eraseFromParent();
2682 return &*NewInst;
2683 };
2684
2685 MachineInstr *LoadStore = SelectLoadStoreAddressingMode();
2686 if (!LoadStore)
2687 return false;
2688
2689 // If we're storing a 0, use WZR/XZR.
2690 if (Opcode == TargetOpcode::G_STORE) {
2691 auto CVal = getConstantVRegValWithLookThrough(
2692 LoadStore->getOperand(0).getReg(), MRI, /*LookThroughInstrs = */ true,
2693 /*HandleFConstants = */ false);
2694 if (CVal && CVal->Value == 0) {
2695 switch (LoadStore->getOpcode()) {
2696 case AArch64::STRWui:
2697 case AArch64::STRHHui:
2698 case AArch64::STRBBui:
2699 LoadStore->getOperand(0).setReg(AArch64::WZR);
2700 break;
2701 case AArch64::STRXui:
2702 LoadStore->getOperand(0).setReg(AArch64::XZR);
2703 break;
2704 }
2705 }
2706 }
2707
2708 if (IsZExtLoad) {
2709 // The zextload from a smaller type to i32 should be handled by the
2710 // importer.
2711 if (MRI.getType(LoadStore->getOperand(0).getReg()).getSizeInBits() != 64)
2712 return false;
2713 // If we have a ZEXTLOAD then change the load's type to be a narrower reg
2714 // and zero_extend with SUBREG_TO_REG.
2715 Register LdReg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
2716 Register DstReg = LoadStore->getOperand(0).getReg();
2717 LoadStore->getOperand(0).setReg(LdReg);
2718
2719 MIB.setInsertPt(MIB.getMBB(), std::next(LoadStore->getIterator()));
2720 MIB.buildInstr(AArch64::SUBREG_TO_REG, {DstReg}, {})
2721 .addImm(0)
2722 .addUse(LdReg)
2723 .addImm(AArch64::sub_32);
2724 constrainSelectedInstRegOperands(*LoadStore, TII, TRI, RBI);
2725 return RBI.constrainGenericRegister(DstReg, AArch64::GPR64allRegClass,
2726 MRI);
2727 }
2728 return constrainSelectedInstRegOperands(*LoadStore, TII, TRI, RBI);
2729 }
2730
2731 case TargetOpcode::G_SMULH:
2732 case TargetOpcode::G_UMULH: {
2733 // Reject the various things we don't support yet.
2734 if (unsupportedBinOp(I, RBI, MRI, TRI))
2735 return false;
2736
2737 const Register DefReg = I.getOperand(0).getReg();
2738 const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
2739
2740 if (RB.getID() != AArch64::GPRRegBankID) {
2741 LLVM_DEBUG(dbgs() << "G_[SU]MULH on bank: " << RB << ", expected: GPR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_[SU]MULH on bank: " <<
RB << ", expected: GPR\n"; } } while (false)
;
2742 return false;
2743 }
2744
2745 if (Ty != LLT::scalar(64)) {
2746 LLVM_DEBUG(dbgs() << "G_[SU]MULH has type: " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_[SU]MULH has type: " <<
Ty << ", expected: " << LLT::scalar(64) <<
'\n'; } } while (false)
2747 << ", expected: " << LLT::scalar(64) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_[SU]MULH has type: " <<
Ty << ", expected: " << LLT::scalar(64) <<
'\n'; } } while (false)
;
2748 return false;
2749 }
2750
2751 unsigned NewOpc = I.getOpcode() == TargetOpcode::G_SMULH ? AArch64::SMULHrr
2752 : AArch64::UMULHrr;
2753 I.setDesc(TII.get(NewOpc));
2754
2755 // Now that we selected an opcode, we need to constrain the register
2756 // operands to use appropriate classes.
2757 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2758 }
2759 case TargetOpcode::G_LSHR:
2760 case TargetOpcode::G_ASHR:
2761 if (MRI.getType(I.getOperand(0).getReg()).isVector())
2762 return selectVectorAshrLshr(I, MRI);
2763 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2764 case TargetOpcode::G_SHL:
2765 if (Opcode == TargetOpcode::G_SHL &&
2766 MRI.getType(I.getOperand(0).getReg()).isVector())
2767 return selectVectorSHL(I, MRI);
2768 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2769 case TargetOpcode::G_FADD:
2770 case TargetOpcode::G_FSUB:
2771 case TargetOpcode::G_FMUL:
2772 case TargetOpcode::G_FDIV:
2773 case TargetOpcode::G_OR: {
2774 // Reject the various things we don't support yet.
2775 if (unsupportedBinOp(I, RBI, MRI, TRI))
2776 return false;
2777
2778 const unsigned OpSize = Ty.getSizeInBits();
2779
2780 const Register DefReg = I.getOperand(0).getReg();
2781 const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
2782
2783 const unsigned NewOpc = selectBinaryOp(I.getOpcode(), RB.getID(), OpSize);
2784 if (NewOpc == I.getOpcode())
2785 return false;
2786
2787 I.setDesc(TII.get(NewOpc));
2788 // FIXME: Should the type be always reset in setDesc?
2789
2790 // Now that we selected an opcode, we need to constrain the register
2791 // operands to use appropriate classes.
2792 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2793 }
2794
2795 case TargetOpcode::G_PTR_ADD: {
2796 MachineIRBuilder MIRBuilder(I);
2797 emitADD(I.getOperand(0).getReg(), I.getOperand(1), I.getOperand(2),
2798 MIRBuilder);
2799 I.eraseFromParent();
2800 return true;
2801 }
2802 case TargetOpcode::G_SADDO:
2803 case TargetOpcode::G_UADDO:
2804 case TargetOpcode::G_SSUBO:
2805 case TargetOpcode::G_USUBO: {
2806 // Emit the operation and get the correct condition code.
2807 MachineIRBuilder MIRBuilder(I);
2808 auto OpAndCC = emitOverflowOp(Opcode, I.getOperand(0).getReg(),
2809 I.getOperand(2), I.getOperand(3), MIRBuilder);
2810
2811 // Now, put the overflow result in the register given by the first operand
2812 // to the overflow op. CSINC increments the result when the predicate is
2813 // false, so to get the increment when it's true, we need to use the
2814 // inverse. In this case, we want to increment when carry is set.
2815 Register ZReg = AArch64::WZR;
2816 auto CsetMI = MIRBuilder
2817 .buildInstr(AArch64::CSINCWr, {I.getOperand(1).getReg()},
2818 {ZReg, ZReg})
2819 .addImm(getInvertedCondCode(OpAndCC.second));
2820 constrainSelectedInstRegOperands(*CsetMI, TII, TRI, RBI);
2821 I.eraseFromParent();
2822 return true;
2823 }
2824
2825 case TargetOpcode::G_PTRMASK: {
2826 Register MaskReg = I.getOperand(2).getReg();
2827 Optional<int64_t> MaskVal = getConstantVRegSExtVal(MaskReg, MRI);
2828 // TODO: Implement arbitrary cases
2829 if (!MaskVal || !isShiftedMask_64(*MaskVal))
2830 return false;
2831
2832 uint64_t Mask = *MaskVal;
2833 I.setDesc(TII.get(AArch64::ANDXri));
2834 I.getOperand(2).ChangeToImmediate(
2835 AArch64_AM::encodeLogicalImmediate(Mask, 64));
2836
2837 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2838 }
2839 case TargetOpcode::G_PTRTOINT:
2840 case TargetOpcode::G_TRUNC: {
2841 const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
2842 const LLT SrcTy = MRI.getType(I.getOperand(1).getReg());
2843
2844 const Register DstReg = I.getOperand(0).getReg();
2845 const Register SrcReg = I.getOperand(1).getReg();
2846
2847 const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
2848 const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
2849
2850 if (DstRB.getID() != SrcRB.getID()) {
2851 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_TRUNC/G_PTRTOINT input/output on different banks\n"
; } } while (false)
2852 dbgs() << "G_TRUNC/G_PTRTOINT input/output on different banks\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_TRUNC/G_PTRTOINT input/output on different banks\n"
; } } while (false)
;
2853 return false;
2854 }
2855
2856 if (DstRB.getID() == AArch64::GPRRegBankID) {
2857 const TargetRegisterClass *DstRC =
2858 getRegClassForTypeOnBank(DstTy, DstRB, RBI);
2859 if (!DstRC)
2860 return false;
2861
2862 const TargetRegisterClass *SrcRC =
2863 getRegClassForTypeOnBank(SrcTy, SrcRB, RBI);
2864 if (!SrcRC)
2865 return false;
2866
2867 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
2868 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
2869 LLVM_DEBUG(dbgs() << "Failed to constrain G_TRUNC/G_PTRTOINT\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain G_TRUNC/G_PTRTOINT\n"
; } } while (false)
;
2870 return false;
2871 }
2872
2873 if (DstRC == SrcRC) {
2874 // Nothing to be done
2875 } else if (Opcode == TargetOpcode::G_TRUNC && DstTy == LLT::scalar(32) &&
2876 SrcTy == LLT::scalar(64)) {
2877 llvm_unreachable("TableGen can import this case")::llvm::llvm_unreachable_internal("TableGen can import this case"
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2877)
;
2878 return false;
2879 } else if (DstRC == &AArch64::GPR32RegClass &&
2880 SrcRC == &AArch64::GPR64RegClass) {
2881 I.getOperand(1).setSubReg(AArch64::sub_32);
2882 } else {
2883 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n"
; } } while (false)
2884 dbgs() << "Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n"
; } } while (false)
;
2885 return false;
2886 }
2887
2888 I.setDesc(TII.get(TargetOpcode::COPY));
2889 return true;
2890 } else if (DstRB.getID() == AArch64::FPRRegBankID) {
2891 if (DstTy == LLT::vector(4, 16) && SrcTy == LLT::vector(4, 32)) {
2892 I.setDesc(TII.get(AArch64::XTNv4i16));
2893 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2894 return true;
2895 }
2896
2897 if (!SrcTy.isVector() && SrcTy.getSizeInBits() == 128) {
2898 MachineIRBuilder MIB(I);
2899 MachineInstr *Extract = emitExtractVectorElt(
2900 DstReg, DstRB, LLT::scalar(DstTy.getSizeInBits()), SrcReg, 0, MIB);
2901 if (!Extract)
2902 return false;
2903 I.eraseFromParent();
2904 return true;
2905 }
2906
2907 // We might have a vector G_PTRTOINT, in which case just emit a COPY.
2908 if (Opcode == TargetOpcode::G_PTRTOINT) {
2909 assert(DstTy.isVector() && "Expected an FPR ptrtoint to be a vector")((DstTy.isVector() && "Expected an FPR ptrtoint to be a vector"
) ? static_cast<void> (0) : __assert_fail ("DstTy.isVector() && \"Expected an FPR ptrtoint to be a vector\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2909, __PRETTY_FUNCTION__))
;
2910 I.setDesc(TII.get(TargetOpcode::COPY));
2911 return true;
2912 }
2913 }
2914
2915 return false;
2916 }
2917
2918 case TargetOpcode::G_ANYEXT: {
2919 const Register DstReg = I.getOperand(0).getReg();
2920 const Register SrcReg = I.getOperand(1).getReg();
2921
2922 const RegisterBank &RBDst = *RBI.getRegBank(DstReg, MRI, TRI);
2923 if (RBDst.getID() != AArch64::GPRRegBankID) {
2924 LLVM_DEBUG(dbgs() << "G_ANYEXT on bank: " << RBDstdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT on bank: " <<
RBDst << ", expected: GPR\n"; } } while (false)
2925 << ", expected: GPR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT on bank: " <<
RBDst << ", expected: GPR\n"; } } while (false)
;
2926 return false;
2927 }
2928
2929 const RegisterBank &RBSrc = *RBI.getRegBank(SrcReg, MRI, TRI);
2930 if (RBSrc.getID() != AArch64::GPRRegBankID) {
2931 LLVM_DEBUG(dbgs() << "G_ANYEXT on bank: " << RBSrcdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT on bank: " <<
RBSrc << ", expected: GPR\n"; } } while (false)
2932 << ", expected: GPR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT on bank: " <<
RBSrc << ", expected: GPR\n"; } } while (false)
;
2933 return false;
2934 }
2935
2936 const unsigned DstSize = MRI.getType(DstReg).getSizeInBits();
2937
2938 if (DstSize == 0) {
2939 LLVM_DEBUG(dbgs() << "G_ANYEXT operand has no size, not a gvreg?\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT operand has no size, not a gvreg?\n"
; } } while (false)
;
2940 return false;
2941 }
2942
2943 if (DstSize != 64 && DstSize > 32) {
2944 LLVM_DEBUG(dbgs() << "G_ANYEXT to size: " << DstSizedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT to size: " <<
DstSize << ", expected: 32 or 64\n"; } } while (false)
2945 << ", expected: 32 or 64\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT to size: " <<
DstSize << ", expected: 32 or 64\n"; } } while (false)
;
2946 return false;
2947 }
2948 // At this point G_ANYEXT is just like a plain COPY, but we need
2949 // to explicitly form the 64-bit value if any.
2950 if (DstSize > 32) {
2951 Register ExtSrc = MRI.createVirtualRegister(&AArch64::GPR64allRegClass);
2952 BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::SUBREG_TO_REG))
2953 .addDef(ExtSrc)
2954 .addImm(0)
2955 .addUse(SrcReg)
2956 .addImm(AArch64::sub_32);
2957 I.getOperand(1).setReg(ExtSrc);
2958 }
2959 return selectCopy(I, TII, MRI, TRI, RBI);
2960 }
2961
2962 case TargetOpcode::G_ZEXT:
2963 case TargetOpcode::G_SEXT_INREG:
2964 case TargetOpcode::G_SEXT: {
2965 unsigned Opcode = I.getOpcode();
2966 const bool IsSigned = Opcode != TargetOpcode::G_ZEXT;
2967 const Register DefReg = I.getOperand(0).getReg();
2968 Register SrcReg = I.getOperand(1).getReg();
2969 const LLT DstTy = MRI.getType(DefReg);
2970 const LLT SrcTy = MRI.getType(SrcReg);
2971 unsigned DstSize = DstTy.getSizeInBits();
2972 unsigned SrcSize = SrcTy.getSizeInBits();
2973
2974 // SEXT_INREG has the same src reg size as dst, the size of the value to be
2975 // extended is encoded in the imm.
2976 if (Opcode == TargetOpcode::G_SEXT_INREG)
2977 SrcSize = I.getOperand(2).getImm();
2978
2979 if (DstTy.isVector())
2980 return false; // Should be handled by imported patterns.
2981
2982 assert((*RBI.getRegBank(DefReg, MRI, TRI)).getID() ==(((*RBI.getRegBank(DefReg, MRI, TRI)).getID() == AArch64::GPRRegBankID
&& "Unexpected ext regbank") ? static_cast<void>
(0) : __assert_fail ("(*RBI.getRegBank(DefReg, MRI, TRI)).getID() == AArch64::GPRRegBankID && \"Unexpected ext regbank\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2984, __PRETTY_FUNCTION__))
2983 AArch64::GPRRegBankID &&(((*RBI.getRegBank(DefReg, MRI, TRI)).getID() == AArch64::GPRRegBankID
&& "Unexpected ext regbank") ? static_cast<void>
(0) : __assert_fail ("(*RBI.getRegBank(DefReg, MRI, TRI)).getID() == AArch64::GPRRegBankID && \"Unexpected ext regbank\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2984, __PRETTY_FUNCTION__))
2984 "Unexpected ext regbank")(((*RBI.getRegBank(DefReg, MRI, TRI)).getID() == AArch64::GPRRegBankID
&& "Unexpected ext regbank") ? static_cast<void>
(0) : __assert_fail ("(*RBI.getRegBank(DefReg, MRI, TRI)).getID() == AArch64::GPRRegBankID && \"Unexpected ext regbank\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2984, __PRETTY_FUNCTION__))
;
2985
2986 MachineIRBuilder MIB(I);
2987 MachineInstr *ExtI;
2988
2989 // First check if we're extending the result of a load which has a dest type
2990 // smaller than 32 bits, then this zext is redundant. GPR32 is the smallest
2991 // GPR register on AArch64 and all loads which are smaller automatically
2992 // zero-extend the upper bits. E.g.
2993 // %v(s8) = G_LOAD %p, :: (load 1)
2994 // %v2(s32) = G_ZEXT %v(s8)
2995 if (!IsSigned) {
2996 auto *LoadMI = getOpcodeDef(TargetOpcode::G_LOAD, SrcReg, MRI);
2997 bool IsGPR =
2998 RBI.getRegBank(SrcReg, MRI, TRI)->getID() == AArch64::GPRRegBankID;
2999 if (LoadMI && IsGPR) {
3000 const MachineMemOperand *MemOp = *LoadMI->memoperands_begin();
3001 unsigned BytesLoaded = MemOp->getSize();
3002 if (BytesLoaded < 4 && SrcTy.getSizeInBytes() == BytesLoaded)
3003 return selectCopy(I, TII, MRI, TRI, RBI);
3004 }
3005
3006 // If we are zero extending from 32 bits to 64 bits, it's possible that
3007 // the instruction implicitly does the zero extend for us. In that case,
3008 // we can just emit a SUBREG_TO_REG.
3009 if (IsGPR && SrcSize == 32 && DstSize == 64) {
3010 // Unlike with the G_LOAD case, we don't want to look through copies
3011 // here.
3012 MachineInstr *Def = MRI.getVRegDef(SrcReg);
3013 if (Def && isDef32(*Def)) {
3014 MIB.buildInstr(AArch64::SUBREG_TO_REG, {DefReg}, {})
3015 .addImm(0)
3016 .addUse(SrcReg)
3017 .addImm(AArch64::sub_32);
3018
3019 if (!RBI.constrainGenericRegister(DefReg, AArch64::GPR64RegClass,
3020 MRI)) {
3021 LLVM_DEBUG(dbgs() << "Failed to constrain G_ZEXT destination\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain G_ZEXT destination\n"
; } } while (false)
;
3022 return false;
3023 }
3024
3025 if (!RBI.constrainGenericRegister(SrcReg, AArch64::GPR32RegClass,
3026 MRI)) {
3027 LLVM_DEBUG(dbgs() << "Failed to constrain G_ZEXT source\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain G_ZEXT source\n"
; } } while (false)
;
3028 return false;
3029 }
3030
3031 I.eraseFromParent();
3032 return true;
3033 }
3034 }
3035 }
3036
3037 if (DstSize == 64) {
3038 if (Opcode != TargetOpcode::G_SEXT_INREG) {
3039 // FIXME: Can we avoid manually doing this?
3040 if (!RBI.constrainGenericRegister(SrcReg, AArch64::GPR32RegClass,
3041 MRI)) {
3042 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(Opcode)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain " <<
TII.getName(Opcode) << " operand\n"; } } while (false)
3043 << " operand\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain " <<
TII.getName(Opcode) << " operand\n"; } } while (false)
;
3044 return false;
3045 }
3046 SrcReg = MIB.buildInstr(AArch64::SUBREG_TO_REG,
3047 {&AArch64::GPR64RegClass}, {})
3048 .addImm(0)
3049 .addUse(SrcReg)
3050 .addImm(AArch64::sub_32)
3051 .getReg(0);
3052 }
3053
3054 ExtI = MIB.buildInstr(IsSigned ? AArch64::SBFMXri : AArch64::UBFMXri,
3055 {DefReg}, {SrcReg})
3056 .addImm(0)
3057 .addImm(SrcSize - 1);
3058 } else if (DstSize <= 32) {
3059 ExtI = MIB.buildInstr(IsSigned ? AArch64::SBFMWri : AArch64::UBFMWri,
3060 {DefReg}, {SrcReg})
3061 .addImm(0)
3062 .addImm(SrcSize - 1);
3063 } else {
3064 return false;
3065 }
3066
3067 constrainSelectedInstRegOperands(*ExtI, TII, TRI, RBI);
3068 I.eraseFromParent();
3069 return true;
3070 }
3071
3072 case TargetOpcode::G_SITOFP:
3073 case TargetOpcode::G_UITOFP:
3074 case TargetOpcode::G_FPTOSI:
3075 case TargetOpcode::G_FPTOUI: {
3076 const LLT DstTy = MRI.getType(I.getOperand(0).getReg()),
3077 SrcTy = MRI.getType(I.getOperand(1).getReg());
3078 const unsigned NewOpc = selectFPConvOpc(Opcode, DstTy, SrcTy);
3079 if (NewOpc == Opcode)
3080 return false;
3081
3082 I.setDesc(TII.get(NewOpc));
3083 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
3084
3085 return true;
3086 }
3087
3088 case TargetOpcode::G_FREEZE:
3089 return selectCopy(I, TII, MRI, TRI, RBI);
3090
3091 case TargetOpcode::G_INTTOPTR:
3092 // The importer is currently unable to import pointer types since they
3093 // didn't exist in SelectionDAG.
3094 return selectCopy(I, TII, MRI, TRI, RBI);
3095
3096 case TargetOpcode::G_BITCAST:
3097 // Imported SelectionDAG rules can handle every bitcast except those that
3098 // bitcast from a type to the same type. Ideally, these shouldn't occur
3099 // but we might not run an optimizer that deletes them. The other exception
3100 // is bitcasts involving pointer types, as SelectionDAG has no knowledge
3101 // of them.
3102 return selectCopy(I, TII, MRI, TRI, RBI);
3103
3104 case TargetOpcode::G_SELECT: {
3105 if (MRI.getType(I.getOperand(1).getReg()) != LLT::scalar(1)) {
3106 LLVM_DEBUG(dbgs() << "G_SELECT cond has type: " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_SELECT cond has type: "
<< Ty << ", expected: " << LLT::scalar(1) <<
'\n'; } } while (false)
3107 << ", expected: " << LLT::scalar(1) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_SELECT cond has type: "
<< Ty << ", expected: " << LLT::scalar(1) <<
'\n'; } } while (false)
;
3108 return false;
3109 }
3110
3111 const Register CondReg = I.getOperand(1).getReg();
3112 const Register TReg = I.getOperand(2).getReg();
3113 const Register FReg = I.getOperand(3).getReg();
3114
3115 if (tryOptSelect(I))
3116 return true;
3117
3118 // Make sure to use an unused vreg instead of wzr, so that the peephole
3119 // optimizations will be able to optimize these.
3120 MachineIRBuilder MIB(I);
3121 Register DeadVReg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
3122 auto TstMI = MIB.buildInstr(AArch64::ANDSWri, {DeadVReg}, {CondReg})
3123 .addImm(AArch64_AM::encodeLogicalImmediate(1, 32));
3124 constrainSelectedInstRegOperands(*TstMI, TII, TRI, RBI);
3125 if (!emitSelect(I.getOperand(0).getReg(), TReg, FReg, AArch64CC::NE, MIB))
3126 return false;
3127 I.eraseFromParent();
3128 return true;
3129 }
3130 case TargetOpcode::G_ICMP: {
3131 if (Ty.isVector())
3132 return selectVectorICmp(I, MRI);
3133
3134 if (Ty != LLT::scalar(32)) {
3135 LLVM_DEBUG(dbgs() << "G_ICMP result has type: " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ICMP result has type: "
<< Ty << ", expected: " << LLT::scalar(32)
<< '\n'; } } while (false)
3136 << ", expected: " << LLT::scalar(32) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ICMP result has type: "
<< Ty << ", expected: " << LLT::scalar(32)
<< '\n'; } } while (false)
;
3137 return false;
3138 }
3139
3140 MachineIRBuilder MIRBuilder(I);
3141 auto Pred = static_cast<CmpInst::Predicate>(I.getOperand(1).getPredicate());
3142 emitIntegerCompare(I.getOperand(2), I.getOperand(3), I.getOperand(1),
3143 MIRBuilder);
3144 emitCSetForICMP(I.getOperand(0).getReg(), Pred, MIRBuilder);
3145 I.eraseFromParent();
3146 return true;
3147 }
3148
3149 case TargetOpcode::G_FCMP: {
3150 MachineIRBuilder MIRBuilder(I);
3151 CmpInst::Predicate Pred =
3152 static_cast<CmpInst::Predicate>(I.getOperand(1).getPredicate());
3153 if (!emitFPCompare(I.getOperand(2).getReg(), I.getOperand(3).getReg(),
3154 MIRBuilder, Pred) ||
3155 !emitCSetForFCmp(I.getOperand(0).getReg(), Pred, MIRBuilder))
3156 return false;
3157 I.eraseFromParent();
3158 return true;
3159 }
3160 case TargetOpcode::G_VASTART:
3161 return STI.isTargetDarwin() ? selectVaStartDarwin(I, MF, MRI)
3162 : selectVaStartAAPCS(I, MF, MRI);
3163 case TargetOpcode::G_INTRINSIC:
3164 return selectIntrinsic(I, MRI);
3165 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
3166 return selectIntrinsicWithSideEffects(I, MRI);
3167 case TargetOpcode::G_IMPLICIT_DEF: {
3168 I.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF));
3169 const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
3170 const Register DstReg = I.getOperand(0).getReg();
3171 const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
3172 const TargetRegisterClass *DstRC =
3173 getRegClassForTypeOnBank(DstTy, DstRB, RBI);
3174 RBI.constrainGenericRegister(DstReg, *DstRC, MRI);
3175 return true;
3176 }
3177 case TargetOpcode::G_BLOCK_ADDR: {
3178 if (TM.getCodeModel() == CodeModel::Large) {
3179 materializeLargeCMVal(I, I.getOperand(1).getBlockAddress(), 0);
3180 I.eraseFromParent();
3181 return true;
3182 } else {
3183 I.setDesc(TII.get(AArch64::MOVaddrBA));
3184 auto MovMI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::MOVaddrBA),
3185 I.getOperand(0).getReg())
3186 .addBlockAddress(I.getOperand(1).getBlockAddress(),
3187 /* Offset */ 0, AArch64II::MO_PAGE)
3188 .addBlockAddress(
3189 I.getOperand(1).getBlockAddress(), /* Offset */ 0,
3190 AArch64II::MO_NC | AArch64II::MO_PAGEOFF);
3191 I.eraseFromParent();
3192 return constrainSelectedInstRegOperands(*MovMI, TII, TRI, RBI);
3193 }
3194 }
3195 case AArch64::G_DUP: {
3196 // When the scalar of G_DUP is an s8/s16 gpr, they can't be selected by
3197 // imported patterns. Do it manually here. Avoiding generating s16 gpr is
3198 // difficult because at RBS we may end up pessimizing the fpr case if we
3199 // decided to add an anyextend to fix this. Manual selection is the most
3200 // robust solution for now.
3201 if (RBI.getRegBank(I.getOperand(1).getReg(), MRI, TRI)->getID() !=
3202 AArch64::GPRRegBankID)
3203 return false; // We expect the fpr regbank case to be imported.
3204 LLT VecTy = MRI.getType(I.getOperand(0).getReg());
3205 if (VecTy == LLT::vector(8, 8))
3206 I.setDesc(TII.get(AArch64::DUPv8i8gpr));
3207 else if (VecTy == LLT::vector(16, 8))
3208 I.setDesc(TII.get(AArch64::DUPv16i8gpr));
3209 else if (VecTy == LLT::vector(4, 16))
3210 I.setDesc(TII.get(AArch64::DUPv4i16gpr));
3211 else if (VecTy == LLT::vector(8, 16))
3212 I.setDesc(TII.get(AArch64::DUPv8i16gpr));
3213 else
3214 return false;
3215 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
3216 }
3217 case TargetOpcode::G_INTRINSIC_TRUNC:
3218 return selectIntrinsicTrunc(I, MRI);
3219 case TargetOpcode::G_INTRINSIC_ROUND:
3220 return selectIntrinsicRound(I, MRI);
3221 case TargetOpcode::G_BUILD_VECTOR:
3222 return selectBuildVector(I, MRI);
3223 case TargetOpcode::G_MERGE_VALUES:
3224 return selectMergeValues(I, MRI);
3225 case TargetOpcode::G_UNMERGE_VALUES:
3226 return selectUnmergeValues(I, MRI);
3227 case TargetOpcode::G_SHUFFLE_VECTOR:
3228 return selectShuffleVector(I, MRI);
3229 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
3230 return selectExtractElt(I, MRI);
3231 case TargetOpcode::G_INSERT_VECTOR_ELT:
3232 return selectInsertElt(I, MRI);
3233 case TargetOpcode::G_CONCAT_VECTORS:
3234 return selectConcatVectors(I, MRI);
3235 case TargetOpcode::G_JUMP_TABLE:
3236 return selectJumpTable(I, MRI);
3237 case TargetOpcode::G_VECREDUCE_FADD:
3238 case TargetOpcode::G_VECREDUCE_ADD:
3239 return selectReduction(I, MRI);
3240 }
3241
3242 return false;
3243}
3244
3245bool AArch64InstructionSelector::selectReduction(
3246 MachineInstr &I, MachineRegisterInfo &MRI) const {
3247 Register VecReg = I.getOperand(1).getReg();
3248 LLT VecTy = MRI.getType(VecReg);
3249 if (I.getOpcode() == TargetOpcode::G_VECREDUCE_ADD) {
3250 // For <2 x i32> ADDPv2i32 generates an FPR64 value, so we need to emit
3251 // a subregister copy afterwards.
3252 if (VecTy == LLT::vector(2, 32)) {
3253 MachineIRBuilder MIB(I);
3254 Register DstReg = I.getOperand(0).getReg();
3255 auto AddP = MIB.buildInstr(AArch64::ADDPv2i32, {&AArch64::FPR64RegClass},
3256 {VecReg, VecReg});
3257 auto Copy = MIB.buildInstr(TargetOpcode::COPY, {DstReg}, {})
3258 .addReg(AddP.getReg(0), 0, AArch64::ssub)
3259 .getReg(0);
3260 RBI.constrainGenericRegister(Copy, AArch64::FPR32RegClass, MRI);
3261 I.eraseFromParent();
3262 return constrainSelectedInstRegOperands(*AddP, TII, TRI, RBI);
3263 }
3264
3265 unsigned Opc = 0;
3266 if (VecTy == LLT::vector(16, 8))
3267 Opc = AArch64::ADDVv16i8v;
3268 else if (VecTy == LLT::vector(8, 16))
3269 Opc = AArch64::ADDVv8i16v;
3270 else if (VecTy == LLT::vector(4, 32))
3271 Opc = AArch64::ADDVv4i32v;
3272 else if (VecTy == LLT::vector(2, 64))
3273 Opc = AArch64::ADDPv2i64p;
3274 else {
3275 LLVM_DEBUG(dbgs() << "Unhandled type for add reduction")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled type for add reduction"
; } } while (false)
;
3276 return false;
3277 }
3278 I.setDesc(TII.get(Opc));
3279 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
3280 }
3281
3282 if (I.getOpcode() == TargetOpcode::G_VECREDUCE_FADD) {
3283 unsigned Opc = 0;
3284 if (VecTy == LLT::vector(2, 32))
3285 Opc = AArch64::FADDPv2i32p;
3286 else if (VecTy == LLT::vector(2, 64))
3287 Opc = AArch64::FADDPv2i64p;
3288 else {
3289 LLVM_DEBUG(dbgs() << "Unhandled type for fadd reduction")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled type for fadd reduction"
; } } while (false)
;
3290 return false;
3291 }
3292 I.setDesc(TII.get(Opc));
3293 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
3294 }
3295 return false;
3296}
3297
3298bool AArch64InstructionSelector::selectBrJT(MachineInstr &I,
3299 MachineRegisterInfo &MRI) const {
3300 assert(I.getOpcode() == TargetOpcode::G_BRJT && "Expected G_BRJT")((I.getOpcode() == TargetOpcode::G_BRJT && "Expected G_BRJT"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BRJT && \"Expected G_BRJT\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3300, __PRETTY_FUNCTION__))
;
3301 Register JTAddr = I.getOperand(0).getReg();
3302 unsigned JTI = I.getOperand(1).getIndex();
3303 Register Index = I.getOperand(2).getReg();
3304 MachineIRBuilder MIB(I);
3305
3306 Register TargetReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
3307 Register ScratchReg = MRI.createVirtualRegister(&AArch64::GPR64spRegClass);
3308
3309 MF->getInfo<AArch64FunctionInfo>()->setJumpTableEntryInfo(JTI, 4, nullptr);
3310 auto JumpTableInst = MIB.buildInstr(AArch64::JumpTableDest32,
3311 {TargetReg, ScratchReg}, {JTAddr, Index})
3312 .addJumpTableIndex(JTI);
3313 // Build the indirect branch.
3314 MIB.buildInstr(AArch64::BR, {}, {TargetReg});
3315 I.eraseFromParent();
3316 return constrainSelectedInstRegOperands(*JumpTableInst, TII, TRI, RBI);
3317}
3318
3319bool AArch64InstructionSelector::selectJumpTable(
3320 MachineInstr &I, MachineRegisterInfo &MRI) const {
3321 assert(I.getOpcode() == TargetOpcode::G_JUMP_TABLE && "Expected jump table")((I.getOpcode() == TargetOpcode::G_JUMP_TABLE && "Expected jump table"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_JUMP_TABLE && \"Expected jump table\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3321, __PRETTY_FUNCTION__))
;
3322 assert(I.getOperand(1).isJTI() && "Jump table op should have a JTI!")((I.getOperand(1).isJTI() && "Jump table op should have a JTI!"
) ? static_cast<void> (0) : __assert_fail ("I.getOperand(1).isJTI() && \"Jump table op should have a JTI!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3322, __PRETTY_FUNCTION__))
;
3323
3324 Register DstReg = I.getOperand(0).getReg();
3325 unsigned JTI = I.getOperand(1).getIndex();
3326 // We generate a MOVaddrJT which will get expanded to an ADRP + ADD later.
3327 MachineIRBuilder MIB(I);
3328 auto MovMI =
3329 MIB.buildInstr(AArch64::MOVaddrJT, {DstReg}, {})
3330 .addJumpTableIndex(JTI, AArch64II::MO_PAGE)
3331 .addJumpTableIndex(JTI, AArch64II::MO_NC | AArch64II::MO_PAGEOFF);
3332 I.eraseFromParent();
3333 return constrainSelectedInstRegOperands(*MovMI, TII, TRI, RBI);
3334}
3335
3336bool AArch64InstructionSelector::selectTLSGlobalValue(
3337 MachineInstr &I, MachineRegisterInfo &MRI) const {
3338 if (!STI.isTargetMachO())
3339 return false;
3340 MachineFunction &MF = *I.getParent()->getParent();
3341 MF.getFrameInfo().setAdjustsStack(true);
3342
3343 const GlobalValue &GV = *I.getOperand(1).getGlobal();
3344 MachineIRBuilder MIB(I);
3345
3346 auto LoadGOT =
3347 MIB.buildInstr(AArch64::LOADgot, {&AArch64::GPR64commonRegClass}, {})
3348 .addGlobalAddress(&GV, 0, AArch64II::MO_TLS);
3349
3350 auto Load = MIB.buildInstr(AArch64::LDRXui, {&AArch64::GPR64commonRegClass},
3351 {LoadGOT.getReg(0)})
3352 .addImm(0);
3353
3354 MIB.buildCopy(Register(AArch64::X0), LoadGOT.getReg(0));
3355 // TLS calls preserve all registers except those that absolutely must be
3356 // trashed: X0 (it takes an argument), LR (it's a call) and NZCV (let's not be
3357 // silly).
3358 MIB.buildInstr(getBLRCallOpcode(MF), {}, {Load})
3359 .addUse(AArch64::X0, RegState::Implicit)
3360 .addDef(AArch64::X0, RegState::Implicit)
3361 .addRegMask(TRI.getTLSCallPreservedMask());
3362
3363 MIB.buildCopy(I.getOperand(0).getReg(), Register(AArch64::X0));
3364 RBI.constrainGenericRegister(I.getOperand(0).getReg(), AArch64::GPR64RegClass,
3365 MRI);
3366 I.eraseFromParent();
3367 return true;
3368}
3369
3370bool AArch64InstructionSelector::selectIntrinsicTrunc(
3371 MachineInstr &I, MachineRegisterInfo &MRI) const {
3372 const LLT SrcTy = MRI.getType(I.getOperand(0).getReg());
3373
3374 // Select the correct opcode.
3375 unsigned Opc = 0;
3376 if (!SrcTy.isVector()) {
3377 switch (SrcTy.getSizeInBits()) {
3378 default:
3379 case 16:
3380 Opc = AArch64::FRINTZHr;
3381 break;
3382 case 32:
3383 Opc = AArch64::FRINTZSr;
3384 break;
3385 case 64:
3386 Opc = AArch64::FRINTZDr;
3387 break;
3388 }
3389 } else {
3390 unsigned NumElts = SrcTy.getNumElements();
3391 switch (SrcTy.getElementType().getSizeInBits()) {
3392 default:
3393 break;
3394 case 16:
3395 if (NumElts == 4)
3396 Opc = AArch64::FRINTZv4f16;
3397 else if (NumElts == 8)
3398 Opc = AArch64::FRINTZv8f16;
3399 break;
3400 case 32:
3401 if (NumElts == 2)
3402 Opc = AArch64::FRINTZv2f32;
3403 else if (NumElts == 4)
3404 Opc = AArch64::FRINTZv4f32;
3405 break;
3406 case 64:
3407 if (NumElts == 2)
3408 Opc = AArch64::FRINTZv2f64;
3409 break;
3410 }
3411 }
3412
3413 if (!Opc) {
3414 // Didn't get an opcode above, bail.
3415 LLVM_DEBUG(dbgs() << "Unsupported type for G_INTRINSIC_TRUNC!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported type for G_INTRINSIC_TRUNC!\n"
; } } while (false)
;
3416 return false;
3417 }
3418
3419 // Legalization would have set us up perfectly for this; we just need to
3420 // set the opcode and move on.
3421 I.setDesc(TII.get(Opc));
3422 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
3423}
3424
3425bool AArch64InstructionSelector::selectIntrinsicRound(
3426 MachineInstr &I, MachineRegisterInfo &MRI) const {
3427 const LLT SrcTy = MRI.getType(I.getOperand(0).getReg());
3428
3429 // Select the correct opcode.
3430 unsigned Opc = 0;
3431 if (!SrcTy.isVector()) {
3432 switch (SrcTy.getSizeInBits()) {
3433 default:
3434 case 16:
3435 Opc = AArch64::FRINTAHr;
3436 break;
3437 case 32:
3438 Opc = AArch64::FRINTASr;
3439 break;
3440 case 64:
3441 Opc = AArch64::FRINTADr;
3442 break;
3443 }
3444 } else {
3445 unsigned NumElts = SrcTy.getNumElements();
3446 switch (SrcTy.getElementType().getSizeInBits()) {
3447 default:
3448 break;
3449 case 16:
3450 if (NumElts == 4)
3451 Opc = AArch64::FRINTAv4f16;
3452 else if (NumElts == 8)
3453 Opc = AArch64::FRINTAv8f16;
3454 break;
3455 case 32:
3456 if (NumElts == 2)
3457 Opc = AArch64::FRINTAv2f32;
3458 else if (NumElts == 4)
3459 Opc = AArch64::FRINTAv4f32;
3460 break;
3461 case 64:
3462 if (NumElts == 2)
3463 Opc = AArch64::FRINTAv2f64;
3464 break;
3465 }
3466 }
3467
3468 if (!Opc) {
3469 // Didn't get an opcode above, bail.
3470 LLVM_DEBUG(dbgs() << "Unsupported type for G_INTRINSIC_ROUND!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported type for G_INTRINSIC_ROUND!\n"
; } } while (false)
;
3471 return false;
3472 }
3473
3474 // Legalization would have set us up perfectly for this; we just need to
3475 // set the opcode and move on.
3476 I.setDesc(TII.get(Opc));
3477 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
3478}
3479
3480bool AArch64InstructionSelector::selectVectorICmp(
3481 MachineInstr &I, MachineRegisterInfo &MRI) const {
3482 Register DstReg = I.getOperand(0).getReg();
3483 LLT DstTy = MRI.getType(DstReg);
3484 Register SrcReg = I.getOperand(2).getReg();
3485 Register Src2Reg = I.getOperand(3).getReg();
3486 LLT SrcTy = MRI.getType(SrcReg);
3487
3488 unsigned SrcEltSize = SrcTy.getElementType().getSizeInBits();
3489 unsigned NumElts = DstTy.getNumElements();
3490
3491 // First index is element size, 0 == 8b, 1 == 16b, 2 == 32b, 3 == 64b
3492 // Second index is num elts, 0 == v2, 1 == v4, 2 == v8, 3 == v16
3493 // Third index is cc opcode:
3494 // 0 == eq
3495 // 1 == ugt
3496 // 2 == uge
3497 // 3 == ult
3498 // 4 == ule
3499 // 5 == sgt
3500 // 6 == sge
3501 // 7 == slt
3502 // 8 == sle
3503 // ne is done by negating 'eq' result.
3504
3505 // This table below assumes that for some comparisons the operands will be
3506 // commuted.
3507 // ult op == commute + ugt op
3508 // ule op == commute + uge op
3509 // slt op == commute + sgt op
3510 // sle op == commute + sge op
3511 unsigned PredIdx = 0;
3512 bool SwapOperands = false;
3513 CmpInst::Predicate Pred = (CmpInst::Predicate)I.getOperand(1).getPredicate();
3514 switch (Pred) {
3515 case CmpInst::ICMP_NE:
3516 case CmpInst::ICMP_EQ:
3517 PredIdx = 0;
3518 break;
3519 case CmpInst::ICMP_UGT:
3520 PredIdx = 1;
3521 break;
3522 case CmpInst::ICMP_UGE:
3523 PredIdx = 2;
3524 break;
3525 case CmpInst::ICMP_ULT:
3526 PredIdx = 3;
3527 SwapOperands = true;
3528 break;
3529 case CmpInst::ICMP_ULE:
3530 PredIdx = 4;
3531 SwapOperands = true;
3532 break;
3533 case CmpInst::ICMP_SGT:
3534 PredIdx = 5;
3535 break;
3536 case CmpInst::ICMP_SGE:
3537 PredIdx = 6;
3538 break;
3539 case CmpInst::ICMP_SLT:
3540 PredIdx = 7;
3541 SwapOperands = true;
3542 break;
3543 case CmpInst::ICMP_SLE:
3544 PredIdx = 8;
3545 SwapOperands = true;
3546 break;
3547 default:
3548 llvm_unreachable("Unhandled icmp predicate")::llvm::llvm_unreachable_internal("Unhandled icmp predicate",
"/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3548)
;
3549 return false;
3550 }
3551
3552 // This table obviously should be tablegen'd when we have our GISel native
3553 // tablegen selector.
3554
3555 static const unsigned OpcTable[4][4][9] = {
3556 {
3557 {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3558 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3559 0 /* invalid */},
3560 {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3561 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3562 0 /* invalid */},
3563 {AArch64::CMEQv8i8, AArch64::CMHIv8i8, AArch64::CMHSv8i8,
3564 AArch64::CMHIv8i8, AArch64::CMHSv8i8, AArch64::CMGTv8i8,
3565 AArch64::CMGEv8i8, AArch64::CMGTv8i8, AArch64::CMGEv8i8},
3566 {AArch64::CMEQv16i8, AArch64::CMHIv16i8, AArch64::CMHSv16i8,
3567 AArch64::CMHIv16i8, AArch64::CMHSv16i8, AArch64::CMGTv16i8,
3568 AArch64::CMGEv16i8, AArch64::CMGTv16i8, AArch64::CMGEv16i8}
3569 },
3570 {
3571 {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3572 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3573 0 /* invalid */},
3574 {AArch64::CMEQv4i16, AArch64::CMHIv4i16, AArch64::CMHSv4i16,
3575 AArch64::CMHIv4i16, AArch64::CMHSv4i16, AArch64::CMGTv4i16,
3576 AArch64::CMGEv4i16, AArch64::CMGTv4i16, AArch64::CMGEv4i16},
3577 {AArch64::CMEQv8i16, AArch64::CMHIv8i16, AArch64::CMHSv8i16,
3578 AArch64::CMHIv8i16, AArch64::CMHSv8i16, AArch64::CMGTv8i16,
3579 AArch64::CMGEv8i16, AArch64::CMGTv8i16, AArch64::CMGEv8i16},
3580 {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3581 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3582 0 /* invalid */}
3583 },
3584 {
3585 {AArch64::CMEQv2i32, AArch64::CMHIv2i32, AArch64::CMHSv2i32,
3586 AArch64::CMHIv2i32, AArch64::CMHSv2i32, AArch64::CMGTv2i32,
3587 AArch64::CMGEv2i32, AArch64::CMGTv2i32, AArch64::CMGEv2i32},
3588 {AArch64::CMEQv4i32, AArch64::CMHIv4i32, AArch64::CMHSv4i32,
3589 AArch64::CMHIv4i32, AArch64::CMHSv4i32, AArch64::CMGTv4i32,
3590 AArch64::CMGEv4i32, AArch64::CMGTv4i32, AArch64::CMGEv4i32},
3591 {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3592 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3593 0 /* invalid */},
3594 {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3595 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3596 0 /* invalid */}
3597 },
3598 {
3599 {AArch64::CMEQv2i64, AArch64::CMHIv2i64, AArch64::CMHSv2i64,
3600 AArch64::CMHIv2i64, AArch64::CMHSv2i64, AArch64::CMGTv2i64,
3601 AArch64::CMGEv2i64, AArch64::CMGTv2i64, AArch64::CMGEv2i64},
3602 {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3603 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3604 0 /* invalid */},
3605 {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3606 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3607 0 /* invalid */},
3608 {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3609 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
3610 0 /* invalid */}
3611 },
3612 };
3613 unsigned EltIdx = Log2_32(SrcEltSize / 8);
3614 unsigned NumEltsIdx = Log2_32(NumElts / 2);
3615 unsigned Opc = OpcTable[EltIdx][NumEltsIdx][PredIdx];
3616 if (!Opc) {
3617 LLVM_DEBUG(dbgs() << "Could not map G_ICMP to cmp opcode")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not map G_ICMP to cmp opcode"
; } } while (false)
;
3618 return false;
3619 }
3620
3621 const RegisterBank &VecRB = *RBI.getRegBank(SrcReg, MRI, TRI);
3622 const TargetRegisterClass *SrcRC =
3623 getRegClassForTypeOnBank(SrcTy, VecRB, RBI, true);
3624 if (!SrcRC) {
3625 LLVM_DEBUG(dbgs() << "Could not determine source register class.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not determine source register class.\n"
; } } while (false)
;
3626 return false;
3627 }
3628
3629 unsigned NotOpc = Pred == ICmpInst::ICMP_NE ? AArch64::NOTv8i8 : 0;
3630 if (SrcTy.getSizeInBits() == 128)
3631 NotOpc = NotOpc ? AArch64::NOTv16i8 : 0;
3632
3633 if (SwapOperands)
3634 std::swap(SrcReg, Src2Reg);
3635
3636 MachineIRBuilder MIB(I);
3637 auto Cmp = MIB.buildInstr(Opc, {SrcRC}, {SrcReg, Src2Reg});
3638 constrainSelectedInstRegOperands(*Cmp, TII, TRI, RBI);
3639
3640 // Invert if we had a 'ne' cc.
3641 if (NotOpc) {
3642 Cmp = MIB.buildInstr(NotOpc, {DstReg}, {Cmp});
3643 constrainSelectedInstRegOperands(*Cmp, TII, TRI, RBI);
3644 } else {
3645 MIB.buildCopy(DstReg, Cmp.getReg(0));
3646 }
3647 RBI.constrainGenericRegister(DstReg, *SrcRC, MRI);
3648 I.eraseFromParent();
3649 return true;
3650}
3651
3652MachineInstr *AArch64InstructionSelector::emitScalarToVector(
3653 unsigned EltSize, const TargetRegisterClass *DstRC, Register Scalar,
3654 MachineIRBuilder &MIRBuilder) const {
3655 auto Undef = MIRBuilder.buildInstr(TargetOpcode::IMPLICIT_DEF, {DstRC}, {});
3656
3657 auto BuildFn = [&](unsigned SubregIndex) {
3658 auto Ins =
3659 MIRBuilder
3660 .buildInstr(TargetOpcode::INSERT_SUBREG, {DstRC}, {Undef, Scalar})
3661 .addImm(SubregIndex);
3662 constrainSelectedInstRegOperands(*Undef, TII, TRI, RBI);
3663 constrainSelectedInstRegOperands(*Ins, TII, TRI, RBI);
3664 return &*Ins;
3665 };
3666
3667 switch (EltSize) {
3668 case 16:
3669 return BuildFn(AArch64::hsub);
3670 case 32:
3671 return BuildFn(AArch64::ssub);
3672 case 64:
3673 return BuildFn(AArch64::dsub);
3674 default:
3675 return nullptr;
3676 }
3677}
3678
3679bool AArch64InstructionSelector::selectMergeValues(
3680 MachineInstr &I, MachineRegisterInfo &MRI) const {
3681 assert(I.getOpcode() == TargetOpcode::G_MERGE_VALUES && "unexpected opcode")((I.getOpcode() == TargetOpcode::G_MERGE_VALUES && "unexpected opcode"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_MERGE_VALUES && \"unexpected opcode\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3681, __PRETTY_FUNCTION__))
;
3682 const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
3683 const LLT SrcTy = MRI.getType(I.getOperand(1).getReg());
3684 assert(!DstTy.isVector() && !SrcTy.isVector() && "invalid merge operation")((!DstTy.isVector() && !SrcTy.isVector() && "invalid merge operation"
) ? static_cast<void> (0) : __assert_fail ("!DstTy.isVector() && !SrcTy.isVector() && \"invalid merge operation\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3684, __PRETTY_FUNCTION__))
;
3685 const RegisterBank &RB = *RBI.getRegBank(I.getOperand(1).getReg(), MRI, TRI);
3686
3687 if (I.getNumOperands() != 3)
3688 return false;
3689
3690 // Merging 2 s64s into an s128.
3691 if (DstTy == LLT::scalar(128)) {
3692 if (SrcTy.getSizeInBits() != 64)
3693 return false;
3694 MachineIRBuilder MIB(I);
3695 Register DstReg = I.getOperand(0).getReg();
3696 Register Src1Reg = I.getOperand(1).getReg();
3697 Register Src2Reg = I.getOperand(2).getReg();
3698 auto Tmp = MIB.buildInstr(TargetOpcode::IMPLICIT_DEF, {DstTy}, {});
3699 MachineInstr *InsMI =
3700 emitLaneInsert(None, Tmp.getReg(0), Src1Reg, /* LaneIdx */ 0, RB, MIB);
3701 if (!InsMI)
3702 return false;
3703 MachineInstr *Ins2MI = emitLaneInsert(DstReg, InsMI->getOperand(0).getReg(),
3704 Src2Reg, /* LaneIdx */ 1, RB, MIB);
3705 if (!Ins2MI)
3706 return false;
3707 constrainSelectedInstRegOperands(*InsMI, TII, TRI, RBI);
3708 constrainSelectedInstRegOperands(*Ins2MI, TII, TRI, RBI);
3709 I.eraseFromParent();
3710 return true;
3711 }
3712
3713 if (RB.getID() != AArch64::GPRRegBankID)
3714 return false;
3715
3716 if (DstTy.getSizeInBits() != 64 || SrcTy.getSizeInBits() != 32)
3717 return false;
3718
3719 auto *DstRC = &AArch64::GPR64RegClass;
3720 Register SubToRegDef = MRI.createVirtualRegister(DstRC);
3721 MachineInstr &SubRegMI = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
3722 TII.get(TargetOpcode::SUBREG_TO_REG))
3723 .addDef(SubToRegDef)
3724 .addImm(0)
3725 .addUse(I.getOperand(1).getReg())
3726 .addImm(AArch64::sub_32);
3727 Register SubToRegDef2 = MRI.createVirtualRegister(DstRC);
3728 // Need to anyext the second scalar before we can use bfm
3729 MachineInstr &SubRegMI2 = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
3730 TII.get(TargetOpcode::SUBREG_TO_REG))
3731 .addDef(SubToRegDef2)
3732 .addImm(0)
3733 .addUse(I.getOperand(2).getReg())
3734 .addImm(AArch64::sub_32);
3735 MachineInstr &BFM =
3736 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::BFMXri))
3737 .addDef(I.getOperand(0).getReg())
3738 .addUse(SubToRegDef)
3739 .addUse(SubToRegDef2)
3740 .addImm(32)
3741 .addImm(31);
3742 constrainSelectedInstRegOperands(SubRegMI, TII, TRI, RBI);
3743 constrainSelectedInstRegOperands(SubRegMI2, TII, TRI, RBI);
3744 constrainSelectedInstRegOperands(BFM, TII, TRI, RBI);
3745 I.eraseFromParent();
3746 return true;
3747}
3748
3749static bool getLaneCopyOpcode(unsigned &CopyOpc, unsigned &ExtractSubReg,
3750 const unsigned EltSize) {
3751 // Choose a lane copy opcode and subregister based off of the size of the
3752 // vector's elements.
3753 switch (EltSize) {
3754 case 16:
3755 CopyOpc = AArch64::CPYi16;
3756 ExtractSubReg = AArch64::hsub;
3757 break;
3758 case 32:
3759 CopyOpc = AArch64::CPYi32;
3760 ExtractSubReg = AArch64::ssub;
3761 break;
3762 case 64:
3763 CopyOpc = AArch64::CPYi64;
3764 ExtractSubReg = AArch64::dsub;
3765 break;
3766 default:
3767 // Unknown size, bail out.
3768 LLVM_DEBUG(dbgs() << "Elt size '" << EltSize << "' unsupported.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Elt size '" << EltSize
<< "' unsupported.\n"; } } while (false)
;
3769 return false;
3770 }
3771 return true;
3772}
3773
3774MachineInstr *AArch64InstructionSelector::emitExtractVectorElt(
3775 Optional<Register> DstReg, const RegisterBank &DstRB, LLT ScalarTy,
3776 Register VecReg, unsigned LaneIdx, MachineIRBuilder &MIRBuilder) const {
3777 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
3778 unsigned CopyOpc = 0;
3779 unsigned ExtractSubReg = 0;
3780 if (!getLaneCopyOpcode(CopyOpc, ExtractSubReg, ScalarTy.getSizeInBits())) {
3781 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't determine lane copy opcode for instruction.\n"
; } } while (false)
3782 dbgs() << "Couldn't determine lane copy opcode for instruction.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't determine lane copy opcode for instruction.\n"
; } } while (false)
;
3783 return nullptr;
3784 }
3785
3786 const TargetRegisterClass *DstRC =
3787 getRegClassForTypeOnBank(ScalarTy, DstRB, RBI, true);
3788 if (!DstRC) {
3789 LLVM_DEBUG(dbgs() << "Could not determine destination register class.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not determine destination register class.\n"
; } } while (false)
;
3790 return nullptr;
3791 }
3792
3793 const RegisterBank &VecRB = *RBI.getRegBank(VecReg, MRI, TRI);
3794 const LLT &VecTy = MRI.getType(VecReg);
3795 const TargetRegisterClass *VecRC =
3796 getRegClassForTypeOnBank(VecTy, VecRB, RBI, true);
3797 if (!VecRC) {
3798 LLVM_DEBUG(dbgs() << "Could not determine source register class.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not determine source register class.\n"
; } } while (false)
;
3799 return nullptr;
3800 }
3801
3802 // The register that we're going to copy into.
3803 Register InsertReg = VecReg;
3804 if (!DstReg)
3805 DstReg = MRI.createVirtualRegister(DstRC);
3806 // If the lane index is 0, we just use a subregister COPY.
3807 if (LaneIdx == 0) {
3808 auto Copy = MIRBuilder.buildInstr(TargetOpcode::COPY, {*DstReg}, {})
3809 .addReg(VecReg, 0, ExtractSubReg);
3810 RBI.constrainGenericRegister(*DstReg, *DstRC, MRI);
3811 return &*Copy;
3812 }
3813
3814 // Lane copies require 128-bit wide registers. If we're dealing with an
3815 // unpacked vector, then we need to move up to that width. Insert an implicit
3816 // def and a subregister insert to get us there.
3817 if (VecTy.getSizeInBits() != 128) {
3818 MachineInstr *ScalarToVector = emitScalarToVector(
3819 VecTy.getSizeInBits(), &AArch64::FPR128RegClass, VecReg, MIRBuilder);
3820 if (!ScalarToVector)
3821 return nullptr;
3822 InsertReg = ScalarToVector->getOperand(0).getReg();
3823 }
3824
3825 MachineInstr *LaneCopyMI =
3826 MIRBuilder.buildInstr(CopyOpc, {*DstReg}, {InsertReg}).addImm(LaneIdx);
3827 constrainSelectedInstRegOperands(*LaneCopyMI, TII, TRI, RBI);
3828
3829 // Make sure that we actually constrain the initial copy.
3830 RBI.constrainGenericRegister(*DstReg, *DstRC, MRI);
3831 return LaneCopyMI;
3832}
3833
3834bool AArch64InstructionSelector::selectExtractElt(
3835 MachineInstr &I, MachineRegisterInfo &MRI) const {
3836 assert(I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT &&((I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT &&
"unexpected opcode!") ? static_cast<void> (0) : __assert_fail
("I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT && \"unexpected opcode!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3837, __PRETTY_FUNCTION__))
3837 "unexpected opcode!")((I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT &&
"unexpected opcode!") ? static_cast<void> (0) : __assert_fail
("I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT && \"unexpected opcode!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3837, __PRETTY_FUNCTION__))
;
3838 Register DstReg = I.getOperand(0).getReg();
3839 const LLT NarrowTy = MRI.getType(DstReg);
3840 const Register SrcReg = I.getOperand(1).getReg();
3841 const LLT WideTy = MRI.getType(SrcReg);
3842 (void)WideTy;
3843 assert(WideTy.getSizeInBits() >= NarrowTy.getSizeInBits() &&((WideTy.getSizeInBits() >= NarrowTy.getSizeInBits() &&
"source register size too small!") ? static_cast<void>
(0) : __assert_fail ("WideTy.getSizeInBits() >= NarrowTy.getSizeInBits() && \"source register size too small!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3844, __PRETTY_FUNCTION__))
3844 "source register size too small!")((WideTy.getSizeInBits() >= NarrowTy.getSizeInBits() &&
"source register size too small!") ? static_cast<void>
(0) : __assert_fail ("WideTy.getSizeInBits() >= NarrowTy.getSizeInBits() && \"source register size too small!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3844, __PRETTY_FUNCTION__))
;
3845 assert(!NarrowTy.isVector() && "cannot extract vector into vector!")((!NarrowTy.isVector() && "cannot extract vector into vector!"
) ? static_cast<void> (0) : __assert_fail ("!NarrowTy.isVector() && \"cannot extract vector into vector!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3845, __PRETTY_FUNCTION__))
;
3846
3847 // Need the lane index to determine the correct copy opcode.
3848 MachineOperand &LaneIdxOp = I.getOperand(2);
3849 assert(LaneIdxOp.isReg() && "Lane index operand was not a register?")((LaneIdxOp.isReg() && "Lane index operand was not a register?"
) ? static_cast<void> (0) : __assert_fail ("LaneIdxOp.isReg() && \"Lane index operand was not a register?\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3849, __PRETTY_FUNCTION__))
;
3850
3851 if (RBI.getRegBank(DstReg, MRI, TRI)->getID() != AArch64::FPRRegBankID) {
3852 LLVM_DEBUG(dbgs() << "Cannot extract into GPR.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Cannot extract into GPR.\n"
; } } while (false)
;
3853 return false;
3854 }
3855
3856 // Find the index to extract from.
3857 auto VRegAndVal = getConstantVRegValWithLookThrough(LaneIdxOp.getReg(), MRI);
3858 if (!VRegAndVal)
3859 return false;
3860 unsigned LaneIdx = VRegAndVal->Value.getSExtValue();
3861
3862 MachineIRBuilder MIRBuilder(I);
3863
3864 const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
3865 MachineInstr *Extract = emitExtractVectorElt(DstReg, DstRB, NarrowTy, SrcReg,
3866 LaneIdx, MIRBuilder);
3867 if (!Extract)
3868 return false;
3869
3870 I.eraseFromParent();
3871 return true;
3872}
3873
3874bool AArch64InstructionSelector::selectSplitVectorUnmerge(
3875 MachineInstr &I, MachineRegisterInfo &MRI) const {
3876 unsigned NumElts = I.getNumOperands() - 1;
3877 Register SrcReg = I.getOperand(NumElts).getReg();
3878 const LLT NarrowTy = MRI.getType(I.getOperand(0).getReg());
3879 const LLT SrcTy = MRI.getType(SrcReg);
3880
3881 assert(NarrowTy.isVector() && "Expected an unmerge into vectors")((NarrowTy.isVector() && "Expected an unmerge into vectors"
) ? static_cast<void> (0) : __assert_fail ("NarrowTy.isVector() && \"Expected an unmerge into vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3881, __PRETTY_FUNCTION__))
;
3882 if (SrcTy.getSizeInBits() > 128) {
3883 LLVM_DEBUG(dbgs() << "Unexpected vector type for vec split unmerge")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unexpected vector type for vec split unmerge"
; } } while (false)
;
3884 return false;
3885 }
3886
3887 MachineIRBuilder MIB(I);
3888
3889 // We implement a split vector operation by treating the sub-vectors as
3890 // scalars and extracting them.
3891 const RegisterBank &DstRB =
3892 *RBI.getRegBank(I.getOperand(0).getReg(), MRI, TRI);
3893 for (unsigned OpIdx = 0; OpIdx < NumElts; ++OpIdx) {
3894 Register Dst = I.getOperand(OpIdx).getReg();
3895 MachineInstr *Extract =
3896 emitExtractVectorElt(Dst, DstRB, NarrowTy, SrcReg, OpIdx, MIB);
3897 if (!Extract)
3898 return false;
3899 }
3900 I.eraseFromParent();
3901 return true;
3902}
3903
3904bool AArch64InstructionSelector::selectUnmergeValues(
3905 MachineInstr &I, MachineRegisterInfo &MRI) const {
3906 assert(I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&((I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES && "unexpected opcode"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES && \"unexpected opcode\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3907, __PRETTY_FUNCTION__))
3907 "unexpected opcode")((I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES && "unexpected opcode"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES && \"unexpected opcode\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3907, __PRETTY_FUNCTION__))
;
3908
3909 // TODO: Handle unmerging into GPRs and from scalars to scalars.
3910 if (RBI.getRegBank(I.getOperand(0).getReg(), MRI, TRI)->getID() !=
3911 AArch64::FPRRegBankID ||
3912 RBI.getRegBank(I.getOperand(1).getReg(), MRI, TRI)->getID() !=
3913 AArch64::FPRRegBankID) {
3914 LLVM_DEBUG(dbgs() << "Unmerging vector-to-gpr and scalar-to-scalar "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unmerging vector-to-gpr and scalar-to-scalar "
"currently unsupported.\n"; } } while (false)
3915 "currently unsupported.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unmerging vector-to-gpr and scalar-to-scalar "
"currently unsupported.\n"; } } while (false)
;
3916 return false;
3917 }
3918
3919 // The last operand is the vector source register, and every other operand is
3920 // a register to unpack into.
3921 unsigned NumElts = I.getNumOperands() - 1;
3922 Register SrcReg = I.getOperand(NumElts).getReg();
3923 const LLT NarrowTy = MRI.getType(I.getOperand(0).getReg());
3924 const LLT WideTy = MRI.getType(SrcReg);
3925 (void)WideTy;
3926 assert((WideTy.isVector() || WideTy.getSizeInBits() == 128) &&(((WideTy.isVector() || WideTy.getSizeInBits() == 128) &&
"can only unmerge from vector or s128 types!") ? static_cast
<void> (0) : __assert_fail ("(WideTy.isVector() || WideTy.getSizeInBits() == 128) && \"can only unmerge from vector or s128 types!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3927, __PRETTY_FUNCTION__))
3927 "can only unmerge from vector or s128 types!")(((WideTy.isVector() || WideTy.getSizeInBits() == 128) &&
"can only unmerge from vector or s128 types!") ? static_cast
<void> (0) : __assert_fail ("(WideTy.isVector() || WideTy.getSizeInBits() == 128) && \"can only unmerge from vector or s128 types!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3927, __PRETTY_FUNCTION__))
;
3928 assert(WideTy.getSizeInBits() > NarrowTy.getSizeInBits() &&((WideTy.getSizeInBits() > NarrowTy.getSizeInBits() &&
"source register size too small!") ? static_cast<void>
(0) : __assert_fail ("WideTy.getSizeInBits() > NarrowTy.getSizeInBits() && \"source register size too small!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3929, __PRETTY_FUNCTION__))
3929 "source register size too small!")((WideTy.getSizeInBits() > NarrowTy.getSizeInBits() &&
"source register size too small!") ? static_cast<void>
(0) : __assert_fail ("WideTy.getSizeInBits() > NarrowTy.getSizeInBits() && \"source register size too small!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3929, __PRETTY_FUNCTION__))
;
3930
3931 if (!NarrowTy.isScalar())
3932 return selectSplitVectorUnmerge(I, MRI);
3933
3934 MachineIRBuilder MIB(I);
3935
3936 // Choose a lane copy opcode and subregister based off of the size of the
3937 // vector's elements.
3938 unsigned CopyOpc = 0;
3939 unsigned ExtractSubReg = 0;
3940 if (!getLaneCopyOpcode(CopyOpc, ExtractSubReg, NarrowTy.getSizeInBits()))
3941 return false;
3942
3943 // Set up for the lane copies.
3944 MachineBasicBlock &MBB = *I.getParent();
3945
3946 // Stores the registers we'll be copying from.
3947 SmallVector<Register, 4> InsertRegs;
3948
3949 // We'll use the first register twice, so we only need NumElts-1 registers.
3950 unsigned NumInsertRegs = NumElts - 1;
3951
3952 // If our elements fit into exactly 128 bits, then we can copy from the source
3953 // directly. Otherwise, we need to do a bit of setup with some subregister
3954 // inserts.
3955 if (NarrowTy.getSizeInBits() * NumElts == 128) {
3956 InsertRegs = SmallVector<Register, 4>(NumInsertRegs, SrcReg);
3957 } else {
3958 // No. We have to perform subregister inserts. For each insert, create an
3959 // implicit def and a subregister insert, and save the register we create.
3960 for (unsigned Idx = 0; Idx < NumInsertRegs; ++Idx) {
3961 Register ImpDefReg = MRI.createVirtualRegister(&AArch64::FPR128RegClass);
3962 MachineInstr &ImpDefMI =
3963 *BuildMI(MBB, I, I.getDebugLoc(), TII.get(TargetOpcode::IMPLICIT_DEF),
3964 ImpDefReg);
3965
3966 // Now, create the subregister insert from SrcReg.
3967 Register InsertReg = MRI.createVirtualRegister(&AArch64::FPR128RegClass);
3968 MachineInstr &InsMI =
3969 *BuildMI(MBB, I, I.getDebugLoc(),
3970 TII.get(TargetOpcode::INSERT_SUBREG), InsertReg)
3971 .addUse(ImpDefReg)
3972 .addUse(SrcReg)
3973 .addImm(AArch64::dsub);
3974
3975 constrainSelectedInstRegOperands(ImpDefMI, TII, TRI, RBI);
3976 constrainSelectedInstRegOperands(InsMI, TII, TRI, RBI);
3977
3978 // Save the register so that we can copy from it after.
3979 InsertRegs.push_back(InsertReg);
3980 }
3981 }
3982
3983 // Now that we've created any necessary subregister inserts, we can
3984 // create the copies.
3985 //
3986 // Perform the first copy separately as a subregister copy.
3987 Register CopyTo = I.getOperand(0).getReg();
3988 auto FirstCopy = MIB.buildInstr(TargetOpcode::COPY, {CopyTo}, {})
3989 .addReg(InsertRegs[0], 0, ExtractSubReg);
3990 constrainSelectedInstRegOperands(*FirstCopy, TII, TRI, RBI);
3991
3992 // Now, perform the remaining copies as vector lane copies.
3993 unsigned LaneIdx = 1;
3994 for (Register InsReg : InsertRegs) {
3995 Register CopyTo = I.getOperand(LaneIdx).getReg();
3996 MachineInstr &CopyInst =
3997 *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CopyOpc), CopyTo)
3998 .addUse(InsReg)
3999 .addImm(LaneIdx);
4000 constrainSelectedInstRegOperands(CopyInst, TII, TRI, RBI);
4001 ++LaneIdx;
4002 }
4003
4004 // Separately constrain the first copy's destination. Because of the
4005 // limitation in constrainOperandRegClass, we can't guarantee that this will
4006 // actually be constrained. So, do it ourselves using the second operand.
4007 const TargetRegisterClass *RC =
4008 MRI.getRegClassOrNull(I.getOperand(1).getReg());
4009 if (!RC) {
4010 LLVM_DEBUG(dbgs() << "Couldn't constrain copy destination.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't constrain copy destination.\n"
; } } while (false)
;
4011 return false;
4012 }
4013
4014 RBI.constrainGenericRegister(CopyTo, *RC, MRI);
4015 I.eraseFromParent();
4016 return true;
4017}
4018
4019bool AArch64InstructionSelector::selectConcatVectors(
4020 MachineInstr &I, MachineRegisterInfo &MRI) const {
4021 assert(I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS &&((I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS && "Unexpected opcode"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS && \"Unexpected opcode\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4022, __PRETTY_FUNCTION__))
4022 "Unexpected opcode")((I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS && "Unexpected opcode"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS && \"Unexpected opcode\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4022, __PRETTY_FUNCTION__))
;
4023 Register Dst = I.getOperand(0).getReg();
4024 Register Op1 = I.getOperand(1).getReg();
4025 Register Op2 = I.getOperand(2).getReg();
4026 MachineIRBuilder MIRBuilder(I);
4027 MachineInstr *ConcatMI = emitVectorConcat(Dst, Op1, Op2, MIRBuilder);
4028 if (!ConcatMI)
4029 return false;
4030 I.eraseFromParent();
4031 return true;
4032}
4033
4034unsigned
4035AArch64InstructionSelector::emitConstantPoolEntry(const Constant *CPVal,
4036 MachineFunction &MF) const {
4037 Type *CPTy = CPVal->getType();
4038 Align Alignment = MF.getDataLayout().getPrefTypeAlign(CPTy);
4039
4040 MachineConstantPool *MCP = MF.getConstantPool();
4041 return MCP->getConstantPoolIndex(CPVal, Alignment);
4042}
4043
4044MachineInstr *AArch64InstructionSelector::emitLoadFromConstantPool(
4045 const Constant *CPVal, MachineIRBuilder &MIRBuilder) const {
4046 unsigned CPIdx = emitConstantPoolEntry(CPVal, MIRBuilder.getMF());
4047
4048 auto Adrp =
4049 MIRBuilder.buildInstr(AArch64::ADRP, {&AArch64::GPR64RegClass}, {})
4050 .addConstantPoolIndex(CPIdx, 0, AArch64II::MO_PAGE);
4051
4052 MachineInstr *LoadMI = nullptr;
4053 switch (MIRBuilder.getDataLayout().getTypeStoreSize(CPVal->getType())) {
4054 case 16:
4055 LoadMI =
4056 &*MIRBuilder
4057 .buildInstr(AArch64::LDRQui, {&AArch64::FPR128RegClass}, {Adrp})
4058 .addConstantPoolIndex(CPIdx, 0,
4059 AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
4060 break;
4061 case 8:
4062 LoadMI =
4063 &*MIRBuilder
4064 .buildInstr(AArch64::LDRDui, {&AArch64::FPR64RegClass}, {Adrp})
4065 .addConstantPoolIndex(CPIdx, 0,
4066 AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
4067 break;
4068 case 4:
4069 LoadMI =
4070 &*MIRBuilder
4071 .buildInstr(AArch64::LDRSui, {&AArch64::FPR32RegClass}, {Adrp})
4072 .addConstantPoolIndex(CPIdx, 0,
4073 AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
4074 break;
4075 default:
4076 LLVM_DEBUG(dbgs() << "Could not load from constant pool of type "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not load from constant pool of type "
<< *CPVal->getType(); } } while (false)
4077 << *CPVal->getType())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not load from constant pool of type "
<< *CPVal->getType(); } } while (false)
;
4078 return nullptr;
4079 }
4080 constrainSelectedInstRegOperands(*Adrp, TII, TRI, RBI);
4081 constrainSelectedInstRegOperands(*LoadMI, TII, TRI, RBI);
4082 return LoadMI;
4083}
4084
4085/// Return an <Opcode, SubregIndex> pair to do an vector elt insert of a given
4086/// size and RB.
4087static std::pair<unsigned, unsigned>
4088getInsertVecEltOpInfo(const RegisterBank &RB, unsigned EltSize) {
4089 unsigned Opc, SubregIdx;
4090 if (RB.getID() == AArch64::GPRRegBankID) {
4091 if (EltSize == 16) {
4092 Opc = AArch64::INSvi16gpr;
4093 SubregIdx = AArch64::ssub;
4094 } else if (EltSize == 32) {
4095 Opc = AArch64::INSvi32gpr;
4096 SubregIdx = AArch64::ssub;
4097 } else if (EltSize == 64) {
4098 Opc = AArch64::INSvi64gpr;
4099 SubregIdx = AArch64::dsub;
4100 } else {
4101 llvm_unreachable("invalid elt size!")::llvm::llvm_unreachable_internal("invalid elt size!", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4101)
;
4102 }
4103 } else {
4104 if (EltSize == 8) {
4105 Opc = AArch64::INSvi8lane;
4106 SubregIdx = AArch64::bsub;
4107 } else if (EltSize == 16) {
4108 Opc = AArch64::INSvi16lane;
4109 SubregIdx = AArch64::hsub;
4110 } else if (EltSize == 32) {
4111 Opc = AArch64::INSvi32lane;
4112 SubregIdx = AArch64::ssub;
4113 } else if (EltSize == 64) {
4114 Opc = AArch64::INSvi64lane;
4115 SubregIdx = AArch64::dsub;
4116 } else {
4117 llvm_unreachable("invalid elt size!")::llvm::llvm_unreachable_internal("invalid elt size!", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4117)
;
4118 }
4119 }
4120 return std::make_pair(Opc, SubregIdx);
4121}
4122
4123MachineInstr *AArch64InstructionSelector::emitInstr(
4124 unsigned Opcode, std::initializer_list<llvm::DstOp> DstOps,
4125 std::initializer_list<llvm::SrcOp> SrcOps, MachineIRBuilder &MIRBuilder,
4126 const ComplexRendererFns &RenderFns) const {
4127 assert(Opcode && "Expected an opcode?")((Opcode && "Expected an opcode?") ? static_cast<void
> (0) : __assert_fail ("Opcode && \"Expected an opcode?\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4127, __PRETTY_FUNCTION__))
;
4128 assert(!isPreISelGenericOpcode(Opcode) &&((!isPreISelGenericOpcode(Opcode) && "Function should only be used to produce selected instructions!"
) ? static_cast<void> (0) : __assert_fail ("!isPreISelGenericOpcode(Opcode) && \"Function should only be used to produce selected instructions!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4129, __PRETTY_FUNCTION__))
4129 "Function should only be used to produce selected instructions!")((!isPreISelGenericOpcode(Opcode) && "Function should only be used to produce selected instructions!"
) ? static_cast<void> (0) : __assert_fail ("!isPreISelGenericOpcode(Opcode) && \"Function should only be used to produce selected instructions!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4129, __PRETTY_FUNCTION__))
;
4130 auto MI = MIRBuilder.buildInstr(Opcode, DstOps, SrcOps);
4131 if (RenderFns)
4132 for (auto &Fn : *RenderFns)
4133 Fn(MI);
4134 constrainSelectedInstRegOperands(*MI, TII, TRI, RBI);
4135 return &*MI;
4136}
4137
4138MachineInstr *AArch64InstructionSelector::emitAddSub(
4139 const std::array<std::array<unsigned, 2>, 5> &AddrModeAndSizeToOpcode,
4140 Register Dst, MachineOperand &LHS, MachineOperand &RHS,
4141 MachineIRBuilder &MIRBuilder) const {
4142 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4143 assert(LHS.isReg() && RHS.isReg() && "Expected register operands?")((LHS.isReg() && RHS.isReg() && "Expected register operands?"
) ? static_cast<void> (0) : __assert_fail ("LHS.isReg() && RHS.isReg() && \"Expected register operands?\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4143, __PRETTY_FUNCTION__))
;
2
'?' condition is true
4144 auto Ty = MRI.getType(LHS.getReg());
4145 assert(!Ty.isVector() && "Expected a scalar or pointer?")((!Ty.isVector() && "Expected a scalar or pointer?") ?
static_cast<void> (0) : __assert_fail ("!Ty.isVector() && \"Expected a scalar or pointer?\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4145, __PRETTY_FUNCTION__))
;
3
'?' condition is true
4146 unsigned Size = Ty.getSizeInBits();
4147 assert((Size == 32 || Size == 64) && "Expected a 32-bit or 64-bit type only")(((Size == 32 || Size == 64) && "Expected a 32-bit or 64-bit type only"
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Expected a 32-bit or 64-bit type only\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4147, __PRETTY_FUNCTION__))
;
4
Assuming 'Size' is not equal to 32
5
Assuming 'Size' is equal to 64
6
'?' condition is true
4148 bool Is32Bit = Size == 32;
4149
4150 // INSTRri form with positive arithmetic immediate.
4151 if (auto Fns = selectArithImmed(RHS))
7
Taking false branch
4152 return emitInstr(AddrModeAndSizeToOpcode[0][Is32Bit], {Dst}, {LHS},
4153 MIRBuilder, Fns);
4154
4155 // INSTRri form with negative arithmetic immediate.
4156 if (auto Fns = selectNegArithImmed(RHS))
8
Taking false branch
4157 return emitInstr(AddrModeAndSizeToOpcode[3][Is32Bit], {Dst}, {LHS},
4158 MIRBuilder, Fns);
4159
4160 // INSTRrx form.
4161 if (auto Fns = selectArithExtendedRegister(RHS))
9
Calling 'AArch64InstructionSelector::selectArithExtendedRegister'
4162 return emitInstr(AddrModeAndSizeToOpcode[4][Is32Bit], {Dst}, {LHS},
4163 MIRBuilder, Fns);
4164
4165 // INSTRrs form.
4166 if (auto Fns = selectShiftedRegister(RHS))
4167 return emitInstr(AddrModeAndSizeToOpcode[1][Is32Bit], {Dst}, {LHS},
4168 MIRBuilder, Fns);
4169 return emitInstr(AddrModeAndSizeToOpcode[2][Is32Bit], {Dst}, {LHS, RHS},
4170 MIRBuilder);
4171}
4172
4173MachineInstr *
4174AArch64InstructionSelector::emitADD(Register DefReg, MachineOperand &LHS,
4175 MachineOperand &RHS,
4176 MachineIRBuilder &MIRBuilder) const {
4177 const std::array<std::array<unsigned, 2>, 5> OpcTable{
4178 {{AArch64::ADDXri, AArch64::ADDWri},
4179 {AArch64::ADDXrs, AArch64::ADDWrs},
4180 {AArch64::ADDXrr, AArch64::ADDWrr},
4181 {AArch64::SUBXri, AArch64::SUBWri},
4182 {AArch64::ADDXrx, AArch64::ADDWrx}}};
4183 return emitAddSub(OpcTable, DefReg, LHS, RHS, MIRBuilder);
1
Calling 'AArch64InstructionSelector::emitAddSub'
4184}
4185
4186MachineInstr *
4187AArch64InstructionSelector::emitADDS(Register Dst, MachineOperand &LHS,
4188 MachineOperand &RHS,
4189 MachineIRBuilder &MIRBuilder) const {
4190 const std::array<std::array<unsigned, 2>, 5> OpcTable{
4191 {{AArch64::ADDSXri, AArch64::ADDSWri},
4192 {AArch64::ADDSXrs, AArch64::ADDSWrs},
4193 {AArch64::ADDSXrr, AArch64::ADDSWrr},
4194 {AArch64::SUBSXri, AArch64::SUBSWri},
4195 {AArch64::ADDSXrx, AArch64::ADDSWrx}}};
4196 return emitAddSub(OpcTable, Dst, LHS, RHS, MIRBuilder);
4197}
4198
4199MachineInstr *
4200AArch64InstructionSelector::emitSUBS(Register Dst, MachineOperand &LHS,
4201 MachineOperand &RHS,
4202 MachineIRBuilder &MIRBuilder) const {
4203 const std::array<std::array<unsigned, 2>, 5> OpcTable{
4204 {{AArch64::SUBSXri, AArch64::SUBSWri},
4205 {AArch64::SUBSXrs, AArch64::SUBSWrs},
4206 {AArch64::SUBSXrr, AArch64::SUBSWrr},
4207 {AArch64::ADDSXri, AArch64::ADDSWri},
4208 {AArch64::SUBSXrx, AArch64::SUBSWrx}}};
4209 return emitAddSub(OpcTable, Dst, LHS, RHS, MIRBuilder);
4210}
4211
4212MachineInstr *
4213AArch64InstructionSelector::emitCMN(MachineOperand &LHS, MachineOperand &RHS,
4214 MachineIRBuilder &MIRBuilder) const {
4215 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4216 bool Is32Bit = (MRI.getType(LHS.getReg()).getSizeInBits() == 32);
4217 auto RC = Is32Bit ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass;
4218 return emitADDS(MRI.createVirtualRegister(RC), LHS, RHS, MIRBuilder);
4219}
4220
4221MachineInstr *
4222AArch64InstructionSelector::emitTST(MachineOperand &LHS, MachineOperand &RHS,
4223 MachineIRBuilder &MIRBuilder) const {
4224 assert(LHS.isReg() && RHS.isReg() && "Expected register operands?")((LHS.isReg() && RHS.isReg() && "Expected register operands?"
) ? static_cast<void> (0) : __assert_fail ("LHS.isReg() && RHS.isReg() && \"Expected register operands?\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4224, __PRETTY_FUNCTION__))
;
4225 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4226 LLT Ty = MRI.getType(LHS.getReg());
4227 unsigned RegSize = Ty.getSizeInBits();
4228 bool Is32Bit = (RegSize == 32);
4229 const unsigned OpcTable[3][2] = {{AArch64::ANDSXri, AArch64::ANDSWri},
4230 {AArch64::ANDSXrs, AArch64::ANDSWrs},
4231 {AArch64::ANDSXrr, AArch64::ANDSWrr}};
4232 // ANDS needs a logical immediate for its immediate form. Check if we can
4233 // fold one in.
4234 if (auto ValAndVReg = getConstantVRegValWithLookThrough(RHS.getReg(), MRI)) {
4235 int64_t Imm = ValAndVReg->Value.getSExtValue();
4236
4237 if (AArch64_AM::isLogicalImmediate(Imm, RegSize)) {
4238 auto TstMI = MIRBuilder.buildInstr(OpcTable[0][Is32Bit], {Ty}, {LHS});
4239 TstMI.addImm(AArch64_AM::encodeLogicalImmediate(Imm, RegSize));
4240 constrainSelectedInstRegOperands(*TstMI, TII, TRI, RBI);
4241 return &*TstMI;
4242 }
4243 }
4244
4245 if (auto Fns = selectLogicalShiftedRegister(RHS))
4246 return emitInstr(OpcTable[1][Is32Bit], {Ty}, {LHS}, MIRBuilder, Fns);
4247 return emitInstr(OpcTable[2][Is32Bit], {Ty}, {LHS, RHS}, MIRBuilder);
4248}
4249
4250MachineInstr *AArch64InstructionSelector::emitIntegerCompare(
4251 MachineOperand &LHS, MachineOperand &RHS, MachineOperand &Predicate,
4252 MachineIRBuilder &MIRBuilder) const {
4253 assert(LHS.isReg() && RHS.isReg() && "Expected LHS and RHS to be registers!")((LHS.isReg() && RHS.isReg() && "Expected LHS and RHS to be registers!"
) ? static_cast<void> (0) : __assert_fail ("LHS.isReg() && RHS.isReg() && \"Expected LHS and RHS to be registers!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4253, __PRETTY_FUNCTION__))
;
4254 assert(Predicate.isPredicate() && "Expected predicate?")((Predicate.isPredicate() && "Expected predicate?") ?
static_cast<void> (0) : __assert_fail ("Predicate.isPredicate() && \"Expected predicate?\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4254, __PRETTY_FUNCTION__))
;
4255 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4256 LLT CmpTy = MRI.getType(LHS.getReg());
4257 assert(!CmpTy.isVector() && "Expected scalar or pointer")((!CmpTy.isVector() && "Expected scalar or pointer") ?
static_cast<void> (0) : __assert_fail ("!CmpTy.isVector() && \"Expected scalar or pointer\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4257, __PRETTY_FUNCTION__))
;
4258 unsigned Size = CmpTy.getSizeInBits();
4259 (void)Size;
4260 assert((Size == 32 || Size == 64) && "Expected a 32-bit or 64-bit LHS/RHS?")(((Size == 32 || Size == 64) && "Expected a 32-bit or 64-bit LHS/RHS?"
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Expected a 32-bit or 64-bit LHS/RHS?\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4260, __PRETTY_FUNCTION__))
;
4261 // Fold the compare into a cmn or tst if possible.
4262 if (auto FoldCmp = tryFoldIntegerCompare(LHS, RHS, Predicate, MIRBuilder))
4263 return FoldCmp;
4264 auto Dst = MRI.cloneVirtualRegister(LHS.getReg());
4265 return emitSUBS(Dst, LHS, RHS, MIRBuilder);
4266}
4267
4268MachineInstr *AArch64InstructionSelector::emitCSetForFCmp(
4269 Register Dst, CmpInst::Predicate Pred, MachineIRBuilder &MIRBuilder) const {
4270 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4271#ifndef NDEBUG
4272 LLT Ty = MRI.getType(Dst);
4273 assert(!Ty.isVector() && Ty.getSizeInBits() == 32 &&((!Ty.isVector() && Ty.getSizeInBits() == 32 &&
"Expected a 32-bit scalar register?") ? static_cast<void>
(0) : __assert_fail ("!Ty.isVector() && Ty.getSizeInBits() == 32 && \"Expected a 32-bit scalar register?\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4274, __PRETTY_FUNCTION__))
4274 "Expected a 32-bit scalar register?")((!Ty.isVector() && Ty.getSizeInBits() == 32 &&
"Expected a 32-bit scalar register?") ? static_cast<void>
(0) : __assert_fail ("!Ty.isVector() && Ty.getSizeInBits() == 32 && \"Expected a 32-bit scalar register?\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4274, __PRETTY_FUNCTION__))
;
4275#endif
4276 const Register ZeroReg = AArch64::WZR;
4277 auto EmitCSet = [&](Register CsetDst, AArch64CC::CondCode CC) {
4278 auto CSet =
4279 MIRBuilder.buildInstr(AArch64::CSINCWr, {CsetDst}, {ZeroReg, ZeroReg})
4280 .addImm(getInvertedCondCode(CC));
4281 constrainSelectedInstRegOperands(*CSet, TII, TRI, RBI);
4282 return &*CSet;
4283 };
4284
4285 AArch64CC::CondCode CC1, CC2;
4286 changeFCMPPredToAArch64CC(Pred, CC1, CC2);
4287 if (CC2 == AArch64CC::AL)
4288 return EmitCSet(Dst, CC1);
4289
4290 const TargetRegisterClass *RC = &AArch64::GPR32RegClass;
4291 Register Def1Reg = MRI.createVirtualRegister(RC);
4292 Register Def2Reg = MRI.createVirtualRegister(RC);
4293 EmitCSet(Def1Reg, CC1);
4294 EmitCSet(Def2Reg, CC2);
4295 auto OrMI = MIRBuilder.buildInstr(AArch64::ORRWrr, {Dst}, {Def1Reg, Def2Reg});
4296 constrainSelectedInstRegOperands(*OrMI, TII, TRI, RBI);
4297 return &*OrMI;
4298}
4299
4300MachineInstr *
4301AArch64InstructionSelector::emitFPCompare(Register LHS, Register RHS,
4302 MachineIRBuilder &MIRBuilder,
4303 Optional<CmpInst::Predicate> Pred) const {
4304 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4305 LLT Ty = MRI.getType(LHS);
4306 if (Ty.isVector())
4307 return nullptr;
4308 unsigned OpSize = Ty.getSizeInBits();
4309 if (OpSize != 32 && OpSize != 64)
4310 return nullptr;
4311
4312 // If this is a compare against +0.0, then we don't have
4313 // to explicitly materialize a constant.
4314 const ConstantFP *FPImm = getConstantFPVRegVal(RHS, MRI);
4315 bool ShouldUseImm = FPImm && (FPImm->isZero() && !FPImm->isNegative());
4316
4317 auto IsEqualityPred = [](CmpInst::Predicate P) {
4318 return P == CmpInst::FCMP_OEQ || P == CmpInst::FCMP_ONE ||
4319 P == CmpInst::FCMP_UEQ || P == CmpInst::FCMP_UNE;
4320 };
4321 if (!ShouldUseImm && Pred && IsEqualityPred(*Pred)) {
4322 // Try commutating the operands.
4323 const ConstantFP *LHSImm = getConstantFPVRegVal(LHS, MRI);
4324 if (LHSImm && (LHSImm->isZero() && !LHSImm->isNegative())) {
4325 ShouldUseImm = true;
4326 std::swap(LHS, RHS);
4327 }
4328 }
4329 unsigned CmpOpcTbl[2][2] = {{AArch64::FCMPSrr, AArch64::FCMPDrr},
4330 {AArch64::FCMPSri, AArch64::FCMPDri}};
4331 unsigned CmpOpc = CmpOpcTbl[ShouldUseImm][OpSize == 64];
4332
4333 // Partially build the compare. Decide if we need to add a use for the
4334 // third operand based off whether or not we're comparing against 0.0.
4335 auto CmpMI = MIRBuilder.buildInstr(CmpOpc).addUse(LHS);
4336 if (!ShouldUseImm)
4337 CmpMI.addUse(RHS);
4338 constrainSelectedInstRegOperands(*CmpMI, TII, TRI, RBI);
4339 return &*CmpMI;
4340}
4341
4342MachineInstr *AArch64InstructionSelector::emitVectorConcat(
4343 Optional<Register> Dst, Register Op1, Register Op2,
4344 MachineIRBuilder &MIRBuilder) const {
4345 // We implement a vector concat by:
4346 // 1. Use scalar_to_vector to insert the lower vector into the larger dest
4347 // 2. Insert the upper vector into the destination's upper element
4348 // TODO: some of this code is common with G_BUILD_VECTOR handling.
4349 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4350
4351 const LLT Op1Ty = MRI.getType(Op1);
4352 const LLT Op2Ty = MRI.getType(Op2);
4353
4354 if (Op1Ty != Op2Ty) {
4355 LLVM_DEBUG(dbgs() << "Could not do vector concat of differing vector tys")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not do vector concat of differing vector tys"
; } } while (false)
;
4356 return nullptr;
4357 }
4358 assert(Op1Ty.isVector() && "Expected a vector for vector concat")((Op1Ty.isVector() && "Expected a vector for vector concat"
) ? static_cast<void> (0) : __assert_fail ("Op1Ty.isVector() && \"Expected a vector for vector concat\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4358, __PRETTY_FUNCTION__))
;
4359
4360 if (Op1Ty.getSizeInBits() >= 128) {
4361 LLVM_DEBUG(dbgs() << "Vector concat not supported for full size vectors")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Vector concat not supported for full size vectors"
; } } while (false)
;
4362 return nullptr;
4363 }
4364
4365 // At the moment we just support 64 bit vector concats.
4366 if (Op1Ty.getSizeInBits() != 64) {
4367 LLVM_DEBUG(dbgs() << "Vector concat supported for 64b vectors")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Vector concat supported for 64b vectors"
; } } while (false)
;
4368 return nullptr;
4369 }
4370
4371 const LLT ScalarTy = LLT::scalar(Op1Ty.getSizeInBits());
4372 const RegisterBank &FPRBank = *RBI.getRegBank(Op1, MRI, TRI);
4373 const TargetRegisterClass *DstRC =
4374 getMinClassForRegBank(FPRBank, Op1Ty.getSizeInBits() * 2);
4375
4376 MachineInstr *WidenedOp1 =
4377 emitScalarToVector(ScalarTy.getSizeInBits(), DstRC, Op1, MIRBuilder);
4378 MachineInstr *WidenedOp2 =
4379 emitScalarToVector(ScalarTy.getSizeInBits(), DstRC, Op2, MIRBuilder);
4380 if (!WidenedOp1 || !WidenedOp2) {
4381 LLVM_DEBUG(dbgs() << "Could not emit a vector from scalar value")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not emit a vector from scalar value"
; } } while (false)
;
4382 return nullptr;
4383 }
4384
4385 // Now do the insert of the upper element.
4386 unsigned InsertOpc, InsSubRegIdx;
4387 std::tie(InsertOpc, InsSubRegIdx) =
4388 getInsertVecEltOpInfo(FPRBank, ScalarTy.getSizeInBits());
4389
4390 if (!Dst)
4391 Dst = MRI.createVirtualRegister(DstRC);
4392 auto InsElt =
4393 MIRBuilder
4394 .buildInstr(InsertOpc, {*Dst}, {WidenedOp1->getOperand(0).getReg()})
4395 .addImm(1) /* Lane index */
4396 .addUse(WidenedOp2->getOperand(0).getReg())
4397 .addImm(0);
4398 constrainSelectedInstRegOperands(*InsElt, TII, TRI, RBI);
4399 return &*InsElt;
4400}
4401
4402MachineInstr *
4403AArch64InstructionSelector::emitCSetForICMP(Register DefReg, unsigned Pred,
4404 MachineIRBuilder &MIRBuilder,
4405 Register SrcReg) const {
4406 // CSINC increments the result when the predicate is false. Invert it.
4407 const AArch64CC::CondCode InvCC = changeICMPPredToAArch64CC(
4408 CmpInst::getInversePredicate((CmpInst::Predicate)Pred));
4409 auto I = MIRBuilder.buildInstr(AArch64::CSINCWr, {DefReg}, {SrcReg, SrcReg})
4410 .addImm(InvCC);
4411 constrainSelectedInstRegOperands(*I, TII, TRI, RBI);
4412 return &*I;
4413}
4414
4415std::pair<MachineInstr *, AArch64CC::CondCode>
4416AArch64InstructionSelector::emitOverflowOp(unsigned Opcode, Register Dst,
4417 MachineOperand &LHS,
4418 MachineOperand &RHS,
4419 MachineIRBuilder &MIRBuilder) const {
4420 switch (Opcode) {
4421 default:
4422 llvm_unreachable("Unexpected opcode!")::llvm::llvm_unreachable_internal("Unexpected opcode!", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4422)
;
4423 case TargetOpcode::G_SADDO:
4424 return std::make_pair(emitADDS(Dst, LHS, RHS, MIRBuilder), AArch64CC::VS);
4425 case TargetOpcode::G_UADDO:
4426 return std::make_pair(emitADDS(Dst, LHS, RHS, MIRBuilder), AArch64CC::HS);
4427 case TargetOpcode::G_SSUBO:
4428 return std::make_pair(emitSUBS(Dst, LHS, RHS, MIRBuilder), AArch64CC::VS);
4429 case TargetOpcode::G_USUBO:
4430 return std::make_pair(emitSUBS(Dst, LHS, RHS, MIRBuilder), AArch64CC::LO);
4431 }
4432}
4433
4434bool AArch64InstructionSelector::tryOptSelect(MachineInstr &I) const {
4435 MachineIRBuilder MIB(I);
4436 MachineRegisterInfo &MRI = *MIB.getMRI();
4437 // We want to recognize this pattern:
4438 //
4439 // $z = G_FCMP pred, $x, $y
4440 // ...
4441 // $w = G_SELECT $z, $a, $b
4442 //
4443 // Where the value of $z is *only* ever used by the G_SELECT (possibly with
4444 // some copies/truncs in between.)
4445 //
4446 // If we see this, then we can emit something like this:
4447 //
4448 // fcmp $x, $y
4449 // fcsel $w, $a, $b, pred
4450 //
4451 // Rather than emitting both of the rather long sequences in the standard
4452 // G_FCMP/G_SELECT select methods.
4453
4454 // First, check if the condition is defined by a compare.
4455 MachineInstr *CondDef = MRI.getVRegDef(I.getOperand(1).getReg());
4456 while (CondDef) {
4457 // We can only fold if all of the defs have one use.
4458 Register CondDefReg = CondDef->getOperand(0).getReg();
4459 if (!MRI.hasOneNonDBGUse(CondDefReg)) {
4460 // Unless it's another select.
4461 for (const MachineInstr &UI : MRI.use_nodbg_instructions(CondDefReg)) {
4462 if (CondDef == &UI)
4463 continue;
4464 if (UI.getOpcode() != TargetOpcode::G_SELECT)
4465 return false;
4466 }
4467 }
4468
4469 // We can skip over G_TRUNC since the condition is 1-bit.
4470 // Truncating/extending can have no impact on the value.
4471 unsigned Opc = CondDef->getOpcode();
4472 if (Opc != TargetOpcode::COPY && Opc != TargetOpcode::G_TRUNC)
4473 break;
4474
4475 // Can't see past copies from physregs.
4476 if (Opc == TargetOpcode::COPY &&
4477 Register::isPhysicalRegister(CondDef->getOperand(1).getReg()))
4478 return false;
4479
4480 CondDef = MRI.getVRegDef(CondDef->getOperand(1).getReg());
4481 }
4482
4483 // Is the condition defined by a compare?
4484 if (!CondDef)
4485 return false;
4486
4487 unsigned CondOpc = CondDef->getOpcode();
4488 if (CondOpc != TargetOpcode::G_ICMP && CondOpc != TargetOpcode::G_FCMP)
4489 return false;
4490
4491 AArch64CC::CondCode CondCode;
4492 if (CondOpc == TargetOpcode::G_ICMP) {
4493 auto Pred =
4494 static_cast<CmpInst::Predicate>(CondDef->getOperand(1).getPredicate());
4495 CondCode = changeICMPPredToAArch64CC(Pred);
4496 emitIntegerCompare(CondDef->getOperand(2), CondDef->getOperand(3),
4497 CondDef->getOperand(1), MIB);
4498 } else {
4499 // Get the condition code for the select.
4500 auto Pred =
4501 static_cast<CmpInst::Predicate>(CondDef->getOperand(1).getPredicate());
4502 AArch64CC::CondCode CondCode2;
4503 changeFCMPPredToAArch64CC(Pred, CondCode, CondCode2);
4504
4505 // changeFCMPPredToAArch64CC sets CondCode2 to AL when we require two
4506 // instructions to emit the comparison.
4507 // TODO: Handle FCMP_UEQ and FCMP_ONE. After that, this check will be
4508 // unnecessary.
4509 if (CondCode2 != AArch64CC::AL)
4510 return false;
4511
4512 if (!emitFPCompare(CondDef->getOperand(2).getReg(),
4513 CondDef->getOperand(3).getReg(), MIB)) {
4514 LLVM_DEBUG(dbgs() << "Couldn't emit compare for select!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't emit compare for select!\n"
; } } while (false)
;
4515 return false;
4516 }
4517 }
4518
4519 // Emit the select.
4520 emitSelect(I.getOperand(0).getReg(), I.getOperand(2).getReg(),
4521 I.getOperand(3).getReg(), CondCode, MIB);
4522 I.eraseFromParent();
4523 return true;
4524}
4525
4526MachineInstr *AArch64InstructionSelector::tryFoldIntegerCompare(
4527 MachineOperand &LHS, MachineOperand &RHS, MachineOperand &Predicate,
4528 MachineIRBuilder &MIRBuilder) const {
4529 assert(LHS.isReg() && RHS.isReg() && Predicate.isPredicate() &&((LHS.isReg() && RHS.isReg() && Predicate.isPredicate
() && "Unexpected MachineOperand") ? static_cast<void
> (0) : __assert_fail ("LHS.isReg() && RHS.isReg() && Predicate.isPredicate() && \"Unexpected MachineOperand\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4530, __PRETTY_FUNCTION__))
4530 "Unexpected MachineOperand")((LHS.isReg() && RHS.isReg() && Predicate.isPredicate
() && "Unexpected MachineOperand") ? static_cast<void
> (0) : __assert_fail ("LHS.isReg() && RHS.isReg() && Predicate.isPredicate() && \"Unexpected MachineOperand\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4530, __PRETTY_FUNCTION__))
;
4531 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4532 // We want to find this sort of thing:
4533 // x = G_SUB 0, y
4534 // G_ICMP z, x
4535 //
4536 // In this case, we can fold the G_SUB into the G_ICMP using a CMN instead.
4537 // e.g:
4538 //
4539 // cmn z, y
4540
4541 // Helper lambda to detect the subtract followed by the compare.
4542 // Takes in the def of the LHS or RHS, and checks if it's a subtract from 0.
4543 auto IsCMN = [&](MachineInstr *DefMI, const AArch64CC::CondCode &CC) {
4544 if (!DefMI || DefMI->getOpcode() != TargetOpcode::G_SUB)
4545 return false;
4546
4547 // Need to make sure NZCV is the same at the end of the transformation.
4548 if (CC != AArch64CC::EQ && CC != AArch64CC::NE)
4549 return false;
4550
4551 // We want to match against SUBs.
4552 if (DefMI->getOpcode() != TargetOpcode::G_SUB)
4553 return false;
4554
4555 // Make sure that we're getting
4556 // x = G_SUB 0, y
4557 auto ValAndVReg =
4558 getConstantVRegValWithLookThrough(DefMI->getOperand(1).getReg(), MRI);
4559 if (!ValAndVReg || ValAndVReg->Value != 0)
4560 return false;
4561
4562 // This can safely be represented as a CMN.
4563 return true;
4564 };
4565
4566 // Check if the RHS or LHS of the G_ICMP is defined by a SUB
4567 MachineInstr *LHSDef = getDefIgnoringCopies(LHS.getReg(), MRI);
4568 MachineInstr *RHSDef = getDefIgnoringCopies(RHS.getReg(), MRI);
4569 CmpInst::Predicate P = (CmpInst::Predicate)Predicate.getPredicate();
4570 const AArch64CC::CondCode CC = changeICMPPredToAArch64CC(P);
4571
4572 // Given this:
4573 //
4574 // x = G_SUB 0, y
4575 // G_ICMP x, z
4576 //
4577 // Produce this:
4578 //
4579 // cmn y, z
4580 if (IsCMN(LHSDef, CC))
4581 return emitCMN(LHSDef->getOperand(2), RHS, MIRBuilder);
4582
4583 // Same idea here, but with the RHS of the compare instead:
4584 //
4585 // Given this:
4586 //
4587 // x = G_SUB 0, y
4588 // G_ICMP z, x
4589 //
4590 // Produce this:
4591 //
4592 // cmn z, y
4593 if (IsCMN(RHSDef, CC))
4594 return emitCMN(LHS, RHSDef->getOperand(2), MIRBuilder);
4595
4596 // Given this:
4597 //
4598 // z = G_AND x, y
4599 // G_ICMP z, 0
4600 //
4601 // Produce this if the compare is signed:
4602 //
4603 // tst x, y
4604 if (!CmpInst::isUnsigned(P) && LHSDef &&
4605 LHSDef->getOpcode() == TargetOpcode::G_AND) {
4606 // Make sure that the RHS is 0.
4607 auto ValAndVReg = getConstantVRegValWithLookThrough(RHS.getReg(), MRI);
4608 if (!ValAndVReg || ValAndVReg->Value != 0)
4609 return nullptr;
4610
4611 return emitTST(LHSDef->getOperand(1),
4612 LHSDef->getOperand(2), MIRBuilder);
4613 }
4614
4615 return nullptr;
4616}
4617
4618bool AArch64InstructionSelector::selectShuffleVector(
4619 MachineInstr &I, MachineRegisterInfo &MRI) const {
4620 const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
4621 Register Src1Reg = I.getOperand(1).getReg();
4622 const LLT Src1Ty = MRI.getType(Src1Reg);
4623 Register Src2Reg = I.getOperand(2).getReg();
4624 const LLT Src2Ty = MRI.getType(Src2Reg);
4625 ArrayRef<int> Mask = I.getOperand(3).getShuffleMask();
4626
4627 MachineBasicBlock &MBB = *I.getParent();
4628 MachineFunction &MF = *MBB.getParent();
4629 LLVMContext &Ctx = MF.getFunction().getContext();
4630
4631 // G_SHUFFLE_VECTOR is weird in that the source operands can be scalars, if
4632 // it's originated from a <1 x T> type. Those should have been lowered into
4633 // G_BUILD_VECTOR earlier.
4634 if (!Src1Ty.isVector() || !Src2Ty.isVector()) {
4635 LLVM_DEBUG(dbgs() << "Could not select a \"scalar\" G_SHUFFLE_VECTOR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not select a \"scalar\" G_SHUFFLE_VECTOR\n"
; } } while (false)
;
4636 return false;
4637 }
4638
4639 unsigned BytesPerElt = DstTy.getElementType().getSizeInBits() / 8;
4640
4641 SmallVector<Constant *, 64> CstIdxs;
4642 for (int Val : Mask) {
4643 // For now, any undef indexes we'll just assume to be 0. This should be
4644 // optimized in future, e.g. to select DUP etc.
4645 Val = Val < 0 ? 0 : Val;
4646 for (unsigned Byte = 0; Byte < BytesPerElt; ++Byte) {
4647 unsigned Offset = Byte + Val * BytesPerElt;
4648 CstIdxs.emplace_back(ConstantInt::get(Type::getInt8Ty(Ctx), Offset));
4649 }
4650 }
4651
4652 MachineIRBuilder MIRBuilder(I);
4653
4654 // Use a constant pool to load the index vector for TBL.
4655 Constant *CPVal = ConstantVector::get(CstIdxs);
4656 MachineInstr *IndexLoad = emitLoadFromConstantPool(CPVal, MIRBuilder);
4657 if (!IndexLoad) {
4658 LLVM_DEBUG(dbgs() << "Could not load from a constant pool")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not load from a constant pool"
; } } while (false)
;
4659 return false;
4660 }
4661
4662 if (DstTy.getSizeInBits() != 128) {
4663 assert(DstTy.getSizeInBits() == 64 && "Unexpected shuffle result ty")((DstTy.getSizeInBits() == 64 && "Unexpected shuffle result ty"
) ? static_cast<void> (0) : __assert_fail ("DstTy.getSizeInBits() == 64 && \"Unexpected shuffle result ty\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4663, __PRETTY_FUNCTION__))
;
4664 // This case can be done with TBL1.
4665 MachineInstr *Concat = emitVectorConcat(None, Src1Reg, Src2Reg, MIRBuilder);
4666 if (!Concat) {
4667 LLVM_DEBUG(dbgs() << "Could not do vector concat for tbl1")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not do vector concat for tbl1"
; } } while (false)
;
4668 return false;
4669 }
4670
4671 // The constant pool load will be 64 bits, so need to convert to FPR128 reg.
4672 IndexLoad =
4673 emitScalarToVector(64, &AArch64::FPR128RegClass,
4674 IndexLoad->getOperand(0).getReg(), MIRBuilder);
4675
4676 auto TBL1 = MIRBuilder.buildInstr(
4677 AArch64::TBLv16i8One, {&AArch64::FPR128RegClass},
4678 {Concat->getOperand(0).getReg(), IndexLoad->getOperand(0).getReg()});
4679 constrainSelectedInstRegOperands(*TBL1, TII, TRI, RBI);
4680
4681 auto Copy =
4682 MIRBuilder
4683 .buildInstr(TargetOpcode::COPY, {I.getOperand(0).getReg()}, {})
4684 .addReg(TBL1.getReg(0), 0, AArch64::dsub);
4685 RBI.constrainGenericRegister(Copy.getReg(0), AArch64::FPR64RegClass, MRI);
4686 I.eraseFromParent();
4687 return true;
4688 }
4689
4690 // For TBL2 we need to emit a REG_SEQUENCE to tie together two consecutive
4691 // Q registers for regalloc.
4692 auto RegSeq = MIRBuilder
4693 .buildInstr(TargetOpcode::REG_SEQUENCE,
4694 {&AArch64::QQRegClass}, {Src1Reg})
4695 .addImm(AArch64::qsub0)
4696 .addUse(Src2Reg)
4697 .addImm(AArch64::qsub1);
4698
4699 auto TBL2 = MIRBuilder.buildInstr(AArch64::TBLv16i8Two, {I.getOperand(0)},
4700 {RegSeq, IndexLoad->getOperand(0)});
4701 constrainSelectedInstRegOperands(*RegSeq, TII, TRI, RBI);
4702 constrainSelectedInstRegOperands(*TBL2, TII, TRI, RBI);
4703 I.eraseFromParent();
4704 return true;
4705}
4706
4707MachineInstr *AArch64InstructionSelector::emitLaneInsert(
4708 Optional<Register> DstReg, Register SrcReg, Register EltReg,
4709 unsigned LaneIdx, const RegisterBank &RB,
4710 MachineIRBuilder &MIRBuilder) const {
4711 MachineInstr *InsElt = nullptr;
4712 const TargetRegisterClass *DstRC = &AArch64::FPR128RegClass;
4713 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4714
4715 // Create a register to define with the insert if one wasn't passed in.
4716 if (!DstReg)
4717 DstReg = MRI.createVirtualRegister(DstRC);
4718
4719 unsigned EltSize = MRI.getType(EltReg).getSizeInBits();
4720 unsigned Opc = getInsertVecEltOpInfo(RB, EltSize).first;
4721
4722 if (RB.getID() == AArch64::FPRRegBankID) {
4723 auto InsSub = emitScalarToVector(EltSize, DstRC, EltReg, MIRBuilder);
4724 InsElt = MIRBuilder.buildInstr(Opc, {*DstReg}, {SrcReg})
4725 .addImm(LaneIdx)
4726 .addUse(InsSub->getOperand(0).getReg())
4727 .addImm(0);
4728 } else {
4729 InsElt = MIRBuilder.buildInstr(Opc, {*DstReg}, {SrcReg})
4730 .addImm(LaneIdx)
4731 .addUse(EltReg);
4732 }
4733
4734 constrainSelectedInstRegOperands(*InsElt, TII, TRI, RBI);
4735 return InsElt;
4736}
4737
4738bool AArch64InstructionSelector::selectInsertElt(
4739 MachineInstr &I, MachineRegisterInfo &MRI) const {
4740 assert(I.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT)((I.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT) ? static_cast
<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT"
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4740, __PRETTY_FUNCTION__))
;
4741
4742 // Get information on the destination.
4743 Register DstReg = I.getOperand(0).getReg();
4744 const LLT DstTy = MRI.getType(DstReg);
4745 unsigned VecSize = DstTy.getSizeInBits();
4746
4747 // Get information on the element we want to insert into the destination.
4748 Register EltReg = I.getOperand(2).getReg();
4749 const LLT EltTy = MRI.getType(EltReg);
4750 unsigned EltSize = EltTy.getSizeInBits();
4751 if (EltSize < 16 || EltSize > 64)
4752 return false; // Don't support all element types yet.
4753
4754 // Find the definition of the index. Bail out if it's not defined by a
4755 // G_CONSTANT.
4756 Register IdxReg = I.getOperand(3).getReg();
4757 auto VRegAndVal = getConstantVRegValWithLookThrough(IdxReg, MRI);
4758 if (!VRegAndVal)
4759 return false;
4760 unsigned LaneIdx = VRegAndVal->Value.getSExtValue();
4761
4762 // Perform the lane insert.
4763 Register SrcReg = I.getOperand(1).getReg();
4764 const RegisterBank &EltRB = *RBI.getRegBank(EltReg, MRI, TRI);
4765 MachineIRBuilder MIRBuilder(I);
4766
4767 if (VecSize < 128) {
4768 // If the vector we're inserting into is smaller than 128 bits, widen it
4769 // to 128 to do the insert.
4770 MachineInstr *ScalarToVec = emitScalarToVector(
4771 VecSize, &AArch64::FPR128RegClass, SrcReg, MIRBuilder);
4772 if (!ScalarToVec)
4773 return false;
4774 SrcReg = ScalarToVec->getOperand(0).getReg();
4775 }
4776
4777 // Create an insert into a new FPR128 register.
4778 // Note that if our vector is already 128 bits, we end up emitting an extra
4779 // register.
4780 MachineInstr *InsMI =
4781 emitLaneInsert(None, SrcReg, EltReg, LaneIdx, EltRB, MIRBuilder);
4782
4783 if (VecSize < 128) {
4784 // If we had to widen to perform the insert, then we have to demote back to
4785 // the original size to get the result we want.
4786 Register DemoteVec = InsMI->getOperand(0).getReg();
4787 const TargetRegisterClass *RC =
4788 getMinClassForRegBank(*RBI.getRegBank(DemoteVec, MRI, TRI), VecSize);
4789 if (RC != &AArch64::FPR32RegClass && RC != &AArch64::FPR64RegClass) {
4790 LLVM_DEBUG(dbgs() << "Unsupported register class!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported register class!\n"
; } } while (false)
;
4791 return false;
4792 }
4793 unsigned SubReg = 0;
4794 if (!getSubRegForClass(RC, TRI, SubReg))
4795 return false;
4796 if (SubReg != AArch64::ssub && SubReg != AArch64::dsub) {
4797 LLVM_DEBUG(dbgs() << "Unsupported destination size! (" << VecSizedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported destination size! ("
<< VecSize << "\n"; } } while (false)
4798 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported destination size! ("
<< VecSize << "\n"; } } while (false)
;
4799 return false;
4800 }
4801 MIRBuilder.buildInstr(TargetOpcode::COPY, {DstReg}, {})
4802 .addReg(DemoteVec, 0, SubReg);
4803 RBI.constrainGenericRegister(DstReg, *RC, MRI);
4804 } else {
4805 // No widening needed.
4806 InsMI->getOperand(0).setReg(DstReg);
4807 constrainSelectedInstRegOperands(*InsMI, TII, TRI, RBI);
4808 }
4809
4810 I.eraseFromParent();
4811 return true;
4812}
4813
4814bool AArch64InstructionSelector::tryOptConstantBuildVec(
4815 MachineInstr &I, LLT DstTy, MachineRegisterInfo &MRI) const {
4816 assert(I.getOpcode() == TargetOpcode::G_BUILD_VECTOR)((I.getOpcode() == TargetOpcode::G_BUILD_VECTOR) ? static_cast
<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BUILD_VECTOR"
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4816, __PRETTY_FUNCTION__))
;
4817 unsigned DstSize = DstTy.getSizeInBits();
4818 assert(DstSize <= 128 && "Unexpected build_vec type!")((DstSize <= 128 && "Unexpected build_vec type!") ?
static_cast<void> (0) : __assert_fail ("DstSize <= 128 && \"Unexpected build_vec type!\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4818, __PRETTY_FUNCTION__))
;
4819 if (DstSize < 32)
4820 return false;
4821 // Check if we're building a constant vector, in which case we want to
4822 // generate a constant pool load instead of a vector insert sequence.
4823 SmallVector<Constant *, 16> Csts;
4824 for (unsigned Idx = 1; Idx < I.getNumOperands(); ++Idx) {
4825 // Try to find G_CONSTANT or G_FCONSTANT
4826 auto *OpMI =
4827 getOpcodeDef(TargetOpcode::G_CONSTANT, I.getOperand(Idx).getReg(), MRI);
4828 if (OpMI)
4829 Csts.emplace_back(
4830 const_cast<ConstantInt *>(OpMI->getOperand(1).getCImm()));
4831 else if ((OpMI = getOpcodeDef(TargetOpcode::G_FCONSTANT,
4832 I.getOperand(Idx).getReg(), MRI)))
4833 Csts.emplace_back(
4834 const_cast<ConstantFP *>(OpMI->getOperand(1).getFPImm()));
4835 else
4836 return false;
4837 }
4838 Constant *CV = ConstantVector::get(Csts);
4839 MachineIRBuilder MIB(I);
4840 if (CV->isNullValue()) {
4841 // Until the importer can support immAllZerosV in pattern leaf nodes,
4842 // select a zero move manually here.
4843 Register DstReg = I.getOperand(0).getReg();
4844 if (DstSize == 128) {
4845 auto Mov = MIB.buildInstr(AArch64::MOVIv2d_ns, {DstReg}, {}).addImm(0);
4846 I.eraseFromParent();
4847 return constrainSelectedInstRegOperands(*Mov, TII, TRI, RBI);
4848 } else if (DstSize == 64) {
4849 auto Mov =
4850 MIB.buildInstr(AArch64::MOVIv2d_ns, {&AArch64::FPR128RegClass}, {})
4851 .addImm(0);
4852 MIB.buildInstr(TargetOpcode::COPY, {DstReg}, {})
4853 .addReg(Mov.getReg(0), 0, AArch64::dsub);
4854 I.eraseFromParent();
4855 return RBI.constrainGenericRegister(DstReg, AArch64::FPR64RegClass, MRI);
4856 }
4857 }
4858 auto *CPLoad = emitLoadFromConstantPool(CV, MIB);
4859 if (!CPLoad) {
4860 LLVM_DEBUG(dbgs() << "Could not generate cp load for build_vector")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not generate cp load for build_vector"
; } } while (false)
;
4861 return false;
4862 }
4863 MIB.buildCopy(I.getOperand(0), CPLoad->getOperand(0));
4864 RBI.constrainGenericRegister(I.getOperand(0).getReg(),
4865 *MRI.getRegClass(CPLoad->getOperand(0).getReg()),
4866 MRI);
4867 I.eraseFromParent();
4868 return true;
4869}
4870
4871bool AArch64InstructionSelector::selectBuildVector(
4872 MachineInstr &I, MachineRegisterInfo &MRI) const {
4873 assert(I.getOpcode() == TargetOpcode::G_BUILD_VECTOR)((I.getOpcode() == TargetOpcode::G_BUILD_VECTOR) ? static_cast
<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BUILD_VECTOR"
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4873, __PRETTY_FUNCTION__))
;
4874 // Until we port more of the optimized selections, for now just use a vector
4875 // insert sequence.
4876 const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
4877 const LLT EltTy = MRI.getType(I.getOperand(1).getReg());
4878 unsigned EltSize = EltTy.getSizeInBits();
4879
4880 if (tryOptConstantBuildVec(I, DstTy, MRI))
4881 return true;
4882 if (EltSize < 16 || EltSize > 64)
4883 return false; // Don't support all element types yet.
4884 const RegisterBank &RB = *RBI.getRegBank(I.getOperand(1).getReg(), MRI, TRI);
4885 MachineIRBuilder MIRBuilder(I);
4886
4887 const TargetRegisterClass *DstRC = &AArch64::FPR128RegClass;
4888 MachineInstr *ScalarToVec =
4889 emitScalarToVector(DstTy.getElementType().getSizeInBits(), DstRC,
4890 I.getOperand(1).getReg(), MIRBuilder);
4891 if (!ScalarToVec)
4892 return false;
4893
4894 Register DstVec = ScalarToVec->getOperand(0).getReg();
4895 unsigned DstSize = DstTy.getSizeInBits();
4896
4897 // Keep track of the last MI we inserted. Later on, we might be able to save
4898 // a copy using it.
4899 MachineInstr *PrevMI = nullptr;
4900 for (unsigned i = 2, e = DstSize / EltSize + 1; i < e; ++i) {
4901 // Note that if we don't do a subregister copy, we can end up making an
4902 // extra register.
4903 PrevMI = &*emitLaneInsert(None, DstVec, I.getOperand(i).getReg(), i - 1, RB,
4904 MIRBuilder);
4905 DstVec = PrevMI->getOperand(0).getReg();
4906 }
4907
4908 // If DstTy's size in bits is less than 128, then emit a subregister copy
4909 // from DstVec to the last register we've defined.
4910 if (DstSize < 128) {
4911 // Force this to be FPR using the destination vector.
4912 const TargetRegisterClass *RC =
4913 getMinClassForRegBank(*RBI.getRegBank(DstVec, MRI, TRI), DstSize);
4914 if (!RC)
4915 return false;
4916 if (RC != &AArch64::FPR32RegClass && RC != &AArch64::FPR64RegClass) {
4917 LLVM_DEBUG(dbgs() << "Unsupported register class!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported register class!\n"
; } } while (false)
;
4918 return false;
4919 }
4920
4921 unsigned SubReg = 0;
4922 if (!getSubRegForClass(RC, TRI, SubReg))
4923 return false;
4924 if (SubReg != AArch64::ssub && SubReg != AArch64::dsub) {
4925 LLVM_DEBUG(dbgs() << "Unsupported destination size! (" << DstSizedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported destination size! ("
<< DstSize << "\n"; } } while (false)
4926 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported destination size! ("
<< DstSize << "\n"; } } while (false)
;
4927 return false;
4928 }
4929
4930 Register Reg = MRI.createVirtualRegister(RC);
4931 Register DstReg = I.getOperand(0).getReg();
4932
4933 MIRBuilder.buildInstr(TargetOpcode::COPY, {DstReg}, {})
4934 .addReg(DstVec, 0, SubReg);
4935 MachineOperand &RegOp = I.getOperand(1);
4936 RegOp.setReg(Reg);
4937 RBI.constrainGenericRegister(DstReg, *RC, MRI);
4938 } else {
4939 // We don't need a subregister copy. Save a copy by re-using the
4940 // destination register on the final insert.
4941 assert(PrevMI && "PrevMI was null?")((PrevMI && "PrevMI was null?") ? static_cast<void
> (0) : __assert_fail ("PrevMI && \"PrevMI was null?\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4941, __PRETTY_FUNCTION__))
;
4942 PrevMI->getOperand(0).setReg(I.getOperand(0).getReg());
4943 constrainSelectedInstRegOperands(*PrevMI, TII, TRI, RBI);
4944 }
4945
4946 I.eraseFromParent();
4947 return true;
4948}
4949
4950/// Helper function to find an intrinsic ID on an a MachineInstr. Returns the
4951/// ID if it exists, and 0 otherwise.
4952static unsigned findIntrinsicID(MachineInstr &I) {
4953 auto IntrinOp = find_if(I.operands(), [&](const MachineOperand &Op) {
4954 return Op.isIntrinsicID();
4955 });
4956 if (IntrinOp == I.operands_end())
4957 return 0;
4958 return IntrinOp->getIntrinsicID();
4959}
4960
4961bool AArch64InstructionSelector::selectIntrinsicWithSideEffects(
4962 MachineInstr &I, MachineRegisterInfo &MRI) const {
4963 // Find the intrinsic ID.
4964 unsigned IntrinID = findIntrinsicID(I);
4965 if (!IntrinID)
4966 return false;
4967 MachineIRBuilder MIRBuilder(I);
4968
4969 // Select the instruction.
4970 switch (IntrinID) {
4971 default:
4972 return false;
4973 case Intrinsic::trap:
4974 MIRBuilder.buildInstr(AArch64::BRK, {}, {}).addImm(1);
4975 break;
4976 case Intrinsic::debugtrap:
4977 MIRBuilder.buildInstr(AArch64::BRK, {}, {}).addImm(0xF000);
4978 break;
4979 case Intrinsic::ubsantrap:
4980 MIRBuilder.buildInstr(AArch64::BRK, {}, {})
4981 .addImm(I.getOperand(1).getImm() | ('U' << 8));
4982 break;
4983 }
4984
4985 I.eraseFromParent();
4986 return true;
4987}
4988
4989bool AArch64InstructionSelector::selectIntrinsic(MachineInstr &I,
4990 MachineRegisterInfo &MRI) {
4991 unsigned IntrinID = findIntrinsicID(I);
4992 if (!IntrinID)
4993 return false;
4994 MachineIRBuilder MIRBuilder(I);
4995
4996 switch (IntrinID) {
4997 default:
4998 break;
4999 case Intrinsic::aarch64_crypto_sha1h: {
5000 Register DstReg = I.getOperand(0).getReg();
5001 Register SrcReg = I.getOperand(2).getReg();
5002
5003 // FIXME: Should this be an assert?
5004 if (MRI.getType(DstReg).getSizeInBits() != 32 ||
5005 MRI.getType(SrcReg).getSizeInBits() != 32)
5006 return false;
5007
5008 // The operation has to happen on FPRs. Set up some new FPR registers for
5009 // the source and destination if they are on GPRs.
5010 if (RBI.getRegBank(SrcReg, MRI, TRI)->getID() != AArch64::FPRRegBankID) {
5011 SrcReg = MRI.createVirtualRegister(&AArch64::FPR32RegClass);
5012 MIRBuilder.buildCopy({SrcReg}, {I.getOperand(2)});
5013
5014 // Make sure the copy ends up getting constrained properly.
5015 RBI.constrainGenericRegister(I.getOperand(2).getReg(),
5016 AArch64::GPR32RegClass, MRI);
5017 }
5018
5019 if (RBI.getRegBank(DstReg, MRI, TRI)->getID() != AArch64::FPRRegBankID)
5020 DstReg = MRI.createVirtualRegister(&AArch64::FPR32RegClass);
5021
5022 // Actually insert the instruction.
5023 auto SHA1Inst = MIRBuilder.buildInstr(AArch64::SHA1Hrr, {DstReg}, {SrcReg});
5024 constrainSelectedInstRegOperands(*SHA1Inst, TII, TRI, RBI);
5025
5026 // Did we create a new register for the destination?
5027 if (DstReg != I.getOperand(0).getReg()) {
5028 // Yep. Copy the result of the instruction back into the original
5029 // destination.
5030 MIRBuilder.buildCopy({I.getOperand(0)}, {DstReg});
5031 RBI.constrainGenericRegister(I.getOperand(0).getReg(),
5032 AArch64::GPR32RegClass, MRI);
5033 }
5034
5035 I.eraseFromParent();
5036 return true;
5037 }
5038 case Intrinsic::frameaddress:
5039 case Intrinsic::returnaddress: {
5040 MachineFunction &MF = *I.getParent()->getParent();
5041 MachineFrameInfo &MFI = MF.getFrameInfo();
5042
5043 unsigned Depth = I.getOperand(2).getImm();
5044 Register DstReg = I.getOperand(0).getReg();
5045 RBI.constrainGenericRegister(DstReg, AArch64::GPR64RegClass, MRI);
5046
5047 if (Depth == 0 && IntrinID == Intrinsic::returnaddress) {
5048 if (!MFReturnAddr) {
5049 // Insert the copy from LR/X30 into the entry block, before it can be
5050 // clobbered by anything.
5051 MFI.setReturnAddressIsTaken(true);
5052 MFReturnAddr = getFunctionLiveInPhysReg(MF, TII, AArch64::LR,
5053 AArch64::GPR64RegClass);
5054 }
5055
5056 if (STI.hasPAuth()) {
5057 MIRBuilder.buildInstr(AArch64::XPACI, {DstReg}, {MFReturnAddr});
5058 } else {
5059 MIRBuilder.buildCopy({Register(AArch64::LR)}, {MFReturnAddr});
5060 MIRBuilder.buildInstr(AArch64::XPACLRI);
5061 MIRBuilder.buildCopy({DstReg}, {Register(AArch64::LR)});
5062 }
5063
5064 I.eraseFromParent();
5065 return true;
5066 }
5067
5068 MFI.setFrameAddressIsTaken(true);
5069 Register FrameAddr(AArch64::FP);
5070 while (Depth--) {
5071 Register NextFrame = MRI.createVirtualRegister(&AArch64::GPR64spRegClass);
5072 auto Ldr =
5073 MIRBuilder.buildInstr(AArch64::LDRXui, {NextFrame}, {FrameAddr})
5074 .addImm(0);
5075 constrainSelectedInstRegOperands(*Ldr, TII, TRI, RBI);
5076 FrameAddr = NextFrame;
5077 }
5078
5079 if (IntrinID == Intrinsic::frameaddress)
5080 MIRBuilder.buildCopy({DstReg}, {FrameAddr});
5081 else {
5082 MFI.setReturnAddressIsTaken(true);
5083
5084 if (STI.hasPAuth()) {
5085 Register TmpReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
5086 MIRBuilder.buildInstr(AArch64::LDRXui, {TmpReg}, {FrameAddr}).addImm(1);
5087 MIRBuilder.buildInstr(AArch64::XPACI, {DstReg}, {TmpReg});
5088 } else {
5089 MIRBuilder.buildInstr(AArch64::LDRXui, {Register(AArch64::LR)}, {FrameAddr}).addImm(1);
5090 MIRBuilder.buildInstr(AArch64::XPACLRI);
5091 MIRBuilder.buildCopy({DstReg}, {Register(AArch64::LR)});
5092 }
5093 }
5094
5095 I.eraseFromParent();
5096 return true;
5097 }
5098 }
5099 return false;
5100}
5101
5102InstructionSelector::ComplexRendererFns
5103AArch64InstructionSelector::selectShiftA_32(const MachineOperand &Root) const {
5104 auto MaybeImmed = getImmedFromMO(Root);
5105 if (MaybeImmed == None || *MaybeImmed > 31)
5106 return None;
5107 uint64_t Enc = (32 - *MaybeImmed) & 0x1f;
5108 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}};
5109}
5110
5111InstructionSelector::ComplexRendererFns
5112AArch64InstructionSelector::selectShiftB_32(const MachineOperand &Root) const {
5113 auto MaybeImmed = getImmedFromMO(Root);
5114 if (MaybeImmed == None || *MaybeImmed > 31)
5115 return None;
5116 uint64_t Enc = 31 - *MaybeImmed;
5117 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}};
5118}
5119
5120InstructionSelector::ComplexRendererFns
5121AArch64InstructionSelector::selectShiftA_64(const MachineOperand &Root) const {
5122 auto MaybeImmed = getImmedFromMO(Root);
5123 if (MaybeImmed == None || *MaybeImmed > 63)
5124 return None;
5125 uint64_t Enc = (64 - *MaybeImmed) & 0x3f;
5126 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}};
5127}
5128
5129InstructionSelector::ComplexRendererFns
5130AArch64InstructionSelector::selectShiftB_64(const MachineOperand &Root) const {
5131 auto MaybeImmed = getImmedFromMO(Root);
5132 if (MaybeImmed == None || *MaybeImmed > 63)
5133 return None;
5134 uint64_t Enc = 63 - *MaybeImmed;
5135 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}};
5136}
5137
5138/// Helper to select an immediate value that can be represented as a 12-bit
5139/// value shifted left by either 0 or 12. If it is possible to do so, return
5140/// the immediate and shift value. If not, return None.
5141///
5142/// Used by selectArithImmed and selectNegArithImmed.
5143InstructionSelector::ComplexRendererFns
5144AArch64InstructionSelector::select12BitValueWithLeftShift(
5145 uint64_t Immed) const {
5146 unsigned ShiftAmt;
5147 if (Immed >> 12 == 0) {
5148 ShiftAmt = 0;
5149 } else if ((Immed & 0xfff) == 0 && Immed >> 24 == 0) {
5150 ShiftAmt = 12;
5151 Immed = Immed >> 12;
5152 } else
5153 return None;
5154
5155 unsigned ShVal = AArch64_AM::getShifterImm(AArch64_AM::LSL, ShiftAmt);
5156 return {{
5157 [=](MachineInstrBuilder &MIB) { MIB.addImm(Immed); },
5158 [=](MachineInstrBuilder &MIB) { MIB.addImm(ShVal); },
5159 }};
5160}
5161
5162/// SelectArithImmed - Select an immediate value that can be represented as
5163/// a 12-bit value shifted left by either 0 or 12. If so, return true with
5164/// Val set to the 12-bit value and Shift set to the shifter operand.
5165InstructionSelector::ComplexRendererFns
5166AArch64InstructionSelector::selectArithImmed(MachineOperand &Root) const {
5167 // This function is called from the addsub_shifted_imm ComplexPattern,
5168 // which lists [imm] as the list of opcode it's interested in, however
5169 // we still need to check whether the operand is actually an immediate
5170 // here because the ComplexPattern opcode list is only used in
5171 // root-level opcode matching.
5172 auto MaybeImmed = getImmedFromMO(Root);
5173 if (MaybeImmed == None)
5174 return None;
5175 return select12BitValueWithLeftShift(*MaybeImmed);
5176}
5177
5178/// SelectNegArithImmed - As above, but negates the value before trying to
5179/// select it.
5180InstructionSelector::ComplexRendererFns
5181AArch64InstructionSelector::selectNegArithImmed(MachineOperand &Root) const {
5182 // We need a register here, because we need to know if we have a 64 or 32
5183 // bit immediate.
5184 if (!Root.isReg())
5185 return None;
5186 auto MaybeImmed = getImmedFromMO(Root);
5187 if (MaybeImmed == None)
5188 return None;
5189 uint64_t Immed = *MaybeImmed;
5190
5191 // This negation is almost always valid, but "cmp wN, #0" and "cmn wN, #0"
5192 // have the opposite effect on the C flag, so this pattern mustn't match under
5193 // those circumstances.
5194 if (Immed == 0)
5195 return None;
5196
5197 // Check if we're dealing with a 32-bit type on the root or a 64-bit type on
5198 // the root.
5199 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
5200 if (MRI.getType(Root.getReg()).getSizeInBits() == 32)
5201 Immed = ~((uint32_t)Immed) + 1;
5202 else
5203 Immed = ~Immed + 1ULL;
5204
5205 if (Immed & 0xFFFFFFFFFF000000ULL)
5206 return None;
5207
5208 Immed &= 0xFFFFFFULL;
5209 return select12BitValueWithLeftShift(Immed);
5210}
5211
5212/// Return true if it is worth folding MI into an extended register. That is,
5213/// if it's safe to pull it into the addressing mode of a load or store as a
5214/// shift.
5215bool AArch64InstructionSelector::isWorthFoldingIntoExtendedReg(
5216 MachineInstr &MI, const MachineRegisterInfo &MRI) const {
5217 // Always fold if there is one use, or if we're optimizing for size.
5218 Register DefReg = MI.getOperand(0).getReg();
5219 if (MRI.hasOneNonDBGUse(DefReg) ||
5220 MI.getParent()->getParent()->getFunction().hasOptSize())
5221 return true;
5222
5223 // It's better to avoid folding and recomputing shifts when we don't have a
5224 // fastpath.
5225 if (!STI.hasLSLFast())
5226 return false;
5227
5228 // We have a fastpath, so folding a shift in and potentially computing it
5229 // many times may be beneficial. Check if this is only used in memory ops.
5230 // If it is, then we should fold.
5231 return all_of(MRI.use_nodbg_instructions(DefReg),
5232 [](MachineInstr &Use) { return Use.mayLoadOrStore(); });
5233}
5234
5235static bool isSignExtendShiftType(AArch64_AM::ShiftExtendType Type) {
5236 switch (Type) {
5237 case AArch64_AM::SXTB:
5238 case AArch64_AM::SXTH:
5239 case AArch64_AM::SXTW:
5240 return true;
5241 default:
5242 return false;
5243 }
5244}
5245
5246InstructionSelector::ComplexRendererFns
5247AArch64InstructionSelector::selectExtendedSHL(
5248 MachineOperand &Root, MachineOperand &Base, MachineOperand &Offset,
5249 unsigned SizeInBytes, bool WantsExt) const {
5250 assert(Base.isReg() && "Expected base to be a register operand")((Base.isReg() && "Expected base to be a register operand"
) ? static_cast<void> (0) : __assert_fail ("Base.isReg() && \"Expected base to be a register operand\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5250, __PRETTY_FUNCTION__))
;
5251 assert(Offset.isReg() && "Expected offset to be a register operand")((Offset.isReg() && "Expected offset to be a register operand"
) ? static_cast<void> (0) : __assert_fail ("Offset.isReg() && \"Expected offset to be a register operand\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5251, __PRETTY_FUNCTION__))
;
5252
5253 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
5254 MachineInstr *OffsetInst = MRI.getVRegDef(Offset.getReg());
5255 if (!OffsetInst)
5256 return None;
5257
5258 unsigned OffsetOpc = OffsetInst->getOpcode();
5259 bool LookedThroughZExt = false;
5260 if (OffsetOpc != TargetOpcode::G_SHL && OffsetOpc != TargetOpcode::G_MUL) {
5261 // Try to look through a ZEXT.
5262 if (OffsetOpc != TargetOpcode::G_ZEXT || !WantsExt)
5263 return None;
5264
5265 OffsetInst = MRI.getVRegDef(OffsetInst->getOperand(1).getReg());
5266 OffsetOpc = OffsetInst->getOpcode();
5267 LookedThroughZExt = true;
5268
5269 if (OffsetOpc != TargetOpcode::G_SHL && OffsetOpc != TargetOpcode::G_MUL)
5270 return None;
5271 }
5272 // Make sure that the memory op is a valid size.
5273 int64_t LegalShiftVal = Log2_32(SizeInBytes);
5274 if (LegalShiftVal == 0)
5275 return None;
5276 if (!isWorthFoldingIntoExtendedReg(*OffsetInst, MRI))
5277 return None;
5278
5279 // Now, try to find the specific G_CONSTANT. Start by assuming that the
5280 // register we will offset is the LHS, and the register containing the
5281 // constant is the RHS.
5282 Register OffsetReg = OffsetInst->getOperand(1).getReg();
5283 Register ConstantReg = OffsetInst->getOperand(2).getReg();
5284 auto ValAndVReg = getConstantVRegValWithLookThrough(ConstantReg, MRI);
5285 if (!ValAndVReg) {
5286 // We didn't get a constant on the RHS. If the opcode is a shift, then
5287 // we're done.
5288 if (OffsetOpc == TargetOpcode::G_SHL)
5289 return None;
5290
5291 // If we have a G_MUL, we can use either register. Try looking at the RHS.
5292 std::swap(OffsetReg, ConstantReg);
5293 ValAndVReg = getConstantVRegValWithLookThrough(ConstantReg, MRI);
5294 if (!ValAndVReg)
5295 return None;
5296 }
5297
5298 // The value must fit into 3 bits, and must be positive. Make sure that is
5299 // true.
5300 int64_t ImmVal = ValAndVReg->Value.getSExtValue();
5301
5302 // Since we're going to pull this into a shift, the constant value must be
5303 // a power of 2. If we got a multiply, then we need to check this.
5304 if (OffsetOpc == TargetOpcode::G_MUL) {
5305 if (!isPowerOf2_32(ImmVal))
5306 return None;
5307
5308 // Got a power of 2. So, the amount we'll shift is the log base-2 of that.
5309 ImmVal = Log2_32(ImmVal);
5310 }
5311
5312 if ((ImmVal & 0x7) != ImmVal)
5313 return None;
5314
5315 // We are only allowed to shift by LegalShiftVal. This shift value is built
5316 // into the instruction, so we can't just use whatever we want.
5317 if (ImmVal != LegalShiftVal)
5318 return None;
5319
5320 unsigned SignExtend = 0;
5321 if (WantsExt) {
5322 // Check if the offset is defined by an extend, unless we looked through a
5323 // G_ZEXT earlier.
5324 if (!LookedThroughZExt) {
5325 MachineInstr *ExtInst = getDefIgnoringCopies(OffsetReg, MRI);
5326 auto Ext = getExtendTypeForInst(*ExtInst, MRI, true);
5327 if (Ext == AArch64_AM::InvalidShiftExtend)
5328 return None;
5329
5330 SignExtend = isSignExtendShiftType(Ext) ? 1 : 0;
5331 // We only support SXTW for signed extension here.
5332 if (SignExtend && Ext != AArch64_AM::SXTW)
5333 return None;
5334 OffsetReg = ExtInst->getOperand(1).getReg();
5335 }
5336
5337 // Need a 32-bit wide register here.
5338 MachineIRBuilder MIB(*MRI.getVRegDef(Root.getReg()));
5339 OffsetReg = moveScalarRegClass(OffsetReg, AArch64::GPR32RegClass, MIB);
5340 }
5341
5342 // We can use the LHS of the GEP as the base, and the LHS of the shift as an
5343 // offset. Signify that we are shifting by setting the shift flag to 1.
5344 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(Base.getReg()); },
5345 [=](MachineInstrBuilder &MIB) { MIB.addUse(OffsetReg); },
5346 [=](MachineInstrBuilder &MIB) {
5347 // Need to add both immediates here to make sure that they are both
5348 // added to the instruction.
5349 MIB.addImm(SignExtend);
5350 MIB.addImm(1);
5351 }}};
5352}
5353
5354/// This is used for computing addresses like this:
5355///
5356/// ldr x1, [x2, x3, lsl #3]
5357///
5358/// Where x2 is the base register, and x3 is an offset register. The shift-left
5359/// is a constant value specific to this load instruction. That is, we'll never
5360/// see anything other than a 3 here (which corresponds to the size of the
5361/// element being loaded.)
5362InstructionSelector::ComplexRendererFns
5363AArch64InstructionSelector::selectAddrModeShiftedExtendXReg(
5364 MachineOperand &Root, unsigned SizeInBytes) const {
5365 if (!Root.isReg())
5366 return None;
5367 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
5368
5369 // We want to find something like this:
5370 //
5371 // val = G_CONSTANT LegalShiftVal
5372 // shift = G_SHL off_reg val
5373 // ptr = G_PTR_ADD base_reg shift
5374 // x = G_LOAD ptr
5375 //
5376 // And fold it into this addressing mode:
5377 //
5378 // ldr x, [base_reg, off_reg, lsl #LegalShiftVal]
5379
5380 // Check if we can find the G_PTR_ADD.
5381 MachineInstr *PtrAdd =
5382 getOpcodeDef(TargetOpcode::G_PTR_ADD, Root.getReg(), MRI);
5383 if (!PtrAdd || !isWorthFoldingIntoExtendedReg(*PtrAdd, MRI))
5384 return None;
5385
5386 // Now, try to match an opcode which will match our specific offset.
5387 // We want a G_SHL or a G_MUL.
5388 MachineInstr *OffsetInst =
5389 getDefIgnoringCopies(PtrAdd->getOperand(2).getReg(), MRI);
5390 return selectExtendedSHL(Root, PtrAdd->getOperand(1),
5391 OffsetInst->getOperand(0), SizeInBytes,
5392 /*WantsExt=*/false);
5393}
5394
5395/// This is used for computing addresses like this:
5396///
5397/// ldr x1, [x2, x3]
5398///
5399/// Where x2 is the base register, and x3 is an offset register.
5400///
5401/// When possible (or profitable) to fold a G_PTR_ADD into the address calculation,
5402/// this will do so. Otherwise, it will return None.
5403InstructionSelector::ComplexRendererFns
5404AArch64InstructionSelector::selectAddrModeRegisterOffset(
5405 MachineOperand &Root) const {
5406 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
5407
5408 // We need a GEP.
5409 MachineInstr *Gep = MRI.getVRegDef(Root.getReg());
5410 if (!Gep || Gep->getOpcode() != TargetOpcode::G_PTR_ADD)
5411 return None;
5412
5413 // If this is used more than once, let's not bother folding.
5414 // TODO: Check if they are memory ops. If they are, then we can still fold
5415 // without having to recompute anything.
5416 if (!MRI.hasOneNonDBGUse(Gep->getOperand(0).getReg()))
5417 return None;
5418
5419 // Base is the GEP's LHS, offset is its RHS.
5420 return {{[=](MachineInstrBuilder &MIB) {
5421 MIB.addUse(Gep->getOperand(1).getReg());
5422 },
5423 [=](MachineInstrBuilder &MIB) {
5424 MIB.addUse(Gep->getOperand(2).getReg());
5425 },
5426 [=](MachineInstrBuilder &MIB) {
5427 // Need to add both immediates here to make sure that they are both
5428 // added to the instruction.
5429 MIB.addImm(0);
5430 MIB.addImm(0);
5431 }}};
5432}
5433
5434/// This is intended to be equivalent to selectAddrModeXRO in
5435/// AArch64ISelDAGtoDAG. It's used for selecting X register offset loads.
5436InstructionSelector::ComplexRendererFns
5437AArch64InstructionSelector::selectAddrModeXRO(MachineOperand &Root,
5438 unsigned SizeInBytes) const {
5439 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
5440 if (!Root.isReg())
5441 return None;
5442 MachineInstr *PtrAdd =
5443 getOpcodeDef(TargetOpcode::G_PTR_ADD, Root.getReg(), MRI);
5444 if (!PtrAdd)
5445 return None;
5446
5447 // Check for an immediates which cannot be encoded in the [base + imm]
5448 // addressing mode, and can't be encoded in an add/sub. If this happens, we'll
5449 // end up with code like:
5450 //
5451 // mov x0, wide
5452 // add x1 base, x0
5453 // ldr x2, [x1, x0]
5454 //
5455 // In this situation, we can use the [base, xreg] addressing mode to save an
5456 // add/sub:
5457 //
5458 // mov x0, wide
5459 // ldr x2, [base, x0]
5460 auto ValAndVReg =
5461 getConstantVRegValWithLookThrough(PtrAdd->getOperand(2).getReg(), MRI);
5462 if (ValAndVReg) {
5463 unsigned Scale = Log2_32(SizeInBytes);
5464 int64_t ImmOff = ValAndVReg->Value.getSExtValue();
5465
5466 // Skip immediates that can be selected in the load/store addresing
5467 // mode.
5468 if (ImmOff % SizeInBytes == 0 && ImmOff >= 0 &&
5469 ImmOff < (0x1000 << Scale))
5470 return None;
5471
5472 // Helper lambda to decide whether or not it is preferable to emit an add.
5473 auto isPreferredADD = [](int64_t ImmOff) {
5474 // Constants in [0x0, 0xfff] can be encoded in an add.
5475 if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL)
5476 return true;
5477
5478 // Can it be encoded in an add lsl #12?
5479 if ((ImmOff & 0xffffffffff000fffLL) != 0x0LL)
5480 return false;
5481
5482 // It can be encoded in an add lsl #12, but we may not want to. If it is
5483 // possible to select this as a single movz, then prefer that. A single
5484 // movz is faster than an add with a shift.
5485 return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL &&
5486 (ImmOff & 0xffffffffffff0fffLL) != 0x0LL;
5487 };
5488
5489 // If the immediate can be encoded in a single add/sub, then bail out.
5490 if (isPreferredADD(ImmOff) || isPreferredADD(-ImmOff))
5491 return None;
5492 }
5493
5494 // Try to fold shifts into the addressing mode.
5495 auto AddrModeFns = selectAddrModeShiftedExtendXReg(Root, SizeInBytes);
5496 if (AddrModeFns)
5497 return AddrModeFns;
5498
5499 // If that doesn't work, see if it's possible to fold in registers from
5500 // a GEP.
5501 return selectAddrModeRegisterOffset(Root);
5502}
5503
5504/// This is used for computing addresses like this:
5505///
5506/// ldr x0, [xBase, wOffset, sxtw #LegalShiftVal]
5507///
5508/// Where we have a 64-bit base register, a 32-bit offset register, and an
5509/// extend (which may or may not be signed).
5510InstructionSelector::ComplexRendererFns
5511AArch64InstructionSelector::selectAddrModeWRO(MachineOperand &Root,
5512 unsigned SizeInBytes) const {
5513 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
5514
5515 MachineInstr *PtrAdd =
5516 getOpcodeDef(TargetOpcode::G_PTR_ADD, Root.getReg(), MRI);
5517 if (!PtrAdd || !isWorthFoldingIntoExtendedReg(*PtrAdd, MRI))
5518 return None;
5519
5520 MachineOperand &LHS = PtrAdd->getOperand(1);
5521 MachineOperand &RHS = PtrAdd->getOperand(2);
5522 MachineInstr *OffsetInst = getDefIgnoringCopies(RHS.getReg(), MRI);
5523
5524 // The first case is the same as selectAddrModeXRO, except we need an extend.
5525 // In this case, we try to find a shift and extend, and fold them into the
5526 // addressing mode.
5527 //
5528 // E.g.
5529 //
5530 // off_reg = G_Z/S/ANYEXT ext_reg
5531 // val = G_CONSTANT LegalShiftVal
5532 // shift = G_SHL off_reg val
5533 // ptr = G_PTR_ADD base_reg shift
5534 // x = G_LOAD ptr
5535 //
5536 // In this case we can get a load like this:
5537 //
5538 // ldr x0, [base_reg, ext_reg, sxtw #LegalShiftVal]
5539 auto ExtendedShl = selectExtendedSHL(Root, LHS, OffsetInst->getOperand(0),
5540 SizeInBytes, /*WantsExt=*/true);
5541 if (ExtendedShl)
5542 return ExtendedShl;
5543
5544 // There was no shift. We can try and fold a G_Z/S/ANYEXT in alone though.
5545 //
5546 // e.g.
5547 // ldr something, [base_reg, ext_reg, sxtw]
5548 if (!isWorthFoldingIntoExtendedReg(*OffsetInst, MRI))
5549 return None;
5550
5551 // Check if this is an extend. We'll get an extend type if it is.
5552 AArch64_AM::ShiftExtendType Ext =
5553 getExtendTypeForInst(*OffsetInst, MRI, /*IsLoadStore=*/true);
5554 if (Ext == AArch64_AM::InvalidShiftExtend)
5555 return None;
5556
5557 // Need a 32-bit wide register.
5558 MachineIRBuilder MIB(*PtrAdd);
5559 Register ExtReg = moveScalarRegClass(OffsetInst->getOperand(1).getReg(),
5560 AArch64::GPR32RegClass, MIB);
5561 unsigned SignExtend = Ext == AArch64_AM::SXTW;
5562
5563 // Base is LHS, offset is ExtReg.
5564 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(LHS.getReg()); },
5565 [=](MachineInstrBuilder &MIB) { MIB.addUse(ExtReg); },
5566 [=](MachineInstrBuilder &MIB) {
5567 MIB.addImm(SignExtend);
5568 MIB.addImm(0);
5569 }}};
5570}
5571
5572/// Select a "register plus unscaled signed 9-bit immediate" address. This
5573/// should only match when there is an offset that is not valid for a scaled
5574/// immediate addressing mode. The "Size" argument is the size in bytes of the
5575/// memory reference, which is needed here to know what is valid for a scaled
5576/// immediate.
5577InstructionSelector::ComplexRendererFns
5578AArch64InstructionSelector::selectAddrModeUnscaled(MachineOperand &Root,
5579 unsigned Size) const {
5580 MachineRegisterInfo &MRI =
5581 Root.getParent()->getParent()->getParent()->getRegInfo();
5582
5583 if (!Root.isReg())
5584 return None;
5585
5586 if (!isBaseWithConstantOffset(Root, MRI))
5587 return None;
5588
5589 MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());
5590 if (!RootDef)
5591 return None;
5592
5593 MachineOperand &OffImm = RootDef->getOperand(2);
5594 if (!OffImm.isReg())
5595 return None;
5596 MachineInstr *RHS = MRI.getVRegDef(OffImm.getReg());
5597 if (!RHS || RHS->getOpcode() != TargetOpcode::G_CONSTANT)
5598 return None;
5599 int64_t RHSC;
5600 MachineOperand &RHSOp1 = RHS->getOperand(1);
5601 if (!RHSOp1.isCImm() || RHSOp1.getCImm()->getBitWidth() > 64)
5602 return None;
5603 RHSC = RHSOp1.getCImm()->getSExtValue();
5604
5605 // If the offset is valid as a scaled immediate, don't match here.
5606 if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Log2_32(Size)))
5607 return None;
5608 if (RHSC >= -256 && RHSC < 256) {
5609 MachineOperand &Base = RootDef->getOperand(1);
5610 return {{
5611 [=](MachineInstrBuilder &MIB) { MIB.add(Base); },
5612 [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); },
5613 }};
5614 }
5615 return None;
5616}
5617
5618InstructionSelector::ComplexRendererFns
5619AArch64InstructionSelector::tryFoldAddLowIntoImm(MachineInstr &RootDef,
5620 unsigned Size,