LLVM 24.0.0git
AArch64InstructionSelector.cpp
Go to the documentation of this file.
1//===- AArch64InstructionSelector.cpp ----------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// AArch64.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "AArch64ExpandImm.h"
16#include "AArch64InstrInfo.h"
19#include "AArch64RegisterInfo.h"
20#include "AArch64Subtarget.h"
43#include "llvm/IR/Constants.h"
46#include "llvm/IR/IntrinsicsAArch64.h"
47#include "llvm/IR/Type.h"
48#include "llvm/Pass.h"
49#include "llvm/Support/Debug.h"
51#include <optional>
52
53#define DEBUG_TYPE "aarch64-isel"
54
55using namespace llvm;
56using namespace MIPatternMatch;
57using namespace AArch64GISelUtils;
58
59namespace llvm {
62}
63
64namespace {
65
66#define GET_GLOBALISEL_PREDICATE_BITSET
67#include "AArch64GenGlobalISel.inc"
68#undef GET_GLOBALISEL_PREDICATE_BITSET
69
70
71class AArch64InstructionSelector : public InstructionSelector {
72public:
73 AArch64InstructionSelector(const AArch64TargetMachine &TM,
74 const AArch64Subtarget &STI,
75 const AArch64RegisterBankInfo &RBI);
76
77 bool select(MachineInstr &I) override;
78 static const char *getName() { return DEBUG_TYPE; }
79
80 void setupMF(MachineFunction &MF, GISelValueTracking *VT,
81 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
82 BlockFrequencyInfo *BFI) override {
83 InstructionSelector::setupMF(MF, VT, CoverageInfo, PSI, BFI);
84 MIB.setMF(MF);
85
86 // hasFnAttribute() is expensive to call on every BRCOND selection, so
87 // cache it here for each run of the selector.
88 ProduceNonFlagSettingCondBr =
89 !MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening);
90 MFReturnAddr = Register();
91
92 processPHIs(MF);
93 }
94
95private:
96 /// tblgen-erated 'select' implementation, used as the initial selector for
97 /// the patterns that don't require complex C++.
98 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
99
100 // A lowering phase that runs before any selection attempts.
101 // Returns true if the instruction was modified.
102 bool preISelLower(MachineInstr &I);
103
104 // An early selection function that runs before the selectImpl() call.
105 bool earlySelect(MachineInstr &I);
106
107 /// Save state that is shared between select calls, call select on \p I and
108 /// then restore the saved state. This can be used to recursively call select
109 /// within a select call.
110 bool selectAndRestoreState(MachineInstr &I);
111
112 // Do some preprocessing of G_PHIs before we begin selection.
113 void processPHIs(MachineFunction &MF);
114
115 bool earlySelectSHL(MachineInstr &I, MachineRegisterInfo &MRI);
116
117 /// Eliminate same-sized cross-bank copies into stores before selectImpl().
118 bool contractCrossBankCopyIntoStore(MachineInstr &I,
120
121 bool convertPtrAddToAdd(MachineInstr &I, MachineRegisterInfo &MRI);
122
123 bool selectVaStartAAPCS(MachineInstr &I, MachineFunction &MF,
124 MachineRegisterInfo &MRI) const;
125 bool selectVaStartDarwin(MachineInstr &I, MachineFunction &MF,
126 MachineRegisterInfo &MRI) const;
127
128 ///@{
129 /// Helper functions for selectCompareBranch.
130 bool selectCompareBranchFedByFCmp(MachineInstr &I, MachineInstr &FCmp,
131 MachineIRBuilder &MIB) const;
132 bool selectCompareBranchFedByICmp(MachineInstr &I, MachineInstr &ICmp,
133 MachineIRBuilder &MIB) const;
134 bool tryOptCompareBranchFedByICmp(MachineInstr &I, MachineInstr &ICmp,
135 MachineIRBuilder &MIB) const;
136 bool tryOptAndIntoCompareBranch(MachineInstr &AndInst, bool Invert,
137 MachineBasicBlock *DstMBB,
138 MachineIRBuilder &MIB) const;
139 ///@}
140
141 bool selectCompareBranch(MachineInstr &I, MachineFunction &MF,
143
144 bool selectVectorAshrLshr(MachineInstr &I, MachineRegisterInfo &MRI);
145 bool selectVectorSHL(MachineInstr &I, MachineRegisterInfo &MRI);
146
147 // Helper to generate an equivalent of scalar_to_vector into a new register,
148 // returned via 'Dst'.
149 MachineInstr *emitScalarToVector(unsigned EltSize,
150 const TargetRegisterClass *DstRC,
151 Register Scalar,
152 MachineIRBuilder &MIRBuilder) const;
153 /// Helper to narrow vector that was widened by emitScalarToVector.
154 /// Copy lowest part of 128-bit or 64-bit vector to 64-bit or 32-bit
155 /// vector, correspondingly.
156 MachineInstr *emitNarrowVector(Register DstReg, Register SrcReg,
157 MachineIRBuilder &MIRBuilder,
158 MachineRegisterInfo &MRI) const;
159
160 /// Emit a lane insert into \p DstReg, or a new vector register if
161 /// std::nullopt is provided.
162 ///
163 /// The lane inserted into is defined by \p LaneIdx. The vector source
164 /// register is given by \p SrcReg. The register containing the element is
165 /// given by \p EltReg.
166 MachineInstr *emitLaneInsert(std::optional<Register> DstReg, Register SrcReg,
167 Register EltReg, unsigned LaneIdx,
168 const RegisterBank &RB,
169 MachineIRBuilder &MIRBuilder) const;
170
171 /// Emit a sequence of instructions representing a constant \p Imm for a
172 /// vector register \p Dst via MOVI.
173 MachineInstr *emitConstantMOVVector(DstOp Dst, APInt Imm,
175
176 /// Emit a sequence of instructions representing a constant \p CV for a
177 /// vector register \p Dst. (E.g. a MOV, or a load from a constant pool.)
178 ///
179 /// \returns the last instruction in the sequence on success, and nullptr
180 /// otherwise.
181 MachineInstr *emitConstantVector(Register Dst, Constant *CV, APInt Imm,
183
184 bool tryOptConstantBuildVec(MachineInstr &MI, LLT DstTy,
186 /// \returns true if a G_BUILD_VECTOR instruction \p MI can be selected as a
187 /// SUBREG_TO_REG.
188 bool tryOptBuildVecToSubregToReg(MachineInstr &MI, MachineRegisterInfo &MRI);
189 bool selectBuildVector(MachineInstr &I, MachineRegisterInfo &MRI);
192
193 bool selectShuffleVector(MachineInstr &I, MachineRegisterInfo &MRI);
194 bool selectExtractElt(MachineInstr &I, MachineRegisterInfo &MRI);
195 bool selectConcatVectors(MachineInstr &I, MachineRegisterInfo &MRI);
196 bool selectSplitVectorUnmerge(MachineInstr &I, MachineRegisterInfo &MRI);
197
198 /// Helper function to select vector load intrinsics like
199 /// @llvm.aarch64.neon.ld2.*, @llvm.aarch64.neon.ld4.*, etc.
200 /// \p Opc is the opcode that the selected instruction should use.
201 /// \p NumVecs is the number of vector destinations for the instruction.
202 /// \p I is the original G_INTRINSIC_W_SIDE_EFFECTS instruction.
203 bool selectVectorLoadIntrinsic(unsigned Opc, unsigned NumVecs,
204 MachineInstr &I);
205 bool selectVectorLoadLaneIntrinsic(unsigned Opc, unsigned NumVecs,
206 MachineInstr &I);
207 void selectVectorStoreIntrinsic(MachineInstr &I, unsigned NumVecs,
208 unsigned Opc);
209 bool selectVectorStoreLaneIntrinsic(MachineInstr &I, unsigned NumVecs,
210 unsigned Opc);
211 bool selectIntrinsicWithSideEffects(MachineInstr &I,
213 bool selectIntrinsic(MachineInstr &I, MachineRegisterInfo &MRI);
214 bool selectJumpTable(MachineInstr &I, MachineRegisterInfo &MRI);
215 bool selectBrJT(MachineInstr &I, MachineRegisterInfo &MRI);
216 bool selectTLSGlobalValueELF(MachineInstr &I, MachineRegisterInfo &MRI);
217 bool selectTLSLocalExecELF(const GlobalValue *GV, MachineInstr &I,
219 bool selectTLSGlobalValueMachO(MachineInstr &I, MachineRegisterInfo &MRI);
220 bool selectTLSGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI);
221 bool selectPtrAuthGlobalValue(MachineInstr &I,
222 MachineRegisterInfo &MRI) const;
223 bool selectMOPS(MachineInstr &I, MachineRegisterInfo &MRI);
224 bool selectUSMovFromExtend(MachineInstr &I, MachineRegisterInfo &MRI);
225 void SelectTable(MachineInstr &I, MachineRegisterInfo &MRI, unsigned NumVecs,
226 unsigned Opc1, unsigned Opc2, bool isExt);
227
228 bool selectIndexedExtLoad(MachineInstr &I, MachineRegisterInfo &MRI);
229 bool selectIndexedLoad(MachineInstr &I, MachineRegisterInfo &MRI);
230 bool selectIndexedStore(GIndexedStore &I, MachineRegisterInfo &MRI);
231
232 unsigned emitConstantPoolEntry(const Constant *CPVal,
233 MachineFunction &MF) const;
235 MachineIRBuilder &MIRBuilder) const;
236
237 // Emit a vector concat operation.
238 MachineInstr *emitVectorConcat(std::optional<Register> Dst, Register Op1,
239 Register Op2,
240 MachineIRBuilder &MIRBuilder) const;
241
242 // Emit an integer compare between LHS and RHS, which checks for Predicate.
243 MachineInstr *emitIntegerCompare(MachineOperand &LHS, MachineOperand &RHS,
245 MachineIRBuilder &MIRBuilder) const;
246
247 /// Emit a floating point comparison between \p LHS and \p RHS.
248 /// \p Pred if given is the intended predicate to use.
250 emitFPCompare(Register LHS, Register RHS, MachineIRBuilder &MIRBuilder,
251 std::optional<CmpInst::Predicate> = std::nullopt) const;
252
254 emitInstr(unsigned Opcode, std::initializer_list<llvm::DstOp> DstOps,
255 std::initializer_list<llvm::SrcOp> SrcOps,
256 MachineIRBuilder &MIRBuilder,
257 const ComplexRendererFns &RenderFns = std::nullopt) const;
258 /// Helper function to emit an add or sub instruction.
259 ///
260 /// \p AddrModeAndSizeToOpcode must contain each of the opcode variants above
261 /// in a specific order.
262 ///
263 /// Below is an example of the expected input to \p AddrModeAndSizeToOpcode.
264 ///
265 /// \code
266 /// const std::array<std::array<unsigned, 2>, 4> Table {
267 /// {{AArch64::ADDXri, AArch64::ADDWri},
268 /// {AArch64::ADDXrs, AArch64::ADDWrs},
269 /// {AArch64::ADDXrr, AArch64::ADDWrr},
270 /// {AArch64::SUBXri, AArch64::SUBWri},
271 /// {AArch64::ADDXrx, AArch64::ADDWrx}}};
272 /// \endcode
273 ///
274 /// Each row in the table corresponds to a different addressing mode. Each
275 /// column corresponds to a different register size.
276 ///
277 /// \attention Rows must be structured as follows:
278 /// - Row 0: The ri opcode variants
279 /// - Row 1: The rs opcode variants
280 /// - Row 2: The rr opcode variants
281 /// - Row 3: The ri opcode variants for negative immediates
282 /// - Row 4: The rx opcode variants
283 ///
284 /// \attention Columns must be structured as follows:
285 /// - Column 0: The 64-bit opcode variants
286 /// - Column 1: The 32-bit opcode variants
287 ///
288 /// \p Dst is the destination register of the binop to emit.
289 /// \p LHS is the left-hand operand of the binop to emit.
290 /// \p RHS is the right-hand operand of the binop to emit.
291 MachineInstr *emitAddSub(
292 const std::array<std::array<unsigned, 2>, 5> &AddrModeAndSizeToOpcode,
294 MachineIRBuilder &MIRBuilder) const;
295 MachineInstr *emitADD(Register DefReg, MachineOperand &LHS,
297 MachineIRBuilder &MIRBuilder) const;
299 MachineIRBuilder &MIRBuilder) const;
301 MachineIRBuilder &MIRBuilder) const;
303 MachineIRBuilder &MIRBuilder) const;
305 MachineIRBuilder &MIRBuilder) const;
307 MachineIRBuilder &MIRBuilder) const;
309 MachineIRBuilder &MIRBuilder) const;
311 MachineIRBuilder &MIRBuilder) const;
314 MachineIRBuilder &MIRBuilder) const;
315 MachineInstr *emitExtractVectorElt(std::optional<Register> DstReg,
316 const RegisterBank &DstRB, LLT ScalarTy,
317 Register VecReg, unsigned LaneIdx,
318 MachineIRBuilder &MIRBuilder) const;
319 MachineInstr *emitCSINC(Register Dst, Register Src1, Register Src2,
321 MachineIRBuilder &MIRBuilder) const;
322 /// Emit a CSet for a FP compare.
323 ///
324 /// \p Dst is expected to be a 32-bit scalar register.
325 MachineInstr *emitCSetForFCmp(Register Dst, CmpInst::Predicate Pred,
326 MachineIRBuilder &MIRBuilder) const;
327
328 /// Emit an instruction that sets NZCV to the carry-in expected by \p I.
329 /// Might elide the instruction if the previous instruction already sets NZCV
330 /// correctly.
331 MachineInstr *emitCarryIn(MachineInstr &I, Register CarryReg);
332
333 /// Emit the overflow op for \p Opcode.
334 ///
335 /// \p Opcode is expected to be an overflow op's opcode, e.g. G_UADDO,
336 /// G_USUBO, etc.
337 std::pair<MachineInstr *, AArch64CC::CondCode>
338 emitOverflowOp(unsigned Opcode, Register Dst, MachineOperand &LHS,
339 MachineOperand &RHS, MachineIRBuilder &MIRBuilder) const;
340
341 bool selectOverflowOp(MachineInstr &I, MachineRegisterInfo &MRI);
342
343 /// Emit expression as a conjunction (a series of CCMP/CFCMP ops).
344 /// In some cases this is even possible with OR operations in the expression.
346 MachineIRBuilder &MIB) const;
351 MachineIRBuilder &MIB) const;
353 bool Negate, Register CCOp,
355 MachineIRBuilder &MIB) const;
356
357 /// Emit a TB(N)Z instruction which tests \p Bit in \p TestReg.
358 /// \p IsNegative is true if the test should be "not zero".
359 /// This will also optimize the test bit instruction when possible.
360 MachineInstr *emitTestBit(Register TestReg, uint64_t Bit, bool IsNegative,
361 MachineBasicBlock *DstMBB,
362 MachineIRBuilder &MIB) const;
363
364 /// Emit a CB(N)Z instruction which branches to \p DestMBB.
365 MachineInstr *emitCBZ(Register CompareReg, bool IsNegative,
366 MachineBasicBlock *DestMBB,
367 MachineIRBuilder &MIB) const;
368
369 // Equivalent to the i32shift_a and friends from AArch64InstrInfo.td.
370 // We use these manually instead of using the importer since it doesn't
371 // support SDNodeXForm.
372 ComplexRendererFns selectShiftA_32(const MachineOperand &Root) const;
373 ComplexRendererFns selectShiftB_32(const MachineOperand &Root) const;
374 ComplexRendererFns selectShiftA_64(const MachineOperand &Root) const;
375 ComplexRendererFns selectShiftB_64(const MachineOperand &Root) const;
376
377 template <unsigned ShiftWidth>
378 ComplexRendererFns selectShiftMask(MachineOperand &Root) const;
379 ComplexRendererFns select12BitValueWithLeftShift(uint64_t Immed) const;
380 ComplexRendererFns selectArithImmed(MachineOperand &Root) const;
381 ComplexRendererFns selectNegArithImmed(MachineOperand &Root) const;
382
383 ComplexRendererFns selectAddrModeUnscaled(MachineOperand &Root,
384 unsigned Size) const;
385
386 ComplexRendererFns selectAddrModeUnscaled8(MachineOperand &Root) const {
387 return selectAddrModeUnscaled(Root, 1);
388 }
389 ComplexRendererFns selectAddrModeUnscaled16(MachineOperand &Root) const {
390 return selectAddrModeUnscaled(Root, 2);
391 }
392 ComplexRendererFns selectAddrModeUnscaled32(MachineOperand &Root) const {
393 return selectAddrModeUnscaled(Root, 4);
394 }
395 ComplexRendererFns selectAddrModeUnscaled64(MachineOperand &Root) const {
396 return selectAddrModeUnscaled(Root, 8);
397 }
398 ComplexRendererFns selectAddrModeUnscaled128(MachineOperand &Root) const {
399 return selectAddrModeUnscaled(Root, 16);
400 }
401
402 /// Helper to try to fold in a GISEL_ADD_LOW into an immediate, to be used
403 /// from complex pattern matchers like selectAddrModeIndexed().
404 ComplexRendererFns tryFoldAddLowIntoImm(MachineInstr &RootDef, unsigned Size,
405 MachineRegisterInfo &MRI) const;
406
407 ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root,
408 unsigned Size) const;
409 template <int Width>
410 ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root) const {
411 return selectAddrModeIndexed(Root, Width / 8);
412 }
413
414 std::optional<bool>
415 isWorthFoldingIntoAddrMode(const MachineInstr &MI,
416 const MachineRegisterInfo &MRI) const;
417
418 bool isWorthFoldingIntoExtendedReg(const MachineInstr &MI,
420 bool IsAddrOperand) const;
421 ComplexRendererFns
422 selectAddrModeShiftedExtendXReg(MachineOperand &Root,
423 unsigned SizeInBytes) const;
424
425 /// Returns a \p ComplexRendererFns which contains a base, offset, and whether
426 /// or not a shift + extend should be folded into an addressing mode. Returns
427 /// None when this is not profitable or possible.
428 ComplexRendererFns
429 selectExtendedSHL(MachineOperand &Root, MachineOperand &Base,
430 MachineOperand &Offset, unsigned SizeInBytes,
431 bool WantsExt) const;
432 ComplexRendererFns selectAddrModeRegisterOffset(MachineOperand &Root) const;
433 ComplexRendererFns selectAddrModeXRO(MachineOperand &Root,
434 unsigned SizeInBytes) const;
435 template <int Width>
436 ComplexRendererFns selectAddrModeXRO(MachineOperand &Root) const {
437 return selectAddrModeXRO(Root, Width / 8);
438 }
439
440 ComplexRendererFns selectAddrModeWRO(MachineOperand &Root,
441 unsigned SizeInBytes) const;
442 template <int Width>
443 ComplexRendererFns selectAddrModeWRO(MachineOperand &Root) const {
444 return selectAddrModeWRO(Root, Width / 8);
445 }
446
447 ComplexRendererFns selectShiftedRegister(MachineOperand &Root,
448 bool AllowROR = false) const;
449
450 ComplexRendererFns selectArithShiftedRegister(MachineOperand &Root) const {
451 return selectShiftedRegister(Root);
452 }
453
454 ComplexRendererFns selectLogicalShiftedRegister(MachineOperand &Root) const {
455 return selectShiftedRegister(Root, true);
456 }
457
458 /// Given an extend instruction, determine the correct shift-extend type for
459 /// that instruction.
460 ///
461 /// If the instruction is going to be used in a load or store, pass
462 /// \p IsLoadStore = true.
464 getExtendTypeForInst(MachineInstr &MI, MachineRegisterInfo &MRI,
465 bool IsLoadStore = false) const;
466
467 /// Move \p Reg to \p RC if \p Reg is not already on \p RC.
468 ///
469 /// \returns Either \p Reg if no change was necessary, or the new register
470 /// created by moving \p Reg.
471 ///
472 /// Note: This uses emitCopy right now.
473 Register moveScalarRegClass(Register Reg, const TargetRegisterClass &RC,
474 MachineIRBuilder &MIB) const;
475
476 ComplexRendererFns selectArithExtendedRegister(MachineOperand &Root) const;
477
478 ComplexRendererFns selectExtractHigh(MachineOperand &Root) const;
479 template <unsigned Width>
480 ComplexRendererFns selectCVTFixedPoint(MachineOperand &Root) const;
481 template <unsigned Width>
482 ComplexRendererFns selectCVTFixedPosRecipOperand(MachineOperand &Root) const;
483 ComplexRendererFns selectCVTFixedPointBase(const MachineOperand &Root,
484 unsigned width,
485 bool isReciprocal = false) const;
486 ComplexRendererFns selectCVTFixedPointVec(MachineOperand &Root) const;
487 ComplexRendererFns
488 selectCVTFixedPosRecipOperandVec(MachineOperand &Root) const;
489 void renderFixedPointScalarXForm(MachineInstrBuilder &MIB,
490 const MachineInstr &MI, int OpIdx) const;
491 unsigned getFixedPointWidthFromOperand(const MachineOperand &Root) const;
492 void renderFixedPointXForm(MachineInstrBuilder &MIB, const MachineInstr &MI,
493 int OpIdx = -1) const;
494 void renderFixedPointRecipXForm(MachineInstrBuilder &MIB,
495 const MachineInstr &MI, int OpIdx = -1) const;
496 void renderFixedPointImm(MachineInstrBuilder &MIB, const MachineOperand &Root,
497 unsigned Width, bool isReciprocal) const;
498 void renderTruncImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
499 int OpIdx = -1) const;
500 void renderLogicalImm32(MachineInstrBuilder &MIB, const MachineInstr &I,
501 int OpIdx = -1) const;
502 void renderLogicalImm64(MachineInstrBuilder &MIB, const MachineInstr &I,
503 int OpIdx = -1) const;
504 void renderUbsanTrap(MachineInstrBuilder &MIB, const MachineInstr &MI,
505 int OpIdx) const;
506 void renderFPImm16(MachineInstrBuilder &MIB, const MachineInstr &MI,
507 int OpIdx = -1) const;
508 void renderFPImm32(MachineInstrBuilder &MIB, const MachineInstr &MI,
509 int OpIdx = -1) const;
510 void renderFPImm64(MachineInstrBuilder &MIB, const MachineInstr &MI,
511 int OpIdx = -1) const;
512
513 // Materialize a GlobalValue or BlockAddress using a movz+movk sequence.
514 void materializeLargeCMVal(MachineInstr &I, const Value *V, unsigned OpFlags);
515
516 // Optimization methods.
517 bool tryOptSelect(GSelect &Sel);
518 bool tryOptSelectConjunction(GSelect &Sel, MachineInstr &CondMI);
519 MachineInstr *tryFoldIntegerCompare(MachineOperand &LHS, MachineOperand &RHS,
521 MachineIRBuilder &MIRBuilder) const;
522
523 /// Return true if \p MI is a load or store of \p NumBytes bytes.
524 bool isLoadStoreOfNumBytes(const MachineInstr &MI, unsigned NumBytes) const;
525
526 /// Returns true if \p MI is guaranteed to have the high-half of a 64-bit
527 /// register zeroed out. In other words, the result of MI has been explicitly
528 /// zero extended.
529 bool isDef32(const MachineInstr &MI) const;
530
531 const AArch64TargetMachine &TM;
532 const AArch64Subtarget &STI;
533 const AArch64InstrInfo &TII;
535 const AArch64RegisterBankInfo &RBI;
536
537 bool ProduceNonFlagSettingCondBr = false;
538
539 // Some cached values used during selection.
540 // We use LR as a live-in register, and we keep track of it here as it can be
541 // clobbered by calls.
542 Register MFReturnAddr;
543
545
546#define GET_GLOBALISEL_PREDICATES_DECL
547#include "AArch64GenGlobalISel.inc"
548#undef GET_GLOBALISEL_PREDICATES_DECL
549
550// We declare the temporaries used by selectImpl() in the class to minimize the
551// cost of constructing placeholder values.
552#define GET_GLOBALISEL_TEMPORARIES_DECL
553#include "AArch64GenGlobalISel.inc"
554#undef GET_GLOBALISEL_TEMPORARIES_DECL
555};
556
557} // end anonymous namespace
558
559#define GET_GLOBALISEL_IMPL
560#include "AArch64GenGlobalISel.inc"
561#undef GET_GLOBALISEL_IMPL
562
563AArch64InstructionSelector::AArch64InstructionSelector(
564 const AArch64TargetMachine &TM, const AArch64Subtarget &STI,
565 const AArch64RegisterBankInfo &RBI)
566 : TM(TM), STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()),
567 RBI(RBI),
569#include "AArch64GenGlobalISel.inc"
572#include "AArch64GenGlobalISel.inc"
574{
575}
576
577// FIXME: This should be target-independent, inferred from the types declared
578// for each class in the bank.
579//
580/// Given a register bank, and a type, return the smallest register class that
581/// can represent that combination.
582static const TargetRegisterClass *
583getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB,
584 bool GetAllRegSet = false) {
585 if (RB.getID() == AArch64::GPRRegBankID) {
586 if (Ty.getSizeInBits() <= 32)
587 return GetAllRegSet ? &AArch64::GPR32allRegClass
588 : &AArch64::GPR32RegClass;
589 if (Ty.getSizeInBits() == 64)
590 return GetAllRegSet ? &AArch64::GPR64allRegClass
591 : &AArch64::GPR64RegClass;
592 if (Ty.getSizeInBits() == 128)
593 return &AArch64::XSeqPairsClassRegClass;
594 return nullptr;
595 }
596
597 if (RB.getID() == AArch64::FPRRegBankID) {
598 switch (Ty.getSizeInBits()) {
599 case 8:
600 return &AArch64::FPR8RegClass;
601 case 16:
602 return &AArch64::FPR16RegClass;
603 case 32:
604 return &AArch64::FPR32RegClass;
605 case 64:
606 return &AArch64::FPR64RegClass;
607 case 128:
608 return &AArch64::FPR128RegClass;
609 }
610 return nullptr;
611 }
612
613 return nullptr;
614}
615
616/// Given a register bank, and size in bits, return the smallest register class
617/// that can represent that combination.
618static const TargetRegisterClass *
620 bool GetAllRegSet = false) {
621 if (SizeInBits.isScalable()) {
622 assert(RB.getID() == AArch64::FPRRegBankID &&
623 "Expected FPR regbank for scalable type size");
624 return &AArch64::ZPRRegClass;
625 }
626
627 unsigned RegBankID = RB.getID();
628
629 if (RegBankID == AArch64::GPRRegBankID) {
630 assert(!SizeInBits.isScalable() && "Unexpected scalable register size");
631 if (SizeInBits <= 32)
632 return GetAllRegSet ? &AArch64::GPR32allRegClass
633 : &AArch64::GPR32RegClass;
634 if (SizeInBits == 64)
635 return GetAllRegSet ? &AArch64::GPR64allRegClass
636 : &AArch64::GPR64RegClass;
637 if (SizeInBits == 128)
638 return &AArch64::XSeqPairsClassRegClass;
639 }
640
641 if (RegBankID == AArch64::FPRRegBankID) {
642 if (SizeInBits.isScalable()) {
643 assert(SizeInBits == TypeSize::getScalable(128) &&
644 "Unexpected scalable register size");
645 return &AArch64::ZPRRegClass;
646 }
647
648 switch (SizeInBits) {
649 default:
650 return nullptr;
651 case 8:
652 return &AArch64::FPR8RegClass;
653 case 16:
654 return &AArch64::FPR16RegClass;
655 case 32:
656 return &AArch64::FPR32RegClass;
657 case 64:
658 return &AArch64::FPR64RegClass;
659 case 128:
660 return &AArch64::FPR128RegClass;
661 }
662 }
663
664 return nullptr;
665}
666
667/// Returns the correct subregister to use for a given register class.
669 const TargetRegisterInfo &TRI, unsigned &SubReg) {
670 switch (TRI.getRegSizeInBits(*RC)) {
671 case 8:
672 SubReg = AArch64::bsub;
673 break;
674 case 16:
675 SubReg = AArch64::hsub;
676 break;
677 case 32:
678 if (RC != &AArch64::FPR32RegClass)
679 SubReg = AArch64::sub_32;
680 else
681 SubReg = AArch64::ssub;
682 break;
683 case 64:
684 SubReg = AArch64::dsub;
685 break;
686 default:
688 dbgs() << "Couldn't find appropriate subregister for register class.");
689 return false;
690 }
691
692 return true;
693}
694
695/// Returns the minimum size the given register bank can hold.
696static unsigned getMinSizeForRegBank(const RegisterBank &RB) {
697 switch (RB.getID()) {
698 case AArch64::GPRRegBankID:
699 return 32;
700 case AArch64::FPRRegBankID:
701 return 8;
702 default:
703 llvm_unreachable("Tried to get minimum size for unknown register bank.");
704 }
705}
706
707/// Create a REG_SEQUENCE instruction using the registers in \p Regs.
708/// Helper function for functions like createDTuple and createQTuple.
709///
710/// \p RegClassIDs - The list of register class IDs available for some tuple of
711/// a scalar class. E.g. QQRegClassID, QQQRegClassID, QQQQRegClassID. This is
712/// expected to contain between 2 and 4 tuple classes.
713///
714/// \p SubRegs - The list of subregister classes associated with each register
715/// class ID in \p RegClassIDs. E.g., QQRegClassID should use the qsub0
716/// subregister class. The index of each subregister class is expected to
717/// correspond with the index of each register class.
718///
719/// \returns Either the destination register of REG_SEQUENCE instruction that
720/// was created, or the 0th element of \p Regs if \p Regs contains a single
721/// element.
723 const unsigned RegClassIDs[],
724 const unsigned SubRegs[], MachineIRBuilder &MIB) {
725 unsigned NumRegs = Regs.size();
726 if (NumRegs == 1)
727 return Regs[0];
728 assert(NumRegs >= 2 && NumRegs <= 4 &&
729 "Only support between two and 4 registers in a tuple!");
731 auto *DesiredClass = TRI->getRegClass(RegClassIDs[NumRegs - 2]);
732 auto RegSequence =
733 MIB.buildInstr(TargetOpcode::REG_SEQUENCE, {DesiredClass}, {});
734 for (unsigned I = 0, E = Regs.size(); I < E; ++I) {
735 RegSequence.addUse(Regs[I]);
736 RegSequence.addImm(SubRegs[I]);
737 }
738 return RegSequence.getReg(0);
739}
740
741/// Create a tuple of D-registers using the registers in \p Regs.
743 static const unsigned RegClassIDs[] = {
744 AArch64::DDRegClassID, AArch64::DDDRegClassID, AArch64::DDDDRegClassID};
745 static const unsigned SubRegs[] = {AArch64::dsub0, AArch64::dsub1,
746 AArch64::dsub2, AArch64::dsub3};
747 return createTuple(Regs, RegClassIDs, SubRegs, MIB);
748}
749
750/// Create a tuple of Q-registers using the registers in \p Regs.
752 static const unsigned RegClassIDs[] = {
753 AArch64::QQRegClassID, AArch64::QQQRegClassID, AArch64::QQQQRegClassID};
754 static const unsigned SubRegs[] = {AArch64::qsub0, AArch64::qsub1,
755 AArch64::qsub2, AArch64::qsub3};
756 return createTuple(Regs, RegClassIDs, SubRegs, MIB);
757}
758
759static std::optional<uint64_t> getImmedFromMO(const MachineOperand &Root) {
760 auto &MI = *Root.getParent();
761 auto &MBB = *MI.getParent();
762 auto &MF = *MBB.getParent();
763 auto &MRI = MF.getRegInfo();
764 uint64_t Immed;
765 if (Root.isImm())
766 Immed = Root.getImm();
767 else if (Root.isCImm())
768 Immed = Root.getCImm()->getZExtValue();
769 else if (Root.isReg()) {
770 auto ValAndVReg =
772 if (!ValAndVReg)
773 return std::nullopt;
774 Immed = ValAndVReg->Value.getSExtValue();
775 } else
776 return std::nullopt;
777 return Immed;
778}
779
780/// Select the AArch64 opcode for the basic binary operation \p GenericOpc,
781/// appropriate for the register bank \p RegBankID and of size \p OpSize.
782/// \returns \p GenericOpc if the combination is unsupported.
783static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID,
784 unsigned OpSize) {
785 if (RegBankID == AArch64::GPRRegBankID) {
786 if (OpSize == 32) {
787 switch (GenericOpc) {
788 case TargetOpcode::G_SHL:
789 return AArch64::LSLVWr;
790 case TargetOpcode::G_LSHR:
791 return AArch64::LSRVWr;
792 case TargetOpcode::G_ASHR:
793 return AArch64::ASRVWr;
794 default:
795 return GenericOpc;
796 }
797 } else if (OpSize == 64) {
798 switch (GenericOpc) {
799 case TargetOpcode::G_SHL:
800 return AArch64::LSLVXr;
801 case TargetOpcode::G_LSHR:
802 return AArch64::LSRVXr;
803 case TargetOpcode::G_ASHR:
804 return AArch64::ASRVXr;
805 default:
806 return GenericOpc;
807 }
808 }
809 }
810 return GenericOpc;
811}
812
813/// Select the AArch64 opcode for the G_LOAD or G_STORE operation \p GenericOpc,
814/// appropriate for the (value) register bank \p RegBankID and of memory access
815/// size \p OpSize. This returns the variant with the base+unsigned-immediate
816/// addressing mode (e.g., LDRXui).
817/// \returns \p GenericOpc if the combination is unsupported.
818static unsigned selectLoadStoreUIOp(unsigned GenericOpc, unsigned RegBankID,
819 unsigned OpSize) {
820 const bool isStore = GenericOpc == TargetOpcode::G_STORE;
821 switch (RegBankID) {
822 case AArch64::GPRRegBankID:
823 switch (OpSize) {
824 case 8:
825 return isStore ? AArch64::STRBBui : AArch64::LDRBBui;
826 case 16:
827 return isStore ? AArch64::STRHHui : AArch64::LDRHHui;
828 case 32:
829 return isStore ? AArch64::STRWui : AArch64::LDRWui;
830 case 64:
831 return isStore ? AArch64::STRXui : AArch64::LDRXui;
832 }
833 break;
834 case AArch64::FPRRegBankID:
835 switch (OpSize) {
836 case 8:
837 return isStore ? AArch64::STRBui : AArch64::LDRBui;
838 case 16:
839 return isStore ? AArch64::STRHui : AArch64::LDRHui;
840 case 32:
841 return isStore ? AArch64::STRSui : AArch64::LDRSui;
842 case 64:
843 return isStore ? AArch64::STRDui : AArch64::LDRDui;
844 case 128:
845 return isStore ? AArch64::STRQui : AArch64::LDRQui;
846 }
847 break;
848 }
849 return GenericOpc;
850}
851
852/// Helper function for selectCopy. Inserts a subregister copy from \p SrcReg
853/// to \p *To.
854///
855/// E.g "To = COPY SrcReg:SubReg"
857 const RegisterBankInfo &RBI, Register SrcReg,
858 const TargetRegisterClass *To, unsigned SubReg) {
859 assert(SrcReg.isValid() && "Expected a valid source register?");
860 assert(To && "Destination register class cannot be null");
861 assert(SubReg && "Expected a valid subregister");
862
863 MachineIRBuilder MIB(I);
864 auto SubRegCopy =
865 MIB.buildInstr(TargetOpcode::COPY, {To}, {}).addReg(SrcReg, {}, SubReg);
866 MachineOperand &RegOp = I.getOperand(1);
867 RegOp.setReg(SubRegCopy.getReg(0));
868
869 // It's possible that the destination register won't be constrained. Make
870 // sure that happens.
871 if (!I.getOperand(0).getReg().isPhysical())
872 RBI.constrainGenericRegister(I.getOperand(0).getReg(), *To, MRI);
873
874 return true;
875}
876
879 const RegisterBankInfo &RBI) {
880 Register DstReg = I.getOperand(0).getReg();
881 Register SrcReg = I.getOperand(1).getReg();
882 const RegisterBank &DstRegBank = *RBI.getRegBank(DstReg, MRI, TRI);
883 const RegisterBank &SrcRegBank = *RBI.getRegBank(SrcReg, MRI, TRI);
884
885 TypeSize DstRegSize = TRI.getRegSizeInBits(DstReg, MRI);
886 TypeSize SrcRegSize = TRI.getRegSizeInBits(SrcReg, MRI);
887
888 // Special casing for cross-bank copies of s1s. We can technically represent
889 // a 1-bit value with any size of register. The minimum size for a GPR is 32
890 // bits. So, we need to put the FPR on 32 bits as well.
891 //
892 // FIXME: I'm not sure if this case holds true outside of copies. If it does,
893 // then we can pull it into the helpers that get the appropriate class for a
894 // register bank. Or make a new helper that carries along some constraint
895 // information.
896 if (SrcRegBank != DstRegBank && (DstRegSize == TypeSize::getFixed(1) &&
897 SrcRegSize == TypeSize::getFixed(1)))
898 SrcRegSize = DstRegSize = TypeSize::getFixed(32);
899
900 // Find the correct register classes for the source and destination registers.
901 const TargetRegisterClass *SrcRC =
902 getMinClassForRegBank(SrcRegBank, SrcRegSize, true);
903 const TargetRegisterClass *DstRC =
904 getMinClassForRegBank(DstRegBank, DstRegSize, true);
905
906 if (!DstRC) {
907 LLVM_DEBUG(dbgs() << "Unexpected dest size "
908 << TRI.getRegSizeInBits(DstReg, MRI) << '\n');
909 return false;
910 }
911
912 if (I.getOpcode() == TargetOpcode::G_BITCAST &&
913 TRI.getRegSizeInBits(DstReg, MRI) == TypeSize::getFixed(16)) {
914 if (DstRegBank.getID() == AArch64::FPRRegBankID &&
915 SrcRegBank.getID() == AArch64::GPRRegBankID) {
916 if (!SrcReg.isPhysical() &&
917 !RBI.constrainGenericRegister(SrcReg, AArch64::GPR32RegClass, MRI))
918 return false;
919 if (!DstReg.isPhysical() &&
920 !RBI.constrainGenericRegister(DstReg, AArch64::FPR16RegClass, MRI))
921 return false;
922
923 Register FPR32 = MRI.createVirtualRegister(&AArch64::FPR32RegClass);
924 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::FMOVWSr))
925 .addDef(FPR32)
926 .addUse(SrcReg);
927 I.setDesc(TII.get(TargetOpcode::COPY));
928 I.getOperand(1).setReg(FPR32);
929 I.getOperand(1).setSubReg(AArch64::hsub);
930 return true;
931 }
932
933 if (DstRegBank.getID() == AArch64::GPRRegBankID &&
934 SrcRegBank.getID() == AArch64::FPRRegBankID) {
935 if (!SrcReg.isPhysical() &&
936 !RBI.constrainGenericRegister(SrcReg, AArch64::FPR16RegClass, MRI))
937 return false;
938 if (!DstReg.isPhysical() &&
939 !RBI.constrainGenericRegister(DstReg, AArch64::GPR32RegClass, MRI))
940 return false;
941
942 Register FPR32 = MRI.createVirtualRegister(&AArch64::FPR32RegClass);
943 BuildMI(*I.getParent(), I, I.getDebugLoc(),
944 TII.get(TargetOpcode::SUBREG_TO_REG))
945 .addDef(FPR32)
946 .addUse(SrcReg)
947 .addImm(AArch64::hsub);
948 I.setDesc(TII.get(AArch64::FMOVSWr));
949 I.getOperand(1).setReg(FPR32);
950 return true;
951 }
952 }
953
954 // Is this a copy? If so, then we may need to insert a subregister copy.
955 if (I.isCopy()) {
956 // Yes. Check if there's anything to fix up.
957 if (!SrcRC) {
958 LLVM_DEBUG(dbgs() << "Couldn't determine source register class\n");
959 return false;
960 }
961
962 const TypeSize SrcSize = TRI.getRegSizeInBits(*SrcRC);
963 const TypeSize DstSize = TRI.getRegSizeInBits(*DstRC);
964 unsigned SrcSubReg = I.getOperand(1).getSubReg();
965 unsigned SubReg;
966
967 if (SrcSubReg)
968 return RBI.constrainGenericRegister(DstReg, *DstRC, MRI);
969
970 // If the source bank doesn't support a subregister copy small enough,
971 // then we first need to copy to the destination bank.
972 if (getMinSizeForRegBank(SrcRegBank) > DstSize) {
973 const TargetRegisterClass *DstTempRC =
974 getMinClassForRegBank(DstRegBank, SrcSize, /* GetAllRegSet */ true);
975 getSubRegForClass(DstRC, TRI, SubReg);
976
977 MachineIRBuilder MIB(I);
978 auto Copy = MIB.buildCopy({DstTempRC}, {SrcReg});
979 copySubReg(I, MRI, RBI, Copy.getReg(0), DstRC, SubReg);
980 } else if (SrcSize > DstSize) {
981 // If the source register is bigger than the destination we need to
982 // perform a subregister copy.
983 const TargetRegisterClass *SubRegRC =
984 getMinClassForRegBank(SrcRegBank, DstSize, /* GetAllRegSet */ true);
985 getSubRegForClass(SubRegRC, TRI, SubReg);
986 copySubReg(I, MRI, RBI, SrcReg, DstRC, SubReg);
987 } else if (DstSize > SrcSize) {
988 // If the destination register is bigger than the source we need to do
989 // a promotion using SUBREG_TO_REG.
990 const TargetRegisterClass *PromotionRC =
991 getMinClassForRegBank(SrcRegBank, DstSize, /* GetAllRegSet */ true);
992 getSubRegForClass(SrcRC, TRI, SubReg);
993
994 Register PromoteReg = MRI.createVirtualRegister(PromotionRC);
995 BuildMI(*I.getParent(), I, I.getDebugLoc(),
996 TII.get(AArch64::SUBREG_TO_REG), PromoteReg)
997 .addUse(SrcReg)
998 .addImm(SubReg);
999 MachineOperand &RegOp = I.getOperand(1);
1000 RegOp.setReg(PromoteReg);
1001 }
1002
1003 // If the destination is a physical register, then there's nothing to
1004 // change, so we're done.
1005 if (DstReg.isPhysical())
1006 return true;
1007 }
1008
1009 // No need to constrain SrcReg. It will get constrained when we hit another
1010 // of its use or its defs. Copies do not have constraints.
1011 if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
1012 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1013 << " operand\n");
1014 return false;
1015 }
1016
1017 // If this a GPR ZEXT that we want to just reduce down into a copy.
1018 // The sizes will be mismatched with the source < 32b but that's ok.
1019 if (I.getOpcode() == TargetOpcode::G_ZEXT) {
1020 I.setDesc(TII.get(AArch64::COPY));
1021 assert(SrcRegBank.getID() == AArch64::GPRRegBankID);
1022 return selectCopy(I, TII, MRI, TRI, RBI);
1023 }
1024
1025 I.setDesc(TII.get(AArch64::COPY));
1026 return true;
1027}
1028
1030AArch64InstructionSelector::emitSelect(Register Dst, Register True,
1032 MachineIRBuilder &MIB) const {
1033 MachineRegisterInfo &MRI = *MIB.getMRI();
1034 assert(RBI.getRegBank(False, MRI, TRI)->getID() ==
1035 RBI.getRegBank(True, MRI, TRI)->getID() &&
1036 "Expected both select operands to have the same regbank?");
1037 LLT Ty = MRI.getType(True);
1038 if (Ty.isVector())
1039 return nullptr;
1040 const unsigned Size = Ty.getSizeInBits();
1041 assert((Size == 32 || Size == 64) &&
1042 "Expected 32 bit or 64 bit select only?");
1043 const bool Is32Bit = Size == 32;
1044 if (RBI.getRegBank(True, MRI, TRI)->getID() != AArch64::GPRRegBankID) {
1045 unsigned Opc = Is32Bit ? AArch64::FCSELSrrr : AArch64::FCSELDrrr;
1046 auto FCSel = MIB.buildInstr(Opc, {Dst}, {True, False}).addImm(CC);
1048 return &*FCSel;
1049 }
1050
1051 // By default, we'll try and emit a CSEL.
1052 unsigned Opc = Is32Bit ? AArch64::CSELWr : AArch64::CSELXr;
1053 bool Optimized = false;
1054 auto TryFoldBinOpIntoSelect = [&Opc, Is32Bit, &CC, &MRI,
1055 &Optimized](Register &Reg, Register &OtherReg,
1056 bool Invert) {
1057 if (Optimized)
1058 return false;
1059
1060 // Attempt to fold:
1061 //
1062 // %sub = G_SUB 0, %x
1063 // %select = G_SELECT cc, %reg, %sub
1064 //
1065 // Into:
1066 // %select = CSNEG %reg, %x, cc
1067 Register MatchReg;
1068 if (mi_match(Reg, MRI, m_Neg(m_Reg(MatchReg)))) {
1069 Opc = Is32Bit ? AArch64::CSNEGWr : AArch64::CSNEGXr;
1070 Reg = MatchReg;
1071 if (Invert) {
1073 std::swap(Reg, OtherReg);
1074 }
1075 return true;
1076 }
1077
1078 // Attempt to fold:
1079 //
1080 // %xor = G_XOR %x, -1
1081 // %select = G_SELECT cc, %reg, %xor
1082 //
1083 // Into:
1084 // %select = CSINV %reg, %x, cc
1085 if (mi_match(Reg, MRI, m_Not(m_Reg(MatchReg)))) {
1086 Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
1087 Reg = MatchReg;
1088 if (Invert) {
1090 std::swap(Reg, OtherReg);
1091 }
1092 return true;
1093 }
1094
1095 // Attempt to fold:
1096 //
1097 // %add = G_ADD %x, 1
1098 // %select = G_SELECT cc, %reg, %add
1099 //
1100 // Into:
1101 // %select = CSINC %reg, %x, cc
1102 if (mi_match(Reg, MRI,
1103 m_any_of(m_GAdd(m_Reg(MatchReg), m_SpecificICst(1)),
1104 m_GPtrAdd(m_Reg(MatchReg), m_SpecificICst(1))))) {
1105 Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
1106 Reg = MatchReg;
1107 if (Invert) {
1109 std::swap(Reg, OtherReg);
1110 }
1111 return true;
1112 }
1113
1114 return false;
1115 };
1116
1117 // Helper lambda which tries to use CSINC/CSINV for the instruction when its
1118 // true/false values are constants.
1119 // FIXME: All of these patterns already exist in tablegen. We should be
1120 // able to import these.
1121 auto TryOptSelectCst = [&Opc, &True, &False, &CC, Is32Bit, &MRI,
1122 &Optimized]() {
1123 if (Optimized)
1124 return false;
1127 if (!TrueCst && !FalseCst)
1128 return false;
1129
1130 Register ZReg = Is32Bit ? AArch64::WZR : AArch64::XZR;
1131 if (TrueCst && FalseCst) {
1132 int64_t T = TrueCst->Value.getSExtValue();
1133 int64_t F = FalseCst->Value.getSExtValue();
1134
1135 if (T == 0 && F == 1) {
1136 // G_SELECT cc, 0, 1 -> CSINC zreg, zreg, cc
1137 Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
1138 True = ZReg;
1139 False = ZReg;
1140 return true;
1141 }
1142
1143 if (T == 0 && F == -1) {
1144 // G_SELECT cc 0, -1 -> CSINV zreg, zreg cc
1145 Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
1146 True = ZReg;
1147 False = ZReg;
1148 return true;
1149 }
1150 }
1151
1152 if (TrueCst) {
1153 int64_t T = TrueCst->Value.getSExtValue();
1154 if (T == 1) {
1155 // G_SELECT cc, 1, f -> CSINC f, zreg, inv_cc
1156 Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
1157 True = False;
1158 False = ZReg;
1160 return true;
1161 }
1162
1163 if (T == -1) {
1164 // G_SELECT cc, -1, f -> CSINV f, zreg, inv_cc
1165 Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
1166 True = False;
1167 False = ZReg;
1169 return true;
1170 }
1171 }
1172
1173 if (FalseCst) {
1174 int64_t F = FalseCst->Value.getSExtValue();
1175 if (F == 1) {
1176 // G_SELECT cc, t, 1 -> CSINC t, zreg, cc
1177 Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
1178 False = ZReg;
1179 return true;
1180 }
1181
1182 if (F == -1) {
1183 // G_SELECT cc, t, -1 -> CSINC t, zreg, cc
1184 Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
1185 False = ZReg;
1186 return true;
1187 }
1188 }
1189 return false;
1190 };
1191
1192 Optimized |= TryFoldBinOpIntoSelect(False, True, /*Invert = */ false);
1193 Optimized |= TryFoldBinOpIntoSelect(True, False, /*Invert = */ true);
1194 Optimized |= TryOptSelectCst();
1195 auto SelectInst = MIB.buildInstr(Opc, {Dst}, {True, False}).addImm(CC);
1196 constrainSelectedInstRegOperands(*SelectInst, TII, TRI, RBI);
1197 return &*SelectInst;
1198}
1199
1202 MachineRegisterInfo *MRI = nullptr) {
1203 switch (P) {
1204 default:
1205 llvm_unreachable("Unknown condition code!");
1206 case CmpInst::ICMP_NE:
1207 return AArch64CC::NE;
1208 case CmpInst::ICMP_EQ:
1209 return AArch64CC::EQ;
1210 case CmpInst::ICMP_SGT:
1211 return AArch64CC::GT;
1212 case CmpInst::ICMP_SGE:
1213 if (RHS && MRI) {
1214 auto ValAndVReg = getIConstantVRegValWithLookThrough(RHS, *MRI);
1215 if (ValAndVReg && ValAndVReg->Value == 0)
1216 return AArch64CC::PL;
1217 }
1218 return AArch64CC::GE;
1219 case CmpInst::ICMP_SLT:
1220 if (RHS && MRI) {
1221 auto ValAndVReg = getIConstantVRegValWithLookThrough(RHS, *MRI);
1222 if (ValAndVReg && ValAndVReg->Value == 0)
1223 return AArch64CC::MI;
1224 }
1225 return AArch64CC::LT;
1226 case CmpInst::ICMP_SLE:
1227 return AArch64CC::LE;
1228 case CmpInst::ICMP_UGT:
1229 return AArch64CC::HI;
1230 case CmpInst::ICMP_UGE:
1231 return AArch64CC::HS;
1232 case CmpInst::ICMP_ULT:
1233 return AArch64CC::LO;
1234 case CmpInst::ICMP_ULE:
1235 return AArch64CC::LS;
1236 }
1237}
1238
1239/// changeFPCCToORAArch64CC - Convert an IR fp condition code to an AArch64 CC.
1241 AArch64CC::CondCode &CondCode,
1242 AArch64CC::CondCode &CondCode2) {
1243 CondCode2 = AArch64CC::AL;
1244 switch (CC) {
1245 default:
1246 llvm_unreachable("Unknown FP condition!");
1247 case CmpInst::FCMP_OEQ:
1248 CondCode = AArch64CC::EQ;
1249 break;
1250 case CmpInst::FCMP_OGT:
1251 CondCode = AArch64CC::GT;
1252 break;
1253 case CmpInst::FCMP_OGE:
1254 CondCode = AArch64CC::GE;
1255 break;
1256 case CmpInst::FCMP_OLT:
1257 CondCode = AArch64CC::MI;
1258 break;
1259 case CmpInst::FCMP_OLE:
1260 CondCode = AArch64CC::LS;
1261 break;
1262 case CmpInst::FCMP_ONE:
1263 CondCode = AArch64CC::MI;
1264 CondCode2 = AArch64CC::GT;
1265 break;
1266 case CmpInst::FCMP_ORD:
1267 CondCode = AArch64CC::VC;
1268 break;
1269 case CmpInst::FCMP_UNO:
1270 CondCode = AArch64CC::VS;
1271 break;
1272 case CmpInst::FCMP_UEQ:
1273 CondCode = AArch64CC::EQ;
1274 CondCode2 = AArch64CC::VS;
1275 break;
1276 case CmpInst::FCMP_UGT:
1277 CondCode = AArch64CC::HI;
1278 break;
1279 case CmpInst::FCMP_UGE:
1280 CondCode = AArch64CC::PL;
1281 break;
1282 case CmpInst::FCMP_ULT:
1283 CondCode = AArch64CC::LT;
1284 break;
1285 case CmpInst::FCMP_ULE:
1286 CondCode = AArch64CC::LE;
1287 break;
1288 case CmpInst::FCMP_UNE:
1289 CondCode = AArch64CC::NE;
1290 break;
1291 }
1292}
1293
1294/// Convert an IR fp condition code to an AArch64 CC.
1295/// This differs from changeFPCCToAArch64CC in that it returns cond codes that
1296/// should be AND'ed instead of OR'ed.
1298 AArch64CC::CondCode &CondCode,
1299 AArch64CC::CondCode &CondCode2) {
1300 CondCode2 = AArch64CC::AL;
1301 switch (CC) {
1302 default:
1303 changeFPCCToORAArch64CC(CC, CondCode, CondCode2);
1304 assert(CondCode2 == AArch64CC::AL);
1305 break;
1306 case CmpInst::FCMP_ONE:
1307 // (a one b)
1308 // == ((a olt b) || (a ogt b))
1309 // == ((a ord b) && (a une b))
1310 CondCode = AArch64CC::VC;
1311 CondCode2 = AArch64CC::NE;
1312 break;
1313 case CmpInst::FCMP_UEQ:
1314 // (a ueq b)
1315 // == ((a uno b) || (a oeq b))
1316 // == ((a ule b) && (a uge b))
1317 CondCode = AArch64CC::PL;
1318 CondCode2 = AArch64CC::LE;
1319 break;
1320 }
1321}
1322
1323/// Return a register which can be used as a bit to test in a TB(N)Z.
1324static Register getTestBitReg(Register Reg, uint64_t &Bit, bool &Invert,
1326 assert(Reg.isValid() && "Expected valid register!");
1327 bool HasZext = false;
1329 unsigned Opc = MI->getOpcode();
1330
1331 if (!MI->getOperand(0).isReg() ||
1332 !MRI.hasOneNonDBGUse(MI->getOperand(0).getReg()))
1333 break;
1334
1335 // (tbz (any_ext x), b) -> (tbz x, b) and
1336 // (tbz (zext x), b) -> (tbz x, b) if we don't use the extended bits.
1337 //
1338 // (tbz (trunc x), b) -> (tbz x, b) is always safe, because the bit number
1339 // on the truncated x is the same as the bit number on x.
1340 if (Opc == TargetOpcode::G_ANYEXT || Opc == TargetOpcode::G_ZEXT ||
1341 Opc == TargetOpcode::G_TRUNC) {
1342 if (Opc == TargetOpcode::G_ZEXT)
1343 HasZext = true;
1344
1345 Register NextReg = MI->getOperand(1).getReg();
1346 // Did we find something worth folding?
1347 if (!NextReg.isValid() || !MRI.hasOneNonDBGUse(NextReg))
1348 break;
1349 TypeSize InSize = MRI.getType(NextReg).getSizeInBits();
1350 if (Bit >= InSize)
1351 break;
1352
1353 // NextReg is worth folding. Keep looking.
1354 Reg = NextReg;
1355 continue;
1356 }
1357
1358 // Attempt to find a suitable operation with a constant on one side.
1359 std::optional<uint64_t> C;
1360 Register TestReg;
1361 switch (Opc) {
1362 default:
1363 break;
1364 case TargetOpcode::G_AND:
1365 case TargetOpcode::G_XOR: {
1366 TestReg = MI->getOperand(1).getReg();
1367 Register ConstantReg = MI->getOperand(2).getReg();
1368 auto VRegAndVal = getIConstantVRegValWithLookThrough(ConstantReg, MRI);
1369 if (!VRegAndVal) {
1370 // AND commutes, check the other side for a constant.
1371 // FIXME: Can we canonicalize the constant so that it's always on the
1372 // same side at some point earlier?
1373 std::swap(ConstantReg, TestReg);
1374 VRegAndVal = getIConstantVRegValWithLookThrough(ConstantReg, MRI);
1375 }
1376 if (VRegAndVal) {
1377 if (HasZext)
1378 C = VRegAndVal->Value.getZExtValue();
1379 else
1380 C = VRegAndVal->Value.getSExtValue();
1381 }
1382 break;
1383 }
1384 case TargetOpcode::G_ASHR:
1385 case TargetOpcode::G_LSHR:
1386 case TargetOpcode::G_SHL: {
1387 TestReg = MI->getOperand(1).getReg();
1388 auto VRegAndVal =
1389 getIConstantVRegValWithLookThrough(MI->getOperand(2).getReg(), MRI);
1390 if (VRegAndVal)
1391 C = VRegAndVal->Value.getSExtValue();
1392 break;
1393 }
1394 }
1395
1396 // Didn't find a constant or viable register. Bail out of the loop.
1397 if (!C || !TestReg.isValid())
1398 break;
1399
1400 // We found a suitable instruction with a constant. Check to see if we can
1401 // walk through the instruction.
1402 Register NextReg;
1403 unsigned TestRegSize = MRI.getType(TestReg).getSizeInBits();
1404 switch (Opc) {
1405 default:
1406 break;
1407 case TargetOpcode::G_AND:
1408 // (tbz (and x, m), b) -> (tbz x, b) when the b-th bit of m is set.
1409 if ((*C >> Bit) & 1)
1410 NextReg = TestReg;
1411 break;
1412 case TargetOpcode::G_SHL:
1413 // (tbz (shl x, c), b) -> (tbz x, b-c) when b-c is positive and fits in
1414 // the type of the register.
1415 if (*C <= Bit && (Bit - *C) < TestRegSize) {
1416 NextReg = TestReg;
1417 Bit = Bit - *C;
1418 }
1419 break;
1420 case TargetOpcode::G_ASHR:
1421 // (tbz (ashr x, c), b) -> (tbz x, b+c) or (tbz x, msb) if b+c is > # bits
1422 // in x
1423 NextReg = TestReg;
1424 Bit = Bit + *C;
1425 if (Bit >= TestRegSize)
1426 Bit = TestRegSize - 1;
1427 break;
1428 case TargetOpcode::G_LSHR:
1429 // (tbz (lshr x, c), b) -> (tbz x, b+c) when b + c is < # bits in x
1430 if ((Bit + *C) < TestRegSize) {
1431 NextReg = TestReg;
1432 Bit = Bit + *C;
1433 }
1434 break;
1435 case TargetOpcode::G_XOR:
1436 // We can walk through a G_XOR by inverting whether we use tbz/tbnz when
1437 // appropriate.
1438 //
1439 // e.g. If x' = xor x, c, and the b-th bit is set in c then
1440 //
1441 // tbz x', b -> tbnz x, b
1442 //
1443 // Because x' only has the b-th bit set if x does not.
1444 if ((*C >> Bit) & 1)
1445 Invert = !Invert;
1446 NextReg = TestReg;
1447 break;
1448 }
1449
1450 // Check if we found anything worth folding.
1451 if (!NextReg.isValid())
1452 return Reg;
1453 Reg = NextReg;
1454 }
1455
1456 return Reg;
1457}
1458
1459MachineInstr *AArch64InstructionSelector::emitTestBit(
1460 Register TestReg, uint64_t Bit, bool IsNegative, MachineBasicBlock *DstMBB,
1461 MachineIRBuilder &MIB) const {
1462 assert(TestReg.isValid());
1463 assert(ProduceNonFlagSettingCondBr &&
1464 "Cannot emit TB(N)Z with speculation tracking!");
1465 MachineRegisterInfo &MRI = *MIB.getMRI();
1466
1467 // Attempt to optimize the test bit by walking over instructions.
1468 TestReg = getTestBitReg(TestReg, Bit, IsNegative, MRI);
1469 LLT Ty = MRI.getType(TestReg);
1470 unsigned Size = Ty.getSizeInBits();
1471 assert(!Ty.isVector() && "Expected a scalar!");
1472 assert(Bit < 64 && "Bit is too large!");
1473
1474 // When the test register is a 64-bit register, we have to narrow to make
1475 // TBNZW work.
1476 bool UseWReg = Bit < 32;
1477 unsigned NecessarySize = UseWReg ? 32 : 64;
1478 if (Size != NecessarySize)
1479 TestReg = moveScalarRegClass(
1480 TestReg, UseWReg ? AArch64::GPR32RegClass : AArch64::GPR64RegClass,
1481 MIB);
1482
1483 static const unsigned OpcTable[2][2] = {{AArch64::TBZX, AArch64::TBNZX},
1484 {AArch64::TBZW, AArch64::TBNZW}};
1485 unsigned Opc = OpcTable[UseWReg][IsNegative];
1486 auto TestBitMI =
1487 MIB.buildInstr(Opc).addReg(TestReg).addImm(Bit).addMBB(DstMBB);
1488 constrainSelectedInstRegOperands(*TestBitMI, TII, TRI, RBI);
1489 return &*TestBitMI;
1490}
1491
1492bool AArch64InstructionSelector::tryOptAndIntoCompareBranch(
1493 MachineInstr &AndInst, bool Invert, MachineBasicBlock *DstMBB,
1494 MachineIRBuilder &MIB) const {
1495 assert(AndInst.getOpcode() == TargetOpcode::G_AND && "Expected G_AND only?");
1496 // Given something like this:
1497 //
1498 // %x = ...Something...
1499 // %one = G_CONSTANT i64 1
1500 // %zero = G_CONSTANT i64 0
1501 // %and = G_AND %x, %one
1502 // %cmp = G_ICMP intpred(ne), %and, %zero
1503 // %cmp_trunc = G_TRUNC %cmp
1504 // G_BRCOND %cmp_trunc, %bb.3
1505 //
1506 // We want to try and fold the AND into the G_BRCOND and produce either a
1507 // TBNZ (when we have intpred(ne)) or a TBZ (when we have intpred(eq)).
1508 //
1509 // In this case, we'd get
1510 //
1511 // TBNZ %x %bb.3
1512 //
1513
1514 // Check if the AND has a constant on its RHS which we can use as a mask.
1515 // If it's a power of 2, then it's the same as checking a specific bit.
1516 // (e.g, ANDing with 8 == ANDing with 000...100 == testing if bit 3 is set)
1517 auto MaybeBit = getIConstantVRegValWithLookThrough(
1518 AndInst.getOperand(2).getReg(), *MIB.getMRI());
1519 if (!MaybeBit)
1520 return false;
1521
1522 int32_t Bit = MaybeBit->Value.exactLogBase2();
1523 if (Bit < 0)
1524 return false;
1525
1526 Register TestReg = AndInst.getOperand(1).getReg();
1527
1528 // Emit a TB(N)Z.
1529 emitTestBit(TestReg, Bit, Invert, DstMBB, MIB);
1530 return true;
1531}
1532
1533MachineInstr *AArch64InstructionSelector::emitCBZ(Register CompareReg,
1534 bool IsNegative,
1535 MachineBasicBlock *DestMBB,
1536 MachineIRBuilder &MIB) const {
1537 assert(ProduceNonFlagSettingCondBr && "CBZ does not set flags!");
1538 MachineRegisterInfo &MRI = *MIB.getMRI();
1539 assert(RBI.getRegBank(CompareReg, MRI, TRI)->getID() ==
1540 AArch64::GPRRegBankID &&
1541 "Expected GPRs only?");
1542 auto Ty = MRI.getType(CompareReg);
1543 unsigned Width = Ty.getSizeInBits();
1544 assert(!Ty.isVector() && "Expected scalar only?");
1545 assert(Width <= 64 && "Expected width to be at most 64?");
1546 static const unsigned OpcTable[2][2] = {{AArch64::CBZW, AArch64::CBZX},
1547 {AArch64::CBNZW, AArch64::CBNZX}};
1548 unsigned Opc = OpcTable[IsNegative][Width == 64];
1549 auto BranchMI = MIB.buildInstr(Opc, {}, {CompareReg}).addMBB(DestMBB);
1550 constrainSelectedInstRegOperands(*BranchMI, TII, TRI, RBI);
1551 return &*BranchMI;
1552}
1553
1554bool AArch64InstructionSelector::selectCompareBranchFedByFCmp(
1555 MachineInstr &I, MachineInstr &FCmp, MachineIRBuilder &MIB) const {
1556 assert(FCmp.getOpcode() == TargetOpcode::G_FCMP);
1557 assert(I.getOpcode() == TargetOpcode::G_BRCOND);
1558 // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't
1559 // totally clean. Some of them require two branches to implement.
1561 emitFPCompare(FCmp.getOperand(2).getReg(), FCmp.getOperand(3).getReg(), MIB,
1562 Pred);
1563 AArch64CC::CondCode CC1, CC2;
1565 MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
1566 MIB.buildInstr(AArch64::Bcc, {}, {}).addImm(CC1).addMBB(DestMBB);
1567 if (CC2 != AArch64CC::AL)
1568 MIB.buildInstr(AArch64::Bcc, {}, {}).addImm(CC2).addMBB(DestMBB);
1569 I.eraseFromParent();
1570 return true;
1571}
1572
1573bool AArch64InstructionSelector::tryOptCompareBranchFedByICmp(
1574 MachineInstr &I, MachineInstr &ICmp, MachineIRBuilder &MIB) const {
1575 assert(ICmp.getOpcode() == TargetOpcode::G_ICMP);
1576 assert(I.getOpcode() == TargetOpcode::G_BRCOND);
1577 // Attempt to optimize the G_BRCOND + G_ICMP into a TB(N)Z/CB(N)Z.
1578 //
1579 // Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z
1580 // instructions will not be produced, as they are conditional branch
1581 // instructions that do not set flags.
1582 if (!ProduceNonFlagSettingCondBr)
1583 return false;
1584
1585 MachineRegisterInfo &MRI = *MIB.getMRI();
1586 MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
1587 auto Pred =
1588 static_cast<CmpInst::Predicate>(ICmp.getOperand(1).getPredicate());
1589 Register LHS = ICmp.getOperand(2).getReg();
1590 Register RHS = ICmp.getOperand(3).getReg();
1591
1592 // We're allowed to emit a TB(N)Z/CB(N)Z. Try to do that.
1593 auto VRegAndVal = getIConstantVRegValWithLookThrough(RHS, MRI);
1594 MachineInstr *AndInst = getOpcodeDef(TargetOpcode::G_AND, LHS, MRI);
1595
1596 // When we can emit a TB(N)Z, prefer that.
1597 //
1598 // Handle non-commutative condition codes first.
1599 // Note that we don't want to do this when we have a G_AND because it can
1600 // become a tst. The tst will make the test bit in the TB(N)Z redundant.
1601 if (VRegAndVal && !AndInst) {
1602 int64_t C = VRegAndVal->Value.getSExtValue();
1603
1604 // When we have a greater-than comparison, we can just test if the msb is
1605 // zero.
1606 if (C == -1 && Pred == CmpInst::ICMP_SGT) {
1608 emitTestBit(LHS, Bit, /*IsNegative = */ false, DestMBB, MIB);
1609 I.eraseFromParent();
1610 return true;
1611 }
1612
1613 // When we have a less than comparison, we can just test if the msb is not
1614 // zero.
1615 if (C == 0 && Pred == CmpInst::ICMP_SLT) {
1617 emitTestBit(LHS, Bit, /*IsNegative = */ true, DestMBB, MIB);
1618 I.eraseFromParent();
1619 return true;
1620 }
1621
1622 // Inversely, if we have a signed greater-than-or-equal comparison to zero,
1623 // we can test if the msb is zero.
1624 if (C == 0 && Pred == CmpInst::ICMP_SGE) {
1626 emitTestBit(LHS, Bit, /*IsNegative = */ false, DestMBB, MIB);
1627 I.eraseFromParent();
1628 return true;
1629 }
1630 }
1631
1632 // Attempt to handle commutative condition codes. Right now, that's only
1633 // eq/ne.
1635 if (!VRegAndVal) {
1636 std::swap(RHS, LHS);
1638 AndInst = getOpcodeDef(TargetOpcode::G_AND, LHS, MRI);
1639 }
1640
1641 if (VRegAndVal && VRegAndVal->Value == 0) {
1642 // If there's a G_AND feeding into this branch, try to fold it away by
1643 // emitting a TB(N)Z instead.
1644 //
1645 // Note: If we have LT, then it *is* possible to fold, but it wouldn't be
1646 // beneficial. When we have an AND and LT, we need a TST/ANDS, so folding
1647 // would be redundant.
1648 if (AndInst &&
1649 tryOptAndIntoCompareBranch(
1650 *AndInst, /*Invert = */ Pred == CmpInst::ICMP_NE, DestMBB, MIB)) {
1651 I.eraseFromParent();
1652 return true;
1653 }
1654
1655 // Otherwise, try to emit a CB(N)Z instead.
1656 auto LHSTy = MRI.getType(LHS);
1657 if (!LHSTy.isVector() && LHSTy.getSizeInBits() <= 64) {
1658 emitCBZ(LHS, /*IsNegative = */ Pred == CmpInst::ICMP_NE, DestMBB, MIB);
1659 I.eraseFromParent();
1660 return true;
1661 }
1662 }
1663 }
1664
1665 return false;
1666}
1667
1668bool AArch64InstructionSelector::selectCompareBranchFedByICmp(
1669 MachineInstr &I, MachineInstr &ICmp, MachineIRBuilder &MIB) const {
1670 assert(ICmp.getOpcode() == TargetOpcode::G_ICMP);
1671 assert(I.getOpcode() == TargetOpcode::G_BRCOND);
1672 if (tryOptCompareBranchFedByICmp(I, ICmp, MIB))
1673 return true;
1674
1675 // Couldn't optimize. Emit a compare + a Bcc.
1676 MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
1677 auto &PredOp = ICmp.getOperand(1);
1678 emitIntegerCompare(ICmp.getOperand(2), ICmp.getOperand(3), PredOp, MIB);
1680 static_cast<CmpInst::Predicate>(PredOp.getPredicate()),
1681 ICmp.getOperand(3).getReg(), MIB.getMRI());
1682 MIB.buildInstr(AArch64::Bcc, {}, {}).addImm(CC).addMBB(DestMBB);
1683 I.eraseFromParent();
1684 return true;
1685}
1686
1687bool AArch64InstructionSelector::selectCompareBranch(
1688 MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) {
1689 Register CondReg = I.getOperand(0).getReg();
1690 MachineInstr *CCMI = MRI.getVRegDef(CondReg);
1691 // Try to select the G_BRCOND using whatever is feeding the condition if
1692 // possible.
1693 unsigned CCMIOpc = CCMI->getOpcode();
1694 if (CCMIOpc == TargetOpcode::G_FCMP)
1695 return selectCompareBranchFedByFCmp(I, *CCMI, MIB);
1696 if (CCMIOpc == TargetOpcode::G_ICMP)
1697 return selectCompareBranchFedByICmp(I, *CCMI, MIB);
1698
1699 // Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z
1700 // instructions will not be produced, as they are conditional branch
1701 // instructions that do not set flags.
1702 if (ProduceNonFlagSettingCondBr) {
1703 emitTestBit(CondReg, /*Bit = */ 0, /*IsNegative = */ true,
1704 I.getOperand(1).getMBB(), MIB);
1705 I.eraseFromParent();
1706 return true;
1707 }
1708
1709 // Can't emit TB(N)Z/CB(N)Z. Emit a tst + bcc instead.
1710 auto TstMI =
1711 MIB.buildInstr(AArch64::ANDSWri, {LLT::scalar(32)}, {CondReg}).addImm(1);
1713 auto Bcc = MIB.buildInstr(AArch64::Bcc)
1715 .addMBB(I.getOperand(1).getMBB());
1716 I.eraseFromParent();
1718 return true;
1719}
1720
1721/// Returns the element immediate value of a vector shift operand if found.
1722/// This needs to detect a splat-like operation, e.g. a G_BUILD_VECTOR.
1723static std::optional<APInt> getVectorShiftImm(Register Reg,
1725 assert(MRI.getType(Reg).isVector() && "Expected a *vector* shift operand");
1726 MachineInstr *OpMI = MRI.getVRegDef(Reg);
1727 std::optional<Register> Splat = getVectorSplat(*OpMI, MRI);
1728 if (!Splat)
1729 return std::nullopt;
1730 return getIConstantVRegVal(*Splat, MRI);
1731}
1732
1733/// Matches and returns the shift immediate value for a SHL instruction given
1734/// a shift operand.
1735static std::optional<uint64_t> getVectorSHLImm(LLT SrcTy, Register Reg,
1737 std::optional<APInt> ShiftImm = getVectorShiftImm(Reg, MRI);
1738 if (!ShiftImm)
1739 return std::nullopt;
1740 // Check the immediate is in range for a SHL.
1741 uint64_t Imm = ShiftImm->getZExtValue();
1742 if (Imm >= SrcTy.getElementType().getSizeInBits())
1743 return std::nullopt;
1744 return Imm;
1745}
1746
1747bool AArch64InstructionSelector::selectVectorSHL(MachineInstr &I,
1748 MachineRegisterInfo &MRI) {
1749 assert(I.getOpcode() == TargetOpcode::G_SHL);
1750 Register DstReg = I.getOperand(0).getReg();
1751 const LLT Ty = MRI.getType(DstReg);
1752 Register Src1Reg = I.getOperand(1).getReg();
1753 Register Src2Reg = I.getOperand(2).getReg();
1754
1755 if (!Ty.isVector())
1756 return false;
1757
1758 // Check if we have a vector of constants on RHS that we can select as the
1759 // immediate form.
1760 std::optional<uint64_t> ImmVal = getVectorSHLImm(Ty, Src2Reg, MRI);
1761
1762 unsigned Opc = 0;
1763 if (Ty == LLT::fixed_vector(2, 64)) {
1764 Opc = ImmVal ? AArch64::SHLv2i64_shift : AArch64::USHLv2i64;
1765 } else if (Ty == LLT::fixed_vector(4, 32)) {
1766 Opc = ImmVal ? AArch64::SHLv4i32_shift : AArch64::USHLv4i32;
1767 } else if (Ty == LLT::fixed_vector(2, 32)) {
1768 Opc = ImmVal ? AArch64::SHLv2i32_shift : AArch64::USHLv2i32;
1769 } else if (Ty == LLT::fixed_vector(4, 16)) {
1770 Opc = ImmVal ? AArch64::SHLv4i16_shift : AArch64::USHLv4i16;
1771 } else if (Ty == LLT::fixed_vector(8, 16)) {
1772 Opc = ImmVal ? AArch64::SHLv8i16_shift : AArch64::USHLv8i16;
1773 } else if (Ty == LLT::fixed_vector(16, 8)) {
1774 Opc = ImmVal ? AArch64::SHLv16i8_shift : AArch64::USHLv16i8;
1775 } else if (Ty == LLT::fixed_vector(8, 8)) {
1776 Opc = ImmVal ? AArch64::SHLv8i8_shift : AArch64::USHLv8i8;
1777 } else {
1778 LLVM_DEBUG(dbgs() << "Unhandled G_SHL type");
1779 return false;
1780 }
1781
1782 auto Shl = MIB.buildInstr(Opc, {DstReg}, {Src1Reg});
1783 if (ImmVal)
1784 Shl.addImm(*ImmVal);
1785 else
1786 Shl.addUse(Src2Reg);
1788 I.eraseFromParent();
1789 return true;
1790}
1791
1792bool AArch64InstructionSelector::selectVectorAshrLshr(
1793 MachineInstr &I, MachineRegisterInfo &MRI) {
1794 assert(I.getOpcode() == TargetOpcode::G_ASHR ||
1795 I.getOpcode() == TargetOpcode::G_LSHR);
1796 Register DstReg = I.getOperand(0).getReg();
1797 const LLT Ty = MRI.getType(DstReg);
1798 Register Src1Reg = I.getOperand(1).getReg();
1799 Register Src2Reg = I.getOperand(2).getReg();
1800
1801 if (!Ty.isVector())
1802 return false;
1803
1804 bool IsASHR = I.getOpcode() == TargetOpcode::G_ASHR;
1805
1806 // We expect the immediate case to be lowered in the PostLegalCombiner to
1807 // AArch64ISD::VASHR or AArch64ISD::VLSHR equivalents.
1808
1809 // There is not a shift right register instruction, but the shift left
1810 // register instruction takes a signed value, where negative numbers specify a
1811 // right shift.
1812
1813 unsigned Opc = 0;
1814 unsigned NegOpc = 0;
1815 const TargetRegisterClass *RC =
1816 getRegClassForTypeOnBank(Ty, RBI.getRegBank(AArch64::FPRRegBankID));
1817 if (Ty == LLT::fixed_vector(2, 64)) {
1818 Opc = IsASHR ? AArch64::SSHLv2i64 : AArch64::USHLv2i64;
1819 NegOpc = AArch64::NEGv2i64;
1820 } else if (Ty == LLT::fixed_vector(4, 32)) {
1821 Opc = IsASHR ? AArch64::SSHLv4i32 : AArch64::USHLv4i32;
1822 NegOpc = AArch64::NEGv4i32;
1823 } else if (Ty == LLT::fixed_vector(2, 32)) {
1824 Opc = IsASHR ? AArch64::SSHLv2i32 : AArch64::USHLv2i32;
1825 NegOpc = AArch64::NEGv2i32;
1826 } else if (Ty == LLT::fixed_vector(4, 16)) {
1827 Opc = IsASHR ? AArch64::SSHLv4i16 : AArch64::USHLv4i16;
1828 NegOpc = AArch64::NEGv4i16;
1829 } else if (Ty == LLT::fixed_vector(8, 16)) {
1830 Opc = IsASHR ? AArch64::SSHLv8i16 : AArch64::USHLv8i16;
1831 NegOpc = AArch64::NEGv8i16;
1832 } else if (Ty == LLT::fixed_vector(16, 8)) {
1833 Opc = IsASHR ? AArch64::SSHLv16i8 : AArch64::USHLv16i8;
1834 NegOpc = AArch64::NEGv16i8;
1835 } else if (Ty == LLT::fixed_vector(8, 8)) {
1836 Opc = IsASHR ? AArch64::SSHLv8i8 : AArch64::USHLv8i8;
1837 NegOpc = AArch64::NEGv8i8;
1838 } else {
1839 LLVM_DEBUG(dbgs() << "Unhandled G_ASHR type");
1840 return false;
1841 }
1842
1843 auto Neg = MIB.buildInstr(NegOpc, {RC}, {Src2Reg});
1845 auto SShl = MIB.buildInstr(Opc, {DstReg}, {Src1Reg, Neg});
1847 I.eraseFromParent();
1848 return true;
1849}
1850
1851bool AArch64InstructionSelector::selectVaStartAAPCS(
1852 MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
1853
1855 MF.getFunction().isVarArg()))
1856 return false;
1857
1858 // The layout of the va_list struct is specified in the AArch64 Procedure Call
1859 // Standard, section 10.1.5.
1860
1861 const AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
1862 const unsigned PtrSize = STI.isTargetILP32() ? 4 : 8;
1863 const auto *PtrRegClass =
1864 STI.isTargetILP32() ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass;
1865
1866 const MCInstrDesc &MCIDAddAddr =
1867 TII.get(STI.isTargetILP32() ? AArch64::ADDWri : AArch64::ADDXri);
1868 const MCInstrDesc &MCIDStoreAddr =
1869 TII.get(STI.isTargetILP32() ? AArch64::STRWui : AArch64::STRXui);
1870
1871 /*
1872 * typedef struct va_list {
1873 * void * stack; // next stack param
1874 * void * gr_top; // end of GP arg reg save area
1875 * void * vr_top; // end of FP/SIMD arg reg save area
1876 * int gr_offs; // offset from gr_top to next GP register arg
1877 * int vr_offs; // offset from vr_top to next FP/SIMD register arg
1878 * } va_list;
1879 */
1880 const auto VAList = I.getOperand(0).getReg();
1881
1882 // Our current offset in bytes from the va_list struct (VAList).
1883 unsigned OffsetBytes = 0;
1884
1885 // Helper function to store (FrameIndex + Imm) to VAList at offset OffsetBytes
1886 // and increment OffsetBytes by PtrSize.
1887 const auto PushAddress = [&](const int FrameIndex, const int64_t Imm) {
1888 const Register Top = MRI.createVirtualRegister(PtrRegClass);
1889 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), MCIDAddAddr)
1890 .addDef(Top)
1891 .addFrameIndex(FrameIndex)
1892 .addImm(Imm)
1893 .addImm(0);
1895
1896 const auto *MMO = *I.memoperands_begin();
1897 MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), MCIDStoreAddr)
1898 .addUse(Top)
1899 .addUse(VAList)
1900 .addImm(OffsetBytes / PtrSize)
1902 MMO->getPointerInfo().getWithOffset(OffsetBytes),
1903 MachineMemOperand::MOStore, PtrSize, MMO->getBaseAlign()));
1905
1906 OffsetBytes += PtrSize;
1907 };
1908
1909 // void* stack at offset 0
1910 PushAddress(FuncInfo->getVarArgsStackIndex(), 0);
1911
1912 // void* gr_top at offset 8 (4 on ILP32)
1913 const unsigned GPRSize = FuncInfo->getVarArgsGPRSize();
1914 PushAddress(FuncInfo->getVarArgsGPRIndex(), GPRSize);
1915
1916 // void* vr_top at offset 16 (8 on ILP32)
1917 const unsigned FPRSize = FuncInfo->getVarArgsFPRSize();
1918 PushAddress(FuncInfo->getVarArgsFPRIndex(), FPRSize);
1919
1920 // Helper function to store a 4-byte integer constant to VAList at offset
1921 // OffsetBytes, and increment OffsetBytes by 4.
1922 const auto PushIntConstant = [&](const int32_t Value) {
1923 constexpr int IntSize = 4;
1924 const Register Temp = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
1925 auto MIB =
1926 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::MOVi32imm))
1927 .addDef(Temp)
1928 .addImm(Value);
1930
1931 const auto *MMO = *I.memoperands_begin();
1932 MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::STRWui))
1933 .addUse(Temp)
1934 .addUse(VAList)
1935 .addImm(OffsetBytes / IntSize)
1937 MMO->getPointerInfo().getWithOffset(OffsetBytes),
1938 MachineMemOperand::MOStore, IntSize, MMO->getBaseAlign()));
1940 OffsetBytes += IntSize;
1941 };
1942
1943 // int gr_offs at offset 24 (12 on ILP32)
1944 PushIntConstant(-static_cast<int32_t>(GPRSize));
1945
1946 // int vr_offs at offset 28 (16 on ILP32)
1947 PushIntConstant(-static_cast<int32_t>(FPRSize));
1948
1949 assert(OffsetBytes == (STI.isTargetILP32() ? 20 : 32) && "Unexpected offset");
1950
1951 I.eraseFromParent();
1952 return true;
1953}
1954
1955bool AArch64InstructionSelector::selectVaStartDarwin(
1956 MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
1957 AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
1958 Register ListReg = I.getOperand(0).getReg();
1959
1960 Register ArgsAddrReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
1961
1962 int FrameIdx = FuncInfo->getVarArgsStackIndex();
1963 if (MF.getSubtarget<AArch64Subtarget>().isCallingConvWin64(
1965 FrameIdx = FuncInfo->getVarArgsGPRSize() > 0
1966 ? FuncInfo->getVarArgsGPRIndex()
1967 : FuncInfo->getVarArgsStackIndex();
1968 }
1969
1970 auto MIB =
1971 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::ADDXri))
1972 .addDef(ArgsAddrReg)
1973 .addFrameIndex(FrameIdx)
1974 .addImm(0)
1975 .addImm(0);
1976
1978
1979 MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::STRXui))
1980 .addUse(ArgsAddrReg)
1981 .addUse(ListReg)
1982 .addImm(0)
1983 .addMemOperand(*I.memoperands_begin());
1984
1986 I.eraseFromParent();
1987 return true;
1988}
1989
1990void AArch64InstructionSelector::materializeLargeCMVal(
1991 MachineInstr &I, const Value *V, unsigned OpFlags) {
1992 MachineBasicBlock &MBB = *I.getParent();
1993 MachineFunction &MF = *MBB.getParent();
1994 MachineRegisterInfo &MRI = MF.getRegInfo();
1995
1996 auto MovZ = MIB.buildInstr(AArch64::MOVZXi, {&AArch64::GPR64RegClass}, {});
1997 MovZ->addOperand(MF, I.getOperand(1));
1998 MovZ->getOperand(1).setTargetFlags(OpFlags | AArch64II::MO_G0 |
2000 MovZ->addOperand(MF, MachineOperand::CreateImm(0));
2002
2003 auto BuildMovK = [&](Register SrcReg, unsigned char Flags, unsigned Offset,
2004 Register ForceDstReg) {
2005 Register DstReg = ForceDstReg
2006 ? ForceDstReg
2007 : MRI.createVirtualRegister(&AArch64::GPR64RegClass);
2008 auto MovI = MIB.buildInstr(AArch64::MOVKXi).addDef(DstReg).addUse(SrcReg);
2009 if (auto *GV = dyn_cast<GlobalValue>(V)) {
2010 MovI->addOperand(MF, MachineOperand::CreateGA(
2011 GV, MovZ->getOperand(1).getOffset(), Flags));
2012 } else {
2013 MovI->addOperand(
2015 MovZ->getOperand(1).getOffset(), Flags));
2016 }
2019 return DstReg;
2020 };
2021 Register DstReg = BuildMovK(MovZ.getReg(0),
2023 DstReg = BuildMovK(DstReg, AArch64II::MO_G2 | AArch64II::MO_NC, 32, 0);
2024 BuildMovK(DstReg, AArch64II::MO_G3, 48, I.getOperand(0).getReg());
2025}
2026
2027bool AArch64InstructionSelector::preISelLower(MachineInstr &I) {
2028 MachineBasicBlock &MBB = *I.getParent();
2029 MachineFunction &MF = *MBB.getParent();
2030 MachineRegisterInfo &MRI = MF.getRegInfo();
2031
2032 switch (I.getOpcode()) {
2033 case TargetOpcode::G_CONSTANT: {
2034 Register DefReg = I.getOperand(0).getReg();
2035 const LLT DefTy = MRI.getType(DefReg);
2036 if (!DefTy.isPointer()) {
2037 if (DefTy.getSizeInBits() >= 32 ||
2038 RBI.getRegBank(DefReg, MRI, TRI)->getID() != AArch64::GPRRegBankID)
2039 return false;
2040 // Widen narrow GPR constants to s32 so imported patterns can match.
2041 APInt Val = I.getOperand(1).getCImm()->getValue().zext(32);
2042 I.getOperand(1).setCImm(
2043 ConstantInt::get(MF.getFunction().getContext(), Val));
2044
2046 MRI.setRegBank(WideReg, RBI.getRegBank(AArch64::GPRRegBankID));
2047 I.getOperand(0).setReg(WideReg);
2048
2049 MIB.setInsertPt(MBB, std::next(I.getIterator()));
2050 auto Copy = MIB.buildCopy(DefReg, WideReg);
2051 selectCopy(*Copy, TII, MRI, TRI, RBI);
2052 MIB.setInstr(I);
2053 return true;
2054 }
2055 const unsigned PtrSize = DefTy.getSizeInBits();
2056 if (PtrSize != 32 && PtrSize != 64)
2057 return false;
2058 // Convert pointer typed constants to integers so TableGen can select.
2059 MRI.setType(DefReg, LLT::integer(PtrSize));
2060 return true;
2061 }
2062 case TargetOpcode::G_STORE: {
2063 bool Changed = contractCrossBankCopyIntoStore(I, MRI);
2064 MachineOperand &SrcOp = I.getOperand(0);
2065 if (MRI.getType(SrcOp.getReg()).isPointer()) {
2066 // Allow matching with imported patterns for stores of pointers. Unlike
2067 // G_LOAD/G_PTR_ADD, we may not have selected all users. So, emit a copy
2068 // and constrain.
2069 auto Copy = MIB.buildCopy(LLT::scalar(64), SrcOp);
2070 Register NewSrc = Copy.getReg(0);
2071 SrcOp.setReg(NewSrc);
2072 RBI.constrainGenericRegister(NewSrc, AArch64::GPR64RegClass, MRI);
2073 Changed = true;
2074 }
2075 return Changed;
2076 }
2077 case TargetOpcode::G_PTR_ADD: {
2078 // If Checked Pointer Arithmetic (FEAT_CPA) is present, preserve the pointer
2079 // arithmetic semantics instead of falling back to regular arithmetic.
2080 const auto &TL = STI.getTargetLowering();
2081 if (TL->shouldPreservePtrArith(MF.getFunction(), EVT()))
2082 return false;
2083 return convertPtrAddToAdd(I, MRI);
2084 }
2085 case TargetOpcode::G_LOAD: {
2086 // For scalar loads of pointers, we try to convert the dest type from p0
2087 // to s64 so that our imported patterns can match. Like with the G_PTR_ADD
2088 // conversion, this should be ok because all users should have been
2089 // selected already, so the type doesn't matter for them.
2090 Register DstReg = I.getOperand(0).getReg();
2091 const LLT DstTy = MRI.getType(DstReg);
2092 if (!DstTy.isPointer())
2093 return false;
2094 MRI.setType(DstReg, LLT::scalar(64));
2095 return true;
2096 }
2097 case TargetOpcode::G_VECREDUCE_ADD:
2098 case TargetOpcode::G_VECREDUCE_SMAX:
2099 case TargetOpcode::G_VECREDUCE_SMIN:
2100 case TargetOpcode::G_VECREDUCE_UMAX:
2101 case TargetOpcode::G_VECREDUCE_UMIN: {
2102 // Imported patterns require an FPR result. For a GPR, use a temporary FPR
2103 // and insert a cross-bank copy.
2104 Register DstReg = I.getOperand(0).getReg();
2105 const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
2106 if (DstRB.getID() != AArch64::GPRRegBankID)
2107 return false;
2108
2109 LLT DstTy = MRI.getType(DstReg);
2110 const TargetRegisterClass *DstRC =
2111 getRegClassForTypeOnBank(DstTy, DstRB, /*GetAllRegSet=*/true);
2112 if (!DstRC || !RBI.constrainGenericRegister(DstReg, *DstRC, MRI))
2113 return false;
2114
2116 MRI.setRegBank(FPRDst, RBI.getRegBank(AArch64::FPRRegBankID));
2117 I.getOperand(0).setReg(FPRDst);
2118
2119 BuildMI(MBB, std::next(I.getIterator()), MIMetadata(I),
2120 TII.get(TargetOpcode::COPY), DstReg)
2121 .addReg(FPRDst);
2122 return true;
2123 }
2124 case TargetOpcode::G_SPLAT_VECTOR: {
2125 // Convert the type from p0 to s64 to help selection.
2126 LLT DstTy = MRI.getType(I.getOperand(0).getReg());
2127 if (!DstTy.isPointerVector())
2128 return false;
2129 auto NewSrc = MIB.buildCopy(LLT::scalar(64), I.getOperand(1).getReg());
2130 MRI.setType(I.getOperand(0).getReg(),
2131 DstTy.changeElementType(LLT::scalar(64)));
2132 MRI.setRegClass(NewSrc.getReg(0), &AArch64::GPR64RegClass);
2133 I.getOperand(1).setReg(NewSrc.getReg(0));
2134 return true;
2135 }
2136 case AArch64::G_INSERT_VECTOR_ELT: {
2137 LLT DstTy = MRI.getType(I.getOperand(0).getReg());
2138 LLT SrcVecTy = MRI.getType(I.getOperand(1).getReg());
2139 if (SrcVecTy.isPointerVector()) {
2140 // Convert the type from p0 to s64 to help selection.
2141 auto NewSrc = MIB.buildCopy(LLT::scalar(64), I.getOperand(2).getReg());
2142 MRI.setType(I.getOperand(1).getReg(),
2143 DstTy.changeElementType(LLT::scalar(64)));
2144 MRI.setType(I.getOperand(0).getReg(),
2145 DstTy.changeElementType(LLT::scalar(64)));
2146 MRI.setRegClass(NewSrc.getReg(0), &AArch64::GPR64RegClass);
2147 I.getOperand(2).setReg(NewSrc.getReg(0));
2148 return true;
2149 }
2150
2151 Register EltReg = I.getOperand(2).getReg();
2152 LLT EltTy = MRI.getType(EltReg);
2153 if (EltTy.isScalar() &&
2154 (EltTy.getSizeInBits() == 8 || EltTy.getSizeInBits() == 16) &&
2155 RBI.getRegBank(EltReg, MRI, TRI)->getID() == AArch64::GPRRegBankID) {
2156 // Convert the type from s8/s16 to s32 to help selection.
2157 auto NewElt = MIB.buildCopy(LLT::scalar(32), EltReg);
2158 MRI.setRegClass(NewElt.getReg(0), &AArch64::GPR32RegClass);
2159 I.getOperand(2).setReg(NewElt.getReg(0));
2160 return true;
2161 }
2162 return false;
2163 }
2164 case TargetOpcode::G_UITOFP:
2165 case TargetOpcode::G_SITOFP: {
2166 // If both source and destination regbanks are FPR, then convert the opcode
2167 // to G_SITOF so that the importer can select it to an fpr variant.
2168 // Otherwise, it ends up matching an fpr/gpr variant and adding a cross-bank
2169 // copy.
2170 Register SrcReg = I.getOperand(1).getReg();
2171 LLT SrcTy = MRI.getType(SrcReg);
2172 LLT DstTy = MRI.getType(I.getOperand(0).getReg());
2173 if (SrcTy.isVector() || SrcTy.getSizeInBits() != DstTy.getSizeInBits())
2174 return false;
2175
2176 if (RBI.getRegBank(SrcReg, MRI, TRI)->getID() == AArch64::FPRRegBankID) {
2177 // Need to add a copy to change the type so that the existing patterns can
2178 // match when there is an integer on an FPR bank.
2179 if (SrcTy.getScalarType().isInteger()) {
2180 auto Copy = MIB.buildCopy(DstTy, SrcReg);
2181 I.getOperand(1).setReg(Copy.getReg(0));
2182 MRI.setRegClass(Copy.getReg(0),
2183 getRegClassForTypeOnBank(
2184 SrcTy, RBI.getRegBank(AArch64::FPRRegBankID)));
2185 }
2186 if (I.getOpcode() == TargetOpcode::G_SITOFP)
2187 I.setDesc(TII.get(AArch64::G_SITOF));
2188 else
2189 I.setDesc(TII.get(AArch64::G_UITOF));
2190 return true;
2191 }
2192 return false;
2193 }
2194 default:
2195 return false;
2196 }
2197}
2198
2199/// This lowering tries to look for G_PTR_ADD instructions and then converts
2200/// them to a standard G_ADD with a COPY on the source.
2201///
2202/// The motivation behind this is to expose the add semantics to the imported
2203/// tablegen patterns. We shouldn't need to check for uses being loads/stores,
2204/// because the selector works bottom up, uses before defs. By the time we
2205/// end up trying to select a G_PTR_ADD, we should have already attempted to
2206/// fold this into addressing modes and were therefore unsuccessful.
2207bool AArch64InstructionSelector::convertPtrAddToAdd(
2208 MachineInstr &I, MachineRegisterInfo &MRI) {
2209 assert(I.getOpcode() == TargetOpcode::G_PTR_ADD && "Expected G_PTR_ADD");
2210 Register DstReg = I.getOperand(0).getReg();
2211 Register AddOp1Reg = I.getOperand(1).getReg();
2212 const LLT PtrTy = MRI.getType(DstReg);
2213 if (PtrTy.getAddressSpace() != 0)
2214 return false;
2215
2216 const LLT CastPtrTy = PtrTy.isVector()
2218 : LLT::integer(64);
2219 auto PtrToInt = MIB.buildPtrToInt(CastPtrTy, AddOp1Reg);
2220 // Set regbanks on the registers.
2221 if (PtrTy.isVector())
2222 MRI.setRegBank(PtrToInt.getReg(0), RBI.getRegBank(AArch64::FPRRegBankID));
2223 else
2224 MRI.setRegBank(PtrToInt.getReg(0), RBI.getRegBank(AArch64::GPRRegBankID));
2225
2226 // Now turn the %dst(p0) = G_PTR_ADD %base, off into:
2227 // %dst(intty) = G_ADD %intbase, off
2228 I.setDesc(TII.get(TargetOpcode::G_ADD));
2229 MRI.setType(DstReg, CastPtrTy);
2230 I.getOperand(1).setReg(PtrToInt.getReg(0));
2231 if (!select(*PtrToInt)) {
2232 LLVM_DEBUG(dbgs() << "Failed to select G_PTRTOINT in convertPtrAddToAdd");
2233 return false;
2234 }
2235
2236 // Also take the opportunity here to try to do some optimization.
2237 // Try to convert this into a G_SUB if the offset is a 0-x negate idiom.
2238 Register NegatedReg;
2239 if (!mi_match(I.getOperand(2).getReg(), MRI, m_Neg(m_Reg(NegatedReg))))
2240 return true;
2241 I.getOperand(2).setReg(NegatedReg);
2242 I.setDesc(TII.get(TargetOpcode::G_SUB));
2243 return true;
2244}
2245
2246bool AArch64InstructionSelector::earlySelectSHL(MachineInstr &I,
2247 MachineRegisterInfo &MRI) {
2248 // We try to match the immediate variant of LSL, which is actually an alias
2249 // for a special case of UBFM. Otherwise, we fall back to the imported
2250 // selector which will match the register variant.
2251 assert(I.getOpcode() == TargetOpcode::G_SHL && "unexpected op");
2252 const auto &MO = I.getOperand(2);
2253 auto VRegAndVal = getIConstantVRegVal(MO.getReg(), MRI);
2254 if (!VRegAndVal)
2255 return false;
2256
2257 const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
2258 if (DstTy.isVector())
2259 return false;
2260 bool Is64Bit = DstTy.getSizeInBits() == 64;
2261 auto Imm1Fn = Is64Bit ? selectShiftA_64(MO) : selectShiftA_32(MO);
2262 auto Imm2Fn = Is64Bit ? selectShiftB_64(MO) : selectShiftB_32(MO);
2263
2264 if (!Imm1Fn || !Imm2Fn)
2265 return false;
2266
2267 auto NewI =
2268 MIB.buildInstr(Is64Bit ? AArch64::UBFMXri : AArch64::UBFMWri,
2269 {I.getOperand(0).getReg()}, {I.getOperand(1).getReg()});
2270
2271 for (auto &RenderFn : *Imm1Fn)
2272 RenderFn(NewI);
2273 for (auto &RenderFn : *Imm2Fn)
2274 RenderFn(NewI);
2275
2276 I.eraseFromParent();
2278 return true;
2279}
2280
2281bool AArch64InstructionSelector::contractCrossBankCopyIntoStore(
2282 MachineInstr &I, MachineRegisterInfo &MRI) {
2283 assert(I.getOpcode() == TargetOpcode::G_STORE && "Expected G_STORE");
2284 // If we're storing a scalar, it doesn't matter what register bank that
2285 // scalar is on. All that matters is the size.
2286 //
2287 // So, if we see something like this (with a 32-bit scalar as an example):
2288 //
2289 // %x:gpr(s32) = ... something ...
2290 // %y:fpr(s32) = COPY %x:gpr(s32)
2291 // G_STORE %y:fpr(s32)
2292 //
2293 // We can fix this up into something like this:
2294 //
2295 // G_STORE %x:gpr(s32)
2296 //
2297 // And then continue the selection process normally.
2298 Register DefDstReg = getSrcRegIgnoringCopies(I.getOperand(0).getReg(), MRI);
2299 if (!DefDstReg.isValid())
2300 return false;
2301 LLT DefDstTy = MRI.getType(DefDstReg);
2302 Register StoreSrcReg = I.getOperand(0).getReg();
2303 LLT StoreSrcTy = MRI.getType(StoreSrcReg);
2304
2305 // If we get something strange like a physical register, then we shouldn't
2306 // go any further.
2307 if (!DefDstTy.isValid())
2308 return false;
2309
2310 // Are the source and dst types the same size?
2311 if (DefDstTy.getSizeInBits() != StoreSrcTy.getSizeInBits())
2312 return false;
2313
2314 if (RBI.getRegBank(StoreSrcReg, MRI, TRI) ==
2315 RBI.getRegBank(DefDstReg, MRI, TRI))
2316 return false;
2317
2318 // We have a cross-bank copy, which is entering a store. Let's fold it.
2319 I.getOperand(0).setReg(DefDstReg);
2320 return true;
2321}
2322
2323bool AArch64InstructionSelector::earlySelect(MachineInstr &I) {
2324 assert(I.getParent() && "Instruction should be in a basic block!");
2325 assert(I.getParent()->getParent() && "Instruction should be in a function!");
2326
2327 MachineBasicBlock &MBB = *I.getParent();
2328 MachineFunction &MF = *MBB.getParent();
2329 MachineRegisterInfo &MRI = MF.getRegInfo();
2330
2331 switch (I.getOpcode()) {
2332 case TargetOpcode::G_SPLAT_VECTOR: {
2333 Register Dst = I.getOperand(0).getReg();
2334 LLT DstTy = MRI.getType(Dst);
2335 if (DstTy.isScalableVector())
2336 return false;
2337 // Before selecting a DUP instruction, check if it is better selected as a
2338 // MOV or load from a constant pool.
2339 Register Src = I.getOperand(1).getReg();
2340 auto ValAndVReg = getAnyConstantVRegValWithLookThrough(
2341 Src, MRI, /*LookThroughInstrs=*/true, /*LookThroughAnyExt=*/true);
2342 if (!ValAndVReg)
2343 return false;
2344 LLVMContext &Ctx = MF.getFunction().getContext();
2346 DstTy.getNumElements(),
2347 ConstantInt::get(Type::getIntNTy(Ctx, DstTy.getScalarSizeInBits()),
2348 ValAndVReg->Value.trunc(DstTy.getScalarSizeInBits())));
2349 APInt Imm =
2351 ValAndVReg->Value.trunc(DstTy.getScalarSizeInBits()));
2352 if (!emitConstantVector(Dst, CV, Imm, MRI))
2353 return false;
2354 I.eraseFromParent();
2355 return true;
2356 }
2357 case TargetOpcode::G_SEXT:
2358 // Check for i64 sext(i32 vector_extract) prior to tablegen to select SMOV
2359 // over a normal extend.
2360 if (selectUSMovFromExtend(I, MRI))
2361 return true;
2362 return false;
2363 case TargetOpcode::G_BR:
2364 return false;
2365 case TargetOpcode::G_SHL:
2366 return earlySelectSHL(I, MRI);
2367 case TargetOpcode::G_CONSTANT: {
2368 bool IsZero = false;
2369 if (I.getOperand(1).isCImm())
2370 IsZero = I.getOperand(1).getCImm()->isZero();
2371 else if (I.getOperand(1).isImm())
2372 IsZero = I.getOperand(1).getImm() == 0;
2373
2374 if (!IsZero)
2375 return false;
2376
2377 Register DefReg = I.getOperand(0).getReg();
2378 LLT Ty = MRI.getType(DefReg);
2379 if (Ty.getSizeInBits() == 64) {
2380 I.getOperand(1).ChangeToRegister(AArch64::XZR, false);
2381 RBI.constrainGenericRegister(DefReg, AArch64::GPR64RegClass, MRI);
2382 } else if (Ty.getSizeInBits() <= 32) {
2383 I.getOperand(1).ChangeToRegister(AArch64::WZR, false);
2384 RBI.constrainGenericRegister(DefReg, AArch64::GPR32RegClass, MRI);
2385 } else
2386 return false;
2387
2388 I.setDesc(TII.get(TargetOpcode::COPY));
2389 return true;
2390 }
2391
2392 case TargetOpcode::G_ADD: {
2393 // Check if this is being fed by a G_ICMP on either side.
2394 //
2395 // (cmp pred, x, y) + z
2396 //
2397 // In the above case, when the cmp is true, we increment z by 1. So, we can
2398 // fold the add into the cset for the cmp by using cinc.
2399 //
2400 // FIXME: This would probably be a lot nicer in PostLegalizerLowering.
2401 Register AddDst = I.getOperand(0).getReg();
2402 Register AddLHS = I.getOperand(1).getReg();
2403 Register AddRHS = I.getOperand(2).getReg();
2404 // Only handle scalars.
2405 LLT Ty = MRI.getType(AddLHS);
2406 if (Ty.isVector())
2407 return false;
2408 // Since G_ICMP is modeled as ADDS/SUBS/ANDS, we can handle 32 bits or 64
2409 // bits.
2410 unsigned Size = Ty.getSizeInBits();
2411 if (Size != 32 && Size != 64)
2412 return false;
2413 auto MatchCmp = [&](Register Reg) -> MachineInstr * {
2414 if (!MRI.hasOneNonDBGUse(Reg))
2415 return nullptr;
2416 // If the LHS of the add is 32 bits, then we want to fold a 32-bit
2417 // compare.
2418 if (Size == 32)
2419 return getOpcodeDef(TargetOpcode::G_ICMP, Reg, MRI);
2420 // We model scalar compares using 32-bit destinations right now.
2421 // If it's a 64-bit compare, it'll have 64-bit sources.
2422 Register ZExt;
2423 if (!mi_match(Reg, MRI, m_OneUse(m_GZExt(m_OneUse(m_Reg(ZExt))))))
2424 return nullptr;
2425 auto *Cmp = getOpcodeDef(TargetOpcode::G_ICMP, ZExt, MRI);
2426 if (!Cmp ||
2427 MRI.getType(Cmp->getOperand(2).getReg()).getSizeInBits() != 64)
2428 return nullptr;
2429 return Cmp;
2430 };
2431 // Try to match
2432 // z + (cmp pred, x, y)
2433 MachineInstr *Cmp = MatchCmp(AddRHS);
2434 if (!Cmp) {
2435 // (cmp pred, x, y) + z
2436 std::swap(AddLHS, AddRHS);
2437 Cmp = MatchCmp(AddRHS);
2438 if (!Cmp)
2439 return false;
2440 }
2441 auto &PredOp = Cmp->getOperand(1);
2443 emitIntegerCompare(/*LHS=*/Cmp->getOperand(2),
2444 /*RHS=*/Cmp->getOperand(3), PredOp, MIB);
2445 auto Pred = static_cast<CmpInst::Predicate>(PredOp.getPredicate());
2447 CmpInst::getInversePredicate(Pred), Cmp->getOperand(3).getReg(), &MRI);
2448 emitCSINC(/*Dst=*/AddDst, /*Src =*/AddLHS, /*Src2=*/AddLHS, InvCC, MIB);
2449 I.eraseFromParent();
2450 return true;
2451 }
2452 case TargetOpcode::G_OR: {
2453 // Look for operations that take the lower `Width=Size-ShiftImm` bits of
2454 // `ShiftSrc` and insert them into the upper `Width` bits of `MaskSrc` via
2455 // shifting and masking that we can replace with a BFI (encoded as a BFM).
2456 Register Dst = I.getOperand(0).getReg();
2457 LLT Ty = MRI.getType(Dst);
2458
2459 if (!Ty.isScalar())
2460 return false;
2461
2462 unsigned Size = Ty.getSizeInBits();
2463 if (Size != 32 && Size != 64)
2464 return false;
2465
2466 Register ShiftSrc;
2467 int64_t ShiftImm;
2468 Register MaskSrc;
2469 int64_t MaskImm;
2470 if (!mi_match(Dst, MRI,
2471 m_GOr(m_OneUse(m_GShl(m_Reg(ShiftSrc), m_ICst(ShiftImm))),
2472 m_OneUse(m_GAnd(m_Reg(MaskSrc), m_ICst(MaskImm))))))
2473 return false;
2474
2475 if (ShiftImm > Size || ((1ULL << ShiftImm) - 1ULL) != uint64_t(MaskImm))
2476 return false;
2477
2478 int64_t Immr = Size - ShiftImm;
2479 int64_t Imms = Size - ShiftImm - 1;
2480 unsigned Opc = Size == 32 ? AArch64::BFMWri : AArch64::BFMXri;
2481 emitInstr(Opc, {Dst}, {MaskSrc, ShiftSrc, Immr, Imms}, MIB);
2482 I.eraseFromParent();
2483 return true;
2484 }
2485 case TargetOpcode::G_FENCE: {
2486 if (I.getOperand(1).getImm() == 0)
2487 BuildMI(MBB, I, MIMetadata(I), TII.get(TargetOpcode::MEMBARRIER));
2488 else
2489 BuildMI(MBB, I, MIMetadata(I), TII.get(AArch64::DMB))
2490 .addImm(I.getOperand(0).getImm() == 4 ? 0x9 : 0xb);
2491 I.eraseFromParent();
2492 return true;
2493 }
2494 case TargetOpcode::G_BRINDIRECT: {
2495 const Function &Fn = MF.getFunction();
2496 if (std::optional<uint16_t> BADisc =
2498 auto MI = MIB.buildInstr(AArch64::BRA, {}, {I.getOperand(0).getReg()});
2500 MI.addImm(*BADisc);
2501 MI.addReg(/*AddrDisc=*/AArch64::XZR);
2502 I.eraseFromParent();
2504 return true;
2505 }
2506 // Use table-based selection.
2507 return false;
2508 }
2509
2510 default:
2511 return false;
2512 }
2513}
2514
2515bool AArch64InstructionSelector::select(MachineInstr &I) {
2516 assert(I.getParent() && "Instruction should be in a basic block!");
2517 assert(I.getParent()->getParent() && "Instruction should be in a function!");
2518
2519 MachineBasicBlock &MBB = *I.getParent();
2520 MachineFunction &MF = *MBB.getParent();
2521 MachineRegisterInfo &MRI = MF.getRegInfo();
2522
2523 const AArch64Subtarget *Subtarget = &MF.getSubtarget<AArch64Subtarget>();
2524 if (Subtarget->requiresStrictAlign()) {
2525 // We don't support this feature yet.
2526 LLVM_DEBUG(dbgs() << "AArch64 GISel does not support strict-align yet\n");
2527 return false;
2528 }
2529
2531
2532 unsigned Opcode = I.getOpcode();
2533 // G_PHI requires same handling as PHI
2534 if (!I.isPreISelOpcode() || Opcode == TargetOpcode::G_PHI) {
2535 // Certain non-generic instructions also need some special handling.
2536
2537 if (Opcode == TargetOpcode::LOAD_STACK_GUARD) {
2539 return true;
2540 }
2541
2542 if (Opcode == TargetOpcode::PHI || Opcode == TargetOpcode::G_PHI) {
2543 const Register DefReg = I.getOperand(0).getReg();
2544 const LLT DefTy = MRI.getType(DefReg);
2545
2546 const RegClassOrRegBank &RegClassOrBank =
2547 MRI.getRegClassOrRegBank(DefReg);
2548
2549 const TargetRegisterClass *DefRC =
2551 if (!DefRC) {
2552 if (!DefTy.isValid()) {
2553 LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
2554 return false;
2555 }
2556 const RegisterBank &RB = *cast<const RegisterBank *>(RegClassOrBank);
2557 DefRC = getRegClassForTypeOnBank(DefTy, RB);
2558 if (!DefRC) {
2559 LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
2560 return false;
2561 }
2562 }
2563
2564 I.setDesc(TII.get(TargetOpcode::PHI));
2565
2566 return RBI.constrainGenericRegister(DefReg, *DefRC, MRI);
2567 }
2568
2569 if (I.isCopy())
2570 return selectCopy(I, TII, MRI, TRI, RBI);
2571
2572 return true;
2573 }
2574
2575
2576 if (I.getNumOperands() != I.getNumExplicitOperands()) {
2577 LLVM_DEBUG(
2578 dbgs() << "Generic instruction has unexpected implicit operands\n");
2579 return false;
2580 }
2581
2582 // Try to do some lowering before we start instruction selecting. These
2583 // lowerings are purely transformations on the input G_MIR and so selection
2584 // must continue after any modification of the instruction.
2585 if (preISelLower(I)) {
2586 Opcode = I.getOpcode(); // The opcode may have been modified, refresh it.
2587 }
2588
2589 // There may be patterns where the importer can't deal with them optimally,
2590 // but does select it to a suboptimal sequence so our custom C++ selection
2591 // code later never has a chance to work on it. Therefore, we have an early
2592 // selection attempt here to give priority to certain selection routines
2593 // over the imported ones.
2594 if (earlySelect(I))
2595 return true;
2596
2597 if (selectImpl(I, *CoverageInfo))
2598 return true;
2599
2600 LLT Ty =
2601 I.getOperand(0).isReg() ? MRI.getType(I.getOperand(0).getReg()) : LLT{};
2602
2603 switch (Opcode) {
2604 case TargetOpcode::G_SBFX:
2605 case TargetOpcode::G_UBFX: {
2606 static const unsigned OpcTable[2][2] = {
2607 {AArch64::UBFMWri, AArch64::UBFMXri},
2608 {AArch64::SBFMWri, AArch64::SBFMXri}};
2609 bool IsSigned = Opcode == TargetOpcode::G_SBFX;
2610 unsigned Size = Ty.getSizeInBits();
2611 unsigned Opc = OpcTable[IsSigned][Size == 64];
2612 auto Cst1 =
2613 getIConstantVRegValWithLookThrough(I.getOperand(2).getReg(), MRI);
2614 assert(Cst1 && "Should have gotten a constant for src 1?");
2615 auto Cst2 =
2616 getIConstantVRegValWithLookThrough(I.getOperand(3).getReg(), MRI);
2617 assert(Cst2 && "Should have gotten a constant for src 2?");
2618 auto LSB = Cst1->Value.getZExtValue();
2619 auto Width = Cst2->Value.getZExtValue();
2620 auto BitfieldInst =
2621 MIB.buildInstr(Opc, {I.getOperand(0)}, {I.getOperand(1)})
2622 .addImm(LSB)
2623 .addImm(LSB + Width - 1);
2624 I.eraseFromParent();
2625 constrainSelectedInstRegOperands(*BitfieldInst, TII, TRI, RBI);
2626 return true;
2627 }
2628 case TargetOpcode::G_BRCOND:
2629 return selectCompareBranch(I, MF, MRI);
2630
2631 case TargetOpcode::G_BRJT:
2632 return selectBrJT(I, MRI);
2633
2634 case AArch64::G_ADD_LOW: {
2635 // This op may have been separated from it's ADRP companion by the localizer
2636 // or some other code motion pass. Given that many CPUs will try to
2637 // macro fuse these operations anyway, select this into a MOVaddr pseudo
2638 // which will later be expanded into an ADRP+ADD pair after scheduling.
2639 MachineInstr *BaseMI = MRI.getVRegDef(I.getOperand(1).getReg());
2640 if (BaseMI->getOpcode() != AArch64::ADRP) {
2641 I.setDesc(TII.get(AArch64::ADDXri));
2642 I.addOperand(MachineOperand::CreateImm(0));
2644 return true;
2645 }
2647 "Expected small code model");
2648 auto Op1 = BaseMI->getOperand(1);
2649 auto Op2 = I.getOperand(2);
2650 auto MovAddr = MIB.buildInstr(AArch64::MOVaddr, {I.getOperand(0)}, {})
2651 .addGlobalAddress(Op1.getGlobal(), Op1.getOffset(),
2652 Op1.getTargetFlags())
2653 .addGlobalAddress(Op2.getGlobal(), Op2.getOffset(),
2654 Op2.getTargetFlags());
2655 I.eraseFromParent();
2656 constrainSelectedInstRegOperands(*MovAddr, TII, TRI, RBI);
2657 return true;
2658 }
2659
2660 case TargetOpcode::G_FCONSTANT: {
2661 const Register DefReg = I.getOperand(0).getReg();
2662 const LLT DefTy = MRI.getType(DefReg);
2663 const unsigned DefSize = DefTy.getSizeInBits();
2664 const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
2665
2666 const TargetRegisterClass &FPRRC = *getRegClassForTypeOnBank(DefTy, RB);
2667 assert(
2668 (DefSize == 16 || DefSize == 32 || DefSize == 64 || DefSize == 128) &&
2669 "Unexpected destination size for G_FCONSTANT?");
2670
2671 // Try to emit as a vector movi.
2672 const ConstantFP *FPImm = I.getOperand(1).getFPImm();
2673 APInt Imm = DefSize >= 64
2674 ? FPImm->getValue().bitcastToAPInt()
2675 : APInt::getSplat(64, FPImm->getValue().bitcastToAPInt());
2676 if (auto *Mov = emitConstantMOVVector(
2677 DefSize >= 64 ? DstOp(DefReg) : DstOp(&AArch64::FPR64RegClass), Imm,
2678 MRI)) {
2679 if (DefSize < 64) {
2680 MIB.buildInstr(TargetOpcode::COPY, {DefReg}, {})
2681 .addReg(Mov->getOperand(0).getReg(), {},
2682 DefSize == 16 ? AArch64::hsub : AArch64::ssub);
2683 RBI.constrainGenericRegister(DefReg, FPRRC, MRI);
2684 }
2685 I.eraseFromParent();
2686 return true;
2687 }
2688
2689 // If TLI says that this fpimm is legal then prefer a copy from GPR movimm,
2690 // otherwise we'll expand to a constant pool load
2691 bool OptForSize = shouldOptForSize(&MF);
2692 const auto &TLI = MF.getSubtarget().getTargetLowering();
2693 if ((DefSize == 32 || DefSize == 64) &&
2694 TLI->isFPImmLegal(I.getOperand(1).getFPImm()->getValueAPF(),
2695 EVT::getFloatingPointVT(DefSize), OptForSize)) {
2696 // Emit a copy to emit a normal movimm.
2697 const Register DefGPRReg = MRI.createVirtualRegister(
2698 DefSize == 64 ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass);
2699 MachineOperand &RegOp = I.getOperand(0);
2700 RegOp.setReg(DefGPRReg);
2701 MIB.setInsertPt(MIB.getMBB(), std::next(I.getIterator()));
2702 MIB.buildCopy({DefReg}, {DefGPRReg});
2703
2704 if (!RBI.constrainGenericRegister(DefReg, FPRRC, MRI)) {
2705 LLVM_DEBUG(dbgs() << "Failed to constrain G_FCONSTANT def operand\n");
2706 return false;
2707 }
2708
2709 MachineOperand &ImmOp = I.getOperand(1);
2710 ImmOp.ChangeToImmediate(
2712
2713 const unsigned MovOpc =
2714 DefSize == 64 ? AArch64::MOVi64imm : AArch64::MOVi32imm;
2715 I.setDesc(TII.get(MovOpc));
2717 return true;
2718 }
2719
2720 // Expand to a constant pool.
2721 auto *LoadMI = emitLoadFromConstantPool(FPImm, MIB);
2722 if (!LoadMI) {
2723 LLVM_DEBUG(dbgs() << "Failed to load double constant pool entry\n");
2724 return false;
2725 }
2726 MIB.buildCopy({DefReg}, {LoadMI->getOperand(0).getReg()});
2727 I.eraseFromParent();
2728 return RBI.constrainGenericRegister(DefReg, FPRRC, MRI);
2729 }
2730 case TargetOpcode::G_EXTRACT: {
2731 Register DstReg = I.getOperand(0).getReg();
2732 Register SrcReg = I.getOperand(1).getReg();
2733 LLT SrcTy = MRI.getType(SrcReg);
2734 LLT DstTy = MRI.getType(DstReg);
2735 (void)DstTy;
2736 unsigned SrcSize = SrcTy.getSizeInBits();
2737
2738 if (SrcTy.getSizeInBits() > 64) {
2739 // This should be an extract of an s128, which is like a vector extract.
2740 if (SrcTy.getSizeInBits() != 128)
2741 return false;
2742 // Only support extracting 64 bits from an s128 at the moment.
2743 if (DstTy.getSizeInBits() != 64)
2744 return false;
2745
2746 unsigned Offset = I.getOperand(2).getImm();
2747 if (Offset % 64 != 0)
2748 return false;
2749
2750 // Check we have the right regbank always.
2751 const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
2752 const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
2753 assert(SrcRB.getID() == DstRB.getID() && "Wrong extract regbank!");
2754
2755 if (SrcRB.getID() == AArch64::GPRRegBankID) {
2756 auto NewI =
2757 MIB.buildInstr(TargetOpcode::COPY, {DstReg}, {})
2758 .addUse(SrcReg, {},
2759 Offset == 0 ? AArch64::sube64 : AArch64::subo64);
2760 constrainOperandRegClass(MF, TRI, MRI, TII, RBI, *NewI,
2761 AArch64::GPR64RegClass, NewI->getOperand(0));
2762 I.eraseFromParent();
2763 return true;
2764 }
2765
2766 // Emit the same code as a vector extract.
2767 // Offset must be a multiple of 64.
2768 unsigned LaneIdx = Offset / 64;
2769 MachineInstr *Extract = emitExtractVectorElt(
2770 DstReg, DstRB, LLT::scalar(64), SrcReg, LaneIdx, MIB);
2771 if (!Extract)
2772 return false;
2773 I.eraseFromParent();
2774 return true;
2775 }
2776
2777 I.setDesc(TII.get(SrcSize == 64 ? AArch64::UBFMXri : AArch64::UBFMWri));
2778 MachineInstrBuilder(MF, I).addImm(I.getOperand(2).getImm() +
2779 Ty.getSizeInBits() - 1);
2780
2781 if (SrcSize < 64) {
2782 assert(SrcSize == 32 && DstTy.getSizeInBits() == 16 &&
2783 "unexpected G_EXTRACT types");
2785 return true;
2786 }
2787
2789 MIB.setInsertPt(MIB.getMBB(), std::next(I.getIterator()));
2790 MIB.buildInstr(TargetOpcode::COPY, {I.getOperand(0).getReg()}, {})
2791 .addReg(DstReg, {}, AArch64::sub_32);
2792 RBI.constrainGenericRegister(I.getOperand(0).getReg(),
2793 AArch64::GPR32RegClass, MRI);
2794 I.getOperand(0).setReg(DstReg);
2795
2797 return true;
2798 }
2799
2800 case TargetOpcode::G_INSERT: {
2801 LLT SrcTy = MRI.getType(I.getOperand(2).getReg());
2802 LLT DstTy = MRI.getType(I.getOperand(0).getReg());
2803 unsigned DstSize = DstTy.getSizeInBits();
2804 // Larger inserts are vectors, same-size ones should be something else by
2805 // now (split up or turned into COPYs).
2806 if (Ty.getSizeInBits() > 64 || SrcTy.getSizeInBits() > 32)
2807 return false;
2808
2809 I.setDesc(TII.get(DstSize == 64 ? AArch64::BFMXri : AArch64::BFMWri));
2810 unsigned LSB = I.getOperand(3).getImm();
2811 unsigned Width = MRI.getType(I.getOperand(2).getReg()).getSizeInBits();
2812 I.getOperand(3).setImm((DstSize - LSB) % DstSize);
2813 MachineInstrBuilder(MF, I).addImm(Width - 1);
2814
2815 if (DstSize < 64) {
2816 assert(DstSize == 32 && SrcTy.getSizeInBits() == 16 &&
2817 "unexpected G_INSERT types");
2819 return true;
2820 }
2821
2823 BuildMI(MBB, I.getIterator(), I.getDebugLoc(),
2824 TII.get(AArch64::SUBREG_TO_REG))
2825 .addDef(SrcReg)
2826 .addUse(I.getOperand(2).getReg())
2827 .addImm(AArch64::sub_32);
2828 RBI.constrainGenericRegister(I.getOperand(2).getReg(),
2829 AArch64::GPR32RegClass, MRI);
2830 I.getOperand(2).setReg(SrcReg);
2831
2833 return true;
2834 }
2835 case TargetOpcode::G_FRAME_INDEX: {
2836 // allocas and G_FRAME_INDEX are only supported in addrspace(0).
2837 if (Ty != LLT::pointer(0, 64)) {
2838 LLVM_DEBUG(dbgs() << "G_FRAME_INDEX pointer has type: " << Ty
2839 << ", expected: " << LLT::pointer(0, 64) << '\n');
2840 return false;
2841 }
2842 I.setDesc(TII.get(AArch64::ADDXri));
2843
2844 // MOs for a #0 shifted immediate.
2845 I.addOperand(MachineOperand::CreateImm(0));
2846 I.addOperand(MachineOperand::CreateImm(0));
2847
2849 return true;
2850 }
2851
2852 case TargetOpcode::G_GLOBAL_VALUE: {
2853 const GlobalValue *GV = nullptr;
2854 unsigned OpFlags;
2855 if (I.getOperand(1).isSymbol()) {
2856 OpFlags = I.getOperand(1).getTargetFlags();
2857 // Currently only used by "RtLibUseGOT".
2858 assert(OpFlags == AArch64II::MO_GOT);
2859 } else {
2860 GV = I.getOperand(1).getGlobal();
2861 if (GV->isThreadLocal())
2862 return selectTLSGlobalValue(I, MRI);
2863
2864 OpFlags = STI.ClassifyGlobalReference(GV, TM);
2865 }
2866
2867 if (OpFlags & AArch64II::MO_GOT) {
2868 bool IsGOTSigned = MF.getInfo<AArch64FunctionInfo>()->hasELFSignedGOT();
2869 I.setDesc(TII.get(IsGOTSigned ? AArch64::LOADgotAUTH : AArch64::LOADgot));
2870 I.getOperand(1).setTargetFlags(OpFlags);
2871 I.addImplicitDefUseOperands(MF);
2872 I.setImplicitPhysRegDefsDead();
2873 } else if (TM.getCodeModel() == CodeModel::Large &&
2874 !TM.isPositionIndependent()) {
2875 // Materialize the global using movz/movk instructions.
2876 materializeLargeCMVal(I, GV, OpFlags);
2877 I.eraseFromParent();
2878 return true;
2879 } else if (TM.getCodeModel() == CodeModel::Tiny) {
2880 I.setDesc(TII.get(AArch64::ADR));
2881 I.getOperand(1).setTargetFlags(OpFlags);
2882 } else {
2883 I.setDesc(TII.get(AArch64::MOVaddr));
2884 I.getOperand(1).setTargetFlags(OpFlags | AArch64II::MO_PAGE);
2885 MachineInstrBuilder MIB(MF, I);
2886 MIB.addGlobalAddress(GV, I.getOperand(1).getOffset(),
2888 }
2890 return true;
2891 }
2892
2893 case TargetOpcode::G_PTRAUTH_GLOBAL_VALUE:
2894 return selectPtrAuthGlobalValue(I, MRI);
2895
2896 case TargetOpcode::G_ZEXTLOAD:
2897 case TargetOpcode::G_LOAD:
2898 case TargetOpcode::G_STORE: {
2899 GLoadStore &LdSt = cast<GLoadStore>(I);
2900 bool IsZExtLoad = I.getOpcode() == TargetOpcode::G_ZEXTLOAD;
2901 LLT PtrTy = MRI.getType(LdSt.getPointerReg());
2902
2903 // Can only handle AddressSpace 0, 64-bit pointers.
2904 if (PtrTy != LLT::pointer(0, 64)) {
2905 return false;
2906 }
2907
2908 uint64_t MemSizeInBytes = LdSt.getMemSize().getValue();
2909 unsigned MemSizeInBits = LdSt.getMemSizeInBits().getValue();
2910 AtomicOrdering Order = LdSt.getMMO().getSuccessOrdering();
2911
2912 // Need special instructions for atomics that affect ordering.
2913 if (isStrongerThanMonotonic(Order)) {
2914 assert(!isa<GZExtLoad>(LdSt));
2915 assert(MemSizeInBytes <= 8 &&
2916 "128-bit atomics should already be custom-legalized");
2917
2918 if (isa<GLoad>(LdSt)) {
2919 static constexpr unsigned LDAPROpcodes[] = {
2920 AArch64::LDAPRB, AArch64::LDAPRH, AArch64::LDAPRW, AArch64::LDAPRX};
2921 static constexpr unsigned LDAROpcodes[] = {
2922 AArch64::LDARB, AArch64::LDARH, AArch64::LDARW, AArch64::LDARX};
2923 ArrayRef<unsigned> Opcodes =
2924 STI.hasRCPC() && Order != AtomicOrdering::SequentiallyConsistent
2925 ? LDAPROpcodes
2926 : LDAROpcodes;
2927 I.setDesc(TII.get(Opcodes[Log2_32(MemSizeInBytes)]));
2928 } else {
2929 static constexpr unsigned Opcodes[] = {AArch64::STLRB, AArch64::STLRH,
2930 AArch64::STLRW, AArch64::STLRX};
2931 Register ValReg = LdSt.getReg(0);
2932 if (MRI.getType(ValReg).getSizeInBits() == 64 && MemSizeInBits != 64) {
2933 // Emit a subreg copy of 32 bits.
2934 Register NewVal = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
2935 MIB.buildInstr(TargetOpcode::COPY, {NewVal}, {})
2936 .addReg(I.getOperand(0).getReg(), {}, AArch64::sub_32);
2937 I.getOperand(0).setReg(NewVal);
2938 }
2939 I.setDesc(TII.get(Opcodes[Log2_32(MemSizeInBytes)]));
2940 }
2942 return true;
2943 }
2944
2945#ifndef NDEBUG
2946 const Register PtrReg = LdSt.getPointerReg();
2947 const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, MRI, TRI);
2948 // Check that the pointer register is valid.
2949 assert(PtrRB.getID() == AArch64::GPRRegBankID &&
2950 "Load/Store pointer operand isn't a GPR");
2951 assert(MRI.getType(PtrReg).isPointer() &&
2952 "Load/Store pointer operand isn't a pointer");
2953#endif
2954
2955 const Register ValReg = LdSt.getReg(0);
2956 const RegisterBank &RB = *RBI.getRegBank(ValReg, MRI, TRI);
2957 LLT ValTy = MRI.getType(ValReg);
2958
2959 // The code below doesn't support truncating stores, so we need to split it
2960 // again.
2961 if (isa<GStore>(LdSt) && ValTy.getSizeInBits() > MemSizeInBits &&
2962 RB.getID() == AArch64::FPRRegBankID) {
2963 unsigned SubReg;
2964 LLT MemTy = LdSt.getMMO().getMemoryType();
2965 auto *RC = getRegClassForTypeOnBank(MemTy, RB);
2966 if (!getSubRegForClass(RC, TRI, SubReg))
2967 return false;
2968
2969 // Generate a subreg copy.
2970 auto Copy = MIB.buildInstr(TargetOpcode::COPY, {MemTy}, {})
2971 .addReg(ValReg, {}, SubReg)
2972 .getReg(0);
2973 RBI.constrainGenericRegister(Copy, *RC, MRI);
2974 LdSt.getOperand(0).setReg(Copy);
2975 } else if (isa<GLoad>(LdSt) && ValTy.getSizeInBits() > MemSizeInBits) {
2976 // If this is an any-extending load from the FPR bank, split it into a regular
2977 // load + extend.
2978 if (RB.getID() == AArch64::FPRRegBankID) {
2979 unsigned SubReg;
2980 LLT MemTy = LdSt.getMMO().getMemoryType();
2981 auto *RC = getRegClassForTypeOnBank(MemTy, RB);
2982 if (!getSubRegForClass(RC, TRI, SubReg))
2983 return false;
2984 Register OldDst = LdSt.getReg(0);
2985 Register NewDst =
2987 LdSt.getOperand(0).setReg(NewDst);
2988 MRI.setRegBank(NewDst, RB);
2989 // Generate a SUBREG_TO_REG to extend it.
2990 MIB.setInsertPt(MIB.getMBB(), std::next(LdSt.getIterator()));
2991 MIB.buildInstr(AArch64::SUBREG_TO_REG, {OldDst}, {})
2992 .addUse(NewDst)
2993 .addImm(SubReg);
2994 auto SubRegRC = getRegClassForTypeOnBank(MRI.getType(OldDst), RB);
2995 RBI.constrainGenericRegister(OldDst, *SubRegRC, MRI);
2996 MIB.setInstr(LdSt);
2997 ValTy = MemTy; // This is no longer an extending load.
2998 }
2999 }
3000
3001 // Helper lambda for partially selecting I. Either returns the original
3002 // instruction with an updated opcode, or a new instruction.
3003 auto SelectLoadStoreAddressingMode = [&]() -> MachineInstr * {
3004 bool IsStore = isa<GStore>(I);
3005 const unsigned NewOpc =
3006 selectLoadStoreUIOp(I.getOpcode(), RB.getID(), MemSizeInBits);
3007 if (NewOpc == I.getOpcode())
3008 return nullptr;
3009 // Check if we can fold anything into the addressing mode.
3010 auto AddrModeFns =
3011 selectAddrModeIndexed(I.getOperand(1), MemSizeInBytes);
3012 if (!AddrModeFns) {
3013 // Can't fold anything. Use the original instruction.
3014 I.setDesc(TII.get(NewOpc));
3015 I.addOperand(MachineOperand::CreateImm(0));
3016 return &I;
3017 }
3018
3019 // Folded something. Create a new instruction and return it.
3020 auto NewInst = MIB.buildInstr(NewOpc, {}, {}, I.getFlags());
3021 Register CurValReg = I.getOperand(0).getReg();
3022 IsStore ? NewInst.addUse(CurValReg) : NewInst.addDef(CurValReg);
3023 NewInst.cloneMemRefs(I);
3024 for (auto &Fn : *AddrModeFns)
3025 Fn(NewInst);
3026 I.eraseFromParent();
3027 return &*NewInst;
3028 };
3029
3030 MachineInstr *LoadStore = SelectLoadStoreAddressingMode();
3031 if (!LoadStore)
3032 return false;
3033
3034 // If we're storing a 0, use WZR/XZR.
3035 if (Opcode == TargetOpcode::G_STORE) {
3037 LoadStore->getOperand(0).getReg(), MRI);
3038 if (CVal && CVal->Value == 0) {
3039 switch (LoadStore->getOpcode()) {
3040 case AArch64::STRWui:
3041 case AArch64::STRHHui:
3042 case AArch64::STRBBui:
3043 LoadStore->getOperand(0).setReg(AArch64::WZR);
3044 break;
3045 case AArch64::STRXui:
3046 LoadStore->getOperand(0).setReg(AArch64::XZR);
3047 break;
3048 }
3049 }
3050 }
3051
3052 if (IsZExtLoad || (Opcode == TargetOpcode::G_LOAD &&
3053 ValTy == LLT::scalar(64) && MemSizeInBits == 32)) {
3054 // The any/zextload from a smaller type to i32 should be handled by the
3055 // importer.
3056 if (MRI.getType(LoadStore->getOperand(0).getReg()).getSizeInBits() != 64)
3057 return false;
3058 // If we have an extending load then change the load's type to be a
3059 // narrower reg and zero_extend with SUBREG_TO_REG.
3060 Register LdReg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
3061 Register DstReg = LoadStore->getOperand(0).getReg();
3062 LoadStore->getOperand(0).setReg(LdReg);
3063
3064 MIB.setInsertPt(MIB.getMBB(), std::next(LoadStore->getIterator()));
3065 MIB.buildInstr(AArch64::SUBREG_TO_REG, {DstReg}, {})
3066 .addUse(LdReg)
3067 .addImm(AArch64::sub_32);
3068 constrainSelectedInstRegOperands(*LoadStore, TII, TRI, RBI);
3069 return RBI.constrainGenericRegister(DstReg, AArch64::GPR64allRegClass,
3070 MRI);
3071 }
3072 constrainSelectedInstRegOperands(*LoadStore, TII, TRI, RBI);
3073 return true;
3074 }
3075
3076 case TargetOpcode::G_INDEXED_ZEXTLOAD:
3077 case TargetOpcode::G_INDEXED_SEXTLOAD:
3078 return selectIndexedExtLoad(I, MRI);
3079 case TargetOpcode::G_INDEXED_LOAD:
3080 return selectIndexedLoad(I, MRI);
3081 case TargetOpcode::G_INDEXED_STORE:
3082 return selectIndexedStore(cast<GIndexedStore>(I), MRI);
3083
3084 case TargetOpcode::G_LSHR:
3085 case TargetOpcode::G_ASHR:
3086 if (MRI.getType(I.getOperand(0).getReg()).isVector())
3087 return selectVectorAshrLshr(I, MRI);
3088 [[fallthrough]];
3089 case TargetOpcode::G_SHL: {
3090 if (Opcode == TargetOpcode::G_SHL &&
3091 MRI.getType(I.getOperand(0).getReg()).isVector())
3092 return selectVectorSHL(I, MRI);
3093
3094 // These shifts were legalized to have 64 bit shift amounts because we
3095 // want to take advantage of the selection patterns that assume the
3096 // immediates are s64s, however, selectBinaryOp will assume both operands
3097 // will have the same bit size.
3098 {
3099 Register SrcReg = I.getOperand(1).getReg();
3100 Register ShiftReg = I.getOperand(2).getReg();
3101 const LLT ShiftTy = MRI.getType(ShiftReg);
3102 const LLT SrcTy = MRI.getType(SrcReg);
3103 if (!SrcTy.isVector() && SrcTy.getSizeInBits() == 32 &&
3104 ShiftTy.getSizeInBits() == 64) {
3105 assert(!ShiftTy.isVector() && "unexpected vector shift ty");
3106 // Insert a subregister copy to implement a 64->32 trunc
3107 auto Trunc = MIB.buildInstr(TargetOpcode::COPY, {SrcTy}, {})
3108 .addReg(ShiftReg, {}, AArch64::sub_32);
3109 MRI.setRegBank(Trunc.getReg(0), RBI.getRegBank(AArch64::GPRRegBankID));
3110 I.getOperand(2).setReg(Trunc.getReg(0));
3111 }
3112 }
3113
3114 const unsigned OpSize = Ty.getSizeInBits();
3115 const Register DefReg = I.getOperand(0).getReg();
3116 const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
3117
3118 const unsigned NewOpc = selectBinaryOp(I.getOpcode(), RB.getID(), OpSize);
3119 if (NewOpc == I.getOpcode())
3120 return false;
3121
3122 I.setDesc(TII.get(NewOpc));
3123 // FIXME: Should the type be always reset in setDesc?
3124
3125 // Now that we selected an opcode, we need to constrain the register
3126 // operands to use appropriate classes.
3128 return true;
3129 }
3130 case TargetOpcode::G_PTR_ADD: {
3131 emitADD(I.getOperand(0).getReg(), I.getOperand(1), I.getOperand(2), MIB);
3132 I.eraseFromParent();
3133 return true;
3134 }
3135
3136 case TargetOpcode::G_SADDE:
3137 case TargetOpcode::G_UADDE:
3138 case TargetOpcode::G_SSUBE:
3139 case TargetOpcode::G_USUBE:
3140 case TargetOpcode::G_SADDO:
3141 case TargetOpcode::G_UADDO:
3142 case TargetOpcode::G_SSUBO:
3143 case TargetOpcode::G_USUBO:
3144 return selectOverflowOp(I, MRI);
3145
3146 case TargetOpcode::G_PTRMASK: {
3147 Register MaskReg = I.getOperand(2).getReg();
3148 std::optional<int64_t> MaskVal = getIConstantVRegSExtVal(MaskReg, MRI);
3149 // TODO: Implement arbitrary cases
3150 if (!MaskVal || !isShiftedMask_64(*MaskVal))
3151 return false;
3152
3153 uint64_t Mask = *MaskVal;
3154 I.setDesc(TII.get(AArch64::ANDXri));
3155 I.getOperand(2).ChangeToImmediate(
3157
3159 return true;
3160 }
3161 case TargetOpcode::G_PTRTOINT:
3162 case TargetOpcode::G_TRUNC: {
3163 const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
3164 const LLT SrcTy = MRI.getType(I.getOperand(1).getReg());
3165
3166 const Register DstReg = I.getOperand(0).getReg();
3167 const Register SrcReg = I.getOperand(1).getReg();
3168
3169 const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
3170 const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
3171
3172 if (DstRB.getID() != SrcRB.getID()) {
3173 LLVM_DEBUG(
3174 dbgs() << "G_TRUNC/G_PTRTOINT input/output on different banks\n");
3175 return false;
3176 }
3177
3178 if (DstRB.getID() == AArch64::GPRRegBankID) {
3179 const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(DstTy, DstRB);
3180 if (!DstRC)
3181 return false;
3182
3183 const TargetRegisterClass *SrcRC = getRegClassForTypeOnBank(SrcTy, SrcRB);
3184 if (!SrcRC)
3185 return false;
3186
3187 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
3188 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
3189 LLVM_DEBUG(dbgs() << "Failed to constrain G_TRUNC/G_PTRTOINT\n");
3190 return false;
3191 }
3192
3193 if (DstRC == SrcRC) {
3194 // Nothing to be done
3195 } else if (Opcode == TargetOpcode::G_TRUNC && DstTy == LLT::scalar(32) &&
3196 SrcTy == LLT::scalar(64)) {
3197 llvm_unreachable("TableGen can import this case");
3198 return false;
3199 } else if (DstRC == &AArch64::GPR32RegClass &&
3200 SrcRC == &AArch64::GPR64RegClass) {
3201 I.getOperand(1).setSubReg(AArch64::sub_32);
3202 } else {
3203 LLVM_DEBUG(
3204 dbgs() << "Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n");
3205 return false;
3206 }
3207
3208 I.setDesc(TII.get(TargetOpcode::COPY));
3209 return true;
3210 } else if (DstRB.getID() == AArch64::FPRRegBankID) {
3211 if (DstTy == LLT::fixed_vector(4, 16) &&
3212 SrcTy == LLT::fixed_vector(4, 32)) {
3213 I.setDesc(TII.get(AArch64::XTNv4i16));
3215 return true;
3216 }
3217
3218 if (!SrcTy.isVector() && SrcTy.getSizeInBits() == 128) {
3219 MachineInstr *Extract = emitExtractVectorElt(
3220 DstReg, DstRB, LLT::scalar(DstTy.getSizeInBits()), SrcReg, 0, MIB);
3221 if (!Extract)
3222 return false;
3223 I.eraseFromParent();
3224 return true;
3225 }
3226
3227 // We might have a vector G_PTRTOINT, in which case just emit a COPY.
3228 if (Opcode == TargetOpcode::G_PTRTOINT) {
3229 assert(DstTy.isVector() && "Expected an FPR ptrtoint to be a vector");
3230 I.setDesc(TII.get(TargetOpcode::COPY));
3231 return selectCopy(I, TII, MRI, TRI, RBI);
3232 }
3233 }
3234
3235 return false;
3236 }
3237
3238 case TargetOpcode::G_ANYEXT: {
3239 if (selectUSMovFromExtend(I, MRI))
3240 return true;
3241
3242 const Register DstReg = I.getOperand(0).getReg();
3243 const Register SrcReg = I.getOperand(1).getReg();
3244
3245 const RegisterBank &RBDst = *RBI.getRegBank(DstReg, MRI, TRI);
3246 if (RBDst.getID() != AArch64::GPRRegBankID) {
3247 LLVM_DEBUG(dbgs() << "G_ANYEXT on bank: " << RBDst
3248 << ", expected: GPR\n");
3249 return false;
3250 }
3251
3252 const RegisterBank &RBSrc = *RBI.getRegBank(SrcReg, MRI, TRI);
3253 if (RBSrc.getID() != AArch64::GPRRegBankID) {
3254 LLVM_DEBUG(dbgs() << "G_ANYEXT on bank: " << RBSrc
3255 << ", expected: GPR\n");
3256 return false;
3257 }
3258
3259 const unsigned DstSize = MRI.getType(DstReg).getSizeInBits();
3260
3261 if (DstSize == 0) {
3262 LLVM_DEBUG(dbgs() << "G_ANYEXT operand has no size, not a gvreg?\n");
3263 return false;
3264 }
3265
3266 if (DstSize != 64 && DstSize > 32) {
3267 LLVM_DEBUG(dbgs() << "G_ANYEXT to size: " << DstSize
3268 << ", expected: 32 or 64\n");
3269 return false;
3270 }
3271 // At this point G_ANYEXT is just like a plain COPY, but we need
3272 // to explicitly form the 64-bit value if any.
3273 if (DstSize > 32) {
3274 Register ExtSrc = MRI.createVirtualRegister(&AArch64::GPR64allRegClass);
3275 BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::SUBREG_TO_REG))
3276 .addDef(ExtSrc)
3277 .addUse(SrcReg)
3278 .addImm(AArch64::sub_32);
3279 I.getOperand(1).setReg(ExtSrc);
3280 }
3281 return selectCopy(I, TII, MRI, TRI, RBI);
3282 }
3283
3284 case TargetOpcode::G_ZEXT:
3285 case TargetOpcode::G_SEXT_INREG:
3286 case TargetOpcode::G_SEXT: {
3287 if (selectUSMovFromExtend(I, MRI))
3288 return true;
3289
3290 unsigned Opcode = I.getOpcode();
3291 const bool IsSigned = Opcode != TargetOpcode::G_ZEXT;
3292 const Register DefReg = I.getOperand(0).getReg();
3293 Register SrcReg = I.getOperand(1).getReg();
3294 const LLT DstTy = MRI.getType(DefReg);
3295 const LLT SrcTy = MRI.getType(SrcReg);
3296 unsigned DstSize = DstTy.getSizeInBits();
3297 unsigned SrcSize = SrcTy.getSizeInBits();
3298
3299 // SEXT_INREG has the same src reg size as dst, the size of the value to be
3300 // extended is encoded in the imm.
3301 if (Opcode == TargetOpcode::G_SEXT_INREG)
3302 SrcSize = I.getOperand(2).getImm();
3303
3304 if (DstTy.isVector())
3305 return false; // Should be handled by imported patterns.
3306
3307 assert((*RBI.getRegBank(DefReg, MRI, TRI)).getID() ==
3308 AArch64::GPRRegBankID &&
3309 "Unexpected ext regbank");
3310
3311 MachineInstr *ExtI;
3312
3313 // First check if we're extending the result of a load which has a dest type
3314 // smaller than 32 bits, then this zext is redundant. GPR32 is the smallest
3315 // GPR register on AArch64 and all loads which are smaller automatically
3316 // zero-extend the upper bits. E.g.
3317 // %v(s8) = G_LOAD %p, :: (load 1)
3318 // %v2(s32) = G_ZEXT %v(s8)
3319 if (!IsSigned) {
3320 auto *LoadMI = getOpcodeDef(TargetOpcode::G_LOAD, SrcReg, MRI);
3321 bool IsGPR =
3322 RBI.getRegBank(SrcReg, MRI, TRI)->getID() == AArch64::GPRRegBankID;
3323 if (LoadMI && IsGPR) {
3324 const MachineMemOperand *MemOp = *LoadMI->memoperands_begin();
3325 unsigned BytesLoaded = MemOp->getSize().getValue();
3326 if (BytesLoaded < 4 && SrcTy.getSizeInBytes() == BytesLoaded)
3327 return selectCopy(I, TII, MRI, TRI, RBI);
3328 }
3329
3330 // For the 32-bit -> 64-bit case, we can emit a mov (ORRWrs)
3331 // + SUBREG_TO_REG.
3332 if (IsGPR && SrcSize == 32 && DstSize == 64) {
3333 Register SubregToRegSrc =
3334 MRI.createVirtualRegister(&AArch64::GPR32RegClass);
3335 const Register ZReg = AArch64::WZR;
3336 MIB.buildInstr(AArch64::ORRWrs, {SubregToRegSrc}, {ZReg, SrcReg})
3337 .addImm(0);
3338
3339 MIB.buildInstr(AArch64::SUBREG_TO_REG, {DefReg}, {})
3340 .addUse(SubregToRegSrc)
3341 .addImm(AArch64::sub_32);
3342
3343 if (!RBI.constrainGenericRegister(DefReg, AArch64::GPR64RegClass,
3344 MRI)) {
3345 LLVM_DEBUG(dbgs() << "Failed to constrain G_ZEXT destination\n");
3346 return false;
3347 }
3348
3349 if (!RBI.constrainGenericRegister(SrcReg, AArch64::GPR32RegClass,
3350 MRI)) {
3351 LLVM_DEBUG(dbgs() << "Failed to constrain G_ZEXT source\n");
3352 return false;
3353 }
3354
3355 I.eraseFromParent();
3356 return true;
3357 }
3358 }
3359
3360 if (DstSize == 64) {
3361 if (Opcode != TargetOpcode::G_SEXT_INREG) {
3362 // FIXME: Can we avoid manually doing this?
3363 if (!RBI.constrainGenericRegister(SrcReg, AArch64::GPR32RegClass,
3364 MRI)) {
3365 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(Opcode)
3366 << " operand\n");
3367 return false;
3368 }
3369 SrcReg = MIB.buildInstr(AArch64::SUBREG_TO_REG,
3370 {&AArch64::GPR64RegClass}, {})
3371 .addUse(SrcReg)
3372 .addImm(AArch64::sub_32)
3373 .getReg(0);
3374 }
3375
3376 ExtI = MIB.buildInstr(IsSigned ? AArch64::SBFMXri : AArch64::UBFMXri,
3377 {DefReg}, {SrcReg})
3378 .addImm(0)
3379 .addImm(SrcSize - 1);
3380 } else if (DstSize <= 32) {
3381 ExtI = MIB.buildInstr(IsSigned ? AArch64::SBFMWri : AArch64::UBFMWri,
3382 {DefReg}, {SrcReg})
3383 .addImm(0)
3384 .addImm(SrcSize - 1);
3385 } else {
3386 return false;
3387 }
3388
3390 I.eraseFromParent();
3391 return true;
3392 }
3393
3394 case TargetOpcode::G_FREEZE:
3395 return selectCopy(I, TII, MRI, TRI, RBI);
3396
3397 case TargetOpcode::G_INTTOPTR:
3398 // The importer is currently unable to import pointer types since they
3399 // didn't exist in SelectionDAG.
3400 return selectCopy(I, TII, MRI, TRI, RBI);
3401
3402 case TargetOpcode::G_BITCAST:
3403 // Imported SelectionDAG rules can handle every bitcast except those that
3404 // bitcast from a type to the same regbank. The other exception is bitcasts
3405 // involving pointer types, as SelectionDAG has no knowledge of them.
3406 return selectCopy(I, TII, MRI, TRI, RBI);
3407
3408 case TargetOpcode::G_SELECT: {
3409 auto &Sel = cast<GSelect>(I);
3410 const Register CondReg = Sel.getCondReg();
3411 const Register TReg = Sel.getTrueReg();
3412 const Register FReg = Sel.getFalseReg();
3413
3414 if (tryOptSelect(Sel))
3415 return true;
3416
3417 // Make sure to use an unused vreg instead of wzr, so that the peephole
3418 // optimizations will be able to optimize these.
3419 Register DeadVReg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
3420 auto TstMI = MIB.buildInstr(AArch64::ANDSWri, {DeadVReg}, {CondReg})
3421 .addImm(AArch64_AM::encodeLogicalImmediate(1, 32));
3423 if (!emitSelect(Sel.getReg(0), TReg, FReg, AArch64CC::NE, MIB))
3424 return false;
3425 Sel.eraseFromParent();
3426 return true;
3427 }
3428 case TargetOpcode::G_ICMP: {
3429 if (Ty.isVector())
3430 return false;
3431
3432 if (Ty != LLT::scalar(32)) {
3433 LLVM_DEBUG(dbgs() << "G_ICMP result has type: " << Ty
3434 << ", expected: " << LLT::scalar(32) << '\n');
3435 return false;
3436 }
3437
3438 auto &PredOp = I.getOperand(1);
3439 emitIntegerCompare(I.getOperand(2), I.getOperand(3), PredOp, MIB);
3440 auto Pred = static_cast<CmpInst::Predicate>(PredOp.getPredicate());
3442 CmpInst::getInversePredicate(Pred), I.getOperand(3).getReg(), &MRI);
3443 emitCSINC(/*Dst=*/I.getOperand(0).getReg(), /*Src1=*/AArch64::WZR,
3444 /*Src2=*/AArch64::WZR, InvCC, MIB);
3445 I.eraseFromParent();
3446 return true;
3447 }
3448
3449 case TargetOpcode::G_FCMP: {
3451 static_cast<CmpInst::Predicate>(I.getOperand(1).getPredicate());
3452 if (!emitFPCompare(I.getOperand(2).getReg(), I.getOperand(3).getReg(), MIB,
3453 Pred) ||
3454 !emitCSetForFCmp(I.getOperand(0).getReg(), Pred, MIB))
3455 return false;
3456 I.eraseFromParent();
3457 return true;
3458 }
3459 case TargetOpcode::G_VASTART:
3460 return STI.isTargetDarwin() ? selectVaStartDarwin(I, MF, MRI)
3461 : selectVaStartAAPCS(I, MF, MRI);
3462 case TargetOpcode::G_INTRINSIC:
3463 return selectIntrinsic(I, MRI);
3464 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
3465 return selectIntrinsicWithSideEffects(I, MRI);
3466 case TargetOpcode::G_IMPLICIT_DEF: {
3467 I.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF));
3468 const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
3469 const Register DstReg = I.getOperand(0).getReg();
3470 const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
3471 const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(DstTy, DstRB);
3472 RBI.constrainGenericRegister(DstReg, *DstRC, MRI);
3473 return true;
3474 }
3475 case TargetOpcode::G_BLOCK_ADDR: {
3476 Function *BAFn = I.getOperand(1).getBlockAddress()->getFunction();
3477 if (std::optional<uint16_t> BADisc =
3479 MIB.buildInstr(AArch64::MOVaddrPAC)
3480 .addBlockAddress(I.getOperand(1).getBlockAddress())
3482 .addReg(/*AddrDisc=*/AArch64::XZR)
3483 .addImm(*BADisc)
3484 .constrainAllUses(TII, TRI, RBI);
3485 MIB.buildCopy(I.getOperand(0).getReg(), Register(AArch64::X16));
3486 RBI.constrainGenericRegister(I.getOperand(0).getReg(),
3487 AArch64::GPR64RegClass, MRI);
3488 I.eraseFromParent();
3489 return true;
3490 }
3492 materializeLargeCMVal(I, I.getOperand(1).getBlockAddress(), 0);
3493 I.eraseFromParent();
3494 return true;
3495 } else {
3496 I.setDesc(TII.get(AArch64::MOVaddrBA));
3497 auto MovMI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::MOVaddrBA),
3498 I.getOperand(0).getReg())
3499 .addBlockAddress(I.getOperand(1).getBlockAddress(),
3500 /* Offset */ 0, AArch64II::MO_PAGE)
3502 I.getOperand(1).getBlockAddress(), /* Offset */ 0,
3504 I.eraseFromParent();
3506 return true;
3507 }
3508 }
3509 case TargetOpcode::G_BUILD_VECTOR:
3510 return selectBuildVector(I, MRI);
3511 case TargetOpcode::G_MERGE_VALUES:
3512 return selectMergeValues(I, MRI);
3513 case TargetOpcode::G_UNMERGE_VALUES:
3514 return selectUnmergeValues(I, MRI);
3515 case TargetOpcode::G_SHUFFLE_VECTOR:
3516 return selectShuffleVector(I, MRI);
3517 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
3518 return selectExtractElt(I, MRI);
3519 case TargetOpcode::G_CONCAT_VECTORS:
3520 return selectConcatVectors(I, MRI);
3521 case TargetOpcode::G_JUMP_TABLE:
3522 return selectJumpTable(I, MRI);
3523 case TargetOpcode::G_MEMCPY:
3524 case TargetOpcode::G_MEMCPY_INLINE:
3525 case TargetOpcode::G_MEMMOVE:
3526 case TargetOpcode::G_MEMSET:
3527 case TargetOpcode::G_MEMSET_INLINE:
3528 assert(STI.hasMOPS() && "Shouldn't get here without +mops feature");
3529 return selectMOPS(I, MRI);
3530 }
3531
3532 return false;
3533}
3534
3535bool AArch64InstructionSelector::selectAndRestoreState(MachineInstr &I) {
3536 MachineIRBuilderState OldMIBState = MIB.getState();
3537 bool Success = select(I);
3538 MIB.setState(OldMIBState);
3539 return Success;
3540}
3541
3542bool AArch64InstructionSelector::selectMOPS(MachineInstr &GI,
3543 MachineRegisterInfo &MRI) {
3544 unsigned Mopcode;
3545 switch (GI.getOpcode()) {
3546 case TargetOpcode::G_MEMCPY:
3547 case TargetOpcode::G_MEMCPY_INLINE:
3548 Mopcode = AArch64::MOPSMemoryCopyPseudo;
3549 break;
3550 case TargetOpcode::G_MEMMOVE:
3551 Mopcode = AArch64::MOPSMemoryMovePseudo;
3552 break;
3553 case TargetOpcode::G_MEMSET:
3554 case TargetOpcode::G_MEMSET_INLINE:
3555 // For tagged memset see llvm.aarch64.mops.memset.tag
3556 Mopcode = AArch64::MOPSMemorySetPseudo;
3557 break;
3558 }
3559
3560 auto &DstPtr = GI.getOperand(0);
3561 auto &SrcOrVal = GI.getOperand(1);
3562 auto &Size = GI.getOperand(2);
3563
3564 // Create copies of the registers that can be clobbered.
3565 const Register DstPtrCopy = MRI.cloneVirtualRegister(DstPtr.getReg());
3566 const Register SrcValCopy = MRI.cloneVirtualRegister(SrcOrVal.getReg());
3567 const Register SizeCopy = MRI.cloneVirtualRegister(Size.getReg());
3568
3569 const bool IsSet = Mopcode == AArch64::MOPSMemorySetPseudo;
3570 const auto &SrcValRegClass =
3571 IsSet ? AArch64::GPR64RegClass : AArch64::GPR64commonRegClass;
3572
3573 // Constrain to specific registers
3574 RBI.constrainGenericRegister(DstPtrCopy, AArch64::GPR64commonRegClass, MRI);
3575 RBI.constrainGenericRegister(SrcValCopy, SrcValRegClass, MRI);
3576 RBI.constrainGenericRegister(SizeCopy, AArch64::GPR64RegClass, MRI);
3577
3578 MIB.buildCopy(DstPtrCopy, DstPtr);
3579 MIB.buildCopy(SrcValCopy, SrcOrVal);
3580 MIB.buildCopy(SizeCopy, Size);
3581
3582 // New instruction uses the copied registers because it must update them.
3583 // The defs are not used since they don't exist in G_MEM*. They are still
3584 // tied.
3585 // Note: order of operands is different from G_MEMSET, G_MEMCPY, G_MEMMOVE
3586 Register DefDstPtr = MRI.createVirtualRegister(&AArch64::GPR64commonRegClass);
3587 Register DefSize = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
3588 if (IsSet) {
3589 MIB.buildInstr(Mopcode, {DefDstPtr, DefSize},
3590 {DstPtrCopy, SizeCopy, SrcValCopy})
3591 .setOperandDead(5); // implicit-def $nzcv
3592 } else {
3593 Register DefSrcPtr = MRI.createVirtualRegister(&SrcValRegClass);
3594 MIB.buildInstr(Mopcode, {DefDstPtr, DefSrcPtr, DefSize},
3595 {DstPtrCopy, SrcValCopy, SizeCopy})
3596 .setOperandDead(6); // implicit-def $nzcv
3597 }
3598
3599 GI.eraseFromParent();
3600 return true;
3601}
3602
3603bool AArch64InstructionSelector::selectBrJT(MachineInstr &I,
3604 MachineRegisterInfo &MRI) {
3605 assert(I.getOpcode() == TargetOpcode::G_BRJT && "Expected G_BRJT");
3606 Register JTAddr = I.getOperand(0).getReg();
3607 unsigned JTI = I.getOperand(1).getIndex();
3608 Register Index = I.getOperand(2).getReg();
3609
3610 MF->getInfo<AArch64FunctionInfo>()->setJumpTableEntryInfo(JTI, 4, nullptr);
3611
3612 // With aarch64-jump-table-hardening, we only expand the jump table dispatch
3613 // sequence later, to guarantee the integrity of the intermediate values.
3614 if (MF->getFunction().hasFnAttribute("aarch64-jump-table-hardening")) {
3616 if (STI.isTargetMachO()) {
3617 if (CM != CodeModel::Small && CM != CodeModel::Large)
3618 report_fatal_error("Unsupported code-model for hardened jump-table");
3619 } else {
3620 // Note that COFF support would likely also need JUMP_TABLE_DEBUG_INFO.
3621 assert(STI.isTargetELF() &&
3622 "jump table hardening only supported on MachO/ELF");
3623 if (CM != CodeModel::Small)
3624 report_fatal_error("Unsupported code-model for hardened jump-table");
3625 }
3626
3627 MIB.buildCopy({AArch64::X16}, I.getOperand(2).getReg());
3628 MIB.buildInstr(AArch64::BR_JumpTable)
3629 .addJumpTableIndex(I.getOperand(1).getIndex());
3630 I.eraseFromParent();
3631 return true;
3632 }
3633
3634 Register TargetReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
3635 Register ScratchReg = MRI.createVirtualRegister(&AArch64::GPR64spRegClass);
3636
3637 auto JumpTableInst = MIB.buildInstr(AArch64::JumpTableDest32,
3638 {TargetReg, ScratchReg}, {JTAddr, Index})
3639 .addJumpTableIndex(JTI);
3640 // Save the jump table info.
3641 MIB.buildInstr(TargetOpcode::JUMP_TABLE_DEBUG_INFO, {},
3642 {static_cast<int64_t>(JTI)});
3643 // Build the indirect branch.
3644 MIB.buildInstr(AArch64::BR, {}, {TargetReg});
3645 I.eraseFromParent();
3646 constrainSelectedInstRegOperands(*JumpTableInst, TII, TRI, RBI);
3647 return true;
3648}
3649
3650bool AArch64InstructionSelector::selectJumpTable(MachineInstr &I,
3651 MachineRegisterInfo &MRI) {
3652 assert(I.getOpcode() == TargetOpcode::G_JUMP_TABLE && "Expected jump table");
3653 assert(I.getOperand(1).isJTI() && "Jump table op should have a JTI!");
3654
3655 Register DstReg = I.getOperand(0).getReg();
3656 unsigned JTI = I.getOperand(1).getIndex();
3657 // We generate a MOVaddrJT which will get expanded to an ADRP + ADD later.
3658 auto MovMI =
3659 MIB.buildInstr(AArch64::MOVaddrJT, {DstReg}, {})
3660 .addJumpTableIndex(JTI, AArch64II::MO_PAGE)
3662 I.eraseFromParent();
3664 return true;
3665}
3666
3667bool AArch64InstructionSelector::selectTLSLocalExecELF(
3668 const GlobalValue *GV, MachineInstr &I, MachineRegisterInfo &MRI) {
3669 auto ConstrainRegOps = [&](MachineInstrBuilder MIB) {
3671 };
3673 ConstrainRegOps(MIB.buildInstr(AArch64::MOVbaseTLS, {ThreadBase}, {}));
3674
3675 switch (MF->getTarget().Options.TLSSize) {
3676 default:
3677 llvm_unreachable("Unexpected TLS size");
3678 case 12: {
3679 // add x0, x0, :tprel_lo12:a
3680 ConstrainRegOps(
3681 MIB.buildInstr(AArch64::ADDXri, {I.getOperand(0).getReg()},
3682 {ThreadBase})
3684 .addImm(0));
3685 break;
3686 }
3687 case 24: {
3688 // add x0, x0, :tprel_hi12:a
3689 // add x0, x0, :tprel_lo12_nc:a
3690 Register Addr = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
3691 ConstrainRegOps(
3692 MIB.buildInstr(AArch64::ADDXri, {Addr}, {ThreadBase})
3694 .addImm(0));
3695 ConstrainRegOps(
3696 MIB.buildInstr(AArch64::ADDXri, {I.getOperand(0).getReg()}, {Addr})
3697 .addGlobalAddress(GV, 0,
3700 .addImm(0));
3701 break;
3702 }
3703 case 32: {
3704 // movz x0, #:tprel_g1:a
3705 // movk x0, #:tprel_g0_nc:a
3706 // add x0, x1, x0
3707 Register Addr = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
3708 ConstrainRegOps(
3709 MIB.buildInstr(AArch64::MOVZXi, {Addr}, {})
3711 .addImm(16));
3712 Register Addr2 = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
3713 ConstrainRegOps(MIB.buildInstr(AArch64::MOVKXi, {Addr2}, {Addr})
3714 .addGlobalAddress(GV, 0,
3717 .addImm(0));
3718 ConstrainRegOps(MIB.buildInstr(AArch64::ADDXrr, {I.getOperand(0).getReg()},
3719 {ThreadBase, Addr2}));
3720 break;
3721 }
3722 case 48: {
3723 // movz x0, #:tprel_g2:a
3724 // movk x0, #:tprel_g1_nc:a
3725 // movk x0, #:tprel_g0_nc:a
3726 // add x0, x1, x0
3727 Register Addr = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
3728 ConstrainRegOps(
3729 MIB.buildInstr(AArch64::MOVZXi, {Addr}, {})
3731 .addImm(32));
3732 Register Addr2 = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
3733 ConstrainRegOps(MIB.buildInstr(AArch64::MOVKXi, {Addr2}, {Addr})
3734 .addGlobalAddress(GV, 0,
3737 .addImm(16));
3738 Register Addr3 = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
3739 ConstrainRegOps(MIB.buildInstr(AArch64::MOVKXi, {Addr3}, {Addr2})
3740 .addGlobalAddress(GV, 0,
3743 .addImm(0));
3744 ConstrainRegOps(MIB.buildInstr(AArch64::ADDXrr, {I.getOperand(0).getReg()},
3745 {ThreadBase, Addr3}));
3746 break;
3747 }
3748 }
3749 I.eraseFromParent();
3750 return true;
3751}
3752
3753// TLS lowering below mirrors the corresponding DAGISel implementation.
3754// See LowerELFTLSModel() for details.
3755bool AArch64InstructionSelector::selectTLSGlobalValueELF(
3756 MachineInstr &I, MachineRegisterInfo &MRI) {
3757 const GlobalValue *GV = I.getOperand(1).getGlobal();
3758 auto *FuncInfo = MF->getInfo<AArch64FunctionInfo>();
3760 AArch64::getELFTLSModel(GV, TM, FuncInfo->hasELFSignedGOT());
3761
3762 Register TPOff = MRI.createVirtualRegister(&AArch64::GPR64commonRegClass);
3763 switch (Model) {
3765 return selectTLSLocalExecELF(GV, I, MRI);
3767 MIB.buildInstr(AArch64::LOADgot, {TPOff}, {})
3768 .addGlobalAddress(GV, 0, AArch64II::MO_TLS);
3769 break;
3772#ifndef NDEBUG
3773 SMEAttrs Attrs = MF->getInfo<AArch64FunctionInfo>()->getSMEFnAttrs();
3774 assert(!Attrs.hasZAState() && !Attrs.hasStreamingInterfaceOrBody() &&
3775 !Attrs.hasStreamingCompatibleInterface() &&
3776 "unsupported SME features reached GlobalISel TLS lowering");
3777#endif
3778 unsigned Opcode = FuncInfo->hasELFSignedGOT()
3779 ? AArch64::TLSDESC_AUTH_CALLSEQ
3780 : AArch64::TLSDESC_CALLSEQ;
3781
3782 if (TLSModel::GeneralDynamic == Model) {
3783 MIB.buildInstr(Opcode, {}, {}).addGlobalAddress(GV, 0, AArch64II::MO_TLS);
3784 MIB.buildCopy(TPOff, Register(AArch64::X0));
3785 break;
3786 }
3788 // These accesses will need deduplicating if there's more than one.
3790
3791 MIB.buildInstr(Opcode, {}, {})
3792 .addExternalSymbol("_TLS_MODULE_BASE_", AArch64II::MO_TLS);
3793 auto Copy = MIB.buildCopy(LLT::scalar(64), Register(AArch64::X0));
3794 auto Add1 =
3795 MIB.buildInstr(AArch64::ADDXri, {LLT::scalar(64)}, {Copy.getReg(0)})
3796 .addGlobalAddress(GV, 0, AArch64II::MO_TLS | AArch64II::MO_HI12)
3797 .addImm(0);
3798 auto Add2 =
3799 MIB.buildInstr(AArch64::ADDXri, {TPOff}, {Add1.getReg(0)})
3800 .addGlobalAddress(GV, 0,
3803 .addImm(0);
3806 }
3807 }
3809 MIB.buildInstr(AArch64::MOVbaseTLS, {ThreadBase}, {});
3810 auto Add = MIB.buildInstr(AArch64::ADDXrr, {I.getOperand(0).getReg()},
3811 {ThreadBase, TPOff});
3813
3814 I.eraseFromParent();
3815 return true;
3816}
3817
3818bool AArch64InstructionSelector::selectTLSGlobalValueMachO(
3819 MachineInstr &I, MachineRegisterInfo &MRI) {
3820 const auto &GlobalOp = I.getOperand(1);
3821 assert(GlobalOp.getOffset() == 0 &&
3822 "Shouldn't have an offset on TLS globals!");
3823
3824 const GlobalValue &GV = *GlobalOp.getGlobal();
3825 MF->getFrameInfo().setAdjustsStack(true);
3826 auto LoadGOT =
3827 MIB.buildInstr(AArch64::LOADgot, {&AArch64::GPR64commonRegClass}, {})
3828 .addGlobalAddress(&GV, 0, AArch64II::MO_TLS);
3829
3830 auto Load = MIB.buildInstr(AArch64::LDRXui, {&AArch64::GPR64commonRegClass},
3831 {LoadGOT.getReg(0)})
3832 .addImm(0);
3833
3834 MIB.buildCopy(Register(AArch64::X0), LoadGOT.getReg(0));
3835 // TLS calls preserve all registers except those that absolutely must be
3836 // trashed: X0 (it takes an argument), LR (it's a call) and NZCV (let's not be
3837 // silly).
3838 unsigned Opcode = getBLRCallOpcode(*MF);
3839
3840 // With ptrauth-calls, the tlv access thunk pointer is authenticated (IA, 0).
3841 if (MF->getFunction().hasFnAttribute("ptrauth-calls")) {
3842 assert(Opcode == AArch64::BLR);
3843 Opcode = AArch64::BLRAAZ;
3844 }
3845
3846 MIB.buildInstr(Opcode, {}, {Load})
3847 .setOperandDead(1) // implicit-def $lr
3848 .addUse(AArch64::X0, RegState::Implicit)
3849 .addDef(AArch64::X0, RegState::Implicit)
3850 .addRegMask(TRI.getTLSCallPreservedMask());
3851
3852 MIB.buildCopy(I.getOperand(0).getReg(), Register(AArch64::X0));
3853 RBI.constrainGenericRegister(I.getOperand(0).getReg(), AArch64::GPR64RegClass,
3854 MRI);
3855 I.eraseFromParent();
3856 return true;
3857}
3858
3859bool AArch64InstructionSelector::selectTLSGlobalValue(
3860 MachineInstr &I, MachineRegisterInfo &MRI) {
3861 // We don't support instructions with emulated TLS variables yet.
3862 if (TM.useEmulatedTLS())
3863 return false;
3864
3865 if (STI.isTargetELF())
3866 return selectTLSGlobalValueELF(I, MRI);
3867
3868 if (STI.isTargetMachO())
3869 return selectTLSGlobalValueMachO(I, MRI);
3870
3871 return false;
3872}
3873
3874MachineInstr *AArch64InstructionSelector::emitScalarToVector(
3875 unsigned EltSize, const TargetRegisterClass *DstRC, Register Scalar,
3876 MachineIRBuilder &MIRBuilder) const {
3877 auto Undef = MIRBuilder.buildInstr(TargetOpcode::IMPLICIT_DEF, {DstRC}, {});
3878
3879 auto BuildFn = [&](unsigned SubregIndex) {
3880 auto Ins =
3881 MIRBuilder
3882 .buildInstr(TargetOpcode::INSERT_SUBREG, {DstRC}, {Undef, Scalar})
3883 .addImm(SubregIndex);
3886 return &*Ins;
3887 };
3888
3889 switch (EltSize) {
3890 case 8:
3891 return BuildFn(AArch64::bsub);
3892 case 16:
3893 return BuildFn(AArch64::hsub);
3894 case 32:
3895 return BuildFn(AArch64::ssub);
3896 case 64:
3897 return BuildFn(AArch64::dsub);
3898 default:
3899 return nullptr;
3900 }
3901}
3902
3903MachineInstr *
3904AArch64InstructionSelector::emitNarrowVector(Register DstReg, Register SrcReg,
3905 MachineIRBuilder &MIB,
3906 MachineRegisterInfo &MRI) const {
3907 LLT DstTy = MRI.getType(DstReg);
3908 const TargetRegisterClass *RC =
3909 getRegClassForTypeOnBank(DstTy, *RBI.getRegBank(SrcReg, MRI, TRI));
3910 if (RC != &AArch64::FPR32RegClass && RC != &AArch64::FPR64RegClass) {
3911 LLVM_DEBUG(dbgs() << "Unsupported register class!\n");
3912 return nullptr;
3913 }
3914 unsigned SubReg = 0;
3915 if (!getSubRegForClass(RC, TRI, SubReg))
3916 return nullptr;
3917 if (SubReg != AArch64::ssub && SubReg != AArch64::dsub) {
3918 LLVM_DEBUG(dbgs() << "Unsupported destination size! ("
3919 << DstTy.getSizeInBits() << "\n");
3920 return nullptr;
3921 }
3922 auto Copy = MIB.buildInstr(TargetOpcode::COPY, {DstReg}, {})
3923 .addReg(SrcReg, {}, SubReg);
3924 RBI.constrainGenericRegister(DstReg, *RC, MRI);
3925 return Copy;
3926}
3927
3928bool AArch64InstructionSelector::selectMergeValues(
3929 MachineInstr &I, MachineRegisterInfo &MRI) {
3930 assert(I.getOpcode() == TargetOpcode::G_MERGE_VALUES && "unexpected opcode");
3931 const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
3932 const LLT SrcTy = MRI.getType(I.getOperand(1).getReg());
3933 assert(!DstTy.isVector() && !SrcTy.isVector() && "invalid merge operation");
3934 const RegisterBank &RB = *RBI.getRegBank(I.getOperand(1).getReg(), MRI, TRI);
3935
3936 if (I.getNumOperands() != 3)
3937 return false;
3938
3939 // Merging 2 s64s into an s128.
3940 if (DstTy == LLT::scalar(128)) {
3941 if (SrcTy.getSizeInBits() != 64)
3942 return false;
3943 Register DstReg = I.getOperand(0).getReg();
3944 Register Src1Reg = I.getOperand(1).getReg();
3945 Register Src2Reg = I.getOperand(2).getReg();
3946 auto Tmp = MIB.buildInstr(TargetOpcode::IMPLICIT_DEF, {DstTy}, {});
3947 MachineInstr *InsMI = emitLaneInsert(std::nullopt, Tmp.getReg(0), Src1Reg,
3948 /* LaneIdx */ 0, RB, MIB);
3949 if (!InsMI)
3950 return false;
3951 MachineInstr *Ins2MI = emitLaneInsert(DstReg, InsMI->getOperand(0).getReg(),
3952 Src2Reg, /* LaneIdx */ 1, RB, MIB);
3953 if (!Ins2MI)
3954 return false;
3957 I.eraseFromParent();
3958 return true;
3959 }
3960
3961 if (RB.getID() != AArch64::GPRRegBankID)
3962 return false;
3963
3964 if (DstTy.getSizeInBits() != 64 || SrcTy.getSizeInBits() != 32)
3965 return false;
3966
3967 auto *DstRC = &AArch64::GPR64RegClass;
3968 Register SubToRegDef = MRI.createVirtualRegister(DstRC);
3969 MachineInstr &SubRegMI = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
3970 TII.get(TargetOpcode::SUBREG_TO_REG))
3971 .addDef(SubToRegDef)
3972 .addUse(I.getOperand(1).getReg())
3973 .addImm(AArch64::sub_32);
3974 Register SubToRegDef2 = MRI.createVirtualRegister(DstRC);
3975 // Need to anyext the second scalar before we can use bfm
3976 MachineInstr &SubRegMI2 = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
3977 TII.get(TargetOpcode::SUBREG_TO_REG))
3978 .addDef(SubToRegDef2)
3979 .addUse(I.getOperand(2).getReg())
3980 .addImm(AArch64::sub_32);
3981 MachineInstr &BFM =
3982 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::BFMXri))
3983 .addDef(I.getOperand(0).getReg())
3984 .addUse(SubToRegDef)
3985 .addUse(SubToRegDef2)
3986 .addImm(32)
3987 .addImm(31);
3988 constrainSelectedInstRegOperands(SubRegMI, TII, TRI, RBI);
3989 constrainSelectedInstRegOperands(SubRegMI2, TII, TRI, RBI);
3991 I.eraseFromParent();
3992 return true;
3993}
3994
3995static bool getLaneCopyOpcode(unsigned &CopyOpc, unsigned &ExtractSubReg,
3996 const unsigned EltSize) {
3997 // Choose a lane copy opcode and subregister based off of the size of the
3998 // vector's elements.
3999 switch (EltSize) {
4000 case 8:
4001 CopyOpc = AArch64::DUPi8;
4002 ExtractSubReg = AArch64::bsub;
4003 break;
4004 case 16:
4005 CopyOpc = AArch64::DUPi16;
4006 ExtractSubReg = AArch64::hsub;
4007 break;
4008 case 32:
4009 CopyOpc = AArch64::DUPi32;
4010 ExtractSubReg = AArch64::ssub;
4011 break;
4012 case 64:
4013 CopyOpc = AArch64::DUPi64;
4014 ExtractSubReg = AArch64::dsub;
4015 break;
4016 default:
4017 // Unknown size, bail out.
4018 LLVM_DEBUG(dbgs() << "Elt size '" << EltSize << "' unsupported.\n");
4019 return false;
4020 }
4021 return true;
4022}
4023
4024MachineInstr *AArch64InstructionSelector::emitExtractVectorElt(
4025 std::optional<Register> DstReg, const RegisterBank &DstRB, LLT ScalarTy,
4026 Register VecReg, unsigned LaneIdx, MachineIRBuilder &MIRBuilder) const {
4027 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4028 unsigned CopyOpc = 0;
4029 unsigned ExtractSubReg = 0;
4030 if (!getLaneCopyOpcode(CopyOpc, ExtractSubReg, ScalarTy.getSizeInBits())) {
4031 LLVM_DEBUG(
4032 dbgs() << "Couldn't determine lane copy opcode for instruction.\n");
4033 return nullptr;
4034 }
4035
4036 const TargetRegisterClass *DstRC =
4037 getRegClassForTypeOnBank(ScalarTy, DstRB, true);
4038 if (!DstRC) {
4039 LLVM_DEBUG(dbgs() << "Could not determine destination register class.\n");
4040 return nullptr;
4041 }
4042
4043 const RegisterBank &VecRB = *RBI.getRegBank(VecReg, MRI, TRI);
4044 const LLT &VecTy = MRI.getType(VecReg);
4045 const TargetRegisterClass *VecRC =
4046 getRegClassForTypeOnBank(VecTy, VecRB, true);
4047 if (!VecRC) {
4048 LLVM_DEBUG(dbgs() << "Could not determine source register class.\n");
4049 return nullptr;
4050 }
4051
4052 // The register that we're going to copy into.
4053 Register InsertReg = VecReg;
4054 if (!DstReg)
4055 DstReg = MRI.createVirtualRegister(DstRC);
4056 // If the lane index is 0, we just use a subregister COPY.
4057 if (LaneIdx == 0) {
4058 auto Copy = MIRBuilder.buildInstr(TargetOpcode::COPY, {*DstReg}, {})
4059 .addReg(VecReg, {}, ExtractSubReg);
4060 RBI.constrainGenericRegister(*DstReg, *DstRC, MRI);
4061 return &*Copy;
4062 }
4063
4064 // Lane copies require 128-bit wide registers. If we're dealing with an
4065 // unpacked vector, then we need to move up to that width. Insert an implicit
4066 // def and a subregister insert to get us there.
4067 if (VecTy.getSizeInBits() != 128) {
4068 MachineInstr *ScalarToVector = emitScalarToVector(
4069 VecTy.getSizeInBits(), &AArch64::FPR128RegClass, VecReg, MIRBuilder);
4070 if (!ScalarToVector)
4071 return nullptr;
4072 InsertReg = ScalarToVector->getOperand(0).getReg();
4073 }
4074
4075 MachineInstr *LaneCopyMI =
4076 MIRBuilder.buildInstr(CopyOpc, {*DstReg}, {InsertReg}).addImm(LaneIdx);
4077 constrainSelectedInstRegOperands(*LaneCopyMI, TII, TRI, RBI);
4078
4079 // Make sure that we actually constrain the initial copy.
4080 RBI.constrainGenericRegister(*DstReg, *DstRC, MRI);
4081 return LaneCopyMI;
4082}
4083
4084bool AArch64InstructionSelector::selectExtractElt(
4085 MachineInstr &I, MachineRegisterInfo &MRI) {
4086 assert(I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT &&
4087 "unexpected opcode!");
4088 Register DstReg = I.getOperand(0).getReg();
4089 const LLT NarrowTy = MRI.getType(DstReg);
4090 const Register SrcReg = I.getOperand(1).getReg();
4091 const LLT WideTy = MRI.getType(SrcReg);
4092 assert(WideTy.getSizeInBits() >= NarrowTy.getSizeInBits() &&
4093 "source register size too small!");
4094 assert(!NarrowTy.isVector() && "cannot extract vector into vector!");
4095
4096 // Need the lane index to determine the correct copy opcode.
4097 MachineOperand &LaneIdxOp = I.getOperand(2);
4098 assert(LaneIdxOp.isReg() && "Lane index operand was not a register?");
4099
4100 // Find the index to extract from.
4101 auto VRegAndVal = getIConstantVRegValWithLookThrough(LaneIdxOp.getReg(), MRI);
4102 if (!VRegAndVal)
4103 return false;
4104 unsigned LaneIdx = VRegAndVal->Value.getSExtValue();
4105
4106 const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
4107 if (DstRB.getID() == AArch64::GPRRegBankID) {
4108 unsigned Opcode;
4109 switch (WideTy.getScalarSizeInBits()) {
4110 case 8:
4111 Opcode = AArch64::UMOVvi8;
4112 break;
4113 case 16:
4114 Opcode = AArch64::UMOVvi16;
4115 break;
4116 case 32:
4117 Opcode = AArch64::UMOVvi32;
4118 break;
4119 default:
4120 return false;
4121 }
4122
4123 if (WideTy.getSizeInBits() != 128) {
4124 MachineInstr *ScalarToVector = emitScalarToVector(
4125 WideTy.getSizeInBits(), &AArch64::FPR128RegClass, SrcReg, MIB);
4126 assert(ScalarToVector && "Didn't expect emitScalarToVector to fail!");
4127 I.getOperand(1).setReg(ScalarToVector->getOperand(0).getReg());
4128 }
4129
4130 I.setDesc(TII.get(Opcode));
4131 I.getOperand(2).ChangeToImmediate(LaneIdx);
4133 return true;
4134 }
4135
4136 MachineInstr *Extract = emitExtractVectorElt(DstReg, DstRB, NarrowTy, SrcReg,
4137 LaneIdx, MIB);
4138 if (!Extract)
4139 return false;
4140
4141 I.eraseFromParent();
4142 return true;
4143}
4144
4145bool AArch64InstructionSelector::selectSplitVectorUnmerge(
4146 MachineInstr &I, MachineRegisterInfo &MRI) {
4147 unsigned NumElts = I.getNumOperands() - 1;
4148 Register SrcReg = I.getOperand(NumElts).getReg();
4149 const LLT NarrowTy = MRI.getType(I.getOperand(0).getReg());
4150 const LLT SrcTy = MRI.getType(SrcReg);
4151
4152 assert(NarrowTy.isVector() && "Expected an unmerge into vectors");
4153 if (SrcTy.getSizeInBits() > 128) {
4154 LLVM_DEBUG(dbgs() << "Unexpected vector type for vec split unmerge");
4155 return false;
4156 }
4157
4158 // We implement a split vector operation by treating the sub-vectors as
4159 // scalars and extracting them.
4160 const RegisterBank &DstRB =
4161 *RBI.getRegBank(I.getOperand(0).getReg(), MRI, TRI);
4162 for (unsigned OpIdx = 0; OpIdx < NumElts; ++OpIdx) {
4163 Register Dst = I.getOperand(OpIdx).getReg();
4164 MachineInstr *Extract =
4165 emitExtractVectorElt(Dst, DstRB, NarrowTy, SrcReg, OpIdx, MIB);
4166 if (!Extract)
4167 return false;
4168 }
4169 I.eraseFromParent();
4170 return true;
4171}
4172
4173bool AArch64InstructionSelector::selectUnmergeValues(MachineInstr &I,
4174 MachineRegisterInfo &MRI) {
4175 assert(I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
4176 "unexpected opcode");
4177
4178 // The last operand is the vector source register, and every other operand is
4179 // a register to unpack into.
4180 unsigned NumElts = I.getNumOperands() - 1;
4181 Register SrcReg = I.getOperand(NumElts).getReg();
4182 Register LoReg = I.getOperand(0).getReg();
4183 Register HiReg = I.getOperand(1).getReg();
4184 const LLT NarrowTy = MRI.getType(LoReg);
4185 const LLT WideTy = MRI.getType(SrcReg);
4186 const RegisterBank &LoRB = *RBI.getRegBank(LoReg, MRI, TRI);
4187 const RegisterBank &HiRB = *RBI.getRegBank(HiReg, MRI, TRI);
4188 const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
4189
4190 // Handle unmerging a 128-bit FPR value into two 64-bit GPR values.
4191 if (NarrowTy == LLT::scalar(64) && WideTy == LLT::scalar(128) &&
4192 LoRB.getID() == AArch64::GPRRegBankID &&
4193 HiRB.getID() == AArch64::GPRRegBankID &&
4194 SrcRB.getID() == AArch64::FPRRegBankID) {
4195 MachineInstr &Lo = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
4196 TII.get(AArch64::UMOVvi64), LoReg)
4197 .addUse(SrcReg)
4198 .addImm(0);
4199 MachineInstr &Hi = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
4200 TII.get(AArch64::UMOVvi64), HiReg)
4201 .addUse(SrcReg)
4202 .addImm(1);
4205 I.eraseFromParent();
4206 return true;
4207 }
4208
4209 // TODO: Handle other unmerges into GPRs and from scalars to scalars.
4210 if (LoRB.getID() != AArch64::FPRRegBankID ||
4211 HiRB.getID() != AArch64::FPRRegBankID) {
4212 LLVM_DEBUG(dbgs() << "Unmerging vector-to-gpr and scalar-to-scalar "
4213 "currently unsupported.\n");
4214 return false;
4215 }
4216
4217 assert(WideTy.getSizeInBits() > NarrowTy.getSizeInBits() &&
4218 "source register size too small!");
4219
4220 if (!NarrowTy.isScalar())
4221 return selectSplitVectorUnmerge(I, MRI);
4222
4223 // Choose a lane copy opcode and subregister based off of the size of the
4224 // vector's elements.
4225 unsigned CopyOpc = 0;
4226 unsigned ExtractSubReg = 0;
4227 if (!getLaneCopyOpcode(CopyOpc, ExtractSubReg, NarrowTy.getSizeInBits()))
4228 return false;
4229
4230 // Set up for the lane copies.
4231 MachineBasicBlock &MBB = *I.getParent();
4232
4233 // Stores the registers we'll be copying from.
4234 SmallVector<Register, 4> InsertRegs;
4235
4236 // We'll use the first register twice, so we only need NumElts-1 registers.
4237 unsigned NumInsertRegs = NumElts - 1;
4238
4239 // If our elements fit into exactly 128 bits, then we can copy from the source
4240 // directly. Otherwise, we need to do a bit of setup with some subregister
4241 // inserts.
4242 if (NarrowTy.getSizeInBits() * NumElts == 128) {
4243 InsertRegs.assign(NumInsertRegs, SrcReg);
4244 } else {
4245 // No. We have to perform subregister inserts. For each insert, create an
4246 // implicit def and a subregister insert, and save the register we create.
4247 // For scalar sources, treat as a pseudo-vector of NarrowTy elements.
4248 unsigned EltSize = WideTy.isVector() ? WideTy.getScalarSizeInBits()
4249 : NarrowTy.getSizeInBits();
4250 const TargetRegisterClass *RC = getRegClassForTypeOnBank(
4251 LLT::fixed_vector(NumElts, EltSize), *RBI.getRegBank(SrcReg, MRI, TRI));
4252 unsigned SubReg = 0;
4253 bool Found = getSubRegForClass(RC, TRI, SubReg);
4254 (void)Found;
4255 assert(Found && "expected to find last operand's subeg idx");
4256 for (unsigned Idx = 0; Idx < NumInsertRegs; ++Idx) {
4257 Register ImpDefReg = MRI.createVirtualRegister(&AArch64::FPR128RegClass);
4258 MachineInstr &ImpDefMI =
4259 *BuildMI(MBB, I, I.getDebugLoc(), TII.get(TargetOpcode::IMPLICIT_DEF),
4260 ImpDefReg);
4261
4262 // Now, create the subregister insert from SrcReg.
4263 Register InsertReg = MRI.createVirtualRegister(&AArch64::FPR128RegClass);
4264 MachineInstr &InsMI =
4265 *BuildMI(MBB, I, I.getDebugLoc(),
4266 TII.get(TargetOpcode::INSERT_SUBREG), InsertReg)
4267 .addUse(ImpDefReg)
4268 .addUse(SrcReg)
4269 .addImm(SubReg);
4270
4271 constrainSelectedInstRegOperands(ImpDefMI, TII, TRI, RBI);
4273
4274 // Save the register so that we can copy from it after.
4275 InsertRegs.push_back(InsertReg);
4276 }
4277 }
4278
4279 // Now that we've created any necessary subregister inserts, we can
4280 // create the copies.
4281 //
4282 // Perform the first copy separately as a subregister copy.
4283 Register CopyTo = I.getOperand(0).getReg();
4284 auto FirstCopy = MIB.buildInstr(TargetOpcode::COPY, {CopyTo}, {})
4285 .addReg(InsertRegs[0], {}, ExtractSubReg);
4286 constrainSelectedInstRegOperands(*FirstCopy, TII, TRI, RBI);
4287
4288 // Now, perform the remaining copies as vector lane copies.
4289 unsigned LaneIdx = 1;
4290 for (Register InsReg : InsertRegs) {
4291 Register CopyTo = I.getOperand(LaneIdx).getReg();
4292 MachineInstr &CopyInst =
4293 *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CopyOpc), CopyTo)
4294 .addUse(InsReg)
4295 .addImm(LaneIdx);
4296 constrainSelectedInstRegOperands(CopyInst, TII, TRI, RBI);
4297 ++LaneIdx;
4298 }
4299
4300 // Separately constrain the first copy's destination. Because of the
4301 // limitation in constrainOperandRegClass, we can't guarantee that this will
4302 // actually be constrained. So, do it ourselves using the second operand.
4303 const TargetRegisterClass *RC =
4304 MRI.getRegClassOrNull(I.getOperand(1).getReg());
4305 if (!RC) {
4306 LLVM_DEBUG(dbgs() << "Couldn't constrain copy destination.\n");
4307 return false;
4308 }
4309
4310 RBI.constrainGenericRegister(CopyTo, *RC, MRI);
4311 I.eraseFromParent();
4312 return true;
4313}
4314
4315bool AArch64InstructionSelector::selectConcatVectors(
4316 MachineInstr &I, MachineRegisterInfo &MRI) {
4317 assert(I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS &&
4318 "Unexpected opcode");
4319 Register Dst = I.getOperand(0).getReg();
4320 Register Op1 = I.getOperand(1).getReg();
4321 Register Op2 = I.getOperand(2).getReg();
4322 MachineInstr *ConcatMI = emitVectorConcat(Dst, Op1, Op2, MIB);
4323 if (!ConcatMI)
4324 return false;
4325 I.eraseFromParent();
4326 return true;
4327}
4328
4329unsigned
4330AArch64InstructionSelector::emitConstantPoolEntry(const Constant *CPVal,
4331 MachineFunction &MF) const {
4332 Type *CPTy = CPVal->getType();
4334
4335 MachineConstantPool *MCP = MF.getConstantPool();
4336 return MCP->getConstantPoolIndex(CPVal, Alignment);
4337}
4338
4339MachineInstr *AArch64InstructionSelector::emitLoadFromConstantPool(
4340 const Constant *CPVal, MachineIRBuilder &MIRBuilder) const {
4341 const TargetRegisterClass *RC;
4342 unsigned Opc;
4343 bool IsTiny = TM.getCodeModel() == CodeModel::Tiny;
4344 unsigned Size = MIRBuilder.getDataLayout().getTypeStoreSize(CPVal->getType());
4345 switch (Size) {
4346 case 16:
4347 RC = &AArch64::FPR128RegClass;
4348 Opc = IsTiny ? AArch64::LDRQl : AArch64::LDRQui;
4349 break;
4350 case 8:
4351 RC = &AArch64::FPR64RegClass;
4352 Opc = IsTiny ? AArch64::LDRDl : AArch64::LDRDui;
4353 break;
4354 case 4:
4355 RC = &AArch64::FPR32RegClass;
4356 Opc = IsTiny ? AArch64::LDRSl : AArch64::LDRSui;
4357 break;
4358 case 2:
4359 RC = &AArch64::FPR16RegClass;
4360 Opc = AArch64::LDRHui;
4361 break;
4362 default:
4363 LLVM_DEBUG(dbgs() << "Could not load from constant pool of type "
4364 << *CPVal->getType());
4365 return nullptr;
4366 }
4367
4368 MachineInstr *LoadMI = nullptr;
4369 auto &MF = MIRBuilder.getMF();
4370 unsigned CPIdx = emitConstantPoolEntry(CPVal, MF);
4371 if (IsTiny && (Size == 16 || Size == 8 || Size == 4)) {
4372 // Use load(literal) for tiny code model.
4373 LoadMI = &*MIRBuilder.buildInstr(Opc, {RC}, {}).addConstantPoolIndex(CPIdx);
4374 } else {
4375 auto Adrp =
4376 MIRBuilder.buildInstr(AArch64::ADRP, {&AArch64::GPR64RegClass}, {})
4377 .addConstantPoolIndex(CPIdx, 0, AArch64II::MO_PAGE);
4378
4379 LoadMI = &*MIRBuilder.buildInstr(Opc, {RC}, {Adrp})
4380 .addConstantPoolIndex(
4382
4384 }
4385
4386 MachinePointerInfo PtrInfo = MachinePointerInfo::getConstantPool(MF);
4387 LoadMI->addMemOperand(MF, MF.getMachineMemOperand(PtrInfo,
4389 Size, Align(Size)));
4391 return LoadMI;
4392}
4393
4394/// Return an <Opcode, SubregIndex> pair to do an vector elt insert of a given
4395/// size and RB.
4396static std::pair<unsigned, unsigned>
4397getInsertVecEltOpInfo(const RegisterBank &RB, unsigned EltSize) {
4398 unsigned Opc, SubregIdx;
4399 if (RB.getID() == AArch64::GPRRegBankID) {
4400 if (EltSize == 8) {
4401 Opc = AArch64::INSvi8gpr;
4402 SubregIdx = AArch64::bsub;
4403 } else if (EltSize == 16) {
4404 Opc = AArch64::INSvi16gpr;
4405 SubregIdx = AArch64::ssub;
4406 } else if (EltSize == 32) {
4407 Opc = AArch64::INSvi32gpr;
4408 SubregIdx = AArch64::ssub;
4409 } else if (EltSize == 64) {
4410 Opc = AArch64::INSvi64gpr;
4411 SubregIdx = AArch64::dsub;
4412 } else {
4413 llvm_unreachable("invalid elt size!");
4414 }
4415 } else {
4416 if (EltSize == 8) {
4417 Opc = AArch64::INSvi8lane;
4418 SubregIdx = AArch64::bsub;
4419 } else if (EltSize == 16) {
4420 Opc = AArch64::INSvi16lane;
4421 SubregIdx = AArch64::hsub;
4422 } else if (EltSize == 32) {
4423 Opc = AArch64::INSvi32lane;
4424 SubregIdx = AArch64::ssub;
4425 } else if (EltSize == 64) {
4426 Opc = AArch64::INSvi64lane;
4427 SubregIdx = AArch64::dsub;
4428 } else {
4429 llvm_unreachable("invalid elt size!");
4430 }
4431 }
4432 return std::make_pair(Opc, SubregIdx);
4433}
4434
4435MachineInstr *AArch64InstructionSelector::emitInstr(
4436 unsigned Opcode, std::initializer_list<llvm::DstOp> DstOps,
4437 std::initializer_list<llvm::SrcOp> SrcOps, MachineIRBuilder &MIRBuilder,
4438 const ComplexRendererFns &RenderFns) const {
4439 assert(Opcode && "Expected an opcode?");
4440 assert(!isPreISelGenericOpcode(Opcode) &&
4441 "Function should only be used to produce selected instructions!");
4442 auto MI = MIRBuilder.buildInstr(Opcode, DstOps, SrcOps);
4443 if (RenderFns)
4444 for (auto &Fn : *RenderFns)
4445 Fn(MI);
4447 return &*MI;
4448}
4449
4450MachineInstr *AArch64InstructionSelector::emitAddSub(
4451 const std::array<std::array<unsigned, 2>, 5> &AddrModeAndSizeToOpcode,
4452 Register Dst, MachineOperand &LHS, MachineOperand &RHS,
4453 MachineIRBuilder &MIRBuilder) const {
4454 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4455 assert(LHS.isReg() && RHS.isReg() && "Expected register operands?");
4456 auto Ty = MRI.getType(LHS.getReg());
4457 assert(!Ty.isVector() && "Expected a scalar or pointer?");
4458 unsigned Size = Ty.getSizeInBits();
4459 assert((Size == 32 || Size == 64) && "Expected a 32-bit or 64-bit type only");
4460 bool Is32Bit = Size == 32;
4461
4462 // INSTRri form with positive arithmetic immediate.
4463 if (auto Fns = selectArithImmed(RHS))
4464 return emitInstr(AddrModeAndSizeToOpcode[0][Is32Bit], {Dst}, {LHS},
4465 MIRBuilder, Fns);
4466
4467 // INSTRri form with negative arithmetic immediate.
4468 if (auto Fns = selectNegArithImmed(RHS))
4469 return emitInstr(AddrModeAndSizeToOpcode[3][Is32Bit], {Dst}, {LHS},
4470 MIRBuilder, Fns);
4471
4472 // INSTRrx form.
4473 if (auto Fns = selectArithExtendedRegister(RHS))
4474 return emitInstr(AddrModeAndSizeToOpcode[4][Is32Bit], {Dst}, {LHS},
4475 MIRBuilder, Fns);
4476
4477 // INSTRrs form.
4478 if (auto Fns = selectShiftedRegister(RHS))
4479 return emitInstr(AddrModeAndSizeToOpcode[1][Is32Bit], {Dst}, {LHS},
4480 MIRBuilder, Fns);
4481 return emitInstr(AddrModeAndSizeToOpcode[2][Is32Bit], {Dst}, {LHS, RHS},
4482 MIRBuilder);
4483}
4484
4485MachineInstr *
4486AArch64InstructionSelector::emitADD(Register DefReg, MachineOperand &LHS,
4487 MachineOperand &RHS,
4488 MachineIRBuilder &MIRBuilder) const {
4489 const std::array<std::array<unsigned, 2>, 5> OpcTable{
4490 {{AArch64::ADDXri, AArch64::ADDWri},
4491 {AArch64::ADDXrs, AArch64::ADDWrs},
4492 {AArch64::ADDXrr, AArch64::ADDWrr},
4493 {AArch64::SUBXri, AArch64::SUBWri},
4494 {AArch64::ADDXrx, AArch64::ADDWrx}}};
4495 return emitAddSub(OpcTable, DefReg, LHS, RHS, MIRBuilder);
4496}
4497
4498MachineInstr *
4499AArch64InstructionSelector::emitADDS(Register Dst, MachineOperand &LHS,
4500 MachineOperand &RHS,
4501 MachineIRBuilder &MIRBuilder) const {
4502 const std::array<std::array<unsigned, 2>, 5> OpcTable{
4503 {{AArch64::ADDSXri, AArch64::ADDSWri},
4504 {AArch64::ADDSXrs, AArch64::ADDSWrs},
4505 {AArch64::ADDSXrr, AArch64::ADDSWrr},
4506 {AArch64::SUBSXri, AArch64::SUBSWri},
4507 {AArch64::ADDSXrx, AArch64::ADDSWrx}}};
4508 return emitAddSub(OpcTable, Dst, LHS, RHS, MIRBuilder);
4509}
4510
4511MachineInstr *
4512AArch64InstructionSelector::emitSUBS(Register Dst, MachineOperand &LHS,
4513 MachineOperand &RHS,
4514 MachineIRBuilder &MIRBuilder) const {
4515 const std::array<std::array<unsigned, 2>, 5> OpcTable{
4516 {{AArch64::SUBSXri, AArch64::SUBSWri},
4517 {AArch64::SUBSXrs, AArch64::SUBSWrs},
4518 {AArch64::SUBSXrr, AArch64::SUBSWrr},
4519 {AArch64::ADDSXri, AArch64::ADDSWri},
4520 {AArch64::SUBSXrx, AArch64::SUBSWrx}}};
4521 return emitAddSub(OpcTable, Dst, LHS, RHS, MIRBuilder);
4522}
4523
4524MachineInstr *
4525AArch64InstructionSelector::emitADCS(Register Dst, MachineOperand &LHS,
4526 MachineOperand &RHS,
4527 MachineIRBuilder &MIRBuilder) const {
4528 assert(LHS.isReg() && RHS.isReg() && "Expected register operands?");
4529 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
4530 bool Is32Bit = (MRI->getType(LHS.getReg()).getSizeInBits() == 32);
4531 static const unsigned OpcTable[2] = {AArch64::ADCSXr, AArch64::ADCSWr};
4532 return emitInstr(OpcTable[Is32Bit], {Dst}, {LHS, RHS}, MIRBuilder);
4533}
4534
4535MachineInstr *
4536AArch64InstructionSelector::emitSBCS(Register Dst, MachineOperand &LHS,
4537 MachineOperand &RHS,
4538 MachineIRBuilder &MIRBuilder) const {
4539 assert(LHS.isReg() && RHS.isReg() && "Expected register operands?");
4540 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
4541 bool Is32Bit = (MRI->getType(LHS.getReg()).getSizeInBits() == 32);
4542 static const unsigned OpcTable[2] = {AArch64::SBCSXr, AArch64::SBCSWr};
4543 return emitInstr(OpcTable[Is32Bit], {Dst}, {LHS, RHS}, MIRBuilder);
4544}
4545
4546MachineInstr *
4547AArch64InstructionSelector::emitCMP(MachineOperand &LHS, MachineOperand &RHS,
4548 MachineIRBuilder &MIRBuilder) const {
4549 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4550 bool Is32Bit = MRI.getType(LHS.getReg()).getSizeInBits() == 32;
4551 auto RC = Is32Bit ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass;
4552 return emitSUBS(MRI.createVirtualRegister(RC), LHS, RHS, MIRBuilder);
4553}
4554
4555MachineInstr *
4556AArch64InstructionSelector::emitCMN(MachineOperand &LHS, MachineOperand &RHS,
4557 MachineIRBuilder &MIRBuilder) const {
4558 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4559 bool Is32Bit = (MRI.getType(LHS.getReg()).getSizeInBits() == 32);
4560 auto RC = Is32Bit ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass;
4561 return emitADDS(MRI.createVirtualRegister(RC), LHS, RHS, MIRBuilder);
4562}
4563
4564MachineInstr *
4565AArch64InstructionSelector::emitTST(MachineOperand &LHS, MachineOperand &RHS,
4566 MachineIRBuilder &MIRBuilder) const {
4567 assert(LHS.isReg() && RHS.isReg() && "Expected register operands?");
4568 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4569 LLT Ty = MRI.getType(LHS.getReg());
4570 unsigned RegSize = Ty.getSizeInBits();
4571 bool Is32Bit = (RegSize == 32);
4572 const unsigned OpcTable[3][2] = {{AArch64::ANDSXri, AArch64::ANDSWri},
4573 {AArch64::ANDSXrs, AArch64::ANDSWrs},
4574 {AArch64::ANDSXrr, AArch64::ANDSWrr}};
4575 // ANDS needs a logical immediate for its immediate form. Check if we can
4576 // fold one in.
4577 if (auto ValAndVReg = getIConstantVRegValWithLookThrough(RHS.getReg(), MRI)) {
4578 int64_t Imm = ValAndVReg->Value.getSExtValue();
4579
4581 auto TstMI = MIRBuilder.buildInstr(OpcTable[0][Is32Bit], {Ty}, {LHS});
4584 return &*TstMI;
4585 }
4586 }
4587
4588 if (auto Fns = selectLogicalShiftedRegister(RHS))
4589 return emitInstr(OpcTable[1][Is32Bit], {Ty}, {LHS}, MIRBuilder, Fns);
4590 return emitInstr(OpcTable[2][Is32Bit], {Ty}, {LHS, RHS}, MIRBuilder);
4591}
4592
4593MachineInstr *AArch64InstructionSelector::emitIntegerCompare(
4594 MachineOperand &LHS, MachineOperand &RHS, MachineOperand &Predicate,
4595 MachineIRBuilder &MIRBuilder) const {
4596 assert(LHS.isReg() && RHS.isReg() && "Expected LHS and RHS to be registers!");
4597 assert(Predicate.isPredicate() && "Expected predicate?");
4598 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4599 LLT CmpTy = MRI.getType(LHS.getReg());
4600 assert(!CmpTy.isVector() && "Expected scalar or pointer");
4601 unsigned Size = CmpTy.getSizeInBits();
4602 (void)Size;
4603 assert((Size == 32 || Size == 64) && "Expected a 32-bit or 64-bit LHS/RHS?");
4604 // Fold the compare into a cmn or tst if possible.
4605 if (auto FoldCmp = tryFoldIntegerCompare(LHS, RHS, Predicate, MIRBuilder))
4606 return FoldCmp;
4607 return emitCMP(LHS, RHS, MIRBuilder);
4608}
4609
4610MachineInstr *AArch64InstructionSelector::emitCSetForFCmp(
4611 Register Dst, CmpInst::Predicate Pred, MachineIRBuilder &MIRBuilder) const {
4612 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4613#ifndef NDEBUG
4614 LLT Ty = MRI.getType(Dst);
4615 assert(!Ty.isVector() && Ty.getSizeInBits() == 32 &&
4616 "Expected a 32-bit scalar register?");
4617#endif
4618 const Register ZReg = AArch64::WZR;
4619 AArch64CC::CondCode CC1, CC2;
4621 auto InvCC1 = AArch64CC::getInvertedCondCode(CC1);
4622 if (CC2 == AArch64CC::AL)
4623 return emitCSINC(/*Dst=*/Dst, /*Src1=*/ZReg, /*Src2=*/ZReg, InvCC1,
4624 MIRBuilder);
4625 const TargetRegisterClass *RC = &AArch64::GPR32RegClass;
4626 Register Def1Reg = MRI.createVirtualRegister(RC);
4627 Register Def2Reg = MRI.createVirtualRegister(RC);
4628 auto InvCC2 = AArch64CC::getInvertedCondCode(CC2);
4629 emitCSINC(/*Dst=*/Def1Reg, /*Src1=*/ZReg, /*Src2=*/ZReg, InvCC1, MIRBuilder);
4630 emitCSINC(/*Dst=*/Def2Reg, /*Src1=*/ZReg, /*Src2=*/ZReg, InvCC2, MIRBuilder);
4631 auto OrMI = MIRBuilder.buildInstr(AArch64::ORRWrr, {Dst}, {Def1Reg, Def2Reg});
4633 return &*OrMI;
4634}
4635
4636MachineInstr *AArch64InstructionSelector::emitFPCompare(
4637 Register LHS, Register RHS, MachineIRBuilder &MIRBuilder,
4638 std::optional<CmpInst::Predicate> Pred) const {
4639 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4640 LLT Ty = MRI.getType(LHS);
4641 if (Ty.isVector())
4642 return nullptr;
4643 unsigned OpSize = Ty.getSizeInBits();
4644 assert(OpSize == 16 || OpSize == 32 || OpSize == 64);
4645
4646 // If this is a compare against +0.0, then we don't have
4647 // to explicitly materialize a constant.
4648 bool ShouldUseImm = mi_match(RHS, MRI, m_PosZeroFP());
4649
4650 auto IsEqualityPred = [](CmpInst::Predicate P) {
4651 return P == CmpInst::FCMP_OEQ || P == CmpInst::FCMP_ONE ||
4653 };
4654 if (!ShouldUseImm && Pred && IsEqualityPred(*Pred)) {
4655 // Try commuting the operands.
4656 if (mi_match(LHS, MRI, m_PosZeroFP())) {
4657 ShouldUseImm = true;
4658 std::swap(LHS, RHS);
4659 }
4660 }
4661 unsigned CmpOpcTbl[2][3] = {
4662 {AArch64::FCMPHrr, AArch64::FCMPSrr, AArch64::FCMPDrr},
4663 {AArch64::FCMPHri, AArch64::FCMPSri, AArch64::FCMPDri}};
4664 unsigned CmpOpc =
4665 CmpOpcTbl[ShouldUseImm][OpSize == 16 ? 0 : (OpSize == 32 ? 1 : 2)];
4666
4667 // Partially build the compare. Decide if we need to add a use for the
4668 // third operand based off whether or not we're comparing against 0.0.
4669 auto CmpMI = MIRBuilder.buildInstr(CmpOpc).addUse(LHS);
4671 if (!ShouldUseImm)
4672 CmpMI.addUse(RHS);
4674 return &*CmpMI;
4675}
4676
4677MachineInstr *AArch64InstructionSelector::emitVectorConcat(
4678 std::optional<Register> Dst, Register Op1, Register Op2,
4679 MachineIRBuilder &MIRBuilder) const {
4680 // We implement a vector concat by:
4681 // 1. Use scalar_to_vector to insert the lower vector into the larger dest
4682 // 2. Insert the upper vector into the destination's upper element
4683 // TODO: some of this code is common with G_BUILD_VECTOR handling.
4684 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4685
4686 const LLT Op1Ty = MRI.getType(Op1);
4687 const LLT Op2Ty = MRI.getType(Op2);
4688
4689 if (Op1Ty != Op2Ty) {
4690 LLVM_DEBUG(dbgs() << "Could not do vector concat of differing vector tys");
4691 return nullptr;
4692 }
4693 assert(Op1Ty.isVector() && "Expected a vector for vector concat");
4694
4695 if (Op1Ty.getSizeInBits() >= 128) {
4696 LLVM_DEBUG(dbgs() << "Vector concat not supported for full size vectors");
4697 return nullptr;
4698 }
4699
4700 // At the moment we just support 64 bit vector concats.
4701 if (Op1Ty.getSizeInBits() != 64) {
4702 LLVM_DEBUG(dbgs() << "Vector concat supported for 64b vectors");
4703 return nullptr;
4704 }
4705
4706 const LLT ScalarTy = LLT::scalar(Op1Ty.getSizeInBits());
4707 const RegisterBank &FPRBank = *RBI.getRegBank(Op1, MRI, TRI);
4708 const TargetRegisterClass *DstRC =
4709 getRegClassForTypeOnBank(Op1Ty.multiplyElements(2), FPRBank);
4710
4711 MachineInstr *WidenedOp1 =
4712 emitScalarToVector(ScalarTy.getSizeInBits(), DstRC, Op1, MIRBuilder);
4713 MachineInstr *WidenedOp2 =
4714 emitScalarToVector(ScalarTy.getSizeInBits(), DstRC, Op2, MIRBuilder);
4715 if (!WidenedOp1 || !WidenedOp2) {
4716 LLVM_DEBUG(dbgs() << "Could not emit a vector from scalar value");
4717 return nullptr;
4718 }
4719
4720 // Now do the insert of the upper element.
4721 unsigned InsertOpc, InsSubRegIdx;
4722 std::tie(InsertOpc, InsSubRegIdx) =
4723 getInsertVecEltOpInfo(FPRBank, ScalarTy.getSizeInBits());
4724
4725 if (!Dst)
4726 Dst = MRI.createVirtualRegister(DstRC);
4727 auto InsElt =
4728 MIRBuilder
4729 .buildInstr(InsertOpc, {*Dst}, {WidenedOp1->getOperand(0).getReg()})
4730 .addImm(1) /* Lane index */
4731 .addUse(WidenedOp2->getOperand(0).getReg())
4732 .addImm(0);
4734 return &*InsElt;
4735}
4736
4737MachineInstr *
4738AArch64InstructionSelector::emitCSINC(Register Dst, Register Src1,
4740 MachineIRBuilder &MIRBuilder) const {
4741 auto &MRI = *MIRBuilder.getMRI();
4742 const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Dst);
4743 // If we used a register class, then this won't necessarily have an LLT.
4744 // Compute the size based off whether or not we have a class or bank.
4745 unsigned Size;
4746 if (const auto *RC = dyn_cast<const TargetRegisterClass *>(RegClassOrBank))
4747 Size = TRI.getRegSizeInBits(*RC);
4748 else
4749 Size = MRI.getType(Dst).getSizeInBits();
4750 // Some opcodes use s1.
4751 assert(Size <= 64 && "Expected 64 bits or less only!");
4752 static const unsigned OpcTable[2] = {AArch64::CSINCWr, AArch64::CSINCXr};
4753 unsigned Opc = OpcTable[Size == 64];
4754 auto CSINC = MIRBuilder.buildInstr(Opc, {Dst}, {Src1, Src2}).addImm(Pred);
4756 return &*CSINC;
4757}
4758
4759MachineInstr *AArch64InstructionSelector::emitCarryIn(MachineInstr &I,
4760 Register CarryReg) {
4761 MachineRegisterInfo *MRI = MIB.getMRI();
4762 unsigned Opcode = I.getOpcode();
4763
4764 // If the instruction is a SUB, we need to negate the carry,
4765 // because borrowing is indicated by carry-flag == 0.
4766 bool NeedsNegatedCarry =
4767 (Opcode == TargetOpcode::G_USUBE || Opcode == TargetOpcode::G_SSUBE);
4768
4769 // If the previous instruction will already produce the correct carry, do not
4770 // emit a carry generating instruction. E.g. for G_UADDE/G_USUBE sequences
4771 // generated during legalization of wide add/sub. This optimization depends on
4772 // these sequences not being interrupted by other instructions.
4773 // We have to select the previous instruction before the carry-using
4774 // instruction is deleted by the calling function, otherwise the previous
4775 // instruction might become dead and would get deleted.
4776 MachineInstr *SrcMI = MRI->getVRegDef(CarryReg);
4777
4778 // Debug instructions (e.g., DBG_VALUE) do not modify NZCV, so skip all that
4779 // immediately precede I.
4780 MachineInstr *PrevMI = I.getPrevNode();
4781 while (PrevMI && PrevMI->isDebugInstr())
4782 PrevMI = PrevMI->getPrevNode();
4783
4784 if (SrcMI == PrevMI) {
4785 if (auto *CarrySrcMI = dyn_cast<GAddSubCarryOut>(SrcMI)) {
4786 bool ProducesNegatedCarry = CarrySrcMI->isSub();
4787 if (NeedsNegatedCarry == ProducesNegatedCarry &&
4788 CarrySrcMI->isUnsigned() &&
4789 CarrySrcMI->getCarryOutReg() == CarryReg &&
4790 selectAndRestoreState(*SrcMI))
4791 return nullptr;
4792 }
4793 }
4794
4795 Register DeadReg = MRI->createVirtualRegister(&AArch64::GPR32RegClass);
4796
4797 if (NeedsNegatedCarry) {
4798 // (0 - Carry) sets !C in NZCV when Carry == 1
4799 Register ZReg = AArch64::WZR;
4800 return emitInstr(AArch64::SUBSWrr, {DeadReg}, {ZReg, CarryReg}, MIB);
4801 }
4802
4803 // (Carry - 1) sets !C in NZCV when Carry == 0
4804 auto Fns = select12BitValueWithLeftShift(1);
4805 return emitInstr(AArch64::SUBSWri, {DeadReg}, {CarryReg}, MIB, Fns);
4806}
4807
4808bool AArch64InstructionSelector::selectOverflowOp(MachineInstr &I,
4809 MachineRegisterInfo &MRI) {
4810 auto &CarryMI = cast<GAddSubCarryOut>(I);
4811
4812 if (auto *CarryInMI = dyn_cast<GAddSubCarryInOut>(&I)) {
4813 // Set NZCV carry according to carry-in VReg
4814 emitCarryIn(I, CarryInMI->getCarryInReg());
4815 }
4816
4817 // Emit the operation and get the correct condition code.
4818 auto OpAndCC = emitOverflowOp(I.getOpcode(), CarryMI.getDstReg(),
4819 CarryMI.getLHS(), CarryMI.getRHS(), MIB);
4820
4821 Register CarryOutReg = CarryMI.getCarryOutReg();
4822
4823 // Don't convert carry-out to VReg if it is never used
4824 if (MRI.use_nodbg_empty(CarryOutReg)) {
4825 OpAndCC.first->addRegisterDead(AArch64::NZCV, &TRI);
4826 } else {
4827 // Now, put the overflow result in the register given by the first operand
4828 // to the overflow op. CSINC increments the result when the predicate is
4829 // false, so to get the increment when it's true, we need to use the
4830 // inverse. In this case, we want to increment when carry is set.
4831 Register ZReg = AArch64::WZR;
4832 emitCSINC(/*Dst=*/CarryOutReg, /*Src1=*/ZReg, /*Src2=*/ZReg,
4833 getInvertedCondCode(OpAndCC.second), MIB);
4834 }
4835
4836 I.eraseFromParent();
4837 return true;
4838}
4839
4840std::pair<MachineInstr *, AArch64CC::CondCode>
4841AArch64InstructionSelector::emitOverflowOp(unsigned Opcode, Register Dst,
4842 MachineOperand &LHS,
4843 MachineOperand &RHS,
4844 MachineIRBuilder &MIRBuilder) const {
4845 switch (Opcode) {
4846 default:
4847 llvm_unreachable("Unexpected opcode!");
4848 case TargetOpcode::G_SADDO:
4849 return std::make_pair(emitADDS(Dst, LHS, RHS, MIRBuilder), AArch64CC::VS);
4850 case TargetOpcode::G_UADDO:
4851 return std::make_pair(emitADDS(Dst, LHS, RHS, MIRBuilder), AArch64CC::HS);
4852 case TargetOpcode::G_SSUBO:
4853 return std::make_pair(emitSUBS(Dst, LHS, RHS, MIRBuilder), AArch64CC::VS);
4854 case TargetOpcode::G_USUBO:
4855 return std::make_pair(emitSUBS(Dst, LHS, RHS, MIRBuilder), AArch64CC::LO);
4856 case TargetOpcode::G_SADDE:
4857 return std::make_pair(emitADCS(Dst, LHS, RHS, MIRBuilder), AArch64CC::VS);
4858 case TargetOpcode::G_UADDE:
4859 return std::make_pair(emitADCS(Dst, LHS, RHS, MIRBuilder), AArch64CC::HS);
4860 case TargetOpcode::G_SSUBE:
4861 return std::make_pair(emitSBCS(Dst, LHS, RHS, MIRBuilder), AArch64CC::VS);
4862 case TargetOpcode::G_USUBE:
4863 return std::make_pair(emitSBCS(Dst, LHS, RHS, MIRBuilder), AArch64CC::LO);
4864 }
4865}
4866
4867/// Returns true if @p Val is a tree of AND/OR/CMP operations that can be
4868/// expressed as a conjunction.
4869/// \param CanNegate Set to true if we can negate the whole sub-tree just by
4870/// changing the conditions on the CMP tests.
4871/// (this means we can call emitConjunctionRec() with
4872/// Negate==true on this sub-tree)
4873/// \param MustBeFirst Set to true if this subtree needs to be negated and we
4874/// cannot do the negation naturally. We are required to
4875/// emit the subtree first in this case.
4876/// \param WillNegate Is true if are called when the result of this
4877/// subexpression must be negated. This happens when the
4878/// outer expression is an OR. We can use this fact to know
4879/// that we have a double negation (or (or ...) ...) that
4880/// can be implemented for free.
4881static bool canEmitConjunction(Register Val, bool &CanNegate, bool &MustBeFirst,
4882 bool WillNegate, MachineRegisterInfo &MRI,
4883 unsigned Depth = 0) {
4884 if (!MRI.hasOneNonDBGUse(Val))
4885 return false;
4886 MachineInstr *ValDef = MRI.getVRegDef(Val);
4887 unsigned Opcode = ValDef->getOpcode();
4888 if (isa<GAnyCmp>(ValDef)) {
4889 CanNegate = true;
4890 MustBeFirst = false;
4891 return true;
4892 }
4893 // Protect against exponential runtime and stack overflow.
4894 if (Depth > 6)
4895 return false;
4896 if (Opcode == TargetOpcode::G_AND || Opcode == TargetOpcode::G_OR) {
4897 bool IsOR = Opcode == TargetOpcode::G_OR;
4898 Register O0 = ValDef->getOperand(1).getReg();
4899 Register O1 = ValDef->getOperand(2).getReg();
4900 bool CanNegateL;
4901 bool MustBeFirstL;
4902 if (!canEmitConjunction(O0, CanNegateL, MustBeFirstL, IsOR, MRI, Depth + 1))
4903 return false;
4904 bool CanNegateR;
4905 bool MustBeFirstR;
4906 if (!canEmitConjunction(O1, CanNegateR, MustBeFirstR, IsOR, MRI, Depth + 1))
4907 return false;
4908
4909 if (MustBeFirstL && MustBeFirstR)
4910 return false;
4911
4912 if (IsOR) {
4913 // For an OR expression we need to be able to naturally negate at least
4914 // one side or we cannot do the transformation at all.
4915 if (!CanNegateL && !CanNegateR)
4916 return false;
4917 // If we the result of the OR will be negated and we can naturally negate
4918 // the leaves, then this sub-tree as a whole negates naturally.
4919 CanNegate = WillNegate && CanNegateL && CanNegateR;
4920 // If we cannot naturally negate the whole sub-tree, then this must be
4921 // emitted first.
4922 MustBeFirst = !CanNegate;
4923 } else {
4924 assert(Opcode == TargetOpcode::G_AND && "Must be G_AND");
4925 // We cannot naturally negate an AND operation.
4926 CanNegate = false;
4927 MustBeFirst = MustBeFirstL || MustBeFirstR;
4928 }
4929 return true;
4930 }
4931 return false;
4932}
4933
4934MachineInstr *AArch64InstructionSelector::emitConditionalComparison(
4937 MachineIRBuilder &MIB) const {
4938 auto &MRI = *MIB.getMRI();
4939 LLT OpTy = MRI.getType(LHS);
4940 unsigned CCmpOpc;
4941 std::optional<ValueAndVReg> C;
4943 assert(OpTy.getSizeInBits() == 32 || OpTy.getSizeInBits() == 64);
4945 if (!C || C->Value.sgt(31) || C->Value.slt(-31))
4946 CCmpOpc = OpTy.getSizeInBits() == 32 ? AArch64::CCMPWr : AArch64::CCMPXr;
4947 else if (C->Value.ule(31))
4948 CCmpOpc = OpTy.getSizeInBits() == 32 ? AArch64::CCMPWi : AArch64::CCMPXi;
4949 else
4950 CCmpOpc = OpTy.getSizeInBits() == 32 ? AArch64::CCMNWi : AArch64::CCMNXi;
4951 } else {
4952 assert(OpTy.getSizeInBits() == 16 || OpTy.getSizeInBits() == 32 ||
4953 OpTy.getSizeInBits() == 64);
4954 switch (OpTy.getSizeInBits()) {
4955 case 16:
4956 assert(STI.hasFullFP16() && "Expected Full FP16 for fp16 comparisons");
4957 CCmpOpc = AArch64::FCCMPHrr;
4958 break;
4959 case 32:
4960 CCmpOpc = AArch64::FCCMPSrr;
4961 break;
4962 case 64:
4963 CCmpOpc = AArch64::FCCMPDrr;
4964 break;
4965 default:
4966 return nullptr;
4967 }
4968 }
4970 unsigned NZCV = AArch64CC::getNZCVToSatisfyCondCode(InvOutCC);
4971 auto CCmp =
4972 MIB.buildInstr(CCmpOpc, {}, {LHS});
4973 if (CCmpOpc == AArch64::CCMPWi || CCmpOpc == AArch64::CCMPXi)
4974 CCmp.addImm(C->Value.getZExtValue());
4975 else if (CCmpOpc == AArch64::CCMNWi || CCmpOpc == AArch64::CCMNXi)
4976 CCmp.addImm(C->Value.abs().getZExtValue());
4977 else
4978 CCmp.addReg(RHS);
4979 CCmp.addImm(NZCV).addImm(Predicate);
4981 return &*CCmp;
4982}
4983
4984MachineInstr *AArch64InstructionSelector::emitConjunctionRec(
4985 Register Val, AArch64CC::CondCode &OutCC, bool Negate, Register CCOp,
4986 AArch64CC::CondCode Predicate, MachineIRBuilder &MIB) const {
4987 // We're at a tree leaf, produce a conditional comparison operation.
4988 auto &MRI = *MIB.getMRI();
4989 MachineInstr *ValDef = MRI.getVRegDef(Val);
4990 unsigned Opcode = ValDef->getOpcode();
4991 if (auto *Cmp = dyn_cast<GAnyCmp>(ValDef)) {
4992 Register LHS = Cmp->getLHSReg();
4993 Register RHS = Cmp->getRHSReg();
4994 CmpInst::Predicate CC = Cmp->getCond();
4995 if (Negate)
4997 if (isa<GICmp>(Cmp)) {
4998 OutCC = changeICMPPredToAArch64CC(CC, RHS, MIB.getMRI());
4999 } else {
5000 // Handle special FP cases.
5001 AArch64CC::CondCode ExtraCC;
5002 changeFPCCToANDAArch64CC(CC, OutCC, ExtraCC);
5003 // Some floating point conditions can't be tested with a single condition
5004 // code. Construct an additional comparison in this case.
5005 if (ExtraCC != AArch64CC::AL) {
5006 MachineInstr *ExtraCmp;
5007 if (!CCOp)
5008 ExtraCmp = emitFPCompare(LHS, RHS, MIB, CC);
5009 else
5010 ExtraCmp =
5011 emitConditionalComparison(LHS, RHS, CC, Predicate, ExtraCC, MIB);
5012 CCOp = ExtraCmp->getOperand(0).getReg();
5013 Predicate = ExtraCC;
5014 }
5015 }
5016
5017 // Produce a normal comparison if we are first in the chain
5018 if (!CCOp) {
5019 if (isa<GICmp>(Cmp))
5020 return emitCMP(Cmp->getOperand(2), Cmp->getOperand(3), MIB);
5021 return emitFPCompare(Cmp->getOperand(2).getReg(),
5022 Cmp->getOperand(3).getReg(), MIB);
5023 }
5024 // Otherwise produce a ccmp.
5025 return emitConditionalComparison(LHS, RHS, CC, Predicate, OutCC, MIB);
5026 }
5027 assert(MRI.hasOneNonDBGUse(Val) && "Valid conjunction/disjunction tree");
5028
5029 bool IsOR = Opcode == TargetOpcode::G_OR;
5030
5031 Register LHS = ValDef->getOperand(1).getReg();
5032 bool CanNegateL;
5033 bool MustBeFirstL;
5034 bool ValidL = canEmitConjunction(LHS, CanNegateL, MustBeFirstL, IsOR, MRI);
5035 assert(ValidL && "Valid conjunction/disjunction tree");
5036 (void)ValidL;
5037
5038 Register RHS = ValDef->getOperand(2).getReg();
5039 bool CanNegateR;
5040 bool MustBeFirstR;
5041 bool ValidR = canEmitConjunction(RHS, CanNegateR, MustBeFirstR, IsOR, MRI);
5042 assert(ValidR && "Valid conjunction/disjunction tree");
5043 (void)ValidR;
5044
5045 // Swap sub-tree that must come first to the right side.
5046 if (MustBeFirstL) {
5047 assert(!MustBeFirstR && "Valid conjunction/disjunction tree");
5048 std::swap(LHS, RHS);
5049 std::swap(CanNegateL, CanNegateR);
5050 std::swap(MustBeFirstL, MustBeFirstR);
5051 }
5052
5053 bool NegateR;
5054 bool NegateAfterR;
5055 bool NegateL;
5056 bool NegateAfterAll;
5057 if (Opcode == TargetOpcode::G_OR) {
5058 // Swap the sub-tree that we can negate naturally to the left.
5059 if (!CanNegateL) {
5060 assert(CanNegateR && "at least one side must be negatable");
5061 assert(!MustBeFirstR && "invalid conjunction/disjunction tree");
5062 assert(!Negate);
5063 std::swap(LHS, RHS);
5064 NegateR = false;
5065 NegateAfterR = true;
5066 } else {
5067 // Negate the left sub-tree if possible, otherwise negate the result.
5068 NegateR = CanNegateR;
5069 NegateAfterR = !CanNegateR;
5070 }
5071 NegateL = true;
5072 NegateAfterAll = !Negate;
5073 } else {
5074 assert(Opcode == TargetOpcode::G_AND &&
5075 "Valid conjunction/disjunction tree");
5076 assert(!Negate && "Valid conjunction/disjunction tree");
5077
5078 NegateL = false;
5079 NegateR = false;
5080 NegateAfterR = false;
5081 NegateAfterAll = false;
5082 }
5083
5084 // Emit sub-trees.
5085 AArch64CC::CondCode RHSCC;
5086 MachineInstr *CmpR =
5087 emitConjunctionRec(RHS, RHSCC, NegateR, CCOp, Predicate, MIB);
5088 if (NegateAfterR)
5089 RHSCC = AArch64CC::getInvertedCondCode(RHSCC);
5090 MachineInstr *CmpL = emitConjunctionRec(
5091 LHS, OutCC, NegateL, CmpR->getOperand(0).getReg(), RHSCC, MIB);
5092 if (NegateAfterAll)
5093 OutCC = AArch64CC::getInvertedCondCode(OutCC);
5094 return CmpL;
5095}
5096
5097MachineInstr *AArch64InstructionSelector::emitConjunction(
5098 Register Val, AArch64CC::CondCode &OutCC, MachineIRBuilder &MIB) const {
5099 bool DummyCanNegate;
5100 bool DummyMustBeFirst;
5101 if (!canEmitConjunction(Val, DummyCanNegate, DummyMustBeFirst, false,
5102 *MIB.getMRI()))
5103 return nullptr;
5104 return emitConjunctionRec(Val, OutCC, false, Register(), AArch64CC::AL, MIB);
5105}
5106
5107bool AArch64InstructionSelector::tryOptSelectConjunction(GSelect &SelI,
5108 MachineInstr &CondMI) {
5109 AArch64CC::CondCode AArch64CC;
5110 MachineInstr *ConjMI = emitConjunction(SelI.getCondReg(), AArch64CC, MIB);
5111 if (!ConjMI)
5112 return false;
5113
5114 emitSelect(SelI.getReg(0), SelI.getTrueReg(), SelI.getFalseReg(), AArch64CC, MIB);
5115 SelI.eraseFromParent();
5116 return true;
5117}
5118
5119bool AArch64InstructionSelector::tryOptSelect(GSelect &I) {
5120 MachineRegisterInfo &MRI = *MIB.getMRI();
5121 // We want to recognize this pattern:
5122 //
5123 // $z = G_FCMP pred, $x, $y
5124 // ...
5125 // $w = G_SELECT $z, $a, $b
5126 //
5127 // Where the value of $z is *only* ever used by the G_SELECT (possibly with
5128 // some copies/truncs in between.)
5129 //
5130 // If we see this, then we can emit something like this:
5131 //
5132 // fcmp $x, $y
5133 // fcsel $w, $a, $b, pred
5134 //
5135 // Rather than emitting both of the rather long sequences in the standard
5136 // G_FCMP/G_SELECT select methods.
5137
5138 // First, check if the condition is defined by a compare.
5139 MachineInstr *CondDef = MRI.getVRegDef(I.getOperand(1).getReg());
5140
5141 // We can only fold if all of the defs have one use.
5142 Register CondDefReg = CondDef->getOperand(0).getReg();
5143 if (!MRI.hasOneNonDBGUse(CondDefReg)) {
5144 // Unless it's another select.
5145 for (const MachineInstr &UI : MRI.use_nodbg_instructions(CondDefReg)) {
5146 if (CondDef == &UI)
5147 continue;
5148 if (UI.getOpcode() != TargetOpcode::G_SELECT)
5149 return false;
5150 }
5151 }
5152
5153 // Is the condition defined by a compare?
5154 unsigned CondOpc = CondDef->getOpcode();
5155 if (CondOpc != TargetOpcode::G_ICMP && CondOpc != TargetOpcode::G_FCMP) {
5156 if (tryOptSelectConjunction(I, *CondDef))
5157 return true;
5158 return false;
5159 }
5160
5162 if (CondOpc == TargetOpcode::G_ICMP) {
5163 auto &PredOp = CondDef->getOperand(1);
5164 emitIntegerCompare(CondDef->getOperand(2), CondDef->getOperand(3), PredOp,
5165 MIB);
5166 auto Pred = static_cast<CmpInst::Predicate>(PredOp.getPredicate());
5167 CondCode =
5169 } else {
5170 // Get the condition code for the select.
5171 auto Pred =
5172 static_cast<CmpInst::Predicate>(CondDef->getOperand(1).getPredicate());
5173 AArch64CC::CondCode CondCode2;
5174 changeFCMPPredToAArch64CC(Pred, CondCode, CondCode2);
5175
5176 // changeFCMPPredToAArch64CC sets CondCode2 to AL when we require two
5177 // instructions to emit the comparison.
5178 // TODO: Handle FCMP_UEQ and FCMP_ONE. After that, this check will be
5179 // unnecessary.
5180 if (CondCode2 != AArch64CC::AL)
5181 return false;
5182
5183 if (!emitFPCompare(CondDef->getOperand(2).getReg(),
5184 CondDef->getOperand(3).getReg(), MIB)) {
5185 LLVM_DEBUG(dbgs() << "Couldn't emit compare for select!\n");
5186 return false;
5187 }
5188 }
5189
5190 // Emit the select.
5191 emitSelect(I.getOperand(0).getReg(), I.getOperand(2).getReg(),
5192 I.getOperand(3).getReg(), CondCode, MIB);
5193 I.eraseFromParent();
5194 return true;
5195}
5196
5197MachineInstr *AArch64InstructionSelector::tryFoldIntegerCompare(
5198 MachineOperand &LHS, MachineOperand &RHS, MachineOperand &Predicate,
5199 MachineIRBuilder &MIRBuilder) const {
5200 assert(LHS.isReg() && RHS.isReg() && Predicate.isPredicate() &&
5201 "Unexpected MachineOperand");
5202 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
5203 // We want to find this sort of thing:
5204 // x = G_SUB 0, y
5205 // G_ICMP z, x
5206 //
5207 // In this case, we can fold the G_SUB into the G_ICMP using a CMN instead.
5208 // e.g:
5209 //
5210 // cmn z, y
5211
5212 // Check if the RHS or LHS of the G_ICMP is defined by a SUB
5213 MachineInstr *LHSDef = getDefIgnoringCopies(LHS.getReg(), MRI);
5214 MachineInstr *RHSDef = getDefIgnoringCopies(RHS.getReg(), MRI);
5215 auto P = static_cast<CmpInst::Predicate>(Predicate.getPredicate());
5216
5217 // Given this:
5218 //
5219 // x = G_SUB 0, y
5220 // G_ICMP z, x
5221 //
5222 // Produce this:
5223 //
5224 // cmn z, y
5225 if (isCMN(RHSDef, P, MRI))
5226 return emitCMN(LHS, RHSDef->getOperand(2), MIRBuilder);
5227
5228 // Same idea here, but with the LHS of the compare instead:
5229 //
5230 // Given this:
5231 //
5232 // x = G_SUB 0, y
5233 // G_ICMP x, z
5234 //
5235 // Produce this:
5236 //
5237 // cmn y, z
5238 //
5239 // But be careful! We need to swap the predicate!
5240 if (isCMN(LHSDef, P, MRI)) {
5241 if (!CmpInst::isEquality(P)) {
5244 }
5245 return emitCMN(LHSDef->getOperand(2), RHS, MIRBuilder);
5246 }
5247
5248 // Given this:
5249 //
5250 // z = G_AND x, y
5251 // G_ICMP z, 0
5252 //
5253 // Produce this if the compare is signed:
5254 //
5255 // tst x, y
5256 if (!CmpInst::isUnsigned(P) && LHSDef &&
5257 LHSDef->getOpcode() == TargetOpcode::G_AND) {
5258 // Make sure that the RHS is 0.
5259 auto ValAndVReg = getIConstantVRegValWithLookThrough(RHS.getReg(), MRI);
5260 if (!ValAndVReg || ValAndVReg->Value != 0)
5261 return nullptr;
5262
5263 return emitTST(LHSDef->getOperand(1),
5264 LHSDef->getOperand(2), MIRBuilder);
5265 }
5266
5267 return nullptr;
5268}
5269
5270bool AArch64InstructionSelector::selectShuffleVector(
5271 MachineInstr &I, MachineRegisterInfo &MRI) {
5272 const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
5273 Register Src1Reg = I.getOperand(1).getReg();
5274 Register Src2Reg = I.getOperand(2).getReg();
5275 ArrayRef<int> Mask = I.getOperand(3).getShuffleMask();
5276 assert(DstTy == MRI.getType(Src1Reg) &&
5277 "Expected equal shuffle types during selection");
5278
5279 MachineBasicBlock &MBB = *I.getParent();
5280 MachineFunction &MF = *MBB.getParent();
5281 LLVMContext &Ctx = MF.getFunction().getContext();
5282
5283 unsigned BytesPerElt = DstTy.getElementType().getSizeInBits() / 8;
5284 int NumElts = DstTy.getNumElements();
5285
5286 SmallVector<int> NewMask;
5287 bool FirstUsed = false;
5288 bool SecondUsed = false;
5289 for (int M : Mask) {
5290 // Map any undef or zero lanes to 255.
5291 if (M < 0 || VT->getKnownBits(M < NumElts ? Src1Reg : Src2Reg,
5292 APInt::getOneBitSet(NumElts, M % NumElts))
5293 .isZero()) {
5294 for (unsigned Byte = 0; Byte < BytesPerElt; ++Byte)
5295 NewMask.push_back(255);
5296 continue;
5297 }
5298
5299 FirstUsed |= M < NumElts;
5300 SecondUsed |= M >= NumElts;
5301 for (unsigned Byte = 0; Byte < BytesPerElt; ++Byte) {
5302 unsigned Offset = Byte + M * BytesPerElt;
5303 NewMask.push_back(Offset);
5304 }
5305 }
5306
5307 // If the first is unused or all zeros, use the second src in a tbl1.
5308 if (!FirstUsed) {
5309 int ByteLanes = DstTy.getSizeInBits() == 128 ? 16 : 8;
5310 for (int &M : NewMask) {
5311 if (M != 255) {
5312 assert(M >= ByteLanes && M < 2 * ByteLanes);
5313 M -= ByteLanes;
5314 }
5315 }
5316 std::swap(Src1Reg, Src2Reg);
5317 std::swap(FirstUsed, SecondUsed);
5318 }
5319
5320 // Use a constant pool to load the index vector for TBL.
5322 transform(NewMask, std::back_inserter(CstIdxs), [&Ctx](int M) {
5323 return ConstantInt::get(Type::getInt8Ty(Ctx), M);
5324 });
5325 Constant *CPVal = ConstantVector::get(CstIdxs);
5326 MachineInstr *IndexLoad = emitLoadFromConstantPool(CPVal, MIB);
5327 if (!IndexLoad) {
5328 LLVM_DEBUG(dbgs() << "Could not load from a constant pool");
5329 return false;
5330 }
5331
5332 if (DstTy.getSizeInBits() != 128) {
5333 assert(DstTy.getSizeInBits() == 64 && "Unexpected shuffle result ty");
5334 // This case can be done with TBL1.
5335 MachineInstr *Concat =
5336 emitVectorConcat(std::nullopt, Src1Reg, Src2Reg, MIB);
5337 if (!Concat) {
5338 LLVM_DEBUG(dbgs() << "Could not do vector concat for tbl1");
5339 return false;
5340 }
5341
5342 // The constant pool load will be 64 bits, so need to convert to FPR128 reg.
5343 IndexLoad = emitScalarToVector(64, &AArch64::FPR128RegClass,
5344 IndexLoad->getOperand(0).getReg(), MIB);
5345
5346 auto TBL1 = MIB.buildInstr(
5347 AArch64::TBLv16i8One, {&AArch64::FPR128RegClass},
5348 {Concat->getOperand(0).getReg(), IndexLoad->getOperand(0).getReg()});
5350
5351 auto Copy =
5352 MIB.buildInstr(TargetOpcode::COPY, {I.getOperand(0).getReg()}, {})
5353 .addReg(TBL1.getReg(0), {}, AArch64::dsub);
5354 RBI.constrainGenericRegister(Copy.getReg(0), AArch64::FPR64RegClass, MRI);
5355 I.eraseFromParent();
5356 return true;
5357 }
5358
5359 if (!SecondUsed) {
5360 auto TBL1 = MIB.buildInstr(AArch64::TBLv16i8One, {I.getOperand(0)},
5361 {Src1Reg, IndexLoad->getOperand(0)});
5363 I.eraseFromParent();
5364 return true;
5365 }
5366
5367 // For TBL2 we need to emit a REG_SEQUENCE to tie together two consecutive
5368 // Q registers for regalloc.
5369 SmallVector<Register, 2> Regs = {Src1Reg, Src2Reg};
5370 auto RegSeq = createQTuple(Regs, MIB);
5371 auto TBL2 = MIB.buildInstr(AArch64::TBLv16i8Two, {I.getOperand(0)},
5372 {RegSeq, IndexLoad->getOperand(0)});
5374 I.eraseFromParent();
5375 return true;
5376}
5377
5378MachineInstr *AArch64InstructionSelector::emitLaneInsert(
5379 std::optional<Register> DstReg, Register SrcReg, Register EltReg,
5380 unsigned LaneIdx, const RegisterBank &RB,
5381 MachineIRBuilder &MIRBuilder) const {
5382 MachineInstr *InsElt = nullptr;
5383 const TargetRegisterClass *DstRC = &AArch64::FPR128RegClass;
5384 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
5385
5386 // Create a register to define with the insert if one wasn't passed in.
5387 if (!DstReg)
5388 DstReg = MRI.createVirtualRegister(DstRC);
5389
5390 unsigned EltSize = MRI.getType(EltReg).getSizeInBits();
5391 unsigned Opc = getInsertVecEltOpInfo(RB, EltSize).first;
5392
5393 if (RB.getID() == AArch64::FPRRegBankID) {
5394 auto InsSub = emitScalarToVector(EltSize, DstRC, EltReg, MIRBuilder);
5395 InsElt = MIRBuilder.buildInstr(Opc, {*DstReg}, {SrcReg})
5396 .addImm(LaneIdx)
5397 .addUse(InsSub->getOperand(0).getReg())
5398 .addImm(0);
5399 } else {
5400 InsElt = MIRBuilder.buildInstr(Opc, {*DstReg}, {SrcReg})
5401 .addImm(LaneIdx)
5402 .addUse(EltReg);
5403 }
5404
5406 return InsElt;
5407}
5408
5409bool AArch64InstructionSelector::selectUSMovFromExtend(
5410 MachineInstr &MI, MachineRegisterInfo &MRI) {
5411 if (MI.getOpcode() != TargetOpcode::G_SEXT &&
5412 MI.getOpcode() != TargetOpcode::G_ZEXT &&
5413 MI.getOpcode() != TargetOpcode::G_ANYEXT)
5414 return false;
5415 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SEXT;
5416 const Register DefReg = MI.getOperand(0).getReg();
5417 const LLT DstTy = MRI.getType(DefReg);
5418 unsigned DstSize = DstTy.getSizeInBits();
5419
5420 if (DstSize != 32 && DstSize != 64)
5421 return false;
5422
5423 MachineInstr *Extract = getOpcodeDef(TargetOpcode::G_EXTRACT_VECTOR_ELT,
5424 MI.getOperand(1).getReg(), MRI);
5425 int64_t Lane;
5426 if (!Extract || !mi_match(Extract->getOperand(2).getReg(), MRI, m_ICst(Lane)))
5427 return false;
5428 Register Src0 = Extract->getOperand(1).getReg();
5429
5430 const LLT VecTy = MRI.getType(Src0);
5431 if (VecTy.isScalableVector())
5432 return false;
5433
5434 if (VecTy.getSizeInBits() != 128) {
5435 const MachineInstr *ScalarToVector = emitScalarToVector(
5436 VecTy.getSizeInBits(), &AArch64::FPR128RegClass, Src0, MIB);
5437 assert(ScalarToVector && "Didn't expect emitScalarToVector to fail!");
5438 Src0 = ScalarToVector->getOperand(0).getReg();
5439 }
5440
5441 unsigned Opcode;
5442 if (DstSize == 64 && VecTy.getScalarSizeInBits() == 32)
5443 Opcode = IsSigned ? AArch64::SMOVvi32to64 : AArch64::UMOVvi32;
5444 else if (DstSize == 64 && VecTy.getScalarSizeInBits() == 16)
5445 Opcode = IsSigned ? AArch64::SMOVvi16to64 : AArch64::UMOVvi16;
5446 else if (DstSize == 64 && VecTy.getScalarSizeInBits() == 8)
5447 Opcode = IsSigned ? AArch64::SMOVvi8to64 : AArch64::UMOVvi8;
5448 else if (DstSize == 32 && VecTy.getScalarSizeInBits() == 16)
5449 Opcode = IsSigned ? AArch64::SMOVvi16to32 : AArch64::UMOVvi16;
5450 else if (DstSize == 32 && VecTy.getScalarSizeInBits() == 8)
5451 Opcode = IsSigned ? AArch64::SMOVvi8to32 : AArch64::UMOVvi8;
5452 else
5453 llvm_unreachable("Unexpected type combo for S/UMov!");
5454
5455 // We may need to generate one of these, depending on the type and sign of the
5456 // input:
5457 // DstReg = SMOV Src0, Lane;
5458 // NewReg = UMOV Src0, Lane; DstReg = SUBREG_TO_REG NewReg, sub_32;
5459 MachineInstr *ExtI = nullptr;
5460 if (DstSize == 64 && !IsSigned) {
5461 Register NewReg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
5462 MIB.buildInstr(Opcode, {NewReg}, {Src0}).addImm(Lane);
5463 ExtI = MIB.buildInstr(AArch64::SUBREG_TO_REG, {DefReg}, {})
5464 .addUse(NewReg)
5465 .addImm(AArch64::sub_32);
5466 RBI.constrainGenericRegister(DefReg, AArch64::GPR64RegClass, MRI);
5467 } else
5468 ExtI = MIB.buildInstr(Opcode, {DefReg}, {Src0}).addImm(Lane);
5469
5471 MI.eraseFromParent();
5472 return true;
5473}
5474
5475bool AArch64InstructionSelector::selectIndexedExtLoad(
5476 MachineInstr &MI, MachineRegisterInfo &MRI) {
5477 auto &ExtLd = cast<GIndexedAnyExtLoad>(MI);
5478 Register Dst = ExtLd.getDstReg();
5479 Register WriteBack = ExtLd.getWritebackReg();
5480 Register Base = ExtLd.getBaseReg();
5481 Register Offset = ExtLd.getOffsetReg();
5482 LLT Ty = MRI.getType(Dst);
5483 assert(Ty.getSizeInBits() <= 64); // Only for scalar GPRs.
5484 unsigned MemSizeBits = ExtLd.getMMO().getMemoryType().getSizeInBits();
5485 bool IsPre = ExtLd.isPre();
5486 bool IsSExt = isa<GIndexedSExtLoad>(ExtLd);
5487 unsigned InsertIntoSubReg = 0;
5488 bool IsDst64 = Ty.getSizeInBits() == 64;
5489
5490 // ZExt/SExt should be on gpr but can handle extload and zextload of fpr, so
5491 // long as they are scalar.
5492 bool IsFPR = RBI.getRegBank(Dst, MRI, TRI)->getID() == AArch64::FPRRegBankID;
5493 if ((IsSExt && IsFPR) || Ty.isVector())
5494 return false;
5495
5496 unsigned Opc = 0;
5497 LLT NewLdDstTy;
5498 LLT s32 = LLT::scalar(32);
5499 LLT s64 = LLT::scalar(64);
5500
5501 if (MemSizeBits == 8) {
5502 if (IsSExt) {
5503 if (IsDst64)
5504 Opc = IsPre ? AArch64::LDRSBXpre : AArch64::LDRSBXpost;
5505 else
5506 Opc = IsPre ? AArch64::LDRSBWpre : AArch64::LDRSBWpost;
5507 NewLdDstTy = IsDst64 ? s64 : s32;
5508 } else if (IsFPR) {
5509 Opc = IsPre ? AArch64::LDRBpre : AArch64::LDRBpost;
5510 InsertIntoSubReg = AArch64::bsub;
5511 NewLdDstTy = LLT::scalar(MemSizeBits);
5512 } else {
5513 Opc = IsPre ? AArch64::LDRBBpre : AArch64::LDRBBpost;
5514 InsertIntoSubReg = IsDst64 ? AArch64::sub_32 : 0;
5515 NewLdDstTy = s32;
5516 }
5517 } else if (MemSizeBits == 16) {
5518 if (IsSExt) {
5519 if (IsDst64)
5520 Opc = IsPre ? AArch64::LDRSHXpre : AArch64::LDRSHXpost;
5521 else
5522 Opc = IsPre ? AArch64::LDRSHWpre : AArch64::LDRSHWpost;
5523 NewLdDstTy = IsDst64 ? s64 : s32;
5524 } else if (IsFPR) {
5525 Opc = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
5526 InsertIntoSubReg = AArch64::hsub;
5527 NewLdDstTy = LLT::scalar(MemSizeBits);
5528 } else {
5529 Opc = IsPre ? AArch64::LDRHHpre : AArch64::LDRHHpost;
5530 InsertIntoSubReg = IsDst64 ? AArch64::sub_32 : 0;
5531 NewLdDstTy = s32;
5532 }
5533 } else if (MemSizeBits == 32) {
5534 if (IsSExt) {
5535 Opc = IsPre ? AArch64::LDRSWpre : AArch64::LDRSWpost;
5536 NewLdDstTy = s64;
5537 } else if (IsFPR) {
5538 Opc = IsPre ? AArch64::LDRSpre : AArch64::LDRSpost;
5539 InsertIntoSubReg = AArch64::ssub;
5540 NewLdDstTy = LLT::scalar(MemSizeBits);
5541 } else {
5542 Opc = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
5543 InsertIntoSubReg = IsDst64 ? AArch64::sub_32 : 0;
5544 NewLdDstTy = s32;
5545 }
5546 } else {
5547 llvm_unreachable("Unexpected size for indexed load");
5548 }
5549
5550 auto Cst = getIConstantVRegVal(Offset, MRI);
5551 if (!Cst)
5552 return false; // Shouldn't happen, but just in case.
5553
5554 auto LdMI = MIB.buildInstr(Opc, {WriteBack, NewLdDstTy}, {Base})
5555 .addImm(Cst->getSExtValue());
5556 LdMI.cloneMemRefs(ExtLd);
5558 // Make sure to select the load with the MemTy as the dest type, and then
5559 // insert into a larger reg if needed.
5560 if (InsertIntoSubReg) {
5561 // Generate a SUBREG_TO_REG.
5562 auto SubToReg = MIB.buildInstr(TargetOpcode::SUBREG_TO_REG, {Dst}, {})
5563 .addUse(LdMI.getReg(1))
5564 .addImm(InsertIntoSubReg);
5566 SubToReg.getReg(0),
5567 *getRegClassForTypeOnBank(MRI.getType(Dst),
5568 *RBI.getRegBank(Dst, MRI, TRI)),
5569 MRI);
5570 } else {
5571 auto Copy = MIB.buildCopy(Dst, LdMI.getReg(1));
5572 selectCopy(*Copy, TII, MRI, TRI, RBI);
5573 }
5574 MI.eraseFromParent();
5575
5576 return true;
5577}
5578
5579bool AArch64InstructionSelector::selectIndexedLoad(MachineInstr &MI,
5580 MachineRegisterInfo &MRI) {
5581 auto &Ld = cast<GIndexedLoad>(MI);
5582 Register Dst = Ld.getDstReg();
5583 Register WriteBack = Ld.getWritebackReg();
5584 Register Base = Ld.getBaseReg();
5585 Register Offset = Ld.getOffsetReg();
5586 assert(MRI.getType(Dst).getSizeInBits() <= 128 &&
5587 "Unexpected type for indexed load");
5588 unsigned MemSize = Ld.getMMO().getMemoryType().getSizeInBytes();
5589
5590 if (MemSize < MRI.getType(Dst).getSizeInBytes())
5591 return selectIndexedExtLoad(MI, MRI);
5592
5593 unsigned Opc = 0;
5594 if (Ld.isPre()) {
5595 static constexpr unsigned GPROpcodes[] = {
5596 AArch64::LDRBBpre, AArch64::LDRHHpre, AArch64::LDRWpre,
5597 AArch64::LDRXpre};
5598 static constexpr unsigned FPROpcodes[] = {
5599 AArch64::LDRBpre, AArch64::LDRHpre, AArch64::LDRSpre, AArch64::LDRDpre,
5600 AArch64::LDRQpre};
5601 Opc = (RBI.getRegBank(Dst, MRI, TRI)->getID() == AArch64::FPRRegBankID)
5602 ? FPROpcodes[Log2_32(MemSize)]
5603 : GPROpcodes[Log2_32(MemSize)];
5604 ;
5605 } else {
5606 static constexpr unsigned GPROpcodes[] = {
5607 AArch64::LDRBBpost, AArch64::LDRHHpost, AArch64::LDRWpost,
5608 AArch64::LDRXpost};
5609 static constexpr unsigned FPROpcodes[] = {
5610 AArch64::LDRBpost, AArch64::LDRHpost, AArch64::LDRSpost,
5611 AArch64::LDRDpost, AArch64::LDRQpost};
5612 Opc = (RBI.getRegBank(Dst, MRI, TRI)->getID() == AArch64::FPRRegBankID)
5613 ? FPROpcodes[Log2_32(MemSize)]
5614 : GPROpcodes[Log2_32(MemSize)];
5615 ;
5616 }
5617 auto Cst = getIConstantVRegVal(Offset, MRI);
5618 if (!Cst)
5619 return false; // Shouldn't happen, but just in case.
5620 auto LdMI =
5621 MIB.buildInstr(Opc, {WriteBack, Dst}, {Base}).addImm(Cst->getSExtValue());
5622 LdMI.cloneMemRefs(Ld);
5624 MI.eraseFromParent();
5625 return true;
5626}
5627
5628bool AArch64InstructionSelector::selectIndexedStore(GIndexedStore &I,
5629 MachineRegisterInfo &MRI) {
5630 Register Dst = I.getWritebackReg();
5631 Register Val = I.getValueReg();
5632 Register Base = I.getBaseReg();
5633 Register Offset = I.getOffsetReg();
5634 assert(MRI.getType(Val).getSizeInBits() <= 128 &&
5635 "Unexpected type for indexed store");
5636
5637 LocationSize MemSize = I.getMMO().getSize();
5638 unsigned MemSizeInBytes = MemSize.getValue();
5639
5640 assert(MemSizeInBytes && MemSizeInBytes <= 16 &&
5641 "Unexpected indexed store size");
5642 unsigned MemSizeLog2 = Log2_32(MemSizeInBytes);
5643
5644 unsigned Opc = 0;
5645 if (I.isPre()) {
5646 static constexpr unsigned GPROpcodes[] = {
5647 AArch64::STRBBpre, AArch64::STRHHpre, AArch64::STRWpre,
5648 AArch64::STRXpre};
5649 static constexpr unsigned FPROpcodes[] = {
5650 AArch64::STRBpre, AArch64::STRHpre, AArch64::STRSpre, AArch64::STRDpre,
5651 AArch64::STRQpre};
5652
5653 if (RBI.getRegBank(Val, MRI, TRI)->getID() == AArch64::FPRRegBankID)
5654 Opc = FPROpcodes[MemSizeLog2];
5655 else
5656 Opc = GPROpcodes[MemSizeLog2];
5657 } else {
5658 static constexpr unsigned GPROpcodes[] = {
5659 AArch64::STRBBpost, AArch64::STRHHpost, AArch64::STRWpost,
5660 AArch64::STRXpost};
5661 static constexpr unsigned FPROpcodes[] = {
5662 AArch64::STRBpost, AArch64::STRHpost, AArch64::STRSpost,
5663 AArch64::STRDpost, AArch64::STRQpost};
5664
5665 if (RBI.getRegBank(Val, MRI, TRI)->getID() == AArch64::FPRRegBankID)
5666 Opc = FPROpcodes[MemSizeLog2];
5667 else
5668 Opc = GPROpcodes[MemSizeLog2];
5669 }
5670
5671 auto Cst = getIConstantVRegVal(Offset, MRI);
5672 if (!Cst)
5673 return false; // Shouldn't happen, but just in case.
5674 auto Str =
5675 MIB.buildInstr(Opc, {Dst}, {Val, Base}).addImm(Cst->getSExtValue());
5676 Str.cloneMemRefs(I);
5678 I.eraseFromParent();
5679 return true;
5680}
5681
5682MachineInstr *
5683AArch64InstructionSelector::emitConstantMOVVector(DstOp Dst, APInt Imm,
5684 MachineRegisterInfo &MRI) {
5685 unsigned DstSize = Imm.getBitWidth();
5686 assert((DstSize == 64 || DstSize == 128) &&
5687 "Unexpected vector constant size");
5688
5690 if (AArch64_IMM::expandVectorMOVImm(Imm, Insns, STI)) {
5691 for (AArch64_IMM::ImmInsnModel Insn : Insns) {
5692 switch (Insn.Opcode) {
5693 case AArch64::FMOVD0: {
5694 // Currently emit a MOVIv2d_ns in both cases to match SDAG.
5695 auto Mov =
5696 MIB.buildInstr(
5697 AArch64::MOVIv2d_ns,
5698 {DstSize == 128 ? Dst : DstOp(&AArch64::FPR128RegClass)}, {})
5699 .addImm(0);
5701 if (DstSize != 128) {
5702 Mov = MIB.buildInstr(TargetOpcode::COPY, {Dst}, {})
5703 .addReg(Mov.getReg(0), {}, AArch64::dsub);
5704 RBI.constrainGenericRegister(Mov.getReg(0), AArch64::FPR64RegClass,
5705 MRI);
5706 }
5707 return &*Mov;
5708 }
5709 case AArch64::FMOVDi:
5710 case AArch64::FMOVv2f32_ns:
5711 case AArch64::FMOVv2f64_ns:
5712 case AArch64::FMOVv4f32_ns:
5713 case AArch64::FMOVv4f16_ns:
5714 case AArch64::FMOVv8f16_ns:
5715 case AArch64::MOVID:
5716 case AArch64::MOVIv16b_ns:
5717 case AArch64::MOVIv2d_ns:
5718 case AArch64::MOVIv2i32:
5719 case AArch64::MOVIv2s_msl:
5720 case AArch64::MOVIv4i16:
5721 case AArch64::MOVIv4i32:
5722 case AArch64::MOVIv4s_msl:
5723 case AArch64::MOVIv8b_ns:
5724 case AArch64::MOVIv8i16:
5725 case AArch64::MVNIv2i32:
5726 case AArch64::MVNIv2s_msl:
5727 case AArch64::MVNIv4i16:
5728 case AArch64::MVNIv4i32:
5729 case AArch64::MVNIv4s_msl:
5730 case AArch64::MVNIv8i16: {
5731 auto Mov = MIB.buildInstr(Insn.Opcode, {Dst}, {});
5732 if (Insn.Op1)
5733 Mov.addImm(*Insn.Op1);
5734 if (Insn.Op2)
5735 Mov.addImm(*Insn.Op2);
5737 return &*Mov;
5738 }
5739 case AArch64::DUPM_ZI: {
5740 auto Mov = MIB.buildInstr(Insn.Opcode, {&AArch64::ZPRRegClass}, {})
5741 .addImm(*Insn.Op1);
5743 Mov = MIB.buildInstr(TargetOpcode::COPY, {Dst}, {})
5744 .addReg(Mov.getReg(0), {},
5745 DstSize == 64 ? AArch64::dsub : AArch64::zsub);
5746 RBI.constrainGenericRegister(Mov.getReg(0),
5747 DstSize == 64 ? AArch64::FPR64RegClass
5748 : AArch64::FPR128RegClass,
5749 MRI);
5750 return &*Mov;
5751 }
5752 case AArch64::DUP_ZI_D: {
5753 auto Mov = MIB.buildInstr(Insn.Opcode, {&AArch64::ZPRRegClass}, {})
5754 .addImm(*Insn.Op1)
5755 .addImm(*Insn.Op2);
5757 Mov = MIB.buildInstr(TargetOpcode::COPY, {Dst}, {})
5758 .addReg(Mov.getReg(0), {},
5759 DstSize == 64 ? AArch64::dsub : AArch64::zsub);
5760 RBI.constrainGenericRegister(Mov.getReg(0),
5761 DstSize == 64 ? AArch64::FPR64RegClass
5762 : AArch64::FPR128RegClass,
5763 MRI);
5764 return &*Mov;
5765 }
5766 default:
5767 llvm_unreachable("Unexpected node in expandVectorMOVImm\n");
5768 }
5769 }
5770 }
5771 return nullptr;
5772}
5773
5774MachineInstr *AArch64InstructionSelector::emitConstantVector(
5775 Register Dst, Constant *CV, APInt Imm, MachineRegisterInfo &MRI) {
5776 if (auto *MovI = emitConstantMOVVector(Dst, Imm, MRI))
5777 return MovI;
5778
5779 auto *CPLoad = emitLoadFromConstantPool(CV, MIB);
5780 if (!CPLoad) {
5781 LLVM_DEBUG(dbgs() << "Could not generate cp load for constant vector!");
5782 return nullptr;
5783 }
5784
5785 auto Copy = MIB.buildCopy(Dst, CPLoad->getOperand(0));
5787 Dst, *MRI.getRegClass(CPLoad->getOperand(0).getReg()), MRI);
5788 return &*Copy;
5789}
5790
5791bool AArch64InstructionSelector::tryOptConstantBuildVec(
5792 MachineInstr &I, LLT DstTy, MachineRegisterInfo &MRI) {
5793 assert(I.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
5794 unsigned DstSize = DstTy.getSizeInBits();
5795 assert(DstSize <= 128 && "Unexpected build_vec type!");
5796 if (DstSize < 32)
5797 return false;
5798 // Check if we're building a constant vector, in which case we want to
5799 // generate a constant pool load instead of a vector insert sequence.
5801 APInt Imm(DstSize, 0);
5802 for (unsigned Idx = 1; Idx < I.getNumOperands(); ++Idx) {
5803 Register OpReg = I.getOperand(Idx).getReg();
5804 if (auto AnyConst = getAnyConstantVRegValWithLookThrough(
5805 OpReg, MRI, /*LookThroughInstrs=*/true,
5806 /*LookThroughAnyExt=*/true)) {
5807 MachineInstr *DefMI = MRI.getVRegDef(AnyConst->VReg);
5808 Imm |= AnyConst->Value.trunc(DstTy.getScalarSizeInBits())
5809 .zext(DstSize)
5810 .shl((Idx - 1) * DstTy.getScalarSizeInBits());
5811
5812 if (DefMI->getOpcode() == TargetOpcode::G_CONSTANT) {
5813 Csts.emplace_back(
5814 ConstantInt::get(MIB.getMF().getFunction().getContext(),
5815 std::move(AnyConst->Value)));
5816 continue;
5817 }
5818
5819 if (DefMI->getOpcode() == TargetOpcode::G_FCONSTANT) {
5820 Csts.emplace_back(
5821 const_cast<ConstantFP *>(DefMI->getOperand(1).getFPImm()));
5822 continue;
5823 }
5824 }
5825 return false;
5826 }
5827 Constant *CV = ConstantVector::get(Csts);
5828 if (!emitConstantVector(I.getOperand(0).getReg(), CV, Imm, MRI))
5829 return false;
5830 I.eraseFromParent();
5831 return true;
5832}
5833
5834bool AArch64InstructionSelector::tryOptBuildVecToSubregToReg(
5835 MachineInstr &I, MachineRegisterInfo &MRI) {
5836 // Given:
5837 // %vec = G_BUILD_VECTOR %elt, %undef, %undef, ... %undef
5838 //
5839 // Select the G_BUILD_VECTOR as a SUBREG_TO_REG from %elt.
5840 Register Dst = I.getOperand(0).getReg();
5841 Register EltReg = I.getOperand(1).getReg();
5842 LLT EltTy = MRI.getType(EltReg);
5843 // If the index isn't on the same bank as its elements, then this can't be a
5844 // SUBREG_TO_REG.
5845 const RegisterBank &EltRB = *RBI.getRegBank(EltReg, MRI, TRI);
5846 const RegisterBank &DstRB = *RBI.getRegBank(Dst, MRI, TRI);
5847 if (EltRB != DstRB)
5848 return false;
5849 if (any_of(drop_begin(I.operands(), 2), [&MRI](const MachineOperand &Op) {
5850 return !getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, Op.getReg(), MRI);
5851 }))
5852 return false;
5853 unsigned SubReg;
5854 const TargetRegisterClass *EltRC = getRegClassForTypeOnBank(EltTy, EltRB);
5855 if (!EltRC)
5856 return false;
5857 const TargetRegisterClass *DstRC =
5858 getRegClassForTypeOnBank(MRI.getType(Dst), DstRB);
5859 if (!DstRC)
5860 return false;
5861 if (!getSubRegForClass(EltRC, TRI, SubReg))
5862 return false;
5863 auto SubregToReg = MIB.buildInstr(AArch64::SUBREG_TO_REG, {Dst}, {})
5864 .addUse(EltReg)
5865 .addImm(SubReg);
5866 I.eraseFromParent();
5867 constrainSelectedInstRegOperands(*SubregToReg, TII, TRI, RBI);
5868 return RBI.constrainGenericRegister(Dst, *DstRC, MRI);
5869}
5870
5871bool AArch64InstructionSelector::selectBuildVector(MachineInstr &I,
5872 MachineRegisterInfo &MRI) {
5873 assert(I.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
5874 // Until we port more of the optimized selections, for now just use a vector
5875 // insert sequence.
5876 const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
5877 const LLT EltTy = MRI.getType(I.getOperand(1).getReg());
5878 unsigned EltSize = EltTy.getSizeInBits();
5879
5880 if (tryOptConstantBuildVec(I, DstTy, MRI))
5881 return true;
5882 if (tryOptBuildVecToSubregToReg(I, MRI))
5883 return true;
5884
5885 if (EltSize != 8 && EltSize != 16 && EltSize != 32 && EltSize != 64)
5886 return false; // Don't support all element types yet.
5887 const RegisterBank &RB = *RBI.getRegBank(I.getOperand(1).getReg(), MRI, TRI);
5888
5889 const TargetRegisterClass *DstRC = &AArch64::FPR128RegClass;
5890 MachineInstr *ScalarToVec =
5891 emitScalarToVector(DstTy.getElementType().getSizeInBits(), DstRC,
5892 I.getOperand(1).getReg(), MIB);
5893 if (!ScalarToVec)
5894 return false;
5895
5896 Register DstVec = ScalarToVec->getOperand(0).getReg();
5897 unsigned DstSize = DstTy.getSizeInBits();
5898
5899 // Keep track of the last MI we inserted. Later on, we might be able to save
5900 // a copy using it.
5901 MachineInstr *PrevMI = ScalarToVec;
5902 for (unsigned i = 2, e = DstSize / EltSize + 1; i < e; ++i) {
5903 // Note that if we don't do a subregister copy, we can end up making an
5904 // extra register.
5905 Register OpReg = I.getOperand(i).getReg();
5906 // Do not emit inserts for undefs
5907 if (!getOpcodeDef<GImplicitDef>(OpReg, MRI)) {
5908 PrevMI = &*emitLaneInsert(std::nullopt, DstVec, OpReg, i - 1, RB, MIB);
5909 DstVec = PrevMI->getOperand(0).getReg();
5910 }
5911 }
5912
5913 // If DstTy's size in bits is less than 128, then emit a subregister copy
5914 // from DstVec to the last register we've defined.
5915 if (DstSize < 128) {
5916 // Force this to be FPR using the destination vector.
5917 const TargetRegisterClass *RC =
5918 getRegClassForTypeOnBank(DstTy, *RBI.getRegBank(DstVec, MRI, TRI));
5919 if (!RC)
5920 return false;
5921 if (RC != &AArch64::FPR32RegClass && RC != &AArch64::FPR64RegClass) {
5922 LLVM_DEBUG(dbgs() << "Unsupported register class!\n");
5923 return false;
5924 }
5925
5926 unsigned SubReg = 0;
5927 if (!getSubRegForClass(RC, TRI, SubReg))
5928 return false;
5929 if (SubReg != AArch64::ssub && SubReg != AArch64::dsub) {
5930 LLVM_DEBUG(dbgs() << "Unsupported destination size! (" << DstSize
5931 << "\n");
5932 return false;
5933 }
5934
5936 Register DstReg = I.getOperand(0).getReg();
5937
5938 MIB.buildInstr(TargetOpcode::COPY, {DstReg}, {}).addReg(DstVec, {}, SubReg);
5939 MachineOperand &RegOp = I.getOperand(1);
5940 RegOp.setReg(Reg);
5941 RBI.constrainGenericRegister(DstReg, *RC, MRI);
5942 } else {
5943 // We either have a vector with all elements (except the first one) undef or
5944 // at least one non-undef non-first element. In the first case, we need to
5945 // constrain the output register ourselves as we may have generated an
5946 // INSERT_SUBREG operation which is a generic operation for which the
5947 // output regclass cannot be automatically chosen.
5948 //
5949 // In the second case, there is no need to do this as it may generate an
5950 // instruction like INSvi32gpr where the regclass can be automatically
5951 // chosen.
5952 //
5953 // Also, we save a copy by re-using the destination register on the final
5954 // insert.
5955 PrevMI->getOperand(0).setReg(I.getOperand(0).getReg());
5957
5958 Register DstReg = PrevMI->getOperand(0).getReg();
5959 if (PrevMI == ScalarToVec && DstReg.isVirtual()) {
5960 const TargetRegisterClass *RC =
5961 getRegClassForTypeOnBank(DstTy, *RBI.getRegBank(DstVec, MRI, TRI));
5962 RBI.constrainGenericRegister(DstReg, *RC, MRI);
5963 }
5964 }
5965
5967 return true;
5968}
5969
5970bool AArch64InstructionSelector::selectVectorLoadIntrinsic(unsigned Opc,
5971 unsigned NumVecs,
5972 MachineInstr &I) {
5973 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS);
5974 assert(Opc && "Expected an opcode?");
5975 assert(NumVecs > 1 && NumVecs < 5 && "Only support 2, 3, or 4 vectors");
5976 auto &MRI = *MIB.getMRI();
5977 LLT Ty = MRI.getType(I.getOperand(0).getReg());
5978 unsigned Size = Ty.getSizeInBits();
5979 assert((Size == 64 || Size == 128) &&
5980 "Destination must be 64 bits or 128 bits?");
5981 unsigned SubReg = Size == 64 ? AArch64::dsub0 : AArch64::qsub0;
5982 auto Ptr = I.getOperand(I.getNumOperands() - 1).getReg();
5983 assert(MRI.getType(Ptr).isPointer() && "Expected a pointer type?");
5984 auto Load = MIB.buildInstr(Opc, {Ty}, {Ptr});
5987 Register SelectedLoadDst = Load->getOperand(0).getReg();
5988 for (unsigned Idx = 0; Idx < NumVecs; ++Idx) {
5989 auto Vec = MIB.buildInstr(TargetOpcode::COPY, {I.getOperand(Idx)}, {})
5990 .addReg(SelectedLoadDst, {}, SubReg + Idx);
5991 // Emit the subreg copies and immediately select them.
5992 // FIXME: We should refactor our copy code into an emitCopy helper and
5993 // clean up uses of this pattern elsewhere in the selector.
5994 selectCopy(*Vec, TII, MRI, TRI, RBI);
5995 }
5996 return true;
5997}
5998
5999bool AArch64InstructionSelector::selectVectorLoadLaneIntrinsic(
6000 unsigned Opc, unsigned NumVecs, MachineInstr &I) {
6001 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS);
6002 assert(Opc && "Expected an opcode?");
6003 assert(NumVecs > 1 && NumVecs < 5 && "Only support 2, 3, or 4 vectors");
6004 auto &MRI = *MIB.getMRI();
6005 LLT Ty = MRI.getType(I.getOperand(0).getReg());
6006 bool Narrow = Ty.getSizeInBits() == 64;
6007
6008 auto FirstSrcRegIt = I.operands_begin() + NumVecs + 1;
6009 SmallVector<Register, 4> Regs(NumVecs);
6010 std::transform(FirstSrcRegIt, FirstSrcRegIt + NumVecs, Regs.begin(),
6011 [](auto MO) { return MO.getReg(); });
6012
6013 if (Narrow) {
6014 transform(Regs, Regs.begin(), [this](Register Reg) {
6015 return emitScalarToVector(64, &AArch64::FPR128RegClass, Reg, MIB)
6016 ->getOperand(0)
6017 .getReg();
6018 });
6019 Ty = Ty.multiplyElements(2);
6020 }
6021
6022 Register Tuple = createQTuple(Regs, MIB);
6023 auto LaneNo = getIConstantVRegVal((FirstSrcRegIt + NumVecs)->getReg(), MRI);
6024 if (!LaneNo)
6025 return false;
6026
6027 Register Ptr = (FirstSrcRegIt + NumVecs + 1)->getReg();
6028 auto Load = MIB.buildInstr(Opc, {Ty}, {})
6029 .addReg(Tuple)
6030 .addImm(LaneNo->getZExtValue())
6031 .addReg(Ptr);
6034 Register SelectedLoadDst = Load->getOperand(0).getReg();
6035 unsigned SubReg = AArch64::qsub0;
6036 for (unsigned Idx = 0; Idx < NumVecs; ++Idx) {
6037 auto Vec = MIB.buildInstr(TargetOpcode::COPY,
6038 {Narrow ? DstOp(&AArch64::FPR128RegClass)
6039 : DstOp(I.getOperand(Idx).getReg())},
6040 {})
6041 .addReg(SelectedLoadDst, {}, SubReg + Idx);
6042 Register WideReg = Vec.getReg(0);
6043 // Emit the subreg copies and immediately select them.
6044 selectCopy(*Vec, TII, MRI, TRI, RBI);
6045 if (Narrow &&
6046 !emitNarrowVector(I.getOperand(Idx).getReg(), WideReg, MIB, MRI))
6047 return false;
6048 }
6049 return true;
6050}
6051
6052void AArch64InstructionSelector::selectVectorStoreIntrinsic(MachineInstr &I,
6053 unsigned NumVecs,
6054 unsigned Opc) {
6055 MachineRegisterInfo &MRI = I.getParent()->getParent()->getRegInfo();
6056 LLT Ty = MRI.getType(I.getOperand(1).getReg());
6057 Register Ptr = I.getOperand(1 + NumVecs).getReg();
6058
6059 SmallVector<Register, 2> Regs(NumVecs);
6060 std::transform(I.operands_begin() + 1, I.operands_begin() + 1 + NumVecs,
6061 Regs.begin(), [](auto MO) { return MO.getReg(); });
6062
6063 Register Tuple = Ty.getSizeInBits() == 128 ? createQTuple(Regs, MIB)
6064 : createDTuple(Regs, MIB);
6065 auto Store = MIB.buildInstr(Opc, {}, {Tuple, Ptr});
6068}
6069
6070bool AArch64InstructionSelector::selectVectorStoreLaneIntrinsic(
6071 MachineInstr &I, unsigned NumVecs, unsigned Opc) {
6072 MachineRegisterInfo &MRI = I.getParent()->getParent()->getRegInfo();
6073 LLT Ty = MRI.getType(I.getOperand(1).getReg());
6074 bool Narrow = Ty.getSizeInBits() == 64;
6075
6076 SmallVector<Register, 2> Regs(NumVecs);
6077 std::transform(I.operands_begin() + 1, I.operands_begin() + 1 + NumVecs,
6078 Regs.begin(), [](auto MO) { return MO.getReg(); });
6079
6080 if (Narrow)
6081 transform(Regs, Regs.begin(), [this](Register Reg) {
6082 return emitScalarToVector(64, &AArch64::FPR128RegClass, Reg, MIB)
6083 ->getOperand(0)
6084 .getReg();
6085 });
6086
6087 Register Tuple = createQTuple(Regs, MIB);
6088
6089 auto LaneNo = getIConstantVRegVal(I.getOperand(1 + NumVecs).getReg(), MRI);
6090 if (!LaneNo)
6091 return false;
6092 Register Ptr = I.getOperand(1 + NumVecs + 1).getReg();
6093 auto Store = MIB.buildInstr(Opc, {}, {})
6094 .addReg(Tuple)
6095 .addImm(LaneNo->getZExtValue())
6096 .addReg(Ptr);
6099 return true;
6100}
6101
6102bool AArch64InstructionSelector::selectIntrinsicWithSideEffects(
6103 MachineInstr &I, MachineRegisterInfo &MRI) {
6104 // Find the intrinsic ID.
6105 unsigned IntrinID = cast<GIntrinsic>(I).getIntrinsicID();
6106
6107 const LLT S8 = LLT::scalar(8);
6108 const LLT S16 = LLT::scalar(16);
6109 const LLT S32 = LLT::scalar(32);
6110 const LLT S64 = LLT::scalar(64);
6111 const LLT P0 = LLT::pointer(0, 64);
6112 // Select the instruction.
6113 switch (IntrinID) {
6114 default:
6115 return false;
6116 case Intrinsic::aarch64_ldxp:
6117 case Intrinsic::aarch64_ldaxp: {
6118 auto NewI = MIB.buildInstr(
6119 IntrinID == Intrinsic::aarch64_ldxp ? AArch64::LDXPX : AArch64::LDAXPX,
6120 {I.getOperand(0).getReg(), I.getOperand(1).getReg()},
6121 {I.getOperand(3)});
6122 NewI.cloneMemRefs(I);
6124 break;
6125 }
6126 case Intrinsic::aarch64_neon_ld1x2: {
6127 LLT Ty = MRI.getType(I.getOperand(0).getReg());
6128 unsigned Opc = 0;
6129 if (Ty == LLT::fixed_vector(8, S8))
6130 Opc = AArch64::LD1Twov8b;
6131 else if (Ty == LLT::fixed_vector(16, S8))
6132 Opc = AArch64::LD1Twov16b;
6133 else if (Ty == LLT::fixed_vector(4, S16))
6134 Opc = AArch64::LD1Twov4h;
6135 else if (Ty == LLT::fixed_vector(8, S16))
6136 Opc = AArch64::LD1Twov8h;
6137 else if (Ty == LLT::fixed_vector(2, S32))
6138 Opc = AArch64::LD1Twov2s;
6139 else if (Ty == LLT::fixed_vector(4, S32))
6140 Opc = AArch64::LD1Twov4s;
6141 else if (Ty == LLT::fixed_vector(2, S64) || Ty == LLT::fixed_vector(2, P0))
6142 Opc = AArch64::LD1Twov2d;
6143 else if (Ty == S64 || Ty == P0)
6144 Opc = AArch64::LD1Twov1d;
6145 else
6146 llvm_unreachable("Unexpected type for ld1x2!");
6147 selectVectorLoadIntrinsic(Opc, 2, I);
6148 break;
6149 }
6150 case Intrinsic::aarch64_neon_ld1x3: {
6151 LLT Ty = MRI.getType(I.getOperand(0).getReg());
6152 unsigned Opc = 0;
6153 if (Ty == LLT::fixed_vector(8, S8))
6154 Opc = AArch64::LD1Threev8b;
6155 else if (Ty == LLT::fixed_vector(16, S8))
6156 Opc = AArch64::LD1Threev16b;
6157 else if (Ty == LLT::fixed_vector(4, S16))
6158 Opc = AArch64::LD1Threev4h;
6159 else if (Ty == LLT::fixed_vector(8, S16))
6160 Opc = AArch64::LD1Threev8h;
6161 else if (Ty == LLT::fixed_vector(2, S32))
6162 Opc = AArch64::LD1Threev2s;
6163 else if (Ty == LLT::fixed_vector(4, S32))
6164 Opc = AArch64::LD1Threev4s;
6165 else if (Ty == LLT::fixed_vector(2, S64) || Ty == LLT::fixed_vector(2, P0))
6166 Opc = AArch64::LD1Threev2d;
6167 else if (Ty == S64 || Ty == P0)
6168 Opc = AArch64::LD1Threev1d;
6169 else
6170 llvm_unreachable("Unexpected type for ld1x3!");
6171 selectVectorLoadIntrinsic(Opc, 3, I);
6172 break;
6173 }
6174 case Intrinsic::aarch64_neon_ld1x4: {
6175 LLT Ty = MRI.getType(I.getOperand(0).getReg());
6176 unsigned Opc = 0;
6177 if (Ty == LLT::fixed_vector(8, S8))
6178 Opc = AArch64::LD1Fourv8b;
6179 else if (Ty == LLT::fixed_vector(16, S8))
6180 Opc = AArch64::LD1Fourv16b;
6181 else if (Ty == LLT::fixed_vector(4, S16))
6182 Opc = AArch64::LD1Fourv4h;
6183 else if (Ty == LLT::fixed_vector(8, S16))
6184 Opc = AArch64::LD1Fourv8h;
6185 else if (Ty == LLT::fixed_vector(2, S32))
6186 Opc = AArch64::LD1Fourv2s;
6187 else if (Ty == LLT::fixed_vector(4, S32))
6188 Opc = AArch64::LD1Fourv4s;
6189 else if (Ty == LLT::fixed_vector(2, S64) || Ty == LLT::fixed_vector(2, P0))
6190 Opc = AArch64::LD1Fourv2d;
6191 else if (Ty == S64 || Ty == P0)
6192 Opc = AArch64::LD1Fourv1d;
6193 else
6194 llvm_unreachable("Unexpected type for ld1x4!");
6195 selectVectorLoadIntrinsic(Opc, 4, I);
6196 break;
6197 }
6198 case Intrinsic::aarch64_neon_ld2: {
6199 LLT Ty = MRI.getType(I.getOperand(0).getReg());
6200 unsigned Opc = 0;
6201 if (Ty == LLT::fixed_vector(8, S8))
6202 Opc = AArch64::LD2Twov8b;
6203 else if (Ty == LLT::fixed_vector(16, S8))
6204 Opc = AArch64::LD2Twov16b;
6205 else if (Ty == LLT::fixed_vector(4, S16))
6206 Opc = AArch64::LD2Twov4h;
6207 else if (Ty == LLT::fixed_vector(8, S16))
6208 Opc = AArch64::LD2Twov8h;
6209 else if (Ty == LLT::fixed_vector(2, S32))
6210 Opc = AArch64::LD2Twov2s;
6211 else if (Ty == LLT::fixed_vector(4, S32))
6212 Opc = AArch64::LD2Twov4s;
6213 else if (Ty == LLT::fixed_vector(2, S64) || Ty == LLT::fixed_vector(2, P0))
6214 Opc = AArch64::LD2Twov2d;
6215 else if (Ty == S64 || Ty == P0)
6216 Opc = AArch64::LD1Twov1d;
6217 else
6218 llvm_unreachable("Unexpected type for ld2!");
6219 selectVectorLoadIntrinsic(Opc, 2, I);
6220 break;
6221 }
6222 case Intrinsic::aarch64_neon_ld2lane: {
6223 LLT Ty = MRI.getType(I.getOperand(0).getReg());
6224 unsigned Opc;
6225 if (Ty == LLT::fixed_vector(8, S8) || Ty == LLT::fixed_vector(16, S8))
6226 Opc = AArch64::LD2i8;
6227 else if (Ty == LLT::fixed_vector(4, S16) || Ty == LLT::fixed_vector(8, S16))
6228 Opc = AArch64::LD2i16;
6229 else if (Ty == LLT::fixed_vector(2, S32) || Ty == LLT::fixed_vector(4, S32))
6230 Opc = AArch64::LD2i32;
6231 else if (Ty == LLT::fixed_vector(2, S64) ||
6232 Ty == LLT::fixed_vector(2, P0) || Ty == S64 || Ty == P0)
6233 Opc = AArch64::LD2i64;
6234 else
6235 llvm_unreachable("Unexpected type for ld2lane!");
6236 if (!selectVectorLoadLaneIntrinsic(Opc, 2, I))
6237 return false;
6238 break;
6239 }
6240 case Intrinsic::aarch64_neon_ld2r: {
6241 LLT Ty = MRI.getType(I.getOperand(0).getReg());
6242 unsigned Opc = 0;
6243 if (Ty == LLT::fixed_vector(8, S8))
6244 Opc = AArch64::LD2Rv8b;
6245 else if (Ty == LLT::fixed_vector(16, S8))
6246 Opc = AArch64::LD2Rv16b;
6247 else if (Ty == LLT::fixed_vector(4, S16))
6248 Opc = AArch64::LD2Rv4h;
6249 else if (Ty == LLT::fixed_vector(8, S16))
6250 Opc = AArch64::LD2Rv8h;
6251 else if (Ty == LLT::fixed_vector(2, S32))
6252 Opc = AArch64::LD2Rv2s;
6253 else if (Ty == LLT::fixed_vector(4, S32))
6254 Opc = AArch64::LD2Rv4s;
6255 else if (Ty == LLT::fixed_vector(2, S64) || Ty == LLT::fixed_vector(2, P0))
6256 Opc = AArch64::LD2Rv2d;
6257 else if (Ty == S64 || Ty == P0)
6258 Opc = AArch64::LD2Rv1d;
6259 else
6260 llvm_unreachable("Unexpected type for ld2r!");
6261 selectVectorLoadIntrinsic(Opc, 2, I);
6262 break;
6263 }
6264 case Intrinsic::aarch64_neon_ld3: {
6265 LLT Ty = MRI.getType(I.getOperand(0).getReg());
6266 unsigned Opc = 0;
6267 if (Ty == LLT::fixed_vector(8, S8))
6268 Opc = AArch64::LD3Threev8b;
6269 else if (Ty == LLT::fixed_vector(16, S8))
6270 Opc = AArch64::LD3Threev16b;
6271 else if (Ty == LLT::fixed_vector(4, S16))
6272 Opc = AArch64::LD3Threev4h;
6273 else if (Ty == LLT::fixed_vector(8, S16))
6274 Opc = AArch64::LD3Threev8h;
6275 else if (Ty == LLT::fixed_vector(2, S32))
6276 Opc = AArch64::LD3Threev2s;
6277 else if (Ty == LLT::fixed_vector(4, S32))
6278 Opc = AArch64::LD3Threev4s;
6279 else if (Ty == LLT::fixed_vector(2, S64) || Ty == LLT::fixed_vector(2, P0))
6280 Opc = AArch64::LD3Threev2d;
6281 else if (Ty == S64 || Ty == P0)
6282 Opc = AArch64::LD1Threev1d;
6283 else
6284 llvm_unreachable("Unexpected type for ld3!");
6285 selectVectorLoadIntrinsic(Opc, 3, I);
6286 break;
6287 }
6288 case Intrinsic::aarch64_neon_ld3lane: {
6289 LLT Ty = MRI.getType(I.getOperand(0).getReg());
6290 unsigned Opc;
6291 if (Ty == LLT::fixed_vector(8, S8) || Ty == LLT::fixed_vector(16, S8))
6292 Opc = AArch64::LD3i8;
6293 else if (Ty == LLT::fixed_vector(4, S16) || Ty == LLT::fixed_vector(8, S16))
6294 Opc = AArch64::LD3i16;
6295 else if (Ty == LLT::fixed_vector(2, S32) || Ty == LLT::fixed_vector(4, S32))
6296 Opc = AArch64::LD3i32;
6297 else if (Ty == LLT::fixed_vector(2, S64) ||
6298 Ty == LLT::fixed_vector(2, P0) || Ty == S64 || Ty == P0)
6299 Opc = AArch64::LD3i64;
6300 else
6301 llvm_unreachable("Unexpected type for ld3lane!");
6302 if (!selectVectorLoadLaneIntrinsic(Opc, 3, I))
6303 return false;
6304 break;
6305 }
6306 case Intrinsic::aarch64_neon_ld3r: {
6307 LLT Ty = MRI.getType(I.getOperand(0).getReg());
6308 unsigned Opc = 0;
6309 if (Ty == LLT::fixed_vector(8, S8))
6310 Opc = AArch64::LD3Rv8b;
6311 else if (Ty == LLT::fixed_vector(16, S8))
6312 Opc = AArch64::LD3Rv16b;
6313 else if (Ty == LLT::fixed_vector(4, S16))
6314 Opc = AArch64::LD3Rv4h;
6315 else if (Ty == LLT::fixed_vector(8, S16))
6316 Opc = AArch64::LD3Rv8h;
6317 else if (Ty == LLT::fixed_vector(2, S32))
6318 Opc = AArch64::LD3Rv2s;
6319 else if (Ty == LLT::fixed_vector(4, S32))
6320 Opc = AArch64::LD3Rv4s;
6321 else if (Ty == LLT::fixed_vector(2, S64) || Ty == LLT::fixed_vector(2, P0))
6322 Opc = AArch64::LD3Rv2d;
6323 else if (Ty == S64 || Ty == P0)
6324 Opc = AArch64::LD3Rv1d;
6325 else
6326 llvm_unreachable("Unexpected type for ld3r!");
6327 selectVectorLoadIntrinsic(Opc, 3, I);
6328 break;
6329 }
6330 case Intrinsic::aarch64_neon_ld4: {
6331 LLT Ty = MRI.getType(I.getOperand(0).getReg());
6332 unsigned Opc = 0;
6333 if (Ty == LLT::fixed_vector(8, S8))
6334 Opc = AArch64::LD4Fourv8b;
6335 else if (Ty == LLT::fixed_vector(16, S8))
6336 Opc = AArch64::LD4Fourv16b;
6337 else if (Ty == LLT::fixed_vector(4, S16))
6338 Opc = AArch64::LD4Fourv4h;
6339 else if (Ty == LLT::fixed_vector(8, S16))
6340 Opc = AArch64::LD4Fourv8h;
6341 else if (Ty == LLT::fixed_vector(2, S32))
6342 Opc = AArch64::LD4Fourv2s;
6343 else if (Ty == LLT::fixed_vector(4, S32))
6344 Opc = AArch64::LD4Fourv4s;
6345 else if (Ty == LLT::fixed_vector(2, S64) || Ty == LLT::fixed_vector(2, P0))
6346 Opc = AArch64::LD4Fourv2d;
6347 else if (Ty == S64 || Ty == P0)
6348 Opc = AArch64::LD1Fourv1d;
6349 else
6350 llvm_unreachable("Unexpected type for ld4!");
6351 selectVectorLoadIntrinsic(Opc, 4, I);
6352 break;
6353 }
6354 case Intrinsic::aarch64_neon_ld4lane: {
6355 LLT Ty = MRI.getType(I.getOperand(0).getReg());
6356 unsigned Opc;
6357 if (Ty == LLT::fixed_vector(8, S8) || Ty == LLT::fixed_vector(16, S8))
6358 Opc = AArch64::LD4i8;
6359 else if (Ty == LLT::fixed_vector(4, S16) || Ty == LLT::fixed_vector(8, S16))
6360 Opc = AArch64::LD4i16;
6361 else if (Ty == LLT::fixed_vector(2, S32) || Ty == LLT::fixed_vector(4, S32))
6362 Opc = AArch64::LD4i32;
6363 else if (Ty == LLT::fixed_vector(2, S64) ||
6364 Ty == LLT::fixed_vector(2, P0) || Ty == S64 || Ty == P0)
6365 Opc = AArch64::LD4i64;
6366 else
6367 llvm_unreachable("Unexpected type for ld4lane!");
6368 if (!selectVectorLoadLaneIntrinsic(Opc, 4, I))
6369 return false;
6370 break;
6371 }
6372 case Intrinsic::aarch64_neon_ld4r: {
6373 LLT Ty = MRI.getType(I.getOperand(0).getReg());
6374 unsigned Opc = 0;
6375 if (Ty == LLT::fixed_vector(8, S8))
6376 Opc = AArch64::LD4Rv8b;
6377 else if (Ty == LLT::fixed_vector(16, S8))
6378 Opc = AArch64::LD4Rv16b;
6379 else if (Ty == LLT::fixed_vector(4, S16))
6380 Opc = AArch64::LD4Rv4h;
6381 else if (Ty == LLT::fixed_vector(8, S16))
6382 Opc = AArch64::LD4Rv8h;
6383 else if (Ty == LLT::fixed_vector(2, S32))
6384 Opc = AArch64::LD4Rv2s;
6385 else if (Ty == LLT::fixed_vector(4, S32))
6386 Opc = AArch64::LD4Rv4s;
6387 else if (Ty == LLT::fixed_vector(2, S64) || Ty == LLT::fixed_vector(2, P0))
6388 Opc = AArch64::LD4Rv2d;
6389 else if (Ty == S64 || Ty == P0)
6390 Opc = AArch64::LD4Rv1d;
6391 else
6392 llvm_unreachable("Unexpected type for ld4r!");
6393 selectVectorLoadIntrinsic(Opc, 4, I);
6394 break;
6395 }
6396 case Intrinsic::aarch64_neon_st1x2: {
6397 LLT Ty = MRI.getType(I.getOperand(1).getReg());
6398 unsigned Opc;
6399 if (Ty == LLT::fixed_vector(8, S8))
6400 Opc = AArch64::ST1Twov8b;
6401 else if (Ty == LLT::fixed_vector(16, S8))
6402 Opc = AArch64::ST1Twov16b;
6403 else if (Ty == LLT::fixed_vector(4, S16))
6404 Opc = AArch64::ST1Twov4h;
6405 else if (Ty == LLT::fixed_vector(8, S16))
6406 Opc = AArch64::ST1Twov8h;
6407 else if (Ty == LLT::fixed_vector(2, S32))
6408 Opc = AArch64::ST1Twov2s;
6409 else if (Ty == LLT::fixed_vector(4, S32))
6410 Opc = AArch64::ST1Twov4s;
6411 else if (Ty == LLT::fixed_vector(2, S64) || Ty == LLT::fixed_vector(2, P0))
6412 Opc = AArch64::ST1Twov2d;
6413 else if (Ty == S64 || Ty == P0)
6414 Opc = AArch64::ST1Twov1d;
6415 else
6416 llvm_unreachable("Unexpected type for st1x2!");
6417 selectVectorStoreIntrinsic(I, 2, Opc);
6418 break;
6419 }
6420 case Intrinsic::aarch64_neon_st1x3: {
6421 LLT Ty = MRI.getType(I.getOperand(1).getReg());
6422 unsigned Opc;
6423 if (Ty == LLT::fixed_vector(8, S8))
6424 Opc = AArch64::ST1Threev8b;
6425 else if (Ty == LLT::fixed_vector(16, S8))
6426 Opc = AArch64::ST1Threev16b;
6427 else if (Ty == LLT::fixed_vector(4, S16))
6428 Opc = AArch64::ST1Threev4h;
6429 else if (Ty == LLT::fixed_vector(8, S16))
6430 Opc = AArch64::ST1Threev8h;
6431 else if (Ty == LLT::fixed_vector(2, S32))
6432 Opc = AArch64::ST1Threev2s;
6433 else if (Ty == LLT::fixed_vector(4, S32))
6434 Opc = AArch64::ST1Threev4s;
6435 else if (Ty == LLT::fixed_vector(2, S64) || Ty == LLT::fixed_vector(2, P0))
6436 Opc = AArch64::ST1Threev2d;
6437 else if (Ty == S64 || Ty == P0)
6438 Opc = AArch64::ST1Threev1d;
6439 else
6440 llvm_unreachable("Unexpected type for st1x3!");
6441 selectVectorStoreIntrinsic(I, 3, Opc);
6442 break;
6443 }
6444 case Intrinsic::aarch64_neon_st1x4: {
6445 LLT Ty = MRI.getType(I.getOperand(1).getReg());
6446 unsigned Opc;
6447 if (Ty == LLT::fixed_vector(8, S8))
6448 Opc = AArch64::ST1Fourv8b;
6449 else if (Ty == LLT::fixed_vector(16, S8))
6450 Opc = AArch64::ST1Fourv16b;
6451 else if (Ty == LLT::fixed_vector(4, S16))
6452 Opc = AArch64::ST1Fourv4h;
6453 else if (Ty == LLT::fixed_vector(8, S16))
6454 Opc = AArch64::ST1Fourv8h;
6455 else if (Ty == LLT::fixed_vector(2, S32))
6456 Opc = AArch64::ST1Fourv2s;
6457 else if (Ty == LLT::fixed_vector(4, S32))
6458 Opc = AArch64::ST1Fourv4s;
6459 else if (Ty == LLT::fixed_vector(2, S64) || Ty == LLT::fixed_vector(2, P0))
6460 Opc = AArch64::ST1Fourv2d;
6461 else if (Ty == S64 || Ty == P0)
6462 Opc = AArch64::ST1Fourv1d;
6463 else
6464 llvm_unreachable("Unexpected type for st1x4!");
6465 selectVectorStoreIntrinsic(I, 4, Opc);
6466 break;
6467 }
6468 case Intrinsic::aarch64_neon_st2: {
6469 LLT Ty = MRI.getType(I.getOperand(1).getReg());
6470 unsigned Opc;
6471 if (Ty == LLT::fixed_vector(8, S8))
6472 Opc = AArch64::ST2Twov8b;
6473 else if (Ty == LLT::fixed_vector(16, S8))
6474 Opc = AArch64::ST2Twov16b;
6475 else if (Ty == LLT::fixed_vector(4, S16))
6476 Opc = AArch64::ST2Twov4h;
6477 else if (Ty == LLT::fixed_vector(8, S16))
6478 Opc = AArch64::ST2Twov8h;
6479 else if (Ty == LLT::fixed_vector(2, S32))
6480 Opc = AArch64::ST2Twov2s;
6481 else if (Ty == LLT::fixed_vector(4, S32))
6482 Opc = AArch64::ST2Twov4s;
6483 else if (Ty == LLT::fixed_vector(2, S64) || Ty == LLT::fixed_vector(2, P0))
6484 Opc = AArch64::ST2Twov2d;
6485 else if (Ty == S64 || Ty == P0)
6486 Opc = AArch64::ST1Twov1d;
6487 else
6488 llvm_unreachable("Unexpected type for st2!");
6489 selectVectorStoreIntrinsic(I, 2, Opc);
6490 break;
6491 }
6492 case Intrinsic::aarch64_neon_st3: {
6493 LLT Ty = MRI.getType(I.getOperand(1).getReg());
6494 unsigned Opc;
6495 if (Ty == LLT::fixed_vector(8, S8))
6496 Opc = AArch64::ST3Threev8b;
6497 else if (Ty == LLT::fixed_vector(16, S8))
6498 Opc = AArch64::ST3Threev16b;
6499 else if (Ty == LLT::fixed_vector(4, S16))
6500 Opc = AArch64::ST3Threev4h;
6501 else if (Ty == LLT::fixed_vector(8, S16))
6502 Opc = AArch64::ST3Threev8h;
6503 else if (Ty == LLT::fixed_vector(2, S32))
6504 Opc = AArch64::ST3Threev2s;
6505 else if (Ty == LLT::fixed_vector(4, S32))
6506 Opc = AArch64::ST3Threev4s;
6507 else if (Ty == LLT::fixed_vector(2, S64) || Ty == LLT::fixed_vector(2, P0))
6508 Opc = AArch64::ST3Threev2d;
6509 else if (Ty == S64 || Ty == P0)
6510 Opc = AArch64::ST1Threev1d;
6511 else
6512 llvm_unreachable("Unexpected type for st3!");
6513 selectVectorStoreIntrinsic(I, 3, Opc);
6514 break;
6515 }
6516 case Intrinsic::aarch64_neon_st4: {
6517 LLT Ty = MRI.getType(I.getOperand(1).getReg());
6518 unsigned Opc;
6519 if (Ty == LLT::fixed_vector(8, S8))
6520 Opc = AArch64::ST4Fourv8b;
6521 else if (Ty == LLT::fixed_vector(16, S8))
6522 Opc = AArch64::ST4Fourv16b;
6523 else if (Ty == LLT::fixed_vector(4, S16))
6524 Opc = AArch64::ST4Fourv4h;
6525 else if (Ty == LLT::fixed_vector(8, S16))
6526 Opc = AArch64::ST4Fourv8h;
6527 else if (Ty == LLT::fixed_vector(2, S32))
6528 Opc = AArch64::ST4Fourv2s;
6529 else if (Ty == LLT::fixed_vector(4, S32))
6530 Opc = AArch64::ST4Fourv4s;
6531 else if (Ty == LLT::fixed_vector(2, S64) || Ty == LLT::fixed_vector(2, P0))
6532 Opc = AArch64::ST4Fourv2d;
6533 else if (Ty == S64 || Ty == P0)
6534 Opc = AArch64::ST1Fourv1d;
6535 else
6536 llvm_unreachable("Unexpected type for st4!");
6537 selectVectorStoreIntrinsic(I, 4, Opc);
6538 break;
6539 }
6540 case Intrinsic::aarch64_neon_st2lane: {
6541 LLT Ty = MRI.getType(I.getOperand(1).getReg());
6542 unsigned Opc;
6543 if (Ty == LLT::fixed_vector(8, S8) || Ty == LLT::fixed_vector(16, S8))
6544 Opc = AArch64::ST2i8;
6545 else if (Ty == LLT::fixed_vector(4, S16) || Ty == LLT::fixed_vector(8, S16))
6546 Opc = AArch64::ST2i16;
6547 else if (Ty == LLT::fixed_vector(2, S32) || Ty == LLT::fixed_vector(4, S32))
6548 Opc = AArch64::ST2i32;
6549 else if (Ty == LLT::fixed_vector(2, S64) ||
6550 Ty == LLT::fixed_vector(2, P0) || Ty == S64 || Ty == P0)
6551 Opc = AArch64::ST2i64;
6552 else
6553 llvm_unreachable("Unexpected type for st2lane!");
6554 if (!selectVectorStoreLaneIntrinsic(I, 2, Opc))
6555 return false;
6556 break;
6557 }
6558 case Intrinsic::aarch64_neon_st3lane: {
6559 LLT Ty = MRI.getType(I.getOperand(1).getReg());
6560 unsigned Opc;
6561 if (Ty == LLT::fixed_vector(8, S8) || Ty == LLT::fixed_vector(16, S8))
6562 Opc = AArch64::ST3i8;
6563 else if (Ty == LLT::fixed_vector(4, S16) || Ty == LLT::fixed_vector(8, S16))
6564 Opc = AArch64::ST3i16;
6565 else if (Ty == LLT::fixed_vector(2, S32) || Ty == LLT::fixed_vector(4, S32))
6566 Opc = AArch64::ST3i32;
6567 else if (Ty == LLT::fixed_vector(2, S64) ||
6568 Ty == LLT::fixed_vector(2, P0) || Ty == S64 || Ty == P0)
6569 Opc = AArch64::ST3i64;
6570 else
6571 llvm_unreachable("Unexpected type for st3lane!");
6572 if (!selectVectorStoreLaneIntrinsic(I, 3, Opc))
6573 return false;
6574 break;
6575 }
6576 case Intrinsic::aarch64_neon_st4lane: {
6577 LLT Ty = MRI.getType(I.getOperand(1).getReg());
6578 unsigned Opc;
6579 if (Ty == LLT::fixed_vector(8, S8) || Ty == LLT::fixed_vector(16, S8))
6580 Opc = AArch64::ST4i8;
6581 else if (Ty == LLT::fixed_vector(4, S16) || Ty == LLT::fixed_vector(8, S16))
6582 Opc = AArch64::ST4i16;
6583 else if (Ty == LLT::fixed_vector(2, S32) || Ty == LLT::fixed_vector(4, S32))
6584 Opc = AArch64::ST4i32;
6585 else if (Ty == LLT::fixed_vector(2, S64) ||
6586 Ty == LLT::fixed_vector(2, P0) || Ty == S64 || Ty == P0)
6587 Opc = AArch64::ST4i64;
6588 else
6589 llvm_unreachable("Unexpected type for st4lane!");
6590 if (!selectVectorStoreLaneIntrinsic(I, 4, Opc))
6591 return false;
6592 break;
6593 }
6594 case Intrinsic::aarch64_mops_memset_tag: {
6595 // Transform
6596 // %dst:gpr(p0) = \
6597 // G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.aarch64.mops.memset.tag),
6598 // \ %dst:gpr(p0), %val:gpr(s64), %n:gpr(s64)
6599 // where %dst is updated, into
6600 // %Rd:GPR64common, %Rn:GPR64) = \
6601 // MOPSMemorySetTaggingPseudo \
6602 // %Rd:GPR64common, %Rn:GPR64, %Rm:GPR64
6603 // where Rd and Rn are tied.
6604 // It is expected that %val has been extended to s64 in legalization.
6605 // Note that the order of the size/value operands are swapped.
6606
6607 Register DstDef = I.getOperand(0).getReg();
6608 // I.getOperand(1) is the intrinsic function
6609 Register DstUse = I.getOperand(2).getReg();
6610 Register ValUse = I.getOperand(3).getReg();
6611 Register SizeUse = I.getOperand(4).getReg();
6612
6613 // MOPSMemorySetTaggingPseudo has two defs; the intrinsic call has only one.
6614 // Therefore an additional virtual register is required for the updated size
6615 // operand. This value is not accessible via the semantics of the intrinsic.
6617
6618 auto Memset = MIB.buildInstr(AArch64::MOPSMemorySetTaggingPseudo,
6619 {DstDef, SizeDef}, {DstUse, SizeUse, ValUse});
6620 Memset.cloneMemRefs(I);
6621 Memset.setOperandDead(5); // implicit-def $nzcv
6623 break;
6624 }
6625 case Intrinsic::ptrauth_resign_load_relative: {
6626 Register DstReg = I.getOperand(0).getReg();
6627 Register ValReg = I.getOperand(2).getReg();
6628 uint64_t AUTKey = I.getOperand(3).getImm();
6629 Register AUTDisc = I.getOperand(4).getReg();
6630 uint64_t PACKey = I.getOperand(5).getImm();
6631 Register PACDisc = I.getOperand(6).getReg();
6632 int64_t Addend = I.getOperand(7).getImm();
6633
6634 Register AUTAddrDisc = AUTDisc;
6635 uint16_t AUTConstDiscC = 0;
6636 std::tie(AUTConstDiscC, AUTAddrDisc) =
6638
6639 Register PACAddrDisc = PACDisc;
6640 uint16_t PACConstDiscC = 0;
6641 std::tie(PACConstDiscC, PACAddrDisc) =
6643
6644 MIB.buildCopy({AArch64::X16}, {ValReg});
6645
6646 MIB.buildInstr(AArch64::AUTRELLOADPAC)
6647 .addImm(AUTKey)
6648 .addImm(AUTConstDiscC)
6649 .addUse(AUTAddrDisc)
6650 .addImm(PACKey)
6651 .addImm(PACConstDiscC)
6652 .addUse(PACAddrDisc)
6653 .addImm(Addend)
6654 .setOperandDead(8) // implicit-def $x17
6655 .setOperandDead(9) // implicit-def $nzcv
6656 .constrainAllUses(TII, TRI, RBI);
6657 MIB.buildCopy({DstReg}, Register(AArch64::X16));
6658
6659 RBI.constrainGenericRegister(DstReg, AArch64::GPR64RegClass, MRI);
6660 I.eraseFromParent();
6661 return true;
6662 }
6663 }
6664
6665 I.eraseFromParent();
6666 return true;
6667}
6668
6669bool AArch64InstructionSelector::selectIntrinsic(MachineInstr &I,
6670 MachineRegisterInfo &MRI) {
6671 unsigned IntrinID = cast<GIntrinsic>(I).getIntrinsicID();
6672
6673 switch (IntrinID) {
6674 default:
6675 break;
6676 case Intrinsic::ptrauth_resign: {
6677 Register DstReg = I.getOperand(0).getReg();
6678 Register ValReg = I.getOperand(2).getReg();
6679 uint64_t AUTKey = I.getOperand(3).getImm();
6680 Register AUTDisc = I.getOperand(4).getReg();
6681 uint64_t PACKey = I.getOperand(5).getImm();
6682 Register PACDisc = I.getOperand(6).getReg();
6683
6684 Register AUTAddrDisc = AUTDisc;
6685 uint16_t AUTConstDiscC = 0;
6686 std::tie(AUTConstDiscC, AUTAddrDisc) =
6688
6689 Register PACAddrDisc = PACDisc;
6690 uint16_t PACConstDiscC = 0;
6691 std::tie(PACConstDiscC, PACAddrDisc) =
6693
6694 MIB.buildCopy({AArch64::X16}, {ValReg});
6695 MIB.buildInstr(AArch64::AUTPAC)
6696 .addImm(AUTKey)
6697 .addImm(AUTConstDiscC)
6698 .addUse(AUTAddrDisc)
6699 .addImm(PACKey)
6700 .addImm(PACConstDiscC)
6701 .addUse(PACAddrDisc)
6702 .setOperandDead(7) // implicit-def $x17
6703 .setOperandDead(8) // implicit-def $nzcv
6704 .constrainAllUses(TII, TRI, RBI);
6705 MIB.buildCopy({DstReg}, Register(AArch64::X16));
6706
6707 RBI.constrainGenericRegister(DstReg, AArch64::GPR64RegClass, MRI);
6708 I.eraseFromParent();
6709 return true;
6710 }
6711 case Intrinsic::ptrauth_auth_with_pc_and_resign: {
6712 Register DstReg = I.getOperand(0).getReg();
6713 Register ValReg = I.getOperand(2).getReg();
6714 uint64_t AUTKey = I.getOperand(3).getImm();
6715 Register AUTDisc = I.getOperand(4).getReg();
6716 Register AUTPC = I.getOperand(5).getReg();
6717 uint64_t PACKey = I.getOperand(6).getImm();
6718 Register PACDisc = I.getOperand(7).getReg();
6719
6720 assert((AUTKey == AArch64PACKey::IA || AUTKey == AArch64PACKey::IB) &&
6721 "auth_with_pc_and_resign only supports IA and IB keys");
6722
6723 uint16_t PACConstDiscC = 0;
6724 Register PACAddrDisc;
6725 std::tie(PACConstDiscC, PACAddrDisc) =
6727
6728 if (!PACAddrDisc.isValid())
6729 PACAddrDisc = AArch64::XZR;
6730
6731 MIB.buildCopy({AArch64::X17}, {ValReg});
6732 MIB.buildCopy({AArch64::X16}, {AUTDisc});
6733 MIB.buildCopy({AArch64::X15}, {AUTPC});
6734
6735 MIB.buildInstr(AArch64::AUTPCPAC)
6736 .addImm(AUTKey)
6737 .addImm(PACKey)
6738 .addImm(PACConstDiscC)
6739 .addUse(PACAddrDisc)
6740 .setOperandDead(5) // implicit-def $x15
6741 .setOperandDead(6) // implicit-def $x16
6742 .setOperandDead(7) // implicit-def $nzcv
6743 .constrainAllUses(TII, TRI, RBI);
6744
6745 MIB.buildCopy({DstReg}, Register(AArch64::X17));
6746 RBI.constrainGenericRegister(DstReg, AArch64::GPR64RegClass, MRI);
6747 I.eraseFromParent();
6748 return true;
6749 }
6750 case Intrinsic::ptrauth_auth: {
6751 Register DstReg = I.getOperand(0).getReg();
6752 Register ValReg = I.getOperand(2).getReg();
6753 uint64_t AUTKey = I.getOperand(3).getImm();
6754 Register AUTDisc = I.getOperand(4).getReg();
6755
6756 Register AUTAddrDisc = AUTDisc;
6757 uint16_t AUTConstDiscC = 0;
6758 std::tie(AUTConstDiscC, AUTAddrDisc) =
6760
6761 if (STI.isX16X17Safer()) {
6762 MIB.buildCopy({AArch64::X16}, {ValReg});
6763 MIB.buildInstr(AArch64::AUTx16x17)
6764 .addImm(AUTKey)
6765 .addImm(AUTConstDiscC)
6766 .addUse(AUTAddrDisc)
6767 .setOperandDead(4) // implicit-def $x17
6768 .setOperandDead(5) // implicit-def $nzcv
6769 .constrainAllUses(TII, TRI, RBI);
6770 MIB.buildCopy({DstReg}, Register(AArch64::X16));
6771 } else {
6772 Register ScratchReg =
6773 MRI.createVirtualRegister(&AArch64::GPR64commonRegClass);
6774 auto Auth = MIB.buildInstr(AArch64::AUTxMxN)
6775 .addDef(DstReg)
6776 .addDef(ScratchReg)
6777 .addUse(ValReg)
6778 .addImm(AUTKey)
6779 .addImm(AUTConstDiscC)
6780 .addUse(AUTAddrDisc);
6782 Auth.constrainAllUses(TII, TRI, RBI);
6783 }
6784
6785 RBI.constrainGenericRegister(DstReg, AArch64::GPR64RegClass, MRI);
6786 I.eraseFromParent();
6787 return true;
6788 }
6789 case Intrinsic::frameaddress:
6790 case Intrinsic::returnaddress: {
6791 MachineFunction &MF = *I.getParent()->getParent();
6792 MachineFrameInfo &MFI = MF.getFrameInfo();
6793
6794 unsigned Depth = I.getOperand(2).getImm();
6795 Register DstReg = I.getOperand(0).getReg();
6796 RBI.constrainGenericRegister(DstReg, AArch64::GPR64RegClass, MRI);
6797
6798 if (Depth == 0 && IntrinID == Intrinsic::returnaddress) {
6799 if (!MFReturnAddr) {
6800 // Insert the copy from LR/X30 into the entry block, before it can be
6801 // clobbered by anything.
6802 MFI.setReturnAddressIsTaken(true);
6803 MFReturnAddr = getFunctionLiveInPhysReg(
6804 MF, TII, AArch64::LR, AArch64::GPR64RegClass, I.getDebugLoc());
6805 }
6806
6807 if (STI.hasPAuth()) {
6808 MIB.buildInstr(AArch64::XPACI, {DstReg}, {MFReturnAddr});
6809 } else {
6810 MIB.buildCopy({Register(AArch64::LR)}, {MFReturnAddr});
6811 MIB.buildInstr(AArch64::XPACLRI);
6812 MIB.buildCopy({DstReg}, {Register(AArch64::LR)});
6813 }
6814
6815 I.eraseFromParent();
6816 return true;
6817 }
6818
6819 MFI.setFrameAddressIsTaken(true);
6820 Register FrameAddr(AArch64::FP);
6821 while (Depth--) {
6822 Register NextFrame = MRI.createVirtualRegister(&AArch64::GPR64spRegClass);
6823 auto Ldr =
6824 MIB.buildInstr(AArch64::LDRXui, {NextFrame}, {FrameAddr}).addImm(0);
6826 FrameAddr = NextFrame;
6827 }
6828
6829 if (IntrinID == Intrinsic::frameaddress)
6830 MIB.buildCopy({DstReg}, {FrameAddr});
6831 else {
6832 MFI.setReturnAddressIsTaken(true);
6833
6834 if (STI.hasPAuth()) {
6835 Register TmpReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
6836 MIB.buildInstr(AArch64::LDRXui, {TmpReg}, {FrameAddr}).addImm(1);
6837 MIB.buildInstr(AArch64::XPACI, {DstReg}, {TmpReg});
6838 } else {
6839 MIB.buildInstr(AArch64::LDRXui, {Register(AArch64::LR)}, {FrameAddr})
6840 .addImm(1);
6841 MIB.buildInstr(AArch64::XPACLRI);
6842 MIB.buildCopy({DstReg}, {Register(AArch64::LR)});
6843 }
6844 }
6845
6846 I.eraseFromParent();
6847 return true;
6848 }
6849 case Intrinsic::aarch64_neon_tbl2:
6850 SelectTable(I, MRI, 2, AArch64::TBLv8i8Two, AArch64::TBLv16i8Two, false);
6851 return true;
6852 case Intrinsic::aarch64_neon_tbl3:
6853 SelectTable(I, MRI, 3, AArch64::TBLv8i8Three, AArch64::TBLv16i8Three,
6854 false);
6855 return true;
6856 case Intrinsic::aarch64_neon_tbl4:
6857 SelectTable(I, MRI, 4, AArch64::TBLv8i8Four, AArch64::TBLv16i8Four, false);
6858 return true;
6859 case Intrinsic::aarch64_neon_tbx2:
6860 SelectTable(I, MRI, 2, AArch64::TBXv8i8Two, AArch64::TBXv16i8Two, true);
6861 return true;
6862 case Intrinsic::aarch64_neon_tbx3:
6863 SelectTable(I, MRI, 3, AArch64::TBXv8i8Three, AArch64::TBXv16i8Three, true);
6864 return true;
6865 case Intrinsic::aarch64_neon_tbx4:
6866 SelectTable(I, MRI, 4, AArch64::TBXv8i8Four, AArch64::TBXv16i8Four, true);
6867 return true;
6868 case Intrinsic::swift_async_context_addr:
6869 auto Sub = MIB.buildInstr(AArch64::SUBXri, {I.getOperand(0).getReg()},
6870 {Register(AArch64::FP)})
6871 .addImm(8)
6872 .addImm(0);
6874
6876 MF->getInfo<AArch64FunctionInfo>()->setHasSwiftAsyncContext(true);
6877 I.eraseFromParent();
6878 return true;
6879 }
6880 return false;
6881}
6882
6883// G_PTRAUTH_GLOBAL_VALUE lowering
6884//
6885// We have 3 lowering alternatives to choose from:
6886// - MOVaddrPAC: similar to MOVaddr, with added PAC.
6887// If the GV doesn't need a GOT load (i.e., is locally defined)
6888// materialize the pointer using adrp+add+pac. See LowerMOVaddrPAC.
6889//
6890// - LOADgotPAC: similar to LOADgot, with added PAC.
6891// If the GV needs a GOT load, materialize the pointer using the usual
6892// GOT adrp+ldr, +pac. Pointers in GOT are assumed to be not signed, the GOT
6893// section is assumed to be read-only (for example, via relro mechanism). See
6894// LowerMOVaddrPAC.
6895//
6896// - LOADauthptrstatic: similar to LOADgot, but use a
6897// special stub slot instead of a GOT slot.
6898// Load a signed pointer for symbol 'sym' from a stub slot named
6899// 'sym$auth_ptr$key$disc' filled by dynamic linker during relocation
6900// resolving. This usually lowers to adrp+ldr, but also emits an entry into
6901// .data with an
6902// @AUTH relocation. See LowerLOADauthptrstatic.
6903//
6904// All 3 are pseudos that are expand late to longer sequences: this lets us
6905// provide integrity guarantees on the to-be-signed intermediate values.
6906//
6907// LOADauthptrstatic is undesirable because it requires a large section filled
6908// with often similarly-signed pointers, making it a good harvesting target.
6909// Thus, it's only used for ptrauth references to extern_weak to avoid null
6910// checks.
6911
6912bool AArch64InstructionSelector::selectPtrAuthGlobalValue(
6913 MachineInstr &I, MachineRegisterInfo &MRI) const {
6914 Register DefReg = I.getOperand(0).getReg();
6915 Register Addr = I.getOperand(1).getReg();
6916 uint64_t Key = I.getOperand(2).getImm();
6917 Register AddrDisc = I.getOperand(3).getReg();
6918 uint64_t Disc = I.getOperand(4).getImm();
6919 int64_t Offset = 0;
6920
6922 report_fatal_error("key in ptrauth global out of range [0, " +
6923 Twine((int)AArch64PACKey::LAST) + "]");
6924
6925 // Blend only works if the integer discriminator is 16-bit wide.
6926 if (!isUInt<16>(Disc))
6928 "constant discriminator in ptrauth global out of range [0, 0xffff]");
6929
6930 // Choosing between 3 lowering alternatives is target-specific.
6931 if (!STI.isTargetELF() && !STI.isTargetMachO())
6932 report_fatal_error("ptrauth global lowering only supported on MachO/ELF");
6933
6934 if (!MRI.hasOneDef(Addr))
6935 return false;
6936
6937 // First match any offset we take from the real global.
6938 const MachineInstr *DefMI = &*MRI.def_instr_begin(Addr);
6939 if (DefMI->getOpcode() == TargetOpcode::G_PTR_ADD) {
6940 Register OffsetReg = DefMI->getOperand(2).getReg();
6941 if (!MRI.hasOneDef(OffsetReg))
6942 return false;
6943 const MachineInstr &OffsetMI = *MRI.def_instr_begin(OffsetReg);
6944 if (OffsetMI.getOpcode() != TargetOpcode::G_CONSTANT)
6945 return false;
6946
6947 Addr = DefMI->getOperand(1).getReg();
6948 if (!MRI.hasOneDef(Addr))
6949 return false;
6950
6951 DefMI = &*MRI.def_instr_begin(Addr);
6952 Offset = OffsetMI.getOperand(1).getCImm()->getSExtValue();
6953 }
6954
6955 // We should be left with a genuine unauthenticated GlobalValue.
6956 const GlobalValue *GV;
6957 if (DefMI->getOpcode() == TargetOpcode::G_GLOBAL_VALUE) {
6958 GV = DefMI->getOperand(1).getGlobal();
6960 } else if (DefMI->getOpcode() == AArch64::G_ADD_LOW) {
6961 GV = DefMI->getOperand(2).getGlobal();
6963 } else {
6964 return false;
6965 }
6966
6967 MachineIRBuilder MIB(I);
6968
6969 // Classify the reference to determine whether it needs a GOT load.
6970 unsigned OpFlags = STI.ClassifyGlobalReference(GV, TM);
6971 const bool NeedsGOTLoad = ((OpFlags & AArch64II::MO_GOT) != 0);
6972 assert(((OpFlags & (~AArch64II::MO_GOT)) == 0) &&
6973 "unsupported non-GOT op flags on ptrauth global reference");
6974 assert((!GV->hasExternalWeakLinkage() || NeedsGOTLoad) &&
6975 "unsupported non-GOT reference to weak ptrauth global");
6976
6977 std::optional<APInt> AddrDiscVal = getIConstantVRegVal(AddrDisc, MRI);
6978 bool HasAddrDisc = !AddrDiscVal || *AddrDiscVal != 0;
6979
6980 // Non-extern_weak:
6981 // - No GOT load needed -> MOVaddrPAC
6982 // - GOT load for non-extern_weak -> LOADgotPAC
6983 // Note that we disallow extern_weak refs to avoid null checks later.
6984 if (!GV->hasExternalWeakLinkage()) {
6985 MIB.buildInstr(NeedsGOTLoad ? AArch64::LOADgotPAC : AArch64::MOVaddrPAC)
6987 .addImm(Key)
6988 .addReg(HasAddrDisc ? AddrDisc : AArch64::XZR)
6989 .addImm(Disc)
6990 .constrainAllUses(TII, TRI, RBI);
6991 MIB.buildCopy(DefReg, Register(AArch64::X16));
6992 RBI.constrainGenericRegister(DefReg, AArch64::GPR64RegClass, MRI);
6993 I.eraseFromParent();
6994 return true;
6995 }
6996
6997 // extern_weak -> LOADauthptrstatic
6998
6999 // Offsets and extern_weak don't mix well: ptrauth aside, you'd get the
7000 // offset alone as a pointer if the symbol wasn't available, which would
7001 // probably break null checks in users. Ptrauth complicates things further:
7002 // error out.
7003 if (Offset != 0)
7005 "unsupported non-zero offset in weak ptrauth global reference");
7006
7007 if (HasAddrDisc)
7008 report_fatal_error("unsupported weak addr-div ptrauth global");
7009
7010 MIB.buildInstr(AArch64::LOADauthptrstatic, {DefReg}, {})
7011 .addGlobalAddress(GV, Offset)
7012 .addImm(Key)
7013 .addImm(Disc);
7014 RBI.constrainGenericRegister(DefReg, AArch64::GPR64RegClass, MRI);
7015
7016 I.eraseFromParent();
7017 return true;
7018}
7019
7020void AArch64InstructionSelector::SelectTable(MachineInstr &I,
7021 MachineRegisterInfo &MRI,
7022 unsigned NumVec, unsigned Opc1,
7023 unsigned Opc2, bool isExt) {
7024 Register DstReg = I.getOperand(0).getReg();
7025 unsigned Opc = MRI.getType(DstReg) == LLT::fixed_vector(8, 8) ? Opc1 : Opc2;
7026
7027 // Create the REG_SEQUENCE
7029 for (unsigned i = 0; i < NumVec; i++)
7030 Regs.push_back(I.getOperand(i + 2 + isExt).getReg());
7031 Register RegSeq = createQTuple(Regs, MIB);
7032
7033 Register IdxReg = I.getOperand(2 + NumVec + isExt).getReg();
7034 MachineInstrBuilder Instr;
7035 if (isExt) {
7036 Register Reg = I.getOperand(2).getReg();
7037 Instr = MIB.buildInstr(Opc, {DstReg}, {Reg, RegSeq, IdxReg});
7038 } else
7039 Instr = MIB.buildInstr(Opc, {DstReg}, {RegSeq, IdxReg});
7041 I.eraseFromParent();
7042}
7043
7044InstructionSelector::ComplexRendererFns
7045AArch64InstructionSelector::selectShiftA_32(const MachineOperand &Root) const {
7046 auto MaybeImmed = getImmedFromMO(Root);
7047 if (MaybeImmed == std::nullopt || *MaybeImmed > 31)
7048 return std::nullopt;
7049 uint64_t Enc = (32 - *MaybeImmed) & 0x1f;
7050 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}};
7051}
7052
7053InstructionSelector::ComplexRendererFns
7054AArch64InstructionSelector::selectShiftB_32(const MachineOperand &Root) const {
7055 auto MaybeImmed = getImmedFromMO(Root);
7056 if (MaybeImmed == std::nullopt || *MaybeImmed > 31)
7057 return std::nullopt;
7058 uint64_t Enc = 31 - *MaybeImmed;
7059 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}};
7060}
7061
7062InstructionSelector::ComplexRendererFns
7063AArch64InstructionSelector::selectShiftA_64(const MachineOperand &Root) const {
7064 auto MaybeImmed = getImmedFromMO(Root);
7065 if (MaybeImmed == std::nullopt || *MaybeImmed > 63)
7066 return std::nullopt;
7067 uint64_t Enc = (64 - *MaybeImmed) & 0x3f;
7068 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}};
7069}
7070
7071InstructionSelector::ComplexRendererFns
7072AArch64InstructionSelector::selectShiftB_64(const MachineOperand &Root) const {
7073 auto MaybeImmed = getImmedFromMO(Root);
7074 if (MaybeImmed == std::nullopt || *MaybeImmed > 63)
7075 return std::nullopt;
7076 uint64_t Enc = 63 - *MaybeImmed;
7077 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}};
7078}
7079
7080template <unsigned ShiftWidth>
7081InstructionSelector::ComplexRendererFns
7082AArch64InstructionSelector::selectShiftMask(MachineOperand &Root) const {
7083 if (!Root.isReg())
7084 return std::nullopt;
7085
7086 MachineRegisterInfo &MRI =
7087 Root.getParent()->getParent()->getParent()->getRegInfo();
7088
7089 Register ShAmtReg = Root.getReg();
7090
7091 // Peek through zext for i32 shifts only. For i64 shifts the zext case
7092 // is already handled by existing patterns in the Shift multiclass.
7093 if (ShiftWidth == 32) {
7094 Register ZExtSrcReg;
7095 if (mi_match(ShAmtReg, MRI, m_GZExt(m_Reg(ZExtSrcReg))))
7096 ShAmtReg = ZExtSrcReg;
7097 }
7098
7099 // Remove AND if the mask covers at least the low log2(ShiftWidth) bits.
7100 APInt AndMask;
7101 Register AndSrcReg;
7102 if (mi_match(ShAmtReg, MRI, m_GAnd(m_Reg(AndSrcReg), m_ICst(AndMask))) &&
7103 MRI.getType(ShAmtReg).getSizeInBits() == ShiftWidth) {
7104 if (AndMask.countr_one() >= Log2_32(ShiftWidth))
7105 ShAmtReg = AndSrcReg;
7106 }
7107
7108 // If shifting by X+/-N where N == 0 mod ShiftWidth, then just shift by X
7109 // to avoid the ADD/SUB. The low log2(ShiftWidth) bits are unchanged, so the
7110 // shift can use X directly; the original ADD/SUB stays for any other users.
7111 Register AddSrcReg;
7112 int64_t AddImm;
7113 if ((mi_match(ShAmtReg, MRI,
7114 m_GAdd(m_Reg(AddSrcReg), m_ICstOrSplat(AddImm))) ||
7115 mi_match(ShAmtReg, MRI,
7116 m_GSub(m_Reg(AddSrcReg), m_ICstOrSplat(AddImm)))) &&
7117 (AddImm % ShiftWidth == 0)) {
7118 ShAmtReg = AddSrcReg;
7119 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(ShAmtReg); }}};
7120 }
7121
7122 // If shifting by N-X where N == 0 mod ShiftWidth, then just shift by -X
7123 // to generate a NEG instead of a SUB from a constant.
7124 Register SubSrcReg;
7125 int64_t SubImm;
7126 if (MRI.hasOneNonDBGUse(ShAmtReg) &&
7127 mi_match(ShAmtReg, MRI, m_GSub(m_ICst(SubImm), m_Reg(SubSrcReg))) &&
7128 SubImm != 0 && (SubImm % ShiftWidth == 0)) {
7129 return {{[=](MachineInstrBuilder &MIB) {
7130 MachineInstr *I = MIB.getInstr();
7131 MachineRegisterInfo &MRI2 = I->getMF()->getRegInfo();
7132 const TargetRegisterClass &RC =
7133 ShiftWidth == 32 ? AArch64::GPR32RegClass : AArch64::GPR64RegClass;
7134 unsigned SubOpc = ShiftWidth == 32 ? AArch64::SUBWrr : AArch64::SUBXrr;
7135 Register ZeroReg = ShiftWidth == 32 ? AArch64::WZR : AArch64::XZR;
7136 Register NegReg = MRI2.createVirtualRegister(&RC);
7137 auto NegMI = BuildMI(*I->getParent(), *I, I->getDebugLoc(),
7138 TII.get(SubOpc), NegReg)
7139 .addReg(ZeroReg)
7140 .addReg(SubSrcReg);
7142 MIB.addReg(NegReg);
7143 }}};
7144 }
7145
7146 // If shifting by N-X where N == -1 mod ShiftWidth, then just shift by ~X
7147 // to generate a NOT (MVN) instead of a SUB from a constant.
7148 Register NotSrcReg;
7149 int64_t NotImm;
7150 if (MRI.hasOneNonDBGUse(ShAmtReg) &&
7151 mi_match(ShAmtReg, MRI, m_GSub(m_ICst(NotImm), m_Reg(NotSrcReg))) &&
7152 (NotImm % ShiftWidth == ShiftWidth - 1)) {
7153 return {{[=](MachineInstrBuilder &MIB) {
7154 MachineInstr *I = MIB.getInstr();
7155 MachineRegisterInfo &MRI2 = I->getMF()->getRegInfo();
7156 const TargetRegisterClass &RC =
7157 ShiftWidth == 32 ? AArch64::GPR32RegClass : AArch64::GPR64RegClass;
7158 unsigned NotOpc = ShiftWidth == 32 ? AArch64::ORNWrr : AArch64::ORNXrr;
7159 Register ZeroReg = ShiftWidth == 32 ? AArch64::WZR : AArch64::XZR;
7160 Register NotReg = MRI2.createVirtualRegister(&RC);
7161 auto NotMI = BuildMI(*I->getParent(), *I, I->getDebugLoc(),
7162 TII.get(NotOpc), NotReg)
7163 .addReg(ZeroReg)
7164 .addReg(NotSrcReg);
7166 MIB.addReg(NotReg);
7167 }}};
7168 }
7169
7170 // Only succeed if we changed the shift amount; otherwise let other
7171 // patterns (e.g. zext GPR32 -> SUBREG_TO_REG) match instead.
7172 if (ShAmtReg == Root.getReg())
7173 return std::nullopt;
7174
7175 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(ShAmtReg); }}};
7176}
7177
7178/// Helper to select an immediate value that can be represented as a 12-bit
7179/// value shifted left by either 0 or 12. If it is possible to do so, return
7180/// the immediate and shift value. If not, return std::nullopt.
7181///
7182/// Used by selectArithImmed and selectNegArithImmed.
7183InstructionSelector::ComplexRendererFns
7184AArch64InstructionSelector::select12BitValueWithLeftShift(
7185 uint64_t Immed) const {
7186 unsigned ShiftAmt;
7187 if (Immed >> 12 == 0) {
7188 ShiftAmt = 0;
7189 } else if ((Immed & 0xfff) == 0 && Immed >> 24 == 0) {
7190 ShiftAmt = 12;
7191 Immed = Immed >> 12;
7192 } else
7193 return std::nullopt;
7194
7195 unsigned ShVal = AArch64_AM::getShifterImm(AArch64_AM::LSL, ShiftAmt);
7196 return {{
7197 [=](MachineInstrBuilder &MIB) { MIB.addImm(Immed); },
7198 [=](MachineInstrBuilder &MIB) { MIB.addImm(ShVal); },
7199 }};
7200}
7201
7202/// SelectArithImmed - Select an immediate value that can be represented as
7203/// a 12-bit value shifted left by either 0 or 12. If so, return true with
7204/// Val set to the 12-bit value and Shift set to the shifter operand.
7205InstructionSelector::ComplexRendererFns
7206AArch64InstructionSelector::selectArithImmed(MachineOperand &Root) const {
7207 // This function is called from the addsub_shifted_imm ComplexPattern,
7208 // which lists [imm] as the list of opcode it's interested in, however
7209 // we still need to check whether the operand is actually an immediate
7210 // here because the ComplexPattern opcode list is only used in
7211 // root-level opcode matching.
7212 auto MaybeImmed = getImmedFromMO(Root);
7213 if (MaybeImmed == std::nullopt)
7214 return std::nullopt;
7215 return select12BitValueWithLeftShift(*MaybeImmed);
7216}
7217
7218/// SelectNegArithImmed - As above, but negates the value before trying to
7219/// select it.
7220InstructionSelector::ComplexRendererFns
7221AArch64InstructionSelector::selectNegArithImmed(MachineOperand &Root) const {
7222 // We need a register here, because we need to know if we have a 64 or 32
7223 // bit immediate.
7224 if (!Root.isReg())
7225 return std::nullopt;
7226 auto MaybeImmed = getImmedFromMO(Root);
7227 if (MaybeImmed == std::nullopt)
7228 return std::nullopt;
7229 uint64_t Immed = *MaybeImmed;
7230
7231 // This negation is almost always valid, but "cmp wN, #0" and "cmn wN, #0"
7232 // have the opposite effect on the C flag, so this pattern mustn't match under
7233 // those circumstances.
7234 if (Immed == 0)
7235 return std::nullopt;
7236
7237 // Check if we're dealing with a 32-bit type on the root or a 64-bit type on
7238 // the root.
7239 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
7240 if (MRI.getType(Root.getReg()).getSizeInBits() == 32)
7241 Immed = ~((uint32_t)Immed) + 1;
7242 else
7243 Immed = ~Immed + 1ULL;
7244
7245 if (Immed & 0xFFFFFFFFFF000000ULL)
7246 return std::nullopt;
7247
7248 Immed &= 0xFFFFFFULL;
7249 return select12BitValueWithLeftShift(Immed);
7250}
7251
7252/// Checks if we are sure that folding MI into load/store addressing mode is
7253/// beneficial or not.
7254///
7255/// Returns:
7256/// - true if folding MI would be beneficial.
7257/// - false if folding MI would be bad.
7258/// - std::nullopt if it is not sure whether folding MI is beneficial.
7259///
7260/// \p MI can be the offset operand of G_PTR_ADD, e.g. G_SHL in the example:
7261///
7262/// %13:gpr(s64) = G_CONSTANT i64 1
7263/// %8:gpr(s64) = G_SHL %6, %13(s64)
7264/// %9:gpr(p0) = G_PTR_ADD %0, %8(s64)
7265/// %12:gpr(s32) = G_LOAD %9(p0) :: (load (s16))
7266std::optional<bool> AArch64InstructionSelector::isWorthFoldingIntoAddrMode(
7267 const MachineInstr &MI, const MachineRegisterInfo &MRI) const {
7268 if (MI.getOpcode() == AArch64::G_SHL) {
7269 // Address operands with shifts are free, except for running on subtargets
7270 // with AddrLSLSlow14.
7271 if (const auto ValAndVeg = getIConstantVRegValWithLookThrough(
7272 MI.getOperand(2).getReg(), MRI)) {
7273 const APInt ShiftVal = ValAndVeg->Value;
7274
7275 // Don't fold if we know this will be slow.
7276 return !(STI.hasAddrLSLSlow14() && (ShiftVal == 1 || ShiftVal == 4));
7277 }
7278 }
7279 return std::nullopt;
7280}
7281
7282/// Return true if it is worth folding MI into an extended register. That is,
7283/// if it's safe to pull it into the addressing mode of a load or store as a
7284/// shift.
7285/// \p IsAddrOperand whether the def of MI is used as an address operand
7286/// (e.g. feeding into an LDR/STR).
7287bool AArch64InstructionSelector::isWorthFoldingIntoExtendedReg(
7288 const MachineInstr &MI, const MachineRegisterInfo &MRI,
7289 bool IsAddrOperand) const {
7290
7291 // Always fold if there is one use, or if we're optimizing for size.
7292 Register DefReg = MI.getOperand(0).getReg();
7293 if (MRI.hasOneNonDBGUse(DefReg) ||
7294 MI.getParent()->getParent()->getFunction().hasOptSize())
7295 return true;
7296
7297 if (IsAddrOperand) {
7298 // If we are already sure that folding MI is good or bad, return the result.
7299 if (const auto Worth = isWorthFoldingIntoAddrMode(MI, MRI))
7300 return *Worth;
7301
7302 // Fold G_PTR_ADD if its offset operand can be folded
7303 if (MI.getOpcode() == AArch64::G_PTR_ADD) {
7304 MachineInstr *OffsetInst =
7305 getDefIgnoringCopies(MI.getOperand(2).getReg(), MRI);
7306
7307 // Note, we already know G_PTR_ADD is used by at least two instructions.
7308 // If we are also sure about whether folding is beneficial or not,
7309 // return the result.
7310 if (const auto Worth = isWorthFoldingIntoAddrMode(*OffsetInst, MRI))
7311 return *Worth;
7312 }
7313 }
7314
7315 // FIXME: Consider checking HasALULSLFast as appropriate.
7316
7317 // We have a fastpath, so folding a shift in and potentially computing it
7318 // many times may be beneficial. Check if this is only used in memory ops.
7319 // If it is, then we should fold.
7320 return all_of(MRI.use_nodbg_instructions(DefReg),
7321 [](MachineInstr &Use) { return Use.mayLoadOrStore(); });
7322}
7323
7324InstructionSelector::ComplexRendererFns
7325AArch64InstructionSelector::selectExtendedSHL(
7326 MachineOperand &Root, MachineOperand &Base, MachineOperand &Offset,
7327 unsigned SizeInBytes, bool WantsExt) const {
7328 assert(Base.isReg() && "Expected base to be a register operand");
7329 assert(Offset.isReg() && "Expected offset to be a register operand");
7330
7331 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
7332 MachineInstr *OffsetInst = MRI.getVRegDef(Offset.getReg());
7333
7334 unsigned OffsetOpc = OffsetInst->getOpcode();
7335 bool LookedThroughZExt = false;
7336 if (OffsetOpc != TargetOpcode::G_SHL && OffsetOpc != TargetOpcode::G_MUL) {
7337 // Try to look through a ZEXT.
7338 if (OffsetOpc != TargetOpcode::G_ZEXT || !WantsExt)
7339 return std::nullopt;
7340
7341 OffsetInst = MRI.getVRegDef(OffsetInst->getOperand(1).getReg());
7342 OffsetOpc = OffsetInst->getOpcode();
7343 LookedThroughZExt = true;
7344
7345 if (OffsetOpc != TargetOpcode::G_SHL && OffsetOpc != TargetOpcode::G_MUL)
7346 return std::nullopt;
7347 }
7348 // Make sure that the memory op is a valid size.
7349 int64_t LegalShiftVal = Log2_32(SizeInBytes);
7350 if (LegalShiftVal == 0)
7351 return std::nullopt;
7352 if (!isWorthFoldingIntoExtendedReg(*OffsetInst, MRI, true))
7353 return std::nullopt;
7354
7355 // Now, try to find the specific G_CONSTANT. Start by assuming that the
7356 // register we will offset is the LHS, and the register containing the
7357 // constant is the RHS.
7358 Register OffsetReg = OffsetInst->getOperand(1).getReg();
7359 Register ConstantReg = OffsetInst->getOperand(2).getReg();
7360 auto ValAndVReg = getIConstantVRegValWithLookThrough(ConstantReg, MRI);
7361 if (!ValAndVReg) {
7362 // We didn't get a constant on the RHS. If the opcode is a shift, then
7363 // we're done.
7364 if (OffsetOpc == TargetOpcode::G_SHL)
7365 return std::nullopt;
7366
7367 // If we have a G_MUL, we can use either register. Try looking at the RHS.
7368 std::swap(OffsetReg, ConstantReg);
7369 ValAndVReg = getIConstantVRegValWithLookThrough(ConstantReg, MRI);
7370 if (!ValAndVReg)
7371 return std::nullopt;
7372 }
7373
7374 // The value must fit into 3 bits, and must be positive. Make sure that is
7375 // true.
7376 int64_t ImmVal = ValAndVReg->Value.getSExtValue();
7377
7378 // Since we're going to pull this into a shift, the constant value must be
7379 // a power of 2. If we got a multiply, then we need to check this.
7380 if (OffsetOpc == TargetOpcode::G_MUL) {
7381 if (!llvm::has_single_bit<uint32_t>(ImmVal))
7382 return std::nullopt;
7383
7384 // Got a power of 2. So, the amount we'll shift is the log base-2 of that.
7385 ImmVal = Log2_32(ImmVal);
7386 }
7387
7388 if ((ImmVal & 0x7) != ImmVal)
7389 return std::nullopt;
7390
7391 // We are only allowed to shift by LegalShiftVal. This shift value is built
7392 // into the instruction, so we can't just use whatever we want.
7393 if (ImmVal != LegalShiftVal)
7394 return std::nullopt;
7395
7396 unsigned SignExtend = 0;
7397 if (WantsExt) {
7398 // Check if the offset is defined by an extend, unless we looked through a
7399 // G_ZEXT earlier.
7400 if (!LookedThroughZExt) {
7401 MachineInstr *ExtInst = getDefIgnoringCopies(OffsetReg, MRI);
7402 auto Ext = getExtendTypeForInst(*ExtInst, MRI, true);
7404 return std::nullopt;
7405
7406 SignExtend = AArch64_AM::isSignExtendShiftType(Ext) ? 1 : 0;
7407 // We only support SXTW for signed extension here.
7408 if (SignExtend && Ext != AArch64_AM::SXTW)
7409 return std::nullopt;
7410 OffsetReg = ExtInst->getOperand(1).getReg();
7411 }
7412
7413 // Need a 32-bit wide register here.
7414 MachineIRBuilder MIB(*MRI.getVRegDef(Root.getReg()));
7415 OffsetReg = moveScalarRegClass(OffsetReg, AArch64::GPR32RegClass, MIB);
7416 }
7417
7418 // We can use the LHS of the GEP as the base, and the LHS of the shift as an
7419 // offset. Signify that we are shifting by setting the shift flag to 1.
7420 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(Base.getReg()); },
7421 [=](MachineInstrBuilder &MIB) { MIB.addUse(OffsetReg); },
7422 [=](MachineInstrBuilder &MIB) {
7423 // Need to add both immediates here to make sure that they are both
7424 // added to the instruction.
7425 MIB.addImm(SignExtend);
7426 MIB.addImm(1);
7427 }}};
7428}
7429
7430/// This is used for computing addresses like this:
7431///
7432/// ldr x1, [x2, x3, lsl #3]
7433///
7434/// Where x2 is the base register, and x3 is an offset register. The shift-left
7435/// is a constant value specific to this load instruction. That is, we'll never
7436/// see anything other than a 3 here (which corresponds to the size of the
7437/// element being loaded.)
7438InstructionSelector::ComplexRendererFns
7439AArch64InstructionSelector::selectAddrModeShiftedExtendXReg(
7440 MachineOperand &Root, unsigned SizeInBytes) const {
7441 if (!Root.isReg())
7442 return std::nullopt;
7443 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
7444
7445 // We want to find something like this:
7446 //
7447 // val = G_CONSTANT LegalShiftVal
7448 // shift = G_SHL off_reg val
7449 // ptr = G_PTR_ADD base_reg shift
7450 // x = G_LOAD ptr
7451 //
7452 // And fold it into this addressing mode:
7453 //
7454 // ldr x, [base_reg, off_reg, lsl #LegalShiftVal]
7455
7456 // Check if we can find the G_PTR_ADD.
7457 MachineInstr *PtrAdd =
7458 getOpcodeDef(TargetOpcode::G_PTR_ADD, Root.getReg(), MRI);
7459 if (!PtrAdd || !isWorthFoldingIntoExtendedReg(*PtrAdd, MRI, true))
7460 return std::nullopt;
7461
7462 // Now, try to match an opcode which will match our specific offset.
7463 // We want a G_SHL or a G_MUL.
7464 MachineInstr *OffsetInst =
7466 return selectExtendedSHL(Root, PtrAdd->getOperand(1),
7467 OffsetInst->getOperand(0), SizeInBytes,
7468 /*WantsExt=*/false);
7469}
7470
7471/// This is used for computing addresses like this:
7472///
7473/// ldr x1, [x2, x3]
7474///
7475/// Where x2 is the base register, and x3 is an offset register.
7476///
7477/// When possible (or profitable) to fold a G_PTR_ADD into the address
7478/// calculation, this will do so. Otherwise, it will return std::nullopt.
7479InstructionSelector::ComplexRendererFns
7480AArch64InstructionSelector::selectAddrModeRegisterOffset(
7481 MachineOperand &Root) const {
7482 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
7483
7484 // We need a GEP.
7486 if (!mi_match(Root.getReg(), MRI, m_GPtrAdd(m_Reg(Base), m_Reg(Offset))))
7487 return std::nullopt;
7488
7489 // If this is used more than once, let's not bother folding.
7490 // TODO: Check if they are memory ops. If they are, then we can still fold
7491 // without having to recompute anything.
7492 if (!MRI.hasOneNonDBGUse(Root.getReg()))
7493 return std::nullopt;
7494
7495 // Base is the GEP's LHS, offset is its RHS.
7496 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(Base); },
7497 [=](MachineInstrBuilder &MIB) { MIB.addUse(Offset); },
7498 [=](MachineInstrBuilder &MIB) {
7499 // Need to add both immediates here to make sure that they are both
7500 // added to the instruction.
7501 MIB.addImm(0);
7502 MIB.addImm(0);
7503 }}};
7504}
7505
7506/// This is intended to be equivalent to selectAddrModeXRO in
7507/// AArch64ISelDAGtoDAG. It's used for selecting X register offset loads.
7508InstructionSelector::ComplexRendererFns
7509AArch64InstructionSelector::selectAddrModeXRO(MachineOperand &Root,
7510 unsigned SizeInBytes) const {
7511 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
7512 if (!Root.isReg())
7513 return std::nullopt;
7514 MachineInstr *PtrAdd =
7515 getOpcodeDef(TargetOpcode::G_PTR_ADD, Root.getReg(), MRI);
7516 if (!PtrAdd)
7517 return std::nullopt;
7518
7519 // Check for an immediates which cannot be encoded in the [base + imm]
7520 // addressing mode, and can't be encoded in an add/sub. If this happens, we'll
7521 // end up with code like:
7522 //
7523 // mov x0, wide
7524 // add x1 base, x0
7525 // ldr x2, [x1, x0]
7526 //
7527 // In this situation, we can use the [base, xreg] addressing mode to save an
7528 // add/sub:
7529 //
7530 // mov x0, wide
7531 // ldr x2, [base, x0]
7532 auto ValAndVReg =
7534 if (ValAndVReg) {
7535 unsigned Scale = Log2_32(SizeInBytes);
7536 int64_t ImmOff = ValAndVReg->Value.getSExtValue();
7537
7538 // Skip immediates that can be selected in the load/store addressing
7539 // mode.
7540 if (ImmOff % SizeInBytes == 0 && ImmOff >= 0 &&
7541 ImmOff < (0x1000 << Scale))
7542 return std::nullopt;
7543
7544 // Helper lambda to decide whether or not it is preferable to emit an add.
7545 auto isPreferredADD = [](int64_t ImmOff) {
7546 // Constants in [0x0, 0xfff] can be encoded in an add.
7547 if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL)
7548 return true;
7549
7550 // Can it be encoded in an add lsl #12?
7551 if ((ImmOff & 0xffffffffff000fffLL) != 0x0LL)
7552 return false;
7553
7554 // It can be encoded in an add lsl #12, but we may not want to. If it is
7555 // possible to select this as a single movz, then prefer that. A single
7556 // movz is faster than an add with a shift.
7557 return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL &&
7558 (ImmOff & 0xffffffffffff0fffLL) != 0x0LL;
7559 };
7560
7561 // If the immediate can be encoded in a single add/sub, then bail out.
7562 if (isPreferredADD(ImmOff) || isPreferredADD(-ImmOff))
7563 return std::nullopt;
7564 }
7565
7566 // Try to fold shifts into the addressing mode.
7567 auto AddrModeFns = selectAddrModeShiftedExtendXReg(Root, SizeInBytes);
7568 if (AddrModeFns)
7569 return AddrModeFns;
7570
7571 // If that doesn't work, see if it's possible to fold in registers from
7572 // a GEP.
7573 return selectAddrModeRegisterOffset(Root);
7574}
7575
7576/// This is used for computing addresses like this:
7577///
7578/// ldr x0, [xBase, wOffset, sxtw #LegalShiftVal]
7579///
7580/// Where we have a 64-bit base register, a 32-bit offset register, and an
7581/// extend (which may or may not be signed).
7582InstructionSelector::ComplexRendererFns
7583AArch64InstructionSelector::selectAddrModeWRO(MachineOperand &Root,
7584 unsigned SizeInBytes) const {
7585 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
7586
7587 MachineInstr *PtrAdd =
7588 getOpcodeDef(TargetOpcode::G_PTR_ADD, Root.getReg(), MRI);
7589 if (!PtrAdd || !isWorthFoldingIntoExtendedReg(*PtrAdd, MRI, true))
7590 return std::nullopt;
7591
7592 MachineOperand &LHS = PtrAdd->getOperand(1);
7593 MachineOperand &RHS = PtrAdd->getOperand(2);
7594 MachineInstr *OffsetInst = getDefIgnoringCopies(RHS.getReg(), MRI);
7595
7596 // The first case is the same as selectAddrModeXRO, except we need an extend.
7597 // In this case, we try to find a shift and extend, and fold them into the
7598 // addressing mode.
7599 //
7600 // E.g.
7601 //
7602 // off_reg = G_Z/S/ANYEXT ext_reg
7603 // val = G_CONSTANT LegalShiftVal
7604 // shift = G_SHL off_reg val
7605 // ptr = G_PTR_ADD base_reg shift
7606 // x = G_LOAD ptr
7607 //
7608 // In this case we can get a load like this:
7609 //
7610 // ldr x0, [base_reg, ext_reg, sxtw #LegalShiftVal]
7611 auto ExtendedShl = selectExtendedSHL(Root, LHS, OffsetInst->getOperand(0),
7612 SizeInBytes, /*WantsExt=*/true);
7613 if (ExtendedShl)
7614 return ExtendedShl;
7615
7616 // There was no shift. We can try and fold a G_Z/S/ANYEXT in alone though.
7617 //
7618 // e.g.
7619 // ldr something, [base_reg, ext_reg, sxtw]
7620 if (!isWorthFoldingIntoExtendedReg(*OffsetInst, MRI, true))
7621 return std::nullopt;
7622
7623 // Check if this is an extend. We'll get an extend type if it is.
7625 getExtendTypeForInst(*OffsetInst, MRI, /*IsLoadStore=*/true);
7627 return std::nullopt;
7628
7629 // Need a 32-bit wide register.
7630 MachineIRBuilder MIB(*PtrAdd);
7631 Register ExtReg = moveScalarRegClass(OffsetInst->getOperand(1).getReg(),
7632 AArch64::GPR32RegClass, MIB);
7633 unsigned SignExtend = Ext == AArch64_AM::SXTW;
7634
7635 // Base is LHS, offset is ExtReg.
7636 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(LHS.getReg()); },
7637 [=](MachineInstrBuilder &MIB) { MIB.addUse(ExtReg); },
7638 [=](MachineInstrBuilder &MIB) {
7639 MIB.addImm(SignExtend);
7640 MIB.addImm(0);
7641 }}};
7642}
7643
7644/// Select a "register plus unscaled signed 9-bit immediate" address. This
7645/// should only match when there is an offset that is not valid for a scaled
7646/// immediate addressing mode. The "Size" argument is the size in bytes of the
7647/// memory reference, which is needed here to know what is valid for a scaled
7648/// immediate.
7649InstructionSelector::ComplexRendererFns
7650AArch64InstructionSelector::selectAddrModeUnscaled(MachineOperand &Root,
7651 unsigned Size) const {
7652 MachineRegisterInfo &MRI =
7653 Root.getParent()->getParent()->getParent()->getRegInfo();
7654
7655 if (!Root.isReg())
7656 return std::nullopt;
7657
7658 if (!isBaseWithConstantOffset(Root, MRI))
7659 return std::nullopt;
7660
7661 MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());
7662
7663 MachineOperand &OffImm = RootDef->getOperand(2);
7664 if (!OffImm.isReg())
7665 return std::nullopt;
7666 MachineInstr *RHS = MRI.getVRegDef(OffImm.getReg());
7667 if (RHS->getOpcode() != TargetOpcode::G_CONSTANT)
7668 return std::nullopt;
7669 int64_t RHSC;
7670 MachineOperand &RHSOp1 = RHS->getOperand(1);
7671 if (!RHSOp1.isCImm() || RHSOp1.getCImm()->getBitWidth() > 64)
7672 return std::nullopt;
7673 RHSC = RHSOp1.getCImm()->getSExtValue();
7674
7675 if (RHSC >= -256 && RHSC < 256) {
7676 MachineOperand &Base = RootDef->getOperand(1);
7677 return {{
7678 [=](MachineInstrBuilder &MIB) { MIB.add(Base); },
7679 [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); },
7680 }};
7681 }
7682 return std::nullopt;
7683}
7684
7685InstructionSelector::ComplexRendererFns
7686AArch64InstructionSelector::tryFoldAddLowIntoImm(MachineInstr &RootDef,
7687 unsigned Size,
7688 MachineRegisterInfo &MRI) const {
7689 if (RootDef.getOpcode() != AArch64::G_ADD_LOW)
7690 return std::nullopt;
7691 MachineInstr &Adrp = *MRI.getVRegDef(RootDef.getOperand(1).getReg());
7692 if (Adrp.getOpcode() != AArch64::ADRP)
7693 return std::nullopt;
7694
7695 // TODO: add heuristics like isWorthFoldingADDlow() from SelectionDAG.
7696 auto Offset = Adrp.getOperand(1).getOffset();
7697 if (Offset % Size != 0)
7698 return std::nullopt;
7699
7700 auto GV = Adrp.getOperand(1).getGlobal();
7701 if (GV->isThreadLocal())
7702 return std::nullopt;
7703
7704 auto &MF = *RootDef.getParent()->getParent();
7705 if (GV->getPointerAlignment(MF.getDataLayout()) < Size)
7706 return std::nullopt;
7707
7708 unsigned OpFlags = STI.ClassifyGlobalReference(GV, MF.getTarget());
7709 Register AdrpReg = Adrp.getOperand(0).getReg();
7710 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(AdrpReg); },
7711 [=](MachineInstrBuilder &MIB) {
7712 MIB.addGlobalAddress(GV, Offset,
7713 OpFlags | AArch64II::MO_PAGEOFF |
7715 }}};
7716}
7717
7718/// Select a "register plus scaled unsigned 12-bit immediate" address. The
7719/// "Size" argument is the size in bytes of the memory reference, which
7720/// determines the scale.
7721InstructionSelector::ComplexRendererFns
7722AArch64InstructionSelector::selectAddrModeIndexed(MachineOperand &Root,
7723 unsigned Size) const {
7724 MachineFunction &MF = *Root.getParent()->getParent()->getParent();
7725 MachineRegisterInfo &MRI = MF.getRegInfo();
7726
7727 if (!Root.isReg())
7728 return std::nullopt;
7729
7730 MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());
7731 if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
7732 return {{
7733 [=](MachineInstrBuilder &MIB) { MIB.add(RootDef->getOperand(1)); },
7734 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
7735 }};
7736 }
7737
7739 // Check if we can fold in the ADD of small code model ADRP + ADD address.
7740 // HACK: ld64 on Darwin doesn't support relocations on PRFM, so we can't fold
7741 // globals into the offset.
7742 MachineInstr *RootParent = Root.getParent();
7743 if (CM == CodeModel::Small &&
7744 !(RootParent->getOpcode() == AArch64::G_AARCH64_PREFETCH &&
7745 STI.isTargetDarwin())) {
7746 auto OpFns = tryFoldAddLowIntoImm(*RootDef, Size, MRI);
7747 if (OpFns)
7748 return OpFns;
7749 }
7750
7751 if (isBaseWithConstantOffset(Root, MRI)) {
7752 MachineOperand &LHS = RootDef->getOperand(1);
7753 MachineOperand &RHS = RootDef->getOperand(2);
7754 MachineInstr *LHSDef = MRI.getVRegDef(LHS.getReg());
7755 MachineInstr *RHSDef = MRI.getVRegDef(RHS.getReg());
7756
7757 int64_t RHSC = (int64_t)RHSDef->getOperand(1).getCImm()->getZExtValue();
7758 unsigned Scale = Log2_32(Size);
7759 if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Scale)) {
7760 if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
7761 return {{
7762 [=](MachineInstrBuilder &MIB) { MIB.add(LHSDef->getOperand(1)); },
7763 [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC >> Scale); },
7764 }};
7765
7766 return {{
7767 [=](MachineInstrBuilder &MIB) { MIB.add(LHS); },
7768 [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC >> Scale); },
7769 }};
7770 }
7771 }
7772
7773 // Before falling back to our general case, check if the unscaled
7774 // instructions can handle this. If so, that's preferable.
7775 if (selectAddrModeUnscaled(Root, Size))
7776 return std::nullopt;
7777
7778 return {{
7779 [=](MachineInstrBuilder &MIB) { MIB.add(Root); },
7780 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
7781 }};
7782}
7783
7784/// Given a shift instruction, return the correct shift type for that
7785/// instruction.
7787 switch (MI.getOpcode()) {
7788 default:
7790 case TargetOpcode::G_SHL:
7791 return AArch64_AM::LSL;
7792 case TargetOpcode::G_LSHR:
7793 return AArch64_AM::LSR;
7794 case TargetOpcode::G_ASHR:
7795 return AArch64_AM::ASR;
7796 case TargetOpcode::G_ROTR:
7797 return AArch64_AM::ROR;
7798 }
7799}
7800
7801/// Select a "shifted register" operand. If the value is not shifted, set the
7802/// shift operand to a default value of "lsl 0".
7803InstructionSelector::ComplexRendererFns
7804AArch64InstructionSelector::selectShiftedRegister(MachineOperand &Root,
7805 bool AllowROR) const {
7806 if (!Root.isReg())
7807 return std::nullopt;
7808 MachineRegisterInfo &MRI =
7809 Root.getParent()->getParent()->getParent()->getRegInfo();
7810
7811 // Check if the operand is defined by an instruction which corresponds to
7812 // a ShiftExtendType. E.g. a G_SHL, G_LSHR, etc.
7813 MachineInstr *ShiftInst = MRI.getVRegDef(Root.getReg());
7815 if (ShType == AArch64_AM::InvalidShiftExtend)
7816 return std::nullopt;
7817 if (ShType == AArch64_AM::ROR && !AllowROR)
7818 return std::nullopt;
7819 if (!isWorthFoldingIntoExtendedReg(*ShiftInst, MRI, false))
7820 return std::nullopt;
7821
7822 // Need an immediate on the RHS.
7823 MachineOperand &ShiftRHS = ShiftInst->getOperand(2);
7824 auto Immed = getImmedFromMO(ShiftRHS);
7825 if (!Immed)
7826 return std::nullopt;
7827
7828 // We have something that we can fold. Fold in the shift's LHS and RHS into
7829 // the instruction.
7830 MachineOperand &ShiftLHS = ShiftInst->getOperand(1);
7831 Register ShiftReg = ShiftLHS.getReg();
7832
7833 unsigned NumBits = MRI.getType(ShiftReg).getSizeInBits();
7834 unsigned Val = *Immed & (NumBits - 1);
7835 unsigned ShiftVal = AArch64_AM::getShifterImm(ShType, Val);
7836
7837 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(ShiftReg); },
7838 [=](MachineInstrBuilder &MIB) { MIB.addImm(ShiftVal); }}};
7839}
7840
7841AArch64_AM::ShiftExtendType AArch64InstructionSelector::getExtendTypeForInst(
7842 MachineInstr &MI, MachineRegisterInfo &MRI, bool IsLoadStore) const {
7843 unsigned Opc = MI.getOpcode();
7844
7845 // Handle explicit extend instructions first.
7846 if (Opc == TargetOpcode::G_SEXT || Opc == TargetOpcode::G_SEXT_INREG) {
7847 unsigned Size;
7848 if (Opc == TargetOpcode::G_SEXT)
7849 Size = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
7850 else
7851 Size = MI.getOperand(2).getImm();
7852 assert(Size != 64 && "Extend from 64 bits?");
7853 switch (Size) {
7854 case 8:
7855 return IsLoadStore ? AArch64_AM::InvalidShiftExtend : AArch64_AM::SXTB;
7856 case 16:
7857 return IsLoadStore ? AArch64_AM::InvalidShiftExtend : AArch64_AM::SXTH;
7858 case 32:
7859 return AArch64_AM::SXTW;
7860 default:
7862 }
7863 }
7864
7865 if (Opc == TargetOpcode::G_ZEXT || Opc == TargetOpcode::G_ANYEXT) {
7866 unsigned Size = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
7867 assert(Size != 64 && "Extend from 64 bits?");
7868 switch (Size) {
7869 case 8:
7870 return IsLoadStore ? AArch64_AM::InvalidShiftExtend : AArch64_AM::UXTB;
7871 case 16:
7872 return IsLoadStore ? AArch64_AM::InvalidShiftExtend : AArch64_AM::UXTH;
7873 case 32:
7874 return AArch64_AM::UXTW;
7875 default:
7877 }
7878 }
7879
7880 // Don't have an explicit extend. Try to handle a G_AND with a constant mask
7881 // on the RHS.
7882 if (Opc != TargetOpcode::G_AND)
7884
7885 std::optional<uint64_t> MaybeAndMask = getImmedFromMO(MI.getOperand(2));
7886 if (!MaybeAndMask)
7888 uint64_t AndMask = *MaybeAndMask;
7889 switch (AndMask) {
7890 default:
7892 case 0xFF:
7893 return !IsLoadStore ? AArch64_AM::UXTB : AArch64_AM::InvalidShiftExtend;
7894 case 0xFFFF:
7895 return !IsLoadStore ? AArch64_AM::UXTH : AArch64_AM::InvalidShiftExtend;
7896 case 0xFFFFFFFF:
7897 return AArch64_AM::UXTW;
7898 }
7899}
7900
7901Register AArch64InstructionSelector::moveScalarRegClass(
7902 Register Reg, const TargetRegisterClass &RC, MachineIRBuilder &MIB) const {
7903 MachineRegisterInfo &MRI = *MIB.getMRI();
7904 auto Ty = MRI.getType(Reg);
7905 assert(!Ty.isVector() && "Expected scalars only!");
7906 if (Ty.getSizeInBits() == TRI.getRegSizeInBits(RC))
7907 return Reg;
7908
7909 // Create a copy and immediately select it.
7910 // FIXME: We should have an emitCopy function?
7911 auto Copy = MIB.buildCopy({&RC}, {Reg});
7912 selectCopy(*Copy, TII, MRI, TRI, RBI);
7913 return Copy.getReg(0);
7914}
7915
7916/// Select an "extended register" operand. This operand folds in an extend
7917/// followed by an optional left shift.
7918InstructionSelector::ComplexRendererFns
7919AArch64InstructionSelector::selectArithExtendedRegister(
7920 MachineOperand &Root) const {
7921 if (!Root.isReg())
7922 return std::nullopt;
7923 MachineRegisterInfo &MRI =
7924 Root.getParent()->getParent()->getParent()->getRegInfo();
7925
7926 uint64_t ShiftVal = 0;
7927 Register ExtReg;
7929 MachineInstr *RootDef = getDefIgnoringCopies(Root.getReg(), MRI);
7930 if (!RootDef)
7931 return std::nullopt;
7932
7933 if (!isWorthFoldingIntoExtendedReg(*RootDef, MRI, false))
7934 return std::nullopt;
7935
7936 // Check if we can fold a shift and an extend.
7937 if (RootDef->getOpcode() == TargetOpcode::G_SHL) {
7938 // Look for a constant on the RHS of the shift.
7939 MachineOperand &RHS = RootDef->getOperand(2);
7940 std::optional<uint64_t> MaybeShiftVal = getImmedFromMO(RHS);
7941 if (!MaybeShiftVal)
7942 return std::nullopt;
7943 ShiftVal = *MaybeShiftVal;
7944 if (ShiftVal > 4)
7945 return std::nullopt;
7946 // Look for a valid extend instruction on the LHS of the shift.
7947 MachineOperand &LHS = RootDef->getOperand(1);
7948 MachineInstr *ExtDef = getDefIgnoringCopies(LHS.getReg(), MRI);
7949 if (!ExtDef)
7950 return std::nullopt;
7951 Ext = getExtendTypeForInst(*ExtDef, MRI);
7953 return std::nullopt;
7954 ExtReg = ExtDef->getOperand(1).getReg();
7955 } else {
7956 // Didn't get a shift. Try just folding an extend.
7957 Ext = getExtendTypeForInst(*RootDef, MRI);
7959 return std::nullopt;
7960 ExtReg = RootDef->getOperand(1).getReg();
7961
7962 // If we have a 32 bit instruction which zeroes out the high half of a
7963 // register, we get an implicit zero extend for free. Check if we have one.
7964 // FIXME: We actually emit the extend right now even though we don't have
7965 // to.
7966 if (Ext == AArch64_AM::UXTW && MRI.getType(ExtReg).getSizeInBits() == 32) {
7967 MachineInstr *ExtInst = MRI.getVRegDef(ExtReg);
7968 if (isDef32(*ExtInst))
7969 return std::nullopt;
7970 }
7971 }
7972
7973 // We require a GPR32 here. Narrow the ExtReg if needed using a subregister
7974 // copy.
7975 MachineIRBuilder MIB(*RootDef);
7976 ExtReg = moveScalarRegClass(ExtReg, AArch64::GPR32RegClass, MIB);
7977
7978 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(ExtReg); },
7979 [=](MachineInstrBuilder &MIB) {
7980 MIB.addImm(getArithExtendImm(Ext, ShiftVal));
7981 }}};
7982}
7983
7984InstructionSelector::ComplexRendererFns
7985AArch64InstructionSelector::selectExtractHigh(MachineOperand &Root) const {
7986 if (!Root.isReg())
7987 return std::nullopt;
7988 MachineRegisterInfo &MRI =
7989 Root.getParent()->getParent()->getParent()->getRegInfo();
7990
7991 auto Extract = getDefSrcRegIgnoringCopies(Root.getReg(), MRI);
7992 while (Extract && Extract->MI->getOpcode() == TargetOpcode::G_BITCAST &&
7993 STI.isLittleEndian())
7994 Extract =
7995 getDefSrcRegIgnoringCopies(Extract->MI->getOperand(1).getReg(), MRI);
7996 if (!Extract)
7997 return std::nullopt;
7998
7999 if (auto *Unmerge = dyn_cast<GUnmerge>(Extract->MI)) {
8000 if (Unmerge->getNumDefs() == 2 &&
8001 Extract->Reg == Unmerge->getOperand(1).getReg()) {
8002 Register ExtReg = Unmerge->getSourceReg();
8003 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(ExtReg); }}};
8004 }
8005 }
8006 if (auto *ExtElt = dyn_cast<GExtractVectorElement>(Extract->MI)) {
8007 LLT SrcTy = MRI.getType(ExtElt->getVectorReg());
8008 auto LaneIdx =
8009 getIConstantVRegValWithLookThrough(ExtElt->getIndexReg(), MRI);
8010 if (LaneIdx && SrcTy == LLT::fixed_vector(2, 64) &&
8011 LaneIdx->Value.getSExtValue() == 1) {
8012 Register ExtReg = ExtElt->getVectorReg();
8013 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(ExtReg); }}};
8014 }
8015 }
8016 if (auto *Subvec = dyn_cast<GExtractSubvector>(Extract->MI)) {
8017 LLT SrcTy = MRI.getType(Subvec->getSrcVec());
8018 auto LaneIdx = Subvec->getIndexImm();
8019 if (LaneIdx == SrcTy.getNumElements() / 2) {
8020 Register ExtReg = Subvec->getSrcVec();
8021 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(ExtReg); }}};
8022 }
8023 }
8024
8025 return std::nullopt;
8026}
8027
8028InstructionSelector::ComplexRendererFns
8029AArch64InstructionSelector::selectCVTFixedPointBase(const MachineOperand &Root,
8030 unsigned DstElemWidth,
8031 bool isReciprocal) const {
8032 if (!Root.isReg())
8033 return std::nullopt;
8034 const MachineRegisterInfo &MRI =
8035 Root.getParent()->getParent()->getParent()->getRegInfo();
8036
8037 Register Reg = Root.getReg();
8038 MachineInstr *Dup = getDefIgnoringCopies(Reg, MRI);
8039
8040 if (Dup && Dup->getOpcode() == TargetOpcode::G_SPLAT_VECTOR)
8041 Reg = Dup->getOperand(1).getReg();
8042
8043 std::optional<ValueAndVReg> CstVal =
8045
8046 if (!CstVal)
8047 return std::nullopt;
8048
8049 unsigned CstElemWidth = MRI.getType(Reg).getScalarSizeInBits();
8050 APFloat FVal(0.0);
8051 switch (CstElemWidth) {
8052 case 16:
8053 FVal = APFloat(APFloat::IEEEhalf(), CstVal->Value.trunc(CstElemWidth));
8054 break;
8055 case 32:
8056 FVal = APFloat(APFloat::IEEEsingle(), CstVal->Value);
8057 break;
8058 case 64:
8059 FVal = APFloat(APFloat::IEEEdouble(), CstVal->Value);
8060 break;
8061 default:
8062 return std::nullopt;
8063 };
8064 if (unsigned FBits =
8065 CheckFixedPointOperandConstant(FVal, DstElemWidth, isReciprocal))
8066 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(FBits); }}};
8067
8068 return std::nullopt;
8069}
8070
8071unsigned AArch64InstructionSelector::getFixedPointWidthFromOperand(
8072 const MachineOperand &Root) const {
8073 return Root.getParent()
8074 ->getMF()
8075 ->getRegInfo()
8076 .getType(Root.getReg())
8078}
8079
8080template <unsigned Width>
8081InstructionSelector::ComplexRendererFns
8082AArch64InstructionSelector::selectCVTFixedPoint(MachineOperand &Root) const {
8083 return selectCVTFixedPointBase(Root, Width, /*isReciprocal*/ false);
8084}
8085
8086template <unsigned Width>
8087InstructionSelector::ComplexRendererFns
8088AArch64InstructionSelector::selectCVTFixedPosRecipOperand(
8089 MachineOperand &Root) const {
8090 return selectCVTFixedPointBase(Root, Width, /*isReciprocal*/ true);
8091}
8092
8093InstructionSelector::ComplexRendererFns
8094AArch64InstructionSelector::selectCVTFixedPointVec(MachineOperand &Root) const {
8095 return selectCVTFixedPointBase(Root, getFixedPointWidthFromOperand(Root),
8096 /*isReciprocal*/ false);
8097}
8098
8099InstructionSelector::ComplexRendererFns
8100AArch64InstructionSelector::selectCVTFixedPosRecipOperandVec(
8101 MachineOperand &Root) const {
8102 return selectCVTFixedPointBase(Root, getFixedPointWidthFromOperand(Root),
8103 /*isReciprocal*/ true);
8104}
8105
8106void AArch64InstructionSelector::renderFixedPointScalarXForm(
8107 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
8108 assert(OpIdx == 3 && MI.getOperand(OpIdx).isImm() &&
8109 "Expected vecshift immediate operand");
8110 MIB.addImm(MI.getOperand(OpIdx).getImm());
8111}
8112
8113void AArch64InstructionSelector::renderFixedPointImm(MachineInstrBuilder &MIB,
8114 const MachineOperand &Root,
8115 unsigned Width,
8116 bool isReciprocal) const {
8117 // FIXME: This is only needed to satisfy the type checking in tablegen, and
8118 // should be able to reuse the Renderers already calculated by
8119 // selectCVTFixedPointBase.
8120 InstructionSelector::ComplexRendererFns Renderer =
8121 selectCVTFixedPointBase(Root, Width, isReciprocal);
8122 assert((Renderer && Renderer->size() == 1) &&
8123 "Expected selectCVTFixedPointBase to provide a function\n");
8124 (Renderer->front())(MIB);
8125}
8126
8127void AArch64InstructionSelector::renderFixedPointXForm(MachineInstrBuilder &MIB,
8128 const MachineInstr &MI,
8129 int OpIdx) const {
8130 const MachineOperand &Root = MI.getOperand(OpIdx);
8131 renderFixedPointImm(MIB, Root, getFixedPointWidthFromOperand(Root),
8132 /*isReciprocal*/ false);
8133}
8134
8135void AArch64InstructionSelector::renderFixedPointRecipXForm(
8136 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
8137 const MachineOperand &Root = MI.getOperand(OpIdx);
8138 renderFixedPointImm(MIB, Root, getFixedPointWidthFromOperand(Root),
8139 /*isReciprocal*/ true);
8140}
8141
8142void AArch64InstructionSelector::renderTruncImm(MachineInstrBuilder &MIB,
8143 const MachineInstr &MI,
8144 int OpIdx) const {
8145 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
8146 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
8147 "Expected G_CONSTANT");
8148 std::optional<int64_t> CstVal =
8149 getIConstantVRegSExtVal(MI.getOperand(0).getReg(), MRI);
8150 assert(CstVal && "Expected constant value");
8151 MIB.addImm(*CstVal);
8152}
8153
8154void AArch64InstructionSelector::renderLogicalImm32(
8155 MachineInstrBuilder &MIB, const MachineInstr &I, int OpIdx) const {
8156 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
8157 "Expected G_CONSTANT");
8158 uint64_t CstVal = I.getOperand(1).getCImm()->getZExtValue();
8160 MIB.addImm(Enc);
8161}
8162
8163void AArch64InstructionSelector::renderLogicalImm64(
8164 MachineInstrBuilder &MIB, const MachineInstr &I, int OpIdx) const {
8165 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
8166 "Expected G_CONSTANT");
8167 uint64_t CstVal = I.getOperand(1).getCImm()->getZExtValue();
8169 MIB.addImm(Enc);
8170}
8171
8172void AArch64InstructionSelector::renderUbsanTrap(MachineInstrBuilder &MIB,
8173 const MachineInstr &MI,
8174 int OpIdx) const {
8175 assert(MI.getOpcode() == TargetOpcode::G_UBSANTRAP && OpIdx == 0 &&
8176 "Expected G_UBSANTRAP");
8177 MIB.addImm(MI.getOperand(0).getImm() | ('U' << 8));
8178}
8179
8180void AArch64InstructionSelector::renderFPImm16(MachineInstrBuilder &MIB,
8181 const MachineInstr &MI,
8182 int OpIdx) const {
8183 assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&
8184 "Expected G_FCONSTANT");
8185 MIB.addImm(
8186 AArch64_AM::getFP16Imm(MI.getOperand(1).getFPImm()->getValueAPF()));
8187}
8188
8189void AArch64InstructionSelector::renderFPImm32(MachineInstrBuilder &MIB,
8190 const MachineInstr &MI,
8191 int OpIdx) const {
8192 assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&
8193 "Expected G_FCONSTANT");
8194 MIB.addImm(
8195 AArch64_AM::getFP32Imm(MI.getOperand(1).getFPImm()->getValueAPF()));
8196}
8197
8198void AArch64InstructionSelector::renderFPImm64(MachineInstrBuilder &MIB,
8199 const MachineInstr &MI,
8200 int OpIdx) const {
8201 assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&
8202 "Expected G_FCONSTANT");
8203 MIB.addImm(
8204 AArch64_AM::getFP64Imm(MI.getOperand(1).getFPImm()->getValueAPF()));
8205}
8206
8207bool AArch64InstructionSelector::isLoadStoreOfNumBytes(
8208 const MachineInstr &MI, unsigned NumBytes) const {
8209 if (!MI.mayLoadOrStore())
8210 return false;
8211 assert(MI.hasOneMemOperand() &&
8212 "Expected load/store to have only one mem op!");
8213 return (*MI.memoperands_begin())->getSize() == NumBytes;
8214}
8215
8216bool AArch64InstructionSelector::isDef32(const MachineInstr &MI) const {
8217 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
8218 if (MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() != 32)
8219 return false;
8220
8221 // Only return true if we know the operation will zero-out the high half of
8222 // the 64-bit register. Truncates can be subregister copies, which don't
8223 // zero out the high bits. Copies and other copy-like instructions can be
8224 // fed by truncates, or could be lowered as subregister copies.
8225 switch (MI.getOpcode()) {
8226 default:
8227 return true;
8228 case TargetOpcode::COPY:
8229 case TargetOpcode::G_BITCAST:
8230 case TargetOpcode::G_TRUNC:
8231 case TargetOpcode::G_PHI:
8232 return false;
8233 }
8234}
8235
8236
8237// Perform fixups on the given PHI instruction's operands to force them all
8238// to be the same as the destination regbank.
8240 const AArch64RegisterBankInfo &RBI) {
8241 assert(MI.getOpcode() == TargetOpcode::G_PHI && "Expected a G_PHI");
8242 Register DstReg = MI.getOperand(0).getReg();
8243 const RegisterBank *DstRB = MRI.getRegBankOrNull(DstReg);
8244 assert(DstRB && "Expected PHI dst to have regbank assigned");
8245 MachineIRBuilder MIB(MI);
8246
8247 // Go through each operand and ensure it has the same regbank.
8248 for (MachineOperand &MO : llvm::drop_begin(MI.operands())) {
8249 if (!MO.isReg())
8250 continue;
8251 Register OpReg = MO.getReg();
8252 const RegisterBank *RB = MRI.getRegBankOrNull(OpReg);
8253 if (RB != DstRB) {
8254 // Insert a cross-bank copy.
8255 auto *OpDef = MRI.getVRegDef(OpReg);
8256 const LLT &Ty = MRI.getType(OpReg);
8257 MachineBasicBlock &OpDefBB = *OpDef->getParent();
8258
8259 // Any instruction we insert must appear after all PHIs in the block
8260 // for the block to be valid MIR.
8261 MachineBasicBlock::iterator InsertPt = std::next(OpDef->getIterator());
8262 if (InsertPt != OpDefBB.end() && InsertPt->isPHI())
8263 InsertPt = OpDefBB.getFirstNonPHI();
8264 MIB.setInsertPt(*OpDef->getParent(), InsertPt);
8265 auto Copy = MIB.buildCopy(Ty, OpReg);
8266 MRI.setRegBank(Copy.getReg(0), *DstRB);
8267 MO.setReg(Copy.getReg(0));
8268 }
8269 }
8270}
8271
8272void AArch64InstructionSelector::processPHIs(MachineFunction &MF) {
8273 // We're looking for PHIs, build a list so we don't invalidate iterators.
8274 MachineRegisterInfo &MRI = MF.getRegInfo();
8276 for (auto &BB : MF) {
8277 for (auto &MI : BB) {
8278 if (MI.getOpcode() == TargetOpcode::G_PHI)
8279 Phis.emplace_back(&MI);
8280 }
8281 }
8282
8283 for (auto *MI : Phis) {
8284 // We need to do some work here if the operand types are < 16 bit and they
8285 // are split across fpr/gpr banks. Since all types <32b on gpr
8286 // end up being assigned gpr32 regclasses, we can end up with PHIs here
8287 // which try to select between a gpr32 and an fpr16. Ideally RBS shouldn't
8288 // be selecting heterogenous regbanks for operands if possible, but we
8289 // still need to be able to deal with it here.
8290 //
8291 // To fix this, if we have a gpr-bank operand < 32b in size and at least
8292 // one other operand is on the fpr bank, then we add cross-bank copies
8293 // to homogenize the operand banks. For simplicity the bank that we choose
8294 // to settle on is whatever bank the def operand has. For example:
8295 //
8296 // %endbb:
8297 // %dst:gpr(s16) = G_PHI %in1:gpr(s16), %bb1, %in2:fpr(s16), %bb2
8298 // =>
8299 // %bb2:
8300 // ...
8301 // %in2_copy:gpr(s16) = COPY %in2:fpr(s16)
8302 // ...
8303 // %endbb:
8304 // %dst:gpr(s16) = G_PHI %in1:gpr(s16), %bb1, %in2_copy:gpr(s16), %bb2
8305 bool HasGPROp = false, HasFPROp = false;
8306 for (const MachineOperand &MO : llvm::drop_begin(MI->operands())) {
8307 if (!MO.isReg())
8308 continue;
8309 const LLT &Ty = MRI.getType(MO.getReg());
8310 if (!Ty.isValid() || !Ty.isScalar())
8311 break;
8312 if (Ty.getSizeInBits() >= 32)
8313 break;
8314 const RegisterBank *RB = MRI.getRegBankOrNull(MO.getReg());
8315 // If for some reason we don't have a regbank yet. Don't try anything.
8316 if (!RB)
8317 break;
8318
8319 if (RB->getID() == AArch64::GPRRegBankID)
8320 HasGPROp = true;
8321 else
8322 HasFPROp = true;
8323 }
8324 // We have heterogenous regbanks, need to fixup.
8325 if (HasGPROp && HasFPROp)
8326 fixupPHIOpBanks(*MI, MRI, RBI);
8327 }
8328}
8329
8330namespace llvm {
8331InstructionSelector *
8334 const AArch64RegisterBankInfo &RBI) {
8335 return new AArch64InstructionSelector(TM, Subtarget, RBI);
8336}
8337}
#define Success
MachineInstrBuilder MachineInstrBuilder & DefMI
static std::tuple< SDValue, SDValue > extractPtrauthBlendDiscriminators(SDValue Disc, SelectionDAG *DAG)
static bool isPreferredADD(int64_t ImmOff)
static SDValue emitConditionalComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue CCOp, AArch64CC::CondCode Predicate, AArch64CC::CondCode OutCC, const SDLoc &DL, SelectionDAG &DAG)
can be transformed to: not (and (not (and (setCC (cmp C)) (setCD (cmp D)))) (and (not (setCA (cmp A))...
static bool isCMN(SDValue Op, ISD::CondCode CC, SelectionDAG &DAG)
static SDValue emitConjunctionRec(SelectionDAG &DAG, SDValue Val, AArch64CC::CondCode &OutCC, bool Negate, SDValue CCOp, AArch64CC::CondCode Predicate)
Emit conjunction or disjunction tree with the CMP/FCMP followed by a chain of CCMP/CFCMP ops.
static void changeFPCCToANDAArch64CC(ISD::CondCode CC, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2)
Convert a DAG fp condition code to an AArch64 CC.
const AArch64Subtarget & Subtarget
static bool canEmitConjunction(SelectionDAG &DAG, const SDValue Val, bool &CanNegate, bool &MustBeFirst, bool &PreferFirst, bool WillNegate, unsigned Depth=0)
Returns true if Val is a tree of AND/OR/SETCC operations that can be expressed as a conjunction.
const TargetLowering & TLI
SDValue Pred
static SDValue emitConjunction(SelectionDAG &DAG, SDValue Val, AArch64CC::CondCode &OutCC)
Emit expression as a conjunction (a series of CCMP/CFCMP ops).
#define GET_GLOBALISEL_PREDICATES_INIT
#define GET_GLOBALISEL_TEMPORARIES_INIT
static Register getTestBitReg(Register Reg, uint64_t &Bit, bool &Invert, MachineRegisterInfo &MRI)
Return a register which can be used as a bit to test in a TB(N)Z.
static unsigned getMinSizeForRegBank(const RegisterBank &RB)
Returns the minimum size the given register bank can hold.
static unsigned selectLoadStoreUIOp(unsigned GenericOpc, unsigned RegBankID, unsigned OpSize)
Select the AArch64 opcode for the G_LOAD or G_STORE operation GenericOpc, appropriate for the (value)...
static const TargetRegisterClass * getMinClassForRegBank(const RegisterBank &RB, TypeSize SizeInBits, bool GetAllRegSet=false)
Given a register bank, and size in bits, return the smallest register class that can represent that c...
static std::optional< APInt > getVectorShiftImm(Register Reg, MachineRegisterInfo &MRI)
Returns the element immediate value of a vector shift operand if found.
static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID, unsigned OpSize)
Select the AArch64 opcode for the basic binary operation GenericOpc, appropriate for the register ban...
static bool getSubRegForClass(const TargetRegisterClass *RC, const TargetRegisterInfo &TRI, unsigned &SubReg)
Returns the correct subregister to use for a given register class.
static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
static bool copySubReg(MachineInstr &I, MachineRegisterInfo &MRI, const RegisterBankInfo &RBI, Register SrcReg, const TargetRegisterClass *To, unsigned SubReg)
Helper function for selectCopy.
static AArch64CC::CondCode changeICMPPredToAArch64CC(CmpInst::Predicate P, Register RHS={}, MachineRegisterInfo *MRI=nullptr)
static std::optional< uint64_t > getVectorSHLImm(LLT SrcTy, Register Reg, MachineRegisterInfo &MRI)
Matches and returns the shift immediate value for a SHL instruction given a shift operand.
static Register createDTuple(ArrayRef< Register > Regs, MachineIRBuilder &MIB)
Create a tuple of D-registers using the registers in Regs.
static void fixupPHIOpBanks(MachineInstr &MI, MachineRegisterInfo &MRI, const AArch64RegisterBankInfo &RBI)
static AArch64_AM::ShiftExtendType getShiftTypeForInst(MachineInstr &MI)
Given a shift instruction, return the correct shift type for that instruction.
static bool getLaneCopyOpcode(unsigned &CopyOpc, unsigned &ExtractSubReg, const unsigned EltSize)
static Register createQTuple(ArrayRef< Register > Regs, MachineIRBuilder &MIB)
Create a tuple of Q-registers using the registers in Regs.
static std::optional< uint64_t > getImmedFromMO(const MachineOperand &Root)
static std::pair< unsigned, unsigned > getInsertVecEltOpInfo(const RegisterBank &RB, unsigned EltSize)
Return an <Opcode, SubregIndex> pair to do an vector elt insert of a given size and RB.
static Register createTuple(ArrayRef< Register > Regs, const unsigned RegClassIDs[], const unsigned SubRegs[], MachineIRBuilder &MIB)
Create a REG_SEQUENCE instruction using the registers in Regs.
static void changeFPCCToORAArch64CC(CmpInst::Predicate CC, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2)
changeFPCCToORAArch64CC - Convert an IR fp condition code to an AArch64 CC.
unsigned RegSize
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
This file declares the targeting of the RegisterBankInfo class for AArch64.
uint64_t uint64_t
uint64_t Imm
constexpr LLT S16
constexpr LLT S32
constexpr LLT S64
constexpr LLT S8
static bool isStore(int Opcode)
static bool selectMergeValues(MachineInstrBuilder &MIB, const ARMBaseInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
static bool selectUnmergeValues(MachineInstrBuilder &MIB, const ARMBaseInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
MachineBasicBlock & MBB
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file contains constants used for implementing Dwarf debug support.
Provides analysis for querying information about KnownBits during GISel passes.
#define DEBUG_TYPE
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
static void emitLoadFromConstantPool(Register DstReg, const Constant *ConstVal, MachineIRBuilder &MIRBuilder)
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition Lint.cpp:550
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
Contains matchers for matching SSA Machine Instructions.
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
This file declares the MachineIRBuilder class.
Register Reg
Register const TargetRegisterInfo * TRI
Promote Memory to Register
Definition Mem2Reg.cpp:110
#define T
static MCRegister getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
#define P(N)
static MachineBasicBlock * emitSelect(MachineInstr &MI, MachineBasicBlock *BB, const TargetInstrInfo *TII, const PPCSubtarget &Subtarget)
Emit SELECT instruction, using ISEL if available, otherwise use branch-based control flow.
if(PassOpts->AAPipeline)
static StringRef getName(Value *V)
RISCVCC::CondCode CC
MachineRegisterInfo & MRI
MachineInstr *& Lo
#define LLVM_DEBUG(...)
Definition Debug.h:119
static constexpr int Concat[]
Value * RHS
Value * LHS
This class provides the information for the target register banks.
std::optional< uint16_t > getPtrAuthBlockAddressDiscriminatorIfEnabled(const Function &ParentFn) const
Compute the integer discriminator for a given BlockAddress constant, if blockaddress signing is enabl...
const AArch64TargetLowering * getTargetLowering() const override
unsigned ClassifyGlobalReference(const GlobalValue *GV, const TargetMachine &TM) const
ClassifyGlobalReference - Find the target operand flags that describe how a global value should be re...
bool isX16X17Safer() const
Returns whether the operating system makes it safer to store sensitive values in x16 and x17 as oppos...
bool isCallingConvWin64(CallingConv::ID CC, bool IsVarArg) const
APInt bitcastToAPInt() const
Definition APFloat.h:1475
Class for arbitrary precision integers.
Definition APInt.h:78
uint64_t getZExtValue() const
Get zero extended value.
Definition APInt.h:1560
static LLVM_ABI APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
Definition APInt.cpp:648
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition APInt.h:235
unsigned countr_one() const
Count the number of trailing one bits.
Definition APInt.h:1676
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
size_t size() const
Get the array size.
Definition ArrayRef.h:141
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
bool isEquality() const
Determine if this is an equals/not equals predicate.
Definition InstrTypes.h:978
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition InstrTypes.h:743
@ ICMP_SLT
signed less than
Definition InstrTypes.h:769
@ ICMP_SLE
signed less or equal
Definition InstrTypes.h:770
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition InstrTypes.h:746
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
Definition InstrTypes.h:755
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition InstrTypes.h:744
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
Definition InstrTypes.h:745
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:764
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:763
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:767
@ FCMP_ULT
1 1 0 0 True if unordered or less than
Definition InstrTypes.h:754
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition InstrTypes.h:748
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition InstrTypes.h:751
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
Definition InstrTypes.h:752
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition InstrTypes.h:747
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition InstrTypes.h:749
@ ICMP_NE
not equal
Definition InstrTypes.h:762
@ ICMP_SGE
signed greater or equal
Definition InstrTypes.h:768
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
Definition InstrTypes.h:756
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:766
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
Definition InstrTypes.h:753
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition InstrTypes.h:750
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
Definition InstrTypes.h:890
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
Definition InstrTypes.h:852
bool isIntPredicate() const
Definition InstrTypes.h:846
bool isUnsigned() const
Definition InstrTypes.h:999
static LLVM_ABI Constant * getSplat(unsigned NumElts, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
const APFloat & getValue() const
Definition Constants.h:464
const APFloat & getValueAPF() const
Definition Constants.h:463
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
Definition Constants.h:174
unsigned getBitWidth() const
getBitWidth - Return the scalar bitwidth of this constant.
Definition Constants.h:162
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:168
static LLVM_ABI Constant * get(ArrayRef< Constant * > V)
This is an important base class in LLVM.
Definition Constant.h:44
TypeSize getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type.
Definition DataLayout.h:582
LLVM_ABI Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition Function.h:273
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition Function.cpp:352
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
Definition Function.h:230
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition Function.cpp:731
virtual void setupMF(MachineFunction &mf, GISelValueTracking *vt, CodeGenCoverage *covinfo=nullptr, ProfileSummaryInfo *psi=nullptr, BlockFrequencyInfo *bfi=nullptr)
Setup per-MF executor state.
Represents indexed stores.
Register getPointerReg() const
Get the source register of the pointer value.
MachineMemOperand & getMMO() const
Get the MachineMemOperand on this instruction.
LocationSize getMemSize() const
Returns the size in bytes of the memory access.
LocationSize getMemSizeInBits() const
Returns the size in bits of the memory access.
Represents a G_SELECT.
Register getCondReg() const
Register getFalseReg() const
Register getTrueReg() const
Register getReg(unsigned Idx) const
Access the Idx'th operand as a register and return it.
bool isThreadLocal() const
If the value is "Thread Local", its value isn't shared by the threads.
bool hasExternalWeakLinkage() const
bool isEquality() const
Return true if this predicate is either EQ or NE.
constexpr bool isScalableVector() const
Returns true if the LLT is a scalable vector.
constexpr unsigned getScalarSizeInBits() const
constexpr bool isScalar() const
LLT multiplyElements(int Factor) const
Produce a vector type that is Factor times bigger, preserving the element type.
constexpr LLT changeElementType(LLT NewEltTy) const
If this type is a vector, return a vector with the same number of elements but the new element type.
LLT getScalarType() const
constexpr bool isPointerVector() const
constexpr bool isInteger() const
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr bool isValid() const
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
constexpr bool isVector() const
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
constexpr bool isPointer() const
constexpr unsigned getAddressSpace() const
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
static LLT integer(unsigned SizeInBits)
constexpr TypeSize getSizeInBytes() const
Returns the total size of the type in bytes, i.e.
LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
TypeSize getValue() const
LLVM_ABI iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineInstrBundleIterator< MachineInstr > iterator
LLVM_ABI unsigned getConstantPoolIndex(const Constant *C, Align Alignment)
getConstantPoolIndex - Create a new entry in the constant pool or return an existing one.
void setFrameAddressIsTaken(bool T)
void setReturnAddressIsTaken(bool s)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
MachineConstantPool * getConstantPool()
getConstantPool - Return the constant pool object for the current function.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Helper class to build MachineInstr.
void setInsertPt(MachineBasicBlock &MBB, MachineBasicBlock::iterator II)
Set the insertion point before the specified position.
void setInstr(MachineInstr &MI)
Set the insertion point to before MI.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineFunction & getMF()
Getter for the function we currently build.
void setInstrAndDebugLoc(MachineInstr &MI)
Set the insertion point to before MI, and set the debug loc to MI's loc.
const MachineBasicBlock & getMBB() const
Getter for the basic block we currently build.
MachineRegisterInfo * getMRI()
Getter for MRI.
MachineIRBuilderState & getState()
Getter for the State.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
const DataLayout & getDataLayout() const
void setState(const MachineIRBuilderState &NewState)
Setter for the State.
MachineInstrBuilder buildPtrToInt(const DstOp &Dst, const SrcOp &Src)
Build and insert a G_PTRTOINT instruction.
Register getReg(unsigned Idx) const
Get the register for the operand index.
void constrainAllUses(const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI) const
const MachineInstrBuilder & setOperandDead(unsigned OpIdx) const
const MachineInstrBuilder & addUse(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addBlockAddress(const BlockAddress *BA, int64_t Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addRegMask(const uint32_t *Mask) const
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addJumpTableIndex(unsigned Idx, unsigned TargetFlags=0) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addDef(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register definition operand.
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
const MachineBasicBlock * getParent() const
bool isDebugInstr() const
LLVM_ABI void addOperand(MachineFunction &MF, const MachineOperand &Op)
Add the specified operand to the instruction.
void setImplicitPhysRegDefsDead()
Mark the implicit physreg defs named by the instruction description as dead.
LLVM_ABI const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
const MachineOperand & getOperand(unsigned i) const
LLVM_ABI MachineInstrBundleIterator< MachineInstr > eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
LLVM_ABI void addMemOperand(MachineFunction &MF, MachineMemOperand *MO)
Add a MachineMemOperand to the machine instruction.
LLT getMemoryType() const
Return the memory type of the memory reference.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
AtomicOrdering getSuccessOrdering() const
Return the atomic ordering requirements for this memory operation.
MachineOperand class - Representation of each machine instruction operand.
const GlobalValue * getGlobal() const
const ConstantInt * getCImm() const
bool isCImm() const
isCImm - Test if this is a MO_CImmediate operand.
int64_t getImm() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
LLVM_ABI void setReg(Register Reg)
Change the register this operand corresponds to.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
LLVM_ABI void ChangeToImmediate(int64_t ImmVal, unsigned TargetFlags=0)
ChangeToImmediate - Replace this operand with a new immediate operand of the specified value.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
static MachineOperand CreatePredicate(unsigned Pred)
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateGA(const GlobalValue *GV, int64_t Offset, unsigned TargetFlags=0)
static MachineOperand CreateBA(const BlockAddress *BA, int64_t Offset, unsigned TargetFlags=0)
const ConstantFP * getFPImm() const
unsigned getPredicate() const
int64_t getOffset() const
Return the offset from the symbol in this operand.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLVM_ABI bool hasOneNonDBGUse(Register RegNo) const
hasOneNonDBGUse - Return true if there is exactly one non-Debug use of the specified register.
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
LLVM_ABI LLVM_READONLY MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
bool use_nodbg_empty(Register RegNo) const
use_nodbg_empty - Return true if there are no non-Debug instructions using the specified register.
const RegClassOrRegBank & getRegClassOrRegBank(Register Reg) const
Return the register bank or register class of Reg.
LLVM_ABI Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
def_instr_iterator def_instr_begin(Register RegNo) const
LLT getType(Register Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register.
const RegisterBank * getRegBankOrNull(Register Reg) const
Return the register bank of Reg, or null if Reg has not been assigned a register bank or has been ass...
LLVM_ABI void setRegBank(Register Reg, const RegisterBank &RegBank)
Set the register bank to RegBank for Reg.
iterator_range< use_instr_nodbg_iterator > use_nodbg_instructions(Register Reg) const
LLVM_ABI void setType(Register VReg, LLT Ty)
Set the low-level type of VReg to Ty.
const MachineFunction & getMF() const
bool hasOneDef(Register RegNo) const
Return true if there is exactly one operand defining the specified register.
LLVM_ABI void setRegClass(Register Reg, const TargetRegisterClass *RC)
setRegClass - Set the register class of the specified virtual register.
LLVM_ABI Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
const TargetRegisterClass * getRegClassOrNull(Register Reg) const
Return the register class of Reg, or null if Reg has not been assigned a register class yet.
LLVM_ABI Register cloneVirtualRegister(Register VReg, StringRef Name="")
Create and return a new virtual register in the function with the same attributes as the given regist...
Analysis providing profile information.
Holds all the information related to register banks.
static const TargetRegisterClass * constrainGenericRegister(Register Reg, const TargetRegisterClass &RC, MachineRegisterInfo &MRI)
Constrain the (possibly generic) virtual register Reg to RC.
const RegisterBank & getRegBank(unsigned ID)
Get the register bank identified by ID.
This class implements the register bank concept.
unsigned getID() const
Get the identifier of this register bank.
Wrapper class representing virtual and physical registers.
Definition Register.h:20
constexpr bool isValid() const
Definition Register.h:112
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Definition Register.h:79
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition Register.h:83
void assign(size_type NumElts, ValueParamT Elt)
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
LLVM_NO_SANITIZE_CFI_UNRELATED_CAST iterator begin()
TargetInstrInfo - Interface to description of machine instruction set.
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize=false) const
Returns true if the specified FP immediate can be materialized without a load from the constant pool.
bool isPositionIndependent() const
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
TargetOptions Options
CodeModel::Model getCodeModel() const
Returns the code model.
unsigned TLSSize
Bit size of immediate TLS offsets (0 == use the default).
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const =0
Return the target's register information.
virtual const TargetLowering * getTargetLowering() const
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition TypeSize.h:352
static constexpr TypeSize getScalable(ScalarTy MinimumSize)
Definition TypeSize.h:355
Value * getOperand(unsigned i) const
Definition User.h:207
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
LLVM_ABI Align getPointerAlignment(const DataLayout &DL) const
Returns an alignment of the pointer value.
Definition Value.cpp:985
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition TypeSize.h:169
self_iterator getIterator()
Definition ilist_node.h:123
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
static CondCode getInvertedCondCode(CondCode Code)
static unsigned getNZCVToSatisfyCondCode(CondCode Code)
Given a condition code, return NZCV flags that would satisfy that condition.
void changeFCMPPredToAArch64CC(const CmpInst::Predicate P, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2)
Find the AArch64 condition codes necessary to represent P for a scalar floating point comparison.
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
@ MO_G1
MO_G1 - A symbol operand with this flag (granule 1) represents the bits 16-31 of a 64-bit address,...
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
@ MO_G0
MO_G0 - A symbol operand with this flag (granule 0) represents the bits 0-15 of a 64-bit address,...
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
@ MO_HI12
MO_HI12 - This flag indicates that a symbol operand represents the bits 13-24 of a 64-bit address,...
@ MO_TLS
MO_TLS - Indicates that the operand being accessed is some kind of thread-local symbol.
@ MO_G2
MO_G2 - A symbol operand with this flag (granule 2) represents the bits 32-47 of a 64-bit address,...
@ MO_G3
MO_G3 - A symbol operand with this flag (granule 3) represents the high 16-bits of a 64-bit address,...
static bool isLogicalImmediate(uint64_t imm, unsigned regSize)
isLogicalImmediate - Return true if the immediate is valid for a logical immediate instruction of the...
static int getFP32Imm(const APInt &Imm)
getFP32Imm - Return an 8-bit floating-point version of the 32-bit floating-point value.
static uint64_t encodeLogicalImmediate(uint64_t imm, unsigned regSize)
encodeLogicalImmediate - Return the encoded immediate value for a logical immediate instruction of th...
static int getFP64Imm(const APInt &Imm)
getFP64Imm - Return an 8-bit floating-point version of the 64-bit floating-point value.
static int getFP16Imm(const APInt &Imm)
getFP16Imm - Return an 8-bit floating-point version of the 16-bit floating-point value.
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
static bool isSignExtendShiftType(AArch64_AM::ShiftExtendType Type)
isSignExtendShiftType - Returns true if Type is sign extending.
bool expandVectorMOVImm(APInt Imm, SmallVectorImpl< ImmInsnModel > &Insn, const AArch64Subtarget &ST)
TLSModel::Model getELFTLSModel(const GlobalValue *GV, const AArch64TargetMachine &TM, bool HasELFSignedGOT)
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Attrs[]
Key for Kernel::Metadata::mAttrs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
operand_type_match m_Reg()
SpecificConstantMatch m_SpecificICst(const APInt &RequestedValue)
Matches a constant equal to RequestedValue.
UnaryOp_match< SrcTy, TargetOpcode::G_ZEXT > m_GZExt(const SrcTy &Src)
ConstantMatch< APInt > m_ICst(APInt &Cst)
BinaryOp_match< LHS, RHS, TargetOpcode::G_ADD, true > m_GAdd(const LHS &L, const RHS &R)
auto m_PosZeroFP()
Matches a floating-point positive zero.
BinaryOp_match< LHS, RHS, TargetOpcode::G_OR, true > m_GOr(const LHS &L, const RHS &R)
BinaryOp_match< SpecificConstantMatch, SrcTy, TargetOpcode::G_SUB > m_Neg(const SrcTy &&Src)
Matches a register negated by a G_SUB.
ICstOrSplatMatch< APInt > m_ICstOrSplat(APInt &Cst)
BinaryOp_match< SrcTy, SpecificConstantMatch, TargetOpcode::G_XOR, true > m_Not(const SrcTy &&Src)
Matches a register not-ed by a G_XOR.
BinaryOp_match< LHS, RHS, TargetOpcode::G_SUB > m_GSub(const LHS &L, const RHS &R)
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
BinaryOp_match< LHS, RHS, TargetOpcode::G_PTR_ADD, false > m_GPtrAdd(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_SHL, false > m_GShl(const LHS &L, const RHS &R)
Or< Preds... > m_any_of(Preds &&... preds)
BinaryOp_match< LHS, RHS, TargetOpcode::G_AND, true > m_GAnd(const LHS &L, const RHS &R)
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
Predicate
Predicate - These are "(BI << 5) | BO" for various predicates.
Predicate getPredicate(unsigned Condition, unsigned Hint)
Return predicate consisting of specified condition and hint bits.
constexpr double e
NodeAddr< InstrNode * > Instr
Definition RDFGraph.h:389
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI Register getFunctionLiveInPhysReg(MachineFunction &MF, const TargetInstrInfo &TII, MCRegister PhysReg, const TargetRegisterClass &RC, const DebugLoc &DL, LLT RegTy=LLT())
Return a virtual register corresponding to the incoming argument register PhysReg.
Definition Utils.cpp:896
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:316
@ Offset
Definition DWP.cpp:577
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
LLVM_ABI Register constrainOperandRegClass(const MachineFunction &MF, const TargetRegisterInfo &TRI, MachineRegisterInfo &MRI, const TargetInstrInfo &TII, const RegisterBankInfo &RBI, MachineInstr &InsertPt, const TargetRegisterClass &RegClass, MachineOperand &RegMO)
Constrain the Register operand OpIdx, so that it is now constrained to the TargetRegisterClass passed...
Definition Utils.cpp:64
LLVM_ABI MachineInstr * getOpcodeDef(unsigned Opcode, Register Reg, const MachineRegisterInfo &MRI)
See if Reg is defined by an single def instruction that is Opcode.
Definition Utils.cpp:679
PointerUnion< const TargetRegisterClass *, const RegisterBank * > RegClassOrRegBank
Convenient type to represent either a register class or a register bank.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
LLVM_ABI std::optional< APInt > getIConstantVRegVal(Register VReg, const MachineRegisterInfo &MRI)
If VReg is defined by a G_CONSTANT, return the corresponding value.
Definition Utils.cpp:301
unsigned CheckFixedPointOperandConstant(APFloat &FVal, unsigned RegWidth, bool isReciprocal)
@ Undef
Value of the register doesn't matter.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
bool isStrongerThanMonotonic(AtomicOrdering AO)
LLVM_ABI void constrainSelectedInstRegOperands(MachineInstr &I, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
Mutate the newly-selected instruction I to constrain its (possibly generic) virtual register operands...
Definition Utils.cpp:163
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
bool isPreISelGenericOpcode(unsigned Opcode)
Check whether the given Opcode is a generic opcode that is not supposed to appear after ISel.
unsigned getBLRCallOpcode(const MachineFunction &MF)
Return opcode to be used for indirect calls.
@ O1
Optimize quickly without destroying debuggability.
@ O0
Disable as many optimizations as possible.
LLVM_ABI MachineInstr * getDefIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the def instruction for Reg, folding away any trivial copies.
Definition Utils.cpp:520
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
LLVM_ABI std::optional< int64_t > getIConstantVRegSExtVal(Register VReg, const MachineRegisterInfo &MRI)
If VReg is defined by a G_CONSTANT fits in int64_t returns it.
Definition Utils.cpp:321
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
Definition MathExtras.h:274
InstructionSelector * createAArch64InstructionSelector(const AArch64TargetMachine &, const AArch64Subtarget &, const AArch64RegisterBankInfo &)
OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere.
Definition STLExtras.h:2042
constexpr bool has_single_bit(T Value) noexcept
Definition bit.h:149
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition MathExtras.h:326
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
LLVM_ABI std::optional< ValueAndVReg > getAnyConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true, bool LookThroughAnyExt=false)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT or G_FCONST...
Definition Utils.cpp:446
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
Definition MathExtras.h:190
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Sub
Subtraction of integers.
@ Add
Sum of integers.
DWARFExpression::Operation Op
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
LLVM_ABI std::optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...
Definition Utils.cpp:440
LLVM_ABI std::optional< DefinitionAndSourceRegister > getDefSrcRegIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the def instruction for Reg, and underlying value Register folding away any copies.
Definition Utils.cpp:495
LLVM_ABI std::optional< Register > getVectorSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI)
Definition Utils.cpp:1605
LLVM_ABI Register getSrcRegIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the source register for Reg, folding away any trivial copies.
Definition Utils.cpp:527
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
static EVT getFloatingPointVT(unsigned BitWidth)
Returns the EVT that represents a floating-point type with the given number of bits.
Definition ValueTypes.h:55
static LLVM_ABI MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.