Line data Source code
1 : //===- AArch64InstructionSelector.cpp ----------------------------*- C++ -*-==//
2 : //
3 : // The LLVM Compiler Infrastructure
4 : //
5 : // This file is distributed under the University of Illinois Open Source
6 : // License. See LICENSE.TXT for details.
7 : //
8 : //===----------------------------------------------------------------------===//
9 : /// \file
10 : /// This file implements the targeting of the InstructionSelector class for
11 : /// AArch64.
12 : /// \todo This should be generated by TableGen.
13 : //===----------------------------------------------------------------------===//
14 :
15 : #include "AArch64InstrInfo.h"
16 : #include "AArch64MachineFunctionInfo.h"
17 : #include "AArch64RegisterBankInfo.h"
18 : #include "AArch64RegisterInfo.h"
19 : #include "AArch64Subtarget.h"
20 : #include "AArch64TargetMachine.h"
21 : #include "MCTargetDesc/AArch64AddressingModes.h"
22 : #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
23 : #include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h"
24 : #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
25 : #include "llvm/CodeGen/GlobalISel/Utils.h"
26 : #include "llvm/CodeGen/MachineBasicBlock.h"
27 : #include "llvm/CodeGen/MachineFunction.h"
28 : #include "llvm/CodeGen/MachineInstr.h"
29 : #include "llvm/CodeGen/MachineInstrBuilder.h"
30 : #include "llvm/CodeGen/MachineOperand.h"
31 : #include "llvm/CodeGen/MachineRegisterInfo.h"
32 : #include "llvm/IR/Type.h"
33 : #include "llvm/Support/Debug.h"
34 : #include "llvm/Support/raw_ostream.h"
35 :
36 : #define DEBUG_TYPE "aarch64-isel"
37 :
38 : using namespace llvm;
39 :
40 : namespace {
41 :
42 : #define GET_GLOBALISEL_PREDICATE_BITSET
43 : #include "AArch64GenGlobalISel.inc"
44 : #undef GET_GLOBALISEL_PREDICATE_BITSET
45 :
46 : class AArch64InstructionSelector : public InstructionSelector {
47 : public:
48 : AArch64InstructionSelector(const AArch64TargetMachine &TM,
49 : const AArch64Subtarget &STI,
50 : const AArch64RegisterBankInfo &RBI);
51 :
52 : bool select(MachineInstr &I, CodeGenCoverage &CoverageInfo) const override;
53 : static const char *getName() { return DEBUG_TYPE; }
54 :
55 : private:
56 : /// tblgen-erated 'select' implementation, used as the initial selector for
57 : /// the patterns that don't require complex C++.
58 : bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
59 :
60 : bool selectVaStartAAPCS(MachineInstr &I, MachineFunction &MF,
61 : MachineRegisterInfo &MRI) const;
62 : bool selectVaStartDarwin(MachineInstr &I, MachineFunction &MF,
63 : MachineRegisterInfo &MRI) const;
64 :
65 : bool selectCompareBranch(MachineInstr &I, MachineFunction &MF,
66 : MachineRegisterInfo &MRI) const;
67 :
68 : ComplexRendererFns selectArithImmed(MachineOperand &Root) const;
69 :
70 : ComplexRendererFns selectAddrModeUnscaled(MachineOperand &Root,
71 : unsigned Size) const;
72 :
73 0 : ComplexRendererFns selectAddrModeUnscaled8(MachineOperand &Root) const {
74 0 : return selectAddrModeUnscaled(Root, 1);
75 : }
76 0 : ComplexRendererFns selectAddrModeUnscaled16(MachineOperand &Root) const {
77 0 : return selectAddrModeUnscaled(Root, 2);
78 : }
79 0 : ComplexRendererFns selectAddrModeUnscaled32(MachineOperand &Root) const {
80 0 : return selectAddrModeUnscaled(Root, 4);
81 : }
82 0 : ComplexRendererFns selectAddrModeUnscaled64(MachineOperand &Root) const {
83 0 : return selectAddrModeUnscaled(Root, 8);
84 : }
85 0 : ComplexRendererFns selectAddrModeUnscaled128(MachineOperand &Root) const {
86 0 : return selectAddrModeUnscaled(Root, 16);
87 : }
88 :
89 : ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root,
90 : unsigned Size) const;
91 : template <int Width>
92 297 : ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root) const {
93 297 : return selectAddrModeIndexed(Root, Width / 8);
94 : }
95 4 :
96 4 : void renderTruncImm(MachineInstrBuilder &MIB, const MachineInstr &MI) const;
97 :
98 218 : // Materialize a GlobalValue or BlockAddress using a movz+movk sequence.
99 218 : void materializeLargeCMVal(MachineInstr &I, const Value *V,
100 : unsigned char OpFlags) const;
101 58 :
102 58 : const AArch64TargetMachine &TM;
103 : const AArch64Subtarget &STI;
104 7 : const AArch64InstrInfo &TII;
105 7 : const AArch64RegisterInfo &TRI;
106 : const AArch64RegisterBankInfo &RBI;
107 10 :
108 10 : #define GET_GLOBALISEL_PREDICATES_DECL
109 : #include "AArch64GenGlobalISel.inc"
110 : #undef GET_GLOBALISEL_PREDICATES_DECL
111 :
112 : // We declare the temporaries used by selectImpl() in the class to minimize the
113 : // cost of constructing placeholder values.
114 : #define GET_GLOBALISEL_TEMPORARIES_DECL
115 : #include "AArch64GenGlobalISel.inc"
116 : #undef GET_GLOBALISEL_TEMPORARIES_DECL
117 : };
118 :
119 : } // end anonymous namespace
120 :
121 : #define GET_GLOBALISEL_IMPL
122 : #include "AArch64GenGlobalISel.inc"
123 : #undef GET_GLOBALISEL_IMPL
124 :
125 : AArch64InstructionSelector::AArch64InstructionSelector(
126 : const AArch64TargetMachine &TM, const AArch64Subtarget &STI,
127 : const AArch64RegisterBankInfo &RBI)
128 : : InstructionSelector(), TM(TM), STI(STI), TII(*STI.getInstrInfo()),
129 : TRI(*STI.getRegisterInfo()), RBI(RBI),
130 : #define GET_GLOBALISEL_PREDICATES_INIT
131 : #include "AArch64GenGlobalISel.inc"
132 : #undef GET_GLOBALISEL_PREDICATES_INIT
133 : #define GET_GLOBALISEL_TEMPORARIES_INIT
134 : #include "AArch64GenGlobalISel.inc"
135 : #undef GET_GLOBALISEL_TEMPORARIES_INIT
136 : {
137 : }
138 :
139 : // FIXME: This should be target-independent, inferred from the types declared
140 1570 : // for each class in the bank.
141 : static const TargetRegisterClass *
142 1570 : getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB,
143 : const RegisterBankInfo &RBI,
144 : bool GetAllRegSet = false) {
145 : if (RB.getID() == AArch64::GPRRegBankID) {
146 : if (Ty.getSizeInBits() <= 32)
147 : return GetAllRegSet ? &AArch64::GPR32allRegClass
148 : : &AArch64::GPR32RegClass;
149 : if (Ty.getSizeInBits() == 64)
150 : return GetAllRegSet ? &AArch64::GPR64allRegClass
151 : : &AArch64::GPR64RegClass;
152 1570 : return nullptr;
153 : }
154 :
155 : if (RB.getID() == AArch64::FPRRegBankID) {
156 : if (Ty.getSizeInBits() <= 16)
157 909 : return &AArch64::FPR16RegClass;
158 : if (Ty.getSizeInBits() == 32)
159 : return &AArch64::FPR32RegClass;
160 909 : if (Ty.getSizeInBits() == 64)
161 534 : return &AArch64::FPR64RegClass;
162 293 : if (Ty.getSizeInBits() == 128)
163 293 : return &AArch64::FPR128RegClass;
164 241 : return nullptr;
165 241 : }
166 241 :
167 : return nullptr;
168 : }
169 :
170 375 : /// Check whether \p I is a currently unsupported binary operation:
171 375 : /// - it has an unsized type
172 : /// - an operand is not a vreg
173 363 : /// - all operands are not in the same bank
174 : /// These are checks that should someday live in the verifier, but right now,
175 303 : /// these are mostly limitations of the aarch64 selector.
176 : static bool unsupportedBinOp(const MachineInstr &I,
177 116 : const AArch64RegisterBankInfo &RBI,
178 : const MachineRegisterInfo &MRI,
179 0 : const AArch64RegisterInfo &TRI) {
180 : LLT Ty = MRI.getType(I.getOperand(0).getReg());
181 : if (!Ty.isValid()) {
182 : LLVM_DEBUG(dbgs() << "Generic binop register should be typed\n");
183 : return true;
184 : }
185 :
186 : const RegisterBank *PrevOpBank = nullptr;
187 : for (auto &MO : I.operands()) {
188 : // FIXME: Support non-register operands.
189 : if (!MO.isReg()) {
190 : LLVM_DEBUG(dbgs() << "Generic inst non-reg operands are unsupported\n");
191 16 : return true;
192 : }
193 :
194 : // FIXME: Can generic operations have physical registers operands? If
195 16 : // so, this will need to be taught about that, and we'll need to get the
196 16 : // bank out of the minimal class for the register.
197 : // Either way, this needs to be documented (and possibly verified).
198 : if (!TargetRegisterInfo::isVirtualRegister(MO.getReg())) {
199 : LLVM_DEBUG(dbgs() << "Generic inst has physical register operand\n");
200 : return true;
201 : }
202 64 :
203 : const RegisterBank *OpBank = RBI.getRegBank(MO.getReg(), MRI, TRI);
204 48 : if (!OpBank) {
205 : LLVM_DEBUG(dbgs() << "Generic register has no bank or class\n");
206 : return true;
207 : }
208 :
209 : if (PrevOpBank && OpBank != PrevOpBank) {
210 : LLVM_DEBUG(dbgs() << "Generic inst operands have different banks\n");
211 : return true;
212 : }
213 96 : PrevOpBank = OpBank;
214 : }
215 : return false;
216 : }
217 :
218 48 : /// Select the AArch64 opcode for the basic binary operation \p GenericOpc
219 48 : /// (such as G_OR or G_SDIV), appropriate for the register bank \p RegBankID
220 : /// and of size \p OpSize.
221 : /// \returns \p GenericOpc if the combination is unsupported.
222 : static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID,
223 : unsigned OpSize) {
224 48 : switch (RegBankID) {
225 : case AArch64::GPRRegBankID:
226 : if (OpSize == 32) {
227 : switch (GenericOpc) {
228 : case TargetOpcode::G_SHL:
229 : return AArch64::LSLVWr;
230 : case TargetOpcode::G_LSHR:
231 : return AArch64::LSRVWr;
232 : case TargetOpcode::G_ASHR:
233 : return AArch64::ASRVWr;
234 : default:
235 : return GenericOpc;
236 : }
237 14 : } else if (OpSize == 64) {
238 : switch (GenericOpc) {
239 14 : case TargetOpcode::G_GEP:
240 14 : return AArch64::ADDXrr;
241 14 : case TargetOpcode::G_SHL:
242 4 : return AArch64::LSLVXr;
243 : case TargetOpcode::G_LSHR:
244 : return AArch64::LSRVXr;
245 1 : case TargetOpcode::G_ASHR:
246 1 : return AArch64::ASRVXr;
247 1 : default:
248 1 : return GenericOpc;
249 0 : }
250 0 : }
251 : break;
252 10 : case AArch64::FPRRegBankID:
253 10 : switch (OpSize) {
254 : case 32:
255 : switch (GenericOpc) {
256 0 : case TargetOpcode::G_FADD:
257 0 : return AArch64::FADDSrr;
258 0 : case TargetOpcode::G_FSUB:
259 0 : return AArch64::FSUBSrr;
260 0 : case TargetOpcode::G_FMUL:
261 0 : return AArch64::FMULSrr;
262 0 : case TargetOpcode::G_FDIV:
263 0 : return AArch64::FDIVSrr;
264 : default:
265 : return GenericOpc;
266 : }
267 0 : case 64:
268 : switch (GenericOpc) {
269 0 : case TargetOpcode::G_FADD:
270 0 : return AArch64::FADDDrr;
271 : case TargetOpcode::G_FSUB:
272 : return AArch64::FSUBDrr;
273 0 : case TargetOpcode::G_FMUL:
274 0 : return AArch64::FMULDrr;
275 0 : case TargetOpcode::G_FDIV:
276 0 : return AArch64::FDIVDrr;
277 0 : case TargetOpcode::G_OR:
278 0 : return AArch64::ORRv8i8;
279 0 : default:
280 0 : return GenericOpc;
281 : }
282 0 : }
283 0 : break;
284 : }
285 : return GenericOpc;
286 0 : }
287 0 :
288 0 : /// Select the AArch64 opcode for the G_LOAD or G_STORE operation \p GenericOpc,
289 0 : /// appropriate for the (value) register bank \p RegBankID and of memory access
290 0 : /// size \p OpSize. This returns the variant with the base+unsigned-immediate
291 0 : /// addressing mode (e.g., LDRXui).
292 0 : /// \returns \p GenericOpc if the combination is unsupported.
293 0 : static unsigned selectLoadStoreUIOp(unsigned GenericOpc, unsigned RegBankID,
294 0 : unsigned OpSize) {
295 0 : const bool isStore = GenericOpc == TargetOpcode::G_STORE;
296 : switch (RegBankID) {
297 : case AArch64::GPRRegBankID:
298 : switch (OpSize) {
299 : case 8:
300 : return isStore ? AArch64::STRBBui : AArch64::LDRBBui;
301 : case 16:
302 : return isStore ? AArch64::STRHHui : AArch64::LDRHHui;
303 : case 32:
304 : return isStore ? AArch64::STRWui : AArch64::LDRWui;
305 : case 64:
306 : return isStore ? AArch64::STRXui : AArch64::LDRXui;
307 : }
308 76 : break;
309 : case AArch64::FPRRegBankID:
310 : switch (OpSize) {
311 76 : case 8:
312 73 : return isStore ? AArch64::STRBui : AArch64::LDRBui;
313 : case 16:
314 45 : return isStore ? AArch64::STRHui : AArch64::LDRHui;
315 45 : case 32:
316 7 : return isStore ? AArch64::STRSui : AArch64::LDRSui;
317 7 : case 64:
318 0 : return isStore ? AArch64::STRDui : AArch64::LDRDui;
319 0 : }
320 21 : break;
321 21 : }
322 : return GenericOpc;
323 : }
324 3 :
325 : static bool selectFP16CopyFromGPR32(MachineInstr &I, const TargetInstrInfo &TII,
326 3 : MachineRegisterInfo &MRI, unsigned SrcReg) {
327 3 : // Copies from gpr32 to fpr16 need to use a sub-register copy.
328 0 : unsigned CopyReg = MRI.createVirtualRegister(&AArch64::FPR32RegClass);
329 0 : BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::COPY))
330 0 : .addDef(CopyReg)
331 0 : .addUse(SrcReg);
332 0 : unsigned SubRegCopy = MRI.createVirtualRegister(&AArch64::FPR16RegClass);
333 0 : BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY))
334 : .addDef(SubRegCopy)
335 : .addUse(CopyReg, 0, AArch64::hsub);
336 :
337 : MachineOperand &RegOp = I.getOperand(1);
338 : RegOp.setReg(SubRegCopy);
339 : return true;
340 4 : }
341 :
342 : static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII,
343 4 : MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
344 12 : const RegisterBankInfo &RBI) {
345 :
346 : unsigned DstReg = I.getOperand(0).getReg();
347 4 : unsigned SrcReg = I.getOperand(1).getReg();
348 8 :
349 : if (TargetRegisterInfo::isPhysicalRegister(DstReg)) {
350 : if (TRI.getRegClass(AArch64::FPR16RegClassID)->contains(DstReg) &&
351 : !TargetRegisterInfo::isPhysicalRegister(SrcReg)) {
352 4 : const RegisterBank &RegBank = *RBI.getRegBank(SrcReg, MRI, TRI);
353 4 : const TargetRegisterClass *SrcRC = getRegClassForTypeOnBank(
354 4 : MRI.getType(SrcReg), RegBank, RBI, /* GetAllRegSet */ true);
355 : if (SrcRC == &AArch64::GPR32allRegClass)
356 : return selectFP16CopyFromGPR32(I, TII, MRI, SrcReg);
357 955 : }
358 : assert(I.isCopy() && "Generic operators do not allow physical registers");
359 : return true;
360 : }
361 955 :
362 955 : const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI);
363 : const unsigned DstSize = MRI.getType(DstReg).getSizeInBits();
364 955 : (void)DstSize;
365 524 : const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI);
366 : (void)SrcSize;
367 8 : assert((!TargetRegisterInfo::isPhysicalRegister(SrcReg) || I.isCopy()) &&
368 8 : "No phys reg on generic operators");
369 : assert(
370 8 : (DstSize == SrcSize ||
371 3 : // Copies are a mean to setup initial types, the number of
372 : // bits may not exactly match.
373 : (TargetRegisterInfo::isPhysicalRegister(SrcReg) &&
374 259 : DstSize <= RBI.getSizeInBits(SrcReg, MRI, TRI)) ||
375 : // Copies are a mean to copy bits around, as long as we are
376 : // on the same register class, that's fine. Otherwise, that
377 693 : // means we need some SUBREG_TO_REG or AND & co.
378 693 : (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) &&
379 : "Copy with different width?!");
380 693 : assert((DstSize <= 64 || RegBank.getID() == AArch64::FPRRegBankID) &&
381 : "GPRs cannot get more than 64-bit width values");
382 :
383 : const TargetRegisterClass *RC = getRegClassForTypeOnBank(
384 : MRI.getType(DstReg), RegBank, RBI, /* GetAllRegSet */ true);
385 : if (!RC) {
386 : LLVM_DEBUG(dbgs() << "Unexpected bitcast size " << DstSize << '\n');
387 : return false;
388 : }
389 :
390 : if (!TargetRegisterInfo::isPhysicalRegister(SrcReg)) {
391 : const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(SrcReg);
392 : const TargetRegisterClass *SrcRC =
393 : RegClassOrBank.dyn_cast<const TargetRegisterClass *>();
394 : const RegisterBank *RB = nullptr;
395 : if (!SrcRC) {
396 : RB = RegClassOrBank.get<const RegisterBank *>();
397 : SrcRC = getRegClassForTypeOnBank(MRI.getType(SrcReg), *RB, RBI, true);
398 693 : }
399 : // Copies from fpr16 to gpr32 need to use SUBREG_TO_REG.
400 693 : if (RC == &AArch64::GPR32allRegClass && SrcRC == &AArch64::FPR16RegClass) {
401 : unsigned PromoteReg = MRI.createVirtualRegister(&AArch64::FPR32RegClass);
402 : BuildMI(*I.getParent(), I, I.getDebugLoc(),
403 : TII.get(AArch64::SUBREG_TO_REG))
404 : .addDef(PromoteReg)
405 693 : .addImm(0)
406 : .addUse(SrcReg)
407 : .addImm(AArch64::hsub);
408 : MachineOperand &RegOp = I.getOperand(1);
409 : RegOp.setReg(PromoteReg);
410 2 : } else if (RC == &AArch64::FPR16RegClass &&
411 : SrcRC == &AArch64::GPR32allRegClass) {
412 82 : selectFP16CopyFromGPR32(I, TII, MRI, SrcReg);
413 : }
414 : }
415 84 :
416 2 : // No need to constrain SrcReg. It will get constrained when
417 4 : // we hit another of its use or its defs.
418 4 : // Copies do not have constraints.
419 : if (!RBI.constrainGenericRegister(DstReg, *RC, MRI)) {
420 : LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
421 : << " operand\n");
422 : return false;
423 2 : }
424 2 : I.setDesc(TII.get(AArch64::COPY));
425 82 : return true;
426 : }
427 1 :
428 : static unsigned selectFPConvOpc(unsigned GenericOpc, LLT DstTy, LLT SrcTy) {
429 : if (!DstTy.isScalar() || !SrcTy.isScalar())
430 : return GenericOpc;
431 :
432 : const unsigned DstSize = DstTy.getSizeInBits();
433 : const unsigned SrcSize = SrcTy.getSizeInBits();
434 693 :
435 : switch (DstSize) {
436 : case 32:
437 : switch (SrcSize) {
438 : case 32:
439 693 : switch (GenericOpc) {
440 693 : case TargetOpcode::G_SITOFP:
441 : return AArch64::SCVTFUWSri;
442 : case TargetOpcode::G_UITOFP:
443 0 : return AArch64::UCVTFUWSri;
444 0 : case TargetOpcode::G_FPTOSI:
445 0 : return AArch64::FCVTZSUWSr;
446 : case TargetOpcode::G_FPTOUI:
447 0 : return AArch64::FCVTZUUWSr;
448 0 : default:
449 : return GenericOpc;
450 0 : }
451 0 : case 64:
452 0 : switch (GenericOpc) {
453 0 : case TargetOpcode::G_SITOFP:
454 : return AArch64::SCVTFUXSri;
455 : case TargetOpcode::G_UITOFP:
456 : return AArch64::UCVTFUXSri;
457 : case TargetOpcode::G_FPTOSI:
458 : return AArch64::FCVTZSUWDr;
459 : case TargetOpcode::G_FPTOUI:
460 : return AArch64::FCVTZUUWDr;
461 : default:
462 : return GenericOpc;
463 : }
464 : default:
465 : return GenericOpc;
466 0 : }
467 : case 64:
468 : switch (SrcSize) {
469 : case 32:
470 : switch (GenericOpc) {
471 : case TargetOpcode::G_SITOFP:
472 : return AArch64::SCVTFUWDri;
473 : case TargetOpcode::G_UITOFP:
474 : return AArch64::UCVTFUWDri;
475 : case TargetOpcode::G_FPTOSI:
476 : return AArch64::FCVTZSUXSr;
477 : case TargetOpcode::G_FPTOUI:
478 : return AArch64::FCVTZUUXSr;
479 : default:
480 : return GenericOpc;
481 : }
482 0 : case 64:
483 0 : switch (GenericOpc) {
484 0 : case TargetOpcode::G_SITOFP:
485 : return AArch64::SCVTFUXDri;
486 : case TargetOpcode::G_UITOFP:
487 : return AArch64::UCVTFUXDri;
488 : case TargetOpcode::G_FPTOSI:
489 : return AArch64::FCVTZSUXDr;
490 : case TargetOpcode::G_FPTOUI:
491 : return AArch64::FCVTZUUXDr;
492 : default:
493 : return GenericOpc;
494 : }
495 : default:
496 : return GenericOpc;
497 0 : }
498 : default:
499 : return GenericOpc;
500 : };
501 : return GenericOpc;
502 : }
503 :
504 : static AArch64CC::CondCode changeICMPPredToAArch64CC(CmpInst::Predicate P) {
505 : switch (P) {
506 : default:
507 : llvm_unreachable("Unknown condition code!");
508 : case CmpInst::ICMP_NE:
509 : return AArch64CC::NE;
510 : case CmpInst::ICMP_EQ:
511 : return AArch64CC::EQ;
512 : case CmpInst::ICMP_SGT:
513 : return AArch64CC::GT;
514 : case CmpInst::ICMP_SGE:
515 : return AArch64CC::GE;
516 : case CmpInst::ICMP_SLT:
517 : return AArch64CC::LT;
518 : case CmpInst::ICMP_SLE:
519 : return AArch64CC::LE;
520 : case CmpInst::ICMP_UGT:
521 0 : return AArch64CC::HI;
522 0 : case CmpInst::ICMP_UGE:
523 : return AArch64CC::HS;
524 : case CmpInst::ICMP_ULT:
525 : return AArch64CC::LO;
526 : case CmpInst::ICMP_ULE:
527 : return AArch64CC::LS;
528 : }
529 : }
530 :
531 : static void changeFCMPPredToAArch64CC(CmpInst::Predicate P,
532 : AArch64CC::CondCode &CondCode,
533 : AArch64CC::CondCode &CondCode2) {
534 : CondCode2 = AArch64CC::AL;
535 : switch (P) {
536 : default:
537 : llvm_unreachable("Unknown FP condition!");
538 : case CmpInst::FCMP_OEQ:
539 : CondCode = AArch64CC::EQ;
540 : break;
541 : case CmpInst::FCMP_OGT:
542 : CondCode = AArch64CC::GT;
543 : break;
544 : case CmpInst::FCMP_OGE:
545 : CondCode = AArch64CC::GE;
546 6 : break;
547 : case CmpInst::FCMP_OLT:
548 : CondCode = AArch64CC::MI;
549 6 : break;
550 6 : case CmpInst::FCMP_OLE:
551 0 : CondCode = AArch64CC::LS;
552 0 : break;
553 0 : case CmpInst::FCMP_ONE:
554 0 : CondCode = AArch64CC::MI;
555 0 : CondCode2 = AArch64CC::GT;
556 0 : break;
557 0 : case CmpInst::FCMP_ORD:
558 0 : CondCode = AArch64CC::VC;
559 0 : break;
560 0 : case CmpInst::FCMP_UNO:
561 0 : CondCode = AArch64CC::VS;
562 0 : break;
563 0 : case CmpInst::FCMP_UEQ:
564 0 : CondCode = AArch64CC::EQ;
565 0 : CondCode2 = AArch64CC::VS;
566 0 : break;
567 0 : case CmpInst::FCMP_UGT:
568 3 : CondCode = AArch64CC::HI;
569 3 : break;
570 3 : case CmpInst::FCMP_UGE:
571 3 : CondCode = AArch64CC::PL;
572 0 : break;
573 0 : case CmpInst::FCMP_ULT:
574 0 : CondCode = AArch64CC::LT;
575 0 : break;
576 0 : case CmpInst::FCMP_ULE:
577 0 : CondCode = AArch64CC::LE;
578 0 : break;
579 0 : case CmpInst::FCMP_UNE:
580 0 : CondCode = AArch64CC::NE;
581 0 : break;
582 0 : }
583 0 : }
584 0 :
585 3 : bool AArch64InstructionSelector::selectCompareBranch(
586 3 : MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
587 3 :
588 0 : const unsigned CondReg = I.getOperand(0).getReg();
589 0 : MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
590 0 : MachineInstr *CCMI = MRI.getVRegDef(CondReg);
591 0 : if (CCMI->getOpcode() == TargetOpcode::G_TRUNC)
592 0 : CCMI = MRI.getVRegDef(CCMI->getOperand(1).getReg());
593 0 : if (CCMI->getOpcode() != TargetOpcode::G_ICMP)
594 0 : return false;
595 0 :
596 0 : unsigned LHS = CCMI->getOperand(2).getReg();
597 : unsigned RHS = CCMI->getOperand(3).getReg();
598 6 : if (!getConstantVRegVal(RHS, MRI))
599 : std::swap(RHS, LHS);
600 0 :
601 : const auto RHSImm = getConstantVRegVal(RHS, MRI);
602 : if (!RHSImm || *RHSImm != 0)
603 0 : return false;
604 0 :
605 0 : const RegisterBank &RB = *RBI.getRegBank(LHS, MRI, TRI);
606 0 : if (RB.getID() != AArch64::GPRRegBankID)
607 0 : return false;
608 0 :
609 0 : const auto Pred = (CmpInst::Predicate)CCMI->getOperand(1).getPredicate();
610 : if (Pred != CmpInst::ICMP_NE && Pred != CmpInst::ICMP_EQ)
611 0 : return false;
612 0 :
613 0 : const unsigned CmpWidth = MRI.getType(LHS).getSizeInBits();
614 : unsigned CBOpc = 0;
615 : if (CmpWidth <= 32)
616 0 : CBOpc = (Pred == CmpInst::ICMP_EQ ? AArch64::CBZW : AArch64::CBNZW);
617 0 : else if (CmpWidth == 64)
618 0 : CBOpc = (Pred == CmpInst::ICMP_EQ ? AArch64::CBZX : AArch64::CBNZX);
619 : else
620 0 : return false;
621 0 :
622 0 : BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CBOpc))
623 : .addUse(LHS)
624 0 : .addMBB(DestMBB)
625 0 : .constrainAllUses(TII, TRI, RBI);
626 0 :
627 : I.eraseFromParent();
628 0 : return true;
629 : }
630 0 :
631 0 : bool AArch64InstructionSelector::selectVaStartAAPCS(
632 0 : MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
633 0 : return false;
634 : }
635 0 :
636 : bool AArch64InstructionSelector::selectVaStartDarwin(
637 0 : MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
638 : AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
639 : unsigned ListReg = I.getOperand(0).getReg();
640 0 :
641 : unsigned ArgsAddrReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
642 0 :
643 0 : auto MIB =
644 : BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::ADDXri))
645 : .addDef(ArgsAddrReg)
646 0 : .addFrameIndex(FuncInfo->getVarArgsStackIndex())
647 : .addImm(0)
648 0 : .addImm(0);
649 :
650 : constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
651 1 :
652 : MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::STRXui))
653 1 : .addUse(ArgsAddrReg)
654 1 : .addUse(ListReg)
655 : .addImm(0)
656 1 : .addMemOperand(*I.memoperands_begin());
657 :
658 : constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
659 2 : I.eraseFromParent();
660 : return true;
661 1 : }
662 :
663 1 : void AArch64InstructionSelector::materializeLargeCMVal(
664 : MachineInstr &I, const Value *V, unsigned char OpFlags) const {
665 1 : MachineBasicBlock &MBB = *I.getParent();
666 : MachineFunction &MF = *MBB.getParent();
667 2 : MachineRegisterInfo &MRI = MF.getRegInfo();
668 : MachineIRBuilder MIB(I);
669 :
670 : auto MovZ = MIB.buildInstr(AArch64::MOVZXi, &AArch64::GPR64RegClass);
671 1 : MovZ->addOperand(MF, I.getOperand(1));
672 : MovZ->getOperand(1).setTargetFlags(OpFlags | AArch64II::MO_G0 |
673 1 : AArch64II::MO_NC);
674 1 : MovZ->addOperand(MF, MachineOperand::CreateImm(0));
675 1 : constrainSelectedInstRegOperands(*MovZ, TII, TRI, RBI);
676 :
677 : auto BuildMovK = [&](unsigned SrcReg, unsigned char Flags, unsigned Offset,
678 4 : unsigned ForceDstReg) {
679 : unsigned DstReg = ForceDstReg
680 4 : ? ForceDstReg
681 4 : : MRI.createVirtualRegister(&AArch64::GPR64RegClass);
682 4 : auto MovI = MIB.buildInstr(AArch64::MOVKXi).addDef(DstReg).addUse(SrcReg);
683 : if (auto *GV = dyn_cast<GlobalValue>(V)) {
684 : MovI->addOperand(MF, MachineOperand::CreateGA(
685 4 : GV, MovZ->getOperand(1).getOffset(), Flags));
686 8 : } else {
687 4 : MovI->addOperand(
688 : MF, MachineOperand::CreateBA(cast<BlockAddress>(V),
689 4 : MovZ->getOperand(1).getOffset(), Flags));
690 4 : }
691 : MovI->addOperand(MF, MachineOperand::CreateImm(Offset));
692 : constrainSelectedInstRegOperands(*MovI, TII, TRI, RBI);
693 : return DstReg;
694 : };
695 : unsigned DstReg = BuildMovK(MovZ->getOperand(0).getReg(),
696 : AArch64II::MO_G1 | AArch64II::MO_NC, 16, 0);
697 : DstReg = BuildMovK(DstReg, AArch64II::MO_G2 | AArch64II::MO_NC, 32, 0);
698 : BuildMovK(DstReg, AArch64II::MO_G3, 48, I.getOperand(0).getReg());
699 : return;
700 : }
701 :
702 : bool AArch64InstructionSelector::select(MachineInstr &I,
703 : CodeGenCoverage &CoverageInfo) const {
704 : assert(I.getParent() && "Instruction should be in a basic block!");
705 : assert(I.getParent()->getParent() && "Instruction should be in a function!");
706 :
707 : MachineBasicBlock &MBB = *I.getParent();
708 : MachineFunction &MF = *MBB.getParent();
709 4 : MachineRegisterInfo &MRI = MF.getRegInfo();
710 4 :
711 : unsigned Opcode = I.getOpcode();
712 4 : // G_PHI requires same handling as PHI
713 4 : if (!isPreISelGenericOpcode(Opcode) || Opcode == TargetOpcode::G_PHI) {
714 4 : // Certain non-generic instructions also need some special handling.
715 :
716 : if (Opcode == TargetOpcode::LOAD_STACK_GUARD)
717 2311 : return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
718 :
719 : if (Opcode == TargetOpcode::PHI || Opcode == TargetOpcode::G_PHI) {
720 : const unsigned DefReg = I.getOperand(0).getReg();
721 : const LLT DefTy = MRI.getType(DefReg);
722 2311 :
723 2311 : const TargetRegisterClass *DefRC = nullptr;
724 2311 : if (TargetRegisterInfo::isPhysicalRegister(DefReg)) {
725 : DefRC = TRI.getRegClass(DefReg);
726 2311 : } else {
727 : const RegClassOrRegBank &RegClassOrBank =
728 2311 : MRI.getRegClassOrRegBank(DefReg);
729 :
730 : DefRC = RegClassOrBank.dyn_cast<const TargetRegisterClass *>();
731 1229 : if (!DefRC) {
732 0 : if (!DefTy.isValid()) {
733 : LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
734 1229 : return false;
735 9 : }
736 : const RegisterBank &RB = *RegClassOrBank.get<const RegisterBank *>();
737 : DefRC = getRegClassForTypeOnBank(DefTy, RB, RBI);
738 : if (!DefRC) {
739 9 : LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
740 0 : return false;
741 : }
742 : }
743 : }
744 : I.setDesc(TII.get(TargetOpcode::PHI));
745 :
746 2 : return RBI.constrainGenericRegister(DefReg, *DefRC, MRI);
747 7 : }
748 :
749 : if (I.isCopy())
750 : return selectCopy(I, TII, MRI, TRI, RBI);
751 :
752 7 : return true;
753 7 : }
754 :
755 :
756 : if (I.getNumOperands() != I.getNumExplicitOperands()) {
757 : LLVM_DEBUG(
758 : dbgs() << "Generic instruction has unexpected implicit operands\n");
759 9 : return false;
760 : }
761 9 :
762 : if (selectImpl(I, CoverageInfo))
763 : return true;
764 1220 :
765 921 : LLT Ty =
766 : I.getOperand(0).isReg() ? MRI.getType(I.getOperand(0).getReg()) : LLT{};
767 :
768 : switch (Opcode) {
769 : case TargetOpcode::G_BRCOND: {
770 : if (Ty.getSizeInBits() > 32) {
771 1082 : // We shouldn't need this on AArch64, but it would be implemented as an
772 : // EXTRACT_SUBREG followed by a TBNZW because TBNZX has no encoding if the
773 : // bit being tested is < 32.
774 : LLVM_DEBUG(dbgs() << "G_BRCOND has type: " << Ty
775 : << ", expected at most 32-bits");
776 : return false;
777 1082 : }
778 :
779 : const unsigned CondReg = I.getOperand(0).getReg();
780 : MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
781 1172 :
782 : if (selectCompareBranch(I, MF, MRI))
783 391 : return true;
784 17 :
785 17 : auto MIB = BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::TBNZW))
786 : .addUse(CondReg)
787 : .addImm(/*bit offset=*/0)
788 : .addMBB(DestMBB);
789 :
790 : I.eraseFromParent();
791 : return constrainSelectedInstRegOperands(*MIB.getInstr(), TII, TRI, RBI);
792 : }
793 :
794 17 : case TargetOpcode::G_BRINDIRECT: {
795 17 : I.setDesc(TII.get(AArch64::BR));
796 : return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
797 17 : }
798 :
799 : case TargetOpcode::G_FCONSTANT:
800 26 : case TargetOpcode::G_CONSTANT: {
801 : const bool isFP = Opcode == TargetOpcode::G_FCONSTANT;
802 :
803 13 : const LLT s32 = LLT::scalar(32);
804 : const LLT s64 = LLT::scalar(64);
805 13 : const LLT p0 = LLT::pointer(0, 64);
806 13 :
807 : const unsigned DefReg = I.getOperand(0).getReg();
808 : const LLT DefTy = MRI.getType(DefReg);
809 4 : const unsigned DefSize = DefTy.getSizeInBits();
810 4 : const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
811 4 :
812 : // FIXME: Redundant check, but even less readable when factored out.
813 : if (isFP) {
814 28 : if (Ty != s32 && Ty != s64) {
815 : LLVM_DEBUG(dbgs() << "Unable to materialize FP " << Ty
816 : << " constant, expected: " << s32 << " or " << s64
817 : << '\n');
818 : return false;
819 : }
820 :
821 : if (RB.getID() != AArch64::FPRRegBankID) {
822 28 : LLVM_DEBUG(dbgs() << "Unable to materialize FP " << Ty
823 28 : << " constant on bank: " << RB
824 28 : << ", expected: FPR\n");
825 28 : return false;
826 : }
827 :
828 28 : // The case when we have 0.0 is covered by tablegen. Reject it here so we
829 42 : // can be sure tablegen works correctly and isn't rescued by this code.
830 : if (I.getOperand(1).getFPImm()->getValueAPF().isExactlyValue(0.0))
831 : return false;
832 : } else {
833 0 : // s32 and s64 are covered by tablegen.
834 : if (Ty != p0) {
835 : LLVM_DEBUG(dbgs() << "Unable to materialize integer " << Ty
836 28 : << " constant, expected: " << s32 << ", " << s64
837 : << ", or " << p0 << '\n');
838 : return false;
839 : }
840 :
841 : if (RB.getID() != AArch64::GPRRegBankID) {
842 : LLVM_DEBUG(dbgs() << "Unable to materialize integer " << Ty
843 : << " constant on bank: " << RB
844 : << ", expected: GPR\n");
845 56 : return false;
846 : }
847 : }
848 :
849 0 : const unsigned MovOpc =
850 : DefSize == 32 ? AArch64::MOVi32imm : AArch64::MOVi64imm;
851 :
852 : I.setDesc(TII.get(MovOpc));
853 0 :
854 : if (isFP) {
855 : const TargetRegisterClass &GPRRC =
856 0 : DefSize == 32 ? AArch64::GPR32RegClass : AArch64::GPR64RegClass;
857 : const TargetRegisterClass &FPRRC =
858 : DefSize == 32 ? AArch64::FPR32RegClass : AArch64::FPR64RegClass;
859 :
860 : const unsigned DefGPRReg = MRI.createVirtualRegister(&GPRRC);
861 : MachineOperand &RegOp = I.getOperand(0);
862 : RegOp.setReg(DefGPRReg);
863 :
864 : BuildMI(MBB, std::next(I.getIterator()), I.getDebugLoc(),
865 28 : TII.get(AArch64::COPY))
866 : .addDef(DefReg)
867 28 : .addUse(DefGPRReg);
868 :
869 28 : if (!RBI.constrainGenericRegister(DefReg, FPRRC, MRI)) {
870 28 : LLVM_DEBUG(dbgs() << "Failed to constrain G_FCONSTANT def operand\n");
871 : return false;
872 28 : }
873 :
874 : MachineOperand &ImmOp = I.getOperand(1);
875 28 : // FIXME: Is going through int64_t always correct?
876 28 : ImmOp.ChangeToImmediate(
877 28 : ImmOp.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue());
878 : } else if (I.getOperand(1).isCImm()) {
879 28 : uint64_t Val = I.getOperand(1).getCImm()->getZExtValue();
880 84 : I.getOperand(1).ChangeToImmediate(Val);
881 : } else if (I.getOperand(1).isImm()) {
882 : uint64_t Val = I.getOperand(1).getImm();
883 : I.getOperand(1).ChangeToImmediate(Val);
884 28 : }
885 :
886 : constrainSelectedInstRegOperands(I, TII, TRI, RBI);
887 : return true;
888 : }
889 28 : case TargetOpcode::G_EXTRACT: {
890 : LLT SrcTy = MRI.getType(I.getOperand(1).getReg());
891 28 : LLT DstTy = MRI.getType(I.getOperand(0).getReg());
892 84 : (void)DstTy;
893 0 : unsigned SrcSize = SrcTy.getSizeInBits();
894 0 : // Larger extracts are vectors, same-size extracts should be something else
895 0 : // by now (either split up or simplified to a COPY).
896 0 : if (SrcTy.getSizeInBits() > 64 || Ty.getSizeInBits() > 32)
897 0 : return false;
898 0 :
899 : I.setDesc(TII.get(SrcSize == 64 ? AArch64::UBFMXri : AArch64::UBFMWri));
900 : MachineInstrBuilder(MF, I).addImm(I.getOperand(2).getImm() +
901 28 : Ty.getSizeInBits() - 1);
902 28 :
903 : if (SrcSize < 64) {
904 4 : assert(SrcSize == 32 && DstTy.getSizeInBits() == 16 &&
905 4 : "unexpected G_EXTRACT types");
906 : return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
907 : }
908 4 :
909 : unsigned DstReg = MRI.createGenericVirtualRegister(LLT::scalar(64));
910 : BuildMI(MBB, std::next(I.getIterator()), I.getDebugLoc(),
911 4 : TII.get(AArch64::COPY))
912 : .addDef(I.getOperand(0).getReg())
913 : .addUse(DstReg, 0, AArch64::sub_32);
914 6 : RBI.constrainGenericRegister(I.getOperand(0).getReg(),
915 8 : AArch64::GPR32RegClass, MRI);
916 4 : I.getOperand(0).setReg(DstReg);
917 :
918 4 : return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
919 : }
920 :
921 2 : case TargetOpcode::G_INSERT: {
922 : LLT SrcTy = MRI.getType(I.getOperand(2).getReg());
923 : LLT DstTy = MRI.getType(I.getOperand(0).getReg());
924 2 : unsigned DstSize = DstTy.getSizeInBits();
925 2 : // Larger inserts are vectors, same-size ones should be something else by
926 6 : // now (split up or turned into COPYs).
927 2 : if (Ty.getSizeInBits() > 64 || SrcTy.getSizeInBits() > 32)
928 : return false;
929 2 :
930 : I.setDesc(TII.get(DstSize == 64 ? AArch64::BFMXri : AArch64::BFMWri));
931 2 : unsigned LSB = I.getOperand(3).getImm();
932 : unsigned Width = MRI.getType(I.getOperand(2).getReg()).getSizeInBits();
933 2 : I.getOperand(3).setImm((DstSize - LSB) % DstSize);
934 : MachineInstrBuilder(MF, I).addImm(Width - 1);
935 :
936 6 : if (DstSize < 64) {
937 6 : assert(DstSize == 32 && SrcTy.getSizeInBits() == 16 &&
938 6 : "unexpected G_INSERT types");
939 6 : return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
940 : }
941 :
942 6 : unsigned SrcReg = MRI.createGenericVirtualRegister(LLT::scalar(64));
943 : BuildMI(MBB, I.getIterator(), I.getDebugLoc(),
944 : TII.get(AArch64::SUBREG_TO_REG))
945 10 : .addDef(SrcReg)
946 6 : .addImm(0)
947 6 : .addUse(I.getOperand(2).getReg())
948 6 : .addImm(AArch64::sub_32);
949 6 : RBI.constrainGenericRegister(I.getOperand(2).getReg(),
950 : AArch64::GPR32RegClass, MRI);
951 6 : I.getOperand(2).setReg(SrcReg);
952 :
953 : return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
954 4 : }
955 : case TargetOpcode::G_FRAME_INDEX: {
956 : // allocas and G_FRAME_INDEX are only supported in addrspace(0).
957 2 : if (Ty != LLT::pointer(0, 64)) {
958 2 : LLVM_DEBUG(dbgs() << "G_FRAME_INDEX pointer has type: " << Ty
959 4 : << ", expected: " << LLT::pointer(0, 64) << '\n');
960 : return false;
961 : }
962 2 : I.setDesc(TII.get(AArch64::ADDXri));
963 :
964 2 : // MOs for a #0 shifted immediate.
965 : I.addOperand(MachineOperand::CreateImm(0));
966 4 : I.addOperand(MachineOperand::CreateImm(0));
967 :
968 2 : return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
969 : }
970 :
971 : case TargetOpcode::G_GLOBAL_VALUE: {
972 3 : auto GV = I.getOperand(1).getGlobal();
973 : if (GV->isThreadLocal()) {
974 : // FIXME: we don't support TLS yet.
975 0 : return false;
976 : }
977 3 : unsigned char OpFlags = STI.ClassifyGlobalReference(GV, TM);
978 : if (OpFlags & AArch64II::MO_GOT) {
979 : I.setDesc(TII.get(AArch64::LOADgot));
980 3 : I.getOperand(1).setTargetFlags(OpFlags);
981 3 : } else if (TM.getCodeModel() == CodeModel::Large) {
982 : // Materialize the global using movz/movk instructions.
983 3 : materializeLargeCMVal(I, GV, OpFlags);
984 : I.eraseFromParent();
985 : return true;
986 85 : } else if (TM.getCodeModel() == CodeModel::Tiny) {
987 85 : I.setDesc(TII.get(AArch64::ADR));
988 85 : I.getOperand(1).setTargetFlags(OpFlags);
989 : } else {
990 : I.setDesc(TII.get(AArch64::MOVaddr));
991 : I.getOperand(1).setTargetFlags(OpFlags | AArch64II::MO_PAGE);
992 85 : MachineInstrBuilder MIB(MF, I);
993 85 : MIB.addGlobalAddress(GV, I.getOperand(1).getOffset(),
994 18 : OpFlags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
995 18 : }
996 67 : return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
997 : }
998 3 :
999 3 : case TargetOpcode::G_LOAD:
1000 3 : case TargetOpcode::G_STORE: {
1001 64 : LLT PtrTy = MRI.getType(I.getOperand(1).getReg());
1002 34 :
1003 34 : if (PtrTy != LLT::pointer(0, 64)) {
1004 : LLVM_DEBUG(dbgs() << "Load/Store pointer has type: " << PtrTy
1005 30 : << ", expected: " << LLT::pointer(0, 64) << '\n');
1006 30 : return false;
1007 : }
1008 30 :
1009 30 : auto &MemOp = **I.memoperands_begin();
1010 : if (MemOp.getOrdering() != AtomicOrdering::NotAtomic) {
1011 82 : LLVM_DEBUG(dbgs() << "Atomic load/store not supported yet\n");
1012 : return false;
1013 : }
1014 80 : unsigned MemSizeInBits = MemOp.getSize() * 8;
1015 :
1016 80 : // FIXME: PR36018: Volatile loads in some cases are incorrectly selected by
1017 : // folding with an extend. Until we have a G_SEXTLOAD solution bail out if
1018 : // we hit one.
1019 : if (Opcode == TargetOpcode::G_LOAD && MemOp.isVolatile())
1020 : return false;
1021 :
1022 : const unsigned PtrReg = I.getOperand(1).getReg();
1023 : #ifndef NDEBUG
1024 80 : const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, MRI, TRI);
1025 80 : // Sanity-check the pointer register.
1026 : assert(PtrRB.getID() == AArch64::GPRRegBankID &&
1027 : "Load/Store pointer operand isn't a GPR");
1028 : assert(MRI.getType(PtrReg).isPointer() &&
1029 76 : "Load/Store pointer operand isn't a pointer");
1030 : #endif
1031 :
1032 : const unsigned ValReg = I.getOperand(0).getReg();
1033 : const RegisterBank &RB = *RBI.getRegBank(ValReg, MRI, TRI);
1034 76 :
1035 : const unsigned NewOpc =
1036 : selectLoadStoreUIOp(I.getOpcode(), RB.getID(), MemSizeInBits);
1037 76 : if (NewOpc == I.getOpcode())
1038 : return false;
1039 :
1040 : I.setDesc(TII.get(NewOpc));
1041 :
1042 : uint64_t Offset = 0;
1043 : auto *PtrMI = MRI.getVRegDef(PtrReg);
1044 :
1045 : // Try to fold a GEP into our unsigned immediate addressing mode.
1046 : if (PtrMI->getOpcode() == TargetOpcode::G_GEP) {
1047 76 : if (auto COff = getConstantVRegVal(PtrMI->getOperand(2).getReg(), MRI)) {
1048 76 : int64_t Imm = *COff;
1049 : const unsigned Size = MemSizeInBits / 8;
1050 : const unsigned Scale = Log2_32(Size);
1051 76 : if ((Imm & (Size - 1)) == 0 && Imm >= 0 && Imm < (0x1000 << Scale)) {
1052 76 : unsigned Ptr2Reg = PtrMI->getOperand(1).getReg();
1053 : I.getOperand(1).setReg(Ptr2Reg);
1054 : PtrMI = MRI.getVRegDef(Ptr2Reg);
1055 76 : Offset = Imm / Size;
1056 : }
1057 : }
1058 76 : }
1059 :
1060 : // If we haven't folded anything into our addressing mode yet, try to fold
1061 152 : // a frame index into the base+offset.
1062 19 : if (!Offset && PtrMI->getOpcode() == TargetOpcode::G_FRAME_INDEX)
1063 19 : I.getOperand(1).ChangeToFrameIndex(PtrMI->getOperand(1).getIndex());
1064 19 :
1065 : I.addOperand(MachineOperand::CreateImm(Offset));
1066 19 :
1067 19 : // If we're storing a 0, use WZR/XZR.
1068 38 : if (auto CVal = getConstantVRegVal(ValReg, MRI)) {
1069 19 : if (*CVal == 0 && Opcode == TargetOpcode::G_STORE) {
1070 19 : if (I.getOpcode() == AArch64::STRWui)
1071 : I.getOperand(0).setReg(AArch64::WZR);
1072 : else if (I.getOpcode() == AArch64::STRXui)
1073 : I.getOperand(0).setReg(AArch64::XZR);
1074 : }
1075 : }
1076 :
1077 133 : return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1078 6 : }
1079 :
1080 152 : case TargetOpcode::G_SMULH:
1081 : case TargetOpcode::G_UMULH: {
1082 : // Reject the various things we don't support yet.
1083 76 : if (unsupportedBinOp(I, RBI, MRI, TRI))
1084 0 : return false;
1085 0 :
1086 0 : const unsigned DefReg = I.getOperand(0).getReg();
1087 0 : const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
1088 0 :
1089 : if (RB.getID() != AArch64::GPRRegBankID) {
1090 : LLVM_DEBUG(dbgs() << "G_[SU]MULH on bank: " << RB << ", expected: GPR\n");
1091 : return false;
1092 76 : }
1093 :
1094 : if (Ty != LLT::scalar(64)) {
1095 2 : LLVM_DEBUG(dbgs() << "G_[SU]MULH has type: " << Ty
1096 : << ", expected: " << LLT::scalar(64) << '\n');
1097 : return false;
1098 2 : }
1099 :
1100 : unsigned NewOpc = I.getOpcode() == TargetOpcode::G_SMULH ? AArch64::SMULHrr
1101 2 : : AArch64::UMULHrr;
1102 2 : I.setDesc(TII.get(NewOpc));
1103 :
1104 2 : // Now that we selected an opcode, we need to constrain the register
1105 : // operands to use appropriate classes.
1106 : return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1107 : }
1108 : case TargetOpcode::G_FADD:
1109 2 : case TargetOpcode::G_FSUB:
1110 : case TargetOpcode::G_FMUL:
1111 : case TargetOpcode::G_FDIV:
1112 0 :
1113 : case TargetOpcode::G_OR:
1114 : case TargetOpcode::G_SHL:
1115 4 : case TargetOpcode::G_LSHR:
1116 : case TargetOpcode::G_ASHR:
1117 2 : case TargetOpcode::G_GEP: {
1118 : // Reject the various things we don't support yet.
1119 : if (unsupportedBinOp(I, RBI, MRI, TRI))
1120 : return false;
1121 2 :
1122 : const unsigned OpSize = Ty.getSizeInBits();
1123 14 :
1124 : const unsigned DefReg = I.getOperand(0).getReg();
1125 : const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
1126 :
1127 : const unsigned NewOpc = selectBinaryOp(I.getOpcode(), RB.getID(), OpSize);
1128 : if (NewOpc == I.getOpcode())
1129 : return false;
1130 :
1131 : I.setDesc(TII.get(NewOpc));
1132 : // FIXME: Should the type be always reset in setDesc?
1133 :
1134 14 : // Now that we selected an opcode, we need to constrain the register
1135 : // operands to use appropriate classes.
1136 : return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1137 14 : }
1138 :
1139 14 : case TargetOpcode::G_PTR_MASK: {
1140 14 : uint64_t Align = I.getOperand(2).getImm();
1141 : if (Align >= 64 || Align == 0)
1142 14 : return false;
1143 14 :
1144 : uint64_t Mask = ~((1ULL << Align) - 1);
1145 : I.setDesc(TII.get(AArch64::ANDXri));
1146 14 : I.getOperand(2).setImm(AArch64_AM::encodeLogicalImmediate(Mask, 64));
1147 :
1148 : return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1149 : }
1150 : case TargetOpcode::G_PTRTOINT:
1151 14 : case TargetOpcode::G_TRUNC: {
1152 : const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
1153 : const LLT SrcTy = MRI.getType(I.getOperand(1).getReg());
1154 3 :
1155 3 : const unsigned DstReg = I.getOperand(0).getReg();
1156 3 : const unsigned SrcReg = I.getOperand(1).getReg();
1157 :
1158 : const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
1159 3 : const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
1160 3 :
1161 3 : if (DstRB.getID() != SrcRB.getID()) {
1162 : LLVM_DEBUG(
1163 3 : dbgs() << "G_TRUNC/G_PTRTOINT input/output on different banks\n");
1164 : return false;
1165 56 : }
1166 :
1167 56 : if (DstRB.getID() == AArch64::GPRRegBankID) {
1168 56 : const TargetRegisterClass *DstRC =
1169 : getRegClassForTypeOnBank(DstTy, DstRB, RBI);
1170 : if (!DstRC)
1171 : return false;
1172 :
1173 56 : const TargetRegisterClass *SrcRC =
1174 56 : getRegClassForTypeOnBank(SrcTy, SrcRB, RBI);
1175 : if (!SrcRC)
1176 56 : return false;
1177 :
1178 : if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
1179 : !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
1180 : LLVM_DEBUG(dbgs() << "Failed to constrain G_TRUNC/G_PTRTOINT\n");
1181 : return false;
1182 56 : }
1183 :
1184 56 : if (DstRC == SrcRC) {
1185 56 : // Nothing to be done
1186 : } else if (Opcode == TargetOpcode::G_TRUNC && DstTy == LLT::scalar(32) &&
1187 : SrcTy == LLT::scalar(64)) {
1188 : llvm_unreachable("TableGen can import this case");
1189 56 : return false;
1190 56 : } else if (DstRC == &AArch64::GPR32RegClass &&
1191 : SrcRC == &AArch64::GPR64RegClass) {
1192 : I.getOperand(1).setSubReg(AArch64::sub_32);
1193 112 : } else {
1194 56 : LLVM_DEBUG(
1195 : dbgs() << "Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n");
1196 0 : return false;
1197 : }
1198 :
1199 56 : I.setDesc(TII.get(TargetOpcode::COPY));
1200 : return true;
1201 5 : } else if (DstRB.getID() == AArch64::FPRRegBankID) {
1202 : if (DstTy == LLT::vector(4, 16) && SrcTy == LLT::vector(4, 32)) {
1203 0 : I.setDesc(TII.get(AArch64::XTNv4i16));
1204 : constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1205 5 : return true;
1206 : }
1207 5 : }
1208 :
1209 : return false;
1210 : }
1211 :
1212 : case TargetOpcode::G_ANYEXT: {
1213 : const unsigned DstReg = I.getOperand(0).getReg();
1214 56 : const unsigned SrcReg = I.getOperand(1).getReg();
1215 56 :
1216 0 : const RegisterBank &RBDst = *RBI.getRegBank(DstReg, MRI, TRI);
1217 : if (RBDst.getID() != AArch64::GPRRegBankID) {
1218 0 : LLVM_DEBUG(dbgs() << "G_ANYEXT on bank: " << RBDst
1219 0 : << ", expected: GPR\n");
1220 0 : return false;
1221 : }
1222 :
1223 : const RegisterBank &RBSrc = *RBI.getRegBank(SrcReg, MRI, TRI);
1224 : if (RBSrc.getID() != AArch64::GPRRegBankID) {
1225 : LLVM_DEBUG(dbgs() << "G_ANYEXT on bank: " << RBSrc
1226 : << ", expected: GPR\n");
1227 30 : return false;
1228 30 : }
1229 30 :
1230 : const unsigned DstSize = MRI.getType(DstReg).getSizeInBits();
1231 30 :
1232 30 : if (DstSize == 0) {
1233 : LLVM_DEBUG(dbgs() << "G_ANYEXT operand has no size, not a gvreg?\n");
1234 : return false;
1235 : }
1236 :
1237 : if (DstSize != 64 && DstSize > 32) {
1238 30 : LLVM_DEBUG(dbgs() << "G_ANYEXT to size: " << DstSize
1239 30 : << ", expected: 32 or 64\n");
1240 : return false;
1241 : }
1242 : // At this point G_ANYEXT is just like a plain COPY, but we need
1243 : // to explicitly form the 64-bit value if any.
1244 : if (DstSize > 32) {
1245 30 : unsigned ExtSrc = MRI.createVirtualRegister(&AArch64::GPR64allRegClass);
1246 : BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::SUBREG_TO_REG))
1247 30 : .addDef(ExtSrc)
1248 : .addImm(0)
1249 : .addUse(SrcReg)
1250 : .addImm(AArch64::sub_32);
1251 : I.getOperand(1).setReg(ExtSrc);
1252 30 : }
1253 : return selectCopy(I, TII, MRI, TRI, RBI);
1254 : }
1255 :
1256 : case TargetOpcode::G_ZEXT:
1257 : case TargetOpcode::G_SEXT: {
1258 : unsigned Opcode = I.getOpcode();
1259 30 : const LLT DstTy = MRI.getType(I.getOperand(0).getReg()),
1260 2 : SrcTy = MRI.getType(I.getOperand(1).getReg());
1261 4 : const bool isSigned = Opcode == TargetOpcode::G_SEXT;
1262 : const unsigned DefReg = I.getOperand(0).getReg();
1263 : const unsigned SrcReg = I.getOperand(1).getReg();
1264 : const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
1265 :
1266 4 : if (RB.getID() != AArch64::GPRRegBankID) {
1267 : LLVM_DEBUG(dbgs() << TII.getName(I.getOpcode()) << " on bank: " << RB
1268 30 : << ", expected: GPR\n");
1269 : return false;
1270 : }
1271 14 :
1272 : MachineInstr *ExtI;
1273 14 : if (DstTy == LLT::scalar(64)) {
1274 14 : // FIXME: Can we avoid manually doing this?
1275 14 : if (!RBI.constrainGenericRegister(SrcReg, AArch64::GPR32RegClass, MRI)) {
1276 : LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(Opcode)
1277 : << " operand\n");
1278 : return false;
1279 14 : }
1280 :
1281 14 : const unsigned SrcXReg =
1282 : MRI.createVirtualRegister(&AArch64::GPR64RegClass);
1283 : BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::SUBREG_TO_REG))
1284 : .addDef(SrcXReg)
1285 : .addImm(0)
1286 : .addUse(SrcReg)
1287 : .addImm(AArch64::sub_32);
1288 9 :
1289 : const unsigned NewOpc = isSigned ? AArch64::SBFMXri : AArch64::UBFMXri;
1290 5 : ExtI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(NewOpc))
1291 : .addDef(DefReg)
1292 : .addUse(SrcXReg)
1293 : .addImm(0)
1294 : .addImm(SrcTy.getSizeInBits() - 1);
1295 : } else if (DstTy.isScalar() && DstTy.getSizeInBits() <= 32) {
1296 : const unsigned NewOpc = isSigned ? AArch64::SBFMWri : AArch64::UBFMWri;
1297 5 : ExtI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(NewOpc))
1298 10 : .addDef(DefReg)
1299 : .addUse(SrcReg)
1300 : .addImm(0)
1301 : .addImm(SrcTy.getSizeInBits() - 1);
1302 : } else {
1303 : return false;
1304 5 : }
1305 10 :
1306 : constrainSelectedInstRegOperands(*ExtI, TII, TRI, RBI);
1307 :
1308 : I.eraseFromParent();
1309 5 : return true;
1310 9 : }
1311 9 :
1312 18 : case TargetOpcode::G_SITOFP:
1313 : case TargetOpcode::G_UITOFP:
1314 : case TargetOpcode::G_FPTOSI:
1315 : case TargetOpcode::G_FPTOUI: {
1316 9 : const LLT DstTy = MRI.getType(I.getOperand(0).getReg()),
1317 : SrcTy = MRI.getType(I.getOperand(1).getReg());
1318 0 : const unsigned NewOpc = selectFPConvOpc(Opcode, DstTy, SrcTy);
1319 : if (NewOpc == Opcode)
1320 : return false;
1321 14 :
1322 : I.setDesc(TII.get(NewOpc));
1323 14 : constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1324 14 :
1325 : return true;
1326 : }
1327 0 :
1328 :
1329 : case TargetOpcode::G_INTTOPTR:
1330 : // The importer is currently unable to import pointer types since they
1331 0 : // didn't exist in SelectionDAG.
1332 0 : return selectCopy(I, TII, MRI, TRI, RBI);
1333 0 :
1334 0 : case TargetOpcode::G_BITCAST:
1335 : // Imported SelectionDAG rules can handle every bitcast except those that
1336 : // bitcast from a type to the same type. Ideally, these shouldn't occur
1337 0 : // but we might not run an optimizer that deletes them.
1338 0 : if (MRI.getType(I.getOperand(0).getReg()) ==
1339 : MRI.getType(I.getOperand(1).getReg()))
1340 0 : return selectCopy(I, TII, MRI, TRI, RBI);
1341 : return false;
1342 :
1343 : case TargetOpcode::G_SELECT: {
1344 1 : if (MRI.getType(I.getOperand(1).getReg()) != LLT::scalar(1)) {
1345 : LLVM_DEBUG(dbgs() << "G_SELECT cond has type: " << Ty
1346 : << ", expected: " << LLT::scalar(1) << '\n');
1347 1 : return false;
1348 : }
1349 3 :
1350 : const unsigned CondReg = I.getOperand(1).getReg();
1351 : const unsigned TReg = I.getOperand(2).getReg();
1352 : const unsigned FReg = I.getOperand(3).getReg();
1353 3 :
1354 3 : unsigned CSelOpc = 0;
1355 3 :
1356 : if (Ty == LLT::scalar(32)) {
1357 : CSelOpc = AArch64::CSELWr;
1358 : } else if (Ty == LLT::scalar(64) || Ty == LLT::pointer(0, 64)) {
1359 9 : CSelOpc = AArch64::CSELXr;
1360 : } else {
1361 : return false;
1362 : }
1363 :
1364 : MachineInstr &TstMI =
1365 : *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::ANDSWri))
1366 9 : .addDef(AArch64::WZR)
1367 9 : .addUse(CondReg)
1368 : .addImm(AArch64_AM::encodeLogicalImmediate(1, 32));
1369 :
1370 : MachineInstr &CSelMI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CSelOpc))
1371 9 : .addDef(I.getOperand(0).getReg())
1372 : .addUse(TReg)
1373 9 : .addUse(FReg)
1374 : .addImm(AArch64CC::NE);
1375 :
1376 0 : constrainSelectedInstRegOperands(TstMI, TII, TRI, RBI);
1377 : constrainSelectedInstRegOperands(CSelMI, TII, TRI, RBI);
1378 :
1379 : I.eraseFromParent();
1380 18 : return true;
1381 : }
1382 : case TargetOpcode::G_ICMP: {
1383 9 : if (Ty != LLT::scalar(32)) {
1384 : LLVM_DEBUG(dbgs() << "G_ICMP result has type: " << Ty
1385 18 : << ", expected: " << LLT::scalar(32) << '\n');
1386 9 : return false;
1387 : }
1388 :
1389 9 : unsigned CmpOpc = 0;
1390 : unsigned ZReg = 0;
1391 9 :
1392 9 : LLT CmpTy = MRI.getType(I.getOperand(2).getReg());
1393 : if (CmpTy == LLT::scalar(32)) {
1394 9 : CmpOpc = AArch64::SUBSWrr;
1395 9 : ZReg = AArch64::WZR;
1396 : } else if (CmpTy == LLT::scalar(64) || CmpTy.isPointer()) {
1397 : CmpOpc = AArch64::SUBSXrr;
1398 14 : ZReg = AArch64::XZR;
1399 : } else {
1400 : return false;
1401 0 : }
1402 :
1403 : // CSINC increments the result by one when the condition code is false.
1404 : // Therefore, we have to invert the predicate to get an increment by 1 when
1405 : // the predicate is true.
1406 : const AArch64CC::CondCode invCC =
1407 14 : changeICMPPredToAArch64CC(CmpInst::getInversePredicate(
1408 : (CmpInst::Predicate)I.getOperand(1).getPredicate()));
1409 :
1410 : MachineInstr &CmpMI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CmpOpc))
1411 : .addDef(ZReg)
1412 : .addUse(I.getOperand(2).getReg())
1413 : .addUse(I.getOperand(3).getReg());
1414 :
1415 0 : MachineInstr &CSetMI =
1416 : *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr))
1417 : .addDef(I.getOperand(0).getReg())
1418 : .addUse(AArch64::WZR)
1419 : .addUse(AArch64::WZR)
1420 : .addImm(invCC);
1421 :
1422 14 : constrainSelectedInstRegOperands(CmpMI, TII, TRI, RBI);
1423 14 : constrainSelectedInstRegOperands(CSetMI, TII, TRI, RBI);
1424 :
1425 28 : I.eraseFromParent();
1426 : return true;
1427 14 : }
1428 14 :
1429 : case TargetOpcode::G_FCMP: {
1430 : if (Ty != LLT::scalar(32)) {
1431 28 : LLVM_DEBUG(dbgs() << "G_FCMP result has type: " << Ty
1432 14 : << ", expected: " << LLT::scalar(32) << '\n');
1433 : return false;
1434 : }
1435 14 :
1436 : unsigned CmpOpc = 0;
1437 14 : LLT CmpTy = MRI.getType(I.getOperand(2).getReg());
1438 14 : if (CmpTy == LLT::scalar(32)) {
1439 : CmpOpc = AArch64::FCMPSrr;
1440 14 : } else if (CmpTy == LLT::scalar(64)) {
1441 14 : CmpOpc = AArch64::FCMPDrr;
1442 : } else {
1443 : return false;
1444 : }
1445 6 :
1446 : // FIXME: regbank
1447 :
1448 0 : AArch64CC::CondCode CC1, CC2;
1449 : changeFCMPPredToAArch64CC(
1450 : (CmpInst::Predicate)I.getOperand(1).getPredicate(), CC1, CC2);
1451 :
1452 6 : MachineInstr &CmpMI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CmpOpc))
1453 : .addUse(I.getOperand(2).getReg())
1454 : .addUse(I.getOperand(3).getReg());
1455 :
1456 : const unsigned DefReg = I.getOperand(0).getReg();
1457 : unsigned Def1Reg = DefReg;
1458 : if (CC2 != AArch64CC::AL)
1459 : Def1Reg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
1460 :
1461 : MachineInstr &CSetMI =
1462 : *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr))
1463 : .addDef(Def1Reg)
1464 6 : .addUse(AArch64::WZR)
1465 6 : .addUse(AArch64::WZR)
1466 : .addImm(getInvertedCondCode(CC1));
1467 12 :
1468 6 : if (CC2 != AArch64CC::AL) {
1469 6 : unsigned Def2Reg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
1470 : MachineInstr &CSet2MI =
1471 6 : *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr))
1472 : .addDef(Def2Reg)
1473 6 : .addUse(AArch64::WZR)
1474 3 : .addUse(AArch64::WZR)
1475 : .addImm(getInvertedCondCode(CC2));
1476 : MachineInstr &OrMI =
1477 12 : *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::ORRWrr))
1478 : .addDef(DefReg)
1479 : .addUse(Def1Reg)
1480 : .addUse(Def2Reg);
1481 12 : constrainSelectedInstRegOperands(OrMI, TII, TRI, RBI);
1482 : constrainSelectedInstRegOperands(CSet2MI, TII, TRI, RBI);
1483 6 : }
1484 3 :
1485 : constrainSelectedInstRegOperands(CmpMI, TII, TRI, RBI);
1486 6 : constrainSelectedInstRegOperands(CSetMI, TII, TRI, RBI);
1487 :
1488 : I.eraseFromParent();
1489 : return true;
1490 6 : }
1491 : case TargetOpcode::G_VASTART:
1492 6 : return STI.isTargetDarwin() ? selectVaStartDarwin(I, MF, MRI)
1493 : : selectVaStartAAPCS(I, MF, MRI);
1494 : case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
1495 3 : if (!I.getOperand(0).isIntrinsicID())
1496 3 : return false;
1497 3 : if (I.getOperand(0).getIntrinsicID() != Intrinsic::trap)
1498 : return false;
1499 : BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::BRK))
1500 6 : .addImm(1);
1501 6 : I.eraseFromParent();
1502 : return true;
1503 6 : case TargetOpcode::G_IMPLICIT_DEF: {
1504 6 : I.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF));
1505 : const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
1506 1 : const unsigned DstReg = I.getOperand(0).getReg();
1507 2 : const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
1508 : const TargetRegisterClass *DstRC =
1509 2 : getRegClassForTypeOnBank(DstTy, DstRB, RBI);
1510 2 : RBI.constrainGenericRegister(DstReg, *DstRC, MRI);
1511 : return true;
1512 1 : }
1513 : case TargetOpcode::G_BLOCK_ADDR: {
1514 2 : if (TM.getCodeModel() == CodeModel::Large) {
1515 : materializeLargeCMVal(I, I.getOperand(1).getBlockAddress(), 0);
1516 1 : I.eraseFromParent();
1517 1 : return true;
1518 7 : } else {
1519 7 : I.setDesc(TII.get(AArch64::MOVaddrBA));
1520 7 : auto MovMI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::MOVaddrBA),
1521 : I.getOperand(0).getReg())
1522 7 : .addBlockAddress(I.getOperand(1).getBlockAddress(),
1523 : /* Offset */ 0, AArch64II::MO_PAGE)
1524 7 : .addBlockAddress(
1525 7 : I.getOperand(1).getBlockAddress(), /* Offset */ 0,
1526 : AArch64II::MO_NC | AArch64II::MO_PAGEOFF);
1527 : I.eraseFromParent();
1528 2 : return constrainSelectedInstRegOperands(*MovMI, TII, TRI, RBI);
1529 2 : }
1530 1 : }
1531 1 : }
1532 1 :
1533 : return false;
1534 1 : }
1535 :
1536 2 : /// SelectArithImmed - Select an immediate value that can be represented as
1537 1 : /// a 12-bit value shifted left by either 0 or 12. If so, return true with
1538 1 : /// Val set to the 12-bit value and Shift set to the shifter operand.
1539 : InstructionSelector::ComplexRendererFns
1540 1 : AArch64InstructionSelector::selectArithImmed(MachineOperand &Root) const {
1541 1 : MachineInstr &MI = *Root.getParent();
1542 1 : MachineBasicBlock &MBB = *MI.getParent();
1543 1 : MachineFunction &MF = *MBB.getParent();
1544 : MachineRegisterInfo &MRI = MF.getRegInfo();
1545 :
1546 : // This function is called from the addsub_shifted_imm ComplexPattern,
1547 : // which lists [imm] as the list of opcode it's interested in, however
1548 : // we still need to check whether the operand is actually an immediate
1549 : // here because the ComplexPattern opcode list is only used in
1550 : // root-level opcode matching.
1551 : uint64_t Immed;
1552 : if (Root.isImm())
1553 : Immed = Root.getImm();
1554 : else if (Root.isCImm())
1555 40 : Immed = Root.getCImm()->getZExtValue();
1556 40 : else if (Root.isReg()) {
1557 40 : MachineInstr *Def = MRI.getVRegDef(Root.getReg());
1558 40 : if (Def->getOpcode() != TargetOpcode::G_CONSTANT)
1559 40 : return None;
1560 : MachineOperand &Op1 = Def->getOperand(1);
1561 : if (!Op1.isCImm() || Op1.getCImm()->getBitWidth() > 64)
1562 : return None;
1563 : Immed = Op1.getCImm()->getZExtValue();
1564 : } else
1565 : return None;
1566 :
1567 40 : unsigned ShiftAmt;
1568 0 :
1569 40 : if (Immed >> 12 == 0) {
1570 0 : ShiftAmt = 0;
1571 40 : } else if ((Immed & 0xfff) == 0 && Immed >> 24 == 0) {
1572 40 : ShiftAmt = 12;
1573 80 : Immed = Immed >> 12;
1574 : } else
1575 11 : return None;
1576 11 :
1577 : unsigned ShVal = AArch64_AM::getShifterImm(AArch64_AM::LSL, ShiftAmt);
1578 : return {{
1579 : [=](MachineInstrBuilder &MIB) { MIB.addImm(Immed); },
1580 : [=](MachineInstrBuilder &MIB) { MIB.addImm(ShVal); },
1581 : }};
1582 : }
1583 :
1584 11 : /// Select a "register plus unscaled signed 9-bit immediate" address. This
1585 : /// should only match when there is an offset that is not valid for a scaled
1586 0 : /// immediate addressing mode. The "Size" argument is the size in bytes of the
1587 : /// memory reference, which is needed here to know what is valid for a scaled
1588 : /// immediate.
1589 : InstructionSelector::ComplexRendererFns
1590 : AArch64InstructionSelector::selectAddrModeUnscaled(MachineOperand &Root,
1591 : unsigned Size) const {
1592 : MachineRegisterInfo &MRI =
1593 : Root.getParent()->getParent()->getParent()->getRegInfo();
1594 11 :
1595 11 : if (!Root.isReg())
1596 44 : return None;
1597 :
1598 : if (!isBaseWithConstantOffset(Root, MRI))
1599 : return None;
1600 :
1601 : MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());
1602 : if (!RootDef)
1603 : return None;
1604 :
1605 91 : MachineOperand &OffImm = RootDef->getOperand(2);
1606 : if (!OffImm.isReg())
1607 : return None;
1608 91 : MachineInstr *RHS = MRI.getVRegDef(OffImm.getReg());
1609 : if (!RHS || RHS->getOpcode() != TargetOpcode::G_CONSTANT)
1610 91 : return None;
1611 : int64_t RHSC;
1612 : MachineOperand &RHSOp1 = RHS->getOperand(1);
1613 91 : if (!RHSOp1.isCImm() || RHSOp1.getCImm()->getBitWidth() > 64)
1614 : return None;
1615 : RHSC = RHSOp1.getCImm()->getSExtValue();
1616 3 :
1617 3 : // If the offset is valid as a scaled immediate, don't match here.
1618 : if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Log2_32(Size)))
1619 : return None;
1620 3 : if (RHSC >= -256 && RHSC < 256) {
1621 3 : MachineOperand &Base = RootDef->getOperand(1);
1622 : return {{
1623 3 : [=](MachineInstrBuilder &MIB) { MIB.add(Base); },
1624 3 : [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); },
1625 : }};
1626 : }
1627 3 : return None;
1628 3 : }
1629 :
1630 : /// Select a "register plus scaled unsigned 12-bit immediate" address. The
1631 : /// "Size" argument is the size in bytes of the memory reference, which
1632 : /// determines the scale.
1633 6 : InstructionSelector::ComplexRendererFns
1634 : AArch64InstructionSelector::selectAddrModeIndexed(MachineOperand &Root,
1635 3 : unsigned Size) const {
1636 0 : MachineRegisterInfo &MRI =
1637 : Root.getParent()->getParent()->getParent()->getRegInfo();
1638 0 :
1639 : if (!Root.isReg())
1640 0 : return None;
1641 :
1642 : MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());
1643 : if (!RootDef)
1644 : return None;
1645 :
1646 : if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
1647 : return {{
1648 : [=](MachineInstrBuilder &MIB) { MIB.add(RootDef->getOperand(1)); },
1649 297 : [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
1650 : }};
1651 : }
1652 297 :
1653 : if (isBaseWithConstantOffset(Root, MRI)) {
1654 297 : MachineOperand &LHS = RootDef->getOperand(1);
1655 : MachineOperand &RHS = RootDef->getOperand(2);
1656 : MachineInstr *LHSDef = MRI.getVRegDef(LHS.getReg());
1657 297 : MachineInstr *RHSDef = MRI.getVRegDef(RHS.getReg());
1658 297 : if (LHSDef && RHSDef) {
1659 : int64_t RHSC = (int64_t)RHSDef->getOperand(1).getCImm()->getZExtValue();
1660 : unsigned Scale = Log2_32(Size);
1661 594 : if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Scale)) {
1662 : if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
1663 23 : return {{
1664 : [=](MachineInstrBuilder &MIB) { MIB.add(LHSDef->getOperand(1)); },
1665 92 : [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC >> Scale); },
1666 : }};
1667 :
1668 274 : return {{
1669 186 : [=](MachineInstrBuilder &MIB) { MIB.add(LHS); },
1670 : [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC >> Scale); },
1671 186 : }};
1672 186 : }
1673 186 : }
1674 372 : }
1675 :
1676 186 : // Before falling back to our general case, check if the unscaled
1677 366 : // instructions can handle this. If so, that's preferable.
1678 : if (selectAddrModeUnscaled(Root, Size).hasValue())
1679 0 : return None;
1680 0 :
1681 0 : return {{
1682 : [=](MachineInstrBuilder &MIB) { MIB.add(Root); },
1683 : [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
1684 183 : }};
1685 183 : }
1686 732 :
1687 : void AArch64InstructionSelector::renderTruncImm(MachineInstrBuilder &MIB,
1688 : const MachineInstr &MI) const {
1689 : const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
1690 : assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
1691 : Optional<int64_t> CstVal = getConstantVRegVal(MI.getOperand(0).getReg(), MRI);
1692 : assert(CstVal && "Expected constant value");
1693 91 : MIB.addImm(CstVal.getValue());
1694 : }
1695 :
1696 : namespace llvm {
1697 91 : InstructionSelector *
1698 : createAArch64InstructionSelector(const AArch64TargetMachine &TM,
1699 364 : AArch64Subtarget &Subtarget,
1700 : AArch64RegisterBankInfo &RBI) {
1701 : return new AArch64InstructionSelector(TM, Subtarget, RBI);
1702 1 : }
1703 : }
|
