LLVM  9.0.0svn
X86InstructionSelector.cpp
Go to the documentation of this file.
1 //===- X86InstructionSelector.cpp -----------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 /// This file implements the targeting of the InstructionSelector class for
10 /// X86.
11 /// \todo This should be generated by TableGen.
12 //===----------------------------------------------------------------------===//
13 
15 #include "X86InstrBuilder.h"
16 #include "X86InstrInfo.h"
17 #include "X86RegisterBankInfo.h"
18 #include "X86RegisterInfo.h"
19 #include "X86Subtarget.h"
20 #include "X86TargetMachine.h"
35 #include "llvm/IR/DataLayout.h"
36 #include "llvm/IR/InstrTypes.h"
38 #include "llvm/Support/CodeGen.h"
39 #include "llvm/Support/Debug.h"
44 #include <cassert>
45 #include <cstdint>
46 #include <tuple>
47 
48 #define DEBUG_TYPE "X86-isel"
49 
50 using namespace llvm;
51 
52 namespace {
53 
54 #define GET_GLOBALISEL_PREDICATE_BITSET
55 #include "X86GenGlobalISel.inc"
56 #undef GET_GLOBALISEL_PREDICATE_BITSET
57 
58 class X86InstructionSelector : public InstructionSelector {
59 public:
60  X86InstructionSelector(const X86TargetMachine &TM, const X86Subtarget &STI,
61  const X86RegisterBankInfo &RBI);
62 
63  bool select(MachineInstr &I, CodeGenCoverage &CoverageInfo) const override;
64  static const char *getName() { return DEBUG_TYPE; }
65 
66 private:
67  /// tblgen-erated 'select' implementation, used as the initial selector for
68  /// the patterns that don't require complex C++.
69  bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
70 
71  // TODO: remove after supported by Tablegen-erated instruction selection.
72  unsigned getLoadStoreOp(const LLT &Ty, const RegisterBank &RB, unsigned Opc,
73  uint64_t Alignment) const;
74 
75  bool selectLoadStoreOp(MachineInstr &I, MachineRegisterInfo &MRI,
76  MachineFunction &MF) const;
77  bool selectFrameIndexOrGep(MachineInstr &I, MachineRegisterInfo &MRI,
78  MachineFunction &MF) const;
79  bool selectGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI,
80  MachineFunction &MF) const;
81  bool selectConstant(MachineInstr &I, MachineRegisterInfo &MRI,
82  MachineFunction &MF) const;
83  bool selectTruncOrPtrToInt(MachineInstr &I, MachineRegisterInfo &MRI,
84  MachineFunction &MF) const;
85  bool selectZext(MachineInstr &I, MachineRegisterInfo &MRI,
86  MachineFunction &MF) const;
87  bool selectAnyext(MachineInstr &I, MachineRegisterInfo &MRI,
88  MachineFunction &MF) const;
89  bool selectCmp(MachineInstr &I, MachineRegisterInfo &MRI,
90  MachineFunction &MF) const;
91  bool selectFCmp(MachineInstr &I, MachineRegisterInfo &MRI,
92  MachineFunction &MF) const;
93  bool selectUadde(MachineInstr &I, MachineRegisterInfo &MRI,
94  MachineFunction &MF) const;
95  bool selectCopy(MachineInstr &I, MachineRegisterInfo &MRI) const;
97  MachineFunction &MF,
98  CodeGenCoverage &CoverageInfo) const;
100  MachineFunction &MF,
101  CodeGenCoverage &CoverageInfo) const;
102  bool selectInsert(MachineInstr &I, MachineRegisterInfo &MRI,
103  MachineFunction &MF) const;
104  bool selectExtract(MachineInstr &I, MachineRegisterInfo &MRI,
105  MachineFunction &MF) const;
106  bool selectCondBranch(MachineInstr &I, MachineRegisterInfo &MRI,
107  MachineFunction &MF) const;
108  bool selectTurnIntoCOPY(MachineInstr &I, MachineRegisterInfo &MRI,
109  const unsigned DstReg,
110  const TargetRegisterClass *DstRC,
111  const unsigned SrcReg,
112  const TargetRegisterClass *SrcRC) const;
113  bool materializeFP(MachineInstr &I, MachineRegisterInfo &MRI,
114  MachineFunction &MF) const;
115  bool selectImplicitDefOrPHI(MachineInstr &I, MachineRegisterInfo &MRI) const;
116  bool selectShift(MachineInstr &I, MachineRegisterInfo &MRI,
117  MachineFunction &MF) const;
118  bool selectDivRem(MachineInstr &I, MachineRegisterInfo &MRI,
119  MachineFunction &MF) const;
120  bool selectIntrinsicWSideEffects(MachineInstr &I, MachineRegisterInfo &MRI,
121  MachineFunction &MF) const;
122 
123  // emit insert subreg instruction and insert it before MachineInstr &I
124  bool emitInsertSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I,
125  MachineRegisterInfo &MRI, MachineFunction &MF) const;
126  // emit extract subreg instruction and insert it before MachineInstr &I
127  bool emitExtractSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I,
128  MachineRegisterInfo &MRI, MachineFunction &MF) const;
129 
130  const TargetRegisterClass *getRegClass(LLT Ty, const RegisterBank &RB) const;
131  const TargetRegisterClass *getRegClass(LLT Ty, unsigned Reg,
132  MachineRegisterInfo &MRI) const;
133 
134  const X86TargetMachine &TM;
135  const X86Subtarget &STI;
136  const X86InstrInfo &TII;
137  const X86RegisterInfo &TRI;
138  const X86RegisterBankInfo &RBI;
139 
140 #define GET_GLOBALISEL_PREDICATES_DECL
141 #include "X86GenGlobalISel.inc"
142 #undef GET_GLOBALISEL_PREDICATES_DECL
143 
144 #define GET_GLOBALISEL_TEMPORARIES_DECL
145 #include "X86GenGlobalISel.inc"
146 #undef GET_GLOBALISEL_TEMPORARIES_DECL
147 };
148 
149 } // end anonymous namespace
150 
151 #define GET_GLOBALISEL_IMPL
152 #include "X86GenGlobalISel.inc"
153 #undef GET_GLOBALISEL_IMPL
154 
155 X86InstructionSelector::X86InstructionSelector(const X86TargetMachine &TM,
156  const X86Subtarget &STI,
157  const X86RegisterBankInfo &RBI)
158  : InstructionSelector(), TM(TM), STI(STI), TII(*STI.getInstrInfo()),
159  TRI(*STI.getRegisterInfo()), RBI(RBI),
161 #include "X86GenGlobalISel.inc"
164 #include "X86GenGlobalISel.inc"
166 {
167 }
168 
169 // FIXME: This should be target-independent, inferred from the types declared
170 // for each class in the bank.
171 const TargetRegisterClass *
173  if (RB.getID() == X86::GPRRegBankID) {
174  if (Ty.getSizeInBits() <= 8)
175  return &X86::GR8RegClass;
176  if (Ty.getSizeInBits() == 16)
177  return &X86::GR16RegClass;
178  if (Ty.getSizeInBits() == 32)
179  return &X86::GR32RegClass;
180  if (Ty.getSizeInBits() == 64)
181  return &X86::GR64RegClass;
182  }
183  if (RB.getID() == X86::VECRRegBankID) {
184  if (Ty.getSizeInBits() == 32)
185  return STI.hasAVX512() ? &X86::FR32XRegClass : &X86::FR32RegClass;
186  if (Ty.getSizeInBits() == 64)
187  return STI.hasAVX512() ? &X86::FR64XRegClass : &X86::FR64RegClass;
188  if (Ty.getSizeInBits() == 128)
189  return STI.hasAVX512() ? &X86::VR128XRegClass : &X86::VR128RegClass;
190  if (Ty.getSizeInBits() == 256)
191  return STI.hasAVX512() ? &X86::VR256XRegClass : &X86::VR256RegClass;
192  if (Ty.getSizeInBits() == 512)
193  return &X86::VR512RegClass;
194  }
195 
196  llvm_unreachable("Unknown RegBank!");
197 }
198 
199 const TargetRegisterClass *
201  MachineRegisterInfo &MRI) const {
202  const RegisterBank &RegBank = *RBI.getRegBank(Reg, MRI, TRI);
203  return getRegClass(Ty, RegBank);
204 }
205 
206 static unsigned getSubRegIndex(const TargetRegisterClass *RC) {
207  unsigned SubIdx = X86::NoSubRegister;
208  if (RC == &X86::GR32RegClass) {
209  SubIdx = X86::sub_32bit;
210  } else if (RC == &X86::GR16RegClass) {
211  SubIdx = X86::sub_16bit;
212  } else if (RC == &X86::GR8RegClass) {
213  SubIdx = X86::sub_8bit;
214  }
215 
216  return SubIdx;
217 }
218 
219 static const TargetRegisterClass *getRegClassFromGRPhysReg(unsigned Reg) {
220  assert(TargetRegisterInfo::isPhysicalRegister(Reg));
221  if (X86::GR64RegClass.contains(Reg))
222  return &X86::GR64RegClass;
223  if (X86::GR32RegClass.contains(Reg))
224  return &X86::GR32RegClass;
225  if (X86::GR16RegClass.contains(Reg))
226  return &X86::GR16RegClass;
227  if (X86::GR8RegClass.contains(Reg))
228  return &X86::GR8RegClass;
229 
230  llvm_unreachable("Unknown RegClass for PhysReg!");
231 }
232 
233 // Set X86 Opcode and constrain DestReg.
235  MachineRegisterInfo &MRI) const {
236  unsigned DstReg = I.getOperand(0).getReg();
237  const unsigned DstSize = RBI.getSizeInBits(DstReg, MRI, TRI);
238  const RegisterBank &DstRegBank = *RBI.getRegBank(DstReg, MRI, TRI);
239 
240  unsigned SrcReg = I.getOperand(1).getReg();
241  const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI);
242  const RegisterBank &SrcRegBank = *RBI.getRegBank(SrcReg, MRI, TRI);
243 
244  if (TargetRegisterInfo::isPhysicalRegister(DstReg)) {
245  assert(I.isCopy() && "Generic operators do not allow physical registers");
246 
247  if (DstSize > SrcSize && SrcRegBank.getID() == X86::GPRRegBankID &&
248  DstRegBank.getID() == X86::GPRRegBankID) {
249 
250  const TargetRegisterClass *SrcRC =
251  getRegClass(MRI.getType(SrcReg), SrcRegBank);
252  const TargetRegisterClass *DstRC = getRegClassFromGRPhysReg(DstReg);
253 
254  if (SrcRC != DstRC) {
255  // This case can be generated by ABI lowering, performe anyext
256  unsigned ExtSrc = MRI.createVirtualRegister(DstRC);
257  BuildMI(*I.getParent(), I, I.getDebugLoc(),
258  TII.get(TargetOpcode::SUBREG_TO_REG))
259  .addDef(ExtSrc)
260  .addImm(0)
261  .addReg(SrcReg)
262  .addImm(getSubRegIndex(SrcRC));
263 
264  I.getOperand(1).setReg(ExtSrc);
265  }
266  }
267 
268  return true;
269  }
270 
271  assert((!TargetRegisterInfo::isPhysicalRegister(SrcReg) || I.isCopy()) &&
272  "No phys reg on generic operators");
273  assert((DstSize == SrcSize ||
274  // Copies are a mean to setup initial types, the number of
275  // bits may not exactly match.
276  (TargetRegisterInfo::isPhysicalRegister(SrcReg) &&
277  DstSize <= RBI.getSizeInBits(SrcReg, MRI, TRI))) &&
278  "Copy with different width?!");
279 
280  const TargetRegisterClass *DstRC =
281  getRegClass(MRI.getType(DstReg), DstRegBank);
282 
283  if (SrcRegBank.getID() == X86::GPRRegBankID &&
284  DstRegBank.getID() == X86::GPRRegBankID && SrcSize > DstSize &&
285  TargetRegisterInfo::isPhysicalRegister(SrcReg)) {
286  // Change the physical register to performe truncate.
287 
288  const TargetRegisterClass *SrcRC = getRegClassFromGRPhysReg(SrcReg);
289 
290  if (DstRC != SrcRC) {
291  I.getOperand(1).setSubReg(getSubRegIndex(DstRC));
292  I.getOperand(1).substPhysReg(SrcReg, TRI);
293  }
294  }
295 
296  // No need to constrain SrcReg. It will get constrained when
297  // we hit another of its use or its defs.
298  // Copies do not have constraints.
299  const TargetRegisterClass *OldRC = MRI.getRegClassOrNull(DstReg);
300  if (!OldRC || !DstRC->hasSubClassEq(OldRC)) {
301  if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
302  LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
303  << " operand\n");
304  return false;
305  }
306  }
307  I.setDesc(TII.get(X86::COPY));
308  return true;
309 }
310 
311 bool X86InstructionSelector::select(MachineInstr &I,
312  CodeGenCoverage &CoverageInfo) const {
313  assert(I.getParent() && "Instruction should be in a basic block!");
314  assert(I.getParent()->getParent() && "Instruction should be in a function!");
315 
316  MachineBasicBlock &MBB = *I.getParent();
317  MachineFunction &MF = *MBB.getParent();
318  MachineRegisterInfo &MRI = MF.getRegInfo();
319 
320  unsigned Opcode = I.getOpcode();
321  if (!isPreISelGenericOpcode(Opcode)) {
322  // Certain non-generic instructions also need some special handling.
323 
324  if (Opcode == TargetOpcode::LOAD_STACK_GUARD)
325  return false;
326 
327  if (I.isCopy())
328  return selectCopy(I, MRI);
329 
330  return true;
331  }
332 
334  "Generic instruction has unexpected implicit operands\n");
335 
336  if (selectImpl(I, CoverageInfo))
337  return true;
338 
339  LLVM_DEBUG(dbgs() << " C++ instruction selection: "; I.print(dbgs()));
340 
341  // TODO: This should be implemented by tblgen.
342  switch (I.getOpcode()) {
343  default:
344  return false;
345  case TargetOpcode::G_STORE:
346  case TargetOpcode::G_LOAD:
347  return selectLoadStoreOp(I, MRI, MF);
348  case TargetOpcode::G_GEP:
349  case TargetOpcode::G_FRAME_INDEX:
350  return selectFrameIndexOrGep(I, MRI, MF);
351  case TargetOpcode::G_GLOBAL_VALUE:
352  return selectGlobalValue(I, MRI, MF);
353  case TargetOpcode::G_CONSTANT:
354  return selectConstant(I, MRI, MF);
355  case TargetOpcode::G_FCONSTANT:
356  return materializeFP(I, MRI, MF);
357  case TargetOpcode::G_PTRTOINT:
358  case TargetOpcode::G_TRUNC:
359  return selectTruncOrPtrToInt(I, MRI, MF);
360  case TargetOpcode::G_INTTOPTR:
361  return selectCopy(I, MRI);
362  case TargetOpcode::G_ZEXT:
363  return selectZext(I, MRI, MF);
364  case TargetOpcode::G_ANYEXT:
365  return selectAnyext(I, MRI, MF);
366  case TargetOpcode::G_ICMP:
367  return selectCmp(I, MRI, MF);
368  case TargetOpcode::G_FCMP:
369  return selectFCmp(I, MRI, MF);
370  case TargetOpcode::G_UADDE:
371  return selectUadde(I, MRI, MF);
372  case TargetOpcode::G_UNMERGE_VALUES:
373  return selectUnmergeValues(I, MRI, MF, CoverageInfo);
374  case TargetOpcode::G_MERGE_VALUES:
375  case TargetOpcode::G_CONCAT_VECTORS:
376  return selectMergeValues(I, MRI, MF, CoverageInfo);
377  case TargetOpcode::G_EXTRACT:
378  return selectExtract(I, MRI, MF);
379  case TargetOpcode::G_INSERT:
380  return selectInsert(I, MRI, MF);
381  case TargetOpcode::G_BRCOND:
382  return selectCondBranch(I, MRI, MF);
383  case TargetOpcode::G_IMPLICIT_DEF:
384  case TargetOpcode::G_PHI:
385  return selectImplicitDefOrPHI(I, MRI);
386  case TargetOpcode::G_SHL:
387  case TargetOpcode::G_ASHR:
388  case TargetOpcode::G_LSHR:
389  return selectShift(I, MRI, MF);
390  case TargetOpcode::G_SDIV:
391  case TargetOpcode::G_UDIV:
392  case TargetOpcode::G_SREM:
393  case TargetOpcode::G_UREM:
394  return selectDivRem(I, MRI, MF);
395  case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
396  return selectIntrinsicWSideEffects(I, MRI, MF);
397  }
398 
399  return false;
400 }
401 
402 unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty,
403  const RegisterBank &RB,
404  unsigned Opc,
405  uint64_t Alignment) const {
406  bool Isload = (Opc == TargetOpcode::G_LOAD);
407  bool HasAVX = STI.hasAVX();
408  bool HasAVX512 = STI.hasAVX512();
409  bool HasVLX = STI.hasVLX();
410 
411  if (Ty == LLT::scalar(8)) {
412  if (X86::GPRRegBankID == RB.getID())
413  return Isload ? X86::MOV8rm : X86::MOV8mr;
414  } else if (Ty == LLT::scalar(16)) {
415  if (X86::GPRRegBankID == RB.getID())
416  return Isload ? X86::MOV16rm : X86::MOV16mr;
417  } else if (Ty == LLT::scalar(32) || Ty == LLT::pointer(0, 32)) {
418  if (X86::GPRRegBankID == RB.getID())
419  return Isload ? X86::MOV32rm : X86::MOV32mr;
420  if (X86::VECRRegBankID == RB.getID())
421  return Isload ? (HasAVX512 ? X86::VMOVSSZrm
422  : HasAVX ? X86::VMOVSSrm : X86::MOVSSrm)
423  : (HasAVX512 ? X86::VMOVSSZmr
424  : HasAVX ? X86::VMOVSSmr : X86::MOVSSmr);
425  } else if (Ty == LLT::scalar(64) || Ty == LLT::pointer(0, 64)) {
426  if (X86::GPRRegBankID == RB.getID())
427  return Isload ? X86::MOV64rm : X86::MOV64mr;
428  if (X86::VECRRegBankID == RB.getID())
429  return Isload ? (HasAVX512 ? X86::VMOVSDZrm
430  : HasAVX ? X86::VMOVSDrm : X86::MOVSDrm)
431  : (HasAVX512 ? X86::VMOVSDZmr
432  : HasAVX ? X86::VMOVSDmr : X86::MOVSDmr);
433  } else if (Ty.isVector() && Ty.getSizeInBits() == 128) {
434  if (Alignment >= 16)
435  return Isload ? (HasVLX ? X86::VMOVAPSZ128rm
436  : HasAVX512
437  ? X86::VMOVAPSZ128rm_NOVLX
438  : HasAVX ? X86::VMOVAPSrm : X86::MOVAPSrm)
439  : (HasVLX ? X86::VMOVAPSZ128mr
440  : HasAVX512
441  ? X86::VMOVAPSZ128mr_NOVLX
442  : HasAVX ? X86::VMOVAPSmr : X86::MOVAPSmr);
443  else
444  return Isload ? (HasVLX ? X86::VMOVUPSZ128rm
445  : HasAVX512
446  ? X86::VMOVUPSZ128rm_NOVLX
447  : HasAVX ? X86::VMOVUPSrm : X86::MOVUPSrm)
448  : (HasVLX ? X86::VMOVUPSZ128mr
449  : HasAVX512
450  ? X86::VMOVUPSZ128mr_NOVLX
451  : HasAVX ? X86::VMOVUPSmr : X86::MOVUPSmr);
452  } else if (Ty.isVector() && Ty.getSizeInBits() == 256) {
453  if (Alignment >= 32)
454  return Isload ? (HasVLX ? X86::VMOVAPSZ256rm
455  : HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX
456  : X86::VMOVAPSYrm)
457  : (HasVLX ? X86::VMOVAPSZ256mr
458  : HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX
459  : X86::VMOVAPSYmr);
460  else
461  return Isload ? (HasVLX ? X86::VMOVUPSZ256rm
462  : HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX
463  : X86::VMOVUPSYrm)
464  : (HasVLX ? X86::VMOVUPSZ256mr
465  : HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX
466  : X86::VMOVUPSYmr);
467  } else if (Ty.isVector() && Ty.getSizeInBits() == 512) {
468  if (Alignment >= 64)
469  return Isload ? X86::VMOVAPSZrm : X86::VMOVAPSZmr;
470  else
471  return Isload ? X86::VMOVUPSZrm : X86::VMOVUPSZmr;
472  }
473  return Opc;
474 }
475 
476 // Fill in an address from the given instruction.
477 static void X86SelectAddress(const MachineInstr &I,
478  const MachineRegisterInfo &MRI,
479  X86AddressMode &AM) {
480  assert(I.getOperand(0).isReg() && "unsupported opperand.");
481  assert(MRI.getType(I.getOperand(0).getReg()).isPointer() &&
482  "unsupported type.");
483 
484  if (I.getOpcode() == TargetOpcode::G_GEP) {
485  if (auto COff = getConstantVRegVal(I.getOperand(2).getReg(), MRI)) {
486  int64_t Imm = *COff;
487  if (isInt<32>(Imm)) { // Check for displacement overflow.
488  AM.Disp = static_cast<int32_t>(Imm);
489  AM.Base.Reg = I.getOperand(1).getReg();
490  return;
491  }
492  }
493  } else if (I.getOpcode() == TargetOpcode::G_FRAME_INDEX) {
494  AM.Base.FrameIndex = I.getOperand(1).getIndex();
495  AM.BaseType = X86AddressMode::FrameIndexBase;
496  return;
497  }
498 
499  // Default behavior.
500  AM.Base.Reg = I.getOperand(0).getReg();
501 }
502 
503 bool X86InstructionSelector::selectLoadStoreOp(MachineInstr &I,
504  MachineRegisterInfo &MRI,
505  MachineFunction &MF) const {
506  unsigned Opc = I.getOpcode();
507 
508  assert((Opc == TargetOpcode::G_STORE || Opc == TargetOpcode::G_LOAD) &&
509  "unexpected instruction");
510 
511  const unsigned DefReg = I.getOperand(0).getReg();
512  LLT Ty = MRI.getType(DefReg);
513  const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
514 
516  auto &MemOp = **I.memoperands_begin();
517  if (MemOp.isAtomic()) {
518  // Note: for unordered operations, we rely on the fact the appropriate MMO
519  // is already on the instruction we're mutating, and thus we don't need to
520  // make any changes. So long as we select an opcode which is capable of
521  // loading or storing the appropriate size atomically, the rest of the
522  // backend is required to respect the MMO state.
523  if (!MemOp.isUnordered()) {
524  LLVM_DEBUG(dbgs() << "Atomic ordering not supported yet\n");
525  return false;
526  }
527  if (MemOp.getAlignment() < Ty.getSizeInBits()/8) {
528  LLVM_DEBUG(dbgs() << "Unaligned atomics not supported yet\n");
529  return false;
530  }
531  }
532 
533  unsigned NewOpc = getLoadStoreOp(Ty, RB, Opc, MemOp.getAlignment());
534  if (NewOpc == Opc)
535  return false;
536 
537  X86AddressMode AM;
538  X86SelectAddress(*MRI.getVRegDef(I.getOperand(1).getReg()), MRI, AM);
539 
540  I.setDesc(TII.get(NewOpc));
541  MachineInstrBuilder MIB(MF, I);
542  if (Opc == TargetOpcode::G_LOAD) {
543  I.RemoveOperand(1);
544  addFullAddress(MIB, AM);
545  } else {
546  // G_STORE (VAL, Addr), X86Store instruction (Addr, VAL)
547  I.RemoveOperand(1);
548  I.RemoveOperand(0);
549  addFullAddress(MIB, AM).addUse(DefReg);
550  }
551  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
552 }
553 
554 static unsigned getLeaOP(LLT Ty, const X86Subtarget &STI) {
555  if (Ty == LLT::pointer(0, 64))
556  return X86::LEA64r;
557  else if (Ty == LLT::pointer(0, 32))
558  return STI.isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r;
559  else
560  llvm_unreachable("Can't get LEA opcode. Unsupported type.");
561 }
562 
563 bool X86InstructionSelector::selectFrameIndexOrGep(MachineInstr &I,
564  MachineRegisterInfo &MRI,
565  MachineFunction &MF) const {
566  unsigned Opc = I.getOpcode();
567 
568  assert((Opc == TargetOpcode::G_FRAME_INDEX || Opc == TargetOpcode::G_GEP) &&
569  "unexpected instruction");
570 
571  const unsigned DefReg = I.getOperand(0).getReg();
572  LLT Ty = MRI.getType(DefReg);
573 
574  // Use LEA to calculate frame index and GEP
575  unsigned NewOpc = getLeaOP(Ty, STI);
576  I.setDesc(TII.get(NewOpc));
577  MachineInstrBuilder MIB(MF, I);
578 
579  if (Opc == TargetOpcode::G_FRAME_INDEX) {
580  addOffset(MIB, 0);
581  } else {
582  MachineOperand &InxOp = I.getOperand(2);
583  I.addOperand(InxOp); // set IndexReg
584  InxOp.ChangeToImmediate(1); // set Scale
585  MIB.addImm(0).addReg(0);
586  }
587 
588  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
589 }
590 
591 bool X86InstructionSelector::selectGlobalValue(MachineInstr &I,
592  MachineRegisterInfo &MRI,
593  MachineFunction &MF) const {
594  assert((I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE) &&
595  "unexpected instruction");
596 
597  auto GV = I.getOperand(1).getGlobal();
598  if (GV->isThreadLocal()) {
599  return false; // TODO: we don't support TLS yet.
600  }
601 
602  // Can't handle alternate code models yet.
603  if (TM.getCodeModel() != CodeModel::Small)
604  return false;
605 
606  X86AddressMode AM;
607  AM.GV = GV;
608  AM.GVOpFlags = STI.classifyGlobalReference(GV);
609 
610  // TODO: The ABI requires an extra load. not supported yet.
612  return false;
613 
614  // TODO: This reference is relative to the pic base. not supported yet.
616  return false;
617 
618  if (STI.isPICStyleRIPRel()) {
619  // Use rip-relative addressing.
620  assert(AM.Base.Reg == 0 && AM.IndexReg == 0);
621  AM.Base.Reg = X86::RIP;
622  }
623 
624  const unsigned DefReg = I.getOperand(0).getReg();
625  LLT Ty = MRI.getType(DefReg);
626  unsigned NewOpc = getLeaOP(Ty, STI);
627 
628  I.setDesc(TII.get(NewOpc));
629  MachineInstrBuilder MIB(MF, I);
630 
631  I.RemoveOperand(1);
632  addFullAddress(MIB, AM);
633 
634  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
635 }
636 
637 bool X86InstructionSelector::selectConstant(MachineInstr &I,
638  MachineRegisterInfo &MRI,
639  MachineFunction &MF) const {
640  assert((I.getOpcode() == TargetOpcode::G_CONSTANT) &&
641  "unexpected instruction");
642 
643  const unsigned DefReg = I.getOperand(0).getReg();
644  LLT Ty = MRI.getType(DefReg);
645 
646  if (RBI.getRegBank(DefReg, MRI, TRI)->getID() != X86::GPRRegBankID)
647  return false;
648 
649  uint64_t Val = 0;
650  if (I.getOperand(1).isCImm()) {
651  Val = I.getOperand(1).getCImm()->getZExtValue();
652  I.getOperand(1).ChangeToImmediate(Val);
653  } else if (I.getOperand(1).isImm()) {
654  Val = I.getOperand(1).getImm();
655  } else
656  llvm_unreachable("Unsupported operand type.");
657 
658  unsigned NewOpc;
659  switch (Ty.getSizeInBits()) {
660  case 8:
661  NewOpc = X86::MOV8ri;
662  break;
663  case 16:
664  NewOpc = X86::MOV16ri;
665  break;
666  case 32:
667  NewOpc = X86::MOV32ri;
668  break;
669  case 64:
670  // TODO: in case isUInt<32>(Val), X86::MOV32ri can be used
671  if (isInt<32>(Val))
672  NewOpc = X86::MOV64ri32;
673  else
674  NewOpc = X86::MOV64ri;
675  break;
676  default:
677  llvm_unreachable("Can't select G_CONSTANT, unsupported type.");
678  }
679 
680  I.setDesc(TII.get(NewOpc));
681  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
682 }
683 
684 // Helper function for selectTruncOrPtrToInt and selectAnyext.
685 // Returns true if DstRC lives on a floating register class and
686 // SrcRC lives on a 128-bit vector class.
687 static bool canTurnIntoCOPY(const TargetRegisterClass *DstRC,
688  const TargetRegisterClass *SrcRC) {
689  return (DstRC == &X86::FR32RegClass || DstRC == &X86::FR32XRegClass ||
690  DstRC == &X86::FR64RegClass || DstRC == &X86::FR64XRegClass) &&
691  (SrcRC == &X86::VR128RegClass || SrcRC == &X86::VR128XRegClass);
692 }
693 
694 bool X86InstructionSelector::selectTurnIntoCOPY(
695  MachineInstr &I, MachineRegisterInfo &MRI, const unsigned DstReg,
696  const TargetRegisterClass *DstRC, const unsigned SrcReg,
697  const TargetRegisterClass *SrcRC) const {
698 
699  if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
700  !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
701  LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
702  << " operand\n");
703  return false;
704  }
705  I.setDesc(TII.get(X86::COPY));
706  return true;
707 }
708 
709 bool X86InstructionSelector::selectTruncOrPtrToInt(MachineInstr &I,
710  MachineRegisterInfo &MRI,
711  MachineFunction &MF) const {
712  assert((I.getOpcode() == TargetOpcode::G_TRUNC ||
713  I.getOpcode() == TargetOpcode::G_PTRTOINT) &&
714  "unexpected instruction");
715 
716  const unsigned DstReg = I.getOperand(0).getReg();
717  const unsigned SrcReg = I.getOperand(1).getReg();
718 
719  const LLT DstTy = MRI.getType(DstReg);
720  const LLT SrcTy = MRI.getType(SrcReg);
721 
722  const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
723  const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
724 
725  if (DstRB.getID() != SrcRB.getID()) {
726  LLVM_DEBUG(dbgs() << TII.getName(I.getOpcode())
727  << " input/output on different banks\n");
728  return false;
729  }
730 
731  const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB);
732  const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB);
733 
734  if (!DstRC || !SrcRC)
735  return false;
736 
737  // If that's truncation of the value that lives on the vector class and goes
738  // into the floating class, just replace it with copy, as we are able to
739  // select it as a regular move.
740  if (canTurnIntoCOPY(DstRC, SrcRC))
741  return selectTurnIntoCOPY(I, MRI, DstReg, DstRC, SrcReg, SrcRC);
742 
743  if (DstRB.getID() != X86::GPRRegBankID)
744  return false;
745 
746  unsigned SubIdx;
747  if (DstRC == SrcRC) {
748  // Nothing to be done
749  SubIdx = X86::NoSubRegister;
750  } else if (DstRC == &X86::GR32RegClass) {
751  SubIdx = X86::sub_32bit;
752  } else if (DstRC == &X86::GR16RegClass) {
753  SubIdx = X86::sub_16bit;
754  } else if (DstRC == &X86::GR8RegClass) {
755  SubIdx = X86::sub_8bit;
756  } else {
757  return false;
758  }
759 
760  SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubIdx);
761 
762  if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
763  !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
764  LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
765  << "\n");
766  return false;
767  }
768 
769  I.getOperand(1).setSubReg(SubIdx);
770 
771  I.setDesc(TII.get(X86::COPY));
772  return true;
773 }
774 
775 bool X86InstructionSelector::selectZext(MachineInstr &I,
776  MachineRegisterInfo &MRI,
777  MachineFunction &MF) const {
778  assert((I.getOpcode() == TargetOpcode::G_ZEXT) && "unexpected instruction");
779 
780  const unsigned DstReg = I.getOperand(0).getReg();
781  const unsigned SrcReg = I.getOperand(1).getReg();
782 
783  const LLT DstTy = MRI.getType(DstReg);
784  const LLT SrcTy = MRI.getType(SrcReg);
785 
786  assert(!(SrcTy == LLT::scalar(8) && DstTy == LLT::scalar(32)) &&
787  "8=>32 Zext is handled by tablegen");
788  assert(!(SrcTy == LLT::scalar(16) && DstTy == LLT::scalar(32)) &&
789  "16=>32 Zext is handled by tablegen");
790 
791  const static struct ZextEntry {
792  LLT SrcTy;
793  LLT DstTy;
794  unsigned MovOp;
795  bool NeedSubregToReg;
796  } OpTable[] = {
797  {LLT::scalar(8), LLT::scalar(16), X86::MOVZX16rr8, false}, // i8 => i16
798  {LLT::scalar(8), LLT::scalar(64), X86::MOVZX32rr8, true}, // i8 => i64
799  {LLT::scalar(16), LLT::scalar(64), X86::MOVZX32rr16, true}, // i16 => i64
800  {LLT::scalar(32), LLT::scalar(64), 0, true} // i32 => i64
801  };
802 
803  auto ZextEntryIt =
804  std::find_if(std::begin(OpTable), std::end(OpTable),
805  [SrcTy, DstTy](const ZextEntry &El) {
806  return El.DstTy == DstTy && El.SrcTy == SrcTy;
807  });
808 
809  // Here we try to select Zext into a MOVZ and/or SUBREG_TO_REG instruction.
810  if (ZextEntryIt != std::end(OpTable)) {
811  const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
812  const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
813  const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB);
814  const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB);
815 
816  if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
817  !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
818  LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
819  << " operand\n");
820  return false;
821  }
822 
823  unsigned TransitRegTo = DstReg;
824  unsigned TransitRegFrom = SrcReg;
825  if (ZextEntryIt->MovOp) {
826  // If we select Zext into MOVZ + SUBREG_TO_REG, we need to have
827  // a transit register in between: create it here.
828  if (ZextEntryIt->NeedSubregToReg) {
829  TransitRegFrom = MRI.createVirtualRegister(
830  getRegClass(LLT::scalar(32), DstReg, MRI));
831  TransitRegTo = TransitRegFrom;
832  }
833 
834  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(ZextEntryIt->MovOp))
835  .addDef(TransitRegTo)
836  .addReg(SrcReg);
837  }
838  if (ZextEntryIt->NeedSubregToReg) {
839  BuildMI(*I.getParent(), I, I.getDebugLoc(),
840  TII.get(TargetOpcode::SUBREG_TO_REG))
841  .addDef(DstReg)
842  .addImm(0)
843  .addReg(TransitRegFrom)
844  .addImm(X86::sub_32bit);
845  }
846  I.eraseFromParent();
847  return true;
848  }
849 
850  if (SrcTy != LLT::scalar(1))
851  return false;
852 
853  unsigned AndOpc;
854  if (DstTy == LLT::scalar(8))
855  AndOpc = X86::AND8ri;
856  else if (DstTy == LLT::scalar(16))
857  AndOpc = X86::AND16ri8;
858  else if (DstTy == LLT::scalar(32))
859  AndOpc = X86::AND32ri8;
860  else if (DstTy == LLT::scalar(64))
861  AndOpc = X86::AND64ri8;
862  else
863  return false;
864 
865  unsigned DefReg = SrcReg;
866  if (DstTy != LLT::scalar(8)) {
867  DefReg = MRI.createVirtualRegister(getRegClass(DstTy, DstReg, MRI));
868  BuildMI(*I.getParent(), I, I.getDebugLoc(),
869  TII.get(TargetOpcode::SUBREG_TO_REG), DefReg)
870  .addImm(0)
871  .addReg(SrcReg)
872  .addImm(X86::sub_8bit);
873  }
874 
875  MachineInstr &AndInst =
876  *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AndOpc), DstReg)
877  .addReg(DefReg)
878  .addImm(1);
879 
880  constrainSelectedInstRegOperands(AndInst, TII, TRI, RBI);
881 
882  I.eraseFromParent();
883  return true;
884 }
885 
886 bool X86InstructionSelector::selectAnyext(MachineInstr &I,
887  MachineRegisterInfo &MRI,
888  MachineFunction &MF) const {
889  assert((I.getOpcode() == TargetOpcode::G_ANYEXT) && "unexpected instruction");
890 
891  const unsigned DstReg = I.getOperand(0).getReg();
892  const unsigned SrcReg = I.getOperand(1).getReg();
893 
894  const LLT DstTy = MRI.getType(DstReg);
895  const LLT SrcTy = MRI.getType(SrcReg);
896 
897  const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
898  const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
899 
900  assert(DstRB.getID() == SrcRB.getID() &&
901  "G_ANYEXT input/output on different banks\n");
902 
903  assert(DstTy.getSizeInBits() > SrcTy.getSizeInBits() &&
904  "G_ANYEXT incorrect operand size");
905 
906  const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB);
907  const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB);
908 
909  // If that's ANY_EXT of the value that lives on the floating class and goes
910  // into the vector class, just replace it with copy, as we are able to select
911  // it as a regular move.
912  if (canTurnIntoCOPY(SrcRC, DstRC))
913  return selectTurnIntoCOPY(I, MRI, SrcReg, SrcRC, DstReg, DstRC);
914 
915  if (DstRB.getID() != X86::GPRRegBankID)
916  return false;
917 
918  if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
919  !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
920  LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
921  << " operand\n");
922  return false;
923  }
924 
925  if (SrcRC == DstRC) {
926  I.setDesc(TII.get(X86::COPY));
927  return true;
928  }
929 
930  BuildMI(*I.getParent(), I, I.getDebugLoc(),
931  TII.get(TargetOpcode::SUBREG_TO_REG))
932  .addDef(DstReg)
933  .addImm(0)
934  .addReg(SrcReg)
935  .addImm(getSubRegIndex(SrcRC));
936 
937  I.eraseFromParent();
938  return true;
939 }
940 
941 bool X86InstructionSelector::selectCmp(MachineInstr &I,
942  MachineRegisterInfo &MRI,
943  MachineFunction &MF) const {
944  assert((I.getOpcode() == TargetOpcode::G_ICMP) && "unexpected instruction");
945 
946  X86::CondCode CC;
947  bool SwapArgs;
948  std::tie(CC, SwapArgs) = X86::getX86ConditionCode(
950 
951  unsigned LHS = I.getOperand(2).getReg();
952  unsigned RHS = I.getOperand(3).getReg();
953 
954  if (SwapArgs)
955  std::swap(LHS, RHS);
956 
957  unsigned OpCmp;
958  LLT Ty = MRI.getType(LHS);
959 
960  switch (Ty.getSizeInBits()) {
961  default:
962  return false;
963  case 8:
964  OpCmp = X86::CMP8rr;
965  break;
966  case 16:
967  OpCmp = X86::CMP16rr;
968  break;
969  case 32:
970  OpCmp = X86::CMP32rr;
971  break;
972  case 64:
973  OpCmp = X86::CMP64rr;
974  break;
975  }
976 
978  *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp))
979  .addReg(LHS)
980  .addReg(RHS);
981 
982  MachineInstr &SetInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
983  TII.get(X86::SETCCr), I.getOperand(0).getReg()).addImm(CC);
984 
985  constrainSelectedInstRegOperands(CmpInst, TII, TRI, RBI);
986  constrainSelectedInstRegOperands(SetInst, TII, TRI, RBI);
987 
988  I.eraseFromParent();
989  return true;
990 }
991 
992 bool X86InstructionSelector::selectFCmp(MachineInstr &I,
993  MachineRegisterInfo &MRI,
994  MachineFunction &MF) const {
995  assert((I.getOpcode() == TargetOpcode::G_FCMP) && "unexpected instruction");
996 
997  unsigned LhsReg = I.getOperand(2).getReg();
998  unsigned RhsReg = I.getOperand(3).getReg();
1001 
1002  // FCMP_OEQ and FCMP_UNE cannot be checked with a single instruction.
1003  static const uint16_t SETFOpcTable[2][3] = {
1004  {X86::COND_E, X86::COND_NP, X86::AND8rr},
1005  {X86::COND_NE, X86::COND_P, X86::OR8rr}};
1006  const uint16_t *SETFOpc = nullptr;
1007  switch (Predicate) {
1008  default:
1009  break;
1010  case CmpInst::FCMP_OEQ:
1011  SETFOpc = &SETFOpcTable[0][0];
1012  break;
1013  case CmpInst::FCMP_UNE:
1014  SETFOpc = &SETFOpcTable[1][0];
1015  break;
1016  }
1017 
1018  // Compute the opcode for the CMP instruction.
1019  unsigned OpCmp;
1020  LLT Ty = MRI.getType(LhsReg);
1021  switch (Ty.getSizeInBits()) {
1022  default:
1023  return false;
1024  case 32:
1025  OpCmp = X86::UCOMISSrr;
1026  break;
1027  case 64:
1028  OpCmp = X86::UCOMISDrr;
1029  break;
1030  }
1031 
1032  unsigned ResultReg = I.getOperand(0).getReg();
1033  RBI.constrainGenericRegister(
1034  ResultReg,
1035  *getRegClass(LLT::scalar(8), *RBI.getRegBank(ResultReg, MRI, TRI)), MRI);
1036  if (SETFOpc) {
1038  *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp))
1039  .addReg(LhsReg)
1040  .addReg(RhsReg);
1041 
1042  unsigned FlagReg1 = MRI.createVirtualRegister(&X86::GR8RegClass);
1043  unsigned FlagReg2 = MRI.createVirtualRegister(&X86::GR8RegClass);
1044  MachineInstr &Set1 = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1045  TII.get(X86::SETCCr), FlagReg1).addImm(SETFOpc[0]);
1046  MachineInstr &Set2 = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1047  TII.get(X86::SETCCr), FlagReg2).addImm(SETFOpc[1]);
1048  MachineInstr &Set3 = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1049  TII.get(SETFOpc[2]), ResultReg)
1050  .addReg(FlagReg1)
1051  .addReg(FlagReg2);
1052  constrainSelectedInstRegOperands(CmpInst, TII, TRI, RBI);
1056 
1057  I.eraseFromParent();
1058  return true;
1059  }
1060 
1061  X86::CondCode CC;
1062  bool SwapArgs;
1063  std::tie(CC, SwapArgs) = X86::getX86ConditionCode(Predicate);
1064  assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code.");
1065 
1066  if (SwapArgs)
1067  std::swap(LhsReg, RhsReg);
1068 
1069  // Emit a compare of LHS/RHS.
1071  *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp))
1072  .addReg(LhsReg)
1073  .addReg(RhsReg);
1074 
1075  MachineInstr &Set =
1076  *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::SETCCr), ResultReg).addImm(CC);
1077  constrainSelectedInstRegOperands(CmpInst, TII, TRI, RBI);
1079  I.eraseFromParent();
1080  return true;
1081 }
1082 
1083 bool X86InstructionSelector::selectUadde(MachineInstr &I,
1084  MachineRegisterInfo &MRI,
1085  MachineFunction &MF) const {
1086  assert((I.getOpcode() == TargetOpcode::G_UADDE) && "unexpected instruction");
1087 
1088  const unsigned DstReg = I.getOperand(0).getReg();
1089  const unsigned CarryOutReg = I.getOperand(1).getReg();
1090  const unsigned Op0Reg = I.getOperand(2).getReg();
1091  const unsigned Op1Reg = I.getOperand(3).getReg();
1092  unsigned CarryInReg = I.getOperand(4).getReg();
1093 
1094  const LLT DstTy = MRI.getType(DstReg);
1095 
1096  if (DstTy != LLT::scalar(32))
1097  return false;
1098 
1099  // find CarryIn def instruction.
1100  MachineInstr *Def = MRI.getVRegDef(CarryInReg);
1101  while (Def->getOpcode() == TargetOpcode::G_TRUNC) {
1102  CarryInReg = Def->getOperand(1).getReg();
1103  Def = MRI.getVRegDef(CarryInReg);
1104  }
1105 
1106  unsigned Opcode;
1107  if (Def->getOpcode() == TargetOpcode::G_UADDE) {
1108  // carry set by prev ADD.
1109 
1110  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), X86::EFLAGS)
1111  .addReg(CarryInReg);
1112 
1113  if (!RBI.constrainGenericRegister(CarryInReg, X86::GR32RegClass, MRI))
1114  return false;
1115 
1116  Opcode = X86::ADC32rr;
1117  } else if (auto val = getConstantVRegVal(CarryInReg, MRI)) {
1118  // carry is constant, support only 0.
1119  if (*val != 0)
1120  return false;
1121 
1122  Opcode = X86::ADD32rr;
1123  } else
1124  return false;
1125 
1126  MachineInstr &AddInst =
1127  *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode), DstReg)
1128  .addReg(Op0Reg)
1129  .addReg(Op1Reg);
1130 
1131  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), CarryOutReg)
1132  .addReg(X86::EFLAGS);
1133 
1134  if (!constrainSelectedInstRegOperands(AddInst, TII, TRI, RBI) ||
1135  !RBI.constrainGenericRegister(CarryOutReg, X86::GR32RegClass, MRI))
1136  return false;
1137 
1138  I.eraseFromParent();
1139  return true;
1140 }
1141 
1142 bool X86InstructionSelector::selectExtract(MachineInstr &I,
1143  MachineRegisterInfo &MRI,
1144  MachineFunction &MF) const {
1145  assert((I.getOpcode() == TargetOpcode::G_EXTRACT) &&
1146  "unexpected instruction");
1147 
1148  const unsigned DstReg = I.getOperand(0).getReg();
1149  const unsigned SrcReg = I.getOperand(1).getReg();
1150  int64_t Index = I.getOperand(2).getImm();
1151 
1152  const LLT DstTy = MRI.getType(DstReg);
1153  const LLT SrcTy = MRI.getType(SrcReg);
1154 
1155  // Meanwile handle vector type only.
1156  if (!DstTy.isVector())
1157  return false;
1158 
1159  if (Index % DstTy.getSizeInBits() != 0)
1160  return false; // Not extract subvector.
1161 
1162  if (Index == 0) {
1163  // Replace by extract subreg copy.
1164  if (!emitExtractSubreg(DstReg, SrcReg, I, MRI, MF))
1165  return false;
1166 
1167  I.eraseFromParent();
1168  return true;
1169  }
1170 
1171  bool HasAVX = STI.hasAVX();
1172  bool HasAVX512 = STI.hasAVX512();
1173  bool HasVLX = STI.hasVLX();
1174 
1175  if (SrcTy.getSizeInBits() == 256 && DstTy.getSizeInBits() == 128) {
1176  if (HasVLX)
1177  I.setDesc(TII.get(X86::VEXTRACTF32x4Z256rr));
1178  else if (HasAVX)
1179  I.setDesc(TII.get(X86::VEXTRACTF128rr));
1180  else
1181  return false;
1182  } else if (SrcTy.getSizeInBits() == 512 && HasAVX512) {
1183  if (DstTy.getSizeInBits() == 128)
1184  I.setDesc(TII.get(X86::VEXTRACTF32x4Zrr));
1185  else if (DstTy.getSizeInBits() == 256)
1186  I.setDesc(TII.get(X86::VEXTRACTF64x4Zrr));
1187  else
1188  return false;
1189  } else
1190  return false;
1191 
1192  // Convert to X86 VEXTRACT immediate.
1193  Index = Index / DstTy.getSizeInBits();
1194  I.getOperand(2).setImm(Index);
1195 
1196  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1197 }
1198 
1199 bool X86InstructionSelector::emitExtractSubreg(unsigned DstReg, unsigned SrcReg,
1200  MachineInstr &I,
1201  MachineRegisterInfo &MRI,
1202  MachineFunction &MF) const {
1203  const LLT DstTy = MRI.getType(DstReg);
1204  const LLT SrcTy = MRI.getType(SrcReg);
1205  unsigned SubIdx = X86::NoSubRegister;
1206 
1207  if (!DstTy.isVector() || !SrcTy.isVector())
1208  return false;
1209 
1210  assert(SrcTy.getSizeInBits() > DstTy.getSizeInBits() &&
1211  "Incorrect Src/Dst register size");
1212 
1213  if (DstTy.getSizeInBits() == 128)
1214  SubIdx = X86::sub_xmm;
1215  else if (DstTy.getSizeInBits() == 256)
1216  SubIdx = X86::sub_ymm;
1217  else
1218  return false;
1219 
1220  const TargetRegisterClass *DstRC = getRegClass(DstTy, DstReg, MRI);
1221  const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcReg, MRI);
1222 
1223  SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubIdx);
1224 
1225  if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
1226  !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
1227  LLVM_DEBUG(dbgs() << "Failed to constrain G_TRUNC\n");
1228  return false;
1229  }
1230 
1231  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), DstReg)
1232  .addReg(SrcReg, 0, SubIdx);
1233 
1234  return true;
1235 }
1236 
1237 bool X86InstructionSelector::emitInsertSubreg(unsigned DstReg, unsigned SrcReg,
1238  MachineInstr &I,
1239  MachineRegisterInfo &MRI,
1240  MachineFunction &MF) const {
1241  const LLT DstTy = MRI.getType(DstReg);
1242  const LLT SrcTy = MRI.getType(SrcReg);
1243  unsigned SubIdx = X86::NoSubRegister;
1244 
1245  // TODO: support scalar types
1246  if (!DstTy.isVector() || !SrcTy.isVector())
1247  return false;
1248 
1249  assert(SrcTy.getSizeInBits() < DstTy.getSizeInBits() &&
1250  "Incorrect Src/Dst register size");
1251 
1252  if (SrcTy.getSizeInBits() == 128)
1253  SubIdx = X86::sub_xmm;
1254  else if (SrcTy.getSizeInBits() == 256)
1255  SubIdx = X86::sub_ymm;
1256  else
1257  return false;
1258 
1259  const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcReg, MRI);
1260  const TargetRegisterClass *DstRC = getRegClass(DstTy, DstReg, MRI);
1261 
1262  if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
1263  !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
1264  LLVM_DEBUG(dbgs() << "Failed to constrain INSERT_SUBREG\n");
1265  return false;
1266  }
1267 
1268  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY))
1269  .addReg(DstReg, RegState::DefineNoRead, SubIdx)
1270  .addReg(SrcReg);
1271 
1272  return true;
1273 }
1274 
1275 bool X86InstructionSelector::selectInsert(MachineInstr &I,
1276  MachineRegisterInfo &MRI,
1277  MachineFunction &MF) const {
1278  assert((I.getOpcode() == TargetOpcode::G_INSERT) && "unexpected instruction");
1279 
1280  const unsigned DstReg = I.getOperand(0).getReg();
1281  const unsigned SrcReg = I.getOperand(1).getReg();
1282  const unsigned InsertReg = I.getOperand(2).getReg();
1283  int64_t Index = I.getOperand(3).getImm();
1284 
1285  const LLT DstTy = MRI.getType(DstReg);
1286  const LLT InsertRegTy = MRI.getType(InsertReg);
1287 
1288  // Meanwile handle vector type only.
1289  if (!DstTy.isVector())
1290  return false;
1291 
1292  if (Index % InsertRegTy.getSizeInBits() != 0)
1293  return false; // Not insert subvector.
1294 
1295  if (Index == 0 && MRI.getVRegDef(SrcReg)->isImplicitDef()) {
1296  // Replace by subreg copy.
1297  if (!emitInsertSubreg(DstReg, InsertReg, I, MRI, MF))
1298  return false;
1299 
1300  I.eraseFromParent();
1301  return true;
1302  }
1303 
1304  bool HasAVX = STI.hasAVX();
1305  bool HasAVX512 = STI.hasAVX512();
1306  bool HasVLX = STI.hasVLX();
1307 
1308  if (DstTy.getSizeInBits() == 256 && InsertRegTy.getSizeInBits() == 128) {
1309  if (HasVLX)
1310  I.setDesc(TII.get(X86::VINSERTF32x4Z256rr));
1311  else if (HasAVX)
1312  I.setDesc(TII.get(X86::VINSERTF128rr));
1313  else
1314  return false;
1315  } else if (DstTy.getSizeInBits() == 512 && HasAVX512) {
1316  if (InsertRegTy.getSizeInBits() == 128)
1317  I.setDesc(TII.get(X86::VINSERTF32x4Zrr));
1318  else if (InsertRegTy.getSizeInBits() == 256)
1319  I.setDesc(TII.get(X86::VINSERTF64x4Zrr));
1320  else
1321  return false;
1322  } else
1323  return false;
1324 
1325  // Convert to X86 VINSERT immediate.
1326  Index = Index / InsertRegTy.getSizeInBits();
1327 
1328  I.getOperand(3).setImm(Index);
1329 
1330  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1331 }
1332 
1335  CodeGenCoverage &CoverageInfo) const {
1336  assert((I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES) &&
1337  "unexpected instruction");
1338 
1339  // Split to extracts.
1340  unsigned NumDefs = I.getNumOperands() - 1;
1341  unsigned SrcReg = I.getOperand(NumDefs).getReg();
1342  unsigned DefSize = MRI.getType(I.getOperand(0).getReg()).getSizeInBits();
1343 
1344  for (unsigned Idx = 0; Idx < NumDefs; ++Idx) {
1345  MachineInstr &ExtrInst =
1346  *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1347  TII.get(TargetOpcode::G_EXTRACT), I.getOperand(Idx).getReg())
1348  .addReg(SrcReg)
1349  .addImm(Idx * DefSize);
1350 
1351  if (!select(ExtrInst, CoverageInfo))
1352  return false;
1353  }
1354 
1355  I.eraseFromParent();
1356  return true;
1357 }
1358 
1361  CodeGenCoverage &CoverageInfo) const {
1362  assert((I.getOpcode() == TargetOpcode::G_MERGE_VALUES ||
1363  I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS) &&
1364  "unexpected instruction");
1365 
1366  // Split to inserts.
1367  unsigned DstReg = I.getOperand(0).getReg();
1368  unsigned SrcReg0 = I.getOperand(1).getReg();
1369 
1370  const LLT DstTy = MRI.getType(DstReg);
1371  const LLT SrcTy = MRI.getType(SrcReg0);
1372  unsigned SrcSize = SrcTy.getSizeInBits();
1373 
1374  const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI);
1375 
1376  // For the first src use insertSubReg.
1377  unsigned DefReg = MRI.createGenericVirtualRegister(DstTy);
1378  MRI.setRegBank(DefReg, RegBank);
1379  if (!emitInsertSubreg(DefReg, I.getOperand(1).getReg(), I, MRI, MF))
1380  return false;
1381 
1382  for (unsigned Idx = 2; Idx < I.getNumOperands(); ++Idx) {
1383  unsigned Tmp = MRI.createGenericVirtualRegister(DstTy);
1384  MRI.setRegBank(Tmp, RegBank);
1385 
1386  MachineInstr &InsertInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1387  TII.get(TargetOpcode::G_INSERT), Tmp)
1388  .addReg(DefReg)
1389  .addReg(I.getOperand(Idx).getReg())
1390  .addImm((Idx - 1) * SrcSize);
1391 
1392  DefReg = Tmp;
1393 
1394  if (!select(InsertInst, CoverageInfo))
1395  return false;
1396  }
1397 
1398  MachineInstr &CopyInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1399  TII.get(TargetOpcode::COPY), DstReg)
1400  .addReg(DefReg);
1401 
1402  if (!select(CopyInst, CoverageInfo))
1403  return false;
1404 
1405  I.eraseFromParent();
1406  return true;
1407 }
1408 
1409 bool X86InstructionSelector::selectCondBranch(MachineInstr &I,
1410  MachineRegisterInfo &MRI,
1411  MachineFunction &MF) const {
1412  assert((I.getOpcode() == TargetOpcode::G_BRCOND) && "unexpected instruction");
1413 
1414  const unsigned CondReg = I.getOperand(0).getReg();
1415  MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
1416 
1417  MachineInstr &TestInst =
1418  *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::TEST8ri))
1419  .addReg(CondReg)
1420  .addImm(1);
1421  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::JCC_1))
1422  .addMBB(DestMBB).addImm(X86::COND_NE);
1423 
1424  constrainSelectedInstRegOperands(TestInst, TII, TRI, RBI);
1425 
1426  I.eraseFromParent();
1427  return true;
1428 }
1429 
1430 bool X86InstructionSelector::materializeFP(MachineInstr &I,
1431  MachineRegisterInfo &MRI,
1432  MachineFunction &MF) const {
1433  assert((I.getOpcode() == TargetOpcode::G_FCONSTANT) &&
1434  "unexpected instruction");
1435 
1436  // Can't handle alternate code models yet.
1437  CodeModel::Model CM = TM.getCodeModel();
1438  if (CM != CodeModel::Small && CM != CodeModel::Large)
1439  return false;
1440 
1441  const unsigned DstReg = I.getOperand(0).getReg();
1442  const LLT DstTy = MRI.getType(DstReg);
1443  const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI);
1444  unsigned Align = DstTy.getSizeInBits();
1445  const DebugLoc &DbgLoc = I.getDebugLoc();
1446 
1447  unsigned Opc = getLoadStoreOp(DstTy, RegBank, TargetOpcode::G_LOAD, Align);
1448 
1449  // Create the load from the constant pool.
1450  const ConstantFP *CFP = I.getOperand(1).getFPImm();
1451  unsigned CPI = MF.getConstantPool()->getConstantPoolIndex(CFP, Align);
1452  MachineInstr *LoadInst = nullptr;
1453  unsigned char OpFlag = STI.classifyLocalReference(nullptr);
1454 
1455  if (CM == CodeModel::Large && STI.is64Bit()) {
1456  // Under X86-64 non-small code model, GV (and friends) are 64-bits, so
1457  // they cannot be folded into immediate fields.
1458 
1459  unsigned AddrReg = MRI.createVirtualRegister(&X86::GR64RegClass);
1460  BuildMI(*I.getParent(), I, DbgLoc, TII.get(X86::MOV64ri), AddrReg)
1461  .addConstantPoolIndex(CPI, 0, OpFlag);
1462 
1464  MachinePointerInfo::getConstantPool(MF), MachineMemOperand::MOLoad,
1466 
1467  LoadInst =
1468  addDirectMem(BuildMI(*I.getParent(), I, DbgLoc, TII.get(Opc), DstReg),
1469  AddrReg)
1470  .addMemOperand(MMO);
1471 
1472  } else if (CM == CodeModel::Small || !STI.is64Bit()) {
1473  // Handle the case when globals fit in our immediate field.
1474  // This is true for X86-32 always and X86-64 when in -mcmodel=small mode.
1475 
1476  // x86-32 PIC requires a PIC base register for constant pools.
1477  unsigned PICBase = 0;
1478  if (OpFlag == X86II::MO_PIC_BASE_OFFSET || OpFlag == X86II::MO_GOTOFF) {
1479  // PICBase can be allocated by TII.getGlobalBaseReg(&MF).
1480  // In DAGISEL the code that initialize it generated by the CGBR pass.
1481  return false; // TODO support the mode.
1482  } else if (STI.is64Bit() && TM.getCodeModel() == CodeModel::Small)
1483  PICBase = X86::RIP;
1484 
1485  LoadInst = addConstantPoolReference(
1486  BuildMI(*I.getParent(), I, DbgLoc, TII.get(Opc), DstReg), CPI, PICBase,
1487  OpFlag);
1488  } else
1489  return false;
1490 
1491  constrainSelectedInstRegOperands(*LoadInst, TII, TRI, RBI);
1492  I.eraseFromParent();
1493  return true;
1494 }
1495 
1496 bool X86InstructionSelector::selectImplicitDefOrPHI(
1497  MachineInstr &I, MachineRegisterInfo &MRI) const {
1498  assert((I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF ||
1499  I.getOpcode() == TargetOpcode::G_PHI) &&
1500  "unexpected instruction");
1501 
1502  unsigned DstReg = I.getOperand(0).getReg();
1503 
1504  if (!MRI.getRegClassOrNull(DstReg)) {
1505  const LLT DstTy = MRI.getType(DstReg);
1506  const TargetRegisterClass *RC = getRegClass(DstTy, DstReg, MRI);
1507 
1508  if (!RBI.constrainGenericRegister(DstReg, *RC, MRI)) {
1509  LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1510  << " operand\n");
1511  return false;
1512  }
1513  }
1514 
1515  if (I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
1516  I.setDesc(TII.get(X86::IMPLICIT_DEF));
1517  else
1518  I.setDesc(TII.get(X86::PHI));
1519 
1520  return true;
1521 }
1522 
1523 // Currently GlobalIsel TableGen generates patterns for shift imm and shift 1,
1524 // but with shiftCount i8. In G_LSHR/G_ASHR/G_SHL like LLVM-IR both arguments
1525 // has the same type, so for now only shift i8 can use auto generated
1526 // TableGen patterns.
1527 bool X86InstructionSelector::selectShift(MachineInstr &I,
1528  MachineRegisterInfo &MRI,
1529  MachineFunction &MF) const {
1530 
1531  assert((I.getOpcode() == TargetOpcode::G_SHL ||
1532  I.getOpcode() == TargetOpcode::G_ASHR ||
1533  I.getOpcode() == TargetOpcode::G_LSHR) &&
1534  "unexpected instruction");
1535 
1536  unsigned DstReg = I.getOperand(0).getReg();
1537  const LLT DstTy = MRI.getType(DstReg);
1538  const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
1539 
1540  const static struct ShiftEntry {
1541  unsigned SizeInBits;
1542  unsigned OpLSHR;
1543  unsigned OpASHR;
1544  unsigned OpSHL;
1545  } OpTable[] = {
1546  {8, X86::SHR8rCL, X86::SAR8rCL, X86::SHL8rCL}, // i8
1547  {16, X86::SHR16rCL, X86::SAR16rCL, X86::SHL16rCL}, // i16
1548  {32, X86::SHR32rCL, X86::SAR32rCL, X86::SHL32rCL}, // i32
1549  {64, X86::SHR64rCL, X86::SAR64rCL, X86::SHL64rCL} // i64
1550  };
1551 
1552  if (DstRB.getID() != X86::GPRRegBankID)
1553  return false;
1554 
1555  auto ShiftEntryIt = std::find_if(
1556  std::begin(OpTable), std::end(OpTable), [DstTy](const ShiftEntry &El) {
1557  return El.SizeInBits == DstTy.getSizeInBits();
1558  });
1559  if (ShiftEntryIt == std::end(OpTable))
1560  return false;
1561 
1562  unsigned Opcode = 0;
1563  switch (I.getOpcode()) {
1564  case TargetOpcode::G_SHL:
1565  Opcode = ShiftEntryIt->OpSHL;
1566  break;
1567  case TargetOpcode::G_ASHR:
1568  Opcode = ShiftEntryIt->OpASHR;
1569  break;
1570  case TargetOpcode::G_LSHR:
1571  Opcode = ShiftEntryIt->OpLSHR;
1572  break;
1573  default:
1574  return false;
1575  }
1576 
1577  unsigned Op0Reg = I.getOperand(1).getReg();
1578  unsigned Op1Reg = I.getOperand(2).getReg();
1579 
1580  assert(MRI.getType(Op1Reg).getSizeInBits() == 8);
1581 
1582  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY),
1583  X86::CL)
1584  .addReg(Op1Reg);
1585 
1586  MachineInstr &ShiftInst =
1587  *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode), DstReg)
1588  .addReg(Op0Reg);
1589 
1590  constrainSelectedInstRegOperands(ShiftInst, TII, TRI, RBI);
1591  I.eraseFromParent();
1592  return true;
1593 }
1594 
1595 bool X86InstructionSelector::selectDivRem(MachineInstr &I,
1596  MachineRegisterInfo &MRI,
1597  MachineFunction &MF) const {
1598  // The implementation of this function is taken from X86FastISel.
1599  assert((I.getOpcode() == TargetOpcode::G_SDIV ||
1600  I.getOpcode() == TargetOpcode::G_SREM ||
1601  I.getOpcode() == TargetOpcode::G_UDIV ||
1602  I.getOpcode() == TargetOpcode::G_UREM) &&
1603  "unexpected instruction");
1604 
1605  const unsigned DstReg = I.getOperand(0).getReg();
1606  const unsigned Op1Reg = I.getOperand(1).getReg();
1607  const unsigned Op2Reg = I.getOperand(2).getReg();
1608 
1609  const LLT RegTy = MRI.getType(DstReg);
1610  assert(RegTy == MRI.getType(Op1Reg) && RegTy == MRI.getType(Op2Reg) &&
1611  "Arguments and return value types must match");
1612 
1613  const RegisterBank *RegRB = RBI.getRegBank(DstReg, MRI, TRI);
1614  if (!RegRB || RegRB->getID() != X86::GPRRegBankID)
1615  return false;
1616 
1617  const static unsigned NumTypes = 4; // i8, i16, i32, i64
1618  const static unsigned NumOps = 4; // SDiv, SRem, UDiv, URem
1619  const static bool S = true; // IsSigned
1620  const static bool U = false; // !IsSigned
1621  const static unsigned Copy = TargetOpcode::COPY;
1622  // For the X86 IDIV instruction, in most cases the dividend
1623  // (numerator) must be in a specific register pair highreg:lowreg,
1624  // producing the quotient in lowreg and the remainder in highreg.
1625  // For most data types, to set up the instruction, the dividend is
1626  // copied into lowreg, and lowreg is sign-extended into highreg. The
1627  // exception is i8, where the dividend is defined as a single register rather
1628  // than a register pair, and we therefore directly sign-extend the dividend
1629  // into lowreg, instead of copying, and ignore the highreg.
1630  const static struct DivRemEntry {
1631  // The following portion depends only on the data type.
1632  unsigned SizeInBits;
1633  unsigned LowInReg; // low part of the register pair
1634  unsigned HighInReg; // high part of the register pair
1635  // The following portion depends on both the data type and the operation.
1636  struct DivRemResult {
1637  unsigned OpDivRem; // The specific DIV/IDIV opcode to use.
1638  unsigned OpSignExtend; // Opcode for sign-extending lowreg into
1639  // highreg, or copying a zero into highreg.
1640  unsigned OpCopy; // Opcode for copying dividend into lowreg, or
1641  // zero/sign-extending into lowreg for i8.
1642  unsigned DivRemResultReg; // Register containing the desired result.
1643  bool IsOpSigned; // Whether to use signed or unsigned form.
1644  } ResultTable[NumOps];
1645  } OpTable[NumTypes] = {
1646  {8,
1647  X86::AX,
1648  0,
1649  {
1650  {X86::IDIV8r, 0, X86::MOVSX16rr8, X86::AL, S}, // SDiv
1651  {X86::IDIV8r, 0, X86::MOVSX16rr8, X86::AH, S}, // SRem
1652  {X86::DIV8r, 0, X86::MOVZX16rr8, X86::AL, U}, // UDiv
1653  {X86::DIV8r, 0, X86::MOVZX16rr8, X86::AH, U}, // URem
1654  }}, // i8
1655  {16,
1656  X86::AX,
1657  X86::DX,
1658  {
1659  {X86::IDIV16r, X86::CWD, Copy, X86::AX, S}, // SDiv
1660  {X86::IDIV16r, X86::CWD, Copy, X86::DX, S}, // SRem
1661  {X86::DIV16r, X86::MOV32r0, Copy, X86::AX, U}, // UDiv
1662  {X86::DIV16r, X86::MOV32r0, Copy, X86::DX, U}, // URem
1663  }}, // i16
1664  {32,
1665  X86::EAX,
1666  X86::EDX,
1667  {
1668  {X86::IDIV32r, X86::CDQ, Copy, X86::EAX, S}, // SDiv
1669  {X86::IDIV32r, X86::CDQ, Copy, X86::EDX, S}, // SRem
1670  {X86::DIV32r, X86::MOV32r0, Copy, X86::EAX, U}, // UDiv
1671  {X86::DIV32r, X86::MOV32r0, Copy, X86::EDX, U}, // URem
1672  }}, // i32
1673  {64,
1674  X86::RAX,
1675  X86::RDX,
1676  {
1677  {X86::IDIV64r, X86::CQO, Copy, X86::RAX, S}, // SDiv
1678  {X86::IDIV64r, X86::CQO, Copy, X86::RDX, S}, // SRem
1679  {X86::DIV64r, X86::MOV32r0, Copy, X86::RAX, U}, // UDiv
1680  {X86::DIV64r, X86::MOV32r0, Copy, X86::RDX, U}, // URem
1681  }}, // i64
1682  };
1683 
1684  auto OpEntryIt = std::find_if(std::begin(OpTable), std::end(OpTable),
1685  [RegTy](const DivRemEntry &El) {
1686  return El.SizeInBits == RegTy.getSizeInBits();
1687  });
1688  if (OpEntryIt == std::end(OpTable))
1689  return false;
1690 
1691  unsigned OpIndex;
1692  switch (I.getOpcode()) {
1693  default:
1694  llvm_unreachable("Unexpected div/rem opcode");
1695  case TargetOpcode::G_SDIV:
1696  OpIndex = 0;
1697  break;
1698  case TargetOpcode::G_SREM:
1699  OpIndex = 1;
1700  break;
1701  case TargetOpcode::G_UDIV:
1702  OpIndex = 2;
1703  break;
1704  case TargetOpcode::G_UREM:
1705  OpIndex = 3;
1706  break;
1707  }
1708 
1709  const DivRemEntry &TypeEntry = *OpEntryIt;
1710  const DivRemEntry::DivRemResult &OpEntry = TypeEntry.ResultTable[OpIndex];
1711 
1712  const TargetRegisterClass *RegRC = getRegClass(RegTy, *RegRB);
1713  if (!RBI.constrainGenericRegister(Op1Reg, *RegRC, MRI) ||
1714  !RBI.constrainGenericRegister(Op2Reg, *RegRC, MRI) ||
1715  !RBI.constrainGenericRegister(DstReg, *RegRC, MRI)) {
1716  LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1717  << " operand\n");
1718  return false;
1719  }
1720 
1721  // Move op1 into low-order input register.
1722  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpEntry.OpCopy),
1723  TypeEntry.LowInReg)
1724  .addReg(Op1Reg);
1725  // Zero-extend or sign-extend into high-order input register.
1726  if (OpEntry.OpSignExtend) {
1727  if (OpEntry.IsOpSigned)
1728  BuildMI(*I.getParent(), I, I.getDebugLoc(),
1729  TII.get(OpEntry.OpSignExtend));
1730  else {
1731  unsigned Zero32 = MRI.createVirtualRegister(&X86::GR32RegClass);
1732  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::MOV32r0),
1733  Zero32);
1734 
1735  // Copy the zero into the appropriate sub/super/identical physical
1736  // register. Unfortunately the operations needed are not uniform enough
1737  // to fit neatly into the table above.
1738  if (RegTy.getSizeInBits() == 16) {
1739  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Copy),
1740  TypeEntry.HighInReg)
1741  .addReg(Zero32, 0, X86::sub_16bit);
1742  } else if (RegTy.getSizeInBits() == 32) {
1743  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Copy),
1744  TypeEntry.HighInReg)
1745  .addReg(Zero32);
1746  } else if (RegTy.getSizeInBits() == 64) {
1747  BuildMI(*I.getParent(), I, I.getDebugLoc(),
1748  TII.get(TargetOpcode::SUBREG_TO_REG), TypeEntry.HighInReg)
1749  .addImm(0)
1750  .addReg(Zero32)
1751  .addImm(X86::sub_32bit);
1752  }
1753  }
1754  }
1755  // Generate the DIV/IDIV instruction.
1756  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpEntry.OpDivRem))
1757  .addReg(Op2Reg);
1758  // For i8 remainder, we can't reference ah directly, as we'll end
1759  // up with bogus copies like %r9b = COPY %ah. Reference ax
1760  // instead to prevent ah references in a rex instruction.
1761  //
1762  // The current assumption of the fast register allocator is that isel
1763  // won't generate explicit references to the GR8_NOREX registers. If
1764  // the allocator and/or the backend get enhanced to be more robust in
1765  // that regard, this can be, and should be, removed.
1766  if ((I.getOpcode() == Instruction::SRem ||
1767  I.getOpcode() == Instruction::URem) &&
1768  OpEntry.DivRemResultReg == X86::AH && STI.is64Bit()) {
1769  unsigned SourceSuperReg = MRI.createVirtualRegister(&X86::GR16RegClass);
1770  unsigned ResultSuperReg = MRI.createVirtualRegister(&X86::GR16RegClass);
1771  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Copy), SourceSuperReg)
1772  .addReg(X86::AX);
1773 
1774  // Shift AX right by 8 bits instead of using AH.
1775  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::SHR16ri),
1776  ResultSuperReg)
1777  .addReg(SourceSuperReg)
1778  .addImm(8);
1779 
1780  // Now reference the 8-bit subreg of the result.
1781  BuildMI(*I.getParent(), I, I.getDebugLoc(),
1782  TII.get(TargetOpcode::SUBREG_TO_REG))
1783  .addDef(DstReg)
1784  .addImm(0)
1785  .addReg(ResultSuperReg)
1786  .addImm(X86::sub_8bit);
1787  } else {
1788  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY),
1789  DstReg)
1790  .addReg(OpEntry.DivRemResultReg);
1791  }
1792  I.eraseFromParent();
1793  return true;
1794 }
1795 
1796 bool X86InstructionSelector::selectIntrinsicWSideEffects(
1797  MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) const {
1798 
1799  assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS &&
1800  "unexpected instruction");
1801 
1802  if (I.getOperand(0).getIntrinsicID() != Intrinsic::trap)
1803  return false;
1804 
1805  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::TRAP));
1806 
1807  I.eraseFromParent();
1808  return true;
1809 }
1810 
1813  X86Subtarget &Subtarget,
1814  X86RegisterBankInfo &RBI) {
1815  return new X86InstructionSelector(TM, Subtarget, RBI);
1816 }
const_iterator end(StringRef path)
Get end iterator over path.
Definition: Path.cpp:233
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:636
union llvm::X86AddressMode::@501 Base
const_iterator begin(StringRef path, Style style=Style::native)
Get begin iterator over path.
Definition: Path.cpp:224
MachineBasicBlock * getMBB() const
Atomic ordering constants.
static bool selectMergeValues(MachineInstrBuilder &MIB, const ARMBaseInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
This class represents lattice values for constants.
Definition: AllocatorList.h:23
static const MachineInstrBuilder & addConstantPoolReference(const MachineInstrBuilder &MIB, unsigned CPI, unsigned GlobalBaseReg, unsigned char OpFlags)
addConstantPoolReference - This function is used to add a reference to the base of a constant value s...
enum llvm::X86AddressMode::@500 BaseType
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition: MachineInstr.h:382
unsigned getReg() const
getReg - Returns the register number.
unsigned Reg
static uint64_t selectImpl(uint64_t CandidateMask, uint64_t &NextInSequenceMask)
LLT getType(unsigned Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register...
unsigned const TargetRegisterInfo * TRI
A debug info location.
Definition: DebugLoc.h:33
const GlobalValue * GV
An instruction for reading from memory.
Definition: Instructions.h:167
void setRegBank(unsigned Reg, const RegisterBank &RegBank)
Set the register bank to RegBank for Reg.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
LegalityPredicate isPointer(unsigned TypeIdx)
True iff the specified type index is a pointer (with any address space).
static const TargetRegisterClass * getRegClassFromGRPhysReg(unsigned Reg)
return AArch64::GPR64RegClass contains(Reg)
#define DEBUG_TYPE
bool isVector() const
A description of a memory reference used in the backend.
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
const HexagonInstrInfo * TII
const ConstantFP * getFPImm() const
unsigned getNumOperands() const
Retuns the total number of operands.
Definition: MachineInstr.h:411
This class provides the information for the target register banks.
const MachineInstrBuilder & addUse(unsigned RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
static bool isGlobalStubReference(unsigned char TargetFlag)
isGlobalStubReference - Return true if the specified TargetFlag operand is a reference to a stub for ...
Definition: X86InstrInfo.h:74
void eraseFromParent()
Unlink &#39;this&#39; from the containing basic block and delete it.
static StringRef getName(Value *V)
static int getRegClass(RegisterKind Is, unsigned RegWidth)
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:408
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineInstr * getVRegDef(unsigned Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
static bool isGlobalRelativeToPICBase(unsigned char TargetFlag)
isGlobalRelativeToPICBase - Return true if the specified global value reference is relative to a 32-b...
Definition: X86InstrInfo.h:91
static bool canTurnIntoCOPY(const TargetRegisterClass *DstRC, const TargetRegisterClass *SrcRC)
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
void ChangeToImmediate(int64_t ImmVal)
ChangeToImmediate - Replace this operand with a new immediate operand of the specified value...
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
std::pair< CondCode, bool > getX86ConditionCode(CmpInst::Predicate Predicate)
Return a pair of condition code for the given predicate and whether the instruction operands should b...
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:148
unsigned const MachineRegisterInfo * MRI
#define GET_GLOBALISEL_PREDICATES_INIT
unsigned getPointerSize(unsigned AS=0) const
Layout pointer size FIXME: The defaults need to be removed once all of the backends/clients are updat...
Definition: DataLayout.cpp:648
const GlobalValue * getGlobal() const
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:263
bool isCImm() const
isCImm - Test if this is a MO_CImmediate operand.
bool hasOneMemOperand() const
Return true if this instruction has exactly one MachineMemOperand.
Definition: MachineInstr.h:548
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:646
int64_t addOffset(int64_t LHS, int64_t RHS)
void substPhysReg(unsigned Reg, const TargetRegisterInfo &)
substPhysReg - Substitute the current register with the physical register Reg, taking any existing Su...
void setImm(int64_t immVal)
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:771
static bool selectUnmergeValues(MachineInstrBuilder &MIB, const ARMBaseInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
void print(raw_ostream &OS, bool IsStandalone=true, bool SkipOpers=false, bool SkipDebugLoc=false, bool AddNewLine=true, const TargetInstrInfo *TII=nullptr) const
Print this MI to OS.
auto find_if(R &&Range, UnaryPredicate P) -> decltype(adl_begin(Range))
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1213
static void X86SelectAddress(const MachineInstr &I, const MachineRegisterInfo &MRI, X86AddressMode &AM)
MachineConstantPool * getConstantPool()
getConstantPool - Return the constant pool object for the current function.
bool isCopy() const
bool isImplicitDef() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned getNumExplicitOperands() const
Returns the number of non-implicit operands.
bool hasSubClassEq(const TargetRegisterClass *RC) const
Returns true if RC is a sub-class of or equal to this class.
This file declares the targeting of the RegisterBankInfo class for X86.
unsigned createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
void setDesc(const MCInstrDesc &tid)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one...
constexpr bool isInt< 32 >(int64_t x)
Definition: MathExtras.h:308
void addOperand(MachineFunction &MF, const MachineOperand &Op)
Add the specified operand to the instruction.
mmo_iterator memoperands_begin() const
Access to memory operands of the instruction.
Definition: MachineInstr.h:533
MachineOperand class - Representation of each machine instruction operand.
Intrinsic::ID getIntrinsicID() const
Predicate
Predicate - These are "(BI << 5) | BO" for various predicates.
Definition: PPCPredicates.h:26
unsigned getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
bool constrainSelectedInstRegOperands(MachineInstr &I, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
Mutate the newly-selected instruction I to constrain its (possibly generic) virtual register operands...
Definition: Utils.cpp:87
InstructionSelector * createX86InstructionSelector(const X86TargetMachine &TM, X86Subtarget &, X86RegisterBankInfo &)
This class implements the register bank concept.
Definition: RegisterBank.h:28
int64_t getImm() const
bool isTarget64BitILP32() const
Is this x86_64 with the ILP32 programming model (x32 ABI)?
Definition: X86Subtarget.h:540
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:940
Optional< int64_t > getConstantVRegVal(unsigned VReg, const MachineRegisterInfo &MRI)
If VReg is defined by a G_CONSTANT fits in int64_t returns it.
Definition: Utils.cpp:184
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:253
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
Provides the logic to select generic machine instructions.
Representation of each machine instruction.
Definition: MachineInstr.h:63
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MO_GOTOFF - On a symbol operand this indicates that the immediate is the offset to the location of th...
Definition: X86BaseInfo.h:139
static const MachineInstrBuilder & addDirectMem(const MachineInstrBuilder &MIB, unsigned Reg)
addDirectMem - This function is used to add a direct memory reference to the current instruction – t...
static unsigned getLeaOP(LLT Ty, const X86Subtarget &STI)
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
void setReg(unsigned Reg)
Change the register this operand corresponds to.
#define I(x, y, z)
Definition: MD5.cpp:58
void setSubReg(unsigned subReg)
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
#define GET_GLOBALISEL_TEMPORARIES_INIT
const TargetRegisterClass * getRegClassOrNull(unsigned Reg) const
Return the register class of Reg, or null if Reg has not been assigned a register class yet...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
bool isPreISelGenericOpcode(unsigned Opcode)
Check whether the given Opcode is a generic opcode that is not supposed to appear after ISel...
Definition: TargetOpcodes.h:30
unsigned getSizeInBits(unsigned Reg, const MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI) const
Get the size in bits of Reg.
MO_PIC_BASE_OFFSET - On a symbol operand this indicates that the immediate should get the value of th...
Definition: X86BaseInfo.h:125
void RemoveOperand(unsigned OpNo)
Erase an operand from an instruction, leaving it with one fewer operand than it started with...
X86AddressMode - This struct holds a generalized full x86 address mode.
static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
#define LLVM_DEBUG(X)
Definition: Debug.h:122
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:413
const ConstantInt * getCImm() const
static const MachineInstrBuilder & addFullAddress(const MachineInstrBuilder &MIB, const X86AddressMode &AM)
unsigned getConstantPoolIndex(const Constant *C, unsigned Alignment)
getConstantPoolIndex - Create a new entry in the constant pool or return an existing one...
unsigned createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
unsigned getID() const
Get the identifier of this register bank.
Definition: RegisterBank.h:47
unsigned getPredicate() const