LLVM  9.0.0svn
LegalizerHelper.cpp
Go to the documentation of this file.
1 //===-- llvm/CodeGen/GlobalISel/LegalizerHelper.cpp -----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file This file implements the LegalizerHelper class to legalize
10 /// individual instructions and the LegalizeMachineIR wrapper pass for the
11 /// primary legalization.
12 //
13 //===----------------------------------------------------------------------===//
14 
23 #include "llvm/Support/Debug.h"
26 
27 #define DEBUG_TYPE "legalizer"
28 
29 using namespace llvm;
30 using namespace LegalizeActions;
31 
32 /// Try to break down \p OrigTy into \p NarrowTy sized pieces.
33 ///
34 /// Returns the number of \p NarrowTy elements needed to reconstruct \p OrigTy,
35 /// with any leftover piece as type \p LeftoverTy
36 ///
37 /// Returns -1 in the first element of the pair if the breakdown is not
38 /// satisfiable.
39 static std::pair<int, int>
40 getNarrowTypeBreakDown(LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy) {
41  assert(!LeftoverTy.isValid() && "this is an out argument");
42 
43  unsigned Size = OrigTy.getSizeInBits();
44  unsigned NarrowSize = NarrowTy.getSizeInBits();
45  unsigned NumParts = Size / NarrowSize;
46  unsigned LeftoverSize = Size - NumParts * NarrowSize;
47  assert(Size > NarrowSize);
48 
49  if (LeftoverSize == 0)
50  return {NumParts, 0};
51 
52  if (NarrowTy.isVector()) {
53  unsigned EltSize = OrigTy.getScalarSizeInBits();
54  if (LeftoverSize % EltSize != 0)
55  return {-1, -1};
56  LeftoverTy = LLT::scalarOrVector(LeftoverSize / EltSize, EltSize);
57  } else {
58  LeftoverTy = LLT::scalar(LeftoverSize);
59  }
60 
61  int NumLeftover = LeftoverSize / LeftoverTy.getSizeInBits();
62  return std::make_pair(NumParts, NumLeftover);
63 }
64 
66  GISelChangeObserver &Observer,
67  MachineIRBuilder &Builder)
68  : MIRBuilder(Builder), MRI(MF.getRegInfo()),
69  LI(*MF.getSubtarget().getLegalizerInfo()), Observer(Observer) {
70  MIRBuilder.setMF(MF);
71  MIRBuilder.setChangeObserver(Observer);
72 }
73 
75  GISelChangeObserver &Observer,
77  : MIRBuilder(B), MRI(MF.getRegInfo()), LI(LI), Observer(Observer) {
78  MIRBuilder.setMF(MF);
79  MIRBuilder.setChangeObserver(Observer);
80 }
83  LLVM_DEBUG(dbgs() << "Legalizing: "; MI.print(dbgs()));
84 
85  if (MI.getOpcode() == TargetOpcode::G_INTRINSIC ||
86  MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS)
87  return LI.legalizeIntrinsic(MI, MRI, MIRBuilder) ? Legalized
89  auto Step = LI.getAction(MI, MRI);
90  switch (Step.Action) {
91  case Legal:
92  LLVM_DEBUG(dbgs() << ".. Already legal\n");
93  return AlreadyLegal;
94  case Libcall:
95  LLVM_DEBUG(dbgs() << ".. Convert to libcall\n");
96  return libcall(MI);
97  case NarrowScalar:
98  LLVM_DEBUG(dbgs() << ".. Narrow scalar\n");
99  return narrowScalar(MI, Step.TypeIdx, Step.NewType);
100  case WidenScalar:
101  LLVM_DEBUG(dbgs() << ".. Widen scalar\n");
102  return widenScalar(MI, Step.TypeIdx, Step.NewType);
103  case Lower:
104  LLVM_DEBUG(dbgs() << ".. Lower\n");
105  return lower(MI, Step.TypeIdx, Step.NewType);
106  case FewerElements:
107  LLVM_DEBUG(dbgs() << ".. Reduce number of elements\n");
108  return fewerElementsVector(MI, Step.TypeIdx, Step.NewType);
109  case MoreElements:
110  LLVM_DEBUG(dbgs() << ".. Increase number of elements\n");
111  return moreElementsVector(MI, Step.TypeIdx, Step.NewType);
112  case Custom:
113  LLVM_DEBUG(dbgs() << ".. Custom legalization\n");
114  return LI.legalizeCustom(MI, MRI, MIRBuilder, Observer) ? Legalized
116  default:
117  LLVM_DEBUG(dbgs() << ".. Unable to legalize\n");
118  return UnableToLegalize;
119  }
120 }
121 
122 void LegalizerHelper::extractParts(Register Reg, LLT Ty, int NumParts,
123  SmallVectorImpl<Register> &VRegs) {
124  for (int i = 0; i < NumParts; ++i)
126  MIRBuilder.buildUnmerge(VRegs, Reg);
127 }
128 
129 bool LegalizerHelper::extractParts(Register Reg, LLT RegTy,
130  LLT MainTy, LLT &LeftoverTy,
132  SmallVectorImpl<Register> &LeftoverRegs) {
133  assert(!LeftoverTy.isValid() && "this is an out argument");
134 
135  unsigned RegSize = RegTy.getSizeInBits();
136  unsigned MainSize = MainTy.getSizeInBits();
137  unsigned NumParts = RegSize / MainSize;
138  unsigned LeftoverSize = RegSize - NumParts * MainSize;
139 
140  // Use an unmerge when possible.
141  if (LeftoverSize == 0) {
142  for (unsigned I = 0; I < NumParts; ++I)
143  VRegs.push_back(MRI.createGenericVirtualRegister(MainTy));
144  MIRBuilder.buildUnmerge(VRegs, Reg);
145  return true;
146  }
147 
148  if (MainTy.isVector()) {
149  unsigned EltSize = MainTy.getScalarSizeInBits();
150  if (LeftoverSize % EltSize != 0)
151  return false;
152  LeftoverTy = LLT::scalarOrVector(LeftoverSize / EltSize, EltSize);
153  } else {
154  LeftoverTy = LLT::scalar(LeftoverSize);
155  }
156 
157  // For irregular sizes, extract the individual parts.
158  for (unsigned I = 0; I != NumParts; ++I) {
159  Register NewReg = MRI.createGenericVirtualRegister(MainTy);
160  VRegs.push_back(NewReg);
161  MIRBuilder.buildExtract(NewReg, Reg, MainSize * I);
162  }
163 
164  for (unsigned Offset = MainSize * NumParts; Offset < RegSize;
165  Offset += LeftoverSize) {
166  Register NewReg = MRI.createGenericVirtualRegister(LeftoverTy);
167  LeftoverRegs.push_back(NewReg);
168  MIRBuilder.buildExtract(NewReg, Reg, Offset);
169  }
170 
171  return true;
172 }
173 
174 void LegalizerHelper::insertParts(Register DstReg,
175  LLT ResultTy, LLT PartTy,
176  ArrayRef<Register> PartRegs,
177  LLT LeftoverTy,
178  ArrayRef<Register> LeftoverRegs) {
179  if (!LeftoverTy.isValid()) {
180  assert(LeftoverRegs.empty());
181 
182  if (!ResultTy.isVector()) {
183  MIRBuilder.buildMerge(DstReg, PartRegs);
184  return;
185  }
186 
187  if (PartTy.isVector())
188  MIRBuilder.buildConcatVectors(DstReg, PartRegs);
189  else
190  MIRBuilder.buildBuildVector(DstReg, PartRegs);
191  return;
192  }
193 
194  unsigned PartSize = PartTy.getSizeInBits();
195  unsigned LeftoverPartSize = LeftoverTy.getSizeInBits();
196 
197  Register CurResultReg = MRI.createGenericVirtualRegister(ResultTy);
198  MIRBuilder.buildUndef(CurResultReg);
199 
200  unsigned Offset = 0;
201  for (Register PartReg : PartRegs) {
202  Register NewResultReg = MRI.createGenericVirtualRegister(ResultTy);
203  MIRBuilder.buildInsert(NewResultReg, CurResultReg, PartReg, Offset);
204  CurResultReg = NewResultReg;
205  Offset += PartSize;
206  }
207 
208  for (unsigned I = 0, E = LeftoverRegs.size(); I != E; ++I) {
209  // Use the original output register for the final insert to avoid a copy.
210  Register NewResultReg = (I + 1 == E) ?
211  DstReg : MRI.createGenericVirtualRegister(ResultTy);
212 
213  MIRBuilder.buildInsert(NewResultReg, CurResultReg, LeftoverRegs[I], Offset);
214  CurResultReg = NewResultReg;
215  Offset += LeftoverPartSize;
216  }
217 }
218 
219 static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
220  switch (Opcode) {
221  case TargetOpcode::G_SDIV:
222  assert((Size == 32 || Size == 64) && "Unsupported size");
223  return Size == 64 ? RTLIB::SDIV_I64 : RTLIB::SDIV_I32;
224  case TargetOpcode::G_UDIV:
225  assert((Size == 32 || Size == 64) && "Unsupported size");
226  return Size == 64 ? RTLIB::UDIV_I64 : RTLIB::UDIV_I32;
227  case TargetOpcode::G_SREM:
228  assert((Size == 32 || Size == 64) && "Unsupported size");
229  return Size == 64 ? RTLIB::SREM_I64 : RTLIB::SREM_I32;
230  case TargetOpcode::G_UREM:
231  assert((Size == 32 || Size == 64) && "Unsupported size");
232  return Size == 64 ? RTLIB::UREM_I64 : RTLIB::UREM_I32;
233  case TargetOpcode::G_CTLZ_ZERO_UNDEF:
234  assert(Size == 32 && "Unsupported size");
235  return RTLIB::CTLZ_I32;
236  case TargetOpcode::G_FADD:
237  assert((Size == 32 || Size == 64) && "Unsupported size");
238  return Size == 64 ? RTLIB::ADD_F64 : RTLIB::ADD_F32;
239  case TargetOpcode::G_FSUB:
240  assert((Size == 32 || Size == 64) && "Unsupported size");
241  return Size == 64 ? RTLIB::SUB_F64 : RTLIB::SUB_F32;
242  case TargetOpcode::G_FMUL:
243  assert((Size == 32 || Size == 64) && "Unsupported size");
244  return Size == 64 ? RTLIB::MUL_F64 : RTLIB::MUL_F32;
245  case TargetOpcode::G_FDIV:
246  assert((Size == 32 || Size == 64) && "Unsupported size");
247  return Size == 64 ? RTLIB::DIV_F64 : RTLIB::DIV_F32;
248  case TargetOpcode::G_FEXP:
249  assert((Size == 32 || Size == 64) && "Unsupported size");
250  return Size == 64 ? RTLIB::EXP_F64 : RTLIB::EXP_F32;
251  case TargetOpcode::G_FEXP2:
252  assert((Size == 32 || Size == 64) && "Unsupported size");
253  return Size == 64 ? RTLIB::EXP2_F64 : RTLIB::EXP2_F32;
254  case TargetOpcode::G_FREM:
255  return Size == 64 ? RTLIB::REM_F64 : RTLIB::REM_F32;
256  case TargetOpcode::G_FPOW:
257  return Size == 64 ? RTLIB::POW_F64 : RTLIB::POW_F32;
258  case TargetOpcode::G_FMA:
259  assert((Size == 32 || Size == 64) && "Unsupported size");
260  return Size == 64 ? RTLIB::FMA_F64 : RTLIB::FMA_F32;
261  case TargetOpcode::G_FSIN:
262  assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
263  return Size == 128 ? RTLIB::SIN_F128
264  : Size == 64 ? RTLIB::SIN_F64 : RTLIB::SIN_F32;
265  case TargetOpcode::G_FCOS:
266  assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
267  return Size == 128 ? RTLIB::COS_F128
268  : Size == 64 ? RTLIB::COS_F64 : RTLIB::COS_F32;
269  case TargetOpcode::G_FLOG10:
270  assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
271  return Size == 128 ? RTLIB::LOG10_F128
272  : Size == 64 ? RTLIB::LOG10_F64 : RTLIB::LOG10_F32;
273  case TargetOpcode::G_FLOG:
274  assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
275  return Size == 128 ? RTLIB::LOG_F128
276  : Size == 64 ? RTLIB::LOG_F64 : RTLIB::LOG_F32;
277  case TargetOpcode::G_FLOG2:
278  assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
279  return Size == 128 ? RTLIB::LOG2_F128
280  : Size == 64 ? RTLIB::LOG2_F64 : RTLIB::LOG2_F32;
281  case TargetOpcode::G_FCEIL:
282  assert((Size == 32 || Size == 64) && "Unsupported size");
283  return Size == 64 ? RTLIB::CEIL_F64 : RTLIB::CEIL_F32;
284  case TargetOpcode::G_FFLOOR:
285  assert((Size == 32 || Size == 64) && "Unsupported size");
286  return Size == 64 ? RTLIB::FLOOR_F64 : RTLIB::FLOOR_F32;
287  }
288  llvm_unreachable("Unknown libcall function");
289 }
290 
293  const CallLowering::ArgInfo &Result,
295  auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
296  auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
297  const char *Name = TLI.getLibcallName(Libcall);
298 
299  MIRBuilder.getMF().getFrameInfo().setHasCalls(true);
300  if (!CLI.lowerCall(MIRBuilder, TLI.getLibcallCallingConv(Libcall),
301  MachineOperand::CreateES(Name), Result, Args))
303 
305 }
306 
307 // Useful for libcalls where all operands have the same type.
310  Type *OpType) {
311  auto Libcall = getRTLibDesc(MI.getOpcode(), Size);
312 
314  for (unsigned i = 1; i < MI.getNumOperands(); i++)
315  Args.push_back({MI.getOperand(i).getReg(), OpType});
316  return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), OpType},
317  Args);
318 }
319 
320 static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType,
321  Type *FromType) {
322  auto ToMVT = MVT::getVT(ToType);
323  auto FromMVT = MVT::getVT(FromType);
324 
325  switch (Opcode) {
326  case TargetOpcode::G_FPEXT:
327  return RTLIB::getFPEXT(FromMVT, ToMVT);
328  case TargetOpcode::G_FPTRUNC:
329  return RTLIB::getFPROUND(FromMVT, ToMVT);
330  case TargetOpcode::G_FPTOSI:
331  return RTLIB::getFPTOSINT(FromMVT, ToMVT);
332  case TargetOpcode::G_FPTOUI:
333  return RTLIB::getFPTOUINT(FromMVT, ToMVT);
334  case TargetOpcode::G_SITOFP:
335  return RTLIB::getSINTTOFP(FromMVT, ToMVT);
336  case TargetOpcode::G_UITOFP:
337  return RTLIB::getUINTTOFP(FromMVT, ToMVT);
338  }
339  llvm_unreachable("Unsupported libcall function");
340 }
341 
344  Type *FromType) {
346  return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), ToType},
347  {{MI.getOperand(1).getReg(), FromType}});
348 }
349 
352  LLT LLTy = MRI.getType(MI.getOperand(0).getReg());
353  unsigned Size = LLTy.getSizeInBits();
354  auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
355 
356  MIRBuilder.setInstr(MI);
357 
358  switch (MI.getOpcode()) {
359  default:
360  return UnableToLegalize;
361  case TargetOpcode::G_SDIV:
362  case TargetOpcode::G_UDIV:
363  case TargetOpcode::G_SREM:
364  case TargetOpcode::G_UREM:
365  case TargetOpcode::G_CTLZ_ZERO_UNDEF: {
366  Type *HLTy = IntegerType::get(Ctx, Size);
367  auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
368  if (Status != Legalized)
369  return Status;
370  break;
371  }
372  case TargetOpcode::G_FADD:
373  case TargetOpcode::G_FSUB:
374  case TargetOpcode::G_FMUL:
375  case TargetOpcode::G_FDIV:
376  case TargetOpcode::G_FMA:
377  case TargetOpcode::G_FPOW:
378  case TargetOpcode::G_FREM:
379  case TargetOpcode::G_FCOS:
380  case TargetOpcode::G_FSIN:
381  case TargetOpcode::G_FLOG10:
382  case TargetOpcode::G_FLOG:
383  case TargetOpcode::G_FLOG2:
384  case TargetOpcode::G_FEXP:
385  case TargetOpcode::G_FEXP2:
386  case TargetOpcode::G_FCEIL:
387  case TargetOpcode::G_FFLOOR: {
388  if (Size > 64) {
389  LLVM_DEBUG(dbgs() << "Size " << Size << " too large to legalize.\n");
390  return UnableToLegalize;
391  }
392  Type *HLTy = Size == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx);
393  auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
394  if (Status != Legalized)
395  return Status;
396  break;
397  }
398  case TargetOpcode::G_FPEXT: {
399  // FIXME: Support other floating point types (half, fp128 etc)
400  unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
401  unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
402  if (ToSize != 64 || FromSize != 32)
403  return UnableToLegalize;
406  if (Status != Legalized)
407  return Status;
408  break;
409  }
410  case TargetOpcode::G_FPTRUNC: {
411  // FIXME: Support other floating point types (half, fp128 etc)
412  unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
413  unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
414  if (ToSize != 32 || FromSize != 64)
415  return UnableToLegalize;
418  if (Status != Legalized)
419  return Status;
420  break;
421  }
422  case TargetOpcode::G_FPTOSI:
423  case TargetOpcode::G_FPTOUI: {
424  // FIXME: Support other types
425  unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
426  unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
427  if ((ToSize != 32 && ToSize != 64) || (FromSize != 32 && FromSize != 64))
428  return UnableToLegalize;
430  MI, MIRBuilder,
431  ToSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx),
432  FromSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx));
433  if (Status != Legalized)
434  return Status;
435  break;
436  }
437  case TargetOpcode::G_SITOFP:
438  case TargetOpcode::G_UITOFP: {
439  // FIXME: Support other types
440  unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
441  unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
442  if ((FromSize != 32 && FromSize != 64) || (ToSize != 32 && ToSize != 64))
443  return UnableToLegalize;
445  MI, MIRBuilder,
446  ToSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx),
447  FromSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx));
448  if (Status != Legalized)
449  return Status;
450  break;
451  }
452  }
453 
454  MI.eraseFromParent();
455  return Legalized;
456 }
457 
459  unsigned TypeIdx,
460  LLT NarrowTy) {
461  MIRBuilder.setInstr(MI);
462 
463  uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
464  uint64_t NarrowSize = NarrowTy.getSizeInBits();
465 
466  switch (MI.getOpcode()) {
467  default:
468  return UnableToLegalize;
469  case TargetOpcode::G_IMPLICIT_DEF: {
470  // FIXME: add support for when SizeOp0 isn't an exact multiple of
471  // NarrowSize.
472  if (SizeOp0 % NarrowSize != 0)
473  return UnableToLegalize;
474  int NumParts = SizeOp0 / NarrowSize;
475 
476  SmallVector<Register, 2> DstRegs;
477  for (int i = 0; i < NumParts; ++i)
478  DstRegs.push_back(
479  MIRBuilder.buildUndef(NarrowTy)->getOperand(0).getReg());
480 
481  Register DstReg = MI.getOperand(0).getReg();
482  if(MRI.getType(DstReg).isVector())
483  MIRBuilder.buildBuildVector(DstReg, DstRegs);
484  else
485  MIRBuilder.buildMerge(DstReg, DstRegs);
486  MI.eraseFromParent();
487  return Legalized;
488  }
489  case TargetOpcode::G_CONSTANT: {
490  LLT Ty = MRI.getType(MI.getOperand(0).getReg());
491  const APInt &Val = MI.getOperand(1).getCImm()->getValue();
492  unsigned TotalSize = Ty.getSizeInBits();
493  unsigned NarrowSize = NarrowTy.getSizeInBits();
494  int NumParts = TotalSize / NarrowSize;
495 
496  SmallVector<Register, 4> PartRegs;
497  for (int I = 0; I != NumParts; ++I) {
498  unsigned Offset = I * NarrowSize;
499  auto K = MIRBuilder.buildConstant(NarrowTy,
500  Val.lshr(Offset).trunc(NarrowSize));
501  PartRegs.push_back(K.getReg(0));
502  }
503 
504  LLT LeftoverTy;
505  unsigned LeftoverBits = TotalSize - NumParts * NarrowSize;
506  SmallVector<Register, 1> LeftoverRegs;
507  if (LeftoverBits != 0) {
508  LeftoverTy = LLT::scalar(LeftoverBits);
509  auto K = MIRBuilder.buildConstant(
510  LeftoverTy,
511  Val.lshr(NumParts * NarrowSize).trunc(LeftoverBits));
512  LeftoverRegs.push_back(K.getReg(0));
513  }
514 
515  insertParts(MI.getOperand(0).getReg(),
516  Ty, NarrowTy, PartRegs, LeftoverTy, LeftoverRegs);
517 
518  MI.eraseFromParent();
519  return Legalized;
520  }
521  case TargetOpcode::G_ADD: {
522  // FIXME: add support for when SizeOp0 isn't an exact multiple of
523  // NarrowSize.
524  if (SizeOp0 % NarrowSize != 0)
525  return UnableToLegalize;
526  // Expand in terms of carry-setting/consuming G_ADDE instructions.
527  int NumParts = SizeOp0 / NarrowTy.getSizeInBits();
528 
529  SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs;
530  extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
531  extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
532 
534  MIRBuilder.buildConstant(CarryIn, 0);
535 
536  for (int i = 0; i < NumParts; ++i) {
537  Register DstReg = MRI.createGenericVirtualRegister(NarrowTy);
539 
540  MIRBuilder.buildUAdde(DstReg, CarryOut, Src1Regs[i],
541  Src2Regs[i], CarryIn);
542 
543  DstRegs.push_back(DstReg);
544  CarryIn = CarryOut;
545  }
546  Register DstReg = MI.getOperand(0).getReg();
547  if(MRI.getType(DstReg).isVector())
548  MIRBuilder.buildBuildVector(DstReg, DstRegs);
549  else
550  MIRBuilder.buildMerge(DstReg, DstRegs);
551  MI.eraseFromParent();
552  return Legalized;
553  }
554  case TargetOpcode::G_SUB: {
555  // FIXME: add support for when SizeOp0 isn't an exact multiple of
556  // NarrowSize.
557  if (SizeOp0 % NarrowSize != 0)
558  return UnableToLegalize;
559 
560  int NumParts = SizeOp0 / NarrowTy.getSizeInBits();
561 
562  SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs;
563  extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
564  extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
565 
566  Register DstReg = MRI.createGenericVirtualRegister(NarrowTy);
568  MIRBuilder.buildInstr(TargetOpcode::G_USUBO, {DstReg, BorrowOut},
569  {Src1Regs[0], Src2Regs[0]});
570  DstRegs.push_back(DstReg);
571  Register BorrowIn = BorrowOut;
572  for (int i = 1; i < NumParts; ++i) {
573  DstReg = MRI.createGenericVirtualRegister(NarrowTy);
574  BorrowOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
575 
576  MIRBuilder.buildInstr(TargetOpcode::G_USUBE, {DstReg, BorrowOut},
577  {Src1Regs[i], Src2Regs[i], BorrowIn});
578 
579  DstRegs.push_back(DstReg);
580  BorrowIn = BorrowOut;
581  }
582  MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
583  MI.eraseFromParent();
584  return Legalized;
585  }
586  case TargetOpcode::G_MUL:
587  case TargetOpcode::G_UMULH:
588  return narrowScalarMul(MI, NarrowTy);
589  case TargetOpcode::G_EXTRACT:
590  return narrowScalarExtract(MI, TypeIdx, NarrowTy);
591  case TargetOpcode::G_INSERT:
592  return narrowScalarInsert(MI, TypeIdx, NarrowTy);
593  case TargetOpcode::G_LOAD: {
594  const auto &MMO = **MI.memoperands_begin();
595  Register DstReg = MI.getOperand(0).getReg();
596  LLT DstTy = MRI.getType(DstReg);
597  if (DstTy.isVector())
598  return UnableToLegalize;
599 
600  if (8 * MMO.getSize() != DstTy.getSizeInBits()) {
601  Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
602  auto &MMO = **MI.memoperands_begin();
603  MIRBuilder.buildLoad(TmpReg, MI.getOperand(1).getReg(), MMO);
604  MIRBuilder.buildAnyExt(DstReg, TmpReg);
605  MI.eraseFromParent();
606  return Legalized;
607  }
608 
609  return reduceLoadStoreWidth(MI, TypeIdx, NarrowTy);
610  }
611  case TargetOpcode::G_ZEXTLOAD:
612  case TargetOpcode::G_SEXTLOAD: {
613  bool ZExt = MI.getOpcode() == TargetOpcode::G_ZEXTLOAD;
614  Register DstReg = MI.getOperand(0).getReg();
615  Register PtrReg = MI.getOperand(1).getReg();
616 
617  Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
618  auto &MMO = **MI.memoperands_begin();
619  if (MMO.getSizeInBits() == NarrowSize) {
620  MIRBuilder.buildLoad(TmpReg, PtrReg, MMO);
621  } else {
622  unsigned ExtLoad = ZExt ? TargetOpcode::G_ZEXTLOAD
623  : TargetOpcode::G_SEXTLOAD;
624  MIRBuilder.buildInstr(ExtLoad)
625  .addDef(TmpReg)
626  .addUse(PtrReg)
627  .addMemOperand(&MMO);
628  }
629 
630  if (ZExt)
631  MIRBuilder.buildZExt(DstReg, TmpReg);
632  else
633  MIRBuilder.buildSExt(DstReg, TmpReg);
634 
635  MI.eraseFromParent();
636  return Legalized;
637  }
638  case TargetOpcode::G_STORE: {
639  const auto &MMO = **MI.memoperands_begin();
640 
641  Register SrcReg = MI.getOperand(0).getReg();
642  LLT SrcTy = MRI.getType(SrcReg);
643  if (SrcTy.isVector())
644  return UnableToLegalize;
645 
646  int NumParts = SizeOp0 / NarrowSize;
647  unsigned HandledSize = NumParts * NarrowTy.getSizeInBits();
648  unsigned LeftoverBits = SrcTy.getSizeInBits() - HandledSize;
649  if (SrcTy.isVector() && LeftoverBits != 0)
650  return UnableToLegalize;
651 
652  if (8 * MMO.getSize() != SrcTy.getSizeInBits()) {
653  Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
654  auto &MMO = **MI.memoperands_begin();
655  MIRBuilder.buildTrunc(TmpReg, SrcReg);
656  MIRBuilder.buildStore(TmpReg, MI.getOperand(1).getReg(), MMO);
657  MI.eraseFromParent();
658  return Legalized;
659  }
660 
661  return reduceLoadStoreWidth(MI, 0, NarrowTy);
662  }
663  case TargetOpcode::G_SELECT:
664  return narrowScalarSelect(MI, TypeIdx, NarrowTy);
665  case TargetOpcode::G_AND:
666  case TargetOpcode::G_OR:
667  case TargetOpcode::G_XOR: {
668  // Legalize bitwise operation:
669  // A = BinOp<Ty> B, C
670  // into:
671  // B1, ..., BN = G_UNMERGE_VALUES B
672  // C1, ..., CN = G_UNMERGE_VALUES C
673  // A1 = BinOp<Ty/N> B1, C2
674  // ...
675  // AN = BinOp<Ty/N> BN, CN
676  // A = G_MERGE_VALUES A1, ..., AN
677  return narrowScalarBasic(MI, TypeIdx, NarrowTy);
678  }
679  case TargetOpcode::G_SHL:
680  case TargetOpcode::G_LSHR:
681  case TargetOpcode::G_ASHR:
682  return narrowScalarShift(MI, TypeIdx, NarrowTy);
683  case TargetOpcode::G_CTLZ:
684  case TargetOpcode::G_CTLZ_ZERO_UNDEF:
685  case TargetOpcode::G_CTTZ:
686  case TargetOpcode::G_CTTZ_ZERO_UNDEF:
687  case TargetOpcode::G_CTPOP:
688  if (TypeIdx != 0)
689  return UnableToLegalize; // TODO
690 
691  Observer.changingInstr(MI);
692  narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT);
693  Observer.changedInstr(MI);
694  return Legalized;
695  case TargetOpcode::G_INTTOPTR:
696  if (TypeIdx != 1)
697  return UnableToLegalize;
698 
699  Observer.changingInstr(MI);
700  narrowScalarSrc(MI, NarrowTy, 1);
701  Observer.changedInstr(MI);
702  return Legalized;
703  case TargetOpcode::G_PTRTOINT:
704  if (TypeIdx != 0)
705  return UnableToLegalize;
706 
707  Observer.changingInstr(MI);
708  narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT);
709  Observer.changedInstr(MI);
710  return Legalized;
711  case TargetOpcode::G_PHI: {
712  unsigned NumParts = SizeOp0 / NarrowSize;
713  SmallVector<Register, 2> DstRegs;
715  DstRegs.resize(NumParts);
716  SrcRegs.resize(MI.getNumOperands() / 2);
717  Observer.changingInstr(MI);
718  for (unsigned i = 1; i < MI.getNumOperands(); i += 2) {
719  MachineBasicBlock &OpMBB = *MI.getOperand(i + 1).getMBB();
720  MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
721  extractParts(MI.getOperand(i).getReg(), NarrowTy, NumParts,
722  SrcRegs[i / 2]);
723  }
724  MachineBasicBlock &MBB = *MI.getParent();
725  MIRBuilder.setInsertPt(MBB, MI);
726  for (unsigned i = 0; i < NumParts; ++i) {
727  DstRegs[i] = MRI.createGenericVirtualRegister(NarrowTy);
728  MachineInstrBuilder MIB =
729  MIRBuilder.buildInstr(TargetOpcode::G_PHI).addDef(DstRegs[i]);
730  for (unsigned j = 1; j < MI.getNumOperands(); j += 2)
731  MIB.addUse(SrcRegs[j / 2][i]).add(MI.getOperand(j + 1));
732  }
733  MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI());
734  MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
735  Observer.changedInstr(MI);
736  MI.eraseFromParent();
737  return Legalized;
738  }
739  case TargetOpcode::G_EXTRACT_VECTOR_ELT:
740  case TargetOpcode::G_INSERT_VECTOR_ELT: {
741  if (TypeIdx != 2)
742  return UnableToLegalize;
743 
744  int OpIdx = MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3;
745  Observer.changingInstr(MI);
746  narrowScalarSrc(MI, NarrowTy, OpIdx);
747  Observer.changedInstr(MI);
748  return Legalized;
749  }
750  }
751 }
752 
753 void LegalizerHelper::widenScalarSrc(MachineInstr &MI, LLT WideTy,
754  unsigned OpIdx, unsigned ExtOpcode) {
755  MachineOperand &MO = MI.getOperand(OpIdx);
756  auto ExtB = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MO.getReg()});
757  MO.setReg(ExtB->getOperand(0).getReg());
758 }
759 
760 void LegalizerHelper::narrowScalarSrc(MachineInstr &MI, LLT NarrowTy,
761  unsigned OpIdx) {
762  MachineOperand &MO = MI.getOperand(OpIdx);
763  auto ExtB = MIRBuilder.buildInstr(TargetOpcode::G_TRUNC, {NarrowTy},
764  {MO.getReg()});
765  MO.setReg(ExtB->getOperand(0).getReg());
766 }
767 
768 void LegalizerHelper::widenScalarDst(MachineInstr &MI, LLT WideTy,
769  unsigned OpIdx, unsigned TruncOpcode) {
770  MachineOperand &MO = MI.getOperand(OpIdx);
771  Register DstExt = MRI.createGenericVirtualRegister(WideTy);
773  MIRBuilder.buildInstr(TruncOpcode, {MO.getReg()}, {DstExt});
774  MO.setReg(DstExt);
775 }
776 
777 void LegalizerHelper::narrowScalarDst(MachineInstr &MI, LLT NarrowTy,
778  unsigned OpIdx, unsigned ExtOpcode) {
779  MachineOperand &MO = MI.getOperand(OpIdx);
780  Register DstTrunc = MRI.createGenericVirtualRegister(NarrowTy);
782  MIRBuilder.buildInstr(ExtOpcode, {MO.getReg()}, {DstTrunc});
783  MO.setReg(DstTrunc);
784 }
785 
786 void LegalizerHelper::moreElementsVectorDst(MachineInstr &MI, LLT WideTy,
787  unsigned OpIdx) {
788  MachineOperand &MO = MI.getOperand(OpIdx);
789  Register DstExt = MRI.createGenericVirtualRegister(WideTy);
791  MIRBuilder.buildExtract(MO.getReg(), DstExt, 0);
792  MO.setReg(DstExt);
793 }
794 
795 void LegalizerHelper::moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy,
796  unsigned OpIdx) {
797  MachineOperand &MO = MI.getOperand(OpIdx);
798 
799  LLT OldTy = MRI.getType(MO.getReg());
800  unsigned OldElts = OldTy.getNumElements();
801  unsigned NewElts = MoreTy.getNumElements();
802 
803  unsigned NumParts = NewElts / OldElts;
804 
805  // Use concat_vectors if the result is a multiple of the number of elements.
806  if (NumParts * OldElts == NewElts) {
808  Parts.push_back(MO.getReg());
809 
810  Register ImpDef = MIRBuilder.buildUndef(OldTy).getReg(0);
811  for (unsigned I = 1; I != NumParts; ++I)
812  Parts.push_back(ImpDef);
813 
814  auto Concat = MIRBuilder.buildConcatVectors(MoreTy, Parts);
815  MO.setReg(Concat.getReg(0));
816  return;
817  }
818 
819  Register MoreReg = MRI.createGenericVirtualRegister(MoreTy);
820  Register ImpDef = MIRBuilder.buildUndef(MoreTy).getReg(0);
821  MIRBuilder.buildInsert(MoreReg, ImpDef, MO.getReg(), 0);
822  MO.setReg(MoreReg);
823 }
824 
826 LegalizerHelper::widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx,
827  LLT WideTy) {
828  if (TypeIdx != 1)
829  return UnableToLegalize;
830 
831  Register DstReg = MI.getOperand(0).getReg();
832  LLT DstTy = MRI.getType(DstReg);
833  if (DstTy.isVector())
834  return UnableToLegalize;
835 
836  Register Src1 = MI.getOperand(1).getReg();
837  LLT SrcTy = MRI.getType(Src1);
838  int NumMerge = DstTy.getSizeInBits() / WideTy.getSizeInBits();
839 
840  // Try to turn this into a merge of merges if we can use the requested type as
841  // the source.
842 
843  // TODO: Pad with undef if DstTy > WideTy
844  if (NumMerge > 1 && WideTy.getSizeInBits() % SrcTy.getSizeInBits() == 0) {
845  int PartsPerMerge = WideTy.getSizeInBits() / SrcTy.getSizeInBits();
847  SmallVector<Register, 4> SubMerges;
848 
849  for (int I = 0; I != NumMerge; ++I) {
850  for (int J = 0; J != PartsPerMerge; ++J)
851  Parts.push_back(MI.getOperand(I * PartsPerMerge + J + 1).getReg());
852 
853  auto SubMerge = MIRBuilder.buildMerge(WideTy, Parts);
854  SubMerges.push_back(SubMerge.getReg(0));
855  Parts.clear();
856  }
857 
858  MIRBuilder.buildMerge(DstReg, SubMerges);
859  MI.eraseFromParent();
860  return Legalized;
861  }
862 
863  unsigned NumOps = MI.getNumOperands();
864  unsigned NumSrc = MI.getNumOperands() - 1;
865  unsigned PartSize = DstTy.getSizeInBits() / NumSrc;
866 
867  Register ResultReg = MIRBuilder.buildZExt(DstTy, Src1).getReg(0);
868 
869  for (unsigned I = 2; I != NumOps; ++I) {
870  const unsigned Offset = (I - 1) * PartSize;
871 
872  Register SrcReg = MI.getOperand(I).getReg();
873  assert(MRI.getType(SrcReg) == LLT::scalar(PartSize));
874 
875  auto ZextInput = MIRBuilder.buildZExt(DstTy, SrcReg);
876 
877  Register NextResult = I + 1 == NumOps ? DstReg :
878  MRI.createGenericVirtualRegister(DstTy);
879 
880  auto ShiftAmt = MIRBuilder.buildConstant(DstTy, Offset);
881  auto Shl = MIRBuilder.buildShl(DstTy, ZextInput, ShiftAmt);
882  MIRBuilder.buildOr(NextResult, ResultReg, Shl);
883  ResultReg = NextResult;
884  }
885 
886  MI.eraseFromParent();
887  return Legalized;
888 }
889 
891 LegalizerHelper::widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx,
892  LLT WideTy) {
893  if (TypeIdx != 0)
894  return UnableToLegalize;
895 
896  unsigned NumDst = MI.getNumOperands() - 1;
897  Register SrcReg = MI.getOperand(NumDst).getReg();
898  LLT SrcTy = MRI.getType(SrcReg);
899  if (!SrcTy.isScalar())
900  return UnableToLegalize;
901 
902  Register Dst0Reg = MI.getOperand(0).getReg();
903  LLT DstTy = MRI.getType(Dst0Reg);
904  if (!DstTy.isScalar())
905  return UnableToLegalize;
906 
907  unsigned NewSrcSize = NumDst * WideTy.getSizeInBits();
908  LLT NewSrcTy = LLT::scalar(NewSrcSize);
909  unsigned SizeDiff = WideTy.getSizeInBits() - DstTy.getSizeInBits();
910 
911  auto WideSrc = MIRBuilder.buildZExt(NewSrcTy, SrcReg);
912 
913  for (unsigned I = 1; I != NumDst; ++I) {
914  auto ShiftAmt = MIRBuilder.buildConstant(NewSrcTy, SizeDiff * I);
915  auto Shl = MIRBuilder.buildShl(NewSrcTy, WideSrc, ShiftAmt);
916  WideSrc = MIRBuilder.buildOr(NewSrcTy, WideSrc, Shl);
917  }
918 
919  Observer.changingInstr(MI);
920 
921  MI.getOperand(NumDst).setReg(WideSrc->getOperand(0).getReg());
922  for (unsigned I = 0; I != NumDst; ++I)
923  widenScalarDst(MI, WideTy, I);
924 
925  Observer.changedInstr(MI);
926 
927  return Legalized;
928 }
929 
931 LegalizerHelper::widenScalarExtract(MachineInstr &MI, unsigned TypeIdx,
932  LLT WideTy) {
933  Register DstReg = MI.getOperand(0).getReg();
934  Register SrcReg = MI.getOperand(1).getReg();
935  LLT SrcTy = MRI.getType(SrcReg);
936 
937  LLT DstTy = MRI.getType(DstReg);
938  unsigned Offset = MI.getOperand(2).getImm();
939 
940  if (TypeIdx == 0) {
941  if (SrcTy.isVector() || DstTy.isVector())
942  return UnableToLegalize;
943 
944  SrcOp Src(SrcReg);
945  if (SrcTy.isPointer()) {
946  // Extracts from pointers can be handled only if they are really just
947  // simple integers.
948  const DataLayout &DL = MIRBuilder.getDataLayout();
950  return UnableToLegalize;
951 
952  LLT SrcAsIntTy = LLT::scalar(SrcTy.getSizeInBits());
953  Src = MIRBuilder.buildPtrToInt(SrcAsIntTy, Src);
954  SrcTy = SrcAsIntTy;
955  }
956 
957  if (DstTy.isPointer())
958  return UnableToLegalize;
959 
960  if (Offset == 0) {
961  // Avoid a shift in the degenerate case.
962  MIRBuilder.buildTrunc(DstReg,
963  MIRBuilder.buildAnyExtOrTrunc(WideTy, Src));
964  MI.eraseFromParent();
965  return Legalized;
966  }
967 
968  // Do a shift in the source type.
969  LLT ShiftTy = SrcTy;
970  if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) {
971  Src = MIRBuilder.buildAnyExt(WideTy, Src);
972  ShiftTy = WideTy;
973  } else if (WideTy.getSizeInBits() > SrcTy.getSizeInBits())
974  return UnableToLegalize;
975 
976  auto LShr = MIRBuilder.buildLShr(
977  ShiftTy, Src, MIRBuilder.buildConstant(ShiftTy, Offset));
978  MIRBuilder.buildTrunc(DstReg, LShr);
979  MI.eraseFromParent();
980  return Legalized;
981  }
982 
983  if (SrcTy.isScalar()) {
984  Observer.changingInstr(MI);
985  widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
986  Observer.changedInstr(MI);
987  return Legalized;
988  }
989 
990  if (!SrcTy.isVector())
991  return UnableToLegalize;
992 
993  if (DstTy != SrcTy.getElementType())
994  return UnableToLegalize;
995 
996  if (Offset % SrcTy.getScalarSizeInBits() != 0)
997  return UnableToLegalize;
998 
999  Observer.changingInstr(MI);
1000  widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1001 
1002  MI.getOperand(2).setImm((WideTy.getSizeInBits() / SrcTy.getSizeInBits()) *
1003  Offset);
1004  widenScalarDst(MI, WideTy.getScalarType(), 0);
1005  Observer.changedInstr(MI);
1006  return Legalized;
1007 }
1008 
1010 LegalizerHelper::widenScalarInsert(MachineInstr &MI, unsigned TypeIdx,
1011  LLT WideTy) {
1012  if (TypeIdx != 0)
1013  return UnableToLegalize;
1014  Observer.changingInstr(MI);
1015  widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1016  widenScalarDst(MI, WideTy);
1017  Observer.changedInstr(MI);
1018  return Legalized;
1019 }
1020 
1022 LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
1023  MIRBuilder.setInstr(MI);
1024 
1025  switch (MI.getOpcode()) {
1026  default:
1027  return UnableToLegalize;
1028  case TargetOpcode::G_EXTRACT:
1029  return widenScalarExtract(MI, TypeIdx, WideTy);
1030  case TargetOpcode::G_INSERT:
1031  return widenScalarInsert(MI, TypeIdx, WideTy);
1032  case TargetOpcode::G_MERGE_VALUES:
1033  return widenScalarMergeValues(MI, TypeIdx, WideTy);
1034  case TargetOpcode::G_UNMERGE_VALUES:
1035  return widenScalarUnmergeValues(MI, TypeIdx, WideTy);
1036  case TargetOpcode::G_UADDO:
1037  case TargetOpcode::G_USUBO: {
1038  if (TypeIdx == 1)
1039  return UnableToLegalize; // TODO
1040  auto LHSZext = MIRBuilder.buildInstr(TargetOpcode::G_ZEXT, {WideTy},
1041  {MI.getOperand(2).getReg()});
1042  auto RHSZext = MIRBuilder.buildInstr(TargetOpcode::G_ZEXT, {WideTy},
1043  {MI.getOperand(3).getReg()});
1044  unsigned Opcode = MI.getOpcode() == TargetOpcode::G_UADDO
1045  ? TargetOpcode::G_ADD
1046  : TargetOpcode::G_SUB;
1047  // Do the arithmetic in the larger type.
1048  auto NewOp = MIRBuilder.buildInstr(Opcode, {WideTy}, {LHSZext, RHSZext});
1049  LLT OrigTy = MRI.getType(MI.getOperand(0).getReg());
1051  auto AndOp = MIRBuilder.buildInstr(
1052  TargetOpcode::G_AND, {WideTy},
1053  {NewOp, MIRBuilder.buildConstant(WideTy, Mask.getZExtValue())});
1054  // There is no overflow if the AndOp is the same as NewOp.
1056  AndOp);
1057  // Now trunc the NewOp to the original result.
1058  MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), NewOp);
1059  MI.eraseFromParent();
1060  return Legalized;
1061  }
1062  case TargetOpcode::G_CTTZ:
1063  case TargetOpcode::G_CTTZ_ZERO_UNDEF:
1064  case TargetOpcode::G_CTLZ:
1065  case TargetOpcode::G_CTLZ_ZERO_UNDEF:
1066  case TargetOpcode::G_CTPOP: {
1067  if (TypeIdx == 0) {
1068  Observer.changingInstr(MI);
1069  widenScalarDst(MI, WideTy, 0);
1070  Observer.changedInstr(MI);
1071  return Legalized;
1072  }
1073 
1074  Register SrcReg = MI.getOperand(1).getReg();
1075 
1076  // First ZEXT the input.
1077  auto MIBSrc = MIRBuilder.buildZExt(WideTy, SrcReg);
1078  LLT CurTy = MRI.getType(SrcReg);
1079  if (MI.getOpcode() == TargetOpcode::G_CTTZ) {
1080  // The count is the same in the larger type except if the original
1081  // value was zero. This can be handled by setting the bit just off
1082  // the top of the original type.
1083  auto TopBit =
1084  APInt::getOneBitSet(WideTy.getSizeInBits(), CurTy.getSizeInBits());
1085  MIBSrc = MIRBuilder.buildOr(
1086  WideTy, MIBSrc, MIRBuilder.buildConstant(WideTy, TopBit));
1087  }
1088 
1089  // Perform the operation at the larger size.
1090  auto MIBNewOp = MIRBuilder.buildInstr(MI.getOpcode(), {WideTy}, {MIBSrc});
1091  // This is already the correct result for CTPOP and CTTZs
1092  if (MI.getOpcode() == TargetOpcode::G_CTLZ ||
1093  MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF) {
1094  // The correct result is NewOp - (Difference in widety and current ty).
1095  unsigned SizeDiff = WideTy.getSizeInBits() - CurTy.getSizeInBits();
1096  MIBNewOp = MIRBuilder.buildInstr(
1097  TargetOpcode::G_SUB, {WideTy},
1098  {MIBNewOp, MIRBuilder.buildConstant(WideTy, SizeDiff)});
1099  }
1100 
1101  MIRBuilder.buildZExtOrTrunc(MI.getOperand(0), MIBNewOp);
1102  MI.eraseFromParent();
1103  return Legalized;
1104  }
1105  case TargetOpcode::G_BSWAP: {
1106  Observer.changingInstr(MI);
1107  Register DstReg = MI.getOperand(0).getReg();
1108 
1109  Register ShrReg = MRI.createGenericVirtualRegister(WideTy);
1110  Register DstExt = MRI.createGenericVirtualRegister(WideTy);
1111  Register ShiftAmtReg = MRI.createGenericVirtualRegister(WideTy);
1112  widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1113 
1114  MI.getOperand(0).setReg(DstExt);
1115 
1117 
1118  LLT Ty = MRI.getType(DstReg);
1119  unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits();
1120  MIRBuilder.buildConstant(ShiftAmtReg, DiffBits);
1121  MIRBuilder.buildInstr(TargetOpcode::G_LSHR)
1122  .addDef(ShrReg)
1123  .addUse(DstExt)
1124  .addUse(ShiftAmtReg);
1125 
1126  MIRBuilder.buildTrunc(DstReg, ShrReg);
1127  Observer.changedInstr(MI);
1128  return Legalized;
1129  }
1130  case TargetOpcode::G_ADD:
1131  case TargetOpcode::G_AND:
1132  case TargetOpcode::G_MUL:
1133  case TargetOpcode::G_OR:
1134  case TargetOpcode::G_XOR:
1135  case TargetOpcode::G_SUB:
1136  // Perform operation at larger width (any extension is fines here, high bits
1137  // don't affect the result) and then truncate the result back to the
1138  // original type.
1139  Observer.changingInstr(MI);
1140  widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1141  widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
1142  widenScalarDst(MI, WideTy);
1143  Observer.changedInstr(MI);
1144  return Legalized;
1145 
1146  case TargetOpcode::G_SHL:
1147  Observer.changingInstr(MI);
1148 
1149  if (TypeIdx == 0) {
1150  widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1151  widenScalarDst(MI, WideTy);
1152  } else {
1153  assert(TypeIdx == 1);
1154  // The "number of bits to shift" operand must preserve its value as an
1155  // unsigned integer:
1156  widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
1157  }
1158 
1159  Observer.changedInstr(MI);
1160  return Legalized;
1161 
1162  case TargetOpcode::G_SDIV:
1163  case TargetOpcode::G_SREM:
1164  case TargetOpcode::G_SMIN:
1165  case TargetOpcode::G_SMAX:
1166  Observer.changingInstr(MI);
1167  widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
1168  widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
1169  widenScalarDst(MI, WideTy);
1170  Observer.changedInstr(MI);
1171  return Legalized;
1172 
1173  case TargetOpcode::G_ASHR:
1174  case TargetOpcode::G_LSHR:
1175  Observer.changingInstr(MI);
1176 
1177  if (TypeIdx == 0) {
1178  unsigned CvtOp = MI.getOpcode() == TargetOpcode::G_ASHR ?
1179  TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
1180 
1181  widenScalarSrc(MI, WideTy, 1, CvtOp);
1182  widenScalarDst(MI, WideTy);
1183  } else {
1184  assert(TypeIdx == 1);
1185  // The "number of bits to shift" operand must preserve its value as an
1186  // unsigned integer:
1187  widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
1188  }
1189 
1190  Observer.changedInstr(MI);
1191  return Legalized;
1192  case TargetOpcode::G_UDIV:
1193  case TargetOpcode::G_UREM:
1194  case TargetOpcode::G_UMIN:
1195  case TargetOpcode::G_UMAX:
1196  Observer.changingInstr(MI);
1197  widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
1198  widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
1199  widenScalarDst(MI, WideTy);
1200  Observer.changedInstr(MI);
1201  return Legalized;
1202 
1203  case TargetOpcode::G_SELECT:
1204  Observer.changingInstr(MI);
1205  if (TypeIdx == 0) {
1206  // Perform operation at larger width (any extension is fine here, high
1207  // bits don't affect the result) and then truncate the result back to the
1208  // original type.
1209  widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
1210  widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ANYEXT);
1211  widenScalarDst(MI, WideTy);
1212  } else {
1213  bool IsVec = MRI.getType(MI.getOperand(1).getReg()).isVector();
1214  // Explicit extension is required here since high bits affect the result.
1215  widenScalarSrc(MI, WideTy, 1, MIRBuilder.getBoolExtOp(IsVec, false));
1216  }
1217  Observer.changedInstr(MI);
1218  return Legalized;
1219 
1220  case TargetOpcode::G_FPTOSI:
1221  case TargetOpcode::G_FPTOUI:
1222  if (TypeIdx != 0)
1223  return UnableToLegalize;
1224  Observer.changingInstr(MI);
1225  widenScalarDst(MI, WideTy);
1226  Observer.changedInstr(MI);
1227  return Legalized;
1228 
1229  case TargetOpcode::G_SITOFP:
1230  if (TypeIdx != 1)
1231  return UnableToLegalize;
1232  Observer.changingInstr(MI);
1233  widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
1234  Observer.changedInstr(MI);
1235  return Legalized;
1236 
1237  case TargetOpcode::G_UITOFP:
1238  if (TypeIdx != 1)
1239  return UnableToLegalize;
1240  Observer.changingInstr(MI);
1241  widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
1242  Observer.changedInstr(MI);
1243  return Legalized;
1244 
1245  case TargetOpcode::G_LOAD:
1246  case TargetOpcode::G_SEXTLOAD:
1247  case TargetOpcode::G_ZEXTLOAD:
1248  Observer.changingInstr(MI);
1249  widenScalarDst(MI, WideTy);
1250  Observer.changedInstr(MI);
1251  return Legalized;
1252 
1253  case TargetOpcode::G_STORE: {
1254  if (TypeIdx != 0)
1255  return UnableToLegalize;
1256 
1257  LLT Ty = MRI.getType(MI.getOperand(0).getReg());
1258  if (!isPowerOf2_32(Ty.getSizeInBits()))
1259  return UnableToLegalize;
1260 
1261  Observer.changingInstr(MI);
1262 
1263  unsigned ExtType = Ty.getScalarSizeInBits() == 1 ?
1264  TargetOpcode::G_ZEXT : TargetOpcode::G_ANYEXT;
1265  widenScalarSrc(MI, WideTy, 0, ExtType);
1266 
1267  Observer.changedInstr(MI);
1268  return Legalized;
1269  }
1270  case TargetOpcode::G_CONSTANT: {
1271  MachineOperand &SrcMO = MI.getOperand(1);
1273  const APInt &Val = SrcMO.getCImm()->getValue().sext(WideTy.getSizeInBits());
1274  Observer.changingInstr(MI);
1275  SrcMO.setCImm(ConstantInt::get(Ctx, Val));
1276 
1277  widenScalarDst(MI, WideTy);
1278  Observer.changedInstr(MI);
1279  return Legalized;
1280  }
1281  case TargetOpcode::G_FCONSTANT: {
1282  MachineOperand &SrcMO = MI.getOperand(1);
1284  APFloat Val = SrcMO.getFPImm()->getValueAPF();
1285  bool LosesInfo;
1286  switch (WideTy.getSizeInBits()) {
1287  case 32:
1289  &LosesInfo);
1290  break;
1291  case 64:
1293  &LosesInfo);
1294  break;
1295  default:
1296  return UnableToLegalize;
1297  }
1298 
1299  assert(!LosesInfo && "extend should always be lossless");
1300 
1301  Observer.changingInstr(MI);
1302  SrcMO.setFPImm(ConstantFP::get(Ctx, Val));
1303 
1304  widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
1305  Observer.changedInstr(MI);
1306  return Legalized;
1307  }
1308  case TargetOpcode::G_IMPLICIT_DEF: {
1309  Observer.changingInstr(MI);
1310  widenScalarDst(MI, WideTy);
1311  Observer.changedInstr(MI);
1312  return Legalized;
1313  }
1314  case TargetOpcode::G_BRCOND:
1315  Observer.changingInstr(MI);
1316  widenScalarSrc(MI, WideTy, 0, MIRBuilder.getBoolExtOp(false, false));
1317  Observer.changedInstr(MI);
1318  return Legalized;
1319 
1320  case TargetOpcode::G_FCMP:
1321  Observer.changingInstr(MI);
1322  if (TypeIdx == 0)
1323  widenScalarDst(MI, WideTy);
1324  else {
1325  widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_FPEXT);
1326  widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_FPEXT);
1327  }
1328  Observer.changedInstr(MI);
1329  return Legalized;
1330 
1331  case TargetOpcode::G_ICMP:
1332  Observer.changingInstr(MI);
1333  if (TypeIdx == 0)
1334  widenScalarDst(MI, WideTy);
1335  else {
1336  unsigned ExtOpcode = CmpInst::isSigned(static_cast<CmpInst::Predicate>(
1337  MI.getOperand(1).getPredicate()))
1338  ? TargetOpcode::G_SEXT
1339  : TargetOpcode::G_ZEXT;
1340  widenScalarSrc(MI, WideTy, 2, ExtOpcode);
1341  widenScalarSrc(MI, WideTy, 3, ExtOpcode);
1342  }
1343  Observer.changedInstr(MI);
1344  return Legalized;
1345 
1346  case TargetOpcode::G_GEP:
1347  assert(TypeIdx == 1 && "unable to legalize pointer of GEP");
1348  Observer.changingInstr(MI);
1349  widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
1350  Observer.changedInstr(MI);
1351  return Legalized;
1352 
1353  case TargetOpcode::G_PHI: {
1354  assert(TypeIdx == 0 && "Expecting only Idx 0");
1355 
1356  Observer.changingInstr(MI);
1357  for (unsigned I = 1; I < MI.getNumOperands(); I += 2) {
1358  MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
1359  MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
1360  widenScalarSrc(MI, WideTy, I, TargetOpcode::G_ANYEXT);
1361  }
1362 
1363  MachineBasicBlock &MBB = *MI.getParent();
1364  MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI());
1365  widenScalarDst(MI, WideTy);
1366  Observer.changedInstr(MI);
1367  return Legalized;
1368  }
1369  case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
1370  if (TypeIdx == 0) {
1371  Register VecReg = MI.getOperand(1).getReg();
1372  LLT VecTy = MRI.getType(VecReg);
1373  Observer.changingInstr(MI);
1374 
1375  widenScalarSrc(MI, LLT::vector(VecTy.getNumElements(),
1376  WideTy.getSizeInBits()),
1377  1, TargetOpcode::G_SEXT);
1378 
1379  widenScalarDst(MI, WideTy, 0);
1380  Observer.changedInstr(MI);
1381  return Legalized;
1382  }
1383 
1384  if (TypeIdx != 2)
1385  return UnableToLegalize;
1386  Observer.changingInstr(MI);
1387  widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
1388  Observer.changedInstr(MI);
1389  return Legalized;
1390  }
1391  case TargetOpcode::G_FADD:
1392  case TargetOpcode::G_FMUL:
1393  case TargetOpcode::G_FSUB:
1394  case TargetOpcode::G_FMA:
1395  case TargetOpcode::G_FNEG:
1396  case TargetOpcode::G_FABS:
1397  case TargetOpcode::G_FCANONICALIZE:
1398  case TargetOpcode::G_FMINNUM:
1399  case TargetOpcode::G_FMAXNUM:
1400  case TargetOpcode::G_FMINNUM_IEEE:
1401  case TargetOpcode::G_FMAXNUM_IEEE:
1402  case TargetOpcode::G_FMINIMUM:
1403  case TargetOpcode::G_FMAXIMUM:
1404  case TargetOpcode::G_FDIV:
1405  case TargetOpcode::G_FREM:
1406  case TargetOpcode::G_FCEIL:
1407  case TargetOpcode::G_FFLOOR:
1408  case TargetOpcode::G_FCOS:
1409  case TargetOpcode::G_FSIN:
1410  case TargetOpcode::G_FLOG10:
1411  case TargetOpcode::G_FLOG:
1412  case TargetOpcode::G_FLOG2:
1413  case TargetOpcode::G_FRINT:
1414  case TargetOpcode::G_FNEARBYINT:
1415  case TargetOpcode::G_FSQRT:
1416  case TargetOpcode::G_FEXP:
1417  case TargetOpcode::G_FEXP2:
1418  case TargetOpcode::G_FPOW:
1419  case TargetOpcode::G_INTRINSIC_TRUNC:
1420  case TargetOpcode::G_INTRINSIC_ROUND:
1421  assert(TypeIdx == 0);
1422  Observer.changingInstr(MI);
1423 
1424  for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I)
1425  widenScalarSrc(MI, WideTy, I, TargetOpcode::G_FPEXT);
1426 
1427  widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
1428  Observer.changedInstr(MI);
1429  return Legalized;
1430  case TargetOpcode::G_INTTOPTR:
1431  if (TypeIdx != 1)
1432  return UnableToLegalize;
1433 
1434  Observer.changingInstr(MI);
1435  widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
1436  Observer.changedInstr(MI);
1437  return Legalized;
1438  case TargetOpcode::G_PTRTOINT:
1439  if (TypeIdx != 0)
1440  return UnableToLegalize;
1441 
1442  Observer.changingInstr(MI);
1443  widenScalarDst(MI, WideTy, 0);
1444  Observer.changedInstr(MI);
1445  return Legalized;
1446  case TargetOpcode::G_BUILD_VECTOR: {
1447  Observer.changingInstr(MI);
1448 
1449  const LLT WideEltTy = TypeIdx == 1 ? WideTy : WideTy.getElementType();
1450  for (int I = 1, E = MI.getNumOperands(); I != E; ++I)
1451  widenScalarSrc(MI, WideEltTy, I, TargetOpcode::G_ANYEXT);
1452 
1453  // Avoid changing the result vector type if the source element type was
1454  // requested.
1455  if (TypeIdx == 1) {
1456  auto &TII = *MI.getMF()->getSubtarget().getInstrInfo();
1457  MI.setDesc(TII.get(TargetOpcode::G_BUILD_VECTOR_TRUNC));
1458  } else {
1459  widenScalarDst(MI, WideTy, 0);
1460  }
1461 
1462  Observer.changedInstr(MI);
1463  return Legalized;
1464  }
1465  }
1466 }
1467 
1469 LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
1470  using namespace TargetOpcode;
1471  MIRBuilder.setInstr(MI);
1472 
1473  switch(MI.getOpcode()) {
1474  default:
1475  return UnableToLegalize;
1476  case TargetOpcode::G_SREM:
1477  case TargetOpcode::G_UREM: {
1478  Register QuotReg = MRI.createGenericVirtualRegister(Ty);
1479  MIRBuilder.buildInstr(MI.getOpcode() == G_SREM ? G_SDIV : G_UDIV)
1480  .addDef(QuotReg)
1481  .addUse(MI.getOperand(1).getReg())
1482  .addUse(MI.getOperand(2).getReg());
1483 
1484  Register ProdReg = MRI.createGenericVirtualRegister(Ty);
1485  MIRBuilder.buildMul(ProdReg, QuotReg, MI.getOperand(2).getReg());
1487  ProdReg);
1488  MI.eraseFromParent();
1489  return Legalized;
1490  }
1491  case TargetOpcode::G_SMULO:
1492  case TargetOpcode::G_UMULO: {
1493  // Generate G_UMULH/G_SMULH to check for overflow and a normal G_MUL for the
1494  // result.
1495  Register Res = MI.getOperand(0).getReg();
1496  Register Overflow = MI.getOperand(1).getReg();
1497  Register LHS = MI.getOperand(2).getReg();
1498  Register RHS = MI.getOperand(3).getReg();
1499 
1500  MIRBuilder.buildMul(Res, LHS, RHS);
1501 
1502  unsigned Opcode = MI.getOpcode() == TargetOpcode::G_SMULO
1503  ? TargetOpcode::G_SMULH
1504  : TargetOpcode::G_UMULH;
1505 
1506  Register HiPart = MRI.createGenericVirtualRegister(Ty);
1507  MIRBuilder.buildInstr(Opcode)
1508  .addDef(HiPart)
1509  .addUse(LHS)
1510  .addUse(RHS);
1511 
1512  Register Zero = MRI.createGenericVirtualRegister(Ty);
1513  MIRBuilder.buildConstant(Zero, 0);
1514 
1515  // For *signed* multiply, overflow is detected by checking:
1516  // (hi != (lo >> bitwidth-1))
1517  if (Opcode == TargetOpcode::G_SMULH) {
1518  Register Shifted = MRI.createGenericVirtualRegister(Ty);
1519  Register ShiftAmt = MRI.createGenericVirtualRegister(Ty);
1520  MIRBuilder.buildConstant(ShiftAmt, Ty.getSizeInBits() - 1);
1521  MIRBuilder.buildInstr(TargetOpcode::G_ASHR)
1522  .addDef(Shifted)
1523  .addUse(Res)
1524  .addUse(ShiftAmt);
1525  MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Shifted);
1526  } else {
1527  MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Zero);
1528  }
1529  MI.eraseFromParent();
1530  return Legalized;
1531  }
1532  case TargetOpcode::G_FNEG: {
1533  // TODO: Handle vector types once we are able to
1534  // represent them.
1535  if (Ty.isVector())
1536  return UnableToLegalize;
1537  Register Res = MI.getOperand(0).getReg();
1538  Type *ZeroTy;
1540  switch (Ty.getSizeInBits()) {
1541  case 16:
1542  ZeroTy = Type::getHalfTy(Ctx);
1543  break;
1544  case 32:
1545  ZeroTy = Type::getFloatTy(Ctx);
1546  break;
1547  case 64:
1548  ZeroTy = Type::getDoubleTy(Ctx);
1549  break;
1550  case 128:
1551  ZeroTy = Type::getFP128Ty(Ctx);
1552  break;
1553  default:
1554  llvm_unreachable("unexpected floating-point type");
1555  }
1556  ConstantFP &ZeroForNegation =
1557  *cast<ConstantFP>(ConstantFP::getZeroValueForNegation(ZeroTy));
1558  auto Zero = MIRBuilder.buildFConstant(Ty, ZeroForNegation);
1559  Register SubByReg = MI.getOperand(1).getReg();
1560  Register ZeroReg = Zero->getOperand(0).getReg();
1561  MIRBuilder.buildInstr(TargetOpcode::G_FSUB, {Res}, {ZeroReg, SubByReg},
1562  MI.getFlags());
1563  MI.eraseFromParent();
1564  return Legalized;
1565  }
1566  case TargetOpcode::G_FSUB: {
1567  // Lower (G_FSUB LHS, RHS) to (G_FADD LHS, (G_FNEG RHS)).
1568  // First, check if G_FNEG is marked as Lower. If so, we may
1569  // end up with an infinite loop as G_FSUB is used to legalize G_FNEG.
1570  if (LI.getAction({G_FNEG, {Ty}}).Action == Lower)
1571  return UnableToLegalize;
1572  Register Res = MI.getOperand(0).getReg();
1573  Register LHS = MI.getOperand(1).getReg();
1574  Register RHS = MI.getOperand(2).getReg();
1575  Register Neg = MRI.createGenericVirtualRegister(Ty);
1576  MIRBuilder.buildInstr(TargetOpcode::G_FNEG).addDef(Neg).addUse(RHS);
1577  MIRBuilder.buildInstr(TargetOpcode::G_FADD, {Res}, {LHS, Neg}, MI.getFlags());
1578  MI.eraseFromParent();
1579  return Legalized;
1580  }
1581  case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
1582  Register OldValRes = MI.getOperand(0).getReg();
1583  Register SuccessRes = MI.getOperand(1).getReg();
1584  Register Addr = MI.getOperand(2).getReg();
1585  Register CmpVal = MI.getOperand(3).getReg();
1586  Register NewVal = MI.getOperand(4).getReg();
1587  MIRBuilder.buildAtomicCmpXchg(OldValRes, Addr, CmpVal, NewVal,
1588  **MI.memoperands_begin());
1589  MIRBuilder.buildICmp(CmpInst::ICMP_EQ, SuccessRes, OldValRes, CmpVal);
1590  MI.eraseFromParent();
1591  return Legalized;
1592  }
1593  case TargetOpcode::G_LOAD:
1594  case TargetOpcode::G_SEXTLOAD:
1595  case TargetOpcode::G_ZEXTLOAD: {
1596  // Lower to a memory-width G_LOAD and a G_SEXT/G_ZEXT/G_ANYEXT
1597  Register DstReg = MI.getOperand(0).getReg();
1598  Register PtrReg = MI.getOperand(1).getReg();
1599  LLT DstTy = MRI.getType(DstReg);
1600  auto &MMO = **MI.memoperands_begin();
1601 
1602  if (DstTy.getSizeInBits() == MMO.getSize() /* in bytes */ * 8) {
1603  // In the case of G_LOAD, this was a non-extending load already and we're
1604  // about to lower to the same instruction.
1605  if (MI.getOpcode() == TargetOpcode::G_LOAD)
1606  return UnableToLegalize;
1607  MIRBuilder.buildLoad(DstReg, PtrReg, MMO);
1608  MI.eraseFromParent();
1609  return Legalized;
1610  }
1611 
1612  if (DstTy.isScalar()) {
1613  Register TmpReg =
1614  MRI.createGenericVirtualRegister(LLT::scalar(MMO.getSizeInBits()));
1615  MIRBuilder.buildLoad(TmpReg, PtrReg, MMO);
1616  switch (MI.getOpcode()) {
1617  default:
1618  llvm_unreachable("Unexpected opcode");
1619  case TargetOpcode::G_LOAD:
1620  MIRBuilder.buildAnyExt(DstReg, TmpReg);
1621  break;
1622  case TargetOpcode::G_SEXTLOAD:
1623  MIRBuilder.buildSExt(DstReg, TmpReg);
1624  break;
1625  case TargetOpcode::G_ZEXTLOAD:
1626  MIRBuilder.buildZExt(DstReg, TmpReg);
1627  break;
1628  }
1629  MI.eraseFromParent();
1630  return Legalized;
1631  }
1632 
1633  return UnableToLegalize;
1634  }
1635  case TargetOpcode::G_CTLZ_ZERO_UNDEF:
1636  case TargetOpcode::G_CTTZ_ZERO_UNDEF:
1637  case TargetOpcode::G_CTLZ:
1638  case TargetOpcode::G_CTTZ:
1639  case TargetOpcode::G_CTPOP:
1640  return lowerBitCount(MI, TypeIdx, Ty);
1641  case G_UADDO: {
1642  Register Res = MI.getOperand(0).getReg();
1643  Register CarryOut = MI.getOperand(1).getReg();
1644  Register LHS = MI.getOperand(2).getReg();
1645  Register RHS = MI.getOperand(3).getReg();
1646 
1647  MIRBuilder.buildAdd(Res, LHS, RHS);
1648  MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, RHS);
1649 
1650  MI.eraseFromParent();
1651  return Legalized;
1652  }
1653  case G_UADDE: {
1654  Register Res = MI.getOperand(0).getReg();
1655  Register CarryOut = MI.getOperand(1).getReg();
1656  Register LHS = MI.getOperand(2).getReg();
1657  Register RHS = MI.getOperand(3).getReg();
1658  Register CarryIn = MI.getOperand(4).getReg();
1659 
1660  Register TmpRes = MRI.createGenericVirtualRegister(Ty);
1661  Register ZExtCarryIn = MRI.createGenericVirtualRegister(Ty);
1662 
1663  MIRBuilder.buildAdd(TmpRes, LHS, RHS);
1664  MIRBuilder.buildZExt(ZExtCarryIn, CarryIn);
1665  MIRBuilder.buildAdd(Res, TmpRes, ZExtCarryIn);
1666  MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, LHS);
1667 
1668  MI.eraseFromParent();
1669  return Legalized;
1670  }
1671  case G_USUBO: {
1672  Register Res = MI.getOperand(0).getReg();
1673  Register BorrowOut = MI.getOperand(1).getReg();
1674  Register LHS = MI.getOperand(2).getReg();
1675  Register RHS = MI.getOperand(3).getReg();
1676 
1677  MIRBuilder.buildSub(Res, LHS, RHS);
1678  MIRBuilder.buildICmp(CmpInst::ICMP_ULT, BorrowOut, LHS, RHS);
1679 
1680  MI.eraseFromParent();
1681  return Legalized;
1682  }
1683  case G_USUBE: {
1684  Register Res = MI.getOperand(0).getReg();
1685  Register BorrowOut = MI.getOperand(1).getReg();
1686  Register LHS = MI.getOperand(2).getReg();
1687  Register RHS = MI.getOperand(3).getReg();
1688  Register BorrowIn = MI.getOperand(4).getReg();
1689 
1690  Register TmpRes = MRI.createGenericVirtualRegister(Ty);
1691  Register ZExtBorrowIn = MRI.createGenericVirtualRegister(Ty);
1692  Register LHS_EQ_RHS = MRI.createGenericVirtualRegister(LLT::scalar(1));
1693  Register LHS_ULT_RHS = MRI.createGenericVirtualRegister(LLT::scalar(1));
1694 
1695  MIRBuilder.buildSub(TmpRes, LHS, RHS);
1696  MIRBuilder.buildZExt(ZExtBorrowIn, BorrowIn);
1697  MIRBuilder.buildSub(Res, TmpRes, ZExtBorrowIn);
1698  MIRBuilder.buildICmp(CmpInst::ICMP_EQ, LHS_EQ_RHS, LHS, RHS);
1699  MIRBuilder.buildICmp(CmpInst::ICMP_ULT, LHS_ULT_RHS, LHS, RHS);
1700  MIRBuilder.buildSelect(BorrowOut, LHS_EQ_RHS, BorrowIn, LHS_ULT_RHS);
1701 
1702  MI.eraseFromParent();
1703  return Legalized;
1704  }
1705  case G_UITOFP:
1706  return lowerUITOFP(MI, TypeIdx, Ty);
1707  case G_SITOFP:
1708  return lowerSITOFP(MI, TypeIdx, Ty);
1709  case G_SMIN:
1710  case G_SMAX:
1711  case G_UMIN:
1712  case G_UMAX:
1713  return lowerMinMax(MI, TypeIdx, Ty);
1714  case G_FCOPYSIGN:
1715  return lowerFCopySign(MI, TypeIdx, Ty);
1716  case G_FMINNUM:
1717  case G_FMAXNUM:
1718  return lowerFMinNumMaxNum(MI);
1719  }
1720 }
1721 
1723  MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) {
1724  SmallVector<Register, 2> DstRegs;
1725 
1726  unsigned NarrowSize = NarrowTy.getSizeInBits();
1727  Register DstReg = MI.getOperand(0).getReg();
1728  unsigned Size = MRI.getType(DstReg).getSizeInBits();
1729  int NumParts = Size / NarrowSize;
1730  // FIXME: Don't know how to handle the situation where the small vectors
1731  // aren't all the same size yet.
1732  if (Size % NarrowSize != 0)
1733  return UnableToLegalize;
1734 
1735  for (int i = 0; i < NumParts; ++i) {
1736  Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
1737  MIRBuilder.buildUndef(TmpReg);
1738  DstRegs.push_back(TmpReg);
1739  }
1740 
1741  if (NarrowTy.isVector())
1742  MIRBuilder.buildConcatVectors(DstReg, DstRegs);
1743  else
1744  MIRBuilder.buildBuildVector(DstReg, DstRegs);
1745 
1746  MI.eraseFromParent();
1747  return Legalized;
1748 }
1749 
1752  LLT NarrowTy) {
1753  const unsigned Opc = MI.getOpcode();
1754  const unsigned NumOps = MI.getNumOperands() - 1;
1755  const unsigned NarrowSize = NarrowTy.getSizeInBits();
1756  const Register DstReg = MI.getOperand(0).getReg();
1757  const unsigned Flags = MI.getFlags();
1758  const LLT DstTy = MRI.getType(DstReg);
1759  const unsigned Size = DstTy.getSizeInBits();
1760  const int NumParts = Size / NarrowSize;
1761  const LLT EltTy = DstTy.getElementType();
1762  const unsigned EltSize = EltTy.getSizeInBits();
1763  const unsigned BitsForNumParts = NarrowSize * NumParts;
1764 
1765  // Check if we have any leftovers. If we do, then only handle the case where
1766  // the leftover is one element.
1767  if (BitsForNumParts != Size && BitsForNumParts + EltSize != Size)
1768  return UnableToLegalize;
1769 
1770  if (BitsForNumParts != Size) {
1771  Register AccumDstReg = MRI.createGenericVirtualRegister(DstTy);
1772  MIRBuilder.buildUndef(AccumDstReg);
1773 
1774  // Handle the pieces which evenly divide into the requested type with
1775  // extract/op/insert sequence.
1776  for (unsigned Offset = 0; Offset < BitsForNumParts; Offset += NarrowSize) {
1777  SmallVector<SrcOp, 4> SrcOps;
1778  for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
1779  Register PartOpReg = MRI.createGenericVirtualRegister(NarrowTy);
1780  MIRBuilder.buildExtract(PartOpReg, MI.getOperand(I).getReg(), Offset);
1781  SrcOps.push_back(PartOpReg);
1782  }
1783 
1784  Register PartDstReg = MRI.createGenericVirtualRegister(NarrowTy);
1785  MIRBuilder.buildInstr(Opc, {PartDstReg}, SrcOps, Flags);
1786 
1787  Register PartInsertReg = MRI.createGenericVirtualRegister(DstTy);
1788  MIRBuilder.buildInsert(PartInsertReg, AccumDstReg, PartDstReg, Offset);
1789  AccumDstReg = PartInsertReg;
1790  }
1791 
1792  // Handle the remaining element sized leftover piece.
1793  SmallVector<SrcOp, 4> SrcOps;
1794  for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
1795  Register PartOpReg = MRI.createGenericVirtualRegister(EltTy);
1796  MIRBuilder.buildExtract(PartOpReg, MI.getOperand(I).getReg(),
1797  BitsForNumParts);
1798  SrcOps.push_back(PartOpReg);
1799  }
1800 
1801  Register PartDstReg = MRI.createGenericVirtualRegister(EltTy);
1802  MIRBuilder.buildInstr(Opc, {PartDstReg}, SrcOps, Flags);
1803  MIRBuilder.buildInsert(DstReg, AccumDstReg, PartDstReg, BitsForNumParts);
1804  MI.eraseFromParent();
1805 
1806  return Legalized;
1807  }
1808 
1809  SmallVector<Register, 2> DstRegs, Src0Regs, Src1Regs, Src2Regs;
1810 
1811  extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src0Regs);
1812 
1813  if (NumOps >= 2)
1814  extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src1Regs);
1815 
1816  if (NumOps >= 3)
1817  extractParts(MI.getOperand(3).getReg(), NarrowTy, NumParts, Src2Regs);
1818 
1819  for (int i = 0; i < NumParts; ++i) {
1820  Register DstReg = MRI.createGenericVirtualRegister(NarrowTy);
1821 
1822  if (NumOps == 1)
1823  MIRBuilder.buildInstr(Opc, {DstReg}, {Src0Regs[i]}, Flags);
1824  else if (NumOps == 2) {
1825  MIRBuilder.buildInstr(Opc, {DstReg}, {Src0Regs[i], Src1Regs[i]}, Flags);
1826  } else if (NumOps == 3) {
1827  MIRBuilder.buildInstr(Opc, {DstReg},
1828  {Src0Regs[i], Src1Regs[i], Src2Regs[i]}, Flags);
1829  }
1830 
1831  DstRegs.push_back(DstReg);
1832  }
1833 
1834  if (NarrowTy.isVector())
1835  MIRBuilder.buildConcatVectors(DstReg, DstRegs);
1836  else
1837  MIRBuilder.buildBuildVector(DstReg, DstRegs);
1838 
1839  MI.eraseFromParent();
1840  return Legalized;
1841 }
1842 
1843 // Handle splitting vector operations which need to have the same number of
1844 // elements in each type index, but each type index may have a different element
1845 // type.
1846 //
1847 // e.g. <4 x s64> = G_SHL <4 x s64>, <4 x s32> ->
1848 // <2 x s64> = G_SHL <2 x s64>, <2 x s32>
1849 // <2 x s64> = G_SHL <2 x s64>, <2 x s32>
1850 //
1851 // Also handles some irregular breakdown cases, e.g.
1852 // e.g. <3 x s64> = G_SHL <3 x s64>, <3 x s32> ->
1853 // <2 x s64> = G_SHL <2 x s64>, <2 x s32>
1854 // s64 = G_SHL s64, s32
1857  MachineInstr &MI, unsigned TypeIdx, LLT NarrowTyArg) {
1858  if (TypeIdx != 0)
1859  return UnableToLegalize;
1860 
1861  const LLT NarrowTy0 = NarrowTyArg;
1862  const unsigned NewNumElts =
1863  NarrowTy0.isVector() ? NarrowTy0.getNumElements() : 1;
1864 
1865  const Register DstReg = MI.getOperand(0).getReg();
1866  LLT DstTy = MRI.getType(DstReg);
1867  LLT LeftoverTy0;
1868 
1869  // All of the operands need to have the same number of elements, so if we can
1870  // determine a type breakdown for the result type, we can for all of the
1871  // source types.
1872  int NumParts = getNarrowTypeBreakDown(DstTy, NarrowTy0, LeftoverTy0).first;
1873  if (NumParts < 0)
1874  return UnableToLegalize;
1875 
1877 
1878  SmallVector<Register, 4> DstRegs, LeftoverDstRegs;
1879  SmallVector<Register, 4> PartRegs, LeftoverRegs;
1880 
1881  for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
1882  LLT LeftoverTy;
1883  Register SrcReg = MI.getOperand(I).getReg();
1884  LLT SrcTyI = MRI.getType(SrcReg);
1885  LLT NarrowTyI = LLT::scalarOrVector(NewNumElts, SrcTyI.getScalarType());
1886  LLT LeftoverTyI;
1887 
1888  // Split this operand into the requested typed registers, and any leftover
1889  // required to reproduce the original type.
1890  if (!extractParts(SrcReg, SrcTyI, NarrowTyI, LeftoverTyI, PartRegs,
1891  LeftoverRegs))
1892  return UnableToLegalize;
1893 
1894  if (I == 1) {
1895  // For the first operand, create an instruction for each part and setup
1896  // the result.
1897  for (Register PartReg : PartRegs) {
1898  Register PartDstReg = MRI.createGenericVirtualRegister(NarrowTy0);
1900  .addDef(PartDstReg)
1901  .addUse(PartReg));
1902  DstRegs.push_back(PartDstReg);
1903  }
1904 
1905  for (Register LeftoverReg : LeftoverRegs) {
1906  Register PartDstReg = MRI.createGenericVirtualRegister(LeftoverTy0);
1908  .addDef(PartDstReg)
1909  .addUse(LeftoverReg));
1910  LeftoverDstRegs.push_back(PartDstReg);
1911  }
1912  } else {
1913  assert(NewInsts.size() == PartRegs.size() + LeftoverRegs.size());
1914 
1915  // Add the newly created operand splits to the existing instructions. The
1916  // odd-sized pieces are ordered after the requested NarrowTyArg sized
1917  // pieces.
1918  unsigned InstCount = 0;
1919  for (unsigned J = 0, JE = PartRegs.size(); J != JE; ++J)
1920  NewInsts[InstCount++].addUse(PartRegs[J]);
1921  for (unsigned J = 0, JE = LeftoverRegs.size(); J != JE; ++J)
1922  NewInsts[InstCount++].addUse(LeftoverRegs[J]);
1923  }
1924 
1925  PartRegs.clear();
1926  LeftoverRegs.clear();
1927  }
1928 
1929  // Insert the newly built operations and rebuild the result register.
1930  for (auto &MIB : NewInsts)
1931  MIRBuilder.insertInstr(MIB);
1932 
1933  insertParts(DstReg, DstTy, NarrowTy0, DstRegs, LeftoverTy0, LeftoverDstRegs);
1934 
1935  MI.eraseFromParent();
1936  return Legalized;
1937 }
1938 
1941  LLT NarrowTy) {
1942  if (TypeIdx != 0)
1943  return UnableToLegalize;
1944 
1945  Register DstReg = MI.getOperand(0).getReg();
1946  Register SrcReg = MI.getOperand(1).getReg();
1947  LLT DstTy = MRI.getType(DstReg);
1948  LLT SrcTy = MRI.getType(SrcReg);
1949 
1950  LLT NarrowTy0 = NarrowTy;
1951  LLT NarrowTy1;
1952  unsigned NumParts;
1953 
1954  if (NarrowTy.isVector()) {
1955  // Uneven breakdown not handled.
1956  NumParts = DstTy.getNumElements() / NarrowTy.getNumElements();
1957  if (NumParts * NarrowTy.getNumElements() != DstTy.getNumElements())
1958  return UnableToLegalize;
1959 
1960  NarrowTy1 = LLT::vector(NumParts, SrcTy.getElementType().getSizeInBits());
1961  } else {
1962  NumParts = DstTy.getNumElements();
1963  NarrowTy1 = SrcTy.getElementType();
1964  }
1965 
1966  SmallVector<Register, 4> SrcRegs, DstRegs;
1967  extractParts(SrcReg, NarrowTy1, NumParts, SrcRegs);
1968 
1969  for (unsigned I = 0; I < NumParts; ++I) {
1970  Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
1971  MachineInstr *NewInst = MIRBuilder.buildInstr(MI.getOpcode())
1972  .addDef(DstReg)
1973  .addUse(SrcRegs[I]);
1974 
1975  NewInst->setFlags(MI.getFlags());
1976  DstRegs.push_back(DstReg);
1977  }
1978 
1979  if (NarrowTy.isVector())
1980  MIRBuilder.buildConcatVectors(DstReg, DstRegs);
1981  else
1982  MIRBuilder.buildBuildVector(DstReg, DstRegs);
1983 
1984  MI.eraseFromParent();
1985  return Legalized;
1986 }
1987 
1990  LLT NarrowTy) {
1991  Register DstReg = MI.getOperand(0).getReg();
1992  Register Src0Reg = MI.getOperand(2).getReg();
1993  LLT DstTy = MRI.getType(DstReg);
1994  LLT SrcTy = MRI.getType(Src0Reg);
1995 
1996  unsigned NumParts;
1997  LLT NarrowTy0, NarrowTy1;
1998 
1999  if (TypeIdx == 0) {
2000  unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
2001  unsigned OldElts = DstTy.getNumElements();
2002 
2003  NarrowTy0 = NarrowTy;
2004  NumParts = NarrowTy.isVector() ? (OldElts / NewElts) : DstTy.getNumElements();
2005  NarrowTy1 = NarrowTy.isVector() ?
2006  LLT::vector(NarrowTy.getNumElements(), SrcTy.getScalarSizeInBits()) :
2007  SrcTy.getElementType();
2008 
2009  } else {
2010  unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
2011  unsigned OldElts = SrcTy.getNumElements();
2012 
2013  NumParts = NarrowTy.isVector() ? (OldElts / NewElts) :
2014  NarrowTy.getNumElements();
2015  NarrowTy0 = LLT::vector(NarrowTy.getNumElements(),
2016  DstTy.getScalarSizeInBits());
2017  NarrowTy1 = NarrowTy;
2018  }
2019 
2020  // FIXME: Don't know how to handle the situation where the small vectors
2021  // aren't all the same size yet.
2022  if (NarrowTy1.isVector() &&
2023  NarrowTy1.getNumElements() * NumParts != DstTy.getNumElements())
2024  return UnableToLegalize;
2025 
2026  CmpInst::Predicate Pred
2027  = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
2028 
2029  SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs;
2030  extractParts(MI.getOperand(2).getReg(), NarrowTy1, NumParts, Src1Regs);
2031  extractParts(MI.getOperand(3).getReg(), NarrowTy1, NumParts, Src2Regs);
2032 
2033  for (unsigned I = 0; I < NumParts; ++I) {
2034  Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
2035  DstRegs.push_back(DstReg);
2036 
2037  if (MI.getOpcode() == TargetOpcode::G_ICMP)
2038  MIRBuilder.buildICmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]);
2039  else {
2040  MachineInstr *NewCmp
2041  = MIRBuilder.buildFCmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]);
2042  NewCmp->setFlags(MI.getFlags());
2043  }
2044  }
2045 
2046  if (NarrowTy1.isVector())
2047  MIRBuilder.buildConcatVectors(DstReg, DstRegs);
2048  else
2049  MIRBuilder.buildBuildVector(DstReg, DstRegs);
2050 
2051  MI.eraseFromParent();
2052  return Legalized;
2053 }
2054 
2057  LLT NarrowTy) {
2058  Register DstReg = MI.getOperand(0).getReg();
2059  Register CondReg = MI.getOperand(1).getReg();
2060 
2061  unsigned NumParts = 0;
2062  LLT NarrowTy0, NarrowTy1;
2063 
2064  LLT DstTy = MRI.getType(DstReg);
2065  LLT CondTy = MRI.getType(CondReg);
2066  unsigned Size = DstTy.getSizeInBits();
2067 
2068  assert(TypeIdx == 0 || CondTy.isVector());
2069 
2070  if (TypeIdx == 0) {
2071  NarrowTy0 = NarrowTy;
2072  NarrowTy1 = CondTy;
2073 
2074  unsigned NarrowSize = NarrowTy0.getSizeInBits();
2075  // FIXME: Don't know how to handle the situation where the small vectors
2076  // aren't all the same size yet.
2077  if (Size % NarrowSize != 0)
2078  return UnableToLegalize;
2079 
2080  NumParts = Size / NarrowSize;
2081 
2082  // Need to break down the condition type
2083  if (CondTy.isVector()) {
2084  if (CondTy.getNumElements() == NumParts)
2085  NarrowTy1 = CondTy.getElementType();
2086  else
2087  NarrowTy1 = LLT::vector(CondTy.getNumElements() / NumParts,
2088  CondTy.getScalarSizeInBits());
2089  }
2090  } else {
2091  NumParts = CondTy.getNumElements();
2092  if (NarrowTy.isVector()) {
2093  // TODO: Handle uneven breakdown.
2094  if (NumParts * NarrowTy.getNumElements() != CondTy.getNumElements())
2095  return UnableToLegalize;
2096 
2097  return UnableToLegalize;
2098  } else {
2099  NarrowTy0 = DstTy.getElementType();
2100  NarrowTy1 = NarrowTy;
2101  }
2102  }
2103 
2104  SmallVector<Register, 2> DstRegs, Src0Regs, Src1Regs, Src2Regs;
2105  if (CondTy.isVector())
2106  extractParts(MI.getOperand(1).getReg(), NarrowTy1, NumParts, Src0Regs);
2107 
2108  extractParts(MI.getOperand(2).getReg(), NarrowTy0, NumParts, Src1Regs);
2109  extractParts(MI.getOperand(3).getReg(), NarrowTy0, NumParts, Src2Regs);
2110 
2111  for (unsigned i = 0; i < NumParts; ++i) {
2112  Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
2113  MIRBuilder.buildSelect(DstReg, CondTy.isVector() ? Src0Regs[i] : CondReg,
2114  Src1Regs[i], Src2Regs[i]);
2115  DstRegs.push_back(DstReg);
2116  }
2117 
2118  if (NarrowTy0.isVector())
2119  MIRBuilder.buildConcatVectors(DstReg, DstRegs);
2120  else
2121  MIRBuilder.buildBuildVector(DstReg, DstRegs);
2122 
2123  MI.eraseFromParent();
2124  return Legalized;
2125 }
2126 
2129  LLT NarrowTy) {
2130  const Register DstReg = MI.getOperand(0).getReg();
2131  LLT PhiTy = MRI.getType(DstReg);
2132  LLT LeftoverTy;
2133 
2134  // All of the operands need to have the same number of elements, so if we can
2135  // determine a type breakdown for the result type, we can for all of the
2136  // source types.
2137  int NumParts, NumLeftover;
2138  std::tie(NumParts, NumLeftover)
2139  = getNarrowTypeBreakDown(PhiTy, NarrowTy, LeftoverTy);
2140  if (NumParts < 0)
2141  return UnableToLegalize;
2142 
2143  SmallVector<Register, 4> DstRegs, LeftoverDstRegs;
2145 
2146  const int TotalNumParts = NumParts + NumLeftover;
2147 
2148  // Insert the new phis in the result block first.
2149  for (int I = 0; I != TotalNumParts; ++I) {
2150  LLT Ty = I < NumParts ? NarrowTy : LeftoverTy;
2151  Register PartDstReg = MRI.createGenericVirtualRegister(Ty);
2152  NewInsts.push_back(MIRBuilder.buildInstr(TargetOpcode::G_PHI)
2153  .addDef(PartDstReg));
2154  if (I < NumParts)
2155  DstRegs.push_back(PartDstReg);
2156  else
2157  LeftoverDstRegs.push_back(PartDstReg);
2158  }
2159 
2160  MachineBasicBlock *MBB = MI.getParent();
2161  MIRBuilder.setInsertPt(*MBB, MBB->getFirstNonPHI());
2162  insertParts(DstReg, PhiTy, NarrowTy, DstRegs, LeftoverTy, LeftoverDstRegs);
2163 
2164  SmallVector<Register, 4> PartRegs, LeftoverRegs;
2165 
2166  // Insert code to extract the incoming values in each predecessor block.
2167  for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
2168  PartRegs.clear();
2169  LeftoverRegs.clear();
2170 
2171  Register SrcReg = MI.getOperand(I).getReg();
2172  MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
2173  MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
2174 
2175  LLT Unused;
2176  if (!extractParts(SrcReg, PhiTy, NarrowTy, Unused, PartRegs,
2177  LeftoverRegs))
2178  return UnableToLegalize;
2179 
2180  // Add the newly created operand splits to the existing instructions. The
2181  // odd-sized pieces are ordered after the requested NarrowTyArg sized
2182  // pieces.
2183  for (int J = 0; J != TotalNumParts; ++J) {
2184  MachineInstrBuilder MIB = NewInsts[J];
2185  MIB.addUse(J < NumParts ? PartRegs[J] : LeftoverRegs[J - NumParts]);
2186  MIB.addMBB(&OpMBB);
2187  }
2188  }
2189 
2190  MI.eraseFromParent();
2191  return Legalized;
2192 }
2193 
2196  LLT NarrowTy) {
2197  // FIXME: Don't know how to handle secondary types yet.
2198  if (TypeIdx != 0)
2199  return UnableToLegalize;
2200 
2201  MachineMemOperand *MMO = *MI.memoperands_begin();
2202 
2203  // This implementation doesn't work for atomics. Give up instead of doing
2204  // something invalid.
2205  if (MMO->getOrdering() != AtomicOrdering::NotAtomic ||
2207  return UnableToLegalize;
2208 
2209  bool IsLoad = MI.getOpcode() == TargetOpcode::G_LOAD;
2210  Register ValReg = MI.getOperand(0).getReg();
2211  Register AddrReg = MI.getOperand(1).getReg();
2212  LLT ValTy = MRI.getType(ValReg);
2213 
2214  int NumParts = -1;
2215  int NumLeftover = -1;
2216  LLT LeftoverTy;
2217  SmallVector<Register, 8> NarrowRegs, NarrowLeftoverRegs;
2218  if (IsLoad) {
2219  std::tie(NumParts, NumLeftover) = getNarrowTypeBreakDown(ValTy, NarrowTy, LeftoverTy);
2220  } else {
2221  if (extractParts(ValReg, ValTy, NarrowTy, LeftoverTy, NarrowRegs,
2222  NarrowLeftoverRegs)) {
2223  NumParts = NarrowRegs.size();
2224  NumLeftover = NarrowLeftoverRegs.size();
2225  }
2226  }
2227 
2228  if (NumParts == -1)
2229  return UnableToLegalize;
2230 
2231  const LLT OffsetTy = LLT::scalar(MRI.getType(AddrReg).getScalarSizeInBits());
2232 
2233  unsigned TotalSize = ValTy.getSizeInBits();
2234 
2235  // Split the load/store into PartTy sized pieces starting at Offset. If this
2236  // is a load, return the new registers in ValRegs. For a store, each elements
2237  // of ValRegs should be PartTy. Returns the next offset that needs to be
2238  // handled.
2239  auto splitTypePieces = [=](LLT PartTy, SmallVectorImpl<Register> &ValRegs,
2240  unsigned Offset) -> unsigned {
2242  unsigned PartSize = PartTy.getSizeInBits();
2243  for (unsigned Idx = 0, E = NumParts; Idx != E && Offset < TotalSize;
2244  Offset += PartSize, ++Idx) {
2245  unsigned ByteSize = PartSize / 8;
2246  unsigned ByteOffset = Offset / 8;
2247  Register NewAddrReg;
2248 
2249  MIRBuilder.materializeGEP(NewAddrReg, AddrReg, OffsetTy, ByteOffset);
2250 
2251  MachineMemOperand *NewMMO =
2252  MF.getMachineMemOperand(MMO, ByteOffset, ByteSize);
2253 
2254  if (IsLoad) {
2255  Register Dst = MRI.createGenericVirtualRegister(PartTy);
2256  ValRegs.push_back(Dst);
2257  MIRBuilder.buildLoad(Dst, NewAddrReg, *NewMMO);
2258  } else {
2259  MIRBuilder.buildStore(ValRegs[Idx], NewAddrReg, *NewMMO);
2260  }
2261  }
2262 
2263  return Offset;
2264  };
2265 
2266  unsigned HandledOffset = splitTypePieces(NarrowTy, NarrowRegs, 0);
2267 
2268  // Handle the rest of the register if this isn't an even type breakdown.
2269  if (LeftoverTy.isValid())
2270  splitTypePieces(LeftoverTy, NarrowLeftoverRegs, HandledOffset);
2271 
2272  if (IsLoad) {
2273  insertParts(ValReg, ValTy, NarrowTy, NarrowRegs,
2274  LeftoverTy, NarrowLeftoverRegs);
2275  }
2276 
2277  MI.eraseFromParent();
2278  return Legalized;
2279 }
2280 
2283  LLT NarrowTy) {
2284  using namespace TargetOpcode;
2285 
2286  MIRBuilder.setInstr(MI);
2287  switch (MI.getOpcode()) {
2288  case G_IMPLICIT_DEF:
2289  return fewerElementsVectorImplicitDef(MI, TypeIdx, NarrowTy);
2290  case G_AND:
2291  case G_OR:
2292  case G_XOR:
2293  case G_ADD:
2294  case G_SUB:
2295  case G_MUL:
2296  case G_SMULH:
2297  case G_UMULH:
2298  case G_FADD:
2299  case G_FMUL:
2300  case G_FSUB:
2301  case G_FNEG:
2302  case G_FABS:
2303  case G_FCANONICALIZE:
2304  case G_FDIV:
2305  case G_FREM:
2306  case G_FMA:
2307  case G_FPOW:
2308  case G_FEXP:
2309  case G_FEXP2:
2310  case G_FLOG:
2311  case G_FLOG2:
2312  case G_FLOG10:
2313  case G_FNEARBYINT:
2314  case G_FCEIL:
2315  case G_FFLOOR:
2316  case G_FRINT:
2317  case G_INTRINSIC_ROUND:
2318  case G_INTRINSIC_TRUNC:
2319  case G_FCOS:
2320  case G_FSIN:
2321  case G_FSQRT:
2322  case G_BSWAP:
2323  case G_SDIV:
2324  case G_SMIN:
2325  case G_SMAX:
2326  case G_UMIN:
2327  case G_UMAX:
2328  case G_FMINNUM:
2329  case G_FMAXNUM:
2330  case G_FMINNUM_IEEE:
2331  case G_FMAXNUM_IEEE:
2332  case G_FMINIMUM:
2333  case G_FMAXIMUM:
2334  return fewerElementsVectorBasic(MI, TypeIdx, NarrowTy);
2335  case G_SHL:
2336  case G_LSHR:
2337  case G_ASHR:
2338  case G_CTLZ:
2339  case G_CTLZ_ZERO_UNDEF:
2340  case G_CTTZ:
2341  case G_CTTZ_ZERO_UNDEF:
2342  case G_CTPOP:
2343  case G_FCOPYSIGN:
2344  return fewerElementsVectorMultiEltType(MI, TypeIdx, NarrowTy);
2345  case G_ZEXT:
2346  case G_SEXT:
2347  case G_ANYEXT:
2348  case G_FPEXT:
2349  case G_FPTRUNC:
2350  case G_SITOFP:
2351  case G_UITOFP:
2352  case G_FPTOSI:
2353  case G_FPTOUI:
2354  case G_INTTOPTR:
2355  case G_PTRTOINT:
2356  case G_ADDRSPACE_CAST:
2357  return fewerElementsVectorCasts(MI, TypeIdx, NarrowTy);
2358  case G_ICMP:
2359  case G_FCMP:
2360  return fewerElementsVectorCmp(MI, TypeIdx, NarrowTy);
2361  case G_SELECT:
2362  return fewerElementsVectorSelect(MI, TypeIdx, NarrowTy);
2363  case G_PHI:
2364  return fewerElementsVectorPhi(MI, TypeIdx, NarrowTy);
2365  case G_LOAD:
2366  case G_STORE:
2367  return reduceLoadStoreWidth(MI, TypeIdx, NarrowTy);
2368  default:
2369  return UnableToLegalize;
2370  }
2371 }
2372 
2375  const LLT HalfTy, const LLT AmtTy) {
2376 
2377  Register InL = MRI.createGenericVirtualRegister(HalfTy);
2378  Register InH = MRI.createGenericVirtualRegister(HalfTy);
2379  MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1).getReg());
2380 
2381  if (Amt.isNullValue()) {
2382  MIRBuilder.buildMerge(MI.getOperand(0).getReg(), {InL, InH});
2383  MI.eraseFromParent();
2384  return Legalized;
2385  }
2386 
2387  LLT NVT = HalfTy;
2388  unsigned NVTBits = HalfTy.getSizeInBits();
2389  unsigned VTBits = 2 * NVTBits;
2390 
2391  SrcOp Lo(Register(0)), Hi(Register(0));
2392  if (MI.getOpcode() == TargetOpcode::G_SHL) {
2393  if (Amt.ugt(VTBits)) {
2394  Lo = Hi = MIRBuilder.buildConstant(NVT, 0);
2395  } else if (Amt.ugt(NVTBits)) {
2396  Lo = MIRBuilder.buildConstant(NVT, 0);
2397  Hi = MIRBuilder.buildShl(NVT, InL,
2398  MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
2399  } else if (Amt == NVTBits) {
2400  Lo = MIRBuilder.buildConstant(NVT, 0);
2401  Hi = InL;
2402  } else {
2403  Lo = MIRBuilder.buildShl(NVT, InL, MIRBuilder.buildConstant(AmtTy, Amt));
2404  auto OrLHS =
2405  MIRBuilder.buildShl(NVT, InH, MIRBuilder.buildConstant(AmtTy, Amt));
2406  auto OrRHS = MIRBuilder.buildLShr(
2407  NVT, InL, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
2408  Hi = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
2409  }
2410  } else if (MI.getOpcode() == TargetOpcode::G_LSHR) {
2411  if (Amt.ugt(VTBits)) {
2412  Lo = Hi = MIRBuilder.buildConstant(NVT, 0);
2413  } else if (Amt.ugt(NVTBits)) {
2414  Lo = MIRBuilder.buildLShr(NVT, InH,
2415  MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
2416  Hi = MIRBuilder.buildConstant(NVT, 0);
2417  } else if (Amt == NVTBits) {
2418  Lo = InH;
2419  Hi = MIRBuilder.buildConstant(NVT, 0);
2420  } else {
2421  auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt);
2422 
2423  auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst);
2424  auto OrRHS = MIRBuilder.buildShl(
2425  NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
2426 
2427  Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
2428  Hi = MIRBuilder.buildLShr(NVT, InH, ShiftAmtConst);
2429  }
2430  } else {
2431  if (Amt.ugt(VTBits)) {
2432  Hi = Lo = MIRBuilder.buildAShr(
2433  NVT, InH, MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
2434  } else if (Amt.ugt(NVTBits)) {
2435  Lo = MIRBuilder.buildAShr(NVT, InH,
2436  MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
2437  Hi = MIRBuilder.buildAShr(NVT, InH,
2438  MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
2439  } else if (Amt == NVTBits) {
2440  Lo = InH;
2441  Hi = MIRBuilder.buildAShr(NVT, InH,
2442  MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
2443  } else {
2444  auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt);
2445 
2446  auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst);
2447  auto OrRHS = MIRBuilder.buildShl(
2448  NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
2449 
2450  Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
2451  Hi = MIRBuilder.buildAShr(NVT, InH, ShiftAmtConst);
2452  }
2453  }
2454 
2455  MIRBuilder.buildMerge(MI.getOperand(0).getReg(), {Lo.getReg(), Hi.getReg()});
2456  MI.eraseFromParent();
2457 
2458  return Legalized;
2459 }
2460 
2461 // TODO: Optimize if constant shift amount.
2464  LLT RequestedTy) {
2465  if (TypeIdx == 1) {
2466  Observer.changingInstr(MI);
2467  narrowScalarSrc(MI, RequestedTy, 2);
2468  Observer.changedInstr(MI);
2469  return Legalized;
2470  }
2471 
2472  Register DstReg = MI.getOperand(0).getReg();
2473  LLT DstTy = MRI.getType(DstReg);
2474  if (DstTy.isVector())
2475  return UnableToLegalize;
2476 
2477  Register Amt = MI.getOperand(2).getReg();
2478  LLT ShiftAmtTy = MRI.getType(Amt);
2479  const unsigned DstEltSize = DstTy.getScalarSizeInBits();
2480  if (DstEltSize % 2 != 0)
2481  return UnableToLegalize;
2482 
2483  // Ignore the input type. We can only go to exactly half the size of the
2484  // input. If that isn't small enough, the resulting pieces will be further
2485  // legalized.
2486  const unsigned NewBitSize = DstEltSize / 2;
2487  const LLT HalfTy = LLT::scalar(NewBitSize);
2488  const LLT CondTy = LLT::scalar(1);
2489 
2490  if (const MachineInstr *KShiftAmt =
2491  getOpcodeDef(TargetOpcode::G_CONSTANT, Amt, MRI)) {
2493  MI, KShiftAmt->getOperand(1).getCImm()->getValue(), HalfTy, ShiftAmtTy);
2494  }
2495 
2496  // TODO: Expand with known bits.
2497 
2498  // Handle the fully general expansion by an unknown amount.
2499  auto NewBits = MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize);
2500 
2501  Register InL = MRI.createGenericVirtualRegister(HalfTy);
2502  Register InH = MRI.createGenericVirtualRegister(HalfTy);
2503  MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1).getReg());
2504 
2505  auto AmtExcess = MIRBuilder.buildSub(ShiftAmtTy, Amt, NewBits);
2506  auto AmtLack = MIRBuilder.buildSub(ShiftAmtTy, NewBits, Amt);
2507 
2508  auto Zero = MIRBuilder.buildConstant(ShiftAmtTy, 0);
2509  auto IsShort = MIRBuilder.buildICmp(ICmpInst::ICMP_ULT, CondTy, Amt, NewBits);
2510  auto IsZero = MIRBuilder.buildICmp(ICmpInst::ICMP_EQ, CondTy, Amt, Zero);
2511 
2512  Register ResultRegs[2];
2513  switch (MI.getOpcode()) {
2514  case TargetOpcode::G_SHL: {
2515  // Short: ShAmt < NewBitSize
2516  auto LoS = MIRBuilder.buildShl(HalfTy, InH, Amt);
2517 
2518  auto OrLHS = MIRBuilder.buildShl(HalfTy, InH, Amt);
2519  auto OrRHS = MIRBuilder.buildLShr(HalfTy, InL, AmtLack);
2520  auto HiS = MIRBuilder.buildOr(HalfTy, OrLHS, OrRHS);
2521 
2522  // Long: ShAmt >= NewBitSize
2523  auto LoL = MIRBuilder.buildConstant(HalfTy, 0); // Lo part is zero.
2524  auto HiL = MIRBuilder.buildShl(HalfTy, InL, AmtExcess); // Hi from Lo part.
2525 
2526  auto Lo = MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL);
2527  auto Hi = MIRBuilder.buildSelect(
2528  HalfTy, IsZero, InH, MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL));
2529 
2530  ResultRegs[0] = Lo.getReg(0);
2531  ResultRegs[1] = Hi.getReg(0);
2532  break;
2533  }
2534  case TargetOpcode::G_LSHR: {
2535  // Short: ShAmt < NewBitSize
2536  auto HiS = MIRBuilder.buildLShr(HalfTy, InH, Amt);
2537 
2538  auto OrLHS = MIRBuilder.buildLShr(HalfTy, InL, Amt);
2539  auto OrRHS = MIRBuilder.buildShl(HalfTy, InH, AmtLack);
2540  auto LoS = MIRBuilder.buildOr(HalfTy, OrLHS, OrRHS);
2541 
2542  // Long: ShAmt >= NewBitSize
2543  auto HiL = MIRBuilder.buildConstant(HalfTy, 0); // Hi part is zero.
2544  auto LoL = MIRBuilder.buildLShr(HalfTy, InH, AmtExcess); // Lo from Hi part.
2545 
2546  auto Lo = MIRBuilder.buildSelect(
2547  HalfTy, IsZero, InL, MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL));
2548  auto Hi = MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL);
2549 
2550  ResultRegs[0] = Lo.getReg(0);
2551  ResultRegs[1] = Hi.getReg(0);
2552  break;
2553  }
2554  case TargetOpcode::G_ASHR: {
2555  // Short: ShAmt < NewBitSize
2556  auto HiS = MIRBuilder.buildAShr(HalfTy, InH, Amt);
2557 
2558  auto OrLHS = MIRBuilder.buildLShr(HalfTy, InL, Amt);
2559  auto OrRHS = MIRBuilder.buildLShr(HalfTy, InH, AmtLack);
2560  auto LoS = MIRBuilder.buildOr(HalfTy, OrLHS, OrRHS);
2561 
2562  // Long: ShAmt >= NewBitSize
2563 
2564  // Sign of Hi part.
2565  auto HiL = MIRBuilder.buildAShr(
2566  HalfTy, InH, MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize - 1));
2567 
2568  auto LoL = MIRBuilder.buildAShr(HalfTy, InH, AmtExcess); // Lo from Hi part.
2569 
2570  auto Lo = MIRBuilder.buildSelect(
2571  HalfTy, IsZero, InL, MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL));
2572 
2573  auto Hi = MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL);
2574 
2575  ResultRegs[0] = Lo.getReg(0);
2576  ResultRegs[1] = Hi.getReg(0);
2577  break;
2578  }
2579  default:
2580  llvm_unreachable("not a shift");
2581  }
2582 
2583  MIRBuilder.buildMerge(DstReg, ResultRegs);
2584  MI.eraseFromParent();
2585  return Legalized;
2586 }
2587 
2590  LLT MoreTy) {
2591  assert(TypeIdx == 0 && "Expecting only Idx 0");
2592 
2593  Observer.changingInstr(MI);
2594  for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
2595  MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
2596  MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
2597  moreElementsVectorSrc(MI, MoreTy, I);
2598  }
2599 
2600  MachineBasicBlock &MBB = *MI.getParent();
2601  MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI());
2602  moreElementsVectorDst(MI, MoreTy, 0);
2603  Observer.changedInstr(MI);
2604  return Legalized;
2605 }
2606 
2609  LLT MoreTy) {
2610  MIRBuilder.setInstr(MI);
2611  unsigned Opc = MI.getOpcode();
2612  switch (Opc) {
2613  case TargetOpcode::G_IMPLICIT_DEF: {
2614  Observer.changingInstr(MI);
2615  moreElementsVectorDst(MI, MoreTy, 0);
2616  Observer.changedInstr(MI);
2617  return Legalized;
2618  }
2619  case TargetOpcode::G_AND:
2620  case TargetOpcode::G_OR:
2621  case TargetOpcode::G_XOR:
2622  case TargetOpcode::G_SMIN:
2623  case TargetOpcode::G_SMAX:
2624  case TargetOpcode::G_UMIN:
2625  case TargetOpcode::G_UMAX: {
2626  Observer.changingInstr(MI);
2627  moreElementsVectorSrc(MI, MoreTy, 1);
2628  moreElementsVectorSrc(MI, MoreTy, 2);
2629  moreElementsVectorDst(MI, MoreTy, 0);
2630  Observer.changedInstr(MI);
2631  return Legalized;
2632  }
2633  case TargetOpcode::G_EXTRACT:
2634  if (TypeIdx != 1)
2635  return UnableToLegalize;
2636  Observer.changingInstr(MI);
2637  moreElementsVectorSrc(MI, MoreTy, 1);
2638  Observer.changedInstr(MI);
2639  return Legalized;
2640  case TargetOpcode::G_INSERT:
2641  if (TypeIdx != 0)
2642  return UnableToLegalize;
2643  Observer.changingInstr(MI);
2644  moreElementsVectorSrc(MI, MoreTy, 1);
2645  moreElementsVectorDst(MI, MoreTy, 0);
2646  Observer.changedInstr(MI);
2647  return Legalized;
2648  case TargetOpcode::G_SELECT:
2649  if (TypeIdx != 0)
2650  return UnableToLegalize;
2651  if (MRI.getType(MI.getOperand(1).getReg()).isVector())
2652  return UnableToLegalize;
2653 
2654  Observer.changingInstr(MI);
2655  moreElementsVectorSrc(MI, MoreTy, 2);
2656  moreElementsVectorSrc(MI, MoreTy, 3);
2657  moreElementsVectorDst(MI, MoreTy, 0);
2658  Observer.changedInstr(MI);
2659  return Legalized;
2660  case TargetOpcode::G_PHI:
2661  return moreElementsVectorPhi(MI, TypeIdx, MoreTy);
2662  default:
2663  return UnableToLegalize;
2664  }
2665 }
2666 
2667 void LegalizerHelper::multiplyRegisters(SmallVectorImpl<Register> &DstRegs,
2668  ArrayRef<Register> Src1Regs,
2669  ArrayRef<Register> Src2Regs,
2670  LLT NarrowTy) {
2672  unsigned SrcParts = Src1Regs.size();
2673  unsigned DstParts = DstRegs.size();
2674 
2675  unsigned DstIdx = 0; // Low bits of the result.
2676  Register FactorSum =
2677  B.buildMul(NarrowTy, Src1Regs[DstIdx], Src2Regs[DstIdx]).getReg(0);
2678  DstRegs[DstIdx] = FactorSum;
2679 
2680  unsigned CarrySumPrevDstIdx;
2681  SmallVector<Register, 4> Factors;
2682 
2683  for (DstIdx = 1; DstIdx < DstParts; DstIdx++) {
2684  // Collect low parts of muls for DstIdx.
2685  for (unsigned i = DstIdx + 1 < SrcParts ? 0 : DstIdx - SrcParts + 1;
2686  i <= std::min(DstIdx, SrcParts - 1); ++i) {
2687  MachineInstrBuilder Mul =
2688  B.buildMul(NarrowTy, Src1Regs[DstIdx - i], Src2Regs[i]);
2689  Factors.push_back(Mul.getReg(0));
2690  }
2691  // Collect high parts of muls from previous DstIdx.
2692  for (unsigned i = DstIdx < SrcParts ? 0 : DstIdx - SrcParts;
2693  i <= std::min(DstIdx - 1, SrcParts - 1); ++i) {
2694  MachineInstrBuilder Umulh =
2695  B.buildUMulH(NarrowTy, Src1Regs[DstIdx - 1 - i], Src2Regs[i]);
2696  Factors.push_back(Umulh.getReg(0));
2697  }
2698  // Add CarrySum from additons calculated for previous DstIdx.
2699  if (DstIdx != 1) {
2700  Factors.push_back(CarrySumPrevDstIdx);
2701  }
2702 
2703  Register CarrySum;
2704  // Add all factors and accumulate all carries into CarrySum.
2705  if (DstIdx != DstParts - 1) {
2706  MachineInstrBuilder Uaddo =
2707  B.buildUAddo(NarrowTy, LLT::scalar(1), Factors[0], Factors[1]);
2708  FactorSum = Uaddo.getReg(0);
2709  CarrySum = B.buildZExt(NarrowTy, Uaddo.getReg(1)).getReg(0);
2710  for (unsigned i = 2; i < Factors.size(); ++i) {
2711  MachineInstrBuilder Uaddo =
2712  B.buildUAddo(NarrowTy, LLT::scalar(1), FactorSum, Factors[i]);
2713  FactorSum = Uaddo.getReg(0);
2714  MachineInstrBuilder Carry = B.buildZExt(NarrowTy, Uaddo.getReg(1));
2715  CarrySum = B.buildAdd(NarrowTy, CarrySum, Carry).getReg(0);
2716  }
2717  } else {
2718  // Since value for the next index is not calculated, neither is CarrySum.
2719  FactorSum = B.buildAdd(NarrowTy, Factors[0], Factors[1]).getReg(0);
2720  for (unsigned i = 2; i < Factors.size(); ++i)
2721  FactorSum = B.buildAdd(NarrowTy, FactorSum, Factors[i]).getReg(0);
2722  }
2723 
2724  CarrySumPrevDstIdx = CarrySum;
2725  DstRegs[DstIdx] = FactorSum;
2726  Factors.clear();
2727  }
2728 }
2729 
2732  Register DstReg = MI.getOperand(0).getReg();
2733  Register Src1 = MI.getOperand(1).getReg();
2734  Register Src2 = MI.getOperand(2).getReg();
2735 
2736  LLT Ty = MRI.getType(DstReg);
2737  if (Ty.isVector())
2738  return UnableToLegalize;
2739 
2740  unsigned SrcSize = MRI.getType(Src1).getSizeInBits();
2741  unsigned DstSize = Ty.getSizeInBits();
2742  unsigned NarrowSize = NarrowTy.getSizeInBits();
2743  if (DstSize % NarrowSize != 0 || SrcSize % NarrowSize != 0)
2744  return UnableToLegalize;
2745 
2746  unsigned NumDstParts = DstSize / NarrowSize;
2747  unsigned NumSrcParts = SrcSize / NarrowSize;
2748  bool IsMulHigh = MI.getOpcode() == TargetOpcode::G_UMULH;
2749  unsigned DstTmpParts = NumDstParts * (IsMulHigh ? 2 : 1);
2750 
2751  SmallVector<Register, 2> Src1Parts, Src2Parts, DstTmpRegs;
2752  extractParts(Src1, NarrowTy, NumSrcParts, Src1Parts);
2753  extractParts(Src2, NarrowTy, NumSrcParts, Src2Parts);
2754  DstTmpRegs.resize(DstTmpParts);
2755  multiplyRegisters(DstTmpRegs, Src1Parts, Src2Parts, NarrowTy);
2756 
2757  // Take only high half of registers if this is high mul.
2758  ArrayRef<Register> DstRegs(
2759  IsMulHigh ? &DstTmpRegs[DstTmpParts / 2] : &DstTmpRegs[0], NumDstParts);
2760  MIRBuilder.buildMerge(DstReg, DstRegs);
2761  MI.eraseFromParent();
2762  return Legalized;
2763 }
2764 
2767  LLT NarrowTy) {
2768  if (TypeIdx != 1)
2769  return UnableToLegalize;
2770 
2771  uint64_t NarrowSize = NarrowTy.getSizeInBits();
2772 
2773  int64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
2774  // FIXME: add support for when SizeOp1 isn't an exact multiple of
2775  // NarrowSize.
2776  if (SizeOp1 % NarrowSize != 0)
2777  return UnableToLegalize;
2778  int NumParts = SizeOp1 / NarrowSize;
2779 
2780  SmallVector<Register, 2> SrcRegs, DstRegs;
2781  SmallVector<uint64_t, 2> Indexes;
2782  extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
2783 
2784  Register OpReg = MI.getOperand(0).getReg();
2785  uint64_t OpStart = MI.getOperand(2).getImm();
2786  uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
2787  for (int i = 0; i < NumParts; ++i) {
2788  unsigned SrcStart = i * NarrowSize;
2789 
2790  if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) {
2791  // No part of the extract uses this subregister, ignore it.
2792  continue;
2793  } else if (SrcStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
2794  // The entire subregister is extracted, forward the value.
2795  DstRegs.push_back(SrcRegs[i]);
2796  continue;
2797  }
2798 
2799  // OpSegStart is where this destination segment would start in OpReg if it
2800  // extended infinitely in both directions.
2801  int64_t ExtractOffset;
2802  uint64_t SegSize;
2803  if (OpStart < SrcStart) {
2804  ExtractOffset = 0;
2805  SegSize = std::min(NarrowSize, OpStart + OpSize - SrcStart);
2806  } else {
2807  ExtractOffset = OpStart - SrcStart;
2808  SegSize = std::min(SrcStart + NarrowSize - OpStart, OpSize);
2809  }
2810 
2811  Register SegReg = SrcRegs[i];
2812  if (ExtractOffset != 0 || SegSize != NarrowSize) {
2813  // A genuine extract is needed.
2814  SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
2815  MIRBuilder.buildExtract(SegReg, SrcRegs[i], ExtractOffset);
2816  }
2817 
2818  DstRegs.push_back(SegReg);
2819  }
2820 
2821  Register DstReg = MI.getOperand(0).getReg();
2822  if(MRI.getType(DstReg).isVector())
2823  MIRBuilder.buildBuildVector(DstReg, DstRegs);
2824  else
2825  MIRBuilder.buildMerge(DstReg, DstRegs);
2826  MI.eraseFromParent();
2827  return Legalized;
2828 }
2829 
2832  LLT NarrowTy) {
2833  // FIXME: Don't know how to handle secondary types yet.
2834  if (TypeIdx != 0)
2835  return UnableToLegalize;
2836 
2837  uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
2838  uint64_t NarrowSize = NarrowTy.getSizeInBits();
2839 
2840  // FIXME: add support for when SizeOp0 isn't an exact multiple of
2841  // NarrowSize.
2842  if (SizeOp0 % NarrowSize != 0)
2843  return UnableToLegalize;
2844 
2845  int NumParts = SizeOp0 / NarrowSize;
2846 
2847  SmallVector<Register, 2> SrcRegs, DstRegs;
2848  SmallVector<uint64_t, 2> Indexes;
2849  extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
2850 
2851  Register OpReg = MI.getOperand(2).getReg();
2852  uint64_t OpStart = MI.getOperand(3).getImm();
2853  uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
2854  for (int i = 0; i < NumParts; ++i) {
2855  unsigned DstStart = i * NarrowSize;
2856 
2857  if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) {
2858  // No part of the insert affects this subregister, forward the original.
2859  DstRegs.push_back(SrcRegs[i]);
2860  continue;
2861  } else if (DstStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
2862  // The entire subregister is defined by this insert, forward the new
2863  // value.
2864  DstRegs.push_back(OpReg);
2865  continue;
2866  }
2867 
2868  // OpSegStart is where this destination segment would start in OpReg if it
2869  // extended infinitely in both directions.
2870  int64_t ExtractOffset, InsertOffset;
2871  uint64_t SegSize;
2872  if (OpStart < DstStart) {
2873  InsertOffset = 0;
2874  ExtractOffset = DstStart - OpStart;
2875  SegSize = std::min(NarrowSize, OpStart + OpSize - DstStart);
2876  } else {
2877  InsertOffset = OpStart - DstStart;
2878  ExtractOffset = 0;
2879  SegSize =
2880  std::min(NarrowSize - InsertOffset, OpStart + OpSize - DstStart);
2881  }
2882 
2883  Register SegReg = OpReg;
2884  if (ExtractOffset != 0 || SegSize != OpSize) {
2885  // A genuine extract is needed.
2886  SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
2887  MIRBuilder.buildExtract(SegReg, OpReg, ExtractOffset);
2888  }
2889 
2890  Register DstReg = MRI.createGenericVirtualRegister(NarrowTy);
2891  MIRBuilder.buildInsert(DstReg, SrcRegs[i], SegReg, InsertOffset);
2892  DstRegs.push_back(DstReg);
2893  }
2894 
2895  assert(DstRegs.size() == (unsigned)NumParts && "not all parts covered");
2896  Register DstReg = MI.getOperand(0).getReg();
2897  if(MRI.getType(DstReg).isVector())
2898  MIRBuilder.buildBuildVector(DstReg, DstRegs);
2899  else
2900  MIRBuilder.buildMerge(DstReg, DstRegs);
2901  MI.eraseFromParent();
2902  return Legalized;
2903 }
2904 
2907  LLT NarrowTy) {
2908  Register DstReg = MI.getOperand(0).getReg();
2909  LLT DstTy = MRI.getType(DstReg);
2910 
2911  assert(MI.getNumOperands() == 3 && TypeIdx == 0);
2912 
2913  SmallVector<Register, 4> DstRegs, DstLeftoverRegs;
2914  SmallVector<Register, 4> Src0Regs, Src0LeftoverRegs;
2915  SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs;
2916  LLT LeftoverTy;
2917  if (!extractParts(MI.getOperand(1).getReg(), DstTy, NarrowTy, LeftoverTy,
2918  Src0Regs, Src0LeftoverRegs))
2919  return UnableToLegalize;
2920 
2921  LLT Unused;
2922  if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, Unused,
2923  Src1Regs, Src1LeftoverRegs))
2924  llvm_unreachable("inconsistent extractParts result");
2925 
2926  for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) {
2927  auto Inst = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy},
2928  {Src0Regs[I], Src1Regs[I]});
2929  DstRegs.push_back(Inst->getOperand(0).getReg());
2930  }
2931 
2932  for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) {
2933  auto Inst = MIRBuilder.buildInstr(
2934  MI.getOpcode(),
2935  {LeftoverTy}, {Src0LeftoverRegs[I], Src1LeftoverRegs[I]});
2936  DstLeftoverRegs.push_back(Inst->getOperand(0).getReg());
2937  }
2938 
2939  insertParts(DstReg, DstTy, NarrowTy, DstRegs,
2940  LeftoverTy, DstLeftoverRegs);
2941 
2942  MI.eraseFromParent();
2943  return Legalized;
2944 }
2945 
2948  LLT NarrowTy) {
2949  if (TypeIdx != 0)
2950  return UnableToLegalize;
2951 
2952  Register CondReg = MI.getOperand(1).getReg();
2953  LLT CondTy = MRI.getType(CondReg);
2954  if (CondTy.isVector()) // TODO: Handle vselect
2955  return UnableToLegalize;
2956 
2957  Register DstReg = MI.getOperand(0).getReg();
2958  LLT DstTy = MRI.getType(DstReg);
2959 
2960  SmallVector<Register, 4> DstRegs, DstLeftoverRegs;
2961  SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs;
2962  SmallVector<Register, 4> Src2Regs, Src2LeftoverRegs;
2963  LLT LeftoverTy;
2964  if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, LeftoverTy,
2965  Src1Regs, Src1LeftoverRegs))
2966  return UnableToLegalize;
2967 
2968  LLT Unused;
2969  if (!extractParts(MI.getOperand(3).getReg(), DstTy, NarrowTy, Unused,
2970  Src2Regs, Src2LeftoverRegs))
2971  llvm_unreachable("inconsistent extractParts result");
2972 
2973  for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) {
2974  auto Select = MIRBuilder.buildSelect(NarrowTy,
2975  CondReg, Src1Regs[I], Src2Regs[I]);
2976  DstRegs.push_back(Select->getOperand(0).getReg());
2977  }
2978 
2979  for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) {
2980  auto Select = MIRBuilder.buildSelect(
2981  LeftoverTy, CondReg, Src1LeftoverRegs[I], Src2LeftoverRegs[I]);
2982  DstLeftoverRegs.push_back(Select->getOperand(0).getReg());
2983  }
2984 
2985  insertParts(DstReg, DstTy, NarrowTy, DstRegs,
2986  LeftoverTy, DstLeftoverRegs);
2987 
2988  MI.eraseFromParent();
2989  return Legalized;
2990 }
2991 
2994  unsigned Opc = MI.getOpcode();
2995  auto &TII = *MI.getMF()->getSubtarget().getInstrInfo();
2996  auto isSupported = [this](const LegalityQuery &Q) {
2997  auto QAction = LI.getAction(Q).Action;
2998  return QAction == Legal || QAction == Libcall || QAction == Custom;
2999  };
3000  switch (Opc) {
3001  default:
3002  return UnableToLegalize;
3003  case TargetOpcode::G_CTLZ_ZERO_UNDEF: {
3004  // This trivially expands to CTLZ.
3005  Observer.changingInstr(MI);
3006  MI.setDesc(TII.get(TargetOpcode::G_CTLZ));
3007  Observer.changedInstr(MI);
3008  return Legalized;
3009  }
3010  case TargetOpcode::G_CTLZ: {
3011  Register SrcReg = MI.getOperand(1).getReg();
3012  unsigned Len = Ty.getSizeInBits();
3013  if (isSupported({TargetOpcode::G_CTLZ_ZERO_UNDEF, {Ty, Ty}})) {
3014  // If CTLZ_ZERO_UNDEF is supported, emit that and a select for zero.
3015  auto MIBCtlzZU = MIRBuilder.buildInstr(TargetOpcode::G_CTLZ_ZERO_UNDEF,
3016  {Ty}, {SrcReg});
3017  auto MIBZero = MIRBuilder.buildConstant(Ty, 0);
3018  auto MIBLen = MIRBuilder.buildConstant(Ty, Len);
3019  auto MIBICmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1),
3020  SrcReg, MIBZero);
3021  MIRBuilder.buildSelect(MI.getOperand(0).getReg(), MIBICmp, MIBLen,
3022  MIBCtlzZU);
3023  MI.eraseFromParent();
3024  return Legalized;
3025  }
3026  // for now, we do this:
3027  // NewLen = NextPowerOf2(Len);
3028  // x = x | (x >> 1);
3029  // x = x | (x >> 2);
3030  // ...
3031  // x = x | (x >>16);
3032  // x = x | (x >>32); // for 64-bit input
3033  // Upto NewLen/2
3034  // return Len - popcount(x);
3035  //
3036  // Ref: "Hacker's Delight" by Henry Warren
3037  Register Op = SrcReg;
3038  unsigned NewLen = PowerOf2Ceil(Len);
3039  for (unsigned i = 0; (1U << i) <= (NewLen / 2); ++i) {
3040  auto MIBShiftAmt = MIRBuilder.buildConstant(Ty, 1ULL << i);
3041  auto MIBOp = MIRBuilder.buildInstr(
3042  TargetOpcode::G_OR, {Ty},
3043  {Op, MIRBuilder.buildInstr(TargetOpcode::G_LSHR, {Ty},
3044  {Op, MIBShiftAmt})});
3045  Op = MIBOp->getOperand(0).getReg();
3046  }
3047  auto MIBPop = MIRBuilder.buildInstr(TargetOpcode::G_CTPOP, {Ty}, {Op});
3048  MIRBuilder.buildInstr(TargetOpcode::G_SUB, {MI.getOperand(0).getReg()},
3049  {MIRBuilder.buildConstant(Ty, Len), MIBPop});
3050  MI.eraseFromParent();
3051  return Legalized;
3052  }
3053  case TargetOpcode::G_CTTZ_ZERO_UNDEF: {
3054  // This trivially expands to CTTZ.
3055  Observer.changingInstr(MI);
3056  MI.setDesc(TII.get(TargetOpcode::G_CTTZ));
3057  Observer.changedInstr(MI);
3058  return Legalized;
3059  }
3060  case TargetOpcode::G_CTTZ: {
3061  Register SrcReg = MI.getOperand(1).getReg();
3062  unsigned Len = Ty.getSizeInBits();
3063  if (isSupported({TargetOpcode::G_CTTZ_ZERO_UNDEF, {Ty, Ty}})) {
3064  // If CTTZ_ZERO_UNDEF is legal or custom, emit that and a select with
3065  // zero.
3066  auto MIBCttzZU = MIRBuilder.buildInstr(TargetOpcode::G_CTTZ_ZERO_UNDEF,
3067  {Ty}, {SrcReg});
3068  auto MIBZero = MIRBuilder.buildConstant(Ty, 0);
3069  auto MIBLen = MIRBuilder.buildConstant(Ty, Len);
3070  auto MIBICmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1),
3071  SrcReg, MIBZero);
3072  MIRBuilder.buildSelect(MI.getOperand(0).getReg(), MIBICmp, MIBLen,
3073  MIBCttzZU);
3074  MI.eraseFromParent();
3075  return Legalized;
3076  }
3077  // for now, we use: { return popcount(~x & (x - 1)); }
3078  // unless the target has ctlz but not ctpop, in which case we use:
3079  // { return 32 - nlz(~x & (x-1)); }
3080  // Ref: "Hacker's Delight" by Henry Warren
3081  auto MIBCstNeg1 = MIRBuilder.buildConstant(Ty, -1);
3082  auto MIBNot =
3083  MIRBuilder.buildInstr(TargetOpcode::G_XOR, {Ty}, {SrcReg, MIBCstNeg1});
3084  auto MIBTmp = MIRBuilder.buildInstr(
3085  TargetOpcode::G_AND, {Ty},
3086  {MIBNot, MIRBuilder.buildInstr(TargetOpcode::G_ADD, {Ty},
3087  {SrcReg, MIBCstNeg1})});
3088  if (!isSupported({TargetOpcode::G_CTPOP, {Ty, Ty}}) &&
3089  isSupported({TargetOpcode::G_CTLZ, {Ty, Ty}})) {
3090  auto MIBCstLen = MIRBuilder.buildConstant(Ty, Len);
3092  TargetOpcode::G_SUB, {MI.getOperand(0).getReg()},
3093  {MIBCstLen,
3094  MIRBuilder.buildInstr(TargetOpcode::G_CTLZ, {Ty}, {MIBTmp})});
3095  MI.eraseFromParent();
3096  return Legalized;
3097  }
3098  MI.setDesc(TII.get(TargetOpcode::G_CTPOP));
3099  MI.getOperand(1).setReg(MIBTmp->getOperand(0).getReg());
3100  return Legalized;
3101  }
3102  }
3103 }
3104 
3105 // Expand s32 = G_UITOFP s64 using bit operations to an IEEE float
3106 // representation.
3109  Register Dst = MI.getOperand(0).getReg();
3110  Register Src = MI.getOperand(1).getReg();
3111  const LLT S64 = LLT::scalar(64);
3112  const LLT S32 = LLT::scalar(32);
3113  const LLT S1 = LLT::scalar(1);
3114 
3115  assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S32);
3116 
3117  // unsigned cul2f(ulong u) {
3118  // uint lz = clz(u);
3119  // uint e = (u != 0) ? 127U + 63U - lz : 0;
3120  // u = (u << lz) & 0x7fffffffffffffffUL;
3121  // ulong t = u & 0xffffffffffUL;
3122  // uint v = (e << 23) | (uint)(u >> 40);
3123  // uint r = t > 0x8000000000UL ? 1U : (t == 0x8000000000UL ? v & 1U : 0U);
3124  // return as_float(v + r);
3125  // }
3126 
3127  auto Zero32 = MIRBuilder.buildConstant(S32, 0);
3128  auto Zero64 = MIRBuilder.buildConstant(S64, 0);
3129 
3130  auto LZ = MIRBuilder.buildCTLZ_ZERO_UNDEF(S32, Src);
3131 
3132  auto K = MIRBuilder.buildConstant(S32, 127U + 63U);
3133  auto Sub = MIRBuilder.buildSub(S32, K, LZ);
3134 
3135  auto NotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, Src, Zero64);
3136  auto E = MIRBuilder.buildSelect(S32, NotZero, Sub, Zero32);
3137 
3138  auto Mask0 = MIRBuilder.buildConstant(S64, (-1ULL) >> 1);
3139  auto ShlLZ = MIRBuilder.buildShl(S64, Src, LZ);
3140 
3141  auto U = MIRBuilder.buildAnd(S64, ShlLZ, Mask0);
3142 
3143  auto Mask1 = MIRBuilder.buildConstant(S64, 0xffffffffffULL);
3144  auto T = MIRBuilder.buildAnd(S64, U, Mask1);
3145 
3146  auto UShl = MIRBuilder.buildLShr(S64, U, MIRBuilder.buildConstant(S64, 40));
3147  auto ShlE = MIRBuilder.buildShl(S32, E, MIRBuilder.buildConstant(S32, 23));
3148  auto V = MIRBuilder.buildOr(S32, ShlE, MIRBuilder.buildTrunc(S32, UShl));
3149 
3150  auto C = MIRBuilder.buildConstant(S64, 0x8000000000ULL);
3151  auto RCmp = MIRBuilder.buildICmp(CmpInst::ICMP_UGT, S1, T, C);
3152  auto TCmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1, T, C);
3153  auto One = MIRBuilder.buildConstant(S32, 1);
3154 
3155  auto VTrunc1 = MIRBuilder.buildAnd(S32, V, One);
3156  auto Select0 = MIRBuilder.buildSelect(S32, TCmp, VTrunc1, Zero32);
3157  auto R = MIRBuilder.buildSelect(S32, RCmp, One, Select0);
3158  MIRBuilder.buildAdd(Dst, V, R);
3159 
3160  return Legalized;
3161 }
3162 
3164 LegalizerHelper::lowerUITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
3165  Register Dst = MI.getOperand(0).getReg();
3166  Register Src = MI.getOperand(1).getReg();
3167  LLT DstTy = MRI.getType(Dst);
3168  LLT SrcTy = MRI.getType(Src);
3169 
3170  if (SrcTy != LLT::scalar(64))
3171  return UnableToLegalize;
3172 
3173  if (DstTy == LLT::scalar(32)) {
3174  // TODO: SelectionDAG has several alternative expansions to port which may
3175  // be more reasonble depending on the available instructions. If a target
3176  // has sitofp, does not have CTLZ, or can efficiently use f64 as an
3177  // intermediate type, this is probably worse.
3178  return lowerU64ToF32BitOps(MI);
3179  }
3180 
3181  return UnableToLegalize;
3182 }
3183 
3185 LegalizerHelper::lowerSITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
3186  Register Dst = MI.getOperand(0).getReg();
3187  Register Src = MI.getOperand(1).getReg();
3188  LLT DstTy = MRI.getType(Dst);
3189  LLT SrcTy = MRI.getType(Src);
3190 
3191  const LLT S64 = LLT::scalar(64);
3192  const LLT S32 = LLT::scalar(32);
3193  const LLT S1 = LLT::scalar(1);
3194 
3195  if (SrcTy != S64)
3196  return UnableToLegalize;
3197 
3198  if (DstTy == S32) {
3199  // signed cl2f(long l) {
3200  // long s = l >> 63;
3201  // float r = cul2f((l + s) ^ s);
3202  // return s ? -r : r;
3203  // }
3204  Register L = Src;
3205  auto SignBit = MIRBuilder.buildConstant(S64, 63);
3206  auto S = MIRBuilder.buildAShr(S64, L, SignBit);
3207 
3208  auto LPlusS = MIRBuilder.buildAdd(S64, L, S);
3209  auto Xor = MIRBuilder.buildXor(S64, LPlusS, S);
3210  auto R = MIRBuilder.buildUITOFP(S32, Xor);
3211 
3212  auto RNeg = MIRBuilder.buildFNeg(S32, R);
3213  auto SignNotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, S,
3214  MIRBuilder.buildConstant(S64, 0));
3215  MIRBuilder.buildSelect(Dst, SignNotZero, RNeg, R);
3216  return Legalized;
3217  }
3218 
3219  return UnableToLegalize;
3220 }
3221 
3222 static CmpInst::Predicate minMaxToCompare(unsigned Opc) {
3223  switch (Opc) {
3224  case TargetOpcode::G_SMIN:
3225  return CmpInst::ICMP_SLT;
3226  case TargetOpcode::G_SMAX:
3227  return CmpInst::ICMP_SGT;
3228  case TargetOpcode::G_UMIN:
3229  return CmpInst::ICMP_ULT;
3230  case TargetOpcode::G_UMAX:
3231  return CmpInst::ICMP_UGT;
3232  default:
3233  llvm_unreachable("not in integer min/max");
3234  }
3235 }
3236 
3238 LegalizerHelper::lowerMinMax(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
3239  Register Dst = MI.getOperand(0).getReg();
3240  Register Src0 = MI.getOperand(1).getReg();
3241  Register Src1 = MI.getOperand(2).getReg();
3242 
3243  const CmpInst::Predicate Pred = minMaxToCompare(MI.getOpcode());
3244  LLT CmpType = MRI.getType(Dst).changeElementSize(1);
3245 
3246  auto Cmp = MIRBuilder.buildICmp(Pred, CmpType, Src0, Src1);
3247  MIRBuilder.buildSelect(Dst, Cmp, Src0, Src1);
3248 
3249  MI.eraseFromParent();
3250  return Legalized;
3251 }
3252 
3255  Register Dst = MI.getOperand(0).getReg();
3256  Register Src0 = MI.getOperand(1).getReg();
3257  Register Src1 = MI.getOperand(2).getReg();
3258 
3259  const LLT Src0Ty = MRI.getType(Src0);
3260  const LLT Src1Ty = MRI.getType(Src1);
3261 
3262  const int Src0Size = Src0Ty.getScalarSizeInBits();
3263  const int Src1Size = Src1Ty.getScalarSizeInBits();
3264 
3265  auto SignBitMask = MIRBuilder.buildConstant(
3266  Src0Ty, APInt::getSignMask(Src0Size));
3267 
3268  auto NotSignBitMask = MIRBuilder.buildConstant(
3269  Src0Ty, APInt::getLowBitsSet(Src0Size, Src0Size - 1));
3270 
3271  auto And0 = MIRBuilder.buildAnd(Src0Ty, Src0, NotSignBitMask);
3272  MachineInstr *Or;
3273 
3274  if (Src0Ty == Src1Ty) {
3275  auto And1 = MIRBuilder.buildAnd(Src1Ty, Src0, SignBitMask);
3276  Or = MIRBuilder.buildOr(Dst, And0, And1);
3277  } else if (Src0Size > Src1Size) {
3278  auto ShiftAmt = MIRBuilder.buildConstant(Src0Ty, Src0Size - Src1Size);
3279  auto Zext = MIRBuilder.buildZExt(Src0Ty, Src1);
3280  auto Shift = MIRBuilder.buildShl(Src0Ty, Zext, ShiftAmt);
3281  auto And1 = MIRBuilder.buildAnd(Src0Ty, Shift, SignBitMask);
3282  Or = MIRBuilder.buildOr(Dst, And0, And1);
3283  } else {
3284  auto ShiftAmt = MIRBuilder.buildConstant(Src1Ty, Src1Size - Src0Size);
3285  auto Shift = MIRBuilder.buildLShr(Src1Ty, Src1, ShiftAmt);
3286  auto Trunc = MIRBuilder.buildTrunc(Src0Ty, Shift);
3287  auto And1 = MIRBuilder.buildAnd(Src0Ty, Trunc, SignBitMask);
3288  Or = MIRBuilder.buildOr(Dst, And0, And1);
3289  }
3290 
3291  // Be careful about setting nsz/nnan/ninf on every instruction, since the
3292  // constants are a nan and -0.0, but the final result should preserve
3293  // everything.
3294  if (unsigned Flags = MI.getFlags())
3295  Or->setFlags(Flags);
3296 
3297  MI.eraseFromParent();
3298  return Legalized;
3299 }
3300 
3303  unsigned NewOp = MI.getOpcode() == TargetOpcode::G_FMINNUM ?
3304  TargetOpcode::G_FMINNUM_IEEE : TargetOpcode::G_FMAXNUM_IEEE;
3305 
3306  Register Dst = MI.getOperand(0).getReg();
3307  Register Src0 = MI.getOperand(1).getReg();
3308  Register Src1 = MI.getOperand(2).getReg();
3309  LLT Ty = MRI.getType(Dst);
3310 
3311  if (!MI.getFlag(MachineInstr::FmNoNans)) {
3312  // Insert canonicalizes if it's possible we need to quiet to get correct
3313  // sNaN behavior.
3314 
3315  // Note this must be done here, and not as an optimization combine in the
3316  // absence of a dedicate quiet-snan instruction as we're using an
3317  // omni-purpose G_FCANONICALIZE.
3318  if (!isKnownNeverSNaN(Src0, MRI))
3319  Src0 = MIRBuilder.buildFCanonicalize(Ty, Src0, MI.getFlags()).getReg(0);
3320 
3321  if (!isKnownNeverSNaN(Src1, MRI))
3322  Src1 = MIRBuilder.buildFCanonicalize(Ty, Src1, MI.getFlags()).getReg(0);
3323  }
3324 
3325  // If there are no nans, it's safe to simply replace this with the non-IEEE
3326  // version.
3327  MIRBuilder.buildInstr(NewOp, {Dst}, {Src0, Src1}, MI.getFlags());
3328  MI.eraseFromParent();
3329  return Legalized;
3330 }
static LegalizerHelper::LegalizeResult simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size, Type *OpType)
uint64_t CallInst * C
LegalizeResult fewerElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
const MachineInstrBuilder & add(const MachineOperand &MO) const
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:110
static Type * getDoubleTy(LLVMContext &C)
Definition: Type.cpp:164
uint64_t getZExtValue() const
Get zero extended value.
Definition: APInt.h:1562
APInt sext(unsigned width) const
Sign extend to a new width.
Definition: APInt.cpp:836
MachineInstrBuilder buildZExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
MachineInstrBuilder buildUnmerge(ArrayRef< LLT > Res, const SrcOp &Op)
Build and insert Res0, ...
static APInt getAllOnesValue(unsigned numBits)
Get the all-ones value.
Definition: APInt.h:561
MachineInstrBuilder buildXor(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_XOR Op0, Op1.
MachineBasicBlock * getMBB() const
LegalizeResult lowerUITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
This class represents lattice values for constants.
Definition: AllocatorList.h:23
MachineInstrBuilder buildInsert(Register Res, Register Src, Register Op, unsigned Index)
Register getReg(unsigned Idx) const
Get the register for the operand index.
amdgpu Simplify well known AMD library false FunctionCallee Value const Twine & Name
unsigned getScalarSizeInBits() const
LegalizeResult fewerElementsVectorMultiEltType(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
Legalize a instruction with a vector type where each operand may have a different element type...
The operation should be implemented in terms of a wider scalar base-type.
Definition: LegalizerInfo.h:57
unsigned getSizeInBits(Register Reg, const MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI) const
Get the size in bits of Reg.
void setFPImm(const ConstantFP *CFP)
LegalizeResult lowerMinMax(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
AtomicOrdering getFailureOrdering() const
For cmpxchg atomic operations, return the atomic ordering requirements when store does not occur...
The LegalityQuery object bundles together all the information that&#39;s needed to decide whether a given...
bool isScalar() const
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
LegalizeResult fewerElementsVectorCasts(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
unsigned Reg
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Get a value with low bits set.
Definition: APInt.h:647
The (vector) operation should be implemented by splitting it into sub-vectors where the operation is ...
Definition: LegalizerInfo.h:62
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit...
virtual const TargetLowering * getTargetLowering() const
unsigned less than
Definition: InstrTypes.h:757
LLT getScalarType() const
MachineInstrBuilder buildOr(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_OR Op0, Op1.
LLT getType(unsigned Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register...
APInt trunc(unsigned width) const
Truncate to new width.
Definition: APInt.cpp:813
bool isNonIntegralAddressSpace(unsigned AddrSpace) const
Definition: DataLayout.h:371
static uint32_t Concat[]
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:176
LegalizeResult lowerBitCount(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
MachineInstrBuilder buildUAddo(const DstOp &Res, const DstOp &CarryOut, const SrcOp &Op0, const SrcOp &Op1)
Build and insert Res, CarryOut = G_UADDO Op0, Op1.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
LegalityPredicate isVector(unsigned TypeIdx)
True iff the specified type index is a vector.
Optional< MachineInstrBuilder > materializeGEP(Register &Res, Register Op0, const LLT &ValueTy, uint64_t Value)
Materialize and insert Res = G_GEP Op0, (G_CONSTANT Value)
bool isKnownNeverSNaN(Register Val, const MachineRegisterInfo &MRI)
Returns true if Val can be assumed to never be a signaling NaN.
Definition: Utils.h:160
MachineInstrBuilder buildExtract(const DstOp &Res, const SrcOp &Src, uint64_t Index)
Build and insert `Res0, ...
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
The operation should be synthesized from multiple instructions acting on a narrower scalar base-type...
Definition: LegalizerInfo.h:52
bool isVector() const
void setMF(MachineFunction &MF)
LegalizeActionStep getAction(const LegalityQuery &Query) const
Determine what action should be taken to legalize the described instruction.
A description of a memory reference used in the backend.
bool isSigned() const
Definition: InstrTypes.h:902
LegalizeResult narrowScalarMul(MachineInstr &MI, LLT Ty)
void setInsertPt(MachineBasicBlock &MBB, MachineBasicBlock::iterator II)
Set the insertion point before the specified position.
MachineIRBuilder & MIRBuilder
Expose MIRBuilder so clients can set their own RecordInsertInstruction functions. ...
MachineInstrBuilder buildAnyExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ANYEXT Op0.
LegalizeResult widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy)
Legalize an instruction by performing the operation on a wider scalar type (for example a 16-bit addi...
LegalizeResult lowerFMinNumMaxNum(MachineInstr &MI)
const HexagonInstrInfo * TII
static Type * getFloatTy(LLVMContext &C)
Definition: Type.cpp:163
const ConstantFP * getFPImm() const
unsigned getNumOperands() const
Retuns the total number of operands.
Definition: MachineInstr.h:414
MachineInstrBuilder buildUAdde(const DstOp &Res, const DstOp &CarryOut, const SrcOp &Op0, const SrcOp &Op1, const SrcOp &CarryIn)
Build and insert Res, CarryOut = G_UADDE Op0, Op1, CarryIn.
const MachineInstrBuilder & addUse(unsigned RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:41
void eraseFromParent()
Unlink &#39;this&#39; from the containing basic block and delete it.
LegalizeResult narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
MachineInstrBuilder buildAnyExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Res = COPY Op depending on the differing sizes of Res and Op.
LegalizeResult narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:411
LegalizeResult moreElementsVector(MachineInstr &MI, unsigned TypeIdx, LLT MoreTy)
Legalize a vector instruction by increasing the number of vector elements involved and ignoring the a...
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
LLT getElementType() const
Returns the vector&#39;s element type. Only valid for vector types.
LegalizerHelper::LegalizeResult createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall, const CallLowering::ArgInfo &Result, ArrayRef< CallLowering::ArgInfo > Args)
Helper function that creates the given libcall.
AtomicOrdering getOrdering() const
Return the atomic ordering requirements for this memory operation.
The (vector) operation should be implemented by widening the input vector and ignoring the lanes adde...
Definition: LegalizerInfo.h:68
opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
Definition: APFloat.cpp:4483
MachineInstrBuilder buildFNeg(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_FNEG Op0.
MachineInstrBuilder buildInstrNoInsert(unsigned Opcode)
Build but don&#39;t insert <empty> = Opcode <empty>.
MachineFunction & getMF()
Getter for the function we currently build.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:32
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition: Constants.h:137
MachineInstrBuilder buildLShr(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
virtual const TargetInstrInfo * getInstrInfo() const
static const fltSemantics & IEEEdouble() LLVM_READNONE
Definition: APFloat.cpp:158
static LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
MachineInstrBuilder buildSub(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
Build and insert Res = G_SUB Op0, Op1.
LegalizeResult narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt, LLT HalfTy, LLT ShiftAmtTy)
void setChangeObserver(GISelChangeObserver &Observer)
MachineBasicBlock::iterator getInsertPt()
Current insertion point for new instructions.
LegalizerHelper(MachineFunction &MF, GISelChangeObserver &Observer, MachineIRBuilder &B)
LegalizeResult narrowScalarShift(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
MachineInstrBuilder buildSExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_SEXT Op.
Abstract class that contains various methods for clients to notify about changes. ...
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
unsigned const MachineRegisterInfo * MRI
static LLT scalarOrVector(uint16_t NumElements, LLT ScalarTy)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:428
LegalizeResult legalizeInstrStep(MachineInstr &MI)
Replace MI by a sequence of legal instructions that can implement the same operation.
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:64
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
LegalizeResult narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineInstrBuilder buildZExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op.
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:148
MachineInstrBuilder buildCTLZ_ZERO_UNDEF(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTLZ_ZERO_UNDEF Op0, Src0.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
MachineInstrBuilder buildBuildVector(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_BUILD_VECTOR Op0, ...
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:263
virtual bool legalizeIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder) const
Return true if MI is either legal or has been legalized and false if not legal.
Helper class to build MachineInstr.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition: APInt.h:587
MachineInstrBuilder buildUMulH(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
LegalizeResult reduceLoadStoreWidth(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
void setInstr(MachineInstr &MI)
Set the insertion point to before MI.
bool isValid() const
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:732
void setImm(int64_t immVal)
virtual const CallLowering * getCallLowering() const
unsigned getAddressSpace() const
static MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
Definition: ValueTypes.cpp:312
void print(raw_ostream &OS, bool IsStandalone=true, bool SkipOpers=false, bool SkipDebugLoc=false, bool AddNewLine=true, const TargetInstrInfo *TII=nullptr) const
Print this MI to OS.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
Definition: Function.cpp:205
LegalizeResult moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx, LLT MoreTy)
Some kind of error has occurred and we could not legalize this instruction.
static CmpInst::Predicate minMaxToCompare(unsigned Opc)
MachineInstrBuilder buildTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_TRUNC Op.
Instruction was already legal and no change was made to the MachineFunction.
MachineInstr * getOpcodeDef(unsigned Opcode, Register Reg, const MachineRegisterInfo &MRI)
See if Reg is defined by an single def instruction that is Opcode.
Definition: Utils.cpp:303
size_t size() const
Definition: SmallVector.h:52
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
MachineInstrBuilder buildUITOFP(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_UITOFP Src0.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
Definition: APInt.h:970
signed greater than
Definition: InstrTypes.h:759
static Type * getFP128Ty(LLVMContext &C)
Definition: Type.cpp:168
LegalizeResult lowerFCopySign(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
const APFloat & getValueAPF() const
Definition: Constants.h:302
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition: APInt.h:554
static std::pair< int, int > getNarrowTypeBreakDown(LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy)
Try to break down OrigTy into NarrowTy sized pieces.
static Type * getHalfTy(LLVMContext &C)
Definition: Type.cpp:162
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:239
static const fltSemantics & IEEEsingle() LLVM_READNONE
Definition: APFloat.cpp:155
MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = G_LOAD Addr, MMO.
MachineInstrBuilder buildMul(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
Build and insert Res = G_MUL Op0, Op1.
MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert a Res = G_ICMP Pred, Op0, Op1.
void setDesc(const MCInstrDesc &tid)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one...
virtual MachineInstrBuilder buildFConstant(const DstOp &Res, const ConstantFP &Val)
Build and insert Res = G_FCONSTANT Val.
LegalizeResult libcall(MachineInstr &MI)
Legalize an instruction by emiting a runtime library call instead.
mmo_iterator memoperands_begin() const
Access to memory operands of the instruction.
Definition: MachineInstr.h:536
void setFlags(unsigned flags)
Definition: MachineInstr.h:306
static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size)
MachineOperand class - Representation of each machine instruction operand.
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:837
LegalizeResult lowerSITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
unsigned getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
signed less than
Definition: InstrTypes.h:761
Promote Memory to Register
Definition: Mem2Reg.cpp:109
LegalizeResult fewerElementsVectorBasic(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
Legalize a simple vector instruction where all operands are the same type by splitting into multiple ...
The target wants to do something special with this combination of operand and type.
Definition: LegalizerInfo.h:81
LegalizeResult lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
Legalize an instruction by splitting it into simpler parts, hopefully understood by the target...
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:631
MachineInstrBuilder buildSelect(const DstOp &Res, const SrcOp &Tst, const SrcOp &Op0, const SrcOp &Op1)
Build and insert a Res = G_SELECT Tst, Op0, Op1.
static Constant * get(Type *Ty, double V)
This returns a ConstantFP, or a vector containing a splat of a ConstantFP, for the specified value in...
Definition: Constants.cpp:694
LegalizeResult fewerElementsVector(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
Legalize a vector instruction by splitting into multiple components, each acting on the same scalar t...
int64_t getImm() const
const Function & getFunction() const
Return the LLVM function that this machine code represents.
virtual bool legalizeCustom(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder, GISelChangeObserver &Observer) const
MachineInstrBuilder buildAShr(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
LegalizeResult fewerElementsVectorSelect(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
Class for arbitrary precision integers.
Definition: APInt.h:69
static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType, Type *FromType)
virtual void changingInstr(MachineInstr &MI)=0
This instruction is about to be mutated in some way.
static unsigned getReg(const void *D, unsigned RC, unsigned RegNo)
static MachineOperand CreateES(const char *SymName, unsigned char TargetFlags=0)
Register getReg() const
unsigned getBoolExtOp(bool IsVec, bool IsFP) const
static LegalizerHelper::LegalizeResult conversionLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, Type *ToType, Type *FromType)
bool isPointer() const
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:256
LegalizeResult lowerU64ToF32BitOps(MachineInstr &MI)
Representation of each machine instruction.
Definition: MachineInstr.h:64
bool ugt(const APInt &RHS) const
Unsigned greather than comparison.
Definition: APInt.h:1254
Instruction has been legalized and the MachineFunction changed.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
MachineInstrBuilder buildFCmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert a Res = G_FCMP PredOp0, Op1.
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:175
MachineInstrBuilder buildFCanonicalize(const DstOp &Dst, const SrcOp &Src0, Optional< unsigned > Flags=None)
Build and insert Dst = G_FCANONICALIZE Src0.
MachineInstrBuilder buildAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
Build and insert Res = G_ADD Op0, Op1.
const MachineBasicBlock & getMBB() const
Getter for the basic block we currently build.
LegalizeResult fewerElementsVectorImplicitDef(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
MachineInstrBuilder buildPtrToInt(const DstOp &Dst, const SrcOp &Src)
Build and insert a G_PTRTOINT instruction.
void setReg(unsigned Reg)
Change the register this operand corresponds to.
#define I(x, y, z)
Definition: MD5.cpp:58
static Constant * getZeroValueForNegation(Type *Ty)
Floating point negation must be implemented with f(x) = -0.0 - x.
Definition: Constants.cpp:780
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
uint32_t Size
Definition: Profile.cpp:46
void setCImm(const ConstantInt *CI)
const DataLayout & getDataLayout() const
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
uint16_t getFlags() const
Return the MI flags bitvector.
Definition: MachineInstr.h:292
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
LegalizeResult narrowScalar(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
Legalize an instruction by reducing the width of the underlying scalar type.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
MachineInstrBuilder insertInstr(MachineInstrBuilder MIB)
Insert an existing instruction at the insertion point.
uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
This file describes how to lower LLVM calls to machine code calls.
LegalizeResult narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
MachineInstrBuilder buildShl(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E&#39;s largest value.
Definition: BitmaskEnum.h:80
MachineInstrBuilder buildAnd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_AND Op0, Op1.
MachineInstrBuilder buildMerge(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_MERGE_VALUES Op0, ...
IRTranslator LLVM IR MI
unsigned greater than
Definition: InstrTypes.h:755
const MachineInstrBuilder & addDef(unsigned RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
MachineInstrBuilder buildUndef(const DstOp &Res)
Build and insert Res = IMPLICIT_DEF.
static LLT vector(uint16_t NumElements, unsigned ScalarSizeInBits)
Get a low-level vector of some number of elements and element width.
Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
Register getReg() const
getReg - Returns the register number.
MachineInstrBuilder buildConcatVectors(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_CONCAT_VECTORS Op0, ...
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned char TargetFlags=0) const
#define LLVM_DEBUG(X)
Definition: Debug.h:122
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:416
const ConstantInt * getCImm() const
MachineInstrBuilder buildAtomicCmpXchg(Register OldValRes, Register Addr, Register CmpVal, Register NewVal, MachineMemOperand &MMO)
Build and insert OldValRes<def> = G_ATOMIC_CMPXCHG Addr, CmpVal, NewVal, MMO.
The operation is expected to be selectable directly by the target, and no transformation is necessary...
Definition: LegalizerInfo.h:47
bool getFlag(MIFlag Flag) const
Return whether an MI flag is set.
Definition: MachineInstr.h:297
virtual void changedInstr(MachineInstr &MI)=0
This instruction was mutated in some way.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
LegalizeResult fewerElementsVectorCmp(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
bool isNullValue() const
Determine if all bits are clear.
Definition: APInt.h:405
Wrapper class representing virtual and physical registers.
Definition: Register.h:18
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:143
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
Definition: MathExtras.h:658
This file describes how to lower LLVM code to machine code.
unsigned getPredicate() const
void resize(size_type N)
Definition: SmallVector.h:344