LLVM  8.0.0svn
AMDGPUTargetTransformInfo.cpp
Go to the documentation of this file.
1 //===- AMDGPUTargetTransformInfo.cpp - AMDGPU specific TTI pass -----------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // \file
11 // This file implements a TargetTransformInfo analysis pass specific to the
12 // AMDGPU target machine. It uses the target's detailed information to provide
13 // more precise answers to certain TTI queries, while letting the target
14 // independent and default TTI implementations handle the rest.
15 //
16 //===----------------------------------------------------------------------===//
17 
19 #include "AMDGPUSubtarget.h"
20 #include "Utils/AMDGPUBaseInfo.h"
21 #include "llvm/ADT/STLExtras.h"
22 #include "llvm/Analysis/LoopInfo.h"
27 #include "llvm/IR/Argument.h"
28 #include "llvm/IR/Attributes.h"
29 #include "llvm/IR/BasicBlock.h"
30 #include "llvm/IR/CallingConv.h"
31 #include "llvm/IR/DataLayout.h"
32 #include "llvm/IR/DerivedTypes.h"
33 #include "llvm/IR/Function.h"
34 #include "llvm/IR/Instruction.h"
35 #include "llvm/IR/Instructions.h"
36 #include "llvm/IR/IntrinsicInst.h"
37 #include "llvm/IR/Module.h"
38 #include "llvm/IR/PatternMatch.h"
39 #include "llvm/IR/Type.h"
40 #include "llvm/IR/Value.h"
42 #include "llvm/Support/Casting.h"
44 #include "llvm/Support/Debug.h"
49 #include <algorithm>
50 #include <cassert>
51 #include <limits>
52 #include <utility>
53 
54 using namespace llvm;
55 
56 #define DEBUG_TYPE "AMDGPUtti"
57 
59  "amdgpu-unroll-threshold-private",
60  cl::desc("Unroll threshold for AMDGPU if private memory used in a loop"),
61  cl::init(2500), cl::Hidden);
62 
64  "amdgpu-unroll-threshold-local",
65  cl::desc("Unroll threshold for AMDGPU if local memory used in a loop"),
66  cl::init(1000), cl::Hidden);
67 
69  "amdgpu-unroll-threshold-if",
70  cl::desc("Unroll threshold increment for AMDGPU for each if statement inside loop"),
71  cl::init(150), cl::Hidden);
72 
73 static bool dependsOnLocalPhi(const Loop *L, const Value *Cond,
74  unsigned Depth = 0) {
75  const Instruction *I = dyn_cast<Instruction>(Cond);
76  if (!I)
77  return false;
78 
79  for (const Value *V : I->operand_values()) {
80  if (!L->contains(I))
81  continue;
82  if (const PHINode *PHI = dyn_cast<PHINode>(V)) {
83  if (llvm::none_of(L->getSubLoops(), [PHI](const Loop* SubLoop) {
84  return SubLoop->contains(PHI); }))
85  return true;
86  } else if (Depth < 10 && dependsOnLocalPhi(L, V, Depth+1))
87  return true;
88  }
89  return false;
90 }
91 
94  UP.Threshold = 300; // Twice the default.
96  UP.Partial = true;
97 
98  // TODO: Do we want runtime unrolling?
99 
100  // Maximum alloca size than can fit registers. Reserve 16 registers.
101  const unsigned MaxAlloca = (256 - 16) * 4;
102  unsigned ThresholdPrivate = UnrollThresholdPrivate;
103  unsigned ThresholdLocal = UnrollThresholdLocal;
104  unsigned MaxBoost = std::max(ThresholdPrivate, ThresholdLocal);
105  for (const BasicBlock *BB : L->getBlocks()) {
106  const DataLayout &DL = BB->getModule()->getDataLayout();
107  unsigned LocalGEPsSeen = 0;
108 
109  if (llvm::any_of(L->getSubLoops(), [BB](const Loop* SubLoop) {
110  return SubLoop->contains(BB); }))
111  continue; // Block belongs to an inner loop.
112 
113  for (const Instruction &I : *BB) {
114  // Unroll a loop which contains an "if" statement whose condition
115  // defined by a PHI belonging to the loop. This may help to eliminate
116  // if region and potentially even PHI itself, saving on both divergence
117  // and registers used for the PHI.
118  // Add a small bonus for each of such "if" statements.
119  if (const BranchInst *Br = dyn_cast<BranchInst>(&I)) {
120  if (UP.Threshold < MaxBoost && Br->isConditional()) {
121  if (L->isLoopExiting(Br->getSuccessor(0)) ||
122  L->isLoopExiting(Br->getSuccessor(1)))
123  continue;
124  if (dependsOnLocalPhi(L, Br->getCondition())) {
126  LLVM_DEBUG(dbgs() << "Set unroll threshold " << UP.Threshold
127  << " for loop:\n"
128  << *L << " due to " << *Br << '\n');
129  if (UP.Threshold >= MaxBoost)
130  return;
131  }
132  }
133  continue;
134  }
135 
137  if (!GEP)
138  continue;
139 
140  unsigned AS = GEP->getAddressSpace();
141  unsigned Threshold = 0;
142  if (AS == AMDGPUAS::PRIVATE_ADDRESS)
143  Threshold = ThresholdPrivate;
144  else if (AS == AMDGPUAS::LOCAL_ADDRESS)
145  Threshold = ThresholdLocal;
146  else
147  continue;
148 
149  if (UP.Threshold >= Threshold)
150  continue;
151 
152  if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
153  const Value *Ptr = GEP->getPointerOperand();
154  const AllocaInst *Alloca =
156  if (!Alloca || !Alloca->isStaticAlloca())
157  continue;
158  Type *Ty = Alloca->getAllocatedType();
159  unsigned AllocaSize = Ty->isSized() ? DL.getTypeAllocSize(Ty) : 0;
160  if (AllocaSize > MaxAlloca)
161  continue;
162  } else if (AS == AMDGPUAS::LOCAL_ADDRESS) {
163  LocalGEPsSeen++;
164  // Inhibit unroll for local memory if we have seen addressing not to
165  // a variable, most likely we will be unable to combine it.
166  // Do not unroll too deep inner loops for local memory to give a chance
167  // to unroll an outer loop for a more important reason.
168  if (LocalGEPsSeen > 1 || L->getLoopDepth() > 2 ||
169  (!isa<GlobalVariable>(GEP->getPointerOperand()) &&
170  !isa<Argument>(GEP->getPointerOperand())))
171  continue;
172  }
173 
174  // Check if GEP depends on a value defined by this loop itself.
175  bool HasLoopDef = false;
176  for (const Value *Op : GEP->operands()) {
177  const Instruction *Inst = dyn_cast<Instruction>(Op);
178  if (!Inst || L->isLoopInvariant(Op))
179  continue;
180 
181  if (llvm::any_of(L->getSubLoops(), [Inst](const Loop* SubLoop) {
182  return SubLoop->contains(Inst); }))
183  continue;
184  HasLoopDef = true;
185  break;
186  }
187  if (!HasLoopDef)
188  continue;
189 
190  // We want to do whatever we can to limit the number of alloca
191  // instructions that make it through to the code generator. allocas
192  // require us to use indirect addressing, which is slow and prone to
193  // compiler bugs. If this loop does an address calculation on an
194  // alloca ptr, then we want to use a higher than normal loop unroll
195  // threshold. This will give SROA a better chance to eliminate these
196  // allocas.
197  //
198  // We also want to have more unrolling for local memory to let ds
199  // instructions with different offsets combine.
200  //
201  // Don't use the maximum allowed value here as it will make some
202  // programs way too big.
203  UP.Threshold = Threshold;
204  LLVM_DEBUG(dbgs() << "Set unroll threshold " << Threshold
205  << " for loop:\n"
206  << *L << " due to " << *GEP << '\n');
207  if (UP.Threshold >= MaxBoost)
208  return;
209  }
210  }
211 }
212 
214  // The concept of vector registers doesn't really exist. Some packed vector
215  // operations operate on the normal 32-bit registers.
216  return 256;
217 }
218 
219 unsigned GCNTTIImpl::getNumberOfRegisters(bool Vec) const {
220  // This is really the number of registers to fill when vectorizing /
221  // interleaving loops, so we lie to avoid trying to use all registers.
222  return getHardwareNumberOfRegisters(Vec) >> 3;
223 }
224 
225 unsigned GCNTTIImpl::getRegisterBitWidth(bool Vector) const {
226  return 32;
227 }
228 
230  return 32;
231 }
232 
233 unsigned GCNTTIImpl::getLoadVectorFactor(unsigned VF, unsigned LoadSize,
234  unsigned ChainSizeInBytes,
235  VectorType *VecTy) const {
236  unsigned VecRegBitWidth = VF * LoadSize;
237  if (VecRegBitWidth > 128 && VecTy->getScalarSizeInBits() < 32)
238  // TODO: Support element-size less than 32bit?
239  return 128 / LoadSize;
240 
241  return VF;
242 }
243 
244 unsigned GCNTTIImpl::getStoreVectorFactor(unsigned VF, unsigned StoreSize,
245  unsigned ChainSizeInBytes,
246  VectorType *VecTy) const {
247  unsigned VecRegBitWidth = VF * StoreSize;
248  if (VecRegBitWidth > 128)
249  return 128 / StoreSize;
250 
251  return VF;
252 }
253 
254 unsigned GCNTTIImpl::getLoadStoreVecRegBitWidth(unsigned AddrSpace) const {
255  if (AddrSpace == AMDGPUAS::GLOBAL_ADDRESS ||
256  AddrSpace == AMDGPUAS::CONSTANT_ADDRESS ||
257  AddrSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT) {
258  return 512;
259  }
260 
261  if (AddrSpace == AMDGPUAS::FLAT_ADDRESS ||
262  AddrSpace == AMDGPUAS::LOCAL_ADDRESS ||
263  AddrSpace == AMDGPUAS::REGION_ADDRESS)
264  return 128;
265 
266  if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS)
267  return 8 * ST->getMaxPrivateElementSize();
268 
269  llvm_unreachable("unhandled address space");
270 }
271 
272 bool GCNTTIImpl::isLegalToVectorizeMemChain(unsigned ChainSizeInBytes,
273  unsigned Alignment,
274  unsigned AddrSpace) const {
275  // We allow vectorization of flat stores, even though we may need to decompose
276  // them later if they may access private memory. We don't have enough context
277  // here, and legalization can handle it.
278  if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS) {
279  return (Alignment >= 4 || ST->hasUnalignedScratchAccess()) &&
280  ChainSizeInBytes <= ST->getMaxPrivateElementSize();
281  }
282  return true;
283 }
284 
285 bool GCNTTIImpl::isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes,
286  unsigned Alignment,
287  unsigned AddrSpace) const {
288  return isLegalToVectorizeMemChain(ChainSizeInBytes, Alignment, AddrSpace);
289 }
290 
291 bool GCNTTIImpl::isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes,
292  unsigned Alignment,
293  unsigned AddrSpace) const {
294  return isLegalToVectorizeMemChain(ChainSizeInBytes, Alignment, AddrSpace);
295 }
296 
297 unsigned GCNTTIImpl::getMaxInterleaveFactor(unsigned VF) {
298  // Disable unrolling if the loop is not vectorized.
299  // TODO: Enable this again.
300  if (VF == 1)
301  return 1;
302 
303  return 8;
304 }
305 
307  MemIntrinsicInfo &Info) const {
308  switch (Inst->getIntrinsicID()) {
309  case Intrinsic::amdgcn_atomic_inc:
310  case Intrinsic::amdgcn_atomic_dec:
311  case Intrinsic::amdgcn_ds_fadd:
312  case Intrinsic::amdgcn_ds_fmin:
313  case Intrinsic::amdgcn_ds_fmax: {
314  auto *Ordering = dyn_cast<ConstantInt>(Inst->getArgOperand(2));
315  auto *Volatile = dyn_cast<ConstantInt>(Inst->getArgOperand(4));
316  if (!Ordering || !Volatile)
317  return false; // Invalid.
318 
319  unsigned OrderingVal = Ordering->getZExtValue();
320  if (OrderingVal > static_cast<unsigned>(AtomicOrdering::SequentiallyConsistent))
321  return false;
322 
323  Info.PtrVal = Inst->getArgOperand(0);
324  Info.Ordering = static_cast<AtomicOrdering>(OrderingVal);
325  Info.ReadMem = true;
326  Info.WriteMem = true;
327  Info.IsVolatile = !Volatile->isNullValue();
328  return true;
329  }
330  default:
331  return false;
332  }
333 }
334 
336  unsigned Opcode, Type *Ty, TTI::OperandValueKind Opd1Info,
337  TTI::OperandValueKind Opd2Info, TTI::OperandValueProperties Opd1PropInfo,
339  EVT OrigTy = TLI->getValueType(DL, Ty);
340  if (!OrigTy.isSimple()) {
341  return BaseT::getArithmeticInstrCost(Opcode, Ty, Opd1Info, Opd2Info,
342  Opd1PropInfo, Opd2PropInfo);
343  }
344 
345  // Legalize the type.
346  std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, Ty);
347  int ISD = TLI->InstructionOpcodeToISD(Opcode);
348 
349  // Because we don't have any legal vector operations, but the legal types, we
350  // need to account for split vectors.
351  unsigned NElts = LT.second.isVector() ?
352  LT.second.getVectorNumElements() : 1;
353 
354  MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;
355 
356  switch (ISD) {
357  case ISD::SHL:
358  case ISD::SRL:
359  case ISD::SRA:
360  if (SLT == MVT::i64)
361  return get64BitInstrCost() * LT.first * NElts;
362 
363  // i32
364  return getFullRateInstrCost() * LT.first * NElts;
365  case ISD::ADD:
366  case ISD::SUB:
367  case ISD::AND:
368  case ISD::OR:
369  case ISD::XOR:
370  if (SLT == MVT::i64){
371  // and, or and xor are typically split into 2 VALU instructions.
372  return 2 * getFullRateInstrCost() * LT.first * NElts;
373  }
374 
375  return LT.first * NElts * getFullRateInstrCost();
376  case ISD::MUL: {
377  const int QuarterRateCost = getQuarterRateInstrCost();
378  if (SLT == MVT::i64) {
379  const int FullRateCost = getFullRateInstrCost();
380  return (4 * QuarterRateCost + (2 * 2) * FullRateCost) * LT.first * NElts;
381  }
382 
383  // i32
384  return QuarterRateCost * NElts * LT.first;
385  }
386  case ISD::FADD:
387  case ISD::FSUB:
388  case ISD::FMUL:
389  if (SLT == MVT::f64)
390  return LT.first * NElts * get64BitInstrCost();
391 
392  if (SLT == MVT::f32 || SLT == MVT::f16)
393  return LT.first * NElts * getFullRateInstrCost();
394  break;
395  case ISD::FDIV:
396  case ISD::FREM:
397  // FIXME: frem should be handled separately. The fdiv in it is most of it,
398  // but the current lowering is also not entirely correct.
399  if (SLT == MVT::f64) {
400  int Cost = 4 * get64BitInstrCost() + 7 * getQuarterRateInstrCost();
401  // Add cost of workaround.
402  if (ST->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS)
403  Cost += 3 * getFullRateInstrCost();
404 
405  return LT.first * Cost * NElts;
406  }
407 
408  if (!Args.empty() && match(Args[0], PatternMatch::m_FPOne())) {
409  // TODO: This is more complicated, unsafe flags etc.
410  if ((SLT == MVT::f32 && !ST->hasFP32Denormals()) ||
411  (SLT == MVT::f16 && ST->has16BitInsts())) {
412  return LT.first * getQuarterRateInstrCost() * NElts;
413  }
414  }
415 
416  if (SLT == MVT::f16 && ST->has16BitInsts()) {
417  // 2 x v_cvt_f32_f16
418  // f32 rcp
419  // f32 fmul
420  // v_cvt_f16_f32
421  // f16 div_fixup
422  int Cost = 4 * getFullRateInstrCost() + 2 * getQuarterRateInstrCost();
423  return LT.first * Cost * NElts;
424  }
425 
426  if (SLT == MVT::f32 || SLT == MVT::f16) {
427  int Cost = 7 * getFullRateInstrCost() + 1 * getQuarterRateInstrCost();
428 
429  if (!ST->hasFP32Denormals()) {
430  // FP mode switches.
431  Cost += 2 * getFullRateInstrCost();
432  }
433 
434  return LT.first * NElts * Cost;
435  }
436  break;
437  default:
438  break;
439  }
440 
441  return BaseT::getArithmeticInstrCost(Opcode, Ty, Opd1Info, Opd2Info,
442  Opd1PropInfo, Opd2PropInfo);
443 }
444 
445 unsigned GCNTTIImpl::getCFInstrCost(unsigned Opcode) {
446  // XXX - For some reason this isn't called for switch.
447  switch (Opcode) {
448  case Instruction::Br:
449  case Instruction::Ret:
450  return 10;
451  default:
452  return BaseT::getCFInstrCost(Opcode);
453  }
454 }
455 
457  bool IsPairwise) {
458  EVT OrigTy = TLI->getValueType(DL, Ty);
459 
460  // Computes cost on targets that have packed math instructions(which support
461  // 16-bit types only).
462  if (IsPairwise ||
463  !ST->hasVOP3PInsts() ||
464  OrigTy.getScalarSizeInBits() != 16)
465  return BaseT::getArithmeticReductionCost(Opcode, Ty, IsPairwise);
466 
467  std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, Ty);
468  return LT.first * getFullRateInstrCost();
469 }
470 
472  bool IsPairwise,
473  bool IsUnsigned) {
474  EVT OrigTy = TLI->getValueType(DL, Ty);
475 
476  // Computes cost on targets that have packed math instructions(which support
477  // 16-bit types only).
478  if (IsPairwise ||
479  !ST->hasVOP3PInsts() ||
480  OrigTy.getScalarSizeInBits() != 16)
481  return BaseT::getMinMaxReductionCost(Ty, CondTy, IsPairwise, IsUnsigned);
482 
483  std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, Ty);
484  return LT.first * getHalfRateInstrCost();
485 }
486 
487 int GCNTTIImpl::getVectorInstrCost(unsigned Opcode, Type *ValTy,
488  unsigned Index) {
489  switch (Opcode) {
490  case Instruction::ExtractElement:
491  case Instruction::InsertElement: {
492  unsigned EltSize
493  = DL.getTypeSizeInBits(cast<VectorType>(ValTy)->getElementType());
494  if (EltSize < 32) {
495  if (EltSize == 16 && Index == 0 && ST->has16BitInsts())
496  return 0;
497  return BaseT::getVectorInstrCost(Opcode, ValTy, Index);
498  }
499 
500  // Extracts are just reads of a subregister, so are free. Inserts are
501  // considered free because we don't want to have any cost for scalarizing
502  // operations, and we don't have to copy into a different register class.
503 
504  // Dynamic indexing isn't free and is best avoided.
505  return Index == ~0u ? 2 : 0;
506  }
507  default:
508  return BaseT::getVectorInstrCost(Opcode, ValTy, Index);
509  }
510 }
511 
512 
513 
514 static bool isArgPassedInSGPR(const Argument *A) {
515  const Function *F = A->getParent();
516 
517  // Arguments to compute shaders are never a source of divergence.
519  switch (CC) {
522  return true;
530  // For non-compute shaders, SGPR inputs are marked with either inreg or byval.
531  // Everything else is in VGPRs.
532  return F->getAttributes().hasParamAttribute(A->getArgNo(), Attribute::InReg) ||
533  F->getAttributes().hasParamAttribute(A->getArgNo(), Attribute::ByVal);
534  default:
535  // TODO: Should calls support inreg for SGPR inputs?
536  return false;
537  }
538 }
539 
540 /// \returns true if the result of the value could potentially be
541 /// different across workitems in a wavefront.
543  if (const Argument *A = dyn_cast<Argument>(V))
544  return !isArgPassedInSGPR(A);
545 
546  // Loads from the private and flat address spaces are divergent, because
547  // threads can execute the load instruction with the same inputs and get
548  // different results.
549  //
550  // All other loads are not divergent, because if threads issue loads with the
551  // same arguments, they will always get the same result.
552  if (const LoadInst *Load = dyn_cast<LoadInst>(V))
553  return Load->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS ||
554  Load->getPointerAddressSpace() == AMDGPUAS::FLAT_ADDRESS;
555 
556  // Atomics are divergent because they are executed sequentially: when an
557  // atomic operation refers to the same address in each thread, then each
558  // thread after the first sees the value written by the previous thread as
559  // original value.
560  if (isa<AtomicRMWInst>(V) || isa<AtomicCmpXchgInst>(V))
561  return true;
562 
563  if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(V))
564  return AMDGPU::isIntrinsicSourceOfDivergence(Intrinsic->getIntrinsicID());
565 
566  // Assume all function calls are a source of divergence.
567  if (isa<CallInst>(V) || isa<InvokeInst>(V))
568  return true;
569 
570  return false;
571 }
572 
573 bool GCNTTIImpl::isAlwaysUniform(const Value *V) const {
574  if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(V)) {
575  switch (Intrinsic->getIntrinsicID()) {
576  default:
577  return false;
578  case Intrinsic::amdgcn_readfirstlane:
579  case Intrinsic::amdgcn_readlane:
580  return true;
581  }
582  }
583  return false;
584 }
585 
587  Type *SubTp) {
588  if (ST->hasVOP3PInsts()) {
589  VectorType *VT = cast<VectorType>(Tp);
590  if (VT->getNumElements() == 2 &&
591  DL.getTypeSizeInBits(VT->getElementType()) == 16) {
592  // With op_sel VOP3P instructions freely can access the low half or high
593  // half of a register, so any swizzle is free.
594 
595  switch (Kind) {
596  case TTI::SK_Broadcast:
597  case TTI::SK_Reverse:
599  return 0;
600  default:
601  break;
602  }
603  }
604  }
605 
606  return BaseT::getShuffleCost(Kind, Tp, Index, SubTp);
607 }
608 
610  const Function *Callee) const {
611  const TargetMachine &TM = getTLI()->getTargetMachine();
612  const FeatureBitset &CallerBits =
613  TM.getSubtargetImpl(*Caller)->getFeatureBits();
614  const FeatureBitset &CalleeBits =
615  TM.getSubtargetImpl(*Callee)->getFeatureBits();
616 
617  FeatureBitset RealCallerBits = CallerBits & ~InlineFeatureIgnoreList;
618  FeatureBitset RealCalleeBits = CalleeBits & ~InlineFeatureIgnoreList;
619  return ((RealCallerBits & RealCalleeBits) == RealCalleeBits);
620 }
621 
624  CommonTTI.getUnrollingPreferences(L, SE, UP);
625 }
626 
628  return 4 * 128; // XXX - 4 channels. Should these count as vector instead?
629 }
630 
631 unsigned R600TTIImpl::getNumberOfRegisters(bool Vec) const {
632  return getHardwareNumberOfRegisters(Vec);
633 }
634 
635 unsigned R600TTIImpl::getRegisterBitWidth(bool Vector) const {
636  return 32;
637 }
638 
640  return 32;
641 }
642 
643 unsigned R600TTIImpl::getLoadStoreVecRegBitWidth(unsigned AddrSpace) const {
644  if (AddrSpace == AMDGPUAS::GLOBAL_ADDRESS ||
645  AddrSpace == AMDGPUAS::CONSTANT_ADDRESS)
646  return 128;
647  if (AddrSpace == AMDGPUAS::LOCAL_ADDRESS ||
648  AddrSpace == AMDGPUAS::REGION_ADDRESS)
649  return 64;
650  if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS)
651  return 32;
652 
653  if ((AddrSpace == AMDGPUAS::PARAM_D_ADDRESS ||
654  AddrSpace == AMDGPUAS::PARAM_I_ADDRESS ||
655  (AddrSpace >= AMDGPUAS::CONSTANT_BUFFER_0 &&
656  AddrSpace <= AMDGPUAS::CONSTANT_BUFFER_15)))
657  return 128;
658  llvm_unreachable("unhandled address space");
659 }
660 
661 bool R600TTIImpl::isLegalToVectorizeMemChain(unsigned ChainSizeInBytes,
662  unsigned Alignment,
663  unsigned AddrSpace) const {
664  // We allow vectorization of flat stores, even though we may need to decompose
665  // them later if they may access private memory. We don't have enough context
666  // here, and legalization can handle it.
667  return (AddrSpace != AMDGPUAS::PRIVATE_ADDRESS);
668 }
669 
670 bool R600TTIImpl::isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes,
671  unsigned Alignment,
672  unsigned AddrSpace) const {
673  return isLegalToVectorizeMemChain(ChainSizeInBytes, Alignment, AddrSpace);
674 }
675 
676 bool R600TTIImpl::isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes,
677  unsigned Alignment,
678  unsigned AddrSpace) const {
679  return isLegalToVectorizeMemChain(ChainSizeInBytes, Alignment, AddrSpace);
680 }
681 
682 unsigned R600TTIImpl::getMaxInterleaveFactor(unsigned VF) {
683  // Disable unrolling if the loop is not vectorized.
684  // TODO: Enable this again.
685  if (VF == 1)
686  return 1;
687 
688  return 8;
689 }
690 
691 unsigned R600TTIImpl::getCFInstrCost(unsigned Opcode) {
692  // XXX - For some reason this isn't called for switch.
693  switch (Opcode) {
694  case Instruction::Br:
695  case Instruction::Ret:
696  return 10;
697  default:
698  return BaseT::getCFInstrCost(Opcode);
699  }
700 }
701 
702 int R600TTIImpl::getVectorInstrCost(unsigned Opcode, Type *ValTy,
703  unsigned Index) {
704  switch (Opcode) {
705  case Instruction::ExtractElement:
706  case Instruction::InsertElement: {
707  unsigned EltSize
708  = DL.getTypeSizeInBits(cast<VectorType>(ValTy)->getElementType());
709  if (EltSize < 32) {
710  return BaseT::getVectorInstrCost(Opcode, ValTy, Index);
711  }
712 
713  // Extracts are just reads of a subregister, so are free. Inserts are
714  // considered free because we don't want to have any cost for scalarizing
715  // operations, and we don't have to copy into a different register class.
716 
717  // Dynamic indexing isn't free and is best avoided.
718  return Index == ~0u ? 2 : 0;
719  }
720  default:
721  return BaseT::getVectorInstrCost(Opcode, ValTy, Index);
722  }
723 }
724 
727  CommonTTI.getUnrollingPreferences(L, SE, UP);
728 }
Calling convention used for Mesa/AMDPAL hull shaders (= tessellation control shaders).
Definition: CallingConv.h:208
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:111
bool Partial
Allow partial unrolling (unrolling of loops to expand the size of the loop body, not only to eliminat...
unsigned getArithmeticInstrCost(unsigned Opcode, Type *Ty, TTI::OperandValueKind Opd1Info=TTI::OK_AnyValue, TTI::OperandValueKind Opd2Info=TTI::OK_AnyValue, TTI::OperandValueProperties Opd1PropInfo=TTI::OP_None, TTI::OperandValueProperties Opd2PropInfo=TTI::OP_None, ArrayRef< const Value * > Args=ArrayRef< const Value * >())
Definition: BasicTTIImpl.h:568
bool isLegalToVectorizeMemChain(unsigned ChainSizeInBytes, unsigned Alignment, unsigned AddrSpace) const
bool isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes, unsigned Alignment, unsigned AddrSpace) const
This file a TargetTransformInfo::Concept conforming object specific to the AMDGPU target machine...
GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2)
This class represents an incoming formal argument to a Function.
Definition: Argument.h:30
AMDGPU specific subclass of TargetSubtarget.
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
Address space for constant memory (VTX2)
Definition: AMDGPU.h:243
bool isSized(SmallPtrSetImpl< Type *> *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
Definition: Type.h:265
static cl::opt< unsigned > UnrollThresholdIf("amdgpu-unroll-threshold-if", cl::desc("Unroll threshold increment for AMDGPU for each if statement inside loop"), cl::init(150), cl::Hidden)
unsigned getCFInstrCost(unsigned Opcode)
unsigned getLoopDepth() const
Return the nesting level of this loop.
Definition: LoopInfo.h:92
The main scalar evolution driver.
bool isLegalToVectorizeMemChain(unsigned ChainSizeInBytes, unsigned Alignment, unsigned AddrSpace) const
Address space for flat memory.
Definition: AMDGPU.h:239
int getVectorInstrCost(unsigned Opcode, Type *ValTy, unsigned Index)
Calling convention used for Mesa/AMDPAL pixel shaders.
Definition: CallingConv.h:195
unsigned getMinVectorRegisterBitWidth() const
Calling convention used for Mesa/AMDPAL compute shaders.
Definition: CallingConv.h:198
void getUnrollingPreferences(Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP)
F(f)
An instruction for reading from memory.
Definition: Instructions.h:168
int getArithmeticReductionCost(unsigned Opcode, Type *Ty, bool IsPairwise)
Hexagon Common GEP
Address space for region memory.
Definition: AMDGPU.h:241
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:49
unsigned getLoadStoreVecRegBitWidth(unsigned AddrSpace) const
const FeatureBitset & getFeatureBits() const
Address space for local memory.
Definition: AMDGPU.h:244
bool isAlwaysUniform(const Value *V) const
Shift and rotation operations.
Definition: ISDOpcodes.h:399
unsigned getNumberOfRegisters(bool Vector) const
bool isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes, unsigned Alignment, unsigned AddrSpace) const
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1062
This file contains the simple types necessary to represent the attributes associated with functions a...
unsigned getArithmeticReductionCost(unsigned Opcode, Type *Ty, bool IsPairwise)
Try to calculate arithmetic and shuffle op costs for reduction operations.
Address space for 32-bit constant memory.
Definition: AMDGPU.h:247
uint64_t getNumElements() const
Definition: DerivedTypes.h:359
AtomicOrdering
Atomic ordering for LLVM&#39;s memory model.
Calling convention used for Mesa/AMDPAL geometry shaders.
Definition: CallingConv.h:192
unsigned getScalarSizeInBits() const
Definition: ValueTypes.h:298
bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info) const
bool isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes, unsigned Alignment, unsigned AddrSpace) const
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:201
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:33
unsigned getRegisterBitWidth(bool Vector) const
unsigned getMaxInterleaveFactor(unsigned VF)
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition: Function.h:224
static bool dependsOnLocalPhi(const Loop *L, const Value *Cond, unsigned Depth=0)
Reverse the order of the vector.
unsigned getAddressSpace() const
Returns the address space of this instruction&#39;s pointer type.
Definition: Instructions.h:952
unsigned getCFInstrCost(unsigned Opcode)
an instruction for type-safe pointer arithmetic to access elements of arrays and structs ...
Definition: Instructions.h:843
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:419
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:149
Container class for subtarget features.
LLVM Basic Block Representation.
Definition: BasicBlock.h:58
bool isLoopExiting(const BlockT *BB) const
True if terminator in the block can branch to another block that is outside of the current loop...
Definition: LoopInfo.h:203
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:46
Simple binary floating point operators.
Definition: ISDOpcodes.h:276
Conditional or Unconditional Branch instruction.
unsigned getMinMaxReductionCost(Type *Ty, Type *CondTy, bool IsPairwise, bool)
Try to calculate op costs for min/max reduction operations.
bool isIntrinsicSourceOfDivergence(unsigned IntrID)
unsigned getStoreVectorFactor(unsigned VF, unsigned StoreSize, unsigned ChainSizeInBytes, VectorType *VecTy) const
bool any_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1055
unsigned getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index, Type *SubTp)
unsigned getNumberOfRegisters(bool Vec) const
op_range operands()
Definition: User.h:238
static bool isArgPassedInSGPR(const Argument *A)
Calling convention used for Mesa vertex shaders, or AMDPAL last shader stage before rasterization (ve...
Definition: CallingConv.h:189
Extended Value Type.
Definition: ValueTypes.h:34
unsigned getHardwareNumberOfRegisters(bool Vec) const
unsigned getLoadStoreVecRegBitWidth(unsigned AddrSpace) const
Value * GetUnderlyingObject(Value *V, const DataLayout &DL, unsigned MaxLookup=6)
This method strips off any GEP address adjustments and pointer casts from the specified value...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
Definition: Instructions.h:106
bool isLoopInvariant(const Value *V) const
Return true if the specified value is loop invariant.
Definition: LoopInfo.cpp:58
OperandValueProperties
Additional properties of an operand&#39;s values.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:51
unsigned getCFInstrCost(unsigned Opcode)
Definition: BasicTTIImpl.h:767
unsigned getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index, Type *SubTp)
Definition: BasicTTIImpl.h:615
bool contains(const LoopT *L) const
Return true if the specified loop is contained within in this loop.
Definition: LoopInfo.h:110
unsigned getLoadVectorFactor(unsigned VF, unsigned LoadSize, unsigned ChainSizeInBytes, VectorType *VecTy) const
Address space for private memory.
Definition: AMDGPU.h:245
bool hasParamAttribute(unsigned ArgNo, Attribute::AttrKind Kind) const
Equivalent to hasAttribute(ArgNo + FirstArgIndex, Kind).
This is the shared class of boolean and integer constants.
Definition: Constants.h:84
bool isSourceOfDivergence(const Value *V) const
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type...
Definition: Type.cpp:130
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:213
Module.h This file contains the declarations for the Module class.
int getArithmeticInstrCost(unsigned Opcode, Type *Ty, TTI::OperandValueKind Opd1Info=TTI::OK_AnyValue, TTI::OperandValueKind Opd2Info=TTI::OK_AnyValue, TTI::OperandValueProperties Opd1PropInfo=TTI::OP_None, TTI::OperandValueProperties Opd2PropInfo=TTI::OP_None, ArrayRef< const Value *> Args=ArrayRef< const Value *>())
int getMinMaxReductionCost(Type *Ty, Type *CondTy, bool IsPairwiseForm, bool IsUnsigned)
bool isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes, unsigned Alignment, unsigned AddrSpace) const
Address space for direct addressible parameter memory (CONST0)
Definition: AMDGPU.h:250
virtual const TargetSubtargetInfo * getSubtargetImpl(const Function &) const
Virtual method implemented by subclasses that returns a reference to that target&#39;s TargetSubtargetInf...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:133
unsigned getMaxInterleaveFactor(unsigned VF)
Class to represent vector types.
Definition: DerivedTypes.h:393
bool areInlineCompatible(const Function *Caller, const Function *Callee) const
unsigned getArgNo() const
Return the index of this formal argument in its containing function.
Definition: Argument.h:48
int getVectorInstrCost(unsigned Opcode, Type *ValTy, unsigned Index)
Calling convention for AMDGPU code object kernels.
Definition: CallingConv.h:201
uint64_t getTypeSizeInBits(Type *Ty) const
Size examples:
Definition: DataLayout.h:560
uint64_t getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Definition: DataLayout.h:428
const std::vector< LoopT * > & getSubLoops() const
Return the loops contained entirely within this loop.
Definition: LoopInfo.h:131
specific_fpval m_FPOne()
Match a float 1.0 or vector with all elements equal to 1.0.
Definition: PatternMatch.h:540
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:380
unsigned Threshold
The cost threshold for the unrolled loop.
const Function * getParent() const
Definition: Argument.h:42
unsigned getHardwareNumberOfRegisters(bool Vector) const
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:459
ArrayRef< BlockT * > getBlocks() const
Get a list of the basic blocks which make up this loop.
Definition: LoopInfo.h:149
Parameters that control the generic loop unrolling transformation.
void getUnrollingPreferences(Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP)
#define I(x, y, z)
Definition: MD5.cpp:58
iterator_range< value_op_iterator > operand_values()
Definition: User.h:262
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:323
unsigned getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index)
Definition: BasicTTIImpl.h:812
SPIR_KERNEL - Calling convention for SPIR kernel functions.
Definition: CallingConv.h:137
static cl::opt< unsigned > UnrollThresholdLocal("amdgpu-unroll-threshold-local", cl::desc("Unroll threshold for AMDGPU if local memory used in a loop"), cl::init(1000), cl::Hidden)
static int const Threshold
TODO: Write a new FunctionPass AliasAnalysis so that it can keep a cache.
const unsigned Kind
Value * getArgOperand(unsigned i) const
getArgOperand/setArgOperand - Return/set the i-th call argument.
Calling convention used for AMDPAL vertex shader if tessellation is in use.
Definition: CallingConv.h:216
LLVM Value Representation.
Definition: Value.h:73
static cl::opt< unsigned > UnrollThresholdPrivate("amdgpu-unroll-threshold-private", cl::desc("Unroll threshold for AMDGPU if private memory used in a loop"), cl::init(2500), cl::Hidden)
Broadcast element 0 to all other elements.
Address space for global memory (RAT0, VTX0).
Definition: AMDGPU.h:240
unsigned getMinVectorRegisterBitWidth() const
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:59
Type * getElementType() const
Definition: DerivedTypes.h:360
Value * PtrVal
This is the pointer that the intrinsic is loading from or storing to.
OperandValueKind
Additional information about an operand&#39;s possible values.
This pass exposes codegen information to IR-level passes.
bool isStaticAlloca() const
Return true if this alloca is in the entry block of the function and is a constant size...
Address space for indirect addressible parameter memory (VTX1)
Definition: AMDGPU.h:252
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:126
void getUnrollingPreferences(Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP)
#define LLVM_DEBUG(X)
Definition: Debug.h:123
Information about a load/store intrinsic defined by the target.
Calling convention used for AMDPAL shader stage before geometry shader if geometry is in use...
Definition: CallingConv.h:221
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
unsigned getRegisterBitWidth(bool Vector) const
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:144
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:44
an instruction to allocate memory on the stack
Definition: Instructions.h:60
ShuffleKind
The various kinds of shuffle patterns for vector queries.
Shuffle elements of single source vector with any shuffle mask.