LLVM  8.0.0svn
AMDGPUCodeGenPrepare.cpp
Go to the documentation of this file.
1 //===-- AMDGPUCodeGenPrepare.cpp ------------------------------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file
11 /// This pass does misc. AMDGPU optimizations on IR before instruction
12 /// selection.
13 //
14 //===----------------------------------------------------------------------===//
15 
16 #include "AMDGPU.h"
17 #include "AMDGPUSubtarget.h"
18 #include "AMDGPUTargetMachine.h"
19 #include "llvm/ADT/StringRef.h"
22 #include "llvm/Analysis/Loads.h"
24 #include "llvm/CodeGen/Passes.h"
26 #include "llvm/IR/Attributes.h"
27 #include "llvm/IR/BasicBlock.h"
28 #include "llvm/IR/Constants.h"
29 #include "llvm/IR/DerivedTypes.h"
30 #include "llvm/IR/Function.h"
31 #include "llvm/IR/IRBuilder.h"
32 #include "llvm/IR/InstVisitor.h"
33 #include "llvm/IR/InstrTypes.h"
34 #include "llvm/IR/Instruction.h"
35 #include "llvm/IR/Instructions.h"
36 #include "llvm/IR/IntrinsicInst.h"
37 #include "llvm/IR/Intrinsics.h"
38 #include "llvm/IR/LLVMContext.h"
39 #include "llvm/IR/Operator.h"
40 #include "llvm/IR/Type.h"
41 #include "llvm/IR/Value.h"
42 #include "llvm/Pass.h"
43 #include "llvm/Support/Casting.h"
44 #include <cassert>
45 #include <iterator>
46 
47 #define DEBUG_TYPE "amdgpu-codegenprepare"
48 
49 using namespace llvm;
50 
51 namespace {
52 
53 static cl::opt<bool> WidenLoads(
54  "amdgpu-codegenprepare-widen-constant-loads",
55  cl::desc("Widen sub-dword constant address space loads in AMDGPUCodeGenPrepare"),
57  cl::init(true));
58 
59 class AMDGPUCodeGenPrepare : public FunctionPass,
60  public InstVisitor<AMDGPUCodeGenPrepare, bool> {
61  const GCNSubtarget *ST = nullptr;
62  AssumptionCache *AC = nullptr;
63  LegacyDivergenceAnalysis *DA = nullptr;
64  Module *Mod = nullptr;
65  bool HasUnsafeFPMath = false;
66 
67  /// Copies exact/nsw/nuw flags (if any) from binary operation \p I to
68  /// binary operation \p V.
69  ///
70  /// \returns Binary operation \p V.
71  /// \returns \p T's base element bit width.
72  unsigned getBaseElementBitWidth(const Type *T) const;
73 
74  /// \returns Equivalent 32 bit integer type for given type \p T. For example,
75  /// if \p T is i7, then i32 is returned; if \p T is <3 x i12>, then <3 x i32>
76  /// is returned.
77  Type *getI32Ty(IRBuilder<> &B, const Type *T) const;
78 
79  /// \returns True if binary operation \p I is a signed binary operation, false
80  /// otherwise.
81  bool isSigned(const BinaryOperator &I) const;
82 
83  /// \returns True if the condition of 'select' operation \p I comes from a
84  /// signed 'icmp' operation, false otherwise.
85  bool isSigned(const SelectInst &I) const;
86 
87  /// \returns True if type \p T needs to be promoted to 32 bit integer type,
88  /// false otherwise.
89  bool needsPromotionToI32(const Type *T) const;
90 
91  /// Promotes uniform binary operation \p I to equivalent 32 bit binary
92  /// operation.
93  ///
94  /// \details \p I's base element bit width must be greater than 1 and less
95  /// than or equal 16. Promotion is done by sign or zero extending operands to
96  /// 32 bits, replacing \p I with equivalent 32 bit binary operation, and
97  /// truncating the result of 32 bit binary operation back to \p I's original
98  /// type. Division operation is not promoted.
99  ///
100  /// \returns True if \p I is promoted to equivalent 32 bit binary operation,
101  /// false otherwise.
102  bool promoteUniformOpToI32(BinaryOperator &I) const;
103 
104  /// Promotes uniform 'icmp' operation \p I to 32 bit 'icmp' operation.
105  ///
106  /// \details \p I's base element bit width must be greater than 1 and less
107  /// than or equal 16. Promotion is done by sign or zero extending operands to
108  /// 32 bits, and replacing \p I with 32 bit 'icmp' operation.
109  ///
110  /// \returns True.
111  bool promoteUniformOpToI32(ICmpInst &I) const;
112 
113  /// Promotes uniform 'select' operation \p I to 32 bit 'select'
114  /// operation.
115  ///
116  /// \details \p I's base element bit width must be greater than 1 and less
117  /// than or equal 16. Promotion is done by sign or zero extending operands to
118  /// 32 bits, replacing \p I with 32 bit 'select' operation, and truncating the
119  /// result of 32 bit 'select' operation back to \p I's original type.
120  ///
121  /// \returns True.
122  bool promoteUniformOpToI32(SelectInst &I) const;
123 
124  /// Promotes uniform 'bitreverse' intrinsic \p I to 32 bit 'bitreverse'
125  /// intrinsic.
126  ///
127  /// \details \p I's base element bit width must be greater than 1 and less
128  /// than or equal 16. Promotion is done by zero extending the operand to 32
129  /// bits, replacing \p I with 32 bit 'bitreverse' intrinsic, shifting the
130  /// result of 32 bit 'bitreverse' intrinsic to the right with zero fill (the
131  /// shift amount is 32 minus \p I's base element bit width), and truncating
132  /// the result of the shift operation back to \p I's original type.
133  ///
134  /// \returns True.
135  bool promoteUniformBitreverseToI32(IntrinsicInst &I) const;
136 
137  /// Expands 24 bit div or rem.
138  Value* expandDivRem24(IRBuilder<> &Builder, BinaryOperator &I,
139  Value *Num, Value *Den,
140  bool IsDiv, bool IsSigned) const;
141 
142  /// Expands 32 bit div or rem.
143  Value* expandDivRem32(IRBuilder<> &Builder, BinaryOperator &I,
144  Value *Num, Value *Den) const;
145 
146  /// Widen a scalar load.
147  ///
148  /// \details \p Widen scalar load for uniform, small type loads from constant
149  // memory / to a full 32-bits and then truncate the input to allow a scalar
150  // load instead of a vector load.
151  //
152  /// \returns True.
153 
154  bool canWidenScalarExtLoad(LoadInst &I) const;
155 
156 public:
157  static char ID;
158 
159  AMDGPUCodeGenPrepare() : FunctionPass(ID) {}
160 
161  bool visitFDiv(BinaryOperator &I);
162 
163  bool visitInstruction(Instruction &I) { return false; }
164  bool visitBinaryOperator(BinaryOperator &I);
165  bool visitLoadInst(LoadInst &I);
166  bool visitICmpInst(ICmpInst &I);
167  bool visitSelectInst(SelectInst &I);
168 
169  bool visitIntrinsicInst(IntrinsicInst &I);
170  bool visitBitreverseIntrinsicInst(IntrinsicInst &I);
171 
172  bool doInitialization(Module &M) override;
173  bool runOnFunction(Function &F) override;
174 
175  StringRef getPassName() const override { return "AMDGPU IR optimizations"; }
176 
177  void getAnalysisUsage(AnalysisUsage &AU) const override {
180  AU.setPreservesAll();
181  }
182 };
183 
184 } // end anonymous namespace
185 
186 unsigned AMDGPUCodeGenPrepare::getBaseElementBitWidth(const Type *T) const {
187  assert(needsPromotionToI32(T) && "T does not need promotion to i32");
188 
189  if (T->isIntegerTy())
190  return T->getIntegerBitWidth();
191  return cast<VectorType>(T)->getElementType()->getIntegerBitWidth();
192 }
193 
194 Type *AMDGPUCodeGenPrepare::getI32Ty(IRBuilder<> &B, const Type *T) const {
195  assert(needsPromotionToI32(T) && "T does not need promotion to i32");
196 
197  if (T->isIntegerTy())
198  return B.getInt32Ty();
199  return VectorType::get(B.getInt32Ty(), cast<VectorType>(T)->getNumElements());
200 }
201 
202 bool AMDGPUCodeGenPrepare::isSigned(const BinaryOperator &I) const {
203  return I.getOpcode() == Instruction::AShr ||
204  I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::SRem;
205 }
206 
207 bool AMDGPUCodeGenPrepare::isSigned(const SelectInst &I) const {
208  return isa<ICmpInst>(I.getOperand(0)) ?
209  cast<ICmpInst>(I.getOperand(0))->isSigned() : false;
210 }
211 
212 bool AMDGPUCodeGenPrepare::needsPromotionToI32(const Type *T) const {
213  const IntegerType *IntTy = dyn_cast<IntegerType>(T);
214  if (IntTy && IntTy->getBitWidth() > 1 && IntTy->getBitWidth() <= 16)
215  return true;
216 
217  if (const VectorType *VT = dyn_cast<VectorType>(T)) {
218  // TODO: The set of packed operations is more limited, so may want to
219  // promote some anyway.
220  if (ST->hasVOP3PInsts())
221  return false;
222 
223  return needsPromotionToI32(VT->getElementType());
224  }
225 
226  return false;
227 }
228 
229 // Return true if the op promoted to i32 should have nsw set.
230 static bool promotedOpIsNSW(const Instruction &I) {
231  switch (I.getOpcode()) {
232  case Instruction::Shl:
233  case Instruction::Add:
234  case Instruction::Sub:
235  return true;
236  case Instruction::Mul:
237  return I.hasNoUnsignedWrap();
238  default:
239  return false;
240  }
241 }
242 
243 // Return true if the op promoted to i32 should have nuw set.
244 static bool promotedOpIsNUW(const Instruction &I) {
245  switch (I.getOpcode()) {
246  case Instruction::Shl:
247  case Instruction::Add:
248  case Instruction::Mul:
249  return true;
250  case Instruction::Sub:
251  return I.hasNoUnsignedWrap();
252  default:
253  return false;
254  }
255 }
256 
257 bool AMDGPUCodeGenPrepare::canWidenScalarExtLoad(LoadInst &I) const {
258  Type *Ty = I.getType();
259  const DataLayout &DL = Mod->getDataLayout();
260  int TySize = DL.getTypeSizeInBits(Ty);
261  unsigned Align = I.getAlignment() ?
262  I.getAlignment() : DL.getABITypeAlignment(Ty);
263 
264  return I.isSimple() && TySize < 32 && Align >= 4 && DA->isUniform(&I);
265 }
266 
267 bool AMDGPUCodeGenPrepare::promoteUniformOpToI32(BinaryOperator &I) const {
268  assert(needsPromotionToI32(I.getType()) &&
269  "I does not need promotion to i32");
270 
271  if (I.getOpcode() == Instruction::SDiv ||
272  I.getOpcode() == Instruction::UDiv ||
273  I.getOpcode() == Instruction::SRem ||
274  I.getOpcode() == Instruction::URem)
275  return false;
276 
277  IRBuilder<> Builder(&I);
279 
280  Type *I32Ty = getI32Ty(Builder, I.getType());
281  Value *ExtOp0 = nullptr;
282  Value *ExtOp1 = nullptr;
283  Value *ExtRes = nullptr;
284  Value *TruncRes = nullptr;
285 
286  if (isSigned(I)) {
287  ExtOp0 = Builder.CreateSExt(I.getOperand(0), I32Ty);
288  ExtOp1 = Builder.CreateSExt(I.getOperand(1), I32Ty);
289  } else {
290  ExtOp0 = Builder.CreateZExt(I.getOperand(0), I32Ty);
291  ExtOp1 = Builder.CreateZExt(I.getOperand(1), I32Ty);
292  }
293 
294  ExtRes = Builder.CreateBinOp(I.getOpcode(), ExtOp0, ExtOp1);
295  if (Instruction *Inst = dyn_cast<Instruction>(ExtRes)) {
296  if (promotedOpIsNSW(cast<Instruction>(I)))
297  Inst->setHasNoSignedWrap();
298 
299  if (promotedOpIsNUW(cast<Instruction>(I)))
300  Inst->setHasNoUnsignedWrap();
301 
302  if (const auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I))
303  Inst->setIsExact(ExactOp->isExact());
304  }
305 
306  TruncRes = Builder.CreateTrunc(ExtRes, I.getType());
307 
308  I.replaceAllUsesWith(TruncRes);
309  I.eraseFromParent();
310 
311  return true;
312 }
313 
314 bool AMDGPUCodeGenPrepare::promoteUniformOpToI32(ICmpInst &I) const {
315  assert(needsPromotionToI32(I.getOperand(0)->getType()) &&
316  "I does not need promotion to i32");
317 
318  IRBuilder<> Builder(&I);
320 
321  Type *I32Ty = getI32Ty(Builder, I.getOperand(0)->getType());
322  Value *ExtOp0 = nullptr;
323  Value *ExtOp1 = nullptr;
324  Value *NewICmp = nullptr;
325 
326  if (I.isSigned()) {
327  ExtOp0 = Builder.CreateSExt(I.getOperand(0), I32Ty);
328  ExtOp1 = Builder.CreateSExt(I.getOperand(1), I32Ty);
329  } else {
330  ExtOp0 = Builder.CreateZExt(I.getOperand(0), I32Ty);
331  ExtOp1 = Builder.CreateZExt(I.getOperand(1), I32Ty);
332  }
333  NewICmp = Builder.CreateICmp(I.getPredicate(), ExtOp0, ExtOp1);
334 
335  I.replaceAllUsesWith(NewICmp);
336  I.eraseFromParent();
337 
338  return true;
339 }
340 
341 bool AMDGPUCodeGenPrepare::promoteUniformOpToI32(SelectInst &I) const {
342  assert(needsPromotionToI32(I.getType()) &&
343  "I does not need promotion to i32");
344 
345  IRBuilder<> Builder(&I);
347 
348  Type *I32Ty = getI32Ty(Builder, I.getType());
349  Value *ExtOp1 = nullptr;
350  Value *ExtOp2 = nullptr;
351  Value *ExtRes = nullptr;
352  Value *TruncRes = nullptr;
353 
354  if (isSigned(I)) {
355  ExtOp1 = Builder.CreateSExt(I.getOperand(1), I32Ty);
356  ExtOp2 = Builder.CreateSExt(I.getOperand(2), I32Ty);
357  } else {
358  ExtOp1 = Builder.CreateZExt(I.getOperand(1), I32Ty);
359  ExtOp2 = Builder.CreateZExt(I.getOperand(2), I32Ty);
360  }
361  ExtRes = Builder.CreateSelect(I.getOperand(0), ExtOp1, ExtOp2);
362  TruncRes = Builder.CreateTrunc(ExtRes, I.getType());
363 
364  I.replaceAllUsesWith(TruncRes);
365  I.eraseFromParent();
366 
367  return true;
368 }
369 
370 bool AMDGPUCodeGenPrepare::promoteUniformBitreverseToI32(
371  IntrinsicInst &I) const {
372  assert(I.getIntrinsicID() == Intrinsic::bitreverse &&
373  "I must be bitreverse intrinsic");
374  assert(needsPromotionToI32(I.getType()) &&
375  "I does not need promotion to i32");
376 
377  IRBuilder<> Builder(&I);
379 
380  Type *I32Ty = getI32Ty(Builder, I.getType());
381  Function *I32 =
382  Intrinsic::getDeclaration(Mod, Intrinsic::bitreverse, { I32Ty });
383  Value *ExtOp = Builder.CreateZExt(I.getOperand(0), I32Ty);
384  Value *ExtRes = Builder.CreateCall(I32, { ExtOp });
385  Value *LShrOp =
386  Builder.CreateLShr(ExtRes, 32 - getBaseElementBitWidth(I.getType()));
387  Value *TruncRes =
388  Builder.CreateTrunc(LShrOp, I.getType());
389 
390  I.replaceAllUsesWith(TruncRes);
391  I.eraseFromParent();
392 
393  return true;
394 }
395 
396 static bool shouldKeepFDivF32(Value *Num, bool UnsafeDiv, bool HasDenormals) {
397  const ConstantFP *CNum = dyn_cast<ConstantFP>(Num);
398  if (!CNum)
399  return HasDenormals;
400 
401  if (UnsafeDiv)
402  return true;
403 
404  bool IsOne = CNum->isExactlyValue(+1.0) || CNum->isExactlyValue(-1.0);
405 
406  // Reciprocal f32 is handled separately without denormals.
407  return HasDenormals ^ IsOne;
408 }
409 
410 // Insert an intrinsic for fast fdiv for safe math situations where we can
411 // reduce precision. Leave fdiv for situations where the generic node is
412 // expected to be optimized.
413 bool AMDGPUCodeGenPrepare::visitFDiv(BinaryOperator &FDiv) {
414  Type *Ty = FDiv.getType();
415 
416  if (!Ty->getScalarType()->isFloatTy())
417  return false;
418 
419  MDNode *FPMath = FDiv.getMetadata(LLVMContext::MD_fpmath);
420  if (!FPMath)
421  return false;
422 
423  const FPMathOperator *FPOp = cast<const FPMathOperator>(&FDiv);
424  float ULP = FPOp->getFPAccuracy();
425  if (ULP < 2.5f)
426  return false;
427 
428  FastMathFlags FMF = FPOp->getFastMathFlags();
429  bool UnsafeDiv = HasUnsafeFPMath || FMF.isFast() ||
430  FMF.allowReciprocal();
431 
432  // With UnsafeDiv node will be optimized to just rcp and mul.
433  if (UnsafeDiv)
434  return false;
435 
436  IRBuilder<> Builder(FDiv.getParent(), std::next(FDiv.getIterator()), FPMath);
437  Builder.setFastMathFlags(FMF);
438  Builder.SetCurrentDebugLocation(FDiv.getDebugLoc());
439 
440  Function *Decl = Intrinsic::getDeclaration(Mod, Intrinsic::amdgcn_fdiv_fast);
441 
442  Value *Num = FDiv.getOperand(0);
443  Value *Den = FDiv.getOperand(1);
444 
445  Value *NewFDiv = nullptr;
446 
447  bool HasDenormals = ST->hasFP32Denormals();
448  if (VectorType *VT = dyn_cast<VectorType>(Ty)) {
449  NewFDiv = UndefValue::get(VT);
450 
451  // FIXME: Doesn't do the right thing for cases where the vector is partially
452  // constant. This works when the scalarizer pass is run first.
453  for (unsigned I = 0, E = VT->getNumElements(); I != E; ++I) {
454  Value *NumEltI = Builder.CreateExtractElement(Num, I);
455  Value *DenEltI = Builder.CreateExtractElement(Den, I);
456  Value *NewElt;
457 
458  if (shouldKeepFDivF32(NumEltI, UnsafeDiv, HasDenormals)) {
459  NewElt = Builder.CreateFDiv(NumEltI, DenEltI);
460  } else {
461  NewElt = Builder.CreateCall(Decl, { NumEltI, DenEltI });
462  }
463 
464  NewFDiv = Builder.CreateInsertElement(NewFDiv, NewElt, I);
465  }
466  } else {
467  if (!shouldKeepFDivF32(Num, UnsafeDiv, HasDenormals))
468  NewFDiv = Builder.CreateCall(Decl, { Num, Den });
469  }
470 
471  if (NewFDiv) {
472  FDiv.replaceAllUsesWith(NewFDiv);
473  NewFDiv->takeName(&FDiv);
474  FDiv.eraseFromParent();
475  }
476 
477  return !!NewFDiv;
478 }
479 
480 static bool hasUnsafeFPMath(const Function &F) {
481  Attribute Attr = F.getFnAttribute("unsafe-fp-math");
482  return Attr.getValueAsString() == "true";
483 }
484 
485 static std::pair<Value*, Value*> getMul64(IRBuilder<> &Builder,
486  Value *LHS, Value *RHS) {
487  Type *I32Ty = Builder.getInt32Ty();
488  Type *I64Ty = Builder.getInt64Ty();
489 
490  Value *LHS_EXT64 = Builder.CreateZExt(LHS, I64Ty);
491  Value *RHS_EXT64 = Builder.CreateZExt(RHS, I64Ty);
492  Value *MUL64 = Builder.CreateMul(LHS_EXT64, RHS_EXT64);
493  Value *Lo = Builder.CreateTrunc(MUL64, I32Ty);
494  Value *Hi = Builder.CreateLShr(MUL64, Builder.getInt64(32));
495  Hi = Builder.CreateTrunc(Hi, I32Ty);
496  return std::make_pair(Lo, Hi);
497 }
498 
499 static Value* getMulHu(IRBuilder<> &Builder, Value *LHS, Value *RHS) {
500  return getMul64(Builder, LHS, RHS).second;
501 }
502 
503 // The fractional part of a float is enough to accurately represent up to
504 // a 24-bit signed integer.
505 Value* AMDGPUCodeGenPrepare::expandDivRem24(IRBuilder<> &Builder,
506  BinaryOperator &I,
507  Value *Num, Value *Den,
508  bool IsDiv, bool IsSigned) const {
509  assert(Num->getType()->isIntegerTy(32));
510 
511  const DataLayout &DL = Mod->getDataLayout();
512  unsigned LHSSignBits = ComputeNumSignBits(Num, DL, 0, AC, &I);
513  if (LHSSignBits < 9)
514  return nullptr;
515 
516  unsigned RHSSignBits = ComputeNumSignBits(Den, DL, 0, AC, &I);
517  if (RHSSignBits < 9)
518  return nullptr;
519 
520 
521  unsigned SignBits = std::min(LHSSignBits, RHSSignBits);
522  unsigned DivBits = 32 - SignBits;
523  if (IsSigned)
524  ++DivBits;
525 
526  Type *Ty = Num->getType();
527  Type *I32Ty = Builder.getInt32Ty();
528  Type *F32Ty = Builder.getFloatTy();
529  ConstantInt *One = Builder.getInt32(1);
530  Value *JQ = One;
531 
532  if (IsSigned) {
533  // char|short jq = ia ^ ib;
534  JQ = Builder.CreateXor(Num, Den);
535 
536  // jq = jq >> (bitsize - 2)
537  JQ = Builder.CreateAShr(JQ, Builder.getInt32(30));
538 
539  // jq = jq | 0x1
540  JQ = Builder.CreateOr(JQ, One);
541  }
542 
543  // int ia = (int)LHS;
544  Value *IA = Num;
545 
546  // int ib, (int)RHS;
547  Value *IB = Den;
548 
549  // float fa = (float)ia;
550  Value *FA = IsSigned ? Builder.CreateSIToFP(IA, F32Ty)
551  : Builder.CreateUIToFP(IA, F32Ty);
552 
553  // float fb = (float)ib;
554  Value *FB = IsSigned ? Builder.CreateSIToFP(IB,F32Ty)
555  : Builder.CreateUIToFP(IB,F32Ty);
556 
557  Value *RCP = Builder.CreateFDiv(ConstantFP::get(F32Ty, 1.0), FB);
558  Value *FQM = Builder.CreateFMul(FA, RCP);
559 
560  // fq = trunc(fqm);
561  CallInst* FQ = Builder.CreateIntrinsic(Intrinsic::trunc, { FQM });
562  FQ->copyFastMathFlags(Builder.getFastMathFlags());
563 
564  // float fqneg = -fq;
565  Value *FQNeg = Builder.CreateFNeg(FQ);
566 
567  // float fr = mad(fqneg, fb, fa);
568  Value *FR = Builder.CreateIntrinsic(Intrinsic::amdgcn_fmad_ftz,
569  { FQNeg, FB, FA }, FQ);
570 
571  // int iq = (int)fq;
572  Value *IQ = IsSigned ? Builder.CreateFPToSI(FQ, I32Ty)
573  : Builder.CreateFPToUI(FQ, I32Ty);
574 
575  // fr = fabs(fr);
576  FR = Builder.CreateIntrinsic(Intrinsic::fabs, { FR }, FQ);
577 
578  // fb = fabs(fb);
579  FB = Builder.CreateIntrinsic(Intrinsic::fabs, { FB }, FQ);
580 
581  // int cv = fr >= fb;
582  Value *CV = Builder.CreateFCmpOGE(FR, FB);
583 
584  // jq = (cv ? jq : 0);
585  JQ = Builder.CreateSelect(CV, JQ, Builder.getInt32(0));
586 
587  // dst = iq + jq;
588  Value *Div = Builder.CreateAdd(IQ, JQ);
589 
590  Value *Res = Div;
591  if (!IsDiv) {
592  // Rem needs compensation, it's easier to recompute it
593  Value *Rem = Builder.CreateMul(Div, Den);
594  Res = Builder.CreateSub(Num, Rem);
595  }
596 
597  // Truncate to number of bits this divide really is.
598  if (IsSigned) {
599  Res = Builder.CreateTrunc(Res, Builder.getIntNTy(DivBits));
600  Res = Builder.CreateSExt(Res, Ty);
601  } else {
602  ConstantInt *TruncMask = Builder.getInt32((UINT64_C(1) << DivBits) - 1);
603  Res = Builder.CreateAnd(Res, TruncMask);
604  }
605 
606  return Res;
607 }
608 
609 Value* AMDGPUCodeGenPrepare::expandDivRem32(IRBuilder<> &Builder,
610  BinaryOperator &I,
611  Value *Num, Value *Den) const {
613  assert(Opc == Instruction::URem || Opc == Instruction::UDiv ||
614  Opc == Instruction::SRem || Opc == Instruction::SDiv);
615 
616  FastMathFlags FMF;
617  FMF.setFast();
618  Builder.setFastMathFlags(FMF);
619 
620  if (isa<Constant>(Den))
621  return nullptr; // Keep it for optimization
622 
623  bool IsDiv = Opc == Instruction::UDiv || Opc == Instruction::SDiv;
624  bool IsSigned = Opc == Instruction::SRem || Opc == Instruction::SDiv;
625 
626  Type *Ty = Num->getType();
627  Type *I32Ty = Builder.getInt32Ty();
628  Type *F32Ty = Builder.getFloatTy();
629 
630  if (Ty->getScalarSizeInBits() < 32) {
631  if (IsSigned) {
632  Num = Builder.CreateSExt(Num, I32Ty);
633  Den = Builder.CreateSExt(Den, I32Ty);
634  } else {
635  Num = Builder.CreateZExt(Num, I32Ty);
636  Den = Builder.CreateZExt(Den, I32Ty);
637  }
638  }
639 
640  if (Value *Res = expandDivRem24(Builder, I, Num, Den, IsDiv, IsSigned)) {
641  Res = Builder.CreateTrunc(Res, Ty);
642  return Res;
643  }
644 
645  ConstantInt *Zero = Builder.getInt32(0);
646  ConstantInt *One = Builder.getInt32(1);
647  ConstantInt *MinusOne = Builder.getInt32(~0);
648 
649  Value *Sign = nullptr;
650  if (IsSigned) {
651  ConstantInt *K31 = Builder.getInt32(31);
652  Value *LHSign = Builder.CreateAShr(Num, K31);
653  Value *RHSign = Builder.CreateAShr(Den, K31);
654  // Remainder sign is the same as LHS
655  Sign = IsDiv ? Builder.CreateXor(LHSign, RHSign) : LHSign;
656 
657  Num = Builder.CreateAdd(Num, LHSign);
658  Den = Builder.CreateAdd(Den, RHSign);
659 
660  Num = Builder.CreateXor(Num, LHSign);
661  Den = Builder.CreateXor(Den, RHSign);
662  }
663 
664  // RCP = URECIP(Den) = 2^32 / Den + e
665  // e is rounding error.
666  Value *DEN_F32 = Builder.CreateUIToFP(Den, F32Ty);
667  Value *RCP_F32 = Builder.CreateFDiv(ConstantFP::get(F32Ty, 1.0), DEN_F32);
668  Constant *UINT_MAX_PLUS_1 = ConstantFP::get(F32Ty, BitsToFloat(0x4f800000));
669  Value *RCP_SCALE = Builder.CreateFMul(RCP_F32, UINT_MAX_PLUS_1);
670  Value *RCP = Builder.CreateFPToUI(RCP_SCALE, I32Ty);
671 
672  // RCP_LO, RCP_HI = mul(RCP, Den) */
673  Value *RCP_LO, *RCP_HI;
674  std::tie(RCP_LO, RCP_HI) = getMul64(Builder, RCP, Den);
675 
676  // NEG_RCP_LO = -RCP_LO
677  Value *NEG_RCP_LO = Builder.CreateNeg(RCP_LO);
678 
679  // ABS_RCP_LO = (RCP_HI == 0 ? NEG_RCP_LO : RCP_LO)
680  Value *RCP_HI_0_CC = Builder.CreateICmpEQ(RCP_HI, Zero);
681  Value *ABS_RCP_LO = Builder.CreateSelect(RCP_HI_0_CC, NEG_RCP_LO, RCP_LO);
682 
683  // Calculate the rounding error from the URECIP instruction
684  // E = mulhu(ABS_RCP_LO, RCP)
685  Value *E = getMulHu(Builder, ABS_RCP_LO, RCP);
686 
687  // RCP_A_E = RCP + E
688  Value *RCP_A_E = Builder.CreateAdd(RCP, E);
689 
690  // RCP_S_E = RCP - E
691  Value *RCP_S_E = Builder.CreateSub(RCP, E);
692 
693  // Tmp0 = (RCP_HI == 0 ? RCP_A_E : RCP_SUB_E)
694  Value *Tmp0 = Builder.CreateSelect(RCP_HI_0_CC, RCP_A_E, RCP_S_E);
695 
696  // Quotient = mulhu(Tmp0, Num)
697  Value *Quotient = getMulHu(Builder, Tmp0, Num);
698 
699  // Num_S_Remainder = Quotient * Den
700  Value *Num_S_Remainder = Builder.CreateMul(Quotient, Den);
701 
702  // Remainder = Num - Num_S_Remainder
703  Value *Remainder = Builder.CreateSub(Num, Num_S_Remainder);
704 
705  // Remainder_GE_Den = (Remainder >= Den ? -1 : 0)
706  Value *Rem_GE_Den_CC = Builder.CreateICmpUGE(Remainder, Den);
707  Value *Remainder_GE_Den = Builder.CreateSelect(Rem_GE_Den_CC, MinusOne, Zero);
708 
709  // Remainder_GE_Zero = (Num >= Num_S_Remainder ? -1 : 0)
710  Value *Num_GE_Num_S_Rem_CC = Builder.CreateICmpUGE(Num, Num_S_Remainder);
711  Value *Remainder_GE_Zero = Builder.CreateSelect(Num_GE_Num_S_Rem_CC,
712  MinusOne, Zero);
713 
714  // Tmp1 = Remainder_GE_Den & Remainder_GE_Zero
715  Value *Tmp1 = Builder.CreateAnd(Remainder_GE_Den, Remainder_GE_Zero);
716  Value *Tmp1_0_CC = Builder.CreateICmpEQ(Tmp1, Zero);
717 
718  Value *Res;
719  if (IsDiv) {
720  // Quotient_A_One = Quotient + 1
721  Value *Quotient_A_One = Builder.CreateAdd(Quotient, One);
722 
723  // Quotient_S_One = Quotient - 1
724  Value *Quotient_S_One = Builder.CreateSub(Quotient, One);
725 
726  // Div = (Tmp1 == 0 ? Quotient : Quotient_A_One)
727  Value *Div = Builder.CreateSelect(Tmp1_0_CC, Quotient, Quotient_A_One);
728 
729  // Div = (Remainder_GE_Zero == 0 ? Quotient_S_One : Div)
730  Res = Builder.CreateSelect(Num_GE_Num_S_Rem_CC, Div, Quotient_S_One);
731  } else {
732  // Remainder_S_Den = Remainder - Den
733  Value *Remainder_S_Den = Builder.CreateSub(Remainder, Den);
734 
735  // Remainder_A_Den = Remainder + Den
736  Value *Remainder_A_Den = Builder.CreateAdd(Remainder, Den);
737 
738  // Rem = (Tmp1 == 0 ? Remainder : Remainder_S_Den)
739  Value *Rem = Builder.CreateSelect(Tmp1_0_CC, Remainder, Remainder_S_Den);
740 
741  // Rem = (Remainder_GE_Zero == 0 ? Remainder_A_Den : Rem)
742  Res = Builder.CreateSelect(Num_GE_Num_S_Rem_CC, Rem, Remainder_A_Den);
743  }
744 
745  if (IsSigned) {
746  Res = Builder.CreateXor(Res, Sign);
747  Res = Builder.CreateSub(Res, Sign);
748  }
749 
750  Res = Builder.CreateTrunc(Res, Ty);
751 
752  return Res;
753 }
754 
755 bool AMDGPUCodeGenPrepare::visitBinaryOperator(BinaryOperator &I) {
756  if (ST->has16BitInsts() && needsPromotionToI32(I.getType()) &&
757  DA->isUniform(&I) && promoteUniformOpToI32(I))
758  return true;
759 
760  bool Changed = false;
762  Type *Ty = I.getType();
763  Value *NewDiv = nullptr;
764  if ((Opc == Instruction::URem || Opc == Instruction::UDiv ||
765  Opc == Instruction::SRem || Opc == Instruction::SDiv) &&
766  Ty->getScalarSizeInBits() <= 32) {
767  Value *Num = I.getOperand(0);
768  Value *Den = I.getOperand(1);
769  IRBuilder<> Builder(&I);
771 
772  if (VectorType *VT = dyn_cast<VectorType>(Ty)) {
773  NewDiv = UndefValue::get(VT);
774 
775  for (unsigned N = 0, E = VT->getNumElements(); N != E; ++N) {
776  Value *NumEltN = Builder.CreateExtractElement(Num, N);
777  Value *DenEltN = Builder.CreateExtractElement(Den, N);
778  Value *NewElt = expandDivRem32(Builder, I, NumEltN, DenEltN);
779  if (!NewElt)
780  NewElt = Builder.CreateBinOp(Opc, NumEltN, DenEltN);
781  NewDiv = Builder.CreateInsertElement(NewDiv, NewElt, N);
782  }
783  } else {
784  NewDiv = expandDivRem32(Builder, I, Num, Den);
785  }
786 
787  if (NewDiv) {
788  I.replaceAllUsesWith(NewDiv);
789  I.eraseFromParent();
790  Changed = true;
791  }
792  }
793 
794  return Changed;
795 }
796 
797 bool AMDGPUCodeGenPrepare::visitLoadInst(LoadInst &I) {
798  if (!WidenLoads)
799  return false;
800 
803  canWidenScalarExtLoad(I)) {
804  IRBuilder<> Builder(&I);
806 
807  Type *I32Ty = Builder.getInt32Ty();
808  Type *PT = PointerType::get(I32Ty, I.getPointerAddressSpace());
809  Value *BitCast= Builder.CreateBitCast(I.getPointerOperand(), PT);
810  LoadInst *WidenLoad = Builder.CreateLoad(BitCast);
811  WidenLoad->copyMetadata(I);
812 
813  // If we have range metadata, we need to convert the type, and not make
814  // assumptions about the high bits.
815  if (auto *Range = WidenLoad->getMetadata(LLVMContext::MD_range)) {
816  ConstantInt *Lower =
817  mdconst::extract<ConstantInt>(Range->getOperand(0));
818 
819  if (Lower->getValue().isNullValue()) {
820  WidenLoad->setMetadata(LLVMContext::MD_range, nullptr);
821  } else {
822  Metadata *LowAndHigh[] = {
824  // Don't make assumptions about the high bits.
826  };
827 
829  MDNode::get(Mod->getContext(), LowAndHigh));
830  }
831  }
832 
833  int TySize = Mod->getDataLayout().getTypeSizeInBits(I.getType());
834  Type *IntNTy = Builder.getIntNTy(TySize);
835  Value *ValTrunc = Builder.CreateTrunc(WidenLoad, IntNTy);
836  Value *ValOrig = Builder.CreateBitCast(ValTrunc, I.getType());
837  I.replaceAllUsesWith(ValOrig);
838  I.eraseFromParent();
839  return true;
840  }
841 
842  return false;
843 }
844 
845 bool AMDGPUCodeGenPrepare::visitICmpInst(ICmpInst &I) {
846  bool Changed = false;
847 
848  if (ST->has16BitInsts() && needsPromotionToI32(I.getOperand(0)->getType()) &&
849  DA->isUniform(&I))
850  Changed |= promoteUniformOpToI32(I);
851 
852  return Changed;
853 }
854 
855 bool AMDGPUCodeGenPrepare::visitSelectInst(SelectInst &I) {
856  bool Changed = false;
857 
858  if (ST->has16BitInsts() && needsPromotionToI32(I.getType()) &&
859  DA->isUniform(&I))
860  Changed |= promoteUniformOpToI32(I);
861 
862  return Changed;
863 }
864 
865 bool AMDGPUCodeGenPrepare::visitIntrinsicInst(IntrinsicInst &I) {
866  switch (I.getIntrinsicID()) {
867  case Intrinsic::bitreverse:
868  return visitBitreverseIntrinsicInst(I);
869  default:
870  return false;
871  }
872 }
873 
874 bool AMDGPUCodeGenPrepare::visitBitreverseIntrinsicInst(IntrinsicInst &I) {
875  bool Changed = false;
876 
877  if (ST->has16BitInsts() && needsPromotionToI32(I.getType()) &&
878  DA->isUniform(&I))
879  Changed |= promoteUniformBitreverseToI32(I);
880 
881  return Changed;
882 }
883 
884 bool AMDGPUCodeGenPrepare::doInitialization(Module &M) {
885  Mod = &M;
886  return false;
887 }
888 
890  if (skipFunction(F))
891  return false;
892 
893  auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
894  if (!TPC)
895  return false;
896 
897  const AMDGPUTargetMachine &TM = TPC->getTM<AMDGPUTargetMachine>();
898  ST = &TM.getSubtarget<GCNSubtarget>(F);
899  AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
900  DA = &getAnalysis<LegacyDivergenceAnalysis>();
901  HasUnsafeFPMath = hasUnsafeFPMath(F);
902 
903  bool MadeChange = false;
904 
905  for (BasicBlock &BB : F) {
907  for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E; I = Next) {
908  Next = std::next(I);
909  MadeChange |= visit(*I);
910  }
911  }
912 
913  return MadeChange;
914 }
915 
916 INITIALIZE_PASS_BEGIN(AMDGPUCodeGenPrepare, DEBUG_TYPE,
917  "AMDGPU IR optimizations", false, false)
921  false, false)
922 
923 char AMDGPUCodeGenPrepare::ID = 0;
924 
926  return new AMDGPUCodeGenPrepare();
927 }
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks &#39;this&#39; from the containing basic block and deletes it.
Definition: Instruction.cpp:68
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:111
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
void copyFastMathFlags(FastMathFlags FMF)
Convenience function for transferring all fast-math flag values to this instruction, which must be an operator which supports these flags.
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1846
bool isSimple() const
Definition: Instructions.h:266
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:1246
FastMathFlags getFastMathFlags() const
Get the flags to be applied to created floating point ops.
Definition: IRBuilder.h:211
Base class for instruction visitors.
Definition: InstVisitor.h:81
AMDGPU specific subclass of TargetSubtarget.
void setFast(bool B=true)
Definition: Operator.h:231
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
BinaryOps getOpcode() const
Definition: InstrTypes.h:355
Value * CreateXor(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1148
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:63
APInt zext(unsigned width) const
Zero extend to a new width.
Definition: APInt.cpp:858
This class represents a function call, abstracting a target machine&#39;s calling convention.
An immutable pass that tracks lazily created AssumptionCache objects.
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space...
Definition: Type.cpp:617
float BitsToFloat(uint32_t Bits)
This function takes a 32-bit integer and returns the bit equivalent float.
Definition: MathExtras.h:581
A cache of @llvm.assume calls within a function.
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1560
FunctionPass * createAMDGPUCodeGenPreparePass()
Metadata node.
Definition: Metadata.h:864
F(f)
An instruction for reading from memory.
Definition: Instructions.h:168
static bool promotedOpIsNUW(const Instruction &I)
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
Definition: IRBuilder.h:347
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:51
bool isSigned() const
Definition: InstrTypes.h:854
This class represents the LLVM &#39;select&#39; instruction.
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
Definition: IRBuilder.h:352
AMDGPU IR optimizations
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:197
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:731
This file contains the simple types necessary to represent the attributes associated with functions a...
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:962
Metadata * LowAndHigh[]
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1629
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:245
#define T
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:217
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition: Constants.h:138
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:126
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:979
Value * CreateSIToFP(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1606
void SetCurrentDebugLocation(DebugLoc L)
Set location information used by debugging information.
Definition: IRBuilder.h:151
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:439
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1556
unsigned getBitWidth() const
Get the number of bits in this IntegerType.
Definition: DerivedTypes.h:66
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:301
static ConstantAsMetadata * get(Constant *C)
Definition: Metadata.h:410
Function * getDeclaration(Module *M, ID id, ArrayRef< Type *> Tys=None)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:1021
Value * getOperand(unsigned i) const
Definition: User.h:170
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1130
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return &#39;this&#39;.
Definition: Type.h:304
Value * getOperand(unsigned i_nocapture) const
bool isFloatTy() const
Return true if this is &#39;float&#39;, a 32-bit IEEE fp type.
Definition: Type.h:147
LoadInst * CreateLoad(Value *Ptr, const char *Name)
Provided to resolve &#39;CreateLoad(Ptr, "...")&#39; correctly, instead of converting the string to &#39;bool&#39; fo...
Definition: IRBuilder.h:1305
static bool promotedOpIsNSW(const Instruction &I)
FastMathFlags getFastMathFlags() const
Convenience function for getting all the fast-math flags.
Definition: Operator.h:358
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata *> MDs)
Definition: Metadata.h:1166
static bool runOnFunction(Function &F, bool PostInlining)
unsigned ComputeNumSignBits(const Value *Op, const DataLayout &DL, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true)
Return the number of times the sign bit of the register is replicated into the other bits...
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:410
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Value * CreateFMul(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Definition: IRBuilder.h:1195
Value * CreateAShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Definition: IRBuilder.h:1092
LLVM Basic Block Representation.
Definition: BasicBlock.h:59
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:46
CallInst * CreateIntrinsic(Intrinsic::ID ID, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with no operands.
Definition: IRBuilder.cpp:742
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This is an important base class in LLVM.
Definition: Constant.h:42
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
Definition: IRBuilder.h:1901
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:264
Value * CreateFPToUI(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1594
bool isFast() const
&#39;Fast&#39; means all bits are set.
Definition: Operator.h:207
Represent the analysis usage information of a pass.
Value * CreateNeg(Value *V, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1256
This instruction compares its operands according to the predicate given to the constructor.
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:285
Value * getPointerOperand()
Definition: Instructions.h:274
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1736
bool allowReciprocal() const
Definition: Operator.h:203
self_iterator getIterator()
Definition: ilist_node.h:82
Class to represent integer types.
Definition: DerivedTypes.h:40
ConstantInt * getInt64(uint64_t C)
Get a constant 64-bit value.
Definition: IRBuilder.h:312
IntegerType * getIntNTy(unsigned N)
Fetch the type representing an N-bit integer.
Definition: IRBuilder.h:360
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
Definition: IRBuilder.h:1921
The AMDGPU TargetMachine interface definition for hw codgen targets.
static bool shouldKeepFDivF32(Value *Num, bool UnsafeDiv, bool HasDenormals)
static Value * getMulHu(IRBuilder<> &Builder, Value *LHS, Value *RHS)
static UndefValue * get(Type *T)
Static factory methods - Return an &#39;undef&#39; object of the specified type.
Definition: Constants.cpp:1392
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE, "Assign register bank of generic virtual registers", false, false) RegBankSelect
#define DEBUG_TYPE
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition: Metadata.cpp:1226
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:996
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1552
Value * CreateUIToFP(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1602
Value * CreateFCmpOGE(Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:1786
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:51
Iterator for intrusive lists based on ilist_node.
This is the shared class of boolean and integer constants.
Definition: Constants.h:84
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type...
Definition: Type.cpp:130
Value * CreateICmpUGE(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1748
Utility class for floating point operations which can have information about relaxed accuracy require...
Definition: Operator.h:240
Value * CreateInsertElement(Value *Vec, Value *NewElt, Value *Idx, const Twine &Name="")
Definition: IRBuilder.h:1934
unsigned getABITypeAlignment(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
Definition: DataLayout.cpp:722
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Definition: IRBuilder.h:307
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:621
static Constant * get(Type *Ty, double V)
This returns a ConstantFP, or a vector containing a splat of a ConstantFP, for the specified value in...
Definition: Constants.cpp:684
static bool hasUnsafeFPMath(const Function &F)
The access may modify the value stored in memory.
static std::pair< Value *, Value * > getMul64(IRBuilder<> &Builder, Value *LHS, Value *RHS)
Class to represent vector types.
Definition: DerivedTypes.h:393
void setPreservesAll()
Set by analyses that do not transform their input at all.
uint64_t getTypeSizeInBits(Type *Ty) const
Size examples:
Definition: DataLayout.h:560
Predicate getPredicate() const
Return the predicate for this instruction.
Definition: InstrTypes.h:759
Address space for 32-bit constant memory.
Definition: AMDGPU.h:237
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:307
unsigned getAlignment() const
Return the alignment of the access that is being performed.
Definition: Instructions.h:230
unsigned getIntegerBitWidth() const
Definition: DerivedTypes.h:97
StringRef getValueAsString() const
Return the attribute&#39;s value as a string.
Definition: Attributes.cpp:195
float getFPAccuracy() const
Get the maximum error permitted by this operation in ULPs.
#define I(x, y, z)
Definition: MD5.cpp:58
#define N
Value * CreateFPToSI(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1598
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:323
Type * getFloatTy()
Fetch the type representing a 32-bit floating point value.
Definition: IRBuilder.h:370
bool hasNoUnsignedWrap() const
Determine whether the no unsigned wrap flag is set.
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1112
Value * CreateFDiv(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Definition: IRBuilder.h:1212
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Definition: Instructions.h:280
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
LLVM Value Representation.
Definition: Value.h:73
void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
static VectorType * get(Type *ElementType, unsigned NumElements)
This static method is the primary way to construct an VectorType.
Definition: Type.cpp:593
void setFastMathFlags(FastMathFlags NewFMF)
Set the fast-math flags to be used with generated fp-math operators.
Definition: IRBuilder.h:220
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition: Function.h:331
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Definition: IRBuilder.h:1072
bool isExactlyValue(const APFloat &V) const
We don&#39;t rely on operator== working on double values, as it returns true for things that are clearly ...
Definition: Constants.cpp:801
Convenience struct for specifying and reasoning about fast-math flags.
Definition: Operator.h:160
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:49
INITIALIZE_PASS_BEGIN(AMDGPUCodeGenPrepare, DEBUG_TYPE, "AMDGPU IR optimizations", false, false) INITIALIZE_PASS_END(AMDGPUCodeGenPrepare
const STC & getSubtarget(const Function &F) const
This method returns a pointer to the specified type of TargetSubtargetInfo.
Root of the metadata hierarchy.
Definition: Metadata.h:58
Statically lint checks LLVM IR
Definition: Lint.cpp:193
Address space for constant memory (VTX2)
Definition: AMDGPU.h:233
Value * CreateFNeg(Value *V, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:1274
bool isNullValue() const
Determine if all bits are clear.
Definition: APInt.h:406
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:44
const BasicBlock * getParent() const
Definition: Instruction.h:67
CallInst * CreateCall(Value *Callee, ArrayRef< Value *> Args=None, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:1871