LLVM  10.0.0svn
AMDGPULibCalls.cpp
Go to the documentation of this file.
1 //===- AMDGPULibCalls.cpp -------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// This file does AMD library function optimizations.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #define DEBUG_TYPE "amdgpu-simplifylib"
15 
16 #include "AMDGPU.h"
17 #include "AMDGPULibFunc.h"
18 #include "AMDGPUSubtarget.h"
20 #include "llvm/Analysis/Loads.h"
21 #include "llvm/ADT/StringSet.h"
22 #include "llvm/ADT/StringRef.h"
23 #include "llvm/IR/Constants.h"
24 #include "llvm/IR/DerivedTypes.h"
25 #include "llvm/IR/Instructions.h"
26 #include "llvm/IR/Intrinsics.h"
27 #include "llvm/IR/IRBuilder.h"
28 #include "llvm/IR/Function.h"
29 #include "llvm/IR/LLVMContext.h"
30 #include "llvm/IR/Module.h"
32 #include "llvm/Support/Debug.h"
37 #include <vector>
38 #include <cmath>
39 
40 using namespace llvm;
41 
42 static cl::opt<bool> EnablePreLink("amdgpu-prelink",
43  cl::desc("Enable pre-link mode optimizations"),
44  cl::init(false),
45  cl::Hidden);
46 
47 static cl::list<std::string> UseNative("amdgpu-use-native",
48  cl::desc("Comma separated list of functions to replace with native, or all"),
50  cl::Hidden);
51 
52 #define MATH_PI numbers::pi
53 #define MATH_E numbers::e
54 #define MATH_SQRT2 numbers::sqrt2
55 #define MATH_SQRT1_2 numbers::inv_sqrt2
56 
57 namespace llvm {
58 
60 private:
61 
63 
64  const TargetMachine *TM;
65 
66  // -fuse-native.
67  bool AllNative = false;
68 
69  bool useNativeFunc(const StringRef F) const;
70 
71  // Return a pointer (pointer expr) to the function if function defintion with
72  // "FuncName" exists. It may create a new function prototype in pre-link mode.
73  FunctionCallee getFunction(Module *M, const FuncInfo &fInfo);
74 
75  // Replace a normal function with its native version.
76  bool replaceWithNative(CallInst *CI, const FuncInfo &FInfo);
77 
78  bool parseFunctionName(const StringRef& FMangledName,
79  FuncInfo *FInfo=nullptr /*out*/);
80 
81  bool TDOFold(CallInst *CI, const FuncInfo &FInfo);
82 
83  /* Specialized optimizations */
84 
85  // recip (half or native)
86  bool fold_recip(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
87 
88  // divide (half or native)
89  bool fold_divide(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
90 
91  // pow/powr/pown
92  bool fold_pow(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
93 
94  // rootn
95  bool fold_rootn(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
96 
97  // fma/mad
98  bool fold_fma_mad(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
99 
100  // -fuse-native for sincos
101  bool sincosUseNative(CallInst *aCI, const FuncInfo &FInfo);
102 
103  // evaluate calls if calls' arguments are constants.
104  bool evaluateScalarMathFunc(FuncInfo &FInfo, double& Res0,
105  double& Res1, Constant *copr0, Constant *copr1, Constant *copr2);
106  bool evaluateCall(CallInst *aCI, FuncInfo &FInfo);
107 
108  // exp
109  bool fold_exp(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
110 
111  // exp2
112  bool fold_exp2(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
113 
114  // exp10
115  bool fold_exp10(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
116 
117  // log
118  bool fold_log(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
119 
120  // log2
121  bool fold_log2(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
122 
123  // log10
124  bool fold_log10(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
125 
126  // sqrt
127  bool fold_sqrt(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
128 
129  // sin/cos
130  bool fold_sincos(CallInst * CI, IRBuilder<> &B, AliasAnalysis * AA);
131 
132  // __read_pipe/__write_pipe
133  bool fold_read_write_pipe(CallInst *CI, IRBuilder<> &B, FuncInfo &FInfo);
134 
135  // llvm.amdgcn.wavefrontsize
136  bool fold_wavefrontsize(CallInst *CI, IRBuilder<> &B);
137 
138  // Get insertion point at entry.
139  BasicBlock::iterator getEntryIns(CallInst * UI);
140  // Insert an Alloc instruction.
141  AllocaInst* insertAlloca(CallInst * UI, IRBuilder<> &B, const char *prefix);
142  // Get a scalar native builtin signle argument FP function
143  FunctionCallee getNativeFunction(Module *M, const FuncInfo &FInfo);
144 
145 protected:
147 
148  bool isUnsafeMath(const CallInst *CI) const;
149 
150  void replaceCall(Value *With) {
151  CI->replaceAllUsesWith(With);
152  CI->eraseFromParent();
153  }
154 
155 public:
156  AMDGPULibCalls(const TargetMachine *TM_ = nullptr) : TM(TM_) {}
157 
158  bool fold(CallInst *CI, AliasAnalysis *AA = nullptr);
159 
160  void initNativeFuncs();
161 
162  // Replace a normal math function call with that native version
163  bool useNative(CallInst *CI);
164 };
165 
166 } // end llvm namespace
167 
168 namespace {
169 
170  class AMDGPUSimplifyLibCalls : public FunctionPass {
171 
172  const TargetOptions Options;
173 
174  AMDGPULibCalls Simplifier;
175 
176  public:
177  static char ID; // Pass identification
178 
179  AMDGPUSimplifyLibCalls(const TargetOptions &Opt = TargetOptions(),
180  const TargetMachine *TM = nullptr)
181  : FunctionPass(ID), Options(Opt), Simplifier(TM) {
183  }
184 
185  void getAnalysisUsage(AnalysisUsage &AU) const override {
187  }
188 
189  bool runOnFunction(Function &M) override;
190  };
191 
192  class AMDGPUUseNativeCalls : public FunctionPass {
193 
194  AMDGPULibCalls Simplifier;
195 
196  public:
197  static char ID; // Pass identification
198 
199  AMDGPUUseNativeCalls() : FunctionPass(ID) {
201  Simplifier.initNativeFuncs();
202  }
203 
204  bool runOnFunction(Function &F) override;
205  };
206 
207 } // end anonymous namespace.
208 
210 char AMDGPUUseNativeCalls::ID = 0;
211 
212 INITIALIZE_PASS_BEGIN(AMDGPUSimplifyLibCalls, "amdgpu-simplifylib",
213  "Simplify well-known AMD library calls", false, false)
215 INITIALIZE_PASS_END(AMDGPUSimplifyLibCalls, "amdgpu-simplifylib",
216  "Simplify well-known AMD library calls", false, false)
217 
218 INITIALIZE_PASS(AMDGPUUseNativeCalls, "amdgpu-usenative",
219  "Replace builtin math calls with that native versions.",
220  false, false)
221 
222 template <typename IRB>
223 static CallInst *CreateCallEx(IRB &B, FunctionCallee Callee, Value *Arg,
224  const Twine &Name = "") {
225  CallInst *R = B.CreateCall(Callee, Arg, Name);
226  if (Function *F = dyn_cast<Function>(Callee.getCallee()))
227  R->setCallingConv(F->getCallingConv());
228  return R;
229 }
230 
231 template <typename IRB>
232 static CallInst *CreateCallEx2(IRB &B, FunctionCallee Callee, Value *Arg1,
233  Value *Arg2, const Twine &Name = "") {
234  CallInst *R = B.CreateCall(Callee, {Arg1, Arg2}, Name);
235  if (Function *F = dyn_cast<Function>(Callee.getCallee()))
236  R->setCallingConv(F->getCallingConv());
237  return R;
238 }
239 
240 // Data structures for table-driven optimizations.
241 // FuncTbl works for both f32 and f64 functions with 1 input argument
242 
243 struct TableEntry {
244  double result;
245  double input;
246 };
247 
248 /* a list of {result, input} */
249 static const TableEntry tbl_acos[] = {
250  {MATH_PI / 2.0, 0.0},
251  {MATH_PI / 2.0, -0.0},
252  {0.0, 1.0},
253  {MATH_PI, -1.0}
254 };
255 static const TableEntry tbl_acosh[] = {
256  {0.0, 1.0}
257 };
258 static const TableEntry tbl_acospi[] = {
259  {0.5, 0.0},
260  {0.5, -0.0},
261  {0.0, 1.0},
262  {1.0, -1.0}
263 };
264 static const TableEntry tbl_asin[] = {
265  {0.0, 0.0},
266  {-0.0, -0.0},
267  {MATH_PI / 2.0, 1.0},
268  {-MATH_PI / 2.0, -1.0}
269 };
270 static const TableEntry tbl_asinh[] = {
271  {0.0, 0.0},
272  {-0.0, -0.0}
273 };
274 static const TableEntry tbl_asinpi[] = {
275  {0.0, 0.0},
276  {-0.0, -0.0},
277  {0.5, 1.0},
278  {-0.5, -1.0}
279 };
280 static const TableEntry tbl_atan[] = {
281  {0.0, 0.0},
282  {-0.0, -0.0},
283  {MATH_PI / 4.0, 1.0},
284  {-MATH_PI / 4.0, -1.0}
285 };
286 static const TableEntry tbl_atanh[] = {
287  {0.0, 0.0},
288  {-0.0, -0.0}
289 };
290 static const TableEntry tbl_atanpi[] = {
291  {0.0, 0.0},
292  {-0.0, -0.0},
293  {0.25, 1.0},
294  {-0.25, -1.0}
295 };
296 static const TableEntry tbl_cbrt[] = {
297  {0.0, 0.0},
298  {-0.0, -0.0},
299  {1.0, 1.0},
300  {-1.0, -1.0},
301 };
302 static const TableEntry tbl_cos[] = {
303  {1.0, 0.0},
304  {1.0, -0.0}
305 };
306 static const TableEntry tbl_cosh[] = {
307  {1.0, 0.0},
308  {1.0, -0.0}
309 };
310 static const TableEntry tbl_cospi[] = {
311  {1.0, 0.0},
312  {1.0, -0.0}
313 };
314 static const TableEntry tbl_erfc[] = {
315  {1.0, 0.0},
316  {1.0, -0.0}
317 };
318 static const TableEntry tbl_erf[] = {
319  {0.0, 0.0},
320  {-0.0, -0.0}
321 };
322 static const TableEntry tbl_exp[] = {
323  {1.0, 0.0},
324  {1.0, -0.0},
325  {MATH_E, 1.0}
326 };
327 static const TableEntry tbl_exp2[] = {
328  {1.0, 0.0},
329  {1.0, -0.0},
330  {2.0, 1.0}
331 };
332 static const TableEntry tbl_exp10[] = {
333  {1.0, 0.0},
334  {1.0, -0.0},
335  {10.0, 1.0}
336 };
337 static const TableEntry tbl_expm1[] = {
338  {0.0, 0.0},
339  {-0.0, -0.0}
340 };
341 static const TableEntry tbl_log[] = {
342  {0.0, 1.0},
343  {1.0, MATH_E}
344 };
345 static const TableEntry tbl_log2[] = {
346  {0.0, 1.0},
347  {1.0, 2.0}
348 };
349 static const TableEntry tbl_log10[] = {
350  {0.0, 1.0},
351  {1.0, 10.0}
352 };
353 static const TableEntry tbl_rsqrt[] = {
354  {1.0, 1.0},
355  {MATH_SQRT1_2, 2.0}
356 };
357 static const TableEntry tbl_sin[] = {
358  {0.0, 0.0},
359  {-0.0, -0.0}
360 };
361 static const TableEntry tbl_sinh[] = {
362  {0.0, 0.0},
363  {-0.0, -0.0}
364 };
365 static const TableEntry tbl_sinpi[] = {
366  {0.0, 0.0},
367  {-0.0, -0.0}
368 };
369 static const TableEntry tbl_sqrt[] = {
370  {0.0, 0.0},
371  {1.0, 1.0},
372  {MATH_SQRT2, 2.0}
373 };
374 static const TableEntry tbl_tan[] = {
375  {0.0, 0.0},
376  {-0.0, -0.0}
377 };
378 static const TableEntry tbl_tanh[] = {
379  {0.0, 0.0},
380  {-0.0, -0.0}
381 };
382 static const TableEntry tbl_tanpi[] = {
383  {0.0, 0.0},
384  {-0.0, -0.0}
385 };
386 static const TableEntry tbl_tgamma[] = {
387  {1.0, 1.0},
388  {1.0, 2.0},
389  {2.0, 3.0},
390  {6.0, 4.0}
391 };
392 
394  switch(id) {
410  return true;
411  default:;
412  }
413  return false;
414 }
415 
416 struct TableRef {
417  size_t size;
418  const TableEntry *table; // variable size: from 0 to (size - 1)
419 
420  TableRef() : size(0), table(nullptr) {}
421 
422  template <size_t N>
423  TableRef(const TableEntry (&tbl)[N]) : size(N), table(&tbl[0]) {}
424 };
425 
427  switch(id) {
428  case AMDGPULibFunc::EI_ACOS: return TableRef(tbl_acos);
429  case AMDGPULibFunc::EI_ACOSH: return TableRef(tbl_acosh);
430  case AMDGPULibFunc::EI_ACOSPI: return TableRef(tbl_acospi);
431  case AMDGPULibFunc::EI_ASIN: return TableRef(tbl_asin);
432  case AMDGPULibFunc::EI_ASINH: return TableRef(tbl_asinh);
433  case AMDGPULibFunc::EI_ASINPI: return TableRef(tbl_asinpi);
434  case AMDGPULibFunc::EI_ATAN: return TableRef(tbl_atan);
435  case AMDGPULibFunc::EI_ATANH: return TableRef(tbl_atanh);
436  case AMDGPULibFunc::EI_ATANPI: return TableRef(tbl_atanpi);
437  case AMDGPULibFunc::EI_CBRT: return TableRef(tbl_cbrt);
439  case AMDGPULibFunc::EI_COS: return TableRef(tbl_cos);
440  case AMDGPULibFunc::EI_COSH: return TableRef(tbl_cosh);
441  case AMDGPULibFunc::EI_COSPI: return TableRef(tbl_cospi);
442  case AMDGPULibFunc::EI_ERFC: return TableRef(tbl_erfc);
443  case AMDGPULibFunc::EI_ERF: return TableRef(tbl_erf);
444  case AMDGPULibFunc::EI_EXP: return TableRef(tbl_exp);
446  case AMDGPULibFunc::EI_EXP2: return TableRef(tbl_exp2);
447  case AMDGPULibFunc::EI_EXP10: return TableRef(tbl_exp10);
448  case AMDGPULibFunc::EI_EXPM1: return TableRef(tbl_expm1);
449  case AMDGPULibFunc::EI_LOG: return TableRef(tbl_log);
451  case AMDGPULibFunc::EI_LOG2: return TableRef(tbl_log2);
452  case AMDGPULibFunc::EI_LOG10: return TableRef(tbl_log10);
454  case AMDGPULibFunc::EI_RSQRT: return TableRef(tbl_rsqrt);
456  case AMDGPULibFunc::EI_SIN: return TableRef(tbl_sin);
457  case AMDGPULibFunc::EI_SINH: return TableRef(tbl_sinh);
458  case AMDGPULibFunc::EI_SINPI: return TableRef(tbl_sinpi);
460  case AMDGPULibFunc::EI_SQRT: return TableRef(tbl_sqrt);
461  case AMDGPULibFunc::EI_TAN: return TableRef(tbl_tan);
462  case AMDGPULibFunc::EI_TANH: return TableRef(tbl_tanh);
463  case AMDGPULibFunc::EI_TANPI: return TableRef(tbl_tanpi);
464  case AMDGPULibFunc::EI_TGAMMA: return TableRef(tbl_tgamma);
465  default:;
466  }
467  return TableRef();
468 }
469 
470 static inline int getVecSize(const AMDGPULibFunc& FInfo) {
471  return FInfo.getLeads()[0].VectorSize;
472 }
473 
474 static inline AMDGPULibFunc::EType getArgType(const AMDGPULibFunc& FInfo) {
475  return (AMDGPULibFunc::EType)FInfo.getLeads()[0].ArgType;
476 }
477 
478 FunctionCallee AMDGPULibCalls::getFunction(Module *M, const FuncInfo &fInfo) {
479  // If we are doing PreLinkOpt, the function is external. So it is safe to
480  // use getOrInsertFunction() at this stage.
481 
483  : AMDGPULibFunc::getFunction(M, fInfo);
484 }
485 
486 bool AMDGPULibCalls::parseFunctionName(const StringRef& FMangledName,
487  FuncInfo *FInfo) {
488  return AMDGPULibFunc::parse(FMangledName, *FInfo);
489 }
490 
492  if (auto Op = dyn_cast<FPMathOperator>(CI))
493  if (Op->isFast())
494  return true;
495  const Function *F = CI->getParent()->getParent();
496  Attribute Attr = F->getFnAttribute("unsafe-fp-math");
497  return Attr.getValueAsString() == "true";
498 }
499 
500 bool AMDGPULibCalls::useNativeFunc(const StringRef F) const {
501  return AllNative ||
502  std::find(UseNative.begin(), UseNative.end(), F) != UseNative.end();
503 }
504 
506  AllNative = useNativeFunc("all") ||
507  (UseNative.getNumOccurrences() && UseNative.size() == 1 &&
508  UseNative.begin()->empty());
509 }
510 
511 bool AMDGPULibCalls::sincosUseNative(CallInst *aCI, const FuncInfo &FInfo) {
512  bool native_sin = useNativeFunc("sin");
513  bool native_cos = useNativeFunc("cos");
514 
515  if (native_sin && native_cos) {
516  Module *M = aCI->getModule();
517  Value *opr0 = aCI->getArgOperand(0);
518 
519  AMDGPULibFunc nf;
520  nf.getLeads()[0].ArgType = FInfo.getLeads()[0].ArgType;
521  nf.getLeads()[0].VectorSize = FInfo.getLeads()[0].VectorSize;
522 
525  FunctionCallee sinExpr = getFunction(M, nf);
526 
529  FunctionCallee cosExpr = getFunction(M, nf);
530  if (sinExpr && cosExpr) {
531  Value *sinval = CallInst::Create(sinExpr, opr0, "splitsin", aCI);
532  Value *cosval = CallInst::Create(cosExpr, opr0, "splitcos", aCI);
533  new StoreInst(cosval, aCI->getArgOperand(1), aCI);
534 
535  DEBUG_WITH_TYPE("usenative", dbgs() << "<useNative> replace " << *aCI
536  << " with native version of sin/cos");
537 
538  replaceCall(sinval);
539  return true;
540  }
541  }
542  return false;
543 }
544 
546  CI = aCI;
547  Function *Callee = aCI->getCalledFunction();
548 
549  FuncInfo FInfo;
550  if (!parseFunctionName(Callee->getName(), &FInfo) || !FInfo.isMangled() ||
551  FInfo.getPrefix() != AMDGPULibFunc::NOPFX ||
552  getArgType(FInfo) == AMDGPULibFunc::F64 || !HasNative(FInfo.getId()) ||
553  !(AllNative || useNativeFunc(FInfo.getName()))) {
554  return false;
555  }
556 
557  if (FInfo.getId() == AMDGPULibFunc::EI_SINCOS)
558  return sincosUseNative(aCI, FInfo);
559 
561  FunctionCallee F = getFunction(aCI->getModule(), FInfo);
562  if (!F)
563  return false;
564 
565  aCI->setCalledFunction(F);
566  DEBUG_WITH_TYPE("usenative", dbgs() << "<useNative> replace " << *aCI
567  << " with native version");
568  return true;
569 }
570 
571 // Clang emits call of __read_pipe_2 or __read_pipe_4 for OpenCL read_pipe
572 // builtin, with appended type size and alignment arguments, where 2 or 4
573 // indicates the original number of arguments. The library has optimized version
574 // of __read_pipe_2/__read_pipe_4 when the type size and alignment has the same
575 // power of 2 value. This function transforms __read_pipe_2 to __read_pipe_2_N
576 // for such cases where N is the size in bytes of the type (N = 1, 2, 4, 8, ...,
577 // 128). The same for __read_pipe_4, write_pipe_2, and write_pipe_4.
578 bool AMDGPULibCalls::fold_read_write_pipe(CallInst *CI, IRBuilder<> &B,
579  FuncInfo &FInfo) {
580  auto *Callee = CI->getCalledFunction();
581  if (!Callee->isDeclaration())
582  return false;
583 
584  assert(Callee->hasName() && "Invalid read_pipe/write_pipe function");
585  auto *M = Callee->getParent();
586  auto &Ctx = M->getContext();
587  std::string Name = Callee->getName();
588  auto NumArg = CI->getNumArgOperands();
589  if (NumArg != 4 && NumArg != 6)
590  return false;
591  auto *PacketSize = CI->getArgOperand(NumArg - 2);
592  auto *PacketAlign = CI->getArgOperand(NumArg - 1);
593  if (!isa<ConstantInt>(PacketSize) || !isa<ConstantInt>(PacketAlign))
594  return false;
595  unsigned Size = cast<ConstantInt>(PacketSize)->getZExtValue();
596  unsigned Align = cast<ConstantInt>(PacketAlign)->getZExtValue();
597  if (Size != Align || !isPowerOf2_32(Size))
598  return false;
599 
600  Type *PtrElemTy;
601  if (Size <= 8)
602  PtrElemTy = Type::getIntNTy(Ctx, Size * 8);
603  else
604  PtrElemTy = VectorType::get(Type::getInt64Ty(Ctx), Size / 8);
605  unsigned PtrArgLoc = CI->getNumArgOperands() - 3;
606  auto PtrArg = CI->getArgOperand(PtrArgLoc);
607  unsigned PtrArgAS = PtrArg->getType()->getPointerAddressSpace();
608  auto *PtrTy = llvm::PointerType::get(PtrElemTy, PtrArgAS);
609 
611  for (unsigned I = 0; I != PtrArgLoc; ++I)
612  ArgTys.push_back(CI->getArgOperand(I)->getType());
613  ArgTys.push_back(PtrTy);
614 
615  Name = Name + "_" + std::to_string(Size);
616  auto *FTy = FunctionType::get(Callee->getReturnType(),
617  ArrayRef<Type *>(ArgTys), false);
618  AMDGPULibFunc NewLibFunc(Name, FTy);
620  if (!F)
621  return false;
622 
623  auto *BCast = B.CreatePointerCast(PtrArg, PtrTy);
625  for (unsigned I = 0; I != PtrArgLoc; ++I)
626  Args.push_back(CI->getArgOperand(I));
627  Args.push_back(BCast);
628 
629  auto *NCI = B.CreateCall(F, Args);
630  NCI->setAttributes(CI->getAttributes());
631  CI->replaceAllUsesWith(NCI);
632  CI->dropAllReferences();
633  CI->eraseFromParent();
634 
635  return true;
636 }
637 
638 // This function returns false if no change; return true otherwise.
640  this->CI = CI;
641  Function *Callee = CI->getCalledFunction();
642 
643  // Ignore indirect calls.
644  if (Callee == 0) return false;
645 
646  BasicBlock *BB = CI->getParent();
648  IRBuilder<> B(Context);
649 
650  // Set the builder to the instruction after the call.
651  B.SetInsertPoint(BB, CI->getIterator());
652 
653  // Copy fast flags from the original call.
654  if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(CI))
655  B.setFastMathFlags(FPOp->getFastMathFlags());
656 
657  switch (Callee->getIntrinsicID()) {
658  default:
659  break;
660  case Intrinsic::amdgcn_wavefrontsize:
661  return !EnablePreLink && fold_wavefrontsize(CI, B);
662  }
663 
664  FuncInfo FInfo;
665  if (!parseFunctionName(Callee->getName(), &FInfo))
666  return false;
667 
668  // Further check the number of arguments to see if they match.
669  if (CI->getNumArgOperands() != FInfo.getNumArgs())
670  return false;
671 
672  if (TDOFold(CI, FInfo))
673  return true;
674 
675  // Under unsafe-math, evaluate calls if possible.
676  // According to Brian Sumner, we can do this for all f32 function calls
677  // using host's double function calls.
678  if (isUnsafeMath(CI) && evaluateCall(CI, FInfo))
679  return true;
680 
681  // Specilized optimizations for each function call
682  switch (FInfo.getId()) {
684  // skip vector function
685  assert ((FInfo.getPrefix() == AMDGPULibFunc::NATIVE ||
686  FInfo.getPrefix() == AMDGPULibFunc::HALF) &&
687  "recip must be an either native or half function");
688  return (getVecSize(FInfo) != 1) ? false : fold_recip(CI, B, FInfo);
689 
691  // skip vector function
692  assert ((FInfo.getPrefix() == AMDGPULibFunc::NATIVE ||
693  FInfo.getPrefix() == AMDGPULibFunc::HALF) &&
694  "divide must be an either native or half function");
695  return (getVecSize(FInfo) != 1) ? false : fold_divide(CI, B, FInfo);
696 
700  return fold_pow(CI, B, FInfo);
701 
703  // skip vector function
704  return (getVecSize(FInfo) != 1) ? false : fold_rootn(CI, B, FInfo);
705 
709  // skip vector function
710  return (getVecSize(FInfo) != 1) ? false : fold_fma_mad(CI, B, FInfo);
711 
713  return isUnsafeMath(CI) && fold_sqrt(CI, B, FInfo);
716  if ((getArgType(FInfo) == AMDGPULibFunc::F32 ||
717  getArgType(FInfo) == AMDGPULibFunc::F64)
718  && (FInfo.getPrefix() == AMDGPULibFunc::NOPFX))
719  return fold_sincos(CI, B, AA);
720 
721  break;
726  return fold_read_write_pipe(CI, B, FInfo);
727 
728  default:
729  break;
730  }
731 
732  return false;
733 }
734 
735 bool AMDGPULibCalls::TDOFold(CallInst *CI, const FuncInfo &FInfo) {
736  // Table-Driven optimization
737  const TableRef tr = getOptTable(FInfo.getId());
738  if (tr.size==0)
739  return false;
740 
741  int const sz = (int)tr.size;
742  const TableEntry * const ftbl = tr.table;
743  Value *opr0 = CI->getArgOperand(0);
744 
745  if (getVecSize(FInfo) > 1) {
746  if (ConstantDataVector *CV = dyn_cast<ConstantDataVector>(opr0)) {
748  for (int eltNo = 0; eltNo < getVecSize(FInfo); ++eltNo) {
749  ConstantFP *eltval = dyn_cast<ConstantFP>(
750  CV->getElementAsConstant((unsigned)eltNo));
751  assert(eltval && "Non-FP arguments in math function!");
752  bool found = false;
753  for (int i=0; i < sz; ++i) {
754  if (eltval->isExactlyValue(ftbl[i].input)) {
755  DVal.push_back(ftbl[i].result);
756  found = true;
757  break;
758  }
759  }
760  if (!found) {
761  // This vector constants not handled yet.
762  return false;
763  }
764  }
765  LLVMContext &context = CI->getParent()->getParent()->getContext();
766  Constant *nval;
767  if (getArgType(FInfo) == AMDGPULibFunc::F32) {
769  for (unsigned i = 0; i < DVal.size(); ++i) {
770  FVal.push_back((float)DVal[i]);
771  }
772  ArrayRef<float> tmp(FVal);
773  nval = ConstantDataVector::get(context, tmp);
774  } else { // F64
775  ArrayRef<double> tmp(DVal);
776  nval = ConstantDataVector::get(context, tmp);
777  }
778  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n");
779  replaceCall(nval);
780  return true;
781  }
782  } else {
783  // Scalar version
784  if (ConstantFP *CF = dyn_cast<ConstantFP>(opr0)) {
785  for (int i = 0; i < sz; ++i) {
786  if (CF->isExactlyValue(ftbl[i].input)) {
787  Value *nval = ConstantFP::get(CF->getType(), ftbl[i].result);
788  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n");
789  replaceCall(nval);
790  return true;
791  }
792  }
793  }
794  }
795 
796  return false;
797 }
798 
799 bool AMDGPULibCalls::replaceWithNative(CallInst *CI, const FuncInfo &FInfo) {
800  Module *M = CI->getModule();
801  if (getArgType(FInfo) != AMDGPULibFunc::F32 ||
802  FInfo.getPrefix() != AMDGPULibFunc::NOPFX ||
803  !HasNative(FInfo.getId()))
804  return false;
805 
806  AMDGPULibFunc nf = FInfo;
808  if (FunctionCallee FPExpr = getFunction(M, nf)) {
809  LLVM_DEBUG(dbgs() << "AMDIC: " << *CI << " ---> ");
810 
811  CI->setCalledFunction(FPExpr);
812 
813  LLVM_DEBUG(dbgs() << *CI << '\n');
814 
815  return true;
816  }
817  return false;
818 }
819 
820 // [native_]half_recip(c) ==> 1.0/c
821 bool AMDGPULibCalls::fold_recip(CallInst *CI, IRBuilder<> &B,
822  const FuncInfo &FInfo) {
823  Value *opr0 = CI->getArgOperand(0);
824  if (ConstantFP *CF = dyn_cast<ConstantFP>(opr0)) {
825  // Just create a normal div. Later, InstCombine will be able
826  // to compute the divide into a constant (avoid check float infinity
827  // or subnormal at this point).
828  Value *nval = B.CreateFDiv(ConstantFP::get(CF->getType(), 1.0),
829  opr0,
830  "recip2div");
831  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n");
832  replaceCall(nval);
833  return true;
834  }
835  return false;
836 }
837 
838 // [native_]half_divide(x, c) ==> x/c
839 bool AMDGPULibCalls::fold_divide(CallInst *CI, IRBuilder<> &B,
840  const FuncInfo &FInfo) {
841  Value *opr0 = CI->getArgOperand(0);
842  Value *opr1 = CI->getArgOperand(1);
843  ConstantFP *CF0 = dyn_cast<ConstantFP>(opr0);
844  ConstantFP *CF1 = dyn_cast<ConstantFP>(opr1);
845 
846  if ((CF0 && CF1) || // both are constants
847  (CF1 && (getArgType(FInfo) == AMDGPULibFunc::F32)))
848  // CF1 is constant && f32 divide
849  {
850  Value *nval1 = B.CreateFDiv(ConstantFP::get(opr1->getType(), 1.0),
851  opr1, "__div2recip");
852  Value *nval = B.CreateFMul(opr0, nval1, "__div2mul");
853  replaceCall(nval);
854  return true;
855  }
856  return false;
857 }
858 
859 namespace llvm {
860 static double log2(double V) {
861 #if _XOPEN_SOURCE >= 600 || defined(_ISOC99_SOURCE) || _POSIX_C_SOURCE >= 200112L
862  return ::log2(V);
863 #else
864  return log(V) / numbers::ln2;
865 #endif
866 }
867 }
868 
869 bool AMDGPULibCalls::fold_pow(CallInst *CI, IRBuilder<> &B,
870  const FuncInfo &FInfo) {
871  assert((FInfo.getId() == AMDGPULibFunc::EI_POW ||
872  FInfo.getId() == AMDGPULibFunc::EI_POWR ||
873  FInfo.getId() == AMDGPULibFunc::EI_POWN) &&
874  "fold_pow: encounter a wrong function call");
875 
876  Value *opr0, *opr1;
877  ConstantFP *CF;
878  ConstantInt *CINT;
879  ConstantAggregateZero *CZero;
880  Type *eltType;
881 
882  opr0 = CI->getArgOperand(0);
883  opr1 = CI->getArgOperand(1);
884  CZero = dyn_cast<ConstantAggregateZero>(opr1);
885  if (getVecSize(FInfo) == 1) {
886  eltType = opr0->getType();
887  CF = dyn_cast<ConstantFP>(opr1);
888  CINT = dyn_cast<ConstantInt>(opr1);
889  } else {
890  VectorType *VTy = dyn_cast<VectorType>(opr0->getType());
891  assert(VTy && "Oprand of vector function should be of vectortype");
892  eltType = VTy->getElementType();
894 
895  // Now, only Handle vector const whose elements have the same value.
896  CF = CDV ? dyn_cast_or_null<ConstantFP>(CDV->getSplatValue()) : nullptr;
897  CINT = CDV ? dyn_cast_or_null<ConstantInt>(CDV->getSplatValue()) : nullptr;
898  }
899 
900  // No unsafe math , no constant argument, do nothing
901  if (!isUnsafeMath(CI) && !CF && !CINT && !CZero)
902  return false;
903 
904  // 0x1111111 means that we don't do anything for this call.
905  int ci_opr1 = (CINT ? (int)CINT->getSExtValue() : 0x1111111);
906 
907  if ((CF && CF->isZero()) || (CINT && ci_opr1 == 0) || CZero) {
908  // pow/powr/pown(x, 0) == 1
909  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> 1\n");
910  Constant *cnval = ConstantFP::get(eltType, 1.0);
911  if (getVecSize(FInfo) > 1) {
912  cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval);
913  }
914  replaceCall(cnval);
915  return true;
916  }
917  if ((CF && CF->isExactlyValue(1.0)) || (CINT && ci_opr1 == 1)) {
918  // pow/powr/pown(x, 1.0) = x
919  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << "\n");
920  replaceCall(opr0);
921  return true;
922  }
923  if ((CF && CF->isExactlyValue(2.0)) || (CINT && ci_opr1 == 2)) {
924  // pow/powr/pown(x, 2.0) = x*x
925  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << " * " << *opr0
926  << "\n");
927  Value *nval = B.CreateFMul(opr0, opr0, "__pow2");
928  replaceCall(nval);
929  return true;
930  }
931  if ((CF && CF->isExactlyValue(-1.0)) || (CINT && ci_opr1 == -1)) {
932  // pow/powr/pown(x, -1.0) = 1.0/x
933  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> 1 / " << *opr0 << "\n");
934  Constant *cnval = ConstantFP::get(eltType, 1.0);
935  if (getVecSize(FInfo) > 1) {
936  cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval);
937  }
938  Value *nval = B.CreateFDiv(cnval, opr0, "__powrecip");
939  replaceCall(nval);
940  return true;
941  }
942 
943  Module *M = CI->getModule();
944  if (CF && (CF->isExactlyValue(0.5) || CF->isExactlyValue(-0.5))) {
945  // pow[r](x, [-]0.5) = sqrt(x)
946  bool issqrt = CF->isExactlyValue(0.5);
947  if (FunctionCallee FPExpr =
948  getFunction(M, AMDGPULibFunc(issqrt ? AMDGPULibFunc::EI_SQRT
950  FInfo))) {
951  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> "
952  << FInfo.getName().c_str() << "(" << *opr0 << ")\n");
953  Value *nval = CreateCallEx(B,FPExpr, opr0, issqrt ? "__pow2sqrt"
954  : "__pow2rsqrt");
955  replaceCall(nval);
956  return true;
957  }
958  }
959 
960  if (!isUnsafeMath(CI))
961  return false;
962 
963  // Unsafe Math optimization
964 
965  // Remember that ci_opr1 is set if opr1 is integral
966  if (CF) {
967  double dval = (getArgType(FInfo) == AMDGPULibFunc::F32)
968  ? (double)CF->getValueAPF().convertToFloat()
969  : CF->getValueAPF().convertToDouble();
970  int ival = (int)dval;
971  if ((double)ival == dval) {
972  ci_opr1 = ival;
973  } else
974  ci_opr1 = 0x11111111;
975  }
976 
977  // pow/powr/pown(x, c) = [1/](x*x*..x); where
978  // trunc(c) == c && the number of x == c && |c| <= 12
979  unsigned abs_opr1 = (ci_opr1 < 0) ? -ci_opr1 : ci_opr1;
980  if (abs_opr1 <= 12) {
981  Constant *cnval;
982  Value *nval;
983  if (abs_opr1 == 0) {
984  cnval = ConstantFP::get(eltType, 1.0);
985  if (getVecSize(FInfo) > 1) {
986  cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval);
987  }
988  nval = cnval;
989  } else {
990  Value *valx2 = nullptr;
991  nval = nullptr;
992  while (abs_opr1 > 0) {
993  valx2 = valx2 ? B.CreateFMul(valx2, valx2, "__powx2") : opr0;
994  if (abs_opr1 & 1) {
995  nval = nval ? B.CreateFMul(nval, valx2, "__powprod") : valx2;
996  }
997  abs_opr1 >>= 1;
998  }
999  }
1000 
1001  if (ci_opr1 < 0) {
1002  cnval = ConstantFP::get(eltType, 1.0);
1003  if (getVecSize(FInfo) > 1) {
1004  cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval);
1005  }
1006  nval = B.CreateFDiv(cnval, nval, "__1powprod");
1007  }
1008  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> "
1009  << ((ci_opr1 < 0) ? "1/prod(" : "prod(") << *opr0
1010  << ")\n");
1011  replaceCall(nval);
1012  return true;
1013  }
1014 
1015  // powr ---> exp2(y * log2(x))
1016  // pown/pow ---> powr(fabs(x), y) | (x & ((int)y << 31))
1017  FunctionCallee ExpExpr =
1018  getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_EXP2, FInfo));
1019  if (!ExpExpr)
1020  return false;
1021 
1022  bool needlog = false;
1023  bool needabs = false;
1024  bool needcopysign = false;
1025  Constant *cnval = nullptr;
1026  if (getVecSize(FInfo) == 1) {
1027  CF = dyn_cast<ConstantFP>(opr0);
1028 
1029  if (CF) {
1030  double V = (getArgType(FInfo) == AMDGPULibFunc::F32)
1031  ? (double)CF->getValueAPF().convertToFloat()
1032  : CF->getValueAPF().convertToDouble();
1033 
1034  V = log2(std::abs(V));
1035  cnval = ConstantFP::get(eltType, V);
1036  needcopysign = (FInfo.getId() != AMDGPULibFunc::EI_POWR) &&
1037  CF->isNegative();
1038  } else {
1039  needlog = true;
1040  needcopysign = needabs = FInfo.getId() != AMDGPULibFunc::EI_POWR &&
1041  (!CF || CF->isNegative());
1042  }
1043  } else {
1045 
1046  if (!CDV) {
1047  needlog = true;
1048  needcopysign = needabs = FInfo.getId() != AMDGPULibFunc::EI_POWR;
1049  } else {
1050  assert ((int)CDV->getNumElements() == getVecSize(FInfo) &&
1051  "Wrong vector size detected");
1052 
1054  for (int i=0; i < getVecSize(FInfo); ++i) {
1055  double V = (getArgType(FInfo) == AMDGPULibFunc::F32)
1056  ? (double)CDV->getElementAsFloat(i)
1057  : CDV->getElementAsDouble(i);
1058  if (V < 0.0) needcopysign = true;
1059  V = log2(std::abs(V));
1060  DVal.push_back(V);
1061  }
1062  if (getArgType(FInfo) == AMDGPULibFunc::F32) {
1063  SmallVector<float, 0> FVal;
1064  for (unsigned i=0; i < DVal.size(); ++i) {
1065  FVal.push_back((float)DVal[i]);
1066  }
1067  ArrayRef<float> tmp(FVal);
1068  cnval = ConstantDataVector::get(M->getContext(), tmp);
1069  } else {
1070  ArrayRef<double> tmp(DVal);
1071  cnval = ConstantDataVector::get(M->getContext(), tmp);
1072  }
1073  }
1074  }
1075 
1076  if (needcopysign && (FInfo.getId() == AMDGPULibFunc::EI_POW)) {
1077  // We cannot handle corner cases for a general pow() function, give up
1078  // unless y is a constant integral value. Then proceed as if it were pown.
1079  if (getVecSize(FInfo) == 1) {
1080  if (const ConstantFP *CF = dyn_cast<ConstantFP>(opr1)) {
1081  double y = (getArgType(FInfo) == AMDGPULibFunc::F32)
1082  ? (double)CF->getValueAPF().convertToFloat()
1083  : CF->getValueAPF().convertToDouble();
1084  if (y != (double)(int64_t)y)
1085  return false;
1086  } else
1087  return false;
1088  } else {
1089  if (const ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(opr1)) {
1090  for (int i=0; i < getVecSize(FInfo); ++i) {
1091  double y = (getArgType(FInfo) == AMDGPULibFunc::F32)
1092  ? (double)CDV->getElementAsFloat(i)
1093  : CDV->getElementAsDouble(i);
1094  if (y != (double)(int64_t)y)
1095  return false;
1096  }
1097  } else
1098  return false;
1099  }
1100  }
1101 
1102  Value *nval;
1103  if (needabs) {
1104  FunctionCallee AbsExpr =
1105  getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_FABS, FInfo));
1106  if (!AbsExpr)
1107  return false;
1108  nval = CreateCallEx(B, AbsExpr, opr0, "__fabs");
1109  } else {
1110  nval = cnval ? cnval : opr0;
1111  }
1112  if (needlog) {
1113  FunctionCallee LogExpr =
1114  getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_LOG2, FInfo));
1115  if (!LogExpr)
1116  return false;
1117  nval = CreateCallEx(B,LogExpr, nval, "__log2");
1118  }
1119 
1120  if (FInfo.getId() == AMDGPULibFunc::EI_POWN) {
1121  // convert int(32) to fp(f32 or f64)
1122  opr1 = B.CreateSIToFP(opr1, nval->getType(), "pownI2F");
1123  }
1124  nval = B.CreateFMul(opr1, nval, "__ylogx");
1125  nval = CreateCallEx(B,ExpExpr, nval, "__exp2");
1126 
1127  if (needcopysign) {
1128  Value *opr_n;
1129  Type* rTy = opr0->getType();
1130  Type* nTyS = eltType->isDoubleTy() ? B.getInt64Ty() : B.getInt32Ty();
1131  Type *nTy = nTyS;
1132  if (const VectorType *vTy = dyn_cast<VectorType>(rTy))
1133  nTy = VectorType::get(nTyS, vTy->getNumElements());
1134  unsigned size = nTy->getScalarSizeInBits();
1135  opr_n = CI->getArgOperand(1);
1136  if (opr_n->getType()->isIntegerTy())
1137  opr_n = B.CreateZExtOrBitCast(opr_n, nTy, "__ytou");
1138  else
1139  opr_n = B.CreateFPToSI(opr1, nTy, "__ytou");
1140 
1141  Value *sign = B.CreateShl(opr_n, size-1, "__yeven");
1142  sign = B.CreateAnd(B.CreateBitCast(opr0, nTy), sign, "__pow_sign");
1143  nval = B.CreateOr(B.CreateBitCast(nval, nTy), sign);
1144  nval = B.CreateBitCast(nval, opr0->getType());
1145  }
1146 
1147  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> "
1148  << "exp2(" << *opr1 << " * log2(" << *opr0 << "))\n");
1149  replaceCall(nval);
1150 
1151  return true;
1152 }
1153 
1154 bool AMDGPULibCalls::fold_rootn(CallInst *CI, IRBuilder<> &B,
1155  const FuncInfo &FInfo) {
1156  Value *opr0 = CI->getArgOperand(0);
1157  Value *opr1 = CI->getArgOperand(1);
1158 
1159  ConstantInt *CINT = dyn_cast<ConstantInt>(opr1);
1160  if (!CINT) {
1161  return false;
1162  }
1163  int ci_opr1 = (int)CINT->getSExtValue();
1164  if (ci_opr1 == 1) { // rootn(x, 1) = x
1165  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << "\n");
1166  replaceCall(opr0);
1167  return true;
1168  }
1169  if (ci_opr1 == 2) { // rootn(x, 2) = sqrt(x)
1170  std::vector<const Type*> ParamsTys;
1171  ParamsTys.push_back(opr0->getType());
1172  Module *M = CI->getModule();
1173  if (FunctionCallee FPExpr =
1174  getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_SQRT, FInfo))) {
1175  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> sqrt(" << *opr0 << ")\n");
1176  Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2sqrt");
1177  replaceCall(nval);
1178  return true;
1179  }
1180  } else if (ci_opr1 == 3) { // rootn(x, 3) = cbrt(x)
1181  Module *M = CI->getModule();
1182  if (FunctionCallee FPExpr =
1183  getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_CBRT, FInfo))) {
1184  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> cbrt(" << *opr0 << ")\n");
1185  Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2cbrt");
1186  replaceCall(nval);
1187  return true;
1188  }
1189  } else if (ci_opr1 == -1) { // rootn(x, -1) = 1.0/x
1190  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> 1.0 / " << *opr0 << "\n");
1191  Value *nval = B.CreateFDiv(ConstantFP::get(opr0->getType(), 1.0),
1192  opr0,
1193  "__rootn2div");
1194  replaceCall(nval);
1195  return true;
1196  } else if (ci_opr1 == -2) { // rootn(x, -2) = rsqrt(x)
1197  std::vector<const Type*> ParamsTys;
1198  ParamsTys.push_back(opr0->getType());
1199  Module *M = CI->getModule();
1200  if (FunctionCallee FPExpr =
1201  getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_RSQRT, FInfo))) {
1202  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> rsqrt(" << *opr0
1203  << ")\n");
1204  Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2rsqrt");
1205  replaceCall(nval);
1206  return true;
1207  }
1208  }
1209  return false;
1210 }
1211 
1212 bool AMDGPULibCalls::fold_fma_mad(CallInst *CI, IRBuilder<> &B,
1213  const FuncInfo &FInfo) {
1214  Value *opr0 = CI->getArgOperand(0);
1215  Value *opr1 = CI->getArgOperand(1);
1216  Value *opr2 = CI->getArgOperand(2);
1217 
1218  ConstantFP *CF0 = dyn_cast<ConstantFP>(opr0);
1219  ConstantFP *CF1 = dyn_cast<ConstantFP>(opr1);
1220  if ((CF0 && CF0->isZero()) || (CF1 && CF1->isZero())) {
1221  // fma/mad(a, b, c) = c if a=0 || b=0
1222  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr2 << "\n");
1223  replaceCall(opr2);
1224  return true;
1225  }
1226  if (CF0 && CF0->isExactlyValue(1.0f)) {
1227  // fma/mad(a, b, c) = b+c if a=1
1228  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr1 << " + " << *opr2
1229  << "\n");
1230  Value *nval = B.CreateFAdd(opr1, opr2, "fmaadd");
1231  replaceCall(nval);
1232  return true;
1233  }
1234  if (CF1 && CF1->isExactlyValue(1.0f)) {
1235  // fma/mad(a, b, c) = a+c if b=1
1236  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << " + " << *opr2
1237  << "\n");
1238  Value *nval = B.CreateFAdd(opr0, opr2, "fmaadd");
1239  replaceCall(nval);
1240  return true;
1241  }
1242  if (ConstantFP *CF = dyn_cast<ConstantFP>(opr2)) {
1243  if (CF->isZero()) {
1244  // fma/mad(a, b, c) = a*b if c=0
1245  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << " * "
1246  << *opr1 << "\n");
1247  Value *nval = B.CreateFMul(opr0, opr1, "fmamul");
1248  replaceCall(nval);
1249  return true;
1250  }
1251  }
1252 
1253  return false;
1254 }
1255 
1256 // Get a scalar native builtin signle argument FP function
1257 FunctionCallee AMDGPULibCalls::getNativeFunction(Module *M,
1258  const FuncInfo &FInfo) {
1259  if (getArgType(FInfo) == AMDGPULibFunc::F64 || !HasNative(FInfo.getId()))
1260  return nullptr;
1261  FuncInfo nf = FInfo;
1262  nf.setPrefix(AMDGPULibFunc::NATIVE);
1263  return getFunction(M, nf);
1264 }
1265 
1266 // fold sqrt -> native_sqrt (x)
1267 bool AMDGPULibCalls::fold_sqrt(CallInst *CI, IRBuilder<> &B,
1268  const FuncInfo &FInfo) {
1269  if (getArgType(FInfo) == AMDGPULibFunc::F32 && (getVecSize(FInfo) == 1) &&
1270  (FInfo.getPrefix() != AMDGPULibFunc::NATIVE)) {
1271  if (FunctionCallee FPExpr = getNativeFunction(
1273  Value *opr0 = CI->getArgOperand(0);
1274  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> "
1275  << "sqrt(" << *opr0 << ")\n");
1276  Value *nval = CreateCallEx(B,FPExpr, opr0, "__sqrt");
1277  replaceCall(nval);
1278  return true;
1279  }
1280  }
1281  return false;
1282 }
1283 
1284 // fold sin, cos -> sincos.
1285 bool AMDGPULibCalls::fold_sincos(CallInst *CI, IRBuilder<> &B,
1286  AliasAnalysis *AA) {
1287  AMDGPULibFunc fInfo;
1288  if (!AMDGPULibFunc::parse(CI->getCalledFunction()->getName(), fInfo))
1289  return false;
1290 
1291  assert(fInfo.getId() == AMDGPULibFunc::EI_SIN ||
1292  fInfo.getId() == AMDGPULibFunc::EI_COS);
1293  bool const isSin = fInfo.getId() == AMDGPULibFunc::EI_SIN;
1294 
1295  Value *CArgVal = CI->getArgOperand(0);
1296  BasicBlock * const CBB = CI->getParent();
1297 
1298  int const MaxScan = 30;
1299 
1300  { // fold in load value.
1301  LoadInst *LI = dyn_cast<LoadInst>(CArgVal);
1302  if (LI && LI->getParent() == CBB) {
1303  BasicBlock::iterator BBI = LI->getIterator();
1304  Value *AvailableVal = FindAvailableLoadedValue(LI, CBB, BBI, MaxScan, AA);
1305  if (AvailableVal) {
1306  CArgVal->replaceAllUsesWith(AvailableVal);
1307  if (CArgVal->getNumUses() == 0)
1308  LI->eraseFromParent();
1309  CArgVal = CI->getArgOperand(0);
1310  }
1311  }
1312  }
1313 
1314  Module *M = CI->getModule();
1316  std::string const PairName = fInfo.mangle();
1317 
1318  CallInst *UI = nullptr;
1319  for (User* U : CArgVal->users()) {
1320  CallInst *XI = dyn_cast_or_null<CallInst>(U);
1321  if (!XI || XI == CI || XI->getParent() != CBB)
1322  continue;
1323 
1324  Function *UCallee = XI->getCalledFunction();
1325  if (!UCallee || !UCallee->getName().equals(PairName))
1326  continue;
1327 
1328  BasicBlock::iterator BBI = CI->getIterator();
1329  if (BBI == CI->getParent()->begin())
1330  break;
1331  --BBI;
1332  for (int I = MaxScan; I > 0 && BBI != CBB->begin(); --BBI, --I) {
1333  if (cast<Instruction>(BBI) == XI) {
1334  UI = XI;
1335  break;
1336  }
1337  }
1338  if (UI) break;
1339  }
1340 
1341  if (!UI) return false;
1342 
1343  // Merge the sin and cos.
1344 
1345  // for OpenCL 2.0 we have only generic implementation of sincos
1346  // function.
1349  FunctionCallee Fsincos = getFunction(M, nf);
1350  if (!Fsincos) return false;
1351 
1353  AllocaInst *Alloc = insertAlloca(UI, B, "__sincos_");
1354  B.SetInsertPoint(UI);
1355 
1356  Value *P = Alloc;
1357  Type *PTy = Fsincos.getFunctionType()->getParamType(1);
1358  // The allocaInst allocates the memory in private address space. This need
1359  // to be bitcasted to point to the address space of cos pointer type.
1360  // In OpenCL 2.0 this is generic, while in 1.2 that is private.
1362  P = B.CreateAddrSpaceCast(Alloc, PTy);
1363  CallInst *Call = CreateCallEx2(B, Fsincos, UI->getArgOperand(0), P);
1364 
1365  LLVM_DEBUG(errs() << "AMDIC: fold_sincos (" << *CI << ", " << *UI << ") with "
1366  << *Call << "\n");
1367 
1368  if (!isSin) { // CI->cos, UI->sin
1369  B.SetInsertPoint(&*ItOld);
1370  UI->replaceAllUsesWith(&*Call);
1371  Instruction *Reload = B.CreateLoad(Alloc->getAllocatedType(), Alloc);
1372  CI->replaceAllUsesWith(Reload);
1373  UI->eraseFromParent();
1374  CI->eraseFromParent();
1375  } else { // CI->sin, UI->cos
1376  Instruction *Reload = B.CreateLoad(Alloc->getAllocatedType(), Alloc);
1377  UI->replaceAllUsesWith(Reload);
1378  CI->replaceAllUsesWith(Call);
1379  UI->eraseFromParent();
1380  CI->eraseFromParent();
1381  }
1382  return true;
1383 }
1384 
1385 bool AMDGPULibCalls::fold_wavefrontsize(CallInst *CI, IRBuilder<> &B) {
1386  if (!TM)
1387  return false;
1388 
1389  StringRef CPU = TM->getTargetCPU();
1390  StringRef Features = TM->getTargetFeatureString();
1391  if ((CPU.empty() || CPU.equals_lower("generic")) &&
1392  (Features.empty() ||
1393  Features.find_lower("wavefrontsize") == StringRef::npos))
1394  return false;
1395 
1396  Function *F = CI->getParent()->getParent();
1397  const GCNSubtarget &ST = TM->getSubtarget<GCNSubtarget>(*F);
1398  unsigned N = ST.getWavefrontSize();
1399 
1400  LLVM_DEBUG(errs() << "AMDIC: fold_wavefrontsize (" << *CI << ") with "
1401  << N << "\n");
1402 
1404  CI->eraseFromParent();
1405  return true;
1406 }
1407 
1408 // Get insertion point at entry.
1409 BasicBlock::iterator AMDGPULibCalls::getEntryIns(CallInst * UI) {
1410  Function * Func = UI->getParent()->getParent();
1411  BasicBlock * BB = &Func->getEntryBlock();
1412  assert(BB && "Entry block not found!");
1413  BasicBlock::iterator ItNew = BB->begin();
1414  return ItNew;
1415 }
1416 
1417 // Insert a AllocsInst at the beginning of function entry block.
1418 AllocaInst* AMDGPULibCalls::insertAlloca(CallInst *UI, IRBuilder<> &B,
1419  const char *prefix) {
1420  BasicBlock::iterator ItNew = getEntryIns(UI);
1421  Function *UCallee = UI->getCalledFunction();
1422  Type *RetType = UCallee->getReturnType();
1423  B.SetInsertPoint(&*ItNew);
1424  AllocaInst *Alloc = B.CreateAlloca(RetType, 0,
1425  std::string(prefix) + UI->getName());
1426  Alloc->setAlignment(MaybeAlign(
1427  UCallee->getParent()->getDataLayout().getTypeAllocSize(RetType)));
1428  return Alloc;
1429 }
1430 
1431 bool AMDGPULibCalls::evaluateScalarMathFunc(FuncInfo &FInfo,
1432  double& Res0, double& Res1,
1433  Constant *copr0, Constant *copr1,
1434  Constant *copr2) {
1435  // By default, opr0/opr1/opr3 holds values of float/double type.
1436  // If they are not float/double, each function has to its
1437  // operand separately.
1438  double opr0=0.0, opr1=0.0, opr2=0.0;
1439  ConstantFP *fpopr0 = dyn_cast_or_null<ConstantFP>(copr0);
1440  ConstantFP *fpopr1 = dyn_cast_or_null<ConstantFP>(copr1);
1441  ConstantFP *fpopr2 = dyn_cast_or_null<ConstantFP>(copr2);
1442  if (fpopr0) {
1443  opr0 = (getArgType(FInfo) == AMDGPULibFunc::F64)
1444  ? fpopr0->getValueAPF().convertToDouble()
1445  : (double)fpopr0->getValueAPF().convertToFloat();
1446  }
1447 
1448  if (fpopr1) {
1449  opr1 = (getArgType(FInfo) == AMDGPULibFunc::F64)
1450  ? fpopr1->getValueAPF().convertToDouble()
1451  : (double)fpopr1->getValueAPF().convertToFloat();
1452  }
1453 
1454  if (fpopr2) {
1455  opr2 = (getArgType(FInfo) == AMDGPULibFunc::F64)
1456  ? fpopr2->getValueAPF().convertToDouble()
1457  : (double)fpopr2->getValueAPF().convertToFloat();
1458  }
1459 
1460  switch (FInfo.getId()) {
1461  default : return false;
1462 
1464  Res0 = acos(opr0);
1465  return true;
1466 
1468  // acosh(x) == log(x + sqrt(x*x - 1))
1469  Res0 = log(opr0 + sqrt(opr0*opr0 - 1.0));
1470  return true;
1471 
1473  Res0 = acos(opr0) / MATH_PI;
1474  return true;
1475 
1477  Res0 = asin(opr0);
1478  return true;
1479 
1481  // asinh(x) == log(x + sqrt(x*x + 1))
1482  Res0 = log(opr0 + sqrt(opr0*opr0 + 1.0));
1483  return true;
1484 
1486  Res0 = asin(opr0) / MATH_PI;
1487  return true;
1488 
1490  Res0 = atan(opr0);
1491  return true;
1492 
1494  // atanh(x) == (log(x+1) - log(x-1))/2;
1495  Res0 = (log(opr0 + 1.0) - log(opr0 - 1.0))/2.0;
1496  return true;
1497 
1499  Res0 = atan(opr0) / MATH_PI;
1500  return true;
1501 
1503  Res0 = (opr0 < 0.0) ? -pow(-opr0, 1.0/3.0) : pow(opr0, 1.0/3.0);
1504  return true;
1505 
1506  case AMDGPULibFunc::EI_COS:
1507  Res0 = cos(opr0);
1508  return true;
1509 
1511  Res0 = cosh(opr0);
1512  return true;
1513 
1515  Res0 = cos(MATH_PI * opr0);
1516  return true;
1517 
1518  case AMDGPULibFunc::EI_EXP:
1519  Res0 = exp(opr0);
1520  return true;
1521 
1523  Res0 = pow(2.0, opr0);
1524  return true;
1525 
1527  Res0 = pow(10.0, opr0);
1528  return true;
1529 
1531  Res0 = exp(opr0) - 1.0;
1532  return true;
1533 
1534  case AMDGPULibFunc::EI_LOG:
1535  Res0 = log(opr0);
1536  return true;
1537 
1539  Res0 = log(opr0) / log(2.0);
1540  return true;
1541 
1543  Res0 = log(opr0) / log(10.0);
1544  return true;
1545 
1547  Res0 = 1.0 / sqrt(opr0);
1548  return true;
1549 
1550  case AMDGPULibFunc::EI_SIN:
1551  Res0 = sin(opr0);
1552  return true;
1553 
1555  Res0 = sinh(opr0);
1556  return true;
1557 
1559  Res0 = sin(MATH_PI * opr0);
1560  return true;
1561 
1563  Res0 = sqrt(opr0);
1564  return true;
1565 
1566  case AMDGPULibFunc::EI_TAN:
1567  Res0 = tan(opr0);
1568  return true;
1569 
1571  Res0 = tanh(opr0);
1572  return true;
1573 
1575  Res0 = tan(MATH_PI * opr0);
1576  return true;
1577 
1579  Res0 = 1.0 / opr0;
1580  return true;
1581 
1582  // two-arg functions
1584  Res0 = opr0 / opr1;
1585  return true;
1586 
1587  case AMDGPULibFunc::EI_POW:
1589  Res0 = pow(opr0, opr1);
1590  return true;
1591 
1592  case AMDGPULibFunc::EI_POWN: {
1593  if (ConstantInt *iopr1 = dyn_cast_or_null<ConstantInt>(copr1)) {
1594  double val = (double)iopr1->getSExtValue();
1595  Res0 = pow(opr0, val);
1596  return true;
1597  }
1598  return false;
1599  }
1600 
1601  case AMDGPULibFunc::EI_ROOTN: {
1602  if (ConstantInt *iopr1 = dyn_cast_or_null<ConstantInt>(copr1)) {
1603  double val = (double)iopr1->getSExtValue();
1604  Res0 = pow(opr0, 1.0 / val);
1605  return true;
1606  }
1607  return false;
1608  }
1609 
1610  // with ptr arg
1612  Res0 = sin(opr0);
1613  Res1 = cos(opr0);
1614  return true;
1615 
1616  // three-arg functions
1617  case AMDGPULibFunc::EI_FMA:
1618  case AMDGPULibFunc::EI_MAD:
1619  Res0 = opr0 * opr1 + opr2;
1620  return true;
1621  }
1622 
1623  return false;
1624 }
1625 
1626 bool AMDGPULibCalls::evaluateCall(CallInst *aCI, FuncInfo &FInfo) {
1627  int numArgs = (int)aCI->getNumArgOperands();
1628  if (numArgs > 3)
1629  return false;
1630 
1631  Constant *copr0 = nullptr;
1632  Constant *copr1 = nullptr;
1633  Constant *copr2 = nullptr;
1634  if (numArgs > 0) {
1635  if ((copr0 = dyn_cast<Constant>(aCI->getArgOperand(0))) == nullptr)
1636  return false;
1637  }
1638 
1639  if (numArgs > 1) {
1640  if ((copr1 = dyn_cast<Constant>(aCI->getArgOperand(1))) == nullptr) {
1641  if (FInfo.getId() != AMDGPULibFunc::EI_SINCOS)
1642  return false;
1643  }
1644  }
1645 
1646  if (numArgs > 2) {
1647  if ((copr2 = dyn_cast<Constant>(aCI->getArgOperand(2))) == nullptr)
1648  return false;
1649  }
1650 
1651  // At this point, all arguments to aCI are constants.
1652 
1653  // max vector size is 16, and sincos will generate two results.
1654  double DVal0[16], DVal1[16];
1655  bool hasTwoResults = (FInfo.getId() == AMDGPULibFunc::EI_SINCOS);
1656  if (getVecSize(FInfo) == 1) {
1657  if (!evaluateScalarMathFunc(FInfo, DVal0[0],
1658  DVal1[0], copr0, copr1, copr2)) {
1659  return false;
1660  }
1661  } else {
1662  ConstantDataVector *CDV0 = dyn_cast_or_null<ConstantDataVector>(copr0);
1663  ConstantDataVector *CDV1 = dyn_cast_or_null<ConstantDataVector>(copr1);
1664  ConstantDataVector *CDV2 = dyn_cast_or_null<ConstantDataVector>(copr2);
1665  for (int i=0; i < getVecSize(FInfo); ++i) {
1666  Constant *celt0 = CDV0 ? CDV0->getElementAsConstant(i) : nullptr;
1667  Constant *celt1 = CDV1 ? CDV1->getElementAsConstant(i) : nullptr;
1668  Constant *celt2 = CDV2 ? CDV2->getElementAsConstant(i) : nullptr;
1669  if (!evaluateScalarMathFunc(FInfo, DVal0[i],
1670  DVal1[i], celt0, celt1, celt2)) {
1671  return false;
1672  }
1673  }
1674  }
1675 
1676  LLVMContext &context = CI->getParent()->getParent()->getContext();
1677  Constant *nval0, *nval1;
1678  if (getVecSize(FInfo) == 1) {
1679  nval0 = ConstantFP::get(CI->getType(), DVal0[0]);
1680  if (hasTwoResults)
1681  nval1 = ConstantFP::get(CI->getType(), DVal1[0]);
1682  } else {
1683  if (getArgType(FInfo) == AMDGPULibFunc::F32) {
1684  SmallVector <float, 0> FVal0, FVal1;
1685  for (int i=0; i < getVecSize(FInfo); ++i)
1686  FVal0.push_back((float)DVal0[i]);
1687  ArrayRef<float> tmp0(FVal0);
1688  nval0 = ConstantDataVector::get(context, tmp0);
1689  if (hasTwoResults) {
1690  for (int i=0; i < getVecSize(FInfo); ++i)
1691  FVal1.push_back((float)DVal1[i]);
1692  ArrayRef<float> tmp1(FVal1);
1693  nval1 = ConstantDataVector::get(context, tmp1);
1694  }
1695  } else {
1696  ArrayRef<double> tmp0(DVal0);
1697  nval0 = ConstantDataVector::get(context, tmp0);
1698  if (hasTwoResults) {
1699  ArrayRef<double> tmp1(DVal1);
1700  nval1 = ConstantDataVector::get(context, tmp1);
1701  }
1702  }
1703  }
1704 
1705  if (hasTwoResults) {
1706  // sincos
1707  assert(FInfo.getId() == AMDGPULibFunc::EI_SINCOS &&
1708  "math function with ptr arg not supported yet");
1709  new StoreInst(nval1, aCI->getArgOperand(1), aCI);
1710  }
1711 
1712  replaceCall(nval0);
1713  return true;
1714 }
1715 
1716 // Public interface to the Simplify LibCalls pass.
1718  const TargetMachine *TM) {
1719  return new AMDGPUSimplifyLibCalls(Opt, TM);
1720 }
1721 
1723  return new AMDGPUUseNativeCalls();
1724 }
1725 
1726 static bool setFastFlags(Function &F, const TargetOptions &Options) {
1727  AttrBuilder B;
1728 
1729  if (Options.UnsafeFPMath || Options.NoInfsFPMath)
1730  B.addAttribute("no-infs-fp-math", "true");
1731  if (Options.UnsafeFPMath || Options.NoNaNsFPMath)
1732  B.addAttribute("no-nans-fp-math", "true");
1733  if (Options.UnsafeFPMath) {
1734  B.addAttribute("less-precise-fpmad", "true");
1735  B.addAttribute("unsafe-fp-math", "true");
1736  }
1737 
1738  if (!B.hasAttributes())
1739  return false;
1740 
1742 
1743  return true;
1744 }
1745 
1747  if (skipFunction(F))
1748  return false;
1749 
1750  bool Changed = false;
1751  auto AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
1752 
1753  LLVM_DEBUG(dbgs() << "AMDIC: process function ";
1754  F.printAsOperand(dbgs(), false, F.getParent()); dbgs() << '\n';);
1755 
1756  if (!EnablePreLink)
1757  Changed |= setFastFlags(F, Options);
1758 
1759  for (auto &BB : F) {
1760  for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E; ) {
1761  // Ignore non-calls.
1762  CallInst *CI = dyn_cast<CallInst>(I);
1763  ++I;
1764  if (!CI) continue;
1765 
1766  // Ignore indirect calls.
1767  Function *Callee = CI->getCalledFunction();
1768  if (Callee == 0) continue;
1769 
1770  LLVM_DEBUG(dbgs() << "AMDIC: try folding " << *CI << "\n";
1771  dbgs().flush());
1772  if(Simplifier.fold(CI, AA))
1773  Changed = true;
1774  }
1775  }
1776  return Changed;
1777 }
1778 
1780  if (skipFunction(F) || UseNative.empty())
1781  return false;
1782 
1783  bool Changed = false;
1784  for (auto &BB : F) {
1785  for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E; ) {
1786  // Ignore non-calls.
1787  CallInst *CI = dyn_cast<CallInst>(I);
1788  ++I;
1789  if (!CI) continue;
1790 
1791  // Ignore indirect calls.
1792  Function *Callee = CI->getCalledFunction();
1793  if (Callee == 0) continue;
1794 
1795  if(Simplifier.useNative(CI))
1796  Changed = true;
1797  }
1798  }
1799  return Changed;
1800 }
A vector constant whose element type is a simple 1/2/4/8-byte integer or float/double, and whose elements are just simple data values (i.e.
Definition: Constants.h:761
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks &#39;this&#39; from the containing basic block and deletes it.
Definition: Instruction.cpp:67
static const TableEntry tbl_sqrt[]
static const TableEntry tbl_log10[]
raw_ostream & errs()
This returns a reference to a raw_ostream for standard error.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
LLVMContext & Context
AllocaInst * CreateAlloca(Type *Ty, unsigned AddrSpace, Value *ArraySize=nullptr, const Twine &Name="")
Definition: IRBuilder.h:1562
AMDGPU specific subclass of TargetSubtarget.
void dropAllReferences()
Drop all references to operands.
Definition: User.h:294
EFuncId getId() const
Value * CreateAddrSpaceCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1968
This class represents lattice values for constants.
Definition: AllocatorList.h:23
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition: DerivedTypes.h:140
Constant * getElementAsConstant(unsigned i) const
Return a Constant for a specified index&#39;s element.
Definition: Constants.cpp:2773
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve &#39;CreateLoad(Ty, Ptr, "...")&#39; correctly, instead of converting the string to &#39;bool...
Definition: IRBuilder.h:1575
LLVM_NODISCARD bool equals_lower(StringRef RHS) const
equals_lower - Check for string equality, ignoring case.
Definition: StringRef.h:181
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:66
amdgpu Simplify well known AMD library false FunctionCallee Value const Twine & Name
static const TableEntry tbl_cospi[]
amdgpu simplifylib
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:170
static const TableEntry tbl_cbrt[]
static bool parse(StringRef MangledName, AMDGPULibFunc &Ptr)
void initializeAMDGPUSimplifyLibCallsPass(PassRegistry &)
This class represents a function call, abstracting a target machine&#39;s calling convention.
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space...
Definition: Type.cpp:637
bool isUnsafeMath(const CallInst *CI) const
float convertToFloat() const
Definition: APFloat.h:1113
F(f)
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition: DerivedTypes.h:635
An instruction for reading from memory.
Definition: Instructions.h:169
static const TableEntry tbl_erf[]
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:181
AttrBuilder & addAttribute(Attribute::AttrKind Val)
Add an attribute to the builder.
void setPrefix(ENamePrefix PFX)
LLVMContext & getContext() const
Get the context in which this basic block lives.
Definition: BasicBlock.cpp:32
static const TableEntry tbl_acosh[]
static FunctionCallee getOrInsertFunction(llvm::Module *M, const AMDGPULibFunc &fInfo)
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:273
#define DEBUG_WITH_TYPE(TYPE, X)
DEBUG_WITH_TYPE macro - This macro should be used by passes to emit debug information.
Definition: Debug.h:64
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
Definition: IRBuilder.h:383
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1241
static const TableEntry tbl_atanh[]
std::string getName() const
Get unmangled name for mangled library function and name for unmangled library function.
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:50
bool hasAttributes() const
Return true if the builder has IR-level attributes.
const DataLayout & getDataLayout() const
Get the data layout for the module&#39;s target platform.
Definition: Module.cpp:369
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:80
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
Definition: IRBuilder.h:388
Value * FindAvailableLoadedValue(LoadInst *Load, BasicBlock *ScanBB, BasicBlock::iterator &ScanFrom, unsigned MaxInstsToScan=DefMaxInstsToScan, AliasAnalysis *AA=nullptr, bool *IsLoadCSE=nullptr, unsigned *NumScanedInst=nullptr)
Scan backwards to see if we have the value of the given load available locally within a small number ...
Definition: Loads.cpp:361
LLVMContext & getContext() const
Get the global data context.
Definition: Module.h:245
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:197
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:779
Param * getLeads()
Get leading parameters for mangled lib functions.
static const TableEntry tbl_sinh[]
LLVM_NODISCARD bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:140
unsigned getNumArgs() const
INITIALIZE_PASS_BEGIN(AMDGPUSimplifyLibCalls, "amdgpu-simplifylib", "Simplify well-known AMD library calls", false, false) INITIALIZE_PASS_END(AMDGPUSimplifyLibCalls
All zero aggregate value.
Definition: Constants.h:340
void setId(EFuncId Id)
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1963
unsigned NoNaNsFPMath
NoNaNsFPMath - This flag is enabled when the -enable-no-nans-fp-math flag is specified on the command...
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:246
amdgpu Simplify well known AMD library false
static const TableEntry tbl_sinpi[]
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
Definition: InstrTypes.h:1323
bool isNegative() const
Return true if the sign bit is set.
Definition: Constants.h:308
static const TableEntry tbl_asinh[]
static const TableEntry tbl_tanpi[]
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:32
Value * CreateSIToFP(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1933
An instruction for storing to memory.
Definition: Instructions.h:325
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:429
Value * CreateZExtOrBitCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1973
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block...
Definition: IRBuilder.h:132
static const TableEntry tbl_log2[]
constexpr double ln2
Definition: MathExtras.h:59
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1294
static AMDGPULibFunc::EType getArgType(const AMDGPULibFunc &FInfo)
Address space for private memory.
Definition: AMDGPU.h:275
const BasicBlock & getEntryBlock() const
Definition: Function.h:669
static bool runOnFunction(Function &F, bool PostInlining)
#define P(N)
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:432
TableRef(const TableEntry(&tbl)[N])
Type * getReturnType() const
Returns the type of the ret val.
Definition: Function.h:168
bool fold(CallInst *CI, AliasAnalysis *AA=nullptr)
unsigned UnsafeFPMath
UnsafeFPMath - This flag is enabled when the -enable-unsafe-fp-math flag is specified on the command ...
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Value * CreateFMul(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Definition: IRBuilder.h:1383
static const TableEntry tbl_exp2[]
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:465
LLVM Basic Block Representation.
Definition: BasicBlock.h:57
static Function * getFunction(llvm::Module *M, const AMDGPULibFunc &fInfo)
Constant * getSplatValue() const
If this is a splat constant, meaning that all of the elements have the same value, return that value.
Definition: Constants.cpp:2810
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:46
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:64
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This is an important base class in LLVM.
Definition: Constant.h:41
Address space for flat memory.
Definition: AMDGPU.h:269
This file contains the declarations for the subclasses of Constant, which represent the different fla...
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Definition: DataLayout.h:486
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:263
static cl::opt< bool > EnablePreLink("amdgpu-prelink", cl::desc("Enable pre-link mode optimizations"), cl::init(false), cl::Hidden)
double convertToDouble() const
Definition: APFloat.h:1112
float getElementAsFloat(unsigned i) const
If this is an sequential container of floats, return the specified element as a float.
Definition: Constants.cpp:2761
static const TableEntry tbl_atan[]
static const TableEntry tbl_sin[]
bool isMangled() const
Represent the analysis usage information of a pass.
static const TableEntry tbl_rsqrt[]
static const TableEntry tbl_acospi[]
void setCallingConv(CallingConv::ID CC)
Definition: InstrTypes.h:1348
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:284
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
static FunctionType * get(Type *Result, ArrayRef< Type *> Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
Definition: Type.cpp:301
Wrapper class for AMDGPULIbFuncImpl.
static const TableEntry tbl_expm1[]
self_iterator getIterator()
Definition: ilist_node.h:81
static int getVecSize(const AMDGPULibFunc &FInfo)
static Constant * getSplat(unsigned NumElts, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
Definition: Constants.cpp:2650
unsigned NoInfsFPMath
NoInfsFPMath - This flag is enabled when the -enable-no-infs-fp-math flag is specified on the command...
static double log2(double V)
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
Definition: Function.cpp:205
static unsigned getEPtrKindFromAddrSpace(unsigned AS)
size_t size() const
Definition: SmallVector.h:52
void initializeAMDGPUUseNativeCallsPass(PassRegistry &)
auto find(R &&Range, const T &Val) -> decltype(adl_begin(Range))
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1186
void printAsOperand(raw_ostream &O, bool PrintType=true, const Module *M=nullptr) const
Print the name of this Value out to the specified raw_ostream.
Definition: AsmWriter.cpp:4356
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE, "Assign register bank of generic virtual registers", false, false) RegBankSelect
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:40
static const TableEntry tbl_atanpi[]
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
Definition: Instructions.h:105
const APFloat & getValueAPF() const
Definition: Constants.h:302
unsigned getWavefrontSize() const
static const TableEntry tbl_exp10[]
AMDGPULibCalls(const TargetMachine *TM_=nullptr)
Iterator for intrusive lists based on ilist_node.
TargetPassConfig.
This is the shared class of boolean and integer constants.
Definition: Constants.h:83
static const TableEntry tbl_asinpi[]
static bool setFastFlags(Function &F, const TargetOptions &Options)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type...
Definition: Type.cpp:134
This struct is a compact representation of a valid (power of two) or undefined (0) alignment...
Definition: Alignment.h:117
ENamePrefix getPrefix() const
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:837
Utility class for floating point operations which can have information about relaxed accuracy require...
Definition: Operator.h:245
Module.h This file contains the declarations for the Module class.
LLVM_NODISCARD size_t find_lower(char C, size_t From=0) const
Search for the first character C in the string, ignoring case.
Definition: StringRef.cpp:57
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Definition: Type.cpp:184
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:653
static Constant * get(Type *Ty, double V)
This returns a ConstantFP, or a vector containing a splat of a ConstantFP, for the specified value in...
Definition: Constants.cpp:716
const TableEntry * table
AttributeList getAttributes() const
Return the parameter attributes for this call.
Definition: InstrTypes.h:1366
static const TableEntry tbl_log[]
void setAlignment(MaybeAlign Align)
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition: Function.h:193
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
Class to represent vector types.
Definition: DerivedTypes.h:432
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:55
static const TableEntry tbl_cosh[]
amdgpu Simplify well known AMD library false FunctionCallee Callee
iterator_range< user_iterator > users()
Definition: Value.h:420
InstListType::iterator iterator
Instruction iterators...
Definition: BasicBlock.h:89
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1207
Value * CreatePointerCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2009
LLVM_NODISCARD bool equals(StringRef RHS) const
equals - Check for string equality, this is more efficient than compare() when the relative ordering ...
Definition: StringRef.h:174
unsigned getNumUses() const
This method computes the number of uses of this Value.
Definition: Value.cpp:160
unsigned getNumArgOperands() const
Definition: InstrTypes.h:1239
static const TableEntry tbl_exp[]
static const size_t npos
Definition: StringRef.h:50
StringRef getValueAsString() const
Return the attribute&#39;s value as a string.
Definition: Attributes.cpp:220
static const TableEntry tbl_tanh[]
bool isZero() const
Return true if the value is positive or negative zero.
Definition: Constants.h:305
static VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
Definition: Type.cpp:614
double getElementAsDouble(unsigned i) const
If this is an sequential container of doubles, return the specified element as a double.
Definition: Constants.cpp:2767
StringRef getName() const
Return a constant reference to the value&#39;s name.
Definition: Value.cpp:214
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation.
Definition: InstrTypes.h:1287
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:106
#define I(x, y, z)
Definition: MD5.cpp:58
static TableRef getOptTable(AMDGPULibFunc::EFuncId id)
#define N
#define MATH_PI
APFloat abs(APFloat X)
Returns the absolute value of the argument.
Definition: APFloat.h:1228
static const TableEntry tbl_asin[]
Value * CreateFPToSI(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1922
void setAttributes(AttributeList A)
Set the parameter attributes for this call.
Definition: InstrTypes.h:1370
FunctionType * getFunctionType()
Definition: DerivedTypes.h:187
static cl::list< std::string > UseNative("amdgpu-use-native", cl::desc("Comma separated list of functions to replace with native, or all"), cl::CommaSeparated, cl::ValueOptional, cl::Hidden)
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:332
uint32_t Size
Definition: Profile.cpp:46
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value *> Args=None, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2239
#define MATH_SQRT2
static const TableEntry tbl_tan[]
static const TableEntry tbl_cos[]
Value * CreateFAdd(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Definition: IRBuilder.h:1333
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1268
static bool HasNative(AMDGPULibFunc::EFuncId id)
Value * CreateFDiv(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Definition: IRBuilder.h:1408
const std::string to_string(const T &Value)
Definition: ScopedPrinter.h:61
void replaceCall(Value *With)
void addAttributes(unsigned i, const AttrBuilder &Attrs)
adds the attributes to the list of attributes.
Definition: Function.cpp:393
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
FunctionPass * createAMDGPUSimplifyLibCallsPass(const TargetOptions &, const TargetMachine *)
FunctionPass * createAMDGPUUseNativeCallsPass()
aarch64 promote const
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:575
LLVM Value Representation.
Definition: Value.h:74
amdgpu Simplify well known AMD library calls
#define MATH_SQRT1_2
static const TableEntry tbl_erfc[]
void setFastMathFlags(FastMathFlags NewFMF)
Set the fast-math flags to be used with generated fp-math operators.
Definition: IRBuilder.h:225
BasicBlock::iterator GetInsertPoint() const
Definition: IRBuilder.h:127
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition: Function.h:333
std::string mangle() const
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:65
Type * getElementType() const
Definition: DerivedTypes.h:399
bool isExactlyValue(const APFloat &V) const
We don&#39;t rely on operator== working on double values, as it returns true for things that are clearly ...
Definition: Constants.cpp:855
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
unsigned getNumElements() const
Return the number of elements in the array or vector.
Definition: Constants.cpp:2456
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object...
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
Definition: Constants.h:156
static Constant * get(LLVMContext &Context, ArrayRef< uint8_t > Elts)
get() constructors - Return a constant with vector type with an element count and element type matchi...
Definition: Constants.cpp:2595
#define LLVM_DEBUG(X)
Definition: Debug.h:122
bool isDoubleTy() const
Return true if this is &#39;double&#39;, a 64-bit IEEE fp type.
Definition: Type.h:150
static const TableEntry tbl_tgamma[]
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
bool useNative(CallInst *CI)
static const TableEntry tbl_acos[]
const BasicBlock * getParent() const
Definition: Instruction.h:66
an instruction to allocate memory on the stack
Definition: Instructions.h:59
#define MATH_E