LLVM  4.0.0
LowerExpectIntrinsic.cpp
Go to the documentation of this file.
1 //===- LowerExpectIntrinsic.cpp - Lower expect intrinsic ------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This pass lowers the 'expect' intrinsic to LLVM metadata.
11 //
12 //===----------------------------------------------------------------------===//
13 
15 #include "llvm/ADT/SmallVector.h"
16 #include "llvm/ADT/Statistic.h"
17 #include "llvm/IR/BasicBlock.h"
18 #include "llvm/IR/Constants.h"
19 #include "llvm/IR/Function.h"
20 #include "llvm/IR/Instructions.h"
21 #include "llvm/IR/Intrinsics.h"
22 #include "llvm/IR/LLVMContext.h"
23 #include "llvm/IR/MDBuilder.h"
24 #include "llvm/IR/Metadata.h"
25 #include "llvm/Pass.h"
27 #include "llvm/Support/Debug.h"
28 #include "llvm/Transforms/Scalar.h"
29 
30 using namespace llvm;
31 
32 #define DEBUG_TYPE "lower-expect-intrinsic"
33 
34 STATISTIC(ExpectIntrinsicsHandled,
35  "Number of 'expect' intrinsic instructions handled");
36 
37 // These default values are chosen to represent an extremely skewed outcome for
38 // a condition, but they leave some room for interpretation by later passes.
39 //
40 // If the documentation for __builtin_expect() was made explicit that it should
41 // only be used in extreme cases, we could make this ratio higher. As it stands,
42 // programmers may be using __builtin_expect() / llvm.expect to annotate that a
43 // branch is likely or unlikely to be taken.
44 //
45 // There is a known dependency on this ratio in CodeGenPrepare when transforming
46 // 'select' instructions. It may be worthwhile to hoist these values to some
47 // shared space, so they can be used directly by other passes.
48 
50  "likely-branch-weight", cl::Hidden, cl::init(2000),
51  cl::desc("Weight of the branch likely to be taken (default = 2000)"));
53  "unlikely-branch-weight", cl::Hidden, cl::init(1),
54  cl::desc("Weight of the branch unlikely to be taken (default = 1)"));
55 
56 static bool handleSwitchExpect(SwitchInst &SI) {
58  if (!CI)
59  return false;
60 
61  Function *Fn = CI->getCalledFunction();
62  if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect)
63  return false;
64 
65  Value *ArgValue = CI->getArgOperand(0);
66  ConstantInt *ExpectedValue = dyn_cast<ConstantInt>(CI->getArgOperand(1));
67  if (!ExpectedValue)
68  return false;
69 
70  SwitchInst::CaseIt Case = SI.findCaseValue(ExpectedValue);
71  unsigned n = SI.getNumCases(); // +1 for default case.
73 
74  if (Case == SI.case_default())
75  Weights[0] = LikelyBranchWeight;
76  else
77  Weights[Case.getCaseIndex() + 1] = LikelyBranchWeight;
78 
80  MDBuilder(CI->getContext()).createBranchWeights(Weights));
81 
82  SI.setCondition(ArgValue);
83  return true;
84 }
85 
86 // Handle both BranchInst and SelectInst.
87 template <class BrSelInst> static bool handleBrSelExpect(BrSelInst &BSI) {
88 
89  // Handle non-optimized IR code like:
90  // %expval = call i64 @llvm.expect.i64(i64 %conv1, i64 1)
91  // %tobool = icmp ne i64 %expval, 0
92  // br i1 %tobool, label %if.then, label %if.end
93  //
94  // Or the following simpler case:
95  // %expval = call i1 @llvm.expect.i1(i1 %cmp, i1 1)
96  // br i1 %expval, label %if.then, label %if.end
97 
98  CallInst *CI;
99 
100  ICmpInst *CmpI = dyn_cast<ICmpInst>(BSI.getCondition());
101  if (!CmpI) {
102  CI = dyn_cast<CallInst>(BSI.getCondition());
103  } else {
104  if (CmpI->getPredicate() != CmpInst::ICMP_NE)
105  return false;
106  CI = dyn_cast<CallInst>(CmpI->getOperand(0));
107  }
108 
109  if (!CI)
110  return false;
111 
112  Function *Fn = CI->getCalledFunction();
113  if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect)
114  return false;
115 
116  Value *ArgValue = CI->getArgOperand(0);
117  ConstantInt *ExpectedValue = dyn_cast<ConstantInt>(CI->getArgOperand(1));
118  if (!ExpectedValue)
119  return false;
120 
121  MDBuilder MDB(CI->getContext());
122  MDNode *Node;
123 
124  // If expect value is equal to 1 it means that we are more likely to take
125  // branch 0, in other case more likely is branch 1.
126  if (ExpectedValue->isOne())
127  Node = MDB.createBranchWeights(LikelyBranchWeight, UnlikelyBranchWeight);
128  else
129  Node = MDB.createBranchWeights(UnlikelyBranchWeight, LikelyBranchWeight);
130 
131  BSI.setMetadata(LLVMContext::MD_prof, Node);
132 
133  if (CmpI)
134  CmpI->setOperand(0, ArgValue);
135  else
136  BSI.setCondition(ArgValue);
137  return true;
138 }
139 
140 static bool handleBranchExpect(BranchInst &BI) {
141  if (BI.isUnconditional())
142  return false;
143 
144  return handleBrSelExpect<BranchInst>(BI);
145 }
146 
148  bool Changed = false;
149 
150  for (BasicBlock &BB : F) {
151  // Create "block_weights" metadata.
152  if (BranchInst *BI = dyn_cast<BranchInst>(BB.getTerminator())) {
153  if (handleBranchExpect(*BI))
154  ExpectIntrinsicsHandled++;
155  } else if (SwitchInst *SI = dyn_cast<SwitchInst>(BB.getTerminator())) {
156  if (handleSwitchExpect(*SI))
157  ExpectIntrinsicsHandled++;
158  }
159 
160  // Remove llvm.expect intrinsics. Iterate backwards in order
161  // to process select instructions before the intrinsic gets
162  // removed.
163  for (auto BI = BB.rbegin(), BE = BB.rend(); BI != BE;) {
164  Instruction *Inst = &*BI++;
165  CallInst *CI = dyn_cast<CallInst>(Inst);
166  if (!CI) {
167  if (SelectInst *SI = dyn_cast<SelectInst>(Inst)) {
168  if (handleBrSelExpect(*SI))
169  ExpectIntrinsicsHandled++;
170  }
171  continue;
172  }
173 
174  Function *Fn = CI->getCalledFunction();
175  if (Fn && Fn->getIntrinsicID() == Intrinsic::expect) {
176  Value *Exp = CI->getArgOperand(0);
177  CI->replaceAllUsesWith(Exp);
178  CI->eraseFromParent();
179  Changed = true;
180  }
181  }
182  }
183 
184  return Changed;
185 }
186 
189  if (lowerExpectIntrinsic(F))
190  return PreservedAnalyses::none();
191 
192  return PreservedAnalyses::all();
193 }
194 
195 namespace {
196 /// \brief Legacy pass for lowering expect intrinsics out of the IR.
197 ///
198 /// When this pass is run over a function it uses expect intrinsics which feed
199 /// branches and switches to provide branch weight metadata for those
200 /// terminators. It then removes the expect intrinsics from the IR so the rest
201 /// of the optimizer can ignore them.
202 class LowerExpectIntrinsic : public FunctionPass {
203 public:
204  static char ID;
205  LowerExpectIntrinsic() : FunctionPass(ID) {
207  }
208 
209  bool runOnFunction(Function &F) override { return lowerExpectIntrinsic(F); }
210 };
211 }
212 
213 char LowerExpectIntrinsic::ID = 0;
214 INITIALIZE_PASS(LowerExpectIntrinsic, "lower-expect",
215  "Lower 'expect' Intrinsics", false, false)
216 
218  return new LowerExpectIntrinsic();
219 }
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:76
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
STATISTIC(NumFunctions,"Total number of functions")
INITIALIZE_PASS(LowerExpectIntrinsic,"lower-expect","Lower 'expect' Intrinsics", false, false) FunctionPass *llvm
This class represents a function call, abstracting a target machine's calling convention.
This file contains the declarations for metadata subclasses.
Metadata node.
Definition: Metadata.h:830
PreservedAnalyses run(Function &F, FunctionAnalysisManager &)
Run the pass over the function.
bool isUnconditional() const
This class represents the LLVM 'select' instruction.
#define F(x, y, z)
Definition: MD5.cpp:51
static bool lowerExpectIntrinsic(Function &F)
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:401
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: PassManager.h:110
static cl::opt< uint32_t > UnlikelyBranchWeight("unlikely-branch-weight", cl::Hidden, cl::init(1), cl::desc("Weight of the branch unlikely to be taken (default = 1)"))
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:395
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:107
unsigned getCaseIndex() const
Returns number of current case.
void initializeLowerExpectIntrinsicPass(PassRegistry &)
LLVM Basic Block Representation.
Definition: BasicBlock.h:51
Conditional or Unconditional Branch instruction.
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This instruction compares its operands according to the predicate given to the constructor.
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:298
Value * getOperand(unsigned i) const
Definition: User.h:145
Predicate getPredicate() const
Return the predicate for this instruction.
Definition: InstrTypes.h:960
static cl::opt< uint32_t > LikelyBranchWeight("likely-branch-weight", cl::Hidden, cl::init(2000), cl::desc("Weight of the branch likely to be taken (default = 2000)"))
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:113
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:654
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition: Metadata.cpp:1183
static bool handleBrSelExpect(BrSelInst &BSI)
This is the shared class of boolean and integer constants.
Definition: Constants.h:88
Function * getCalledFunction() const
Return the function called, or null if this is an indirect function invocation.
CaseIt findCaseValue(const ConstantInt *C)
Search all of the case values for the specified constant.
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition: Function.h:146
void setOperand(unsigned i, Value *Val)
Definition: User.h:150
Value * getArgOperand(unsigned i) const
getArgOperand/setArgOperand - Return/set the i-th call argument.
static bool handleBranchExpect(BranchInst &BI)
Value * getCondition() const
void setCondition(Value *V)
The header file for the LowerExpectIntrinsic pass as used by the new pass manager.
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:287
CaseIt case_default()
Returns an iterator that points to the default case.
unsigned getNumCases() const
Return the number of 'cases' in this switch instruction, excluding the default case.
Multiway switch.
LLVM Value Representation.
Definition: Value.h:71
A container for analyses that lazily runs them and caches their results.
static bool handleSwitchExpect(SwitchInst &SI)
bool isOne() const
This is just a convenience method to make client code smaller for a common case.
Definition: Constants.h:206
FunctionPass * createLowerExpectIntrinsicPass()