18#include "llvm/IR/IntrinsicsBPF.h"
27#define DEBUG_TYPE "bpf-adjust-opt"
34 cl::desc(
"BPF: Disable Serializing ICMP insns."),
39 cl::desc(
"BPF: Disable Avoiding Speculative Code Motion."),
52class BPFAdjustOptImpl {
53 struct PassThroughInfo {
58 : Input(
I), UsedInst(
U), OpIdx(
Idx) {}
62 BPFAdjustOptImpl(
Module *M) :
M(
M) {}
70 bool adjustICmpToBuiltin();
76 bool insertPassThrough();
81char BPFAdjustOpt::ID = 0;
87bool BPFAdjustOpt::runOnModule(
Module &M) {
return BPFAdjustOptImpl(&M).run(); }
89bool BPFAdjustOptImpl::run() {
90 bool Changed = adjustICmpToBuiltin();
98 return insertPassThrough() || Changed;
109bool BPFAdjustOptImpl::adjustICmpToBuiltin() {
110 bool Changed =
false;
117 ToBeDeleted =
nullptr;
120 auto *Icmp = dyn_cast<ICmpInst>(&
I);
124 Value *Op0 = Icmp->getOperand(0);
125 if (!isa<TruncInst>(Op0))
128 auto ConstOp1 = dyn_cast<ConstantInt>(Icmp->getOperand(1));
132 auto ConstOp1Val = ConstOp1->getValue().getZExtValue();
133 auto Op = Icmp->getPredicate();
134 if (Op == ICmpInst::ICMP_ULT || Op == ICmpInst::ICMP_UGE) {
135 if ((ConstOp1Val - 1) & ConstOp1Val)
137 }
else if (Op == ICmpInst::ICMP_ULE || Op == ICmpInst::ICMP_UGT) {
138 if (ConstOp1Val & (ConstOp1Val + 1))
147 M, Intrinsic::bpf_compare, {Op0->
getType(), ConstOp1->getType()});
150 Icmp->replaceAllUsesWith(NewInst);
158bool BPFAdjustOptImpl::insertPassThrough() {
159 for (
auto &Info : PassThroughs) {
161 M,
Info.UsedInst->getParent(),
Info.Input,
Info.UsedInst);
162 Info.UsedInst->setOperand(
Info.OpIdx, CI);
165 return !PassThroughs.empty();
170bool BPFAdjustOptImpl::serializeICMPInBB(
Instruction &
I) {
184 auto *Icmp1 = dyn_cast<ICmpInst>(Op0);
187 auto *Icmp2 = dyn_cast<ICmpInst>(Op1);
191 Value *Icmp1Op0 = Icmp1->getOperand(0);
192 Value *Icmp2Op0 = Icmp2->getOperand(0);
193 if (Icmp1Op0 != Icmp2Op0)
198 PassThroughInfo
Info(Icmp1, &
I, 0);
199 PassThroughs.push_back(Info);
205bool BPFAdjustOptImpl::serializeICMPCrossBB(
BasicBlock &BB) {
239 auto *BI = dyn_cast<BranchInst>(TI);
240 if (!BI || !BI->isConditional())
242 auto *
Cond = dyn_cast<ICmpInst>(BI->getCondition());
246 auto Cond2Op =
Cond->getPredicate();
249 BI = dyn_cast<BranchInst>(TI);
250 if (!BI || !BI->isConditional())
252 Cond = dyn_cast<ICmpInst>(BI->getCondition());
256 auto Cond1Op =
Cond->getPredicate();
261 if (Cond1Op == ICmpInst::ICMP_SGT || Cond1Op == ICmpInst::ICMP_SGE) {
262 if (Cond2Op != ICmpInst::ICMP_SLT && Cond2Op != ICmpInst::ICMP_SLE)
264 }
else if (Cond1Op == ICmpInst::ICMP_SLT || Cond1Op == ICmpInst::ICMP_SLE) {
265 if (Cond2Op != ICmpInst::ICMP_SGT && Cond2Op != ICmpInst::ICMP_SGE)
267 }
else if (Cond1Op == ICmpInst::ICMP_ULT || Cond1Op == ICmpInst::ICMP_ULE) {
268 if (Cond2Op != ICmpInst::ICMP_UGT && Cond2Op != ICmpInst::ICMP_UGE)
270 }
else if (Cond1Op == ICmpInst::ICMP_UGT || Cond1Op == ICmpInst::ICMP_UGE) {
271 if (Cond2Op != ICmpInst::ICMP_ULT && Cond2Op != ICmpInst::ICMP_ULE)
278 PassThroughs.push_back(Info);
285bool BPFAdjustOptImpl::avoidSpeculation(
Instruction &
I) {
286 if (
auto *LdInst = dyn_cast<LoadInst>(&
I)) {
287 if (
auto *GV = dyn_cast<GlobalVariable>(LdInst->getOperand(0))) {
294 if (!isa<LoadInst>(&
I) && !isa<CallInst>(&
I))
318 for (
User *U :
I.users()) {
325 if (
auto *Icmp1 = dyn_cast<ICmpInst>(Inst)) {
326 Value *Icmp1Op1 = Icmp1->getOperand(1);
327 if (!isa<Constant>(Icmp1Op1))
341 if (isa<CallInst>(&I2))
343 if (isa<LoadInst>(&I2) || isa<StoreInst>(&I2))
351 if (Inst->
getOpcode() == Instruction::ZExt ||
352 Inst->
getOpcode() == Instruction::SExt) {
353 PassThroughInfo
Info(&
I, Inst, 0);
355 }
else if (
auto *GI = dyn_cast<GetElementPtrInst>(Inst)) {
358 for (i = 1, e = GI->getNumOperands(); i != e; ++i) {
359 Value *
V = GI->getOperand(i);
366 PassThroughInfo
Info(&
I, GI, i);
378void BPFAdjustOptImpl::adjustBasicBlock(
BasicBlock &BB) {
static cl::opt< bool > DisableBPFserializeICMP("bpf-disable-serialize-icmp", cl::Hidden, cl::desc("BPF: Disable Serializing ICMP insns."), cl::init(false))
static cl::opt< bool > DisableBPFavoidSpeculation("bpf-disable-avoid-speculation", cl::Hidden, cl::desc("BPF: Disable Avoiding Speculative Code Motion."), cl::init(false))
SmallVector< MachineOperand, 4 > Cond
Analysis containing CSE Info
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static bool isCandidate(const MachineInstr *MI, Register &DefedReg, Register FrameReg)
Module.h This file contains the declarations for the Module class.
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
A container for analyses that lazily runs them and caches their results.
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
static constexpr StringRef TypeIdAttr
The attribute attached to globals representing a type id.
static Instruction * insertPassThrough(Module *M, BasicBlock *BB, Instruction *Input, Instruction *Before)
Insert a bpf passthrough builtin function.
static constexpr StringRef AmaAttr
The attribute attached to globals representing a field access.
LLVM Basic Block Representation.
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
This is an important base class in LLVM.
This instruction compares its operands according to the predicate given to the constructor.
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction.
const BasicBlock * getParent() const
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
virtual bool runOnModule(Module &M)=0
runOnModule - Virtual method overriden by subclasses to process the module being operated on.
A Module instance is used to store all the information related to an LLVM module.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
static IntegerType * getInt32Ty(LLVMContext &C)
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
bool match(Val *V, const Pattern &P)
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
initializer< Ty > init(const Ty &Val)
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
This is an optimization pass for GlobalISel generic memory operations.
void append_range(Container &C, Range &&R)
Wrapper function to append a range to a container.
ModulePass * createBPFAdjustOpt()