118#include "llvm/IR/IntrinsicsBPF.h"
122#define DEBUG_TYPE "bpf-preserve-static-offset"
130 if (
auto *Call = dyn_cast<CallInst>(
I))
131 if (
Function *Func = Call->getCalledFunction())
132 return Func->getIntrinsicID() == Id;
142 return cast<CallInst>(
I);
148 return cast<CallInst>(
I);
152template <
class T = Instruction>
161 Intrinsic::BPFIntrinsics Intrinsic,
192 Type *SourceElementType;
196 GEPChainInfo() { reset(); }
200 SourceElementType =
nullptr;
207template <
class T = std::disjunction<LoadInst, StoreInst>>
213 unsigned AlignShiftValue =
Log2_64(
Insn->getAlign().value());
214 Args.push_back(
GEP.Members[0]->getPointerOperand());
215 Args.push_back(ConstantInt::get(Int1Ty,
Insn->isVolatile()));
216 Args.push_back(ConstantInt::get(Int8Ty, (
unsigned)
Insn->getOrdering()));
217 Args.push_back(ConstantInt::get(Int8Ty, (
unsigned)
Insn->getSyncScopeID()));
218 Args.push_back(ConstantInt::get(Int8Ty, AlignShiftValue));
219 Args.push_back(ConstantInt::get(Int1Ty,
GEP.InBounds));
220 Args.append(
GEP.Indices.begin(),
GEP.Indices.end());
228 {Load->getType()}, Args);
231 Call->setName((*
GEP.Members.rbegin())->getName());
232 if (Load->isUnordered()) {
233 Call->setOnlyReadsMemory();
234 Call->setOnlyAccessesArgMemory();
238 Call->addParamAttr(
I, Attribute::ImmArg);
239 Call->setAAMetadata(Load->getAAMetadata());
246 Args.push_back(Store->getValueOperand());
250 {Store->getValueOperand()->
getType()}, Args);
252 if (Store->getValueOperand()->getType()->isPointerTy())
255 Store->getDebugLoc());
256 if (Store->isUnordered()) {
257 Call->setOnlyWritesMemory();
258 Call->setOnlyAccessesArgMemory();
262 Call->addParamAttr(
I, Attribute::ImmArg);
263 Call->setAAMetadata(Store->getAAMetadata());
268 if (
auto *
Int = dyn_cast<ConstantInt>(Call->getOperand(ArgNo)))
269 return Int->getValue().getZExtValue();
272 ReportS <<
"Expecting ConstantInt as argument #" << ArgNo <<
" of " << *Call
279 Indices.
append(Call->data_operands_begin() + 6 + Delta,
280 Call->data_operands_end());
281 Type *GEPPointeeType = Call->getParamElementType(Delta);
289template <
class T = std::disjunction<LoadInst, StoreInst>>
296 Insn->setAlignment(
Align(1ULL << AlignShiftValue));
297 GEP->setDebugLoc(Call->getDebugLoc());
298 Insn->setDebugLoc(Call->getDebugLoc());
299 Insn->setAAMetadata(Call->getAAMetadata());
302std::pair<GetElementPtrInst *, LoadInst *>
305 Type *ReturnType = Call->getFunctionType()->getReturnType();
310 return std::pair{
GEP, Load};
313std::pair<GetElementPtrInst *, StoreInst *>
320 return std::pair{
GEP, Store};
324 auto *CI = dyn_cast<ConstantInt>(V);
325 return CI && CI->isZero();
332 GEPChainInfo &Info) {
337 return GEP->hasAllConstantIndices();
343 Info.SourceElementType =
First->getSourceElementType();
344 Type *ResultElementType =
First->getResultElementType();
348 for (
auto *Iter = GEPs.
begin() + 1; Iter != GEPs.
end(); ++Iter) {
354 if (!
GEP->getSourceElementType() ||
355 GEP->getSourceElementType() != ResultElementType) {
359 Info.InBounds &=
GEP->isInBounds();
360 Info.Indices.append(
GEP->idx_begin() + 1,
GEP->idx_end());
362 ResultElementType =
GEP->getResultElementType();
372 GEPChainInfo &Info) {
379 Type *PtrTy =
First->getType()->getScalarType();
386 Info.InBounds &=
GEP->isInBounds();
390 Info.Indices.push_back(ConstantInt::get(
C,
Offset));
397 *
Insn->getFunction(),
398 Twine(
"Non-constant offset in access to a field of a type marked "
399 "with preserve_static_offset might be rejected by BPF verifier")
402 :
" (pass -g option to get exact location)"),
404 Insn->getContext().diagnose(Msg);
409 return GEP->hasAllZeroIndices();
416 GEPChainInfo GEPChain;
422 if (
auto *Load = dyn_cast<LoadInst>(LoadOrStoreTemplate)) {
427 if (
auto *Store = dyn_cast<StoreInst>(LoadOrStoreTemplate)) {
436 if (
auto *L = dyn_cast<LoadInst>(U))
437 return L->getPointerOperand() ==
I;
438 if (
auto *S = dyn_cast<StoreInst>(U))
439 return S->getPointerOperand() ==
I;
440 if (
auto *
GEP = dyn_cast<GetElementPtrInst>(U))
441 return GEP->getPointerOperand() ==
I;
443 return Call->getArgOperand(0) ==
I;
445 return Call->getArgOperand(1) ==
I;
450 if (
auto *Call = dyn_cast<CallInst>(U))
451 return Call->hasFnAttr(Attribute::InlineHint);
458 bool AllowPatial,
bool &StillUsed);
465 auto *UI = dyn_cast<Instruction>(U);
471 llvm::dbgs() <<
"unsupported usage in BPFPreserveStaticOffsetPass:\n";
504 bool AllowPatial,
bool &StillUsed) {
505 auto MarkAndTraverseUses = [&]() {
509 auto TryToReplace = [&](
Instruction *LoadOrStore) {
523 if (isa<LoadInst>(
Insn) || isa<StoreInst>(
Insn)) {
544 }
else if (
auto *
GEP = dyn_cast<GetElementPtrInst>(
Insn)) {
546 MarkAndTraverseUses();
549 MarkAndTraverseUses();
564 Twine(
"Unexpected rewriteAccessChain Insn = ").
concat(Buf));
577 bool StillUsed =
false;
578 rewriteUses(Marker, GEPs, Visited, AllowPatial, StillUsed);
581 for (
auto V = Visited.
rbegin(); V != Visited.
rend(); ++V) {
584 RemovedMarkers.
insert(*V);
585 }
else if ((*V)->use_empty()) {
586 (*V)->eraseFromParent();
592static std::vector<Instruction *>
594 std::vector<Instruction *> Calls;
597 Calls.push_back(&
Insn);
615 if (
auto *
GEP = dyn_cast<GetElementPtrInst>(U))
616 return GEP->getPointerOperand() ==
Op;
619 return cast<CallInst>(U)->getArgOperand(0) ==
Op;
627 for (
User *U : V->users())
628 if (IsPointerOperand(V, U))
630 auto *Call = dyn_cast<CallInst>(V);
639 }
while (!WorkList.
empty());
649 LLVM_DEBUG(
dbgs() <<
"********** BPFPreserveStaticOffsetPass (AllowPartial="
650 << AllowPartial <<
") ************\n");
656 <<
" preserve.static.offset calls\n");
658 if (MarkerCalls.empty())
661 for (
auto *Call : MarkerCalls)
664 for (
auto *Call : MarkerCalls) {
668 if (!StillUsed || !AllowPartial)
SmallVector< AArch64_IMM::ImmInsnModel, 4 > Insn
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Expand Atomic instructions
This file contains the simple types necessary to represent the attributes associated with functions a...
static CallInst * isGEPAndLoad(Value *I)
bool isPreserveUnionIndex(Value *V)
bool isPreserveArrayIndex(Value *V)
static bool isPreserveStaticOffsetCall(Value *I)
static void setParamElementType(CallInst *Call, unsigned ArgNo, Type *Type)
static bool foldGEPChainAsU8Access(SmallVector< GetElementPtrInst * > &GEPs, GEPChainInfo &Info)
static void fillCommonArgs(LLVMContext &C, SmallVector< Value * > &Args, GEPChainInfo &GEP, T *Insn)
static void removePAICalls(Instruction *Marker)
static void reportNonStaticGEPChain(Instruction *Insn)
static bool foldGEPChainAsStructAccess(SmallVector< GetElementPtrInst * > &GEPs, GEPChainInfo &Info)
static const unsigned GepAndStoreFirstIdxArg
static void removeMarkerCall(Instruction *Marker)
static Instruction * makeGEPAndStore(Module *M, GEPChainInfo &GEP, StoreInst *Store)
static void rewriteUses(Instruction *Insn, SmallVector< GetElementPtrInst * > &GEPs, SmallVector< Instruction * > &Visited, bool AllowPatial, bool &StillUsed)
static void setParamReadNone(CallInst *Call, unsigned ArgNo)
static Instruction * makeGEPAndLoad(Module *M, GEPChainInfo &GEP, LoadInst *Load)
static unsigned getOperandAsUnsigned(CallInst *Call, unsigned ArgNo)
bool isPreserveStructIndex(Value *V)
static void setParamReadOnly(CallInst *Call, unsigned ArgNo)
static void rewriteAccessChain(Instruction *Insn, SmallVector< GetElementPtrInst * > &GEPs, SmallVector< Instruction * > &Visited, bool AllowPatial, bool &StillUsed)
static bool isInlineableCall(User *U)
static DILocation * mergeDILocations(SmallVector< T * > &Insns)
static const unsigned GepAndLoadFirstIdxArg
static GetElementPtrInst * reconstructGEP(CallInst *Call, int Delta)
static CallInst * makeIntrinsicCall(Module *M, Intrinsic::BPFIntrinsics Intrinsic, ArrayRef< Type * > Types, ArrayRef< Value * > Args)
static bool allZeroIndices(SmallVector< GetElementPtrInst * > &GEPs)
static std::vector< Instruction * > collectPreserveStaticOffsetCalls(Function &F)
static bool rewriteFunction(Function &F, bool AllowPartial)
static bool tryToReplaceWithGEPBuiltin(Instruction *LoadOrStoreTemplate, SmallVector< GetElementPtrInst * > &GEPs, Instruction *InsnToReplace)
static void reconstructCommon(CallInst *Call, GetElementPtrInst *GEP, T *Insn, int Delta)
static CallInst * isGEPAndStore(Value *I)
static void setParamWriteOnly(CallInst *Call, unsigned ArgNo)
static bool isIntrinsicCall(Value *I, Intrinsic::ID Id)
static bool isPointerOperand(Value *I, User *U)
static bool isIntrinsicCall(const CallBase *Call, Intrinsic::ID IID)
Analysis containing CSE Info
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
Return the first found DebugLoc that has a DILocation, given a range of instructions.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
static SymbolRef::Type getType(const Symbol *Sym)
Class for arbitrary precision integers.
A container for analyses that lazily runs them and caches their results.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
static void removeArrayAccessCall(CallInst *Call)
static void removeStructAccessCall(CallInst *Call)
static void removeUnionAccessCall(CallInst *Call)
static std::pair< GetElementPtrInst *, StoreInst * > reconstructStore(CallInst *Call)
static std::pair< GetElementPtrInst *, LoadInst * > reconstructLoad(CallInst *Call)
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr, BasicBlock::iterator InsertBefore)
static DILocation * getMergedLocation(DILocation *LocA, DILocation *LocB)
When two instructions are combined into a single instruction we also need to combine the original loc...
A parsed version of the target data layout string in and methods for querying it.
Diagnostic information for unsupported feature in backend.
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
static GetElementPtrInst * Create(Type *PointeeType, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &NameStr, BasicBlock::iterator InsertBefore)
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction.
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
A Module instance is used to store all the information related to an LLVM module.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
reverse_iterator rbegin()
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
static IntegerType * getInt1Ty(LLVMContext &C)
static IntegerType * getInt8Ty(LLVMContext &C)
Value * getOperand(unsigned i) const
LLVM Value Representation.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
A raw_ostream that writes to an std::string.
A raw_ostream that writes to an SmallVector or SmallString.
@ C
The default llvm calling convention, compatible with C.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
This is an optimization pass for GlobalISel generic memory operations.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
detail::concat_range< ValueT, RangeTs... > concat(RangeTs &&... Ranges)
Concatenated range across two or more ranges.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
DWARFExpression::Operation Op
This struct is a compact representation of a valid (non-zero power of two) alignment.