181 using namespace llvm;
182 using namespace llvm::PatternMatch;
185 "disable-separate-const-offset-from-gep",
cl::init(
false),
186 cl::desc(
"Do not separate the constant offset from a GEP instruction"),
193 cl::desc(
"Verify this pass produces no dead code"),
211 class ConstantOffsetExtractor {
230 :
IP(InsertionPt),
DL(InsertionPt->getModule()->getDataLayout()), DT(DT) {
246 APInt find(
Value *V,
bool SignExtended,
bool ZeroExtended,
bool NonNegative);
265 Value *rebuildWithoutConstOffset();
281 Value *distributeExtsAndCloneChain(
unsigned ChainIndex);
283 Value *removeConstOffset(
unsigned ChainIndex);
296 bool CanTraceInto(
bool SignExtended,
bool ZeroExtended,
BinaryOperator *BO,
317 class SeparateConstOffsetFromGEP :
public FunctionPass {
321 bool LowerGEP =
false)
335 bool doInitialization(
Module &M)
override {
339 bool runOnFunction(
Function &
F)
override;
353 int64_t AccumulativeByteOffset);
362 int64_t AccumulativeByteOffset);
403 bool hasMoreThanOneUseInLoop(
Value *v,
Loop *
L);
426 SeparateConstOffsetFromGEP,
"separate-const-offset-from-gep",
427 "Split GEPs to a variadic base and a constant offset for better CSE",
false,
435 SeparateConstOffsetFromGEP, "separate-
const-offset-from-
gep",
436 "
Split GEPs to a variadic base and a constant offset
for better
CSE",
false,
442 return new SeparateConstOffsetFromGEP(TM, LowerGEP);
445 bool ConstantOffsetExtractor::CanTraceInto(
bool SignExtended,
485 if (
ConstantInt *ConstLHS = dyn_cast<ConstantInt>(LHS)) {
486 if (!ConstLHS->isNegative())
489 if (
ConstantInt *ConstRHS = dyn_cast<ConstantInt>(RHS)) {
490 if (!ConstRHS->isNegative())
520 if (ConstantOffset != 0)
return ConstantOffset;
521 ConstantOffset =
find(BO->
getOperand(1), SignExtended, ZeroExtended,
526 ConstantOffset = -ConstantOffset;
527 return ConstantOffset;
531 bool ZeroExtended,
bool NonNegative) {
539 if (U ==
nullptr)
return APInt(BitWidth, 0);
541 APInt ConstantOffset(BitWidth, 0);
544 ConstantOffset = CI->getValue();
547 if (CanTraceInto(SignExtended, ZeroExtended, BO, NonNegative))
548 ConstantOffset = findInEitherOperand(BO, SignExtended, ZeroExtended);
549 }
else if (isa<SExtInst>(V)) {
551 ZeroExtended, NonNegative).sext(BitWidth);
552 }
else if (isa<ZExtInst>(V)) {
559 true,
false).zext(BitWidth);
565 if (ConstantOffset != 0)
566 UserChain.push_back(U);
567 return ConstantOffset;
570 Value *ConstantOffsetExtractor::applyExts(
Value *V) {
574 for (
auto I = ExtInsts.rbegin(),
E = ExtInsts.rend();
I !=
E; ++
I) {
575 if (
Constant *
C = dyn_cast<Constant>(Current)) {
589 Value *ConstantOffsetExtractor::rebuildWithoutConstOffset() {
590 distributeExtsAndCloneChain(UserChain.size() - 1);
592 unsigned NewSize = 0;
593 for (
User *
I : UserChain) {
595 UserChain[NewSize] =
I;
599 UserChain.resize(NewSize);
600 return removeConstOffset(UserChain.size() - 1);
604 ConstantOffsetExtractor::distributeExtsAndCloneChain(
unsigned ChainIndex) {
605 User *U = UserChain[ChainIndex];
606 if (ChainIndex == 0) {
607 assert(isa<ConstantInt>(U));
609 return UserChain[ChainIndex] = cast<ConstantInt>(applyExts(U));
612 if (
CastInst *Cast = dyn_cast<CastInst>(U)) {
613 assert((isa<SExtInst>(Cast) || isa<ZExtInst>(Cast)) &&
614 "We only traced into two types of CastInst: sext and zext");
615 ExtInsts.push_back(Cast);
616 UserChain[ChainIndex] =
nullptr;
617 return distributeExtsAndCloneChain(ChainIndex - 1);
623 unsigned OpNo = (BO->
getOperand(0) == UserChain[ChainIndex - 1] ? 0 : 1);
625 Value *NextInChain = distributeExtsAndCloneChain(ChainIndex - 1);
635 return UserChain[ChainIndex] = NewBO;
638 Value *ConstantOffsetExtractor::removeConstOffset(
unsigned ChainIndex) {
639 if (ChainIndex == 0) {
640 assert(isa<ConstantInt>(UserChain[ChainIndex]));
646 "distributeExtsAndCloneChain clones each BinaryOperator in "
647 "UserChain, so no one should be used more than "
650 unsigned OpNo = (BO->
getOperand(0) == UserChain[ChainIndex - 1] ? 0 : 1);
652 Value *NextInChain = removeConstOffset(ChainIndex - 1);
657 if (
ConstantInt *CI = dyn_cast<ConstantInt>(NextInChain)) {
658 if (CI->isZero() && !(BO->
getOpcode() == Instruction::Sub && OpNo == 0))
691 User *&UserChainTail,
693 ConstantOffsetExtractor Extractor(GEP, DT);
695 APInt ConstantOffset =
696 Extractor.find(Idx,
false,
false,
698 if (ConstantOffset == 0) {
699 UserChainTail =
nullptr;
703 Value *IdxWithoutConstOffset = Extractor.rebuildWithoutConstOffset();
704 UserChainTail = Extractor.UserChain.back();
705 return IdxWithoutConstOffset;
711 return ConstantOffsetExtractor(GEP, DT)
712 .find(Idx,
false,
false,
717 bool SeparateConstOffsetFromGEP::canonicalizeArrayIndicesToPointerSize(
719 bool Changed =
false;
723 I !=
E; ++
I, ++GTI) {
726 if ((*I)->getType() != IntPtrTy) {
737 bool &NeedsExtraction) {
738 NeedsExtraction =
false;
739 int64_t AccumulativeByteOffset = 0;
744 int64_t ConstantOffset =
746 if (ConstantOffset != 0) {
747 NeedsExtraction =
true;
751 AccumulativeByteOffset +=
754 }
else if (LowerGEP) {
759 NeedsExtraction =
true;
760 AccumulativeByteOffset +=
761 DL->getStructLayout(StTy)->getElementOffset(Field);
765 return AccumulativeByteOffset;
768 void SeparateConstOffsetFromGEP::lowerToSingleIndexGEPs(
778 bool isSwapCandidate =
780 !hasMoreThanOneUseInLoop(ResultPtr, L);
781 Value *FirstResult =
nullptr;
783 if (ResultPtr->
getType() != I8PtrTy)
784 ResultPtr = Builder.CreateBitCast(ResultPtr, I8PtrTy);
800 if (ElementSize != 1) {
802 Idx = Builder.CreateShl(
810 Builder.CreateGEP(Builder.getInt8Ty(), ResultPtr, Idx,
"uglygep");
811 if (FirstResult ==
nullptr)
812 FirstResult = ResultPtr;
817 if (AccumulativeByteOffset != 0) {
820 Builder.CreateGEP(Builder.getInt8Ty(), ResultPtr,
Offset,
"uglygep");
822 isSwapCandidate =
false;
829 if (isSwapCandidate && isLegalToSwapOperand(FirstGEP, SecondGEP, L))
830 swapGEPOperand(FirstGEP, SecondGEP);
833 ResultPtr = Builder.CreateBitCast(ResultPtr, Variadic->
getType());
841 int64_t AccumulativeByteOffset) {
845 Value *ResultPtr = Builder.CreatePtrToInt(Variadic->
getOperand(0), IntPtrTy);
861 if (ElementSize != 1) {
863 Idx = Builder.CreateShl(
870 ResultPtr = Builder.CreateAdd(ResultPtr, Idx);
875 if (AccumulativeByteOffset != 0) {
876 ResultPtr = Builder.CreateAdd(
880 ResultPtr = Builder.CreateIntToPtr(ResultPtr, Variadic->
getType());
895 bool Changed = canonicalizeArrayIndicesToPointerSize(GEP);
897 bool NeedsExtraction;
898 int64_t AccumulativeByteOffset = accumulateByteOffset(GEP, NeedsExtraction);
900 if (!NeedsExtraction)
911 getAnalysis<TargetTransformInfoWrapperPass>().getTTI(
915 nullptr, AccumulativeByteOffset,
937 ConstantOffsetExtractor::Extract(OldIdx, GEP, UserChainTail, DT);
938 if (NewIdx !=
nullptr) {
976 lowerToSingleIndexGEPs(GEP, AccumulativeByteOffset);
978 lowerToArithmetics(GEP, AccumulativeByteOffset);
983 if (AccumulativeByteOffset == 0)
1020 int64_t ElementTypeSizeOfGEP =
static_cast<int64_t
>(
1023 if (AccumulativeByteOffset % ElementTypeSizeOfGEP == 0) {
1026 int64_t Index = AccumulativeByteOffset / ElementTypeSizeOfGEP;
1031 cast<GetElementPtrInst>(NewGEP)->setIsInBounds(GEPWasInBounds);
1049 NewGEP =
new BitCastInst(NewGEP, I8PtrTy,
"", GEP);
1055 cast<GetElementPtrInst>(NewGEP)->setIsInBounds(GEPWasInBounds);
1056 if (GEP->
getType() != I8PtrTy)
1066 bool SeparateConstOffsetFromGEP::runOnFunction(
Function &
F) {
1067 if (skipFunction(F))
1073 DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
1074 SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
1075 LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
1076 TLI = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
1077 bool Changed =
false;
1081 Changed |= splitGEP(GEP);
1086 Changed |= reuniteExts(F);
1089 verifyNoDeadCode(F);
1094 Instruction *SeparateConstOffsetFromGEP::findClosestMatchingDominator(
1096 auto Pos = DominatingExprs.find(Key);
1097 if (Pos == DominatingExprs.end())
1100 auto &Candidates = Pos->second;
1105 while (!Candidates.empty()) {
1107 if (DT->
dominates(Candidate, Dominatee))
1109 Candidates.pop_back();
1114 bool SeparateConstOffsetFromGEP::reuniteExts(
Instruction *
I) {
1115 if (!SE->isSCEVable(I->
getType()))
1122 Value *LHS =
nullptr, *RHS =
nullptr;
1125 if (LHS->
getType() == RHS->getType()) {
1127 SE->getAddExpr(SE->getUnknown(LHS), SE->getUnknown(RHS));
1128 if (
auto *Dom = findClosestMatchingDominator(Key, I)) {
1143 SE->getAddExpr(SE->getUnknown(LHS), SE->getUnknown(RHS));
1144 DominatingExprs[Key].push_back(I);
1150 bool SeparateConstOffsetFromGEP::reuniteExts(
Function &F) {
1151 bool Changed =
false;
1152 DominatingExprs.clear();
1155 for (
auto I = BB->
begin(); I != BB->
end(); ) {
1157 Changed |= reuniteExts(Cur);
1163 void SeparateConstOffsetFromGEP::verifyNoDeadCode(
Function &F) {
1167 std::string ErrMessage;
1169 RSO <<
"Dead instruction detected!\n" << I <<
"\n";
1176 bool SeparateConstOffsetFromGEP::isLegalToSwapOperand(
1178 if (!FirstGEP || !FirstGEP->
hasOneUse())
1184 if (FirstGEP == SecondGEP)
1190 if (FirstNum != SecondNum || FirstNum != 2)
1215 if (FirstOffsetDef && FirstOffsetDef->
isShift() &&
1216 isa<ConstantInt>(FirstOffsetDef->
getOperand(1)))
1222 if (
BinaryOperator *BO = dyn_cast<BinaryOperator>(FirstOffsetDef)) {
1232 bool SeparateConstOffsetFromGEP::hasMoreThanOneUseInLoop(
Value *V,
Loop *L) {
1237 if (++UsesInLoop > 1)
1253 cast<PointerType>(First->
getType())->getAddressSpace()),
1257 uint64_t ObjectSize;
1259 Offset.ugt(ObjectSize)) {
static unsigned getBitWidth(Type *Ty, const DataLayout &DL)
Returns the bitwidth of the given scalar or pointer type (if unknown returns 0).
INITIALIZE_PASS_BEGIN(SeparateConstOffsetFromGEP,"separate-const-offset-from-gep","Split GEPs to a variadic base and a constant offset for better CSE", false, false) INITIALIZE_PASS_END(SeparateConstOffsetFromGEP
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
OverflowingBinaryOp_match< LHS, RHS, Instruction::Sub, OverflowingBinaryOperator::NoSignedWrap > m_NSWSub(const LHS &L, const RHS &R)
A parsed version of the target data layout string in and methods for querying it. ...
Type * getIndexedType() const
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
BinaryOp_match< LHS, RHS, Instruction::Sub > m_Sub(const LHS &L, const RHS &R)
A Module instance is used to store all the information related to an LLVM module. ...
bool isKnownNotFullPoison(const Instruction *PoisonI)
Return true if this function can prove that if PoisonI is executed and yields a full-poison value (al...
unsigned getNumOperands() const
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
The main scalar evolution driver.
static cl::opt< bool > VerifyNoDeadCode("reassociate-geps-verify-no-dead-code", cl::init(false), cl::desc("Verify this pass produces no dead code"), cl::Hidden)
const Function * getParent() const
Return the enclosing method, or null if none.
This class represents a sign extension of integer types.
FunctionType * getType(LLVMContext &Context, ID id, ArrayRef< Type * > Tys=None)
Return the function type for an intrinsic.
bool isSequential() const
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
StringRef getName() const
Return a constant reference to the value's name.
iterator begin()
Instruction iterator methods.
bool match(Val *V, const Pattern &P)
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
This is the base class for all instructions that perform data casts.
Class to represent struct types.
A Use represents the edge between a Value definition and its users.
void setIsInBounds(bool b=true)
Set or clear the inbounds flag on this GEP instruction.
separate const offset from Split GEPs to a variadic base and a constant offset for better CSE
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
StructType * getStructType() const
bool isLoopInvariant(const Value *V) const
Return true if the specified value is loop invariant.
Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following: ...
Value * stripAndAccumulateInBoundsConstantOffsets(const DataLayout &DL, APInt &Offset)
Accumulate offsets from stripInBoundsConstantOffsets().
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
This class represents a no-op cast from one type to another.
static GCRegistry::Add< OcamlGC > B("ocaml","ocaml 3.10-compatible GC")
bool getObjectSize(const Value *Ptr, uint64_t &Size, const DataLayout &DL, const TargetLibraryInfo *TLI, bool RoundToAlign=false, ObjSizeMode Mode=ObjSizeMode::Exact)
Compute the size of the object pointed by Ptr.
bool haveNoCommonBitsSet(const Value *LHS, const Value *RHS, const DataLayout &DL, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr)
Return true if LHS and RHS have no common bits set.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
void takeName(Value *V)
Transfer the name from V to this value.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
void initializeSeparateConstOffsetFromGEPPass(PassRegistry &)
bool isInBounds() const
Determine whether the GEP has the inbounds flag.
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
an instruction for type-safe pointer arithmetic to access elements of arrays and structs ...
initializer< Ty > init(const Ty &Val)
FunctionPass * createSeparateConstOffsetFromGEPPass(const TargetMachine *TM=nullptr, bool LowerGEP=false)
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction...
LLVM Basic Block Representation.
The instances of the Type class are immutable: once they are created, they are never changed...
bool isVectorTy() const
True if this is an instance of VectorType.
This is an important base class in LLVM.
static const SubtargetFeatureKV * Find(StringRef S, ArrayRef< SubtargetFeatureKV > A)
Find KV in array using binary search.
This file contains the declarations for the subclasses of Constant, which represent the different fla...
APInt Or(const APInt &LHS, const APInt &RHS)
Bitwise OR function for APInt.
bool hasNoSignedWrap() const
Determine whether the no signed wrap flag is set.
Represent the analysis usage information of a pass.
bool contains(const LoopT *L) const
Return true if the specified loop is contained within in this loop.
static cl::opt< bool > DisableSeparateConstOffsetFromGEP("disable-separate-const-offset-from-gep", cl::init(false), cl::desc("Do not separate the constant offset from a GEP instruction"), cl::Hidden)
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE,"Assign register bank of generic virtual registers", false, false) RegBankSelect
for(unsigned i=0, e=MI->getNumOperands();i!=e;++i)
FunctionPass class - This class is used to implement most global optimizations.
Value * getOperand(unsigned i) const
unsigned getIntegerBitWidth() const
bool RecursivelyDeleteTriviallyDeadInstructions(Value *V, const TargetLibraryInfo *TLI=nullptr)
If the specified value is a trivially dead instruction, delete it.
bool hasAllConstantIndices() const
Return true if all of the indices of this GEP are constant integers.
LLVMContext & getContext() const
All values hold a context through their type.
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static GetElementPtrInst * Create(Type *PointeeType, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
bool dominates(const Instruction *Def, const Use &U) const
Return true if Def dominates a use in User.
CastClass_match< OpTy, Instruction::SExt > m_SExt(const OpTy &Op)
Matches SExt.
BinaryOps getOpcode() const
static CastInst * CreateIntegerCast(Value *S, Type *Ty, bool isSigned, const Twine &Name="", Instruction *InsertBefore=nullptr)
Create a ZExt, BitCast, or Trunc for int -> int casts.
Iterator for intrusive lists based on ilist_node.
This is the shared class of boolean and integer constants.
auto find(R &&Range, const T &Val) -> decltype(std::begin(Range))
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly...
bool hasNoUnsignedWrap() const
Determine whether the no unsigned wrap flag is set.
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
unsigned logBase2() const
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Module.h This file contains the declarations for the Module class.
Type * getType() const
All values are typed, get the type of this value.
Provides information about what library functions are available for the current target.
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
OverflowingBinaryOp_match< LHS, RHS, Instruction::Add, OverflowingBinaryOperator::NoSignedWrap > m_NSWAdd(const LHS &L, const RHS &R)
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
void setOperand(unsigned i, Value *Val)
Class for arbitrary precision integers.
static BinaryOperator * Create(BinaryOps Op, Value *S1, Value *S2, const Twine &Name=Twine(), Instruction *InsertBefore=nullptr)
Construct a binary instruction, given the opcode and the two operands.
iterator_range< user_iterator > users()
static Constant * getCast(unsigned ops, Constant *C, Type *Ty, bool OnlyIfReduced=false)
Convenience function for getting a Cast operation.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
This class represents an analyzed expression in the program.
Represents a single loop in the control flow graph.
bool hasOneUse() const
Return true if there is exactly one user of this value.
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
iterator_range< df_iterator< T > > depth_first(const T &G)
separate const offset from gep
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
A raw_ostream that writes to an std::string.
bool isInstructionTriviallyDead(Instruction *I, const TargetLibraryInfo *TLI=nullptr)
Return true if the result produced by the instruction is not used, and the instruction has no side ef...
LLVM Value Representation.
Primary interface to the complete machine description for the target machine.
The legacy pass manager's analysis pass to compute loop information.
Legacy analysis pass which computes a DominatorTree.
unsigned getNumUses() const
This method computes the number of uses of this Value.
static void Split(std::vector< std::string > &V, StringRef S)
Split - Splits a string of comma separated items in to a vector of strings.
separate const offset from Split GEPs to a variadic base and a constant offset for better false
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
static IntegerType * getInt8Ty(LLVMContext &C)
const BasicBlock * getParent() const
Type * getResultElementType() const
gep_type_iterator gep_type_begin(const User *GEP)