75 using namespace PatternMatch;
83 struct Candidate :
public ilist_node<Candidate> {
92 : CandidateKind(
Invalid), Base(nullptr), Index(nullptr),
93 Stride(nullptr),
Ins(nullptr), Basis(nullptr) {}
96 : CandidateKind(CT), Base(B), Index(Idx), Stride(S),
Ins(I),
127 StraightLineStrengthReduce()
140 bool doInitialization(
Module &M)
override {
145 bool runOnFunction(
Function &
F)
override;
150 bool isBasisFor(
const Candidate &Basis,
const Candidate &C);
156 bool isSimplestForm(
const Candidate &C);
161 void allocateCandidatesAndFindBasisForAdd(
Instruction *
I);
164 void allocateCandidatesAndFindBasisForAdd(
Value *LHS,
Value *RHS,
167 void allocateCandidatesAndFindBasisForMul(
Instruction *
I);
170 void allocateCandidatesAndFindBasisForMul(
Value *LHS,
Value *RHS,
176 void allocateCandidatesAndFindBasisForGEP(
const SCEV *B,
ConstantInt *Idx,
177 Value *S, uint64_t ElementSize,
185 void rewriteCandidateWithBasis(
const Candidate &C,
const Candidate &Basis);
189 void factorArrayIndex(
Value *ArrayIdx,
const SCEV *Base, uint64_t ElementSize,
195 static Value *emitBump(
const Candidate &Basis,
const Candidate &C,
197 bool &BumpWithUglyGEP);
207 std::vector<Instruction *> UnlinkedInstructions;
213 "Straight line strength reduction",
false,
false)
221 return new StraightLineStrengthReduce();
224 bool StraightLineStrengthReduce::isBasisFor(
const Candidate &Basis,
225 const Candidate &C) {
226 return (Basis.Ins != C.Ins &&
229 Basis.Ins->getType() == C.Ins->getType() &&
231 DT->dominates(Basis.Ins->getParent(), C.Ins->getParent()) &&
233 Basis.Base == C.Base && Basis.Stride == C.Stride &&
234 Basis.CandidateKind == C.CandidateKind);
241 int64_t BaseOffset = 0;
242 bool HasBaseReg =
false;
252 if (isa<SequentialType>(*GTI)) {
254 if (
ConstantInt *ConstIdx = dyn_cast<ConstantInt>(*
I)) {
255 BaseOffset += ConstIdx->getSExtValue() * ElementSize;
266 uint64_t
Field = cast<ConstantInt>(*I)->getZExtValue();
273 BaseOffset, HasBaseReg, Scale, AddrSpace);
283 bool StraightLineStrengthReduce::isFoldable(
const Candidate &C,
286 if (C.CandidateKind == Candidate::Add)
289 return isGEPFoldable(cast<GetElementPtrInst>(C.Ins), TTI, DL);
295 unsigned NumNonZeroIndices = 0;
298 if (ConstIdx ==
nullptr || !ConstIdx->
isZero())
301 return NumNonZeroIndices <= 1;
304 bool StraightLineStrengthReduce::isSimplestForm(
const Candidate &C) {
305 if (C.CandidateKind == Candidate::Add) {
307 return C.Index->isOne() || C.Index->isMinusOne();
309 if (C.CandidateKind == Candidate::Mul) {
311 return C.Index->isZero();
315 return ((C.Index->isOne() || C.Index->isMinusOne()) &&
328 void StraightLineStrengthReduce::allocateCandidatesAndFindBasis(
331 Candidate C(CT, B, Idx, S, I);
345 if (!isFoldable(C, TTI, DL) && !isSimplestForm(C)) {
347 unsigned NumIterations = 0;
349 static const unsigned MaxNumIterations = 50;
350 for (
auto Basis = Candidates.rbegin();
351 Basis != Candidates.rend() && NumIterations < MaxNumIterations;
352 ++Basis, ++NumIterations) {
353 if (isBasisFor(*Basis, C)) {
361 Candidates.push_back(C);
364 void StraightLineStrengthReduce::allocateCandidatesAndFindBasis(
367 case Instruction::Add:
368 allocateCandidatesAndFindBasisForAdd(I);
370 case Instruction::Mul:
371 allocateCandidatesAndFindBasisForMul(I);
373 case Instruction::GetElementPtr:
374 allocateCandidatesAndFindBasisForGEP(cast<GetElementPtrInst>(I));
379 void StraightLineStrengthReduce::allocateCandidatesAndFindBasisForAdd(
382 if (!isa<IntegerType>(I->
getType()))
387 allocateCandidatesAndFindBasisForAdd(LHS, RHS, I);
389 allocateCandidatesAndFindBasisForAdd(RHS, LHS, I);
392 void StraightLineStrengthReduce::allocateCandidatesAndFindBasisForAdd(
398 allocateCandidatesAndFindBasis(Candidate::Add, SE->getSCEV(LHS), Idx, S,
I);
403 allocateCandidatesAndFindBasis(Candidate::Add, SE->getSCEV(LHS), Idx, S,
I);
407 allocateCandidatesAndFindBasis(Candidate::Add, SE->getSCEV(LHS),
One, RHS,
424 void StraightLineStrengthReduce::allocateCandidatesAndFindBasisForMul(
431 allocateCandidatesAndFindBasis(Candidate::Mul, SE->getSCEV(B), Idx, RHS,
I);
437 allocateCandidatesAndFindBasis(Candidate::Mul, SE->getSCEV(B), Idx, RHS,
I);
441 allocateCandidatesAndFindBasis(Candidate::Mul, SE->getSCEV(LHS), Zero, RHS,
446 void StraightLineStrengthReduce::allocateCandidatesAndFindBasisForMul(
450 if (!isa<IntegerType>(I->
getType()))
455 allocateCandidatesAndFindBasisForMul(LHS, RHS, I);
458 allocateCandidatesAndFindBasisForMul(RHS, LHS, I);
462 void StraightLineStrengthReduce::allocateCandidatesAndFindBasisForGEP(
471 IntPtrTy, Idx->
getSExtValue() * (int64_t)ElementSize,
true);
472 allocateCandidatesAndFindBasis(
Candidate::GEP, B, ScaledIdx, S, I);
475 void StraightLineStrengthReduce::factorArrayIndex(
Value *ArrayIdx,
477 uint64_t ElementSize,
480 allocateCandidatesAndFindBasisForGEP(
482 ArrayIdx, ElementSize,
GEP);
483 Value *LHS =
nullptr;
499 allocateCandidatesAndFindBasisForGEP(Base, RHS, LHS, ElementSize, GEP);
506 allocateCandidatesAndFindBasisForGEP(Base, PowerOf2, LHS, ElementSize, GEP);
510 void StraightLineStrengthReduce::allocateCandidatesAndFindBasisForGEP(
522 if (!isa<SequentialType>(*GTI++))
525 const SCEV *OrigIndexExpr = IndexExprs[I - 1];
526 IndexExprs[I - 1] = SE->getConstant(OrigIndexExpr->
getType(), 0);
535 factorArrayIndex(ArrayIdx, BaseExpr, ElementSize, GEP);
539 Value *TruncatedArrayIdx =
nullptr;
541 factorArrayIndex(TruncatedArrayIdx, BaseExpr, ElementSize, GEP);
543 IndexExprs[I - 1] = OrigIndexExpr;
555 Value *StraightLineStrengthReduce::emitBump(
const Candidate &Basis,
559 bool &BumpWithUglyGEP) {
560 APInt Idx = C.Index->getValue(), BasisIdx = Basis.Index->getValue();
562 APInt IndexOffset = Idx - BasisIdx;
564 BumpWithUglyGEP =
false;
569 cast<GetElementPtrInst>(Basis.Ins)->getType()->getElementType()));
575 BumpWithUglyGEP =
true;
594 return Builder.
CreateShl(ExtendedStride, Exponent);
596 if ((-IndexOffset).isPowerOf2()) {
603 return Builder.
CreateMul(ExtendedStride, Delta);
606 void StraightLineStrengthReduce::rewriteCandidateWithBasis(
607 const Candidate &C,
const Candidate &Basis) {
608 assert(C.CandidateKind == Basis.CandidateKind && C.Base == Basis.Base &&
609 C.Stride == Basis.Stride);
612 assert(Basis.Ins->getParent() !=
nullptr &&
"the basis is unlinked");
618 if (!C.Ins->getParent())
622 bool BumpWithUglyGEP;
623 Value *Bump = emitBump(Basis, C, Builder, DL, BumpWithUglyGEP);
624 Value *Reduced =
nullptr;
625 switch (C.CandidateKind) {
646 Reduced = Builder.
CreateAdd(Basis.Ins, Bump);
652 bool InBounds = cast<GetElementPtrInst>(C.Ins)->isInBounds();
653 if (BumpWithUglyGEP) {
655 unsigned AS = Basis.Ins->getType()->getPointerAddressSpace();
671 Reduced = Builder.
CreateGEP(
nullptr, Basis.Ins, Bump);
678 Reduced->takeName(C.Ins);
679 C.Ins->replaceAllUsesWith(Reduced);
682 C.Ins->removeFromParent();
683 UnlinkedInstructions.push_back(C.Ins);
686 bool StraightLineStrengthReduce::runOnFunction(
Function &
F) {
687 if (skipOptnoneFunction(F))
690 TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
691 DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
692 SE = &getAnalysis<ScalarEvolution>();
697 for (
auto &I : *node->getBlock())
698 allocateCandidatesAndFindBasis(&I);
703 while (!Candidates.empty()) {
704 const Candidate &C = Candidates.back();
705 if (C.Basis !=
nullptr) {
706 rewriteCandidateWithBasis(C, *C.Basis);
708 Candidates.pop_back();
712 for (
auto *UnlinkedInst : UnlinkedInstructions) {
713 for (
unsigned I = 0, E = UnlinkedInst->
getNumOperands(); I != E; ++
I) {
714 Value *Op = UnlinkedInst->getOperand(I);
715 UnlinkedInst->setOperand(I,
nullptr);
720 bool Ret = !UnlinkedInstructions.empty();
721 UnlinkedInstructions.clear();
FunctionPass * createStraightLineStrengthReducePass()
Value * CreateGEP(Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="")
void push_back(const T &Elt)
A parsed version of the target data layout string in and methods for querying it. ...
Type * getIndexedType() const
Value * getPointerOperand()
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Value * CreateSExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")
Create a SExt or Trunc from the integer value V to DestTy.
Type * getSourceElementType() const
A Module instance is used to store all the information related to an LLVM module. ...
DominatorTree GraphTraits specialization so the DominatorTree can be iterable by generic graph iterat...
unsigned getBitWidth() const
getBitWidth - Return the bitwidth of this constant.
unsigned getNumOperands() const
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
BinaryOp_match< LHS, RHS, Instruction::Mul > m_Mul(const LHS &L, const RHS &R)
ScalarEvolution - This class is the main scalar evolution driver.
static void unifyBitWidth(APInt &A, APInt &B)
bool haveNoCommonBitsSet(Value *LHS, Value *RHS, const DataLayout &DL, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr)
Returns true if LHS and RHS have no common bits set.
static void sdivrem(const APInt &LHS, const APInt &RHS, APInt &Quotient, APInt &Remainder)
static bool matchesAdd(Value *A, Value *&B, ConstantInt *&C)
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
bool match(Val *V, const Pattern &P)
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
static const Value * getNegArgument(const Value *BinOp)
Helper functions to extract the unary argument of a NEG, FNEG or NOT operation implemented via Sub...
static bool matchesOr(Value *A, Value *&B, ConstantInt *&C)
const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
const APInt & getValue() const
Return the constant as an APInt value reference.
StructType - Class to represent struct types.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
unsigned logBase2(const APInt &APIVal)
Returns the floor log base 2 of the specified APInt value.
static bool hasOnlyOneNonZeroIndex(GetElementPtrInst *GEP)
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
void initializeStraightLineStrengthReducePass(PassRegistry &)
class_match< ConstantInt > m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
INITIALIZE_PASS_BEGIN(StraightLineStrengthReduce,"slsr","Straight line strength reduction", false, false) INITIALIZE_PASS_END(StraightLineStrengthReduce
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
Type * getElementType() const
bool isInBounds() const
isInBounds - Determine whether the GEP has the inbounds flag.
uint64_t getElementOffset(unsigned Idx) const
GetElementPtrInst - an instruction for type-safe pointer arithmetic to access elements of arrays and ...
* if(!EatIfPresent(lltok::kw_thread_local)) return false
ParseOptionalThreadLocal := /*empty.
The instances of the Type class are immutable: once they are created, they are never changed...
BinaryOp_match< LHS, RHS, Instruction::Or > m_Or(const LHS &L, const RHS &R)
Type * getType() const
getType - Return the LLVM type of this SCEV expression.
bool isVectorTy() const
isVectorTy - True if this is an instance of VectorType.
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
This is an important base class in LLVM.
Straight line strength reduction
int64_t getSExtValue() const
Get sign extended value.
APInt LLVM_ATTRIBUTE_UNUSED_RESULT sext(unsigned width) const
Sign extend to a new width.
Represent the analysis usage information of a pass.
BinaryOp_match< LHS, RHS, Instruction::Shl > m_Shl(const LHS &L, const RHS &R)
unsigned getBitWidth() const
Return the number of bits in the APInt.
FunctionPass class - This class is used to implement most global optimizations.
Value * getOperand(unsigned i) const
Class to represent integer types.
Value * CreateInBoundsGEP(Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="")
bool RecursivelyDeleteTriviallyDeadInstructions(Value *V, const TargetLibraryInfo *TLI=nullptr)
RecursivelyDeleteTriviallyDeadInstructions - If the specified value is a trivially dead instruction...
LLVMContext & getContext() const
All values hold a context through their type.
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
CastClass_match< OpTy, Instruction::SExt > m_SExt(const OpTy &Op)
Matches SExt.
IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space...
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
static bool isGEPFoldable(GetElementPtrInst *GEP, const TargetTransformInfo *TTI, const DataLayout *DL)
This is the shared class of boolean and integer constants.
uint64_t getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
unsigned logBase2() const
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Module.h This file contains the declarations for the Module class.
Type * getType() const
All values are typed, get the type of this value.
SequentialType * getType() const
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Straight line strength false
static bool isNeg(const Value *V)
Check if the given Value is a NEG, FNeg, or NOT instruction.
Class for arbitrary precision integers.
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
LLVM_ATTRIBUTE_UNUSED_RESULT std::enable_if< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
SCEV - This class represents an analyzed expression in the program.
ilist_node - Base class that provides next/prev services for nodes that use ilist_nextprev_traits or ...
OverflowingBinaryOp_match< LHS, RHS, Instruction::Shl, OverflowingBinaryOperator::NoSignedWrap > m_NSWShl(const LHS &L, const RHS &R)
const ARM::ArchExtKind Kind
Value * CreateNeg(Value *V, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
LLVM Value Representation.
unsigned getOpcode() const
getOpcode() returns a member of one of the enums like Instruction::Add.
OverflowingBinaryOp_match< LHS, RHS, Instruction::Mul, OverflowingBinaryOperator::NoSignedWrap > m_NSWMul(const LHS &L, const RHS &R)
Legacy analysis pass which computes a DominatorTree.
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
static bool isAddFoldable(const SCEV *Base, ConstantInt *Index, Value *Stride, TargetTransformInfo *TTI)
gep_type_iterator gep_type_begin(const User *GEP)