58 #define DEBUG_TYPE "guard-widening"
62 class GuardWideningImpl {
79 bool eliminateGuardViaWidening(
102 static StringRef scoreTypeToString(WideningScore WS);
107 WideningScore computeWideningScore(
IntrinsicInst *DominatedGuard,
108 Loop *DominatedGuardLoop,
110 Loop *DominatingGuardLoop);
115 return isAvailableAt(V, InsertPos, Visited);
147 : Base(Base),
Offset(Offset), Length(Length), CheckInst(CheckInst) {}
149 void setBase(
Value *NewBase) { Base = NewBase; }
150 void setOffset(
ConstantInt *NewOffset) { Offset = NewOffset; }
152 Value *getBase()
const {
return Base; }
154 const APInt &getOffsetValue()
const {
return getOffset()->getValue(); }
155 Value *getLength()
const {
return Length; };
156 ICmpInst *getCheckInst()
const {
return CheckInst; }
178 return parseRangeChecks(CheckCond, Checks, Visited);
193 bool isWideningCondProfitable(
Value *Cond0,
Value *Cond1) {
195 return widenCondCommon(Cond0, Cond1,
nullptr, ResultUnused);
202 widenCondCommon(ToWiden->
getArgOperand(0), NewCondition, ToWiden, Result);
203 ToWiden->setArgOperand(0, Result);
209 : DT(DT), PDT(PDT), LI(LI) {}
223 bool runOnFunction(
Function &
F)
override {
226 return GuardWideningImpl(
227 getAnalysis<DominatorTreeWrapperPass>().getDomTree(),
228 getAnalysis<PostDominatorTreeWrapperPass>().getPostDomTree(),
229 getAnalysis<LoopInfoWrapperPass>().getLoopInfo()).run();
242 bool GuardWideningImpl::run() {
243 using namespace llvm::PatternMatch;
246 bool Changed =
false;
248 for (
auto DFI =
df_begin(DT.getRootNode()), DFE =
df_end(DT.getRootNode());
250 auto *BB = (*DFI)->getBlock();
251 auto &CurrentList = GuardsInBlock[BB];
254 if (
match(&
I, m_Intrinsic<Intrinsic::experimental_guard>()))
255 CurrentList.push_back(cast<IntrinsicInst>(&
I));
257 for (
auto *II : CurrentList)
258 Changed |= eliminateGuardViaWidening(II, DFI, GuardsInBlock);
261 for (
auto *II : EliminatedGuards)
262 if (!WidenedGuards.count(II))
263 II->eraseFromParent();
268 bool GuardWideningImpl::eliminateGuardViaWidening(
273 auto BestScoreSoFar = WS_IllegalOrNegative;
274 auto *GuardInstLoop = LI.getLoopFor(GuardInst->
getParent());
279 auto *CurBB = DFSI.
getPath(
i)->getBlock();
280 auto *CurLoop = LI.getLoopFor(CurBB);
281 assert(GuardsInBlock.count(CurBB) &&
"Must have been populated by now!");
282 const auto &GuardsInCurBB = GuardsInBlock.find(CurBB)->second;
284 auto I = GuardsInCurBB.begin();
285 auto E = GuardsInCurBB.end();
290 for (
auto &
I : *CurBB) {
291 if (Index == GuardsInCurBB.size())
293 if (GuardsInCurBB[Index] == &
I)
296 assert(Index == GuardsInCurBB.size() &&
297 "Guards expected to be in order!");
307 assert(NewEnd !=
E &&
"GuardInst not in its own block?");
313 computeWideningScore(GuardInst, GuardInstLoop, Candidate, CurLoop);
315 <<
" and " << *Candidate->getArgOperand(0) <<
" is "
316 << scoreTypeToString(Score) <<
"\n");
317 if (Score > BestScoreSoFar) {
318 BestScoreSoFar = Score;
319 BestSoFar = Candidate;
324 if (BestScoreSoFar == WS_IllegalOrNegative) {
325 DEBUG(
dbgs() <<
"Did not eliminate guard " << *GuardInst <<
"\n");
329 assert(BestSoFar != GuardInst &&
"Should have never visited same guard!");
330 assert(DT.dominates(BestSoFar, GuardInst) &&
"Should be!");
332 DEBUG(
dbgs() <<
"Widening " << *GuardInst <<
" into " << *BestSoFar
333 <<
" with score " << scoreTypeToString(BestScoreSoFar) <<
"\n");
336 EliminatedGuards.push_back(GuardInst);
337 WidenedGuards.insert(BestSoFar);
341 GuardWideningImpl::WideningScore GuardWideningImpl::computeWideningScore(
344 bool HoistingOutOfLoop =
false;
346 if (DominatingGuardLoop != DominatedGuardLoop) {
347 if (DominatingGuardLoop &&
348 !DominatingGuardLoop->
contains(DominatedGuardLoop))
349 return WS_IllegalOrNegative;
351 HoistingOutOfLoop =
true;
354 if (!isAvailableAt(DominatedGuard->
getArgOperand(0), DominatingGuard))
355 return WS_IllegalOrNegative;
357 bool HoistingOutOfIf =
360 if (isWideningCondProfitable(DominatedGuard->
getArgOperand(0),
362 return HoistingOutOfLoop ? WS_VeryPositive : WS_Positive;
364 if (HoistingOutOfLoop)
367 return HoistingOutOfIf ? WS_IllegalOrNegative : WS_Neutral;
373 if (!Inst || DT.dominates(Inst, Loc) || Visited.
count(Inst))
377 Inst->mayReadFromMemory())
383 assert(!isa<PHINode>(Loc) &&
384 "PHIs should return false for isSafeToSpeculativelyExecute");
385 assert(DT.isReachableFromEntry(Inst->getParent()) &&
386 "We did a DFS from the block entry!");
387 return all_of(Inst->operands(),
388 [&](
Value *
Op) {
return isAvailableAt(
Op, Loc, Visited); });
393 if (!Inst || DT.dominates(Inst, Loc))
397 !Inst->mayReadFromMemory() &&
"Should've checked with isAvailableAt!");
399 for (
Value *
Op : Inst->operands())
400 makeAvailableAt(
Op, Loc);
402 Inst->moveBefore(Loc);
405 bool GuardWideningImpl::widenCondCommon(
Value *Cond0,
Value *Cond1,
407 using namespace llvm::PatternMatch;
436 if (SubsetIntersect == SupersetIntersect &&
437 SubsetIntersect.getEquivalentICmp(Pred, NewRHSAP)) {
440 Result =
new ICmpInst(InsertPt, Pred, LHS, NewRHS,
"wide.chk");
449 if (parseRangeChecks(Cond0, Checks) && parseRangeChecks(Cond1, Checks) &&
450 combineRangeChecks(Checks, CombinedChecks)) {
453 for (
auto &RC : CombinedChecks) {
454 makeAvailableAt(RC.getCheckInst(), InsertPt);
456 Result = BinaryOperator::CreateAnd(RC.getCheckInst(), Result,
"",
459 Result = RC.getCheckInst();
471 makeAvailableAt(Cond0, InsertPt);
472 makeAvailableAt(Cond1, InsertPt);
474 Result = BinaryOperator::CreateAnd(Cond0, Cond1,
"wide.chk", InsertPt);
481 bool GuardWideningImpl::parseRangeChecks(
484 if (!Visited.
insert(CheckCond).second)
487 using namespace llvm::PatternMatch;
490 Value *AndLHS, *AndRHS;
492 return parseRangeChecks(AndLHS, Checks) &&
493 parseRangeChecks(AndRHS, Checks);
497 if (!IC || !IC->getOperand(0)->getType()->isIntegerTy() ||
502 Value *CmpLHS = IC->getOperand(0), *CmpRHS = IC->getOperand(1);
506 auto &
DL = IC->getModule()->getDataLayout();
508 GuardWideningImpl::RangeCheck
Check(
528 assert((!BaseInst || DT.isReachableFromEntry(BaseInst->getParent())) &&
529 "Unreachable instruction?");
533 Check.setBase(OpLHS);
540 APInt KnownZero(BitWidth, 0), KnownOne(BitWidth, 0);
543 Check.setBase(OpLHS);
555 bool GuardWideningImpl::combineRangeChecks(
558 unsigned OldCount = Checks.
size();
559 while (!Checks.
empty()) {
562 Value *CurrentBase = Checks.
front().getBase();
563 Value *CurrentLength = Checks.
front().getLength();
567 auto IsCurrentCheck = [&](GuardWideningImpl::RangeCheck &RC) {
568 return RC.getBase() == CurrentBase && RC.getLength() == CurrentLength;
571 std::copy_if(Checks.
begin(), Checks.
end(),
572 std::back_inserter(CurrentChecks), IsCurrentCheck);
575 assert(CurrentChecks.
size() != 0 &&
"We know we have at least one!");
577 if (CurrentChecks.
size() < 3) {
578 RangeChecksOut.
insert(RangeChecksOut.
end(), CurrentChecks.
begin(),
579 CurrentChecks.
end());
586 std::sort(CurrentChecks.
begin(), CurrentChecks.
end(),
587 [&](
const GuardWideningImpl::RangeCheck &LHS,
588 const GuardWideningImpl::RangeCheck &RHS) {
589 return LHS.getOffsetValue().slt(RHS.getOffsetValue());
595 *MaxOffset = CurrentChecks.
back().getOffset();
597 unsigned BitWidth = MaxOffset->getValue().getBitWidth();
598 if ((MaxOffset->getValue() - MinOffset->getValue())
602 APInt MaxDiff = MaxOffset->getValue() - MinOffset->getValue();
603 const APInt &HighOffset = MaxOffset->getValue();
604 auto OffsetOK = [&](
const GuardWideningImpl::RangeCheck &RC) {
605 return (HighOffset - RC.getOffsetValue()).ult(MaxDiff);
652 assert(RangeChecksOut.
size() <= OldCount &&
"We pessimized!");
653 return RangeChecksOut.
size() != OldCount;
661 bool Changed = GuardWideningImpl(DT, PDT, LI).
run();
665 StringRef GuardWideningImpl::scoreTypeToString(WideningScore WS) {
667 case WS_IllegalOrNegative:
668 return "IllegalOrNegative";
673 case WS_VeryPositive:
674 return "VeryPositive";
691 return new GuardWideningLegacyPass();
static bool Check(DecodeStatus &Out, DecodeStatus In)
PostDominatorTree run(Function &F, FunctionAnalysisManager &)
Run the analysis pass over a function and produce a post dominator tree.
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
void computeKnownBits(const Value *V, APInt &KnownZero, APInt &KnownOne, const DataLayout &DL, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
void push_back(const T &Elt)
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds...
auto remove_if(R &&Range, UnaryPredicate P) -> decltype(std::begin(Range))
Provide wrappers to std::remove_if which take ranges instead of having to pass begin/end explicitly...
Implements a dense probed hash-table based set.
void initializeGuardWideningLegacyPassPass(PassRegistry &)
size_type count(PtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly...
Analysis pass which computes a DominatorTree.
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool match(Val *V, const Pattern &P)
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
const APInt & getValue() const
Return the constant as an APInt value reference.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void setName(const Twine &Name)
Change the name of the value.
Analysis pass that exposes the LoopInfo for a function.
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
LLVM_NODISCARD bool empty() const
void printAsOperand(raw_ostream &O, bool PrintType=true, const Module *M=nullptr) const
Print the name of this Value out to the specified raw_ostream.
ConstantRange unionWith(const ConstantRange &CR) const
Return the range that results from the union of this range with another range.
class_match< ConstantInt > m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
NodeRef getPath(unsigned n) const
getPath - Return the n'th node in the path from the entry node to the current node.
A set of analyses that are preserved following a run of a transformation pass.
ConstantRange intersectWith(const ConstantRange &CR) const
Return the range that results from the intersection of this range with another range.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs...ExtraArgs)
Get the result of an analysis pass for a given IR unit.
LLVM Basic Block Representation.
BinaryOp_match< LHS, RHS, Instruction::Or > m_Or(const LHS &L, const RHS &R)
bool isKnownNonNegative(const Value *V, const DataLayout &DL, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr)
Returns true if the give value is known to be non-negative.
df_iterator< T > df_end(const T &G)
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
static void PrintTypes(formatted_raw_ostream &OS, ArrayRef< MVT > Types)
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
Represent the analysis usage information of a pass.
bool contains(const LoopT *L) const
Return true if the specified loop is contained within in this loop.
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
This instruction compares its operands according to the predicate given to the constructor.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE,"Assign register bank of generic virtual registers", false, false) RegBankSelect
FunctionPass class - This class is used to implement most global optimizations.
INITIALIZE_PASS_BEGIN(GuardWideningLegacyPass,"guard-widening","Widen guards", false, false) INITIALIZE_PASS_END(GuardWideningLegacyPass
iterator erase(const_iterator CI)
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
LLVMContext & getContext() const
All values hold a context through their type.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
static SDValue Widen(SelectionDAG *CurDAG, SDValue N)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
static ConstantRange makeExactICmpRegion(CmpInst::Predicate Pred, const APInt &Other)
Produce the exact range such that all values in the returned range satisfy the given predicate with a...
This is the shared class of boolean and integer constants.
auto find(R &&Range, const T &Val) -> decltype(std::begin(Range))
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly...
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type...
ConstantRange inverse() const
Return a new range that is the logical not of the current set.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Analysis pass which computes a PostDominatorTree.
Type * getType() const
All values are typed, get the type of this value.
This class represents a range of values.
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
static ConstantInt * getTrue(LLVMContext &Context)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
df_iterator< T > df_begin(const T &G)
Value * getArgOperand(unsigned i) const
getArgOperand/setArgOperand - Return/set the i-th call argument.
Class for arbitrary precision integers.
PostDominatorTree Class - Concrete subclass of DominatorTree that is used to compute the post-dominat...
iterator insert(iterator I, T &&Elt)
bool isMinValue() const
Determine if this is the smallest unsigned value.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
void emplace_back(ArgTypes &&...Args)
Represents a single loop in the control flow graph.
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
void setArgOperand(unsigned i, Value *v)
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
bool isSafeToSpeculativelyExecute(const Value *V, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr)
Return true if the instruction does not have any effects besides calculating the result and does not ...
LLVM Value Representation.
FunctionPass * createGuardWideningPass()
This class implements an extremely fast bulk output stream that can only output to a stream...
The legacy pass manager's analysis pass to compute loop information.
StringRef - Represent a constant reference to a string, i.e.
A container for analyses that lazily runs them and caches their results.
Legacy analysis pass which computes a DominatorTree.
unsigned getPathLength() const
getPathLength - Return the length of the path from the entry node to the current node, counting both nodes.
const BasicBlock * getParent() const
A wrapper class for inspecting calls to intrinsic functions.
CmpClass_match< LHS, RHS, ICmpInst, ICmpInst::Predicate > m_ICmp(ICmpInst::Predicate &Pred, const LHS &L, const RHS &R)