/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp

Bug Summary

File:	llvm/lib/Transforms/Scalar/LICM.cpp
Warning:	line 1266, column 41 Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

Show analyzer invocation

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name LICM.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/build-llvm/lib/Transforms/Scalar -resource-dir /usr/lib/llvm-13/lib/clang/13.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/build-llvm/lib/Transforms/Scalar -I /build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar -I /build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/build-llvm/include -I /build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-13/lib/clang/13.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/build-llvm/lib/Transforms/Scalar -fdebug-prefix-map=/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d=. -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-05-07-005843-9350-1 -x c++ /build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp

/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp

→

1//===-- LICM.cpp - Loop Invariant Code Motion Pass ------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass performs loop invariant code motion, attempting to remove as much
10// code from the body of a loop as possible.  It does this by either hoisting
11// code into the preheader block, or by sinking code to the exit blocks if it is
12// safe.  This pass also promotes must-aliased memory locations in the loop to
13// live in registers, thus hoisting and sinking "invariant" loads and stores.
14//
15// Hoisting operations out of loops is a canonicalization transform.  It
16// enables and simplifies subsequent optimizations in the middle-end.
17// Rematerialization of hoisted instructions to reduce register pressure is the
18// responsibility of the back-end, which has more accurate information about
19// register pressure and also handles other optimizations than LICM that
20// increase live-ranges.
21//
22// This pass uses alias analysis for two purposes:
23//
24//  1. Moving loop invariant loads and calls out of loops.  If we can determine
25//     that a load or call inside of a loop never aliases anything stored to,
26//     we can hoist it or sink it like any other instruction.
27//  2. Scalar Promotion of Memory - If there is a store instruction inside of
28//     the loop, we try to move the store to happen AFTER the loop instead of
29//     inside of the loop.  This can only happen if a few conditions are true:
30//       A. The pointer stored through is loop invariant
31//       B. There are no stores or loads in the loop which _may_ alias the
32//          pointer.  There are no calls in the loop which mod/ref the pointer.
33//     If these conditions are true, we can promote the loads and stores in the
34//     loop of the pointer to use a temporary alloca'd variable.  We then use
35//     the SSAUpdater to construct the appropriate SSA form for the value.
36//
37//===----------------------------------------------------------------------===//

39#include "llvm/Transforms/Scalar/LICM.h"
40#include "llvm/ADT/SetOperations.h"
41#include "llvm/ADT/Statistic.h"
42#include "llvm/Analysis/AliasAnalysis.h"
43#include "llvm/Analysis/AliasSetTracker.h"
44#include "llvm/Analysis/BasicAliasAnalysis.h"
45#include "llvm/Analysis/BlockFrequencyInfo.h"
46#include "llvm/Analysis/CaptureTracking.h"
47#include "llvm/Analysis/ConstantFolding.h"
48#include "llvm/Analysis/GlobalsModRef.h"
49#include "llvm/Analysis/GuardUtils.h"
50#include "llvm/Analysis/LazyBlockFrequencyInfo.h"
51#include "llvm/Analysis/Loads.h"
52#include "llvm/Analysis/LoopInfo.h"
53#include "llvm/Analysis/LoopIterator.h"
54#include "llvm/Analysis/LoopPass.h"
55#include "llvm/Analysis/MemoryBuiltins.h"
56#include "llvm/Analysis/MemorySSA.h"
57#include "llvm/Analysis/MemorySSAUpdater.h"
58#include "llvm/Analysis/MustExecute.h"
59#include "llvm/Analysis/OptimizationRemarkEmitter.h"
60#include "llvm/Analysis/ScalarEvolution.h"
61#include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
62#include "llvm/Analysis/TargetLibraryInfo.h"
63#include "llvm/Analysis/ValueTracking.h"
64#include "llvm/IR/CFG.h"
65#include "llvm/IR/Constants.h"
66#include "llvm/IR/DataLayout.h"
67#include "llvm/IR/DebugInfoMetadata.h"
68#include "llvm/IR/DerivedTypes.h"
69#include "llvm/IR/Dominators.h"
70#include "llvm/IR/Instructions.h"
71#include "llvm/IR/IntrinsicInst.h"
72#include "llvm/IR/LLVMContext.h"
73#include "llvm/IR/Metadata.h"
74#include "llvm/IR/PatternMatch.h"
75#include "llvm/IR/PredIteratorCache.h"
76#include "llvm/InitializePasses.h"
77#include "llvm/Support/CommandLine.h"
78#include "llvm/Support/Debug.h"
79#include "llvm/Support/raw_ostream.h"
80#include "llvm/Transforms/Scalar.h"
81#include "llvm/Transforms/Scalar/LoopPassManager.h"
82#include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
83#include "llvm/Transforms/Utils/BasicBlockUtils.h"
84#include "llvm/Transforms/Utils/Local.h"
85#include "llvm/Transforms/Utils/LoopUtils.h"
86#include "llvm/Transforms/Utils/SSAUpdater.h"
87#include <algorithm>
88#include <utility>
89using namespace llvm;

91#define DEBUG_TYPE"licm" "licm"

93STATISTIC(NumCreatedBlocks, "Number of blocks created")static llvm::Statistic NumCreatedBlocks = {"licm", "NumCreatedBlocks"
, "Number of blocks created"};
94STATISTIC(NumClonedBranches, "Number of branches cloned")static llvm::Statistic NumClonedBranches = {"licm", "NumClonedBranches"
, "Number of branches cloned"};
95STATISTIC(NumSunk, "Number of instructions sunk out of loop")static llvm::Statistic NumSunk = {"licm", "NumSunk", "Number of instructions sunk out of loop"
};
96STATISTIC(NumHoisted, "Number of instructions hoisted out of loop")static llvm::Statistic NumHoisted = {"licm", "NumHoisted", "Number of instructions hoisted out of loop"
};
97STATISTIC(NumMovedLoads, "Number of load insts hoisted or sunk")static llvm::Statistic NumMovedLoads = {"licm", "NumMovedLoads"
, "Number of load insts hoisted or sunk"};
98STATISTIC(NumMovedCalls, "Number of call insts hoisted or sunk")static llvm::Statistic NumMovedCalls = {"licm", "NumMovedCalls"
, "Number of call insts hoisted or sunk"};
99STATISTIC(NumPromoted, "Number of memory locations promoted to registers")static llvm::Statistic NumPromoted = {"licm", "NumPromoted", "Number of memory locations promoted to registers"
};

101/// Memory promotion is enabled by default.
102static cl::opt<bool>
  DisablePromotion("disable-licm-promotion", cl::Hidden, cl::init(false),
                   cl::desc("Disable memory promotion in LICM pass"));

106static cl::opt<bool> ControlFlowHoisting(
  "licm-control-flow-hoisting", cl::Hidden, cl::init(false),
  cl::desc("Enable control flow (and PHI) hoisting in LICM"));

110static cl::opt<unsigned> HoistSinkColdnessThreshold(
  "licm-coldness-threshold", cl::Hidden, cl::init(4),
  cl::desc("Relative coldness Threshold of hoisting/sinking destination "
           "block for LICM to be considered beneficial"));

115static cl::opt<uint32_t> MaxNumUsesTraversed(
  "licm-max-num-uses-traversed", cl::Hidden, cl::init(8),
  cl::desc("Max num uses visited for identifying load "
           "invariance in loop using invariant start (default = 8)"));

120// Default value of zero implies we use the regular alias set tracker mechanism
121// instead of the cross product using AA to identify aliasing of the memory
122// location we are interested in.
123static cl::opt<int>
124LICMN2Theshold("licm-n2-threshold", cl::Hidden, cl::init(0),
             cl::desc("How many instruction to cross product using AA"));

127// Experimental option to allow imprecision in LICM in pathological cases, in
128// exchange for faster compile. This is to be removed if MemorySSA starts to
129// address the same issue. This flag applies only when LICM uses MemorySSA
130// instead on AliasSetTracker. LICM calls MemorySSAWalker's
131// getClobberingMemoryAccess, up to the value of the Cap, getting perfect
132// accuracy. Afterwards, LICM will call into MemorySSA's getDefiningAccess,
133// which may not be precise, since optimizeUses is capped. The result is
134// correct, but we may not get as "far up" as possible to get which access is
135// clobbering the one queried.
136cl::opt<unsigned> llvm::SetLicmMssaOptCap(
  "licm-mssa-optimization-cap", cl::init(100), cl::Hidden,
  cl::desc("Enable imprecision in LICM in pathological cases, in exchange "
           "for faster compile. Caps the MemorySSA clobbering calls."));

141// Experimentally, memory promotion carries less importance than sinking and
142// hoisting. Limit when we do promotion when using MemorySSA, in order to save
143// compile time.
144cl::opt<unsigned> llvm::SetLicmMssaNoAccForPromotionCap(
  "licm-mssa-max-acc-promotion", cl::init(250), cl::Hidden,
  cl::desc("[LICM & MemorySSA] When MSSA in LICM is disabled, this has no "
           "effect. When MSSA in LICM is enabled, then this is the maximum "
           "number of accesses allowed to be present in a loop in order to "
           "enable memory promotion."));

151static bool inSubLoop(BasicBlock *BB, Loop *CurLoop, LoopInfo *LI);
152static bool isNotUsedOrFreeInLoop(const Instruction &I, const Loop *CurLoop,
                                const LoopSafetyInfo *SafetyInfo,
                                TargetTransformInfo *TTI, bool &FreeInLoop);
155static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop,
                BasicBlock *Dest, ICFLoopSafetyInfo *SafetyInfo,
                MemorySSAUpdater *MSSAU, ScalarEvolution *SE,
                OptimizationRemarkEmitter *ORE);
159static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT,
               BlockFrequencyInfo *BFI, const Loop *CurLoop,
               ICFLoopSafetyInfo *SafetyInfo, MemorySSAUpdater *MSSAU,
               OptimizationRemarkEmitter *ORE);
163static bool isSafeToExecuteUnconditionally(Instruction &Inst,
                                         const DominatorTree *DT,
                                         const TargetLibraryInfo *TLI,
                                         const Loop *CurLoop,
                                         const LoopSafetyInfo *SafetyInfo,
                                         OptimizationRemarkEmitter *ORE,
                                         const Instruction *CtxI = nullptr);
170static bool pointerInvalidatedByLoop(MemoryLocation MemLoc,
                                   AliasSetTracker *CurAST, Loop *CurLoop,
                                   AAResults *AA);
173static bool pointerInvalidatedByLoopWithMSSA(MemorySSA *MSSA, MemoryUse *MU,
                                           Loop *CurLoop, Instruction &I,
                                           SinkAndHoistLICMFlags &Flags);
176static bool pointerInvalidatedByBlockWithMSSA(BasicBlock &BB, MemorySSA &MSSA,
                                            MemoryUse &MU);
178static Instruction *cloneInstructionInExitBlock(
  Instruction &I, BasicBlock &ExitBlock, PHINode &PN, const LoopInfo *LI,
  const LoopSafetyInfo *SafetyInfo, MemorySSAUpdater *MSSAU);

182static void eraseInstruction(Instruction &I, ICFLoopSafetyInfo &SafetyInfo,
                           AliasSetTracker *AST, MemorySSAUpdater *MSSAU);

185static void moveInstructionBefore(Instruction &I, Instruction &Dest,
                                ICFLoopSafetyInfo &SafetyInfo,
                                MemorySSAUpdater *MSSAU, ScalarEvolution *SE);

189static void foreachMemoryAccess(MemorySSA *MSSA, Loop *L,
                              function_ref<void(Instruction *)> Fn);
191static SmallVector<SmallSetVector<Value *, 8>, 0>
192collectPromotionCandidates(MemorySSA *MSSA, AliasAnalysis *AA, Loop *L,
                         SmallVectorImpl<Instruction *> &MaybePromotable);

195namespace {
196struct LoopInvariantCodeMotion {
bool runOnLoop(Loop *L, AAResults *AA, LoopInfo *LI, DominatorTree *DT,
               BlockFrequencyInfo *BFI, TargetLibraryInfo *TLI,
               TargetTransformInfo *TTI, ScalarEvolution *SE, MemorySSA *MSSA,
               OptimizationRemarkEmitter *ORE);

LoopInvariantCodeMotion(unsigned LicmMssaOptCap,
                        unsigned LicmMssaNoAccForPromotionCap)
    : LicmMssaOptCap(LicmMssaOptCap),
      LicmMssaNoAccForPromotionCap(LicmMssaNoAccForPromotionCap) {}

207private:
unsigned LicmMssaOptCap;
unsigned LicmMssaNoAccForPromotionCap;

std::unique_ptr<AliasSetTracker>
collectAliasInfoForLoop(Loop *L, LoopInfo *LI, AAResults *AA);
213};

215struct LegacyLICMPass : public LoopPass {
static char ID; // Pass identification, replacement for typeid
LegacyLICMPass(
    unsigned LicmMssaOptCap = SetLicmMssaOptCap,
    unsigned LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap)
    : LoopPass(ID), LICM(LicmMssaOptCap, LicmMssaNoAccForPromotionCap) {
  initializeLegacyLICMPassPass(*PassRegistry::getPassRegistry());
}

bool runOnLoop(Loop *L, LPPassManager &LPM) override {
  if (skipLoop(L))
    return false;

  LLVM_DEBUG(dbgs() << "Perform LICM on Loop with header at block "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "Perform LICM on Loop with header at block "
 << L->getHeader()->getNameOrAsOperand() <<
 "\n"; } } while (false)
                    << L->getHeader()->getNameOrAsOperand() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "Perform LICM on Loop with header at block "
 << L->getHeader()->getNameOrAsOperand() <<
 "\n"; } } while (false);

  auto *SE = getAnalysisIfAvailable<ScalarEvolutionWrapperPass>();
  MemorySSA *MSSA = EnableMSSALoopDependency
                        ? (&getAnalysis<MemorySSAWrapperPass>().getMSSA())
                        : nullptr;
  bool hasProfileData = L->getHeader()->getParent()->hasProfileData();
  BlockFrequencyInfo *BFI =
      hasProfileData ? &getAnalysis<LazyBlockFrequencyInfoPass>().getBFI()
                     : nullptr;
  // For the old PM, we can't use OptimizationRemarkEmitter as an analysis
  // pass. Function analyses need to be preserved across loop transformations
  // but ORE cannot be preserved (see comment before the pass definition).
  OptimizationRemarkEmitter ORE(L->getHeader()->getParent());
  return LICM.runOnLoop(
      L, &getAnalysis<AAResultsWrapperPass>().getAAResults(),
      &getAnalysis<LoopInfoWrapperPass>().getLoopInfo(),
      &getAnalysis<DominatorTreeWrapperPass>().getDomTree(), BFI,
      &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(
          *L->getHeader()->getParent()),
      &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(
          *L->getHeader()->getParent()),
      SE ? &SE->getSE() : nullptr, MSSA, &ORE);
}

/// This transformation requires natural loop information & requires that
/// loop preheaders be inserted into the CFG...
///
void getAnalysisUsage(AnalysisUsage &AU) const override {
  AU.addPreserved<DominatorTreeWrapperPass>();
  AU.addPreserved<LoopInfoWrapperPass>();
  AU.addRequired<TargetLibraryInfoWrapperPass>();
  if (EnableMSSALoopDependency) {
    AU.addRequired<MemorySSAWrapperPass>();
    AU.addPreserved<MemorySSAWrapperPass>();
  }
  AU.addRequired<TargetTransformInfoWrapperPass>();
  getLoopAnalysisUsage(AU);
  LazyBlockFrequencyInfoPass::getLazyBFIAnalysisUsage(AU);
  AU.addPreserved<LazyBlockFrequencyInfoPass>();
  AU.addPreserved<LazyBranchProbabilityInfoPass>();
}

272private:
LoopInvariantCodeMotion LICM;
274};
275} // namespace

277PreservedAnalyses LICMPass::run(Loop &L, LoopAnalysisManager &AM,
                              LoopStandardAnalysisResults &AR, LPMUpdater &) {
// For the new PM, we also can't use OptimizationRemarkEmitter as an analysis
// pass.  Function analyses need to be preserved across loop transformations
// but ORE cannot be preserved (see comment before the pass definition).
OptimizationRemarkEmitter ORE(L.getHeader()->getParent());

LoopInvariantCodeMotion LICM(LicmMssaOptCap, LicmMssaNoAccForPromotionCap);
if (!LICM.runOnLoop(&L, &AR.AA, &AR.LI, &AR.DT, AR.BFI, &AR.TLI, &AR.TTI,
                    &AR.SE, AR.MSSA, &ORE))
  return PreservedAnalyses::all();

auto PA = getLoopPassPreservedAnalyses();

PA.preserve<DominatorTreeAnalysis>();
PA.preserve<LoopAnalysis>();
if (AR.MSSA)
  PA.preserve<MemorySSAAnalysis>();

return PA;
297}

299char LegacyLICMPass::ID = 0;
300INITIALIZE_PASS_BEGIN(LegacyLICMPass, "licm", "Loop Invariant Code Motion",static void *initializeLegacyLICMPassPassOnce(PassRegistry &
Registry) {
                    false, false)static void *initializeLegacyLICMPassPassOnce(PassRegistry &
Registry) {
302INITIALIZE_PASS_DEPENDENCY(LoopPass)initializeLoopPassPass(Registry);
303INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)initializeTargetLibraryInfoWrapperPassPass(Registry);
304INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)initializeTargetTransformInfoWrapperPassPass(Registry);
305INITIALIZE_PASS_DEPENDENCY(MemorySSAWrapperPass)initializeMemorySSAWrapperPassPass(Registry);
306INITIALIZE_PASS_DEPENDENCY(LazyBFIPass)initializeLazyBFIPassPass(Registry);
307INITIALIZE_PASS_END(LegacyLICMPass, "licm", "Loop Invariant Code Motion", false,PassInfo *PI = new PassInfo( "Loop Invariant Code Motion", "licm"
, &LegacyLICMPass::ID, PassInfo::NormalCtor_t(callDefaultCtor
<LegacyLICMPass>), false, false); Registry.registerPass
(*PI, true); return PI; } static llvm::once_flag InitializeLegacyLICMPassPassFlag
; void llvm::initializeLegacyLICMPassPass(PassRegistry &Registry
) { llvm::call_once(InitializeLegacyLICMPassPassFlag, initializeLegacyLICMPassPassOnce
, std::ref(Registry)); }
                  false)PassInfo *PI = new PassInfo( "Loop Invariant Code Motion", "licm"
, &LegacyLICMPass::ID, PassInfo::NormalCtor_t(callDefaultCtor
<LegacyLICMPass>), false, false); Registry.registerPass
(*PI, true); return PI; } static llvm::once_flag InitializeLegacyLICMPassPassFlag
; void llvm::initializeLegacyLICMPassPass(PassRegistry &Registry
) { llvm::call_once(InitializeLegacyLICMPassPassFlag, initializeLegacyLICMPassPassOnce
, std::ref(Registry)); }

310Pass *llvm::createLICMPass() { return new LegacyLICMPass(); }
311Pass *llvm::createLICMPass(unsigned LicmMssaOptCap,
                         unsigned LicmMssaNoAccForPromotionCap) {
return new LegacyLICMPass(LicmMssaOptCap, LicmMssaNoAccForPromotionCap);
314}

316llvm::SinkAndHoistLICMFlags::SinkAndHoistLICMFlags(bool IsSink, Loop *L,
                                                 MemorySSA *MSSA)
  : SinkAndHoistLICMFlags(SetLicmMssaOptCap, SetLicmMssaNoAccForPromotionCap,
                          IsSink, L, MSSA) {}

321llvm::SinkAndHoistLICMFlags::SinkAndHoistLICMFlags(
  unsigned LicmMssaOptCap, unsigned LicmMssaNoAccForPromotionCap, bool IsSink,
  Loop *L, MemorySSA *MSSA)
  : LicmMssaOptCap(LicmMssaOptCap),
    LicmMssaNoAccForPromotionCap(LicmMssaNoAccForPromotionCap),
    IsSink(IsSink) {
assert(((L != nullptr) == (MSSA != nullptr)) &&(static_cast <bool> (((L != nullptr) == (MSSA != nullptr
)) && "Unexpected values for SinkAndHoistLICMFlags") ?
 void (0) : __assert_fail ("((L != nullptr) == (MSSA != nullptr)) && \"Unexpected values for SinkAndHoistLICMFlags\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 328, __extension__ __PRETTY_FUNCTION__))
       "Unexpected values for SinkAndHoistLICMFlags")(static_cast <bool> (((L != nullptr) == (MSSA != nullptr
)) && "Unexpected values for SinkAndHoistLICMFlags") ?
 void (0) : __assert_fail ("((L != nullptr) == (MSSA != nullptr)) && \"Unexpected values for SinkAndHoistLICMFlags\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 328, __extension__ __PRETTY_FUNCTION__));
if (!MSSA)
  return;

unsigned AccessCapCount = 0;
for (auto *BB : L->getBlocks())
  if (const auto *Accesses = MSSA->getBlockAccesses(BB))
    for (const auto &MA : *Accesses) {
      (void)MA;
      ++AccessCapCount;
      if (AccessCapCount > LicmMssaNoAccForPromotionCap) {
        NoOfMemAccTooLarge = true;
        return;
      }
    }
343}

345/// Hoist expressions out of the specified loop. Note, alias info for inner
346/// loop is not preserved so it is not a good idea to run LICM multiple
347/// times on one loop.
348bool LoopInvariantCodeMotion::runOnLoop(
  Loop *L, AAResults *AA, LoopInfo *LI, DominatorTree *DT,
  BlockFrequencyInfo *BFI, TargetLibraryInfo *TLI, TargetTransformInfo *TTI,
  ScalarEvolution *SE, MemorySSA *MSSA, OptimizationRemarkEmitter *ORE) {
bool Changed = false;

assert(L->isLCSSAForm(*DT) && "Loop is not in LCSSA form.")(static_cast <bool> (L->isLCSSAForm(*DT) && "Loop is not in LCSSA form."
) ? void (0) : __assert_fail ("L->isLCSSAForm(*DT) && \"Loop is not in LCSSA form.\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 354, __extension__ __PRETTY_FUNCTION__));

// If this loop has metadata indicating that LICM is not to be performed then
// just exit.
if (hasDisableLICMTransformsHint(L)) {
  return false;
}

std::unique_ptr<AliasSetTracker> CurAST;
std::unique_ptr<MemorySSAUpdater> MSSAU;
std::unique_ptr<SinkAndHoistLICMFlags> Flags;

// Don't sink stores from loops with coroutine suspend instructions.
// LICM would sink instructions into the default destination of
// the coroutine switch. The default destination of the switch is to
// handle the case where the coroutine is suspended, by which point the
// coroutine frame may have been destroyed. No instruction can be sunk there.
// FIXME: This would unfortunately hurt the performance of coroutines, however
// there is currently no general solution for this. Similar issues could also
// potentially happen in other passes where instructions are being moved
// across that edge.
bool HasCoroSuspendInst = llvm::any_of(L->getBlocks(), [](BasicBlock *BB) {
  return llvm::any_of(*BB, [](Instruction &I) {
    IntrinsicInst *II = dyn_cast<IntrinsicInst>(&I);
    return II && II->getIntrinsicID() == Intrinsic::coro_suspend;
  });
});

if (!MSSA) {
  LLVM_DEBUG(dbgs() << "LICM: Using Alias Set Tracker.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM: Using Alias Set Tracker.\n"
; } } while (false);
  CurAST = collectAliasInfoForLoop(L, LI, AA);
  Flags = std::make_unique<SinkAndHoistLICMFlags>(
      LicmMssaOptCap, LicmMssaNoAccForPromotionCap, /*IsSink=*/true);
} else {
  LLVM_DEBUG(dbgs() << "LICM: Using MemorySSA.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM: Using MemorySSA.\n"; } } while
 (false);
  MSSAU = std::make_unique<MemorySSAUpdater>(MSSA);
  Flags = std::make_unique<SinkAndHoistLICMFlags>(
      LicmMssaOptCap, LicmMssaNoAccForPromotionCap, /*IsSink=*/true, L, MSSA);
}

// Get the preheader block to move instructions into...
BasicBlock *Preheader = L->getLoopPreheader();

// Compute loop safety information.
ICFLoopSafetyInfo SafetyInfo;
SafetyInfo.computeLoopSafetyInfo(L);

// We want to visit all of the instructions in this loop... that are not parts
// of our subloops (they have already had their invariants hoisted out of
// their loop, into this loop, so there is no need to process the BODIES of
// the subloops).
//
// Traverse the body of the loop in depth first order on the dominator tree so
// that we are guaranteed to see definitions before we see uses.  This allows
// us to sink instructions in one pass, without iteration.  After sinking
// instructions, we perform another pass to hoist them out of the loop.
if (L->hasDedicatedExits())
  Changed |=
      sinkRegion(DT->getNode(L->getHeader()), AA, LI, DT, BFI, TLI, TTI, L,
                 CurAST.get(), MSSAU.get(), &SafetyInfo, *Flags.get(), ORE);
Flags->setIsSink(false);
if (Preheader)
  Changed |= hoistRegion(DT->getNode(L->getHeader()), AA, LI, DT, BFI, TLI, L,
                         CurAST.get(), MSSAU.get(), SE, &SafetyInfo,
                         *Flags.get(), ORE);

// Now that all loop invariants have been removed from the loop, promote any
// memory references to scalars that we can.
// Don't sink stores from loops without dedicated block exits. Exits
// containing indirect branches are not transformed by loop simplify,
// make sure we catch that. An additional load may be generated in the
// preheader for SSA updater, so also avoid sinking when no preheader
// is available.
if (!DisablePromotion && Preheader && L->hasDedicatedExits() &&
    !Flags->tooManyMemoryAccesses() && !HasCoroSuspendInst) {
  // Figure out the loop exits and their insertion points
  SmallVector<BasicBlock *, 8> ExitBlocks;
  L->getUniqueExitBlocks(ExitBlocks);

  // We can't insert into a catchswitch.
  bool HasCatchSwitch = llvm::any_of(ExitBlocks, [](BasicBlock *Exit) {
    return isa<CatchSwitchInst>(Exit->getTerminator());
  });

  if (!HasCatchSwitch) {
    SmallVector<Instruction *, 8> InsertPts;
    SmallVector<MemoryAccess *, 8> MSSAInsertPts;
    InsertPts.reserve(ExitBlocks.size());
    if (MSSAU)
      MSSAInsertPts.reserve(ExitBlocks.size());
    for (BasicBlock *ExitBlock : ExitBlocks) {
      InsertPts.push_back(&*ExitBlock->getFirstInsertionPt());
      if (MSSAU)
        MSSAInsertPts.push_back(nullptr);
    }

    PredIteratorCache PIC;

    bool Promoted = false;
    if (CurAST.get()) {
      // Loop over all of the alias sets in the tracker object.
      for (AliasSet &AS : *CurAST) {
        // We can promote this alias set if it has a store, if it is a "Must"
        // alias set, if the pointer is loop invariant, and if we are not
        // eliminating any volatile loads or stores.
        if (AS.isForwardingAliasSet() || !AS.isMod() || !AS.isMustAlias() ||
            !L->isLoopInvariant(AS.begin()->getValue()))
          continue;

        assert((static_cast <bool> (!AS.empty() && "Must alias set should have at least one pointer element in it!"
) ? void (0) : __assert_fail ("!AS.empty() && \"Must alias set should have at least one pointer element in it!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 465, __extension__ __PRETTY_FUNCTION__))
            !AS.empty() &&(static_cast <bool> (!AS.empty() && "Must alias set should have at least one pointer element in it!"
) ? void (0) : __assert_fail ("!AS.empty() && \"Must alias set should have at least one pointer element in it!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 465, __extension__ __PRETTY_FUNCTION__))
            "Must alias set should have at least one pointer element in it!")(static_cast <bool> (!AS.empty() && "Must alias set should have at least one pointer element in it!"
) ? void (0) : __assert_fail ("!AS.empty() && \"Must alias set should have at least one pointer element in it!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 465, __extension__ __PRETTY_FUNCTION__));

        SmallSetVector<Value *, 8> PointerMustAliases;
        for (const auto &ASI : AS)
          PointerMustAliases.insert(ASI.getValue());

        Promoted |= promoteLoopAccessesToScalars(
            PointerMustAliases, ExitBlocks, InsertPts, MSSAInsertPts, PIC, LI,
            DT, TLI, L, CurAST.get(), MSSAU.get(), &SafetyInfo, ORE);
      }
    } else {
      SmallVector<Instruction *, 16> MaybePromotable;
      foreachMemoryAccess(MSSA, L, [&](Instruction *I) {
        MaybePromotable.push_back(I);
      });

      // Promoting one set of accesses may make the pointers for another set
      // loop invariant, so run this in a loop (with the MaybePromotable set
      // decreasing in size over time).
      bool LocalPromoted;
      do {
        LocalPromoted = false;
        for (const SmallSetVector<Value *, 8> &PointerMustAliases :
             collectPromotionCandidates(MSSA, AA, L, MaybePromotable)) {
          LocalPromoted |= promoteLoopAccessesToScalars(
              PointerMustAliases, ExitBlocks, InsertPts, MSSAInsertPts, PIC,
              LI, DT, TLI, L, /*AST*/nullptr, MSSAU.get(), &SafetyInfo, ORE);
        }
        Promoted |= LocalPromoted;
      } while (LocalPromoted);
    }

    // Once we have promoted values across the loop body we have to
    // recursively reform LCSSA as any nested loop may now have values defined
    // within the loop used in the outer loop.
    // FIXME: This is really heavy handed. It would be a bit better to use an
    // SSAUpdater strategy during promotion that was LCSSA aware and reformed
    // it as it went.
    if (Promoted)
      formLCSSARecursively(*L, *DT, LI, SE);

    Changed |= Promoted;
  }
}

// Check that neither this loop nor its parent have had LCSSA broken. LICM is
// specifically moving instructions across the loop boundary and so it is
// especially in need of sanity checking here.
assert(L->isLCSSAForm(*DT) && "Loop not left in LCSSA form after LICM!")(static_cast <bool> (L->isLCSSAForm(*DT) && "Loop not left in LCSSA form after LICM!"
) ? void (0) : __assert_fail ("L->isLCSSAForm(*DT) && \"Loop not left in LCSSA form after LICM!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 513, __extension__ __PRETTY_FUNCTION__));
assert((L->isOutermost() || L->getParentLoop()->isLCSSAForm(*DT)) &&(static_cast <bool> ((L->isOutermost() || L->getParentLoop
()->isLCSSAForm(*DT)) && "Parent loop not left in LCSSA form after LICM!"
) ? void (0) : __assert_fail ("(L->isOutermost() || L->getParentLoop()->isLCSSAForm(*DT)) && \"Parent loop not left in LCSSA form after LICM!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 515, __extension__ __PRETTY_FUNCTION__))
       "Parent loop not left in LCSSA form after LICM!")(static_cast <bool> ((L->isOutermost() || L->getParentLoop
()->isLCSSAForm(*DT)) && "Parent loop not left in LCSSA form after LICM!"
) ? void (0) : __assert_fail ("(L->isOutermost() || L->getParentLoop()->isLCSSAForm(*DT)) && \"Parent loop not left in LCSSA form after LICM!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 515, __extension__ __PRETTY_FUNCTION__));

if (MSSAU.get() && VerifyMemorySSA)
  MSSAU->getMemorySSA()->verifyMemorySSA();

if (Changed && SE)
  SE->forgetLoopDispositions(L);
return Changed;
523}

525/// Walk the specified region of the CFG (defined by all blocks dominated by
526/// the specified block, and that are in the current loop) in reverse depth
527/// first order w.r.t the DominatorTree.  This allows us to visit uses before
528/// definitions, allowing us to sink a loop body in one pass without iteration.
529///
530bool llvm::sinkRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI,
                    DominatorTree *DT, BlockFrequencyInfo *BFI,
                    TargetLibraryInfo *TLI, TargetTransformInfo *TTI,
                    Loop *CurLoop, AliasSetTracker *CurAST,
                    MemorySSAUpdater *MSSAU, ICFLoopSafetyInfo *SafetyInfo,
                    SinkAndHoistLICMFlags &Flags,
                    OptimizationRemarkEmitter *ORE) {

// Verify inputs.
assert(N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr &&(static_cast <bool> (N != nullptr && AA != nullptr
 && LI != nullptr && DT != nullptr &&
 CurLoop != nullptr && SafetyInfo != nullptr &&
 "Unexpected input to sinkRegion.") ? void (0) : __assert_fail
 ("N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected input to sinkRegion.\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 541, __extension__ __PRETTY_FUNCTION__))
       CurLoop != nullptr && SafetyInfo != nullptr &&(static_cast <bool> (N != nullptr && AA != nullptr
 && LI != nullptr && DT != nullptr &&
 CurLoop != nullptr && SafetyInfo != nullptr &&
 "Unexpected input to sinkRegion.") ? void (0) : __assert_fail
 ("N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected input to sinkRegion.\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 541, __extension__ __PRETTY_FUNCTION__))
       "Unexpected input to sinkRegion.")(static_cast <bool> (N != nullptr && AA != nullptr
 && LI != nullptr && DT != nullptr &&
 CurLoop != nullptr && SafetyInfo != nullptr &&
 "Unexpected input to sinkRegion.") ? void (0) : __assert_fail
 ("N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected input to sinkRegion.\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 541, __extension__ __PRETTY_FUNCTION__));
assert(((CurAST != nullptr) ^ (MSSAU != nullptr)) &&(static_cast <bool> (((CurAST != nullptr) ^ (MSSAU != nullptr
)) && "Either AliasSetTracker or MemorySSA should be initialized."
) ? void (0) : __assert_fail ("((CurAST != nullptr) ^ (MSSAU != nullptr)) && \"Either AliasSetTracker or MemorySSA should be initialized.\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 543, __extension__ __PRETTY_FUNCTION__))
       "Either AliasSetTracker or MemorySSA should be initialized.")(static_cast <bool> (((CurAST != nullptr) ^ (MSSAU != nullptr
)) && "Either AliasSetTracker or MemorySSA should be initialized."
) ? void (0) : __assert_fail ("((CurAST != nullptr) ^ (MSSAU != nullptr)) && \"Either AliasSetTracker or MemorySSA should be initialized.\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 543, __extension__ __PRETTY_FUNCTION__));

// We want to visit children before parents. We will enque all the parents
// before their children in the worklist and process the worklist in reverse
// order.
SmallVector<DomTreeNode *, 16> Worklist = collectChildrenInLoop(N, CurLoop);

bool Changed = false;
for (DomTreeNode *DTN : reverse(Worklist)) {
  BasicBlock *BB = DTN->getBlock();
  // Only need to process the contents of this block if it is not part of a
  // subloop (which would already have been processed).
  if (inSubLoop(BB, CurLoop, LI))
    continue;

  for (BasicBlock::iterator II = BB->end(); II != BB->begin();) {
    Instruction &I = *--II;

    // The instruction is not used in the loop if it is dead.  In this case,
    // we just delete it instead of sinking it.
    if (isInstructionTriviallyDead(&I, TLI)) {
      LLVM_DEBUG(dbgs() << "LICM deleting dead inst: " << I << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM deleting dead inst: " <<
 I << '\n'; } } while (false);
      salvageKnowledge(&I);
      salvageDebugInfo(I);
      ++II;
      eraseInstruction(I, *SafetyInfo, CurAST, MSSAU);
      Changed = true;
      continue;
    }

    // Check to see if we can sink this instruction to the exit blocks
    // of the loop.  We can do this if the all users of the instruction are
    // outside of the loop.  In this case, it doesn't even matter if the
    // operands of the instruction are loop invariant.
    //
    bool FreeInLoop = false;
    if (!I.mayHaveSideEffects() &&
        isNotUsedOrFreeInLoop(I, CurLoop, SafetyInfo, TTI, FreeInLoop) &&
        canSinkOrHoistInst(I, AA, DT, CurLoop, CurAST, MSSAU, true, &Flags,
                           ORE)) {
      if (sink(I, LI, DT, BFI, CurLoop, SafetyInfo, MSSAU, ORE)) {
        if (!FreeInLoop) {
          ++II;
          salvageDebugInfo(I);
          eraseInstruction(I, *SafetyInfo, CurAST, MSSAU);
        }
        Changed = true;
      }
    }
  }
}
if (MSSAU && VerifyMemorySSA)
  MSSAU->getMemorySSA()->verifyMemorySSA();
return Changed;
597}

599namespace {
600// This is a helper class for hoistRegion to make it able to hoist control flow
601// in order to be able to hoist phis. The way this works is that we initially
602// start hoisting to the loop preheader, and when we see a loop invariant branch
603// we make note of this. When we then come to hoist an instruction that's
604// conditional on such a branch we duplicate the branch and the relevant control
605// flow, then hoist the instruction into the block corresponding to its original
606// block in the duplicated control flow.
607class ControlFlowHoister {
608private:
// Information about the loop we are hoisting from
LoopInfo *LI;
DominatorTree *DT;
Loop *CurLoop;
MemorySSAUpdater *MSSAU;

// A map of blocks in the loop to the block their instructions will be hoisted
// to.
DenseMap<BasicBlock *, BasicBlock *> HoistDestinationMap;

// The branches that we can hoist, mapped to the block that marks a
// convergence point of their control flow.
DenseMap<BranchInst *, BasicBlock *> HoistableBranches;

623public:
ControlFlowHoister(LoopInfo *LI, DominatorTree *DT, Loop *CurLoop,
                   MemorySSAUpdater *MSSAU)
    : LI(LI), DT(DT), CurLoop(CurLoop), MSSAU(MSSAU) {}

void registerPossiblyHoistableBranch(BranchInst *BI) {
  // We can only hoist conditional branches with loop invariant operands.
  if (!ControlFlowHoisting || !BI->isConditional() ||
      !CurLoop->hasLoopInvariantOperands(BI))
    return;

  // The branch destinations need to be in the loop, and we don't gain
  // anything by duplicating conditional branches with duplicate successors,
  // as it's essentially the same as an unconditional branch.
  BasicBlock *TrueDest = BI->getSuccessor(0);
  BasicBlock *FalseDest = BI->getSuccessor(1);
  if (!CurLoop->contains(TrueDest) || !CurLoop->contains(FalseDest) ||
      TrueDest == FalseDest)
    return;

  // We can hoist BI if one branch destination is the successor of the other,
  // or both have common successor which we check by seeing if the
  // intersection of their successors is non-empty.
  // TODO: This could be expanded to allowing branches where both ends
  // eventually converge to a single block.
  SmallPtrSet<BasicBlock *, 4> TrueDestSucc, FalseDestSucc;
  TrueDestSucc.insert(succ_begin(TrueDest), succ_end(TrueDest));
  FalseDestSucc.insert(succ_begin(FalseDest), succ_end(FalseDest));
  BasicBlock *CommonSucc = nullptr;
  if (TrueDestSucc.count(FalseDest)) {
    CommonSucc = FalseDest;
  } else if (FalseDestSucc.count(TrueDest)) {
    CommonSucc = TrueDest;
  } else {
    set_intersect(TrueDestSucc, FalseDestSucc);
    // If there's one common successor use that.
    if (TrueDestSucc.size() == 1)
      CommonSucc = *TrueDestSucc.begin();
    // If there's more than one pick whichever appears first in the block list
    // (we can't use the value returned by TrueDestSucc.begin() as it's
    // unpredicatable which element gets returned).
    else if (!TrueDestSucc.empty()) {
      Function *F = TrueDest->getParent();
      auto IsSucc = [&](BasicBlock &BB) { return TrueDestSucc.count(&BB); };
      auto It = llvm::find_if(*F, IsSucc);
      assert(It != F->end() && "Could not find successor in function")(static_cast <bool> (It != F->end() && "Could not find successor in function"
) ? void (0) : __assert_fail ("It != F->end() && \"Could not find successor in function\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 668, __extension__ __PRETTY_FUNCTION__));
      CommonSucc = &*It;
    }
  }
  // The common successor has to be dominated by the branch, as otherwise
  // there will be some other path to the successor that will not be
  // controlled by this branch so any phi we hoist would be controlled by the
  // wrong condition. This also takes care of avoiding hoisting of loop back
  // edges.
  // TODO: In some cases this could be relaxed if the successor is dominated
  // by another block that's been hoisted and we can guarantee that the
  // control flow has been replicated exactly.
  if (CommonSucc && DT->dominates(BI, CommonSucc))
    HoistableBranches[BI] = CommonSucc;
}

bool canHoistPHI(PHINode *PN) {
  // The phi must have loop invariant operands.
  if (!ControlFlowHoisting || !CurLoop->hasLoopInvariantOperands(PN))
    return false;
  // We can hoist phis if the block they are in is the target of hoistable
  // branches which cover all of the predecessors of the block.
  SmallPtrSet<BasicBlock *, 8> PredecessorBlocks;
  BasicBlock *BB = PN->getParent();
  for (BasicBlock *PredBB : predecessors(BB))
    PredecessorBlocks.insert(PredBB);
  // If we have less predecessor blocks than predecessors then the phi will
  // have more than one incoming value for the same block which we can't
  // handle.
  // TODO: This could be handled be erasing some of the duplicate incoming
  // values.
  if (PredecessorBlocks.size() != pred_size(BB))
    return false;
  for (auto &Pair : HoistableBranches) {
    if (Pair.second == BB) {
      // Which blocks are predecessors via this branch depends on if the
      // branch is triangle-like or diamond-like.
      if (Pair.first->getSuccessor(0) == BB) {
        PredecessorBlocks.erase(Pair.first->getParent());
        PredecessorBlocks.erase(Pair.first->getSuccessor(1));
      } else if (Pair.first->getSuccessor(1) == BB) {
        PredecessorBlocks.erase(Pair.first->getParent());
        PredecessorBlocks.erase(Pair.first->getSuccessor(0));
      } else {
        PredecessorBlocks.erase(Pair.first->getSuccessor(0));
        PredecessorBlocks.erase(Pair.first->getSuccessor(1));
      }
    }
  }
  // PredecessorBlocks will now be empty if for every predecessor of BB we
  // found a hoistable branch source.
  return PredecessorBlocks.empty();
}

BasicBlock *getOrCreateHoistedBlock(BasicBlock *BB) {
  if (!ControlFlowHoisting)
    return CurLoop->getLoopPreheader();
  // If BB has already been hoisted, return that
  if (HoistDestinationMap.count(BB))
    return HoistDestinationMap[BB];

  // Check if this block is conditional based on a pending branch
  auto HasBBAsSuccessor =
      [&](DenseMap<BranchInst *, BasicBlock *>::value_type &Pair) {
        return BB != Pair.second && (Pair.first->getSuccessor(0) == BB ||
                                     Pair.first->getSuccessor(1) == BB);
      };
  auto It = llvm::find_if(HoistableBranches, HasBBAsSuccessor);

  // If not involved in a pending branch, hoist to preheader
  BasicBlock *InitialPreheader = CurLoop->getLoopPreheader();
  if (It == HoistableBranches.end()) {
    LLVM_DEBUG(dbgs() << "LICM using "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM using " << InitialPreheader
->getNameOrAsOperand() << " as hoist destination for "
 << BB->getNameOrAsOperand() << "\n"; } } while
 (false)
                      << InitialPreheader->getNameOrAsOperand()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM using " << InitialPreheader
->getNameOrAsOperand() << " as hoist destination for "
 << BB->getNameOrAsOperand() << "\n"; } } while
 (false)
                      << " as hoist destination for "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM using " << InitialPreheader
->getNameOrAsOperand() << " as hoist destination for "
 << BB->getNameOrAsOperand() << "\n"; } } while
 (false)
                      << BB->getNameOrAsOperand() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM using " << InitialPreheader
->getNameOrAsOperand() << " as hoist destination for "
 << BB->getNameOrAsOperand() << "\n"; } } while
 (false);
    HoistDestinationMap[BB] = InitialPreheader;
    return InitialPreheader;
  }
  BranchInst *BI = It->first;
  assert(std::find_if(++It, HoistableBranches.end(), HasBBAsSuccessor) ==(static_cast <bool> (std::find_if(++It, HoistableBranches
.end(), HasBBAsSuccessor) == HoistableBranches.end() &&
 "BB is expected to be the target of at most one branch") ? void
 (0) : __assert_fail ("std::find_if(++It, HoistableBranches.end(), HasBBAsSuccessor) == HoistableBranches.end() && \"BB is expected to be the target of at most one branch\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 750, __extension__ __PRETTY_FUNCTION__))
             HoistableBranches.end() &&(static_cast <bool> (std::find_if(++It, HoistableBranches
.end(), HasBBAsSuccessor) == HoistableBranches.end() &&
 "BB is expected to be the target of at most one branch") ? void
 (0) : __assert_fail ("std::find_if(++It, HoistableBranches.end(), HasBBAsSuccessor) == HoistableBranches.end() && \"BB is expected to be the target of at most one branch\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 750, __extension__ __PRETTY_FUNCTION__))
         "BB is expected to be the target of at most one branch")(static_cast <bool> (std::find_if(++It, HoistableBranches
.end(), HasBBAsSuccessor) == HoistableBranches.end() &&
 "BB is expected to be the target of at most one branch") ? void
 (0) : __assert_fail ("std::find_if(++It, HoistableBranches.end(), HasBBAsSuccessor) == HoistableBranches.end() && \"BB is expected to be the target of at most one branch\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 750, __extension__ __PRETTY_FUNCTION__));

  LLVMContext &C = BB->getContext();
  BasicBlock *TrueDest = BI->getSuccessor(0);
  BasicBlock *FalseDest = BI->getSuccessor(1);
  BasicBlock *CommonSucc = HoistableBranches[BI];
  BasicBlock *HoistTarget = getOrCreateHoistedBlock(BI->getParent());

  // Create hoisted versions of blocks that currently don't have them
  auto CreateHoistedBlock = [&](BasicBlock *Orig) {
    if (HoistDestinationMap.count(Orig))
      return HoistDestinationMap[Orig];
    BasicBlock *New =
        BasicBlock::Create(C, Orig->getName() + ".licm", Orig->getParent());
    HoistDestinationMap[Orig] = New;
    DT->addNewBlock(New, HoistTarget);
    if (CurLoop->getParentLoop())
      CurLoop->getParentLoop()->addBasicBlockToLoop(New, *LI);
    ++NumCreatedBlocks;
    LLVM_DEBUG(dbgs() << "LICM created " << New->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM created " << New->
getName() << " as hoist destination for " << Orig
->getName() << "\n"; } } while (false)
                      << " as hoist destination for " << Orig->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM created " << New->
getName() << " as hoist destination for " << Orig
->getName() << "\n"; } } while (false)
                      << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM created " << New->
getName() << " as hoist destination for " << Orig
->getName() << "\n"; } } while (false);
    return New;
  };
  BasicBlock *HoistTrueDest = CreateHoistedBlock(TrueDest);
  BasicBlock *HoistFalseDest = CreateHoistedBlock(FalseDest);
  BasicBlock *HoistCommonSucc = CreateHoistedBlock(CommonSucc);

  // Link up these blocks with branches.
  if (!HoistCommonSucc->getTerminator()) {
    // The new common successor we've generated will branch to whatever that
    // hoist target branched to.
    BasicBlock *TargetSucc = HoistTarget->getSingleSuccessor();
    assert(TargetSucc && "Expected hoist target to have a single successor")(static_cast <bool> (TargetSucc && "Expected hoist target to have a single successor"
) ? void (0) : __assert_fail ("TargetSucc && \"Expected hoist target to have a single successor\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 783, __extension__ __PRETTY_FUNCTION__));
    HoistCommonSucc->moveBefore(TargetSucc);
    BranchInst::Create(TargetSucc, HoistCommonSucc);
  }
  if (!HoistTrueDest->getTerminator()) {
    HoistTrueDest->moveBefore(HoistCommonSucc);
    BranchInst::Create(HoistCommonSucc, HoistTrueDest);
  }
  if (!HoistFalseDest->getTerminator()) {
    HoistFalseDest->moveBefore(HoistCommonSucc);
    BranchInst::Create(HoistCommonSucc, HoistFalseDest);
  }

  // If BI is being cloned to what was originally the preheader then
  // HoistCommonSucc will now be the new preheader.
  if (HoistTarget == InitialPreheader) {
    // Phis in the loop header now need to use the new preheader.
    InitialPreheader->replaceSuccessorsPhiUsesWith(HoistCommonSucc);
    if (MSSAU)
      MSSAU->wireOldPredecessorsToNewImmediatePredecessor(
          HoistTarget->getSingleSuccessor(), HoistCommonSucc, {HoistTarget});
    // The new preheader dominates the loop header.
    DomTreeNode *PreheaderNode = DT->getNode(HoistCommonSucc);
    DomTreeNode *HeaderNode = DT->getNode(CurLoop->getHeader());
    DT->changeImmediateDominator(HeaderNode, PreheaderNode);
    // The preheader hoist destination is now the new preheader, with the
    // exception of the hoist destination of this branch.
    for (auto &Pair : HoistDestinationMap)
      if (Pair.second == InitialPreheader && Pair.first != BI->getParent())
        Pair.second = HoistCommonSucc;
  }

  // Now finally clone BI.
  ReplaceInstWithInst(
      HoistTarget->getTerminator(),
      BranchInst::Create(HoistTrueDest, HoistFalseDest, BI->getCondition()));
  ++NumClonedBranches;

  assert(CurLoop->getLoopPreheader() &&(static_cast <bool> (CurLoop->getLoopPreheader() &&
 "Hoisting blocks should not have destroyed preheader") ? void
 (0) : __assert_fail ("CurLoop->getLoopPreheader() && \"Hoisting blocks should not have destroyed preheader\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 822, __extension__ __PRETTY_FUNCTION__))
         "Hoisting blocks should not have destroyed preheader")(static_cast <bool> (CurLoop->getLoopPreheader() &&
 "Hoisting blocks should not have destroyed preheader") ? void
 (0) : __assert_fail ("CurLoop->getLoopPreheader() && \"Hoisting blocks should not have destroyed preheader\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 822, __extension__ __PRETTY_FUNCTION__));
  return HoistDestinationMap[BB];
}
825};
826} // namespace

828// Hoisting/sinking instruction out of a loop isn't always beneficial. It's only
829// only worthwhile if the destination block is actually colder than current
830// block.
831static bool worthSinkOrHoistInst(Instruction &I, BasicBlock *DstBlock,
                               OptimizationRemarkEmitter *ORE,
                               BlockFrequencyInfo *BFI) {
// Check block frequency only when runtime profile is available
// to avoid pathological cases. With static profile, lean towards
// hosting because it helps canonicalize the loop for vectorizer.
if (!DstBlock->getParent()->hasProfileData())
  return true;

if (!HoistSinkColdnessThreshold || !BFI)
  return true;

BasicBlock *SrcBlock = I.getParent();
if (BFI->getBlockFreq(DstBlock).getFrequency() / HoistSinkColdnessThreshold >
    BFI->getBlockFreq(SrcBlock).getFrequency()) {
  ORE->emit([&]() {
    return OptimizationRemarkMissed(DEBUG_TYPE"licm", "SinkHoistInst", &I)
           << "failed to sink or hoist instruction because containing block "
              "has lower frequency than destination block";
  });
  return false;
}

return true;
855}

857/// Walk the specified region of the CFG (defined by all blocks dominated by
858/// the specified block, and that are in the current loop) in depth first
859/// order w.r.t the DominatorTree.  This allows us to visit definitions before
860/// uses, allowing us to hoist a loop body in one pass without iteration.
861///
862bool llvm::hoistRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI,
                     DominatorTree *DT, BlockFrequencyInfo *BFI,
                     TargetLibraryInfo *TLI, Loop *CurLoop,
                     AliasSetTracker *CurAST, MemorySSAUpdater *MSSAU,
                     ScalarEvolution *SE, ICFLoopSafetyInfo *SafetyInfo,
                     SinkAndHoistLICMFlags &Flags,
                     OptimizationRemarkEmitter *ORE) {
// Verify inputs.
assert(N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr &&(static_cast <bool> (N != nullptr && AA != nullptr
 && LI != nullptr && DT != nullptr &&
 CurLoop != nullptr && SafetyInfo != nullptr &&
 "Unexpected input to hoistRegion.") ? void (0) : __assert_fail
 ("N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected input to hoistRegion.\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 872, __extension__ __PRETTY_FUNCTION__))
       CurLoop != nullptr && SafetyInfo != nullptr &&(static_cast <bool> (N != nullptr && AA != nullptr
 && LI != nullptr && DT != nullptr &&
 CurLoop != nullptr && SafetyInfo != nullptr &&
 "Unexpected input to hoistRegion.") ? void (0) : __assert_fail
 ("N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected input to hoistRegion.\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 872, __extension__ __PRETTY_FUNCTION__))
       "Unexpected input to hoistRegion.")(static_cast <bool> (N != nullptr && AA != nullptr
 && LI != nullptr && DT != nullptr &&
 CurLoop != nullptr && SafetyInfo != nullptr &&
 "Unexpected input to hoistRegion.") ? void (0) : __assert_fail
 ("N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected input to hoistRegion.\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 872, __extension__ __PRETTY_FUNCTION__));
assert(((CurAST != nullptr) ^ (MSSAU != nullptr)) &&(static_cast <bool> (((CurAST != nullptr) ^ (MSSAU != nullptr
)) && "Either AliasSetTracker or MemorySSA should be initialized."
) ? void (0) : __assert_fail ("((CurAST != nullptr) ^ (MSSAU != nullptr)) && \"Either AliasSetTracker or MemorySSA should be initialized.\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 874, __extension__ __PRETTY_FUNCTION__))
       "Either AliasSetTracker or MemorySSA should be initialized.")(static_cast <bool> (((CurAST != nullptr) ^ (MSSAU != nullptr
)) && "Either AliasSetTracker or MemorySSA should be initialized."
) ? void (0) : __assert_fail ("((CurAST != nullptr) ^ (MSSAU != nullptr)) && \"Either AliasSetTracker or MemorySSA should be initialized.\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 874, __extension__ __PRETTY_FUNCTION__));

ControlFlowHoister CFH(LI, DT, CurLoop, MSSAU);

// Keep track of instructions that have been hoisted, as they may need to be
// re-hoisted if they end up not dominating all of their uses.
SmallVector<Instruction *, 16> HoistedInstructions;

// For PHI hoisting to work we need to hoist blocks before their successors.
// We can do this by iterating through the blocks in the loop in reverse
// post-order.
LoopBlocksRPO Worklist(CurLoop);
Worklist.perform(LI);
bool Changed = false;
for (BasicBlock *BB : Worklist) {
  // Only need to process the contents of this block if it is not part of a
  // subloop (which would already have been processed).
  if (inSubLoop(BB, CurLoop, LI))
    continue;

  for (BasicBlock::iterator II = BB->begin(), E = BB->end(); II != E;) {
    Instruction &I = *II++;
    // Try constant folding this instruction.  If all the operands are
    // constants, it is technically hoistable, but it would be better to
    // just fold it.
    if (Constant *C = ConstantFoldInstruction(
            &I, I.getModule()->getDataLayout(), TLI)) {
      LLVM_DEBUG(dbgs() << "LICM folding inst: " << I << "  --> " << *Cdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM folding inst: " << I <<
 "  --> " << *C << '\n'; } } while (false)
                        << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM folding inst: " << I <<
 "  --> " << *C << '\n'; } } while (false);
      if (CurAST)
        CurAST->copyValue(&I, C);
      // FIXME MSSA: Such replacements may make accesses unoptimized (D51960).
      I.replaceAllUsesWith(C);
      if (isInstructionTriviallyDead(&I, TLI))
        eraseInstruction(I, *SafetyInfo, CurAST, MSSAU);
      Changed = true;
      continue;
    }

    // Try hoisting the instruction out to the preheader.  We can only do
    // this if all of the operands of the instruction are loop invariant and
    // if it is safe to hoist the instruction. We also check block frequency
    // to make sure instruction only gets hoisted into colder blocks.
    // TODO: It may be safe to hoist if we are hoisting to a conditional block
    // and we have accurately duplicated the control flow from the loop header
    // to that block.
    if (CurLoop->hasLoopInvariantOperands(&I) &&
        canSinkOrHoistInst(I, AA, DT, CurLoop, CurAST, MSSAU, true, &Flags,
                           ORE) &&
        worthSinkOrHoistInst(I, CurLoop->getLoopPreheader(), ORE, BFI) &&
        isSafeToExecuteUnconditionally(
            I, DT, TLI, CurLoop, SafetyInfo, ORE,
            CurLoop->getLoopPreheader()->getTerminator())) {
      hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo,
            MSSAU, SE, ORE);
      HoistedInstructions.push_back(&I);
      Changed = true;
      continue;
    }

    // Attempt to remove floating point division out of the loop by
    // converting it to a reciprocal multiplication.
    if (I.getOpcode() == Instruction::FDiv && I.hasAllowReciprocal() &&
        CurLoop->isLoopInvariant(I.getOperand(1))) {
      auto Divisor = I.getOperand(1);
      auto One = llvm::ConstantFP::get(Divisor->getType(), 1.0);
      auto ReciprocalDivisor = BinaryOperator::CreateFDiv(One, Divisor);
      ReciprocalDivisor->setFastMathFlags(I.getFastMathFlags());
      SafetyInfo->insertInstructionTo(ReciprocalDivisor, I.getParent());
      ReciprocalDivisor->insertBefore(&I);

      auto Product =
          BinaryOperator::CreateFMul(I.getOperand(0), ReciprocalDivisor);
      Product->setFastMathFlags(I.getFastMathFlags());
      SafetyInfo->insertInstructionTo(Product, I.getParent());
      Product->insertAfter(&I);
      I.replaceAllUsesWith(Product);
      eraseInstruction(I, *SafetyInfo, CurAST, MSSAU);

      hoist(*ReciprocalDivisor, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB),
            SafetyInfo, MSSAU, SE, ORE);
      HoistedInstructions.push_back(ReciprocalDivisor);
      Changed = true;
      continue;
    }

    auto IsInvariantStart = [&](Instruction &I) {
      using namespace PatternMatch;
      return I.use_empty() &&
             match(&I, m_Intrinsic<Intrinsic::invariant_start>());
    };
    auto MustExecuteWithoutWritesBefore = [&](Instruction &I) {
      return SafetyInfo->isGuaranteedToExecute(I, DT, CurLoop) &&
             SafetyInfo->doesNotWriteMemoryBefore(I, CurLoop);
    };
    if ((IsInvariantStart(I) || isGuard(&I)) &&
        CurLoop->hasLoopInvariantOperands(&I) &&
        MustExecuteWithoutWritesBefore(I)) {
      hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo,
            MSSAU, SE, ORE);
      HoistedInstructions.push_back(&I);
      Changed = true;
      continue;
    }

    if (PHINode *PN = dyn_cast<PHINode>(&I)) {
      if (CFH.canHoistPHI(PN)) {
        // Redirect incoming blocks first to ensure that we create hoisted
        // versions of those blocks before we hoist the phi.
        for (unsigned int i = 0; i < PN->getNumIncomingValues(); ++i)
          PN->setIncomingBlock(
              i, CFH.getOrCreateHoistedBlock(PN->getIncomingBlock(i)));
        hoist(*PN, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo,
              MSSAU, SE, ORE);
        assert(DT->dominates(PN, BB) && "Conditional PHIs not expected")(static_cast <bool> (DT->dominates(PN, BB) &&
 "Conditional PHIs not expected") ? void (0) : __assert_fail (
"DT->dominates(PN, BB) && \"Conditional PHIs not expected\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 988, __extension__ __PRETTY_FUNCTION__));
        Changed = true;
        continue;
      }
    }

    // Remember possibly hoistable branches so we can actually hoist them
    // later if needed.
    if (BranchInst *BI = dyn_cast<BranchInst>(&I))
      CFH.registerPossiblyHoistableBranch(BI);
  }
}

// If we hoisted instructions to a conditional block they may not dominate
// their uses that weren't hoisted (such as phis where some operands are not
// loop invariant). If so make them unconditional by moving them to their
// immediate dominator. We iterate through the instructions in reverse order
// which ensures that when we rehoist an instruction we rehoist its operands,
// and also keep track of where in the block we are rehoisting to to make sure
// that we rehoist instructions before the instructions that use them.
Instruction *HoistPoint = nullptr;
if (ControlFlowHoisting) {
  for (Instruction *I : reverse(HoistedInstructions)) {
    if (!llvm::all_of(I->uses(),
                      [&](Use &U) { return DT->dominates(I, U); })) {
      BasicBlock *Dominator =
          DT->getNode(I->getParent())->getIDom()->getBlock();
      if (!HoistPoint || !DT->dominates(HoistPoint->getParent(), Dominator)) {
        if (HoistPoint)
          assert(DT->dominates(Dominator, HoistPoint->getParent()) &&(static_cast <bool> (DT->dominates(Dominator, HoistPoint
->getParent()) && "New hoist point expected to dominate old hoist point"
) ? void (0) : __assert_fail ("DT->dominates(Dominator, HoistPoint->getParent()) && \"New hoist point expected to dominate old hoist point\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1018, __extension__ __PRETTY_FUNCTION__))
                 "New hoist point expected to dominate old hoist point")(static_cast <bool> (DT->dominates(Dominator, HoistPoint
->getParent()) && "New hoist point expected to dominate old hoist point"
) ? void (0) : __assert_fail ("DT->dominates(Dominator, HoistPoint->getParent()) && \"New hoist point expected to dominate old hoist point\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1018, __extension__ __PRETTY_FUNCTION__));
        HoistPoint = Dominator->getTerminator();
      }
      LLVM_DEBUG(dbgs() << "LICM rehoisting to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM rehoisting to " << HoistPoint
->getParent()->getNameOrAsOperand() << ": " <<
 *I << "\n"; } } while (false)
                        << HoistPoint->getParent()->getNameOrAsOperand()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM rehoisting to " << HoistPoint
->getParent()->getNameOrAsOperand() << ": " <<
 *I << "\n"; } } while (false)
                        << ": " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM rehoisting to " << HoistPoint
->getParent()->getNameOrAsOperand() << ": " <<
 *I << "\n"; } } while (false);
      moveInstructionBefore(*I, *HoistPoint, *SafetyInfo, MSSAU, SE);
      HoistPoint = I;
      Changed = true;
    }
  }
}
if (MSSAU && VerifyMemorySSA)
  MSSAU->getMemorySSA()->verifyMemorySSA();

  // Now that we've finished hoisting make sure that LI and DT are still
  // valid.
1035#ifdef EXPENSIVE_CHECKS
if (Changed) {
  assert(DT->verify(DominatorTree::VerificationLevel::Fast) &&(static_cast <bool> (DT->verify(DominatorTree::VerificationLevel
::Fast) && "Dominator tree verification failed") ? void
 (0) : __assert_fail ("DT->verify(DominatorTree::VerificationLevel::Fast) && \"Dominator tree verification failed\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1038, __extension__ __PRETTY_FUNCTION__))
         "Dominator tree verification failed")(static_cast <bool> (DT->verify(DominatorTree::VerificationLevel
::Fast) && "Dominator tree verification failed") ? void
 (0) : __assert_fail ("DT->verify(DominatorTree::VerificationLevel::Fast) && \"Dominator tree verification failed\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1038, __extension__ __PRETTY_FUNCTION__));
  LI->verify(*DT);
}
1041#endif

return Changed;
1044}

1046// Return true if LI is invariant within scope of the loop. LI is invariant if
1047// CurLoop is dominated by an invariant.start representing the same memory
1048// location and size as the memory location LI loads from, and also the
1049// invariant.start has no uses.
1050static bool isLoadInvariantInLoop(LoadInst *LI, DominatorTree *DT,
                                Loop *CurLoop) {
Value *Addr = LI->getOperand(0);
const DataLayout &DL = LI->getModule()->getDataLayout();
const TypeSize LocSizeInBits = DL.getTypeSizeInBits(LI->getType());

// It is not currently possible for clang to generate an invariant.start
// intrinsic with scalable vector types because we don't support thread local
// sizeless types and we don't permit sizeless types in structs or classes.
// Furthermore, even if support is added for this in future the intrinsic
// itself is defined to have a size of -1 for variable sized objects. This
// makes it impossible to verify if the intrinsic envelops our region of
// interest. For example, both <vscale x 32 x i8> and <vscale x 16 x i8>
// types would have a -1 parameter, but the former is clearly double the size
// of the latter.
if (LocSizeInBits.isScalable())
  return false;

// if the type is i8 addrspace(x)*, we know this is the type of
// llvm.invariant.start operand
auto *PtrInt8Ty = PointerType::get(Type::getInt8Ty(LI->getContext()),
                                   LI->getPointerAddressSpace());
unsigned BitcastsVisited = 0;
// Look through bitcasts until we reach the i8* type (this is invariant.start
// operand type).
while (Addr->getType() != PtrInt8Ty) {
  auto *BC = dyn_cast<BitCastInst>(Addr);
  // Avoid traversing high number of bitcast uses.
  if (++BitcastsVisited > MaxNumUsesTraversed || !BC)
    return false;
  Addr = BC->getOperand(0);
}

unsigned UsesVisited = 0;
// Traverse all uses of the load operand value, to see if invariant.start is
// one of the uses, and whether it dominates the load instruction.
for (auto *U : Addr->users()) {
  // Avoid traversing for Load operand with high number of users.
  if (++UsesVisited > MaxNumUsesTraversed)
    return false;
  IntrinsicInst *II = dyn_cast<IntrinsicInst>(U);
  // If there are escaping uses of invariant.start instruction, the load maybe
  // non-invariant.
  if (!II || II->getIntrinsicID() != Intrinsic::invariant_start ||
      !II->use_empty())
    continue;
  ConstantInt *InvariantSize = cast<ConstantInt>(II->getArgOperand(0));
  // The intrinsic supports having a -1 argument for variable sized objects
  // so we should check for that here.
  if (InvariantSize->isNegative())
    continue;
  uint64_t InvariantSizeInBits = InvariantSize->getSExtValue() * 8;
  // Confirm the invariant.start location size contains the load operand size
  // in bits. Also, the invariant.start should dominate the load, and we
  // should not hoist the load out of a loop that contains this dominating
  // invariant.start.
  if (LocSizeInBits.getFixedSize() <= InvariantSizeInBits &&
      DT->properlyDominates(II->getParent(), CurLoop->getHeader()))
    return true;
}

return false;
1112}

1114namespace {
1115/// Return true if-and-only-if we know how to (mechanically) both hoist and
1116/// sink a given instruction out of a loop.  Does not address legality
1117/// concerns such as aliasing or speculation safety.
1118bool isHoistableAndSinkableInst(Instruction &I) {
// Only these instructions are hoistable/sinkable.
return (isa<LoadInst>(I) || isa<StoreInst>(I) || isa<CallInst>(I) ||
5
←
Assuming 'I' is a 'LoadInst'→
6
←
Returning the value 1, which participates in a condition later→
        isa<FenceInst>(I) || isa<CastInst>(I) || isa<UnaryOperator>(I) ||
        isa<BinaryOperator>(I) || isa<SelectInst>(I) ||
        isa<GetElementPtrInst>(I) || isa<CmpInst>(I) ||
        isa<InsertElementInst>(I) || isa<ExtractElementInst>(I) ||
        isa<ShuffleVectorInst>(I) || isa<ExtractValueInst>(I) ||
        isa<InsertValueInst>(I) || isa<FreezeInst>(I));
1127}
1128/// Return true if all of the alias sets within this AST are known not to
1129/// contain a Mod, or if MSSA knows there are no MemoryDefs in the loop.
1130bool isReadOnly(AliasSetTracker *CurAST, const MemorySSAUpdater *MSSAU,
              const Loop *L) {
if (CurAST) {
  for (AliasSet &AS : *CurAST) {
    if (!AS.isForwardingAliasSet() && AS.isMod()) {
      return false;
    }
  }
  return true;
} else { /*MSSAU*/
  for (auto *BB : L->getBlocks())
    if (MSSAU->getMemorySSA()->getBlockDefs(BB))
      return false;
  return true;
}
1145}

1147/// Return true if I is the only Instruction with a MemoryAccess in L.
1148bool isOnlyMemoryAccess(const Instruction *I, const Loop *L,
                      const MemorySSAUpdater *MSSAU) {
for (auto *BB : L->getBlocks())
  if (auto *Accs = MSSAU->getMemorySSA()->getBlockAccesses(BB)) {
    int NotAPhi = 0;
    for (const auto &Acc : *Accs) {
      if (isa<MemoryPhi>(&Acc))
        continue;
      const auto *MUD = cast<MemoryUseOrDef>(&Acc);
      if (MUD->getMemoryInst() != I || NotAPhi++ == 1)
        return false;
    }
  }
return true;
1162}
1163}

1165bool llvm::canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
                            Loop *CurLoop, AliasSetTracker *CurAST,
                            MemorySSAUpdater *MSSAU,
                            bool TargetExecutesOncePerLoop,
                            SinkAndHoistLICMFlags *Flags,
                            OptimizationRemarkEmitter *ORE) {
assert(((CurAST != nullptr) ^ (MSSAU != nullptr)) &&(static_cast <bool> (((CurAST != nullptr) ^ (MSSAU != nullptr
)) && "Either AliasSetTracker or MemorySSA should be initialized."
) ? void (0) : __assert_fail ("((CurAST != nullptr) ^ (MSSAU != nullptr)) && \"Either AliasSetTracker or MemorySSA should be initialized.\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1172, __extension__ __PRETTY_FUNCTION__))
1
Assuming pointer value is null→
2
←
Assuming the condition is true→
3
←
'?' condition is true→
       "Either AliasSetTracker or MemorySSA should be initialized.")(static_cast <bool> (((CurAST != nullptr) ^ (MSSAU != nullptr
)) && "Either AliasSetTracker or MemorySSA should be initialized."
) ? void (0) : __assert_fail ("((CurAST != nullptr) ^ (MSSAU != nullptr)) && \"Either AliasSetTracker or MemorySSA should be initialized.\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1172, __extension__ __PRETTY_FUNCTION__));

// If we don't understand the instruction, bail early.
if (!isHoistableAndSinkableInst(I))
4
←
Calling 'isHoistableAndSinkableInst'→
7
←
Returning from 'isHoistableAndSinkableInst'→
8
←
Taking false branch→
  return false;

MemorySSA *MSSA = MSSAU8.1
'MSSAU' is non-null
1
'MSSAU' is non-null
1
'MSSAU' is non-null
1
'MSSAU' is non-null
1
'MSSAU' is non-null
 ? MSSAU->getMemorySSA() : nullptr;
9
←
'?' condition is true→
10
←
'MSSA' initialized here→
if (MSSA)
11
←
Assuming 'MSSA' is null→
12
←
Taking false branch→
  assert(Flags != nullptr && "Flags cannot be null.")(static_cast <bool> (Flags != nullptr && "Flags cannot be null."
) ? void (0) : __assert_fail ("Flags != nullptr && \"Flags cannot be null.\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1180, __extension__ __PRETTY_FUNCTION__));

// Loads have extra constraints we have to verify before we can hoist them.
if (LoadInst *LI13.1
'LI' is null
1
'LI' is null
1
'LI' is null
1
'LI' is null
1
'LI' is null
 = dyn_cast<LoadInst>(&I)) {
13
←
Assuming the object is not a 'LoadInst'→
14
←
Taking false branch→
  if (!LI->isUnordered())
    return false; // Don't sink/hoist volatile or ordered atomic loads!

  // Loads from constant memory are always safe to move, even if they end up
  // in the same alias set as something that ends up being modified.
  if (AA->pointsToConstantMemory(LI->getOperand(0)))
    return true;
  if (LI->hasMetadata(LLVMContext::MD_invariant_load))
    return true;

  if (LI->isAtomic() && !TargetExecutesOncePerLoop)
    return false; // Don't risk duplicating unordered loads

  // This checks for an invariant.start dominating the load.
  if (isLoadInvariantInLoop(LI, DT, CurLoop))
    return true;

  // Stores with an invariant.group metadata are ok to sink/hoist.
  if (LI->hasMetadata(LLVMContext::MD_invariant_group))
    return true;

  bool Invalidated;
  if (CurAST)
    Invalidated = pointerInvalidatedByLoop(MemoryLocation::get(LI), CurAST,
                                           CurLoop, AA);
  else
    Invalidated = pointerInvalidatedByLoopWithMSSA(
        MSSA, cast<MemoryUse>(MSSA->getMemoryAccess(LI)), CurLoop, I, *Flags);
  // Check loop-invariant address because this may also be a sinkable load
  // whose address is not necessarily loop-invariant.
  if (ORE && Invalidated && CurLoop->isLoopInvariant(LI->getPointerOperand()))
    ORE->emit([&]() {
      return OptimizationRemarkMissed(
                 DEBUG_TYPE"licm", "LoadWithLoopInvariantAddressInvalidated", LI)
             << "failed to move load with loop-invariant address "
                "because the loop may invalidate its value";
    });

  return !Invalidated;
} else if (CallInst *CI15.1
'CI' is non-null
1
'CI' is non-null
1
'CI' is non-null
1
'CI' is non-null
1
'CI' is non-null
 = dyn_cast<CallInst>(&I)) {
15
←
Assuming the object is a 'CallInst'→
16
←
Taking true branch→
  // Don't sink or hoist dbg info; it's legal, but not useful.
  if (isa<DbgInfoIntrinsic>(I))
17
←
Assuming 'I' is not a 'DbgInfoIntrinsic'→
18
←
Taking false branch→
    return false;

  // Don't sink calls which can throw.
  if (CI->mayThrow())
19
←
Assuming the condition is false→
20
←
Taking false branch→
    return false;

  // Convergent attribute has been used on operations that involve
  // inter-thread communication which results are implicitly affected by the
  // enclosing control flows. It is not safe to hoist or sink such operations
  // across control flow.
  if (CI->isConvergent())
21
←
Calling 'CallBase::isConvergent'→
37
←
Returning from 'CallBase::isConvergent'→
38
←
Assuming the condition is false→
39
←
Taking false branch→
    return false;

  using namespace PatternMatch;
  if (match(CI, m_Intrinsic<Intrinsic::assume>()))
40
←
Calling 'match<llvm::CallInst, llvm::PatternMatch::IntrinsicID_match>'→
47
←
Returning from 'match<llvm::CallInst, llvm::PatternMatch::IntrinsicID_match>'→
48
←
Taking false branch→
    // Assumes don't actually alias anything or throw
    return true;

  if (match(CI, m_Intrinsic<Intrinsic::experimental_widenable_condition>()))
49
←
Calling 'match<llvm::CallInst, llvm::PatternMatch::IntrinsicID_match>'→
56
←
Returning from 'match<llvm::CallInst, llvm::PatternMatch::IntrinsicID_match>'→
57
←
Taking false branch→
    // Widenable conditions don't actually alias anything or throw
    return true;

  // Handle simple cases by querying alias analysis.
  FunctionModRefBehavior Behavior = AA->getModRefBehavior(CI);
  if (Behavior == FMRB_DoesNotAccessMemory)
58
←
Assuming 'Behavior' is not equal to FMRB_DoesNotAccessMemory→
59
←
Taking false branch→
    return true;
  if (AAResults::onlyReadsMemory(Behavior)) {
60
←
Calling 'AAResults::onlyReadsMemory'→
63
←
Returning from 'AAResults::onlyReadsMemory'→
64
←
Taking true branch→
    // A readonly argmemonly function only reads from memory pointed to by
    // it's arguments with arbitrary offsets.  If we can prove there are no
    // writes to this memory in the loop, we can hoist or sink.
    if (AAResults::onlyAccessesArgPointees(Behavior)) {
65
←
Calling 'AAResults::onlyAccessesArgPointees'→
68
←
Returning from 'AAResults::onlyAccessesArgPointees'→
69
←
Taking true branch→
      // TODO: expand to writeable arguments
      for (Value *Op : CI->arg_operands())
70
←
Assuming '__begin5' is not equal to '__end5'→
        if (Op->getType()->isPointerTy()) {
71
←
Calling 'Type::isPointerTy'→
74
←
Returning from 'Type::isPointerTy'→
75
←
Taking true branch→
          bool Invalidated;
          if (CurAST75.1
'CurAST' is null
1
'CurAST' is null
1
'CurAST' is null
1
'CurAST' is null
1
'CurAST' is null
)
76
←
Taking false branch→
            Invalidated = pointerInvalidatedByLoop(
                MemoryLocation::getBeforeOrAfter(Op), CurAST, CurLoop, AA);
          else
            Invalidated = pointerInvalidatedByLoopWithMSSA(
                MSSA, cast<MemoryUse>(MSSA->getMemoryAccess(CI)), CurLoop, I,
77
←
Called C++ object pointer is null
                *Flags);
          if (Invalidated)
            return false;
        }
      return true;
    }

    // If this call only reads from memory and there are no writes to memory
    // in the loop, we can hoist or sink the call as appropriate.
    if (isReadOnly(CurAST, MSSAU, CurLoop))
      return true;
  }

  // FIXME: This should use mod/ref information to see if we can hoist or
  // sink the call.

  return false;
} else if (auto *FI = dyn_cast<FenceInst>(&I)) {
  // Fences alias (most) everything to provide ordering.  For the moment,
  // just give up if there are any other memory operations in the loop.
  if (CurAST) {
    auto Begin = CurAST->begin();
    assert(Begin != CurAST->end() && "must contain FI")(static_cast <bool> (Begin != CurAST->end() &&
 "must contain FI") ? void (0) : __assert_fail ("Begin != CurAST->end() && \"must contain FI\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1289, __extension__ __PRETTY_FUNCTION__));
    if (std::next(Begin) != CurAST->end())
      // constant memory for instance, TODO: handle better
      return false;
    auto *UniqueI = Begin->getUniqueInstruction();
    if (!UniqueI)
      // other memory op, give up
      return false;
    (void)FI; // suppress unused variable warning
    assert(UniqueI == FI && "AS must contain FI")(static_cast <bool> (UniqueI == FI && "AS must contain FI"
) ? void (0) : __assert_fail ("UniqueI == FI && \"AS must contain FI\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1298, __extension__ __PRETTY_FUNCTION__));
    return true;
  } else // MSSAU
    return isOnlyMemoryAccess(FI, CurLoop, MSSAU);
} else if (auto *SI = dyn_cast<StoreInst>(&I)) {
  if (!SI->isUnordered())
    return false; // Don't sink/hoist volatile or ordered atomic store!

  // We can only hoist a store that we can prove writes a value which is not
  // read or overwritten within the loop.  For those cases, we fallback to
  // load store promotion instead.  TODO: We can extend this to cases where
  // there is exactly one write to the location and that write dominates an
  // arbitrary number of reads in the loop.
  if (CurAST) {
    auto &AS = CurAST->getAliasSetFor(MemoryLocation::get(SI));

    if (AS.isRef() || !AS.isMustAlias())
      // Quick exit test, handled by the full path below as well.
      return false;
    auto *UniqueI = AS.getUniqueInstruction();
    if (!UniqueI)
      // other memory op, give up
      return false;
    assert(UniqueI == SI && "AS must contain SI")(static_cast <bool> (UniqueI == SI && "AS must contain SI"
) ? void (0) : __assert_fail ("UniqueI == SI && \"AS must contain SI\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1321, __extension__ __PRETTY_FUNCTION__));
    return true;
  } else { // MSSAU
    if (isOnlyMemoryAccess(SI, CurLoop, MSSAU))
      return true;
    // If there are more accesses than the Promotion cap or no "quota" to
    // check clobber, then give up as we're not walking a list that long.
    if (Flags->tooManyMemoryAccesses() || Flags->tooManyClobberingCalls())
      return false;
    // If there are interfering Uses (i.e. their defining access is in the
    // loop), or ordered loads (stored as Defs!), don't move this store.
    // Could do better here, but this is conservatively correct.
    // TODO: Cache set of Uses on the first walk in runOnLoop, update when
    // moving accesses. Can also extend to dominating uses.
    auto *SIMD = MSSA->getMemoryAccess(SI);
    for (auto *BB : CurLoop->getBlocks())
      if (auto *Accesses = MSSA->getBlockAccesses(BB)) {
        for (const auto &MA : *Accesses)
          if (const auto *MU = dyn_cast<MemoryUse>(&MA)) {
            auto *MD = MU->getDefiningAccess();
            if (!MSSA->isLiveOnEntryDef(MD) &&
                CurLoop->contains(MD->getBlock()))
              return false;
            // Disable hoisting past potentially interfering loads. Optimized
            // Uses may point to an access outside the loop, as getClobbering
            // checks the previous iteration when walking the backedge.
            // FIXME: More precise: no Uses that alias SI.
            if (!Flags->getIsSink() && !MSSA->dominates(SIMD, MU))
              return false;
          } else if (const auto *MD = dyn_cast<MemoryDef>(&MA)) {
            if (auto *LI = dyn_cast<LoadInst>(MD->getMemoryInst())) {
              (void)LI; // Silence warning.
              assert(!LI->isUnordered() && "Expected unordered load")(static_cast <bool> (!LI->isUnordered() && "Expected unordered load"
) ? void (0) : __assert_fail ("!LI->isUnordered() && \"Expected unordered load\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1353, __extension__ __PRETTY_FUNCTION__));
              return false;
            }
            // Any call, while it may not be clobbering SI, it may be a use.
            if (auto *CI = dyn_cast<CallInst>(MD->getMemoryInst())) {
              // Check if the call may read from the memory location written
              // to by SI. Check CI's attributes and arguments; the number of
              // such checks performed is limited above by NoOfMemAccTooLarge.
              ModRefInfo MRI = AA->getModRefInfo(CI, MemoryLocation::get(SI));
              if (isModOrRefSet(MRI))
                return false;
            }
          }
      }
    auto *Source = MSSA->getSkipSelfWalker()->getClobberingMemoryAccess(SI);
    Flags->incrementClobberingCalls();
    // If there are no clobbering Defs in the loop, store is safe to hoist.
    return MSSA->isLiveOnEntryDef(Source) ||
           !CurLoop->contains(Source->getBlock());
  }
}

assert(!I.mayReadOrWriteMemory() && "unhandled aliasing")(static_cast <bool> (!I.mayReadOrWriteMemory() &&
 "unhandled aliasing") ? void (0) : __assert_fail ("!I.mayReadOrWriteMemory() && \"unhandled aliasing\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1375, __extension__ __PRETTY_FUNCTION__));

// We've established mechanical ability and aliasing, it's up to the caller
// to check fault safety
return true;
1380}

1382/// Returns true if a PHINode is a trivially replaceable with an
1383/// Instruction.
1384/// This is true when all incoming values are that instruction.
1385/// This pattern occurs most often with LCSSA PHI nodes.
1386///
1387static bool isTriviallyReplaceablePHI(const PHINode &PN, const Instruction &I) {
for (const Value *IncValue : PN.incoming_values())
  if (IncValue != &I)
    return false;

return true;
1393}

1395/// Return true if the instruction is free in the loop.
1396static bool isFreeInLoop(const Instruction &I, const Loop *CurLoop,
                       const TargetTransformInfo *TTI) {

if (const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(&I)) {
  if (TTI->getUserCost(GEP, TargetTransformInfo::TCK_SizeAndLatency) !=
      TargetTransformInfo::TCC_Free)
    return false;
  // For a GEP, we cannot simply use getUserCost because currently it
  // optimistically assume that a GEP will fold into addressing mode
  // regardless of its users.
  const BasicBlock *BB = GEP->getParent();
  for (const User *U : GEP->users()) {
    const Instruction *UI = cast<Instruction>(U);
    if (CurLoop->contains(UI) &&
        (BB != UI->getParent() ||
         (!isa<StoreInst>(UI) && !isa<LoadInst>(UI))))
      return false;
  }
  return true;
} else
  return TTI->getUserCost(&I, TargetTransformInfo::TCK_SizeAndLatency) ==
         TargetTransformInfo::TCC_Free;
1418}

1420/// Return true if the only users of this instruction are outside of
1421/// the loop. If this is true, we can sink the instruction to the exit
1422/// blocks of the loop.
1423///
1424/// We also return true if the instruction could be folded away in lowering.
1425/// (e.g.,  a GEP can be folded into a load as an addressing mode in the loop).
1426static bool isNotUsedOrFreeInLoop(const Instruction &I, const Loop *CurLoop,
                                const LoopSafetyInfo *SafetyInfo,
                                TargetTransformInfo *TTI, bool &FreeInLoop) {
const auto &BlockColors = SafetyInfo->getBlockColors();
bool IsFree = isFreeInLoop(I, CurLoop, TTI);
for (const User *U : I.users()) {
  const Instruction *UI = cast<Instruction>(U);
  if (const PHINode *PN = dyn_cast<PHINode>(UI)) {
    const BasicBlock *BB = PN->getParent();
    // We cannot sink uses in catchswitches.
    if (isa<CatchSwitchInst>(BB->getTerminator()))
      return false;

    // We need to sink a callsite to a unique funclet.  Avoid sinking if the
    // phi use is too muddled.
    if (isa<CallInst>(I))
      if (!BlockColors.empty() &&
          BlockColors.find(const_cast<BasicBlock *>(BB))->second.size() != 1)
        return false;
  }

  if (CurLoop->contains(UI)) {
    if (IsFree) {
      FreeInLoop = true;
      continue;
    }
    return false;
  }
}
return true;
1456}

1458static Instruction *cloneInstructionInExitBlock(
  Instruction &I, BasicBlock &ExitBlock, PHINode &PN, const LoopInfo *LI,
  const LoopSafetyInfo *SafetyInfo, MemorySSAUpdater *MSSAU) {
Instruction *New;
if (auto *CI = dyn_cast<CallInst>(&I)) {
  const auto &BlockColors = SafetyInfo->getBlockColors();

  // Sinking call-sites need to be handled differently from other
  // instructions.  The cloned call-site needs a funclet bundle operand
  // appropriate for its location in the CFG.
  SmallVector<OperandBundleDef, 1> OpBundles;
  for (unsigned BundleIdx = 0, BundleEnd = CI->getNumOperandBundles();
       BundleIdx != BundleEnd; ++BundleIdx) {
    OperandBundleUse Bundle = CI->getOperandBundleAt(BundleIdx);
    if (Bundle.getTagID() == LLVMContext::OB_funclet)
      continue;

    OpBundles.emplace_back(Bundle);
  }

  if (!BlockColors.empty()) {
    const ColorVector &CV = BlockColors.find(&ExitBlock)->second;
    assert(CV.size() == 1 && "non-unique color for exit block!")(static_cast <bool> (CV.size() == 1 && "non-unique color for exit block!"
) ? void (0) : __assert_fail ("CV.size() == 1 && \"non-unique color for exit block!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1480, __extension__ __PRETTY_FUNCTION__));
    BasicBlock *BBColor = CV.front();
    Instruction *EHPad = BBColor->getFirstNonPHI();
    if (EHPad->isEHPad())
      OpBundles.emplace_back("funclet", EHPad);
  }

  New = CallInst::Create(CI, OpBundles);
} else {
  New = I.clone();
}

ExitBlock.getInstList().insert(ExitBlock.getFirstInsertionPt(), New);
if (!I.getName().empty())
  New->setName(I.getName() + ".le");

if (MSSAU && MSSAU->getMemorySSA()->getMemoryAccess(&I)) {
  // Create a new MemoryAccess and let MemorySSA set its defining access.
  MemoryAccess *NewMemAcc = MSSAU->createMemoryAccessInBB(
      New, nullptr, New->getParent(), MemorySSA::Beginning);
  if (NewMemAcc) {
    if (auto *MemDef = dyn_cast<MemoryDef>(NewMemAcc))
      MSSAU->insertDef(MemDef, /*RenameUses=*/true);
    else {
      auto *MemUse = cast<MemoryUse>(NewMemAcc);
      MSSAU->insertUse(MemUse, /*RenameUses=*/true);
    }
  }
}

// Build LCSSA PHI nodes for any in-loop operands (if legal).  Note that
// this is particularly cheap because we can rip off the PHI node that we're
// replacing for the number and blocks of the predecessors.
// OPT: If this shows up in a profile, we can instead finish sinking all
// invariant instructions, and then walk their operands to re-establish
// LCSSA. That will eliminate creating PHI nodes just to nuke them when
// sinking bottom-up.
for (Use &Op : New->operands())
  if (LI->wouldBeOutOfLoopUseRequiringLCSSA(Op.get(), PN.getParent())) {
    auto *OInst = cast<Instruction>(Op.get());
    PHINode *OpPN =
      PHINode::Create(OInst->getType(), PN.getNumIncomingValues(),
                      OInst->getName() + ".lcssa", &ExitBlock.front());
    for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i)
      OpPN->addIncoming(OInst, PN.getIncomingBlock(i));
    Op = OpPN;
  }
return New;
1528}

1530static void eraseInstruction(Instruction &I, ICFLoopSafetyInfo &SafetyInfo,
                           AliasSetTracker *AST, MemorySSAUpdater *MSSAU) {
if (AST)
  AST->deleteValue(&I);
if (MSSAU)
  MSSAU->removeMemoryAccess(&I);
SafetyInfo.removeInstruction(&I);
I.eraseFromParent();
1538}

1540static void moveInstructionBefore(Instruction &I, Instruction &Dest,
                                ICFLoopSafetyInfo &SafetyInfo,
                                MemorySSAUpdater *MSSAU,
                                ScalarEvolution *SE) {
SafetyInfo.removeInstruction(&I);
SafetyInfo.insertInstructionTo(&I, Dest.getParent());
I.moveBefore(&Dest);
if (MSSAU)
  if (MemoryUseOrDef *OldMemAcc = cast_or_null<MemoryUseOrDef>(
          MSSAU->getMemorySSA()->getMemoryAccess(&I)))
    MSSAU->moveToPlace(OldMemAcc, Dest.getParent(),
                       MemorySSA::BeforeTerminator);
if (SE)
  SE->forgetValue(&I);
1554}

1556static Instruction *sinkThroughTriviallyReplaceablePHI(
  PHINode *TPN, Instruction *I, LoopInfo *LI,
  SmallDenseMap<BasicBlock *, Instruction *, 32> &SunkCopies,
  const LoopSafetyInfo *SafetyInfo, const Loop *CurLoop,
  MemorySSAUpdater *MSSAU) {
assert(isTriviallyReplaceablePHI(*TPN, *I) &&(static_cast <bool> (isTriviallyReplaceablePHI(*TPN, *I
) && "Expect only trivially replaceable PHI") ? void (
0) : __assert_fail ("isTriviallyReplaceablePHI(*TPN, *I) && \"Expect only trivially replaceable PHI\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1562, __extension__ __PRETTY_FUNCTION__))
       "Expect only trivially replaceable PHI")(static_cast <bool> (isTriviallyReplaceablePHI(*TPN, *I
) && "Expect only trivially replaceable PHI") ? void (
0) : __assert_fail ("isTriviallyReplaceablePHI(*TPN, *I) && \"Expect only trivially replaceable PHI\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1562, __extension__ __PRETTY_FUNCTION__));
BasicBlock *ExitBlock = TPN->getParent();
Instruction *New;
auto It = SunkCopies.find(ExitBlock);
if (It != SunkCopies.end())
  New = It->second;
else
  New = SunkCopies[ExitBlock] = cloneInstructionInExitBlock(
      *I, *ExitBlock, *TPN, LI, SafetyInfo, MSSAU);
return New;
1572}

1574static bool canSplitPredecessors(PHINode *PN, LoopSafetyInfo *SafetyInfo) {
BasicBlock *BB = PN->getParent();
if (!BB->canSplitPredecessors())
  return false;
// It's not impossible to split EHPad blocks, but if BlockColors already exist
// it require updating BlockColors for all offspring blocks accordingly. By
// skipping such corner case, we can make updating BlockColors after splitting
// predecessor fairly simple.
if (!SafetyInfo->getBlockColors().empty() && BB->getFirstNonPHI()->isEHPad())
  return false;
for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI) {
  BasicBlock *BBPred = *PI;
  if (isa<IndirectBrInst>(BBPred->getTerminator()) ||
      isa<CallBrInst>(BBPred->getTerminator()))
    return false;
}
return true;
1591}

1593static void splitPredecessorsOfLoopExit(PHINode *PN, DominatorTree *DT,
                                      LoopInfo *LI, const Loop *CurLoop,
                                      LoopSafetyInfo *SafetyInfo,
                                      MemorySSAUpdater *MSSAU) {
1597#ifndef NDEBUG
SmallVector<BasicBlock *, 32> ExitBlocks;
CurLoop->getUniqueExitBlocks(ExitBlocks);
SmallPtrSet<BasicBlock *, 32> ExitBlockSet(ExitBlocks.begin(),
                                           ExitBlocks.end());
1602#endif
BasicBlock *ExitBB = PN->getParent();
assert(ExitBlockSet.count(ExitBB) && "Expect the PHI is in an exit block.")(static_cast <bool> (ExitBlockSet.count(ExitBB) &&
 "Expect the PHI is in an exit block.") ? void (0) : __assert_fail
 ("ExitBlockSet.count(ExitBB) && \"Expect the PHI is in an exit block.\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1604, __extension__ __PRETTY_FUNCTION__));

// Split predecessors of the loop exit to make instructions in the loop are
// exposed to exit blocks through trivially replaceable PHIs while keeping the
// loop in the canonical form where each predecessor of each exit block should
// be contained within the loop. For example, this will convert the loop below
// from
//
// LB1:
//   %v1 =
//   br %LE, %LB2
// LB2:
//   %v2 =
//   br %LE, %LB1
// LE:
//   %p = phi [%v1, %LB1], [%v2, %LB2] <-- non-trivially replaceable
//
// to
//
// LB1:
//   %v1 =
//   br %LE.split, %LB2
// LB2:
//   %v2 =
//   br %LE.split2, %LB1
// LE.split:
//   %p1 = phi [%v1, %LB1]  <-- trivially replaceable
//   br %LE
// LE.split2:
//   %p2 = phi [%v2, %LB2]  <-- trivially replaceable
//   br %LE
// LE:
//   %p = phi [%p1, %LE.split], [%p2, %LE.split2]
//
const auto &BlockColors = SafetyInfo->getBlockColors();
SmallSetVector<BasicBlock *, 8> PredBBs(pred_begin(ExitBB), pred_end(ExitBB));
while (!PredBBs.empty()) {
  BasicBlock *PredBB = *PredBBs.begin();
  assert(CurLoop->contains(PredBB) &&(static_cast <bool> (CurLoop->contains(PredBB) &&
 "Expect all predecessors are in the loop") ? void (0) : __assert_fail
 ("CurLoop->contains(PredBB) && \"Expect all predecessors are in the loop\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1643, __extension__ __PRETTY_FUNCTION__))
         "Expect all predecessors are in the loop")(static_cast <bool> (CurLoop->contains(PredBB) &&
 "Expect all predecessors are in the loop") ? void (0) : __assert_fail
 ("CurLoop->contains(PredBB) && \"Expect all predecessors are in the loop\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1643, __extension__ __PRETTY_FUNCTION__));
  if (PN->getBasicBlockIndex(PredBB) >= 0) {
    BasicBlock *NewPred = SplitBlockPredecessors(
        ExitBB, PredBB, ".split.loop.exit", DT, LI, MSSAU, true);
    // Since we do not allow splitting EH-block with BlockColors in
    // canSplitPredecessors(), we can simply assign predecessor's color to
    // the new block.
    if (!BlockColors.empty())
      // Grab a reference to the ColorVector to be inserted before getting the
      // reference to the vector we are copying because inserting the new
      // element in BlockColors might cause the map to be reallocated.
      SafetyInfo->copyColors(NewPred, PredBB);
  }
  PredBBs.remove(PredBB);
}
1658}

1660/// When an instruction is found to only be used outside of the loop, this
1661/// function moves it to the exit blocks and patches up SSA form as needed.
1662/// This method is guaranteed to remove the original instruction from its
1663/// position, and may either delete it or move it to outside of the loop.
1664///
1665static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT,
               BlockFrequencyInfo *BFI, const Loop *CurLoop,
               ICFLoopSafetyInfo *SafetyInfo, MemorySSAUpdater *MSSAU,
               OptimizationRemarkEmitter *ORE) {
bool Changed = false;
LLVM_DEBUG(dbgs() << "LICM sinking instruction: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM sinking instruction: " <<
 I << "\n"; } } while (false);

// Iterate over users to be ready for actual sinking. Replace users via
// unreachable blocks with undef and make all user PHIs trivially replaceable.
SmallPtrSet<Instruction *, 8> VisitedUsers;
for (Value::user_iterator UI = I.user_begin(), UE = I.user_end(); UI != UE;) {
  auto *User = cast<Instruction>(*UI);
  Use &U = UI.getUse();
  ++UI;

  if (VisitedUsers.count(User) || CurLoop->contains(User))
    continue;

  if (!DT->isReachableFromEntry(User->getParent())) {
    U = UndefValue::get(I.getType());
    Changed = true;
    continue;
  }

  // The user must be a PHI node.
  PHINode *PN = cast<PHINode>(User);

  // Surprisingly, instructions can be used outside of loops without any
  // exits.  This can only happen in PHI nodes if the incoming block is
  // unreachable.
  BasicBlock *BB = PN->getIncomingBlock(U);
  if (!DT->isReachableFromEntry(BB)) {
    U = UndefValue::get(I.getType());
    Changed = true;
    continue;
  }

  VisitedUsers.insert(PN);
  if (isTriviallyReplaceablePHI(*PN, I))
    continue;

  if (!canSplitPredecessors(PN, SafetyInfo))
    return Changed;

  // Split predecessors of the PHI so that we can make users trivially
  // replaceable.
  splitPredecessorsOfLoopExit(PN, DT, LI, CurLoop, SafetyInfo, MSSAU);

  // Should rebuild the iterators, as they may be invalidated by
  // splitPredecessorsOfLoopExit().
  UI = I.user_begin();
  UE = I.user_end();
}

if (VisitedUsers.empty())
  return Changed;

ORE->emit([&]() {
  return OptimizationRemark(DEBUG_TYPE"licm", "InstSunk", &I)
         << "sinking " << ore::NV("Inst", &I);
});
if (isa<LoadInst>(I))
  ++NumMovedLoads;
else if (isa<CallInst>(I))
  ++NumMovedCalls;
++NumSunk;

1732#ifndef NDEBUG
SmallVector<BasicBlock *, 32> ExitBlocks;
CurLoop->getUniqueExitBlocks(ExitBlocks);
SmallPtrSet<BasicBlock *, 32> ExitBlockSet(ExitBlocks.begin(),
                                           ExitBlocks.end());
1737#endif

// Clones of this instruction. Don't create more than one per exit block!
SmallDenseMap<BasicBlock *, Instruction *, 32> SunkCopies;

// If this instruction is only used outside of the loop, then all users are
// PHI nodes in exit blocks due to LCSSA form. Just RAUW them with clones of
// the instruction.
// First check if I is worth sinking for all uses. Sink only when it is worth
// across all uses.
SmallSetVector<User*, 8> Users(I.user_begin(), I.user_end());
SmallVector<PHINode *, 8> ExitPNs;
for (auto *UI : Users) {
  auto *User = cast<Instruction>(UI);

  if (CurLoop->contains(User))
    continue;

  PHINode *PN = cast<PHINode>(User);
  assert(ExitBlockSet.count(PN->getParent()) &&(static_cast <bool> (ExitBlockSet.count(PN->getParent
()) && "The LCSSA PHI is not in an exit block!") ? void
 (0) : __assert_fail ("ExitBlockSet.count(PN->getParent()) && \"The LCSSA PHI is not in an exit block!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1757, __extension__ __PRETTY_FUNCTION__))
         "The LCSSA PHI is not in an exit block!")(static_cast <bool> (ExitBlockSet.count(PN->getParent
()) && "The LCSSA PHI is not in an exit block!") ? void
 (0) : __assert_fail ("ExitBlockSet.count(PN->getParent()) && \"The LCSSA PHI is not in an exit block!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1757, __extension__ __PRETTY_FUNCTION__));
  if (!worthSinkOrHoistInst(I, PN->getParent(), ORE, BFI)) {
    return Changed;
  }

  ExitPNs.push_back(PN);
}

for (auto *PN : ExitPNs) {

  // The PHI must be trivially replaceable.
  Instruction *New = sinkThroughTriviallyReplaceablePHI(
      PN, &I, LI, SunkCopies, SafetyInfo, CurLoop, MSSAU);
  PN->replaceAllUsesWith(New);
  eraseInstruction(*PN, *SafetyInfo, nullptr, nullptr);
  Changed = true;
}
return Changed;
1775}

1777/// When an instruction is found to only use loop invariant operands that
1778/// is safe to hoist, this instruction is called to do the dirty work.
1779///
1780static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop,
                BasicBlock *Dest, ICFLoopSafetyInfo *SafetyInfo,
                MemorySSAUpdater *MSSAU, ScalarEvolution *SE,
                OptimizationRemarkEmitter *ORE) {
LLVM_DEBUG(dbgs() << "LICM hoisting to " << Dest->getNameOrAsOperand() << ": "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM hoisting to " << Dest
->getNameOrAsOperand() << ": " << I << "\n"
; } } while (false)
                  << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM hoisting to " << Dest
->getNameOrAsOperand() << ": " << I << "\n"
; } } while (false);
ORE->emit([&]() {
  return OptimizationRemark(DEBUG_TYPE"licm", "Hoisted", &I) << "hoisting "
                                                       << ore::NV("Inst", &I);
});

// Metadata can be dependent on conditions we are hoisting above.
// Conservatively strip all metadata on the instruction unless we were
// guaranteed to execute I if we entered the loop, in which case the metadata
// is valid in the loop preheader.
if (I.hasMetadataOtherThanDebugLoc() &&
    // The check on hasMetadataOtherThanDebugLoc is to prevent us from burning
    // time in isGuaranteedToExecute if we don't actually have anything to
    // drop.  It is a compile time optimization, not required for correctness.
    !SafetyInfo->isGuaranteedToExecute(I, DT, CurLoop))
  I.dropUnknownNonDebugMetadata();

if (isa<PHINode>(I))
  // Move the new node to the end of the phi list in the destination block.
  moveInstructionBefore(I, *Dest->getFirstNonPHI(), *SafetyInfo, MSSAU, SE);
else
  // Move the new node to the destination block, before its terminator.
  moveInstructionBefore(I, *Dest->getTerminator(), *SafetyInfo, MSSAU, SE);

I.updateLocationAfterHoist();

if (isa<LoadInst>(I))
  ++NumMovedLoads;
else if (isa<CallInst>(I))
  ++NumMovedCalls;
++NumHoisted;
1816}

1818/// Only sink or hoist an instruction if it is not a trapping instruction,
1819/// or if the instruction is known not to trap when moved to the preheader.
1820/// or if it is a trapping instruction and is guaranteed to execute.
1821static bool isSafeToExecuteUnconditionally(Instruction &Inst,
                                         const DominatorTree *DT,
                                         const TargetLibraryInfo *TLI,
                                         const Loop *CurLoop,
                                         const LoopSafetyInfo *SafetyInfo,
                                         OptimizationRemarkEmitter *ORE,
                                         const Instruction *CtxI) {
if (isSafeToSpeculativelyExecute(&Inst, CtxI, DT, TLI))
  return true;

bool GuaranteedToExecute =
    SafetyInfo->isGuaranteedToExecute(Inst, DT, CurLoop);

if (!GuaranteedToExecute) {
  auto *LI = dyn_cast<LoadInst>(&Inst);
  if (LI && CurLoop->isLoopInvariant(LI->getPointerOperand()))
    ORE->emit([&]() {
      return OptimizationRemarkMissed(
                 DEBUG_TYPE"licm", "LoadWithLoopInvariantAddressCondExecuted", LI)
             << "failed to hoist load with loop-invariant address "
                "because load is conditionally executed";
    });
}

return GuaranteedToExecute;
1846}

1848namespace {
1849class LoopPromoter : public LoadAndStorePromoter {
Value *SomePtr; // Designated pointer to store to.
const SmallSetVector<Value *, 8> &PointerMustAliases;
SmallVectorImpl<BasicBlock *> &LoopExitBlocks;
SmallVectorImpl<Instruction *> &LoopInsertPts;
SmallVectorImpl<MemoryAccess *> &MSSAInsertPts;
PredIteratorCache &PredCache;
AliasSetTracker *AST;
MemorySSAUpdater *MSSAU;
LoopInfo &LI;
DebugLoc DL;
int Alignment;
bool UnorderedAtomic;
AAMDNodes AATags;
ICFLoopSafetyInfo &SafetyInfo;

// We're about to add a use of V in a loop exit block.  Insert an LCSSA phi
// (if legal) if doing so would add an out-of-loop use to an instruction
// defined in-loop.
Value *maybeInsertLCSSAPHI(Value *V, BasicBlock *BB) const {
  if (!LI.wouldBeOutOfLoopUseRequiringLCSSA(V, BB))
    return V;

  Instruction *I = cast<Instruction>(V);
  // We need to create an LCSSA PHI node for the incoming value and
  // store that.
  PHINode *PN = PHINode::Create(I->getType(), PredCache.size(BB),
                                I->getName() + ".lcssa", &BB->front());
  for (BasicBlock *Pred : PredCache.get(BB))
    PN->addIncoming(I, Pred);
  return PN;
}

1882public:
LoopPromoter(Value *SP, ArrayRef<const Instruction *> Insts, SSAUpdater &S,
             const SmallSetVector<Value *, 8> &PMA,
             SmallVectorImpl<BasicBlock *> &LEB,
             SmallVectorImpl<Instruction *> &LIP,
             SmallVectorImpl<MemoryAccess *> &MSSAIP, PredIteratorCache &PIC,
             AliasSetTracker *ast, MemorySSAUpdater *MSSAU, LoopInfo &li,
             DebugLoc dl, int alignment, bool UnorderedAtomic,
             const AAMDNodes &AATags, ICFLoopSafetyInfo &SafetyInfo)
    : LoadAndStorePromoter(Insts, S), SomePtr(SP), PointerMustAliases(PMA),
      LoopExitBlocks(LEB), LoopInsertPts(LIP), MSSAInsertPts(MSSAIP),
      PredCache(PIC), AST(ast), MSSAU(MSSAU), LI(li), DL(std::move(dl)),
      Alignment(alignment), UnorderedAtomic(UnorderedAtomic), AATags(AATags),
      SafetyInfo(SafetyInfo) {}

bool isInstInList(Instruction *I,
                  const SmallVectorImpl<Instruction *> &) const override {
  Value *Ptr;
  if (LoadInst *LI = dyn_cast<LoadInst>(I))
    Ptr = LI->getOperand(0);
  else
    Ptr = cast<StoreInst>(I)->getPointerOperand();
  return PointerMustAliases.count(Ptr);
}

void doExtraRewritesBeforeFinalDeletion() override {
  // Insert stores after in the loop exit blocks.  Each exit block gets a
  // store of the live-out values that feed them.  Since we've already told
  // the SSA updater about the defs in the loop and the preheader
  // definition, it is all set and we can start using it.
  for (unsigned i = 0, e = LoopExitBlocks.size(); i != e; ++i) {
    BasicBlock *ExitBlock = LoopExitBlocks[i];
    Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock);
    LiveInValue = maybeInsertLCSSAPHI(LiveInValue, ExitBlock);
    Value *Ptr = maybeInsertLCSSAPHI(SomePtr, ExitBlock);
    Instruction *InsertPos = LoopInsertPts[i];
    StoreInst *NewSI = new StoreInst(LiveInValue, Ptr, InsertPos);
    if (UnorderedAtomic)
      NewSI->setOrdering(AtomicOrdering::Unordered);
    NewSI->setAlignment(Align(Alignment));
    NewSI->setDebugLoc(DL);
    if (AATags)
      NewSI->setAAMetadata(AATags);

    if (MSSAU) {
      MemoryAccess *MSSAInsertPoint = MSSAInsertPts[i];
      MemoryAccess *NewMemAcc;
      if (!MSSAInsertPoint) {
        NewMemAcc = MSSAU->createMemoryAccessInBB(
            NewSI, nullptr, NewSI->getParent(), MemorySSA::Beginning);
      } else {
        NewMemAcc =
            MSSAU->createMemoryAccessAfter(NewSI, nullptr, MSSAInsertPoint);
      }
      MSSAInsertPts[i] = NewMemAcc;
      MSSAU->insertDef(cast<MemoryDef>(NewMemAcc), true);
      // FIXME: true for safety, false may still be correct.
    }
  }
}

void replaceLoadWithValue(LoadInst *LI, Value *V) const override {
  // Update alias analysis.
  if (AST)
    AST->copyValue(LI, V);
}
void instructionDeleted(Instruction *I) const override {
  SafetyInfo.removeInstruction(I);
  if (AST)
    AST->deleteValue(I);
  if (MSSAU)
    MSSAU->removeMemoryAccess(I);
}
1955};

1957bool isNotCapturedBeforeOrInLoop(const Value *V, const Loop *L,
                               DominatorTree *DT) {
// We can perform the captured-before check against any instruction in the
// loop header, as the loop header is reachable from any instruction inside
// the loop.
// TODO: ReturnCaptures=true shouldn't be necessary here.
return !PointerMayBeCapturedBefore(V, /* ReturnCaptures */ true,
                                   /* StoreCaptures */ true,
                                   L->getHeader()->getTerminator(), DT);
1966}

1968/// Return true iff we can prove that a caller of this function can not inspect
1969/// the contents of the provided object in a well defined program.
1970bool isKnownNonEscaping(Value *Object, const Loop *L,
                      const TargetLibraryInfo *TLI, DominatorTree *DT) {
if (isa<AllocaInst>(Object))
  // Since the alloca goes out of scope, we know the caller can't retain a
  // reference to it and be well defined.  Thus, we don't need to check for
  // capture.
  return true;

// For all other objects we need to know that the caller can't possibly
// have gotten a reference to the object.  There are two components of
// that:
//   1) Object can't be escaped by this function.  This is what
//      PointerMayBeCaptured checks.
//   2) Object can't have been captured at definition site.  For this, we
//      need to know the return value is noalias.  At the moment, we use a
//      weaker condition and handle only AllocLikeFunctions (which are
//      known to be noalias).  TODO
return isAllocLikeFn(Object, TLI) &&
       isNotCapturedBeforeOrInLoop(Object, L, DT);
1989}

1991} // namespace

1993/// Try to promote memory values to scalars by sinking stores out of the
1994/// loop and moving loads to before the loop.  We do this by looping over
1995/// the stores in the loop, looking for stores to Must pointers which are
1996/// loop invariant.
1997///
1998bool llvm::promoteLoopAccessesToScalars(
  const SmallSetVector<Value *, 8> &PointerMustAliases,
  SmallVectorImpl<BasicBlock *> &ExitBlocks,
  SmallVectorImpl<Instruction *> &InsertPts,
  SmallVectorImpl<MemoryAccess *> &MSSAInsertPts, PredIteratorCache &PIC,
  LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI,
  Loop *CurLoop, AliasSetTracker *CurAST, MemorySSAUpdater *MSSAU,
  ICFLoopSafetyInfo *SafetyInfo, OptimizationRemarkEmitter *ORE) {
// Verify inputs.
assert(LI != nullptr && DT != nullptr && CurLoop != nullptr &&(static_cast <bool> (LI != nullptr && DT != nullptr
 && CurLoop != nullptr && SafetyInfo != nullptr
 && "Unexpected Input to promoteLoopAccessesToScalars"
) ? void (0) : __assert_fail ("LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected Input to promoteLoopAccessesToScalars\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 2009, __extension__ __PRETTY_FUNCTION__))
       SafetyInfo != nullptr &&(static_cast <bool> (LI != nullptr && DT != nullptr
 && CurLoop != nullptr && SafetyInfo != nullptr
 && "Unexpected Input to promoteLoopAccessesToScalars"
) ? void (0) : __assert_fail ("LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected Input to promoteLoopAccessesToScalars\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 2009, __extension__ __PRETTY_FUNCTION__))
       "Unexpected Input to promoteLoopAccessesToScalars")(static_cast <bool> (LI != nullptr && DT != nullptr
 && CurLoop != nullptr && SafetyInfo != nullptr
 && "Unexpected Input to promoteLoopAccessesToScalars"
) ? void (0) : __assert_fail ("LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected Input to promoteLoopAccessesToScalars\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 2009, __extension__ __PRETTY_FUNCTION__));

Value *SomePtr = *PointerMustAliases.begin();
BasicBlock *Preheader = CurLoop->getLoopPreheader();

// It is not safe to promote a load/store from the loop if the load/store is
// conditional.  For example, turning:
//
//    for () { if (c) *P += 1; }
//
// into:
//
//    tmp = *P;  for () { if (c) tmp +=1; } *P = tmp;
//
// is not safe, because *P may only be valid to access if 'c' is true.
//
// The safety property divides into two parts:
// p1) The memory may not be dereferenceable on entry to the loop.  In this
//    case, we can't insert the required load in the preheader.
// p2) The memory model does not allow us to insert a store along any dynamic
//    path which did not originally have one.
//
// If at least one store is guaranteed to execute, both properties are
// satisfied, and promotion is legal.
//
// This, however, is not a necessary condition. Even if no store/load is
// guaranteed to execute, we can still establish these properties.
// We can establish (p1) by proving that hoisting the load into the preheader
// is safe (i.e. proving dereferenceability on all paths through the loop). We
// can use any access within the alias set to prove dereferenceability,
// since they're all must alias.
//
// There are two ways establish (p2):
// a) Prove the location is thread-local. In this case the memory model
// requirement does not apply, and stores are safe to insert.
// b) Prove a store dominates every exit block. In this case, if an exit
// blocks is reached, the original dynamic path would have taken us through
// the store, so inserting a store into the exit block is safe. Note that this
// is different from the store being guaranteed to execute. For instance,
// if an exception is thrown on the first iteration of the loop, the original
// store is never executed, but the exit blocks are not executed either.

bool DereferenceableInPH = false;
bool SafeToInsertStore = false;

SmallVector<Instruction *, 64> LoopUses;

// We start with an alignment of one and try to find instructions that allow
// us to prove better alignment.
Align Alignment;
// Keep track of which types of access we see
bool SawUnorderedAtomic = false;
bool SawNotAtomic = false;
AAMDNodes AATags;

const DataLayout &MDL = Preheader->getModule()->getDataLayout();

bool IsKnownThreadLocalObject = false;
if (SafetyInfo->anyBlockMayThrow()) {
  // If a loop can throw, we have to insert a store along each unwind edge.
  // That said, we can't actually make the unwind edge explicit. Therefore,
  // we have to prove that the store is dead along the unwind edge.  We do
  // this by proving that the caller can't have a reference to the object
  // after return and thus can't possibly load from the object.
  Value *Object = getUnderlyingObject(SomePtr);
  if (!isKnownNonEscaping(Object, CurLoop, TLI, DT))
    return false;
  // Subtlety: Alloca's aren't visible to callers, but *are* potentially
  // visible to other threads if captured and used during their lifetimes.
  IsKnownThreadLocalObject = !isa<AllocaInst>(Object);
}

// Check that all of the pointers in the alias set have the same type.  We
// cannot (yet) promote a memory location that is loaded and stored in
// different sizes.  While we are at it, collect alignment and AA info.
for (Value *ASIV : PointerMustAliases) {
  // Check that all of the pointers in the alias set have the same type.  We
  // cannot (yet) promote a memory location that is loaded and stored in
  // different sizes.
  if (SomePtr->getType() != ASIV->getType())
    return false;

  for (User *U : ASIV->users()) {
    // Ignore instructions that are outside the loop.
    Instruction *UI = dyn_cast<Instruction>(U);
    if (!UI || !CurLoop->contains(UI))
      continue;

    // If there is an non-load/store instruction in the loop, we can't promote
    // it.
    if (LoadInst *Load = dyn_cast<LoadInst>(UI)) {
      if (!Load->isUnordered())
        return false;

      SawUnorderedAtomic |= Load->isAtomic();
      SawNotAtomic |= !Load->isAtomic();

      Align InstAlignment = Load->getAlign();

      // Note that proving a load safe to speculate requires proving
      // sufficient alignment at the target location.  Proving it guaranteed
      // to execute does as well.  Thus we can increase our guaranteed
      // alignment as well.
      if (!DereferenceableInPH || (InstAlignment > Alignment))
        if (isSafeToExecuteUnconditionally(*Load, DT, TLI, CurLoop,
                                           SafetyInfo, ORE,
                                           Preheader->getTerminator())) {
          DereferenceableInPH = true;
          Alignment = std::max(Alignment, InstAlignment);
        }
    } else if (const StoreInst *Store = dyn_cast<StoreInst>(UI)) {
      // Stores *of* the pointer are not interesting, only stores *to* the
      // pointer.
      if (UI->getOperand(1) != ASIV)
        continue;
      if (!Store->isUnordered())
        return false;

      SawUnorderedAtomic |= Store->isAtomic();
      SawNotAtomic |= !Store->isAtomic();

      // If the store is guaranteed to execute, both properties are satisfied.
      // We may want to check if a store is guaranteed to execute even if we
      // already know that promotion is safe, since it may have higher
      // alignment than any other guaranteed stores, in which case we can
      // raise the alignment on the promoted store.
      Align InstAlignment = Store->getAlign();

      if (!DereferenceableInPH || !SafeToInsertStore ||
          (InstAlignment > Alignment)) {
        if (SafetyInfo->isGuaranteedToExecute(*UI, DT, CurLoop)) {
          DereferenceableInPH = true;
          SafeToInsertStore = true;
          Alignment = std::max(Alignment, InstAlignment);
        }
      }

      // If a store dominates all exit blocks, it is safe to sink.
      // As explained above, if an exit block was executed, a dominating
      // store must have been executed at least once, so we are not
      // introducing stores on paths that did not have them.
      // Note that this only looks at explicit exit blocks. If we ever
      // start sinking stores into unwind edges (see above), this will break.
      if (!SafeToInsertStore)
        SafeToInsertStore = llvm::all_of(ExitBlocks, [&](BasicBlock *Exit) {
          return DT->dominates(Store->getParent(), Exit);
        });

      // If the store is not guaranteed to execute, we may still get
      // deref info through it.
      if (!DereferenceableInPH) {
        DereferenceableInPH = isDereferenceableAndAlignedPointer(
            Store->getPointerOperand(), Store->getValueOperand()->getType(),
            Store->getAlign(), MDL, Preheader->getTerminator(), DT, TLI);
      }
    } else
      return false; // Not a load or store.

    // Merge the AA tags.
    if (LoopUses.empty()) {
      // On the first load/store, just take its AA tags.
      UI->getAAMetadata(AATags);
    } else if (AATags) {
      UI->getAAMetadata(AATags, /* Merge = */ true);
    }

    LoopUses.push_back(UI);
  }
}

// If we found both an unordered atomic instruction and a non-atomic memory
// access, bail.  We can't blindly promote non-atomic to atomic since we
// might not be able to lower the result.  We can't downgrade since that
// would violate memory model.  Also, align 0 is an error for atomics.
if (SawUnorderedAtomic && SawNotAtomic)
  return false;

// If we're inserting an atomic load in the preheader, we must be able to
// lower it.  We're only guaranteed to be able to lower naturally aligned
// atomics.
auto *SomePtrElemType = SomePtr->getType()->getPointerElementType();
if (SawUnorderedAtomic &&
    Alignment < MDL.getTypeStoreSize(SomePtrElemType))
  return false;

// If we couldn't prove we can hoist the load, bail.
if (!DereferenceableInPH)
  return false;

// We know we can hoist the load, but don't have a guaranteed store.
// Check whether the location is thread-local. If it is, then we can insert
// stores along paths which originally didn't have them without violating the
// memory model.
if (!SafeToInsertStore) {
  if (IsKnownThreadLocalObject)
    SafeToInsertStore = true;
  else {
    Value *Object = getUnderlyingObject(SomePtr);
    SafeToInsertStore =
        (isAllocLikeFn(Object, TLI) || isa<AllocaInst>(Object)) &&
        isNotCapturedBeforeOrInLoop(Object, CurLoop, DT);
  }
}

// If we've still failed to prove we can sink the store, give up.
if (!SafeToInsertStore)
  return false;

// Otherwise, this is safe to promote, lets do it!
LLVM_DEBUG(dbgs() << "LICM: Promoting value stored to in loop: " << *SomePtrdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM: Promoting value stored to in loop: "
 << *SomePtr << '\n'; } } while (false)
                  << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM: Promoting value stored to in loop: "
 << *SomePtr << '\n'; } } while (false);
ORE->emit([&]() {
  return OptimizationRemark(DEBUG_TYPE"licm", "PromoteLoopAccessesToScalar",
                            LoopUses[0])
         << "Moving accesses to memory location out of the loop";
});
++NumPromoted;

// Look at all the loop uses, and try to merge their locations.
std::vector<const DILocation *> LoopUsesLocs;
for (auto U : LoopUses)
  LoopUsesLocs.push_back(U->getDebugLoc().get());
auto DL = DebugLoc(DILocation::getMergedLocations(LoopUsesLocs));

// We use the SSAUpdater interface to insert phi nodes as required.
SmallVector<PHINode *, 16> NewPHIs;
SSAUpdater SSA(&NewPHIs);
LoopPromoter Promoter(SomePtr, LoopUses, SSA, PointerMustAliases, ExitBlocks,
                      InsertPts, MSSAInsertPts, PIC, CurAST, MSSAU, *LI, DL,
                      Alignment.value(), SawUnorderedAtomic, AATags,
                      *SafetyInfo);

// Set up the preheader to have a definition of the value.  It is the live-out
// value from the preheader that uses in the loop will use.
LoadInst *PreheaderLoad = new LoadInst(
    SomePtr->getType()->getPointerElementType(), SomePtr,
    SomePtr->getName() + ".promoted", Preheader->getTerminator());
if (SawUnorderedAtomic)
  PreheaderLoad->setOrdering(AtomicOrdering::Unordered);
PreheaderLoad->setAlignment(Alignment);
PreheaderLoad->setDebugLoc(DebugLoc());
if (AATags)
  PreheaderLoad->setAAMetadata(AATags);
SSA.AddAvailableValue(Preheader, PreheaderLoad);

if (MSSAU) {
  MemoryAccess *PreheaderLoadMemoryAccess = MSSAU->createMemoryAccessInBB(
      PreheaderLoad, nullptr, PreheaderLoad->getParent(), MemorySSA::End);
  MemoryUse *NewMemUse = cast<MemoryUse>(PreheaderLoadMemoryAccess);
  MSSAU->insertUse(NewMemUse, /*RenameUses=*/true);
}

if (MSSAU && VerifyMemorySSA)
  MSSAU->getMemorySSA()->verifyMemorySSA();
// Rewrite all the loads in the loop and remember all the definitions from
// stores in the loop.
Promoter.run(LoopUses);

if (MSSAU && VerifyMemorySSA)
  MSSAU->getMemorySSA()->verifyMemorySSA();
// If the SSAUpdater didn't use the load in the preheader, just zap it now.
if (PreheaderLoad->use_empty())
  eraseInstruction(*PreheaderLoad, *SafetyInfo, CurAST, MSSAU);

return true;
2274}

2276static void foreachMemoryAccess(MemorySSA *MSSA, Loop *L,
                              function_ref<void(Instruction *)> Fn) {
for (const BasicBlock *BB : L->blocks())
  if (const auto *Accesses = MSSA->getBlockAccesses(BB))
    for (const auto &Access : *Accesses)
      if (const auto *MUD = dyn_cast<MemoryUseOrDef>(&Access))
        Fn(MUD->getMemoryInst());
2283}

2285static SmallVector<SmallSetVector<Value *, 8>, 0>
2286collectPromotionCandidates(MemorySSA *MSSA, AliasAnalysis *AA, Loop *L,
                         SmallVectorImpl<Instruction *> &MaybePromotable) {
AliasSetTracker AST(*AA);

auto IsPotentiallyPromotable = [L](const Instruction *I) {
  if (const auto *SI = dyn_cast<StoreInst>(I))
    return L->isLoopInvariant(SI->getPointerOperand());
  if (const auto *LI = dyn_cast<LoadInst>(I))
    return L->isLoopInvariant(LI->getPointerOperand());
  return false;
};

// Populate AST with potentially promotable accesses and remove them from
// MaybePromotable, so they will not be checked again on the next iteration.
SmallPtrSet<Value *, 16> AttemptingPromotion;
llvm::erase_if(MaybePromotable, [&](Instruction *I) {
  if (IsPotentiallyPromotable(I)) {
    AttemptingPromotion.insert(I);
    AST.add(I);
    return true;
  }
  return false;
});

// We're only interested in must-alias sets that contain a mod.
SmallVector<const AliasSet *, 8> Sets;
for (AliasSet &AS : AST)
  if (!AS.isForwardingAliasSet() && AS.isMod() && AS.isMustAlias())
    Sets.push_back(&AS);

if (Sets.empty())
  return {}; // Nothing to promote...

// Discard any sets for which there is an aliasing non-promotable access.
foreachMemoryAccess(MSSA, L, [&](Instruction *I) {
  if (AttemptingPromotion.contains(I))
    return;

  llvm::erase_if(Sets, [&](const AliasSet *AS) {
    return AS->aliasesUnknownInst(I, *AA);
  });
});

SmallVector<SmallSetVector<Value *, 8>, 0> Result;
for (const AliasSet *Set : Sets) {
  SmallSetVector<Value *, 8> PointerMustAliases;
  for (const auto &ASI : *Set)
    PointerMustAliases.insert(ASI.getValue());
  Result.push_back(std::move(PointerMustAliases));
}

return Result;
2338}

2340/// Returns an owning pointer to an alias set which incorporates aliasing info
2341/// from L and all subloops of L.
2342std::unique_ptr<AliasSetTracker>
2343LoopInvariantCodeMotion::collectAliasInfoForLoop(Loop *L, LoopInfo *LI,
                                               AAResults *AA) {
auto CurAST = std::make_unique<AliasSetTracker>(*AA);

// Add everything from all the sub loops.
for (Loop *InnerL : L->getSubLoops())
  for (BasicBlock *BB : InnerL->blocks())
    CurAST->add(*BB);

// And merge in this loop (without anything from inner loops).
for (BasicBlock *BB : L->blocks())
  if (LI->getLoopFor(BB) == L)
    CurAST->add(*BB);

return CurAST;
2358}

2360static bool pointerInvalidatedByLoop(MemoryLocation MemLoc,
                                   AliasSetTracker *CurAST, Loop *CurLoop,
                                   AAResults *AA) {
// First check to see if any of the basic blocks in CurLoop invalidate *V.
bool isInvalidatedAccordingToAST = CurAST->getAliasSetFor(MemLoc).isMod();

if (!isInvalidatedAccordingToAST || !LICMN2Theshold)
  return isInvalidatedAccordingToAST;

// Check with a diagnostic analysis if we can refine the information above.
// This is to identify the limitations of using the AST.
// The alias set mechanism used by LICM has a major weakness in that it
// combines all things which may alias into a single set *before* asking
// modref questions. As a result, a single readonly call within a loop will
// collapse all loads and stores into a single alias set and report
// invalidation if the loop contains any store. For example, readonly calls
// with deopt states have this form and create a general alias set with all
// loads and stores.  In order to get any LICM in loops containing possible
// deopt states we need a more precise invalidation of checking the mod ref
// info of each instruction within the loop and LI. This has a complexity of
// O(N^2), so currently, it is used only as a diagnostic tool since the
// default value of LICMN2Threshold is zero.

// Don't look at nested loops.
if (CurLoop->begin() != CurLoop->end())
  return true;

int N = 0;
for (BasicBlock *BB : CurLoop->getBlocks())
  for (Instruction &I : *BB) {
    if (N >= LICMN2Theshold) {
      LLVM_DEBUG(dbgs() << "Alasing N2 threshold exhausted for "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "Alasing N2 threshold exhausted for "
 << *(MemLoc.Ptr) << "\n"; } } while (false)
                        << *(MemLoc.Ptr) << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "Alasing N2 threshold exhausted for "
 << *(MemLoc.Ptr) << "\n"; } } while (false);
      return true;
    }
    N++;
    auto Res = AA->getModRefInfo(&I, MemLoc);
    if (isModSet(Res)) {
      LLVM_DEBUG(dbgs() << "Aliasing failed on " << I << " for "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "Aliasing failed on " << I <<
 " for " << *(MemLoc.Ptr) << "\n"; } } while (false
)
                        << *(MemLoc.Ptr) << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "Aliasing failed on " << I <<
 " for " << *(MemLoc.Ptr) << "\n"; } } while (false
);
      return true;
    }
  }
LLVM_DEBUG(dbgs() << "Aliasing okay for " << *(MemLoc.Ptr) << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "Aliasing okay for " << *(MemLoc
.Ptr) << "\n"; } } while (false);
return false;
2405}

2407bool pointerInvalidatedByLoopWithMSSA(MemorySSA *MSSA, MemoryUse *MU,
                                    Loop *CurLoop, Instruction &I,
                                    SinkAndHoistLICMFlags &Flags) {
// For hoisting, use the walker to determine safety
if (!Flags.getIsSink()) {
  MemoryAccess *Source;
  // See declaration of SetLicmMssaOptCap for usage details.
  if (Flags.tooManyClobberingCalls())
    Source = MU->getDefiningAccess();
  else {
    Source = MSSA->getSkipSelfWalker()->getClobberingMemoryAccess(MU);
    Flags.incrementClobberingCalls();
  }
  return !MSSA->isLiveOnEntryDef(Source) &&
         CurLoop->contains(Source->getBlock());
}

// For sinking, we'd need to check all Defs below this use. The getClobbering
// call will look on the backedge of the loop, but will check aliasing with
// the instructions on the previous iteration.
// For example:
// for (i ... )
//   load a[i] ( Use (LoE)
//   store a[i] ( 1 = Def (2), with 2 = Phi for the loop.
//   i++;
// The load sees no clobbering inside the loop, as the backedge alias check
// does phi translation, and will check aliasing against store a[i-1].
// However sinking the load outside the loop, below the store is incorrect.

// For now, only sink if there are no Defs in the loop, and the existing ones
// precede the use and are in the same block.
// FIXME: Increase precision: Safe to sink if Use post dominates the Def;
// needs PostDominatorTreeAnalysis.
// FIXME: More precise: no Defs that alias this Use.
if (Flags.tooManyMemoryAccesses())
  return true;
for (auto *BB : CurLoop->getBlocks())
  if (pointerInvalidatedByBlockWithMSSA(*BB, *MSSA, *MU))
    return true;
// When sinking, the source block may not be part of the loop so check it.
if (!CurLoop->contains(&I))
  return pointerInvalidatedByBlockWithMSSA(*I.getParent(), *MSSA, *MU);

return false;
2451}

2453bool pointerInvalidatedByBlockWithMSSA(BasicBlock &BB, MemorySSA &MSSA,
                                     MemoryUse &MU) {
if (const auto *Accesses = MSSA.getBlockDefs(&BB))
  for (const auto &MA : *Accesses)
    if (const auto *MD = dyn_cast<MemoryDef>(&MA))
      if (MU.getBlock() != MD->getBlock() || !MSSA.locallyDominates(MD, &MU))
        return true;
return false;
2461}

2463/// Little predicate that returns true if the specified basic block is in
2464/// a subloop of the current one, not the current one itself.
2465///
2466static bool inSubLoop(BasicBlock *BB, Loop *CurLoop, LoopInfo *LI) {
assert(CurLoop->contains(BB) && "Only valid if BB is IN the loop")(static_cast <bool> (CurLoop->contains(BB) &&
 "Only valid if BB is IN the loop") ? void (0) : __assert_fail
 ("CurLoop->contains(BB) && \"Only valid if BB is IN the loop\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/Transforms/Scalar/LICM.cpp"
, 2467, __extension__ __PRETTY_FUNCTION__));
return LI->getLoopFor(BB) != CurLoop;
2469}

←

/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h

→

1//===- llvm/InstrTypes.h - Important Instruction subclasses -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines various meta classes of instructions that exist in the VM
10// representation.  Specific concrete subclasses of these may be found in the
11// i*.h files...
12//
13//===----------------------------------------------------------------------===//

15#ifndef LLVM_IR_INSTRTYPES_H
16#define LLVM_IR_INSTRTYPES_H

18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/None.h"
20#include "llvm/ADT/Optional.h"
21#include "llvm/ADT/STLExtras.h"
22#include "llvm/ADT/StringMap.h"
23#include "llvm/ADT/StringRef.h"
24#include "llvm/ADT/Twine.h"
25#include "llvm/ADT/iterator_range.h"
26#include "llvm/IR/Attributes.h"
27#include "llvm/IR/CallingConv.h"
28#include "llvm/IR/Constants.h"
29#include "llvm/IR/DerivedTypes.h"
30#include "llvm/IR/Function.h"
31#include "llvm/IR/Instruction.h"
32#include "llvm/IR/LLVMContext.h"
33#include "llvm/IR/OperandTraits.h"
34#include "llvm/IR/Type.h"
35#include "llvm/IR/User.h"
36#include "llvm/IR/Value.h"
37#include "llvm/Support/Casting.h"
38#include "llvm/Support/ErrorHandling.h"
39#include <algorithm>
40#include <cassert>
41#include <cstddef>
42#include <cstdint>
43#include <iterator>
44#include <string>
45#include <vector>

47namespace llvm {

49namespace Intrinsic {
50typedef unsigned ID;
51}

53//===----------------------------------------------------------------------===//
54//                          UnaryInstruction Class
55//===----------------------------------------------------------------------===//

57class UnaryInstruction : public Instruction {
58protected:
UnaryInstruction(Type *Ty, unsigned iType, Value *V,
                 Instruction *IB = nullptr)
  : Instruction(Ty, iType, &Op<0>(), 1, IB) {
  Op<0>() = V;
}
UnaryInstruction(Type *Ty, unsigned iType, Value *V, BasicBlock *IAE)
  : Instruction(Ty, iType, &Op<0>(), 1, IAE) {
  Op<0>() = V;
}

69public:
// allocate space for exactly one operand
void *operator new(size_t s) {
  return User::operator new(s, 1);
}

/// Transparently provide more efficient getOperand methods.
DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
 setOperand(unsigned, Value*); inline op_iterator op_begin();
 inline const_op_iterator op_begin() const; inline op_iterator
 op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
 getNumOperands() const;

// Methods for support type inquiry through isa, cast, and dyn_cast:
static bool classof(const Instruction *I) {
  return I->isUnaryOp() ||
         I->getOpcode() == Instruction::Alloca ||
         I->getOpcode() == Instruction::Load ||
         I->getOpcode() == Instruction::VAArg ||
         I->getOpcode() == Instruction::ExtractValue ||
         (I->getOpcode() >= CastOpsBegin && I->getOpcode() < CastOpsEnd);
}
static bool classof(const Value *V) {
  return isa<Instruction>(V) && classof(cast<Instruction>(V));
}
90};

92template <>
93struct OperandTraits<UnaryInstruction> :
public FixedNumOperandTraits<UnaryInstruction, 1> {
95};

97DEFINE_TRANSPARENT_OPERAND_ACCESSORS(UnaryInstruction, Value)UnaryInstruction::op_iterator UnaryInstruction::op_begin() { return
 OperandTraits<UnaryInstruction>::op_begin(this); } UnaryInstruction
::const_op_iterator UnaryInstruction::op_begin() const { return
 OperandTraits<UnaryInstruction>::op_begin(const_cast<
UnaryInstruction*>(this)); } UnaryInstruction::op_iterator
 UnaryInstruction::op_end() { return OperandTraits<UnaryInstruction
>::op_end(this); } UnaryInstruction::const_op_iterator UnaryInstruction
::op_end() const { return OperandTraits<UnaryInstruction>
::op_end(const_cast<UnaryInstruction*>(this)); } Value *
UnaryInstruction::getOperand(unsigned i_nocapture) const { (static_cast
 <bool> (i_nocapture < OperandTraits<UnaryInstruction
>::operands(this) && "getOperand() out of range!")
 ? void (0) : __assert_fail ("i_nocapture < OperandTraits<UnaryInstruction>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 97, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<UnaryInstruction>::op_begin
(const_cast<UnaryInstruction*>(this))[i_nocapture].get(
)); } void UnaryInstruction::setOperand(unsigned i_nocapture,
 Value *Val_nocapture) { (static_cast <bool> (i_nocapture
 < OperandTraits<UnaryInstruction>::operands(this) &&
 "setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<UnaryInstruction>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 97, __extension__ __PRETTY_FUNCTION__)); OperandTraits<UnaryInstruction
>::op_begin(this)[i_nocapture] = Val_nocapture; } unsigned
 UnaryInstruction::getNumOperands() const { return OperandTraits
<UnaryInstruction>::operands(this); } template <int Idx_nocapture
> Use &UnaryInstruction::Op() { return this->OpFrom
<Idx_nocapture>(this); } template <int Idx_nocapture
> const Use &UnaryInstruction::Op() const { return this
->OpFrom<Idx_nocapture>(this); }

99//===----------------------------------------------------------------------===//
100//                                UnaryOperator Class
101//===----------------------------------------------------------------------===//

103class UnaryOperator : public UnaryInstruction {
void AssertOK();

106protected:
UnaryOperator(UnaryOps iType, Value *S, Type *Ty,
              const Twine &Name, Instruction *InsertBefore);
UnaryOperator(UnaryOps iType, Value *S, Type *Ty,
              const Twine &Name, BasicBlock *InsertAtEnd);

// Note: Instruction needs to be a friend here to call cloneImpl.
friend class Instruction;

UnaryOperator *cloneImpl() const;

117public:

/// Construct a unary instruction, given the opcode and an operand.
/// Optionally (if InstBefore is specified) insert the instruction
/// into a BasicBlock right before the specified instruction.  The specified
/// Instruction is allowed to be a dereferenced end iterator.
///
static UnaryOperator *Create(UnaryOps Op, Value *S,
                             const Twine &Name = Twine(),
                             Instruction *InsertBefore = nullptr);

/// Construct a unary instruction, given the opcode and an operand.
/// Also automatically insert this instruction to the end of the
/// BasicBlock specified.
///
static UnaryOperator *Create(UnaryOps Op, Value *S,
                             const Twine &Name,
                             BasicBlock *InsertAtEnd);

/// These methods just forward to Create, and are useful when you
/// statically know what type of instruction you're going to create.  These
/// helpers just save some typing.
139#define HANDLE_UNARY_INST(N, OPC, CLASS) \
static UnaryOperator *Create##OPC(Value *V, const Twine &Name = "") {\
  return Create(Instruction::OPC, V, Name);\
}
143#include "llvm/IR/Instruction.def"
144#define HANDLE_UNARY_INST(N, OPC, CLASS) \
static UnaryOperator *Create##OPC(Value *V, const Twine &Name, \
                                  BasicBlock *BB) {\
  return Create(Instruction::OPC, V, Name, BB);\
}
149#include "llvm/IR/Instruction.def"
150#define HANDLE_UNARY_INST(N, OPC, CLASS) \
static UnaryOperator *Create##OPC(Value *V, const Twine &Name, \
                                  Instruction *I) {\
  return Create(Instruction::OPC, V, Name, I);\
}
155#include "llvm/IR/Instruction.def"

static UnaryOperator *
CreateWithCopiedFlags(UnaryOps Opc, Value *V, Instruction *CopyO,
                      const Twine &Name = "",
                      Instruction *InsertBefore = nullptr) {
  UnaryOperator *UO = Create(Opc, V, Name, InsertBefore);
  UO->copyIRFlags(CopyO);
  return UO;
}

static UnaryOperator *CreateFNegFMF(Value *Op, Instruction *FMFSource,
                                    const Twine &Name = "",
                                    Instruction *InsertBefore = nullptr) {
  return CreateWithCopiedFlags(Instruction::FNeg, Op, FMFSource, Name,
                               InsertBefore);
}

UnaryOps getOpcode() const {
  return static_cast<UnaryOps>(Instruction::getOpcode());
}

// Methods for support type inquiry through isa, cast, and dyn_cast:
static bool classof(const Instruction *I) {
  return I->isUnaryOp();
}
static bool classof(const Value *V) {
  return isa<Instruction>(V) && classof(cast<Instruction>(V));
}
184};

186//===----------------------------------------------------------------------===//
187//                           BinaryOperator Class
188//===----------------------------------------------------------------------===//

190class BinaryOperator : public Instruction {
void AssertOK();

193protected:
BinaryOperator(BinaryOps iType, Value *S1, Value *S2, Type *Ty,
               const Twine &Name, Instruction *InsertBefore);
BinaryOperator(BinaryOps iType, Value *S1, Value *S2, Type *Ty,
               const Twine &Name, BasicBlock *InsertAtEnd);

// Note: Instruction needs to be a friend here to call cloneImpl.
friend class Instruction;

BinaryOperator *cloneImpl() const;

204public:
// allocate space for exactly two operands
void *operator new(size_t s) {
  return User::operator new(s, 2);
}

/// Transparently provide more efficient getOperand methods.
DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
 setOperand(unsigned, Value*); inline op_iterator op_begin();
 inline const_op_iterator op_begin() const; inline op_iterator
 op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
 getNumOperands() const;

/// Construct a binary instruction, given the opcode and the two
/// operands.  Optionally (if InstBefore is specified) insert the instruction
/// into a BasicBlock right before the specified instruction.  The specified
/// Instruction is allowed to be a dereferenced end iterator.
///
static BinaryOperator *Create(BinaryOps Op, Value *S1, Value *S2,
                              const Twine &Name = Twine(),
                              Instruction *InsertBefore = nullptr);

/// Construct a binary instruction, given the opcode and the two
/// operands.  Also automatically insert this instruction to the end of the
/// BasicBlock specified.
///
static BinaryOperator *Create(BinaryOps Op, Value *S1, Value *S2,
                              const Twine &Name, BasicBlock *InsertAtEnd);

/// These methods just forward to Create, and are useful when you
/// statically know what type of instruction you're going to create.  These
/// helpers just save some typing.
232#define HANDLE_BINARY_INST(N, OPC, CLASS) \
static BinaryOperator *Create##OPC(Value *V1, Value *V2, \
                                   const Twine &Name = "") {\
  return Create(Instruction::OPC, V1, V2, Name);\
}
237#include "llvm/IR/Instruction.def"
238#define HANDLE_BINARY_INST(N, OPC, CLASS) \
static BinaryOperator *Create##OPC(Value *V1, Value *V2, \
                                   const Twine &Name, BasicBlock *BB) {\
  return Create(Instruction::OPC, V1, V2, Name, BB);\
}
243#include "llvm/IR/Instruction.def"
244#define HANDLE_BINARY_INST(N, OPC, CLASS) \
static BinaryOperator *Create##OPC(Value *V1, Value *V2, \
                                   const Twine &Name, Instruction *I) {\
  return Create(Instruction::OPC, V1, V2, Name, I);\
}
249#include "llvm/IR/Instruction.def"

static BinaryOperator *CreateWithCopiedFlags(BinaryOps Opc,
                                             Value *V1, Value *V2,
                                             Instruction *CopyO,
                                             const Twine &Name = "") {
  BinaryOperator *BO = Create(Opc, V1, V2, Name);
  BO->copyIRFlags(CopyO);
  return BO;
}

static BinaryOperator *CreateFAddFMF(Value *V1, Value *V2,
                                     Instruction *FMFSource,
                                     const Twine &Name = "") {
  return CreateWithCopiedFlags(Instruction::FAdd, V1, V2, FMFSource, Name);
}
static BinaryOperator *CreateFSubFMF(Value *V1, Value *V2,
                                     Instruction *FMFSource,
                                     const Twine &Name = "") {
  return CreateWithCopiedFlags(Instruction::FSub, V1, V2, FMFSource, Name);
}
static BinaryOperator *CreateFMulFMF(Value *V1, Value *V2,
                                     Instruction *FMFSource,
                                     const Twine &Name = "") {
  return CreateWithCopiedFlags(Instruction::FMul, V1, V2, FMFSource, Name);
}
static BinaryOperator *CreateFDivFMF(Value *V1, Value *V2,
                                     Instruction *FMFSource,
                                     const Twine &Name = "") {
  return CreateWithCopiedFlags(Instruction::FDiv, V1, V2, FMFSource, Name);
}
static BinaryOperator *CreateFRemFMF(Value *V1, Value *V2,
                                     Instruction *FMFSource,
                                     const Twine &Name = "") {
  return CreateWithCopiedFlags(Instruction::FRem, V1, V2, FMFSource, Name);
}

static BinaryOperator *CreateNSW(BinaryOps Opc, Value *V1, Value *V2,
                                 const Twine &Name = "") {
  BinaryOperator *BO = Create(Opc, V1, V2, Name);
  BO->setHasNoSignedWrap(true);
  return BO;
}
static BinaryOperator *CreateNSW(BinaryOps Opc, Value *V1, Value *V2,
                                 const Twine &Name, BasicBlock *BB) {
  BinaryOperator *BO = Create(Opc, V1, V2, Name, BB);
  BO->setHasNoSignedWrap(true);
  return BO;
}
static BinaryOperator *CreateNSW(BinaryOps Opc, Value *V1, Value *V2,
                                 const Twine &Name, Instruction *I) {
  BinaryOperator *BO = Create(Opc, V1, V2, Name, I);
  BO->setHasNoSignedWrap(true);
  return BO;
}

static BinaryOperator *CreateNUW(BinaryOps Opc, Value *V1, Value *V2,
                                 const Twine &Name = "") {
  BinaryOperator *BO = Create(Opc, V1, V2, Name);
  BO->setHasNoUnsignedWrap(true);
  return BO;
}
static BinaryOperator *CreateNUW(BinaryOps Opc, Value *V1, Value *V2,
                                 const Twine &Name, BasicBlock *BB) {
  BinaryOperator *BO = Create(Opc, V1, V2, Name, BB);
  BO->setHasNoUnsignedWrap(true);
  return BO;
}
static BinaryOperator *CreateNUW(BinaryOps Opc, Value *V1, Value *V2,
                                 const Twine &Name, Instruction *I) {
  BinaryOperator *BO = Create(Opc, V1, V2, Name, I);
  BO->setHasNoUnsignedWrap(true);
  return BO;
}

static BinaryOperator *CreateExact(BinaryOps Opc, Value *V1, Value *V2,
                                   const Twine &Name = "") {
  BinaryOperator *BO = Create(Opc, V1, V2, Name);
  BO->setIsExact(true);
  return BO;
}
static BinaryOperator *CreateExact(BinaryOps Opc, Value *V1, Value *V2,
                                   const Twine &Name, BasicBlock *BB) {
  BinaryOperator *BO = Create(Opc, V1, V2, Name, BB);
  BO->setIsExact(true);
  return BO;
}
static BinaryOperator *CreateExact(BinaryOps Opc, Value *V1, Value *V2,
                                   const Twine &Name, Instruction *I) {
  BinaryOperator *BO = Create(Opc, V1, V2, Name, I);
  BO->setIsExact(true);
  return BO;
}

343#define DEFINE_HELPERS(OPC, NUWNSWEXACT)                                       \
static BinaryOperator *Create##NUWNSWEXACT##OPC(Value *V1, Value *V2,        \
                                                const Twine &Name = "") {    \
  return Create##NUWNSWEXACT(Instruction::OPC, V1, V2, Name);                \
}                                                                            \
static BinaryOperator *Create##NUWNSWEXACT##OPC(                             \
    Value *V1, Value *V2, const Twine &Name, BasicBlock *BB) {               \
  return Create##NUWNSWEXACT(Instruction::OPC, V1, V2, Name, BB);            \
}                                                                            \
static BinaryOperator *Create##NUWNSWEXACT##OPC(                             \
    Value *V1, Value *V2, const Twine &Name, Instruction *I) {               \
  return Create##NUWNSWEXACT(Instruction::OPC, V1, V2, Name, I);             \
}

DEFINE_HELPERS(Add, NSW) // CreateNSWAdd
DEFINE_HELPERS(Add, NUW) // CreateNUWAdd
DEFINE_HELPERS(Sub, NSW) // CreateNSWSub
DEFINE_HELPERS(Sub, NUW) // CreateNUWSub
DEFINE_HELPERS(Mul, NSW) // CreateNSWMul
DEFINE_HELPERS(Mul, NUW) // CreateNUWMul
DEFINE_HELPERS(Shl, NSW) // CreateNSWShl
DEFINE_HELPERS(Shl, NUW) // CreateNUWShl

DEFINE_HELPERS(SDiv, Exact)  // CreateExactSDiv
DEFINE_HELPERS(UDiv, Exact)  // CreateExactUDiv
DEFINE_HELPERS(AShr, Exact)  // CreateExactAShr
DEFINE_HELPERS(LShr, Exact)  // CreateExactLShr

371#undef DEFINE_HELPERS

/// Helper functions to construct and inspect unary operations (NEG and NOT)
/// via binary operators SUB and XOR:
///
/// Create the NEG and NOT instructions out of SUB and XOR instructions.
///
static BinaryOperator *CreateNeg(Value *Op, const Twine &Name = "",
                                 Instruction *InsertBefore = nullptr);
static BinaryOperator *CreateNeg(Value *Op, const Twine &Name,
                                 BasicBlock *InsertAtEnd);
static BinaryOperator *CreateNSWNeg(Value *Op, const Twine &Name = "",
                                    Instruction *InsertBefore = nullptr);
static BinaryOperator *CreateNSWNeg(Value *Op, const Twine &Name,
                                    BasicBlock *InsertAtEnd);
static BinaryOperator *CreateNUWNeg(Value *Op, const Twine &Name = "",
                                    Instruction *InsertBefore = nullptr);
static BinaryOperator *CreateNUWNeg(Value *Op, const Twine &Name,
                                    BasicBlock *InsertAtEnd);
static BinaryOperator *CreateNot(Value *Op, const Twine &Name = "",
                                 Instruction *InsertBefore = nullptr);
static BinaryOperator *CreateNot(Value *Op, const Twine &Name,
                                 BasicBlock *InsertAtEnd);

BinaryOps getOpcode() const {
  return static_cast<BinaryOps>(Instruction::getOpcode());
}

/// Exchange the two operands to this instruction.
/// This instruction is safe to use on any binary instruction and
/// does not modify the semantics of the instruction.  If the instruction
/// cannot be reversed (ie, it's a Div), then return true.
///
bool swapOperands();

// Methods for support type inquiry through isa, cast, and dyn_cast:
static bool classof(const Instruction *I) {
  return I->isBinaryOp();
}
static bool classof(const Value *V) {
  return isa<Instruction>(V) && classof(cast<Instruction>(V));
}
413};

415template <>
416struct OperandTraits<BinaryOperator> :
public FixedNumOperandTraits<BinaryOperator, 2> {
418};

420DEFINE_TRANSPARENT_OPERAND_ACCESSORS(BinaryOperator, Value)BinaryOperator::op_iterator BinaryOperator::op_begin() { return
 OperandTraits<BinaryOperator>::op_begin(this); } BinaryOperator
::const_op_iterator BinaryOperator::op_begin() const { return
 OperandTraits<BinaryOperator>::op_begin(const_cast<
BinaryOperator*>(this)); } BinaryOperator::op_iterator BinaryOperator
::op_end() { return OperandTraits<BinaryOperator>::op_end
(this); } BinaryOperator::const_op_iterator BinaryOperator::op_end
() const { return OperandTraits<BinaryOperator>::op_end
(const_cast<BinaryOperator*>(this)); } Value *BinaryOperator
::getOperand(unsigned i_nocapture) const { (static_cast <bool
> (i_nocapture < OperandTraits<BinaryOperator>::operands
(this) && "getOperand() out of range!") ? void (0) : __assert_fail
 ("i_nocapture < OperandTraits<BinaryOperator>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 420, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<BinaryOperator>::op_begin(
const_cast<BinaryOperator*>(this))[i_nocapture].get());
 } void BinaryOperator::setOperand(unsigned i_nocapture, Value
 *Val_nocapture) { (static_cast <bool> (i_nocapture <
 OperandTraits<BinaryOperator>::operands(this) &&
 "setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<BinaryOperator>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 420, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
BinaryOperator>::op_begin(this)[i_nocapture] = Val_nocapture
; } unsigned BinaryOperator::getNumOperands() const { return OperandTraits
<BinaryOperator>::operands(this); } template <int Idx_nocapture
> Use &BinaryOperator::Op() { return this->OpFrom<
Idx_nocapture>(this); } template <int Idx_nocapture>
 const Use &BinaryOperator::Op() const { return this->
OpFrom<Idx_nocapture>(this); }

422//===----------------------------------------------------------------------===//
423//                               CastInst Class
424//===----------------------------------------------------------------------===//

426/// This is the base class for all instructions that perform data
427/// casts. It is simply provided so that instruction category testing
428/// can be performed with code like:
429///
430/// if (isa<CastInst>(Instr)) { ... }
431/// Base class of casting instructions.
432class CastInst : public UnaryInstruction {
433protected:
/// Constructor with insert-before-instruction semantics for subclasses
CastInst(Type *Ty, unsigned iType, Value *S,
         const Twine &NameStr = "", Instruction *InsertBefore = nullptr)
  : UnaryInstruction(Ty, iType, S, InsertBefore) {
  setName(NameStr);
}
/// Constructor with insert-at-end-of-block semantics for subclasses
CastInst(Type *Ty, unsigned iType, Value *S,
         const Twine &NameStr, BasicBlock *InsertAtEnd)
  : UnaryInstruction(Ty, iType, S, InsertAtEnd) {
  setName(NameStr);
}

447public:
/// Provides a way to construct any of the CastInst subclasses using an
/// opcode instead of the subclass's constructor. The opcode must be in the
/// CastOps category (Instruction::isCast(opcode) returns true). This
/// constructor has insert-before-instruction semantics to automatically
/// insert the new CastInst before InsertBefore (if it is non-null).
/// Construct any of the CastInst subclasses
static CastInst *Create(
  Instruction::CastOps,    ///< The opcode of the cast instruction
  Value *S,                ///< The value to be casted (operand 0)
  Type *Ty,          ///< The type to which cast should be made
  const Twine &Name = "", ///< Name for the instruction
  Instruction *InsertBefore = nullptr ///< Place to insert the instruction
);
/// Provides a way to construct any of the CastInst subclasses using an
/// opcode instead of the subclass's constructor. The opcode must be in the
/// CastOps category. This constructor has insert-at-end-of-block semantics
/// to automatically insert the new CastInst at the end of InsertAtEnd (if
/// its non-null).
/// Construct any of the CastInst subclasses
static CastInst *Create(
  Instruction::CastOps,    ///< The opcode for the cast instruction
  Value *S,                ///< The value to be casted (operand 0)
  Type *Ty,          ///< The type to which operand is casted
  const Twine &Name, ///< The name for the instruction
  BasicBlock *InsertAtEnd  ///< The block to insert the instruction into
);

/// Create a ZExt or BitCast cast instruction
static CastInst *CreateZExtOrBitCast(
  Value *S,                ///< The value to be casted (operand 0)
  Type *Ty,          ///< The type to which cast should be made
  const Twine &Name = "", ///< Name for the instruction
  Instruction *InsertBefore = nullptr ///< Place to insert the instruction
);

/// Create a ZExt or BitCast cast instruction
static CastInst *CreateZExtOrBitCast(
  Value *S,                ///< The value to be casted (operand 0)
  Type *Ty,          ///< The type to which operand is casted
  const Twine &Name, ///< The name for the instruction
  BasicBlock *InsertAtEnd  ///< The block to insert the instruction into
);

/// Create a SExt or BitCast cast instruction
static CastInst *CreateSExtOrBitCast(
  Value *S,                ///< The value to be casted (operand 0)
  Type *Ty,          ///< The type to which cast should be made
  const Twine &Name = "", ///< Name for the instruction
  Instruction *InsertBefore = nullptr ///< Place to insert the instruction
);

/// Create a SExt or BitCast cast instruction
static CastInst *CreateSExtOrBitCast(
  Value *S,                ///< The value to be casted (operand 0)
  Type *Ty,          ///< The type to which operand is casted
  const Twine &Name, ///< The name for the instruction
  BasicBlock *InsertAtEnd  ///< The block to insert the instruction into
);

/// Create a BitCast AddrSpaceCast, or a PtrToInt cast instruction.
static CastInst *CreatePointerCast(
  Value *S,                ///< The pointer value to be casted (operand 0)
  Type *Ty,          ///< The type to which operand is casted
  const Twine &Name, ///< The name for the instruction
  BasicBlock *InsertAtEnd  ///< The block to insert the instruction into
);

/// Create a BitCast, AddrSpaceCast or a PtrToInt cast instruction.
static CastInst *CreatePointerCast(
  Value *S,                ///< The pointer value to be casted (operand 0)
  Type *Ty,          ///< The type to which cast should be made
  const Twine &Name = "", ///< Name for the instruction
  Instruction *InsertBefore = nullptr ///< Place to insert the instruction
);

/// Create a BitCast or an AddrSpaceCast cast instruction.
static CastInst *CreatePointerBitCastOrAddrSpaceCast(
  Value *S,                ///< The pointer value to be casted (operand 0)
  Type *Ty,          ///< The type to which operand is casted
  const Twine &Name, ///< The name for the instruction
  BasicBlock *InsertAtEnd  ///< The block to insert the instruction into
);

/// Create a BitCast or an AddrSpaceCast cast instruction.
static CastInst *CreatePointerBitCastOrAddrSpaceCast(
  Value *S,                ///< The pointer value to be casted (operand 0)
  Type *Ty,          ///< The type to which cast should be made
  const Twine &Name = "", ///< Name for the instruction
  Instruction *InsertBefore = nullptr ///< Place to insert the instruction
);

/// Create a BitCast, a PtrToInt, or an IntToPTr cast instruction.
///
/// If the value is a pointer type and the destination an integer type,
/// creates a PtrToInt cast. If the value is an integer type and the
/// destination a pointer type, creates an IntToPtr cast. Otherwise, creates
/// a bitcast.
static CastInst *CreateBitOrPointerCast(
  Value *S,                ///< The pointer value to be casted (operand 0)
  Type *Ty,          ///< The type to which cast should be made
  const Twine &Name = "", ///< Name for the instruction
  Instruction *InsertBefore = nullptr ///< Place to insert the instruction
);

/// Create a ZExt, BitCast, or Trunc for int -> int casts.
static CastInst *CreateIntegerCast(
  Value *S,                ///< The pointer value to be casted (operand 0)
  Type *Ty,          ///< The type to which cast should be made
  bool isSigned,           ///< Whether to regard S as signed or not
  const Twine &Name = "", ///< Name for the instruction
  Instruction *InsertBefore = nullptr ///< Place to insert the instruction
);

/// Create a ZExt, BitCast, or Trunc for int -> int casts.
static CastInst *CreateIntegerCast(
  Value *S,                ///< The integer value to be casted (operand 0)
  Type *Ty,          ///< The integer type to which operand is casted
  bool isSigned,           ///< Whether to regard S as signed or not
  const Twine &Name, ///< The name for the instruction
  BasicBlock *InsertAtEnd  ///< The block to insert the instruction into
);

/// Create an FPExt, BitCast, or FPTrunc for fp -> fp casts
static CastInst *CreateFPCast(
  Value *S,                ///< The floating point value to be casted
  Type *Ty,          ///< The floating point type to cast to
  const Twine &Name = "", ///< Name for the instruction
  Instruction *InsertBefore = nullptr ///< Place to insert the instruction
);

/// Create an FPExt, BitCast, or FPTrunc for fp -> fp casts
static CastInst *CreateFPCast(
  Value *S,                ///< The floating point value to be casted
  Type *Ty,          ///< The floating point type to cast to
  const Twine &Name, ///< The name for the instruction
  BasicBlock *InsertAtEnd  ///< The block to insert the instruction into
);

/// Create a Trunc or BitCast cast instruction
static CastInst *CreateTruncOrBitCast(
  Value *S,                ///< The value to be casted (operand 0)
  Type *Ty,          ///< The type to which cast should be made
  const Twine &Name = "", ///< Name for the instruction
  Instruction *InsertBefore = nullptr ///< Place to insert the instruction
);

/// Create a Trunc or BitCast cast instruction
static CastInst *CreateTruncOrBitCast(
  Value *S,                ///< The value to be casted (operand 0)
  Type *Ty,          ///< The type to which operand is casted
  const Twine &Name, ///< The name for the instruction
  BasicBlock *InsertAtEnd  ///< The block to insert the instruction into
);

/// Check whether a bitcast between these types is valid
static bool isBitCastable(
  Type *SrcTy, ///< The Type from which the value should be cast.
  Type *DestTy ///< The Type to which the value should be cast.
);

/// Check whether a bitcast, inttoptr, or ptrtoint cast between these
/// types is valid and a no-op.
///
/// This ensures that any pointer<->integer cast has enough bits in the
/// integer and any other cast is a bitcast.
static bool isBitOrNoopPointerCastable(
    Type *SrcTy,  ///< The Type from which the value should be cast.
    Type *DestTy, ///< The Type to which the value should be cast.
    const DataLayout &DL);

/// Returns the opcode necessary to cast Val into Ty using usual casting
/// rules.
/// Infer the opcode for cast operand and type
static Instruction::CastOps getCastOpcode(
  const Value *Val, ///< The value to cast
  bool SrcIsSigned, ///< Whether to treat the source as signed
  Type *Ty,   ///< The Type to which the value should be casted
  bool DstIsSigned  ///< Whether to treate the dest. as signed
);

/// There are several places where we need to know if a cast instruction
/// only deals with integer source and destination types. To simplify that
/// logic, this method is provided.
/// @returns true iff the cast has only integral typed operand and dest type.
/// Determine if this is an integer-only cast.
bool isIntegerCast() const;

/// A lossless cast is one that does not alter the basic value. It implies
/// a no-op cast but is more stringent, preventing things like int->float,
/// long->double, or int->ptr.
/// @returns true iff the cast is lossless.
/// Determine if this is a lossless cast.
bool isLosslessCast() const;

/// A no-op cast is one that can be effected without changing any bits.
/// It implies that the source and destination types are the same size. The
/// DataLayout argument is to determine the pointer size when examining casts
/// involving Integer and Pointer types. They are no-op casts if the integer
/// is the same size as the pointer. However, pointer size varies with
/// platform.  Note that a precondition of this method is that the cast is
/// legal - i.e. the instruction formed with these operands would verify.
static bool isNoopCast(
  Instruction::CastOps Opcode, ///< Opcode of cast
  Type *SrcTy,         ///< SrcTy of cast
  Type *DstTy,         ///< DstTy of cast
  const DataLayout &DL ///< DataLayout to get the Int Ptr type from.
);

/// Determine if this cast is a no-op cast.
///
/// \param DL is the DataLayout to determine pointer size.
bool isNoopCast(const DataLayout &DL) const;

/// Determine how a pair of casts can be eliminated, if they can be at all.
/// This is a helper function for both CastInst and ConstantExpr.
/// @returns 0 if the CastInst pair can't be eliminated, otherwise
/// returns Instruction::CastOps value for a cast that can replace
/// the pair, casting SrcTy to DstTy.
/// Determine if a cast pair is eliminable
static unsigned isEliminableCastPair(
  Instruction::CastOps firstOpcode,  ///< Opcode of first cast
  Instruction::CastOps secondOpcode, ///< Opcode of second cast
  Type *SrcTy, ///< SrcTy of 1st cast
  Type *MidTy, ///< DstTy of 1st cast & SrcTy of 2nd cast
  Type *DstTy, ///< DstTy of 2nd cast
  Type *SrcIntPtrTy, ///< Integer type corresponding to Ptr SrcTy, or null
  Type *MidIntPtrTy, ///< Integer type corresponding to Ptr MidTy, or null
  Type *DstIntPtrTy  ///< Integer type corresponding to Ptr DstTy, or null
);

/// Return the opcode of this CastInst
Instruction::CastOps getOpcode() const {
  return Instruction::CastOps(Instruction::getOpcode());
}

/// Return the source type, as a convenience
Type* getSrcTy() const { return getOperand(0)->getType(); }
/// Return the destination type, as a convenience
Type* getDestTy() const { return getType(); }

/// This method can be used to determine if a cast from SrcTy to DstTy using
/// Opcode op is valid or not.
/// @returns true iff the proposed cast is valid.
/// Determine if a cast is valid without creating one.
static bool castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy);
static bool castIsValid(Instruction::CastOps op, Value *S, Type *DstTy) {
  return castIsValid(op, S->getType(), DstTy);
}

/// Methods for support type inquiry through isa, cast, and dyn_cast:
static bool classof(const Instruction *I) {
  return I->isCast();
}
static bool classof(const Value *V) {
  return isa<Instruction>(V) && classof(cast<Instruction>(V));
}
704};

706//===----------------------------------------------------------------------===//
707//                               CmpInst Class
708//===----------------------------------------------------------------------===//

710/// This class is the base class for the comparison instructions.
711/// Abstract base class of comparison instructions.
712class CmpInst : public Instruction {
713public:
/// This enumeration lists the possible predicates for CmpInst subclasses.
/// Values in the range 0-31 are reserved for FCmpInst, while values in the
/// range 32-64 are reserved for ICmpInst. This is necessary to ensure the
/// predicate values are not overlapping between the classes.
///
/// Some passes (e.g. InstCombine) depend on the bit-wise characteristics of
/// FCMP_* values. Changing the bit patterns requires a potential change to
/// those passes.
enum Predicate : unsigned {
  // Opcode            U L G E    Intuitive operation
  FCMP_FALSE = 0, ///< 0 0 0 0    Always false (always folded)
  FCMP_OEQ = 1,   ///< 0 0 0 1    True if ordered and equal
  FCMP_OGT = 2,   ///< 0 0 1 0    True if ordered and greater than
  FCMP_OGE = 3,   ///< 0 0 1 1    True if ordered and greater than or equal
  FCMP_OLT = 4,   ///< 0 1 0 0    True if ordered and less than
  FCMP_OLE = 5,   ///< 0 1 0 1    True if ordered and less than or equal
  FCMP_ONE = 6,   ///< 0 1 1 0    True if ordered and operands are unequal
  FCMP_ORD = 7,   ///< 0 1 1 1    True if ordered (no nans)
  FCMP_UNO = 8,   ///< 1 0 0 0    True if unordered: isnan(X) | isnan(Y)
  FCMP_UEQ = 9,   ///< 1 0 0 1    True if unordered or equal
  FCMP_UGT = 10,  ///< 1 0 1 0    True if unordered or greater than
  FCMP_UGE = 11,  ///< 1 0 1 1    True if unordered, greater than, or equal
  FCMP_ULT = 12,  ///< 1 1 0 0    True if unordered or less than
  FCMP_ULE = 13,  ///< 1 1 0 1    True if unordered, less than, or equal
  FCMP_UNE = 14,  ///< 1 1 1 0    True if unordered or not equal
  FCMP_TRUE = 15, ///< 1 1 1 1    Always true (always folded)
  FIRST_FCMP_PREDICATE = FCMP_FALSE,
  LAST_FCMP_PREDICATE = FCMP_TRUE,
  BAD_FCMP_PREDICATE = FCMP_TRUE + 1,
  ICMP_EQ = 32,  ///< equal
  ICMP_NE = 33,  ///< not equal
  ICMP_UGT = 34, ///< unsigned greater than
  ICMP_UGE = 35, ///< unsigned greater or equal
  ICMP_ULT = 36, ///< unsigned less than
  ICMP_ULE = 37, ///< unsigned less or equal
  ICMP_SGT = 38, ///< signed greater than
  ICMP_SGE = 39, ///< signed greater or equal
  ICMP_SLT = 40, ///< signed less than
  ICMP_SLE = 41, ///< signed less or equal
  FIRST_ICMP_PREDICATE = ICMP_EQ,
  LAST_ICMP_PREDICATE = ICMP_SLE,
  BAD_ICMP_PREDICATE = ICMP_SLE + 1
};
using PredicateField =
    Bitfield::Element<Predicate, 0, 6, LAST_ICMP_PREDICATE>;

760protected:
CmpInst(Type *ty, Instruction::OtherOps op, Predicate pred,
        Value *LHS, Value *RHS, const Twine &Name = "",
        Instruction *InsertBefore = nullptr,
        Instruction *FlagsSource = nullptr);

CmpInst(Type *ty, Instruction::OtherOps op, Predicate pred,
        Value *LHS, Value *RHS, const Twine &Name,
        BasicBlock *InsertAtEnd);

770public:
// allocate space for exactly two operands
void *operator new(size_t s) {
  return User::operator new(s, 2);
}

/// Construct a compare instruction, given the opcode, the predicate and
/// the two operands.  Optionally (if InstBefore is specified) insert the
/// instruction into a BasicBlock right before the specified instruction.
/// The specified Instruction is allowed to be a dereferenced end iterator.
/// Create a CmpInst
static CmpInst *Create(OtherOps Op,
                       Predicate predicate, Value *S1,
                       Value *S2, const Twine &Name = "",
                       Instruction *InsertBefore = nullptr);

/// Construct a compare instruction, given the opcode, the predicate and the
/// two operands.  Also automatically insert this instruction to the end of
/// the BasicBlock specified.
/// Create a CmpInst
static CmpInst *Create(OtherOps Op, Predicate predicate, Value *S1,
                       Value *S2, const Twine &Name, BasicBlock *InsertAtEnd);

/// Get the opcode casted to the right type
OtherOps getOpcode() const {
  return static_cast<OtherOps>(Instruction::getOpcode());
}

/// Return the predicate for this instruction.
Predicate getPredicate() const { return getSubclassData<PredicateField>(); }

/// Set the predicate for this instruction to the specified value.
void setPredicate(Predicate P) { setSubclassData<PredicateField>(P); }

static bool isFPPredicate(Predicate P) {
  static_assert(FIRST_FCMP_PREDICATE == 0,
                "FIRST_FCMP_PREDICATE is required to be 0");
  return P <= LAST_FCMP_PREDICATE;
}

static bool isIntPredicate(Predicate P) {
  return P >= FIRST_ICMP_PREDICATE && P <= LAST_ICMP_PREDICATE;
}

static StringRef getPredicateName(Predicate P);

bool isFPPredicate() const { return isFPPredicate(getPredicate()); }
bool isIntPredicate() const { return isIntPredicate(getPredicate()); }

/// For example, EQ -> NE, UGT -> ULE, SLT -> SGE,
///              OEQ -> UNE, UGT -> OLE, OLT -> UGE, etc.
/// @returns the inverse predicate for the instruction's current predicate.
/// Return the inverse of the instruction's predicate.
Predicate getInversePredicate() const {
  return getInversePredicate(getPredicate());
}

/// For example, EQ -> NE, UGT -> ULE, SLT -> SGE,
///              OEQ -> UNE, UGT -> OLE, OLT -> UGE, etc.
/// @returns the inverse predicate for predicate provided in \p pred.
/// Return the inverse of a given predicate
static Predicate getInversePredicate(Predicate pred);

/// For example, EQ->EQ, SLE->SGE, ULT->UGT,
///              OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
/// @returns the predicate that would be the result of exchanging the two
/// operands of the CmpInst instruction without changing the result
/// produced.
/// Return the predicate as if the operands were swapped
Predicate getSwappedPredicate() const {
  return getSwappedPredicate(getPredicate());
}

/// This is a static version that you can use without an instruction
/// available.
/// Return the predicate as if the operands were swapped.
static Predicate getSwappedPredicate(Predicate pred);

/// This is a static version that you can use without an instruction
/// available.
/// @returns true if the comparison predicate is strict, false otherwise.
static bool isStrictPredicate(Predicate predicate);

/// @returns true if the comparison predicate is strict, false otherwise.
/// Determine if this instruction is using an strict comparison predicate.
bool isStrictPredicate() const { return isStrictPredicate(getPredicate()); }

/// This is a static version that you can use without an instruction
/// available.
/// @returns true if the comparison predicate is non-strict, false otherwise.
static bool isNonStrictPredicate(Predicate predicate);

/// @returns true if the comparison predicate is non-strict, false otherwise.
/// Determine if this instruction is using an non-strict comparison predicate.
bool isNonStrictPredicate() const {
  return isNonStrictPredicate(getPredicate());
}

/// For example, SGE -> SGT, SLE -> SLT, ULE -> ULT, UGE -> UGT.
/// Returns the strict version of non-strict comparisons.
Predicate getStrictPredicate() const {
  return getStrictPredicate(getPredicate());
}

/// This is a static version that you can use without an instruction
/// available.
/// @returns the strict version of comparison provided in \p pred.
/// If \p pred is not a strict comparison predicate, returns \p pred.
/// Returns the strict version of non-strict comparisons.
static Predicate getStrictPredicate(Predicate pred);

/// For example, SGT -> SGE, SLT -> SLE, ULT -> ULE, UGT -> UGE.
/// Returns the non-strict version of strict comparisons.
Predicate getNonStrictPredicate() const {
  return getNonStrictPredicate(getPredicate());
}

/// This is a static version that you can use without an instruction
/// available.
/// @returns the non-strict version of comparison provided in \p pred.
/// If \p pred is not a strict comparison predicate, returns \p pred.
/// Returns the non-strict version of strict comparisons.
static Predicate getNonStrictPredicate(Predicate pred);

/// This is a static version that you can use without an instruction
/// available.
/// Return the flipped strictness of predicate
static Predicate getFlippedStrictnessPredicate(Predicate pred);

/// For predicate of kind "is X or equal to 0" returns the predicate "is X".
/// For predicate of kind "is X" returns the predicate "is X or equal to 0".
/// does not support other kind of predicates.
/// @returns the predicate that does not contains is equal to zero if
/// it had and vice versa.
/// Return the flipped strictness of predicate
Predicate getFlippedStrictnessPredicate() const {
  return getFlippedStrictnessPredicate(getPredicate());
}

/// Provide more efficient getOperand methods.
DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
 setOperand(unsigned, Value*); inline op_iterator op_begin();
 inline const_op_iterator op_begin() const; inline op_iterator
 op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
 getNumOperands() const;

/// This is just a convenience that dispatches to the subclasses.
/// Swap the operands and adjust predicate accordingly to retain
/// the same comparison.
void swapOperands();

/// This is just a convenience that dispatches to the subclasses.
/// Determine if this CmpInst is commutative.
bool isCommutative() const;

/// Determine if this is an equals/not equals predicate.
/// This is a static version that you can use without an instruction
/// available.
static bool isEquality(Predicate pred);

/// Determine if this is an equals/not equals predicate.
bool isEquality() const { return isEquality(getPredicate()); }

/// Return true if the predicate is relational (not EQ or NE).
static bool isRelational(Predicate P) { return !isEquality(P); }

/// Return true if the predicate is relational (not EQ or NE).
bool isRelational() const { return !isEquality(); }

/// @returns true if the comparison is signed, false otherwise.
/// Determine if this instruction is using a signed comparison.
bool isSigned() const {
  return isSigned(getPredicate());
}

/// @returns true if the comparison is unsigned, false otherwise.
/// Determine if this instruction is using an unsigned comparison.
bool isUnsigned() const {
  return isUnsigned(getPredicate());
}

/// For example, ULT->SLT, ULE->SLE, UGT->SGT, UGE->SGE, SLT->Failed assert
/// @returns the signed version of the unsigned predicate pred.
/// return the signed version of a predicate
static Predicate getSignedPredicate(Predicate pred);

/// For example, ULT->SLT, ULE->SLE, UGT->SGT, UGE->SGE, SLT->Failed assert
/// @returns the signed version of the predicate for this instruction (which
/// has to be an unsigned predicate).
/// return the signed version of a predicate
Predicate getSignedPredicate() {
  return getSignedPredicate(getPredicate());
}

/// For example, SLT->ULT, SLE->ULE, SGT->UGT, SGE->UGE, ULT->Failed assert
/// @returns the unsigned version of the signed predicate pred.
static Predicate getUnsignedPredicate(Predicate pred);

/// For example, SLT->ULT, SLE->ULE, SGT->UGT, SGE->UGE, ULT->Failed assert
/// @returns the unsigned version of the predicate for this instruction (which
/// has to be an signed predicate).
/// return the unsigned version of a predicate
Predicate getUnsignedPredicate() {
  return getUnsignedPredicate(getPredicate());
}

/// For example, SLT->ULT, ULT->SLT, SLE->ULE, ULE->SLE, EQ->Failed assert
/// @returns the unsigned version of the signed predicate pred or
///          the signed version of the signed predicate pred.
static Predicate getFlippedSignednessPredicate(Predicate pred);

/// For example, SLT->ULT, ULT->SLT, SLE->ULE, ULE->SLE, EQ->Failed assert
/// @returns the unsigned version of the signed predicate pred or
///          the signed version of the signed predicate pred.
Predicate getFlippedSignednessPredicate() {
  return getFlippedSignednessPredicate(getPredicate());
}

/// This is just a convenience.
/// Determine if this is true when both operands are the same.
bool isTrueWhenEqual() const {
  return isTrueWhenEqual(getPredicate());
}

/// This is just a convenience.
/// Determine if this is false when both operands are the same.
bool isFalseWhenEqual() const {
  return isFalseWhenEqual(getPredicate());
}

/// @returns true if the predicate is unsigned, false otherwise.
/// Determine if the predicate is an unsigned operation.
static bool isUnsigned(Predicate predicate);

/// @returns true if the predicate is signed, false otherwise.
/// Determine if the predicate is an signed operation.
static bool isSigned(Predicate predicate);

/// Determine if the predicate is an ordered operation.
static bool isOrdered(Predicate predicate);

/// Determine if the predicate is an unordered operation.
static bool isUnordered(Predicate predicate);

/// Determine if the predicate is true when comparing a value with itself.
static bool isTrueWhenEqual(Predicate predicate);

/// Determine if the predicate is false when comparing a value with itself.
static bool isFalseWhenEqual(Predicate predicate);

/// Determine if Pred1 implies Pred2 is true when two compares have matching
/// operands.
static bool isImpliedTrueByMatchingCmp(Predicate Pred1, Predicate Pred2);

/// Determine if Pred1 implies Pred2 is false when two compares have matching
/// operands.
static bool isImpliedFalseByMatchingCmp(Predicate Pred1, Predicate Pred2);

/// Methods for support type inquiry through isa, cast, and dyn_cast:
static bool classof(const Instruction *I) {
  return I->getOpcode() == Instruction::ICmp ||
         I->getOpcode() == Instruction::FCmp;
}
static bool classof(const Value *V) {
  return isa<Instruction>(V) && classof(cast<Instruction>(V));
}

/// Create a result type for fcmp/icmp
static Type* makeCmpResultType(Type* opnd_type) {
  if (VectorType* vt = dyn_cast<VectorType>(opnd_type)) {
    return VectorType::get(Type::getInt1Ty(opnd_type->getContext()),
                           vt->getElementCount());
  }
  return Type::getInt1Ty(opnd_type->getContext());
}

1042private:
// Shadow Value::setValueSubclassData with a private forwarding method so that
// subclasses cannot accidentally use it.
void setValueSubclassData(unsigned short D) {
  Value::setValueSubclassData(D);
}
1048};

1050// FIXME: these are redundant if CmpInst < BinaryOperator
1051template <>
1052struct OperandTraits<CmpInst> : public FixedNumOperandTraits<CmpInst, 2> {
1053};

1055DEFINE_TRANSPARENT_OPERAND_ACCESSORS(CmpInst, Value)CmpInst::op_iterator CmpInst::op_begin() { return OperandTraits
<CmpInst>::op_begin(this); } CmpInst::const_op_iterator
 CmpInst::op_begin() const { return OperandTraits<CmpInst>
::op_begin(const_cast<CmpInst*>(this)); } CmpInst::op_iterator
 CmpInst::op_end() { return OperandTraits<CmpInst>::op_end
(this); } CmpInst::const_op_iterator CmpInst::op_end() const {
 return OperandTraits<CmpInst>::op_end(const_cast<CmpInst
*>(this)); } Value *CmpInst::getOperand(unsigned i_nocapture
) const { (static_cast <bool> (i_nocapture < OperandTraits
<CmpInst>::operands(this) && "getOperand() out of range!"
) ? void (0) : __assert_fail ("i_nocapture < OperandTraits<CmpInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1055, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<CmpInst>::op_begin(const_cast
<CmpInst*>(this))[i_nocapture].get()); } void CmpInst::
setOperand(unsigned i_nocapture, Value *Val_nocapture) { (static_cast
 <bool> (i_nocapture < OperandTraits<CmpInst>::
operands(this) && "setOperand() out of range!") ? void
 (0) : __assert_fail ("i_nocapture < OperandTraits<CmpInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1055, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
CmpInst>::op_begin(this)[i_nocapture] = Val_nocapture; } unsigned
 CmpInst::getNumOperands() const { return OperandTraits<CmpInst
>::operands(this); } template <int Idx_nocapture> Use
 &CmpInst::Op() { return this->OpFrom<Idx_nocapture
>(this); } template <int Idx_nocapture> const Use &
CmpInst::Op() const { return this->OpFrom<Idx_nocapture
>(this); }

1057/// A lightweight accessor for an operand bundle meant to be passed
1058/// around by value.
1059struct OperandBundleUse {
ArrayRef<Use> Inputs;

OperandBundleUse() = default;
explicit OperandBundleUse(StringMapEntry<uint32_t> *Tag, ArrayRef<Use> Inputs)
    : Inputs(Inputs), Tag(Tag) {}

/// Return true if the operand at index \p Idx in this operand bundle
/// has the attribute A.
bool operandHasAttr(unsigned Idx, Attribute::AttrKind A) const {
  if (isDeoptOperandBundle())
    if (A == Attribute::ReadOnly || A == Attribute::NoCapture)
      return Inputs[Idx]->getType()->isPointerTy();

  // Conservative answer:  no operands have any attributes.
  return false;
}

/// Return the tag of this operand bundle as a string.
StringRef getTagName() const {
  return Tag->getKey();
}

/// Return the tag of this operand bundle as an integer.
///
/// Operand bundle tags are interned by LLVMContextImpl::getOrInsertBundleTag,
/// and this function returns the unique integer getOrInsertBundleTag
/// associated the tag of this operand bundle to.
uint32_t getTagID() const {
  return Tag->getValue();
}

/// Return true if this is a "deopt" operand bundle.
bool isDeoptOperandBundle() const {
  return getTagID() == LLVMContext::OB_deopt;
}

/// Return true if this is a "funclet" operand bundle.
bool isFuncletOperandBundle() const {
  return getTagID() == LLVMContext::OB_funclet;
}

/// Return true if this is a "cfguardtarget" operand bundle.
bool isCFGuardTargetOperandBundle() const {
  return getTagID() == LLVMContext::OB_cfguardtarget;
}

1106private:
/// Pointer to an entry in LLVMContextImpl::getOrInsertBundleTag.
StringMapEntry<uint32_t> *Tag;
1109};

1111/// A container for an operand bundle being viewed as a set of values
1112/// rather than a set of uses.
1113///
1114/// Unlike OperandBundleUse, OperandBundleDefT owns the memory it carries, and
1115/// so it is possible to create and pass around "self-contained" instances of
1116/// OperandBundleDef and ConstOperandBundleDef.
1117template <typename InputTy> class OperandBundleDefT {
std::string Tag;
std::vector<InputTy> Inputs;

1121public:
explicit OperandBundleDefT(std::string Tag, std::vector<InputTy> Inputs)
    : Tag(std::move(Tag)), Inputs(std::move(Inputs)) {}
explicit OperandBundleDefT(std::string Tag, ArrayRef<InputTy> Inputs)
    : Tag(std::move(Tag)), Inputs(Inputs) {}

explicit OperandBundleDefT(const OperandBundleUse &OBU) {
  Tag = std::string(OBU.getTagName());
  llvm::append_range(Inputs, OBU.Inputs);
}

ArrayRef<InputTy> inputs() const { return Inputs; }

using input_iterator = typename std::vector<InputTy>::const_iterator;

size_t input_size() const { return Inputs.size(); }
input_iterator input_begin() const { return Inputs.begin(); }
input_iterator input_end() const { return Inputs.end(); }

StringRef getTag() const { return Tag; }
1141};

1143using OperandBundleDef = OperandBundleDefT<Value *>;
1144using ConstOperandBundleDef = OperandBundleDefT<const Value *>;

1146//===----------------------------------------------------------------------===//
1147//                               CallBase Class
1148//===----------------------------------------------------------------------===//

1150/// Base class for all callable instructions (InvokeInst and CallInst)
1151/// Holds everything related to calling a function.
1152///
1153/// All call-like instructions are required to use a common operand layout:
1154/// - Zero or more arguments to the call,
1155/// - Zero or more operand bundles with zero or more operand inputs each
1156///   bundle,
1157/// - Zero or more subclass controlled operands
1158/// - The called function.
1159///
1160/// This allows this base class to easily access the called function and the
1161/// start of the arguments without knowing how many other operands a particular
1162/// subclass requires. Note that accessing the end of the argument list isn't
1163/// as cheap as most other operations on the base class.
1164class CallBase : public Instruction {
1165protected:
// The first two bits are reserved by CallInst for fast retrieval,
using CallInstReservedField = Bitfield::Element<unsigned, 0, 2>;
using CallingConvField =
    Bitfield::Element<CallingConv::ID, CallInstReservedField::NextBit, 10,
                      CallingConv::MaxID>;
static_assert(
    Bitfield::areContiguous<CallInstReservedField, CallingConvField>(),
    "Bitfields must be contiguous");

/// The last operand is the called operand.
static constexpr int CalledOperandOpEndIdx = -1;

AttributeList Attrs; ///< parameter attributes for callable
FunctionType *FTy;

template <class... ArgsTy>
CallBase(AttributeList const &A, FunctionType *FT, ArgsTy &&... Args)
    : Instruction(std::forward<ArgsTy>(Args)...), Attrs(A), FTy(FT) {}

using Instruction::Instruction;

bool hasDescriptor() const { return Value::HasDescriptor; }

unsigned getNumSubclassExtraOperands() const {
  switch (getOpcode()) {
  case Instruction::Call:
    return 0;
  case Instruction::Invoke:
    return 2;
  case Instruction::CallBr:
    return getNumSubclassExtraOperandsDynamic();
  }
  llvm_unreachable("Invalid opcode!")::llvm::llvm_unreachable_internal("Invalid opcode!", "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1198);
}

/// Get the number of extra operands for instructions that don't have a fixed
/// number of extra operands.
unsigned getNumSubclassExtraOperandsDynamic() const;

1205public:
using Instruction::getContext;

/// Create a clone of \p CB with a different set of operand bundles and
/// insert it before \p InsertPt.
///
/// The returned call instruction is identical \p CB in every way except that
/// the operand bundles for the new instruction are set to the operand bundles
/// in \p Bundles.
static CallBase *Create(CallBase *CB, ArrayRef<OperandBundleDef> Bundles,
                        Instruction *InsertPt = nullptr);

/// Create a clone of \p CB with the operand bundle with the tag matching
/// \p Bundle's tag replaced with Bundle, and insert it before \p InsertPt.
///
/// The returned call instruction is identical \p CI in every way except that
/// the specified operand bundle has been replaced.
static CallBase *Create(CallBase *CB,
                        OperandBundleDef Bundle,
                        Instruction *InsertPt = nullptr);

/// Create a clone of \p CB with operand bundle \p OB added.
static CallBase *addOperandBundle(CallBase *CB, uint32_t ID,
                                  OperandBundleDef OB,
                                  Instruction *InsertPt = nullptr);

/// Create a clone of \p CB with operand bundle \p ID removed.
static CallBase *removeOperandBundle(CallBase *CB, uint32_t ID,
                                     Instruction *InsertPt = nullptr);

static bool classof(const Instruction *I) {
  return I->getOpcode() == Instruction::Call ||
         I->getOpcode() == Instruction::Invoke ||
         I->getOpcode() == Instruction::CallBr;
}
static bool classof(const Value *V) {
  return isa<Instruction>(V) && classof(cast<Instruction>(V));
}

FunctionType *getFunctionType() const { return FTy; }

void mutateFunctionType(FunctionType *FTy) {
  Value::mutateType(FTy->getReturnType());
  this->FTy = FTy;
}

DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
 setOperand(unsigned, Value*); inline op_iterator op_begin();
 inline const_op_iterator op_begin() const; inline op_iterator
 op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
 getNumOperands() const;

/// data_operands_begin/data_operands_end - Return iterators iterating over
/// the call / invoke argument list and bundle operands.  For invokes, this is
/// the set of instruction operands except the invoke target and the two
/// successor blocks; and for calls this is the set of instruction operands
/// except the call target.
User::op_iterator data_operands_begin() { return op_begin(); }
User::const_op_iterator data_operands_begin() const {
  return const_cast<CallBase *>(this)->data_operands_begin();
}
User::op_iterator data_operands_end() {
  // Walk from the end of the operands over the called operand and any
  // subclass operands.
  return op_end() - getNumSubclassExtraOperands() - 1;
}
User::const_op_iterator data_operands_end() const {
  return const_cast<CallBase *>(this)->data_operands_end();
}
iterator_range<User::op_iterator> data_ops() {
  return make_range(data_operands_begin(), data_operands_end());
}
iterator_range<User::const_op_iterator> data_ops() const {
  return make_range(data_operands_begin(), data_operands_end());
}
bool data_operands_empty() const {
  return data_operands_end() == data_operands_begin();
}
unsigned data_operands_size() const {
  return std::distance(data_operands_begin(), data_operands_end());
}

bool isDataOperand(const Use *U) const {
  assert(this == U->getUser() &&(static_cast <bool> (this == U->getUser() &&
 "Only valid to query with a use of this instruction!") ? void
 (0) : __assert_fail ("this == U->getUser() && \"Only valid to query with a use of this instruction!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1285, __extension__ __PRETTY_FUNCTION__))
         "Only valid to query with a use of this instruction!")(static_cast <bool> (this == U->getUser() &&
 "Only valid to query with a use of this instruction!") ? void
 (0) : __assert_fail ("this == U->getUser() && \"Only valid to query with a use of this instruction!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1285, __extension__ __PRETTY_FUNCTION__));
  return data_operands_begin() <= U && U < data_operands_end();
}
bool isDataOperand(Value::const_user_iterator UI) const {
  return isDataOperand(&UI.getUse());
}

/// Given a value use iterator, return the data operand corresponding to it.
/// Iterator must actually correspond to a data operand.
unsigned getDataOperandNo(Value::const_user_iterator UI) const {
  return getDataOperandNo(&UI.getUse());
}

/// Given a use for a data operand, get the data operand number that
/// corresponds to it.
unsigned getDataOperandNo(const Use *U) const {
  assert(isDataOperand(U) && "Data operand # out of range!")(static_cast <bool> (isDataOperand(U) && "Data operand # out of range!"
) ? void (0) : __assert_fail ("isDataOperand(U) && \"Data operand # out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1301, __extension__ __PRETTY_FUNCTION__));
  return U - data_operands_begin();
}

/// Return the iterator pointing to the beginning of the argument list.
User::op_iterator arg_begin() { return op_begin(); }
User::const_op_iterator arg_begin() const {
  return const_cast<CallBase *>(this)->arg_begin();
}

/// Return the iterator pointing to the end of the argument list.
User::op_iterator arg_end() {
  // From the end of the data operands, walk backwards past the bundle
  // operands.
  return data_operands_end() - getNumTotalBundleOperands();
}
User::const_op_iterator arg_end() const {
  return const_cast<CallBase *>(this)->arg_end();
}

/// Iteration adapter for range-for loops.
iterator_range<User::op_iterator> args() {
  return make_range(arg_begin(), arg_end());
}
iterator_range<User::const_op_iterator> args() const {
  return make_range(arg_begin(), arg_end());
}
bool arg_empty() const { return arg_end() == arg_begin(); }
unsigned arg_size() const { return arg_end() - arg_begin(); }

// Legacy API names that duplicate the above and will be removed once users
// are migrated.
iterator_range<User::op_iterator> arg_operands() {
  return make_range(arg_begin(), arg_end());
}
iterator_range<User::const_op_iterator> arg_operands() const {
  return make_range(arg_begin(), arg_end());
}
unsigned getNumArgOperands() const { return arg_size(); }

Value *getArgOperand(unsigned i) const {
  assert(i < getNumArgOperands() && "Out of bounds!")(static_cast <bool> (i < getNumArgOperands() &&
 "Out of bounds!") ? void (0) : __assert_fail ("i < getNumArgOperands() && \"Out of bounds!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1342, __extension__ __PRETTY_FUNCTION__));
  return getOperand(i);
}

void setArgOperand(unsigned i, Value *v) {
  assert(i < getNumArgOperands() && "Out of bounds!")(static_cast <bool> (i < getNumArgOperands() &&
 "Out of bounds!") ? void (0) : __assert_fail ("i < getNumArgOperands() && \"Out of bounds!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1347, __extension__ __PRETTY_FUNCTION__));
  setOperand(i, v);
}

/// Wrappers for getting the \c Use of a call argument.
const Use &getArgOperandUse(unsigned i) const {
  assert(i < getNumArgOperands() && "Out of bounds!")(static_cast <bool> (i < getNumArgOperands() &&
 "Out of bounds!") ? void (0) : __assert_fail ("i < getNumArgOperands() && \"Out of bounds!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1353, __extension__ __PRETTY_FUNCTION__));
  return User::getOperandUse(i);
}
Use &getArgOperandUse(unsigned i) {
  assert(i < getNumArgOperands() && "Out of bounds!")(static_cast <bool> (i < getNumArgOperands() &&
 "Out of bounds!") ? void (0) : __assert_fail ("i < getNumArgOperands() && \"Out of bounds!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1357, __extension__ __PRETTY_FUNCTION__));
  return User::getOperandUse(i);
}

bool isArgOperand(const Use *U) const {
  assert(this == U->getUser() &&(static_cast <bool> (this == U->getUser() &&
 "Only valid to query with a use of this instruction!") ? void
 (0) : __assert_fail ("this == U->getUser() && \"Only valid to query with a use of this instruction!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1363, __extension__ __PRETTY_FUNCTION__))
         "Only valid to query with a use of this instruction!")(static_cast <bool> (this == U->getUser() &&
 "Only valid to query with a use of this instruction!") ? void
 (0) : __assert_fail ("this == U->getUser() && \"Only valid to query with a use of this instruction!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1363, __extension__ __PRETTY_FUNCTION__));
  return arg_begin() <= U && U < arg_end();
}
bool isArgOperand(Value::const_user_iterator UI) const {
  return isArgOperand(&UI.getUse());
}

/// Given a use for a arg operand, get the arg operand number that
/// corresponds to it.
unsigned getArgOperandNo(const Use *U) const {
  assert(isArgOperand(U) && "Arg operand # out of range!")(static_cast <bool> (isArgOperand(U) && "Arg operand # out of range!"
) ? void (0) : __assert_fail ("isArgOperand(U) && \"Arg operand # out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1373, __extension__ __PRETTY_FUNCTION__));
  return U - arg_begin();
}

/// Given a value use iterator, return the arg operand number corresponding to
/// it. Iterator must actually correspond to a data operand.
unsigned getArgOperandNo(Value::const_user_iterator UI) const {
  return getArgOperandNo(&UI.getUse());
}

/// Returns true if this CallSite passes the given Value* as an argument to
/// the called function.
bool hasArgument(const Value *V) const {
  return llvm::is_contained(args(), V);
}

Value *getCalledOperand() const { return Op<CalledOperandOpEndIdx>(); }

const Use &getCalledOperandUse() const { return Op<CalledOperandOpEndIdx>(); }
Use &getCalledOperandUse() { return Op<CalledOperandOpEndIdx>(); }

/// Returns the function called, or null if this is an
/// indirect function invocation.
Function *getCalledFunction() const {
  return dyn_cast_or_null<Function>(getCalledOperand());
}

/// Return true if the callsite is an indirect call.
bool isIndirectCall() const;

/// Determine whether the passed iterator points to the callee operand's Use.
bool isCallee(Value::const_user_iterator UI) const {
  return isCallee(&UI.getUse());
}

/// Determine whether this Use is the callee operand's Use.
bool isCallee(const Use *U) const { return &getCalledOperandUse() == U; }

/// Helper to get the caller (the parent function).
Function *getCaller();
const Function *getCaller() const {
  return const_cast<CallBase *>(this)->getCaller();
}

/// Tests if this call site must be tail call optimized. Only a CallInst can
/// be tail call optimized.
bool isMustTailCall() const;

/// Tests if this call site is marked as a tail call.
bool isTailCall() const;

/// Returns the intrinsic ID of the intrinsic called or
/// Intrinsic::not_intrinsic if the called function is not an intrinsic, or if
/// this is an indirect call.
Intrinsic::ID getIntrinsicID() const;

void setCalledOperand(Value *V) { Op<CalledOperandOpEndIdx>() = V; }

/// Sets the function called, including updating the function type.
void setCalledFunction(Function *Fn) {
  setCalledFunction(Fn->getFunctionType(), Fn);
}

/// Sets the function called, including updating the function type.
void setCalledFunction(FunctionCallee Fn) {
  setCalledFunction(Fn.getFunctionType(), Fn.getCallee());
}

/// Sets the function called, including updating to the specified function
/// type.
void setCalledFunction(FunctionType *FTy, Value *Fn) {
  this->FTy = FTy;
  assert(FTy == cast<FunctionType>((static_cast <bool> (FTy == cast<FunctionType>( cast
<PointerType>(Fn->getType())->getElementType())) ?
 void (0) : __assert_fail ("FTy == cast<FunctionType>( cast<PointerType>(Fn->getType())->getElementType())"
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1446, __extension__ __PRETTY_FUNCTION__))
                    cast<PointerType>(Fn->getType())->getElementType()))(static_cast <bool> (FTy == cast<FunctionType>( cast
<PointerType>(Fn->getType())->getElementType())) ?
 void (0) : __assert_fail ("FTy == cast<FunctionType>( cast<PointerType>(Fn->getType())->getElementType())"
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1446, __extension__ __PRETTY_FUNCTION__));
  // This function doesn't mutate the return type, only the function
  // type. Seems broken, but I'm just gonna stick an assert in for now.
  assert(getType() == FTy->getReturnType())(static_cast <bool> (getType() == FTy->getReturnType
()) ? void (0) : __assert_fail ("getType() == FTy->getReturnType()"
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1449, __extension__ __PRETTY_FUNCTION__));
  setCalledOperand(Fn);
}

CallingConv::ID getCallingConv() const {
  return getSubclassData<CallingConvField>();
}

void setCallingConv(CallingConv::ID CC) {
  setSubclassData<CallingConvField>(CC);
}

/// Check if this call is an inline asm statement.
bool isInlineAsm() const { return isa<InlineAsm>(getCalledOperand()); }

/// \name Attribute API
///
/// These methods access and modify attributes on this call (including
/// looking through to the attributes on the called function when necessary).
///@{

/// Return the parameter attributes for this call.
///
AttributeList getAttributes() const { return Attrs; }

/// Set the parameter attributes for this call.
///
void setAttributes(AttributeList A) { Attrs = A; }

/// Determine whether this call has the given attribute. If it does not
/// then determine if the called function has the attribute, but only if
/// the attribute is allowed for the call.
bool hasFnAttr(Attribute::AttrKind Kind) const {
  assert(Kind != Attribute::NoBuiltin &&(static_cast <bool> (Kind != Attribute::NoBuiltin &&
 "Use CallBase::isNoBuiltin() to check for Attribute::NoBuiltin"
) ? void (0) : __assert_fail ("Kind != Attribute::NoBuiltin && \"Use CallBase::isNoBuiltin() to check for Attribute::NoBuiltin\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1483, __extension__ __PRETTY_FUNCTION__))
23
←
'?' condition is true→
         "Use CallBase::isNoBuiltin() to check for Attribute::NoBuiltin")(static_cast <bool> (Kind != Attribute::NoBuiltin &&
 "Use CallBase::isNoBuiltin() to check for Attribute::NoBuiltin"
) ? void (0) : __assert_fail ("Kind != Attribute::NoBuiltin && \"Use CallBase::isNoBuiltin() to check for Attribute::NoBuiltin\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1483, __extension__ __PRETTY_FUNCTION__));
  return hasFnAttrImpl(Kind);
24
←
Calling 'CallBase::hasFnAttrImpl'→
33
←
Returning from 'CallBase::hasFnAttrImpl'→
34
←
Returning value, which participates in a condition later→
}

/// Determine whether this call has the given attribute. If it does not
/// then determine if the called function has the attribute, but only if
/// the attribute is allowed for the call.
bool hasFnAttr(StringRef Kind) const { return hasFnAttrImpl(Kind); }

/// adds the attribute to the list of attributes.
void addAttribute(unsigned i, Attribute::AttrKind Kind) {
  AttributeList PAL = getAttributes();
  PAL = PAL.addAttribute(getContext(), i, Kind);
  setAttributes(PAL);
}

/// adds the attribute to the list of attributes.
void addAttribute(unsigned i, Attribute Attr) {
  AttributeList PAL = getAttributes();
  PAL = PAL.addAttribute(getContext(), i, Attr);
  setAttributes(PAL);
}

/// Adds the attribute to the indicated argument
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind) {
  assert(ArgNo < getNumArgOperands() && "Out of bounds")(static_cast <bool> (ArgNo < getNumArgOperands() &&
 "Out of bounds") ? void (0) : __assert_fail ("ArgNo < getNumArgOperands() && \"Out of bounds\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1508, __extension__ __PRETTY_FUNCTION__));
  AttributeList PAL = getAttributes();
  PAL = PAL.addParamAttribute(getContext(), ArgNo, Kind);
  setAttributes(PAL);
}

/// Adds the attribute to the indicated argument
void addParamAttr(unsigned ArgNo, Attribute Attr) {
  assert(ArgNo < getNumArgOperands() && "Out of bounds")(static_cast <bool> (ArgNo < getNumArgOperands() &&
 "Out of bounds") ? void (0) : __assert_fail ("ArgNo < getNumArgOperands() && \"Out of bounds\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1516, __extension__ __PRETTY_FUNCTION__));
  AttributeList PAL = getAttributes();
  PAL = PAL.addParamAttribute(getContext(), ArgNo, Attr);
  setAttributes(PAL);
}

/// removes the attribute from the list of attributes.
void removeAttribute(unsigned i, Attribute::AttrKind Kind) {
  AttributeList PAL = getAttributes();
  PAL = PAL.removeAttribute(getContext(), i, Kind);
  setAttributes(PAL);
}

/// removes the attribute from the list of attributes.
void removeAttribute(unsigned i, StringRef Kind) {
  AttributeList PAL = getAttributes();
  PAL = PAL.removeAttribute(getContext(), i, Kind);
  setAttributes(PAL);
}

void removeAttributes(unsigned i, const AttrBuilder &Attrs) {
  AttributeList PAL = getAttributes();
  PAL = PAL.removeAttributes(getContext(), i, Attrs);
  setAttributes(PAL);
}

/// Removes the attribute from the given argument
void removeParamAttr(unsigned ArgNo, Attribute::AttrKind Kind) {
  assert(ArgNo < getNumArgOperands() && "Out of bounds")(static_cast <bool> (ArgNo < getNumArgOperands() &&
 "Out of bounds") ? void (0) : __assert_fail ("ArgNo < getNumArgOperands() && \"Out of bounds\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1544, __extension__ __PRETTY_FUNCTION__));
  AttributeList PAL = getAttributes();
  PAL = PAL.removeParamAttribute(getContext(), ArgNo, Kind);
  setAttributes(PAL);
}

/// Removes the attribute from the given argument
void removeParamAttr(unsigned ArgNo, StringRef Kind) {
  assert(ArgNo < getNumArgOperands() && "Out of bounds")(static_cast <bool> (ArgNo < getNumArgOperands() &&
 "Out of bounds") ? void (0) : __assert_fail ("ArgNo < getNumArgOperands() && \"Out of bounds\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1552, __extension__ __PRETTY_FUNCTION__));
  AttributeList PAL = getAttributes();
  PAL = PAL.removeParamAttribute(getContext(), ArgNo, Kind);
  setAttributes(PAL);
}

/// Removes noundef and other attributes that imply undefined behavior if a
/// `undef` or `poison` value is passed from the given argument.
void removeParamUndefImplyingAttrs(unsigned ArgNo) {
  assert(ArgNo < getNumArgOperands() && "Out of bounds")(static_cast <bool> (ArgNo < getNumArgOperands() &&
 "Out of bounds") ? void (0) : __assert_fail ("ArgNo < getNumArgOperands() && \"Out of bounds\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1561, __extension__ __PRETTY_FUNCTION__));
  AttributeList PAL = getAttributes();
  PAL = PAL.removeParamUndefImplyingAttributes(getContext(), ArgNo);
  setAttributes(PAL);
}

/// adds the dereferenceable attribute to the list of attributes.
void addDereferenceableAttr(unsigned i, uint64_t Bytes) {
  AttributeList PAL = getAttributes();
  PAL = PAL.addDereferenceableAttr(getContext(), i, Bytes);
  setAttributes(PAL);
}

/// adds the dereferenceable_or_null attribute to the list of
/// attributes.
void addDereferenceableOrNullAttr(unsigned i, uint64_t Bytes) {
  AttributeList PAL = getAttributes();
  PAL = PAL.addDereferenceableOrNullAttr(getContext(), i, Bytes);
  setAttributes(PAL);
}

/// Determine whether the return value has the given attribute.
bool hasRetAttr(Attribute::AttrKind Kind) const {
  return hasRetAttrImpl(Kind);
}
/// Determine whether the return value has the given attribute.
bool hasRetAttr(StringRef Kind) const { return hasRetAttrImpl(Kind); }

/// Determine whether the argument or parameter has the given attribute.
bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const;

/// Get the attribute of a given kind at a position.
Attribute getAttribute(unsigned i, Attribute::AttrKind Kind) const {
  return getAttributes().getAttribute(i, Kind);
}

/// Get the attribute of a given kind at a position.
Attribute getAttribute(unsigned i, StringRef Kind) const {
  return getAttributes().getAttribute(i, Kind);
}

/// Get the attribute of a given kind from a given arg
Attribute getParamAttr(unsigned ArgNo, Attribute::AttrKind Kind) const {
  assert(ArgNo < getNumArgOperands() && "Out of bounds")(static_cast <bool> (ArgNo < getNumArgOperands() &&
 "Out of bounds") ? void (0) : __assert_fail ("ArgNo < getNumArgOperands() && \"Out of bounds\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1604, __extension__ __PRETTY_FUNCTION__));
  return getAttributes().getParamAttr(ArgNo, Kind);
}

/// Get the attribute of a given kind from a given arg
Attribute getParamAttr(unsigned ArgNo, StringRef Kind) const {
  assert(ArgNo < getNumArgOperands() && "Out of bounds")(static_cast <bool> (ArgNo < getNumArgOperands() &&
 "Out of bounds") ? void (0) : __assert_fail ("ArgNo < getNumArgOperands() && \"Out of bounds\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1610, __extension__ __PRETTY_FUNCTION__));
  return getAttributes().getParamAttr(ArgNo, Kind);
}

/// Return true if the data operand at index \p i has the attribute \p
/// A.
///
/// Data operands include call arguments and values used in operand bundles,
/// but does not include the callee operand.  This routine dispatches to the
/// underlying AttributeList or the OperandBundleUser as appropriate.
///
/// The index \p i is interpreted as
///
///  \p i == Attribute::ReturnIndex  -> the return value
///  \p i in [1, arg_size + 1)  -> argument number (\p i - 1)
///  \p i in [arg_size + 1, data_operand_size + 1) -> bundle operand at index
///     (\p i - 1) in the operand list.
bool dataOperandHasImpliedAttr(unsigned i, Attribute::AttrKind Kind) const {
  // Note that we have to add one because `i` isn't zero-indexed.
  assert(i < (getNumArgOperands() + getNumTotalBundleOperands() + 1) &&(static_cast <bool> (i < (getNumArgOperands() + getNumTotalBundleOperands
() + 1) && "Data operand index out of bounds!") ? void
 (0) : __assert_fail ("i < (getNumArgOperands() + getNumTotalBundleOperands() + 1) && \"Data operand index out of bounds!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1630, __extension__ __PRETTY_FUNCTION__))
         "Data operand index out of bounds!")(static_cast <bool> (i < (getNumArgOperands() + getNumTotalBundleOperands
() + 1) && "Data operand index out of bounds!") ? void
 (0) : __assert_fail ("i < (getNumArgOperands() + getNumTotalBundleOperands() + 1) && \"Data operand index out of bounds!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1630, __extension__ __PRETTY_FUNCTION__));

  // The attribute A can either be directly specified, if the operand in
  // question is a call argument; or be indirectly implied by the kind of its
  // containing operand bundle, if the operand is a bundle operand.

  if (i == AttributeList::ReturnIndex)
    return hasRetAttr(Kind);

  // FIXME: Avoid these i - 1 calculations and update the API to use
  // zero-based indices.
  if (i < (getNumArgOperands() + 1))
    return paramHasAttr(i - 1, Kind);

  assert(hasOperandBundles() && i >= (getBundleOperandsStartIndex() + 1) &&(static_cast <bool> (hasOperandBundles() && i >=
 (getBundleOperandsStartIndex() + 1) && "Must be either a call argument or an operand bundle!"
) ? void (0) : __assert_fail ("hasOperandBundles() && i >= (getBundleOperandsStartIndex() + 1) && \"Must be either a call argument or an operand bundle!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1645, __extension__ __PRETTY_FUNCTION__))
         "Must be either a call argument or an operand bundle!")(static_cast <bool> (hasOperandBundles() && i >=
 (getBundleOperandsStartIndex() + 1) && "Must be either a call argument or an operand bundle!"
) ? void (0) : __assert_fail ("hasOperandBundles() && i >= (getBundleOperandsStartIndex() + 1) && \"Must be either a call argument or an operand bundle!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1645, __extension__ __PRETTY_FUNCTION__));
  return bundleOperandHasAttr(i - 1, Kind);
}

/// Determine whether this data operand is not captured.
// FIXME: Once this API is no longer duplicated in `CallSite`, rename this to
// better indicate that this may return a conservative answer.
bool doesNotCapture(unsigned OpNo) const {
  return dataOperandHasImpliedAttr(OpNo + 1, Attribute::NoCapture);
}

/// Determine whether this argument is passed by value.
bool isByValArgument(unsigned ArgNo) const {
  return paramHasAttr(ArgNo, Attribute::ByVal);
}

/// Determine whether this argument is passed in an alloca.
bool isInAllocaArgument(unsigned ArgNo) const {
  return paramHasAttr(ArgNo, Attribute::InAlloca);
}

/// Determine whether this argument is passed by value, in an alloca, or is
/// preallocated.
bool isPassPointeeByValueArgument(unsigned ArgNo) const {
  return paramHasAttr(ArgNo, Attribute::ByVal) ||
         paramHasAttr(ArgNo, Attribute::InAlloca) ||
         paramHasAttr(ArgNo, Attribute::Preallocated);
}

/// Determine whether passing undef to this argument is undefined behavior.
/// If passing undef to this argument is UB, passing poison is UB as well
/// because poison is more undefined than undef.
bool isPassingUndefUB(unsigned ArgNo) const {
  return paramHasAttr(ArgNo, Attribute::NoUndef) ||
         // dereferenceable implies noundef.
         paramHasAttr(ArgNo, Attribute::Dereferenceable) ||
         // dereferenceable implies noundef, and null is a well-defined value.
         paramHasAttr(ArgNo, Attribute::DereferenceableOrNull);
}

/// Determine if there are is an inalloca argument. Only the last argument can
/// have the inalloca attribute.
bool hasInAllocaArgument() const {
  return !arg_empty() && paramHasAttr(arg_size() - 1, Attribute::InAlloca);
}

// FIXME: Once this API is no longer duplicated in `CallSite`, rename this to
// better indicate that this may return a conservative answer.
bool doesNotAccessMemory(unsigned OpNo) const {
  return dataOperandHasImpliedAttr(OpNo + 1, Attribute::ReadNone);
}

// FIXME: Once this API is no longer duplicated in `CallSite`, rename this to
// better indicate that this may return a conservative answer.
bool onlyReadsMemory(unsigned OpNo) const {
  return dataOperandHasImpliedAttr(OpNo + 1, Attribute::ReadOnly) ||
         dataOperandHasImpliedAttr(OpNo + 1, Attribute::ReadNone);
}

// FIXME: Once this API is no longer duplicated in `CallSite`, rename this to
// better indicate that this may return a conservative answer.
bool doesNotReadMemory(unsigned OpNo) const {
  return dataOperandHasImpliedAttr(OpNo + 1, Attribute::WriteOnly) ||
         dataOperandHasImpliedAttr(OpNo + 1, Attribute::ReadNone);
}

LLVM_ATTRIBUTE_DEPRECATED(unsigned getRetAlignment() const,[[deprecated("Use getRetAlign() instead")]] unsigned getRetAlignment
() const
                          "Use getRetAlign() instead")[[deprecated("Use getRetAlign() instead")]] unsigned getRetAlignment
() const {
  if (const auto MA = Attrs.getRetAlignment())
    return MA->value();
  return 0;
}

/// Extract the alignment of the return value.
MaybeAlign getRetAlign() const { return Attrs.getRetAlignment(); }

/// Extract the alignment for a call or parameter (0=unknown).
LLVM_ATTRIBUTE_DEPRECATED(unsigned getParamAlignment(unsigned ArgNo) const,[[deprecated("Use getParamAlign() instead")]] unsigned getParamAlignment
(unsigned ArgNo) const
                          "Use getParamAlign() instead")[[deprecated("Use getParamAlign() instead")]] unsigned getParamAlignment
(unsigned ArgNo) const {
  if (const auto MA = Attrs.getParamAlignment(ArgNo))
    return MA->value();
  return 0;
}

/// Extract the alignment for a call or parameter (0=unknown).
MaybeAlign getParamAlign(unsigned ArgNo) const {
  return Attrs.getParamAlignment(ArgNo);
}

MaybeAlign getParamStackAlign(unsigned ArgNo) const {
  return Attrs.getParamStackAlignment(ArgNo);
}

/// Extract the byval type for a call or parameter.
Type *getParamByValType(unsigned ArgNo) const {
  Type *Ty = Attrs.getParamByValType(ArgNo);
  return Ty ? Ty : getArgOperand(ArgNo)->getType()->getPointerElementType();
}

/// Extract the preallocated type for a call or parameter.
Type *getParamPreallocatedType(unsigned ArgNo) const {
  Type *Ty = Attrs.getParamPreallocatedType(ArgNo);
  return Ty ? Ty : getArgOperand(ArgNo)->getType()->getPointerElementType();
}

/// Extract the number of dereferenceable bytes for a call or
/// parameter (0=unknown).
uint64_t getDereferenceableBytes(unsigned i) const {
  return Attrs.getDereferenceableBytes(i);
}

/// Extract the number of dereferenceable_or_null bytes for a call or
/// parameter (0=unknown).
uint64_t getDereferenceableOrNullBytes(unsigned i) const {
  return Attrs.getDereferenceableOrNullBytes(i);
}

/// Return true if the return value is known to be not null.
/// This may be because it has the nonnull attribute, or because at least
/// one byte is dereferenceable and the pointer is in addrspace(0).
bool isReturnNonNull() const;

/// Determine if the return value is marked with NoAlias attribute.
bool returnDoesNotAlias() const {
  return Attrs.hasAttribute(AttributeList::ReturnIndex, Attribute::NoAlias);
}

/// If one of the arguments has the 'returned' attribute, returns its
/// operand value. Otherwise, return nullptr.
Value *getReturnedArgOperand() const;

/// Return true if the call should not be treated as a call to a
/// builtin.
bool isNoBuiltin() const {
  return hasFnAttrImpl(Attribute::NoBuiltin) &&
         !hasFnAttrImpl(Attribute::Builtin);
}

/// Determine if the call requires strict floating point semantics.
bool isStrictFP() const { return hasFnAttr(Attribute::StrictFP); }

/// Return true if the call should not be inlined.
bool isNoInline() const { return hasFnAttr(Attribute::NoInline); }
void setIsNoInline() {
  addAttribute(AttributeList::FunctionIndex, Attribute::NoInline);
}
/// Determine if the call does not access memory.
bool doesNotAccessMemory() const { return hasFnAttr(Attribute::ReadNone); }
void setDoesNotAccessMemory() {
  addAttribute(AttributeList::FunctionIndex, Attribute::ReadNone);
}

/// Determine if the call does not access or only reads memory.
bool onlyReadsMemory() const {
  return doesNotAccessMemory() || hasFnAttr(Attribute::ReadOnly);
}

void setOnlyReadsMemory() {
  addAttribute(AttributeList::FunctionIndex, Attribute::ReadOnly);
}

/// Determine if the call does not access or only writes memory.
bool doesNotReadMemory() const {
  return doesNotAccessMemory() || hasFnAttr(Attribute::WriteOnly);
}
void setDoesNotReadMemory() {
  addAttribute(AttributeList::FunctionIndex, Attribute::WriteOnly);
}

/// Determine if the call can access memmory only using pointers based
/// on its arguments.
bool onlyAccessesArgMemory() const {
  return hasFnAttr(Attribute::ArgMemOnly);
}
void setOnlyAccessesArgMemory() {
  addAttribute(AttributeList::FunctionIndex, Attribute::ArgMemOnly);
}

/// Determine if the function may only access memory that is
/// inaccessible from the IR.
bool onlyAccessesInaccessibleMemory() const {
  return hasFnAttr(Attribute::InaccessibleMemOnly);
}
void setOnlyAccessesInaccessibleMemory() {
  addAttribute(AttributeList::FunctionIndex, Attribute::InaccessibleMemOnly);
}

/// Determine if the function may only access memory that is
/// either inaccessible from the IR or pointed to by its arguments.
bool onlyAccessesInaccessibleMemOrArgMem() const {
  return hasFnAttr(Attribute::InaccessibleMemOrArgMemOnly);
}
void setOnlyAccessesInaccessibleMemOrArgMem() {
  addAttribute(AttributeList::FunctionIndex,
               Attribute::InaccessibleMemOrArgMemOnly);
}
/// Determine if the call cannot return.
bool doesNotReturn() const { return hasFnAttr(Attribute::NoReturn); }
void setDoesNotReturn() {
  addAttribute(AttributeList::FunctionIndex, Attribute::NoReturn);
}

/// Determine if the call should not perform indirect branch tracking.
bool doesNoCfCheck() const { return hasFnAttr(Attribute::NoCfCheck); }

/// Determine if the call cannot unwind.
bool doesNotThrow() const { return hasFnAttr(Attribute::NoUnwind); }
void setDoesNotThrow() {
  addAttribute(AttributeList::FunctionIndex, Attribute::NoUnwind);
}

/// Determine if the invoke cannot be duplicated.
bool cannotDuplicate() const { return hasFnAttr(Attribute::NoDuplicate); }
void setCannotDuplicate() {
  addAttribute(AttributeList::FunctionIndex, Attribute::NoDuplicate);
}

/// Determine if the call cannot be tail merged.
bool cannotMerge() const { return hasFnAttr(Attribute::NoMerge); }
void setCannotMerge() {
  addAttribute(AttributeList::FunctionIndex, Attribute::NoMerge);
}

/// Determine if the invoke is convergent
bool isConvergent() const { return hasFnAttr(Attribute::Convergent); }
22
←
Calling 'CallBase::hasFnAttr'→
35
←
Returning from 'CallBase::hasFnAttr'→
36
←
Returning value, which participates in a condition later→
void setConvergent() {
  addAttribute(AttributeList::FunctionIndex, Attribute::Convergent);
}
void setNotConvergent() {
  removeAttribute(AttributeList::FunctionIndex, Attribute::Convergent);
}

/// Determine if the call returns a structure through first
/// pointer argument.
bool hasStructRetAttr() const {
  if (getNumArgOperands() == 0)
    return false;

  // Be friendly and also check the callee.
  return paramHasAttr(0, Attribute::StructRet);
}

/// Determine if any call argument is an aggregate passed by value.
bool hasByValArgument() const {
  return Attrs.hasAttrSomewhere(Attribute::ByVal);
}

///@{
// End of attribute API.

/// \name Operand Bundle API
///
/// This group of methods provides the API to access and manipulate operand
/// bundles on this call.
/// @{

/// Return the number of operand bundles associated with this User.
unsigned getNumOperandBundles() const {
  return std::distance(bundle_op_info_begin(), bundle_op_info_end());
}

/// Return true if this User has any operand bundles.
bool hasOperandBundles() const { return getNumOperandBundles() != 0; }

/// Return the index of the first bundle operand in the Use array.
unsigned getBundleOperandsStartIndex() const {
  assert(hasOperandBundles() && "Don't call otherwise!")(static_cast <bool> (hasOperandBundles() && "Don't call otherwise!"
) ? void (0) : __assert_fail ("hasOperandBundles() && \"Don't call otherwise!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1911, __extension__ __PRETTY_FUNCTION__));
  return bundle_op_info_begin()->Begin;
}

/// Return the index of the last bundle operand in the Use array.
unsigned getBundleOperandsEndIndex() const {
  assert(hasOperandBundles() && "Don't call otherwise!")(static_cast <bool> (hasOperandBundles() && "Don't call otherwise!"
) ? void (0) : __assert_fail ("hasOperandBundles() && \"Don't call otherwise!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1917, __extension__ __PRETTY_FUNCTION__));
  return bundle_op_info_end()[-1].End;
}

/// Return true if the operand at index \p Idx is a bundle operand.
bool isBundleOperand(unsigned Idx) const {
  return hasOperandBundles() && Idx >= getBundleOperandsStartIndex() &&
         Idx < getBundleOperandsEndIndex();
}

/// Returns true if the use is a bundle operand.
bool isBundleOperand(const Use *U) const {
  assert(this == U->getUser() &&(static_cast <bool> (this == U->getUser() &&
 "Only valid to query with a use of this instruction!") ? void
 (0) : __assert_fail ("this == U->getUser() && \"Only valid to query with a use of this instruction!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1930, __extension__ __PRETTY_FUNCTION__))
         "Only valid to query with a use of this instruction!")(static_cast <bool> (this == U->getUser() &&
 "Only valid to query with a use of this instruction!") ? void
 (0) : __assert_fail ("this == U->getUser() && \"Only valid to query with a use of this instruction!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1930, __extension__ __PRETTY_FUNCTION__));
  return hasOperandBundles() && isBundleOperand(U - op_begin());
}
bool isBundleOperand(Value::const_user_iterator UI) const {
  return isBundleOperand(&UI.getUse());
}

/// Return the total number operands (not operand bundles) used by
/// every operand bundle in this OperandBundleUser.
unsigned getNumTotalBundleOperands() const {
  if (!hasOperandBundles())
    return 0;

  unsigned Begin = getBundleOperandsStartIndex();
  unsigned End = getBundleOperandsEndIndex();

  assert(Begin <= End && "Should be!")(static_cast <bool> (Begin <= End && "Should be!"
) ? void (0) : __assert_fail ("Begin <= End && \"Should be!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1946, __extension__ __PRETTY_FUNCTION__));
  return End - Begin;
}

/// Return the operand bundle at a specific index.
OperandBundleUse getOperandBundleAt(unsigned Index) const {
  assert(Index < getNumOperandBundles() && "Index out of bounds!")(static_cast <bool> (Index < getNumOperandBundles() &&
 "Index out of bounds!") ? void (0) : __assert_fail ("Index < getNumOperandBundles() && \"Index out of bounds!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1952, __extension__ __PRETTY_FUNCTION__));
  return operandBundleFromBundleOpInfo(*(bundle_op_info_begin() + Index));
}

/// Return the number of operand bundles with the tag Name attached to
/// this instruction.
unsigned countOperandBundlesOfType(StringRef Name) const {
  unsigned Count = 0;
  for (unsigned i = 0, e = getNumOperandBundles(); i != e; ++i)
    if (getOperandBundleAt(i).getTagName() == Name)
      Count++;

  return Count;
}

/// Return the number of operand bundles with the tag ID attached to
/// this instruction.
unsigned countOperandBundlesOfType(uint32_t ID) const {
  unsigned Count = 0;
  for (unsigned i = 0, e = getNumOperandBundles(); i != e; ++i)
    if (getOperandBundleAt(i).getTagID() == ID)
      Count++;

  return Count;
}

/// Return an operand bundle by name, if present.
///
/// It is an error to call this for operand bundle types that may have
/// multiple instances of them on the same instruction.
Optional<OperandBundleUse> getOperandBundle(StringRef Name) const {
  assert(countOperandBundlesOfType(Name) < 2 && "Precondition violated!")(static_cast <bool> (countOperandBundlesOfType(Name) <
&& "Precondition violated!") ? void (0) : __assert_fail
 ("countOperandBundlesOfType(Name) < 2 && \"Precondition violated!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1983, __extension__ __PRETTY_FUNCTION__));

  for (unsigned i = 0, e = getNumOperandBundles(); i != e; ++i) {
    OperandBundleUse U = getOperandBundleAt(i);
    if (U.getTagName() == Name)
      return U;
  }

  return None;
}

/// Return an operand bundle by tag ID, if present.
///
/// It is an error to call this for operand bundle types that may have
/// multiple instances of them on the same instruction.
Optional<OperandBundleUse> getOperandBundle(uint32_t ID) const {
  assert(countOperandBundlesOfType(ID) < 2 && "Precondition violated!")(static_cast <bool> (countOperandBundlesOfType(ID) <
&& "Precondition violated!") ? void (0) : __assert_fail
 ("countOperandBundlesOfType(ID) < 2 && \"Precondition violated!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 1999, __extension__ __PRETTY_FUNCTION__));

  for (unsigned i = 0, e = getNumOperandBundles(); i != e; ++i) {
    OperandBundleUse U = getOperandBundleAt(i);
    if (U.getTagID() == ID)
      return U;
  }

  return None;
}

/// Return the list of operand bundles attached to this instruction as
/// a vector of OperandBundleDefs.
///
/// This function copies the OperandBundeUse instances associated with this
/// OperandBundleUser to a vector of OperandBundleDefs.  Note:
/// OperandBundeUses and OperandBundleDefs are non-trivially *different*
/// representations of operand bundles (see documentation above).
void getOperandBundlesAsDefs(SmallVectorImpl<OperandBundleDef> &Defs) const;

/// Return the operand bundle for the operand at index OpIdx.
///
/// It is an error to call this with an OpIdx that does not correspond to an
/// bundle operand.
OperandBundleUse getOperandBundleForOperand(unsigned OpIdx) const {
  return operandBundleFromBundleOpInfo(getBundleOpInfoForOperand(OpIdx));
}

/// Return true if this operand bundle user has operand bundles that
/// may read from the heap.
bool hasReadingOperandBundles() const;

/// Return true if this operand bundle user has operand bundles that
/// may write to the heap.
bool hasClobberingOperandBundles() const {
  for (auto &BOI : bundle_op_infos()) {
    if (BOI.Tag->second == LLVMContext::OB_deopt ||
        BOI.Tag->second == LLVMContext::OB_funclet)
      continue;

    // This instruction has an operand bundle that is not known to us.
    // Assume the worst.
    return true;
  }

  return false;
}

/// Return true if the bundle operand at index \p OpIdx has the
/// attribute \p A.
bool bundleOperandHasAttr(unsigned OpIdx,  Attribute::AttrKind A) const {
  auto &BOI = getBundleOpInfoForOperand(OpIdx);
  auto OBU = operandBundleFromBundleOpInfo(BOI);
  return OBU.operandHasAttr(OpIdx - BOI.Begin, A);
}

/// Return true if \p Other has the same sequence of operand bundle
/// tags with the same number of operands on each one of them as this
/// OperandBundleUser.
bool hasIdenticalOperandBundleSchema(const CallBase &Other) const {
  if (getNumOperandBundles() != Other.getNumOperandBundles())
    return false;

  return std::equal(bundle_op_info_begin(), bundle_op_info_end(),
                    Other.bundle_op_info_begin());
}

/// Return true if this operand bundle user contains operand bundles
/// with tags other than those specified in \p IDs.
bool hasOperandBundlesOtherThan(ArrayRef<uint32_t> IDs) const {
  for (unsigned i = 0, e = getNumOperandBundles(); i != e; ++i) {
    uint32_t ID = getOperandBundleAt(i).getTagID();
    if (!is_contained(IDs, ID))
      return true;
  }
  return false;
}

/// Is the function attribute S disallowed by some operand bundle on
/// this operand bundle user?
bool isFnAttrDisallowedByOpBundle(StringRef S) const {
  // Operand bundles only possibly disallow readnone, readonly and argmemonly
  // attributes.  All String attributes are fine.
  return false;
}

/// Is the function attribute A disallowed by some operand bundle on
/// this operand bundle user?
bool isFnAttrDisallowedByOpBundle(Attribute::AttrKind A) const {
  switch (A) {
28
←
Control jumps to the 'default' case at line 2089→
  default:
    return false;
29
←
Returning zero, which participates in a condition later→

  case Attribute::InaccessibleMemOrArgMemOnly:
    return hasReadingOperandBundles();

  case Attribute::InaccessibleMemOnly:
    return hasReadingOperandBundles();

  case Attribute::ArgMemOnly:
    return hasReadingOperandBundles();

  case Attribute::ReadNone:
    return hasReadingOperandBundles();

  case Attribute::ReadOnly:
    return hasClobberingOperandBundles();
  }

  llvm_unreachable("switch has a default case!")::llvm::llvm_unreachable_internal("switch has a default case!"
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 2108);
}

/// Used to keep track of an operand bundle.  See the main comment on
/// OperandBundleUser above.
struct BundleOpInfo {
  /// The operand bundle tag, interned by
  /// LLVMContextImpl::getOrInsertBundleTag.
  StringMapEntry<uint32_t> *Tag;

  /// The index in the Use& vector where operands for this operand
  /// bundle starts.
  uint32_t Begin;

  /// The index in the Use& vector where operands for this operand
  /// bundle ends.
  uint32_t End;

  bool operator==(const BundleOpInfo &Other) const {
    return Tag == Other.Tag && Begin == Other.Begin && End == Other.End;
  }
};

/// Simple helper function to map a BundleOpInfo to an
/// OperandBundleUse.
OperandBundleUse
operandBundleFromBundleOpInfo(const BundleOpInfo &BOI) const {
  auto begin = op_begin();
  ArrayRef<Use> Inputs(begin + BOI.Begin, begin + BOI.End);
  return OperandBundleUse(BOI.Tag, Inputs);
}

using bundle_op_iterator = BundleOpInfo *;
using const_bundle_op_iterator = const BundleOpInfo *;

/// Return the start of the list of BundleOpInfo instances associated
/// with this OperandBundleUser.
///
/// OperandBundleUser uses the descriptor area co-allocated with the host User
/// to store some meta information about which operands are "normal" operands,
/// and which ones belong to some operand bundle.
///
/// The layout of an operand bundle user is
///
///          +-----------uint32_t End-------------------------------------+
///          |                                                            |
///          |  +--------uint32_t Begin--------------------+              |
///          |  |                                          |              |
///          ^  ^                                          v              v
///  |------|------|----|----|----|----|----|---------|----|---------|----|-----
///  | BOI0 | BOI1 | .. | DU | U0 | U1 | .. | BOI0_U0 | .. | BOI1_U0 | .. | Un
///  |------|------|----|----|----|----|----|---------|----|---------|----|-----
///   v  v                                  ^              ^
///   |  |                                  |              |
///   |  +--------uint32_t Begin------------+              |
///   |                                                    |
///   +-----------uint32_t End-----------------------------+
///
///
/// BOI0, BOI1 ... are descriptions of operand bundles in this User's use
/// list. These descriptions are installed and managed by this class, and
/// they're all instances of OperandBundleUser<T>::BundleOpInfo.
///
/// DU is an additional descriptor installed by User's 'operator new' to keep
/// track of the 'BOI0 ... BOIN' co-allocation.  OperandBundleUser does not
/// access or modify DU in any way, it's an implementation detail private to
/// User.
///
/// The regular Use& vector for the User starts at U0.  The operand bundle
/// uses are part of the Use& vector, just like normal uses.  In the diagram
/// above, the operand bundle uses start at BOI0_U0.  Each instance of
/// BundleOpInfo has information about a contiguous set of uses constituting
/// an operand bundle, and the total set of operand bundle uses themselves
/// form a contiguous set of uses (i.e. there are no gaps between uses
/// corresponding to individual operand bundles).
///
/// This class does not know the location of the set of operand bundle uses
/// within the use list -- that is decided by the User using this class via
/// the BeginIdx argument in populateBundleOperandInfos.
///
/// Currently operand bundle users with hung-off operands are not supported.
bundle_op_iterator bundle_op_info_begin() {
  if (!hasDescriptor())
    return nullptr;

  uint8_t *BytesBegin = getDescriptor().begin();
  return reinterpret_cast<bundle_op_iterator>(BytesBegin);
}

/// Return the start of the list of BundleOpInfo instances associated
/// with this OperandBundleUser.
const_bundle_op_iterator bundle_op_info_begin() const {
  auto *NonConstThis = const_cast<CallBase *>(this);
  return NonConstThis->bundle_op_info_begin();
}

/// Return the end of the list of BundleOpInfo instances associated
/// with this OperandBundleUser.
bundle_op_iterator bundle_op_info_end() {
  if (!hasDescriptor())
    return nullptr;

  uint8_t *BytesEnd = getDescriptor().end();
  return reinterpret_cast<bundle_op_iterator>(BytesEnd);
}

/// Return the end of the list of BundleOpInfo instances associated
/// with this OperandBundleUser.
const_bundle_op_iterator bundle_op_info_end() const {
  auto *NonConstThis = const_cast<CallBase *>(this);
  return NonConstThis->bundle_op_info_end();
}

/// Return the range [\p bundle_op_info_begin, \p bundle_op_info_end).
iterator_range<bundle_op_iterator> bundle_op_infos() {
  return make_range(bundle_op_info_begin(), bundle_op_info_end());
}

/// Return the range [\p bundle_op_info_begin, \p bundle_op_info_end).
iterator_range<const_bundle_op_iterator> bundle_op_infos() const {
  return make_range(bundle_op_info_begin(), bundle_op_info_end());
}

/// Populate the BundleOpInfo instances and the Use& vector from \p
/// Bundles.  Return the op_iterator pointing to the Use& one past the last
/// last bundle operand use.
///
/// Each \p OperandBundleDef instance is tracked by a OperandBundleInfo
/// instance allocated in this User's descriptor.
op_iterator populateBundleOperandInfos(ArrayRef<OperandBundleDef> Bundles,
                                       const unsigned BeginIndex);

2240public:
/// Return the BundleOpInfo for the operand at index OpIdx.
///
/// It is an error to call this with an OpIdx that does not correspond to an
/// bundle operand.
BundleOpInfo &getBundleOpInfoForOperand(unsigned OpIdx);
const BundleOpInfo &getBundleOpInfoForOperand(unsigned OpIdx) const {
  return const_cast<CallBase *>(this)->getBundleOpInfoForOperand(OpIdx);
}

2250protected:
/// Return the total number of values used in \p Bundles.
static unsigned CountBundleInputs(ArrayRef<OperandBundleDef> Bundles) {
  unsigned Total = 0;
  for (auto &B : Bundles)
    Total += B.input_size();
  return Total;
}

/// @}
// End of operand bundle API.

2262private:
bool hasFnAttrOnCalledFunction(Attribute::AttrKind Kind) const;
bool hasFnAttrOnCalledFunction(StringRef Kind) const;

template <typename AttrKind> bool hasFnAttrImpl(AttrKind Kind) const {
  if (Attrs.hasFnAttribute(Kind))
25
←
Assuming the condition is false→
26
←
Taking false branch→
    return true;

  // Operand bundles override attributes on the called function, but don't
  // override attributes directly present on the call instruction.
  if (isFnAttrDisallowedByOpBundle(Kind))
27
←
Calling 'CallBase::isFnAttrDisallowedByOpBundle'→
30
←
Returning from 'CallBase::isFnAttrDisallowedByOpBundle'→
31
←
Taking false branch→
    return false;

  return hasFnAttrOnCalledFunction(Kind);
32
←
Returning value, which participates in a condition later→
}

/// Determine whether the return value has the given attribute. Supports
/// Attribute::AttrKind and StringRef as \p AttrKind types.
template <typename AttrKind> bool hasRetAttrImpl(AttrKind Kind) const {
  if (Attrs.hasAttribute(AttributeList::ReturnIndex, Kind))
    return true;

  // Look at the callee, if available.
  if (const Function *F = getCalledFunction())
    return F->getAttributes().hasAttribute(AttributeList::ReturnIndex, Kind);
  return false;
}
2289};

2291template <>
2292struct OperandTraits<CallBase> : public VariadicOperandTraits<CallBase, 1> {};

2294DEFINE_TRANSPARENT_OPERAND_ACCESSORS(CallBase, Value)CallBase::op_iterator CallBase::op_begin() { return OperandTraits
<CallBase>::op_begin(this); } CallBase::const_op_iterator
 CallBase::op_begin() const { return OperandTraits<CallBase
>::op_begin(const_cast<CallBase*>(this)); } CallBase
::op_iterator CallBase::op_end() { return OperandTraits<CallBase
>::op_end(this); } CallBase::const_op_iterator CallBase::op_end
() const { return OperandTraits<CallBase>::op_end(const_cast
<CallBase*>(this)); } Value *CallBase::getOperand(unsigned
 i_nocapture) const { (static_cast <bool> (i_nocapture <
 OperandTraits<CallBase>::operands(this) && "getOperand() out of range!"
) ? void (0) : __assert_fail ("i_nocapture < OperandTraits<CallBase>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 2294, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<CallBase>::op_begin(const_cast
<CallBase*>(this))[i_nocapture].get()); } void CallBase
::setOperand(unsigned i_nocapture, Value *Val_nocapture) { (static_cast
 <bool> (i_nocapture < OperandTraits<CallBase>
::operands(this) && "setOperand() out of range!") ? void
 (0) : __assert_fail ("i_nocapture < OperandTraits<CallBase>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 2294, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
CallBase>::op_begin(this)[i_nocapture] = Val_nocapture; } unsigned
 CallBase::getNumOperands() const { return OperandTraits<CallBase
>::operands(this); } template <int Idx_nocapture> Use
 &CallBase::Op() { return this->OpFrom<Idx_nocapture
>(this); } template <int Idx_nocapture> const Use &
CallBase::Op() const { return this->OpFrom<Idx_nocapture
>(this); }

2296//===----------------------------------------------------------------------===//
2297//                           FuncletPadInst Class
2298//===----------------------------------------------------------------------===//
2299class FuncletPadInst : public Instruction {
2300private:
FuncletPadInst(const FuncletPadInst &CPI);

explicit FuncletPadInst(Instruction::FuncletPadOps Op, Value *ParentPad,
                        ArrayRef<Value *> Args, unsigned Values,
                        const Twine &NameStr, Instruction *InsertBefore);
explicit FuncletPadInst(Instruction::FuncletPadOps Op, Value *ParentPad,
                        ArrayRef<Value *> Args, unsigned Values,
                        const Twine &NameStr, BasicBlock *InsertAtEnd);

void init(Value *ParentPad, ArrayRef<Value *> Args, const Twine &NameStr);

2312protected:
// Note: Instruction needs to be a friend here to call cloneImpl.
friend class Instruction;
friend class CatchPadInst;
friend class CleanupPadInst;

FuncletPadInst *cloneImpl() const;

2320public:
/// Provide fast operand accessors
DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
 setOperand(unsigned, Value*); inline op_iterator op_begin();
 inline const_op_iterator op_begin() const; inline op_iterator
 op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
 getNumOperands() const;

/// getNumArgOperands - Return the number of funcletpad arguments.
///
unsigned getNumArgOperands() const { return getNumOperands() - 1; }

/// Convenience accessors

/// Return the outer EH-pad this funclet is nested within.
///
/// Note: This returns the associated CatchSwitchInst if this FuncletPadInst
/// is a CatchPadInst.
Value *getParentPad() const { return Op<-1>(); }
void setParentPad(Value *ParentPad) {
  assert(ParentPad)(static_cast <bool> (ParentPad) ? void (0) : __assert_fail
 ("ParentPad", "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 2336, __extension__ __PRETTY_FUNCTION__));
  Op<-1>() = ParentPad;
}

/// getArgOperand/setArgOperand - Return/set the i-th funcletpad argument.
///
Value *getArgOperand(unsigned i) const { return getOperand(i); }
void setArgOperand(unsigned i, Value *v) { setOperand(i, v); }

/// arg_operands - iteration adapter for range-for loops.
op_range arg_operands() { return op_range(op_begin(), op_end() - 1); }

/// arg_operands - iteration adapter for range-for loops.
const_op_range arg_operands() const {
  return const_op_range(op_begin(), op_end() - 1);
}

// Methods for support type inquiry through isa, cast, and dyn_cast:
static bool classof(const Instruction *I) { return I->isFuncletPad(); }
static bool classof(const Value *V) {
  return isa<Instruction>(V) && classof(cast<Instruction>(V));
}
2358};

2360template <>
2361struct OperandTraits<FuncletPadInst>
  : public VariadicOperandTraits<FuncletPadInst, /*MINARITY=*/1> {};

2364DEFINE_TRANSPARENT_OPERAND_ACCESSORS(FuncletPadInst, Value)FuncletPadInst::op_iterator FuncletPadInst::op_begin() { return
 OperandTraits<FuncletPadInst>::op_begin(this); } FuncletPadInst
::const_op_iterator FuncletPadInst::op_begin() const { return
 OperandTraits<FuncletPadInst>::op_begin(const_cast<
FuncletPadInst*>(this)); } FuncletPadInst::op_iterator FuncletPadInst
::op_end() { return OperandTraits<FuncletPadInst>::op_end
(this); } FuncletPadInst::const_op_iterator FuncletPadInst::op_end
() const { return OperandTraits<FuncletPadInst>::op_end
(const_cast<FuncletPadInst*>(this)); } Value *FuncletPadInst
::getOperand(unsigned i_nocapture) const { (static_cast <bool
> (i_nocapture < OperandTraits<FuncletPadInst>::operands
(this) && "getOperand() out of range!") ? void (0) : __assert_fail
 ("i_nocapture < OperandTraits<FuncletPadInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 2364, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<FuncletPadInst>::op_begin(
const_cast<FuncletPadInst*>(this))[i_nocapture].get());
 } void FuncletPadInst::setOperand(unsigned i_nocapture, Value
 *Val_nocapture) { (static_cast <bool> (i_nocapture <
 OperandTraits<FuncletPadInst>::operands(this) &&
 "setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<FuncletPadInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/InstrTypes.h"
, 2364, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
FuncletPadInst>::op_begin(this)[i_nocapture] = Val_nocapture
; } unsigned FuncletPadInst::getNumOperands() const { return OperandTraits
<FuncletPadInst>::operands(this); } template <int Idx_nocapture
> Use &FuncletPadInst::Op() { return this->OpFrom<
Idx_nocapture>(this); } template <int Idx_nocapture>
 const Use &FuncletPadInst::Op() const { return this->
OpFrom<Idx_nocapture>(this); }

2366} // end namespace llvm

2368#endif // LLVM_IR_INSTRTYPES_H

←

/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/PatternMatch.h

→

1//===- PatternMatch.h - Match on the LLVM IR --------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file provides a simple and efficient mechanism for performing general
10// tree-based pattern matches on the LLVM IR. The power of these routines is
11// that it allows you to write concise patterns that are expressive and easy to
12// understand. The other major advantage of this is that it allows you to
13// trivially capture/bind elements in the pattern to variables. For example,
14// you can do something like this:
15//
16//  Value *Exp = ...
17//  Value *X, *Y;  ConstantInt *C1, *C2;      // (X & C1) | (Y & C2)
18//  if (match(Exp, m_Or(m_And(m_Value(X), m_ConstantInt(C1)),
19//                      m_And(m_Value(Y), m_ConstantInt(C2))))) {
20//    ... Pattern is matched and variables are bound ...
21//  }
22//
23// This is primarily useful to things like the instruction combiner, but can
24// also be useful for static analysis tools or code generators.
25//
26//===----------------------------------------------------------------------===//
27 
28#ifndef LLVM_IR_PATTERNMATCH_H
29#define LLVM_IR_PATTERNMATCH_H
30 
31#include "llvm/ADT/APFloat.h"
32#include "llvm/ADT/APInt.h"
33#include "llvm/IR/Constant.h"
34#include "llvm/IR/Constants.h"
35#include "llvm/IR/DataLayout.h"
36#include "llvm/IR/InstrTypes.h"
37#include "llvm/IR/Instruction.h"
38#include "llvm/IR/Instructions.h"
39#include "llvm/IR/IntrinsicInst.h"
40#include "llvm/IR/Intrinsics.h"
41#include "llvm/IR/Operator.h"
42#include "llvm/IR/Value.h"
43#include "llvm/Support/Casting.h"
44#include <cstdint>
45 
46namespace llvm {
47namespace PatternMatch {
48 
49template <typename Val, typename Pattern> bool match(Val *V, const Pattern &P) {
50  return const_cast<Pattern &>(P).match(V);
41
←
Calling 'IntrinsicID_match::match'→
45
←
Returning from 'IntrinsicID_match::match'→
46
←
Returning zero, which participates in a condition later→
50
←
Calling 'IntrinsicID_match::match'→
54
←
Returning from 'IntrinsicID_match::match'→
55
←
Returning zero, which participates in a condition later→
51}
52 
53template <typename Pattern> bool match(ArrayRef<int> Mask, const Pattern &P) {
54  return const_cast<Pattern &>(P).match(Mask);
55}
56 
57template <typename SubPattern_t> struct OneUse_match {
58  SubPattern_t SubPattern;
59 
60  OneUse_match(const SubPattern_t &SP) : SubPattern(SP) {}
61 
62  template <typename OpTy> bool match(OpTy *V) {
63    return V->hasOneUse() && SubPattern.match(V);
64  }
65};
66 
67template <typename T> inline OneUse_match<T> m_OneUse(const T &SubPattern) {
68  return SubPattern;
69}
70 
71template <typename Class> struct class_match {
72  template <typename ITy> bool match(ITy *V) { return isa<Class>(V); }
73};
74 
75/// Match an arbitrary value and ignore it.
76inline class_match<Value> m_Value() { return class_match<Value>(); }
77 
78/// Match an arbitrary unary operation and ignore it.
79inline class_match<UnaryOperator> m_UnOp() {
80  return class_match<UnaryOperator>();
81}
82 
83/// Match an arbitrary binary operation and ignore it.
84inline class_match<BinaryOperator> m_BinOp() {
85  return class_match<BinaryOperator>();
86}
87 
88/// Matches any compare instruction and ignore it.
89inline class_match<CmpInst> m_Cmp() { return class_match<CmpInst>(); }
90 
91struct undef_match {
92  static bool check(const Value *V) {
93    if (isa<UndefValue>(V))
94      return true;
95 
96    const auto *CA = dyn_cast<ConstantAggregate>(V);
97    if (!CA)
98      return false;
99 
100    SmallPtrSet<const ConstantAggregate *, 8> Seen;
101    SmallVector<const ConstantAggregate *, 8> Worklist;
102 
103    // Either UndefValue, PoisonValue, or an aggregate that only contains
104    // these is accepted by matcher.
105    // CheckValue returns false if CA cannot satisfy this constraint.
106    auto CheckValue = [&](const ConstantAggregate *CA) {
107      for (const Value *Op : CA->operand_values()) {
108        if (isa<UndefValue>(Op))
109          continue;
110 
111        const auto *CA = dyn_cast<ConstantAggregate>(Op);
112        if (!CA)
113          return false;
114        if (Seen.insert(CA).second)
115          Worklist.emplace_back(CA);
116      }
117 
118      return true;
119    };
120 
121    if (!CheckValue(CA))
122      return false;
123 
124    while (!Worklist.empty()) {
125      if (!CheckValue(Worklist.pop_back_val()))
126        return false;
127    }
128    return true;
129  }
130  template <typename ITy> bool match(ITy *V) { return check(V); }
131};
132 
133/// Match an arbitrary undef constant. This matches poison as well.
134/// If this is an aggregate and contains a non-aggregate element that is
135/// neither undef nor poison, the aggregate is not matched.
136inline auto m_Undef() { return undef_match(); }
137 
138/// Match an arbitrary poison constant.
139inline class_match<PoisonValue> m_Poison() { return class_match<PoisonValue>(); }
140 
141/// Match an arbitrary Constant and ignore it.
142inline class_match<Constant> m_Constant() { return class_match<Constant>(); }
143 
144/// Match an arbitrary ConstantInt and ignore it.
145inline class_match<ConstantInt> m_ConstantInt() {
146  return class_match<ConstantInt>();
147}
148 
149/// Match an arbitrary ConstantFP and ignore it.
150inline class_match<ConstantFP> m_ConstantFP() {
151  return class_match<ConstantFP>();
152}
153 
154/// Match an arbitrary ConstantExpr and ignore it.
155inline class_match<ConstantExpr> m_ConstantExpr() {
156  return class_match<ConstantExpr>();
157}
158 
159/// Match an arbitrary basic block value and ignore it.
160inline class_match<BasicBlock> m_BasicBlock() {
161  return class_match<BasicBlock>();
162}
163 
164/// Inverting matcher
165template <typename Ty> struct match_unless {
166  Ty M;
167 
168  match_unless(const Ty &Matcher) : M(Matcher) {}
169 
170  template <typename ITy> bool match(ITy *V) { return !M.match(V); }
171};
172 
173/// Match if the inner matcher does *NOT* match.
174template <typename Ty> inline match_unless<Ty> m_Unless(const Ty &M) {
175  return match_unless<Ty>(M);
176}
177 
178/// Matching combinators
179template <typename LTy, typename RTy> struct match_combine_or {
180  LTy L;
181  RTy R;
182 
183  match_combine_or(const LTy &Left, const RTy &Right) : L(Left), R(Right) {}
184 
185  template <typename ITy> bool match(ITy *V) {
186    if (L.match(V))
187      return true;
188    if (R.match(V))
189      return true;
190    return false;
191  }
192};
193 
194template <typename LTy, typename RTy> struct match_combine_and {
195  LTy L;
196  RTy R;
197 
198  match_combine_and(const LTy &Left, const RTy &Right) : L(Left), R(Right) {}
199 
200  template <typename ITy> bool match(ITy *V) {
201    if (L.match(V))
202      if (R.match(V))
203        return true;
204    return false;
205  }
206};
207 
208/// Combine two pattern matchers matching L || R
209template <typename LTy, typename RTy>
210inline match_combine_or<LTy, RTy> m_CombineOr(const LTy &L, const RTy &R) {
211  return match_combine_or<LTy, RTy>(L, R);
212}
213 
214/// Combine two pattern matchers matching L && R
215template <typename LTy, typename RTy>
216inline match_combine_and<LTy, RTy> m_CombineAnd(const LTy &L, const RTy &R) {
217  return match_combine_and<LTy, RTy>(L, R);
218}
219 
220struct apint_match {
221  const APInt *&Res;
222  bool AllowUndef;
223 
224  apint_match(const APInt *&Res, bool AllowUndef)
225    : Res(Res), AllowUndef(AllowUndef) {}
226 
227  template <typename ITy> bool match(ITy *V) {
228    if (auto *CI = dyn_cast<ConstantInt>(V)) {
229      Res = &CI->getValue();
230      return true;
231    }
232    if (V->getType()->isVectorTy())
233      if (const auto *C = dyn_cast<Constant>(V))
234        if (auto *CI = dyn_cast_or_null<ConstantInt>(
235                C->getSplatValue(AllowUndef))) {
236          Res = &CI->getValue();
237          return true;
238        }
239    return false;
240  }
241};
242// Either constexpr if or renaming ConstantFP::getValueAPF to
243// ConstantFP::getValue is needed to do it via single template
244// function for both apint/apfloat.
245struct apfloat_match {
246  const APFloat *&Res;
247  bool AllowUndef;
248 
249  apfloat_match(const APFloat *&Res, bool AllowUndef)
250      : Res(Res), AllowUndef(AllowUndef) {}
251 
252  template <typename ITy> bool match(ITy *V) {
253    if (auto *CI = dyn_cast<ConstantFP>(V)) {
254      Res = &CI->getValueAPF();
255      return true;
256    }
257    if (V->getType()->isVectorTy())
258      if (const auto *C = dyn_cast<Constant>(V))
259        if (auto *CI = dyn_cast_or_null<ConstantFP>(
260                C->getSplatValue(AllowUndef))) {
261          Res = &CI->getValueAPF();
262          return true;
263        }
264    return false;
265  }
266};
267 
268/// Match a ConstantInt or splatted ConstantVector, binding the
269/// specified pointer to the contained APInt.
270inline apint_match m_APInt(const APInt *&Res) {
271  // Forbid undefs by default to maintain previous behavior.
272  return apint_match(Res, /* AllowUndef */ false);
273}
274 
275/// Match APInt while allowing undefs in splat vector constants.
276inline apint_match m_APIntAllowUndef(const APInt *&Res) {
277  return apint_match(Res, /* AllowUndef */ true);
278}
279 
280/// Match APInt while forbidding undefs in splat vector constants.
281inline apint_match m_APIntForbidUndef(const APInt *&Res) {
282  return apint_match(Res, /* AllowUndef */ false);
283}
284 
285/// Match a ConstantFP or splatted ConstantVector, binding the
286/// specified pointer to the contained APFloat.
287inline apfloat_match m_APFloat(const APFloat *&Res) {
288  // Forbid undefs by default to maintain previous behavior.
289  return apfloat_match(Res, /* AllowUndef */ false);
290}
291 
292/// Match APFloat while allowing undefs in splat vector constants.
293inline apfloat_match m_APFloatAllowUndef(const APFloat *&Res) {
294  return apfloat_match(Res, /* AllowUndef */ true);
295}
296 
297/// Match APFloat while forbidding undefs in splat vector constants.
298inline apfloat_match m_APFloatForbidUndef(const APFloat *&Res) {
299  return apfloat_match(Res, /* AllowUndef */ false);
300}
301 
302template <int64_t Val> struct constantint_match {
303  template <typename ITy> bool match(ITy *V) {
304    if (const auto *CI = dyn_cast<ConstantInt>(V)) {
305      const APInt &CIV = CI->getValue();
306      if (Val >= 0)
307        return CIV == static_cast<uint64_t>(Val);
308      // If Val is negative, and CI is shorter than it, truncate to the right
309      // number of bits.  If it is larger, then we have to sign extend.  Just
310      // compare their negated values.
311      return -CIV == -Val;
312    }
313    return false;
314  }
315};
316 
317/// Match a ConstantInt with a specific value.
318template <int64_t Val> inline constantint_match<Val> m_ConstantInt() {
319  return constantint_match<Val>();
320}
321 
322/// This helper class is used to match constant scalars, vector splats,
323/// and fixed width vectors that satisfy a specified predicate.
324/// For fixed width vector constants, undefined elements are ignored.
325template <typename Predicate, typename ConstantVal>
326struct cstval_pred_ty : public Predicate {
327  template <typename ITy> bool match(ITy *V) {
328    if (const auto *CV = dyn_cast<ConstantVal>(V))
329      return this->isValue(CV->getValue());
330    if (const auto *VTy = dyn_cast<VectorType>(V->getType())) {
331      if (const auto *C = dyn_cast<Constant>(V)) {
332        if (const auto *CV = dyn_cast_or_null<ConstantVal>(C->getSplatValue()))
333          return this->isValue(CV->getValue());
334 
335        // Number of elements of a scalable vector unknown at compile time
336        auto *FVTy = dyn_cast<FixedVectorType>(VTy);
337        if (!FVTy)
338          return false;
339 
340        // Non-splat vector constant: check each element for a match.
341        unsigned NumElts = FVTy->getNumElements();
342        assert(NumElts != 0 && "Constant vector with no elements?")(static_cast <bool> (NumElts != 0 && "Constant vector with no elements?"
) ? void (0) : __assert_fail ("NumElts != 0 && \"Constant vector with no elements?\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/PatternMatch.h"
, 342, __extension__ __PRETTY_FUNCTION__));
343        bool HasNonUndefElements = false;
344        for (unsigned i = 0; i != NumElts; ++i) {
345          Constant *Elt = C->getAggregateElement(i);
346          if (!Elt)
347            return false;
348          if (isa<UndefValue>(Elt))
349            continue;
350          auto *CV = dyn_cast<ConstantVal>(Elt);
351          if (!CV || !this->isValue(CV->getValue()))
352            return false;
353          HasNonUndefElements = true;
354        }
355        return HasNonUndefElements;
356      }
357    }
358    return false;
359  }
360};
361 
362/// specialization of cstval_pred_ty for ConstantInt
363template <typename Predicate>
364using cst_pred_ty = cstval_pred_ty<Predicate, ConstantInt>;
365 
366/// specialization of cstval_pred_ty for ConstantFP
367template <typename Predicate>
368using cstfp_pred_ty = cstval_pred_ty<Predicate, ConstantFP>;
369 
370/// This helper class is used to match scalar and vector constants that
371/// satisfy a specified predicate, and bind them to an APInt.
372template <typename Predicate> struct api_pred_ty : public Predicate {
373  const APInt *&Res;
374 
375  api_pred_ty(const APInt *&R) : Res(R) {}
376 
377  template <typename ITy> bool match(ITy *V) {
378    if (const auto *CI = dyn_cast<ConstantInt>(V))
379      if (this->isValue(CI->getValue())) {
380        Res = &CI->getValue();
381        return true;
382      }
383    if (V->getType()->isVectorTy())
384      if (const auto *C = dyn_cast<Constant>(V))
385        if (auto *CI = dyn_cast_or_null<ConstantInt>(C->getSplatValue()))
386          if (this->isValue(CI->getValue())) {
387            Res = &CI->getValue();
388            return true;
389          }
390 
391    return false;
392  }
393};
394 
395/// This helper class is used to match scalar and vector constants that
396/// satisfy a specified predicate, and bind them to an APFloat.
397/// Undefs are allowed in splat vector constants.
398template <typename Predicate> struct apf_pred_ty : public Predicate {
399  const APFloat *&Res;
400 
401  apf_pred_ty(const APFloat *&R) : Res(R) {}
402 
403  template <typename ITy> bool match(ITy *V) {
404    if (const auto *CI = dyn_cast<ConstantFP>(V))
405      if (this->isValue(CI->getValue())) {
406        Res = &CI->getValue();
407        return true;
408      }
409    if (V->getType()->isVectorTy())
410      if (const auto *C = dyn_cast<Constant>(V))
411        if (auto *CI = dyn_cast_or_null<ConstantFP>(
412                C->getSplatValue(/* AllowUndef */ true)))
413          if (this->isValue(CI->getValue())) {
414            Res = &CI->getValue();
415            return true;
416          }
417 
418    return false;
419  }
420};
421 
422///////////////////////////////////////////////////////////////////////////////
423//
424// Encapsulate constant value queries for use in templated predicate matchers.
425// This allows checking if constants match using compound predicates and works
426// with vector constants, possibly with relaxed constraints. For example, ignore
427// undef values.
428//
429///////////////////////////////////////////////////////////////////////////////
430 
431struct is_any_apint {
432  bool isValue(const APInt &C) { return true; }
433};
434/// Match an integer or vector with any integral constant.
435/// For vectors, this includes constants with undefined elements.
436inline cst_pred_ty<is_any_apint> m_AnyIntegralConstant() {
437  return cst_pred_ty<is_any_apint>();
438}
439 
440struct is_all_ones {
441  bool isValue(const APInt &C) { return C.isAllOnesValue(); }
442};
443/// Match an integer or vector with all bits set.
444/// For vectors, this includes constants with undefined elements.
445inline cst_pred_ty<is_all_ones> m_AllOnes() {
446  return cst_pred_ty<is_all_ones>();
447}
448 
449struct is_maxsignedvalue {
450  bool isValue(const APInt &C) { return C.isMaxSignedValue(); }
451};
452/// Match an integer or vector with values having all bits except for the high
453/// bit set (0x7f...).
454/// For vectors, this includes constants with undefined elements.
455inline cst_pred_ty<is_maxsignedvalue> m_MaxSignedValue() {
456  return cst_pred_ty<is_maxsignedvalue>();
457}
458inline api_pred_ty<is_maxsignedvalue> m_MaxSignedValue(const APInt *&V) {
459  return V;
460}
461 
462struct is_negative {
463  bool isValue(const APInt &C) { return C.isNegative(); }
464};
465/// Match an integer or vector of negative values.
466/// For vectors, this includes constants with undefined elements.
467inline cst_pred_ty<is_negative> m_Negative() {
468  return cst_pred_ty<is_negative>();
469}
470inline api_pred_ty<is_negative> m_Negative(const APInt *&V) {
471  return V;
472}
473 
474struct is_nonnegative {
475  bool isValue(const APInt &C) { return C.isNonNegative(); }
476};
477/// Match an integer or vector of non-negative values.
478/// For vectors, this includes constants with undefined elements.
479inline cst_pred_ty<is_nonnegative> m_NonNegative() {
480  return cst_pred_ty<is_nonnegative>();
481}
482inline api_pred_ty<is_nonnegative> m_NonNegative(const APInt *&V) {
483  return V;
484}
485 
486struct is_strictlypositive {
487  bool isValue(const APInt &C) { return C.isStrictlyPositive(); }
488};
489/// Match an integer or vector of strictly positive values.
490/// For vectors, this includes constants with undefined elements.
491inline cst_pred_ty<is_strictlypositive> m_StrictlyPositive() {
492  return cst_pred_ty<is_strictlypositive>();
493}
494inline api_pred_ty<is_strictlypositive> m_StrictlyPositive(const APInt *&V) {
495  return V;
496}
497 
498struct is_nonpositive {
499  bool isValue(const APInt &C) { return C.isNonPositive(); }
500};
501/// Match an integer or vector of non-positive values.
502/// For vectors, this includes constants with undefined elements.
503inline cst_pred_ty<is_nonpositive> m_NonPositive() {
504  return cst_pred_ty<is_nonpositive>();
505}
506inline api_pred_ty<is_nonpositive> m_NonPositive(const APInt *&V) { return V; }
507 
508struct is_one {
509  bool isValue(const APInt &C) { return C.isOneValue(); }
510};
511/// Match an integer 1 or a vector with all elements equal to 1.
512/// For vectors, this includes constants with undefined elements.
513inline cst_pred_ty<is_one> m_One() {
514  return cst_pred_ty<is_one>();
515}
516 
517struct is_zero_int {
518  bool isValue(const APInt &C) { return C.isNullValue(); }
519};
520/// Match an integer 0 or a vector with all elements equal to 0.
521/// For vectors, this includes constants with undefined elements.
522inline cst_pred_ty<is_zero_int> m_ZeroInt() {
523  return cst_pred_ty<is_zero_int>();
524}
525 
526struct is_zero {
527  template <typename ITy> bool match(ITy *V) {
528    auto *C = dyn_cast<Constant>(V);
529    // FIXME: this should be able to do something for scalable vectors
530    return C && (C->isNullValue() || cst_pred_ty<is_zero_int>().match(C));
531  }
532};
533/// Match any null constant or a vector with all elements equal to 0.
534/// For vectors, this includes constants with undefined elements.
535inline is_zero m_Zero() {
536  return is_zero();
537}
538 
539struct is_power2 {
540  bool isValue(const APInt &C) { return C.isPowerOf2(); }
541};
542/// Match an integer or vector power-of-2.
543/// For vectors, this includes constants with undefined elements.
544inline cst_pred_ty<is_power2> m_Power2() {
545  return cst_pred_ty<is_power2>();
546}
547inline api_pred_ty<is_power2> m_Power2(const APInt *&V) {
548  return V;
549}
550 
551struct is_negated_power2 {
552  bool isValue(const APInt &C) { return (-C).isPowerOf2(); }
553};
554/// Match a integer or vector negated power-of-2.
555/// For vectors, this includes constants with undefined elements.
556inline cst_pred_ty<is_negated_power2> m_NegatedPower2() {
557  return cst_pred_ty<is_negated_power2>();
558}
559inline api_pred_ty<is_negated_power2> m_NegatedPower2(const APInt *&V) {
560  return V;
561}
562 
563struct is_power2_or_zero {
564  bool isValue(const APInt &C) { return !C || C.isPowerOf2(); }
565};
566/// Match an integer or vector of 0 or power-of-2 values.
567/// For vectors, this includes constants with undefined elements.
568inline cst_pred_ty<is_power2_or_zero> m_Power2OrZero() {
569  return cst_pred_ty<is_power2_or_zero>();
570}
571inline api_pred_ty<is_power2_or_zero> m_Power2OrZero(const APInt *&V) {
572  return V;
573}
574 
575struct is_sign_mask {
576  bool isValue(const APInt &C) { return C.isSignMask(); }
577};
578/// Match an integer or vector with only the sign bit(s) set.
579/// For vectors, this includes constants with undefined elements.
580inline cst_pred_ty<is_sign_mask> m_SignMask() {
581  return cst_pred_ty<is_sign_mask>();
582}
583 
584struct is_lowbit_mask {
585  bool isValue(const APInt &C) { return C.isMask(); }
586};
587/// Match an integer or vector with only the low bit(s) set.
588/// For vectors, this includes constants with undefined elements.
589inline cst_pred_ty<is_lowbit_mask> m_LowBitMask() {
590  return cst_pred_ty<is_lowbit_mask>();
591}
592 
593struct icmp_pred_with_threshold {
594  ICmpInst::Predicate Pred;
595  const APInt *Thr;
596  bool isValue(const APInt &C) {
597    switch (Pred) {
598    case ICmpInst::Predicate::ICMP_EQ:
599      return C.eq(*Thr);
600    case ICmpInst::Predicate::ICMP_NE:
601      return C.ne(*Thr);
602    case ICmpInst::Predicate::ICMP_UGT:
603      return C.ugt(*Thr);
604    case ICmpInst::Predicate::ICMP_UGE:
605      return C.uge(*Thr);
606    case ICmpInst::Predicate::ICMP_ULT:
607      return C.ult(*Thr);
608    case ICmpInst::Predicate::ICMP_ULE:
609      return C.ule(*Thr);
610    case ICmpInst::Predicate::ICMP_SGT:
611      return C.sgt(*Thr);
612    case ICmpInst::Predicate::ICMP_SGE:
613      return C.sge(*Thr);
614    case ICmpInst::Predicate::ICMP_SLT:
615      return C.slt(*Thr);
616    case ICmpInst::Predicate::ICMP_SLE:
617      return C.sle(*Thr);
618    default:
619      llvm_unreachable("Unhandled ICmp predicate")::llvm::llvm_unreachable_internal("Unhandled ICmp predicate",
 "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/PatternMatch.h"
, 619);
620    }
621  }
622};
623/// Match an integer or vector with every element comparing 'pred' (eg/ne/...)
624/// to Threshold. For vectors, this includes constants with undefined elements.
625inline cst_pred_ty<icmp_pred_with_threshold>
626m_SpecificInt_ICMP(ICmpInst::Predicate Predicate, const APInt &Threshold) {
627  cst_pred_ty<icmp_pred_with_threshold> P;
628  P.Pred = Predicate;
629  P.Thr = &Threshold;
630  return P;
631}
632 
633struct is_nan {
634  bool isValue(const APFloat &C) { return C.isNaN(); }
635};
636/// Match an arbitrary NaN constant. This includes quiet and signalling nans.
637/// For vectors, this includes constants with undefined elements.
638inline cstfp_pred_ty<is_nan> m_NaN() {
639  return cstfp_pred_ty<is_nan>();
640}
641 
642struct is_nonnan {
643  bool isValue(const APFloat &C) { return !C.isNaN(); }
644};
645/// Match a non-NaN FP constant.
646/// For vectors, this includes constants with undefined elements.
647inline cstfp_pred_ty<is_nonnan> m_NonNaN() {
648  return cstfp_pred_ty<is_nonnan>();
649}
650 
651struct is_inf {
652  bool isValue(const APFloat &C) { return C.isInfinity(); }
653};
654/// Match a positive or negative infinity FP constant.
655/// For vectors, this includes constants with undefined elements.
656inline cstfp_pred_ty<is_inf> m_Inf() {
657  return cstfp_pred_ty<is_inf>();
658}
659 
660struct is_noninf {
661  bool isValue(const APFloat &C) { return !C.isInfinity(); }
662};
663/// Match a non-infinity FP constant, i.e. finite or NaN.
664/// For vectors, this includes constants with undefined elements.
665inline cstfp_pred_ty<is_noninf> m_NonInf() {
666  return cstfp_pred_ty<is_noninf>();
667}
668 
669struct is_finite {
670  bool isValue(const APFloat &C) { return C.isFinite(); }
671};
672/// Match a finite FP constant, i.e. not infinity or NaN.
673/// For vectors, this includes constants with undefined elements.
674inline cstfp_pred_ty<is_finite> m_Finite() {
675  return cstfp_pred_ty<is_finite>();
676}
677inline apf_pred_ty<is_finite> m_Finite(const APFloat *&V) { return V; }
678 
679struct is_finitenonzero {
680  bool isValue(const APFloat &C) { return C.isFiniteNonZero(); }
681};
682/// Match a finite non-zero FP constant.
683/// For vectors, this includes constants with undefined elements.
684inline cstfp_pred_ty<is_finitenonzero> m_FiniteNonZero() {
685  return cstfp_pred_ty<is_finitenonzero>();
686}
687inline apf_pred_ty<is_finitenonzero> m_FiniteNonZero(const APFloat *&V) {
688  return V;
689}
690 
691struct is_any_zero_fp {
692  bool isValue(const APFloat &C) { return C.isZero(); }
693};
694/// Match a floating-point negative zero or positive zero.
695/// For vectors, this includes constants with undefined elements.
696inline cstfp_pred_ty<is_any_zero_fp> m_AnyZeroFP() {
697  return cstfp_pred_ty<is_any_zero_fp>();
698}
699 
700struct is_pos_zero_fp {
701  bool isValue(const APFloat &C) { return C.isPosZero(); }
702};
703/// Match a floating-point positive zero.
704/// For vectors, this includes constants with undefined elements.
705inline cstfp_pred_ty<is_pos_zero_fp> m_PosZeroFP() {
706  return cstfp_pred_ty<is_pos_zero_fp>();
707}
708 
709struct is_neg_zero_fp {
710  bool isValue(const APFloat &C) { return C.isNegZero(); }
711};
712/// Match a floating-point negative zero.
713/// For vectors, this includes constants with undefined elements.
714inline cstfp_pred_ty<is_neg_zero_fp> m_NegZeroFP() {
715  return cstfp_pred_ty<is_neg_zero_fp>();
716}
717 
718struct is_non_zero_fp {
719  bool isValue(const APFloat &C) { return C.isNonZero(); }
720};
721/// Match a floating-point non-zero.
722/// For vectors, this includes constants with undefined elements.
723inline cstfp_pred_ty<is_non_zero_fp> m_NonZeroFP() {
724  return cstfp_pred_ty<is_non_zero_fp>();
725}
726 
727///////////////////////////////////////////////////////////////////////////////
728 
729template <typename Class> struct bind_ty {
730  Class *&VR;
731 
732  bind_ty(Class *&V) : VR(V) {}
733 
734  template <typename ITy> bool match(ITy *V) {
735    if (auto *CV = dyn_cast<Class>(V)) {
736      VR = CV;
737      return true;
738    }
739    return false;
740  }
741};
742 
743/// Match a value, capturing it if we match.
744inline bind_ty<Value> m_Value(Value *&V) { return V; }
745inline bind_ty<const Value> m_Value(const Value *&V) { return V; }
746 
747/// Match an instruction, capturing it if we match.
748inline bind_ty<Instruction> m_Instruction(Instruction *&I) { return I; }
749/// Match a unary operator, capturing it if we match.
750inline bind_ty<UnaryOperator> m_UnOp(UnaryOperator *&I) { return I; }
751/// Match a binary operator, capturing it if we match.
752inline bind_ty<BinaryOperator> m_BinOp(BinaryOperator *&I) { return I; }
753/// Match a with overflow intrinsic, capturing it if we match.
754inline bind_ty<WithOverflowInst> m_WithOverflowInst(WithOverflowInst *&I) { return I; }
755inline bind_ty<const WithOverflowInst>
756m_WithOverflowInst(const WithOverflowInst *&I) {
757  return I;
758}
759 
760/// Match a Constant, capturing the value if we match.
761inline bind_ty<Constant> m_Constant(Constant *&C) { return C; }
762 
763/// Match a ConstantInt, capturing the value if we match.
764inline bind_ty<ConstantInt> m_ConstantInt(ConstantInt *&CI) { return CI; }
765 
766/// Match a ConstantFP, capturing the value if we match.
767inline bind_ty<ConstantFP> m_ConstantFP(ConstantFP *&C) { return C; }
768 
769/// Match a ConstantExpr, capturing the value if we match.
770inline bind_ty<ConstantExpr> m_ConstantExpr(ConstantExpr *&C) { return C; }
771 
772/// Match a basic block value, capturing it if we match.
773inline bind_ty<BasicBlock> m_BasicBlock(BasicBlock *&V) { return V; }
774inline bind_ty<const BasicBlock> m_BasicBlock(const BasicBlock *&V) {
775  return V;
776}
777 
778/// Match an arbitrary immediate Constant and ignore it.
779inline match_combine_and<class_match<Constant>,
780                         match_unless<class_match<ConstantExpr>>>
781m_ImmConstant() {
782  return m_CombineAnd(m_Constant(), m_Unless(m_ConstantExpr()));
783}
784 
785/// Match an immediate Constant, capturing the value if we match.
786inline match_combine_and<bind_ty<Constant>,
787                         match_unless<class_match<ConstantExpr>>>
788m_ImmConstant(Constant *&C) {
789  return m_CombineAnd(m_Constant(C), m_Unless(m_ConstantExpr()));
790}
791 
792/// Match a specified Value*.
793struct specificval_ty {
794  const Value *Val;
795 
796  specificval_ty(const Value *V) : Val(V) {}
797 
798  template <typename ITy> bool match(ITy *V) { return V == Val; }
799};
800 
801/// Match if we have a specific specified value.
802inline specificval_ty m_Specific(const Value *V) { return V; }
803 
804/// Stores a reference to the Value *, not the Value * itself,
805/// thus can be used in commutative matchers.
806template <typename Class> struct deferredval_ty {
807  Class *const &Val;
808 
809  deferredval_ty(Class *const &V) : Val(V) {}
810 
811  template <typename ITy> bool match(ITy *const V) { return V == Val; }
812};
813 
814/// Like m_Specific(), but works if the specific value to match is determined
815/// as part of the same match() expression. For example:
816/// m_Add(m_Value(X), m_Specific(X)) is incorrect, because m_Specific() will
817/// bind X before the pattern match starts.
818/// m_Add(m_Value(X), m_Deferred(X)) is correct, and will check against
819/// whichever value m_Value(X) populated.
820inline deferredval_ty<Value> m_Deferred(Value *const &V) { return V; }
821inline deferredval_ty<const Value> m_Deferred(const Value *const &V) {
822  return V;
823}
824 
825/// Match a specified floating point value or vector of all elements of
826/// that value.
827struct specific_fpval {
828  double Val;
829 
830  specific_fpval(double V) : Val(V) {}
831 
832  template <typename ITy> bool match(ITy *V) {
833    if (const auto *CFP = dyn_cast<ConstantFP>(V))
834      return CFP->isExactlyValue(Val);
835    if (V->getType()->isVectorTy())
836      if (const auto *C = dyn_cast<Constant>(V))
837        if (auto *CFP = dyn_cast_or_null<ConstantFP>(C->getSplatValue()))
838          return CFP->isExactlyValue(Val);
839    return false;
840  }
841};
842 
843/// Match a specific floating point value or vector with all elements
844/// equal to the value.
845inline specific_fpval m_SpecificFP(double V) { return specific_fpval(V); }
846 
847/// Match a float 1.0 or vector with all elements equal to 1.0.
848inline specific_fpval m_FPOne() { return m_SpecificFP(1.0); }
849 
850struct bind_const_intval_ty {
851  uint64_t &VR;
852 
853  bind_const_intval_ty(uint64_t &V) : VR(V) {}
854 
855  template <typename ITy> bool match(ITy *V) {
856    if (const auto *CV = dyn_cast<ConstantInt>(V))
857      if (CV->getValue().ule(UINT64_MAX(18446744073709551615UL))) {
858        VR = CV->getZExtValue();
859        return true;
860      }
861    return false;
862  }
863};
864 
865/// Match a specified integer value or vector of all elements of that
866/// value.
867template <bool AllowUndefs>
868struct specific_intval {
869  APInt Val;
870 
871  specific_intval(APInt V) : Val(std::move(V)) {}
872 
873  template <typename ITy> bool match(ITy *V) {
874    const auto *CI = dyn_cast<ConstantInt>(V);
875    if (!CI && V->getType()->isVectorTy())
876      if (const auto *C = dyn_cast<Constant>(V))
877        CI = dyn_cast_or_null<ConstantInt>(C->getSplatValue(AllowUndefs));
878 
879    return CI && APInt::isSameValue(CI->getValue(), Val);
880  }
881};
882 
883/// Match a specific integer value or vector with all elements equal to
884/// the value.
885inline specific_intval<false> m_SpecificInt(APInt V) {
886  return specific_intval<false>(std::move(V));
887}
888 
889inline specific_intval<false> m_SpecificInt(uint64_t V) {
890  return m_SpecificInt(APInt(64, V));
891}
892 
893inline specific_intval<true> m_SpecificIntAllowUndef(APInt V) {
894  return specific_intval<true>(std::move(V));
895}
896 
897inline specific_intval<true> m_SpecificIntAllowUndef(uint64_t V) {
898  return m_SpecificIntAllowUndef(APInt(64, V));
899}
900 
901/// Match a ConstantInt and bind to its value.  This does not match
902/// ConstantInts wider than 64-bits.
903inline bind_const_intval_ty m_ConstantInt(uint64_t &V) { return V; }
904 
905/// Match a specified basic block value.
906struct specific_bbval {
907  BasicBlock *Val;
908 
909  specific_bbval(BasicBlock *Val) : Val(Val) {}
910 
911  template <typename ITy> bool match(ITy *V) {
912    const auto *BB = dyn_cast<BasicBlock>(V);
913    return BB && BB == Val;
914  }
915};
916 
917/// Match a specific basic block value.
918inline specific_bbval m_SpecificBB(BasicBlock *BB) {
919  return specific_bbval(BB);
920}
921 
922/// A commutative-friendly version of m_Specific().
923inline deferredval_ty<BasicBlock> m_Deferred(BasicBlock *const &BB) {
924  return BB;
925}
926inline deferredval_ty<const BasicBlock>
927m_Deferred(const BasicBlock *const &BB) {
928  return BB;
929}
930 
931//===----------------------------------------------------------------------===//
932// Matcher for any binary operator.
933//
934template <typename LHS_t, typename RHS_t, bool Commutable = false>
935struct AnyBinaryOp_match {
936  LHS_t L;
937  RHS_t R;
938 
939  // The evaluation order is always stable, regardless of Commutability.
940  // The LHS is always matched first.
941  AnyBinaryOp_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {}
942 
943  template <typename OpTy> bool match(OpTy *V) {
944    if (auto *I = dyn_cast<BinaryOperator>(V))
945      return (L.match(I->getOperand(0)) && R.match(I->getOperand(1))) ||
946             (Commutable && L.match(I->getOperand(1)) &&
947              R.match(I->getOperand(0)));
948    return false;
949  }
950};
951 
952template <typename LHS, typename RHS>
953inline AnyBinaryOp_match<LHS, RHS> m_BinOp(const LHS &L, const RHS &R) {
954  return AnyBinaryOp_match<LHS, RHS>(L, R);
955}
956 
957//===----------------------------------------------------------------------===//
958// Matcher for any unary operator.
959// TODO fuse unary, binary matcher into n-ary matcher
960//
961template <typename OP_t> struct AnyUnaryOp_match {
962  OP_t X;
963 
964  AnyUnaryOp_match(const OP_t &X) : X(X) {}
965 
966  template <typename OpTy> bool match(OpTy *V) {
967    if (auto *I = dyn_cast<UnaryOperator>(V))
968      return X.match(I->getOperand(0));
969    return false;
970  }
971};
972 
973template <typename OP_t> inline AnyUnaryOp_match<OP_t> m_UnOp(const OP_t &X) {
974  return AnyUnaryOp_match<OP_t>(X);
975}
976 
977//===----------------------------------------------------------------------===//
978// Matchers for specific binary operators.
979//
980 
981template <typename LHS_t, typename RHS_t, unsigned Opcode,
982          bool Commutable = false>
983struct BinaryOp_match {
984  LHS_t L;
985  RHS_t R;
986 
987  // The evaluation order is always stable, regardless of Commutability.
988  // The LHS is always matched first.
989  BinaryOp_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {}
990 
991  template <typename OpTy> bool match(OpTy *V) {
992    if (V->getValueID() == Value::InstructionVal + Opcode) {
993      auto *I = cast<BinaryOperator>(V);
994      return (L.match(I->getOperand(0)) && R.match(I->getOperand(1))) ||
995             (Commutable && L.match(I->getOperand(1)) &&
996              R.match(I->getOperand(0)));
997    }
998    if (auto *CE = dyn_cast<ConstantExpr>(V))
999      return CE->getOpcode() == Opcode &&
1000             ((L.match(CE->getOperand(0)) && R.match(CE->getOperand(1))) ||
1001              (Commutable && L.match(CE->getOperand(1)) &&
1002               R.match(CE->getOperand(0))));
1003    return false;
1004  }
1005};
1006 
1007template <typename LHS, typename RHS>
1008inline BinaryOp_match<LHS, RHS, Instruction::Add> m_Add(const LHS &L,
1009                                                        const RHS &R) {
1010  return BinaryOp_match<LHS, RHS, Instruction::Add>(L, R);
1011}
1012 
1013template <typename LHS, typename RHS>
1014inline BinaryOp_match<LHS, RHS, Instruction::FAdd> m_FAdd(const LHS &L,
1015                                                          const RHS &R) {
1016  return BinaryOp_match<LHS, RHS, Instruction::FAdd>(L, R);
1017}
1018 
1019template <typename LHS, typename RHS>
1020inline BinaryOp_match<LHS, RHS, Instruction::Sub> m_Sub(const LHS &L,
1021                                                        const RHS &R) {
1022  return BinaryOp_match<LHS, RHS, Instruction::Sub>(L, R);
1023}
1024 
1025template <typename LHS, typename RHS>
1026inline BinaryOp_match<LHS, RHS, Instruction::FSub> m_FSub(const LHS &L,
1027                                                          const RHS &R) {
1028  return BinaryOp_match<LHS, RHS, Instruction::FSub>(L, R);
1029}
1030 
1031template <typename Op_t> struct FNeg_match {
1032  Op_t X;
1033 
1034  FNeg_match(const Op_t &Op) : X(Op) {}
1035  template <typename OpTy> bool match(OpTy *V) {
1036    auto *FPMO = dyn_cast<FPMathOperator>(V);
1037    if (!FPMO) return false;
1038 
1039    if (FPMO->getOpcode() == Instruction::FNeg)
1040      return X.match(FPMO->getOperand(0));
1041 
1042    if (FPMO->getOpcode() == Instruction::FSub) {
1043      if (FPMO->hasNoSignedZeros()) {
1044        // With 'nsz', any zero goes.
1045        if (!cstfp_pred_ty<is_any_zero_fp>().match(FPMO->getOperand(0)))
1046          return false;
1047      } else {
1048        // Without 'nsz', we need fsub -0.0, X exactly.
1049        if (!cstfp_pred_ty<is_neg_zero_fp>().match(FPMO->getOperand(0)))
1050          return false;
1051      }
1052 
1053      return X.match(FPMO->getOperand(1));
1054    }
1055 
1056    return false;
1057  }
1058};
1059 
1060/// Match 'fneg X' as 'fsub -0.0, X'.
1061template <typename OpTy>
1062inline FNeg_match<OpTy>
1063m_FNeg(const OpTy &X) {
1064  return FNeg_match<OpTy>(X);
1065}
1066 
1067/// Match 'fneg X' as 'fsub +-0.0, X'.
1068template <typename RHS>
1069inline BinaryOp_match<cstfp_pred_ty<is_any_zero_fp>, RHS, Instruction::FSub>
1070m_FNegNSZ(const RHS &X) {
1071  return m_FSub(m_AnyZeroFP(), X);
1072}
1073 
1074template <typename LHS, typename RHS>
1075inline BinaryOp_match<LHS, RHS, Instruction::Mul> m_Mul(const LHS &L,
1076                                                        const RHS &R) {
1077  return BinaryOp_match<LHS, RHS, Instruction::Mul>(L, R);
1078}
1079 
1080template <typename LHS, typename RHS>
1081inline BinaryOp_match<LHS, RHS, Instruction::FMul> m_FMul(const LHS &L,
1082                                                          const RHS &R) {
1083  return BinaryOp_match<LHS, RHS, Instruction::FMul>(L, R);
1084}
1085 
1086template <typename LHS, typename RHS>
1087inline BinaryOp_match<LHS, RHS, Instruction::UDiv> m_UDiv(const LHS &L,
1088                                                          const RHS &R) {
1089  return BinaryOp_match<LHS, RHS, Instruction::UDiv>(L, R);
1090}
1091 
1092template <typename LHS, typename RHS>
1093inline BinaryOp_match<LHS, RHS, Instruction::SDiv> m_SDiv(const LHS &L,
1094                                                          const RHS &R) {
1095  return BinaryOp_match<LHS, RHS, Instruction::SDiv>(L, R);
1096}
1097 
1098template <typename LHS, typename RHS>
1099inline BinaryOp_match<LHS, RHS, Instruction::FDiv> m_FDiv(const LHS &L,
1100                                                          const RHS &R) {
1101  return BinaryOp_match<LHS, RHS, Instruction::FDiv>(L, R);
1102}
1103 
1104template <typename LHS, typename RHS>
1105inline BinaryOp_match<LHS, RHS, Instruction::URem> m_URem(const LHS &L,
1106                                                          const RHS &R) {
1107  return BinaryOp_match<LHS, RHS, Instruction::URem>(L, R);
1108}
1109 
1110template <typename LHS, typename RHS>
1111inline BinaryOp_match<LHS, RHS, Instruction::SRem> m_SRem(const LHS &L,
1112                                                          const RHS &R) {
1113  return BinaryOp_match<LHS, RHS, Instruction::SRem>(L, R);
1114}
1115 
1116template <typename LHS, typename RHS>
1117inline BinaryOp_match<LHS, RHS, Instruction::FRem> m_FRem(const LHS &L,
1118                                                          const RHS &R) {
1119  return BinaryOp_match<LHS, RHS, Instruction::FRem>(L, R);
1120}
1121 
1122template <typename LHS, typename RHS>
1123inline BinaryOp_match<LHS, RHS, Instruction::And> m_And(const LHS &L,
1124                                                        const RHS &R) {
1125  return BinaryOp_match<LHS, RHS, Instruction::And>(L, R);
1126}
1127 
1128template <typename LHS, typename RHS>
1129inline BinaryOp_match<LHS, RHS, Instruction::Or> m_Or(const LHS &L,
1130                                                      const RHS &R) {
1131  return BinaryOp_match<LHS, RHS, Instruction::Or>(L, R);
1132}
1133 
1134template <typename LHS, typename RHS>
1135inline BinaryOp_match<LHS, RHS, Instruction::Xor> m_Xor(const LHS &L,
1136                                                        const RHS &R) {
1137  return BinaryOp_match<LHS, RHS, Instruction::Xor>(L, R);
1138}
1139 
1140template <typename LHS, typename RHS>
1141inline BinaryOp_match<LHS, RHS, Instruction::Shl> m_Shl(const LHS &L,
1142                                                        const RHS &R) {
1143  return BinaryOp_match<LHS, RHS, Instruction::Shl>(L, R);
1144}
1145 
1146template <typename LHS, typename RHS>
1147inline BinaryOp_match<LHS, RHS, Instruction::LShr> m_LShr(const LHS &L,
1148                                                          const RHS &R) {
1149  return BinaryOp_match<LHS, RHS, Instruction::LShr>(L, R);
1150}
1151 
1152template <typename LHS, typename RHS>
1153inline BinaryOp_match<LHS, RHS, Instruction::AShr> m_AShr(const LHS &L,
1154                                                          const RHS &R) {
1155  return BinaryOp_match<LHS, RHS, Instruction::AShr>(L, R);
1156}
1157 
1158template <typename LHS_t, typename RHS_t, unsigned Opcode,
1159          unsigned WrapFlags = 0>
1160struct OverflowingBinaryOp_match {
1161  LHS_t L;
1162  RHS_t R;
1163 
1164  OverflowingBinaryOp_match(const LHS_t &LHS, const RHS_t &RHS)
1165      : L(LHS), R(RHS) {}
1166 
1167  template <typename OpTy> bool match(OpTy *V) {
1168    if (auto *Op = dyn_cast<OverflowingBinaryOperator>(V)) {
1169      if (Op->getOpcode() != Opcode)
1170        return false;
1171      if (WrapFlags & OverflowingBinaryOperator::NoUnsignedWrap &&
1172          !Op->hasNoUnsignedWrap())
1173        return false;
1174      if (WrapFlags & OverflowingBinaryOperator::NoSignedWrap &&
1175          !Op->hasNoSignedWrap())
1176        return false;
1177      return L.match(Op->getOperand(0)) && R.match(Op->getOperand(1));
1178    }
1179    return false;
1180  }
1181};
1182 
1183template <typename LHS, typename RHS>
1184inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Add,
1185                                 OverflowingBinaryOperator::NoSignedWrap>
1186m_NSWAdd(const LHS &L, const RHS &R) {
1187  return OverflowingBinaryOp_match<LHS, RHS, Instruction::Add,
1188                                   OverflowingBinaryOperator::NoSignedWrap>(
1189      L, R);
1190}
1191template <typename LHS, typename RHS>
1192inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub,
1193                                 OverflowingBinaryOperator::NoSignedWrap>
1194m_NSWSub(const LHS &L, const RHS &R) {
1195  return OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub,
1196                                   OverflowingBinaryOperator::NoSignedWrap>(
1197      L, R);
1198}
1199template <typename LHS, typename RHS>
1200inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul,
1201                                 OverflowingBinaryOperator::NoSignedWrap>
1202m_NSWMul(const LHS &L, const RHS &R) {
1203  return OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul,
1204                                   OverflowingBinaryOperator::NoSignedWrap>(
1205      L, R);
1206}
1207template <typename LHS, typename RHS>
1208inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl,
1209                                 OverflowingBinaryOperator::NoSignedWrap>
1210m_NSWShl(const LHS &L, const RHS &R) {
1211  return OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl,
1212                                   OverflowingBinaryOperator::NoSignedWrap>(
1213      L, R);
1214}
1215 
1216template <typename LHS, typename RHS>
1217inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Add,
1218                                 OverflowingBinaryOperator::NoUnsignedWrap>
1219m_NUWAdd(const LHS &L, const RHS &R) {
1220  return OverflowingBinaryOp_match<LHS, RHS, Instruction::Add,
1221                                   OverflowingBinaryOperator::NoUnsignedWrap>(
1222      L, R);
1223}
1224template <typename LHS, typename RHS>
1225inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub,
1226                                 OverflowingBinaryOperator::NoUnsignedWrap>
1227m_NUWSub(const LHS &L, const RHS &R) {
1228  return OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub,
1229                                   OverflowingBinaryOperator::NoUnsignedWrap>(
1230      L, R);
1231}
1232template <typename LHS, typename RHS>
1233inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul,
1234                                 OverflowingBinaryOperator::NoUnsignedWrap>
1235m_NUWMul(const LHS &L, const RHS &R) {
1236  return OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul,
1237                                   OverflowingBinaryOperator::NoUnsignedWrap>(
1238      L, R);
1239}
1240template <typename LHS, typename RHS>
1241inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl,
1242                                 OverflowingBinaryOperator::NoUnsignedWrap>
1243m_NUWShl(const LHS &L, const RHS &R) {
1244  return OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl,
1245                                   OverflowingBinaryOperator::NoUnsignedWrap>(
1246      L, R);
1247}
1248 
1249//===----------------------------------------------------------------------===//
1250// Class that matches a group of binary opcodes.
1251//
1252template <typename LHS_t, typename RHS_t, typename Predicate>
1253struct BinOpPred_match : Predicate {
1254  LHS_t L;
1255  RHS_t R;
1256 
1257  BinOpPred_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {}
1258 
1259  template <typename OpTy> bool match(OpTy *V) {
1260    if (auto *I = dyn_cast<Instruction>(V))
1261      return this->isOpType(I->getOpcode()) && L.match(I->getOperand(0)) &&
1262             R.match(I->getOperand(1));
1263    if (auto *CE = dyn_cast<ConstantExpr>(V))
1264      return this->isOpType(CE->getOpcode()) && L.match(CE->getOperand(0)) &&
1265             R.match(CE->getOperand(1));
1266    return false;
1267  }
1268};
1269 
1270struct is_shift_op {
1271  bool isOpType(unsigned Opcode) { return Instruction::isShift(Opcode); }
1272};
1273 
1274struct is_right_shift_op {
1275  bool isOpType(unsigned Opcode) {
1276    return Opcode == Instruction::LShr || Opcode == Instruction::AShr;
1277  }
1278};
1279 
1280struct is_logical_shift_op {
1281  bool isOpType(unsigned Opcode) {
1282    return Opcode == Instruction::LShr || Opcode == Instruction::Shl;
1283  }
1284};
1285 
1286struct is_bitwiselogic_op {
1287  bool isOpType(unsigned Opcode) {
1288    return Instruction::isBitwiseLogicOp(Opcode);
1289  }
1290};
1291 
1292struct is_idiv_op {
1293  bool isOpType(unsigned Opcode) {
1294    return Opcode == Instruction::SDiv || Opcode == Instruction::UDiv;
1295  }
1296};
1297 
1298struct is_irem_op {
1299  bool isOpType(unsigned Opcode) {
1300    return Opcode == Instruction::SRem || Opcode == Instruction::URem;
1301  }
1302};
1303 
1304/// Matches shift operations.
1305template <typename LHS, typename RHS>
1306inline BinOpPred_match<LHS, RHS, is_shift_op> m_Shift(const LHS &L,
1307                                                      const RHS &R) {
1308  return BinOpPred_match<LHS, RHS, is_shift_op>(L, R);
1309}
1310 
1311/// Matches logical shift operations.
1312template <typename LHS, typename RHS>
1313inline BinOpPred_match<LHS, RHS, is_right_shift_op> m_Shr(const LHS &L,
1314                                                          const RHS &R) {
1315  return BinOpPred_match<LHS, RHS, is_right_shift_op>(L, R);
1316}
1317 
1318/// Matches logical shift operations.
1319template <typename LHS, typename RHS>
1320inline BinOpPred_match<LHS, RHS, is_logical_shift_op>
1321m_LogicalShift(const LHS &L, const RHS &R) {
1322  return BinOpPred_match<LHS, RHS, is_logical_shift_op>(L, R);
1323}
1324 
1325/// Matches bitwise logic operations.
1326template <typename LHS, typename RHS>
1327inline BinOpPred_match<LHS, RHS, is_bitwiselogic_op>
1328m_BitwiseLogic(const LHS &L, const RHS &R) {
1329  return BinOpPred_match<LHS, RHS, is_bitwiselogic_op>(L, R);
1330}
1331 
1332/// Matches integer division operations.
1333template <typename LHS, typename RHS>
1334inline BinOpPred_match<LHS, RHS, is_idiv_op> m_IDiv(const LHS &L,
1335                                                    const RHS &R) {
1336  return BinOpPred_match<LHS, RHS, is_idiv_op>(L, R);
1337}
1338 
1339/// Matches integer remainder operations.
1340template <typename LHS, typename RHS>
1341inline BinOpPred_match<LHS, RHS, is_irem_op> m_IRem(const LHS &L,
1342                                                    const RHS &R) {
1343  return BinOpPred_match<LHS, RHS, is_irem_op>(L, R);
1344}
1345 
1346//===----------------------------------------------------------------------===//
1347// Class that matches exact binary ops.
1348//
1349template <typename SubPattern_t> struct Exact_match {
1350  SubPattern_t SubPattern;
1351 
1352  Exact_match(const SubPattern_t &SP) : SubPattern(SP) {}
1353 
1354  template <typename OpTy> bool match(OpTy *V) {
1355    if (auto *PEO = dyn_cast<PossiblyExactOperator>(V))
1356      return PEO->isExact() && SubPattern.match(V);
1357    return false;
1358  }
1359};
1360 
1361template <typename T> inline Exact_match<T> m_Exact(const T &SubPattern) {
1362  return SubPattern;
1363}
1364 
1365//===----------------------------------------------------------------------===//
1366// Matchers for CmpInst classes
1367//
1368 
1369template <typename LHS_t, typename RHS_t, typename Class, typename PredicateTy,
1370          bool Commutable = false>
1371struct CmpClass_match {
1372  PredicateTy &Predicate;
1373  LHS_t L;
1374  RHS_t R;
1375 
1376  // The evaluation order is always stable, regardless of Commutability.
1377  // The LHS is always matched first.
1378  CmpClass_match(PredicateTy &Pred, const LHS_t &LHS, const RHS_t &RHS)
1379      : Predicate(Pred), L(LHS), R(RHS) {}
1380 
1381  template <typename OpTy> bool match(OpTy *V) {
1382    if (auto *I = dyn_cast<Class>(V)) {
1383      if (L.match(I->getOperand(0)) && R.match(I->getOperand(1))) {
1384        Predicate = I->getPredicate();
1385        return true;
1386      } else if (Commutable && L.match(I->getOperand(1)) &&
1387           R.match(I->getOperand(0))) {
1388        Predicate = I->getSwappedPredicate();
1389        return true;
1390      }
1391    }
1392    return false;
1393  }
1394};
1395 
1396template <typename LHS, typename RHS>
1397inline CmpClass_match<LHS, RHS, CmpInst, CmpInst::Predicate>
1398m_Cmp(CmpInst::Predicate &Pred, const LHS &L, const RHS &R) {
1399  return CmpClass_match<LHS, RHS, CmpInst, CmpInst::Predicate>(Pred, L, R);
1400}
1401 
1402template <typename LHS, typename RHS>
1403inline CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate>
1404m_ICmp(ICmpInst::Predicate &Pred, const LHS &L, const RHS &R) {
1405  return CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate>(Pred, L, R);
1406}
1407 
1408template <typename LHS, typename RHS>
1409inline CmpClass_match<LHS, RHS, FCmpInst, FCmpInst::Predicate>
1410m_FCmp(FCmpInst::Predicate &Pred, const LHS &L, const RHS &R) {
1411  return CmpClass_match<LHS, RHS, FCmpInst, FCmpInst::Predicate>(Pred, L, R);
1412}
1413 
1414//===----------------------------------------------------------------------===//
1415// Matchers for instructions with a given opcode and number of operands.
1416//
1417 
1418/// Matches instructions with Opcode and three operands.
1419template <typename T0, unsigned Opcode> struct OneOps_match {
1420  T0 Op1;
1421 
1422  OneOps_match(const T0 &Op1) : Op1(Op1) {}
1423 
1424  template <typename OpTy> bool match(OpTy *V) {
1425    if (V->getValueID() == Value::InstructionVal + Opcode) {
1426      auto *I = cast<Instruction>(V);
1427      return Op1.match(I->getOperand(0));
1428    }
1429    return false;
1430  }
1431};
1432 
1433/// Matches instructions with Opcode and three operands.
1434template <typename T0, typename T1, unsigned Opcode> struct TwoOps_match {
1435  T0 Op1;
1436  T1 Op2;
1437 
1438  TwoOps_match(const T0 &Op1, const T1 &Op2) : Op1(Op1), Op2(Op2) {}
1439 
1440  template <typename OpTy> bool match(OpTy *V) {
1441    if (V->getValueID() == Value::InstructionVal + Opcode) {
1442      auto *I = cast<Instruction>(V);
1443      return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1));
1444    }
1445    return false;
1446  }
1447};
1448 
1449/// Matches instructions with Opcode and three operands.
1450template <typename T0, typename T1, typename T2, unsigned Opcode>
1451struct ThreeOps_match {
1452  T0 Op1;
1453  T1 Op2;
1454  T2 Op3;
1455 
1456  ThreeOps_match(const T0 &Op1, const T1 &Op2, const T2 &Op3)
1457      : Op1(Op1), Op2(Op2), Op3(Op3) {}
1458 
1459  template <typename OpTy> bool match(OpTy *V) {
1460    if (V->getValueID() == Value::InstructionVal + Opcode) {
1461      auto *I = cast<Instruction>(V);
1462      return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1)) &&
1463             Op3.match(I->getOperand(2));
1464    }
1465    return false;
1466  }
1467};
1468 
1469/// Matches SelectInst.
1470template <typename Cond, typename LHS, typename RHS>
1471inline ThreeOps_match<Cond, LHS, RHS, Instruction::Select>
1472m_Select(const Cond &C, const LHS &L, const RHS &R) {
1473  return ThreeOps_match<Cond, LHS, RHS, Instruction::Select>(C, L, R);
1474}
1475 
1476/// This matches a select of two constants, e.g.:
1477/// m_SelectCst<-1, 0>(m_Value(V))
1478template <int64_t L, int64_t R, typename Cond>
1479inline ThreeOps_match<Cond, constantint_match<L>, constantint_match<R>,
1480                      Instruction::Select>
1481m_SelectCst(const Cond &C) {
1482  return m_Select(C, m_ConstantInt<L>(), m_ConstantInt<R>());
1483}
1484 
1485/// Matches FreezeInst.
1486template <typename OpTy>
1487inline OneOps_match<OpTy, Instruction::Freeze> m_Freeze(const OpTy &Op) {
1488  return OneOps_match<OpTy, Instruction::Freeze>(Op);
1489}
1490 
1491/// Matches InsertElementInst.
1492template <typename Val_t, typename Elt_t, typename Idx_t>
1493inline ThreeOps_match<Val_t, Elt_t, Idx_t, Instruction::InsertElement>
1494m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx) {
1495  return ThreeOps_match<Val_t, Elt_t, Idx_t, Instruction::InsertElement>(
1496      Val, Elt, Idx);
1497}
1498 
1499/// Matches ExtractElementInst.
1500template <typename Val_t, typename Idx_t>
1501inline TwoOps_match<Val_t, Idx_t, Instruction::ExtractElement>
1502m_ExtractElt(const Val_t &Val, const Idx_t &Idx) {
1503  return TwoOps_match<Val_t, Idx_t, Instruction::ExtractElement>(Val, Idx);
1504}
1505 
1506/// Matches shuffle.
1507template <typename T0, typename T1, typename T2> struct Shuffle_match {
1508  T0 Op1;
1509  T1 Op2;
1510  T2 Mask;
1511 
1512  Shuffle_match(const T0 &Op1, const T1 &Op2, const T2 &Mask)
1513      : Op1(Op1), Op2(Op2), Mask(Mask) {}
1514 
1515  template <typename OpTy> bool match(OpTy *V) {
1516    if (auto *I = dyn_cast<ShuffleVectorInst>(V)) {
1517      return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1)) &&
1518             Mask.match(I->getShuffleMask());
1519    }
1520    return false;
1521  }
1522};
1523 
1524struct m_Mask {
1525  ArrayRef<int> &MaskRef;
1526  m_Mask(ArrayRef<int> &MaskRef) : MaskRef(MaskRef) {}
1527  bool match(ArrayRef<int> Mask) {
1528    MaskRef = Mask;
1529    return true;
1530  }
1531};
1532 
1533struct m_ZeroMask {
1534  bool match(ArrayRef<int> Mask) {
1535    return all_of(Mask, [](int Elem) { return Elem == 0 || Elem == -1; });
1536  }
1537};
1538 
1539struct m_SpecificMask {
1540  ArrayRef<int> &MaskRef;
1541  m_SpecificMask(ArrayRef<int> &MaskRef) : MaskRef(MaskRef) {}
1542  bool match(ArrayRef<int> Mask) { return MaskRef == Mask; }
1543};
1544 
1545struct m_SplatOrUndefMask {
1546  int &SplatIndex;
1547  m_SplatOrUndefMask(int &SplatIndex) : SplatIndex(SplatIndex) {}
1548  bool match(ArrayRef<int> Mask) {
1549    auto First = find_if(Mask, [](int Elem) { return Elem != -1; });
1550    if (First == Mask.end())
1551      return false;
1552    SplatIndex = *First;
1553    return all_of(Mask,
1554                  [First](int Elem) { return Elem == *First || Elem == -1; });
1555  }
1556};
1557 
1558/// Matches ShuffleVectorInst independently of mask value.
1559template <typename V1_t, typename V2_t>
1560inline TwoOps_match<V1_t, V2_t, Instruction::ShuffleVector>
1561m_Shuffle(const V1_t &v1, const V2_t &v2) {
1562  return TwoOps_match<V1_t, V2_t, Instruction::ShuffleVector>(v1, v2);
1563}
1564 
1565template <typename V1_t, typename V2_t, typename Mask_t>
1566inline Shuffle_match<V1_t, V2_t, Mask_t>
1567m_Shuffle(const V1_t &v1, const V2_t &v2, const Mask_t &mask) {
1568  return Shuffle_match<V1_t, V2_t, Mask_t>(v1, v2, mask);
1569}
1570 
1571/// Matches LoadInst.
1572template <typename OpTy>
1573inline OneOps_match<OpTy, Instruction::Load> m_Load(const OpTy &Op) {
1574  return OneOps_match<OpTy, Instruction::Load>(Op);
1575}
1576 
1577/// Matches StoreInst.
1578template <typename ValueOpTy, typename PointerOpTy>
1579inline TwoOps_match<ValueOpTy, PointerOpTy, Instruction::Store>
1580m_Store(const ValueOpTy &ValueOp, const PointerOpTy &PointerOp) {
1581  return TwoOps_match<ValueOpTy, PointerOpTy, Instruction::Store>(ValueOp,
1582                                                                  PointerOp);
1583}
1584 
1585//===----------------------------------------------------------------------===//
1586// Matchers for CastInst classes
1587//
1588 
1589template <typename Op_t, unsigned Opcode> struct CastClass_match {
1590  Op_t Op;
1591 
1592  CastClass_match(const Op_t &OpMatch) : Op(OpMatch) {}
1593 
1594  template <typename OpTy> bool match(OpTy *V) {
1595    if (auto *O = dyn_cast<Operator>(V))
1596      return O->getOpcode() == Opcode && Op.match(O->getOperand(0));
1597    return false;
1598  }
1599};
1600 
1601/// Matches BitCast.
1602template <typename OpTy>
1603inline CastClass_match<OpTy, Instruction::BitCast> m_BitCast(const OpTy &Op) {
1604  return CastClass_match<OpTy, Instruction::BitCast>(Op);
1605}
1606 
1607/// Matches PtrToInt.
1608template <typename OpTy>
1609inline CastClass_match<OpTy, Instruction::PtrToInt> m_PtrToInt(const OpTy &Op) {
1610  return CastClass_match<OpTy, Instruction::PtrToInt>(Op);
1611}
1612 
1613/// Matches IntToPtr.
1614template <typename OpTy>
1615inline CastClass_match<OpTy, Instruction::IntToPtr> m_IntToPtr(const OpTy &Op) {
1616  return CastClass_match<OpTy, Instruction::IntToPtr>(Op);
1617}
1618 
1619/// Matches Trunc.
1620template <typename OpTy>
1621inline CastClass_match<OpTy, Instruction::Trunc> m_Trunc(const OpTy &Op) {
1622  return CastClass_match<OpTy, Instruction::Trunc>(Op);
1623}
1624 
1625template <typename OpTy>
1626inline match_combine_or<CastClass_match<OpTy, Instruction::Trunc>, OpTy>
1627m_TruncOrSelf(const OpTy &Op) {
1628  return m_CombineOr(m_Trunc(Op), Op);
1629}
1630 
1631/// Matches SExt.
1632template <typename OpTy>
1633inline CastClass_match<OpTy, Instruction::SExt> m_SExt(const OpTy &Op) {
1634  return CastClass_match<OpTy, Instruction::SExt>(Op);
1635}
1636 
1637/// Matches ZExt.
1638template <typename OpTy>
1639inline CastClass_match<OpTy, Instruction::ZExt> m_ZExt(const OpTy &Op) {
1640  return CastClass_match<OpTy, Instruction::ZExt>(Op);
1641}
1642 
1643template <typename OpTy>
1644inline match_combine_or<CastClass_match<OpTy, Instruction::ZExt>, OpTy>
1645m_ZExtOrSelf(const OpTy &Op) {
1646  return m_CombineOr(m_ZExt(Op), Op);
1647}
1648 
1649template <typename OpTy>
1650inline match_combine_or<CastClass_match<OpTy, Instruction::SExt>, OpTy>
1651m_SExtOrSelf(const OpTy &Op) {
1652  return m_CombineOr(m_SExt(Op), Op);
1653}
1654 
1655template <typename OpTy>
1656inline match_combine_or<CastClass_match<OpTy, Instruction::ZExt>,
1657                        CastClass_match<OpTy, Instruction::SExt>>
1658m_ZExtOrSExt(const OpTy &Op) {
1659  return m_CombineOr(m_ZExt(Op), m_SExt(Op));
1660}
1661 
1662template <typename OpTy>
1663inline match_combine_or<
1664    match_combine_or<CastClass_match<OpTy, Instruction::ZExt>,
1665                     CastClass_match<OpTy, Instruction::SExt>>,
1666    OpTy>
1667m_ZExtOrSExtOrSelf(const OpTy &Op) {
1668  return m_CombineOr(m_ZExtOrSExt(Op), Op);
1669}
1670 
1671template <typename OpTy>
1672inline CastClass_match<OpTy, Instruction::UIToFP> m_UIToFP(const OpTy &Op) {
1673  return CastClass_match<OpTy, Instruction::UIToFP>(Op);
1674}
1675 
1676template <typename OpTy>
1677inline CastClass_match<OpTy, Instruction::SIToFP> m_SIToFP(const OpTy &Op) {
1678  return CastClass_match<OpTy, Instruction::SIToFP>(Op);
1679}
1680 
1681template <typename OpTy>
1682inline CastClass_match<OpTy, Instruction::FPToUI> m_FPToUI(const OpTy &Op) {
1683  return CastClass_match<OpTy, Instruction::FPToUI>(Op);
1684}
1685 
1686template <typename OpTy>
1687inline CastClass_match<OpTy, Instruction::FPToSI> m_FPToSI(const OpTy &Op) {
1688  return CastClass_match<OpTy, Instruction::FPToSI>(Op);
1689}
1690 
1691template <typename OpTy>
1692inline CastClass_match<OpTy, Instruction::FPTrunc> m_FPTrunc(const OpTy &Op) {
1693  return CastClass_match<OpTy, Instruction::FPTrunc>(Op);
1694}
1695 
1696template <typename OpTy>
1697inline CastClass_match<OpTy, Instruction::FPExt> m_FPExt(const OpTy &Op) {
1698  return CastClass_match<OpTy, Instruction::FPExt>(Op);
1699}
1700 
1701//===----------------------------------------------------------------------===//
1702// Matchers for control flow.
1703//
1704 
1705struct br_match {
1706  BasicBlock *&Succ;
1707 
1708  br_match(BasicBlock *&Succ) : Succ(Succ) {}
1709 
1710  template <typename OpTy> bool match(OpTy *V) {
1711    if (auto *BI = dyn_cast<BranchInst>(V))
1712      if (BI->isUnconditional()) {
1713        Succ = BI->getSuccessor(0);
1714        return true;
1715      }
1716    return false;
1717  }
1718};
1719 
1720inline br_match m_UnconditionalBr(BasicBlock *&Succ) { return br_match(Succ); }
1721 
1722template <typename Cond_t, typename TrueBlock_t, typename FalseBlock_t>
1723struct brc_match {
1724  Cond_t Cond;
1725  TrueBlock_t T;
1726  FalseBlock_t F;
1727 
1728  brc_match(const Cond_t &C, const TrueBlock_t &t, const FalseBlock_t &f)
1729      : Cond(C), T(t), F(f) {}
1730 
1731  template <typename OpTy> bool match(OpTy *V) {
1732    if (auto *BI = dyn_cast<BranchInst>(V))
1733      if (BI->isConditional() && Cond.match(BI->getCondition()))
1734        return T.match(BI->getSuccessor(0)) && F.match(BI->getSuccessor(1));
1735    return false;
1736  }
1737};
1738 
1739template <typename Cond_t>
1740inline brc_match<Cond_t, bind_ty<BasicBlock>, bind_ty<BasicBlock>>
1741m_Br(const Cond_t &C, BasicBlock *&T, BasicBlock *&F) {
1742  return brc_match<Cond_t, bind_ty<BasicBlock>, bind_ty<BasicBlock>>(
1743      C, m_BasicBlock(T), m_BasicBlock(F));
1744}
1745 
1746template <typename Cond_t, typename TrueBlock_t, typename FalseBlock_t>
1747inline brc_match<Cond_t, TrueBlock_t, FalseBlock_t>
1748m_Br(const Cond_t &C, const TrueBlock_t &T, const FalseBlock_t &F) {
1749  return brc_match<Cond_t, TrueBlock_t, FalseBlock_t>(C, T, F);
1750}
1751 
1752//===----------------------------------------------------------------------===//
1753// Matchers for max/min idioms, eg: "select (sgt x, y), x, y" -> smax(x,y).
1754//
1755 
1756template <typename CmpInst_t, typename LHS_t, typename RHS_t, typename Pred_t,
1757          bool Commutable = false>
1758struct MaxMin_match {
1759  using PredType = Pred_t;
1760  LHS_t L;
1761  RHS_t R;
1762 
1763  // The evaluation order is always stable, regardless of Commutability.
1764  // The LHS is always matched first.
1765  MaxMin_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {}
1766 
1767  template <typename OpTy> bool match(OpTy *V) {
1768    if (auto *II = dyn_cast<IntrinsicInst>(V)) {
1769      Intrinsic::ID IID = II->getIntrinsicID();
1770      if ((IID == Intrinsic::smax && Pred_t::match(ICmpInst::ICMP_SGT)) ||
1771          (IID == Intrinsic::smin && Pred_t::match(ICmpInst::ICMP_SLT)) ||
1772          (IID == Intrinsic::umax && Pred_t::match(ICmpInst::ICMP_UGT)) ||
1773          (IID == Intrinsic::umin && Pred_t::match(ICmpInst::ICMP_ULT))) {
1774        Value *LHS = II->getOperand(0), *RHS = II->getOperand(1);
1775        return (L.match(LHS) && R.match(RHS)) ||
1776               (Commutable && L.match(RHS) && R.match(LHS));
1777      }
1778    }
1779    // Look for "(x pred y) ? x : y" or "(x pred y) ? y : x".
1780    auto *SI = dyn_cast<SelectInst>(V);
1781    if (!SI)
1782      return false;
1783    auto *Cmp = dyn_cast<CmpInst_t>(SI->getCondition());
1784    if (!Cmp)
1785      return false;
1786    // At this point we have a select conditioned on a comparison.  Check that
1787    // it is the values returned by the select that are being compared.
1788    auto *TrueVal = SI->getTrueValue();
1789    auto *FalseVal = SI->getFalseValue();
1790    auto *LHS = Cmp->getOperand(0);
1791    auto *RHS = Cmp->getOperand(1);
1792    if ((TrueVal != LHS || FalseVal != RHS) &&
1793        (TrueVal != RHS || FalseVal != LHS))
1794      return false;
1795    typename CmpInst_t::Predicate Pred =
1796        LHS == TrueVal ? Cmp->getPredicate() : Cmp->getInversePredicate();
1797    // Does "(x pred y) ? x : y" represent the desired max/min operation?
1798    if (!Pred_t::match(Pred))
1799      return false;
1800    // It does!  Bind the operands.
1801    return (L.match(LHS) && R.match(RHS)) ||
1802           (Commutable && L.match(RHS) && R.match(LHS));
1803  }
1804};
1805 
1806/// Helper class for identifying signed max predicates.
1807struct smax_pred_ty {
1808  static bool match(ICmpInst::Predicate Pred) {
1809    return Pred == CmpInst::ICMP_SGT || Pred == CmpInst::ICMP_SGE;
1810  }
1811};
1812 
1813/// Helper class for identifying signed min predicates.
1814struct smin_pred_ty {
1815  static bool match(ICmpInst::Predicate Pred) {
1816    return Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_SLE;
1817  }
1818};
1819 
1820/// Helper class for identifying unsigned max predicates.
1821struct umax_pred_ty {
1822  static bool match(ICmpInst::Predicate Pred) {
1823    return Pred == CmpInst::ICMP_UGT || Pred == CmpInst::ICMP_UGE;
1824  }
1825};
1826 
1827/// Helper class for identifying unsigned min predicates.
1828struct umin_pred_ty {
1829  static bool match(ICmpInst::Predicate Pred) {
1830    return Pred == CmpInst::ICMP_ULT || Pred == CmpInst::ICMP_ULE;
1831  }
1832};
1833 
1834/// Helper class for identifying ordered max predicates.
1835struct ofmax_pred_ty {
1836  static bool match(FCmpInst::Predicate Pred) {
1837    return Pred == CmpInst::FCMP_OGT || Pred == CmpInst::FCMP_OGE;
1838  }
1839};
1840 
1841/// Helper class for identifying ordered min predicates.
1842struct ofmin_pred_ty {
1843  static bool match(FCmpInst::Predicate Pred) {
1844    return Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_OLE;
1845  }
1846};
1847 
1848/// Helper class for identifying unordered max predicates.
1849struct ufmax_pred_ty {
1850  static bool match(FCmpInst::Predicate Pred) {
1851    return Pred == CmpInst::FCMP_UGT || Pred == CmpInst::FCMP_UGE;
1852  }
1853};
1854 
1855/// Helper class for identifying unordered min predicates.
1856struct ufmin_pred_ty {
1857  static bool match(FCmpInst::Predicate Pred) {
1858    return Pred == CmpInst::FCMP_ULT || Pred == CmpInst::FCMP_ULE;
1859  }
1860};
1861 
1862template <typename LHS, typename RHS>
1863inline MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty> m_SMax(const LHS &L,
1864                                                             const RHS &R) {
1865  return MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty>(L, R);
1866}
1867 
1868template <typename LHS, typename RHS>
1869inline MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty> m_SMin(const LHS &L,
1870                                                             const RHS &R) {
1871  return MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty>(L, R);
1872}
1873 
1874template <typename LHS, typename RHS>
1875inline MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty> m_UMax(const LHS &L,
1876                                                             const RHS &R) {
1877  return MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty>(L, R);
1878}
1879 
1880template <typename LHS, typename RHS>
1881inline MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty> m_UMin(const LHS &L,
1882                                                             const RHS &R) {
1883  return MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty>(L, R);
1884}
1885 
1886template <typename LHS, typename RHS>
1887inline match_combine_or<
1888    match_combine_or<MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty>,
1889                     MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty>>,
1890    match_combine_or<MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty>,
1891                     MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty>>>
1892m_MaxOrMin(const LHS &L, const RHS &R) {
1893  return m_CombineOr(m_CombineOr(m_SMax(L, R), m_SMin(L, R)),
1894                     m_CombineOr(m_UMax(L, R), m_UMin(L, R)));
1895}
1896 
1897/// Match an 'ordered' floating point maximum function.
1898/// Floating point has one special value 'NaN'. Therefore, there is no total
1899/// order. However, if we can ignore the 'NaN' value (for example, because of a
1900/// 'no-nans-float-math' flag) a combination of a fcmp and select has 'maximum'
1901/// semantics. In the presence of 'NaN' we have to preserve the original
1902/// select(fcmp(ogt/ge, L, R), L, R) semantics matched by this predicate.
1903///
1904///                         max(L, R)  iff L and R are not NaN
1905///  m_OrdFMax(L, R) =      R          iff L or R are NaN
1906template <typename LHS, typename RHS>
1907inline MaxMin_match<FCmpInst, LHS, RHS, ofmax_pred_ty> m_OrdFMax(const LHS &L,
1908                                                                 const RHS &R) {
1909  return MaxMin_match<FCmpInst, LHS, RHS, ofmax_pred_ty>(L, R);
1910}
1911 
1912/// Match an 'ordered' floating point minimum function.
1913/// Floating point has one special value 'NaN'. Therefore, there is no total
1914/// order. However, if we can ignore the 'NaN' value (for example, because of a
1915/// 'no-nans-float-math' flag) a combination of a fcmp and select has 'minimum'
1916/// semantics. In the presence of 'NaN' we have to preserve the original
1917/// select(fcmp(olt/le, L, R), L, R) semantics matched by this predicate.
1918///
1919///                         min(L, R)  iff L and R are not NaN
1920///  m_OrdFMin(L, R) =      R          iff L or R are NaN
1921template <typename LHS, typename RHS>
1922inline MaxMin_match<FCmpInst, LHS, RHS, ofmin_pred_ty> m_OrdFMin(const LHS &L,
1923                                                                 const RHS &R) {
1924  return MaxMin_match<FCmpInst, LHS, RHS, ofmin_pred_ty>(L, R);
1925}
1926 
1927/// Match an 'unordered' floating point maximum function.
1928/// Floating point has one special value 'NaN'. Therefore, there is no total
1929/// order. However, if we can ignore the 'NaN' value (for example, because of a
1930/// 'no-nans-float-math' flag) a combination of a fcmp and select has 'maximum'
1931/// semantics. In the presence of 'NaN' we have to preserve the original
1932/// select(fcmp(ugt/ge, L, R), L, R) semantics matched by this predicate.
1933///
1934///                         max(L, R)  iff L and R are not NaN
1935///  m_UnordFMax(L, R) =    L          iff L or R are NaN
1936template <typename LHS, typename RHS>
1937inline MaxMin_match<FCmpInst, LHS, RHS, ufmax_pred_ty>
1938m_UnordFMax(const LHS &L, const RHS &R) {
1939  return MaxMin_match<FCmpInst, LHS, RHS, ufmax_pred_ty>(L, R);
1940}
1941 
1942/// Match an 'unordered' floating point minimum function.
1943/// Floating point has one special value 'NaN'. Therefore, there is no total
1944/// order. However, if we can ignore the 'NaN' value (for example, because of a
1945/// 'no-nans-float-math' flag) a combination of a fcmp and select has 'minimum'
1946/// semantics. In the presence of 'NaN' we have to preserve the original
1947/// select(fcmp(ult/le, L, R), L, R) semantics matched by this predicate.
1948///
1949///                          min(L, R)  iff L and R are not NaN
1950///  m_UnordFMin(L, R) =     L          iff L or R are NaN
1951template <typename LHS, typename RHS>
1952inline MaxMin_match<FCmpInst, LHS, RHS, ufmin_pred_ty>
1953m_UnordFMin(const LHS &L, const RHS &R) {
1954  return MaxMin_match<FCmpInst, LHS, RHS, ufmin_pred_ty>(L, R);
1955}
1956 
1957//===----------------------------------------------------------------------===//
1958// Matchers for overflow check patterns: e.g. (a + b) u< a, (a ^ -1) <u b
1959// Note that S might be matched to other instructions than AddInst.
1960//
1961 
1962template <typename LHS_t, typename RHS_t, typename Sum_t>
1963struct UAddWithOverflow_match {
1964  LHS_t L;
1965  RHS_t R;
1966  Sum_t S;
1967 
1968  UAddWithOverflow_match(const LHS_t &L, const RHS_t &R, const Sum_t &S)
1969      : L(L), R(R), S(S) {}
1970 
1971  template <typename OpTy> bool match(OpTy *V) {
1972    Value *ICmpLHS, *ICmpRHS;
1973    ICmpInst::Predicate Pred;
1974    if (!m_ICmp(Pred, m_Value(ICmpLHS), m_Value(ICmpRHS)).match(V))
1975      return false;
1976 
1977    Value *AddLHS, *AddRHS;
1978    auto AddExpr = m_Add(m_Value(AddLHS), m_Value(AddRHS));
1979 
1980    // (a + b) u< a, (a + b) u< b
1981    if (Pred == ICmpInst::ICMP_ULT)
1982      if (AddExpr.match(ICmpLHS) && (ICmpRHS == AddLHS || ICmpRHS == AddRHS))
1983        return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpLHS);
1984 
1985    // a >u (a + b), b >u (a + b)
1986    if (Pred == ICmpInst::ICMP_UGT)
1987      if (AddExpr.match(ICmpRHS) && (ICmpLHS == AddLHS || ICmpLHS == AddRHS))
1988        return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpRHS);
1989 
1990    Value *Op1;
1991    auto XorExpr = m_OneUse(m_Xor(m_Value(Op1), m_AllOnes()));
1992    // (a ^ -1) <u b
1993    if (Pred == ICmpInst::ICMP_ULT) {
1994      if (XorExpr.match(ICmpLHS))
1995        return L.match(Op1) && R.match(ICmpRHS) && S.match(ICmpLHS);
1996    }
1997    //  b > u (a ^ -1)
1998    if (Pred == ICmpInst::ICMP_UGT) {
1999      if (XorExpr.match(ICmpRHS))
2000        return L.match(Op1) && R.match(ICmpLHS) && S.match(ICmpRHS);
2001    }
2002 
2003    // Match special-case for increment-by-1.
2004    if (Pred == ICmpInst::ICMP_EQ) {
2005      // (a + 1) == 0
2006      // (1 + a) == 0
2007      if (AddExpr.match(ICmpLHS) && m_ZeroInt().match(ICmpRHS) &&
2008          (m_One().match(AddLHS) || m_One().match(AddRHS)))
2009        return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpLHS);
2010      // 0 == (a + 1)
2011      // 0 == (1 + a)
2012      if (m_ZeroInt().match(ICmpLHS) && AddExpr.match(ICmpRHS) &&
2013          (m_One().match(AddLHS) || m_One().match(AddRHS)))
2014        return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpRHS);
2015    }
2016 
2017    return false;
2018  }
2019};
2020 
2021/// Match an icmp instruction checking for unsigned overflow on addition.
2022///
2023/// S is matched to the addition whose result is being checked for overflow, and
2024/// L and R are matched to the LHS and RHS of S.
2025template <typename LHS_t, typename RHS_t, typename Sum_t>
2026UAddWithOverflow_match<LHS_t, RHS_t, Sum_t>
2027m_UAddWithOverflow(const LHS_t &L, const RHS_t &R, const Sum_t &S) {
2028  return UAddWithOverflow_match<LHS_t, RHS_t, Sum_t>(L, R, S);
2029}
2030 
2031template <typename Opnd_t> struct Argument_match {
2032  unsigned OpI;
2033  Opnd_t Val;
2034 
2035  Argument_match(unsigned OpIdx, const Opnd_t &V) : OpI(OpIdx), Val(V) {}
2036 
2037  template <typename OpTy> bool match(OpTy *V) {
2038    // FIXME: Should likely be switched to use `CallBase`.
2039    if (const auto *CI = dyn_cast<CallInst>(V))
2040      return Val.match(CI->getArgOperand(OpI));
2041    return false;
2042  }
2043};
2044 
2045/// Match an argument.
2046template <unsigned OpI, typename Opnd_t>
2047inline Argument_match<Opnd_t> m_Argument(const Opnd_t &Op) {
2048  return Argument_match<Opnd_t>(OpI, Op);
2049}
2050 
2051/// Intrinsic matchers.
2052struct IntrinsicID_match {
2053  unsigned ID;
2054 
2055  IntrinsicID_match(Intrinsic::ID IntrID) : ID(IntrID) {}
2056 
2057  template <typename OpTy> bool match(OpTy *V) {
2058    if (const auto *CI42.1
'CI' is null
51.1
'CI' is null
42.1
'CI' is null
51.1
'CI' is null
42.1
'CI' is null
51.1
'CI' is null
42.1
'CI' is null
51.1
'CI' is null
42.1
'CI' is null
51.1
'CI' is null
 = dyn_cast<CallInst>(V))
42
←
Assuming 'V' is not a 'CallInst'→
43
←
Taking false branch→
51
←
'V' is not a 'CallInst'→
52
←
Taking false branch→
2059      if (const auto *F = CI->getCalledFunction())
2060        return F->getIntrinsicID() == ID;
2061    return false;
44
←
Returning zero, which participates in a condition later→
53
←
Returning zero, which participates in a condition later→
2062  }
2063};
2064 
2065/// Intrinsic matches are combinations of ID matchers, and argument
2066/// matchers. Higher arity matcher are defined recursively in terms of and-ing
2067/// them with lower arity matchers. Here's some convenient typedefs for up to
2068/// several arguments, and more can be added as needed
2069template <typename T0 = void, typename T1 = void, typename T2 = void,
2070          typename T3 = void, typename T4 = void, typename T5 = void,
2071          typename T6 = void, typename T7 = void, typename T8 = void,
2072          typename T9 = void, typename T10 = void>
2073struct m_Intrinsic_Ty;
2074template <typename T0> struct m_Intrinsic_Ty<T0> {
2075  using Ty = match_combine_and<IntrinsicID_match, Argument_match<T0>>;
2076};
2077template <typename T0, typename T1> struct m_Intrinsic_Ty<T0, T1> {
2078  using Ty =
2079      match_combine_and<typename m_Intrinsic_Ty<T0>::Ty, Argument_match<T1>>;
2080};
2081template <typename T0, typename T1, typename T2>
2082struct m_Intrinsic_Ty<T0, T1, T2> {
2083  using Ty =
2084      match_combine_and<typename m_Intrinsic_Ty<T0, T1>::Ty,
2085                        Argument_match<T2>>;
2086};
2087template <typename T0, typename T1, typename T2, typename T3>
2088struct m_Intrinsic_Ty<T0, T1, T2, T3> {
2089  using Ty =
2090      match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2>::Ty,
2091                        Argument_match<T3>>;
2092};
2093 
2094template <typename T0, typename T1, typename T2, typename T3, typename T4>
2095struct m_Intrinsic_Ty<T0, T1, T2, T3, T4> {
2096  using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2, T3>::Ty,
2097                               Argument_match<T4>>;
2098};
2099 
2100template <typename T0, typename T1, typename T2, typename T3, typename T4, typename T5>
2101struct m_Intrinsic_Ty<T0, T1, T2, T3, T4, T5> {
2102  using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2, T3, T4>::Ty,
2103                               Argument_match<T5>>;
2104};
2105 
2106/// Match intrinsic calls like this:
2107/// m_Intrinsic<Intrinsic::fabs>(m_Value(X))
2108template <Intrinsic::ID IntrID> inline IntrinsicID_match m_Intrinsic() {
2109  return IntrinsicID_match(IntrID);
2110}
2111 
2112template <Intrinsic::ID IntrID, typename T0>
2113inline typename m_Intrinsic_Ty<T0>::Ty m_Intrinsic(const T0 &Op0) {
2114  return m_CombineAnd(m_Intrinsic<IntrID>(), m_Argument<0>(Op0));
2115}
2116 
2117template <Intrinsic::ID IntrID, typename T0, typename T1>
2118inline typename m_Intrinsic_Ty<T0, T1>::Ty m_Intrinsic(const T0 &Op0,
2119                                                       const T1 &Op1) {
2120  return m_CombineAnd(m_Intrinsic<IntrID>(Op0), m_Argument<1>(Op1));
2121}
2122 
2123template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2>
2124inline typename m_Intrinsic_Ty<T0, T1, T2>::Ty
2125m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2) {
2126  return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1), m_Argument<2>(Op2));
2127}
2128 
2129template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2,
2130          typename T3>
2131inline typename m_Intrinsic_Ty<T0, T1, T2, T3>::Ty
2132m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3) {
2133  return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2), m_Argument<3>(Op3));
2134}
2135 
2136template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2,
2137          typename T3, typename T4>
2138inline typename m_Intrinsic_Ty<T0, T1, T2, T3, T4>::Ty
2139m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3,
2140            const T4 &Op4) {
2141  return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2, Op3),
2142                      m_Argument<4>(Op4));
2143}
2144 
2145template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2,
2146          typename T3, typename T4, typename T5>
2147inline typename m_Intrinsic_Ty<T0, T1, T2, T3, T4, T5>::Ty
2148m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3,
2149            const T4 &Op4, const T5 &Op5) {
2150  return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2, Op3, Op4),
2151                      m_Argument<5>(Op5));
2152}
2153 
2154// Helper intrinsic matching specializations.
2155template <typename Opnd0>
2156inline typename m_Intrinsic_Ty<Opnd0>::Ty m_BitReverse(const Opnd0 &Op0) {
2157  return m_Intrinsic<Intrinsic::bitreverse>(Op0);
2158}
2159 
2160template <typename Opnd0>
2161inline typename m_Intrinsic_Ty<Opnd0>::Ty m_BSwap(const Opnd0 &Op0) {
2162  return m_Intrinsic<Intrinsic::bswap>(Op0);
2163}
2164 
2165template <typename Opnd0>
2166inline typename m_Intrinsic_Ty<Opnd0>::Ty m_FAbs(const Opnd0 &Op0) {
2167  return m_Intrinsic<Intrinsic::fabs>(Op0);
2168}
2169 
2170template <typename Opnd0>
2171inline typename m_Intrinsic_Ty<Opnd0>::Ty m_FCanonicalize(const Opnd0 &Op0) {
2172  return m_Intrinsic<Intrinsic::canonicalize>(Op0);
2173}
2174 
2175template <typename Opnd0, typename Opnd1>
2176inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty m_FMin(const Opnd0 &Op0,
2177                                                        const Opnd1 &Op1) {
2178  return m_Intrinsic<Intrinsic::minnum>(Op0, Op1);
2179}
2180 
2181template <typename Opnd0, typename Opnd1>
2182inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty m_FMax(const Opnd0 &Op0,
2183                                                        const Opnd1 &Op1) {
2184  return m_Intrinsic<Intrinsic::maxnum>(Op0, Op1);
2185}
2186 
2187template <typename Opnd0, typename Opnd1, typename Opnd2>
2188inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2>::Ty
2189m_FShl(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2) {
2190  return m_Intrinsic<Intrinsic::fshl>(Op0, Op1, Op2);
2191}
2192 
2193template <typename Opnd0, typename Opnd1, typename Opnd2>
2194inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2>::Ty
2195m_FShr(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2) {
2196  return m_Intrinsic<Intrinsic::fshr>(Op0, Op1, Op2);
2197}
2198 
2199//===----------------------------------------------------------------------===//
2200// Matchers for two-operands operators with the operators in either order
2201//
2202 
2203/// Matches a BinaryOperator with LHS and RHS in either order.
2204template <typename LHS, typename RHS>
2205inline AnyBinaryOp_match<LHS, RHS, true> m_c_BinOp(const LHS &L, const RHS &R) {
2206  return AnyBinaryOp_match<LHS, RHS, true>(L, R);
2207}
2208 
2209/// Matches an ICmp with a predicate over LHS and RHS in either order.
2210/// Swaps the predicate if operands are commuted.
2211template <typename LHS, typename RHS>
2212inline CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate, true>
2213m_c_ICmp(ICmpInst::Predicate &Pred, const LHS &L, const RHS &R) {
2214  return CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate, true>(Pred, L,
2215                                                                       R);
2216}
2217 
2218/// Matches a Add with LHS and RHS in either order.
2219template <typename LHS, typename RHS>
2220inline BinaryOp_match<LHS, RHS, Instruction::Add, true> m_c_Add(const LHS &L,
2221                                                                const RHS &R) {
2222  return BinaryOp_match<LHS, RHS, Instruction::Add, true>(L, R);
2223}
2224 
2225/// Matches a Mul with LHS and RHS in either order.
2226template <typename LHS, typename RHS>
2227inline BinaryOp_match<LHS, RHS, Instruction::Mul, true> m_c_Mul(const LHS &L,
2228                                                                const RHS &R) {
2229  return BinaryOp_match<LHS, RHS, Instruction::Mul, true>(L, R);
2230}
2231 
2232/// Matches an And with LHS and RHS in either order.
2233template <typename LHS, typename RHS>
2234inline BinaryOp_match<LHS, RHS, Instruction::And, true> m_c_And(const LHS &L,
2235                                                                const RHS &R) {
2236  return BinaryOp_match<LHS, RHS, Instruction::And, true>(L, R);
2237}
2238 
2239/// Matches an Or with LHS and RHS in either order.
2240template <typename LHS, typename RHS>
2241inline BinaryOp_match<LHS, RHS, Instruction::Or, true> m_c_Or(const LHS &L,
2242                                                              const RHS &R) {
2243  return BinaryOp_match<LHS, RHS, Instruction::Or, true>(L, R);
2244}
2245 
2246/// Matches an Xor with LHS and RHS in either order.
2247template <typename LHS, typename RHS>
2248inline BinaryOp_match<LHS, RHS, Instruction::Xor, true> m_c_Xor(const LHS &L,
2249                                                                const RHS &R) {
2250  return BinaryOp_match<LHS, RHS, Instruction::Xor, true>(L, R);
2251}
2252 
2253/// Matches a 'Neg' as 'sub 0, V'.
2254template <typename ValTy>
2255inline BinaryOp_match<cst_pred_ty<is_zero_int>, ValTy, Instruction::Sub>
2256m_Neg(const ValTy &V) {
2257  return m_Sub(m_ZeroInt(), V);
2258}
2259 
2260/// Matches a 'Neg' as 'sub nsw 0, V'.
2261template <typename ValTy>
2262inline OverflowingBinaryOp_match<cst_pred_ty<is_zero_int>, ValTy,
2263                                 Instruction::Sub,
2264                                 OverflowingBinaryOperator::NoSignedWrap>
2265m_NSWNeg(const ValTy &V) {
2266  return m_NSWSub(m_ZeroInt(), V);
2267}
2268 
2269/// Matches a 'Not' as 'xor V, -1' or 'xor -1, V'.
2270template <typename ValTy>
2271inline BinaryOp_match<ValTy, cst_pred_ty<is_all_ones>, Instruction::Xor, true>
2272m_Not(const ValTy &V) {
2273  return m_c_Xor(V, m_AllOnes());
2274}
2275 
2276/// Matches an SMin with LHS and RHS in either order.
2277template <typename LHS, typename RHS>
2278inline MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty, true>
2279m_c_SMin(const LHS &L, const RHS &R) {
2280  return MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty, true>(L, R);
2281}
2282/// Matches an SMax with LHS and RHS in either order.
2283template <typename LHS, typename RHS>
2284inline MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty, true>
2285m_c_SMax(const LHS &L, const RHS &R) {
2286  return MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty, true>(L, R);
2287}
2288/// Matches a UMin with LHS and RHS in either order.
2289template <typename LHS, typename RHS>
2290inline MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty, true>
2291m_c_UMin(const LHS &L, const RHS &R) {
2292  return MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty, true>(L, R);
2293}
2294/// Matches a UMax with LHS and RHS in either order.
2295template <typename LHS, typename RHS>
2296inline MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty, true>
2297m_c_UMax(const LHS &L, const RHS &R) {
2298  return MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty, true>(L, R);
2299}
2300 
2301template <typename LHS, typename RHS>
2302inline match_combine_or<
2303    match_combine_or<MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty, true>,
2304                     MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty, true>>,
2305    match_combine_or<MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty, true>,
2306                     MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty, true>>>
2307m_c_MaxOrMin(const LHS &L, const RHS &R) {
2308  return m_CombineOr(m_CombineOr(m_c_SMax(L, R), m_c_SMin(L, R)),
2309                     m_CombineOr(m_c_UMax(L, R), m_c_UMin(L, R)));
2310}
2311 
2312/// Matches FAdd with LHS and RHS in either order.
2313template <typename LHS, typename RHS>
2314inline BinaryOp_match<LHS, RHS, Instruction::FAdd, true>
2315m_c_FAdd(const LHS &L, const RHS &R) {
2316  return BinaryOp_match<LHS, RHS, Instruction::FAdd, true>(L, R);
2317}
2318 
2319/// Matches FMul with LHS and RHS in either order.
2320template <typename LHS, typename RHS>
2321inline BinaryOp_match<LHS, RHS, Instruction::FMul, true>
2322m_c_FMul(const LHS &L, const RHS &R) {
2323  return BinaryOp_match<LHS, RHS, Instruction::FMul, true>(L, R);
2324}
2325 
2326template <typename Opnd_t> struct Signum_match {
2327  Opnd_t Val;
2328  Signum_match(const Opnd_t &V) : Val(V) {}
2329 
2330  template <typename OpTy> bool match(OpTy *V) {
2331    unsigned TypeSize = V->getType()->getScalarSizeInBits();
2332    if (TypeSize == 0)
2333      return false;
2334 
2335    unsigned ShiftWidth = TypeSize - 1;
2336    Value *OpL = nullptr, *OpR = nullptr;
2337 
2338    // This is the representation of signum we match:
2339    //
2340    //  signum(x) == (x >> 63) | (-x >>u 63)
2341    //
2342    // An i1 value is its own signum, so it's correct to match
2343    //
2344    //  signum(x) == (x >> 0)  | (-x >>u 0)
2345    //
2346    // for i1 values.
2347 
2348    auto LHS = m_AShr(m_Value(OpL), m_SpecificInt(ShiftWidth));
2349    auto RHS = m_LShr(m_Neg(m_Value(OpR)), m_SpecificInt(ShiftWidth));
2350    auto Signum = m_Or(LHS, RHS);
2351 
2352    return Signum.match(V) && OpL == OpR && Val.match(OpL);
2353  }
2354};
2355 
2356/// Matches a signum pattern.
2357///
2358/// signum(x) =
2359///      x >  0  ->  1
2360///      x == 0  ->  0
2361///      x <  0  -> -1
2362template <typename Val_t> inline Signum_match<Val_t> m_Signum(const Val_t &V) {
2363  return Signum_match<Val_t>(V);
2364}
2365 
2366template <int Ind, typename Opnd_t> struct ExtractValue_match {
2367  Opnd_t Val;
2368  ExtractValue_match(const Opnd_t &V) : Val(V) {}
2369 
2370  template <typename OpTy> bool match(OpTy *V) {
2371    if (auto *I = dyn_cast<ExtractValueInst>(V)) {
2372      // If Ind is -1, don't inspect indices
2373      if (Ind != -1 &&
2374          !(I->getNumIndices() == 1 && I->getIndices()[0] == (unsigned)Ind))
2375        return false;
2376      return Val.match(I->getAggregateOperand());
2377    }
2378    return false;
2379  }
2380};
2381 
2382/// Match a single index ExtractValue instruction.
2383/// For example m_ExtractValue<1>(...)
2384template <int Ind, typename Val_t>
2385inline ExtractValue_match<Ind, Val_t> m_ExtractValue(const Val_t &V) {
2386  return ExtractValue_match<Ind, Val_t>(V);
2387}
2388 
2389/// Match an ExtractValue instruction with any index.
2390/// For example m_ExtractValue(...)
2391template <typename Val_t>
2392inline ExtractValue_match<-1, Val_t> m_ExtractValue(const Val_t &V) {
2393  return ExtractValue_match<-1, Val_t>(V);
2394}
2395 
2396/// Matcher for a single index InsertValue instruction.
2397template <int Ind, typename T0, typename T1> struct InsertValue_match {
2398  T0 Op0;
2399  T1 Op1;
2400 
2401  InsertValue_match(const T0 &Op0, const T1 &Op1) : Op0(Op0), Op1(Op1) {}
2402 
2403  template <typename OpTy> bool match(OpTy *V) {
2404    if (auto *I = dyn_cast<InsertValueInst>(V)) {
2405      return Op0.match(I->getOperand(0)) && Op1.match(I->getOperand(1)) &&
2406             I->getNumIndices() == 1 && Ind == I->getIndices()[0];
2407    }
2408    return false;
2409  }
2410};
2411 
2412/// Matches a single index InsertValue instruction.
2413template <int Ind, typename Val_t, typename Elt_t>
2414inline InsertValue_match<Ind, Val_t, Elt_t> m_InsertValue(const Val_t &Val,
2415                                                          const Elt_t &Elt) {
2416  return InsertValue_match<Ind, Val_t, Elt_t>(Val, Elt);
2417}
2418 
2419/// Matches patterns for `vscale`. This can either be a call to `llvm.vscale` or
2420/// the constant expression
2421///  `ptrtoint(gep <vscale x 1 x i8>, <vscale x 1 x i8>* null, i32 1>`
2422/// under the right conditions determined by DataLayout.
2423struct VScaleVal_match {
2424private:
2425  template <typename Base, typename Offset>
2426  inline BinaryOp_match<Base, Offset, Instruction::GetElementPtr>
2427  m_OffsetGep(const Base &B, const Offset &O) {
2428    return BinaryOp_match<Base, Offset, Instruction::GetElementPtr>(B, O);
2429  }
2430 
2431public:
2432  const DataLayout &DL;
2433  VScaleVal_match(const DataLayout &DL) : DL(DL) {}
2434 
2435  template <typename ITy> bool match(ITy *V) {
2436    if (m_Intrinsic<Intrinsic::vscale>().match(V))
2437      return true;
2438 
2439    if (m_PtrToInt(m_OffsetGep(m_Zero(), m_SpecificInt(1))).match(V)) {
2440      Type *PtrTy = cast<Operator>(V)->getOperand(0)->getType();
2441      auto *DerefTy = PtrTy->getPointerElementType();
2442      if (isa<ScalableVectorType>(DerefTy) &&
2443          DL.getTypeAllocSizeInBits(DerefTy).getKnownMinSize() == 8)
2444        return true;
2445    }
2446 
2447    return false;
2448  }
2449};
2450 
2451inline VScaleVal_match m_VScale(const DataLayout &DL) {
2452  return VScaleVal_match(DL);
2453}
2454 
2455template <typename LHS, typename RHS, unsigned Opcode>
2456struct LogicalOp_match {
2457  LHS L;
2458  RHS R;
2459 
2460  LogicalOp_match(const LHS &L, const RHS &R) : L(L), R(R) {}
2461 
2462  template <typename T> bool match(T *V) {
2463    if (auto *I = dyn_cast<Instruction>(V)) {
2464      if (!I->getType()->isIntOrIntVectorTy(1))
2465        return false;
2466 
2467      if (I->getOpcode() == Opcode && L.match(I->getOperand(0)) &&
2468          R.match(I->getOperand(1)))
2469        return true;
2470 
2471      if (auto *SI = dyn_cast<SelectInst>(I)) {
2472        if (Opcode == Instruction::And) {
2473          if (const auto *C = dyn_cast<Constant>(SI->getFalseValue()))
2474            if (C->isNullValue() && L.match(SI->getCondition()) &&
2475                R.match(SI->getTrueValue()))
2476              return true;
2477        } else {
2478          assert(Opcode == Instruction::Or)(static_cast <bool> (Opcode == Instruction::Or) ? void (
0) : __assert_fail ("Opcode == Instruction::Or", "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/PatternMatch.h"
, 2478, __extension__ __PRETTY_FUNCTION__));
2479          if (const auto *C = dyn_cast<Constant>(SI->getTrueValue()))
2480            if (C->isOneValue() && L.match(SI->getCondition()) &&
2481                R.match(SI->getFalseValue()))
2482              return true;
2483        }
2484      }
2485    }
2486 
2487    return false;
2488  }
2489};
2490 
2491/// Matches L && R either in the form of L & R or L ? R : false.
2492/// Note that the latter form is poison-blocking.
2493template <typename LHS, typename RHS>
2494inline LogicalOp_match<LHS, RHS, Instruction::And>
2495m_LogicalAnd(const LHS &L, const RHS &R) {
2496  return LogicalOp_match<LHS, RHS, Instruction::And>(L, R);
2497}
2498 
2499/// Matches L && R where L and R are arbitrary values.
2500inline auto m_LogicalAnd() { return m_LogicalAnd(m_Value(), m_Value()); }
2501 
2502/// Matches L || R either in the form of L | R or L ? true : R.
2503/// Note that the latter form is poison-blocking.
2504template <typename LHS, typename RHS>
2505inline LogicalOp_match<LHS, RHS, Instruction::Or>
2506m_LogicalOr(const LHS &L, const RHS &R) {
2507  return LogicalOp_match<LHS, RHS, Instruction::Or>(L, R);
2508}
2509 
2510/// Matches L || R where L and R are arbitrary values.
2511inline auto m_LogicalOr() {
2512  return m_LogicalOr(m_Value(), m_Value());
2513}
2514 
2515} // end namespace PatternMatch
2516} // end namespace llvm
2517 
2518#endif // LLVM_IR_PATTERNMATCH_H

←

/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/Analysis/AliasAnalysis.h

→

1//===- llvm/Analysis/AliasAnalysis.h - Alias Analysis Interface -*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the generic AliasAnalysis interface, which is used as the
10// common interface used by all clients of alias analysis information, and
11// implemented by all alias analysis implementations.  Mod/Ref information is
12// also captured by this interface.
13//
14// Implementations of this interface must implement the various virtual methods,
15// which automatically provides functionality for the entire suite of client
16// APIs.
17//
18// This API identifies memory regions with the MemoryLocation class. The pointer
19// component specifies the base memory address of the region. The Size specifies
20// the maximum size (in address units) of the memory region, or
21// MemoryLocation::UnknownSize if the size is not known. The TBAA tag
22// identifies the "type" of the memory reference; see the
23// TypeBasedAliasAnalysis class for details.
24//
25// Some non-obvious details include:
26//  - Pointers that point to two completely different objects in memory never
27//    alias, regardless of the value of the Size component.
28//  - NoAlias doesn't imply inequal pointers. The most obvious example of this
29//    is two pointers to constant memory. Even if they are equal, constant
30//    memory is never stored to, so there will never be any dependencies.
31//    In this and other situations, the pointers may be both NoAlias and
32//    MustAlias at the same time. The current API can only return one result,
33//    though this is rarely a problem in practice.
34//
35//===----------------------------------------------------------------------===//

37#ifndef LLVM_ANALYSIS_ALIASANALYSIS_H
38#define LLVM_ANALYSIS_ALIASANALYSIS_H

40#include "llvm/ADT/DenseMap.h"
41#include "llvm/ADT/None.h"
42#include "llvm/ADT/Optional.h"
43#include "llvm/ADT/SmallVector.h"
44#include "llvm/Analysis/MemoryLocation.h"
45#include "llvm/IR/PassManager.h"
46#include "llvm/Pass.h"
47#include <cstdint>
48#include <functional>
49#include <memory>
50#include <vector>

52namespace llvm {

54class AnalysisUsage;
55class AtomicCmpXchgInst;
56class BasicAAResult;
57class BasicBlock;
58class CatchPadInst;
59class CatchReturnInst;
60class DominatorTree;
61class FenceInst;
62class Function;
63class InvokeInst;
64class PreservedAnalyses;
65class TargetLibraryInfo;
66class Value;

68/// The possible results of an alias query.
69///
70/// These results are always computed between two MemoryLocation objects as
71/// a query to some alias analysis.
72///
73/// Note that these are unscoped enumerations because we would like to support
74/// implicitly testing a result for the existence of any possible aliasing with
75/// a conversion to bool, but an "enum class" doesn't support this. The
76/// canonical names from the literature are suffixed and unique anyways, and so
77/// they serve as global constants in LLVM for these results.
78///
79/// See docs/AliasAnalysis.html for more information on the specific meanings
80/// of these values.
81class AliasResult {
82private:
static const int OffsetBits = 23;
static const int AliasBits = 8;
static_assert(AliasBits + 1 + OffsetBits <= 32,
              "AliasResult size is intended to be 4 bytes!");

unsigned int Alias : AliasBits;
unsigned int HasOffset : 1;
signed int Offset : OffsetBits;

92public:
enum Kind : uint8_t {
  /// The two locations do not alias at all.
  ///
  /// This value is arranged to convert to false, while all other values
  /// convert to true. This allows a boolean context to convert the result to
  /// a binary flag indicating whether there is the possibility of aliasing.
  NoAlias = 0,
  /// The two locations may or may not alias. This is the least precise
  /// result.
  MayAlias,
  /// The two locations alias, but only due to a partial overlap.
  PartialAlias,
  /// The two locations precisely alias each other.
  MustAlias,
};
static_assert(MustAlias < (1 << AliasBits),
              "Not enough bit field size for the enum!");

explicit AliasResult() = delete;
constexpr AliasResult(const Kind &Alias)
    : Alias(Alias), HasOffset(false), Offset(0) {}

operator Kind() const { return static_cast<Kind>(Alias); }

constexpr bool hasOffset() const { return HasOffset; }
constexpr int32_t getOffset() const {
  assert(HasOffset && "No offset!")(static_cast <bool> (HasOffset && "No offset!")
 ? void (0) : __assert_fail ("HasOffset && \"No offset!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/Analysis/AliasAnalysis.h"
, 119, __extension__ __PRETTY_FUNCTION__));
  return Offset;
}
void setOffset(int32_t NewOffset) {
  if (isInt<OffsetBits>(NewOffset)) {
    HasOffset = true;
    Offset = NewOffset;
  }
}

/// Helper for processing AliasResult for swapped memory location pairs.
void swap(bool DoSwap = true) {
  if (DoSwap && hasOffset())
    setOffset(-getOffset());
}
134};

136static_assert(sizeof(AliasResult) == 4,
            "AliasResult size is intended to be 4 bytes!");

139/// << operator for AliasResult.
140raw_ostream &operator<<(raw_ostream &OS, AliasResult AR);

142/// Flags indicating whether a memory access modifies or references memory.
143///
144/// This is no access at all, a modification, a reference, or both
145/// a modification and a reference. These are specifically structured such that
146/// they form a three bit matrix and bit-tests for 'mod' or 'ref' or 'must'
147/// work with any of the possible values.
148enum class ModRefInfo : uint8_t {
/// Must is provided for completeness, but no routines will return only
/// Must today. See definition of Must below.
Must = 0,
/// The access may reference the value stored in memory,
/// a mustAlias relation was found, and no mayAlias or partialAlias found.
MustRef = 1,
/// The access may modify the value stored in memory,
/// a mustAlias relation was found, and no mayAlias or partialAlias found.
MustMod = 2,
/// The access may reference, modify or both the value stored in memory,
/// a mustAlias relation was found, and no mayAlias or partialAlias found.
MustModRef = MustRef | MustMod,
/// The access neither references nor modifies the value stored in memory.
NoModRef = 4,
/// The access may reference the value stored in memory.
Ref = NoModRef | MustRef,
/// The access may modify the value stored in memory.
Mod = NoModRef | MustMod,
/// The access may reference and may modify the value stored in memory.
ModRef = Ref | Mod,

/// About Must:
/// Must is set in a best effort manner.
/// We usually do not try our best to infer Must, instead it is merely
/// another piece of "free" information that is presented when available.
/// Must set means there was certainly a MustAlias found. For calls,
/// where multiple arguments are checked (argmemonly), this translates to
/// only MustAlias or NoAlias was found.
/// Must is not set for RAR accesses, even if the two locations must
/// alias. The reason is that two read accesses translate to an early return
/// of NoModRef. An additional alias check to set Must may be
/// expensive. Other cases may also not set Must(e.g. callCapturesBefore).
/// We refer to Must being *set* when the most significant bit is *cleared*.
/// Conversely we *clear* Must information by *setting* the Must bit to 1.
183};

185LLVM_NODISCARD[[clang::warn_unused_result]] inline bool isNoModRef(const ModRefInfo MRI) {
return (static_cast<int>(MRI) & static_cast<int>(ModRefInfo::MustModRef)) ==
       static_cast<int>(ModRefInfo::Must);
188}
189LLVM_NODISCARD[[clang::warn_unused_result]] inline bool isModOrRefSet(const ModRefInfo MRI) {
return static_cast<int>(MRI) & static_cast<int>(ModRefInfo::MustModRef);
191}
192LLVM_NODISCARD[[clang::warn_unused_result]] inline bool isModAndRefSet(const ModRefInfo MRI) {
return (static_cast<int>(MRI) & static_cast<int>(ModRefInfo::MustModRef)) ==
       static_cast<int>(ModRefInfo::MustModRef);
195}
196LLVM_NODISCARD[[clang::warn_unused_result]] inline bool isModSet(const ModRefInfo MRI) {
return static_cast<int>(MRI) & static_cast<int>(ModRefInfo::MustMod);
198}
199LLVM_NODISCARD[[clang::warn_unused_result]] inline bool isRefSet(const ModRefInfo MRI) {
return static_cast<int>(MRI) & static_cast<int>(ModRefInfo::MustRef);
201}
202LLVM_NODISCARD[[clang::warn_unused_result]] inline bool isMustSet(const ModRefInfo MRI) {
return !(static_cast<int>(MRI) & static_cast<int>(ModRefInfo::NoModRef));
204}

206LLVM_NODISCARD[[clang::warn_unused_result]] inline ModRefInfo setMod(const ModRefInfo MRI) {
return ModRefInfo(static_cast<int>(MRI) |
                  static_cast<int>(ModRefInfo::MustMod));
209}
210LLVM_NODISCARD[[clang::warn_unused_result]] inline ModRefInfo setRef(const ModRefInfo MRI) {
return ModRefInfo(static_cast<int>(MRI) |
                  static_cast<int>(ModRefInfo::MustRef));
213}
214LLVM_NODISCARD[[clang::warn_unused_result]] inline ModRefInfo setMust(const ModRefInfo MRI) {
return ModRefInfo(static_cast<int>(MRI) &
                  static_cast<int>(ModRefInfo::MustModRef));
217}
218LLVM_NODISCARD[[clang::warn_unused_result]] inline ModRefInfo setModAndRef(const ModRefInfo MRI) {
return ModRefInfo(static_cast<int>(MRI) |
                  static_cast<int>(ModRefInfo::MustModRef));
221}
222LLVM_NODISCARD[[clang::warn_unused_result]] inline ModRefInfo clearMod(const ModRefInfo MRI) {
return ModRefInfo(static_cast<int>(MRI) & static_cast<int>(ModRefInfo::Ref));
224}
225LLVM_NODISCARD[[clang::warn_unused_result]] inline ModRefInfo clearRef(const ModRefInfo MRI) {
return ModRefInfo(static_cast<int>(MRI) & static_cast<int>(ModRefInfo::Mod));
227}
228LLVM_NODISCARD[[clang::warn_unused_result]] inline ModRefInfo clearMust(const ModRefInfo MRI) {
return ModRefInfo(static_cast<int>(MRI) |
                  static_cast<int>(ModRefInfo::NoModRef));
231}
232LLVM_NODISCARD[[clang::warn_unused_result]] inline ModRefInfo unionModRef(const ModRefInfo MRI1,
                                           const ModRefInfo MRI2) {
return ModRefInfo(static_cast<int>(MRI1) | static_cast<int>(MRI2));
235}
236LLVM_NODISCARD[[clang::warn_unused_result]] inline ModRefInfo intersectModRef(const ModRefInfo MRI1,
                                               const ModRefInfo MRI2) {
return ModRefInfo(static_cast<int>(MRI1) & static_cast<int>(MRI2));
239}

241/// The locations at which a function might access memory.
242///
243/// These are primarily used in conjunction with the \c AccessKind bits to
244/// describe both the nature of access and the locations of access for a
245/// function call.
246enum FunctionModRefLocation {
/// Base case is no access to memory.
FMRL_Nowhere = 0,
/// Access to memory via argument pointers.
FMRL_ArgumentPointees = 8,
/// Memory that is inaccessible via LLVM IR.
FMRL_InaccessibleMem = 16,
/// Access to any memory.
FMRL_Anywhere = 32 | FMRL_InaccessibleMem | FMRL_ArgumentPointees
255};

257/// Summary of how a function affects memory in the program.
258///
259/// Loads from constant globals are not considered memory accesses for this
260/// interface. Also, functions may freely modify stack space local to their
261/// invocation without having to report it through these interfaces.
262enum FunctionModRefBehavior {
/// This function does not perform any non-local loads or stores to memory.
///
/// This property corresponds to the GCC 'const' attribute.
/// This property corresponds to the LLVM IR 'readnone' attribute.
/// This property corresponds to the IntrNoMem LLVM intrinsic flag.
FMRB_DoesNotAccessMemory =
    FMRL_Nowhere | static_cast<int>(ModRefInfo::NoModRef),

/// The only memory references in this function (if it has any) are
/// non-volatile loads from objects pointed to by its pointer-typed
/// arguments, with arbitrary offsets.
///
/// This property corresponds to the combination of the IntrReadMem
/// and IntrArgMemOnly LLVM intrinsic flags.
FMRB_OnlyReadsArgumentPointees =
    FMRL_ArgumentPointees | static_cast<int>(ModRefInfo::Ref),

/// The only memory references in this function (if it has any) are
/// non-volatile stores from objects pointed to by its pointer-typed
/// arguments, with arbitrary offsets.
///
/// This property corresponds to the combination of the IntrWriteMem
/// and IntrArgMemOnly LLVM intrinsic flags.
FMRB_OnlyWritesArgumentPointees =
    FMRL_ArgumentPointees | static_cast<int>(ModRefInfo::Mod),

/// The only memory references in this function (if it has any) are
/// non-volatile loads and stores from objects pointed to by its
/// pointer-typed arguments, with arbitrary offsets.
///
/// This property corresponds to the IntrArgMemOnly LLVM intrinsic flag.
FMRB_OnlyAccessesArgumentPointees =
    FMRL_ArgumentPointees | static_cast<int>(ModRefInfo::ModRef),

/// The only memory references in this function (if it has any) are
/// reads of memory that is otherwise inaccessible via LLVM IR.
///
/// This property corresponds to the LLVM IR inaccessiblememonly attribute.
FMRB_OnlyReadsInaccessibleMem =
    FMRL_InaccessibleMem | static_cast<int>(ModRefInfo::Ref),

/// The only memory references in this function (if it has any) are
/// writes to memory that is otherwise inaccessible via LLVM IR.
///
/// This property corresponds to the LLVM IR inaccessiblememonly attribute.
FMRB_OnlyWritesInaccessibleMem =
    FMRL_InaccessibleMem | static_cast<int>(ModRefInfo::Mod),

/// The only memory references in this function (if it has any) are
/// references of memory that is otherwise inaccessible via LLVM IR.
///
/// This property corresponds to the LLVM IR inaccessiblememonly attribute.
FMRB_OnlyAccessesInaccessibleMem =
    FMRL_InaccessibleMem | static_cast<int>(ModRefInfo::ModRef),

/// The function may perform non-volatile loads from objects pointed
/// to by its pointer-typed arguments, with arbitrary offsets, and
/// it may also perform loads of memory that is otherwise
/// inaccessible via LLVM IR.
///
/// This property corresponds to the LLVM IR
/// inaccessiblemem_or_argmemonly attribute.
FMRB_OnlyReadsInaccessibleOrArgMem = FMRL_InaccessibleMem |
                                     FMRL_ArgumentPointees |
                                     static_cast<int>(ModRefInfo::Ref),

/// The function may perform non-volatile stores to objects pointed
/// to by its pointer-typed arguments, with arbitrary offsets, and
/// it may also perform stores of memory that is otherwise
/// inaccessible via LLVM IR.
///
/// This property corresponds to the LLVM IR
/// inaccessiblemem_or_argmemonly attribute.
FMRB_OnlyWritesInaccessibleOrArgMem = FMRL_InaccessibleMem |
                                      FMRL_ArgumentPointees |
                                      static_cast<int>(ModRefInfo::Mod),

/// The function may perform non-volatile loads and stores of objects
/// pointed to by its pointer-typed arguments, with arbitrary offsets, and
/// it may also perform loads and stores of memory that is otherwise
/// inaccessible via LLVM IR.
///
/// This property corresponds to the LLVM IR
/// inaccessiblemem_or_argmemonly attribute.
FMRB_OnlyAccessesInaccessibleOrArgMem = FMRL_InaccessibleMem |
                                        FMRL_ArgumentPointees |
                                        static_cast<int>(ModRefInfo::ModRef),

/// This function does not perform any non-local stores or volatile loads,
/// but may read from any memory location.
///
/// This property corresponds to the GCC 'pure' attribute.
/// This property corresponds to the LLVM IR 'readonly' attribute.
/// This property corresponds to the IntrReadMem LLVM intrinsic flag.
FMRB_OnlyReadsMemory = FMRL_Anywhere | static_cast<int>(ModRefInfo::Ref),

// This function does not read from memory anywhere, but may write to any
// memory location.
//
// This property corresponds to the LLVM IR 'writeonly' attribute.
// This property corresponds to the IntrWriteMem LLVM intrinsic flag.
FMRB_OnlyWritesMemory = FMRL_Anywhere | static_cast<int>(ModRefInfo::Mod),

/// This indicates that the function could not be classified into one of the
/// behaviors above.
FMRB_UnknownModRefBehavior =
    FMRL_Anywhere | static_cast<int>(ModRefInfo::ModRef)
370};

372// Wrapper method strips bits significant only in FunctionModRefBehavior,
373// to obtain a valid ModRefInfo. The benefit of using the wrapper is that if
374// ModRefInfo enum changes, the wrapper can be updated to & with the new enum
375// entry with all bits set to 1.
376LLVM_NODISCARD[[clang::warn_unused_result]] inline ModRefInfo
377createModRefInfo(const FunctionModRefBehavior FMRB) {
return ModRefInfo(FMRB & static_cast<int>(ModRefInfo::ModRef));
379}

381/// Reduced version of MemoryLocation that only stores a pointer and size.
382/// Used for caching AATags independent BasicAA results.
383struct AACacheLoc {
const Value *Ptr;
LocationSize Size;
386};

388template <> struct DenseMapInfo<AACacheLoc> {
static inline AACacheLoc getEmptyKey() {
  return {DenseMapInfo<const Value *>::getEmptyKey(),
          DenseMapInfo<LocationSize>::getEmptyKey()};
}
static inline AACacheLoc getTombstoneKey() {
  return {DenseMapInfo<const Value *>::getTombstoneKey(),
          DenseMapInfo<LocationSize>::getTombstoneKey()};
}
static unsigned getHashValue(const AACacheLoc &Val) {
  return DenseMapInfo<const Value *>::getHashValue(Val.Ptr) ^
         DenseMapInfo<LocationSize>::getHashValue(Val.Size);
}
static bool isEqual(const AACacheLoc &LHS, const AACacheLoc &RHS) {
  return LHS.Ptr == RHS.Ptr && LHS.Size == RHS.Size;
}
404};

406/// This class stores info we want to provide to or retain within an alias
407/// query. By default, the root query is stateless and starts with a freshly
408/// constructed info object. Specific alias analyses can use this query info to
409/// store per-query state that is important for recursive or nested queries to
410/// avoid recomputing. To enable preserving this state across multiple queries
411/// where safe (due to the IR not changing), use a `BatchAAResults` wrapper.
412/// The information stored in an `AAQueryInfo` is currently limitted to the
413/// caches used by BasicAA, but can further be extended to fit other AA needs.
414class AAQueryInfo {
415public:
using LocPair = std::pair<AACacheLoc, AACacheLoc>;
struct CacheEntry {
  AliasResult Result;
  /// Number of times a NoAlias assumption has been used.
  /// 0 for assumptions that have not been used, -1 for definitive results.
  int NumAssumptionUses;
  /// Whether this is a definitive (non-assumption) result.
  bool isDefinitive() const { return NumAssumptionUses < 0; }
};
using AliasCacheT = SmallDenseMap<LocPair, CacheEntry, 8>;
AliasCacheT AliasCache;

using IsCapturedCacheT = SmallDenseMap<const Value *, bool, 8>;
IsCapturedCacheT IsCapturedCache;

/// Query depth used to distinguish recursive queries.
unsigned Depth = 0;

/// How many active NoAlias assumption uses there are.
int NumAssumptionUses = 0;

/// Location pairs for which an assumption based result is currently stored.
/// Used to remove all potentially incorrect results from the cache if an
/// assumption is disproven.
SmallVector<AAQueryInfo::LocPair, 4> AssumptionBasedResults;

AAQueryInfo() : AliasCache(), IsCapturedCache() {}

/// Create a new AAQueryInfo based on this one, but with the cache cleared.
/// This is used for recursive queries across phis, where cache results may
/// not be valid.
AAQueryInfo withEmptyCache() {
  AAQueryInfo NewAAQI;
  NewAAQI.Depth = Depth;
  return NewAAQI;
}
452};

454class BatchAAResults;

456class AAResults {
457public:
// Make these results default constructable and movable. We have to spell
// these out because MSVC won't synthesize them.
AAResults(const TargetLibraryInfo &TLI) : TLI(TLI) {}
AAResults(AAResults &&Arg);
~AAResults();

/// Register a specific AA result.
template <typename AAResultT> void addAAResult(AAResultT &AAResult) {
  // FIXME: We should use a much lighter weight system than the usual
  // polymorphic pattern because we don't own AAResult. It should
  // ideally involve two pointers and no separate allocation.
  AAs.emplace_back(new Model<AAResultT>(AAResult, *this));
}

/// Register a function analysis ID that the results aggregation depends on.
///
/// This is used in the new pass manager to implement the invalidation logic
/// where we must invalidate the results aggregation if any of our component
/// analyses become invalid.
void addAADependencyID(AnalysisKey *ID) { AADeps.push_back(ID); }

/// Handle invalidation events in the new pass manager.
///
/// The aggregation is invalidated if any of the underlying analyses is
/// invalidated.
bool invalidate(Function &F, const PreservedAnalyses &PA,
                FunctionAnalysisManager::Invalidator &Inv);

//===--------------------------------------------------------------------===//
/// \name Alias Queries
/// @{

/// The main low level interface to the alias analysis implementation.
/// Returns an AliasResult indicating whether the two pointers are aliased to
/// each other. This is the interface that must be implemented by specific
/// alias analysis implementations.
AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB);

/// A convenience wrapper around the primary \c alias interface.
AliasResult alias(const Value *V1, LocationSize V1Size, const Value *V2,
                  LocationSize V2Size) {
  return alias(MemoryLocation(V1, V1Size), MemoryLocation(V2, V2Size));
}

/// A convenience wrapper around the primary \c alias interface.
AliasResult alias(const Value *V1, const Value *V2) {
  return alias(MemoryLocation::getBeforeOrAfter(V1),
               MemoryLocation::getBeforeOrAfter(V2));
}

/// A trivial helper function to check to see if the specified pointers are
/// no-alias.
bool isNoAlias(const MemoryLocation &LocA, const MemoryLocation &LocB) {
  return alias(LocA, LocB) == AliasResult::NoAlias;
}

/// A convenience wrapper around the \c isNoAlias helper interface.
bool isNoAlias(const Value *V1, LocationSize V1Size, const Value *V2,
               LocationSize V2Size) {
  return isNoAlias(MemoryLocation(V1, V1Size), MemoryLocation(V2, V2Size));
}

/// A convenience wrapper around the \c isNoAlias helper interface.
bool isNoAlias(const Value *V1, const Value *V2) {
  return isNoAlias(MemoryLocation::getBeforeOrAfter(V1),
                   MemoryLocation::getBeforeOrAfter(V2));
}

/// A trivial helper function to check to see if the specified pointers are
/// must-alias.
bool isMustAlias(const MemoryLocation &LocA, const MemoryLocation &LocB) {
  return alias(LocA, LocB) == AliasResult::MustAlias;
}

/// A convenience wrapper around the \c isMustAlias helper interface.
bool isMustAlias(const Value *V1, const Value *V2) {
  return alias(V1, LocationSize::precise(1), V2, LocationSize::precise(1)) ==
         AliasResult::MustAlias;
}

/// Checks whether the given location points to constant memory, or if
/// \p OrLocal is true whether it points to a local alloca.
bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal = false);

/// A convenience wrapper around the primary \c pointsToConstantMemory
/// interface.
bool pointsToConstantMemory(const Value *P, bool OrLocal = false) {
  return pointsToConstantMemory(MemoryLocation::getBeforeOrAfter(P), OrLocal);
}

/// @}
//===--------------------------------------------------------------------===//
/// \name Simple mod/ref information
/// @{

/// Get the ModRef info associated with a pointer argument of a call. The
/// result's bits are set to indicate the allowed aliasing ModRef kinds. Note
/// that these bits do not necessarily account for the overall behavior of
/// the function, but rather only provide additional per-argument
/// information. This never sets ModRefInfo::Must.
ModRefInfo getArgModRefInfo(const CallBase *Call, unsigned ArgIdx);

/// Return the behavior of the given call site.
FunctionModRefBehavior getModRefBehavior(const CallBase *Call);

/// Return the behavior when calling the given function.
FunctionModRefBehavior getModRefBehavior(const Function *F);

/// Checks if the specified call is known to never read or write memory.
///
/// Note that if the call only reads from known-constant memory, it is also
/// legal to return true. Also, calls that unwind the stack are legal for
/// this predicate.
///
/// Many optimizations (such as CSE and LICM) can be performed on such calls
/// without worrying about aliasing properties, and many calls have this
/// property (e.g. calls to 'sin' and 'cos').
///
/// This property corresponds to the GCC 'const' attribute.
bool doesNotAccessMemory(const CallBase *Call) {
  return getModRefBehavior(Call) == FMRB_DoesNotAccessMemory;
}

/// Checks if the specified function is known to never read or write memory.
///
/// Note that if the function only reads from known-constant memory, it is
/// also legal to return true. Also, function that unwind the stack are legal
/// for this predicate.
///
/// Many optimizations (such as CSE and LICM) can be performed on such calls
/// to such functions without worrying about aliasing properties, and many
/// functions have this property (e.g. 'sin' and 'cos').
///
/// This property corresponds to the GCC 'const' attribute.
bool doesNotAccessMemory(const Function *F) {
  return getModRefBehavior(F) == FMRB_DoesNotAccessMemory;
}

/// Checks if the specified call is known to only read from non-volatile
/// memory (or not access memory at all).
///
/// Calls that unwind the stack are legal for this predicate.
///
/// This property allows many common optimizations to be performed in the
/// absence of interfering store instructions, such as CSE of strlen calls.
///
/// This property corresponds to the GCC 'pure' attribute.
bool onlyReadsMemory(const CallBase *Call) {
  return onlyReadsMemory(getModRefBehavior(Call));
}

/// Checks if the specified function is known to only read from non-volatile
/// memory (or not access memory at all).
///
/// Functions that unwind the stack are legal for this predicate.
///
/// This property allows many common optimizations to be performed in the
/// absence of interfering store instructions, such as CSE of strlen calls.
///
/// This property corresponds to the GCC 'pure' attribute.
bool onlyReadsMemory(const Function *F) {
  return onlyReadsMemory(getModRefBehavior(F));
}

/// Checks if functions with the specified behavior are known to only read
/// from non-volatile memory (or not access memory at all).
static bool onlyReadsMemory(FunctionModRefBehavior MRB) {
  return !isModSet(createModRefInfo(MRB));
61
←
Assuming the condition is true→
62
←
Returning the value 1, which participates in a condition later→
}

/// Checks if functions with the specified behavior are known to only write
/// memory (or not access memory at all).
static bool doesNotReadMemory(FunctionModRefBehavior MRB) {
  return !isRefSet(createModRefInfo(MRB));
}

/// Checks if functions with the specified behavior are known to read and
/// write at most from objects pointed to by their pointer-typed arguments
/// (with arbitrary offsets).
static bool onlyAccessesArgPointees(FunctionModRefBehavior MRB) {
  return !((unsigned)MRB & FMRL_Anywhere & ~FMRL_ArgumentPointees);
66
←
Assuming the condition is true→
67
←
Returning the value 1, which participates in a condition later→
}

/// Checks if functions with the specified behavior are known to potentially
/// read or write from objects pointed to be their pointer-typed arguments
/// (with arbitrary offsets).
static bool doesAccessArgPointees(FunctionModRefBehavior MRB) {
  return isModOrRefSet(createModRefInfo(MRB)) &&
         ((unsigned)MRB & FMRL_ArgumentPointees);
}

/// Checks if functions with the specified behavior are known to read and
/// write at most from memory that is inaccessible from LLVM IR.
static bool onlyAccessesInaccessibleMem(FunctionModRefBehavior MRB) {
  return !((unsigned)MRB & FMRL_Anywhere & ~FMRL_InaccessibleMem);
}

/// Checks if functions with the specified behavior are known to potentially
/// read or write from memory that is inaccessible from LLVM IR.
static bool doesAccessInaccessibleMem(FunctionModRefBehavior MRB) {
  return isModOrRefSet(createModRefInfo(MRB)) &&
           ((unsigned)MRB & FMRL_InaccessibleMem);
}

/// Checks if functions with the specified behavior are known to read and
/// write at most from memory that is inaccessible from LLVM IR or objects
/// pointed to by their pointer-typed arguments (with arbitrary offsets).
static bool onlyAccessesInaccessibleOrArgMem(FunctionModRefBehavior MRB) {
  return !((unsigned)MRB & FMRL_Anywhere &
           ~(FMRL_InaccessibleMem | FMRL_ArgumentPointees));
}

/// getModRefInfo (for call sites) - Return information about whether
/// a particular call site modifies or reads the specified memory location.
ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc);

/// getModRefInfo (for call sites) - A convenience wrapper.
ModRefInfo getModRefInfo(const CallBase *Call, const Value *P,
                         LocationSize Size) {
  return getModRefInfo(Call, MemoryLocation(P, Size));
}

/// getModRefInfo (for loads) - Return information about whether
/// a particular load modifies or reads the specified memory location.
ModRefInfo getModRefInfo(const LoadInst *L, const MemoryLocation &Loc);

/// getModRefInfo (for loads) - A convenience wrapper.
ModRefInfo getModRefInfo(const LoadInst *L, const Value *P,
                         LocationSize Size) {
  return getModRefInfo(L, MemoryLocation(P, Size));
}

/// getModRefInfo (for stores) - Return information about whether
/// a particular store modifies or reads the specified memory location.
ModRefInfo getModRefInfo(const StoreInst *S, const MemoryLocation &Loc);

/// getModRefInfo (for stores) - A convenience wrapper.
ModRefInfo getModRefInfo(const StoreInst *S, const Value *P,
                         LocationSize Size) {
  return getModRefInfo(S, MemoryLocation(P, Size));
}

/// getModRefInfo (for fences) - Return information about whether
/// a particular store modifies or reads the specified memory location.
ModRefInfo getModRefInfo(const FenceInst *S, const MemoryLocation &Loc);

/// getModRefInfo (for fences) - A convenience wrapper.
ModRefInfo getModRefInfo(const FenceInst *S, const Value *P,
                         LocationSize Size) {
  return getModRefInfo(S, MemoryLocation(P, Size));
}

/// getModRefInfo (for cmpxchges) - Return information about whether
/// a particular cmpxchg modifies or reads the specified memory location.
ModRefInfo getModRefInfo(const AtomicCmpXchgInst *CX,
                         const MemoryLocation &Loc);

/// getModRefInfo (for cmpxchges) - A convenience wrapper.
ModRefInfo getModRefInfo(const AtomicCmpXchgInst *CX, const Value *P,
                         LocationSize Size) {
  return getModRefInfo(CX, MemoryLocation(P, Size));
}

/// getModRefInfo (for atomicrmws) - Return information about whether
/// a particular atomicrmw modifies or reads the specified memory location.
ModRefInfo getModRefInfo(const AtomicRMWInst *RMW, const MemoryLocation &Loc);

/// getModRefInfo (for atomicrmws) - A convenience wrapper.
ModRefInfo getModRefInfo(const AtomicRMWInst *RMW, const Value *P,
                         LocationSize Size) {
  return getModRefInfo(RMW, MemoryLocation(P, Size));
}

/// getModRefInfo (for va_args) - Return information about whether
/// a particular va_arg modifies or reads the specified memory location.
ModRefInfo getModRefInfo(const VAArgInst *I, const MemoryLocation &Loc);

/// getModRefInfo (for va_args) - A convenience wrapper.
ModRefInfo getModRefInfo(const VAArgInst *I, const Value *P,
                         LocationSize Size) {
  return getModRefInfo(I, MemoryLocation(P, Size));
}

/// getModRefInfo (for catchpads) - Return information about whether
/// a particular catchpad modifies or reads the specified memory location.
ModRefInfo getModRefInfo(const CatchPadInst *I, const MemoryLocation &Loc);

/// getModRefInfo (for catchpads) - A convenience wrapper.
ModRefInfo getModRefInfo(const CatchPadInst *I, const Value *P,
                         LocationSize Size) {
  return getModRefInfo(I, MemoryLocation(P, Size));
}

/// getModRefInfo (for catchrets) - Return information about whether
/// a particular catchret modifies or reads the specified memory location.
ModRefInfo getModRefInfo(const CatchReturnInst *I, const MemoryLocation &Loc);

/// getModRefInfo (for catchrets) - A convenience wrapper.
ModRefInfo getModRefInfo(const CatchReturnInst *I, const Value *P,
                         LocationSize Size) {
  return getModRefInfo(I, MemoryLocation(P, Size));
}

/// Check whether or not an instruction may read or write the optionally
/// specified memory location.
///
///
/// An instruction that doesn't read or write memory may be trivially LICM'd
/// for example.
///
/// For function calls, this delegates to the alias-analysis specific
/// call-site mod-ref behavior queries. Otherwise it delegates to the specific
/// helpers above.
ModRefInfo getModRefInfo(const Instruction *I,
                         const Optional<MemoryLocation> &OptLoc) {
  AAQueryInfo AAQIP;
  return getModRefInfo(I, OptLoc, AAQIP);
}

/// A convenience wrapper for constructing the memory location.
ModRefInfo getModRefInfo(const Instruction *I, const Value *P,
                         LocationSize Size) {
  return getModRefInfo(I, MemoryLocation(P, Size));
}

/// Return information about whether a call and an instruction may refer to
/// the same memory locations.
ModRefInfo getModRefInfo(Instruction *I, const CallBase *Call);

/// Return information about whether two call sites may refer to the same set
/// of memory locations. See the AA documentation for details:
///   http://llvm.org/docs/AliasAnalysis.html#ModRefInfo
ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2);

/// Return information about whether a particular call site modifies
/// or reads the specified memory location \p MemLoc before instruction \p I
/// in a BasicBlock.
/// Early exits in callCapturesBefore may lead to ModRefInfo::Must not being
/// set.
ModRefInfo callCapturesBefore(const Instruction *I,
                              const MemoryLocation &MemLoc, DominatorTree *DT);

/// A convenience wrapper to synthesize a memory location.
ModRefInfo callCapturesBefore(const Instruction *I, const Value *P,
                              LocationSize Size, DominatorTree *DT) {
  return callCapturesBefore(I, MemoryLocation(P, Size), DT);
}

/// @}
//===--------------------------------------------------------------------===//
/// \name Higher level methods for querying mod/ref information.
/// @{

/// Check if it is possible for execution of the specified basic block to
/// modify the location Loc.
bool canBasicBlockModify(const BasicBlock &BB, const MemoryLocation &Loc);

/// A convenience wrapper synthesizing a memory location.
bool canBasicBlockModify(const BasicBlock &BB, const Value *P,
                         LocationSize Size) {
  return canBasicBlockModify(BB, MemoryLocation(P, Size));
}

/// Check if it is possible for the execution of the specified instructions
/// to mod\ref (according to the mode) the location Loc.
///
/// The instructions to consider are all of the instructions in the range of
/// [I1,I2] INCLUSIVE. I1 and I2 must be in the same basic block.
bool canInstructionRangeModRef(const Instruction &I1, const Instruction &I2,
                               const MemoryLocation &Loc,
                               const ModRefInfo Mode);

/// A convenience wrapper synthesizing a memory location.
bool canInstructionRangeModRef(const Instruction &I1, const Instruction &I2,
                               const Value *Ptr, LocationSize Size,
                               const ModRefInfo Mode) {
  return canInstructionRangeModRef(I1, I2, MemoryLocation(Ptr, Size), Mode);
}

837private:
AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB,
                  AAQueryInfo &AAQI);
bool pointsToConstantMemory(const MemoryLocation &Loc, AAQueryInfo &AAQI,
                            bool OrLocal = false);
ModRefInfo getModRefInfo(Instruction *I, const CallBase *Call2,
                         AAQueryInfo &AAQIP);
ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc,
                         AAQueryInfo &AAQI);
ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2,
                         AAQueryInfo &AAQI);
ModRefInfo getModRefInfo(const VAArgInst *V, const MemoryLocation &Loc,
                         AAQueryInfo &AAQI);
ModRefInfo getModRefInfo(const LoadInst *L, const MemoryLocation &Loc,
                         AAQueryInfo &AAQI);
ModRefInfo getModRefInfo(const StoreInst *S, const MemoryLocation &Loc,
                         AAQueryInfo &AAQI);
ModRefInfo getModRefInfo(const FenceInst *S, const MemoryLocation &Loc,
                         AAQueryInfo &AAQI);
ModRefInfo getModRefInfo(const AtomicCmpXchgInst *CX,
                         const MemoryLocation &Loc, AAQueryInfo &AAQI);
ModRefInfo getModRefInfo(const AtomicRMWInst *RMW, const MemoryLocation &Loc,
                         AAQueryInfo &AAQI);
ModRefInfo getModRefInfo(const CatchPadInst *I, const MemoryLocation &Loc,
                         AAQueryInfo &AAQI);
ModRefInfo getModRefInfo(const CatchReturnInst *I, const MemoryLocation &Loc,
                         AAQueryInfo &AAQI);
ModRefInfo getModRefInfo(const Instruction *I,
                         const Optional<MemoryLocation> &OptLoc,
                         AAQueryInfo &AAQIP);

class Concept;

template <typename T> class Model;

template <typename T> friend class AAResultBase;

const TargetLibraryInfo &TLI;

std::vector<std::unique_ptr<Concept>> AAs;

std::vector<AnalysisKey *> AADeps;

friend class BatchAAResults;
881};

883/// This class is a wrapper over an AAResults, and it is intended to be used
884/// only when there are no IR changes inbetween queries. BatchAAResults is
885/// reusing the same `AAQueryInfo` to preserve the state across queries,
886/// esentially making AA work in "batch mode". The internal state cannot be
887/// cleared, so to go "out-of-batch-mode", the user must either use AAResults,
888/// or create a new BatchAAResults.
889class BatchAAResults {
AAResults &AA;
AAQueryInfo AAQI;

893public:
BatchAAResults(AAResults &AAR) : AA(AAR), AAQI() {}
AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB) {
  return AA.alias(LocA, LocB, AAQI);
}
bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal = false) {
  return AA.pointsToConstantMemory(Loc, AAQI, OrLocal);
}
ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc) {
  return AA.getModRefInfo(Call, Loc, AAQI);
}
ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2) {
  return AA.getModRefInfo(Call1, Call2, AAQI);
}
ModRefInfo getModRefInfo(const Instruction *I,
                         const Optional<MemoryLocation> &OptLoc) {
  return AA.getModRefInfo(I, OptLoc, AAQI);
}
ModRefInfo getModRefInfo(Instruction *I, const CallBase *Call2) {
  return AA.getModRefInfo(I, Call2, AAQI);
}
ModRefInfo getArgModRefInfo(const CallBase *Call, unsigned ArgIdx) {
  return AA.getArgModRefInfo(Call, ArgIdx);
}
FunctionModRefBehavior getModRefBehavior(const CallBase *Call) {
  return AA.getModRefBehavior(Call);
}
bool isMustAlias(const MemoryLocation &LocA, const MemoryLocation &LocB) {
  return alias(LocA, LocB) == AliasResult::MustAlias;
}
bool isMustAlias(const Value *V1, const Value *V2) {
  return alias(MemoryLocation(V1, LocationSize::precise(1)),
               MemoryLocation(V2, LocationSize::precise(1))) ==
         AliasResult::MustAlias;
}
928};

930/// Temporary typedef for legacy code that uses a generic \c AliasAnalysis
931/// pointer or reference.
932using AliasAnalysis = AAResults;

934/// A private abstract base class describing the concept of an individual alias
935/// analysis implementation.
936///
937/// This interface is implemented by any \c Model instantiation. It is also the
938/// interface which a type used to instantiate the model must provide.
939///
940/// All of these methods model methods by the same name in the \c
941/// AAResults class. Only differences and specifics to how the
942/// implementations are called are documented here.
943class AAResults::Concept {
944public:
virtual ~Concept() = 0;

/// An update API used internally by the AAResults to provide
/// a handle back to the top level aggregation.
virtual void setAAResults(AAResults *NewAAR) = 0;

//===--------------------------------------------------------------------===//
/// \name Alias Queries
/// @{

/// The main low level interface to the alias analysis implementation.
/// Returns an AliasResult indicating whether the two pointers are aliased to
/// each other. This is the interface that must be implemented by specific
/// alias analysis implementations.
virtual AliasResult alias(const MemoryLocation &LocA,
                          const MemoryLocation &LocB, AAQueryInfo &AAQI) = 0;

/// Checks whether the given location points to constant memory, or if
/// \p OrLocal is true whether it points to a local alloca.
virtual bool pointsToConstantMemory(const MemoryLocation &Loc,
                                    AAQueryInfo &AAQI, bool OrLocal) = 0;

/// @}
//===--------------------------------------------------------------------===//
/// \name Simple mod/ref information
/// @{

/// Get the ModRef info associated with a pointer argument of a callsite. The
/// result's bits are set to indicate the allowed aliasing ModRef kinds. Note
/// that these bits do not necessarily account for the overall behavior of
/// the function, but rather only provide additional per-argument
/// information.
virtual ModRefInfo getArgModRefInfo(const CallBase *Call,
                                    unsigned ArgIdx) = 0;

/// Return the behavior of the given call site.
virtual FunctionModRefBehavior getModRefBehavior(const CallBase *Call) = 0;

/// Return the behavior when calling the given function.
virtual FunctionModRefBehavior getModRefBehavior(const Function *F) = 0;

/// getModRefInfo (for call sites) - Return information about whether
/// a particular call site modifies or reads the specified memory location.
virtual ModRefInfo getModRefInfo(const CallBase *Call,
                                 const MemoryLocation &Loc,
                                 AAQueryInfo &AAQI) = 0;

/// Return information about whether two call sites may refer to the same set
/// of memory locations. See the AA documentation for details:
///   http://llvm.org/docs/AliasAnalysis.html#ModRefInfo
virtual ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2,
                                 AAQueryInfo &AAQI) = 0;

/// @}
999};

1001/// A private class template which derives from \c Concept and wraps some other
1002/// type.
1003///
1004/// This models the concept by directly forwarding each interface point to the
1005/// wrapped type which must implement a compatible interface. This provides
1006/// a type erased binding.
1007template <typename AAResultT> class AAResults::Model final : public Concept {
AAResultT &Result;

1010public:
explicit Model(AAResultT &Result, AAResults &AAR) : Result(Result) {
  Result.setAAResults(&AAR);
}
~Model() override = default;

void setAAResults(AAResults *NewAAR) override { Result.setAAResults(NewAAR); }

AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB,
                  AAQueryInfo &AAQI) override {
  return Result.alias(LocA, LocB, AAQI);
}

bool pointsToConstantMemory(const MemoryLocation &Loc, AAQueryInfo &AAQI,
                            bool OrLocal) override {
  return Result.pointsToConstantMemory(Loc, AAQI, OrLocal);
}

ModRefInfo getArgModRefInfo(const CallBase *Call, unsigned ArgIdx) override {
  return Result.getArgModRefInfo(Call, ArgIdx);
}

FunctionModRefBehavior getModRefBehavior(const CallBase *Call) override {
  return Result.getModRefBehavior(Call);
}

FunctionModRefBehavior getModRefBehavior(const Function *F) override {
  return Result.getModRefBehavior(F);
}

ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc,
                         AAQueryInfo &AAQI) override {
  return Result.getModRefInfo(Call, Loc, AAQI);
}

ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2,
                         AAQueryInfo &AAQI) override {
  return Result.getModRefInfo(Call1, Call2, AAQI);
}
1049};

1051/// A CRTP-driven "mixin" base class to help implement the function alias
1052/// analysis results concept.
1053///
1054/// Because of the nature of many alias analysis implementations, they often
1055/// only implement a subset of the interface. This base class will attempt to
1056/// implement the remaining portions of the interface in terms of simpler forms
1057/// of the interface where possible, and otherwise provide conservatively
1058/// correct fallback implementations.
1059///
1060/// Implementors of an alias analysis should derive from this CRTP, and then
1061/// override specific methods that they wish to customize. There is no need to
1062/// use virtual anywhere, the CRTP base class does static dispatch to the
1063/// derived type passed into it.
1064template <typename DerivedT> class AAResultBase {
// Expose some parts of the interface only to the AAResults::Model
// for wrapping. Specifically, this allows the model to call our
// setAAResults method without exposing it as a fully public API.
friend class AAResults::Model<DerivedT>;

/// A pointer to the AAResults object that this AAResult is
/// aggregated within. May be null if not aggregated.
AAResults *AAR = nullptr;

/// Helper to dispatch calls back through the derived type.
DerivedT &derived() { return static_cast<DerivedT &>(*this); }

/// A setter for the AAResults pointer, which is used to satisfy the
/// AAResults::Model contract.
void setAAResults(AAResults *NewAAR) { AAR = NewAAR; }

1081protected:
/// This proxy class models a common pattern where we delegate to either the
/// top-level \c AAResults aggregation if one is registered, or to the
/// current result if none are registered.
class AAResultsProxy {
  AAResults *AAR;
  DerivedT &CurrentResult;

public:
  AAResultsProxy(AAResults *AAR, DerivedT &CurrentResult)
      : AAR(AAR), CurrentResult(CurrentResult) {}

  AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB,
                    AAQueryInfo &AAQI) {
    return AAR ? AAR->alias(LocA, LocB, AAQI)
               : CurrentResult.alias(LocA, LocB, AAQI);
  }

  bool pointsToConstantMemory(const MemoryLocation &Loc, AAQueryInfo &AAQI,
                              bool OrLocal) {
    return AAR ? AAR->pointsToConstantMemory(Loc, AAQI, OrLocal)
               : CurrentResult.pointsToConstantMemory(Loc, AAQI, OrLocal);
  }

  ModRefInfo getArgModRefInfo(const CallBase *Call, unsigned ArgIdx) {
    return AAR ? AAR->getArgModRefInfo(Call, ArgIdx)
               : CurrentResult.getArgModRefInfo(Call, ArgIdx);
  }

  FunctionModRefBehavior getModRefBehavior(const CallBase *Call) {
    return AAR ? AAR->getModRefBehavior(Call)
               : CurrentResult.getModRefBehavior(Call);
  }

  FunctionModRefBehavior getModRefBehavior(const Function *F) {
    return AAR ? AAR->getModRefBehavior(F) : CurrentResult.getModRefBehavior(F);
  }

  ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc,
                           AAQueryInfo &AAQI) {
    return AAR ? AAR->getModRefInfo(Call, Loc, AAQI)
               : CurrentResult.getModRefInfo(Call, Loc, AAQI);
  }

  ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2,
                           AAQueryInfo &AAQI) {
    return AAR ? AAR->getModRefInfo(Call1, Call2, AAQI)
               : CurrentResult.getModRefInfo(Call1, Call2, AAQI);
  }
};

explicit AAResultBase() = default;

// Provide all the copy and move constructors so that derived types aren't
// constrained.
AAResultBase(const AAResultBase &Arg) {}
AAResultBase(AAResultBase &&Arg) {}

/// Get a proxy for the best AA result set to query at this time.
///
/// When this result is part of a larger aggregation, this will proxy to that
/// aggregation. When this result is used in isolation, it will just delegate
/// back to the derived class's implementation.
///
/// Note that callers of this need to take considerable care to not cause
/// performance problems when they use this routine, in the case of a large
/// number of alias analyses being aggregated, it can be expensive to walk
/// back across the chain.
AAResultsProxy getBestAAResults() { return AAResultsProxy(AAR, derived()); }

1151public:
AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB,
                  AAQueryInfo &AAQI) {
  return AliasResult::MayAlias;
}

bool pointsToConstantMemory(const MemoryLocation &Loc, AAQueryInfo &AAQI,
                            bool OrLocal) {
  return false;
}

ModRefInfo getArgModRefInfo(const CallBase *Call, unsigned ArgIdx) {
  return ModRefInfo::ModRef;
}

FunctionModRefBehavior getModRefBehavior(const CallBase *Call) {
  return FMRB_UnknownModRefBehavior;
}

FunctionModRefBehavior getModRefBehavior(const Function *F) {
  return FMRB_UnknownModRefBehavior;
}

ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc,
                         AAQueryInfo &AAQI) {
  return ModRefInfo::ModRef;
}

ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2,
                         AAQueryInfo &AAQI) {
  return ModRefInfo::ModRef;
}
1183};

1185/// Return true if this pointer is returned by a noalias function.
1186bool isNoAliasCall(const Value *V);

1188/// Return true if this pointer refers to a distinct and identifiable object.
1189/// This returns true for:
1190///    Global Variables and Functions (but not Global Aliases)
1191///    Allocas
1192///    ByVal and NoAlias Arguments
1193///    NoAlias returns (e.g. calls to malloc)
1194///
1195bool isIdentifiedObject(const Value *V);

1197/// Return true if V is umabigously identified at the function-level.
1198/// Different IdentifiedFunctionLocals can't alias.
1199/// Further, an IdentifiedFunctionLocal can not alias with any function
1200/// arguments other than itself, which is not necessarily true for
1201/// IdentifiedObjects.
1202bool isIdentifiedFunctionLocal(const Value *V);

1204/// A manager for alias analyses.
1205///
1206/// This class can have analyses registered with it and when run, it will run
1207/// all of them and aggregate their results into single AA results interface
1208/// that dispatches across all of the alias analysis results available.
1209///
1210/// Note that the order in which analyses are registered is very significant.
1211/// That is the order in which the results will be aggregated and queried.
1212///
1213/// This manager effectively wraps the AnalysisManager for registering alias
1214/// analyses. When you register your alias analysis with this manager, it will
1215/// ensure the analysis itself is registered with its AnalysisManager.
1216///
1217/// The result of this analysis is only invalidated if one of the particular
1218/// aggregated AA results end up being invalidated. This removes the need to
1219/// explicitly preserve the results of `AAManager`. Note that analyses should no
1220/// longer be registered once the `AAManager` is run.
1221class AAManager : public AnalysisInfoMixin<AAManager> {
1222public:
using Result = AAResults;

/// Register a specific AA result.
template <typename AnalysisT> void registerFunctionAnalysis() {
  ResultGetters.push_back(&getFunctionAAResultImpl<AnalysisT>);
}

/// Register a specific AA result.
template <typename AnalysisT> void registerModuleAnalysis() {
  ResultGetters.push_back(&getModuleAAResultImpl<AnalysisT>);
}

Result run(Function &F, FunctionAnalysisManager &AM);

1237private:
friend AnalysisInfoMixin<AAManager>;

static AnalysisKey Key;

SmallVector<void (*)(Function &F, FunctionAnalysisManager &AM,
                     AAResults &AAResults),
            4> ResultGetters;

template <typename AnalysisT>
static void getFunctionAAResultImpl(Function &F,
                                    FunctionAnalysisManager &AM,
                                    AAResults &AAResults) {
  AAResults.addAAResult(AM.template getResult<AnalysisT>(F));
  AAResults.addAADependencyID(AnalysisT::ID());
}

template <typename AnalysisT>
static void getModuleAAResultImpl(Function &F, FunctionAnalysisManager &AM,
                                  AAResults &AAResults) {
  auto &MAMProxy = AM.getResult<ModuleAnalysisManagerFunctionProxy>(F);
  if (auto *R =
          MAMProxy.template getCachedResult<AnalysisT>(*F.getParent())) {
    AAResults.addAAResult(*R);
    MAMProxy
        .template registerOuterAnalysisInvalidation<AnalysisT, AAManager>();
  }
}
1265};

1267/// A wrapper pass to provide the legacy pass manager access to a suitably
1268/// prepared AAResults object.
1269class AAResultsWrapperPass : public FunctionPass {
std::unique_ptr<AAResults> AAR;

1272public:
static char ID;

AAResultsWrapperPass();

AAResults &getAAResults() { return *AAR; }
const AAResults &getAAResults() const { return *AAR; }

bool runOnFunction(Function &F) override;

void getAnalysisUsage(AnalysisUsage &AU) const override;
1283};

1285/// A wrapper pass for external alias analyses. This just squirrels away the
1286/// callback used to run any analyses and register their results.
1287struct ExternalAAWrapperPass : ImmutablePass {
using CallbackT = std::function<void(Pass &, Function &, AAResults &)>;

CallbackT CB;

static char ID;

ExternalAAWrapperPass();

explicit ExternalAAWrapperPass(CallbackT CB);

void getAnalysisUsage(AnalysisUsage &AU) const override {
  AU.setPreservesAll();
}
1301};

1303FunctionPass *createAAResultsWrapperPass();

1305/// A wrapper pass around a callback which can be used to populate the
1306/// AAResults in the AAResultsWrapperPass from an external AA.
1307///
1308/// The callback provided here will be used each time we prepare an AAResults
1309/// object, and will receive a reference to the function wrapper pass, the
1310/// function, and the AAResults object to populate. This should be used when
1311/// setting up a custom pass pipeline to inject a hook into the AA results.
1312ImmutablePass *createExternalAAWrapperPass(
  std::function<void(Pass &, Function &, AAResults &)> Callback);

1315/// A helper for the legacy pass manager to create a \c AAResults
1316/// object populated to the best of our ability for a particular function when
1317/// inside of a \c ModulePass or a \c CallGraphSCCPass.
1318///
1319/// If a \c ModulePass or a \c CallGraphSCCPass calls \p
1320/// createLegacyPMAAResults, it also needs to call \p addUsedAAAnalyses in \p
1321/// getAnalysisUsage.
1322AAResults createLegacyPMAAResults(Pass &P, Function &F, BasicAAResult &BAR);

1324/// A helper for the legacy pass manager to populate \p AU to add uses to make
1325/// sure the analyses required by \p createLegacyPMAAResults are available.
1326void getAAResultsAnalysisUsage(AnalysisUsage &AU);

1328} // end namespace llvm

1330#endif // LLVM_ANALYSIS_ALIASANALYSIS_H

←

/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/Type.h

1//===- llvm/Type.h - Classes for handling data types ------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the declaration of the Type class.  For more "Type"
10// stuff, look in DerivedTypes.h.
11//
12//===----------------------------------------------------------------------===//

14#ifndef LLVM_IR_TYPE_H
15#define LLVM_IR_TYPE_H

17#include "llvm/ADT/APFloat.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/SmallPtrSet.h"
20#include "llvm/Support/CBindingWrapping.h"
21#include "llvm/Support/Casting.h"
22#include "llvm/Support/Compiler.h"
23#include "llvm/Support/ErrorHandling.h"
24#include "llvm/Support/TypeSize.h"
25#include <cassert>
26#include <cstdint>
27#include <iterator>

29namespace llvm {

31template<class GraphType> struct GraphTraits;
32class IntegerType;
33class LLVMContext;
34class PointerType;
35class raw_ostream;
36class StringRef;

38/// The instances of the Type class are immutable: once they are created,
39/// they are never changed.  Also note that only one instance of a particular
40/// type is ever created.  Thus seeing if two types are equal is a matter of
41/// doing a trivial pointer comparison. To enforce that no two equal instances
42/// are created, Type instances can only be created via static factory methods
43/// in class Type and in derived classes.  Once allocated, Types are never
44/// free'd.
45///
46class Type {
47public:
//===--------------------------------------------------------------------===//
/// Definitions of all of the base types for the Type system.  Based on this
/// value, you can cast to a class defined in DerivedTypes.h.
/// Note: If you add an element to this, you need to add an element to the
/// Type::getPrimitiveType function, or else things will break!
/// Also update LLVMTypeKind and LLVMGetTypeKind () in the C binding.
///
enum TypeID {
  // PrimitiveTypes
  HalfTyID = 0,  ///< 16-bit floating point type
  BFloatTyID,    ///< 16-bit floating point type (7-bit significand)
  FloatTyID,     ///< 32-bit floating point type
  DoubleTyID,    ///< 64-bit floating point type
  X86_FP80TyID,  ///< 80-bit floating point type (X87)
  FP128TyID,     ///< 128-bit floating point type (112-bit significand)
  PPC_FP128TyID, ///< 128-bit floating point type (two 64-bits, PowerPC)
  VoidTyID,      ///< type with no size
  LabelTyID,     ///< Labels
  MetadataTyID,  ///< Metadata
  X86_MMXTyID,   ///< MMX vectors (64 bits, X86 specific)
  X86_AMXTyID,   ///< AMX vectors (8192 bits, X86 specific)
  TokenTyID,     ///< Tokens

  // Derived types... see DerivedTypes.h file.
  IntegerTyID,       ///< Arbitrary bit width integers
  FunctionTyID,      ///< Functions
  PointerTyID,       ///< Pointers
  StructTyID,        ///< Structures
  ArrayTyID,         ///< Arrays
  FixedVectorTyID,   ///< Fixed width SIMD vector type
  ScalableVectorTyID ///< Scalable SIMD vector type
};

81private:
/// This refers to the LLVMContext in which this type was uniqued.
LLVMContext &Context;

TypeID   ID : 8;            // The current base type of this type.
unsigned SubclassData : 24; // Space for subclasses to store data.
                            // Note that this should be synchronized with
                            // MAX_INT_BITS value in IntegerType class.

90protected:
friend class LLVMContextImpl;

explicit Type(LLVMContext &C, TypeID tid)
  : Context(C), ID(tid), SubclassData(0) {}
~Type() = default;

unsigned getSubclassData() const { return SubclassData; }

void setSubclassData(unsigned val) {
  SubclassData = val;
  // Ensure we don't have any accidental truncation.
  assert(getSubclassData() == val && "Subclass data too large for field")(static_cast <bool> (getSubclassData() == val &&
 "Subclass data too large for field") ? void (0) : __assert_fail
 ("getSubclassData() == val && \"Subclass data too large for field\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/Type.h"
, 102, __extension__ __PRETTY_FUNCTION__));
}

/// Keeps track of how many Type*'s there are in the ContainedTys list.
unsigned NumContainedTys = 0;

/// A pointer to the array of Types contained by this Type. For example, this
/// includes the arguments of a function type, the elements of a structure,
/// the pointee of a pointer, the element type of an array, etc. This pointer
/// may be 0 for types that don't contain other types (Integer, Double,
/// Float).
Type * const *ContainedTys = nullptr;

115public:
/// Print the current type.
/// Omit the type details if \p NoDetails == true.
/// E.g., let %st = type { i32, i16 }
/// When \p NoDetails is true, we only print %st.
/// Put differently, \p NoDetails prints the type as if
/// inlined with the operands when printing an instruction.
void print(raw_ostream &O, bool IsForDebug = false,
           bool NoDetails = false) const;

void dump() const;

/// Return the LLVMContext in which this type was uniqued.
LLVMContext &getContext() const { return Context; }

//===--------------------------------------------------------------------===//
// Accessors for working with types.
//

/// Return the type id for the type. This will return one of the TypeID enum
/// elements defined above.
TypeID getTypeID() const { return ID; }

/// Return true if this is 'void'.
bool isVoidTy() const { return getTypeID() == VoidTyID; }

/// Return true if this is 'half', a 16-bit IEEE fp type.
bool isHalfTy() const { return getTypeID() == HalfTyID; }

/// Return true if this is 'bfloat', a 16-bit bfloat type.
bool isBFloatTy() const { return getTypeID() == BFloatTyID; }

/// Return true if this is 'float', a 32-bit IEEE fp type.
bool isFloatTy() const { return getTypeID() == FloatTyID; }

/// Return true if this is 'double', a 64-bit IEEE fp type.
bool isDoubleTy() const { return getTypeID() == DoubleTyID; }

/// Return true if this is x86 long double.
bool isX86_FP80Ty() const { return getTypeID() == X86_FP80TyID; }

/// Return true if this is 'fp128'.
bool isFP128Ty() const { return getTypeID() == FP128TyID; }

/// Return true if this is powerpc long double.
bool isPPC_FP128Ty() const { return getTypeID() == PPC_FP128TyID; }

/// Return true if this is one of the six floating-point types
bool isFloatingPointTy() const {
  return getTypeID() == HalfTyID || getTypeID() == BFloatTyID ||
         getTypeID() == FloatTyID || getTypeID() == DoubleTyID ||
         getTypeID() == X86_FP80TyID || getTypeID() == FP128TyID ||
         getTypeID() == PPC_FP128TyID;
}

const fltSemantics &getFltSemantics() const {
  switch (getTypeID()) {
  case HalfTyID: return APFloat::IEEEhalf();
  case BFloatTyID: return APFloat::BFloat();
  case FloatTyID: return APFloat::IEEEsingle();
  case DoubleTyID: return APFloat::IEEEdouble();
  case X86_FP80TyID: return APFloat::x87DoubleExtended();
  case FP128TyID: return APFloat::IEEEquad();
  case PPC_FP128TyID: return APFloat::PPCDoubleDouble();
  default: llvm_unreachable("Invalid floating type")::llvm::llvm_unreachable_internal("Invalid floating type", "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/Type.h"
, 179);
  }
}

/// Return true if this is X86 MMX.
bool isX86_MMXTy() const { return getTypeID() == X86_MMXTyID; }

/// Return true if this is X86 AMX.
bool isX86_AMXTy() const { return getTypeID() == X86_AMXTyID; }

/// Return true if this is a FP type or a vector of FP.
bool isFPOrFPVectorTy() const { return getScalarType()->isFloatingPointTy(); }

/// Return true if this is 'label'.
bool isLabelTy() const { return getTypeID() == LabelTyID; }

/// Return true if this is 'metadata'.
bool isMetadataTy() const { return getTypeID() == MetadataTyID; }

/// Return true if this is 'token'.
bool isTokenTy() const { return getTypeID() == TokenTyID; }

/// True if this is an instance of IntegerType.
bool isIntegerTy() const { return getTypeID() == IntegerTyID; }

/// Return true if this is an IntegerType of the given width.
bool isIntegerTy(unsigned Bitwidth) const;

/// Return true if this is an integer type or a vector of integer types.
bool isIntOrIntVectorTy() const { return getScalarType()->isIntegerTy(); }

/// Return true if this is an integer type or a vector of integer types of
/// the given width.
bool isIntOrIntVectorTy(unsigned BitWidth) const {
  return getScalarType()->isIntegerTy(BitWidth);
}

/// Return true if this is an integer type or a pointer type.
bool isIntOrPtrTy() const { return isIntegerTy() || isPointerTy(); }

/// True if this is an instance of FunctionType.
bool isFunctionTy() const { return getTypeID() == FunctionTyID; }

/// True if this is an instance of StructType.
bool isStructTy() const { return getTypeID() == StructTyID; }

/// True if this is an instance of ArrayType.
bool isArrayTy() const { return getTypeID() == ArrayTyID; }

/// True if this is an instance of PointerType.
bool isPointerTy() const { return getTypeID() == PointerTyID; }
72
←
Assuming the condition is true→
73
←
Returning the value 1, which participates in a condition later→

/// Return true if this is a pointer type or a vector of pointer types.
bool isPtrOrPtrVectorTy() const { return getScalarType()->isPointerTy(); }

/// True if this is an instance of VectorType.
inline bool isVectorTy() const {
  return getTypeID() == ScalableVectorTyID || getTypeID() == FixedVectorTyID;
}

/// Return true if this type could be converted with a lossless BitCast to
/// type 'Ty'. For example, i8* to i32*. BitCasts are valid for types of the
/// same size only where no re-interpretation of the bits is done.
/// Determine if this type could be losslessly bitcast to Ty
bool canLosslesslyBitCastTo(Type *Ty) const;

/// Return true if this type is empty, that is, it has no elements or all of
/// its elements are empty.
bool isEmptyTy() const;

/// Return true if the type is "first class", meaning it is a valid type for a
/// Value.
bool isFirstClassType() const {
  return getTypeID() != FunctionTyID && getTypeID() != VoidTyID;
}

/// Return true if the type is a valid type for a register in codegen. This
/// includes all first-class types except struct and array types.
bool isSingleValueType() const {
  return isFloatingPointTy() || isX86_MMXTy() || isIntegerTy() ||
         isPointerTy() || isVectorTy() || isX86_AMXTy();
}

/// Return true if the type is an aggregate type. This means it is valid as
/// the first operand of an insertvalue or extractvalue instruction. This
/// includes struct and array types, but does not include vector types.
bool isAggregateType() const {
  return getTypeID() == StructTyID || getTypeID() == ArrayTyID;
}

/// Return true if it makes sense to take the size of this type. To get the
/// actual size for a particular target, it is reasonable to use the
/// DataLayout subsystem to do this.
bool isSized(SmallPtrSetImpl<Type*> *Visited = nullptr) const {
  // If it's a primitive, it is always sized.
  if (getTypeID() == IntegerTyID || isFloatingPointTy() ||
      getTypeID() == PointerTyID || getTypeID() == X86_MMXTyID ||
      getTypeID() == X86_AMXTyID)
    return true;
  // If it is not something that can have a size (e.g. a function or label),
  // it doesn't have a size.
  if (getTypeID() != StructTyID && getTypeID() != ArrayTyID && !isVectorTy())
    return false;
  // Otherwise we have to try harder to decide.
  return isSizedDerivedType(Visited);
}

/// Return the basic size of this type if it is a primitive type. These are
/// fixed by LLVM and are not target-dependent.
/// This will return zero if the type does not have a size or is not a
/// primitive type.
///
/// If this is a scalable vector type, the scalable property will be set and
/// the runtime size will be a positive integer multiple of the base size.
///
/// Note that this may not reflect the size of memory allocated for an
/// instance of the type or the number of bytes that are written when an
/// instance of the type is stored to memory. The DataLayout class provides
/// additional query functions to provide this information.
///
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY__attribute__((__pure__));

/// If this is a vector type, return the getPrimitiveSizeInBits value for the
/// element type. Otherwise return the getPrimitiveSizeInBits value for this
/// type.
unsigned getScalarSizeInBits() const LLVM_READONLY__attribute__((__pure__));

/// Return the width of the mantissa of this type. This is only valid on
/// floating-point types. If the FP type does not have a stable mantissa (e.g.
/// ppc long double), this method returns -1.
int getFPMantissaWidth() const;

/// Return whether the type is IEEE compatible, as defined by the eponymous
/// method in APFloat.
bool isIEEE() const { return APFloat::getZero(getFltSemantics()).isIEEE(); }

/// If this is a vector type, return the element type, otherwise return
/// 'this'.
inline Type *getScalarType() const {
  if (isVectorTy())
    return getContainedType(0);
  return const_cast<Type *>(this);
}

//===--------------------------------------------------------------------===//
// Type Iteration support.
//
using subtype_iterator = Type * const *;

subtype_iterator subtype_begin() const { return ContainedTys; }
subtype_iterator subtype_end() const { return &ContainedTys[NumContainedTys];}
ArrayRef<Type*> subtypes() const {
  return makeArrayRef(subtype_begin(), subtype_end());
}

using subtype_reverse_iterator = std::reverse_iterator<subtype_iterator>;

subtype_reverse_iterator subtype_rbegin() const {
  return subtype_reverse_iterator(subtype_end());
}
subtype_reverse_iterator subtype_rend() const {
  return subtype_reverse_iterator(subtype_begin());
}

/// This method is used to implement the type iterator (defined at the end of
/// the file). For derived types, this returns the types 'contained' in the
/// derived type.
Type *getContainedType(unsigned i) const {
  assert(i < NumContainedTys && "Index out of range!")(static_cast <bool> (i < NumContainedTys && "Index out of range!"
) ? void (0) : __assert_fail ("i < NumContainedTys && \"Index out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/Type.h"
, 347, __extension__ __PRETTY_FUNCTION__));
  return ContainedTys[i];
}

/// Return the number of types in the derived type.
unsigned getNumContainedTypes() const { return NumContainedTys; }

//===--------------------------------------------------------------------===//
// Helper methods corresponding to subclass methods.  This forces a cast to
// the specified subclass and calls its accessor.  "getArrayNumElements" (for
// example) is shorthand for cast<ArrayType>(Ty)->getNumElements().  This is
// only intended to cover the core methods that are frequently used, helper
// methods should not be added here.

inline unsigned getIntegerBitWidth() const;

inline Type *getFunctionParamType(unsigned i) const;
inline unsigned getFunctionNumParams() const;
inline bool isFunctionVarArg() const;

inline StringRef getStructName() const;
inline unsigned getStructNumElements() const;
inline Type *getStructElementType(unsigned N) const;

inline uint64_t getArrayNumElements() const;

Type *getArrayElementType() const {
  assert(getTypeID() == ArrayTyID)(static_cast <bool> (getTypeID() == ArrayTyID) ? void (
0) : __assert_fail ("getTypeID() == ArrayTyID", "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/Type.h"
, 374, __extension__ __PRETTY_FUNCTION__));
  return ContainedTys[0];
}

Type *getPointerElementType() const {
  assert(getTypeID() == PointerTyID)(static_cast <bool> (getTypeID() == PointerTyID) ? void
 (0) : __assert_fail ("getTypeID() == PointerTyID", "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/Type.h"
, 379, __extension__ __PRETTY_FUNCTION__));
  return ContainedTys[0];
}

/// Given vector type, change the element type,
/// whilst keeping the old number of elements.
/// For non-vectors simply returns \p EltTy.
inline Type *getWithNewType(Type *EltTy) const;

/// Given an integer or vector type, change the lane bitwidth to NewBitwidth,
/// whilst keeping the old number of lanes.
inline Type *getWithNewBitWidth(unsigned NewBitWidth) const;

/// Given scalar/vector integer type, returns a type with elements twice as
/// wide as in the original type. For vectors, preserves element count.
inline Type *getExtendedType() const;

/// Get the address space of this pointer or pointer vector type.
inline unsigned getPointerAddressSpace() const;

//===--------------------------------------------------------------------===//
// Static members exported by the Type class itself.  Useful for getting
// instances of Type.
//

/// Return a type based on an identifier.
static Type *getPrimitiveType(LLVMContext &C, TypeID IDNumber);

//===--------------------------------------------------------------------===//
// These are the builtin types that are always available.
//
static Type *getVoidTy(LLVMContext &C);
static Type *getLabelTy(LLVMContext &C);
static Type *getHalfTy(LLVMContext &C);
static Type *getBFloatTy(LLVMContext &C);
static Type *getFloatTy(LLVMContext &C);
static Type *getDoubleTy(LLVMContext &C);
static Type *getMetadataTy(LLVMContext &C);
static Type *getX86_FP80Ty(LLVMContext &C);
static Type *getFP128Ty(LLVMContext &C);
static Type *getPPC_FP128Ty(LLVMContext &C);
static Type *getX86_MMXTy(LLVMContext &C);
static Type *getX86_AMXTy(LLVMContext &C);
static Type *getTokenTy(LLVMContext &C);
static IntegerType *getIntNTy(LLVMContext &C, unsigned N);
static IntegerType *getInt1Ty(LLVMContext &C);
static IntegerType *getInt8Ty(LLVMContext &C);
static IntegerType *getInt16Ty(LLVMContext &C);
static IntegerType *getInt32Ty(LLVMContext &C);
static IntegerType *getInt64Ty(LLVMContext &C);
static IntegerType *getInt128Ty(LLVMContext &C);
template <typename ScalarTy> static Type *getScalarTy(LLVMContext &C) {
  int noOfBits = sizeof(ScalarTy) * CHAR_BIT8;
  if (std::is_integral<ScalarTy>::value) {
    return (Type*) Type::getIntNTy(C, noOfBits);
  } else if (std::is_floating_point<ScalarTy>::value) {
    switch (noOfBits) {
    case 32:
      return Type::getFloatTy(C);
    case 64:
      return Type::getDoubleTy(C);
    }
  }
  llvm_unreachable("Unsupported type in Type::getScalarTy")::llvm::llvm_unreachable_internal("Unsupported type in Type::getScalarTy"
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/Type.h"
, 442);
}
static Type *getFloatingPointTy(LLVMContext &C, const fltSemantics &S) {
  Type *Ty;
  if (&S == &APFloat::IEEEhalf())
    Ty = Type::getHalfTy(C);
  else if (&S == &APFloat::BFloat())
    Ty = Type::getBFloatTy(C);
  else if (&S == &APFloat::IEEEsingle())
    Ty = Type::getFloatTy(C);
  else if (&S == &APFloat::IEEEdouble())
    Ty = Type::getDoubleTy(C);
  else if (&S == &APFloat::x87DoubleExtended())
    Ty = Type::getX86_FP80Ty(C);
  else if (&S == &APFloat::IEEEquad())
    Ty = Type::getFP128Ty(C);
  else {
    assert(&S == &APFloat::PPCDoubleDouble() && "Unknown FP format")(static_cast <bool> (&S == &APFloat::PPCDoubleDouble
() && "Unknown FP format") ? void (0) : __assert_fail
 ("&S == &APFloat::PPCDoubleDouble() && \"Unknown FP format\""
, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/IR/Type.h"
, 459, __extension__ __PRETTY_FUNCTION__));
    Ty = Type::getPPC_FP128Ty(C);
  }
  return Ty;
}

//===--------------------------------------------------------------------===//
// Convenience methods for getting pointer types with one of the above builtin
// types as pointee.
//
static PointerType *getHalfPtrTy(LLVMContext &C, unsigned AS = 0);
static PointerType *getBFloatPtrTy(LLVMContext &C, unsigned AS = 0);
static PointerType *getFloatPtrTy(LLVMContext &C, unsigned AS = 0);
static PointerType *getDoublePtrTy(LLVMContext &C, unsigned AS = 0);
static PointerType *getX86_FP80PtrTy(LLVMContext &C, unsigned AS = 0);
static PointerType *getFP128PtrTy(LLVMContext &C, unsigned AS = 0);
static PointerType *getPPC_FP128PtrTy(LLVMContext &C, unsigned AS = 0);
static PointerType *getX86_MMXPtrTy(LLVMContext &C, unsigned AS = 0);
static PointerType *getX86_AMXPtrTy(LLVMContext &C, unsigned AS = 0);
static PointerType *getIntNPtrTy(LLVMContext &C, unsigned N, unsigned AS = 0);
static PointerType *getInt1PtrTy(LLVMContext &C, unsigned AS = 0);
static PointerType *getInt8PtrTy(LLVMContext &C, unsigned AS = 0);
static PointerType *getInt16PtrTy(LLVMContext &C, unsigned AS = 0);
static PointerType *getInt32PtrTy(LLVMContext &C, unsigned AS = 0);
static PointerType *getInt64PtrTy(LLVMContext &C, unsigned AS = 0);

/// Return a pointer to the current type. This is equivalent to
/// PointerType::get(Foo, AddrSpace).
PointerType *getPointerTo(unsigned AddrSpace = 0) const;

489private:
/// Derived types like structures and arrays are sized iff all of the members
/// of the type are sized as well. Since asking for their size is relatively
/// uncommon, move this operation out-of-line.
bool isSizedDerivedType(SmallPtrSetImpl<Type*> *Visited = nullptr) const;
494};

496// Printing of types.
497inline raw_ostream &operator<<(raw_ostream &OS, const Type &T) {
T.print(OS);
return OS;
500}

502// allow isa<PointerType>(x) to work without DerivedTypes.h included.
503template <> struct isa_impl<PointerType, Type> {
static inline bool doit(const Type &Ty) {
  return Ty.getTypeID() == Type::PointerTyID;
}
507};

509// Create wrappers for C Binding types (see CBindingWrapping.h).
510DEFINE_ISA_CONVERSION_FUNCTIONS(Type, LLVMTypeRef)inline Type *unwrap(LLVMTypeRef P) { return reinterpret_cast<
Type*>(P); } inline LLVMTypeRef wrap(const Type *P) { return
 reinterpret_cast<LLVMTypeRef>(const_cast<Type*>(
P)); } template<typename T> inline T *unwrap(LLVMTypeRef
 P) { return cast<T>(unwrap(P)); }

512/* Specialized opaque type conversions.
*/
514inline Type **unwrap(LLVMTypeRef* Tys) {
return reinterpret_cast<Type**>(Tys);
516}

518inline LLVMTypeRef *wrap(Type **Tys) {
return reinterpret_cast<LLVMTypeRef*>(const_cast<Type**>(Tys));
520}

522} // end namespace llvm

524#endif // LLVM_IR_TYPE_H