/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp

Bug Summary

File:	llvm/lib/Transforms/Scalar/LICM.cpp
Warning:	line 1216, column 41 Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

Show analyzer invocation

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name LICM.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -fno-split-dwarf-inlining -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-12/lib/clang/12.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/build-llvm/lib/Transforms/Scalar -I /build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar -I /build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/build-llvm/include -I /build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-12/lib/clang/12.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/build-llvm/lib/Transforms/Scalar -fdebug-prefix-map=/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac=. -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2020-11-24-172238-38865-1 -x c++ /build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp

/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp

→

1//===-- LICM.cpp - Loop Invariant Code Motion Pass ------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass performs loop invariant code motion, attempting to remove as much
10// code from the body of a loop as possible.  It does this by either hoisting
11// code into the preheader block, or by sinking code to the exit blocks if it is
12// safe.  This pass also promotes must-aliased memory locations in the loop to
13// live in registers, thus hoisting and sinking "invariant" loads and stores.
14//
15// This pass uses alias analysis for two purposes:
16//
17//  1. Moving loop invariant loads and calls out of loops.  If we can determine
18//     that a load or call inside of a loop never aliases anything stored to,
19//     we can hoist it or sink it like any other instruction.
20//  2. Scalar Promotion of Memory - If there is a store instruction inside of
21//     the loop, we try to move the store to happen AFTER the loop instead of
22//     inside of the loop.  This can only happen if a few conditions are true:
23//       A. The pointer stored through is loop invariant
24//       B. There are no stores or loads in the loop which _may_ alias the
25//          pointer.  There are no calls in the loop which mod/ref the pointer.
26//     If these conditions are true, we can promote the loads and stores in the
27//     loop of the pointer to use a temporary alloca'd variable.  We then use
28//     the SSAUpdater to construct the appropriate SSA form for the value.
29//
30//===----------------------------------------------------------------------===//

32#include "llvm/Transforms/Scalar/LICM.h"
33#include "llvm/ADT/SetOperations.h"
34#include "llvm/ADT/Statistic.h"
35#include "llvm/Analysis/AliasAnalysis.h"
36#include "llvm/Analysis/AliasSetTracker.h"
37#include "llvm/Analysis/BasicAliasAnalysis.h"
38#include "llvm/Analysis/BlockFrequencyInfo.h"
39#include "llvm/Analysis/CaptureTracking.h"
40#include "llvm/Analysis/ConstantFolding.h"
41#include "llvm/Analysis/GlobalsModRef.h"
42#include "llvm/Analysis/GuardUtils.h"
43#include "llvm/Analysis/LazyBlockFrequencyInfo.h"
44#include "llvm/Analysis/Loads.h"
45#include "llvm/Analysis/LoopInfo.h"
46#include "llvm/Analysis/LoopIterator.h"
47#include "llvm/Analysis/LoopPass.h"
48#include "llvm/Analysis/MemoryBuiltins.h"
49#include "llvm/Analysis/MemorySSA.h"
50#include "llvm/Analysis/MemorySSAUpdater.h"
51#include "llvm/Analysis/MustExecute.h"
52#include "llvm/Analysis/OptimizationRemarkEmitter.h"
53#include "llvm/Analysis/ScalarEvolution.h"
54#include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
55#include "llvm/Analysis/TargetLibraryInfo.h"
56#include "llvm/Analysis/ValueTracking.h"
57#include "llvm/IR/CFG.h"
58#include "llvm/IR/Constants.h"
59#include "llvm/IR/DataLayout.h"
60#include "llvm/IR/DebugInfoMetadata.h"
61#include "llvm/IR/DerivedTypes.h"
62#include "llvm/IR/Dominators.h"
63#include "llvm/IR/Instructions.h"
64#include "llvm/IR/IntrinsicInst.h"
65#include "llvm/IR/LLVMContext.h"
66#include "llvm/IR/Metadata.h"
67#include "llvm/IR/PatternMatch.h"
68#include "llvm/IR/PredIteratorCache.h"
69#include "llvm/InitializePasses.h"
70#include "llvm/Support/CommandLine.h"
71#include "llvm/Support/Debug.h"
72#include "llvm/Support/raw_ostream.h"
73#include "llvm/Transforms/Scalar.h"
74#include "llvm/Transforms/Scalar/LoopPassManager.h"
75#include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
76#include "llvm/Transforms/Utils/BasicBlockUtils.h"
77#include "llvm/Transforms/Utils/Local.h"
78#include "llvm/Transforms/Utils/LoopUtils.h"
79#include "llvm/Transforms/Utils/SSAUpdater.h"
80#include <algorithm>
81#include <utility>
82using namespace llvm;

84#define DEBUG_TYPE"licm" "licm"

86STATISTIC(NumCreatedBlocks, "Number of blocks created")static llvm::Statistic NumCreatedBlocks = {"licm", "NumCreatedBlocks"
, "Number of blocks created"};
87STATISTIC(NumClonedBranches, "Number of branches cloned")static llvm::Statistic NumClonedBranches = {"licm", "NumClonedBranches"
, "Number of branches cloned"};
88STATISTIC(NumSunk, "Number of instructions sunk out of loop")static llvm::Statistic NumSunk = {"licm", "NumSunk", "Number of instructions sunk out of loop"
};
89STATISTIC(NumHoisted, "Number of instructions hoisted out of loop")static llvm::Statistic NumHoisted = {"licm", "NumHoisted", "Number of instructions hoisted out of loop"
};
90STATISTIC(NumMovedLoads, "Number of load insts hoisted or sunk")static llvm::Statistic NumMovedLoads = {"licm", "NumMovedLoads"
, "Number of load insts hoisted or sunk"};
91STATISTIC(NumMovedCalls, "Number of call insts hoisted or sunk")static llvm::Statistic NumMovedCalls = {"licm", "NumMovedCalls"
, "Number of call insts hoisted or sunk"};
92STATISTIC(NumPromoted, "Number of memory locations promoted to registers")static llvm::Statistic NumPromoted = {"licm", "NumPromoted", "Number of memory locations promoted to registers"
};

94/// Memory promotion is enabled by default.
95static cl::opt<bool>
  DisablePromotion("disable-licm-promotion", cl::Hidden, cl::init(false),
                   cl::desc("Disable memory promotion in LICM pass"));

99static cl::opt<bool> ControlFlowHoisting(
  "licm-control-flow-hoisting", cl::Hidden, cl::init(false),
  cl::desc("Enable control flow (and PHI) hoisting in LICM"));

103static cl::opt<unsigned> HoistSinkColdnessThreshold(
  "licm-coldness-threshold", cl::Hidden, cl::init(4),
  cl::desc("Relative coldness Threshold of hoisting/sinking destination "
           "block for LICM to be considered beneficial"));

108static cl::opt<uint32_t> MaxNumUsesTraversed(
  "licm-max-num-uses-traversed", cl::Hidden, cl::init(8),
  cl::desc("Max num uses visited for identifying load "
           "invariance in loop using invariant start (default = 8)"));

113// Default value of zero implies we use the regular alias set tracker mechanism
114// instead of the cross product using AA to identify aliasing of the memory
115// location we are interested in.
116static cl::opt<int>
117LICMN2Theshold("licm-n2-threshold", cl::Hidden, cl::init(0),
             cl::desc("How many instruction to cross product using AA"));

120// Experimental option to allow imprecision in LICM in pathological cases, in
121// exchange for faster compile. This is to be removed if MemorySSA starts to
122// address the same issue. This flag applies only when LICM uses MemorySSA
123// instead on AliasSetTracker. LICM calls MemorySSAWalker's
124// getClobberingMemoryAccess, up to the value of the Cap, getting perfect
125// accuracy. Afterwards, LICM will call into MemorySSA's getDefiningAccess,
126// which may not be precise, since optimizeUses is capped. The result is
127// correct, but we may not get as "far up" as possible to get which access is
128// clobbering the one queried.
129cl::opt<unsigned> llvm::SetLicmMssaOptCap(
  "licm-mssa-optimization-cap", cl::init(100), cl::Hidden,
  cl::desc("Enable imprecision in LICM in pathological cases, in exchange "
           "for faster compile. Caps the MemorySSA clobbering calls."));

134// Experimentally, memory promotion carries less importance than sinking and
135// hoisting. Limit when we do promotion when using MemorySSA, in order to save
136// compile time.
137cl::opt<unsigned> llvm::SetLicmMssaNoAccForPromotionCap(
  "licm-mssa-max-acc-promotion", cl::init(250), cl::Hidden,
  cl::desc("[LICM & MemorySSA] When MSSA in LICM is disabled, this has no "
           "effect. When MSSA in LICM is enabled, then this is the maximum "
           "number of accesses allowed to be present in a loop in order to "
           "enable memory promotion."));

144static bool inSubLoop(BasicBlock *BB, Loop *CurLoop, LoopInfo *LI);
145static bool isNotUsedOrFreeInLoop(const Instruction &I, const Loop *CurLoop,
                                const LoopSafetyInfo *SafetyInfo,
                                TargetTransformInfo *TTI, bool &FreeInLoop);
148static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop,
                BasicBlock *Dest, ICFLoopSafetyInfo *SafetyInfo,
                MemorySSAUpdater *MSSAU, ScalarEvolution *SE,
                OptimizationRemarkEmitter *ORE);
152static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT,
               BlockFrequencyInfo *BFI, const Loop *CurLoop,
               ICFLoopSafetyInfo *SafetyInfo, MemorySSAUpdater *MSSAU,
               OptimizationRemarkEmitter *ORE);
156static bool isSafeToExecuteUnconditionally(Instruction &Inst,
                                         const DominatorTree *DT,
                                         const Loop *CurLoop,
                                         const LoopSafetyInfo *SafetyInfo,
                                         OptimizationRemarkEmitter *ORE,
                                         const Instruction *CtxI = nullptr);
162static bool pointerInvalidatedByLoop(MemoryLocation MemLoc,
                                   AliasSetTracker *CurAST, Loop *CurLoop,
                                   AAResults *AA);
165static bool pointerInvalidatedByLoopWithMSSA(MemorySSA *MSSA, MemoryUse *MU,
                                           Loop *CurLoop, Instruction &I,
                                           SinkAndHoistLICMFlags &Flags);
168static bool pointerInvalidatedByBlockWithMSSA(BasicBlock &BB, MemorySSA &MSSA,
                                            MemoryUse &MU);
170static Instruction *cloneInstructionInExitBlock(
  Instruction &I, BasicBlock &ExitBlock, PHINode &PN, const LoopInfo *LI,
  const LoopSafetyInfo *SafetyInfo, MemorySSAUpdater *MSSAU);

174static void eraseInstruction(Instruction &I, ICFLoopSafetyInfo &SafetyInfo,
                           AliasSetTracker *AST, MemorySSAUpdater *MSSAU);

177static void moveInstructionBefore(Instruction &I, Instruction &Dest,
                                ICFLoopSafetyInfo &SafetyInfo,
                                MemorySSAUpdater *MSSAU, ScalarEvolution *SE);

181namespace {
182struct LoopInvariantCodeMotion {
bool runOnLoop(Loop *L, AAResults *AA, LoopInfo *LI, DominatorTree *DT,
               BlockFrequencyInfo *BFI, TargetLibraryInfo *TLI,
               TargetTransformInfo *TTI, ScalarEvolution *SE, MemorySSA *MSSA,
               OptimizationRemarkEmitter *ORE);

LoopInvariantCodeMotion(unsigned LicmMssaOptCap,
                        unsigned LicmMssaNoAccForPromotionCap)
    : LicmMssaOptCap(LicmMssaOptCap),
      LicmMssaNoAccForPromotionCap(LicmMssaNoAccForPromotionCap) {}

193private:
unsigned LicmMssaOptCap;
unsigned LicmMssaNoAccForPromotionCap;

std::unique_ptr<AliasSetTracker>
collectAliasInfoForLoop(Loop *L, LoopInfo *LI, AAResults *AA);
std::unique_ptr<AliasSetTracker>
collectAliasInfoForLoopWithMSSA(Loop *L, AAResults *AA,
                                MemorySSAUpdater *MSSAU);
202};

204struct LegacyLICMPass : public LoopPass {
static char ID; // Pass identification, replacement for typeid
LegacyLICMPass(
    unsigned LicmMssaOptCap = SetLicmMssaOptCap,
    unsigned LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap)
    : LoopPass(ID), LICM(LicmMssaOptCap, LicmMssaNoAccForPromotionCap) {
  initializeLegacyLICMPassPass(*PassRegistry::getPassRegistry());
}

bool runOnLoop(Loop *L, LPPassManager &LPM) override {
  if (skipLoop(L))
    return false;

  auto *SE = getAnalysisIfAvailable<ScalarEvolutionWrapperPass>();
  MemorySSA *MSSA = EnableMSSALoopDependency
                        ? (&getAnalysis<MemorySSAWrapperPass>().getMSSA())
                        : nullptr;
  bool hasProfileData = L->getHeader()->getParent()->hasProfileData();
  BlockFrequencyInfo *BFI =
      hasProfileData ? &getAnalysis<LazyBlockFrequencyInfoPass>().getBFI()
                     : nullptr;
  // For the old PM, we can't use OptimizationRemarkEmitter as an analysis
  // pass. Function analyses need to be preserved across loop transformations
  // but ORE cannot be preserved (see comment before the pass definition).
  OptimizationRemarkEmitter ORE(L->getHeader()->getParent());
  return LICM.runOnLoop(
      L, &getAnalysis<AAResultsWrapperPass>().getAAResults(),
      &getAnalysis<LoopInfoWrapperPass>().getLoopInfo(),
      &getAnalysis<DominatorTreeWrapperPass>().getDomTree(), BFI,
      &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(
          *L->getHeader()->getParent()),
      &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(
          *L->getHeader()->getParent()),
      SE ? &SE->getSE() : nullptr, MSSA, &ORE);
}

/// This transformation requires natural loop information & requires that
/// loop preheaders be inserted into the CFG...
///
void getAnalysisUsage(AnalysisUsage &AU) const override {
  AU.addPreserved<DominatorTreeWrapperPass>();
  AU.addPreserved<LoopInfoWrapperPass>();
  AU.addRequired<TargetLibraryInfoWrapperPass>();
  if (EnableMSSALoopDependency) {
    AU.addRequired<MemorySSAWrapperPass>();
    AU.addPreserved<MemorySSAWrapperPass>();
  }
  AU.addRequired<TargetTransformInfoWrapperPass>();
  getLoopAnalysisUsage(AU);
  LazyBlockFrequencyInfoPass::getLazyBFIAnalysisUsage(AU);
  AU.addPreserved<LazyBlockFrequencyInfoPass>();
  AU.addPreserved<LazyBranchProbabilityInfoPass>();
}

258private:
LoopInvariantCodeMotion LICM;
260};
261} // namespace

263PreservedAnalyses LICMPass::run(Loop &L, LoopAnalysisManager &AM,
                              LoopStandardAnalysisResults &AR, LPMUpdater &) {
// For the new PM, we also can't use OptimizationRemarkEmitter as an analysis
// pass.  Function analyses need to be preserved across loop transformations
// but ORE cannot be preserved (see comment before the pass definition).
OptimizationRemarkEmitter ORE(L.getHeader()->getParent());

LoopInvariantCodeMotion LICM(LicmMssaOptCap, LicmMssaNoAccForPromotionCap);
if (!LICM.runOnLoop(&L, &AR.AA, &AR.LI, &AR.DT, AR.BFI, &AR.TLI, &AR.TTI,
                    &AR.SE, AR.MSSA, &ORE))
  return PreservedAnalyses::all();

auto PA = getLoopPassPreservedAnalyses();

PA.preserve<DominatorTreeAnalysis>();
PA.preserve<LoopAnalysis>();
if (AR.MSSA)
  PA.preserve<MemorySSAAnalysis>();

return PA;
283}

285char LegacyLICMPass::ID = 0;
286INITIALIZE_PASS_BEGIN(LegacyLICMPass, "licm", "Loop Invariant Code Motion",static void *initializeLegacyLICMPassPassOnce(PassRegistry &
Registry) {
                    false, false)static void *initializeLegacyLICMPassPassOnce(PassRegistry &
Registry) {
288INITIALIZE_PASS_DEPENDENCY(LoopPass)initializeLoopPassPass(Registry);
289INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)initializeTargetLibraryInfoWrapperPassPass(Registry);
290INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)initializeTargetTransformInfoWrapperPassPass(Registry);
291INITIALIZE_PASS_DEPENDENCY(MemorySSAWrapperPass)initializeMemorySSAWrapperPassPass(Registry);
292INITIALIZE_PASS_DEPENDENCY(LazyBFIPass)initializeLazyBFIPassPass(Registry);
293INITIALIZE_PASS_END(LegacyLICMPass, "licm", "Loop Invariant Code Motion", false,PassInfo *PI = new PassInfo( "Loop Invariant Code Motion", "licm"
, &LegacyLICMPass::ID, PassInfo::NormalCtor_t(callDefaultCtor
<LegacyLICMPass>), false, false); Registry.registerPass
(*PI, true); return PI; } static llvm::once_flag InitializeLegacyLICMPassPassFlag
; void llvm::initializeLegacyLICMPassPass(PassRegistry &Registry
) { llvm::call_once(InitializeLegacyLICMPassPassFlag, initializeLegacyLICMPassPassOnce
, std::ref(Registry)); }
                  false)PassInfo *PI = new PassInfo( "Loop Invariant Code Motion", "licm"
, &LegacyLICMPass::ID, PassInfo::NormalCtor_t(callDefaultCtor
<LegacyLICMPass>), false, false); Registry.registerPass
(*PI, true); return PI; } static llvm::once_flag InitializeLegacyLICMPassPassFlag
; void llvm::initializeLegacyLICMPassPass(PassRegistry &Registry
) { llvm::call_once(InitializeLegacyLICMPassPassFlag, initializeLegacyLICMPassPassOnce
, std::ref(Registry)); }

296Pass *llvm::createLICMPass() { return new LegacyLICMPass(); }
297Pass *llvm::createLICMPass(unsigned LicmMssaOptCap,
                         unsigned LicmMssaNoAccForPromotionCap) {
return new LegacyLICMPass(LicmMssaOptCap, LicmMssaNoAccForPromotionCap);
300}

302llvm::SinkAndHoistLICMFlags::SinkAndHoistLICMFlags(bool IsSink, Loop *L,
                                                 MemorySSA *MSSA)
  : SinkAndHoistLICMFlags(SetLicmMssaOptCap, SetLicmMssaNoAccForPromotionCap,
                          IsSink, L, MSSA) {}

307llvm::SinkAndHoistLICMFlags::SinkAndHoistLICMFlags(
  unsigned LicmMssaOptCap, unsigned LicmMssaNoAccForPromotionCap, bool IsSink,
  Loop *L, MemorySSA *MSSA)
  : LicmMssaOptCap(LicmMssaOptCap),
    LicmMssaNoAccForPromotionCap(LicmMssaNoAccForPromotionCap),
    IsSink(IsSink) {
assert(((L != nullptr) == (MSSA != nullptr)) &&((((L != nullptr) == (MSSA != nullptr)) && "Unexpected values for SinkAndHoistLICMFlags"
) ? static_cast<void> (0) : __assert_fail ("((L != nullptr) == (MSSA != nullptr)) && \"Unexpected values for SinkAndHoistLICMFlags\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 314, __PRETTY_FUNCTION__))
       "Unexpected values for SinkAndHoistLICMFlags")((((L != nullptr) == (MSSA != nullptr)) && "Unexpected values for SinkAndHoistLICMFlags"
) ? static_cast<void> (0) : __assert_fail ("((L != nullptr) == (MSSA != nullptr)) && \"Unexpected values for SinkAndHoistLICMFlags\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 314, __PRETTY_FUNCTION__));
if (!MSSA)
  return;

unsigned AccessCapCount = 0;
for (auto *BB : L->getBlocks())
  if (const auto *Accesses = MSSA->getBlockAccesses(BB))
    for (const auto &MA : *Accesses) {
      (void)MA;
      ++AccessCapCount;
      if (AccessCapCount > LicmMssaNoAccForPromotionCap) {
        NoOfMemAccTooLarge = true;
        return;
      }
    }
329}

331/// Hoist expressions out of the specified loop. Note, alias info for inner
332/// loop is not preserved so it is not a good idea to run LICM multiple
333/// times on one loop.
334bool LoopInvariantCodeMotion::runOnLoop(
  Loop *L, AAResults *AA, LoopInfo *LI, DominatorTree *DT,
  BlockFrequencyInfo *BFI, TargetLibraryInfo *TLI, TargetTransformInfo *TTI,
  ScalarEvolution *SE, MemorySSA *MSSA, OptimizationRemarkEmitter *ORE) {
bool Changed = false;

assert(L->isLCSSAForm(*DT) && "Loop is not in LCSSA form.")((L->isLCSSAForm(*DT) && "Loop is not in LCSSA form."
) ? static_cast<void> (0) : __assert_fail ("L->isLCSSAForm(*DT) && \"Loop is not in LCSSA form.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 340, __PRETTY_FUNCTION__));

// If this loop has metadata indicating that LICM is not to be performed then
// just exit.
if (hasDisableLICMTransformsHint(L)) {
  return false;
}

std::unique_ptr<AliasSetTracker> CurAST;
std::unique_ptr<MemorySSAUpdater> MSSAU;
std::unique_ptr<SinkAndHoistLICMFlags> Flags;

if (!MSSA) {
  LLVM_DEBUG(dbgs() << "LICM: Using Alias Set Tracker.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM: Using Alias Set Tracker.\n"
; } } while (false);
  CurAST = collectAliasInfoForLoop(L, LI, AA);
  Flags = std::make_unique<SinkAndHoistLICMFlags>(
      LicmMssaOptCap, LicmMssaNoAccForPromotionCap, /*IsSink=*/true);
} else {
  LLVM_DEBUG(dbgs() << "LICM: Using MemorySSA.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM: Using MemorySSA.\n"; } } while
 (false);
  MSSAU = std::make_unique<MemorySSAUpdater>(MSSA);
  Flags = std::make_unique<SinkAndHoistLICMFlags>(
      LicmMssaOptCap, LicmMssaNoAccForPromotionCap, /*IsSink=*/true, L, MSSA);
}

// Get the preheader block to move instructions into...
BasicBlock *Preheader = L->getLoopPreheader();

// Compute loop safety information.
ICFLoopSafetyInfo SafetyInfo;
SafetyInfo.computeLoopSafetyInfo(L);

// We want to visit all of the instructions in this loop... that are not parts
// of our subloops (they have already had their invariants hoisted out of
// their loop, into this loop, so there is no need to process the BODIES of
// the subloops).
//
// Traverse the body of the loop in depth first order on the dominator tree so
// that we are guaranteed to see definitions before we see uses.  This allows
// us to sink instructions in one pass, without iteration.  After sinking
// instructions, we perform another pass to hoist them out of the loop.
if (L->hasDedicatedExits())
  Changed |=
      sinkRegion(DT->getNode(L->getHeader()), AA, LI, DT, BFI, TLI, TTI, L,
                 CurAST.get(), MSSAU.get(), &SafetyInfo, *Flags.get(), ORE);
Flags->setIsSink(false);
if (Preheader)
  Changed |= hoistRegion(DT->getNode(L->getHeader()), AA, LI, DT, BFI, TLI, L,
                         CurAST.get(), MSSAU.get(), SE, &SafetyInfo,
                         *Flags.get(), ORE);

// Now that all loop invariants have been removed from the loop, promote any
// memory references to scalars that we can.
// Don't sink stores from loops without dedicated block exits. Exits
// containing indirect branches are not transformed by loop simplify,
// make sure we catch that. An additional load may be generated in the
// preheader for SSA updater, so also avoid sinking when no preheader
// is available.
if (!DisablePromotion && Preheader && L->hasDedicatedExits() &&
    !Flags->tooManyMemoryAccesses()) {
  // Figure out the loop exits and their insertion points
  SmallVector<BasicBlock *, 8> ExitBlocks;
  L->getUniqueExitBlocks(ExitBlocks);

  // We can't insert into a catchswitch.
  bool HasCatchSwitch = llvm::any_of(ExitBlocks, [](BasicBlock *Exit) {
    return isa<CatchSwitchInst>(Exit->getTerminator());
  });

  if (!HasCatchSwitch) {
    SmallVector<Instruction *, 8> InsertPts;
    SmallVector<MemoryAccess *, 8> MSSAInsertPts;
    InsertPts.reserve(ExitBlocks.size());
    if (MSSAU)
      MSSAInsertPts.reserve(ExitBlocks.size());
    for (BasicBlock *ExitBlock : ExitBlocks) {
      InsertPts.push_back(&*ExitBlock->getFirstInsertionPt());
      if (MSSAU)
        MSSAInsertPts.push_back(nullptr);
    }

    PredIteratorCache PIC;

    bool Promoted = false;

    // Build an AST using MSSA.
    if (!CurAST.get())
      CurAST = collectAliasInfoForLoopWithMSSA(L, AA, MSSAU.get());

    // Loop over all of the alias sets in the tracker object.
    for (AliasSet &AS : *CurAST) {
      // We can promote this alias set if it has a store, if it is a "Must"
      // alias set, if the pointer is loop invariant, and if we are not
      // eliminating any volatile loads or stores.
      if (AS.isForwardingAliasSet() || !AS.isMod() || !AS.isMustAlias() ||
          !L->isLoopInvariant(AS.begin()->getValue()))
        continue;

      assert(((!AS.empty() && "Must alias set should have at least one pointer element in it!"
) ? static_cast<void> (0) : __assert_fail ("!AS.empty() && \"Must alias set should have at least one pointer element in it!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 439, __PRETTY_FUNCTION__))
          !AS.empty() &&((!AS.empty() && "Must alias set should have at least one pointer element in it!"
) ? static_cast<void> (0) : __assert_fail ("!AS.empty() && \"Must alias set should have at least one pointer element in it!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 439, __PRETTY_FUNCTION__))
          "Must alias set should have at least one pointer element in it!")((!AS.empty() && "Must alias set should have at least one pointer element in it!"
) ? static_cast<void> (0) : __assert_fail ("!AS.empty() && \"Must alias set should have at least one pointer element in it!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 439, __PRETTY_FUNCTION__));

      SmallSetVector<Value *, 8> PointerMustAliases;
      for (const auto &ASI : AS)
        PointerMustAliases.insert(ASI.getValue());

      Promoted |= promoteLoopAccessesToScalars(
          PointerMustAliases, ExitBlocks, InsertPts, MSSAInsertPts, PIC, LI,
          DT, TLI, L, CurAST.get(), MSSAU.get(), &SafetyInfo, ORE);
    }

    // Once we have promoted values across the loop body we have to
    // recursively reform LCSSA as any nested loop may now have values defined
    // within the loop used in the outer loop.
    // FIXME: This is really heavy handed. It would be a bit better to use an
    // SSAUpdater strategy during promotion that was LCSSA aware and reformed
    // it as it went.
    if (Promoted)
      formLCSSARecursively(*L, *DT, LI, SE);

    Changed |= Promoted;
  }
}

// Check that neither this loop nor its parent have had LCSSA broken. LICM is
// specifically moving instructions across the loop boundary and so it is
// especially in need of sanity checking here.
assert(L->isLCSSAForm(*DT) && "Loop not left in LCSSA form after LICM!")((L->isLCSSAForm(*DT) && "Loop not left in LCSSA form after LICM!"
) ? static_cast<void> (0) : __assert_fail ("L->isLCSSAForm(*DT) && \"Loop not left in LCSSA form after LICM!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 466, __PRETTY_FUNCTION__));
assert((L->isOutermost() || L->getParentLoop()->isLCSSAForm(*DT)) &&(((L->isOutermost() || L->getParentLoop()->isLCSSAForm
(*DT)) && "Parent loop not left in LCSSA form after LICM!"
) ? static_cast<void> (0) : __assert_fail ("(L->isOutermost() || L->getParentLoop()->isLCSSAForm(*DT)) && \"Parent loop not left in LCSSA form after LICM!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 468, __PRETTY_FUNCTION__))
       "Parent loop not left in LCSSA form after LICM!")(((L->isOutermost() || L->getParentLoop()->isLCSSAForm
(*DT)) && "Parent loop not left in LCSSA form after LICM!"
) ? static_cast<void> (0) : __assert_fail ("(L->isOutermost() || L->getParentLoop()->isLCSSAForm(*DT)) && \"Parent loop not left in LCSSA form after LICM!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 468, __PRETTY_FUNCTION__));

if (MSSAU.get() && VerifyMemorySSA)
  MSSAU->getMemorySSA()->verifyMemorySSA();

if (Changed && SE)
  SE->forgetLoopDispositions(L);
return Changed;
476}

478/// Walk the specified region of the CFG (defined by all blocks dominated by
479/// the specified block, and that are in the current loop) in reverse depth
480/// first order w.r.t the DominatorTree.  This allows us to visit uses before
481/// definitions, allowing us to sink a loop body in one pass without iteration.
482///
483bool llvm::sinkRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI,
                    DominatorTree *DT, BlockFrequencyInfo *BFI,
                    TargetLibraryInfo *TLI, TargetTransformInfo *TTI,
                    Loop *CurLoop, AliasSetTracker *CurAST,
                    MemorySSAUpdater *MSSAU, ICFLoopSafetyInfo *SafetyInfo,
                    SinkAndHoistLICMFlags &Flags,
                    OptimizationRemarkEmitter *ORE) {

// Verify inputs.
assert(N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr &&((N != nullptr && AA != nullptr && LI != nullptr
 && DT != nullptr && CurLoop != nullptr &&
 SafetyInfo != nullptr && "Unexpected input to sinkRegion."
) ? static_cast<void> (0) : __assert_fail ("N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected input to sinkRegion.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 494, __PRETTY_FUNCTION__))
       CurLoop != nullptr && SafetyInfo != nullptr &&((N != nullptr && AA != nullptr && LI != nullptr
 && DT != nullptr && CurLoop != nullptr &&
 SafetyInfo != nullptr && "Unexpected input to sinkRegion."
) ? static_cast<void> (0) : __assert_fail ("N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected input to sinkRegion.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 494, __PRETTY_FUNCTION__))
       "Unexpected input to sinkRegion.")((N != nullptr && AA != nullptr && LI != nullptr
 && DT != nullptr && CurLoop != nullptr &&
 SafetyInfo != nullptr && "Unexpected input to sinkRegion."
) ? static_cast<void> (0) : __assert_fail ("N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected input to sinkRegion.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 494, __PRETTY_FUNCTION__));
assert(((CurAST != nullptr) ^ (MSSAU != nullptr)) &&((((CurAST != nullptr) ^ (MSSAU != nullptr)) && "Either AliasSetTracker or MemorySSA should be initialized."
) ? static_cast<void> (0) : __assert_fail ("((CurAST != nullptr) ^ (MSSAU != nullptr)) && \"Either AliasSetTracker or MemorySSA should be initialized.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 496, __PRETTY_FUNCTION__))
       "Either AliasSetTracker or MemorySSA should be initialized.")((((CurAST != nullptr) ^ (MSSAU != nullptr)) && "Either AliasSetTracker or MemorySSA should be initialized."
) ? static_cast<void> (0) : __assert_fail ("((CurAST != nullptr) ^ (MSSAU != nullptr)) && \"Either AliasSetTracker or MemorySSA should be initialized.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 496, __PRETTY_FUNCTION__));

// We want to visit children before parents. We will enque all the parents
// before their children in the worklist and process the worklist in reverse
// order.
SmallVector<DomTreeNode *, 16> Worklist = collectChildrenInLoop(N, CurLoop);

bool Changed = false;
for (DomTreeNode *DTN : reverse(Worklist)) {
  BasicBlock *BB = DTN->getBlock();
  // Only need to process the contents of this block if it is not part of a
  // subloop (which would already have been processed).
  if (inSubLoop(BB, CurLoop, LI))
    continue;

  for (BasicBlock::iterator II = BB->end(); II != BB->begin();) {
    Instruction &I = *--II;

    // If the instruction is dead, we would try to sink it because it isn't
    // used in the loop, instead, just delete it.
    if (isInstructionTriviallyDead(&I, TLI)) {
      LLVM_DEBUG(dbgs() << "LICM deleting dead inst: " << I << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM deleting dead inst: " <<
 I << '\n'; } } while (false);
      salvageKnowledge(&I);
      salvageDebugInfo(I);
      ++II;
      eraseInstruction(I, *SafetyInfo, CurAST, MSSAU);
      Changed = true;
      continue;
    }

    // Check to see if we can sink this instruction to the exit blocks
    // of the loop.  We can do this if the all users of the instruction are
    // outside of the loop.  In this case, it doesn't even matter if the
    // operands of the instruction are loop invariant.
    //
    bool FreeInLoop = false;
    if (!I.mayHaveSideEffects() &&
        isNotUsedOrFreeInLoop(I, CurLoop, SafetyInfo, TTI, FreeInLoop) &&
        canSinkOrHoistInst(I, AA, DT, CurLoop, CurAST, MSSAU, true, &Flags,
                           ORE)) {
      if (sink(I, LI, DT, BFI, CurLoop, SafetyInfo, MSSAU, ORE)) {
        if (!FreeInLoop) {
          ++II;
          salvageDebugInfo(I);
          eraseInstruction(I, *SafetyInfo, CurAST, MSSAU);
        }
        Changed = true;
      }
    }
  }
}
if (MSSAU && VerifyMemorySSA)
  MSSAU->getMemorySSA()->verifyMemorySSA();
return Changed;
550}

552namespace {
553// This is a helper class for hoistRegion to make it able to hoist control flow
554// in order to be able to hoist phis. The way this works is that we initially
555// start hoisting to the loop preheader, and when we see a loop invariant branch
556// we make note of this. When we then come to hoist an instruction that's
557// conditional on such a branch we duplicate the branch and the relevant control
558// flow, then hoist the instruction into the block corresponding to its original
559// block in the duplicated control flow.
560class ControlFlowHoister {
561private:
// Information about the loop we are hoisting from
LoopInfo *LI;
DominatorTree *DT;
Loop *CurLoop;
MemorySSAUpdater *MSSAU;

// A map of blocks in the loop to the block their instructions will be hoisted
// to.
DenseMap<BasicBlock *, BasicBlock *> HoistDestinationMap;

// The branches that we can hoist, mapped to the block that marks a
// convergence point of their control flow.
DenseMap<BranchInst *, BasicBlock *> HoistableBranches;

576public:
ControlFlowHoister(LoopInfo *LI, DominatorTree *DT, Loop *CurLoop,
                   MemorySSAUpdater *MSSAU)
    : LI(LI), DT(DT), CurLoop(CurLoop), MSSAU(MSSAU) {}

void registerPossiblyHoistableBranch(BranchInst *BI) {
  // We can only hoist conditional branches with loop invariant operands.
  if (!ControlFlowHoisting || !BI->isConditional() ||
      !CurLoop->hasLoopInvariantOperands(BI))
    return;

  // The branch destinations need to be in the loop, and we don't gain
  // anything by duplicating conditional branches with duplicate successors,
  // as it's essentially the same as an unconditional branch.
  BasicBlock *TrueDest = BI->getSuccessor(0);
  BasicBlock *FalseDest = BI->getSuccessor(1);
  if (!CurLoop->contains(TrueDest) || !CurLoop->contains(FalseDest) ||
      TrueDest == FalseDest)
    return;

  // We can hoist BI if one branch destination is the successor of the other,
  // or both have common successor which we check by seeing if the
  // intersection of their successors is non-empty.
  // TODO: This could be expanded to allowing branches where both ends
  // eventually converge to a single block.
  SmallPtrSet<BasicBlock *, 4> TrueDestSucc, FalseDestSucc;
  TrueDestSucc.insert(succ_begin(TrueDest), succ_end(TrueDest));
  FalseDestSucc.insert(succ_begin(FalseDest), succ_end(FalseDest));
  BasicBlock *CommonSucc = nullptr;
  if (TrueDestSucc.count(FalseDest)) {
    CommonSucc = FalseDest;
  } else if (FalseDestSucc.count(TrueDest)) {
    CommonSucc = TrueDest;
  } else {
    set_intersect(TrueDestSucc, FalseDestSucc);
    // If there's one common successor use that.
    if (TrueDestSucc.size() == 1)
      CommonSucc = *TrueDestSucc.begin();
    // If there's more than one pick whichever appears first in the block list
    // (we can't use the value returned by TrueDestSucc.begin() as it's
    // unpredicatable which element gets returned).
    else if (!TrueDestSucc.empty()) {
      Function *F = TrueDest->getParent();
      auto IsSucc = [&](BasicBlock &BB) { return TrueDestSucc.count(&BB); };
      auto It = std::find_if(F->begin(), F->end(), IsSucc);
      assert(It != F->end() && "Could not find successor in function")((It != F->end() && "Could not find successor in function"
) ? static_cast<void> (0) : __assert_fail ("It != F->end() && \"Could not find successor in function\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 621, __PRETTY_FUNCTION__));
      CommonSucc = &*It;
    }
  }
  // The common successor has to be dominated by the branch, as otherwise
  // there will be some other path to the successor that will not be
  // controlled by this branch so any phi we hoist would be controlled by the
  // wrong condition. This also takes care of avoiding hoisting of loop back
  // edges.
  // TODO: In some cases this could be relaxed if the successor is dominated
  // by another block that's been hoisted and we can guarantee that the
  // control flow has been replicated exactly.
  if (CommonSucc && DT->dominates(BI, CommonSucc))
    HoistableBranches[BI] = CommonSucc;
}

bool canHoistPHI(PHINode *PN) {
  // The phi must have loop invariant operands.
  if (!ControlFlowHoisting || !CurLoop->hasLoopInvariantOperands(PN))
    return false;
  // We can hoist phis if the block they are in is the target of hoistable
  // branches which cover all of the predecessors of the block.
  SmallPtrSet<BasicBlock *, 8> PredecessorBlocks;
  BasicBlock *BB = PN->getParent();
  for (BasicBlock *PredBB : predecessors(BB))
    PredecessorBlocks.insert(PredBB);
  // If we have less predecessor blocks than predecessors then the phi will
  // have more than one incoming value for the same block which we can't
  // handle.
  // TODO: This could be handled be erasing some of the duplicate incoming
  // values.
  if (PredecessorBlocks.size() != pred_size(BB))
    return false;
  for (auto &Pair : HoistableBranches) {
    if (Pair.second == BB) {
      // Which blocks are predecessors via this branch depends on if the
      // branch is triangle-like or diamond-like.
      if (Pair.first->getSuccessor(0) == BB) {
        PredecessorBlocks.erase(Pair.first->getParent());
        PredecessorBlocks.erase(Pair.first->getSuccessor(1));
      } else if (Pair.first->getSuccessor(1) == BB) {
        PredecessorBlocks.erase(Pair.first->getParent());
        PredecessorBlocks.erase(Pair.first->getSuccessor(0));
      } else {
        PredecessorBlocks.erase(Pair.first->getSuccessor(0));
        PredecessorBlocks.erase(Pair.first->getSuccessor(1));
      }
    }
  }
  // PredecessorBlocks will now be empty if for every predecessor of BB we
  // found a hoistable branch source.
  return PredecessorBlocks.empty();
}

BasicBlock *getOrCreateHoistedBlock(BasicBlock *BB) {
  if (!ControlFlowHoisting)
    return CurLoop->getLoopPreheader();
  // If BB has already been hoisted, return that
  if (HoistDestinationMap.count(BB))
    return HoistDestinationMap[BB];

  // Check if this block is conditional based on a pending branch
  auto HasBBAsSuccessor =
      [&](DenseMap<BranchInst *, BasicBlock *>::value_type &Pair) {
        return BB != Pair.second && (Pair.first->getSuccessor(0) == BB ||
                                     Pair.first->getSuccessor(1) == BB);
      };
  auto It = std::find_if(HoistableBranches.begin(), HoistableBranches.end(),
                         HasBBAsSuccessor);

  // If not involved in a pending branch, hoist to preheader
  BasicBlock *InitialPreheader = CurLoop->getLoopPreheader();
  if (It == HoistableBranches.end()) {
    LLVM_DEBUG(dbgs() << "LICM using " << InitialPreheader->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM using " << InitialPreheader
->getName() << " as hoist destination for " <<
 BB->getName() << "\n"; } } while (false)
                      << " as hoist destination for " << BB->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM using " << InitialPreheader
->getName() << " as hoist destination for " <<
 BB->getName() << "\n"; } } while (false)
                      << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM using " << InitialPreheader
->getName() << " as hoist destination for " <<
 BB->getName() << "\n"; } } while (false);
    HoistDestinationMap[BB] = InitialPreheader;
    return InitialPreheader;
  }
  BranchInst *BI = It->first;
  assert(std::find_if(++It, HoistableBranches.end(), HasBBAsSuccessor) ==((std::find_if(++It, HoistableBranches.end(), HasBBAsSuccessor
) == HoistableBranches.end() && "BB is expected to be the target of at most one branch"
) ? static_cast<void> (0) : __assert_fail ("std::find_if(++It, HoistableBranches.end(), HasBBAsSuccessor) == HoistableBranches.end() && \"BB is expected to be the target of at most one branch\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 703, __PRETTY_FUNCTION__))
             HoistableBranches.end() &&((std::find_if(++It, HoistableBranches.end(), HasBBAsSuccessor
) == HoistableBranches.end() && "BB is expected to be the target of at most one branch"
) ? static_cast<void> (0) : __assert_fail ("std::find_if(++It, HoistableBranches.end(), HasBBAsSuccessor) == HoistableBranches.end() && \"BB is expected to be the target of at most one branch\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 703, __PRETTY_FUNCTION__))
         "BB is expected to be the target of at most one branch")((std::find_if(++It, HoistableBranches.end(), HasBBAsSuccessor
) == HoistableBranches.end() && "BB is expected to be the target of at most one branch"
) ? static_cast<void> (0) : __assert_fail ("std::find_if(++It, HoistableBranches.end(), HasBBAsSuccessor) == HoistableBranches.end() && \"BB is expected to be the target of at most one branch\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 703, __PRETTY_FUNCTION__));

  LLVMContext &C = BB->getContext();
  BasicBlock *TrueDest = BI->getSuccessor(0);
  BasicBlock *FalseDest = BI->getSuccessor(1);
  BasicBlock *CommonSucc = HoistableBranches[BI];
  BasicBlock *HoistTarget = getOrCreateHoistedBlock(BI->getParent());

  // Create hoisted versions of blocks that currently don't have them
  auto CreateHoistedBlock = [&](BasicBlock *Orig) {
    if (HoistDestinationMap.count(Orig))
      return HoistDestinationMap[Orig];
    BasicBlock *New =
        BasicBlock::Create(C, Orig->getName() + ".licm", Orig->getParent());
    HoistDestinationMap[Orig] = New;
    DT->addNewBlock(New, HoistTarget);
    if (CurLoop->getParentLoop())
      CurLoop->getParentLoop()->addBasicBlockToLoop(New, *LI);
    ++NumCreatedBlocks;
    LLVM_DEBUG(dbgs() << "LICM created " << New->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM created " << New->
getName() << " as hoist destination for " << Orig
->getName() << "\n"; } } while (false)
                      << " as hoist destination for " << Orig->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM created " << New->
getName() << " as hoist destination for " << Orig
->getName() << "\n"; } } while (false)
                      << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM created " << New->
getName() << " as hoist destination for " << Orig
->getName() << "\n"; } } while (false);
    return New;
  };
  BasicBlock *HoistTrueDest = CreateHoistedBlock(TrueDest);
  BasicBlock *HoistFalseDest = CreateHoistedBlock(FalseDest);
  BasicBlock *HoistCommonSucc = CreateHoistedBlock(CommonSucc);

  // Link up these blocks with branches.
  if (!HoistCommonSucc->getTerminator()) {
    // The new common successor we've generated will branch to whatever that
    // hoist target branched to.
    BasicBlock *TargetSucc = HoistTarget->getSingleSuccessor();
    assert(TargetSucc && "Expected hoist target to have a single successor")((TargetSucc && "Expected hoist target to have a single successor"
) ? static_cast<void> (0) : __assert_fail ("TargetSucc && \"Expected hoist target to have a single successor\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 736, __PRETTY_FUNCTION__));
    HoistCommonSucc->moveBefore(TargetSucc);
    BranchInst::Create(TargetSucc, HoistCommonSucc);
  }
  if (!HoistTrueDest->getTerminator()) {
    HoistTrueDest->moveBefore(HoistCommonSucc);
    BranchInst::Create(HoistCommonSucc, HoistTrueDest);
  }
  if (!HoistFalseDest->getTerminator()) {
    HoistFalseDest->moveBefore(HoistCommonSucc);
    BranchInst::Create(HoistCommonSucc, HoistFalseDest);
  }

  // If BI is being cloned to what was originally the preheader then
  // HoistCommonSucc will now be the new preheader.
  if (HoistTarget == InitialPreheader) {
    // Phis in the loop header now need to use the new preheader.
    InitialPreheader->replaceSuccessorsPhiUsesWith(HoistCommonSucc);
    if (MSSAU)
      MSSAU->wireOldPredecessorsToNewImmediatePredecessor(
          HoistTarget->getSingleSuccessor(), HoistCommonSucc, {HoistTarget});
    // The new preheader dominates the loop header.
    DomTreeNode *PreheaderNode = DT->getNode(HoistCommonSucc);
    DomTreeNode *HeaderNode = DT->getNode(CurLoop->getHeader());
    DT->changeImmediateDominator(HeaderNode, PreheaderNode);
    // The preheader hoist destination is now the new preheader, with the
    // exception of the hoist destination of this branch.
    for (auto &Pair : HoistDestinationMap)
      if (Pair.second == InitialPreheader && Pair.first != BI->getParent())
        Pair.second = HoistCommonSucc;
  }

  // Now finally clone BI.
  ReplaceInstWithInst(
      HoistTarget->getTerminator(),
      BranchInst::Create(HoistTrueDest, HoistFalseDest, BI->getCondition()));
  ++NumClonedBranches;

  assert(CurLoop->getLoopPreheader() &&((CurLoop->getLoopPreheader() && "Hoisting blocks should not have destroyed preheader"
) ? static_cast<void> (0) : __assert_fail ("CurLoop->getLoopPreheader() && \"Hoisting blocks should not have destroyed preheader\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 775, __PRETTY_FUNCTION__))
         "Hoisting blocks should not have destroyed preheader")((CurLoop->getLoopPreheader() && "Hoisting blocks should not have destroyed preheader"
) ? static_cast<void> (0) : __assert_fail ("CurLoop->getLoopPreheader() && \"Hoisting blocks should not have destroyed preheader\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 775, __PRETTY_FUNCTION__));
  return HoistDestinationMap[BB];
}
778};
779} // namespace

781// Hoisting/sinking instruction out of a loop isn't always beneficial. It's only
782// only worthwhile if the destination block is actually colder than current
783// block.
784static bool worthSinkOrHoistInst(Instruction &I, BasicBlock *DstBlock,
                               OptimizationRemarkEmitter *ORE,
                               BlockFrequencyInfo *BFI) {
// Check block frequency only when runtime profile is available
// to avoid pathological cases. With static profile, lean towards
// hosting because it helps canonicalize the loop for vectorizer.
if (!DstBlock->getParent()->hasProfileData())
  return true;

if (!HoistSinkColdnessThreshold || !BFI)
  return true;

BasicBlock *SrcBlock = I.getParent();
if (BFI->getBlockFreq(DstBlock).getFrequency() / HoistSinkColdnessThreshold >
    BFI->getBlockFreq(SrcBlock).getFrequency()) {
  ORE->emit([&]() {
    return OptimizationRemarkMissed(DEBUG_TYPE"licm", "SinkHoistInst", &I)
           << "failed to sink or hoist instruction because containing block "
              "has lower frequency than destination block";
  });
  return false;
}

return true;
808}

810/// Walk the specified region of the CFG (defined by all blocks dominated by
811/// the specified block, and that are in the current loop) in depth first
812/// order w.r.t the DominatorTree.  This allows us to visit definitions before
813/// uses, allowing us to hoist a loop body in one pass without iteration.
814///
815bool llvm::hoistRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI,
                     DominatorTree *DT, BlockFrequencyInfo *BFI,
                     TargetLibraryInfo *TLI, Loop *CurLoop,
                     AliasSetTracker *CurAST, MemorySSAUpdater *MSSAU,
                     ScalarEvolution *SE, ICFLoopSafetyInfo *SafetyInfo,
                     SinkAndHoistLICMFlags &Flags,
                     OptimizationRemarkEmitter *ORE) {
// Verify inputs.
assert(N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr &&((N != nullptr && AA != nullptr && LI != nullptr
 && DT != nullptr && CurLoop != nullptr &&
 SafetyInfo != nullptr && "Unexpected input to hoistRegion."
) ? static_cast<void> (0) : __assert_fail ("N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected input to hoistRegion.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 825, __PRETTY_FUNCTION__))
       CurLoop != nullptr && SafetyInfo != nullptr &&((N != nullptr && AA != nullptr && LI != nullptr
 && DT != nullptr && CurLoop != nullptr &&
 SafetyInfo != nullptr && "Unexpected input to hoistRegion."
) ? static_cast<void> (0) : __assert_fail ("N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected input to hoistRegion.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 825, __PRETTY_FUNCTION__))
       "Unexpected input to hoistRegion.")((N != nullptr && AA != nullptr && LI != nullptr
 && DT != nullptr && CurLoop != nullptr &&
 SafetyInfo != nullptr && "Unexpected input to hoistRegion."
) ? static_cast<void> (0) : __assert_fail ("N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected input to hoistRegion.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 825, __PRETTY_FUNCTION__));
assert(((CurAST != nullptr) ^ (MSSAU != nullptr)) &&((((CurAST != nullptr) ^ (MSSAU != nullptr)) && "Either AliasSetTracker or MemorySSA should be initialized."
) ? static_cast<void> (0) : __assert_fail ("((CurAST != nullptr) ^ (MSSAU != nullptr)) && \"Either AliasSetTracker or MemorySSA should be initialized.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 827, __PRETTY_FUNCTION__))
       "Either AliasSetTracker or MemorySSA should be initialized.")((((CurAST != nullptr) ^ (MSSAU != nullptr)) && "Either AliasSetTracker or MemorySSA should be initialized."
) ? static_cast<void> (0) : __assert_fail ("((CurAST != nullptr) ^ (MSSAU != nullptr)) && \"Either AliasSetTracker or MemorySSA should be initialized.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 827, __PRETTY_FUNCTION__));

ControlFlowHoister CFH(LI, DT, CurLoop, MSSAU);

// Keep track of instructions that have been hoisted, as they may need to be
// re-hoisted if they end up not dominating all of their uses.
SmallVector<Instruction *, 16> HoistedInstructions;

// For PHI hoisting to work we need to hoist blocks before their successors.
// We can do this by iterating through the blocks in the loop in reverse
// post-order.
LoopBlocksRPO Worklist(CurLoop);
Worklist.perform(LI);
bool Changed = false;
for (BasicBlock *BB : Worklist) {
  // Only need to process the contents of this block if it is not part of a
  // subloop (which would already have been processed).
  if (inSubLoop(BB, CurLoop, LI))
    continue;

  for (BasicBlock::iterator II = BB->begin(), E = BB->end(); II != E;) {
    Instruction &I = *II++;
    // Try constant folding this instruction.  If all the operands are
    // constants, it is technically hoistable, but it would be better to
    // just fold it.
    if (Constant *C = ConstantFoldInstruction(
            &I, I.getModule()->getDataLayout(), TLI)) {
      LLVM_DEBUG(dbgs() << "LICM folding inst: " << I << "  --> " << *Cdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM folding inst: " << I <<
 "  --> " << *C << '\n'; } } while (false)
                        << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM folding inst: " << I <<
 "  --> " << *C << '\n'; } } while (false);
      if (CurAST)
        CurAST->copyValue(&I, C);
      // FIXME MSSA: Such replacements may make accesses unoptimized (D51960).
      I.replaceAllUsesWith(C);
      if (isInstructionTriviallyDead(&I, TLI))
        eraseInstruction(I, *SafetyInfo, CurAST, MSSAU);
      Changed = true;
      continue;
    }

    // Try hoisting the instruction out to the preheader.  We can only do
    // this if all of the operands of the instruction are loop invariant and
    // if it is safe to hoist the instruction. We also check block frequency
    // to make sure instruction only gets hoisted into colder blocks.
    // TODO: It may be safe to hoist if we are hoisting to a conditional block
    // and we have accurately duplicated the control flow from the loop header
    // to that block.
    if (CurLoop->hasLoopInvariantOperands(&I) &&
        canSinkOrHoistInst(I, AA, DT, CurLoop, CurAST, MSSAU, true, &Flags,
                           ORE) &&
        worthSinkOrHoistInst(I, CurLoop->getLoopPreheader(), ORE, BFI) &&
        isSafeToExecuteUnconditionally(
            I, DT, CurLoop, SafetyInfo, ORE,
            CurLoop->getLoopPreheader()->getTerminator())) {
      hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo,
            MSSAU, SE, ORE);
      HoistedInstructions.push_back(&I);
      Changed = true;
      continue;
    }

    // Attempt to remove floating point division out of the loop by
    // converting it to a reciprocal multiplication.
    if (I.getOpcode() == Instruction::FDiv && I.hasAllowReciprocal() &&
        CurLoop->isLoopInvariant(I.getOperand(1))) {
      auto Divisor = I.getOperand(1);
      auto One = llvm::ConstantFP::get(Divisor->getType(), 1.0);
      auto ReciprocalDivisor = BinaryOperator::CreateFDiv(One, Divisor);
      ReciprocalDivisor->setFastMathFlags(I.getFastMathFlags());
      SafetyInfo->insertInstructionTo(ReciprocalDivisor, I.getParent());
      ReciprocalDivisor->insertBefore(&I);

      auto Product =
          BinaryOperator::CreateFMul(I.getOperand(0), ReciprocalDivisor);
      Product->setFastMathFlags(I.getFastMathFlags());
      SafetyInfo->insertInstructionTo(Product, I.getParent());
      Product->insertAfter(&I);
      I.replaceAllUsesWith(Product);
      eraseInstruction(I, *SafetyInfo, CurAST, MSSAU);

      hoist(*ReciprocalDivisor, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB),
            SafetyInfo, MSSAU, SE, ORE);
      HoistedInstructions.push_back(ReciprocalDivisor);
      Changed = true;
      continue;
    }

    auto IsInvariantStart = [&](Instruction &I) {
      using namespace PatternMatch;
      return I.use_empty() &&
             match(&I, m_Intrinsic<Intrinsic::invariant_start>());
    };
    auto MustExecuteWithoutWritesBefore = [&](Instruction &I) {
      return SafetyInfo->isGuaranteedToExecute(I, DT, CurLoop) &&
             SafetyInfo->doesNotWriteMemoryBefore(I, CurLoop);
    };
    if ((IsInvariantStart(I) || isGuard(&I)) &&
        CurLoop->hasLoopInvariantOperands(&I) &&
        MustExecuteWithoutWritesBefore(I)) {
      hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo,
            MSSAU, SE, ORE);
      HoistedInstructions.push_back(&I);
      Changed = true;
      continue;
    }

    if (PHINode *PN = dyn_cast<PHINode>(&I)) {
      if (CFH.canHoistPHI(PN)) {
        // Redirect incoming blocks first to ensure that we create hoisted
        // versions of those blocks before we hoist the phi.
        for (unsigned int i = 0; i < PN->getNumIncomingValues(); ++i)
          PN->setIncomingBlock(
              i, CFH.getOrCreateHoistedBlock(PN->getIncomingBlock(i)));
        hoist(*PN, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo,
              MSSAU, SE, ORE);
        assert(DT->dominates(PN, BB) && "Conditional PHIs not expected")((DT->dominates(PN, BB) && "Conditional PHIs not expected"
) ? static_cast<void> (0) : __assert_fail ("DT->dominates(PN, BB) && \"Conditional PHIs not expected\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 941, __PRETTY_FUNCTION__));
        Changed = true;
        continue;
      }
    }

    // Remember possibly hoistable branches so we can actually hoist them
    // later if needed.
    if (BranchInst *BI = dyn_cast<BranchInst>(&I))
      CFH.registerPossiblyHoistableBranch(BI);
  }
}

// If we hoisted instructions to a conditional block they may not dominate
// their uses that weren't hoisted (such as phis where some operands are not
// loop invariant). If so make them unconditional by moving them to their
// immediate dominator. We iterate through the instructions in reverse order
// which ensures that when we rehoist an instruction we rehoist its operands,
// and also keep track of where in the block we are rehoisting to to make sure
// that we rehoist instructions before the instructions that use them.
Instruction *HoistPoint = nullptr;
if (ControlFlowHoisting) {
  for (Instruction *I : reverse(HoistedInstructions)) {
    if (!llvm::all_of(I->uses(),
                      [&](Use &U) { return DT->dominates(I, U); })) {
      BasicBlock *Dominator =
          DT->getNode(I->getParent())->getIDom()->getBlock();
      if (!HoistPoint || !DT->dominates(HoistPoint->getParent(), Dominator)) {
        if (HoistPoint)
          assert(DT->dominates(Dominator, HoistPoint->getParent()) &&((DT->dominates(Dominator, HoistPoint->getParent()) &&
 "New hoist point expected to dominate old hoist point") ? static_cast
<void> (0) : __assert_fail ("DT->dominates(Dominator, HoistPoint->getParent()) && \"New hoist point expected to dominate old hoist point\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 971, __PRETTY_FUNCTION__))
                 "New hoist point expected to dominate old hoist point")((DT->dominates(Dominator, HoistPoint->getParent()) &&
 "New hoist point expected to dominate old hoist point") ? static_cast
<void> (0) : __assert_fail ("DT->dominates(Dominator, HoistPoint->getParent()) && \"New hoist point expected to dominate old hoist point\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 971, __PRETTY_FUNCTION__));
        HoistPoint = Dominator->getTerminator();
      }
      LLVM_DEBUG(dbgs() << "LICM rehoisting to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM rehoisting to " << HoistPoint
->getParent()->getName() << ": " << *I <<
 "\n"; } } while (false)
                        << HoistPoint->getParent()->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM rehoisting to " << HoistPoint
->getParent()->getName() << ": " << *I <<
 "\n"; } } while (false)
                        << ": " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM rehoisting to " << HoistPoint
->getParent()->getName() << ": " << *I <<
 "\n"; } } while (false);
      moveInstructionBefore(*I, *HoistPoint, *SafetyInfo, MSSAU, SE);
      HoistPoint = I;
      Changed = true;
    }
  }
}
if (MSSAU && VerifyMemorySSA)
  MSSAU->getMemorySSA()->verifyMemorySSA();

  // Now that we've finished hoisting make sure that LI and DT are still
  // valid.
988#ifdef EXPENSIVE_CHECKS
if (Changed) {
  assert(DT->verify(DominatorTree::VerificationLevel::Fast) &&((DT->verify(DominatorTree::VerificationLevel::Fast) &&
 "Dominator tree verification failed") ? static_cast<void>
 (0) : __assert_fail ("DT->verify(DominatorTree::VerificationLevel::Fast) && \"Dominator tree verification failed\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 991, __PRETTY_FUNCTION__))
         "Dominator tree verification failed")((DT->verify(DominatorTree::VerificationLevel::Fast) &&
 "Dominator tree verification failed") ? static_cast<void>
 (0) : __assert_fail ("DT->verify(DominatorTree::VerificationLevel::Fast) && \"Dominator tree verification failed\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 991, __PRETTY_FUNCTION__));
  LI->verify(*DT);
}
994#endif

return Changed;
997}

999// Return true if LI is invariant within scope of the loop. LI is invariant if
1000// CurLoop is dominated by an invariant.start representing the same memory
1001// location and size as the memory location LI loads from, and also the
1002// invariant.start has no uses.
1003static bool isLoadInvariantInLoop(LoadInst *LI, DominatorTree *DT,
                                Loop *CurLoop) {
Value *Addr = LI->getOperand(0);
const DataLayout &DL = LI->getModule()->getDataLayout();
const TypeSize LocSizeInBits = DL.getTypeSizeInBits(LI->getType());

// It is not currently possible for clang to generate an invariant.start
// intrinsic with scalable vector types because we don't support thread local
// sizeless types and we don't permit sizeless types in structs or classes.
// Furthermore, even if support is added for this in future the intrinsic
// itself is defined to have a size of -1 for variable sized objects. This
// makes it impossible to verify if the intrinsic envelops our region of
// interest. For example, both <vscale x 32 x i8> and <vscale x 16 x i8>
// types would have a -1 parameter, but the former is clearly double the size
// of the latter.
if (LocSizeInBits.isScalable())
  return false;

// if the type is i8 addrspace(x)*, we know this is the type of
// llvm.invariant.start operand
auto *PtrInt8Ty = PointerType::get(Type::getInt8Ty(LI->getContext()),
                                   LI->getPointerAddressSpace());
unsigned BitcastsVisited = 0;
// Look through bitcasts until we reach the i8* type (this is invariant.start
// operand type).
while (Addr->getType() != PtrInt8Ty) {
  auto *BC = dyn_cast<BitCastInst>(Addr);
  // Avoid traversing high number of bitcast uses.
  if (++BitcastsVisited > MaxNumUsesTraversed || !BC)
    return false;
  Addr = BC->getOperand(0);
}

unsigned UsesVisited = 0;
// Traverse all uses of the load operand value, to see if invariant.start is
// one of the uses, and whether it dominates the load instruction.
for (auto *U : Addr->users()) {
  // Avoid traversing for Load operand with high number of users.
  if (++UsesVisited > MaxNumUsesTraversed)
    return false;
  IntrinsicInst *II = dyn_cast<IntrinsicInst>(U);
  // If there are escaping uses of invariant.start instruction, the load maybe
  // non-invariant.
  if (!II || II->getIntrinsicID() != Intrinsic::invariant_start ||
      !II->use_empty())
    continue;
  ConstantInt *InvariantSize = cast<ConstantInt>(II->getArgOperand(0));
  // The intrinsic supports having a -1 argument for variable sized objects
  // so we should check for that here.
  if (InvariantSize->isNegative())
    continue;
  uint64_t InvariantSizeInBits = InvariantSize->getSExtValue() * 8;
  // Confirm the invariant.start location size contains the load operand size
  // in bits. Also, the invariant.start should dominate the load, and we
  // should not hoist the load out of a loop that contains this dominating
  // invariant.start.
  if (LocSizeInBits.getFixedSize() <= InvariantSizeInBits &&
      DT->properlyDominates(II->getParent(), CurLoop->getHeader()))
    return true;
}

return false;
1065}

1067namespace {
1068/// Return true if-and-only-if we know how to (mechanically) both hoist and
1069/// sink a given instruction out of a loop.  Does not address legality
1070/// concerns such as aliasing or speculation safety.
1071bool isHoistableAndSinkableInst(Instruction &I) {
// Only these instructions are hoistable/sinkable.
return (isa<LoadInst>(I) || isa<StoreInst>(I) || isa<CallInst>(I) ||
5
←
Assuming 'I' is a 'LoadInst'→
6
←
Returning the value 1, which participates in a condition later→
        isa<FenceInst>(I) || isa<CastInst>(I) || isa<UnaryOperator>(I) ||
        isa<BinaryOperator>(I) || isa<SelectInst>(I) ||
        isa<GetElementPtrInst>(I) || isa<CmpInst>(I) ||
        isa<InsertElementInst>(I) || isa<ExtractElementInst>(I) ||
        isa<ShuffleVectorInst>(I) || isa<ExtractValueInst>(I) ||
        isa<InsertValueInst>(I) || isa<FreezeInst>(I));
1080}
1081/// Return true if all of the alias sets within this AST are known not to
1082/// contain a Mod, or if MSSA knows thare are no MemoryDefs in the loop.
1083bool isReadOnly(AliasSetTracker *CurAST, const MemorySSAUpdater *MSSAU,
              const Loop *L) {
if (CurAST) {
  for (AliasSet &AS : *CurAST) {
    if (!AS.isForwardingAliasSet() && AS.isMod()) {
      return false;
    }
  }
  return true;
} else { /*MSSAU*/
  for (auto *BB : L->getBlocks())
    if (MSSAU->getMemorySSA()->getBlockDefs(BB))
      return false;
  return true;
}
1098}

1100/// Return true if I is the only Instruction with a MemoryAccess in L.
1101bool isOnlyMemoryAccess(const Instruction *I, const Loop *L,
                      const MemorySSAUpdater *MSSAU) {
for (auto *BB : L->getBlocks())
  if (auto *Accs = MSSAU->getMemorySSA()->getBlockAccesses(BB)) {
    int NotAPhi = 0;
    for (const auto &Acc : *Accs) {
      if (isa<MemoryPhi>(&Acc))
        continue;
      const auto *MUD = cast<MemoryUseOrDef>(&Acc);
      if (MUD->getMemoryInst() != I || NotAPhi++ == 1)
        return false;
    }
  }
return true;
1115}
1116}

1118bool llvm::canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
                            Loop *CurLoop, AliasSetTracker *CurAST,
                            MemorySSAUpdater *MSSAU,
                            bool TargetExecutesOncePerLoop,
                            SinkAndHoistLICMFlags *Flags,
                            OptimizationRemarkEmitter *ORE) {
assert(((CurAST != nullptr) ^ (MSSAU != nullptr)) &&((((CurAST != nullptr) ^ (MSSAU != nullptr)) && "Either AliasSetTracker or MemorySSA should be initialized."
) ? static_cast<void> (0) : __assert_fail ("((CurAST != nullptr) ^ (MSSAU != nullptr)) && \"Either AliasSetTracker or MemorySSA should be initialized.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1125, __PRETTY_FUNCTION__))
1
Assuming pointer value is null→
2
←
Assuming the condition is true→
3
←
'?' condition is true→
       "Either AliasSetTracker or MemorySSA should be initialized.")((((CurAST != nullptr) ^ (MSSAU != nullptr)) && "Either AliasSetTracker or MemorySSA should be initialized."
) ? static_cast<void> (0) : __assert_fail ("((CurAST != nullptr) ^ (MSSAU != nullptr)) && \"Either AliasSetTracker or MemorySSA should be initialized.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1125, __PRETTY_FUNCTION__));

// If we don't understand the instruction, bail early.
if (!isHoistableAndSinkableInst(I))
4
←
Calling 'isHoistableAndSinkableInst'→
7
←
Returning from 'isHoistableAndSinkableInst'→
8
←
Taking false branch→
  return false;

MemorySSA *MSSA = MSSAU8.1
'MSSAU' is non-null
1
'MSSAU' is non-null
1
'MSSAU' is non-null
1
'MSSAU' is non-null
1
'MSSAU' is non-null
 ? MSSAU->getMemorySSA() : nullptr;
9
←
'?' condition is true→
10
←
'MSSA' initialized here→
if (MSSA)
11
←
Assuming 'MSSA' is null→
12
←
Taking false branch→
  assert(Flags != nullptr && "Flags cannot be null.")((Flags != nullptr && "Flags cannot be null.") ? static_cast
<void> (0) : __assert_fail ("Flags != nullptr && \"Flags cannot be null.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1133, __PRETTY_FUNCTION__));

// Loads have extra constraints we have to verify before we can hoist them.
if (LoadInst *LI13.1
'LI' is null
1
'LI' is null
1
'LI' is null
1
'LI' is null
1
'LI' is null
 = dyn_cast<LoadInst>(&I)) {
13
←
Assuming the object is not a 'LoadInst'→
14
←
Taking false branch→
  if (!LI->isUnordered())
    return false; // Don't sink/hoist volatile or ordered atomic loads!

  // Loads from constant memory are always safe to move, even if they end up
  // in the same alias set as something that ends up being modified.
  if (AA->pointsToConstantMemory(LI->getOperand(0)))
    return true;
  if (LI->hasMetadata(LLVMContext::MD_invariant_load))
    return true;

  if (LI->isAtomic() && !TargetExecutesOncePerLoop)
    return false; // Don't risk duplicating unordered loads

  // This checks for an invariant.start dominating the load.
  if (isLoadInvariantInLoop(LI, DT, CurLoop))
    return true;

  bool Invalidated;
  if (CurAST)
    Invalidated = pointerInvalidatedByLoop(MemoryLocation::get(LI), CurAST,
                                           CurLoop, AA);
  else
    Invalidated = pointerInvalidatedByLoopWithMSSA(
        MSSA, cast<MemoryUse>(MSSA->getMemoryAccess(LI)), CurLoop, I, *Flags);
  // Check loop-invariant address because this may also be a sinkable load
  // whose address is not necessarily loop-invariant.
  if (ORE && Invalidated && CurLoop->isLoopInvariant(LI->getPointerOperand()))
    ORE->emit([&]() {
      return OptimizationRemarkMissed(
                 DEBUG_TYPE"licm", "LoadWithLoopInvariantAddressInvalidated", LI)
             << "failed to move load with loop-invariant address "
                "because the loop may invalidate its value";
    });

  return !Invalidated;
} else if (CallInst *CI15.1
'CI' is non-null
1
'CI' is non-null
1
'CI' is non-null
1
'CI' is non-null
1
'CI' is non-null
 = dyn_cast<CallInst>(&I)) {
15
←
Assuming the object is a 'CallInst'→
16
←
Taking true branch→
  // Don't sink or hoist dbg info; it's legal, but not useful.
  if (isa<DbgInfoIntrinsic>(I))
17
←
Assuming 'I' is not a 'DbgInfoIntrinsic'→
18
←
Taking false branch→
    return false;

  // Don't sink calls which can throw.
  if (CI->mayThrow())
19
←
Assuming the condition is false→
20
←
Taking false branch→
    return false;

  // Convergent attribute has been used on operations that involve
  // inter-thread communication which results are implicitly affected by the
  // enclosing control flows. It is not safe to hoist or sink such operations
  // across control flow.
  if (CI->isConvergent())
21
←
Calling 'CallBase::isConvergent'→
37
←
Returning from 'CallBase::isConvergent'→
38
←
Assuming the condition is false→
39
←
Taking false branch→
    return false;

  using namespace PatternMatch;
  if (match(CI, m_Intrinsic<Intrinsic::assume>()))
40
←
Calling 'match<llvm::CallInst, llvm::PatternMatch::IntrinsicID_match>'→
47
←
Returning from 'match<llvm::CallInst, llvm::PatternMatch::IntrinsicID_match>'→
48
←
Taking false branch→
    // Assumes don't actually alias anything or throw
    return true;

  if (match(CI, m_Intrinsic<Intrinsic::experimental_widenable_condition>()))
49
←
Calling 'match<llvm::CallInst, llvm::PatternMatch::IntrinsicID_match>'→
56
←
Returning from 'match<llvm::CallInst, llvm::PatternMatch::IntrinsicID_match>'→
57
←
Taking false branch→
    // Widenable conditions don't actually alias anything or throw
    return true;

  // Handle simple cases by querying alias analysis.
  FunctionModRefBehavior Behavior = AA->getModRefBehavior(CI);
  if (Behavior == FMRB_DoesNotAccessMemory)
58
←
Assuming 'Behavior' is not equal to FMRB_DoesNotAccessMemory→
59
←
Taking false branch→
    return true;
  if (AAResults::onlyReadsMemory(Behavior)) {
60
←
Calling 'AAResults::onlyReadsMemory'→
63
←
Returning from 'AAResults::onlyReadsMemory'→
64
←
Taking true branch→
    // A readonly argmemonly function only reads from memory pointed to by
    // it's arguments with arbitrary offsets.  If we can prove there are no
    // writes to this memory in the loop, we can hoist or sink.
    if (AAResults::onlyAccessesArgPointees(Behavior)) {
65
←
Calling 'AAResults::onlyAccessesArgPointees'→
68
←
Returning from 'AAResults::onlyAccessesArgPointees'→
69
←
Taking true branch→
      // TODO: expand to writeable arguments
      for (Value *Op : CI->arg_operands())
70
←
Assuming '__begin5' is not equal to '__end5'→
        if (Op->getType()->isPointerTy()) {
71
←
Calling 'Type::isPointerTy'→
74
←
Returning from 'Type::isPointerTy'→
75
←
Taking true branch→
          bool Invalidated;
          if (CurAST75.1
'CurAST' is null
1
'CurAST' is null
1
'CurAST' is null
1
'CurAST' is null
1
'CurAST' is null
)
76
←
Taking false branch→
            Invalidated = pointerInvalidatedByLoop(
                MemoryLocation(Op, LocationSize::unknown(), AAMDNodes()),
                CurAST, CurLoop, AA);
          else
            Invalidated = pointerInvalidatedByLoopWithMSSA(
                MSSA, cast<MemoryUse>(MSSA->getMemoryAccess(CI)), CurLoop, I,
77
←
Called C++ object pointer is null
                *Flags);
          if (Invalidated)
            return false;
        }
      return true;
    }

    // If this call only reads from memory and there are no writes to memory
    // in the loop, we can hoist or sink the call as appropriate.
    if (isReadOnly(CurAST, MSSAU, CurLoop))
      return true;
  }

  // FIXME: This should use mod/ref information to see if we can hoist or
  // sink the call.

  return false;
} else if (auto *FI = dyn_cast<FenceInst>(&I)) {
  // Fences alias (most) everything to provide ordering.  For the moment,
  // just give up if there are any other memory operations in the loop.
  if (CurAST) {
    auto Begin = CurAST->begin();
    assert(Begin != CurAST->end() && "must contain FI")((Begin != CurAST->end() && "must contain FI") ? static_cast
<void> (0) : __assert_fail ("Begin != CurAST->end() && \"must contain FI\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1239, __PRETTY_FUNCTION__));
    if (std::next(Begin) != CurAST->end())
      // constant memory for instance, TODO: handle better
      return false;
    auto *UniqueI = Begin->getUniqueInstruction();
    if (!UniqueI)
      // other memory op, give up
      return false;
    (void)FI; // suppress unused variable warning
    assert(UniqueI == FI && "AS must contain FI")((UniqueI == FI && "AS must contain FI") ? static_cast
<void> (0) : __assert_fail ("UniqueI == FI && \"AS must contain FI\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1248, __PRETTY_FUNCTION__));
    return true;
  } else // MSSAU
    return isOnlyMemoryAccess(FI, CurLoop, MSSAU);
} else if (auto *SI = dyn_cast<StoreInst>(&I)) {
  if (!SI->isUnordered())
    return false; // Don't sink/hoist volatile or ordered atomic store!

  // We can only hoist a store that we can prove writes a value which is not
  // read or overwritten within the loop.  For those cases, we fallback to
  // load store promotion instead.  TODO: We can extend this to cases where
  // there is exactly one write to the location and that write dominates an
  // arbitrary number of reads in the loop.
  if (CurAST) {
    auto &AS = CurAST->getAliasSetFor(MemoryLocation::get(SI));

    if (AS.isRef() || !AS.isMustAlias())
      // Quick exit test, handled by the full path below as well.
      return false;
    auto *UniqueI = AS.getUniqueInstruction();
    if (!UniqueI)
      // other memory op, give up
      return false;
    assert(UniqueI == SI && "AS must contain SI")((UniqueI == SI && "AS must contain SI") ? static_cast
<void> (0) : __assert_fail ("UniqueI == SI && \"AS must contain SI\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1271, __PRETTY_FUNCTION__));
    return true;
  } else { // MSSAU
    if (isOnlyMemoryAccess(SI, CurLoop, MSSAU))
      return true;
    // If there are more accesses than the Promotion cap or no "quota" to
    // check clobber, then give up as we're not walking a list that long.
    if (Flags->tooManyMemoryAccesses() || Flags->tooManyClobberingCalls())
      return false;
    // If there are interfering Uses (i.e. their defining access is in the
    // loop), or ordered loads (stored as Defs!), don't move this store.
    // Could do better here, but this is conservatively correct.
    // TODO: Cache set of Uses on the first walk in runOnLoop, update when
    // moving accesses. Can also extend to dominating uses.
    auto *SIMD = MSSA->getMemoryAccess(SI);
    for (auto *BB : CurLoop->getBlocks())
      if (auto *Accesses = MSSA->getBlockAccesses(BB)) {
        for (const auto &MA : *Accesses)
          if (const auto *MU = dyn_cast<MemoryUse>(&MA)) {
            auto *MD = MU->getDefiningAccess();
            if (!MSSA->isLiveOnEntryDef(MD) &&
                CurLoop->contains(MD->getBlock()))
              return false;
            // Disable hoisting past potentially interfering loads. Optimized
            // Uses may point to an access outside the loop, as getClobbering
            // checks the previous iteration when walking the backedge.
            // FIXME: More precise: no Uses that alias SI.
            if (!Flags->getIsSink() && !MSSA->dominates(SIMD, MU))
              return false;
          } else if (const auto *MD = dyn_cast<MemoryDef>(&MA)) {
            if (auto *LI = dyn_cast<LoadInst>(MD->getMemoryInst())) {
              (void)LI; // Silence warning.
              assert(!LI->isUnordered() && "Expected unordered load")((!LI->isUnordered() && "Expected unordered load")
 ? static_cast<void> (0) : __assert_fail ("!LI->isUnordered() && \"Expected unordered load\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1303, __PRETTY_FUNCTION__));
              return false;
            }
            // Any call, while it may not be clobbering SI, it may be a use.
            if (auto *CI = dyn_cast<CallInst>(MD->getMemoryInst())) {
              // Check if the call may read from the memory locattion written
              // to by SI. Check CI's attributes and arguments; the number of
              // such checks performed is limited above by NoOfMemAccTooLarge.
              ModRefInfo MRI = AA->getModRefInfo(CI, MemoryLocation::get(SI));
              if (isModOrRefSet(MRI))
                return false;
            }
          }
      }
    auto *Source = MSSA->getSkipSelfWalker()->getClobberingMemoryAccess(SI);
    Flags->incrementClobberingCalls();
    // If there are no clobbering Defs in the loop, store is safe to hoist.
    return MSSA->isLiveOnEntryDef(Source) ||
           !CurLoop->contains(Source->getBlock());
  }
}

assert(!I.mayReadOrWriteMemory() && "unhandled aliasing")((!I.mayReadOrWriteMemory() && "unhandled aliasing") ?
 static_cast<void> (0) : __assert_fail ("!I.mayReadOrWriteMemory() && \"unhandled aliasing\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1325, __PRETTY_FUNCTION__));

// We've established mechanical ability and aliasing, it's up to the caller
// to check fault safety
return true;
1330}

1332/// Returns true if a PHINode is a trivially replaceable with an
1333/// Instruction.
1334/// This is true when all incoming values are that instruction.
1335/// This pattern occurs most often with LCSSA PHI nodes.
1336///
1337static bool isTriviallyReplaceablePHI(const PHINode &PN, const Instruction &I) {
for (const Value *IncValue : PN.incoming_values())
  if (IncValue != &I)
    return false;

return true;
1343}

1345/// Return true if the instruction is free in the loop.
1346static bool isFreeInLoop(const Instruction &I, const Loop *CurLoop,
                       const TargetTransformInfo *TTI) {

if (const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(&I)) {
  if (TTI->getUserCost(GEP, TargetTransformInfo::TCK_SizeAndLatency) !=
      TargetTransformInfo::TCC_Free)
    return false;
  // For a GEP, we cannot simply use getUserCost because currently it
  // optimistically assume that a GEP will fold into addressing mode
  // regardless of its users.
  const BasicBlock *BB = GEP->getParent();
  for (const User *U : GEP->users()) {
    const Instruction *UI = cast<Instruction>(U);
    if (CurLoop->contains(UI) &&
        (BB != UI->getParent() ||
         (!isa<StoreInst>(UI) && !isa<LoadInst>(UI))))
      return false;
  }
  return true;
} else
  return TTI->getUserCost(&I, TargetTransformInfo::TCK_SizeAndLatency) ==
         TargetTransformInfo::TCC_Free;
1368}

1370/// Return true if the only users of this instruction are outside of
1371/// the loop. If this is true, we can sink the instruction to the exit
1372/// blocks of the loop.
1373///
1374/// We also return true if the instruction could be folded away in lowering.
1375/// (e.g.,  a GEP can be folded into a load as an addressing mode in the loop).
1376static bool isNotUsedOrFreeInLoop(const Instruction &I, const Loop *CurLoop,
                                const LoopSafetyInfo *SafetyInfo,
                                TargetTransformInfo *TTI, bool &FreeInLoop) {
const auto &BlockColors = SafetyInfo->getBlockColors();
bool IsFree = isFreeInLoop(I, CurLoop, TTI);
for (const User *U : I.users()) {
  const Instruction *UI = cast<Instruction>(U);
  if (const PHINode *PN = dyn_cast<PHINode>(UI)) {
    const BasicBlock *BB = PN->getParent();
    // We cannot sink uses in catchswitches.
    if (isa<CatchSwitchInst>(BB->getTerminator()))
      return false;

    // We need to sink a callsite to a unique funclet.  Avoid sinking if the
    // phi use is too muddled.
    if (isa<CallInst>(I))
      if (!BlockColors.empty() &&
          BlockColors.find(const_cast<BasicBlock *>(BB))->second.size() != 1)
        return false;
  }

  if (CurLoop->contains(UI)) {
    if (IsFree) {
      FreeInLoop = true;
      continue;
    }
    return false;
  }
}
return true;
1406}

1408static Instruction *cloneInstructionInExitBlock(
  Instruction &I, BasicBlock &ExitBlock, PHINode &PN, const LoopInfo *LI,
  const LoopSafetyInfo *SafetyInfo, MemorySSAUpdater *MSSAU) {
Instruction *New;
if (auto *CI = dyn_cast<CallInst>(&I)) {
  const auto &BlockColors = SafetyInfo->getBlockColors();

  // Sinking call-sites need to be handled differently from other
  // instructions.  The cloned call-site needs a funclet bundle operand
  // appropriate for its location in the CFG.
  SmallVector<OperandBundleDef, 1> OpBundles;
  for (unsigned BundleIdx = 0, BundleEnd = CI->getNumOperandBundles();
       BundleIdx != BundleEnd; ++BundleIdx) {
    OperandBundleUse Bundle = CI->getOperandBundleAt(BundleIdx);
    if (Bundle.getTagID() == LLVMContext::OB_funclet)
      continue;

    OpBundles.emplace_back(Bundle);
  }

  if (!BlockColors.empty()) {
    const ColorVector &CV = BlockColors.find(&ExitBlock)->second;
    assert(CV.size() == 1 && "non-unique color for exit block!")((CV.size() == 1 && "non-unique color for exit block!"
) ? static_cast<void> (0) : __assert_fail ("CV.size() == 1 && \"non-unique color for exit block!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1430, __PRETTY_FUNCTION__));
    BasicBlock *BBColor = CV.front();
    Instruction *EHPad = BBColor->getFirstNonPHI();
    if (EHPad->isEHPad())
      OpBundles.emplace_back("funclet", EHPad);
  }

  New = CallInst::Create(CI, OpBundles);
} else {
  New = I.clone();
}

ExitBlock.getInstList().insert(ExitBlock.getFirstInsertionPt(), New);
if (!I.getName().empty())
  New->setName(I.getName() + ".le");

if (MSSAU && MSSAU->getMemorySSA()->getMemoryAccess(&I)) {
  // Create a new MemoryAccess and let MemorySSA set its defining access.
  MemoryAccess *NewMemAcc = MSSAU->createMemoryAccessInBB(
      New, nullptr, New->getParent(), MemorySSA::Beginning);
  if (NewMemAcc) {
    if (auto *MemDef = dyn_cast<MemoryDef>(NewMemAcc))
      MSSAU->insertDef(MemDef, /*RenameUses=*/true);
    else {
      auto *MemUse = cast<MemoryUse>(NewMemAcc);
      MSSAU->insertUse(MemUse, /*RenameUses=*/true);
    }
  }
}

// Build LCSSA PHI nodes for any in-loop operands. Note that this is
// particularly cheap because we can rip off the PHI node that we're
// replacing for the number and blocks of the predecessors.
// OPT: If this shows up in a profile, we can instead finish sinking all
// invariant instructions, and then walk their operands to re-establish
// LCSSA. That will eliminate creating PHI nodes just to nuke them when
// sinking bottom-up.
for (User::op_iterator OI = New->op_begin(), OE = New->op_end(); OI != OE;
     ++OI)
  if (Instruction *OInst = dyn_cast<Instruction>(*OI))
    if (Loop *OLoop = LI->getLoopFor(OInst->getParent()))
      if (!OLoop->contains(&PN)) {
        PHINode *OpPN =
            PHINode::Create(OInst->getType(), PN.getNumIncomingValues(),
                            OInst->getName() + ".lcssa", &ExitBlock.front());
        for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i)
          OpPN->addIncoming(OInst, PN.getIncomingBlock(i));
        *OI = OpPN;
      }
return New;
1480}

1482static void eraseInstruction(Instruction &I, ICFLoopSafetyInfo &SafetyInfo,
                           AliasSetTracker *AST, MemorySSAUpdater *MSSAU) {
if (AST)
  AST->deleteValue(&I);
if (MSSAU)
  MSSAU->removeMemoryAccess(&I);
SafetyInfo.removeInstruction(&I);
I.eraseFromParent();
1490}

1492static void moveInstructionBefore(Instruction &I, Instruction &Dest,
                                ICFLoopSafetyInfo &SafetyInfo,
                                MemorySSAUpdater *MSSAU,
                                ScalarEvolution *SE) {
SafetyInfo.removeInstruction(&I);
SafetyInfo.insertInstructionTo(&I, Dest.getParent());
I.moveBefore(&Dest);
if (MSSAU)
  if (MemoryUseOrDef *OldMemAcc = cast_or_null<MemoryUseOrDef>(
          MSSAU->getMemorySSA()->getMemoryAccess(&I)))
    MSSAU->moveToPlace(OldMemAcc, Dest.getParent(),
                       MemorySSA::BeforeTerminator);
if (SE)
  SE->forgetValue(&I);
1506}

1508static Instruction *sinkThroughTriviallyReplaceablePHI(
  PHINode *TPN, Instruction *I, LoopInfo *LI,
  SmallDenseMap<BasicBlock *, Instruction *, 32> &SunkCopies,
  const LoopSafetyInfo *SafetyInfo, const Loop *CurLoop,
  MemorySSAUpdater *MSSAU) {
assert(isTriviallyReplaceablePHI(*TPN, *I) &&((isTriviallyReplaceablePHI(*TPN, *I) && "Expect only trivially replaceable PHI"
) ? static_cast<void> (0) : __assert_fail ("isTriviallyReplaceablePHI(*TPN, *I) && \"Expect only trivially replaceable PHI\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1514, __PRETTY_FUNCTION__))
       "Expect only trivially replaceable PHI")((isTriviallyReplaceablePHI(*TPN, *I) && "Expect only trivially replaceable PHI"
) ? static_cast<void> (0) : __assert_fail ("isTriviallyReplaceablePHI(*TPN, *I) && \"Expect only trivially replaceable PHI\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1514, __PRETTY_FUNCTION__));
BasicBlock *ExitBlock = TPN->getParent();
Instruction *New;
auto It = SunkCopies.find(ExitBlock);
if (It != SunkCopies.end())
  New = It->second;
else
  New = SunkCopies[ExitBlock] = cloneInstructionInExitBlock(
      *I, *ExitBlock, *TPN, LI, SafetyInfo, MSSAU);
return New;
1524}

1526static bool canSplitPredecessors(PHINode *PN, LoopSafetyInfo *SafetyInfo) {
BasicBlock *BB = PN->getParent();
if (!BB->canSplitPredecessors())
  return false;
// It's not impossible to split EHPad blocks, but if BlockColors already exist
// it require updating BlockColors for all offspring blocks accordingly. By
// skipping such corner case, we can make updating BlockColors after splitting
// predecessor fairly simple.
if (!SafetyInfo->getBlockColors().empty() && BB->getFirstNonPHI()->isEHPad())
  return false;
for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI) {
  BasicBlock *BBPred = *PI;
  if (isa<IndirectBrInst>(BBPred->getTerminator()) ||
      isa<CallBrInst>(BBPred->getTerminator()))
    return false;
}
return true;
1543}

1545static void splitPredecessorsOfLoopExit(PHINode *PN, DominatorTree *DT,
                                      LoopInfo *LI, const Loop *CurLoop,
                                      LoopSafetyInfo *SafetyInfo,
                                      MemorySSAUpdater *MSSAU) {
1549#ifndef NDEBUG
SmallVector<BasicBlock *, 32> ExitBlocks;
CurLoop->getUniqueExitBlocks(ExitBlocks);
SmallPtrSet<BasicBlock *, 32> ExitBlockSet(ExitBlocks.begin(),
                                           ExitBlocks.end());
1554#endif
BasicBlock *ExitBB = PN->getParent();
assert(ExitBlockSet.count(ExitBB) && "Expect the PHI is in an exit block.")((ExitBlockSet.count(ExitBB) && "Expect the PHI is in an exit block."
) ? static_cast<void> (0) : __assert_fail ("ExitBlockSet.count(ExitBB) && \"Expect the PHI is in an exit block.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1556, __PRETTY_FUNCTION__));

// Split predecessors of the loop exit to make instructions in the loop are
// exposed to exit blocks through trivially replaceable PHIs while keeping the
// loop in the canonical form where each predecessor of each exit block should
// be contained within the loop. For example, this will convert the loop below
// from
//
// LB1:
//   %v1 =
//   br %LE, %LB2
// LB2:
//   %v2 =
//   br %LE, %LB1
// LE:
//   %p = phi [%v1, %LB1], [%v2, %LB2] <-- non-trivially replaceable
//
// to
//
// LB1:
//   %v1 =
//   br %LE.split, %LB2
// LB2:
//   %v2 =
//   br %LE.split2, %LB1
// LE.split:
//   %p1 = phi [%v1, %LB1]  <-- trivially replaceable
//   br %LE
// LE.split2:
//   %p2 = phi [%v2, %LB2]  <-- trivially replaceable
//   br %LE
// LE:
//   %p = phi [%p1, %LE.split], [%p2, %LE.split2]
//
const auto &BlockColors = SafetyInfo->getBlockColors();
SmallSetVector<BasicBlock *, 8> PredBBs(pred_begin(ExitBB), pred_end(ExitBB));
while (!PredBBs.empty()) {
  BasicBlock *PredBB = *PredBBs.begin();
  assert(CurLoop->contains(PredBB) &&((CurLoop->contains(PredBB) && "Expect all predecessors are in the loop"
) ? static_cast<void> (0) : __assert_fail ("CurLoop->contains(PredBB) && \"Expect all predecessors are in the loop\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1595, __PRETTY_FUNCTION__))
         "Expect all predecessors are in the loop")((CurLoop->contains(PredBB) && "Expect all predecessors are in the loop"
) ? static_cast<void> (0) : __assert_fail ("CurLoop->contains(PredBB) && \"Expect all predecessors are in the loop\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1595, __PRETTY_FUNCTION__));
  if (PN->getBasicBlockIndex(PredBB) >= 0) {
    BasicBlock *NewPred = SplitBlockPredecessors(
        ExitBB, PredBB, ".split.loop.exit", DT, LI, MSSAU, true);
    // Since we do not allow splitting EH-block with BlockColors in
    // canSplitPredecessors(), we can simply assign predecessor's color to
    // the new block.
    if (!BlockColors.empty())
      // Grab a reference to the ColorVector to be inserted before getting the
      // reference to the vector we are copying because inserting the new
      // element in BlockColors might cause the map to be reallocated.
      SafetyInfo->copyColors(NewPred, PredBB);
  }
  PredBBs.remove(PredBB);
}
1610}

1612/// When an instruction is found to only be used outside of the loop, this
1613/// function moves it to the exit blocks and patches up SSA form as needed.
1614/// This method is guaranteed to remove the original instruction from its
1615/// position, and may either delete it or move it to outside of the loop.
1616///
1617static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT,
               BlockFrequencyInfo *BFI, const Loop *CurLoop,
               ICFLoopSafetyInfo *SafetyInfo, MemorySSAUpdater *MSSAU,
               OptimizationRemarkEmitter *ORE) {
LLVM_DEBUG(dbgs() << "LICM sinking instruction: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM sinking instruction: " <<
 I << "\n"; } } while (false);
ORE->emit([&]() {
  return OptimizationRemark(DEBUG_TYPE"licm", "InstSunk", &I)
         << "sinking " << ore::NV("Inst", &I);
});
bool Changed = false;
if (isa<LoadInst>(I))
  ++NumMovedLoads;
else if (isa<CallInst>(I))
  ++NumMovedCalls;
++NumSunk;

// Iterate over users to be ready for actual sinking. Replace users via
// unreachable blocks with undef and make all user PHIs trivially replaceable.
SmallPtrSet<Instruction *, 8> VisitedUsers;
for (Value::user_iterator UI = I.user_begin(), UE = I.user_end(); UI != UE;) {
  auto *User = cast<Instruction>(*UI);
  Use &U = UI.getUse();
  ++UI;

  if (VisitedUsers.count(User) || CurLoop->contains(User))
    continue;

  if (!DT->isReachableFromEntry(User->getParent())) {
    U = UndefValue::get(I.getType());
    Changed = true;
    continue;
  }

  // The user must be a PHI node.
  PHINode *PN = cast<PHINode>(User);

  // Surprisingly, instructions can be used outside of loops without any
  // exits.  This can only happen in PHI nodes if the incoming block is
  // unreachable.
  BasicBlock *BB = PN->getIncomingBlock(U);
  if (!DT->isReachableFromEntry(BB)) {
    U = UndefValue::get(I.getType());
    Changed = true;
    continue;
  }

  VisitedUsers.insert(PN);
  if (isTriviallyReplaceablePHI(*PN, I))
    continue;

  if (!canSplitPredecessors(PN, SafetyInfo))
    return Changed;

  // Split predecessors of the PHI so that we can make users trivially
  // replaceable.
  splitPredecessorsOfLoopExit(PN, DT, LI, CurLoop, SafetyInfo, MSSAU);

  // Should rebuild the iterators, as they may be invalidated by
  // splitPredecessorsOfLoopExit().
  UI = I.user_begin();
  UE = I.user_end();
}

if (VisitedUsers.empty())
  return Changed;

1683#ifndef NDEBUG
SmallVector<BasicBlock *, 32> ExitBlocks;
CurLoop->getUniqueExitBlocks(ExitBlocks);
SmallPtrSet<BasicBlock *, 32> ExitBlockSet(ExitBlocks.begin(),
                                           ExitBlocks.end());
1688#endif

// Clones of this instruction. Don't create more than one per exit block!
SmallDenseMap<BasicBlock *, Instruction *, 32> SunkCopies;

// If this instruction is only used outside of the loop, then all users are
// PHI nodes in exit blocks due to LCSSA form. Just RAUW them with clones of
// the instruction.
// First check if I is worth sinking for all uses. Sink only when it is worth
// across all uses.
SmallSetVector<User*, 8> Users(I.user_begin(), I.user_end());
SmallVector<PHINode *, 8> ExitPNs;
for (auto *UI : Users) {
  auto *User = cast<Instruction>(UI);

  if (CurLoop->contains(User))
    continue;

  PHINode *PN = cast<PHINode>(User);
  assert(ExitBlockSet.count(PN->getParent()) &&((ExitBlockSet.count(PN->getParent()) && "The LCSSA PHI is not in an exit block!"
) ? static_cast<void> (0) : __assert_fail ("ExitBlockSet.count(PN->getParent()) && \"The LCSSA PHI is not in an exit block!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1708, __PRETTY_FUNCTION__))
         "The LCSSA PHI is not in an exit block!")((ExitBlockSet.count(PN->getParent()) && "The LCSSA PHI is not in an exit block!"
) ? static_cast<void> (0) : __assert_fail ("ExitBlockSet.count(PN->getParent()) && \"The LCSSA PHI is not in an exit block!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1708, __PRETTY_FUNCTION__));
  if (!worthSinkOrHoistInst(I, PN->getParent(), ORE, BFI)) {
    return Changed;
  }

  ExitPNs.push_back(PN);
}

for (auto *PN : ExitPNs) {

  // The PHI must be trivially replaceable.
  Instruction *New = sinkThroughTriviallyReplaceablePHI(
      PN, &I, LI, SunkCopies, SafetyInfo, CurLoop, MSSAU);
  PN->replaceAllUsesWith(New);
  eraseInstruction(*PN, *SafetyInfo, nullptr, nullptr);
  Changed = true;
}
return Changed;
1726}

1728/// When an instruction is found to only use loop invariant operands that
1729/// is safe to hoist, this instruction is called to do the dirty work.
1730///
1731static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop,
                BasicBlock *Dest, ICFLoopSafetyInfo *SafetyInfo,
                MemorySSAUpdater *MSSAU, ScalarEvolution *SE,
                OptimizationRemarkEmitter *ORE) {
LLVM_DEBUG(dbgs() << "LICM hoisting to " << Dest->getName() << ": " << Ido { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM hoisting to " << Dest
->getName() << ": " << I << "\n"; } } while
 (false)
                  << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM hoisting to " << Dest
->getName() << ": " << I << "\n"; } } while
 (false);
ORE->emit([&]() {
  return OptimizationRemark(DEBUG_TYPE"licm", "Hoisted", &I) << "hoisting "
                                                       << ore::NV("Inst", &I);
});

// Metadata can be dependent on conditions we are hoisting above.
// Conservatively strip all metadata on the instruction unless we were
// guaranteed to execute I if we entered the loop, in which case the metadata
// is valid in the loop preheader.
if (I.hasMetadataOtherThanDebugLoc() &&
    // The check on hasMetadataOtherThanDebugLoc is to prevent us from burning
    // time in isGuaranteedToExecute if we don't actually have anything to
    // drop.  It is a compile time optimization, not required for correctness.
    !SafetyInfo->isGuaranteedToExecute(I, DT, CurLoop))
  I.dropUnknownNonDebugMetadata();

if (isa<PHINode>(I))
  // Move the new node to the end of the phi list in the destination block.
  moveInstructionBefore(I, *Dest->getFirstNonPHI(), *SafetyInfo, MSSAU, SE);
else
  // Move the new node to the destination block, before its terminator.
  moveInstructionBefore(I, *Dest->getTerminator(), *SafetyInfo, MSSAU, SE);

I.updateLocationAfterHoist();

if (isa<LoadInst>(I))
  ++NumMovedLoads;
else if (isa<CallInst>(I))
  ++NumMovedCalls;
++NumHoisted;
1767}

1769/// Only sink or hoist an instruction if it is not a trapping instruction,
1770/// or if the instruction is known not to trap when moved to the preheader.
1771/// or if it is a trapping instruction and is guaranteed to execute.
1772static bool isSafeToExecuteUnconditionally(Instruction &Inst,
                                         const DominatorTree *DT,
                                         const Loop *CurLoop,
                                         const LoopSafetyInfo *SafetyInfo,
                                         OptimizationRemarkEmitter *ORE,
                                         const Instruction *CtxI) {
if (isSafeToSpeculativelyExecute(&Inst, CtxI, DT))
  return true;

bool GuaranteedToExecute =
    SafetyInfo->isGuaranteedToExecute(Inst, DT, CurLoop);

if (!GuaranteedToExecute) {
  auto *LI = dyn_cast<LoadInst>(&Inst);
  if (LI && CurLoop->isLoopInvariant(LI->getPointerOperand()))
    ORE->emit([&]() {
      return OptimizationRemarkMissed(
                 DEBUG_TYPE"licm", "LoadWithLoopInvariantAddressCondExecuted", LI)
             << "failed to hoist load with loop-invariant address "
                "because load is conditionally executed";
    });
}

return GuaranteedToExecute;
1796}

1798namespace {
1799class LoopPromoter : public LoadAndStorePromoter {
Value *SomePtr; // Designated pointer to store to.
const SmallSetVector<Value *, 8> &PointerMustAliases;
SmallVectorImpl<BasicBlock *> &LoopExitBlocks;
SmallVectorImpl<Instruction *> &LoopInsertPts;
SmallVectorImpl<MemoryAccess *> &MSSAInsertPts;
PredIteratorCache &PredCache;
AliasSetTracker *AST;
MemorySSAUpdater *MSSAU;
LoopInfo &LI;
DebugLoc DL;
int Alignment;
bool UnorderedAtomic;
AAMDNodes AATags;
ICFLoopSafetyInfo &SafetyInfo;

Value *maybeInsertLCSSAPHI(Value *V, BasicBlock *BB) const {
  if (Instruction *I = dyn_cast<Instruction>(V))
    if (Loop *L = LI.getLoopFor(I->getParent()))
      if (!L->contains(BB)) {
        // We need to create an LCSSA PHI node for the incoming value and
        // store that.
        PHINode *PN = PHINode::Create(I->getType(), PredCache.size(BB),
                                      I->getName() + ".lcssa", &BB->front());
        for (BasicBlock *Pred : PredCache.get(BB))
          PN->addIncoming(I, Pred);
        return PN;
      }
  return V;
}

1830public:
LoopPromoter(Value *SP, ArrayRef<const Instruction *> Insts, SSAUpdater &S,
             const SmallSetVector<Value *, 8> &PMA,
             SmallVectorImpl<BasicBlock *> &LEB,
             SmallVectorImpl<Instruction *> &LIP,
             SmallVectorImpl<MemoryAccess *> &MSSAIP, PredIteratorCache &PIC,
             AliasSetTracker *ast, MemorySSAUpdater *MSSAU, LoopInfo &li,
             DebugLoc dl, int alignment, bool UnorderedAtomic,
             const AAMDNodes &AATags, ICFLoopSafetyInfo &SafetyInfo)
    : LoadAndStorePromoter(Insts, S), SomePtr(SP), PointerMustAliases(PMA),
      LoopExitBlocks(LEB), LoopInsertPts(LIP), MSSAInsertPts(MSSAIP),
      PredCache(PIC), AST(ast), MSSAU(MSSAU), LI(li), DL(std::move(dl)),
      Alignment(alignment), UnorderedAtomic(UnorderedAtomic), AATags(AATags),
      SafetyInfo(SafetyInfo) {}

bool isInstInList(Instruction *I,
                  const SmallVectorImpl<Instruction *> &) const override {
  Value *Ptr;
  if (LoadInst *LI = dyn_cast<LoadInst>(I))
    Ptr = LI->getOperand(0);
  else
    Ptr = cast<StoreInst>(I)->getPointerOperand();
  return PointerMustAliases.count(Ptr);
}

void doExtraRewritesBeforeFinalDeletion() override {
  // Insert stores after in the loop exit blocks.  Each exit block gets a
  // store of the live-out values that feed them.  Since we've already told
  // the SSA updater about the defs in the loop and the preheader
  // definition, it is all set and we can start using it.
  for (unsigned i = 0, e = LoopExitBlocks.size(); i != e; ++i) {
    BasicBlock *ExitBlock = LoopExitBlocks[i];
    Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock);
    LiveInValue = maybeInsertLCSSAPHI(LiveInValue, ExitBlock);
    Value *Ptr = maybeInsertLCSSAPHI(SomePtr, ExitBlock);
    Instruction *InsertPos = LoopInsertPts[i];
    StoreInst *NewSI = new StoreInst(LiveInValue, Ptr, InsertPos);
    if (UnorderedAtomic)
      NewSI->setOrdering(AtomicOrdering::Unordered);
    NewSI->setAlignment(Align(Alignment));
    NewSI->setDebugLoc(DL);
    if (AATags)
      NewSI->setAAMetadata(AATags);

    if (MSSAU) {
      MemoryAccess *MSSAInsertPoint = MSSAInsertPts[i];
      MemoryAccess *NewMemAcc;
      if (!MSSAInsertPoint) {
        NewMemAcc = MSSAU->createMemoryAccessInBB(
            NewSI, nullptr, NewSI->getParent(), MemorySSA::Beginning);
      } else {
        NewMemAcc =
            MSSAU->createMemoryAccessAfter(NewSI, nullptr, MSSAInsertPoint);
      }
      MSSAInsertPts[i] = NewMemAcc;
      MSSAU->insertDef(cast<MemoryDef>(NewMemAcc), true);
      // FIXME: true for safety, false may still be correct.
    }
  }
}

void replaceLoadWithValue(LoadInst *LI, Value *V) const override {
  // Update alias analysis.
  if (AST)
    AST->copyValue(LI, V);
}
void instructionDeleted(Instruction *I) const override {
  SafetyInfo.removeInstruction(I);
  if (AST)
    AST->deleteValue(I);
  if (MSSAU)
    MSSAU->removeMemoryAccess(I);
}
1903};


1906/// Return true iff we can prove that a caller of this function can not inspect
1907/// the contents of the provided object in a well defined program.
1908bool isKnownNonEscaping(Value *Object, const TargetLibraryInfo *TLI) {
if (isa<AllocaInst>(Object))
  // Since the alloca goes out of scope, we know the caller can't retain a
  // reference to it and be well defined.  Thus, we don't need to check for
  // capture.
  return true;

// For all other objects we need to know that the caller can't possibly
// have gotten a reference to the object.  There are two components of
// that:
//   1) Object can't be escaped by this function.  This is what
//      PointerMayBeCaptured checks.
//   2) Object can't have been captured at definition site.  For this, we
//      need to know the return value is noalias.  At the moment, we use a
//      weaker condition and handle only AllocLikeFunctions (which are
//      known to be noalias).  TODO
return isAllocLikeFn(Object, TLI) &&
  !PointerMayBeCaptured(Object, true, true);
1926}

1928} // namespace

1930/// Try to promote memory values to scalars by sinking stores out of the
1931/// loop and moving loads to before the loop.  We do this by looping over
1932/// the stores in the loop, looking for stores to Must pointers which are
1933/// loop invariant.
1934///
1935bool llvm::promoteLoopAccessesToScalars(
  const SmallSetVector<Value *, 8> &PointerMustAliases,
  SmallVectorImpl<BasicBlock *> &ExitBlocks,
  SmallVectorImpl<Instruction *> &InsertPts,
  SmallVectorImpl<MemoryAccess *> &MSSAInsertPts, PredIteratorCache &PIC,
  LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI,
  Loop *CurLoop, AliasSetTracker *CurAST, MemorySSAUpdater *MSSAU,
  ICFLoopSafetyInfo *SafetyInfo, OptimizationRemarkEmitter *ORE) {
// Verify inputs.
assert(LI != nullptr && DT != nullptr && CurLoop != nullptr &&((LI != nullptr && DT != nullptr && CurLoop !=
 nullptr && SafetyInfo != nullptr && "Unexpected Input to promoteLoopAccessesToScalars"
) ? static_cast<void> (0) : __assert_fail ("LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected Input to promoteLoopAccessesToScalars\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1946, __PRETTY_FUNCTION__))
       SafetyInfo != nullptr &&((LI != nullptr && DT != nullptr && CurLoop !=
 nullptr && SafetyInfo != nullptr && "Unexpected Input to promoteLoopAccessesToScalars"
) ? static_cast<void> (0) : __assert_fail ("LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected Input to promoteLoopAccessesToScalars\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1946, __PRETTY_FUNCTION__))
       "Unexpected Input to promoteLoopAccessesToScalars")((LI != nullptr && DT != nullptr && CurLoop !=
 nullptr && SafetyInfo != nullptr && "Unexpected Input to promoteLoopAccessesToScalars"
) ? static_cast<void> (0) : __assert_fail ("LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected Input to promoteLoopAccessesToScalars\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1946, __PRETTY_FUNCTION__));

Value *SomePtr = *PointerMustAliases.begin();
BasicBlock *Preheader = CurLoop->getLoopPreheader();

// It is not safe to promote a load/store from the loop if the load/store is
// conditional.  For example, turning:
//
//    for () { if (c) *P += 1; }
//
// into:
//
//    tmp = *P;  for () { if (c) tmp +=1; } *P = tmp;
//
// is not safe, because *P may only be valid to access if 'c' is true.
//
// The safety property divides into two parts:
// p1) The memory may not be dereferenceable on entry to the loop.  In this
//    case, we can't insert the required load in the preheader.
// p2) The memory model does not allow us to insert a store along any dynamic
//    path which did not originally have one.
//
// If at least one store is guaranteed to execute, both properties are
// satisfied, and promotion is legal.
//
// This, however, is not a necessary condition. Even if no store/load is
// guaranteed to execute, we can still establish these properties.
// We can establish (p1) by proving that hoisting the load into the preheader
// is safe (i.e. proving dereferenceability on all paths through the loop). We
// can use any access within the alias set to prove dereferenceability,
// since they're all must alias.
//
// There are two ways establish (p2):
// a) Prove the location is thread-local. In this case the memory model
// requirement does not apply, and stores are safe to insert.
// b) Prove a store dominates every exit block. In this case, if an exit
// blocks is reached, the original dynamic path would have taken us through
// the store, so inserting a store into the exit block is safe. Note that this
// is different from the store being guaranteed to execute. For instance,
// if an exception is thrown on the first iteration of the loop, the original
// store is never executed, but the exit blocks are not executed either.

bool DereferenceableInPH = false;
bool SafeToInsertStore = false;

SmallVector<Instruction *, 64> LoopUses;

// We start with an alignment of one and try to find instructions that allow
// us to prove better alignment.
Align Alignment;
// Keep track of which types of access we see
bool SawUnorderedAtomic = false;
bool SawNotAtomic = false;
AAMDNodes AATags;

const DataLayout &MDL = Preheader->getModule()->getDataLayout();

bool IsKnownThreadLocalObject = false;
if (SafetyInfo->anyBlockMayThrow()) {
  // If a loop can throw, we have to insert a store along each unwind edge.
  // That said, we can't actually make the unwind edge explicit. Therefore,
  // we have to prove that the store is dead along the unwind edge.  We do
  // this by proving that the caller can't have a reference to the object
  // after return and thus can't possibly load from the object.
  Value *Object = getUnderlyingObject(SomePtr);
  if (!isKnownNonEscaping(Object, TLI))
    return false;
  // Subtlety: Alloca's aren't visible to callers, but *are* potentially
  // visible to other threads if captured and used during their lifetimes.
  IsKnownThreadLocalObject = !isa<AllocaInst>(Object);
}

// Check that all of the pointers in the alias set have the same type.  We
// cannot (yet) promote a memory location that is loaded and stored in
// different sizes.  While we are at it, collect alignment and AA info.
for (Value *ASIV : PointerMustAliases) {
  // Check that all of the pointers in the alias set have the same type.  We
  // cannot (yet) promote a memory location that is loaded and stored in
  // different sizes.
  if (SomePtr->getType() != ASIV->getType())
    return false;

  for (User *U : ASIV->users()) {
    // Ignore instructions that are outside the loop.
    Instruction *UI = dyn_cast<Instruction>(U);
    if (!UI || !CurLoop->contains(UI))
      continue;

    // If there is an non-load/store instruction in the loop, we can't promote
    // it.
    if (LoadInst *Load = dyn_cast<LoadInst>(UI)) {
      if (!Load->isUnordered())
        return false;

      SawUnorderedAtomic |= Load->isAtomic();
      SawNotAtomic |= !Load->isAtomic();

      Align InstAlignment = Load->getAlign();

      // Note that proving a load safe to speculate requires proving
      // sufficient alignment at the target location.  Proving it guaranteed
      // to execute does as well.  Thus we can increase our guaranteed
      // alignment as well. 
      if (!DereferenceableInPH || (InstAlignment > Alignment))
        if (isSafeToExecuteUnconditionally(*Load, DT, CurLoop, SafetyInfo,
                                           ORE, Preheader->getTerminator())) {
          DereferenceableInPH = true;
          Alignment = std::max(Alignment, InstAlignment);
        }
    } else if (const StoreInst *Store = dyn_cast<StoreInst>(UI)) {
      // Stores *of* the pointer are not interesting, only stores *to* the
      // pointer.
      if (UI->getOperand(1) != ASIV)
        continue;
      if (!Store->isUnordered())
        return false;

      SawUnorderedAtomic |= Store->isAtomic();
      SawNotAtomic |= !Store->isAtomic();

      // If the store is guaranteed to execute, both properties are satisfied.
      // We may want to check if a store is guaranteed to execute even if we
      // already know that promotion is safe, since it may have higher
      // alignment than any other guaranteed stores, in which case we can
      // raise the alignment on the promoted store.
      Align InstAlignment = Store->getAlign();

      if (!DereferenceableInPH || !SafeToInsertStore ||
          (InstAlignment > Alignment)) {
        if (SafetyInfo->isGuaranteedToExecute(*UI, DT, CurLoop)) {
          DereferenceableInPH = true;
          SafeToInsertStore = true;
          Alignment = std::max(Alignment, InstAlignment);
        }
      }

      // If a store dominates all exit blocks, it is safe to sink.
      // As explained above, if an exit block was executed, a dominating
      // store must have been executed at least once, so we are not
      // introducing stores on paths that did not have them.
      // Note that this only looks at explicit exit blocks. If we ever
      // start sinking stores into unwind edges (see above), this will break.
      if (!SafeToInsertStore)
        SafeToInsertStore = llvm::all_of(ExitBlocks, [&](BasicBlock *Exit) {
          return DT->dominates(Store->getParent(), Exit);
        });

      // If the store is not guaranteed to execute, we may still get
      // deref info through it.
      if (!DereferenceableInPH) {
        DereferenceableInPH = isDereferenceableAndAlignedPointer(
            Store->getPointerOperand(), Store->getValueOperand()->getType(),
            Store->getAlign(), MDL, Preheader->getTerminator(), DT);
      }
    } else
      return false; // Not a load or store.

    // Merge the AA tags.
    if (LoopUses.empty()) {
      // On the first load/store, just take its AA tags.
      UI->getAAMetadata(AATags);
    } else if (AATags) {
      UI->getAAMetadata(AATags, /* Merge = */ true);
    }

    LoopUses.push_back(UI);
  }
}

// If we found both an unordered atomic instruction and a non-atomic memory
// access, bail.  We can't blindly promote non-atomic to atomic since we
// might not be able to lower the result.  We can't downgrade since that
// would violate memory model.  Also, align 0 is an error for atomics.
if (SawUnorderedAtomic && SawNotAtomic)
  return false;

// If we're inserting an atomic load in the preheader, we must be able to
// lower it.  We're only guaranteed to be able to lower naturally aligned
// atomics.
auto *SomePtrElemType = SomePtr->getType()->getPointerElementType();
if (SawUnorderedAtomic &&
    Alignment < MDL.getTypeStoreSize(SomePtrElemType))
  return false;

// If we couldn't prove we can hoist the load, bail.
if (!DereferenceableInPH)
  return false;

// We know we can hoist the load, but don't have a guaranteed store.
// Check whether the location is thread-local. If it is, then we can insert
// stores along paths which originally didn't have them without violating the
// memory model.
if (!SafeToInsertStore) {
  if (IsKnownThreadLocalObject)
    SafeToInsertStore = true;
  else {
    Value *Object = getUnderlyingObject(SomePtr);
    SafeToInsertStore =
        (isAllocLikeFn(Object, TLI) || isa<AllocaInst>(Object)) &&
        !PointerMayBeCaptured(Object, true, true);
  }
}

// If we've still failed to prove we can sink the store, give up.
if (!SafeToInsertStore)
  return false;

// Otherwise, this is safe to promote, lets do it!
LLVM_DEBUG(dbgs() << "LICM: Promoting value stored to in loop: " << *SomePtrdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM: Promoting value stored to in loop: "
 << *SomePtr << '\n'; } } while (false)
                  << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM: Promoting value stored to in loop: "
 << *SomePtr << '\n'; } } while (false);
ORE->emit([&]() {
  return OptimizationRemark(DEBUG_TYPE"licm", "PromoteLoopAccessesToScalar",
                            LoopUses[0])
         << "Moving accesses to memory location out of the loop";
});
++NumPromoted;

// Look at all the loop uses, and try to merge their locations.
std::vector<const DILocation *> LoopUsesLocs;
for (auto U : LoopUses)
  LoopUsesLocs.push_back(U->getDebugLoc().get());
auto DL = DebugLoc(DILocation::getMergedLocations(LoopUsesLocs));

// We use the SSAUpdater interface to insert phi nodes as required.
SmallVector<PHINode *, 16> NewPHIs;
SSAUpdater SSA(&NewPHIs);
LoopPromoter Promoter(SomePtr, LoopUses, SSA, PointerMustAliases, ExitBlocks,
                      InsertPts, MSSAInsertPts, PIC, CurAST, MSSAU, *LI, DL,
                      Alignment.value(), SawUnorderedAtomic, AATags,
                      *SafetyInfo);

// Set up the preheader to have a definition of the value.  It is the live-out
// value from the preheader that uses in the loop will use.
LoadInst *PreheaderLoad = new LoadInst(
    SomePtr->getType()->getPointerElementType(), SomePtr,
    SomePtr->getName() + ".promoted", Preheader->getTerminator());
if (SawUnorderedAtomic)
  PreheaderLoad->setOrdering(AtomicOrdering::Unordered);
PreheaderLoad->setAlignment(Alignment);
PreheaderLoad->setDebugLoc(DebugLoc());
if (AATags)
  PreheaderLoad->setAAMetadata(AATags);
SSA.AddAvailableValue(Preheader, PreheaderLoad);

if (MSSAU) {
  MemoryAccess *PreheaderLoadMemoryAccess = MSSAU->createMemoryAccessInBB(
      PreheaderLoad, nullptr, PreheaderLoad->getParent(), MemorySSA::End);
  MemoryUse *NewMemUse = cast<MemoryUse>(PreheaderLoadMemoryAccess);
  MSSAU->insertUse(NewMemUse, /*RenameUses=*/true);
}

if (MSSAU && VerifyMemorySSA)
  MSSAU->getMemorySSA()->verifyMemorySSA();
// Rewrite all the loads in the loop and remember all the definitions from
// stores in the loop.
Promoter.run(LoopUses);

if (MSSAU && VerifyMemorySSA)
  MSSAU->getMemorySSA()->verifyMemorySSA();
// If the SSAUpdater didn't use the load in the preheader, just zap it now.
if (PreheaderLoad->use_empty())
  eraseInstruction(*PreheaderLoad, *SafetyInfo, CurAST, MSSAU);

return true;
2210}

2212/// Returns an owning pointer to an alias set which incorporates aliasing info
2213/// from L and all subloops of L.
2214std::unique_ptr<AliasSetTracker>
2215LoopInvariantCodeMotion::collectAliasInfoForLoop(Loop *L, LoopInfo *LI,
                                               AAResults *AA) {
auto CurAST = std::make_unique<AliasSetTracker>(*AA);

// Add everything from all the sub loops.
for (Loop *InnerL : L->getSubLoops())
  for (BasicBlock *BB : InnerL->blocks())
    CurAST->add(*BB);

// And merge in this loop (without anything from inner loops).
for (BasicBlock *BB : L->blocks())
  if (LI->getLoopFor(BB) == L)
    CurAST->add(*BB);

return CurAST;
2230}

2232std::unique_ptr<AliasSetTracker>
2233LoopInvariantCodeMotion::collectAliasInfoForLoopWithMSSA(
  Loop *L, AAResults *AA, MemorySSAUpdater *MSSAU) {
auto *MSSA = MSSAU->getMemorySSA();
auto CurAST = std::make_unique<AliasSetTracker>(*AA, MSSA, L);
CurAST->addAllInstructionsInLoopUsingMSSA();
return CurAST;
2239}

2241static bool pointerInvalidatedByLoop(MemoryLocation MemLoc,
                                   AliasSetTracker *CurAST, Loop *CurLoop,
                                   AAResults *AA) {
// First check to see if any of the basic blocks in CurLoop invalidate *V.
bool isInvalidatedAccordingToAST = CurAST->getAliasSetFor(MemLoc).isMod();

if (!isInvalidatedAccordingToAST || !LICMN2Theshold)
  return isInvalidatedAccordingToAST;

// Check with a diagnostic analysis if we can refine the information above.
// This is to identify the limitations of using the AST.
// The alias set mechanism used by LICM has a major weakness in that it
// combines all things which may alias into a single set *before* asking
// modref questions. As a result, a single readonly call within a loop will
// collapse all loads and stores into a single alias set and report
// invalidation if the loop contains any store. For example, readonly calls
// with deopt states have this form and create a general alias set with all
// loads and stores.  In order to get any LICM in loops containing possible
// deopt states we need a more precise invalidation of checking the mod ref
// info of each instruction within the loop and LI. This has a complexity of
// O(N^2), so currently, it is used only as a diagnostic tool since the
// default value of LICMN2Threshold is zero.

// Don't look at nested loops.
if (CurLoop->begin() != CurLoop->end())
  return true;

int N = 0;
for (BasicBlock *BB : CurLoop->getBlocks())
  for (Instruction &I : *BB) {
    if (N >= LICMN2Theshold) {
      LLVM_DEBUG(dbgs() << "Alasing N2 threshold exhausted for "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "Alasing N2 threshold exhausted for "
 << *(MemLoc.Ptr) << "\n"; } } while (false)
                        << *(MemLoc.Ptr) << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "Alasing N2 threshold exhausted for "
 << *(MemLoc.Ptr) << "\n"; } } while (false);
      return true;
    }
    N++;
    auto Res = AA->getModRefInfo(&I, MemLoc);
    if (isModSet(Res)) {
      LLVM_DEBUG(dbgs() << "Aliasing failed on " << I << " for "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "Aliasing failed on " << I <<
 " for " << *(MemLoc.Ptr) << "\n"; } } while (false
)
                        << *(MemLoc.Ptr) << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "Aliasing failed on " << I <<
 " for " << *(MemLoc.Ptr) << "\n"; } } while (false
);
      return true;
    }
  }
LLVM_DEBUG(dbgs() << "Aliasing okay for " << *(MemLoc.Ptr) << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "Aliasing okay for " << *(MemLoc
.Ptr) << "\n"; } } while (false);
return false;
2286}

2288bool pointerInvalidatedByLoopWithMSSA(MemorySSA *MSSA, MemoryUse *MU,
                                    Loop *CurLoop, Instruction &I,
                                    SinkAndHoistLICMFlags &Flags) {
// For hoisting, use the walker to determine safety
if (!Flags.getIsSink()) {
  MemoryAccess *Source;
  // See declaration of SetLicmMssaOptCap for usage details.
  if (Flags.tooManyClobberingCalls())
    Source = MU->getDefiningAccess();
  else {
    Source = MSSA->getSkipSelfWalker()->getClobberingMemoryAccess(MU);
    Flags.incrementClobberingCalls();
  }
  return !MSSA->isLiveOnEntryDef(Source) &&
         CurLoop->contains(Source->getBlock());
}

// For sinking, we'd need to check all Defs below this use. The getClobbering
// call will look on the backedge of the loop, but will check aliasing with
// the instructions on the previous iteration.
// For example:
// for (i ... )
//   load a[i] ( Use (LoE)
//   store a[i] ( 1 = Def (2), with 2 = Phi for the loop.
//   i++;
// The load sees no clobbering inside the loop, as the backedge alias check
// does phi translation, and will check aliasing against store a[i-1].
// However sinking the load outside the loop, below the store is incorrect.

// For now, only sink if there are no Defs in the loop, and the existing ones
// precede the use and are in the same block.
// FIXME: Increase precision: Safe to sink if Use post dominates the Def;
// needs PostDominatorTreeAnalysis.
// FIXME: More precise: no Defs that alias this Use.
if (Flags.tooManyMemoryAccesses())
  return true;
for (auto *BB : CurLoop->getBlocks())
  if (pointerInvalidatedByBlockWithMSSA(*BB, *MSSA, *MU))
    return true;
// When sinking, the source block may not be part of the loop so check it.
if (!CurLoop->contains(&I))
  return pointerInvalidatedByBlockWithMSSA(*I.getParent(), *MSSA, *MU);

return false;
2332}

2334bool pointerInvalidatedByBlockWithMSSA(BasicBlock &BB, MemorySSA &MSSA,
                                     MemoryUse &MU) {
if (const auto *Accesses = MSSA.getBlockDefs(&BB))
  for (const auto &MA : *Accesses)
    if (const auto *MD = dyn_cast<MemoryDef>(&MA))
      if (MU.getBlock() != MD->getBlock() || !MSSA.locallyDominates(MD, &MU))
        return true;
return false;
2342}

2344/// Little predicate that returns true if the specified basic block is in
2345/// a subloop of the current one, not the current one itself.
2346///
2347static bool inSubLoop(BasicBlock *BB, Loop *CurLoop, LoopInfo *LI) {
assert(CurLoop->contains(BB) && "Only valid if BB is IN the loop")((CurLoop->contains(BB) && "Only valid if BB is IN the loop"
) ? static_cast<void> (0) : __assert_fail ("CurLoop->contains(BB) && \"Only valid if BB is IN the loop\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Scalar/LICM.cpp"
, 2348, __PRETTY_FUNCTION__));
return LI->getLoopFor(BB) != CurLoop;
2350}

←

/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h

→

1//===- llvm/InstrTypes.h - Important Instruction subclasses -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines various meta classes of instructions that exist in the VM
10// representation.  Specific concrete subclasses of these may be found in the
11// i*.h files...
12//
13//===----------------------------------------------------------------------===//

15#ifndef LLVM_IR_INSTRTYPES_H
16#define LLVM_IR_INSTRTYPES_H

18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/None.h"
20#include "llvm/ADT/Optional.h"
21#include "llvm/ADT/STLExtras.h"
22#include "llvm/ADT/StringMap.h"
23#include "llvm/ADT/StringRef.h"
24#include "llvm/ADT/Twine.h"
25#include "llvm/ADT/iterator_range.h"
26#include "llvm/IR/Attributes.h"
27#include "llvm/IR/CallingConv.h"
28#include "llvm/IR/Constants.h"
29#include "llvm/IR/DerivedTypes.h"
30#include "llvm/IR/Function.h"
31#include "llvm/IR/Instruction.h"
32#include "llvm/IR/LLVMContext.h"
33#include "llvm/IR/OperandTraits.h"
34#include "llvm/IR/Type.h"
35#include "llvm/IR/User.h"
36#include "llvm/IR/Value.h"
37#include "llvm/Support/Casting.h"
38#include "llvm/Support/ErrorHandling.h"
39#include <algorithm>
40#include <cassert>
41#include <cstddef>
42#include <cstdint>
43#include <iterator>
44#include <string>
45#include <vector>

47namespace llvm {

49namespace Intrinsic {
50typedef unsigned ID;
51}

53//===----------------------------------------------------------------------===//
54//                          UnaryInstruction Class
55//===----------------------------------------------------------------------===//

57class UnaryInstruction : public Instruction {
58protected:
UnaryInstruction(Type *Ty, unsigned iType, Value *V,
                 Instruction *IB = nullptr)
  : Instruction(Ty, iType, &Op<0>(), 1, IB) {
  Op<0>() = V;
}
UnaryInstruction(Type *Ty, unsigned iType, Value *V, BasicBlock *IAE)
  : Instruction(Ty, iType, &Op<0>(), 1, IAE) {
  Op<0>() = V;
}

69public:
// allocate space for exactly one operand
void *operator new(size_t s) {
  return User::operator new(s, 1);
}

/// Transparently provide more efficient getOperand methods.
DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
 setOperand(unsigned, Value*); inline op_iterator op_begin();
 inline const_op_iterator op_begin() const; inline op_iterator
 op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
 getNumOperands() const;

// Methods for support type inquiry through isa, cast, and dyn_cast:
static bool classof(const Instruction *I) {
  return I->isUnaryOp() ||
         I->getOpcode() == Instruction::Alloca ||
         I->getOpcode() == Instruction::Load ||
         I->getOpcode() == Instruction::VAArg ||
         I->getOpcode() == Instruction::ExtractValue ||
         (I->getOpcode() >= CastOpsBegin && I->getOpcode() < CastOpsEnd);
}
static bool classof(const Value *V) {
  return isa<Instruction>(V) && classof(cast<Instruction>(V));
}
90};

92template <>
93struct OperandTraits<UnaryInstruction> :
public FixedNumOperandTraits<UnaryInstruction, 1> {
95};

97DEFINE_TRANSPARENT_OPERAND_ACCESSORS(UnaryInstruction, Value)UnaryInstruction::op_iterator UnaryInstruction::op_begin() { return
 OperandTraits<UnaryInstruction>::op_begin(this); } UnaryInstruction
::const_op_iterator UnaryInstruction::op_begin() const { return
 OperandTraits<UnaryInstruction>::op_begin(const_cast<
UnaryInstruction*>(this)); } UnaryInstruction::op_iterator
 UnaryInstruction::op_end() { return OperandTraits<UnaryInstruction
>::op_end(this); } UnaryInstruction::const_op_iterator UnaryInstruction
::op_end() const { return OperandTraits<UnaryInstruction>
::op_end(const_cast<UnaryInstruction*>(this)); } Value *
UnaryInstruction::getOperand(unsigned i_nocapture) const { ((
i_nocapture < OperandTraits<UnaryInstruction>::operands
(this) && "getOperand() out of range!") ? static_cast
<void> (0) : __assert_fail ("i_nocapture < OperandTraits<UnaryInstruction>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 97, __PRETTY_FUNCTION__)); return cast_or_null<Value>
( OperandTraits<UnaryInstruction>::op_begin(const_cast<
UnaryInstruction*>(this))[i_nocapture].get()); } void UnaryInstruction
::setOperand(unsigned i_nocapture, Value *Val_nocapture) { ((
i_nocapture < OperandTraits<UnaryInstruction>::operands
(this) && "setOperand() out of range!") ? static_cast
<void> (0) : __assert_fail ("i_nocapture < OperandTraits<UnaryInstruction>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 97, __PRETTY_FUNCTION__)); OperandTraits<UnaryInstruction
>::op_begin(this)[i_nocapture] = Val_nocapture; } unsigned
 UnaryInstruction::getNumOperands() const { return OperandTraits
<UnaryInstruction>::operands(this); } template <int Idx_nocapture
> Use &UnaryInstruction::Op() { return this->OpFrom
<Idx_nocapture>(this); } template <int Idx_nocapture
> const Use &UnaryInstruction::Op() const { return this
->OpFrom<Idx_nocapture>(this); }

99//===----------------------------------------------------------------------===//
100//                                UnaryOperator Class
101//===----------------------------------------------------------------------===//

103class UnaryOperator : public UnaryInstruction {
void AssertOK();

106protected:
UnaryOperator(UnaryOps iType, Value *S, Type *Ty,
              const Twine &Name, Instruction *InsertBefore);
UnaryOperator(UnaryOps iType, Value *S, Type *Ty,
              const Twine &Name, BasicBlock *InsertAtEnd);

// Note: Instruction needs to be a friend here to call cloneImpl.
friend class Instruction;

UnaryOperator *cloneImpl() const;

117public:

/// Construct a unary instruction, given the opcode and an operand.
/// Optionally (if InstBefore is specified) insert the instruction
/// into a BasicBlock right before the specified instruction.  The specified
/// Instruction is allowed to be a dereferenced end iterator.
///
static UnaryOperator *Create(UnaryOps Op, Value *S,
                             const Twine &Name = Twine(),
                             Instruction *InsertBefore = nullptr);

/// Construct a unary instruction, given the opcode and an operand.
/// Also automatically insert this instruction to the end of the
/// BasicBlock specified.
///
static UnaryOperator *Create(UnaryOps Op, Value *S,
                             const Twine &Name,
                             BasicBlock *InsertAtEnd);

/// These methods just forward to Create, and are useful when you
/// statically know what type of instruction you're going to create.  These
/// helpers just save some typing.
139#define HANDLE_UNARY_INST(N, OPC, CLASS) \
static UnaryOperator *Create##OPC(Value *V, const Twine &Name = "") {\
  return Create(Instruction::OPC, V, Name);\
}
143#include "llvm/IR/Instruction.def"
144#define HANDLE_UNARY_INST(N, OPC, CLASS) \
static UnaryOperator *Create##OPC(Value *V, const Twine &Name, \
                                  BasicBlock *BB) {\
  return Create(Instruction::OPC, V, Name, BB);\
}
149#include "llvm/IR/Instruction.def"
150#define HANDLE_UNARY_INST(N, OPC, CLASS) \
static UnaryOperator *Create##OPC(Value *V, const Twine &Name, \
                                  Instruction *I) {\
  return Create(Instruction::OPC, V, Name, I);\
}
155#include "llvm/IR/Instruction.def"

static UnaryOperator *
CreateWithCopiedFlags(UnaryOps Opc, Value *V, Instruction *CopyO,
                      const Twine &Name = "",
                      Instruction *InsertBefore = nullptr) {
  UnaryOperator *UO = Create(Opc, V, Name, InsertBefore);
  UO->copyIRFlags(CopyO);
  return UO;
}

static UnaryOperator *CreateFNegFMF(Value *Op, Instruction *FMFSource,
                                    const Twine &Name = "",
                                    Instruction *InsertBefore = nullptr) {
  return CreateWithCopiedFlags(Instruction::FNeg, Op, FMFSource, Name,
                               InsertBefore);
}

UnaryOps getOpcode() const {
  return static_cast<UnaryOps>(Instruction::getOpcode());
}

// Methods for support type inquiry through isa, cast, and dyn_cast:
static bool classof(const Instruction *I) {
  return I->isUnaryOp();
}
static bool classof(const Value *V) {
  return isa<Instruction>(V) && classof(cast<Instruction>(V));
}
184};

186//===----------------------------------------------------------------------===//
187//                           BinaryOperator Class
188//===----------------------------------------------------------------------===//

190class BinaryOperator : public Instruction {
void AssertOK();

193protected:
BinaryOperator(BinaryOps iType, Value *S1, Value *S2, Type *Ty,
               const Twine &Name, Instruction *InsertBefore);
BinaryOperator(BinaryOps iType, Value *S1, Value *S2, Type *Ty,
               const Twine &Name, BasicBlock *InsertAtEnd);

// Note: Instruction needs to be a friend here to call cloneImpl.
friend class Instruction;

BinaryOperator *cloneImpl() const;

204public:
// allocate space for exactly two operands
void *operator new(size_t s) {
  return User::operator new(s, 2);
}

/// Transparently provide more efficient getOperand methods.
DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
 setOperand(unsigned, Value*); inline op_iterator op_begin();
 inline const_op_iterator op_begin() const; inline op_iterator
 op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
 getNumOperands() const;

/// Construct a binary instruction, given the opcode and the two
/// operands.  Optionally (if InstBefore is specified) insert the instruction
/// into a BasicBlock right before the specified instruction.  The specified
/// Instruction is allowed to be a dereferenced end iterator.
///
static BinaryOperator *Create(BinaryOps Op, Value *S1, Value *S2,
                              const Twine &Name = Twine(),
                              Instruction *InsertBefore = nullptr);

/// Construct a binary instruction, given the opcode and the two
/// operands.  Also automatically insert this instruction to the end of the
/// BasicBlock specified.
///
static BinaryOperator *Create(BinaryOps Op, Value *S1, Value *S2,
                              const Twine &Name, BasicBlock *InsertAtEnd);

/// These methods just forward to Create, and are useful when you
/// statically know what type of instruction you're going to create.  These
/// helpers just save some typing.
232#define HANDLE_BINARY_INST(N, OPC, CLASS) \
static BinaryOperator *Create##OPC(Value *V1, Value *V2, \
                                   const Twine &Name = "") {\
  return Create(Instruction::OPC, V1, V2, Name);\
}
237#include "llvm/IR/Instruction.def"
238#define HANDLE_BINARY_INST(N, OPC, CLASS) \
static BinaryOperator *Create##OPC(Value *V1, Value *V2, \
                                   const Twine &Name, BasicBlock *BB) {\
  return Create(Instruction::OPC, V1, V2, Name, BB);\
}
243#include "llvm/IR/Instruction.def"
244#define HANDLE_BINARY_INST(N, OPC, CLASS) \
static BinaryOperator *Create##OPC(Value *V1, Value *V2, \
                                   const Twine &Name, Instruction *I) {\
  return Create(Instruction::OPC, V1, V2, Name, I);\
}
249#include "llvm/IR/Instruction.def"

static BinaryOperator *CreateWithCopiedFlags(BinaryOps Opc,
                                             Value *V1, Value *V2,
                                             Instruction *CopyO,
                                             const Twine &Name = "") {
  BinaryOperator *BO = Create(Opc, V1, V2, Name);
  BO->copyIRFlags(CopyO);
  return BO;
}

static BinaryOperator *CreateFAddFMF(Value *V1, Value *V2,
                                     Instruction *FMFSource,
                                     const Twine &Name = "") {
  return CreateWithCopiedFlags(Instruction::FAdd, V1, V2, FMFSource, Name);
}
static BinaryOperator *CreateFSubFMF(Value *V1, Value *V2,
                                     Instruction *FMFSource,
                                     const Twine &Name = "") {
  return CreateWithCopiedFlags(Instruction::FSub, V1, V2, FMFSource, Name);
}
static BinaryOperator *CreateFMulFMF(Value *V1, Value *V2,
                                     Instruction *FMFSource,
                                     const Twine &Name = "") {
  return CreateWithCopiedFlags(Instruction::FMul, V1, V2, FMFSource, Name);
}
static BinaryOperator *CreateFDivFMF(Value *V1, Value *V2,
                                     Instruction *FMFSource,
                                     const Twine &Name = "") {
  return CreateWithCopiedFlags(Instruction::FDiv, V1, V2, FMFSource, Name);
}
static BinaryOperator *CreateFRemFMF(Value *V1, Value *V2,
                                     Instruction *FMFSource,
                                     const Twine &Name = "") {
  return CreateWithCopiedFlags(Instruction::FRem, V1, V2, FMFSource, Name);
}

static BinaryOperator *CreateNSW(BinaryOps Opc, Value *V1, Value *V2,
                                 const Twine &Name = "") {
  BinaryOperator *BO = Create(Opc, V1, V2, Name);
  BO->setHasNoSignedWrap(true);
  return BO;
}
static BinaryOperator *CreateNSW(BinaryOps Opc, Value *V1, Value *V2,
                                 const Twine &Name, BasicBlock *BB) {
  BinaryOperator *BO = Create(Opc, V1, V2, Name, BB);
  BO->setHasNoSignedWrap(true);
  return BO;
}
static BinaryOperator *CreateNSW(BinaryOps Opc, Value *V1, Value *V2,
                                 const Twine &Name, Instruction *I) {
  BinaryOperator *BO = Create(Opc, V1, V2, Name, I);
  BO->setHasNoSignedWrap(true);
  return BO;
}

static BinaryOperator *CreateNUW(BinaryOps Opc, Value *V1, Value *V2,
                                 const Twine &Name = "") {
  BinaryOperator *BO = Create(Opc, V1, V2, Name);
  BO->setHasNoUnsignedWrap(true);
  return BO;
}
static BinaryOperator *CreateNUW(BinaryOps Opc, Value *V1, Value *V2,
                                 const Twine &Name, BasicBlock *BB) {
  BinaryOperator *BO = Create(Opc, V1, V2, Name, BB);
  BO->setHasNoUnsignedWrap(true);
  return BO;
}
static BinaryOperator *CreateNUW(BinaryOps Opc, Value *V1, Value *V2,
                                 const Twine &Name, Instruction *I) {
  BinaryOperator *BO = Create(Opc, V1, V2, Name, I);
  BO->setHasNoUnsignedWrap(true);
  return BO;
}

static BinaryOperator *CreateExact(BinaryOps Opc, Value *V1, Value *V2,
                                   const Twine &Name = "") {
  BinaryOperator *BO = Create(Opc, V1, V2, Name);
  BO->setIsExact(true);
  return BO;
}
static BinaryOperator *CreateExact(BinaryOps Opc, Value *V1, Value *V2,
                                   const Twine &Name, BasicBlock *BB) {
  BinaryOperator *BO = Create(Opc, V1, V2, Name, BB);
  BO->setIsExact(true);
  return BO;
}
static BinaryOperator *CreateExact(BinaryOps Opc, Value *V1, Value *V2,
                                   const Twine &Name, Instruction *I) {
  BinaryOperator *BO = Create(Opc, V1, V2, Name, I);
  BO->setIsExact(true);
  return BO;
}

343#define DEFINE_HELPERS(OPC, NUWNSWEXACT)                                       \
static BinaryOperator *Create##NUWNSWEXACT##OPC(Value *V1, Value *V2,        \
                                                const Twine &Name = "") {    \
  return Create##NUWNSWEXACT(Instruction::OPC, V1, V2, Name);                \
}                                                                            \
static BinaryOperator *Create##NUWNSWEXACT##OPC(                             \
    Value *V1, Value *V2, const Twine &Name, BasicBlock *BB) {               \
  return Create##NUWNSWEXACT(Instruction::OPC, V1, V2, Name, BB);            \
}                                                                            \
static BinaryOperator *Create##NUWNSWEXACT##OPC(                             \
    Value *V1, Value *V2, const Twine &Name, Instruction *I) {               \
  return Create##NUWNSWEXACT(Instruction::OPC, V1, V2, Name, I);             \
}

DEFINE_HELPERS(Add, NSW) // CreateNSWAdd
DEFINE_HELPERS(Add, NUW) // CreateNUWAdd
DEFINE_HELPERS(Sub, NSW) // CreateNSWSub
DEFINE_HELPERS(Sub, NUW) // CreateNUWSub
DEFINE_HELPERS(Mul, NSW) // CreateNSWMul
DEFINE_HELPERS(Mul, NUW) // CreateNUWMul
DEFINE_HELPERS(Shl, NSW) // CreateNSWShl
DEFINE_HELPERS(Shl, NUW) // CreateNUWShl

DEFINE_HELPERS(SDiv, Exact)  // CreateExactSDiv
DEFINE_HELPERS(UDiv, Exact)  // CreateExactUDiv
DEFINE_HELPERS(AShr, Exact)  // CreateExactAShr
DEFINE_HELPERS(LShr, Exact)  // CreateExactLShr

371#undef DEFINE_HELPERS

/// Helper functions to construct and inspect unary operations (NEG and NOT)
/// via binary operators SUB and XOR:
///
/// Create the NEG and NOT instructions out of SUB and XOR instructions.
///
static BinaryOperator *CreateNeg(Value *Op, const Twine &Name = "",
                                 Instruction *InsertBefore = nullptr);
static BinaryOperator *CreateNeg(Value *Op, const Twine &Name,
                                 BasicBlock *InsertAtEnd);
static BinaryOperator *CreateNSWNeg(Value *Op, const Twine &Name = "",
                                    Instruction *InsertBefore = nullptr);
static BinaryOperator *CreateNSWNeg(Value *Op, const Twine &Name,
                                    BasicBlock *InsertAtEnd);
static BinaryOperator *CreateNUWNeg(Value *Op, const Twine &Name = "",
                                    Instruction *InsertBefore = nullptr);
static BinaryOperator *CreateNUWNeg(Value *Op, const Twine &Name,
                                    BasicBlock *InsertAtEnd);
static BinaryOperator *CreateNot(Value *Op, const Twine &Name = "",
                                 Instruction *InsertBefore = nullptr);
static BinaryOperator *CreateNot(Value *Op, const Twine &Name,
                                 BasicBlock *InsertAtEnd);

BinaryOps getOpcode() const {
  return static_cast<BinaryOps>(Instruction::getOpcode());
}

/// Exchange the two operands to this instruction.
/// This instruction is safe to use on any binary instruction and
/// does not modify the semantics of the instruction.  If the instruction
/// cannot be reversed (ie, it's a Div), then return true.
///
bool swapOperands();

// Methods for support type inquiry through isa, cast, and dyn_cast:
static bool classof(const Instruction *I) {
  return I->isBinaryOp();
}
static bool classof(const Value *V) {
  return isa<Instruction>(V) && classof(cast<Instruction>(V));
}
413};

415template <>
416struct OperandTraits<BinaryOperator> :
public FixedNumOperandTraits<BinaryOperator, 2> {
418};

420DEFINE_TRANSPARENT_OPERAND_ACCESSORS(BinaryOperator, Value)BinaryOperator::op_iterator BinaryOperator::op_begin() { return
 OperandTraits<BinaryOperator>::op_begin(this); } BinaryOperator
::const_op_iterator BinaryOperator::op_begin() const { return
 OperandTraits<BinaryOperator>::op_begin(const_cast<
BinaryOperator*>(this)); } BinaryOperator::op_iterator BinaryOperator
::op_end() { return OperandTraits<BinaryOperator>::op_end
(this); } BinaryOperator::const_op_iterator BinaryOperator::op_end
() const { return OperandTraits<BinaryOperator>::op_end
(const_cast<BinaryOperator*>(this)); } Value *BinaryOperator
::getOperand(unsigned i_nocapture) const { ((i_nocapture <
 OperandTraits<BinaryOperator>::operands(this) &&
 "getOperand() out of range!") ? static_cast<void> (0) :
 __assert_fail ("i_nocapture < OperandTraits<BinaryOperator>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 420, __PRETTY_FUNCTION__)); return cast_or_null<Value>
( OperandTraits<BinaryOperator>::op_begin(const_cast<
BinaryOperator*>(this))[i_nocapture].get()); } void BinaryOperator
::setOperand(unsigned i_nocapture, Value *Val_nocapture) { ((
i_nocapture < OperandTraits<BinaryOperator>::operands
(this) && "setOperand() out of range!") ? static_cast
<void> (0) : __assert_fail ("i_nocapture < OperandTraits<BinaryOperator>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 420, __PRETTY_FUNCTION__)); OperandTraits<BinaryOperator
>::op_begin(this)[i_nocapture] = Val_nocapture; } unsigned
 BinaryOperator::getNumOperands() const { return OperandTraits
<BinaryOperator>::operands(this); } template <int Idx_nocapture
> Use &BinaryOperator::Op() { return this->OpFrom<
Idx_nocapture>(this); } template <int Idx_nocapture>
 const Use &BinaryOperator::Op() const { return this->
OpFrom<Idx_nocapture>(this); }

422//===----------------------------------------------------------------------===//
423//                               CastInst Class
424//===----------------------------------------------------------------------===//

426/// This is the base class for all instructions that perform data
427/// casts. It is simply provided so that instruction category testing
428/// can be performed with code like:
429///
430/// if (isa<CastInst>(Instr)) { ... }
431/// Base class of casting instructions.
432class CastInst : public UnaryInstruction {
433protected:
/// Constructor with insert-before-instruction semantics for subclasses
CastInst(Type *Ty, unsigned iType, Value *S,
         const Twine &NameStr = "", Instruction *InsertBefore = nullptr)
  : UnaryInstruction(Ty, iType, S, InsertBefore) {
  setName(NameStr);
}
/// Constructor with insert-at-end-of-block semantics for subclasses
CastInst(Type *Ty, unsigned iType, Value *S,
         const Twine &NameStr, BasicBlock *InsertAtEnd)
  : UnaryInstruction(Ty, iType, S, InsertAtEnd) {
  setName(NameStr);
}

447public:
/// Provides a way to construct any of the CastInst subclasses using an
/// opcode instead of the subclass's constructor. The opcode must be in the
/// CastOps category (Instruction::isCast(opcode) returns true). This
/// constructor has insert-before-instruction semantics to automatically
/// insert the new CastInst before InsertBefore (if it is non-null).
/// Construct any of the CastInst subclasses
static CastInst *Create(
  Instruction::CastOps,    ///< The opcode of the cast instruction
  Value *S,                ///< The value to be casted (operand 0)
  Type *Ty,          ///< The type to which cast should be made
  const Twine &Name = "", ///< Name for the instruction
  Instruction *InsertBefore = nullptr ///< Place to insert the instruction
);
/// Provides a way to construct any of the CastInst subclasses using an
/// opcode instead of the subclass's constructor. The opcode must be in the
/// CastOps category. This constructor has insert-at-end-of-block semantics
/// to automatically insert the new CastInst at the end of InsertAtEnd (if
/// its non-null).
/// Construct any of the CastInst subclasses
static CastInst *Create(
  Instruction::CastOps,    ///< The opcode for the cast instruction
  Value *S,                ///< The value to be casted (operand 0)
  Type *Ty,          ///< The type to which operand is casted
  const Twine &Name, ///< The name for the instruction
  BasicBlock *InsertAtEnd  ///< The block to insert the instruction into
);

/// Create a ZExt or BitCast cast instruction
static CastInst *CreateZExtOrBitCast(
  Value *S,                ///< The value to be casted (operand 0)
  Type *Ty,          ///< The type to which cast should be made
  const Twine &Name = "", ///< Name for the instruction
  Instruction *InsertBefore = nullptr ///< Place to insert the instruction
);

/// Create a ZExt or BitCast cast instruction
static CastInst *CreateZExtOrBitCast(
  Value *S,                ///< The value to be casted (operand 0)
  Type *Ty,          ///< The type to which operand is casted
  const Twine &Name, ///< The name for the instruction
  BasicBlock *InsertAtEnd  ///< The block to insert the instruction into
);

/// Create a SExt or BitCast cast instruction
static CastInst *CreateSExtOrBitCast(
  Value *S,                ///< The value to be casted (operand 0)
  Type *Ty,          ///< The type to which cast should be made
  const Twine &Name = "", ///< Name for the instruction
  Instruction *InsertBefore = nullptr ///< Place to insert the instruction
);

/// Create a SExt or BitCast cast instruction
static CastInst *CreateSExtOrBitCast(
  Value *S,                ///< The value to be casted (operand 0)
  Type *Ty,          ///< The type to which operand is casted
  const Twine &Name, ///< The name for the instruction
  BasicBlock *InsertAtEnd  ///< The block to insert the instruction into
);

/// Create a BitCast AddrSpaceCast, or a PtrToInt cast instruction.
static CastInst *CreatePointerCast(
  Value *S,                ///< The pointer value to be casted (operand 0)
  Type *Ty,          ///< The type to which operand is casted
  const Twine &Name, ///< The name for the instruction
  BasicBlock *InsertAtEnd  ///< The block to insert the instruction into
);

/// Create a BitCast, AddrSpaceCast or a PtrToInt cast instruction.
static CastInst *CreatePointerCast(
  Value *S,                ///< The pointer value to be casted (operand 0)
  Type *Ty,          ///< The type to which cast should be made
  const Twine &Name = "", ///< Name for the instruction
  Instruction *InsertBefore = nullptr ///< Place to insert the instruction
);

/// Create a BitCast or an AddrSpaceCast cast instruction.
static CastInst *CreatePointerBitCastOrAddrSpaceCast(
  Value *S,                ///< The pointer value to be casted (operand 0)
  Type *Ty,          ///< The type to which operand is casted
  const Twine &Name, ///< The name for the instruction
  BasicBlock *InsertAtEnd  ///< The block to insert the instruction into
);

/// Create a BitCast or an AddrSpaceCast cast instruction.
static CastInst *CreatePointerBitCastOrAddrSpaceCast(
  Value *S,                ///< The pointer value to be casted (operand 0)
  Type *Ty,          ///< The type to which cast should be made
  const Twine &Name = "", ///< Name for the instruction
  Instruction *InsertBefore = nullptr ///< Place to insert the instruction
);

/// Create a BitCast, a PtrToInt, or an IntToPTr cast instruction.
///
/// If the value is a pointer type and the destination an integer type,
/// creates a PtrToInt cast. If the value is an integer type and the
/// destination a pointer type, creates an IntToPtr cast. Otherwise, creates
/// a bitcast.
static CastInst *CreateBitOrPointerCast(
  Value *S,                ///< The pointer value to be casted (operand 0)
  Type *Ty,          ///< The type to which cast should be made
  const Twine &Name = "", ///< Name for the instruction
  Instruction *InsertBefore = nullptr ///< Place to insert the instruction
);

/// Create a ZExt, BitCast, or Trunc for int -> int casts.
static CastInst *CreateIntegerCast(
  Value *S,                ///< The pointer value to be casted (operand 0)
  Type *Ty,          ///< The type to which cast should be made
  bool isSigned,           ///< Whether to regard S as signed or not
  const Twine &Name = "", ///< Name for the instruction
  Instruction *InsertBefore = nullptr ///< Place to insert the instruction
);

/// Create a ZExt, BitCast, or Trunc for int -> int casts.
static CastInst *CreateIntegerCast(
  Value *S,                ///< The integer value to be casted (operand 0)
  Type *Ty,          ///< The integer type to which operand is casted
  bool isSigned,           ///< Whether to regard S as signed or not
  const Twine &Name, ///< The name for the instruction
  BasicBlock *InsertAtEnd  ///< The block to insert the instruction into
);

/// Create an FPExt, BitCast, or FPTrunc for fp -> fp casts
static CastInst *CreateFPCast(
  Value *S,                ///< The floating point value to be casted
  Type *Ty,          ///< The floating point type to cast to
  const Twine &Name = "", ///< Name for the instruction
  Instruction *InsertBefore = nullptr ///< Place to insert the instruction
);

/// Create an FPExt, BitCast, or FPTrunc for fp -> fp casts
static CastInst *CreateFPCast(
  Value *S,                ///< The floating point value to be casted
  Type *Ty,          ///< The floating point type to cast to
  const Twine &Name, ///< The name for the instruction
  BasicBlock *InsertAtEnd  ///< The block to insert the instruction into
);

/// Create a Trunc or BitCast cast instruction
static CastInst *CreateTruncOrBitCast(
  Value *S,                ///< The value to be casted (operand 0)
  Type *Ty,          ///< The type to which cast should be made
  const Twine &Name = "", ///< Name for the instruction
  Instruction *InsertBefore = nullptr ///< Place to insert the instruction
);

/// Create a Trunc or BitCast cast instruction
static CastInst *CreateTruncOrBitCast(
  Value *S,                ///< The value to be casted (operand 0)
  Type *Ty,          ///< The type to which operand is casted
  const Twine &Name, ///< The name for the instruction
  BasicBlock *InsertAtEnd  ///< The block to insert the instruction into
);

/// Check whether it is valid to call getCastOpcode for these types.
static bool isCastable(
  Type *SrcTy, ///< The Type from which the value should be cast.
  Type *DestTy ///< The Type to which the value should be cast.
);

/// Check whether a bitcast between these types is valid
static bool isBitCastable(
  Type *SrcTy, ///< The Type from which the value should be cast.
  Type *DestTy ///< The Type to which the value should be cast.
);

/// Check whether a bitcast, inttoptr, or ptrtoint cast between these
/// types is valid and a no-op.
///
/// This ensures that any pointer<->integer cast has enough bits in the
/// integer and any other cast is a bitcast.
static bool isBitOrNoopPointerCastable(
    Type *SrcTy,  ///< The Type from which the value should be cast.
    Type *DestTy, ///< The Type to which the value should be cast.
    const DataLayout &DL);

/// Returns the opcode necessary to cast Val into Ty using usual casting
/// rules.
/// Infer the opcode for cast operand and type
static Instruction::CastOps getCastOpcode(
  const Value *Val, ///< The value to cast
  bool SrcIsSigned, ///< Whether to treat the source as signed
  Type *Ty,   ///< The Type to which the value should be casted
  bool DstIsSigned  ///< Whether to treate the dest. as signed
);

/// There are several places where we need to know if a cast instruction
/// only deals with integer source and destination types. To simplify that
/// logic, this method is provided.
/// @returns true iff the cast has only integral typed operand and dest type.
/// Determine if this is an integer-only cast.
bool isIntegerCast() const;

/// A lossless cast is one that does not alter the basic value. It implies
/// a no-op cast but is more stringent, preventing things like int->float,
/// long->double, or int->ptr.
/// @returns true iff the cast is lossless.
/// Determine if this is a lossless cast.
bool isLosslessCast() const;

/// A no-op cast is one that can be effected without changing any bits.
/// It implies that the source and destination types are the same size. The
/// DataLayout argument is to determine the pointer size when examining casts
/// involving Integer and Pointer types. They are no-op casts if the integer
/// is the same size as the pointer. However, pointer size varies with
/// platform.  Note that a precondition of this method is that the cast is
/// legal - i.e. the instruction formed with these operands would verify.
static bool isNoopCast(
  Instruction::CastOps Opcode, ///< Opcode of cast
  Type *SrcTy,         ///< SrcTy of cast
  Type *DstTy,         ///< DstTy of cast
  const DataLayout &DL ///< DataLayout to get the Int Ptr type from.
);

/// Determine if this cast is a no-op cast.
///
/// \param DL is the DataLayout to determine pointer size.
bool isNoopCast(const DataLayout &DL) const;

/// Determine how a pair of casts can be eliminated, if they can be at all.
/// This is a helper function for both CastInst and ConstantExpr.
/// @returns 0 if the CastInst pair can't be eliminated, otherwise
/// returns Instruction::CastOps value for a cast that can replace
/// the pair, casting SrcTy to DstTy.
/// Determine if a cast pair is eliminable
static unsigned isEliminableCastPair(
  Instruction::CastOps firstOpcode,  ///< Opcode of first cast
  Instruction::CastOps secondOpcode, ///< Opcode of second cast
  Type *SrcTy, ///< SrcTy of 1st cast
  Type *MidTy, ///< DstTy of 1st cast & SrcTy of 2nd cast
  Type *DstTy, ///< DstTy of 2nd cast
  Type *SrcIntPtrTy, ///< Integer type corresponding to Ptr SrcTy, or null
  Type *MidIntPtrTy, ///< Integer type corresponding to Ptr MidTy, or null
  Type *DstIntPtrTy  ///< Integer type corresponding to Ptr DstTy, or null
);

/// Return the opcode of this CastInst
Instruction::CastOps getOpcode() const {
  return Instruction::CastOps(Instruction::getOpcode());
}

/// Return the source type, as a convenience
Type* getSrcTy() const { return getOperand(0)->getType(); }
/// Return the destination type, as a convenience
Type* getDestTy() const { return getType(); }

/// This method can be used to determine if a cast from SrcTy to DstTy using
/// Opcode op is valid or not.
/// @returns true iff the proposed cast is valid.
/// Determine if a cast is valid without creating one.
static bool castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy);
static bool castIsValid(Instruction::CastOps op, Value *S, Type *DstTy) {
  return castIsValid(op, S->getType(), DstTy);
}

/// Methods for support type inquiry through isa, cast, and dyn_cast:
static bool classof(const Instruction *I) {
  return I->isCast();
}
static bool classof(const Value *V) {
  return isa<Instruction>(V) && classof(cast<Instruction>(V));
}
710};

712//===----------------------------------------------------------------------===//
713//                               CmpInst Class
714//===----------------------------------------------------------------------===//

716/// This class is the base class for the comparison instructions.
717/// Abstract base class of comparison instructions.
718class CmpInst : public Instruction {
719public:
/// This enumeration lists the possible predicates for CmpInst subclasses.
/// Values in the range 0-31 are reserved for FCmpInst, while values in the
/// range 32-64 are reserved for ICmpInst. This is necessary to ensure the
/// predicate values are not overlapping between the classes.
///
/// Some passes (e.g. InstCombine) depend on the bit-wise characteristics of
/// FCMP_* values. Changing the bit patterns requires a potential change to
/// those passes.
enum Predicate : unsigned {
  // Opcode            U L G E    Intuitive operation
  FCMP_FALSE = 0, ///< 0 0 0 0    Always false (always folded)
  FCMP_OEQ = 1,   ///< 0 0 0 1    True if ordered and equal
  FCMP_OGT = 2,   ///< 0 0 1 0    True if ordered and greater than
  FCMP_OGE = 3,   ///< 0 0 1 1    True if ordered and greater than or equal
  FCMP_OLT = 4,   ///< 0 1 0 0    True if ordered and less than
  FCMP_OLE = 5,   ///< 0 1 0 1    True if ordered and less than or equal
  FCMP_ONE = 6,   ///< 0 1 1 0    True if ordered and operands are unequal
  FCMP_ORD = 7,   ///< 0 1 1 1    True if ordered (no nans)
  FCMP_UNO = 8,   ///< 1 0 0 0    True if unordered: isnan(X) | isnan(Y)
  FCMP_UEQ = 9,   ///< 1 0 0 1    True if unordered or equal
  FCMP_UGT = 10,  ///< 1 0 1 0    True if unordered or greater than
  FCMP_UGE = 11,  ///< 1 0 1 1    True if unordered, greater than, or equal
  FCMP_ULT = 12,  ///< 1 1 0 0    True if unordered or less than
  FCMP_ULE = 13,  ///< 1 1 0 1    True if unordered, less than, or equal
  FCMP_UNE = 14,  ///< 1 1 1 0    True if unordered or not equal
  FCMP_TRUE = 15, ///< 1 1 1 1    Always true (always folded)
  FIRST_FCMP_PREDICATE = FCMP_FALSE,
  LAST_FCMP_PREDICATE = FCMP_TRUE,
  BAD_FCMP_PREDICATE = FCMP_TRUE + 1,
  ICMP_EQ = 32,  ///< equal
  ICMP_NE = 33,  ///< not equal
  ICMP_UGT = 34, ///< unsigned greater than
  ICMP_UGE = 35, ///< unsigned greater or equal
  ICMP_ULT = 36, ///< unsigned less than
  ICMP_ULE = 37, ///< unsigned less or equal
  ICMP_SGT = 38, ///< signed greater than
  ICMP_SGE = 39, ///< signed greater or equal
  ICMP_SLT = 40, ///< signed less than
  ICMP_SLE = 41, ///< signed less or equal
  FIRST_ICMP_PREDICATE = ICMP_EQ,
  LAST_ICMP_PREDICATE = ICMP_SLE,
  BAD_ICMP_PREDICATE = ICMP_SLE + 1
};
using PredicateField =
    Bitfield::Element<Predicate, 0, 6, LAST_ICMP_PREDICATE>;

766protected:
CmpInst(Type *ty, Instruction::OtherOps op, Predicate pred,
        Value *LHS, Value *RHS, const Twine &Name = "",
        Instruction *InsertBefore = nullptr,
        Instruction *FlagsSource = nullptr);

CmpInst(Type *ty, Instruction::OtherOps op, Predicate pred,
        Value *LHS, Value *RHS, const Twine &Name,
        BasicBlock *InsertAtEnd);

776public:
// allocate space for exactly two operands
void *operator new(size_t s) {
  return User::operator new(s, 2);
}

/// Construct a compare instruction, given the opcode, the predicate and
/// the two operands.  Optionally (if InstBefore is specified) insert the
/// instruction into a BasicBlock right before the specified instruction.
/// The specified Instruction is allowed to be a dereferenced end iterator.
/// Create a CmpInst
static CmpInst *Create(OtherOps Op,
                       Predicate predicate, Value *S1,
                       Value *S2, const Twine &Name = "",
                       Instruction *InsertBefore = nullptr);

/// Construct a compare instruction, given the opcode, the predicate and the
/// two operands.  Also automatically insert this instruction to the end of
/// the BasicBlock specified.
/// Create a CmpInst
static CmpInst *Create(OtherOps Op, Predicate predicate, Value *S1,
                       Value *S2, const Twine &Name, BasicBlock *InsertAtEnd);

/// Get the opcode casted to the right type
OtherOps getOpcode() const {
  return static_cast<OtherOps>(Instruction::getOpcode());
}

/// Return the predicate for this instruction.
Predicate getPredicate() const { return getSubclassData<PredicateField>(); }

/// Set the predicate for this instruction to the specified value.
void setPredicate(Predicate P) { setSubclassData<PredicateField>(P); }

static bool isFPPredicate(Predicate P) {
  assert(FIRST_FCMP_PREDICATE == 0 &&((FIRST_FCMP_PREDICATE == 0 && "FIRST_FCMP_PREDICATE is required to be 0"
) ? static_cast<void> (0) : __assert_fail ("FIRST_FCMP_PREDICATE == 0 && \"FIRST_FCMP_PREDICATE is required to be 0\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 812, __PRETTY_FUNCTION__))
         "FIRST_FCMP_PREDICATE is required to be 0")((FIRST_FCMP_PREDICATE == 0 && "FIRST_FCMP_PREDICATE is required to be 0"
) ? static_cast<void> (0) : __assert_fail ("FIRST_FCMP_PREDICATE == 0 && \"FIRST_FCMP_PREDICATE is required to be 0\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 812, __PRETTY_FUNCTION__));
  return P <= LAST_FCMP_PREDICATE;
}

static bool isIntPredicate(Predicate P) {
  return P >= FIRST_ICMP_PREDICATE && P <= LAST_ICMP_PREDICATE;
}

static StringRef getPredicateName(Predicate P);

bool isFPPredicate() const { return isFPPredicate(getPredicate()); }
bool isIntPredicate() const { return isIntPredicate(getPredicate()); }

/// For example, EQ -> NE, UGT -> ULE, SLT -> SGE,
///              OEQ -> UNE, UGT -> OLE, OLT -> UGE, etc.
/// @returns the inverse predicate for the instruction's current predicate.
/// Return the inverse of the instruction's predicate.
Predicate getInversePredicate() const {
  return getInversePredicate(getPredicate());
}

/// For example, EQ -> NE, UGT -> ULE, SLT -> SGE,
///              OEQ -> UNE, UGT -> OLE, OLT -> UGE, etc.
/// @returns the inverse predicate for predicate provided in \p pred.
/// Return the inverse of a given predicate
static Predicate getInversePredicate(Predicate pred);

/// For example, EQ->EQ, SLE->SGE, ULT->UGT,
///              OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
/// @returns the predicate that would be the result of exchanging the two
/// operands of the CmpInst instruction without changing the result
/// produced.
/// Return the predicate as if the operands were swapped
Predicate getSwappedPredicate() const {
  return getSwappedPredicate(getPredicate());
}

/// This is a static version that you can use without an instruction
/// available.
/// Return the predicate as if the operands were swapped.
static Predicate getSwappedPredicate(Predicate pred);

/// For predicate of kind "is X or equal to 0" returns the predicate "is X".
/// For predicate of kind "is X" returns the predicate "is X or equal to 0".
/// does not support other kind of predicates.
/// @returns the predicate that does not contains is equal to zero if
/// it had and vice versa.
/// Return the flipped strictness of predicate
Predicate getFlippedStrictnessPredicate() const {
  return getFlippedStrictnessPredicate(getPredicate());
}

/// This is a static version that you can use without an instruction
/// available.
/// Return the flipped strictness of predicate
static Predicate getFlippedStrictnessPredicate(Predicate pred);

/// For example, SGT -> SGE, SLT -> SLE, ULT -> ULE, UGT -> UGE.
/// Returns the non-strict version of strict comparisons.
Predicate getNonStrictPredicate() const {
  return getNonStrictPredicate(getPredicate());
}

/// This is a static version that you can use without an instruction
/// available.
/// @returns the non-strict version of comparison provided in \p pred.
/// If \p pred is not a strict comparison predicate, returns \p pred.
/// Returns the non-strict version of strict comparisons.
static Predicate getNonStrictPredicate(Predicate pred);

/// Provide more efficient getOperand methods.
DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
 setOperand(unsigned, Value*); inline op_iterator op_begin();
 inline const_op_iterator op_begin() const; inline op_iterator
 op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
 getNumOperands() const;

/// This is just a convenience that dispatches to the subclasses.
/// Swap the operands and adjust predicate accordingly to retain
/// the same comparison.
void swapOperands();

/// This is just a convenience that dispatches to the subclasses.
/// Determine if this CmpInst is commutative.
bool isCommutative() const;

/// Determine if this is an equals/not equals predicate.
/// This is a static version that you can use without an instruction
/// available.
static bool isEquality(Predicate pred);

/// Determine if this is an equals/not equals predicate.
bool isEquality() const { return isEquality(getPredicate()); }

/// Return true if the predicate is relational (not EQ or NE).
static bool isRelational(Predicate P) { return !isEquality(P); }

/// Return true if the predicate is relational (not EQ or NE).
bool isRelational() const { return !isEquality(); }

/// @returns true if the comparison is signed, false otherwise.
/// Determine if this instruction is using a signed comparison.
bool isSigned() const {
  return isSigned(getPredicate());
}

/// @returns true if the comparison is unsigned, false otherwise.
/// Determine if this instruction is using an unsigned comparison.
bool isUnsigned() const {
  return isUnsigned(getPredicate());
}

/// For example, ULT->SLT, ULE->SLE, UGT->SGT, UGE->SGE, SLT->Failed assert
/// @returns the signed version of the unsigned predicate pred.
/// return the signed version of a predicate
static Predicate getSignedPredicate(Predicate pred);

/// For example, ULT->SLT, ULE->SLE, UGT->SGT, UGE->SGE, SLT->Failed assert
/// @returns the signed version of the predicate for this instruction (which
/// has to be an unsigned predicate).
/// return the signed version of a predicate
Predicate getSignedPredicate() {
  return getSignedPredicate(getPredicate());
}

/// For example, SLT->ULT, SLE->ULE, SGT->UGT, SGE->UGE, ULT->Failed assert
/// @returns the unsigned version of the signed predicate pred.
static Predicate getUnsignedPredicate(Predicate pred);

/// For example, SLT->ULT, SLE->ULE, SGT->UGT, SGE->UGE, ULT->Failed assert
/// @returns the unsigned version of the predicate for this instruction (which
/// has to be an signed predicate).
/// return the unsigned version of a predicate
Predicate getUnsignedPredicate() {
  return getUnsignedPredicate(getPredicate());
}

/// For example, SLT->ULT, ULT->SLT, SLE->ULE, ULE->SLE, EQ->Failed assert
/// @returns the unsigned version of the signed predicate pred or
///          the signed version of the signed predicate pred.
static Predicate getFlippedSignednessPredicate(Predicate pred);

/// For example, SLT->ULT, ULT->SLT, SLE->ULE, ULE->SLE, EQ->Failed assert
/// @returns the unsigned version of the signed predicate pred or
///          the signed version of the signed predicate pred.
Predicate getFlippedSignednessPredicate() {
  return getFlippedSignednessPredicate(getPredicate());
}

/// This is just a convenience.
/// Determine if this is true when both operands are the same.
bool isTrueWhenEqual() const {
  return isTrueWhenEqual(getPredicate());
}

/// This is just a convenience.
/// Determine if this is false when both operands are the same.
bool isFalseWhenEqual() const {
  return isFalseWhenEqual(getPredicate());
}

/// @returns true if the predicate is unsigned, false otherwise.
/// Determine if the predicate is an unsigned operation.
static bool isUnsigned(Predicate predicate);

/// @returns true if the predicate is signed, false otherwise.
/// Determine if the predicate is an signed operation.
static bool isSigned(Predicate predicate);

/// Determine if the predicate is an ordered operation.
static bool isOrdered(Predicate predicate);

/// Determine if the predicate is an unordered operation.
static bool isUnordered(Predicate predicate);

/// Determine if the predicate is true when comparing a value with itself.
static bool isTrueWhenEqual(Predicate predicate);

/// Determine if the predicate is false when comparing a value with itself.
static bool isFalseWhenEqual(Predicate predicate);

/// Determine if Pred1 implies Pred2 is true when two compares have matching
/// operands.
static bool isImpliedTrueByMatchingCmp(Predicate Pred1, Predicate Pred2);

/// Determine if Pred1 implies Pred2 is false when two compares have matching
/// operands.
static bool isImpliedFalseByMatchingCmp(Predicate Pred1, Predicate Pred2);

/// Methods for support type inquiry through isa, cast, and dyn_cast:
static bool classof(const Instruction *I) {
  return I->getOpcode() == Instruction::ICmp ||
         I->getOpcode() == Instruction::FCmp;
}
static bool classof(const Value *V) {
  return isa<Instruction>(V) && classof(cast<Instruction>(V));
}

/// Create a result type for fcmp/icmp
static Type* makeCmpResultType(Type* opnd_type) {
  if (VectorType* vt = dyn_cast<VectorType>(opnd_type)) {
    return VectorType::get(Type::getInt1Ty(opnd_type->getContext()),
                           vt->getElementCount());
  }
  return Type::getInt1Ty(opnd_type->getContext());
}

1015private:
// Shadow Value::setValueSubclassData with a private forwarding method so that
// subclasses cannot accidentally use it.
void setValueSubclassData(unsigned short D) {
  Value::setValueSubclassData(D);
}
1021};

1023// FIXME: these are redundant if CmpInst < BinaryOperator
1024template <>
1025struct OperandTraits<CmpInst> : public FixedNumOperandTraits<CmpInst, 2> {
1026};

1028DEFINE_TRANSPARENT_OPERAND_ACCESSORS(CmpInst, Value)CmpInst::op_iterator CmpInst::op_begin() { return OperandTraits
<CmpInst>::op_begin(this); } CmpInst::const_op_iterator
 CmpInst::op_begin() const { return OperandTraits<CmpInst>
::op_begin(const_cast<CmpInst*>(this)); } CmpInst::op_iterator
 CmpInst::op_end() { return OperandTraits<CmpInst>::op_end
(this); } CmpInst::const_op_iterator CmpInst::op_end() const {
 return OperandTraits<CmpInst>::op_end(const_cast<CmpInst
*>(this)); } Value *CmpInst::getOperand(unsigned i_nocapture
) const { ((i_nocapture < OperandTraits<CmpInst>::operands
(this) && "getOperand() out of range!") ? static_cast
<void> (0) : __assert_fail ("i_nocapture < OperandTraits<CmpInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1028, __PRETTY_FUNCTION__)); return cast_or_null<Value>
( OperandTraits<CmpInst>::op_begin(const_cast<CmpInst
*>(this))[i_nocapture].get()); } void CmpInst::setOperand(
unsigned i_nocapture, Value *Val_nocapture) { ((i_nocapture <
 OperandTraits<CmpInst>::operands(this) && "setOperand() out of range!"
) ? static_cast<void> (0) : __assert_fail ("i_nocapture < OperandTraits<CmpInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1028, __PRETTY_FUNCTION__)); OperandTraits<CmpInst>::
op_begin(this)[i_nocapture] = Val_nocapture; } unsigned CmpInst
::getNumOperands() const { return OperandTraits<CmpInst>
::operands(this); } template <int Idx_nocapture> Use &
CmpInst::Op() { return this->OpFrom<Idx_nocapture>(this
); } template <int Idx_nocapture> const Use &CmpInst
::Op() const { return this->OpFrom<Idx_nocapture>(this
); }

1030/// A lightweight accessor for an operand bundle meant to be passed
1031/// around by value.
1032struct OperandBundleUse {
ArrayRef<Use> Inputs;

OperandBundleUse() = default;
explicit OperandBundleUse(StringMapEntry<uint32_t> *Tag, ArrayRef<Use> Inputs)
    : Inputs(Inputs), Tag(Tag) {}

/// Return true if the operand at index \p Idx in this operand bundle
/// has the attribute A.
bool operandHasAttr(unsigned Idx, Attribute::AttrKind A) const {
  if (isDeoptOperandBundle())
    if (A == Attribute::ReadOnly || A == Attribute::NoCapture)
      return Inputs[Idx]->getType()->isPointerTy();

  // Conservative answer:  no operands have any attributes.
  return false;
}

/// Return the tag of this operand bundle as a string.
StringRef getTagName() const {
  return Tag->getKey();
}

/// Return the tag of this operand bundle as an integer.
///
/// Operand bundle tags are interned by LLVMContextImpl::getOrInsertBundleTag,
/// and this function returns the unique integer getOrInsertBundleTag
/// associated the tag of this operand bundle to.
uint32_t getTagID() const {
  return Tag->getValue();
}

/// Return true if this is a "deopt" operand bundle.
bool isDeoptOperandBundle() const {
  return getTagID() == LLVMContext::OB_deopt;
}

/// Return true if this is a "funclet" operand bundle.
bool isFuncletOperandBundle() const {
  return getTagID() == LLVMContext::OB_funclet;
}

/// Return true if this is a "cfguardtarget" operand bundle.
bool isCFGuardTargetOperandBundle() const {
  return getTagID() == LLVMContext::OB_cfguardtarget;
}

1079private:
/// Pointer to an entry in LLVMContextImpl::getOrInsertBundleTag.
StringMapEntry<uint32_t> *Tag;
1082};

1084/// A container for an operand bundle being viewed as a set of values
1085/// rather than a set of uses.
1086///
1087/// Unlike OperandBundleUse, OperandBundleDefT owns the memory it carries, and
1088/// so it is possible to create and pass around "self-contained" instances of
1089/// OperandBundleDef and ConstOperandBundleDef.
1090template <typename InputTy> class OperandBundleDefT {
std::string Tag;
std::vector<InputTy> Inputs;

1094public:
explicit OperandBundleDefT(std::string Tag, std::vector<InputTy> Inputs)
    : Tag(std::move(Tag)), Inputs(std::move(Inputs)) {}
explicit OperandBundleDefT(std::string Tag, ArrayRef<InputTy> Inputs)
    : Tag(std::move(Tag)), Inputs(Inputs) {}

explicit OperandBundleDefT(const OperandBundleUse &OBU) {
  Tag = std::string(OBU.getTagName());
  Inputs.insert(Inputs.end(), OBU.Inputs.begin(), OBU.Inputs.end());
}

ArrayRef<InputTy> inputs() const { return Inputs; }

using input_iterator = typename std::vector<InputTy>::const_iterator;

size_t input_size() const { return Inputs.size(); }
input_iterator input_begin() const { return Inputs.begin(); }
input_iterator input_end() const { return Inputs.end(); }

StringRef getTag() const { return Tag; }
1114};

1116using OperandBundleDef = OperandBundleDefT<Value *>;
1117using ConstOperandBundleDef = OperandBundleDefT<const Value *>;

1119//===----------------------------------------------------------------------===//
1120//                               CallBase Class
1121//===----------------------------------------------------------------------===//

1123/// Base class for all callable instructions (InvokeInst and CallInst)
1124/// Holds everything related to calling a function.
1125///
1126/// All call-like instructions are required to use a common operand layout:
1127/// - Zero or more arguments to the call,
1128/// - Zero or more operand bundles with zero or more operand inputs each
1129///   bundle,
1130/// - Zero or more subclass controlled operands
1131/// - The called function.
1132///
1133/// This allows this base class to easily access the called function and the
1134/// start of the arguments without knowing how many other operands a particular
1135/// subclass requires. Note that accessing the end of the argument list isn't
1136/// as cheap as most other operations on the base class.
1137class CallBase : public Instruction {
1138protected:
// The first two bits are reserved by CallInst for fast retrieval,
using CallInstReservedField = Bitfield::Element<unsigned, 0, 2>;
using CallingConvField =
    Bitfield::Element<CallingConv::ID, CallInstReservedField::NextBit, 10,
                      CallingConv::MaxID>;
static_assert(
    Bitfield::areContiguous<CallInstReservedField, CallingConvField>(),
    "Bitfields must be contiguous");

/// The last operand is the called operand.
static constexpr int CalledOperandOpEndIdx = -1;

AttributeList Attrs; ///< parameter attributes for callable
FunctionType *FTy;

template <class... ArgsTy>
CallBase(AttributeList const &A, FunctionType *FT, ArgsTy &&... Args)
    : Instruction(std::forward<ArgsTy>(Args)...), Attrs(A), FTy(FT) {}

using Instruction::Instruction;

bool hasDescriptor() const { return Value::HasDescriptor; }

unsigned getNumSubclassExtraOperands() const {
  switch (getOpcode()) {
  case Instruction::Call:
    return 0;
  case Instruction::Invoke:
    return 2;
  case Instruction::CallBr:
    return getNumSubclassExtraOperandsDynamic();
  }
  llvm_unreachable("Invalid opcode!")::llvm::llvm_unreachable_internal("Invalid opcode!", "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1171);
}

/// Get the number of extra operands for instructions that don't have a fixed
/// number of extra operands.
unsigned getNumSubclassExtraOperandsDynamic() const;

1178public:
using Instruction::getContext;

/// Create a clone of \p CB with a different set of operand bundles and
/// insert it before \p InsertPt.
///
/// The returned call instruction is identical \p CB in every way except that
/// the operand bundles for the new instruction are set to the operand bundles
/// in \p Bundles.
static CallBase *Create(CallBase *CB, ArrayRef<OperandBundleDef> Bundles,
                        Instruction *InsertPt = nullptr);

static bool classof(const Instruction *I) {
  return I->getOpcode() == Instruction::Call ||
         I->getOpcode() == Instruction::Invoke ||
         I->getOpcode() == Instruction::CallBr;
}
static bool classof(const Value *V) {
  return isa<Instruction>(V) && classof(cast<Instruction>(V));
}

FunctionType *getFunctionType() const { return FTy; }

void mutateFunctionType(FunctionType *FTy) {
  Value::mutateType(FTy->getReturnType());
  this->FTy = FTy;
}

DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
 setOperand(unsigned, Value*); inline op_iterator op_begin();
 inline const_op_iterator op_begin() const; inline op_iterator
 op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
 getNumOperands() const;

/// data_operands_begin/data_operands_end - Return iterators iterating over
/// the call / invoke argument list and bundle operands.  For invokes, this is
/// the set of instruction operands except the invoke target and the two
/// successor blocks; and for calls this is the set of instruction operands
/// except the call target.
User::op_iterator data_operands_begin() { return op_begin(); }
User::const_op_iterator data_operands_begin() const {
  return const_cast<CallBase *>(this)->data_operands_begin();
}
User::op_iterator data_operands_end() {
  // Walk from the end of the operands over the called operand and any
  // subclass operands.
  return op_end() - getNumSubclassExtraOperands() - 1;
}
User::const_op_iterator data_operands_end() const {
  return const_cast<CallBase *>(this)->data_operands_end();
}
iterator_range<User::op_iterator> data_ops() {
  return make_range(data_operands_begin(), data_operands_end());
}
iterator_range<User::const_op_iterator> data_ops() const {
  return make_range(data_operands_begin(), data_operands_end());
}
bool data_operands_empty() const {
  return data_operands_end() == data_operands_begin();
}
unsigned data_operands_size() const {
  return std::distance(data_operands_begin(), data_operands_end());
}

bool isDataOperand(const Use *U) const {
  assert(this == U->getUser() &&((this == U->getUser() && "Only valid to query with a use of this instruction!"
) ? static_cast<void> (0) : __assert_fail ("this == U->getUser() && \"Only valid to query with a use of this instruction!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1240, __PRETTY_FUNCTION__))
         "Only valid to query with a use of this instruction!")((this == U->getUser() && "Only valid to query with a use of this instruction!"
) ? static_cast<void> (0) : __assert_fail ("this == U->getUser() && \"Only valid to query with a use of this instruction!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1240, __PRETTY_FUNCTION__));
  return data_operands_begin() <= U && U < data_operands_end();
}
bool isDataOperand(Value::const_user_iterator UI) const {
  return isDataOperand(&UI.getUse());
}

/// Given a value use iterator, return the data operand corresponding to it.
/// Iterator must actually correspond to a data operand.
unsigned getDataOperandNo(Value::const_user_iterator UI) const {
  return getDataOperandNo(&UI.getUse());
}

/// Given a use for a data operand, get the data operand number that
/// corresponds to it.
unsigned getDataOperandNo(const Use *U) const {
  assert(isDataOperand(U) && "Data operand # out of range!")((isDataOperand(U) && "Data operand # out of range!")
 ? static_cast<void> (0) : __assert_fail ("isDataOperand(U) && \"Data operand # out of range!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1256, __PRETTY_FUNCTION__));
  return U - data_operands_begin();
}

/// Return the iterator pointing to the beginning of the argument list.
User::op_iterator arg_begin() { return op_begin(); }
User::const_op_iterator arg_begin() const {
  return const_cast<CallBase *>(this)->arg_begin();
}

/// Return the iterator pointing to the end of the argument list.
User::op_iterator arg_end() {
  // From the end of the data operands, walk backwards past the bundle
  // operands.
  return data_operands_end() - getNumTotalBundleOperands();
}
User::const_op_iterator arg_end() const {
  return const_cast<CallBase *>(this)->arg_end();
}

/// Iteration adapter for range-for loops.
iterator_range<User::op_iterator> args() {
  return make_range(arg_begin(), arg_end());
}
iterator_range<User::const_op_iterator> args() const {
  return make_range(arg_begin(), arg_end());
}
bool arg_empty() const { return arg_end() == arg_begin(); }
unsigned arg_size() const { return arg_end() - arg_begin(); }

// Legacy API names that duplicate the above and will be removed once users
// are migrated.
iterator_range<User::op_iterator> arg_operands() {
  return make_range(arg_begin(), arg_end());
}
iterator_range<User::const_op_iterator> arg_operands() const {
  return make_range(arg_begin(), arg_end());
}
unsigned getNumArgOperands() const { return arg_size(); }

Value *getArgOperand(unsigned i) const {
  assert(i < getNumArgOperands() && "Out of bounds!")((i < getNumArgOperands() && "Out of bounds!") ? static_cast
<void> (0) : __assert_fail ("i < getNumArgOperands() && \"Out of bounds!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1297, __PRETTY_FUNCTION__));
  return getOperand(i);
}

void setArgOperand(unsigned i, Value *v) {
  assert(i < getNumArgOperands() && "Out of bounds!")((i < getNumArgOperands() && "Out of bounds!") ? static_cast
<void> (0) : __assert_fail ("i < getNumArgOperands() && \"Out of bounds!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1302, __PRETTY_FUNCTION__));
  setOperand(i, v);
}

/// Wrappers for getting the \c Use of a call argument.
const Use &getArgOperandUse(unsigned i) const {
  assert(i < getNumArgOperands() && "Out of bounds!")((i < getNumArgOperands() && "Out of bounds!") ? static_cast
<void> (0) : __assert_fail ("i < getNumArgOperands() && \"Out of bounds!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1308, __PRETTY_FUNCTION__));
  return User::getOperandUse(i);
}
Use &getArgOperandUse(unsigned i) {
  assert(i < getNumArgOperands() && "Out of bounds!")((i < getNumArgOperands() && "Out of bounds!") ? static_cast
<void> (0) : __assert_fail ("i < getNumArgOperands() && \"Out of bounds!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1312, __PRETTY_FUNCTION__));
  return User::getOperandUse(i);
}

bool isArgOperand(const Use *U) const {
  assert(this == U->getUser() &&((this == U->getUser() && "Only valid to query with a use of this instruction!"
) ? static_cast<void> (0) : __assert_fail ("this == U->getUser() && \"Only valid to query with a use of this instruction!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1318, __PRETTY_FUNCTION__))
         "Only valid to query with a use of this instruction!")((this == U->getUser() && "Only valid to query with a use of this instruction!"
) ? static_cast<void> (0) : __assert_fail ("this == U->getUser() && \"Only valid to query with a use of this instruction!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1318, __PRETTY_FUNCTION__));
  return arg_begin() <= U && U < arg_end();
}
bool isArgOperand(Value::const_user_iterator UI) const {
  return isArgOperand(&UI.getUse());
}

/// Given a use for a arg operand, get the arg operand number that
/// corresponds to it.
unsigned getArgOperandNo(const Use *U) const {
  assert(isArgOperand(U) && "Arg operand # out of range!")((isArgOperand(U) && "Arg operand # out of range!") ?
 static_cast<void> (0) : __assert_fail ("isArgOperand(U) && \"Arg operand # out of range!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1328, __PRETTY_FUNCTION__));
  return U - arg_begin();
}

/// Given a value use iterator, return the arg operand number corresponding to
/// it. Iterator must actually correspond to a data operand.
unsigned getArgOperandNo(Value::const_user_iterator UI) const {
  return getArgOperandNo(&UI.getUse());
}

/// Returns true if this CallSite passes the given Value* as an argument to
/// the called function.
bool hasArgument(const Value *V) const {
  return llvm::any_of(args(), [V](const Value *Arg) { return Arg == V; });
}

Value *getCalledOperand() const { return Op<CalledOperandOpEndIdx>(); }

const Use &getCalledOperandUse() const { return Op<CalledOperandOpEndIdx>(); }
Use &getCalledOperandUse() { return Op<CalledOperandOpEndIdx>(); }

/// Returns the function called, or null if this is an
/// indirect function invocation.
Function *getCalledFunction() const {
  return dyn_cast_or_null<Function>(getCalledOperand());
}

/// Return true if the callsite is an indirect call.
bool isIndirectCall() const;

/// Determine whether the passed iterator points to the callee operand's Use.
bool isCallee(Value::const_user_iterator UI) const {
  return isCallee(&UI.getUse());
}

/// Determine whether this Use is the callee operand's Use.
bool isCallee(const Use *U) const { return &getCalledOperandUse() == U; }

/// Helper to get the caller (the parent function).
Function *getCaller();
const Function *getCaller() const {
  return const_cast<CallBase *>(this)->getCaller();
}

/// Tests if this call site must be tail call optimized. Only a CallInst can
/// be tail call optimized.
bool isMustTailCall() const;

/// Tests if this call site is marked as a tail call.
bool isTailCall() const;

/// Returns the intrinsic ID of the intrinsic called or
/// Intrinsic::not_intrinsic if the called function is not an intrinsic, or if
/// this is an indirect call.
Intrinsic::ID getIntrinsicID() const;

void setCalledOperand(Value *V) { Op<CalledOperandOpEndIdx>() = V; }

/// Sets the function called, including updating the function type.
void setCalledFunction(Function *Fn) {
  setCalledFunction(Fn->getFunctionType(), Fn);
}

/// Sets the function called, including updating the function type.
void setCalledFunction(FunctionCallee Fn) {
  setCalledFunction(Fn.getFunctionType(), Fn.getCallee());
}

/// Sets the function called, including updating to the specified function
/// type.
void setCalledFunction(FunctionType *FTy, Value *Fn) {
  this->FTy = FTy;
  assert(FTy == cast<FunctionType>(((FTy == cast<FunctionType>( cast<PointerType>(Fn
->getType())->getElementType())) ? static_cast<void>
 (0) : __assert_fail ("FTy == cast<FunctionType>( cast<PointerType>(Fn->getType())->getElementType())"
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1401, __PRETTY_FUNCTION__))
                    cast<PointerType>(Fn->getType())->getElementType()))((FTy == cast<FunctionType>( cast<PointerType>(Fn
->getType())->getElementType())) ? static_cast<void>
 (0) : __assert_fail ("FTy == cast<FunctionType>( cast<PointerType>(Fn->getType())->getElementType())"
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1401, __PRETTY_FUNCTION__));
  // This function doesn't mutate the return type, only the function
  // type. Seems broken, but I'm just gonna stick an assert in for now.
  assert(getType() == FTy->getReturnType())((getType() == FTy->getReturnType()) ? static_cast<void
> (0) : __assert_fail ("getType() == FTy->getReturnType()"
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1404, __PRETTY_FUNCTION__));
  setCalledOperand(Fn);
}

CallingConv::ID getCallingConv() const {
  return getSubclassData<CallingConvField>();
}

void setCallingConv(CallingConv::ID CC) {
  setSubclassData<CallingConvField>(CC);
}

/// Check if this call is an inline asm statement.
bool isInlineAsm() const { return isa<InlineAsm>(getCalledOperand()); }

/// \name Attribute API
///
/// These methods access and modify attributes on this call (including
/// looking through to the attributes on the called function when necessary).
///@{

/// Return the parameter attributes for this call.
///
AttributeList getAttributes() const { return Attrs; }

/// Set the parameter attributes for this call.
///
void setAttributes(AttributeList A) { Attrs = A; }

/// Determine whether this call has the given attribute. If it does not
/// then determine if the called function has the attribute, but only if
/// the attribute is allowed for the call.
bool hasFnAttr(Attribute::AttrKind Kind) const {
  assert(Kind != Attribute::NoBuiltin &&((Kind != Attribute::NoBuiltin && "Use CallBase::isNoBuiltin() to check for Attribute::NoBuiltin"
) ? static_cast<void> (0) : __assert_fail ("Kind != Attribute::NoBuiltin && \"Use CallBase::isNoBuiltin() to check for Attribute::NoBuiltin\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1438, __PRETTY_FUNCTION__))
23
←
'?' condition is true→
         "Use CallBase::isNoBuiltin() to check for Attribute::NoBuiltin")((Kind != Attribute::NoBuiltin && "Use CallBase::isNoBuiltin() to check for Attribute::NoBuiltin"
) ? static_cast<void> (0) : __assert_fail ("Kind != Attribute::NoBuiltin && \"Use CallBase::isNoBuiltin() to check for Attribute::NoBuiltin\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1438, __PRETTY_FUNCTION__));
  return hasFnAttrImpl(Kind);
24
←
Calling 'CallBase::hasFnAttrImpl'→
33
←
Returning from 'CallBase::hasFnAttrImpl'→
34
←
Returning value, which participates in a condition later→
}

/// Determine whether this call has the given attribute. If it does not
/// then determine if the called function has the attribute, but only if
/// the attribute is allowed for the call.
bool hasFnAttr(StringRef Kind) const { return hasFnAttrImpl(Kind); }

/// adds the attribute to the list of attributes.
void addAttribute(unsigned i, Attribute::AttrKind Kind) {
  AttributeList PAL = getAttributes();
  PAL = PAL.addAttribute(getContext(), i, Kind);
  setAttributes(PAL);
}

/// adds the attribute to the list of attributes.
void addAttribute(unsigned i, Attribute Attr) {
  AttributeList PAL = getAttributes();
  PAL = PAL.addAttribute(getContext(), i, Attr);
  setAttributes(PAL);
}

/// Adds the attribute to the indicated argument
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind) {
  assert(ArgNo < getNumArgOperands() && "Out of bounds")((ArgNo < getNumArgOperands() && "Out of bounds") ?
 static_cast<void> (0) : __assert_fail ("ArgNo < getNumArgOperands() && \"Out of bounds\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1463, __PRETTY_FUNCTION__));
  AttributeList PAL = getAttributes();
  PAL = PAL.addParamAttribute(getContext(), ArgNo, Kind);
  setAttributes(PAL);
}

/// Adds the attribute to the indicated argument
void addParamAttr(unsigned ArgNo, Attribute Attr) {
  assert(ArgNo < getNumArgOperands() && "Out of bounds")((ArgNo < getNumArgOperands() && "Out of bounds") ?
 static_cast<void> (0) : __assert_fail ("ArgNo < getNumArgOperands() && \"Out of bounds\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1471, __PRETTY_FUNCTION__));
  AttributeList PAL = getAttributes();
  PAL = PAL.addParamAttribute(getContext(), ArgNo, Attr);
  setAttributes(PAL);
}

/// removes the attribute from the list of attributes.
void removeAttribute(unsigned i, Attribute::AttrKind Kind) {
  AttributeList PAL = getAttributes();
  PAL = PAL.removeAttribute(getContext(), i, Kind);
  setAttributes(PAL);
}

/// removes the attribute from the list of attributes.
void removeAttribute(unsigned i, StringRef Kind) {
  AttributeList PAL = getAttributes();
  PAL = PAL.removeAttribute(getContext(), i, Kind);
  setAttributes(PAL);
}

void removeAttributes(unsigned i, const AttrBuilder &Attrs) {
  AttributeList PAL = getAttributes();
  PAL = PAL.removeAttributes(getContext(), i, Attrs);
  setAttributes(PAL);
}

/// Removes the attribute from the given argument
void removeParamAttr(unsigned ArgNo, Attribute::AttrKind Kind) {
  assert(ArgNo < getNumArgOperands() && "Out of bounds")((ArgNo < getNumArgOperands() && "Out of bounds") ?
 static_cast<void> (0) : __assert_fail ("ArgNo < getNumArgOperands() && \"Out of bounds\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1499, __PRETTY_FUNCTION__));
  AttributeList PAL = getAttributes();
  PAL = PAL.removeParamAttribute(getContext(), ArgNo, Kind);
  setAttributes(PAL);
}

/// Removes the attribute from the given argument
void removeParamAttr(unsigned ArgNo, StringRef Kind) {
  assert(ArgNo < getNumArgOperands() && "Out of bounds")((ArgNo < getNumArgOperands() && "Out of bounds") ?
 static_cast<void> (0) : __assert_fail ("ArgNo < getNumArgOperands() && \"Out of bounds\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1507, __PRETTY_FUNCTION__));
  AttributeList PAL = getAttributes();
  PAL = PAL.removeParamAttribute(getContext(), ArgNo, Kind);
  setAttributes(PAL);
}

/// adds the dereferenceable attribute to the list of attributes.
void addDereferenceableAttr(unsigned i, uint64_t Bytes) {
  AttributeList PAL = getAttributes();
  PAL = PAL.addDereferenceableAttr(getContext(), i, Bytes);
  setAttributes(PAL);
}

/// adds the dereferenceable_or_null attribute to the list of
/// attributes.
void addDereferenceableOrNullAttr(unsigned i, uint64_t Bytes) {
  AttributeList PAL = getAttributes();
  PAL = PAL.addDereferenceableOrNullAttr(getContext(), i, Bytes);
  setAttributes(PAL);
}

/// Determine whether the return value has the given attribute.
bool hasRetAttr(Attribute::AttrKind Kind) const;

/// Determine whether the argument or parameter has the given attribute.
bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const;

/// Get the attribute of a given kind at a position.
Attribute getAttribute(unsigned i, Attribute::AttrKind Kind) const {
  return getAttributes().getAttribute(i, Kind);
}

/// Get the attribute of a given kind at a position.
Attribute getAttribute(unsigned i, StringRef Kind) const {
  return getAttributes().getAttribute(i, Kind);
}

/// Get the attribute of a given kind from a given arg
Attribute getParamAttr(unsigned ArgNo, Attribute::AttrKind Kind) const {
  assert(ArgNo < getNumArgOperands() && "Out of bounds")((ArgNo < getNumArgOperands() && "Out of bounds") ?
 static_cast<void> (0) : __assert_fail ("ArgNo < getNumArgOperands() && \"Out of bounds\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1546, __PRETTY_FUNCTION__));
  return getAttributes().getParamAttr(ArgNo, Kind);
}

/// Get the attribute of a given kind from a given arg
Attribute getParamAttr(unsigned ArgNo, StringRef Kind) const {
  assert(ArgNo < getNumArgOperands() && "Out of bounds")((ArgNo < getNumArgOperands() && "Out of bounds") ?
 static_cast<void> (0) : __assert_fail ("ArgNo < getNumArgOperands() && \"Out of bounds\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1552, __PRETTY_FUNCTION__));
  return getAttributes().getParamAttr(ArgNo, Kind);
}

/// Return true if the data operand at index \p i has the attribute \p
/// A.
///
/// Data operands include call arguments and values used in operand bundles,
/// but does not include the callee operand.  This routine dispatches to the
/// underlying AttributeList or the OperandBundleUser as appropriate.
///
/// The index \p i is interpreted as
///
///  \p i == Attribute::ReturnIndex  -> the return value
///  \p i in [1, arg_size + 1)  -> argument number (\p i - 1)
///  \p i in [arg_size + 1, data_operand_size + 1) -> bundle operand at index
///     (\p i - 1) in the operand list.
bool dataOperandHasImpliedAttr(unsigned i, Attribute::AttrKind Kind) const {
  // Note that we have to add one because `i` isn't zero-indexed.
  assert(i < (getNumArgOperands() + getNumTotalBundleOperands() + 1) &&((i < (getNumArgOperands() + getNumTotalBundleOperands() +
 1) && "Data operand index out of bounds!") ? static_cast
<void> (0) : __assert_fail ("i < (getNumArgOperands() + getNumTotalBundleOperands() + 1) && \"Data operand index out of bounds!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1572, __PRETTY_FUNCTION__))
         "Data operand index out of bounds!")((i < (getNumArgOperands() + getNumTotalBundleOperands() +
 1) && "Data operand index out of bounds!") ? static_cast
<void> (0) : __assert_fail ("i < (getNumArgOperands() + getNumTotalBundleOperands() + 1) && \"Data operand index out of bounds!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1572, __PRETTY_FUNCTION__));

  // The attribute A can either be directly specified, if the operand in
  // question is a call argument; or be indirectly implied by the kind of its
  // containing operand bundle, if the operand is a bundle operand.

  if (i == AttributeList::ReturnIndex)
    return hasRetAttr(Kind);

  // FIXME: Avoid these i - 1 calculations and update the API to use
  // zero-based indices.
  if (i < (getNumArgOperands() + 1))
    return paramHasAttr(i - 1, Kind);

  assert(hasOperandBundles() && i >= (getBundleOperandsStartIndex() + 1) &&((hasOperandBundles() && i >= (getBundleOperandsStartIndex
() + 1) && "Must be either a call argument or an operand bundle!"
) ? static_cast<void> (0) : __assert_fail ("hasOperandBundles() && i >= (getBundleOperandsStartIndex() + 1) && \"Must be either a call argument or an operand bundle!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1587, __PRETTY_FUNCTION__))
         "Must be either a call argument or an operand bundle!")((hasOperandBundles() && i >= (getBundleOperandsStartIndex
() + 1) && "Must be either a call argument or an operand bundle!"
) ? static_cast<void> (0) : __assert_fail ("hasOperandBundles() && i >= (getBundleOperandsStartIndex() + 1) && \"Must be either a call argument or an operand bundle!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1587, __PRETTY_FUNCTION__));
  return bundleOperandHasAttr(i - 1, Kind);
}

/// Determine whether this data operand is not captured.
// FIXME: Once this API is no longer duplicated in `CallSite`, rename this to
// better indicate that this may return a conservative answer.
bool doesNotCapture(unsigned OpNo) const {
  return dataOperandHasImpliedAttr(OpNo + 1, Attribute::NoCapture);
}

/// Determine whether this argument is passed by value.
bool isByValArgument(unsigned ArgNo) const {
  return paramHasAttr(ArgNo, Attribute::ByVal);
}

/// Determine whether this argument is passed in an alloca.
bool isInAllocaArgument(unsigned ArgNo) const {
  return paramHasAttr(ArgNo, Attribute::InAlloca);
}

/// Determine whether this argument is passed by value, in an alloca, or is
/// preallocated.
bool isPassPointeeByValueArgument(unsigned ArgNo) const {
  return paramHasAttr(ArgNo, Attribute::ByVal) ||
         paramHasAttr(ArgNo, Attribute::InAlloca) ||
         paramHasAttr(ArgNo, Attribute::Preallocated);
}

/// Determine if there are is an inalloca argument. Only the last argument can
/// have the inalloca attribute.
bool hasInAllocaArgument() const {
  return !arg_empty() && paramHasAttr(arg_size() - 1, Attribute::InAlloca);
}

// FIXME: Once this API is no longer duplicated in `CallSite`, rename this to
// better indicate that this may return a conservative answer.
bool doesNotAccessMemory(unsigned OpNo) const {
  return dataOperandHasImpliedAttr(OpNo + 1, Attribute::ReadNone);
}

// FIXME: Once this API is no longer duplicated in `CallSite`, rename this to
// better indicate that this may return a conservative answer.
bool onlyReadsMemory(unsigned OpNo) const {
  return dataOperandHasImpliedAttr(OpNo + 1, Attribute::ReadOnly) ||
         dataOperandHasImpliedAttr(OpNo + 1, Attribute::ReadNone);
}

// FIXME: Once this API is no longer duplicated in `CallSite`, rename this to
// better indicate that this may return a conservative answer.
bool doesNotReadMemory(unsigned OpNo) const {
  return dataOperandHasImpliedAttr(OpNo + 1, Attribute::WriteOnly) ||
         dataOperandHasImpliedAttr(OpNo + 1, Attribute::ReadNone);
}

LLVM_ATTRIBUTE_DEPRECATED(unsigned getRetAlignment() const,unsigned getRetAlignment() const __attribute__((deprecated("Use getRetAlign() instead"
)))
                          "Use getRetAlign() instead")unsigned getRetAlignment() const __attribute__((deprecated("Use getRetAlign() instead"
))) {
  if (const auto MA = Attrs.getRetAlignment())
    return MA->value();
  return 0;
}

/// Extract the alignment of the return value.
MaybeAlign getRetAlign() const { return Attrs.getRetAlignment(); }

/// Extract the alignment for a call or parameter (0=unknown).
LLVM_ATTRIBUTE_DEPRECATED(unsigned getParamAlignment(unsigned ArgNo) const,unsigned getParamAlignment(unsigned ArgNo) const __attribute__
((deprecated("Use getParamAlign() instead")))
                          "Use getParamAlign() instead")unsigned getParamAlignment(unsigned ArgNo) const __attribute__
((deprecated("Use getParamAlign() instead"))) {
  if (const auto MA = Attrs.getParamAlignment(ArgNo))
    return MA->value();
  return 0;
}

/// Extract the alignment for a call or parameter (0=unknown).
MaybeAlign getParamAlign(unsigned ArgNo) const {
  return Attrs.getParamAlignment(ArgNo);
}

/// Extract the byval type for a call or parameter.
Type *getParamByValType(unsigned ArgNo) const {
  Type *Ty = Attrs.getParamByValType(ArgNo);
  return Ty ? Ty : getArgOperand(ArgNo)->getType()->getPointerElementType();
}

/// Extract the preallocated type for a call or parameter.
Type *getParamPreallocatedType(unsigned ArgNo) const {
  Type *Ty = Attrs.getParamPreallocatedType(ArgNo);
  return Ty ? Ty : getArgOperand(ArgNo)->getType()->getPointerElementType();
}

/// Extract the number of dereferenceable bytes for a call or
/// parameter (0=unknown).
uint64_t getDereferenceableBytes(unsigned i) const {
  return Attrs.getDereferenceableBytes(i);
}

/// Extract the number of dereferenceable_or_null bytes for a call or
/// parameter (0=unknown).
uint64_t getDereferenceableOrNullBytes(unsigned i) const {
  return Attrs.getDereferenceableOrNullBytes(i);
}

/// Return true if the return value is known to be not null.
/// This may be because it has the nonnull attribute, or because at least
/// one byte is dereferenceable and the pointer is in addrspace(0).
bool isReturnNonNull() const;

/// Determine if the return value is marked with NoAlias attribute.
bool returnDoesNotAlias() const {
  return Attrs.hasAttribute(AttributeList::ReturnIndex, Attribute::NoAlias);
}

/// If one of the arguments has the 'returned' attribute, returns its
/// operand value. Otherwise, return nullptr.
Value *getReturnedArgOperand() const;

/// Return true if the call should not be treated as a call to a
/// builtin.
bool isNoBuiltin() const {
  return hasFnAttrImpl(Attribute::NoBuiltin) &&
         !hasFnAttrImpl(Attribute::Builtin);
}

/// Determine if the call requires strict floating point semantics.
bool isStrictFP() const { return hasFnAttr(Attribute::StrictFP); }

/// Return true if the call should not be inlined.
bool isNoInline() const { return hasFnAttr(Attribute::NoInline); }
void setIsNoInline() {
  addAttribute(AttributeList::FunctionIndex, Attribute::NoInline);
}
/// Determine if the call does not access memory.
bool doesNotAccessMemory() const { return hasFnAttr(Attribute::ReadNone); }
void setDoesNotAccessMemory() {
  addAttribute(AttributeList::FunctionIndex, Attribute::ReadNone);
}

/// Determine if the call does not access or only reads memory.
bool onlyReadsMemory() const {
  return doesNotAccessMemory() || hasFnAttr(Attribute::ReadOnly);
}
void setOnlyReadsMemory() {
  addAttribute(AttributeList::FunctionIndex, Attribute::ReadOnly);
}

/// Determine if the call does not access or only writes memory.
bool doesNotReadMemory() const {
  return doesNotAccessMemory() || hasFnAttr(Attribute::WriteOnly);
}
void setDoesNotReadMemory() {
  addAttribute(AttributeList::FunctionIndex, Attribute::WriteOnly);
}

/// Determine if the call can access memmory only using pointers based
/// on its arguments.
bool onlyAccessesArgMemory() const {
  return hasFnAttr(Attribute::ArgMemOnly);
}
void setOnlyAccessesArgMemory() {
  addAttribute(AttributeList::FunctionIndex, Attribute::ArgMemOnly);
}

/// Determine if the function may only access memory that is
/// inaccessible from the IR.
bool onlyAccessesInaccessibleMemory() const {
  return hasFnAttr(Attribute::InaccessibleMemOnly);
}
void setOnlyAccessesInaccessibleMemory() {
  addAttribute(AttributeList::FunctionIndex, Attribute::InaccessibleMemOnly);
}

/// Determine if the function may only access memory that is
/// either inaccessible from the IR or pointed to by its arguments.
bool onlyAccessesInaccessibleMemOrArgMem() const {
  return hasFnAttr(Attribute::InaccessibleMemOrArgMemOnly);
}
void setOnlyAccessesInaccessibleMemOrArgMem() {
  addAttribute(AttributeList::FunctionIndex,
               Attribute::InaccessibleMemOrArgMemOnly);
}
/// Determine if the call cannot return.
bool doesNotReturn() const { return hasFnAttr(Attribute::NoReturn); }
void setDoesNotReturn() {
  addAttribute(AttributeList::FunctionIndex, Attribute::NoReturn);
}

/// Determine if the call should not perform indirect branch tracking.
bool doesNoCfCheck() const { return hasFnAttr(Attribute::NoCfCheck); }

/// Determine if the call cannot unwind.
bool doesNotThrow() const { return hasFnAttr(Attribute::NoUnwind); }
void setDoesNotThrow() {
  addAttribute(AttributeList::FunctionIndex, Attribute::NoUnwind);
}

/// Determine if the invoke cannot be duplicated.
bool cannotDuplicate() const { return hasFnAttr(Attribute::NoDuplicate); }
void setCannotDuplicate() {
  addAttribute(AttributeList::FunctionIndex, Attribute::NoDuplicate);
}

/// Determine if the call cannot be tail merged.
bool cannotMerge() const { return hasFnAttr(Attribute::NoMerge); }
void setCannotMerge() {
  addAttribute(AttributeList::FunctionIndex, Attribute::NoMerge);
}

/// Determine if the invoke is convergent
bool isConvergent() const { return hasFnAttr(Attribute::Convergent); }
22
←
Calling 'CallBase::hasFnAttr'→
35
←
Returning from 'CallBase::hasFnAttr'→
36
←
Returning value, which participates in a condition later→
void setConvergent() {
  addAttribute(AttributeList::FunctionIndex, Attribute::Convergent);
}
void setNotConvergent() {
  removeAttribute(AttributeList::FunctionIndex, Attribute::Convergent);
}

/// Determine if the call returns a structure through first
/// pointer argument.
bool hasStructRetAttr() const {
  if (getNumArgOperands() == 0)
    return false;

  // Be friendly and also check the callee.
  return paramHasAttr(0, Attribute::StructRet);
}

/// Determine if any call argument is an aggregate passed by value.
bool hasByValArgument() const {
  return Attrs.hasAttrSomewhere(Attribute::ByVal);
}

///@{
// End of attribute API.

/// \name Operand Bundle API
///
/// This group of methods provides the API to access and manipulate operand
/// bundles on this call.
/// @{

/// Return the number of operand bundles associated with this User.
unsigned getNumOperandBundles() const {
  return std::distance(bundle_op_info_begin(), bundle_op_info_end());
}

/// Return true if this User has any operand bundles.
bool hasOperandBundles() const { return getNumOperandBundles() != 0; }

/// Return the index of the first bundle operand in the Use array.
unsigned getBundleOperandsStartIndex() const {
  assert(hasOperandBundles() && "Don't call otherwise!")((hasOperandBundles() && "Don't call otherwise!") ? static_cast
<void> (0) : __assert_fail ("hasOperandBundles() && \"Don't call otherwise!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1837, __PRETTY_FUNCTION__));
  return bundle_op_info_begin()->Begin;
}

/// Return the index of the last bundle operand in the Use array.
unsigned getBundleOperandsEndIndex() const {
  assert(hasOperandBundles() && "Don't call otherwise!")((hasOperandBundles() && "Don't call otherwise!") ? static_cast
<void> (0) : __assert_fail ("hasOperandBundles() && \"Don't call otherwise!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1843, __PRETTY_FUNCTION__));
  return bundle_op_info_end()[-1].End;
}

/// Return true if the operand at index \p Idx is a bundle operand.
bool isBundleOperand(unsigned Idx) const {
  return hasOperandBundles() && Idx >= getBundleOperandsStartIndex() &&
         Idx < getBundleOperandsEndIndex();
}

/// Returns true if the use is a bundle operand.
bool isBundleOperand(const Use *U) const {
  assert(this == U->getUser() &&((this == U->getUser() && "Only valid to query with a use of this instruction!"
) ? static_cast<void> (0) : __assert_fail ("this == U->getUser() && \"Only valid to query with a use of this instruction!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1856, __PRETTY_FUNCTION__))
         "Only valid to query with a use of this instruction!")((this == U->getUser() && "Only valid to query with a use of this instruction!"
) ? static_cast<void> (0) : __assert_fail ("this == U->getUser() && \"Only valid to query with a use of this instruction!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1856, __PRETTY_FUNCTION__));
  return hasOperandBundles() && isBundleOperand(U - op_begin());
}
bool isBundleOperand(Value::const_user_iterator UI) const {
  return isBundleOperand(&UI.getUse());
}

/// Return the total number operands (not operand bundles) used by
/// every operand bundle in this OperandBundleUser.
unsigned getNumTotalBundleOperands() const {
  if (!hasOperandBundles())
    return 0;

  unsigned Begin = getBundleOperandsStartIndex();
  unsigned End = getBundleOperandsEndIndex();

  assert(Begin <= End && "Should be!")((Begin <= End && "Should be!") ? static_cast<void
> (0) : __assert_fail ("Begin <= End && \"Should be!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1872, __PRETTY_FUNCTION__));
  return End - Begin;
}

/// Return the operand bundle at a specific index.
OperandBundleUse getOperandBundleAt(unsigned Index) const {
  assert(Index < getNumOperandBundles() && "Index out of bounds!")((Index < getNumOperandBundles() && "Index out of bounds!"
) ? static_cast<void> (0) : __assert_fail ("Index < getNumOperandBundles() && \"Index out of bounds!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1878, __PRETTY_FUNCTION__));
  return operandBundleFromBundleOpInfo(*(bundle_op_info_begin() + Index));
}

/// Return the number of operand bundles with the tag Name attached to
/// this instruction.
unsigned countOperandBundlesOfType(StringRef Name) const {
  unsigned Count = 0;
  for (unsigned i = 0, e = getNumOperandBundles(); i != e; ++i)
    if (getOperandBundleAt(i).getTagName() == Name)
      Count++;

  return Count;
}

/// Return the number of operand bundles with the tag ID attached to
/// this instruction.
unsigned countOperandBundlesOfType(uint32_t ID) const {
  unsigned Count = 0;
  for (unsigned i = 0, e = getNumOperandBundles(); i != e; ++i)
    if (getOperandBundleAt(i).getTagID() == ID)
      Count++;

  return Count;
}

/// Return an operand bundle by name, if present.
///
/// It is an error to call this for operand bundle types that may have
/// multiple instances of them on the same instruction.
Optional<OperandBundleUse> getOperandBundle(StringRef Name) const {
  assert(countOperandBundlesOfType(Name) < 2 && "Precondition violated!")((countOperandBundlesOfType(Name) < 2 && "Precondition violated!"
) ? static_cast<void> (0) : __assert_fail ("countOperandBundlesOfType(Name) < 2 && \"Precondition violated!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1909, __PRETTY_FUNCTION__));

  for (unsigned i = 0, e = getNumOperandBundles(); i != e; ++i) {
    OperandBundleUse U = getOperandBundleAt(i);
    if (U.getTagName() == Name)
      return U;
  }

  return None;
}

/// Return an operand bundle by tag ID, if present.
///
/// It is an error to call this for operand bundle types that may have
/// multiple instances of them on the same instruction.
Optional<OperandBundleUse> getOperandBundle(uint32_t ID) const {
  assert(countOperandBundlesOfType(ID) < 2 && "Precondition violated!")((countOperandBundlesOfType(ID) < 2 && "Precondition violated!"
) ? static_cast<void> (0) : __assert_fail ("countOperandBundlesOfType(ID) < 2 && \"Precondition violated!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 1925, __PRETTY_FUNCTION__));

  for (unsigned i = 0, e = getNumOperandBundles(); i != e; ++i) {
    OperandBundleUse U = getOperandBundleAt(i);
    if (U.getTagID() == ID)
      return U;
  }

  return None;
}

/// Return the list of operand bundles attached to this instruction as
/// a vector of OperandBundleDefs.
///
/// This function copies the OperandBundeUse instances associated with this
/// OperandBundleUser to a vector of OperandBundleDefs.  Note:
/// OperandBundeUses and OperandBundleDefs are non-trivially *different*
/// representations of operand bundles (see documentation above).
void getOperandBundlesAsDefs(SmallVectorImpl<OperandBundleDef> &Defs) const;

/// Return the operand bundle for the operand at index OpIdx.
///
/// It is an error to call this with an OpIdx that does not correspond to an
/// bundle operand.
OperandBundleUse getOperandBundleForOperand(unsigned OpIdx) const {
  return operandBundleFromBundleOpInfo(getBundleOpInfoForOperand(OpIdx));
}

/// Return true if this operand bundle user has operand bundles that
/// may read from the heap.
bool hasReadingOperandBundles() const {
  // Implementation note: this is a conservative implementation of operand
  // bundle semantics, where *any* operand bundle forces a callsite to be at
  // least readonly.
  return hasOperandBundles();
}

/// Return true if this operand bundle user has operand bundles that
/// may write to the heap.
bool hasClobberingOperandBundles() const {
  for (auto &BOI : bundle_op_infos()) {
    if (BOI.Tag->second == LLVMContext::OB_deopt ||
        BOI.Tag->second == LLVMContext::OB_funclet)
      continue;

    // This instruction has an operand bundle that is not known to us.
    // Assume the worst.
    return true;
  }

  return false;
}

/// Return true if the bundle operand at index \p OpIdx has the
/// attribute \p A.
bool bundleOperandHasAttr(unsigned OpIdx,  Attribute::AttrKind A) const {
  auto &BOI = getBundleOpInfoForOperand(OpIdx);
  auto OBU = operandBundleFromBundleOpInfo(BOI);
  return OBU.operandHasAttr(OpIdx - BOI.Begin, A);
}

/// Return true if \p Other has the same sequence of operand bundle
/// tags with the same number of operands on each one of them as this
/// OperandBundleUser.
bool hasIdenticalOperandBundleSchema(const CallBase &Other) const {
  if (getNumOperandBundles() != Other.getNumOperandBundles())
    return false;

  return std::equal(bundle_op_info_begin(), bundle_op_info_end(),
                    Other.bundle_op_info_begin());
}

/// Return true if this operand bundle user contains operand bundles
/// with tags other than those specified in \p IDs.
bool hasOperandBundlesOtherThan(ArrayRef<uint32_t> IDs) const {
  for (unsigned i = 0, e = getNumOperandBundles(); i != e; ++i) {
    uint32_t ID = getOperandBundleAt(i).getTagID();
    if (!is_contained(IDs, ID))
      return true;
  }
  return false;
}

/// Is the function attribute S disallowed by some operand bundle on
/// this operand bundle user?
bool isFnAttrDisallowedByOpBundle(StringRef S) const {
  // Operand bundles only possibly disallow readnone, readonly and argmemonly
  // attributes.  All String attributes are fine.
  return false;
}

/// Is the function attribute A disallowed by some operand bundle on
/// this operand bundle user?
bool isFnAttrDisallowedByOpBundle(Attribute::AttrKind A) const {
  switch (A) {
28
←
Control jumps to the 'default' case at line 2020→
  default:
    return false;
29
←
Returning zero, which participates in a condition later→

  case Attribute::InaccessibleMemOrArgMemOnly:
    return hasReadingOperandBundles();

  case Attribute::InaccessibleMemOnly:
    return hasReadingOperandBundles();

  case Attribute::ArgMemOnly:
    return hasReadingOperandBundles();

  case Attribute::ReadNone:
    return hasReadingOperandBundles();

  case Attribute::ReadOnly:
    return hasClobberingOperandBundles();
  }

  llvm_unreachable("switch has a default case!")::llvm::llvm_unreachable_internal("switch has a default case!"
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 2039);
}

/// Used to keep track of an operand bundle.  See the main comment on
/// OperandBundleUser above.
struct BundleOpInfo {
  /// The operand bundle tag, interned by
  /// LLVMContextImpl::getOrInsertBundleTag.
  StringMapEntry<uint32_t> *Tag;

  /// The index in the Use& vector where operands for this operand
  /// bundle starts.
  uint32_t Begin;

  /// The index in the Use& vector where operands for this operand
  /// bundle ends.
  uint32_t End;

  bool operator==(const BundleOpInfo &Other) const {
    return Tag == Other.Tag && Begin == Other.Begin && End == Other.End;
  }
};

/// Simple helper function to map a BundleOpInfo to an
/// OperandBundleUse.
OperandBundleUse
operandBundleFromBundleOpInfo(const BundleOpInfo &BOI) const {
  auto begin = op_begin();
  ArrayRef<Use> Inputs(begin + BOI.Begin, begin + BOI.End);
  return OperandBundleUse(BOI.Tag, Inputs);
}

using bundle_op_iterator = BundleOpInfo *;
using const_bundle_op_iterator = const BundleOpInfo *;

/// Return the start of the list of BundleOpInfo instances associated
/// with this OperandBundleUser.
///
/// OperandBundleUser uses the descriptor area co-allocated with the host User
/// to store some meta information about which operands are "normal" operands,
/// and which ones belong to some operand bundle.
///
/// The layout of an operand bundle user is
///
///          +-----------uint32_t End-------------------------------------+
///          |                                                            |
///          |  +--------uint32_t Begin--------------------+              |
///          |  |                                          |              |
///          ^  ^                                          v              v
///  |------|------|----|----|----|----|----|---------|----|---------|----|-----
///  | BOI0 | BOI1 | .. | DU | U0 | U1 | .. | BOI0_U0 | .. | BOI1_U0 | .. | Un
///  |------|------|----|----|----|----|----|---------|----|---------|----|-----
///   v  v                                  ^              ^
///   |  |                                  |              |
///   |  +--------uint32_t Begin------------+              |
///   |                                                    |
///   +-----------uint32_t End-----------------------------+
///
///
/// BOI0, BOI1 ... are descriptions of operand bundles in this User's use
/// list. These descriptions are installed and managed by this class, and
/// they're all instances of OperandBundleUser<T>::BundleOpInfo.
///
/// DU is an additional descriptor installed by User's 'operator new' to keep
/// track of the 'BOI0 ... BOIN' co-allocation.  OperandBundleUser does not
/// access or modify DU in any way, it's an implementation detail private to
/// User.
///
/// The regular Use& vector for the User starts at U0.  The operand bundle
/// uses are part of the Use& vector, just like normal uses.  In the diagram
/// above, the operand bundle uses start at BOI0_U0.  Each instance of
/// BundleOpInfo has information about a contiguous set of uses constituting
/// an operand bundle, and the total set of operand bundle uses themselves
/// form a contiguous set of uses (i.e. there are no gaps between uses
/// corresponding to individual operand bundles).
///
/// This class does not know the location of the set of operand bundle uses
/// within the use list -- that is decided by the User using this class via
/// the BeginIdx argument in populateBundleOperandInfos.
///
/// Currently operand bundle users with hung-off operands are not supported.
bundle_op_iterator bundle_op_info_begin() {
  if (!hasDescriptor())
    return nullptr;

  uint8_t *BytesBegin = getDescriptor().begin();
  return reinterpret_cast<bundle_op_iterator>(BytesBegin);
}

/// Return the start of the list of BundleOpInfo instances associated
/// with this OperandBundleUser.
const_bundle_op_iterator bundle_op_info_begin() const {
  auto *NonConstThis = const_cast<CallBase *>(this);
  return NonConstThis->bundle_op_info_begin();
}

/// Return the end of the list of BundleOpInfo instances associated
/// with this OperandBundleUser.
bundle_op_iterator bundle_op_info_end() {
  if (!hasDescriptor())
    return nullptr;

  uint8_t *BytesEnd = getDescriptor().end();
  return reinterpret_cast<bundle_op_iterator>(BytesEnd);
}

/// Return the end of the list of BundleOpInfo instances associated
/// with this OperandBundleUser.
const_bundle_op_iterator bundle_op_info_end() const {
  auto *NonConstThis = const_cast<CallBase *>(this);
  return NonConstThis->bundle_op_info_end();
}

/// Return the range [\p bundle_op_info_begin, \p bundle_op_info_end).
iterator_range<bundle_op_iterator> bundle_op_infos() {
  return make_range(bundle_op_info_begin(), bundle_op_info_end());
}

/// Return the range [\p bundle_op_info_begin, \p bundle_op_info_end).
iterator_range<const_bundle_op_iterator> bundle_op_infos() const {
  return make_range(bundle_op_info_begin(), bundle_op_info_end());
}

/// Populate the BundleOpInfo instances and the Use& vector from \p
/// Bundles.  Return the op_iterator pointing to the Use& one past the last
/// last bundle operand use.
///
/// Each \p OperandBundleDef instance is tracked by a OperandBundleInfo
/// instance allocated in this User's descriptor.
op_iterator populateBundleOperandInfos(ArrayRef<OperandBundleDef> Bundles,
                                       const unsigned BeginIndex);

2171public:
/// Return the BundleOpInfo for the operand at index OpIdx.
///
/// It is an error to call this with an OpIdx that does not correspond to an
/// bundle operand.
BundleOpInfo &getBundleOpInfoForOperand(unsigned OpIdx);
const BundleOpInfo &getBundleOpInfoForOperand(unsigned OpIdx) const {
  return const_cast<CallBase *>(this)->getBundleOpInfoForOperand(OpIdx);
}

2181protected:
/// Return the total number of values used in \p Bundles.
static unsigned CountBundleInputs(ArrayRef<OperandBundleDef> Bundles) {
  unsigned Total = 0;
  for (auto &B : Bundles)
    Total += B.input_size();
  return Total;
}

/// @}
// End of operand bundle API.

2193private:
bool hasFnAttrOnCalledFunction(Attribute::AttrKind Kind) const;
bool hasFnAttrOnCalledFunction(StringRef Kind) const;

template <typename AttrKind> bool hasFnAttrImpl(AttrKind Kind) const {
  if (Attrs.hasFnAttribute(Kind))
25
←
Assuming the condition is false→
26
←
Taking false branch→
    return true;

  // Operand bundles override attributes on the called function, but don't
  // override attributes directly present on the call instruction.
  if (isFnAttrDisallowedByOpBundle(Kind))
27
←
Calling 'CallBase::isFnAttrDisallowedByOpBundle'→
30
←
Returning from 'CallBase::isFnAttrDisallowedByOpBundle'→
31
←
Taking false branch→
    return false;

  return hasFnAttrOnCalledFunction(Kind);
32
←
Returning value, which participates in a condition later→
}
2208};

2210template <>
2211struct OperandTraits<CallBase> : public VariadicOperandTraits<CallBase, 1> {};

2213DEFINE_TRANSPARENT_OPERAND_ACCESSORS(CallBase, Value)CallBase::op_iterator CallBase::op_begin() { return OperandTraits
<CallBase>::op_begin(this); } CallBase::const_op_iterator
 CallBase::op_begin() const { return OperandTraits<CallBase
>::op_begin(const_cast<CallBase*>(this)); } CallBase
::op_iterator CallBase::op_end() { return OperandTraits<CallBase
>::op_end(this); } CallBase::const_op_iterator CallBase::op_end
() const { return OperandTraits<CallBase>::op_end(const_cast
<CallBase*>(this)); } Value *CallBase::getOperand(unsigned
 i_nocapture) const { ((i_nocapture < OperandTraits<CallBase
>::operands(this) && "getOperand() out of range!")
 ? static_cast<void> (0) : __assert_fail ("i_nocapture < OperandTraits<CallBase>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 2213, __PRETTY_FUNCTION__)); return cast_or_null<Value>
( OperandTraits<CallBase>::op_begin(const_cast<CallBase
*>(this))[i_nocapture].get()); } void CallBase::setOperand
(unsigned i_nocapture, Value *Val_nocapture) { ((i_nocapture <
 OperandTraits<CallBase>::operands(this) && "setOperand() out of range!"
) ? static_cast<void> (0) : __assert_fail ("i_nocapture < OperandTraits<CallBase>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 2213, __PRETTY_FUNCTION__)); OperandTraits<CallBase>::
op_begin(this)[i_nocapture] = Val_nocapture; } unsigned CallBase
::getNumOperands() const { return OperandTraits<CallBase>
::operands(this); } template <int Idx_nocapture> Use &
CallBase::Op() { return this->OpFrom<Idx_nocapture>(
this); } template <int Idx_nocapture> const Use &CallBase
::Op() const { return this->OpFrom<Idx_nocapture>(this
); }

2215//===----------------------------------------------------------------------===//
2216//                           FuncletPadInst Class
2217//===----------------------------------------------------------------------===//
2218class FuncletPadInst : public Instruction {
2219private:
FuncletPadInst(const FuncletPadInst &CPI);

explicit FuncletPadInst(Instruction::FuncletPadOps Op, Value *ParentPad,
                        ArrayRef<Value *> Args, unsigned Values,
                        const Twine &NameStr, Instruction *InsertBefore);
explicit FuncletPadInst(Instruction::FuncletPadOps Op, Value *ParentPad,
                        ArrayRef<Value *> Args, unsigned Values,
                        const Twine &NameStr, BasicBlock *InsertAtEnd);

void init(Value *ParentPad, ArrayRef<Value *> Args, const Twine &NameStr);

2231protected:
// Note: Instruction needs to be a friend here to call cloneImpl.
friend class Instruction;
friend class CatchPadInst;
friend class CleanupPadInst;

FuncletPadInst *cloneImpl() const;

2239public:
/// Provide fast operand accessors
DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
 setOperand(unsigned, Value*); inline op_iterator op_begin();
 inline const_op_iterator op_begin() const; inline op_iterator
 op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
 getNumOperands() const;

/// getNumArgOperands - Return the number of funcletpad arguments.
///
unsigned getNumArgOperands() const { return getNumOperands() - 1; }

/// Convenience accessors

/// Return the outer EH-pad this funclet is nested within.
///
/// Note: This returns the associated CatchSwitchInst if this FuncletPadInst
/// is a CatchPadInst.
Value *getParentPad() const { return Op<-1>(); }
void setParentPad(Value *ParentPad) {
  assert(ParentPad)((ParentPad) ? static_cast<void> (0) : __assert_fail ("ParentPad"
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 2255, __PRETTY_FUNCTION__));
  Op<-1>() = ParentPad;
}

/// getArgOperand/setArgOperand - Return/set the i-th funcletpad argument.
///
Value *getArgOperand(unsigned i) const { return getOperand(i); }
void setArgOperand(unsigned i, Value *v) { setOperand(i, v); }

/// arg_operands - iteration adapter for range-for loops.
op_range arg_operands() { return op_range(op_begin(), op_end() - 1); }

/// arg_operands - iteration adapter for range-for loops.
const_op_range arg_operands() const {
  return const_op_range(op_begin(), op_end() - 1);
}

// Methods for support type inquiry through isa, cast, and dyn_cast:
static bool classof(const Instruction *I) { return I->isFuncletPad(); }
static bool classof(const Value *V) {
  return isa<Instruction>(V) && classof(cast<Instruction>(V));
}
2277};

2279template <>
2280struct OperandTraits<FuncletPadInst>
  : public VariadicOperandTraits<FuncletPadInst, /*MINARITY=*/1> {};

2283DEFINE_TRANSPARENT_OPERAND_ACCESSORS(FuncletPadInst, Value)FuncletPadInst::op_iterator FuncletPadInst::op_begin() { return
 OperandTraits<FuncletPadInst>::op_begin(this); } FuncletPadInst
::const_op_iterator FuncletPadInst::op_begin() const { return
 OperandTraits<FuncletPadInst>::op_begin(const_cast<
FuncletPadInst*>(this)); } FuncletPadInst::op_iterator FuncletPadInst
::op_end() { return OperandTraits<FuncletPadInst>::op_end
(this); } FuncletPadInst::const_op_iterator FuncletPadInst::op_end
() const { return OperandTraits<FuncletPadInst>::op_end
(const_cast<FuncletPadInst*>(this)); } Value *FuncletPadInst
::getOperand(unsigned i_nocapture) const { ((i_nocapture <
 OperandTraits<FuncletPadInst>::operands(this) &&
 "getOperand() out of range!") ? static_cast<void> (0) :
 __assert_fail ("i_nocapture < OperandTraits<FuncletPadInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 2283, __PRETTY_FUNCTION__)); return cast_or_null<Value>
( OperandTraits<FuncletPadInst>::op_begin(const_cast<
FuncletPadInst*>(this))[i_nocapture].get()); } void FuncletPadInst
::setOperand(unsigned i_nocapture, Value *Val_nocapture) { ((
i_nocapture < OperandTraits<FuncletPadInst>::operands
(this) && "setOperand() out of range!") ? static_cast
<void> (0) : __assert_fail ("i_nocapture < OperandTraits<FuncletPadInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/InstrTypes.h"
, 2283, __PRETTY_FUNCTION__)); OperandTraits<FuncletPadInst
>::op_begin(this)[i_nocapture] = Val_nocapture; } unsigned
 FuncletPadInst::getNumOperands() const { return OperandTraits
<FuncletPadInst>::operands(this); } template <int Idx_nocapture
> Use &FuncletPadInst::Op() { return this->OpFrom<
Idx_nocapture>(this); } template <int Idx_nocapture>
 const Use &FuncletPadInst::Op() const { return this->
OpFrom<Idx_nocapture>(this); }

2285} // end namespace llvm

2287#endif // LLVM_IR_INSTRTYPES_H

←

/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/PatternMatch.h

→

1//===- PatternMatch.h - Match on the LLVM IR --------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file provides a simple and efficient mechanism for performing general
10// tree-based pattern matches on the LLVM IR. The power of these routines is
11// that it allows you to write concise patterns that are expressive and easy to
12// understand. The other major advantage of this is that it allows you to
13// trivially capture/bind elements in the pattern to variables. For example,
14// you can do something like this:
15//
16//  Value *Exp = ...
17//  Value *X, *Y;  ConstantInt *C1, *C2;      // (X & C1) | (Y & C2)
18//  if (match(Exp, m_Or(m_And(m_Value(X), m_ConstantInt(C1)),
19//                      m_And(m_Value(Y), m_ConstantInt(C2))))) {
20//    ... Pattern is matched and variables are bound ...
21//  }
22//
23// This is primarily useful to things like the instruction combiner, but can
24// also be useful for static analysis tools or code generators.
25//
26//===----------------------------------------------------------------------===//
27 
28#ifndef LLVM_IR_PATTERNMATCH_H
29#define LLVM_IR_PATTERNMATCH_H
30 
31#include "llvm/ADT/APFloat.h"
32#include "llvm/ADT/APInt.h"
33#include "llvm/IR/Constant.h"
34#include "llvm/IR/Constants.h"
35#include "llvm/IR/DataLayout.h"
36#include "llvm/IR/InstrTypes.h"
37#include "llvm/IR/Instruction.h"
38#include "llvm/IR/Instructions.h"
39#include "llvm/IR/IntrinsicInst.h"
40#include "llvm/IR/Intrinsics.h"
41#include "llvm/IR/Operator.h"
42#include "llvm/IR/Value.h"
43#include "llvm/Support/Casting.h"
44#include <cstdint>
45 
46namespace llvm {
47namespace PatternMatch {
48 
49template <typename Val, typename Pattern> bool match(Val *V, const Pattern &P) {
50  return const_cast<Pattern &>(P).match(V);
41
←
Calling 'IntrinsicID_match::match'→
45
←
Returning from 'IntrinsicID_match::match'→
46
←
Returning zero, which participates in a condition later→
50
←
Calling 'IntrinsicID_match::match'→
54
←
Returning from 'IntrinsicID_match::match'→
55
←
Returning zero, which participates in a condition later→
51}
52 
53template <typename Pattern> bool match(ArrayRef<int> Mask, const Pattern &P) {
54  return const_cast<Pattern &>(P).match(Mask);
55}
56 
57template <typename SubPattern_t> struct OneUse_match {
58  SubPattern_t SubPattern;
59 
60  OneUse_match(const SubPattern_t &SP) : SubPattern(SP) {}
61 
62  template <typename OpTy> bool match(OpTy *V) {
63    return V->hasOneUse() && SubPattern.match(V);
64  }
65};
66 
67template <typename T> inline OneUse_match<T> m_OneUse(const T &SubPattern) {
68  return SubPattern;
69}
70 
71template <typename Class> struct class_match {
72  template <typename ITy> bool match(ITy *V) { return isa<Class>(V); }
73};
74 
75/// Match an arbitrary value and ignore it.
76inline class_match<Value> m_Value() { return class_match<Value>(); }
77 
78/// Match an arbitrary unary operation and ignore it.
79inline class_match<UnaryOperator> m_UnOp() {
80  return class_match<UnaryOperator>();
81}
82 
83/// Match an arbitrary binary operation and ignore it.
84inline class_match<BinaryOperator> m_BinOp() {
85  return class_match<BinaryOperator>();
86}
87 
88/// Matches any compare instruction and ignore it.
89inline class_match<CmpInst> m_Cmp() { return class_match<CmpInst>(); }
90 
91/// Match an arbitrary ConstantInt and ignore it.
92inline class_match<ConstantInt> m_ConstantInt() {
93  return class_match<ConstantInt>();
94}
95 
96/// Match an arbitrary undef constant.
97inline class_match<UndefValue> m_Undef() { return class_match<UndefValue>(); }
98 
99/// Match an arbitrary Constant and ignore it.
100inline class_match<Constant> m_Constant() { return class_match<Constant>(); }
101 
102/// Match an arbitrary basic block value and ignore it.
103inline class_match<BasicBlock> m_BasicBlock() {
104  return class_match<BasicBlock>();
105}
106 
107/// Inverting matcher
108template <typename Ty> struct match_unless {
109  Ty M;
110 
111  match_unless(const Ty &Matcher) : M(Matcher) {}
112 
113  template <typename ITy> bool match(ITy *V) { return !M.match(V); }
114};
115 
116/// Match if the inner matcher does *NOT* match.
117template <typename Ty> inline match_unless<Ty> m_Unless(const Ty &M) {
118  return match_unless<Ty>(M);
119}
120 
121/// Matching combinators
122template <typename LTy, typename RTy> struct match_combine_or {
123  LTy L;
124  RTy R;
125 
126  match_combine_or(const LTy &Left, const RTy &Right) : L(Left), R(Right) {}
127 
128  template <typename ITy> bool match(ITy *V) {
129    if (L.match(V))
130      return true;
131    if (R.match(V))
132      return true;
133    return false;
134  }
135};
136 
137template <typename LTy, typename RTy> struct match_combine_and {
138  LTy L;
139  RTy R;
140 
141  match_combine_and(const LTy &Left, const RTy &Right) : L(Left), R(Right) {}
142 
143  template <typename ITy> bool match(ITy *V) {
144    if (L.match(V))
145      if (R.match(V))
146        return true;
147    return false;
148  }
149};
150 
151/// Combine two pattern matchers matching L || R
152template <typename LTy, typename RTy>
153inline match_combine_or<LTy, RTy> m_CombineOr(const LTy &L, const RTy &R) {
154  return match_combine_or<LTy, RTy>(L, R);
155}
156 
157/// Combine two pattern matchers matching L && R
158template <typename LTy, typename RTy>
159inline match_combine_and<LTy, RTy> m_CombineAnd(const LTy &L, const RTy &R) {
160  return match_combine_and<LTy, RTy>(L, R);
161}
162 
163struct apint_match {
164  const APInt *&Res;
165  bool AllowUndef;
166 
167  apint_match(const APInt *&Res, bool AllowUndef)
168    : Res(Res), AllowUndef(AllowUndef) {}
169 
170  template <typename ITy> bool match(ITy *V) {
171    if (auto *CI = dyn_cast<ConstantInt>(V)) {
172      Res = &CI->getValue();
173      return true;
174    }
175    if (V->getType()->isVectorTy())
176      if (const auto *C = dyn_cast<Constant>(V))
177        if (auto *CI = dyn_cast_or_null<ConstantInt>(
178                C->getSplatValue(AllowUndef))) {
179          Res = &CI->getValue();
180          return true;
181        }
182    return false;
183  }
184};
185// Either constexpr if or renaming ConstantFP::getValueAPF to
186// ConstantFP::getValue is needed to do it via single template
187// function for both apint/apfloat.
188struct apfloat_match {
189  const APFloat *&Res;
190  bool AllowUndef;
191 
192  apfloat_match(const APFloat *&Res, bool AllowUndef)
193      : Res(Res), AllowUndef(AllowUndef) {}
194 
195  template <typename ITy> bool match(ITy *V) {
196    if (auto *CI = dyn_cast<ConstantFP>(V)) {
197      Res = &CI->getValueAPF();
198      return true;
199    }
200    if (V->getType()->isVectorTy())
201      if (const auto *C = dyn_cast<Constant>(V))
202        if (auto *CI = dyn_cast_or_null<ConstantFP>(
203                C->getSplatValue(AllowUndef))) {
204          Res = &CI->getValueAPF();
205          return true;
206        }
207    return false;
208  }
209};
210 
211/// Match a ConstantInt or splatted ConstantVector, binding the
212/// specified pointer to the contained APInt.
213inline apint_match m_APInt(const APInt *&Res) {
214  // Forbid undefs by default to maintain previous behavior.
215  return apint_match(Res, /* AllowUndef */ false);
216}
217 
218/// Match APInt while allowing undefs in splat vector constants.
219inline apint_match m_APIntAllowUndef(const APInt *&Res) {
220  return apint_match(Res, /* AllowUndef */ true);
221}
222 
223/// Match APInt while forbidding undefs in splat vector constants.
224inline apint_match m_APIntForbidUndef(const APInt *&Res) {
225  return apint_match(Res, /* AllowUndef */ false);
226}
227 
228/// Match a ConstantFP or splatted ConstantVector, binding the
229/// specified pointer to the contained APFloat.
230inline apfloat_match m_APFloat(const APFloat *&Res) {
231  // Forbid undefs by default to maintain previous behavior.
232  return apfloat_match(Res, /* AllowUndef */ false);
233}
234 
235/// Match APFloat while allowing undefs in splat vector constants.
236inline apfloat_match m_APFloatAllowUndef(const APFloat *&Res) {
237  return apfloat_match(Res, /* AllowUndef */ true);
238}
239 
240/// Match APFloat while forbidding undefs in splat vector constants.
241inline apfloat_match m_APFloatForbidUndef(const APFloat *&Res) {
242  return apfloat_match(Res, /* AllowUndef */ false);
243}
244 
245template <int64_t Val> struct constantint_match {
246  template <typename ITy> bool match(ITy *V) {
247    if (const auto *CI = dyn_cast<ConstantInt>(V)) {
248      const APInt &CIV = CI->getValue();
249      if (Val >= 0)
250        return CIV == static_cast<uint64_t>(Val);
251      // If Val is negative, and CI is shorter than it, truncate to the right
252      // number of bits.  If it is larger, then we have to sign extend.  Just
253      // compare their negated values.
254      return -CIV == -Val;
255    }
256    return false;
257  }
258};
259 
260/// Match a ConstantInt with a specific value.
261template <int64_t Val> inline constantint_match<Val> m_ConstantInt() {
262  return constantint_match<Val>();
263}
264 
265/// This helper class is used to match constant scalars, vector splats,
266/// and fixed width vectors that satisfy a specified predicate.
267/// For fixed width vector constants, undefined elements are ignored.
268template <typename Predicate, typename ConstantVal>
269struct cstval_pred_ty : public Predicate {
270  template <typename ITy> bool match(ITy *V) {
271    if (const auto *CV = dyn_cast<ConstantVal>(V))
272      return this->isValue(CV->getValue());
273    if (const auto *VTy = dyn_cast<VectorType>(V->getType())) {
274      if (const auto *C = dyn_cast<Constant>(V)) {
275        if (const auto *CV = dyn_cast_or_null<ConstantVal>(C->getSplatValue()))
276          return this->isValue(CV->getValue());
277 
278        // Number of elements of a scalable vector unknown at compile time
279        auto *FVTy = dyn_cast<FixedVectorType>(VTy);
280        if (!FVTy)
281          return false;
282 
283        // Non-splat vector constant: check each element for a match.
284        unsigned NumElts = FVTy->getNumElements();
285        assert(NumElts != 0 && "Constant vector with no elements?")((NumElts != 0 && "Constant vector with no elements?"
) ? static_cast<void> (0) : __assert_fail ("NumElts != 0 && \"Constant vector with no elements?\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/PatternMatch.h"
, 285, __PRETTY_FUNCTION__));
286        bool HasNonUndefElements = false;
287        for (unsigned i = 0; i != NumElts; ++i) {
288          Constant *Elt = C->getAggregateElement(i);
289          if (!Elt)
290            return false;
291          if (isa<UndefValue>(Elt))
292            continue;
293          auto *CV = dyn_cast<ConstantVal>(Elt);
294          if (!CV || !this->isValue(CV->getValue()))
295            return false;
296          HasNonUndefElements = true;
297        }
298        return HasNonUndefElements;
299      }
300    }
301    return false;
302  }
303};
304 
305/// specialization of cstval_pred_ty for ConstantInt
306template <typename Predicate>
307using cst_pred_ty = cstval_pred_ty<Predicate, ConstantInt>;
308 
309/// specialization of cstval_pred_ty for ConstantFP
310template <typename Predicate>
311using cstfp_pred_ty = cstval_pred_ty<Predicate, ConstantFP>;
312 
313/// This helper class is used to match scalar and vector constants that
314/// satisfy a specified predicate, and bind them to an APInt.
315template <typename Predicate> struct api_pred_ty : public Predicate {
316  const APInt *&Res;
317 
318  api_pred_ty(const APInt *&R) : Res(R) {}
319 
320  template <typename ITy> bool match(ITy *V) {
321    if (const auto *CI = dyn_cast<ConstantInt>(V))
322      if (this->isValue(CI->getValue())) {
323        Res = &CI->getValue();
324        return true;
325      }
326    if (V->getType()->isVectorTy())
327      if (const auto *C = dyn_cast<Constant>(V))
328        if (auto *CI = dyn_cast_or_null<ConstantInt>(C->getSplatValue()))
329          if (this->isValue(CI->getValue())) {
330            Res = &CI->getValue();
331            return true;
332          }
333 
334    return false;
335  }
336};
337 
338/// This helper class is used to match scalar and vector constants that
339/// satisfy a specified predicate, and bind them to an APFloat.
340/// Undefs are allowed in splat vector constants.
341template <typename Predicate> struct apf_pred_ty : public Predicate {
342  const APFloat *&Res;
343 
344  apf_pred_ty(const APFloat *&R) : Res(R) {}
345 
346  template <typename ITy> bool match(ITy *V) {
347    if (const auto *CI = dyn_cast<ConstantFP>(V))
348      if (this->isValue(CI->getValue())) {
349        Res = &CI->getValue();
350        return true;
351      }
352    if (V->getType()->isVectorTy())
353      if (const auto *C = dyn_cast<Constant>(V))
354        if (auto *CI = dyn_cast_or_null<ConstantFP>(
355                C->getSplatValue(/* AllowUndef */ true)))
356          if (this->isValue(CI->getValue())) {
357            Res = &CI->getValue();
358            return true;
359          }
360 
361    return false;
362  }
363};
364 
365///////////////////////////////////////////////////////////////////////////////
366//
367// Encapsulate constant value queries for use in templated predicate matchers.
368// This allows checking if constants match using compound predicates and works
369// with vector constants, possibly with relaxed constraints. For example, ignore
370// undef values.
371//
372///////////////////////////////////////////////////////////////////////////////
373 
374struct is_any_apint {
375  bool isValue(const APInt &C) { return true; }
376};
377/// Match an integer or vector with any integral constant.
378/// For vectors, this includes constants with undefined elements.
379inline cst_pred_ty<is_any_apint> m_AnyIntegralConstant() {
380  return cst_pred_ty<is_any_apint>();
381}
382 
383struct is_all_ones {
384  bool isValue(const APInt &C) { return C.isAllOnesValue(); }
385};
386/// Match an integer or vector with all bits set.
387/// For vectors, this includes constants with undefined elements.
388inline cst_pred_ty<is_all_ones> m_AllOnes() {
389  return cst_pred_ty<is_all_ones>();
390}
391 
392struct is_maxsignedvalue {
393  bool isValue(const APInt &C) { return C.isMaxSignedValue(); }
394};
395/// Match an integer or vector with values having all bits except for the high
396/// bit set (0x7f...).
397/// For vectors, this includes constants with undefined elements.
398inline cst_pred_ty<is_maxsignedvalue> m_MaxSignedValue() {
399  return cst_pred_ty<is_maxsignedvalue>();
400}
401inline api_pred_ty<is_maxsignedvalue> m_MaxSignedValue(const APInt *&V) {
402  return V;
403}
404 
405struct is_negative {
406  bool isValue(const APInt &C) { return C.isNegative(); }
407};
408/// Match an integer or vector of negative values.
409/// For vectors, this includes constants with undefined elements.
410inline cst_pred_ty<is_negative> m_Negative() {
411  return cst_pred_ty<is_negative>();
412}
413inline api_pred_ty<is_negative> m_Negative(const APInt *&V) {
414  return V;
415}
416 
417struct is_nonnegative {
418  bool isValue(const APInt &C) { return C.isNonNegative(); }
419};
420/// Match an integer or vector of non-negative values.
421/// For vectors, this includes constants with undefined elements.
422inline cst_pred_ty<is_nonnegative> m_NonNegative() {
423  return cst_pred_ty<is_nonnegative>();
424}
425inline api_pred_ty<is_nonnegative> m_NonNegative(const APInt *&V) {
426  return V;
427}
428 
429struct is_strictlypositive {
430  bool isValue(const APInt &C) { return C.isStrictlyPositive(); }
431};
432/// Match an integer or vector of strictly positive values.
433/// For vectors, this includes constants with undefined elements.
434inline cst_pred_ty<is_strictlypositive> m_StrictlyPositive() {
435  return cst_pred_ty<is_strictlypositive>();
436}
437inline api_pred_ty<is_strictlypositive> m_StrictlyPositive(const APInt *&V) {
438  return V;
439}
440 
441struct is_nonpositive {
442  bool isValue(const APInt &C) { return C.isNonPositive(); }
443};
444/// Match an integer or vector of non-positive values.
445/// For vectors, this includes constants with undefined elements.
446inline cst_pred_ty<is_nonpositive> m_NonPositive() {
447  return cst_pred_ty<is_nonpositive>();
448}
449inline api_pred_ty<is_nonpositive> m_NonPositive(const APInt *&V) { return V; }
450 
451struct is_one {
452  bool isValue(const APInt &C) { return C.isOneValue(); }
453};
454/// Match an integer 1 or a vector with all elements equal to 1.
455/// For vectors, this includes constants with undefined elements.
456inline cst_pred_ty<is_one> m_One() {
457  return cst_pred_ty<is_one>();
458}
459 
460struct is_zero_int {
461  bool isValue(const APInt &C) { return C.isNullValue(); }
462};
463/// Match an integer 0 or a vector with all elements equal to 0.
464/// For vectors, this includes constants with undefined elements.
465inline cst_pred_ty<is_zero_int> m_ZeroInt() {
466  return cst_pred_ty<is_zero_int>();
467}
468 
469struct is_zero {
470  template <typename ITy> bool match(ITy *V) {
471    auto *C = dyn_cast<Constant>(V);
472    // FIXME: this should be able to do something for scalable vectors
473    return C && (C->isNullValue() || cst_pred_ty<is_zero_int>().match(C));
474  }
475};
476/// Match any null constant or a vector with all elements equal to 0.
477/// For vectors, this includes constants with undefined elements.
478inline is_zero m_Zero() {
479  return is_zero();
480}
481 
482struct is_power2 {
483  bool isValue(const APInt &C) { return C.isPowerOf2(); }
484};
485/// Match an integer or vector power-of-2.
486/// For vectors, this includes constants with undefined elements.
487inline cst_pred_ty<is_power2> m_Power2() {
488  return cst_pred_ty<is_power2>();
489}
490inline api_pred_ty<is_power2> m_Power2(const APInt *&V) {
491  return V;
492}
493 
494struct is_negated_power2 {
495  bool isValue(const APInt &C) { return (-C).isPowerOf2(); }
496};
497/// Match a integer or vector negated power-of-2.
498/// For vectors, this includes constants with undefined elements.
499inline cst_pred_ty<is_negated_power2> m_NegatedPower2() {
500  return cst_pred_ty<is_negated_power2>();
501}
502inline api_pred_ty<is_negated_power2> m_NegatedPower2(const APInt *&V) {
503  return V;
504}
505 
506struct is_power2_or_zero {
507  bool isValue(const APInt &C) { return !C || C.isPowerOf2(); }
508};
509/// Match an integer or vector of 0 or power-of-2 values.
510/// For vectors, this includes constants with undefined elements.
511inline cst_pred_ty<is_power2_or_zero> m_Power2OrZero() {
512  return cst_pred_ty<is_power2_or_zero>();
513}
514inline api_pred_ty<is_power2_or_zero> m_Power2OrZero(const APInt *&V) {
515  return V;
516}
517 
518struct is_sign_mask {
519  bool isValue(const APInt &C) { return C.isSignMask(); }
520};
521/// Match an integer or vector with only the sign bit(s) set.
522/// For vectors, this includes constants with undefined elements.
523inline cst_pred_ty<is_sign_mask> m_SignMask() {
524  return cst_pred_ty<is_sign_mask>();
525}
526 
527struct is_lowbit_mask {
528  bool isValue(const APInt &C) { return C.isMask(); }
529};
530/// Match an integer or vector with only the low bit(s) set.
531/// For vectors, this includes constants with undefined elements.
532inline cst_pred_ty<is_lowbit_mask> m_LowBitMask() {
533  return cst_pred_ty<is_lowbit_mask>();
534}
535 
536struct icmp_pred_with_threshold {
537  ICmpInst::Predicate Pred;
538  const APInt *Thr;
539  bool isValue(const APInt &C) {
540    switch (Pred) {
541    case ICmpInst::Predicate::ICMP_EQ:
542      return C.eq(*Thr);
543    case ICmpInst::Predicate::ICMP_NE:
544      return C.ne(*Thr);
545    case ICmpInst::Predicate::ICMP_UGT:
546      return C.ugt(*Thr);
547    case ICmpInst::Predicate::ICMP_UGE:
548      return C.uge(*Thr);
549    case ICmpInst::Predicate::ICMP_ULT:
550      return C.ult(*Thr);
551    case ICmpInst::Predicate::ICMP_ULE:
552      return C.ule(*Thr);
553    case ICmpInst::Predicate::ICMP_SGT:
554      return C.sgt(*Thr);
555    case ICmpInst::Predicate::ICMP_SGE:
556      return C.sge(*Thr);
557    case ICmpInst::Predicate::ICMP_SLT:
558      return C.slt(*Thr);
559    case ICmpInst::Predicate::ICMP_SLE:
560      return C.sle(*Thr);
561    default:
562      llvm_unreachable("Unhandled ICmp predicate")::llvm::llvm_unreachable_internal("Unhandled ICmp predicate",
 "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/PatternMatch.h"
, 562);
563    }
564  }
565};
566/// Match an integer or vector with every element comparing 'pred' (eg/ne/...)
567/// to Threshold. For vectors, this includes constants with undefined elements.
568inline cst_pred_ty<icmp_pred_with_threshold>
569m_SpecificInt_ICMP(ICmpInst::Predicate Predicate, const APInt &Threshold) {
570  cst_pred_ty<icmp_pred_with_threshold> P;
571  P.Pred = Predicate;
572  P.Thr = &Threshold;
573  return P;
574}
575 
576struct is_nan {
577  bool isValue(const APFloat &C) { return C.isNaN(); }
578};
579/// Match an arbitrary NaN constant. This includes quiet and signalling nans.
580/// For vectors, this includes constants with undefined elements.
581inline cstfp_pred_ty<is_nan> m_NaN() {
582  return cstfp_pred_ty<is_nan>();
583}
584 
585struct is_nonnan {
586  bool isValue(const APFloat &C) { return !C.isNaN(); }
587};
588/// Match a non-NaN FP constant.
589/// For vectors, this includes constants with undefined elements.
590inline cstfp_pred_ty<is_nonnan> m_NonNaN() {
591  return cstfp_pred_ty<is_nonnan>();
592}
593 
594struct is_inf {
595  bool isValue(const APFloat &C) { return C.isInfinity(); }
596};
597/// Match a positive or negative infinity FP constant.
598/// For vectors, this includes constants with undefined elements.
599inline cstfp_pred_ty<is_inf> m_Inf() {
600  return cstfp_pred_ty<is_inf>();
601}
602 
603struct is_noninf {
604  bool isValue(const APFloat &C) { return !C.isInfinity(); }
605};
606/// Match a non-infinity FP constant, i.e. finite or NaN.
607/// For vectors, this includes constants with undefined elements.
608inline cstfp_pred_ty<is_noninf> m_NonInf() {
609  return cstfp_pred_ty<is_noninf>();
610}
611 
612struct is_finite {
613  bool isValue(const APFloat &C) { return C.isFinite(); }
614};
615/// Match a finite FP constant, i.e. not infinity or NaN.
616/// For vectors, this includes constants with undefined elements.
617inline cstfp_pred_ty<is_finite> m_Finite() {
618  return cstfp_pred_ty<is_finite>();
619}
620inline apf_pred_ty<is_finite> m_Finite(const APFloat *&V) { return V; }
621 
622struct is_finitenonzero {
623  bool isValue(const APFloat &C) { return C.isFiniteNonZero(); }
624};
625/// Match a finite non-zero FP constant.
626/// For vectors, this includes constants with undefined elements.
627inline cstfp_pred_ty<is_finitenonzero> m_FiniteNonZero() {
628  return cstfp_pred_ty<is_finitenonzero>();
629}
630inline apf_pred_ty<is_finitenonzero> m_FiniteNonZero(const APFloat *&V) {
631  return V;
632}
633 
634struct is_any_zero_fp {
635  bool isValue(const APFloat &C) { return C.isZero(); }
636};
637/// Match a floating-point negative zero or positive zero.
638/// For vectors, this includes constants with undefined elements.
639inline cstfp_pred_ty<is_any_zero_fp> m_AnyZeroFP() {
640  return cstfp_pred_ty<is_any_zero_fp>();
641}
642 
643struct is_pos_zero_fp {
644  bool isValue(const APFloat &C) { return C.isPosZero(); }
645};
646/// Match a floating-point positive zero.
647/// For vectors, this includes constants with undefined elements.
648inline cstfp_pred_ty<is_pos_zero_fp> m_PosZeroFP() {
649  return cstfp_pred_ty<is_pos_zero_fp>();
650}
651 
652struct is_neg_zero_fp {
653  bool isValue(const APFloat &C) { return C.isNegZero(); }
654};
655/// Match a floating-point negative zero.
656/// For vectors, this includes constants with undefined elements.
657inline cstfp_pred_ty<is_neg_zero_fp> m_NegZeroFP() {
658  return cstfp_pred_ty<is_neg_zero_fp>();
659}
660 
661struct is_non_zero_fp {
662  bool isValue(const APFloat &C) { return C.isNonZero(); }
663};
664/// Match a floating-point non-zero.
665/// For vectors, this includes constants with undefined elements.
666inline cstfp_pred_ty<is_non_zero_fp> m_NonZeroFP() {
667  return cstfp_pred_ty<is_non_zero_fp>();
668}
669 
670///////////////////////////////////////////////////////////////////////////////
671 
672template <typename Class> struct bind_ty {
673  Class *&VR;
674 
675  bind_ty(Class *&V) : VR(V) {}
676 
677  template <typename ITy> bool match(ITy *V) {
678    if (auto *CV = dyn_cast<Class>(V)) {
679      VR = CV;
680      return true;
681    }
682    return false;
683  }
684};
685 
686/// Match a value, capturing it if we match.
687inline bind_ty<Value> m_Value(Value *&V) { return V; }
688inline bind_ty<const Value> m_Value(const Value *&V) { return V; }
689 
690/// Match an instruction, capturing it if we match.
691inline bind_ty<Instruction> m_Instruction(Instruction *&I) { return I; }
692/// Match a unary operator, capturing it if we match.
693inline bind_ty<UnaryOperator> m_UnOp(UnaryOperator *&I) { return I; }
694/// Match a binary operator, capturing it if we match.
695inline bind_ty<BinaryOperator> m_BinOp(BinaryOperator *&I) { return I; }
696/// Match a with overflow intrinsic, capturing it if we match.
697inline bind_ty<WithOverflowInst> m_WithOverflowInst(WithOverflowInst *&I) { return I; }
698 
699/// Match a ConstantInt, capturing the value if we match.
700inline bind_ty<ConstantInt> m_ConstantInt(ConstantInt *&CI) { return CI; }
701 
702/// Match a Constant, capturing the value if we match.
703inline bind_ty<Constant> m_Constant(Constant *&C) { return C; }
704 
705/// Match a ConstantFP, capturing the value if we match.
706inline bind_ty<ConstantFP> m_ConstantFP(ConstantFP *&C) { return C; }
707 
708/// Match a basic block value, capturing it if we match.
709inline bind_ty<BasicBlock> m_BasicBlock(BasicBlock *&V) { return V; }
710inline bind_ty<const BasicBlock> m_BasicBlock(const BasicBlock *&V) {
711  return V;
712}
713 
714/// Match a specified Value*.
715struct specificval_ty {
716  const Value *Val;
717 
718  specificval_ty(const Value *V) : Val(V) {}
719 
720  template <typename ITy> bool match(ITy *V) { return V == Val; }
721};
722 
723/// Match if we have a specific specified value.
724inline specificval_ty m_Specific(const Value *V) { return V; }
725 
726/// Stores a reference to the Value *, not the Value * itself,
727/// thus can be used in commutative matchers.
728template <typename Class> struct deferredval_ty {
729  Class *const &Val;
730 
731  deferredval_ty(Class *const &V) : Val(V) {}
732 
733  template <typename ITy> bool match(ITy *const V) { return V == Val; }
734};
735 
736/// A commutative-friendly version of m_Specific().
737inline deferredval_ty<Value> m_Deferred(Value *const &V) { return V; }
738inline deferredval_ty<const Value> m_Deferred(const Value *const &V) {
739  return V;
740}
741 
742/// Match a specified floating point value or vector of all elements of
743/// that value.
744struct specific_fpval {
745  double Val;
746 
747  specific_fpval(double V) : Val(V) {}
748 
749  template <typename ITy> bool match(ITy *V) {
750    if (const auto *CFP = dyn_cast<ConstantFP>(V))
751      return CFP->isExactlyValue(Val);
752    if (V->getType()->isVectorTy())
753      if (const auto *C = dyn_cast<Constant>(V))
754        if (auto *CFP = dyn_cast_or_null<ConstantFP>(C->getSplatValue()))
755          return CFP->isExactlyValue(Val);
756    return false;
757  }
758};
759 
760/// Match a specific floating point value or vector with all elements
761/// equal to the value.
762inline specific_fpval m_SpecificFP(double V) { return specific_fpval(V); }
763 
764/// Match a float 1.0 or vector with all elements equal to 1.0.
765inline specific_fpval m_FPOne() { return m_SpecificFP(1.0); }
766 
767struct bind_const_intval_ty {
768  uint64_t &VR;
769 
770  bind_const_intval_ty(uint64_t &V) : VR(V) {}
771 
772  template <typename ITy> bool match(ITy *V) {
773    if (const auto *CV = dyn_cast<ConstantInt>(V))
774      if (CV->getValue().ule(UINT64_MAX(18446744073709551615UL))) {
775        VR = CV->getZExtValue();
776        return true;
777      }
778    return false;
779  }
780};
781 
782/// Match a specified integer value or vector of all elements of that
783/// value.
784template <bool AllowUndefs>
785struct specific_intval {
786  APInt Val;
787 
788  specific_intval(APInt V) : Val(std::move(V)) {}
789 
790  template <typename ITy> bool match(ITy *V) {
791    const auto *CI = dyn_cast<ConstantInt>(V);
792    if (!CI && V->getType()->isVectorTy())
793      if (const auto *C = dyn_cast<Constant>(V))
794        CI = dyn_cast_or_null<ConstantInt>(C->getSplatValue(AllowUndefs));
795 
796    return CI && APInt::isSameValue(CI->getValue(), Val);
797  }
798};
799 
800/// Match a specific integer value or vector with all elements equal to
801/// the value.
802inline specific_intval<false> m_SpecificInt(APInt V) {
803  return specific_intval<false>(std::move(V));
804}
805 
806inline specific_intval<false> m_SpecificInt(uint64_t V) {
807  return m_SpecificInt(APInt(64, V));
808}
809 
810inline specific_intval<true> m_SpecificIntAllowUndef(APInt V) {
811  return specific_intval<true>(std::move(V));
812}
813 
814inline specific_intval<true> m_SpecificIntAllowUndef(uint64_t V) {
815  return m_SpecificIntAllowUndef(APInt(64, V));
816}
817 
818/// Match a ConstantInt and bind to its value.  This does not match
819/// ConstantInts wider than 64-bits.
820inline bind_const_intval_ty m_ConstantInt(uint64_t &V) { return V; }
821 
822/// Match a specified basic block value.
823struct specific_bbval {
824  BasicBlock *Val;
825 
826  specific_bbval(BasicBlock *Val) : Val(Val) {}
827 
828  template <typename ITy> bool match(ITy *V) {
829    const auto *BB = dyn_cast<BasicBlock>(V);
830    return BB && BB == Val;
831  }
832};
833 
834/// Match a specific basic block value.
835inline specific_bbval m_SpecificBB(BasicBlock *BB) {
836  return specific_bbval(BB);
837}
838 
839/// A commutative-friendly version of m_Specific().
840inline deferredval_ty<BasicBlock> m_Deferred(BasicBlock *const &BB) {
841  return BB;
842}
843inline deferredval_ty<const BasicBlock>
844m_Deferred(const BasicBlock *const &BB) {
845  return BB;
846}
847 
848//===----------------------------------------------------------------------===//
849// Matcher for any binary operator.
850//
851template <typename LHS_t, typename RHS_t, bool Commutable = false>
852struct AnyBinaryOp_match {
853  LHS_t L;
854  RHS_t R;
855 
856  // The evaluation order is always stable, regardless of Commutability.
857  // The LHS is always matched first.
858  AnyBinaryOp_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {}
859 
860  template <typename OpTy> bool match(OpTy *V) {
861    if (auto *I = dyn_cast<BinaryOperator>(V))
862      return (L.match(I->getOperand(0)) && R.match(I->getOperand(1))) ||
863             (Commutable && L.match(I->getOperand(1)) &&
864              R.match(I->getOperand(0)));
865    return false;
866  }
867};
868 
869template <typename LHS, typename RHS>
870inline AnyBinaryOp_match<LHS, RHS> m_BinOp(const LHS &L, const RHS &R) {
871  return AnyBinaryOp_match<LHS, RHS>(L, R);
872}
873 
874//===----------------------------------------------------------------------===//
875// Matcher for any unary operator.
876// TODO fuse unary, binary matcher into n-ary matcher
877//
878template <typename OP_t> struct AnyUnaryOp_match {
879  OP_t X;
880 
881  AnyUnaryOp_match(const OP_t &X) : X(X) {}
882 
883  template <typename OpTy> bool match(OpTy *V) {
884    if (auto *I = dyn_cast<UnaryOperator>(V))
885      return X.match(I->getOperand(0));
886    return false;
887  }
888};
889 
890template <typename OP_t> inline AnyUnaryOp_match<OP_t> m_UnOp(const OP_t &X) {
891  return AnyUnaryOp_match<OP_t>(X);
892}
893 
894//===----------------------------------------------------------------------===//
895// Matchers for specific binary operators.
896//
897 
898template <typename LHS_t, typename RHS_t, unsigned Opcode,
899          bool Commutable = false>
900struct BinaryOp_match {
901  LHS_t L;
902  RHS_t R;
903 
904  // The evaluation order is always stable, regardless of Commutability.
905  // The LHS is always matched first.
906  BinaryOp_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {}
907 
908  template <typename OpTy> bool match(OpTy *V) {
909    if (V->getValueID() == Value::InstructionVal + Opcode) {
910      auto *I = cast<BinaryOperator>(V);
911      return (L.match(I->getOperand(0)) && R.match(I->getOperand(1))) ||
912             (Commutable && L.match(I->getOperand(1)) &&
913              R.match(I->getOperand(0)));
914    }
915    if (auto *CE = dyn_cast<ConstantExpr>(V))
916      return CE->getOpcode() == Opcode &&
917             ((L.match(CE->getOperand(0)) && R.match(CE->getOperand(1))) ||
918              (Commutable && L.match(CE->getOperand(1)) &&
919               R.match(CE->getOperand(0))));
920    return false;
921  }
922};
923 
924template <typename LHS, typename RHS>
925inline BinaryOp_match<LHS, RHS, Instruction::Add> m_Add(const LHS &L,
926                                                        const RHS &R) {
927  return BinaryOp_match<LHS, RHS, Instruction::Add>(L, R);
928}
929 
930template <typename LHS, typename RHS>
931inline BinaryOp_match<LHS, RHS, Instruction::FAdd> m_FAdd(const LHS &L,
932                                                          const RHS &R) {
933  return BinaryOp_match<LHS, RHS, Instruction::FAdd>(L, R);
934}
935 
936template <typename LHS, typename RHS>
937inline BinaryOp_match<LHS, RHS, Instruction::Sub> m_Sub(const LHS &L,
938                                                        const RHS &R) {
939  return BinaryOp_match<LHS, RHS, Instruction::Sub>(L, R);
940}
941 
942template <typename LHS, typename RHS>
943inline BinaryOp_match<LHS, RHS, Instruction::FSub> m_FSub(const LHS &L,
944                                                          const RHS &R) {
945  return BinaryOp_match<LHS, RHS, Instruction::FSub>(L, R);
946}
947 
948template <typename Op_t> struct FNeg_match {
949  Op_t X;
950 
951  FNeg_match(const Op_t &Op) : X(Op) {}
952  template <typename OpTy> bool match(OpTy *V) {
953    auto *FPMO = dyn_cast<FPMathOperator>(V);
954    if (!FPMO) return false;
955 
956    if (FPMO->getOpcode() == Instruction::FNeg)
957      return X.match(FPMO->getOperand(0));
958 
959    if (FPMO->getOpcode() == Instruction::FSub) {
960      if (FPMO->hasNoSignedZeros()) {
961        // With 'nsz', any zero goes.
962        if (!cstfp_pred_ty<is_any_zero_fp>().match(FPMO->getOperand(0)))
963          return false;
964      } else {
965        // Without 'nsz', we need fsub -0.0, X exactly.
966        if (!cstfp_pred_ty<is_neg_zero_fp>().match(FPMO->getOperand(0)))
967          return false;
968      }
969 
970      return X.match(FPMO->getOperand(1));
971    }
972 
973    return false;
974  }
975};
976 
977/// Match 'fneg X' as 'fsub -0.0, X'.
978template <typename OpTy>
979inline FNeg_match<OpTy>
980m_FNeg(const OpTy &X) {
981  return FNeg_match<OpTy>(X);
982}
983 
984/// Match 'fneg X' as 'fsub +-0.0, X'.
985template <typename RHS>
986inline BinaryOp_match<cstfp_pred_ty<is_any_zero_fp>, RHS, Instruction::FSub>
987m_FNegNSZ(const RHS &X) {
988  return m_FSub(m_AnyZeroFP(), X);
989}
990 
991template <typename LHS, typename RHS>
992inline BinaryOp_match<LHS, RHS, Instruction::Mul> m_Mul(const LHS &L,
993                                                        const RHS &R) {
994  return BinaryOp_match<LHS, RHS, Instruction::Mul>(L, R);
995}
996 
997template <typename LHS, typename RHS>
998inline BinaryOp_match<LHS, RHS, Instruction::FMul> m_FMul(const LHS &L,
999                                                          const RHS &R) {
1000  return BinaryOp_match<LHS, RHS, Instruction::FMul>(L, R);
1001}
1002 
1003template <typename LHS, typename RHS>
1004inline BinaryOp_match<LHS, RHS, Instruction::UDiv> m_UDiv(const LHS &L,
1005                                                          const RHS &R) {
1006  return BinaryOp_match<LHS, RHS, Instruction::UDiv>(L, R);
1007}
1008 
1009template <typename LHS, typename RHS>
1010inline BinaryOp_match<LHS, RHS, Instruction::SDiv> m_SDiv(const LHS &L,
1011                                                          const RHS &R) {
1012  return BinaryOp_match<LHS, RHS, Instruction::SDiv>(L, R);
1013}
1014 
1015template <typename LHS, typename RHS>
1016inline BinaryOp_match<LHS, RHS, Instruction::FDiv> m_FDiv(const LHS &L,
1017                                                          const RHS &R) {
1018  return BinaryOp_match<LHS, RHS, Instruction::FDiv>(L, R);
1019}
1020 
1021template <typename LHS, typename RHS>
1022inline BinaryOp_match<LHS, RHS, Instruction::URem> m_URem(const LHS &L,
1023                                                          const RHS &R) {
1024  return BinaryOp_match<LHS, RHS, Instruction::URem>(L, R);
1025}
1026 
1027template <typename LHS, typename RHS>
1028inline BinaryOp_match<LHS, RHS, Instruction::SRem> m_SRem(const LHS &L,
1029                                                          const RHS &R) {
1030  return BinaryOp_match<LHS, RHS, Instruction::SRem>(L, R);
1031}
1032 
1033template <typename LHS, typename RHS>
1034inline BinaryOp_match<LHS, RHS, Instruction::FRem> m_FRem(const LHS &L,
1035                                                          const RHS &R) {
1036  return BinaryOp_match<LHS, RHS, Instruction::FRem>(L, R);
1037}
1038 
1039template <typename LHS, typename RHS>
1040inline BinaryOp_match<LHS, RHS, Instruction::And> m_And(const LHS &L,
1041                                                        const RHS &R) {
1042  return BinaryOp_match<LHS, RHS, Instruction::And>(L, R);
1043}
1044 
1045template <typename LHS, typename RHS>
1046inline BinaryOp_match<LHS, RHS, Instruction::Or> m_Or(const LHS &L,
1047                                                      const RHS &R) {
1048  return BinaryOp_match<LHS, RHS, Instruction::Or>(L, R);
1049}
1050 
1051template <typename LHS, typename RHS>
1052inline BinaryOp_match<LHS, RHS, Instruction::Xor> m_Xor(const LHS &L,
1053                                                        const RHS &R) {
1054  return BinaryOp_match<LHS, RHS, Instruction::Xor>(L, R);
1055}
1056 
1057template <typename LHS, typename RHS>
1058inline BinaryOp_match<LHS, RHS, Instruction::Shl> m_Shl(const LHS &L,
1059                                                        const RHS &R) {
1060  return BinaryOp_match<LHS, RHS, Instruction::Shl>(L, R);
1061}
1062 
1063template <typename LHS, typename RHS>
1064inline BinaryOp_match<LHS, RHS, Instruction::LShr> m_LShr(const LHS &L,
1065                                                          const RHS &R) {
1066  return BinaryOp_match<LHS, RHS, Instruction::LShr>(L, R);
1067}
1068 
1069template <typename LHS, typename RHS>
1070inline BinaryOp_match<LHS, RHS, Instruction::AShr> m_AShr(const LHS &L,
1071                                                          const RHS &R) {
1072  return BinaryOp_match<LHS, RHS, Instruction::AShr>(L, R);
1073}
1074 
1075template <typename LHS_t, typename RHS_t, unsigned Opcode,
1076          unsigned WrapFlags = 0>
1077struct OverflowingBinaryOp_match {
1078  LHS_t L;
1079  RHS_t R;
1080 
1081  OverflowingBinaryOp_match(const LHS_t &LHS, const RHS_t &RHS)
1082      : L(LHS), R(RHS) {}
1083 
1084  template <typename OpTy> bool match(OpTy *V) {
1085    if (auto *Op = dyn_cast<OverflowingBinaryOperator>(V)) {
1086      if (Op->getOpcode() != Opcode)
1087        return false;
1088      if (WrapFlags & OverflowingBinaryOperator::NoUnsignedWrap &&
1089          !Op->hasNoUnsignedWrap())
1090        return false;
1091      if (WrapFlags & OverflowingBinaryOperator::NoSignedWrap &&
1092          !Op->hasNoSignedWrap())
1093        return false;
1094      return L.match(Op->getOperand(0)) && R.match(Op->getOperand(1));
1095    }
1096    return false;
1097  }
1098};
1099 
1100template <typename LHS, typename RHS>
1101inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Add,
1102                                 OverflowingBinaryOperator::NoSignedWrap>
1103m_NSWAdd(const LHS &L, const RHS &R) {
1104  return OverflowingBinaryOp_match<LHS, RHS, Instruction::Add,
1105                                   OverflowingBinaryOperator::NoSignedWrap>(
1106      L, R);
1107}
1108template <typename LHS, typename RHS>
1109inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub,
1110                                 OverflowingBinaryOperator::NoSignedWrap>
1111m_NSWSub(const LHS &L, const RHS &R) {
1112  return OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub,
1113                                   OverflowingBinaryOperator::NoSignedWrap>(
1114      L, R);
1115}
1116template <typename LHS, typename RHS>
1117inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul,
1118                                 OverflowingBinaryOperator::NoSignedWrap>
1119m_NSWMul(const LHS &L, const RHS &R) {
1120  return OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul,
1121                                   OverflowingBinaryOperator::NoSignedWrap>(
1122      L, R);
1123}
1124template <typename LHS, typename RHS>
1125inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl,
1126                                 OverflowingBinaryOperator::NoSignedWrap>
1127m_NSWShl(const LHS &L, const RHS &R) {
1128  return OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl,
1129                                   OverflowingBinaryOperator::NoSignedWrap>(
1130      L, R);
1131}
1132 
1133template <typename LHS, typename RHS>
1134inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Add,
1135                                 OverflowingBinaryOperator::NoUnsignedWrap>
1136m_NUWAdd(const LHS &L, const RHS &R) {
1137  return OverflowingBinaryOp_match<LHS, RHS, Instruction::Add,
1138                                   OverflowingBinaryOperator::NoUnsignedWrap>(
1139      L, R);
1140}
1141template <typename LHS, typename RHS>
1142inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub,
1143                                 OverflowingBinaryOperator::NoUnsignedWrap>
1144m_NUWSub(const LHS &L, const RHS &R) {
1145  return OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub,
1146                                   OverflowingBinaryOperator::NoUnsignedWrap>(
1147      L, R);
1148}
1149template <typename LHS, typename RHS>
1150inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul,
1151                                 OverflowingBinaryOperator::NoUnsignedWrap>
1152m_NUWMul(const LHS &L, const RHS &R) {
1153  return OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul,
1154                                   OverflowingBinaryOperator::NoUnsignedWrap>(
1155      L, R);
1156}
1157template <typename LHS, typename RHS>
1158inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl,
1159                                 OverflowingBinaryOperator::NoUnsignedWrap>
1160m_NUWShl(const LHS &L, const RHS &R) {
1161  return OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl,
1162                                   OverflowingBinaryOperator::NoUnsignedWrap>(
1163      L, R);
1164}
1165 
1166//===----------------------------------------------------------------------===//
1167// Class that matches a group of binary opcodes.
1168//
1169template <typename LHS_t, typename RHS_t, typename Predicate>
1170struct BinOpPred_match : Predicate {
1171  LHS_t L;
1172  RHS_t R;
1173 
1174  BinOpPred_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {}
1175 
1176  template <typename OpTy> bool match(OpTy *V) {
1177    if (auto *I = dyn_cast<Instruction>(V))
1178      return this->isOpType(I->getOpcode()) && L.match(I->getOperand(0)) &&
1179             R.match(I->getOperand(1));
1180    if (auto *CE = dyn_cast<ConstantExpr>(V))
1181      return this->isOpType(CE->getOpcode()) && L.match(CE->getOperand(0)) &&
1182             R.match(CE->getOperand(1));
1183    return false;
1184  }
1185};
1186 
1187struct is_shift_op {
1188  bool isOpType(unsigned Opcode) { return Instruction::isShift(Opcode); }
1189};
1190 
1191struct is_right_shift_op {
1192  bool isOpType(unsigned Opcode) {
1193    return Opcode == Instruction::LShr || Opcode == Instruction::AShr;
1194  }
1195};
1196 
1197struct is_logical_shift_op {
1198  bool isOpType(unsigned Opcode) {
1199    return Opcode == Instruction::LShr || Opcode == Instruction::Shl;
1200  }
1201};
1202 
1203struct is_bitwiselogic_op {
1204  bool isOpType(unsigned Opcode) {
1205    return Instruction::isBitwiseLogicOp(Opcode);
1206  }
1207};
1208 
1209struct is_idiv_op {
1210  bool isOpType(unsigned Opcode) {
1211    return Opcode == Instruction::SDiv || Opcode == Instruction::UDiv;
1212  }
1213};
1214 
1215struct is_irem_op {
1216  bool isOpType(unsigned Opcode) {
1217    return Opcode == Instruction::SRem || Opcode == Instruction::URem;
1218  }
1219};
1220 
1221/// Matches shift operations.
1222template <typename LHS, typename RHS>
1223inline BinOpPred_match<LHS, RHS, is_shift_op> m_Shift(const LHS &L,
1224                                                      const RHS &R) {
1225  return BinOpPred_match<LHS, RHS, is_shift_op>(L, R);
1226}
1227 
1228/// Matches logical shift operations.
1229template <typename LHS, typename RHS>
1230inline BinOpPred_match<LHS, RHS, is_right_shift_op> m_Shr(const LHS &L,
1231                                                          const RHS &R) {
1232  return BinOpPred_match<LHS, RHS, is_right_shift_op>(L, R);
1233}
1234 
1235/// Matches logical shift operations.
1236template <typename LHS, typename RHS>
1237inline BinOpPred_match<LHS, RHS, is_logical_shift_op>
1238m_LogicalShift(const LHS &L, const RHS &R) {
1239  return BinOpPred_match<LHS, RHS, is_logical_shift_op>(L, R);
1240}
1241 
1242/// Matches bitwise logic operations.
1243template <typename LHS, typename RHS>
1244inline BinOpPred_match<LHS, RHS, is_bitwiselogic_op>
1245m_BitwiseLogic(const LHS &L, const RHS &R) {
1246  return BinOpPred_match<LHS, RHS, is_bitwiselogic_op>(L, R);
1247}
1248 
1249/// Matches integer division operations.
1250template <typename LHS, typename RHS>
1251inline BinOpPred_match<LHS, RHS, is_idiv_op> m_IDiv(const LHS &L,
1252                                                    const RHS &R) {
1253  return BinOpPred_match<LHS, RHS, is_idiv_op>(L, R);
1254}
1255 
1256/// Matches integer remainder operations.
1257template <typename LHS, typename RHS>
1258inline BinOpPred_match<LHS, RHS, is_irem_op> m_IRem(const LHS &L,
1259                                                    const RHS &R) {
1260  return BinOpPred_match<LHS, RHS, is_irem_op>(L, R);
1261}
1262 
1263//===----------------------------------------------------------------------===//
1264// Class that matches exact binary ops.
1265//
1266template <typename SubPattern_t> struct Exact_match {
1267  SubPattern_t SubPattern;
1268 
1269  Exact_match(const SubPattern_t &SP) : SubPattern(SP) {}
1270 
1271  template <typename OpTy> bool match(OpTy *V) {
1272    if (auto *PEO = dyn_cast<PossiblyExactOperator>(V))
1273      return PEO->isExact() && SubPattern.match(V);
1274    return false;
1275  }
1276};
1277 
1278template <typename T> inline Exact_match<T> m_Exact(const T &SubPattern) {
1279  return SubPattern;
1280}
1281 
1282//===----------------------------------------------------------------------===//
1283// Matchers for CmpInst classes
1284//
1285 
1286template <typename LHS_t, typename RHS_t, typename Class, typename PredicateTy,
1287          bool Commutable = false>
1288struct CmpClass_match {
1289  PredicateTy &Predicate;
1290  LHS_t L;
1291  RHS_t R;
1292 
1293  // The evaluation order is always stable, regardless of Commutability.
1294  // The LHS is always matched first.
1295  CmpClass_match(PredicateTy &Pred, const LHS_t &LHS, const RHS_t &RHS)
1296      : Predicate(Pred), L(LHS), R(RHS) {}
1297 
1298  template <typename OpTy> bool match(OpTy *V) {
1299    if (auto *I = dyn_cast<Class>(V)) {
1300      if (L.match(I->getOperand(0)) && R.match(I->getOperand(1))) {
1301        Predicate = I->getPredicate();
1302        return true;
1303      } else if (Commutable && L.match(I->getOperand(1)) &&
1304           R.match(I->getOperand(0))) {
1305        Predicate = I->getSwappedPredicate();
1306        return true;
1307      }
1308    }
1309    return false;
1310  }
1311};
1312 
1313template <typename LHS, typename RHS>
1314inline CmpClass_match<LHS, RHS, CmpInst, CmpInst::Predicate>
1315m_Cmp(CmpInst::Predicate &Pred, const LHS &L, const RHS &R) {
1316  return CmpClass_match<LHS, RHS, CmpInst, CmpInst::Predicate>(Pred, L, R);
1317}
1318 
1319template <typename LHS, typename RHS>
1320inline CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate>
1321m_ICmp(ICmpInst::Predicate &Pred, const LHS &L, const RHS &R) {
1322  return CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate>(Pred, L, R);
1323}
1324 
1325template <typename LHS, typename RHS>
1326inline CmpClass_match<LHS, RHS, FCmpInst, FCmpInst::Predicate>
1327m_FCmp(FCmpInst::Predicate &Pred, const LHS &L, const RHS &R) {
1328  return CmpClass_match<LHS, RHS, FCmpInst, FCmpInst::Predicate>(Pred, L, R);
1329}
1330 
1331//===----------------------------------------------------------------------===//
1332// Matchers for instructions with a given opcode and number of operands.
1333//
1334 
1335/// Matches instructions with Opcode and three operands.
1336template <typename T0, unsigned Opcode> struct OneOps_match {
1337  T0 Op1;
1338 
1339  OneOps_match(const T0 &Op1) : Op1(Op1) {}
1340 
1341  template <typename OpTy> bool match(OpTy *V) {
1342    if (V->getValueID() == Value::InstructionVal + Opcode) {
1343      auto *I = cast<Instruction>(V);
1344      return Op1.match(I->getOperand(0));
1345    }
1346    return false;
1347  }
1348};
1349 
1350/// Matches instructions with Opcode and three operands.
1351template <typename T0, typename T1, unsigned Opcode> struct TwoOps_match {
1352  T0 Op1;
1353  T1 Op2;
1354 
1355  TwoOps_match(const T0 &Op1, const T1 &Op2) : Op1(Op1), Op2(Op2) {}
1356 
1357  template <typename OpTy> bool match(OpTy *V) {
1358    if (V->getValueID() == Value::InstructionVal + Opcode) {
1359      auto *I = cast<Instruction>(V);
1360      return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1));
1361    }
1362    return false;
1363  }
1364};
1365 
1366/// Matches instructions with Opcode and three operands.
1367template <typename T0, typename T1, typename T2, unsigned Opcode>
1368struct ThreeOps_match {
1369  T0 Op1;
1370  T1 Op2;
1371  T2 Op3;
1372 
1373  ThreeOps_match(const T0 &Op1, const T1 &Op2, const T2 &Op3)
1374      : Op1(Op1), Op2(Op2), Op3(Op3) {}
1375 
1376  template <typename OpTy> bool match(OpTy *V) {
1377    if (V->getValueID() == Value::InstructionVal + Opcode) {
1378      auto *I = cast<Instruction>(V);
1379      return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1)) &&
1380             Op3.match(I->getOperand(2));
1381    }
1382    return false;
1383  }
1384};
1385 
1386/// Matches SelectInst.
1387template <typename Cond, typename LHS, typename RHS>
1388inline ThreeOps_match<Cond, LHS, RHS, Instruction::Select>
1389m_Select(const Cond &C, const LHS &L, const RHS &R) {
1390  return ThreeOps_match<Cond, LHS, RHS, Instruction::Select>(C, L, R);
1391}
1392 
1393/// This matches a select of two constants, e.g.:
1394/// m_SelectCst<-1, 0>(m_Value(V))
1395template <int64_t L, int64_t R, typename Cond>
1396inline ThreeOps_match<Cond, constantint_match<L>, constantint_match<R>,
1397                      Instruction::Select>
1398m_SelectCst(const Cond &C) {
1399  return m_Select(C, m_ConstantInt<L>(), m_ConstantInt<R>());
1400}
1401 
1402/// Matches FreezeInst.
1403template <typename OpTy>
1404inline OneOps_match<OpTy, Instruction::Freeze> m_Freeze(const OpTy &Op) {
1405  return OneOps_match<OpTy, Instruction::Freeze>(Op);
1406}
1407 
1408/// Matches InsertElementInst.
1409template <typename Val_t, typename Elt_t, typename Idx_t>
1410inline ThreeOps_match<Val_t, Elt_t, Idx_t, Instruction::InsertElement>
1411m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx) {
1412  return ThreeOps_match<Val_t, Elt_t, Idx_t, Instruction::InsertElement>(
1413      Val, Elt, Idx);
1414}
1415 
1416/// Matches ExtractElementInst.
1417template <typename Val_t, typename Idx_t>
1418inline TwoOps_match<Val_t, Idx_t, Instruction::ExtractElement>
1419m_ExtractElt(const Val_t &Val, const Idx_t &Idx) {
1420  return TwoOps_match<Val_t, Idx_t, Instruction::ExtractElement>(Val, Idx);
1421}
1422 
1423/// Matches shuffle.
1424template <typename T0, typename T1, typename T2> struct Shuffle_match {
1425  T0 Op1;
1426  T1 Op2;
1427  T2 Mask;
1428 
1429  Shuffle_match(const T0 &Op1, const T1 &Op2, const T2 &Mask)
1430      : Op1(Op1), Op2(Op2), Mask(Mask) {}
1431 
1432  template <typename OpTy> bool match(OpTy *V) {
1433    if (auto *I = dyn_cast<ShuffleVectorInst>(V)) {
1434      return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1)) &&
1435             Mask.match(I->getShuffleMask());
1436    }
1437    return false;
1438  }
1439};
1440 
1441struct m_Mask {
1442  ArrayRef<int> &MaskRef;
1443  m_Mask(ArrayRef<int> &MaskRef) : MaskRef(MaskRef) {}
1444  bool match(ArrayRef<int> Mask) {
1445    MaskRef = Mask;
1446    return true;
1447  }
1448};
1449 
1450struct m_ZeroMask {
1451  bool match(ArrayRef<int> Mask) {
1452    return all_of(Mask, [](int Elem) { return Elem == 0 || Elem == -1; });
1453  }
1454};
1455 
1456struct m_SpecificMask {
1457  ArrayRef<int> &MaskRef;
1458  m_SpecificMask(ArrayRef<int> &MaskRef) : MaskRef(MaskRef) {}
1459  bool match(ArrayRef<int> Mask) { return MaskRef == Mask; }
1460};
1461 
1462struct m_SplatOrUndefMask {
1463  int &SplatIndex;
1464  m_SplatOrUndefMask(int &SplatIndex) : SplatIndex(SplatIndex) {}
1465  bool match(ArrayRef<int> Mask) {
1466    auto First = find_if(Mask, [](int Elem) { return Elem != -1; });
1467    if (First == Mask.end())
1468      return false;
1469    SplatIndex = *First;
1470    return all_of(Mask,
1471                  [First](int Elem) { return Elem == *First || Elem == -1; });
1472  }
1473};
1474 
1475/// Matches ShuffleVectorInst independently of mask value.
1476template <typename V1_t, typename V2_t>
1477inline TwoOps_match<V1_t, V2_t, Instruction::ShuffleVector>
1478m_Shuffle(const V1_t &v1, const V2_t &v2) {
1479  return TwoOps_match<V1_t, V2_t, Instruction::ShuffleVector>(v1, v2);
1480}
1481 
1482template <typename V1_t, typename V2_t, typename Mask_t>
1483inline Shuffle_match<V1_t, V2_t, Mask_t>
1484m_Shuffle(const V1_t &v1, const V2_t &v2, const Mask_t &mask) {
1485  return Shuffle_match<V1_t, V2_t, Mask_t>(v1, v2, mask);
1486}
1487 
1488/// Matches LoadInst.
1489template <typename OpTy>
1490inline OneOps_match<OpTy, Instruction::Load> m_Load(const OpTy &Op) {
1491  return OneOps_match<OpTy, Instruction::Load>(Op);
1492}
1493 
1494/// Matches StoreInst.
1495template <typename ValueOpTy, typename PointerOpTy>
1496inline TwoOps_match<ValueOpTy, PointerOpTy, Instruction::Store>
1497m_Store(const ValueOpTy &ValueOp, const PointerOpTy &PointerOp) {
1498  return TwoOps_match<ValueOpTy, PointerOpTy, Instruction::Store>(ValueOp,
1499                                                                  PointerOp);
1500}
1501 
1502//===----------------------------------------------------------------------===//
1503// Matchers for CastInst classes
1504//
1505 
1506template <typename Op_t, unsigned Opcode> struct CastClass_match {
1507  Op_t Op;
1508 
1509  CastClass_match(const Op_t &OpMatch) : Op(OpMatch) {}
1510 
1511  template <typename OpTy> bool match(OpTy *V) {
1512    if (auto *O = dyn_cast<Operator>(V))
1513      return O->getOpcode() == Opcode && Op.match(O->getOperand(0));
1514    return false;
1515  }
1516};
1517 
1518/// Matches BitCast.
1519template <typename OpTy>
1520inline CastClass_match<OpTy, Instruction::BitCast> m_BitCast(const OpTy &Op) {
1521  return CastClass_match<OpTy, Instruction::BitCast>(Op);
1522}
1523 
1524/// Matches PtrToInt.
1525template <typename OpTy>
1526inline CastClass_match<OpTy, Instruction::PtrToInt> m_PtrToInt(const OpTy &Op) {
1527  return CastClass_match<OpTy, Instruction::PtrToInt>(Op);
1528}
1529 
1530/// Matches IntToPtr.
1531template <typename OpTy>
1532inline CastClass_match<OpTy, Instruction::IntToPtr> m_IntToPtr(const OpTy &Op) {
1533  return CastClass_match<OpTy, Instruction::IntToPtr>(Op);
1534}
1535 
1536/// Matches Trunc.
1537template <typename OpTy>
1538inline CastClass_match<OpTy, Instruction::Trunc> m_Trunc(const OpTy &Op) {
1539  return CastClass_match<OpTy, Instruction::Trunc>(Op);
1540}
1541 
1542template <typename OpTy>
1543inline match_combine_or<CastClass_match<OpTy, Instruction::Trunc>, OpTy>
1544m_TruncOrSelf(const OpTy &Op) {
1545  return m_CombineOr(m_Trunc(Op), Op);
1546}
1547 
1548/// Matches SExt.
1549template <typename OpTy>
1550inline CastClass_match<OpTy, Instruction::SExt> m_SExt(const OpTy &Op) {
1551  return CastClass_match<OpTy, Instruction::SExt>(Op);
1552}
1553 
1554/// Matches ZExt.
1555template <typename OpTy>
1556inline CastClass_match<OpTy, Instruction::ZExt> m_ZExt(const OpTy &Op) {
1557  return CastClass_match<OpTy, Instruction::ZExt>(Op);
1558}
1559 
1560template <typename OpTy>
1561inline match_combine_or<CastClass_match<OpTy, Instruction::ZExt>, OpTy>
1562m_ZExtOrSelf(const OpTy &Op) {
1563  return m_CombineOr(m_ZExt(Op), Op);
1564}
1565 
1566template <typename OpTy>
1567inline match_combine_or<CastClass_match<OpTy, Instruction::SExt>, OpTy>
1568m_SExtOrSelf(const OpTy &Op) {
1569  return m_CombineOr(m_SExt(Op), Op);
1570}
1571 
1572template <typename OpTy>
1573inline match_combine_or<CastClass_match<OpTy, Instruction::ZExt>,
1574                        CastClass_match<OpTy, Instruction::SExt>>
1575m_ZExtOrSExt(const OpTy &Op) {
1576  return m_CombineOr(m_ZExt(Op), m_SExt(Op));
1577}
1578 
1579template <typename OpTy>
1580inline match_combine_or<
1581    match_combine_or<CastClass_match<OpTy, Instruction::ZExt>,
1582                     CastClass_match<OpTy, Instruction::SExt>>,
1583    OpTy>
1584m_ZExtOrSExtOrSelf(const OpTy &Op) {
1585  return m_CombineOr(m_ZExtOrSExt(Op), Op);
1586}
1587 
1588template <typename OpTy>
1589inline CastClass_match<OpTy, Instruction::UIToFP> m_UIToFP(const OpTy &Op) {
1590  return CastClass_match<OpTy, Instruction::UIToFP>(Op);
1591}
1592 
1593template <typename OpTy>
1594inline CastClass_match<OpTy, Instruction::SIToFP> m_SIToFP(const OpTy &Op) {
1595  return CastClass_match<OpTy, Instruction::SIToFP>(Op);
1596}
1597 
1598template <typename OpTy>
1599inline CastClass_match<OpTy, Instruction::FPToUI> m_FPToUI(const OpTy &Op) {
1600  return CastClass_match<OpTy, Instruction::FPToUI>(Op);
1601}
1602 
1603template <typename OpTy>
1604inline CastClass_match<OpTy, Instruction::FPToSI> m_FPToSI(const OpTy &Op) {
1605  return CastClass_match<OpTy, Instruction::FPToSI>(Op);
1606}
1607 
1608template <typename OpTy>
1609inline CastClass_match<OpTy, Instruction::FPTrunc> m_FPTrunc(const OpTy &Op) {
1610  return CastClass_match<OpTy, Instruction::FPTrunc>(Op);
1611}
1612 
1613template <typename OpTy>
1614inline CastClass_match<OpTy, Instruction::FPExt> m_FPExt(const OpTy &Op) {
1615  return CastClass_match<OpTy, Instruction::FPExt>(Op);
1616}
1617 
1618//===----------------------------------------------------------------------===//
1619// Matchers for control flow.
1620//
1621 
1622struct br_match {
1623  BasicBlock *&Succ;
1624 
1625  br_match(BasicBlock *&Succ) : Succ(Succ) {}
1626 
1627  template <typename OpTy> bool match(OpTy *V) {
1628    if (auto *BI = dyn_cast<BranchInst>(V))
1629      if (BI->isUnconditional()) {
1630        Succ = BI->getSuccessor(0);
1631        return true;
1632      }
1633    return false;
1634  }
1635};
1636 
1637inline br_match m_UnconditionalBr(BasicBlock *&Succ) { return br_match(Succ); }
1638 
1639template <typename Cond_t, typename TrueBlock_t, typename FalseBlock_t>
1640struct brc_match {
1641  Cond_t Cond;
1642  TrueBlock_t T;
1643  FalseBlock_t F;
1644 
1645  brc_match(const Cond_t &C, const TrueBlock_t &t, const FalseBlock_t &f)
1646      : Cond(C), T(t), F(f) {}
1647 
1648  template <typename OpTy> bool match(OpTy *V) {
1649    if (auto *BI = dyn_cast<BranchInst>(V))
1650      if (BI->isConditional() && Cond.match(BI->getCondition()))
1651        return T.match(BI->getSuccessor(0)) && F.match(BI->getSuccessor(1));
1652    return false;
1653  }
1654};
1655 
1656template <typename Cond_t>
1657inline brc_match<Cond_t, bind_ty<BasicBlock>, bind_ty<BasicBlock>>
1658m_Br(const Cond_t &C, BasicBlock *&T, BasicBlock *&F) {
1659  return brc_match<Cond_t, bind_ty<BasicBlock>, bind_ty<BasicBlock>>(
1660      C, m_BasicBlock(T), m_BasicBlock(F));
1661}
1662 
1663template <typename Cond_t, typename TrueBlock_t, typename FalseBlock_t>
1664inline brc_match<Cond_t, TrueBlock_t, FalseBlock_t>
1665m_Br(const Cond_t &C, const TrueBlock_t &T, const FalseBlock_t &F) {
1666  return brc_match<Cond_t, TrueBlock_t, FalseBlock_t>(C, T, F);
1667}
1668 
1669//===----------------------------------------------------------------------===//
1670// Matchers for max/min idioms, eg: "select (sgt x, y), x, y" -> smax(x,y).
1671//
1672 
1673template <typename CmpInst_t, typename LHS_t, typename RHS_t, typename Pred_t,
1674          bool Commutable = false>
1675struct MaxMin_match {
1676  LHS_t L;
1677  RHS_t R;
1678 
1679  // The evaluation order is always stable, regardless of Commutability.
1680  // The LHS is always matched first.
1681  MaxMin_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {}
1682 
1683  template <typename OpTy> bool match(OpTy *V) {
1684    if (auto *II = dyn_cast<IntrinsicInst>(V)) {
1685      Intrinsic::ID IID = II->getIntrinsicID();
1686      if ((IID == Intrinsic::smax && Pred_t::match(ICmpInst::ICMP_SGT)) ||
1687          (IID == Intrinsic::smin && Pred_t::match(ICmpInst::ICMP_SLT)) ||
1688          (IID == Intrinsic::umax && Pred_t::match(ICmpInst::ICMP_UGT)) ||
1689          (IID == Intrinsic::umin && Pred_t::match(ICmpInst::ICMP_ULT))) {
1690        Value *LHS = II->getOperand(0), *RHS = II->getOperand(1);
1691        return (L.match(LHS) && R.match(RHS)) ||
1692               (Commutable && L.match(RHS) && R.match(LHS));
1693      }
1694    }
1695    // Look for "(x pred y) ? x : y" or "(x pred y) ? y : x".
1696    auto *SI = dyn_cast<SelectInst>(V);
1697    if (!SI)
1698      return false;
1699    auto *Cmp = dyn_cast<CmpInst_t>(SI->getCondition());
1700    if (!Cmp)
1701      return false;
1702    // At this point we have a select conditioned on a comparison.  Check that
1703    // it is the values returned by the select that are being compared.
1704    Value *TrueVal = SI->getTrueValue();
1705    Value *FalseVal = SI->getFalseValue();
1706    Value *LHS = Cmp->getOperand(0);
1707    Value *RHS = Cmp->getOperand(1);
1708    if ((TrueVal != LHS || FalseVal != RHS) &&
1709        (TrueVal != RHS || FalseVal != LHS))
1710      return false;
1711    typename CmpInst_t::Predicate Pred =
1712        LHS == TrueVal ? Cmp->getPredicate() : Cmp->getInversePredicate();
1713    // Does "(x pred y) ? x : y" represent the desired max/min operation?
1714    if (!Pred_t::match(Pred))
1715      return false;
1716    // It does!  Bind the operands.
1717    return (L.match(LHS) && R.match(RHS)) ||
1718           (Commutable && L.match(RHS) && R.match(LHS));
1719  }
1720};
1721 
1722/// Helper class for identifying signed max predicates.
1723struct smax_pred_ty {
1724  static bool match(ICmpInst::Predicate Pred) {
1725    return Pred == CmpInst::ICMP_SGT || Pred == CmpInst::ICMP_SGE;
1726  }
1727};
1728 
1729/// Helper class for identifying signed min predicates.
1730struct smin_pred_ty {
1731  static bool match(ICmpInst::Predicate Pred) {
1732    return Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_SLE;
1733  }
1734};
1735 
1736/// Helper class for identifying unsigned max predicates.
1737struct umax_pred_ty {
1738  static bool match(ICmpInst::Predicate Pred) {
1739    return Pred == CmpInst::ICMP_UGT || Pred == CmpInst::ICMP_UGE;
1740  }
1741};
1742 
1743/// Helper class for identifying unsigned min predicates.
1744struct umin_pred_ty {
1745  static bool match(ICmpInst::Predicate Pred) {
1746    return Pred == CmpInst::ICMP_ULT || Pred == CmpInst::ICMP_ULE;
1747  }
1748};
1749 
1750/// Helper class for identifying ordered max predicates.
1751struct ofmax_pred_ty {
1752  static bool match(FCmpInst::Predicate Pred) {
1753    return Pred == CmpInst::FCMP_OGT || Pred == CmpInst::FCMP_OGE;
1754  }
1755};
1756 
1757/// Helper class for identifying ordered min predicates.
1758struct ofmin_pred_ty {
1759  static bool match(FCmpInst::Predicate Pred) {
1760    return Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_OLE;
1761  }
1762};
1763 
1764/// Helper class for identifying unordered max predicates.
1765struct ufmax_pred_ty {
1766  static bool match(FCmpInst::Predicate Pred) {
1767    return Pred == CmpInst::FCMP_UGT || Pred == CmpInst::FCMP_UGE;
1768  }
1769};
1770 
1771/// Helper class for identifying unordered min predicates.
1772struct ufmin_pred_ty {
1773  static bool match(FCmpInst::Predicate Pred) {
1774    return Pred == CmpInst::FCMP_ULT || Pred == CmpInst::FCMP_ULE;
1775  }
1776};
1777 
1778template <typename LHS, typename RHS>
1779inline MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty> m_SMax(const LHS &L,
1780                                                             const RHS &R) {
1781  return MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty>(L, R);
1782}
1783 
1784template <typename LHS, typename RHS>
1785inline MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty> m_SMin(const LHS &L,
1786                                                             const RHS &R) {
1787  return MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty>(L, R);
1788}
1789 
1790template <typename LHS, typename RHS>
1791inline MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty> m_UMax(const LHS &L,
1792                                                             const RHS &R) {
1793  return MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty>(L, R);
1794}
1795 
1796template <typename LHS, typename RHS>
1797inline MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty> m_UMin(const LHS &L,
1798                                                             const RHS &R) {
1799  return MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty>(L, R);
1800}
1801 
1802template <typename LHS, typename RHS>
1803inline match_combine_or<
1804    match_combine_or<MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty>,
1805                     MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty>>,
1806    match_combine_or<MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty>,
1807                     MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty>>>
1808m_MaxOrMin(const LHS &L, const RHS &R) {
1809  return m_CombineOr(m_CombineOr(m_SMax(L, R), m_SMin(L, R)),
1810                     m_CombineOr(m_UMax(L, R), m_UMin(L, R)));
1811}
1812 
1813/// Match an 'ordered' floating point maximum function.
1814/// Floating point has one special value 'NaN'. Therefore, there is no total
1815/// order. However, if we can ignore the 'NaN' value (for example, because of a
1816/// 'no-nans-float-math' flag) a combination of a fcmp and select has 'maximum'
1817/// semantics. In the presence of 'NaN' we have to preserve the original
1818/// select(fcmp(ogt/ge, L, R), L, R) semantics matched by this predicate.
1819///
1820///                         max(L, R)  iff L and R are not NaN
1821///  m_OrdFMax(L, R) =      R          iff L or R are NaN
1822template <typename LHS, typename RHS>
1823inline MaxMin_match<FCmpInst, LHS, RHS, ofmax_pred_ty> m_OrdFMax(const LHS &L,
1824                                                                 const RHS &R) {
1825  return MaxMin_match<FCmpInst, LHS, RHS, ofmax_pred_ty>(L, R);
1826}
1827 
1828/// Match an 'ordered' floating point minimum function.
1829/// Floating point has one special value 'NaN'. Therefore, there is no total
1830/// order. However, if we can ignore the 'NaN' value (for example, because of a
1831/// 'no-nans-float-math' flag) a combination of a fcmp and select has 'minimum'
1832/// semantics. In the presence of 'NaN' we have to preserve the original
1833/// select(fcmp(olt/le, L, R), L, R) semantics matched by this predicate.
1834///
1835///                         min(L, R)  iff L and R are not NaN
1836///  m_OrdFMin(L, R) =      R          iff L or R are NaN
1837template <typename LHS, typename RHS>
1838inline MaxMin_match<FCmpInst, LHS, RHS, ofmin_pred_ty> m_OrdFMin(const LHS &L,
1839                                                                 const RHS &R) {
1840  return MaxMin_match<FCmpInst, LHS, RHS, ofmin_pred_ty>(L, R);
1841}
1842 
1843/// Match an 'unordered' floating point maximum function.
1844/// Floating point has one special value 'NaN'. Therefore, there is no total
1845/// order. However, if we can ignore the 'NaN' value (for example, because of a
1846/// 'no-nans-float-math' flag) a combination of a fcmp and select has 'maximum'
1847/// semantics. In the presence of 'NaN' we have to preserve the original
1848/// select(fcmp(ugt/ge, L, R), L, R) semantics matched by this predicate.
1849///
1850///                         max(L, R)  iff L and R are not NaN
1851///  m_UnordFMax(L, R) =    L          iff L or R are NaN
1852template <typename LHS, typename RHS>
1853inline MaxMin_match<FCmpInst, LHS, RHS, ufmax_pred_ty>
1854m_UnordFMax(const LHS &L, const RHS &R) {
1855  return MaxMin_match<FCmpInst, LHS, RHS, ufmax_pred_ty>(L, R);
1856}
1857 
1858/// Match an 'unordered' floating point minimum function.
1859/// Floating point has one special value 'NaN'. Therefore, there is no total
1860/// order. However, if we can ignore the 'NaN' value (for example, because of a
1861/// 'no-nans-float-math' flag) a combination of a fcmp and select has 'minimum'
1862/// semantics. In the presence of 'NaN' we have to preserve the original
1863/// select(fcmp(ult/le, L, R), L, R) semantics matched by this predicate.
1864///
1865///                          min(L, R)  iff L and R are not NaN
1866///  m_UnordFMin(L, R) =     L          iff L or R are NaN
1867template <typename LHS, typename RHS>
1868inline MaxMin_match<FCmpInst, LHS, RHS, ufmin_pred_ty>
1869m_UnordFMin(const LHS &L, const RHS &R) {
1870  return MaxMin_match<FCmpInst, LHS, RHS, ufmin_pred_ty>(L, R);
1871}
1872 
1873//===----------------------------------------------------------------------===//
1874// Matchers for overflow check patterns: e.g. (a + b) u< a, (a ^ -1) <u b
1875// Note that S might be matched to other instructions than AddInst.
1876//
1877 
1878template <typename LHS_t, typename RHS_t, typename Sum_t>
1879struct UAddWithOverflow_match {
1880  LHS_t L;
1881  RHS_t R;
1882  Sum_t S;
1883 
1884  UAddWithOverflow_match(const LHS_t &L, const RHS_t &R, const Sum_t &S)
1885      : L(L), R(R), S(S) {}
1886 
1887  template <typename OpTy> bool match(OpTy *V) {
1888    Value *ICmpLHS, *ICmpRHS;
1889    ICmpInst::Predicate Pred;
1890    if (!m_ICmp(Pred, m_Value(ICmpLHS), m_Value(ICmpRHS)).match(V))
1891      return false;
1892 
1893    Value *AddLHS, *AddRHS;
1894    auto AddExpr = m_Add(m_Value(AddLHS), m_Value(AddRHS));
1895 
1896    // (a + b) u< a, (a + b) u< b
1897    if (Pred == ICmpInst::ICMP_ULT)
1898      if (AddExpr.match(ICmpLHS) && (ICmpRHS == AddLHS || ICmpRHS == AddRHS))
1899        return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpLHS);
1900 
1901    // a >u (a + b), b >u (a + b)
1902    if (Pred == ICmpInst::ICMP_UGT)
1903      if (AddExpr.match(ICmpRHS) && (ICmpLHS == AddLHS || ICmpLHS == AddRHS))
1904        return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpRHS);
1905 
1906    Value *Op1;
1907    auto XorExpr = m_OneUse(m_Xor(m_Value(Op1), m_AllOnes()));
1908    // (a ^ -1) <u b
1909    if (Pred == ICmpInst::ICMP_ULT) {
1910      if (XorExpr.match(ICmpLHS))
1911        return L.match(Op1) && R.match(ICmpRHS) && S.match(ICmpLHS);
1912    }
1913    //  b > u (a ^ -1)
1914    if (Pred == ICmpInst::ICMP_UGT) {
1915      if (XorExpr.match(ICmpRHS))
1916        return L.match(Op1) && R.match(ICmpLHS) && S.match(ICmpRHS);
1917    }
1918 
1919    // Match special-case for increment-by-1.
1920    if (Pred == ICmpInst::ICMP_EQ) {
1921      // (a + 1) == 0
1922      // (1 + a) == 0
1923      if (AddExpr.match(ICmpLHS) && m_ZeroInt().match(ICmpRHS) &&
1924          (m_One().match(AddLHS) || m_One().match(AddRHS)))
1925        return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpLHS);
1926      // 0 == (a + 1)
1927      // 0 == (1 + a)
1928      if (m_ZeroInt().match(ICmpLHS) && AddExpr.match(ICmpRHS) &&
1929          (m_One().match(AddLHS) || m_One().match(AddRHS)))
1930        return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpRHS);
1931    }
1932 
1933    return false;
1934  }
1935};
1936 
1937/// Match an icmp instruction checking for unsigned overflow on addition.
1938///
1939/// S is matched to the addition whose result is being checked for overflow, and
1940/// L and R are matched to the LHS and RHS of S.
1941template <typename LHS_t, typename RHS_t, typename Sum_t>
1942UAddWithOverflow_match<LHS_t, RHS_t, Sum_t>
1943m_UAddWithOverflow(const LHS_t &L, const RHS_t &R, const Sum_t &S) {
1944  return UAddWithOverflow_match<LHS_t, RHS_t, Sum_t>(L, R, S);
1945}
1946 
1947template <typename Opnd_t> struct Argument_match {
1948  unsigned OpI;
1949  Opnd_t Val;
1950 
1951  Argument_match(unsigned OpIdx, const Opnd_t &V) : OpI(OpIdx), Val(V) {}
1952 
1953  template <typename OpTy> bool match(OpTy *V) {
1954    // FIXME: Should likely be switched to use `CallBase`.
1955    if (const auto *CI = dyn_cast<CallInst>(V))
1956      return Val.match(CI->getArgOperand(OpI));
1957    return false;
1958  }
1959};
1960 
1961/// Match an argument.
1962template <unsigned OpI, typename Opnd_t>
1963inline Argument_match<Opnd_t> m_Argument(const Opnd_t &Op) {
1964  return Argument_match<Opnd_t>(OpI, Op);
1965}
1966 
1967/// Intrinsic matchers.
1968struct IntrinsicID_match {
1969  unsigned ID;
1970 
1971  IntrinsicID_match(Intrinsic::ID IntrID) : ID(IntrID) {}
1972 
1973  template <typename OpTy> bool match(OpTy *V) {
1974    if (const auto *CI42.1
'CI' is null
51.1
'CI' is null
42.1
'CI' is null
51.1
'CI' is null
42.1
'CI' is null
51.1
'CI' is null
42.1
'CI' is null
51.1
'CI' is null
42.1
'CI' is null
51.1
'CI' is null
 = dyn_cast<CallInst>(V))
42
←
Assuming 'V' is not a 'CallInst'→
43
←
Taking false branch→
51
←
'V' is not a 'CallInst'→
52
←
Taking false branch→
1975      if (const auto *F = CI->getCalledFunction())
1976        return F->getIntrinsicID() == ID;
1977    return false;
44
←
Returning zero, which participates in a condition later→
53
←
Returning zero, which participates in a condition later→
1978  }
1979};
1980 
1981/// Intrinsic matches are combinations of ID matchers, and argument
1982/// matchers. Higher arity matcher are defined recursively in terms of and-ing
1983/// them with lower arity matchers. Here's some convenient typedefs for up to
1984/// several arguments, and more can be added as needed
1985template <typename T0 = void, typename T1 = void, typename T2 = void,
1986          typename T3 = void, typename T4 = void, typename T5 = void,
1987          typename T6 = void, typename T7 = void, typename T8 = void,
1988          typename T9 = void, typename T10 = void>
1989struct m_Intrinsic_Ty;
1990template <typename T0> struct m_Intrinsic_Ty<T0> {
1991  using Ty = match_combine_and<IntrinsicID_match, Argument_match<T0>>;
1992};
1993template <typename T0, typename T1> struct m_Intrinsic_Ty<T0, T1> {
1994  using Ty =
1995      match_combine_and<typename m_Intrinsic_Ty<T0>::Ty, Argument_match<T1>>;
1996};
1997template <typename T0, typename T1, typename T2>
1998struct m_Intrinsic_Ty<T0, T1, T2> {
1999  using Ty =
2000      match_combine_and<typename m_Intrinsic_Ty<T0, T1>::Ty,
2001                        Argument_match<T2>>;
2002};
2003template <typename T0, typename T1, typename T2, typename T3>
2004struct m_Intrinsic_Ty<T0, T1, T2, T3> {
2005  using Ty =
2006      match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2>::Ty,
2007                        Argument_match<T3>>;
2008};
2009 
2010template <typename T0, typename T1, typename T2, typename T3, typename T4>
2011struct m_Intrinsic_Ty<T0, T1, T2, T3, T4> {
2012  using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2, T3>::Ty,
2013                               Argument_match<T4>>;
2014};
2015 
2016template <typename T0, typename T1, typename T2, typename T3, typename T4, typename T5>
2017struct m_Intrinsic_Ty<T0, T1, T2, T3, T4, T5> {
2018  using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2, T3, T4>::Ty,
2019                               Argument_match<T5>>;
2020};
2021 
2022/// Match intrinsic calls like this:
2023/// m_Intrinsic<Intrinsic::fabs>(m_Value(X))
2024template <Intrinsic::ID IntrID> inline IntrinsicID_match m_Intrinsic() {
2025  return IntrinsicID_match(IntrID);
2026}
2027 
2028template <Intrinsic::ID IntrID, typename T0>
2029inline typename m_Intrinsic_Ty<T0>::Ty m_Intrinsic(const T0 &Op0) {
2030  return m_CombineAnd(m_Intrinsic<IntrID>(), m_Argument<0>(Op0));
2031}
2032 
2033template <Intrinsic::ID IntrID, typename T0, typename T1>
2034inline typename m_Intrinsic_Ty<T0, T1>::Ty m_Intrinsic(const T0 &Op0,
2035                                                       const T1 &Op1) {
2036  return m_CombineAnd(m_Intrinsic<IntrID>(Op0), m_Argument<1>(Op1));
2037}
2038 
2039template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2>
2040inline typename m_Intrinsic_Ty<T0, T1, T2>::Ty
2041m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2) {
2042  return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1), m_Argument<2>(Op2));
2043}
2044 
2045template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2,
2046          typename T3>
2047inline typename m_Intrinsic_Ty<T0, T1, T2, T3>::Ty
2048m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3) {
2049  return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2), m_Argument<3>(Op3));
2050}
2051 
2052template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2,
2053          typename T3, typename T4>
2054inline typename m_Intrinsic_Ty<T0, T1, T2, T3, T4>::Ty
2055m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3,
2056            const T4 &Op4) {
2057  return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2, Op3),
2058                      m_Argument<4>(Op4));
2059}
2060 
2061template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2,
2062          typename T3, typename T4, typename T5>
2063inline typename m_Intrinsic_Ty<T0, T1, T2, T3, T4, T5>::Ty
2064m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3,
2065            const T4 &Op4, const T5 &Op5) {
2066  return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2, Op3, Op4),
2067                      m_Argument<5>(Op5));
2068}
2069 
2070// Helper intrinsic matching specializations.
2071template <typename Opnd0>
2072inline typename m_Intrinsic_Ty<Opnd0>::Ty m_BitReverse(const Opnd0 &Op0) {
2073  return m_Intrinsic<Intrinsic::bitreverse>(Op0);
2074}
2075 
2076template <typename Opnd0>
2077inline typename m_Intrinsic_Ty<Opnd0>::Ty m_BSwap(const Opnd0 &Op0) {
2078  return m_Intrinsic<Intrinsic::bswap>(Op0);
2079}
2080 
2081template <typename Opnd0>
2082inline typename m_Intrinsic_Ty<Opnd0>::Ty m_FAbs(const Opnd0 &Op0) {
2083  return m_Intrinsic<Intrinsic::fabs>(Op0);
2084}
2085 
2086template <typename Opnd0>
2087inline typename m_Intrinsic_Ty<Opnd0>::Ty m_FCanonicalize(const Opnd0 &Op0) {
2088  return m_Intrinsic<Intrinsic::canonicalize>(Op0);
2089}
2090 
2091template <typename Opnd0, typename Opnd1>
2092inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty m_FMin(const Opnd0 &Op0,
2093                                                        const Opnd1 &Op1) {
2094  return m_Intrinsic<Intrinsic::minnum>(Op0, Op1);
2095}
2096 
2097template <typename Opnd0, typename Opnd1>
2098inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty m_FMax(const Opnd0 &Op0,
2099                                                        const Opnd1 &Op1) {
2100  return m_Intrinsic<Intrinsic::maxnum>(Op0, Op1);
2101}
2102 
2103template <typename Opnd0, typename Opnd1, typename Opnd2>
2104inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2>::Ty
2105m_FShl(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2) {
2106  return m_Intrinsic<Intrinsic::fshl>(Op0, Op1, Op2);
2107}
2108 
2109template <typename Opnd0, typename Opnd1, typename Opnd2>
2110inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2>::Ty
2111m_FShr(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2) {
2112  return m_Intrinsic<Intrinsic::fshr>(Op0, Op1, Op2);
2113}
2114 
2115//===----------------------------------------------------------------------===//
2116// Matchers for two-operands operators with the operators in either order
2117//
2118 
2119/// Matches a BinaryOperator with LHS and RHS in either order.
2120template <typename LHS, typename RHS>
2121inline AnyBinaryOp_match<LHS, RHS, true> m_c_BinOp(const LHS &L, const RHS &R) {
2122  return AnyBinaryOp_match<LHS, RHS, true>(L, R);
2123}
2124 
2125/// Matches an ICmp with a predicate over LHS and RHS in either order.
2126/// Swaps the predicate if operands are commuted.
2127template <typename LHS, typename RHS>
2128inline CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate, true>
2129m_c_ICmp(ICmpInst::Predicate &Pred, const LHS &L, const RHS &R) {
2130  return CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate, true>(Pred, L,
2131                                                                       R);
2132}
2133 
2134/// Matches a Add with LHS and RHS in either order.
2135template <typename LHS, typename RHS>
2136inline BinaryOp_match<LHS, RHS, Instruction::Add, true> m_c_Add(const LHS &L,
2137                                                                const RHS &R) {
2138  return BinaryOp_match<LHS, RHS, Instruction::Add, true>(L, R);
2139}
2140 
2141/// Matches a Mul with LHS and RHS in either order.
2142template <typename LHS, typename RHS>
2143inline BinaryOp_match<LHS, RHS, Instruction::Mul, true> m_c_Mul(const LHS &L,
2144                                                                const RHS &R) {
2145  return BinaryOp_match<LHS, RHS, Instruction::Mul, true>(L, R);
2146}
2147 
2148/// Matches an And with LHS and RHS in either order.
2149template <typename LHS, typename RHS>
2150inline BinaryOp_match<LHS, RHS, Instruction::And, true> m_c_And(const LHS &L,
2151                                                                const RHS &R) {
2152  return BinaryOp_match<LHS, RHS, Instruction::And, true>(L, R);
2153}
2154 
2155/// Matches an Or with LHS and RHS in either order.
2156template <typename LHS, typename RHS>
2157inline BinaryOp_match<LHS, RHS, Instruction::Or, true> m_c_Or(const LHS &L,
2158                                                              const RHS &R) {
2159  return BinaryOp_match<LHS, RHS, Instruction::Or, true>(L, R);
2160}
2161 
2162/// Matches an Xor with LHS and RHS in either order.
2163template <typename LHS, typename RHS>
2164inline BinaryOp_match<LHS, RHS, Instruction::Xor, true> m_c_Xor(const LHS &L,
2165                                                                const RHS &R) {
2166  return BinaryOp_match<LHS, RHS, Instruction::Xor, true>(L, R);
2167}
2168 
2169/// Matches a 'Neg' as 'sub 0, V'.
2170template <typename ValTy>
2171inline BinaryOp_match<cst_pred_ty<is_zero_int>, ValTy, Instruction::Sub>
2172m_Neg(const ValTy &V) {
2173  return m_Sub(m_ZeroInt(), V);
2174}
2175 
2176/// Matches a 'Neg' as 'sub nsw 0, V'.
2177template <typename ValTy>
2178inline OverflowingBinaryOp_match<cst_pred_ty<is_zero_int>, ValTy,
2179                                 Instruction::Sub,
2180                                 OverflowingBinaryOperator::NoSignedWrap>
2181m_NSWNeg(const ValTy &V) {
2182  return m_NSWSub(m_ZeroInt(), V);
2183}
2184 
2185/// Matches a 'Not' as 'xor V, -1' or 'xor -1, V'.
2186template <typename ValTy>
2187inline BinaryOp_match<ValTy, cst_pred_ty<is_all_ones>, Instruction::Xor, true>
2188m_Not(const ValTy &V) {
2189  return m_c_Xor(V, m_AllOnes());
2190}
2191 
2192/// Matches an SMin with LHS and RHS in either order.
2193template <typename LHS, typename RHS>
2194inline MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty, true>
2195m_c_SMin(const LHS &L, const RHS &R) {
2196  return MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty, true>(L, R);
2197}
2198/// Matches an SMax with LHS and RHS in either order.
2199template <typename LHS, typename RHS>
2200inline MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty, true>
2201m_c_SMax(const LHS &L, const RHS &R) {
2202  return MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty, true>(L, R);
2203}
2204/// Matches a UMin with LHS and RHS in either order.
2205template <typename LHS, typename RHS>
2206inline MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty, true>
2207m_c_UMin(const LHS &L, const RHS &R) {
2208  return MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty, true>(L, R);
2209}
2210/// Matches a UMax with LHS and RHS in either order.
2211template <typename LHS, typename RHS>
2212inline MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty, true>
2213m_c_UMax(const LHS &L, const RHS &R) {
2214  return MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty, true>(L, R);
2215}
2216 
2217template <typename LHS, typename RHS>
2218inline match_combine_or<
2219    match_combine_or<MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty, true>,
2220                     MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty, true>>,
2221    match_combine_or<MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty, true>,
2222                     MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty, true>>>
2223m_c_MaxOrMin(const LHS &L, const RHS &R) {
2224  return m_CombineOr(m_CombineOr(m_c_SMax(L, R), m_c_SMin(L, R)),
2225                     m_CombineOr(m_c_UMax(L, R), m_c_UMin(L, R)));
2226}
2227 
2228/// Matches FAdd with LHS and RHS in either order.
2229template <typename LHS, typename RHS>
2230inline BinaryOp_match<LHS, RHS, Instruction::FAdd, true>
2231m_c_FAdd(const LHS &L, const RHS &R) {
2232  return BinaryOp_match<LHS, RHS, Instruction::FAdd, true>(L, R);
2233}
2234 
2235/// Matches FMul with LHS and RHS in either order.
2236template <typename LHS, typename RHS>
2237inline BinaryOp_match<LHS, RHS, Instruction::FMul, true>
2238m_c_FMul(const LHS &L, const RHS &R) {
2239  return BinaryOp_match<LHS, RHS, Instruction::FMul, true>(L, R);
2240}
2241 
2242template <typename Opnd_t> struct Signum_match {
2243  Opnd_t Val;
2244  Signum_match(const Opnd_t &V) : Val(V) {}
2245 
2246  template <typename OpTy> bool match(OpTy *V) {
2247    unsigned TypeSize = V->getType()->getScalarSizeInBits();
2248    if (TypeSize == 0)
2249      return false;
2250 
2251    unsigned ShiftWidth = TypeSize - 1;
2252    Value *OpL = nullptr, *OpR = nullptr;
2253 
2254    // This is the representation of signum we match:
2255    //
2256    //  signum(x) == (x >> 63) | (-x >>u 63)
2257    //
2258    // An i1 value is its own signum, so it's correct to match
2259    //
2260    //  signum(x) == (x >> 0)  | (-x >>u 0)
2261    //
2262    // for i1 values.
2263 
2264    auto LHS = m_AShr(m_Value(OpL), m_SpecificInt(ShiftWidth));
2265    auto RHS = m_LShr(m_Neg(m_Value(OpR)), m_SpecificInt(ShiftWidth));
2266    auto Signum = m_Or(LHS, RHS);
2267 
2268    return Signum.match(V) && OpL == OpR && Val.match(OpL);
2269  }
2270};
2271 
2272/// Matches a signum pattern.
2273///
2274/// signum(x) =
2275///      x >  0  ->  1
2276///      x == 0  ->  0
2277///      x <  0  -> -1
2278template <typename Val_t> inline Signum_match<Val_t> m_Signum(const Val_t &V) {
2279  return Signum_match<Val_t>(V);
2280}
2281 
2282template <int Ind, typename Opnd_t> struct ExtractValue_match {
2283  Opnd_t Val;
2284  ExtractValue_match(const Opnd_t &V) : Val(V) {}
2285 
2286  template <typename OpTy> bool match(OpTy *V) {
2287    if (auto *I = dyn_cast<ExtractValueInst>(V))
2288      return I->getNumIndices() == 1 && I->getIndices()[0] == Ind &&
2289             Val.match(I->getAggregateOperand());
2290    return false;
2291  }
2292};
2293 
2294/// Match a single index ExtractValue instruction.
2295/// For example m_ExtractValue<1>(...)
2296template <int Ind, typename Val_t>
2297inline ExtractValue_match<Ind, Val_t> m_ExtractValue(const Val_t &V) {
2298  return ExtractValue_match<Ind, Val_t>(V);
2299}
2300 
2301/// Matcher for a single index InsertValue instruction.
2302template <int Ind, typename T0, typename T1> struct InsertValue_match {
2303  T0 Op0;
2304  T1 Op1;
2305 
2306  InsertValue_match(const T0 &Op0, const T1 &Op1) : Op0(Op0), Op1(Op1) {}
2307 
2308  template <typename OpTy> bool match(OpTy *V) {
2309    if (auto *I = dyn_cast<InsertValueInst>(V)) {
2310      return Op0.match(I->getOperand(0)) && Op1.match(I->getOperand(1)) &&
2311             I->getNumIndices() == 1 && Ind == I->getIndices()[0];
2312    }
2313    return false;
2314  }
2315};
2316 
2317/// Matches a single index InsertValue instruction.
2318template <int Ind, typename Val_t, typename Elt_t>
2319inline InsertValue_match<Ind, Val_t, Elt_t> m_InsertValue(const Val_t &Val,
2320                                                          const Elt_t &Elt) {
2321  return InsertValue_match<Ind, Val_t, Elt_t>(Val, Elt);
2322}
2323 
2324/// Matches patterns for `vscale`. This can either be a call to `llvm.vscale` or
2325/// the constant expression
2326///  `ptrtoint(gep <vscale x 1 x i8>, <vscale x 1 x i8>* null, i32 1>`
2327/// under the right conditions determined by DataLayout.
2328struct VScaleVal_match {
2329private:
2330  template <typename Base, typename Offset>
2331  inline BinaryOp_match<Base, Offset, Instruction::GetElementPtr>
2332  m_OffsetGep(const Base &B, const Offset &O) {
2333    return BinaryOp_match<Base, Offset, Instruction::GetElementPtr>(B, O);
2334  }
2335 
2336public:
2337  const DataLayout &DL;
2338  VScaleVal_match(const DataLayout &DL) : DL(DL) {}
2339 
2340  template <typename ITy> bool match(ITy *V) {
2341    if (m_Intrinsic<Intrinsic::vscale>().match(V))
2342      return true;
2343 
2344    if (m_PtrToInt(m_OffsetGep(m_Zero(), m_SpecificInt(1))).match(V)) {
2345      Type *PtrTy = cast<Operator>(V)->getOperand(0)->getType();
2346      auto *DerefTy = PtrTy->getPointerElementType();
2347      if (isa<ScalableVectorType>(DerefTy) &&
2348          DL.getTypeAllocSizeInBits(DerefTy).getKnownMinSize() == 8)
2349        return true;
2350    }
2351 
2352    return false;
2353  }
2354};
2355 
2356inline VScaleVal_match m_VScale(const DataLayout &DL) {
2357  return VScaleVal_match(DL);
2358}
2359 
2360} // end namespace PatternMatch
2361} // end namespace llvm
2362 
2363#endif // LLVM_IR_PATTERNMATCH_H

←

/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/Analysis/AliasAnalysis.h

→

1//===- llvm/Analysis/AliasAnalysis.h - Alias Analysis Interface -*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the generic AliasAnalysis interface, which is used as the
10// common interface used by all clients of alias analysis information, and
11// implemented by all alias analysis implementations.  Mod/Ref information is
12// also captured by this interface.
13//
14// Implementations of this interface must implement the various virtual methods,
15// which automatically provides functionality for the entire suite of client
16// APIs.
17//
18// This API identifies memory regions with the MemoryLocation class. The pointer
19// component specifies the base memory address of the region. The Size specifies
20// the maximum size (in address units) of the memory region, or
21// MemoryLocation::UnknownSize if the size is not known. The TBAA tag
22// identifies the "type" of the memory reference; see the
23// TypeBasedAliasAnalysis class for details.
24//
25// Some non-obvious details include:
26//  - Pointers that point to two completely different objects in memory never
27//    alias, regardless of the value of the Size component.
28//  - NoAlias doesn't imply inequal pointers. The most obvious example of this
29//    is two pointers to constant memory. Even if they are equal, constant
30//    memory is never stored to, so there will never be any dependencies.
31//    In this and other situations, the pointers may be both NoAlias and
32//    MustAlias at the same time. The current API can only return one result,
33//    though this is rarely a problem in practice.
34//
35//===----------------------------------------------------------------------===//

37#ifndef LLVM_ANALYSIS_ALIASANALYSIS_H
38#define LLVM_ANALYSIS_ALIASANALYSIS_H

40#include "llvm/ADT/DenseMap.h"
41#include "llvm/ADT/None.h"
42#include "llvm/ADT/Optional.h"
43#include "llvm/ADT/SmallVector.h"
44#include "llvm/Analysis/MemoryLocation.h"
45#include "llvm/Analysis/TargetLibraryInfo.h"
46#include "llvm/IR/Function.h"
47#include "llvm/IR/Instruction.h"
48#include "llvm/IR/Instructions.h"
49#include "llvm/IR/PassManager.h"
50#include "llvm/Pass.h"
51#include <cstdint>
52#include <functional>
53#include <memory>
54#include <vector>

56namespace llvm {

58class AnalysisUsage;
59class BasicAAResult;
60class BasicBlock;
61class DominatorTree;
62class Value;

64/// The possible results of an alias query.
65///
66/// These results are always computed between two MemoryLocation objects as
67/// a query to some alias analysis.
68///
69/// Note that these are unscoped enumerations because we would like to support
70/// implicitly testing a result for the existence of any possible aliasing with
71/// a conversion to bool, but an "enum class" doesn't support this. The
72/// canonical names from the literature are suffixed and unique anyways, and so
73/// they serve as global constants in LLVM for these results.
74///
75/// See docs/AliasAnalysis.html for more information on the specific meanings
76/// of these values.
77enum AliasResult : uint8_t {
/// The two locations do not alias at all.
///
/// This value is arranged to convert to false, while all other values
/// convert to true. This allows a boolean context to convert the result to
/// a binary flag indicating whether there is the possibility of aliasing.
NoAlias = 0,
/// The two locations may or may not alias. This is the least precise result.
MayAlias,
/// The two locations alias, but only due to a partial overlap.
PartialAlias,
/// The two locations precisely alias each other.
MustAlias,
90};

92/// << operator for AliasResult.
93raw_ostream &operator<<(raw_ostream &OS, AliasResult AR);

95/// Flags indicating whether a memory access modifies or references memory.
96///
97/// This is no access at all, a modification, a reference, or both
98/// a modification and a reference. These are specifically structured such that
99/// they form a three bit matrix and bit-tests for 'mod' or 'ref' or 'must'
100/// work with any of the possible values.
101enum class ModRefInfo : uint8_t {
/// Must is provided for completeness, but no routines will return only
/// Must today. See definition of Must below.
Must = 0,
/// The access may reference the value stored in memory,
/// a mustAlias relation was found, and no mayAlias or partialAlias found.
MustRef = 1,
/// The access may modify the value stored in memory,
/// a mustAlias relation was found, and no mayAlias or partialAlias found.
MustMod = 2,
/// The access may reference, modify or both the value stored in memory,
/// a mustAlias relation was found, and no mayAlias or partialAlias found.
MustModRef = MustRef | MustMod,
/// The access neither references nor modifies the value stored in memory.
NoModRef = 4,
/// The access may reference the value stored in memory.
Ref = NoModRef | MustRef,
/// The access may modify the value stored in memory.
Mod = NoModRef | MustMod,
/// The access may reference and may modify the value stored in memory.
ModRef = Ref | Mod,

/// About Must:
/// Must is set in a best effort manner.
/// We usually do not try our best to infer Must, instead it is merely
/// another piece of "free" information that is presented when available.
/// Must set means there was certainly a MustAlias found. For calls,
/// where multiple arguments are checked (argmemonly), this translates to
/// only MustAlias or NoAlias was found.
/// Must is not set for RAR accesses, even if the two locations must
/// alias. The reason is that two read accesses translate to an early return
/// of NoModRef. An additional alias check to set Must may be
/// expensive. Other cases may also not set Must(e.g. callCapturesBefore).
/// We refer to Must being *set* when the most significant bit is *cleared*.
/// Conversely we *clear* Must information by *setting* the Must bit to 1.
136};

138LLVM_NODISCARD[[clang::warn_unused_result]] inline bool isNoModRef(const ModRefInfo MRI) {
return (static_cast<int>(MRI) & static_cast<int>(ModRefInfo::MustModRef)) ==
       static_cast<int>(ModRefInfo::Must);
141}
142LLVM_NODISCARD[[clang::warn_unused_result]] inline bool isModOrRefSet(const ModRefInfo MRI) {
return static_cast<int>(MRI) & static_cast<int>(ModRefInfo::MustModRef);
144}
145LLVM_NODISCARD[[clang::warn_unused_result]] inline bool isModAndRefSet(const ModRefInfo MRI) {
return (static_cast<int>(MRI) & static_cast<int>(ModRefInfo::MustModRef)) ==
       static_cast<int>(ModRefInfo::MustModRef);
148}
149LLVM_NODISCARD[[clang::warn_unused_result]] inline bool isModSet(const ModRefInfo MRI) {
return static_cast<int>(MRI) & static_cast<int>(ModRefInfo::MustMod);
151}
152LLVM_NODISCARD[[clang::warn_unused_result]] inline bool isRefSet(const ModRefInfo MRI) {
return static_cast<int>(MRI) & static_cast<int>(ModRefInfo::MustRef);
154}
155LLVM_NODISCARD[[clang::warn_unused_result]] inline bool isMustSet(const ModRefInfo MRI) {
return !(static_cast<int>(MRI) & static_cast<int>(ModRefInfo::NoModRef));
157}

159LLVM_NODISCARD[[clang::warn_unused_result]] inline ModRefInfo setMod(const ModRefInfo MRI) {
return ModRefInfo(static_cast<int>(MRI) |
                  static_cast<int>(ModRefInfo::MustMod));
162}
163LLVM_NODISCARD[[clang::warn_unused_result]] inline ModRefInfo setRef(const ModRefInfo MRI) {
return ModRefInfo(static_cast<int>(MRI) |
                  static_cast<int>(ModRefInfo::MustRef));
166}
167LLVM_NODISCARD[[clang::warn_unused_result]] inline ModRefInfo setMust(const ModRefInfo MRI) {
return ModRefInfo(static_cast<int>(MRI) &
                  static_cast<int>(ModRefInfo::MustModRef));
170}
171LLVM_NODISCARD[[clang::warn_unused_result]] inline ModRefInfo setModAndRef(const ModRefInfo MRI) {
return ModRefInfo(static_cast<int>(MRI) |
                  static_cast<int>(ModRefInfo::MustModRef));
174}
175LLVM_NODISCARD[[clang::warn_unused_result]] inline ModRefInfo clearMod(const ModRefInfo MRI) {
return ModRefInfo(static_cast<int>(MRI) & static_cast<int>(ModRefInfo::Ref));
177}
178LLVM_NODISCARD[[clang::warn_unused_result]] inline ModRefInfo clearRef(const ModRefInfo MRI) {
return ModRefInfo(static_cast<int>(MRI) & static_cast<int>(ModRefInfo::Mod));
180}
181LLVM_NODISCARD[[clang::warn_unused_result]] inline ModRefInfo clearMust(const ModRefInfo MRI) {
return ModRefInfo(static_cast<int>(MRI) |
                  static_cast<int>(ModRefInfo::NoModRef));
184}
185LLVM_NODISCARD[[clang::warn_unused_result]] inline ModRefInfo unionModRef(const ModRefInfo MRI1,
                                           const ModRefInfo MRI2) {
return ModRefInfo(static_cast<int>(MRI1) | static_cast<int>(MRI2));
188}
189LLVM_NODISCARD[[clang::warn_unused_result]] inline ModRefInfo intersectModRef(const ModRefInfo MRI1,
                                               const ModRefInfo MRI2) {
return ModRefInfo(static_cast<int>(MRI1) & static_cast<int>(MRI2));
192}

194/// The locations at which a function might access memory.
195///
196/// These are primarily used in conjunction with the \c AccessKind bits to
197/// describe both the nature of access and the locations of access for a
198/// function call.
199enum FunctionModRefLocation {
/// Base case is no access to memory.
FMRL_Nowhere = 0,
/// Access to memory via argument pointers.
FMRL_ArgumentPointees = 8,
/// Memory that is inaccessible via LLVM IR.
FMRL_InaccessibleMem = 16,
/// Access to any memory.
FMRL_Anywhere = 32 | FMRL_InaccessibleMem | FMRL_ArgumentPointees
208};

210/// Summary of how a function affects memory in the program.
211///
212/// Loads from constant globals are not considered memory accesses for this
213/// interface. Also, functions may freely modify stack space local to their
214/// invocation without having to report it through these interfaces.
215enum FunctionModRefBehavior {
/// This function does not perform any non-local loads or stores to memory.
///
/// This property corresponds to the GCC 'const' attribute.
/// This property corresponds to the LLVM IR 'readnone' attribute.
/// This property corresponds to the IntrNoMem LLVM intrinsic flag.
FMRB_DoesNotAccessMemory =
    FMRL_Nowhere | static_cast<int>(ModRefInfo::NoModRef),

/// The only memory references in this function (if it has any) are
/// non-volatile loads from objects pointed to by its pointer-typed
/// arguments, with arbitrary offsets.
///
/// This property corresponds to the combination of the IntrReadMem
/// and IntrArgMemOnly LLVM intrinsic flags.
FMRB_OnlyReadsArgumentPointees =
    FMRL_ArgumentPointees | static_cast<int>(ModRefInfo::Ref),

/// The only memory references in this function (if it has any) are
/// non-volatile stores from objects pointed to by its pointer-typed
/// arguments, with arbitrary offsets.
///
/// This property corresponds to the combination of the IntrWriteMem
/// and IntrArgMemOnly LLVM intrinsic flags.
FMRB_OnlyWritesArgumentPointees =
    FMRL_ArgumentPointees | static_cast<int>(ModRefInfo::Mod),

/// The only memory references in this function (if it has any) are
/// non-volatile loads and stores from objects pointed to by its
/// pointer-typed arguments, with arbitrary offsets.
///
/// This property corresponds to the IntrArgMemOnly LLVM intrinsic flag.
FMRB_OnlyAccessesArgumentPointees =
    FMRL_ArgumentPointees | static_cast<int>(ModRefInfo::ModRef),

/// The only memory references in this function (if it has any) are
/// reads of memory that is otherwise inaccessible via LLVM IR.
///
/// This property corresponds to the LLVM IR inaccessiblememonly attribute.
FMRB_OnlyReadsInaccessibleMem =
    FMRL_InaccessibleMem | static_cast<int>(ModRefInfo::Ref),

/// The only memory references in this function (if it has any) are
/// writes to memory that is otherwise inaccessible via LLVM IR.
///
/// This property corresponds to the LLVM IR inaccessiblememonly attribute.
FMRB_OnlyWritesInaccessibleMem =
    FMRL_InaccessibleMem | static_cast<int>(ModRefInfo::Mod),

/// The only memory references in this function (if it has any) are
/// references of memory that is otherwise inaccessible via LLVM IR.
///
/// This property corresponds to the LLVM IR inaccessiblememonly attribute.
FMRB_OnlyAccessesInaccessibleMem =
    FMRL_InaccessibleMem | static_cast<int>(ModRefInfo::ModRef),

/// The function may perform non-volatile loads from objects pointed
/// to by its pointer-typed arguments, with arbitrary offsets, and
/// it may also perform loads of memory that is otherwise
/// inaccessible via LLVM IR.
///
/// This property corresponds to the LLVM IR
/// inaccessiblemem_or_argmemonly attribute.
FMRB_OnlyReadsInaccessibleOrArgMem = FMRL_InaccessibleMem |
                                     FMRL_ArgumentPointees |
                                     static_cast<int>(ModRefInfo::Ref),

/// The function may perform non-volatile stores to objects pointed
/// to by its pointer-typed arguments, with arbitrary offsets, and
/// it may also perform stores of memory that is otherwise
/// inaccessible via LLVM IR.
///
/// This property corresponds to the LLVM IR
/// inaccessiblemem_or_argmemonly attribute.
FMRB_OnlyWritesInaccessibleOrArgMem = FMRL_InaccessibleMem |
                                      FMRL_ArgumentPointees |
                                      static_cast<int>(ModRefInfo::Mod),

/// The function may perform non-volatile loads and stores of objects
/// pointed to by its pointer-typed arguments, with arbitrary offsets, and
/// it may also perform loads and stores of memory that is otherwise
/// inaccessible via LLVM IR.
///
/// This property corresponds to the LLVM IR
/// inaccessiblemem_or_argmemonly attribute.
FMRB_OnlyAccessesInaccessibleOrArgMem = FMRL_InaccessibleMem |
                                        FMRL_ArgumentPointees |
                                        static_cast<int>(ModRefInfo::ModRef),

/// This function does not perform any non-local stores or volatile loads,
/// but may read from any memory location.
///
/// This property corresponds to the GCC 'pure' attribute.
/// This property corresponds to the LLVM IR 'readonly' attribute.
/// This property corresponds to the IntrReadMem LLVM intrinsic flag.
FMRB_OnlyReadsMemory = FMRL_Anywhere | static_cast<int>(ModRefInfo::Ref),

// This function does not read from memory anywhere, but may write to any
// memory location.
//
// This property corresponds to the LLVM IR 'writeonly' attribute.
// This property corresponds to the IntrWriteMem LLVM intrinsic flag.
FMRB_OnlyWritesMemory = FMRL_Anywhere | static_cast<int>(ModRefInfo::Mod),

/// This indicates that the function could not be classified into one of the
/// behaviors above.
FMRB_UnknownModRefBehavior =
    FMRL_Anywhere | static_cast<int>(ModRefInfo::ModRef)
323};

325// Wrapper method strips bits significant only in FunctionModRefBehavior,
326// to obtain a valid ModRefInfo. The benefit of using the wrapper is that if
327// ModRefInfo enum changes, the wrapper can be updated to & with the new enum
328// entry with all bits set to 1.
329LLVM_NODISCARD[[clang::warn_unused_result]] inline ModRefInfo
330createModRefInfo(const FunctionModRefBehavior FMRB) {
return ModRefInfo(FMRB & static_cast<int>(ModRefInfo::ModRef));
332}

334/// This class stores info we want to provide to or retain within an alias
335/// query. By default, the root query is stateless and starts with a freshly
336/// constructed info object. Specific alias analyses can use this query info to
337/// store per-query state that is important for recursive or nested queries to
338/// avoid recomputing. To enable preserving this state across multiple queries
339/// where safe (due to the IR not changing), use a `BatchAAResults` wrapper.
340/// The information stored in an `AAQueryInfo` is currently limitted to the
341/// caches used by BasicAA, but can further be extended to fit other AA needs.
342class AAQueryInfo {
343public:
using LocPair = std::pair<MemoryLocation, MemoryLocation>;
using AliasCacheT = SmallDenseMap<LocPair, AliasResult, 8>;
AliasCacheT AliasCache;

using IsCapturedCacheT = SmallDenseMap<const Value *, bool, 8>;
IsCapturedCacheT IsCapturedCache;

AAQueryInfo() : AliasCache(), IsCapturedCache() {}

AliasResult updateResult(const LocPair &Locs, AliasResult Result) {
  auto It = AliasCache.find(Locs);
  assert(It != AliasCache.end() && "Entry must have existed")((It != AliasCache.end() && "Entry must have existed"
) ? static_cast<void> (0) : __assert_fail ("It != AliasCache.end() && \"Entry must have existed\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/Analysis/AliasAnalysis.h"
, 355, __PRETTY_FUNCTION__));
  return It->second = Result;
}
358};

360class BatchAAResults;

362class AAResults {
363public:
// Make these results default constructable and movable. We have to spell
// these out because MSVC won't synthesize them.
AAResults(const TargetLibraryInfo &TLI) : TLI(TLI) {}
AAResults(AAResults &&Arg);
~AAResults();

/// Register a specific AA result.
template <typename AAResultT> void addAAResult(AAResultT &AAResult) {
  // FIXME: We should use a much lighter weight system than the usual
  // polymorphic pattern because we don't own AAResult. It should
  // ideally involve two pointers and no separate allocation.
  AAs.emplace_back(new Model<AAResultT>(AAResult, *this));
}

/// Register a function analysis ID that the results aggregation depends on.
///
/// This is used in the new pass manager to implement the invalidation logic
/// where we must invalidate the results aggregation if any of our component
/// analyses become invalid.
void addAADependencyID(AnalysisKey *ID) { AADeps.push_back(ID); }

/// Handle invalidation events in the new pass manager.
///
/// The aggregation is invalidated if any of the underlying analyses is
/// invalidated.
bool invalidate(Function &F, const PreservedAnalyses &PA,
                FunctionAnalysisManager::Invalidator &Inv);

//===--------------------------------------------------------------------===//
/// \name Alias Queries
/// @{

/// The main low level interface to the alias analysis implementation.
/// Returns an AliasResult indicating whether the two pointers are aliased to
/// each other. This is the interface that must be implemented by specific
/// alias analysis implementations.
AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB);

/// A convenience wrapper around the primary \c alias interface.
AliasResult alias(const Value *V1, LocationSize V1Size, const Value *V2,
                  LocationSize V2Size) {
  return alias(MemoryLocation(V1, V1Size), MemoryLocation(V2, V2Size));
}

/// A convenience wrapper around the primary \c alias interface.
AliasResult alias(const Value *V1, const Value *V2) {
  return alias(V1, LocationSize::unknown(), V2, LocationSize::unknown());
}

/// A trivial helper function to check to see if the specified pointers are
/// no-alias.
bool isNoAlias(const MemoryLocation &LocA, const MemoryLocation &LocB) {
  return alias(LocA, LocB) == NoAlias;
}

/// A convenience wrapper around the \c isNoAlias helper interface.
bool isNoAlias(const Value *V1, LocationSize V1Size, const Value *V2,
               LocationSize V2Size) {
  return isNoAlias(MemoryLocation(V1, V1Size), MemoryLocation(V2, V2Size));
}

/// A convenience wrapper around the \c isNoAlias helper interface.
bool isNoAlias(const Value *V1, const Value *V2) {
  return isNoAlias(MemoryLocation(V1, LocationSize::unknown()),
                   MemoryLocation(V2, LocationSize::unknown()));
}

/// A trivial helper function to check to see if the specified pointers are
/// must-alias.
bool isMustAlias(const MemoryLocation &LocA, const MemoryLocation &LocB) {
  return alias(LocA, LocB) == MustAlias;
}

/// A convenience wrapper around the \c isMustAlias helper interface.
bool isMustAlias(const Value *V1, const Value *V2) {
  return alias(V1, LocationSize::precise(1), V2, LocationSize::precise(1)) ==
         MustAlias;
}

/// Checks whether the given location points to constant memory, or if
/// \p OrLocal is true whether it points to a local alloca.
bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal = false);

/// A convenience wrapper around the primary \c pointsToConstantMemory
/// interface.
bool pointsToConstantMemory(const Value *P, bool OrLocal = false) {
  return pointsToConstantMemory(MemoryLocation(P, LocationSize::unknown()),
                                OrLocal);
}

/// @}
//===--------------------------------------------------------------------===//
/// \name Simple mod/ref information
/// @{

/// Get the ModRef info associated with a pointer argument of a call. The
/// result's bits are set to indicate the allowed aliasing ModRef kinds. Note
/// that these bits do not necessarily account for the overall behavior of
/// the function, but rather only provide additional per-argument
/// information. This never sets ModRefInfo::Must.
ModRefInfo getArgModRefInfo(const CallBase *Call, unsigned ArgIdx);

/// Return the behavior of the given call site.
FunctionModRefBehavior getModRefBehavior(const CallBase *Call);

/// Return the behavior when calling the given function.
FunctionModRefBehavior getModRefBehavior(const Function *F);

/// Checks if the specified call is known to never read or write memory.
///
/// Note that if the call only reads from known-constant memory, it is also
/// legal to return true. Also, calls that unwind the stack are legal for
/// this predicate.
///
/// Many optimizations (such as CSE and LICM) can be performed on such calls
/// without worrying about aliasing properties, and many calls have this
/// property (e.g. calls to 'sin' and 'cos').
///
/// This property corresponds to the GCC 'const' attribute.
bool doesNotAccessMemory(const CallBase *Call) {
  return getModRefBehavior(Call) == FMRB_DoesNotAccessMemory;
}

/// Checks if the specified function is known to never read or write memory.
///
/// Note that if the function only reads from known-constant memory, it is
/// also legal to return true. Also, function that unwind the stack are legal
/// for this predicate.
///
/// Many optimizations (such as CSE and LICM) can be performed on such calls
/// to such functions without worrying about aliasing properties, and many
/// functions have this property (e.g. 'sin' and 'cos').
///
/// This property corresponds to the GCC 'const' attribute.
bool doesNotAccessMemory(const Function *F) {
  return getModRefBehavior(F) == FMRB_DoesNotAccessMemory;
}

/// Checks if the specified call is known to only read from non-volatile
/// memory (or not access memory at all).
///
/// Calls that unwind the stack are legal for this predicate.
///
/// This property allows many common optimizations to be performed in the
/// absence of interfering store instructions, such as CSE of strlen calls.
///
/// This property corresponds to the GCC 'pure' attribute.
bool onlyReadsMemory(const CallBase *Call) {
  return onlyReadsMemory(getModRefBehavior(Call));
}

/// Checks if the specified function is known to only read from non-volatile
/// memory (or not access memory at all).
///
/// Functions that unwind the stack are legal for this predicate.
///
/// This property allows many common optimizations to be performed in the
/// absence of interfering store instructions, such as CSE of strlen calls.
///
/// This property corresponds to the GCC 'pure' attribute.
bool onlyReadsMemory(const Function *F) {
  return onlyReadsMemory(getModRefBehavior(F));
}

/// Checks if functions with the specified behavior are known to only read
/// from non-volatile memory (or not access memory at all).
static bool onlyReadsMemory(FunctionModRefBehavior MRB) {
  return !isModSet(createModRefInfo(MRB));
61
←
Assuming the condition is true→
62
←
Returning the value 1, which participates in a condition later→
}

/// Checks if functions with the specified behavior are known to only write
/// memory (or not access memory at all).
static bool doesNotReadMemory(FunctionModRefBehavior MRB) {
  return !isRefSet(createModRefInfo(MRB));
}

/// Checks if functions with the specified behavior are known to read and
/// write at most from objects pointed to by their pointer-typed arguments
/// (with arbitrary offsets).
static bool onlyAccessesArgPointees(FunctionModRefBehavior MRB) {
  return !(MRB & FMRL_Anywhere & ~FMRL_ArgumentPointees);
66
←
Assuming the condition is true→
67
←
Returning the value 1, which participates in a condition later→
}

/// Checks if functions with the specified behavior are known to potentially
/// read or write from objects pointed to be their pointer-typed arguments
/// (with arbitrary offsets).
static bool doesAccessArgPointees(FunctionModRefBehavior MRB) {
  return isModOrRefSet(createModRefInfo(MRB)) &&
         (MRB & FMRL_ArgumentPointees);
}

/// Checks if functions with the specified behavior are known to read and
/// write at most from memory that is inaccessible from LLVM IR.
static bool onlyAccessesInaccessibleMem(FunctionModRefBehavior MRB) {
  return !(MRB & FMRL_Anywhere & ~FMRL_InaccessibleMem);
}

/// Checks if functions with the specified behavior are known to potentially
/// read or write from memory that is inaccessible from LLVM IR.
static bool doesAccessInaccessibleMem(FunctionModRefBehavior MRB) {
  return isModOrRefSet(createModRefInfo(MRB)) && (MRB & FMRL_InaccessibleMem);
}

/// Checks if functions with the specified behavior are known to read and
/// write at most from memory that is inaccessible from LLVM IR or objects
/// pointed to by their pointer-typed arguments (with arbitrary offsets).
static bool onlyAccessesInaccessibleOrArgMem(FunctionModRefBehavior MRB) {
  return !(MRB & FMRL_Anywhere &
           ~(FMRL_InaccessibleMem | FMRL_ArgumentPointees));
}

/// getModRefInfo (for call sites) - Return information about whether
/// a particular call site modifies or reads the specified memory location.
ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc);

/// getModRefInfo (for call sites) - A convenience wrapper.
ModRefInfo getModRefInfo(const CallBase *Call, const Value *P,
                         LocationSize Size) {
  return getModRefInfo(Call, MemoryLocation(P, Size));
}

/// getModRefInfo (for loads) - Return information about whether
/// a particular load modifies or reads the specified memory location.
ModRefInfo getModRefInfo(const LoadInst *L, const MemoryLocation &Loc);

/// getModRefInfo (for loads) - A convenience wrapper.
ModRefInfo getModRefInfo(const LoadInst *L, const Value *P,
                         LocationSize Size) {
  return getModRefInfo(L, MemoryLocation(P, Size));
}

/// getModRefInfo (for stores) - Return information about whether
/// a particular store modifies or reads the specified memory location.
ModRefInfo getModRefInfo(const StoreInst *S, const MemoryLocation &Loc);

/// getModRefInfo (for stores) - A convenience wrapper.
ModRefInfo getModRefInfo(const StoreInst *S, const Value *P,
                         LocationSize Size) {
  return getModRefInfo(S, MemoryLocation(P, Size));
}

/// getModRefInfo (for fences) - Return information about whether
/// a particular store modifies or reads the specified memory location.
ModRefInfo getModRefInfo(const FenceInst *S, const MemoryLocation &Loc);

/// getModRefInfo (for fences) - A convenience wrapper.
ModRefInfo getModRefInfo(const FenceInst *S, const Value *P,
                         LocationSize Size) {
  return getModRefInfo(S, MemoryLocation(P, Size));
}

/// getModRefInfo (for cmpxchges) - Return information about whether
/// a particular cmpxchg modifies or reads the specified memory location.
ModRefInfo getModRefInfo(const AtomicCmpXchgInst *CX,
                         const MemoryLocation &Loc);

/// getModRefInfo (for cmpxchges) - A convenience wrapper.
ModRefInfo getModRefInfo(const AtomicCmpXchgInst *CX, const Value *P,
                         LocationSize Size) {
  return getModRefInfo(CX, MemoryLocation(P, Size));
}

/// getModRefInfo (for atomicrmws) - Return information about whether
/// a particular atomicrmw modifies or reads the specified memory location.
ModRefInfo getModRefInfo(const AtomicRMWInst *RMW, const MemoryLocation &Loc);

/// getModRefInfo (for atomicrmws) - A convenience wrapper.
ModRefInfo getModRefInfo(const AtomicRMWInst *RMW, const Value *P,
                         LocationSize Size) {
  return getModRefInfo(RMW, MemoryLocation(P, Size));
}

/// getModRefInfo (for va_args) - Return information about whether
/// a particular va_arg modifies or reads the specified memory location.
ModRefInfo getModRefInfo(const VAArgInst *I, const MemoryLocation &Loc);

/// getModRefInfo (for va_args) - A convenience wrapper.
ModRefInfo getModRefInfo(const VAArgInst *I, const Value *P,
                         LocationSize Size) {
  return getModRefInfo(I, MemoryLocation(P, Size));
}

/// getModRefInfo (for catchpads) - Return information about whether
/// a particular catchpad modifies or reads the specified memory location.
ModRefInfo getModRefInfo(const CatchPadInst *I, const MemoryLocation &Loc);

/// getModRefInfo (for catchpads) - A convenience wrapper.
ModRefInfo getModRefInfo(const CatchPadInst *I, const Value *P,
                         LocationSize Size) {
  return getModRefInfo(I, MemoryLocation(P, Size));
}

/// getModRefInfo (for catchrets) - Return information about whether
/// a particular catchret modifies or reads the specified memory location.
ModRefInfo getModRefInfo(const CatchReturnInst *I, const MemoryLocation &Loc);

/// getModRefInfo (for catchrets) - A convenience wrapper.
ModRefInfo getModRefInfo(const CatchReturnInst *I, const Value *P,
                         LocationSize Size) {
  return getModRefInfo(I, MemoryLocation(P, Size));
}

/// Check whether or not an instruction may read or write the optionally
/// specified memory location.
///
///
/// An instruction that doesn't read or write memory may be trivially LICM'd
/// for example.
///
/// For function calls, this delegates to the alias-analysis specific
/// call-site mod-ref behavior queries. Otherwise it delegates to the specific
/// helpers above.
ModRefInfo getModRefInfo(const Instruction *I,
                         const Optional<MemoryLocation> &OptLoc) {
  AAQueryInfo AAQIP;
  return getModRefInfo(I, OptLoc, AAQIP);
}

/// A convenience wrapper for constructing the memory location.
ModRefInfo getModRefInfo(const Instruction *I, const Value *P,
                         LocationSize Size) {
  return getModRefInfo(I, MemoryLocation(P, Size));
}

/// Return information about whether a call and an instruction may refer to
/// the same memory locations.
ModRefInfo getModRefInfo(Instruction *I, const CallBase *Call);

/// Return information about whether two call sites may refer to the same set
/// of memory locations. See the AA documentation for details:
///   http://llvm.org/docs/AliasAnalysis.html#ModRefInfo
ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2);

/// Return information about whether a particular call site modifies
/// or reads the specified memory location \p MemLoc before instruction \p I
/// in a BasicBlock.
/// Early exits in callCapturesBefore may lead to ModRefInfo::Must not being
/// set.
ModRefInfo callCapturesBefore(const Instruction *I,
                              const MemoryLocation &MemLoc, DominatorTree *DT);

/// A convenience wrapper to synthesize a memory location.
ModRefInfo callCapturesBefore(const Instruction *I, const Value *P,
                              LocationSize Size, DominatorTree *DT) {
  return callCapturesBefore(I, MemoryLocation(P, Size), DT);
}

/// @}
//===--------------------------------------------------------------------===//
/// \name Higher level methods for querying mod/ref information.
/// @{

/// Check if it is possible for execution of the specified basic block to
/// modify the location Loc.
bool canBasicBlockModify(const BasicBlock &BB, const MemoryLocation &Loc);

/// A convenience wrapper synthesizing a memory location.
bool canBasicBlockModify(const BasicBlock &BB, const Value *P,
                         LocationSize Size) {
  return canBasicBlockModify(BB, MemoryLocation(P, Size));
}

/// Check if it is possible for the execution of the specified instructions
/// to mod\ref (according to the mode) the location Loc.
///
/// The instructions to consider are all of the instructions in the range of
/// [I1,I2] INCLUSIVE. I1 and I2 must be in the same basic block.
bool canInstructionRangeModRef(const Instruction &I1, const Instruction &I2,
                               const MemoryLocation &Loc,
                               const ModRefInfo Mode);

/// A convenience wrapper synthesizing a memory location.
bool canInstructionRangeModRef(const Instruction &I1, const Instruction &I2,
                               const Value *Ptr, LocationSize Size,
                               const ModRefInfo Mode) {
  return canInstructionRangeModRef(I1, I2, MemoryLocation(Ptr, Size), Mode);
}

742private:
AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB,
                  AAQueryInfo &AAQI);
bool pointsToConstantMemory(const MemoryLocation &Loc, AAQueryInfo &AAQI,
                            bool OrLocal = false);
ModRefInfo getModRefInfo(Instruction *I, const CallBase *Call2,
                         AAQueryInfo &AAQIP);
ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc,
                         AAQueryInfo &AAQI);
ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2,
                         AAQueryInfo &AAQI);
ModRefInfo getModRefInfo(const VAArgInst *V, const MemoryLocation &Loc,
                         AAQueryInfo &AAQI);
ModRefInfo getModRefInfo(const LoadInst *L, const MemoryLocation &Loc,
                         AAQueryInfo &AAQI);
ModRefInfo getModRefInfo(const StoreInst *S, const MemoryLocation &Loc,
                         AAQueryInfo &AAQI);
ModRefInfo getModRefInfo(const FenceInst *S, const MemoryLocation &Loc,
                         AAQueryInfo &AAQI);
ModRefInfo getModRefInfo(const AtomicCmpXchgInst *CX,
                         const MemoryLocation &Loc, AAQueryInfo &AAQI);
ModRefInfo getModRefInfo(const AtomicRMWInst *RMW, const MemoryLocation &Loc,
                         AAQueryInfo &AAQI);
ModRefInfo getModRefInfo(const CatchPadInst *I, const MemoryLocation &Loc,
                         AAQueryInfo &AAQI);
ModRefInfo getModRefInfo(const CatchReturnInst *I, const MemoryLocation &Loc,
                         AAQueryInfo &AAQI);
ModRefInfo getModRefInfo(const Instruction *I,
                         const Optional<MemoryLocation> &OptLoc,
                         AAQueryInfo &AAQIP) {
  if (OptLoc == None) {
    if (const auto *Call = dyn_cast<CallBase>(I)) {
      return createModRefInfo(getModRefBehavior(Call));
    }
  }

  const MemoryLocation &Loc = OptLoc.getValueOr(MemoryLocation());

  switch (I->getOpcode()) {
  case Instruction::VAArg:
    return getModRefInfo((const VAArgInst *)I, Loc, AAQIP);
  case Instruction::Load:
    return getModRefInfo((const LoadInst *)I, Loc, AAQIP);
  case Instruction::Store:
    return getModRefInfo((const StoreInst *)I, Loc, AAQIP);
  case Instruction::Fence:
    return getModRefInfo((const FenceInst *)I, Loc, AAQIP);
  case Instruction::AtomicCmpXchg:
    return getModRefInfo((const AtomicCmpXchgInst *)I, Loc, AAQIP);
  case Instruction::AtomicRMW:
    return getModRefInfo((const AtomicRMWInst *)I, Loc, AAQIP);
  case Instruction::Call:
    return getModRefInfo((const CallInst *)I, Loc, AAQIP);
  case Instruction::Invoke:
    return getModRefInfo((const InvokeInst *)I, Loc, AAQIP);
  case Instruction::CatchPad:
    return getModRefInfo((const CatchPadInst *)I, Loc, AAQIP);
  case Instruction::CatchRet:
    return getModRefInfo((const CatchReturnInst *)I, Loc, AAQIP);
  default:
    return ModRefInfo::NoModRef;
  }
}

class Concept;

template <typename T> class Model;

template <typename T> friend class AAResultBase;

const TargetLibraryInfo &TLI;

std::vector<std::unique_ptr<Concept>> AAs;

std::vector<AnalysisKey *> AADeps;

friend class BatchAAResults;
819};

821/// This class is a wrapper over an AAResults, and it is intended to be used
822/// only when there are no IR changes inbetween queries. BatchAAResults is
823/// reusing the same `AAQueryInfo` to preserve the state across queries,
824/// esentially making AA work in "batch mode". The internal state cannot be
825/// cleared, so to go "out-of-batch-mode", the user must either use AAResults,
826/// or create a new BatchAAResults.
827class BatchAAResults {
AAResults &AA;
AAQueryInfo AAQI;

831public:
BatchAAResults(AAResults &AAR) : AA(AAR), AAQI() {}
AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB) {
  return AA.alias(LocA, LocB, AAQI);
}
bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal = false) {
  return AA.pointsToConstantMemory(Loc, AAQI, OrLocal);
}
ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc) {
  return AA.getModRefInfo(Call, Loc, AAQI);
}
ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2) {
  return AA.getModRefInfo(Call1, Call2, AAQI);
}
ModRefInfo getModRefInfo(const Instruction *I,
                         const Optional<MemoryLocation> &OptLoc) {
  return AA.getModRefInfo(I, OptLoc, AAQI);
}
ModRefInfo getModRefInfo(Instruction *I, const CallBase *Call2) {
  return AA.getModRefInfo(I, Call2, AAQI);
}
ModRefInfo getArgModRefInfo(const CallBase *Call, unsigned ArgIdx) {
  return AA.getArgModRefInfo(Call, ArgIdx);
}
FunctionModRefBehavior getModRefBehavior(const CallBase *Call) {
  return AA.getModRefBehavior(Call);
}
bool isMustAlias(const MemoryLocation &LocA, const MemoryLocation &LocB) {
  return alias(LocA, LocB) == MustAlias;
}
bool isMustAlias(const Value *V1, const Value *V2) {
  return alias(MemoryLocation(V1, LocationSize::precise(1)),
               MemoryLocation(V2, LocationSize::precise(1))) == MustAlias;
}
865};

867/// Temporary typedef for legacy code that uses a generic \c AliasAnalysis
868/// pointer or reference.
869using AliasAnalysis = AAResults;

871/// A private abstract base class describing the concept of an individual alias
872/// analysis implementation.
873///
874/// This interface is implemented by any \c Model instantiation. It is also the
875/// interface which a type used to instantiate the model must provide.
876///
877/// All of these methods model methods by the same name in the \c
878/// AAResults class. Only differences and specifics to how the
879/// implementations are called are documented here.
880class AAResults::Concept {
881public:
virtual ~Concept() = 0;

/// An update API used internally by the AAResults to provide
/// a handle back to the top level aggregation.
virtual void setAAResults(AAResults *NewAAR) = 0;

//===--------------------------------------------------------------------===//
/// \name Alias Queries
/// @{

/// The main low level interface to the alias analysis implementation.
/// Returns an AliasResult indicating whether the two pointers are aliased to
/// each other. This is the interface that must be implemented by specific
/// alias analysis implementations.
virtual AliasResult alias(const MemoryLocation &LocA,
                          const MemoryLocation &LocB, AAQueryInfo &AAQI) = 0;

/// Checks whether the given location points to constant memory, or if
/// \p OrLocal is true whether it points to a local alloca.
virtual bool pointsToConstantMemory(const MemoryLocation &Loc,
                                    AAQueryInfo &AAQI, bool OrLocal) = 0;

/// @}
//===--------------------------------------------------------------------===//
/// \name Simple mod/ref information
/// @{

/// Get the ModRef info associated with a pointer argument of a callsite. The
/// result's bits are set to indicate the allowed aliasing ModRef kinds. Note
/// that these bits do not necessarily account for the overall behavior of
/// the function, but rather only provide additional per-argument
/// information.
virtual ModRefInfo getArgModRefInfo(const CallBase *Call,
                                    unsigned ArgIdx) = 0;

/// Return the behavior of the given call site.
virtual FunctionModRefBehavior getModRefBehavior(const CallBase *Call) = 0;

/// Return the behavior when calling the given function.
virtual FunctionModRefBehavior getModRefBehavior(const Function *F) = 0;

/// getModRefInfo (for call sites) - Return information about whether
/// a particular call site modifies or reads the specified memory location.
virtual ModRefInfo getModRefInfo(const CallBase *Call,
                                 const MemoryLocation &Loc,
                                 AAQueryInfo &AAQI) = 0;

/// Return information about whether two call sites may refer to the same set
/// of memory locations. See the AA documentation for details:
///   http://llvm.org/docs/AliasAnalysis.html#ModRefInfo
virtual ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2,
                                 AAQueryInfo &AAQI) = 0;

/// @}
936};

938/// A private class template which derives from \c Concept and wraps some other
939/// type.
940///
941/// This models the concept by directly forwarding each interface point to the
942/// wrapped type which must implement a compatible interface. This provides
943/// a type erased binding.
944template <typename AAResultT> class AAResults::Model final : public Concept {
AAResultT &Result;

947public:
explicit Model(AAResultT &Result, AAResults &AAR) : Result(Result) {
  Result.setAAResults(&AAR);
}
~Model() override = default;

void setAAResults(AAResults *NewAAR) override { Result.setAAResults(NewAAR); }

AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB,
                  AAQueryInfo &AAQI) override {
  return Result.alias(LocA, LocB, AAQI);
}

bool pointsToConstantMemory(const MemoryLocation &Loc, AAQueryInfo &AAQI,
                            bool OrLocal) override {
  return Result.pointsToConstantMemory(Loc, AAQI, OrLocal);
}

ModRefInfo getArgModRefInfo(const CallBase *Call, unsigned ArgIdx) override {
  return Result.getArgModRefInfo(Call, ArgIdx);
}

FunctionModRefBehavior getModRefBehavior(const CallBase *Call) override {
  return Result.getModRefBehavior(Call);
}

FunctionModRefBehavior getModRefBehavior(const Function *F) override {
  return Result.getModRefBehavior(F);
}

ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc,
                         AAQueryInfo &AAQI) override {
  return Result.getModRefInfo(Call, Loc, AAQI);
}

ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2,
                         AAQueryInfo &AAQI) override {
  return Result.getModRefInfo(Call1, Call2, AAQI);
}
986};

988/// A CRTP-driven "mixin" base class to help implement the function alias
989/// analysis results concept.
990///
991/// Because of the nature of many alias analysis implementations, they often
992/// only implement a subset of the interface. This base class will attempt to
993/// implement the remaining portions of the interface in terms of simpler forms
994/// of the interface where possible, and otherwise provide conservatively
995/// correct fallback implementations.
996///
997/// Implementors of an alias analysis should derive from this CRTP, and then
998/// override specific methods that they wish to customize. There is no need to
999/// use virtual anywhere, the CRTP base class does static dispatch to the
1000/// derived type passed into it.
1001template <typename DerivedT> class AAResultBase {
// Expose some parts of the interface only to the AAResults::Model
// for wrapping. Specifically, this allows the model to call our
// setAAResults method without exposing it as a fully public API.
friend class AAResults::Model<DerivedT>;

/// A pointer to the AAResults object that this AAResult is
/// aggregated within. May be null if not aggregated.
AAResults *AAR = nullptr;

/// Helper to dispatch calls back through the derived type.
DerivedT &derived() { return static_cast<DerivedT &>(*this); }

/// A setter for the AAResults pointer, which is used to satisfy the
/// AAResults::Model contract.
void setAAResults(AAResults *NewAAR) { AAR = NewAAR; }

1018protected:
/// This proxy class models a common pattern where we delegate to either the
/// top-level \c AAResults aggregation if one is registered, or to the
/// current result if none are registered.
class AAResultsProxy {
  AAResults *AAR;
  DerivedT &CurrentResult;

public:
  AAResultsProxy(AAResults *AAR, DerivedT &CurrentResult)
      : AAR(AAR), CurrentResult(CurrentResult) {}

  AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB,
                    AAQueryInfo &AAQI) {
    return AAR ? AAR->alias(LocA, LocB, AAQI)
               : CurrentResult.alias(LocA, LocB, AAQI);
  }

  bool pointsToConstantMemory(const MemoryLocation &Loc, AAQueryInfo &AAQI,
                              bool OrLocal) {
    return AAR ? AAR->pointsToConstantMemory(Loc, AAQI, OrLocal)
               : CurrentResult.pointsToConstantMemory(Loc, AAQI, OrLocal);
  }

  ModRefInfo getArgModRefInfo(const CallBase *Call, unsigned ArgIdx) {
    return AAR ? AAR->getArgModRefInfo(Call, ArgIdx)
               : CurrentResult.getArgModRefInfo(Call, ArgIdx);
  }

  FunctionModRefBehavior getModRefBehavior(const CallBase *Call) {
    return AAR ? AAR->getModRefBehavior(Call)
               : CurrentResult.getModRefBehavior(Call);
  }

  FunctionModRefBehavior getModRefBehavior(const Function *F) {
    return AAR ? AAR->getModRefBehavior(F) : CurrentResult.getModRefBehavior(F);
  }

  ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc,
                           AAQueryInfo &AAQI) {
    return AAR ? AAR->getModRefInfo(Call, Loc, AAQI)
               : CurrentResult.getModRefInfo(Call, Loc, AAQI);
  }

  ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2,
                           AAQueryInfo &AAQI) {
    return AAR ? AAR->getModRefInfo(Call1, Call2, AAQI)
               : CurrentResult.getModRefInfo(Call1, Call2, AAQI);
  }
};

explicit AAResultBase() = default;

// Provide all the copy and move constructors so that derived types aren't
// constrained.
AAResultBase(const AAResultBase &Arg) {}
AAResultBase(AAResultBase &&Arg) {}

/// Get a proxy for the best AA result set to query at this time.
///
/// When this result is part of a larger aggregation, this will proxy to that
/// aggregation. When this result is used in isolation, it will just delegate
/// back to the derived class's implementation.
///
/// Note that callers of this need to take considerable care to not cause
/// performance problems when they use this routine, in the case of a large
/// number of alias analyses being aggregated, it can be expensive to walk
/// back across the chain.
AAResultsProxy getBestAAResults() { return AAResultsProxy(AAR, derived()); }

1088public:
AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB,
                  AAQueryInfo &AAQI) {
  return MayAlias;
}

bool pointsToConstantMemory(const MemoryLocation &Loc, AAQueryInfo &AAQI,
                            bool OrLocal) {
  return false;
}

ModRefInfo getArgModRefInfo(const CallBase *Call, unsigned ArgIdx) {
  return ModRefInfo::ModRef;
}

FunctionModRefBehavior getModRefBehavior(const CallBase *Call) {
  return FMRB_UnknownModRefBehavior;
}

FunctionModRefBehavior getModRefBehavior(const Function *F) {
  return FMRB_UnknownModRefBehavior;
}

ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc,
                         AAQueryInfo &AAQI) {
  return ModRefInfo::ModRef;
}

ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2,
                         AAQueryInfo &AAQI) {
  return ModRefInfo::ModRef;
}
1120};

1122/// Return true if this pointer is returned by a noalias function.
1123bool isNoAliasCall(const Value *V);

1125/// Return true if this is an argument with the noalias attribute.
1126bool isNoAliasArgument(const Value *V);

1128/// Return true if this pointer refers to a distinct and identifiable object.
1129/// This returns true for:
1130///    Global Variables and Functions (but not Global Aliases)
1131///    Allocas
1132///    ByVal and NoAlias Arguments
1133///    NoAlias returns (e.g. calls to malloc)
1134///
1135bool isIdentifiedObject(const Value *V);

1137/// Return true if V is umabigously identified at the function-level.
1138/// Different IdentifiedFunctionLocals can't alias.
1139/// Further, an IdentifiedFunctionLocal can not alias with any function
1140/// arguments other than itself, which is not necessarily true for
1141/// IdentifiedObjects.
1142bool isIdentifiedFunctionLocal(const Value *V);

1144/// A manager for alias analyses.
1145///
1146/// This class can have analyses registered with it and when run, it will run
1147/// all of them and aggregate their results into single AA results interface
1148/// that dispatches across all of the alias analysis results available.
1149///
1150/// Note that the order in which analyses are registered is very significant.
1151/// That is the order in which the results will be aggregated and queried.
1152///
1153/// This manager effectively wraps the AnalysisManager for registering alias
1154/// analyses. When you register your alias analysis with this manager, it will
1155/// ensure the analysis itself is registered with its AnalysisManager.
1156///
1157/// The result of this analysis is only invalidated if one of the particular
1158/// aggregated AA results end up being invalidated. This removes the need to
1159/// explicitly preserve the results of `AAManager`. Note that analyses should no
1160/// longer be registered once the `AAManager` is run.
1161class AAManager : public AnalysisInfoMixin<AAManager> {
1162public:
using Result = AAResults;

/// Register a specific AA result.
template <typename AnalysisT> void registerFunctionAnalysis() {
  ResultGetters.push_back(&getFunctionAAResultImpl<AnalysisT>);
}

/// Register a specific AA result.
template <typename AnalysisT> void registerModuleAnalysis() {
  ResultGetters.push_back(&getModuleAAResultImpl<AnalysisT>);
}

Result run(Function &F, FunctionAnalysisManager &AM) {
  Result R(AM.getResult<TargetLibraryAnalysis>(F));
  for (auto &Getter : ResultGetters)
    (*Getter)(F, AM, R);
  return R;
}

1182private:
friend AnalysisInfoMixin<AAManager>;

static AnalysisKey Key;

SmallVector<void (*)(Function &F, FunctionAnalysisManager &AM,
                     AAResults &AAResults),
            4> ResultGetters;

template <typename AnalysisT>
static void getFunctionAAResultImpl(Function &F,
                                    FunctionAnalysisManager &AM,
                                    AAResults &AAResults) {
  AAResults.addAAResult(AM.template getResult<AnalysisT>(F));
  AAResults.addAADependencyID(AnalysisT::ID());
}

template <typename AnalysisT>
static void getModuleAAResultImpl(Function &F, FunctionAnalysisManager &AM,
                                  AAResults &AAResults) {
  auto &MAMProxy = AM.getResult<ModuleAnalysisManagerFunctionProxy>(F);
  if (auto *R =
          MAMProxy.template getCachedResult<AnalysisT>(*F.getParent())) {
    AAResults.addAAResult(*R);
    MAMProxy
        .template registerOuterAnalysisInvalidation<AnalysisT, AAManager>();
  }
}
1210};

1212/// A wrapper pass to provide the legacy pass manager access to a suitably
1213/// prepared AAResults object.
1214class AAResultsWrapperPass : public FunctionPass {
std::unique_ptr<AAResults> AAR;

1217public:
static char ID;

AAResultsWrapperPass();

AAResults &getAAResults() { return *AAR; }
const AAResults &getAAResults() const { return *AAR; }

bool runOnFunction(Function &F) override;

void getAnalysisUsage(AnalysisUsage &AU) const override;
1228};

1230/// A wrapper pass for external alias analyses. This just squirrels away the
1231/// callback used to run any analyses and register their results.
1232struct ExternalAAWrapperPass : ImmutablePass {
using CallbackT = std::function<void(Pass &, Function &, AAResults &)>;

CallbackT CB;

static char ID;

ExternalAAWrapperPass();

explicit ExternalAAWrapperPass(CallbackT CB);

void getAnalysisUsage(AnalysisUsage &AU) const override {
  AU.setPreservesAll();
}
1246};

1248FunctionPass *createAAResultsWrapperPass();

1250/// A wrapper pass around a callback which can be used to populate the
1251/// AAResults in the AAResultsWrapperPass from an external AA.
1252///
1253/// The callback provided here will be used each time we prepare an AAResults
1254/// object, and will receive a reference to the function wrapper pass, the
1255/// function, and the AAResults object to populate. This should be used when
1256/// setting up a custom pass pipeline to inject a hook into the AA results.
1257ImmutablePass *createExternalAAWrapperPass(
  std::function<void(Pass &, Function &, AAResults &)> Callback);

1260/// A helper for the legacy pass manager to create a \c AAResults
1261/// object populated to the best of our ability for a particular function when
1262/// inside of a \c ModulePass or a \c CallGraphSCCPass.
1263///
1264/// If a \c ModulePass or a \c CallGraphSCCPass calls \p
1265/// createLegacyPMAAResults, it also needs to call \p addUsedAAAnalyses in \p
1266/// getAnalysisUsage.
1267AAResults createLegacyPMAAResults(Pass &P, Function &F, BasicAAResult &BAR);

1269/// A helper for the legacy pass manager to populate \p AU to add uses to make
1270/// sure the analyses required by \p createLegacyPMAAResults are available.
1271void getAAResultsAnalysisUsage(AnalysisUsage &AU);

1273} // end namespace llvm

1275#endif // LLVM_ANALYSIS_ALIASANALYSIS_H

←

/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/Type.h

1//===- llvm/Type.h - Classes for handling data types ------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the declaration of the Type class.  For more "Type"
10// stuff, look in DerivedTypes.h.
11//
12//===----------------------------------------------------------------------===//

14#ifndef LLVM_IR_TYPE_H
15#define LLVM_IR_TYPE_H

17#include "llvm/ADT/APFloat.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/SmallPtrSet.h"
20#include "llvm/Support/CBindingWrapping.h"
21#include "llvm/Support/Casting.h"
22#include "llvm/Support/Compiler.h"
23#include "llvm/Support/ErrorHandling.h"
24#include "llvm/Support/TypeSize.h"
25#include <cassert>
26#include <cstdint>
27#include <iterator>

29namespace llvm {

31template<class GraphType> struct GraphTraits;
32class IntegerType;
33class LLVMContext;
34class PointerType;
35class raw_ostream;
36class StringRef;

38/// The instances of the Type class are immutable: once they are created,
39/// they are never changed.  Also note that only one instance of a particular
40/// type is ever created.  Thus seeing if two types are equal is a matter of
41/// doing a trivial pointer comparison. To enforce that no two equal instances
42/// are created, Type instances can only be created via static factory methods
43/// in class Type and in derived classes.  Once allocated, Types are never
44/// free'd.
45///
46class Type {
47public:
//===--------------------------------------------------------------------===//
/// Definitions of all of the base types for the Type system.  Based on this
/// value, you can cast to a class defined in DerivedTypes.h.
/// Note: If you add an element to this, you need to add an element to the
/// Type::getPrimitiveType function, or else things will break!
/// Also update LLVMTypeKind and LLVMGetTypeKind () in the C binding.
///
enum TypeID {
  // PrimitiveTypes
  HalfTyID = 0,  ///< 16-bit floating point type
  BFloatTyID,    ///< 16-bit floating point type (7-bit significand)
  FloatTyID,     ///< 32-bit floating point type
  DoubleTyID,    ///< 64-bit floating point type
  X86_FP80TyID,  ///< 80-bit floating point type (X87)
  FP128TyID,     ///< 128-bit floating point type (112-bit significand)
  PPC_FP128TyID, ///< 128-bit floating point type (two 64-bits, PowerPC)
  VoidTyID,      ///< type with no size
  LabelTyID,     ///< Labels
  MetadataTyID,  ///< Metadata
  X86_MMXTyID,   ///< MMX vectors (64 bits, X86 specific)
  TokenTyID,     ///< Tokens

  // Derived types... see DerivedTypes.h file.
  IntegerTyID,       ///< Arbitrary bit width integers
  FunctionTyID,      ///< Functions
  PointerTyID,       ///< Pointers
  StructTyID,        ///< Structures
  ArrayTyID,         ///< Arrays
  FixedVectorTyID,   ///< Fixed width SIMD vector type
  ScalableVectorTyID ///< Scalable SIMD vector type
};

80private:
/// This refers to the LLVMContext in which this type was uniqued.
LLVMContext &Context;

TypeID   ID : 8;            // The current base type of this type.
unsigned SubclassData : 24; // Space for subclasses to store data.
                            // Note that this should be synchronized with
                            // MAX_INT_BITS value in IntegerType class.

89protected:
friend class LLVMContextImpl;

explicit Type(LLVMContext &C, TypeID tid)
  : Context(C), ID(tid), SubclassData(0) {}
~Type() = default;

unsigned getSubclassData() const { return SubclassData; }

void setSubclassData(unsigned val) {
  SubclassData = val;
  // Ensure we don't have any accidental truncation.
  assert(getSubclassData() == val && "Subclass data too large for field")((getSubclassData() == val && "Subclass data too large for field"
) ? static_cast<void> (0) : __assert_fail ("getSubclassData() == val && \"Subclass data too large for field\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/Type.h"
, 101, __PRETTY_FUNCTION__));
}

/// Keeps track of how many Type*'s there are in the ContainedTys list.
unsigned NumContainedTys = 0;

/// A pointer to the array of Types contained by this Type. For example, this
/// includes the arguments of a function type, the elements of a structure,
/// the pointee of a pointer, the element type of an array, etc. This pointer
/// may be 0 for types that don't contain other types (Integer, Double,
/// Float).
Type * const *ContainedTys = nullptr;

114public:
/// Print the current type.
/// Omit the type details if \p NoDetails == true.
/// E.g., let %st = type { i32, i16 }
/// When \p NoDetails is true, we only print %st.
/// Put differently, \p NoDetails prints the type as if
/// inlined with the operands when printing an instruction.
void print(raw_ostream &O, bool IsForDebug = false,
           bool NoDetails = false) const;

void dump() const;

/// Return the LLVMContext in which this type was uniqued.
LLVMContext &getContext() const { return Context; }

//===--------------------------------------------------------------------===//
// Accessors for working with types.
//

/// Return the type id for the type. This will return one of the TypeID enum
/// elements defined above.
TypeID getTypeID() const { return ID; }

/// Return true if this is 'void'.
bool isVoidTy() const { return getTypeID() == VoidTyID; }

/// Return true if this is 'half', a 16-bit IEEE fp type.
bool isHalfTy() const { return getTypeID() == HalfTyID; }

/// Return true if this is 'bfloat', a 16-bit bfloat type.
bool isBFloatTy() const { return getTypeID() == BFloatTyID; }

/// Return true if this is 'float', a 32-bit IEEE fp type.
bool isFloatTy() const { return getTypeID() == FloatTyID; }

/// Return true if this is 'double', a 64-bit IEEE fp type.
bool isDoubleTy() const { return getTypeID() == DoubleTyID; }

/// Return true if this is x86 long double.
bool isX86_FP80Ty() const { return getTypeID() == X86_FP80TyID; }

/// Return true if this is 'fp128'.
bool isFP128Ty() const { return getTypeID() == FP128TyID; }

/// Return true if this is powerpc long double.
bool isPPC_FP128Ty() const { return getTypeID() == PPC_FP128TyID; }

/// Return true if this is one of the six floating-point types
bool isFloatingPointTy() const {
  return getTypeID() == HalfTyID || getTypeID() == BFloatTyID ||
         getTypeID() == FloatTyID || getTypeID() == DoubleTyID ||
         getTypeID() == X86_FP80TyID || getTypeID() == FP128TyID ||
         getTypeID() == PPC_FP128TyID;
}

const fltSemantics &getFltSemantics() const {
  switch (getTypeID()) {
  case HalfTyID: return APFloat::IEEEhalf();
  case BFloatTyID: return APFloat::BFloat();
  case FloatTyID: return APFloat::IEEEsingle();
  case DoubleTyID: return APFloat::IEEEdouble();
  case X86_FP80TyID: return APFloat::x87DoubleExtended();
  case FP128TyID: return APFloat::IEEEquad();
  case PPC_FP128TyID: return APFloat::PPCDoubleDouble();
  default: llvm_unreachable("Invalid floating type")::llvm::llvm_unreachable_internal("Invalid floating type", "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/Type.h"
, 178);
  }
}

/// Return true if this is X86 MMX.
bool isX86_MMXTy() const { return getTypeID() == X86_MMXTyID; }

/// Return true if this is a FP type or a vector of FP.
bool isFPOrFPVectorTy() const { return getScalarType()->isFloatingPointTy(); }

/// Return true if this is 'label'.
bool isLabelTy() const { return getTypeID() == LabelTyID; }

/// Return true if this is 'metadata'.
bool isMetadataTy() const { return getTypeID() == MetadataTyID; }

/// Return true if this is 'token'.
bool isTokenTy() const { return getTypeID() == TokenTyID; }

/// True if this is an instance of IntegerType.
bool isIntegerTy() const { return getTypeID() == IntegerTyID; }

/// Return true if this is an IntegerType of the given width.
bool isIntegerTy(unsigned Bitwidth) const;

/// Return true if this is an integer type or a vector of integer types.
bool isIntOrIntVectorTy() const { return getScalarType()->isIntegerTy(); }

/// Return true if this is an integer type or a vector of integer types of
/// the given width.
bool isIntOrIntVectorTy(unsigned BitWidth) const {
  return getScalarType()->isIntegerTy(BitWidth);
}

/// Return true if this is an integer type or a pointer type.
bool isIntOrPtrTy() const { return isIntegerTy() || isPointerTy(); }

/// True if this is an instance of FunctionType.
bool isFunctionTy() const { return getTypeID() == FunctionTyID; }

/// True if this is an instance of StructType.
bool isStructTy() const { return getTypeID() == StructTyID; }

/// True if this is an instance of ArrayType.
bool isArrayTy() const { return getTypeID() == ArrayTyID; }

/// True if this is an instance of PointerType.
bool isPointerTy() const { return getTypeID() == PointerTyID; }
72
←
Assuming the condition is true→
73
←
Returning the value 1, which participates in a condition later→

/// Return true if this is a pointer type or a vector of pointer types.
bool isPtrOrPtrVectorTy() const { return getScalarType()->isPointerTy(); }

/// True if this is an instance of VectorType.
inline bool isVectorTy() const {
  return getTypeID() == ScalableVectorTyID || getTypeID() == FixedVectorTyID;
}

/// Return true if this type could be converted with a lossless BitCast to
/// type 'Ty'. For example, i8* to i32*. BitCasts are valid for types of the
/// same size only where no re-interpretation of the bits is done.
/// Determine if this type could be losslessly bitcast to Ty
bool canLosslesslyBitCastTo(Type *Ty) const;

/// Return true if this type is empty, that is, it has no elements or all of
/// its elements are empty.
bool isEmptyTy() const;

/// Return true if the type is "first class", meaning it is a valid type for a
/// Value.
bool isFirstClassType() const {
  return getTypeID() != FunctionTyID && getTypeID() != VoidTyID;
}

/// Return true if the type is a valid type for a register in codegen. This
/// includes all first-class types except struct and array types.
bool isSingleValueType() const {
  return isFloatingPointTy() || isX86_MMXTy() || isIntegerTy() ||
         isPointerTy() || isVectorTy();
}

/// Return true if the type is an aggregate type. This means it is valid as
/// the first operand of an insertvalue or extractvalue instruction. This
/// includes struct and array types, but does not include vector types.
bool isAggregateType() const {
  return getTypeID() == StructTyID || getTypeID() == ArrayTyID;
}

/// Return true if it makes sense to take the size of this type. To get the
/// actual size for a particular target, it is reasonable to use the
/// DataLayout subsystem to do this.
bool isSized(SmallPtrSetImpl<Type*> *Visited = nullptr) const {
  // If it's a primitive, it is always sized.
  if (getTypeID() == IntegerTyID || isFloatingPointTy() ||
      getTypeID() == PointerTyID ||
      getTypeID() == X86_MMXTyID)
    return true;
  // If it is not something that can have a size (e.g. a function or label),
  // it doesn't have a size.
  if (getTypeID() != StructTyID && getTypeID() != ArrayTyID && !isVectorTy())
    return false;
  // Otherwise we have to try harder to decide.
  return isSizedDerivedType(Visited);
}

/// Return the basic size of this type if it is a primitive type. These are
/// fixed by LLVM and are not target-dependent.
/// This will return zero if the type does not have a size or is not a
/// primitive type.
///
/// If this is a scalable vector type, the scalable property will be set and
/// the runtime size will be a positive integer multiple of the base size.
///
/// Note that this may not reflect the size of memory allocated for an
/// instance of the type or the number of bytes that are written when an
/// instance of the type is stored to memory. The DataLayout class provides
/// additional query functions to provide this information.
///
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY__attribute__((__pure__));

/// If this is a vector type, return the getPrimitiveSizeInBits value for the
/// element type. Otherwise return the getPrimitiveSizeInBits value for this
/// type.
unsigned getScalarSizeInBits() const LLVM_READONLY__attribute__((__pure__));

/// Return the width of the mantissa of this type. This is only valid on
/// floating-point types. If the FP type does not have a stable mantissa (e.g.
/// ppc long double), this method returns -1.
int getFPMantissaWidth() const;

/// If this is a vector type, return the element type, otherwise return
/// 'this'.
inline Type *getScalarType() const {
  if (isVectorTy())
    return getContainedType(0);
  return const_cast<Type *>(this);
}

//===--------------------------------------------------------------------===//
// Type Iteration support.
//
using subtype_iterator = Type * const *;

subtype_iterator subtype_begin() const { return ContainedTys; }
subtype_iterator subtype_end() const { return &ContainedTys[NumContainedTys];}
ArrayRef<Type*> subtypes() const {
  return makeArrayRef(subtype_begin(), subtype_end());
}

using subtype_reverse_iterator = std::reverse_iterator<subtype_iterator>;

subtype_reverse_iterator subtype_rbegin() const {
  return subtype_reverse_iterator(subtype_end());
}
subtype_reverse_iterator subtype_rend() const {
  return subtype_reverse_iterator(subtype_begin());
}

/// This method is used to implement the type iterator (defined at the end of
/// the file). For derived types, this returns the types 'contained' in the
/// derived type.
Type *getContainedType(unsigned i) const {
  assert(i < NumContainedTys && "Index out of range!")((i < NumContainedTys && "Index out of range!") ? static_cast
<void> (0) : __assert_fail ("i < NumContainedTys && \"Index out of range!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/Type.h"
, 339, __PRETTY_FUNCTION__));
  return ContainedTys[i];
}

/// Return the number of types in the derived type.
unsigned getNumContainedTypes() const { return NumContainedTys; }

//===--------------------------------------------------------------------===//
// Helper methods corresponding to subclass methods.  This forces a cast to
// the specified subclass and calls its accessor.  "getArrayNumElements" (for
// example) is shorthand for cast<ArrayType>(Ty)->getNumElements().  This is
// only intended to cover the core methods that are frequently used, helper
// methods should not be added here.

inline unsigned getIntegerBitWidth() const;

inline Type *getFunctionParamType(unsigned i) const;
inline unsigned getFunctionNumParams() const;
inline bool isFunctionVarArg() const;

inline StringRef getStructName() const;
inline unsigned getStructNumElements() const;
inline Type *getStructElementType(unsigned N) const;

inline uint64_t getArrayNumElements() const;

Type *getArrayElementType() const {
  assert(getTypeID() == ArrayTyID)((getTypeID() == ArrayTyID) ? static_cast<void> (0) : __assert_fail
 ("getTypeID() == ArrayTyID", "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/Type.h"
, 366, __PRETTY_FUNCTION__));
  return ContainedTys[0];
}

Type *getPointerElementType() const {
  assert(getTypeID() == PointerTyID)((getTypeID() == PointerTyID) ? static_cast<void> (0) :
 __assert_fail ("getTypeID() == PointerTyID", "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/Type.h"
, 371, __PRETTY_FUNCTION__));
  return ContainedTys[0];
}

/// Given an integer or vector type, change the lane bitwidth to NewBitwidth,
/// whilst keeping the old number of lanes.
inline Type *getWithNewBitWidth(unsigned NewBitWidth) const;

/// Given scalar/vector integer type, returns a type with elements twice as
/// wide as in the original type. For vectors, preserves element count.
inline Type *getExtendedType() const;

/// Get the address space of this pointer or pointer vector type.
inline unsigned getPointerAddressSpace() const;

//===--------------------------------------------------------------------===//
// Static members exported by the Type class itself.  Useful for getting
// instances of Type.
//

/// Return a type based on an identifier.
static Type *getPrimitiveType(LLVMContext &C, TypeID IDNumber);

//===--------------------------------------------------------------------===//
// These are the builtin types that are always available.
//
static Type *getVoidTy(LLVMContext &C);
static Type *getLabelTy(LLVMContext &C);
static Type *getHalfTy(LLVMContext &C);
static Type *getBFloatTy(LLVMContext &C);
static Type *getFloatTy(LLVMContext &C);
static Type *getDoubleTy(LLVMContext &C);
static Type *getMetadataTy(LLVMContext &C);
static Type *getX86_FP80Ty(LLVMContext &C);
static Type *getFP128Ty(LLVMContext &C);
static Type *getPPC_FP128Ty(LLVMContext &C);
static Type *getX86_MMXTy(LLVMContext &C);
static Type *getTokenTy(LLVMContext &C);
static IntegerType *getIntNTy(LLVMContext &C, unsigned N);
static IntegerType *getInt1Ty(LLVMContext &C);
static IntegerType *getInt8Ty(LLVMContext &C);
static IntegerType *getInt16Ty(LLVMContext &C);
static IntegerType *getInt32Ty(LLVMContext &C);
static IntegerType *getInt64Ty(LLVMContext &C);
static IntegerType *getInt128Ty(LLVMContext &C);
template <typename ScalarTy> static Type *getScalarTy(LLVMContext &C) {
  int noOfBits = sizeof(ScalarTy) * CHAR_BIT8;
  if (std::is_integral<ScalarTy>::value) {
    return (Type*) Type::getIntNTy(C, noOfBits);
  } else if (std::is_floating_point<ScalarTy>::value) {
    switch (noOfBits) {
    case 32:
      return Type::getFloatTy(C);
    case 64:
      return Type::getDoubleTy(C);
    }
  }
  llvm_unreachable("Unsupported type in Type::getScalarTy")::llvm::llvm_unreachable_internal("Unsupported type in Type::getScalarTy"
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/Type.h"
, 428);
}
static Type *getFloatingPointTy(LLVMContext &C, const fltSemantics &S) {
  Type *Ty;
  if (&S == &APFloat::IEEEhalf())
    Ty = Type::getHalfTy(C);
  else if (&S == &APFloat::BFloat())
    Ty = Type::getBFloatTy(C);
  else if (&S == &APFloat::IEEEsingle())
    Ty = Type::getFloatTy(C);
  else if (&S == &APFloat::IEEEdouble())
    Ty = Type::getDoubleTy(C);
  else if (&S == &APFloat::x87DoubleExtended())
    Ty = Type::getX86_FP80Ty(C);
  else if (&S == &APFloat::IEEEquad())
    Ty = Type::getFP128Ty(C);
  else {
    assert(&S == &APFloat::PPCDoubleDouble() && "Unknown FP format")((&S == &APFloat::PPCDoubleDouble() && "Unknown FP format"
) ? static_cast<void> (0) : __assert_fail ("&S == &APFloat::PPCDoubleDouble() && \"Unknown FP format\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include/llvm/IR/Type.h"
, 445, __PRETTY_FUNCTION__));
    Ty = Type::getPPC_FP128Ty(C);
  }
  return Ty;
}

//===--------------------------------------------------------------------===//
// Convenience methods for getting pointer types with one of the above builtin
// types as pointee.
//
static PointerType *getHalfPtrTy(LLVMContext &C, unsigned AS = 0);
static PointerType *getBFloatPtrTy(LLVMContext &C, unsigned AS = 0);
static PointerType *getFloatPtrTy(LLVMContext &C, unsigned AS = 0);
static PointerType *getDoublePtrTy(LLVMContext &C, unsigned AS = 0);
static PointerType *getX86_FP80PtrTy(LLVMContext &C, unsigned AS = 0);
static PointerType *getFP128PtrTy(LLVMContext &C, unsigned AS = 0);
static PointerType *getPPC_FP128PtrTy(LLVMContext &C, unsigned AS = 0);
static PointerType *getX86_MMXPtrTy(LLVMContext &C, unsigned AS = 0);
static PointerType *getIntNPtrTy(LLVMContext &C, unsigned N, unsigned AS = 0);
static PointerType *getInt1PtrTy(LLVMContext &C, unsigned AS = 0);
static PointerType *getInt8PtrTy(LLVMContext &C, unsigned AS = 0);
static PointerType *getInt16PtrTy(LLVMContext &C, unsigned AS = 0);
static PointerType *getInt32PtrTy(LLVMContext &C, unsigned AS = 0);
static PointerType *getInt64PtrTy(LLVMContext &C, unsigned AS = 0);

/// Return a pointer to the current type. This is equivalent to
/// PointerType::get(Foo, AddrSpace).
PointerType *getPointerTo(unsigned AddrSpace = 0) const;

474private:
/// Derived types like structures and arrays are sized iff all of the members
/// of the type are sized as well. Since asking for their size is relatively
/// uncommon, move this operation out-of-line.
bool isSizedDerivedType(SmallPtrSetImpl<Type*> *Visited = nullptr) const;
479};

481// Printing of types.
482inline raw_ostream &operator<<(raw_ostream &OS, const Type &T) {
T.print(OS);
return OS;
485}

487// allow isa<PointerType>(x) to work without DerivedTypes.h included.
488template <> struct isa_impl<PointerType, Type> {
static inline bool doit(const Type &Ty) {
  return Ty.getTypeID() == Type::PointerTyID;
}
492};

494// Create wrappers for C Binding types (see CBindingWrapping.h).
495DEFINE_ISA_CONVERSION_FUNCTIONS(Type, LLVMTypeRef)inline Type *unwrap(LLVMTypeRef P) { return reinterpret_cast<
Type*>(P); } inline LLVMTypeRef wrap(const Type *P) { return
 reinterpret_cast<LLVMTypeRef>(const_cast<Type*>(
P)); } template<typename T> inline T *unwrap(LLVMTypeRef
 P) { return cast<T>(unwrap(P)); }

497/* Specialized opaque type conversions.
*/
499inline Type **unwrap(LLVMTypeRef* Tys) {
return reinterpret_cast<Type**>(Tys);
501}

503inline LLVMTypeRef *wrap(Type **Tys) {
return reinterpret_cast<LLVMTypeRef*>(const_cast<Type**>(Tys));
505}

507} // end namespace llvm

509#endif // LLVM_IR_TYPE_H