/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h

Bug Summary

File:	llvm/include/llvm/IR/IRBuilder.h
Warning:	line 1694, column 30 Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

Show analyzer invocation

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name OMPIRBuilder.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/build-llvm/lib/Frontend/OpenMP -resource-dir /usr/lib/llvm-13/lib/clang/13.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/build-llvm/lib/Frontend/OpenMP -I /build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP -I /build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/build-llvm/include -I /build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../x86_64-linux-gnu/include -internal-isystem /usr/lib/llvm-13/lib/clang/13.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/build-llvm/lib/Frontend/OpenMP -fdebug-prefix-map=/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4=. -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-04-05-202135-9119-1 -x c++ /build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp

/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp

→

1//===- OpenMPIRBuilder.cpp - Builder for LLVM-IR for OpenMP directives ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9///
10/// This file implements the OpenMPIRBuilder class, which is used as a
11/// convenient way to create LLVM instructions for OpenMP directives.
12///
13//===----------------------------------------------------------------------===//

15#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"

17#include "llvm/ADT/StringRef.h"
18#include "llvm/ADT/Triple.h"
19#include "llvm/IR/CFG.h"
20#include "llvm/IR/DebugInfo.h"
21#include "llvm/IR/IRBuilder.h"
22#include "llvm/IR/MDBuilder.h"
23#include "llvm/Support/CommandLine.h"
24#include "llvm/Support/Error.h"
25#include "llvm/Transforms/Utils/BasicBlockUtils.h"
26#include "llvm/Transforms/Utils/CodeExtractor.h"

28#include <sstream>

30#define DEBUG_TYPE"openmp-ir-builder" "openmp-ir-builder"

32using namespace llvm;
33using namespace omp;

35static cl::opt<bool>
  OptimisticAttributes("openmp-ir-builder-optimistic-attributes", cl::Hidden,
                       cl::desc("Use optimistic attributes describing "
                                "'as-if' properties of runtime calls."),
                       cl::init(false));

41void OpenMPIRBuilder::addAttributes(omp::RuntimeFunction FnID, Function &Fn) {
LLVMContext &Ctx = Fn.getContext();

// Get the function's current attributes.
auto Attrs = Fn.getAttributes();
auto FnAttrs = Attrs.getFnAttributes();
auto RetAttrs = Attrs.getRetAttributes();
SmallVector<AttributeSet, 4> ArgAttrs;
for (size_t ArgNo = 0; ArgNo < Fn.arg_size(); ++ArgNo)
  ArgAttrs.emplace_back(Attrs.getParamAttributes(ArgNo));

52#define OMP_ATTRS_SET(VarName, AttrSet) AttributeSet VarName = AttrSet;
53#include "llvm/Frontend/OpenMP/OMPKinds.def"

// Add attributes to the function declaration.
switch (FnID) {
57#define OMP_RTL_ATTRS(Enum, FnAttrSet, RetAttrSet, ArgAttrSets)                \
case Enum:                                                                   \
  FnAttrs = FnAttrs.addAttributes(Ctx, FnAttrSet);                           \
  RetAttrs = RetAttrs.addAttributes(Ctx, RetAttrSet);                        \
  for (size_t ArgNo = 0; ArgNo < ArgAttrSets.size(); ++ArgNo)                \
    ArgAttrs[ArgNo] =                                                        \
        ArgAttrs[ArgNo].addAttributes(Ctx, ArgAttrSets[ArgNo]);              \
  Fn.setAttributes(AttributeList::get(Ctx, FnAttrs, RetAttrs, ArgAttrs));    \
  break;
66#include "llvm/Frontend/OpenMP/OMPKinds.def"
default:
  // Attributes are optional.
  break;
}
71}

73FunctionCallee
74OpenMPIRBuilder::getOrCreateRuntimeFunction(Module &M, RuntimeFunction FnID) {
FunctionType *FnTy = nullptr;
Function *Fn = nullptr;

// Try to find the declation in the module first.
switch (FnID) {
80#define OMP_RTL(Enum, Str, IsVarArg, ReturnType, ...)                          \
case Enum:                                                                   \
  FnTy = FunctionType::get(ReturnType, ArrayRef<Type *>{__VA_ARGS__},        \
                           IsVarArg);                                        \
  Fn = M.getFunction(Str);                                                   \
  break;
86#include "llvm/Frontend/OpenMP/OMPKinds.def"
}

if (!Fn) {
  // Create a new declaration if we need one.
  switch (FnID) {
92#define OMP_RTL(Enum, Str, ...)                                                \
case Enum:                                                                   \
  Fn = Function::Create(FnTy, GlobalValue::ExternalLinkage, Str, M);         \
  break;
96#include "llvm/Frontend/OpenMP/OMPKinds.def"
  }

  // Add information if the runtime function takes a callback function
  if (FnID == OMPRTL___kmpc_fork_call || FnID == OMPRTL___kmpc_fork_teams) {
    if (!Fn->hasMetadata(LLVMContext::MD_callback)) {
      LLVMContext &Ctx = Fn->getContext();
      MDBuilder MDB(Ctx);
      // Annotate the callback behavior of the runtime function:
      //  - The callback callee is argument number 2 (microtask).
      //  - The first two arguments of the callback callee are unknown (-1).
      //  - All variadic arguments to the runtime function are passed to the
      //    callback callee.
      Fn->addMetadata(
          LLVMContext::MD_callback,
          *MDNode::get(Ctx, {MDB.createCallbackEncoding(
                                2, {-1, -1}, /* VarArgsArePassed */ true)}));
    }
  }

  LLVM_DEBUG(dbgs() << "Created OpenMP runtime function " << Fn->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Created OpenMP runtime function "
 << Fn->getName() << " with type " << *Fn
->getFunctionType() << "\n"; } } while (false)
                    << " with type " << *Fn->getFunctionType() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Created OpenMP runtime function "
 << Fn->getName() << " with type " << *Fn
->getFunctionType() << "\n"; } } while (false);
  addAttributes(FnID, *Fn);

} else {
  LLVM_DEBUG(dbgs() << "Found OpenMP runtime function " << Fn->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Found OpenMP runtime function "
 << Fn->getName() << " with type " << *Fn
->getFunctionType() << "\n"; } } while (false)
                    << " with type " << *Fn->getFunctionType() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Found OpenMP runtime function "
 << Fn->getName() << " with type " << *Fn
->getFunctionType() << "\n"; } } while (false);
}

assert(Fn && "Failed to create OpenMP runtime function")((Fn && "Failed to create OpenMP runtime function") ?
 static_cast<void> (0) : __assert_fail ("Fn && \"Failed to create OpenMP runtime function\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 125, __PRETTY_FUNCTION__));

// Cast the function to the expected type if necessary
Constant *C = ConstantExpr::getBitCast(Fn, FnTy->getPointerTo());
return {FnTy, C};
130}

132Function *OpenMPIRBuilder::getOrCreateRuntimeFunctionPtr(RuntimeFunction FnID) {
FunctionCallee RTLFn = getOrCreateRuntimeFunction(M, FnID);
auto *Fn = dyn_cast<llvm::Function>(RTLFn.getCallee());
assert(Fn && "Failed to create OpenMP runtime function pointer")((Fn && "Failed to create OpenMP runtime function pointer"
) ? static_cast<void> (0) : __assert_fail ("Fn && \"Failed to create OpenMP runtime function pointer\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 135, __PRETTY_FUNCTION__));
return Fn;
137}

139void OpenMPIRBuilder::initialize() { initializeTypes(M); }

141void OpenMPIRBuilder::finalize(Function *Fn, bool AllowExtractorSinking) {
SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
SmallVector<BasicBlock *, 32> Blocks;
SmallVector<OutlineInfo, 16> DeferredOutlines;
for (OutlineInfo &OI : OutlineInfos) {
  // Skip functions that have not finalized yet; may happen with nested
  // function generation.
  if (Fn && OI.getFunction() != Fn) {
    DeferredOutlines.push_back(OI);
    continue;
  }

  ParallelRegionBlockSet.clear();
  Blocks.clear();
  OI.collectBlocks(ParallelRegionBlockSet, Blocks);

  Function *OuterFn = OI.getFunction();
  CodeExtractorAnalysisCache CEAC(*OuterFn);
  CodeExtractor Extractor(Blocks, /* DominatorTree */ nullptr,
                          /* AggregateArgs */ false,
                          /* BlockFrequencyInfo */ nullptr,
                          /* BranchProbabilityInfo */ nullptr,
                          /* AssumptionCache */ nullptr,
                          /* AllowVarArgs */ true,
                          /* AllowAlloca */ true,
                          /* Suffix */ ".omp_par");

  LLVM_DEBUG(dbgs() << "Before     outlining: " << *OuterFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Before     outlining: "
 << *OuterFn << "\n"; } } while (false);
  LLVM_DEBUG(dbgs() << "Entry " << OI.EntryBB->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Entry " << OI
.EntryBB->getName() << " Exit: " << OI.ExitBB->
getName() << "\n"; } } while (false)
                    << " Exit: " << OI.ExitBB->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Entry " << OI
.EntryBB->getName() << " Exit: " << OI.ExitBB->
getName() << "\n"; } } while (false);
  assert(Extractor.isEligible() &&((Extractor.isEligible() && "Expected OpenMP outlining to be possible!"
) ? static_cast<void> (0) : __assert_fail ("Extractor.isEligible() && \"Expected OpenMP outlining to be possible!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 172, __PRETTY_FUNCTION__))
         "Expected OpenMP outlining to be possible!")((Extractor.isEligible() && "Expected OpenMP outlining to be possible!"
) ? static_cast<void> (0) : __assert_fail ("Extractor.isEligible() && \"Expected OpenMP outlining to be possible!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 172, __PRETTY_FUNCTION__));

  Function *OutlinedFn = Extractor.extractCodeRegion(CEAC);

  LLVM_DEBUG(dbgs() << "After      outlining: " << *OuterFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "After      outlining: "
 << *OuterFn << "\n"; } } while (false);
  LLVM_DEBUG(dbgs() << "   Outlined function: " << *OutlinedFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "   Outlined function: "
 << *OutlinedFn << "\n"; } } while (false);
  assert(OutlinedFn->getReturnType()->isVoidTy() &&((OutlinedFn->getReturnType()->isVoidTy() && "OpenMP outlined functions should not return a value!"
) ? static_cast<void> (0) : __assert_fail ("OutlinedFn->getReturnType()->isVoidTy() && \"OpenMP outlined functions should not return a value!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 179, __PRETTY_FUNCTION__))
         "OpenMP outlined functions should not return a value!")((OutlinedFn->getReturnType()->isVoidTy() && "OpenMP outlined functions should not return a value!"
) ? static_cast<void> (0) : __assert_fail ("OutlinedFn->getReturnType()->isVoidTy() && \"OpenMP outlined functions should not return a value!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 179, __PRETTY_FUNCTION__));

  // For compability with the clang CG we move the outlined function after the
  // one with the parallel region.
  OutlinedFn->removeFromParent();
  M.getFunctionList().insertAfter(OuterFn->getIterator(), OutlinedFn);

  // Remove the artificial entry introduced by the extractor right away, we
  // made our own entry block after all.
  {
    BasicBlock &ArtificialEntry = OutlinedFn->getEntryBlock();
    assert(ArtificialEntry.getUniqueSuccessor() == OI.EntryBB)((ArtificialEntry.getUniqueSuccessor() == OI.EntryBB) ? static_cast
<void> (0) : __assert_fail ("ArtificialEntry.getUniqueSuccessor() == OI.EntryBB"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 190, __PRETTY_FUNCTION__));
    assert(OI.EntryBB->getUniquePredecessor() == &ArtificialEntry)((OI.EntryBB->getUniquePredecessor() == &ArtificialEntry
) ? static_cast<void> (0) : __assert_fail ("OI.EntryBB->getUniquePredecessor() == &ArtificialEntry"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 191, __PRETTY_FUNCTION__));
    if (AllowExtractorSinking) {
      // Move instructions from the to-be-deleted ArtificialEntry to the entry
      // basic block of the parallel region. CodeExtractor may have sunk
      // allocas/bitcasts for values that are solely used in the outlined
      // region and do not escape.
      assert(!ArtificialEntry.empty() &&((!ArtificialEntry.empty() && "Expected instructions to sink in the outlined region"
) ? static_cast<void> (0) : __assert_fail ("!ArtificialEntry.empty() && \"Expected instructions to sink in the outlined region\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 198, __PRETTY_FUNCTION__))
             "Expected instructions to sink in the outlined region")((!ArtificialEntry.empty() && "Expected instructions to sink in the outlined region"
) ? static_cast<void> (0) : __assert_fail ("!ArtificialEntry.empty() && \"Expected instructions to sink in the outlined region\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 198, __PRETTY_FUNCTION__));
      for (BasicBlock::iterator It = ArtificialEntry.begin(),
                                End = ArtificialEntry.end();
           It != End;) {
        Instruction &I = *It;
        It++;

        if (I.isTerminator())
          continue;

        I.moveBefore(*OI.EntryBB, OI.EntryBB->getFirstInsertionPt());
      }
    }
    OI.EntryBB->moveBefore(&ArtificialEntry);
    ArtificialEntry.eraseFromParent();
  }
  assert(&OutlinedFn->getEntryBlock() == OI.EntryBB)((&OutlinedFn->getEntryBlock() == OI.EntryBB) ? static_cast
<void> (0) : __assert_fail ("&OutlinedFn->getEntryBlock() == OI.EntryBB"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 214, __PRETTY_FUNCTION__));
  assert(OutlinedFn && OutlinedFn->getNumUses() == 1)((OutlinedFn && OutlinedFn->getNumUses() == 1) ? static_cast
<void> (0) : __assert_fail ("OutlinedFn && OutlinedFn->getNumUses() == 1"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 215, __PRETTY_FUNCTION__));

  // Run a user callback, e.g. to add attributes.
  if (OI.PostOutlineCB)
    OI.PostOutlineCB(*OutlinedFn);
}

// Remove work items that have been completed.
OutlineInfos = std::move(DeferredOutlines);
224}

226OpenMPIRBuilder::~OpenMPIRBuilder() {
assert(OutlineInfos.empty() && "There must be no outstanding outlinings")((OutlineInfos.empty() && "There must be no outstanding outlinings"
) ? static_cast<void> (0) : __assert_fail ("OutlineInfos.empty() && \"There must be no outstanding outlinings\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 227, __PRETTY_FUNCTION__));
228}

230Value *OpenMPIRBuilder::getOrCreateIdent(Constant *SrcLocStr,
                                       IdentFlag LocFlags,
                                       unsigned Reserve2Flags) {
// Enable "C-mode".
LocFlags |= OMP_IDENT_FLAG_KMPC;

Value *&Ident =
    IdentMap[{SrcLocStr, uint64_t(LocFlags) << 31 | Reserve2Flags}];
if (!Ident) {
  Constant *I32Null = ConstantInt::getNullValue(Int32);
  Constant *IdentData[] = {
      I32Null, ConstantInt::get(Int32, uint32_t(LocFlags)),
      ConstantInt::get(Int32, Reserve2Flags), I32Null, SrcLocStr};
  Constant *Initializer = ConstantStruct::get(
      cast<StructType>(IdentPtr->getPointerElementType()), IdentData);

  // Look for existing encoding of the location + flags, not needed but
  // minimizes the difference to the existing solution while we transition.
  for (GlobalVariable &GV : M.getGlobalList())
    if (GV.getType() == IdentPtr && GV.hasInitializer())
      if (GV.getInitializer() == Initializer)
        return Ident = &GV;

  auto *GV = new GlobalVariable(M, IdentPtr->getPointerElementType(),
                                /* isConstant = */ true,
                                GlobalValue::PrivateLinkage, Initializer);
  GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
  GV->setAlignment(Align(8));
  Ident = GV;
}
return Builder.CreatePointerCast(Ident, IdentPtr);
261}

263Type *OpenMPIRBuilder::getLanemaskType() {
LLVMContext &Ctx = M.getContext();
Triple triple(M.getTargetTriple());

// This test is adequate until deviceRTL has finer grained lane widths
return triple.isAMDGCN() ? Type::getInt64Ty(Ctx) : Type::getInt32Ty(Ctx);
269}

271Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(StringRef LocStr) {
Constant *&SrcLocStr = SrcLocStrMap[LocStr];
if (!SrcLocStr) {
  Constant *Initializer =
      ConstantDataArray::getString(M.getContext(), LocStr);

  // Look for existing encoding of the location, not needed but minimizes the
  // difference to the existing solution while we transition.
  for (GlobalVariable &GV : M.getGlobalList())
    if (GV.isConstant() && GV.hasInitializer() &&
        GV.getInitializer() == Initializer)
      return SrcLocStr = ConstantExpr::getPointerCast(&GV, Int8Ptr);

  SrcLocStr = Builder.CreateGlobalStringPtr(LocStr, /* Name */ "",
                                            /* AddressSpace */ 0, &M);
}
return SrcLocStr;
288}

290Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(StringRef FunctionName,
                                              StringRef FileName,
                                              unsigned Line,
                                              unsigned Column) {
SmallString<128> Buffer;
Buffer.push_back(';');
Buffer.append(FileName);
Buffer.push_back(';');
Buffer.append(FunctionName);
Buffer.push_back(';');
Buffer.append(std::to_string(Line));
Buffer.push_back(';');
Buffer.append(std::to_string(Column));
Buffer.push_back(';');
Buffer.push_back(';');
return getOrCreateSrcLocStr(Buffer.str());
306}

308Constant *OpenMPIRBuilder::getOrCreateDefaultSrcLocStr() {
return getOrCreateSrcLocStr(";unknown;unknown;0;0;;");
310}

312Constant *
313OpenMPIRBuilder::getOrCreateSrcLocStr(const LocationDescription &Loc) {
DILocation *DIL = Loc.DL.get();
if (!DIL)
  return getOrCreateDefaultSrcLocStr();
StringRef FileName = M.getName();
if (DIFile *DIF = DIL->getFile())
  if (Optional<StringRef> Source = DIF->getSource())
    FileName = *Source;
StringRef Function = DIL->getScope()->getSubprogram()->getName();
Function =
    !Function.empty() ? Function : Loc.IP.getBlock()->getParent()->getName();
return getOrCreateSrcLocStr(Function, FileName, DIL->getLine(),
                            DIL->getColumn());
326}

328Value *OpenMPIRBuilder::getOrCreateThreadID(Value *Ident) {
return Builder.CreateCall(
    getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_global_thread_num), Ident,
    "omp_global_thread_num");
332}

334OpenMPIRBuilder::InsertPointTy
335OpenMPIRBuilder::createBarrier(const LocationDescription &Loc, Directive DK,
                             bool ForceSimpleCall, bool CheckCancelFlag) {
if (!updateToLocation(Loc))
  return Loc.IP;
return emitBarrierImpl(Loc, DK, ForceSimpleCall, CheckCancelFlag);
340}

342OpenMPIRBuilder::InsertPointTy
343OpenMPIRBuilder::emitBarrierImpl(const LocationDescription &Loc, Directive Kind,
                               bool ForceSimpleCall, bool CheckCancelFlag) {
// Build call __kmpc_cancel_barrier(loc, thread_id) or
//            __kmpc_barrier(loc, thread_id);

IdentFlag BarrierLocFlags;
switch (Kind) {
case OMPD_for:
  BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_FOR;
  break;
case OMPD_sections:
  BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SECTIONS;
  break;
case OMPD_single:
  BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SINGLE;
  break;
case OMPD_barrier:
  BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_EXPL;
  break;
default:
  BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL;
  break;
}

Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
Value *Args[] = {getOrCreateIdent(SrcLocStr, BarrierLocFlags),
                 getOrCreateThreadID(getOrCreateIdent(SrcLocStr))};

// If we are in a cancellable parallel region, barriers are cancellation
// points.
// TODO: Check why we would force simple calls or to ignore the cancel flag.
bool UseCancelBarrier =
    !ForceSimpleCall && isLastFinalizationInfoCancellable(OMPD_parallel);

Value *Result =
    Builder.CreateCall(getOrCreateRuntimeFunctionPtr(
                           UseCancelBarrier ? OMPRTL___kmpc_cancel_barrier
                                            : OMPRTL___kmpc_barrier),
                       Args);

if (UseCancelBarrier && CheckCancelFlag)
  emitCancelationCheckImpl(Result, OMPD_parallel);

return Builder.saveIP();
387}

389OpenMPIRBuilder::InsertPointTy
390OpenMPIRBuilder::createCancel(const LocationDescription &Loc,
                            Value *IfCondition,
                            omp::Directive CanceledDirective) {
if (!updateToLocation(Loc))
  return Loc.IP;

// LLVM utilities like blocks with terminators.
auto *UI = Builder.CreateUnreachable();

Instruction *ThenTI = UI, *ElseTI = nullptr;
if (IfCondition)
  SplitBlockAndInsertIfThenElse(IfCondition, UI, &ThenTI, &ElseTI);
Builder.SetInsertPoint(ThenTI);

Value *CancelKind = nullptr;
switch (CanceledDirective) {
406#define OMP_CANCEL_KIND(Enum, Str, DirectiveEnum, Value)                       \
case DirectiveEnum:                                                          \
  CancelKind = Builder.getInt32(Value);                                      \
  break;
410#include "llvm/Frontend/OpenMP/OMPKinds.def"
default:
  llvm_unreachable("Unknown cancel kind!")::llvm::llvm_unreachable_internal("Unknown cancel kind!", "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 412);
}

Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
Value *Ident = getOrCreateIdent(SrcLocStr);
Value *Args[] = {Ident, getOrCreateThreadID(Ident), CancelKind};
Value *Result = Builder.CreateCall(
    getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_cancel), Args);

// The actual cancel logic is shared with others, e.g., cancel_barriers.
emitCancelationCheckImpl(Result, CanceledDirective);

// Update the insertion point and remove the terminator we introduced.
Builder.SetInsertPoint(UI->getParent());
UI->eraseFromParent();

return Builder.saveIP();
429}

431void OpenMPIRBuilder::emitCancelationCheckImpl(
  Value *CancelFlag, omp::Directive CanceledDirective) {
assert(isLastFinalizationInfoCancellable(CanceledDirective) &&((isLastFinalizationInfoCancellable(CanceledDirective) &&
 "Unexpected cancellation!") ? static_cast<void> (0) : __assert_fail
 ("isLastFinalizationInfoCancellable(CanceledDirective) && \"Unexpected cancellation!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 434, __PRETTY_FUNCTION__))
       "Unexpected cancellation!")((isLastFinalizationInfoCancellable(CanceledDirective) &&
 "Unexpected cancellation!") ? static_cast<void> (0) : __assert_fail
 ("isLastFinalizationInfoCancellable(CanceledDirective) && \"Unexpected cancellation!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 434, __PRETTY_FUNCTION__));

// For a cancel barrier we create two new blocks.
BasicBlock *BB = Builder.GetInsertBlock();
BasicBlock *NonCancellationBlock;
if (Builder.GetInsertPoint() == BB->end()) {
  // TODO: This branch will not be needed once we moved to the
  // OpenMPIRBuilder codegen completely.
  NonCancellationBlock = BasicBlock::Create(
      BB->getContext(), BB->getName() + ".cont", BB->getParent());
} else {
  NonCancellationBlock = SplitBlock(BB, &*Builder.GetInsertPoint());
  BB->getTerminator()->eraseFromParent();
  Builder.SetInsertPoint(BB);
}
BasicBlock *CancellationBlock = BasicBlock::Create(
    BB->getContext(), BB->getName() + ".cncl", BB->getParent());

// Jump to them based on the return value.
Value *Cmp = Builder.CreateIsNull(CancelFlag);
Builder.CreateCondBr(Cmp, NonCancellationBlock, CancellationBlock,
                     /* TODO weight */ nullptr, nullptr);

// From the cancellation block we finalize all variables and go to the
// post finalization block that is known to the FiniCB callback.
Builder.SetInsertPoint(CancellationBlock);
auto &FI = FinalizationStack.back();
FI.FiniCB(Builder.saveIP());

// The continuation block is where code generation continues.
Builder.SetInsertPoint(NonCancellationBlock, NonCancellationBlock->begin());
465}

467IRBuilder<>::InsertPoint OpenMPIRBuilder::createParallel(
  const LocationDescription &Loc, InsertPointTy OuterAllocaIP,
  BodyGenCallbackTy BodyGenCB, PrivatizeCallbackTy PrivCB,
  FinalizeCallbackTy FiniCB, Value *IfCondition, Value *NumThreads,
  omp::ProcBindKind ProcBind, bool IsCancellable) {
if (!updateToLocation(Loc))
1
Taking false branch→
  return Loc.IP;

Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
Value *Ident = getOrCreateIdent(SrcLocStr);
Value *ThreadID = getOrCreateThreadID(Ident);

if (NumThreads) {
2
←
Assuming 'NumThreads' is null→
3
←
Taking false branch→
  // Build call __kmpc_push_num_threads(&Ident, global_tid, num_threads)
  Value *Args[] = {
      Ident, ThreadID,
      Builder.CreateIntCast(NumThreads, Int32, /*isSigned*/ false)};
  Builder.CreateCall(
      getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_push_num_threads), Args);
}

if (ProcBind != OMP_PROC_BIND_default) {
4
←
Assuming 'ProcBind' is equal to 'OMP_PROC_BIND_default'→
5
←
Taking false branch→
  // Build call __kmpc_push_proc_bind(&Ident, global_tid, proc_bind)
  Value *Args[] = {
      Ident, ThreadID,
      ConstantInt::get(Int32, unsigned(ProcBind), /*isSigned=*/true)};
  Builder.CreateCall(
      getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_push_proc_bind), Args);
}

BasicBlock *InsertBB = Builder.GetInsertBlock();
Function *OuterFn = InsertBB->getParent();

// Save the outer alloca block because the insertion iterator may get
// invalidated and we still need this later.
BasicBlock *OuterAllocaBlock = OuterAllocaIP.getBlock();

// Vector to remember instructions we used only during the modeling but which
// we want to delete at the end.
SmallVector<Instruction *, 4> ToBeDeleted;

// Change the location to the outer alloca insertion point to create and
// initialize the allocas we pass into the parallel region.
Builder.restoreIP(OuterAllocaIP);
AllocaInst *TIDAddr = Builder.CreateAlloca(Int32, nullptr, "tid.addr");
AllocaInst *ZeroAddr = Builder.CreateAlloca(Int32, nullptr, "zero.addr");

// If there is an if condition we actually use the TIDAddr and ZeroAddr in the
// program, otherwise we only need them for modeling purposes to get the
// associated arguments in the outlined function. In the former case,
// initialize the allocas properly, in the latter case, delete them later.
if (IfCondition) {
6
←
Assuming 'IfCondition' is null→
7
←
Taking false branch→
  Builder.CreateStore(Constant::getNullValue(Int32), TIDAddr);
  Builder.CreateStore(Constant::getNullValue(Int32), ZeroAddr);
} else {
  ToBeDeleted.push_back(TIDAddr);
  ToBeDeleted.push_back(ZeroAddr);
}

// Create an artificial insertion point that will also ensure the blocks we
// are about to split are not degenerated.
auto *UI = new UnreachableInst(Builder.getContext(), InsertBB);

Instruction *ThenTI = UI, *ElseTI = nullptr;
if (IfCondition7.1
'IfCondition' is null
1
'IfCondition' is null
1
'IfCondition' is null
)
8
←
Taking false branch→
  SplitBlockAndInsertIfThenElse(IfCondition, UI, &ThenTI, &ElseTI);

BasicBlock *ThenBB = ThenTI->getParent();
BasicBlock *PRegEntryBB = ThenBB->splitBasicBlock(ThenTI, "omp.par.entry");
BasicBlock *PRegBodyBB =
    PRegEntryBB->splitBasicBlock(ThenTI, "omp.par.region");
BasicBlock *PRegPreFiniBB =
    PRegBodyBB->splitBasicBlock(ThenTI, "omp.par.pre_finalize");
BasicBlock *PRegExitBB =
    PRegPreFiniBB->splitBasicBlock(ThenTI, "omp.par.exit");

auto FiniCBWrapper = [&](InsertPointTy IP) {
  // Hide "open-ended" blocks from the given FiniCB by setting the right jump
  // target to the region exit block.
  if (IP.getBlock()->end() == IP.getPoint()) {
    IRBuilder<>::InsertPointGuard IPG(Builder);
    Builder.restoreIP(IP);
    Instruction *I = Builder.CreateBr(PRegExitBB);
    IP = InsertPointTy(I->getParent(), I->getIterator());
  }
  assert(IP.getBlock()->getTerminator()->getNumSuccessors() == 1 &&((IP.getBlock()->getTerminator()->getNumSuccessors() ==
&& IP.getBlock()->getTerminator()->getSuccessor
(0) == PRegExitBB && "Unexpected insertion point for finalization call!"
) ? static_cast<void> (0) : __assert_fail ("IP.getBlock()->getTerminator()->getNumSuccessors() == 1 && IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB && \"Unexpected insertion point for finalization call!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 554, __PRETTY_FUNCTION__))
         IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB &&((IP.getBlock()->getTerminator()->getNumSuccessors() ==
&& IP.getBlock()->getTerminator()->getSuccessor
(0) == PRegExitBB && "Unexpected insertion point for finalization call!"
) ? static_cast<void> (0) : __assert_fail ("IP.getBlock()->getTerminator()->getNumSuccessors() == 1 && IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB && \"Unexpected insertion point for finalization call!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 554, __PRETTY_FUNCTION__))
         "Unexpected insertion point for finalization call!")((IP.getBlock()->getTerminator()->getNumSuccessors() ==
&& IP.getBlock()->getTerminator()->getSuccessor
(0) == PRegExitBB && "Unexpected insertion point for finalization call!"
) ? static_cast<void> (0) : __assert_fail ("IP.getBlock()->getTerminator()->getNumSuccessors() == 1 && IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB && \"Unexpected insertion point for finalization call!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 554, __PRETTY_FUNCTION__));
  return FiniCB(IP);
};

FinalizationStack.push_back({FiniCBWrapper, OMPD_parallel, IsCancellable});

// Generate the privatization allocas in the block that will become the entry
// of the outlined function.
Builder.SetInsertPoint(PRegEntryBB->getTerminator());
InsertPointTy InnerAllocaIP = Builder.saveIP();

AllocaInst *PrivTIDAddr =
    Builder.CreateAlloca(Int32, nullptr, "tid.addr.local");
Instruction *PrivTID = Builder.CreateLoad(Int32, PrivTIDAddr, "tid");

// Add some fake uses for OpenMP provided arguments.
ToBeDeleted.push_back(Builder.CreateLoad(Int32, TIDAddr, "tid.addr.use"));
Instruction *ZeroAddrUse = Builder.CreateLoad(Int32, ZeroAddr,
                                              "zero.addr.use");
ToBeDeleted.push_back(ZeroAddrUse);

// ThenBB
//   |
//   V
// PRegionEntryBB         <- Privatization allocas are placed here.
//   |
//   V
// PRegionBodyBB          <- BodeGen is invoked here.
//   |
//   V
// PRegPreFiniBB          <- The block we will start finalization from.
//   |
//   V
// PRegionExitBB          <- A common exit to simplify block collection.
//

LLVM_DEBUG(dbgs() << "Before body codegen: " << *OuterFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Before body codegen: "
 << *OuterFn << "\n"; } } while (false);
9
←
Assuming 'DebugFlag' is false→
10
←
Loop condition is false.  Exiting loop→

// Let the caller create the body.
assert(BodyGenCB && "Expected body generation callback!")((BodyGenCB && "Expected body generation callback!") ?
 static_cast<void> (0) : __assert_fail ("BodyGenCB && \"Expected body generation callback!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 593, __PRETTY_FUNCTION__));
11
←
Assuming the condition is true→
12
←
'?' condition is true→
InsertPointTy CodeGenIP(PRegBodyBB, PRegBodyBB->begin());
BodyGenCB(InnerAllocaIP, CodeGenIP, *PRegPreFiniBB);

LLVM_DEBUG(dbgs() << "After  body codegen: " << *OuterFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "After  body codegen: "
 << *OuterFn << "\n"; } } while (false);
13
←
Assuming 'DebugFlag' is false→
14
←
Loop condition is false.  Exiting loop→

FunctionCallee RTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_call);
if (auto *F15.1
'F' is null
1
'F' is null
1
'F' is null
 = dyn_cast<llvm::Function>(RTLFn.getCallee())) {
15
←
Assuming the object is not a 'Function'→
16
←
Taking false branch→
  if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) {
    llvm::LLVMContext &Ctx = F->getContext();
    MDBuilder MDB(Ctx);
    // Annotate the callback behavior of the __kmpc_fork_call:
    //  - The callback callee is argument number 2 (microtask).
    //  - The first two arguments of the callback callee are unknown (-1).
    //  - All variadic arguments to the __kmpc_fork_call are passed to the
    //    callback callee.
    F->addMetadata(
        llvm::LLVMContext::MD_callback,
        *llvm::MDNode::get(
            Ctx, {MDB.createCallbackEncoding(2, {-1, -1},
                                             /* VarArgsArePassed */ true)}));
  }
}

OutlineInfo OI;
OI.PostOutlineCB = [=](Function &OutlinedFn) {
  // Add some known attributes.
  OutlinedFn.addParamAttr(0, Attribute::NoAlias);
  OutlinedFn.addParamAttr(1, Attribute::NoAlias);
  OutlinedFn.addFnAttr(Attribute::NoUnwind);
  OutlinedFn.addFnAttr(Attribute::NoRecurse);

  assert(OutlinedFn.arg_size() >= 2 &&((OutlinedFn.arg_size() >= 2 && "Expected at least tid and bounded tid as arguments"
) ? static_cast<void> (0) : __assert_fail ("OutlinedFn.arg_size() >= 2 && \"Expected at least tid and bounded tid as arguments\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 626, __PRETTY_FUNCTION__))
         "Expected at least tid and bounded tid as arguments")((OutlinedFn.arg_size() >= 2 && "Expected at least tid and bounded tid as arguments"
) ? static_cast<void> (0) : __assert_fail ("OutlinedFn.arg_size() >= 2 && \"Expected at least tid and bounded tid as arguments\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 626, __PRETTY_FUNCTION__));
  unsigned NumCapturedVars =
      OutlinedFn.arg_size() - /* tid & bounded tid */ 2;

  CallInst *CI = cast<CallInst>(OutlinedFn.user_back());
  CI->getParent()->setName("omp_parallel");
  Builder.SetInsertPoint(CI);

  // Build call __kmpc_fork_call(Ident, n, microtask, var1, .., varn);
  Value *ForkCallArgs[] = {
      Ident, Builder.getInt32(NumCapturedVars),
      Builder.CreateBitCast(&OutlinedFn, ParallelTaskPtr)};

  SmallVector<Value *, 16> RealArgs;
  RealArgs.append(std::begin(ForkCallArgs), std::end(ForkCallArgs));
  RealArgs.append(CI->arg_begin() + /* tid & bound tid */ 2, CI->arg_end());

  Builder.CreateCall(RTLFn, RealArgs);

  LLVM_DEBUG(dbgs() << "With fork_call placed: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "With fork_call placed: "
 << *Builder.GetInsertBlock()->getParent() << "\n"
; } } while (false)
                    << *Builder.GetInsertBlock()->getParent() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "With fork_call placed: "
 << *Builder.GetInsertBlock()->getParent() << "\n"
; } } while (false);

  InsertPointTy ExitIP(PRegExitBB, PRegExitBB->end());

  // Initialize the local TID stack location with the argument value.
  Builder.SetInsertPoint(PrivTID);
  Function::arg_iterator OutlinedAI = OutlinedFn.arg_begin();
  Builder.CreateStore(Builder.CreateLoad(Int32, OutlinedAI), PrivTIDAddr);

  // If no "if" clause was present we do not need the call created during
  // outlining, otherwise we reuse it in the serialized parallel region.
  if (!ElseTI) {
    CI->eraseFromParent();
  } else {

    // If an "if" clause was present we are now generating the serialized
    // version into the "else" branch.
    Builder.SetInsertPoint(ElseTI);

    // Build calls __kmpc_serialized_parallel(&Ident, GTid);
    Value *SerializedParallelCallArgs[] = {Ident, ThreadID};
    Builder.CreateCall(
        getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_serialized_parallel),
        SerializedParallelCallArgs);

    // OutlinedFn(&GTid, &zero, CapturedStruct);
    CI->removeFromParent();
    Builder.Insert(CI);

    // __kmpc_end_serialized_parallel(&Ident, GTid);
    Value *EndArgs[] = {Ident, ThreadID};
    Builder.CreateCall(
        getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_serialized_parallel),
        EndArgs);

    LLVM_DEBUG(dbgs() << "With serialized parallel region: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "With serialized parallel region: "
 << *Builder.GetInsertBlock()->getParent() << "\n"
; } } while (false)
                      << *Builder.GetInsertBlock()->getParent() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "With serialized parallel region: "
 << *Builder.GetInsertBlock()->getParent() << "\n"
; } } while (false);
  }

  for (Instruction *I : ToBeDeleted)
    I->eraseFromParent();
};

// Adjust the finalization stack, verify the adjustment, and call the
// finalize function a last time to finalize values between the pre-fini
// block and the exit block if we left the parallel "the normal way".
auto FiniInfo = FinalizationStack.pop_back_val();
(void)FiniInfo;
assert(FiniInfo.DK == OMPD_parallel &&((FiniInfo.DK == OMPD_parallel && "Unexpected finalization stack state!"
) ? static_cast<void> (0) : __assert_fail ("FiniInfo.DK == OMPD_parallel && \"Unexpected finalization stack state!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 695, __PRETTY_FUNCTION__))
17
←
Assuming 'OMPD_parallel' is equal to field 'DK'→
18
←
'?' condition is true→
       "Unexpected finalization stack state!")((FiniInfo.DK == OMPD_parallel && "Unexpected finalization stack state!"
) ? static_cast<void> (0) : __assert_fail ("FiniInfo.DK == OMPD_parallel && \"Unexpected finalization stack state!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 695, __PRETTY_FUNCTION__));

Instruction *PRegPreFiniTI = PRegPreFiniBB->getTerminator();

InsertPointTy PreFiniIP(PRegPreFiniBB, PRegPreFiniTI->getIterator());
FiniCB(PreFiniIP);

OI.EntryBB = PRegEntryBB;
OI.ExitBB = PRegExitBB;

SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
SmallVector<BasicBlock *, 32> Blocks;
OI.collectBlocks(ParallelRegionBlockSet, Blocks);

// Ensure a single exit node for the outlined region by creating one.
// We might have multiple incoming edges to the exit now due to finalizations,
// e.g., cancel calls that cause the control flow to leave the region.
BasicBlock *PRegOutlinedExitBB = PRegExitBB;
PRegExitBB = SplitBlock(PRegExitBB, &*PRegExitBB->getFirstInsertionPt());
PRegOutlinedExitBB->setName("omp.par.outlined.exit");
Blocks.push_back(PRegOutlinedExitBB);

CodeExtractorAnalysisCache CEAC(*OuterFn);
CodeExtractor Extractor(Blocks, /* DominatorTree */ nullptr,
                        /* AggregateArgs */ false,
                        /* BlockFrequencyInfo */ nullptr,
                        /* BranchProbabilityInfo */ nullptr,
                        /* AssumptionCache */ nullptr,
                        /* AllowVarArgs */ true,
                        /* AllowAlloca */ true,
                        /* Suffix */ ".omp_par");

// Find inputs to, outputs from the code region.
BasicBlock *CommonExit = nullptr;
SetVector<Value *> Inputs, Outputs, SinkingCands, HoistingCands;
Extractor.findAllocas(CEAC, SinkingCands, HoistingCands, CommonExit);
Extractor.findInputsOutputs(Inputs, Outputs, SinkingCands);

LLVM_DEBUG(dbgs() << "Before privatization: " << *OuterFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Before privatization: "
 << *OuterFn << "\n"; } } while (false);
19
←
Assuming 'DebugFlag' is false→
20
←
Loop condition is false.  Exiting loop→

FunctionCallee TIDRTLFn =
    getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_global_thread_num);

auto PrivHelper = [&](Value &V) {
  if (&V == TIDAddr || &V == ZeroAddr)
24
←
Assuming the condition is false→
25
←
Assuming the condition is false→
26
←
Taking false branch→
    return;

  SetVector<Use *> Uses;
  for (Use &U : V.uses())
    if (auto *UserI = dyn_cast<Instruction>(U.getUser()))
      if (ParallelRegionBlockSet.count(UserI->getParent()))
        Uses.insert(&U);

  // __kmpc_fork_call expects extra arguments as pointers. If the input
  // already has a pointer type, everything is fine. Otherwise, store the
  // value onto stack and load it back inside the to-be-outlined region. This
  // will ensure only the pointer will be passed to the function.
  // FIXME: if there are more than 15 trailing arguments, they must be
  // additionally packed in a struct.
  Value *Inner = &V;
  if (!V.getType()->isPointerTy()) {
27
←
Taking true branch→
    IRBuilder<>::InsertPointGuard Guard(Builder);
    LLVM_DEBUG(llvm::dbgs() << "Forwarding input as pointer: " << V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { llvm::dbgs() << "Forwarding input as pointer: "
 << V << "\n"; } } while (false);
28
←
Assuming 'DebugFlag' is false→
29
←
Loop condition is false.  Exiting loop→

    Builder.restoreIP(OuterAllocaIP);
    Value *Ptr =
        Builder.CreateAlloca(V.getType(), nullptr, V.getName() + ".reloaded");

    // Store to stack at end of the block that currently branches to the entry
    // block of the to-be-outlined region.
    Builder.SetInsertPoint(InsertBB,
                           InsertBB->getTerminator()->getIterator());
    Builder.CreateStore(&V, Ptr);

    // Load back next to allocations in the to-be-outlined region.
    Builder.restoreIP(InnerAllocaIP);
30
←
Calling 'IRBuilderBase::restoreIP'→
35
←
Returning from 'IRBuilderBase::restoreIP'→
    Inner = Builder.CreateLoad(V.getType(), Ptr);
36
←
Calling 'IRBuilderBase::CreateLoad'→
  }

  Value *ReplacementValue = nullptr;
  CallInst *CI = dyn_cast<CallInst>(&V);
  if (CI && CI->getCalledFunction() == TIDRTLFn.getCallee()) {
    ReplacementValue = PrivTID;
  } else {
    Builder.restoreIP(
        PrivCB(InnerAllocaIP, Builder.saveIP(), V, *Inner, ReplacementValue));
    assert(ReplacementValue &&((ReplacementValue && "Expected copy/create callback to set replacement value!"
) ? static_cast<void> (0) : __assert_fail ("ReplacementValue && \"Expected copy/create callback to set replacement value!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 782, __PRETTY_FUNCTION__))
           "Expected copy/create callback to set replacement value!")((ReplacementValue && "Expected copy/create callback to set replacement value!"
) ? static_cast<void> (0) : __assert_fail ("ReplacementValue && \"Expected copy/create callback to set replacement value!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 782, __PRETTY_FUNCTION__));
    if (ReplacementValue == &V)
      return;
  }

  for (Use *UPtr : Uses)
    UPtr->set(ReplacementValue);
};

// Reset the inner alloca insertion as it will be used for loading the values
// wrapped into pointers before passing them into the to-be-outlined region.
// Configure it to insert immediately after the fake use of zero address so
// that they are available in the generated body and so that the
// OpenMP-related values (thread ID and zero address pointers) remain leading
// in the argument list.
InnerAllocaIP = IRBuilder<>::InsertPoint(
    ZeroAddrUse->getParent(), ZeroAddrUse->getNextNode()->getIterator());

// Reset the outer alloca insertion point to the entry of the relevant block
// in case it was invalidated.
OuterAllocaIP = IRBuilder<>::InsertPoint(
    OuterAllocaBlock, OuterAllocaBlock->getFirstInsertionPt());

for (Value *Input : Inputs) {
  LLVM_DEBUG(dbgs() << "Captured input: " << *Input << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Captured input: " <<
 *Input << "\n"; } } while (false);
21
←
Assuming 'DebugFlag' is false→
22
←
Loop condition is false.  Exiting loop→
  PrivHelper(*Input);
23
←
Calling 'operator()'→
}
LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (Value *Output : Outputs) do {
 if (::llvm::DebugFlag && ::llvm::isCurrentDebugType(
"openmp-ir-builder")) { dbgs() << "Captured output: " <<
 *Output << "\n"; } } while (false); }; } } while (false
)
  for (Value *Output : Outputs)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (Value *Output : Outputs) do {
 if (::llvm::DebugFlag && ::llvm::isCurrentDebugType(
"openmp-ir-builder")) { dbgs() << "Captured output: " <<
 *Output << "\n"; } } while (false); }; } } while (false
)
    LLVM_DEBUG(dbgs() << "Captured output: " << *Output << "\n");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (Value *Output : Outputs) do {
 if (::llvm::DebugFlag && ::llvm::isCurrentDebugType(
"openmp-ir-builder")) { dbgs() << "Captured output: " <<
 *Output << "\n"; } } while (false); }; } } while (false
)
})do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (Value *Output : Outputs) do {
 if (::llvm::DebugFlag && ::llvm::isCurrentDebugType(
"openmp-ir-builder")) { dbgs() << "Captured output: " <<
 *Output << "\n"; } } while (false); }; } } while (false
);
assert(Outputs.empty() &&((Outputs.empty() && "OpenMP outlining should not produce live-out values!"
) ? static_cast<void> (0) : __assert_fail ("Outputs.empty() && \"OpenMP outlining should not produce live-out values!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 814, __PRETTY_FUNCTION__))
       "OpenMP outlining should not produce live-out values!")((Outputs.empty() && "OpenMP outlining should not produce live-out values!"
) ? static_cast<void> (0) : __assert_fail ("Outputs.empty() && \"OpenMP outlining should not produce live-out values!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 814, __PRETTY_FUNCTION__));

LLVM_DEBUG(dbgs() << "After  privatization: " << *OuterFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "After  privatization: "
 << *OuterFn << "\n"; } } while (false);
LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (auto *BB : Blocks) dbgs() <<
 " PBR: " << BB->getName() << "\n"; }; } } while
 (false)
  for (auto *BB : Blocks)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (auto *BB : Blocks) dbgs() <<
 " PBR: " << BB->getName() << "\n"; }; } } while
 (false)
    dbgs() << " PBR: " << BB->getName() << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (auto *BB : Blocks) dbgs() <<
 " PBR: " << BB->getName() << "\n"; }; } } while
 (false)
})do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (auto *BB : Blocks) dbgs() <<
 " PBR: " << BB->getName() << "\n"; }; } } while
 (false);

// Register the outlined info.
addOutlineInfo(std::move(OI));

InsertPointTy AfterIP(UI->getParent(), UI->getParent()->end());
UI->eraseFromParent();

return AfterIP;
829}

831void OpenMPIRBuilder::emitFlush(const LocationDescription &Loc) {
// Build call void __kmpc_flush(ident_t *loc)
Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
Value *Args[] = {getOrCreateIdent(SrcLocStr)};

Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_flush), Args);
837}

839void OpenMPIRBuilder::createFlush(const LocationDescription &Loc) {
if (!updateToLocation(Loc))
  return;
emitFlush(Loc);
843}

845void OpenMPIRBuilder::emitTaskwaitImpl(const LocationDescription &Loc) {
// Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
// global_tid);
Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
Value *Ident = getOrCreateIdent(SrcLocStr);
Value *Args[] = {Ident, getOrCreateThreadID(Ident)};

// Ignore return result until untied tasks are supported.
Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_taskwait),
                   Args);
855}

857void OpenMPIRBuilder::createTaskwait(const LocationDescription &Loc) {
if (!updateToLocation(Loc))
  return;
emitTaskwaitImpl(Loc);
861}

863void OpenMPIRBuilder::emitTaskyieldImpl(const LocationDescription &Loc) {
// Build call __kmpc_omp_taskyield(loc, thread_id, 0);
Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
Value *Ident = getOrCreateIdent(SrcLocStr);
Constant *I32Null = ConstantInt::getNullValue(Int32);
Value *Args[] = {Ident, getOrCreateThreadID(Ident), I32Null};

Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_taskyield),
                   Args);
872}

874void OpenMPIRBuilder::createTaskyield(const LocationDescription &Loc) {
if (!updateToLocation(Loc))
  return;
emitTaskyieldImpl(Loc);
878}

880OpenMPIRBuilder::InsertPointTy
881OpenMPIRBuilder::createMaster(const LocationDescription &Loc,
                            BodyGenCallbackTy BodyGenCB,
                            FinalizeCallbackTy FiniCB) {

if (!updateToLocation(Loc))
  return Loc.IP;

Directive OMPD = Directive::OMPD_master;
Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
Value *Ident = getOrCreateIdent(SrcLocStr);
Value *ThreadId = getOrCreateThreadID(Ident);
Value *Args[] = {Ident, ThreadId};

Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_master);
Instruction *EntryCall = Builder.CreateCall(EntryRTLFn, Args);

Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_master);
Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args);

return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
                            /*Conditional*/ true, /*hasFinalize*/ true);
902}

904CanonicalLoopInfo *OpenMPIRBuilder::createLoopSkeleton(
  DebugLoc DL, Value *TripCount, Function *F, BasicBlock *PreInsertBefore,
  BasicBlock *PostInsertBefore, const Twine &Name) {
Module *M = F->getParent();
LLVMContext &Ctx = M->getContext();
Type *IndVarTy = TripCount->getType();

// Create the basic block structure.
BasicBlock *Preheader =
    BasicBlock::Create(Ctx, "omp_" + Name + ".preheader", F, PreInsertBefore);
BasicBlock *Header =
    BasicBlock::Create(Ctx, "omp_" + Name + ".header", F, PreInsertBefore);
BasicBlock *Cond =
    BasicBlock::Create(Ctx, "omp_" + Name + ".cond", F, PreInsertBefore);
BasicBlock *Body =
    BasicBlock::Create(Ctx, "omp_" + Name + ".body", F, PreInsertBefore);
BasicBlock *Latch =
    BasicBlock::Create(Ctx, "omp_" + Name + ".inc", F, PostInsertBefore);
BasicBlock *Exit =
    BasicBlock::Create(Ctx, "omp_" + Name + ".exit", F, PostInsertBefore);
BasicBlock *After =
    BasicBlock::Create(Ctx, "omp_" + Name + ".after", F, PostInsertBefore);

// Use specified DebugLoc for new instructions.
Builder.SetCurrentDebugLocation(DL);

Builder.SetInsertPoint(Preheader);
Builder.CreateBr(Header);

Builder.SetInsertPoint(Header);
PHINode *IndVarPHI = Builder.CreatePHI(IndVarTy, 2, "omp_" + Name + ".iv");
IndVarPHI->addIncoming(ConstantInt::get(IndVarTy, 0), Preheader);
Builder.CreateBr(Cond);

Builder.SetInsertPoint(Cond);
Value *Cmp =
    Builder.CreateICmpULT(IndVarPHI, TripCount, "omp_" + Name + ".cmp");
Builder.CreateCondBr(Cmp, Body, Exit);

Builder.SetInsertPoint(Body);
Builder.CreateBr(Latch);

Builder.SetInsertPoint(Latch);
Value *Next = Builder.CreateAdd(IndVarPHI, ConstantInt::get(IndVarTy, 1),
                                "omp_" + Name + ".next", /*HasNUW=*/true);
Builder.CreateBr(Header);
IndVarPHI->addIncoming(Next, Latch);

Builder.SetInsertPoint(Exit);
Builder.CreateBr(After);

// Remember and return the canonical control flow.
LoopInfos.emplace_front();
CanonicalLoopInfo *CL = &LoopInfos.front();

CL->Preheader = Preheader;
CL->Header = Header;
CL->Cond = Cond;
CL->Body = Body;
CL->Latch = Latch;
CL->Exit = Exit;
CL->After = After;

CL->IsValid = true;

969#ifndef NDEBUG
CL->assertOK();
971#endif
return CL;
973}

975CanonicalLoopInfo *
976OpenMPIRBuilder::createCanonicalLoop(const LocationDescription &Loc,
                                   LoopBodyGenCallbackTy BodyGenCB,
                                   Value *TripCount, const Twine &Name) {
BasicBlock *BB = Loc.IP.getBlock();
BasicBlock *NextBB = BB->getNextNode();

CanonicalLoopInfo *CL = createLoopSkeleton(Loc.DL, TripCount, BB->getParent(),
                                           NextBB, NextBB, Name);
BasicBlock *After = CL->getAfter();

// If location is not set, don't connect the loop.
if (updateToLocation(Loc)) {
  // Split the loop at the insertion point: Branch to the preheader and move
  // every following instruction to after the loop (the After BB). Also, the
  // new successor is the loop's after block.
  Builder.CreateBr(CL->Preheader);
  After->getInstList().splice(After->begin(), BB->getInstList(),
                              Builder.GetInsertPoint(), BB->end());
  After->replaceSuccessorsPhiUsesWith(BB, After);
}

// Emit the body content. We do it after connecting the loop to the CFG to
// avoid that the callback encounters degenerate BBs.
BodyGenCB(CL->getBodyIP(), CL->getIndVar());

1001#ifndef NDEBUG
CL->assertOK();
1003#endif
return CL;
1005}

1007CanonicalLoopInfo *OpenMPIRBuilder::createCanonicalLoop(
  const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB,
  Value *Start, Value *Stop, Value *Step, bool IsSigned, bool InclusiveStop,
  InsertPointTy ComputeIP, const Twine &Name) {

// Consider the following difficulties (assuming 8-bit signed integers):
//  * Adding \p Step to the loop counter which passes \p Stop may overflow:
//      DO I = 1, 100, 50
///  * A \p Step of INT_MIN cannot not be normalized to a positive direction:
//      DO I = 100, 0, -128

// Start, Stop and Step must be of the same integer type.
auto *IndVarTy = cast<IntegerType>(Start->getType());
assert(IndVarTy == Stop->getType() && "Stop type mismatch")((IndVarTy == Stop->getType() && "Stop type mismatch"
) ? static_cast<void> (0) : __assert_fail ("IndVarTy == Stop->getType() && \"Stop type mismatch\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1020, __PRETTY_FUNCTION__));
assert(IndVarTy == Step->getType() && "Step type mismatch")((IndVarTy == Step->getType() && "Step type mismatch"
) ? static_cast<void> (0) : __assert_fail ("IndVarTy == Step->getType() && \"Step type mismatch\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1021, __PRETTY_FUNCTION__));

LocationDescription ComputeLoc =
    ComputeIP.isSet() ? LocationDescription(ComputeIP, Loc.DL) : Loc;
updateToLocation(ComputeLoc);

ConstantInt *Zero = ConstantInt::get(IndVarTy, 0);
ConstantInt *One = ConstantInt::get(IndVarTy, 1);

// Like Step, but always positive.
Value *Incr = Step;

// Distance between Start and Stop; always positive.
Value *Span;

// Condition whether there are no iterations are executed at all, e.g. because
// UB < LB.
Value *ZeroCmp;

if (IsSigned) {
  // Ensure that increment is positive. If not, negate and invert LB and UB.
  Value *IsNeg = Builder.CreateICmpSLT(Step, Zero);
  Incr = Builder.CreateSelect(IsNeg, Builder.CreateNeg(Step), Step);
  Value *LB = Builder.CreateSelect(IsNeg, Stop, Start);
  Value *UB = Builder.CreateSelect(IsNeg, Start, Stop);
  Span = Builder.CreateSub(UB, LB, "", false, true);
  ZeroCmp = Builder.CreateICmp(
      InclusiveStop ? CmpInst::ICMP_SLT : CmpInst::ICMP_SLE, UB, LB);
} else {
  Span = Builder.CreateSub(Stop, Start, "", true);
  ZeroCmp = Builder.CreateICmp(
      InclusiveStop ? CmpInst::ICMP_ULT : CmpInst::ICMP_ULE, Stop, Start);
}

Value *CountIfLooping;
if (InclusiveStop) {
  CountIfLooping = Builder.CreateAdd(Builder.CreateUDiv(Span, Incr), One);
} else {
  // Avoid incrementing past stop since it could overflow.
  Value *CountIfTwo = Builder.CreateAdd(
      Builder.CreateUDiv(Builder.CreateSub(Span, One), Incr), One);
  Value *OneCmp = Builder.CreateICmp(
      InclusiveStop ? CmpInst::ICMP_ULT : CmpInst::ICMP_ULE, Span, Incr);
  CountIfLooping = Builder.CreateSelect(OneCmp, One, CountIfTwo);
}
Value *TripCount = Builder.CreateSelect(ZeroCmp, Zero, CountIfLooping,
                                        "omp_" + Name + ".tripcount");

auto BodyGen = [=](InsertPointTy CodeGenIP, Value *IV) {
  Builder.restoreIP(CodeGenIP);
  Value *Span = Builder.CreateMul(IV, Step);
  Value *IndVar = Builder.CreateAdd(Span, Start);
  BodyGenCB(Builder.saveIP(), IndVar);
};
LocationDescription LoopLoc = ComputeIP.isSet() ? Loc.IP : Builder.saveIP();
return createCanonicalLoop(LoopLoc, BodyGen, TripCount, Name);
1077}

1079// Returns an LLVM function to call for initializing loop bounds using OpenMP
1080// static scheduling depending on `type`. Only i32 and i64 are supported by the
1081// runtime. Always interpret integers as unsigned similarly to
1082// CanonicalLoopInfo.
1083static FunctionCallee getKmpcForStaticInitForType(Type *Ty, Module &M,
                                                OpenMPIRBuilder &OMPBuilder) {
unsigned Bitwidth = Ty->getIntegerBitWidth();
if (Bitwidth == 32)
  return OMPBuilder.getOrCreateRuntimeFunction(
      M, omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_4u);
if (Bitwidth == 64)
  return OMPBuilder.getOrCreateRuntimeFunction(
      M, omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_8u);
llvm_unreachable("unknown OpenMP loop iterator bitwidth")::llvm::llvm_unreachable_internal("unknown OpenMP loop iterator bitwidth"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1092);
1093}

1095// Sets the number of loop iterations to the given value. This value must be
1096// valid in the condition block (i.e., defined in the preheader) and is
1097// interpreted as an unsigned integer.
1098void setCanonicalLoopTripCount(CanonicalLoopInfo *CLI, Value *TripCount) {
Instruction *CmpI = &CLI->getCond()->front();
assert(isa<CmpInst>(CmpI) && "First inst must compare IV with TripCount")((isa<CmpInst>(CmpI) && "First inst must compare IV with TripCount"
) ? static_cast<void> (0) : __assert_fail ("isa<CmpInst>(CmpI) && \"First inst must compare IV with TripCount\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1100, __PRETTY_FUNCTION__));
CmpI->setOperand(1, TripCount);
CLI->assertOK();
1103}

1105CanonicalLoopInfo *OpenMPIRBuilder::createStaticWorkshareLoop(
  const LocationDescription &Loc, CanonicalLoopInfo *CLI,
  InsertPointTy AllocaIP, bool NeedsBarrier, Value *Chunk) {
// Set up the source location value for OpenMP runtime.
if (!updateToLocation(Loc))
  return nullptr;

Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
Value *SrcLoc = getOrCreateIdent(SrcLocStr);

// Declare useful OpenMP runtime functions.
Value *IV = CLI->getIndVar();
Type *IVTy = IV->getType();
FunctionCallee StaticInit = getKmpcForStaticInitForType(IVTy, M, *this);
FunctionCallee StaticFini =
    getOrCreateRuntimeFunction(M, omp::OMPRTL___kmpc_for_static_fini);

// Allocate space for computed loop bounds as expected by the "init" function.
Builder.restoreIP(AllocaIP);
Type *I32Type = Type::getInt32Ty(M.getContext());
Value *PLastIter = Builder.CreateAlloca(I32Type, nullptr, "p.lastiter");
Value *PLowerBound = Builder.CreateAlloca(IVTy, nullptr, "p.lowerbound");
Value *PUpperBound = Builder.CreateAlloca(IVTy, nullptr, "p.upperbound");
Value *PStride = Builder.CreateAlloca(IVTy, nullptr, "p.stride");

// At the end of the preheader, prepare for calling the "init" function by
// storing the current loop bounds into the allocated space. A canonical loop
// always iterates from 0 to trip-count with step 1. Note that "init" expects
// and produces an inclusive upper bound.
Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());
Constant *Zero = ConstantInt::get(IVTy, 0);
Constant *One = ConstantInt::get(IVTy, 1);
Builder.CreateStore(Zero, PLowerBound);
Value *UpperBound = Builder.CreateSub(CLI->getTripCount(), One);
Builder.CreateStore(UpperBound, PUpperBound);
Builder.CreateStore(One, PStride);

if (!Chunk)
  Chunk = One;

Value *ThreadNum = getOrCreateThreadID(SrcLoc);

// TODO: extract scheduling type and map it to OMP constant. This is curently
// happening in kmp.h and its ilk and needs to be moved to OpenMP.td first.
constexpr int StaticSchedType = 34;
Constant *SchedulingType = ConstantInt::get(I32Type, StaticSchedType);

// Call the "init" function and update the trip count of the loop with the
// value it produced.
Builder.CreateCall(StaticInit,
                   {SrcLoc, ThreadNum, SchedulingType, PLastIter, PLowerBound,
                    PUpperBound, PStride, One, Chunk});
Value *LowerBound = Builder.CreateLoad(IVTy, PLowerBound);
Value *InclusiveUpperBound = Builder.CreateLoad(IVTy, PUpperBound);
Value *TripCountMinusOne = Builder.CreateSub(InclusiveUpperBound, LowerBound);
Value *TripCount = Builder.CreateAdd(TripCountMinusOne, One);
setCanonicalLoopTripCount(CLI, TripCount);

// Update all uses of the induction variable except the one in the condition
// block that compares it with the actual upper bound, and the increment in
// the latch block.
// TODO: this can eventually move to CanonicalLoopInfo or to a new
// CanonicalLoopInfoUpdater interface.
Builder.SetInsertPoint(CLI->getBody(), CLI->getBody()->getFirstInsertionPt());
Value *UpdatedIV = Builder.CreateAdd(IV, LowerBound);
IV->replaceUsesWithIf(UpdatedIV, [&](Use &U) {
  auto *Instr = dyn_cast<Instruction>(U.getUser());
  return !Instr ||
         (Instr->getParent() != CLI->getCond() &&
          Instr->getParent() != CLI->getLatch() && Instr != UpdatedIV);
});

// In the "exit" block, call the "fini" function.
Builder.SetInsertPoint(CLI->getExit(),
                       CLI->getExit()->getTerminator()->getIterator());
Builder.CreateCall(StaticFini, {SrcLoc, ThreadNum});

// Add the barrier if requested.
if (NeedsBarrier)
  createBarrier(LocationDescription(Builder.saveIP(), Loc.DL),
                omp::Directive::OMPD_for, /* ForceSimpleCall */ false,
                /* CheckCancelFlag */ false);

CLI->assertOK();
return CLI;
1190}

1192CanonicalLoopInfo *OpenMPIRBuilder::createWorkshareLoop(
  const LocationDescription &Loc, CanonicalLoopInfo *CLI,
  InsertPointTy AllocaIP, bool NeedsBarrier) {
// Currently only supports static schedules.
return createStaticWorkshareLoop(Loc, CLI, AllocaIP, NeedsBarrier);
1197}

1199/// Make \p Source branch to \p Target.
1200///
1201/// Handles two situations:
1202/// * \p Source already has an unconditional branch.
1203/// * \p Source is a degenerate block (no terminator because the BB is
1204///             the current head of the IR construction).
1205static void redirectTo(BasicBlock *Source, BasicBlock *Target, DebugLoc DL) {
if (Instruction *Term = Source->getTerminator()) {
  auto *Br = cast<BranchInst>(Term);
  assert(!Br->isConditional() &&((!Br->isConditional() && "BB's terminator must be an unconditional branch (or degenerate)"
) ? static_cast<void> (0) : __assert_fail ("!Br->isConditional() && \"BB's terminator must be an unconditional branch (or degenerate)\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1209, __PRETTY_FUNCTION__))
         "BB's terminator must be an unconditional branch (or degenerate)")((!Br->isConditional() && "BB's terminator must be an unconditional branch (or degenerate)"
) ? static_cast<void> (0) : __assert_fail ("!Br->isConditional() && \"BB's terminator must be an unconditional branch (or degenerate)\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1209, __PRETTY_FUNCTION__));
  BasicBlock *Succ = Br->getSuccessor(0);
  Succ->removePredecessor(Source, /*KeepOneInputPHIs=*/true);
  Br->setSuccessor(0, Target);
  return;
}

auto *NewBr = BranchInst::Create(Target, Source);
NewBr->setDebugLoc(DL);
1218}

1220/// Redirect all edges that branch to \p OldTarget to \p NewTarget. That is,
1221/// after this \p OldTarget will be orphaned.
1222static void redirectAllPredecessorsTo(BasicBlock *OldTarget,
                                    BasicBlock *NewTarget, DebugLoc DL) {
for (BasicBlock *Pred : make_early_inc_range(predecessors(OldTarget)))
  redirectTo(Pred, NewTarget, DL);
1226}

1228/// Determine which blocks in \p BBs are reachable from outside and remove the
1229/// ones that are not reachable from the function.
1230static void removeUnusedBlocksFromParent(ArrayRef<BasicBlock *> BBs) {
SmallPtrSet<BasicBlock *, 6> BBsToErase{BBs.begin(), BBs.end()};
auto HasRemainingUses = [&BBsToErase](BasicBlock *BB) {
  for (Use &U : BB->uses()) {
    auto *UseInst = dyn_cast<Instruction>(U.getUser());
    if (!UseInst)
      continue;
    if (BBsToErase.count(UseInst->getParent()))
      continue;
    return true;
  }
  return false;
};

while (true) {
  bool Changed = false;
  for (BasicBlock *BB : make_early_inc_range(BBsToErase)) {
    if (HasRemainingUses(BB)) {
      BBsToErase.erase(BB);
      Changed = true;
    }
  }
  if (!Changed)
    break;
}

SmallVector<BasicBlock *, 7> BBVec(BBsToErase.begin(), BBsToErase.end());
DeleteDeadBlocks(BBVec);
1258}

1260CanonicalLoopInfo *
1261OpenMPIRBuilder::collapseLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
                             InsertPointTy ComputeIP) {
assert(Loops.size() >= 1 && "At least one loop required")((Loops.size() >= 1 && "At least one loop required"
) ? static_cast<void> (0) : __assert_fail ("Loops.size() >= 1 && \"At least one loop required\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1263, __PRETTY_FUNCTION__));
size_t NumLoops = Loops.size();

// Nothing to do if there is already just one loop.
if (NumLoops == 1)
  return Loops.front();

CanonicalLoopInfo *Outermost = Loops.front();
CanonicalLoopInfo *Innermost = Loops.back();
BasicBlock *OrigPreheader = Outermost->getPreheader();
BasicBlock *OrigAfter = Outermost->getAfter();
Function *F = OrigPreheader->getParent();

// Setup the IRBuilder for inserting the trip count computation.
Builder.SetCurrentDebugLocation(DL);
if (ComputeIP.isSet())
  Builder.restoreIP(ComputeIP);
else
  Builder.restoreIP(Outermost->getPreheaderIP());

// Derive the collapsed' loop trip count.
// TODO: Find common/largest indvar type.
Value *CollapsedTripCount = nullptr;
for (CanonicalLoopInfo *L : Loops) {
  Value *OrigTripCount = L->getTripCount();
  if (!CollapsedTripCount) {
    CollapsedTripCount = OrigTripCount;
    continue;
  }

  // TODO: Enable UndefinedSanitizer to diagnose an overflow here.
  CollapsedTripCount = Builder.CreateMul(CollapsedTripCount, OrigTripCount,
                                         {}, /*HasNUW=*/true);
}

// Create the collapsed loop control flow.
CanonicalLoopInfo *Result =
    createLoopSkeleton(DL, CollapsedTripCount, F,
                       OrigPreheader->getNextNode(), OrigAfter, "collapsed");

// Build the collapsed loop body code.
// Start with deriving the input loop induction variables from the collapsed
// one, using a divmod scheme. To preserve the original loops' order, the
// innermost loop use the least significant bits.
Builder.restoreIP(Result->getBodyIP());

Value *Leftover = Result->getIndVar();
SmallVector<Value *> NewIndVars;
NewIndVars.set_size(NumLoops);
for (int i = NumLoops - 1; i >= 1; --i) {
  Value *OrigTripCount = Loops[i]->getTripCount();

  Value *NewIndVar = Builder.CreateURem(Leftover, OrigTripCount);
  NewIndVars[i] = NewIndVar;

  Leftover = Builder.CreateUDiv(Leftover, OrigTripCount);
}
// Outermost loop gets all the remaining bits.
NewIndVars[0] = Leftover;

// Construct the loop body control flow.
// We progressively construct the branch structure following in direction of
// the control flow, from the leading in-between code, the loop nest body, the
// trailing in-between code, and rejoining the collapsed loop's latch.
// ContinueBlock and ContinuePred keep track of the source(s) of next edge. If
// the ContinueBlock is set, continue with that block. If ContinuePred, use
// its predecessors as sources.
BasicBlock *ContinueBlock = Result->getBody();
BasicBlock *ContinuePred = nullptr;
auto ContinueWith = [&ContinueBlock, &ContinuePred, DL](BasicBlock *Dest,
                                                        BasicBlock *NextSrc) {
  if (ContinueBlock)
    redirectTo(ContinueBlock, Dest, DL);
  else
    redirectAllPredecessorsTo(ContinuePred, Dest, DL);

  ContinueBlock = nullptr;
  ContinuePred = NextSrc;
};

// The code before the nested loop of each level.
// Because we are sinking it into the nest, it will be executed more often
// that the original loop. More sophisticated schemes could keep track of what
// the in-between code is and instantiate it only once per thread.
for (size_t i = 0; i < NumLoops - 1; ++i)
  ContinueWith(Loops[i]->getBody(), Loops[i + 1]->getHeader());

// Connect the loop nest body.
ContinueWith(Innermost->getBody(), Innermost->getLatch());

// The code after the nested loop at each level.
for (size_t i = NumLoops - 1; i > 0; --i)
  ContinueWith(Loops[i]->getAfter(), Loops[i - 1]->getLatch());

// Connect the finished loop to the collapsed loop latch.
ContinueWith(Result->getLatch(), nullptr);

// Replace the input loops with the new collapsed loop.
redirectTo(Outermost->getPreheader(), Result->getPreheader(), DL);
redirectTo(Result->getAfter(), Outermost->getAfter(), DL);

// Replace the input loop indvars with the derived ones.
for (size_t i = 0; i < NumLoops; ++i)
  Loops[i]->getIndVar()->replaceAllUsesWith(NewIndVars[i]);

// Remove unused parts of the input loops.
SmallVector<BasicBlock *, 12> OldControlBBs;
OldControlBBs.reserve(6 * Loops.size());
for (CanonicalLoopInfo *Loop : Loops)
  Loop->collectControlBlocks(OldControlBBs);
removeUnusedBlocksFromParent(OldControlBBs);

1375#ifndef NDEBUG
Result->assertOK();
1377#endif
return Result;
1379}

1381std::vector<CanonicalLoopInfo *>
1382OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
                         ArrayRef<Value *> TileSizes) {
assert(TileSizes.size() == Loops.size() &&((TileSizes.size() == Loops.size() && "Must pass as many tile sizes as there are loops"
) ? static_cast<void> (0) : __assert_fail ("TileSizes.size() == Loops.size() && \"Must pass as many tile sizes as there are loops\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1385, __PRETTY_FUNCTION__))
       "Must pass as many tile sizes as there are loops")((TileSizes.size() == Loops.size() && "Must pass as many tile sizes as there are loops"
) ? static_cast<void> (0) : __assert_fail ("TileSizes.size() == Loops.size() && \"Must pass as many tile sizes as there are loops\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1385, __PRETTY_FUNCTION__));
int NumLoops = Loops.size();
assert(NumLoops >= 1 && "At least one loop to tile required")((NumLoops >= 1 && "At least one loop to tile required"
) ? static_cast<void> (0) : __assert_fail ("NumLoops >= 1 && \"At least one loop to tile required\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1387, __PRETTY_FUNCTION__));

CanonicalLoopInfo *OutermostLoop = Loops.front();
CanonicalLoopInfo *InnermostLoop = Loops.back();
Function *F = OutermostLoop->getBody()->getParent();
BasicBlock *InnerEnter = InnermostLoop->getBody();
BasicBlock *InnerLatch = InnermostLoop->getLatch();

// Collect original trip counts and induction variable to be accessible by
// index. Also, the structure of the original loops is not preserved during
// the construction of the tiled loops, so do it before we scavenge the BBs of
// any original CanonicalLoopInfo.
SmallVector<Value *, 4> OrigTripCounts, OrigIndVars;
for (CanonicalLoopInfo *L : Loops) {
  OrigTripCounts.push_back(L->getTripCount());
  OrigIndVars.push_back(L->getIndVar());
}

// Collect the code between loop headers. These may contain SSA definitions
// that are used in the loop nest body. To be usable with in the innermost
// body, these BasicBlocks will be sunk into the loop nest body. That is,
// these instructions may be executed more often than before the tiling.
// TODO: It would be sufficient to only sink them into body of the
// corresponding tile loop.
SmallVector<std::pair<BasicBlock *, BasicBlock *>, 4> InbetweenCode;
for (int i = 0; i < NumLoops - 1; ++i) {
  CanonicalLoopInfo *Surrounding = Loops[i];
  CanonicalLoopInfo *Nested = Loops[i + 1];

  BasicBlock *EnterBB = Surrounding->getBody();
  BasicBlock *ExitBB = Nested->getHeader();
  InbetweenCode.emplace_back(EnterBB, ExitBB);
}

// Compute the trip counts of the floor loops.
Builder.SetCurrentDebugLocation(DL);
Builder.restoreIP(OutermostLoop->getPreheaderIP());
SmallVector<Value *, 4> FloorCount, FloorRems;
for (int i = 0; i < NumLoops; ++i) {
  Value *TileSize = TileSizes[i];
  Value *OrigTripCount = OrigTripCounts[i];
  Type *IVType = OrigTripCount->getType();

  Value *FloorTripCount = Builder.CreateUDiv(OrigTripCount, TileSize);
  Value *FloorTripRem = Builder.CreateURem(OrigTripCount, TileSize);

  // 0 if tripcount divides the tilesize, 1 otherwise.
  // 1 means we need an additional iteration for a partial tile.
  //
  // Unfortunately we cannot just use the roundup-formula
  //   (tripcount + tilesize - 1)/tilesize
  // because the summation might overflow. We do not want introduce undefined
  // behavior when the untiled loop nest did not.
  Value *FloorTripOverflow =
      Builder.CreateICmpNE(FloorTripRem, ConstantInt::get(IVType, 0));

  FloorTripOverflow = Builder.CreateZExt(FloorTripOverflow, IVType);
  FloorTripCount =
      Builder.CreateAdd(FloorTripCount, FloorTripOverflow,
                        "omp_floor" + Twine(i) + ".tripcount", true);

  // Remember some values for later use.
  FloorCount.push_back(FloorTripCount);
  FloorRems.push_back(FloorTripRem);
}

// Generate the new loop nest, from the outermost to the innermost.
std::vector<CanonicalLoopInfo *> Result;
Result.reserve(NumLoops * 2);

// The basic block of the surrounding loop that enters the nest generated
// loop.
BasicBlock *Enter = OutermostLoop->getPreheader();

// The basic block of the surrounding loop where the inner code should
// continue.
BasicBlock *Continue = OutermostLoop->getAfter();

// Where the next loop basic block should be inserted.
BasicBlock *OutroInsertBefore = InnermostLoop->getExit();

auto EmbeddNewLoop =
    [this, DL, F, InnerEnter, &Enter, &Continue, &OutroInsertBefore](
        Value *TripCount, const Twine &Name) -> CanonicalLoopInfo * {
  CanonicalLoopInfo *EmbeddedLoop = createLoopSkeleton(
      DL, TripCount, F, InnerEnter, OutroInsertBefore, Name);
  redirectTo(Enter, EmbeddedLoop->getPreheader(), DL);
  redirectTo(EmbeddedLoop->getAfter(), Continue, DL);

  // Setup the position where the next embedded loop connects to this loop.
  Enter = EmbeddedLoop->getBody();
  Continue = EmbeddedLoop->getLatch();
  OutroInsertBefore = EmbeddedLoop->getLatch();
  return EmbeddedLoop;
};

auto EmbeddNewLoops = [&Result, &EmbeddNewLoop](ArrayRef<Value *> TripCounts,
                                                const Twine &NameBase) {
  for (auto P : enumerate(TripCounts)) {
    CanonicalLoopInfo *EmbeddedLoop =
        EmbeddNewLoop(P.value(), NameBase + Twine(P.index()));
    Result.push_back(EmbeddedLoop);
  }
};

EmbeddNewLoops(FloorCount, "floor");

// Within the innermost floor loop, emit the code that computes the tile
// sizes.
Builder.SetInsertPoint(Enter->getTerminator());
SmallVector<Value *, 4> TileCounts;
for (int i = 0; i < NumLoops; ++i) {
  CanonicalLoopInfo *FloorLoop = Result[i];
  Value *TileSize = TileSizes[i];

  Value *FloorIsEpilogue =
      Builder.CreateICmpEQ(FloorLoop->getIndVar(), FloorCount[i]);
  Value *TileTripCount =
      Builder.CreateSelect(FloorIsEpilogue, FloorRems[i], TileSize);

  TileCounts.push_back(TileTripCount);
}

// Create the tile loops.
EmbeddNewLoops(TileCounts, "tile");

// Insert the inbetween code into the body.
BasicBlock *BodyEnter = Enter;
BasicBlock *BodyEntered = nullptr;
for (std::pair<BasicBlock *, BasicBlock *> P : InbetweenCode) {
  BasicBlock *EnterBB = P.first;
  BasicBlock *ExitBB = P.second;

  if (BodyEnter)
    redirectTo(BodyEnter, EnterBB, DL);
  else
    redirectAllPredecessorsTo(BodyEntered, EnterBB, DL);

  BodyEnter = nullptr;
  BodyEntered = ExitBB;
}

// Append the original loop nest body into the generated loop nest body.
if (BodyEnter)
  redirectTo(BodyEnter, InnerEnter, DL);
else
  redirectAllPredecessorsTo(BodyEntered, InnerEnter, DL);
redirectAllPredecessorsTo(InnerLatch, Continue, DL);

// Replace the original induction variable with an induction variable computed
// from the tile and floor induction variables.
Builder.restoreIP(Result.back()->getBodyIP());
for (int i = 0; i < NumLoops; ++i) {
  CanonicalLoopInfo *FloorLoop = Result[i];
  CanonicalLoopInfo *TileLoop = Result[NumLoops + i];
  Value *OrigIndVar = OrigIndVars[i];
  Value *Size = TileSizes[i];

  Value *Scale =
      Builder.CreateMul(Size, FloorLoop->getIndVar(), {}, /*HasNUW=*/true);
  Value *Shift =
      Builder.CreateAdd(Scale, TileLoop->getIndVar(), {}, /*HasNUW=*/true);
  OrigIndVar->replaceAllUsesWith(Shift);
}

// Remove unused parts of the original loops.
SmallVector<BasicBlock *, 12> OldControlBBs;
OldControlBBs.reserve(6 * Loops.size());
for (CanonicalLoopInfo *Loop : Loops)
  Loop->collectControlBlocks(OldControlBBs);
removeUnusedBlocksFromParent(OldControlBBs);

1559#ifndef NDEBUG
for (CanonicalLoopInfo *GenL : Result)
  GenL->assertOK();
1562#endif
return Result;
1564}

1566OpenMPIRBuilder::InsertPointTy
1567OpenMPIRBuilder::createCopyPrivate(const LocationDescription &Loc,
                                 llvm::Value *BufSize, llvm::Value *CpyBuf,
                                 llvm::Value *CpyFn, llvm::Value *DidIt) {
if (!updateToLocation(Loc))
  return Loc.IP;

Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
Value *Ident = getOrCreateIdent(SrcLocStr);
Value *ThreadId = getOrCreateThreadID(Ident);

llvm::Value *DidItLD = Builder.CreateLoad(Builder.getInt32Ty(), DidIt);

Value *Args[] = {Ident, ThreadId, BufSize, CpyBuf, CpyFn, DidItLD};

Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_copyprivate);
Builder.CreateCall(Fn, Args);

return Builder.saveIP();
1585}

1587OpenMPIRBuilder::InsertPointTy
1588OpenMPIRBuilder::createSingle(const LocationDescription &Loc,
                            BodyGenCallbackTy BodyGenCB,
                            FinalizeCallbackTy FiniCB, llvm::Value *DidIt) {

if (!updateToLocation(Loc))
  return Loc.IP;

// If needed (i.e. not null), initialize `DidIt` with 0
if (DidIt) {
  Builder.CreateStore(Builder.getInt32(0), DidIt);
}

Directive OMPD = Directive::OMPD_single;
Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
Value *Ident = getOrCreateIdent(SrcLocStr);
Value *ThreadId = getOrCreateThreadID(Ident);
Value *Args[] = {Ident, ThreadId};

Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_single);
Instruction *EntryCall = Builder.CreateCall(EntryRTLFn, Args);

Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_single);
Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args);

// generates the following:
// if (__kmpc_single()) {
//		.... single region ...
// 		__kmpc_end_single
// }

return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
                            /*Conditional*/ true, /*hasFinalize*/ true);
1620}

1622OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createCritical(
  const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
  FinalizeCallbackTy FiniCB, StringRef CriticalName, Value *HintInst) {

if (!updateToLocation(Loc))
  return Loc.IP;

Directive OMPD = Directive::OMPD_critical;
Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
Value *Ident = getOrCreateIdent(SrcLocStr);
Value *ThreadId = getOrCreateThreadID(Ident);
Value *LockVar = getOMPCriticalRegionLock(CriticalName);
Value *Args[] = {Ident, ThreadId, LockVar};

SmallVector<llvm::Value *, 4> EnterArgs(std::begin(Args), std::end(Args));
Function *RTFn = nullptr;
if (HintInst) {
  // Add Hint to entry Args and create call
  EnterArgs.push_back(HintInst);
  RTFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_critical_with_hint);
} else {
  RTFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_critical);
}
Instruction *EntryCall = Builder.CreateCall(RTFn, EnterArgs);

Function *ExitRTLFn =
    getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_critical);
Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args);

return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
                            /*Conditional*/ false, /*hasFinalize*/ true);
1653}

1655OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::EmitOMPInlinedRegion(
  Directive OMPD, Instruction *EntryCall, Instruction *ExitCall,
  BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, bool Conditional,
  bool HasFinalize) {

if (HasFinalize)
  FinalizationStack.push_back({FiniCB, OMPD, /*IsCancellable*/ false});

// Create inlined region's entry and body blocks, in preparation
// for conditional creation
BasicBlock *EntryBB = Builder.GetInsertBlock();
Instruction *SplitPos = EntryBB->getTerminator();
if (!isa_and_nonnull<BranchInst>(SplitPos))
  SplitPos = new UnreachableInst(Builder.getContext(), EntryBB);
BasicBlock *ExitBB = EntryBB->splitBasicBlock(SplitPos, "omp_region.end");
BasicBlock *FiniBB =
    EntryBB->splitBasicBlock(EntryBB->getTerminator(), "omp_region.finalize");

Builder.SetInsertPoint(EntryBB->getTerminator());
emitCommonDirectiveEntry(OMPD, EntryCall, ExitBB, Conditional);

// generate body
BodyGenCB(/* AllocaIP */ InsertPointTy(),
          /* CodeGenIP */ Builder.saveIP(), *FiniBB);

// If we didn't emit a branch to FiniBB during body generation, it means
// FiniBB is unreachable (e.g. while(1);). stop generating all the
// unreachable blocks, and remove anything we are not going to use.
auto SkipEmittingRegion = FiniBB->hasNPredecessors(0);
if (SkipEmittingRegion) {
  FiniBB->eraseFromParent();
  ExitCall->eraseFromParent();
  // Discard finalization if we have it.
  if (HasFinalize) {
    assert(!FinalizationStack.empty() &&((!FinalizationStack.empty() && "Unexpected finalization stack state!"
) ? static_cast<void> (0) : __assert_fail ("!FinalizationStack.empty() && \"Unexpected finalization stack state!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1690, __PRETTY_FUNCTION__))
           "Unexpected finalization stack state!")((!FinalizationStack.empty() && "Unexpected finalization stack state!"
) ? static_cast<void> (0) : __assert_fail ("!FinalizationStack.empty() && \"Unexpected finalization stack state!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1690, __PRETTY_FUNCTION__));
    FinalizationStack.pop_back();
  }
} else {
  // emit exit call and do any needed finalization.
  auto FinIP = InsertPointTy(FiniBB, FiniBB->getFirstInsertionPt());
  assert(FiniBB->getTerminator()->getNumSuccessors() == 1 &&((FiniBB->getTerminator()->getNumSuccessors() == 1 &&
 FiniBB->getTerminator()->getSuccessor(0) == ExitBB &&
 "Unexpected control flow graph state!!") ? static_cast<void
> (0) : __assert_fail ("FiniBB->getTerminator()->getNumSuccessors() == 1 && FiniBB->getTerminator()->getSuccessor(0) == ExitBB && \"Unexpected control flow graph state!!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1698, __PRETTY_FUNCTION__))
         FiniBB->getTerminator()->getSuccessor(0) == ExitBB &&((FiniBB->getTerminator()->getNumSuccessors() == 1 &&
 FiniBB->getTerminator()->getSuccessor(0) == ExitBB &&
 "Unexpected control flow graph state!!") ? static_cast<void
> (0) : __assert_fail ("FiniBB->getTerminator()->getNumSuccessors() == 1 && FiniBB->getTerminator()->getSuccessor(0) == ExitBB && \"Unexpected control flow graph state!!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1698, __PRETTY_FUNCTION__))
         "Unexpected control flow graph state!!")((FiniBB->getTerminator()->getNumSuccessors() == 1 &&
 FiniBB->getTerminator()->getSuccessor(0) == ExitBB &&
 "Unexpected control flow graph state!!") ? static_cast<void
> (0) : __assert_fail ("FiniBB->getTerminator()->getNumSuccessors() == 1 && FiniBB->getTerminator()->getSuccessor(0) == ExitBB && \"Unexpected control flow graph state!!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1698, __PRETTY_FUNCTION__));
  emitCommonDirectiveExit(OMPD, FinIP, ExitCall, HasFinalize);
  assert(FiniBB->getUniquePredecessor()->getUniqueSuccessor() == FiniBB &&((FiniBB->getUniquePredecessor()->getUniqueSuccessor() ==
 FiniBB && "Unexpected Control Flow State!") ? static_cast
<void> (0) : __assert_fail ("FiniBB->getUniquePredecessor()->getUniqueSuccessor() == FiniBB && \"Unexpected Control Flow State!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1701, __PRETTY_FUNCTION__))
         "Unexpected Control Flow State!")((FiniBB->getUniquePredecessor()->getUniqueSuccessor() ==
 FiniBB && "Unexpected Control Flow State!") ? static_cast
<void> (0) : __assert_fail ("FiniBB->getUniquePredecessor()->getUniqueSuccessor() == FiniBB && \"Unexpected Control Flow State!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1701, __PRETTY_FUNCTION__));
  MergeBlockIntoPredecessor(FiniBB);
}

// If we are skipping the region of a non conditional, remove the exit
// block, and clear the builder's insertion point.
assert(SplitPos->getParent() == ExitBB &&((SplitPos->getParent() == ExitBB && "Unexpected Insertion point location!"
) ? static_cast<void> (0) : __assert_fail ("SplitPos->getParent() == ExitBB && \"Unexpected Insertion point location!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1708, __PRETTY_FUNCTION__))
       "Unexpected Insertion point location!")((SplitPos->getParent() == ExitBB && "Unexpected Insertion point location!"
) ? static_cast<void> (0) : __assert_fail ("SplitPos->getParent() == ExitBB && \"Unexpected Insertion point location!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1708, __PRETTY_FUNCTION__));
if (!Conditional && SkipEmittingRegion) {
  ExitBB->eraseFromParent();
  Builder.ClearInsertionPoint();
} else {
  auto merged = MergeBlockIntoPredecessor(ExitBB);
  BasicBlock *ExitPredBB = SplitPos->getParent();
  auto InsertBB = merged ? ExitPredBB : ExitBB;
  if (!isa_and_nonnull<BranchInst>(SplitPos))
    SplitPos->eraseFromParent();
  Builder.SetInsertPoint(InsertBB);
}

return Builder.saveIP();
1722}

1724OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveEntry(
  Directive OMPD, Value *EntryCall, BasicBlock *ExitBB, bool Conditional) {

// if nothing to do, Return current insertion point.
if (!Conditional)
  return Builder.saveIP();

BasicBlock *EntryBB = Builder.GetInsertBlock();
Value *CallBool = Builder.CreateIsNotNull(EntryCall);
auto *ThenBB = BasicBlock::Create(M.getContext(), "omp_region.body");
auto *UI = new UnreachableInst(Builder.getContext(), ThenBB);

// Emit thenBB and set the Builder's insertion point there for
// body generation next. Place the block after the current block.
Function *CurFn = EntryBB->getParent();
CurFn->getBasicBlockList().insertAfter(EntryBB->getIterator(), ThenBB);

// Move Entry branch to end of ThenBB, and replace with conditional
// branch (If-stmt)
Instruction *EntryBBTI = EntryBB->getTerminator();
Builder.CreateCondBr(CallBool, ThenBB, ExitBB);
EntryBBTI->removeFromParent();
Builder.SetInsertPoint(UI);
Builder.Insert(EntryBBTI);
UI->eraseFromParent();
Builder.SetInsertPoint(ThenBB->getTerminator());

// return an insertion point to ExitBB.
return IRBuilder<>::InsertPoint(ExitBB, ExitBB->getFirstInsertionPt());
1753}

1755OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveExit(
  omp::Directive OMPD, InsertPointTy FinIP, Instruction *ExitCall,
  bool HasFinalize) {

Builder.restoreIP(FinIP);

// If there is finalization to do, emit it before the exit call
if (HasFinalize) {
  assert(!FinalizationStack.empty() &&((!FinalizationStack.empty() && "Unexpected finalization stack state!"
) ? static_cast<void> (0) : __assert_fail ("!FinalizationStack.empty() && \"Unexpected finalization stack state!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1764, __PRETTY_FUNCTION__))
         "Unexpected finalization stack state!")((!FinalizationStack.empty() && "Unexpected finalization stack state!"
) ? static_cast<void> (0) : __assert_fail ("!FinalizationStack.empty() && \"Unexpected finalization stack state!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1764, __PRETTY_FUNCTION__));

  FinalizationInfo Fi = FinalizationStack.pop_back_val();
  assert(Fi.DK == OMPD && "Unexpected Directive for Finalization call!")((Fi.DK == OMPD && "Unexpected Directive for Finalization call!"
) ? static_cast<void> (0) : __assert_fail ("Fi.DK == OMPD && \"Unexpected Directive for Finalization call!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1767, __PRETTY_FUNCTION__));

  Fi.FiniCB(FinIP);

  BasicBlock *FiniBB = FinIP.getBlock();
  Instruction *FiniBBTI = FiniBB->getTerminator();

  // set Builder IP for call creation
  Builder.SetInsertPoint(FiniBBTI);
}

// place the Exitcall as last instruction before Finalization block terminator
ExitCall->removeFromParent();
Builder.Insert(ExitCall);

return IRBuilder<>::InsertPoint(ExitCall->getParent(),
                                ExitCall->getIterator());
1784}

1786OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createCopyinClauseBlocks(
  InsertPointTy IP, Value *MasterAddr, Value *PrivateAddr,
  llvm::IntegerType *IntPtrTy, bool BranchtoEnd) {
if (!IP.isSet())
  return IP;

IRBuilder<>::InsertPointGuard IPG(Builder);

// creates the following CFG structure
//	   OMP_Entry : (MasterAddr != PrivateAddr)?
//       F     T
//       |      \
//       |     copin.not.master
//       |      /
//       v     /
//   copyin.not.master.end
//		     |
//         v
//   OMP.Entry.Next

BasicBlock *OMP_Entry = IP.getBlock();
Function *CurFn = OMP_Entry->getParent();
BasicBlock *CopyBegin =
    BasicBlock::Create(M.getContext(), "copyin.not.master", CurFn);
BasicBlock *CopyEnd = nullptr;

// If entry block is terminated, split to preserve the branch to following
// basic block (i.e. OMP.Entry.Next), otherwise, leave everything as is.
if (isa_and_nonnull<BranchInst>(OMP_Entry->getTerminator())) {
  CopyEnd = OMP_Entry->splitBasicBlock(OMP_Entry->getTerminator(),
                                       "copyin.not.master.end");
  OMP_Entry->getTerminator()->eraseFromParent();
} else {
  CopyEnd =
      BasicBlock::Create(M.getContext(), "copyin.not.master.end", CurFn);
}

Builder.SetInsertPoint(OMP_Entry);
Value *MasterPtr = Builder.CreatePtrToInt(MasterAddr, IntPtrTy);
Value *PrivatePtr = Builder.CreatePtrToInt(PrivateAddr, IntPtrTy);
Value *cmp = Builder.CreateICmpNE(MasterPtr, PrivatePtr);
Builder.CreateCondBr(cmp, CopyBegin, CopyEnd);

Builder.SetInsertPoint(CopyBegin);
if (BranchtoEnd)
  Builder.SetInsertPoint(Builder.CreateBr(CopyEnd));

return Builder.saveIP();
1834}

1836CallInst *OpenMPIRBuilder::createOMPAlloc(const LocationDescription &Loc,
                                        Value *Size, Value *Allocator,
                                        std::string Name) {
IRBuilder<>::InsertPointGuard IPG(Builder);
Builder.restoreIP(Loc.IP);

Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
Value *Ident = getOrCreateIdent(SrcLocStr);
Value *ThreadId = getOrCreateThreadID(Ident);
Value *Args[] = {ThreadId, Size, Allocator};

Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_alloc);

return Builder.CreateCall(Fn, Args, Name);
1850}

1852CallInst *OpenMPIRBuilder::createOMPFree(const LocationDescription &Loc,
                                       Value *Addr, Value *Allocator,
                                       std::string Name) {
IRBuilder<>::InsertPointGuard IPG(Builder);
Builder.restoreIP(Loc.IP);

Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
Value *Ident = getOrCreateIdent(SrcLocStr);
Value *ThreadId = getOrCreateThreadID(Ident);
Value *Args[] = {ThreadId, Addr, Allocator};
Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_free);
return Builder.CreateCall(Fn, Args, Name);
1864}

1866CallInst *OpenMPIRBuilder::createCachedThreadPrivate(
  const LocationDescription &Loc, llvm::Value *Pointer,
  llvm::ConstantInt *Size, const llvm::Twine &Name) {
IRBuilder<>::InsertPointGuard IPG(Builder);
Builder.restoreIP(Loc.IP);

Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
Value *Ident = getOrCreateIdent(SrcLocStr);
Value *ThreadId = getOrCreateThreadID(Ident);
Constant *ThreadPrivateCache =
    getOrCreateOMPInternalVariable(Int8PtrPtr, Name);
llvm::Value *Args[] = {Ident, ThreadId, Pointer, Size, ThreadPrivateCache};

Function *Fn =
		getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_threadprivate_cached);

return Builder.CreateCall(Fn, Args);
1883}

1885std::string OpenMPIRBuilder::getNameWithSeparators(ArrayRef<StringRef> Parts,
                                                 StringRef FirstSeparator,
                                                 StringRef Separator) {
SmallString<128> Buffer;
llvm::raw_svector_ostream OS(Buffer);
StringRef Sep = FirstSeparator;
for (StringRef Part : Parts) {
  OS << Sep << Part;
  Sep = Separator;
}
return OS.str().str();
1896}

1898Constant *OpenMPIRBuilder::getOrCreateOMPInternalVariable(
  llvm::Type *Ty, const llvm::Twine &Name, unsigned AddressSpace) {
// TODO: Replace the twine arg with stringref to get rid of the conversion
// logic. However This is taken from current implementation in clang as is.
// Since this method is used in many places exclusively for OMP internal use
// we will keep it as is for temporarily until we move all users to the
// builder and then, if possible, fix it everywhere in one go.
SmallString<256> Buffer;
llvm::raw_svector_ostream Out(Buffer);
Out << Name;
StringRef RuntimeName = Out.str();
auto &Elem = *InternalVars.try_emplace(RuntimeName, nullptr).first;
if (Elem.second) {
  assert(Elem.second->getType()->getPointerElementType() == Ty &&((Elem.second->getType()->getPointerElementType() == Ty
 && "OMP internal variable has different type than requested"
) ? static_cast<void> (0) : __assert_fail ("Elem.second->getType()->getPointerElementType() == Ty && \"OMP internal variable has different type than requested\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1912, __PRETTY_FUNCTION__))
         "OMP internal variable has different type than requested")((Elem.second->getType()->getPointerElementType() == Ty
 && "OMP internal variable has different type than requested"
) ? static_cast<void> (0) : __assert_fail ("Elem.second->getType()->getPointerElementType() == Ty && \"OMP internal variable has different type than requested\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1912, __PRETTY_FUNCTION__));
} else {
  // TODO: investigate the appropriate linkage type used for the global
  // variable for possibly changing that to internal or private, or maybe
  // create different versions of the function for different OMP internal
  // variables.
  Elem.second = new llvm::GlobalVariable(
      M, Ty, /*IsConstant*/ false, llvm::GlobalValue::CommonLinkage,
      llvm::Constant::getNullValue(Ty), Elem.first(),
      /*InsertBefore=*/nullptr, llvm::GlobalValue::NotThreadLocal,
      AddressSpace);
}

return Elem.second;
1926}

1928Value *OpenMPIRBuilder::getOMPCriticalRegionLock(StringRef CriticalName) {
std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
std::string Name = getNameWithSeparators({Prefix, "var"}, ".", ".");
return getOrCreateOMPInternalVariable(KmpCriticalNameTy, Name);
1932}

1934// Create all simple and struct types exposed by the runtime and remember
1935// the llvm::PointerTypes of them for easy access later.
1936void OpenMPIRBuilder::initializeTypes(Module &M) {
LLVMContext &Ctx = M.getContext();
StructType *T;
1939#define OMP_TYPE(VarName, InitValue) VarName = InitValue;
1940#define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize)                             \
VarName##Ty = ArrayType::get(ElemTy, ArraySize);                             \
VarName##PtrTy = PointerType::getUnqual(VarName##Ty);
1943#define OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, ...)                  \
VarName = FunctionType::get(ReturnType, {__VA_ARGS__}, IsVarArg);            \
VarName##Ptr = PointerType::getUnqual(VarName);
1946#define OMP_STRUCT_TYPE(VarName, StructName, ...)                              \
T = StructType::getTypeByName(Ctx, StructName);                              \
if (!T)                                                                      \
  T = StructType::create(Ctx, {__VA_ARGS__}, StructName);                    \
VarName = T;                                                                 \
VarName##Ptr = PointerType::getUnqual(T);
1952#include "llvm/Frontend/OpenMP/OMPKinds.def"
1953}

1955void OpenMPIRBuilder::OutlineInfo::collectBlocks(
  SmallPtrSetImpl<BasicBlock *> &BlockSet,
  SmallVectorImpl<BasicBlock *> &BlockVector) {
SmallVector<BasicBlock *, 32> Worklist;
BlockSet.insert(EntryBB);
BlockSet.insert(ExitBB);

Worklist.push_back(EntryBB);
while (!Worklist.empty()) {
  BasicBlock *BB = Worklist.pop_back_val();
  BlockVector.push_back(BB);
  for (BasicBlock *SuccBB : successors(BB))
    if (BlockSet.insert(SuccBB).second)
      Worklist.push_back(SuccBB);
}
1970}

1972void CanonicalLoopInfo::collectControlBlocks(
  SmallVectorImpl<BasicBlock *> &BBs) {
// We only count those BBs as control block for which we do not need to
// reverse the CFG, i.e. not the loop body which can contain arbitrary control
// flow. For consistency, this also means we do not add the Body block, which
// is just the entry to the body code.
BBs.reserve(BBs.size() + 6);
BBs.append({Preheader, Header, Cond, Latch, Exit, After});
1980}

1982void CanonicalLoopInfo::assertOK() const {
1983#ifndef NDEBUG
if (!IsValid)
  return;

// Verify standard control-flow we use for OpenMP loops.
assert(Preheader)((Preheader) ? static_cast<void> (0) : __assert_fail ("Preheader"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1988, __PRETTY_FUNCTION__));
assert(isa<BranchInst>(Preheader->getTerminator()) &&((isa<BranchInst>(Preheader->getTerminator()) &&
 "Preheader must terminate with unconditional branch") ? static_cast
<void> (0) : __assert_fail ("isa<BranchInst>(Preheader->getTerminator()) && \"Preheader must terminate with unconditional branch\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1990, __PRETTY_FUNCTION__))
       "Preheader must terminate with unconditional branch")((isa<BranchInst>(Preheader->getTerminator()) &&
 "Preheader must terminate with unconditional branch") ? static_cast
<void> (0) : __assert_fail ("isa<BranchInst>(Preheader->getTerminator()) && \"Preheader must terminate with unconditional branch\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1990, __PRETTY_FUNCTION__));
assert(Preheader->getSingleSuccessor() == Header &&((Preheader->getSingleSuccessor() == Header && "Preheader must jump to header"
) ? static_cast<void> (0) : __assert_fail ("Preheader->getSingleSuccessor() == Header && \"Preheader must jump to header\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1992, __PRETTY_FUNCTION__))
       "Preheader must jump to header")((Preheader->getSingleSuccessor() == Header && "Preheader must jump to header"
) ? static_cast<void> (0) : __assert_fail ("Preheader->getSingleSuccessor() == Header && \"Preheader must jump to header\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1992, __PRETTY_FUNCTION__));

assert(Header)((Header) ? static_cast<void> (0) : __assert_fail ("Header"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1994, __PRETTY_FUNCTION__));
assert(isa<BranchInst>(Header->getTerminator()) &&((isa<BranchInst>(Header->getTerminator()) &&
 "Header must terminate with unconditional branch") ? static_cast
<void> (0) : __assert_fail ("isa<BranchInst>(Header->getTerminator()) && \"Header must terminate with unconditional branch\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1996, __PRETTY_FUNCTION__))
       "Header must terminate with unconditional branch")((isa<BranchInst>(Header->getTerminator()) &&
 "Header must terminate with unconditional branch") ? static_cast
<void> (0) : __assert_fail ("isa<BranchInst>(Header->getTerminator()) && \"Header must terminate with unconditional branch\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1996, __PRETTY_FUNCTION__));
assert(Header->getSingleSuccessor() == Cond &&((Header->getSingleSuccessor() == Cond && "Header must jump to exiting block"
) ? static_cast<void> (0) : __assert_fail ("Header->getSingleSuccessor() == Cond && \"Header must jump to exiting block\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1998, __PRETTY_FUNCTION__))
       "Header must jump to exiting block")((Header->getSingleSuccessor() == Cond && "Header must jump to exiting block"
) ? static_cast<void> (0) : __assert_fail ("Header->getSingleSuccessor() == Cond && \"Header must jump to exiting block\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 1998, __PRETTY_FUNCTION__));

assert(Cond)((Cond) ? static_cast<void> (0) : __assert_fail ("Cond"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2000, __PRETTY_FUNCTION__));
assert(Cond->getSinglePredecessor() == Header &&((Cond->getSinglePredecessor() == Header && "Exiting block only reachable from header"
) ? static_cast<void> (0) : __assert_fail ("Cond->getSinglePredecessor() == Header && \"Exiting block only reachable from header\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2002, __PRETTY_FUNCTION__))
       "Exiting block only reachable from header")((Cond->getSinglePredecessor() == Header && "Exiting block only reachable from header"
) ? static_cast<void> (0) : __assert_fail ("Cond->getSinglePredecessor() == Header && \"Exiting block only reachable from header\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2002, __PRETTY_FUNCTION__));

assert(isa<BranchInst>(Cond->getTerminator()) &&((isa<BranchInst>(Cond->getTerminator()) && "Exiting block must terminate with conditional branch"
) ? static_cast<void> (0) : __assert_fail ("isa<BranchInst>(Cond->getTerminator()) && \"Exiting block must terminate with conditional branch\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2005, __PRETTY_FUNCTION__))
       "Exiting block must terminate with conditional branch")((isa<BranchInst>(Cond->getTerminator()) && "Exiting block must terminate with conditional branch"
) ? static_cast<void> (0) : __assert_fail ("isa<BranchInst>(Cond->getTerminator()) && \"Exiting block must terminate with conditional branch\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2005, __PRETTY_FUNCTION__));
assert(size(successors(Cond)) == 2 &&((size(successors(Cond)) == 2 && "Exiting block must have two successors"
) ? static_cast<void> (0) : __assert_fail ("size(successors(Cond)) == 2 && \"Exiting block must have two successors\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2007, __PRETTY_FUNCTION__))
       "Exiting block must have two successors")((size(successors(Cond)) == 2 && "Exiting block must have two successors"
) ? static_cast<void> (0) : __assert_fail ("size(successors(Cond)) == 2 && \"Exiting block must have two successors\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2007, __PRETTY_FUNCTION__));
assert(cast<BranchInst>(Cond->getTerminator())->getSuccessor(0) == Body &&((cast<BranchInst>(Cond->getTerminator())->getSuccessor
(0) == Body && "Exiting block's first successor jump to the body"
) ? static_cast<void> (0) : __assert_fail ("cast<BranchInst>(Cond->getTerminator())->getSuccessor(0) == Body && \"Exiting block's first successor jump to the body\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2009, __PRETTY_FUNCTION__))
       "Exiting block's first successor jump to the body")((cast<BranchInst>(Cond->getTerminator())->getSuccessor
(0) == Body && "Exiting block's first successor jump to the body"
) ? static_cast<void> (0) : __assert_fail ("cast<BranchInst>(Cond->getTerminator())->getSuccessor(0) == Body && \"Exiting block's first successor jump to the body\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2009, __PRETTY_FUNCTION__));
assert(cast<BranchInst>(Cond->getTerminator())->getSuccessor(1) == Exit &&((cast<BranchInst>(Cond->getTerminator())->getSuccessor
(1) == Exit && "Exiting block's second successor must exit the loop"
) ? static_cast<void> (0) : __assert_fail ("cast<BranchInst>(Cond->getTerminator())->getSuccessor(1) == Exit && \"Exiting block's second successor must exit the loop\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2011, __PRETTY_FUNCTION__))
       "Exiting block's second successor must exit the loop")((cast<BranchInst>(Cond->getTerminator())->getSuccessor
(1) == Exit && "Exiting block's second successor must exit the loop"
) ? static_cast<void> (0) : __assert_fail ("cast<BranchInst>(Cond->getTerminator())->getSuccessor(1) == Exit && \"Exiting block's second successor must exit the loop\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2011, __PRETTY_FUNCTION__));

assert(Body)((Body) ? static_cast<void> (0) : __assert_fail ("Body"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2013, __PRETTY_FUNCTION__));
assert(Body->getSinglePredecessor() == Cond &&((Body->getSinglePredecessor() == Cond && "Body only reachable from exiting block"
) ? static_cast<void> (0) : __assert_fail ("Body->getSinglePredecessor() == Cond && \"Body only reachable from exiting block\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2015, __PRETTY_FUNCTION__))
       "Body only reachable from exiting block")((Body->getSinglePredecessor() == Cond && "Body only reachable from exiting block"
) ? static_cast<void> (0) : __assert_fail ("Body->getSinglePredecessor() == Cond && \"Body only reachable from exiting block\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2015, __PRETTY_FUNCTION__));
assert(!isa<PHINode>(Body->front()))((!isa<PHINode>(Body->front())) ? static_cast<void
> (0) : __assert_fail ("!isa<PHINode>(Body->front())"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2016, __PRETTY_FUNCTION__));

assert(Latch)((Latch) ? static_cast<void> (0) : __assert_fail ("Latch"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2018, __PRETTY_FUNCTION__));
assert(isa<BranchInst>(Latch->getTerminator()) &&((isa<BranchInst>(Latch->getTerminator()) &&
 "Latch must terminate with unconditional branch") ? static_cast
<void> (0) : __assert_fail ("isa<BranchInst>(Latch->getTerminator()) && \"Latch must terminate with unconditional branch\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2020, __PRETTY_FUNCTION__))
       "Latch must terminate with unconditional branch")((isa<BranchInst>(Latch->getTerminator()) &&
 "Latch must terminate with unconditional branch") ? static_cast
<void> (0) : __assert_fail ("isa<BranchInst>(Latch->getTerminator()) && \"Latch must terminate with unconditional branch\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2020, __PRETTY_FUNCTION__));
assert(Latch->getSingleSuccessor() == Header && "Latch must jump to header")((Latch->getSingleSuccessor() == Header && "Latch must jump to header"
) ? static_cast<void> (0) : __assert_fail ("Latch->getSingleSuccessor() == Header && \"Latch must jump to header\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2021, __PRETTY_FUNCTION__));
// TODO: To support simple redirecting of the end of the body code that has
// multiple; introduce another auxiliary basic block like preheader and after.
assert(Latch->getSinglePredecessor() != nullptr)((Latch->getSinglePredecessor() != nullptr) ? static_cast<
void> (0) : __assert_fail ("Latch->getSinglePredecessor() != nullptr"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2024, __PRETTY_FUNCTION__));
assert(!isa<PHINode>(Latch->front()))((!isa<PHINode>(Latch->front())) ? static_cast<void
> (0) : __assert_fail ("!isa<PHINode>(Latch->front())"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2025, __PRETTY_FUNCTION__));

assert(Exit)((Exit) ? static_cast<void> (0) : __assert_fail ("Exit"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2027, __PRETTY_FUNCTION__));
assert(isa<BranchInst>(Exit->getTerminator()) &&((isa<BranchInst>(Exit->getTerminator()) && "Exit block must terminate with unconditional branch"
) ? static_cast<void> (0) : __assert_fail ("isa<BranchInst>(Exit->getTerminator()) && \"Exit block must terminate with unconditional branch\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2029, __PRETTY_FUNCTION__))
       "Exit block must terminate with unconditional branch")((isa<BranchInst>(Exit->getTerminator()) && "Exit block must terminate with unconditional branch"
) ? static_cast<void> (0) : __assert_fail ("isa<BranchInst>(Exit->getTerminator()) && \"Exit block must terminate with unconditional branch\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2029, __PRETTY_FUNCTION__));
assert(Exit->getSingleSuccessor() == After &&((Exit->getSingleSuccessor() == After && "Exit block must jump to after block"
) ? static_cast<void> (0) : __assert_fail ("Exit->getSingleSuccessor() == After && \"Exit block must jump to after block\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2031, __PRETTY_FUNCTION__))
       "Exit block must jump to after block")((Exit->getSingleSuccessor() == After && "Exit block must jump to after block"
) ? static_cast<void> (0) : __assert_fail ("Exit->getSingleSuccessor() == After && \"Exit block must jump to after block\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2031, __PRETTY_FUNCTION__));

assert(After)((After) ? static_cast<void> (0) : __assert_fail ("After"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2033, __PRETTY_FUNCTION__));
assert(After->getSinglePredecessor() == Exit &&((After->getSinglePredecessor() == Exit && "After block only reachable from exit block"
) ? static_cast<void> (0) : __assert_fail ("After->getSinglePredecessor() == Exit && \"After block only reachable from exit block\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2035, __PRETTY_FUNCTION__))
       "After block only reachable from exit block")((After->getSinglePredecessor() == Exit && "After block only reachable from exit block"
) ? static_cast<void> (0) : __assert_fail ("After->getSinglePredecessor() == Exit && \"After block only reachable from exit block\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2035, __PRETTY_FUNCTION__));
assert(After->empty() || !isa<PHINode>(After->front()))((After->empty() || !isa<PHINode>(After->front())
) ? static_cast<void> (0) : __assert_fail ("After->empty() || !isa<PHINode>(After->front())"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2036, __PRETTY_FUNCTION__));

Instruction *IndVar = getIndVar();
assert(IndVar && "Canonical induction variable not found?")((IndVar && "Canonical induction variable not found?"
) ? static_cast<void> (0) : __assert_fail ("IndVar && \"Canonical induction variable not found?\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2039, __PRETTY_FUNCTION__));
assert(isa<IntegerType>(IndVar->getType()) &&((isa<IntegerType>(IndVar->getType()) && "Induction variable must be an integer"
) ? static_cast<void> (0) : __assert_fail ("isa<IntegerType>(IndVar->getType()) && \"Induction variable must be an integer\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2041, __PRETTY_FUNCTION__))
       "Induction variable must be an integer")((isa<IntegerType>(IndVar->getType()) && "Induction variable must be an integer"
) ? static_cast<void> (0) : __assert_fail ("isa<IntegerType>(IndVar->getType()) && \"Induction variable must be an integer\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2041, __PRETTY_FUNCTION__));
assert(cast<PHINode>(IndVar)->getParent() == Header &&((cast<PHINode>(IndVar)->getParent() == Header &&
 "Induction variable must be a PHI in the loop header") ? static_cast
<void> (0) : __assert_fail ("cast<PHINode>(IndVar)->getParent() == Header && \"Induction variable must be a PHI in the loop header\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2043, __PRETTY_FUNCTION__))
       "Induction variable must be a PHI in the loop header")((cast<PHINode>(IndVar)->getParent() == Header &&
 "Induction variable must be a PHI in the loop header") ? static_cast
<void> (0) : __assert_fail ("cast<PHINode>(IndVar)->getParent() == Header && \"Induction variable must be a PHI in the loop header\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2043, __PRETTY_FUNCTION__));
assert(cast<PHINode>(IndVar)->getIncomingBlock(0) == Preheader)((cast<PHINode>(IndVar)->getIncomingBlock(0) == Preheader
) ? static_cast<void> (0) : __assert_fail ("cast<PHINode>(IndVar)->getIncomingBlock(0) == Preheader"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2044, __PRETTY_FUNCTION__));
assert(((cast<ConstantInt>(cast<PHINode>(IndVar)->getIncomingValue
(0))->isZero()) ? static_cast<void> (0) : __assert_fail
 ("cast<ConstantInt>(cast<PHINode>(IndVar)->getIncomingValue(0))->isZero()"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2046, __PRETTY_FUNCTION__))
    cast<ConstantInt>(cast<PHINode>(IndVar)->getIncomingValue(0))->isZero())((cast<ConstantInt>(cast<PHINode>(IndVar)->getIncomingValue
(0))->isZero()) ? static_cast<void> (0) : __assert_fail
 ("cast<ConstantInt>(cast<PHINode>(IndVar)->getIncomingValue(0))->isZero()"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2046, __PRETTY_FUNCTION__));
assert(cast<PHINode>(IndVar)->getIncomingBlock(1) == Latch)((cast<PHINode>(IndVar)->getIncomingBlock(1) == Latch
) ? static_cast<void> (0) : __assert_fail ("cast<PHINode>(IndVar)->getIncomingBlock(1) == Latch"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2047, __PRETTY_FUNCTION__));

auto *NextIndVar = cast<PHINode>(IndVar)->getIncomingValue(1);
assert(cast<Instruction>(NextIndVar)->getParent() == Latch)((cast<Instruction>(NextIndVar)->getParent() == Latch
) ? static_cast<void> (0) : __assert_fail ("cast<Instruction>(NextIndVar)->getParent() == Latch"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2050, __PRETTY_FUNCTION__));
assert(cast<BinaryOperator>(NextIndVar)->getOpcode() == BinaryOperator::Add)((cast<BinaryOperator>(NextIndVar)->getOpcode() == BinaryOperator
::Add) ? static_cast<void> (0) : __assert_fail ("cast<BinaryOperator>(NextIndVar)->getOpcode() == BinaryOperator::Add"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2051, __PRETTY_FUNCTION__));
assert(cast<BinaryOperator>(NextIndVar)->getOperand(0) == IndVar)((cast<BinaryOperator>(NextIndVar)->getOperand(0) ==
 IndVar) ? static_cast<void> (0) : __assert_fail ("cast<BinaryOperator>(NextIndVar)->getOperand(0) == IndVar"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2052, __PRETTY_FUNCTION__));
assert(cast<ConstantInt>(cast<BinaryOperator>(NextIndVar)->getOperand(1))((cast<ConstantInt>(cast<BinaryOperator>(NextIndVar
)->getOperand(1)) ->isOne()) ? static_cast<void> (
0) : __assert_fail ("cast<ConstantInt>(cast<BinaryOperator>(NextIndVar)->getOperand(1)) ->isOne()"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2054, __PRETTY_FUNCTION__))
           ->isOne())((cast<ConstantInt>(cast<BinaryOperator>(NextIndVar
)->getOperand(1)) ->isOne()) ? static_cast<void> (
0) : __assert_fail ("cast<ConstantInt>(cast<BinaryOperator>(NextIndVar)->getOperand(1)) ->isOne()"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2054, __PRETTY_FUNCTION__));

Value *TripCount = getTripCount();
assert(TripCount && "Loop trip count not found?")((TripCount && "Loop trip count not found?") ? static_cast
<void> (0) : __assert_fail ("TripCount && \"Loop trip count not found?\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2057, __PRETTY_FUNCTION__));
assert(IndVar->getType() == TripCount->getType() &&((IndVar->getType() == TripCount->getType() && "Trip count and induction variable must have the same type"
) ? static_cast<void> (0) : __assert_fail ("IndVar->getType() == TripCount->getType() && \"Trip count and induction variable must have the same type\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2059, __PRETTY_FUNCTION__))
       "Trip count and induction variable must have the same type")((IndVar->getType() == TripCount->getType() && "Trip count and induction variable must have the same type"
) ? static_cast<void> (0) : __assert_fail ("IndVar->getType() == TripCount->getType() && \"Trip count and induction variable must have the same type\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2059, __PRETTY_FUNCTION__));

auto *CmpI = cast<CmpInst>(&Cond->front());
assert(CmpI->getPredicate() == CmpInst::ICMP_ULT &&((CmpI->getPredicate() == CmpInst::ICMP_ULT && "Exit condition must be a signed less-than comparison"
) ? static_cast<void> (0) : __assert_fail ("CmpI->getPredicate() == CmpInst::ICMP_ULT && \"Exit condition must be a signed less-than comparison\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2063, __PRETTY_FUNCTION__))
       "Exit condition must be a signed less-than comparison")((CmpI->getPredicate() == CmpInst::ICMP_ULT && "Exit condition must be a signed less-than comparison"
) ? static_cast<void> (0) : __assert_fail ("CmpI->getPredicate() == CmpInst::ICMP_ULT && \"Exit condition must be a signed less-than comparison\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2063, __PRETTY_FUNCTION__));
assert(CmpI->getOperand(0) == IndVar &&((CmpI->getOperand(0) == IndVar && "Exit condition must compare the induction variable"
) ? static_cast<void> (0) : __assert_fail ("CmpI->getOperand(0) == IndVar && \"Exit condition must compare the induction variable\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2065, __PRETTY_FUNCTION__))
       "Exit condition must compare the induction variable")((CmpI->getOperand(0) == IndVar && "Exit condition must compare the induction variable"
) ? static_cast<void> (0) : __assert_fail ("CmpI->getOperand(0) == IndVar && \"Exit condition must compare the induction variable\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2065, __PRETTY_FUNCTION__));
assert(CmpI->getOperand(1) == TripCount &&((CmpI->getOperand(1) == TripCount && "Exit condition must compare with the trip count"
) ? static_cast<void> (0) : __assert_fail ("CmpI->getOperand(1) == TripCount && \"Exit condition must compare with the trip count\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2067, __PRETTY_FUNCTION__))
       "Exit condition must compare with the trip count")((CmpI->getOperand(1) == TripCount && "Exit condition must compare with the trip count"
) ? static_cast<void> (0) : __assert_fail ("CmpI->getOperand(1) == TripCount && \"Exit condition must compare with the trip count\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2067, __PRETTY_FUNCTION__));
2068#endif
2069}

←

/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h

→

1//===- llvm/IRBuilder.h - Builder for LLVM Instructions ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the IRBuilder class, which is used as a convenient way
10// to create LLVM instructions with a consistent and simplified interface.
11//
12//===----------------------------------------------------------------------===//

14#ifndef LLVM_IR_IRBUILDER_H
15#define LLVM_IR_IRBUILDER_H

17#include "llvm-c/Types.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/None.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ADT/Twine.h"
23#include "llvm/IR/BasicBlock.h"
24#include "llvm/IR/Constant.h"
25#include "llvm/IR/ConstantFolder.h"
26#include "llvm/IR/Constants.h"
27#include "llvm/IR/DataLayout.h"
28#include "llvm/IR/DebugInfoMetadata.h"
29#include "llvm/IR/DebugLoc.h"
30#include "llvm/IR/DerivedTypes.h"
31#include "llvm/IR/Function.h"
32#include "llvm/IR/GlobalVariable.h"
33#include "llvm/IR/InstrTypes.h"
34#include "llvm/IR/Instruction.h"
35#include "llvm/IR/Instructions.h"
36#include "llvm/IR/IntrinsicInst.h"
37#include "llvm/IR/LLVMContext.h"
38#include "llvm/IR/Module.h"
39#include "llvm/IR/Operator.h"
40#include "llvm/IR/Type.h"
41#include "llvm/IR/Value.h"
42#include "llvm/IR/ValueHandle.h"
43#include "llvm/Support/AtomicOrdering.h"
44#include "llvm/Support/CBindingWrapping.h"
45#include "llvm/Support/Casting.h"
46#include <cassert>
47#include <cstddef>
48#include <cstdint>
49#include <functional>
50#include <utility>

52namespace llvm {

54class APInt;
55class MDNode;
56class Use;

58/// This provides the default implementation of the IRBuilder
59/// 'InsertHelper' method that is called whenever an instruction is created by
60/// IRBuilder and needs to be inserted.
61///
62/// By default, this inserts the instruction at the insertion point.
63class IRBuilderDefaultInserter {
64public:
virtual ~IRBuilderDefaultInserter();

virtual void InsertHelper(Instruction *I, const Twine &Name,
                          BasicBlock *BB,
                          BasicBlock::iterator InsertPt) const {
  if (BB) BB->getInstList().insert(InsertPt, I);
  I->setName(Name);
}
73};

75/// Provides an 'InsertHelper' that calls a user-provided callback after
76/// performing the default insertion.
77class IRBuilderCallbackInserter : public IRBuilderDefaultInserter {
std::function<void(Instruction *)> Callback;

80public:
virtual ~IRBuilderCallbackInserter();

IRBuilderCallbackInserter(std::function<void(Instruction *)> Callback)
    : Callback(std::move(Callback)) {}

void InsertHelper(Instruction *I, const Twine &Name,
                  BasicBlock *BB,
                  BasicBlock::iterator InsertPt) const override {
  IRBuilderDefaultInserter::InsertHelper(I, Name, BB, InsertPt);
  Callback(I);
}
92};

94/// Common base class shared among various IRBuilders.
95class IRBuilderBase {
/// Pairs of (metadata kind, MDNode *) that should be added to all newly
/// created instructions, like !dbg metadata.
SmallVector<std::pair<unsigned, MDNode *>, 2> MetadataToCopy;

/// Add or update the an entry (Kind, MD) to MetadataToCopy, if \p MD is not
/// null. If \p MD is null, remove the entry with \p Kind.
void AddOrRemoveMetadataToCopy(unsigned Kind, MDNode *MD) {
  if (!MD) {
    erase_if(MetadataToCopy, [Kind](const std::pair<unsigned, MDNode *> &KV) {
      return KV.first == Kind;
    });
    return;
  }

  for (auto &KV : MetadataToCopy)
    if (KV.first == Kind) {
      KV.second = MD;
      return;
    }

  MetadataToCopy.emplace_back(Kind, MD);
}

119protected:
BasicBlock *BB;
BasicBlock::iterator InsertPt;
LLVMContext &Context;
const IRBuilderFolder &Folder;
const IRBuilderDefaultInserter &Inserter;

MDNode *DefaultFPMathTag;
FastMathFlags FMF;

bool IsFPConstrained;
fp::ExceptionBehavior DefaultConstrainedExcept;
RoundingMode DefaultConstrainedRounding;

ArrayRef<OperandBundleDef> DefaultOperandBundles;

135public:
IRBuilderBase(LLVMContext &context, const IRBuilderFolder &Folder,
              const IRBuilderDefaultInserter &Inserter,
              MDNode *FPMathTag, ArrayRef<OperandBundleDef> OpBundles)
    : Context(context), Folder(Folder), Inserter(Inserter),
      DefaultFPMathTag(FPMathTag), IsFPConstrained(false),
      DefaultConstrainedExcept(fp::ebStrict),
      DefaultConstrainedRounding(RoundingMode::Dynamic),
      DefaultOperandBundles(OpBundles) {
  ClearInsertionPoint();
}

/// Insert and return the specified instruction.
template<typename InstTy>
InstTy *Insert(InstTy *I, const Twine &Name = "") const {
  Inserter.InsertHelper(I, Name, BB, InsertPt);
  AddMetadataToInst(I);
  return I;
}

/// No-op overload to handle constants.
Constant *Insert(Constant *C, const Twine& = "") const {
  return C;
}

Value *Insert(Value *V, const Twine &Name = "") const {
  if (Instruction *I = dyn_cast<Instruction>(V))
    return Insert(I, Name);
  assert(isa<Constant>(V))((isa<Constant>(V)) ? static_cast<void> (0) : __assert_fail
 ("isa<Constant>(V)", "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 163, __PRETTY_FUNCTION__));
  return V;
}

//===--------------------------------------------------------------------===//
// Builder configuration methods
//===--------------------------------------------------------------------===//

/// Clear the insertion point: created instructions will not be
/// inserted into a block.
void ClearInsertionPoint() {
  BB = nullptr;
33
←
Null pointer value stored to field 'BB'→
  InsertPt = BasicBlock::iterator();
}

BasicBlock *GetInsertBlock() const { return BB; }
BasicBlock::iterator GetInsertPoint() const { return InsertPt; }
LLVMContext &getContext() const { return Context; }

/// This specifies that created instructions should be appended to the
/// end of the specified block.
void SetInsertPoint(BasicBlock *TheBB) {
  BB = TheBB;
  InsertPt = BB->end();
}

/// This specifies that created instructions should be inserted before
/// the specified instruction.
void SetInsertPoint(Instruction *I) {
  BB = I->getParent();
  InsertPt = I->getIterator();
  assert(InsertPt != BB->end() && "Can't read debug loc from end()")((InsertPt != BB->end() && "Can't read debug loc from end()"
) ? static_cast<void> (0) : __assert_fail ("InsertPt != BB->end() && \"Can't read debug loc from end()\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 194, __PRETTY_FUNCTION__));
  SetCurrentDebugLocation(I->getDebugLoc());
}

/// This specifies that created instructions should be inserted at the
/// specified point.
void SetInsertPoint(BasicBlock *TheBB, BasicBlock::iterator IP) {
  BB = TheBB;
  InsertPt = IP;
  if (IP != TheBB->end())
    SetCurrentDebugLocation(IP->getDebugLoc());
}

/// Set location information used by debugging information.
void SetCurrentDebugLocation(DebugLoc L) {
  AddOrRemoveMetadataToCopy(LLVMContext::MD_dbg, L.getAsMDNode());
}

/// Collect metadata with IDs \p MetadataKinds from \p Src which should be
/// added to all created instructions. Entries present in MedataDataToCopy but
/// not on \p Src will be dropped from MetadataToCopy.
void CollectMetadataToCopy(Instruction *Src,
                           ArrayRef<unsigned> MetadataKinds) {
  for (unsigned K : MetadataKinds)
    AddOrRemoveMetadataToCopy(K, Src->getMetadata(K));
}

/// Get location information used by debugging information.
DebugLoc getCurrentDebugLocation() const {
  for (auto &KV : MetadataToCopy)
    if (KV.first == LLVMContext::MD_dbg)
      return {cast<DILocation>(KV.second)};

  return {};
}

/// If this builder has a current debug location, set it on the
/// specified instruction.
void SetInstDebugLocation(Instruction *I) const {
  for (const auto &KV : MetadataToCopy)
    if (KV.first == LLVMContext::MD_dbg) {
      I->setDebugLoc(DebugLoc(KV.second));
      return;
    }
}

/// Add all entries in MetadataToCopy to \p I.
void AddMetadataToInst(Instruction *I) const {
  for (auto &KV : MetadataToCopy)
    I->setMetadata(KV.first, KV.second);
}

/// Get the return type of the current function that we're emitting
/// into.
Type *getCurrentFunctionReturnType() const;

/// InsertPoint - A saved insertion point.
class InsertPoint {
  BasicBlock *Block = nullptr;
  BasicBlock::iterator Point;

public:
  /// Creates a new insertion point which doesn't point to anything.
  InsertPoint() = default;

  /// Creates a new insertion point at the given location.
  InsertPoint(BasicBlock *InsertBlock, BasicBlock::iterator InsertPoint)
      : Block(InsertBlock), Point(InsertPoint) {}

  /// Returns true if this insert point is set.
  bool isSet() const { return (Block != nullptr); }

  BasicBlock *getBlock() const { return Block; }
  BasicBlock::iterator getPoint() const { return Point; }
};

/// Returns the current insert point.
InsertPoint saveIP() const {
  return InsertPoint(GetInsertBlock(), GetInsertPoint());
}

/// Returns the current insert point, clearing it in the process.
InsertPoint saveAndClearIP() {
  InsertPoint IP(GetInsertBlock(), GetInsertPoint());
  ClearInsertionPoint();
  return IP;
}

/// Sets the current insert point to a previously-saved location.
void restoreIP(InsertPoint IP) {
  if (IP.isSet())
31
←
Taking false branch→
    SetInsertPoint(IP.getBlock(), IP.getPoint());
  else
    ClearInsertionPoint();
32
←
Calling 'IRBuilderBase::ClearInsertionPoint'→
34
←
Returning from 'IRBuilderBase::ClearInsertionPoint'→
}

/// Get the floating point math metadata being used.
MDNode *getDefaultFPMathTag() const { return DefaultFPMathTag; }

/// Get the flags to be applied to created floating point ops
FastMathFlags getFastMathFlags() const { return FMF; }

FastMathFlags &getFastMathFlags() { return FMF; }

/// Clear the fast-math flags.
void clearFastMathFlags() { FMF.clear(); }

/// Set the floating point math metadata to be used.
void setDefaultFPMathTag(MDNode *FPMathTag) { DefaultFPMathTag = FPMathTag; }

/// Set the fast-math flags to be used with generated fp-math operators
void setFastMathFlags(FastMathFlags NewFMF) { FMF = NewFMF; }

/// Enable/Disable use of constrained floating point math. When
/// enabled the CreateF<op>() calls instead create constrained
/// floating point intrinsic calls. Fast math flags are unaffected
/// by this setting.
void setIsFPConstrained(bool IsCon) { IsFPConstrained = IsCon; }

/// Query for the use of constrained floating point math
bool getIsFPConstrained() { return IsFPConstrained; }

/// Set the exception handling to be used with constrained floating point
void setDefaultConstrainedExcept(fp::ExceptionBehavior NewExcept) {
318#ifndef NDEBUG
  Optional<StringRef> ExceptStr = ExceptionBehaviorToStr(NewExcept);
  assert(ExceptStr.hasValue() && "Garbage strict exception behavior!")((ExceptStr.hasValue() && "Garbage strict exception behavior!"
) ? static_cast<void> (0) : __assert_fail ("ExceptStr.hasValue() && \"Garbage strict exception behavior!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 320, __PRETTY_FUNCTION__));
321#endif
  DefaultConstrainedExcept = NewExcept;
}

/// Set the rounding mode handling to be used with constrained floating point
void setDefaultConstrainedRounding(RoundingMode NewRounding) {
327#ifndef NDEBUG
  Optional<StringRef> RoundingStr = RoundingModeToStr(NewRounding);
  assert(RoundingStr.hasValue() && "Garbage strict rounding mode!")((RoundingStr.hasValue() && "Garbage strict rounding mode!"
) ? static_cast<void> (0) : __assert_fail ("RoundingStr.hasValue() && \"Garbage strict rounding mode!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 329, __PRETTY_FUNCTION__));
330#endif
  DefaultConstrainedRounding = NewRounding;
}

/// Get the exception handling used with constrained floating point
fp::ExceptionBehavior getDefaultConstrainedExcept() {
  return DefaultConstrainedExcept;
}

/// Get the rounding mode handling used with constrained floating point
RoundingMode getDefaultConstrainedRounding() {
  return DefaultConstrainedRounding;
}

void setConstrainedFPFunctionAttr() {
  assert(BB && "Must have a basic block to set any function attributes!")((BB && "Must have a basic block to set any function attributes!"
) ? static_cast<void> (0) : __assert_fail ("BB && \"Must have a basic block to set any function attributes!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 345, __PRETTY_FUNCTION__));

  Function *F = BB->getParent();
  if (!F->hasFnAttribute(Attribute::StrictFP)) {
    F->addFnAttr(Attribute::StrictFP);
  }
}

void setConstrainedFPCallAttr(CallBase *I) {
  I->addAttribute(AttributeList::FunctionIndex, Attribute::StrictFP);
}

void setDefaultOperandBundles(ArrayRef<OperandBundleDef> OpBundles) {
  DefaultOperandBundles = OpBundles;
}

//===--------------------------------------------------------------------===//
// RAII helpers.
//===--------------------------------------------------------------------===//

// RAII object that stores the current insertion point and restores it
// when the object is destroyed. This includes the debug location.
class InsertPointGuard {
  IRBuilderBase &Builder;
  AssertingVH<BasicBlock> Block;
  BasicBlock::iterator Point;
  DebugLoc DbgLoc;

public:
  InsertPointGuard(IRBuilderBase &B)
      : Builder(B), Block(B.GetInsertBlock()), Point(B.GetInsertPoint()),
        DbgLoc(B.getCurrentDebugLocation()) {}

  InsertPointGuard(const InsertPointGuard &) = delete;
  InsertPointGuard &operator=(const InsertPointGuard &) = delete;

  ~InsertPointGuard() {
    Builder.restoreIP(InsertPoint(Block, Point));
    Builder.SetCurrentDebugLocation(DbgLoc);
  }
};

// RAII object that stores the current fast math settings and restores
// them when the object is destroyed.
class FastMathFlagGuard {
  IRBuilderBase &Builder;
  FastMathFlags FMF;
  MDNode *FPMathTag;
  bool IsFPConstrained;
  fp::ExceptionBehavior DefaultConstrainedExcept;
  RoundingMode DefaultConstrainedRounding;

public:
  FastMathFlagGuard(IRBuilderBase &B)
      : Builder(B), FMF(B.FMF), FPMathTag(B.DefaultFPMathTag),
        IsFPConstrained(B.IsFPConstrained),
        DefaultConstrainedExcept(B.DefaultConstrainedExcept),
        DefaultConstrainedRounding(B.DefaultConstrainedRounding) {}

  FastMathFlagGuard(const FastMathFlagGuard &) = delete;
  FastMathFlagGuard &operator=(const FastMathFlagGuard &) = delete;

  ~FastMathFlagGuard() {
    Builder.FMF = FMF;
    Builder.DefaultFPMathTag = FPMathTag;
    Builder.IsFPConstrained = IsFPConstrained;
    Builder.DefaultConstrainedExcept = DefaultConstrainedExcept;
    Builder.DefaultConstrainedRounding = DefaultConstrainedRounding;
  }
};

// RAII object that stores the current default operand bundles and restores
// them when the object is destroyed.
class OperandBundlesGuard {
  IRBuilderBase &Builder;
  ArrayRef<OperandBundleDef> DefaultOperandBundles;

public:
  OperandBundlesGuard(IRBuilderBase &B)
      : Builder(B), DefaultOperandBundles(B.DefaultOperandBundles) {}

  OperandBundlesGuard(const OperandBundlesGuard &) = delete;
  OperandBundlesGuard &operator=(const OperandBundlesGuard &) = delete;

  ~OperandBundlesGuard() {
    Builder.DefaultOperandBundles = DefaultOperandBundles;
  }
};


//===--------------------------------------------------------------------===//
// Miscellaneous creation methods.
//===--------------------------------------------------------------------===//

/// Make a new global variable with initializer type i8*
///
/// Make a new global variable with an initializer that has array of i8 type
/// filled in with the null terminated string value specified.  The new global
/// variable will be marked mergable with any others of the same contents.  If
/// Name is specified, it is the name of the global variable created.
///
/// If no module is given via \p M, it is take from the insertion point basic
/// block.
GlobalVariable *CreateGlobalString(StringRef Str, const Twine &Name = "",
                                   unsigned AddressSpace = 0,
                                   Module *M = nullptr);

/// Get a constant value representing either true or false.
ConstantInt *getInt1(bool V) {
  return ConstantInt::get(getInt1Ty(), V);
}

/// Get the constant value for i1 true.
ConstantInt *getTrue() {
  return ConstantInt::getTrue(Context);
}

/// Get the constant value for i1 false.
ConstantInt *getFalse() {
  return ConstantInt::getFalse(Context);
}

/// Get a constant 8-bit value.
ConstantInt *getInt8(uint8_t C) {
  return ConstantInt::get(getInt8Ty(), C);
}

/// Get a constant 16-bit value.
ConstantInt *getInt16(uint16_t C) {
  return ConstantInt::get(getInt16Ty(), C);
}

/// Get a constant 32-bit value.
ConstantInt *getInt32(uint32_t C) {
  return ConstantInt::get(getInt32Ty(), C);
}

/// Get a constant 64-bit value.
ConstantInt *getInt64(uint64_t C) {
  return ConstantInt::get(getInt64Ty(), C);
}

/// Get a constant N-bit value, zero extended or truncated from
/// a 64-bit value.
ConstantInt *getIntN(unsigned N, uint64_t C) {
  return ConstantInt::get(getIntNTy(N), C);
}

/// Get a constant integer value.
ConstantInt *getInt(const APInt &AI) {
  return ConstantInt::get(Context, AI);
}

//===--------------------------------------------------------------------===//
// Type creation methods
//===--------------------------------------------------------------------===//

/// Fetch the type representing a single bit
IntegerType *getInt1Ty() {
  return Type::getInt1Ty(Context);
}

/// Fetch the type representing an 8-bit integer.
IntegerType *getInt8Ty() {
  return Type::getInt8Ty(Context);
}

/// Fetch the type representing a 16-bit integer.
IntegerType *getInt16Ty() {
  return Type::getInt16Ty(Context);
}

/// Fetch the type representing a 32-bit integer.
IntegerType *getInt32Ty() {
  return Type::getInt32Ty(Context);
}

/// Fetch the type representing a 64-bit integer.
IntegerType *getInt64Ty() {
  return Type::getInt64Ty(Context);
}

/// Fetch the type representing a 128-bit integer.
IntegerType *getInt128Ty() { return Type::getInt128Ty(Context); }

/// Fetch the type representing an N-bit integer.
IntegerType *getIntNTy(unsigned N) {
  return Type::getIntNTy(Context, N);
}

/// Fetch the type representing a 16-bit floating point value.
Type *getHalfTy() {
  return Type::getHalfTy(Context);
}

/// Fetch the type representing a 16-bit brain floating point value.
Type *getBFloatTy() {
  return Type::getBFloatTy(Context);
}

/// Fetch the type representing a 32-bit floating point value.
Type *getFloatTy() {
  return Type::getFloatTy(Context);
}

/// Fetch the type representing a 64-bit floating point value.
Type *getDoubleTy() {
  return Type::getDoubleTy(Context);
}

/// Fetch the type representing void.
Type *getVoidTy() {
  return Type::getVoidTy(Context);
}

/// Fetch the type representing a pointer to an 8-bit integer value.
PointerType *getInt8PtrTy(unsigned AddrSpace = 0) {
  return Type::getInt8PtrTy(Context, AddrSpace);
}

/// Fetch the type representing a pointer to an integer value.
IntegerType *getIntPtrTy(const DataLayout &DL, unsigned AddrSpace = 0) {
  return DL.getIntPtrType(Context, AddrSpace);
}

//===--------------------------------------------------------------------===//
// Intrinsic creation methods
//===--------------------------------------------------------------------===//

/// Create and insert a memset to the specified pointer and the
/// specified value.
///
/// If the pointer isn't an i8*, it will be converted. If a TBAA tag is
/// specified, it will be added to the instruction. Likewise with alias.scope
/// and noalias tags.
CallInst *CreateMemSet(Value *Ptr, Value *Val, uint64_t Size,
                       MaybeAlign Align, bool isVolatile = false,
                       MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr,
                       MDNode *NoAliasTag = nullptr) {
  return CreateMemSet(Ptr, Val, getInt64(Size), Align, isVolatile,
                      TBAATag, ScopeTag, NoAliasTag);
}

CallInst *CreateMemSet(Value *Ptr, Value *Val, Value *Size, MaybeAlign Align,
                       bool isVolatile = false, MDNode *TBAATag = nullptr,
                       MDNode *ScopeTag = nullptr,
                       MDNode *NoAliasTag = nullptr);

/// Create and insert an element unordered-atomic memset of the region of
/// memory starting at the given pointer to the given value.
///
/// If the pointer isn't an i8*, it will be converted. If a TBAA tag is
/// specified, it will be added to the instruction. Likewise with alias.scope
/// and noalias tags.
CallInst *CreateElementUnorderedAtomicMemSet(Value *Ptr, Value *Val,
                                             uint64_t Size, Align Alignment,
                                             uint32_t ElementSize,
                                             MDNode *TBAATag = nullptr,
                                             MDNode *ScopeTag = nullptr,
                                             MDNode *NoAliasTag = nullptr) {
  return CreateElementUnorderedAtomicMemSet(Ptr, Val, getInt64(Size),
                                            Align(Alignment), ElementSize,
                                            TBAATag, ScopeTag, NoAliasTag);
}

CallInst *CreateElementUnorderedAtomicMemSet(Value *Ptr, Value *Val,
                                             Value *Size, Align Alignment,
                                             uint32_t ElementSize,
                                             MDNode *TBAATag = nullptr,
                                             MDNode *ScopeTag = nullptr,
                                             MDNode *NoAliasTag = nullptr);

/// Create and insert a memcpy between the specified pointers.
///
/// If the pointers aren't i8*, they will be converted.  If a TBAA tag is
/// specified, it will be added to the instruction. Likewise with alias.scope
/// and noalias tags.
CallInst *CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src,
                       MaybeAlign SrcAlign, uint64_t Size,
                       bool isVolatile = false, MDNode *TBAATag = nullptr,
                       MDNode *TBAAStructTag = nullptr,
                       MDNode *ScopeTag = nullptr,
                       MDNode *NoAliasTag = nullptr) {
  return CreateMemCpy(Dst, DstAlign, Src, SrcAlign, getInt64(Size),
                      isVolatile, TBAATag, TBAAStructTag, ScopeTag,
                      NoAliasTag);
}

CallInst *CreateMemTransferInst(
    Intrinsic::ID IntrID, Value *Dst, MaybeAlign DstAlign, Value *Src,
    MaybeAlign SrcAlign, Value *Size, bool isVolatile = false,
    MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr,
    MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr);

CallInst *CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src,
                       MaybeAlign SrcAlign, Value *Size,
                       bool isVolatile = false, MDNode *TBAATag = nullptr,
                       MDNode *TBAAStructTag = nullptr,
                       MDNode *ScopeTag = nullptr,
                       MDNode *NoAliasTag = nullptr) {
  return CreateMemTransferInst(Intrinsic::memcpy, Dst, DstAlign, Src,
                               SrcAlign, Size, isVolatile, TBAATag,
                               TBAAStructTag, ScopeTag, NoAliasTag);
}

CallInst *CreateMemCpyInline(Value *Dst, MaybeAlign DstAlign, Value *Src,
                             MaybeAlign SrcAlign, Value *Size);

/// Create and insert an element unordered-atomic memcpy between the
/// specified pointers.
///
/// DstAlign/SrcAlign are the alignments of the Dst/Src pointers, respectively.
///
/// If the pointers aren't i8*, they will be converted.  If a TBAA tag is
/// specified, it will be added to the instruction. Likewise with alias.scope
/// and noalias tags.
CallInst *CreateElementUnorderedAtomicMemCpy(
    Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
    uint32_t ElementSize, MDNode *TBAATag = nullptr,
    MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr,
    MDNode *NoAliasTag = nullptr);

CallInst *CreateMemMove(Value *Dst, MaybeAlign DstAlign, Value *Src,
                        MaybeAlign SrcAlign, uint64_t Size,
                        bool isVolatile = false, MDNode *TBAATag = nullptr,
                        MDNode *ScopeTag = nullptr,
                        MDNode *NoAliasTag = nullptr) {
  return CreateMemMove(Dst, DstAlign, Src, SrcAlign, getInt64(Size),
                       isVolatile, TBAATag, ScopeTag, NoAliasTag);
}

CallInst *CreateMemMove(Value *Dst, MaybeAlign DstAlign, Value *Src,
                        MaybeAlign SrcAlign, Value *Size,
                        bool isVolatile = false, MDNode *TBAATag = nullptr,
                        MDNode *ScopeTag = nullptr,
                        MDNode *NoAliasTag = nullptr);

/// \brief Create and insert an element unordered-atomic memmove between the
/// specified pointers.
///
/// DstAlign/SrcAlign are the alignments of the Dst/Src pointers,
/// respectively.
///
/// If the pointers aren't i8*, they will be converted.  If a TBAA tag is
/// specified, it will be added to the instruction. Likewise with alias.scope
/// and noalias tags.
CallInst *CreateElementUnorderedAtomicMemMove(
    Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
    uint32_t ElementSize, MDNode *TBAATag = nullptr,
    MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr,
    MDNode *NoAliasTag = nullptr);

/// Create a vector fadd reduction intrinsic of the source vector.
/// The first parameter is a scalar accumulator value for ordered reductions.
CallInst *CreateFAddReduce(Value *Acc, Value *Src);

/// Create a vector fmul reduction intrinsic of the source vector.
/// The first parameter is a scalar accumulator value for ordered reductions.
CallInst *CreateFMulReduce(Value *Acc, Value *Src);

/// Create a vector int add reduction intrinsic of the source vector.
CallInst *CreateAddReduce(Value *Src);

/// Create a vector int mul reduction intrinsic of the source vector.
CallInst *CreateMulReduce(Value *Src);

/// Create a vector int AND reduction intrinsic of the source vector.
CallInst *CreateAndReduce(Value *Src);

/// Create a vector int OR reduction intrinsic of the source vector.
CallInst *CreateOrReduce(Value *Src);

/// Create a vector int XOR reduction intrinsic of the source vector.
CallInst *CreateXorReduce(Value *Src);

/// Create a vector integer max reduction intrinsic of the source
/// vector.
CallInst *CreateIntMaxReduce(Value *Src, bool IsSigned = false);

/// Create a vector integer min reduction intrinsic of the source
/// vector.
CallInst *CreateIntMinReduce(Value *Src, bool IsSigned = false);

/// Create a vector float max reduction intrinsic of the source
/// vector.
CallInst *CreateFPMaxReduce(Value *Src);

/// Create a vector float min reduction intrinsic of the source
/// vector.
CallInst *CreateFPMinReduce(Value *Src);

/// Create a lifetime.start intrinsic.
///
/// If the pointer isn't i8* it will be converted.
CallInst *CreateLifetimeStart(Value *Ptr, ConstantInt *Size = nullptr);

/// Create a lifetime.end intrinsic.
///
/// If the pointer isn't i8* it will be converted.
CallInst *CreateLifetimeEnd(Value *Ptr, ConstantInt *Size = nullptr);

/// Create a call to invariant.start intrinsic.
///
/// If the pointer isn't i8* it will be converted.
CallInst *CreateInvariantStart(Value *Ptr, ConstantInt *Size = nullptr);

/// Create a call to Masked Load intrinsic
CallInst *CreateMaskedLoad(Value *Ptr, Align Alignment, Value *Mask,
                           Value *PassThru = nullptr, const Twine &Name = "");

/// Create a call to Masked Store intrinsic
CallInst *CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment,
                            Value *Mask);

/// Create a call to Masked Gather intrinsic
CallInst *CreateMaskedGather(Value *Ptrs, Align Alignment,
                             Value *Mask = nullptr, Value *PassThru = nullptr,
                             const Twine &Name = "");

/// Create a call to Masked Scatter intrinsic
CallInst *CreateMaskedScatter(Value *Val, Value *Ptrs, Align Alignment,
                              Value *Mask = nullptr);

/// Create an assume intrinsic call that allows the optimizer to
/// assume that the provided condition will be true.
///
/// The optional argument \p OpBundles specifies operand bundles that are
/// added to the call instruction.
CallInst *CreateAssumption(Value *Cond,
                           ArrayRef<OperandBundleDef> OpBundles = llvm::None);

/// Create a llvm.experimental.noalias.scope.decl intrinsic call.
Instruction *CreateNoAliasScopeDeclaration(Value *Scope);
Instruction *CreateNoAliasScopeDeclaration(MDNode *ScopeTag) {
  return CreateNoAliasScopeDeclaration(
      MetadataAsValue::get(Context, ScopeTag));
}

/// Create a call to the experimental.gc.statepoint intrinsic to
/// start a new statepoint sequence.
CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes,
                                 Value *ActualCallee,
                                 ArrayRef<Value *> CallArgs,
                                 Optional<ArrayRef<Value *>> DeoptArgs,
                                 ArrayRef<Value *> GCArgs,
                                 const Twine &Name = "");

/// Create a call to the experimental.gc.statepoint intrinsic to
/// start a new statepoint sequence.
CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes,
                                 Value *ActualCallee, uint32_t Flags,
                                 ArrayRef<Value *> CallArgs,
                                 Optional<ArrayRef<Use>> TransitionArgs,
                                 Optional<ArrayRef<Use>> DeoptArgs,
                                 ArrayRef<Value *> GCArgs,
                                 const Twine &Name = "");

/// Conveninence function for the common case when CallArgs are filled
/// in using makeArrayRef(CS.arg_begin(), CS.arg_end()); Use needs to be
/// .get()'ed to get the Value pointer.
CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes,
                                 Value *ActualCallee, ArrayRef<Use> CallArgs,
                                 Optional<ArrayRef<Value *>> DeoptArgs,
                                 ArrayRef<Value *> GCArgs,
                                 const Twine &Name = "");

/// Create an invoke to the experimental.gc.statepoint intrinsic to
/// start a new statepoint sequence.
InvokeInst *
CreateGCStatepointInvoke(uint64_t ID, uint32_t NumPatchBytes,
                         Value *ActualInvokee, BasicBlock *NormalDest,
                         BasicBlock *UnwindDest, ArrayRef<Value *> InvokeArgs,
                         Optional<ArrayRef<Value *>> DeoptArgs,
                         ArrayRef<Value *> GCArgs, const Twine &Name = "");

/// Create an invoke to the experimental.gc.statepoint intrinsic to
/// start a new statepoint sequence.
InvokeInst *CreateGCStatepointInvoke(
    uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
    BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
    ArrayRef<Value *> InvokeArgs, Optional<ArrayRef<Use>> TransitionArgs,
    Optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
    const Twine &Name = "");

// Convenience function for the common case when CallArgs are filled in using
// makeArrayRef(CS.arg_begin(), CS.arg_end()); Use needs to be .get()'ed to
// get the Value *.
InvokeInst *
CreateGCStatepointInvoke(uint64_t ID, uint32_t NumPatchBytes,
                         Value *ActualInvokee, BasicBlock *NormalDest,
                         BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
                         Optional<ArrayRef<Value *>> DeoptArgs,
                         ArrayRef<Value *> GCArgs, const Twine &Name = "");

/// Create a call to the experimental.gc.result intrinsic to extract
/// the result from a call wrapped in a statepoint.
CallInst *CreateGCResult(Instruction *Statepoint,
                         Type *ResultType,
                         const Twine &Name = "");

/// Create a call to the experimental.gc.relocate intrinsics to
/// project the relocated value of one pointer from the statepoint.
CallInst *CreateGCRelocate(Instruction *Statepoint,
                           int BaseOffset,
                           int DerivedOffset,
                           Type *ResultType,
                           const Twine &Name = "");

/// Create a call to llvm.vscale, multiplied by \p Scaling. The type of VScale
/// will be the same type as that of \p Scaling.
Value *CreateVScale(Constant *Scaling, const Twine &Name = "");

/// Creates a vector of type \p DstType with the linear sequence <0, 1, ...>
Value *CreateStepVector(Type *DstType, const Twine &Name = "");

/// Create a call to intrinsic \p ID with 1 operand which is mangled on its
/// type.
CallInst *CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
                               Instruction *FMFSource = nullptr,
                               const Twine &Name = "");

/// Create a call to intrinsic \p ID with 2 operands which is mangled on the
/// first type.
CallInst *CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS,
                                Instruction *FMFSource = nullptr,
                                const Twine &Name = "");

/// Create a call to intrinsic \p ID with \p args, mangled using \p Types. If
/// \p FMFSource is provided, copy fast-math-flags from that instruction to
/// the intrinsic.
CallInst *CreateIntrinsic(Intrinsic::ID ID, ArrayRef<Type *> Types,
                          ArrayRef<Value *> Args,
                          Instruction *FMFSource = nullptr,
                          const Twine &Name = "");

/// Create call to the minnum intrinsic.
CallInst *CreateMinNum(Value *LHS, Value *RHS, const Twine &Name = "") {
  return CreateBinaryIntrinsic(Intrinsic::minnum, LHS, RHS, nullptr, Name);
}

/// Create call to the maxnum intrinsic.
CallInst *CreateMaxNum(Value *LHS, Value *RHS, const Twine &Name = "") {
  return CreateBinaryIntrinsic(Intrinsic::maxnum, LHS, RHS, nullptr, Name);
}

/// Create call to the minimum intrinsic.
CallInst *CreateMinimum(Value *LHS, Value *RHS, const Twine &Name = "") {
  return CreateBinaryIntrinsic(Intrinsic::minimum, LHS, RHS, nullptr, Name);
}

/// Create call to the maximum intrinsic.
CallInst *CreateMaximum(Value *LHS, Value *RHS, const Twine &Name = "") {
  return CreateBinaryIntrinsic(Intrinsic::maximum, LHS, RHS, nullptr, Name);
}

/// Create a call to the experimental.vector.extract intrinsic.
CallInst *CreateExtractVector(Type *DstType, Value *SrcVec, Value *Idx,
                              const Twine &Name = "") {
  return CreateIntrinsic(Intrinsic::experimental_vector_extract,
                         {DstType, SrcVec->getType()}, {SrcVec, Idx}, nullptr,
                         Name);
}

/// Create a call to the experimental.vector.insert intrinsic.
CallInst *CreateInsertVector(Type *DstType, Value *SrcVec, Value *SubVec,
                             Value *Idx, const Twine &Name = "") {
  return CreateIntrinsic(Intrinsic::experimental_vector_insert,
                         {DstType, SubVec->getType()}, {SrcVec, SubVec, Idx},
                         nullptr, Name);
}

916private:
/// Create a call to a masked intrinsic with given Id.
CallInst *CreateMaskedIntrinsic(Intrinsic::ID Id, ArrayRef<Value *> Ops,
                                ArrayRef<Type *> OverloadedTypes,
                                const Twine &Name = "");

Value *getCastedInt8PtrValue(Value *Ptr);

//===--------------------------------------------------------------------===//
// Instruction creation methods: Terminators
//===--------------------------------------------------------------------===//

928private:
/// Helper to add branch weight and unpredictable metadata onto an
/// instruction.
/// \returns The annotated instruction.
template <typename InstTy>
InstTy *addBranchMetadata(InstTy *I, MDNode *Weights, MDNode *Unpredictable) {
  if (Weights)
    I->setMetadata(LLVMContext::MD_prof, Weights);
  if (Unpredictable)
    I->setMetadata(LLVMContext::MD_unpredictable, Unpredictable);
  return I;
}

941public:
/// Create a 'ret void' instruction.
ReturnInst *CreateRetVoid() {
  return Insert(ReturnInst::Create(Context));
}

/// Create a 'ret <val>' instruction.
ReturnInst *CreateRet(Value *V) {
  return Insert(ReturnInst::Create(Context, V));
}

/// Create a sequence of N insertvalue instructions,
/// with one Value from the retVals array each, that build a aggregate
/// return value one value at a time, and a ret instruction to return
/// the resulting aggregate value.
///
/// This is a convenience function for code that uses aggregate return values
/// as a vehicle for having multiple return values.
ReturnInst *CreateAggregateRet(Value *const *retVals, unsigned N) {
  Value *V = UndefValue::get(getCurrentFunctionReturnType());
  for (unsigned i = 0; i != N; ++i)
    V = CreateInsertValue(V, retVals[i], i, "mrv");
  return Insert(ReturnInst::Create(Context, V));
}

/// Create an unconditional 'br label X' instruction.
BranchInst *CreateBr(BasicBlock *Dest) {
  return Insert(BranchInst::Create(Dest));
}

/// Create a conditional 'br Cond, TrueDest, FalseDest'
/// instruction.
BranchInst *CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False,
                         MDNode *BranchWeights = nullptr,
                         MDNode *Unpredictable = nullptr) {
  return Insert(addBranchMetadata(BranchInst::Create(True, False, Cond),
                                  BranchWeights, Unpredictable));
}

/// Create a conditional 'br Cond, TrueDest, FalseDest'
/// instruction. Copy branch meta data if available.
BranchInst *CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False,
                         Instruction *MDSrc) {
  BranchInst *Br = BranchInst::Create(True, False, Cond);
  if (MDSrc) {
    unsigned WL[4] = {LLVMContext::MD_prof, LLVMContext::MD_unpredictable,
                      LLVMContext::MD_make_implicit, LLVMContext::MD_dbg};
    Br->copyMetadata(*MDSrc, makeArrayRef(&WL[0], 4));
  }
  return Insert(Br);
}

/// Create a switch instruction with the specified value, default dest,
/// and with a hint for the number of cases that will be added (for efficient
/// allocation).
SwitchInst *CreateSwitch(Value *V, BasicBlock *Dest, unsigned NumCases = 10,
                         MDNode *BranchWeights = nullptr,
                         MDNode *Unpredictable = nullptr) {
  return Insert(addBranchMetadata(SwitchInst::Create(V, Dest, NumCases),
                                  BranchWeights, Unpredictable));
}

/// Create an indirect branch instruction with the specified address
/// operand, with an optional hint for the number of destinations that will be
/// added (for efficient allocation).
IndirectBrInst *CreateIndirectBr(Value *Addr, unsigned NumDests = 10) {
  return Insert(IndirectBrInst::Create(Addr, NumDests));
}

/// Create an invoke instruction.
InvokeInst *CreateInvoke(FunctionType *Ty, Value *Callee,
                         BasicBlock *NormalDest, BasicBlock *UnwindDest,
                         ArrayRef<Value *> Args,
                         ArrayRef<OperandBundleDef> OpBundles,
                         const Twine &Name = "") {
  InvokeInst *II =
      InvokeInst::Create(Ty, Callee, NormalDest, UnwindDest, Args, OpBundles);
  if (IsFPConstrained)
    setConstrainedFPCallAttr(II);
  return Insert(II, Name);
}
InvokeInst *CreateInvoke(FunctionType *Ty, Value *Callee,
                         BasicBlock *NormalDest, BasicBlock *UnwindDest,
                         ArrayRef<Value *> Args = None,
                         const Twine &Name = "") {
  InvokeInst *II =
      InvokeInst::Create(Ty, Callee, NormalDest, UnwindDest, Args);
  if (IsFPConstrained)
    setConstrainedFPCallAttr(II);
  return Insert(II, Name);
}

InvokeInst *CreateInvoke(FunctionCallee Callee, BasicBlock *NormalDest,
                         BasicBlock *UnwindDest, ArrayRef<Value *> Args,
                         ArrayRef<OperandBundleDef> OpBundles,
                         const Twine &Name = "") {
  return CreateInvoke(Callee.getFunctionType(), Callee.getCallee(),
                      NormalDest, UnwindDest, Args, OpBundles, Name);
}

InvokeInst *CreateInvoke(FunctionCallee Callee, BasicBlock *NormalDest,
                         BasicBlock *UnwindDest,
                         ArrayRef<Value *> Args = None,
                         const Twine &Name = "") {
  return CreateInvoke(Callee.getFunctionType(), Callee.getCallee(),
                      NormalDest, UnwindDest, Args, Name);
}

/// \brief Create a callbr instruction.
CallBrInst *CreateCallBr(FunctionType *Ty, Value *Callee,
                         BasicBlock *DefaultDest,
                         ArrayRef<BasicBlock *> IndirectDests,
                         ArrayRef<Value *> Args = None,
                         const Twine &Name = "") {
  return Insert(CallBrInst::Create(Ty, Callee, DefaultDest, IndirectDests,
                                   Args), Name);
}
CallBrInst *CreateCallBr(FunctionType *Ty, Value *Callee,
                         BasicBlock *DefaultDest,
                         ArrayRef<BasicBlock *> IndirectDests,
                         ArrayRef<Value *> Args,
                         ArrayRef<OperandBundleDef> OpBundles,
                         const Twine &Name = "") {
  return Insert(
      CallBrInst::Create(Ty, Callee, DefaultDest, IndirectDests, Args,
                         OpBundles), Name);
}

CallBrInst *CreateCallBr(FunctionCallee Callee, BasicBlock *DefaultDest,
                         ArrayRef<BasicBlock *> IndirectDests,
                         ArrayRef<Value *> Args = None,
                         const Twine &Name = "") {
  return CreateCallBr(Callee.getFunctionType(), Callee.getCallee(),
                      DefaultDest, IndirectDests, Args, Name);
}
CallBrInst *CreateCallBr(FunctionCallee Callee, BasicBlock *DefaultDest,
                         ArrayRef<BasicBlock *> IndirectDests,
                         ArrayRef<Value *> Args,
                         ArrayRef<OperandBundleDef> OpBundles,
                         const Twine &Name = "") {
  return CreateCallBr(Callee.getFunctionType(), Callee.getCallee(),
                      DefaultDest, IndirectDests, Args, Name);
}

ResumeInst *CreateResume(Value *Exn) {
  return Insert(ResumeInst::Create(Exn));
}

CleanupReturnInst *CreateCleanupRet(CleanupPadInst *CleanupPad,
                                    BasicBlock *UnwindBB = nullptr) {
  return Insert(CleanupReturnInst::Create(CleanupPad, UnwindBB));
}

CatchSwitchInst *CreateCatchSwitch(Value *ParentPad, BasicBlock *UnwindBB,
                                   unsigned NumHandlers,
                                   const Twine &Name = "") {
  return Insert(CatchSwitchInst::Create(ParentPad, UnwindBB, NumHandlers),
                Name);
}

CatchPadInst *CreateCatchPad(Value *ParentPad, ArrayRef<Value *> Args,
                             const Twine &Name = "") {
  return Insert(CatchPadInst::Create(ParentPad, Args), Name);
}

CleanupPadInst *CreateCleanupPad(Value *ParentPad,
                                 ArrayRef<Value *> Args = None,
                                 const Twine &Name = "") {
  return Insert(CleanupPadInst::Create(ParentPad, Args), Name);
}

CatchReturnInst *CreateCatchRet(CatchPadInst *CatchPad, BasicBlock *BB) {
  return Insert(CatchReturnInst::Create(CatchPad, BB));
}

UnreachableInst *CreateUnreachable() {
  return Insert(new UnreachableInst(Context));
}

//===--------------------------------------------------------------------===//
// Instruction creation methods: Binary Operators
//===--------------------------------------------------------------------===//
1123private:
BinaryOperator *CreateInsertNUWNSWBinOp(BinaryOperator::BinaryOps Opc,
                                        Value *LHS, Value *RHS,
                                        const Twine &Name,
                                        bool HasNUW, bool HasNSW) {
  BinaryOperator *BO = Insert(BinaryOperator::Create(Opc, LHS, RHS), Name);
  if (HasNUW) BO->setHasNoUnsignedWrap();
  if (HasNSW) BO->setHasNoSignedWrap();
  return BO;
}

Instruction *setFPAttrs(Instruction *I, MDNode *FPMD,
                        FastMathFlags FMF) const {
  if (!FPMD)
    FPMD = DefaultFPMathTag;
  if (FPMD)
    I->setMetadata(LLVMContext::MD_fpmath, FPMD);
  I->setFastMathFlags(FMF);
  return I;
}

Value *foldConstant(Instruction::BinaryOps Opc, Value *L,
                    Value *R, const Twine &Name) const {
  auto *LC = dyn_cast<Constant>(L);
  auto *RC = dyn_cast<Constant>(R);
  return (LC && RC) ? Insert(Folder.CreateBinOp(Opc, LC, RC), Name) : nullptr;
}

Value *getConstrainedFPRounding(Optional<RoundingMode> Rounding) {
  RoundingMode UseRounding = DefaultConstrainedRounding;

  if (Rounding.hasValue())
    UseRounding = Rounding.getValue();

  Optional<StringRef> RoundingStr = RoundingModeToStr(UseRounding);
  assert(RoundingStr.hasValue() && "Garbage strict rounding mode!")((RoundingStr.hasValue() && "Garbage strict rounding mode!"
) ? static_cast<void> (0) : __assert_fail ("RoundingStr.hasValue() && \"Garbage strict rounding mode!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 1158, __PRETTY_FUNCTION__));
  auto *RoundingMDS = MDString::get(Context, RoundingStr.getValue());

  return MetadataAsValue::get(Context, RoundingMDS);
}

Value *getConstrainedFPExcept(Optional<fp::ExceptionBehavior> Except) {
  fp::ExceptionBehavior UseExcept = DefaultConstrainedExcept;

  if (Except.hasValue())
    UseExcept = Except.getValue();

  Optional<StringRef> ExceptStr = ExceptionBehaviorToStr(UseExcept);
  assert(ExceptStr.hasValue() && "Garbage strict exception behavior!")((ExceptStr.hasValue() && "Garbage strict exception behavior!"
) ? static_cast<void> (0) : __assert_fail ("ExceptStr.hasValue() && \"Garbage strict exception behavior!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 1171, __PRETTY_FUNCTION__));
  auto *ExceptMDS = MDString::get(Context, ExceptStr.getValue());

  return MetadataAsValue::get(Context, ExceptMDS);
}

Value *getConstrainedFPPredicate(CmpInst::Predicate Predicate) {
  assert(CmpInst::isFPPredicate(Predicate) &&((CmpInst::isFPPredicate(Predicate) && Predicate != CmpInst
::FCMP_FALSE && Predicate != CmpInst::FCMP_TRUE &&
 "Invalid constrained FP comparison predicate!") ? static_cast
<void> (0) : __assert_fail ("CmpInst::isFPPredicate(Predicate) && Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst::FCMP_TRUE && \"Invalid constrained FP comparison predicate!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 1181, __PRETTY_FUNCTION__))
         Predicate != CmpInst::FCMP_FALSE &&((CmpInst::isFPPredicate(Predicate) && Predicate != CmpInst
::FCMP_FALSE && Predicate != CmpInst::FCMP_TRUE &&
 "Invalid constrained FP comparison predicate!") ? static_cast
<void> (0) : __assert_fail ("CmpInst::isFPPredicate(Predicate) && Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst::FCMP_TRUE && \"Invalid constrained FP comparison predicate!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 1181, __PRETTY_FUNCTION__))
         Predicate != CmpInst::FCMP_TRUE &&((CmpInst::isFPPredicate(Predicate) && Predicate != CmpInst
::FCMP_FALSE && Predicate != CmpInst::FCMP_TRUE &&
 "Invalid constrained FP comparison predicate!") ? static_cast
<void> (0) : __assert_fail ("CmpInst::isFPPredicate(Predicate) && Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst::FCMP_TRUE && \"Invalid constrained FP comparison predicate!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 1181, __PRETTY_FUNCTION__))
         "Invalid constrained FP comparison predicate!")((CmpInst::isFPPredicate(Predicate) && Predicate != CmpInst
::FCMP_FALSE && Predicate != CmpInst::FCMP_TRUE &&
 "Invalid constrained FP comparison predicate!") ? static_cast
<void> (0) : __assert_fail ("CmpInst::isFPPredicate(Predicate) && Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst::FCMP_TRUE && \"Invalid constrained FP comparison predicate!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 1181, __PRETTY_FUNCTION__));

  StringRef PredicateStr = CmpInst::getPredicateName(Predicate);
  auto *PredicateMDS = MDString::get(Context, PredicateStr);

  return MetadataAsValue::get(Context, PredicateMDS);
}

1189public:
Value *CreateAdd(Value *LHS, Value *RHS, const Twine &Name = "",
                 bool HasNUW = false, bool HasNSW = false) {
  if (auto *LC = dyn_cast<Constant>(LHS))
    if (auto *RC = dyn_cast<Constant>(RHS))
      return Insert(Folder.CreateAdd(LC, RC, HasNUW, HasNSW), Name);
  return CreateInsertNUWNSWBinOp(Instruction::Add, LHS, RHS, Name,
                                 HasNUW, HasNSW);
}

Value *CreateNSWAdd(Value *LHS, Value *RHS, const Twine &Name = "") {
  return CreateAdd(LHS, RHS, Name, false, true);
}

Value *CreateNUWAdd(Value *LHS, Value *RHS, const Twine &Name = "") {
  return CreateAdd(LHS, RHS, Name, true, false);
}

Value *CreateSub(Value *LHS, Value *RHS, const Twine &Name = "",
                 bool HasNUW = false, bool HasNSW = false) {
  if (auto *LC = dyn_cast<Constant>(LHS))
    if (auto *RC = dyn_cast<Constant>(RHS))
      return Insert(Folder.CreateSub(LC, RC, HasNUW, HasNSW), Name);
  return CreateInsertNUWNSWBinOp(Instruction::Sub, LHS, RHS, Name,
                                 HasNUW, HasNSW);
}

Value *CreateNSWSub(Value *LHS, Value *RHS, const Twine &Name = "") {
  return CreateSub(LHS, RHS, Name, false, true);
}

Value *CreateNUWSub(Value *LHS, Value *RHS, const Twine &Name = "") {
  return CreateSub(LHS, RHS, Name, true, false);
}

Value *CreateMul(Value *LHS, Value *RHS, const Twine &Name = "",
                 bool HasNUW = false, bool HasNSW = false) {
  if (auto *LC = dyn_cast<Constant>(LHS))
    if (auto *RC = dyn_cast<Constant>(RHS))
      return Insert(Folder.CreateMul(LC, RC, HasNUW, HasNSW), Name);
  return CreateInsertNUWNSWBinOp(Instruction::Mul, LHS, RHS, Name,
                                 HasNUW, HasNSW);
}

Value *CreateNSWMul(Value *LHS, Value *RHS, const Twine &Name = "") {
  return CreateMul(LHS, RHS, Name, false, true);
}

Value *CreateNUWMul(Value *LHS, Value *RHS, const Twine &Name = "") {
  return CreateMul(LHS, RHS, Name, true, false);
}

Value *CreateUDiv(Value *LHS, Value *RHS, const Twine &Name = "",
                  bool isExact = false) {
  if (auto *LC = dyn_cast<Constant>(LHS))
    if (auto *RC = dyn_cast<Constant>(RHS))
      return Insert(Folder.CreateUDiv(LC, RC, isExact), Name);
  if (!isExact)
    return Insert(BinaryOperator::CreateUDiv(LHS, RHS), Name);
  return Insert(BinaryOperator::CreateExactUDiv(LHS, RHS), Name);
}

Value *CreateExactUDiv(Value *LHS, Value *RHS, const Twine &Name = "") {
  return CreateUDiv(LHS, RHS, Name, true);
}

Value *CreateSDiv(Value *LHS, Value *RHS, const Twine &Name = "",
                  bool isExact = false) {
  if (auto *LC = dyn_cast<Constant>(LHS))
    if (auto *RC = dyn_cast<Constant>(RHS))
      return Insert(Folder.CreateSDiv(LC, RC, isExact), Name);
  if (!isExact)
    return Insert(BinaryOperator::CreateSDiv(LHS, RHS), Name);
  return Insert(BinaryOperator::CreateExactSDiv(LHS, RHS), Name);
}

Value *CreateExactSDiv(Value *LHS, Value *RHS, const Twine &Name = "") {
  return CreateSDiv(LHS, RHS, Name, true);
}

Value *CreateURem(Value *LHS, Value *RHS, const Twine &Name = "") {
  if (Value *V = foldConstant(Instruction::URem, LHS, RHS, Name)) return V;
  return Insert(BinaryOperator::CreateURem(LHS, RHS), Name);
}

Value *CreateSRem(Value *LHS, Value *RHS, const Twine &Name = "") {
  if (Value *V = foldConstant(Instruction::SRem, LHS, RHS, Name)) return V;
  return Insert(BinaryOperator::CreateSRem(LHS, RHS), Name);
}

Value *CreateShl(Value *LHS, Value *RHS, const Twine &Name = "",
                 bool HasNUW = false, bool HasNSW = false) {
  if (auto *LC = dyn_cast<Constant>(LHS))
    if (auto *RC = dyn_cast<Constant>(RHS))
      return Insert(Folder.CreateShl(LC, RC, HasNUW, HasNSW), Name);
  return CreateInsertNUWNSWBinOp(Instruction::Shl, LHS, RHS, Name,
                                 HasNUW, HasNSW);
}

Value *CreateShl(Value *LHS, const APInt &RHS, const Twine &Name = "",
                 bool HasNUW = false, bool HasNSW = false) {
  return CreateShl(LHS, ConstantInt::get(LHS->getType(), RHS), Name,
                   HasNUW, HasNSW);
}

Value *CreateShl(Value *LHS, uint64_t RHS, const Twine &Name = "",
                 bool HasNUW = false, bool HasNSW = false) {
  return CreateShl(LHS, ConstantInt::get(LHS->getType(), RHS), Name,
                   HasNUW, HasNSW);
}

Value *CreateLShr(Value *LHS, Value *RHS, const Twine &Name = "",
                  bool isExact = false) {
  if (auto *LC = dyn_cast<Constant>(LHS))
    if (auto *RC = dyn_cast<Constant>(RHS))
      return Insert(Folder.CreateLShr(LC, RC, isExact), Name);
  if (!isExact)
    return Insert(BinaryOperator::CreateLShr(LHS, RHS), Name);
  return Insert(BinaryOperator::CreateExactLShr(LHS, RHS), Name);
}

Value *CreateLShr(Value *LHS, const APInt &RHS, const Twine &Name = "",
                  bool isExact = false) {
  return CreateLShr(LHS, ConstantInt::get(LHS->getType(), RHS), Name,isExact);
}

Value *CreateLShr(Value *LHS, uint64_t RHS, const Twine &Name = "",
                  bool isExact = false) {
  return CreateLShr(LHS, ConstantInt::get(LHS->getType(), RHS), Name,isExact);
}

Value *CreateAShr(Value *LHS, Value *RHS, const Twine &Name = "",
                  bool isExact = false) {
  if (auto *LC = dyn_cast<Constant>(LHS))
    if (auto *RC = dyn_cast<Constant>(RHS))
      return Insert(Folder.CreateAShr(LC, RC, isExact), Name);
  if (!isExact)
    return Insert(BinaryOperator::CreateAShr(LHS, RHS), Name);
  return Insert(BinaryOperator::CreateExactAShr(LHS, RHS), Name);
}

Value *CreateAShr(Value *LHS, const APInt &RHS, const Twine &Name = "",
                  bool isExact = false) {
  return CreateAShr(LHS, ConstantInt::get(LHS->getType(), RHS), Name,isExact);
}

Value *CreateAShr(Value *LHS, uint64_t RHS, const Twine &Name = "",
                  bool isExact = false) {
  return CreateAShr(LHS, ConstantInt::get(LHS->getType(), RHS), Name,isExact);
}

Value *CreateAnd(Value *LHS, Value *RHS, const Twine &Name = "") {
  if (auto *RC = dyn_cast<Constant>(RHS)) {
    if (isa<ConstantInt>(RC) && cast<ConstantInt>(RC)->isMinusOne())
      return LHS;  // LHS & -1 -> LHS
    if (auto *LC = dyn_cast<Constant>(LHS))
      return Insert(Folder.CreateAnd(LC, RC), Name);
  }
  return Insert(BinaryOperator::CreateAnd(LHS, RHS), Name);
}

Value *CreateAnd(Value *LHS, const APInt &RHS, const Twine &Name = "") {
  return CreateAnd(LHS, ConstantInt::get(LHS->getType(), RHS), Name);
}

Value *CreateAnd(Value *LHS, uint64_t RHS, const Twine &Name = "") {
  return CreateAnd(LHS, ConstantInt::get(LHS->getType(), RHS), Name);
}

Value *CreateAnd(ArrayRef<Value*> Ops) {
  assert(!Ops.empty())((!Ops.empty()) ? static_cast<void> (0) : __assert_fail
 ("!Ops.empty()", "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 1359, __PRETTY_FUNCTION__));
  Value *Accum = Ops[0];
  for (unsigned i = 1; i < Ops.size(); i++)
    Accum = CreateAnd(Accum, Ops[i]);
  return Accum;
}

Value *CreateOr(Value *LHS, Value *RHS, const Twine &Name = "") {
  if (auto *RC = dyn_cast<Constant>(RHS)) {
    if (RC->isNullValue())
      return LHS;  // LHS | 0 -> LHS
    if (auto *LC = dyn_cast<Constant>(LHS))
      return Insert(Folder.CreateOr(LC, RC), Name);
  }
  return Insert(BinaryOperator::CreateOr(LHS, RHS), Name);
}

Value *CreateOr(Value *LHS, const APInt &RHS, const Twine &Name = "") {
  return CreateOr(LHS, ConstantInt::get(LHS->getType(), RHS), Name);
}

Value *CreateOr(Value *LHS, uint64_t RHS, const Twine &Name = "") {
  return CreateOr(LHS, ConstantInt::get(LHS->getType(), RHS), Name);
}

Value *CreateOr(ArrayRef<Value*> Ops) {
  assert(!Ops.empty())((!Ops.empty()) ? static_cast<void> (0) : __assert_fail
 ("!Ops.empty()", "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 1385, __PRETTY_FUNCTION__));
  Value *Accum = Ops[0];
  for (unsigned i = 1; i < Ops.size(); i++)
    Accum = CreateOr(Accum, Ops[i]);
  return Accum;
}

Value *CreateXor(Value *LHS, Value *RHS, const Twine &Name = "") {
  if (Value *V = foldConstant(Instruction::Xor, LHS, RHS, Name)) return V;
  return Insert(BinaryOperator::CreateXor(LHS, RHS), Name);
}

Value *CreateXor(Value *LHS, const APInt &RHS, const Twine &Name = "") {
  return CreateXor(LHS, ConstantInt::get(LHS->getType(), RHS), Name);
}

Value *CreateXor(Value *LHS, uint64_t RHS, const Twine &Name = "") {
  return CreateXor(LHS, ConstantInt::get(LHS->getType(), RHS), Name);
}

Value *CreateFAdd(Value *L, Value *R, const Twine &Name = "",
                  MDNode *FPMD = nullptr) {
  if (IsFPConstrained)
    return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fadd,
                                    L, R, nullptr, Name, FPMD);

  if (Value *V = foldConstant(Instruction::FAdd, L, R, Name)) return V;
  Instruction *I = setFPAttrs(BinaryOperator::CreateFAdd(L, R), FPMD, FMF);
  return Insert(I, Name);
}

/// Copy fast-math-flags from an instruction rather than using the builder's
/// default FMF.
Value *CreateFAddFMF(Value *L, Value *R, Instruction *FMFSource,
                     const Twine &Name = "") {
  if (IsFPConstrained)
    return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fadd,
                                    L, R, FMFSource, Name);

  if (Value *V = foldConstant(Instruction::FAdd, L, R, Name)) return V;
  Instruction *I = setFPAttrs(BinaryOperator::CreateFAdd(L, R), nullptr,
                              FMFSource->getFastMathFlags());
  return Insert(I, Name);
}

Value *CreateFSub(Value *L, Value *R, const Twine &Name = "",
                  MDNode *FPMD = nullptr) {
  if (IsFPConstrained)
    return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fsub,
                                    L, R, nullptr, Name, FPMD);

  if (Value *V = foldConstant(Instruction::FSub, L, R, Name)) return V;
  Instruction *I = setFPAttrs(BinaryOperator::CreateFSub(L, R), FPMD, FMF);
  return Insert(I, Name);
}

/// Copy fast-math-flags from an instruction rather than using the builder's
/// default FMF.
Value *CreateFSubFMF(Value *L, Value *R, Instruction *FMFSource,
                     const Twine &Name = "") {
  if (IsFPConstrained)
    return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fsub,
                                    L, R, FMFSource, Name);

  if (Value *V = foldConstant(Instruction::FSub, L, R, Name)) return V;
  Instruction *I = setFPAttrs(BinaryOperator::CreateFSub(L, R), nullptr,
                              FMFSource->getFastMathFlags());
  return Insert(I, Name);
}

Value *CreateFMul(Value *L, Value *R, const Twine &Name = "",
                  MDNode *FPMD = nullptr) {
  if (IsFPConstrained)
    return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fmul,
                                    L, R, nullptr, Name, FPMD);

  if (Value *V = foldConstant(Instruction::FMul, L, R, Name)) return V;
  Instruction *I = setFPAttrs(BinaryOperator::CreateFMul(L, R), FPMD, FMF);
  return Insert(I, Name);
}

/// Copy fast-math-flags from an instruction rather than using the builder's
/// default FMF.
Value *CreateFMulFMF(Value *L, Value *R, Instruction *FMFSource,
                     const Twine &Name = "") {
  if (IsFPConstrained)
    return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fmul,
                                    L, R, FMFSource, Name);

  if (Value *V = foldConstant(Instruction::FMul, L, R, Name)) return V;
  Instruction *I = setFPAttrs(BinaryOperator::CreateFMul(L, R), nullptr,
                              FMFSource->getFastMathFlags());
  return Insert(I, Name);
}

Value *CreateFDiv(Value *L, Value *R, const Twine &Name = "",
                  MDNode *FPMD = nullptr) {
  if (IsFPConstrained)
    return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fdiv,
                                    L, R, nullptr, Name, FPMD);

  if (Value *V = foldConstant(Instruction::FDiv, L, R, Name)) return V;
  Instruction *I = setFPAttrs(BinaryOperator::CreateFDiv(L, R), FPMD, FMF);
  return Insert(I, Name);
}

/// Copy fast-math-flags from an instruction rather than using the builder's
/// default FMF.
Value *CreateFDivFMF(Value *L, Value *R, Instruction *FMFSource,
                     const Twine &Name = "") {
  if (IsFPConstrained)
    return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fdiv,
                                    L, R, FMFSource, Name);

  if (Value *V = foldConstant(Instruction::FDiv, L, R, Name)) return V;
  Instruction *I = setFPAttrs(BinaryOperator::CreateFDiv(L, R), nullptr,
                              FMFSource->getFastMathFlags());
  return Insert(I, Name);
}

Value *CreateFRem(Value *L, Value *R, const Twine &Name = "",
                  MDNode *FPMD = nullptr) {
  if (IsFPConstrained)
    return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_frem,
                                    L, R, nullptr, Name, FPMD);

  if (Value *V = foldConstant(Instruction::FRem, L, R, Name)) return V;
  Instruction *I = setFPAttrs(BinaryOperator::CreateFRem(L, R), FPMD, FMF);
  return Insert(I, Name);
}

/// Copy fast-math-flags from an instruction rather than using the builder's
/// default FMF.
Value *CreateFRemFMF(Value *L, Value *R, Instruction *FMFSource,
                     const Twine &Name = "") {
  if (IsFPConstrained)
    return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_frem,
                                    L, R, FMFSource, Name);

  if (Value *V = foldConstant(Instruction::FRem, L, R, Name)) return V;
  Instruction *I = setFPAttrs(BinaryOperator::CreateFRem(L, R), nullptr,
                              FMFSource->getFastMathFlags());
  return Insert(I, Name);
}

Value *CreateBinOp(Instruction::BinaryOps Opc,
                   Value *LHS, Value *RHS, const Twine &Name = "",
                   MDNode *FPMathTag = nullptr) {
  if (Value *V = foldConstant(Opc, LHS, RHS, Name)) return V;
  Instruction *BinOp = BinaryOperator::Create(Opc, LHS, RHS);
  if (isa<FPMathOperator>(BinOp))
    setFPAttrs(BinOp, FPMathTag, FMF);
  return Insert(BinOp, Name);
}

Value *CreateLogicalAnd(Value *Cond1, Value *Cond2, const Twine &Name = "") {
  assert(Cond2->getType()->isIntOrIntVectorTy(1))((Cond2->getType()->isIntOrIntVectorTy(1)) ? static_cast
<void> (0) : __assert_fail ("Cond2->getType()->isIntOrIntVectorTy(1)"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 1541, __PRETTY_FUNCTION__));
  return CreateSelect(Cond1, Cond2,
                      ConstantInt::getNullValue(Cond2->getType()), Name);
}

Value *CreateLogicalOr(Value *Cond1, Value *Cond2, const Twine &Name = "") {
  assert(Cond2->getType()->isIntOrIntVectorTy(1))((Cond2->getType()->isIntOrIntVectorTy(1)) ? static_cast
<void> (0) : __assert_fail ("Cond2->getType()->isIntOrIntVectorTy(1)"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 1547, __PRETTY_FUNCTION__));
  return CreateSelect(Cond1, ConstantInt::getAllOnesValue(Cond2->getType()),
                      Cond2, Name);
}

CallInst *CreateConstrainedFPBinOp(
    Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource = nullptr,
    const Twine &Name = "", MDNode *FPMathTag = nullptr,
    Optional<RoundingMode> Rounding = None,
    Optional<fp::ExceptionBehavior> Except = None);

Value *CreateNeg(Value *V, const Twine &Name = "",
                 bool HasNUW = false, bool HasNSW = false) {
  if (auto *VC = dyn_cast<Constant>(V))
    return Insert(Folder.CreateNeg(VC, HasNUW, HasNSW), Name);
  BinaryOperator *BO = Insert(BinaryOperator::CreateNeg(V), Name);
  if (HasNUW) BO->setHasNoUnsignedWrap();
  if (HasNSW) BO->setHasNoSignedWrap();
  return BO;
}

Value *CreateNSWNeg(Value *V, const Twine &Name = "") {
  return CreateNeg(V, Name, false, true);
}

Value *CreateNUWNeg(Value *V, const Twine &Name = "") {
  return CreateNeg(V, Name, true, false);
}

Value *CreateFNeg(Value *V, const Twine &Name = "",
                  MDNode *FPMathTag = nullptr) {
  if (auto *VC = dyn_cast<Constant>(V))
    return Insert(Folder.CreateFNeg(VC), Name);
  return Insert(setFPAttrs(UnaryOperator::CreateFNeg(V), FPMathTag, FMF),
                Name);
}

/// Copy fast-math-flags from an instruction rather than using the builder's
/// default FMF.
Value *CreateFNegFMF(Value *V, Instruction *FMFSource,
                     const Twine &Name = "") {
 if (auto *VC = dyn_cast<Constant>(V))
   return Insert(Folder.CreateFNeg(VC), Name);
 return Insert(setFPAttrs(UnaryOperator::CreateFNeg(V), nullptr,
                          FMFSource->getFastMathFlags()),
               Name);
}

Value *CreateNot(Value *V, const Twine &Name = "") {
  if (auto *VC = dyn_cast<Constant>(V))
    return Insert(Folder.CreateNot(VC), Name);
  return Insert(BinaryOperator::CreateNot(V), Name);
}

Value *CreateUnOp(Instruction::UnaryOps Opc,
                  Value *V, const Twine &Name = "",
                  MDNode *FPMathTag = nullptr) {
  if (auto *VC = dyn_cast<Constant>(V))
    return Insert(Folder.CreateUnOp(Opc, VC), Name);
  Instruction *UnOp = UnaryOperator::Create(Opc, V);
  if (isa<FPMathOperator>(UnOp))
    setFPAttrs(UnOp, FPMathTag, FMF);
  return Insert(UnOp, Name);
}

/// Create either a UnaryOperator or BinaryOperator depending on \p Opc.
/// Correct number of operands must be passed accordingly.
Value *CreateNAryOp(unsigned Opc, ArrayRef<Value *> Ops,
                    const Twine &Name = "", MDNode *FPMathTag = nullptr);

//===--------------------------------------------------------------------===//
// Instruction creation methods: Memory Instructions
//===--------------------------------------------------------------------===//

AllocaInst *CreateAlloca(Type *Ty, unsigned AddrSpace,
                         Value *ArraySize = nullptr, const Twine &Name = "") {
  const DataLayout &DL = BB->getModule()->getDataLayout();
  Align AllocaAlign = DL.getPrefTypeAlign(Ty);
  return Insert(new AllocaInst(Ty, AddrSpace, ArraySize, AllocaAlign), Name);
}

AllocaInst *CreateAlloca(Type *Ty, Value *ArraySize = nullptr,
                         const Twine &Name = "") {
  const DataLayout &DL = BB->getModule()->getDataLayout();
  Align AllocaAlign = DL.getPrefTypeAlign(Ty);
  unsigned AddrSpace = DL.getAllocaAddrSpace();
  return Insert(new AllocaInst(Ty, AddrSpace, ArraySize, AllocaAlign), Name);
}

/// Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of
/// converting the string to 'bool' for the isVolatile parameter.
LoadInst *CreateLoad(Type *Ty, Value *Ptr, const char *Name) {
  return CreateAlignedLoad(Ty, Ptr, MaybeAlign(), Name);
}

LoadInst *CreateLoad(Type *Ty, Value *Ptr, const Twine &Name = "") {
  return CreateAlignedLoad(Ty, Ptr, MaybeAlign(), Name);
37
←
Calling 'IRBuilderBase::CreateAlignedLoad'→
}

LoadInst *CreateLoad(Type *Ty, Value *Ptr, bool isVolatile,
                     const Twine &Name = "") {
  return CreateAlignedLoad(Ty, Ptr, MaybeAlign(), isVolatile, Name);
}

// Deprecated [opaque pointer types]
LLVM_ATTRIBUTE_DEPRECATED(LoadInst *CreateLoad(Value *Ptr,[[deprecated("Use the version that explicitly specifies the "
 "loaded type instead")]] LoadInst *CreateLoad(Value *Ptr, const
 char *Name)
                                               const char *Name),[[deprecated("Use the version that explicitly specifies the "
 "loaded type instead")]] LoadInst *CreateLoad(Value *Ptr, const
 char *Name)
                          "Use the version that explicitly specifies the "[[deprecated("Use the version that explicitly specifies the "
 "loaded type instead")]] LoadInst *CreateLoad(Value *Ptr, const
 char *Name)
                          "loaded type instead")[[deprecated("Use the version that explicitly specifies the "
 "loaded type instead")]] LoadInst *CreateLoad(Value *Ptr, const
 char *Name) {
  return CreateLoad(Ptr->getType()->getPointerElementType(), Ptr, Name);
}

// Deprecated [opaque pointer types]
LLVM_ATTRIBUTE_DEPRECATED(LoadInst *CreateLoad(Value *Ptr,[[deprecated("Use the version that explicitly specifies the "
 "loaded type instead")]] LoadInst *CreateLoad(Value *Ptr, const
 Twine &Name = "")
                                               const Twine &Name = ""),[[deprecated("Use the version that explicitly specifies the "
 "loaded type instead")]] LoadInst *CreateLoad(Value *Ptr, const
 Twine &Name = "")
                          "Use the version that explicitly specifies the "[[deprecated("Use the version that explicitly specifies the "
 "loaded type instead")]] LoadInst *CreateLoad(Value *Ptr, const
 Twine &Name = "")
                          "loaded type instead")[[deprecated("Use the version that explicitly specifies the "
 "loaded type instead")]] LoadInst *CreateLoad(Value *Ptr, const
 Twine &Name = "") {
  return CreateLoad(Ptr->getType()->getPointerElementType(), Ptr, Name);
}

// Deprecated [opaque pointer types]
LLVM_ATTRIBUTE_DEPRECATED(LoadInst *CreateLoad(Value *Ptr,[[deprecated("Use the version that explicitly specifies the "
 "loaded type instead")]] LoadInst *CreateLoad(Value *Ptr, bool
 isVolatile, const Twine &Name = "")
                                               bool isVolatile,[[deprecated("Use the version that explicitly specifies the "
 "loaded type instead")]] LoadInst *CreateLoad(Value *Ptr, bool
 isVolatile, const Twine &Name = "")
                                               const Twine &Name = ""),[[deprecated("Use the version that explicitly specifies the "
 "loaded type instead")]] LoadInst *CreateLoad(Value *Ptr, bool
 isVolatile, const Twine &Name = "")
                          "Use the version that explicitly specifies the "[[deprecated("Use the version that explicitly specifies the "
 "loaded type instead")]] LoadInst *CreateLoad(Value *Ptr, bool
 isVolatile, const Twine &Name = "")
                          "loaded type instead")[[deprecated("Use the version that explicitly specifies the "
 "loaded type instead")]] LoadInst *CreateLoad(Value *Ptr, bool
 isVolatile, const Twine &Name = "") {
  return CreateLoad(Ptr->getType()->getPointerElementType(), Ptr, isVolatile,
                    Name);
}

StoreInst *CreateStore(Value *Val, Value *Ptr, bool isVolatile = false) {
  return CreateAlignedStore(Val, Ptr, MaybeAlign(), isVolatile);
}

LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align,
                            const char *Name) {
  return CreateAlignedLoad(Ty, Ptr, Align, /*isVolatile*/false, Name);
}

LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align,
                            const Twine &Name = "") {
  return CreateAlignedLoad(Ty, Ptr, Align, /*isVolatile*/false, Name);
38
←
Calling 'IRBuilderBase::CreateAlignedLoad'→
}

LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align,
                            bool isVolatile, const Twine &Name = "") {
  if (!Align) {
39
←
Calling 'Optional::operator bool'→
47
←
Returning from 'Optional::operator bool'→
48
←
Taking true branch→
    const DataLayout &DL = BB->getModule()->getDataLayout();
49
←
Called C++ object pointer is null
    Align = DL.getABITypeAlign(Ty);
  }
  return Insert(new LoadInst(Ty, Ptr, Twine(), isVolatile, *Align), Name);
}

// Deprecated [opaque pointer types]
LLVM_ATTRIBUTE_DEPRECATED(LoadInst *CreateAlignedLoad(Value *Ptr,[[deprecated("Use the version that explicitly specifies the "
 "loaded type instead")]] LoadInst *CreateAlignedLoad(Value *
Ptr, MaybeAlign Align, const char *Name)
                                                      MaybeAlign Align,[[deprecated("Use the version that explicitly specifies the "
 "loaded type instead")]] LoadInst *CreateAlignedLoad(Value *
Ptr, MaybeAlign Align, const char *Name)
                                                      const char *Name),[[deprecated("Use the version that explicitly specifies the "
 "loaded type instead")]] LoadInst *CreateAlignedLoad(Value *
Ptr, MaybeAlign Align, const char *Name)
                          "Use the version that explicitly specifies the "[[deprecated("Use the version that explicitly specifies the "
 "loaded type instead")]] LoadInst *CreateAlignedLoad(Value *
Ptr, MaybeAlign Align, const char *Name)
                          "loaded type instead")[[deprecated("Use the version that explicitly specifies the "
 "loaded type instead")]] LoadInst *CreateAlignedLoad(Value *
Ptr, MaybeAlign Align, const char *Name) {
  return CreateAlignedLoad(Ptr->getType()->getPointerElementType(), Ptr,
                           Align, Name);
}
// Deprecated [opaque pointer types]
LLVM_ATTRIBUTE_DEPRECATED(LoadInst *CreateAlignedLoad(Value *Ptr,[[deprecated("Use the version that explicitly specifies the "
 "loaded type instead")]] LoadInst *CreateAlignedLoad(Value *
Ptr, MaybeAlign Align, const Twine &Name = "")
                                                      MaybeAlign Align,[[deprecated("Use the version that explicitly specifies the "
 "loaded type instead")]] LoadInst *CreateAlignedLoad(Value *
Ptr, MaybeAlign Align, const Twine &Name = "")
                                                      const Twine &Name = ""),[[deprecated("Use the version that explicitly specifies the "
 "loaded type instead")]] LoadInst *CreateAlignedLoad(Value *
Ptr, MaybeAlign Align, const Twine &Name = "")
                          "Use the version that explicitly specifies the "[[deprecated("Use the version that explicitly specifies the "
 "loaded type instead")]] LoadInst *CreateAlignedLoad(Value *
Ptr, MaybeAlign Align, const Twine &Name = "")
                          "loaded type instead")[[deprecated("Use the version that explicitly specifies the "
 "loaded type instead")]] LoadInst *CreateAlignedLoad(Value *
Ptr, MaybeAlign Align, const Twine &Name = "") {
  return CreateAlignedLoad(Ptr->getType()->getPointerElementType(), Ptr,
                           Align, Name);
}
// Deprecated [opaque pointer types]
LLVM_ATTRIBUTE_DEPRECATED(LoadInst *CreateAlignedLoad(Value *Ptr,[[deprecated("Use the version that explicitly specifies the "
 "loaded type instead")]] LoadInst *CreateAlignedLoad(Value *
Ptr, MaybeAlign Align, bool isVolatile, const Twine &Name
 = "")
                                                      MaybeAlign Align,[[deprecated("Use the version that explicitly specifies the "
 "loaded type instead")]] LoadInst *CreateAlignedLoad(Value *
Ptr, MaybeAlign Align, bool isVolatile, const Twine &Name
 = "")
                                                      bool isVolatile,[[deprecated("Use the version that explicitly specifies the "
 "loaded type instead")]] LoadInst *CreateAlignedLoad(Value *
Ptr, MaybeAlign Align, bool isVolatile, const Twine &Name
 = "")
                                                      const Twine &Name = ""),[[deprecated("Use the version that explicitly specifies the "
 "loaded type instead")]] LoadInst *CreateAlignedLoad(Value *
Ptr, MaybeAlign Align, bool isVolatile, const Twine &Name
 = "")
                          "Use the version that explicitly specifies the "[[deprecated("Use the version that explicitly specifies the "
 "loaded type instead")]] LoadInst *CreateAlignedLoad(Value *
Ptr, MaybeAlign Align, bool isVolatile, const Twine &Name
 = "")
                          "loaded type instead")[[deprecated("Use the version that explicitly specifies the "
 "loaded type instead")]] LoadInst *CreateAlignedLoad(Value *
Ptr, MaybeAlign Align, bool isVolatile, const Twine &Name
 = "") {
  return CreateAlignedLoad(Ptr->getType()->getPointerElementType(), Ptr,
                           Align, isVolatile, Name);
}

StoreInst *CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align,
                              bool isVolatile = false) {
  if (!Align) {
    const DataLayout &DL = BB->getModule()->getDataLayout();
    Align = DL.getABITypeAlign(Val->getType());
  }
  return Insert(new StoreInst(Val, Ptr, isVolatile, *Align));
}
FenceInst *CreateFence(AtomicOrdering Ordering,
                       SyncScope::ID SSID = SyncScope::System,
                       const Twine &Name = "") {
  return Insert(new FenceInst(Context, Ordering, SSID), Name);
}

AtomicCmpXchgInst *
CreateAtomicCmpXchg(Value *Ptr, Value *Cmp, Value *New, MaybeAlign Align,
                    AtomicOrdering SuccessOrdering,
                    AtomicOrdering FailureOrdering,
                    SyncScope::ID SSID = SyncScope::System) {
  if (!Align) {
    const DataLayout &DL = BB->getModule()->getDataLayout();
    Align = llvm::Align(DL.getTypeStoreSize(New->getType()));
  }

  return Insert(new AtomicCmpXchgInst(Ptr, Cmp, New, *Align, SuccessOrdering,
                                      FailureOrdering, SSID));
}

AtomicRMWInst *CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr,
                               Value *Val, MaybeAlign Align,
                               AtomicOrdering Ordering,
                               SyncScope::ID SSID = SyncScope::System) {
  if (!Align) {
    const DataLayout &DL = BB->getModule()->getDataLayout();
    Align = llvm::Align(DL.getTypeStoreSize(Val->getType()));
  }

  return Insert(new AtomicRMWInst(Op, Ptr, Val, *Align, Ordering, SSID));
}

Value *CreateGEP(Value *Ptr, ArrayRef<Value *> IdxList,
                 const Twine &Name = "") {
  return CreateGEP(nullptr, Ptr, IdxList, Name);
}

Value *CreateGEP(Type *Ty, Value *Ptr, ArrayRef<Value *> IdxList,
                 const Twine &Name = "") {
  if (auto *PC = dyn_cast<Constant>(Ptr)) {
    // Every index must be constant.
    size_t i, e;
    for (i = 0, e = IdxList.size(); i != e; ++i)
      if (!isa<Constant>(IdxList[i]))
        break;
    if (i == e)
      return Insert(Folder.CreateGetElementPtr(Ty, PC, IdxList), Name);
  }
  return Insert(GetElementPtrInst::Create(Ty, Ptr, IdxList), Name);
}

Value *CreateInBoundsGEP(Value *Ptr, ArrayRef<Value *> IdxList,
                         const Twine &Name = "") {
  return CreateInBoundsGEP(nullptr, Ptr, IdxList, Name);
}

Value *CreateInBoundsGEP(Type *Ty, Value *Ptr, ArrayRef<Value *> IdxList,
                         const Twine &Name = "") {
  if (auto *PC = dyn_cast<Constant>(Ptr)) {
    // Every index must be constant.
    size_t i, e;
    for (i = 0, e = IdxList.size(); i != e; ++i)
      if (!isa<Constant>(IdxList[i]))
        break;
    if (i == e)
      return Insert(Folder.CreateInBoundsGetElementPtr(Ty, PC, IdxList),
                    Name);
  }
  return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, IdxList), Name);
}

Value *CreateGEP(Value *Ptr, Value *Idx, const Twine &Name = "") {
  return CreateGEP(nullptr, Ptr, Idx, Name);
}

Value *CreateGEP(Type *Ty, Value *Ptr, Value *Idx, const Twine &Name = "") {
  if (auto *PC = dyn_cast<Constant>(Ptr))
    if (auto *IC = dyn_cast<Constant>(Idx))
      return Insert(Folder.CreateGetElementPtr(Ty, PC, IC), Name);
  return Insert(GetElementPtrInst::Create(Ty, Ptr, Idx), Name);
}

Value *CreateInBoundsGEP(Type *Ty, Value *Ptr, Value *Idx,
                         const Twine &Name = "") {
  if (auto *PC = dyn_cast<Constant>(Ptr))
    if (auto *IC = dyn_cast<Constant>(Idx))
      return Insert(Folder.CreateInBoundsGetElementPtr(Ty, PC, IC), Name);
  return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idx), Name);
}

Value *CreateConstGEP1_32(Value *Ptr, unsigned Idx0, const Twine &Name = "") {
  return CreateConstGEP1_32(nullptr, Ptr, Idx0, Name);
}

Value *CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0,
                          const Twine &Name = "") {
  Value *Idx = ConstantInt::get(Type::getInt32Ty(Context), Idx0);

  if (auto *PC = dyn_cast<Constant>(Ptr))
    return Insert(Folder.CreateGetElementPtr(Ty, PC, Idx), Name);

  return Insert(GetElementPtrInst::Create(Ty, Ptr, Idx), Name);
}

Value *CreateConstInBoundsGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0,
                                  const Twine &Name = "") {
  Value *Idx = ConstantInt::get(Type::getInt32Ty(Context), Idx0);

  if (auto *PC = dyn_cast<Constant>(Ptr))
    return Insert(Folder.CreateInBoundsGetElementPtr(Ty, PC, Idx), Name);

  return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idx), Name);
}

Value *CreateConstGEP2_32(Type *Ty, Value *Ptr, unsigned Idx0, unsigned Idx1,
                          const Twine &Name = "") {
  Value *Idxs[] = {
    ConstantInt::get(Type::getInt32Ty(Context), Idx0),
    ConstantInt::get(Type::getInt32Ty(Context), Idx1)
  };

  if (auto *PC = dyn_cast<Constant>(Ptr))
    return Insert(Folder.CreateGetElementPtr(Ty, PC, Idxs), Name);

  return Insert(GetElementPtrInst::Create(Ty, Ptr, Idxs), Name);
}

Value *CreateConstInBoundsGEP2_32(Type *Ty, Value *Ptr, unsigned Idx0,
                                  unsigned Idx1, const Twine &Name = "") {
  Value *Idxs[] = {
    ConstantInt::get(Type::getInt32Ty(Context), Idx0),
    ConstantInt::get(Type::getInt32Ty(Context), Idx1)
  };

  if (auto *PC = dyn_cast<Constant>(Ptr))
    return Insert(Folder.CreateInBoundsGetElementPtr(Ty, PC, Idxs), Name);

  return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idxs), Name);
}

Value *CreateConstGEP1_64(Type *Ty, Value *Ptr, uint64_t Idx0,
                          const Twine &Name = "") {
  Value *Idx = ConstantInt::get(Type::getInt64Ty(Context), Idx0);

  if (auto *PC = dyn_cast<Constant>(Ptr))
    return Insert(Folder.CreateGetElementPtr(Ty, PC, Idx), Name);

  return Insert(GetElementPtrInst::Create(Ty, Ptr, Idx), Name);
}

Value *CreateConstGEP1_64(Value *Ptr, uint64_t Idx0, const Twine &Name = "") {
  return CreateConstGEP1_64(nullptr, Ptr, Idx0, Name);
}

Value *CreateConstInBoundsGEP1_64(Type *Ty, Value *Ptr, uint64_t Idx0,
                                  const Twine &Name = "") {
  Value *Idx = ConstantInt::get(Type::getInt64Ty(Context), Idx0);

  if (auto *PC = dyn_cast<Constant>(Ptr))
    return Insert(Folder.CreateInBoundsGetElementPtr(Ty, PC, Idx), Name);

  return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idx), Name);
}

Value *CreateConstInBoundsGEP1_64(Value *Ptr, uint64_t Idx0,
                                  const Twine &Name = "") {
  return CreateConstInBoundsGEP1_64(nullptr, Ptr, Idx0, Name);
}

Value *CreateConstGEP2_64(Type *Ty, Value *Ptr, uint64_t Idx0, uint64_t Idx1,
                          const Twine &Name = "") {
  Value *Idxs[] = {
    ConstantInt::get(Type::getInt64Ty(Context), Idx0),
    ConstantInt::get(Type::getInt64Ty(Context), Idx1)
  };

  if (auto *PC = dyn_cast<Constant>(Ptr))
    return Insert(Folder.CreateGetElementPtr(Ty, PC, Idxs), Name);

  return Insert(GetElementPtrInst::Create(Ty, Ptr, Idxs), Name);
}

Value *CreateConstGEP2_64(Value *Ptr, uint64_t Idx0, uint64_t Idx1,
                          const Twine &Name = "") {
  return CreateConstGEP2_64(nullptr, Ptr, Idx0, Idx1, Name);
}

Value *CreateConstInBoundsGEP2_64(Type *Ty, Value *Ptr, uint64_t Idx0,
                                  uint64_t Idx1, const Twine &Name = "") {
  Value *Idxs[] = {
    ConstantInt::get(Type::getInt64Ty(Context), Idx0),
    ConstantInt::get(Type::getInt64Ty(Context), Idx1)
  };

  if (auto *PC = dyn_cast<Constant>(Ptr))
    return Insert(Folder.CreateInBoundsGetElementPtr(Ty, PC, Idxs), Name);

  return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idxs), Name);
}

Value *CreateConstInBoundsGEP2_64(Value *Ptr, uint64_t Idx0, uint64_t Idx1,
                                  const Twine &Name = "") {
  return CreateConstInBoundsGEP2_64(nullptr, Ptr, Idx0, Idx1, Name);
}

Value *CreateStructGEP(Type *Ty, Value *Ptr, unsigned Idx,
                       const Twine &Name = "") {
  return CreateConstInBoundsGEP2_32(Ty, Ptr, 0, Idx, Name);
}

Value *CreateStructGEP(Value *Ptr, unsigned Idx, const Twine &Name = "") {
  return CreateConstInBoundsGEP2_32(nullptr, Ptr, 0, Idx, Name);
}

/// Same as CreateGlobalString, but return a pointer with "i8*" type
/// instead of a pointer to array of i8.
///
/// If no module is given via \p M, it is take from the insertion point basic
/// block.
Constant *CreateGlobalStringPtr(StringRef Str, const Twine &Name = "",
                                unsigned AddressSpace = 0,
                                Module *M = nullptr) {
  GlobalVariable *GV = CreateGlobalString(Str, Name, AddressSpace, M);
  Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0);
  Constant *Indices[] = {Zero, Zero};
  return ConstantExpr::getInBoundsGetElementPtr(GV->getValueType(), GV,
                                                Indices);
}

//===--------------------------------------------------------------------===//
// Instruction creation methods: Cast/Conversion Operators
//===--------------------------------------------------------------------===//

Value *CreateTrunc(Value *V, Type *DestTy, const Twine &Name = "") {
  return CreateCast(Instruction::Trunc, V, DestTy, Name);
}

Value *CreateZExt(Value *V, Type *DestTy, const Twine &Name = "") {
  return CreateCast(Instruction::ZExt, V, DestTy, Name);
}

Value *CreateSExt(Value *V, Type *DestTy, const Twine &Name = "") {
  return CreateCast(Instruction::SExt, V, DestTy, Name);
}

/// Create a ZExt or Trunc from the integer value V to DestTy. Return
/// the value untouched if the type of V is already DestTy.
Value *CreateZExtOrTrunc(Value *V, Type *DestTy,
                         const Twine &Name = "") {
  assert(V->getType()->isIntOrIntVectorTy() &&((V->getType()->isIntOrIntVectorTy() && DestTy->
isIntOrIntVectorTy() && "Can only zero extend/truncate integers!"
) ? static_cast<void> (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only zero extend/truncate integers!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 1988, __PRETTY_FUNCTION__))
         DestTy->isIntOrIntVectorTy() &&((V->getType()->isIntOrIntVectorTy() && DestTy->
isIntOrIntVectorTy() && "Can only zero extend/truncate integers!"
) ? static_cast<void> (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only zero extend/truncate integers!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 1988, __PRETTY_FUNCTION__))
         "Can only zero extend/truncate integers!")((V->getType()->isIntOrIntVectorTy() && DestTy->
isIntOrIntVectorTy() && "Can only zero extend/truncate integers!"
) ? static_cast<void> (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only zero extend/truncate integers!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 1988, __PRETTY_FUNCTION__));
  Type *VTy = V->getType();
  if (VTy->getScalarSizeInBits() < DestTy->getScalarSizeInBits())
    return CreateZExt(V, DestTy, Name);
  if (VTy->getScalarSizeInBits() > DestTy->getScalarSizeInBits())
    return CreateTrunc(V, DestTy, Name);
  return V;
}

/// Create a SExt or Trunc from the integer value V to DestTy. Return
/// the value untouched if the type of V is already DestTy.
Value *CreateSExtOrTrunc(Value *V, Type *DestTy,
                         const Twine &Name = "") {
  assert(V->getType()->isIntOrIntVectorTy() &&((V->getType()->isIntOrIntVectorTy() && DestTy->
isIntOrIntVectorTy() && "Can only sign extend/truncate integers!"
) ? static_cast<void> (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only sign extend/truncate integers!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 2003, __PRETTY_FUNCTION__))
         DestTy->isIntOrIntVectorTy() &&((V->getType()->isIntOrIntVectorTy() && DestTy->
isIntOrIntVectorTy() && "Can only sign extend/truncate integers!"
) ? static_cast<void> (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only sign extend/truncate integers!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 2003, __PRETTY_FUNCTION__))
         "Can only sign extend/truncate integers!")((V->getType()->isIntOrIntVectorTy() && DestTy->
isIntOrIntVectorTy() && "Can only sign extend/truncate integers!"
) ? static_cast<void> (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only sign extend/truncate integers!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/IR/IRBuilder.h"
, 2003, __PRETTY_FUNCTION__));
  Type *VTy = V->getType();
  if (VTy->getScalarSizeInBits() < DestTy->getScalarSizeInBits())
    return CreateSExt(V, DestTy, Name);
  if (VTy->getScalarSizeInBits() > DestTy->getScalarSizeInBits())
    return CreateTrunc(V, DestTy, Name);
  return V;
}

Value *CreateFPToUI(Value *V, Type *DestTy, const Twine &Name = "") {
  if (IsFPConstrained)
    return CreateConstrainedFPCast(Intrinsic::experimental_constrained_fptoui,
                                   V, DestTy, nullptr, Name);
  return CreateCast(Instruction::FPToUI, V, DestTy, Name);
}

Value *CreateFPToSI(Value *V, Type *DestTy, const Twine &Name = "") {
  if (IsFPConstrained)
    return CreateConstrainedFPCast(Intrinsic::experimental_constrained_fptosi,
                                   V, DestTy, nullptr, Name);
  return CreateCast(Instruction::FPToSI, V, DestTy, Name);
}

Value *CreateUIToFP(Value *V, Type *DestTy, const Twine &Name = ""){
  if (IsFPConstrained)
    return CreateConstrainedFPCast(Intrinsic::experimental_constrained_uitofp,
                                   V, DestTy, nullptr, Name);
  return CreateCast(Instruction::UIToFP, V, DestTy, Name);
}

Value *CreateSIToFP(Value *V, Type *DestTy, const Twine &Name = ""){
  if (IsFPConstrained)
    return CreateConstrainedFPCast(Intrinsic::experimental_constrained_sitofp,
                                   V, DestTy, nullptr, Name);
  return CreateCast(Instruction::SIToFP, V, DestTy, Name);
}

Value *CreateFPTrunc(Value *V, Type *DestTy,
                     const Twine &Name = "") {
  if (IsFPConstrained)
    return CreateConstrainedFPCast(
        Intrinsic::experimental_constrained_fptrunc, V, DestTy, nullptr,
        Name);
  return CreateCast(Instruction::FPTrunc, V, DestTy, Name);
}

Value *CreateFPExt(Value *V, Type *DestTy, const Twine &Name = "") {
  if (IsFPConstrained)
    return CreateConstrainedFPCast(Intrinsic::experimental_constrained_fpext,
                                   V, DestTy, nullptr, Name);
  return CreateCast(Instruction::FPExt, V, DestTy, Name);
}

Value *CreatePtrToInt(Value *V, Type *DestTy,
                      const Twine &Name = "") {
  return CreateCast(Instruction::PtrToInt, V, DestTy, Name);
}

Value *CreateIntToPtr(Value *V, Type *DestTy,
                      const Twine &Name = "") {
  return CreateCast(Instruction::IntToPtr, V, DestTy, Name);
}

Value *CreateBitCast(Value *V, Type *DestTy,
                     const Twine &Name = "") {
  return CreateCast(Instruction::BitCast, V, DestTy, Name);
}

Value *CreateAddrSpaceCast(Value *V, Type *DestTy,
                           const Twine &Name = "") {
  return CreateCast(Instruction::AddrSpaceCast, V, DestTy, Name);
}

Value *CreateZExtOrBitCast(Value *V, Type *DestTy,
                           const Twine &Name = "") {
  if (V->getType() == DestTy)
    return V;
  if (auto *VC = dyn_cast<Constant>(V))
    return Insert(Folder.CreateZExtOrBitCast(VC, DestTy), Name);
  return Insert(CastInst::CreateZExtOrBitCast(V, DestTy), Name);
}

Value *CreateSExtOrBitCast(Value *V, Type *DestTy,
                           const Twine &Name = "") {
  if (V->getType() == DestTy)
    return V;
  if (auto *VC = dyn_cast<Constant>(V))
    return Insert(Folder.CreateSExtOrBitCast(VC, DestTy), Name);
  return Insert(CastInst::CreateSExtOrBitCast(V, DestTy), Name);
}

Value *CreateTruncOrBitCast(Value *V, Type *DestTy,
                            const Twine &Name = "") {
  if (V->getType() == DestTy)
    return V;
  if (auto *VC = dyn_cast<Constant>(V))
    return Insert(Folder.CreateTruncOrBitCast(VC, DestTy), Name);
  return Insert(CastInst::CreateTruncOrBitCast(V, DestTy), Name);
}

Value *CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy,
                  const Twine &Name = "") {
  if (V->getType() == DestTy)
    return V;
  if (auto *VC = dyn_cast<Constant>(V))
    return Insert(Folder.CreateCast(Op, VC, DestTy), Name);
  return Insert(CastInst::Create(Op, V, DestTy), Name);
}

Value *CreatePointerCast(Value *V, Type *DestTy,
                         const Twine &Name = "") {
  if (V->getType() == DestTy)
    return V;
  if (auto *VC = dyn_cast<Constant>(V))
    return Insert(Folder.CreatePointerCast(VC, DestTy), Name);
  return Insert(CastInst::CreatePointerCast(V, DestTy), Name);
}

Value *CreatePointerBitCastOrAddrSpaceCast(Value *V, Type *DestTy,
                                           const Twine &Name = "") {
  if (V->getType() == DestTy)
    return V;

  if (auto *VC = dyn_cast<Constant>(V)) {
    return Insert(Folder.CreatePointerBitCastOrAddrSpaceCast(VC, DestTy),
                  Name);
  }

  return Insert(CastInst::CreatePointerBitCastOrAddrSpaceCast(V, DestTy),
                Name);
}

Value *CreateIntCast(Value *V, Type *DestTy, bool isSigned,
                     const Twine &Name = "") {
  if (V->getType() == DestTy)
    return V;
  if (auto *VC = dyn_cast<Constant>(V))
    return Insert(Folder.CreateIntCast(VC, DestTy, isSigned), Name);
  return Insert(CastInst::CreateIntegerCast(V, DestTy, isSigned), Name);
}

Value *CreateBitOrPointerCast(Value *V, Type *DestTy,
                              const Twine &Name = "") {
  if (V->getType() == DestTy)
    return V;
  if (V->getType()->isPtrOrPtrVectorTy() && DestTy->isIntOrIntVectorTy())
    return CreatePtrToInt(V, DestTy, Name);
  if (V->getType()->isIntOrIntVectorTy() && DestTy->isPtrOrPtrVectorTy())
    return CreateIntToPtr(V, DestTy, Name);

  return CreateBitCast(V, DestTy, Name);
}

Value *CreateFPCast(Value *V, Type *DestTy, const Twine &Name = "") {
  if (V->getType() == DestTy)
    return V;
  if (auto *VC = dyn_cast<Constant>(V))
    return Insert(Folder.CreateFPCast(VC, DestTy), Name);
  return Insert(CastInst::CreateFPCast(V, DestTy), Name);
}

CallInst *CreateConstrainedFPCast(
    Intrinsic::ID ID, Value *V, Type *DestTy,
    Instruction *FMFSource = nullptr, const Twine &Name = "",
    MDNode *FPMathTag = nullptr,
    Optional<RoundingMode> Rounding = None,
    Optional<fp::ExceptionBehavior> Except = None);

// Provided to resolve 'CreateIntCast(Ptr, Ptr, "...")', giving a
// compile time error, instead of converting the string to bool for the
// isSigned parameter.
Value *CreateIntCast(Value *, Type *, const char *) = delete;

//===--------------------------------------------------------------------===//
// Instruction creation methods: Compare Instructions
//===--------------------------------------------------------------------===//

Value *CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name = "") {
  return CreateICmp(ICmpInst::ICMP_EQ, LHS, RHS, Name);
}

Value *CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name = "") {
  return CreateICmp(ICmpInst::ICMP_NE, LHS, RHS, Name);
}

Value *CreateICmpUGT(Value *LHS, Value *RHS, const Twine &Name = "") {
  return CreateICmp(ICmpInst::ICMP_UGT, LHS, RHS, Name);
}

Value *CreateICmpUGE(Value *LHS, Value *RHS, const Twine &Name = "") {
  return CreateICmp(ICmpInst::ICMP_UGE, LHS, RHS, Name);
}

Value *CreateICmpULT(Value *LHS, Value *RHS, const Twine &Name = "") {
  return CreateICmp(ICmpInst::ICMP_ULT, LHS, RHS, Name);
}

Value *CreateICmpULE(Value *LHS, Value *RHS, const Twine &Name = "") {
  return CreateICmp(ICmpInst::ICMP_ULE, LHS, RHS, Name);
}

Value *CreateICmpSGT(Value *LHS, Value *RHS, const Twine &Name = "") {
  return CreateICmp(ICmpInst::ICMP_SGT, LHS, RHS, Name);
}

Value *CreateICmpSGE(Value *LHS, Value *RHS, const Twine &Name = "") {
  return CreateICmp(ICmpInst::ICMP_SGE, LHS, RHS, Name);
}

Value *CreateICmpSLT(Value *LHS, Value *RHS, const Twine &Name = "") {
  return CreateICmp(ICmpInst::ICMP_SLT, LHS, RHS, Name);
}

Value *CreateICmpSLE(Value *LHS, Value *RHS, const Twine &Name = "") {
  return CreateICmp(ICmpInst::ICMP_SLE, LHS, RHS, Name);
}

Value *CreateFCmpOEQ(Value *LHS, Value *RHS, const Twine &Name = "",
                     MDNode *FPMathTag = nullptr) {
  return CreateFCmp(FCmpInst::FCMP_OEQ, LHS, RHS, Name, FPMathTag);
}

Value *CreateFCmpOGT(Value *LHS, Value *RHS, const Twine &Name = "",
                     MDNode *FPMathTag = nullptr) {
  return CreateFCmp(FCmpInst::FCMP_OGT, LHS, RHS, Name, FPMathTag);
}

Value *CreateFCmpOGE(Value *LHS, Value *RHS, const Twine &Name = "",
                     MDNode *FPMathTag = nullptr) {
  return CreateFCmp(FCmpInst::FCMP_OGE, LHS, RHS, Name, FPMathTag);
}

Value *CreateFCmpOLT(Value *LHS, Value *RHS, const Twine &Name = "",
                     MDNode *FPMathTag = nullptr) {
  return CreateFCmp(FCmpInst::FCMP_OLT, LHS, RHS, Name, FPMathTag);
}

Value *CreateFCmpOLE(Value *LHS, Value *RHS, const Twine &Name = "",
                     MDNode *FPMathTag = nullptr) {
  return CreateFCmp(FCmpInst::FCMP_OLE, LHS, RHS, Name, FPMathTag);
}

Value *CreateFCmpONE(Value *LHS, Value *RHS, const Twine &Name = "",
                     MDNode *FPMathTag = nullptr) {
  return CreateFCmp(FCmpInst::FCMP_ONE, LHS, RHS, Name, FPMathTag);
}

Value *CreateFCmpORD(Value *LHS, Value *RHS, const Twine &Name = "",
                     MDNode *FPMathTag = nullptr) {
  return CreateFCmp(FCmpInst::FCMP_ORD, LHS, RHS, Name, FPMathTag);
}

Value *CreateFCmpUNO(Value *LHS, Value *RHS, const Twine &Name = "",
                     MDNode *FPMathTag = nullptr) {
  return CreateFCmp(FCmpInst::FCMP_UNO, LHS, RHS, Name, FPMathTag);
}

Value *CreateFCmpUEQ(Value *LHS, Value *RHS, const Twine &Name = "",
                     MDNode *FPMathTag = nullptr) {
  return CreateFCmp(FCmpInst::FCMP_UEQ, LHS, RHS, Name, FPMathTag);
}

Value *CreateFCmpUGT(Value *LHS, Value *RHS, const Twine &Name = "",
                     MDNode *FPMathTag = nullptr) {
  return CreateFCmp(FCmpInst::FCMP_UGT, LHS, RHS, Name, FPMathTag);
}

Value *CreateFCmpUGE(Value *LHS, Value *RHS, const Twine &Name = "",
                     MDNode *FPMathTag = nullptr) {
  return CreateFCmp(FCmpInst::FCMP_UGE, LHS, RHS, Name, FPMathTag);
}

Value *CreateFCmpULT(Value *LHS, Value *RHS, const Twine &Name = "",
                     MDNode *FPMathTag = nullptr) {
  return CreateFCmp(FCmpInst::FCMP_ULT, LHS, RHS, Name, FPMathTag);
}

Value *CreateFCmpULE(Value *LHS, Value *RHS, const Twine &Name = "",
                     MDNode *FPMathTag = nullptr) {
  return CreateFCmp(FCmpInst::FCMP_ULE, LHS, RHS, Name, FPMathTag);
}

Value *CreateFCmpUNE(Value *LHS, Value *RHS, const Twine &Name = "",
                     MDNode *FPMathTag = nullptr) {
  return CreateFCmp(FCmpInst::FCMP_UNE, LHS, RHS, Name, FPMathTag);
}

Value *CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS,
                  const Twine &Name = "") {
  if (auto *LC = dyn_cast<Constant>(LHS))
    if (auto *RC = dyn_cast<Constant>(RHS))
      return Insert(Folder.CreateICmp(P, LC, RC), Name);
  return Insert(new ICmpInst(P, LHS, RHS), Name);
}

// Create a quiet floating-point comparison (i.e. one that raises an FP
// exception only in the case where an input is a signaling NaN).
// Note that this differs from CreateFCmpS only if IsFPConstrained is true.
Value *CreateFCmp(CmpInst::Predicate P, Value *LHS, Value *RHS,
                  const Twine &Name = "", MDNode *FPMathTag = nullptr) {
  return CreateFCmpHelper(P, LHS, RHS, Name, FPMathTag, false);
}

Value *CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS,
                 const Twine &Name = "", MDNode *FPMathTag = nullptr) {
  return CmpInst::isFPPredicate(Pred)
             ? CreateFCmp(Pred, LHS, RHS, Name, FPMathTag)
             : CreateICmp(Pred, LHS, RHS, Name);
}

// Create a signaling floating-point comparison (i.e. one that raises an FP
// exception whenever an input is any NaN, signaling or quiet).
// Note that this differs from CreateFCmp only if IsFPConstrained is true.
Value *CreateFCmpS(CmpInst::Predicate P, Value *LHS, Value *RHS,
                   const Twine &Name = "", MDNode *FPMathTag = nullptr) {
  return CreateFCmpHelper(P, LHS, RHS, Name, FPMathTag, true);
}

2321private:
// Helper routine to create either a signaling or a quiet FP comparison.
Value *CreateFCmpHelper(CmpInst::Predicate P, Value *LHS, Value *RHS,
                        const Twine &Name, MDNode *FPMathTag,
                        bool IsSignaling);

2327public:
CallInst *CreateConstrainedFPCmp(
    Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R,
    const Twine &Name = "", Optional<fp::ExceptionBehavior> Except = None);

//===--------------------------------------------------------------------===//
// Instruction creation methods: Other Instructions
//===--------------------------------------------------------------------===//

PHINode *CreatePHI(Type *Ty, unsigned NumReservedValues,
                   const Twine &Name = "") {
  PHINode *Phi = PHINode::Create(Ty, NumReservedValues);
  if (isa<FPMathOperator>(Phi))
    setFPAttrs(Phi, nullptr /* MDNode* */, FMF);
  return Insert(Phi, Name);
}

CallInst *CreateCall(FunctionType *FTy, Value *Callee,
                     ArrayRef<Value *> Args = None, const Twine &Name = "",
                     MDNode *FPMathTag = nullptr) {
  CallInst *CI = CallInst::Create(FTy, Callee, Args, DefaultOperandBundles);
  if (IsFPConstrained)
    setConstrainedFPCallAttr(CI);
  if (isa<FPMathOperator>(CI))
    setFPAttrs(CI, FPMathTag, FMF);
  return Insert(CI, Name);
}

CallInst *CreateCall(FunctionType *FTy, Value *Callee, ArrayRef<Value *> Args,
                     ArrayRef<OperandBundleDef> OpBundles,
                     const Twine &Name = "", MDNode *FPMathTag = nullptr) {
  CallInst *CI = CallInst::Create(FTy, Callee, Args, OpBundles);
  if (IsFPConstrained)
    setConstrainedFPCallAttr(CI);
  if (isa<FPMathOperator>(CI))
    setFPAttrs(CI, FPMathTag, FMF);
  return Insert(CI, Name);
}

CallInst *CreateCall(FunctionCallee Callee, ArrayRef<Value *> Args = None,
                     const Twine &Name = "", MDNode *FPMathTag = nullptr) {
  return CreateCall(Callee.getFunctionType(), Callee.getCallee(), Args, Name,
                    FPMathTag);
}

CallInst *CreateCall(FunctionCallee Callee, ArrayRef<Value *> Args,
                     ArrayRef<OperandBundleDef> OpBundles,
                     const Twine &Name = "", MDNode *FPMathTag = nullptr) {
  return CreateCall(Callee.getFunctionType(), Callee.getCallee(), Args,
                    OpBundles, Name, FPMathTag);
}

CallInst *CreateConstrainedFPCall(
    Function *Callee, ArrayRef<Value *> Args, const Twine &Name = "",
    Optional<RoundingMode> Rounding = None,
    Optional<fp::ExceptionBehavior> Except = None);

Value *CreateSelect(Value *C, Value *True, Value *False,
                    const Twine &Name = "", Instruction *MDFrom = nullptr);

VAArgInst *CreateVAArg(Value *List, Type *Ty, const Twine &Name = "") {
  return Insert(new VAArgInst(List, Ty), Name);
}

Value *CreateExtractElement(Value *Vec, Value *Idx,
                            const Twine &Name = "") {
  if (auto *VC = dyn_cast<Constant>(Vec))
    if (auto *IC = dyn_cast<Constant>(Idx))
      return Insert(Folder.CreateExtractElement(VC, IC), Name);
  return Insert(ExtractElementInst::Create(Vec, Idx), Name);
}

Value *CreateExtractElement(Value *Vec, uint64_t Idx,
                            const Twine &Name = "") {
  return CreateExtractElement(Vec, getInt64(Idx), Name);
}

Value *CreateInsertElement(Value *Vec, Value *NewElt, Value *Idx,
                           const Twine &Name = "") {
  if (auto *VC = dyn_cast<Constant>(Vec))
    if (auto *NC = dyn_cast<Constant>(NewElt))
      if (auto *IC = dyn_cast<Constant>(Idx))
        return Insert(Folder.CreateInsertElement(VC, NC, IC), Name);
  return Insert(InsertElementInst::Create(Vec, NewElt, Idx), Name);
}

Value *CreateInsertElement(Value *Vec, Value *NewElt, uint64_t Idx,
                           const Twine &Name = "") {
  return CreateInsertElement(Vec, NewElt, getInt64(Idx), Name);
}

Value *CreateShuffleVector(Value *V1, Value *V2, Value *Mask,
                           const Twine &Name = "") {
  SmallVector<int, 16> IntMask;
  ShuffleVectorInst::getShuffleMask(cast<Constant>(Mask), IntMask);
  return CreateShuffleVector(V1, V2, IntMask, Name);
}

LLVM_ATTRIBUTE_DEPRECATED(Value *CreateShuffleVector(Value *V1, Value *V2,[[deprecated("Pass indices as 'int' instead")]] Value *CreateShuffleVector
(Value *V1, Value *V2, ArrayRef<uint32_t> Mask, const Twine
 &Name = "")
                                                     ArrayRef<uint32_t> Mask,[[deprecated("Pass indices as 'int' instead")]] Value *CreateShuffleVector
(Value *V1, Value *V2, ArrayRef<uint32_t> Mask, const Twine
 &Name = "")
                                                     const Twine &Name = ""),[[deprecated("Pass indices as 'int' instead")]] Value *CreateShuffleVector
(Value *V1, Value *V2, ArrayRef<uint32_t> Mask, const Twine
 &Name = "")
                          "Pass indices as 'int' instead")[[deprecated("Pass indices as 'int' instead")]] Value *CreateShuffleVector
(Value *V1, Value *V2, ArrayRef<uint32_t> Mask, const Twine
 &Name = "") {
  SmallVector<int, 16> IntMask;
  IntMask.assign(Mask.begin(), Mask.end());
  return CreateShuffleVector(V1, V2, IntMask, Name);
}

/// See class ShuffleVectorInst for a description of the mask representation.
Value *CreateShuffleVector(Value *V1, Value *V2, ArrayRef<int> Mask,
                           const Twine &Name = "") {
  if (auto *V1C = dyn_cast<Constant>(V1))
    if (auto *V2C = dyn_cast<Constant>(V2))
      return Insert(Folder.CreateShuffleVector(V1C, V2C, Mask), Name);
  return Insert(new ShuffleVectorInst(V1, V2, Mask), Name);
}

/// Create a unary shuffle. The second vector operand of the IR instruction
/// is poison.
Value *CreateShuffleVector(Value *V, ArrayRef<int> Mask,
                           const Twine &Name = "") {
  return CreateShuffleVector(V, PoisonValue::get(V->getType()), Mask, Name);
}

Value *CreateExtractValue(Value *Agg,
                          ArrayRef<unsigned> Idxs,
                          const Twine &Name = "") {
  if (auto *AggC = dyn_cast<Constant>(Agg))
    return Insert(Folder.CreateExtractValue(AggC, Idxs), Name);
  return Insert(ExtractValueInst::Create(Agg, Idxs), Name);
}

Value *CreateInsertValue(Value *Agg, Value *Val,
                         ArrayRef<unsigned> Idxs,
                         const Twine &Name = "") {
  if (auto *AggC = dyn_cast<Constant>(Agg))
    if (auto *ValC = dyn_cast<Constant>(Val))
      return Insert(Folder.CreateInsertValue(AggC, ValC, Idxs), Name);
  return Insert(InsertValueInst::Create(Agg, Val, Idxs), Name);
}

LandingPadInst *CreateLandingPad(Type *Ty, unsigned NumClauses,
                                 const Twine &Name = "") {
  return Insert(LandingPadInst::Create(Ty, NumClauses), Name);
}

Value *CreateFreeze(Value *V, const Twine &Name = "") {
  return Insert(new FreezeInst(V), Name);
}

//===--------------------------------------------------------------------===//
// Utility creation methods
//===--------------------------------------------------------------------===//

/// Return an i1 value testing if \p Arg is null.
Value *CreateIsNull(Value *Arg, const Twine &Name = "") {
  return CreateICmpEQ(Arg, Constant::getNullValue(Arg->getType()),
                      Name);
}

/// Return an i1 value testing if \p Arg is not null.
Value *CreateIsNotNull(Value *Arg, const Twine &Name = "") {
  return CreateICmpNE(Arg, Constant::getNullValue(Arg->getType()),
                      Name);
}

/// Return the i64 difference between two pointer values, dividing out
/// the size of the pointed-to objects.
///
/// This is intended to implement C-style pointer subtraction. As such, the
/// pointers must be appropriately aligned for their element types and
/// pointing into the same object.
Value *CreatePtrDiff(Value *LHS, Value *RHS, const Twine &Name = "");

/// Create a launder.invariant.group intrinsic call. If Ptr type is
/// different from pointer to i8, it's casted to pointer to i8 in the same
/// address space before call and casted back to Ptr type after call.
Value *CreateLaunderInvariantGroup(Value *Ptr);

/// \brief Create a strip.invariant.group intrinsic call. If Ptr type is
/// different from pointer to i8, it's casted to pointer to i8 in the same
/// address space before call and casted back to Ptr type after call.
Value *CreateStripInvariantGroup(Value *Ptr);

/// Return a vector value that contains the vector V reversed
Value *CreateVectorReverse(Value *V, const Twine &Name = "");

/// Return a vector value that contains \arg V broadcasted to \p
/// NumElts elements.
Value *CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name = "");

/// Return a vector value that contains \arg V broadcasted to \p
/// EC elements.
Value *CreateVectorSplat(ElementCount EC, Value *V, const Twine &Name = "");

/// Return a value that has been extracted from a larger integer type.
Value *CreateExtractInteger(const DataLayout &DL, Value *From,
                            IntegerType *ExtractedTy, uint64_t Offset,
                            const Twine &Name);

Value *CreatePreserveArrayAccessIndex(Type *ElTy, Value *Base,
                                      unsigned Dimension, unsigned LastIndex,
                                      MDNode *DbgInfo);

Value *CreatePreserveUnionAccessIndex(Value *Base, unsigned FieldIndex,
                                      MDNode *DbgInfo);

Value *CreatePreserveStructAccessIndex(Type *ElTy, Value *Base,
                                       unsigned Index, unsigned FieldIndex,
                                       MDNode *DbgInfo);

2537private:
/// Helper function that creates an assume intrinsic call that
/// represents an alignment assumption on the provided pointer \p PtrValue
/// with offset \p OffsetValue and alignment value \p AlignValue.
CallInst *CreateAlignmentAssumptionHelper(const DataLayout &DL,
                                          Value *PtrValue, Value *AlignValue,
                                          Value *OffsetValue);

2545public:
/// Create an assume intrinsic call that represents an alignment
/// assumption on the provided pointer.
///
/// An optional offset can be provided, and if it is provided, the offset
/// must be subtracted from the provided pointer to get the pointer with the
/// specified alignment.
CallInst *CreateAlignmentAssumption(const DataLayout &DL, Value *PtrValue,
                                    unsigned Alignment,
                                    Value *OffsetValue = nullptr);

/// Create an assume intrinsic call that represents an alignment
/// assumption on the provided pointer.
///
/// An optional offset can be provided, and if it is provided, the offset
/// must be subtracted from the provided pointer to get the pointer with the
/// specified alignment.
///
/// This overload handles the condition where the Alignment is dependent
/// on an existing value rather than a static value.
CallInst *CreateAlignmentAssumption(const DataLayout &DL, Value *PtrValue,
                                    Value *Alignment,
                                    Value *OffsetValue = nullptr);
2568};

2570/// This provides a uniform API for creating instructions and inserting
2571/// them into a basic block: either at the end of a BasicBlock, or at a specific
2572/// iterator location in a block.
2573///
2574/// Note that the builder does not expose the full generality of LLVM
2575/// instructions.  For access to extra instruction properties, use the mutators
2576/// (e.g. setVolatile) on the instructions after they have been
2577/// created. Convenience state exists to specify fast-math flags and fp-math
2578/// tags.
2579///
2580/// The first template argument specifies a class to use for creating constants.
2581/// This defaults to creating minimally folded constants.  The second template
2582/// argument allows clients to specify custom insertion hooks that are called on
2583/// every newly created insertion.
2584template <typename FolderTy = ConstantFolder,
        typename InserterTy = IRBuilderDefaultInserter>
2586class IRBuilder : public IRBuilderBase {
2587private:
FolderTy Folder;
InserterTy Inserter;

2591public:
IRBuilder(LLVMContext &C, FolderTy Folder, InserterTy Inserter = InserterTy(),
          MDNode *FPMathTag = nullptr,
          ArrayRef<OperandBundleDef> OpBundles = None)
    : IRBuilderBase(C, this->Folder, this->Inserter, FPMathTag, OpBundles),
      Folder(Folder), Inserter(Inserter) {}

explicit IRBuilder(LLVMContext &C, MDNode *FPMathTag = nullptr,
                   ArrayRef<OperandBundleDef> OpBundles = None)
    : IRBuilderBase(C, this->Folder, this->Inserter, FPMathTag, OpBundles) {}

explicit IRBuilder(BasicBlock *TheBB, FolderTy Folder,
                   MDNode *FPMathTag = nullptr,
                   ArrayRef<OperandBundleDef> OpBundles = None)
    : IRBuilderBase(TheBB->getContext(), this->Folder, this->Inserter,
                    FPMathTag, OpBundles), Folder(Folder) {
  SetInsertPoint(TheBB);
}

explicit IRBuilder(BasicBlock *TheBB, MDNode *FPMathTag = nullptr,
                   ArrayRef<OperandBundleDef> OpBundles = None)
    : IRBuilderBase(TheBB->getContext(), this->Folder, this->Inserter,
                    FPMathTag, OpBundles) {
  SetInsertPoint(TheBB);
}

explicit IRBuilder(Instruction *IP, MDNode *FPMathTag = nullptr,
                   ArrayRef<OperandBundleDef> OpBundles = None)
    : IRBuilderBase(IP->getContext(), this->Folder, this->Inserter,
                    FPMathTag, OpBundles) {
  SetInsertPoint(IP);
}

IRBuilder(BasicBlock *TheBB, BasicBlock::iterator IP, FolderTy Folder,
          MDNode *FPMathTag = nullptr,
          ArrayRef<OperandBundleDef> OpBundles = None)
    : IRBuilderBase(TheBB->getContext(), this->Folder, this->Inserter,
                    FPMathTag, OpBundles), Folder(Folder) {
  SetInsertPoint(TheBB, IP);
}

IRBuilder(BasicBlock *TheBB, BasicBlock::iterator IP,
          MDNode *FPMathTag = nullptr,
          ArrayRef<OperandBundleDef> OpBundles = None)
    : IRBuilderBase(TheBB->getContext(), this->Folder, this->Inserter,
                    FPMathTag, OpBundles) {
  SetInsertPoint(TheBB, IP);
}

/// Avoid copying the full IRBuilder. Prefer using InsertPointGuard
/// or FastMathFlagGuard instead.
IRBuilder(const IRBuilder &) = delete;

InserterTy &getInserter() { return Inserter; }
2645};

2647// Create wrappers for C Binding types (see CBindingWrapping.h).
2648DEFINE_SIMPLE_CONVERSION_FUNCTIONS(IRBuilder<>, LLVMBuilderRef)inline IRBuilder<> *unwrap(LLVMBuilderRef P) { return reinterpret_cast
<IRBuilder<>*>(P); } inline LLVMBuilderRef wrap(const
 IRBuilder<> *P) { return reinterpret_cast<LLVMBuilderRef
>(const_cast<IRBuilder<>*>(P)); }

2650} // end namespace llvm

2652#endif // LLVM_IR_IRBUILDER_H

←

/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/ADT/Optional.h

1//===- Optional.h - Simple variant for passing optional values --*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9//  This file provides Optional, a template class modeled in the spirit of
10//  OCaml's 'opt' variant.  The idea is to strongly type whether or not
11//  a value can be optional.
12//
13//===----------------------------------------------------------------------===//
14 
15#ifndef LLVM_ADT_OPTIONAL_H
16#define LLVM_ADT_OPTIONAL_H
17 
18#include "llvm/ADT/Hashing.h"
19#include "llvm/ADT/None.h"
20#include "llvm/Support/Compiler.h"
21#include "llvm/Support/type_traits.h"
22#include <cassert>
23#include <memory>
24#include <new>
25#include <utility>
26 
27namespace llvm {
28 
29class raw_ostream;
30 
31namespace optional_detail {
32 
33struct in_place_t {};
34 
35/// Storage for any type.
36//
37// The specialization condition intentionally uses
38// llvm::is_trivially_copy_constructible instead of
39// std::is_trivially_copy_constructible.  GCC versions prior to 7.4 may
40// instantiate the copy constructor of `T` when
41// std::is_trivially_copy_constructible is instantiated.  This causes
42// compilation to fail if we query the trivially copy constructible property of
43// a class which is not copy constructible.
44//
45// The current implementation of OptionalStorage insists that in order to use
46// the trivial specialization, the value_type must be trivially copy
47// constructible and trivially copy assignable due to =default implementations
48// of the copy/move constructor/assignment.  It does not follow that this is
49// necessarily the case std::is_trivially_copyable is true (hence the expanded
50// specialization condition).
51//
52// The move constructible / assignable conditions emulate the remaining behavior
53// of std::is_trivially_copyable.
54template <typename T, bool = (llvm::is_trivially_copy_constructible<T>::value &&
55                              std::is_trivially_copy_assignable<T>::value &&
56                              (std::is_trivially_move_constructible<T>::value ||
57                               !std::is_move_constructible<T>::value) &&
58                              (std::is_trivially_move_assignable<T>::value ||
59                               !std::is_move_assignable<T>::value))>
60class OptionalStorage {
61  union {
62    char empty;
63    T value;
64  };
65  bool hasVal;
66 
67public:
68  ~OptionalStorage() { reset(); }
69 
70  constexpr OptionalStorage() noexcept : empty(), hasVal(false) {}
71 
72  constexpr OptionalStorage(OptionalStorage const &other) : OptionalStorage() {
73    if (other.hasValue()) {
74      emplace(other.value);
75    }
76  }
77  constexpr OptionalStorage(OptionalStorage &&other) : OptionalStorage() {
78    if (other.hasValue()) {
79      emplace(std::move(other.value));
80    }
81  }
82 
83  template <class... Args>
84  constexpr explicit OptionalStorage(in_place_t, Args &&... args)
85      : value(std::forward<Args>(args)...), hasVal(true) {}
86 
87  void reset() noexcept {
88    if (hasVal) {
89      value.~T();
90      hasVal = false;
91    }
92  }
93 
94  constexpr bool hasValue() const noexcept { return hasVal; }
95 
96  T &getValue() LLVM_LVALUE_FUNCTION& noexcept {
97    assert(hasVal)((hasVal) ? static_cast<void> (0) : __assert_fail ("hasVal"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/ADT/Optional.h"
, 97, __PRETTY_FUNCTION__));
98    return value;
99  }
100  constexpr T const &getValue() const LLVM_LVALUE_FUNCTION& noexcept {
101    assert(hasVal)((hasVal) ? static_cast<void> (0) : __assert_fail ("hasVal"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/ADT/Optional.h"
, 101, __PRETTY_FUNCTION__));
102    return value;
103  }
104#if LLVM_HAS_RVALUE_REFERENCE_THIS1
105  T &&getValue() && noexcept {
106    assert(hasVal)((hasVal) ? static_cast<void> (0) : __assert_fail ("hasVal"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/ADT/Optional.h"
, 106, __PRETTY_FUNCTION__));
107    return std::move(value);
108  }
109#endif
110 
111  template <class... Args> void emplace(Args &&... args) {
112    reset();
113    ::new ((void *)std::addressof(value)) T(std::forward<Args>(args)...);
114    hasVal = true;
115  }
116 
117  OptionalStorage &operator=(T const &y) {
118    if (hasValue()) {
119      value = y;
120    } else {
121      ::new ((void *)std::addressof(value)) T(y);
122      hasVal = true;
123    }
124    return *this;
125  }
126  OptionalStorage &operator=(T &&y) {
127    if (hasValue()) {
128      value = std::move(y);
129    } else {
130      ::new ((void *)std::addressof(value)) T(std::move(y));
131      hasVal = true;
132    }
133    return *this;
134  }
135 
136  OptionalStorage &operator=(OptionalStorage const &other) {
137    if (other.hasValue()) {
138      if (hasValue()) {
139        value = other.value;
140      } else {
141        ::new ((void *)std::addressof(value)) T(other.value);
142        hasVal = true;
143      }
144    } else {
145      reset();
146    }
147    return *this;
148  }
149 
150  OptionalStorage &operator=(OptionalStorage &&other) {
151    if (other.hasValue()) {
152      if (hasValue()) {
153        value = std::move(other.value);
154      } else {
155        ::new ((void *)std::addressof(value)) T(std::move(other.value));
156        hasVal = true;
157      }
158    } else {
159      reset();
160    }
161    return *this;
162  }
163};
164 
165template <typename T> class OptionalStorage<T, true> {
166  union {
167    char empty;
168    T value;
169  };
170  bool hasVal = false;
171 
172public:
173  ~OptionalStorage() = default;
174 
175  constexpr OptionalStorage() noexcept : empty{} {}
176 
177  constexpr OptionalStorage(OptionalStorage const &other) = default;
178  constexpr OptionalStorage(OptionalStorage &&other) = default;
179 
180  OptionalStorage &operator=(OptionalStorage const &other) = default;
181  OptionalStorage &operator=(OptionalStorage &&other) = default;
182 
183  template <class... Args>
184  constexpr explicit OptionalStorage(in_place_t, Args &&... args)
185      : value(std::forward<Args>(args)...), hasVal(true) {}
186 
187  void reset() noexcept {
188    if (hasVal) {
189      value.~T();
190      hasVal = false;
191    }
192  }
193 
194  constexpr bool hasValue() const noexcept { return hasVal; }
42
←
Returning zero, which participates in a condition later→
195 
196  T &getValue() LLVM_LVALUE_FUNCTION& noexcept {
197    assert(hasVal)((hasVal) ? static_cast<void> (0) : __assert_fail ("hasVal"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/ADT/Optional.h"
, 197, __PRETTY_FUNCTION__));
198    return value;
199  }
200  constexpr T const &getValue() const LLVM_LVALUE_FUNCTION& noexcept {
201    assert(hasVal)((hasVal) ? static_cast<void> (0) : __assert_fail ("hasVal"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/ADT/Optional.h"
, 201, __PRETTY_FUNCTION__));
202    return value;
203  }
204#if LLVM_HAS_RVALUE_REFERENCE_THIS1
205  T &&getValue() && noexcept {
206    assert(hasVal)((hasVal) ? static_cast<void> (0) : __assert_fail ("hasVal"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include/llvm/ADT/Optional.h"
, 206, __PRETTY_FUNCTION__));
207    return std::move(value);
208  }
209#endif
210 
211  template <class... Args> void emplace(Args &&... args) {
212    reset();
213    ::new ((void *)std::addressof(value)) T(std::forward<Args>(args)...);
214    hasVal = true;
215  }
216 
217  OptionalStorage &operator=(T const &y) {
218    if (hasValue()) {
219      value = y;
220    } else {
221      ::new ((void *)std::addressof(value)) T(y);
222      hasVal = true;
223    }
224    return *this;
225  }
226  OptionalStorage &operator=(T &&y) {
227    if (hasValue()) {
228      value = std::move(y);
229    } else {
230      ::new ((void *)std::addressof(value)) T(std::move(y));
231      hasVal = true;
232    }
233    return *this;
234  }
235};
236 
237} // namespace optional_detail
238 
239template <typename T> class Optional {
240  optional_detail::OptionalStorage<T> Storage;
241 
242public:
243  using value_type = T;
244 
245  constexpr Optional() {}
246  constexpr Optional(NoneType) {}
247 
248  constexpr Optional(const T &y) : Storage(optional_detail::in_place_t{}, y) {}
249  constexpr Optional(const Optional &O) = default;
250 
251  constexpr Optional(T &&y)
252      : Storage(optional_detail::in_place_t{}, std::move(y)) {}
253  constexpr Optional(Optional &&O) = default;
254 
255  Optional &operator=(T &&y) {
256    Storage = std::move(y);
257    return *this;
258  }
259  Optional &operator=(Optional &&O) = default;
260 
261  /// Create a new object by constructing it in place with the given arguments.
262  template <typename... ArgTypes> void emplace(ArgTypes &&... Args) {
263    Storage.emplace(std::forward<ArgTypes>(Args)...);
264  }
265 
266  static constexpr Optional create(const T *y) {
267    return y ? Optional(*y) : Optional();
268  }
269 
270  Optional &operator=(const T &y) {
271    Storage = y;
272    return *this;
273  }
274  Optional &operator=(const Optional &O) = default;
275 
276  void reset() { Storage.reset(); }
277 
278  constexpr const T *getPointer() const { return &Storage.getValue(); }
279  T *getPointer() { return &Storage.getValue(); }
280  constexpr const T &getValue() const LLVM_LVALUE_FUNCTION& {
281    return Storage.getValue();
282  }
283  T &getValue() LLVM_LVALUE_FUNCTION& { return Storage.getValue(); }
284 
285  constexpr explicit operator bool() const { return hasValue(); }
40
←
Calling 'Optional::hasValue'→
45
←
Returning from 'Optional::hasValue'→
46
←
Returning zero, which participates in a condition later→
286  constexpr bool hasValue() const { return Storage.hasValue(); }
41
←
Calling 'OptionalStorage::hasValue'→
43
←
Returning from 'OptionalStorage::hasValue'→
44
←
Returning zero, which participates in a condition later→
287  constexpr const T *operator->() const { return getPointer(); }
288  T *operator->() { return getPointer(); }
289  constexpr const T &operator*() const LLVM_LVALUE_FUNCTION& {
290    return getValue();
291  }
292  T &operator*() LLVM_LVALUE_FUNCTION& { return getValue(); }
293 
294  template <typename U>
295  constexpr T getValueOr(U &&value) const LLVM_LVALUE_FUNCTION& {
296    return hasValue() ? getValue() : std::forward<U>(value);
297  }
298 
299  /// Apply a function to the value if present; otherwise return None.
300  template <class Function>
301  auto map(const Function &F) const LLVM_LVALUE_FUNCTION&
302      -> Optional<decltype(F(getValue()))> {
303    if (*this) return F(getValue());
304    return None;
305  }
306 
307#if LLVM_HAS_RVALUE_REFERENCE_THIS1
308  T &&getValue() && { return std::move(Storage.getValue()); }
309  T &&operator*() && { return std::move(Storage.getValue()); }
310 
311  template <typename U>
312  T getValueOr(U &&value) && {
313    return hasValue() ? std::move(getValue()) : std::forward<U>(value);
314  }
315 
316  /// Apply a function to the value if present; otherwise return None.
317  template <class Function>
318  auto map(const Function &F) &&
319      -> Optional<decltype(F(std::move(*this).getValue()))> {
320    if (*this) return F(std::move(*this).getValue());
321    return None;
322  }
323#endif
324};
325 
326template <class T> llvm::hash_code hash_value(const Optional<T> &O) {
327  return O ? hash_combine(true, *O) : hash_value(false);
328}
329 
330template <typename T, typename U>
331constexpr bool operator==(const Optional<T> &X, const Optional<U> &Y) {
332  if (X && Y)
333    return *X == *Y;
334  return X.hasValue() == Y.hasValue();
335}
336 
337template <typename T, typename U>
338constexpr bool operator!=(const Optional<T> &X, const Optional<U> &Y) {
339  return !(X == Y);
340}
341 
342template <typename T, typename U>
343constexpr bool operator<(const Optional<T> &X, const Optional<U> &Y) {
344  if (X && Y)
345    return *X < *Y;
346  return X.hasValue() < Y.hasValue();
347}
348 
349template <typename T, typename U>
350constexpr bool operator<=(const Optional<T> &X, const Optional<U> &Y) {
351  return !(Y < X);
352}
353 
354template <typename T, typename U>
355constexpr bool operator>(const Optional<T> &X, const Optional<U> &Y) {
356  return Y < X;
357}
358 
359template <typename T, typename U>
360constexpr bool operator>=(const Optional<T> &X, const Optional<U> &Y) {
361  return !(X < Y);
362}
363 
364template <typename T>
365constexpr bool operator==(const Optional<T> &X, NoneType) {
366  return !X;
367}
368 
369template <typename T>
370constexpr bool operator==(NoneType, const Optional<T> &X) {
371  return X == None;
372}
373 
374template <typename T>
375constexpr bool operator!=(const Optional<T> &X, NoneType) {
376  return !(X == None);
377}
378 
379template <typename T>
380constexpr bool operator!=(NoneType, const Optional<T> &X) {
381  return X != None;
382}
383 
384template <typename T> constexpr bool operator<(const Optional<T> &, NoneType) {
385  return false;
386}
387 
388template <typename T> constexpr bool operator<(NoneType, const Optional<T> &X) {
389  return X.hasValue();
390}
391 
392template <typename T>
393constexpr bool operator<=(const Optional<T> &X, NoneType) {
394  return !(None < X);
395}
396 
397template <typename T>
398constexpr bool operator<=(NoneType, const Optional<T> &X) {
399  return !(X < None);
400}
401 
402template <typename T> constexpr bool operator>(const Optional<T> &X, NoneType) {
403  return None < X;
404}
405 
406template <typename T> constexpr bool operator>(NoneType, const Optional<T> &X) {
407  return X < None;
408}
409 
410template <typename T>
411constexpr bool operator>=(const Optional<T> &X, NoneType) {
412  return None <= X;
413}
414 
415template <typename T>
416constexpr bool operator>=(NoneType, const Optional<T> &X) {
417  return X <= None;
418}
419 
420template <typename T>
421constexpr bool operator==(const Optional<T> &X, const T &Y) {
422  return X && *X == Y;
423}
424 
425template <typename T>
426constexpr bool operator==(const T &X, const Optional<T> &Y) {
427  return Y && X == *Y;
428}
429 
430template <typename T>
431constexpr bool operator!=(const Optional<T> &X, const T &Y) {
432  return !(X == Y);
433}
434 
435template <typename T>
436constexpr bool operator!=(const T &X, const Optional<T> &Y) {
437  return !(X == Y);
438}
439 
440template <typename T>
441constexpr bool operator<(const Optional<T> &X, const T &Y) {
442  return !X || *X < Y;
443}
444 
445template <typename T>
446constexpr bool operator<(const T &X, const Optional<T> &Y) {
447  return Y && X < *Y;
448}
449 
450template <typename T>
451constexpr bool operator<=(const Optional<T> &X, const T &Y) {
452  return !(Y < X);
453}
454 
455template <typename T>
456constexpr bool operator<=(const T &X, const Optional<T> &Y) {
457  return !(Y < X);
458}
459 
460template <typename T>
461constexpr bool operator>(const Optional<T> &X, const T &Y) {
462  return Y < X;
463}
464 
465template <typename T>
466constexpr bool operator>(const T &X, const Optional<T> &Y) {
467  return Y < X;
468}
469 
470template <typename T>
471constexpr bool operator>=(const Optional<T> &X, const T &Y) {
472  return !(X < Y);
473}
474 
475template <typename T>
476constexpr bool operator>=(const T &X, const Optional<T> &Y) {
477  return !(X < Y);
478}
479 
480raw_ostream &operator<<(raw_ostream &OS, NoneType);
481 
482template <typename T, typename = decltype(std::declval<raw_ostream &>()
483                                          << std::declval<const T &>())>
484raw_ostream &operator<<(raw_ostream &OS, const Optional<T> &O) {
485  if (O)
486    OS << *O;
487  else
488    OS << None;
489  return OS;
490}
491 
492} // end namespace llvm
493 
494#endif // LLVM_ADT_OPTIONAL_H