Instrumentor.h

Go to the documentation of this file.

//===-- Instrumentor.h - Highly configurable instrumentation pass ---------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// The Instrumentor, a highly configurable instrumentation pass.
//
//===----------------------------------------------------------------------===//
 
#ifndef LLVM_TRANSFORMS_IPO_INSTRUMENTOR_H
#define LLVM_TRANSFORMS_IPO_INSTRUMENTOR_H
 
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/EnumeratedArray.h"
#include "llvm/ADT/IntrusiveRefCntPtr.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/PassManager.h"
#include "llvm/Support/Allocator.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/StringSaver.h"
#include "llvm/Transforms/IPO/InstrumentorUtils.h"
#include "llvm/Transforms/Utils/Instrumentation.h"
 
#include <cstdint>
#include <functional>
#include <memory>
#include <string>
#include <tuple>
 
namespace llvm {
namespace instrumentor {
 
struct InstrumentationConfig;
struct InstrumentationOpportunity;
 
/// Callback type for getting/setting a value for a instrumented opportunity.
///{
using GetterCallbackTy = std::function<Value *(
    Value &, Type &, InstrumentationConfig &, InstrumentorIRBuilderTy &)>;
using SetterCallbackTy = std::function<Value *(
    Value &, Value &, InstrumentationConfig &, InstrumentorIRBuilderTy &)>;
///}
 
/// Helper to represent an argument to an instrumentation runtime function.
struct IRTArg {
  /// Flags describing the possible properties of an argument.
  enum IRArgFlagTy {
    NONE = 0,
    STRING = 1 << 0,
    REPLACABLE = 1 << 1,
    REPLACABLE_CUSTOM = 1 << 2,
    POTENTIALLY_INDIRECT = 1 << 3,
    INDIRECT_HAS_SIZE = 1 << 4,
    VALUE_PACK = 1 << 5,
    LAST,
  };
 
  /// Construct an argument.
  IRTArg(Type *Ty, StringRef Name, StringRef Description, unsigned Flags,
         GetterCallbackTy GetterCB, SetterCallbackTy SetterCB = nullptr,
         bool Enabled = true, bool NoCache = false)
      : Enabled(Enabled), Ty(Ty), Name(Name), Description(Description),
        Flags(Flags), GetterCB(std::move(GetterCB)),
        SetterCB(std::move(SetterCB)), NoCache(NoCache) {}
 
  /// Whether the argument is enabled and should be passed to the function call.
  bool Enabled;
 
  /// The type of the argument.
  Type *Ty;
 
  /// A string with the name of the argument.
  StringRef Name;
 
  /// A string with the description of the argument.
  StringRef Description;
 
  /// The flags that describe the properties of the argument. Multiple flags may
  /// be specified.
  unsigned Flags;
 
  /// The callback for getting the value of the argument.
  GetterCallbackTy GetterCB;
 
  /// The callback for consuming the output value of the argument.
  SetterCallbackTy SetterCB;
 
  /// Whether the argument value can be cached between the PRE and POST calls.
  bool NoCache;
};
 
/// Helper to represent an instrumentation runtime function that is related to
/// an instrumentation opportunity.
struct IRTCallDescription {
  /// Construct an instrumentation function description linked to the \p IO
  /// instrumentation opportunity and \p RetTy return type.
  LLVM_ABI IRTCallDescription(InstrumentationOpportunity &IO,
                              Type *RetTy = nullptr);
 
  /// Create the type of the instrumentation function.
  LLVM_ABI FunctionType *createLLVMSignature(InstrumentationConfig &IConf,
                                             InstrumentorIRBuilderTy &IIRB,
                                             const DataLayout &DL,
                                             bool ForceIndirection);
 
  /// Create a call instruction that calls to the instrumentation function and
  /// passes the corresponding arguments.
  LLVM_ABI CallInst *createLLVMCall(Value *&V, InstrumentationConfig &IConf,
                                    InstrumentorIRBuilderTy &IIRB,
                                    const DataLayout &DL,
                                    InstrumentationCaches &ICaches);
 
  /// Create a string representation of the function declaration in C. Two
  /// strings are returned: the function definition with direct arguments and
  /// the function with any indirect argument.
  LLVM_ABI std::pair<std::string, std::string>
  createCSignature(const InstrumentationConfig &IConf) const;
 
  /// Create a string representation of the function definition in C. The
  /// function body implements a stub and only prints the passed arguments. Two
  /// strings are returned: the function definition with direct arguments and
  /// the function with any indirect argument.
  LLVM_ABI std::pair<std::string, std::string> createCBodies() const;
 
  /// Return whether the \p IRTA argument can be replaced.
  bool isReplacable(IRTArg &IRTA) const {
    return (IRTA.Flags & (IRTArg::REPLACABLE | IRTArg::REPLACABLE_CUSTOM));
  }
 
  /// Return whether the function may have any indirect argument.
  bool isPotentiallyIndirect(IRTArg &IRTA) const {
    return ((IRTA.Flags & IRTArg::POTENTIALLY_INDIRECT) ||
            ((IRTA.Flags & IRTArg::REPLACABLE) && NumReplaceableArgs > 1));
  }
 
  /// Whether the function requires indirection in some argument.
  bool RequiresIndirection = false;
 
  /// Whether any argument may require indirection.
  bool MightRequireIndirection = false;
 
  /// The number of arguments that can be replaced.
  unsigned NumReplaceableArgs = 0;
 
  /// The instrumentation opportunity which it is linked to.
  InstrumentationOpportunity &IO;
 
  /// The return type of the instrumentation function.
  Type *RetTy = nullptr;
};
 
/// Helper to represent an instrumentation location, which is composed of an
/// instrumentation opportunity type and a position.
struct InstrumentationLocation {
  /// The supported location kinds, which are composed of a opportunity type and
  /// position. The PRE position indicates the instrumentation function call is
  /// inserted before the instrumented event occurs. The POST position indicates
  /// the instrumentation call is inserted after the event occurs. Some
  /// opportunity types may only support one position.
  enum KindTy {
    MODULE_PRE,
    MODULE_POST,
    GLOBAL_PRE,
    GLOBAL_POST,
    FUNCTION_PRE,
    FUNCTION_POST,
    BASIC_BLOCK_PRE,
    BASIC_BLOCK_POST,
    INSTRUCTION_PRE,
    INSTRUCTION_POST,
    SPECIAL_VALUE,
    Last = SPECIAL_VALUE,
  };
 
  /// Construct an instrumentation location with the given kind.
  InstrumentationLocation(KindTy Kind) : Kind(Kind) {}
 
  /// Return the type and position.
  KindTy getKind() const { return Kind; }
 
  /// Return the string representation given a location kind. This is the string
  /// used in the configuration file.
  static StringRef getKindStr(KindTy Kind) {
    switch (Kind) {
    case MODULE_PRE:
      return "module_pre";
    case MODULE_POST:
      return "module_post";
    case GLOBAL_PRE:
      return "global_pre";
    case GLOBAL_POST:
      return "global_post";
    case FUNCTION_PRE:
      return "function_pre";
    case FUNCTION_POST:
      return "function_post";
    case BASIC_BLOCK_PRE:
      return "basic_block_pre";
    case BASIC_BLOCK_POST:
      return "basic_block_post";
    case INSTRUCTION_PRE:
      return "instruction_pre";
    case INSTRUCTION_POST:
      return "instruction_post";
    case SPECIAL_VALUE:
      return "special_value";
    }
    llvm_unreachable("Invalid kind!");
  }
 
  /// Return the location kind described by a string.
  static KindTy getKindFromStr(StringRef S) {
    return StringSwitch<KindTy>(S)
        .Case("module_pre", MODULE_PRE)
        .Case("module_post", MODULE_POST)
        .Case("global_pre", GLOBAL_PRE)
        .Case("global_post", GLOBAL_POST)
        .Case("function_pre", FUNCTION_PRE)
        .Case("function_post", FUNCTION_POST)
        .Case("basic_block_pre", BASIC_BLOCK_PRE)
        .Case("basic_block_post", BASIC_BLOCK_POST)
        .Case("instruction_pre", INSTRUCTION_PRE)
        .Case("instruction_post", INSTRUCTION_POST)
        .Case("special_value", SPECIAL_VALUE)
        .Default(Last);
  }
 
  /// Return whether a location kind is positioned before the event occurs.
  static bool isPRE(KindTy Kind) {
    switch (Kind) {
    case MODULE_PRE:
    case GLOBAL_PRE:
    case FUNCTION_PRE:
    case BASIC_BLOCK_PRE:
    case INSTRUCTION_PRE:
      return true;
    case MODULE_POST:
    case GLOBAL_POST:
    case FUNCTION_POST:
    case BASIC_BLOCK_POST:
    case INSTRUCTION_POST:
    case SPECIAL_VALUE:
      return false;
    }
    llvm_unreachable("Invalid kind!");
  }
 
  /// Return whether the instrumentation location is before the event occurs.
  bool isPRE() const { return isPRE(Kind); }
 
private:
  /// The kind (type and position) of the instrumentation location.
  const KindTy Kind;
};
 
/// An option for the base configuration.
struct BaseConfigurationOption {
  /// The possible types of options.
  enum KindTy {
    STRING,
    BOOLEAN,
  };
 
  /// Create a boolean option with \p Name name, \p Description description and
  /// \p DefaultValue as boolean default value.
  LLVM_ABI static std::unique_ptr<BaseConfigurationOption>
  createBoolOption(InstrumentationConfig &IC, StringRef Name,
                   StringRef Description, bool DefaultValue);
 
  /// Create a string option with \p Name name, \p Description description and
  /// \p DefaultValue as string default value.
  LLVM_ABI static std::unique_ptr<BaseConfigurationOption>
  createStringOption(InstrumentationConfig &IC, StringRef Name,
                     StringRef Description, StringRef DefaultValue);
 
  /// Helper union that holds any possible option type.
  union ValueTy {
    bool Bool;
    StringRef String;
  };
 
  /// Set and get of the boolean value. Only valid if it is a boolean option.
  ///{
  void setBool(bool B) {
    assert(Kind == BOOLEAN && "Not a boolean!");
    Value.Bool = B;
  }
  bool getBool() const {
    assert(Kind == BOOLEAN && "Not a boolean!");
    return Value.Bool;
  }
  ///}
 
  /// Set and get the string value. Only valid if it is a boolean option.
  ///{
  void setString(StringRef S) {
    assert(Kind == STRING && "Not a string!");
    Value.String = S;
  }
  StringRef getString() const {
    assert(Kind == STRING && "Not a string!");
    return Value.String;
  }
  ///}
 
  /// The information of the option.
  ///{
  StringRef Name;
  StringRef Description;
  KindTy Kind;
  ValueTy Value = {0};
  ///}
 
  /// Construct a base configuration option.
  BaseConfigurationOption(StringRef Name, StringRef Desc, KindTy Kind)
      : Name(Name), Description(Desc), Kind(Kind) {}
};
 
/// The class that contains the configuration for the instrumentor. It holds the
/// information for each instrumented opportunity, including the base
/// configuration options. Another class may inherit from this one to modify the
/// default behavior.
struct LLVM_ABI InstrumentationConfig {
  virtual ~InstrumentationConfig() {}
 
  /// Construct an instrumentation configuration with the base options.
  InstrumentationConfig() : SS(StringAllocator) {}
 
  /// Initialize the config to a clean base state without loosing cached values
  /// that can be reused across configurations.
  void init(InstrumentorIRBuilderTy &IIRB) {
    // Clear previous configurations but not the caches.
    BaseConfigurationOptions.clear();
    for (auto &Map : IChoices)
      Map.clear();
 
    RuntimePrefix = BaseConfigurationOption::createStringOption(
        *this, "runtime_prefix", "The runtime API prefix.", "__instrumentor_");
    RuntimeStubsFile = BaseConfigurationOption::createStringOption(
        *this, "runtime_stubs_file",
        "The file into which runtime stubs should be written.", "");
    TargetRegex = BaseConfigurationOption::createStringOption(
        *this, "target_regex",
        "Regular expression to be matched against the module target. "
        "Only targets that match this regex will be instrumented.",
        "");
    FunctionRegex = BaseConfigurationOption::createStringOption(
        *this, "function_regex",
        "Regular expression to be matched against a function name. "
        "Only functions that match this regex will be instrumented.",
        "");
    DemangleFunctionNames = BaseConfigurationOption::createBoolOption(
        *this, "demangle_function_names",
        "Demangle functions names passed to the runtime.", true);
    HostEnabled = BaseConfigurationOption::createBoolOption(
        *this, "host_enabled", "Instrument non-GPU targets", true);
    GPUEnabled = BaseConfigurationOption::createBoolOption(
        *this, "gpu_enabled", "Instrument GPU targets", true);
    populate(IIRB);
  }
 
  /// Populate the instrumentation opportunities.
  virtual void populate(InstrumentorIRBuilderTy &IIRB);
 
  /// Get the runtime prefix for the instrumentation runtime functions.
  StringRef getRTName() const { return RuntimePrefix->getString(); }
 
  /// Get the instrumentation function name.
  std::string getRTName(StringRef Prefix, StringRef Name,
                        StringRef Suffix1 = "", StringRef Suffix2 = "") const {
    return (getRTName() + Prefix + Name + Suffix1 + Suffix2).str();
  }
 
  /// Add the base configuration option \p BCO into the list of base options.
  void addBaseChoice(BaseConfigurationOption *BCO) {
    BaseConfigurationOptions.push_back(BCO);
  }
 
  /// Register instrumentation opportunity \p IO.
  void addChoice(InstrumentationOpportunity &IO, LLVMContext &Ctx);
 
  /// Allocate an object of type \p Ty using a bump allocator and construct it
  /// with the \p Args arguments. The object may not be freed manually.
  template <typename Ty, typename... ArgsTy>
  static Ty *allocate(ArgsTy &&...Args) {
    static SpecificBumpPtrAllocator<Ty> Allocator;
    Ty *Obj = Allocator.Allocate();
    new (Obj) Ty(std::forward<ArgsTy>(Args)...);
    return Obj;
  }
 
  /// Map to remember underlying objects for pointers.
  DenseMap<Value *, Value *> UnderlyingObjsMap;
 
  /// Map to remember base pointer info for values in a specific function.
  DenseMap<std::pair<Value *, Function *>, Value *> BasePointerInfoMap;
 
  /// Return the base pointer info for \p V.
  Value *getBasePointerInfo(Value &V, InstrumentorIRBuilderTy &IIRB);
 
  /// Mapping to remember global strings passed to the runtime.
  DenseMap<StringRef, Constant *> GlobalStringsMap;
 
  /// Mapping from constants to globals with the constant as initializer.
  DenseMap<Constant *, GlobalVariable *> ConstantGlobalsCache;
 
  Constant *getGlobalString(StringRef S, InstrumentorIRBuilderTy &IIRB) {
    Constant *&V = GlobalStringsMap[SS.save(S)];
    if (!V) {
      auto &M = *IIRB.IRB.GetInsertBlock()->getModule();
      V = IIRB.IRB.CreateGlobalString(
          S, getRTName() + ".str",
          M.getDataLayout().getDefaultGlobalsAddressSpace(), &M);
      if (V->getType() != IIRB.IRB.getPtrTy())
        V = ConstantExpr::getAddrSpaceCast(V, IIRB.IRB.getPtrTy());
    }
    return V;
  }
  /// The list of enabled base configuration options.
  SmallVector<BaseConfigurationOption *> BaseConfigurationOptions;
 
  /// The base configuration options.
  std::unique_ptr<BaseConfigurationOption> RuntimePrefix;
  std::unique_ptr<BaseConfigurationOption> RuntimeStubsFile;
  std::unique_ptr<BaseConfigurationOption> DemangleFunctionNames;
  std::unique_ptr<BaseConfigurationOption> TargetRegex;
  std::unique_ptr<BaseConfigurationOption> FunctionRegex;
  std::unique_ptr<BaseConfigurationOption> HostEnabled;
  std::unique_ptr<BaseConfigurationOption> GPUEnabled;
 
  /// The map registered instrumentation opportunities. The map is indexed by
  /// the instrumentation location kind and then by the opportunity name. Notice
  /// that an instrumentation location may have more than one instrumentation
  /// opportunity registered.
  EnumeratedArray<MapVector<StringRef, InstrumentationOpportunity *>,
                  InstrumentationLocation::KindTy>
      IChoices;
 
  /// Utilities for allocating and building strings.
  ///{
  BumpPtrAllocator StringAllocator;
  StringSaver SS;
  ///}
};
 
/// Base class for instrumentation opportunities. All opportunities should
/// inherit from this class and implement the virtual class members.
struct InstrumentationOpportunity {
  virtual ~InstrumentationOpportunity() {}
 
  /// Construct an opportunity with location \p IP.
  InstrumentationOpportunity(const InstrumentationLocation IP) : IP(IP) {}
 
  /// The instrumentation location of the opportunity.
  InstrumentationLocation IP;
 
  /// The list of possible arguments for the instrumentation runtime function.
  /// The order within the array determines the order of arguments. Arguments
  /// may be disabled and will not be passed to the function call.
  SmallVector<IRTArg> IRTArgs;
 
  /// Whether the opportunity is enabled.
  bool Enabled = true;
 
  /// A filter expression to be matched against runtime property values. If the
  /// filter is non-empty, only instrumentations matching the filter will be
  /// executed. The filter syntax supports:
  /// - Integer comparisons: ==, !=, <, >, <=, >=
  /// - String comparisons: ==, != (with quoted strings)
  /// - String prefix check: startswith("prefix")
  /// - Logical operators: &&, ||
  /// Examples:
  ///   "sync_scope_id==3 && atomicity_ordering>0"
  ///   "name==\"foo\" || name.startswith(\"test_\")"
  /// If a property value is dynamic (not a constant), the filter is assumed to
  /// pass (true).
  StringRef Filter;
 
  /// Helpers to cast values, pass them to the runtime, and replace them. To be
  /// used as part of the getter/setter of a InstrumentationOpportunity.
  ///{
  LLVM_ABI static Value *forceCast(Value &V, Type &Ty,
                                   InstrumentorIRBuilderTy &IIRB);
  static Value *getValue(Value &V, Type &Ty, InstrumentationConfig &IConf,
                         InstrumentorIRBuilderTy &IIRB) {
    return forceCast(V, Ty, IIRB);
  }
  LLVM_ABI static Value *replaceValue(Value &V, Value &NewV,
                                      InstrumentationConfig &IConf,
                                      InstrumentorIRBuilderTy &IIRB);
  ///}
 
  /// Instrument the value \p V using the configuration \p IConf, and
  /// potentially, the caches \p ICaches.
  virtual Value *instrument(Value *&V, bool &Changed,
                            InstrumentationConfig &IConf,
                            InstrumentorIRBuilderTy &IIRB,
                            InstrumentationCaches &ICaches) {
    if (CB && !CB(*V))
      return nullptr;
 
    // Check if the filter matches before instrumenting
    if (!evaluateFilter(*V, Changed, *this, IConf, IIRB))
      return nullptr;
 
    Changed = true;
    const DataLayout &DL = IIRB.IRB.GetInsertBlock()->getDataLayout();
    IRTCallDescription IRTCallDesc(*this, getRetTy(V->getContext()));
    auto *CI = IRTCallDesc.createLLVMCall(V, IConf, IIRB, DL, ICaches);
    return CI;
  }
 
  /// Get the return type for the instrumentation runtime function.
  virtual Type *getRetTy(LLVMContext &Ctx) const { return nullptr; }
 
  /// Get the name of the instrumentation opportunity.
  virtual StringRef getName() const = 0;
 
  /// Get all opcodes for this instrumentation opportunity. For non-instruction
  /// opportunities, returns an empty array. For instruction opportunities,
  /// returns an array of all opcodes this IO handles.
  virtual ArrayRef<unsigned> getAllOpcodes() const { return {}; }
 
  /// Get the location kind of the instrumentation opportunity.
  InstrumentationLocation::KindTy getLocationKind() const {
    return IP.getKind();
  }
 
  /// An optional callback that takes the value that is about to be
  /// instrumented and can return false if it should be skipped.
  ///{
  using CallbackTy = std::function<bool(Value &)>;
  CallbackTy CB = nullptr;
  ///}
 
  /// Add arguments available in all instrumentation opportunities.
  void addCommonArgs(InstrumentationConfig &IConf, LLVMContext &Ctx,
                     bool PassId) {
    const auto CB = IP.isPRE() ? getIdPre : getIdPost;
    if (PassId) {
      IRTArgs.push_back(
          IRTArg(IntegerType::getInt32Ty(Ctx), "id",
                 "A unique ID associated with the given instrumentor call",
                 IRTArg::NONE, CB, nullptr, true, true));
    }
  }
 
  /// Get the opportunity identifier for the pre and post positions.
  ///{
  LLVM_ABI static Value *getIdPre(Value &V, Type &Ty,
                                  InstrumentationConfig &IConf,
                                  InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *getIdPost(Value &V, Type &Ty,
                                   InstrumentationConfig &IConf,
                                   InstrumentorIRBuilderTy &IIRB);
  ///}
 
  /// Compute the opportunity identifier for the current instrumentation epoch
  /// \p CurrentEpoch. The identifiers are assigned consecutively as the epoch
  /// advances. Epochs may have no identifier assigned (e.g., because no id was
  /// requested). This function always returns the same identifier when called
  /// multiple times with the same epoch.
  static int32_t getIdFromEpoch(uint32_t CurrentEpoch) {
    static DenseMap<uint32_t, int32_t> EpochIdMap;
    static int32_t GlobalId = 0;
    int32_t &EpochId = EpochIdMap[CurrentEpoch];
    if (EpochId == 0)
      EpochId = ++GlobalId;
    return EpochId;
  }
};
 
/// The base instrumentation opportunity class for instruction opportunities.
/// Each instruction opportunity should inherit from this class and implement
/// the virtual class members. If multiple opcodes are provided, all of them
/// are instrumented using the same logic, and a name must be explicitly
/// provided by overriding getName().
template <unsigned... Opcodes>
struct InstructionIO : public InstrumentationOpportunity {
  virtual ~InstructionIO() {}
 
  /// Construct an instruction opportunity.
  InstructionIO(InstrumentationLocation::KindTy Kind)
      : InstrumentationOpportunity(InstrumentationLocation(Kind)) {
    static_assert(sizeof...(Opcodes) >= 1,
                  "InstructionIO must have at least one opcode");
  }
 
  static constexpr std::array<unsigned, sizeof...(Opcodes)> OpcodesArray = {
      Opcodes...};
 
  /// Get all opcodes for this instrumentation opportunity (override).
  ArrayRef<unsigned> getAllOpcodes() const override { return OpcodesArray; }
 
  /// Get the number of opcodes.
  static constexpr size_t getNumOpcodes() { return OpcodesArray.size(); }
 
  /// Get the name of the instruction. For single-opcode IOs, this defaults to
  /// the opcode name. For multi-opcode IOs, getName() MUST be overridden to
  /// provide an explicit name identifying the whole group of opcodes.
  virtual StringRef getName() const override {
    // This method should not be called for multi-opcode IOs.
    // Multi-opcode IOs must override getName().
    assert(sizeof...(Opcodes) == 1 &&
           "Multi-opcode InstructionIO must override getName() to provide an "
           "explicit name instead of using the first opcode");
    // Get the first opcode from the opcodes array.
    return Instruction::getOpcodeName(OpcodesArray[0]);
  }
};
 
/// The instrumentation opportunity for functions.
struct FunctionIO final : public InstrumentationOpportunity {
  FunctionIO(InstrumentationLocation::KindTy Kind)
      : InstrumentationOpportunity(InstrumentationLocation(Kind)) {}
 
  enum ConfigKind {
    PassAddress = 0,
    PassName,
    PassNumArguments,
    PassArguments,
    ReplaceArguments,
    PassIsMain,
    PassId,
    NumConfig,
  };
 
  struct ConfigTy final : public BaseConfigTy<ConfigKind> {
    std::function<bool(Argument &)> ArgFilter;
 
    ConfigTy(bool Enable = true) : BaseConfigTy(Enable) {}
  } Config;
 
  StringRef getName() const override { return "function"; }
 
  LLVM_ABI void init(InstrumentationConfig &IConf,
                     InstrumentorIRBuilderTy &IIRB,
                     ConfigTy *UserConfig = nullptr);
 
  LLVM_ABI static Value *getFunctionAddress(Value &V, Type &Ty,
                                            InstrumentationConfig &IConf,
                                            InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *getFunctionName(Value &V, Type &Ty,
                                         InstrumentationConfig &IConf,
                                         InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI Value *getNumArguments(Value &V, Type &Ty,
                                  InstrumentationConfig &IConf,
                                  InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI Value *getArguments(Value &V, Type &Ty, InstrumentationConfig &IConf,
                               InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI Value *setArguments(Value &V, Value &NewV,
                               InstrumentationConfig &IConf,
                               InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *isMainFunction(Value &V, Type &Ty,
                                        InstrumentationConfig &IConf,
                                        InstrumentorIRBuilderTy &IIRB);
 
  static void populate(InstrumentationConfig &IConf,
                       InstrumentorIRBuilderTy &IIRB) {
    auto *PreIO =
        IConf.allocate<FunctionIO>(InstrumentationLocation::FUNCTION_PRE);
    PreIO->init(IConf, IIRB);
    auto *PostIO =
        IConf.allocate<FunctionIO>(InstrumentationLocation::FUNCTION_POST);
    PostIO->init(IConf, IIRB);
  }
};
 
/// The instrumentation opportunity for alloca instructions.
struct AllocaIO final : public InstructionIO<Instruction::Alloca> {
  AllocaIO(InstrumentationLocation::KindTy Kind) : InstructionIO(Kind) {}
 
  enum ConfigKind {
    PassAddress = 0,
    ReplaceAddress,
    PassSize,
    ReplaceSize,
    PassAlignment,
    PassId,
    NumConfig,
  };
 
  using ConfigTy = BaseConfigTy<ConfigKind>;
  ConfigTy Config;
 
  LLVM_ABI void init(InstrumentationConfig &IConf,
                     InstrumentorIRBuilderTy &IIRB,
                     ConfigTy *UserConfig = nullptr);
 
  LLVM_ABI static Value *getSize(Value &V, Type &Ty,
                                 InstrumentationConfig &IConf,
                                 InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *setSize(Value &V, Value &NewV,
                                 InstrumentationConfig &IConf,
                                 InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *getAlignment(Value &V, Type &Ty,
                                      InstrumentationConfig &IConf,
                                      InstrumentorIRBuilderTy &IIRB);
 
  static void populate(InstrumentationConfig &IConf,
                       InstrumentorIRBuilderTy &IIRB) {
    auto *PreIO =
        IConf.allocate<AllocaIO>(InstrumentationLocation::INSTRUCTION_PRE);
    PreIO->init(IConf, IIRB);
    auto *PostIO =
        IConf.allocate<AllocaIO>(InstrumentationLocation::INSTRUCTION_POST);
    PostIO->init(IConf, IIRB);
  }
};
 
struct UnreachableIO final : public InstructionIO<Instruction::Unreachable> {
  UnreachableIO()
      : InstructionIO<Instruction::Unreachable>(
            InstrumentationLocation::INSTRUCTION_PRE) {}
 
  enum ConfigKind {
    PassId,
    NumConfig,
  };
 
  using ConfigTy = BaseConfigTy<ConfigKind>;
  ConfigTy Config;
 
  LLVM_ABI void init(InstrumentationConfig &IConf,
                     InstrumentorIRBuilderTy &IIRB,
                     ConfigTy *UserConfig = nullptr);
 
  static void populate(InstrumentationConfig &IConf,
                       InstrumentorIRBuilderTy &IIRB) {
    auto *PreIO = IConf.allocate<UnreachableIO>();
    PreIO->init(IConf, IIRB);
  }
};
 
// Special instrumentation opportunity for base pointers of memory operations.
struct BasePointerIO final : public InstrumentationOpportunity {
  BasePointerIO()
      : InstrumentationOpportunity(
            InstrumentationLocation(InstrumentationLocation::SPECIAL_VALUE)) {}
  virtual ~BasePointerIO() {};
 
  enum ConfigKind {
    PassPointer = 0,
    PassPointerKind,
    PassId,
    NumConfig,
  };
 
  using ConfigTy = BaseConfigTy<ConfigKind>;
  ConfigTy Config;
 
  StringRef getName() const override { return "base_pointer_info"; }
 
  LLVM_ABI void init(InstrumentationConfig &IConf,
                     InstrumentorIRBuilderTy &IIRB,
                     ConfigTy *UserConfig = nullptr);
 
  LLVM_ABI static Value *getPointerKind(Value &V, Type &Ty,
                                        InstrumentationConfig &IConf,
                                        InstrumentorIRBuilderTy &IIRB);
 
  /// This is necessary to produce a return value that can be used by other IOs.
  /// No replacement is actually happening.
  static Value *setValueNoop(Value &V, Value &NewV,
                             InstrumentationConfig &IConf,
                             InstrumentorIRBuilderTy &IIRB) {
    return &NewV;
  }
 
  static void populate(InstrumentationConfig &IConf,
                       InstrumentorIRBuilderTy &IIRB) {
    auto *BPIO = IConf.allocate<BasePointerIO>();
    BPIO->init(IConf, IIRB);
  }
};
 
// Module instrumentation opportunity.
struct ModuleIO final : public InstrumentationOpportunity {
  ModuleIO(InstrumentationLocation::KindTy Kind)
      : InstrumentationOpportunity(InstrumentationLocation(Kind)) {}
 
  enum ConfigKind {
    PassId,
    PassName,
    PassTargetTriple,
    NumConfig,
  };
 
  using ConfigTy = BaseConfigTy<ConfigKind>;
  ConfigTy Config;
 
  StringRef getName() const override { return "module"; }
 
  LLVM_ABI void init(InstrumentationConfig &IConf,
                     InstrumentorIRBuilderTy &IIRB,
                     ConfigTy *UserConfig = nullptr);
 
  LLVM_ABI static Value *getModuleName(Value &V, Type &Ty,
                                       InstrumentationConfig &IConf,
                                       InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *getTargetTriple(Value &V, Type &Ty,
                                         InstrumentationConfig &IConf,
                                         InstrumentorIRBuilderTy &IIRB);
 
  static void populate(InstrumentationConfig &IConf,
                       InstrumentorIRBuilderTy &IIRB) {
    auto *PreIO = IConf.allocate<ModuleIO>(InstrumentationLocation::MODULE_PRE);
    PreIO->init(IConf, IIRB);
    auto *PostIO =
        IConf.allocate<ModuleIO>(InstrumentationLocation::MODULE_POST);
    PostIO->init(IConf, IIRB);
  }
};
 
// Global variable instrumentation opportunity.
struct GlobalVarIO final : public InstrumentationOpportunity {
  GlobalVarIO(InstrumentationLocation::KindTy Kind)
      : InstrumentationOpportunity(InstrumentationLocation(Kind)) {}
 
  enum ConfigKind {
    PassAddress = 0,
    ReplaceAddress,
    PassAS,
    PassDeclaredSize,
    PassAlignment,
    PassName,
    PassInitialValue,
    PassIsConstant,
    PassIsDefinition,
    PassId,
    NumConfig,
  };
 
  using ConfigTy = BaseConfigTy<ConfigKind>;
  ConfigTy Config;
 
  StringRef getName() const override { return "global"; }
 
  LLVM_ABI void init(InstrumentationConfig &IConf,
                     InstrumentorIRBuilderTy &IIRB,
                     ConfigTy *UserConfig = nullptr);
 
  LLVM_ABI static Value *getAddress(Value &V, Type &Ty,
                                    InstrumentationConfig &IConf,
                                    InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *setAddress(Value &V, Value &NewV,
                                    InstrumentationConfig &IConf,
                                    InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *getAS(Value &V, Type &Ty, InstrumentationConfig &IConf,
                               InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *getDeclaredSize(Value &V, Type &Ty,
                                         InstrumentationConfig &IConf,
                                         InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *getAlignment(Value &V, Type &Ty,
                                      InstrumentationConfig &IConf,
                                      InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *getSymbolName(Value &V, Type &Ty,
                                       InstrumentationConfig &IConf,
                                       InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *getInitialValue(Value &V, Type &Ty,
                                         InstrumentationConfig &IConf,
                                         InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *isConstant(Value &V, Type &Ty,
                                    InstrumentationConfig &IConf,
                                    InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *isDefinition(Value &V, Type &Ty,
                                      InstrumentationConfig &IConf,
                                      InstrumentorIRBuilderTy &IIRB);
 
  static void populate(InstrumentationConfig &IConf,
                       InstrumentorIRBuilderTy &IIRB) {
    auto *PreIO =
        IConf.allocate<GlobalVarIO>(InstrumentationLocation::GLOBAL_PRE);
    PreIO->init(IConf, IIRB);
    auto *PostIO =
        IConf.allocate<GlobalVarIO>(InstrumentationLocation::GLOBAL_POST);
    PostIO->init(IConf, IIRB);
  }
};
 
/// The instrumentation opportunity for store instructions.
struct StoreIO : public InstructionIO<Instruction::Store> {
  virtual ~StoreIO() {};
 
  /// Construct a store instruction opportunity.
  StoreIO(InstrumentationLocation::KindTy Kind) : InstructionIO(Kind) {}
 
  /// The selector of arguments for store opportunities.
  ///{
  enum ConfigKind {
    PassPointer = 0,
    ReplacePointer,
    PassPointerAS,
    PassBasePointerInfo,
    PassStoredValue,
    PassStoredValueSize,
    PassAlignment,
    PassValueTypeId,
    PassAtomicityOrdering,
    PassSyncScopeId,
    PassIsVolatile,
    PassId,
    NumConfig,
  };
 
  using ConfigTy = BaseConfigTy<ConfigKind>;
  ConfigTy Config;
  ///}
 
  /// Get the type of the stored value.
  virtual Type *getValueType(InstrumentorIRBuilderTy &IIRB) const {
    return IIRB.Int64Ty;
  }
 
  /// Initialize the store opportunity using the instrumentation config \p IConf
  /// and the user config \p UserConfig.
  LLVM_ABI void init(InstrumentationConfig &IConf,
                     InstrumentorIRBuilderTy &IIRB,
                     ConfigTy *UserConfig = nullptr);
 
  /// Getters and setters for the arguments of the instrumentation function for
  /// the store opportunity.
  ///{
  LLVM_ABI static Value *getPointer(Value &V, Type &Ty,
                                    InstrumentationConfig &IConf,
                                    InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *setPointer(Value &V, Value &NewV,
                                    InstrumentationConfig &IConf,
                                    InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *getPointerAS(Value &V, Type &Ty,
                                      InstrumentationConfig &IConf,
                                      InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *getBasePointerInfo(Value &V, Type &Ty,
                                            InstrumentationConfig &IConf,
                                            InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *getValue(Value &V, Type &Ty,
                                  InstrumentationConfig &IConf,
                                  InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *getValueSize(Value &V, Type &Ty,
                                      InstrumentationConfig &IConf,
                                      InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *getAlignment(Value &V, Type &Ty,
                                      InstrumentationConfig &IConf,
                                      InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *getValueTypeId(Value &V, Type &Ty,
                                        InstrumentationConfig &IConf,
                                        InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *getAtomicityOrdering(Value &V, Type &Ty,
                                              InstrumentationConfig &IConf,
                                              InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *getSyncScopeId(Value &V, Type &Ty,
                                        InstrumentationConfig &IConf,
                                        InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *isVolatile(Value &V, Type &Ty,
                                    InstrumentationConfig &IConf,
                                    InstrumentorIRBuilderTy &IIRB);
  ///}
 
  /// Create the store opportunities for pre and post positions. The
  /// opportunities are also initialized with the arguments for their
  /// instrumentation calls.
  static void populate(InstrumentationConfig &IConf,
                       InstrumentorIRBuilderTy &IIRB) {
    auto *PreIO =
        IConf.allocate<StoreIO>(InstrumentationLocation::INSTRUCTION_PRE);
    PreIO->init(IConf, IIRB);
    auto *PostIO =
        IConf.allocate<StoreIO>(InstrumentationLocation::INSTRUCTION_POST);
    PostIO->init(IConf, IIRB);
  }
};
 
/// The instrumentation opportunity for load instructions.
struct LoadIO : public InstructionIO<Instruction::Load> {
  virtual ~LoadIO() {};
 
  /// Construct a load opportunity.
  LoadIO(InstrumentationLocation::KindTy Kind) : InstructionIO(Kind) {}
 
  /// The selector of arguments for load opportunities.
  ///{
  enum ConfigKind {
    PassPointer = 0,
    ReplacePointer,
    PassPointerAS,
    PassBasePointerInfo,
    PassValue,
    ReplaceValue,
    PassValueSize,
    PassAlignment,
    PassValueTypeId,
    PassAtomicityOrdering,
    PassSyncScopeId,
    PassIsVolatile,
    PassId,
    NumConfig,
  };
 
  using ConfigTy = BaseConfigTy<ConfigKind>;
  ConfigTy Config;
  ///}
 
  /// Get the type of the loaded value.
  virtual Type *getValueType(InstrumentorIRBuilderTy &IIRB) const {
    return IIRB.Int64Ty;
  }
 
  /// Initialize the load opportunity using the instrumentation config \p IConf
  /// and the user config \p UserConfig.
  LLVM_ABI void init(InstrumentationConfig &IConf,
                     InstrumentorIRBuilderTy &IIRB,
                     ConfigTy *UserConfig = nullptr);
 
  /// Getters and setters for the arguments of the instrumentation function for
  /// the load opportunity.
  ///{
  LLVM_ABI static Value *getPointer(Value &V, Type &Ty,
                                    InstrumentationConfig &IConf,
                                    InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *setPointer(Value &V, Value &NewV,
                                    InstrumentationConfig &IConf,
                                    InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *getPointerAS(Value &V, Type &Ty,
                                      InstrumentationConfig &IConf,
                                      InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *getBasePointerInfo(Value &V, Type &Ty,
                                            InstrumentationConfig &IConf,
                                            InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *getValue(Value &V, Type &Ty,
                                  InstrumentationConfig &IConf,
                                  InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *getValueSize(Value &V, Type &Ty,
                                      InstrumentationConfig &IConf,
                                      InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *getAlignment(Value &V, Type &Ty,
                                      InstrumentationConfig &IConf,
                                      InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *getValueTypeId(Value &V, Type &Ty,
                                        InstrumentationConfig &IConf,
                                        InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *getAtomicityOrdering(Value &V, Type &Ty,
                                              InstrumentationConfig &IConf,
                                              InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *getSyncScopeId(Value &V, Type &Ty,
                                        InstrumentationConfig &IConf,
                                        InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *isVolatile(Value &V, Type &Ty,
                                    InstrumentationConfig &IConf,
                                    InstrumentorIRBuilderTy &IIRB);
  ///}
 
  /// Create the load opportunities for PRE and POST positions.
  static void populate(InstrumentationConfig &IConf,
                       InstrumentorIRBuilderTy &IIRB) {
    auto *PreIO =
        IConf.allocate<LoadIO>(InstrumentationLocation::INSTRUCTION_PRE);
    PreIO->init(IConf, IIRB);
    auto *PostIO =
        IConf.allocate<LoadIO>(InstrumentationLocation::INSTRUCTION_POST);
    PostIO->init(IConf, IIRB);
  }
};
 
/// The instrumentation opportunity for type cast instructions.
/// This includes PtrToInt, IntToPtr, Trunc, ZExt, SExt, FPToUI, FPToSI,
/// UIToFP, SIToFP, FPTrunc, FPExt, AddrSpaceCast, and BitCast.
struct CastIO final
    : public InstructionIO<
          Instruction::PtrToInt, Instruction::IntToPtr, Instruction::Trunc,
          Instruction::ZExt, Instruction::SExt, Instruction::FPToUI,
          Instruction::FPToSI, Instruction::UIToFP, Instruction::SIToFP,
          Instruction::FPTrunc, Instruction::FPExt, Instruction::AddrSpaceCast,
          Instruction::BitCast> {
  CastIO(InstrumentationLocation::KindTy Kind) : InstructionIO(Kind) {}
 
  enum ConfigKind {
    PassInput,
    PassInputTypeId,
    PassInputSize,
    PassResult,
    ReplaceResult,
    PassResultTypeId,
    PassResultSize,
    PassOpcode,
    PassId,
    NumConfig,
  };
 
  using ConfigTy = BaseConfigTy<ConfigKind>;
  ConfigTy Config;
 
  StringRef getName() const override { return "cast"; }
 
  LLVM_ABI void init(InstrumentationConfig &IConf,
                     InstrumentorIRBuilderTy &IIRB,
                     ConfigTy *UserConfig = nullptr);
 
  LLVM_ABI static Value *getInput(Value &V, Type &Ty,
                                  InstrumentationConfig &IConf,
                                  InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *getInputTypeId(Value &V, Type &Ty,
                                        InstrumentationConfig &IConf,
                                        InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *getInputSize(Value &V, Type &Ty,
                                      InstrumentationConfig &IConf,
                                      InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *getResultTypeId(Value &V, Type &Ty,
                                         InstrumentationConfig &IConf,
                                         InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *getResultSize(Value &V, Type &Ty,
                                       InstrumentationConfig &IConf,
                                       InstrumentorIRBuilderTy &IIRB);
 
  static void populate(InstrumentationConfig &IConf,
                       InstrumentorIRBuilderTy &IIRB) {
    auto *PreIO =
        IConf.allocate<CastIO>(InstrumentationLocation::INSTRUCTION_PRE);
    PreIO->init(IConf, IIRB);
    auto *PostIO =
        IConf.allocate<CastIO>(InstrumentationLocation::INSTRUCTION_POST);
    PostIO->init(IConf, IIRB);
  }
};
 
/// Instrumentation opportunity for numeric operations. This includes Add, FAdd,
/// Sub, FSub, Mul, FMul, UDiv, FDiv, SDiv, URem, SRem, FRem, Shl, LShr, AShr,
/// And, Or, Xor, and FNeg.
struct NumericIO final
    : public InstructionIO<
          Instruction::Add, Instruction::FAdd, Instruction::Sub,
          Instruction::FSub, Instruction::Mul, Instruction::FMul,
          Instruction::UDiv, Instruction::FDiv, Instruction::SDiv,
          Instruction::URem, Instruction::SRem, Instruction::FRem,
          Instruction::Shl, Instruction::LShr, Instruction::AShr,
          Instruction::And, Instruction::Or, Instruction::Xor,
          Instruction::FNeg> {
  NumericIO(InstrumentationLocation::KindTy Kind) : InstructionIO(Kind) {}
 
  enum ConfigKind {
    PassTypeId,
    PassSize,
    PassOpcode,
    PassResult,
    ReplaceResult,
    PassLeft,
    PassRight,
    PassFlags,
    PassId,
    NumConfig,
  };
 
  using ConfigTy = BaseConfigTy<ConfigKind>;
  ConfigTy Config;
 
  StringRef getName() const override { return "numeric"; }
 
  LLVM_ABI void init(InstrumentationConfig &IConf,
                     InstrumentorIRBuilderTy &IIRB,
                     ConfigTy *UserConfig = nullptr);
 
  LLVM_ABI static Value *getLeft(Value &V, Type &Ty,
                                 InstrumentationConfig &IConf,
                                 InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *getRight(Value &V, Type &Ty,
                                  InstrumentationConfig &IConf,
                                  InstrumentorIRBuilderTy &IIRB);
  LLVM_ABI static Value *getFlags(Value &V, Type &Ty,
                                  InstrumentationConfig &IConf,
                                  InstrumentorIRBuilderTy &IIRB);
 
  static void populate(InstrumentationConfig &IConf,
                       InstrumentorIRBuilderTy &IIRB) {
    auto *PreIO =
        IConf.allocate<NumericIO>(InstrumentationLocation::INSTRUCTION_PRE);
    PreIO->init(IConf, IIRB);
    auto *PostIO =
        IConf.allocate<NumericIO>(InstrumentationLocation::INSTRUCTION_POST);
    PostIO->init(IConf, IIRB);
  }
};
 
} // namespace instrumentor
 
/// The Instrumentor pass.
class InstrumentorPass : public RequiredPassInfoMixin<InstrumentorPass> {
  using InstrumentationConfig = instrumentor::InstrumentationConfig;
  using InstrumentorIRBuilderTy = instrumentor::InstrumentorIRBuilderTy;
 
  /// File system to be used for read operations.
  IntrusiveRefCntPtr<vfs::FileSystem> FS;
 
  /// The configuration and IR builder provided by the user.
  InstrumentationConfig *UserIConf;
  InstrumentorIRBuilderTy *UserIIRB;
 
  PreservedAnalyses run(Module &M, InstrumentationConfig &IConf,
                        InstrumentorIRBuilderTy &IIRB, bool ReadConfig);
 
public:
  /// Construct an instrumentor pass that will use the instrumentation
  /// configuration \p IC and the IR builder \p IIRB. If an IR builder is not
  /// provided, a default builder is used. When the configuration is not
  /// provided, it is read from the config file if available and otherwise a
  /// default configuration is used.
  LLVM_ABI InstrumentorPass(IntrusiveRefCntPtr<vfs::FileSystem> FS = nullptr,
                            InstrumentationConfig *IC = nullptr,
                            InstrumentorIRBuilderTy *IIRB = nullptr);
 
  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
};
 
} // end namespace llvm
 
#endif // LLVM_TRANSFORMS_IPO_INSTRUMENTOR_H