X86CodeGenPassBuilder.cpp

Go to the documentation of this file.

//===-- X86CodeGenPassBuilder.cpp ---------------------------------*- C++ -*-=//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
/// \file
/// This file contains X86 CodeGen pipeline builder.
///
//===----------------------------------------------------------------------===//
 
#include "X86.h"
#include "X86AsmPrinter.h"
#include "X86TargetMachine.h"
 
#include "llvm/CodeGen/AtomicExpand.h"
#include "llvm/CodeGen/BreakFalseDeps.h"
#include "llvm/CodeGen/CFGuardLongjmp.h"
#include "llvm/CodeGen/CFIInstrInserter.h"
#include "llvm/CodeGen/EHContGuardTargets.h"
#include "llvm/CodeGen/EarlyIfConversion.h"
#include "llvm/CodeGen/ExecutionDomainFix.h"
#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
#include "llvm/CodeGen/GlobalISel/Legalizer.h"
#include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
#include "llvm/CodeGen/IndirectBrExpand.h"
#include "llvm/CodeGen/InterleavedAccess.h"
#include "llvm/CodeGen/JMCInstrumenter.h"
#include "llvm/CodeGen/KCFI.h"
#include "llvm/CodeGen/MachineCombiner.h"
#include "llvm/CodeGen/PseudoProbeInserter.h"
#include "llvm/CodeGen/RegAllocGreedyPass.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCStreamer.h"
#include "llvm/Pass.h"
#include "llvm/Passes/CodeGenPassBuilder.h"
#include "llvm/Passes/PassBuilder.h"
#include "llvm/Support/CodeGen.h"
#include "llvm/Target/CGPassBuilderOption.h"
#include "llvm/Transforms/CFGuard.h"
 
using namespace llvm;
 
namespace {
 
class X86CodeGenPassBuilder : public CodeGenPassBuilder {
  using Base = CodeGenPassBuilder;
 
  X86TargetMachine &getTM() const {
    return static_cast<X86TargetMachine &>(TM);
  }
 
public:
  explicit X86CodeGenPassBuilder(X86TargetMachine &TM,
                                 const CGPassBuilderOption &Opts,
                                 PassInstrumentationCallbacks *PIC)
      : CodeGenPassBuilder(TM, Opts, PIC) {}
 
  void addIRPasses(PassManagerWrapper &PMW) override;
  void addPreISel(PassManagerWrapper &PMW) override;
  Error addInstSelector(PassManagerWrapper &PMW) override;
 
  Error addIRTranslator(PassManagerWrapper &PMW) override;
  void addPreLegalizeMachineIR(PassManagerWrapper &PMW) override;
  Error addLegalizeMachineIR(PassManagerWrapper &PMW) override;
  void addPreRegBankSelect(PassManagerWrapper &PMW) override;
  Error addRegBankSelect(PassManagerWrapper &PMW) override;
  Error addGlobalInstructionSelect(PassManagerWrapper &PMW) override;
 
  void addMachineSSAOptimization(PassManagerWrapper &PMW) override;
  void addILPOpts(PassManagerWrapper &PMW) override;
  void addPreRegAlloc(PassManagerWrapper &PMW) override;
  Error addRegAssignAndRewriteFast(PassManagerWrapper &PMW) override;
  Expected<bool>
  addRegAssignAndRewriteOptimized(PassManagerWrapper &PMW) override;
  void addPostRegAlloc(PassManagerWrapper &PMW) override;
  void addPreSched2(PassManagerWrapper &PMW) override;
  void addPreEmitPass(PassManagerWrapper &PMW) override;
  void addPreEmitPass2(PassManagerWrapper &PMW) override;
  void addAsmPrinterBegin(PassManagerWrapper &PMW) override;
  void addAsmPrinter(PassManagerWrapper &PMW) override;
  void addAsmPrinterEnd(PassManagerWrapper &PMW) override;
};
 
void X86CodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) {
  addFunctionPass(AtomicExpandPass(TM), PMW);
 
  // We add both pass anyway and when these two passes run, one will be a
  // no-op based on the optimization level/attributes.
  addFunctionPass(X86LowerAMXIntrinsicsPass(&TM), PMW);
  addFunctionPass(X86LowerAMXTypePass(&TM), PMW);
 
  Base::addIRPasses(PMW);
 
  if (getOptLevel() != CodeGenOptLevel::None) {
    addFunctionPass(InterleavedAccessPass(TM), PMW);
    addFunctionPass(X86PartialReductionPass(&getTM()), PMW);
  }
 
  // Add passes that handle indirect branch removal and insertion of a retpoline
  // thunk. These will be a no-op unless a function subtarget has the retpoline
  // feature enabled.
  addFunctionPass(IndirectBrExpandPass(TM), PMW);
 
  // Add Control Flow Guard checks.
  const Triple &TT = TM.getTargetTriple();
  if (TT.isOSWindows())
    addFunctionPass(CFGuardPass(), PMW);
 
  if (TM.Options.JMCInstrument) {
    flushFPMsToMPM(PMW);
    addModulePass(JMCInstrumenterPass(), PMW);
  }
 
  addFunctionPass(X86FenceNonTemporalStoresPass(&getTM()), PMW);
}
 
void X86CodeGenPassBuilder::addPreISel(PassManagerWrapper &PMW) {
  // Only add this pass for 32-bit x86 Windows.
  const Triple &TT = TM.getTargetTriple();
  if (TT.isOSWindows() && TT.isX86_32()) {
    flushFPMsToMPM(PMW);
    addModulePass(X86WinEHStatePass(), PMW);
  }
}
 
Error X86CodeGenPassBuilder::addInstSelector(PassManagerWrapper &PMW) {
  addMachineFunctionPass(X86ISelDAGToDAGPass(getTM()), PMW);
 
  // For ELF, cleanup any local-dynamic TLS accesses
  if (TM.getTargetTriple().isOSBinFormatELF() &&
      getOptLevel() != CodeGenOptLevel::None) {
    addMachineFunctionPass(X86CleanupLocalDynamicTLSPass(), PMW);
  }
 
  addMachineFunctionPass(X86GlobalBaseRegPass(), PMW);
  addMachineFunctionPass(X86ArgumentStackSlotPass(), PMW);
  return Error::success();
}
 
Error X86CodeGenPassBuilder::addIRTranslator(PassManagerWrapper &PMW) {
  addMachineFunctionPass(IRTranslatorPass(getOptLevel()), PMW);
  return Error::success();
}
 
void X86CodeGenPassBuilder::addPreLegalizeMachineIR(PassManagerWrapper &PMW) {
  if (getOptLevel() != CodeGenOptLevel::None)
    addMachineFunctionPass(X86PreLegalizerCombinerPass(), PMW);
}
 
Error X86CodeGenPassBuilder::addLegalizeMachineIR(PassManagerWrapper &PMW) {
  addMachineFunctionPass(LegalizerPass(), PMW);
  return Error::success();
}
 
void X86CodeGenPassBuilder::addPreRegBankSelect(PassManagerWrapper &PMW) {
  if (getOptLevel() != CodeGenOptLevel::None)
    addMachineFunctionPass(X86PostLegalizerCombinerPass(), PMW);
}
 
Error X86CodeGenPassBuilder::addRegBankSelect(PassManagerWrapper &PMW) {
  addMachineFunctionPass(RegBankSelectPass(), PMW);
  return Error::success();
}
 
Error X86CodeGenPassBuilder::addGlobalInstructionSelect(
    PassManagerWrapper &PMW) {
  addMachineFunctionPass(InstructionSelectPass(getOptLevel()), PMW);
  // Add GlobalBaseReg in case there is no SelectionDAG passes afterwards
  if (isGlobalISelAbortEnabled())
    addMachineFunctionPass(X86GlobalBaseRegPass(), PMW);
  return Error::success();
}
 
void X86CodeGenPassBuilder::addMachineSSAOptimization(PassManagerWrapper &PMW) {
  addMachineFunctionPass(X86DomainReassignmentPass(), PMW);
  Base::addMachineSSAOptimization(PMW);
}
 
void X86CodeGenPassBuilder::addILPOpts(PassManagerWrapper &PMW) {
  addMachineFunctionPass(EarlyIfConverterPass(), PMW);
  if (getTM().getCLOpts().machine_combiner)
    addMachineFunctionPass(MachineCombinerPass(), PMW);
  addMachineFunctionPass(X86CmovConversionPass(), PMW);
}
 
void X86CodeGenPassBuilder::addPreRegAlloc(PassManagerWrapper &PMW) {
  if (getOptLevel() != CodeGenOptLevel::None) {
    addMachineFunctionPass(LiveRangeShrinkPass(), PMW);
    addMachineFunctionPass(X86FixupSetCCPass(), PMW);
    addMachineFunctionPass(X86OptimizeLEAsPass(), PMW);
    addMachineFunctionPass(X86CallFrameOptimizationPass(), PMW);
    addMachineFunctionPass(X86AvoidStoreForwardingBlocksPass(), PMW);
  }
 
  addMachineFunctionPass(X86SuppressAPXForRelocationPass(), PMW);
  addMachineFunctionPass(X86SpeculativeLoadHardeningPass(), PMW);
  addMachineFunctionPass(X86FlagsCopyLoweringPass(), PMW);
  addMachineFunctionPass(X86DynAllocaExpanderPass(), PMW);
 
  if (getOptLevel() != CodeGenOptLevel::None)
    addMachineFunctionPass(X86PreTileConfigPass(), PMW);
  else
    addMachineFunctionPass(X86FastPreTileConfigPass(), PMW);
}
 
static bool onlyAllocateTileRegisters(const TargetRegisterInfo &TRI,
                                      const MachineRegisterInfo &MRI,
                                      const Register Reg) {
  const TargetRegisterClass *RC = MRI.getRegClass(Reg);
  return static_cast<const X86RegisterInfo &>(TRI).isTileRegisterClass(RC);
}
 
Error X86CodeGenPassBuilder::addRegAssignAndRewriteFast(
    PassManagerWrapper &PMW) {
  Error PossibleError = Base::addRegAssignAndRewriteFast(PMW);
  addMachineFunctionPass(X86FastTileConfigPass(), PMW);
  return PossibleError;
}
 
Expected<bool> X86CodeGenPassBuilder::addRegAssignAndRewriteOptimized(
    PassManagerWrapper &PMW) {
  // Don't support tile RA when RA is specified by command line "-regalloc".
  if (Opt.RegAlloc == RegAllocType::Default && getTM().getCLOpts().tile_ra) {
    // Allocate tile register first.
    addMachineFunctionPass(
        RAGreedyPass({RAGreedyPass::Options(onlyAllocateTileRegisters)}), PMW);
    addMachineFunctionPass(X86TileConfigPass(), PMW);
  }
  return Base::addRegAssignAndRewriteOptimized(PMW);
}
 
void X86CodeGenPassBuilder::addPostRegAlloc(PassManagerWrapper &PMW) {
  addMachineFunctionPass(X86LowerTileCopyPass(), PMW);
  addMachineFunctionPass(X86FPStackifierPass(), PMW);
  // When -O0 is enabled, the Load Value Injection Hardening pass will fall back
  // to using the Speculative Execution Side Effect Suppression pass for
  // mitigation. This is to prevent slow downs due to
  // analyses needed by the LVIHardening pass when compiling at -O0.
  if (getOptLevel() != CodeGenOptLevel::None) {
    addMachineFunctionPass(X86LoadValueInjectionLoadHardeningPass(), PMW);
  }
}
 
void X86CodeGenPassBuilder::addPreSched2(PassManagerWrapper &PMW) {
  addMachineFunctionPass(X86ExpandPseudoPass(), PMW);
  addMachineFunctionPass(MachineKCFIPass(), PMW);
}
 
void X86CodeGenPassBuilder::addPreEmitPass(PassManagerWrapper &PMW) {
  if (getOptLevel() != CodeGenOptLevel::None) {
    addMachineFunctionPass(ExecutionDomainFixPass(), PMW);
    addMachineFunctionPass(BreakFalseDepsPass(), PMW);
  }
 
  addMachineFunctionPass(X86IndirectBranchTrackingPass(), PMW);
  addMachineFunctionPass(X86InsertVZeroUpperPass(), PMW);
 
  if (getOptLevel() != CodeGenOptLevel::None) {
    addMachineFunctionPass(X86FixupBWInstsPass(), PMW);
    addMachineFunctionPass(X86PadShortFunctionsPass(), PMW);
    addMachineFunctionPass(X86FixupLEAsPass(), PMW);
    addMachineFunctionPass(X86FixupInstTuningPass(), PMW);
    addMachineFunctionPass(X86FixupVectorConstantsPass(), PMW);
  }
  addMachineFunctionPass(X86CompressEVEXPass(), PMW);
  addMachineFunctionPass(X86InsertX87WaitPass(), PMW);
 
  if (TM.getTargetTriple().isLFI())
    addMachineFunctionPass(X86LFIRewritePass(), PMW);
}
 
void X86CodeGenPassBuilder::addPreEmitPass2(PassManagerWrapper &PMW) {
  const Triple &TT = TM.getTargetTriple();
 
  // The X86 Speculative Execution Pass must run after all control
  // flow graph modifying passes. As a result it was listed to run right before
  // the X86 Retpoline Thunks pass. The reason it must run after control flow
  // graph modifications is that the model of LFENCE in LLVM has to be updated
  // (FIXME: https://bugs.llvm.org/show_bug.cgi?id=45167). Currently the
  // placement of this pass was hand checked to ensure that the subsequent
  // passes don't move the code around the LFENCEs in a way that will hurt the
  // correctness of this pass. This placement has been shown to work based on
  // hand inspection of the codegen output.
  addMachineFunctionPass(X86SpeculativeExecutionSideEffectSuppressionPass(),
                         PMW);
  addMachineFunctionPass(X86IndirectThunksPass(), PMW);
  addMachineFunctionPass(X86ReturnThunksPass(), PMW);
 
  // Insert extra int3 instructions after trailing call instructions to avoid
  // issues in the unwinder.
  if (TT.isOSWindows() && TT.isX86_64())
    addMachineFunctionPass(X86AvoidTrailingCallPass(), PMW);
 
  // Verify basic block incoming and outgoing cfa offset and register values and
  // correct CFA calculation rule where needed by inserting appropriate CFI
  // instructions. CFIInstrInserter self-skips targets that do not use DWARF
  // CFI.
  if (!TT.isOSDarwin())
    addMachineFunctionPass(CFIInstrInserterPass(), PMW);
 
  if (TT.isOSWindows()) {
    // Identify valid longjmp targets for Windows Control Flow Guard.
    addMachineFunctionPass(CFGuardLongjmpPass(), PMW);
    // Identify valid eh continuation targets for Windows EHCont Guard.
    addMachineFunctionPass(EHContGuardTargetsPass(), PMW);
  }
 
  addMachineFunctionPass(X86LoadValueInjectionRetHardeningPass(), PMW);
 
  // Insert pseudo probe annotation for callsite profiling
  addMachineFunctionPass(PseudoProbeInserterPass(), PMW);
 
  // KCFI indirect call checks are lowered to a bundle, and on Darwin platforms,
  // also CALL_RVMARKER.
  addMachineFunctionPass(
      UnpackMachineBundlesPass([&TT](const MachineFunction &MF) {
        // Only run bundle expansion if the module uses kcfi, or there are
        // relevant ObjC runtime functions present in the module.
        const Function &F = MF.getFunction();
        const Module *M = F.getParent();
        return M->getModuleFlag("kcfi") ||
               (TT.isOSDarwin() &&
                (M->getFunction("objc_retainAutoreleasedReturnValue") ||
                 M->getFunction("objc_unsafeClaimAutoreleasedReturnValue")));
      }),
      PMW);
 
  // Analyzes and emits pseudos to support Win x64 Unwind V2. This pass must run
  // after all real instructions have been added to the epilog.
  if (TT.isOSWindows() && TT.isX86_64()) {
    addMachineFunctionPass(X86WinEHUnwindV2Pass(), PMW);
  }
}
 
void X86CodeGenPassBuilder::addAsmPrinterBegin(PassManagerWrapper &PMW) {
  addModulePass(X86AsmPrinterBeginPass(), PMW, /*Force=*/true);
}
 
void X86CodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) {
  addMachineFunctionPass(X86AsmPrinterPass(), PMW);
}
 
void X86CodeGenPassBuilder::addAsmPrinterEnd(PassManagerWrapper &PMW) {
  addModulePass(X86AsmPrinterEndPass(), PMW, /*Force=*/true);
}
 
} // namespace
 
void X86TargetMachine::registerPassBuilderCallbacks(PassBuilder &PB){
#define GET_PASS_REGISTRY "X86PassRegistry.def"
#include "llvm/Passes/TargetPassRegistry.inc"
}
 
Error X86TargetMachine::buildCodeGenPipeline(
    ModulePassManager &MPM, ModuleAnalysisManager &MAM, raw_pwrite_stream &Out,
    raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
    const CGPassBuilderOption &Opt, MCContext &Ctx,
    PassInstrumentationCallbacks *PIC) {
  auto CGPB = X86CodeGenPassBuilder(*this, Opt, PIC);
  return CGPB.buildPipeline(MPM, MAM, Out, DwoOut, FileType, Ctx);
}