| File: | build/source/llvm/lib/CodeGen/MachineFunctionSplitter.cpp |
| Warning: | line 100, column 35 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | //===-- MachineFunctionSplitter.cpp - Split machine functions //-----------===// | |||
| 2 | // | |||
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
| 4 | // See https://llvm.org/LICENSE.txt for license information. | |||
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
| 6 | // | |||
| 7 | //===----------------------------------------------------------------------===// | |||
| 8 | // | |||
| 9 | // \file | |||
| 10 | // Uses profile information to split out cold blocks. | |||
| 11 | // | |||
| 12 | // This pass splits out cold machine basic blocks from the parent function. This | |||
| 13 | // implementation leverages the basic block section framework. Blocks marked | |||
| 14 | // cold by this pass are grouped together in a separate section prefixed with | |||
| 15 | // ".text.unlikely.*". The linker can then group these together as a cold | |||
| 16 | // section. The split part of the function is a contiguous region identified by | |||
| 17 | // the symbol "foo.cold". Grouping all cold blocks across functions together | |||
| 18 | // decreases fragmentation and improves icache and itlb utilization. Note that | |||
| 19 | // the overall changes to the binary size are negligible; only a small number of | |||
| 20 | // additional jump instructions may be introduced. | |||
| 21 | // | |||
| 22 | // For the original RFC of this pass please see | |||
| 23 | // https://groups.google.com/d/msg/llvm-dev/RUegaMg-iqc/wFAVxa6fCgAJ | |||
| 24 | //===----------------------------------------------------------------------===// | |||
| 25 | ||||
| 26 | #include "llvm/ADT/SmallVector.h" | |||
| 27 | #include "llvm/Analysis/EHUtils.h" | |||
| 28 | #include "llvm/Analysis/ProfileSummaryInfo.h" | |||
| 29 | #include "llvm/CodeGen/BasicBlockSectionUtils.h" | |||
| 30 | #include "llvm/CodeGen/MachineBasicBlock.h" | |||
| 31 | #include "llvm/CodeGen/MachineBlockFrequencyInfo.h" | |||
| 32 | #include "llvm/CodeGen/MachineFunction.h" | |||
| 33 | #include "llvm/CodeGen/MachineFunctionPass.h" | |||
| 34 | #include "llvm/CodeGen/MachineModuleInfo.h" | |||
| 35 | #include "llvm/CodeGen/Passes.h" | |||
| 36 | #include "llvm/IR/Function.h" | |||
| 37 | #include "llvm/InitializePasses.h" | |||
| 38 | #include "llvm/Support/CommandLine.h" | |||
| 39 | #include <optional> | |||
| 40 | ||||
| 41 | using namespace llvm; | |||
| 42 | ||||
| 43 | // FIXME: This cutoff value is CPU dependent and should be moved to | |||
| 44 | // TargetTransformInfo once we consider enabling this on other platforms. | |||
| 45 | // The value is expressed as a ProfileSummaryInfo integer percentile cutoff. | |||
| 46 | // Defaults to 999950, i.e. all blocks colder than 99.995 percentile are split. | |||
| 47 | // The default was empirically determined to be optimal when considering cutoff | |||
| 48 | // values between 99%-ile to 100%-ile with respect to iTLB and icache metrics on | |||
| 49 | // Intel CPUs. | |||
| 50 | static cl::opt<unsigned> | |||
| 51 | PercentileCutoff("mfs-psi-cutoff", | |||
| 52 | cl::desc("Percentile profile summary cutoff used to " | |||
| 53 | "determine cold blocks. Unused if set to zero."), | |||
| 54 | cl::init(999950), cl::Hidden); | |||
| 55 | ||||
| 56 | static cl::opt<unsigned> ColdCountThreshold( | |||
| 57 | "mfs-count-threshold", | |||
| 58 | cl::desc( | |||
| 59 | "Minimum number of times a block must be executed to be retained."), | |||
| 60 | cl::init(1), cl::Hidden); | |||
| 61 | ||||
| 62 | static cl::opt<bool> SplitAllEHCode( | |||
| 63 | "mfs-split-ehcode", | |||
| 64 | cl::desc("Splits all EH code and it's descendants by default."), | |||
| 65 | cl::init(false), cl::Hidden); | |||
| 66 | ||||
| 67 | namespace { | |||
| 68 | ||||
| 69 | class MachineFunctionSplitter : public MachineFunctionPass { | |||
| 70 | public: | |||
| 71 | static char ID; | |||
| 72 | MachineFunctionSplitter() : MachineFunctionPass(ID) { | |||
| 73 | initializeMachineFunctionSplitterPass(*PassRegistry::getPassRegistry()); | |||
| 74 | } | |||
| 75 | ||||
| 76 | StringRef getPassName() const override { | |||
| 77 | return "Machine Function Splitter Transformation"; | |||
| 78 | } | |||
| 79 | ||||
| 80 | void getAnalysisUsage(AnalysisUsage &AU) const override; | |||
| 81 | ||||
| 82 | bool runOnMachineFunction(MachineFunction &F) override; | |||
| 83 | }; | |||
| 84 | } // end anonymous namespace | |||
| 85 | ||||
| 86 | /// setDescendantEHBlocksCold - This splits all EH pads and blocks reachable | |||
| 87 | /// only by EH pad as cold. This will help mark EH pads statically cold | |||
| 88 | /// instead of relying on profile data. | |||
| 89 | static void setDescendantEHBlocksCold(MachineFunction &MF) { | |||
| 90 | DenseSet<MachineBasicBlock *> EHBlocks; | |||
| 91 | computeEHOnlyBlocks(MF, EHBlocks); | |||
| 92 | for (auto Block : EHBlocks) { | |||
| 93 | Block->setSectionID(MBBSectionID::ColdSectionID); | |||
| 94 | } | |||
| 95 | } | |||
| 96 | ||||
| 97 | static bool isColdBlock(const MachineBasicBlock &MBB, | |||
| 98 | const MachineBlockFrequencyInfo *MBFI, | |||
| 99 | ProfileSummaryInfo *PSI) { | |||
| 100 | std::optional<uint64_t> Count = MBFI->getBlockProfileCount(&MBB); | |||
| ||||
| 101 | if (!Count) | |||
| 102 | return true; | |||
| 103 | ||||
| 104 | if (PercentileCutoff > 0) { | |||
| 105 | return PSI->isColdCountNthPercentile(PercentileCutoff, *Count); | |||
| 106 | } | |||
| 107 | return (*Count < ColdCountThreshold); | |||
| 108 | } | |||
| 109 | ||||
| 110 | bool MachineFunctionSplitter::runOnMachineFunction(MachineFunction &MF) { | |||
| 111 | // We target functions with profile data. Static information in the form | |||
| 112 | // of exception handling code may be split to cold if user passes the | |||
| 113 | // mfs-split-ehcode flag. | |||
| 114 | bool UseProfileData = MF.getFunction().hasProfileData(); | |||
| 115 | if (!UseProfileData && !SplitAllEHCode) | |||
| ||||
| 116 | return false; | |||
| 117 | ||||
| 118 | // TODO: We don't split functions where a section attribute has been set | |||
| 119 | // since the split part may not be placed in a contiguous region. It may also | |||
| 120 | // be more beneficial to augment the linker to ensure contiguous layout of | |||
| 121 | // split functions within the same section as specified by the attribute. | |||
| 122 | if (MF.getFunction().hasSection() || | |||
| 123 | MF.getFunction().hasFnAttribute("implicit-section-name")) | |||
| 124 | return false; | |||
| 125 | ||||
| 126 | // We don't want to proceed further for cold functions | |||
| 127 | // or functions of unknown hotness. Lukewarm functions have no prefix. | |||
| 128 | std::optional<StringRef> SectionPrefix = MF.getFunction().getSectionPrefix(); | |||
| 129 | if (SectionPrefix && | |||
| 130 | (*SectionPrefix == "unlikely" || *SectionPrefix == "unknown")) { | |||
| 131 | return false; | |||
| 132 | } | |||
| 133 | ||||
| 134 | // Renumbering blocks here preserves the order of the blocks as | |||
| 135 | // sortBasicBlocksAndUpdateBranches uses the numeric identifier to sort | |||
| 136 | // blocks. Preserving the order of blocks is essential to retaining decisions | |||
| 137 | // made by prior passes such as MachineBlockPlacement. | |||
| 138 | MF.RenumberBlocks(); | |||
| 139 | MF.setBBSectionsType(BasicBlockSection::Preset); | |||
| 140 | ||||
| 141 | MachineBlockFrequencyInfo *MBFI = nullptr; | |||
| 142 | ProfileSummaryInfo *PSI = nullptr; | |||
| 143 | if (UseProfileData
| |||
| 144 | MBFI = &getAnalysis<MachineBlockFrequencyInfo>(); | |||
| 145 | PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI(); | |||
| 146 | } | |||
| 147 | ||||
| 148 | SmallVector<MachineBasicBlock *, 2> LandingPads; | |||
| 149 | for (auto &MBB : MF) { | |||
| 150 | if (MBB.isEntryBlock()) | |||
| 151 | continue; | |||
| 152 | ||||
| 153 | if (MBB.isEHPad()) | |||
| 154 | LandingPads.push_back(&MBB); | |||
| 155 | else if (UseProfileData && isColdBlock(MBB, MBFI, PSI) && !SplitAllEHCode) | |||
| 156 | MBB.setSectionID(MBBSectionID::ColdSectionID); | |||
| 157 | } | |||
| 158 | ||||
| 159 | // Split all EH code and it's descendant statically by default. | |||
| 160 | if (SplitAllEHCode) | |||
| 161 | setDescendantEHBlocksCold(MF); | |||
| 162 | // We only split out eh pads if all of them are cold. | |||
| 163 | else { | |||
| 164 | bool HasHotLandingPads = false; | |||
| 165 | for (const MachineBasicBlock *LP : LandingPads) { | |||
| 166 | if (!isColdBlock(*LP, MBFI, PSI)) | |||
| 167 | HasHotLandingPads = true; | |||
| 168 | } | |||
| 169 | if (!HasHotLandingPads) { | |||
| 170 | for (MachineBasicBlock *LP : LandingPads) | |||
| 171 | LP->setSectionID(MBBSectionID::ColdSectionID); | |||
| 172 | } | |||
| 173 | } | |||
| 174 | auto Comparator = [](const MachineBasicBlock &X, const MachineBasicBlock &Y) { | |||
| 175 | return X.getSectionID().Type < Y.getSectionID().Type; | |||
| 176 | }; | |||
| 177 | llvm::sortBasicBlocksAndUpdateBranches(MF, Comparator); | |||
| 178 | llvm::avoidZeroOffsetLandingPad(MF); | |||
| 179 | return true; | |||
| 180 | } | |||
| 181 | ||||
| 182 | void MachineFunctionSplitter::getAnalysisUsage(AnalysisUsage &AU) const { | |||
| 183 | AU.addRequired<MachineModuleInfoWrapperPass>(); | |||
| 184 | AU.addRequired<MachineBlockFrequencyInfo>(); | |||
| 185 | AU.addRequired<ProfileSummaryInfoWrapperPass>(); | |||
| 186 | } | |||
| 187 | ||||
| 188 | char MachineFunctionSplitter::ID = 0; | |||
| 189 | INITIALIZE_PASS(MachineFunctionSplitter, "machine-function-splitter",static void *initializeMachineFunctionSplitterPassOnce(PassRegistry &Registry) { PassInfo *PI = new PassInfo( "Split machine functions using profile information" , "machine-function-splitter", &MachineFunctionSplitter:: ID, PassInfo::NormalCtor_t(callDefaultCtor<MachineFunctionSplitter >), false, false); Registry.registerPass(*PI, true); return PI; } static llvm::once_flag InitializeMachineFunctionSplitterPassFlag ; void llvm::initializeMachineFunctionSplitterPass(PassRegistry &Registry) { llvm::call_once(InitializeMachineFunctionSplitterPassFlag , initializeMachineFunctionSplitterPassOnce, std::ref(Registry )); } | |||
| 190 | "Split machine functions using profile information", false,static void *initializeMachineFunctionSplitterPassOnce(PassRegistry &Registry) { PassInfo *PI = new PassInfo( "Split machine functions using profile information" , "machine-function-splitter", &MachineFunctionSplitter:: ID, PassInfo::NormalCtor_t(callDefaultCtor<MachineFunctionSplitter >), false, false); Registry.registerPass(*PI, true); return PI; } static llvm::once_flag InitializeMachineFunctionSplitterPassFlag ; void llvm::initializeMachineFunctionSplitterPass(PassRegistry &Registry) { llvm::call_once(InitializeMachineFunctionSplitterPassFlag , initializeMachineFunctionSplitterPassOnce, std::ref(Registry )); } | |||
| 191 | false)static void *initializeMachineFunctionSplitterPassOnce(PassRegistry &Registry) { PassInfo *PI = new PassInfo( "Split machine functions using profile information" , "machine-function-splitter", &MachineFunctionSplitter:: ID, PassInfo::NormalCtor_t(callDefaultCtor<MachineFunctionSplitter >), false, false); Registry.registerPass(*PI, true); return PI; } static llvm::once_flag InitializeMachineFunctionSplitterPassFlag ; void llvm::initializeMachineFunctionSplitterPass(PassRegistry &Registry) { llvm::call_once(InitializeMachineFunctionSplitterPassFlag , initializeMachineFunctionSplitterPassOnce, std::ref(Registry )); } | |||
| 192 | ||||
| 193 | MachineFunctionPass *llvm::createMachineFunctionSplitterPass() { | |||
| 194 | return new MachineFunctionSplitter(); | |||
| 195 | } |