File: | build/source/llvm/lib/CodeGen/MachineFunctionSplitter.cpp |
Warning: | line 160, column 30 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===-- MachineFunctionSplitter.cpp - Split machine functions //-----------===// | |||
2 | // | |||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
4 | // See https://llvm.org/LICENSE.txt for license information. | |||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
6 | // | |||
7 | //===----------------------------------------------------------------------===// | |||
8 | // | |||
9 | // \file | |||
10 | // Uses profile information to split out cold blocks. | |||
11 | // | |||
12 | // This pass splits out cold machine basic blocks from the parent function. This | |||
13 | // implementation leverages the basic block section framework. Blocks marked | |||
14 | // cold by this pass are grouped together in a separate section prefixed with | |||
15 | // ".text.unlikely.*". The linker can then group these together as a cold | |||
16 | // section. The split part of the function is a contiguous region identified by | |||
17 | // the symbol "foo.cold". Grouping all cold blocks across functions together | |||
18 | // decreases fragmentation and improves icache and itlb utilization. Note that | |||
19 | // the overall changes to the binary size are negligible; only a small number of | |||
20 | // additional jump instructions may be introduced. | |||
21 | // | |||
22 | // For the original RFC of this pass please see | |||
23 | // https://groups.google.com/d/msg/llvm-dev/RUegaMg-iqc/wFAVxa6fCgAJ | |||
24 | //===----------------------------------------------------------------------===// | |||
25 | ||||
26 | #include "llvm/ADT/SmallVector.h" | |||
27 | #include "llvm/Analysis/ProfileSummaryInfo.h" | |||
28 | #include "llvm/CodeGen/BasicBlockSectionUtils.h" | |||
29 | #include "llvm/CodeGen/MachineBasicBlock.h" | |||
30 | #include "llvm/CodeGen/MachineBlockFrequencyInfo.h" | |||
31 | #include "llvm/CodeGen/MachineFunction.h" | |||
32 | #include "llvm/CodeGen/MachineFunctionPass.h" | |||
33 | #include "llvm/CodeGen/MachineModuleInfo.h" | |||
34 | #include "llvm/CodeGen/Passes.h" | |||
35 | #include "llvm/IR/Function.h" | |||
36 | #include "llvm/InitializePasses.h" | |||
37 | #include "llvm/Support/CommandLine.h" | |||
38 | ||||
39 | using namespace llvm; | |||
40 | ||||
41 | // FIXME: This cutoff value is CPU dependent and should be moved to | |||
42 | // TargetTransformInfo once we consider enabling this on other platforms. | |||
43 | // The value is expressed as a ProfileSummaryInfo integer percentile cutoff. | |||
44 | // Defaults to 999950, i.e. all blocks colder than 99.995 percentile are split. | |||
45 | // The default was empirically determined to be optimal when considering cutoff | |||
46 | // values between 99%-ile to 100%-ile with respect to iTLB and icache metrics on | |||
47 | // Intel CPUs. | |||
48 | static cl::opt<unsigned> | |||
49 | PercentileCutoff("mfs-psi-cutoff", | |||
50 | cl::desc("Percentile profile summary cutoff used to " | |||
51 | "determine cold blocks. Unused if set to zero."), | |||
52 | cl::init(999950), cl::Hidden); | |||
53 | ||||
54 | static cl::opt<unsigned> ColdCountThreshold( | |||
55 | "mfs-count-threshold", | |||
56 | cl::desc( | |||
57 | "Minimum number of times a block must be executed to be retained."), | |||
58 | cl::init(1), cl::Hidden); | |||
59 | ||||
60 | static cl::opt<bool> SplitAllEHCode( | |||
61 | "mfs-split-ehcode", | |||
62 | cl::desc("Splits all EH code and it's descendants by default."), | |||
63 | cl::init(false), cl::Hidden); | |||
64 | ||||
65 | namespace { | |||
66 | ||||
67 | class MachineFunctionSplitter : public MachineFunctionPass { | |||
68 | public: | |||
69 | static char ID; | |||
70 | MachineFunctionSplitter() : MachineFunctionPass(ID) { | |||
71 | initializeMachineFunctionSplitterPass(*PassRegistry::getPassRegistry()); | |||
72 | } | |||
73 | ||||
74 | StringRef getPassName() const override { | |||
75 | return "Machine Function Splitter Transformation"; | |||
76 | } | |||
77 | ||||
78 | void getAnalysisUsage(AnalysisUsage &AU) const override; | |||
79 | ||||
80 | bool runOnMachineFunction(MachineFunction &F) override; | |||
81 | }; | |||
82 | } // end anonymous namespace | |||
83 | ||||
84 | /// setDescendantEHBlocksCold - This splits all EH pads and blocks reachable | |||
85 | /// only by EH pad as cold. This will help mark EH pads statically cold instead | |||
86 | /// of relying on profile data. | |||
87 | static void | |||
88 | setDescendantEHBlocksCold(SmallVectorImpl<MachineBasicBlock *> &EHBlocks, | |||
89 | MachineFunction &MF) { | |||
90 | MachineBasicBlock *StartBlock = &MF.front(); | |||
91 | // A block can be unknown if its not reachable from anywhere | |||
92 | // EH if its only reachable from start blocks via some path through EH pads | |||
93 | // NonEH if it's reachable from Non EH blocks as well. | |||
94 | enum Status { Unknown = 0, EH = 1, NonEH = 2 }; | |||
95 | DenseSet<MachineBasicBlock *> WorkList; | |||
96 | DenseMap<MachineBasicBlock *, Status> Statuses; | |||
97 | ||||
98 | auto getStatus = [&](MachineBasicBlock *MBB) { | |||
99 | if (Statuses.find(MBB) != Statuses.end()) | |||
100 | return Statuses[MBB]; | |||
101 | else | |||
102 | return Unknown; | |||
103 | }; | |||
104 | ||||
105 | auto checkPredecessors = [&](MachineBasicBlock *MBB, Status Stat) { | |||
106 | for (auto *PredMBB : MBB->predecessors()) { | |||
107 | Status PredStatus = getStatus(PredMBB); | |||
108 | // If status of predecessor block has gone above current block | |||
109 | // we update current blocks status. | |||
110 | if (PredStatus > Stat) | |||
111 | Stat = PredStatus; | |||
112 | } | |||
113 | return Stat; | |||
114 | }; | |||
115 | ||||
116 | auto addSuccesors = [&](MachineBasicBlock *MBB) { | |||
117 | for (auto *SuccMBB : MBB->successors()) { | |||
118 | if (!SuccMBB->isEHPad()) | |||
119 | WorkList.insert(SuccMBB); | |||
120 | } | |||
121 | }; | |||
122 | ||||
123 | // Insert the successors of start block | |||
124 | // and landing pads successor. | |||
125 | Statuses[StartBlock] = NonEH; | |||
126 | addSuccesors(StartBlock); | |||
127 | for (auto *LP : EHBlocks) { | |||
128 | addSuccesors(LP); | |||
129 | Statuses[LP] = EH; | |||
130 | } | |||
131 | ||||
132 | // Worklist iterative algorithm. | |||
133 | while (!WorkList.empty()) { | |||
134 | auto *MBB = *WorkList.begin(); | |||
135 | WorkList.erase(MBB); | |||
136 | ||||
137 | Status OldStatus = getStatus(MBB); | |||
138 | ||||
139 | // Check on predecessors and check for | |||
140 | // Status update. | |||
141 | Status NewStatus = checkPredecessors(MBB, OldStatus); | |||
142 | ||||
143 | // Did the block status change? | |||
144 | bool changed = OldStatus != NewStatus; | |||
145 | if (changed) { | |||
146 | addSuccesors(MBB); | |||
147 | Statuses[MBB] = NewStatus; | |||
148 | } | |||
149 | } | |||
150 | ||||
151 | for (auto Entry : Statuses) { | |||
152 | if (Entry.second == EH) | |||
153 | Entry.first->setSectionID(MBBSectionID::ColdSectionID); | |||
154 | } | |||
155 | } | |||
156 | ||||
157 | static bool isColdBlock(const MachineBasicBlock &MBB, | |||
158 | const MachineBlockFrequencyInfo *MBFI, | |||
159 | ProfileSummaryInfo *PSI) { | |||
160 | Optional<uint64_t> Count = MBFI->getBlockProfileCount(&MBB); | |||
| ||||
161 | if (!Count) | |||
162 | return true; | |||
163 | ||||
164 | if (PercentileCutoff > 0) { | |||
165 | return PSI->isColdCountNthPercentile(PercentileCutoff, *Count); | |||
166 | } | |||
167 | return (*Count < ColdCountThreshold); | |||
168 | } | |||
169 | ||||
170 | bool MachineFunctionSplitter::runOnMachineFunction(MachineFunction &MF) { | |||
171 | // We target functions with profile data. Static information in the form | |||
172 | // of exception handling code may be split to cold if user passes the | |||
173 | // mfs-split-ehcode flag. | |||
174 | bool UseProfileData = MF.getFunction().hasProfileData(); | |||
175 | if (!UseProfileData && !SplitAllEHCode) | |||
| ||||
176 | return false; | |||
177 | ||||
178 | // TODO: We don't split functions where a section attribute has been set | |||
179 | // since the split part may not be placed in a contiguous region. It may also | |||
180 | // be more beneficial to augment the linker to ensure contiguous layout of | |||
181 | // split functions within the same section as specified by the attribute. | |||
182 | if (MF.getFunction().hasSection() || | |||
183 | MF.getFunction().hasFnAttribute("implicit-section-name")) | |||
184 | return false; | |||
185 | ||||
186 | // We don't want to proceed further for cold functions | |||
187 | // or functions of unknown hotness. Lukewarm functions have no prefix. | |||
188 | std::optional<StringRef> SectionPrefix = MF.getFunction().getSectionPrefix(); | |||
189 | if (SectionPrefix && (SectionPrefix.value().equals("unlikely") || | |||
190 | SectionPrefix.value().equals("unknown"))) { | |||
191 | return false; | |||
192 | } | |||
193 | ||||
194 | // Renumbering blocks here preserves the order of the blocks as | |||
195 | // sortBasicBlocksAndUpdateBranches uses the numeric identifier to sort | |||
196 | // blocks. Preserving the order of blocks is essential to retaining decisions | |||
197 | // made by prior passes such as MachineBlockPlacement. | |||
198 | MF.RenumberBlocks(); | |||
199 | MF.setBBSectionsType(BasicBlockSection::Preset); | |||
200 | ||||
201 | MachineBlockFrequencyInfo *MBFI = nullptr; | |||
202 | ProfileSummaryInfo *PSI = nullptr; | |||
203 | if (UseProfileData
| |||
204 | MBFI = &getAnalysis<MachineBlockFrequencyInfo>(); | |||
205 | PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI(); | |||
206 | } | |||
207 | ||||
208 | SmallVector<MachineBasicBlock *, 2> LandingPads; | |||
209 | for (auto &MBB : MF) { | |||
210 | if (MBB.isEntryBlock()) | |||
211 | continue; | |||
212 | ||||
213 | if (MBB.isEHPad()) | |||
214 | LandingPads.push_back(&MBB); | |||
215 | else if (UseProfileData && isColdBlock(MBB, MBFI, PSI) && !SplitAllEHCode) | |||
216 | MBB.setSectionID(MBBSectionID::ColdSectionID); | |||
217 | } | |||
218 | ||||
219 | // Split all EH code and it's descendant statically by default. | |||
220 | if (SplitAllEHCode) | |||
221 | setDescendantEHBlocksCold(LandingPads, MF); | |||
222 | // We only split out eh pads if all of them are cold. | |||
223 | else { | |||
224 | bool HasHotLandingPads = false; | |||
225 | for (const MachineBasicBlock *LP : LandingPads) { | |||
226 | if (!isColdBlock(*LP, MBFI, PSI)) | |||
227 | HasHotLandingPads = true; | |||
228 | } | |||
229 | if (!HasHotLandingPads) { | |||
230 | for (MachineBasicBlock *LP : LandingPads) | |||
231 | LP->setSectionID(MBBSectionID::ColdSectionID); | |||
232 | } | |||
233 | } | |||
234 | auto Comparator = [](const MachineBasicBlock &X, const MachineBasicBlock &Y) { | |||
235 | return X.getSectionID().Type < Y.getSectionID().Type; | |||
236 | }; | |||
237 | llvm::sortBasicBlocksAndUpdateBranches(MF, Comparator); | |||
238 | llvm::avoidZeroOffsetLandingPad(MF); | |||
239 | return true; | |||
240 | } | |||
241 | ||||
242 | void MachineFunctionSplitter::getAnalysisUsage(AnalysisUsage &AU) const { | |||
243 | AU.addRequired<MachineModuleInfoWrapperPass>(); | |||
244 | AU.addRequired<MachineBlockFrequencyInfo>(); | |||
245 | AU.addRequired<ProfileSummaryInfoWrapperPass>(); | |||
246 | } | |||
247 | ||||
248 | char MachineFunctionSplitter::ID = 0; | |||
249 | INITIALIZE_PASS(MachineFunctionSplitter, "machine-function-splitter",static void *initializeMachineFunctionSplitterPassOnce(PassRegistry &Registry) { PassInfo *PI = new PassInfo( "Split machine functions using profile information" , "machine-function-splitter", &MachineFunctionSplitter:: ID, PassInfo::NormalCtor_t(callDefaultCtor<MachineFunctionSplitter >), false, false); Registry.registerPass(*PI, true); return PI; } static llvm::once_flag InitializeMachineFunctionSplitterPassFlag ; void llvm::initializeMachineFunctionSplitterPass(PassRegistry &Registry) { llvm::call_once(InitializeMachineFunctionSplitterPassFlag , initializeMachineFunctionSplitterPassOnce, std::ref(Registry )); } | |||
250 | "Split machine functions using profile information", false,static void *initializeMachineFunctionSplitterPassOnce(PassRegistry &Registry) { PassInfo *PI = new PassInfo( "Split machine functions using profile information" , "machine-function-splitter", &MachineFunctionSplitter:: ID, PassInfo::NormalCtor_t(callDefaultCtor<MachineFunctionSplitter >), false, false); Registry.registerPass(*PI, true); return PI; } static llvm::once_flag InitializeMachineFunctionSplitterPassFlag ; void llvm::initializeMachineFunctionSplitterPass(PassRegistry &Registry) { llvm::call_once(InitializeMachineFunctionSplitterPassFlag , initializeMachineFunctionSplitterPassOnce, std::ref(Registry )); } | |||
251 | false)static void *initializeMachineFunctionSplitterPassOnce(PassRegistry &Registry) { PassInfo *PI = new PassInfo( "Split machine functions using profile information" , "machine-function-splitter", &MachineFunctionSplitter:: ID, PassInfo::NormalCtor_t(callDefaultCtor<MachineFunctionSplitter >), false, false); Registry.registerPass(*PI, true); return PI; } static llvm::once_flag InitializeMachineFunctionSplitterPassFlag ; void llvm::initializeMachineFunctionSplitterPass(PassRegistry &Registry) { llvm::call_once(InitializeMachineFunctionSplitterPassFlag , initializeMachineFunctionSplitterPassOnce, std::ref(Registry )); } | |||
252 | ||||
253 | MachineFunctionPass *llvm::createMachineFunctionSplitterPass() { | |||
254 | return new MachineFunctionSplitter(); | |||
255 | } |