Bug Summary

File:llvm/lib/Transforms/IPO/SampleProfile.cpp
Warning:line 1661, column 11
4th function call argument is an uninitialized value

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name SampleProfile.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -fno-split-dwarf-inlining -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-12/lib/clang/12.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/build-llvm/lib/Transforms/IPO -I /build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/lib/Transforms/IPO -I /build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/build-llvm/include -I /build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-12/lib/clang/12.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/build-llvm/lib/Transforms/IPO -fdebug-prefix-map=/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5=. -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2021-01-16-002530-32805-1 -x c++ /build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/lib/Transforms/IPO/SampleProfile.cpp

/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/lib/Transforms/IPO/SampleProfile.cpp

1//===- SampleProfile.cpp - Incorporate sample profiles into the IR --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SampleProfileLoader transformation. This pass
10// reads a profile file generated by a sampling profiler (e.g. Linux Perf -
11// http://perf.wiki.kernel.org/) and generates IR metadata to reflect the
12// profile information in the given profile.
13//
14// This pass generates branch weight annotations on the IR:
15//
16// - prof: Represents branch weights. This annotation is added to branches
17// to indicate the weights of each edge coming out of the branch.
18// The weight of each edge is the weight of the target block for
19// that edge. The weight of a block B is computed as the maximum
20// number of samples found in B.
21//
22//===----------------------------------------------------------------------===//
23
24#include "llvm/Transforms/IPO/SampleProfile.h"
25#include "llvm/ADT/ArrayRef.h"
26#include "llvm/ADT/DenseMap.h"
27#include "llvm/ADT/DenseSet.h"
28#include "llvm/ADT/None.h"
29#include "llvm/ADT/SCCIterator.h"
30#include "llvm/ADT/SmallPtrSet.h"
31#include "llvm/ADT/SmallSet.h"
32#include "llvm/ADT/SmallVector.h"
33#include "llvm/ADT/Statistic.h"
34#include "llvm/ADT/StringMap.h"
35#include "llvm/ADT/StringRef.h"
36#include "llvm/ADT/Twine.h"
37#include "llvm/Analysis/AssumptionCache.h"
38#include "llvm/Analysis/CallGraph.h"
39#include "llvm/Analysis/CallGraphSCCPass.h"
40#include "llvm/Analysis/InlineAdvisor.h"
41#include "llvm/Analysis/InlineCost.h"
42#include "llvm/Analysis/LoopInfo.h"
43#include "llvm/Analysis/OptimizationRemarkEmitter.h"
44#include "llvm/Analysis/PostDominators.h"
45#include "llvm/Analysis/ProfileSummaryInfo.h"
46#include "llvm/Analysis/ReplayInlineAdvisor.h"
47#include "llvm/Analysis/TargetLibraryInfo.h"
48#include "llvm/Analysis/TargetTransformInfo.h"
49#include "llvm/IR/BasicBlock.h"
50#include "llvm/IR/CFG.h"
51#include "llvm/IR/DebugInfoMetadata.h"
52#include "llvm/IR/DebugLoc.h"
53#include "llvm/IR/DiagnosticInfo.h"
54#include "llvm/IR/Dominators.h"
55#include "llvm/IR/Function.h"
56#include "llvm/IR/GlobalValue.h"
57#include "llvm/IR/InstrTypes.h"
58#include "llvm/IR/Instruction.h"
59#include "llvm/IR/Instructions.h"
60#include "llvm/IR/IntrinsicInst.h"
61#include "llvm/IR/LLVMContext.h"
62#include "llvm/IR/MDBuilder.h"
63#include "llvm/IR/Module.h"
64#include "llvm/IR/PassManager.h"
65#include "llvm/IR/ValueSymbolTable.h"
66#include "llvm/InitializePasses.h"
67#include "llvm/Pass.h"
68#include "llvm/ProfileData/InstrProf.h"
69#include "llvm/ProfileData/SampleProf.h"
70#include "llvm/ProfileData/SampleProfReader.h"
71#include "llvm/Support/Casting.h"
72#include "llvm/Support/CommandLine.h"
73#include "llvm/Support/Debug.h"
74#include "llvm/Support/ErrorHandling.h"
75#include "llvm/Support/ErrorOr.h"
76#include "llvm/Support/GenericDomTree.h"
77#include "llvm/Support/raw_ostream.h"
78#include "llvm/Transforms/IPO.h"
79#include "llvm/Transforms/IPO/SampleContextTracker.h"
80#include "llvm/Transforms/IPO/SampleProfileProbe.h"
81#include "llvm/Transforms/Instrumentation.h"
82#include "llvm/Transforms/Utils/CallPromotionUtils.h"
83#include "llvm/Transforms/Utils/Cloning.h"
84#include <algorithm>
85#include <cassert>
86#include <cstdint>
87#include <functional>
88#include <limits>
89#include <map>
90#include <memory>
91#include <queue>
92#include <string>
93#include <system_error>
94#include <utility>
95#include <vector>
96
97using namespace llvm;
98using namespace sampleprof;
99using ProfileCount = Function::ProfileCount;
100#define DEBUG_TYPE"sample-profile" "sample-profile"
101#define CSINLINE_DEBUG"sample-profile" "-inline" DEBUG_TYPE"sample-profile" "-inline"
102
103STATISTIC(NumCSInlined,static llvm::Statistic NumCSInlined = {"sample-profile", "NumCSInlined"
, "Number of functions inlined with context sensitive profile"
}
104 "Number of functions inlined with context sensitive profile")static llvm::Statistic NumCSInlined = {"sample-profile", "NumCSInlined"
, "Number of functions inlined with context sensitive profile"
}
;
105STATISTIC(NumCSNotInlined,static llvm::Statistic NumCSNotInlined = {"sample-profile", "NumCSNotInlined"
, "Number of functions not inlined with context sensitive profile"
}
106 "Number of functions not inlined with context sensitive profile")static llvm::Statistic NumCSNotInlined = {"sample-profile", "NumCSNotInlined"
, "Number of functions not inlined with context sensitive profile"
}
;
107STATISTIC(NumMismatchedProfile,static llvm::Statistic NumMismatchedProfile = {"sample-profile"
, "NumMismatchedProfile", "Number of functions with CFG mismatched profile"
}
108 "Number of functions with CFG mismatched profile")static llvm::Statistic NumMismatchedProfile = {"sample-profile"
, "NumMismatchedProfile", "Number of functions with CFG mismatched profile"
}
;
109STATISTIC(NumMatchedProfile, "Number of functions with CFG matched profile")static llvm::Statistic NumMatchedProfile = {"sample-profile",
"NumMatchedProfile", "Number of functions with CFG matched profile"
}
;
110
111// Command line option to specify the file to read samples from. This is
112// mainly used for debugging.
113static cl::opt<std::string> SampleProfileFile(
114 "sample-profile-file", cl::init(""), cl::value_desc("filename"),
115 cl::desc("Profile file loaded by -sample-profile"), cl::Hidden);
116
117// The named file contains a set of transformations that may have been applied
118// to the symbol names between the program from which the sample data was
119// collected and the current program's symbols.
120static cl::opt<std::string> SampleProfileRemappingFile(
121 "sample-profile-remapping-file", cl::init(""), cl::value_desc("filename"),
122 cl::desc("Profile remapping file loaded by -sample-profile"), cl::Hidden);
123
124static cl::opt<unsigned> SampleProfileMaxPropagateIterations(
125 "sample-profile-max-propagate-iterations", cl::init(100),
126 cl::desc("Maximum number of iterations to go through when propagating "
127 "sample block/edge weights through the CFG."));
128
129static cl::opt<unsigned> SampleProfileRecordCoverage(
130 "sample-profile-check-record-coverage", cl::init(0), cl::value_desc("N"),
131 cl::desc("Emit a warning if less than N% of records in the input profile "
132 "are matched to the IR."));
133
134static cl::opt<unsigned> SampleProfileSampleCoverage(
135 "sample-profile-check-sample-coverage", cl::init(0), cl::value_desc("N"),
136 cl::desc("Emit a warning if less than N% of samples in the input profile "
137 "are matched to the IR."));
138
139static cl::opt<bool> NoWarnSampleUnused(
140 "no-warn-sample-unused", cl::init(false), cl::Hidden,
141 cl::desc("Use this option to turn off/on warnings about function with "
142 "samples but without debug information to use those samples. "));
143
144static cl::opt<bool> ProfileSampleAccurate(
145 "profile-sample-accurate", cl::Hidden, cl::init(false),
146 cl::desc("If the sample profile is accurate, we will mark all un-sampled "
147 "callsite and function as having 0 samples. Otherwise, treat "
148 "un-sampled callsites and functions conservatively as unknown. "));
149
150static cl::opt<bool> ProfileAccurateForSymsInList(
151 "profile-accurate-for-symsinlist", cl::Hidden, cl::ZeroOrMore,
152 cl::init(true),
153 cl::desc("For symbols in profile symbol list, regard their profiles to "
154 "be accurate. It may be overriden by profile-sample-accurate. "));
155
156static cl::opt<bool> ProfileMergeInlinee(
157 "sample-profile-merge-inlinee", cl::Hidden, cl::init(true),
158 cl::desc("Merge past inlinee's profile to outline version if sample "
159 "profile loader decided not to inline a call site. It will "
160 "only be enabled when top-down order of profile loading is "
161 "enabled. "));
162
163static cl::opt<bool> ProfileTopDownLoad(
164 "sample-profile-top-down-load", cl::Hidden, cl::init(true),
165 cl::desc("Do profile annotation and inlining for functions in top-down "
166 "order of call graph during sample profile loading. It only "
167 "works for new pass manager. "));
168
169static cl::opt<bool> ProfileSizeInline(
170 "sample-profile-inline-size", cl::Hidden, cl::init(false),
171 cl::desc("Inline cold call sites in profile loader if it's beneficial "
172 "for code size."));
173
174static cl::opt<int> SampleColdCallSiteThreshold(
175 "sample-profile-cold-inline-threshold", cl::Hidden, cl::init(45),
176 cl::desc("Threshold for inlining cold callsites"));
177
178static cl::opt<std::string> ProfileInlineReplayFile(
179 "sample-profile-inline-replay", cl::init(""), cl::value_desc("filename"),
180 cl::desc(
181 "Optimization remarks file containing inline remarks to be replayed "
182 "by inlining from sample profile loader."),
183 cl::Hidden);
184
185namespace {
186
187using BlockWeightMap = DenseMap<const BasicBlock *, uint64_t>;
188using EquivalenceClassMap = DenseMap<const BasicBlock *, const BasicBlock *>;
189using Edge = std::pair<const BasicBlock *, const BasicBlock *>;
190using EdgeWeightMap = DenseMap<Edge, uint64_t>;
191using BlockEdgeMap =
192 DenseMap<const BasicBlock *, SmallVector<const BasicBlock *, 8>>;
193
194class SampleProfileLoader;
195
196class SampleCoverageTracker {
197public:
198 SampleCoverageTracker(SampleProfileLoader &SPL) : SPLoader(SPL){};
199
200 bool markSamplesUsed(const FunctionSamples *FS, uint32_t LineOffset,
201 uint32_t Discriminator, uint64_t Samples);
202 unsigned computeCoverage(unsigned Used, unsigned Total) const;
203 unsigned countUsedRecords(const FunctionSamples *FS,
204 ProfileSummaryInfo *PSI) const;
205 unsigned countBodyRecords(const FunctionSamples *FS,
206 ProfileSummaryInfo *PSI) const;
207 uint64_t getTotalUsedSamples() const { return TotalUsedSamples; }
208 uint64_t countBodySamples(const FunctionSamples *FS,
209 ProfileSummaryInfo *PSI) const;
210
211 void clear() {
212 SampleCoverage.clear();
213 TotalUsedSamples = 0;
214 }
215
216private:
217 using BodySampleCoverageMap = std::map<LineLocation, unsigned>;
218 using FunctionSamplesCoverageMap =
219 DenseMap<const FunctionSamples *, BodySampleCoverageMap>;
220
221 /// Coverage map for sampling records.
222 ///
223 /// This map keeps a record of sampling records that have been matched to
224 /// an IR instruction. This is used to detect some form of staleness in
225 /// profiles (see flag -sample-profile-check-coverage).
226 ///
227 /// Each entry in the map corresponds to a FunctionSamples instance. This is
228 /// another map that counts how many times the sample record at the
229 /// given location has been used.
230 FunctionSamplesCoverageMap SampleCoverage;
231
232 /// Number of samples used from the profile.
233 ///
234 /// When a sampling record is used for the first time, the samples from
235 /// that record are added to this accumulator. Coverage is later computed
236 /// based on the total number of samples available in this function and
237 /// its callsites.
238 ///
239 /// Note that this accumulator tracks samples used from a single function
240 /// and all the inlined callsites. Strictly, we should have a map of counters
241 /// keyed by FunctionSamples pointers, but these stats are cleared after
242 /// every function, so we just need to keep a single counter.
243 uint64_t TotalUsedSamples = 0;
244
245 SampleProfileLoader &SPLoader;
246};
247
248class GUIDToFuncNameMapper {
249public:
250 GUIDToFuncNameMapper(Module &M, SampleProfileReader &Reader,
251 DenseMap<uint64_t, StringRef> &GUIDToFuncNameMap)
252 : CurrentReader(Reader), CurrentModule(M),
253 CurrentGUIDToFuncNameMap(GUIDToFuncNameMap) {
254 if (!CurrentReader.useMD5())
255 return;
256
257 for (const auto &F : CurrentModule) {
258 StringRef OrigName = F.getName();
259 CurrentGUIDToFuncNameMap.insert(
260 {Function::getGUID(OrigName), OrigName});
261
262 // Local to global var promotion used by optimization like thinlto
263 // will rename the var and add suffix like ".llvm.xxx" to the
264 // original local name. In sample profile, the suffixes of function
265 // names are all stripped. Since it is possible that the mapper is
266 // built in post-thin-link phase and var promotion has been done,
267 // we need to add the substring of function name without the suffix
268 // into the GUIDToFuncNameMap.
269 StringRef CanonName = FunctionSamples::getCanonicalFnName(F);
270 if (CanonName != OrigName)
271 CurrentGUIDToFuncNameMap.insert(
272 {Function::getGUID(CanonName), CanonName});
273 }
274
275 // Update GUIDToFuncNameMap for each function including inlinees.
276 SetGUIDToFuncNameMapForAll(&CurrentGUIDToFuncNameMap);
277 }
278
279 ~GUIDToFuncNameMapper() {
280 if (!CurrentReader.useMD5())
281 return;
282
283 CurrentGUIDToFuncNameMap.clear();
284
285 // Reset GUIDToFuncNameMap for of each function as they're no
286 // longer valid at this point.
287 SetGUIDToFuncNameMapForAll(nullptr);
288 }
289
290private:
291 void SetGUIDToFuncNameMapForAll(DenseMap<uint64_t, StringRef> *Map) {
292 std::queue<FunctionSamples *> FSToUpdate;
293 for (auto &IFS : CurrentReader.getProfiles()) {
294 FSToUpdate.push(&IFS.second);
295 }
296
297 while (!FSToUpdate.empty()) {
298 FunctionSamples *FS = FSToUpdate.front();
299 FSToUpdate.pop();
300 FS->GUIDToFuncNameMap = Map;
301 for (const auto &ICS : FS->getCallsiteSamples()) {
302 const FunctionSamplesMap &FSMap = ICS.second;
303 for (auto &IFS : FSMap) {
304 FunctionSamples &FS = const_cast<FunctionSamples &>(IFS.second);
305 FSToUpdate.push(&FS);
306 }
307 }
308 }
309 }
310
311 SampleProfileReader &CurrentReader;
312 Module &CurrentModule;
313 DenseMap<uint64_t, StringRef> &CurrentGUIDToFuncNameMap;
314};
315
316/// Sample profile pass.
317///
318/// This pass reads profile data from the file specified by
319/// -sample-profile-file and annotates every affected function with the
320/// profile information found in that file.
321class SampleProfileLoader {
322public:
323 SampleProfileLoader(
324 StringRef Name, StringRef RemapName, ThinOrFullLTOPhase LTOPhase,
325 std::function<AssumptionCache &(Function &)> GetAssumptionCache,
326 std::function<TargetTransformInfo &(Function &)> GetTargetTransformInfo,
327 std::function<const TargetLibraryInfo &(Function &)> GetTLI)
328 : GetAC(std::move(GetAssumptionCache)),
329 GetTTI(std::move(GetTargetTransformInfo)), GetTLI(std::move(GetTLI)),
330 CoverageTracker(*this), Filename(std::string(Name)),
331 RemappingFilename(std::string(RemapName)), LTOPhase(LTOPhase) {}
332
333 bool doInitialization(Module &M, FunctionAnalysisManager *FAM = nullptr);
334 bool runOnModule(Module &M, ModuleAnalysisManager *AM,
335 ProfileSummaryInfo *_PSI, CallGraph *CG);
336
337 void dump() { Reader->dump(); }
338
339protected:
340 friend class SampleCoverageTracker;
341
342 bool runOnFunction(Function &F, ModuleAnalysisManager *AM);
343 unsigned getFunctionLoc(Function &F);
344 bool emitAnnotations(Function &F);
345 ErrorOr<uint64_t> getInstWeight(const Instruction &I);
346 ErrorOr<uint64_t> getProbeWeight(const Instruction &I);
347 ErrorOr<uint64_t> getBlockWeight(const BasicBlock *BB);
348 const FunctionSamples *findCalleeFunctionSamples(const CallBase &I) const;
349 std::vector<const FunctionSamples *>
350 findIndirectCallFunctionSamples(const Instruction &I, uint64_t &Sum) const;
351 mutable DenseMap<const DILocation *, const FunctionSamples *> DILocation2SampleMap;
352 const FunctionSamples *findFunctionSamples(const Instruction &I) const;
353 bool inlineCallInstruction(CallBase &CB);
354 bool inlineHotFunctions(Function &F,
355 DenseSet<GlobalValue::GUID> &InlinedGUIDs);
356 // Inline cold/small functions in addition to hot ones
357 bool shouldInlineColdCallee(CallBase &CallInst);
358 void emitOptimizationRemarksForInlineCandidates(
359 const SmallVectorImpl<CallBase *> &Candidates, const Function &F,
360 bool Hot);
361 void printEdgeWeight(raw_ostream &OS, Edge E);
362 void printBlockWeight(raw_ostream &OS, const BasicBlock *BB) const;
363 void printBlockEquivalence(raw_ostream &OS, const BasicBlock *BB);
364 bool computeBlockWeights(Function &F);
365 void findEquivalenceClasses(Function &F);
366 template <bool IsPostDom>
367 void findEquivalencesFor(BasicBlock *BB1, ArrayRef<BasicBlock *> Descendants,
368 DominatorTreeBase<BasicBlock, IsPostDom> *DomTree);
369
370 void propagateWeights(Function &F);
371 uint64_t visitEdge(Edge E, unsigned *NumUnknownEdges, Edge *UnknownEdge);
372 void buildEdges(Function &F);
373 std::vector<Function *> buildFunctionOrder(Module &M, CallGraph *CG);
374 bool propagateThroughEdges(Function &F, bool UpdateBlockCount);
375 void computeDominanceAndLoopInfo(Function &F);
376 void clearFunctionData();
377 bool callsiteIsHot(const FunctionSamples *CallsiteFS,
378 ProfileSummaryInfo *PSI);
379
380 /// Map basic blocks to their computed weights.
381 ///
382 /// The weight of a basic block is defined to be the maximum
383 /// of all the instruction weights in that block.
384 BlockWeightMap BlockWeights;
385
386 /// Map edges to their computed weights.
387 ///
388 /// Edge weights are computed by propagating basic block weights in
389 /// SampleProfile::propagateWeights.
390 EdgeWeightMap EdgeWeights;
391
392 /// Set of visited blocks during propagation.
393 SmallPtrSet<const BasicBlock *, 32> VisitedBlocks;
394
395 /// Set of visited edges during propagation.
396 SmallSet<Edge, 32> VisitedEdges;
397
398 /// Equivalence classes for block weights.
399 ///
400 /// Two blocks BB1 and BB2 are in the same equivalence class if they
401 /// dominate and post-dominate each other, and they are in the same loop
402 /// nest. When this happens, the two blocks are guaranteed to execute
403 /// the same number of times.
404 EquivalenceClassMap EquivalenceClass;
405
406 /// Map from function name to Function *. Used to find the function from
407 /// the function name. If the function name contains suffix, additional
408 /// entry is added to map from the stripped name to the function if there
409 /// is one-to-one mapping.
410 StringMap<Function *> SymbolMap;
411
412 /// Dominance, post-dominance and loop information.
413 std::unique_ptr<DominatorTree> DT;
414 std::unique_ptr<PostDominatorTree> PDT;
415 std::unique_ptr<LoopInfo> LI;
416
417 std::function<AssumptionCache &(Function &)> GetAC;
418 std::function<TargetTransformInfo &(Function &)> GetTTI;
419 std::function<const TargetLibraryInfo &(Function &)> GetTLI;
420
421 /// Predecessors for each basic block in the CFG.
422 BlockEdgeMap Predecessors;
423
424 /// Successors for each basic block in the CFG.
425 BlockEdgeMap Successors;
426
427 SampleCoverageTracker CoverageTracker;
428
429 /// Profile reader object.
430 std::unique_ptr<SampleProfileReader> Reader;
431
432 /// Profile tracker for different context.
433 std::unique_ptr<SampleContextTracker> ContextTracker;
434
435 /// Samples collected for the body of this function.
436 FunctionSamples *Samples = nullptr;
437
438 /// Name of the profile file to load.
439 std::string Filename;
440
441 /// Name of the profile remapping file to load.
442 std::string RemappingFilename;
443
444 /// Flag indicating whether the profile input loaded successfully.
445 bool ProfileIsValid = false;
446
447 /// Flag indicating whether input profile is context-sensitive
448 bool ProfileIsCS = false;
449
450 /// Flag indicating which LTO/ThinLTO phase the pass is invoked in.
451 ///
452 /// We need to know the LTO phase because for example in ThinLTOPrelink
453 /// phase, in annotation, we should not promote indirect calls. Instead,
454 /// we will mark GUIDs that needs to be annotated to the function.
455 ThinOrFullLTOPhase LTOPhase;
456
457 /// Profile Summary Info computed from sample profile.
458 ProfileSummaryInfo *PSI = nullptr;
459
460 /// Profle Symbol list tells whether a function name appears in the binary
461 /// used to generate the current profile.
462 std::unique_ptr<ProfileSymbolList> PSL;
463
464 /// Total number of samples collected in this profile.
465 ///
466 /// This is the sum of all the samples collected in all the functions executed
467 /// at runtime.
468 uint64_t TotalCollectedSamples = 0;
469
470 /// Optimization Remark Emitter used to emit diagnostic remarks.
471 OptimizationRemarkEmitter *ORE = nullptr;
472
473 // Information recorded when we declined to inline a call site
474 // because we have determined it is too cold is accumulated for
475 // each callee function. Initially this is just the entry count.
476 struct NotInlinedProfileInfo {
477 uint64_t entryCount;
478 };
479 DenseMap<Function *, NotInlinedProfileInfo> notInlinedCallInfo;
480
481 // GUIDToFuncNameMap saves the mapping from GUID to the symbol name, for
482 // all the function symbols defined or declared in current module.
483 DenseMap<uint64_t, StringRef> GUIDToFuncNameMap;
484
485 // All the Names used in FunctionSamples including outline function
486 // names, inline instance names and call target names.
487 StringSet<> NamesInProfile;
488
489 // For symbol in profile symbol list, whether to regard their profiles
490 // to be accurate. It is mainly decided by existance of profile symbol
491 // list and -profile-accurate-for-symsinlist flag, but it can be
492 // overriden by -profile-sample-accurate or profile-sample-accurate
493 // attribute.
494 bool ProfAccForSymsInList;
495
496 // External inline advisor used to replay inline decision from remarks.
497 std::unique_ptr<ReplayInlineAdvisor> ExternalInlineAdvisor;
498
499 // A pseudo probe helper to correlate the imported sample counts.
500 std::unique_ptr<PseudoProbeManager> ProbeManager;
501};
502
503class SampleProfileLoaderLegacyPass : public ModulePass {
504public:
505 // Class identification, replacement for typeinfo
506 static char ID;
507
508 SampleProfileLoaderLegacyPass(
509 StringRef Name = SampleProfileFile,
510 ThinOrFullLTOPhase LTOPhase = ThinOrFullLTOPhase::None)
511 : ModulePass(ID), SampleLoader(
512 Name, SampleProfileRemappingFile, LTOPhase,
513 [&](Function &F) -> AssumptionCache & {
514 return ACT->getAssumptionCache(F);
515 },
516 [&](Function &F) -> TargetTransformInfo & {
517 return TTIWP->getTTI(F);
518 },
519 [&](Function &F) -> TargetLibraryInfo & {
520 return TLIWP->getTLI(F);
521 }) {
522 initializeSampleProfileLoaderLegacyPassPass(
523 *PassRegistry::getPassRegistry());
524 }
525
526 void dump() { SampleLoader.dump(); }
527
528 bool doInitialization(Module &M) override {
529 return SampleLoader.doInitialization(M);
530 }
531
532 StringRef getPassName() const override { return "Sample profile pass"; }
533 bool runOnModule(Module &M) override;
534
535 void getAnalysisUsage(AnalysisUsage &AU) const override {
536 AU.addRequired<AssumptionCacheTracker>();
537 AU.addRequired<TargetTransformInfoWrapperPass>();
538 AU.addRequired<TargetLibraryInfoWrapperPass>();
539 AU.addRequired<ProfileSummaryInfoWrapperPass>();
540 }
541
542private:
543 SampleProfileLoader SampleLoader;
544 AssumptionCacheTracker *ACT = nullptr;
545 TargetTransformInfoWrapperPass *TTIWP = nullptr;
546 TargetLibraryInfoWrapperPass *TLIWP = nullptr;
547};
548
549} // end anonymous namespace
550
551/// Return true if the given callsite is hot wrt to hot cutoff threshold.
552///
553/// Functions that were inlined in the original binary will be represented
554/// in the inline stack in the sample profile. If the profile shows that
555/// the original inline decision was "good" (i.e., the callsite is executed
556/// frequently), then we will recreate the inline decision and apply the
557/// profile from the inlined callsite.
558///
559/// To decide whether an inlined callsite is hot, we compare the callsite
560/// sample count with the hot cutoff computed by ProfileSummaryInfo, it is
561/// regarded as hot if the count is above the cutoff value.
562///
563/// When ProfileAccurateForSymsInList is enabled and profile symbol list
564/// is present, functions in the profile symbol list but without profile will
565/// be regarded as cold and much less inlining will happen in CGSCC inlining
566/// pass, so we tend to lower the hot criteria here to allow more early
567/// inlining to happen for warm callsites and it is helpful for performance.
568bool SampleProfileLoader::callsiteIsHot(const FunctionSamples *CallsiteFS,
569 ProfileSummaryInfo *PSI) {
570 if (!CallsiteFS)
571 return false; // The callsite was not inlined in the original binary.
572
573 assert(PSI && "PSI is expected to be non null")((PSI && "PSI is expected to be non null") ? static_cast
<void> (0) : __assert_fail ("PSI && \"PSI is expected to be non null\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 573, __PRETTY_FUNCTION__))
;
574 uint64_t CallsiteTotalSamples = CallsiteFS->getTotalSamples();
575 if (ProfAccForSymsInList)
576 return !PSI->isColdCount(CallsiteTotalSamples);
577 else
578 return PSI->isHotCount(CallsiteTotalSamples);
579}
580
581/// Mark as used the sample record for the given function samples at
582/// (LineOffset, Discriminator).
583///
584/// \returns true if this is the first time we mark the given record.
585bool SampleCoverageTracker::markSamplesUsed(const FunctionSamples *FS,
586 uint32_t LineOffset,
587 uint32_t Discriminator,
588 uint64_t Samples) {
589 LineLocation Loc(LineOffset, Discriminator);
590 unsigned &Count = SampleCoverage[FS][Loc];
591 bool FirstTime = (++Count == 1);
592 if (FirstTime)
593 TotalUsedSamples += Samples;
594 return FirstTime;
595}
596
597/// Return the number of sample records that were applied from this profile.
598///
599/// This count does not include records from cold inlined callsites.
600unsigned
601SampleCoverageTracker::countUsedRecords(const FunctionSamples *FS,
602 ProfileSummaryInfo *PSI) const {
603 auto I = SampleCoverage.find(FS);
604
605 // The size of the coverage map for FS represents the number of records
606 // that were marked used at least once.
607 unsigned Count = (I != SampleCoverage.end()) ? I->second.size() : 0;
608
609 // If there are inlined callsites in this function, count the samples found
610 // in the respective bodies. However, do not bother counting callees with 0
611 // total samples, these are callees that were never invoked at runtime.
612 for (const auto &I : FS->getCallsiteSamples())
613 for (const auto &J : I.second) {
614 const FunctionSamples *CalleeSamples = &J.second;
615 if (SPLoader.callsiteIsHot(CalleeSamples, PSI))
616 Count += countUsedRecords(CalleeSamples, PSI);
617 }
618
619 return Count;
620}
621
622/// Return the number of sample records in the body of this profile.
623///
624/// This count does not include records from cold inlined callsites.
625unsigned
626SampleCoverageTracker::countBodyRecords(const FunctionSamples *FS,
627 ProfileSummaryInfo *PSI) const {
628 unsigned Count = FS->getBodySamples().size();
629
630 // Only count records in hot callsites.
631 for (const auto &I : FS->getCallsiteSamples())
632 for (const auto &J : I.second) {
633 const FunctionSamples *CalleeSamples = &J.second;
634 if (SPLoader.callsiteIsHot(CalleeSamples, PSI))
635 Count += countBodyRecords(CalleeSamples, PSI);
636 }
637
638 return Count;
639}
640
641/// Return the number of samples collected in the body of this profile.
642///
643/// This count does not include samples from cold inlined callsites.
644uint64_t
645SampleCoverageTracker::countBodySamples(const FunctionSamples *FS,
646 ProfileSummaryInfo *PSI) const {
647 uint64_t Total = 0;
648 for (const auto &I : FS->getBodySamples())
649 Total += I.second.getSamples();
650
651 // Only count samples in hot callsites.
652 for (const auto &I : FS->getCallsiteSamples())
653 for (const auto &J : I.second) {
654 const FunctionSamples *CalleeSamples = &J.second;
655 if (SPLoader.callsiteIsHot(CalleeSamples, PSI))
656 Total += countBodySamples(CalleeSamples, PSI);
657 }
658
659 return Total;
660}
661
662/// Return the fraction of sample records used in this profile.
663///
664/// The returned value is an unsigned integer in the range 0-100 indicating
665/// the percentage of sample records that were used while applying this
666/// profile to the associated function.
667unsigned SampleCoverageTracker::computeCoverage(unsigned Used,
668 unsigned Total) const {
669 assert(Used <= Total &&((Used <= Total && "number of used records cannot exceed the total number of records"
) ? static_cast<void> (0) : __assert_fail ("Used <= Total && \"number of used records cannot exceed the total number of records\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 670, __PRETTY_FUNCTION__))
670 "number of used records cannot exceed the total number of records")((Used <= Total && "number of used records cannot exceed the total number of records"
) ? static_cast<void> (0) : __assert_fail ("Used <= Total && \"number of used records cannot exceed the total number of records\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 670, __PRETTY_FUNCTION__))
;
671 return Total > 0 ? Used * 100 / Total : 100;
672}
673
674/// Clear all the per-function data used to load samples and propagate weights.
675void SampleProfileLoader::clearFunctionData() {
676 BlockWeights.clear();
677 EdgeWeights.clear();
678 VisitedBlocks.clear();
679 VisitedEdges.clear();
680 EquivalenceClass.clear();
681 DT = nullptr;
682 PDT = nullptr;
683 LI = nullptr;
684 Predecessors.clear();
685 Successors.clear();
686 CoverageTracker.clear();
687}
688
689#ifndef NDEBUG
690/// Print the weight of edge \p E on stream \p OS.
691///
692/// \param OS Stream to emit the output to.
693/// \param E Edge to print.
694void SampleProfileLoader::printEdgeWeight(raw_ostream &OS, Edge E) {
695 OS << "weight[" << E.first->getName() << "->" << E.second->getName()
696 << "]: " << EdgeWeights[E] << "\n";
697}
698
699/// Print the equivalence class of block \p BB on stream \p OS.
700///
701/// \param OS Stream to emit the output to.
702/// \param BB Block to print.
703void SampleProfileLoader::printBlockEquivalence(raw_ostream &OS,
704 const BasicBlock *BB) {
705 const BasicBlock *Equiv = EquivalenceClass[BB];
706 OS << "equivalence[" << BB->getName()
707 << "]: " << ((Equiv) ? EquivalenceClass[BB]->getName() : "NONE") << "\n";
708}
709
710/// Print the weight of block \p BB on stream \p OS.
711///
712/// \param OS Stream to emit the output to.
713/// \param BB Block to print.
714void SampleProfileLoader::printBlockWeight(raw_ostream &OS,
715 const BasicBlock *BB) const {
716 const auto &I = BlockWeights.find(BB);
717 uint64_t W = (I == BlockWeights.end() ? 0 : I->second);
718 OS << "weight[" << BB->getName() << "]: " << W << "\n";
719}
720#endif
721
722/// Get the weight for an instruction.
723///
724/// The "weight" of an instruction \p Inst is the number of samples
725/// collected on that instruction at runtime. To retrieve it, we
726/// need to compute the line number of \p Inst relative to the start of its
727/// function. We use HeaderLineno to compute the offset. We then
728/// look up the samples collected for \p Inst using BodySamples.
729///
730/// \param Inst Instruction to query.
731///
732/// \returns the weight of \p Inst.
733ErrorOr<uint64_t> SampleProfileLoader::getInstWeight(const Instruction &Inst) {
734 if (FunctionSamples::ProfileIsProbeBased)
735 return getProbeWeight(Inst);
736
737 const DebugLoc &DLoc = Inst.getDebugLoc();
738 if (!DLoc)
739 return std::error_code();
740
741 const FunctionSamples *FS = findFunctionSamples(Inst);
742 if (!FS)
743 return std::error_code();
744
745 // Ignore all intrinsics, phinodes and branch instructions.
746 // Branch and phinodes instruction usually contains debug info from sources outside of
747 // the residing basic block, thus we ignore them during annotation.
748 if (isa<BranchInst>(Inst) || isa<IntrinsicInst>(Inst) || isa<PHINode>(Inst))
749 return std::error_code();
750
751 // If a direct call/invoke instruction is inlined in profile
752 // (findCalleeFunctionSamples returns non-empty result), but not inlined here,
753 // it means that the inlined callsite has no sample, thus the call
754 // instruction should have 0 count.
755 if (!ProfileIsCS)
756 if (const auto *CB = dyn_cast<CallBase>(&Inst))
757 if (!CB->isIndirectCall() && findCalleeFunctionSamples(*CB))
758 return 0;
759
760 const DILocation *DIL = DLoc;
761 uint32_t LineOffset = FunctionSamples::getOffset(DIL);
762 uint32_t Discriminator = DIL->getBaseDiscriminator();
763 ErrorOr<uint64_t> R = FS->findSamplesAt(LineOffset, Discriminator);
764 if (R) {
765 bool FirstMark =
766 CoverageTracker.markSamplesUsed(FS, LineOffset, Discriminator, R.get());
767 if (FirstMark) {
768 ORE->emit([&]() {
769 OptimizationRemarkAnalysis Remark(DEBUG_TYPE"sample-profile", "AppliedSamples", &Inst);
770 Remark << "Applied " << ore::NV("NumSamples", *R);
771 Remark << " samples from profile (offset: ";
772 Remark << ore::NV("LineOffset", LineOffset);
773 if (Discriminator) {
774 Remark << ".";
775 Remark << ore::NV("Discriminator", Discriminator);
776 }
777 Remark << ")";
778 return Remark;
779 });
780 }
781 LLVM_DEBUG(dbgs() << " " << DLoc.getLine() << "."do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << " " << DLoc.getLine
() << "." << DIL->getBaseDiscriminator() <<
":" << Inst << " (line offset: " << LineOffset
<< "." << DIL->getBaseDiscriminator() <<
" - weight: " << R.get() << ")\n"; } } while (false
)
782 << DIL->getBaseDiscriminator() << ":" << Instdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << " " << DLoc.getLine
() << "." << DIL->getBaseDiscriminator() <<
":" << Inst << " (line offset: " << LineOffset
<< "." << DIL->getBaseDiscriminator() <<
" - weight: " << R.get() << ")\n"; } } while (false
)
783 << " (line offset: " << LineOffset << "."do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << " " << DLoc.getLine
() << "." << DIL->getBaseDiscriminator() <<
":" << Inst << " (line offset: " << LineOffset
<< "." << DIL->getBaseDiscriminator() <<
" - weight: " << R.get() << ")\n"; } } while (false
)
784 << DIL->getBaseDiscriminator() << " - weight: " << R.get()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << " " << DLoc.getLine
() << "." << DIL->getBaseDiscriminator() <<
":" << Inst << " (line offset: " << LineOffset
<< "." << DIL->getBaseDiscriminator() <<
" - weight: " << R.get() << ")\n"; } } while (false
)
785 << ")\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << " " << DLoc.getLine
() << "." << DIL->getBaseDiscriminator() <<
":" << Inst << " (line offset: " << LineOffset
<< "." << DIL->getBaseDiscriminator() <<
" - weight: " << R.get() << ")\n"; } } while (false
)
;
786 }
787 return R;
788}
789
790ErrorOr<uint64_t> SampleProfileLoader::getProbeWeight(const Instruction &Inst) {
791 assert(FunctionSamples::ProfileIsProbeBased &&((FunctionSamples::ProfileIsProbeBased && "Profile is not pseudo probe based"
) ? static_cast<void> (0) : __assert_fail ("FunctionSamples::ProfileIsProbeBased && \"Profile is not pseudo probe based\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 792, __PRETTY_FUNCTION__))
792 "Profile is not pseudo probe based")((FunctionSamples::ProfileIsProbeBased && "Profile is not pseudo probe based"
) ? static_cast<void> (0) : __assert_fail ("FunctionSamples::ProfileIsProbeBased && \"Profile is not pseudo probe based\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 792, __PRETTY_FUNCTION__))
;
793 Optional<PseudoProbe> Probe = extractProbe(Inst);
794 if (!Probe)
795 return std::error_code();
796
797 const FunctionSamples *FS = findFunctionSamples(Inst);
798 if (!FS)
799 return std::error_code();
800
801 // If a direct call/invoke instruction is inlined in profile
802 // (findCalleeFunctionSamples returns non-empty result), but not inlined here,
803 // it means that the inlined callsite has no sample, thus the call
804 // instruction should have 0 count.
805 if (const auto *CB = dyn_cast<CallBase>(&Inst))
806 if (!CB->isIndirectCall() && findCalleeFunctionSamples(*CB))
807 return 0;
808
809 const ErrorOr<uint64_t> &R = FS->findSamplesAt(Probe->Id, 0);
810 if (R) {
811 uint64_t Samples = R.get();
812 bool FirstMark = CoverageTracker.markSamplesUsed(FS, Probe->Id, 0, Samples);
813 if (FirstMark) {
814 ORE->emit([&]() {
815 OptimizationRemarkAnalysis Remark(DEBUG_TYPE"sample-profile", "AppliedSamples", &Inst);
816 Remark << "Applied " << ore::NV("NumSamples", Samples);
817 Remark << " samples from profile (ProbeId=";
818 Remark << ore::NV("ProbeId", Probe->Id);
819 Remark << ")";
820 return Remark;
821 });
822 }
823
824 LLVM_DEBUG(dbgs() << " " << Probe->Id << ":" << Instdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << " " << Probe->
Id << ":" << Inst << " - weight: " <<
R.get() << ")\n"; } } while (false)
825 << " - weight: " << R.get() << ")\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << " " << Probe->
Id << ":" << Inst << " - weight: " <<
R.get() << ")\n"; } } while (false)
;
826 return Samples;
827 }
828 return R;
829}
830
831/// Compute the weight of a basic block.
832///
833/// The weight of basic block \p BB is the maximum weight of all the
834/// instructions in BB.
835///
836/// \param BB The basic block to query.
837///
838/// \returns the weight for \p BB.
839ErrorOr<uint64_t> SampleProfileLoader::getBlockWeight(const BasicBlock *BB) {
840 uint64_t Max = 0;
841 bool HasWeight = false;
842 for (auto &I : BB->getInstList()) {
843 const ErrorOr<uint64_t> &R = getInstWeight(I);
844 if (R) {
845 Max = std::max(Max, R.get());
846 HasWeight = true;
847 }
848 }
849 return HasWeight ? ErrorOr<uint64_t>(Max) : std::error_code();
850}
851
852/// Compute and store the weights of every basic block.
853///
854/// This populates the BlockWeights map by computing
855/// the weights of every basic block in the CFG.
856///
857/// \param F The function to query.
858bool SampleProfileLoader::computeBlockWeights(Function &F) {
859 bool Changed = false;
860 LLVM_DEBUG(dbgs() << "Block weights\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "Block weights\n"; } } while
(false)
;
861 for (const auto &BB : F) {
862 ErrorOr<uint64_t> Weight = getBlockWeight(&BB);
863 if (Weight) {
864 BlockWeights[&BB] = Weight.get();
865 VisitedBlocks.insert(&BB);
866 Changed = true;
867 }
868 LLVM_DEBUG(printBlockWeight(dbgs(), &BB))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { printBlockWeight(dbgs(), &BB); } } while
(false)
;
869 }
870
871 return Changed;
872}
873
874/// Get the FunctionSamples for a call instruction.
875///
876/// The FunctionSamples of a call/invoke instruction \p Inst is the inlined
877/// instance in which that call instruction is calling to. It contains
878/// all samples that resides in the inlined instance. We first find the
879/// inlined instance in which the call instruction is from, then we
880/// traverse its children to find the callsite with the matching
881/// location.
882///
883/// \param Inst Call/Invoke instruction to query.
884///
885/// \returns The FunctionSamples pointer to the inlined instance.
886const FunctionSamples *
887SampleProfileLoader::findCalleeFunctionSamples(const CallBase &Inst) const {
888 const DILocation *DIL = Inst.getDebugLoc();
889 if (!DIL) {
890 return nullptr;
891 }
892
893 StringRef CalleeName;
894 if (Function *Callee = Inst.getCalledFunction())
895 CalleeName = FunctionSamples::getCanonicalFnName(*Callee);
896
897 if (ProfileIsCS)
898 return ContextTracker->getCalleeContextSamplesFor(Inst, CalleeName);
899
900 const FunctionSamples *FS = findFunctionSamples(Inst);
901 if (FS == nullptr)
902 return nullptr;
903
904 return FS->findFunctionSamplesAt(FunctionSamples::getCallSiteIdentifier(DIL),
905 CalleeName, Reader->getRemapper());
906}
907
908/// Returns a vector of FunctionSamples that are the indirect call targets
909/// of \p Inst. The vector is sorted by the total number of samples. Stores
910/// the total call count of the indirect call in \p Sum.
911std::vector<const FunctionSamples *>
912SampleProfileLoader::findIndirectCallFunctionSamples(
913 const Instruction &Inst, uint64_t &Sum) const {
914 const DILocation *DIL = Inst.getDebugLoc();
915 std::vector<const FunctionSamples *> R;
916
917 if (!DIL) {
69
Assuming 'DIL' is null
70
Taking true branch
918 return R;
71
Returning without writing to 'Sum'
919 }
920
921 const FunctionSamples *FS = findFunctionSamples(Inst);
922 if (FS == nullptr)
923 return R;
924
925 auto CallSite = FunctionSamples::getCallSiteIdentifier(DIL);
926 auto T = FS->findCallTargetMapAt(CallSite);
927 Sum = 0;
928 if (T)
929 for (const auto &T_C : T.get())
930 Sum += T_C.second;
931 if (const FunctionSamplesMap *M = FS->findFunctionSamplesMapAt(CallSite)) {
932 if (M->empty())
933 return R;
934 for (const auto &NameFS : *M) {
935 Sum += NameFS.second.getEntrySamples();
936 R.push_back(&NameFS.second);
937 }
938 llvm::sort(R, [](const FunctionSamples *L, const FunctionSamples *R) {
939 if (L->getEntrySamples() != R->getEntrySamples())
940 return L->getEntrySamples() > R->getEntrySamples();
941 return FunctionSamples::getGUID(L->getName()) <
942 FunctionSamples::getGUID(R->getName());
943 });
944 }
945 return R;
946}
947
948/// Get the FunctionSamples for an instruction.
949///
950/// The FunctionSamples of an instruction \p Inst is the inlined instance
951/// in which that instruction is coming from. We traverse the inline stack
952/// of that instruction, and match it with the tree nodes in the profile.
953///
954/// \param Inst Instruction to query.
955///
956/// \returns the FunctionSamples pointer to the inlined instance.
957const FunctionSamples *
958SampleProfileLoader::findFunctionSamples(const Instruction &Inst) const {
959 if (FunctionSamples::ProfileIsProbeBased) {
48
Assuming 'ProfileIsProbeBased' is false
49
Taking false branch
960 Optional<PseudoProbe> Probe = extractProbe(Inst);
961 if (!Probe)
962 return nullptr;
963 }
964
965 const DILocation *DIL = Inst.getDebugLoc();
966 if (!DIL)
50
Assuming 'DIL' is non-null, which participates in a condition later
51
Taking false branch
967 return Samples;
968
969 auto it = DILocation2SampleMap.try_emplace(DIL,nullptr);
970 if (it.second) {
52
Assuming field 'second' is false
53
Taking false branch
971 if (ProfileIsCS)
972 it.first->second = ContextTracker->getContextSamplesFor(DIL);
973 else
974 it.first->second =
975 Samples->findFunctionSamples(DIL, Reader->getRemapper());
976 }
977 return it.first->second;
54
Returning pointer, which participates in a condition later
978}
979
980bool SampleProfileLoader::inlineCallInstruction(CallBase &CB) {
981 if (ExternalInlineAdvisor) {
982 auto Advice = ExternalInlineAdvisor->getAdvice(CB);
983 if (!Advice->isInliningRecommended()) {
984 Advice->recordUnattemptedInlining();
985 return false;
986 }
987 // Dummy record, we don't use it for replay.
988 Advice->recordInlining();
989 }
990
991 Function *CalledFunction = CB.getCalledFunction();
992 assert(CalledFunction)((CalledFunction) ? static_cast<void> (0) : __assert_fail
("CalledFunction", "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 992, __PRETTY_FUNCTION__))
;
993 DebugLoc DLoc = CB.getDebugLoc();
994 BasicBlock *BB = CB.getParent();
995 InlineParams Params = getInlineParams();
996 Params.ComputeFullInlineCost = true;
997 // Checks if there is anything in the reachable portion of the callee at
998 // this callsite that makes this inlining potentially illegal. Need to
999 // set ComputeFullInlineCost, otherwise getInlineCost may return early
1000 // when cost exceeds threshold without checking all IRs in the callee.
1001 // The acutal cost does not matter because we only checks isNever() to
1002 // see if it is legal to inline the callsite.
1003 InlineCost Cost =
1004 getInlineCost(CB, Params, GetTTI(*CalledFunction), GetAC, GetTLI);
1005 if (Cost.isNever()) {
1006 ORE->emit(OptimizationRemarkAnalysis(CSINLINE_DEBUG"sample-profile" "-inline", "InlineFail", DLoc, BB)
1007 << "incompatible inlining");
1008 return false;
1009 }
1010 InlineFunctionInfo IFI(nullptr, GetAC);
1011 if (InlineFunction(CB, IFI).isSuccess()) {
1012 // The call to InlineFunction erases I, so we can't pass it here.
1013 emitInlinedInto(*ORE, DLoc, BB, *CalledFunction, *BB->getParent(), Cost,
1014 true, CSINLINE_DEBUG"sample-profile" "-inline");
1015 return true;
1016 }
1017 return false;
1018}
1019
1020bool SampleProfileLoader::shouldInlineColdCallee(CallBase &CallInst) {
1021 if (!ProfileSizeInline)
1022 return false;
1023
1024 Function *Callee = CallInst.getCalledFunction();
1025 if (Callee == nullptr)
1026 return false;
1027
1028 InlineCost Cost = getInlineCost(CallInst, getInlineParams(), GetTTI(*Callee),
1029 GetAC, GetTLI);
1030
1031 if (Cost.isNever())
1032 return false;
1033
1034 if (Cost.isAlways())
1035 return true;
1036
1037 return Cost.getCost() <= SampleColdCallSiteThreshold;
1038}
1039
1040void SampleProfileLoader::emitOptimizationRemarksForInlineCandidates(
1041 const SmallVectorImpl<CallBase *> &Candidates, const Function &F,
1042 bool Hot) {
1043 for (auto I : Candidates) {
1044 Function *CalledFunction = I->getCalledFunction();
1045 if (CalledFunction) {
1046 ORE->emit(OptimizationRemarkAnalysis(CSINLINE_DEBUG"sample-profile" "-inline", "InlineAttempt",
1047 I->getDebugLoc(), I->getParent())
1048 << "previous inlining reattempted for "
1049 << (Hot ? "hotness: '" : "size: '")
1050 << ore::NV("Callee", CalledFunction) << "' into '"
1051 << ore::NV("Caller", &F) << "'");
1052 }
1053 }
1054}
1055
1056/// Iteratively inline hot callsites of a function.
1057///
1058/// Iteratively traverse all callsites of the function \p F, and find if
1059/// the corresponding inlined instance exists and is hot in profile. If
1060/// it is hot enough, inline the callsites and adds new callsites of the
1061/// callee into the caller. If the call is an indirect call, first promote
1062/// it to direct call. Each indirect call is limited with a single target.
1063///
1064/// \param F function to perform iterative inlining.
1065/// \param InlinedGUIDs a set to be updated to include all GUIDs that are
1066/// inlined in the profiled binary.
1067///
1068/// \returns True if there is any inline happened.
1069bool SampleProfileLoader::inlineHotFunctions(
1070 Function &F, DenseSet<GlobalValue::GUID> &InlinedGUIDs) {
1071 DenseSet<Instruction *> PromotedInsns;
1072
1073 // ProfAccForSymsInList is used in callsiteIsHot. The assertion makes sure
1074 // Profile symbol list is ignored when profile-sample-accurate is on.
1075 assert((!ProfAccForSymsInList ||(((!ProfAccForSymsInList || (!ProfileSampleAccurate &&
!F.hasFnAttribute("profile-sample-accurate"))) && "ProfAccForSymsInList should be false when profile-sample-accurate "
"is enabled") ? static_cast<void> (0) : __assert_fail (
"(!ProfAccForSymsInList || (!ProfileSampleAccurate && !F.hasFnAttribute(\"profile-sample-accurate\"))) && \"ProfAccForSymsInList should be false when profile-sample-accurate \" \"is enabled\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1079, __PRETTY_FUNCTION__))
1076 (!ProfileSampleAccurate &&(((!ProfAccForSymsInList || (!ProfileSampleAccurate &&
!F.hasFnAttribute("profile-sample-accurate"))) && "ProfAccForSymsInList should be false when profile-sample-accurate "
"is enabled") ? static_cast<void> (0) : __assert_fail (
"(!ProfAccForSymsInList || (!ProfileSampleAccurate && !F.hasFnAttribute(\"profile-sample-accurate\"))) && \"ProfAccForSymsInList should be false when profile-sample-accurate \" \"is enabled\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1079, __PRETTY_FUNCTION__))
1077 !F.hasFnAttribute("profile-sample-accurate"))) &&(((!ProfAccForSymsInList || (!ProfileSampleAccurate &&
!F.hasFnAttribute("profile-sample-accurate"))) && "ProfAccForSymsInList should be false when profile-sample-accurate "
"is enabled") ? static_cast<void> (0) : __assert_fail (
"(!ProfAccForSymsInList || (!ProfileSampleAccurate && !F.hasFnAttribute(\"profile-sample-accurate\"))) && \"ProfAccForSymsInList should be false when profile-sample-accurate \" \"is enabled\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1079, __PRETTY_FUNCTION__))
1078 "ProfAccForSymsInList should be false when profile-sample-accurate "(((!ProfAccForSymsInList || (!ProfileSampleAccurate &&
!F.hasFnAttribute("profile-sample-accurate"))) && "ProfAccForSymsInList should be false when profile-sample-accurate "
"is enabled") ? static_cast<void> (0) : __assert_fail (
"(!ProfAccForSymsInList || (!ProfileSampleAccurate && !F.hasFnAttribute(\"profile-sample-accurate\"))) && \"ProfAccForSymsInList should be false when profile-sample-accurate \" \"is enabled\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1079, __PRETTY_FUNCTION__))
1079 "is enabled")(((!ProfAccForSymsInList || (!ProfileSampleAccurate &&
!F.hasFnAttribute("profile-sample-accurate"))) && "ProfAccForSymsInList should be false when profile-sample-accurate "
"is enabled") ? static_cast<void> (0) : __assert_fail (
"(!ProfAccForSymsInList || (!ProfileSampleAccurate && !F.hasFnAttribute(\"profile-sample-accurate\"))) && \"ProfAccForSymsInList should be false when profile-sample-accurate \" \"is enabled\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1079, __PRETTY_FUNCTION__))
;
1080
1081 DenseMap<CallBase *, const FunctionSamples *> localNotInlinedCallSites;
1082 bool Changed = false;
1083 while (true) {
1084 bool LocalChanged = false;
1085 SmallVector<CallBase *, 10> CIS;
1086 for (auto &BB : F) {
1087 bool Hot = false;
1088 SmallVector<CallBase *, 10> AllCandidates;
1089 SmallVector<CallBase *, 10> ColdCandidates;
1090 for (auto &I : BB.getInstList()) {
1091 const FunctionSamples *FS = nullptr;
1092 if (auto *CB = dyn_cast<CallBase>(&I)) {
1093 if (!isa<IntrinsicInst>(I) && (FS = findCalleeFunctionSamples(*CB))) {
1094 assert((!FunctionSamples::UseMD5 || FS->GUIDToFuncNameMap) &&(((!FunctionSamples::UseMD5 || FS->GUIDToFuncNameMap) &&
"GUIDToFuncNameMap has to be populated") ? static_cast<void
> (0) : __assert_fail ("(!FunctionSamples::UseMD5 || FS->GUIDToFuncNameMap) && \"GUIDToFuncNameMap has to be populated\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1095, __PRETTY_FUNCTION__))
1095 "GUIDToFuncNameMap has to be populated")(((!FunctionSamples::UseMD5 || FS->GUIDToFuncNameMap) &&
"GUIDToFuncNameMap has to be populated") ? static_cast<void
> (0) : __assert_fail ("(!FunctionSamples::UseMD5 || FS->GUIDToFuncNameMap) && \"GUIDToFuncNameMap has to be populated\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1095, __PRETTY_FUNCTION__))
;
1096 AllCandidates.push_back(CB);
1097 if (FS->getEntrySamples() > 0 || ProfileIsCS)
1098 localNotInlinedCallSites.try_emplace(CB, FS);
1099 if (callsiteIsHot(FS, PSI))
1100 Hot = true;
1101 else if (shouldInlineColdCallee(*CB))
1102 ColdCandidates.push_back(CB);
1103 }
1104 }
1105 }
1106 if (Hot || ExternalInlineAdvisor) {
1107 CIS.insert(CIS.begin(), AllCandidates.begin(), AllCandidates.end());
1108 emitOptimizationRemarksForInlineCandidates(AllCandidates, F, true);
1109 } else {
1110 CIS.insert(CIS.begin(), ColdCandidates.begin(), ColdCandidates.end());
1111 emitOptimizationRemarksForInlineCandidates(ColdCandidates, F, false);
1112 }
1113 }
1114 for (CallBase *I : CIS) {
1115 Function *CalledFunction = I->getCalledFunction();
1116 // Do not inline recursive calls.
1117 if (CalledFunction == &F)
1118 continue;
1119 if (I->isIndirectCall()) {
1120 if (PromotedInsns.count(I))
1121 continue;
1122 uint64_t Sum;
1123 for (const auto *FS : findIndirectCallFunctionSamples(*I, Sum)) {
1124 if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) {
1125 FS->findInlinedFunctions(InlinedGUIDs, F.getParent(),
1126 PSI->getOrCompHotCountThreshold());
1127 continue;
1128 }
1129 if (!callsiteIsHot(FS, PSI))
1130 continue;
1131
1132 const char *Reason = "Callee function not available";
1133 // R->getValue() != &F is to prevent promoting a recursive call.
1134 // If it is a recursive call, we do not inline it as it could bloat
1135 // the code exponentially. There is way to better handle this, e.g.
1136 // clone the caller first, and inline the cloned caller if it is
1137 // recursive. As llvm does not inline recursive calls, we will
1138 // simply ignore it instead of handling it explicitly.
1139 auto CalleeFunctionName = FS->getFuncName();
1140 auto R = SymbolMap.find(CalleeFunctionName);
1141 if (R != SymbolMap.end() && R->getValue() &&
1142 !R->getValue()->isDeclaration() &&
1143 R->getValue()->getSubprogram() &&
1144 R->getValue()->hasFnAttribute("use-sample-profile") &&
1145 R->getValue() != &F &&
1146 isLegalToPromote(*I, R->getValue(), &Reason)) {
1147 uint64_t C = FS->getEntrySamples();
1148 auto &DI =
1149 pgo::promoteIndirectCall(*I, R->getValue(), C, Sum, false, ORE);
1150 Sum -= C;
1151 PromotedInsns.insert(I);
1152 // If profile mismatches, we should not attempt to inline DI.
1153 if ((isa<CallInst>(DI) || isa<InvokeInst>(DI)) &&
1154 inlineCallInstruction(cast<CallBase>(DI))) {
1155 if (ProfileIsCS)
1156 ContextTracker->markContextSamplesInlined(FS);
1157 localNotInlinedCallSites.erase(I);
1158 LocalChanged = true;
1159 ++NumCSInlined;
1160 }
1161 } else {
1162 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "\nFailed to promote indirect call to "
<< CalleeFunctionName << " because " << Reason
<< "\n"; } } while (false)
1163 << "\nFailed to promote indirect call to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "\nFailed to promote indirect call to "
<< CalleeFunctionName << " because " << Reason
<< "\n"; } } while (false)
1164 << CalleeFunctionName << " because " << Reason << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "\nFailed to promote indirect call to "
<< CalleeFunctionName << " because " << Reason
<< "\n"; } } while (false)
;
1165 }
1166 }
1167 } else if (CalledFunction && CalledFunction->getSubprogram() &&
1168 !CalledFunction->isDeclaration()) {
1169 if (inlineCallInstruction(*I)) {
1170 if (ProfileIsCS)
1171 ContextTracker->markContextSamplesInlined(
1172 localNotInlinedCallSites[I]);
1173 localNotInlinedCallSites.erase(I);
1174 LocalChanged = true;
1175 ++NumCSInlined;
1176 }
1177 } else if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) {
1178 findCalleeFunctionSamples(*I)->findInlinedFunctions(
1179 InlinedGUIDs, F.getParent(), PSI->getOrCompHotCountThreshold());
1180 }
1181 }
1182 if (LocalChanged) {
1183 Changed = true;
1184 } else {
1185 break;
1186 }
1187 }
1188
1189 // Accumulate not inlined callsite information into notInlinedSamples
1190 for (const auto &Pair : localNotInlinedCallSites) {
1191 CallBase *I = Pair.getFirst();
1192 Function *Callee = I->getCalledFunction();
1193 if (!Callee || Callee->isDeclaration())
1194 continue;
1195
1196 ORE->emit(OptimizationRemarkAnalysis(CSINLINE_DEBUG"sample-profile" "-inline", "NotInline",
1197 I->getDebugLoc(), I->getParent())
1198 << "previous inlining not repeated: '"
1199 << ore::NV("Callee", Callee) << "' into '"
1200 << ore::NV("Caller", &F) << "'");
1201
1202 ++NumCSNotInlined;
1203 const FunctionSamples *FS = Pair.getSecond();
1204 if (FS->getTotalSamples() == 0 && FS->getEntrySamples() == 0) {
1205 continue;
1206 }
1207
1208 if (ProfileMergeInlinee) {
1209 // A function call can be replicated by optimizations like callsite
1210 // splitting or jump threading and the replicates end up sharing the
1211 // sample nested callee profile instead of slicing the original inlinee's
1212 // profile. We want to do merge exactly once by filtering out callee
1213 // profiles with a non-zero head sample count.
1214 if (FS->getHeadSamples() == 0) {
1215 // Use entry samples as head samples during the merge, as inlinees
1216 // don't have head samples.
1217 const_cast<FunctionSamples *>(FS)->addHeadSamples(
1218 FS->getEntrySamples());
1219
1220 // Note that we have to do the merge right after processing function.
1221 // This allows OutlineFS's profile to be used for annotation during
1222 // top-down processing of functions' annotation.
1223 FunctionSamples *OutlineFS = Reader->getOrCreateSamplesFor(*Callee);
1224 OutlineFS->merge(*FS);
1225 }
1226 } else {
1227 auto pair =
1228 notInlinedCallInfo.try_emplace(Callee, NotInlinedProfileInfo{0});
1229 pair.first->second.entryCount += FS->getEntrySamples();
1230 }
1231 }
1232 return Changed;
1233}
1234
1235/// Find equivalence classes for the given block.
1236///
1237/// This finds all the blocks that are guaranteed to execute the same
1238/// number of times as \p BB1. To do this, it traverses all the
1239/// descendants of \p BB1 in the dominator or post-dominator tree.
1240///
1241/// A block BB2 will be in the same equivalence class as \p BB1 if
1242/// the following holds:
1243///
1244/// 1- \p BB1 is a descendant of BB2 in the opposite tree. So, if BB2
1245/// is a descendant of \p BB1 in the dominator tree, then BB2 should
1246/// dominate BB1 in the post-dominator tree.
1247///
1248/// 2- Both BB2 and \p BB1 must be in the same loop.
1249///
1250/// For every block BB2 that meets those two requirements, we set BB2's
1251/// equivalence class to \p BB1.
1252///
1253/// \param BB1 Block to check.
1254/// \param Descendants Descendants of \p BB1 in either the dom or pdom tree.
1255/// \param DomTree Opposite dominator tree. If \p Descendants is filled
1256/// with blocks from \p BB1's dominator tree, then
1257/// this is the post-dominator tree, and vice versa.
1258template <bool IsPostDom>
1259void SampleProfileLoader::findEquivalencesFor(
1260 BasicBlock *BB1, ArrayRef<BasicBlock *> Descendants,
1261 DominatorTreeBase<BasicBlock, IsPostDom> *DomTree) {
1262 const BasicBlock *EC = EquivalenceClass[BB1];
1263 uint64_t Weight = BlockWeights[EC];
1264 for (const auto *BB2 : Descendants) {
1265 bool IsDomParent = DomTree->dominates(BB2, BB1);
1266 bool IsInSameLoop = LI->getLoopFor(BB1) == LI->getLoopFor(BB2);
1267 if (BB1 != BB2 && IsDomParent && IsInSameLoop) {
1268 EquivalenceClass[BB2] = EC;
1269 // If BB2 is visited, then the entire EC should be marked as visited.
1270 if (VisitedBlocks.count(BB2)) {
1271 VisitedBlocks.insert(EC);
1272 }
1273
1274 // If BB2 is heavier than BB1, make BB2 have the same weight
1275 // as BB1.
1276 //
1277 // Note that we don't worry about the opposite situation here
1278 // (when BB2 is lighter than BB1). We will deal with this
1279 // during the propagation phase. Right now, we just want to
1280 // make sure that BB1 has the largest weight of all the
1281 // members of its equivalence set.
1282 Weight = std::max(Weight, BlockWeights[BB2]);
1283 }
1284 }
1285 if (EC == &EC->getParent()->getEntryBlock()) {
1286 BlockWeights[EC] = Samples->getHeadSamples() + 1;
1287 } else {
1288 BlockWeights[EC] = Weight;
1289 }
1290}
1291
1292/// Find equivalence classes.
1293///
1294/// Since samples may be missing from blocks, we can fill in the gaps by setting
1295/// the weights of all the blocks in the same equivalence class to the same
1296/// weight. To compute the concept of equivalence, we use dominance and loop
1297/// information. Two blocks B1 and B2 are in the same equivalence class if B1
1298/// dominates B2, B2 post-dominates B1 and both are in the same loop.
1299///
1300/// \param F The function to query.
1301void SampleProfileLoader::findEquivalenceClasses(Function &F) {
1302 SmallVector<BasicBlock *, 8> DominatedBBs;
1303 LLVM_DEBUG(dbgs() << "\nBlock equivalence classes\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "\nBlock equivalence classes\n"
; } } while (false)
;
1304 // Find equivalence sets based on dominance and post-dominance information.
1305 for (auto &BB : F) {
1306 BasicBlock *BB1 = &BB;
1307
1308 // Compute BB1's equivalence class once.
1309 if (EquivalenceClass.count(BB1)) {
1310 LLVM_DEBUG(printBlockEquivalence(dbgs(), BB1))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { printBlockEquivalence(dbgs(), BB1); } }
while (false)
;
1311 continue;
1312 }
1313
1314 // By default, blocks are in their own equivalence class.
1315 EquivalenceClass[BB1] = BB1;
1316
1317 // Traverse all the blocks dominated by BB1. We are looking for
1318 // every basic block BB2 such that:
1319 //
1320 // 1- BB1 dominates BB2.
1321 // 2- BB2 post-dominates BB1.
1322 // 3- BB1 and BB2 are in the same loop nest.
1323 //
1324 // If all those conditions hold, it means that BB2 is executed
1325 // as many times as BB1, so they are placed in the same equivalence
1326 // class by making BB2's equivalence class be BB1.
1327 DominatedBBs.clear();
1328 DT->getDescendants(BB1, DominatedBBs);
1329 findEquivalencesFor(BB1, DominatedBBs, PDT.get());
1330
1331 LLVM_DEBUG(printBlockEquivalence(dbgs(), BB1))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { printBlockEquivalence(dbgs(), BB1); } }
while (false)
;
1332 }
1333
1334 // Assign weights to equivalence classes.
1335 //
1336 // All the basic blocks in the same equivalence class will execute
1337 // the same number of times. Since we know that the head block in
1338 // each equivalence class has the largest weight, assign that weight
1339 // to all the blocks in that equivalence class.
1340 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "\nAssign the same weight to all blocks in the same class\n"
; } } while (false)
1341 dbgs() << "\nAssign the same weight to all blocks in the same class\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "\nAssign the same weight to all blocks in the same class\n"
; } } while (false)
;
1342 for (auto &BI : F) {
1343 const BasicBlock *BB = &BI;
1344 const BasicBlock *EquivBB = EquivalenceClass[BB];
1345 if (BB != EquivBB)
1346 BlockWeights[BB] = BlockWeights[EquivBB];
1347 LLVM_DEBUG(printBlockWeight(dbgs(), BB))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { printBlockWeight(dbgs(), BB); } } while
(false)
;
1348 }
1349}
1350
1351/// Visit the given edge to decide if it has a valid weight.
1352///
1353/// If \p E has not been visited before, we copy to \p UnknownEdge
1354/// and increment the count of unknown edges.
1355///
1356/// \param E Edge to visit.
1357/// \param NumUnknownEdges Current number of unknown edges.
1358/// \param UnknownEdge Set if E has not been visited before.
1359///
1360/// \returns E's weight, if known. Otherwise, return 0.
1361uint64_t SampleProfileLoader::visitEdge(Edge E, unsigned *NumUnknownEdges,
1362 Edge *UnknownEdge) {
1363 if (!VisitedEdges.count(E)) {
1364 (*NumUnknownEdges)++;
1365 *UnknownEdge = E;
1366 return 0;
1367 }
1368
1369 return EdgeWeights[E];
1370}
1371
1372/// Propagate weights through incoming/outgoing edges.
1373///
1374/// If the weight of a basic block is known, and there is only one edge
1375/// with an unknown weight, we can calculate the weight of that edge.
1376///
1377/// Similarly, if all the edges have a known count, we can calculate the
1378/// count of the basic block, if needed.
1379///
1380/// \param F Function to process.
1381/// \param UpdateBlockCount Whether we should update basic block counts that
1382/// has already been annotated.
1383///
1384/// \returns True if new weights were assigned to edges or blocks.
1385bool SampleProfileLoader::propagateThroughEdges(Function &F,
1386 bool UpdateBlockCount) {
1387 bool Changed = false;
1388 LLVM_DEBUG(dbgs() << "\nPropagation through edges\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "\nPropagation through edges\n"
; } } while (false)
;
1389 for (const auto &BI : F) {
1390 const BasicBlock *BB = &BI;
1391 const BasicBlock *EC = EquivalenceClass[BB];
1392
1393 // Visit all the predecessor and successor edges to determine
1394 // which ones have a weight assigned already. Note that it doesn't
1395 // matter that we only keep track of a single unknown edge. The
1396 // only case we are interested in handling is when only a single
1397 // edge is unknown (see setEdgeOrBlockWeight).
1398 for (unsigned i = 0; i < 2; i++) {
1399 uint64_t TotalWeight = 0;
1400 unsigned NumUnknownEdges = 0, NumTotalEdges = 0;
1401 Edge UnknownEdge, SelfReferentialEdge, SingleEdge;
1402
1403 if (i == 0) {
1404 // First, visit all predecessor edges.
1405 NumTotalEdges = Predecessors[BB].size();
1406 for (auto *Pred : Predecessors[BB]) {
1407 Edge E = std::make_pair(Pred, BB);
1408 TotalWeight += visitEdge(E, &NumUnknownEdges, &UnknownEdge);
1409 if (E.first == E.second)
1410 SelfReferentialEdge = E;
1411 }
1412 if (NumTotalEdges == 1) {
1413 SingleEdge = std::make_pair(Predecessors[BB][0], BB);
1414 }
1415 } else {
1416 // On the second round, visit all successor edges.
1417 NumTotalEdges = Successors[BB].size();
1418 for (auto *Succ : Successors[BB]) {
1419 Edge E = std::make_pair(BB, Succ);
1420 TotalWeight += visitEdge(E, &NumUnknownEdges, &UnknownEdge);
1421 }
1422 if (NumTotalEdges == 1) {
1423 SingleEdge = std::make_pair(BB, Successors[BB][0]);
1424 }
1425 }
1426
1427 // After visiting all the edges, there are three cases that we
1428 // can handle immediately:
1429 //
1430 // - All the edge weights are known (i.e., NumUnknownEdges == 0).
1431 // In this case, we simply check that the sum of all the edges
1432 // is the same as BB's weight. If not, we change BB's weight
1433 // to match. Additionally, if BB had not been visited before,
1434 // we mark it visited.
1435 //
1436 // - Only one edge is unknown and BB has already been visited.
1437 // In this case, we can compute the weight of the edge by
1438 // subtracting the total block weight from all the known
1439 // edge weights. If the edges weight more than BB, then the
1440 // edge of the last remaining edge is set to zero.
1441 //
1442 // - There exists a self-referential edge and the weight of BB is
1443 // known. In this case, this edge can be based on BB's weight.
1444 // We add up all the other known edges and set the weight on
1445 // the self-referential edge as we did in the previous case.
1446 //
1447 // In any other case, we must continue iterating. Eventually,
1448 // all edges will get a weight, or iteration will stop when
1449 // it reaches SampleProfileMaxPropagateIterations.
1450 if (NumUnknownEdges <= 1) {
1451 uint64_t &BBWeight = BlockWeights[EC];
1452 if (NumUnknownEdges == 0) {
1453 if (!VisitedBlocks.count(EC)) {
1454 // If we already know the weight of all edges, the weight of the
1455 // basic block can be computed. It should be no larger than the sum
1456 // of all edge weights.
1457 if (TotalWeight > BBWeight) {
1458 BBWeight = TotalWeight;
1459 Changed = true;
1460 LLVM_DEBUG(dbgs() << "All edge weights for " << BB->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "All edge weights for "
<< BB->getName() << " known. Set weight for block: "
; printBlockWeight(dbgs(), BB);; } } while (false)
1461 << " known. Set weight for block: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "All edge weights for "
<< BB->getName() << " known. Set weight for block: "
; printBlockWeight(dbgs(), BB);; } } while (false)
1462 printBlockWeight(dbgs(), BB);)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "All edge weights for "
<< BB->getName() << " known. Set weight for block: "
; printBlockWeight(dbgs(), BB);; } } while (false)
;
1463 }
1464 } else if (NumTotalEdges == 1 &&
1465 EdgeWeights[SingleEdge] < BlockWeights[EC]) {
1466 // If there is only one edge for the visited basic block, use the
1467 // block weight to adjust edge weight if edge weight is smaller.
1468 EdgeWeights[SingleEdge] = BlockWeights[EC];
1469 Changed = true;
1470 }
1471 } else if (NumUnknownEdges == 1 && VisitedBlocks.count(EC)) {
1472 // If there is a single unknown edge and the block has been
1473 // visited, then we can compute E's weight.
1474 if (BBWeight >= TotalWeight)
1475 EdgeWeights[UnknownEdge] = BBWeight - TotalWeight;
1476 else
1477 EdgeWeights[UnknownEdge] = 0;
1478 const BasicBlock *OtherEC;
1479 if (i == 0)
1480 OtherEC = EquivalenceClass[UnknownEdge.first];
1481 else
1482 OtherEC = EquivalenceClass[UnknownEdge.second];
1483 // Edge weights should never exceed the BB weights it connects.
1484 if (VisitedBlocks.count(OtherEC) &&
1485 EdgeWeights[UnknownEdge] > BlockWeights[OtherEC])
1486 EdgeWeights[UnknownEdge] = BlockWeights[OtherEC];
1487 VisitedEdges.insert(UnknownEdge);
1488 Changed = true;
1489 LLVM_DEBUG(dbgs() << "Set weight for edge: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "Set weight for edge: "
; printEdgeWeight(dbgs(), UnknownEdge); } } while (false)
1490 printEdgeWeight(dbgs(), UnknownEdge))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "Set weight for edge: "
; printEdgeWeight(dbgs(), UnknownEdge); } } while (false)
;
1491 }
1492 } else if (VisitedBlocks.count(EC) && BlockWeights[EC] == 0) {
1493 // If a block Weights 0, all its in/out edges should weight 0.
1494 if (i == 0) {
1495 for (auto *Pred : Predecessors[BB]) {
1496 Edge E = std::make_pair(Pred, BB);
1497 EdgeWeights[E] = 0;
1498 VisitedEdges.insert(E);
1499 }
1500 } else {
1501 for (auto *Succ : Successors[BB]) {
1502 Edge E = std::make_pair(BB, Succ);
1503 EdgeWeights[E] = 0;
1504 VisitedEdges.insert(E);
1505 }
1506 }
1507 } else if (SelfReferentialEdge.first && VisitedBlocks.count(EC)) {
1508 uint64_t &BBWeight = BlockWeights[BB];
1509 // We have a self-referential edge and the weight of BB is known.
1510 if (BBWeight >= TotalWeight)
1511 EdgeWeights[SelfReferentialEdge] = BBWeight - TotalWeight;
1512 else
1513 EdgeWeights[SelfReferentialEdge] = 0;
1514 VisitedEdges.insert(SelfReferentialEdge);
1515 Changed = true;
1516 LLVM_DEBUG(dbgs() << "Set self-referential edge weight to: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "Set self-referential edge weight to: "
; printEdgeWeight(dbgs(), SelfReferentialEdge); } } while (false
)
1517 printEdgeWeight(dbgs(), SelfReferentialEdge))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "Set self-referential edge weight to: "
; printEdgeWeight(dbgs(), SelfReferentialEdge); } } while (false
)
;
1518 }
1519 if (UpdateBlockCount && !VisitedBlocks.count(EC) && TotalWeight > 0) {
1520 BlockWeights[EC] = TotalWeight;
1521 VisitedBlocks.insert(EC);
1522 Changed = true;
1523 }
1524 }
1525 }
1526
1527 return Changed;
1528}
1529
1530/// Build in/out edge lists for each basic block in the CFG.
1531///
1532/// We are interested in unique edges. If a block B1 has multiple
1533/// edges to another block B2, we only add a single B1->B2 edge.
1534void SampleProfileLoader::buildEdges(Function &F) {
1535 for (auto &BI : F) {
1536 BasicBlock *B1 = &BI;
1537
1538 // Add predecessors for B1.
1539 SmallPtrSet<BasicBlock *, 16> Visited;
1540 if (!Predecessors[B1].empty())
1541 llvm_unreachable("Found a stale predecessors list in a basic block.")::llvm::llvm_unreachable_internal("Found a stale predecessors list in a basic block."
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1541)
;
1542 for (pred_iterator PI = pred_begin(B1), PE = pred_end(B1); PI != PE; ++PI) {
1543 BasicBlock *B2 = *PI;
1544 if (Visited.insert(B2).second)
1545 Predecessors[B1].push_back(B2);
1546 }
1547
1548 // Add successors for B1.
1549 Visited.clear();
1550 if (!Successors[B1].empty())
1551 llvm_unreachable("Found a stale successors list in a basic block.")::llvm::llvm_unreachable_internal("Found a stale successors list in a basic block."
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1551)
;
1552 for (succ_iterator SI = succ_begin(B1), SE = succ_end(B1); SI != SE; ++SI) {
1553 BasicBlock *B2 = *SI;
1554 if (Visited.insert(B2).second)
1555 Successors[B1].push_back(B2);
1556 }
1557 }
1558}
1559
1560/// Returns the sorted CallTargetMap \p M by count in descending order.
1561static SmallVector<InstrProfValueData, 2> GetSortedValueDataFromCallTargets(
1562 const SampleRecord::CallTargetMap & M) {
1563 SmallVector<InstrProfValueData, 2> R;
1564 for (const auto &I : SampleRecord::SortCallTargets(M)) {
1565 R.emplace_back(InstrProfValueData{FunctionSamples::getGUID(I.first), I.second});
1566 }
1567 return R;
1568}
1569
1570/// Propagate weights into edges
1571///
1572/// The following rules are applied to every block BB in the CFG:
1573///
1574/// - If BB has a single predecessor/successor, then the weight
1575/// of that edge is the weight of the block.
1576///
1577/// - If all incoming or outgoing edges are known except one, and the
1578/// weight of the block is already known, the weight of the unknown
1579/// edge will be the weight of the block minus the sum of all the known
1580/// edges. If the sum of all the known edges is larger than BB's weight,
1581/// we set the unknown edge weight to zero.
1582///
1583/// - If there is a self-referential edge, and the weight of the block is
1584/// known, the weight for that edge is set to the weight of the block
1585/// minus the weight of the other incoming edges to that block (if
1586/// known).
1587void SampleProfileLoader::propagateWeights(Function &F) {
1588 bool Changed = true;
1589 unsigned I = 0;
1590
1591 // If BB weight is larger than its corresponding loop's header BB weight,
1592 // use the BB weight to replace the loop header BB weight.
1593 for (auto &BI : F) {
1594 BasicBlock *BB = &BI;
1595 Loop *L = LI->getLoopFor(BB);
1596 if (!L) {
1597 continue;
1598 }
1599 BasicBlock *Header = L->getHeader();
1600 if (Header && BlockWeights[BB] > BlockWeights[Header]) {
1601 BlockWeights[Header] = BlockWeights[BB];
1602 }
1603 }
1604
1605 // Before propagation starts, build, for each block, a list of
1606 // unique predecessors and successors. This is necessary to handle
1607 // identical edges in multiway branches. Since we visit all blocks and all
1608 // edges of the CFG, it is cleaner to build these lists once at the start
1609 // of the pass.
1610 buildEdges(F);
1611
1612 // Propagate until we converge or we go past the iteration limit.
1613 while (Changed
26.1
'Changed' is true
26.1
'Changed' is true
26.1
'Changed' is true
26.1
'Changed' is true
&& I++ < SampleProfileMaxPropagateIterations) {
27
Assuming the condition is false
28
Loop condition is false. Execution continues on line 1620
1614 Changed = propagateThroughEdges(F, false);
1615 }
1616
1617 // The first propagation propagates BB counts from annotated BBs to unknown
1618 // BBs. The 2nd propagation pass resets edges weights, and use all BB weights
1619 // to propagate edge weights.
1620 VisitedEdges.clear();
1621 Changed = true;
1622 while (Changed
28.1
'Changed' is true
28.1
'Changed' is true
28.1
'Changed' is true
28.1
'Changed' is true
&& I++ < SampleProfileMaxPropagateIterations) {
29
Assuming the condition is false
30
Loop condition is false. Execution continues on line 1628
1623 Changed = propagateThroughEdges(F, false);
1624 }
1625
1626 // The 3rd propagation pass allows adjust annotated BB weights that are
1627 // obviously wrong.
1628 Changed = true;
1629 while (Changed
30.1
'Changed' is true
30.1
'Changed' is true
30.1
'Changed' is true
30.1
'Changed' is true
&& I++ < SampleProfileMaxPropagateIterations) {
31
Loop condition is false. Execution continues on line 1635
1630 Changed = propagateThroughEdges(F, true);
1631 }
1632
1633 // Generate MD_prof metadata for every branch instruction using the
1634 // edge weights computed during propagation.
1635 LLVM_DEBUG(dbgs() << "\nPropagation complete. Setting branch weights\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "\nPropagation complete. Setting branch weights\n"
; } } while (false)
;
32
Assuming 'DebugFlag' is false
33
Loop condition is false. Exiting loop
1636 LLVMContext &Ctx = F.getContext();
1637 MDBuilder MDB(Ctx);
1638 for (auto &BI : F) {
1639 BasicBlock *BB = &BI;
1640
1641 if (BlockWeights[BB]) {
34
Assuming the condition is true
35
Taking true branch
1642 for (auto &I : BB->getInstList()) {
1643 if (!isa<CallInst>(I) && !isa<InvokeInst>(I))
36
Assuming 'I' is not a 'CallInst'
37
Assuming 'I' is a 'InvokeInst'
38
Taking false branch
1644 continue;
1645 if (!cast<CallBase>(I).getCalledFunction()) {
39
'I' is a 'CallBase'
40
Calling 'CallBase::getCalledFunction'
43
Returning from 'CallBase::getCalledFunction'
44
Taking true branch
1646 const DebugLoc &DLoc = I.getDebugLoc();
1647 if (!DLoc)
45
Assuming the condition is false
46
Taking false branch
1648 continue;
1649 const DILocation *DIL = DLoc;
1650 const FunctionSamples *FS = findFunctionSamples(I);
47
Calling 'SampleProfileLoader::findFunctionSamples'
55
Returning from 'SampleProfileLoader::findFunctionSamples'
1651 if (!FS)
56
Assuming 'FS' is non-null
57
Taking false branch
1652 continue;
1653 auto CallSite = FunctionSamples::getCallSiteIdentifier(DIL);
1654 auto T = FS->findCallTargetMapAt(CallSite);
1655 if (!T || T.get().empty())
58
Calling 'ErrorOr::operator bool'
61
Returning from 'ErrorOr::operator bool'
62
Calling 'StringMapImpl::empty'
65
Returning from 'StringMapImpl::empty'
66
Taking false branch
1656 continue;
1657 SmallVector<InstrProfValueData, 2> SortedCallTargets =
1658 GetSortedValueDataFromCallTargets(T.get());
1659 uint64_t Sum;
67
'Sum' declared without an initial value
1660 findIndirectCallFunctionSamples(I, Sum);
68
Calling 'SampleProfileLoader::findIndirectCallFunctionSamples'
72
Returning from 'SampleProfileLoader::findIndirectCallFunctionSamples'
1661 annotateValueSite(*I.getParent()->getParent()->getParent(), I,
73
4th function call argument is an uninitialized value
1662 SortedCallTargets, Sum, IPVK_IndirectCallTarget,
1663 SortedCallTargets.size());
1664 } else if (!isa<IntrinsicInst>(&I)) {
1665 I.setMetadata(LLVMContext::MD_prof,
1666 MDB.createBranchWeights(
1667 {static_cast<uint32_t>(BlockWeights[BB])}));
1668 }
1669 }
1670 }
1671 Instruction *TI = BB->getTerminator();
1672 if (TI->getNumSuccessors() == 1)
1673 continue;
1674 if (!isa<BranchInst>(TI) && !isa<SwitchInst>(TI))
1675 continue;
1676
1677 DebugLoc BranchLoc = TI->getDebugLoc();
1678 LLVM_DEBUG(dbgs() << "\nGetting weights for branch at line "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "\nGetting weights for branch at line "
<< ((BranchLoc) ? Twine(BranchLoc.getLine()) : Twine("<UNKNOWN LOCATION>"
)) << ".\n"; } } while (false)
1679 << ((BranchLoc) ? Twine(BranchLoc.getLine())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "\nGetting weights for branch at line "
<< ((BranchLoc) ? Twine(BranchLoc.getLine()) : Twine("<UNKNOWN LOCATION>"
)) << ".\n"; } } while (false)
1680 : Twine("<UNKNOWN LOCATION>"))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "\nGetting weights for branch at line "
<< ((BranchLoc) ? Twine(BranchLoc.getLine()) : Twine("<UNKNOWN LOCATION>"
)) << ".\n"; } } while (false)
1681 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "\nGetting weights for branch at line "
<< ((BranchLoc) ? Twine(BranchLoc.getLine()) : Twine("<UNKNOWN LOCATION>"
)) << ".\n"; } } while (false)
;
1682 SmallVector<uint32_t, 4> Weights;
1683 uint32_t MaxWeight = 0;
1684 Instruction *MaxDestInst;
1685 for (unsigned I = 0; I < TI->getNumSuccessors(); ++I) {
1686 BasicBlock *Succ = TI->getSuccessor(I);
1687 Edge E = std::make_pair(BB, Succ);
1688 uint64_t Weight = EdgeWeights[E];
1689 LLVM_DEBUG(dbgs() << "\t"; printEdgeWeight(dbgs(), E))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "\t"; printEdgeWeight(dbgs
(), E); } } while (false)
;
1690 // Use uint32_t saturated arithmetic to adjust the incoming weights,
1691 // if needed. Sample counts in profiles are 64-bit unsigned values,
1692 // but internally branch weights are expressed as 32-bit values.
1693 if (Weight > std::numeric_limits<uint32_t>::max()) {
1694 LLVM_DEBUG(dbgs() << " (saturated due to uint32_t overflow)")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << " (saturated due to uint32_t overflow)"
; } } while (false)
;
1695 Weight = std::numeric_limits<uint32_t>::max();
1696 }
1697 // Weight is added by one to avoid propagation errors introduced by
1698 // 0 weights.
1699 Weights.push_back(static_cast<uint32_t>(Weight + 1));
1700 if (Weight != 0) {
1701 if (Weight > MaxWeight) {
1702 MaxWeight = Weight;
1703 MaxDestInst = Succ->getFirstNonPHIOrDbgOrLifetime();
1704 }
1705 }
1706 }
1707
1708 uint64_t TempWeight;
1709 // Only set weights if there is at least one non-zero weight.
1710 // In any other case, let the analyzer set weights.
1711 // Do not set weights if the weights are present. In ThinLTO, the profile
1712 // annotation is done twice. If the first annotation already set the
1713 // weights, the second pass does not need to set it.
1714 if (MaxWeight > 0 && !TI->extractProfTotalWeight(TempWeight)) {
1715 LLVM_DEBUG(dbgs() << "SUCCESS. Found non-zero weights.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "SUCCESS. Found non-zero weights.\n"
; } } while (false)
;
1716 TI->setMetadata(LLVMContext::MD_prof,
1717 MDB.createBranchWeights(Weights));
1718 ORE->emit([&]() {
1719 return OptimizationRemark(DEBUG_TYPE"sample-profile", "PopularDest", MaxDestInst)
1720 << "most popular destination for conditional branches at "
1721 << ore::NV("CondBranchesLoc", BranchLoc);
1722 });
1723 } else {
1724 LLVM_DEBUG(dbgs() << "SKIPPED. All branch weights are zero.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "SKIPPED. All branch weights are zero.\n"
; } } while (false)
;
1725 }
1726 }
1727}
1728
1729/// Get the line number for the function header.
1730///
1731/// This looks up function \p F in the current compilation unit and
1732/// retrieves the line number where the function is defined. This is
1733/// line 0 for all the samples read from the profile file. Every line
1734/// number is relative to this line.
1735///
1736/// \param F Function object to query.
1737///
1738/// \returns the line number where \p F is defined. If it returns 0,
1739/// it means that there is no debug information available for \p F.
1740unsigned SampleProfileLoader::getFunctionLoc(Function &F) {
1741 if (DISubprogram *S = F.getSubprogram())
1742 return S->getLine();
1743
1744 if (NoWarnSampleUnused)
1745 return 0;
1746
1747 // If the start of \p F is missing, emit a diagnostic to inform the user
1748 // about the missed opportunity.
1749 F.getContext().diagnose(DiagnosticInfoSampleProfile(
1750 "No debug information found in function " + F.getName() +
1751 ": Function profile not used",
1752 DS_Warning));
1753 return 0;
1754}
1755
1756void SampleProfileLoader::computeDominanceAndLoopInfo(Function &F) {
1757 DT.reset(new DominatorTree);
1758 DT->recalculate(F);
1759
1760 PDT.reset(new PostDominatorTree(F));
1761
1762 LI.reset(new LoopInfo);
1763 LI->analyze(*DT);
1764}
1765
1766/// Generate branch weight metadata for all branches in \p F.
1767///
1768/// Branch weights are computed out of instruction samples using a
1769/// propagation heuristic. Propagation proceeds in 3 phases:
1770///
1771/// 1- Assignment of block weights. All the basic blocks in the function
1772/// are initial assigned the same weight as their most frequently
1773/// executed instruction.
1774///
1775/// 2- Creation of equivalence classes. Since samples may be missing from
1776/// blocks, we can fill in the gaps by setting the weights of all the
1777/// blocks in the same equivalence class to the same weight. To compute
1778/// the concept of equivalence, we use dominance and loop information.
1779/// Two blocks B1 and B2 are in the same equivalence class if B1
1780/// dominates B2, B2 post-dominates B1 and both are in the same loop.
1781///
1782/// 3- Propagation of block weights into edges. This uses a simple
1783/// propagation heuristic. The following rules are applied to every
1784/// block BB in the CFG:
1785///
1786/// - If BB has a single predecessor/successor, then the weight
1787/// of that edge is the weight of the block.
1788///
1789/// - If all the edges are known except one, and the weight of the
1790/// block is already known, the weight of the unknown edge will
1791/// be the weight of the block minus the sum of all the known
1792/// edges. If the sum of all the known edges is larger than BB's weight,
1793/// we set the unknown edge weight to zero.
1794///
1795/// - If there is a self-referential edge, and the weight of the block is
1796/// known, the weight for that edge is set to the weight of the block
1797/// minus the weight of the other incoming edges to that block (if
1798/// known).
1799///
1800/// Since this propagation is not guaranteed to finalize for every CFG, we
1801/// only allow it to proceed for a limited number of iterations (controlled
1802/// by -sample-profile-max-propagate-iterations).
1803///
1804/// FIXME: Try to replace this propagation heuristic with a scheme
1805/// that is guaranteed to finalize. A work-list approach similar to
1806/// the standard value propagation algorithm used by SSA-CCP might
1807/// work here.
1808///
1809/// Once all the branch weights are computed, we emit the MD_prof
1810/// metadata on BB using the computed values for each of its branches.
1811///
1812/// \param F The function to query.
1813///
1814/// \returns true if \p F was modified. Returns false, otherwise.
1815bool SampleProfileLoader::emitAnnotations(Function &F) {
1816 bool Changed = false;
1817
1818 if (FunctionSamples::ProfileIsProbeBased) {
19
Assuming 'ProfileIsProbeBased' is false
20
Taking false branch
1819 if (!ProbeManager->profileIsValid(F, *Samples)) {
1820 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "Profile is invalid due to CFG mismatch for Function "
<< F.getName(); } } while (false)
1821 dbgs() << "Profile is invalid due to CFG mismatch for Function "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "Profile is invalid due to CFG mismatch for Function "
<< F.getName(); } } while (false)
1822 << F.getName())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "Profile is invalid due to CFG mismatch for Function "
<< F.getName(); } } while (false)
;
1823 ++NumMismatchedProfile;
1824 return false;
1825 }
1826 ++NumMatchedProfile;
1827 } else {
1828 if (getFunctionLoc(F) == 0)
21
Assuming the condition is false
22
Taking false branch
1829 return false;
1830
1831 LLVM_DEBUG(dbgs() << "Line number for the first instruction in "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "Line number for the first instruction in "
<< F.getName() << ": " << getFunctionLoc(F
) << "\n"; } } while (false)
23
Assuming 'DebugFlag' is false
24
Loop condition is false. Exiting loop
1832 << F.getName() << ": " << getFunctionLoc(F) << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "Line number for the first instruction in "
<< F.getName() << ": " << getFunctionLoc(F
) << "\n"; } } while (false)
;
1833 }
1834
1835 DenseSet<GlobalValue::GUID> InlinedGUIDs;
1836 Changed |= inlineHotFunctions(F, InlinedGUIDs);
1837
1838 // Compute basic block weights.
1839 Changed |= computeBlockWeights(F);
1840
1841 if (Changed
24.1
'Changed' is true
24.1
'Changed' is true
24.1
'Changed' is true
24.1
'Changed' is true
) {
25
Taking true branch
1842 // Add an entry count to the function using the samples gathered at the
1843 // function entry.
1844 // Sets the GUIDs that are inlined in the profiled binary. This is used
1845 // for ThinLink to make correct liveness analysis, and also make the IR
1846 // match the profiled binary before annotation.
1847 F.setEntryCount(
1848 ProfileCount(Samples->getHeadSamples() + 1, Function::PCT_Real),
1849 &InlinedGUIDs);
1850
1851 // Compute dominance and loop info needed for propagation.
1852 computeDominanceAndLoopInfo(F);
1853
1854 // Find equivalence classes.
1855 findEquivalenceClasses(F);
1856
1857 // Propagate weights to all edges.
1858 propagateWeights(F);
26
Calling 'SampleProfileLoader::propagateWeights'
1859 }
1860
1861 // If coverage checking was requested, compute it now.
1862 if (SampleProfileRecordCoverage) {
1863 unsigned Used = CoverageTracker.countUsedRecords(Samples, PSI);
1864 unsigned Total = CoverageTracker.countBodyRecords(Samples, PSI);
1865 unsigned Coverage = CoverageTracker.computeCoverage(Used, Total);
1866 if (Coverage < SampleProfileRecordCoverage) {
1867 F.getContext().diagnose(DiagnosticInfoSampleProfile(
1868 F.getSubprogram()->getFilename(), getFunctionLoc(F),
1869 Twine(Used) + " of " + Twine(Total) + " available profile records (" +
1870 Twine(Coverage) + "%) were applied",
1871 DS_Warning));
1872 }
1873 }
1874
1875 if (SampleProfileSampleCoverage) {
1876 uint64_t Used = CoverageTracker.getTotalUsedSamples();
1877 uint64_t Total = CoverageTracker.countBodySamples(Samples, PSI);
1878 unsigned Coverage = CoverageTracker.computeCoverage(Used, Total);
1879 if (Coverage < SampleProfileSampleCoverage) {
1880 F.getContext().diagnose(DiagnosticInfoSampleProfile(
1881 F.getSubprogram()->getFilename(), getFunctionLoc(F),
1882 Twine(Used) + " of " + Twine(Total) + " available profile samples (" +
1883 Twine(Coverage) + "%) were applied",
1884 DS_Warning));
1885 }
1886 }
1887 return Changed;
1888}
1889
1890char SampleProfileLoaderLegacyPass::ID = 0;
1891
1892INITIALIZE_PASS_BEGIN(SampleProfileLoaderLegacyPass, "sample-profile",static void *initializeSampleProfileLoaderLegacyPassPassOnce(
PassRegistry &Registry) {
1893 "Sample Profile loader", false, false)static void *initializeSampleProfileLoaderLegacyPassPassOnce(
PassRegistry &Registry) {
1894INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)initializeAssumptionCacheTrackerPass(Registry);
1895INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)initializeTargetTransformInfoWrapperPassPass(Registry);
1896INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)initializeTargetLibraryInfoWrapperPassPass(Registry);
1897INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)initializeProfileSummaryInfoWrapperPassPass(Registry);
1898INITIALIZE_PASS_END(SampleProfileLoaderLegacyPass, "sample-profile",PassInfo *PI = new PassInfo( "Sample Profile loader", "sample-profile"
, &SampleProfileLoaderLegacyPass::ID, PassInfo::NormalCtor_t
(callDefaultCtor<SampleProfileLoaderLegacyPass>), false
, false); Registry.registerPass(*PI, true); return PI; } static
llvm::once_flag InitializeSampleProfileLoaderLegacyPassPassFlag
; void llvm::initializeSampleProfileLoaderLegacyPassPass(PassRegistry
&Registry) { llvm::call_once(InitializeSampleProfileLoaderLegacyPassPassFlag
, initializeSampleProfileLoaderLegacyPassPassOnce, std::ref(Registry
)); }
1899 "Sample Profile loader", false, false)PassInfo *PI = new PassInfo( "Sample Profile loader", "sample-profile"
, &SampleProfileLoaderLegacyPass::ID, PassInfo::NormalCtor_t
(callDefaultCtor<SampleProfileLoaderLegacyPass>), false
, false); Registry.registerPass(*PI, true); return PI; } static
llvm::once_flag InitializeSampleProfileLoaderLegacyPassPassFlag
; void llvm::initializeSampleProfileLoaderLegacyPassPass(PassRegistry
&Registry) { llvm::call_once(InitializeSampleProfileLoaderLegacyPassPassFlag
, initializeSampleProfileLoaderLegacyPassPassOnce, std::ref(Registry
)); }
1900
1901std::vector<Function *>
1902SampleProfileLoader::buildFunctionOrder(Module &M, CallGraph *CG) {
1903 std::vector<Function *> FunctionOrderList;
1904 FunctionOrderList.reserve(M.size());
1905
1906 if (!ProfileTopDownLoad || CG == nullptr) {
1907 if (ProfileMergeInlinee) {
1908 // Disable ProfileMergeInlinee if profile is not loaded in top down order,
1909 // because the profile for a function may be used for the profile
1910 // annotation of its outline copy before the profile merging of its
1911 // non-inlined inline instances, and that is not the way how
1912 // ProfileMergeInlinee is supposed to work.
1913 ProfileMergeInlinee = false;
1914 }
1915
1916 for (Function &F : M)
1917 if (!F.isDeclaration() && F.hasFnAttribute("use-sample-profile"))
1918 FunctionOrderList.push_back(&F);
1919 return FunctionOrderList;
1920 }
1921
1922 assert(&CG->getModule() == &M)((&CG->getModule() == &M) ? static_cast<void>
(0) : __assert_fail ("&CG->getModule() == &M", "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1922, __PRETTY_FUNCTION__))
;
1923 scc_iterator<CallGraph *> CGI = scc_begin(CG);
1924 while (!CGI.isAtEnd()) {
1925 for (CallGraphNode *node : *CGI) {
1926 auto F = node->getFunction();
1927 if (F && !F->isDeclaration() && F->hasFnAttribute("use-sample-profile"))
1928 FunctionOrderList.push_back(F);
1929 }
1930 ++CGI;
1931 }
1932
1933 std::reverse(FunctionOrderList.begin(), FunctionOrderList.end());
1934 return FunctionOrderList;
1935}
1936
1937bool SampleProfileLoader::doInitialization(Module &M,
1938 FunctionAnalysisManager *FAM) {
1939 auto &Ctx = M.getContext();
1940
1941 auto ReaderOrErr =
1942 SampleProfileReader::create(Filename, Ctx, RemappingFilename);
1943 if (std::error_code EC = ReaderOrErr.getError()) {
1944 std::string Msg = "Could not open profile: " + EC.message();
1945 Ctx.diagnose(DiagnosticInfoSampleProfile(Filename, Msg));
1946 return false;
1947 }
1948 Reader = std::move(ReaderOrErr.get());
1949 Reader->collectFuncsFrom(M);
1950 ProfileIsValid = (Reader->read() == sampleprof_error::success);
1951 PSL = Reader->getProfileSymbolList();
1952
1953 // While profile-sample-accurate is on, ignore symbol list.
1954 ProfAccForSymsInList =
1955 ProfileAccurateForSymsInList && PSL && !ProfileSampleAccurate;
1956 if (ProfAccForSymsInList) {
1957 NamesInProfile.clear();
1958 if (auto NameTable = Reader->getNameTable())
1959 NamesInProfile.insert(NameTable->begin(), NameTable->end());
1960 }
1961
1962 if (FAM && !ProfileInlineReplayFile.empty()) {
1963 ExternalInlineAdvisor = std::make_unique<ReplayInlineAdvisor>(
1964 *FAM, Ctx, ProfileInlineReplayFile, /*EmitRemarks=*/false);
1965 if (!ExternalInlineAdvisor->areReplayRemarksLoaded())
1966 ExternalInlineAdvisor.reset();
1967 }
1968
1969 // Apply tweaks if context-sensitive profile is available.
1970 if (Reader->profileIsCS()) {
1971 ProfileIsCS = true;
1972 FunctionSamples::ProfileIsCS = true;
1973
1974 // Tracker for profiles under different context
1975 ContextTracker =
1976 std::make_unique<SampleContextTracker>(Reader->getProfiles());
1977 }
1978
1979 // Load pseudo probe descriptors for probe-based function samples.
1980 if (Reader->profileIsProbeBased()) {
1981 ProbeManager = std::make_unique<PseudoProbeManager>(M);
1982 if (!ProbeManager->moduleIsProbed(M)) {
1983 const char *Msg =
1984 "Pseudo-probe-based profile requires SampleProfileProbePass";
1985 Ctx.diagnose(DiagnosticInfoSampleProfile(Filename, Msg));
1986 return false;
1987 }
1988 }
1989
1990 return true;
1991}
1992
1993ModulePass *llvm::createSampleProfileLoaderPass() {
1994 return new SampleProfileLoaderLegacyPass();
1995}
1996
1997ModulePass *llvm::createSampleProfileLoaderPass(StringRef Name) {
1998 return new SampleProfileLoaderLegacyPass(Name);
1999}
2000
2001bool SampleProfileLoader::runOnModule(Module &M, ModuleAnalysisManager *AM,
2002 ProfileSummaryInfo *_PSI, CallGraph *CG) {
2003 if (!ProfileIsValid)
2
Assuming field 'ProfileIsValid' is true
3
Taking false branch
2004 return false;
2005 GUIDToFuncNameMapper Mapper(M, *Reader, GUIDToFuncNameMap);
2006
2007 PSI = _PSI;
2008 if (M.getProfileSummary(/* IsCS */ false) == nullptr) {
4
Assuming the condition is false
5
Taking false branch
2009 M.setProfileSummary(Reader->getSummary().getMD(M.getContext()),
2010 ProfileSummary::PSK_Sample);
2011 PSI->refresh();
2012 }
2013 // Compute the total number of samples collected in this profile.
2014 for (const auto &I : Reader->getProfiles())
2015 TotalCollectedSamples += I.second.getTotalSamples();
2016
2017 auto Remapper = Reader->getRemapper();
2018 // Populate the symbol map.
2019 for (const auto &N_F : M.getValueSymbolTable()) {
2020 StringRef OrigName = N_F.getKey();
2021 Function *F = dyn_cast<Function>(N_F.getValue());
2022 if (F == nullptr)
2023 continue;
2024 SymbolMap[OrigName] = F;
2025 auto pos = OrigName.find('.');
2026 if (pos != StringRef::npos) {
2027 StringRef NewName = OrigName.substr(0, pos);
2028 auto r = SymbolMap.insert(std::make_pair(NewName, F));
2029 // Failiing to insert means there is already an entry in SymbolMap,
2030 // thus there are multiple functions that are mapped to the same
2031 // stripped name. In this case of name conflicting, set the value
2032 // to nullptr to avoid confusion.
2033 if (!r.second)
2034 r.first->second = nullptr;
2035 OrigName = NewName;
2036 }
2037 // Insert the remapped names into SymbolMap.
2038 if (Remapper) {
2039 if (auto MapName = Remapper->lookUpNameInProfile(OrigName)) {
2040 if (*MapName == OrigName)
2041 continue;
2042 SymbolMap.insert(std::make_pair(*MapName, F));
2043 }
2044 }
2045 }
2046
2047 bool retval = false;
2048 for (auto F : buildFunctionOrder(M, CG)) {
2049 assert(!F->isDeclaration())((!F->isDeclaration()) ? static_cast<void> (0) : __assert_fail
("!F->isDeclaration()", "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 2049, __PRETTY_FUNCTION__))
;
6
Assuming the condition is true
7
'?' condition is true
2050 clearFunctionData();
2051 retval |= runOnFunction(*F, AM);
8
Calling 'SampleProfileLoader::runOnFunction'
2052 }
2053
2054 // Account for cold calls not inlined....
2055 if (!ProfileIsCS)
2056 for (const std::pair<Function *, NotInlinedProfileInfo> &pair :
2057 notInlinedCallInfo)
2058 updateProfileCallee(pair.first, pair.second.entryCount);
2059
2060 return retval;
2061}
2062
2063bool SampleProfileLoaderLegacyPass::runOnModule(Module &M) {
2064 ACT = &getAnalysis<AssumptionCacheTracker>();
2065 TTIWP = &getAnalysis<TargetTransformInfoWrapperPass>();
2066 TLIWP = &getAnalysis<TargetLibraryInfoWrapperPass>();
2067 ProfileSummaryInfo *PSI =
2068 &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
2069 return SampleLoader.runOnModule(M, nullptr, PSI, nullptr);
1
Calling 'SampleProfileLoader::runOnModule'
2070}
2071
2072bool SampleProfileLoader::runOnFunction(Function &F, ModuleAnalysisManager *AM) {
2073 DILocation2SampleMap.clear();
2074 // By default the entry count is initialized to -1, which will be treated
2075 // conservatively by getEntryCount as the same as unknown (None). This is
2076 // to avoid newly added code to be treated as cold. If we have samples
2077 // this will be overwritten in emitAnnotations.
2078 uint64_t initialEntryCount = -1;
2079
2080 ProfAccForSymsInList = ProfileAccurateForSymsInList && PSL;
9
Assuming the condition is false
2081 if (ProfileSampleAccurate || F.hasFnAttribute("profile-sample-accurate")) {
10
Assuming the condition is true
11
Taking true branch
2082 // initialize all the function entry counts to 0. It means all the
2083 // functions without profile will be regarded as cold.
2084 initialEntryCount = 0;
2085 // profile-sample-accurate is a user assertion which has a higher precedence
2086 // than symbol list. When profile-sample-accurate is on, ignore symbol list.
2087 ProfAccForSymsInList = false;
2088 }
2089
2090 // PSL -- profile symbol list include all the symbols in sampled binary.
2091 // If ProfileAccurateForSymsInList is enabled, PSL is used to treat
2092 // old functions without samples being cold, without having to worry
2093 // about new and hot functions being mistakenly treated as cold.
2094 if (ProfAccForSymsInList
11.1
Field 'ProfAccForSymsInList' is false
11.1
Field 'ProfAccForSymsInList' is false
11.1
Field 'ProfAccForSymsInList' is false
11.1
Field 'ProfAccForSymsInList' is false
) {
12
Taking false branch
2095 // Initialize the entry count to 0 for functions in the list.
2096 if (PSL->contains(F.getName()))
2097 initialEntryCount = 0;
2098
2099 // Function in the symbol list but without sample will be regarded as
2100 // cold. To minimize the potential negative performance impact it could
2101 // have, we want to be a little conservative here saying if a function
2102 // shows up in the profile, no matter as outline function, inline instance
2103 // or call targets, treat the function as not being cold. This will handle
2104 // the cases such as most callsites of a function are inlined in sampled
2105 // binary but not inlined in current build (because of source code drift,
2106 // imprecise debug information, or the callsites are all cold individually
2107 // but not cold accumulatively...), so the outline function showing up as
2108 // cold in sampled binary will actually not be cold after current build.
2109 StringRef CanonName = FunctionSamples::getCanonicalFnName(F);
2110 if (NamesInProfile.count(CanonName))
2111 initialEntryCount = -1;
2112 }
2113
2114 F.setEntryCount(ProfileCount(initialEntryCount, Function::PCT_Real));
2115 std::unique_ptr<OptimizationRemarkEmitter> OwnedORE;
2116 if (AM
12.1
'AM' is null
12.1
'AM' is null
12.1
'AM' is null
12.1
'AM' is null
) {
13
Taking false branch
2117 auto &FAM =
2118 AM->getResult<FunctionAnalysisManagerModuleProxy>(*F.getParent())
2119 .getManager();
2120 ORE = &FAM.getResult<OptimizationRemarkEmitterAnalysis>(F);
2121 } else {
2122 OwnedORE = std::make_unique<OptimizationRemarkEmitter>(&F);
2123 ORE = OwnedORE.get();
2124 }
2125
2126 if (ProfileIsCS)
14
Assuming field 'ProfileIsCS' is false
15
Taking false branch
2127 Samples = ContextTracker->getBaseSamplesFor(F);
2128 else
2129 Samples = Reader->getSamplesFor(F);
2130
2131 if (Samples && !Samples->empty())
16
Assuming field 'Samples' is non-null
17
Taking true branch
2132 return emitAnnotations(F);
18
Calling 'SampleProfileLoader::emitAnnotations'
2133 return false;
2134}
2135
2136PreservedAnalyses SampleProfileLoaderPass::run(Module &M,
2137 ModuleAnalysisManager &AM) {
2138 FunctionAnalysisManager &FAM =
2139 AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
2140
2141 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
2142 return FAM.getResult<AssumptionAnalysis>(F);
2143 };
2144 auto GetTTI = [&](Function &F) -> TargetTransformInfo & {
2145 return FAM.getResult<TargetIRAnalysis>(F);
2146 };
2147 auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
2148 return FAM.getResult<TargetLibraryAnalysis>(F);
2149 };
2150
2151 SampleProfileLoader SampleLoader(
2152 ProfileFileName.empty() ? SampleProfileFile : ProfileFileName,
2153 ProfileRemappingFileName.empty() ? SampleProfileRemappingFile
2154 : ProfileRemappingFileName,
2155 LTOPhase, GetAssumptionCache, GetTTI, GetTLI);
2156
2157 if (!SampleLoader.doInitialization(M, &FAM))
2158 return PreservedAnalyses::all();
2159
2160 ProfileSummaryInfo *PSI = &AM.getResult<ProfileSummaryAnalysis>(M);
2161 CallGraph &CG = AM.getResult<CallGraphAnalysis>(M);
2162 if (!SampleLoader.runOnModule(M, &AM, PSI, &CG))
2163 return PreservedAnalyses::all();
2164
2165 return PreservedAnalyses::none();
2166}

/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h

1//===- llvm/InstrTypes.h - Important Instruction subclasses -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines various meta classes of instructions that exist in the VM
10// representation. Specific concrete subclasses of these may be found in the
11// i*.h files...
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_IR_INSTRTYPES_H
16#define LLVM_IR_INSTRTYPES_H
17
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/None.h"
20#include "llvm/ADT/Optional.h"
21#include "llvm/ADT/STLExtras.h"
22#include "llvm/ADT/StringMap.h"
23#include "llvm/ADT/StringRef.h"
24#include "llvm/ADT/Twine.h"
25#include "llvm/ADT/iterator_range.h"
26#include "llvm/IR/Attributes.h"
27#include "llvm/IR/CallingConv.h"
28#include "llvm/IR/Constants.h"
29#include "llvm/IR/DerivedTypes.h"
30#include "llvm/IR/Function.h"
31#include "llvm/IR/Instruction.h"
32#include "llvm/IR/LLVMContext.h"
33#include "llvm/IR/OperandTraits.h"
34#include "llvm/IR/Type.h"
35#include "llvm/IR/User.h"
36#include "llvm/IR/Value.h"
37#include "llvm/Support/Casting.h"
38#include "llvm/Support/ErrorHandling.h"
39#include <algorithm>
40#include <cassert>
41#include <cstddef>
42#include <cstdint>
43#include <iterator>
44#include <string>
45#include <vector>
46
47namespace llvm {
48
49namespace Intrinsic {
50typedef unsigned ID;
51}
52
53//===----------------------------------------------------------------------===//
54// UnaryInstruction Class
55//===----------------------------------------------------------------------===//
56
57class UnaryInstruction : public Instruction {
58protected:
59 UnaryInstruction(Type *Ty, unsigned iType, Value *V,
60 Instruction *IB = nullptr)
61 : Instruction(Ty, iType, &Op<0>(), 1, IB) {
62 Op<0>() = V;
63 }
64 UnaryInstruction(Type *Ty, unsigned iType, Value *V, BasicBlock *IAE)
65 : Instruction(Ty, iType, &Op<0>(), 1, IAE) {
66 Op<0>() = V;
67 }
68
69public:
70 // allocate space for exactly one operand
71 void *operator new(size_t s) {
72 return User::operator new(s, 1);
73 }
74
75 /// Transparently provide more efficient getOperand methods.
76 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
77
78 // Methods for support type inquiry through isa, cast, and dyn_cast:
79 static bool classof(const Instruction *I) {
80 return I->isUnaryOp() ||
81 I->getOpcode() == Instruction::Alloca ||
82 I->getOpcode() == Instruction::Load ||
83 I->getOpcode() == Instruction::VAArg ||
84 I->getOpcode() == Instruction::ExtractValue ||
85 (I->getOpcode() >= CastOpsBegin && I->getOpcode() < CastOpsEnd);
86 }
87 static bool classof(const Value *V) {
88 return isa<Instruction>(V) && classof(cast<Instruction>(V));
89 }
90};
91
92template <>
93struct OperandTraits<UnaryInstruction> :
94 public FixedNumOperandTraits<UnaryInstruction, 1> {
95};
96
97DEFINE_TRANSPARENT_OPERAND_ACCESSORS(UnaryInstruction, Value)UnaryInstruction::op_iterator UnaryInstruction::op_begin() { return
OperandTraits<UnaryInstruction>::op_begin(this); } UnaryInstruction
::const_op_iterator UnaryInstruction::op_begin() const { return
OperandTraits<UnaryInstruction>::op_begin(const_cast<
UnaryInstruction*>(this)); } UnaryInstruction::op_iterator
UnaryInstruction::op_end() { return OperandTraits<UnaryInstruction
>::op_end(this); } UnaryInstruction::const_op_iterator UnaryInstruction
::op_end() const { return OperandTraits<UnaryInstruction>
::op_end(const_cast<UnaryInstruction*>(this)); } Value *
UnaryInstruction::getOperand(unsigned i_nocapture) const { ((
i_nocapture < OperandTraits<UnaryInstruction>::operands
(this) && "getOperand() out of range!") ? static_cast
<void> (0) : __assert_fail ("i_nocapture < OperandTraits<UnaryInstruction>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 97, __PRETTY_FUNCTION__)); return cast_or_null<Value>
( OperandTraits<UnaryInstruction>::op_begin(const_cast<
UnaryInstruction*>(this))[i_nocapture].get()); } void UnaryInstruction
::setOperand(unsigned i_nocapture, Value *Val_nocapture) { ((
i_nocapture < OperandTraits<UnaryInstruction>::operands
(this) && "setOperand() out of range!") ? static_cast
<void> (0) : __assert_fail ("i_nocapture < OperandTraits<UnaryInstruction>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 97, __PRETTY_FUNCTION__)); OperandTraits<UnaryInstruction
>::op_begin(this)[i_nocapture] = Val_nocapture; } unsigned
UnaryInstruction::getNumOperands() const { return OperandTraits
<UnaryInstruction>::operands(this); } template <int Idx_nocapture
> Use &UnaryInstruction::Op() { return this->OpFrom
<Idx_nocapture>(this); } template <int Idx_nocapture
> const Use &UnaryInstruction::Op() const { return this
->OpFrom<Idx_nocapture>(this); }
98
99//===----------------------------------------------------------------------===//
100// UnaryOperator Class
101//===----------------------------------------------------------------------===//
102
103class UnaryOperator : public UnaryInstruction {
104 void AssertOK();
105
106protected:
107 UnaryOperator(UnaryOps iType, Value *S, Type *Ty,
108 const Twine &Name, Instruction *InsertBefore);
109 UnaryOperator(UnaryOps iType, Value *S, Type *Ty,
110 const Twine &Name, BasicBlock *InsertAtEnd);
111
112 // Note: Instruction needs to be a friend here to call cloneImpl.
113 friend class Instruction;
114
115 UnaryOperator *cloneImpl() const;
116
117public:
118
119 /// Construct a unary instruction, given the opcode and an operand.
120 /// Optionally (if InstBefore is specified) insert the instruction
121 /// into a BasicBlock right before the specified instruction. The specified
122 /// Instruction is allowed to be a dereferenced end iterator.
123 ///
124 static UnaryOperator *Create(UnaryOps Op, Value *S,
125 const Twine &Name = Twine(),
126 Instruction *InsertBefore = nullptr);
127
128 /// Construct a unary instruction, given the opcode and an operand.
129 /// Also automatically insert this instruction to the end of the
130 /// BasicBlock specified.
131 ///
132 static UnaryOperator *Create(UnaryOps Op, Value *S,
133 const Twine &Name,
134 BasicBlock *InsertAtEnd);
135
136 /// These methods just forward to Create, and are useful when you
137 /// statically know what type of instruction you're going to create. These
138 /// helpers just save some typing.
139#define HANDLE_UNARY_INST(N, OPC, CLASS) \
140 static UnaryOperator *Create##OPC(Value *V, const Twine &Name = "") {\
141 return Create(Instruction::OPC, V, Name);\
142 }
143#include "llvm/IR/Instruction.def"
144#define HANDLE_UNARY_INST(N, OPC, CLASS) \
145 static UnaryOperator *Create##OPC(Value *V, const Twine &Name, \
146 BasicBlock *BB) {\
147 return Create(Instruction::OPC, V, Name, BB);\
148 }
149#include "llvm/IR/Instruction.def"
150#define HANDLE_UNARY_INST(N, OPC, CLASS) \
151 static UnaryOperator *Create##OPC(Value *V, const Twine &Name, \
152 Instruction *I) {\
153 return Create(Instruction::OPC, V, Name, I);\
154 }
155#include "llvm/IR/Instruction.def"
156
157 static UnaryOperator *
158 CreateWithCopiedFlags(UnaryOps Opc, Value *V, Instruction *CopyO,
159 const Twine &Name = "",
160 Instruction *InsertBefore = nullptr) {
161 UnaryOperator *UO = Create(Opc, V, Name, InsertBefore);
162 UO->copyIRFlags(CopyO);
163 return UO;
164 }
165
166 static UnaryOperator *CreateFNegFMF(Value *Op, Instruction *FMFSource,
167 const Twine &Name = "",
168 Instruction *InsertBefore = nullptr) {
169 return CreateWithCopiedFlags(Instruction::FNeg, Op, FMFSource, Name,
170 InsertBefore);
171 }
172
173 UnaryOps getOpcode() const {
174 return static_cast<UnaryOps>(Instruction::getOpcode());
175 }
176
177 // Methods for support type inquiry through isa, cast, and dyn_cast:
178 static bool classof(const Instruction *I) {
179 return I->isUnaryOp();
180 }
181 static bool classof(const Value *V) {
182 return isa<Instruction>(V) && classof(cast<Instruction>(V));
183 }
184};
185
186//===----------------------------------------------------------------------===//
187// BinaryOperator Class
188//===----------------------------------------------------------------------===//
189
190class BinaryOperator : public Instruction {
191 void AssertOK();
192
193protected:
194 BinaryOperator(BinaryOps iType, Value *S1, Value *S2, Type *Ty,
195 const Twine &Name, Instruction *InsertBefore);
196 BinaryOperator(BinaryOps iType, Value *S1, Value *S2, Type *Ty,
197 const Twine &Name, BasicBlock *InsertAtEnd);
198
199 // Note: Instruction needs to be a friend here to call cloneImpl.
200 friend class Instruction;
201
202 BinaryOperator *cloneImpl() const;
203
204public:
205 // allocate space for exactly two operands
206 void *operator new(size_t s) {
207 return User::operator new(s, 2);
208 }
209
210 /// Transparently provide more efficient getOperand methods.
211 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
212
213 /// Construct a binary instruction, given the opcode and the two
214 /// operands. Optionally (if InstBefore is specified) insert the instruction
215 /// into a BasicBlock right before the specified instruction. The specified
216 /// Instruction is allowed to be a dereferenced end iterator.
217 ///
218 static BinaryOperator *Create(BinaryOps Op, Value *S1, Value *S2,
219 const Twine &Name = Twine(),
220 Instruction *InsertBefore = nullptr);
221
222 /// Construct a binary instruction, given the opcode and the two
223 /// operands. Also automatically insert this instruction to the end of the
224 /// BasicBlock specified.
225 ///
226 static BinaryOperator *Create(BinaryOps Op, Value *S1, Value *S2,
227 const Twine &Name, BasicBlock *InsertAtEnd);
228
229 /// These methods just forward to Create, and are useful when you
230 /// statically know what type of instruction you're going to create. These
231 /// helpers just save some typing.
232#define HANDLE_BINARY_INST(N, OPC, CLASS) \
233 static BinaryOperator *Create##OPC(Value *V1, Value *V2, \
234 const Twine &Name = "") {\
235 return Create(Instruction::OPC, V1, V2, Name);\
236 }
237#include "llvm/IR/Instruction.def"
238#define HANDLE_BINARY_INST(N, OPC, CLASS) \
239 static BinaryOperator *Create##OPC(Value *V1, Value *V2, \
240 const Twine &Name, BasicBlock *BB) {\
241 return Create(Instruction::OPC, V1, V2, Name, BB);\
242 }
243#include "llvm/IR/Instruction.def"
244#define HANDLE_BINARY_INST(N, OPC, CLASS) \
245 static BinaryOperator *Create##OPC(Value *V1, Value *V2, \
246 const Twine &Name, Instruction *I) {\
247 return Create(Instruction::OPC, V1, V2, Name, I);\
248 }
249#include "llvm/IR/Instruction.def"
250
251 static BinaryOperator *CreateWithCopiedFlags(BinaryOps Opc,
252 Value *V1, Value *V2,
253 Instruction *CopyO,
254 const Twine &Name = "") {
255 BinaryOperator *BO = Create(Opc, V1, V2, Name);
256 BO->copyIRFlags(CopyO);
257 return BO;
258 }
259
260 static BinaryOperator *CreateFAddFMF(Value *V1, Value *V2,
261 Instruction *FMFSource,
262 const Twine &Name = "") {
263 return CreateWithCopiedFlags(Instruction::FAdd, V1, V2, FMFSource, Name);
264 }
265 static BinaryOperator *CreateFSubFMF(Value *V1, Value *V2,
266 Instruction *FMFSource,
267 const Twine &Name = "") {
268 return CreateWithCopiedFlags(Instruction::FSub, V1, V2, FMFSource, Name);
269 }
270 static BinaryOperator *CreateFMulFMF(Value *V1, Value *V2,
271 Instruction *FMFSource,
272 const Twine &Name = "") {
273 return CreateWithCopiedFlags(Instruction::FMul, V1, V2, FMFSource, Name);
274 }
275 static BinaryOperator *CreateFDivFMF(Value *V1, Value *V2,
276 Instruction *FMFSource,
277 const Twine &Name = "") {
278 return CreateWithCopiedFlags(Instruction::FDiv, V1, V2, FMFSource, Name);
279 }
280 static BinaryOperator *CreateFRemFMF(Value *V1, Value *V2,
281 Instruction *FMFSource,
282 const Twine &Name = "") {
283 return CreateWithCopiedFlags(Instruction::FRem, V1, V2, FMFSource, Name);
284 }
285
286 static BinaryOperator *CreateNSW(BinaryOps Opc, Value *V1, Value *V2,
287 const Twine &Name = "") {
288 BinaryOperator *BO = Create(Opc, V1, V2, Name);
289 BO->setHasNoSignedWrap(true);
290 return BO;
291 }
292 static BinaryOperator *CreateNSW(BinaryOps Opc, Value *V1, Value *V2,
293 const Twine &Name, BasicBlock *BB) {
294 BinaryOperator *BO = Create(Opc, V1, V2, Name, BB);
295 BO->setHasNoSignedWrap(true);
296 return BO;
297 }
298 static BinaryOperator *CreateNSW(BinaryOps Opc, Value *V1, Value *V2,
299 const Twine &Name, Instruction *I) {
300 BinaryOperator *BO = Create(Opc, V1, V2, Name, I);
301 BO->setHasNoSignedWrap(true);
302 return BO;
303 }
304
305 static BinaryOperator *CreateNUW(BinaryOps Opc, Value *V1, Value *V2,
306 const Twine &Name = "") {
307 BinaryOperator *BO = Create(Opc, V1, V2, Name);
308 BO->setHasNoUnsignedWrap(true);
309 return BO;
310 }
311 static BinaryOperator *CreateNUW(BinaryOps Opc, Value *V1, Value *V2,
312 const Twine &Name, BasicBlock *BB) {
313 BinaryOperator *BO = Create(Opc, V1, V2, Name, BB);
314 BO->setHasNoUnsignedWrap(true);
315 return BO;
316 }
317 static BinaryOperator *CreateNUW(BinaryOps Opc, Value *V1, Value *V2,
318 const Twine &Name, Instruction *I) {
319 BinaryOperator *BO = Create(Opc, V1, V2, Name, I);
320 BO->setHasNoUnsignedWrap(true);
321 return BO;
322 }
323
324 static BinaryOperator *CreateExact(BinaryOps Opc, Value *V1, Value *V2,
325 const Twine &Name = "") {
326 BinaryOperator *BO = Create(Opc, V1, V2, Name);
327 BO->setIsExact(true);
328 return BO;
329 }
330 static BinaryOperator *CreateExact(BinaryOps Opc, Value *V1, Value *V2,
331 const Twine &Name, BasicBlock *BB) {
332 BinaryOperator *BO = Create(Opc, V1, V2, Name, BB);
333 BO->setIsExact(true);
334 return BO;
335 }
336 static BinaryOperator *CreateExact(BinaryOps Opc, Value *V1, Value *V2,
337 const Twine &Name, Instruction *I) {
338 BinaryOperator *BO = Create(Opc, V1, V2, Name, I);
339 BO->setIsExact(true);
340 return BO;
341 }
342
343#define DEFINE_HELPERS(OPC, NUWNSWEXACT) \
344 static BinaryOperator *Create##NUWNSWEXACT##OPC(Value *V1, Value *V2, \
345 const Twine &Name = "") { \
346 return Create##NUWNSWEXACT(Instruction::OPC, V1, V2, Name); \
347 } \
348 static BinaryOperator *Create##NUWNSWEXACT##OPC( \
349 Value *V1, Value *V2, const Twine &Name, BasicBlock *BB) { \
350 return Create##NUWNSWEXACT(Instruction::OPC, V1, V2, Name, BB); \
351 } \
352 static BinaryOperator *Create##NUWNSWEXACT##OPC( \
353 Value *V1, Value *V2, const Twine &Name, Instruction *I) { \
354 return Create##NUWNSWEXACT(Instruction::OPC, V1, V2, Name, I); \
355 }
356
357 DEFINE_HELPERS(Add, NSW) // CreateNSWAdd
358 DEFINE_HELPERS(Add, NUW) // CreateNUWAdd
359 DEFINE_HELPERS(Sub, NSW) // CreateNSWSub
360 DEFINE_HELPERS(Sub, NUW) // CreateNUWSub
361 DEFINE_HELPERS(Mul, NSW) // CreateNSWMul
362 DEFINE_HELPERS(Mul, NUW) // CreateNUWMul
363 DEFINE_HELPERS(Shl, NSW) // CreateNSWShl
364 DEFINE_HELPERS(Shl, NUW) // CreateNUWShl
365
366 DEFINE_HELPERS(SDiv, Exact) // CreateExactSDiv
367 DEFINE_HELPERS(UDiv, Exact) // CreateExactUDiv
368 DEFINE_HELPERS(AShr, Exact) // CreateExactAShr
369 DEFINE_HELPERS(LShr, Exact) // CreateExactLShr
370
371#undef DEFINE_HELPERS
372
373 /// Helper functions to construct and inspect unary operations (NEG and NOT)
374 /// via binary operators SUB and XOR:
375 ///
376 /// Create the NEG and NOT instructions out of SUB and XOR instructions.
377 ///
378 static BinaryOperator *CreateNeg(Value *Op, const Twine &Name = "",
379 Instruction *InsertBefore = nullptr);
380 static BinaryOperator *CreateNeg(Value *Op, const Twine &Name,
381 BasicBlock *InsertAtEnd);
382 static BinaryOperator *CreateNSWNeg(Value *Op, const Twine &Name = "",
383 Instruction *InsertBefore = nullptr);
384 static BinaryOperator *CreateNSWNeg(Value *Op, const Twine &Name,
385 BasicBlock *InsertAtEnd);
386 static BinaryOperator *CreateNUWNeg(Value *Op, const Twine &Name = "",
387 Instruction *InsertBefore = nullptr);
388 static BinaryOperator *CreateNUWNeg(Value *Op, const Twine &Name,
389 BasicBlock *InsertAtEnd);
390 static BinaryOperator *CreateNot(Value *Op, const Twine &Name = "",
391 Instruction *InsertBefore = nullptr);
392 static BinaryOperator *CreateNot(Value *Op, const Twine &Name,
393 BasicBlock *InsertAtEnd);
394
395 BinaryOps getOpcode() const {
396 return static_cast<BinaryOps>(Instruction::getOpcode());
397 }
398
399 /// Exchange the two operands to this instruction.
400 /// This instruction is safe to use on any binary instruction and
401 /// does not modify the semantics of the instruction. If the instruction
402 /// cannot be reversed (ie, it's a Div), then return true.
403 ///
404 bool swapOperands();
405
406 // Methods for support type inquiry through isa, cast, and dyn_cast:
407 static bool classof(const Instruction *I) {
408 return I->isBinaryOp();
409 }
410 static bool classof(const Value *V) {
411 return isa<Instruction>(V) && classof(cast<Instruction>(V));
412 }
413};
414
415template <>
416struct OperandTraits<BinaryOperator> :
417 public FixedNumOperandTraits<BinaryOperator, 2> {
418};
419
420DEFINE_TRANSPARENT_OPERAND_ACCESSORS(BinaryOperator, Value)BinaryOperator::op_iterator BinaryOperator::op_begin() { return
OperandTraits<BinaryOperator>::op_begin(this); } BinaryOperator
::const_op_iterator BinaryOperator::op_begin() const { return
OperandTraits<BinaryOperator>::op_begin(const_cast<
BinaryOperator*>(this)); } BinaryOperator::op_iterator BinaryOperator
::op_end() { return OperandTraits<BinaryOperator>::op_end
(this); } BinaryOperator::const_op_iterator BinaryOperator::op_end
() const { return OperandTraits<BinaryOperator>::op_end
(const_cast<BinaryOperator*>(this)); } Value *BinaryOperator
::getOperand(unsigned i_nocapture) const { ((i_nocapture <
OperandTraits<BinaryOperator>::operands(this) &&
"getOperand() out of range!") ? static_cast<void> (0) :
__assert_fail ("i_nocapture < OperandTraits<BinaryOperator>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 420, __PRETTY_FUNCTION__)); return cast_or_null<Value>
( OperandTraits<BinaryOperator>::op_begin(const_cast<
BinaryOperator*>(this))[i_nocapture].get()); } void BinaryOperator
::setOperand(unsigned i_nocapture, Value *Val_nocapture) { ((
i_nocapture < OperandTraits<BinaryOperator>::operands
(this) && "setOperand() out of range!") ? static_cast
<void> (0) : __assert_fail ("i_nocapture < OperandTraits<BinaryOperator>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 420, __PRETTY_FUNCTION__)); OperandTraits<BinaryOperator
>::op_begin(this)[i_nocapture] = Val_nocapture; } unsigned
BinaryOperator::getNumOperands() const { return OperandTraits
<BinaryOperator>::operands(this); } template <int Idx_nocapture
> Use &BinaryOperator::Op() { return this->OpFrom<
Idx_nocapture>(this); } template <int Idx_nocapture>
const Use &BinaryOperator::Op() const { return this->
OpFrom<Idx_nocapture>(this); }
421
422//===----------------------------------------------------------------------===//
423// CastInst Class
424//===----------------------------------------------------------------------===//
425
426/// This is the base class for all instructions that perform data
427/// casts. It is simply provided so that instruction category testing
428/// can be performed with code like:
429///
430/// if (isa<CastInst>(Instr)) { ... }
431/// Base class of casting instructions.
432class CastInst : public UnaryInstruction {
433protected:
434 /// Constructor with insert-before-instruction semantics for subclasses
435 CastInst(Type *Ty, unsigned iType, Value *S,
436 const Twine &NameStr = "", Instruction *InsertBefore = nullptr)
437 : UnaryInstruction(Ty, iType, S, InsertBefore) {
438 setName(NameStr);
439 }
440 /// Constructor with insert-at-end-of-block semantics for subclasses
441 CastInst(Type *Ty, unsigned iType, Value *S,
442 const Twine &NameStr, BasicBlock *InsertAtEnd)
443 : UnaryInstruction(Ty, iType, S, InsertAtEnd) {
444 setName(NameStr);
445 }
446
447public:
448 /// Provides a way to construct any of the CastInst subclasses using an
449 /// opcode instead of the subclass's constructor. The opcode must be in the
450 /// CastOps category (Instruction::isCast(opcode) returns true). This
451 /// constructor has insert-before-instruction semantics to automatically
452 /// insert the new CastInst before InsertBefore (if it is non-null).
453 /// Construct any of the CastInst subclasses
454 static CastInst *Create(
455 Instruction::CastOps, ///< The opcode of the cast instruction
456 Value *S, ///< The value to be casted (operand 0)
457 Type *Ty, ///< The type to which cast should be made
458 const Twine &Name = "", ///< Name for the instruction
459 Instruction *InsertBefore = nullptr ///< Place to insert the instruction
460 );
461 /// Provides a way to construct any of the CastInst subclasses using an
462 /// opcode instead of the subclass's constructor. The opcode must be in the
463 /// CastOps category. This constructor has insert-at-end-of-block semantics
464 /// to automatically insert the new CastInst at the end of InsertAtEnd (if
465 /// its non-null).
466 /// Construct any of the CastInst subclasses
467 static CastInst *Create(
468 Instruction::CastOps, ///< The opcode for the cast instruction
469 Value *S, ///< The value to be casted (operand 0)
470 Type *Ty, ///< The type to which operand is casted
471 const Twine &Name, ///< The name for the instruction
472 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
473 );
474
475 /// Create a ZExt or BitCast cast instruction
476 static CastInst *CreateZExtOrBitCast(
477 Value *S, ///< The value to be casted (operand 0)
478 Type *Ty, ///< The type to which cast should be made
479 const Twine &Name = "", ///< Name for the instruction
480 Instruction *InsertBefore = nullptr ///< Place to insert the instruction
481 );
482
483 /// Create a ZExt or BitCast cast instruction
484 static CastInst *CreateZExtOrBitCast(
485 Value *S, ///< The value to be casted (operand 0)
486 Type *Ty, ///< The type to which operand is casted
487 const Twine &Name, ///< The name for the instruction
488 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
489 );
490
491 /// Create a SExt or BitCast cast instruction
492 static CastInst *CreateSExtOrBitCast(
493 Value *S, ///< The value to be casted (operand 0)
494 Type *Ty, ///< The type to which cast should be made
495 const Twine &Name = "", ///< Name for the instruction
496 Instruction *InsertBefore = nullptr ///< Place to insert the instruction
497 );
498
499 /// Create a SExt or BitCast cast instruction
500 static CastInst *CreateSExtOrBitCast(
501 Value *S, ///< The value to be casted (operand 0)
502 Type *Ty, ///< The type to which operand is casted
503 const Twine &Name, ///< The name for the instruction
504 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
505 );
506
507 /// Create a BitCast AddrSpaceCast, or a PtrToInt cast instruction.
508 static CastInst *CreatePointerCast(
509 Value *S, ///< The pointer value to be casted (operand 0)
510 Type *Ty, ///< The type to which operand is casted
511 const Twine &Name, ///< The name for the instruction
512 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
513 );
514
515 /// Create a BitCast, AddrSpaceCast or a PtrToInt cast instruction.
516 static CastInst *CreatePointerCast(
517 Value *S, ///< The pointer value to be casted (operand 0)
518 Type *Ty, ///< The type to which cast should be made
519 const Twine &Name = "", ///< Name for the instruction
520 Instruction *InsertBefore = nullptr ///< Place to insert the instruction
521 );
522
523 /// Create a BitCast or an AddrSpaceCast cast instruction.
524 static CastInst *CreatePointerBitCastOrAddrSpaceCast(
525 Value *S, ///< The pointer value to be casted (operand 0)
526 Type *Ty, ///< The type to which operand is casted
527 const Twine &Name, ///< The name for the instruction
528 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
529 );
530
531 /// Create a BitCast or an AddrSpaceCast cast instruction.
532 static CastInst *CreatePointerBitCastOrAddrSpaceCast(
533 Value *S, ///< The pointer value to be casted (operand 0)
534 Type *Ty, ///< The type to which cast should be made
535 const Twine &Name = "", ///< Name for the instruction
536 Instruction *InsertBefore = nullptr ///< Place to insert the instruction
537 );
538
539 /// Create a BitCast, a PtrToInt, or an IntToPTr cast instruction.
540 ///
541 /// If the value is a pointer type and the destination an integer type,
542 /// creates a PtrToInt cast. If the value is an integer type and the
543 /// destination a pointer type, creates an IntToPtr cast. Otherwise, creates
544 /// a bitcast.
545 static CastInst *CreateBitOrPointerCast(
546 Value *S, ///< The pointer value to be casted (operand 0)
547 Type *Ty, ///< The type to which cast should be made
548 const Twine &Name = "", ///< Name for the instruction
549 Instruction *InsertBefore = nullptr ///< Place to insert the instruction
550 );
551
552 /// Create a ZExt, BitCast, or Trunc for int -> int casts.
553 static CastInst *CreateIntegerCast(
554 Value *S, ///< The pointer value to be casted (operand 0)
555 Type *Ty, ///< The type to which cast should be made
556 bool isSigned, ///< Whether to regard S as signed or not
557 const Twine &Name = "", ///< Name for the instruction
558 Instruction *InsertBefore = nullptr ///< Place to insert the instruction
559 );
560
561 /// Create a ZExt, BitCast, or Trunc for int -> int casts.
562 static CastInst *CreateIntegerCast(
563 Value *S, ///< The integer value to be casted (operand 0)
564 Type *Ty, ///< The integer type to which operand is casted
565 bool isSigned, ///< Whether to regard S as signed or not
566 const Twine &Name, ///< The name for the instruction
567 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
568 );
569
570 /// Create an FPExt, BitCast, or FPTrunc for fp -> fp casts
571 static CastInst *CreateFPCast(
572 Value *S, ///< The floating point value to be casted
573 Type *Ty, ///< The floating point type to cast to
574 const Twine &Name = "", ///< Name for the instruction
575 Instruction *InsertBefore = nullptr ///< Place to insert the instruction
576 );
577
578 /// Create an FPExt, BitCast, or FPTrunc for fp -> fp casts
579 static CastInst *CreateFPCast(
580 Value *S, ///< The floating point value to be casted
581 Type *Ty, ///< The floating point type to cast to
582 const Twine &Name, ///< The name for the instruction
583 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
584 );
585
586 /// Create a Trunc or BitCast cast instruction
587 static CastInst *CreateTruncOrBitCast(
588 Value *S, ///< The value to be casted (operand 0)
589 Type *Ty, ///< The type to which cast should be made
590 const Twine &Name = "", ///< Name for the instruction
591 Instruction *InsertBefore = nullptr ///< Place to insert the instruction
592 );
593
594 /// Create a Trunc or BitCast cast instruction
595 static CastInst *CreateTruncOrBitCast(
596 Value *S, ///< The value to be casted (operand 0)
597 Type *Ty, ///< The type to which operand is casted
598 const Twine &Name, ///< The name for the instruction
599 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
600 );
601
602 /// Check whether a bitcast between these types is valid
603 static bool isBitCastable(
604 Type *SrcTy, ///< The Type from which the value should be cast.
605 Type *DestTy ///< The Type to which the value should be cast.
606 );
607
608 /// Check whether a bitcast, inttoptr, or ptrtoint cast between these
609 /// types is valid and a no-op.
610 ///
611 /// This ensures that any pointer<->integer cast has enough bits in the
612 /// integer and any other cast is a bitcast.
613 static bool isBitOrNoopPointerCastable(
614 Type *SrcTy, ///< The Type from which the value should be cast.
615 Type *DestTy, ///< The Type to which the value should be cast.
616 const DataLayout &DL);
617
618 /// Returns the opcode necessary to cast Val into Ty using usual casting
619 /// rules.
620 /// Infer the opcode for cast operand and type
621 static Instruction::CastOps getCastOpcode(
622 const Value *Val, ///< The value to cast
623 bool SrcIsSigned, ///< Whether to treat the source as signed
624 Type *Ty, ///< The Type to which the value should be casted
625 bool DstIsSigned ///< Whether to treate the dest. as signed
626 );
627
628 /// There are several places where we need to know if a cast instruction
629 /// only deals with integer source and destination types. To simplify that
630 /// logic, this method is provided.
631 /// @returns true iff the cast has only integral typed operand and dest type.
632 /// Determine if this is an integer-only cast.
633 bool isIntegerCast() const;
634
635 /// A lossless cast is one that does not alter the basic value. It implies
636 /// a no-op cast but is more stringent, preventing things like int->float,
637 /// long->double, or int->ptr.
638 /// @returns true iff the cast is lossless.
639 /// Determine if this is a lossless cast.
640 bool isLosslessCast() const;
641
642 /// A no-op cast is one that can be effected without changing any bits.
643 /// It implies that the source and destination types are the same size. The
644 /// DataLayout argument is to determine the pointer size when examining casts
645 /// involving Integer and Pointer types. They are no-op casts if the integer
646 /// is the same size as the pointer. However, pointer size varies with
647 /// platform. Note that a precondition of this method is that the cast is
648 /// legal - i.e. the instruction formed with these operands would verify.
649 static bool isNoopCast(
650 Instruction::CastOps Opcode, ///< Opcode of cast
651 Type *SrcTy, ///< SrcTy of cast
652 Type *DstTy, ///< DstTy of cast
653 const DataLayout &DL ///< DataLayout to get the Int Ptr type from.
654 );
655
656 /// Determine if this cast is a no-op cast.
657 ///
658 /// \param DL is the DataLayout to determine pointer size.
659 bool isNoopCast(const DataLayout &DL) const;
660
661 /// Determine how a pair of casts can be eliminated, if they can be at all.
662 /// This is a helper function for both CastInst and ConstantExpr.
663 /// @returns 0 if the CastInst pair can't be eliminated, otherwise
664 /// returns Instruction::CastOps value for a cast that can replace
665 /// the pair, casting SrcTy to DstTy.
666 /// Determine if a cast pair is eliminable
667 static unsigned isEliminableCastPair(
668 Instruction::CastOps firstOpcode, ///< Opcode of first cast
669 Instruction::CastOps secondOpcode, ///< Opcode of second cast
670 Type *SrcTy, ///< SrcTy of 1st cast
671 Type *MidTy, ///< DstTy of 1st cast & SrcTy of 2nd cast
672 Type *DstTy, ///< DstTy of 2nd cast
673 Type *SrcIntPtrTy, ///< Integer type corresponding to Ptr SrcTy, or null
674 Type *MidIntPtrTy, ///< Integer type corresponding to Ptr MidTy, or null
675 Type *DstIntPtrTy ///< Integer type corresponding to Ptr DstTy, or null
676 );
677
678 /// Return the opcode of this CastInst
679 Instruction::CastOps getOpcode() const {
680 return Instruction::CastOps(Instruction::getOpcode());
681 }
682
683 /// Return the source type, as a convenience
684 Type* getSrcTy() const { return getOperand(0)->getType(); }
685 /// Return the destination type, as a convenience
686 Type* getDestTy() const { return getType(); }
687
688 /// This method can be used to determine if a cast from SrcTy to DstTy using
689 /// Opcode op is valid or not.
690 /// @returns true iff the proposed cast is valid.
691 /// Determine if a cast is valid without creating one.
692 static bool castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy);
693 static bool castIsValid(Instruction::CastOps op, Value *S, Type *DstTy) {
694 return castIsValid(op, S->getType(), DstTy);
695 }
696
697 /// Methods for support type inquiry through isa, cast, and dyn_cast:
698 static bool classof(const Instruction *I) {
699 return I->isCast();
700 }
701 static bool classof(const Value *V) {
702 return isa<Instruction>(V) && classof(cast<Instruction>(V));
703 }
704};
705
706//===----------------------------------------------------------------------===//
707// CmpInst Class
708//===----------------------------------------------------------------------===//
709
710/// This class is the base class for the comparison instructions.
711/// Abstract base class of comparison instructions.
712class CmpInst : public Instruction {
713public:
714 /// This enumeration lists the possible predicates for CmpInst subclasses.
715 /// Values in the range 0-31 are reserved for FCmpInst, while values in the
716 /// range 32-64 are reserved for ICmpInst. This is necessary to ensure the
717 /// predicate values are not overlapping between the classes.
718 ///
719 /// Some passes (e.g. InstCombine) depend on the bit-wise characteristics of
720 /// FCMP_* values. Changing the bit patterns requires a potential change to
721 /// those passes.
722 enum Predicate : unsigned {
723 // Opcode U L G E Intuitive operation
724 FCMP_FALSE = 0, ///< 0 0 0 0 Always false (always folded)
725 FCMP_OEQ = 1, ///< 0 0 0 1 True if ordered and equal
726 FCMP_OGT = 2, ///< 0 0 1 0 True if ordered and greater than
727 FCMP_OGE = 3, ///< 0 0 1 1 True if ordered and greater than or equal
728 FCMP_OLT = 4, ///< 0 1 0 0 True if ordered and less than
729 FCMP_OLE = 5, ///< 0 1 0 1 True if ordered and less than or equal
730 FCMP_ONE = 6, ///< 0 1 1 0 True if ordered and operands are unequal
731 FCMP_ORD = 7, ///< 0 1 1 1 True if ordered (no nans)
732 FCMP_UNO = 8, ///< 1 0 0 0 True if unordered: isnan(X) | isnan(Y)
733 FCMP_UEQ = 9, ///< 1 0 0 1 True if unordered or equal
734 FCMP_UGT = 10, ///< 1 0 1 0 True if unordered or greater than
735 FCMP_UGE = 11, ///< 1 0 1 1 True if unordered, greater than, or equal
736 FCMP_ULT = 12, ///< 1 1 0 0 True if unordered or less than
737 FCMP_ULE = 13, ///< 1 1 0 1 True if unordered, less than, or equal
738 FCMP_UNE = 14, ///< 1 1 1 0 True if unordered or not equal
739 FCMP_TRUE = 15, ///< 1 1 1 1 Always true (always folded)
740 FIRST_FCMP_PREDICATE = FCMP_FALSE,
741 LAST_FCMP_PREDICATE = FCMP_TRUE,
742 BAD_FCMP_PREDICATE = FCMP_TRUE + 1,
743 ICMP_EQ = 32, ///< equal
744 ICMP_NE = 33, ///< not equal
745 ICMP_UGT = 34, ///< unsigned greater than
746 ICMP_UGE = 35, ///< unsigned greater or equal
747 ICMP_ULT = 36, ///< unsigned less than
748 ICMP_ULE = 37, ///< unsigned less or equal
749 ICMP_SGT = 38, ///< signed greater than
750 ICMP_SGE = 39, ///< signed greater or equal
751 ICMP_SLT = 40, ///< signed less than
752 ICMP_SLE = 41, ///< signed less or equal
753 FIRST_ICMP_PREDICATE = ICMP_EQ,
754 LAST_ICMP_PREDICATE = ICMP_SLE,
755 BAD_ICMP_PREDICATE = ICMP_SLE + 1
756 };
757 using PredicateField =
758 Bitfield::Element<Predicate, 0, 6, LAST_ICMP_PREDICATE>;
759
760protected:
761 CmpInst(Type *ty, Instruction::OtherOps op, Predicate pred,
762 Value *LHS, Value *RHS, const Twine &Name = "",
763 Instruction *InsertBefore = nullptr,
764 Instruction *FlagsSource = nullptr);
765
766 CmpInst(Type *ty, Instruction::OtherOps op, Predicate pred,
767 Value *LHS, Value *RHS, const Twine &Name,
768 BasicBlock *InsertAtEnd);
769
770public:
771 // allocate space for exactly two operands
772 void *operator new(size_t s) {
773 return User::operator new(s, 2);
774 }
775
776 /// Construct a compare instruction, given the opcode, the predicate and
777 /// the two operands. Optionally (if InstBefore is specified) insert the
778 /// instruction into a BasicBlock right before the specified instruction.
779 /// The specified Instruction is allowed to be a dereferenced end iterator.
780 /// Create a CmpInst
781 static CmpInst *Create(OtherOps Op,
782 Predicate predicate, Value *S1,
783 Value *S2, const Twine &Name = "",
784 Instruction *InsertBefore = nullptr);
785
786 /// Construct a compare instruction, given the opcode, the predicate and the
787 /// two operands. Also automatically insert this instruction to the end of
788 /// the BasicBlock specified.
789 /// Create a CmpInst
790 static CmpInst *Create(OtherOps Op, Predicate predicate, Value *S1,
791 Value *S2, const Twine &Name, BasicBlock *InsertAtEnd);
792
793 /// Get the opcode casted to the right type
794 OtherOps getOpcode() const {
795 return static_cast<OtherOps>(Instruction::getOpcode());
796 }
797
798 /// Return the predicate for this instruction.
799 Predicate getPredicate() const { return getSubclassData<PredicateField>(); }
800
801 /// Set the predicate for this instruction to the specified value.
802 void setPredicate(Predicate P) { setSubclassData<PredicateField>(P); }
803
804 static bool isFPPredicate(Predicate P) {
805 assert(FIRST_FCMP_PREDICATE == 0 &&((FIRST_FCMP_PREDICATE == 0 && "FIRST_FCMP_PREDICATE is required to be 0"
) ? static_cast<void> (0) : __assert_fail ("FIRST_FCMP_PREDICATE == 0 && \"FIRST_FCMP_PREDICATE is required to be 0\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 806, __PRETTY_FUNCTION__))
806 "FIRST_FCMP_PREDICATE is required to be 0")((FIRST_FCMP_PREDICATE == 0 && "FIRST_FCMP_PREDICATE is required to be 0"
) ? static_cast<void> (0) : __assert_fail ("FIRST_FCMP_PREDICATE == 0 && \"FIRST_FCMP_PREDICATE is required to be 0\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 806, __PRETTY_FUNCTION__))
;
807 return P <= LAST_FCMP_PREDICATE;
808 }
809
810 static bool isIntPredicate(Predicate P) {
811 return P >= FIRST_ICMP_PREDICATE && P <= LAST_ICMP_PREDICATE;
812 }
813
814 static StringRef getPredicateName(Predicate P);
815
816 bool isFPPredicate() const { return isFPPredicate(getPredicate()); }
817 bool isIntPredicate() const { return isIntPredicate(getPredicate()); }
818
819 /// For example, EQ -> NE, UGT -> ULE, SLT -> SGE,
820 /// OEQ -> UNE, UGT -> OLE, OLT -> UGE, etc.
821 /// @returns the inverse predicate for the instruction's current predicate.
822 /// Return the inverse of the instruction's predicate.
823 Predicate getInversePredicate() const {
824 return getInversePredicate(getPredicate());
825 }
826
827 /// For example, EQ -> NE, UGT -> ULE, SLT -> SGE,
828 /// OEQ -> UNE, UGT -> OLE, OLT -> UGE, etc.
829 /// @returns the inverse predicate for predicate provided in \p pred.
830 /// Return the inverse of a given predicate
831 static Predicate getInversePredicate(Predicate pred);
832
833 /// For example, EQ->EQ, SLE->SGE, ULT->UGT,
834 /// OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
835 /// @returns the predicate that would be the result of exchanging the two
836 /// operands of the CmpInst instruction without changing the result
837 /// produced.
838 /// Return the predicate as if the operands were swapped
839 Predicate getSwappedPredicate() const {
840 return getSwappedPredicate(getPredicate());
841 }
842
843 /// This is a static version that you can use without an instruction
844 /// available.
845 /// Return the predicate as if the operands were swapped.
846 static Predicate getSwappedPredicate(Predicate pred);
847
848 /// This is a static version that you can use without an instruction
849 /// available.
850 /// @returns true if the comparison predicate is strict, false otherwise.
851 static bool isStrictPredicate(Predicate predicate);
852
853 /// @returns true if the comparison predicate is strict, false otherwise.
854 /// Determine if this instruction is using an strict comparison predicate.
855 bool isStrictPredicate() const { return isStrictPredicate(getPredicate()); }
856
857 /// This is a static version that you can use without an instruction
858 /// available.
859 /// @returns true if the comparison predicate is non-strict, false otherwise.
860 static bool isNonStrictPredicate(Predicate predicate);
861
862 /// @returns true if the comparison predicate is non-strict, false otherwise.
863 /// Determine if this instruction is using an non-strict comparison predicate.
864 bool isNonStrictPredicate() const {
865 return isNonStrictPredicate(getPredicate());
866 }
867
868 /// For example, SGE -> SGT, SLE -> SLT, ULE -> ULT, UGE -> UGT.
869 /// Returns the strict version of non-strict comparisons.
870 Predicate getStrictPredicate() const {
871 return getStrictPredicate(getPredicate());
872 }
873
874 /// This is a static version that you can use without an instruction
875 /// available.
876 /// @returns the strict version of comparison provided in \p pred.
877 /// If \p pred is not a strict comparison predicate, returns \p pred.
878 /// Returns the strict version of non-strict comparisons.
879 static Predicate getStrictPredicate(Predicate pred);
880
881 /// For example, SGT -> SGE, SLT -> SLE, ULT -> ULE, UGT -> UGE.
882 /// Returns the non-strict version of strict comparisons.
883 Predicate getNonStrictPredicate() const {
884 return getNonStrictPredicate(getPredicate());
885 }
886
887 /// This is a static version that you can use without an instruction
888 /// available.
889 /// @returns the non-strict version of comparison provided in \p pred.
890 /// If \p pred is not a strict comparison predicate, returns \p pred.
891 /// Returns the non-strict version of strict comparisons.
892 static Predicate getNonStrictPredicate(Predicate pred);
893
894 /// This is a static version that you can use without an instruction
895 /// available.
896 /// Return the flipped strictness of predicate
897 static Predicate getFlippedStrictnessPredicate(Predicate pred);
898
899 /// For predicate of kind "is X or equal to 0" returns the predicate "is X".
900 /// For predicate of kind "is X" returns the predicate "is X or equal to 0".
901 /// does not support other kind of predicates.
902 /// @returns the predicate that does not contains is equal to zero if
903 /// it had and vice versa.
904 /// Return the flipped strictness of predicate
905 Predicate getFlippedStrictnessPredicate() const {
906 return getFlippedStrictnessPredicate(getPredicate());
907 }
908
909 /// Provide more efficient getOperand methods.
910 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
911
912 /// This is just a convenience that dispatches to the subclasses.
913 /// Swap the operands and adjust predicate accordingly to retain
914 /// the same comparison.
915 void swapOperands();
916
917 /// This is just a convenience that dispatches to the subclasses.
918 /// Determine if this CmpInst is commutative.
919 bool isCommutative() const;
920
921 /// Determine if this is an equals/not equals predicate.
922 /// This is a static version that you can use without an instruction
923 /// available.
924 static bool isEquality(Predicate pred);
925
926 /// Determine if this is an equals/not equals predicate.
927 bool isEquality() const { return isEquality(getPredicate()); }
928
929 /// Return true if the predicate is relational (not EQ or NE).
930 static bool isRelational(Predicate P) { return !isEquality(P); }
931
932 /// Return true if the predicate is relational (not EQ or NE).
933 bool isRelational() const { return !isEquality(); }
934
935 /// @returns true if the comparison is signed, false otherwise.
936 /// Determine if this instruction is using a signed comparison.
937 bool isSigned() const {
938 return isSigned(getPredicate());
939 }
940
941 /// @returns true if the comparison is unsigned, false otherwise.
942 /// Determine if this instruction is using an unsigned comparison.
943 bool isUnsigned() const {
944 return isUnsigned(getPredicate());
945 }
946
947 /// For example, ULT->SLT, ULE->SLE, UGT->SGT, UGE->SGE, SLT->Failed assert
948 /// @returns the signed version of the unsigned predicate pred.
949 /// return the signed version of a predicate
950 static Predicate getSignedPredicate(Predicate pred);
951
952 /// For example, ULT->SLT, ULE->SLE, UGT->SGT, UGE->SGE, SLT->Failed assert
953 /// @returns the signed version of the predicate for this instruction (which
954 /// has to be an unsigned predicate).
955 /// return the signed version of a predicate
956 Predicate getSignedPredicate() {
957 return getSignedPredicate(getPredicate());
958 }
959
960 /// For example, SLT->ULT, SLE->ULE, SGT->UGT, SGE->UGE, ULT->Failed assert
961 /// @returns the unsigned version of the signed predicate pred.
962 static Predicate getUnsignedPredicate(Predicate pred);
963
964 /// For example, SLT->ULT, SLE->ULE, SGT->UGT, SGE->UGE, ULT->Failed assert
965 /// @returns the unsigned version of the predicate for this instruction (which
966 /// has to be an signed predicate).
967 /// return the unsigned version of a predicate
968 Predicate getUnsignedPredicate() {
969 return getUnsignedPredicate(getPredicate());
970 }
971
972 /// For example, SLT->ULT, ULT->SLT, SLE->ULE, ULE->SLE, EQ->Failed assert
973 /// @returns the unsigned version of the signed predicate pred or
974 /// the signed version of the signed predicate pred.
975 static Predicate getFlippedSignednessPredicate(Predicate pred);
976
977 /// For example, SLT->ULT, ULT->SLT, SLE->ULE, ULE->SLE, EQ->Failed assert
978 /// @returns the unsigned version of the signed predicate pred or
979 /// the signed version of the signed predicate pred.
980 Predicate getFlippedSignednessPredicate() {
981 return getFlippedSignednessPredicate(getPredicate());
982 }
983
984 /// This is just a convenience.
985 /// Determine if this is true when both operands are the same.
986 bool isTrueWhenEqual() const {
987 return isTrueWhenEqual(getPredicate());
988 }
989
990 /// This is just a convenience.
991 /// Determine if this is false when both operands are the same.
992 bool isFalseWhenEqual() const {
993 return isFalseWhenEqual(getPredicate());
994 }
995
996 /// @returns true if the predicate is unsigned, false otherwise.
997 /// Determine if the predicate is an unsigned operation.
998 static bool isUnsigned(Predicate predicate);
999
1000 /// @returns true if the predicate is signed, false otherwise.
1001 /// Determine if the predicate is an signed operation.
1002 static bool isSigned(Predicate predicate);
1003
1004 /// Determine if the predicate is an ordered operation.
1005 static bool isOrdered(Predicate predicate);
1006
1007 /// Determine if the predicate is an unordered operation.
1008 static bool isUnordered(Predicate predicate);
1009
1010 /// Determine if the predicate is true when comparing a value with itself.
1011 static bool isTrueWhenEqual(Predicate predicate);
1012
1013 /// Determine if the predicate is false when comparing a value with itself.
1014 static bool isFalseWhenEqual(Predicate predicate);
1015
1016 /// Determine if Pred1 implies Pred2 is true when two compares have matching
1017 /// operands.
1018 static bool isImpliedTrueByMatchingCmp(Predicate Pred1, Predicate Pred2);
1019
1020 /// Determine if Pred1 implies Pred2 is false when two compares have matching
1021 /// operands.
1022 static bool isImpliedFalseByMatchingCmp(Predicate Pred1, Predicate Pred2);
1023
1024 /// Methods for support type inquiry through isa, cast, and dyn_cast:
1025 static bool classof(const Instruction *I) {
1026 return I->getOpcode() == Instruction::ICmp ||
1027 I->getOpcode() == Instruction::FCmp;
1028 }
1029 static bool classof(const Value *V) {
1030 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1031 }
1032
1033 /// Create a result type for fcmp/icmp
1034 static Type* makeCmpResultType(Type* opnd_type) {
1035 if (VectorType* vt = dyn_cast<VectorType>(opnd_type)) {
1036 return VectorType::get(Type::getInt1Ty(opnd_type->getContext()),
1037 vt->getElementCount());
1038 }
1039 return Type::getInt1Ty(opnd_type->getContext());
1040 }
1041
1042private:
1043 // Shadow Value::setValueSubclassData with a private forwarding method so that
1044 // subclasses cannot accidentally use it.
1045 void setValueSubclassData(unsigned short D) {
1046 Value::setValueSubclassData(D);
1047 }
1048};
1049
1050// FIXME: these are redundant if CmpInst < BinaryOperator
1051template <>
1052struct OperandTraits<CmpInst> : public FixedNumOperandTraits<CmpInst, 2> {
1053};
1054
1055DEFINE_TRANSPARENT_OPERAND_ACCESSORS(CmpInst, Value)CmpInst::op_iterator CmpInst::op_begin() { return OperandTraits
<CmpInst>::op_begin(this); } CmpInst::const_op_iterator
CmpInst::op_begin() const { return OperandTraits<CmpInst>
::op_begin(const_cast<CmpInst*>(this)); } CmpInst::op_iterator
CmpInst::op_end() { return OperandTraits<CmpInst>::op_end
(this); } CmpInst::const_op_iterator CmpInst::op_end() const {
return OperandTraits<CmpInst>::op_end(const_cast<CmpInst
*>(this)); } Value *CmpInst::getOperand(unsigned i_nocapture
) const { ((i_nocapture < OperandTraits<CmpInst>::operands
(this) && "getOperand() out of range!") ? static_cast
<void> (0) : __assert_fail ("i_nocapture < OperandTraits<CmpInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1055, __PRETTY_FUNCTION__)); return cast_or_null<Value>
( OperandTraits<CmpInst>::op_begin(const_cast<CmpInst
*>(this))[i_nocapture].get()); } void CmpInst::setOperand(
unsigned i_nocapture, Value *Val_nocapture) { ((i_nocapture <
OperandTraits<CmpInst>::operands(this) && "setOperand() out of range!"
) ? static_cast<void> (0) : __assert_fail ("i_nocapture < OperandTraits<CmpInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1055, __PRETTY_FUNCTION__)); OperandTraits<CmpInst>::
op_begin(this)[i_nocapture] = Val_nocapture; } unsigned CmpInst
::getNumOperands() const { return OperandTraits<CmpInst>
::operands(this); } template <int Idx_nocapture> Use &
CmpInst::Op() { return this->OpFrom<Idx_nocapture>(this
); } template <int Idx_nocapture> const Use &CmpInst
::Op() const { return this->OpFrom<Idx_nocapture>(this
); }
1056
1057/// A lightweight accessor for an operand bundle meant to be passed
1058/// around by value.
1059struct OperandBundleUse {
1060 ArrayRef<Use> Inputs;
1061
1062 OperandBundleUse() = default;
1063 explicit OperandBundleUse(StringMapEntry<uint32_t> *Tag, ArrayRef<Use> Inputs)
1064 : Inputs(Inputs), Tag(Tag) {}
1065
1066 /// Return true if the operand at index \p Idx in this operand bundle
1067 /// has the attribute A.
1068 bool operandHasAttr(unsigned Idx, Attribute::AttrKind A) const {
1069 if (isDeoptOperandBundle())
1070 if (A == Attribute::ReadOnly || A == Attribute::NoCapture)
1071 return Inputs[Idx]->getType()->isPointerTy();
1072
1073 // Conservative answer: no operands have any attributes.
1074 return false;
1075 }
1076
1077 /// Return the tag of this operand bundle as a string.
1078 StringRef getTagName() const {
1079 return Tag->getKey();
1080 }
1081
1082 /// Return the tag of this operand bundle as an integer.
1083 ///
1084 /// Operand bundle tags are interned by LLVMContextImpl::getOrInsertBundleTag,
1085 /// and this function returns the unique integer getOrInsertBundleTag
1086 /// associated the tag of this operand bundle to.
1087 uint32_t getTagID() const {
1088 return Tag->getValue();
1089 }
1090
1091 /// Return true if this is a "deopt" operand bundle.
1092 bool isDeoptOperandBundle() const {
1093 return getTagID() == LLVMContext::OB_deopt;
1094 }
1095
1096 /// Return true if this is a "funclet" operand bundle.
1097 bool isFuncletOperandBundle() const {
1098 return getTagID() == LLVMContext::OB_funclet;
1099 }
1100
1101 /// Return true if this is a "cfguardtarget" operand bundle.
1102 bool isCFGuardTargetOperandBundle() const {
1103 return getTagID() == LLVMContext::OB_cfguardtarget;
1104 }
1105
1106private:
1107 /// Pointer to an entry in LLVMContextImpl::getOrInsertBundleTag.
1108 StringMapEntry<uint32_t> *Tag;
1109};
1110
1111/// A container for an operand bundle being viewed as a set of values
1112/// rather than a set of uses.
1113///
1114/// Unlike OperandBundleUse, OperandBundleDefT owns the memory it carries, and
1115/// so it is possible to create and pass around "self-contained" instances of
1116/// OperandBundleDef and ConstOperandBundleDef.
1117template <typename InputTy> class OperandBundleDefT {
1118 std::string Tag;
1119 std::vector<InputTy> Inputs;
1120
1121public:
1122 explicit OperandBundleDefT(std::string Tag, std::vector<InputTy> Inputs)
1123 : Tag(std::move(Tag)), Inputs(std::move(Inputs)) {}
1124 explicit OperandBundleDefT(std::string Tag, ArrayRef<InputTy> Inputs)
1125 : Tag(std::move(Tag)), Inputs(Inputs) {}
1126
1127 explicit OperandBundleDefT(const OperandBundleUse &OBU) {
1128 Tag = std::string(OBU.getTagName());
1129 llvm::append_range(Inputs, OBU.Inputs);
1130 }
1131
1132 ArrayRef<InputTy> inputs() const { return Inputs; }
1133
1134 using input_iterator = typename std::vector<InputTy>::const_iterator;
1135
1136 size_t input_size() const { return Inputs.size(); }
1137 input_iterator input_begin() const { return Inputs.begin(); }
1138 input_iterator input_end() const { return Inputs.end(); }
1139
1140 StringRef getTag() const { return Tag; }
1141};
1142
1143using OperandBundleDef = OperandBundleDefT<Value *>;
1144using ConstOperandBundleDef = OperandBundleDefT<const Value *>;
1145
1146//===----------------------------------------------------------------------===//
1147// CallBase Class
1148//===----------------------------------------------------------------------===//
1149
1150/// Base class for all callable instructions (InvokeInst and CallInst)
1151/// Holds everything related to calling a function.
1152///
1153/// All call-like instructions are required to use a common operand layout:
1154/// - Zero or more arguments to the call,
1155/// - Zero or more operand bundles with zero or more operand inputs each
1156/// bundle,
1157/// - Zero or more subclass controlled operands
1158/// - The called function.
1159///
1160/// This allows this base class to easily access the called function and the
1161/// start of the arguments without knowing how many other operands a particular
1162/// subclass requires. Note that accessing the end of the argument list isn't
1163/// as cheap as most other operations on the base class.
1164class CallBase : public Instruction {
1165protected:
1166 // The first two bits are reserved by CallInst for fast retrieval,
1167 using CallInstReservedField = Bitfield::Element<unsigned, 0, 2>;
1168 using CallingConvField =
1169 Bitfield::Element<CallingConv::ID, CallInstReservedField::NextBit, 10,
1170 CallingConv::MaxID>;
1171 static_assert(
1172 Bitfield::areContiguous<CallInstReservedField, CallingConvField>(),
1173 "Bitfields must be contiguous");
1174
1175 /// The last operand is the called operand.
1176 static constexpr int CalledOperandOpEndIdx = -1;
1177
1178 AttributeList Attrs; ///< parameter attributes for callable
1179 FunctionType *FTy;
1180
1181 template <class... ArgsTy>
1182 CallBase(AttributeList const &A, FunctionType *FT, ArgsTy &&... Args)
1183 : Instruction(std::forward<ArgsTy>(Args)...), Attrs(A), FTy(FT) {}
1184
1185 using Instruction::Instruction;
1186
1187 bool hasDescriptor() const { return Value::HasDescriptor; }
1188
1189 unsigned getNumSubclassExtraOperands() const {
1190 switch (getOpcode()) {
1191 case Instruction::Call:
1192 return 0;
1193 case Instruction::Invoke:
1194 return 2;
1195 case Instruction::CallBr:
1196 return getNumSubclassExtraOperandsDynamic();
1197 }
1198 llvm_unreachable("Invalid opcode!")::llvm::llvm_unreachable_internal("Invalid opcode!", "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1198)
;
1199 }
1200
1201 /// Get the number of extra operands for instructions that don't have a fixed
1202 /// number of extra operands.
1203 unsigned getNumSubclassExtraOperandsDynamic() const;
1204
1205public:
1206 using Instruction::getContext;
1207
1208 /// Create a clone of \p CB with a different set of operand bundles and
1209 /// insert it before \p InsertPt.
1210 ///
1211 /// The returned call instruction is identical \p CB in every way except that
1212 /// the operand bundles for the new instruction are set to the operand bundles
1213 /// in \p Bundles.
1214 static CallBase *Create(CallBase *CB, ArrayRef<OperandBundleDef> Bundles,
1215 Instruction *InsertPt = nullptr);
1216
1217 static bool classof(const Instruction *I) {
1218 return I->getOpcode() == Instruction::Call ||
1219 I->getOpcode() == Instruction::Invoke ||
1220 I->getOpcode() == Instruction::CallBr;
1221 }
1222 static bool classof(const Value *V) {
1223 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1224 }
1225
1226 FunctionType *getFunctionType() const { return FTy; }
1227
1228 void mutateFunctionType(FunctionType *FTy) {
1229 Value::mutateType(FTy->getReturnType());
1230 this->FTy = FTy;
1231 }
1232
1233 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
1234
1235 /// data_operands_begin/data_operands_end - Return iterators iterating over
1236 /// the call / invoke argument list and bundle operands. For invokes, this is
1237 /// the set of instruction operands except the invoke target and the two
1238 /// successor blocks; and for calls this is the set of instruction operands
1239 /// except the call target.
1240 User::op_iterator data_operands_begin() { return op_begin(); }
1241 User::const_op_iterator data_operands_begin() const {
1242 return const_cast<CallBase *>(this)->data_operands_begin();
1243 }
1244 User::op_iterator data_operands_end() {
1245 // Walk from the end of the operands over the called operand and any
1246 // subclass operands.
1247 return op_end() - getNumSubclassExtraOperands() - 1;
1248 }
1249 User::const_op_iterator data_operands_end() const {
1250 return const_cast<CallBase *>(this)->data_operands_end();
1251 }
1252 iterator_range<User::op_iterator> data_ops() {
1253 return make_range(data_operands_begin(), data_operands_end());
1254 }
1255 iterator_range<User::const_op_iterator> data_ops() const {
1256 return make_range(data_operands_begin(), data_operands_end());
1257 }
1258 bool data_operands_empty() const {
1259 return data_operands_end() == data_operands_begin();
1260 }
1261 unsigned data_operands_size() const {
1262 return std::distance(data_operands_begin(), data_operands_end());
1263 }
1264
1265 bool isDataOperand(const Use *U) const {
1266 assert(this == U->getUser() &&((this == U->getUser() && "Only valid to query with a use of this instruction!"
) ? static_cast<void> (0) : __assert_fail ("this == U->getUser() && \"Only valid to query with a use of this instruction!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1267, __PRETTY_FUNCTION__))
1267 "Only valid to query with a use of this instruction!")((this == U->getUser() && "Only valid to query with a use of this instruction!"
) ? static_cast<void> (0) : __assert_fail ("this == U->getUser() && \"Only valid to query with a use of this instruction!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1267, __PRETTY_FUNCTION__))
;
1268 return data_operands_begin() <= U && U < data_operands_end();
1269 }
1270 bool isDataOperand(Value::const_user_iterator UI) const {
1271 return isDataOperand(&UI.getUse());
1272 }
1273
1274 /// Given a value use iterator, return the data operand corresponding to it.
1275 /// Iterator must actually correspond to a data operand.
1276 unsigned getDataOperandNo(Value::const_user_iterator UI) const {
1277 return getDataOperandNo(&UI.getUse());
1278 }
1279
1280 /// Given a use for a data operand, get the data operand number that
1281 /// corresponds to it.
1282 unsigned getDataOperandNo(const Use *U) const {
1283 assert(isDataOperand(U) && "Data operand # out of range!")((isDataOperand(U) && "Data operand # out of range!")
? static_cast<void> (0) : __assert_fail ("isDataOperand(U) && \"Data operand # out of range!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1283, __PRETTY_FUNCTION__))
;
1284 return U - data_operands_begin();
1285 }
1286
1287 /// Return the iterator pointing to the beginning of the argument list.
1288 User::op_iterator arg_begin() { return op_begin(); }
1289 User::const_op_iterator arg_begin() const {
1290 return const_cast<CallBase *>(this)->arg_begin();
1291 }
1292
1293 /// Return the iterator pointing to the end of the argument list.
1294 User::op_iterator arg_end() {
1295 // From the end of the data operands, walk backwards past the bundle
1296 // operands.
1297 return data_operands_end() - getNumTotalBundleOperands();
1298 }
1299 User::const_op_iterator arg_end() const {
1300 return const_cast<CallBase *>(this)->arg_end();
1301 }
1302
1303 /// Iteration adapter for range-for loops.
1304 iterator_range<User::op_iterator> args() {
1305 return make_range(arg_begin(), arg_end());
1306 }
1307 iterator_range<User::const_op_iterator> args() const {
1308 return make_range(arg_begin(), arg_end());
1309 }
1310 bool arg_empty() const { return arg_end() == arg_begin(); }
1311 unsigned arg_size() const { return arg_end() - arg_begin(); }
1312
1313 // Legacy API names that duplicate the above and will be removed once users
1314 // are migrated.
1315 iterator_range<User::op_iterator> arg_operands() {
1316 return make_range(arg_begin(), arg_end());
1317 }
1318 iterator_range<User::const_op_iterator> arg_operands() const {
1319 return make_range(arg_begin(), arg_end());
1320 }
1321 unsigned getNumArgOperands() const { return arg_size(); }
1322
1323 Value *getArgOperand(unsigned i) const {
1324 assert(i < getNumArgOperands() && "Out of bounds!")((i < getNumArgOperands() && "Out of bounds!") ? static_cast
<void> (0) : __assert_fail ("i < getNumArgOperands() && \"Out of bounds!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1324, __PRETTY_FUNCTION__))
;
1325 return getOperand(i);
1326 }
1327
1328 void setArgOperand(unsigned i, Value *v) {
1329 assert(i < getNumArgOperands() && "Out of bounds!")((i < getNumArgOperands() && "Out of bounds!") ? static_cast
<void> (0) : __assert_fail ("i < getNumArgOperands() && \"Out of bounds!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1329, __PRETTY_FUNCTION__))
;
1330 setOperand(i, v);
1331 }
1332
1333 /// Wrappers for getting the \c Use of a call argument.
1334 const Use &getArgOperandUse(unsigned i) const {
1335 assert(i < getNumArgOperands() && "Out of bounds!")((i < getNumArgOperands() && "Out of bounds!") ? static_cast
<void> (0) : __assert_fail ("i < getNumArgOperands() && \"Out of bounds!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1335, __PRETTY_FUNCTION__))
;
1336 return User::getOperandUse(i);
1337 }
1338 Use &getArgOperandUse(unsigned i) {
1339 assert(i < getNumArgOperands() && "Out of bounds!")((i < getNumArgOperands() && "Out of bounds!") ? static_cast
<void> (0) : __assert_fail ("i < getNumArgOperands() && \"Out of bounds!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1339, __PRETTY_FUNCTION__))
;
1340 return User::getOperandUse(i);
1341 }
1342
1343 bool isArgOperand(const Use *U) const {
1344 assert(this == U->getUser() &&((this == U->getUser() && "Only valid to query with a use of this instruction!"
) ? static_cast<void> (0) : __assert_fail ("this == U->getUser() && \"Only valid to query with a use of this instruction!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1345, __PRETTY_FUNCTION__))
1345 "Only valid to query with a use of this instruction!")((this == U->getUser() && "Only valid to query with a use of this instruction!"
) ? static_cast<void> (0) : __assert_fail ("this == U->getUser() && \"Only valid to query with a use of this instruction!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1345, __PRETTY_FUNCTION__))
;
1346 return arg_begin() <= U && U < arg_end();
1347 }
1348 bool isArgOperand(Value::const_user_iterator UI) const {
1349 return isArgOperand(&UI.getUse());
1350 }
1351
1352 /// Given a use for a arg operand, get the arg operand number that
1353 /// corresponds to it.
1354 unsigned getArgOperandNo(const Use *U) const {
1355 assert(isArgOperand(U) && "Arg operand # out of range!")((isArgOperand(U) && "Arg operand # out of range!") ?
static_cast<void> (0) : __assert_fail ("isArgOperand(U) && \"Arg operand # out of range!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1355, __PRETTY_FUNCTION__))
;
1356 return U - arg_begin();
1357 }
1358
1359 /// Given a value use iterator, return the arg operand number corresponding to
1360 /// it. Iterator must actually correspond to a data operand.
1361 unsigned getArgOperandNo(Value::const_user_iterator UI) const {
1362 return getArgOperandNo(&UI.getUse());
1363 }
1364
1365 /// Returns true if this CallSite passes the given Value* as an argument to
1366 /// the called function.
1367 bool hasArgument(const Value *V) const {
1368 return llvm::is_contained(args(), V);
1369 }
1370
1371 Value *getCalledOperand() const { return Op<CalledOperandOpEndIdx>(); }
1372
1373 const Use &getCalledOperandUse() const { return Op<CalledOperandOpEndIdx>(); }
1374 Use &getCalledOperandUse() { return Op<CalledOperandOpEndIdx>(); }
1375
1376 /// Returns the function called, or null if this is an
1377 /// indirect function invocation.
1378 Function *getCalledFunction() const {
1379 return dyn_cast_or_null<Function>(getCalledOperand());
41
Assuming null pointer is passed into cast
42
Returning null pointer, which participates in a condition later
1380 }
1381
1382 /// Return true if the callsite is an indirect call.
1383 bool isIndirectCall() const;
1384
1385 /// Determine whether the passed iterator points to the callee operand's Use.
1386 bool isCallee(Value::const_user_iterator UI) const {
1387 return isCallee(&UI.getUse());
1388 }
1389
1390 /// Determine whether this Use is the callee operand's Use.
1391 bool isCallee(const Use *U) const { return &getCalledOperandUse() == U; }
1392
1393 /// Helper to get the caller (the parent function).
1394 Function *getCaller();
1395 const Function *getCaller() const {
1396 return const_cast<CallBase *>(this)->getCaller();
1397 }
1398
1399 /// Tests if this call site must be tail call optimized. Only a CallInst can
1400 /// be tail call optimized.
1401 bool isMustTailCall() const;
1402
1403 /// Tests if this call site is marked as a tail call.
1404 bool isTailCall() const;
1405
1406 /// Returns the intrinsic ID of the intrinsic called or
1407 /// Intrinsic::not_intrinsic if the called function is not an intrinsic, or if
1408 /// this is an indirect call.
1409 Intrinsic::ID getIntrinsicID() const;
1410
1411 void setCalledOperand(Value *V) { Op<CalledOperandOpEndIdx>() = V; }
1412
1413 /// Sets the function called, including updating the function type.
1414 void setCalledFunction(Function *Fn) {
1415 setCalledFunction(Fn->getFunctionType(), Fn);
1416 }
1417
1418 /// Sets the function called, including updating the function type.
1419 void setCalledFunction(FunctionCallee Fn) {
1420 setCalledFunction(Fn.getFunctionType(), Fn.getCallee());
1421 }
1422
1423 /// Sets the function called, including updating to the specified function
1424 /// type.
1425 void setCalledFunction(FunctionType *FTy, Value *Fn) {
1426 this->FTy = FTy;
1427 assert(FTy == cast<FunctionType>(((FTy == cast<FunctionType>( cast<PointerType>(Fn
->getType())->getElementType())) ? static_cast<void>
(0) : __assert_fail ("FTy == cast<FunctionType>( cast<PointerType>(Fn->getType())->getElementType())"
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1428, __PRETTY_FUNCTION__))
1428 cast<PointerType>(Fn->getType())->getElementType()))((FTy == cast<FunctionType>( cast<PointerType>(Fn
->getType())->getElementType())) ? static_cast<void>
(0) : __assert_fail ("FTy == cast<FunctionType>( cast<PointerType>(Fn->getType())->getElementType())"
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1428, __PRETTY_FUNCTION__))
;
1429 // This function doesn't mutate the return type, only the function
1430 // type. Seems broken, but I'm just gonna stick an assert in for now.
1431 assert(getType() == FTy->getReturnType())((getType() == FTy->getReturnType()) ? static_cast<void
> (0) : __assert_fail ("getType() == FTy->getReturnType()"
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1431, __PRETTY_FUNCTION__))
;
1432 setCalledOperand(Fn);
1433 }
1434
1435 CallingConv::ID getCallingConv() const {
1436 return getSubclassData<CallingConvField>();
1437 }
1438
1439 void setCallingConv(CallingConv::ID CC) {
1440 setSubclassData<CallingConvField>(CC);
1441 }
1442
1443 /// Check if this call is an inline asm statement.
1444 bool isInlineAsm() const { return isa<InlineAsm>(getCalledOperand()); }
1445
1446 /// \name Attribute API
1447 ///
1448 /// These methods access and modify attributes on this call (including
1449 /// looking through to the attributes on the called function when necessary).
1450 ///@{
1451
1452 /// Return the parameter attributes for this call.
1453 ///
1454 AttributeList getAttributes() const { return Attrs; }
1455
1456 /// Set the parameter attributes for this call.
1457 ///
1458 void setAttributes(AttributeList A) { Attrs = A; }
1459
1460 /// Determine whether this call has the given attribute. If it does not
1461 /// then determine if the called function has the attribute, but only if
1462 /// the attribute is allowed for the call.
1463 bool hasFnAttr(Attribute::AttrKind Kind) const {
1464 assert(Kind != Attribute::NoBuiltin &&((Kind != Attribute::NoBuiltin && "Use CallBase::isNoBuiltin() to check for Attribute::NoBuiltin"
) ? static_cast<void> (0) : __assert_fail ("Kind != Attribute::NoBuiltin && \"Use CallBase::isNoBuiltin() to check for Attribute::NoBuiltin\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1465, __PRETTY_FUNCTION__))
1465 "Use CallBase::isNoBuiltin() to check for Attribute::NoBuiltin")((Kind != Attribute::NoBuiltin && "Use CallBase::isNoBuiltin() to check for Attribute::NoBuiltin"
) ? static_cast<void> (0) : __assert_fail ("Kind != Attribute::NoBuiltin && \"Use CallBase::isNoBuiltin() to check for Attribute::NoBuiltin\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1465, __PRETTY_FUNCTION__))
;
1466 return hasFnAttrImpl(Kind);
1467 }
1468
1469 /// Determine whether this call has the given attribute. If it does not
1470 /// then determine if the called function has the attribute, but only if
1471 /// the attribute is allowed for the call.
1472 bool hasFnAttr(StringRef Kind) const { return hasFnAttrImpl(Kind); }
1473
1474 /// adds the attribute to the list of attributes.
1475 void addAttribute(unsigned i, Attribute::AttrKind Kind) {
1476 AttributeList PAL = getAttributes();
1477 PAL = PAL.addAttribute(getContext(), i, Kind);
1478 setAttributes(PAL);
1479 }
1480
1481 /// adds the attribute to the list of attributes.
1482 void addAttribute(unsigned i, Attribute Attr) {
1483 AttributeList PAL = getAttributes();
1484 PAL = PAL.addAttribute(getContext(), i, Attr);
1485 setAttributes(PAL);
1486 }
1487
1488 /// Adds the attribute to the indicated argument
1489 void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind) {
1490 assert(ArgNo < getNumArgOperands() && "Out of bounds")((ArgNo < getNumArgOperands() && "Out of bounds") ?
static_cast<void> (0) : __assert_fail ("ArgNo < getNumArgOperands() && \"Out of bounds\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1490, __PRETTY_FUNCTION__))
;
1491 AttributeList PAL = getAttributes();
1492 PAL = PAL.addParamAttribute(getContext(), ArgNo, Kind);
1493 setAttributes(PAL);
1494 }
1495
1496 /// Adds the attribute to the indicated argument
1497 void addParamAttr(unsigned ArgNo, Attribute Attr) {
1498 assert(ArgNo < getNumArgOperands() && "Out of bounds")((ArgNo < getNumArgOperands() && "Out of bounds") ?
static_cast<void> (0) : __assert_fail ("ArgNo < getNumArgOperands() && \"Out of bounds\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1498, __PRETTY_FUNCTION__))
;
1499 AttributeList PAL = getAttributes();
1500 PAL = PAL.addParamAttribute(getContext(), ArgNo, Attr);
1501 setAttributes(PAL);
1502 }
1503
1504 /// removes the attribute from the list of attributes.
1505 void removeAttribute(unsigned i, Attribute::AttrKind Kind) {
1506 AttributeList PAL = getAttributes();
1507 PAL = PAL.removeAttribute(getContext(), i, Kind);
1508 setAttributes(PAL);
1509 }
1510
1511 /// removes the attribute from the list of attributes.
1512 void removeAttribute(unsigned i, StringRef Kind) {
1513 AttributeList PAL = getAttributes();
1514 PAL = PAL.removeAttribute(getContext(), i, Kind);
1515 setAttributes(PAL);
1516 }
1517
1518 void removeAttributes(unsigned i, const AttrBuilder &Attrs) {
1519 AttributeList PAL = getAttributes();
1520 PAL = PAL.removeAttributes(getContext(), i, Attrs);
1521 setAttributes(PAL);
1522 }
1523
1524 /// Removes the attribute from the given argument
1525 void removeParamAttr(unsigned ArgNo, Attribute::AttrKind Kind) {
1526 assert(ArgNo < getNumArgOperands() && "Out of bounds")((ArgNo < getNumArgOperands() && "Out of bounds") ?
static_cast<void> (0) : __assert_fail ("ArgNo < getNumArgOperands() && \"Out of bounds\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1526, __PRETTY_FUNCTION__))
;
1527 AttributeList PAL = getAttributes();
1528 PAL = PAL.removeParamAttribute(getContext(), ArgNo, Kind);
1529 setAttributes(PAL);
1530 }
1531
1532 /// Removes the attribute from the given argument
1533 void removeParamAttr(unsigned ArgNo, StringRef Kind) {
1534 assert(ArgNo < getNumArgOperands() && "Out of bounds")((ArgNo < getNumArgOperands() && "Out of bounds") ?
static_cast<void> (0) : __assert_fail ("ArgNo < getNumArgOperands() && \"Out of bounds\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1534, __PRETTY_FUNCTION__))
;
1535 AttributeList PAL = getAttributes();
1536 PAL = PAL.removeParamAttribute(getContext(), ArgNo, Kind);
1537 setAttributes(PAL);
1538 }
1539
1540 /// adds the dereferenceable attribute to the list of attributes.
1541 void addDereferenceableAttr(unsigned i, uint64_t Bytes) {
1542 AttributeList PAL = getAttributes();
1543 PAL = PAL.addDereferenceableAttr(getContext(), i, Bytes);
1544 setAttributes(PAL);
1545 }
1546
1547 /// adds the dereferenceable_or_null attribute to the list of
1548 /// attributes.
1549 void addDereferenceableOrNullAttr(unsigned i, uint64_t Bytes) {
1550 AttributeList PAL = getAttributes();
1551 PAL = PAL.addDereferenceableOrNullAttr(getContext(), i, Bytes);
1552 setAttributes(PAL);
1553 }
1554
1555 /// Determine whether the return value has the given attribute.
1556 bool hasRetAttr(Attribute::AttrKind Kind) const {
1557 return hasRetAttrImpl(Kind);
1558 }
1559 /// Determine whether the return value has the given attribute.
1560 bool hasRetAttr(StringRef Kind) const { return hasRetAttrImpl(Kind); }
1561
1562 /// Determine whether the argument or parameter has the given attribute.
1563 bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const;
1564
1565 /// Get the attribute of a given kind at a position.
1566 Attribute getAttribute(unsigned i, Attribute::AttrKind Kind) const {
1567 return getAttributes().getAttribute(i, Kind);
1568 }
1569
1570 /// Get the attribute of a given kind at a position.
1571 Attribute getAttribute(unsigned i, StringRef Kind) const {
1572 return getAttributes().getAttribute(i, Kind);
1573 }
1574
1575 /// Get the attribute of a given kind from a given arg
1576 Attribute getParamAttr(unsigned ArgNo, Attribute::AttrKind Kind) const {
1577 assert(ArgNo < getNumArgOperands() && "Out of bounds")((ArgNo < getNumArgOperands() && "Out of bounds") ?
static_cast<void> (0) : __assert_fail ("ArgNo < getNumArgOperands() && \"Out of bounds\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1577, __PRETTY_FUNCTION__))
;
1578 return getAttributes().getParamAttr(ArgNo, Kind);
1579 }
1580
1581 /// Get the attribute of a given kind from a given arg
1582 Attribute getParamAttr(unsigned ArgNo, StringRef Kind) const {
1583 assert(ArgNo < getNumArgOperands() && "Out of bounds")((ArgNo < getNumArgOperands() && "Out of bounds") ?
static_cast<void> (0) : __assert_fail ("ArgNo < getNumArgOperands() && \"Out of bounds\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1583, __PRETTY_FUNCTION__))
;
1584 return getAttributes().getParamAttr(ArgNo, Kind);
1585 }
1586
1587 /// Return true if the data operand at index \p i has the attribute \p
1588 /// A.
1589 ///
1590 /// Data operands include call arguments and values used in operand bundles,
1591 /// but does not include the callee operand. This routine dispatches to the
1592 /// underlying AttributeList or the OperandBundleUser as appropriate.
1593 ///
1594 /// The index \p i is interpreted as
1595 ///
1596 /// \p i == Attribute::ReturnIndex -> the return value
1597 /// \p i in [1, arg_size + 1) -> argument number (\p i - 1)
1598 /// \p i in [arg_size + 1, data_operand_size + 1) -> bundle operand at index
1599 /// (\p i - 1) in the operand list.
1600 bool dataOperandHasImpliedAttr(unsigned i, Attribute::AttrKind Kind) const {
1601 // Note that we have to add one because `i` isn't zero-indexed.
1602 assert(i < (getNumArgOperands() + getNumTotalBundleOperands() + 1) &&((i < (getNumArgOperands() + getNumTotalBundleOperands() +
1) && "Data operand index out of bounds!") ? static_cast
<void> (0) : __assert_fail ("i < (getNumArgOperands() + getNumTotalBundleOperands() + 1) && \"Data operand index out of bounds!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1603, __PRETTY_FUNCTION__))
1603 "Data operand index out of bounds!")((i < (getNumArgOperands() + getNumTotalBundleOperands() +
1) && "Data operand index out of bounds!") ? static_cast
<void> (0) : __assert_fail ("i < (getNumArgOperands() + getNumTotalBundleOperands() + 1) && \"Data operand index out of bounds!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1603, __PRETTY_FUNCTION__))
;
1604
1605 // The attribute A can either be directly specified, if the operand in
1606 // question is a call argument; or be indirectly implied by the kind of its
1607 // containing operand bundle, if the operand is a bundle operand.
1608
1609 if (i == AttributeList::ReturnIndex)
1610 return hasRetAttr(Kind);
1611
1612 // FIXME: Avoid these i - 1 calculations and update the API to use
1613 // zero-based indices.
1614 if (i < (getNumArgOperands() + 1))
1615 return paramHasAttr(i - 1, Kind);
1616
1617 assert(hasOperandBundles() && i >= (getBundleOperandsStartIndex() + 1) &&((hasOperandBundles() && i >= (getBundleOperandsStartIndex
() + 1) && "Must be either a call argument or an operand bundle!"
) ? static_cast<void> (0) : __assert_fail ("hasOperandBundles() && i >= (getBundleOperandsStartIndex() + 1) && \"Must be either a call argument or an operand bundle!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1618, __PRETTY_FUNCTION__))
1618 "Must be either a call argument or an operand bundle!")((hasOperandBundles() && i >= (getBundleOperandsStartIndex
() + 1) && "Must be either a call argument or an operand bundle!"
) ? static_cast<void> (0) : __assert_fail ("hasOperandBundles() && i >= (getBundleOperandsStartIndex() + 1) && \"Must be either a call argument or an operand bundle!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1618, __PRETTY_FUNCTION__))
;
1619 return bundleOperandHasAttr(i - 1, Kind);
1620 }
1621
1622 /// Determine whether this data operand is not captured.
1623 // FIXME: Once this API is no longer duplicated in `CallSite`, rename this to
1624 // better indicate that this may return a conservative answer.
1625 bool doesNotCapture(unsigned OpNo) const {
1626 return dataOperandHasImpliedAttr(OpNo + 1, Attribute::NoCapture);
1627 }
1628
1629 /// Determine whether this argument is passed by value.
1630 bool isByValArgument(unsigned ArgNo) const {
1631 return paramHasAttr(ArgNo, Attribute::ByVal);
1632 }
1633
1634 /// Determine whether this argument is passed in an alloca.
1635 bool isInAllocaArgument(unsigned ArgNo) const {
1636 return paramHasAttr(ArgNo, Attribute::InAlloca);
1637 }
1638
1639 /// Determine whether this argument is passed by value, in an alloca, or is
1640 /// preallocated.
1641 bool isPassPointeeByValueArgument(unsigned ArgNo) const {
1642 return paramHasAttr(ArgNo, Attribute::ByVal) ||
1643 paramHasAttr(ArgNo, Attribute::InAlloca) ||
1644 paramHasAttr(ArgNo, Attribute::Preallocated);
1645 }
1646
1647 /// Determine if there are is an inalloca argument. Only the last argument can
1648 /// have the inalloca attribute.
1649 bool hasInAllocaArgument() const {
1650 return !arg_empty() && paramHasAttr(arg_size() - 1, Attribute::InAlloca);
1651 }
1652
1653 // FIXME: Once this API is no longer duplicated in `CallSite`, rename this to
1654 // better indicate that this may return a conservative answer.
1655 bool doesNotAccessMemory(unsigned OpNo) const {
1656 return dataOperandHasImpliedAttr(OpNo + 1, Attribute::ReadNone);
1657 }
1658
1659 // FIXME: Once this API is no longer duplicated in `CallSite`, rename this to
1660 // better indicate that this may return a conservative answer.
1661 bool onlyReadsMemory(unsigned OpNo) const {
1662 return dataOperandHasImpliedAttr(OpNo + 1, Attribute::ReadOnly) ||
1663 dataOperandHasImpliedAttr(OpNo + 1, Attribute::ReadNone);
1664 }
1665
1666 // FIXME: Once this API is no longer duplicated in `CallSite`, rename this to
1667 // better indicate that this may return a conservative answer.
1668 bool doesNotReadMemory(unsigned OpNo) const {
1669 return dataOperandHasImpliedAttr(OpNo + 1, Attribute::WriteOnly) ||
1670 dataOperandHasImpliedAttr(OpNo + 1, Attribute::ReadNone);
1671 }
1672
1673 LLVM_ATTRIBUTE_DEPRECATED(unsigned getRetAlignment() const,[[deprecated("Use getRetAlign() instead")]] unsigned getRetAlignment
() const
1674 "Use getRetAlign() instead")[[deprecated("Use getRetAlign() instead")]] unsigned getRetAlignment
() const
{
1675 if (const auto MA = Attrs.getRetAlignment())
1676 return MA->value();
1677 return 0;
1678 }
1679
1680 /// Extract the alignment of the return value.
1681 MaybeAlign getRetAlign() const { return Attrs.getRetAlignment(); }
1682
1683 /// Extract the alignment for a call or parameter (0=unknown).
1684 LLVM_ATTRIBUTE_DEPRECATED(unsigned getParamAlignment(unsigned ArgNo) const,[[deprecated("Use getParamAlign() instead")]] unsigned getParamAlignment
(unsigned ArgNo) const
1685 "Use getParamAlign() instead")[[deprecated("Use getParamAlign() instead")]] unsigned getParamAlignment
(unsigned ArgNo) const
{
1686 if (const auto MA = Attrs.getParamAlignment(ArgNo))
1687 return MA->value();
1688 return 0;
1689 }
1690
1691 /// Extract the alignment for a call or parameter (0=unknown).
1692 MaybeAlign getParamAlign(unsigned ArgNo) const {
1693 return Attrs.getParamAlignment(ArgNo);
1694 }
1695
1696 /// Extract the byval type for a call or parameter.
1697 Type *getParamByValType(unsigned ArgNo) const {
1698 Type *Ty = Attrs.getParamByValType(ArgNo);
1699 return Ty ? Ty : getArgOperand(ArgNo)->getType()->getPointerElementType();
1700 }
1701
1702 /// Extract the preallocated type for a call or parameter.
1703 Type *getParamPreallocatedType(unsigned ArgNo) const {
1704 Type *Ty = Attrs.getParamPreallocatedType(ArgNo);
1705 return Ty ? Ty : getArgOperand(ArgNo)->getType()->getPointerElementType();
1706 }
1707
1708 /// Extract the number of dereferenceable bytes for a call or
1709 /// parameter (0=unknown).
1710 uint64_t getDereferenceableBytes(unsigned i) const {
1711 return Attrs.getDereferenceableBytes(i);
1712 }
1713
1714 /// Extract the number of dereferenceable_or_null bytes for a call or
1715 /// parameter (0=unknown).
1716 uint64_t getDereferenceableOrNullBytes(unsigned i) const {
1717 return Attrs.getDereferenceableOrNullBytes(i);
1718 }
1719
1720 /// Return true if the return value is known to be not null.
1721 /// This may be because it has the nonnull attribute, or because at least
1722 /// one byte is dereferenceable and the pointer is in addrspace(0).
1723 bool isReturnNonNull() const;
1724
1725 /// Determine if the return value is marked with NoAlias attribute.
1726 bool returnDoesNotAlias() const {
1727 return Attrs.hasAttribute(AttributeList::ReturnIndex, Attribute::NoAlias);
1728 }
1729
1730 /// If one of the arguments has the 'returned' attribute, returns its
1731 /// operand value. Otherwise, return nullptr.
1732 Value *getReturnedArgOperand() const;
1733
1734 /// Return true if the call should not be treated as a call to a
1735 /// builtin.
1736 bool isNoBuiltin() const {
1737 return hasFnAttrImpl(Attribute::NoBuiltin) &&
1738 !hasFnAttrImpl(Attribute::Builtin);
1739 }
1740
1741 /// Determine if the call requires strict floating point semantics.
1742 bool isStrictFP() const { return hasFnAttr(Attribute::StrictFP); }
1743
1744 /// Return true if the call should not be inlined.
1745 bool isNoInline() const { return hasFnAttr(Attribute::NoInline); }
1746 void setIsNoInline() {
1747 addAttribute(AttributeList::FunctionIndex, Attribute::NoInline);
1748 }
1749 /// Determine if the call does not access memory.
1750 bool doesNotAccessMemory() const { return hasFnAttr(Attribute::ReadNone); }
1751 void setDoesNotAccessMemory() {
1752 addAttribute(AttributeList::FunctionIndex, Attribute::ReadNone);
1753 }
1754
1755 /// Determine if the call does not access or only reads memory.
1756 bool onlyReadsMemory() const {
1757 return doesNotAccessMemory() || hasFnAttr(Attribute::ReadOnly);
1758 }
1759 void setOnlyReadsMemory() {
1760 addAttribute(AttributeList::FunctionIndex, Attribute::ReadOnly);
1761 }
1762
1763 /// Determine if the call does not access or only writes memory.
1764 bool doesNotReadMemory() const {
1765 return doesNotAccessMemory() || hasFnAttr(Attribute::WriteOnly);
1766 }
1767 void setDoesNotReadMemory() {
1768 addAttribute(AttributeList::FunctionIndex, Attribute::WriteOnly);
1769 }
1770
1771 /// Determine if the call can access memmory only using pointers based
1772 /// on its arguments.
1773 bool onlyAccessesArgMemory() const {
1774 return hasFnAttr(Attribute::ArgMemOnly);
1775 }
1776 void setOnlyAccessesArgMemory() {
1777 addAttribute(AttributeList::FunctionIndex, Attribute::ArgMemOnly);
1778 }
1779
1780 /// Determine if the function may only access memory that is
1781 /// inaccessible from the IR.
1782 bool onlyAccessesInaccessibleMemory() const {
1783 return hasFnAttr(Attribute::InaccessibleMemOnly);
1784 }
1785 void setOnlyAccessesInaccessibleMemory() {
1786 addAttribute(AttributeList::FunctionIndex, Attribute::InaccessibleMemOnly);
1787 }
1788
1789 /// Determine if the function may only access memory that is
1790 /// either inaccessible from the IR or pointed to by its arguments.
1791 bool onlyAccessesInaccessibleMemOrArgMem() const {
1792 return hasFnAttr(Attribute::InaccessibleMemOrArgMemOnly);
1793 }
1794 void setOnlyAccessesInaccessibleMemOrArgMem() {
1795 addAttribute(AttributeList::FunctionIndex,
1796 Attribute::InaccessibleMemOrArgMemOnly);
1797 }
1798 /// Determine if the call cannot return.
1799 bool doesNotReturn() const { return hasFnAttr(Attribute::NoReturn); }
1800 void setDoesNotReturn() {
1801 addAttribute(AttributeList::FunctionIndex, Attribute::NoReturn);
1802 }
1803
1804 /// Determine if the call should not perform indirect branch tracking.
1805 bool doesNoCfCheck() const { return hasFnAttr(Attribute::NoCfCheck); }
1806
1807 /// Determine if the call cannot unwind.
1808 bool doesNotThrow() const { return hasFnAttr(Attribute::NoUnwind); }
1809 void setDoesNotThrow() {
1810 addAttribute(AttributeList::FunctionIndex, Attribute::NoUnwind);
1811 }
1812
1813 /// Determine if the invoke cannot be duplicated.
1814 bool cannotDuplicate() const { return hasFnAttr(Attribute::NoDuplicate); }
1815 void setCannotDuplicate() {
1816 addAttribute(AttributeList::FunctionIndex, Attribute::NoDuplicate);
1817 }
1818
1819 /// Determine if the call cannot be tail merged.
1820 bool cannotMerge() const { return hasFnAttr(Attribute::NoMerge); }
1821 void setCannotMerge() {
1822 addAttribute(AttributeList::FunctionIndex, Attribute::NoMerge);
1823 }
1824
1825 /// Determine if the invoke is convergent
1826 bool isConvergent() const { return hasFnAttr(Attribute::Convergent); }
1827 void setConvergent() {
1828 addAttribute(AttributeList::FunctionIndex, Attribute::Convergent);
1829 }
1830 void setNotConvergent() {
1831 removeAttribute(AttributeList::FunctionIndex, Attribute::Convergent);
1832 }
1833
1834 /// Determine if the call returns a structure through first
1835 /// pointer argument.
1836 bool hasStructRetAttr() const {
1837 if (getNumArgOperands() == 0)
1838 return false;
1839
1840 // Be friendly and also check the callee.
1841 return paramHasAttr(0, Attribute::StructRet);
1842 }
1843
1844 /// Determine if any call argument is an aggregate passed by value.
1845 bool hasByValArgument() const {
1846 return Attrs.hasAttrSomewhere(Attribute::ByVal);
1847 }
1848
1849 ///@{
1850 // End of attribute API.
1851
1852 /// \name Operand Bundle API
1853 ///
1854 /// This group of methods provides the API to access and manipulate operand
1855 /// bundles on this call.
1856 /// @{
1857
1858 /// Return the number of operand bundles associated with this User.
1859 unsigned getNumOperandBundles() const {
1860 return std::distance(bundle_op_info_begin(), bundle_op_info_end());
1861 }
1862
1863 /// Return true if this User has any operand bundles.
1864 bool hasOperandBundles() const { return getNumOperandBundles() != 0; }
1865
1866 /// Return the index of the first bundle operand in the Use array.
1867 unsigned getBundleOperandsStartIndex() const {
1868 assert(hasOperandBundles() && "Don't call otherwise!")((hasOperandBundles() && "Don't call otherwise!") ? static_cast
<void> (0) : __assert_fail ("hasOperandBundles() && \"Don't call otherwise!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1868, __PRETTY_FUNCTION__))
;
1869 return bundle_op_info_begin()->Begin;
1870 }
1871
1872 /// Return the index of the last bundle operand in the Use array.
1873 unsigned getBundleOperandsEndIndex() const {
1874 assert(hasOperandBundles() && "Don't call otherwise!")((hasOperandBundles() && "Don't call otherwise!") ? static_cast
<void> (0) : __assert_fail ("hasOperandBundles() && \"Don't call otherwise!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1874, __PRETTY_FUNCTION__))
;
1875 return bundle_op_info_end()[-1].End;
1876 }
1877
1878 /// Return true if the operand at index \p Idx is a bundle operand.
1879 bool isBundleOperand(unsigned Idx) const {
1880 return hasOperandBundles() && Idx >= getBundleOperandsStartIndex() &&
1881 Idx < getBundleOperandsEndIndex();
1882 }
1883
1884 /// Returns true if the use is a bundle operand.
1885 bool isBundleOperand(const Use *U) const {
1886 assert(this == U->getUser() &&((this == U->getUser() && "Only valid to query with a use of this instruction!"
) ? static_cast<void> (0) : __assert_fail ("this == U->getUser() && \"Only valid to query with a use of this instruction!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1887, __PRETTY_FUNCTION__))
1887 "Only valid to query with a use of this instruction!")((this == U->getUser() && "Only valid to query with a use of this instruction!"
) ? static_cast<void> (0) : __assert_fail ("this == U->getUser() && \"Only valid to query with a use of this instruction!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1887, __PRETTY_FUNCTION__))
;
1888 return hasOperandBundles() && isBundleOperand(U - op_begin());
1889 }
1890 bool isBundleOperand(Value::const_user_iterator UI) const {
1891 return isBundleOperand(&UI.getUse());
1892 }
1893
1894 /// Return the total number operands (not operand bundles) used by
1895 /// every operand bundle in this OperandBundleUser.
1896 unsigned getNumTotalBundleOperands() const {
1897 if (!hasOperandBundles())
1898 return 0;
1899
1900 unsigned Begin = getBundleOperandsStartIndex();
1901 unsigned End = getBundleOperandsEndIndex();
1902
1903 assert(Begin <= End && "Should be!")((Begin <= End && "Should be!") ? static_cast<void
> (0) : __assert_fail ("Begin <= End && \"Should be!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1903, __PRETTY_FUNCTION__))
;
1904 return End - Begin;
1905 }
1906
1907 /// Return the operand bundle at a specific index.
1908 OperandBundleUse getOperandBundleAt(unsigned Index) const {
1909 assert(Index < getNumOperandBundles() && "Index out of bounds!")((Index < getNumOperandBundles() && "Index out of bounds!"
) ? static_cast<void> (0) : __assert_fail ("Index < getNumOperandBundles() && \"Index out of bounds!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1909, __PRETTY_FUNCTION__))
;
1910 return operandBundleFromBundleOpInfo(*(bundle_op_info_begin() + Index));
1911 }
1912
1913 /// Return the number of operand bundles with the tag Name attached to
1914 /// this instruction.
1915 unsigned countOperandBundlesOfType(StringRef Name) const {
1916 unsigned Count = 0;
1917 for (unsigned i = 0, e = getNumOperandBundles(); i != e; ++i)
1918 if (getOperandBundleAt(i).getTagName() == Name)
1919 Count++;
1920
1921 return Count;
1922 }
1923
1924 /// Return the number of operand bundles with the tag ID attached to
1925 /// this instruction.
1926 unsigned countOperandBundlesOfType(uint32_t ID) const {
1927 unsigned Count = 0;
1928 for (unsigned i = 0, e = getNumOperandBundles(); i != e; ++i)
1929 if (getOperandBundleAt(i).getTagID() == ID)
1930 Count++;
1931
1932 return Count;
1933 }
1934
1935 /// Return an operand bundle by name, if present.
1936 ///
1937 /// It is an error to call this for operand bundle types that may have
1938 /// multiple instances of them on the same instruction.
1939 Optional<OperandBundleUse> getOperandBundle(StringRef Name) const {
1940 assert(countOperandBundlesOfType(Name) < 2 && "Precondition violated!")((countOperandBundlesOfType(Name) < 2 && "Precondition violated!"
) ? static_cast<void> (0) : __assert_fail ("countOperandBundlesOfType(Name) < 2 && \"Precondition violated!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1940, __PRETTY_FUNCTION__))
;
1941
1942 for (unsigned i = 0, e = getNumOperandBundles(); i != e; ++i) {
1943 OperandBundleUse U = getOperandBundleAt(i);
1944 if (U.getTagName() == Name)
1945 return U;
1946 }
1947
1948 return None;
1949 }
1950
1951 /// Return an operand bundle by tag ID, if present.
1952 ///
1953 /// It is an error to call this for operand bundle types that may have
1954 /// multiple instances of them on the same instruction.
1955 Optional<OperandBundleUse> getOperandBundle(uint32_t ID) const {
1956 assert(countOperandBundlesOfType(ID) < 2 && "Precondition violated!")((countOperandBundlesOfType(ID) < 2 && "Precondition violated!"
) ? static_cast<void> (0) : __assert_fail ("countOperandBundlesOfType(ID) < 2 && \"Precondition violated!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 1956, __PRETTY_FUNCTION__))
;
1957
1958 for (unsigned i = 0, e = getNumOperandBundles(); i != e; ++i) {
1959 OperandBundleUse U = getOperandBundleAt(i);
1960 if (U.getTagID() == ID)
1961 return U;
1962 }
1963
1964 return None;
1965 }
1966
1967 /// Return the list of operand bundles attached to this instruction as
1968 /// a vector of OperandBundleDefs.
1969 ///
1970 /// This function copies the OperandBundeUse instances associated with this
1971 /// OperandBundleUser to a vector of OperandBundleDefs. Note:
1972 /// OperandBundeUses and OperandBundleDefs are non-trivially *different*
1973 /// representations of operand bundles (see documentation above).
1974 void getOperandBundlesAsDefs(SmallVectorImpl<OperandBundleDef> &Defs) const;
1975
1976 /// Return the operand bundle for the operand at index OpIdx.
1977 ///
1978 /// It is an error to call this with an OpIdx that does not correspond to an
1979 /// bundle operand.
1980 OperandBundleUse getOperandBundleForOperand(unsigned OpIdx) const {
1981 return operandBundleFromBundleOpInfo(getBundleOpInfoForOperand(OpIdx));
1982 }
1983
1984 /// Return true if this operand bundle user has operand bundles that
1985 /// may read from the heap.
1986 bool hasReadingOperandBundles() const {
1987 // Implementation note: this is a conservative implementation of operand
1988 // bundle semantics, where *any* operand bundle forces a callsite to be at
1989 // least readonly.
1990 return hasOperandBundles();
1991 }
1992
1993 /// Return true if this operand bundle user has operand bundles that
1994 /// may write to the heap.
1995 bool hasClobberingOperandBundles() const {
1996 for (auto &BOI : bundle_op_infos()) {
1997 if (BOI.Tag->second == LLVMContext::OB_deopt ||
1998 BOI.Tag->second == LLVMContext::OB_funclet)
1999 continue;
2000
2001 // This instruction has an operand bundle that is not known to us.
2002 // Assume the worst.
2003 return true;
2004 }
2005
2006 return false;
2007 }
2008
2009 /// Return true if the bundle operand at index \p OpIdx has the
2010 /// attribute \p A.
2011 bool bundleOperandHasAttr(unsigned OpIdx, Attribute::AttrKind A) const {
2012 auto &BOI = getBundleOpInfoForOperand(OpIdx);
2013 auto OBU = operandBundleFromBundleOpInfo(BOI);
2014 return OBU.operandHasAttr(OpIdx - BOI.Begin, A);
2015 }
2016
2017 /// Return true if \p Other has the same sequence of operand bundle
2018 /// tags with the same number of operands on each one of them as this
2019 /// OperandBundleUser.
2020 bool hasIdenticalOperandBundleSchema(const CallBase &Other) const {
2021 if (getNumOperandBundles() != Other.getNumOperandBundles())
2022 return false;
2023
2024 return std::equal(bundle_op_info_begin(), bundle_op_info_end(),
2025 Other.bundle_op_info_begin());
2026 }
2027
2028 /// Return true if this operand bundle user contains operand bundles
2029 /// with tags other than those specified in \p IDs.
2030 bool hasOperandBundlesOtherThan(ArrayRef<uint32_t> IDs) const {
2031 for (unsigned i = 0, e = getNumOperandBundles(); i != e; ++i) {
2032 uint32_t ID = getOperandBundleAt(i).getTagID();
2033 if (!is_contained(IDs, ID))
2034 return true;
2035 }
2036 return false;
2037 }
2038
2039 /// Is the function attribute S disallowed by some operand bundle on
2040 /// this operand bundle user?
2041 bool isFnAttrDisallowedByOpBundle(StringRef S) const {
2042 // Operand bundles only possibly disallow readnone, readonly and argmemonly
2043 // attributes. All String attributes are fine.
2044 return false;
2045 }
2046
2047 /// Is the function attribute A disallowed by some operand bundle on
2048 /// this operand bundle user?
2049 bool isFnAttrDisallowedByOpBundle(Attribute::AttrKind A) const {
2050 switch (A) {
2051 default:
2052 return false;
2053
2054 case Attribute::InaccessibleMemOrArgMemOnly:
2055 return hasReadingOperandBundles();
2056
2057 case Attribute::InaccessibleMemOnly:
2058 return hasReadingOperandBundles();
2059
2060 case Attribute::ArgMemOnly:
2061 return hasReadingOperandBundles();
2062
2063 case Attribute::ReadNone:
2064 return hasReadingOperandBundles();
2065
2066 case Attribute::ReadOnly:
2067 return hasClobberingOperandBundles();
2068 }
2069
2070 llvm_unreachable("switch has a default case!")::llvm::llvm_unreachable_internal("switch has a default case!"
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 2070)
;
2071 }
2072
2073 /// Used to keep track of an operand bundle. See the main comment on
2074 /// OperandBundleUser above.
2075 struct BundleOpInfo {
2076 /// The operand bundle tag, interned by
2077 /// LLVMContextImpl::getOrInsertBundleTag.
2078 StringMapEntry<uint32_t> *Tag;
2079
2080 /// The index in the Use& vector where operands for this operand
2081 /// bundle starts.
2082 uint32_t Begin;
2083
2084 /// The index in the Use& vector where operands for this operand
2085 /// bundle ends.
2086 uint32_t End;
2087
2088 bool operator==(const BundleOpInfo &Other) const {
2089 return Tag == Other.Tag && Begin == Other.Begin && End == Other.End;
2090 }
2091 };
2092
2093 /// Simple helper function to map a BundleOpInfo to an
2094 /// OperandBundleUse.
2095 OperandBundleUse
2096 operandBundleFromBundleOpInfo(const BundleOpInfo &BOI) const {
2097 auto begin = op_begin();
2098 ArrayRef<Use> Inputs(begin + BOI.Begin, begin + BOI.End);
2099 return OperandBundleUse(BOI.Tag, Inputs);
2100 }
2101
2102 using bundle_op_iterator = BundleOpInfo *;
2103 using const_bundle_op_iterator = const BundleOpInfo *;
2104
2105 /// Return the start of the list of BundleOpInfo instances associated
2106 /// with this OperandBundleUser.
2107 ///
2108 /// OperandBundleUser uses the descriptor area co-allocated with the host User
2109 /// to store some meta information about which operands are "normal" operands,
2110 /// and which ones belong to some operand bundle.
2111 ///
2112 /// The layout of an operand bundle user is
2113 ///
2114 /// +-----------uint32_t End-------------------------------------+
2115 /// | |
2116 /// | +--------uint32_t Begin--------------------+ |
2117 /// | | | |
2118 /// ^ ^ v v
2119 /// |------|------|----|----|----|----|----|---------|----|---------|----|-----
2120 /// | BOI0 | BOI1 | .. | DU | U0 | U1 | .. | BOI0_U0 | .. | BOI1_U0 | .. | Un
2121 /// |------|------|----|----|----|----|----|---------|----|---------|----|-----
2122 /// v v ^ ^
2123 /// | | | |
2124 /// | +--------uint32_t Begin------------+ |
2125 /// | |
2126 /// +-----------uint32_t End-----------------------------+
2127 ///
2128 ///
2129 /// BOI0, BOI1 ... are descriptions of operand bundles in this User's use
2130 /// list. These descriptions are installed and managed by this class, and
2131 /// they're all instances of OperandBundleUser<T>::BundleOpInfo.
2132 ///
2133 /// DU is an additional descriptor installed by User's 'operator new' to keep
2134 /// track of the 'BOI0 ... BOIN' co-allocation. OperandBundleUser does not
2135 /// access or modify DU in any way, it's an implementation detail private to
2136 /// User.
2137 ///
2138 /// The regular Use& vector for the User starts at U0. The operand bundle
2139 /// uses are part of the Use& vector, just like normal uses. In the diagram
2140 /// above, the operand bundle uses start at BOI0_U0. Each instance of
2141 /// BundleOpInfo has information about a contiguous set of uses constituting
2142 /// an operand bundle, and the total set of operand bundle uses themselves
2143 /// form a contiguous set of uses (i.e. there are no gaps between uses
2144 /// corresponding to individual operand bundles).
2145 ///
2146 /// This class does not know the location of the set of operand bundle uses
2147 /// within the use list -- that is decided by the User using this class via
2148 /// the BeginIdx argument in populateBundleOperandInfos.
2149 ///
2150 /// Currently operand bundle users with hung-off operands are not supported.
2151 bundle_op_iterator bundle_op_info_begin() {
2152 if (!hasDescriptor())
2153 return nullptr;
2154
2155 uint8_t *BytesBegin = getDescriptor().begin();
2156 return reinterpret_cast<bundle_op_iterator>(BytesBegin);
2157 }
2158
2159 /// Return the start of the list of BundleOpInfo instances associated
2160 /// with this OperandBundleUser.
2161 const_bundle_op_iterator bundle_op_info_begin() const {
2162 auto *NonConstThis = const_cast<CallBase *>(this);
2163 return NonConstThis->bundle_op_info_begin();
2164 }
2165
2166 /// Return the end of the list of BundleOpInfo instances associated
2167 /// with this OperandBundleUser.
2168 bundle_op_iterator bundle_op_info_end() {
2169 if (!hasDescriptor())
2170 return nullptr;
2171
2172 uint8_t *BytesEnd = getDescriptor().end();
2173 return reinterpret_cast<bundle_op_iterator>(BytesEnd);
2174 }
2175
2176 /// Return the end of the list of BundleOpInfo instances associated
2177 /// with this OperandBundleUser.
2178 const_bundle_op_iterator bundle_op_info_end() const {
2179 auto *NonConstThis = const_cast<CallBase *>(this);
2180 return NonConstThis->bundle_op_info_end();
2181 }
2182
2183 /// Return the range [\p bundle_op_info_begin, \p bundle_op_info_end).
2184 iterator_range<bundle_op_iterator> bundle_op_infos() {
2185 return make_range(bundle_op_info_begin(), bundle_op_info_end());
2186 }
2187
2188 /// Return the range [\p bundle_op_info_begin, \p bundle_op_info_end).
2189 iterator_range<const_bundle_op_iterator> bundle_op_infos() const {
2190 return make_range(bundle_op_info_begin(), bundle_op_info_end());
2191 }
2192
2193 /// Populate the BundleOpInfo instances and the Use& vector from \p
2194 /// Bundles. Return the op_iterator pointing to the Use& one past the last
2195 /// last bundle operand use.
2196 ///
2197 /// Each \p OperandBundleDef instance is tracked by a OperandBundleInfo
2198 /// instance allocated in this User's descriptor.
2199 op_iterator populateBundleOperandInfos(ArrayRef<OperandBundleDef> Bundles,
2200 const unsigned BeginIndex);
2201
2202public:
2203 /// Return the BundleOpInfo for the operand at index OpIdx.
2204 ///
2205 /// It is an error to call this with an OpIdx that does not correspond to an
2206 /// bundle operand.
2207 BundleOpInfo &getBundleOpInfoForOperand(unsigned OpIdx);
2208 const BundleOpInfo &getBundleOpInfoForOperand(unsigned OpIdx) const {
2209 return const_cast<CallBase *>(this)->getBundleOpInfoForOperand(OpIdx);
2210 }
2211
2212protected:
2213 /// Return the total number of values used in \p Bundles.
2214 static unsigned CountBundleInputs(ArrayRef<OperandBundleDef> Bundles) {
2215 unsigned Total = 0;
2216 for (auto &B : Bundles)
2217 Total += B.input_size();
2218 return Total;
2219 }
2220
2221 /// @}
2222 // End of operand bundle API.
2223
2224private:
2225 bool hasFnAttrOnCalledFunction(Attribute::AttrKind Kind) const;
2226 bool hasFnAttrOnCalledFunction(StringRef Kind) const;
2227
2228 template <typename AttrKind> bool hasFnAttrImpl(AttrKind Kind) const {
2229 if (Attrs.hasFnAttribute(Kind))
2230 return true;
2231
2232 // Operand bundles override attributes on the called function, but don't
2233 // override attributes directly present on the call instruction.
2234 if (isFnAttrDisallowedByOpBundle(Kind))
2235 return false;
2236
2237 return hasFnAttrOnCalledFunction(Kind);
2238 }
2239
2240 /// Determine whether the return value has the given attribute. Supports
2241 /// Attribute::AttrKind and StringRef as \p AttrKind types.
2242 template <typename AttrKind> bool hasRetAttrImpl(AttrKind Kind) const {
2243 if (Attrs.hasAttribute(AttributeList::ReturnIndex, Kind))
2244 return true;
2245
2246 // Look at the callee, if available.
2247 if (const Function *F = getCalledFunction())
2248 return F->getAttributes().hasAttribute(AttributeList::ReturnIndex, Kind);
2249 return false;
2250 }
2251};
2252
2253template <>
2254struct OperandTraits<CallBase> : public VariadicOperandTraits<CallBase, 1> {};
2255
2256DEFINE_TRANSPARENT_OPERAND_ACCESSORS(CallBase, Value)CallBase::op_iterator CallBase::op_begin() { return OperandTraits
<CallBase>::op_begin(this); } CallBase::const_op_iterator
CallBase::op_begin() const { return OperandTraits<CallBase
>::op_begin(const_cast<CallBase*>(this)); } CallBase
::op_iterator CallBase::op_end() { return OperandTraits<CallBase
>::op_end(this); } CallBase::const_op_iterator CallBase::op_end
() const { return OperandTraits<CallBase>::op_end(const_cast
<CallBase*>(this)); } Value *CallBase::getOperand(unsigned
i_nocapture) const { ((i_nocapture < OperandTraits<CallBase
>::operands(this) && "getOperand() out of range!")
? static_cast<void> (0) : __assert_fail ("i_nocapture < OperandTraits<CallBase>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 2256, __PRETTY_FUNCTION__)); return cast_or_null<Value>
( OperandTraits<CallBase>::op_begin(const_cast<CallBase
*>(this))[i_nocapture].get()); } void CallBase::setOperand
(unsigned i_nocapture, Value *Val_nocapture) { ((i_nocapture <
OperandTraits<CallBase>::operands(this) && "setOperand() out of range!"
) ? static_cast<void> (0) : __assert_fail ("i_nocapture < OperandTraits<CallBase>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 2256, __PRETTY_FUNCTION__)); OperandTraits<CallBase>::
op_begin(this)[i_nocapture] = Val_nocapture; } unsigned CallBase
::getNumOperands() const { return OperandTraits<CallBase>
::operands(this); } template <int Idx_nocapture> Use &
CallBase::Op() { return this->OpFrom<Idx_nocapture>(
this); } template <int Idx_nocapture> const Use &CallBase
::Op() const { return this->OpFrom<Idx_nocapture>(this
); }
2257
2258//===----------------------------------------------------------------------===//
2259// FuncletPadInst Class
2260//===----------------------------------------------------------------------===//
2261class FuncletPadInst : public Instruction {
2262private:
2263 FuncletPadInst(const FuncletPadInst &CPI);
2264
2265 explicit FuncletPadInst(Instruction::FuncletPadOps Op, Value *ParentPad,
2266 ArrayRef<Value *> Args, unsigned Values,
2267 const Twine &NameStr, Instruction *InsertBefore);
2268 explicit FuncletPadInst(Instruction::FuncletPadOps Op, Value *ParentPad,
2269 ArrayRef<Value *> Args, unsigned Values,
2270 const Twine &NameStr, BasicBlock *InsertAtEnd);
2271
2272 void init(Value *ParentPad, ArrayRef<Value *> Args, const Twine &NameStr);
2273
2274protected:
2275 // Note: Instruction needs to be a friend here to call cloneImpl.
2276 friend class Instruction;
2277 friend class CatchPadInst;
2278 friend class CleanupPadInst;
2279
2280 FuncletPadInst *cloneImpl() const;
2281
2282public:
2283 /// Provide fast operand accessors
2284 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
2285
2286 /// getNumArgOperands - Return the number of funcletpad arguments.
2287 ///
2288 unsigned getNumArgOperands() const { return getNumOperands() - 1; }
2289
2290 /// Convenience accessors
2291
2292 /// Return the outer EH-pad this funclet is nested within.
2293 ///
2294 /// Note: This returns the associated CatchSwitchInst if this FuncletPadInst
2295 /// is a CatchPadInst.
2296 Value *getParentPad() const { return Op<-1>(); }
2297 void setParentPad(Value *ParentPad) {
2298 assert(ParentPad)((ParentPad) ? static_cast<void> (0) : __assert_fail ("ParentPad"
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 2298, __PRETTY_FUNCTION__))
;
2299 Op<-1>() = ParentPad;
2300 }
2301
2302 /// getArgOperand/setArgOperand - Return/set the i-th funcletpad argument.
2303 ///
2304 Value *getArgOperand(unsigned i) const { return getOperand(i); }
2305 void setArgOperand(unsigned i, Value *v) { setOperand(i, v); }
2306
2307 /// arg_operands - iteration adapter for range-for loops.
2308 op_range arg_operands() { return op_range(op_begin(), op_end() - 1); }
2309
2310 /// arg_operands - iteration adapter for range-for loops.
2311 const_op_range arg_operands() const {
2312 return const_op_range(op_begin(), op_end() - 1);
2313 }
2314
2315 // Methods for support type inquiry through isa, cast, and dyn_cast:
2316 static bool classof(const Instruction *I) { return I->isFuncletPad(); }
2317 static bool classof(const Value *V) {
2318 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2319 }
2320};
2321
2322template <>
2323struct OperandTraits<FuncletPadInst>
2324 : public VariadicOperandTraits<FuncletPadInst, /*MINARITY=*/1> {};
2325
2326DEFINE_TRANSPARENT_OPERAND_ACCESSORS(FuncletPadInst, Value)FuncletPadInst::op_iterator FuncletPadInst::op_begin() { return
OperandTraits<FuncletPadInst>::op_begin(this); } FuncletPadInst
::const_op_iterator FuncletPadInst::op_begin() const { return
OperandTraits<FuncletPadInst>::op_begin(const_cast<
FuncletPadInst*>(this)); } FuncletPadInst::op_iterator FuncletPadInst
::op_end() { return OperandTraits<FuncletPadInst>::op_end
(this); } FuncletPadInst::const_op_iterator FuncletPadInst::op_end
() const { return OperandTraits<FuncletPadInst>::op_end
(const_cast<FuncletPadInst*>(this)); } Value *FuncletPadInst
::getOperand(unsigned i_nocapture) const { ((i_nocapture <
OperandTraits<FuncletPadInst>::operands(this) &&
"getOperand() out of range!") ? static_cast<void> (0) :
__assert_fail ("i_nocapture < OperandTraits<FuncletPadInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 2326, __PRETTY_FUNCTION__)); return cast_or_null<Value>
( OperandTraits<FuncletPadInst>::op_begin(const_cast<
FuncletPadInst*>(this))[i_nocapture].get()); } void FuncletPadInst
::setOperand(unsigned i_nocapture, Value *Val_nocapture) { ((
i_nocapture < OperandTraits<FuncletPadInst>::operands
(this) && "setOperand() out of range!") ? static_cast
<void> (0) : __assert_fail ("i_nocapture < OperandTraits<FuncletPadInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/IR/InstrTypes.h"
, 2326, __PRETTY_FUNCTION__)); OperandTraits<FuncletPadInst
>::op_begin(this)[i_nocapture] = Val_nocapture; } unsigned
FuncletPadInst::getNumOperands() const { return OperandTraits
<FuncletPadInst>::operands(this); } template <int Idx_nocapture
> Use &FuncletPadInst::Op() { return this->OpFrom<
Idx_nocapture>(this); } template <int Idx_nocapture>
const Use &FuncletPadInst::Op() const { return this->
OpFrom<Idx_nocapture>(this); }
2327
2328} // end namespace llvm
2329
2330#endif // LLVM_IR_INSTRTYPES_H

/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/Support/ErrorOr.h

1//===- llvm/Support/ErrorOr.h - Error Smart Pointer -------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10///
11/// Provides ErrorOr<T> smart pointer.
12///
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_SUPPORT_ERROROR_H
16#define LLVM_SUPPORT_ERROROR_H
17
18#include "llvm/Support/AlignOf.h"
19#include <cassert>
20#include <system_error>
21#include <type_traits>
22#include <utility>
23
24namespace llvm {
25
26/// Represents either an error or a value T.
27///
28/// ErrorOr<T> is a pointer-like class that represents the result of an
29/// operation. The result is either an error, or a value of type T. This is
30/// designed to emulate the usage of returning a pointer where nullptr indicates
31/// failure. However instead of just knowing that the operation failed, we also
32/// have an error_code and optional user data that describes why it failed.
33///
34/// It is used like the following.
35/// \code
36/// ErrorOr<Buffer> getBuffer();
37///
38/// auto buffer = getBuffer();
39/// if (error_code ec = buffer.getError())
40/// return ec;
41/// buffer->write("adena");
42/// \endcode
43///
44///
45/// Implicit conversion to bool returns true if there is a usable value. The
46/// unary * and -> operators provide pointer like access to the value. Accessing
47/// the value when there is an error has undefined behavior.
48///
49/// When T is a reference type the behavior is slightly different. The reference
50/// is held in a std::reference_wrapper<std::remove_reference<T>::type>, and
51/// there is special handling to make operator -> work as if T was not a
52/// reference.
53///
54/// T cannot be a rvalue reference.
55template<class T>
56class ErrorOr {
57 template <class OtherT> friend class ErrorOr;
58
59 static constexpr bool isRef = std::is_reference<T>::value;
60
61 using wrap = std::reference_wrapper<std::remove_reference_t<T>>;
62
63public:
64 using storage_type = std::conditional_t<isRef, wrap, T>;
65
66private:
67 using reference = std::remove_reference_t<T> &;
68 using const_reference = const std::remove_reference_t<T> &;
69 using pointer = std::remove_reference_t<T> *;
70 using const_pointer = const std::remove_reference_t<T> *;
71
72public:
73 template <class E>
74 ErrorOr(E ErrorCode,
75 std::enable_if_t<std::is_error_code_enum<E>::value ||
76 std::is_error_condition_enum<E>::value,
77 void *> = nullptr)
78 : HasError(true) {
79 new (getErrorStorage()) std::error_code(make_error_code(ErrorCode));
80 }
81
82 ErrorOr(std::error_code EC) : HasError(true) {
83 new (getErrorStorage()) std::error_code(EC);
84 }
85
86 template <class OtherT>
87 ErrorOr(OtherT &&Val,
88 std::enable_if_t<std::is_convertible<OtherT, T>::value> * = nullptr)
89 : HasError(false) {
90 new (getStorage()) storage_type(std::forward<OtherT>(Val));
91 }
92
93 ErrorOr(const ErrorOr &Other) {
94 copyConstruct(Other);
95 }
96
97 template <class OtherT>
98 ErrorOr(const ErrorOr<OtherT> &Other,
99 std::enable_if_t<std::is_convertible<OtherT, T>::value> * = nullptr) {
100 copyConstruct(Other);
101 }
102
103 template <class OtherT>
104 explicit ErrorOr(
105 const ErrorOr<OtherT> &Other,
106 std::enable_if_t<!std::is_convertible<OtherT, const T &>::value> * =
107 nullptr) {
108 copyConstruct(Other);
109 }
110
111 ErrorOr(ErrorOr &&Other) {
112 moveConstruct(std::move(Other));
113 }
114
115 template <class OtherT>
116 ErrorOr(ErrorOr<OtherT> &&Other,
117 std::enable_if_t<std::is_convertible<OtherT, T>::value> * = nullptr) {
118 moveConstruct(std::move(Other));
119 }
120
121 // This might eventually need SFINAE but it's more complex than is_convertible
122 // & I'm too lazy to write it right now.
123 template <class OtherT>
124 explicit ErrorOr(
125 ErrorOr<OtherT> &&Other,
126 std::enable_if_t<!std::is_convertible<OtherT, T>::value> * = nullptr) {
127 moveConstruct(std::move(Other));
128 }
129
130 ErrorOr &operator=(const ErrorOr &Other) {
131 copyAssign(Other);
132 return *this;
133 }
134
135 ErrorOr &operator=(ErrorOr &&Other) {
136 moveAssign(std::move(Other));
137 return *this;
138 }
139
140 ~ErrorOr() {
141 if (!HasError)
142 getStorage()->~storage_type();
143 }
144
145 /// Return false if there is an error.
146 explicit operator bool() const {
147 return !HasError;
59
Assuming field 'HasError' is false, which participates in a condition later
60
Returning the value 1, which participates in a condition later
148 }
149
150 reference get() { return *getStorage(); }
151 const_reference get() const { return const_cast<ErrorOr<T> *>(this)->get(); }
152
153 std::error_code getError() const {
154 return HasError ? *getErrorStorage() : std::error_code();
155 }
156
157 pointer operator ->() {
158 return toPointer(getStorage());
159 }
160
161 const_pointer operator->() const { return toPointer(getStorage()); }
162
163 reference operator *() {
164 return *getStorage();
165 }
166
167 const_reference operator*() const { return *getStorage(); }
168
169private:
170 template <class OtherT>
171 void copyConstruct(const ErrorOr<OtherT> &Other) {
172 if (!Other.HasError) {
173 // Get the other value.
174 HasError = false;
175 new (getStorage()) storage_type(*Other.getStorage());
176 } else {
177 // Get other's error.
178 HasError = true;
179 new (getErrorStorage()) std::error_code(Other.getError());
180 }
181 }
182
183 template <class T1>
184 static bool compareThisIfSameType(const T1 &a, const T1 &b) {
185 return &a == &b;
186 }
187
188 template <class T1, class T2>
189 static bool compareThisIfSameType(const T1 &a, const T2 &b) {
190 return false;
191 }
192
193 template <class OtherT>
194 void copyAssign(const ErrorOr<OtherT> &Other) {
195 if (compareThisIfSameType(*this, Other))
196 return;
197
198 this->~ErrorOr();
199 new (this) ErrorOr(Other);
200 }
201
202 template <class OtherT>
203 void moveConstruct(ErrorOr<OtherT> &&Other) {
204 if (!Other.HasError) {
205 // Get the other value.
206 HasError = false;
207 new (getStorage()) storage_type(std::move(*Other.getStorage()));
208 } else {
209 // Get other's error.
210 HasError = true;
211 new (getErrorStorage()) std::error_code(Other.getError());
212 }
213 }
214
215 template <class OtherT>
216 void moveAssign(ErrorOr<OtherT> &&Other) {
217 if (compareThisIfSameType(*this, Other))
218 return;
219
220 this->~ErrorOr();
221 new (this) ErrorOr(std::move(Other));
222 }
223
224 pointer toPointer(pointer Val) {
225 return Val;
226 }
227
228 const_pointer toPointer(const_pointer Val) const { return Val; }
229
230 pointer toPointer(wrap *Val) {
231 return &Val->get();
232 }
233
234 const_pointer toPointer(const wrap *Val) const { return &Val->get(); }
235
236 storage_type *getStorage() {
237 assert(!HasError && "Cannot get value when an error exists!")((!HasError && "Cannot get value when an error exists!"
) ? static_cast<void> (0) : __assert_fail ("!HasError && \"Cannot get value when an error exists!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/Support/ErrorOr.h"
, 237, __PRETTY_FUNCTION__))
;
238 return reinterpret_cast<storage_type *>(&TStorage);
239 }
240
241 const storage_type *getStorage() const {
242 assert(!HasError && "Cannot get value when an error exists!")((!HasError && "Cannot get value when an error exists!"
) ? static_cast<void> (0) : __assert_fail ("!HasError && \"Cannot get value when an error exists!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/Support/ErrorOr.h"
, 242, __PRETTY_FUNCTION__))
;
243 return reinterpret_cast<const storage_type *>(&TStorage);
244 }
245
246 std::error_code *getErrorStorage() {
247 assert(HasError && "Cannot get error when a value exists!")((HasError && "Cannot get error when a value exists!"
) ? static_cast<void> (0) : __assert_fail ("HasError && \"Cannot get error when a value exists!\""
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/Support/ErrorOr.h"
, 247, __PRETTY_FUNCTION__))
;
248 return reinterpret_cast<std::error_code *>(&ErrorStorage);
249 }
250
251 const std::error_code *getErrorStorage() const {
252 return const_cast<ErrorOr<T> *>(this)->getErrorStorage();
253 }
254
255 union {
256 AlignedCharArrayUnion<storage_type> TStorage;
257 AlignedCharArrayUnion<std::error_code> ErrorStorage;
258 };
259 bool HasError : 1;
260};
261
262template <class T, class E>
263std::enable_if_t<std::is_error_code_enum<E>::value ||
264 std::is_error_condition_enum<E>::value,
265 bool>
266operator==(const ErrorOr<T> &Err, E Code) {
267 return Err.getError() == Code;
268}
269
270} // end namespace llvm
271
272#endif // LLVM_SUPPORT_ERROROR_H

/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/ADT/StringMap.h

1//===- StringMap.h - String Hash table map interface ------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the StringMap class.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_ADT_STRINGMAP_H
14#define LLVM_ADT_STRINGMAP_H
15
16#include "llvm/ADT/StringMapEntry.h"
17#include "llvm/Support/AllocatorBase.h"
18#include "llvm/Support/PointerLikeTypeTraits.h"
19#include <initializer_list>
20#include <iterator>
21
22namespace llvm {
23
24template <typename ValueTy> class StringMapConstIterator;
25template <typename ValueTy> class StringMapIterator;
26template <typename ValueTy> class StringMapKeyIterator;
27
28/// StringMapImpl - This is the base class of StringMap that is shared among
29/// all of its instantiations.
30class StringMapImpl {
31protected:
32 // Array of NumBuckets pointers to entries, null pointers are holes.
33 // TheTable[NumBuckets] contains a sentinel value for easy iteration. Followed
34 // by an array of the actual hash values as unsigned integers.
35 StringMapEntryBase **TheTable = nullptr;
36 unsigned NumBuckets = 0;
37 unsigned NumItems = 0;
38 unsigned NumTombstones = 0;
39 unsigned ItemSize;
40
41protected:
42 explicit StringMapImpl(unsigned itemSize) : ItemSize(itemSize) {}
43 StringMapImpl(StringMapImpl &&RHS)
44 : TheTable(RHS.TheTable), NumBuckets(RHS.NumBuckets),
45 NumItems(RHS.NumItems), NumTombstones(RHS.NumTombstones),
46 ItemSize(RHS.ItemSize) {
47 RHS.TheTable = nullptr;
48 RHS.NumBuckets = 0;
49 RHS.NumItems = 0;
50 RHS.NumTombstones = 0;
51 }
52
53 StringMapImpl(unsigned InitSize, unsigned ItemSize);
54 unsigned RehashTable(unsigned BucketNo = 0);
55
56 /// LookupBucketFor - Look up the bucket that the specified string should end
57 /// up in. If it already exists as a key in the map, the Item pointer for the
58 /// specified bucket will be non-null. Otherwise, it will be null. In either
59 /// case, the FullHashValue field of the bucket will be set to the hash value
60 /// of the string.
61 unsigned LookupBucketFor(StringRef Key);
62
63 /// FindKey - Look up the bucket that contains the specified key. If it exists
64 /// in the map, return the bucket number of the key. Otherwise return -1.
65 /// This does not modify the map.
66 int FindKey(StringRef Key) const;
67
68 /// RemoveKey - Remove the specified StringMapEntry from the table, but do not
69 /// delete it. This aborts if the value isn't in the table.
70 void RemoveKey(StringMapEntryBase *V);
71
72 /// RemoveKey - Remove the StringMapEntry for the specified key from the
73 /// table, returning it. If the key is not in the table, this returns null.
74 StringMapEntryBase *RemoveKey(StringRef Key);
75
76 /// Allocate the table with the specified number of buckets and otherwise
77 /// setup the map as empty.
78 void init(unsigned Size);
79
80public:
81 static constexpr uintptr_t TombstoneIntVal =
82 static_cast<uintptr_t>(-1)
83 << PointerLikeTypeTraits<StringMapEntryBase *>::NumLowBitsAvailable;
84
85 static StringMapEntryBase *getTombstoneVal() {
86 return reinterpret_cast<StringMapEntryBase *>(TombstoneIntVal);
87 }
88
89 unsigned getNumBuckets() const { return NumBuckets; }
90 unsigned getNumItems() const { return NumItems; }
91
92 bool empty() const { return NumItems == 0; }
63
Assuming field 'NumItems' is not equal to 0
64
Returning zero, which participates in a condition later
93 unsigned size() const { return NumItems; }
94
95 void swap(StringMapImpl &Other) {
96 std::swap(TheTable, Other.TheTable);
97 std::swap(NumBuckets, Other.NumBuckets);
98 std::swap(NumItems, Other.NumItems);
99 std::swap(NumTombstones, Other.NumTombstones);
100 }
101};
102
103/// StringMap - This is an unconventional map that is specialized for handling
104/// keys that are "strings", which are basically ranges of bytes. This does some
105/// funky memory allocation and hashing things to make it extremely efficient,
106/// storing the string data *after* the value in the map.
107template <typename ValueTy, typename AllocatorTy = MallocAllocator>
108class StringMap : public StringMapImpl {
109 AllocatorTy Allocator;
110
111public:
112 using MapEntryTy = StringMapEntry<ValueTy>;
113
114 StringMap() : StringMapImpl(static_cast<unsigned>(sizeof(MapEntryTy))) {}
115
116 explicit StringMap(unsigned InitialSize)
117 : StringMapImpl(InitialSize, static_cast<unsigned>(sizeof(MapEntryTy))) {}
118
119 explicit StringMap(AllocatorTy A)
120 : StringMapImpl(static_cast<unsigned>(sizeof(MapEntryTy))), Allocator(A) {
121 }
122
123 StringMap(unsigned InitialSize, AllocatorTy A)
124 : StringMapImpl(InitialSize, static_cast<unsigned>(sizeof(MapEntryTy))),
125 Allocator(A) {}
126
127 StringMap(std::initializer_list<std::pair<StringRef, ValueTy>> List)
128 : StringMapImpl(List.size(), static_cast<unsigned>(sizeof(MapEntryTy))) {
129 for (const auto &P : List) {
130 insert(P);
131 }
132 }
133
134 StringMap(StringMap &&RHS)
135 : StringMapImpl(std::move(RHS)), Allocator(std::move(RHS.Allocator)) {}
136
137 StringMap(const StringMap &RHS)
138 : StringMapImpl(static_cast<unsigned>(sizeof(MapEntryTy))),
139 Allocator(RHS.Allocator) {
140 if (RHS.empty())
141 return;
142
143 // Allocate TheTable of the same size as RHS's TheTable, and set the
144 // sentinel appropriately (and NumBuckets).
145 init(RHS.NumBuckets);
146 unsigned *HashTable = (unsigned *)(TheTable + NumBuckets + 1),
147 *RHSHashTable = (unsigned *)(RHS.TheTable + NumBuckets + 1);
148
149 NumItems = RHS.NumItems;
150 NumTombstones = RHS.NumTombstones;
151 for (unsigned I = 0, E = NumBuckets; I != E; ++I) {
152 StringMapEntryBase *Bucket = RHS.TheTable[I];
153 if (!Bucket || Bucket == getTombstoneVal()) {
154 TheTable[I] = Bucket;
155 continue;
156 }
157
158 TheTable[I] = MapEntryTy::Create(
159 static_cast<MapEntryTy *>(Bucket)->getKey(), Allocator,
160 static_cast<MapEntryTy *>(Bucket)->getValue());
161 HashTable[I] = RHSHashTable[I];
162 }
163
164 // Note that here we've copied everything from the RHS into this object,
165 // tombstones included. We could, instead, have re-probed for each key to
166 // instantiate this new object without any tombstone buckets. The
167 // assumption here is that items are rarely deleted from most StringMaps,
168 // and so tombstones are rare, so the cost of re-probing for all inputs is
169 // not worthwhile.
170 }
171
172 StringMap &operator=(StringMap RHS) {
173 StringMapImpl::swap(RHS);
174 std::swap(Allocator, RHS.Allocator);
175 return *this;
176 }
177
178 ~StringMap() {
179 // Delete all the elements in the map, but don't reset the elements
180 // to default values. This is a copy of clear(), but avoids unnecessary
181 // work not required in the destructor.
182 if (!empty()) {
183 for (unsigned I = 0, E = NumBuckets; I != E; ++I) {
184 StringMapEntryBase *Bucket = TheTable[I];
185 if (Bucket && Bucket != getTombstoneVal()) {
186 static_cast<MapEntryTy *>(Bucket)->Destroy(Allocator);
187 }
188 }
189 }
190 free(TheTable);
191 }
192
193 AllocatorTy &getAllocator() { return Allocator; }
194 const AllocatorTy &getAllocator() const { return Allocator; }
195
196 using key_type = const char *;
197 using mapped_type = ValueTy;
198 using value_type = StringMapEntry<ValueTy>;
199 using size_type = size_t;
200
201 using const_iterator = StringMapConstIterator<ValueTy>;
202 using iterator = StringMapIterator<ValueTy>;
203
204 iterator begin() { return iterator(TheTable, NumBuckets == 0); }
205 iterator end() { return iterator(TheTable + NumBuckets, true); }
206 const_iterator begin() const {
207 return const_iterator(TheTable, NumBuckets == 0);
208 }
209 const_iterator end() const {
210 return const_iterator(TheTable + NumBuckets, true);
211 }
212
213 iterator_range<StringMapKeyIterator<ValueTy>> keys() const {
214 return make_range(StringMapKeyIterator<ValueTy>(begin()),
215 StringMapKeyIterator<ValueTy>(end()));
216 }
217
218 iterator find(StringRef Key) {
219 int Bucket = FindKey(Key);
220 if (Bucket == -1)
221 return end();
222 return iterator(TheTable + Bucket, true);
223 }
224
225 const_iterator find(StringRef Key) const {
226 int Bucket = FindKey(Key);
227 if (Bucket == -1)
228 return end();
229 return const_iterator(TheTable + Bucket, true);
230 }
231
232 /// lookup - Return the entry for the specified key, or a default
233 /// constructed value if no such entry exists.
234 ValueTy lookup(StringRef Key) const {
235 const_iterator it = find(Key);
236 if (it != end())
237 return it->second;
238 return ValueTy();
239 }
240
241 /// Lookup the ValueTy for the \p Key, or create a default constructed value
242 /// if the key is not in the map.
243 ValueTy &operator[](StringRef Key) { return try_emplace(Key).first->second; }
244
245 /// count - Return 1 if the element is in the map, 0 otherwise.
246 size_type count(StringRef Key) const { return find(Key) == end() ? 0 : 1; }
247
248 template <typename InputTy>
249 size_type count(const StringMapEntry<InputTy> &MapEntry) const {
250 return count(MapEntry.getKey());
251 }
252
253 /// equal - check whether both of the containers are equal.
254 bool operator==(const StringMap &RHS) const {
255 if (size() != RHS.size())
256 return false;
257
258 for (const auto &KeyValue : *this) {
259 auto FindInRHS = RHS.find(KeyValue.getKey());
260
261 if (FindInRHS == RHS.end())
262 return false;
263
264 if (!(KeyValue.getValue() == FindInRHS->getValue()))
265 return false;
266 }
267
268 return true;
269 }
270
271 bool operator!=(const StringMap &RHS) const { return !(*this == RHS); }
272
273 /// insert - Insert the specified key/value pair into the map. If the key
274 /// already exists in the map, return false and ignore the request, otherwise
275 /// insert it and return true.
276 bool insert(MapEntryTy *KeyValue) {
277 unsigned BucketNo = LookupBucketFor(KeyValue->getKey());
278 StringMapEntryBase *&Bucket = TheTable[BucketNo];
279 if (Bucket && Bucket != getTombstoneVal())
280 return false; // Already exists in map.
281
282 if (Bucket == getTombstoneVal())
283 --NumTombstones;
284 Bucket = KeyValue;
285 ++NumItems;
286 assert(NumItems + NumTombstones <= NumBuckets)((NumItems + NumTombstones <= NumBuckets) ? static_cast<
void> (0) : __assert_fail ("NumItems + NumTombstones <= NumBuckets"
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/ADT/StringMap.h"
, 286, __PRETTY_FUNCTION__))
;
287
288 RehashTable();
289 return true;
290 }
291
292 /// insert - Inserts the specified key/value pair into the map if the key
293 /// isn't already in the map. The bool component of the returned pair is true
294 /// if and only if the insertion takes place, and the iterator component of
295 /// the pair points to the element with key equivalent to the key of the pair.
296 std::pair<iterator, bool> insert(std::pair<StringRef, ValueTy> KV) {
297 return try_emplace(KV.first, std::move(KV.second));
298 }
299
300 /// Inserts an element or assigns to the current element if the key already
301 /// exists. The return type is the same as try_emplace.
302 template <typename V>
303 std::pair<iterator, bool> insert_or_assign(StringRef Key, V &&Val) {
304 auto Ret = try_emplace(Key, std::forward<V>(Val));
305 if (!Ret.second)
306 Ret.first->second = std::forward<V>(Val);
307 return Ret;
308 }
309
310 /// Emplace a new element for the specified key into the map if the key isn't
311 /// already in the map. The bool component of the returned pair is true
312 /// if and only if the insertion takes place, and the iterator component of
313 /// the pair points to the element with key equivalent to the key of the pair.
314 template <typename... ArgsTy>
315 std::pair<iterator, bool> try_emplace(StringRef Key, ArgsTy &&... Args) {
316 unsigned BucketNo = LookupBucketFor(Key);
317 StringMapEntryBase *&Bucket = TheTable[BucketNo];
318 if (Bucket && Bucket != getTombstoneVal())
319 return std::make_pair(iterator(TheTable + BucketNo, false),
320 false); // Already exists in map.
321
322 if (Bucket == getTombstoneVal())
323 --NumTombstones;
324 Bucket = MapEntryTy::Create(Key, Allocator, std::forward<ArgsTy>(Args)...);
325 ++NumItems;
326 assert(NumItems + NumTombstones <= NumBuckets)((NumItems + NumTombstones <= NumBuckets) ? static_cast<
void> (0) : __assert_fail ("NumItems + NumTombstones <= NumBuckets"
, "/build/llvm-toolchain-snapshot-12~++20210115100614+a14c36fe27f5/llvm/include/llvm/ADT/StringMap.h"
, 326, __PRETTY_FUNCTION__))
;
327
328 BucketNo = RehashTable(BucketNo);
329 return std::make_pair(iterator(TheTable + BucketNo, false), true);
330 }
331
332 // clear - Empties out the StringMap
333 void clear() {
334 if (empty())
335 return;
336
337 // Zap all values, resetting the keys back to non-present (not tombstone),
338 // which is safe because we're removing all elements.
339 for (unsigned I = 0, E = NumBuckets; I != E; ++I) {
340 StringMapEntryBase *&Bucket = TheTable[I];
341 if (Bucket && Bucket != getTombstoneVal()) {
342 static_cast<MapEntryTy *>(Bucket)->Destroy(Allocator);
343 }
344 Bucket = nullptr;
345 }
346
347 NumItems = 0;
348 NumTombstones = 0;
349 }
350
351 /// remove - Remove the specified key/value pair from the map, but do not
352 /// erase it. This aborts if the key is not in the map.
353 void remove(MapEntryTy *KeyValue) { RemoveKey(KeyValue); }
354
355 void erase(iterator I) {
356 MapEntryTy &V = *I;
357 remove(&V);
358 V.Destroy(Allocator);
359 }
360
361 bool erase(StringRef Key) {
362 iterator I = find(Key);
363 if (I == end())
364 return false;
365 erase(I);
366 return true;
367 }
368};
369
370template <typename DerivedTy, typename ValueTy>
371class StringMapIterBase
372 : public iterator_facade_base<DerivedTy, std::forward_iterator_tag,
373 ValueTy> {
374protected:
375 StringMapEntryBase **Ptr = nullptr;
376
377public:
378 StringMapIterBase() = default;
379
380 explicit StringMapIterBase(StringMapEntryBase **Bucket,
381 bool NoAdvance = false)
382 : Ptr(Bucket) {
383 if (!NoAdvance)
384 AdvancePastEmptyBuckets();
385 }
386
387 DerivedTy &operator=(const DerivedTy &Other) {
388 Ptr = Other.Ptr;
389 return static_cast<DerivedTy &>(*this);
390 }
391
392 friend bool operator==(const DerivedTy &LHS, const DerivedTy &RHS) {
393 return LHS.Ptr == RHS.Ptr;
394 }
395
396 DerivedTy &operator++() { // Preincrement
397 ++Ptr;
398 AdvancePastEmptyBuckets();
399 return static_cast<DerivedTy &>(*this);
400 }
401
402 DerivedTy operator++(int) { // Post-increment
403 DerivedTy Tmp(Ptr);
404 ++*this;
405 return Tmp;
406 }
407
408private:
409 void AdvancePastEmptyBuckets() {
410 while (*Ptr == nullptr || *Ptr == StringMapImpl::getTombstoneVal())
411 ++Ptr;
412 }
413};
414
415template <typename ValueTy>
416class StringMapConstIterator
417 : public StringMapIterBase<StringMapConstIterator<ValueTy>,
418 const StringMapEntry<ValueTy>> {
419 using base = StringMapIterBase<StringMapConstIterator<ValueTy>,
420 const StringMapEntry<ValueTy>>;
421
422public:
423 StringMapConstIterator() = default;
424 explicit StringMapConstIterator(StringMapEntryBase **Bucket,
425 bool NoAdvance = false)
426 : base(Bucket, NoAdvance) {}
427
428 const StringMapEntry<ValueTy> &operator*() const {
429 return *static_cast<const StringMapEntry<ValueTy> *>(*this->Ptr);
430 }
431};
432
433template <typename ValueTy>
434class StringMapIterator : public StringMapIterBase<StringMapIterator<ValueTy>,
435 StringMapEntry<ValueTy>> {
436 using base =
437 StringMapIterBase<StringMapIterator<ValueTy>, StringMapEntry<ValueTy>>;
438
439public:
440 StringMapIterator() = default;
441 explicit StringMapIterator(StringMapEntryBase **Bucket,
442 bool NoAdvance = false)
443 : base(Bucket, NoAdvance) {}
444
445 StringMapEntry<ValueTy> &operator*() const {
446 return *static_cast<StringMapEntry<ValueTy> *>(*this->Ptr);
447 }
448
449 operator StringMapConstIterator<ValueTy>() const {
450 return StringMapConstIterator<ValueTy>(this->Ptr, true);
451 }
452};
453
454template <typename ValueTy>
455class StringMapKeyIterator
456 : public iterator_adaptor_base<StringMapKeyIterator<ValueTy>,
457 StringMapConstIterator<ValueTy>,
458 std::forward_iterator_tag, StringRef> {
459 using base = iterator_adaptor_base<StringMapKeyIterator<ValueTy>,
460 StringMapConstIterator<ValueTy>,
461 std::forward_iterator_tag, StringRef>;
462
463public:
464 StringMapKeyIterator() = default;
465 explicit StringMapKeyIterator(StringMapConstIterator<ValueTy> Iter)
466 : base(std::move(Iter)) {}
467
468 StringRef &operator*() {
469 Key = this->wrapped()->getKey();
470 return Key;
471 }
472
473private:
474 StringRef Key;
475};
476
477} // end namespace llvm
478
479#endif // LLVM_ADT_STRINGMAP_H