1 | //===- InstrProf.h - Instrumented profiling format support ------*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // Instrumentation-based profiling data is generated by instrumented |
10 | // binaries through library functions in compiler-rt, and read by the clang |
11 | // frontend to feed PGO. |
12 | // |
13 | //===----------------------------------------------------------------------===// |
14 | |
15 | #ifndef LLVM_PROFILEDATA_INSTRPROF_H |
16 | #define LLVM_PROFILEDATA_INSTRPROF_H |
17 | |
18 | #include "llvm/ADT/ArrayRef.h" |
19 | #include "llvm/ADT/BitmaskEnum.h" |
20 | #include "llvm/ADT/STLExtras.h" |
21 | #include "llvm/ADT/StringRef.h" |
22 | #include "llvm/ADT/StringSet.h" |
23 | #include "llvm/IR/GlobalValue.h" |
24 | #include "llvm/IR/ProfileSummary.h" |
25 | #include "llvm/ProfileData/InstrProfData.inc" |
26 | #include "llvm/Support/BalancedPartitioning.h" |
27 | #include "llvm/Support/CommandLine.h" |
28 | #include "llvm/Support/Compiler.h" |
29 | #include "llvm/Support/Error.h" |
30 | #include "llvm/Support/ErrorHandling.h" |
31 | #include "llvm/Support/MD5.h" |
32 | #include "llvm/Support/MathExtras.h" |
33 | #include "llvm/Support/raw_ostream.h" |
34 | #include "llvm/TargetParser/Host.h" |
35 | #include "llvm/TargetParser/Triple.h" |
36 | #include <algorithm> |
37 | #include <cassert> |
38 | #include <cstddef> |
39 | #include <cstdint> |
40 | #include <cstring> |
41 | #include <list> |
42 | #include <memory> |
43 | #include <string> |
44 | #include <system_error> |
45 | #include <utility> |
46 | #include <vector> |
47 | |
48 | namespace llvm { |
49 | |
50 | class Function; |
51 | class GlobalVariable; |
52 | struct InstrProfRecord; |
53 | class InstrProfSymtab; |
54 | class Instruction; |
55 | class MDNode; |
56 | class Module; |
57 | |
58 | enum InstrProfSectKind { |
59 | #define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) Kind, |
60 | #include "llvm/ProfileData/InstrProfData.inc" |
61 | }; |
62 | |
63 | /// Return the max count value. We reserver a few large values for special use. |
64 | inline uint64_t getInstrMaxCountValue() { |
65 | return std::numeric_limits<uint64_t>::max() - 2; |
66 | } |
67 | |
68 | /// Return the name of the profile section corresponding to \p IPSK. |
69 | /// |
70 | /// The name of the section depends on the object format type \p OF. If |
71 | /// \p AddSegmentInfo is true, a segment prefix and additional linker hints may |
72 | /// be added to the section name (this is the default). |
73 | std::string getInstrProfSectionName(InstrProfSectKind IPSK, |
74 | Triple::ObjectFormatType OF, |
75 | bool AddSegmentInfo = true); |
76 | |
77 | /// Return the name profile runtime entry point to do value profiling |
78 | /// for a given site. |
79 | inline StringRef getInstrProfValueProfFuncName() { |
80 | return INSTR_PROF_VALUE_PROF_FUNC_STR; |
81 | } |
82 | |
83 | /// Return the name profile runtime entry point to do memop size value |
84 | /// profiling. |
85 | inline StringRef getInstrProfValueProfMemOpFuncName() { |
86 | return INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR; |
87 | } |
88 | |
89 | /// Return the name prefix of variables containing instrumented function names. |
90 | inline StringRef getInstrProfNameVarPrefix() { return "__profn_" ; } |
91 | |
92 | /// Return the name prefix of variables containing per-function control data. |
93 | inline StringRef getInstrProfDataVarPrefix() { return "__profd_" ; } |
94 | |
95 | /// Return the name prefix of profile counter variables. |
96 | inline StringRef getInstrProfCountersVarPrefix() { return "__profc_" ; } |
97 | |
98 | /// Return the name prefix of profile bitmap variables. |
99 | inline StringRef getInstrProfBitmapVarPrefix() { return "__profbm_" ; } |
100 | |
101 | /// Return the name prefix of value profile variables. |
102 | inline StringRef getInstrProfValuesVarPrefix() { return "__profvp_" ; } |
103 | |
104 | /// Return the name of value profile node array variables: |
105 | inline StringRef getInstrProfVNodesVarName() { return "__llvm_prf_vnodes" ; } |
106 | |
107 | /// Return the name of the variable holding the strings (possibly compressed) |
108 | /// of all function's PGO names. |
109 | inline StringRef getInstrProfNamesVarName() { |
110 | return "__llvm_prf_nm" ; |
111 | } |
112 | |
113 | /// Return the name of a covarage mapping variable (internal linkage) |
114 | /// for each instrumented source module. Such variables are allocated |
115 | /// in the __llvm_covmap section. |
116 | inline StringRef getCoverageMappingVarName() { |
117 | return "__llvm_coverage_mapping" ; |
118 | } |
119 | |
120 | /// Return the name of the internal variable recording the array |
121 | /// of PGO name vars referenced by the coverage mapping. The owning |
122 | /// functions of those names are not emitted by FE (e.g, unused inline |
123 | /// functions.) |
124 | inline StringRef getCoverageUnusedNamesVarName() { |
125 | return "__llvm_coverage_names" ; |
126 | } |
127 | |
128 | /// Return the name of function that registers all the per-function control |
129 | /// data at program startup time by calling __llvm_register_function. This |
130 | /// function has internal linkage and is called by __llvm_profile_init |
131 | /// runtime method. This function is not generated for these platforms: |
132 | /// Darwin, Linux, and FreeBSD. |
133 | inline StringRef getInstrProfRegFuncsName() { |
134 | return "__llvm_profile_register_functions" ; |
135 | } |
136 | |
137 | /// Return the name of the runtime interface that registers per-function control |
138 | /// data for one instrumented function. |
139 | inline StringRef getInstrProfRegFuncName() { |
140 | return "__llvm_profile_register_function" ; |
141 | } |
142 | |
143 | /// Return the name of the runtime interface that registers the PGO name strings. |
144 | inline StringRef getInstrProfNamesRegFuncName() { |
145 | return "__llvm_profile_register_names_function" ; |
146 | } |
147 | |
148 | /// Return the name of the runtime initialization method that is generated by |
149 | /// the compiler. The function calls __llvm_profile_register_functions and |
150 | /// __llvm_profile_override_default_filename functions if needed. This function |
151 | /// has internal linkage and invoked at startup time via init_array. |
152 | inline StringRef getInstrProfInitFuncName() { return "__llvm_profile_init" ; } |
153 | |
154 | /// Return the name of the hook variable defined in profile runtime library. |
155 | /// A reference to the variable causes the linker to link in the runtime |
156 | /// initialization module (which defines the hook variable). |
157 | inline StringRef getInstrProfRuntimeHookVarName() { |
158 | return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_RUNTIME_VAR); |
159 | } |
160 | |
161 | /// Return the name of the compiler generated function that references the |
162 | /// runtime hook variable. The function is a weak global. |
163 | inline StringRef getInstrProfRuntimeHookVarUseFuncName() { |
164 | return "__llvm_profile_runtime_user" ; |
165 | } |
166 | |
167 | inline StringRef getInstrProfCounterBiasVarName() { |
168 | return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR); |
169 | } |
170 | |
171 | /// Return the marker used to separate PGO names during serialization. |
172 | inline StringRef getInstrProfNameSeparator() { return "\01" ; } |
173 | |
174 | /// Please use getIRPGOFuncName for LLVM IR instrumentation. This function is |
175 | /// for front-end (Clang, etc) instrumentation. |
176 | /// Return the modified name for function \c F suitable to be |
177 | /// used the key for profile lookup. Variable \c InLTO indicates if this |
178 | /// is called in LTO optimization passes. |
179 | std::string getPGOFuncName(const Function &F, bool InLTO = false, |
180 | uint64_t Version = INSTR_PROF_INDEX_VERSION); |
181 | |
182 | /// Return the modified name for a function suitable to be |
183 | /// used the key for profile lookup. The function's original |
184 | /// name is \c RawFuncName and has linkage of type \c Linkage. |
185 | /// The function is defined in module \c FileName. |
186 | std::string getPGOFuncName(StringRef RawFuncName, |
187 | GlobalValue::LinkageTypes Linkage, |
188 | StringRef FileName, |
189 | uint64_t Version = INSTR_PROF_INDEX_VERSION); |
190 | |
191 | /// \return the modified name for function \c F suitable to be |
192 | /// used as the key for IRPGO profile lookup. \c InLTO indicates if this is |
193 | /// called from LTO optimization passes. |
194 | std::string getIRPGOFuncName(const Function &F, bool InLTO = false); |
195 | |
196 | /// \return the filename and the function name parsed from the output of |
197 | /// \c getIRPGOFuncName() |
198 | std::pair<StringRef, StringRef> getParsedIRPGOName(StringRef IRPGOName); |
199 | |
200 | /// Return the name of the global variable used to store a function |
201 | /// name in PGO instrumentation. \c FuncName is the IRPGO function name |
202 | /// (returned by \c getIRPGOFuncName) for LLVM IR instrumentation and PGO |
203 | /// function name (returned by \c getPGOFuncName) for front-end instrumentation. |
204 | std::string getPGOFuncNameVarName(StringRef FuncName, |
205 | GlobalValue::LinkageTypes Linkage); |
206 | |
207 | /// Create and return the global variable for function name used in PGO |
208 | /// instrumentation. \c FuncName is the IRPGO function name (returned by |
209 | /// \c getIRPGOFuncName) for LLVM IR instrumentation and PGO function name |
210 | /// (returned by \c getPGOFuncName) for front-end instrumentation. |
211 | GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName); |
212 | |
213 | /// Create and return the global variable for function name used in PGO |
214 | /// instrumentation. \c FuncName is the IRPGO function name (returned by |
215 | /// \c getIRPGOFuncName) for LLVM IR instrumentation and PGO function name |
216 | /// (returned by \c getPGOFuncName) for front-end instrumentation. |
217 | GlobalVariable *createPGOFuncNameVar(Module &M, |
218 | GlobalValue::LinkageTypes Linkage, |
219 | StringRef PGOFuncName); |
220 | |
221 | /// Return the initializer in string of the PGO name var \c NameVar. |
222 | StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar); |
223 | |
224 | /// Given a PGO function name, remove the filename prefix and return |
225 | /// the original (static) function name. |
226 | StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, |
227 | StringRef FileName = "<unknown>" ); |
228 | |
229 | /// Given a vector of strings (names of global objects like functions or, |
230 | /// virtual tables) \c NameStrs, the method generates a combined string \c |
231 | /// Result that is ready to be serialized. The \c Result string is comprised of |
232 | /// three fields: The first field is the length of the uncompressed strings, and |
233 | /// the the second field is the length of the zlib-compressed string. Both |
234 | /// fields are encoded in ULEB128. If \c doCompress is false, the |
235 | /// third field is the uncompressed strings; otherwise it is the |
236 | /// compressed string. When the string compression is off, the |
237 | /// second field will have value zero. |
238 | Error collectGlobalObjectNameStrings(ArrayRef<std::string> NameStrs, |
239 | bool doCompression, std::string &Result); |
240 | |
241 | /// Produce \c Result string with the same format described above. The input |
242 | /// is vector of PGO function name variables that are referenced. |
243 | /// The global variable element in 'NameVars' is a string containing the pgo |
244 | /// name of a function. See `createPGOFuncNameVar` that creates these global |
245 | /// variables. |
246 | Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars, |
247 | std::string &Result, bool doCompression = true); |
248 | |
249 | /// Check if INSTR_PROF_RAW_VERSION_VAR is defined. This global is only being |
250 | /// set in IR PGO compilation. |
251 | bool isIRPGOFlagSet(const Module *M); |
252 | |
253 | /// Check if we can safely rename this Comdat function. Instances of the same |
254 | /// comdat function may have different control flows thus can not share the |
255 | /// same counter variable. |
256 | bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken = false); |
257 | |
258 | enum InstrProfValueKind : uint32_t { |
259 | #define VALUE_PROF_KIND(Enumerator, Value, Descr) Enumerator = Value, |
260 | #include "llvm/ProfileData/InstrProfData.inc" |
261 | }; |
262 | |
263 | /// Get the value profile data for value site \p SiteIdx from \p InstrProfR |
264 | /// and annotate the instruction \p Inst with the value profile meta data. |
265 | /// Annotate up to \p MaxMDCount (default 3) number of records per value site. |
266 | void annotateValueSite(Module &M, Instruction &Inst, |
267 | const InstrProfRecord &InstrProfR, |
268 | InstrProfValueKind ValueKind, uint32_t SiteIndx, |
269 | uint32_t MaxMDCount = 3); |
270 | |
271 | /// Same as the above interface but using an ArrayRef, as well as \p Sum. |
272 | void annotateValueSite(Module &M, Instruction &Inst, |
273 | ArrayRef<InstrProfValueData> VDs, uint64_t Sum, |
274 | InstrProfValueKind ValueKind, uint32_t MaxMDCount); |
275 | |
276 | /// Extract the value profile data from \p Inst which is annotated with |
277 | /// value profile meta data. Return false if there is no value data annotated, |
278 | /// otherwise return true. |
279 | bool getValueProfDataFromInst(const Instruction &Inst, |
280 | InstrProfValueKind ValueKind, |
281 | uint32_t MaxNumValueData, |
282 | InstrProfValueData ValueData[], |
283 | uint32_t &ActualNumValueData, uint64_t &TotalC, |
284 | bool GetNoICPValue = false); |
285 | |
286 | inline StringRef getPGOFuncNameMetadataName() { return "PGOFuncName" ; } |
287 | |
288 | /// Return the PGOFuncName meta data associated with a function. |
289 | MDNode *getPGOFuncNameMetadata(const Function &F); |
290 | |
291 | /// Create the PGOFuncName meta data if PGOFuncName is different from |
292 | /// function's raw name. This should only apply to internal linkage functions |
293 | /// declared by users only. |
294 | void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName); |
295 | |
296 | /// Check if we can use Comdat for profile variables. This will eliminate |
297 | /// the duplicated profile variables for Comdat functions. |
298 | bool needsComdatForCounter(const Function &F, const Module &M); |
299 | |
300 | /// An enum describing the attributes of an instrumented profile. |
301 | enum class InstrProfKind { |
302 | Unknown = 0x0, |
303 | // A frontend clang profile, incompatible with other attrs. |
304 | FrontendInstrumentation = 0x1, |
305 | // An IR-level profile (default when -fprofile-generate is used). |
306 | IRInstrumentation = 0x2, |
307 | // A profile with entry basic block instrumentation. |
308 | FunctionEntryInstrumentation = 0x4, |
309 | // A context sensitive IR-level profile. |
310 | ContextSensitive = 0x8, |
311 | // Use single byte probes for coverage. |
312 | SingleByteCoverage = 0x10, |
313 | // Only instrument the function entry basic block. |
314 | FunctionEntryOnly = 0x20, |
315 | // A memory profile collected using -fprofile=memory. |
316 | MemProf = 0x40, |
317 | // A temporal profile. |
318 | TemporalProfile = 0x80, |
319 | LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/TemporalProfile) |
320 | }; |
321 | |
322 | const std::error_category &instrprof_category(); |
323 | |
324 | enum class instrprof_error { |
325 | success = 0, |
326 | eof, |
327 | unrecognized_format, |
328 | bad_magic, |
329 | , |
330 | unsupported_version, |
331 | unsupported_hash_type, |
332 | too_large, |
333 | truncated, |
334 | malformed, |
335 | missing_correlation_info, |
336 | unexpected_correlation_info, |
337 | unable_to_correlate_profile, |
338 | unknown_function, |
339 | invalid_prof, |
340 | hash_mismatch, |
341 | count_mismatch, |
342 | bitmap_mismatch, |
343 | counter_overflow, |
344 | value_site_count_mismatch, |
345 | compress_failed, |
346 | uncompress_failed, |
347 | empty_raw_profile, |
348 | zlib_unavailable, |
349 | raw_profile_version_mismatch, |
350 | counter_value_too_large, |
351 | }; |
352 | |
353 | /// An ordered list of functions identified by their NameRef found in |
354 | /// INSTR_PROF_DATA |
355 | struct TemporalProfTraceTy { |
356 | std::vector<uint64_t> FunctionNameRefs; |
357 | uint64_t Weight; |
358 | TemporalProfTraceTy(std::initializer_list<uint64_t> Trace = {}, |
359 | uint64_t Weight = 1) |
360 | : FunctionNameRefs(Trace), Weight(Weight) {} |
361 | |
362 | /// Use a set of temporal profile traces to create a list of balanced |
363 | /// partitioning function nodes used by BalancedPartitioning to generate a |
364 | /// function order that reduces page faults during startup |
365 | static std::vector<BPFunctionNode> |
366 | createBPFunctionNodes(ArrayRef<TemporalProfTraceTy> Traces); |
367 | }; |
368 | |
369 | inline std::error_code make_error_code(instrprof_error E) { |
370 | return std::error_code(static_cast<int>(E), instrprof_category()); |
371 | } |
372 | |
373 | class InstrProfError : public ErrorInfo<InstrProfError> { |
374 | public: |
375 | InstrProfError(instrprof_error Err, const Twine &ErrStr = Twine()) |
376 | : Err(Err), Msg(ErrStr.str()) { |
377 | assert(Err != instrprof_error::success && "Not an error" ); |
378 | } |
379 | |
380 | std::string message() const override; |
381 | |
382 | void log(raw_ostream &OS) const override { OS << message(); } |
383 | |
384 | std::error_code convertToErrorCode() const override { |
385 | return make_error_code(E: Err); |
386 | } |
387 | |
388 | instrprof_error get() const { return Err; } |
389 | const std::string &getMessage() const { return Msg; } |
390 | |
391 | /// Consume an Error and return the raw enum value contained within it, and |
392 | /// the optional error message. The Error must either be a success value, or |
393 | /// contain a single InstrProfError. |
394 | static std::pair<instrprof_error, std::string> take(Error E) { |
395 | auto Err = instrprof_error::success; |
396 | std::string Msg = "" ; |
397 | handleAllErrors(E: std::move(E), Handlers: [&Err, &Msg](const InstrProfError &IPE) { |
398 | assert(Err == instrprof_error::success && "Multiple errors encountered" ); |
399 | Err = IPE.get(); |
400 | Msg = IPE.getMessage(); |
401 | }); |
402 | return {Err, Msg}; |
403 | } |
404 | |
405 | static char ID; |
406 | |
407 | private: |
408 | instrprof_error Err; |
409 | std::string Msg; |
410 | }; |
411 | |
412 | namespace object { |
413 | |
414 | class SectionRef; |
415 | |
416 | } // end namespace object |
417 | |
418 | namespace IndexedInstrProf { |
419 | |
420 | uint64_t ComputeHash(StringRef K); |
421 | |
422 | } // end namespace IndexedInstrProf |
423 | |
424 | /// A symbol table used for function [IR]PGO name look-up with keys |
425 | /// (such as pointers, md5hash values) to the function. A function's |
426 | /// [IR]PGO name or name's md5hash are used in retrieving the profile |
427 | /// data of the function. See \c getIRPGOFuncName() and \c getPGOFuncName |
428 | /// methods for details how [IR]PGO name is formed. |
429 | class InstrProfSymtab { |
430 | public: |
431 | using AddrHashMap = std::vector<std::pair<uint64_t, uint64_t>>; |
432 | |
433 | private: |
434 | StringRef Data; |
435 | uint64_t Address = 0; |
436 | // Unique name strings. |
437 | StringSet<> NameTab; |
438 | // A map from MD5 keys to function name strings. |
439 | std::vector<std::pair<uint64_t, StringRef>> MD5NameMap; |
440 | // A map from MD5 keys to function define. We only populate this map |
441 | // when build the Symtab from a Module. |
442 | std::vector<std::pair<uint64_t, Function *>> MD5FuncMap; |
443 | // A map from function runtime address to function name MD5 hash. |
444 | // This map is only populated and used by raw instr profile reader. |
445 | AddrHashMap AddrToMD5Map; |
446 | bool Sorted = false; |
447 | |
448 | static StringRef getExternalSymbol() { |
449 | return "** External Symbol **" ; |
450 | } |
451 | |
452 | // Returns the canonial name of the given PGOName. In a canonical name, all |
453 | // suffixes that begins with "." except ".__uniq." are stripped. |
454 | // FIXME: Unify this with `FunctionSamples::getCanonicalFnName`. |
455 | static StringRef getCanonicalName(StringRef PGOName); |
456 | |
457 | // Add the function into the symbol table, by creating the following |
458 | // map entries: |
459 | // name-set = {PGOFuncName} + {getCanonicalName(PGOFuncName)} if the canonical |
460 | // name is different from pgo name |
461 | // - In MD5NameMap: <MD5Hash(name), name> for name in name-set |
462 | // - In MD5FuncMap: <MD5Hash(name), &F> for name in name-set |
463 | Error addFuncWithName(Function &F, StringRef PGOFuncName); |
464 | |
465 | // If the symtab is created by a series of calls to \c addFuncName, \c |
466 | // finalizeSymtab needs to be called before looking up function names. |
467 | // This is required because the underlying map is a vector (for space |
468 | // efficiency) which needs to be sorted. |
469 | inline void finalizeSymtab(); |
470 | |
471 | public: |
472 | InstrProfSymtab() = default; |
473 | |
474 | /// Create InstrProfSymtab from an object file section which |
475 | /// contains function PGO names. When section may contain raw |
476 | /// string data or string data in compressed form. This method |
477 | /// only initialize the symtab with reference to the data and |
478 | /// the section base address. The decompression will be delayed |
479 | /// until before it is used. See also \c create(StringRef) method. |
480 | Error create(object::SectionRef &Section); |
481 | |
482 | /// \c NameStrings is a string composed of one of more sub-strings |
483 | /// encoded in the format described in \c collectPGOFuncNameStrings. |
484 | /// This method is a wrapper to \c readPGOFuncNameStrings method. |
485 | Error create(StringRef NameStrings); |
486 | |
487 | /// This interface is used by reader of CoverageMapping test |
488 | /// format. |
489 | inline Error create(StringRef D, uint64_t BaseAddr); |
490 | |
491 | /// A wrapper interface to populate the PGO symtab with functions |
492 | /// decls from module \c M. This interface is used by transformation |
493 | /// passes such as indirect function call promotion. Variable \c InLTO |
494 | /// indicates if this is called from LTO optimization passes. |
495 | Error create(Module &M, bool InLTO = false); |
496 | |
497 | /// Create InstrProfSymtab from a set of names iteratable from |
498 | /// \p IterRange. This interface is used by IndexedProfReader. |
499 | template <typename NameIterRange> Error create(const NameIterRange &IterRange); |
500 | |
501 | /// Update the symtab by adding \p FuncName to the table. This interface |
502 | /// is used by the raw and text profile readers. |
503 | Error addFuncName(StringRef FuncName) { |
504 | if (FuncName.empty()) |
505 | return make_error<InstrProfError>(Args: instrprof_error::malformed, |
506 | Args: "function name is empty" ); |
507 | auto Ins = NameTab.insert(key: FuncName); |
508 | if (Ins.second) { |
509 | MD5NameMap.push_back(x: std::make_pair( |
510 | x: IndexedInstrProf::ComputeHash(K: FuncName), y: Ins.first->getKey())); |
511 | Sorted = false; |
512 | } |
513 | return Error::success(); |
514 | } |
515 | |
516 | /// Map a function address to its name's MD5 hash. This interface |
517 | /// is only used by the raw profiler reader. |
518 | void mapAddress(uint64_t Addr, uint64_t MD5Val) { |
519 | AddrToMD5Map.push_back(x: std::make_pair(x&: Addr, y&: MD5Val)); |
520 | } |
521 | |
522 | /// Return a function's hash, or 0, if the function isn't in this SymTab. |
523 | uint64_t getFunctionHashFromAddress(uint64_t Address); |
524 | |
525 | /// Return function's PGO name from the function name's symbol |
526 | /// address in the object file. If an error occurs, return |
527 | /// an empty string. |
528 | StringRef getFuncName(uint64_t FuncNameAddress, size_t NameSize); |
529 | |
530 | /// Return name of functions or global variables from the name's md5 hash |
531 | /// value. If not found, return an empty string. |
532 | inline StringRef getFuncOrVarName(uint64_t ValMD5Hash); |
533 | |
534 | /// Just like getFuncOrVarName, except that it will return literal string |
535 | /// 'External Symbol' if the function or global variable is external to |
536 | /// this symbol table. |
537 | inline StringRef getFuncOrVarNameIfDefined(uint64_t ValMD5Hash); |
538 | |
539 | /// True if Symbol is the value used to represent external symbols. |
540 | static bool isExternalSymbol(const StringRef &Symbol) { |
541 | return Symbol == InstrProfSymtab::getExternalSymbol(); |
542 | } |
543 | |
544 | /// Return function from the name's md5 hash. Return nullptr if not found. |
545 | inline Function *getFunction(uint64_t FuncMD5Hash); |
546 | |
547 | /// Return the name section data. |
548 | inline StringRef getNameData() const { return Data; } |
549 | |
550 | /// Dump the symbols in this table. |
551 | void dumpNames(raw_ostream &OS) const; |
552 | }; |
553 | |
554 | Error InstrProfSymtab::create(StringRef D, uint64_t BaseAddr) { |
555 | Data = D; |
556 | Address = BaseAddr; |
557 | return Error::success(); |
558 | } |
559 | |
560 | template <typename NameIterRange> |
561 | Error InstrProfSymtab::create(const NameIterRange &IterRange) { |
562 | for (auto Name : IterRange) |
563 | if (Error E = addFuncName(FuncName: Name)) |
564 | return E; |
565 | |
566 | finalizeSymtab(); |
567 | return Error::success(); |
568 | } |
569 | |
570 | void InstrProfSymtab::finalizeSymtab() { |
571 | if (Sorted) |
572 | return; |
573 | llvm::sort(C&: MD5NameMap, Comp: less_first()); |
574 | llvm::sort(C&: MD5FuncMap, Comp: less_first()); |
575 | llvm::sort(C&: AddrToMD5Map, Comp: less_first()); |
576 | AddrToMD5Map.erase(first: std::unique(first: AddrToMD5Map.begin(), last: AddrToMD5Map.end()), |
577 | last: AddrToMD5Map.end()); |
578 | Sorted = true; |
579 | } |
580 | |
581 | StringRef InstrProfSymtab::getFuncOrVarNameIfDefined(uint64_t MD5Hash) { |
582 | StringRef ret = getFuncOrVarName(ValMD5Hash: MD5Hash); |
583 | if (ret.empty()) |
584 | return InstrProfSymtab::getExternalSymbol(); |
585 | return ret; |
586 | } |
587 | |
588 | StringRef InstrProfSymtab::getFuncOrVarName(uint64_t MD5Hash) { |
589 | finalizeSymtab(); |
590 | auto Result = llvm::lower_bound(Range&: MD5NameMap, Value&: MD5Hash, |
591 | C: [](const std::pair<uint64_t, StringRef> &LHS, |
592 | uint64_t RHS) { return LHS.first < RHS; }); |
593 | if (Result != MD5NameMap.end() && Result->first == MD5Hash) |
594 | return Result->second; |
595 | return StringRef(); |
596 | } |
597 | |
598 | Function* InstrProfSymtab::getFunction(uint64_t FuncMD5Hash) { |
599 | finalizeSymtab(); |
600 | auto Result = llvm::lower_bound(Range&: MD5FuncMap, Value&: FuncMD5Hash, |
601 | C: [](const std::pair<uint64_t, Function *> &LHS, |
602 | uint64_t RHS) { return LHS.first < RHS; }); |
603 | if (Result != MD5FuncMap.end() && Result->first == FuncMD5Hash) |
604 | return Result->second; |
605 | return nullptr; |
606 | } |
607 | |
608 | // To store the sums of profile count values, or the percentage of |
609 | // the sums of the total count values. |
610 | struct CountSumOrPercent { |
611 | uint64_t NumEntries; |
612 | double CountSum; |
613 | double ValueCounts[IPVK_Last - IPVK_First + 1]; |
614 | CountSumOrPercent() : NumEntries(0), CountSum(0.0f), ValueCounts() {} |
615 | void reset() { |
616 | NumEntries = 0; |
617 | CountSum = 0.0f; |
618 | for (double &VC : ValueCounts) |
619 | VC = 0.0f; |
620 | } |
621 | }; |
622 | |
623 | // Function level or program level overlap information. |
624 | struct OverlapStats { |
625 | enum OverlapStatsLevel { ProgramLevel, FunctionLevel }; |
626 | // Sum of the total count values for the base profile. |
627 | CountSumOrPercent Base; |
628 | // Sum of the total count values for the test profile. |
629 | CountSumOrPercent Test; |
630 | // Overlap lap score. Should be in range of [0.0f to 1.0f]. |
631 | CountSumOrPercent Overlap; |
632 | CountSumOrPercent Mismatch; |
633 | CountSumOrPercent Unique; |
634 | OverlapStatsLevel Level; |
635 | const std::string *BaseFilename; |
636 | const std::string *TestFilename; |
637 | StringRef FuncName; |
638 | uint64_t FuncHash; |
639 | bool Valid; |
640 | |
641 | OverlapStats(OverlapStatsLevel L = ProgramLevel) |
642 | : Level(L), BaseFilename(nullptr), TestFilename(nullptr), FuncHash(0), |
643 | Valid(false) {} |
644 | |
645 | void dump(raw_fd_ostream &OS) const; |
646 | |
647 | void setFuncInfo(StringRef Name, uint64_t Hash) { |
648 | FuncName = Name; |
649 | FuncHash = Hash; |
650 | } |
651 | |
652 | Error accumulateCounts(const std::string &BaseFilename, |
653 | const std::string &TestFilename, bool IsCS); |
654 | void addOneMismatch(const CountSumOrPercent &MismatchFunc); |
655 | void addOneUnique(const CountSumOrPercent &UniqueFunc); |
656 | |
657 | static inline double score(uint64_t Val1, uint64_t Val2, double Sum1, |
658 | double Sum2) { |
659 | if (Sum1 < 1.0f || Sum2 < 1.0f) |
660 | return 0.0f; |
661 | return std::min(a: Val1 / Sum1, b: Val2 / Sum2); |
662 | } |
663 | }; |
664 | |
665 | // This is used to filter the functions whose overlap information |
666 | // to be output. |
667 | struct OverlapFuncFilters { |
668 | uint64_t ValueCutoff; |
669 | const std::string NameFilter; |
670 | }; |
671 | |
672 | struct InstrProfValueSiteRecord { |
673 | /// Value profiling data pairs at a given value site. |
674 | std::list<InstrProfValueData> ValueData; |
675 | |
676 | InstrProfValueSiteRecord() { ValueData.clear(); } |
677 | template <class InputIterator> |
678 | InstrProfValueSiteRecord(InputIterator F, InputIterator L) |
679 | : ValueData(F, L) {} |
680 | |
681 | /// Sort ValueData ascending by Value |
682 | void sortByTargetValues() { |
683 | ValueData.sort( |
684 | comp: [](const InstrProfValueData &left, const InstrProfValueData &right) { |
685 | return left.Value < right.Value; |
686 | }); |
687 | } |
688 | /// Sort ValueData Descending by Count |
689 | inline void sortByCount(); |
690 | |
691 | /// Merge data from another InstrProfValueSiteRecord |
692 | /// Optionally scale merged counts by \p Weight. |
693 | void merge(InstrProfValueSiteRecord &Input, uint64_t Weight, |
694 | function_ref<void(instrprof_error)> Warn); |
695 | /// Scale up value profile data counts by N (Numerator) / D (Denominator). |
696 | void scale(uint64_t N, uint64_t D, function_ref<void(instrprof_error)> Warn); |
697 | |
698 | /// Compute the overlap b/w this record and Input record. |
699 | void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind, |
700 | OverlapStats &Overlap, OverlapStats &FuncLevelOverlap); |
701 | }; |
702 | |
703 | /// Profiling information for a single function. |
704 | struct InstrProfRecord { |
705 | std::vector<uint64_t> Counts; |
706 | std::vector<uint8_t> BitmapBytes; |
707 | |
708 | InstrProfRecord() = default; |
709 | InstrProfRecord(std::vector<uint64_t> Counts) : Counts(std::move(Counts)) {} |
710 | InstrProfRecord(std::vector<uint64_t> Counts, |
711 | std::vector<uint8_t> BitmapBytes) |
712 | : Counts(std::move(Counts)), BitmapBytes(std::move(BitmapBytes)) {} |
713 | InstrProfRecord(InstrProfRecord &&) = default; |
714 | InstrProfRecord(const InstrProfRecord &RHS) |
715 | : Counts(RHS.Counts), BitmapBytes(RHS.BitmapBytes), |
716 | ValueData(RHS.ValueData |
717 | ? std::make_unique<ValueProfData>(args&: *RHS.ValueData) |
718 | : nullptr) {} |
719 | InstrProfRecord &operator=(InstrProfRecord &&) = default; |
720 | InstrProfRecord &operator=(const InstrProfRecord &RHS) { |
721 | Counts = RHS.Counts; |
722 | BitmapBytes = RHS.BitmapBytes; |
723 | if (!RHS.ValueData) { |
724 | ValueData = nullptr; |
725 | return *this; |
726 | } |
727 | if (!ValueData) |
728 | ValueData = std::make_unique<ValueProfData>(args&: *RHS.ValueData); |
729 | else |
730 | *ValueData = *RHS.ValueData; |
731 | return *this; |
732 | } |
733 | |
734 | /// Return the number of value profile kinds with non-zero number |
735 | /// of profile sites. |
736 | inline uint32_t getNumValueKinds() const; |
737 | /// Return the number of instrumented sites for ValueKind. |
738 | inline uint32_t getNumValueSites(uint32_t ValueKind) const; |
739 | |
740 | /// Return the total number of ValueData for ValueKind. |
741 | inline uint32_t getNumValueData(uint32_t ValueKind) const; |
742 | |
743 | /// Return the number of value data collected for ValueKind at profiling |
744 | /// site: Site. |
745 | inline uint32_t getNumValueDataForSite(uint32_t ValueKind, |
746 | uint32_t Site) const; |
747 | |
748 | /// Return the array of profiled values at \p Site. If \p TotalC |
749 | /// is not null, the total count of all target values at this site |
750 | /// will be stored in \c *TotalC. |
751 | inline std::unique_ptr<InstrProfValueData[]> |
752 | getValueForSite(uint32_t ValueKind, uint32_t Site, |
753 | uint64_t *TotalC = nullptr) const; |
754 | |
755 | /// Get the target value/counts of kind \p ValueKind collected at site |
756 | /// \p Site and store the result in array \p Dest. Return the total |
757 | /// counts of all target values at this site. |
758 | inline uint64_t getValueForSite(InstrProfValueData Dest[], uint32_t ValueKind, |
759 | uint32_t Site) const; |
760 | |
761 | /// Reserve space for NumValueSites sites. |
762 | inline void reserveSites(uint32_t ValueKind, uint32_t NumValueSites); |
763 | |
764 | /// Add ValueData for ValueKind at value Site. |
765 | void addValueData(uint32_t ValueKind, uint32_t Site, |
766 | InstrProfValueData *VData, uint32_t N, |
767 | InstrProfSymtab *SymTab); |
768 | |
769 | /// Merge the counts in \p Other into this one. |
770 | /// Optionally scale merged counts by \p Weight. |
771 | void merge(InstrProfRecord &Other, uint64_t Weight, |
772 | function_ref<void(instrprof_error)> Warn); |
773 | |
774 | /// Scale up profile counts (including value profile data) by |
775 | /// a factor of (N / D). |
776 | void scale(uint64_t N, uint64_t D, function_ref<void(instrprof_error)> Warn); |
777 | |
778 | /// Sort value profile data (per site) by count. |
779 | void sortValueData() { |
780 | for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) |
781 | for (auto &SR : getValueSitesForKind(ValueKind: Kind)) |
782 | SR.sortByCount(); |
783 | } |
784 | |
785 | /// Clear value data entries and edge counters. |
786 | void Clear() { |
787 | Counts.clear(); |
788 | clearValueData(); |
789 | } |
790 | |
791 | /// Clear value data entries |
792 | void clearValueData() { ValueData = nullptr; } |
793 | |
794 | /// Compute the sums of all counts and store in Sum. |
795 | void accumulateCounts(CountSumOrPercent &Sum) const; |
796 | |
797 | /// Compute the overlap b/w this IntrprofRecord and Other. |
798 | void overlap(InstrProfRecord &Other, OverlapStats &Overlap, |
799 | OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff); |
800 | |
801 | /// Compute the overlap of value profile counts. |
802 | void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src, |
803 | OverlapStats &Overlap, |
804 | OverlapStats &FuncLevelOverlap); |
805 | |
806 | enum CountPseudoKind { |
807 | NotPseudo = 0, |
808 | PseudoHot, |
809 | PseudoWarm, |
810 | }; |
811 | enum PseudoCountVal { |
812 | HotFunctionVal = -1, |
813 | WarmFunctionVal = -2, |
814 | }; |
815 | CountPseudoKind getCountPseudoKind() const { |
816 | uint64_t FirstCount = Counts[0]; |
817 | if (FirstCount == (uint64_t)HotFunctionVal) |
818 | return PseudoHot; |
819 | if (FirstCount == (uint64_t)WarmFunctionVal) |
820 | return PseudoWarm; |
821 | return NotPseudo; |
822 | } |
823 | void setPseudoCount(CountPseudoKind Kind) { |
824 | if (Kind == PseudoHot) |
825 | Counts[0] = (uint64_t)HotFunctionVal; |
826 | else if (Kind == PseudoWarm) |
827 | Counts[0] = (uint64_t)WarmFunctionVal; |
828 | } |
829 | |
830 | private: |
831 | struct ValueProfData { |
832 | std::vector<InstrProfValueSiteRecord> IndirectCallSites; |
833 | std::vector<InstrProfValueSiteRecord> MemOPSizes; |
834 | }; |
835 | std::unique_ptr<ValueProfData> ValueData; |
836 | |
837 | MutableArrayRef<InstrProfValueSiteRecord> |
838 | getValueSitesForKind(uint32_t ValueKind) { |
839 | // Cast to /add/ const (should be an implicit_cast, ideally, if that's ever |
840 | // implemented in LLVM) to call the const overload of this function, then |
841 | // cast away the constness from the result. |
842 | auto AR = const_cast<const InstrProfRecord *>(this)->getValueSitesForKind( |
843 | ValueKind); |
844 | return MutableArrayRef( |
845 | const_cast<InstrProfValueSiteRecord *>(AR.data()), AR.size()); |
846 | } |
847 | ArrayRef<InstrProfValueSiteRecord> |
848 | getValueSitesForKind(uint32_t ValueKind) const { |
849 | if (!ValueData) |
850 | return std::nullopt; |
851 | switch (ValueKind) { |
852 | case IPVK_IndirectCallTarget: |
853 | return ValueData->IndirectCallSites; |
854 | case IPVK_MemOPSize: |
855 | return ValueData->MemOPSizes; |
856 | default: |
857 | llvm_unreachable("Unknown value kind!" ); |
858 | } |
859 | } |
860 | |
861 | std::vector<InstrProfValueSiteRecord> & |
862 | getOrCreateValueSitesForKind(uint32_t ValueKind) { |
863 | if (!ValueData) |
864 | ValueData = std::make_unique<ValueProfData>(); |
865 | switch (ValueKind) { |
866 | case IPVK_IndirectCallTarget: |
867 | return ValueData->IndirectCallSites; |
868 | case IPVK_MemOPSize: |
869 | return ValueData->MemOPSizes; |
870 | default: |
871 | llvm_unreachable("Unknown value kind!" ); |
872 | } |
873 | } |
874 | |
875 | // Map indirect call target name hash to name string. |
876 | uint64_t remapValue(uint64_t Value, uint32_t ValueKind, |
877 | InstrProfSymtab *SymTab); |
878 | |
879 | // Merge Value Profile data from Src record to this record for ValueKind. |
880 | // Scale merged value counts by \p Weight. |
881 | void mergeValueProfData(uint32_t ValkeKind, InstrProfRecord &Src, |
882 | uint64_t Weight, |
883 | function_ref<void(instrprof_error)> Warn); |
884 | |
885 | // Scale up value profile data count by N (Numerator) / D (Denominator). |
886 | void scaleValueProfData(uint32_t ValueKind, uint64_t N, uint64_t D, |
887 | function_ref<void(instrprof_error)> Warn); |
888 | }; |
889 | |
890 | struct NamedInstrProfRecord : InstrProfRecord { |
891 | StringRef Name; |
892 | uint64_t Hash; |
893 | |
894 | // We reserve this bit as the flag for context sensitive profile record. |
895 | static const int CS_FLAG_IN_FUNC_HASH = 60; |
896 | |
897 | NamedInstrProfRecord() = default; |
898 | NamedInstrProfRecord(StringRef Name, uint64_t Hash, |
899 | std::vector<uint64_t> Counts) |
900 | : InstrProfRecord(std::move(Counts)), Name(Name), Hash(Hash) {} |
901 | NamedInstrProfRecord(StringRef Name, uint64_t Hash, |
902 | std::vector<uint64_t> Counts, |
903 | std::vector<uint8_t> BitmapBytes) |
904 | : InstrProfRecord(std::move(Counts), std::move(BitmapBytes)), Name(Name), |
905 | Hash(Hash) {} |
906 | |
907 | static bool hasCSFlagInHash(uint64_t FuncHash) { |
908 | return ((FuncHash >> CS_FLAG_IN_FUNC_HASH) & 1); |
909 | } |
910 | static void setCSFlagInHash(uint64_t &FuncHash) { |
911 | FuncHash |= ((uint64_t)1 << CS_FLAG_IN_FUNC_HASH); |
912 | } |
913 | }; |
914 | |
915 | uint32_t InstrProfRecord::getNumValueKinds() const { |
916 | uint32_t NumValueKinds = 0; |
917 | for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) |
918 | NumValueKinds += !(getValueSitesForKind(ValueKind: Kind).empty()); |
919 | return NumValueKinds; |
920 | } |
921 | |
922 | uint32_t InstrProfRecord::getNumValueData(uint32_t ValueKind) const { |
923 | uint32_t N = 0; |
924 | for (const auto &SR : getValueSitesForKind(ValueKind)) |
925 | N += SR.ValueData.size(); |
926 | return N; |
927 | } |
928 | |
929 | uint32_t InstrProfRecord::getNumValueSites(uint32_t ValueKind) const { |
930 | return getValueSitesForKind(ValueKind).size(); |
931 | } |
932 | |
933 | uint32_t InstrProfRecord::getNumValueDataForSite(uint32_t ValueKind, |
934 | uint32_t Site) const { |
935 | return getValueSitesForKind(ValueKind)[Site].ValueData.size(); |
936 | } |
937 | |
938 | std::unique_ptr<InstrProfValueData[]> |
939 | InstrProfRecord::getValueForSite(uint32_t ValueKind, uint32_t Site, |
940 | uint64_t *TotalC) const { |
941 | uint64_t Dummy = 0; |
942 | uint64_t &TotalCount = (TotalC == nullptr ? Dummy : *TotalC); |
943 | uint32_t N = getNumValueDataForSite(ValueKind, Site); |
944 | if (N == 0) { |
945 | TotalCount = 0; |
946 | return std::unique_ptr<InstrProfValueData[]>(nullptr); |
947 | } |
948 | |
949 | auto VD = std::make_unique<InstrProfValueData[]>(num: N); |
950 | TotalCount = getValueForSite(Dest: VD.get(), ValueKind, Site); |
951 | |
952 | return VD; |
953 | } |
954 | |
955 | uint64_t InstrProfRecord::getValueForSite(InstrProfValueData Dest[], |
956 | uint32_t ValueKind, |
957 | uint32_t Site) const { |
958 | uint32_t I = 0; |
959 | uint64_t TotalCount = 0; |
960 | for (auto V : getValueSitesForKind(ValueKind)[Site].ValueData) { |
961 | Dest[I].Value = V.Value; |
962 | Dest[I].Count = V.Count; |
963 | TotalCount = SaturatingAdd(X: TotalCount, Y: V.Count); |
964 | I++; |
965 | } |
966 | return TotalCount; |
967 | } |
968 | |
969 | void InstrProfRecord::reserveSites(uint32_t ValueKind, uint32_t NumValueSites) { |
970 | if (!NumValueSites) |
971 | return; |
972 | getOrCreateValueSitesForKind(ValueKind).reserve(n: NumValueSites); |
973 | } |
974 | |
975 | // Include definitions for value profile data |
976 | #define INSTR_PROF_VALUE_PROF_DATA |
977 | #include "llvm/ProfileData/InstrProfData.inc" |
978 | |
979 | void InstrProfValueSiteRecord::sortByCount() { |
980 | ValueData.sort( |
981 | comp: [](const InstrProfValueData &left, const InstrProfValueData &right) { |
982 | return left.Count > right.Count; |
983 | }); |
984 | // Now truncate |
985 | size_t max_s = INSTR_PROF_MAX_NUM_VAL_PER_SITE; |
986 | if (ValueData.size() > max_s) |
987 | ValueData.resize(new_size: max_s); |
988 | } |
989 | |
990 | namespace IndexedInstrProf { |
991 | |
992 | enum class HashT : uint32_t { |
993 | MD5, |
994 | Last = MD5 |
995 | }; |
996 | |
997 | inline uint64_t ComputeHash(HashT Type, StringRef K) { |
998 | switch (Type) { |
999 | case HashT::MD5: |
1000 | return MD5Hash(Str: K); |
1001 | } |
1002 | llvm_unreachable("Unhandled hash type" ); |
1003 | } |
1004 | |
1005 | const uint64_t Magic = 0x8169666f72706cff; // "\xfflprofi\x81" |
1006 | |
1007 | enum ProfVersion { |
1008 | // Version 1 is the first version. In this version, the value of |
1009 | // a key/value pair can only include profile data of a single function. |
1010 | // Due to this restriction, the number of block counters for a given |
1011 | // function is not recorded but derived from the length of the value. |
1012 | Version1 = 1, |
1013 | // The version 2 format supports recording profile data of multiple |
1014 | // functions which share the same key in one value field. To support this, |
1015 | // the number block counters is recorded as an uint64_t field right after the |
1016 | // function structural hash. |
1017 | Version2 = 2, |
1018 | // Version 3 supports value profile data. The value profile data is expected |
1019 | // to follow the block counter profile data. |
1020 | Version3 = 3, |
1021 | // In this version, profile summary data \c IndexedInstrProf::Summary is |
1022 | // stored after the profile header. |
1023 | Version4 = 4, |
1024 | // In this version, the frontend PGO stable hash algorithm defaults to V2. |
1025 | Version5 = 5, |
1026 | // In this version, the frontend PGO stable hash algorithm got fixed and |
1027 | // may produce hashes different from Version5. |
1028 | Version6 = 6, |
1029 | // An additional counter is added around logical operators. |
1030 | Version7 = 7, |
1031 | // An additional (optional) memory profile type is added. |
1032 | Version8 = 8, |
1033 | // Binary ids are added. |
1034 | Version9 = 9, |
1035 | // An additional (optional) temporal profile traces section is added. |
1036 | Version10 = 10, |
1037 | // An additional field is used for bitmap bytes. |
1038 | Version11 = 11, |
1039 | // The current version is 11. |
1040 | CurrentVersion = INSTR_PROF_INDEX_VERSION |
1041 | }; |
1042 | const uint64_t Version = ProfVersion::CurrentVersion; |
1043 | |
1044 | const HashT HashType = HashT::MD5; |
1045 | |
1046 | inline uint64_t ComputeHash(StringRef K) { return ComputeHash(Type: HashType, K); } |
1047 | |
1048 | // This structure defines the file header of the LLVM profile |
1049 | // data file in indexed-format. Please update llvm/docs/InstrProfileFormat.rst |
1050 | // as appropriate when updating the indexed profile format. |
1051 | struct { |
1052 | uint64_t ; |
1053 | uint64_t ; |
1054 | uint64_t ; // Becomes unused since version 4 |
1055 | uint64_t ; |
1056 | uint64_t ; |
1057 | uint64_t ; |
1058 | uint64_t ; |
1059 | uint64_t ; |
1060 | // New fields should only be added at the end to ensure that the size |
1061 | // computation is correct. The methods below need to be updated to ensure that |
1062 | // the new field is read correctly. |
1063 | |
1064 | // Reads a header struct from the buffer. |
1065 | static Expected<Header> (const unsigned char *Buffer); |
1066 | |
1067 | // Returns the size of the header in bytes for all valid fields based on the |
1068 | // version. I.e a older version header will return a smaller size. |
1069 | size_t () const; |
1070 | |
1071 | // Returns the format version in little endian. The header retains the version |
1072 | // in native endian of the compiler runtime. |
1073 | uint64_t () const; |
1074 | }; |
1075 | |
1076 | // Profile summary data recorded in the profile data file in indexed |
1077 | // format. It is introduced in version 4. The summary data follows |
1078 | // right after the profile file header. |
1079 | struct Summary { |
1080 | struct Entry { |
1081 | uint64_t Cutoff; ///< The required percentile of total execution count. |
1082 | uint64_t |
1083 | MinBlockCount; ///< The minimum execution count for this percentile. |
1084 | uint64_t NumBlocks; ///< Number of blocks >= the minumum execution count. |
1085 | }; |
1086 | // The field kind enumerator to assigned value mapping should remain |
1087 | // unchanged when a new kind is added or an old kind gets deleted in |
1088 | // the future. |
1089 | enum SummaryFieldKind { |
1090 | /// The total number of functions instrumented. |
1091 | TotalNumFunctions = 0, |
1092 | /// Total number of instrumented blocks/edges. |
1093 | TotalNumBlocks = 1, |
1094 | /// The maximal execution count among all functions. |
1095 | /// This field does not exist for profile data from IR based |
1096 | /// instrumentation. |
1097 | MaxFunctionCount = 2, |
1098 | /// Max block count of the program. |
1099 | MaxBlockCount = 3, |
1100 | /// Max internal block count of the program (excluding entry blocks). |
1101 | MaxInternalBlockCount = 4, |
1102 | /// The sum of all instrumented block counts. |
1103 | TotalBlockCount = 5, |
1104 | NumKinds = TotalBlockCount + 1 |
1105 | }; |
1106 | |
1107 | // The number of summmary fields following the summary header. |
1108 | uint64_t NumSummaryFields; |
1109 | // The number of Cutoff Entries (Summary::Entry) following summary fields. |
1110 | uint64_t NumCutoffEntries; |
1111 | |
1112 | Summary() = delete; |
1113 | Summary(uint32_t Size) { memset(s: this, c: 0, n: Size); } |
1114 | |
1115 | void operator delete(void *ptr) { ::operator delete(ptr); } |
1116 | |
1117 | static uint32_t getSize(uint32_t NumSumFields, uint32_t NumCutoffEntries) { |
1118 | return sizeof(Summary) + NumCutoffEntries * sizeof(Entry) + |
1119 | NumSumFields * sizeof(uint64_t); |
1120 | } |
1121 | |
1122 | const uint64_t *getSummaryDataBase() const { |
1123 | return reinterpret_cast<const uint64_t *>(this + 1); |
1124 | } |
1125 | |
1126 | uint64_t *getSummaryDataBase() { |
1127 | return reinterpret_cast<uint64_t *>(this + 1); |
1128 | } |
1129 | |
1130 | const Entry *getCutoffEntryBase() const { |
1131 | return reinterpret_cast<const Entry *>( |
1132 | &getSummaryDataBase()[NumSummaryFields]); |
1133 | } |
1134 | |
1135 | Entry *getCutoffEntryBase() { |
1136 | return reinterpret_cast<Entry *>(&getSummaryDataBase()[NumSummaryFields]); |
1137 | } |
1138 | |
1139 | uint64_t get(SummaryFieldKind K) const { |
1140 | return getSummaryDataBase()[K]; |
1141 | } |
1142 | |
1143 | void set(SummaryFieldKind K, uint64_t V) { |
1144 | getSummaryDataBase()[K] = V; |
1145 | } |
1146 | |
1147 | const Entry &getEntry(uint32_t I) const { return getCutoffEntryBase()[I]; } |
1148 | |
1149 | void setEntry(uint32_t I, const ProfileSummaryEntry &E) { |
1150 | Entry &ER = getCutoffEntryBase()[I]; |
1151 | ER.Cutoff = E.Cutoff; |
1152 | ER.MinBlockCount = E.MinCount; |
1153 | ER.NumBlocks = E.NumCounts; |
1154 | } |
1155 | }; |
1156 | |
1157 | inline std::unique_ptr<Summary> allocSummary(uint32_t TotalSize) { |
1158 | return std::unique_ptr<Summary>(new (::operator new(TotalSize)) |
1159 | Summary(TotalSize)); |
1160 | } |
1161 | |
1162 | } // end namespace IndexedInstrProf |
1163 | |
1164 | namespace RawInstrProf { |
1165 | |
1166 | // Version 1: First version |
1167 | // Version 2: Added value profile data section. Per-function control data |
1168 | // struct has more fields to describe value profile information. |
1169 | // Version 3: Compressed name section support. Function PGO name reference |
1170 | // from control data struct is changed from raw pointer to Name's MD5 value. |
1171 | // Version 4: ValueDataBegin and ValueDataSizes fields are removed from the |
1172 | // raw header. |
1173 | // Version 5: Bit 60 of FuncHash is reserved for the flag for the context |
1174 | // sensitive records. |
1175 | // Version 6: Added binary id. |
1176 | // Version 7: Reorder binary id and include version in signature. |
1177 | // Version 8: Use relative counter pointer. |
1178 | // Version 9: Added relative bitmap bytes pointer and count used by MC/DC. |
1179 | const uint64_t Version = INSTR_PROF_RAW_VERSION; |
1180 | |
1181 | template <class IntPtrT> inline uint64_t getMagic(); |
1182 | template <> inline uint64_t getMagic<uint64_t>() { |
1183 | return INSTR_PROF_RAW_MAGIC_64; |
1184 | } |
1185 | |
1186 | template <> inline uint64_t getMagic<uint32_t>() { |
1187 | return INSTR_PROF_RAW_MAGIC_32; |
1188 | } |
1189 | |
1190 | // Per-function profile data header/control structure. |
1191 | // The definition should match the structure defined in |
1192 | // compiler-rt/lib/profile/InstrProfiling.h. |
1193 | // It should also match the synthesized type in |
1194 | // Transforms/Instrumentation/InstrProfiling.cpp:getOrCreateRegionCounters. |
1195 | template <class IntPtrT> struct alignas(8) ProfileData { |
1196 | #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Type Name; |
1197 | #include "llvm/ProfileData/InstrProfData.inc" |
1198 | }; |
1199 | |
1200 | // File header structure of the LLVM profile data in raw format. |
1201 | // The definition should match the header referenced in |
1202 | // compiler-rt/lib/profile/InstrProfilingFile.c and |
1203 | // InstrProfilingBuffer.c. |
1204 | struct { |
1205 | #define (Type, Name, Init) const Type Name; |
1206 | #include "llvm/ProfileData/InstrProfData.inc" |
1207 | }; |
1208 | |
1209 | } // end namespace RawInstrProf |
1210 | |
1211 | // Create the variable for the profile file name. |
1212 | void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput); |
1213 | |
1214 | // Whether to compress function names in profile records, and filenames in |
1215 | // code coverage mappings. Used by the Instrumentation library and unit tests. |
1216 | extern cl::opt<bool> DoInstrProfNameCompression; |
1217 | |
1218 | } // end namespace llvm |
1219 | #endif // LLVM_PROFILEDATA_INSTRPROF_H |
1220 | |