1 | //===- InstrProfReader.h - Instrumented profiling readers -------*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file contains support for reading profiling data for instrumentation |
10 | // based PGO and coverage. |
11 | // |
12 | //===----------------------------------------------------------------------===// |
13 | |
14 | #ifndef LLVM_PROFILEDATA_INSTRPROFREADER_H |
15 | #define LLVM_PROFILEDATA_INSTRPROFREADER_H |
16 | |
17 | #include "llvm/ADT/ArrayRef.h" |
18 | #include "llvm/ADT/StringRef.h" |
19 | #include "llvm/IR/ProfileSummary.h" |
20 | #include "llvm/Object/BuildID.h" |
21 | #include "llvm/ProfileData/InstrProf.h" |
22 | #include "llvm/ProfileData/InstrProfCorrelator.h" |
23 | #include "llvm/ProfileData/MemProf.h" |
24 | #include "llvm/Support/Endian.h" |
25 | #include "llvm/Support/Error.h" |
26 | #include "llvm/Support/LineIterator.h" |
27 | #include "llvm/Support/MathExtras.h" |
28 | #include "llvm/Support/MemoryBuffer.h" |
29 | #include "llvm/Support/OnDiskHashTable.h" |
30 | #include "llvm/Support/SwapByteOrder.h" |
31 | #include <algorithm> |
32 | #include <cassert> |
33 | #include <cstddef> |
34 | #include <cstdint> |
35 | #include <iterator> |
36 | #include <memory> |
37 | #include <utility> |
38 | #include <vector> |
39 | |
40 | namespace llvm { |
41 | |
42 | class InstrProfReader; |
43 | |
44 | namespace vfs { |
45 | class FileSystem; |
46 | } // namespace vfs |
47 | |
48 | /// A file format agnostic iterator over profiling data. |
49 | template <class record_type = NamedInstrProfRecord, |
50 | class reader_type = InstrProfReader> |
51 | class InstrProfIterator { |
52 | public: |
53 | using iterator_category = std::input_iterator_tag; |
54 | using value_type = record_type; |
55 | using difference_type = std::ptrdiff_t; |
56 | using pointer = value_type *; |
57 | using reference = value_type &; |
58 | |
59 | private: |
60 | reader_type *Reader = nullptr; |
61 | value_type Record; |
62 | |
63 | void increment() { |
64 | if (Error E = Reader->readNextRecord(Record)) { |
65 | // Handle errors in the reader. |
66 | InstrProfError::take(E: std::move(E)); |
67 | *this = InstrProfIterator(); |
68 | } |
69 | } |
70 | |
71 | public: |
72 | InstrProfIterator() = default; |
73 | InstrProfIterator(reader_type *Reader) : Reader(Reader) { increment(); } |
74 | |
75 | InstrProfIterator &operator++() { |
76 | increment(); |
77 | return *this; |
78 | } |
79 | bool operator==(const InstrProfIterator &RHS) const { |
80 | return Reader == RHS.Reader; |
81 | } |
82 | bool operator!=(const InstrProfIterator &RHS) const { |
83 | return Reader != RHS.Reader; |
84 | } |
85 | value_type &operator*() { return Record; } |
86 | value_type *operator->() { return &Record; } |
87 | }; |
88 | |
89 | /// Base class and interface for reading profiling data of any known instrprof |
90 | /// format. Provides an iterator over NamedInstrProfRecords. |
91 | class InstrProfReader { |
92 | instrprof_error LastError = instrprof_error::success; |
93 | std::string LastErrorMsg; |
94 | |
95 | public: |
96 | InstrProfReader() = default; |
97 | virtual ~InstrProfReader() = default; |
98 | |
99 | /// Read the header. Required before reading first record. |
100 | virtual Error () = 0; |
101 | |
102 | /// Read a single record. |
103 | virtual Error readNextRecord(NamedInstrProfRecord &Record) = 0; |
104 | |
105 | /// Read a list of binary ids. |
106 | virtual Error readBinaryIds(std::vector<llvm::object::BuildID> &BinaryIds) { |
107 | return success(); |
108 | } |
109 | |
110 | /// Print binary ids. |
111 | virtual Error printBinaryIds(raw_ostream &OS) { return success(); }; |
112 | |
113 | /// Iterator over profile data. |
114 | InstrProfIterator<> begin() { return InstrProfIterator<>(this); } |
115 | InstrProfIterator<> end() { return InstrProfIterator<>(); } |
116 | |
117 | /// Return the profile version. |
118 | virtual uint64_t getVersion() const = 0; |
119 | |
120 | virtual bool isIRLevelProfile() const = 0; |
121 | |
122 | virtual bool hasCSIRLevelProfile() const = 0; |
123 | |
124 | virtual bool instrEntryBBEnabled() const = 0; |
125 | |
126 | /// Return true if the profile has single byte counters representing coverage. |
127 | virtual bool hasSingleByteCoverage() const = 0; |
128 | |
129 | /// Return true if the profile only instruments function entries. |
130 | virtual bool functionEntryOnly() const = 0; |
131 | |
132 | /// Return true if profile includes a memory profile. |
133 | virtual bool hasMemoryProfile() const = 0; |
134 | |
135 | /// Return true if this has a temporal profile. |
136 | virtual bool hasTemporalProfile() const = 0; |
137 | |
138 | /// Returns a BitsetEnum describing the attributes of the profile. To check |
139 | /// individual attributes prefer using the helpers above. |
140 | virtual InstrProfKind getProfileKind() const = 0; |
141 | |
142 | /// Return the PGO symtab. There are three different readers: |
143 | /// Raw, Text, and Indexed profile readers. The first two types |
144 | /// of readers are used only by llvm-profdata tool, while the indexed |
145 | /// profile reader is also used by llvm-cov tool and the compiler ( |
146 | /// backend or frontend). Since creating PGO symtab can create |
147 | /// significant runtime and memory overhead (as it touches data |
148 | /// for the whole program), InstrProfSymtab for the indexed profile |
149 | /// reader should be created on demand and it is recommended to be |
150 | /// only used for dumping purpose with llvm-proftool, not with the |
151 | /// compiler. |
152 | virtual InstrProfSymtab &getSymtab() = 0; |
153 | |
154 | /// Compute the sum of counts and return in Sum. |
155 | void accumulateCounts(CountSumOrPercent &Sum, bool IsCS); |
156 | |
157 | protected: |
158 | std::unique_ptr<InstrProfSymtab> Symtab; |
159 | /// A list of temporal profile traces. |
160 | SmallVector<TemporalProfTraceTy> TemporalProfTraces; |
161 | /// The total number of temporal profile traces seen. |
162 | uint64_t TemporalProfTraceStreamSize = 0; |
163 | |
164 | /// Set the current error and return same. |
165 | Error error(instrprof_error Err, const std::string &ErrMsg = "" ) { |
166 | LastError = Err; |
167 | LastErrorMsg = ErrMsg; |
168 | if (Err == instrprof_error::success) |
169 | return Error::success(); |
170 | return make_error<InstrProfError>(Args&: Err, Args: ErrMsg); |
171 | } |
172 | |
173 | Error error(Error &&E) { |
174 | handleAllErrors(E: std::move(E), Handlers: [&](const InstrProfError &IPE) { |
175 | LastError = IPE.get(); |
176 | LastErrorMsg = IPE.getMessage(); |
177 | }); |
178 | return make_error<InstrProfError>(Args&: LastError, Args&: LastErrorMsg); |
179 | } |
180 | |
181 | /// Clear the current error and return a successful one. |
182 | Error success() { return error(Err: instrprof_error::success); } |
183 | |
184 | public: |
185 | /// Return true if the reader has finished reading the profile data. |
186 | bool isEOF() { return LastError == instrprof_error::eof; } |
187 | |
188 | /// Return true if the reader encountered an error reading profiling data. |
189 | bool hasError() { return LastError != instrprof_error::success && !isEOF(); } |
190 | |
191 | /// Get the current error. |
192 | Error getError() { |
193 | if (hasError()) |
194 | return make_error<InstrProfError>(Args&: LastError, Args&: LastErrorMsg); |
195 | return Error::success(); |
196 | } |
197 | |
198 | /// Factory method to create an appropriately typed reader for the given |
199 | /// instrprof file. |
200 | static Expected<std::unique_ptr<InstrProfReader>> |
201 | create(const Twine &Path, vfs::FileSystem &FS, |
202 | const InstrProfCorrelator *Correlator = nullptr, |
203 | std::function<void(Error)> Warn = nullptr); |
204 | |
205 | static Expected<std::unique_ptr<InstrProfReader>> |
206 | create(std::unique_ptr<MemoryBuffer> Buffer, |
207 | const InstrProfCorrelator *Correlator = nullptr, |
208 | std::function<void(Error)> Warn = nullptr); |
209 | |
210 | /// \param Weight for raw profiles use this as the temporal profile trace |
211 | /// weight |
212 | /// \returns a list of temporal profile traces. |
213 | virtual SmallVector<TemporalProfTraceTy> & |
214 | getTemporalProfTraces(std::optional<uint64_t> Weight = {}) { |
215 | // For non-raw profiles we ignore the input weight and instead use the |
216 | // weights already in the traces. |
217 | return TemporalProfTraces; |
218 | } |
219 | /// \returns the total number of temporal profile traces seen. |
220 | uint64_t getTemporalProfTraceStreamSize() { |
221 | return TemporalProfTraceStreamSize; |
222 | } |
223 | }; |
224 | |
225 | /// Reader for the simple text based instrprof format. |
226 | /// |
227 | /// This format is a simple text format that's suitable for test data. Records |
228 | /// are separated by one or more blank lines, and record fields are separated by |
229 | /// new lines. |
230 | /// |
231 | /// Each record consists of a function name, a function hash, a number of |
232 | /// counters, and then each counter value, in that order. |
233 | class TextInstrProfReader : public InstrProfReader { |
234 | private: |
235 | /// The profile data file contents. |
236 | std::unique_ptr<MemoryBuffer> DataBuffer; |
237 | /// Iterator over the profile data. |
238 | line_iterator Line; |
239 | /// The attributes of the current profile. |
240 | InstrProfKind ProfileKind = InstrProfKind::Unknown; |
241 | |
242 | Error readValueProfileData(InstrProfRecord &Record); |
243 | |
244 | Error readTemporalProfTraceData(); |
245 | |
246 | public: |
247 | TextInstrProfReader(std::unique_ptr<MemoryBuffer> DataBuffer_) |
248 | : DataBuffer(std::move(DataBuffer_)), Line(*DataBuffer, true, '#') {} |
249 | TextInstrProfReader(const TextInstrProfReader &) = delete; |
250 | TextInstrProfReader &operator=(const TextInstrProfReader &) = delete; |
251 | |
252 | /// Return true if the given buffer is in text instrprof format. |
253 | static bool hasFormat(const MemoryBuffer &Buffer); |
254 | |
255 | // Text format does not have version, so return 0. |
256 | uint64_t getVersion() const override { return 0; } |
257 | |
258 | bool isIRLevelProfile() const override { |
259 | return static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation); |
260 | } |
261 | |
262 | bool hasCSIRLevelProfile() const override { |
263 | return static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive); |
264 | } |
265 | |
266 | bool instrEntryBBEnabled() const override { |
267 | return static_cast<bool>(ProfileKind & |
268 | InstrProfKind::FunctionEntryInstrumentation); |
269 | } |
270 | |
271 | bool hasSingleByteCoverage() const override { |
272 | return static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage); |
273 | } |
274 | |
275 | bool functionEntryOnly() const override { |
276 | return static_cast<bool>(ProfileKind & InstrProfKind::FunctionEntryOnly); |
277 | } |
278 | |
279 | bool hasMemoryProfile() const override { |
280 | // TODO: Add support for text format memory profiles. |
281 | return false; |
282 | } |
283 | |
284 | bool hasTemporalProfile() const override { |
285 | return static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile); |
286 | } |
287 | |
288 | InstrProfKind getProfileKind() const override { return ProfileKind; } |
289 | |
290 | /// Read the header. |
291 | Error () override; |
292 | |
293 | /// Read a single record. |
294 | Error readNextRecord(NamedInstrProfRecord &Record) override; |
295 | |
296 | InstrProfSymtab &getSymtab() override { |
297 | assert(Symtab); |
298 | return *Symtab; |
299 | } |
300 | }; |
301 | |
302 | /// Reader for the raw instrprof binary format from runtime. |
303 | /// |
304 | /// This format is a raw memory dump of the instrumentation-based profiling data |
305 | /// from the runtime. It has no index. |
306 | /// |
307 | /// Templated on the unsigned type whose size matches pointers on the platform |
308 | /// that wrote the profile. |
309 | template <class IntPtrT> |
310 | class RawInstrProfReader : public InstrProfReader { |
311 | private: |
312 | /// The profile data file contents. |
313 | std::unique_ptr<MemoryBuffer> DataBuffer; |
314 | /// If available, this hold the ProfileData array used to correlate raw |
315 | /// instrumentation data to their functions. |
316 | const InstrProfCorrelatorImpl<IntPtrT> *Correlator; |
317 | /// A list of timestamps paired with a function name reference. |
318 | std::vector<std::pair<uint64_t, uint64_t>> TemporalProfTimestamps; |
319 | bool ShouldSwapBytes; |
320 | // The value of the version field of the raw profile data header. The lower 32 |
321 | // bits specifies the format version and the most significant 32 bits specify |
322 | // the variant types of the profile. |
323 | uint64_t Version; |
324 | uint64_t CountersDelta; |
325 | uint64_t BitmapDelta; |
326 | uint64_t NamesDelta; |
327 | const RawInstrProf::ProfileData<IntPtrT> *Data; |
328 | const RawInstrProf::ProfileData<IntPtrT> *DataEnd; |
329 | const char *; |
330 | const char *CountersEnd; |
331 | const char *BitmapStart; |
332 | const char *BitmapEnd; |
333 | const char *NamesStart; |
334 | const char *NamesEnd; |
335 | // After value profile is all read, this pointer points to |
336 | // the header of next profile data (if exists) |
337 | const uint8_t *ValueDataStart; |
338 | uint32_t ValueKindLast; |
339 | uint32_t CurValueDataSize; |
340 | std::vector<llvm::object::BuildID> BinaryIds; |
341 | |
342 | std::function<void(Error)> Warn; |
343 | |
344 | /// Maxium counter value 2^56. |
345 | static const uint64_t MaxCounterValue = (1ULL << 56); |
346 | |
347 | public: |
348 | RawInstrProfReader(std::unique_ptr<MemoryBuffer> DataBuffer, |
349 | const InstrProfCorrelator *Correlator, |
350 | std::function<void(Error)> Warn) |
351 | : DataBuffer(std::move(DataBuffer)), |
352 | Correlator(dyn_cast_or_null<const InstrProfCorrelatorImpl<IntPtrT>>( |
353 | Correlator)), |
354 | Warn(Warn) {} |
355 | RawInstrProfReader(const RawInstrProfReader &) = delete; |
356 | RawInstrProfReader &operator=(const RawInstrProfReader &) = delete; |
357 | |
358 | static bool hasFormat(const MemoryBuffer &DataBuffer); |
359 | Error () override; |
360 | Error readNextRecord(NamedInstrProfRecord &Record) override; |
361 | Error readBinaryIds(std::vector<llvm::object::BuildID> &BinaryIds) override; |
362 | Error printBinaryIds(raw_ostream &OS) override; |
363 | |
364 | uint64_t getVersion() const override { return Version; } |
365 | |
366 | bool isIRLevelProfile() const override { |
367 | return (Version & VARIANT_MASK_IR_PROF) != 0; |
368 | } |
369 | |
370 | bool hasCSIRLevelProfile() const override { |
371 | return (Version & VARIANT_MASK_CSIR_PROF) != 0; |
372 | } |
373 | |
374 | bool instrEntryBBEnabled() const override { |
375 | return (Version & VARIANT_MASK_INSTR_ENTRY) != 0; |
376 | } |
377 | |
378 | bool hasSingleByteCoverage() const override { |
379 | return (Version & VARIANT_MASK_BYTE_COVERAGE) != 0; |
380 | } |
381 | |
382 | bool functionEntryOnly() const override { |
383 | return (Version & VARIANT_MASK_FUNCTION_ENTRY_ONLY) != 0; |
384 | } |
385 | |
386 | bool hasMemoryProfile() const override { |
387 | // Memory profiles have a separate raw format, so this should never be set. |
388 | assert(!(Version & VARIANT_MASK_MEMPROF)); |
389 | return false; |
390 | } |
391 | |
392 | bool hasTemporalProfile() const override { |
393 | return (Version & VARIANT_MASK_TEMPORAL_PROF) != 0; |
394 | } |
395 | |
396 | /// Returns a BitsetEnum describing the attributes of the raw instr profile. |
397 | InstrProfKind getProfileKind() const override; |
398 | |
399 | InstrProfSymtab &getSymtab() override { |
400 | assert(Symtab.get()); |
401 | return *Symtab.get(); |
402 | } |
403 | |
404 | SmallVector<TemporalProfTraceTy> & |
405 | getTemporalProfTraces(std::optional<uint64_t> Weight = {}) override; |
406 | |
407 | private: |
408 | Error createSymtab(InstrProfSymtab &Symtab); |
409 | Error (const char *CurrentPos); |
410 | Error (const RawInstrProf::Header &); |
411 | |
412 | template <class IntT> IntT swap(IntT Int) const { |
413 | return ShouldSwapBytes ? llvm::byteswap(Int) : Int; |
414 | } |
415 | |
416 | llvm::endianness getDataEndianness() const { |
417 | if (!ShouldSwapBytes) |
418 | return llvm::endianness::native; |
419 | if (llvm::endianness::native == llvm::endianness::little) |
420 | return llvm::endianness::big; |
421 | else |
422 | return llvm::endianness::little; |
423 | } |
424 | |
425 | inline uint8_t getNumPaddingBytes(uint64_t SizeInBytes) { |
426 | return 7 & (sizeof(uint64_t) - SizeInBytes % sizeof(uint64_t)); |
427 | } |
428 | |
429 | Error readName(NamedInstrProfRecord &Record); |
430 | Error readFuncHash(NamedInstrProfRecord &Record); |
431 | Error readRawCounts(InstrProfRecord &Record); |
432 | Error readRawBitmapBytes(InstrProfRecord &Record); |
433 | Error readValueProfilingData(InstrProfRecord &Record); |
434 | bool atEnd() const { return Data == DataEnd; } |
435 | |
436 | void advanceData() { |
437 | // `CountersDelta` is a constant zero when using debug info correlation. |
438 | if (!Correlator) { |
439 | // The initial CountersDelta is the in-memory address difference between |
440 | // the data and counts sections: |
441 | // start(__llvm_prf_cnts) - start(__llvm_prf_data) |
442 | // As we advance to the next record, we maintain the correct CountersDelta |
443 | // with respect to the next record. |
444 | CountersDelta -= sizeof(*Data); |
445 | BitmapDelta -= sizeof(*Data); |
446 | } |
447 | Data++; |
448 | ValueDataStart += CurValueDataSize; |
449 | } |
450 | |
451 | const char *() const { |
452 | assert(atEnd()); |
453 | return (const char *)ValueDataStart; |
454 | } |
455 | |
456 | StringRef getName(uint64_t NameRef) const { |
457 | return Symtab->getFuncOrVarName(MD5Hash: swap(NameRef)); |
458 | } |
459 | |
460 | int getCounterTypeSize() const { |
461 | return hasSingleByteCoverage() ? sizeof(uint8_t) : sizeof(uint64_t); |
462 | } |
463 | }; |
464 | |
465 | using RawInstrProfReader32 = RawInstrProfReader<uint32_t>; |
466 | using RawInstrProfReader64 = RawInstrProfReader<uint64_t>; |
467 | |
468 | namespace IndexedInstrProf { |
469 | |
470 | enum class HashT : uint32_t; |
471 | |
472 | } // end namespace IndexedInstrProf |
473 | |
474 | /// Trait for lookups into the on-disk hash table for the binary instrprof |
475 | /// format. |
476 | class InstrProfLookupTrait { |
477 | std::vector<NamedInstrProfRecord> DataBuffer; |
478 | IndexedInstrProf::HashT HashType; |
479 | unsigned FormatVersion; |
480 | // Endianness of the input value profile data. |
481 | // It should be LE by default, but can be changed |
482 | // for testing purpose. |
483 | llvm::endianness ValueProfDataEndianness = llvm::endianness::little; |
484 | |
485 | public: |
486 | InstrProfLookupTrait(IndexedInstrProf::HashT HashType, unsigned FormatVersion) |
487 | : HashType(HashType), FormatVersion(FormatVersion) {} |
488 | |
489 | using data_type = ArrayRef<NamedInstrProfRecord>; |
490 | |
491 | using internal_key_type = StringRef; |
492 | using external_key_type = StringRef; |
493 | using hash_value_type = uint64_t; |
494 | using offset_type = uint64_t; |
495 | |
496 | static bool EqualKey(StringRef A, StringRef B) { return A == B; } |
497 | static StringRef GetInternalKey(StringRef K) { return K; } |
498 | static StringRef GetExternalKey(StringRef K) { return K; } |
499 | |
500 | hash_value_type ComputeHash(StringRef K); |
501 | |
502 | static std::pair<offset_type, offset_type> |
503 | ReadKeyDataLength(const unsigned char *&D) { |
504 | using namespace support; |
505 | |
506 | offset_type KeyLen = |
507 | endian::readNext<offset_type, llvm::endianness::little, unaligned>(memory&: D); |
508 | offset_type DataLen = |
509 | endian::readNext<offset_type, llvm::endianness::little, unaligned>(memory&: D); |
510 | return std::make_pair(x&: KeyLen, y&: DataLen); |
511 | } |
512 | |
513 | StringRef ReadKey(const unsigned char *D, offset_type N) { |
514 | return StringRef((const char *)D, N); |
515 | } |
516 | |
517 | bool readValueProfilingData(const unsigned char *&D, |
518 | const unsigned char *const End); |
519 | data_type ReadData(StringRef K, const unsigned char *D, offset_type N); |
520 | |
521 | // Used for testing purpose only. |
522 | void setValueProfDataEndianness(llvm::endianness Endianness) { |
523 | ValueProfDataEndianness = Endianness; |
524 | } |
525 | }; |
526 | |
527 | struct InstrProfReaderIndexBase { |
528 | virtual ~InstrProfReaderIndexBase() = default; |
529 | |
530 | // Read all the profile records with the same key pointed to the current |
531 | // iterator. |
532 | virtual Error getRecords(ArrayRef<NamedInstrProfRecord> &Data) = 0; |
533 | |
534 | // Read all the profile records with the key equal to FuncName |
535 | virtual Error getRecords(StringRef FuncName, |
536 | ArrayRef<NamedInstrProfRecord> &Data) = 0; |
537 | virtual void advanceToNextKey() = 0; |
538 | virtual bool atEnd() const = 0; |
539 | virtual void setValueProfDataEndianness(llvm::endianness Endianness) = 0; |
540 | virtual uint64_t getVersion() const = 0; |
541 | virtual bool isIRLevelProfile() const = 0; |
542 | virtual bool hasCSIRLevelProfile() const = 0; |
543 | virtual bool instrEntryBBEnabled() const = 0; |
544 | virtual bool hasSingleByteCoverage() const = 0; |
545 | virtual bool functionEntryOnly() const = 0; |
546 | virtual bool hasMemoryProfile() const = 0; |
547 | virtual bool hasTemporalProfile() const = 0; |
548 | virtual InstrProfKind getProfileKind() const = 0; |
549 | virtual Error populateSymtab(InstrProfSymtab &) = 0; |
550 | }; |
551 | |
552 | using OnDiskHashTableImplV3 = |
553 | OnDiskIterableChainedHashTable<InstrProfLookupTrait>; |
554 | |
555 | using MemProfRecordHashTable = |
556 | OnDiskIterableChainedHashTable<memprof::RecordLookupTrait>; |
557 | using MemProfFrameHashTable = |
558 | OnDiskIterableChainedHashTable<memprof::FrameLookupTrait>; |
559 | |
560 | template <typename HashTableImpl> |
561 | class InstrProfReaderItaniumRemapper; |
562 | |
563 | template <typename HashTableImpl> |
564 | class InstrProfReaderIndex : public InstrProfReaderIndexBase { |
565 | private: |
566 | std::unique_ptr<HashTableImpl> HashTable; |
567 | typename HashTableImpl::data_iterator RecordIterator; |
568 | uint64_t FormatVersion; |
569 | |
570 | friend class InstrProfReaderItaniumRemapper<HashTableImpl>; |
571 | |
572 | public: |
573 | InstrProfReaderIndex(const unsigned char *Buckets, |
574 | const unsigned char *const Payload, |
575 | const unsigned char *const Base, |
576 | IndexedInstrProf::HashT HashType, uint64_t Version); |
577 | ~InstrProfReaderIndex() override = default; |
578 | |
579 | Error getRecords(ArrayRef<NamedInstrProfRecord> &Data) override; |
580 | Error getRecords(StringRef FuncName, |
581 | ArrayRef<NamedInstrProfRecord> &Data) override; |
582 | void advanceToNextKey() override { RecordIterator++; } |
583 | |
584 | bool atEnd() const override { |
585 | return RecordIterator == HashTable->data_end(); |
586 | } |
587 | |
588 | void setValueProfDataEndianness(llvm::endianness Endianness) override { |
589 | HashTable->getInfoObj().setValueProfDataEndianness(Endianness); |
590 | } |
591 | |
592 | uint64_t getVersion() const override { return GET_VERSION(FormatVersion); } |
593 | |
594 | bool isIRLevelProfile() const override { |
595 | return (FormatVersion & VARIANT_MASK_IR_PROF) != 0; |
596 | } |
597 | |
598 | bool hasCSIRLevelProfile() const override { |
599 | return (FormatVersion & VARIANT_MASK_CSIR_PROF) != 0; |
600 | } |
601 | |
602 | bool instrEntryBBEnabled() const override { |
603 | return (FormatVersion & VARIANT_MASK_INSTR_ENTRY) != 0; |
604 | } |
605 | |
606 | bool hasSingleByteCoverage() const override { |
607 | return (FormatVersion & VARIANT_MASK_BYTE_COVERAGE) != 0; |
608 | } |
609 | |
610 | bool functionEntryOnly() const override { |
611 | return (FormatVersion & VARIANT_MASK_FUNCTION_ENTRY_ONLY) != 0; |
612 | } |
613 | |
614 | bool hasMemoryProfile() const override { |
615 | return (FormatVersion & VARIANT_MASK_MEMPROF) != 0; |
616 | } |
617 | |
618 | bool hasTemporalProfile() const override { |
619 | return (FormatVersion & VARIANT_MASK_TEMPORAL_PROF) != 0; |
620 | } |
621 | |
622 | InstrProfKind getProfileKind() const override; |
623 | |
624 | Error populateSymtab(InstrProfSymtab &Symtab) override { |
625 | return Symtab.create(HashTable->keys()); |
626 | } |
627 | }; |
628 | |
629 | /// Name matcher supporting fuzzy matching of symbol names to names in profiles. |
630 | class InstrProfReaderRemapper { |
631 | public: |
632 | virtual ~InstrProfReaderRemapper() = default; |
633 | virtual Error populateRemappings() { return Error::success(); } |
634 | virtual Error getRecords(StringRef FuncName, |
635 | ArrayRef<NamedInstrProfRecord> &Data) = 0; |
636 | }; |
637 | |
638 | /// Reader for the indexed binary instrprof format. |
639 | class IndexedInstrProfReader : public InstrProfReader { |
640 | private: |
641 | /// The profile data file contents. |
642 | std::unique_ptr<MemoryBuffer> DataBuffer; |
643 | /// The profile remapping file contents. |
644 | std::unique_ptr<MemoryBuffer> RemappingBuffer; |
645 | /// The index into the profile data. |
646 | std::unique_ptr<InstrProfReaderIndexBase> Index; |
647 | /// The profile remapping file contents. |
648 | std::unique_ptr<InstrProfReaderRemapper> Remapper; |
649 | /// Profile summary data. |
650 | std::unique_ptr<ProfileSummary> Summary; |
651 | /// Context sensitive profile summary data. |
652 | std::unique_ptr<ProfileSummary> CS_Summary; |
653 | /// MemProf profile schema (if available). |
654 | memprof::MemProfSchema Schema; |
655 | /// MemProf record profile data on-disk indexed via llvm::md5(FunctionName). |
656 | std::unique_ptr<MemProfRecordHashTable> MemProfRecordTable; |
657 | /// MemProf frame profile data on-disk indexed via frame id. |
658 | std::unique_ptr<MemProfFrameHashTable> MemProfFrameTable; |
659 | /// Total size of binary ids. |
660 | uint64_t BinaryIdsSize{0}; |
661 | /// Start address of binary id length and data pairs. |
662 | const uint8_t *BinaryIdsStart = nullptr; |
663 | |
664 | // Index to the current record in the record array. |
665 | unsigned RecordIndex; |
666 | |
667 | // Read the profile summary. Return a pointer pointing to one byte past the |
668 | // end of the summary data if it exists or the input \c Cur. |
669 | // \c UseCS indicates whether to use the context-sensitive profile summary. |
670 | const unsigned char *readSummary(IndexedInstrProf::ProfVersion Version, |
671 | const unsigned char *Cur, bool UseCS); |
672 | |
673 | public: |
674 | IndexedInstrProfReader( |
675 | std::unique_ptr<MemoryBuffer> DataBuffer, |
676 | std::unique_ptr<MemoryBuffer> RemappingBuffer = nullptr) |
677 | : DataBuffer(std::move(DataBuffer)), |
678 | RemappingBuffer(std::move(RemappingBuffer)), RecordIndex(0) {} |
679 | IndexedInstrProfReader(const IndexedInstrProfReader &) = delete; |
680 | IndexedInstrProfReader &operator=(const IndexedInstrProfReader &) = delete; |
681 | |
682 | /// Return the profile version. |
683 | uint64_t getVersion() const override { return Index->getVersion(); } |
684 | bool isIRLevelProfile() const override { return Index->isIRLevelProfile(); } |
685 | bool hasCSIRLevelProfile() const override { |
686 | return Index->hasCSIRLevelProfile(); |
687 | } |
688 | |
689 | bool instrEntryBBEnabled() const override { |
690 | return Index->instrEntryBBEnabled(); |
691 | } |
692 | |
693 | bool hasSingleByteCoverage() const override { |
694 | return Index->hasSingleByteCoverage(); |
695 | } |
696 | |
697 | bool functionEntryOnly() const override { return Index->functionEntryOnly(); } |
698 | |
699 | bool hasMemoryProfile() const override { return Index->hasMemoryProfile(); } |
700 | |
701 | bool hasTemporalProfile() const override { |
702 | return Index->hasTemporalProfile(); |
703 | } |
704 | |
705 | /// Returns a BitsetEnum describing the attributes of the indexed instr |
706 | /// profile. |
707 | InstrProfKind getProfileKind() const override { |
708 | return Index->getProfileKind(); |
709 | } |
710 | |
711 | /// Return true if the given buffer is in an indexed instrprof format. |
712 | static bool hasFormat(const MemoryBuffer &DataBuffer); |
713 | |
714 | /// Read the file header. |
715 | Error () override; |
716 | /// Read a single record. |
717 | Error readNextRecord(NamedInstrProfRecord &Record) override; |
718 | |
719 | /// Return the NamedInstrProfRecord associated with FuncName and FuncHash. |
720 | /// When return a hash_mismatch error and MismatchedFuncSum is not nullptr, |
721 | /// the sum of all counters in the mismatched function will be set to |
722 | /// MismatchedFuncSum. If there are multiple instances of mismatched |
723 | /// functions, MismatchedFuncSum returns the maximum. If \c FuncName is not |
724 | /// found, try to lookup \c DeprecatedFuncName to handle profiles built by |
725 | /// older compilers. |
726 | Expected<InstrProfRecord> |
727 | getInstrProfRecord(StringRef FuncName, uint64_t FuncHash, |
728 | StringRef DeprecatedFuncName = "" , |
729 | uint64_t *MismatchedFuncSum = nullptr); |
730 | |
731 | /// Return the memprof record for the function identified by |
732 | /// llvm::md5(Name). |
733 | Expected<memprof::MemProfRecord> getMemProfRecord(uint64_t FuncNameHash); |
734 | |
735 | /// Fill Counts with the profile data for the given function name. |
736 | Error getFunctionCounts(StringRef FuncName, uint64_t FuncHash, |
737 | std::vector<uint64_t> &Counts); |
738 | |
739 | /// Fill Bitmap with the profile data for the given function name. |
740 | Error getFunctionBitmap(StringRef FuncName, uint64_t FuncHash, |
741 | BitVector &Bitmap); |
742 | |
743 | /// Return the maximum of all known function counts. |
744 | /// \c UseCS indicates whether to use the context-sensitive count. |
745 | uint64_t getMaximumFunctionCount(bool UseCS) { |
746 | if (UseCS) { |
747 | assert(CS_Summary && "No context sensitive profile summary" ); |
748 | return CS_Summary->getMaxFunctionCount(); |
749 | } else { |
750 | assert(Summary && "No profile summary" ); |
751 | return Summary->getMaxFunctionCount(); |
752 | } |
753 | } |
754 | |
755 | /// Factory method to create an indexed reader. |
756 | static Expected<std::unique_ptr<IndexedInstrProfReader>> |
757 | create(const Twine &Path, vfs::FileSystem &FS, |
758 | const Twine &RemappingPath = "" ); |
759 | |
760 | static Expected<std::unique_ptr<IndexedInstrProfReader>> |
761 | create(std::unique_ptr<MemoryBuffer> Buffer, |
762 | std::unique_ptr<MemoryBuffer> RemappingBuffer = nullptr); |
763 | |
764 | // Used for testing purpose only. |
765 | void setValueProfDataEndianness(llvm::endianness Endianness) { |
766 | Index->setValueProfDataEndianness(Endianness); |
767 | } |
768 | |
769 | // See description in the base class. This interface is designed |
770 | // to be used by llvm-profdata (for dumping). Avoid using this when |
771 | // the client is the compiler. |
772 | InstrProfSymtab &getSymtab() override; |
773 | |
774 | /// Return the profile summary. |
775 | /// \c UseCS indicates whether to use the context-sensitive summary. |
776 | ProfileSummary &getSummary(bool UseCS) { |
777 | if (UseCS) { |
778 | assert(CS_Summary && "No context sensitive summary" ); |
779 | return *CS_Summary; |
780 | } else { |
781 | assert(Summary && "No profile summary" ); |
782 | return *Summary; |
783 | } |
784 | } |
785 | |
786 | Error readBinaryIds(std::vector<llvm::object::BuildID> &BinaryIds) override; |
787 | Error printBinaryIds(raw_ostream &OS) override; |
788 | }; |
789 | |
790 | } // end namespace llvm |
791 | |
792 | #endif // LLVM_PROFILEDATA_INSTRPROFREADER_H |
793 | |