1 | #ifndef LLVM_PROFILEDATA_MEMPROF_H_ |
2 | #define LLVM_PROFILEDATA_MEMPROF_H_ |
3 | |
4 | #include "llvm/ADT/MapVector.h" |
5 | #include "llvm/ADT/STLFunctionalExtras.h" |
6 | #include "llvm/ADT/SmallVector.h" |
7 | #include "llvm/IR/GlobalValue.h" |
8 | #include "llvm/ProfileData/MemProfData.inc" |
9 | #include "llvm/Support/Endian.h" |
10 | #include "llvm/Support/EndianStream.h" |
11 | #include "llvm/Support/raw_ostream.h" |
12 | |
13 | #include <cstdint> |
14 | #include <optional> |
15 | |
16 | namespace llvm { |
17 | namespace memprof { |
18 | |
19 | struct MemProfRecord; |
20 | |
21 | // The versions of the indexed MemProf format |
22 | enum IndexedVersion : uint64_t { |
23 | // Version 0: This version didn't have a version field. |
24 | Version0 = 0, |
25 | // Version 1: Added a version field to the header. |
26 | Version1 = 1, |
27 | // Version 2: Added a call stack table. Under development. |
28 | Version2 = 2, |
29 | }; |
30 | |
31 | constexpr uint64_t MinimumSupportedVersion = Version0; |
32 | constexpr uint64_t MaximumSupportedVersion = Version2; |
33 | |
34 | // Verify that the minimum and maximum satisfy the obvious constraint. |
35 | static_assert(MinimumSupportedVersion <= MaximumSupportedVersion); |
36 | |
37 | enum class Meta : uint64_t { |
38 | Start = 0, |
39 | #define MIBEntryDef(NameTag, Name, Type) NameTag, |
40 | #include "llvm/ProfileData/MIBEntryDef.inc" |
41 | #undef MIBEntryDef |
42 | Size |
43 | }; |
44 | |
45 | using MemProfSchema = llvm::SmallVector<Meta, static_cast<int>(Meta::Size)>; |
46 | |
47 | // Holds the actual MemInfoBlock data with all fields. Contents may be read or |
48 | // written partially by providing an appropriate schema to the serialize and |
49 | // deserialize methods. |
50 | struct PortableMemInfoBlock { |
51 | PortableMemInfoBlock() = default; |
52 | explicit PortableMemInfoBlock(const MemInfoBlock &Block) { |
53 | #define MIBEntryDef(NameTag, Name, Type) Name = Block.Name; |
54 | #include "llvm/ProfileData/MIBEntryDef.inc" |
55 | #undef MIBEntryDef |
56 | } |
57 | |
58 | PortableMemInfoBlock(const MemProfSchema &Schema, const unsigned char *Ptr) { |
59 | deserialize(Schema, Ptr); |
60 | } |
61 | |
62 | // Read the contents of \p Ptr based on the \p Schema to populate the |
63 | // MemInfoBlock member. |
64 | void deserialize(const MemProfSchema &Schema, const unsigned char *Ptr) { |
65 | using namespace support; |
66 | |
67 | for (const Meta Id : Schema) { |
68 | switch (Id) { |
69 | #define MIBEntryDef(NameTag, Name, Type) \ |
70 | case Meta::Name: { \ |
71 | Name = endian::readNext<Type, llvm::endianness::little>(Ptr); \ |
72 | } break; |
73 | #include "llvm/ProfileData/MIBEntryDef.inc" |
74 | #undef MIBEntryDef |
75 | default: |
76 | llvm_unreachable("Unknown meta type id, is the profile collected from " |
77 | "a newer version of the runtime?" ); |
78 | } |
79 | } |
80 | } |
81 | |
82 | // Write the contents of the MemInfoBlock based on the \p Schema provided to |
83 | // the raw_ostream \p OS. |
84 | void serialize(const MemProfSchema &Schema, raw_ostream &OS) const { |
85 | using namespace support; |
86 | |
87 | endian::Writer LE(OS, llvm::endianness::little); |
88 | for (const Meta Id : Schema) { |
89 | switch (Id) { |
90 | #define MIBEntryDef(NameTag, Name, Type) \ |
91 | case Meta::Name: { \ |
92 | LE.write<Type>(Name); \ |
93 | } break; |
94 | #include "llvm/ProfileData/MIBEntryDef.inc" |
95 | #undef MIBEntryDef |
96 | default: |
97 | llvm_unreachable("Unknown meta type id, invalid input?" ); |
98 | } |
99 | } |
100 | } |
101 | |
102 | // Print out the contents of the MemInfoBlock in YAML format. |
103 | void printYAML(raw_ostream &OS) const { |
104 | OS << " MemInfoBlock:\n" ; |
105 | #define MIBEntryDef(NameTag, Name, Type) \ |
106 | OS << " " << #Name << ": " << Name << "\n"; |
107 | #include "llvm/ProfileData/MIBEntryDef.inc" |
108 | #undef MIBEntryDef |
109 | } |
110 | |
111 | // Define getters for each type which can be called by analyses. |
112 | #define MIBEntryDef(NameTag, Name, Type) \ |
113 | Type get##Name() const { return Name; } |
114 | #include "llvm/ProfileData/MIBEntryDef.inc" |
115 | #undef MIBEntryDef |
116 | |
117 | void clear() { *this = PortableMemInfoBlock(); } |
118 | |
119 | // Returns the full schema currently in use. |
120 | static MemProfSchema getSchema() { |
121 | MemProfSchema List; |
122 | #define MIBEntryDef(NameTag, Name, Type) List.push_back(Meta::Name); |
123 | #include "llvm/ProfileData/MIBEntryDef.inc" |
124 | #undef MIBEntryDef |
125 | return List; |
126 | } |
127 | |
128 | bool operator==(const PortableMemInfoBlock &Other) const { |
129 | #define MIBEntryDef(NameTag, Name, Type) \ |
130 | if (Other.get##Name() != get##Name()) \ |
131 | return false; |
132 | #include "llvm/ProfileData/MIBEntryDef.inc" |
133 | #undef MIBEntryDef |
134 | return true; |
135 | } |
136 | |
137 | bool operator!=(const PortableMemInfoBlock &Other) const { |
138 | return !operator==(Other); |
139 | } |
140 | |
141 | static size_t serializedSize(const MemProfSchema &Schema) { |
142 | size_t Result = 0; |
143 | |
144 | for (const Meta Id : Schema) { |
145 | switch (Id) { |
146 | #define MIBEntryDef(NameTag, Name, Type) \ |
147 | case Meta::Name: { \ |
148 | Result += sizeof(Type); \ |
149 | } break; |
150 | #include "llvm/ProfileData/MIBEntryDef.inc" |
151 | #undef MIBEntryDef |
152 | default: |
153 | llvm_unreachable("Unknown meta type id, invalid input?" ); |
154 | } |
155 | } |
156 | |
157 | return Result; |
158 | } |
159 | |
160 | private: |
161 | #define MIBEntryDef(NameTag, Name, Type) Type Name = Type(); |
162 | #include "llvm/ProfileData/MIBEntryDef.inc" |
163 | #undef MIBEntryDef |
164 | }; |
165 | |
166 | // A type representing the id generated by hashing the contents of the Frame. |
167 | using FrameId = uint64_t; |
168 | // Describes a call frame for a dynamic allocation context. The contents of |
169 | // the frame are populated by symbolizing the stack depot call frame from the |
170 | // compiler runtime. |
171 | struct Frame { |
172 | // A uuid (uint64_t) identifying the function. It is obtained by |
173 | // llvm::md5(FunctionName) which returns the lower 64 bits. |
174 | GlobalValue::GUID Function; |
175 | // The symbol name for the function. Only populated in the Frame by the reader |
176 | // if requested during initialization. This field should not be serialized. |
177 | std::optional<std::string> SymbolName; |
178 | // The source line offset of the call from the beginning of parent function. |
179 | uint32_t LineOffset; |
180 | // The source column number of the call to help distinguish multiple calls |
181 | // on the same line. |
182 | uint32_t Column; |
183 | // Whether the current frame is inlined. |
184 | bool IsInlineFrame; |
185 | |
186 | Frame(const Frame &Other) { |
187 | Function = Other.Function; |
188 | SymbolName = Other.SymbolName; |
189 | LineOffset = Other.LineOffset; |
190 | Column = Other.Column; |
191 | IsInlineFrame = Other.IsInlineFrame; |
192 | } |
193 | |
194 | Frame(uint64_t Hash, uint32_t Off, uint32_t Col, bool Inline) |
195 | : Function(Hash), LineOffset(Off), Column(Col), IsInlineFrame(Inline) {} |
196 | |
197 | bool operator==(const Frame &Other) const { |
198 | // Ignore the SymbolName field to avoid a string compare. Comparing the |
199 | // function hash serves the same purpose. |
200 | return Other.Function == Function && Other.LineOffset == LineOffset && |
201 | Other.Column == Column && Other.IsInlineFrame == IsInlineFrame; |
202 | } |
203 | |
204 | Frame &operator=(const Frame &Other) { |
205 | Function = Other.Function; |
206 | SymbolName = Other.SymbolName; |
207 | LineOffset = Other.LineOffset; |
208 | Column = Other.Column; |
209 | IsInlineFrame = Other.IsInlineFrame; |
210 | return *this; |
211 | } |
212 | |
213 | bool operator!=(const Frame &Other) const { return !operator==(Other); } |
214 | |
215 | // Write the contents of the frame to the ostream \p OS. |
216 | void serialize(raw_ostream &OS) const { |
217 | using namespace support; |
218 | |
219 | endian::Writer LE(OS, llvm::endianness::little); |
220 | |
221 | // If the type of the GlobalValue::GUID changes, then we need to update |
222 | // the reader and the writer. |
223 | static_assert(std::is_same<GlobalValue::GUID, uint64_t>::value, |
224 | "Expect GUID to be uint64_t." ); |
225 | LE.write<uint64_t>(Val: Function); |
226 | |
227 | LE.write<uint32_t>(Val: LineOffset); |
228 | LE.write<uint32_t>(Val: Column); |
229 | LE.write<bool>(Val: IsInlineFrame); |
230 | } |
231 | |
232 | // Read a frame from char data which has been serialized as little endian. |
233 | static Frame deserialize(const unsigned char *Ptr) { |
234 | using namespace support; |
235 | |
236 | const uint64_t F = |
237 | endian::readNext<uint64_t, llvm::endianness::little>(memory&: Ptr); |
238 | const uint32_t L = |
239 | endian::readNext<uint32_t, llvm::endianness::little>(memory&: Ptr); |
240 | const uint32_t C = |
241 | endian::readNext<uint32_t, llvm::endianness::little>(memory&: Ptr); |
242 | const bool I = endian::readNext<bool, llvm::endianness::little>(memory&: Ptr); |
243 | return Frame(/*Function=*/F, /*LineOffset=*/L, /*Column=*/C, |
244 | /*IsInlineFrame=*/I); |
245 | } |
246 | |
247 | // Returns the size of the frame information. |
248 | static constexpr size_t serializedSize() { |
249 | return sizeof(Frame::Function) + sizeof(Frame::LineOffset) + |
250 | sizeof(Frame::Column) + sizeof(Frame::IsInlineFrame); |
251 | } |
252 | |
253 | // Print the frame information in YAML format. |
254 | void printYAML(raw_ostream &OS) const { |
255 | OS << " -\n" |
256 | << " Function: " << Function << "\n" |
257 | << " SymbolName: " << SymbolName.value_or(u: "<None>" ) << "\n" |
258 | << " LineOffset: " << LineOffset << "\n" |
259 | << " Column: " << Column << "\n" |
260 | << " Inline: " << IsInlineFrame << "\n" ; |
261 | } |
262 | |
263 | // Return a hash value based on the contents of the frame. Here we don't use |
264 | // hashing from llvm ADT since we are going to persist the hash id, the hash |
265 | // combine algorithm in ADT uses a new randomized seed each time. |
266 | inline FrameId hash() const { |
267 | auto HashCombine = [](auto Value, size_t Seed) { |
268 | std::hash<decltype(Value)> Hasher; |
269 | // The constant used below is the 64 bit representation of the fractional |
270 | // part of the golden ratio. Used here for the randomness in their bit |
271 | // pattern. |
272 | return Hasher(Value) + 0x9e3779b97f4a7c15 + (Seed << 6) + (Seed >> 2); |
273 | }; |
274 | |
275 | size_t Result = 0; |
276 | Result ^= HashCombine(Function, Result); |
277 | Result ^= HashCombine(LineOffset, Result); |
278 | Result ^= HashCombine(Column, Result); |
279 | Result ^= HashCombine(IsInlineFrame, Result); |
280 | return static_cast<FrameId>(Result); |
281 | } |
282 | }; |
283 | |
284 | // A type representing the index into the table of call stacks. |
285 | using CallStackId = uint64_t; |
286 | |
287 | // Holds allocation information in a space efficient format where frames are |
288 | // represented using unique identifiers. |
289 | struct IndexedAllocationInfo { |
290 | // The dynamic calling context for the allocation in bottom-up (leaf-to-root) |
291 | // order. Frame contents are stored out-of-line. |
292 | // TODO: Remove once we fully transition to CSId. |
293 | llvm::SmallVector<FrameId> CallStack; |
294 | // Conceptually the same as above. We are going to keep both CallStack and |
295 | // CallStackId while we are transitioning from CallStack to CallStackId. |
296 | CallStackId CSId = 0; |
297 | // The statistics obtained from the runtime for the allocation. |
298 | PortableMemInfoBlock Info; |
299 | |
300 | IndexedAllocationInfo() = default; |
301 | IndexedAllocationInfo(ArrayRef<FrameId> CS, CallStackId CSId, |
302 | const MemInfoBlock &MB) |
303 | : CallStack(CS.begin(), CS.end()), CSId(CSId), Info(MB) {} |
304 | |
305 | // Returns the size in bytes when this allocation info struct is serialized. |
306 | size_t serializedSize(const MemProfSchema &Schema, |
307 | IndexedVersion Version) const; |
308 | |
309 | bool operator==(const IndexedAllocationInfo &Other) const { |
310 | if (Other.Info != Info) |
311 | return false; |
312 | |
313 | if (Other.CSId != CSId) |
314 | return false; |
315 | return true; |
316 | } |
317 | |
318 | bool operator!=(const IndexedAllocationInfo &Other) const { |
319 | return !operator==(Other); |
320 | } |
321 | }; |
322 | |
323 | // Holds allocation information with frame contents inline. The type should |
324 | // be used for temporary in-memory instances. |
325 | struct AllocationInfo { |
326 | // Same as IndexedAllocationInfo::CallStack with the frame contents inline. |
327 | llvm::SmallVector<Frame> CallStack; |
328 | // Same as IndexedAllocationInfo::Info; |
329 | PortableMemInfoBlock Info; |
330 | |
331 | AllocationInfo() = default; |
332 | AllocationInfo( |
333 | const IndexedAllocationInfo &IndexedAI, |
334 | llvm::function_ref<const Frame(const FrameId)> IdToFrameCallback) { |
335 | for (const FrameId &Id : IndexedAI.CallStack) { |
336 | CallStack.push_back(Elt: IdToFrameCallback(Id)); |
337 | } |
338 | Info = IndexedAI.Info; |
339 | } |
340 | |
341 | void printYAML(raw_ostream &OS) const { |
342 | OS << " -\n" ; |
343 | OS << " Callstack:\n" ; |
344 | // TODO: Print out the frame on one line with to make it easier for deep |
345 | // callstacks once we have a test to check valid YAML is generated. |
346 | for (const Frame &F : CallStack) { |
347 | F.printYAML(OS); |
348 | } |
349 | Info.printYAML(OS); |
350 | } |
351 | }; |
352 | |
353 | // Holds the memprof profile information for a function. The internal |
354 | // representation stores frame ids for efficiency. This representation should |
355 | // be used in the profile conversion and manipulation tools. |
356 | struct IndexedMemProfRecord { |
357 | // Memory allocation sites in this function for which we have memory |
358 | // profiling data. |
359 | llvm::SmallVector<IndexedAllocationInfo> AllocSites; |
360 | // Holds call sites in this function which are part of some memory |
361 | // allocation context. We store this as a list of locations, each with its |
362 | // list of inline locations in bottom-up order i.e. from leaf to root. The |
363 | // inline location list may include additional entries, users should pick |
364 | // the last entry in the list with the same function GUID. |
365 | llvm::SmallVector<llvm::SmallVector<FrameId>> CallSites; |
366 | // Conceptually the same as above. We are going to keep both CallSites and |
367 | // CallSiteIds while we are transitioning from CallSites to CallSiteIds. |
368 | llvm::SmallVector<CallStackId> CallSiteIds; |
369 | |
370 | void clear() { |
371 | AllocSites.clear(); |
372 | CallSites.clear(); |
373 | } |
374 | |
375 | void merge(const IndexedMemProfRecord &Other) { |
376 | // TODO: Filter out duplicates which may occur if multiple memprof |
377 | // profiles are merged together using llvm-profdata. |
378 | AllocSites.append(RHS: Other.AllocSites); |
379 | CallSites.append(RHS: Other.CallSites); |
380 | } |
381 | |
382 | size_t serializedSize(const MemProfSchema &Schema, |
383 | IndexedVersion Version) const; |
384 | |
385 | bool operator==(const IndexedMemProfRecord &Other) const { |
386 | if (Other.AllocSites != AllocSites) |
387 | return false; |
388 | |
389 | if (Other.CallSiteIds != CallSiteIds) |
390 | return false; |
391 | return true; |
392 | } |
393 | |
394 | // Serializes the memprof records in \p Records to the ostream \p OS based |
395 | // on the schema provided in \p Schema. |
396 | void serialize(const MemProfSchema &Schema, raw_ostream &OS, |
397 | IndexedVersion Version); |
398 | |
399 | // Deserializes memprof records from the Buffer. |
400 | static IndexedMemProfRecord deserialize(const MemProfSchema &Schema, |
401 | const unsigned char *Buffer, |
402 | IndexedVersion Version); |
403 | |
404 | // Convert IndexedMemProfRecord to MemProfRecord. Callback is used to |
405 | // translate CallStackId to call stacks with frames inline. |
406 | MemProfRecord toMemProfRecord( |
407 | std::function<const llvm::SmallVector<Frame>(const CallStackId)> Callback) |
408 | const; |
409 | |
410 | // Returns the GUID for the function name after canonicalization. For |
411 | // memprof, we remove any .llvm suffix added by LTO. MemProfRecords are |
412 | // mapped to functions using this GUID. |
413 | static GlobalValue::GUID getGUID(const StringRef FunctionName); |
414 | }; |
415 | |
416 | // Holds the memprof profile information for a function. The internal |
417 | // representation stores frame contents inline. This representation should |
418 | // be used for small amount of temporary, in memory instances. |
419 | struct MemProfRecord { |
420 | // Same as IndexedMemProfRecord::AllocSites with frame contents inline. |
421 | llvm::SmallVector<AllocationInfo> AllocSites; |
422 | // Same as IndexedMemProfRecord::CallSites with frame contents inline. |
423 | llvm::SmallVector<llvm::SmallVector<Frame>> CallSites; |
424 | |
425 | MemProfRecord() = default; |
426 | MemProfRecord( |
427 | const IndexedMemProfRecord &Record, |
428 | llvm::function_ref<const Frame(const FrameId Id)> IdToFrameCallback) { |
429 | for (const IndexedAllocationInfo &IndexedAI : Record.AllocSites) { |
430 | AllocSites.emplace_back(Args: IndexedAI, Args&: IdToFrameCallback); |
431 | } |
432 | for (const ArrayRef<FrameId> Site : Record.CallSites) { |
433 | llvm::SmallVector<Frame> Frames; |
434 | for (const FrameId Id : Site) { |
435 | Frames.push_back(Elt: IdToFrameCallback(Id)); |
436 | } |
437 | CallSites.push_back(Elt: Frames); |
438 | } |
439 | } |
440 | |
441 | // Prints out the contents of the memprof record in YAML. |
442 | void print(llvm::raw_ostream &OS) const { |
443 | if (!AllocSites.empty()) { |
444 | OS << " AllocSites:\n" ; |
445 | for (const AllocationInfo &N : AllocSites) |
446 | N.printYAML(OS); |
447 | } |
448 | |
449 | if (!CallSites.empty()) { |
450 | OS << " CallSites:\n" ; |
451 | for (const llvm::SmallVector<Frame> &Frames : CallSites) { |
452 | for (const Frame &F : Frames) { |
453 | OS << " -\n" ; |
454 | F.printYAML(OS); |
455 | } |
456 | } |
457 | } |
458 | } |
459 | }; |
460 | |
461 | // Reads a memprof schema from a buffer. All entries in the buffer are |
462 | // interpreted as uint64_t. The first entry in the buffer denotes the number of |
463 | // ids in the schema. Subsequent entries are integers which map to memprof::Meta |
464 | // enum class entries. After successfully reading the schema, the pointer is one |
465 | // byte past the schema contents. |
466 | Expected<MemProfSchema> readMemProfSchema(const unsigned char *&Buffer); |
467 | |
468 | // Trait for reading IndexedMemProfRecord data from the on-disk hash table. |
469 | class RecordLookupTrait { |
470 | public: |
471 | using data_type = const IndexedMemProfRecord &; |
472 | using internal_key_type = uint64_t; |
473 | using external_key_type = uint64_t; |
474 | using hash_value_type = uint64_t; |
475 | using offset_type = uint64_t; |
476 | |
477 | RecordLookupTrait() = delete; |
478 | RecordLookupTrait(IndexedVersion V, const MemProfSchema &S) |
479 | : Version(V), Schema(S) {} |
480 | |
481 | static bool EqualKey(uint64_t A, uint64_t B) { return A == B; } |
482 | static uint64_t GetInternalKey(uint64_t K) { return K; } |
483 | static uint64_t GetExternalKey(uint64_t K) { return K; } |
484 | |
485 | hash_value_type ComputeHash(uint64_t K) { return K; } |
486 | |
487 | static std::pair<offset_type, offset_type> |
488 | ReadKeyDataLength(const unsigned char *&D) { |
489 | using namespace support; |
490 | |
491 | offset_type KeyLen = |
492 | endian::readNext<offset_type, llvm::endianness::little>(memory&: D); |
493 | offset_type DataLen = |
494 | endian::readNext<offset_type, llvm::endianness::little>(memory&: D); |
495 | return std::make_pair(x&: KeyLen, y&: DataLen); |
496 | } |
497 | |
498 | uint64_t ReadKey(const unsigned char *D, offset_type /*Unused*/) { |
499 | using namespace support; |
500 | return endian::readNext<external_key_type, llvm::endianness::little>(memory&: D); |
501 | } |
502 | |
503 | data_type ReadData(uint64_t K, const unsigned char *D, |
504 | offset_type /*Unused*/) { |
505 | Record = IndexedMemProfRecord::deserialize(Schema, Buffer: D, Version); |
506 | return Record; |
507 | } |
508 | |
509 | private: |
510 | // Holds the MemProf version. |
511 | IndexedVersion Version; |
512 | // Holds the memprof schema used to deserialize records. |
513 | MemProfSchema Schema; |
514 | // Holds the records from one function deserialized from the indexed format. |
515 | IndexedMemProfRecord Record; |
516 | }; |
517 | |
518 | // Trait for writing IndexedMemProfRecord data to the on-disk hash table. |
519 | class RecordWriterTrait { |
520 | public: |
521 | using key_type = uint64_t; |
522 | using key_type_ref = uint64_t; |
523 | |
524 | using data_type = IndexedMemProfRecord; |
525 | using data_type_ref = IndexedMemProfRecord &; |
526 | |
527 | using hash_value_type = uint64_t; |
528 | using offset_type = uint64_t; |
529 | |
530 | private: |
531 | // Pointer to the memprof schema to use for the generator. |
532 | const MemProfSchema *Schema; |
533 | // The MemProf version to use for the serialization. |
534 | IndexedVersion Version; |
535 | |
536 | public: |
537 | // We do not support the default constructor, which does not set Version. |
538 | RecordWriterTrait() = delete; |
539 | RecordWriterTrait(const MemProfSchema *Schema, IndexedVersion V) |
540 | : Schema(Schema), Version(V) {} |
541 | |
542 | static hash_value_type ComputeHash(key_type_ref K) { return K; } |
543 | |
544 | std::pair<offset_type, offset_type> |
545 | EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V) { |
546 | using namespace support; |
547 | |
548 | endian::Writer LE(Out, llvm::endianness::little); |
549 | offset_type N = sizeof(K); |
550 | LE.write<offset_type>(Val: N); |
551 | offset_type M = V.serializedSize(Schema: *Schema, Version); |
552 | LE.write<offset_type>(Val: M); |
553 | return std::make_pair(x&: N, y&: M); |
554 | } |
555 | |
556 | void EmitKey(raw_ostream &Out, key_type_ref K, offset_type /*Unused*/) { |
557 | using namespace support; |
558 | endian::Writer LE(Out, llvm::endianness::little); |
559 | LE.write<uint64_t>(Val: K); |
560 | } |
561 | |
562 | void EmitData(raw_ostream &Out, key_type_ref /*Unused*/, data_type_ref V, |
563 | offset_type /*Unused*/) { |
564 | assert(Schema != nullptr && "MemProf schema is not initialized!" ); |
565 | V.serialize(Schema: *Schema, OS&: Out, Version); |
566 | // Clear the IndexedMemProfRecord which results in clearing/freeing its |
567 | // vectors of allocs and callsites. This is owned by the associated on-disk |
568 | // hash table, but unused after this point. See also the comment added to |
569 | // the client which constructs the on-disk hash table for this trait. |
570 | V.clear(); |
571 | } |
572 | }; |
573 | |
574 | // Trait for writing frame mappings to the on-disk hash table. |
575 | class FrameWriterTrait { |
576 | public: |
577 | using key_type = FrameId; |
578 | using key_type_ref = FrameId; |
579 | |
580 | using data_type = Frame; |
581 | using data_type_ref = Frame &; |
582 | |
583 | using hash_value_type = FrameId; |
584 | using offset_type = uint64_t; |
585 | |
586 | static hash_value_type ComputeHash(key_type_ref K) { return K; } |
587 | |
588 | static std::pair<offset_type, offset_type> |
589 | EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V) { |
590 | using namespace support; |
591 | endian::Writer LE(Out, llvm::endianness::little); |
592 | offset_type N = sizeof(K); |
593 | LE.write<offset_type>(Val: N); |
594 | offset_type M = V.serializedSize(); |
595 | LE.write<offset_type>(Val: M); |
596 | return std::make_pair(x&: N, y&: M); |
597 | } |
598 | |
599 | void EmitKey(raw_ostream &Out, key_type_ref K, offset_type /*Unused*/) { |
600 | using namespace support; |
601 | endian::Writer LE(Out, llvm::endianness::little); |
602 | LE.write<key_type>(Val: K); |
603 | } |
604 | |
605 | void EmitData(raw_ostream &Out, key_type_ref /*Unused*/, data_type_ref V, |
606 | offset_type /*Unused*/) { |
607 | V.serialize(OS&: Out); |
608 | } |
609 | }; |
610 | |
611 | // Trait for reading frame mappings from the on-disk hash table. |
612 | class FrameLookupTrait { |
613 | public: |
614 | using data_type = const Frame; |
615 | using internal_key_type = FrameId; |
616 | using external_key_type = FrameId; |
617 | using hash_value_type = FrameId; |
618 | using offset_type = uint64_t; |
619 | |
620 | static bool EqualKey(internal_key_type A, internal_key_type B) { |
621 | return A == B; |
622 | } |
623 | static uint64_t GetInternalKey(internal_key_type K) { return K; } |
624 | static uint64_t GetExternalKey(external_key_type K) { return K; } |
625 | |
626 | hash_value_type ComputeHash(internal_key_type K) { return K; } |
627 | |
628 | static std::pair<offset_type, offset_type> |
629 | ReadKeyDataLength(const unsigned char *&D) { |
630 | using namespace support; |
631 | |
632 | offset_type KeyLen = |
633 | endian::readNext<offset_type, llvm::endianness::little>(memory&: D); |
634 | offset_type DataLen = |
635 | endian::readNext<offset_type, llvm::endianness::little>(memory&: D); |
636 | return std::make_pair(x&: KeyLen, y&: DataLen); |
637 | } |
638 | |
639 | uint64_t ReadKey(const unsigned char *D, offset_type /*Unused*/) { |
640 | using namespace support; |
641 | return endian::readNext<external_key_type, llvm::endianness::little>(memory&: D); |
642 | } |
643 | |
644 | data_type ReadData(uint64_t K, const unsigned char *D, |
645 | offset_type /*Unused*/) { |
646 | return Frame::deserialize(Ptr: D); |
647 | } |
648 | }; |
649 | |
650 | // Trait for writing call stacks to the on-disk hash table. |
651 | class CallStackWriterTrait { |
652 | public: |
653 | using key_type = CallStackId; |
654 | using key_type_ref = CallStackId; |
655 | |
656 | using data_type = llvm::SmallVector<FrameId>; |
657 | using data_type_ref = llvm::SmallVector<FrameId> &; |
658 | |
659 | using hash_value_type = CallStackId; |
660 | using offset_type = uint64_t; |
661 | |
662 | static hash_value_type ComputeHash(key_type_ref K) { return K; } |
663 | |
664 | static std::pair<offset_type, offset_type> |
665 | EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V) { |
666 | using namespace support; |
667 | endian::Writer LE(Out, llvm::endianness::little); |
668 | // We do not explicitly emit the key length because it is a constant. |
669 | offset_type N = sizeof(K); |
670 | offset_type M = sizeof(FrameId) * V.size(); |
671 | LE.write<offset_type>(Val: M); |
672 | return std::make_pair(x&: N, y&: M); |
673 | } |
674 | |
675 | void EmitKey(raw_ostream &Out, key_type_ref K, offset_type /*Unused*/) { |
676 | using namespace support; |
677 | endian::Writer LE(Out, llvm::endianness::little); |
678 | LE.write<key_type>(Val: K); |
679 | } |
680 | |
681 | void EmitData(raw_ostream &Out, key_type_ref /*Unused*/, data_type_ref V, |
682 | offset_type /*Unused*/) { |
683 | using namespace support; |
684 | endian::Writer LE(Out, llvm::endianness::little); |
685 | // Emit the frames. We do not explicitly emit the length of the vector |
686 | // because it can be inferred from the data length. |
687 | for (FrameId F : V) |
688 | LE.write<FrameId>(Val: F); |
689 | } |
690 | }; |
691 | |
692 | // Trait for reading call stack mappings from the on-disk hash table. |
693 | class CallStackLookupTrait { |
694 | public: |
695 | using data_type = const llvm::SmallVector<FrameId>; |
696 | using internal_key_type = CallStackId; |
697 | using external_key_type = CallStackId; |
698 | using hash_value_type = CallStackId; |
699 | using offset_type = uint64_t; |
700 | |
701 | static bool EqualKey(internal_key_type A, internal_key_type B) { |
702 | return A == B; |
703 | } |
704 | static uint64_t GetInternalKey(internal_key_type K) { return K; } |
705 | static uint64_t GetExternalKey(external_key_type K) { return K; } |
706 | |
707 | hash_value_type ComputeHash(internal_key_type K) { return K; } |
708 | |
709 | static std::pair<offset_type, offset_type> |
710 | ReadKeyDataLength(const unsigned char *&D) { |
711 | using namespace support; |
712 | |
713 | // We do not explicitly read the key length because it is a constant. |
714 | offset_type KeyLen = sizeof(external_key_type); |
715 | offset_type DataLen = |
716 | endian::readNext<offset_type, llvm::endianness::little>(memory&: D); |
717 | return std::make_pair(x&: KeyLen, y&: DataLen); |
718 | } |
719 | |
720 | uint64_t ReadKey(const unsigned char *D, offset_type /*Unused*/) { |
721 | using namespace support; |
722 | return endian::readNext<external_key_type, llvm::endianness::little>(memory&: D); |
723 | } |
724 | |
725 | data_type ReadData(uint64_t K, const unsigned char *D, offset_type Length) { |
726 | using namespace support; |
727 | llvm::SmallVector<FrameId> CS; |
728 | // Derive the number of frames from the data length. |
729 | uint64_t NumFrames = Length / sizeof(FrameId); |
730 | assert(Length % sizeof(FrameId) == 0); |
731 | CS.reserve(N: NumFrames); |
732 | for (size_t I = 0; I != NumFrames; ++I) { |
733 | FrameId F = endian::readNext<FrameId, llvm::endianness::little>(memory&: D); |
734 | CS.push_back(Elt: F); |
735 | } |
736 | return CS; |
737 | } |
738 | }; |
739 | |
740 | // Compute a CallStackId for a given call stack. |
741 | CallStackId hashCallStack(ArrayRef<FrameId> CS); |
742 | |
743 | // Verify that each CallStackId is computed with hashCallStack. This function |
744 | // is intended to help transition from CallStack to CSId in |
745 | // IndexedAllocationInfo. |
746 | void verifyIndexedMemProfRecord(const IndexedMemProfRecord &Record); |
747 | |
748 | // Verify that each CallStackId is computed with hashCallStack. This function |
749 | // is intended to help transition from CallStack to CSId in |
750 | // IndexedAllocationInfo. |
751 | void verifyFunctionProfileData( |
752 | const llvm::MapVector<GlobalValue::GUID, IndexedMemProfRecord> |
753 | &FunctionProfileData); |
754 | } // namespace memprof |
755 | } // namespace llvm |
756 | |
757 | #endif // LLVM_PROFILEDATA_MEMPROF_H_ |
758 | |