1 | //===- InstrProfWriter.cpp - Instrumented profiling writer ----------------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file contains support for writing profiling data for clang's |
10 | // instrumentation based PGO and coverage. |
11 | // |
12 | //===----------------------------------------------------------------------===// |
13 | |
14 | #include "llvm/ProfileData/InstrProfWriter.h" |
15 | #include "llvm/ADT/STLExtras.h" |
16 | #include "llvm/ADT/SetVector.h" |
17 | #include "llvm/ADT/StringRef.h" |
18 | #include "llvm/IR/ProfileSummary.h" |
19 | #include "llvm/ProfileData/InstrProf.h" |
20 | #include "llvm/ProfileData/MemProf.h" |
21 | #include "llvm/ProfileData/ProfileCommon.h" |
22 | #include "llvm/Support/Compression.h" |
23 | #include "llvm/Support/Endian.h" |
24 | #include "llvm/Support/EndianStream.h" |
25 | #include "llvm/Support/Error.h" |
26 | #include "llvm/Support/FormatVariadic.h" |
27 | #include "llvm/Support/MemoryBuffer.h" |
28 | #include "llvm/Support/OnDiskHashTable.h" |
29 | #include "llvm/Support/raw_ostream.h" |
30 | #include <cstdint> |
31 | #include <memory> |
32 | #include <string> |
33 | #include <tuple> |
34 | #include <utility> |
35 | #include <vector> |
36 | |
37 | using namespace llvm; |
38 | |
39 | // A struct to define how the data stream should be patched. For Indexed |
40 | // profiling, only uint64_t data type is needed. |
41 | struct PatchItem { |
42 | uint64_t Pos; // Where to patch. |
43 | uint64_t *D; // Pointer to an array of source data. |
44 | int N; // Number of elements in \c D array. |
45 | }; |
46 | |
47 | namespace llvm { |
48 | |
49 | // A wrapper class to abstract writer stream with support of bytes |
50 | // back patching. |
51 | class ProfOStream { |
52 | public: |
53 | ProfOStream(raw_fd_ostream &FD) |
54 | : IsFDOStream(true), OS(FD), LE(FD, llvm::endianness::little) {} |
55 | ProfOStream(raw_string_ostream &STR) |
56 | : IsFDOStream(false), OS(STR), LE(STR, llvm::endianness::little) {} |
57 | |
58 | uint64_t tell() { return OS.tell(); } |
59 | void write(uint64_t V) { LE.write<uint64_t>(Val: V); } |
60 | void writeByte(uint8_t V) { LE.write<uint8_t>(Val: V); } |
61 | |
62 | // \c patch can only be called when all data is written and flushed. |
63 | // For raw_string_ostream, the patch is done on the target string |
64 | // directly and it won't be reflected in the stream's internal buffer. |
65 | void patch(ArrayRef<PatchItem> P) { |
66 | using namespace support; |
67 | |
68 | if (IsFDOStream) { |
69 | raw_fd_ostream &FDOStream = static_cast<raw_fd_ostream &>(OS); |
70 | const uint64_t LastPos = FDOStream.tell(); |
71 | for (const auto &K : P) { |
72 | FDOStream.seek(off: K.Pos); |
73 | for (int I = 0; I < K.N; I++) |
74 | write(V: K.D[I]); |
75 | } |
76 | // Reset the stream to the last position after patching so that users |
77 | // don't accidentally overwrite data. This makes it consistent with |
78 | // the string stream below which replaces the data directly. |
79 | FDOStream.seek(off: LastPos); |
80 | } else { |
81 | raw_string_ostream &SOStream = static_cast<raw_string_ostream &>(OS); |
82 | std::string &Data = SOStream.str(); // with flush |
83 | for (const auto &K : P) { |
84 | for (int I = 0; I < K.N; I++) { |
85 | uint64_t Bytes = |
86 | endian::byte_swap<uint64_t, llvm::endianness::little>(value: K.D[I]); |
87 | Data.replace(pos: K.Pos + I * sizeof(uint64_t), n1: sizeof(uint64_t), |
88 | s: (const char *)&Bytes, n2: sizeof(uint64_t)); |
89 | } |
90 | } |
91 | } |
92 | } |
93 | |
94 | // If \c OS is an instance of \c raw_fd_ostream, this field will be |
95 | // true. Otherwise, \c OS will be an raw_string_ostream. |
96 | bool IsFDOStream; |
97 | raw_ostream &OS; |
98 | support::endian::Writer LE; |
99 | }; |
100 | |
101 | class InstrProfRecordWriterTrait { |
102 | public: |
103 | using key_type = StringRef; |
104 | using key_type_ref = StringRef; |
105 | |
106 | using data_type = const InstrProfWriter::ProfilingData *const; |
107 | using data_type_ref = const InstrProfWriter::ProfilingData *const; |
108 | |
109 | using hash_value_type = uint64_t; |
110 | using offset_type = uint64_t; |
111 | |
112 | llvm::endianness ValueProfDataEndianness = llvm::endianness::little; |
113 | InstrProfSummaryBuilder *SummaryBuilder; |
114 | InstrProfSummaryBuilder *CSSummaryBuilder; |
115 | |
116 | InstrProfRecordWriterTrait() = default; |
117 | |
118 | static hash_value_type ComputeHash(key_type_ref K) { |
119 | return IndexedInstrProf::ComputeHash(K); |
120 | } |
121 | |
122 | static std::pair<offset_type, offset_type> |
123 | EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V) { |
124 | using namespace support; |
125 | |
126 | endian::Writer LE(Out, llvm::endianness::little); |
127 | |
128 | offset_type N = K.size(); |
129 | LE.write<offset_type>(Val: N); |
130 | |
131 | offset_type M = 0; |
132 | for (const auto &ProfileData : *V) { |
133 | const InstrProfRecord &ProfRecord = ProfileData.second; |
134 | M += sizeof(uint64_t); // The function hash |
135 | M += sizeof(uint64_t); // The size of the Counts vector |
136 | M += ProfRecord.Counts.size() * sizeof(uint64_t); |
137 | M += sizeof(uint64_t); // The size of the Bitmap vector |
138 | M += ProfRecord.BitmapBytes.size() * sizeof(uint64_t); |
139 | |
140 | // Value data |
141 | M += ValueProfData::getSize(Record: ProfileData.second); |
142 | } |
143 | LE.write<offset_type>(Val: M); |
144 | |
145 | return std::make_pair(x&: N, y&: M); |
146 | } |
147 | |
148 | void EmitKey(raw_ostream &Out, key_type_ref K, offset_type N) { |
149 | Out.write(Ptr: K.data(), Size: N); |
150 | } |
151 | |
152 | void EmitData(raw_ostream &Out, key_type_ref, data_type_ref V, offset_type) { |
153 | using namespace support; |
154 | |
155 | endian::Writer LE(Out, llvm::endianness::little); |
156 | for (const auto &ProfileData : *V) { |
157 | const InstrProfRecord &ProfRecord = ProfileData.second; |
158 | if (NamedInstrProfRecord::hasCSFlagInHash(FuncHash: ProfileData.first)) |
159 | CSSummaryBuilder->addRecord(ProfRecord); |
160 | else |
161 | SummaryBuilder->addRecord(ProfRecord); |
162 | |
163 | LE.write<uint64_t>(Val: ProfileData.first); // Function hash |
164 | LE.write<uint64_t>(Val: ProfRecord.Counts.size()); |
165 | for (uint64_t I : ProfRecord.Counts) |
166 | LE.write<uint64_t>(Val: I); |
167 | |
168 | LE.write<uint64_t>(Val: ProfRecord.BitmapBytes.size()); |
169 | for (uint64_t I : ProfRecord.BitmapBytes) |
170 | LE.write<uint64_t>(Val: I); |
171 | |
172 | // Write value data |
173 | std::unique_ptr<ValueProfData> VDataPtr = |
174 | ValueProfData::serializeFrom(Record: ProfileData.second); |
175 | uint32_t S = VDataPtr->getSize(); |
176 | VDataPtr->swapBytesFromHost(Endianness: ValueProfDataEndianness); |
177 | Out.write(Ptr: (const char *)VDataPtr.get(), Size: S); |
178 | } |
179 | } |
180 | }; |
181 | |
182 | } // end namespace llvm |
183 | |
184 | InstrProfWriter::InstrProfWriter( |
185 | bool Sparse, uint64_t TemporalProfTraceReservoirSize, |
186 | uint64_t MaxTemporalProfTraceLength, bool WritePrevVersion, |
187 | memprof::IndexedVersion MemProfVersionRequested) |
188 | : Sparse(Sparse), MaxTemporalProfTraceLength(MaxTemporalProfTraceLength), |
189 | TemporalProfTraceReservoirSize(TemporalProfTraceReservoirSize), |
190 | InfoObj(new InstrProfRecordWriterTrait()), |
191 | WritePrevVersion(WritePrevVersion), |
192 | MemProfVersionRequested(MemProfVersionRequested) {} |
193 | |
194 | InstrProfWriter::~InstrProfWriter() { delete InfoObj; } |
195 | |
196 | // Internal interface for testing purpose only. |
197 | void InstrProfWriter::setValueProfDataEndianness(llvm::endianness Endianness) { |
198 | InfoObj->ValueProfDataEndianness = Endianness; |
199 | } |
200 | |
201 | void InstrProfWriter::setOutputSparse(bool Sparse) { |
202 | this->Sparse = Sparse; |
203 | } |
204 | |
205 | void InstrProfWriter::addRecord(NamedInstrProfRecord &&I, uint64_t Weight, |
206 | function_ref<void(Error)> Warn) { |
207 | auto Name = I.Name; |
208 | auto Hash = I.Hash; |
209 | addRecord(Name, Hash, I: std::move(I), Weight, Warn); |
210 | } |
211 | |
212 | void InstrProfWriter::overlapRecord(NamedInstrProfRecord &&Other, |
213 | OverlapStats &Overlap, |
214 | OverlapStats &FuncLevelOverlap, |
215 | const OverlapFuncFilters &FuncFilter) { |
216 | auto Name = Other.Name; |
217 | auto Hash = Other.Hash; |
218 | Other.accumulateCounts(Sum&: FuncLevelOverlap.Test); |
219 | if (!FunctionData.contains(Key: Name)) { |
220 | Overlap.addOneUnique(UniqueFunc: FuncLevelOverlap.Test); |
221 | return; |
222 | } |
223 | if (FuncLevelOverlap.Test.CountSum < 1.0f) { |
224 | Overlap.Overlap.NumEntries += 1; |
225 | return; |
226 | } |
227 | auto &ProfileDataMap = FunctionData[Name]; |
228 | bool NewFunc; |
229 | ProfilingData::iterator Where; |
230 | std::tie(args&: Where, args&: NewFunc) = |
231 | ProfileDataMap.insert(KV: std::make_pair(x&: Hash, y: InstrProfRecord())); |
232 | if (NewFunc) { |
233 | Overlap.addOneMismatch(MismatchFunc: FuncLevelOverlap.Test); |
234 | return; |
235 | } |
236 | InstrProfRecord &Dest = Where->second; |
237 | |
238 | uint64_t ValueCutoff = FuncFilter.ValueCutoff; |
239 | if (!FuncFilter.NameFilter.empty() && Name.contains(Other: FuncFilter.NameFilter)) |
240 | ValueCutoff = 0; |
241 | |
242 | Dest.overlap(Other, Overlap, FuncLevelOverlap, ValueCutoff); |
243 | } |
244 | |
245 | void InstrProfWriter::addRecord(StringRef Name, uint64_t Hash, |
246 | InstrProfRecord &&I, uint64_t Weight, |
247 | function_ref<void(Error)> Warn) { |
248 | auto &ProfileDataMap = FunctionData[Name]; |
249 | |
250 | bool NewFunc; |
251 | ProfilingData::iterator Where; |
252 | std::tie(args&: Where, args&: NewFunc) = |
253 | ProfileDataMap.insert(KV: std::make_pair(x&: Hash, y: InstrProfRecord())); |
254 | InstrProfRecord &Dest = Where->second; |
255 | |
256 | auto MapWarn = [&](instrprof_error E) { |
257 | Warn(make_error<InstrProfError>(Args&: E)); |
258 | }; |
259 | |
260 | if (NewFunc) { |
261 | // We've never seen a function with this name and hash, add it. |
262 | Dest = std::move(I); |
263 | if (Weight > 1) |
264 | Dest.scale(N: Weight, D: 1, Warn: MapWarn); |
265 | } else { |
266 | // We're updating a function we've seen before. |
267 | Dest.merge(Other&: I, Weight, Warn: MapWarn); |
268 | } |
269 | |
270 | Dest.sortValueData(); |
271 | } |
272 | |
273 | void InstrProfWriter::addMemProfRecord( |
274 | const Function::GUID Id, const memprof::IndexedMemProfRecord &Record) { |
275 | auto [Iter, Inserted] = MemProfRecordData.insert(KV: {Id, Record}); |
276 | // If we inserted a new record then we are done. |
277 | if (Inserted) { |
278 | return; |
279 | } |
280 | memprof::IndexedMemProfRecord &Existing = Iter->second; |
281 | Existing.merge(Other: Record); |
282 | } |
283 | |
284 | bool InstrProfWriter::addMemProfFrame(const memprof::FrameId Id, |
285 | const memprof::Frame &Frame, |
286 | function_ref<void(Error)> Warn) { |
287 | auto [Iter, Inserted] = MemProfFrameData.insert(KV: {Id, Frame}); |
288 | // If a mapping already exists for the current frame id and it does not |
289 | // match the new mapping provided then reset the existing contents and bail |
290 | // out. We don't support the merging of memprof data whose Frame -> Id |
291 | // mapping across profiles is inconsistent. |
292 | if (!Inserted && Iter->second != Frame) { |
293 | Warn(make_error<InstrProfError>(Args: instrprof_error::malformed, |
294 | Args: "frame to id mapping mismatch" )); |
295 | return false; |
296 | } |
297 | return true; |
298 | } |
299 | |
300 | bool InstrProfWriter::addMemProfCallStack( |
301 | const memprof::CallStackId CSId, |
302 | const llvm::SmallVector<memprof::FrameId> &CallStack, |
303 | function_ref<void(Error)> Warn) { |
304 | auto [Iter, Inserted] = MemProfCallStackData.insert(KV: {CSId, CallStack}); |
305 | // If a mapping already exists for the current call stack id and it does not |
306 | // match the new mapping provided then reset the existing contents and bail |
307 | // out. We don't support the merging of memprof data whose CallStack -> Id |
308 | // mapping across profiles is inconsistent. |
309 | if (!Inserted && Iter->second != CallStack) { |
310 | Warn(make_error<InstrProfError>(Args: instrprof_error::malformed, |
311 | Args: "call stack to id mapping mismatch" )); |
312 | return false; |
313 | } |
314 | return true; |
315 | } |
316 | |
317 | void InstrProfWriter::addBinaryIds(ArrayRef<llvm::object::BuildID> BIs) { |
318 | llvm::append_range(C&: BinaryIds, R&: BIs); |
319 | } |
320 | |
321 | void InstrProfWriter::addTemporalProfileTrace(TemporalProfTraceTy Trace) { |
322 | if (Trace.FunctionNameRefs.size() > MaxTemporalProfTraceLength) |
323 | Trace.FunctionNameRefs.resize(new_size: MaxTemporalProfTraceLength); |
324 | if (Trace.FunctionNameRefs.empty()) |
325 | return; |
326 | |
327 | if (TemporalProfTraceStreamSize < TemporalProfTraceReservoirSize) { |
328 | // Simply append the trace if we have not yet hit our reservoir size limit. |
329 | TemporalProfTraces.push_back(Elt: std::move(Trace)); |
330 | } else { |
331 | // Otherwise, replace a random trace in the stream. |
332 | std::uniform_int_distribution<uint64_t> Distribution( |
333 | 0, TemporalProfTraceStreamSize); |
334 | uint64_t RandomIndex = Distribution(RNG); |
335 | if (RandomIndex < TemporalProfTraces.size()) |
336 | TemporalProfTraces[RandomIndex] = std::move(Trace); |
337 | } |
338 | ++TemporalProfTraceStreamSize; |
339 | } |
340 | |
341 | void InstrProfWriter::addTemporalProfileTraces( |
342 | SmallVectorImpl<TemporalProfTraceTy> &SrcTraces, uint64_t SrcStreamSize) { |
343 | // Assume that the source has the same reservoir size as the destination to |
344 | // avoid needing to record it in the indexed profile format. |
345 | bool IsDestSampled = |
346 | (TemporalProfTraceStreamSize > TemporalProfTraceReservoirSize); |
347 | bool IsSrcSampled = (SrcStreamSize > TemporalProfTraceReservoirSize); |
348 | if (!IsDestSampled && IsSrcSampled) { |
349 | // If one of the traces are sampled, ensure that it belongs to Dest. |
350 | std::swap(LHS&: TemporalProfTraces, RHS&: SrcTraces); |
351 | std::swap(a&: TemporalProfTraceStreamSize, b&: SrcStreamSize); |
352 | std::swap(a&: IsDestSampled, b&: IsSrcSampled); |
353 | } |
354 | if (!IsSrcSampled) { |
355 | // If the source stream is not sampled, we add each source trace normally. |
356 | for (auto &Trace : SrcTraces) |
357 | addTemporalProfileTrace(Trace: std::move(Trace)); |
358 | return; |
359 | } |
360 | // Otherwise, we find the traces that would have been removed if we added |
361 | // the whole source stream. |
362 | SmallSetVector<uint64_t, 8> IndicesToReplace; |
363 | for (uint64_t I = 0; I < SrcStreamSize; I++) { |
364 | std::uniform_int_distribution<uint64_t> Distribution( |
365 | 0, TemporalProfTraceStreamSize); |
366 | uint64_t RandomIndex = Distribution(RNG); |
367 | if (RandomIndex < TemporalProfTraces.size()) |
368 | IndicesToReplace.insert(X: RandomIndex); |
369 | ++TemporalProfTraceStreamSize; |
370 | } |
371 | // Then we insert a random sample of the source traces. |
372 | llvm::shuffle(first: SrcTraces.begin(), last: SrcTraces.end(), g&: RNG); |
373 | for (const auto &[Index, Trace] : llvm::zip(t&: IndicesToReplace, u&: SrcTraces)) |
374 | TemporalProfTraces[Index] = std::move(Trace); |
375 | } |
376 | |
377 | void InstrProfWriter::mergeRecordsFromWriter(InstrProfWriter &&IPW, |
378 | function_ref<void(Error)> Warn) { |
379 | for (auto &I : IPW.FunctionData) |
380 | for (auto &Func : I.getValue()) |
381 | addRecord(Name: I.getKey(), Hash: Func.first, I: std::move(Func.second), Weight: 1, Warn); |
382 | |
383 | BinaryIds.reserve(n: BinaryIds.size() + IPW.BinaryIds.size()); |
384 | for (auto &I : IPW.BinaryIds) |
385 | addBinaryIds(BIs: I); |
386 | |
387 | addTemporalProfileTraces(SrcTraces&: IPW.TemporalProfTraces, |
388 | SrcStreamSize: IPW.TemporalProfTraceStreamSize); |
389 | |
390 | MemProfFrameData.reserve(NumEntries: IPW.MemProfFrameData.size()); |
391 | for (auto &[FrameId, Frame] : IPW.MemProfFrameData) { |
392 | // If we weren't able to add the frame mappings then it doesn't make sense |
393 | // to try to merge the records from this profile. |
394 | if (!addMemProfFrame(Id: FrameId, Frame, Warn)) |
395 | return; |
396 | } |
397 | |
398 | MemProfCallStackData.reserve(NumEntries: IPW.MemProfCallStackData.size()); |
399 | for (auto &[CSId, CallStack] : IPW.MemProfCallStackData) { |
400 | if (!addMemProfCallStack(CSId, CallStack, Warn)) |
401 | return; |
402 | } |
403 | |
404 | MemProfRecordData.reserve(NumEntries: IPW.MemProfRecordData.size()); |
405 | for (auto &[GUID, Record] : IPW.MemProfRecordData) { |
406 | addMemProfRecord(Id: GUID, Record); |
407 | } |
408 | } |
409 | |
410 | bool InstrProfWriter::shouldEncodeData(const ProfilingData &PD) { |
411 | if (!Sparse) |
412 | return true; |
413 | for (const auto &Func : PD) { |
414 | const InstrProfRecord &IPR = Func.second; |
415 | if (llvm::any_of(Range: IPR.Counts, P: [](uint64_t Count) { return Count > 0; })) |
416 | return true; |
417 | if (llvm::any_of(Range: IPR.BitmapBytes, P: [](uint8_t Byte) { return Byte > 0; })) |
418 | return true; |
419 | } |
420 | return false; |
421 | } |
422 | |
423 | static void setSummary(IndexedInstrProf::Summary *TheSummary, |
424 | ProfileSummary &PS) { |
425 | using namespace IndexedInstrProf; |
426 | |
427 | const std::vector<ProfileSummaryEntry> &Res = PS.getDetailedSummary(); |
428 | TheSummary->NumSummaryFields = Summary::NumKinds; |
429 | TheSummary->NumCutoffEntries = Res.size(); |
430 | TheSummary->set(K: Summary::MaxFunctionCount, V: PS.getMaxFunctionCount()); |
431 | TheSummary->set(K: Summary::MaxBlockCount, V: PS.getMaxCount()); |
432 | TheSummary->set(K: Summary::MaxInternalBlockCount, V: PS.getMaxInternalCount()); |
433 | TheSummary->set(K: Summary::TotalBlockCount, V: PS.getTotalCount()); |
434 | TheSummary->set(K: Summary::TotalNumBlocks, V: PS.getNumCounts()); |
435 | TheSummary->set(K: Summary::TotalNumFunctions, V: PS.getNumFunctions()); |
436 | for (unsigned I = 0; I < Res.size(); I++) |
437 | TheSummary->setEntry(I, E: Res[I]); |
438 | } |
439 | |
440 | // Serialize Schema. |
441 | static void writeMemProfSchema(ProfOStream &OS, |
442 | const memprof::MemProfSchema &Schema) { |
443 | OS.write(V: static_cast<uint64_t>(Schema.size())); |
444 | for (const auto Id : Schema) |
445 | OS.write(V: static_cast<uint64_t>(Id)); |
446 | } |
447 | |
448 | // Serialize MemProfRecordData. Return RecordTableOffset. |
449 | static uint64_t writeMemProfRecords( |
450 | ProfOStream &OS, |
451 | llvm::MapVector<GlobalValue::GUID, memprof::IndexedMemProfRecord> |
452 | &MemProfRecordData, |
453 | memprof::MemProfSchema *Schema, memprof::IndexedVersion Version) { |
454 | memprof::RecordWriterTrait RecordWriter(Schema, Version); |
455 | OnDiskChainedHashTableGenerator<memprof::RecordWriterTrait> |
456 | RecordTableGenerator; |
457 | for (auto &[GUID, Record] : MemProfRecordData) { |
458 | // Insert the key (func hash) and value (memprof record). |
459 | RecordTableGenerator.insert(Key: GUID, Data&: Record, InfoObj&: RecordWriter); |
460 | } |
461 | // Release the memory of this MapVector as it is no longer needed. |
462 | MemProfRecordData.clear(); |
463 | |
464 | // The call to Emit invokes RecordWriterTrait::EmitData which destructs |
465 | // the memprof record copies owned by the RecordTableGenerator. This works |
466 | // because the RecordTableGenerator is not used after this point. |
467 | return RecordTableGenerator.Emit(Out&: OS.OS, InfoObj&: RecordWriter); |
468 | } |
469 | |
470 | // Serialize MemProfFrameData. Return FrameTableOffset. |
471 | static uint64_t writeMemProfFrames( |
472 | ProfOStream &OS, |
473 | llvm::MapVector<memprof::FrameId, memprof::Frame> &MemProfFrameData) { |
474 | OnDiskChainedHashTableGenerator<memprof::FrameWriterTrait> |
475 | FrameTableGenerator; |
476 | for (auto &[FrameId, Frame] : MemProfFrameData) { |
477 | // Insert the key (frame id) and value (frame contents). |
478 | FrameTableGenerator.insert(Key: FrameId, Data&: Frame); |
479 | } |
480 | // Release the memory of this MapVector as it is no longer needed. |
481 | MemProfFrameData.clear(); |
482 | |
483 | return FrameTableGenerator.Emit(Out&: OS.OS); |
484 | } |
485 | |
486 | static uint64_t writeMemProfCallStacks( |
487 | ProfOStream &OS, |
488 | llvm::MapVector<memprof::CallStackId, llvm::SmallVector<memprof::FrameId>> |
489 | &MemProfCallStackData) { |
490 | OnDiskChainedHashTableGenerator<memprof::CallStackWriterTrait> |
491 | CallStackTableGenerator; |
492 | for (auto &[CSId, CallStack] : MemProfCallStackData) |
493 | CallStackTableGenerator.insert(Key: CSId, Data&: CallStack); |
494 | // Release the memory of this vector as it is no longer needed. |
495 | MemProfCallStackData.clear(); |
496 | |
497 | return CallStackTableGenerator.Emit(Out&: OS.OS); |
498 | } |
499 | |
500 | static Error writeMemProfV0( |
501 | ProfOStream &OS, |
502 | llvm::MapVector<GlobalValue::GUID, memprof::IndexedMemProfRecord> |
503 | &MemProfRecordData, |
504 | llvm::MapVector<memprof::FrameId, memprof::Frame> &MemProfFrameData) { |
505 | uint64_t = OS.tell(); |
506 | OS.write(V: 0ULL); // Reserve space for the memprof record table offset. |
507 | OS.write(V: 0ULL); // Reserve space for the memprof frame payload offset. |
508 | OS.write(V: 0ULL); // Reserve space for the memprof frame table offset. |
509 | |
510 | auto Schema = memprof::PortableMemInfoBlock::getSchema(); |
511 | writeMemProfSchema(OS, Schema); |
512 | |
513 | uint64_t RecordTableOffset = |
514 | writeMemProfRecords(OS, MemProfRecordData, Schema: &Schema, Version: memprof::Version0); |
515 | |
516 | uint64_t FramePayloadOffset = OS.tell(); |
517 | uint64_t FrameTableOffset = writeMemProfFrames(OS, MemProfFrameData); |
518 | |
519 | uint64_t [] = {RecordTableOffset, FramePayloadOffset, FrameTableOffset}; |
520 | OS.patch(P: {{.Pos: HeaderUpdatePos, .D: Header, .N: std::size(Header)}}); |
521 | |
522 | return Error::success(); |
523 | } |
524 | |
525 | static Error writeMemProfV1( |
526 | ProfOStream &OS, |
527 | llvm::MapVector<GlobalValue::GUID, memprof::IndexedMemProfRecord> |
528 | &MemProfRecordData, |
529 | llvm::MapVector<memprof::FrameId, memprof::Frame> &MemProfFrameData) { |
530 | OS.write(V: memprof::Version1); |
531 | uint64_t = OS.tell(); |
532 | OS.write(V: 0ULL); // Reserve space for the memprof record table offset. |
533 | OS.write(V: 0ULL); // Reserve space for the memprof frame payload offset. |
534 | OS.write(V: 0ULL); // Reserve space for the memprof frame table offset. |
535 | |
536 | auto Schema = memprof::PortableMemInfoBlock::getSchema(); |
537 | writeMemProfSchema(OS, Schema); |
538 | |
539 | uint64_t RecordTableOffset = |
540 | writeMemProfRecords(OS, MemProfRecordData, Schema: &Schema, Version: memprof::Version1); |
541 | |
542 | uint64_t FramePayloadOffset = OS.tell(); |
543 | uint64_t FrameTableOffset = writeMemProfFrames(OS, MemProfFrameData); |
544 | |
545 | uint64_t [] = {RecordTableOffset, FramePayloadOffset, FrameTableOffset}; |
546 | OS.patch(P: {{.Pos: HeaderUpdatePos, .D: Header, .N: std::size(Header)}}); |
547 | |
548 | return Error::success(); |
549 | } |
550 | |
551 | static Error writeMemProfV2( |
552 | ProfOStream &OS, |
553 | llvm::MapVector<GlobalValue::GUID, memprof::IndexedMemProfRecord> |
554 | &MemProfRecordData, |
555 | llvm::MapVector<memprof::FrameId, memprof::Frame> &MemProfFrameData, |
556 | llvm::MapVector<memprof::CallStackId, llvm::SmallVector<memprof::FrameId>> |
557 | &MemProfCallStackData) { |
558 | OS.write(V: memprof::Version2); |
559 | uint64_t = OS.tell(); |
560 | OS.write(V: 0ULL); // Reserve space for the memprof record table offset. |
561 | OS.write(V: 0ULL); // Reserve space for the memprof frame payload offset. |
562 | OS.write(V: 0ULL); // Reserve space for the memprof frame table offset. |
563 | OS.write(V: 0ULL); // Reserve space for the memprof call stack payload offset. |
564 | OS.write(V: 0ULL); // Reserve space for the memprof call stack table offset. |
565 | |
566 | auto Schema = memprof::PortableMemInfoBlock::getSchema(); |
567 | writeMemProfSchema(OS, Schema); |
568 | |
569 | uint64_t RecordTableOffset = |
570 | writeMemProfRecords(OS, MemProfRecordData, Schema: &Schema, Version: memprof::Version2); |
571 | |
572 | uint64_t FramePayloadOffset = OS.tell(); |
573 | uint64_t FrameTableOffset = writeMemProfFrames(OS, MemProfFrameData); |
574 | |
575 | uint64_t CallStackPayloadOffset = OS.tell(); |
576 | uint64_t CallStackTableOffset = |
577 | writeMemProfCallStacks(OS, MemProfCallStackData); |
578 | |
579 | uint64_t [] = { |
580 | RecordTableOffset, FramePayloadOffset, FrameTableOffset, |
581 | CallStackPayloadOffset, CallStackTableOffset, |
582 | }; |
583 | OS.patch(P: {{.Pos: HeaderUpdatePos, .D: Header, .N: std::size(Header)}}); |
584 | |
585 | return Error::success(); |
586 | } |
587 | |
588 | // The MemProf profile data includes a simple schema |
589 | // with the format described below followed by the hashtable: |
590 | // uint64_t Version |
591 | // uint64_t RecordTableOffset = RecordTableGenerator.Emit |
592 | // uint64_t FramePayloadOffset = Stream offset before emitting the frame table |
593 | // uint64_t FrameTableOffset = FrameTableGenerator.Emit |
594 | // uint64_t Num schema entries |
595 | // uint64_t Schema entry 0 |
596 | // uint64_t Schema entry 1 |
597 | // .... |
598 | // uint64_t Schema entry N - 1 |
599 | // OnDiskChainedHashTable MemProfRecordData |
600 | // OnDiskChainedHashTable MemProfFrameData |
601 | static Error writeMemProf( |
602 | ProfOStream &OS, |
603 | llvm::MapVector<GlobalValue::GUID, memprof::IndexedMemProfRecord> |
604 | &MemProfRecordData, |
605 | llvm::MapVector<memprof::FrameId, memprof::Frame> &MemProfFrameData, |
606 | llvm::MapVector<memprof::CallStackId, llvm::SmallVector<memprof::FrameId>> |
607 | &MemProfCallStackData, |
608 | memprof::IndexedVersion MemProfVersionRequested) { |
609 | |
610 | switch (MemProfVersionRequested) { |
611 | case memprof::Version0: |
612 | return writeMemProfV0(OS, MemProfRecordData, MemProfFrameData); |
613 | case memprof::Version1: |
614 | return writeMemProfV1(OS, MemProfRecordData, MemProfFrameData); |
615 | case memprof::Version2: |
616 | return writeMemProfV2(OS, MemProfRecordData, MemProfFrameData, |
617 | MemProfCallStackData); |
618 | } |
619 | |
620 | return make_error<InstrProfError>( |
621 | Args: instrprof_error::unsupported_version, |
622 | Args: formatv(Fmt: "MemProf version {} not supported; " |
623 | "requires version between {} and {}, inclusive" , |
624 | Vals&: MemProfVersionRequested, Vals: memprof::MinimumSupportedVersion, |
625 | Vals: memprof::MaximumSupportedVersion)); |
626 | } |
627 | |
628 | Error InstrProfWriter::writeImpl(ProfOStream &OS) { |
629 | using namespace IndexedInstrProf; |
630 | using namespace support; |
631 | |
632 | OnDiskChainedHashTableGenerator<InstrProfRecordWriterTrait> Generator; |
633 | |
634 | InstrProfSummaryBuilder ISB(ProfileSummaryBuilder::DefaultCutoffs); |
635 | InfoObj->SummaryBuilder = &ISB; |
636 | InstrProfSummaryBuilder CSISB(ProfileSummaryBuilder::DefaultCutoffs); |
637 | InfoObj->CSSummaryBuilder = &CSISB; |
638 | |
639 | // Populate the hash table generator. |
640 | SmallVector<std::pair<StringRef, const ProfilingData *>, 0> OrderedData; |
641 | for (const auto &I : FunctionData) |
642 | if (shouldEncodeData(PD: I.getValue())) |
643 | OrderedData.emplace_back(Args: (I.getKey()), Args: &I.getValue()); |
644 | llvm::sort(C&: OrderedData, Comp: less_first()); |
645 | for (const auto &I : OrderedData) |
646 | Generator.insert(Key: I.first, Data: I.second); |
647 | |
648 | // Write the header. |
649 | IndexedInstrProf::Header ; |
650 | Header.Magic = IndexedInstrProf::Magic; |
651 | Header.Version = WritePrevVersion |
652 | ? IndexedInstrProf::ProfVersion::Version11 |
653 | : IndexedInstrProf::ProfVersion::CurrentVersion; |
654 | // The WritePrevVersion handling will either need to be removed or updated |
655 | // if the version is advanced beyond 12. |
656 | assert(IndexedInstrProf::ProfVersion::CurrentVersion == |
657 | IndexedInstrProf::ProfVersion::Version12); |
658 | if (static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation)) |
659 | Header.Version |= VARIANT_MASK_IR_PROF; |
660 | if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) |
661 | Header.Version |= VARIANT_MASK_CSIR_PROF; |
662 | if (static_cast<bool>(ProfileKind & |
663 | InstrProfKind::FunctionEntryInstrumentation)) |
664 | Header.Version |= VARIANT_MASK_INSTR_ENTRY; |
665 | if (static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage)) |
666 | Header.Version |= VARIANT_MASK_BYTE_COVERAGE; |
667 | if (static_cast<bool>(ProfileKind & InstrProfKind::FunctionEntryOnly)) |
668 | Header.Version |= VARIANT_MASK_FUNCTION_ENTRY_ONLY; |
669 | if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf)) |
670 | Header.Version |= VARIANT_MASK_MEMPROF; |
671 | if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile)) |
672 | Header.Version |= VARIANT_MASK_TEMPORAL_PROF; |
673 | |
674 | Header.Unused = 0; |
675 | Header.HashType = static_cast<uint64_t>(IndexedInstrProf::HashType); |
676 | Header.HashOffset = 0; |
677 | Header.MemProfOffset = 0; |
678 | Header.BinaryIdOffset = 0; |
679 | Header.TemporalProfTracesOffset = 0; |
680 | Header.VTableNamesOffset = 0; |
681 | |
682 | // Only write out the first four fields. We need to remember the offset of the |
683 | // remaining fields to allow back patching later. |
684 | for (int I = 0; I < 4; I++) |
685 | OS.write(V: reinterpret_cast<uint64_t *>(&Header)[I]); |
686 | |
687 | // Save the location of Header.HashOffset field in \c OS. |
688 | uint64_t HashTableStartFieldOffset = OS.tell(); |
689 | // Reserve the space for HashOffset field. |
690 | OS.write(V: 0); |
691 | |
692 | // Save the location of MemProf profile data. This is stored in two parts as |
693 | // the schema and as a separate on-disk chained hashtable. |
694 | uint64_t MemProfSectionOffset = OS.tell(); |
695 | // Reserve space for the MemProf table field to be patched later if this |
696 | // profile contains memory profile information. |
697 | OS.write(V: 0); |
698 | |
699 | // Save the location of binary ids section. |
700 | uint64_t BinaryIdSectionOffset = OS.tell(); |
701 | // Reserve space for the BinaryIdOffset field to be patched later if this |
702 | // profile contains binary ids. |
703 | OS.write(V: 0); |
704 | |
705 | uint64_t TemporalProfTracesOffset = OS.tell(); |
706 | OS.write(V: 0); |
707 | |
708 | uint64_t VTableNamesOffset = OS.tell(); |
709 | if (!WritePrevVersion) |
710 | OS.write(V: 0); |
711 | |
712 | // Reserve space to write profile summary data. |
713 | uint32_t NumEntries = ProfileSummaryBuilder::DefaultCutoffs.size(); |
714 | uint32_t SummarySize = Summary::getSize(NumSumFields: Summary::NumKinds, NumCutoffEntries: NumEntries); |
715 | // Remember the summary offset. |
716 | uint64_t SummaryOffset = OS.tell(); |
717 | for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++) |
718 | OS.write(V: 0); |
719 | uint64_t CSSummaryOffset = 0; |
720 | uint64_t CSSummarySize = 0; |
721 | if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) { |
722 | CSSummaryOffset = OS.tell(); |
723 | CSSummarySize = SummarySize / sizeof(uint64_t); |
724 | for (unsigned I = 0; I < CSSummarySize; I++) |
725 | OS.write(V: 0); |
726 | } |
727 | |
728 | // Write the hash table. |
729 | uint64_t HashTableStart = Generator.Emit(Out&: OS.OS, InfoObj&: *InfoObj); |
730 | |
731 | // Write the MemProf profile data if we have it. |
732 | uint64_t MemProfSectionStart = 0; |
733 | if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf)) { |
734 | MemProfSectionStart = OS.tell(); |
735 | if (auto E = writeMemProf(OS, MemProfRecordData, MemProfFrameData, |
736 | MemProfCallStackData, MemProfVersionRequested)) |
737 | return E; |
738 | } |
739 | |
740 | // BinaryIdSection has two parts: |
741 | // 1. uint64_t BinaryIdsSectionSize |
742 | // 2. list of binary ids that consist of: |
743 | // a. uint64_t BinaryIdLength |
744 | // b. uint8_t BinaryIdData |
745 | // c. uint8_t Padding (if necessary) |
746 | uint64_t BinaryIdSectionStart = OS.tell(); |
747 | // Calculate size of binary section. |
748 | uint64_t BinaryIdsSectionSize = 0; |
749 | |
750 | // Remove duplicate binary ids. |
751 | llvm::sort(C&: BinaryIds); |
752 | BinaryIds.erase(first: std::unique(first: BinaryIds.begin(), last: BinaryIds.end()), |
753 | last: BinaryIds.end()); |
754 | |
755 | for (auto BI : BinaryIds) { |
756 | // Increment by binary id length data type size. |
757 | BinaryIdsSectionSize += sizeof(uint64_t); |
758 | // Increment by binary id data length, aligned to 8 bytes. |
759 | BinaryIdsSectionSize += alignToPowerOf2(Value: BI.size(), Align: sizeof(uint64_t)); |
760 | } |
761 | // Write binary ids section size. |
762 | OS.write(V: BinaryIdsSectionSize); |
763 | |
764 | for (auto BI : BinaryIds) { |
765 | uint64_t BILen = BI.size(); |
766 | // Write binary id length. |
767 | OS.write(V: BILen); |
768 | // Write binary id data. |
769 | for (unsigned K = 0; K < BILen; K++) |
770 | OS.writeByte(V: BI[K]); |
771 | // Write padding if necessary. |
772 | uint64_t PaddingSize = alignToPowerOf2(Value: BILen, Align: sizeof(uint64_t)) - BILen; |
773 | for (unsigned K = 0; K < PaddingSize; K++) |
774 | OS.writeByte(V: 0); |
775 | } |
776 | |
777 | uint64_t VTableNamesSectionStart = OS.tell(); |
778 | |
779 | if (!WritePrevVersion) { |
780 | std::vector<std::string> VTableNameStrs; |
781 | for (StringRef VTableName : VTableNames.keys()) |
782 | VTableNameStrs.push_back(x: VTableName.str()); |
783 | |
784 | std::string CompressedVTableNames; |
785 | if (!VTableNameStrs.empty()) |
786 | if (Error E = collectGlobalObjectNameStrings( |
787 | NameStrs: VTableNameStrs, doCompression: compression::zlib::isAvailable(), |
788 | Result&: CompressedVTableNames)) |
789 | return E; |
790 | |
791 | const uint64_t CompressedStringLen = CompressedVTableNames.length(); |
792 | |
793 | // Record the length of compressed string. |
794 | OS.write(V: CompressedStringLen); |
795 | |
796 | // Write the chars in compressed strings. |
797 | for (auto &c : CompressedVTableNames) |
798 | OS.writeByte(V: static_cast<uint8_t>(c)); |
799 | |
800 | // Pad up to a multiple of 8. |
801 | // InstrProfReader could read bytes according to 'CompressedStringLen'. |
802 | const uint64_t PaddedLength = alignTo(Value: CompressedStringLen, Align: 8); |
803 | |
804 | for (uint64_t K = CompressedStringLen; K < PaddedLength; K++) |
805 | OS.writeByte(V: 0); |
806 | } |
807 | |
808 | uint64_t TemporalProfTracesSectionStart = 0; |
809 | if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile)) { |
810 | TemporalProfTracesSectionStart = OS.tell(); |
811 | OS.write(V: TemporalProfTraces.size()); |
812 | OS.write(V: TemporalProfTraceStreamSize); |
813 | for (auto &Trace : TemporalProfTraces) { |
814 | OS.write(V: Trace.Weight); |
815 | OS.write(V: Trace.FunctionNameRefs.size()); |
816 | for (auto &NameRef : Trace.FunctionNameRefs) |
817 | OS.write(V: NameRef); |
818 | } |
819 | } |
820 | |
821 | // Allocate space for data to be serialized out. |
822 | std::unique_ptr<IndexedInstrProf::Summary> TheSummary = |
823 | IndexedInstrProf::allocSummary(TotalSize: SummarySize); |
824 | // Compute the Summary and copy the data to the data |
825 | // structure to be serialized out (to disk or buffer). |
826 | std::unique_ptr<ProfileSummary> PS = ISB.getSummary(); |
827 | setSummary(TheSummary: TheSummary.get(), PS&: *PS); |
828 | InfoObj->SummaryBuilder = nullptr; |
829 | |
830 | // For Context Sensitive summary. |
831 | std::unique_ptr<IndexedInstrProf::Summary> TheCSSummary = nullptr; |
832 | if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) { |
833 | TheCSSummary = IndexedInstrProf::allocSummary(TotalSize: SummarySize); |
834 | std::unique_ptr<ProfileSummary> CSPS = CSISB.getSummary(); |
835 | setSummary(TheSummary: TheCSSummary.get(), PS&: *CSPS); |
836 | } |
837 | InfoObj->CSSummaryBuilder = nullptr; |
838 | |
839 | if (!WritePrevVersion) { |
840 | // Now do the final patch: |
841 | PatchItem PatchItems[] = { |
842 | // Patch the Header.HashOffset field. |
843 | {.Pos: HashTableStartFieldOffset, .D: &HashTableStart, .N: 1}, |
844 | // Patch the Header.MemProfOffset (=0 for profiles without MemProf |
845 | // data). |
846 | {.Pos: MemProfSectionOffset, .D: &MemProfSectionStart, .N: 1}, |
847 | // Patch the Header.BinaryIdSectionOffset. |
848 | {.Pos: BinaryIdSectionOffset, .D: &BinaryIdSectionStart, .N: 1}, |
849 | // Patch the Header.TemporalProfTracesOffset (=0 for profiles without |
850 | // traces). |
851 | {.Pos: TemporalProfTracesOffset, .D: &TemporalProfTracesSectionStart, .N: 1}, |
852 | {.Pos: VTableNamesOffset, .D: &VTableNamesSectionStart, .N: 1}, |
853 | // Patch the summary data. |
854 | {.Pos: SummaryOffset, .D: reinterpret_cast<uint64_t *>(TheSummary.get()), |
855 | .N: (int)(SummarySize / sizeof(uint64_t))}, |
856 | {.Pos: CSSummaryOffset, .D: reinterpret_cast<uint64_t *>(TheCSSummary.get()), |
857 | .N: (int)CSSummarySize}}; |
858 | |
859 | OS.patch(P: PatchItems); |
860 | } else { |
861 | // Now do the final patch: |
862 | PatchItem PatchItems[] = { |
863 | // Patch the Header.HashOffset field. |
864 | {.Pos: HashTableStartFieldOffset, .D: &HashTableStart, .N: 1}, |
865 | // Patch the Header.MemProfOffset (=0 for profiles without MemProf |
866 | // data). |
867 | {.Pos: MemProfSectionOffset, .D: &MemProfSectionStart, .N: 1}, |
868 | // Patch the Header.BinaryIdSectionOffset. |
869 | {.Pos: BinaryIdSectionOffset, .D: &BinaryIdSectionStart, .N: 1}, |
870 | // Patch the Header.TemporalProfTracesOffset (=0 for profiles without |
871 | // traces). |
872 | {.Pos: TemporalProfTracesOffset, .D: &TemporalProfTracesSectionStart, .N: 1}, |
873 | // Patch the summary data. |
874 | {.Pos: SummaryOffset, .D: reinterpret_cast<uint64_t *>(TheSummary.get()), |
875 | .N: (int)(SummarySize / sizeof(uint64_t))}, |
876 | {.Pos: CSSummaryOffset, .D: reinterpret_cast<uint64_t *>(TheCSSummary.get()), |
877 | .N: (int)CSSummarySize}}; |
878 | |
879 | OS.patch(P: PatchItems); |
880 | } |
881 | |
882 | for (const auto &I : FunctionData) |
883 | for (const auto &F : I.getValue()) |
884 | if (Error E = validateRecord(Func: F.second)) |
885 | return E; |
886 | |
887 | return Error::success(); |
888 | } |
889 | |
890 | Error InstrProfWriter::write(raw_fd_ostream &OS) { |
891 | // Write the hash table. |
892 | ProfOStream POS(OS); |
893 | return writeImpl(OS&: POS); |
894 | } |
895 | |
896 | Error InstrProfWriter::write(raw_string_ostream &OS) { |
897 | ProfOStream POS(OS); |
898 | return writeImpl(OS&: POS); |
899 | } |
900 | |
901 | std::unique_ptr<MemoryBuffer> InstrProfWriter::writeBuffer() { |
902 | std::string Data; |
903 | raw_string_ostream OS(Data); |
904 | // Write the hash table. |
905 | if (Error E = write(OS)) |
906 | return nullptr; |
907 | // Return this in an aligned memory buffer. |
908 | return MemoryBuffer::getMemBufferCopy(InputData: Data); |
909 | } |
910 | |
911 | static const char *ValueProfKindStr[] = { |
912 | #define VALUE_PROF_KIND(Enumerator, Value, Descr) #Enumerator, |
913 | #include "llvm/ProfileData/InstrProfData.inc" |
914 | }; |
915 | |
916 | Error InstrProfWriter::validateRecord(const InstrProfRecord &Func) { |
917 | for (uint32_t VK = 0; VK <= IPVK_Last; VK++) { |
918 | uint32_t NS = Func.getNumValueSites(ValueKind: VK); |
919 | if (!NS) |
920 | continue; |
921 | for (uint32_t S = 0; S < NS; S++) { |
922 | uint32_t ND = Func.getNumValueDataForSite(ValueKind: VK, Site: S); |
923 | std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(ValueKind: VK, Site: S); |
924 | DenseSet<uint64_t> SeenValues; |
925 | for (uint32_t I = 0; I < ND; I++) |
926 | if ((VK != IPVK_IndirectCallTarget && VK != IPVK_VTableTarget) && |
927 | !SeenValues.insert(V: VD[I].Value).second) |
928 | return make_error<InstrProfError>(Args: instrprof_error::invalid_prof); |
929 | } |
930 | } |
931 | |
932 | return Error::success(); |
933 | } |
934 | |
935 | void InstrProfWriter::writeRecordInText(StringRef Name, uint64_t Hash, |
936 | const InstrProfRecord &Func, |
937 | InstrProfSymtab &Symtab, |
938 | raw_fd_ostream &OS) { |
939 | OS << Name << "\n" ; |
940 | OS << "# Func Hash:\n" << Hash << "\n" ; |
941 | OS << "# Num Counters:\n" << Func.Counts.size() << "\n" ; |
942 | OS << "# Counter Values:\n" ; |
943 | for (uint64_t Count : Func.Counts) |
944 | OS << Count << "\n" ; |
945 | |
946 | if (Func.BitmapBytes.size() > 0) { |
947 | OS << "# Num Bitmap Bytes:\n$" << Func.BitmapBytes.size() << "\n" ; |
948 | OS << "# Bitmap Byte Values:\n" ; |
949 | for (uint8_t Byte : Func.BitmapBytes) { |
950 | OS << "0x" ; |
951 | OS.write_hex(N: Byte); |
952 | OS << "\n" ; |
953 | } |
954 | OS << "\n" ; |
955 | } |
956 | |
957 | uint32_t NumValueKinds = Func.getNumValueKinds(); |
958 | if (!NumValueKinds) { |
959 | OS << "\n" ; |
960 | return; |
961 | } |
962 | |
963 | OS << "# Num Value Kinds:\n" << Func.getNumValueKinds() << "\n" ; |
964 | for (uint32_t VK = 0; VK < IPVK_Last + 1; VK++) { |
965 | uint32_t NS = Func.getNumValueSites(ValueKind: VK); |
966 | if (!NS) |
967 | continue; |
968 | OS << "# ValueKind = " << ValueProfKindStr[VK] << ":\n" << VK << "\n" ; |
969 | OS << "# NumValueSites:\n" << NS << "\n" ; |
970 | for (uint32_t S = 0; S < NS; S++) { |
971 | uint32_t ND = Func.getNumValueDataForSite(ValueKind: VK, Site: S); |
972 | OS << ND << "\n" ; |
973 | std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(ValueKind: VK, Site: S); |
974 | for (uint32_t I = 0; I < ND; I++) { |
975 | if (VK == IPVK_IndirectCallTarget || VK == IPVK_VTableTarget) |
976 | OS << Symtab.getFuncOrVarNameIfDefined(MD5Hash: VD[I].Value) << ":" |
977 | << VD[I].Count << "\n" ; |
978 | else |
979 | OS << VD[I].Value << ":" << VD[I].Count << "\n" ; |
980 | } |
981 | } |
982 | } |
983 | |
984 | OS << "\n" ; |
985 | } |
986 | |
987 | Error InstrProfWriter::writeText(raw_fd_ostream &OS) { |
988 | // Check CS first since it implies an IR level profile. |
989 | if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) |
990 | OS << "# CSIR level Instrumentation Flag\n:csir\n" ; |
991 | else if (static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation)) |
992 | OS << "# IR level Instrumentation Flag\n:ir\n" ; |
993 | |
994 | if (static_cast<bool>(ProfileKind & |
995 | InstrProfKind::FunctionEntryInstrumentation)) |
996 | OS << "# Always instrument the function entry block\n:entry_first\n" ; |
997 | if (static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage)) |
998 | OS << "# Instrument block coverage\n:single_byte_coverage\n" ; |
999 | InstrProfSymtab Symtab; |
1000 | |
1001 | using FuncPair = detail::DenseMapPair<uint64_t, InstrProfRecord>; |
1002 | using RecordType = std::pair<StringRef, FuncPair>; |
1003 | SmallVector<RecordType, 4> OrderedFuncData; |
1004 | |
1005 | for (const auto &I : FunctionData) { |
1006 | if (shouldEncodeData(PD: I.getValue())) { |
1007 | if (Error E = Symtab.addFuncName(FuncName: I.getKey())) |
1008 | return E; |
1009 | for (const auto &Func : I.getValue()) |
1010 | OrderedFuncData.push_back(Elt: std::make_pair(x: I.getKey(), y: Func)); |
1011 | } |
1012 | } |
1013 | |
1014 | for (const auto &VTableName : VTableNames) |
1015 | if (Error E = Symtab.addVTableName(VTableName: VTableName.getKey())) |
1016 | return E; |
1017 | |
1018 | if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile)) |
1019 | writeTextTemporalProfTraceData(OS, Symtab); |
1020 | |
1021 | llvm::sort(C&: OrderedFuncData, Comp: [](const RecordType &A, const RecordType &B) { |
1022 | return std::tie(args: A.first, args: A.second.first) < |
1023 | std::tie(args: B.first, args: B.second.first); |
1024 | }); |
1025 | |
1026 | for (const auto &record : OrderedFuncData) { |
1027 | const StringRef &Name = record.first; |
1028 | const FuncPair &Func = record.second; |
1029 | writeRecordInText(Name, Hash: Func.first, Func: Func.second, Symtab, OS); |
1030 | } |
1031 | |
1032 | for (const auto &record : OrderedFuncData) { |
1033 | const FuncPair &Func = record.second; |
1034 | if (Error E = validateRecord(Func: Func.second)) |
1035 | return E; |
1036 | } |
1037 | |
1038 | return Error::success(); |
1039 | } |
1040 | |
1041 | void InstrProfWriter::writeTextTemporalProfTraceData(raw_fd_ostream &OS, |
1042 | InstrProfSymtab &Symtab) { |
1043 | OS << ":temporal_prof_traces\n" ; |
1044 | OS << "# Num Temporal Profile Traces:\n" << TemporalProfTraces.size() << "\n" ; |
1045 | OS << "# Temporal Profile Trace Stream Size:\n" |
1046 | << TemporalProfTraceStreamSize << "\n" ; |
1047 | for (auto &Trace : TemporalProfTraces) { |
1048 | OS << "# Weight:\n" << Trace.Weight << "\n" ; |
1049 | for (auto &NameRef : Trace.FunctionNameRefs) |
1050 | OS << Symtab.getFuncOrVarName(MD5Hash: NameRef) << "," ; |
1051 | OS << "\n" ; |
1052 | } |
1053 | OS << "\n" ; |
1054 | } |
1055 | |