1//===- InstrProfWriter.cpp - Instrumented profiling writer ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains support for writing profiling data for clang's
10// instrumentation based PGO and coverage.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/ProfileData/InstrProfWriter.h"
15#include "llvm/ADT/STLExtras.h"
16#include "llvm/ADT/SetVector.h"
17#include "llvm/ADT/StringRef.h"
18#include "llvm/IR/ProfileSummary.h"
19#include "llvm/ProfileData/InstrProf.h"
20#include "llvm/ProfileData/MemProf.h"
21#include "llvm/ProfileData/ProfileCommon.h"
22#include "llvm/Support/Compression.h"
23#include "llvm/Support/Endian.h"
24#include "llvm/Support/EndianStream.h"
25#include "llvm/Support/Error.h"
26#include "llvm/Support/FormatVariadic.h"
27#include "llvm/Support/MemoryBuffer.h"
28#include "llvm/Support/OnDiskHashTable.h"
29#include "llvm/Support/raw_ostream.h"
30#include <cstdint>
31#include <memory>
32#include <string>
33#include <tuple>
34#include <utility>
35#include <vector>
36
37using namespace llvm;
38
39// A struct to define how the data stream should be patched. For Indexed
40// profiling, only uint64_t data type is needed.
41struct PatchItem {
42 uint64_t Pos; // Where to patch.
43 uint64_t *D; // Pointer to an array of source data.
44 int N; // Number of elements in \c D array.
45};
46
47namespace llvm {
48
49// A wrapper class to abstract writer stream with support of bytes
50// back patching.
51class ProfOStream {
52public:
53 ProfOStream(raw_fd_ostream &FD)
54 : IsFDOStream(true), OS(FD), LE(FD, llvm::endianness::little) {}
55 ProfOStream(raw_string_ostream &STR)
56 : IsFDOStream(false), OS(STR), LE(STR, llvm::endianness::little) {}
57
58 uint64_t tell() { return OS.tell(); }
59 void write(uint64_t V) { LE.write<uint64_t>(Val: V); }
60 void writeByte(uint8_t V) { LE.write<uint8_t>(Val: V); }
61
62 // \c patch can only be called when all data is written and flushed.
63 // For raw_string_ostream, the patch is done on the target string
64 // directly and it won't be reflected in the stream's internal buffer.
65 void patch(ArrayRef<PatchItem> P) {
66 using namespace support;
67
68 if (IsFDOStream) {
69 raw_fd_ostream &FDOStream = static_cast<raw_fd_ostream &>(OS);
70 const uint64_t LastPos = FDOStream.tell();
71 for (const auto &K : P) {
72 FDOStream.seek(off: K.Pos);
73 for (int I = 0; I < K.N; I++)
74 write(V: K.D[I]);
75 }
76 // Reset the stream to the last position after patching so that users
77 // don't accidentally overwrite data. This makes it consistent with
78 // the string stream below which replaces the data directly.
79 FDOStream.seek(off: LastPos);
80 } else {
81 raw_string_ostream &SOStream = static_cast<raw_string_ostream &>(OS);
82 std::string &Data = SOStream.str(); // with flush
83 for (const auto &K : P) {
84 for (int I = 0; I < K.N; I++) {
85 uint64_t Bytes =
86 endian::byte_swap<uint64_t, llvm::endianness::little>(value: K.D[I]);
87 Data.replace(pos: K.Pos + I * sizeof(uint64_t), n1: sizeof(uint64_t),
88 s: (const char *)&Bytes, n2: sizeof(uint64_t));
89 }
90 }
91 }
92 }
93
94 // If \c OS is an instance of \c raw_fd_ostream, this field will be
95 // true. Otherwise, \c OS will be an raw_string_ostream.
96 bool IsFDOStream;
97 raw_ostream &OS;
98 support::endian::Writer LE;
99};
100
101class InstrProfRecordWriterTrait {
102public:
103 using key_type = StringRef;
104 using key_type_ref = StringRef;
105
106 using data_type = const InstrProfWriter::ProfilingData *const;
107 using data_type_ref = const InstrProfWriter::ProfilingData *const;
108
109 using hash_value_type = uint64_t;
110 using offset_type = uint64_t;
111
112 llvm::endianness ValueProfDataEndianness = llvm::endianness::little;
113 InstrProfSummaryBuilder *SummaryBuilder;
114 InstrProfSummaryBuilder *CSSummaryBuilder;
115
116 InstrProfRecordWriterTrait() = default;
117
118 static hash_value_type ComputeHash(key_type_ref K) {
119 return IndexedInstrProf::ComputeHash(K);
120 }
121
122 static std::pair<offset_type, offset_type>
123 EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V) {
124 using namespace support;
125
126 endian::Writer LE(Out, llvm::endianness::little);
127
128 offset_type N = K.size();
129 LE.write<offset_type>(Val: N);
130
131 offset_type M = 0;
132 for (const auto &ProfileData : *V) {
133 const InstrProfRecord &ProfRecord = ProfileData.second;
134 M += sizeof(uint64_t); // The function hash
135 M += sizeof(uint64_t); // The size of the Counts vector
136 M += ProfRecord.Counts.size() * sizeof(uint64_t);
137 M += sizeof(uint64_t); // The size of the Bitmap vector
138 M += ProfRecord.BitmapBytes.size() * sizeof(uint64_t);
139
140 // Value data
141 M += ValueProfData::getSize(Record: ProfileData.second);
142 }
143 LE.write<offset_type>(Val: M);
144
145 return std::make_pair(x&: N, y&: M);
146 }
147
148 void EmitKey(raw_ostream &Out, key_type_ref K, offset_type N) {
149 Out.write(Ptr: K.data(), Size: N);
150 }
151
152 void EmitData(raw_ostream &Out, key_type_ref, data_type_ref V, offset_type) {
153 using namespace support;
154
155 endian::Writer LE(Out, llvm::endianness::little);
156 for (const auto &ProfileData : *V) {
157 const InstrProfRecord &ProfRecord = ProfileData.second;
158 if (NamedInstrProfRecord::hasCSFlagInHash(FuncHash: ProfileData.first))
159 CSSummaryBuilder->addRecord(ProfRecord);
160 else
161 SummaryBuilder->addRecord(ProfRecord);
162
163 LE.write<uint64_t>(Val: ProfileData.first); // Function hash
164 LE.write<uint64_t>(Val: ProfRecord.Counts.size());
165 for (uint64_t I : ProfRecord.Counts)
166 LE.write<uint64_t>(Val: I);
167
168 LE.write<uint64_t>(Val: ProfRecord.BitmapBytes.size());
169 for (uint64_t I : ProfRecord.BitmapBytes)
170 LE.write<uint64_t>(Val: I);
171
172 // Write value data
173 std::unique_ptr<ValueProfData> VDataPtr =
174 ValueProfData::serializeFrom(Record: ProfileData.second);
175 uint32_t S = VDataPtr->getSize();
176 VDataPtr->swapBytesFromHost(Endianness: ValueProfDataEndianness);
177 Out.write(Ptr: (const char *)VDataPtr.get(), Size: S);
178 }
179 }
180};
181
182} // end namespace llvm
183
184InstrProfWriter::InstrProfWriter(
185 bool Sparse, uint64_t TemporalProfTraceReservoirSize,
186 uint64_t MaxTemporalProfTraceLength, bool WritePrevVersion,
187 memprof::IndexedVersion MemProfVersionRequested)
188 : Sparse(Sparse), MaxTemporalProfTraceLength(MaxTemporalProfTraceLength),
189 TemporalProfTraceReservoirSize(TemporalProfTraceReservoirSize),
190 InfoObj(new InstrProfRecordWriterTrait()),
191 WritePrevVersion(WritePrevVersion),
192 MemProfVersionRequested(MemProfVersionRequested) {}
193
194InstrProfWriter::~InstrProfWriter() { delete InfoObj; }
195
196// Internal interface for testing purpose only.
197void InstrProfWriter::setValueProfDataEndianness(llvm::endianness Endianness) {
198 InfoObj->ValueProfDataEndianness = Endianness;
199}
200
201void InstrProfWriter::setOutputSparse(bool Sparse) {
202 this->Sparse = Sparse;
203}
204
205void InstrProfWriter::addRecord(NamedInstrProfRecord &&I, uint64_t Weight,
206 function_ref<void(Error)> Warn) {
207 auto Name = I.Name;
208 auto Hash = I.Hash;
209 addRecord(Name, Hash, I: std::move(I), Weight, Warn);
210}
211
212void InstrProfWriter::overlapRecord(NamedInstrProfRecord &&Other,
213 OverlapStats &Overlap,
214 OverlapStats &FuncLevelOverlap,
215 const OverlapFuncFilters &FuncFilter) {
216 auto Name = Other.Name;
217 auto Hash = Other.Hash;
218 Other.accumulateCounts(Sum&: FuncLevelOverlap.Test);
219 if (!FunctionData.contains(Key: Name)) {
220 Overlap.addOneUnique(UniqueFunc: FuncLevelOverlap.Test);
221 return;
222 }
223 if (FuncLevelOverlap.Test.CountSum < 1.0f) {
224 Overlap.Overlap.NumEntries += 1;
225 return;
226 }
227 auto &ProfileDataMap = FunctionData[Name];
228 bool NewFunc;
229 ProfilingData::iterator Where;
230 std::tie(args&: Where, args&: NewFunc) =
231 ProfileDataMap.insert(KV: std::make_pair(x&: Hash, y: InstrProfRecord()));
232 if (NewFunc) {
233 Overlap.addOneMismatch(MismatchFunc: FuncLevelOverlap.Test);
234 return;
235 }
236 InstrProfRecord &Dest = Where->second;
237
238 uint64_t ValueCutoff = FuncFilter.ValueCutoff;
239 if (!FuncFilter.NameFilter.empty() && Name.contains(Other: FuncFilter.NameFilter))
240 ValueCutoff = 0;
241
242 Dest.overlap(Other, Overlap, FuncLevelOverlap, ValueCutoff);
243}
244
245void InstrProfWriter::addRecord(StringRef Name, uint64_t Hash,
246 InstrProfRecord &&I, uint64_t Weight,
247 function_ref<void(Error)> Warn) {
248 auto &ProfileDataMap = FunctionData[Name];
249
250 bool NewFunc;
251 ProfilingData::iterator Where;
252 std::tie(args&: Where, args&: NewFunc) =
253 ProfileDataMap.insert(KV: std::make_pair(x&: Hash, y: InstrProfRecord()));
254 InstrProfRecord &Dest = Where->second;
255
256 auto MapWarn = [&](instrprof_error E) {
257 Warn(make_error<InstrProfError>(Args&: E));
258 };
259
260 if (NewFunc) {
261 // We've never seen a function with this name and hash, add it.
262 Dest = std::move(I);
263 if (Weight > 1)
264 Dest.scale(N: Weight, D: 1, Warn: MapWarn);
265 } else {
266 // We're updating a function we've seen before.
267 Dest.merge(Other&: I, Weight, Warn: MapWarn);
268 }
269
270 Dest.sortValueData();
271}
272
273void InstrProfWriter::addMemProfRecord(
274 const Function::GUID Id, const memprof::IndexedMemProfRecord &Record) {
275 auto [Iter, Inserted] = MemProfRecordData.insert(KV: {Id, Record});
276 // If we inserted a new record then we are done.
277 if (Inserted) {
278 return;
279 }
280 memprof::IndexedMemProfRecord &Existing = Iter->second;
281 Existing.merge(Other: Record);
282}
283
284bool InstrProfWriter::addMemProfFrame(const memprof::FrameId Id,
285 const memprof::Frame &Frame,
286 function_ref<void(Error)> Warn) {
287 auto [Iter, Inserted] = MemProfFrameData.insert(KV: {Id, Frame});
288 // If a mapping already exists for the current frame id and it does not
289 // match the new mapping provided then reset the existing contents and bail
290 // out. We don't support the merging of memprof data whose Frame -> Id
291 // mapping across profiles is inconsistent.
292 if (!Inserted && Iter->second != Frame) {
293 Warn(make_error<InstrProfError>(Args: instrprof_error::malformed,
294 Args: "frame to id mapping mismatch"));
295 return false;
296 }
297 return true;
298}
299
300bool InstrProfWriter::addMemProfCallStack(
301 const memprof::CallStackId CSId,
302 const llvm::SmallVector<memprof::FrameId> &CallStack,
303 function_ref<void(Error)> Warn) {
304 auto [Iter, Inserted] = MemProfCallStackData.insert(KV: {CSId, CallStack});
305 // If a mapping already exists for the current call stack id and it does not
306 // match the new mapping provided then reset the existing contents and bail
307 // out. We don't support the merging of memprof data whose CallStack -> Id
308 // mapping across profiles is inconsistent.
309 if (!Inserted && Iter->second != CallStack) {
310 Warn(make_error<InstrProfError>(Args: instrprof_error::malformed,
311 Args: "call stack to id mapping mismatch"));
312 return false;
313 }
314 return true;
315}
316
317void InstrProfWriter::addBinaryIds(ArrayRef<llvm::object::BuildID> BIs) {
318 llvm::append_range(C&: BinaryIds, R&: BIs);
319}
320
321void InstrProfWriter::addTemporalProfileTrace(TemporalProfTraceTy Trace) {
322 if (Trace.FunctionNameRefs.size() > MaxTemporalProfTraceLength)
323 Trace.FunctionNameRefs.resize(new_size: MaxTemporalProfTraceLength);
324 if (Trace.FunctionNameRefs.empty())
325 return;
326
327 if (TemporalProfTraceStreamSize < TemporalProfTraceReservoirSize) {
328 // Simply append the trace if we have not yet hit our reservoir size limit.
329 TemporalProfTraces.push_back(Elt: std::move(Trace));
330 } else {
331 // Otherwise, replace a random trace in the stream.
332 std::uniform_int_distribution<uint64_t> Distribution(
333 0, TemporalProfTraceStreamSize);
334 uint64_t RandomIndex = Distribution(RNG);
335 if (RandomIndex < TemporalProfTraces.size())
336 TemporalProfTraces[RandomIndex] = std::move(Trace);
337 }
338 ++TemporalProfTraceStreamSize;
339}
340
341void InstrProfWriter::addTemporalProfileTraces(
342 SmallVectorImpl<TemporalProfTraceTy> &SrcTraces, uint64_t SrcStreamSize) {
343 // Assume that the source has the same reservoir size as the destination to
344 // avoid needing to record it in the indexed profile format.
345 bool IsDestSampled =
346 (TemporalProfTraceStreamSize > TemporalProfTraceReservoirSize);
347 bool IsSrcSampled = (SrcStreamSize > TemporalProfTraceReservoirSize);
348 if (!IsDestSampled && IsSrcSampled) {
349 // If one of the traces are sampled, ensure that it belongs to Dest.
350 std::swap(LHS&: TemporalProfTraces, RHS&: SrcTraces);
351 std::swap(a&: TemporalProfTraceStreamSize, b&: SrcStreamSize);
352 std::swap(a&: IsDestSampled, b&: IsSrcSampled);
353 }
354 if (!IsSrcSampled) {
355 // If the source stream is not sampled, we add each source trace normally.
356 for (auto &Trace : SrcTraces)
357 addTemporalProfileTrace(Trace: std::move(Trace));
358 return;
359 }
360 // Otherwise, we find the traces that would have been removed if we added
361 // the whole source stream.
362 SmallSetVector<uint64_t, 8> IndicesToReplace;
363 for (uint64_t I = 0; I < SrcStreamSize; I++) {
364 std::uniform_int_distribution<uint64_t> Distribution(
365 0, TemporalProfTraceStreamSize);
366 uint64_t RandomIndex = Distribution(RNG);
367 if (RandomIndex < TemporalProfTraces.size())
368 IndicesToReplace.insert(X: RandomIndex);
369 ++TemporalProfTraceStreamSize;
370 }
371 // Then we insert a random sample of the source traces.
372 llvm::shuffle(first: SrcTraces.begin(), last: SrcTraces.end(), g&: RNG);
373 for (const auto &[Index, Trace] : llvm::zip(t&: IndicesToReplace, u&: SrcTraces))
374 TemporalProfTraces[Index] = std::move(Trace);
375}
376
377void InstrProfWriter::mergeRecordsFromWriter(InstrProfWriter &&IPW,
378 function_ref<void(Error)> Warn) {
379 for (auto &I : IPW.FunctionData)
380 for (auto &Func : I.getValue())
381 addRecord(Name: I.getKey(), Hash: Func.first, I: std::move(Func.second), Weight: 1, Warn);
382
383 BinaryIds.reserve(n: BinaryIds.size() + IPW.BinaryIds.size());
384 for (auto &I : IPW.BinaryIds)
385 addBinaryIds(BIs: I);
386
387 addTemporalProfileTraces(SrcTraces&: IPW.TemporalProfTraces,
388 SrcStreamSize: IPW.TemporalProfTraceStreamSize);
389
390 MemProfFrameData.reserve(NumEntries: IPW.MemProfFrameData.size());
391 for (auto &[FrameId, Frame] : IPW.MemProfFrameData) {
392 // If we weren't able to add the frame mappings then it doesn't make sense
393 // to try to merge the records from this profile.
394 if (!addMemProfFrame(Id: FrameId, Frame, Warn))
395 return;
396 }
397
398 MemProfCallStackData.reserve(NumEntries: IPW.MemProfCallStackData.size());
399 for (auto &[CSId, CallStack] : IPW.MemProfCallStackData) {
400 if (!addMemProfCallStack(CSId, CallStack, Warn))
401 return;
402 }
403
404 MemProfRecordData.reserve(NumEntries: IPW.MemProfRecordData.size());
405 for (auto &[GUID, Record] : IPW.MemProfRecordData) {
406 addMemProfRecord(Id: GUID, Record);
407 }
408}
409
410bool InstrProfWriter::shouldEncodeData(const ProfilingData &PD) {
411 if (!Sparse)
412 return true;
413 for (const auto &Func : PD) {
414 const InstrProfRecord &IPR = Func.second;
415 if (llvm::any_of(Range: IPR.Counts, P: [](uint64_t Count) { return Count > 0; }))
416 return true;
417 if (llvm::any_of(Range: IPR.BitmapBytes, P: [](uint8_t Byte) { return Byte > 0; }))
418 return true;
419 }
420 return false;
421}
422
423static void setSummary(IndexedInstrProf::Summary *TheSummary,
424 ProfileSummary &PS) {
425 using namespace IndexedInstrProf;
426
427 const std::vector<ProfileSummaryEntry> &Res = PS.getDetailedSummary();
428 TheSummary->NumSummaryFields = Summary::NumKinds;
429 TheSummary->NumCutoffEntries = Res.size();
430 TheSummary->set(K: Summary::MaxFunctionCount, V: PS.getMaxFunctionCount());
431 TheSummary->set(K: Summary::MaxBlockCount, V: PS.getMaxCount());
432 TheSummary->set(K: Summary::MaxInternalBlockCount, V: PS.getMaxInternalCount());
433 TheSummary->set(K: Summary::TotalBlockCount, V: PS.getTotalCount());
434 TheSummary->set(K: Summary::TotalNumBlocks, V: PS.getNumCounts());
435 TheSummary->set(K: Summary::TotalNumFunctions, V: PS.getNumFunctions());
436 for (unsigned I = 0; I < Res.size(); I++)
437 TheSummary->setEntry(I, E: Res[I]);
438}
439
440// Serialize Schema.
441static void writeMemProfSchema(ProfOStream &OS,
442 const memprof::MemProfSchema &Schema) {
443 OS.write(V: static_cast<uint64_t>(Schema.size()));
444 for (const auto Id : Schema)
445 OS.write(V: static_cast<uint64_t>(Id));
446}
447
448// Serialize MemProfRecordData. Return RecordTableOffset.
449static uint64_t writeMemProfRecords(
450 ProfOStream &OS,
451 llvm::MapVector<GlobalValue::GUID, memprof::IndexedMemProfRecord>
452 &MemProfRecordData,
453 memprof::MemProfSchema *Schema, memprof::IndexedVersion Version) {
454 memprof::RecordWriterTrait RecordWriter(Schema, Version);
455 OnDiskChainedHashTableGenerator<memprof::RecordWriterTrait>
456 RecordTableGenerator;
457 for (auto &[GUID, Record] : MemProfRecordData) {
458 // Insert the key (func hash) and value (memprof record).
459 RecordTableGenerator.insert(Key: GUID, Data&: Record, InfoObj&: RecordWriter);
460 }
461 // Release the memory of this MapVector as it is no longer needed.
462 MemProfRecordData.clear();
463
464 // The call to Emit invokes RecordWriterTrait::EmitData which destructs
465 // the memprof record copies owned by the RecordTableGenerator. This works
466 // because the RecordTableGenerator is not used after this point.
467 return RecordTableGenerator.Emit(Out&: OS.OS, InfoObj&: RecordWriter);
468}
469
470// Serialize MemProfFrameData. Return FrameTableOffset.
471static uint64_t writeMemProfFrames(
472 ProfOStream &OS,
473 llvm::MapVector<memprof::FrameId, memprof::Frame> &MemProfFrameData) {
474 OnDiskChainedHashTableGenerator<memprof::FrameWriterTrait>
475 FrameTableGenerator;
476 for (auto &[FrameId, Frame] : MemProfFrameData) {
477 // Insert the key (frame id) and value (frame contents).
478 FrameTableGenerator.insert(Key: FrameId, Data&: Frame);
479 }
480 // Release the memory of this MapVector as it is no longer needed.
481 MemProfFrameData.clear();
482
483 return FrameTableGenerator.Emit(Out&: OS.OS);
484}
485
486static uint64_t writeMemProfCallStacks(
487 ProfOStream &OS,
488 llvm::MapVector<memprof::CallStackId, llvm::SmallVector<memprof::FrameId>>
489 &MemProfCallStackData) {
490 OnDiskChainedHashTableGenerator<memprof::CallStackWriterTrait>
491 CallStackTableGenerator;
492 for (auto &[CSId, CallStack] : MemProfCallStackData)
493 CallStackTableGenerator.insert(Key: CSId, Data&: CallStack);
494 // Release the memory of this vector as it is no longer needed.
495 MemProfCallStackData.clear();
496
497 return CallStackTableGenerator.Emit(Out&: OS.OS);
498}
499
500static Error writeMemProfV0(
501 ProfOStream &OS,
502 llvm::MapVector<GlobalValue::GUID, memprof::IndexedMemProfRecord>
503 &MemProfRecordData,
504 llvm::MapVector<memprof::FrameId, memprof::Frame> &MemProfFrameData) {
505 uint64_t HeaderUpdatePos = OS.tell();
506 OS.write(V: 0ULL); // Reserve space for the memprof record table offset.
507 OS.write(V: 0ULL); // Reserve space for the memprof frame payload offset.
508 OS.write(V: 0ULL); // Reserve space for the memprof frame table offset.
509
510 auto Schema = memprof::PortableMemInfoBlock::getSchema();
511 writeMemProfSchema(OS, Schema);
512
513 uint64_t RecordTableOffset =
514 writeMemProfRecords(OS, MemProfRecordData, Schema: &Schema, Version: memprof::Version0);
515
516 uint64_t FramePayloadOffset = OS.tell();
517 uint64_t FrameTableOffset = writeMemProfFrames(OS, MemProfFrameData);
518
519 uint64_t Header[] = {RecordTableOffset, FramePayloadOffset, FrameTableOffset};
520 OS.patch(P: {{.Pos: HeaderUpdatePos, .D: Header, .N: std::size(Header)}});
521
522 return Error::success();
523}
524
525static Error writeMemProfV1(
526 ProfOStream &OS,
527 llvm::MapVector<GlobalValue::GUID, memprof::IndexedMemProfRecord>
528 &MemProfRecordData,
529 llvm::MapVector<memprof::FrameId, memprof::Frame> &MemProfFrameData) {
530 OS.write(V: memprof::Version1);
531 uint64_t HeaderUpdatePos = OS.tell();
532 OS.write(V: 0ULL); // Reserve space for the memprof record table offset.
533 OS.write(V: 0ULL); // Reserve space for the memprof frame payload offset.
534 OS.write(V: 0ULL); // Reserve space for the memprof frame table offset.
535
536 auto Schema = memprof::PortableMemInfoBlock::getSchema();
537 writeMemProfSchema(OS, Schema);
538
539 uint64_t RecordTableOffset =
540 writeMemProfRecords(OS, MemProfRecordData, Schema: &Schema, Version: memprof::Version1);
541
542 uint64_t FramePayloadOffset = OS.tell();
543 uint64_t FrameTableOffset = writeMemProfFrames(OS, MemProfFrameData);
544
545 uint64_t Header[] = {RecordTableOffset, FramePayloadOffset, FrameTableOffset};
546 OS.patch(P: {{.Pos: HeaderUpdatePos, .D: Header, .N: std::size(Header)}});
547
548 return Error::success();
549}
550
551static Error writeMemProfV2(
552 ProfOStream &OS,
553 llvm::MapVector<GlobalValue::GUID, memprof::IndexedMemProfRecord>
554 &MemProfRecordData,
555 llvm::MapVector<memprof::FrameId, memprof::Frame> &MemProfFrameData,
556 llvm::MapVector<memprof::CallStackId, llvm::SmallVector<memprof::FrameId>>
557 &MemProfCallStackData) {
558 OS.write(V: memprof::Version2);
559 uint64_t HeaderUpdatePos = OS.tell();
560 OS.write(V: 0ULL); // Reserve space for the memprof record table offset.
561 OS.write(V: 0ULL); // Reserve space for the memprof frame payload offset.
562 OS.write(V: 0ULL); // Reserve space for the memprof frame table offset.
563 OS.write(V: 0ULL); // Reserve space for the memprof call stack payload offset.
564 OS.write(V: 0ULL); // Reserve space for the memprof call stack table offset.
565
566 auto Schema = memprof::PortableMemInfoBlock::getSchema();
567 writeMemProfSchema(OS, Schema);
568
569 uint64_t RecordTableOffset =
570 writeMemProfRecords(OS, MemProfRecordData, Schema: &Schema, Version: memprof::Version2);
571
572 uint64_t FramePayloadOffset = OS.tell();
573 uint64_t FrameTableOffset = writeMemProfFrames(OS, MemProfFrameData);
574
575 uint64_t CallStackPayloadOffset = OS.tell();
576 uint64_t CallStackTableOffset =
577 writeMemProfCallStacks(OS, MemProfCallStackData);
578
579 uint64_t Header[] = {
580 RecordTableOffset, FramePayloadOffset, FrameTableOffset,
581 CallStackPayloadOffset, CallStackTableOffset,
582 };
583 OS.patch(P: {{.Pos: HeaderUpdatePos, .D: Header, .N: std::size(Header)}});
584
585 return Error::success();
586}
587
588// The MemProf profile data includes a simple schema
589// with the format described below followed by the hashtable:
590// uint64_t Version
591// uint64_t RecordTableOffset = RecordTableGenerator.Emit
592// uint64_t FramePayloadOffset = Stream offset before emitting the frame table
593// uint64_t FrameTableOffset = FrameTableGenerator.Emit
594// uint64_t Num schema entries
595// uint64_t Schema entry 0
596// uint64_t Schema entry 1
597// ....
598// uint64_t Schema entry N - 1
599// OnDiskChainedHashTable MemProfRecordData
600// OnDiskChainedHashTable MemProfFrameData
601static Error writeMemProf(
602 ProfOStream &OS,
603 llvm::MapVector<GlobalValue::GUID, memprof::IndexedMemProfRecord>
604 &MemProfRecordData,
605 llvm::MapVector<memprof::FrameId, memprof::Frame> &MemProfFrameData,
606 llvm::MapVector<memprof::CallStackId, llvm::SmallVector<memprof::FrameId>>
607 &MemProfCallStackData,
608 memprof::IndexedVersion MemProfVersionRequested) {
609
610 switch (MemProfVersionRequested) {
611 case memprof::Version0:
612 return writeMemProfV0(OS, MemProfRecordData, MemProfFrameData);
613 case memprof::Version1:
614 return writeMemProfV1(OS, MemProfRecordData, MemProfFrameData);
615 case memprof::Version2:
616 return writeMemProfV2(OS, MemProfRecordData, MemProfFrameData,
617 MemProfCallStackData);
618 }
619
620 return make_error<InstrProfError>(
621 Args: instrprof_error::unsupported_version,
622 Args: formatv(Fmt: "MemProf version {} not supported; "
623 "requires version between {} and {}, inclusive",
624 Vals&: MemProfVersionRequested, Vals: memprof::MinimumSupportedVersion,
625 Vals: memprof::MaximumSupportedVersion));
626}
627
628Error InstrProfWriter::writeImpl(ProfOStream &OS) {
629 using namespace IndexedInstrProf;
630 using namespace support;
631
632 OnDiskChainedHashTableGenerator<InstrProfRecordWriterTrait> Generator;
633
634 InstrProfSummaryBuilder ISB(ProfileSummaryBuilder::DefaultCutoffs);
635 InfoObj->SummaryBuilder = &ISB;
636 InstrProfSummaryBuilder CSISB(ProfileSummaryBuilder::DefaultCutoffs);
637 InfoObj->CSSummaryBuilder = &CSISB;
638
639 // Populate the hash table generator.
640 SmallVector<std::pair<StringRef, const ProfilingData *>, 0> OrderedData;
641 for (const auto &I : FunctionData)
642 if (shouldEncodeData(PD: I.getValue()))
643 OrderedData.emplace_back(Args: (I.getKey()), Args: &I.getValue());
644 llvm::sort(C&: OrderedData, Comp: less_first());
645 for (const auto &I : OrderedData)
646 Generator.insert(Key: I.first, Data: I.second);
647
648 // Write the header.
649 IndexedInstrProf::Header Header;
650 Header.Magic = IndexedInstrProf::Magic;
651 Header.Version = WritePrevVersion
652 ? IndexedInstrProf::ProfVersion::Version11
653 : IndexedInstrProf::ProfVersion::CurrentVersion;
654 // The WritePrevVersion handling will either need to be removed or updated
655 // if the version is advanced beyond 12.
656 assert(IndexedInstrProf::ProfVersion::CurrentVersion ==
657 IndexedInstrProf::ProfVersion::Version12);
658 if (static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation))
659 Header.Version |= VARIANT_MASK_IR_PROF;
660 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive))
661 Header.Version |= VARIANT_MASK_CSIR_PROF;
662 if (static_cast<bool>(ProfileKind &
663 InstrProfKind::FunctionEntryInstrumentation))
664 Header.Version |= VARIANT_MASK_INSTR_ENTRY;
665 if (static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage))
666 Header.Version |= VARIANT_MASK_BYTE_COVERAGE;
667 if (static_cast<bool>(ProfileKind & InstrProfKind::FunctionEntryOnly))
668 Header.Version |= VARIANT_MASK_FUNCTION_ENTRY_ONLY;
669 if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf))
670 Header.Version |= VARIANT_MASK_MEMPROF;
671 if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile))
672 Header.Version |= VARIANT_MASK_TEMPORAL_PROF;
673
674 Header.Unused = 0;
675 Header.HashType = static_cast<uint64_t>(IndexedInstrProf::HashType);
676 Header.HashOffset = 0;
677 Header.MemProfOffset = 0;
678 Header.BinaryIdOffset = 0;
679 Header.TemporalProfTracesOffset = 0;
680 Header.VTableNamesOffset = 0;
681
682 // Only write out the first four fields. We need to remember the offset of the
683 // remaining fields to allow back patching later.
684 for (int I = 0; I < 4; I++)
685 OS.write(V: reinterpret_cast<uint64_t *>(&Header)[I]);
686
687 // Save the location of Header.HashOffset field in \c OS.
688 uint64_t HashTableStartFieldOffset = OS.tell();
689 // Reserve the space for HashOffset field.
690 OS.write(V: 0);
691
692 // Save the location of MemProf profile data. This is stored in two parts as
693 // the schema and as a separate on-disk chained hashtable.
694 uint64_t MemProfSectionOffset = OS.tell();
695 // Reserve space for the MemProf table field to be patched later if this
696 // profile contains memory profile information.
697 OS.write(V: 0);
698
699 // Save the location of binary ids section.
700 uint64_t BinaryIdSectionOffset = OS.tell();
701 // Reserve space for the BinaryIdOffset field to be patched later if this
702 // profile contains binary ids.
703 OS.write(V: 0);
704
705 uint64_t TemporalProfTracesOffset = OS.tell();
706 OS.write(V: 0);
707
708 uint64_t VTableNamesOffset = OS.tell();
709 if (!WritePrevVersion)
710 OS.write(V: 0);
711
712 // Reserve space to write profile summary data.
713 uint32_t NumEntries = ProfileSummaryBuilder::DefaultCutoffs.size();
714 uint32_t SummarySize = Summary::getSize(NumSumFields: Summary::NumKinds, NumCutoffEntries: NumEntries);
715 // Remember the summary offset.
716 uint64_t SummaryOffset = OS.tell();
717 for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++)
718 OS.write(V: 0);
719 uint64_t CSSummaryOffset = 0;
720 uint64_t CSSummarySize = 0;
721 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) {
722 CSSummaryOffset = OS.tell();
723 CSSummarySize = SummarySize / sizeof(uint64_t);
724 for (unsigned I = 0; I < CSSummarySize; I++)
725 OS.write(V: 0);
726 }
727
728 // Write the hash table.
729 uint64_t HashTableStart = Generator.Emit(Out&: OS.OS, InfoObj&: *InfoObj);
730
731 // Write the MemProf profile data if we have it.
732 uint64_t MemProfSectionStart = 0;
733 if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf)) {
734 MemProfSectionStart = OS.tell();
735 if (auto E = writeMemProf(OS, MemProfRecordData, MemProfFrameData,
736 MemProfCallStackData, MemProfVersionRequested))
737 return E;
738 }
739
740 // BinaryIdSection has two parts:
741 // 1. uint64_t BinaryIdsSectionSize
742 // 2. list of binary ids that consist of:
743 // a. uint64_t BinaryIdLength
744 // b. uint8_t BinaryIdData
745 // c. uint8_t Padding (if necessary)
746 uint64_t BinaryIdSectionStart = OS.tell();
747 // Calculate size of binary section.
748 uint64_t BinaryIdsSectionSize = 0;
749
750 // Remove duplicate binary ids.
751 llvm::sort(C&: BinaryIds);
752 BinaryIds.erase(first: std::unique(first: BinaryIds.begin(), last: BinaryIds.end()),
753 last: BinaryIds.end());
754
755 for (auto BI : BinaryIds) {
756 // Increment by binary id length data type size.
757 BinaryIdsSectionSize += sizeof(uint64_t);
758 // Increment by binary id data length, aligned to 8 bytes.
759 BinaryIdsSectionSize += alignToPowerOf2(Value: BI.size(), Align: sizeof(uint64_t));
760 }
761 // Write binary ids section size.
762 OS.write(V: BinaryIdsSectionSize);
763
764 for (auto BI : BinaryIds) {
765 uint64_t BILen = BI.size();
766 // Write binary id length.
767 OS.write(V: BILen);
768 // Write binary id data.
769 for (unsigned K = 0; K < BILen; K++)
770 OS.writeByte(V: BI[K]);
771 // Write padding if necessary.
772 uint64_t PaddingSize = alignToPowerOf2(Value: BILen, Align: sizeof(uint64_t)) - BILen;
773 for (unsigned K = 0; K < PaddingSize; K++)
774 OS.writeByte(V: 0);
775 }
776
777 uint64_t VTableNamesSectionStart = OS.tell();
778
779 if (!WritePrevVersion) {
780 std::vector<std::string> VTableNameStrs;
781 for (StringRef VTableName : VTableNames.keys())
782 VTableNameStrs.push_back(x: VTableName.str());
783
784 std::string CompressedVTableNames;
785 if (!VTableNameStrs.empty())
786 if (Error E = collectGlobalObjectNameStrings(
787 NameStrs: VTableNameStrs, doCompression: compression::zlib::isAvailable(),
788 Result&: CompressedVTableNames))
789 return E;
790
791 const uint64_t CompressedStringLen = CompressedVTableNames.length();
792
793 // Record the length of compressed string.
794 OS.write(V: CompressedStringLen);
795
796 // Write the chars in compressed strings.
797 for (auto &c : CompressedVTableNames)
798 OS.writeByte(V: static_cast<uint8_t>(c));
799
800 // Pad up to a multiple of 8.
801 // InstrProfReader could read bytes according to 'CompressedStringLen'.
802 const uint64_t PaddedLength = alignTo(Value: CompressedStringLen, Align: 8);
803
804 for (uint64_t K = CompressedStringLen; K < PaddedLength; K++)
805 OS.writeByte(V: 0);
806 }
807
808 uint64_t TemporalProfTracesSectionStart = 0;
809 if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile)) {
810 TemporalProfTracesSectionStart = OS.tell();
811 OS.write(V: TemporalProfTraces.size());
812 OS.write(V: TemporalProfTraceStreamSize);
813 for (auto &Trace : TemporalProfTraces) {
814 OS.write(V: Trace.Weight);
815 OS.write(V: Trace.FunctionNameRefs.size());
816 for (auto &NameRef : Trace.FunctionNameRefs)
817 OS.write(V: NameRef);
818 }
819 }
820
821 // Allocate space for data to be serialized out.
822 std::unique_ptr<IndexedInstrProf::Summary> TheSummary =
823 IndexedInstrProf::allocSummary(TotalSize: SummarySize);
824 // Compute the Summary and copy the data to the data
825 // structure to be serialized out (to disk or buffer).
826 std::unique_ptr<ProfileSummary> PS = ISB.getSummary();
827 setSummary(TheSummary: TheSummary.get(), PS&: *PS);
828 InfoObj->SummaryBuilder = nullptr;
829
830 // For Context Sensitive summary.
831 std::unique_ptr<IndexedInstrProf::Summary> TheCSSummary = nullptr;
832 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) {
833 TheCSSummary = IndexedInstrProf::allocSummary(TotalSize: SummarySize);
834 std::unique_ptr<ProfileSummary> CSPS = CSISB.getSummary();
835 setSummary(TheSummary: TheCSSummary.get(), PS&: *CSPS);
836 }
837 InfoObj->CSSummaryBuilder = nullptr;
838
839 if (!WritePrevVersion) {
840 // Now do the final patch:
841 PatchItem PatchItems[] = {
842 // Patch the Header.HashOffset field.
843 {.Pos: HashTableStartFieldOffset, .D: &HashTableStart, .N: 1},
844 // Patch the Header.MemProfOffset (=0 for profiles without MemProf
845 // data).
846 {.Pos: MemProfSectionOffset, .D: &MemProfSectionStart, .N: 1},
847 // Patch the Header.BinaryIdSectionOffset.
848 {.Pos: BinaryIdSectionOffset, .D: &BinaryIdSectionStart, .N: 1},
849 // Patch the Header.TemporalProfTracesOffset (=0 for profiles without
850 // traces).
851 {.Pos: TemporalProfTracesOffset, .D: &TemporalProfTracesSectionStart, .N: 1},
852 {.Pos: VTableNamesOffset, .D: &VTableNamesSectionStart, .N: 1},
853 // Patch the summary data.
854 {.Pos: SummaryOffset, .D: reinterpret_cast<uint64_t *>(TheSummary.get()),
855 .N: (int)(SummarySize / sizeof(uint64_t))},
856 {.Pos: CSSummaryOffset, .D: reinterpret_cast<uint64_t *>(TheCSSummary.get()),
857 .N: (int)CSSummarySize}};
858
859 OS.patch(P: PatchItems);
860 } else {
861 // Now do the final patch:
862 PatchItem PatchItems[] = {
863 // Patch the Header.HashOffset field.
864 {.Pos: HashTableStartFieldOffset, .D: &HashTableStart, .N: 1},
865 // Patch the Header.MemProfOffset (=0 for profiles without MemProf
866 // data).
867 {.Pos: MemProfSectionOffset, .D: &MemProfSectionStart, .N: 1},
868 // Patch the Header.BinaryIdSectionOffset.
869 {.Pos: BinaryIdSectionOffset, .D: &BinaryIdSectionStart, .N: 1},
870 // Patch the Header.TemporalProfTracesOffset (=0 for profiles without
871 // traces).
872 {.Pos: TemporalProfTracesOffset, .D: &TemporalProfTracesSectionStart, .N: 1},
873 // Patch the summary data.
874 {.Pos: SummaryOffset, .D: reinterpret_cast<uint64_t *>(TheSummary.get()),
875 .N: (int)(SummarySize / sizeof(uint64_t))},
876 {.Pos: CSSummaryOffset, .D: reinterpret_cast<uint64_t *>(TheCSSummary.get()),
877 .N: (int)CSSummarySize}};
878
879 OS.patch(P: PatchItems);
880 }
881
882 for (const auto &I : FunctionData)
883 for (const auto &F : I.getValue())
884 if (Error E = validateRecord(Func: F.second))
885 return E;
886
887 return Error::success();
888}
889
890Error InstrProfWriter::write(raw_fd_ostream &OS) {
891 // Write the hash table.
892 ProfOStream POS(OS);
893 return writeImpl(OS&: POS);
894}
895
896Error InstrProfWriter::write(raw_string_ostream &OS) {
897 ProfOStream POS(OS);
898 return writeImpl(OS&: POS);
899}
900
901std::unique_ptr<MemoryBuffer> InstrProfWriter::writeBuffer() {
902 std::string Data;
903 raw_string_ostream OS(Data);
904 // Write the hash table.
905 if (Error E = write(OS))
906 return nullptr;
907 // Return this in an aligned memory buffer.
908 return MemoryBuffer::getMemBufferCopy(InputData: Data);
909}
910
911static const char *ValueProfKindStr[] = {
912#define VALUE_PROF_KIND(Enumerator, Value, Descr) #Enumerator,
913#include "llvm/ProfileData/InstrProfData.inc"
914};
915
916Error InstrProfWriter::validateRecord(const InstrProfRecord &Func) {
917 for (uint32_t VK = 0; VK <= IPVK_Last; VK++) {
918 uint32_t NS = Func.getNumValueSites(ValueKind: VK);
919 if (!NS)
920 continue;
921 for (uint32_t S = 0; S < NS; S++) {
922 uint32_t ND = Func.getNumValueDataForSite(ValueKind: VK, Site: S);
923 std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(ValueKind: VK, Site: S);
924 DenseSet<uint64_t> SeenValues;
925 for (uint32_t I = 0; I < ND; I++)
926 if ((VK != IPVK_IndirectCallTarget && VK != IPVK_VTableTarget) &&
927 !SeenValues.insert(V: VD[I].Value).second)
928 return make_error<InstrProfError>(Args: instrprof_error::invalid_prof);
929 }
930 }
931
932 return Error::success();
933}
934
935void InstrProfWriter::writeRecordInText(StringRef Name, uint64_t Hash,
936 const InstrProfRecord &Func,
937 InstrProfSymtab &Symtab,
938 raw_fd_ostream &OS) {
939 OS << Name << "\n";
940 OS << "# Func Hash:\n" << Hash << "\n";
941 OS << "# Num Counters:\n" << Func.Counts.size() << "\n";
942 OS << "# Counter Values:\n";
943 for (uint64_t Count : Func.Counts)
944 OS << Count << "\n";
945
946 if (Func.BitmapBytes.size() > 0) {
947 OS << "# Num Bitmap Bytes:\n$" << Func.BitmapBytes.size() << "\n";
948 OS << "# Bitmap Byte Values:\n";
949 for (uint8_t Byte : Func.BitmapBytes) {
950 OS << "0x";
951 OS.write_hex(N: Byte);
952 OS << "\n";
953 }
954 OS << "\n";
955 }
956
957 uint32_t NumValueKinds = Func.getNumValueKinds();
958 if (!NumValueKinds) {
959 OS << "\n";
960 return;
961 }
962
963 OS << "# Num Value Kinds:\n" << Func.getNumValueKinds() << "\n";
964 for (uint32_t VK = 0; VK < IPVK_Last + 1; VK++) {
965 uint32_t NS = Func.getNumValueSites(ValueKind: VK);
966 if (!NS)
967 continue;
968 OS << "# ValueKind = " << ValueProfKindStr[VK] << ":\n" << VK << "\n";
969 OS << "# NumValueSites:\n" << NS << "\n";
970 for (uint32_t S = 0; S < NS; S++) {
971 uint32_t ND = Func.getNumValueDataForSite(ValueKind: VK, Site: S);
972 OS << ND << "\n";
973 std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(ValueKind: VK, Site: S);
974 for (uint32_t I = 0; I < ND; I++) {
975 if (VK == IPVK_IndirectCallTarget || VK == IPVK_VTableTarget)
976 OS << Symtab.getFuncOrVarNameIfDefined(MD5Hash: VD[I].Value) << ":"
977 << VD[I].Count << "\n";
978 else
979 OS << VD[I].Value << ":" << VD[I].Count << "\n";
980 }
981 }
982 }
983
984 OS << "\n";
985}
986
987Error InstrProfWriter::writeText(raw_fd_ostream &OS) {
988 // Check CS first since it implies an IR level profile.
989 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive))
990 OS << "# CSIR level Instrumentation Flag\n:csir\n";
991 else if (static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation))
992 OS << "# IR level Instrumentation Flag\n:ir\n";
993
994 if (static_cast<bool>(ProfileKind &
995 InstrProfKind::FunctionEntryInstrumentation))
996 OS << "# Always instrument the function entry block\n:entry_first\n";
997 if (static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage))
998 OS << "# Instrument block coverage\n:single_byte_coverage\n";
999 InstrProfSymtab Symtab;
1000
1001 using FuncPair = detail::DenseMapPair<uint64_t, InstrProfRecord>;
1002 using RecordType = std::pair<StringRef, FuncPair>;
1003 SmallVector<RecordType, 4> OrderedFuncData;
1004
1005 for (const auto &I : FunctionData) {
1006 if (shouldEncodeData(PD: I.getValue())) {
1007 if (Error E = Symtab.addFuncName(FuncName: I.getKey()))
1008 return E;
1009 for (const auto &Func : I.getValue())
1010 OrderedFuncData.push_back(Elt: std::make_pair(x: I.getKey(), y: Func));
1011 }
1012 }
1013
1014 for (const auto &VTableName : VTableNames)
1015 if (Error E = Symtab.addVTableName(VTableName: VTableName.getKey()))
1016 return E;
1017
1018 if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile))
1019 writeTextTemporalProfTraceData(OS, Symtab);
1020
1021 llvm::sort(C&: OrderedFuncData, Comp: [](const RecordType &A, const RecordType &B) {
1022 return std::tie(args: A.first, args: A.second.first) <
1023 std::tie(args: B.first, args: B.second.first);
1024 });
1025
1026 for (const auto &record : OrderedFuncData) {
1027 const StringRef &Name = record.first;
1028 const FuncPair &Func = record.second;
1029 writeRecordInText(Name, Hash: Func.first, Func: Func.second, Symtab, OS);
1030 }
1031
1032 for (const auto &record : OrderedFuncData) {
1033 const FuncPair &Func = record.second;
1034 if (Error E = validateRecord(Func: Func.second))
1035 return E;
1036 }
1037
1038 return Error::success();
1039}
1040
1041void InstrProfWriter::writeTextTemporalProfTraceData(raw_fd_ostream &OS,
1042 InstrProfSymtab &Symtab) {
1043 OS << ":temporal_prof_traces\n";
1044 OS << "# Num Temporal Profile Traces:\n" << TemporalProfTraces.size() << "\n";
1045 OS << "# Temporal Profile Trace Stream Size:\n"
1046 << TemporalProfTraceStreamSize << "\n";
1047 for (auto &Trace : TemporalProfTraces) {
1048 OS << "# Weight:\n" << Trace.Weight << "\n";
1049 for (auto &NameRef : Trace.FunctionNameRefs)
1050 OS << Symtab.getFuncOrVarName(MD5Hash: NameRef) << ",";
1051 OS << "\n";
1052 }
1053 OS << "\n";
1054}
1055

source code of llvm/lib/ProfileData/InstrProfWriter.cpp