1//===- SampleProfReader.cpp - Read LLVM sample profile data ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the class that reads LLVM sample profiles. It
10// supports three file formats: text, binary and gcov.
11//
12// The textual representation is useful for debugging and testing purposes. The
13// binary representation is more compact, resulting in smaller file sizes.
14//
15// The gcov encoding is the one generated by GCC's AutoFDO profile creation
16// tool (https://github.com/google/autofdo)
17//
18// All three encodings can be used interchangeably as an input sample profile.
19//
20//===----------------------------------------------------------------------===//
21
22#include "llvm/ProfileData/SampleProfReader.h"
23#include "llvm/ADT/DenseMap.h"
24#include "llvm/ADT/STLExtras.h"
25#include "llvm/ADT/StringRef.h"
26#include "llvm/IR/Module.h"
27#include "llvm/IR/ProfileSummary.h"
28#include "llvm/ProfileData/ProfileCommon.h"
29#include "llvm/ProfileData/SampleProf.h"
30#include "llvm/Support/CommandLine.h"
31#include "llvm/Support/Compression.h"
32#include "llvm/Support/ErrorOr.h"
33#include "llvm/Support/JSON.h"
34#include "llvm/Support/LEB128.h"
35#include "llvm/Support/LineIterator.h"
36#include "llvm/Support/MD5.h"
37#include "llvm/Support/MemoryBuffer.h"
38#include "llvm/Support/VirtualFileSystem.h"
39#include "llvm/Support/raw_ostream.h"
40#include <algorithm>
41#include <cstddef>
42#include <cstdint>
43#include <limits>
44#include <memory>
45#include <system_error>
46#include <vector>
47
48using namespace llvm;
49using namespace sampleprof;
50
51#define DEBUG_TYPE "samplepgo-reader"
52
53// This internal option specifies if the profile uses FS discriminators.
54// It only applies to text, and binary format profiles.
55// For ext-binary format profiles, the flag is set in the summary.
56static cl::opt<bool> ProfileIsFSDisciminator(
57 "profile-isfs", cl::Hidden, cl::init(Val: false),
58 cl::desc("Profile uses flow sensitive discriminators"));
59
60/// Dump the function profile for \p FName.
61///
62/// \param FContext Name + context of the function to print.
63/// \param OS Stream to emit the output to.
64void SampleProfileReader::dumpFunctionProfile(const FunctionSamples &FS,
65 raw_ostream &OS) {
66 OS << "Function: " << FS.getContext().toString() << ": " << FS;
67}
68
69/// Dump all the function profiles found on stream \p OS.
70void SampleProfileReader::dump(raw_ostream &OS) {
71 std::vector<NameFunctionSamples> V;
72 sortFuncProfiles(ProfileMap: Profiles, SortedProfiles&: V);
73 for (const auto &I : V)
74 dumpFunctionProfile(FS: *I.second, OS);
75}
76
77static void dumpFunctionProfileJson(const FunctionSamples &S,
78 json::OStream &JOS, bool TopLevel = false) {
79 auto DumpBody = [&](const BodySampleMap &BodySamples) {
80 for (const auto &I : BodySamples) {
81 const LineLocation &Loc = I.first;
82 const SampleRecord &Sample = I.second;
83 JOS.object(Contents: [&] {
84 JOS.attribute(Key: "line", Contents: Loc.LineOffset);
85 if (Loc.Discriminator)
86 JOS.attribute(Key: "discriminator", Contents: Loc.Discriminator);
87 JOS.attribute(Key: "samples", Contents: Sample.getSamples());
88
89 auto CallTargets = Sample.getSortedCallTargets();
90 if (!CallTargets.empty()) {
91 JOS.attributeArray(Key: "calls", Contents: [&] {
92 for (const auto &J : CallTargets) {
93 JOS.object(Contents: [&] {
94 JOS.attribute(Key: "function", Contents: J.first.str());
95 JOS.attribute(Key: "samples", Contents: J.second);
96 });
97 }
98 });
99 }
100 });
101 }
102 };
103
104 auto DumpCallsiteSamples = [&](const CallsiteSampleMap &CallsiteSamples) {
105 for (const auto &I : CallsiteSamples)
106 for (const auto &FS : I.second) {
107 const LineLocation &Loc = I.first;
108 const FunctionSamples &CalleeSamples = FS.second;
109 JOS.object(Contents: [&] {
110 JOS.attribute(Key: "line", Contents: Loc.LineOffset);
111 if (Loc.Discriminator)
112 JOS.attribute(Key: "discriminator", Contents: Loc.Discriminator);
113 JOS.attributeArray(
114 Key: "samples", Contents: [&] { dumpFunctionProfileJson(S: CalleeSamples, JOS); });
115 });
116 }
117 };
118
119 JOS.object(Contents: [&] {
120 JOS.attribute(Key: "name", Contents: S.getFunction().str());
121 JOS.attribute(Key: "total", Contents: S.getTotalSamples());
122 if (TopLevel)
123 JOS.attribute(Key: "head", Contents: S.getHeadSamples());
124
125 const auto &BodySamples = S.getBodySamples();
126 if (!BodySamples.empty())
127 JOS.attributeArray(Key: "body", Contents: [&] { DumpBody(BodySamples); });
128
129 const auto &CallsiteSamples = S.getCallsiteSamples();
130 if (!CallsiteSamples.empty())
131 JOS.attributeArray(Key: "callsites",
132 Contents: [&] { DumpCallsiteSamples(CallsiteSamples); });
133 });
134}
135
136/// Dump all the function profiles found on stream \p OS in the JSON format.
137void SampleProfileReader::dumpJson(raw_ostream &OS) {
138 std::vector<NameFunctionSamples> V;
139 sortFuncProfiles(ProfileMap: Profiles, SortedProfiles&: V);
140 json::OStream JOS(OS, 2);
141 JOS.arrayBegin();
142 for (const auto &F : V)
143 dumpFunctionProfileJson(S: *F.second, JOS, TopLevel: true);
144 JOS.arrayEnd();
145
146 // Emit a newline character at the end as json::OStream doesn't emit one.
147 OS << "\n";
148}
149
150/// Parse \p Input as function head.
151///
152/// Parse one line of \p Input, and update function name in \p FName,
153/// function's total sample count in \p NumSamples, function's entry
154/// count in \p NumHeadSamples.
155///
156/// \returns true if parsing is successful.
157static bool ParseHead(const StringRef &Input, StringRef &FName,
158 uint64_t &NumSamples, uint64_t &NumHeadSamples) {
159 if (Input[0] == ' ')
160 return false;
161 size_t n2 = Input.rfind(C: ':');
162 size_t n1 = Input.rfind(C: ':', From: n2 - 1);
163 FName = Input.substr(Start: 0, N: n1);
164 if (Input.substr(Start: n1 + 1, N: n2 - n1 - 1).getAsInteger(Radix: 10, Result&: NumSamples))
165 return false;
166 if (Input.substr(Start: n2 + 1).getAsInteger(Radix: 10, Result&: NumHeadSamples))
167 return false;
168 return true;
169}
170
171/// Returns true if line offset \p L is legal (only has 16 bits).
172static bool isOffsetLegal(unsigned L) { return (L & 0xffff) == L; }
173
174/// Parse \p Input that contains metadata.
175/// Possible metadata:
176/// - CFG Checksum information:
177/// !CFGChecksum: 12345
178/// - CFG Checksum information:
179/// !Attributes: 1
180/// Stores the FunctionHash (a.k.a. CFG Checksum) into \p FunctionHash.
181static bool parseMetadata(const StringRef &Input, uint64_t &FunctionHash,
182 uint32_t &Attributes) {
183 if (Input.starts_with(Prefix: "!CFGChecksum:")) {
184 StringRef CFGInfo = Input.substr(Start: strlen(s: "!CFGChecksum:")).trim();
185 return !CFGInfo.getAsInteger(Radix: 10, Result&: FunctionHash);
186 }
187
188 if (Input.starts_with(Prefix: "!Attributes:")) {
189 StringRef Attrib = Input.substr(Start: strlen(s: "!Attributes:")).trim();
190 return !Attrib.getAsInteger(Radix: 10, Result&: Attributes);
191 }
192
193 return false;
194}
195
196enum class LineType {
197 CallSiteProfile,
198 BodyProfile,
199 Metadata,
200};
201
202/// Parse \p Input as line sample.
203///
204/// \param Input input line.
205/// \param LineTy Type of this line.
206/// \param Depth the depth of the inline stack.
207/// \param NumSamples total samples of the line/inlined callsite.
208/// \param LineOffset line offset to the start of the function.
209/// \param Discriminator discriminator of the line.
210/// \param TargetCountMap map from indirect call target to count.
211/// \param FunctionHash the function's CFG hash, used by pseudo probe.
212///
213/// returns true if parsing is successful.
214static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth,
215 uint64_t &NumSamples, uint32_t &LineOffset,
216 uint32_t &Discriminator, StringRef &CalleeName,
217 DenseMap<StringRef, uint64_t> &TargetCountMap,
218 uint64_t &FunctionHash, uint32_t &Attributes) {
219 for (Depth = 0; Input[Depth] == ' '; Depth++)
220 ;
221 if (Depth == 0)
222 return false;
223
224 if (Input[Depth] == '!') {
225 LineTy = LineType::Metadata;
226 return parseMetadata(Input: Input.substr(Start: Depth), FunctionHash, Attributes);
227 }
228
229 size_t n1 = Input.find(C: ':');
230 StringRef Loc = Input.substr(Start: Depth, N: n1 - Depth);
231 size_t n2 = Loc.find(C: '.');
232 if (n2 == StringRef::npos) {
233 if (Loc.getAsInteger(Radix: 10, Result&: LineOffset) || !isOffsetLegal(L: LineOffset))
234 return false;
235 Discriminator = 0;
236 } else {
237 if (Loc.substr(Start: 0, N: n2).getAsInteger(Radix: 10, Result&: LineOffset))
238 return false;
239 if (Loc.substr(Start: n2 + 1).getAsInteger(Radix: 10, Result&: Discriminator))
240 return false;
241 }
242
243 StringRef Rest = Input.substr(Start: n1 + 2);
244 if (isDigit(C: Rest[0])) {
245 LineTy = LineType::BodyProfile;
246 size_t n3 = Rest.find(C: ' ');
247 if (n3 == StringRef::npos) {
248 if (Rest.getAsInteger(Radix: 10, Result&: NumSamples))
249 return false;
250 } else {
251 if (Rest.substr(Start: 0, N: n3).getAsInteger(Radix: 10, Result&: NumSamples))
252 return false;
253 }
254 // Find call targets and their sample counts.
255 // Note: In some cases, there are symbols in the profile which are not
256 // mangled. To accommodate such cases, use colon + integer pairs as the
257 // anchor points.
258 // An example:
259 // _M_construct<char *>:1000 string_view<std::allocator<char> >:437
260 // ":1000" and ":437" are used as anchor points so the string above will
261 // be interpreted as
262 // target: _M_construct<char *>
263 // count: 1000
264 // target: string_view<std::allocator<char> >
265 // count: 437
266 while (n3 != StringRef::npos) {
267 n3 += Rest.substr(Start: n3).find_first_not_of(C: ' ');
268 Rest = Rest.substr(Start: n3);
269 n3 = Rest.find_first_of(C: ':');
270 if (n3 == StringRef::npos || n3 == 0)
271 return false;
272
273 StringRef Target;
274 uint64_t count, n4;
275 while (true) {
276 // Get the segment after the current colon.
277 StringRef AfterColon = Rest.substr(Start: n3 + 1);
278 // Get the target symbol before the current colon.
279 Target = Rest.substr(Start: 0, N: n3);
280 // Check if the word after the current colon is an integer.
281 n4 = AfterColon.find_first_of(C: ' ');
282 n4 = (n4 != StringRef::npos) ? n3 + n4 + 1 : Rest.size();
283 StringRef WordAfterColon = Rest.substr(Start: n3 + 1, N: n4 - n3 - 1);
284 if (!WordAfterColon.getAsInteger(Radix: 10, Result&: count))
285 break;
286
287 // Try to find the next colon.
288 uint64_t n5 = AfterColon.find_first_of(C: ':');
289 if (n5 == StringRef::npos)
290 return false;
291 n3 += n5 + 1;
292 }
293
294 // An anchor point is found. Save the {target, count} pair
295 TargetCountMap[Target] = count;
296 if (n4 == Rest.size())
297 break;
298 // Change n3 to the next blank space after colon + integer pair.
299 n3 = n4;
300 }
301 } else {
302 LineTy = LineType::CallSiteProfile;
303 size_t n3 = Rest.find_last_of(C: ':');
304 CalleeName = Rest.substr(Start: 0, N: n3);
305 if (Rest.substr(Start: n3 + 1).getAsInteger(Radix: 10, Result&: NumSamples))
306 return false;
307 }
308 return true;
309}
310
311/// Load samples from a text file.
312///
313/// See the documentation at the top of the file for an explanation of
314/// the expected format.
315///
316/// \returns true if the file was loaded successfully, false otherwise.
317std::error_code SampleProfileReaderText::readImpl() {
318 line_iterator LineIt(*Buffer, /*SkipBlanks=*/true, '#');
319 sampleprof_error Result = sampleprof_error::success;
320
321 InlineCallStack InlineStack;
322 uint32_t TopLevelProbeProfileCount = 0;
323
324 // DepthMetadata tracks whether we have processed metadata for the current
325 // top-level or nested function profile.
326 uint32_t DepthMetadata = 0;
327
328 ProfileIsFS = ProfileIsFSDisciminator;
329 FunctionSamples::ProfileIsFS = ProfileIsFS;
330 for (; !LineIt.is_at_eof(); ++LineIt) {
331 size_t pos = LineIt->find_first_not_of(C: ' ');
332 if (pos == LineIt->npos || (*LineIt)[pos] == '#')
333 continue;
334 // Read the header of each function.
335 //
336 // Note that for function identifiers we are actually expecting
337 // mangled names, but we may not always get them. This happens when
338 // the compiler decides not to emit the function (e.g., it was inlined
339 // and removed). In this case, the binary will not have the linkage
340 // name for the function, so the profiler will emit the function's
341 // unmangled name, which may contain characters like ':' and '>' in its
342 // name (member functions, templates, etc).
343 //
344 // The only requirement we place on the identifier, then, is that it
345 // should not begin with a number.
346 if ((*LineIt)[0] != ' ') {
347 uint64_t NumSamples, NumHeadSamples;
348 StringRef FName;
349 if (!ParseHead(Input: *LineIt, FName, NumSamples, NumHeadSamples)) {
350 reportError(LineNumber: LineIt.line_number(),
351 Msg: "Expected 'mangled_name:NUM:NUM', found " + *LineIt);
352 return sampleprof_error::malformed;
353 }
354 DepthMetadata = 0;
355 SampleContext FContext(FName, CSNameTable);
356 if (FContext.hasContext())
357 ++CSProfileCount;
358 FunctionSamples &FProfile = Profiles.Create(Ctx: FContext);
359 MergeResult(Accumulator&: Result, Result: FProfile.addTotalSamples(Num: NumSamples));
360 MergeResult(Accumulator&: Result, Result: FProfile.addHeadSamples(Num: NumHeadSamples));
361 InlineStack.clear();
362 InlineStack.push_back(Elt: &FProfile);
363 } else {
364 uint64_t NumSamples;
365 StringRef FName;
366 DenseMap<StringRef, uint64_t> TargetCountMap;
367 uint32_t Depth, LineOffset, Discriminator;
368 LineType LineTy;
369 uint64_t FunctionHash = 0;
370 uint32_t Attributes = 0;
371 if (!ParseLine(Input: *LineIt, LineTy, Depth, NumSamples, LineOffset,
372 Discriminator, CalleeName&: FName, TargetCountMap, FunctionHash,
373 Attributes)) {
374 reportError(LineNumber: LineIt.line_number(),
375 Msg: "Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " +
376 *LineIt);
377 return sampleprof_error::malformed;
378 }
379 if (LineTy != LineType::Metadata && Depth == DepthMetadata) {
380 // Metadata must be put at the end of a function profile.
381 reportError(LineNumber: LineIt.line_number(),
382 Msg: "Found non-metadata after metadata: " + *LineIt);
383 return sampleprof_error::malformed;
384 }
385
386 // Here we handle FS discriminators.
387 Discriminator &= getDiscriminatorMask();
388
389 while (InlineStack.size() > Depth) {
390 InlineStack.pop_back();
391 }
392 switch (LineTy) {
393 case LineType::CallSiteProfile: {
394 FunctionSamples &FSamples = InlineStack.back()->functionSamplesAt(
395 Loc: LineLocation(LineOffset, Discriminator))[FunctionId(FName)];
396 FSamples.setFunction(FunctionId(FName));
397 MergeResult(Accumulator&: Result, Result: FSamples.addTotalSamples(Num: NumSamples));
398 InlineStack.push_back(Elt: &FSamples);
399 DepthMetadata = 0;
400 break;
401 }
402 case LineType::BodyProfile: {
403 while (InlineStack.size() > Depth) {
404 InlineStack.pop_back();
405 }
406 FunctionSamples &FProfile = *InlineStack.back();
407 for (const auto &name_count : TargetCountMap) {
408 MergeResult(Accumulator&: Result, Result: FProfile.addCalledTargetSamples(
409 LineOffset, Discriminator,
410 Func: FunctionId(name_count.first),
411 Num: name_count.second));
412 }
413 MergeResult(Accumulator&: Result, Result: FProfile.addBodySamples(LineOffset, Discriminator,
414 Num: NumSamples));
415 break;
416 }
417 case LineType::Metadata: {
418 FunctionSamples &FProfile = *InlineStack.back();
419 if (FunctionHash) {
420 FProfile.setFunctionHash(FunctionHash);
421 if (Depth == 1)
422 ++TopLevelProbeProfileCount;
423 }
424 FProfile.getContext().setAllAttributes(Attributes);
425 if (Attributes & (uint32_t)ContextShouldBeInlined)
426 ProfileIsPreInlined = true;
427 DepthMetadata = Depth;
428 break;
429 }
430 }
431 }
432 }
433
434 assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&
435 "Cannot have both context-sensitive and regular profile");
436 ProfileIsCS = (CSProfileCount > 0);
437 assert((TopLevelProbeProfileCount == 0 ||
438 TopLevelProbeProfileCount == Profiles.size()) &&
439 "Cannot have both probe-based profiles and regular profiles");
440 ProfileIsProbeBased = (TopLevelProbeProfileCount > 0);
441 FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased;
442 FunctionSamples::ProfileIsCS = ProfileIsCS;
443 FunctionSamples::ProfileIsPreInlined = ProfileIsPreInlined;
444
445 if (Result == sampleprof_error::success)
446 computeSummary();
447
448 return Result;
449}
450
451bool SampleProfileReaderText::hasFormat(const MemoryBuffer &Buffer) {
452 bool result = false;
453
454 // Check that the first non-comment line is a valid function header.
455 line_iterator LineIt(Buffer, /*SkipBlanks=*/true, '#');
456 if (!LineIt.is_at_eof()) {
457 if ((*LineIt)[0] != ' ') {
458 uint64_t NumSamples, NumHeadSamples;
459 StringRef FName;
460 result = ParseHead(Input: *LineIt, FName, NumSamples, NumHeadSamples);
461 }
462 }
463
464 return result;
465}
466
467template <typename T> ErrorOr<T> SampleProfileReaderBinary::readNumber() {
468 unsigned NumBytesRead = 0;
469 uint64_t Val = decodeULEB128(p: Data, n: &NumBytesRead);
470
471 if (Val > std::numeric_limits<T>::max()) {
472 std::error_code EC = sampleprof_error::malformed;
473 reportError(LineNumber: 0, Msg: EC.message());
474 return EC;
475 } else if (Data + NumBytesRead > End) {
476 std::error_code EC = sampleprof_error::truncated;
477 reportError(LineNumber: 0, Msg: EC.message());
478 return EC;
479 }
480
481 Data += NumBytesRead;
482 return static_cast<T>(Val);
483}
484
485ErrorOr<StringRef> SampleProfileReaderBinary::readString() {
486 StringRef Str(reinterpret_cast<const char *>(Data));
487 if (Data + Str.size() + 1 > End) {
488 std::error_code EC = sampleprof_error::truncated;
489 reportError(LineNumber: 0, Msg: EC.message());
490 return EC;
491 }
492
493 Data += Str.size() + 1;
494 return Str;
495}
496
497template <typename T>
498ErrorOr<T> SampleProfileReaderBinary::readUnencodedNumber() {
499 if (Data + sizeof(T) > End) {
500 std::error_code EC = sampleprof_error::truncated;
501 reportError(LineNumber: 0, Msg: EC.message());
502 return EC;
503 }
504
505 using namespace support;
506 T Val = endian::readNext<T, llvm::endianness::little, unaligned>(Data);
507 return Val;
508}
509
510template <typename T>
511inline ErrorOr<size_t> SampleProfileReaderBinary::readStringIndex(T &Table) {
512 auto Idx = readNumber<size_t>();
513 if (std::error_code EC = Idx.getError())
514 return EC;
515 if (*Idx >= Table.size())
516 return sampleprof_error::truncated_name_table;
517 return *Idx;
518}
519
520ErrorOr<FunctionId>
521SampleProfileReaderBinary::readStringFromTable(size_t *RetIdx) {
522 auto Idx = readStringIndex(Table&: NameTable);
523 if (std::error_code EC = Idx.getError())
524 return EC;
525 if (RetIdx)
526 *RetIdx = *Idx;
527 return NameTable[*Idx];
528}
529
530ErrorOr<SampleContextFrames>
531SampleProfileReaderBinary::readContextFromTable(size_t *RetIdx) {
532 auto ContextIdx = readNumber<size_t>();
533 if (std::error_code EC = ContextIdx.getError())
534 return EC;
535 if (*ContextIdx >= CSNameTable.size())
536 return sampleprof_error::truncated_name_table;
537 if (RetIdx)
538 *RetIdx = *ContextIdx;
539 return CSNameTable[*ContextIdx];
540}
541
542ErrorOr<std::pair<SampleContext, uint64_t>>
543SampleProfileReaderBinary::readSampleContextFromTable() {
544 SampleContext Context;
545 size_t Idx;
546 if (ProfileIsCS) {
547 auto FContext(readContextFromTable(RetIdx: &Idx));
548 if (std::error_code EC = FContext.getError())
549 return EC;
550 Context = SampleContext(*FContext);
551 } else {
552 auto FName(readStringFromTable(RetIdx: &Idx));
553 if (std::error_code EC = FName.getError())
554 return EC;
555 Context = SampleContext(*FName);
556 }
557 // Since MD5SampleContextStart may point to the profile's file data, need to
558 // make sure it is reading the same value on big endian CPU.
559 uint64_t Hash = support::endian::read64le(P: MD5SampleContextStart + Idx);
560 // Lazy computing of hash value, write back to the table to cache it. Only
561 // compute the context's hash value if it is being referenced for the first
562 // time.
563 if (Hash == 0) {
564 assert(MD5SampleContextStart == MD5SampleContextTable.data());
565 Hash = Context.getHashCode();
566 support::endian::write64le(P: &MD5SampleContextTable[Idx], V: Hash);
567 }
568 return std::make_pair(x&: Context, y&: Hash);
569}
570
571std::error_code
572SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) {
573 auto NumSamples = readNumber<uint64_t>();
574 if (std::error_code EC = NumSamples.getError())
575 return EC;
576 FProfile.addTotalSamples(Num: *NumSamples);
577
578 // Read the samples in the body.
579 auto NumRecords = readNumber<uint32_t>();
580 if (std::error_code EC = NumRecords.getError())
581 return EC;
582
583 for (uint32_t I = 0; I < *NumRecords; ++I) {
584 auto LineOffset = readNumber<uint64_t>();
585 if (std::error_code EC = LineOffset.getError())
586 return EC;
587
588 if (!isOffsetLegal(L: *LineOffset)) {
589 return std::error_code();
590 }
591
592 auto Discriminator = readNumber<uint64_t>();
593 if (std::error_code EC = Discriminator.getError())
594 return EC;
595
596 auto NumSamples = readNumber<uint64_t>();
597 if (std::error_code EC = NumSamples.getError())
598 return EC;
599
600 auto NumCalls = readNumber<uint32_t>();
601 if (std::error_code EC = NumCalls.getError())
602 return EC;
603
604 // Here we handle FS discriminators:
605 uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();
606
607 for (uint32_t J = 0; J < *NumCalls; ++J) {
608 auto CalledFunction(readStringFromTable());
609 if (std::error_code EC = CalledFunction.getError())
610 return EC;
611
612 auto CalledFunctionSamples = readNumber<uint64_t>();
613 if (std::error_code EC = CalledFunctionSamples.getError())
614 return EC;
615
616 FProfile.addCalledTargetSamples(LineOffset: *LineOffset, Discriminator: DiscriminatorVal,
617 Func: *CalledFunction, Num: *CalledFunctionSamples);
618 }
619
620 FProfile.addBodySamples(LineOffset: *LineOffset, Discriminator: DiscriminatorVal, Num: *NumSamples);
621 }
622
623 // Read all the samples for inlined function calls.
624 auto NumCallsites = readNumber<uint32_t>();
625 if (std::error_code EC = NumCallsites.getError())
626 return EC;
627
628 for (uint32_t J = 0; J < *NumCallsites; ++J) {
629 auto LineOffset = readNumber<uint64_t>();
630 if (std::error_code EC = LineOffset.getError())
631 return EC;
632
633 auto Discriminator = readNumber<uint64_t>();
634 if (std::error_code EC = Discriminator.getError())
635 return EC;
636
637 auto FName(readStringFromTable());
638 if (std::error_code EC = FName.getError())
639 return EC;
640
641 // Here we handle FS discriminators:
642 uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();
643
644 FunctionSamples &CalleeProfile = FProfile.functionSamplesAt(
645 Loc: LineLocation(*LineOffset, DiscriminatorVal))[*FName];
646 CalleeProfile.setFunction(*FName);
647 if (std::error_code EC = readProfile(FProfile&: CalleeProfile))
648 return EC;
649 }
650
651 return sampleprof_error::success;
652}
653
654std::error_code
655SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start) {
656 Data = Start;
657 auto NumHeadSamples = readNumber<uint64_t>();
658 if (std::error_code EC = NumHeadSamples.getError())
659 return EC;
660
661 auto FContextHash(readSampleContextFromTable());
662 if (std::error_code EC = FContextHash.getError())
663 return EC;
664
665 auto &[FContext, Hash] = *FContextHash;
666 // Use the cached hash value for insertion instead of recalculating it.
667 auto Res = Profiles.try_emplace(Hash, Key: FContext, Args: FunctionSamples());
668 FunctionSamples &FProfile = Res.first->second;
669 FProfile.setContext(FContext);
670 FProfile.addHeadSamples(Num: *NumHeadSamples);
671
672 if (FContext.hasContext())
673 CSProfileCount++;
674
675 if (std::error_code EC = readProfile(FProfile))
676 return EC;
677 return sampleprof_error::success;
678}
679
680std::error_code SampleProfileReaderBinary::readImpl() {
681 ProfileIsFS = ProfileIsFSDisciminator;
682 FunctionSamples::ProfileIsFS = ProfileIsFS;
683 while (Data < End) {
684 if (std::error_code EC = readFuncProfile(Start: Data))
685 return EC;
686 }
687
688 return sampleprof_error::success;
689}
690
691std::error_code SampleProfileReaderExtBinaryBase::readOneSection(
692 const uint8_t *Start, uint64_t Size, const SecHdrTableEntry &Entry) {
693 Data = Start;
694 End = Start + Size;
695 switch (Entry.Type) {
696 case SecProfSummary:
697 if (std::error_code EC = readSummary())
698 return EC;
699 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagPartial))
700 Summary->setPartialProfile(true);
701 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagFullContext))
702 FunctionSamples::ProfileIsCS = ProfileIsCS = true;
703 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagIsPreInlined))
704 FunctionSamples::ProfileIsPreInlined = ProfileIsPreInlined = true;
705 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagFSDiscriminator))
706 FunctionSamples::ProfileIsFS = ProfileIsFS = true;
707 break;
708 case SecNameTable: {
709 bool FixedLengthMD5 =
710 hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagFixedLengthMD5);
711 bool UseMD5 = hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagMD5Name);
712 // UseMD5 means if THIS section uses MD5, ProfileIsMD5 means if the entire
713 // profile uses MD5 for function name matching in IPO passes.
714 ProfileIsMD5 = ProfileIsMD5 || UseMD5;
715 FunctionSamples::HasUniqSuffix =
716 hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagUniqSuffix);
717 if (std::error_code EC = readNameTableSec(IsMD5: UseMD5, FixedLengthMD5))
718 return EC;
719 break;
720 }
721 case SecCSNameTable: {
722 if (std::error_code EC = readCSNameTableSec())
723 return EC;
724 break;
725 }
726 case SecLBRProfile:
727 if (std::error_code EC = readFuncProfiles())
728 return EC;
729 break;
730 case SecFuncOffsetTable:
731 // If module is absent, we are using LLVM tools, and need to read all
732 // profiles, so skip reading the function offset table.
733 if (!M) {
734 Data = End;
735 } else {
736 assert((!ProfileIsCS ||
737 hasSecFlag(Entry, SecFuncOffsetFlags::SecFlagOrdered)) &&
738 "func offset table should always be sorted in CS profile");
739 if (std::error_code EC = readFuncOffsetTable())
740 return EC;
741 }
742 break;
743 case SecFuncMetadata: {
744 ProfileIsProbeBased =
745 hasSecFlag(Entry, Flag: SecFuncMetadataFlags::SecFlagIsProbeBased);
746 FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased;
747 bool HasAttribute =
748 hasSecFlag(Entry, Flag: SecFuncMetadataFlags::SecFlagHasAttribute);
749 if (std::error_code EC = readFuncMetadata(ProfileHasAttribute: HasAttribute))
750 return EC;
751 break;
752 }
753 case SecProfileSymbolList:
754 if (std::error_code EC = readProfileSymbolList())
755 return EC;
756 break;
757 default:
758 if (std::error_code EC = readCustomSection(Entry))
759 return EC;
760 break;
761 }
762 return sampleprof_error::success;
763}
764
765bool SampleProfileReaderExtBinaryBase::useFuncOffsetList() const {
766 // If profile is CS, the function offset section is expected to consist of
767 // sequences of contexts in pre-order layout
768 // (e.g. [A, A:1 @ B, A:1 @ B:2.3 @ C] [D, D:1 @ E]), so that when a matched
769 // context in the module is found, the profiles of all its callees are
770 // recursively loaded. A list is needed since the order of profiles matters.
771 if (ProfileIsCS)
772 return true;
773
774 // If the profile is MD5, use the map container to lookup functions in
775 // the module. A remapper has no use on MD5 names.
776 if (useMD5())
777 return false;
778
779 // Profile is not MD5 and if a remapper is present, the remapped name of
780 // every function needed to be matched against the module, so use the list
781 // container since each entry is accessed.
782 if (Remapper)
783 return true;
784
785 // Otherwise use the map container for faster lookup.
786 // TODO: If the cardinality of the function offset section is much smaller
787 // than the number of functions in the module, using the list container can
788 // be always faster, but we need to figure out the constant factor to
789 // determine the cutoff.
790 return false;
791}
792
793
794bool SampleProfileReaderExtBinaryBase::collectFuncsFromModule() {
795 if (!M)
796 return false;
797 FuncsToUse.clear();
798 for (auto &F : *M)
799 FuncsToUse.insert(V: FunctionSamples::getCanonicalFnName(F));
800 return true;
801}
802
803std::error_code SampleProfileReaderExtBinaryBase::readFuncOffsetTable() {
804 // If there are more than one function offset section, the profile associated
805 // with the previous section has to be done reading before next one is read.
806 FuncOffsetTable.clear();
807 FuncOffsetList.clear();
808
809 auto Size = readNumber<uint64_t>();
810 if (std::error_code EC = Size.getError())
811 return EC;
812
813 bool UseFuncOffsetList = useFuncOffsetList();
814 if (UseFuncOffsetList)
815 FuncOffsetList.reserve(n: *Size);
816 else
817 FuncOffsetTable.reserve(NumEntries: *Size);
818
819 for (uint64_t I = 0; I < *Size; ++I) {
820 auto FContextHash(readSampleContextFromTable());
821 if (std::error_code EC = FContextHash.getError())
822 return EC;
823
824 auto &[FContext, Hash] = *FContextHash;
825 auto Offset = readNumber<uint64_t>();
826 if (std::error_code EC = Offset.getError())
827 return EC;
828
829 if (UseFuncOffsetList)
830 FuncOffsetList.emplace_back(args&: FContext, args&: *Offset);
831 else
832 // Because Porfiles replace existing value with new value if collision
833 // happens, we also use the latest offset so that they are consistent.
834 FuncOffsetTable[Hash] = *Offset;
835 }
836
837 return sampleprof_error::success;
838}
839
840std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() {
841 // Collect functions used by current module if the Reader has been
842 // given a module.
843 // collectFuncsFromModule uses FunctionSamples::getCanonicalFnName
844 // which will query FunctionSamples::HasUniqSuffix, so it has to be
845 // called after FunctionSamples::HasUniqSuffix is set, i.e. after
846 // NameTable section is read.
847 bool LoadFuncsToBeUsed = collectFuncsFromModule();
848
849 // When LoadFuncsToBeUsed is false, we are using LLVM tool, need to read all
850 // profiles.
851 const uint8_t *Start = Data;
852 if (!LoadFuncsToBeUsed) {
853 while (Data < End) {
854 if (std::error_code EC = readFuncProfile(Start: Data))
855 return EC;
856 }
857 assert(Data == End && "More data is read than expected");
858 } else {
859 // Load function profiles on demand.
860 if (Remapper) {
861 for (auto Name : FuncsToUse) {
862 Remapper->insert(FunctionName: Name);
863 }
864 }
865
866 if (ProfileIsCS) {
867 assert(useFuncOffsetList());
868 DenseSet<uint64_t> FuncGuidsToUse;
869 if (useMD5()) {
870 for (auto Name : FuncsToUse)
871 FuncGuidsToUse.insert(V: Function::getGUID(GlobalName: Name));
872 }
873
874 // For each function in current module, load all context profiles for
875 // the function as well as their callee contexts which can help profile
876 // guided importing for ThinLTO. This can be achieved by walking
877 // through an ordered context container, where contexts are laid out
878 // as if they were walked in preorder of a context trie. While
879 // traversing the trie, a link to the highest common ancestor node is
880 // kept so that all of its decendants will be loaded.
881 const SampleContext *CommonContext = nullptr;
882 for (const auto &NameOffset : FuncOffsetList) {
883 const auto &FContext = NameOffset.first;
884 FunctionId FName = FContext.getFunction();
885 StringRef FNameString;
886 if (!useMD5())
887 FNameString = FName.stringRef();
888
889 // For function in the current module, keep its farthest ancestor
890 // context. This can be used to load itself and its child and
891 // sibling contexts.
892 if ((useMD5() && FuncGuidsToUse.count(V: FName.getHashCode())) ||
893 (!useMD5() && (FuncsToUse.count(V: FNameString) ||
894 (Remapper && Remapper->exist(FunctionName: FNameString))))) {
895 if (!CommonContext || !CommonContext->IsPrefixOf(That: FContext))
896 CommonContext = &FContext;
897 }
898
899 if (CommonContext == &FContext ||
900 (CommonContext && CommonContext->IsPrefixOf(That: FContext))) {
901 // Load profile for the current context which originated from
902 // the common ancestor.
903 const uint8_t *FuncProfileAddr = Start + NameOffset.second;
904 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr))
905 return EC;
906 }
907 }
908 } else if (useMD5()) {
909 assert(!useFuncOffsetList());
910 for (auto Name : FuncsToUse) {
911 auto GUID = MD5Hash(Str: Name);
912 auto iter = FuncOffsetTable.find(Val: GUID);
913 if (iter == FuncOffsetTable.end())
914 continue;
915 const uint8_t *FuncProfileAddr = Start + iter->second;
916 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr))
917 return EC;
918 }
919 } else if (Remapper) {
920 assert(useFuncOffsetList());
921 for (auto NameOffset : FuncOffsetList) {
922 SampleContext FContext(NameOffset.first);
923 auto FuncName = FContext.getFunction();
924 StringRef FuncNameStr = FuncName.stringRef();
925 if (!FuncsToUse.count(V: FuncNameStr) && !Remapper->exist(FunctionName: FuncNameStr))
926 continue;
927 const uint8_t *FuncProfileAddr = Start + NameOffset.second;
928 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr))
929 return EC;
930 }
931 } else {
932 assert(!useFuncOffsetList());
933 for (auto Name : FuncsToUse) {
934 auto iter = FuncOffsetTable.find(Val: MD5Hash(Str: Name));
935 if (iter == FuncOffsetTable.end())
936 continue;
937 const uint8_t *FuncProfileAddr = Start + iter->second;
938 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr))
939 return EC;
940 }
941 }
942 Data = End;
943 }
944 assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&
945 "Cannot have both context-sensitive and regular profile");
946 assert((!CSProfileCount || ProfileIsCS) &&
947 "Section flag should be consistent with actual profile");
948 return sampleprof_error::success;
949}
950
951std::error_code SampleProfileReaderExtBinaryBase::readProfileSymbolList() {
952 if (!ProfSymList)
953 ProfSymList = std::make_unique<ProfileSymbolList>();
954
955 if (std::error_code EC = ProfSymList->read(Data, ListSize: End - Data))
956 return EC;
957
958 Data = End;
959 return sampleprof_error::success;
960}
961
962std::error_code SampleProfileReaderExtBinaryBase::decompressSection(
963 const uint8_t *SecStart, const uint64_t SecSize,
964 const uint8_t *&DecompressBuf, uint64_t &DecompressBufSize) {
965 Data = SecStart;
966 End = SecStart + SecSize;
967 auto DecompressSize = readNumber<uint64_t>();
968 if (std::error_code EC = DecompressSize.getError())
969 return EC;
970 DecompressBufSize = *DecompressSize;
971
972 auto CompressSize = readNumber<uint64_t>();
973 if (std::error_code EC = CompressSize.getError())
974 return EC;
975
976 if (!llvm::compression::zlib::isAvailable())
977 return sampleprof_error::zlib_unavailable;
978
979 uint8_t *Buffer = Allocator.Allocate<uint8_t>(Num: DecompressBufSize);
980 size_t UCSize = DecompressBufSize;
981 llvm::Error E = compression::zlib::decompress(Input: ArrayRef(Data, *CompressSize),
982 Output: Buffer, UncompressedSize&: UCSize);
983 if (E)
984 return sampleprof_error::uncompress_failed;
985 DecompressBuf = reinterpret_cast<const uint8_t *>(Buffer);
986 return sampleprof_error::success;
987}
988
989std::error_code SampleProfileReaderExtBinaryBase::readImpl() {
990 const uint8_t *BufStart =
991 reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
992
993 for (auto &Entry : SecHdrTable) {
994 // Skip empty section.
995 if (!Entry.Size)
996 continue;
997
998 // Skip sections without context when SkipFlatProf is true.
999 if (SkipFlatProf && hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagFlat))
1000 continue;
1001
1002 const uint8_t *SecStart = BufStart + Entry.Offset;
1003 uint64_t SecSize = Entry.Size;
1004
1005 // If the section is compressed, decompress it into a buffer
1006 // DecompressBuf before reading the actual data. The pointee of
1007 // 'Data' will be changed to buffer hold by DecompressBuf
1008 // temporarily when reading the actual data.
1009 bool isCompressed = hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress);
1010 if (isCompressed) {
1011 const uint8_t *DecompressBuf;
1012 uint64_t DecompressBufSize;
1013 if (std::error_code EC = decompressSection(
1014 SecStart, SecSize, DecompressBuf, DecompressBufSize))
1015 return EC;
1016 SecStart = DecompressBuf;
1017 SecSize = DecompressBufSize;
1018 }
1019
1020 if (std::error_code EC = readOneSection(Start: SecStart, Size: SecSize, Entry))
1021 return EC;
1022 if (Data != SecStart + SecSize)
1023 return sampleprof_error::malformed;
1024
1025 // Change the pointee of 'Data' from DecompressBuf to original Buffer.
1026 if (isCompressed) {
1027 Data = BufStart + Entry.Offset;
1028 End = BufStart + Buffer->getBufferSize();
1029 }
1030 }
1031
1032 return sampleprof_error::success;
1033}
1034
1035std::error_code SampleProfileReaderRawBinary::verifySPMagic(uint64_t Magic) {
1036 if (Magic == SPMagic())
1037 return sampleprof_error::success;
1038 return sampleprof_error::bad_magic;
1039}
1040
1041std::error_code SampleProfileReaderExtBinary::verifySPMagic(uint64_t Magic) {
1042 if (Magic == SPMagic(Format: SPF_Ext_Binary))
1043 return sampleprof_error::success;
1044 return sampleprof_error::bad_magic;
1045}
1046
1047std::error_code SampleProfileReaderBinary::readNameTable() {
1048 auto Size = readNumber<size_t>();
1049 if (std::error_code EC = Size.getError())
1050 return EC;
1051
1052 // Normally if useMD5 is true, the name table should have MD5 values, not
1053 // strings, however in the case that ExtBinary profile has multiple name
1054 // tables mixing string and MD5, all of them have to be normalized to use MD5,
1055 // because optimization passes can only handle either type.
1056 bool UseMD5 = useMD5();
1057
1058 NameTable.clear();
1059 NameTable.reserve(n: *Size);
1060 if (!ProfileIsCS) {
1061 MD5SampleContextTable.clear();
1062 if (UseMD5)
1063 MD5SampleContextTable.reserve(n: *Size);
1064 else
1065 // If we are using strings, delay MD5 computation since only a portion of
1066 // names are used by top level functions. Use 0 to indicate MD5 value is
1067 // to be calculated as no known string has a MD5 value of 0.
1068 MD5SampleContextTable.resize(new_size: *Size);
1069 }
1070 for (size_t I = 0; I < *Size; ++I) {
1071 auto Name(readString());
1072 if (std::error_code EC = Name.getError())
1073 return EC;
1074 if (UseMD5) {
1075 FunctionId FID(*Name);
1076 if (!ProfileIsCS)
1077 MD5SampleContextTable.emplace_back(args: FID.getHashCode());
1078 NameTable.emplace_back(args&: FID);
1079 } else
1080 NameTable.push_back(x: FunctionId(*Name));
1081 }
1082 if (!ProfileIsCS)
1083 MD5SampleContextStart = MD5SampleContextTable.data();
1084 return sampleprof_error::success;
1085}
1086
1087std::error_code
1088SampleProfileReaderExtBinaryBase::readNameTableSec(bool IsMD5,
1089 bool FixedLengthMD5) {
1090 if (FixedLengthMD5) {
1091 if (!IsMD5)
1092 errs() << "If FixedLengthMD5 is true, UseMD5 has to be true";
1093 auto Size = readNumber<size_t>();
1094 if (std::error_code EC = Size.getError())
1095 return EC;
1096
1097 assert(Data + (*Size) * sizeof(uint64_t) == End &&
1098 "Fixed length MD5 name table does not contain specified number of "
1099 "entries");
1100 if (Data + (*Size) * sizeof(uint64_t) > End)
1101 return sampleprof_error::truncated;
1102
1103 NameTable.clear();
1104 NameTable.reserve(n: *Size);
1105 for (size_t I = 0; I < *Size; ++I) {
1106 using namespace support;
1107 uint64_t FID = endian::read<uint64_t, endianness::little, unaligned>(
1108 memory: Data + I * sizeof(uint64_t));
1109 NameTable.emplace_back(args: FunctionId(FID));
1110 }
1111 if (!ProfileIsCS)
1112 MD5SampleContextStart = reinterpret_cast<const uint64_t *>(Data);
1113 Data = Data + (*Size) * sizeof(uint64_t);
1114 return sampleprof_error::success;
1115 }
1116
1117 if (IsMD5) {
1118 assert(!FixedLengthMD5 && "FixedLengthMD5 should be unreachable here");
1119 auto Size = readNumber<size_t>();
1120 if (std::error_code EC = Size.getError())
1121 return EC;
1122
1123 NameTable.clear();
1124 NameTable.reserve(n: *Size);
1125 if (!ProfileIsCS)
1126 MD5SampleContextTable.resize(new_size: *Size);
1127 for (size_t I = 0; I < *Size; ++I) {
1128 auto FID = readNumber<uint64_t>();
1129 if (std::error_code EC = FID.getError())
1130 return EC;
1131 if (!ProfileIsCS)
1132 support::endian::write64le(P: &MD5SampleContextTable[I], V: *FID);
1133 NameTable.emplace_back(args: FunctionId(*FID));
1134 }
1135 if (!ProfileIsCS)
1136 MD5SampleContextStart = MD5SampleContextTable.data();
1137 return sampleprof_error::success;
1138 }
1139
1140 return SampleProfileReaderBinary::readNameTable();
1141}
1142
1143// Read in the CS name table section, which basically contains a list of context
1144// vectors. Each element of a context vector, aka a frame, refers to the
1145// underlying raw function names that are stored in the name table, as well as
1146// a callsite identifier that only makes sense for non-leaf frames.
1147std::error_code SampleProfileReaderExtBinaryBase::readCSNameTableSec() {
1148 auto Size = readNumber<size_t>();
1149 if (std::error_code EC = Size.getError())
1150 return EC;
1151
1152 CSNameTable.clear();
1153 CSNameTable.reserve(n: *Size);
1154 if (ProfileIsCS) {
1155 // Delay MD5 computation of CS context until they are needed. Use 0 to
1156 // indicate MD5 value is to be calculated as no known string has a MD5
1157 // value of 0.
1158 MD5SampleContextTable.clear();
1159 MD5SampleContextTable.resize(new_size: *Size);
1160 MD5SampleContextStart = MD5SampleContextTable.data();
1161 }
1162 for (size_t I = 0; I < *Size; ++I) {
1163 CSNameTable.emplace_back(args: SampleContextFrameVector());
1164 auto ContextSize = readNumber<uint32_t>();
1165 if (std::error_code EC = ContextSize.getError())
1166 return EC;
1167 for (uint32_t J = 0; J < *ContextSize; ++J) {
1168 auto FName(readStringFromTable());
1169 if (std::error_code EC = FName.getError())
1170 return EC;
1171 auto LineOffset = readNumber<uint64_t>();
1172 if (std::error_code EC = LineOffset.getError())
1173 return EC;
1174
1175 if (!isOffsetLegal(L: *LineOffset))
1176 return std::error_code();
1177
1178 auto Discriminator = readNumber<uint64_t>();
1179 if (std::error_code EC = Discriminator.getError())
1180 return EC;
1181
1182 CSNameTable.back().emplace_back(
1183 Args&: FName.get(), Args: LineLocation(LineOffset.get(), Discriminator.get()));
1184 }
1185 }
1186
1187 return sampleprof_error::success;
1188}
1189
1190std::error_code
1191SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute,
1192 FunctionSamples *FProfile) {
1193 if (Data < End) {
1194 if (ProfileIsProbeBased) {
1195 auto Checksum = readNumber<uint64_t>();
1196 if (std::error_code EC = Checksum.getError())
1197 return EC;
1198 if (FProfile)
1199 FProfile->setFunctionHash(*Checksum);
1200 }
1201
1202 if (ProfileHasAttribute) {
1203 auto Attributes = readNumber<uint32_t>();
1204 if (std::error_code EC = Attributes.getError())
1205 return EC;
1206 if (FProfile)
1207 FProfile->getContext().setAllAttributes(*Attributes);
1208 }
1209
1210 if (!ProfileIsCS) {
1211 // Read all the attributes for inlined function calls.
1212 auto NumCallsites = readNumber<uint32_t>();
1213 if (std::error_code EC = NumCallsites.getError())
1214 return EC;
1215
1216 for (uint32_t J = 0; J < *NumCallsites; ++J) {
1217 auto LineOffset = readNumber<uint64_t>();
1218 if (std::error_code EC = LineOffset.getError())
1219 return EC;
1220
1221 auto Discriminator = readNumber<uint64_t>();
1222 if (std::error_code EC = Discriminator.getError())
1223 return EC;
1224
1225 auto FContextHash(readSampleContextFromTable());
1226 if (std::error_code EC = FContextHash.getError())
1227 return EC;
1228
1229 auto &[FContext, Hash] = *FContextHash;
1230 FunctionSamples *CalleeProfile = nullptr;
1231 if (FProfile) {
1232 CalleeProfile = const_cast<FunctionSamples *>(
1233 &FProfile->functionSamplesAt(Loc: LineLocation(
1234 *LineOffset,
1235 *Discriminator))[FContext.getFunction()]);
1236 }
1237 if (std::error_code EC =
1238 readFuncMetadata(ProfileHasAttribute, FProfile: CalleeProfile))
1239 return EC;
1240 }
1241 }
1242 }
1243
1244 return sampleprof_error::success;
1245}
1246
1247std::error_code
1248SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute) {
1249 while (Data < End) {
1250 auto FContextHash(readSampleContextFromTable());
1251 if (std::error_code EC = FContextHash.getError())
1252 return EC;
1253 auto &[FContext, Hash] = *FContextHash;
1254 FunctionSamples *FProfile = nullptr;
1255 auto It = Profiles.find(Ctx: FContext);
1256 if (It != Profiles.end())
1257 FProfile = &It->second;
1258
1259 if (std::error_code EC = readFuncMetadata(ProfileHasAttribute, FProfile))
1260 return EC;
1261 }
1262
1263 assert(Data == End && "More data is read than expected");
1264 return sampleprof_error::success;
1265}
1266
1267std::error_code
1268SampleProfileReaderExtBinaryBase::readSecHdrTableEntry(uint64_t Idx) {
1269 SecHdrTableEntry Entry;
1270 auto Type = readUnencodedNumber<uint64_t>();
1271 if (std::error_code EC = Type.getError())
1272 return EC;
1273 Entry.Type = static_cast<SecType>(*Type);
1274
1275 auto Flags = readUnencodedNumber<uint64_t>();
1276 if (std::error_code EC = Flags.getError())
1277 return EC;
1278 Entry.Flags = *Flags;
1279
1280 auto Offset = readUnencodedNumber<uint64_t>();
1281 if (std::error_code EC = Offset.getError())
1282 return EC;
1283 Entry.Offset = *Offset;
1284
1285 auto Size = readUnencodedNumber<uint64_t>();
1286 if (std::error_code EC = Size.getError())
1287 return EC;
1288 Entry.Size = *Size;
1289
1290 Entry.LayoutIndex = Idx;
1291 SecHdrTable.push_back(x: std::move(Entry));
1292 return sampleprof_error::success;
1293}
1294
1295std::error_code SampleProfileReaderExtBinaryBase::readSecHdrTable() {
1296 auto EntryNum = readUnencodedNumber<uint64_t>();
1297 if (std::error_code EC = EntryNum.getError())
1298 return EC;
1299
1300 for (uint64_t i = 0; i < (*EntryNum); i++)
1301 if (std::error_code EC = readSecHdrTableEntry(Idx: i))
1302 return EC;
1303
1304 return sampleprof_error::success;
1305}
1306
1307std::error_code SampleProfileReaderExtBinaryBase::readHeader() {
1308 const uint8_t *BufStart =
1309 reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
1310 Data = BufStart;
1311 End = BufStart + Buffer->getBufferSize();
1312
1313 if (std::error_code EC = readMagicIdent())
1314 return EC;
1315
1316 if (std::error_code EC = readSecHdrTable())
1317 return EC;
1318
1319 return sampleprof_error::success;
1320}
1321
1322uint64_t SampleProfileReaderExtBinaryBase::getSectionSize(SecType Type) {
1323 uint64_t Size = 0;
1324 for (auto &Entry : SecHdrTable) {
1325 if (Entry.Type == Type)
1326 Size += Entry.Size;
1327 }
1328 return Size;
1329}
1330
1331uint64_t SampleProfileReaderExtBinaryBase::getFileSize() {
1332 // Sections in SecHdrTable is not necessarily in the same order as
1333 // sections in the profile because section like FuncOffsetTable needs
1334 // to be written after section LBRProfile but needs to be read before
1335 // section LBRProfile, so we cannot simply use the last entry in
1336 // SecHdrTable to calculate the file size.
1337 uint64_t FileSize = 0;
1338 for (auto &Entry : SecHdrTable) {
1339 FileSize = std::max(a: Entry.Offset + Entry.Size, b: FileSize);
1340 }
1341 return FileSize;
1342}
1343
1344static std::string getSecFlagsStr(const SecHdrTableEntry &Entry) {
1345 std::string Flags;
1346 if (hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress))
1347 Flags.append(s: "{compressed,");
1348 else
1349 Flags.append(s: "{");
1350
1351 if (hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagFlat))
1352 Flags.append(s: "flat,");
1353
1354 switch (Entry.Type) {
1355 case SecNameTable:
1356 if (hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagFixedLengthMD5))
1357 Flags.append(s: "fixlenmd5,");
1358 else if (hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagMD5Name))
1359 Flags.append(s: "md5,");
1360 if (hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagUniqSuffix))
1361 Flags.append(s: "uniq,");
1362 break;
1363 case SecProfSummary:
1364 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagPartial))
1365 Flags.append(s: "partial,");
1366 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagFullContext))
1367 Flags.append(s: "context,");
1368 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagIsPreInlined))
1369 Flags.append(s: "preInlined,");
1370 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagFSDiscriminator))
1371 Flags.append(s: "fs-discriminator,");
1372 break;
1373 case SecFuncOffsetTable:
1374 if (hasSecFlag(Entry, Flag: SecFuncOffsetFlags::SecFlagOrdered))
1375 Flags.append(s: "ordered,");
1376 break;
1377 case SecFuncMetadata:
1378 if (hasSecFlag(Entry, Flag: SecFuncMetadataFlags::SecFlagIsProbeBased))
1379 Flags.append(s: "probe,");
1380 if (hasSecFlag(Entry, Flag: SecFuncMetadataFlags::SecFlagHasAttribute))
1381 Flags.append(s: "attr,");
1382 break;
1383 default:
1384 break;
1385 }
1386 char &last = Flags.back();
1387 if (last == ',')
1388 last = '}';
1389 else
1390 Flags.append(s: "}");
1391 return Flags;
1392}
1393
1394bool SampleProfileReaderExtBinaryBase::dumpSectionInfo(raw_ostream &OS) {
1395 uint64_t TotalSecsSize = 0;
1396 for (auto &Entry : SecHdrTable) {
1397 OS << getSecName(Type: Entry.Type) << " - Offset: " << Entry.Offset
1398 << ", Size: " << Entry.Size << ", Flags: " << getSecFlagsStr(Entry)
1399 << "\n";
1400 ;
1401 TotalSecsSize += Entry.Size;
1402 }
1403 uint64_t HeaderSize = SecHdrTable.front().Offset;
1404 assert(HeaderSize + TotalSecsSize == getFileSize() &&
1405 "Size of 'header + sections' doesn't match the total size of profile");
1406
1407 OS << "Header Size: " << HeaderSize << "\n";
1408 OS << "Total Sections Size: " << TotalSecsSize << "\n";
1409 OS << "File Size: " << getFileSize() << "\n";
1410 return true;
1411}
1412
1413std::error_code SampleProfileReaderBinary::readMagicIdent() {
1414 // Read and check the magic identifier.
1415 auto Magic = readNumber<uint64_t>();
1416 if (std::error_code EC = Magic.getError())
1417 return EC;
1418 else if (std::error_code EC = verifySPMagic(Magic: *Magic))
1419 return EC;
1420
1421 // Read the version number.
1422 auto Version = readNumber<uint64_t>();
1423 if (std::error_code EC = Version.getError())
1424 return EC;
1425 else if (*Version != SPVersion())
1426 return sampleprof_error::unsupported_version;
1427
1428 return sampleprof_error::success;
1429}
1430
1431std::error_code SampleProfileReaderBinary::readHeader() {
1432 Data = reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
1433 End = Data + Buffer->getBufferSize();
1434
1435 if (std::error_code EC = readMagicIdent())
1436 return EC;
1437
1438 if (std::error_code EC = readSummary())
1439 return EC;
1440
1441 if (std::error_code EC = readNameTable())
1442 return EC;
1443 return sampleprof_error::success;
1444}
1445
1446std::error_code SampleProfileReaderBinary::readSummaryEntry(
1447 std::vector<ProfileSummaryEntry> &Entries) {
1448 auto Cutoff = readNumber<uint64_t>();
1449 if (std::error_code EC = Cutoff.getError())
1450 return EC;
1451
1452 auto MinBlockCount = readNumber<uint64_t>();
1453 if (std::error_code EC = MinBlockCount.getError())
1454 return EC;
1455
1456 auto NumBlocks = readNumber<uint64_t>();
1457 if (std::error_code EC = NumBlocks.getError())
1458 return EC;
1459
1460 Entries.emplace_back(args&: *Cutoff, args&: *MinBlockCount, args&: *NumBlocks);
1461 return sampleprof_error::success;
1462}
1463
1464std::error_code SampleProfileReaderBinary::readSummary() {
1465 auto TotalCount = readNumber<uint64_t>();
1466 if (std::error_code EC = TotalCount.getError())
1467 return EC;
1468
1469 auto MaxBlockCount = readNumber<uint64_t>();
1470 if (std::error_code EC = MaxBlockCount.getError())
1471 return EC;
1472
1473 auto MaxFunctionCount = readNumber<uint64_t>();
1474 if (std::error_code EC = MaxFunctionCount.getError())
1475 return EC;
1476
1477 auto NumBlocks = readNumber<uint64_t>();
1478 if (std::error_code EC = NumBlocks.getError())
1479 return EC;
1480
1481 auto NumFunctions = readNumber<uint64_t>();
1482 if (std::error_code EC = NumFunctions.getError())
1483 return EC;
1484
1485 auto NumSummaryEntries = readNumber<uint64_t>();
1486 if (std::error_code EC = NumSummaryEntries.getError())
1487 return EC;
1488
1489 std::vector<ProfileSummaryEntry> Entries;
1490 for (unsigned i = 0; i < *NumSummaryEntries; i++) {
1491 std::error_code EC = readSummaryEntry(Entries);
1492 if (EC != sampleprof_error::success)
1493 return EC;
1494 }
1495 Summary = std::make_unique<ProfileSummary>(
1496 args: ProfileSummary::PSK_Sample, args&: Entries, args&: *TotalCount, args&: *MaxBlockCount, args: 0,
1497 args&: *MaxFunctionCount, args&: *NumBlocks, args&: *NumFunctions);
1498
1499 return sampleprof_error::success;
1500}
1501
1502bool SampleProfileReaderRawBinary::hasFormat(const MemoryBuffer &Buffer) {
1503 const uint8_t *Data =
1504 reinterpret_cast<const uint8_t *>(Buffer.getBufferStart());
1505 uint64_t Magic = decodeULEB128(p: Data);
1506 return Magic == SPMagic();
1507}
1508
1509bool SampleProfileReaderExtBinary::hasFormat(const MemoryBuffer &Buffer) {
1510 const uint8_t *Data =
1511 reinterpret_cast<const uint8_t *>(Buffer.getBufferStart());
1512 uint64_t Magic = decodeULEB128(p: Data);
1513 return Magic == SPMagic(Format: SPF_Ext_Binary);
1514}
1515
1516std::error_code SampleProfileReaderGCC::skipNextWord() {
1517 uint32_t dummy;
1518 if (!GcovBuffer.readInt(Val&: dummy))
1519 return sampleprof_error::truncated;
1520 return sampleprof_error::success;
1521}
1522
1523template <typename T> ErrorOr<T> SampleProfileReaderGCC::readNumber() {
1524 if (sizeof(T) <= sizeof(uint32_t)) {
1525 uint32_t Val;
1526 if (GcovBuffer.readInt(Val) && Val <= std::numeric_limits<T>::max())
1527 return static_cast<T>(Val);
1528 } else if (sizeof(T) <= sizeof(uint64_t)) {
1529 uint64_t Val;
1530 if (GcovBuffer.readInt64(Val) && Val <= std::numeric_limits<T>::max())
1531 return static_cast<T>(Val);
1532 }
1533
1534 std::error_code EC = sampleprof_error::malformed;
1535 reportError(LineNumber: 0, Msg: EC.message());
1536 return EC;
1537}
1538
1539ErrorOr<StringRef> SampleProfileReaderGCC::readString() {
1540 StringRef Str;
1541 if (!GcovBuffer.readString(str&: Str))
1542 return sampleprof_error::truncated;
1543 return Str;
1544}
1545
1546std::error_code SampleProfileReaderGCC::readHeader() {
1547 // Read the magic identifier.
1548 if (!GcovBuffer.readGCDAFormat())
1549 return sampleprof_error::unrecognized_format;
1550
1551 // Read the version number. Note - the GCC reader does not validate this
1552 // version, but the profile creator generates v704.
1553 GCOV::GCOVVersion version;
1554 if (!GcovBuffer.readGCOVVersion(version))
1555 return sampleprof_error::unrecognized_format;
1556
1557 if (version != GCOV::V407)
1558 return sampleprof_error::unsupported_version;
1559
1560 // Skip the empty integer.
1561 if (std::error_code EC = skipNextWord())
1562 return EC;
1563
1564 return sampleprof_error::success;
1565}
1566
1567std::error_code SampleProfileReaderGCC::readSectionTag(uint32_t Expected) {
1568 uint32_t Tag;
1569 if (!GcovBuffer.readInt(Val&: Tag))
1570 return sampleprof_error::truncated;
1571
1572 if (Tag != Expected)
1573 return sampleprof_error::malformed;
1574
1575 if (std::error_code EC = skipNextWord())
1576 return EC;
1577
1578 return sampleprof_error::success;
1579}
1580
1581std::error_code SampleProfileReaderGCC::readNameTable() {
1582 if (std::error_code EC = readSectionTag(Expected: GCOVTagAFDOFileNames))
1583 return EC;
1584
1585 uint32_t Size;
1586 if (!GcovBuffer.readInt(Val&: Size))
1587 return sampleprof_error::truncated;
1588
1589 for (uint32_t I = 0; I < Size; ++I) {
1590 StringRef Str;
1591 if (!GcovBuffer.readString(str&: Str))
1592 return sampleprof_error::truncated;
1593 Names.push_back(x: std::string(Str));
1594 }
1595
1596 return sampleprof_error::success;
1597}
1598
1599std::error_code SampleProfileReaderGCC::readFunctionProfiles() {
1600 if (std::error_code EC = readSectionTag(Expected: GCOVTagAFDOFunction))
1601 return EC;
1602
1603 uint32_t NumFunctions;
1604 if (!GcovBuffer.readInt(Val&: NumFunctions))
1605 return sampleprof_error::truncated;
1606
1607 InlineCallStack Stack;
1608 for (uint32_t I = 0; I < NumFunctions; ++I)
1609 if (std::error_code EC = readOneFunctionProfile(InlineStack: Stack, Update: true, Offset: 0))
1610 return EC;
1611
1612 computeSummary();
1613 return sampleprof_error::success;
1614}
1615
1616std::error_code SampleProfileReaderGCC::readOneFunctionProfile(
1617 const InlineCallStack &InlineStack, bool Update, uint32_t Offset) {
1618 uint64_t HeadCount = 0;
1619 if (InlineStack.size() == 0)
1620 if (!GcovBuffer.readInt64(Val&: HeadCount))
1621 return sampleprof_error::truncated;
1622
1623 uint32_t NameIdx;
1624 if (!GcovBuffer.readInt(Val&: NameIdx))
1625 return sampleprof_error::truncated;
1626
1627 StringRef Name(Names[NameIdx]);
1628
1629 uint32_t NumPosCounts;
1630 if (!GcovBuffer.readInt(Val&: NumPosCounts))
1631 return sampleprof_error::truncated;
1632
1633 uint32_t NumCallsites;
1634 if (!GcovBuffer.readInt(Val&: NumCallsites))
1635 return sampleprof_error::truncated;
1636
1637 FunctionSamples *FProfile = nullptr;
1638 if (InlineStack.size() == 0) {
1639 // If this is a top function that we have already processed, do not
1640 // update its profile again. This happens in the presence of
1641 // function aliases. Since these aliases share the same function
1642 // body, there will be identical replicated profiles for the
1643 // original function. In this case, we simply not bother updating
1644 // the profile of the original function.
1645 FProfile = &Profiles[FunctionId(Name)];
1646 FProfile->addHeadSamples(Num: HeadCount);
1647 if (FProfile->getTotalSamples() > 0)
1648 Update = false;
1649 } else {
1650 // Otherwise, we are reading an inlined instance. The top of the
1651 // inline stack contains the profile of the caller. Insert this
1652 // callee in the caller's CallsiteMap.
1653 FunctionSamples *CallerProfile = InlineStack.front();
1654 uint32_t LineOffset = Offset >> 16;
1655 uint32_t Discriminator = Offset & 0xffff;
1656 FProfile = &CallerProfile->functionSamplesAt(
1657 Loc: LineLocation(LineOffset, Discriminator))[FunctionId(Name)];
1658 }
1659 FProfile->setFunction(FunctionId(Name));
1660
1661 for (uint32_t I = 0; I < NumPosCounts; ++I) {
1662 uint32_t Offset;
1663 if (!GcovBuffer.readInt(Val&: Offset))
1664 return sampleprof_error::truncated;
1665
1666 uint32_t NumTargets;
1667 if (!GcovBuffer.readInt(Val&: NumTargets))
1668 return sampleprof_error::truncated;
1669
1670 uint64_t Count;
1671 if (!GcovBuffer.readInt64(Val&: Count))
1672 return sampleprof_error::truncated;
1673
1674 // The line location is encoded in the offset as:
1675 // high 16 bits: line offset to the start of the function.
1676 // low 16 bits: discriminator.
1677 uint32_t LineOffset = Offset >> 16;
1678 uint32_t Discriminator = Offset & 0xffff;
1679
1680 InlineCallStack NewStack;
1681 NewStack.push_back(Elt: FProfile);
1682 llvm::append_range(C&: NewStack, R: InlineStack);
1683 if (Update) {
1684 // Walk up the inline stack, adding the samples on this line to
1685 // the total sample count of the callers in the chain.
1686 for (auto *CallerProfile : NewStack)
1687 CallerProfile->addTotalSamples(Num: Count);
1688
1689 // Update the body samples for the current profile.
1690 FProfile->addBodySamples(LineOffset, Discriminator, Num: Count);
1691 }
1692
1693 // Process the list of functions called at an indirect call site.
1694 // These are all the targets that a function pointer (or virtual
1695 // function) resolved at runtime.
1696 for (uint32_t J = 0; J < NumTargets; J++) {
1697 uint32_t HistVal;
1698 if (!GcovBuffer.readInt(Val&: HistVal))
1699 return sampleprof_error::truncated;
1700
1701 if (HistVal != HIST_TYPE_INDIR_CALL_TOPN)
1702 return sampleprof_error::malformed;
1703
1704 uint64_t TargetIdx;
1705 if (!GcovBuffer.readInt64(Val&: TargetIdx))
1706 return sampleprof_error::truncated;
1707 StringRef TargetName(Names[TargetIdx]);
1708
1709 uint64_t TargetCount;
1710 if (!GcovBuffer.readInt64(Val&: TargetCount))
1711 return sampleprof_error::truncated;
1712
1713 if (Update)
1714 FProfile->addCalledTargetSamples(LineOffset, Discriminator,
1715 Func: FunctionId(TargetName),
1716 Num: TargetCount);
1717 }
1718 }
1719
1720 // Process all the inlined callers into the current function. These
1721 // are all the callsites that were inlined into this function.
1722 for (uint32_t I = 0; I < NumCallsites; I++) {
1723 // The offset is encoded as:
1724 // high 16 bits: line offset to the start of the function.
1725 // low 16 bits: discriminator.
1726 uint32_t Offset;
1727 if (!GcovBuffer.readInt(Val&: Offset))
1728 return sampleprof_error::truncated;
1729 InlineCallStack NewStack;
1730 NewStack.push_back(Elt: FProfile);
1731 llvm::append_range(C&: NewStack, R: InlineStack);
1732 if (std::error_code EC = readOneFunctionProfile(InlineStack: NewStack, Update, Offset))
1733 return EC;
1734 }
1735
1736 return sampleprof_error::success;
1737}
1738
1739/// Read a GCC AutoFDO profile.
1740///
1741/// This format is generated by the Linux Perf conversion tool at
1742/// https://github.com/google/autofdo.
1743std::error_code SampleProfileReaderGCC::readImpl() {
1744 assert(!ProfileIsFSDisciminator && "Gcc profiles not support FSDisciminator");
1745 // Read the string table.
1746 if (std::error_code EC = readNameTable())
1747 return EC;
1748
1749 // Read the source profile.
1750 if (std::error_code EC = readFunctionProfiles())
1751 return EC;
1752
1753 return sampleprof_error::success;
1754}
1755
1756bool SampleProfileReaderGCC::hasFormat(const MemoryBuffer &Buffer) {
1757 StringRef Magic(reinterpret_cast<const char *>(Buffer.getBufferStart()));
1758 return Magic == "adcg*704";
1759}
1760
1761void SampleProfileReaderItaniumRemapper::applyRemapping(LLVMContext &Ctx) {
1762 // If the reader uses MD5 to represent string, we can't remap it because
1763 // we don't know what the original function names were.
1764 if (Reader.useMD5()) {
1765 Ctx.diagnose(DI: DiagnosticInfoSampleProfile(
1766 Reader.getBuffer()->getBufferIdentifier(),
1767 "Profile data remapping cannot be applied to profile data "
1768 "using MD5 names (original mangled names are not available).",
1769 DS_Warning));
1770 return;
1771 }
1772
1773 // CSSPGO-TODO: Remapper is not yet supported.
1774 // We will need to remap the entire context string.
1775 assert(Remappings && "should be initialized while creating remapper");
1776 for (auto &Sample : Reader.getProfiles()) {
1777 DenseSet<FunctionId> NamesInSample;
1778 Sample.second.findAllNames(NameSet&: NamesInSample);
1779 for (auto &Name : NamesInSample) {
1780 StringRef NameStr = Name.stringRef();
1781 if (auto Key = Remappings->insert(FunctionName: NameStr))
1782 NameMap.insert(KV: {Key, NameStr});
1783 }
1784 }
1785
1786 RemappingApplied = true;
1787}
1788
1789std::optional<StringRef>
1790SampleProfileReaderItaniumRemapper::lookUpNameInProfile(StringRef Fname) {
1791 if (auto Key = Remappings->lookup(FunctionName: Fname)) {
1792 StringRef Result = NameMap.lookup(Val: Key);
1793 if (!Result.empty())
1794 return Result;
1795 }
1796 return std::nullopt;
1797}
1798
1799/// Prepare a memory buffer for the contents of \p Filename.
1800///
1801/// \returns an error code indicating the status of the buffer.
1802static ErrorOr<std::unique_ptr<MemoryBuffer>>
1803setupMemoryBuffer(const Twine &Filename, vfs::FileSystem &FS) {
1804 auto BufferOrErr = Filename.str() == "-" ? MemoryBuffer::getSTDIN()
1805 : FS.getBufferForFile(Name: Filename);
1806 if (std::error_code EC = BufferOrErr.getError())
1807 return EC;
1808 auto Buffer = std::move(BufferOrErr.get());
1809
1810 return std::move(Buffer);
1811}
1812
1813/// Create a sample profile reader based on the format of the input file.
1814///
1815/// \param Filename The file to open.
1816///
1817/// \param C The LLVM context to use to emit diagnostics.
1818///
1819/// \param P The FSDiscriminatorPass.
1820///
1821/// \param RemapFilename The file used for profile remapping.
1822///
1823/// \returns an error code indicating the status of the created reader.
1824ErrorOr<std::unique_ptr<SampleProfileReader>>
1825SampleProfileReader::create(const std::string Filename, LLVMContext &C,
1826 vfs::FileSystem &FS, FSDiscriminatorPass P,
1827 const std::string RemapFilename) {
1828 auto BufferOrError = setupMemoryBuffer(Filename, FS);
1829 if (std::error_code EC = BufferOrError.getError())
1830 return EC;
1831 return create(B&: BufferOrError.get(), C, FS, P, RemapFilename);
1832}
1833
1834/// Create a sample profile remapper from the given input, to remap the
1835/// function names in the given profile data.
1836///
1837/// \param Filename The file to open.
1838///
1839/// \param Reader The profile reader the remapper is going to be applied to.
1840///
1841/// \param C The LLVM context to use to emit diagnostics.
1842///
1843/// \returns an error code indicating the status of the created reader.
1844ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>>
1845SampleProfileReaderItaniumRemapper::create(const std::string Filename,
1846 vfs::FileSystem &FS,
1847 SampleProfileReader &Reader,
1848 LLVMContext &C) {
1849 auto BufferOrError = setupMemoryBuffer(Filename, FS);
1850 if (std::error_code EC = BufferOrError.getError())
1851 return EC;
1852 return create(B&: BufferOrError.get(), Reader, C);
1853}
1854
1855/// Create a sample profile remapper from the given input, to remap the
1856/// function names in the given profile data.
1857///
1858/// \param B The memory buffer to create the reader from (assumes ownership).
1859///
1860/// \param C The LLVM context to use to emit diagnostics.
1861///
1862/// \param Reader The profile reader the remapper is going to be applied to.
1863///
1864/// \returns an error code indicating the status of the created reader.
1865ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>>
1866SampleProfileReaderItaniumRemapper::create(std::unique_ptr<MemoryBuffer> &B,
1867 SampleProfileReader &Reader,
1868 LLVMContext &C) {
1869 auto Remappings = std::make_unique<SymbolRemappingReader>();
1870 if (Error E = Remappings->read(B&: *B)) {
1871 handleAllErrors(
1872 E: std::move(E), Handlers: [&](const SymbolRemappingParseError &ParseError) {
1873 C.diagnose(DI: DiagnosticInfoSampleProfile(B->getBufferIdentifier(),
1874 ParseError.getLineNum(),
1875 ParseError.getMessage()));
1876 });
1877 return sampleprof_error::malformed;
1878 }
1879
1880 return std::make_unique<SampleProfileReaderItaniumRemapper>(
1881 args: std::move(B), args: std::move(Remappings), args&: Reader);
1882}
1883
1884/// Create a sample profile reader based on the format of the input data.
1885///
1886/// \param B The memory buffer to create the reader from (assumes ownership).
1887///
1888/// \param C The LLVM context to use to emit diagnostics.
1889///
1890/// \param P The FSDiscriminatorPass.
1891///
1892/// \param RemapFilename The file used for profile remapping.
1893///
1894/// \returns an error code indicating the status of the created reader.
1895ErrorOr<std::unique_ptr<SampleProfileReader>>
1896SampleProfileReader::create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C,
1897 vfs::FileSystem &FS, FSDiscriminatorPass P,
1898 const std::string RemapFilename) {
1899 std::unique_ptr<SampleProfileReader> Reader;
1900 if (SampleProfileReaderRawBinary::hasFormat(Buffer: *B))
1901 Reader.reset(p: new SampleProfileReaderRawBinary(std::move(B), C));
1902 else if (SampleProfileReaderExtBinary::hasFormat(Buffer: *B))
1903 Reader.reset(p: new SampleProfileReaderExtBinary(std::move(B), C));
1904 else if (SampleProfileReaderGCC::hasFormat(Buffer: *B))
1905 Reader.reset(p: new SampleProfileReaderGCC(std::move(B), C));
1906 else if (SampleProfileReaderText::hasFormat(Buffer: *B))
1907 Reader.reset(p: new SampleProfileReaderText(std::move(B), C));
1908 else
1909 return sampleprof_error::unrecognized_format;
1910
1911 if (!RemapFilename.empty()) {
1912 auto ReaderOrErr = SampleProfileReaderItaniumRemapper::create(
1913 Filename: RemapFilename, FS, Reader&: *Reader, C);
1914 if (std::error_code EC = ReaderOrErr.getError()) {
1915 std::string Msg = "Could not create remapper: " + EC.message();
1916 C.diagnose(DI: DiagnosticInfoSampleProfile(RemapFilename, Msg));
1917 return EC;
1918 }
1919 Reader->Remapper = std::move(ReaderOrErr.get());
1920 }
1921
1922 if (std::error_code EC = Reader->readHeader()) {
1923 return EC;
1924 }
1925
1926 Reader->setDiscriminatorMaskedBitFrom(P);
1927
1928 return std::move(Reader);
1929}
1930
1931// For text and GCC file formats, we compute the summary after reading the
1932// profile. Binary format has the profile summary in its header.
1933void SampleProfileReader::computeSummary() {
1934 SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
1935 Summary = Builder.computeSummaryForProfiles(Profiles);
1936}
1937

source code of llvm/lib/ProfileData/SampleProfReader.cpp