1 | //===- lib/MC/MCPseudoProbe.cpp - Pseudo probe encoding support ----------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | |
9 | #include "llvm/MC/MCPseudoProbe.h" |
10 | #include "llvm/ADT/STLExtras.h" |
11 | #include "llvm/IR/PseudoProbe.h" |
12 | #include "llvm/MC/MCAsmInfo.h" |
13 | #include "llvm/MC/MCAssembler.h" |
14 | #include "llvm/MC/MCContext.h" |
15 | #include "llvm/MC/MCExpr.h" |
16 | #include "llvm/MC/MCFragment.h" |
17 | #include "llvm/MC/MCObjectFileInfo.h" |
18 | #include "llvm/MC/MCObjectStreamer.h" |
19 | #include "llvm/MC/MCSymbol.h" |
20 | #include "llvm/Support/Endian.h" |
21 | #include "llvm/Support/LEB128.h" |
22 | #include "llvm/Support/MD5.h" |
23 | #include "llvm/Support/raw_ostream.h" |
24 | #include <algorithm> |
25 | #include <cassert> |
26 | #include <limits> |
27 | #include <memory> |
28 | #include <sstream> |
29 | #include <vector> |
30 | |
31 | #define DEBUG_TYPE "mcpseudoprobe" |
32 | |
33 | using namespace llvm; |
34 | using namespace support; |
35 | |
36 | #ifndef NDEBUG |
37 | int MCPseudoProbeTable::DdgPrintIndent = 0; |
38 | #endif |
39 | |
40 | static const MCExpr *buildSymbolDiff(MCObjectStreamer *MCOS, const MCSymbol *A, |
41 | const MCSymbol *B) { |
42 | MCContext &Context = MCOS->getContext(); |
43 | MCSymbolRefExpr::VariantKind Variant = MCSymbolRefExpr::VK_None; |
44 | const MCExpr *ARef = MCSymbolRefExpr::create(Symbol: A, Kind: Variant, Ctx&: Context); |
45 | const MCExpr *BRef = MCSymbolRefExpr::create(Symbol: B, Kind: Variant, Ctx&: Context); |
46 | const MCExpr *AddrDelta = |
47 | MCBinaryExpr::create(Op: MCBinaryExpr::Sub, LHS: ARef, RHS: BRef, Ctx&: Context); |
48 | return AddrDelta; |
49 | } |
50 | |
51 | void MCPseudoProbe::emit(MCObjectStreamer *MCOS, |
52 | const MCPseudoProbe *LastProbe) const { |
53 | bool IsSentinel = isSentinelProbe(Flags: getAttributes()); |
54 | assert((LastProbe || IsSentinel) && |
55 | "Last probe should not be null for non-sentinel probes" ); |
56 | |
57 | // Emit Index |
58 | MCOS->emitULEB128IntValue(Value: Index); |
59 | // Emit Type and the flag: |
60 | // Type (bit 0 to 3), with bit 4 to 6 for attributes. |
61 | // Flag (bit 7, 0 - code address, 1 - address delta). This indicates whether |
62 | // the following field is a symbolic code address or an address delta. |
63 | // Emit FS discriminator |
64 | assert(Type <= 0xF && "Probe type too big to encode, exceeding 15" ); |
65 | auto NewAttributes = Attributes; |
66 | if (Discriminator) |
67 | NewAttributes |= (uint32_t)PseudoProbeAttributes::HasDiscriminator; |
68 | assert(NewAttributes <= 0x7 && |
69 | "Probe attributes too big to encode, exceeding 7" ); |
70 | uint8_t PackedType = Type | (NewAttributes << 4); |
71 | uint8_t Flag = |
72 | !IsSentinel ? ((int8_t)MCPseudoProbeFlag::AddressDelta << 7) : 0; |
73 | MCOS->emitInt8(Value: Flag | PackedType); |
74 | |
75 | if (!IsSentinel) { |
76 | // Emit the delta between the address label and LastProbe. |
77 | const MCExpr *AddrDelta = |
78 | buildSymbolDiff(MCOS, A: Label, B: LastProbe->getLabel()); |
79 | int64_t Delta; |
80 | if (AddrDelta->evaluateAsAbsolute(Res&: Delta, Asm: MCOS->getAssemblerPtr())) { |
81 | MCOS->emitSLEB128IntValue(Value: Delta); |
82 | } else { |
83 | MCOS->insert(F: new MCPseudoProbeAddrFragment(AddrDelta)); |
84 | } |
85 | } else { |
86 | // Emit the GUID of the split function that the sentinel probe represents. |
87 | MCOS->emitInt64(Value: Guid); |
88 | } |
89 | |
90 | if (Discriminator) |
91 | MCOS->emitULEB128IntValue(Value: Discriminator); |
92 | |
93 | LLVM_DEBUG({ |
94 | dbgs().indent(MCPseudoProbeTable::DdgPrintIndent); |
95 | dbgs() << "Probe: " << Index << "\n" ; |
96 | }); |
97 | } |
98 | |
99 | void MCPseudoProbeInlineTree::addPseudoProbe( |
100 | const MCPseudoProbe &Probe, const MCPseudoProbeInlineStack &InlineStack) { |
101 | // The function should not be called on the root. |
102 | assert(isRoot() && "Should only be called on root" ); |
103 | |
104 | // When it comes here, the input look like: |
105 | // Probe: GUID of C, ... |
106 | // InlineStack: [88, A], [66, B] |
107 | // which means, Function A inlines function B at call site with a probe id of |
108 | // 88, and B inlines C at probe 66. The tri-tree expects a tree path like {[0, |
109 | // A], [88, B], [66, C]} to locate the tree node where the probe should be |
110 | // added. Note that the edge [0, A] means A is the top-level function we are |
111 | // emitting probes for. |
112 | |
113 | // Make a [0, A] edge. |
114 | // An empty inline stack means the function that the probe originates from |
115 | // is a top-level function. |
116 | InlineSite Top; |
117 | if (InlineStack.empty()) { |
118 | Top = InlineSite(Probe.getGuid(), 0); |
119 | } else { |
120 | Top = InlineSite(std::get<0>(t: InlineStack.front()), 0); |
121 | } |
122 | |
123 | auto *Cur = getOrAddNode(Site: Top); |
124 | |
125 | // Make interior edges by walking the inline stack. Once it's done, Cur should |
126 | // point to the node that the probe originates from. |
127 | if (!InlineStack.empty()) { |
128 | auto Iter = InlineStack.begin(); |
129 | auto Index = std::get<1>(t: *Iter); |
130 | Iter++; |
131 | for (; Iter != InlineStack.end(); Iter++) { |
132 | // Make an edge by using the previous probe id and current GUID. |
133 | Cur = Cur->getOrAddNode(Site: InlineSite(std::get<0>(t: *Iter), Index)); |
134 | Index = std::get<1>(t: *Iter); |
135 | } |
136 | Cur = Cur->getOrAddNode(Site: InlineSite(Probe.getGuid(), Index)); |
137 | } |
138 | |
139 | Cur->Probes.push_back(x: Probe); |
140 | } |
141 | |
142 | void MCPseudoProbeInlineTree::emit(MCObjectStreamer *MCOS, |
143 | const MCPseudoProbe *&LastProbe) { |
144 | LLVM_DEBUG({ |
145 | dbgs().indent(MCPseudoProbeTable::DdgPrintIndent); |
146 | dbgs() << "Group [\n" ; |
147 | MCPseudoProbeTable::DdgPrintIndent += 2; |
148 | }); |
149 | assert(!isRoot() && "Root should be handled seperately" ); |
150 | |
151 | // Emit probes grouped by GUID. |
152 | LLVM_DEBUG({ |
153 | dbgs().indent(MCPseudoProbeTable::DdgPrintIndent); |
154 | dbgs() << "GUID: " << Guid << "\n" ; |
155 | }); |
156 | // Emit Guid |
157 | MCOS->emitInt64(Value: Guid); |
158 | // Emit number of probes in this node, including a sentinel probe for |
159 | // top-level functions if needed. |
160 | bool NeedSentinel = false; |
161 | if (Parent->isRoot()) { |
162 | assert(isSentinelProbe(LastProbe->getAttributes()) && |
163 | "Starting probe of a top-level function should be a sentinel probe" ); |
164 | // The main body of a split function doesn't need a sentinel probe. |
165 | if (LastProbe->getGuid() != Guid) |
166 | NeedSentinel = true; |
167 | } |
168 | |
169 | MCOS->emitULEB128IntValue(Value: Probes.size() + NeedSentinel); |
170 | // Emit number of direct inlinees |
171 | MCOS->emitULEB128IntValue(Value: Children.size()); |
172 | // Emit sentinel probe for top-level functions |
173 | if (NeedSentinel) |
174 | LastProbe->emit(MCOS, LastProbe: nullptr); |
175 | |
176 | // Emit probes in this group |
177 | for (const auto &Probe : Probes) { |
178 | Probe.emit(MCOS, LastProbe); |
179 | LastProbe = &Probe; |
180 | } |
181 | |
182 | // Emit sorted descendant. InlineSite is unique for each pair, so there will |
183 | // be no ordering of Inlinee based on MCPseudoProbeInlineTree* |
184 | using InlineeType = std::pair<InlineSite, MCPseudoProbeInlineTree *>; |
185 | auto Comparer = [](const InlineeType &A, const InlineeType &B) { |
186 | return A.first < B.first; |
187 | }; |
188 | std::vector<InlineeType> Inlinees; |
189 | for (const auto &Child : Children) |
190 | Inlinees.emplace_back(args: Child.first, args: Child.second.get()); |
191 | std::sort(first: Inlinees.begin(), last: Inlinees.end(), comp: Comparer); |
192 | |
193 | for (const auto &Inlinee : Inlinees) { |
194 | // Emit probe index |
195 | MCOS->emitULEB128IntValue(Value: std::get<1>(t: Inlinee.first)); |
196 | LLVM_DEBUG({ |
197 | dbgs().indent(MCPseudoProbeTable::DdgPrintIndent); |
198 | dbgs() << "InlineSite: " << std::get<1>(Inlinee.first) << "\n" ; |
199 | }); |
200 | // Emit the group |
201 | Inlinee.second->emit(MCOS, LastProbe); |
202 | } |
203 | |
204 | LLVM_DEBUG({ |
205 | MCPseudoProbeTable::DdgPrintIndent -= 2; |
206 | dbgs().indent(MCPseudoProbeTable::DdgPrintIndent); |
207 | dbgs() << "]\n" ; |
208 | }); |
209 | } |
210 | |
211 | void MCPseudoProbeSections::emit(MCObjectStreamer *MCOS) { |
212 | MCContext &Ctx = MCOS->getContext(); |
213 | SmallVector<std::pair<MCSymbol *, MCPseudoProbeInlineTree *>> Vec; |
214 | Vec.reserve(N: MCProbeDivisions.size()); |
215 | for (auto &ProbeSec : MCProbeDivisions) |
216 | Vec.emplace_back(Args: ProbeSec.first, Args: &ProbeSec.second); |
217 | for (auto I : llvm::enumerate(First&: MCOS->getAssembler())) |
218 | I.value().setOrdinal(I.index()); |
219 | llvm::sort(C&: Vec, Comp: [](auto A, auto B) { |
220 | return A.first->getSection().getOrdinal() < |
221 | B.first->getSection().getOrdinal(); |
222 | }); |
223 | for (auto [FuncSym, RootPtr] : Vec) { |
224 | const auto &Root = *RootPtr; |
225 | if (auto *S = Ctx.getObjectFileInfo()->getPseudoProbeSection( |
226 | TextSec: FuncSym->getSection())) { |
227 | // Switch to the .pseudoprobe section or a comdat group. |
228 | MCOS->switchSection(Section: S); |
229 | // Emit probes grouped by GUID. |
230 | // Emit sorted descendant. InlineSite is unique for each pair, so there |
231 | // will be no ordering of Inlinee based on MCPseudoProbeInlineTree* |
232 | using InlineeType = std::pair<InlineSite, MCPseudoProbeInlineTree *>; |
233 | auto Comparer = [](const InlineeType &A, const InlineeType &B) { |
234 | return A.first < B.first; |
235 | }; |
236 | std::vector<InlineeType> Inlinees; |
237 | for (const auto &Child : Root.getChildren()) |
238 | Inlinees.emplace_back(args: Child.first, args: Child.second.get()); |
239 | std::sort(first: Inlinees.begin(), last: Inlinees.end(), comp: Comparer); |
240 | |
241 | for (const auto &Inlinee : Inlinees) { |
242 | // Emit the group guarded by a sentinel probe. |
243 | MCPseudoProbe SentinelProbe( |
244 | const_cast<MCSymbol *>(FuncSym), MD5Hash(Str: FuncSym->getName()), |
245 | (uint32_t)PseudoProbeReservedId::Invalid, |
246 | (uint32_t)PseudoProbeType::Block, |
247 | (uint32_t)PseudoProbeAttributes::Sentinel, 0); |
248 | const MCPseudoProbe *Probe = &SentinelProbe; |
249 | Inlinee.second->emit(MCOS, LastProbe&: Probe); |
250 | } |
251 | } |
252 | } |
253 | } |
254 | |
255 | // |
256 | // This emits the pseudo probe tables. |
257 | // |
258 | void MCPseudoProbeTable::emit(MCObjectStreamer *MCOS) { |
259 | MCContext &Ctx = MCOS->getContext(); |
260 | auto &ProbeTable = Ctx.getMCPseudoProbeTable(); |
261 | |
262 | // Bail out early so we don't switch to the pseudo_probe section needlessly |
263 | // and in doing so create an unnecessary (if empty) section. |
264 | auto &ProbeSections = ProbeTable.getProbeSections(); |
265 | if (ProbeSections.empty()) |
266 | return; |
267 | |
268 | LLVM_DEBUG(MCPseudoProbeTable::DdgPrintIndent = 0); |
269 | |
270 | // Put out the probe. |
271 | ProbeSections.emit(MCOS); |
272 | } |
273 | |
274 | static StringRef getProbeFNameForGUID(const GUIDProbeFunctionMap &GUID2FuncMAP, |
275 | uint64_t GUID) { |
276 | auto It = GUID2FuncMAP.find(x: GUID); |
277 | assert(It != GUID2FuncMAP.end() && |
278 | "Probe function must exist for a valid GUID" ); |
279 | return It->second.FuncName; |
280 | } |
281 | |
282 | void MCPseudoProbeFuncDesc::print(raw_ostream &OS) { |
283 | OS << "GUID: " << FuncGUID << " Name: " << FuncName << "\n" ; |
284 | OS << "Hash: " << FuncHash << "\n" ; |
285 | } |
286 | |
287 | void MCDecodedPseudoProbe::getInlineContext( |
288 | SmallVectorImpl<MCPseduoProbeFrameLocation> &ContextStack, |
289 | const GUIDProbeFunctionMap &GUID2FuncMAP) const { |
290 | uint32_t Begin = ContextStack.size(); |
291 | MCDecodedPseudoProbeInlineTree *Cur = InlineTree; |
292 | // It will add the string of each node's inline site during iteration. |
293 | // Note that it won't include the probe's belonging function(leaf location) |
294 | while (Cur->hasInlineSite()) { |
295 | StringRef FuncName = getProbeFNameForGUID(GUID2FuncMAP, GUID: Cur->Parent->Guid); |
296 | ContextStack.emplace_back( |
297 | Args: MCPseduoProbeFrameLocation(FuncName, std::get<1>(t&: Cur->ISite))); |
298 | Cur = static_cast<MCDecodedPseudoProbeInlineTree *>(Cur->Parent); |
299 | } |
300 | // Make the ContextStack in caller-callee order |
301 | std::reverse(first: ContextStack.begin() + Begin, last: ContextStack.end()); |
302 | } |
303 | |
304 | std::string MCDecodedPseudoProbe::getInlineContextStr( |
305 | const GUIDProbeFunctionMap &GUID2FuncMAP) const { |
306 | std::ostringstream OContextStr; |
307 | SmallVector<MCPseduoProbeFrameLocation, 16> ContextStack; |
308 | getInlineContext(ContextStack, GUID2FuncMAP); |
309 | for (auto &Cxt : ContextStack) { |
310 | if (OContextStr.str().size()) |
311 | OContextStr << " @ " ; |
312 | OContextStr << Cxt.first.str() << ":" << Cxt.second; |
313 | } |
314 | return OContextStr.str(); |
315 | } |
316 | |
317 | static const char *PseudoProbeTypeStr[3] = {"Block" , "IndirectCall" , |
318 | "DirectCall" }; |
319 | |
320 | void MCDecodedPseudoProbe::print(raw_ostream &OS, |
321 | const GUIDProbeFunctionMap &GUID2FuncMAP, |
322 | bool ShowName) const { |
323 | OS << "FUNC: " ; |
324 | if (ShowName) { |
325 | StringRef FuncName = getProbeFNameForGUID(GUID2FuncMAP, GUID: Guid); |
326 | OS << FuncName.str() << " " ; |
327 | } else { |
328 | OS << Guid << " " ; |
329 | } |
330 | OS << "Index: " << Index << " " ; |
331 | if (Discriminator) |
332 | OS << "Discriminator: " << Discriminator << " " ; |
333 | OS << "Type: " << PseudoProbeTypeStr[static_cast<uint8_t>(Type)] << " " ; |
334 | std::string InlineContextStr = getInlineContextStr(GUID2FuncMAP); |
335 | if (InlineContextStr.size()) { |
336 | OS << "Inlined: @ " ; |
337 | OS << InlineContextStr; |
338 | } |
339 | OS << "\n" ; |
340 | } |
341 | |
342 | template <typename T> ErrorOr<T> MCPseudoProbeDecoder::readUnencodedNumber() { |
343 | if (Data + sizeof(T) > End) { |
344 | return std::error_code(); |
345 | } |
346 | T Val = endian::readNext<T, llvm::endianness::little, unaligned>(Data); |
347 | return ErrorOr<T>(Val); |
348 | } |
349 | |
350 | template <typename T> ErrorOr<T> MCPseudoProbeDecoder::readUnsignedNumber() { |
351 | unsigned NumBytesRead = 0; |
352 | uint64_t Val = decodeULEB128(p: Data, n: &NumBytesRead); |
353 | if (Val > std::numeric_limits<T>::max() || (Data + NumBytesRead > End)) { |
354 | return std::error_code(); |
355 | } |
356 | Data += NumBytesRead; |
357 | return ErrorOr<T>(static_cast<T>(Val)); |
358 | } |
359 | |
360 | template <typename T> ErrorOr<T> MCPseudoProbeDecoder::readSignedNumber() { |
361 | unsigned NumBytesRead = 0; |
362 | int64_t Val = decodeSLEB128(p: Data, n: &NumBytesRead); |
363 | if (Val > std::numeric_limits<T>::max() || (Data + NumBytesRead > End)) { |
364 | return std::error_code(); |
365 | } |
366 | Data += NumBytesRead; |
367 | return ErrorOr<T>(static_cast<T>(Val)); |
368 | } |
369 | |
370 | ErrorOr<StringRef> MCPseudoProbeDecoder::readString(uint32_t Size) { |
371 | StringRef Str(reinterpret_cast<const char *>(Data), Size); |
372 | if (Data + Size > End) { |
373 | return std::error_code(); |
374 | } |
375 | Data += Size; |
376 | return ErrorOr<StringRef>(Str); |
377 | } |
378 | |
379 | bool MCPseudoProbeDecoder::buildGUID2FuncDescMap(const uint8_t *Start, |
380 | std::size_t Size) { |
381 | // The pseudo_probe_desc section has a format like: |
382 | // .section .pseudo_probe_desc,"",@progbits |
383 | // .quad -5182264717993193164 // GUID |
384 | // .quad 4294967295 // Hash |
385 | // .uleb 3 // Name size |
386 | // .ascii "foo" // Name |
387 | // .quad -2624081020897602054 |
388 | // .quad 174696971957 |
389 | // .uleb 34 |
390 | // .ascii "main" |
391 | |
392 | Data = Start; |
393 | End = Data + Size; |
394 | |
395 | while (Data < End) { |
396 | auto ErrorOrGUID = readUnencodedNumber<uint64_t>(); |
397 | if (!ErrorOrGUID) |
398 | return false; |
399 | |
400 | auto ErrorOrHash = readUnencodedNumber<uint64_t>(); |
401 | if (!ErrorOrHash) |
402 | return false; |
403 | |
404 | auto ErrorOrNameSize = readUnsignedNumber<uint32_t>(); |
405 | if (!ErrorOrNameSize) |
406 | return false; |
407 | uint32_t NameSize = std::move(*ErrorOrNameSize); |
408 | |
409 | auto ErrorOrName = readString(Size: NameSize); |
410 | if (!ErrorOrName) |
411 | return false; |
412 | |
413 | uint64_t GUID = std::move(*ErrorOrGUID); |
414 | uint64_t Hash = std::move(*ErrorOrHash); |
415 | StringRef Name = std::move(*ErrorOrName); |
416 | |
417 | // Initialize PseudoProbeFuncDesc and populate it into GUID2FuncDescMap |
418 | GUID2FuncDescMap.emplace(args&: GUID, args: MCPseudoProbeFuncDesc(GUID, Hash, Name)); |
419 | } |
420 | assert(Data == End && "Have unprocessed data in pseudo_probe_desc section" ); |
421 | return true; |
422 | } |
423 | |
424 | bool MCPseudoProbeDecoder::buildAddress2ProbeMap( |
425 | MCDecodedPseudoProbeInlineTree *Cur, uint64_t &LastAddr, |
426 | const Uint64Set &GuidFilter, const Uint64Map &FuncStartAddrs) { |
427 | // The pseudo_probe section encodes an inline forest and each tree has a |
428 | // format defined in MCPseudoProbe.h |
429 | |
430 | uint32_t Index = 0; |
431 | bool IsTopLevelFunc = Cur == &DummyInlineRoot; |
432 | if (IsTopLevelFunc) { |
433 | // Use a sequential id for top level inliner. |
434 | Index = Cur->getChildren().size(); |
435 | } else { |
436 | // Read inline site for inlinees |
437 | auto ErrorOrIndex = readUnsignedNumber<uint32_t>(); |
438 | if (!ErrorOrIndex) |
439 | return false; |
440 | Index = std::move(*ErrorOrIndex); |
441 | } |
442 | |
443 | // Read guid |
444 | auto ErrorOrCurGuid = readUnencodedNumber<uint64_t>(); |
445 | if (!ErrorOrCurGuid) |
446 | return false; |
447 | uint64_t Guid = std::move(*ErrorOrCurGuid); |
448 | |
449 | // Decide if top-level node should be disgarded. |
450 | if (IsTopLevelFunc && !GuidFilter.empty() && !GuidFilter.count(V: Guid)) |
451 | Cur = nullptr; |
452 | |
453 | // If the incoming node is null, all its children nodes should be disgarded. |
454 | if (Cur) { |
455 | // Switch/add to a new tree node(inlinee) |
456 | Cur = Cur->getOrAddNode(Site: std::make_tuple(args&: Guid, args&: Index)); |
457 | Cur->Guid = Guid; |
458 | if (IsTopLevelFunc && !EncodingIsAddrBased) { |
459 | if (auto V = FuncStartAddrs.lookup(Val: Guid)) |
460 | LastAddr = V; |
461 | } |
462 | } |
463 | |
464 | // Read number of probes in the current node. |
465 | auto ErrorOrNodeCount = readUnsignedNumber<uint32_t>(); |
466 | if (!ErrorOrNodeCount) |
467 | return false; |
468 | uint32_t NodeCount = std::move(*ErrorOrNodeCount); |
469 | // Read number of direct inlinees |
470 | auto ErrorOrCurChildrenToProcess = readUnsignedNumber<uint32_t>(); |
471 | if (!ErrorOrCurChildrenToProcess) |
472 | return false; |
473 | // Read all probes in this node |
474 | for (std::size_t I = 0; I < NodeCount; I++) { |
475 | // Read index |
476 | auto ErrorOrIndex = readUnsignedNumber<uint32_t>(); |
477 | if (!ErrorOrIndex) |
478 | return false; |
479 | uint32_t Index = std::move(*ErrorOrIndex); |
480 | // Read type | flag. |
481 | auto ErrorOrValue = readUnencodedNumber<uint8_t>(); |
482 | if (!ErrorOrValue) |
483 | return false; |
484 | uint8_t Value = std::move(*ErrorOrValue); |
485 | uint8_t Kind = Value & 0xf; |
486 | uint8_t Attr = (Value & 0x70) >> 4; |
487 | // Read address |
488 | uint64_t Addr = 0; |
489 | if (Value & 0x80) { |
490 | auto ErrorOrOffset = readSignedNumber<int64_t>(); |
491 | if (!ErrorOrOffset) |
492 | return false; |
493 | int64_t Offset = std::move(*ErrorOrOffset); |
494 | Addr = LastAddr + Offset; |
495 | } else { |
496 | auto ErrorOrAddr = readUnencodedNumber<int64_t>(); |
497 | if (!ErrorOrAddr) |
498 | return false; |
499 | Addr = std::move(*ErrorOrAddr); |
500 | if (isSentinelProbe(Flags: Attr)) { |
501 | // For sentinel probe, the addr field actually stores the GUID of the |
502 | // split function. Convert it to the real address. |
503 | if (auto V = FuncStartAddrs.lookup(Val: Addr)) |
504 | Addr = V; |
505 | } else { |
506 | // For now we assume all probe encoding should be either based on |
507 | // leading probe address or function start address. |
508 | // The scheme is for downwards compatibility. |
509 | // TODO: retire this scheme once compatibility is no longer an issue. |
510 | EncodingIsAddrBased = true; |
511 | } |
512 | } |
513 | |
514 | uint32_t Discriminator = 0; |
515 | if (hasDiscriminator(Flags: Attr)) { |
516 | auto ErrorOrDiscriminator = readUnsignedNumber<uint32_t>(); |
517 | if (!ErrorOrDiscriminator) |
518 | return false; |
519 | Discriminator = std::move(*ErrorOrDiscriminator); |
520 | } |
521 | |
522 | if (Cur && !isSentinelProbe(Flags: Attr)) { |
523 | // Populate Address2ProbesMap |
524 | auto &Probes = Address2ProbesMap[Addr]; |
525 | Probes.emplace_back(args&: Addr, args&: Cur->Guid, args&: Index, args: PseudoProbeType(Kind), args&: Attr, |
526 | args&: Discriminator, args&: Cur); |
527 | Cur->addProbes(Probe: &Probes.back()); |
528 | } |
529 | LastAddr = Addr; |
530 | } |
531 | |
532 | uint32_t ChildrenToProcess = std::move(*ErrorOrCurChildrenToProcess); |
533 | for (uint32_t I = 0; I < ChildrenToProcess; I++) { |
534 | buildAddress2ProbeMap(Cur, LastAddr, GuidFilter, FuncStartAddrs); |
535 | } |
536 | |
537 | return true; |
538 | } |
539 | |
540 | bool MCPseudoProbeDecoder::buildAddress2ProbeMap( |
541 | const uint8_t *Start, std::size_t Size, const Uint64Set &GuidFilter, |
542 | const Uint64Map &FuncStartAddrs) { |
543 | Data = Start; |
544 | End = Data + Size; |
545 | uint64_t LastAddr = 0; |
546 | while (Data < End) |
547 | buildAddress2ProbeMap(Cur: &DummyInlineRoot, LastAddr, GuidFilter, |
548 | FuncStartAddrs); |
549 | assert(Data == End && "Have unprocessed data in pseudo_probe section" ); |
550 | return true; |
551 | } |
552 | |
553 | void MCPseudoProbeDecoder::printGUID2FuncDescMap(raw_ostream &OS) { |
554 | OS << "Pseudo Probe Desc:\n" ; |
555 | // Make the output deterministic |
556 | std::map<uint64_t, MCPseudoProbeFuncDesc> OrderedMap(GUID2FuncDescMap.begin(), |
557 | GUID2FuncDescMap.end()); |
558 | for (auto &I : OrderedMap) { |
559 | I.second.print(OS); |
560 | } |
561 | } |
562 | |
563 | void MCPseudoProbeDecoder::printProbeForAddress(raw_ostream &OS, |
564 | uint64_t Address) { |
565 | auto It = Address2ProbesMap.find(x: Address); |
566 | if (It != Address2ProbesMap.end()) { |
567 | for (auto &Probe : It->second) { |
568 | OS << " [Probe]:\t" ; |
569 | Probe.print(OS, GUID2FuncMAP: GUID2FuncDescMap, ShowName: true); |
570 | } |
571 | } |
572 | } |
573 | |
574 | void MCPseudoProbeDecoder::printProbesForAllAddresses(raw_ostream &OS) { |
575 | auto Entries = make_first_range(c&: Address2ProbesMap); |
576 | SmallVector<uint64_t, 0> Addresses(Entries.begin(), Entries.end()); |
577 | llvm::sort(C&: Addresses); |
578 | for (auto K : Addresses) { |
579 | OS << "Address:\t" ; |
580 | OS << K; |
581 | OS << "\n" ; |
582 | printProbeForAddress(OS, Address: K); |
583 | } |
584 | } |
585 | |
586 | const MCDecodedPseudoProbe * |
587 | MCPseudoProbeDecoder::getCallProbeForAddr(uint64_t Address) const { |
588 | auto It = Address2ProbesMap.find(x: Address); |
589 | if (It == Address2ProbesMap.end()) |
590 | return nullptr; |
591 | const auto &Probes = It->second; |
592 | |
593 | const MCDecodedPseudoProbe *CallProbe = nullptr; |
594 | for (const auto &Probe : Probes) { |
595 | if (Probe.isCall()) { |
596 | // Disabling the assert and returning first call probe seen so far. |
597 | // Subsequent call probes, if any, are ignored. Due to the the way |
598 | // .pseudo_probe section is decoded, probes of the same-named independent |
599 | // static functions are merged thus multiple call probes may be seen for a |
600 | // callsite. This should only happen to compiler-generated statics, with |
601 | // -funique-internal-linkage-names where user statics get unique names. |
602 | // |
603 | // TODO: re-enable or narrow down the assert to static functions only. |
604 | // |
605 | // assert(!CallProbe && |
606 | // "There should be only one call probe corresponding to address " |
607 | // "which is a callsite."); |
608 | CallProbe = &Probe; |
609 | break; |
610 | } |
611 | } |
612 | return CallProbe; |
613 | } |
614 | |
615 | const MCPseudoProbeFuncDesc * |
616 | MCPseudoProbeDecoder::getFuncDescForGUID(uint64_t GUID) const { |
617 | auto It = GUID2FuncDescMap.find(x: GUID); |
618 | assert(It != GUID2FuncDescMap.end() && "Function descriptor doesn't exist" ); |
619 | return &It->second; |
620 | } |
621 | |
622 | void MCPseudoProbeDecoder::getInlineContextForProbe( |
623 | const MCDecodedPseudoProbe *Probe, |
624 | SmallVectorImpl<MCPseduoProbeFrameLocation> &InlineContextStack, |
625 | bool IncludeLeaf) const { |
626 | Probe->getInlineContext(ContextStack&: InlineContextStack, GUID2FuncMAP: GUID2FuncDescMap); |
627 | if (!IncludeLeaf) |
628 | return; |
629 | // Note that the context from probe doesn't include leaf frame, |
630 | // hence we need to retrieve and prepend leaf if requested. |
631 | const auto *FuncDesc = getFuncDescForGUID(GUID: Probe->getGuid()); |
632 | InlineContextStack.emplace_back( |
633 | Args: MCPseduoProbeFrameLocation(FuncDesc->FuncName, Probe->getIndex())); |
634 | } |
635 | |
636 | const MCPseudoProbeFuncDesc *MCPseudoProbeDecoder::getInlinerDescForProbe( |
637 | const MCDecodedPseudoProbe *Probe) const { |
638 | MCDecodedPseudoProbeInlineTree *InlinerNode = Probe->getInlineTreeNode(); |
639 | if (!InlinerNode->hasInlineSite()) |
640 | return nullptr; |
641 | return getFuncDescForGUID(GUID: InlinerNode->Parent->Guid); |
642 | } |
643 | |