1//===- Transforms/IPO/SampleProfileProbe.h ----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file provides the interface for the pseudo probe implementation for
11/// AutoFDO.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_TRANSFORMS_IPO_SAMPLEPROFILEPROBE_H
16#define LLVM_TRANSFORMS_IPO_SAMPLEPROFILEPROBE_H
17
18#include "llvm/Analysis/LazyCallGraph.h"
19#include "llvm/IR/PassManager.h"
20#include "llvm/ProfileData/SampleProf.h"
21#include <unordered_map>
22
23namespace llvm {
24class BasicBlock;
25class Function;
26class Instruction;
27class Loop;
28class PassInstrumentationCallbacks;
29class TargetMachine;
30
31class Module;
32
33using namespace sampleprof;
34using BlockIdMap = std::unordered_map<BasicBlock *, uint32_t>;
35using InstructionIdMap = std::unordered_map<Instruction *, uint32_t>;
36// Map from tuples of Probe id and inline stack hash code to distribution
37// factors.
38using ProbeFactorMap = std::unordered_map<std::pair<uint64_t, uint64_t>, float,
39 pair_hash<uint64_t, uint64_t>>;
40using FuncProbeFactorMap = StringMap<ProbeFactorMap>;
41
42
43// A pseudo probe verifier that can be run after each IR passes to detect the
44// violation of updating probe factors. In principle, the sum of distribution
45// factor for a probe should be identical before and after a pass. For a
46// function pass, the factor sum for a probe would be typically 100%.
47class PseudoProbeVerifier {
48public:
49 void registerCallbacks(PassInstrumentationCallbacks &PIC);
50
51 // Implementation of pass instrumentation callbacks for new pass manager.
52 void runAfterPass(StringRef PassID, Any IR);
53
54private:
55 // Allow a little bias due the rounding to integral factors.
56 constexpr static float DistributionFactorVariance = 0.02f;
57 // Distribution factors from last pass.
58 FuncProbeFactorMap FunctionProbeFactors;
59
60 void collectProbeFactors(const BasicBlock *BB, ProbeFactorMap &ProbeFactors);
61 void runAfterPass(const Module *M);
62 void runAfterPass(const LazyCallGraph::SCC *C);
63 void runAfterPass(const Function *F);
64 void runAfterPass(const Loop *L);
65 bool shouldVerifyFunction(const Function *F);
66 void verifyProbeFactors(const Function *F,
67 const ProbeFactorMap &ProbeFactors);
68};
69
70/// Sample profile pseudo prober.
71///
72/// Insert pseudo probes for block sampling and value sampling.
73class SampleProfileProber {
74public:
75 // Give an empty module id when the prober is not used for instrumentation.
76 SampleProfileProber(Function &F, const std::string &CurModuleUniqueId);
77 void instrumentOneFunc(Function &F, TargetMachine *TM);
78
79private:
80 Function *getFunction() const { return F; }
81 uint64_t getFunctionHash() const { return FunctionHash; }
82 uint32_t getBlockId(const BasicBlock *BB) const;
83 uint32_t getCallsiteId(const Instruction *Call) const;
84 void computeCFGHash();
85 void computeProbeIdForBlocks();
86 void computeProbeIdForCallsites();
87
88 Function *F;
89
90 /// The current module ID that is used to name a static object as a comdat
91 /// group.
92 std::string CurModuleUniqueId;
93
94 /// A CFG hash code used to identify a function code changes.
95 uint64_t FunctionHash;
96
97 /// Map basic blocks to the their pseudo probe ids.
98 BlockIdMap BlockProbeIds;
99
100 /// Map indirect calls to the their pseudo probe ids.
101 InstructionIdMap CallProbeIds;
102
103 /// The ID of the last probe, Can be used to number a new probe.
104 uint32_t LastProbeId;
105};
106
107class SampleProfileProbePass : public PassInfoMixin<SampleProfileProbePass> {
108 TargetMachine *TM;
109
110public:
111 SampleProfileProbePass(TargetMachine *TM) : TM(TM) {}
112 PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
113};
114
115// Pseudo probe distribution factor updater.
116// Sample profile annotation can happen in both LTO prelink and postlink. The
117// postlink-time re-annotation can degrade profile quality because of prelink
118// code duplication transformation, such as loop unrolling, jump threading,
119// indirect call promotion etc. As such, samples corresponding to a source
120// location may be aggregated multiple times in postlink. With a concept of
121// distribution factor for pseudo probes, samples can be distributed among
122// duplicated probes reasonable based on the assumption that optimizations
123// duplicating code well-maintain the branch frequency information (BFI). This
124// pass updates distribution factors for each pseudo probe at the end of the
125// prelink pipeline, to reflect an estimated portion of the real execution
126// count.
127class PseudoProbeUpdatePass : public PassInfoMixin<PseudoProbeUpdatePass> {
128 void runOnFunction(Function &F, FunctionAnalysisManager &FAM);
129
130public:
131 PseudoProbeUpdatePass() = default;
132 PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
133};
134
135} // end namespace llvm
136#endif // LLVM_TRANSFORMS_IPO_SAMPLEPROFILEPROBE_H
137

source code of llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h