1 | //===-- CGProfile.cpp -----------------------------------------------------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | |
9 | #include "llvm/Transforms/Instrumentation/CGProfile.h" |
10 | |
11 | #include "llvm/ADT/MapVector.h" |
12 | #include "llvm/Analysis/BlockFrequencyInfo.h" |
13 | #include "llvm/Analysis/LazyBlockFrequencyInfo.h" |
14 | #include "llvm/Analysis/TargetTransformInfo.h" |
15 | #include "llvm/IR/Constants.h" |
16 | #include "llvm/IR/MDBuilder.h" |
17 | #include "llvm/IR/PassManager.h" |
18 | #include "llvm/ProfileData/InstrProf.h" |
19 | #include "llvm/Transforms/Instrumentation.h" |
20 | #include <optional> |
21 | |
22 | using namespace llvm; |
23 | |
24 | static bool |
25 | addModuleFlags(Module &M, |
26 | MapVector<std::pair<Function *, Function *>, uint64_t> &Counts) { |
27 | if (Counts.empty()) |
28 | return false; |
29 | |
30 | LLVMContext &Context = M.getContext(); |
31 | MDBuilder MDB(Context); |
32 | std::vector<Metadata *> Nodes; |
33 | |
34 | for (auto E : Counts) { |
35 | Metadata *Vals[] = {ValueAsMetadata::get(V: E.first.first), |
36 | ValueAsMetadata::get(V: E.first.second), |
37 | MDB.createConstant(C: ConstantInt::get( |
38 | Ty: Type::getInt64Ty(C&: Context), V: E.second))}; |
39 | Nodes.push_back(x: MDNode::get(Context, MDs: Vals)); |
40 | } |
41 | |
42 | M.addModuleFlag(Behavior: Module::Append, Key: "CG Profile" , |
43 | Val: MDTuple::getDistinct(Context, MDs: Nodes)); |
44 | return true; |
45 | } |
46 | |
47 | static bool runCGProfilePass(Module &M, FunctionAnalysisManager &FAM, |
48 | bool InLTO) { |
49 | MapVector<std::pair<Function *, Function *>, uint64_t> Counts; |
50 | InstrProfSymtab Symtab; |
51 | auto UpdateCounts = [&](TargetTransformInfo &TTI, Function *F, |
52 | Function *CalledF, uint64_t NewCount) { |
53 | if (NewCount == 0) |
54 | return; |
55 | if (!CalledF || !TTI.isLoweredToCall(F: CalledF) || |
56 | CalledF->hasDLLImportStorageClass()) |
57 | return; |
58 | uint64_t &Count = Counts[std::make_pair(x&: F, y&: CalledF)]; |
59 | Count = SaturatingAdd(X: Count, Y: NewCount); |
60 | }; |
61 | // Ignore error here. Indirect calls are ignored if this fails. |
62 | (void)(bool)Symtab.create(M, InLTO); |
63 | for (auto &F : M) { |
64 | // Avoid extra cost of running passes for BFI when the function doesn't have |
65 | // entry count. |
66 | if (F.isDeclaration() || !F.getEntryCount()) |
67 | continue; |
68 | auto &BFI = FAM.getResult<BlockFrequencyAnalysis>(IR&: F); |
69 | if (BFI.getEntryFreq() == BlockFrequency(0)) |
70 | continue; |
71 | TargetTransformInfo &TTI = FAM.getResult<TargetIRAnalysis>(IR&: F); |
72 | for (auto &BB : F) { |
73 | std::optional<uint64_t> BBCount = BFI.getBlockProfileCount(BB: &BB); |
74 | if (!BBCount) |
75 | continue; |
76 | for (auto &I : BB) { |
77 | CallBase *CB = dyn_cast<CallBase>(Val: &I); |
78 | if (!CB) |
79 | continue; |
80 | if (CB->isIndirectCall()) { |
81 | InstrProfValueData ValueData[8]; |
82 | uint32_t ActualNumValueData; |
83 | uint64_t TotalC; |
84 | if (!getValueProfDataFromInst(Inst: *CB, ValueKind: IPVK_IndirectCallTarget, MaxNumValueData: 8, |
85 | ValueData, ActualNumValueData, TotalC)) |
86 | continue; |
87 | for (const auto &VD : |
88 | ArrayRef<InstrProfValueData>(ValueData, ActualNumValueData)) { |
89 | UpdateCounts(TTI, &F, Symtab.getFunction(FuncMD5Hash: VD.Value), VD.Count); |
90 | } |
91 | continue; |
92 | } |
93 | UpdateCounts(TTI, &F, CB->getCalledFunction(), *BBCount); |
94 | } |
95 | } |
96 | } |
97 | |
98 | return addModuleFlags(M, Counts); |
99 | } |
100 | |
101 | PreservedAnalyses CGProfilePass::run(Module &M, ModuleAnalysisManager &MAM) { |
102 | FunctionAnalysisManager &FAM = |
103 | MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager(); |
104 | runCGProfilePass(M, FAM, InLTO); |
105 | |
106 | return PreservedAnalyses::all(); |
107 | } |
108 | |