1 | //===- unittest/ProfileData/InstrProfTest.cpp -------------------*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | |
9 | #include "llvm/IR/Function.h" |
10 | #include "llvm/IR/IRBuilder.h" |
11 | #include "llvm/IR/LLVMContext.h" |
12 | #include "llvm/IR/Module.h" |
13 | #include "llvm/ProfileData/InstrProfReader.h" |
14 | #include "llvm/ProfileData/InstrProfWriter.h" |
15 | #include "llvm/ProfileData/MemProf.h" |
16 | #include "llvm/ProfileData/MemProfData.inc" |
17 | #include "llvm/Support/Compression.h" |
18 | #include "llvm/Support/raw_ostream.h" |
19 | #include "llvm/Testing/Support/Error.h" |
20 | #include "gtest/gtest.h" |
21 | #include <cstdarg> |
22 | #include <optional> |
23 | |
24 | using namespace llvm; |
25 | using ::testing::EndsWith; |
26 | using ::testing::IsSubsetOf; |
27 | using ::testing::SizeIs; |
28 | using ::testing::UnorderedElementsAre; |
29 | |
30 | [[nodiscard]] static ::testing::AssertionResult |
31 | ErrorEquals(instrprof_error Expected, Error E) { |
32 | instrprof_error Found; |
33 | std::string FoundMsg; |
34 | handleAllErrors(E: std::move(E), Handlers: [&](const InstrProfError &IPE) { |
35 | Found = IPE.get(); |
36 | FoundMsg = IPE.message(); |
37 | }); |
38 | if (Expected == Found) |
39 | return ::testing::AssertionSuccess(); |
40 | return ::testing::AssertionFailure() << "error: " << FoundMsg << "\n" ; |
41 | } |
42 | |
43 | namespace llvm { |
44 | bool operator==(const TemporalProfTraceTy &lhs, |
45 | const TemporalProfTraceTy &rhs) { |
46 | return lhs.Weight == rhs.Weight && |
47 | lhs.FunctionNameRefs == rhs.FunctionNameRefs; |
48 | } |
49 | } // end namespace llvm |
50 | |
51 | namespace { |
52 | |
53 | struct InstrProfTest : ::testing::Test { |
54 | InstrProfWriter Writer; |
55 | std::unique_ptr<IndexedInstrProfReader> Reader; |
56 | |
57 | void SetUp() override { Writer.setOutputSparse(false); } |
58 | |
59 | void readProfile(std::unique_ptr<MemoryBuffer> Profile, |
60 | std::unique_ptr<MemoryBuffer> Remapping = nullptr) { |
61 | auto ReaderOrErr = IndexedInstrProfReader::create(Buffer: std::move(Profile), |
62 | RemappingBuffer: std::move(Remapping)); |
63 | EXPECT_THAT_ERROR(ReaderOrErr.takeError(), Succeeded()); |
64 | Reader = std::move(ReaderOrErr.get()); |
65 | } |
66 | }; |
67 | |
68 | struct SparseInstrProfTest : public InstrProfTest { |
69 | void SetUp() override { Writer.setOutputSparse(true); } |
70 | }; |
71 | |
72 | struct InstrProfReaderWriterTest |
73 | : public InstrProfTest, |
74 | public ::testing::WithParamInterface< |
75 | std::tuple<bool, uint64_t, llvm::endianness>> { |
76 | void SetUp() override { Writer.setOutputSparse(std::get<0>(t: GetParam())); } |
77 | void TearDown() override { |
78 | // Reset writer value profile data endianness after each test case. Note |
79 | // it's not necessary to reset reader value profile endianness for each test |
80 | // case. Each test case creates a new reader; at reader initialization time, |
81 | // it uses the endianness from hash table object (which is little by |
82 | // default). |
83 | Writer.setValueProfDataEndianness(llvm::endianness::little); |
84 | } |
85 | |
86 | uint64_t getProfWeight() const { return std::get<1>(t: GetParam()); } |
87 | |
88 | llvm::endianness getEndianness() const { return std::get<2>(t: GetParam()); } |
89 | }; |
90 | |
91 | struct MaybeSparseInstrProfTest : public InstrProfTest, |
92 | public ::testing::WithParamInterface<bool> { |
93 | void SetUp() override { Writer.setOutputSparse(GetParam()); } |
94 | }; |
95 | |
96 | TEST_P(MaybeSparseInstrProfTest, write_and_read_empty_profile) { |
97 | auto Profile = Writer.writeBuffer(); |
98 | readProfile(Profile: std::move(Profile)); |
99 | ASSERT_TRUE(Reader->begin() == Reader->end()); |
100 | } |
101 | |
102 | static const auto Err = [](Error E) { |
103 | consumeError(Err: std::move(E)); |
104 | FAIL(); |
105 | }; |
106 | |
107 | TEST_P(MaybeSparseInstrProfTest, write_and_read_one_function) { |
108 | Writer.addRecord(I: {"foo" , 0x1234, {1, 2, 3, 4}}, Warn: Err); |
109 | auto Profile = Writer.writeBuffer(); |
110 | readProfile(Profile: std::move(Profile)); |
111 | |
112 | auto I = Reader->begin(), E = Reader->end(); |
113 | ASSERT_TRUE(I != E); |
114 | ASSERT_EQ(StringRef("foo" ), I->Name); |
115 | ASSERT_EQ(0x1234U, I->Hash); |
116 | ASSERT_EQ(4U, I->Counts.size()); |
117 | ASSERT_EQ(1U, I->Counts[0]); |
118 | ASSERT_EQ(2U, I->Counts[1]); |
119 | ASSERT_EQ(3U, I->Counts[2]); |
120 | ASSERT_EQ(4U, I->Counts[3]); |
121 | ASSERT_TRUE(++I == E); |
122 | } |
123 | |
124 | TEST_P(MaybeSparseInstrProfTest, get_instr_prof_record) { |
125 | Writer.addRecord(I: {"foo" , 0x1234, {1, 2}}, Warn: Err); |
126 | Writer.addRecord(I: {"foo" , 0x1235, {3, 4}}, Warn: Err); |
127 | auto Profile = Writer.writeBuffer(); |
128 | readProfile(Profile: std::move(Profile)); |
129 | |
130 | Expected<InstrProfRecord> R = Reader->getInstrProfRecord(FuncName: "foo" , FuncHash: 0x1234); |
131 | EXPECT_THAT_ERROR(R.takeError(), Succeeded()); |
132 | ASSERT_EQ(2U, R->Counts.size()); |
133 | ASSERT_EQ(1U, R->Counts[0]); |
134 | ASSERT_EQ(2U, R->Counts[1]); |
135 | |
136 | R = Reader->getInstrProfRecord(FuncName: "foo" , FuncHash: 0x1235); |
137 | EXPECT_THAT_ERROR(R.takeError(), Succeeded()); |
138 | ASSERT_EQ(2U, R->Counts.size()); |
139 | ASSERT_EQ(3U, R->Counts[0]); |
140 | ASSERT_EQ(4U, R->Counts[1]); |
141 | |
142 | R = Reader->getInstrProfRecord(FuncName: "foo" , FuncHash: 0x5678); |
143 | ASSERT_TRUE(ErrorEquals(instrprof_error::hash_mismatch, R.takeError())); |
144 | |
145 | R = Reader->getInstrProfRecord(FuncName: "bar" , FuncHash: 0x1234); |
146 | ASSERT_TRUE(ErrorEquals(instrprof_error::unknown_function, R.takeError())); |
147 | } |
148 | |
149 | TEST_P(MaybeSparseInstrProfTest, get_function_counts) { |
150 | Writer.addRecord(I: {"foo" , 0x1234, {1, 2}}, Warn: Err); |
151 | Writer.addRecord(I: {"foo" , 0x1235, {3, 4}}, Warn: Err); |
152 | auto Profile = Writer.writeBuffer(); |
153 | readProfile(Profile: std::move(Profile)); |
154 | |
155 | std::vector<uint64_t> Counts; |
156 | EXPECT_THAT_ERROR(Reader->getFunctionCounts("foo" , 0x1234, Counts), |
157 | Succeeded()); |
158 | ASSERT_EQ(2U, Counts.size()); |
159 | ASSERT_EQ(1U, Counts[0]); |
160 | ASSERT_EQ(2U, Counts[1]); |
161 | |
162 | EXPECT_THAT_ERROR(Reader->getFunctionCounts("foo" , 0x1235, Counts), |
163 | Succeeded()); |
164 | ASSERT_EQ(2U, Counts.size()); |
165 | ASSERT_EQ(3U, Counts[0]); |
166 | ASSERT_EQ(4U, Counts[1]); |
167 | |
168 | Error E1 = Reader->getFunctionCounts(FuncName: "foo" , FuncHash: 0x5678, Counts); |
169 | ASSERT_TRUE(ErrorEquals(instrprof_error::hash_mismatch, std::move(E1))); |
170 | |
171 | Error E2 = Reader->getFunctionCounts(FuncName: "bar" , FuncHash: 0x1234, Counts); |
172 | ASSERT_TRUE(ErrorEquals(instrprof_error::unknown_function, std::move(E2))); |
173 | } |
174 | |
175 | // Profile data is copied from general.proftext |
176 | TEST_F(InstrProfTest, get_profile_summary) { |
177 | Writer.addRecord(I: {"func1" , 0x1234, {97531}}, Warn: Err); |
178 | Writer.addRecord(I: {"func2" , 0x1234, {0, 0}}, Warn: Err); |
179 | Writer.addRecord( |
180 | I: {"func3" , |
181 | 0x1234, |
182 | {2305843009213693952, 1152921504606846976, 576460752303423488, |
183 | 288230376151711744, 144115188075855872, 72057594037927936}}, |
184 | Warn: Err); |
185 | Writer.addRecord(I: {"func4" , 0x1234, {0}}, Warn: Err); |
186 | auto Profile = Writer.writeBuffer(); |
187 | readProfile(Profile: std::move(Profile)); |
188 | |
189 | auto VerifySummary = [](ProfileSummary &IPS) mutable { |
190 | ASSERT_EQ(ProfileSummary::PSK_Instr, IPS.getKind()); |
191 | ASSERT_EQ(2305843009213693952U, IPS.getMaxFunctionCount()); |
192 | ASSERT_EQ(2305843009213693952U, IPS.getMaxCount()); |
193 | ASSERT_EQ(10U, IPS.getNumCounts()); |
194 | ASSERT_EQ(4539628424389557499U, IPS.getTotalCount()); |
195 | const std::vector<ProfileSummaryEntry> &Details = IPS.getDetailedSummary(); |
196 | uint32_t Cutoff = 800000; |
197 | auto Predicate = [&Cutoff](const ProfileSummaryEntry &PE) { |
198 | return PE.Cutoff == Cutoff; |
199 | }; |
200 | auto EightyPerc = find_if(Range: Details, P: Predicate); |
201 | Cutoff = 900000; |
202 | auto NinetyPerc = find_if(Range: Details, P: Predicate); |
203 | Cutoff = 950000; |
204 | auto NinetyFivePerc = find_if(Range: Details, P: Predicate); |
205 | Cutoff = 990000; |
206 | auto NinetyNinePerc = find_if(Range: Details, P: Predicate); |
207 | ASSERT_EQ(576460752303423488U, EightyPerc->MinCount); |
208 | ASSERT_EQ(288230376151711744U, NinetyPerc->MinCount); |
209 | ASSERT_EQ(288230376151711744U, NinetyFivePerc->MinCount); |
210 | ASSERT_EQ(72057594037927936U, NinetyNinePerc->MinCount); |
211 | }; |
212 | ProfileSummary &PS = Reader->getSummary(/* IsCS */ UseCS: false); |
213 | VerifySummary(PS); |
214 | |
215 | // Test that conversion of summary to and from Metadata works. |
216 | LLVMContext Context; |
217 | Metadata *MD = PS.getMD(Context); |
218 | ASSERT_TRUE(MD); |
219 | ProfileSummary *PSFromMD = ProfileSummary::getFromMD(MD); |
220 | ASSERT_TRUE(PSFromMD); |
221 | VerifySummary(*PSFromMD); |
222 | delete PSFromMD; |
223 | |
224 | // Test that summary can be attached to and read back from module. |
225 | Module M("my_module" , Context); |
226 | M.setProfileSummary(M: MD, Kind: ProfileSummary::PSK_Instr); |
227 | MD = M.getProfileSummary(/* IsCS */ false); |
228 | ASSERT_TRUE(MD); |
229 | PSFromMD = ProfileSummary::getFromMD(MD); |
230 | ASSERT_TRUE(PSFromMD); |
231 | VerifySummary(*PSFromMD); |
232 | delete PSFromMD; |
233 | } |
234 | |
235 | TEST_F(InstrProfTest, test_writer_merge) { |
236 | Writer.addRecord(I: {"func1" , 0x1234, {42}}, Warn: Err); |
237 | |
238 | InstrProfWriter Writer2; |
239 | Writer2.addRecord(I: {"func2" , 0x1234, {0, 0}}, Warn: Err); |
240 | |
241 | Writer.mergeRecordsFromWriter(IPW: std::move(Writer2), Warn: Err); |
242 | |
243 | auto Profile = Writer.writeBuffer(); |
244 | readProfile(Profile: std::move(Profile)); |
245 | |
246 | Expected<InstrProfRecord> R = Reader->getInstrProfRecord(FuncName: "func1" , FuncHash: 0x1234); |
247 | EXPECT_THAT_ERROR(R.takeError(), Succeeded()); |
248 | ASSERT_EQ(1U, R->Counts.size()); |
249 | ASSERT_EQ(42U, R->Counts[0]); |
250 | |
251 | R = Reader->getInstrProfRecord(FuncName: "func2" , FuncHash: 0x1234); |
252 | EXPECT_THAT_ERROR(R.takeError(), Succeeded()); |
253 | ASSERT_EQ(2U, R->Counts.size()); |
254 | ASSERT_EQ(0U, R->Counts[0]); |
255 | ASSERT_EQ(0U, R->Counts[1]); |
256 | } |
257 | |
258 | TEST_F(InstrProfTest, test_merge_temporal_prof_traces_truncated) { |
259 | uint64_t ReservoirSize = 10; |
260 | uint64_t MaxTraceLength = 2; |
261 | InstrProfWriter Writer(/*Sparse=*/false, ReservoirSize, MaxTraceLength); |
262 | ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::TemporalProfile), |
263 | Succeeded()); |
264 | |
265 | TemporalProfTraceTy LargeTrace, SmallTrace; |
266 | LargeTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash(K: "foo" ), |
267 | IndexedInstrProf::ComputeHash(K: "bar" ), |
268 | IndexedInstrProf::ComputeHash(K: "goo" )}; |
269 | SmallTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash(K: "foo" ), |
270 | IndexedInstrProf::ComputeHash(K: "bar" )}; |
271 | |
272 | SmallVector<TemporalProfTraceTy, 4> Traces = {LargeTrace, SmallTrace}; |
273 | Writer.addTemporalProfileTraces(SrcTraces&: Traces, SrcStreamSize: 2); |
274 | |
275 | auto Profile = Writer.writeBuffer(); |
276 | readProfile(Profile: std::move(Profile)); |
277 | |
278 | ASSERT_TRUE(Reader->hasTemporalProfile()); |
279 | EXPECT_EQ(Reader->getTemporalProfTraceStreamSize(), 2U); |
280 | EXPECT_THAT(Reader->getTemporalProfTraces(), |
281 | UnorderedElementsAre(SmallTrace, SmallTrace)); |
282 | } |
283 | |
284 | TEST_F(InstrProfTest, test_merge_traces_from_writer) { |
285 | uint64_t ReservoirSize = 10; |
286 | uint64_t MaxTraceLength = 10; |
287 | InstrProfWriter Writer(/*Sparse=*/false, ReservoirSize, MaxTraceLength); |
288 | InstrProfWriter Writer2(/*Sparse=*/false, ReservoirSize, MaxTraceLength); |
289 | ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::TemporalProfile), |
290 | Succeeded()); |
291 | ASSERT_THAT_ERROR(Writer2.mergeProfileKind(InstrProfKind::TemporalProfile), |
292 | Succeeded()); |
293 | |
294 | TemporalProfTraceTy , BarTrace; |
295 | FooTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash(K: "foo" )}; |
296 | BarTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash(K: "bar" )}; |
297 | |
298 | SmallVector<TemporalProfTraceTy, 4> Traces1({FooTrace}), Traces2({BarTrace}); |
299 | Writer.addTemporalProfileTraces(SrcTraces&: Traces1, SrcStreamSize: 1); |
300 | Writer2.addTemporalProfileTraces(SrcTraces&: Traces2, SrcStreamSize: 1); |
301 | Writer.mergeRecordsFromWriter(IPW: std::move(Writer2), Warn: Err); |
302 | |
303 | auto Profile = Writer.writeBuffer(); |
304 | readProfile(Profile: std::move(Profile)); |
305 | |
306 | ASSERT_TRUE(Reader->hasTemporalProfile()); |
307 | EXPECT_EQ(Reader->getTemporalProfTraceStreamSize(), 2U); |
308 | EXPECT_THAT(Reader->getTemporalProfTraces(), |
309 | UnorderedElementsAre(FooTrace, BarTrace)); |
310 | } |
311 | |
312 | TEST_F(InstrProfTest, test_merge_traces_sampled) { |
313 | uint64_t ReservoirSize = 3; |
314 | uint64_t MaxTraceLength = 10; |
315 | InstrProfWriter Writer(/*Sparse=*/false, ReservoirSize, MaxTraceLength); |
316 | ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::TemporalProfile), |
317 | Succeeded()); |
318 | |
319 | TemporalProfTraceTy , BarTrace, GooTrace; |
320 | FooTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash(K: "foo" )}; |
321 | BarTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash(K: "bar" )}; |
322 | GooTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash(K: "Goo" )}; |
323 | |
324 | // Add some sampled traces |
325 | SmallVector<TemporalProfTraceTy, 4> SampledTraces = {FooTrace, BarTrace, |
326 | GooTrace}; |
327 | Writer.addTemporalProfileTraces(SrcTraces&: SampledTraces, SrcStreamSize: 5); |
328 | // Add some unsampled traces |
329 | SmallVector<TemporalProfTraceTy, 4> UnsampledTraces = {BarTrace, GooTrace}; |
330 | Writer.addTemporalProfileTraces(SrcTraces&: UnsampledTraces, SrcStreamSize: 2); |
331 | UnsampledTraces = {FooTrace}; |
332 | Writer.addTemporalProfileTraces(SrcTraces&: UnsampledTraces, SrcStreamSize: 1); |
333 | |
334 | auto Profile = Writer.writeBuffer(); |
335 | readProfile(Profile: std::move(Profile)); |
336 | |
337 | ASSERT_TRUE(Reader->hasTemporalProfile()); |
338 | EXPECT_EQ(Reader->getTemporalProfTraceStreamSize(), 8U); |
339 | // Check that we have a subset of all the traces we added |
340 | EXPECT_THAT(Reader->getTemporalProfTraces(), SizeIs(ReservoirSize)); |
341 | EXPECT_THAT( |
342 | Reader->getTemporalProfTraces(), |
343 | IsSubsetOf({FooTrace, BarTrace, GooTrace, BarTrace, GooTrace, FooTrace})); |
344 | } |
345 | |
346 | using ::llvm::memprof::IndexedMemProfRecord; |
347 | using ::llvm::memprof::MemInfoBlock; |
348 | using FrameIdMapTy = |
349 | llvm::DenseMap<::llvm::memprof::FrameId, ::llvm::memprof::Frame>; |
350 | using CallStackIdMapTy = |
351 | llvm::DenseMap<::llvm::memprof::CallStackId, |
352 | ::llvm::SmallVector<::llvm::memprof::FrameId>>; |
353 | |
354 | static FrameIdMapTy getFrameMapping() { |
355 | FrameIdMapTy Mapping; |
356 | Mapping.insert(KV: {0, {0x123, 1, 2, false}}); |
357 | Mapping.insert(KV: {1, {0x345, 3, 4, true}}); |
358 | Mapping.insert(KV: {2, {0x125, 5, 6, false}}); |
359 | Mapping.insert(KV: {3, {0x567, 7, 8, true}}); |
360 | Mapping.insert(KV: {4, {0x124, 5, 6, false}}); |
361 | Mapping.insert(KV: {5, {0x789, 8, 9, true}}); |
362 | return Mapping; |
363 | } |
364 | |
365 | static CallStackIdMapTy getCallStackMapping() { |
366 | CallStackIdMapTy Mapping; |
367 | Mapping.insert(KV: {0x111, {0, 1}}); |
368 | Mapping.insert(KV: {0x222, {2, 3}}); |
369 | Mapping.insert(KV: {0x333, {4, 5}}); |
370 | return Mapping; |
371 | } |
372 | |
373 | IndexedMemProfRecord makeRecord( |
374 | std::initializer_list<std::initializer_list<::llvm::memprof::FrameId>> |
375 | AllocFrames, |
376 | std::initializer_list<std::initializer_list<::llvm::memprof::FrameId>> |
377 | CallSiteFrames, |
378 | const MemInfoBlock &Block = MemInfoBlock()) { |
379 | llvm::memprof::IndexedMemProfRecord MR; |
380 | for (const auto &Frames : AllocFrames) |
381 | MR.AllocSites.emplace_back(Args: Frames, Args: llvm::memprof::hashCallStack(CS: Frames), |
382 | Args: Block); |
383 | for (const auto &Frames : CallSiteFrames) |
384 | MR.CallSites.push_back(Elt: Frames); |
385 | return MR; |
386 | } |
387 | |
388 | IndexedMemProfRecord |
389 | makeRecordV2(std::initializer_list<::llvm::memprof::CallStackId> AllocFrames, |
390 | std::initializer_list<::llvm::memprof::CallStackId> CallSiteFrames, |
391 | const MemInfoBlock &Block = MemInfoBlock()) { |
392 | llvm::memprof::IndexedMemProfRecord MR; |
393 | for (const auto &CSId : AllocFrames) |
394 | // We don't populate IndexedAllocationInfo::CallStack because we use it only |
395 | // in Version0 and Version1. |
396 | MR.AllocSites.emplace_back(Args: ::llvm::SmallVector<memprof::FrameId>(), Args: CSId, |
397 | Args: Block); |
398 | for (const auto &CSId : CallSiteFrames) |
399 | MR.CallSiteIds.push_back(Elt: CSId); |
400 | return MR; |
401 | } |
402 | |
403 | MATCHER_P(EqualsRecord, Want, "" ) { |
404 | const memprof::MemProfRecord &Got = arg; |
405 | |
406 | auto PrintAndFail = [&]() { |
407 | std::string Buffer; |
408 | llvm::raw_string_ostream OS(Buffer); |
409 | OS << "Want:\n" ; |
410 | Want.print(OS); |
411 | OS << "Got:\n" ; |
412 | Got.print(OS); |
413 | OS.flush(); |
414 | *result_listener << "MemProf Record differs!\n" << Buffer; |
415 | return false; |
416 | }; |
417 | |
418 | if (Want.AllocSites.size() != Got.AllocSites.size()) |
419 | return PrintAndFail(); |
420 | if (Want.CallSites.size() != Got.CallSites.size()) |
421 | return PrintAndFail(); |
422 | |
423 | for (size_t I = 0; I < Got.AllocSites.size(); I++) { |
424 | if (Want.AllocSites[I].Info != Got.AllocSites[I].Info) |
425 | return PrintAndFail(); |
426 | if (Want.AllocSites[I].CallStack != Got.AllocSites[I].CallStack) |
427 | return PrintAndFail(); |
428 | } |
429 | |
430 | for (size_t I = 0; I < Got.CallSites.size(); I++) { |
431 | if (Want.CallSites[I] != Got.CallSites[I]) |
432 | return PrintAndFail(); |
433 | } |
434 | return true; |
435 | } |
436 | |
437 | TEST_F(InstrProfTest, test_memprof_v0) { |
438 | ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::MemProf), |
439 | Succeeded()); |
440 | |
441 | const IndexedMemProfRecord IndexedMR = makeRecord( |
442 | /*AllocFrames=*/ |
443 | { |
444 | {0, 1}, |
445 | {2, 3}, |
446 | }, |
447 | /*CallSiteFrames=*/{ |
448 | {4, 5}, |
449 | }); |
450 | const FrameIdMapTy IdToFrameMap = getFrameMapping(); |
451 | for (const auto &I : IdToFrameMap) { |
452 | Writer.addMemProfFrame(I.first, F: I.getSecond(), Warn: Err); |
453 | } |
454 | Writer.addMemProfRecord(/*Id=*/0x9999, Record: IndexedMR); |
455 | |
456 | auto Profile = Writer.writeBuffer(); |
457 | readProfile(Profile: std::move(Profile)); |
458 | |
459 | auto RecordOr = Reader->getMemProfRecord(FuncNameHash: 0x9999); |
460 | ASSERT_THAT_ERROR(RecordOr.takeError(), Succeeded()); |
461 | const memprof::MemProfRecord &Record = RecordOr.get(); |
462 | |
463 | std::optional<memprof::FrameId> LastUnmappedFrameId; |
464 | auto IdToFrameCallback = [&](const memprof::FrameId Id) { |
465 | auto Iter = IdToFrameMap.find(Val: Id); |
466 | if (Iter == IdToFrameMap.end()) { |
467 | LastUnmappedFrameId = Id; |
468 | return memprof::Frame(0, 0, 0, false); |
469 | } |
470 | return Iter->second; |
471 | }; |
472 | |
473 | const memprof::MemProfRecord WantRecord(IndexedMR, IdToFrameCallback); |
474 | ASSERT_EQ(LastUnmappedFrameId, std::nullopt) |
475 | << "could not map frame id: " << *LastUnmappedFrameId; |
476 | EXPECT_THAT(WantRecord, EqualsRecord(Record)); |
477 | } |
478 | |
479 | TEST_F(InstrProfTest, test_memprof_v2) { |
480 | Writer.setMemProfVersionRequested(memprof::Version2); |
481 | |
482 | ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::MemProf), |
483 | Succeeded()); |
484 | |
485 | const IndexedMemProfRecord IndexedMR = makeRecordV2( |
486 | /*AllocFrames=*/{0x111, 0x222}, |
487 | /*CallSiteFrames=*/{0x333}); |
488 | const FrameIdMapTy IdToFrameMap = getFrameMapping(); |
489 | const auto CSIdToCallStackMap = getCallStackMapping(); |
490 | for (const auto &I : IdToFrameMap) { |
491 | Writer.addMemProfFrame(I.first, F: I.getSecond(), Warn: Err); |
492 | } |
493 | for (const auto &I : CSIdToCallStackMap) { |
494 | Writer.addMemProfCallStack(CSId: I.first, CallStack: I.getSecond(), Warn: Err); |
495 | } |
496 | Writer.addMemProfRecord(/*Id=*/0x9999, Record: IndexedMR); |
497 | |
498 | auto Profile = Writer.writeBuffer(); |
499 | readProfile(Profile: std::move(Profile)); |
500 | |
501 | auto RecordOr = Reader->getMemProfRecord(FuncNameHash: 0x9999); |
502 | ASSERT_THAT_ERROR(RecordOr.takeError(), Succeeded()); |
503 | const memprof::MemProfRecord &Record = RecordOr.get(); |
504 | |
505 | std::optional<memprof::FrameId> LastUnmappedFrameId; |
506 | auto IdToFrameCallback = [&](const memprof::FrameId Id) { |
507 | auto Iter = IdToFrameMap.find(Val: Id); |
508 | if (Iter == IdToFrameMap.end()) { |
509 | LastUnmappedFrameId = Id; |
510 | return memprof::Frame(0, 0, 0, false); |
511 | } |
512 | return Iter->second; |
513 | }; |
514 | |
515 | std::optional<::llvm::memprof::CallStackId> LastUnmappedCSId; |
516 | auto CSIdToCallStackCallback = [&](::llvm::memprof::CallStackId CSId) { |
517 | llvm::SmallVector<memprof::Frame> Frames; |
518 | auto CSIter = CSIdToCallStackMap.find(Val: CSId); |
519 | if (CSIter == CSIdToCallStackMap.end()) { |
520 | LastUnmappedCSId = CSId; |
521 | } else { |
522 | const ::llvm::SmallVector<::llvm::memprof::FrameId> &CS = |
523 | CSIter->getSecond(); |
524 | Frames.reserve(N: CS.size()); |
525 | for (::llvm::memprof::FrameId Id : CS) |
526 | Frames.push_back(Elt: IdToFrameCallback(Id)); |
527 | } |
528 | return Frames; |
529 | }; |
530 | |
531 | const ::llvm::memprof::MemProfRecord WantRecord = |
532 | IndexedMR.toMemProfRecord(Callback: CSIdToCallStackCallback); |
533 | ASSERT_EQ(LastUnmappedFrameId, std::nullopt) |
534 | << "could not map frame id: " << *LastUnmappedFrameId; |
535 | ASSERT_EQ(LastUnmappedCSId, std::nullopt) |
536 | << "could not map call stack id: " << *LastUnmappedCSId; |
537 | EXPECT_THAT(WantRecord, EqualsRecord(Record)); |
538 | } |
539 | |
540 | TEST_F(InstrProfTest, test_memprof_getrecord_error) { |
541 | ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::MemProf), |
542 | Succeeded()); |
543 | |
544 | const IndexedMemProfRecord IndexedMR = makeRecord( |
545 | /*AllocFrames=*/ |
546 | { |
547 | {0, 1}, |
548 | {2, 3}, |
549 | }, |
550 | /*CallSiteFrames=*/{ |
551 | {4, 5}, |
552 | }); |
553 | // We skip adding the frame mappings here unlike the test_memprof unit test |
554 | // above to exercise the failure path when getMemProfRecord is invoked. |
555 | Writer.addMemProfRecord(/*Id=*/0x9999, Record: IndexedMR); |
556 | |
557 | auto Profile = Writer.writeBuffer(); |
558 | readProfile(Profile: std::move(Profile)); |
559 | |
560 | // Missing frames give a hash_mismatch error. |
561 | auto RecordOr = Reader->getMemProfRecord(FuncNameHash: 0x9999); |
562 | ASSERT_TRUE( |
563 | ErrorEquals(instrprof_error::hash_mismatch, RecordOr.takeError())); |
564 | |
565 | // Missing functions give a unknown_function error. |
566 | RecordOr = Reader->getMemProfRecord(FuncNameHash: 0x1111); |
567 | ASSERT_TRUE( |
568 | ErrorEquals(instrprof_error::unknown_function, RecordOr.takeError())); |
569 | } |
570 | |
571 | TEST_F(InstrProfTest, test_memprof_merge) { |
572 | Writer.addRecord(I: {"func1" , 0x1234, {42}}, Warn: Err); |
573 | |
574 | InstrProfWriter Writer2; |
575 | ASSERT_THAT_ERROR(Writer2.mergeProfileKind(InstrProfKind::MemProf), |
576 | Succeeded()); |
577 | |
578 | const IndexedMemProfRecord IndexedMR = makeRecord( |
579 | /*AllocFrames=*/ |
580 | { |
581 | {0, 1}, |
582 | {2, 3}, |
583 | }, |
584 | /*CallSiteFrames=*/{ |
585 | {4, 5}, |
586 | }); |
587 | |
588 | const FrameIdMapTy IdToFrameMap = getFrameMapping(); |
589 | for (const auto &I : IdToFrameMap) { |
590 | Writer.addMemProfFrame(I.first, F: I.getSecond(), Warn: Err); |
591 | } |
592 | Writer2.addMemProfRecord(/*Id=*/0x9999, Record: IndexedMR); |
593 | |
594 | ASSERT_THAT_ERROR(Writer.mergeProfileKind(Writer2.getProfileKind()), |
595 | Succeeded()); |
596 | Writer.mergeRecordsFromWriter(IPW: std::move(Writer2), Warn: Err); |
597 | |
598 | auto Profile = Writer.writeBuffer(); |
599 | readProfile(Profile: std::move(Profile)); |
600 | |
601 | Expected<InstrProfRecord> R = Reader->getInstrProfRecord(FuncName: "func1" , FuncHash: 0x1234); |
602 | EXPECT_THAT_ERROR(R.takeError(), Succeeded()); |
603 | ASSERT_EQ(1U, R->Counts.size()); |
604 | ASSERT_EQ(42U, R->Counts[0]); |
605 | |
606 | auto RecordOr = Reader->getMemProfRecord(FuncNameHash: 0x9999); |
607 | ASSERT_THAT_ERROR(RecordOr.takeError(), Succeeded()); |
608 | const memprof::MemProfRecord &Record = RecordOr.get(); |
609 | |
610 | std::optional<memprof::FrameId> LastUnmappedFrameId; |
611 | |
612 | auto IdToFrameCallback = [&](const memprof::FrameId Id) { |
613 | auto Iter = IdToFrameMap.find(Val: Id); |
614 | if (Iter == IdToFrameMap.end()) { |
615 | LastUnmappedFrameId = Id; |
616 | return memprof::Frame(0, 0, 0, false); |
617 | } |
618 | return Iter->second; |
619 | }; |
620 | |
621 | const memprof::MemProfRecord WantRecord(IndexedMR, IdToFrameCallback); |
622 | ASSERT_EQ(LastUnmappedFrameId, std::nullopt) |
623 | << "could not map frame id: " << *LastUnmappedFrameId; |
624 | EXPECT_THAT(WantRecord, EqualsRecord(Record)); |
625 | } |
626 | |
627 | TEST_F(InstrProfTest, test_irpgo_function_name) { |
628 | LLVMContext Ctx; |
629 | auto M = std::make_unique<Module>(args: "MyModule.cpp" , args&: Ctx); |
630 | auto *FTy = FunctionType::get(Result: Type::getVoidTy(C&: Ctx), /*isVarArg=*/false); |
631 | |
632 | std::vector<std::tuple<StringRef, Function::LinkageTypes, StringRef>> Data; |
633 | Data.emplace_back(args: "ExternalFoo" , args: Function::ExternalLinkage, args: "ExternalFoo" ); |
634 | Data.emplace_back(args: "InternalFoo" , args: Function::InternalLinkage, |
635 | args: "MyModule.cpp;InternalFoo" ); |
636 | Data.emplace_back(args: "\01-[C dynamicFoo:]" , args: Function::ExternalLinkage, |
637 | args: "-[C dynamicFoo:]" ); |
638 | Data.emplace_back(args: "\01-[C internalFoo:]" , args: Function::InternalLinkage, |
639 | args: "MyModule.cpp;-[C internalFoo:]" ); |
640 | |
641 | for (auto &[Name, Linkage, ExpectedIRPGOFuncName] : Data) |
642 | Function::Create(Ty: FTy, Linkage, N: Name, M: M.get()); |
643 | |
644 | for (auto &[Name, Linkage, ExpectedIRPGOFuncName] : Data) { |
645 | auto *F = M->getFunction(Name); |
646 | auto IRPGOFuncName = getIRPGOFuncName(F: *F); |
647 | EXPECT_EQ(IRPGOFuncName, ExpectedIRPGOFuncName); |
648 | |
649 | auto [Filename, ParsedIRPGOFuncName] = getParsedIRPGOName(IRPGOName: IRPGOFuncName); |
650 | StringRef ExpectedParsedIRPGOFuncName = IRPGOFuncName; |
651 | if (ExpectedParsedIRPGOFuncName.consume_front(Prefix: "MyModule.cpp;" )) { |
652 | EXPECT_EQ(Filename, "MyModule.cpp" ); |
653 | } else { |
654 | EXPECT_EQ(Filename, "" ); |
655 | } |
656 | EXPECT_EQ(ParsedIRPGOFuncName, ExpectedParsedIRPGOFuncName); |
657 | } |
658 | } |
659 | |
660 | TEST_F(InstrProfTest, test_pgo_function_name) { |
661 | LLVMContext Ctx; |
662 | auto M = std::make_unique<Module>(args: "MyModule.cpp" , args&: Ctx); |
663 | auto *FTy = FunctionType::get(Result: Type::getVoidTy(C&: Ctx), /*isVarArg=*/false); |
664 | |
665 | std::vector<std::tuple<StringRef, Function::LinkageTypes, StringRef>> Data; |
666 | Data.emplace_back(args: "ExternalFoo" , args: Function::ExternalLinkage, args: "ExternalFoo" ); |
667 | Data.emplace_back(args: "InternalFoo" , args: Function::InternalLinkage, |
668 | args: "MyModule.cpp:InternalFoo" ); |
669 | Data.emplace_back(args: "\01-[C externalFoo:]" , args: Function::ExternalLinkage, |
670 | args: "-[C externalFoo:]" ); |
671 | Data.emplace_back(args: "\01-[C internalFoo:]" , args: Function::InternalLinkage, |
672 | args: "MyModule.cpp:-[C internalFoo:]" ); |
673 | |
674 | for (auto &[Name, Linkage, ExpectedPGOFuncName] : Data) |
675 | Function::Create(Ty: FTy, Linkage, N: Name, M: M.get()); |
676 | |
677 | for (auto &[Name, Linkage, ExpectedPGOFuncName] : Data) { |
678 | auto *F = M->getFunction(Name); |
679 | EXPECT_EQ(getPGOFuncName(*F), ExpectedPGOFuncName); |
680 | } |
681 | } |
682 | |
683 | TEST_F(InstrProfTest, test_irpgo_read_deprecated_names) { |
684 | LLVMContext Ctx; |
685 | auto M = std::make_unique<Module>(args: "MyModule.cpp" , args&: Ctx); |
686 | auto *FTy = FunctionType::get(Result: Type::getVoidTy(C&: Ctx), /*isVarArg=*/false); |
687 | auto *InternalFooF = |
688 | Function::Create(Ty: FTy, Linkage: Function::InternalLinkage, N: "InternalFoo" , M: M.get()); |
689 | auto *ExternalFooF = |
690 | Function::Create(Ty: FTy, Linkage: Function::ExternalLinkage, N: "ExternalFoo" , M: M.get()); |
691 | |
692 | auto *InternalBarF = |
693 | Function::Create(Ty: FTy, Linkage: Function::InternalLinkage, N: "InternalBar" , M: M.get()); |
694 | auto *ExternalBarF = |
695 | Function::Create(Ty: FTy, Linkage: Function::ExternalLinkage, N: "ExternalBar" , M: M.get()); |
696 | |
697 | Writer.addRecord(I: {getIRPGOFuncName(F: *InternalFooF), 0x1234, {1}}, Warn: Err); |
698 | Writer.addRecord(I: {getIRPGOFuncName(F: *ExternalFooF), 0x5678, {1}}, Warn: Err); |
699 | // Write a record with a deprecated name |
700 | Writer.addRecord(I: {getPGOFuncName(F: *InternalBarF), 0x1111, {2}}, Warn: Err); |
701 | Writer.addRecord(I: {getPGOFuncName(F: *ExternalBarF), 0x2222, {2}}, Warn: Err); |
702 | |
703 | auto Profile = Writer.writeBuffer(); |
704 | readProfile(Profile: std::move(Profile)); |
705 | |
706 | EXPECT_THAT_EXPECTED( |
707 | Reader->getInstrProfRecord(getIRPGOFuncName(*InternalFooF), 0x1234, |
708 | getPGOFuncName(*InternalFooF)), |
709 | Succeeded()); |
710 | EXPECT_THAT_EXPECTED( |
711 | Reader->getInstrProfRecord(getIRPGOFuncName(*ExternalFooF), 0x5678, |
712 | getPGOFuncName(*ExternalFooF)), |
713 | Succeeded()); |
714 | // Ensure we can still read this old record name |
715 | EXPECT_THAT_EXPECTED( |
716 | Reader->getInstrProfRecord(getIRPGOFuncName(*InternalBarF), 0x1111, |
717 | getPGOFuncName(*InternalBarF)), |
718 | Succeeded()); |
719 | EXPECT_THAT_EXPECTED( |
720 | Reader->getInstrProfRecord(getIRPGOFuncName(*ExternalBarF), 0x2222, |
721 | getPGOFuncName(*ExternalBarF)), |
722 | Succeeded()); |
723 | } |
724 | |
725 | // callee1 to callee6 are from vtable1 to vtable6 respectively. |
726 | static const char callee1[] = "callee1" ; |
727 | static const char callee2[] = "callee2" ; |
728 | static const char callee3[] = "callee3" ; |
729 | static const char callee4[] = "callee4" ; |
730 | static const char callee5[] = "callee5" ; |
731 | static const char callee6[] = "callee6" ; |
732 | // callee7 and callee8 are not from any vtables. |
733 | static const char callee7[] = "callee7" ; |
734 | static const char callee8[] = "callee8" ; |
735 | // 'callee' is primarily used to create multiple-element vtables. |
736 | static const char callee[] = "callee" ; |
737 | static const uint64_t vtable1[] = {uint64_t(callee), uint64_t(callee1)}; |
738 | static const uint64_t vtable2[] = {uint64_t(callee2), uint64_t(callee)}; |
739 | static const uint64_t vtable3[] = { |
740 | uint64_t(callee), |
741 | uint64_t(callee3), |
742 | }; |
743 | static const uint64_t vtable4[] = {uint64_t(callee4), uint64_t(callee)}; |
744 | static const uint64_t vtable5[] = {uint64_t(callee5), uint64_t(callee)}; |
745 | static const uint64_t vtable6[] = {uint64_t(callee6), uint64_t(callee)}; |
746 | |
747 | // Returns the address of callee with a numbered suffix in vtable. |
748 | static uint64_t getCalleeAddress(const uint64_t *vtableAddr) { |
749 | uint64_t CalleeAddr; |
750 | // Callee with a numbered suffix is the 2nd element in vtable1 and vtable3, |
751 | // and the 1st element in the rest of vtables. |
752 | if (vtableAddr == vtable1 || vtableAddr == vtable3) |
753 | CalleeAddr = uint64_t(vtableAddr) + 8; |
754 | else |
755 | CalleeAddr = uint64_t(vtableAddr); |
756 | return CalleeAddr; |
757 | } |
758 | |
759 | TEST_P(InstrProfReaderWriterTest, icall_and_vtable_data_read_write) { |
760 | NamedInstrProfRecord Record1("caller" , 0x1234, {1, 2}); |
761 | |
762 | // 4 indirect call value sites. |
763 | { |
764 | Record1.reserveSites(ValueKind: IPVK_IndirectCallTarget, NumValueSites: 4); |
765 | InstrProfValueData VD0[] = { |
766 | {.Value: (uint64_t)callee1, .Count: 1}, {.Value: (uint64_t)callee2, .Count: 2}, {.Value: (uint64_t)callee3, .Count: 3}}; |
767 | Record1.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 0, VData: VD0, N: 3, SymTab: nullptr); |
768 | // No value profile data at the second site. |
769 | Record1.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 1, VData: nullptr, N: 0, SymTab: nullptr); |
770 | InstrProfValueData VD2[] = {{.Value: (uint64_t)callee1, .Count: 1}, {.Value: (uint64_t)callee2, .Count: 2}}; |
771 | Record1.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 2, VData: VD2, N: 2, SymTab: nullptr); |
772 | InstrProfValueData VD3[] = {{.Value: (uint64_t)callee7, .Count: 1}, {.Value: (uint64_t)callee8, .Count: 2}}; |
773 | Record1.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 3, VData: VD3, N: 2, SymTab: nullptr); |
774 | } |
775 | |
776 | // 2 vtable value sites. |
777 | { |
778 | InstrProfValueData VD0[] = { |
779 | {.Value: getCalleeAddress(vtableAddr: vtable1), .Count: 1}, |
780 | {.Value: getCalleeAddress(vtableAddr: vtable2), .Count: 2}, |
781 | {.Value: getCalleeAddress(vtableAddr: vtable3), .Count: 3}, |
782 | }; |
783 | InstrProfValueData VD2[] = { |
784 | {.Value: getCalleeAddress(vtableAddr: vtable1), .Count: 1}, |
785 | {.Value: getCalleeAddress(vtableAddr: vtable2), .Count: 2}, |
786 | }; |
787 | Record1.addValueData(ValueKind: IPVK_VTableTarget, Site: 0, VData: VD0, N: 3, SymTab: nullptr); |
788 | Record1.addValueData(ValueKind: IPVK_VTableTarget, Site: 2, VData: VD2, N: 2, SymTab: nullptr); |
789 | } |
790 | |
791 | Writer.addRecord(I: std::move(Record1), Weight: getProfWeight(), Warn: Err); |
792 | Writer.addRecord(I: {"callee1" , 0x1235, {3, 4}}, Warn: Err); |
793 | Writer.addRecord(I: {"callee2" , 0x1235, {3, 4}}, Warn: Err); |
794 | Writer.addRecord(I: {"callee3" , 0x1235, {3, 4}}, Warn: Err); |
795 | Writer.addRecord(I: {"callee7" , 0x1235, {3, 4}}, Warn: Err); |
796 | Writer.addRecord(I: {"callee8" , 0x1235, {3, 4}}, Warn: Err); |
797 | |
798 | // Set writer value prof data endianness. |
799 | Writer.setValueProfDataEndianness(getEndianness()); |
800 | |
801 | auto Profile = Writer.writeBuffer(); |
802 | readProfile(Profile: std::move(Profile)); |
803 | |
804 | // Set reader value prof data endianness. |
805 | Reader->setValueProfDataEndianness(getEndianness()); |
806 | |
807 | Expected<InstrProfRecord> R = Reader->getInstrProfRecord(FuncName: "caller" , FuncHash: 0x1234); |
808 | ASSERT_THAT_ERROR(R.takeError(), Succeeded()); |
809 | |
810 | // Test the number of instrumented indirect call sites and the number of |
811 | // profiled values at each site. |
812 | ASSERT_EQ(4U, R->getNumValueSites(IPVK_IndirectCallTarget)); |
813 | EXPECT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0)); |
814 | EXPECT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1)); |
815 | EXPECT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2)); |
816 | EXPECT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3)); |
817 | |
818 | // Test the number of instrumented vtable sites and the number of profiled |
819 | // values at each site. |
820 | ASSERT_EQ(R->getNumValueSites(IPVK_VTableTarget), 2U); |
821 | EXPECT_EQ(R->getNumValueDataForSite(IPVK_VTableTarget, 0), 3U); |
822 | EXPECT_EQ(R->getNumValueDataForSite(IPVK_VTableTarget, 1), 2U); |
823 | |
824 | // First indirect site. |
825 | { |
826 | uint64_t TotalC; |
827 | auto VD = R->getValueForSite(ValueKind: IPVK_IndirectCallTarget, Site: 0, TotalC: &TotalC); |
828 | |
829 | EXPECT_EQ(VD[0].Count, 3U * getProfWeight()); |
830 | EXPECT_EQ(VD[1].Count, 2U * getProfWeight()); |
831 | EXPECT_EQ(VD[2].Count, 1U * getProfWeight()); |
832 | EXPECT_EQ(TotalC, 6U * getProfWeight()); |
833 | |
834 | EXPECT_STREQ((const char *)VD[0].Value, "callee3" ); |
835 | EXPECT_STREQ((const char *)VD[1].Value, "callee2" ); |
836 | EXPECT_STREQ((const char *)VD[2].Value, "callee1" ); |
837 | } |
838 | |
839 | // First vtable site. |
840 | { |
841 | uint64_t TotalC; |
842 | auto VD = R->getValueForSite(ValueKind: IPVK_VTableTarget, Site: 0, TotalC: &TotalC); |
843 | |
844 | EXPECT_EQ(VD[0].Count, 3U * getProfWeight()); |
845 | EXPECT_EQ(VD[1].Count, 2U * getProfWeight()); |
846 | EXPECT_EQ(VD[2].Count, 1U * getProfWeight()); |
847 | EXPECT_EQ(TotalC, 6U * getProfWeight()); |
848 | |
849 | EXPECT_EQ(VD[0].Value, getCalleeAddress(vtable3)); |
850 | EXPECT_EQ(VD[1].Value, getCalleeAddress(vtable2)); |
851 | EXPECT_EQ(VD[2].Value, getCalleeAddress(vtable1)); |
852 | } |
853 | |
854 | // Second vtable site. |
855 | { |
856 | uint64_t TotalC; |
857 | auto VD = R->getValueForSite(ValueKind: IPVK_VTableTarget, Site: 1, TotalC: &TotalC); |
858 | |
859 | EXPECT_EQ(VD[0].Count, 2U * getProfWeight()); |
860 | EXPECT_EQ(VD[1].Count, 1U * getProfWeight()); |
861 | EXPECT_EQ(TotalC, 3U * getProfWeight()); |
862 | |
863 | EXPECT_EQ(VD[0].Value, getCalleeAddress(vtable2)); |
864 | EXPECT_EQ(VD[1].Value, getCalleeAddress(vtable1)); |
865 | } |
866 | } |
867 | |
868 | INSTANTIATE_TEST_SUITE_P( |
869 | WeightAndEndiannessTest, InstrProfReaderWriterTest, |
870 | ::testing::Combine( |
871 | ::testing::Bool(), /* Sparse */ |
872 | ::testing::Values(1U, 10U), /* ProfWeight */ |
873 | ::testing::Values(llvm::endianness::big, |
874 | llvm::endianness::little) /* Endianness */ |
875 | )); |
876 | |
877 | TEST_P(MaybeSparseInstrProfTest, annotate_vp_data) { |
878 | NamedInstrProfRecord Record("caller" , 0x1234, {1, 2}); |
879 | Record.reserveSites(ValueKind: IPVK_IndirectCallTarget, NumValueSites: 1); |
880 | InstrProfValueData VD0[] = {{.Value: 1000, .Count: 1}, {.Value: 2000, .Count: 2}, {.Value: 3000, .Count: 3}, {.Value: 5000, .Count: 5}, |
881 | {.Value: 4000, .Count: 4}, {.Value: 6000, .Count: 6}}; |
882 | Record.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 0, VData: VD0, N: 6, SymTab: nullptr); |
883 | Writer.addRecord(I: std::move(Record), Warn: Err); |
884 | auto Profile = Writer.writeBuffer(); |
885 | readProfile(Profile: std::move(Profile)); |
886 | Expected<InstrProfRecord> R = Reader->getInstrProfRecord(FuncName: "caller" , FuncHash: 0x1234); |
887 | EXPECT_THAT_ERROR(R.takeError(), Succeeded()); |
888 | |
889 | LLVMContext Ctx; |
890 | std::unique_ptr<Module> M(new Module("MyModule" , Ctx)); |
891 | FunctionType *FTy = FunctionType::get(Result: Type::getVoidTy(C&: Ctx), |
892 | /*isVarArg=*/false); |
893 | Function *F = |
894 | Function::Create(Ty: FTy, Linkage: Function::ExternalLinkage, N: "caller" , M: M.get()); |
895 | BasicBlock *BB = BasicBlock::Create(Context&: Ctx, Name: "" , Parent: F); |
896 | |
897 | IRBuilder<> Builder(BB); |
898 | BasicBlock *TBB = BasicBlock::Create(Context&: Ctx, Name: "" , Parent: F); |
899 | BasicBlock *FBB = BasicBlock::Create(Context&: Ctx, Name: "" , Parent: F); |
900 | |
901 | // Use branch instruction to annotate with value profile data for simplicity |
902 | Instruction *Inst = Builder.CreateCondBr(Cond: Builder.getTrue(), True: TBB, False: FBB); |
903 | Instruction *Inst2 = Builder.CreateCondBr(Cond: Builder.getTrue(), True: TBB, False: FBB); |
904 | annotateValueSite(M&: *M, Inst&: *Inst, InstrProfR: R.get(), ValueKind: IPVK_IndirectCallTarget, SiteIndx: 0); |
905 | |
906 | InstrProfValueData ValueData[5]; |
907 | uint32_t N; |
908 | uint64_t T; |
909 | bool Res = getValueProfDataFromInst(Inst: *Inst, ValueKind: IPVK_IndirectCallTarget, MaxNumValueData: 5, |
910 | ValueData, ActualNumValueData&: N, TotalC&: T); |
911 | ASSERT_TRUE(Res); |
912 | ASSERT_EQ(3U, N); |
913 | ASSERT_EQ(21U, T); |
914 | // The result should be sorted already: |
915 | ASSERT_EQ(6000U, ValueData[0].Value); |
916 | ASSERT_EQ(6U, ValueData[0].Count); |
917 | ASSERT_EQ(5000U, ValueData[1].Value); |
918 | ASSERT_EQ(5U, ValueData[1].Count); |
919 | ASSERT_EQ(4000U, ValueData[2].Value); |
920 | ASSERT_EQ(4U, ValueData[2].Count); |
921 | Res = getValueProfDataFromInst(Inst: *Inst, ValueKind: IPVK_IndirectCallTarget, MaxNumValueData: 1, ValueData, |
922 | ActualNumValueData&: N, TotalC&: T); |
923 | ASSERT_TRUE(Res); |
924 | ASSERT_EQ(1U, N); |
925 | ASSERT_EQ(21U, T); |
926 | |
927 | Res = getValueProfDataFromInst(Inst: *Inst2, ValueKind: IPVK_IndirectCallTarget, MaxNumValueData: 5, ValueData, |
928 | ActualNumValueData&: N, TotalC&: T); |
929 | ASSERT_FALSE(Res); |
930 | |
931 | // Remove the MD_prof metadata |
932 | Inst->setMetadata(KindID: LLVMContext::MD_prof, Node: 0); |
933 | // Annotate 5 records this time. |
934 | annotateValueSite(M&: *M, Inst&: *Inst, InstrProfR: R.get(), ValueKind: IPVK_IndirectCallTarget, SiteIndx: 0, MaxMDCount: 5); |
935 | Res = getValueProfDataFromInst(Inst: *Inst, ValueKind: IPVK_IndirectCallTarget, MaxNumValueData: 5, |
936 | ValueData, ActualNumValueData&: N, TotalC&: T); |
937 | ASSERT_TRUE(Res); |
938 | ASSERT_EQ(5U, N); |
939 | ASSERT_EQ(21U, T); |
940 | ASSERT_EQ(6000U, ValueData[0].Value); |
941 | ASSERT_EQ(6U, ValueData[0].Count); |
942 | ASSERT_EQ(5000U, ValueData[1].Value); |
943 | ASSERT_EQ(5U, ValueData[1].Count); |
944 | ASSERT_EQ(4000U, ValueData[2].Value); |
945 | ASSERT_EQ(4U, ValueData[2].Count); |
946 | ASSERT_EQ(3000U, ValueData[3].Value); |
947 | ASSERT_EQ(3U, ValueData[3].Count); |
948 | ASSERT_EQ(2000U, ValueData[4].Value); |
949 | ASSERT_EQ(2U, ValueData[4].Count); |
950 | |
951 | // Remove the MD_prof metadata |
952 | Inst->setMetadata(KindID: LLVMContext::MD_prof, Node: 0); |
953 | // Annotate with 4 records. |
954 | InstrProfValueData VD0Sorted[] = {{.Value: 1000, .Count: 6}, {.Value: 2000, .Count: 5}, {.Value: 3000, .Count: 4}, {.Value: 4000, .Count: 3}, |
955 | {.Value: 5000, .Count: 2}, {.Value: 6000, .Count: 1}}; |
956 | annotateValueSite(M&: *M, Inst&: *Inst, VDs: ArrayRef(VD0Sorted).slice(N: 2), Sum: 10, |
957 | ValueKind: IPVK_IndirectCallTarget, MaxMDCount: 5); |
958 | Res = getValueProfDataFromInst(Inst: *Inst, ValueKind: IPVK_IndirectCallTarget, MaxNumValueData: 5, |
959 | ValueData, ActualNumValueData&: N, TotalC&: T); |
960 | ASSERT_TRUE(Res); |
961 | ASSERT_EQ(4U, N); |
962 | ASSERT_EQ(10U, T); |
963 | ASSERT_EQ(3000U, ValueData[0].Value); |
964 | ASSERT_EQ(4U, ValueData[0].Count); |
965 | ASSERT_EQ(4000U, ValueData[1].Value); |
966 | ASSERT_EQ(3U, ValueData[1].Count); |
967 | ASSERT_EQ(5000U, ValueData[2].Value); |
968 | ASSERT_EQ(2U, ValueData[2].Count); |
969 | ASSERT_EQ(6000U, ValueData[3].Value); |
970 | ASSERT_EQ(1U, ValueData[3].Count); |
971 | } |
972 | |
973 | TEST_P(MaybeSparseInstrProfTest, icall_and_vtable_data_merge) { |
974 | static const char caller[] = "caller" ; |
975 | NamedInstrProfRecord Record11(caller, 0x1234, {1, 2}); |
976 | NamedInstrProfRecord Record12(caller, 0x1234, {1, 2}); |
977 | |
978 | // 5 value sites for indirect calls. |
979 | { |
980 | Record11.reserveSites(ValueKind: IPVK_IndirectCallTarget, NumValueSites: 5); |
981 | InstrProfValueData VD0[] = {{.Value: uint64_t(callee1), .Count: 1}, |
982 | {.Value: uint64_t(callee2), .Count: 2}, |
983 | {.Value: uint64_t(callee3), .Count: 3}, |
984 | {.Value: uint64_t(callee4), .Count: 4}}; |
985 | Record11.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 0, VData: VD0, N: 4, SymTab: nullptr); |
986 | |
987 | // No value profile data at the second site. |
988 | Record11.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 1, VData: nullptr, N: 0, SymTab: nullptr); |
989 | |
990 | InstrProfValueData VD2[] = { |
991 | {.Value: uint64_t(callee1), .Count: 1}, {.Value: uint64_t(callee2), .Count: 2}, {.Value: uint64_t(callee3), .Count: 3}}; |
992 | Record11.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 2, VData: VD2, N: 3, SymTab: nullptr); |
993 | |
994 | InstrProfValueData VD3[] = {{.Value: uint64_t(callee7), .Count: 1}, {.Value: uint64_t(callee8), .Count: 2}}; |
995 | Record11.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 3, VData: VD3, N: 2, SymTab: nullptr); |
996 | |
997 | InstrProfValueData VD4[] = { |
998 | {.Value: uint64_t(callee1), .Count: 1}, {.Value: uint64_t(callee2), .Count: 2}, {.Value: uint64_t(callee3), .Count: 3}}; |
999 | Record11.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 4, VData: VD4, N: 3, SymTab: nullptr); |
1000 | } |
1001 | // 3 value sites for vtables. |
1002 | { |
1003 | Record11.reserveSites(ValueKind: IPVK_VTableTarget, NumValueSites: 3); |
1004 | InstrProfValueData VD0[] = {{.Value: getCalleeAddress(vtableAddr: vtable1), .Count: 1}, |
1005 | {.Value: getCalleeAddress(vtableAddr: vtable2), .Count: 2}, |
1006 | {.Value: getCalleeAddress(vtableAddr: vtable3), .Count: 3}, |
1007 | {.Value: getCalleeAddress(vtableAddr: vtable4), .Count: 4}}; |
1008 | Record11.addValueData(ValueKind: IPVK_VTableTarget, Site: 0, VData: VD0, N: 4, SymTab: nullptr); |
1009 | |
1010 | InstrProfValueData VD2[] = {{.Value: getCalleeAddress(vtableAddr: vtable1), .Count: 1}, |
1011 | {.Value: getCalleeAddress(vtableAddr: vtable2), .Count: 2}, |
1012 | {.Value: getCalleeAddress(vtableAddr: vtable3), .Count: 3}}; |
1013 | Record11.addValueData(ValueKind: IPVK_VTableTarget, Site: 1, VData: VD2, N: 3, SymTab: nullptr); |
1014 | |
1015 | InstrProfValueData VD4[] = {{.Value: getCalleeAddress(vtableAddr: vtable1), .Count: 1}, |
1016 | {.Value: getCalleeAddress(vtableAddr: vtable2), .Count: 2}, |
1017 | {.Value: getCalleeAddress(vtableAddr: vtable3), .Count: 3}}; |
1018 | Record11.addValueData(ValueKind: IPVK_VTableTarget, Site: 3, VData: VD4, N: 3, SymTab: nullptr); |
1019 | } |
1020 | |
1021 | // A different record for the same caller. |
1022 | Record12.reserveSites(ValueKind: IPVK_IndirectCallTarget, NumValueSites: 5); |
1023 | InstrProfValueData VD02[] = {{.Value: uint64_t(callee2), .Count: 5}, {.Value: uint64_t(callee3), .Count: 3}}; |
1024 | Record12.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 0, VData: VD02, N: 2, SymTab: nullptr); |
1025 | |
1026 | // No value profile data at the second site. |
1027 | Record12.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 1, VData: nullptr, N: 0, SymTab: nullptr); |
1028 | |
1029 | InstrProfValueData VD22[] = { |
1030 | {.Value: uint64_t(callee2), .Count: 1}, {.Value: uint64_t(callee3), .Count: 3}, {.Value: uint64_t(callee4), .Count: 4}}; |
1031 | Record12.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 2, VData: VD22, N: 3, SymTab: nullptr); |
1032 | |
1033 | Record12.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 3, VData: nullptr, N: 0, SymTab: nullptr); |
1034 | |
1035 | InstrProfValueData VD42[] = { |
1036 | {.Value: uint64_t(callee1), .Count: 1}, {.Value: uint64_t(callee2), .Count: 2}, {.Value: uint64_t(callee3), .Count: 3}}; |
1037 | Record12.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 4, VData: VD42, N: 3, SymTab: nullptr); |
1038 | |
1039 | // 3 value sites for vtables. |
1040 | { |
1041 | Record12.reserveSites(ValueKind: IPVK_VTableTarget, NumValueSites: 3); |
1042 | InstrProfValueData VD0[] = {{.Value: getCalleeAddress(vtableAddr: vtable2), .Count: 5}, |
1043 | {.Value: getCalleeAddress(vtableAddr: vtable3), .Count: 3}}; |
1044 | Record12.addValueData(ValueKind: IPVK_VTableTarget, Site: 0, VData: VD0, N: 2, SymTab: nullptr); |
1045 | |
1046 | InstrProfValueData VD2[] = {{.Value: getCalleeAddress(vtableAddr: vtable2), .Count: 1}, |
1047 | {.Value: getCalleeAddress(vtableAddr: vtable3), .Count: 3}, |
1048 | {.Value: getCalleeAddress(vtableAddr: vtable4), .Count: 4}}; |
1049 | Record12.addValueData(ValueKind: IPVK_VTableTarget, Site: 1, VData: VD2, N: 3, SymTab: nullptr); |
1050 | |
1051 | InstrProfValueData VD4[] = {{.Value: getCalleeAddress(vtableAddr: vtable1), .Count: 1}, |
1052 | {.Value: getCalleeAddress(vtableAddr: vtable2), .Count: 2}, |
1053 | {.Value: getCalleeAddress(vtableAddr: vtable3), .Count: 3}}; |
1054 | Record12.addValueData(ValueKind: IPVK_VTableTarget, Site: 3, VData: VD4, N: 3, SymTab: nullptr); |
1055 | } |
1056 | |
1057 | Writer.addRecord(I: std::move(Record11), Warn: Err); |
1058 | // Merge profile data. |
1059 | Writer.addRecord(I: std::move(Record12), Warn: Err); |
1060 | |
1061 | Writer.addRecord(I: {callee1, 0x1235, {3, 4}}, Warn: Err); |
1062 | Writer.addRecord(I: {callee2, 0x1235, {3, 4}}, Warn: Err); |
1063 | Writer.addRecord(I: {callee3, 0x1235, {3, 4}}, Warn: Err); |
1064 | Writer.addRecord(I: {callee3, 0x1235, {3, 4}}, Warn: Err); |
1065 | Writer.addRecord(I: {callee4, 0x1235, {3, 5}}, Warn: Err); |
1066 | Writer.addRecord(I: {callee7, 0x1235, {3, 5}}, Warn: Err); |
1067 | Writer.addRecord(I: {callee8, 0x1235, {3, 5}}, Warn: Err); |
1068 | auto Profile = Writer.writeBuffer(); |
1069 | readProfile(Profile: std::move(Profile)); |
1070 | |
1071 | // Test the number of instrumented value sites and the number of profiled |
1072 | // values for each site. |
1073 | Expected<InstrProfRecord> R = Reader->getInstrProfRecord(FuncName: "caller" , FuncHash: 0x1234); |
1074 | EXPECT_THAT_ERROR(R.takeError(), Succeeded()); |
1075 | // For indirect calls. |
1076 | ASSERT_EQ(5U, R->getNumValueSites(IPVK_IndirectCallTarget)); |
1077 | ASSERT_EQ(4U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0)); |
1078 | ASSERT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1)); |
1079 | ASSERT_EQ(4U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2)); |
1080 | ASSERT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3)); |
1081 | ASSERT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 4)); |
1082 | // For vtables. |
1083 | ASSERT_EQ(R->getNumValueSites(IPVK_VTableTarget), 3U); |
1084 | ASSERT_EQ(R->getNumValueDataForSite(IPVK_VTableTarget, 0), 4U); |
1085 | ASSERT_EQ(R->getNumValueDataForSite(IPVK_VTableTarget, 1), 4U); |
1086 | ASSERT_EQ(R->getNumValueDataForSite(IPVK_VTableTarget, 2), 3U); |
1087 | |
1088 | // Test the merged values for indirect calls. |
1089 | { |
1090 | auto VD = R->getValueForSite(ValueKind: IPVK_IndirectCallTarget, Site: 0); |
1091 | EXPECT_STREQ((const char *)VD[0].Value, "callee2" ); |
1092 | EXPECT_EQ(VD[0].Count, 7U); |
1093 | EXPECT_STREQ((const char *)VD[1].Value, "callee3" ); |
1094 | EXPECT_EQ(VD[1].Count, 6U); |
1095 | EXPECT_STREQ((const char *)VD[2].Value, "callee4" ); |
1096 | EXPECT_EQ(VD[2].Count, 4U); |
1097 | EXPECT_STREQ((const char *)VD[3].Value, "callee1" ); |
1098 | EXPECT_EQ(VD[3].Count, 1U); |
1099 | |
1100 | auto VD_2(R->getValueForSite(ValueKind: IPVK_IndirectCallTarget, Site: 2)); |
1101 | EXPECT_STREQ((const char *)VD_2[0].Value, "callee3" ); |
1102 | EXPECT_EQ(VD_2[0].Count, 6U); |
1103 | EXPECT_STREQ((const char *)VD_2[1].Value, "callee4" ); |
1104 | EXPECT_EQ(VD_2[1].Count, 4U); |
1105 | EXPECT_STREQ((const char *)VD_2[2].Value, "callee2" ); |
1106 | EXPECT_EQ(VD_2[2].Count, 3U); |
1107 | EXPECT_STREQ((const char *)VD_2[3].Value, "callee1" ); |
1108 | EXPECT_EQ(VD_2[3].Count, 1U); |
1109 | |
1110 | auto VD_3(R->getValueForSite(ValueKind: IPVK_IndirectCallTarget, Site: 3)); |
1111 | EXPECT_STREQ((const char *)VD_3[0].Value, "callee8" ); |
1112 | EXPECT_EQ(VD_3[0].Count, 2U); |
1113 | EXPECT_STREQ((const char *)VD_3[1].Value, "callee7" ); |
1114 | EXPECT_EQ(VD_3[1].Count, 1U); |
1115 | |
1116 | auto VD_4(R->getValueForSite(ValueKind: IPVK_IndirectCallTarget, Site: 4)); |
1117 | EXPECT_STREQ((const char *)VD_4[0].Value, "callee3" ); |
1118 | EXPECT_EQ(VD_4[0].Count, 6U); |
1119 | EXPECT_STREQ((const char *)VD_4[1].Value, "callee2" ); |
1120 | EXPECT_EQ(VD_4[1].Count, 4U); |
1121 | EXPECT_STREQ((const char *)VD_4[2].Value, "callee1" ); |
1122 | EXPECT_EQ(VD_4[2].Count, 2U); |
1123 | } |
1124 | |
1125 | // Test the merged values for vtables |
1126 | { |
1127 | auto VD0 = R->getValueForSite(ValueKind: IPVK_VTableTarget, Site: 0); |
1128 | EXPECT_EQ(VD0[0].Value, getCalleeAddress(vtable2)); |
1129 | EXPECT_EQ(VD0[0].Count, 7U); |
1130 | EXPECT_EQ(VD0[1].Value, getCalleeAddress(vtable3)); |
1131 | EXPECT_EQ(VD0[1].Count, 6U); |
1132 | EXPECT_EQ(VD0[2].Value, getCalleeAddress(vtable4)); |
1133 | EXPECT_EQ(VD0[2].Count, 4U); |
1134 | EXPECT_EQ(VD0[3].Value, getCalleeAddress(vtable1)); |
1135 | EXPECT_EQ(VD0[3].Count, 1U); |
1136 | |
1137 | auto VD1 = R->getValueForSite(ValueKind: IPVK_VTableTarget, Site: 1); |
1138 | EXPECT_EQ(VD1[0].Value, getCalleeAddress(vtable3)); |
1139 | EXPECT_EQ(VD1[0].Count, 6U); |
1140 | EXPECT_EQ(VD1[1].Value, getCalleeAddress(vtable4)); |
1141 | EXPECT_EQ(VD1[1].Count, 4U); |
1142 | EXPECT_EQ(VD1[2].Value, getCalleeAddress(vtable2)); |
1143 | EXPECT_EQ(VD1[2].Count, 3U); |
1144 | EXPECT_EQ(VD1[3].Value, getCalleeAddress(vtable1)); |
1145 | EXPECT_EQ(VD1[3].Count, 1U); |
1146 | |
1147 | auto VD2 = R->getValueForSite(ValueKind: IPVK_VTableTarget, Site: 2); |
1148 | EXPECT_EQ(VD2[0].Value, getCalleeAddress(vtable3)); |
1149 | EXPECT_EQ(VD2[0].Count, 6U); |
1150 | EXPECT_EQ(VD2[1].Value, getCalleeAddress(vtable2)); |
1151 | EXPECT_EQ(VD2[1].Count, 4U); |
1152 | EXPECT_EQ(VD2[2].Value, getCalleeAddress(vtable1)); |
1153 | EXPECT_EQ(VD2[2].Count, 2U); |
1154 | } |
1155 | } |
1156 | |
1157 | struct ValueProfileMergeEdgeCaseTest |
1158 | : public InstrProfTest, |
1159 | public ::testing::WithParamInterface<std::tuple<bool, uint32_t>> { |
1160 | void SetUp() override { Writer.setOutputSparse(std::get<0>(t: GetParam())); } |
1161 | |
1162 | uint32_t getValueProfileKind() const { return std::get<1>(t: GetParam()); } |
1163 | }; |
1164 | |
1165 | TEST_P(ValueProfileMergeEdgeCaseTest, value_profile_data_merge_saturation) { |
1166 | const uint32_t ValueKind = getValueProfileKind(); |
1167 | static const char bar[] = "bar" ; |
1168 | const uint64_t ProfiledValue = 0x5678; |
1169 | |
1170 | const uint64_t MaxValCount = std::numeric_limits<uint64_t>::max(); |
1171 | const uint64_t MaxEdgeCount = getInstrMaxCountValue(); |
1172 | |
1173 | instrprof_error Result; |
1174 | auto Err = [&](Error E) { |
1175 | Result = std::get<0>(in: InstrProfError::take(E: std::move(E))); |
1176 | }; |
1177 | Result = instrprof_error::success; |
1178 | Writer.addRecord(I: {"foo" , 0x1234, {1}}, Warn: Err); |
1179 | ASSERT_EQ(Result, instrprof_error::success); |
1180 | |
1181 | // Verify counter overflow. |
1182 | Result = instrprof_error::success; |
1183 | Writer.addRecord(I: {"foo" , 0x1234, {MaxEdgeCount}}, Warn: Err); |
1184 | ASSERT_EQ(Result, instrprof_error::counter_overflow); |
1185 | |
1186 | Result = instrprof_error::success; |
1187 | Writer.addRecord(I: {bar, 0x9012, {8}}, Warn: Err); |
1188 | ASSERT_EQ(Result, instrprof_error::success); |
1189 | |
1190 | NamedInstrProfRecord Record4("baz" , 0x5678, {3, 4}); |
1191 | Record4.reserveSites(ValueKind, NumValueSites: 1); |
1192 | InstrProfValueData VD4[] = {{.Value: ProfiledValue, .Count: 1}}; |
1193 | Record4.addValueData(ValueKind, Site: 0, VData: VD4, N: 1, SymTab: nullptr); |
1194 | Result = instrprof_error::success; |
1195 | Writer.addRecord(I: std::move(Record4), Warn: Err); |
1196 | ASSERT_EQ(Result, instrprof_error::success); |
1197 | |
1198 | // Verify value data counter overflow. |
1199 | NamedInstrProfRecord Record5("baz" , 0x5678, {5, 6}); |
1200 | Record5.reserveSites(ValueKind, NumValueSites: 1); |
1201 | InstrProfValueData VD5[] = {{.Value: ProfiledValue, .Count: MaxValCount}}; |
1202 | Record5.addValueData(ValueKind, Site: 0, VData: VD5, N: 1, SymTab: nullptr); |
1203 | Result = instrprof_error::success; |
1204 | Writer.addRecord(I: std::move(Record5), Warn: Err); |
1205 | ASSERT_EQ(Result, instrprof_error::counter_overflow); |
1206 | |
1207 | auto Profile = Writer.writeBuffer(); |
1208 | readProfile(Profile: std::move(Profile)); |
1209 | |
1210 | // Verify saturation of counts. |
1211 | Expected<InstrProfRecord> ReadRecord1 = |
1212 | Reader->getInstrProfRecord(FuncName: "foo" , FuncHash: 0x1234); |
1213 | ASSERT_THAT_ERROR(ReadRecord1.takeError(), Succeeded()); |
1214 | EXPECT_EQ(MaxEdgeCount, ReadRecord1->Counts[0]); |
1215 | |
1216 | Expected<InstrProfRecord> ReadRecord2 = |
1217 | Reader->getInstrProfRecord(FuncName: "baz" , FuncHash: 0x5678); |
1218 | ASSERT_TRUE(bool(ReadRecord2)); |
1219 | ASSERT_EQ(1U, ReadRecord2->getNumValueSites(ValueKind)); |
1220 | std::unique_ptr<InstrProfValueData[]> VD = |
1221 | ReadRecord2->getValueForSite(ValueKind, Site: 0); |
1222 | EXPECT_EQ(ProfiledValue, VD[0].Value); |
1223 | EXPECT_EQ(MaxValCount, VD[0].Count); |
1224 | } |
1225 | |
1226 | // This test tests that when there are too many values for a given site, the |
1227 | // merged results are properly truncated. |
1228 | TEST_P(ValueProfileMergeEdgeCaseTest, value_profile_data_merge_site_trunc) { |
1229 | const uint32_t ValueKind = getValueProfileKind(); |
1230 | static const char caller[] = "caller" ; |
1231 | |
1232 | NamedInstrProfRecord Record11(caller, 0x1234, {1, 2}); |
1233 | NamedInstrProfRecord Record12(caller, 0x1234, {1, 2}); |
1234 | |
1235 | // 2 value sites. |
1236 | Record11.reserveSites(ValueKind, NumValueSites: 2); |
1237 | InstrProfValueData VD0[255]; |
1238 | for (int I = 0; I < 255; I++) { |
1239 | VD0[I].Value = 2 * I; |
1240 | VD0[I].Count = 2 * I + 1000; |
1241 | } |
1242 | |
1243 | Record11.addValueData(ValueKind, Site: 0, VData: VD0, N: 255, SymTab: nullptr); |
1244 | Record11.addValueData(ValueKind, Site: 1, VData: nullptr, N: 0, SymTab: nullptr); |
1245 | |
1246 | Record12.reserveSites(ValueKind, NumValueSites: 2); |
1247 | InstrProfValueData VD1[255]; |
1248 | for (int I = 0; I < 255; I++) { |
1249 | VD1[I].Value = 2 * I + 1; |
1250 | VD1[I].Count = 2 * I + 1001; |
1251 | } |
1252 | |
1253 | Record12.addValueData(ValueKind, Site: 0, VData: VD1, N: 255, SymTab: nullptr); |
1254 | Record12.addValueData(ValueKind, Site: 1, VData: nullptr, N: 0, SymTab: nullptr); |
1255 | |
1256 | Writer.addRecord(I: std::move(Record11), Warn: Err); |
1257 | // Merge profile data. |
1258 | Writer.addRecord(I: std::move(Record12), Warn: Err); |
1259 | |
1260 | auto Profile = Writer.writeBuffer(); |
1261 | readProfile(Profile: std::move(Profile)); |
1262 | |
1263 | Expected<InstrProfRecord> R = Reader->getInstrProfRecord(FuncName: "caller" , FuncHash: 0x1234); |
1264 | ASSERT_THAT_ERROR(R.takeError(), Succeeded()); |
1265 | std::unique_ptr<InstrProfValueData[]> VD(R->getValueForSite(ValueKind, Site: 0)); |
1266 | ASSERT_EQ(2U, R->getNumValueSites(ValueKind)); |
1267 | EXPECT_EQ(255U, R->getNumValueDataForSite(ValueKind, 0)); |
1268 | for (unsigned I = 0; I < 255; I++) { |
1269 | EXPECT_EQ(VD[I].Value, 509 - I); |
1270 | EXPECT_EQ(VD[I].Count, 1509 - I); |
1271 | } |
1272 | } |
1273 | |
1274 | INSTANTIATE_TEST_SUITE_P( |
1275 | EdgeCaseTest, ValueProfileMergeEdgeCaseTest, |
1276 | ::testing::Combine(::testing::Bool(), /* Sparse */ |
1277 | ::testing::Values(IPVK_IndirectCallTarget, |
1278 | IPVK_MemOPSize, |
1279 | IPVK_VTableTarget) /* ValueKind */ |
1280 | )); |
1281 | |
1282 | static void addValueProfData(InstrProfRecord &Record) { |
1283 | // Add test data for indirect calls. |
1284 | { |
1285 | Record.reserveSites(ValueKind: IPVK_IndirectCallTarget, NumValueSites: 6); |
1286 | InstrProfValueData VD0[] = {{.Value: uint64_t(callee1), .Count: 400}, |
1287 | {.Value: uint64_t(callee2), .Count: 1000}, |
1288 | {.Value: uint64_t(callee3), .Count: 500}, |
1289 | {.Value: uint64_t(callee4), .Count: 300}, |
1290 | {.Value: uint64_t(callee5), .Count: 100}}; |
1291 | Record.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 0, VData: VD0, N: 5, SymTab: nullptr); |
1292 | InstrProfValueData VD1[] = {{.Value: uint64_t(callee5), .Count: 800}, |
1293 | {.Value: uint64_t(callee3), .Count: 1000}, |
1294 | {.Value: uint64_t(callee2), .Count: 2500}, |
1295 | {.Value: uint64_t(callee1), .Count: 1300}}; |
1296 | Record.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 1, VData: VD1, N: 4, SymTab: nullptr); |
1297 | InstrProfValueData VD2[] = {{.Value: uint64_t(callee6), .Count: 800}, |
1298 | {.Value: uint64_t(callee3), .Count: 1000}, |
1299 | {.Value: uint64_t(callee4), .Count: 5500}}; |
1300 | Record.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 2, VData: VD2, N: 3, SymTab: nullptr); |
1301 | InstrProfValueData VD3[] = {{.Value: uint64_t(callee2), .Count: 1800}, |
1302 | {.Value: uint64_t(callee3), .Count: 2000}}; |
1303 | Record.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 3, VData: VD3, N: 2, SymTab: nullptr); |
1304 | Record.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 4, VData: nullptr, N: 0, SymTab: nullptr); |
1305 | InstrProfValueData VD5[] = {{.Value: uint64_t(callee7), .Count: 1234}, |
1306 | {.Value: uint64_t(callee8), .Count: 5678}}; |
1307 | Record.addValueData(ValueKind: IPVK_IndirectCallTarget, Site: 5, VData: VD5, N: 2, SymTab: nullptr); |
1308 | } |
1309 | |
1310 | // Add test data for vtables |
1311 | { |
1312 | Record.reserveSites(ValueKind: IPVK_VTableTarget, NumValueSites: 4); |
1313 | InstrProfValueData VD0[] = { |
1314 | {.Value: getCalleeAddress(vtableAddr: vtable1), .Count: 400}, {.Value: getCalleeAddress(vtableAddr: vtable2), .Count: 1000}, |
1315 | {.Value: getCalleeAddress(vtableAddr: vtable3), .Count: 500}, {.Value: getCalleeAddress(vtableAddr: vtable4), .Count: 300}, |
1316 | {.Value: getCalleeAddress(vtableAddr: vtable5), .Count: 100}, |
1317 | }; |
1318 | InstrProfValueData VD1[] = {{.Value: getCalleeAddress(vtableAddr: vtable5), .Count: 800}, |
1319 | {.Value: getCalleeAddress(vtableAddr: vtable3), .Count: 1000}, |
1320 | {.Value: getCalleeAddress(vtableAddr: vtable2), .Count: 2500}, |
1321 | {.Value: getCalleeAddress(vtableAddr: vtable1), .Count: 1300}}; |
1322 | InstrProfValueData VD2[] = { |
1323 | {.Value: getCalleeAddress(vtableAddr: vtable6), .Count: 800}, |
1324 | {.Value: getCalleeAddress(vtableAddr: vtable3), .Count: 1000}, |
1325 | {.Value: getCalleeAddress(vtableAddr: vtable4), .Count: 5500}, |
1326 | }; |
1327 | InstrProfValueData VD3[] = {{.Value: getCalleeAddress(vtableAddr: vtable2), .Count: 1800}, |
1328 | {.Value: getCalleeAddress(vtableAddr: vtable3), .Count: 2000}}; |
1329 | Record.addValueData(ValueKind: IPVK_VTableTarget, Site: 0, VData: VD0, N: 5, SymTab: nullptr); |
1330 | Record.addValueData(ValueKind: IPVK_VTableTarget, Site: 1, VData: VD1, N: 4, SymTab: nullptr); |
1331 | Record.addValueData(ValueKind: IPVK_VTableTarget, Site: 2, VData: VD2, N: 3, SymTab: nullptr); |
1332 | Record.addValueData(ValueKind: IPVK_VTableTarget, Site: 3, VData: VD3, N: 2, SymTab: nullptr); |
1333 | } |
1334 | } |
1335 | |
1336 | TEST(ValueProfileReadWriteTest, value_prof_data_read_write) { |
1337 | InstrProfRecord SrcRecord({1ULL << 31, 2}); |
1338 | addValueProfData(Record&: SrcRecord); |
1339 | std::unique_ptr<ValueProfData> VPData = |
1340 | ValueProfData::serializeFrom(Record: SrcRecord); |
1341 | |
1342 | InstrProfRecord Record({1ULL << 31, 2}); |
1343 | VPData->deserializeTo(Record, SymTab: nullptr); |
1344 | |
1345 | // Now read data from Record and sanity check the data |
1346 | ASSERT_EQ(6U, Record.getNumValueSites(IPVK_IndirectCallTarget)); |
1347 | ASSERT_EQ(5U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 0)); |
1348 | ASSERT_EQ(4U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 1)); |
1349 | ASSERT_EQ(3U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 2)); |
1350 | ASSERT_EQ(2U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 3)); |
1351 | ASSERT_EQ(0U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 4)); |
1352 | ASSERT_EQ(2U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 5)); |
1353 | |
1354 | auto Cmp = [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) { |
1355 | return VD1.Count > VD2.Count; |
1356 | }; |
1357 | |
1358 | std::unique_ptr<InstrProfValueData[]> VD_0( |
1359 | Record.getValueForSite(ValueKind: IPVK_IndirectCallTarget, Site: 0)); |
1360 | llvm::sort(Start: &VD_0[0], End: &VD_0[5], Comp: Cmp); |
1361 | EXPECT_STREQ((const char *)VD_0[0].Value, "callee2" ); |
1362 | EXPECT_EQ(1000U, VD_0[0].Count); |
1363 | EXPECT_STREQ((const char *)VD_0[1].Value, "callee3" ); |
1364 | EXPECT_EQ(500U, VD_0[1].Count); |
1365 | EXPECT_STREQ((const char *)VD_0[2].Value, "callee1" ); |
1366 | EXPECT_EQ(400U, VD_0[2].Count); |
1367 | EXPECT_STREQ((const char *)VD_0[3].Value, "callee4" ); |
1368 | EXPECT_EQ(300U, VD_0[3].Count); |
1369 | EXPECT_STREQ((const char *)VD_0[4].Value, "callee5" ); |
1370 | EXPECT_EQ(100U, VD_0[4].Count); |
1371 | |
1372 | std::unique_ptr<InstrProfValueData[]> VD_1( |
1373 | Record.getValueForSite(ValueKind: IPVK_IndirectCallTarget, Site: 1)); |
1374 | llvm::sort(Start: &VD_1[0], End: &VD_1[4], Comp: Cmp); |
1375 | EXPECT_STREQ((const char *)VD_1[0].Value, "callee2" ); |
1376 | EXPECT_EQ(VD_1[0].Count, 2500U); |
1377 | EXPECT_STREQ((const char *)VD_1[1].Value, "callee1" ); |
1378 | EXPECT_EQ(VD_1[1].Count, 1300U); |
1379 | EXPECT_STREQ((const char *)VD_1[2].Value, "callee3" ); |
1380 | EXPECT_EQ(VD_1[2].Count, 1000U); |
1381 | EXPECT_STREQ((const char *)VD_1[3].Value, "callee5" ); |
1382 | EXPECT_EQ(VD_1[3].Count, 800U); |
1383 | |
1384 | std::unique_ptr<InstrProfValueData[]> VD_2( |
1385 | Record.getValueForSite(ValueKind: IPVK_IndirectCallTarget, Site: 2)); |
1386 | llvm::sort(Start: &VD_2[0], End: &VD_2[3], Comp: Cmp); |
1387 | EXPECT_STREQ((const char *)VD_2[0].Value, "callee4" ); |
1388 | EXPECT_EQ(VD_2[0].Count, 5500U); |
1389 | EXPECT_STREQ((const char *)VD_2[1].Value, "callee3" ); |
1390 | EXPECT_EQ(VD_2[1].Count, 1000U); |
1391 | EXPECT_STREQ((const char *)VD_2[2].Value, "callee6" ); |
1392 | EXPECT_EQ(VD_2[2].Count, 800U); |
1393 | |
1394 | std::unique_ptr<InstrProfValueData[]> VD_3( |
1395 | Record.getValueForSite(ValueKind: IPVK_IndirectCallTarget, Site: 3)); |
1396 | llvm::sort(Start: &VD_3[0], End: &VD_3[2], Comp: Cmp); |
1397 | EXPECT_STREQ((const char *)VD_3[0].Value, "callee3" ); |
1398 | EXPECT_EQ(VD_3[0].Count, 2000U); |
1399 | EXPECT_STREQ((const char *)VD_3[1].Value, "callee2" ); |
1400 | EXPECT_EQ(VD_3[1].Count, 1800U); |
1401 | |
1402 | ASSERT_EQ(Record.getNumValueSites(IPVK_VTableTarget), 4U); |
1403 | ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 0), 5U); |
1404 | ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 1), 4U); |
1405 | ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 2), 3U); |
1406 | ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 3), 2U); |
1407 | |
1408 | auto VD0(Record.getValueForSite(ValueKind: IPVK_VTableTarget, Site: 0)); |
1409 | llvm::sort(Start: &VD0[0], End: &VD0[5], Comp: Cmp); |
1410 | EXPECT_EQ(VD0[0].Value, getCalleeAddress(vtable2)); |
1411 | EXPECT_EQ(VD0[0].Count, 1000U); |
1412 | EXPECT_EQ(VD0[1].Value, getCalleeAddress(vtable3)); |
1413 | EXPECT_EQ(VD0[1].Count, 500U); |
1414 | EXPECT_EQ(VD0[2].Value, getCalleeAddress(vtable1)); |
1415 | EXPECT_EQ(VD0[2].Count, 400U); |
1416 | EXPECT_EQ(VD0[3].Value, getCalleeAddress(vtable4)); |
1417 | EXPECT_EQ(VD0[3].Count, 300U); |
1418 | EXPECT_EQ(VD0[4].Value, getCalleeAddress(vtable5)); |
1419 | EXPECT_EQ(VD0[4].Count, 100U); |
1420 | |
1421 | auto VD1(Record.getValueForSite(ValueKind: IPVK_VTableTarget, Site: 1)); |
1422 | llvm::sort(Start: &VD1[0], End: &VD1[4], Comp: Cmp); |
1423 | EXPECT_EQ(VD1[0].Value, getCalleeAddress(vtable2)); |
1424 | EXPECT_EQ(VD1[0].Count, 2500U); |
1425 | EXPECT_EQ(VD1[1].Value, getCalleeAddress(vtable1)); |
1426 | EXPECT_EQ(VD1[1].Count, 1300U); |
1427 | EXPECT_EQ(VD1[2].Value, getCalleeAddress(vtable3)); |
1428 | EXPECT_EQ(VD1[2].Count, 1000U); |
1429 | EXPECT_EQ(VD1[3].Value, getCalleeAddress(vtable5)); |
1430 | EXPECT_EQ(VD1[3].Count, 800U); |
1431 | |
1432 | auto VD2(Record.getValueForSite(ValueKind: IPVK_VTableTarget, Site: 2)); |
1433 | llvm::sort(Start: &VD2[0], End: &VD2[3], Comp: Cmp); |
1434 | EXPECT_EQ(VD2[0].Value, getCalleeAddress(vtable4)); |
1435 | EXPECT_EQ(VD2[0].Count, 5500U); |
1436 | EXPECT_EQ(VD2[1].Value, getCalleeAddress(vtable3)); |
1437 | EXPECT_EQ(VD2[1].Count, 1000U); |
1438 | EXPECT_EQ(VD2[2].Value, getCalleeAddress(vtable6)); |
1439 | EXPECT_EQ(VD2[2].Count, 800U); |
1440 | |
1441 | auto VD3(Record.getValueForSite(ValueKind: IPVK_VTableTarget, Site: 3)); |
1442 | llvm::sort(Start: &VD3[0], End: &VD3[2], Comp: Cmp); |
1443 | EXPECT_EQ(VD3[0].Value, getCalleeAddress(vtable3)); |
1444 | EXPECT_EQ(VD3[0].Count, 2000U); |
1445 | EXPECT_EQ(VD3[1].Value, getCalleeAddress(vtable2)); |
1446 | EXPECT_EQ(VD3[1].Count, 1800U); |
1447 | } |
1448 | |
1449 | TEST(ValueProfileReadWriteTest, symtab_mapping) { |
1450 | NamedInstrProfRecord SrcRecord("caller" , 0x1234, {1ULL << 31, 2}); |
1451 | addValueProfData(Record&: SrcRecord); |
1452 | std::unique_ptr<ValueProfData> VPData = |
1453 | ValueProfData::serializeFrom(Record: SrcRecord); |
1454 | |
1455 | NamedInstrProfRecord Record("caller" , 0x1234, {1ULL << 31, 2}); |
1456 | InstrProfSymtab Symtab; |
1457 | Symtab.mapAddress(Addr: uint64_t(callee1), MD5Val: 0x1000ULL); |
1458 | Symtab.mapAddress(Addr: uint64_t(callee2), MD5Val: 0x2000ULL); |
1459 | Symtab.mapAddress(Addr: uint64_t(callee3), MD5Val: 0x3000ULL); |
1460 | Symtab.mapAddress(Addr: uint64_t(callee4), MD5Val: 0x4000ULL); |
1461 | // Missing mapping for callee5 |
1462 | |
1463 | auto getVTableStartAddr = [](const uint64_t *vtable) -> uint64_t { |
1464 | return uint64_t(vtable); |
1465 | }; |
1466 | auto getVTableEndAddr = [](const uint64_t *vtable) -> uint64_t { |
1467 | return uint64_t(vtable) + 16; |
1468 | }; |
1469 | auto getVTableMidAddr = [](const uint64_t *vtable) -> uint64_t { |
1470 | return uint64_t(vtable) + 8; |
1471 | }; |
1472 | // vtable1, vtable2, vtable3, vtable4 get mapped; vtable5, vtable6 are not |
1473 | // mapped. |
1474 | Symtab.mapVTableAddress(StartAddr: getVTableStartAddr(vtable1), |
1475 | EndAddr: getVTableEndAddr(vtable1), MD5Val: MD5Hash(Str: "vtable1" )); |
1476 | Symtab.mapVTableAddress(StartAddr: getVTableStartAddr(vtable2), |
1477 | EndAddr: getVTableEndAddr(vtable2), MD5Val: MD5Hash(Str: "vtable2" )); |
1478 | Symtab.mapVTableAddress(StartAddr: getVTableStartAddr(vtable3), |
1479 | EndAddr: getVTableEndAddr(vtable3), MD5Val: MD5Hash(Str: "vtable3" )); |
1480 | Symtab.mapVTableAddress(StartAddr: getVTableStartAddr(vtable4), |
1481 | EndAddr: getVTableEndAddr(vtable4), MD5Val: MD5Hash(Str: "vtable4" )); |
1482 | |
1483 | VPData->deserializeTo(Record, SymTab: &Symtab); |
1484 | |
1485 | // Now read data from Record and sanity check the data |
1486 | ASSERT_EQ(Record.getNumValueSites(IPVK_IndirectCallTarget), 6U); |
1487 | ASSERT_EQ(Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 0), 5U); |
1488 | |
1489 | // Look up the value correpsonding to the middle of a vtable in symtab and |
1490 | // test that it's the hash of the name. |
1491 | EXPECT_EQ(Symtab.getVTableHashFromAddress(getVTableMidAddr(vtable1)), |
1492 | MD5Hash("vtable1" )); |
1493 | EXPECT_EQ(Symtab.getVTableHashFromAddress(getVTableMidAddr(vtable2)), |
1494 | MD5Hash("vtable2" )); |
1495 | EXPECT_EQ(Symtab.getVTableHashFromAddress(getVTableMidAddr(vtable3)), |
1496 | MD5Hash("vtable3" )); |
1497 | EXPECT_EQ(Symtab.getVTableHashFromAddress(getVTableMidAddr(vtable4)), |
1498 | MD5Hash("vtable4" )); |
1499 | |
1500 | auto Cmp = [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) { |
1501 | return VD1.Count > VD2.Count; |
1502 | }; |
1503 | auto VD_0(Record.getValueForSite(ValueKind: IPVK_IndirectCallTarget, Site: 0)); |
1504 | llvm::sort(Start: &VD_0[0], End: &VD_0[5], Comp: Cmp); |
1505 | ASSERT_EQ(VD_0[0].Value, 0x2000ULL); |
1506 | ASSERT_EQ(VD_0[0].Count, 1000U); |
1507 | ASSERT_EQ(VD_0[1].Value, 0x3000ULL); |
1508 | ASSERT_EQ(VD_0[1].Count, 500U); |
1509 | ASSERT_EQ(VD_0[2].Value, 0x1000ULL); |
1510 | ASSERT_EQ(VD_0[2].Count, 400U); |
1511 | |
1512 | // callee5 does not have a mapped value -- default to 0. |
1513 | ASSERT_EQ(VD_0[4].Value, 0ULL); |
1514 | |
1515 | // Sanity check the vtable value data |
1516 | ASSERT_EQ(Record.getNumValueSites(IPVK_VTableTarget), 4U); |
1517 | |
1518 | { |
1519 | // The first vtable site. |
1520 | auto VD(Record.getValueForSite(ValueKind: IPVK_VTableTarget, Site: 0)); |
1521 | ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 0), 5U); |
1522 | llvm::sort(Start: &VD[0], End: &VD[5], Comp: Cmp); |
1523 | EXPECT_EQ(VD[0].Count, 1000U); |
1524 | EXPECT_EQ(VD[0].Value, MD5Hash("vtable2" )); |
1525 | EXPECT_EQ(VD[1].Count, 500U); |
1526 | EXPECT_EQ(VD[1].Value, MD5Hash("vtable3" )); |
1527 | EXPECT_EQ(VD[2].Value, MD5Hash("vtable1" )); |
1528 | EXPECT_EQ(VD[2].Count, 400U); |
1529 | EXPECT_EQ(VD[3].Value, MD5Hash("vtable4" )); |
1530 | EXPECT_EQ(VD[3].Count, 300U); |
1531 | |
1532 | // vtable5 isn't mapped -- default to 0. |
1533 | EXPECT_EQ(VD[4].Value, 0U); |
1534 | EXPECT_EQ(VD[4].Count, 100U); |
1535 | } |
1536 | |
1537 | { |
1538 | // The second vtable site. |
1539 | auto VD(Record.getValueForSite(ValueKind: IPVK_VTableTarget, Site: 1)); |
1540 | ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 1), 4U); |
1541 | llvm::sort(Start: &VD[0], End: &VD[4], Comp: Cmp); |
1542 | EXPECT_EQ(VD[0].Value, MD5Hash("vtable2" )); |
1543 | EXPECT_EQ(VD[0].Count, 2500U); |
1544 | EXPECT_EQ(VD[1].Value, MD5Hash("vtable1" )); |
1545 | EXPECT_EQ(VD[1].Count, 1300U); |
1546 | |
1547 | EXPECT_EQ(VD[2].Value, MD5Hash("vtable3" )); |
1548 | EXPECT_EQ(VD[2].Count, 1000U); |
1549 | // vtable5 isn't mapped -- default to 0. |
1550 | EXPECT_EQ(VD[3].Value, 0U); |
1551 | EXPECT_EQ(VD[3].Count, 800U); |
1552 | } |
1553 | |
1554 | { |
1555 | // The third vtable site. |
1556 | auto VD(Record.getValueForSite(ValueKind: IPVK_VTableTarget, Site: 2)); |
1557 | ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 2), 3U); |
1558 | llvm::sort(Start: &VD[0], End: &VD[3], Comp: Cmp); |
1559 | EXPECT_EQ(VD[0].Count, 5500U); |
1560 | EXPECT_EQ(VD[0].Value, MD5Hash("vtable4" )); |
1561 | EXPECT_EQ(VD[1].Count, 1000U); |
1562 | EXPECT_EQ(VD[1].Value, MD5Hash("vtable3" )); |
1563 | // vtable6 isn't mapped -- default to 0. |
1564 | EXPECT_EQ(VD[2].Value, 0U); |
1565 | EXPECT_EQ(VD[2].Count, 800U); |
1566 | } |
1567 | |
1568 | { |
1569 | // The fourth vtable site. |
1570 | auto VD(Record.getValueForSite(ValueKind: IPVK_VTableTarget, Site: 3)); |
1571 | ASSERT_EQ(Record.getNumValueDataForSite(IPVK_VTableTarget, 3), 2U); |
1572 | llvm::sort(Start: &VD[0], End: &VD[2], Comp: Cmp); |
1573 | EXPECT_EQ(VD[0].Count, 2000U); |
1574 | EXPECT_EQ(VD[0].Value, MD5Hash("vtable3" )); |
1575 | EXPECT_EQ(VD[1].Count, 1800U); |
1576 | EXPECT_EQ(VD[1].Value, MD5Hash("vtable2" )); |
1577 | } |
1578 | } |
1579 | |
1580 | TEST_P(MaybeSparseInstrProfTest, get_max_function_count) { |
1581 | Writer.addRecord(I: {"foo" , 0x1234, {1ULL << 31, 2}}, Warn: Err); |
1582 | Writer.addRecord(I: {"bar" , 0, {1ULL << 63}}, Warn: Err); |
1583 | Writer.addRecord(I: {"baz" , 0x5678, {0, 0, 0, 0}}, Warn: Err); |
1584 | auto Profile = Writer.writeBuffer(); |
1585 | readProfile(Profile: std::move(Profile)); |
1586 | |
1587 | ASSERT_EQ(1ULL << 63, Reader->getMaximumFunctionCount(/* IsCS */ false)); |
1588 | } |
1589 | |
1590 | TEST_P(MaybeSparseInstrProfTest, get_weighted_function_counts) { |
1591 | Writer.addRecord(I: {"foo" , 0x1234, {1, 2}}, Weight: 3, Warn: Err); |
1592 | Writer.addRecord(I: {"foo" , 0x1235, {3, 4}}, Weight: 5, Warn: Err); |
1593 | auto Profile = Writer.writeBuffer(); |
1594 | readProfile(Profile: std::move(Profile)); |
1595 | |
1596 | std::vector<uint64_t> Counts; |
1597 | EXPECT_THAT_ERROR(Reader->getFunctionCounts("foo" , 0x1234, Counts), |
1598 | Succeeded()); |
1599 | ASSERT_EQ(2U, Counts.size()); |
1600 | ASSERT_EQ(3U, Counts[0]); |
1601 | ASSERT_EQ(6U, Counts[1]); |
1602 | |
1603 | EXPECT_THAT_ERROR(Reader->getFunctionCounts("foo" , 0x1235, Counts), |
1604 | Succeeded()); |
1605 | ASSERT_EQ(2U, Counts.size()); |
1606 | ASSERT_EQ(15U, Counts[0]); |
1607 | ASSERT_EQ(20U, Counts[1]); |
1608 | } |
1609 | |
1610 | // Testing symtab creator interface used by indexed profile reader. |
1611 | TEST(SymtabTest, instr_prof_symtab_test) { |
1612 | std::vector<StringRef> FuncNames; |
1613 | FuncNames.push_back(x: "func1" ); |
1614 | FuncNames.push_back(x: "func2" ); |
1615 | FuncNames.push_back(x: "func3" ); |
1616 | FuncNames.push_back(x: "bar1" ); |
1617 | FuncNames.push_back(x: "bar2" ); |
1618 | FuncNames.push_back(x: "bar3" ); |
1619 | InstrProfSymtab Symtab; |
1620 | EXPECT_THAT_ERROR(Symtab.create(FuncNames), Succeeded()); |
1621 | StringRef R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "func1" )); |
1622 | ASSERT_EQ(StringRef("func1" ), R); |
1623 | R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "func2" )); |
1624 | ASSERT_EQ(StringRef("func2" ), R); |
1625 | R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "func3" )); |
1626 | ASSERT_EQ(StringRef("func3" ), R); |
1627 | R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "bar1" )); |
1628 | ASSERT_EQ(StringRef("bar1" ), R); |
1629 | R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "bar2" )); |
1630 | ASSERT_EQ(StringRef("bar2" ), R); |
1631 | R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "bar3" )); |
1632 | ASSERT_EQ(StringRef("bar3" ), R); |
1633 | |
1634 | // negative tests |
1635 | R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "bar4" )); |
1636 | ASSERT_EQ(StringRef(), R); |
1637 | R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "foo4" )); |
1638 | ASSERT_EQ(StringRef(), R); |
1639 | |
1640 | // Now incrementally update the symtab |
1641 | EXPECT_THAT_ERROR(Symtab.addFuncName("blah_1" ), Succeeded()); |
1642 | EXPECT_THAT_ERROR(Symtab.addFuncName("blah_2" ), Succeeded()); |
1643 | EXPECT_THAT_ERROR(Symtab.addFuncName("blah_3" ), Succeeded()); |
1644 | |
1645 | // Check again |
1646 | R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "blah_1" )); |
1647 | ASSERT_EQ(StringRef("blah_1" ), R); |
1648 | R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "blah_2" )); |
1649 | ASSERT_EQ(StringRef("blah_2" ), R); |
1650 | R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "blah_3" )); |
1651 | ASSERT_EQ(StringRef("blah_3" ), R); |
1652 | R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "func1" )); |
1653 | ASSERT_EQ(StringRef("func1" ), R); |
1654 | R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "func2" )); |
1655 | ASSERT_EQ(StringRef("func2" ), R); |
1656 | R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "func3" )); |
1657 | ASSERT_EQ(StringRef("func3" ), R); |
1658 | R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "bar1" )); |
1659 | ASSERT_EQ(StringRef("bar1" ), R); |
1660 | R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "bar2" )); |
1661 | ASSERT_EQ(StringRef("bar2" ), R); |
1662 | R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: "bar3" )); |
1663 | ASSERT_EQ(StringRef("bar3" ), R); |
1664 | } |
1665 | |
1666 | // Test that we get an error when creating a bogus symtab. |
1667 | TEST(SymtabTest, instr_prof_bogus_symtab_empty_func_name) { |
1668 | InstrProfSymtab Symtab; |
1669 | EXPECT_TRUE(ErrorEquals(instrprof_error::malformed, Symtab.addFuncName("" ))); |
1670 | } |
1671 | |
1672 | // Testing symtab creator interface used by value profile transformer. |
1673 | TEST(SymtabTest, instr_prof_symtab_module_test) { |
1674 | LLVMContext Ctx; |
1675 | std::unique_ptr<Module> M = std::make_unique<Module>(args: "MyModule.cpp" , args&: Ctx); |
1676 | FunctionType *FTy = FunctionType::get(Result: Type::getVoidTy(C&: Ctx), |
1677 | /*isVarArg=*/false); |
1678 | Function::Create(Ty: FTy, Linkage: Function::ExternalLinkage, N: "Gfoo" , M: M.get()); |
1679 | Function::Create(Ty: FTy, Linkage: Function::ExternalLinkage, N: "Gblah" , M: M.get()); |
1680 | Function::Create(Ty: FTy, Linkage: Function::ExternalLinkage, N: "Gbar" , M: M.get()); |
1681 | Function::Create(Ty: FTy, Linkage: Function::InternalLinkage, N: "Ifoo" , M: M.get()); |
1682 | Function::Create(Ty: FTy, Linkage: Function::InternalLinkage, N: "Iblah" , M: M.get()); |
1683 | Function::Create(Ty: FTy, Linkage: Function::InternalLinkage, N: "Ibar" , M: M.get()); |
1684 | Function::Create(Ty: FTy, Linkage: Function::PrivateLinkage, N: "Pfoo" , M: M.get()); |
1685 | Function::Create(Ty: FTy, Linkage: Function::PrivateLinkage, N: "Pblah" , M: M.get()); |
1686 | Function::Create(Ty: FTy, Linkage: Function::PrivateLinkage, N: "Pbar" , M: M.get()); |
1687 | Function::Create(Ty: FTy, Linkage: Function::WeakODRLinkage, N: "Wfoo" , M: M.get()); |
1688 | Function::Create(Ty: FTy, Linkage: Function::WeakODRLinkage, N: "Wblah" , M: M.get()); |
1689 | Function::Create(Ty: FTy, Linkage: Function::WeakODRLinkage, N: "Wbar" , M: M.get()); |
1690 | |
1691 | InstrProfSymtab ProfSymtab; |
1692 | EXPECT_THAT_ERROR(ProfSymtab.create(*M), Succeeded()); |
1693 | |
1694 | StringRef Funcs[] = {"Gfoo" , "Gblah" , "Gbar" , "Ifoo" , "Iblah" , "Ibar" , |
1695 | "Pfoo" , "Pblah" , "Pbar" , "Wfoo" , "Wblah" , "Wbar" }; |
1696 | |
1697 | for (unsigned I = 0; I < std::size(Funcs); I++) { |
1698 | Function *F = M->getFunction(Name: Funcs[I]); |
1699 | |
1700 | std::string IRPGOName = getIRPGOFuncName(F: *F); |
1701 | auto IRPGOFuncName = |
1702 | ProfSymtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: IRPGOName)); |
1703 | EXPECT_EQ(IRPGOName, IRPGOFuncName); |
1704 | EXPECT_EQ(Funcs[I], getParsedIRPGOName(IRPGOFuncName).second); |
1705 | // Ensure we can still read this old record name. |
1706 | std::string PGOName = getPGOFuncName(F: *F); |
1707 | auto PGOFuncName = |
1708 | ProfSymtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: PGOName)); |
1709 | EXPECT_EQ(PGOName, PGOFuncName); |
1710 | EXPECT_THAT(PGOFuncName.str(), EndsWith(Funcs[I].str())); |
1711 | } |
1712 | } |
1713 | |
1714 | // Testing symtab serialization and creator/deserialization interface |
1715 | // used by coverage map reader, and raw profile reader. |
1716 | TEST(SymtabTest, instr_prof_symtab_compression_test) { |
1717 | std::vector<std::string> FuncNames1; |
1718 | std::vector<std::string> FuncNames2; |
1719 | for (int I = 0; I < 3; I++) { |
1720 | std::string str; |
1721 | raw_string_ostream OS(str); |
1722 | OS << "func_" << I; |
1723 | FuncNames1.push_back(x: OS.str()); |
1724 | str.clear(); |
1725 | OS << "f oooooooooooooo_" << I; |
1726 | FuncNames1.push_back(x: OS.str()); |
1727 | str.clear(); |
1728 | OS << "BAR_" << I; |
1729 | FuncNames2.push_back(x: OS.str()); |
1730 | str.clear(); |
1731 | OS << "BlahblahBlahblahBar_" << I; |
1732 | FuncNames2.push_back(x: OS.str()); |
1733 | } |
1734 | |
1735 | for (bool DoCompression : {false, true}) { |
1736 | // Compressing: |
1737 | std::string FuncNameStrings1; |
1738 | EXPECT_THAT_ERROR(collectGlobalObjectNameStrings( |
1739 | FuncNames1, |
1740 | (DoCompression && compression::zlib::isAvailable()), |
1741 | FuncNameStrings1), |
1742 | Succeeded()); |
1743 | |
1744 | // Compressing: |
1745 | std::string FuncNameStrings2; |
1746 | EXPECT_THAT_ERROR(collectGlobalObjectNameStrings( |
1747 | FuncNames2, |
1748 | (DoCompression && compression::zlib::isAvailable()), |
1749 | FuncNameStrings2), |
1750 | Succeeded()); |
1751 | |
1752 | for (int Padding = 0; Padding < 2; Padding++) { |
1753 | // Join with paddings : |
1754 | std::string FuncNameStrings = FuncNameStrings1; |
1755 | for (int P = 0; P < Padding; P++) { |
1756 | FuncNameStrings.push_back(c: '\0'); |
1757 | } |
1758 | FuncNameStrings += FuncNameStrings2; |
1759 | |
1760 | // Now decompress: |
1761 | InstrProfSymtab Symtab; |
1762 | EXPECT_THAT_ERROR(Symtab.create(StringRef(FuncNameStrings)), Succeeded()); |
1763 | |
1764 | // Now do the checks: |
1765 | // First sampling some data points: |
1766 | StringRef R = |
1767 | Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: FuncNames1[0])); |
1768 | ASSERT_EQ(StringRef("func_0" ), R); |
1769 | R = Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: FuncNames1[1])); |
1770 | ASSERT_EQ(StringRef("f oooooooooooooo_0" ), R); |
1771 | for (int I = 0; I < 3; I++) { |
1772 | std::string N[4]; |
1773 | N[0] = FuncNames1[2 * I]; |
1774 | N[1] = FuncNames1[2 * I + 1]; |
1775 | N[2] = FuncNames2[2 * I]; |
1776 | N[3] = FuncNames2[2 * I + 1]; |
1777 | for (int J = 0; J < 4; J++) { |
1778 | StringRef R = |
1779 | Symtab.getFuncOrVarName(MD5Hash: IndexedInstrProf::ComputeHash(K: N[J])); |
1780 | ASSERT_EQ(StringRef(N[J]), R); |
1781 | } |
1782 | } |
1783 | } |
1784 | } |
1785 | } |
1786 | |
1787 | TEST_P(MaybeSparseInstrProfTest, remapping_test) { |
1788 | Writer.addRecord(I: {"_Z3fooi" , 0x1234, {1, 2, 3, 4}}, Warn: Err); |
1789 | Writer.addRecord(I: {"file;_Z3barf" , 0x567, {5, 6, 7}}, Warn: Err); |
1790 | auto Profile = Writer.writeBuffer(); |
1791 | readProfile(Profile: std::move(Profile), Remapping: llvm::MemoryBuffer::getMemBuffer(InputData: R"( |
1792 | type i l |
1793 | name 3bar 4quux |
1794 | )" )); |
1795 | |
1796 | std::vector<uint64_t> Counts; |
1797 | for (StringRef FooName : {"_Z3fooi" , "_Z3fool" }) { |
1798 | EXPECT_THAT_ERROR(Reader->getFunctionCounts(FooName, 0x1234, Counts), |
1799 | Succeeded()); |
1800 | ASSERT_EQ(4u, Counts.size()); |
1801 | EXPECT_EQ(1u, Counts[0]); |
1802 | EXPECT_EQ(2u, Counts[1]); |
1803 | EXPECT_EQ(3u, Counts[2]); |
1804 | EXPECT_EQ(4u, Counts[3]); |
1805 | } |
1806 | |
1807 | for (StringRef BarName : {"file;_Z3barf" , "file;_Z4quuxf" }) { |
1808 | EXPECT_THAT_ERROR(Reader->getFunctionCounts(BarName, 0x567, Counts), |
1809 | Succeeded()); |
1810 | ASSERT_EQ(3u, Counts.size()); |
1811 | EXPECT_EQ(5u, Counts[0]); |
1812 | EXPECT_EQ(6u, Counts[1]); |
1813 | EXPECT_EQ(7u, Counts[2]); |
1814 | } |
1815 | |
1816 | for (StringRef BadName : {"_Z3foof" , "_Z4quuxi" , "_Z3barl" , "" , "_ZZZ" , |
1817 | "_Z3barf" , "otherfile:_Z4quuxf" }) { |
1818 | EXPECT_THAT_ERROR(Reader->getFunctionCounts(BadName, 0x1234, Counts), |
1819 | Failed()); |
1820 | EXPECT_THAT_ERROR(Reader->getFunctionCounts(BadName, 0x567, Counts), |
1821 | Failed()); |
1822 | } |
1823 | } |
1824 | |
1825 | TEST_F(SparseInstrProfTest, preserve_no_records) { |
1826 | Writer.addRecord(I: {"foo" , 0x1234, {0}}, Warn: Err); |
1827 | Writer.addRecord(I: {"bar" , 0x4321, {0, 0}}, Warn: Err); |
1828 | Writer.addRecord(I: {"baz" , 0x4321, {0, 0, 0}}, Warn: Err); |
1829 | |
1830 | auto Profile = Writer.writeBuffer(); |
1831 | readProfile(Profile: std::move(Profile)); |
1832 | |
1833 | auto I = Reader->begin(), E = Reader->end(); |
1834 | ASSERT_TRUE(I == E); |
1835 | } |
1836 | |
1837 | INSTANTIATE_TEST_SUITE_P(MaybeSparse, MaybeSparseInstrProfTest, |
1838 | ::testing::Bool()); |
1839 | |
1840 | #if defined(_LP64) && defined(EXPENSIVE_CHECKS) |
1841 | TEST(ProfileReaderTest, ReadsLargeFiles) { |
1842 | const size_t LargeSize = 1ULL << 32; // 4GB |
1843 | |
1844 | auto RawProfile = WritableMemoryBuffer::getNewUninitMemBuffer(LargeSize); |
1845 | if (!RawProfile) |
1846 | GTEST_SKIP(); |
1847 | auto RawProfileReaderOrErr = InstrProfReader::create(std::move(RawProfile)); |
1848 | ASSERT_TRUE( |
1849 | std::get<0>(InstrProfError::take(RawProfileReaderOrErr.takeError())) == |
1850 | instrprof_error::unrecognized_format); |
1851 | |
1852 | auto IndexedProfile = WritableMemoryBuffer::getNewUninitMemBuffer(LargeSize); |
1853 | if (!IndexedProfile) |
1854 | GTEST_SKIP(); |
1855 | auto IndexedReaderOrErr = |
1856 | IndexedInstrProfReader::create(std::move(IndexedProfile), nullptr); |
1857 | ASSERT_TRUE( |
1858 | std::get<0>(InstrProfError::take(IndexedReaderOrErr.takeError())) == |
1859 | instrprof_error::bad_magic); |
1860 | } |
1861 | #endif |
1862 | |
1863 | } // end anonymous namespace |
1864 | |