1 | /*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\ |
2 | |* |
3 | |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | |* See https://llvm.org/LICENSE.txt for license information. |
5 | |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | |* |
7 | \*===----------------------------------------------------------------------===*/ |
8 | /* |
9 | * This is the main file that defines all the data structure, signature, |
10 | * constant literals that are shared across profiling runtime library, |
11 | * compiler (instrumentation), and host tools (reader/writer). The entities |
12 | * defined in this file affect the profile runtime ABI, the raw profile format, |
13 | * or both. |
14 | * |
15 | * The file has two identical copies. The primary copy lives in LLVM and |
16 | * the other one sits in compiler-rt/lib/profile directory. To make changes |
17 | * in this file, first modify the primary copy and copy it over to compiler-rt. |
18 | * Testing of any change in this file can start only after the two copies are |
19 | * synced up. |
20 | * |
21 | * The first part of the file includes macros that defines types, names, and |
22 | * initializers for the member fields of the core data structures. The field |
23 | * declarations for one structure is enabled by defining the field activation |
24 | * macro associated with that structure. Only one field activation record |
25 | * can be defined at one time and the rest definitions will be filtered out by |
26 | * the preprocessor. |
27 | * |
28 | * Examples of how the template is used to instantiate structure definition: |
29 | * 1. To declare a structure: |
30 | * |
31 | * struct ProfData { |
32 | * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ |
33 | * Type Name; |
34 | * #include "llvm/ProfileData/InstrProfData.inc" |
35 | * }; |
36 | * |
37 | * 2. To construct LLVM type arrays for the struct type: |
38 | * |
39 | * Type *DataTypes[] = { |
40 | * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ |
41 | * LLVMType, |
42 | * #include "llvm/ProfileData/InstrProfData.inc" |
43 | * }; |
44 | * |
45 | * 4. To construct constant array for the initializers: |
46 | * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ |
47 | * Initializer, |
48 | * Constant *ConstantVals[] = { |
49 | * #include "llvm/ProfileData/InstrProfData.inc" |
50 | * }; |
51 | * |
52 | * |
53 | * The second part of the file includes definitions all other entities that |
54 | * are related to runtime ABI and format. When no field activation macro is |
55 | * defined, this file can be included to introduce the definitions. |
56 | * |
57 | \*===----------------------------------------------------------------------===*/ |
58 | |
59 | /* Functions marked with INSTR_PROF_VISIBILITY must have hidden visibility in |
60 | * the compiler runtime. */ |
61 | #ifndef INSTR_PROF_VISIBILITY |
62 | #define INSTR_PROF_VISIBILITY |
63 | #endif |
64 | |
65 | /* INSTR_PROF_DATA start. */ |
66 | /* Definition of member fields of the per-function control structure. */ |
67 | #ifndef INSTR_PROF_DATA |
68 | #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) |
69 | #else |
70 | #define INSTR_PROF_DATA_DEFINED |
71 | #endif |
72 | INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \ |
73 | ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ |
74 | IndexedInstrProf::ComputeHash(getPGOFuncNameVarInitializer(Inc->getName())))) |
75 | INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ |
76 | ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ |
77 | Inc->getHash()->getZExtValue())) |
78 | INSTR_PROF_DATA(const IntPtrT, IntPtrTy, CounterPtr, RelativeCounterPtr) |
79 | INSTR_PROF_DATA(const IntPtrT, IntPtrTy, BitmapPtr, RelativeBitmapPtr) |
80 | /* This is used to map function pointers for the indirect call targets to |
81 | * function name hashes during the conversion from raw to merged profile |
82 | * data. |
83 | */ |
84 | INSTR_PROF_DATA(const IntPtrT, llvm::PointerType::getUnqual(Ctx), FunctionPointer, \ |
85 | FunctionAddr) |
86 | INSTR_PROF_DATA(IntPtrT, llvm::PointerType::getUnqual(Ctx), Values, \ |
87 | ValuesPtrExpr) |
88 | INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \ |
89 | ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters)) |
90 | INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \ |
91 | ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) \ |
92 | INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \ |
93 | ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumBitmapBytes)) |
94 | #undef INSTR_PROF_DATA |
95 | /* INSTR_PROF_DATA end. */ |
96 | |
97 | |
98 | /* This is an internal data structure used by value profiler. It |
99 | * is defined here to allow serialization code sharing by LLVM |
100 | * to be used in unit test. |
101 | * |
102 | * typedef struct ValueProfNode { |
103 | * // InstrProfValueData VData; |
104 | * uint64_t Value; |
105 | * uint64_t Count; |
106 | * struct ValueProfNode *Next; |
107 | * } ValueProfNode; |
108 | */ |
109 | /* INSTR_PROF_VALUE_NODE start. */ |
110 | #ifndef INSTR_PROF_VALUE_NODE |
111 | #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer) |
112 | #else |
113 | #define INSTR_PROF_DATA_DEFINED |
114 | #endif |
115 | INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Value, \ |
116 | ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0)) |
117 | INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \ |
118 | ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0)) |
119 | INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::PointerType::getUnqual(Ctx), Next, \ |
120 | ConstantInt::get(llvm::Type::GetInt8PtrTy(Ctx), 0)) |
121 | #undef INSTR_PROF_VALUE_NODE |
122 | /* INSTR_PROF_VALUE_NODE end. */ |
123 | |
124 | /* INSTR_PROF_RAW_HEADER start */ |
125 | /* Definition of member fields of the raw profile header data structure. */ |
126 | /* Please update llvm/docs/InstrProfileFormat.rst as appropriate when updating |
127 | raw profile format. */ |
128 | #ifndef INSTR_PROF_RAW_HEADER |
129 | #define (Type, Name, Initializer) |
130 | #else |
131 | #define INSTR_PROF_DATA_DEFINED |
132 | #endif |
133 | INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic()) |
134 | INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version()) |
135 | INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL)) |
136 | INSTR_PROF_RAW_HEADER(uint64_t, NumData, NumData) |
137 | INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters) |
138 | INSTR_PROF_RAW_HEADER(uint64_t, NumCounters, NumCounters) |
139 | INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters) |
140 | INSTR_PROF_RAW_HEADER(uint64_t, NumBitmapBytes, NumBitmapBytes) |
141 | INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterBitmapBytes, PaddingBytesAfterBitmapBytes) |
142 | INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) |
143 | INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, |
144 | (uintptr_t)CountersBegin - (uintptr_t)DataBegin) |
145 | INSTR_PROF_RAW_HEADER(uint64_t, BitmapDelta, |
146 | (uintptr_t)BitmapBegin - (uintptr_t)DataBegin) |
147 | INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) |
148 | INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) |
149 | #undef INSTR_PROF_RAW_HEADER |
150 | /* INSTR_PROF_RAW_HEADER end */ |
151 | |
152 | /* VALUE_PROF_FUNC_PARAM start */ |
153 | /* Definition of parameter types of the runtime API used to do value profiling |
154 | * for a given value site. |
155 | */ |
156 | #ifndef VALUE_PROF_FUNC_PARAM |
157 | #define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType) |
158 | #define INSTR_PROF_COMMA |
159 | #else |
160 | #define INSTR_PROF_DATA_DEFINED |
161 | #define INSTR_PROF_COMMA , |
162 | #endif |
163 | VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \ |
164 | INSTR_PROF_COMMA |
165 | VALUE_PROF_FUNC_PARAM(void *, Data, PointerType::getUnqual(Ctx)) INSTR_PROF_COMMA |
166 | VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) |
167 | #undef VALUE_PROF_FUNC_PARAM |
168 | #undef INSTR_PROF_COMMA |
169 | /* VALUE_PROF_FUNC_PARAM end */ |
170 | |
171 | /* VALUE_PROF_KIND start */ |
172 | #ifndef VALUE_PROF_KIND |
173 | #define VALUE_PROF_KIND(Enumerator, Value, Descr) |
174 | #else |
175 | #define INSTR_PROF_DATA_DEFINED |
176 | #endif |
177 | /* For indirect function call value profiling, the addresses of the target |
178 | * functions are profiled by the instrumented code. The target addresses are |
179 | * written in the raw profile data and converted to target function name's MD5 |
180 | * hash by the profile reader during deserialization. Typically, this happens |
181 | * when the raw profile data is read during profile merging. |
182 | * |
183 | * For this remapping the ProfData is used. ProfData contains both the function |
184 | * name hash and the function address. |
185 | */ |
186 | VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target" ) |
187 | /* For memory intrinsic functions size profiling. */ |
188 | VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size" ) |
189 | /* These two kinds must be the last to be |
190 | * declared. This is to make sure the string |
191 | * array created with the template can be |
192 | * indexed with the kind value. |
193 | */ |
194 | VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first" ) |
195 | VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last" ) |
196 | |
197 | #undef VALUE_PROF_KIND |
198 | /* VALUE_PROF_KIND end */ |
199 | |
200 | #undef COVMAP_V2_OR_V3 |
201 | #ifdef COVMAP_V2 |
202 | #define COVMAP_V2_OR_V3 |
203 | #endif |
204 | #ifdef COVMAP_V3 |
205 | #define COVMAP_V2_OR_V3 |
206 | #endif |
207 | |
208 | /* COVMAP_FUNC_RECORD start */ |
209 | /* Definition of member fields of the function record structure in coverage |
210 | * map. |
211 | */ |
212 | #ifndef COVMAP_FUNC_RECORD |
213 | #define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer) |
214 | #else |
215 | #define INSTR_PROF_DATA_DEFINED |
216 | #endif |
217 | #ifdef COVMAP_V1 |
218 | COVMAP_FUNC_RECORD(const IntPtrT, llvm::PointerType::getUnqual(Ctx), \ |
219 | NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \ |
220 | llvm::PointerType::getUnqual(Ctx))) |
221 | COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \ |
222 | llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \ |
223 | NameValue.size())) |
224 | #endif |
225 | #ifdef COVMAP_V2_OR_V3 |
226 | COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \ |
227 | llvm::ConstantInt::get( \ |
228 | llvm::Type::getInt64Ty(Ctx), NameHash)) |
229 | #endif |
230 | COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \ |
231 | llvm::ConstantInt::get( \ |
232 | llvm::Type::getInt32Ty(Ctx), CoverageMapping.size())) |
233 | COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ |
234 | llvm::ConstantInt::get( \ |
235 | llvm::Type::getInt64Ty(Ctx), FuncHash)) |
236 | #ifdef COVMAP_V3 |
237 | COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FilenamesRef, \ |
238 | llvm::ConstantInt::get( \ |
239 | llvm::Type::getInt64Ty(Ctx), FilenamesRef)) |
240 | COVMAP_FUNC_RECORD(const char, \ |
241 | llvm::ArrayType::get(llvm::Type::getInt8Ty(Ctx), \ |
242 | CoverageMapping.size()), \ |
243 | CoverageMapping, |
244 | llvm::ConstantDataArray::getRaw( \ |
245 | CoverageMapping, CoverageMapping.size(), \ |
246 | llvm::Type::getInt8Ty(Ctx))) |
247 | #endif |
248 | #undef COVMAP_FUNC_RECORD |
249 | /* COVMAP_FUNC_RECORD end. */ |
250 | |
251 | /* COVMAP_HEADER start */ |
252 | /* Definition of member fields of coverage map header. |
253 | */ |
254 | #ifndef COVMAP_HEADER |
255 | #define (Type, LLVMType, Name, Initializer) |
256 | #else |
257 | #define INSTR_PROF_DATA_DEFINED |
258 | #endif |
259 | COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \ |
260 | llvm::ConstantInt::get(Int32Ty, NRecords)) |
261 | COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \ |
262 | llvm::ConstantInt::get(Int32Ty, FilenamesSize)) |
263 | COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \ |
264 | llvm::ConstantInt::get(Int32Ty, CoverageMappingSize)) |
265 | COVMAP_HEADER(uint32_t, Int32Ty, Version, \ |
266 | llvm::ConstantInt::get(Int32Ty, CovMapVersion::CurrentVersion)) |
267 | #undef COVMAP_HEADER |
268 | /* COVMAP_HEADER end. */ |
269 | |
270 | |
271 | #ifdef INSTR_PROF_SECT_ENTRY |
272 | #define INSTR_PROF_DATA_DEFINED |
273 | INSTR_PROF_SECT_ENTRY(IPSK_data, \ |
274 | INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \ |
275 | INSTR_PROF_DATA_COFF, "__DATA," ) |
276 | INSTR_PROF_SECT_ENTRY(IPSK_cnts, \ |
277 | INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \ |
278 | INSTR_PROF_CNTS_COFF, "__DATA," ) |
279 | INSTR_PROF_SECT_ENTRY(IPSK_bitmap, \ |
280 | INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON), \ |
281 | INSTR_PROF_BITS_COFF, "__DATA," ) |
282 | INSTR_PROF_SECT_ENTRY(IPSK_name, \ |
283 | INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \ |
284 | INSTR_PROF_NAME_COFF, "__DATA," ) |
285 | INSTR_PROF_SECT_ENTRY(IPSK_vals, \ |
286 | INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \ |
287 | INSTR_PROF_VALS_COFF, "__DATA," ) |
288 | INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \ |
289 | INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \ |
290 | INSTR_PROF_VNODES_COFF, "__DATA," ) |
291 | INSTR_PROF_SECT_ENTRY(IPSK_covmap, \ |
292 | INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \ |
293 | INSTR_PROF_COVMAP_COFF, "__LLVM_COV," ) |
294 | INSTR_PROF_SECT_ENTRY(IPSK_covfun, \ |
295 | INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON), \ |
296 | INSTR_PROF_COVFUN_COFF, "__LLVM_COV," ) |
297 | INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \ |
298 | INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \ |
299 | INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA," ) |
300 | INSTR_PROF_SECT_ENTRY(IPSK_covdata, \ |
301 | INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON), \ |
302 | INSTR_PROF_COVDATA_COFF, "__LLVM_COV," ) |
303 | INSTR_PROF_SECT_ENTRY(IPSK_covname, \ |
304 | INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON), \ |
305 | INSTR_PROF_COVNAME_COFF, "__LLVM_COV," ) |
306 | |
307 | #undef INSTR_PROF_SECT_ENTRY |
308 | #endif |
309 | |
310 | |
311 | #ifdef INSTR_PROF_VALUE_PROF_DATA |
312 | #define INSTR_PROF_DATA_DEFINED |
313 | |
314 | #define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255 |
315 | /*! |
316 | * This is the header of the data structure that defines the on-disk |
317 | * layout of the value profile data of a particular kind for one function. |
318 | */ |
319 | typedef struct ValueProfRecord { |
320 | /* The kind of the value profile record. */ |
321 | uint32_t Kind; |
322 | /* |
323 | * The number of value profile sites. It is guaranteed to be non-zero; |
324 | * otherwise the record for this kind won't be emitted. |
325 | */ |
326 | uint32_t NumValueSites; |
327 | /* |
328 | * The first element of the array that stores the number of profiled |
329 | * values for each value site. The size of the array is NumValueSites. |
330 | * Since NumValueSites is greater than zero, there is at least one |
331 | * element in the array. |
332 | */ |
333 | uint8_t SiteCountArray[1]; |
334 | |
335 | /* |
336 | * The fake declaration is for documentation purpose only. |
337 | * Align the start of next field to be on 8 byte boundaries. |
338 | uint8_t Padding[X]; |
339 | */ |
340 | |
341 | /* The array of value profile data. The size of the array is the sum |
342 | * of all elements in SiteCountArray[]. |
343 | InstrProfValueData ValueData[]; |
344 | */ |
345 | |
346 | #ifdef __cplusplus |
347 | /*! |
348 | * Return the number of value sites. |
349 | */ |
350 | uint32_t getNumValueSites() const { return NumValueSites; } |
351 | /*! |
352 | * Read data from this record and save it to Record. |
353 | */ |
354 | void deserializeTo(InstrProfRecord &Record, |
355 | InstrProfSymtab *SymTab); |
356 | /* |
357 | * In-place byte swap: |
358 | * Do byte swap for this instance. \c Old is the original order before |
359 | * the swap, and \c New is the New byte order. |
360 | */ |
361 | void swapBytes(llvm::endianness Old, llvm::endianness New); |
362 | #endif |
363 | } ValueProfRecord; |
364 | |
365 | /*! |
366 | * Per-function header/control data structure for value profiling |
367 | * data in indexed format. |
368 | */ |
369 | typedef struct ValueProfData { |
370 | /* |
371 | * Total size in bytes including this field. It must be a multiple |
372 | * of sizeof(uint64_t). |
373 | */ |
374 | uint32_t TotalSize; |
375 | /* |
376 | *The number of value profile kinds that has value profile data. |
377 | * In this implementation, a value profile kind is considered to |
378 | * have profile data if the number of value profile sites for the |
379 | * kind is not zero. More aggressively, the implementation can |
380 | * choose to check the actual data value: if none of the value sites |
381 | * has any profiled values, the kind can be skipped. |
382 | */ |
383 | uint32_t NumValueKinds; |
384 | |
385 | /* |
386 | * Following are a sequence of variable length records. The prefix/header |
387 | * of each record is defined by ValueProfRecord type. The number of |
388 | * records is NumValueKinds. |
389 | * ValueProfRecord Record_1; |
390 | * ValueProfRecord Record_N; |
391 | */ |
392 | |
393 | #if __cplusplus |
394 | /*! |
395 | * Return the total size in bytes of the on-disk value profile data |
396 | * given the data stored in Record. |
397 | */ |
398 | static uint32_t getSize(const InstrProfRecord &Record); |
399 | /*! |
400 | * Return a pointer to \c ValueProfData instance ready to be streamed. |
401 | */ |
402 | static std::unique_ptr<ValueProfData> |
403 | serializeFrom(const InstrProfRecord &Record); |
404 | /*! |
405 | * Check the integrity of the record. |
406 | */ |
407 | Error checkIntegrity(); |
408 | /*! |
409 | * Return a pointer to \c ValueProfileData instance ready to be read. |
410 | * All data in the instance are properly byte swapped. The input |
411 | * data is assumed to be in little endian order. |
412 | */ |
413 | static Expected<std::unique_ptr<ValueProfData>> |
414 | getValueProfData(const unsigned char *SrcBuffer, |
415 | const unsigned char *const SrcBufferEnd, |
416 | llvm::endianness SrcDataEndianness); |
417 | /*! |
418 | * Swap byte order from \c Endianness order to host byte order. |
419 | */ |
420 | void swapBytesToHost(llvm::endianness Endianness); |
421 | /*! |
422 | * Swap byte order from host byte order to \c Endianness order. |
423 | */ |
424 | void swapBytesFromHost(llvm::endianness Endianness); |
425 | /*! |
426 | * Return the total size of \c ValueProfileData. |
427 | */ |
428 | uint32_t getSize() const { return TotalSize; } |
429 | /*! |
430 | * Read data from this data and save it to \c Record. |
431 | */ |
432 | void deserializeTo(InstrProfRecord &Record, |
433 | InstrProfSymtab *SymTab); |
434 | void operator delete(void *ptr) { ::operator delete(ptr); } |
435 | #endif |
436 | } ValueProfData; |
437 | |
438 | /* |
439 | * The closure is designed to abstact away two types of value profile data: |
440 | * - InstrProfRecord which is the primary data structure used to |
441 | * represent profile data in host tools (reader, writer, and profile-use) |
442 | * - value profile runtime data structure suitable to be used by C |
443 | * runtime library. |
444 | * |
445 | * Both sources of data need to serialize to disk/memory-buffer in common |
446 | * format: ValueProfData. The abstraction allows compiler-rt's raw profiler |
447 | * writer to share the same format and code with indexed profile writer. |
448 | * |
449 | * For documentation of the member methods below, refer to corresponding methods |
450 | * in class InstrProfRecord. |
451 | */ |
452 | typedef struct ValueProfRecordClosure { |
453 | const void *Record; |
454 | uint32_t (*GetNumValueKinds)(const void *Record); |
455 | uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind); |
456 | uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind); |
457 | uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S); |
458 | |
459 | /* |
460 | * After extracting the value profile data from the value profile record, |
461 | * this method is used to map the in-memory value to on-disk value. If |
462 | * the method is null, value will be written out untranslated. |
463 | */ |
464 | uint64_t (*RemapValueData)(uint32_t, uint64_t Value); |
465 | void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K, |
466 | uint32_t S); |
467 | ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes); |
468 | } ValueProfRecordClosure; |
469 | |
470 | INSTR_PROF_VISIBILITY ValueProfRecord * |
471 | getFirstValueProfRecord(ValueProfData *VPD); |
472 | INSTR_PROF_VISIBILITY ValueProfRecord * |
473 | getValueProfRecordNext(ValueProfRecord *VPR); |
474 | INSTR_PROF_VISIBILITY InstrProfValueData * |
475 | getValueProfRecordValueData(ValueProfRecord *VPR); |
476 | INSTR_PROF_VISIBILITY uint32_t |
477 | getValueProfRecordHeaderSize(uint32_t NumValueSites); |
478 | |
479 | #undef INSTR_PROF_VALUE_PROF_DATA |
480 | #endif /* INSTR_PROF_VALUE_PROF_DATA */ |
481 | |
482 | |
483 | #ifdef INSTR_PROF_COMMON_API_IMPL |
484 | #define INSTR_PROF_DATA_DEFINED |
485 | #ifdef __cplusplus |
486 | #define INSTR_PROF_INLINE inline |
487 | #define INSTR_PROF_NULLPTR nullptr |
488 | #else |
489 | #define INSTR_PROF_INLINE |
490 | #define INSTR_PROF_NULLPTR NULL |
491 | #endif |
492 | |
493 | #ifndef offsetof |
494 | #define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) |
495 | #endif |
496 | |
497 | /*! |
498 | * Return the \c ValueProfRecord header size including the |
499 | * padding bytes. |
500 | */ |
501 | INSTR_PROF_VISIBILITY INSTR_PROF_INLINE |
502 | uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) { |
503 | uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) + |
504 | sizeof(uint8_t) * NumValueSites; |
505 | /* Round the size to multiple of 8 bytes. */ |
506 | Size = (Size + 7) & ~7; |
507 | return Size; |
508 | } |
509 | |
510 | /*! |
511 | * Return the total size of the value profile record including the |
512 | * header and the value data. |
513 | */ |
514 | INSTR_PROF_VISIBILITY INSTR_PROF_INLINE |
515 | uint32_t getValueProfRecordSize(uint32_t NumValueSites, |
516 | uint32_t NumValueData) { |
517 | return getValueProfRecordHeaderSize(NumValueSites) + |
518 | sizeof(InstrProfValueData) * NumValueData; |
519 | } |
520 | |
521 | /*! |
522 | * Return the pointer to the start of value data array. |
523 | */ |
524 | INSTR_PROF_VISIBILITY INSTR_PROF_INLINE |
525 | InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) { |
526 | return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize( |
527 | This->NumValueSites)); |
528 | } |
529 | |
530 | /*! |
531 | * Return the total number of value data for \c This record. |
532 | */ |
533 | INSTR_PROF_VISIBILITY INSTR_PROF_INLINE |
534 | uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) { |
535 | uint32_t NumValueData = 0; |
536 | uint32_t I; |
537 | for (I = 0; I < This->NumValueSites; I++) |
538 | NumValueData += This->SiteCountArray[I]; |
539 | return NumValueData; |
540 | } |
541 | |
542 | /*! |
543 | * Use this method to advance to the next \c This \c ValueProfRecord. |
544 | */ |
545 | INSTR_PROF_VISIBILITY INSTR_PROF_INLINE |
546 | ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) { |
547 | uint32_t NumValueData = getValueProfRecordNumValueData(This); |
548 | return (ValueProfRecord *)((char *)This + |
549 | getValueProfRecordSize(This->NumValueSites, |
550 | NumValueData)); |
551 | } |
552 | |
553 | /*! |
554 | * Return the first \c ValueProfRecord instance. |
555 | */ |
556 | INSTR_PROF_VISIBILITY INSTR_PROF_INLINE |
557 | ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) { |
558 | return (ValueProfRecord *)((char *)This + sizeof(ValueProfData)); |
559 | } |
560 | |
561 | /* Closure based interfaces. */ |
562 | |
563 | /*! |
564 | * Return the total size in bytes of the on-disk value profile data |
565 | * given the data stored in Record. |
566 | */ |
567 | INSTR_PROF_VISIBILITY uint32_t |
568 | getValueProfDataSize(ValueProfRecordClosure *Closure) { |
569 | uint32_t Kind; |
570 | uint32_t TotalSize = sizeof(ValueProfData); |
571 | const void *Record = Closure->Record; |
572 | |
573 | for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { |
574 | uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind); |
575 | if (!NumValueSites) |
576 | continue; |
577 | TotalSize += getValueProfRecordSize(NumValueSites, |
578 | Closure->GetNumValueData(Record, Kind)); |
579 | } |
580 | return TotalSize; |
581 | } |
582 | |
583 | /*! |
584 | * Extract value profile data of a function for the profile kind \c ValueKind |
585 | * from the \c Closure and serialize the data into \c This record instance. |
586 | */ |
587 | INSTR_PROF_VISIBILITY void |
588 | serializeValueProfRecordFrom(ValueProfRecord *This, |
589 | ValueProfRecordClosure *Closure, |
590 | uint32_t ValueKind, uint32_t NumValueSites) { |
591 | uint32_t S; |
592 | const void *Record = Closure->Record; |
593 | This->Kind = ValueKind; |
594 | This->NumValueSites = NumValueSites; |
595 | InstrProfValueData *DstVD = getValueProfRecordValueData(This); |
596 | |
597 | for (S = 0; S < NumValueSites; S++) { |
598 | uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S); |
599 | This->SiteCountArray[S] = ND; |
600 | Closure->GetValueForSite(Record, DstVD, ValueKind, S); |
601 | DstVD += ND; |
602 | } |
603 | } |
604 | |
605 | /*! |
606 | * Extract value profile data of a function from the \c Closure |
607 | * and serialize the data into \c DstData if it is not NULL or heap |
608 | * memory allocated by the \c Closure's allocator method. If \c |
609 | * DstData is not null, the caller is expected to set the TotalSize |
610 | * in DstData. |
611 | */ |
612 | INSTR_PROF_VISIBILITY ValueProfData * |
613 | serializeValueProfDataFrom(ValueProfRecordClosure *Closure, |
614 | ValueProfData *DstData) { |
615 | uint32_t Kind; |
616 | uint32_t TotalSize = |
617 | DstData ? DstData->TotalSize : getValueProfDataSize(Closure); |
618 | |
619 | ValueProfData *VPD = |
620 | DstData ? DstData : Closure->AllocValueProfData(TotalSize); |
621 | |
622 | VPD->TotalSize = TotalSize; |
623 | VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record); |
624 | ValueProfRecord *VR = getFirstValueProfRecord(VPD); |
625 | for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { |
626 | uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind); |
627 | if (!NumValueSites) |
628 | continue; |
629 | serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites); |
630 | VR = getValueProfRecordNext(VR); |
631 | } |
632 | return VPD; |
633 | } |
634 | |
635 | #undef INSTR_PROF_COMMON_API_IMPL |
636 | #endif /* INSTR_PROF_COMMON_API_IMPL */ |
637 | |
638 | /*============================================================================*/ |
639 | |
640 | #ifndef INSTR_PROF_DATA_DEFINED |
641 | |
642 | #ifndef INSTR_PROF_DATA_INC |
643 | #define INSTR_PROF_DATA_INC |
644 | |
645 | /* Helper macros. */ |
646 | #define INSTR_PROF_SIMPLE_QUOTE(x) #x |
647 | #define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x) |
648 | #define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y |
649 | #define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y) |
650 | |
651 | /* Magic number to detect file format and endianness. |
652 | * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0, |
653 | * so that utilities, like strings, don't grab it as a string. 129 is also |
654 | * invalid UTF-8, and high enough to be interesting. |
655 | * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR" |
656 | * for 32-bit platforms. |
657 | */ |
658 | #define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ |
659 | (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ |
660 | (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129 |
661 | #define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ |
662 | (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ |
663 | (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129 |
664 | |
665 | /* Raw profile format version (start from 1). */ |
666 | #define INSTR_PROF_RAW_VERSION 9 |
667 | /* Indexed profile format version (start from 1). */ |
668 | #define INSTR_PROF_INDEX_VERSION 11 |
669 | /* Coverage mapping format version (start from 0). */ |
670 | #define INSTR_PROF_COVMAP_VERSION 6 |
671 | |
672 | /* Profile version is always of type uint64_t. Reserve the upper 32 bits in the |
673 | * version for other variants of profile. We set the 8th most significant bit |
674 | * (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation |
675 | * generated profile, and 0 if this is a Clang FE generated profile. |
676 | * 1 in bit 57 indicates there are context-sensitive records in the profile. |
677 | * The 59th bit indicates whether to use debug info to correlate profiles. |
678 | * The 60th bit indicates single byte coverage instrumentation. |
679 | * The 61st bit indicates function entry instrumentation only. |
680 | * The 62nd bit indicates whether memory profile information is present. |
681 | * The 63rd bit indicates if this is a temporal profile. |
682 | */ |
683 | #define VARIANT_MASKS_ALL 0xffffffff00000000ULL |
684 | #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) |
685 | #define VARIANT_MASK_IR_PROF (0x1ULL << 56) |
686 | #define VARIANT_MASK_CSIR_PROF (0x1ULL << 57) |
687 | #define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58) |
688 | #define VARIANT_MASK_DBG_CORRELATE (0x1ULL << 59) |
689 | #define VARIANT_MASK_BYTE_COVERAGE (0x1ULL << 60) |
690 | #define VARIANT_MASK_FUNCTION_ENTRY_ONLY (0x1ULL << 61) |
691 | #define VARIANT_MASK_MEMPROF (0x1ULL << 62) |
692 | #define VARIANT_MASK_TEMPORAL_PROF (0x1ULL << 63) |
693 | #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version |
694 | #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime |
695 | #define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias |
696 | #define INSTR_PROF_PROFILE_SET_TIMESTAMP __llvm_profile_set_timestamp |
697 | |
698 | /* The variable that holds the name of the profile data |
699 | * specified via command line. */ |
700 | #define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename |
701 | |
702 | /* section name strings common to all targets other |
703 | than WIN32 */ |
704 | #define INSTR_PROF_DATA_COMMON __llvm_prf_data |
705 | #define INSTR_PROF_NAME_COMMON __llvm_prf_names |
706 | #define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts |
707 | #define INSTR_PROF_BITS_COMMON __llvm_prf_bits |
708 | #define INSTR_PROF_VALS_COMMON __llvm_prf_vals |
709 | #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds |
710 | #define INSTR_PROF_COVMAP_COMMON __llvm_covmap |
711 | #define INSTR_PROF_COVFUN_COMMON __llvm_covfun |
712 | #define INSTR_PROF_COVDATA_COMMON __llvm_covdata |
713 | #define INSTR_PROF_COVNAME_COMMON __llvm_covnames |
714 | #define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile |
715 | /* Windows section names. Because these section names contain dollar characters, |
716 | * they must be quoted. |
717 | */ |
718 | #define INSTR_PROF_DATA_COFF ".lprfd$M" |
719 | #define INSTR_PROF_NAME_COFF ".lprfn$M" |
720 | #define INSTR_PROF_CNTS_COFF ".lprfc$M" |
721 | #define INSTR_PROF_BITS_COFF ".lprfb$M" |
722 | #define INSTR_PROF_VALS_COFF ".lprfv$M" |
723 | #define INSTR_PROF_VNODES_COFF ".lprfnd$M" |
724 | #define INSTR_PROF_COVMAP_COFF ".lcovmap$M" |
725 | #define INSTR_PROF_COVFUN_COFF ".lcovfun$M" |
726 | /* Since cov data and cov names sections are not allocated, we don't need to |
727 | * access them at runtime. |
728 | */ |
729 | #define INSTR_PROF_COVDATA_COFF ".lcovd" |
730 | #define INSTR_PROF_COVNAME_COFF ".lcovn" |
731 | #define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M" |
732 | |
733 | #ifdef _WIN32 |
734 | /* Runtime section names and name strings. */ |
735 | #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF |
736 | #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF |
737 | #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF |
738 | #define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_BITS_COFF |
739 | /* Array of pointers. Each pointer points to a list |
740 | * of value nodes associated with one value site. |
741 | */ |
742 | #define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF |
743 | /* Value profile nodes section. */ |
744 | #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF |
745 | #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF |
746 | #define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF |
747 | #define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_COVDATA_COFF |
748 | #define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_COVNAME_COFF |
749 | #define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF |
750 | #else |
751 | /* Runtime section names and name strings. */ |
752 | #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) |
753 | #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON) |
754 | #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON) |
755 | #define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON) |
756 | /* Array of pointers. Each pointer points to a list |
757 | * of value nodes associated with one value site. |
758 | */ |
759 | #define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON) |
760 | /* Value profile nodes section. */ |
761 | #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON) |
762 | #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON) |
763 | #define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON) |
764 | #define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON) |
765 | #define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON) |
766 | /* Order file instrumentation. */ |
767 | #define INSTR_PROF_ORDERFILE_SECT_NAME \ |
768 | INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON) |
769 | #endif |
770 | |
771 | #define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer |
772 | #define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \ |
773 | INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME) |
774 | #define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx |
775 | #define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \ |
776 | INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME) |
777 | |
778 | /* Macros to define start/stop section symbol for a given |
779 | * section on Linux. For instance |
780 | * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will |
781 | * expand to __start___llvm_prof_data |
782 | */ |
783 | #define INSTR_PROF_SECT_START(Sect) \ |
784 | INSTR_PROF_CONCAT(__start_,Sect) |
785 | #define INSTR_PROF_SECT_STOP(Sect) \ |
786 | INSTR_PROF_CONCAT(__stop_,Sect) |
787 | |
788 | /* Value Profiling API linkage name. */ |
789 | #define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target |
790 | #define INSTR_PROF_VALUE_PROF_FUNC_STR \ |
791 | INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC) |
792 | #define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop |
793 | #define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \ |
794 | INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC) |
795 | |
796 | /* InstrProfile per-function control data alignment. */ |
797 | #define INSTR_PROF_DATA_ALIGNMENT 8 |
798 | |
799 | /* The data structure that represents a tracked value by the |
800 | * value profiler. |
801 | */ |
802 | typedef struct InstrProfValueData { |
803 | /* Profiled value. */ |
804 | uint64_t Value; |
805 | /* Number of times the value appears in the training run. */ |
806 | uint64_t Count; |
807 | } InstrProfValueData; |
808 | |
809 | #endif /* INSTR_PROF_DATA_INC */ |
810 | |
811 | #ifndef INSTR_ORDER_FILE_INC |
812 | /* The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). */ |
813 | #define INSTR_ORDER_FILE_BUFFER_SIZE 131072 |
814 | #define INSTR_ORDER_FILE_BUFFER_BITS 17 |
815 | #define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff |
816 | #endif /* INSTR_ORDER_FILE_INC */ |
817 | #else |
818 | #undef INSTR_PROF_DATA_DEFINED |
819 | #endif |
820 | |
821 | #undef COVMAP_V2_OR_V3 |
822 | |
823 | #ifdef INSTR_PROF_VALUE_PROF_MEMOP_API |
824 | |
825 | #ifdef __cplusplus |
826 | #define INSTR_PROF_INLINE inline |
827 | #else |
828 | #define INSTR_PROF_INLINE |
829 | #endif |
830 | |
831 | /* The value range buckets (22 buckets) for the memop size value profiling looks |
832 | * like: |
833 | * |
834 | * [0, 0] |
835 | * [1, 1] |
836 | * [2, 2] |
837 | * [3, 3] |
838 | * [4, 4] |
839 | * [5, 5] |
840 | * [6, 6] |
841 | * [7, 7] |
842 | * [8, 8] |
843 | * [9, 15] |
844 | * [16, 16] |
845 | * [17, 31] |
846 | * [32, 32] |
847 | * [33, 63] |
848 | * [64, 64] |
849 | * [65, 127] |
850 | * [128, 128] |
851 | * [129, 255] |
852 | * [256, 256] |
853 | * [257, 511] |
854 | * [512, 512] |
855 | * [513, UINT64_MAX] |
856 | * |
857 | * Each range has a 'representative value' which is the lower end value of the |
858 | * range and used to store in the runtime profile data records and the VP |
859 | * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127]. |
860 | */ |
861 | #define INSTR_PROF_NUM_BUCKETS 22 |
862 | |
863 | /* |
864 | * Clz and Popcount. This code was copied from |
865 | * compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and |
866 | * llvm/include/llvm/Support/MathExtras.h. |
867 | */ |
868 | #if defined(_MSC_VER) && !defined(__clang__) |
869 | |
870 | #include <intrin.h> |
871 | INSTR_PROF_VISIBILITY INSTR_PROF_INLINE |
872 | int InstProfClzll(unsigned long long X) { |
873 | unsigned long LeadZeroIdx = 0; |
874 | #if !defined(_M_ARM64) && !defined(_M_X64) |
875 | // Scan the high 32 bits. |
876 | if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32))) |
877 | return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset |
878 | // from the MSB. |
879 | // Scan the low 32 bits. |
880 | if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X))) |
881 | return (int)(63 - LeadZeroIdx); |
882 | #else |
883 | if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx; |
884 | #endif |
885 | return 64; |
886 | } |
887 | INSTR_PROF_VISIBILITY INSTR_PROF_INLINE |
888 | int InstProfPopcountll(unsigned long long X) { |
889 | // This code originates from https://reviews.llvm.org/rG30626254510f. |
890 | unsigned long long v = X; |
891 | v = v - ((v >> 1) & 0x5555555555555555ULL); |
892 | v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL); |
893 | v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL; |
894 | return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56); |
895 | } |
896 | |
897 | #else |
898 | |
899 | INSTR_PROF_VISIBILITY INSTR_PROF_INLINE |
900 | int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); } |
901 | INSTR_PROF_VISIBILITY INSTR_PROF_INLINE |
902 | int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); } |
903 | |
904 | #endif /* defined(_MSC_VER) && !defined(__clang__) */ |
905 | |
906 | /* Map an (observed) memop size value to the representative value of its range. |
907 | * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */ |
908 | INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t |
909 | InstrProfGetRangeRepValue(uint64_t Value) { |
910 | if (Value <= 8) |
911 | // The first ranges are individually tracked. Use the value as is. |
912 | return Value; |
913 | else if (Value >= 513) |
914 | // The last range is mapped to its lowest value. |
915 | return 513; |
916 | else if (InstProfPopcountll(Value) == 1) |
917 | // If it's a power of two, use it as is. |
918 | return Value; |
919 | else |
920 | // Otherwise, take to the previous power of two + 1. |
921 | return (UINT64_C(1) << (64 - InstProfClzll(Value) - 1)) + 1; |
922 | } |
923 | |
924 | /* Return true if the range that an (observed) memop size value belongs to has |
925 | * only a single value in the range. For example, 0 -> true, 8 -> true, 10 -> |
926 | * false, 64 -> true, 100 -> false, 513 -> false. */ |
927 | INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned |
928 | InstrProfIsSingleValRange(uint64_t Value) { |
929 | if (Value <= 8) |
930 | // The first ranges are individually tracked. |
931 | return 1; |
932 | else if (InstProfPopcountll(Value) == 1) |
933 | // If it's a power of two, there's only one value. |
934 | return 1; |
935 | else |
936 | // Otherwise, there's more than one value in the range. |
937 | return 0; |
938 | } |
939 | |
940 | #endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */ |
941 | |