1/*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\
2|*
3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4|* See https://llvm.org/LICENSE.txt for license information.
5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6|*
7\*===----------------------------------------------------------------------===*/
8/*
9 * This is the main file that defines all the data structure, signature,
10 * constant literals that are shared across profiling runtime library,
11 * compiler (instrumentation), and host tools (reader/writer). The entities
12 * defined in this file affect the profile runtime ABI, the raw profile format,
13 * or both.
14 *
15 * The file has two identical copies. The primary copy lives in LLVM and
16 * the other one sits in compiler-rt/lib/profile directory. To make changes
17 * in this file, first modify the primary copy and copy it over to compiler-rt.
18 * Testing of any change in this file can start only after the two copies are
19 * synced up.
20 *
21 * The first part of the file includes macros that defines types, names, and
22 * initializers for the member fields of the core data structures. The field
23 * declarations for one structure is enabled by defining the field activation
24 * macro associated with that structure. Only one field activation record
25 * can be defined at one time and the rest definitions will be filtered out by
26 * the preprocessor.
27 *
28 * Examples of how the template is used to instantiate structure definition:
29 * 1. To declare a structure:
30 *
31 * struct ProfData {
32 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
33 * Type Name;
34 * #include "llvm/ProfileData/InstrProfData.inc"
35 * };
36 *
37 * 2. To construct LLVM type arrays for the struct type:
38 *
39 * Type *DataTypes[] = {
40 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
41 * LLVMType,
42 * #include "llvm/ProfileData/InstrProfData.inc"
43 * };
44 *
45 * 4. To construct constant array for the initializers:
46 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
47 * Initializer,
48 * Constant *ConstantVals[] = {
49 * #include "llvm/ProfileData/InstrProfData.inc"
50 * };
51 *
52 *
53 * The second part of the file includes definitions all other entities that
54 * are related to runtime ABI and format. When no field activation macro is
55 * defined, this file can be included to introduce the definitions.
56 *
57\*===----------------------------------------------------------------------===*/
58
59/* Functions marked with INSTR_PROF_VISIBILITY must have hidden visibility in
60 * the compiler runtime. */
61#ifndef INSTR_PROF_VISIBILITY
62#define INSTR_PROF_VISIBILITY
63#endif
64
65/* INSTR_PROF_DATA start. */
66/* Definition of member fields of the per-function control structure. */
67#ifndef INSTR_PROF_DATA
68#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer)
69#else
70#define INSTR_PROF_DATA_DEFINED
71#endif
72INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
73 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
74 IndexedInstrProf::ComputeHash(getPGOFuncNameVarInitializer(Inc->getName()))))
75INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
76 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
77 Inc->getHash()->getZExtValue()))
78INSTR_PROF_DATA(const IntPtrT, IntPtrTy, CounterPtr, RelativeCounterPtr)
79INSTR_PROF_DATA(const IntPtrT, IntPtrTy, BitmapPtr, RelativeBitmapPtr)
80/* This is used to map function pointers for the indirect call targets to
81 * function name hashes during the conversion from raw to merged profile
82 * data.
83 */
84INSTR_PROF_DATA(const IntPtrT, llvm::PointerType::getUnqual(Ctx), FunctionPointer, \
85 FunctionAddr)
86INSTR_PROF_DATA(IntPtrT, llvm::PointerType::getUnqual(Ctx), Values, \
87 ValuesPtrExpr)
88INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \
89 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters))
90INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \
91 ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) \
92INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \
93 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumBitmapBytes))
94#undef INSTR_PROF_DATA
95/* INSTR_PROF_DATA end. */
96
97
98/* This is an internal data structure used by value profiler. It
99 * is defined here to allow serialization code sharing by LLVM
100 * to be used in unit test.
101 *
102 * typedef struct ValueProfNode {
103 * // InstrProfValueData VData;
104 * uint64_t Value;
105 * uint64_t Count;
106 * struct ValueProfNode *Next;
107 * } ValueProfNode;
108 */
109/* INSTR_PROF_VALUE_NODE start. */
110#ifndef INSTR_PROF_VALUE_NODE
111#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer)
112#else
113#define INSTR_PROF_DATA_DEFINED
114#endif
115INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Value, \
116 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
117INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \
118 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
119INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::PointerType::getUnqual(Ctx), Next, \
120 ConstantInt::get(llvm::Type::GetInt8PtrTy(Ctx), 0))
121#undef INSTR_PROF_VALUE_NODE
122/* INSTR_PROF_VALUE_NODE end. */
123
124/* INSTR_PROF_RAW_HEADER start */
125/* Definition of member fields of the raw profile header data structure. */
126/* Please update llvm/docs/InstrProfileFormat.rst as appropriate when updating
127 raw profile format. */
128#ifndef INSTR_PROF_RAW_HEADER
129#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer)
130#else
131#define INSTR_PROF_DATA_DEFINED
132#endif
133INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic())
134INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version())
135INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL))
136INSTR_PROF_RAW_HEADER(uint64_t, NumData, NumData)
137INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters)
138INSTR_PROF_RAW_HEADER(uint64_t, NumCounters, NumCounters)
139INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters)
140INSTR_PROF_RAW_HEADER(uint64_t, NumBitmapBytes, NumBitmapBytes)
141INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterBitmapBytes, PaddingBytesAfterBitmapBytes)
142INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize)
143INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta,
144 (uintptr_t)CountersBegin - (uintptr_t)DataBegin)
145INSTR_PROF_RAW_HEADER(uint64_t, BitmapDelta,
146 (uintptr_t)BitmapBegin - (uintptr_t)DataBegin)
147INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
148INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
149#undef INSTR_PROF_RAW_HEADER
150/* INSTR_PROF_RAW_HEADER end */
151
152/* VALUE_PROF_FUNC_PARAM start */
153/* Definition of parameter types of the runtime API used to do value profiling
154 * for a given value site.
155 */
156#ifndef VALUE_PROF_FUNC_PARAM
157#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType)
158#define INSTR_PROF_COMMA
159#else
160#define INSTR_PROF_DATA_DEFINED
161#define INSTR_PROF_COMMA ,
162#endif
163VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \
164 INSTR_PROF_COMMA
165VALUE_PROF_FUNC_PARAM(void *, Data, PointerType::getUnqual(Ctx)) INSTR_PROF_COMMA
166VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
167#undef VALUE_PROF_FUNC_PARAM
168#undef INSTR_PROF_COMMA
169/* VALUE_PROF_FUNC_PARAM end */
170
171/* VALUE_PROF_KIND start */
172#ifndef VALUE_PROF_KIND
173#define VALUE_PROF_KIND(Enumerator, Value, Descr)
174#else
175#define INSTR_PROF_DATA_DEFINED
176#endif
177/* For indirect function call value profiling, the addresses of the target
178 * functions are profiled by the instrumented code. The target addresses are
179 * written in the raw profile data and converted to target function name's MD5
180 * hash by the profile reader during deserialization. Typically, this happens
181 * when the raw profile data is read during profile merging.
182 *
183 * For this remapping the ProfData is used. ProfData contains both the function
184 * name hash and the function address.
185 */
186VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target")
187/* For memory intrinsic functions size profiling. */
188VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size")
189/* These two kinds must be the last to be
190 * declared. This is to make sure the string
191 * array created with the template can be
192 * indexed with the kind value.
193 */
194VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first")
195VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last")
196
197#undef VALUE_PROF_KIND
198/* VALUE_PROF_KIND end */
199
200#undef COVMAP_V2_OR_V3
201#ifdef COVMAP_V2
202#define COVMAP_V2_OR_V3
203#endif
204#ifdef COVMAP_V3
205#define COVMAP_V2_OR_V3
206#endif
207
208/* COVMAP_FUNC_RECORD start */
209/* Definition of member fields of the function record structure in coverage
210 * map.
211 */
212#ifndef COVMAP_FUNC_RECORD
213#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer)
214#else
215#define INSTR_PROF_DATA_DEFINED
216#endif
217#ifdef COVMAP_V1
218COVMAP_FUNC_RECORD(const IntPtrT, llvm::PointerType::getUnqual(Ctx), \
219 NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \
220 llvm::PointerType::getUnqual(Ctx)))
221COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \
222 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \
223 NameValue.size()))
224#endif
225#ifdef COVMAP_V2_OR_V3
226COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
227 llvm::ConstantInt::get( \
228 llvm::Type::getInt64Ty(Ctx), NameHash))
229#endif
230COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \
231 llvm::ConstantInt::get( \
232 llvm::Type::getInt32Ty(Ctx), CoverageMapping.size()))
233COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
234 llvm::ConstantInt::get( \
235 llvm::Type::getInt64Ty(Ctx), FuncHash))
236#ifdef COVMAP_V3
237COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FilenamesRef, \
238 llvm::ConstantInt::get( \
239 llvm::Type::getInt64Ty(Ctx), FilenamesRef))
240COVMAP_FUNC_RECORD(const char, \
241 llvm::ArrayType::get(llvm::Type::getInt8Ty(Ctx), \
242 CoverageMapping.size()), \
243 CoverageMapping,
244 llvm::ConstantDataArray::getRaw( \
245 CoverageMapping, CoverageMapping.size(), \
246 llvm::Type::getInt8Ty(Ctx)))
247#endif
248#undef COVMAP_FUNC_RECORD
249/* COVMAP_FUNC_RECORD end. */
250
251/* COVMAP_HEADER start */
252/* Definition of member fields of coverage map header.
253 */
254#ifndef COVMAP_HEADER
255#define COVMAP_HEADER(Type, LLVMType, Name, Initializer)
256#else
257#define INSTR_PROF_DATA_DEFINED
258#endif
259COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \
260 llvm::ConstantInt::get(Int32Ty, NRecords))
261COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \
262 llvm::ConstantInt::get(Int32Ty, FilenamesSize))
263COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \
264 llvm::ConstantInt::get(Int32Ty, CoverageMappingSize))
265COVMAP_HEADER(uint32_t, Int32Ty, Version, \
266 llvm::ConstantInt::get(Int32Ty, CovMapVersion::CurrentVersion))
267#undef COVMAP_HEADER
268/* COVMAP_HEADER end. */
269
270
271#ifdef INSTR_PROF_SECT_ENTRY
272#define INSTR_PROF_DATA_DEFINED
273INSTR_PROF_SECT_ENTRY(IPSK_data, \
274 INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \
275 INSTR_PROF_DATA_COFF, "__DATA,")
276INSTR_PROF_SECT_ENTRY(IPSK_cnts, \
277 INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \
278 INSTR_PROF_CNTS_COFF, "__DATA,")
279INSTR_PROF_SECT_ENTRY(IPSK_bitmap, \
280 INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON), \
281 INSTR_PROF_BITS_COFF, "__DATA,")
282INSTR_PROF_SECT_ENTRY(IPSK_name, \
283 INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \
284 INSTR_PROF_NAME_COFF, "__DATA,")
285INSTR_PROF_SECT_ENTRY(IPSK_vals, \
286 INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \
287 INSTR_PROF_VALS_COFF, "__DATA,")
288INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \
289 INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \
290 INSTR_PROF_VNODES_COFF, "__DATA,")
291INSTR_PROF_SECT_ENTRY(IPSK_covmap, \
292 INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \
293 INSTR_PROF_COVMAP_COFF, "__LLVM_COV,")
294INSTR_PROF_SECT_ENTRY(IPSK_covfun, \
295 INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON), \
296 INSTR_PROF_COVFUN_COFF, "__LLVM_COV,")
297INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \
298 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \
299 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,")
300INSTR_PROF_SECT_ENTRY(IPSK_covdata, \
301 INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON), \
302 INSTR_PROF_COVDATA_COFF, "__LLVM_COV,")
303INSTR_PROF_SECT_ENTRY(IPSK_covname, \
304 INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON), \
305 INSTR_PROF_COVNAME_COFF, "__LLVM_COV,")
306
307#undef INSTR_PROF_SECT_ENTRY
308#endif
309
310
311#ifdef INSTR_PROF_VALUE_PROF_DATA
312#define INSTR_PROF_DATA_DEFINED
313
314#define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255
315/*!
316 * This is the header of the data structure that defines the on-disk
317 * layout of the value profile data of a particular kind for one function.
318 */
319typedef struct ValueProfRecord {
320 /* The kind of the value profile record. */
321 uint32_t Kind;
322 /*
323 * The number of value profile sites. It is guaranteed to be non-zero;
324 * otherwise the record for this kind won't be emitted.
325 */
326 uint32_t NumValueSites;
327 /*
328 * The first element of the array that stores the number of profiled
329 * values for each value site. The size of the array is NumValueSites.
330 * Since NumValueSites is greater than zero, there is at least one
331 * element in the array.
332 */
333 uint8_t SiteCountArray[1];
334
335 /*
336 * The fake declaration is for documentation purpose only.
337 * Align the start of next field to be on 8 byte boundaries.
338 uint8_t Padding[X];
339 */
340
341 /* The array of value profile data. The size of the array is the sum
342 * of all elements in SiteCountArray[].
343 InstrProfValueData ValueData[];
344 */
345
346#ifdef __cplusplus
347 /*!
348 * Return the number of value sites.
349 */
350 uint32_t getNumValueSites() const { return NumValueSites; }
351 /*!
352 * Read data from this record and save it to Record.
353 */
354 void deserializeTo(InstrProfRecord &Record,
355 InstrProfSymtab *SymTab);
356 /*
357 * In-place byte swap:
358 * Do byte swap for this instance. \c Old is the original order before
359 * the swap, and \c New is the New byte order.
360 */
361 void swapBytes(llvm::endianness Old, llvm::endianness New);
362#endif
363} ValueProfRecord;
364
365/*!
366 * Per-function header/control data structure for value profiling
367 * data in indexed format.
368 */
369typedef struct ValueProfData {
370 /*
371 * Total size in bytes including this field. It must be a multiple
372 * of sizeof(uint64_t).
373 */
374 uint32_t TotalSize;
375 /*
376 *The number of value profile kinds that has value profile data.
377 * In this implementation, a value profile kind is considered to
378 * have profile data if the number of value profile sites for the
379 * kind is not zero. More aggressively, the implementation can
380 * choose to check the actual data value: if none of the value sites
381 * has any profiled values, the kind can be skipped.
382 */
383 uint32_t NumValueKinds;
384
385 /*
386 * Following are a sequence of variable length records. The prefix/header
387 * of each record is defined by ValueProfRecord type. The number of
388 * records is NumValueKinds.
389 * ValueProfRecord Record_1;
390 * ValueProfRecord Record_N;
391 */
392
393#if __cplusplus
394 /*!
395 * Return the total size in bytes of the on-disk value profile data
396 * given the data stored in Record.
397 */
398 static uint32_t getSize(const InstrProfRecord &Record);
399 /*!
400 * Return a pointer to \c ValueProfData instance ready to be streamed.
401 */
402 static std::unique_ptr<ValueProfData>
403 serializeFrom(const InstrProfRecord &Record);
404 /*!
405 * Check the integrity of the record.
406 */
407 Error checkIntegrity();
408 /*!
409 * Return a pointer to \c ValueProfileData instance ready to be read.
410 * All data in the instance are properly byte swapped. The input
411 * data is assumed to be in little endian order.
412 */
413 static Expected<std::unique_ptr<ValueProfData>>
414 getValueProfData(const unsigned char *SrcBuffer,
415 const unsigned char *const SrcBufferEnd,
416 llvm::endianness SrcDataEndianness);
417 /*!
418 * Swap byte order from \c Endianness order to host byte order.
419 */
420 void swapBytesToHost(llvm::endianness Endianness);
421 /*!
422 * Swap byte order from host byte order to \c Endianness order.
423 */
424 void swapBytesFromHost(llvm::endianness Endianness);
425 /*!
426 * Return the total size of \c ValueProfileData.
427 */
428 uint32_t getSize() const { return TotalSize; }
429 /*!
430 * Read data from this data and save it to \c Record.
431 */
432 void deserializeTo(InstrProfRecord &Record,
433 InstrProfSymtab *SymTab);
434 void operator delete(void *ptr) { ::operator delete(ptr); }
435#endif
436} ValueProfData;
437
438/*
439 * The closure is designed to abstact away two types of value profile data:
440 * - InstrProfRecord which is the primary data structure used to
441 * represent profile data in host tools (reader, writer, and profile-use)
442 * - value profile runtime data structure suitable to be used by C
443 * runtime library.
444 *
445 * Both sources of data need to serialize to disk/memory-buffer in common
446 * format: ValueProfData. The abstraction allows compiler-rt's raw profiler
447 * writer to share the same format and code with indexed profile writer.
448 *
449 * For documentation of the member methods below, refer to corresponding methods
450 * in class InstrProfRecord.
451 */
452typedef struct ValueProfRecordClosure {
453 const void *Record;
454 uint32_t (*GetNumValueKinds)(const void *Record);
455 uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind);
456 uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind);
457 uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S);
458
459 /*
460 * After extracting the value profile data from the value profile record,
461 * this method is used to map the in-memory value to on-disk value. If
462 * the method is null, value will be written out untranslated.
463 */
464 uint64_t (*RemapValueData)(uint32_t, uint64_t Value);
465 void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K,
466 uint32_t S);
467 ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes);
468} ValueProfRecordClosure;
469
470INSTR_PROF_VISIBILITY ValueProfRecord *
471getFirstValueProfRecord(ValueProfData *VPD);
472INSTR_PROF_VISIBILITY ValueProfRecord *
473getValueProfRecordNext(ValueProfRecord *VPR);
474INSTR_PROF_VISIBILITY InstrProfValueData *
475getValueProfRecordValueData(ValueProfRecord *VPR);
476INSTR_PROF_VISIBILITY uint32_t
477getValueProfRecordHeaderSize(uint32_t NumValueSites);
478
479#undef INSTR_PROF_VALUE_PROF_DATA
480#endif /* INSTR_PROF_VALUE_PROF_DATA */
481
482
483#ifdef INSTR_PROF_COMMON_API_IMPL
484#define INSTR_PROF_DATA_DEFINED
485#ifdef __cplusplus
486#define INSTR_PROF_INLINE inline
487#define INSTR_PROF_NULLPTR nullptr
488#else
489#define INSTR_PROF_INLINE
490#define INSTR_PROF_NULLPTR NULL
491#endif
492
493#ifndef offsetof
494#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
495#endif
496
497/*!
498 * Return the \c ValueProfRecord header size including the
499 * padding bytes.
500 */
501INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
502uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) {
503 uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) +
504 sizeof(uint8_t) * NumValueSites;
505 /* Round the size to multiple of 8 bytes. */
506 Size = (Size + 7) & ~7;
507 return Size;
508}
509
510/*!
511 * Return the total size of the value profile record including the
512 * header and the value data.
513 */
514INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
515uint32_t getValueProfRecordSize(uint32_t NumValueSites,
516 uint32_t NumValueData) {
517 return getValueProfRecordHeaderSize(NumValueSites) +
518 sizeof(InstrProfValueData) * NumValueData;
519}
520
521/*!
522 * Return the pointer to the start of value data array.
523 */
524INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
525InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) {
526 return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize(
527 This->NumValueSites));
528}
529
530/*!
531 * Return the total number of value data for \c This record.
532 */
533INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
534uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) {
535 uint32_t NumValueData = 0;
536 uint32_t I;
537 for (I = 0; I < This->NumValueSites; I++)
538 NumValueData += This->SiteCountArray[I];
539 return NumValueData;
540}
541
542/*!
543 * Use this method to advance to the next \c This \c ValueProfRecord.
544 */
545INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
546ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) {
547 uint32_t NumValueData = getValueProfRecordNumValueData(This);
548 return (ValueProfRecord *)((char *)This +
549 getValueProfRecordSize(This->NumValueSites,
550 NumValueData));
551}
552
553/*!
554 * Return the first \c ValueProfRecord instance.
555 */
556INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
557ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) {
558 return (ValueProfRecord *)((char *)This + sizeof(ValueProfData));
559}
560
561/* Closure based interfaces. */
562
563/*!
564 * Return the total size in bytes of the on-disk value profile data
565 * given the data stored in Record.
566 */
567INSTR_PROF_VISIBILITY uint32_t
568getValueProfDataSize(ValueProfRecordClosure *Closure) {
569 uint32_t Kind;
570 uint32_t TotalSize = sizeof(ValueProfData);
571 const void *Record = Closure->Record;
572
573 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
574 uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind);
575 if (!NumValueSites)
576 continue;
577 TotalSize += getValueProfRecordSize(NumValueSites,
578 Closure->GetNumValueData(Record, Kind));
579 }
580 return TotalSize;
581}
582
583/*!
584 * Extract value profile data of a function for the profile kind \c ValueKind
585 * from the \c Closure and serialize the data into \c This record instance.
586 */
587INSTR_PROF_VISIBILITY void
588serializeValueProfRecordFrom(ValueProfRecord *This,
589 ValueProfRecordClosure *Closure,
590 uint32_t ValueKind, uint32_t NumValueSites) {
591 uint32_t S;
592 const void *Record = Closure->Record;
593 This->Kind = ValueKind;
594 This->NumValueSites = NumValueSites;
595 InstrProfValueData *DstVD = getValueProfRecordValueData(This);
596
597 for (S = 0; S < NumValueSites; S++) {
598 uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S);
599 This->SiteCountArray[S] = ND;
600 Closure->GetValueForSite(Record, DstVD, ValueKind, S);
601 DstVD += ND;
602 }
603}
604
605/*!
606 * Extract value profile data of a function from the \c Closure
607 * and serialize the data into \c DstData if it is not NULL or heap
608 * memory allocated by the \c Closure's allocator method. If \c
609 * DstData is not null, the caller is expected to set the TotalSize
610 * in DstData.
611 */
612INSTR_PROF_VISIBILITY ValueProfData *
613serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
614 ValueProfData *DstData) {
615 uint32_t Kind;
616 uint32_t TotalSize =
617 DstData ? DstData->TotalSize : getValueProfDataSize(Closure);
618
619 ValueProfData *VPD =
620 DstData ? DstData : Closure->AllocValueProfData(TotalSize);
621
622 VPD->TotalSize = TotalSize;
623 VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record);
624 ValueProfRecord *VR = getFirstValueProfRecord(VPD);
625 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
626 uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind);
627 if (!NumValueSites)
628 continue;
629 serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites);
630 VR = getValueProfRecordNext(VR);
631 }
632 return VPD;
633}
634
635#undef INSTR_PROF_COMMON_API_IMPL
636#endif /* INSTR_PROF_COMMON_API_IMPL */
637
638/*============================================================================*/
639
640#ifndef INSTR_PROF_DATA_DEFINED
641
642#ifndef INSTR_PROF_DATA_INC
643#define INSTR_PROF_DATA_INC
644
645/* Helper macros. */
646#define INSTR_PROF_SIMPLE_QUOTE(x) #x
647#define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x)
648#define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y
649#define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y)
650
651/* Magic number to detect file format and endianness.
652 * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0,
653 * so that utilities, like strings, don't grab it as a string. 129 is also
654 * invalid UTF-8, and high enough to be interesting.
655 * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR"
656 * for 32-bit platforms.
657 */
658#define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
659 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
660 (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129
661#define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
662 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
663 (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
664
665/* Raw profile format version (start from 1). */
666#define INSTR_PROF_RAW_VERSION 9
667/* Indexed profile format version (start from 1). */
668#define INSTR_PROF_INDEX_VERSION 11
669/* Coverage mapping format version (start from 0). */
670#define INSTR_PROF_COVMAP_VERSION 6
671
672/* Profile version is always of type uint64_t. Reserve the upper 32 bits in the
673 * version for other variants of profile. We set the 8th most significant bit
674 * (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation
675 * generated profile, and 0 if this is a Clang FE generated profile.
676 * 1 in bit 57 indicates there are context-sensitive records in the profile.
677 * The 59th bit indicates whether to use debug info to correlate profiles.
678 * The 60th bit indicates single byte coverage instrumentation.
679 * The 61st bit indicates function entry instrumentation only.
680 * The 62nd bit indicates whether memory profile information is present.
681 * The 63rd bit indicates if this is a temporal profile.
682 */
683#define VARIANT_MASKS_ALL 0xffffffff00000000ULL
684#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
685#define VARIANT_MASK_IR_PROF (0x1ULL << 56)
686#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57)
687#define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58)
688#define VARIANT_MASK_DBG_CORRELATE (0x1ULL << 59)
689#define VARIANT_MASK_BYTE_COVERAGE (0x1ULL << 60)
690#define VARIANT_MASK_FUNCTION_ENTRY_ONLY (0x1ULL << 61)
691#define VARIANT_MASK_MEMPROF (0x1ULL << 62)
692#define VARIANT_MASK_TEMPORAL_PROF (0x1ULL << 63)
693#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version
694#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime
695#define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias
696#define INSTR_PROF_PROFILE_SET_TIMESTAMP __llvm_profile_set_timestamp
697
698/* The variable that holds the name of the profile data
699 * specified via command line. */
700#define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename
701
702/* section name strings common to all targets other
703 than WIN32 */
704#define INSTR_PROF_DATA_COMMON __llvm_prf_data
705#define INSTR_PROF_NAME_COMMON __llvm_prf_names
706#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts
707#define INSTR_PROF_BITS_COMMON __llvm_prf_bits
708#define INSTR_PROF_VALS_COMMON __llvm_prf_vals
709#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds
710#define INSTR_PROF_COVMAP_COMMON __llvm_covmap
711#define INSTR_PROF_COVFUN_COMMON __llvm_covfun
712#define INSTR_PROF_COVDATA_COMMON __llvm_covdata
713#define INSTR_PROF_COVNAME_COMMON __llvm_covnames
714#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile
715/* Windows section names. Because these section names contain dollar characters,
716 * they must be quoted.
717 */
718#define INSTR_PROF_DATA_COFF ".lprfd$M"
719#define INSTR_PROF_NAME_COFF ".lprfn$M"
720#define INSTR_PROF_CNTS_COFF ".lprfc$M"
721#define INSTR_PROF_BITS_COFF ".lprfb$M"
722#define INSTR_PROF_VALS_COFF ".lprfv$M"
723#define INSTR_PROF_VNODES_COFF ".lprfnd$M"
724#define INSTR_PROF_COVMAP_COFF ".lcovmap$M"
725#define INSTR_PROF_COVFUN_COFF ".lcovfun$M"
726/* Since cov data and cov names sections are not allocated, we don't need to
727 * access them at runtime.
728 */
729#define INSTR_PROF_COVDATA_COFF ".lcovd"
730#define INSTR_PROF_COVNAME_COFF ".lcovn"
731#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M"
732
733#ifdef _WIN32
734/* Runtime section names and name strings. */
735#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF
736#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF
737#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF
738#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_BITS_COFF
739/* Array of pointers. Each pointer points to a list
740 * of value nodes associated with one value site.
741 */
742#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF
743/* Value profile nodes section. */
744#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF
745#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF
746#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF
747#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_COVDATA_COFF
748#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_COVNAME_COFF
749#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF
750#else
751/* Runtime section names and name strings. */
752#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON)
753#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON)
754#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON)
755#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON)
756/* Array of pointers. Each pointer points to a list
757 * of value nodes associated with one value site.
758 */
759#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON)
760/* Value profile nodes section. */
761#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON)
762#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON)
763#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON)
764#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON)
765#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON)
766/* Order file instrumentation. */
767#define INSTR_PROF_ORDERFILE_SECT_NAME \
768 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON)
769#endif
770
771#define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer
772#define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \
773 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME)
774#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx
775#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \
776 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME)
777
778/* Macros to define start/stop section symbol for a given
779 * section on Linux. For instance
780 * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will
781 * expand to __start___llvm_prof_data
782 */
783#define INSTR_PROF_SECT_START(Sect) \
784 INSTR_PROF_CONCAT(__start_,Sect)
785#define INSTR_PROF_SECT_STOP(Sect) \
786 INSTR_PROF_CONCAT(__stop_,Sect)
787
788/* Value Profiling API linkage name. */
789#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target
790#define INSTR_PROF_VALUE_PROF_FUNC_STR \
791 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC)
792#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop
793#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \
794 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC)
795
796/* InstrProfile per-function control data alignment. */
797#define INSTR_PROF_DATA_ALIGNMENT 8
798
799/* The data structure that represents a tracked value by the
800 * value profiler.
801 */
802typedef struct InstrProfValueData {
803 /* Profiled value. */
804 uint64_t Value;
805 /* Number of times the value appears in the training run. */
806 uint64_t Count;
807} InstrProfValueData;
808
809#endif /* INSTR_PROF_DATA_INC */
810
811#ifndef INSTR_ORDER_FILE_INC
812/* The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). */
813#define INSTR_ORDER_FILE_BUFFER_SIZE 131072
814#define INSTR_ORDER_FILE_BUFFER_BITS 17
815#define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff
816#endif /* INSTR_ORDER_FILE_INC */
817#else
818#undef INSTR_PROF_DATA_DEFINED
819#endif
820
821#undef COVMAP_V2_OR_V3
822
823#ifdef INSTR_PROF_VALUE_PROF_MEMOP_API
824
825#ifdef __cplusplus
826#define INSTR_PROF_INLINE inline
827#else
828#define INSTR_PROF_INLINE
829#endif
830
831/* The value range buckets (22 buckets) for the memop size value profiling looks
832 * like:
833 *
834 * [0, 0]
835 * [1, 1]
836 * [2, 2]
837 * [3, 3]
838 * [4, 4]
839 * [5, 5]
840 * [6, 6]
841 * [7, 7]
842 * [8, 8]
843 * [9, 15]
844 * [16, 16]
845 * [17, 31]
846 * [32, 32]
847 * [33, 63]
848 * [64, 64]
849 * [65, 127]
850 * [128, 128]
851 * [129, 255]
852 * [256, 256]
853 * [257, 511]
854 * [512, 512]
855 * [513, UINT64_MAX]
856 *
857 * Each range has a 'representative value' which is the lower end value of the
858 * range and used to store in the runtime profile data records and the VP
859 * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127].
860 */
861#define INSTR_PROF_NUM_BUCKETS 22
862
863/*
864 * Clz and Popcount. This code was copied from
865 * compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and
866 * llvm/include/llvm/Support/MathExtras.h.
867 */
868#if defined(_MSC_VER) && !defined(__clang__)
869
870#include <intrin.h>
871INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
872int InstProfClzll(unsigned long long X) {
873 unsigned long LeadZeroIdx = 0;
874#if !defined(_M_ARM64) && !defined(_M_X64)
875 // Scan the high 32 bits.
876 if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32)))
877 return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset
878 // from the MSB.
879 // Scan the low 32 bits.
880 if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X)))
881 return (int)(63 - LeadZeroIdx);
882#else
883 if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx;
884#endif
885 return 64;
886}
887INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
888int InstProfPopcountll(unsigned long long X) {
889 // This code originates from https://reviews.llvm.org/rG30626254510f.
890 unsigned long long v = X;
891 v = v - ((v >> 1) & 0x5555555555555555ULL);
892 v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);
893 v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
894 return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56);
895}
896
897#else
898
899INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
900int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); }
901INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
902int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); }
903
904#endif /* defined(_MSC_VER) && !defined(__clang__) */
905
906/* Map an (observed) memop size value to the representative value of its range.
907 * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */
908INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t
909InstrProfGetRangeRepValue(uint64_t Value) {
910 if (Value <= 8)
911 // The first ranges are individually tracked. Use the value as is.
912 return Value;
913 else if (Value >= 513)
914 // The last range is mapped to its lowest value.
915 return 513;
916 else if (InstProfPopcountll(Value) == 1)
917 // If it's a power of two, use it as is.
918 return Value;
919 else
920 // Otherwise, take to the previous power of two + 1.
921 return (UINT64_C(1) << (64 - InstProfClzll(Value) - 1)) + 1;
922}
923
924/* Return true if the range that an (observed) memop size value belongs to has
925 * only a single value in the range. For example, 0 -> true, 8 -> true, 10 ->
926 * false, 64 -> true, 100 -> false, 513 -> false. */
927INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned
928InstrProfIsSingleValRange(uint64_t Value) {
929 if (Value <= 8)
930 // The first ranges are individually tracked.
931 return 1;
932 else if (InstProfPopcountll(Value) == 1)
933 // If it's a power of two, there's only one value.
934 return 1;
935 else
936 // Otherwise, there's more than one value in the range.
937 return 0;
938}
939
940#endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */
941

source code of llvm/include/llvm/ProfileData/InstrProfData.inc