1/*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\
2|*
3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4|* See https://llvm.org/LICENSE.txt for license information.
5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6|*
7\*===----------------------------------------------------------------------===*/
8/*
9 * This is the main file that defines all the data structure, signature,
10 * constant literals that are shared across profiling runtime library,
11 * compiler (instrumentation), and host tools (reader/writer). The entities
12 * defined in this file affect the profile runtime ABI, the raw profile format,
13 * or both.
14 *
15 * The file has two identical copies. The primary copy lives in LLVM and
16 * the other one sits in compiler-rt/lib/profile directory. To make changes
17 * in this file, first modify the primary copy and copy it over to compiler-rt.
18 * Testing of any change in this file can start only after the two copies are
19 * synced up.
20 *
21 * The first part of the file includes macros that defines types, names, and
22 * initializers for the member fields of the core data structures. The field
23 * declarations for one structure is enabled by defining the field activation
24 * macro associated with that structure. Only one field activation record
25 * can be defined at one time and the rest definitions will be filtered out by
26 * the preprocessor.
27 *
28 * Examples of how the template is used to instantiate structure definition:
29 * 1. To declare a structure:
30 *
31 * struct ProfData {
32 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
33 * Type Name;
34 * #include "llvm/ProfileData/InstrProfData.inc"
35 * };
36 *
37 * 2. To construct LLVM type arrays for the struct type:
38 *
39 * Type *DataTypes[] = {
40 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
41 * LLVMType,
42 * #include "llvm/ProfileData/InstrProfData.inc"
43 * };
44 *
45 * 4. To construct constant array for the initializers:
46 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
47 * Initializer,
48 * Constant *ConstantVals[] = {
49 * #include "llvm/ProfileData/InstrProfData.inc"
50 * };
51 *
52 *
53 * The second part of the file includes definitions all other entities that
54 * are related to runtime ABI and format. When no field activation macro is
55 * defined, this file can be included to introduce the definitions.
56 *
57\*===----------------------------------------------------------------------===*/
58
59/* Functions marked with INSTR_PROF_VISIBILITY must have hidden visibility in
60 * the compiler runtime. */
61#ifndef INSTR_PROF_VISIBILITY
62#define INSTR_PROF_VISIBILITY
63#endif
64
65/* INSTR_PROF_DATA start. */
66/* Definition of member fields of the per-function control structure. */
67#ifndef INSTR_PROF_DATA
68#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer)
69#else
70#define INSTR_PROF_DATA_DEFINED
71#endif
72INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
73 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
74 IndexedInstrProf::ComputeHash(getPGOFuncNameVarInitializer(Inc->getName()))))
75INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
76 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
77 Inc->getHash()->getZExtValue()))
78INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt64PtrTy(Ctx), CounterPtr, \
79 ConstantExpr::getBitCast(CounterPtr, \
80 llvm::Type::getInt64PtrTy(Ctx)))
81/* This is used to map function pointers for the indirect call targets to
82 * function name hashes during the conversion from raw to merged profile
83 * data.
84 */
85INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), FunctionPointer, \
86 FunctionAddr)
87INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \
88 ValuesPtrExpr)
89INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \
90 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters))
91INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \
92 ConstantArray::get(Int16ArrayTy, Int16ArrayVals))
93#undef INSTR_PROF_DATA
94/* INSTR_PROF_DATA end. */
95
96
97/* This is an internal data structure used by value profiler. It
98 * is defined here to allow serialization code sharing by LLVM
99 * to be used in unit test.
100 *
101 * typedef struct ValueProfNode {
102 * // InstrProfValueData VData;
103 * uint64_t Value;
104 * uint64_t Count;
105 * struct ValueProfNode *Next;
106 * } ValueProfNode;
107 */
108/* INSTR_PROF_VALUE_NODE start. */
109#ifndef INSTR_PROF_VALUE_NODE
110#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer)
111#else
112#define INSTR_PROF_DATA_DEFINED
113#endif
114INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Value, \
115 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
116INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \
117 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
118INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::Type::getInt8PtrTy(Ctx), Next, \
119 ConstantInt::get(llvm::Type::GetInt8PtrTy(Ctx), 0))
120#undef INSTR_PROF_VALUE_NODE
121/* INSTR_PROF_VALUE_NODE end. */
122
123/* INSTR_PROF_RAW_HEADER start */
124/* Definition of member fields of the raw profile header data structure. */
125#ifndef INSTR_PROF_RAW_HEADER
126#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer)
127#else
128#define INSTR_PROF_DATA_DEFINED
129#endif
130INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic())
131INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version())
132INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize)
133INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters)
134INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize)
135INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters)
136INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize)
137INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin)
138INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
139INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
140#undef INSTR_PROF_RAW_HEADER
141/* INSTR_PROF_RAW_HEADER end */
142
143/* VALUE_PROF_FUNC_PARAM start */
144/* Definition of parameter types of the runtime API used to do value profiling
145 * for a given value site.
146 */
147#ifndef VALUE_PROF_FUNC_PARAM
148#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType)
149#define INSTR_PROF_COMMA
150#else
151#define INSTR_PROF_DATA_DEFINED
152#define INSTR_PROF_COMMA ,
153#endif
154VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \
155 INSTR_PROF_COMMA
156VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA
157VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
158#undef VALUE_PROF_FUNC_PARAM
159#undef INSTR_PROF_COMMA
160/* VALUE_PROF_FUNC_PARAM end */
161
162/* VALUE_PROF_KIND start */
163#ifndef VALUE_PROF_KIND
164#define VALUE_PROF_KIND(Enumerator, Value, Descr)
165#else
166#define INSTR_PROF_DATA_DEFINED
167#endif
168/* For indirect function call value profiling, the addresses of the target
169 * functions are profiled by the instrumented code. The target addresses are
170 * written in the raw profile data and converted to target function name's MD5
171 * hash by the profile reader during deserialization. Typically, this happens
172 * when the raw profile data is read during profile merging.
173 *
174 * For this remapping the ProfData is used. ProfData contains both the function
175 * name hash and the function address.
176 */
177VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target")
178/* For memory intrinsic functions size profiling. */
179VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size")
180/* These two kinds must be the last to be
181 * declared. This is to make sure the string
182 * array created with the template can be
183 * indexed with the kind value.
184 */
185VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first")
186VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last")
187
188#undef VALUE_PROF_KIND
189/* VALUE_PROF_KIND end */
190
191#undef COVMAP_V2_OR_V3
192#ifdef COVMAP_V2
193#define COVMAP_V2_OR_V3
194#endif
195#ifdef COVMAP_V3
196#define COVMAP_V2_OR_V3
197#endif
198
199/* COVMAP_FUNC_RECORD start */
200/* Definition of member fields of the function record structure in coverage
201 * map.
202 */
203#ifndef COVMAP_FUNC_RECORD
204#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer)
205#else
206#define INSTR_PROF_DATA_DEFINED
207#endif
208#ifdef COVMAP_V1
209COVMAP_FUNC_RECORD(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \
210 NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \
211 llvm::Type::getInt8PtrTy(Ctx)))
212COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \
213 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \
214 NameValue.size()))
215#endif
216#ifdef COVMAP_V2_OR_V3
217COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
218 llvm::ConstantInt::get( \
219 llvm::Type::getInt64Ty(Ctx), NameHash))
220#endif
221COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \
222 llvm::ConstantInt::get( \
223 llvm::Type::getInt32Ty(Ctx), CoverageMapping.size()))
224COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
225 llvm::ConstantInt::get( \
226 llvm::Type::getInt64Ty(Ctx), FuncHash))
227#ifdef COVMAP_V3
228COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FilenamesRef, \
229 llvm::ConstantInt::get( \
230 llvm::Type::getInt64Ty(Ctx), FilenamesRef))
231COVMAP_FUNC_RECORD(const char, \
232 llvm::ArrayType::get(llvm::Type::getInt8Ty(Ctx), \
233 CoverageMapping.size()), \
234 CoverageMapping,
235 llvm::ConstantDataArray::getRaw( \
236 CoverageMapping, CoverageMapping.size(), \
237 llvm::Type::getInt8Ty(Ctx)))
238#endif
239#undef COVMAP_FUNC_RECORD
240/* COVMAP_FUNC_RECORD end. */
241
242/* COVMAP_HEADER start */
243/* Definition of member fields of coverage map header.
244 */
245#ifndef COVMAP_HEADER
246#define COVMAP_HEADER(Type, LLVMType, Name, Initializer)
247#else
248#define INSTR_PROF_DATA_DEFINED
249#endif
250COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \
251 llvm::ConstantInt::get(Int32Ty, NRecords))
252COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \
253 llvm::ConstantInt::get(Int32Ty, FilenamesSize))
254COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \
255 llvm::ConstantInt::get(Int32Ty, CoverageMappingSize))
256COVMAP_HEADER(uint32_t, Int32Ty, Version, \
257 llvm::ConstantInt::get(Int32Ty, CovMapVersion::CurrentVersion))
258#undef COVMAP_HEADER
259/* COVMAP_HEADER end. */
260
261
262#ifdef INSTR_PROF_SECT_ENTRY
263#define INSTR_PROF_DATA_DEFINED
264INSTR_PROF_SECT_ENTRY(IPSK_data, \
265 INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \
266 INSTR_PROF_DATA_COFF, "__DATA,")
267INSTR_PROF_SECT_ENTRY(IPSK_cnts, \
268 INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \
269 INSTR_PROF_CNTS_COFF, "__DATA,")
270INSTR_PROF_SECT_ENTRY(IPSK_name, \
271 INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \
272 INSTR_PROF_NAME_COFF, "__DATA,")
273INSTR_PROF_SECT_ENTRY(IPSK_vals, \
274 INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \
275 INSTR_PROF_VALS_COFF, "__DATA,")
276INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \
277 INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \
278 INSTR_PROF_VNODES_COFF, "__DATA,")
279INSTR_PROF_SECT_ENTRY(IPSK_covmap, \
280 INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \
281 INSTR_PROF_COVMAP_COFF, "__LLVM_COV,")
282INSTR_PROF_SECT_ENTRY(IPSK_covfun, \
283 INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON), \
284 INSTR_PROF_COVFUN_COFF, "__LLVM_COV,")
285INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \
286 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \
287 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,")
288
289#undef INSTR_PROF_SECT_ENTRY
290#endif
291
292
293#ifdef INSTR_PROF_VALUE_PROF_DATA
294#define INSTR_PROF_DATA_DEFINED
295
296#define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255
297/*!
298 * This is the header of the data structure that defines the on-disk
299 * layout of the value profile data of a particular kind for one function.
300 */
301typedef struct ValueProfRecord {
302 /* The kind of the value profile record. */
303 uint32_t Kind;
304 /*
305 * The number of value profile sites. It is guaranteed to be non-zero;
306 * otherwise the record for this kind won't be emitted.
307 */
308 uint32_t NumValueSites;
309 /*
310 * The first element of the array that stores the number of profiled
311 * values for each value site. The size of the array is NumValueSites.
312 * Since NumValueSites is greater than zero, there is at least one
313 * element in the array.
314 */
315 uint8_t SiteCountArray[1];
316
317 /*
318 * The fake declaration is for documentation purpose only.
319 * Align the start of next field to be on 8 byte boundaries.
320 uint8_t Padding[X];
321 */
322
323 /* The array of value profile data. The size of the array is the sum
324 * of all elements in SiteCountArray[].
325 InstrProfValueData ValueData[];
326 */
327
328#ifdef __cplusplus
329 /*!
330 * Return the number of value sites.
331 */
332 uint32_t getNumValueSites() const { return NumValueSites; }
333 /*!
334 * Read data from this record and save it to Record.
335 */
336 void deserializeTo(InstrProfRecord &Record,
337 InstrProfSymtab *SymTab);
338 /*
339 * In-place byte swap:
340 * Do byte swap for this instance. \c Old is the original order before
341 * the swap, and \c New is the New byte order.
342 */
343 void swapBytes(support::endianness Old, support::endianness New);
344#endif
345} ValueProfRecord;
346
347/*!
348 * Per-function header/control data structure for value profiling
349 * data in indexed format.
350 */
351typedef struct ValueProfData {
352 /*
353 * Total size in bytes including this field. It must be a multiple
354 * of sizeof(uint64_t).
355 */
356 uint32_t TotalSize;
357 /*
358 *The number of value profile kinds that has value profile data.
359 * In this implementation, a value profile kind is considered to
360 * have profile data if the number of value profile sites for the
361 * kind is not zero. More aggressively, the implementation can
362 * choose to check the actual data value: if none of the value sites
363 * has any profiled values, the kind can be skipped.
364 */
365 uint32_t NumValueKinds;
366
367 /*
368 * Following are a sequence of variable length records. The prefix/header
369 * of each record is defined by ValueProfRecord type. The number of
370 * records is NumValueKinds.
371 * ValueProfRecord Record_1;
372 * ValueProfRecord Record_N;
373 */
374
375#if __cplusplus
376 /*!
377 * Return the total size in bytes of the on-disk value profile data
378 * given the data stored in Record.
379 */
380 static uint32_t getSize(const InstrProfRecord &Record);
381 /*!
382 * Return a pointer to \c ValueProfData instance ready to be streamed.
383 */
384 static std::unique_ptr<ValueProfData>
385 serializeFrom(const InstrProfRecord &Record);
386 /*!
387 * Check the integrity of the record.
388 */
389 Error checkIntegrity();
390 /*!
391 * Return a pointer to \c ValueProfileData instance ready to be read.
392 * All data in the instance are properly byte swapped. The input
393 * data is assumed to be in little endian order.
394 */
395 static Expected<std::unique_ptr<ValueProfData>>
396 getValueProfData(const unsigned char *SrcBuffer,
397 const unsigned char *const SrcBufferEnd,
398 support::endianness SrcDataEndianness);
399 /*!
400 * Swap byte order from \c Endianness order to host byte order.
401 */
402 void swapBytesToHost(support::endianness Endianness);
403 /*!
404 * Swap byte order from host byte order to \c Endianness order.
405 */
406 void swapBytesFromHost(support::endianness Endianness);
407 /*!
408 * Return the total size of \c ValueProfileData.
409 */
410 uint32_t getSize() const { return TotalSize; }
411 /*!
412 * Read data from this data and save it to \c Record.
413 */
414 void deserializeTo(InstrProfRecord &Record,
415 InstrProfSymtab *SymTab);
416 void operator delete(void *ptr) { ::operator delete(ptr); }
417#endif
418} ValueProfData;
419
420/*
421 * The closure is designed to abstact away two types of value profile data:
422 * - InstrProfRecord which is the primary data structure used to
423 * represent profile data in host tools (reader, writer, and profile-use)
424 * - value profile runtime data structure suitable to be used by C
425 * runtime library.
426 *
427 * Both sources of data need to serialize to disk/memory-buffer in common
428 * format: ValueProfData. The abstraction allows compiler-rt's raw profiler
429 * writer to share the same format and code with indexed profile writer.
430 *
431 * For documentation of the member methods below, refer to corresponding methods
432 * in class InstrProfRecord.
433 */
434typedef struct ValueProfRecordClosure {
435 const void *Record;
436 uint32_t (*GetNumValueKinds)(const void *Record);
437 uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind);
438 uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind);
439 uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S);
440
441 /*
442 * After extracting the value profile data from the value profile record,
443 * this method is used to map the in-memory value to on-disk value. If
444 * the method is null, value will be written out untranslated.
445 */
446 uint64_t (*RemapValueData)(uint32_t, uint64_t Value);
447 void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K,
448 uint32_t S);
449 ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes);
450} ValueProfRecordClosure;
451
452INSTR_PROF_VISIBILITY ValueProfRecord *
453getFirstValueProfRecord(ValueProfData *VPD);
454INSTR_PROF_VISIBILITY ValueProfRecord *
455getValueProfRecordNext(ValueProfRecord *VPR);
456INSTR_PROF_VISIBILITY InstrProfValueData *
457getValueProfRecordValueData(ValueProfRecord *VPR);
458INSTR_PROF_VISIBILITY uint32_t
459getValueProfRecordHeaderSize(uint32_t NumValueSites);
460
461#undef INSTR_PROF_VALUE_PROF_DATA
462#endif /* INSTR_PROF_VALUE_PROF_DATA */
463
464
465#ifdef INSTR_PROF_COMMON_API_IMPL
466#define INSTR_PROF_DATA_DEFINED
467#ifdef __cplusplus
468#define INSTR_PROF_INLINE inline
469#define INSTR_PROF_NULLPTR nullptr
470#else
471#define INSTR_PROF_INLINE
472#define INSTR_PROF_NULLPTR NULL
473#endif
474
475#ifndef offsetof
476#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
477#endif
478
479/*!
480 * Return the \c ValueProfRecord header size including the
481 * padding bytes.
482 */
483INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
484uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) {
485 uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) +
486 sizeof(uint8_t) * NumValueSites;
487 /* Round the size to multiple of 8 bytes. */
488 Size = (Size + 7) & ~7;
489 return Size;
490}
491
492/*!
493 * Return the total size of the value profile record including the
494 * header and the value data.
495 */
496INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
497uint32_t getValueProfRecordSize(uint32_t NumValueSites,
498 uint32_t NumValueData) {
499 return getValueProfRecordHeaderSize(NumValueSites) +
500 sizeof(InstrProfValueData) * NumValueData;
501}
502
503/*!
504 * Return the pointer to the start of value data array.
505 */
506INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
507InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) {
508 return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize(
509 This->NumValueSites));
510}
511
512/*!
513 * Return the total number of value data for \c This record.
514 */
515INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
516uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) {
517 uint32_t NumValueData = 0;
518 uint32_t I;
519 for (I = 0; I < This->NumValueSites; I++)
520 NumValueData += This->SiteCountArray[I];
521 return NumValueData;
522}
523
524/*!
525 * Use this method to advance to the next \c This \c ValueProfRecord.
526 */
527INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
528ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) {
529 uint32_t NumValueData = getValueProfRecordNumValueData(This);
530 return (ValueProfRecord *)((char *)This +
531 getValueProfRecordSize(This->NumValueSites,
532 NumValueData));
533}
534
535/*!
536 * Return the first \c ValueProfRecord instance.
537 */
538INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
539ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) {
540 return (ValueProfRecord *)((char *)This + sizeof(ValueProfData));
541}
542
543/* Closure based interfaces. */
544
545/*!
546 * Return the total size in bytes of the on-disk value profile data
547 * given the data stored in Record.
548 */
549INSTR_PROF_VISIBILITY uint32_t
550getValueProfDataSize(ValueProfRecordClosure *Closure) {
551 uint32_t Kind;
552 uint32_t TotalSize = sizeof(ValueProfData);
553 const void *Record = Closure->Record;
554
555 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
556 uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind);
557 if (!NumValueSites)
558 continue;
559 TotalSize += getValueProfRecordSize(NumValueSites,
560 Closure->GetNumValueData(Record, Kind));
561 }
562 return TotalSize;
563}
564
565/*!
566 * Extract value profile data of a function for the profile kind \c ValueKind
567 * from the \c Closure and serialize the data into \c This record instance.
568 */
569INSTR_PROF_VISIBILITY void
570serializeValueProfRecordFrom(ValueProfRecord *This,
571 ValueProfRecordClosure *Closure,
572 uint32_t ValueKind, uint32_t NumValueSites) {
573 uint32_t S;
574 const void *Record = Closure->Record;
575 This->Kind = ValueKind;
576 This->NumValueSites = NumValueSites;
577 InstrProfValueData *DstVD = getValueProfRecordValueData(This);
578
579 for (S = 0; S < NumValueSites; S++) {
580 uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S);
581 This->SiteCountArray[S] = ND;
582 Closure->GetValueForSite(Record, DstVD, ValueKind, S);
583 DstVD += ND;
584 }
585}
586
587/*!
588 * Extract value profile data of a function from the \c Closure
589 * and serialize the data into \c DstData if it is not NULL or heap
590 * memory allocated by the \c Closure's allocator method. If \c
591 * DstData is not null, the caller is expected to set the TotalSize
592 * in DstData.
593 */
594INSTR_PROF_VISIBILITY ValueProfData *
595serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
596 ValueProfData *DstData) {
597 uint32_t Kind;
598 uint32_t TotalSize =
599 DstData ? DstData->TotalSize : getValueProfDataSize(Closure);
600
601 ValueProfData *VPD =
602 DstData ? DstData : Closure->AllocValueProfData(TotalSize);
603
604 VPD->TotalSize = TotalSize;
605 VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record);
606 ValueProfRecord *VR = getFirstValueProfRecord(VPD);
607 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
608 uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind);
609 if (!NumValueSites)
610 continue;
611 serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites);
612 VR = getValueProfRecordNext(VR);
613 }
614 return VPD;
615}
616
617#undef INSTR_PROF_COMMON_API_IMPL
618#endif /* INSTR_PROF_COMMON_API_IMPL */
619
620/*============================================================================*/
621
622#ifndef INSTR_PROF_DATA_DEFINED
623
624#ifndef INSTR_PROF_DATA_INC
625#define INSTR_PROF_DATA_INC
626
627/* Helper macros. */
628#define INSTR_PROF_SIMPLE_QUOTE(x) #x
629#define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x)
630#define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y
631#define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y)
632
633/* Magic number to detect file format and endianness.
634 * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0,
635 * so that utilities, like strings, don't grab it as a string. 129 is also
636 * invalid UTF-8, and high enough to be interesting.
637 * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR"
638 * for 32-bit platforms.
639 */
640#define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
641 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
642 (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129
643#define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
644 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
645 (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
646
647/* Raw profile format version (start from 1). */
648#define INSTR_PROF_RAW_VERSION 5
649/* Indexed profile format version (start from 1). */
650#define INSTR_PROF_INDEX_VERSION 7
651/* Coverage mapping format version (start from 0). */
652#define INSTR_PROF_COVMAP_VERSION 5
653
654/* Profile version is always of type uint64_t. Reserve the upper 8 bits in the
655 * version for other variants of profile. We set the lowest bit of the upper 8
656 * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton
657 * generated profile, and 0 if this is a Clang FE generated profile.
658 * 1 in bit 57 indicates there are context-sensitive records in the profile.
659 */
660#define VARIANT_MASKS_ALL 0xff00000000000000ULL
661#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
662#define VARIANT_MASK_IR_PROF (0x1ULL << 56)
663#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57)
664#define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58)
665#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version
666#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime
667
668/* The variable that holds the name of the profile data
669 * specified via command line. */
670#define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename
671
672/* section name strings common to all targets other
673 than WIN32 */
674#define INSTR_PROF_DATA_COMMON __llvm_prf_data
675#define INSTR_PROF_NAME_COMMON __llvm_prf_names
676#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts
677#define INSTR_PROF_VALS_COMMON __llvm_prf_vals
678#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds
679#define INSTR_PROF_COVMAP_COMMON __llvm_covmap
680#define INSTR_PROF_COVFUN_COMMON __llvm_covfun
681#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile
682/* Windows section names. Because these section names contain dollar characters,
683 * they must be quoted.
684 */
685#define INSTR_PROF_DATA_COFF ".lprfd$M"
686#define INSTR_PROF_NAME_COFF ".lprfn$M"
687#define INSTR_PROF_CNTS_COFF ".lprfc$M"
688#define INSTR_PROF_VALS_COFF ".lprfv$M"
689#define INSTR_PROF_VNODES_COFF ".lprfnd$M"
690#define INSTR_PROF_COVMAP_COFF ".lcovmap$M"
691#define INSTR_PROF_COVFUN_COFF ".lcovfun$M"
692#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M"
693
694#ifdef _WIN32
695/* Runtime section names and name strings. */
696#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF
697#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF
698#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF
699/* Array of pointers. Each pointer points to a list
700 * of value nodes associated with one value site.
701 */
702#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF
703/* Value profile nodes section. */
704#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF
705#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF
706#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF
707#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF
708#else
709/* Runtime section names and name strings. */
710#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON)
711#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON)
712#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON)
713/* Array of pointers. Each pointer points to a list
714 * of value nodes associated with one value site.
715 */
716#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON)
717/* Value profile nodes section. */
718#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON)
719#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON)
720#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON)
721/* Order file instrumentation. */
722#define INSTR_PROF_ORDERFILE_SECT_NAME \
723 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON)
724#endif
725
726#define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer
727#define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \
728 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME)
729#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx
730#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \
731 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME)
732
733/* Macros to define start/stop section symbol for a given
734 * section on Linux. For instance
735 * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will
736 * expand to __start___llvm_prof_data
737 */
738#define INSTR_PROF_SECT_START(Sect) \
739 INSTR_PROF_CONCAT(__start_,Sect)
740#define INSTR_PROF_SECT_STOP(Sect) \
741 INSTR_PROF_CONCAT(__stop_,Sect)
742
743/* Value Profiling API linkage name. */
744#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target
745#define INSTR_PROF_VALUE_PROF_FUNC_STR \
746 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC)
747#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop
748#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \
749 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC)
750
751/* InstrProfile per-function control data alignment. */
752#define INSTR_PROF_DATA_ALIGNMENT 8
753
754/* The data structure that represents a tracked value by the
755 * value profiler.
756 */
757typedef struct InstrProfValueData {
758 /* Profiled value. */
759 uint64_t Value;
760 /* Number of times the value appears in the training run. */
761 uint64_t Count;
762} InstrProfValueData;
763
764#endif /* INSTR_PROF_DATA_INC */
765
766#ifndef INSTR_ORDER_FILE_INC
767/* The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). */
768#define INSTR_ORDER_FILE_BUFFER_SIZE 131072
769#define INSTR_ORDER_FILE_BUFFER_BITS 17
770#define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff
771#endif /* INSTR_ORDER_FILE_INC */
772#else
773#undef INSTR_PROF_DATA_DEFINED
774#endif
775
776#undef COVMAP_V2_OR_V3
777
778#ifdef INSTR_PROF_VALUE_PROF_MEMOP_API
779
780#ifdef __cplusplus
781#define INSTR_PROF_INLINE inline
782#else
783#define INSTR_PROF_INLINE
784#endif
785
786/* The value range buckets (22 buckets) for the memop size value profiling looks
787 * like:
788 *
789 * [0, 0]
790 * [1, 1]
791 * [2, 2]
792 * [3, 3]
793 * [4, 4]
794 * [5, 5]
795 * [6, 6]
796 * [7, 7]
797 * [8, 8]
798 * [9, 15]
799 * [16, 16]
800 * [17, 31]
801 * [32, 32]
802 * [33, 63]
803 * [64, 64]
804 * [65, 127]
805 * [128, 128]
806 * [129, 255]
807 * [256, 256]
808 * [257, 511]
809 * [512, 512]
810 * [513, UINT64_MAX]
811 *
812 * Each range has a 'representative value' which is the lower end value of the
813 * range and used to store in the runtime profile data records and the VP
814 * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127].
815 */
816#define INSTR_PROF_NUM_BUCKETS 22
817
818/*
819 * Clz and Popcount. This code was copied from
820 * compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and
821 * llvm/include/llvm/Support/MathExtras.h.
822 */
823#if defined(_MSC_VER) && !defined(__clang__)
824
825#include <intrin.h>
826INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
827int InstProfClzll(unsigned long long X) {
828 unsigned long LeadZeroIdx = 0;
829#if !defined(_M_ARM64) && !defined(_M_X64)
830 // Scan the high 32 bits.
831 if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32)))
832 return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset
833 // from the MSB.
834 // Scan the low 32 bits.
835 if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X)))
836 return (int)(63 - LeadZeroIdx);
837#else
838 if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx;
839#endif
840 return 64;
841}
842INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
843int InstProfPopcountll(unsigned long long X) {
844 // This code originates from https://reviews.llvm.org/rG30626254510f.
845 unsigned long long v = X;
846 v = v - ((v >> 1) & 0x5555555555555555ULL);
847 v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);
848 v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
849 return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56);
850}
851
852#else
853
854INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
855int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); }
856INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
857int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); }
858
859#endif /* defined(_MSC_VER) && !defined(__clang__) */
860
861/* Map an (observed) memop size value to the representative value of its range.
862 * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */
863INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t
864InstrProfGetRangeRepValue(uint64_t Value) {
865 if (Value <= 8)
866 // The first ranges are individually tracked. Use the value as is.
867 return Value;
868 else if (Value >= 513)
869 // The last range is mapped to its lowest value.
870 return 513;
871 else if (InstProfPopcountll(Value) == 1)
872 // If it's a power of two, use it as is.
873 return Value;
874 else
875 // Otherwise, take to the previous power of two + 1.
876 return (UINT64_C(1) << (64 - InstProfClzll(Value) - 1)) + 1;
877}
878
879/* Return true if the range that an (observed) memop size value belongs to has
880 * only a single value in the range. For example, 0 -> true, 8 -> true, 10 ->
881 * false, 64 -> true, 100 -> false, 513 -> false. */
882INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned
883InstrProfIsSingleValRange(uint64_t Value) {
884 if (Value <= 8)
885 // The first ranges are individually tracked.
886 return 1;
887 else if (InstProfPopcountll(Value) == 1)
888 // If it's a power of two, there's only one value.
889 return 1;
890 else
891 // Otherwise, there's more than one value in the range.
892 return 0;
893}
894
895#endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */
896