1 | //===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This pass lowers instrprof_* intrinsics emitted by an instrumentor. |
10 | // It also builds the data structures and initialization code needed for |
11 | // updating execution counts and emitting the profile at runtime. |
12 | // |
13 | //===----------------------------------------------------------------------===// |
14 | |
15 | #include "llvm/Transforms/Instrumentation/InstrProfiling.h" |
16 | #include "llvm/ADT/ArrayRef.h" |
17 | #include "llvm/ADT/STLExtras.h" |
18 | #include "llvm/ADT/SmallVector.h" |
19 | #include "llvm/ADT/StringRef.h" |
20 | #include "llvm/ADT/Twine.h" |
21 | #include "llvm/Analysis/BlockFrequencyInfo.h" |
22 | #include "llvm/Analysis/BranchProbabilityInfo.h" |
23 | #include "llvm/Analysis/LoopInfo.h" |
24 | #include "llvm/Analysis/TargetLibraryInfo.h" |
25 | #include "llvm/IR/Attributes.h" |
26 | #include "llvm/IR/BasicBlock.h" |
27 | #include "llvm/IR/CFG.h" |
28 | #include "llvm/IR/Constant.h" |
29 | #include "llvm/IR/Constants.h" |
30 | #include "llvm/IR/DIBuilder.h" |
31 | #include "llvm/IR/DerivedTypes.h" |
32 | #include "llvm/IR/DiagnosticInfo.h" |
33 | #include "llvm/IR/Dominators.h" |
34 | #include "llvm/IR/Function.h" |
35 | #include "llvm/IR/GlobalValue.h" |
36 | #include "llvm/IR/GlobalVariable.h" |
37 | #include "llvm/IR/IRBuilder.h" |
38 | #include "llvm/IR/Instruction.h" |
39 | #include "llvm/IR/Instructions.h" |
40 | #include "llvm/IR/IntrinsicInst.h" |
41 | #include "llvm/IR/Module.h" |
42 | #include "llvm/IR/Type.h" |
43 | #include "llvm/InitializePasses.h" |
44 | #include "llvm/Pass.h" |
45 | #include "llvm/ProfileData/InstrProf.h" |
46 | #include "llvm/ProfileData/InstrProfCorrelator.h" |
47 | #include "llvm/Support/Casting.h" |
48 | #include "llvm/Support/CommandLine.h" |
49 | #include "llvm/Support/Error.h" |
50 | #include "llvm/Support/ErrorHandling.h" |
51 | #include "llvm/TargetParser/Triple.h" |
52 | #include "llvm/Transforms/Instrumentation.h" |
53 | #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h" |
54 | #include "llvm/Transforms/Utils/BasicBlockUtils.h" |
55 | #include "llvm/Transforms/Utils/ModuleUtils.h" |
56 | #include "llvm/Transforms/Utils/SSAUpdater.h" |
57 | #include <algorithm> |
58 | #include <cassert> |
59 | #include <cstdint> |
60 | #include <string> |
61 | |
62 | using namespace llvm; |
63 | |
64 | #define DEBUG_TYPE "instrprof" |
65 | |
66 | namespace llvm { |
67 | // Command line option to enable vtable value profiling. Defined in |
68 | // ProfileData/InstrProf.cpp: -enable-vtable-value-profiling= |
69 | extern cl::opt<bool> EnableVTableValueProfiling; |
70 | // TODO: Remove -debug-info-correlate in next LLVM release, in favor of |
71 | // -profile-correlate=debug-info. |
72 | cl::opt<bool> DebugInfoCorrelate( |
73 | "debug-info-correlate" , |
74 | cl::desc("Use debug info to correlate profiles. (Deprecated, use " |
75 | "-profile-correlate=debug-info)" ), |
76 | cl::init(Val: false)); |
77 | |
78 | cl::opt<InstrProfCorrelator::ProfCorrelatorKind> ProfileCorrelate( |
79 | "profile-correlate" , |
80 | cl::desc("Use debug info or binary file to correlate profiles." ), |
81 | cl::init(Val: InstrProfCorrelator::NONE), |
82 | cl::values(clEnumValN(InstrProfCorrelator::NONE, "" , |
83 | "No profile correlation" ), |
84 | clEnumValN(InstrProfCorrelator::DEBUG_INFO, "debug-info" , |
85 | "Use debug info to correlate" ), |
86 | clEnumValN(InstrProfCorrelator::BINARY, "binary" , |
87 | "Use binary to correlate" ))); |
88 | } // namespace llvm |
89 | |
90 | namespace { |
91 | |
92 | cl::opt<bool> DoHashBasedCounterSplit( |
93 | "hash-based-counter-split" , |
94 | cl::desc("Rename counter variable of a comdat function based on cfg hash" ), |
95 | cl::init(Val: true)); |
96 | |
97 | cl::opt<bool> |
98 | RuntimeCounterRelocation("runtime-counter-relocation" , |
99 | cl::desc("Enable relocating counters at runtime." ), |
100 | cl::init(Val: false)); |
101 | |
102 | cl::opt<bool> ValueProfileStaticAlloc( |
103 | "vp-static-alloc" , |
104 | cl::desc("Do static counter allocation for value profiler" ), |
105 | cl::init(Val: true)); |
106 | |
107 | cl::opt<double> NumCountersPerValueSite( |
108 | "vp-counters-per-site" , |
109 | cl::desc("The average number of profile counters allocated " |
110 | "per value profiling site." ), |
111 | // This is set to a very small value because in real programs, only |
112 | // a very small percentage of value sites have non-zero targets, e.g, 1/30. |
113 | // For those sites with non-zero profile, the average number of targets |
114 | // is usually smaller than 2. |
115 | cl::init(Val: 1.0)); |
116 | |
117 | cl::opt<bool> AtomicCounterUpdateAll( |
118 | "instrprof-atomic-counter-update-all" , |
119 | cl::desc("Make all profile counter updates atomic (for testing only)" ), |
120 | cl::init(Val: false)); |
121 | |
122 | cl::opt<bool> AtomicCounterUpdatePromoted( |
123 | "atomic-counter-update-promoted" , |
124 | cl::desc("Do counter update using atomic fetch add " |
125 | " for promoted counters only" ), |
126 | cl::init(Val: false)); |
127 | |
128 | cl::opt<bool> AtomicFirstCounter( |
129 | "atomic-first-counter" , |
130 | cl::desc("Use atomic fetch add for first counter in a function (usually " |
131 | "the entry counter)" ), |
132 | cl::init(Val: false)); |
133 | |
134 | // If the option is not specified, the default behavior about whether |
135 | // counter promotion is done depends on how instrumentaiton lowering |
136 | // pipeline is setup, i.e., the default value of true of this option |
137 | // does not mean the promotion will be done by default. Explicitly |
138 | // setting this option can override the default behavior. |
139 | cl::opt<bool> DoCounterPromotion("do-counter-promotion" , |
140 | cl::desc("Do counter register promotion" ), |
141 | cl::init(Val: false)); |
142 | cl::opt<unsigned> MaxNumOfPromotionsPerLoop( |
143 | "max-counter-promotions-per-loop" , cl::init(Val: 20), |
144 | cl::desc("Max number counter promotions per loop to avoid" |
145 | " increasing register pressure too much" )); |
146 | |
147 | // A debug option |
148 | cl::opt<int> |
149 | MaxNumOfPromotions("max-counter-promotions" , cl::init(Val: -1), |
150 | cl::desc("Max number of allowed counter promotions" )); |
151 | |
152 | cl::opt<unsigned> SpeculativeCounterPromotionMaxExiting( |
153 | "speculative-counter-promotion-max-exiting" , cl::init(Val: 3), |
154 | cl::desc("The max number of exiting blocks of a loop to allow " |
155 | " speculative counter promotion" )); |
156 | |
157 | cl::opt<bool> SpeculativeCounterPromotionToLoop( |
158 | "speculative-counter-promotion-to-loop" , |
159 | cl::desc("When the option is false, if the target block is in a loop, " |
160 | "the promotion will be disallowed unless the promoted counter " |
161 | " update can be further/iteratively promoted into an acyclic " |
162 | " region." )); |
163 | |
164 | cl::opt<bool> IterativeCounterPromotion( |
165 | "iterative-counter-promotion" , cl::init(Val: true), |
166 | cl::desc("Allow counter promotion across the whole loop nest." )); |
167 | |
168 | cl::opt<bool> SkipRetExitBlock( |
169 | "skip-ret-exit-block" , cl::init(Val: true), |
170 | cl::desc("Suppress counter promotion if exit blocks contain ret." )); |
171 | |
172 | using LoadStorePair = std::pair<Instruction *, Instruction *>; |
173 | |
174 | static uint64_t getIntModuleFlagOrZero(const Module &M, StringRef Flag) { |
175 | auto *MD = dyn_cast_or_null<ConstantAsMetadata>(Val: M.getModuleFlag(Key: Flag)); |
176 | if (!MD) |
177 | return 0; |
178 | |
179 | // If the flag is a ConstantAsMetadata, it should be an integer representable |
180 | // in 64-bits. |
181 | return cast<ConstantInt>(Val: MD->getValue())->getZExtValue(); |
182 | } |
183 | |
184 | static bool enablesValueProfiling(const Module &M) { |
185 | return isIRPGOFlagSet(M: &M) || |
186 | getIntModuleFlagOrZero(M, Flag: "EnableValueProfiling" ) != 0; |
187 | } |
188 | |
189 | // Conservatively returns true if value profiling is enabled. |
190 | static bool profDataReferencedByCode(const Module &M) { |
191 | return enablesValueProfiling(M); |
192 | } |
193 | |
194 | class InstrLowerer final { |
195 | public: |
196 | InstrLowerer(Module &M, const InstrProfOptions &Options, |
197 | std::function<const TargetLibraryInfo &(Function &F)> GetTLI, |
198 | bool IsCS) |
199 | : M(M), Options(Options), TT(Triple(M.getTargetTriple())), IsCS(IsCS), |
200 | GetTLI(GetTLI), DataReferencedByCode(profDataReferencedByCode(M)) {} |
201 | |
202 | bool lower(); |
203 | |
204 | private: |
205 | Module &M; |
206 | const InstrProfOptions Options; |
207 | const Triple TT; |
208 | // Is this lowering for the context-sensitive instrumentation. |
209 | const bool IsCS; |
210 | |
211 | std::function<const TargetLibraryInfo &(Function &F)> GetTLI; |
212 | |
213 | const bool DataReferencedByCode; |
214 | |
215 | struct PerFunctionProfileData { |
216 | uint32_t NumValueSites[IPVK_Last + 1] = {}; |
217 | GlobalVariable *RegionCounters = nullptr; |
218 | GlobalVariable *DataVar = nullptr; |
219 | GlobalVariable *RegionBitmaps = nullptr; |
220 | uint32_t NumBitmapBytes = 0; |
221 | |
222 | PerFunctionProfileData() = default; |
223 | }; |
224 | DenseMap<GlobalVariable *, PerFunctionProfileData> ProfileDataMap; |
225 | // Key is virtual table variable, value is 'VTableProfData' in the form of |
226 | // GlobalVariable. |
227 | DenseMap<GlobalVariable *, GlobalVariable *> VTableDataMap; |
228 | /// If runtime relocation is enabled, this maps functions to the load |
229 | /// instruction that produces the profile relocation bias. |
230 | DenseMap<const Function *, LoadInst *> FunctionToProfileBiasMap; |
231 | std::vector<GlobalValue *> CompilerUsedVars; |
232 | std::vector<GlobalValue *> UsedVars; |
233 | std::vector<GlobalVariable *> ReferencedNames; |
234 | // The list of virtual table variables of which the VTableProfData is |
235 | // collected. |
236 | std::vector<GlobalVariable *> ReferencedVTables; |
237 | GlobalVariable *NamesVar = nullptr; |
238 | size_t NamesSize = 0; |
239 | |
240 | // vector of counter load/store pairs to be register promoted. |
241 | std::vector<LoadStorePair> PromotionCandidates; |
242 | |
243 | int64_t TotalCountersPromoted = 0; |
244 | |
245 | /// Lower instrumentation intrinsics in the function. Returns true if there |
246 | /// any lowering. |
247 | bool lowerIntrinsics(Function *F); |
248 | |
249 | /// Register-promote counter loads and stores in loops. |
250 | void promoteCounterLoadStores(Function *F); |
251 | |
252 | /// Returns true if relocating counters at runtime is enabled. |
253 | bool isRuntimeCounterRelocationEnabled() const; |
254 | |
255 | /// Returns true if profile counter update register promotion is enabled. |
256 | bool isCounterPromotionEnabled() const; |
257 | |
258 | /// Count the number of instrumented value sites for the function. |
259 | void computeNumValueSiteCounts(InstrProfValueProfileInst *Ins); |
260 | |
261 | /// Replace instrprof.value.profile with a call to runtime library. |
262 | void lowerValueProfileInst(InstrProfValueProfileInst *Ins); |
263 | |
264 | /// Replace instrprof.cover with a store instruction to the coverage byte. |
265 | void lowerCover(InstrProfCoverInst *Inc); |
266 | |
267 | /// Replace instrprof.timestamp with a call to |
268 | /// INSTR_PROF_PROFILE_SET_TIMESTAMP. |
269 | void lowerTimestamp(InstrProfTimestampInst *TimestampInstruction); |
270 | |
271 | /// Replace instrprof.increment with an increment of the appropriate value. |
272 | void lowerIncrement(InstrProfIncrementInst *Inc); |
273 | |
274 | /// Force emitting of name vars for unused functions. |
275 | void lowerCoverageData(GlobalVariable *CoverageNamesVar); |
276 | |
277 | /// Replace instrprof.mcdc.tvbitmask.update with a shift and or instruction |
278 | /// using the index represented by the a temp value into a bitmap. |
279 | void lowerMCDCTestVectorBitmapUpdate(InstrProfMCDCTVBitmapUpdate *Ins); |
280 | |
281 | /// Replace instrprof.mcdc.temp.update with a shift and or instruction using |
282 | /// the corresponding condition ID. |
283 | void lowerMCDCCondBitmapUpdate(InstrProfMCDCCondBitmapUpdate *Ins); |
284 | |
285 | /// Compute the address of the counter value that this profiling instruction |
286 | /// acts on. |
287 | Value *getCounterAddress(InstrProfCntrInstBase *I); |
288 | |
289 | /// Get the region counters for an increment, creating them if necessary. |
290 | /// |
291 | /// If the counter array doesn't yet exist, the profile data variables |
292 | /// referring to them will also be created. |
293 | GlobalVariable *getOrCreateRegionCounters(InstrProfCntrInstBase *Inc); |
294 | |
295 | /// Create the region counters. |
296 | GlobalVariable *createRegionCounters(InstrProfCntrInstBase *Inc, |
297 | StringRef Name, |
298 | GlobalValue::LinkageTypes Linkage); |
299 | |
300 | /// Compute the address of the test vector bitmap that this profiling |
301 | /// instruction acts on. |
302 | Value *getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I); |
303 | |
304 | /// Get the region bitmaps for an increment, creating them if necessary. |
305 | /// |
306 | /// If the bitmap array doesn't yet exist, the profile data variables |
307 | /// referring to them will also be created. |
308 | GlobalVariable *getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc); |
309 | |
310 | /// Create the MC/DC bitmap as a byte-aligned array of bytes associated with |
311 | /// an MC/DC Decision region. The number of bytes required is indicated by |
312 | /// the intrinsic used (type InstrProfMCDCBitmapInstBase). This is called |
313 | /// as part of setupProfileSection() and is conceptually very similar to |
314 | /// what is done for profile data counters in createRegionCounters(). |
315 | GlobalVariable *createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc, |
316 | StringRef Name, |
317 | GlobalValue::LinkageTypes Linkage); |
318 | |
319 | /// Set Comdat property of GV, if required. |
320 | void maybeSetComdat(GlobalVariable *GV, GlobalObject *GO, StringRef VarName); |
321 | |
322 | /// Setup the sections into which counters and bitmaps are allocated. |
323 | GlobalVariable *setupProfileSection(InstrProfInstBase *Inc, |
324 | InstrProfSectKind IPSK); |
325 | |
326 | /// Create INSTR_PROF_DATA variable for counters and bitmaps. |
327 | void createDataVariable(InstrProfCntrInstBase *Inc); |
328 | |
329 | /// Get the counters for virtual table values, creating them if necessary. |
330 | void getOrCreateVTableProfData(GlobalVariable *GV); |
331 | |
332 | /// Emit the section with compressed function names. |
333 | void emitNameData(); |
334 | |
335 | /// Emit the section with compressed vtable names. |
336 | void emitVTableNames(); |
337 | |
338 | /// Emit value nodes section for value profiling. |
339 | void emitVNodes(); |
340 | |
341 | /// Emit runtime registration functions for each profile data variable. |
342 | void emitRegistration(); |
343 | |
344 | /// Emit the necessary plumbing to pull in the runtime initialization. |
345 | /// Returns true if a change was made. |
346 | bool emitRuntimeHook(); |
347 | |
348 | /// Add uses of our data variables and runtime hook. |
349 | void emitUses(); |
350 | |
351 | /// Create a static initializer for our data, on platforms that need it, |
352 | /// and for any profile output file that was specified. |
353 | void emitInitialization(); |
354 | }; |
355 | |
356 | /// |
357 | /// A helper class to promote one counter RMW operation in the loop |
358 | /// into register update. |
359 | /// |
360 | /// RWM update for the counter will be sinked out of the loop after |
361 | /// the transformation. |
362 | /// |
363 | class PGOCounterPromoterHelper : public LoadAndStorePromoter { |
364 | public: |
365 | PGOCounterPromoterHelper( |
366 | Instruction *L, Instruction *S, SSAUpdater &SSA, Value *Init, |
367 | BasicBlock *PH, ArrayRef<BasicBlock *> ExitBlocks, |
368 | ArrayRef<Instruction *> InsertPts, |
369 | DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands, |
370 | LoopInfo &LI) |
371 | : LoadAndStorePromoter({L, S}, SSA), Store(S), ExitBlocks(ExitBlocks), |
372 | InsertPts(InsertPts), LoopToCandidates(LoopToCands), LI(LI) { |
373 | assert(isa<LoadInst>(L)); |
374 | assert(isa<StoreInst>(S)); |
375 | SSA.AddAvailableValue(BB: PH, V: Init); |
376 | } |
377 | |
378 | void () override { |
379 | for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) { |
380 | BasicBlock *ExitBlock = ExitBlocks[i]; |
381 | Instruction *InsertPos = InsertPts[i]; |
382 | // Get LiveIn value into the ExitBlock. If there are multiple |
383 | // predecessors, the value is defined by a PHI node in this |
384 | // block. |
385 | Value *LiveInValue = SSA.GetValueInMiddleOfBlock(BB: ExitBlock); |
386 | Value *Addr = cast<StoreInst>(Val: Store)->getPointerOperand(); |
387 | Type *Ty = LiveInValue->getType(); |
388 | IRBuilder<> Builder(InsertPos); |
389 | if (auto *AddrInst = dyn_cast_or_null<IntToPtrInst>(Val: Addr)) { |
390 | // If isRuntimeCounterRelocationEnabled() is true then the address of |
391 | // the store instruction is computed with two instructions in |
392 | // InstrProfiling::getCounterAddress(). We need to copy those |
393 | // instructions to this block to compute Addr correctly. |
394 | // %BiasAdd = add i64 ptrtoint <__profc_>, <__llvm_profile_counter_bias> |
395 | // %Addr = inttoptr i64 %BiasAdd to i64* |
396 | auto *OrigBiasInst = dyn_cast<BinaryOperator>(Val: AddrInst->getOperand(i_nocapture: 0)); |
397 | assert(OrigBiasInst->getOpcode() == Instruction::BinaryOps::Add); |
398 | Value *BiasInst = Builder.Insert(I: OrigBiasInst->clone()); |
399 | Addr = Builder.CreateIntToPtr(V: BiasInst, |
400 | DestTy: PointerType::getUnqual(C&: Ty->getContext())); |
401 | } |
402 | if (AtomicCounterUpdatePromoted) |
403 | // automic update currently can only be promoted across the current |
404 | // loop, not the whole loop nest. |
405 | Builder.CreateAtomicRMW(Op: AtomicRMWInst::Add, Ptr: Addr, Val: LiveInValue, |
406 | Align: MaybeAlign(), |
407 | Ordering: AtomicOrdering::SequentiallyConsistent); |
408 | else { |
409 | LoadInst *OldVal = Builder.CreateLoad(Ty, Ptr: Addr, Name: "pgocount.promoted" ); |
410 | auto *NewVal = Builder.CreateAdd(LHS: OldVal, RHS: LiveInValue); |
411 | auto *NewStore = Builder.CreateStore(Val: NewVal, Ptr: Addr); |
412 | |
413 | // Now update the parent loop's candidate list: |
414 | if (IterativeCounterPromotion) { |
415 | auto *TargetLoop = LI.getLoopFor(BB: ExitBlock); |
416 | if (TargetLoop) |
417 | LoopToCandidates[TargetLoop].emplace_back(Args&: OldVal, Args&: NewStore); |
418 | } |
419 | } |
420 | } |
421 | } |
422 | |
423 | private: |
424 | Instruction *Store; |
425 | ArrayRef<BasicBlock *> ExitBlocks; |
426 | ArrayRef<Instruction *> InsertPts; |
427 | DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates; |
428 | LoopInfo &LI; |
429 | }; |
430 | |
431 | /// A helper class to do register promotion for all profile counter |
432 | /// updates in a loop. |
433 | /// |
434 | class PGOCounterPromoter { |
435 | public: |
436 | PGOCounterPromoter( |
437 | DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands, |
438 | Loop &CurLoop, LoopInfo &LI, BlockFrequencyInfo *BFI) |
439 | : LoopToCandidates(LoopToCands), L(CurLoop), LI(LI), BFI(BFI) { |
440 | |
441 | // Skip collection of ExitBlocks and InsertPts for loops that will not be |
442 | // able to have counters promoted. |
443 | SmallVector<BasicBlock *, 8> LoopExitBlocks; |
444 | SmallPtrSet<BasicBlock *, 8> BlockSet; |
445 | |
446 | L.getExitBlocks(ExitBlocks&: LoopExitBlocks); |
447 | if (!isPromotionPossible(LP: &L, LoopExitBlocks)) |
448 | return; |
449 | |
450 | for (BasicBlock *ExitBlock : LoopExitBlocks) { |
451 | if (BlockSet.insert(Ptr: ExitBlock).second && |
452 | llvm::none_of(Range: predecessors(BB: ExitBlock), P: [&](const BasicBlock *Pred) { |
453 | return llvm::isPresplitCoroSuspendExitEdge(Src: *Pred, Dest: *ExitBlock); |
454 | })) { |
455 | ExitBlocks.push_back(Elt: ExitBlock); |
456 | InsertPts.push_back(Elt: &*ExitBlock->getFirstInsertionPt()); |
457 | } |
458 | } |
459 | } |
460 | |
461 | bool run(int64_t *NumPromoted) { |
462 | // Skip 'infinite' loops: |
463 | if (ExitBlocks.size() == 0) |
464 | return false; |
465 | |
466 | // Skip if any of the ExitBlocks contains a ret instruction. |
467 | // This is to prevent dumping of incomplete profile -- if the |
468 | // the loop is a long running loop and dump is called in the middle |
469 | // of the loop, the result profile is incomplete. |
470 | // FIXME: add other heuristics to detect long running loops. |
471 | if (SkipRetExitBlock) { |
472 | for (auto *BB : ExitBlocks) |
473 | if (isa<ReturnInst>(Val: BB->getTerminator())) |
474 | return false; |
475 | } |
476 | |
477 | unsigned MaxProm = getMaxNumOfPromotionsInLoop(LP: &L); |
478 | if (MaxProm == 0) |
479 | return false; |
480 | |
481 | unsigned Promoted = 0; |
482 | for (auto &Cand : LoopToCandidates[&L]) { |
483 | |
484 | SmallVector<PHINode *, 4> NewPHIs; |
485 | SSAUpdater SSA(&NewPHIs); |
486 | Value *InitVal = ConstantInt::get(Ty: Cand.first->getType(), V: 0); |
487 | |
488 | // If BFI is set, we will use it to guide the promotions. |
489 | if (BFI) { |
490 | auto *BB = Cand.first->getParent(); |
491 | auto InstrCount = BFI->getBlockProfileCount(BB); |
492 | if (!InstrCount) |
493 | continue; |
494 | auto = BFI->getBlockProfileCount(BB: L.getLoopPreheader()); |
495 | // If the average loop trip count is not greater than 1.5, we skip |
496 | // promotion. |
497 | if (PreheaderCount && (*PreheaderCount * 3) >= (*InstrCount * 2)) |
498 | continue; |
499 | } |
500 | |
501 | PGOCounterPromoterHelper Promoter(Cand.first, Cand.second, SSA, InitVal, |
502 | L.getLoopPreheader(), ExitBlocks, |
503 | InsertPts, LoopToCandidates, LI); |
504 | Promoter.run(Insts: SmallVector<Instruction *, 2>({Cand.first, Cand.second})); |
505 | Promoted++; |
506 | if (Promoted >= MaxProm) |
507 | break; |
508 | |
509 | (*NumPromoted)++; |
510 | if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions) |
511 | break; |
512 | } |
513 | |
514 | LLVM_DEBUG(dbgs() << Promoted << " counters promoted for loop (depth=" |
515 | << L.getLoopDepth() << ")\n" ); |
516 | return Promoted != 0; |
517 | } |
518 | |
519 | private: |
520 | bool allowSpeculativeCounterPromotion(Loop *LP) { |
521 | SmallVector<BasicBlock *, 8> ExitingBlocks; |
522 | L.getExitingBlocks(ExitingBlocks); |
523 | // Not considierered speculative. |
524 | if (ExitingBlocks.size() == 1) |
525 | return true; |
526 | if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting) |
527 | return false; |
528 | return true; |
529 | } |
530 | |
531 | // Check whether the loop satisfies the basic conditions needed to perform |
532 | // Counter Promotions. |
533 | bool |
534 | isPromotionPossible(Loop *LP, |
535 | const SmallVectorImpl<BasicBlock *> &LoopExitBlocks) { |
536 | // We can't insert into a catchswitch. |
537 | if (llvm::any_of(Range: LoopExitBlocks, P: [](BasicBlock *Exit) { |
538 | return isa<CatchSwitchInst>(Val: Exit->getTerminator()); |
539 | })) |
540 | return false; |
541 | |
542 | if (!LP->hasDedicatedExits()) |
543 | return false; |
544 | |
545 | BasicBlock *PH = LP->getLoopPreheader(); |
546 | if (!PH) |
547 | return false; |
548 | |
549 | return true; |
550 | } |
551 | |
552 | // Returns the max number of Counter Promotions for LP. |
553 | unsigned getMaxNumOfPromotionsInLoop(Loop *LP) { |
554 | SmallVector<BasicBlock *, 8> LoopExitBlocks; |
555 | LP->getExitBlocks(ExitBlocks&: LoopExitBlocks); |
556 | if (!isPromotionPossible(LP, LoopExitBlocks)) |
557 | return 0; |
558 | |
559 | SmallVector<BasicBlock *, 8> ExitingBlocks; |
560 | LP->getExitingBlocks(ExitingBlocks); |
561 | |
562 | // If BFI is set, we do more aggressive promotions based on BFI. |
563 | if (BFI) |
564 | return (unsigned)-1; |
565 | |
566 | // Not considierered speculative. |
567 | if (ExitingBlocks.size() == 1) |
568 | return MaxNumOfPromotionsPerLoop; |
569 | |
570 | if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting) |
571 | return 0; |
572 | |
573 | // Whether the target block is in a loop does not matter: |
574 | if (SpeculativeCounterPromotionToLoop) |
575 | return MaxNumOfPromotionsPerLoop; |
576 | |
577 | // Now check the target block: |
578 | unsigned MaxProm = MaxNumOfPromotionsPerLoop; |
579 | for (auto *TargetBlock : LoopExitBlocks) { |
580 | auto *TargetLoop = LI.getLoopFor(BB: TargetBlock); |
581 | if (!TargetLoop) |
582 | continue; |
583 | unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(LP: TargetLoop); |
584 | unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size(); |
585 | MaxProm = |
586 | std::min(a: MaxProm, b: std::max(a: MaxPromForTarget, b: PendingCandsInTarget) - |
587 | PendingCandsInTarget); |
588 | } |
589 | return MaxProm; |
590 | } |
591 | |
592 | DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates; |
593 | SmallVector<BasicBlock *, 8> ExitBlocks; |
594 | SmallVector<Instruction *, 8> InsertPts; |
595 | Loop &L; |
596 | LoopInfo &LI; |
597 | BlockFrequencyInfo *BFI; |
598 | }; |
599 | |
600 | enum class ValueProfilingCallType { |
601 | // Individual values are tracked. Currently used for indiret call target |
602 | // profiling. |
603 | Default, |
604 | |
605 | // MemOp: the memop size value profiling. |
606 | MemOp |
607 | }; |
608 | |
609 | } // end anonymous namespace |
610 | |
611 | PreservedAnalyses InstrProfilingLoweringPass::run(Module &M, |
612 | ModuleAnalysisManager &AM) { |
613 | FunctionAnalysisManager &FAM = |
614 | AM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager(); |
615 | auto GetTLI = [&FAM](Function &F) -> TargetLibraryInfo & { |
616 | return FAM.getResult<TargetLibraryAnalysis>(IR&: F); |
617 | }; |
618 | InstrLowerer Lowerer(M, Options, GetTLI, IsCS); |
619 | if (!Lowerer.lower()) |
620 | return PreservedAnalyses::all(); |
621 | |
622 | return PreservedAnalyses::none(); |
623 | } |
624 | |
625 | bool InstrLowerer::lowerIntrinsics(Function *F) { |
626 | bool MadeChange = false; |
627 | PromotionCandidates.clear(); |
628 | for (BasicBlock &BB : *F) { |
629 | for (Instruction &Instr : llvm::make_early_inc_range(Range&: BB)) { |
630 | if (auto *IPIS = dyn_cast<InstrProfIncrementInstStep>(Val: &Instr)) { |
631 | lowerIncrement(Inc: IPIS); |
632 | MadeChange = true; |
633 | } else if (auto *IPI = dyn_cast<InstrProfIncrementInst>(Val: &Instr)) { |
634 | lowerIncrement(Inc: IPI); |
635 | MadeChange = true; |
636 | } else if (auto *IPC = dyn_cast<InstrProfTimestampInst>(Val: &Instr)) { |
637 | lowerTimestamp(TimestampInstruction: IPC); |
638 | MadeChange = true; |
639 | } else if (auto *IPC = dyn_cast<InstrProfCoverInst>(Val: &Instr)) { |
640 | lowerCover(Inc: IPC); |
641 | MadeChange = true; |
642 | } else if (auto *IPVP = dyn_cast<InstrProfValueProfileInst>(Val: &Instr)) { |
643 | lowerValueProfileInst(Ins: IPVP); |
644 | MadeChange = true; |
645 | } else if (auto *IPMP = dyn_cast<InstrProfMCDCBitmapParameters>(Val: &Instr)) { |
646 | IPMP->eraseFromParent(); |
647 | MadeChange = true; |
648 | } else if (auto *IPBU = dyn_cast<InstrProfMCDCTVBitmapUpdate>(Val: &Instr)) { |
649 | lowerMCDCTestVectorBitmapUpdate(Ins: IPBU); |
650 | MadeChange = true; |
651 | } else if (auto *IPTU = dyn_cast<InstrProfMCDCCondBitmapUpdate>(Val: &Instr)) { |
652 | lowerMCDCCondBitmapUpdate(Ins: IPTU); |
653 | MadeChange = true; |
654 | } |
655 | } |
656 | } |
657 | |
658 | if (!MadeChange) |
659 | return false; |
660 | |
661 | promoteCounterLoadStores(F); |
662 | return true; |
663 | } |
664 | |
665 | bool InstrLowerer::isRuntimeCounterRelocationEnabled() const { |
666 | // Mach-O don't support weak external references. |
667 | if (TT.isOSBinFormatMachO()) |
668 | return false; |
669 | |
670 | if (RuntimeCounterRelocation.getNumOccurrences() > 0) |
671 | return RuntimeCounterRelocation; |
672 | |
673 | // Fuchsia uses runtime counter relocation by default. |
674 | return TT.isOSFuchsia(); |
675 | } |
676 | |
677 | bool InstrLowerer::isCounterPromotionEnabled() const { |
678 | if (DoCounterPromotion.getNumOccurrences() > 0) |
679 | return DoCounterPromotion; |
680 | |
681 | return Options.DoCounterPromotion; |
682 | } |
683 | |
684 | void InstrLowerer::promoteCounterLoadStores(Function *F) { |
685 | if (!isCounterPromotionEnabled()) |
686 | return; |
687 | |
688 | DominatorTree DT(*F); |
689 | LoopInfo LI(DT); |
690 | DenseMap<Loop *, SmallVector<LoadStorePair, 8>> LoopPromotionCandidates; |
691 | |
692 | std::unique_ptr<BlockFrequencyInfo> BFI; |
693 | if (Options.UseBFIInPromotion) { |
694 | std::unique_ptr<BranchProbabilityInfo> BPI; |
695 | BPI.reset(p: new BranchProbabilityInfo(*F, LI, &GetTLI(*F))); |
696 | BFI.reset(p: new BlockFrequencyInfo(*F, *BPI, LI)); |
697 | } |
698 | |
699 | for (const auto &LoadStore : PromotionCandidates) { |
700 | auto *CounterLoad = LoadStore.first; |
701 | auto *CounterStore = LoadStore.second; |
702 | BasicBlock *BB = CounterLoad->getParent(); |
703 | Loop *ParentLoop = LI.getLoopFor(BB); |
704 | if (!ParentLoop) |
705 | continue; |
706 | LoopPromotionCandidates[ParentLoop].emplace_back(Args&: CounterLoad, Args&: CounterStore); |
707 | } |
708 | |
709 | SmallVector<Loop *, 4> Loops = LI.getLoopsInPreorder(); |
710 | |
711 | // Do a post-order traversal of the loops so that counter updates can be |
712 | // iteratively hoisted outside the loop nest. |
713 | for (auto *Loop : llvm::reverse(C&: Loops)) { |
714 | PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI, BFI.get()); |
715 | Promoter.run(NumPromoted: &TotalCountersPromoted); |
716 | } |
717 | } |
718 | |
719 | static bool needsRuntimeHookUnconditionally(const Triple &TT) { |
720 | // On Fuchsia, we only need runtime hook if any counters are present. |
721 | if (TT.isOSFuchsia()) |
722 | return false; |
723 | |
724 | return true; |
725 | } |
726 | |
727 | /// Check if the module contains uses of any profiling intrinsics. |
728 | static bool containsProfilingIntrinsics(Module &M) { |
729 | auto containsIntrinsic = [&](int ID) { |
730 | if (auto *F = M.getFunction(Name: Intrinsic::getName(id: ID))) |
731 | return !F->use_empty(); |
732 | return false; |
733 | }; |
734 | return containsIntrinsic(llvm::Intrinsic::instrprof_cover) || |
735 | containsIntrinsic(llvm::Intrinsic::instrprof_increment) || |
736 | containsIntrinsic(llvm::Intrinsic::instrprof_increment_step) || |
737 | containsIntrinsic(llvm::Intrinsic::instrprof_timestamp) || |
738 | containsIntrinsic(llvm::Intrinsic::instrprof_value_profile); |
739 | } |
740 | |
741 | bool InstrLowerer::lower() { |
742 | bool MadeChange = false; |
743 | bool NeedsRuntimeHook = needsRuntimeHookUnconditionally(TT); |
744 | if (NeedsRuntimeHook) |
745 | MadeChange = emitRuntimeHook(); |
746 | |
747 | bool ContainsProfiling = containsProfilingIntrinsics(M); |
748 | GlobalVariable *CoverageNamesVar = |
749 | M.getNamedGlobal(Name: getCoverageUnusedNamesVarName()); |
750 | // Improve compile time by avoiding linear scans when there is no work. |
751 | if (!ContainsProfiling && !CoverageNamesVar) |
752 | return MadeChange; |
753 | |
754 | // We did not know how many value sites there would be inside |
755 | // the instrumented function. This is counting the number of instrumented |
756 | // target value sites to enter it as field in the profile data variable. |
757 | for (Function &F : M) { |
758 | InstrProfCntrInstBase *FirstProfInst = nullptr; |
759 | for (BasicBlock &BB : F) { |
760 | for (auto I = BB.begin(), E = BB.end(); I != E; I++) { |
761 | if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(Val&: I)) |
762 | computeNumValueSiteCounts(Ins: Ind); |
763 | else { |
764 | if (FirstProfInst == nullptr && |
765 | (isa<InstrProfIncrementInst>(Val: I) || isa<InstrProfCoverInst>(Val: I))) |
766 | FirstProfInst = dyn_cast<InstrProfCntrInstBase>(Val&: I); |
767 | // If the MCDCBitmapParameters intrinsic seen, create the bitmaps. |
768 | if (const auto &Params = dyn_cast<InstrProfMCDCBitmapParameters>(Val&: I)) |
769 | static_cast<void>(getOrCreateRegionBitmaps(Inc: Params)); |
770 | } |
771 | } |
772 | } |
773 | |
774 | // Use a profile intrinsic to create the region counters and data variable. |
775 | // Also create the data variable based on the MCDCParams. |
776 | if (FirstProfInst != nullptr) { |
777 | static_cast<void>(getOrCreateRegionCounters(Inc: FirstProfInst)); |
778 | } |
779 | } |
780 | |
781 | if (EnableVTableValueProfiling) |
782 | for (GlobalVariable &GV : M.globals()) |
783 | // Global variables with type metadata are virtual table variables. |
784 | if (GV.hasMetadata(KindID: LLVMContext::MD_type)) |
785 | getOrCreateVTableProfData(GV: &GV); |
786 | |
787 | for (Function &F : M) |
788 | MadeChange |= lowerIntrinsics(F: &F); |
789 | |
790 | if (CoverageNamesVar) { |
791 | lowerCoverageData(CoverageNamesVar); |
792 | MadeChange = true; |
793 | } |
794 | |
795 | if (!MadeChange) |
796 | return false; |
797 | |
798 | emitVNodes(); |
799 | emitNameData(); |
800 | emitVTableNames(); |
801 | |
802 | // Emit runtime hook for the cases where the target does not unconditionally |
803 | // require pulling in profile runtime, and coverage is enabled on code that is |
804 | // not eliminated by the front-end, e.g. unused functions with internal |
805 | // linkage. |
806 | if (!NeedsRuntimeHook && ContainsProfiling) |
807 | emitRuntimeHook(); |
808 | |
809 | emitRegistration(); |
810 | emitUses(); |
811 | emitInitialization(); |
812 | return true; |
813 | } |
814 | |
815 | static FunctionCallee getOrInsertValueProfilingCall( |
816 | Module &M, const TargetLibraryInfo &TLI, |
817 | ValueProfilingCallType CallType = ValueProfilingCallType::Default) { |
818 | LLVMContext &Ctx = M.getContext(); |
819 | auto *ReturnTy = Type::getVoidTy(C&: M.getContext()); |
820 | |
821 | AttributeList AL; |
822 | if (auto AK = TLI.getExtAttrForI32Param(Signed: false)) |
823 | AL = AL.addParamAttribute(C&: M.getContext(), ArgNo: 2, Kind: AK); |
824 | |
825 | assert((CallType == ValueProfilingCallType::Default || |
826 | CallType == ValueProfilingCallType::MemOp) && |
827 | "Must be Default or MemOp" ); |
828 | Type *ParamTypes[] = { |
829 | #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType |
830 | #include "llvm/ProfileData/InstrProfData.inc" |
831 | }; |
832 | auto *ValueProfilingCallTy = |
833 | FunctionType::get(Result: ReturnTy, Params: ArrayRef(ParamTypes), isVarArg: false); |
834 | StringRef FuncName = CallType == ValueProfilingCallType::Default |
835 | ? getInstrProfValueProfFuncName() |
836 | : getInstrProfValueProfMemOpFuncName(); |
837 | return M.getOrInsertFunction(Name: FuncName, T: ValueProfilingCallTy, AttributeList: AL); |
838 | } |
839 | |
840 | void InstrLowerer::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) { |
841 | GlobalVariable *Name = Ind->getName(); |
842 | uint64_t ValueKind = Ind->getValueKind()->getZExtValue(); |
843 | uint64_t Index = Ind->getIndex()->getZExtValue(); |
844 | auto &PD = ProfileDataMap[Name]; |
845 | PD.NumValueSites[ValueKind] = |
846 | std::max(a: PD.NumValueSites[ValueKind], b: (uint32_t)(Index + 1)); |
847 | } |
848 | |
849 | void InstrLowerer::lowerValueProfileInst(InstrProfValueProfileInst *Ind) { |
850 | // TODO: Value profiling heavily depends on the data section which is omitted |
851 | // in lightweight mode. We need to move the value profile pointer to the |
852 | // Counter struct to get this working. |
853 | assert( |
854 | !DebugInfoCorrelate && ProfileCorrelate == InstrProfCorrelator::NONE && |
855 | "Value profiling is not yet supported with lightweight instrumentation" ); |
856 | GlobalVariable *Name = Ind->getName(); |
857 | auto It = ProfileDataMap.find(Val: Name); |
858 | assert(It != ProfileDataMap.end() && It->second.DataVar && |
859 | "value profiling detected in function with no counter incerement" ); |
860 | |
861 | GlobalVariable *DataVar = It->second.DataVar; |
862 | uint64_t ValueKind = Ind->getValueKind()->getZExtValue(); |
863 | uint64_t Index = Ind->getIndex()->getZExtValue(); |
864 | for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind) |
865 | Index += It->second.NumValueSites[Kind]; |
866 | |
867 | IRBuilder<> Builder(Ind); |
868 | bool IsMemOpSize = (Ind->getValueKind()->getZExtValue() == |
869 | llvm::InstrProfValueKind::IPVK_MemOPSize); |
870 | CallInst *Call = nullptr; |
871 | auto *TLI = &GetTLI(*Ind->getFunction()); |
872 | |
873 | // To support value profiling calls within Windows exception handlers, funclet |
874 | // information contained within operand bundles needs to be copied over to |
875 | // the library call. This is required for the IR to be processed by the |
876 | // WinEHPrepare pass. |
877 | SmallVector<OperandBundleDef, 1> OpBundles; |
878 | Ind->getOperandBundlesAsDefs(Defs&: OpBundles); |
879 | if (!IsMemOpSize) { |
880 | Value *Args[3] = {Ind->getTargetValue(), DataVar, Builder.getInt32(C: Index)}; |
881 | Call = Builder.CreateCall(Callee: getOrInsertValueProfilingCall(M, TLI: *TLI), Args, |
882 | OpBundles); |
883 | } else { |
884 | Value *Args[3] = {Ind->getTargetValue(), DataVar, Builder.getInt32(C: Index)}; |
885 | Call = Builder.CreateCall( |
886 | Callee: getOrInsertValueProfilingCall(M, TLI: *TLI, CallType: ValueProfilingCallType::MemOp), |
887 | Args, OpBundles); |
888 | } |
889 | if (auto AK = TLI->getExtAttrForI32Param(Signed: false)) |
890 | Call->addParamAttr(ArgNo: 2, Kind: AK); |
891 | Ind->replaceAllUsesWith(V: Call); |
892 | Ind->eraseFromParent(); |
893 | } |
894 | |
895 | Value *InstrLowerer::getCounterAddress(InstrProfCntrInstBase *I) { |
896 | auto *Counters = getOrCreateRegionCounters(Inc: I); |
897 | IRBuilder<> Builder(I); |
898 | |
899 | if (isa<InstrProfTimestampInst>(Val: I)) |
900 | Counters->setAlignment(Align(8)); |
901 | |
902 | auto *Addr = Builder.CreateConstInBoundsGEP2_32( |
903 | Ty: Counters->getValueType(), Ptr: Counters, Idx0: 0, Idx1: I->getIndex()->getZExtValue()); |
904 | |
905 | if (!isRuntimeCounterRelocationEnabled()) |
906 | return Addr; |
907 | |
908 | Type *Int64Ty = Type::getInt64Ty(C&: M.getContext()); |
909 | Function *Fn = I->getParent()->getParent(); |
910 | LoadInst *&BiasLI = FunctionToProfileBiasMap[Fn]; |
911 | if (!BiasLI) { |
912 | IRBuilder<> EntryBuilder(&Fn->getEntryBlock().front()); |
913 | auto *Bias = M.getGlobalVariable(Name: getInstrProfCounterBiasVarName()); |
914 | if (!Bias) { |
915 | // Compiler must define this variable when runtime counter relocation |
916 | // is being used. Runtime has a weak external reference that is used |
917 | // to check whether that's the case or not. |
918 | Bias = new GlobalVariable( |
919 | M, Int64Ty, false, GlobalValue::LinkOnceODRLinkage, |
920 | Constant::getNullValue(Ty: Int64Ty), getInstrProfCounterBiasVarName()); |
921 | Bias->setVisibility(GlobalVariable::HiddenVisibility); |
922 | // A definition that's weak (linkonce_odr) without being in a COMDAT |
923 | // section wouldn't lead to link errors, but it would lead to a dead |
924 | // data word from every TU but one. Putting it in COMDAT ensures there |
925 | // will be exactly one data slot in the link. |
926 | if (TT.supportsCOMDAT()) |
927 | Bias->setComdat(M.getOrInsertComdat(Name: Bias->getName())); |
928 | } |
929 | BiasLI = EntryBuilder.CreateLoad(Ty: Int64Ty, Ptr: Bias); |
930 | } |
931 | auto *Add = Builder.CreateAdd(LHS: Builder.CreatePtrToInt(V: Addr, DestTy: Int64Ty), RHS: BiasLI); |
932 | return Builder.CreateIntToPtr(V: Add, DestTy: Addr->getType()); |
933 | } |
934 | |
935 | Value *InstrLowerer::getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I) { |
936 | auto *Bitmaps = getOrCreateRegionBitmaps(Inc: I); |
937 | IRBuilder<> Builder(I); |
938 | |
939 | auto *Addr = Builder.CreateConstInBoundsGEP2_32( |
940 | Ty: Bitmaps->getValueType(), Ptr: Bitmaps, Idx0: 0, Idx1: I->getBitmapIndex()->getZExtValue()); |
941 | |
942 | if (isRuntimeCounterRelocationEnabled()) { |
943 | LLVMContext &Ctx = M.getContext(); |
944 | Ctx.diagnose(DI: DiagnosticInfoPGOProfile( |
945 | M.getName().data(), |
946 | Twine("Runtime counter relocation is presently not supported for MC/DC " |
947 | "bitmaps." ), |
948 | DS_Warning)); |
949 | } |
950 | |
951 | return Addr; |
952 | } |
953 | |
954 | void InstrLowerer::lowerCover(InstrProfCoverInst *CoverInstruction) { |
955 | auto *Addr = getCounterAddress(I: CoverInstruction); |
956 | IRBuilder<> Builder(CoverInstruction); |
957 | // We store zero to represent that this block is covered. |
958 | Builder.CreateStore(Val: Builder.getInt8(C: 0), Ptr: Addr); |
959 | CoverInstruction->eraseFromParent(); |
960 | } |
961 | |
962 | void InstrLowerer::lowerTimestamp( |
963 | InstrProfTimestampInst *TimestampInstruction) { |
964 | assert(TimestampInstruction->getIndex()->isZeroValue() && |
965 | "timestamp probes are always the first probe for a function" ); |
966 | auto &Ctx = M.getContext(); |
967 | auto *TimestampAddr = getCounterAddress(I: TimestampInstruction); |
968 | IRBuilder<> Builder(TimestampInstruction); |
969 | auto *CalleeTy = |
970 | FunctionType::get(Result: Type::getVoidTy(C&: Ctx), Params: TimestampAddr->getType(), isVarArg: false); |
971 | auto Callee = M.getOrInsertFunction( |
972 | INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SET_TIMESTAMP), T: CalleeTy); |
973 | Builder.CreateCall(Callee, Args: {TimestampAddr}); |
974 | TimestampInstruction->eraseFromParent(); |
975 | } |
976 | |
977 | void InstrLowerer::lowerIncrement(InstrProfIncrementInst *Inc) { |
978 | auto *Addr = getCounterAddress(I: Inc); |
979 | |
980 | IRBuilder<> Builder(Inc); |
981 | if (Options.Atomic || AtomicCounterUpdateAll || |
982 | (Inc->getIndex()->isZeroValue() && AtomicFirstCounter)) { |
983 | Builder.CreateAtomicRMW(Op: AtomicRMWInst::Add, Ptr: Addr, Val: Inc->getStep(), |
984 | Align: MaybeAlign(), Ordering: AtomicOrdering::Monotonic); |
985 | } else { |
986 | Value *IncStep = Inc->getStep(); |
987 | Value *Load = Builder.CreateLoad(Ty: IncStep->getType(), Ptr: Addr, Name: "pgocount" ); |
988 | auto *Count = Builder.CreateAdd(LHS: Load, RHS: Inc->getStep()); |
989 | auto *Store = Builder.CreateStore(Val: Count, Ptr: Addr); |
990 | if (isCounterPromotionEnabled()) |
991 | PromotionCandidates.emplace_back(args: cast<Instruction>(Val: Load), args&: Store); |
992 | } |
993 | Inc->eraseFromParent(); |
994 | } |
995 | |
996 | void InstrLowerer::lowerCoverageData(GlobalVariable *CoverageNamesVar) { |
997 | ConstantArray *Names = |
998 | cast<ConstantArray>(Val: CoverageNamesVar->getInitializer()); |
999 | for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) { |
1000 | Constant *NC = Names->getOperand(i_nocapture: I); |
1001 | Value *V = NC->stripPointerCasts(); |
1002 | assert(isa<GlobalVariable>(V) && "Missing reference to function name" ); |
1003 | GlobalVariable *Name = cast<GlobalVariable>(Val: V); |
1004 | |
1005 | Name->setLinkage(GlobalValue::PrivateLinkage); |
1006 | ReferencedNames.push_back(x: Name); |
1007 | if (isa<ConstantExpr>(Val: NC)) |
1008 | NC->dropAllReferences(); |
1009 | } |
1010 | CoverageNamesVar->eraseFromParent(); |
1011 | } |
1012 | |
1013 | void InstrLowerer::lowerMCDCTestVectorBitmapUpdate( |
1014 | InstrProfMCDCTVBitmapUpdate *Update) { |
1015 | IRBuilder<> Builder(Update); |
1016 | auto *Int8Ty = Type::getInt8Ty(C&: M.getContext()); |
1017 | auto *Int32Ty = Type::getInt32Ty(C&: M.getContext()); |
1018 | auto *MCDCCondBitmapAddr = Update->getMCDCCondBitmapAddr(); |
1019 | auto *BitmapAddr = getBitmapAddress(I: Update); |
1020 | |
1021 | // Load Temp Val. |
1022 | // %mcdc.temp = load i32, ptr %mcdc.addr, align 4 |
1023 | auto *Temp = Builder.CreateLoad(Ty: Int32Ty, Ptr: MCDCCondBitmapAddr, Name: "mcdc.temp" ); |
1024 | |
1025 | // Calculate byte offset using div8. |
1026 | // %1 = lshr i32 %mcdc.temp, 3 |
1027 | auto *BitmapByteOffset = Builder.CreateLShr(LHS: Temp, RHS: 0x3); |
1028 | |
1029 | // Add byte offset to section base byte address. |
1030 | // %4 = getelementptr inbounds i8, ptr @__profbm_test, i32 %1 |
1031 | auto *BitmapByteAddr = |
1032 | Builder.CreateInBoundsPtrAdd(Ptr: BitmapAddr, Offset: BitmapByteOffset); |
1033 | |
1034 | // Calculate bit offset into bitmap byte by using div8 remainder (AND ~8) |
1035 | // %5 = and i32 %mcdc.temp, 7 |
1036 | // %6 = trunc i32 %5 to i8 |
1037 | auto *BitToSet = Builder.CreateTrunc(V: Builder.CreateAnd(LHS: Temp, RHS: 0x7), DestTy: Int8Ty); |
1038 | |
1039 | // Shift bit offset left to form a bitmap. |
1040 | // %7 = shl i8 1, %6 |
1041 | auto *ShiftedVal = Builder.CreateShl(LHS: Builder.getInt8(C: 0x1), RHS: BitToSet); |
1042 | |
1043 | // Load profile bitmap byte. |
1044 | // %mcdc.bits = load i8, ptr %4, align 1 |
1045 | auto *Bitmap = Builder.CreateLoad(Ty: Int8Ty, Ptr: BitmapByteAddr, Name: "mcdc.bits" ); |
1046 | |
1047 | // Perform logical OR of profile bitmap byte and shifted bit offset. |
1048 | // %8 = or i8 %mcdc.bits, %7 |
1049 | auto *Result = Builder.CreateOr(LHS: Bitmap, RHS: ShiftedVal); |
1050 | |
1051 | // Store the updated profile bitmap byte. |
1052 | // store i8 %8, ptr %3, align 1 |
1053 | Builder.CreateStore(Val: Result, Ptr: BitmapByteAddr); |
1054 | Update->eraseFromParent(); |
1055 | } |
1056 | |
1057 | void InstrLowerer::lowerMCDCCondBitmapUpdate( |
1058 | InstrProfMCDCCondBitmapUpdate *Update) { |
1059 | IRBuilder<> Builder(Update); |
1060 | auto *Int32Ty = Type::getInt32Ty(C&: M.getContext()); |
1061 | auto *MCDCCondBitmapAddr = Update->getMCDCCondBitmapAddr(); |
1062 | |
1063 | // Load the MCDC temporary value from the stack. |
1064 | // %mcdc.temp = load i32, ptr %mcdc.addr, align 4 |
1065 | auto *Temp = Builder.CreateLoad(Ty: Int32Ty, Ptr: MCDCCondBitmapAddr, Name: "mcdc.temp" ); |
1066 | |
1067 | // Zero-extend the evaluated condition boolean value (0 or 1) by 32bits. |
1068 | // %1 = zext i1 %tobool to i32 |
1069 | auto *CondV_32 = Builder.CreateZExt(V: Update->getCondBool(), DestTy: Int32Ty); |
1070 | |
1071 | // Shift the boolean value left (by the condition's ID) to form a bitmap. |
1072 | // %2 = shl i32 %1, <Update->getCondID()> |
1073 | auto *ShiftedVal = Builder.CreateShl(LHS: CondV_32, RHS: Update->getCondID()); |
1074 | |
1075 | // Perform logical OR of the bitmap against the loaded MCDC temporary value. |
1076 | // %3 = or i32 %mcdc.temp, %2 |
1077 | auto *Result = Builder.CreateOr(LHS: Temp, RHS: ShiftedVal); |
1078 | |
1079 | // Store the updated temporary value back to the stack. |
1080 | // store i32 %3, ptr %mcdc.addr, align 4 |
1081 | Builder.CreateStore(Val: Result, Ptr: MCDCCondBitmapAddr); |
1082 | Update->eraseFromParent(); |
1083 | } |
1084 | |
1085 | /// Get the name of a profiling variable for a particular function. |
1086 | static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix, |
1087 | bool &Renamed) { |
1088 | StringRef NamePrefix = getInstrProfNameVarPrefix(); |
1089 | StringRef Name = Inc->getName()->getName().substr(Start: NamePrefix.size()); |
1090 | Function *F = Inc->getParent()->getParent(); |
1091 | Module *M = F->getParent(); |
1092 | if (!DoHashBasedCounterSplit || !isIRPGOFlagSet(M) || |
1093 | !canRenameComdatFunc(F: *F)) { |
1094 | Renamed = false; |
1095 | return (Prefix + Name).str(); |
1096 | } |
1097 | Renamed = true; |
1098 | uint64_t FuncHash = Inc->getHash()->getZExtValue(); |
1099 | SmallVector<char, 24> HashPostfix; |
1100 | if (Name.ends_with(Suffix: (Twine("." ) + Twine(FuncHash)).toStringRef(Out&: HashPostfix))) |
1101 | return (Prefix + Name).str(); |
1102 | return (Prefix + Name + "." + Twine(FuncHash)).str(); |
1103 | } |
1104 | |
1105 | static inline bool shouldRecordFunctionAddr(Function *F) { |
1106 | // Only record function addresses if IR PGO is enabled or if clang value |
1107 | // profiling is enabled. Recording function addresses greatly increases object |
1108 | // file size, because it prevents the inliner from deleting functions that |
1109 | // have been inlined everywhere. |
1110 | if (!profDataReferencedByCode(M: *F->getParent())) |
1111 | return false; |
1112 | |
1113 | // Check the linkage |
1114 | bool HasAvailableExternallyLinkage = F->hasAvailableExternallyLinkage(); |
1115 | if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() && |
1116 | !HasAvailableExternallyLinkage) |
1117 | return true; |
1118 | |
1119 | // A function marked 'alwaysinline' with available_externally linkage can't |
1120 | // have its address taken. Doing so would create an undefined external ref to |
1121 | // the function, which would fail to link. |
1122 | if (HasAvailableExternallyLinkage && |
1123 | F->hasFnAttribute(Attribute::AlwaysInline)) |
1124 | return false; |
1125 | |
1126 | // Prohibit function address recording if the function is both internal and |
1127 | // COMDAT. This avoids the profile data variable referencing internal symbols |
1128 | // in COMDAT. |
1129 | if (F->hasLocalLinkage() && F->hasComdat()) |
1130 | return false; |
1131 | |
1132 | // Check uses of this function for other than direct calls or invokes to it. |
1133 | // Inline virtual functions have linkeOnceODR linkage. When a key method |
1134 | // exists, the vtable will only be emitted in the TU where the key method |
1135 | // is defined. In a TU where vtable is not available, the function won't |
1136 | // be 'addresstaken'. If its address is not recorded here, the profile data |
1137 | // with missing address may be picked by the linker leading to missing |
1138 | // indirect call target info. |
1139 | return F->hasAddressTaken() || F->hasLinkOnceLinkage(); |
1140 | } |
1141 | |
1142 | static inline bool shouldUsePublicSymbol(Function *Fn) { |
1143 | // It isn't legal to make an alias of this function at all |
1144 | if (Fn->isDeclarationForLinker()) |
1145 | return true; |
1146 | |
1147 | // Symbols with local linkage can just use the symbol directly without |
1148 | // introducing relocations |
1149 | if (Fn->hasLocalLinkage()) |
1150 | return true; |
1151 | |
1152 | // PGO + ThinLTO + CFI cause duplicate symbols to be introduced due to some |
1153 | // unfavorable interaction between the new alias and the alias renaming done |
1154 | // in LowerTypeTests under ThinLTO. For comdat functions that would normally |
1155 | // be deduplicated, but the renaming scheme ends up preventing renaming, since |
1156 | // it creates unique names for each alias, resulting in duplicated symbols. In |
1157 | // the future, we should update the CFI related passes to migrate these |
1158 | // aliases to the same module as the jump-table they refer to will be defined. |
1159 | if (Fn->hasMetadata(KindID: LLVMContext::MD_type)) |
1160 | return true; |
1161 | |
1162 | // For comdat functions, an alias would need the same linkage as the original |
1163 | // function and hidden visibility. There is no point in adding an alias with |
1164 | // identical linkage an visibility to avoid introducing symbolic relocations. |
1165 | if (Fn->hasComdat() && |
1166 | (Fn->getVisibility() == GlobalValue::VisibilityTypes::HiddenVisibility)) |
1167 | return true; |
1168 | |
1169 | // its OK to use an alias |
1170 | return false; |
1171 | } |
1172 | |
1173 | static inline Constant *getFuncAddrForProfData(Function *Fn) { |
1174 | auto *Int8PtrTy = PointerType::getUnqual(C&: Fn->getContext()); |
1175 | // Store a nullptr in __llvm_profd, if we shouldn't use a real address |
1176 | if (!shouldRecordFunctionAddr(F: Fn)) |
1177 | return ConstantPointerNull::get(T: Int8PtrTy); |
1178 | |
1179 | // If we can't use an alias, we must use the public symbol, even though this |
1180 | // may require a symbolic relocation. |
1181 | if (shouldUsePublicSymbol(Fn)) |
1182 | return Fn; |
1183 | |
1184 | // When possible use a private alias to avoid symbolic relocations. |
1185 | auto *GA = GlobalAlias::create(Linkage: GlobalValue::LinkageTypes::PrivateLinkage, |
1186 | Name: Fn->getName() + ".local" , Aliasee: Fn); |
1187 | |
1188 | // When the instrumented function is a COMDAT function, we cannot use a |
1189 | // private alias. If we did, we would create reference to a local label in |
1190 | // this function's section. If this version of the function isn't selected by |
1191 | // the linker, then the metadata would introduce a reference to a discarded |
1192 | // section. So, for COMDAT functions, we need to adjust the linkage of the |
1193 | // alias. Using hidden visibility avoids a dynamic relocation and an entry in |
1194 | // the dynamic symbol table. |
1195 | // |
1196 | // Note that this handles COMDAT functions with visibility other than Hidden, |
1197 | // since that case is covered in shouldUsePublicSymbol() |
1198 | if (Fn->hasComdat()) { |
1199 | GA->setLinkage(Fn->getLinkage()); |
1200 | GA->setVisibility(GlobalValue::VisibilityTypes::HiddenVisibility); |
1201 | } |
1202 | |
1203 | // appendToCompilerUsed(*Fn->getParent(), {GA}); |
1204 | |
1205 | return GA; |
1206 | } |
1207 | |
1208 | static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) { |
1209 | // compiler-rt uses linker support to get data/counters/name start/end for |
1210 | // ELF, COFF, Mach-O and XCOFF. |
1211 | if (TT.isOSBinFormatELF() || TT.isOSBinFormatCOFF() || |
1212 | TT.isOSBinFormatMachO() || TT.isOSBinFormatXCOFF()) |
1213 | return false; |
1214 | |
1215 | return true; |
1216 | } |
1217 | |
1218 | void InstrLowerer::maybeSetComdat(GlobalVariable *GV, GlobalObject *GO, |
1219 | StringRef CounterGroupName) { |
1220 | // Place lowered global variables in a comdat group if the associated function |
1221 | // or global variable is a COMDAT. This will make sure that only one copy of |
1222 | // global variable (e.g. function counters) of the COMDAT function will be |
1223 | // emitted after linking. |
1224 | bool NeedComdat = needsComdatForCounter(GV: *GO, M); |
1225 | bool UseComdat = (NeedComdat || TT.isOSBinFormatELF()); |
1226 | |
1227 | if (!UseComdat) |
1228 | return; |
1229 | |
1230 | // Keep in mind that this pass may run before the inliner, so we need to |
1231 | // create a new comdat group (for counters, profiling data, etc). If we use |
1232 | // the comdat of the parent function, that will result in relocations against |
1233 | // discarded sections. |
1234 | // |
1235 | // If the data variable is referenced by code, non-counter variables (notably |
1236 | // profiling data) and counters have to be in different comdats for COFF |
1237 | // because the Visual C++ linker will report duplicate symbol errors if there |
1238 | // are multiple external symbols with the same name marked |
1239 | // IMAGE_COMDAT_SELECT_ASSOCIATIVE. |
1240 | StringRef GroupName = TT.isOSBinFormatCOFF() && DataReferencedByCode |
1241 | ? GV->getName() |
1242 | : CounterGroupName; |
1243 | Comdat *C = M.getOrInsertComdat(Name: GroupName); |
1244 | |
1245 | if (!NeedComdat) { |
1246 | // Object file format must be ELF since `UseComdat && !NeedComdat` is true. |
1247 | // |
1248 | // For ELF, when not using COMDAT, put counters, data and values into a |
1249 | // nodeduplicate COMDAT which is lowered to a zero-flag section group. This |
1250 | // allows -z start-stop-gc to discard the entire group when the function is |
1251 | // discarded. |
1252 | C->setSelectionKind(Comdat::NoDeduplicate); |
1253 | } |
1254 | GV->setComdat(C); |
1255 | // COFF doesn't allow the comdat group leader to have private linkage, so |
1256 | // upgrade private linkage to internal linkage to produce a symbol table |
1257 | // entry. |
1258 | if (TT.isOSBinFormatCOFF() && GV->hasPrivateLinkage()) |
1259 | GV->setLinkage(GlobalValue::InternalLinkage); |
1260 | } |
1261 | |
1262 | static inline bool shouldRecordVTableAddr(GlobalVariable *GV) { |
1263 | if (!profDataReferencedByCode(M: *GV->getParent())) |
1264 | return false; |
1265 | |
1266 | if (!GV->hasLinkOnceLinkage() && !GV->hasLocalLinkage() && |
1267 | !GV->hasAvailableExternallyLinkage()) |
1268 | return true; |
1269 | |
1270 | // This avoids the profile data from referencing internal symbols in |
1271 | // COMDAT. |
1272 | if (GV->hasLocalLinkage() && GV->hasComdat()) |
1273 | return false; |
1274 | |
1275 | return true; |
1276 | } |
1277 | |
1278 | // FIXME: Introduce an internal alias like what's done for functions to reduce |
1279 | // the number of relocation entries. |
1280 | static inline Constant *getVTableAddrForProfData(GlobalVariable *GV) { |
1281 | auto *Int8PtrTy = PointerType::getUnqual(C&: GV->getContext()); |
1282 | |
1283 | // Store a nullptr in __profvt_ if a real address shouldn't be used. |
1284 | if (!shouldRecordVTableAddr(GV)) |
1285 | return ConstantPointerNull::get(T: Int8PtrTy); |
1286 | |
1287 | return ConstantExpr::getBitCast(C: GV, Ty: Int8PtrTy); |
1288 | } |
1289 | |
1290 | void InstrLowerer::getOrCreateVTableProfData(GlobalVariable *GV) { |
1291 | assert(!DebugInfoCorrelate && |
1292 | "Value profiling is not supported with lightweight instrumentation" ); |
1293 | if (GV->isDeclaration() || GV->hasAvailableExternallyLinkage()) |
1294 | return; |
1295 | |
1296 | // Skip llvm internal global variable or __prof variables. |
1297 | if (GV->getName().starts_with(Prefix: "llvm." ) || |
1298 | GV->getName().starts_with(Prefix: "__llvm" ) || |
1299 | GV->getName().starts_with(Prefix: "__prof" )) |
1300 | return; |
1301 | |
1302 | // VTableProfData already created |
1303 | auto It = VTableDataMap.find(Val: GV); |
1304 | if (It != VTableDataMap.end() && It->second) |
1305 | return; |
1306 | |
1307 | GlobalValue::LinkageTypes Linkage = GV->getLinkage(); |
1308 | GlobalValue::VisibilityTypes Visibility = GV->getVisibility(); |
1309 | |
1310 | // This is to keep consistent with per-function profile data |
1311 | // for correctness. |
1312 | if (TT.isOSBinFormatXCOFF()) { |
1313 | Linkage = GlobalValue::InternalLinkage; |
1314 | Visibility = GlobalValue::DefaultVisibility; |
1315 | } |
1316 | |
1317 | LLVMContext &Ctx = M.getContext(); |
1318 | Type *DataTypes[] = { |
1319 | #define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) LLVMType, |
1320 | #include "llvm/ProfileData/InstrProfData.inc" |
1321 | #undef INSTR_PROF_VTABLE_DATA |
1322 | }; |
1323 | |
1324 | auto *DataTy = StructType::get(Context&: Ctx, Elements: ArrayRef(DataTypes)); |
1325 | |
1326 | // Used by INSTR_PROF_VTABLE_DATA MACRO |
1327 | Constant *VTableAddr = getVTableAddrForProfData(GV); |
1328 | const std::string PGOVTableName = getPGOName(V: *GV); |
1329 | // Record the length of the vtable. This is needed since vtable pointers |
1330 | // loaded from C++ objects might be from the middle of a vtable definition. |
1331 | uint32_t VTableSizeVal = |
1332 | M.getDataLayout().getTypeAllocSize(Ty: GV->getValueType()); |
1333 | |
1334 | Constant *DataVals[] = { |
1335 | #define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) Init, |
1336 | #include "llvm/ProfileData/InstrProfData.inc" |
1337 | #undef INSTR_PROF_VTABLE_DATA |
1338 | }; |
1339 | |
1340 | auto *Data = |
1341 | new GlobalVariable(M, DataTy, /*constant=*/false, Linkage, |
1342 | ConstantStruct::get(T: DataTy, V: DataVals), |
1343 | getInstrProfVTableVarPrefix() + PGOVTableName); |
1344 | |
1345 | Data->setVisibility(Visibility); |
1346 | Data->setSection(getInstrProfSectionName(IPSK: IPSK_vtab, OF: TT.getObjectFormat())); |
1347 | Data->setAlignment(Align(8)); |
1348 | |
1349 | maybeSetComdat(GV: Data, GO: GV, CounterGroupName: Data->getName()); |
1350 | |
1351 | VTableDataMap[GV] = Data; |
1352 | |
1353 | ReferencedVTables.push_back(x: GV); |
1354 | |
1355 | // VTable <Hash, Addr> is used by runtime but not referenced by other |
1356 | // sections. Conservatively mark it linker retained. |
1357 | UsedVars.push_back(x: Data); |
1358 | } |
1359 | |
1360 | GlobalVariable *InstrLowerer::setupProfileSection(InstrProfInstBase *Inc, |
1361 | InstrProfSectKind IPSK) { |
1362 | GlobalVariable *NamePtr = Inc->getName(); |
1363 | |
1364 | // Match the linkage and visibility of the name global. |
1365 | Function *Fn = Inc->getParent()->getParent(); |
1366 | GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage(); |
1367 | GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility(); |
1368 | |
1369 | // Use internal rather than private linkage so the counter variable shows up |
1370 | // in the symbol table when using debug info for correlation. |
1371 | if ((DebugInfoCorrelate || |
1372 | ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO) && |
1373 | TT.isOSBinFormatMachO() && Linkage == GlobalValue::PrivateLinkage) |
1374 | Linkage = GlobalValue::InternalLinkage; |
1375 | |
1376 | // Due to the limitation of binder as of 2021/09/28, the duplicate weak |
1377 | // symbols in the same csect won't be discarded. When there are duplicate weak |
1378 | // symbols, we can NOT guarantee that the relocations get resolved to the |
1379 | // intended weak symbol, so we can not ensure the correctness of the relative |
1380 | // CounterPtr, so we have to use private linkage for counter and data symbols. |
1381 | if (TT.isOSBinFormatXCOFF()) { |
1382 | Linkage = GlobalValue::PrivateLinkage; |
1383 | Visibility = GlobalValue::DefaultVisibility; |
1384 | } |
1385 | // Move the name variable to the right section. |
1386 | bool Renamed; |
1387 | GlobalVariable *Ptr; |
1388 | StringRef VarPrefix; |
1389 | std::string VarName; |
1390 | if (IPSK == IPSK_cnts) { |
1391 | VarPrefix = getInstrProfCountersVarPrefix(); |
1392 | VarName = getVarName(Inc, Prefix: VarPrefix, Renamed); |
1393 | InstrProfCntrInstBase *CntrIncrement = dyn_cast<InstrProfCntrInstBase>(Val: Inc); |
1394 | Ptr = createRegionCounters(Inc: CntrIncrement, Name: VarName, Linkage); |
1395 | } else if (IPSK == IPSK_bitmap) { |
1396 | VarPrefix = getInstrProfBitmapVarPrefix(); |
1397 | VarName = getVarName(Inc, Prefix: VarPrefix, Renamed); |
1398 | InstrProfMCDCBitmapInstBase *BitmapUpdate = |
1399 | dyn_cast<InstrProfMCDCBitmapInstBase>(Val: Inc); |
1400 | Ptr = createRegionBitmaps(Inc: BitmapUpdate, Name: VarName, Linkage); |
1401 | } else { |
1402 | llvm_unreachable("Profile Section must be for Counters or Bitmaps" ); |
1403 | } |
1404 | |
1405 | Ptr->setVisibility(Visibility); |
1406 | // Put the counters and bitmaps in their own sections so linkers can |
1407 | // remove unneeded sections. |
1408 | Ptr->setSection(getInstrProfSectionName(IPSK, OF: TT.getObjectFormat())); |
1409 | Ptr->setLinkage(Linkage); |
1410 | maybeSetComdat(GV: Ptr, GO: Fn, CounterGroupName: VarName); |
1411 | return Ptr; |
1412 | } |
1413 | |
1414 | GlobalVariable * |
1415 | InstrLowerer::createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc, |
1416 | StringRef Name, |
1417 | GlobalValue::LinkageTypes Linkage) { |
1418 | uint64_t NumBytes = Inc->getNumBitmapBytes()->getZExtValue(); |
1419 | auto *BitmapTy = ArrayType::get(ElementType: Type::getInt8Ty(C&: M.getContext()), NumElements: NumBytes); |
1420 | auto GV = new GlobalVariable(M, BitmapTy, false, Linkage, |
1421 | Constant::getNullValue(Ty: BitmapTy), Name); |
1422 | GV->setAlignment(Align(1)); |
1423 | return GV; |
1424 | } |
1425 | |
1426 | GlobalVariable * |
1427 | InstrLowerer::getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc) { |
1428 | GlobalVariable *NamePtr = Inc->getName(); |
1429 | auto &PD = ProfileDataMap[NamePtr]; |
1430 | if (PD.RegionBitmaps) |
1431 | return PD.RegionBitmaps; |
1432 | |
1433 | // If RegionBitmaps doesn't already exist, create it by first setting up |
1434 | // the corresponding profile section. |
1435 | auto *BitmapPtr = setupProfileSection(Inc, IPSK: IPSK_bitmap); |
1436 | PD.RegionBitmaps = BitmapPtr; |
1437 | PD.NumBitmapBytes = Inc->getNumBitmapBytes()->getZExtValue(); |
1438 | return PD.RegionBitmaps; |
1439 | } |
1440 | |
1441 | GlobalVariable * |
1442 | InstrLowerer::createRegionCounters(InstrProfCntrInstBase *Inc, StringRef Name, |
1443 | GlobalValue::LinkageTypes Linkage) { |
1444 | uint64_t NumCounters = Inc->getNumCounters()->getZExtValue(); |
1445 | auto &Ctx = M.getContext(); |
1446 | GlobalVariable *GV; |
1447 | if (isa<InstrProfCoverInst>(Val: Inc)) { |
1448 | auto *CounterTy = Type::getInt8Ty(C&: Ctx); |
1449 | auto *CounterArrTy = ArrayType::get(ElementType: CounterTy, NumElements: NumCounters); |
1450 | // TODO: `Constant::getAllOnesValue()` does not yet accept an array type. |
1451 | std::vector<Constant *> InitialValues(NumCounters, |
1452 | Constant::getAllOnesValue(Ty: CounterTy)); |
1453 | GV = new GlobalVariable(M, CounterArrTy, false, Linkage, |
1454 | ConstantArray::get(T: CounterArrTy, V: InitialValues), |
1455 | Name); |
1456 | GV->setAlignment(Align(1)); |
1457 | } else { |
1458 | auto *CounterTy = ArrayType::get(ElementType: Type::getInt64Ty(C&: Ctx), NumElements: NumCounters); |
1459 | GV = new GlobalVariable(M, CounterTy, false, Linkage, |
1460 | Constant::getNullValue(Ty: CounterTy), Name); |
1461 | GV->setAlignment(Align(8)); |
1462 | } |
1463 | return GV; |
1464 | } |
1465 | |
1466 | GlobalVariable * |
1467 | InstrLowerer::getOrCreateRegionCounters(InstrProfCntrInstBase *Inc) { |
1468 | GlobalVariable *NamePtr = Inc->getName(); |
1469 | auto &PD = ProfileDataMap[NamePtr]; |
1470 | if (PD.RegionCounters) |
1471 | return PD.RegionCounters; |
1472 | |
1473 | // If RegionCounters doesn't already exist, create it by first setting up |
1474 | // the corresponding profile section. |
1475 | auto *CounterPtr = setupProfileSection(Inc, IPSK: IPSK_cnts); |
1476 | PD.RegionCounters = CounterPtr; |
1477 | |
1478 | if (DebugInfoCorrelate || |
1479 | ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO) { |
1480 | LLVMContext &Ctx = M.getContext(); |
1481 | Function *Fn = Inc->getParent()->getParent(); |
1482 | if (auto *SP = Fn->getSubprogram()) { |
1483 | DIBuilder DB(M, true, SP->getUnit()); |
1484 | Metadata *FunctionNameAnnotation[] = { |
1485 | MDString::get(Context&: Ctx, Str: InstrProfCorrelator::FunctionNameAttributeName), |
1486 | MDString::get(Context&: Ctx, Str: getPGOFuncNameVarInitializer(NameVar: NamePtr)), |
1487 | }; |
1488 | Metadata *CFGHashAnnotation[] = { |
1489 | MDString::get(Context&: Ctx, Str: InstrProfCorrelator::CFGHashAttributeName), |
1490 | ConstantAsMetadata::get(C: Inc->getHash()), |
1491 | }; |
1492 | Metadata *NumCountersAnnotation[] = { |
1493 | MDString::get(Context&: Ctx, Str: InstrProfCorrelator::NumCountersAttributeName), |
1494 | ConstantAsMetadata::get(C: Inc->getNumCounters()), |
1495 | }; |
1496 | auto Annotations = DB.getOrCreateArray(Elements: { |
1497 | MDNode::get(Context&: Ctx, MDs: FunctionNameAnnotation), |
1498 | MDNode::get(Context&: Ctx, MDs: CFGHashAnnotation), |
1499 | MDNode::get(Context&: Ctx, MDs: NumCountersAnnotation), |
1500 | }); |
1501 | auto *DICounter = DB.createGlobalVariableExpression( |
1502 | Context: SP, Name: CounterPtr->getName(), /*LinkageName=*/StringRef(), File: SP->getFile(), |
1503 | /*LineNo=*/0, Ty: DB.createUnspecifiedType(Name: "Profile Data Type" ), |
1504 | IsLocalToUnit: CounterPtr->hasLocalLinkage(), /*IsDefined=*/isDefined: true, /*Expr=*/nullptr, |
1505 | /*Decl=*/nullptr, /*TemplateParams=*/nullptr, /*AlignInBits=*/0, |
1506 | Annotations); |
1507 | CounterPtr->addDebugInfo(GV: DICounter); |
1508 | DB.finalize(); |
1509 | } |
1510 | |
1511 | // Mark the counter variable as used so that it isn't optimized out. |
1512 | CompilerUsedVars.push_back(x: PD.RegionCounters); |
1513 | } |
1514 | |
1515 | // Create the data variable (if it doesn't already exist). |
1516 | createDataVariable(Inc); |
1517 | |
1518 | return PD.RegionCounters; |
1519 | } |
1520 | |
1521 | void InstrLowerer::createDataVariable(InstrProfCntrInstBase *Inc) { |
1522 | // When debug information is correlated to profile data, a data variable |
1523 | // is not needed. |
1524 | if (DebugInfoCorrelate || ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO) |
1525 | return; |
1526 | |
1527 | GlobalVariable *NamePtr = Inc->getName(); |
1528 | auto &PD = ProfileDataMap[NamePtr]; |
1529 | |
1530 | // Return if data variable was already created. |
1531 | if (PD.DataVar) |
1532 | return; |
1533 | |
1534 | LLVMContext &Ctx = M.getContext(); |
1535 | |
1536 | Function *Fn = Inc->getParent()->getParent(); |
1537 | GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage(); |
1538 | GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility(); |
1539 | |
1540 | // Due to the limitation of binder as of 2021/09/28, the duplicate weak |
1541 | // symbols in the same csect won't be discarded. When there are duplicate weak |
1542 | // symbols, we can NOT guarantee that the relocations get resolved to the |
1543 | // intended weak symbol, so we can not ensure the correctness of the relative |
1544 | // CounterPtr, so we have to use private linkage for counter and data symbols. |
1545 | if (TT.isOSBinFormatXCOFF()) { |
1546 | Linkage = GlobalValue::PrivateLinkage; |
1547 | Visibility = GlobalValue::DefaultVisibility; |
1548 | } |
1549 | |
1550 | bool NeedComdat = needsComdatForCounter(GV: *Fn, M); |
1551 | bool Renamed; |
1552 | |
1553 | // The Data Variable section is anchored to profile counters. |
1554 | std::string CntsVarName = |
1555 | getVarName(Inc, Prefix: getInstrProfCountersVarPrefix(), Renamed); |
1556 | std::string DataVarName = |
1557 | getVarName(Inc, Prefix: getInstrProfDataVarPrefix(), Renamed); |
1558 | |
1559 | auto *Int8PtrTy = PointerType::getUnqual(C&: Ctx); |
1560 | // Allocate statically the array of pointers to value profile nodes for |
1561 | // the current function. |
1562 | Constant *ValuesPtrExpr = ConstantPointerNull::get(T: Int8PtrTy); |
1563 | uint64_t NS = 0; |
1564 | for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) |
1565 | NS += PD.NumValueSites[Kind]; |
1566 | if (NS > 0 && ValueProfileStaticAlloc && |
1567 | !needsRuntimeRegistrationOfSectionRange(TT)) { |
1568 | ArrayType *ValuesTy = ArrayType::get(ElementType: Type::getInt64Ty(C&: Ctx), NumElements: NS); |
1569 | auto *ValuesVar = new GlobalVariable( |
1570 | M, ValuesTy, false, Linkage, Constant::getNullValue(Ty: ValuesTy), |
1571 | getVarName(Inc, Prefix: getInstrProfValuesVarPrefix(), Renamed)); |
1572 | ValuesVar->setVisibility(Visibility); |
1573 | setGlobalVariableLargeSection(TargetTriple: TT, GV&: *ValuesVar); |
1574 | ValuesVar->setSection( |
1575 | getInstrProfSectionName(IPSK: IPSK_vals, OF: TT.getObjectFormat())); |
1576 | ValuesVar->setAlignment(Align(8)); |
1577 | maybeSetComdat(GV: ValuesVar, GO: Fn, CounterGroupName: CntsVarName); |
1578 | ValuesPtrExpr = ValuesVar; |
1579 | } |
1580 | |
1581 | uint64_t NumCounters = Inc->getNumCounters()->getZExtValue(); |
1582 | auto *CounterPtr = PD.RegionCounters; |
1583 | |
1584 | uint64_t NumBitmapBytes = PD.NumBitmapBytes; |
1585 | |
1586 | // Create data variable. |
1587 | auto *IntPtrTy = M.getDataLayout().getIntPtrType(C&: M.getContext()); |
1588 | auto *Int16Ty = Type::getInt16Ty(C&: Ctx); |
1589 | auto *Int16ArrayTy = ArrayType::get(ElementType: Int16Ty, NumElements: IPVK_Last + 1); |
1590 | Type *DataTypes[] = { |
1591 | #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType, |
1592 | #include "llvm/ProfileData/InstrProfData.inc" |
1593 | }; |
1594 | auto *DataTy = StructType::get(Context&: Ctx, Elements: ArrayRef(DataTypes)); |
1595 | |
1596 | Constant *FunctionAddr = getFuncAddrForProfData(Fn); |
1597 | |
1598 | Constant *Int16ArrayVals[IPVK_Last + 1]; |
1599 | for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) |
1600 | Int16ArrayVals[Kind] = ConstantInt::get(Ty: Int16Ty, V: PD.NumValueSites[Kind]); |
1601 | |
1602 | // If the data variable is not referenced by code (if we don't emit |
1603 | // @llvm.instrprof.value.profile, NS will be 0), and the counter keeps the |
1604 | // data variable live under linker GC, the data variable can be private. This |
1605 | // optimization applies to ELF. |
1606 | // |
1607 | // On COFF, a comdat leader cannot be local so we require DataReferencedByCode |
1608 | // to be false. |
1609 | // |
1610 | // If profd is in a deduplicate comdat, NS==0 with a hash suffix guarantees |
1611 | // that other copies must have the same CFG and cannot have value profiling. |
1612 | // If no hash suffix, other profd copies may be referenced by code. |
1613 | if (NS == 0 && !(DataReferencedByCode && NeedComdat && !Renamed) && |
1614 | (TT.isOSBinFormatELF() || |
1615 | (!DataReferencedByCode && TT.isOSBinFormatCOFF()))) { |
1616 | Linkage = GlobalValue::PrivateLinkage; |
1617 | Visibility = GlobalValue::DefaultVisibility; |
1618 | } |
1619 | auto *Data = |
1620 | new GlobalVariable(M, DataTy, false, Linkage, nullptr, DataVarName); |
1621 | Constant *RelativeCounterPtr; |
1622 | GlobalVariable *BitmapPtr = PD.RegionBitmaps; |
1623 | Constant *RelativeBitmapPtr = ConstantInt::get(Ty: IntPtrTy, V: 0); |
1624 | InstrProfSectKind DataSectionKind; |
1625 | // With binary profile correlation, profile data is not loaded into memory. |
1626 | // profile data must reference profile counter with an absolute relocation. |
1627 | if (ProfileCorrelate == InstrProfCorrelator::BINARY) { |
1628 | DataSectionKind = IPSK_covdata; |
1629 | RelativeCounterPtr = ConstantExpr::getPtrToInt(C: CounterPtr, Ty: IntPtrTy); |
1630 | if (BitmapPtr != nullptr) |
1631 | RelativeBitmapPtr = ConstantExpr::getPtrToInt(C: BitmapPtr, Ty: IntPtrTy); |
1632 | } else { |
1633 | // Reference the counter variable with a label difference (link-time |
1634 | // constant). |
1635 | DataSectionKind = IPSK_data; |
1636 | RelativeCounterPtr = |
1637 | ConstantExpr::getSub(C1: ConstantExpr::getPtrToInt(C: CounterPtr, Ty: IntPtrTy), |
1638 | C2: ConstantExpr::getPtrToInt(C: Data, Ty: IntPtrTy)); |
1639 | if (BitmapPtr != nullptr) |
1640 | RelativeBitmapPtr = |
1641 | ConstantExpr::getSub(C1: ConstantExpr::getPtrToInt(C: BitmapPtr, Ty: IntPtrTy), |
1642 | C2: ConstantExpr::getPtrToInt(C: Data, Ty: IntPtrTy)); |
1643 | } |
1644 | |
1645 | Constant *DataVals[] = { |
1646 | #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init, |
1647 | #include "llvm/ProfileData/InstrProfData.inc" |
1648 | }; |
1649 | Data->setInitializer(ConstantStruct::get(T: DataTy, V: DataVals)); |
1650 | |
1651 | Data->setVisibility(Visibility); |
1652 | Data->setSection( |
1653 | getInstrProfSectionName(IPSK: DataSectionKind, OF: TT.getObjectFormat())); |
1654 | Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT)); |
1655 | maybeSetComdat(GV: Data, GO: Fn, CounterGroupName: CntsVarName); |
1656 | |
1657 | PD.DataVar = Data; |
1658 | |
1659 | // Mark the data variable as used so that it isn't stripped out. |
1660 | CompilerUsedVars.push_back(x: Data); |
1661 | // Now that the linkage set by the FE has been passed to the data and counter |
1662 | // variables, reset Name variable's linkage and visibility to private so that |
1663 | // it can be removed later by the compiler. |
1664 | NamePtr->setLinkage(GlobalValue::PrivateLinkage); |
1665 | // Collect the referenced names to be used by emitNameData. |
1666 | ReferencedNames.push_back(x: NamePtr); |
1667 | } |
1668 | |
1669 | void InstrLowerer::emitVNodes() { |
1670 | if (!ValueProfileStaticAlloc) |
1671 | return; |
1672 | |
1673 | // For now only support this on platforms that do |
1674 | // not require runtime registration to discover |
1675 | // named section start/end. |
1676 | if (needsRuntimeRegistrationOfSectionRange(TT)) |
1677 | return; |
1678 | |
1679 | size_t TotalNS = 0; |
1680 | for (auto &PD : ProfileDataMap) { |
1681 | for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) |
1682 | TotalNS += PD.second.NumValueSites[Kind]; |
1683 | } |
1684 | |
1685 | if (!TotalNS) |
1686 | return; |
1687 | |
1688 | uint64_t NumCounters = TotalNS * NumCountersPerValueSite; |
1689 | // Heuristic for small programs with very few total value sites. |
1690 | // The default value of vp-counters-per-site is chosen based on |
1691 | // the observation that large apps usually have a low percentage |
1692 | // of value sites that actually have any profile data, and thus |
1693 | // the average number of counters per site is low. For small |
1694 | // apps with very few sites, this may not be true. Bump up the |
1695 | // number of counters in this case. |
1696 | #define INSTR_PROF_MIN_VAL_COUNTS 10 |
1697 | if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS) |
1698 | NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, b: (int)NumCounters * 2); |
1699 | |
1700 | auto &Ctx = M.getContext(); |
1701 | Type *VNodeTypes[] = { |
1702 | #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType, |
1703 | #include "llvm/ProfileData/InstrProfData.inc" |
1704 | }; |
1705 | auto *VNodeTy = StructType::get(Context&: Ctx, Elements: ArrayRef(VNodeTypes)); |
1706 | |
1707 | ArrayType *VNodesTy = ArrayType::get(ElementType: VNodeTy, NumElements: NumCounters); |
1708 | auto *VNodesVar = new GlobalVariable( |
1709 | M, VNodesTy, false, GlobalValue::PrivateLinkage, |
1710 | Constant::getNullValue(Ty: VNodesTy), getInstrProfVNodesVarName()); |
1711 | setGlobalVariableLargeSection(TargetTriple: TT, GV&: *VNodesVar); |
1712 | VNodesVar->setSection( |
1713 | getInstrProfSectionName(IPSK: IPSK_vnodes, OF: TT.getObjectFormat())); |
1714 | VNodesVar->setAlignment(M.getDataLayout().getABITypeAlign(Ty: VNodesTy)); |
1715 | // VNodesVar is used by runtime but not referenced via relocation by other |
1716 | // sections. Conservatively make it linker retained. |
1717 | UsedVars.push_back(x: VNodesVar); |
1718 | } |
1719 | |
1720 | void InstrLowerer::emitNameData() { |
1721 | std::string UncompressedData; |
1722 | |
1723 | if (ReferencedNames.empty()) |
1724 | return; |
1725 | |
1726 | std::string CompressedNameStr; |
1727 | if (Error E = collectPGOFuncNameStrings(NameVars: ReferencedNames, Result&: CompressedNameStr, |
1728 | doCompression: DoInstrProfNameCompression)) { |
1729 | report_fatal_error(reason: Twine(toString(E: std::move(E))), gen_crash_diag: false); |
1730 | } |
1731 | |
1732 | auto &Ctx = M.getContext(); |
1733 | auto *NamesVal = |
1734 | ConstantDataArray::getString(Context&: Ctx, Initializer: StringRef(CompressedNameStr), AddNull: false); |
1735 | NamesVar = new GlobalVariable(M, NamesVal->getType(), true, |
1736 | GlobalValue::PrivateLinkage, NamesVal, |
1737 | getInstrProfNamesVarName()); |
1738 | NamesSize = CompressedNameStr.size(); |
1739 | setGlobalVariableLargeSection(TargetTriple: TT, GV&: *NamesVar); |
1740 | NamesVar->setSection( |
1741 | ProfileCorrelate == InstrProfCorrelator::BINARY |
1742 | ? getInstrProfSectionName(IPSK: IPSK_covname, OF: TT.getObjectFormat()) |
1743 | : getInstrProfSectionName(IPSK: IPSK_name, OF: TT.getObjectFormat())); |
1744 | // On COFF, it's important to reduce the alignment down to 1 to prevent the |
1745 | // linker from inserting padding before the start of the names section or |
1746 | // between names entries. |
1747 | NamesVar->setAlignment(Align(1)); |
1748 | // NamesVar is used by runtime but not referenced via relocation by other |
1749 | // sections. Conservatively make it linker retained. |
1750 | UsedVars.push_back(x: NamesVar); |
1751 | |
1752 | for (auto *NamePtr : ReferencedNames) |
1753 | NamePtr->eraseFromParent(); |
1754 | } |
1755 | |
1756 | void InstrLowerer::emitVTableNames() { |
1757 | if (!EnableVTableValueProfiling || ReferencedVTables.empty()) |
1758 | return; |
1759 | |
1760 | // Collect the PGO names of referenced vtables and compress them. |
1761 | std::string CompressedVTableNames; |
1762 | if (Error E = collectVTableStrings(VTables: ReferencedVTables, Result&: CompressedVTableNames, |
1763 | doCompression: DoInstrProfNameCompression)) { |
1764 | report_fatal_error(reason: Twine(toString(E: std::move(E))), gen_crash_diag: false); |
1765 | } |
1766 | |
1767 | auto &Ctx = M.getContext(); |
1768 | auto *VTableNamesVal = ConstantDataArray::getString( |
1769 | Context&: Ctx, Initializer: StringRef(CompressedVTableNames), AddNull: false /* AddNull */); |
1770 | GlobalVariable *VTableNamesVar = |
1771 | new GlobalVariable(M, VTableNamesVal->getType(), true /* constant */, |
1772 | GlobalValue::PrivateLinkage, VTableNamesVal, |
1773 | getInstrProfVTableNamesVarName()); |
1774 | VTableNamesVar->setSection( |
1775 | getInstrProfSectionName(IPSK: IPSK_vname, OF: TT.getObjectFormat())); |
1776 | VTableNamesVar->setAlignment(Align(1)); |
1777 | // Make VTableNames linker retained. |
1778 | UsedVars.push_back(x: VTableNamesVar); |
1779 | } |
1780 | |
1781 | void InstrLowerer::emitRegistration() { |
1782 | if (!needsRuntimeRegistrationOfSectionRange(TT)) |
1783 | return; |
1784 | |
1785 | // Construct the function. |
1786 | auto *VoidTy = Type::getVoidTy(C&: M.getContext()); |
1787 | auto *VoidPtrTy = PointerType::getUnqual(C&: M.getContext()); |
1788 | auto *Int64Ty = Type::getInt64Ty(C&: M.getContext()); |
1789 | auto *RegisterFTy = FunctionType::get(Result: VoidTy, isVarArg: false); |
1790 | auto *RegisterF = Function::Create(Ty: RegisterFTy, Linkage: GlobalValue::InternalLinkage, |
1791 | N: getInstrProfRegFuncsName(), M); |
1792 | RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); |
1793 | if (Options.NoRedZone) |
1794 | RegisterF->addFnAttr(Attribute::NoRedZone); |
1795 | |
1796 | auto *RuntimeRegisterTy = FunctionType::get(Result: VoidTy, Params: VoidPtrTy, isVarArg: false); |
1797 | auto *RuntimeRegisterF = |
1798 | Function::Create(Ty: RuntimeRegisterTy, Linkage: GlobalVariable::ExternalLinkage, |
1799 | N: getInstrProfRegFuncName(), M); |
1800 | |
1801 | IRBuilder<> IRB(BasicBlock::Create(Context&: M.getContext(), Name: "" , Parent: RegisterF)); |
1802 | for (Value *Data : CompilerUsedVars) |
1803 | if (!isa<Function>(Val: Data)) |
1804 | IRB.CreateCall(Callee: RuntimeRegisterF, Args: Data); |
1805 | for (Value *Data : UsedVars) |
1806 | if (Data != NamesVar && !isa<Function>(Val: Data)) |
1807 | IRB.CreateCall(Callee: RuntimeRegisterF, Args: Data); |
1808 | |
1809 | if (NamesVar) { |
1810 | Type *ParamTypes[] = {VoidPtrTy, Int64Ty}; |
1811 | auto *NamesRegisterTy = |
1812 | FunctionType::get(Result: VoidTy, Params: ArrayRef(ParamTypes), isVarArg: false); |
1813 | auto *NamesRegisterF = |
1814 | Function::Create(Ty: NamesRegisterTy, Linkage: GlobalVariable::ExternalLinkage, |
1815 | N: getInstrProfNamesRegFuncName(), M); |
1816 | IRB.CreateCall(Callee: NamesRegisterF, Args: {NamesVar, IRB.getInt64(C: NamesSize)}); |
1817 | } |
1818 | |
1819 | IRB.CreateRetVoid(); |
1820 | } |
1821 | |
1822 | bool InstrLowerer::emitRuntimeHook() { |
1823 | // We expect the linker to be invoked with -u<hook_var> flag for Linux |
1824 | // in which case there is no need to emit the external variable. |
1825 | if (TT.isOSLinux() || TT.isOSAIX()) |
1826 | return false; |
1827 | |
1828 | // If the module's provided its own runtime, we don't need to do anything. |
1829 | if (M.getGlobalVariable(Name: getInstrProfRuntimeHookVarName())) |
1830 | return false; |
1831 | |
1832 | // Declare an external variable that will pull in the runtime initialization. |
1833 | auto *Int32Ty = Type::getInt32Ty(C&: M.getContext()); |
1834 | auto *Var = |
1835 | new GlobalVariable(M, Int32Ty, false, GlobalValue::ExternalLinkage, |
1836 | nullptr, getInstrProfRuntimeHookVarName()); |
1837 | Var->setVisibility(GlobalValue::HiddenVisibility); |
1838 | |
1839 | if (TT.isOSBinFormatELF() && !TT.isPS()) { |
1840 | // Mark the user variable as used so that it isn't stripped out. |
1841 | CompilerUsedVars.push_back(x: Var); |
1842 | } else { |
1843 | // Make a function that uses it. |
1844 | auto *User = Function::Create(Ty: FunctionType::get(Result: Int32Ty, isVarArg: false), |
1845 | Linkage: GlobalValue::LinkOnceODRLinkage, |
1846 | N: getInstrProfRuntimeHookVarUseFuncName(), M); |
1847 | User->addFnAttr(Attribute::NoInline); |
1848 | if (Options.NoRedZone) |
1849 | User->addFnAttr(Attribute::NoRedZone); |
1850 | User->setVisibility(GlobalValue::HiddenVisibility); |
1851 | if (TT.supportsCOMDAT()) |
1852 | User->setComdat(M.getOrInsertComdat(Name: User->getName())); |
1853 | |
1854 | IRBuilder<> IRB(BasicBlock::Create(Context&: M.getContext(), Name: "" , Parent: User)); |
1855 | auto *Load = IRB.CreateLoad(Ty: Int32Ty, Ptr: Var); |
1856 | IRB.CreateRet(V: Load); |
1857 | |
1858 | // Mark the function as used so that it isn't stripped out. |
1859 | CompilerUsedVars.push_back(x: User); |
1860 | } |
1861 | return true; |
1862 | } |
1863 | |
1864 | void InstrLowerer::emitUses() { |
1865 | // The metadata sections are parallel arrays. Optimizers (e.g. |
1866 | // GlobalOpt/ConstantMerge) may not discard associated sections as a unit, so |
1867 | // we conservatively retain all unconditionally in the compiler. |
1868 | // |
1869 | // On ELF and Mach-O, the linker can guarantee the associated sections will be |
1870 | // retained or discarded as a unit, so llvm.compiler.used is sufficient. |
1871 | // Similarly on COFF, if prof data is not referenced by code we use one comdat |
1872 | // and ensure this GC property as well. Otherwise, we have to conservatively |
1873 | // make all of the sections retained by the linker. |
1874 | if (TT.isOSBinFormatELF() || TT.isOSBinFormatMachO() || |
1875 | (TT.isOSBinFormatCOFF() && !DataReferencedByCode)) |
1876 | appendToCompilerUsed(M, Values: CompilerUsedVars); |
1877 | else |
1878 | appendToUsed(M, Values: CompilerUsedVars); |
1879 | |
1880 | // We do not add proper references from used metadata sections to NamesVar and |
1881 | // VNodesVar, so we have to be conservative and place them in llvm.used |
1882 | // regardless of the target, |
1883 | appendToUsed(M, Values: UsedVars); |
1884 | } |
1885 | |
1886 | void InstrLowerer::emitInitialization() { |
1887 | // Create ProfileFileName variable. Don't don't this for the |
1888 | // context-sensitive instrumentation lowering: This lowering is after |
1889 | // LTO/ThinLTO linking. Pass PGOInstrumentationGenCreateVar should |
1890 | // have already create the variable before LTO/ThinLTO linking. |
1891 | if (!IsCS) |
1892 | createProfileFileNameVar(M, InstrProfileOutput: Options.InstrProfileOutput); |
1893 | Function *RegisterF = M.getFunction(Name: getInstrProfRegFuncsName()); |
1894 | if (!RegisterF) |
1895 | return; |
1896 | |
1897 | // Create the initialization function. |
1898 | auto *VoidTy = Type::getVoidTy(C&: M.getContext()); |
1899 | auto *F = Function::Create(Ty: FunctionType::get(Result: VoidTy, isVarArg: false), |
1900 | Linkage: GlobalValue::InternalLinkage, |
1901 | N: getInstrProfInitFuncName(), M); |
1902 | F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); |
1903 | F->addFnAttr(Attribute::NoInline); |
1904 | if (Options.NoRedZone) |
1905 | F->addFnAttr(Attribute::NoRedZone); |
1906 | |
1907 | // Add the basic block and the necessary calls. |
1908 | IRBuilder<> IRB(BasicBlock::Create(Context&: M.getContext(), Name: "" , Parent: F)); |
1909 | IRB.CreateCall(Callee: RegisterF, Args: {}); |
1910 | IRB.CreateRetVoid(); |
1911 | |
1912 | appendToGlobalCtors(M, F, Priority: 0); |
1913 | } |
1914 | |