1 | //===- InlineAdvisor.cpp - analysis pass implementation -------------------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file implements InlineAdvisorAnalysis and DefaultInlineAdvisor, and |
10 | // related types. |
11 | // |
12 | //===----------------------------------------------------------------------===// |
13 | |
14 | #include "llvm/Analysis/InlineAdvisor.h" |
15 | #include "llvm/ADT/Statistic.h" |
16 | #include "llvm/ADT/StringExtras.h" |
17 | #include "llvm/Analysis/AssumptionCache.h" |
18 | #include "llvm/Analysis/InlineCost.h" |
19 | #include "llvm/Analysis/OptimizationRemarkEmitter.h" |
20 | #include "llvm/Analysis/ProfileSummaryInfo.h" |
21 | #include "llvm/Analysis/ReplayInlineAdvisor.h" |
22 | #include "llvm/Analysis/TargetLibraryInfo.h" |
23 | #include "llvm/Analysis/TargetTransformInfo.h" |
24 | #include "llvm/Analysis/Utils/ImportedFunctionsInliningStatistics.h" |
25 | #include "llvm/IR/DebugInfoMetadata.h" |
26 | #include "llvm/IR/PassManager.h" |
27 | #include "llvm/Support/CommandLine.h" |
28 | #include "llvm/Support/raw_ostream.h" |
29 | |
30 | using namespace llvm; |
31 | #define DEBUG_TYPE "inline" |
32 | #ifdef LLVM_HAVE_TF_AOT_INLINERSIZEMODEL |
33 | #define LLVM_HAVE_TF_AOT |
34 | #endif |
35 | |
36 | // This weirdly named statistic tracks the number of times that, when attempting |
37 | // to inline a function A into B, we analyze the callers of B in order to see |
38 | // if those would be more profitable and blocked inline steps. |
39 | STATISTIC(NumCallerCallersAnalyzed, "Number of caller-callers analyzed" ); |
40 | |
41 | /// Flag to add inline messages as callsite attributes 'inline-remark'. |
42 | static cl::opt<bool> |
43 | ("inline-remark-attribute" , cl::init(Val: false), |
44 | cl::Hidden, |
45 | cl::desc("Enable adding inline-remark attribute to" |
46 | " callsites processed by inliner but decided" |
47 | " to be not inlined" )); |
48 | |
49 | static cl::opt<bool> EnableInlineDeferral("inline-deferral" , cl::init(Val: false), |
50 | cl::Hidden, |
51 | cl::desc("Enable deferred inlining" )); |
52 | |
53 | // An integer used to limit the cost of inline deferral. The default negative |
54 | // number tells shouldBeDeferred to only take the secondary cost into account. |
55 | static cl::opt<int> |
56 | InlineDeferralScale("inline-deferral-scale" , |
57 | cl::desc("Scale to limit the cost of inline deferral" ), |
58 | cl::init(Val: 2), cl::Hidden); |
59 | |
60 | static cl::opt<bool> |
61 | AnnotateInlinePhase("annotate-inline-phase" , cl::Hidden, cl::init(Val: false), |
62 | cl::desc("If true, annotate inline advisor remarks " |
63 | "with LTO and pass information." )); |
64 | |
65 | namespace llvm { |
66 | extern cl::opt<InlinerFunctionImportStatsOpts> InlinerFunctionImportStats; |
67 | } |
68 | |
69 | namespace { |
70 | using namespace llvm::ore; |
71 | class MandatoryInlineAdvice : public InlineAdvice { |
72 | public: |
73 | MandatoryInlineAdvice(InlineAdvisor *Advisor, CallBase &CB, |
74 | OptimizationRemarkEmitter &ORE, |
75 | bool IsInliningMandatory) |
76 | : InlineAdvice(Advisor, CB, ORE, IsInliningMandatory) {} |
77 | |
78 | private: |
79 | void recordInliningWithCalleeDeletedImpl() override { recordInliningImpl(); } |
80 | |
81 | void recordInliningImpl() override { |
82 | if (IsInliningRecommended) |
83 | emitInlinedInto(ORE, DLoc, Block, Callee: *Callee, Caller: *Caller, IsMandatory: IsInliningRecommended, |
84 | ExtraContext: [&](OptimizationRemark &) { |
85 | Remark << ": always inline attribute" ; |
86 | }); |
87 | } |
88 | |
89 | void recordUnsuccessfulInliningImpl(const InlineResult &Result) override { |
90 | if (IsInliningRecommended) |
91 | ORE.emit(RemarkBuilder: [&]() { |
92 | return OptimizationRemarkMissed(Advisor->getAnnotatedInlinePassName(), |
93 | "NotInlined" , DLoc, Block) |
94 | << "'" << NV("Callee" , Callee) << "' is not AlwaysInline into '" |
95 | << NV("Caller" , Caller) |
96 | << "': " << NV("Reason" , Result.getFailureReason()); |
97 | }); |
98 | } |
99 | |
100 | void recordUnattemptedInliningImpl() override { |
101 | assert(!IsInliningRecommended && "Expected to attempt inlining" ); |
102 | } |
103 | }; |
104 | } // namespace |
105 | |
106 | void DefaultInlineAdvice::recordUnsuccessfulInliningImpl( |
107 | const InlineResult &Result) { |
108 | using namespace ore; |
109 | llvm::setInlineRemark(CB&: *OriginalCB, Message: std::string(Result.getFailureReason()) + |
110 | "; " + inlineCostStr(IC: *OIC)); |
111 | ORE.emit(RemarkBuilder: [&]() { |
112 | return OptimizationRemarkMissed(Advisor->getAnnotatedInlinePassName(), |
113 | "NotInlined" , DLoc, Block) |
114 | << "'" << NV("Callee" , Callee) << "' is not inlined into '" |
115 | << NV("Caller" , Caller) |
116 | << "': " << NV("Reason" , Result.getFailureReason()); |
117 | }); |
118 | } |
119 | |
120 | void DefaultInlineAdvice::recordInliningWithCalleeDeletedImpl() { |
121 | if (EmitRemarks) |
122 | emitInlinedIntoBasedOnCost(ORE, DLoc, Block, Callee: *Callee, Caller: *Caller, IC: *OIC, |
123 | /* ForProfileContext= */ false, |
124 | PassName: Advisor->getAnnotatedInlinePassName()); |
125 | } |
126 | |
127 | void DefaultInlineAdvice::recordInliningImpl() { |
128 | if (EmitRemarks) |
129 | emitInlinedIntoBasedOnCost(ORE, DLoc, Block, Callee: *Callee, Caller: *Caller, IC: *OIC, |
130 | /* ForProfileContext= */ false, |
131 | PassName: Advisor->getAnnotatedInlinePassName()); |
132 | } |
133 | |
134 | std::optional<llvm::InlineCost> static getDefaultInlineAdvice( |
135 | CallBase &CB, FunctionAnalysisManager &FAM, const InlineParams &Params) { |
136 | Function &Caller = *CB.getCaller(); |
137 | ProfileSummaryInfo *PSI = |
138 | FAM.getResult<ModuleAnalysisManagerFunctionProxy>(IR&: Caller) |
139 | .getCachedResult<ProfileSummaryAnalysis>( |
140 | IR&: *CB.getParent()->getParent()->getParent()); |
141 | |
142 | auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: Caller); |
143 | auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & { |
144 | return FAM.getResult<AssumptionAnalysis>(IR&: F); |
145 | }; |
146 | auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & { |
147 | return FAM.getResult<BlockFrequencyAnalysis>(IR&: F); |
148 | }; |
149 | auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & { |
150 | return FAM.getResult<TargetLibraryAnalysis>(IR&: F); |
151 | }; |
152 | |
153 | auto GetInlineCost = [&](CallBase &CB) { |
154 | Function &Callee = *CB.getCalledFunction(); |
155 | auto &CalleeTTI = FAM.getResult<TargetIRAnalysis>(IR&: Callee); |
156 | bool = |
157 | Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled( |
158 | DEBUG_TYPE); |
159 | return getInlineCost(Call&: CB, Params, CalleeTTI, GetAssumptionCache, GetTLI, |
160 | GetBFI, PSI, ORE: RemarksEnabled ? &ORE : nullptr); |
161 | }; |
162 | return llvm::shouldInline( |
163 | CB, GetInlineCost, ORE, |
164 | EnableDeferral: Params.EnableDeferral.value_or(u&: EnableInlineDeferral)); |
165 | } |
166 | |
167 | std::unique_ptr<InlineAdvice> |
168 | DefaultInlineAdvisor::getAdviceImpl(CallBase &CB) { |
169 | auto OIC = getDefaultInlineAdvice(CB, FAM, Params); |
170 | return std::make_unique<DefaultInlineAdvice>( |
171 | args: this, args&: CB, args&: OIC, |
172 | args&: FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: *CB.getCaller())); |
173 | } |
174 | |
175 | InlineAdvice::(InlineAdvisor *Advisor, CallBase &CB, |
176 | OptimizationRemarkEmitter &ORE, |
177 | bool IsInliningRecommended) |
178 | : Advisor(Advisor), Caller(CB.getCaller()), Callee(CB.getCalledFunction()), |
179 | DLoc(CB.getDebugLoc()), Block(CB.getParent()), ORE(ORE), |
180 | IsInliningRecommended(IsInliningRecommended) {} |
181 | |
182 | void InlineAdvice::recordInlineStatsIfNeeded() { |
183 | if (Advisor->ImportedFunctionsStats) |
184 | Advisor->ImportedFunctionsStats->recordInline(Caller: *Caller, Callee: *Callee); |
185 | } |
186 | |
187 | void InlineAdvice::recordInlining() { |
188 | markRecorded(); |
189 | recordInlineStatsIfNeeded(); |
190 | recordInliningImpl(); |
191 | } |
192 | |
193 | void InlineAdvice::recordInliningWithCalleeDeleted() { |
194 | markRecorded(); |
195 | recordInlineStatsIfNeeded(); |
196 | recordInliningWithCalleeDeletedImpl(); |
197 | } |
198 | |
199 | AnalysisKey InlineAdvisorAnalysis::Key; |
200 | AnalysisKey PluginInlineAdvisorAnalysis::Key; |
201 | bool PluginInlineAdvisorAnalysis::HasBeenRegistered = false; |
202 | |
203 | bool InlineAdvisorAnalysis::Result::tryCreate( |
204 | InlineParams Params, InliningAdvisorMode Mode, |
205 | const ReplayInlinerSettings &ReplaySettings, InlineContext IC) { |
206 | auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager(); |
207 | if (PluginInlineAdvisorAnalysis::HasBeenRegistered) { |
208 | auto &DA = MAM.getResult<PluginInlineAdvisorAnalysis>(IR&: M); |
209 | Advisor.reset(p: DA.Factory(M, FAM, Params, IC)); |
210 | return !!Advisor; |
211 | } |
212 | auto GetDefaultAdvice = [&FAM, Params](CallBase &CB) { |
213 | auto OIC = getDefaultInlineAdvice(CB, FAM, Params); |
214 | return OIC.has_value(); |
215 | }; |
216 | switch (Mode) { |
217 | case InliningAdvisorMode::Default: |
218 | LLVM_DEBUG(dbgs() << "Using default inliner heuristic.\n" ); |
219 | Advisor.reset(p: new DefaultInlineAdvisor(M, FAM, Params, IC)); |
220 | // Restrict replay to default advisor, ML advisors are stateful so |
221 | // replay will need augmentations to interleave with them correctly. |
222 | if (!ReplaySettings.ReplayFile.empty()) { |
223 | Advisor = llvm::getReplayInlineAdvisor(M, FAM, Context&: M.getContext(), |
224 | OriginalAdvisor: std::move(Advisor), ReplaySettings, |
225 | /* EmitRemarks =*/true, IC); |
226 | } |
227 | break; |
228 | case InliningAdvisorMode::Development: |
229 | #ifdef LLVM_HAVE_TFLITE |
230 | LLVM_DEBUG(dbgs() << "Using development-mode inliner policy.\n" ); |
231 | Advisor = llvm::getDevelopmentModeAdvisor(M, MAM, GetDefaultAdvice); |
232 | #endif |
233 | break; |
234 | case InliningAdvisorMode::Release: |
235 | LLVM_DEBUG(dbgs() << "Using release-mode inliner policy.\n" ); |
236 | Advisor = llvm::getReleaseModeAdvisor(M, MAM, GetDefaultAdvice); |
237 | break; |
238 | } |
239 | |
240 | return !!Advisor; |
241 | } |
242 | |
243 | /// Return true if inlining of CB can block the caller from being |
244 | /// inlined which is proved to be more beneficial. \p IC is the |
245 | /// estimated inline cost associated with callsite \p CB. |
246 | /// \p TotalSecondaryCost will be set to the estimated cost of inlining the |
247 | /// caller if \p CB is suppressed for inlining. |
248 | static bool |
249 | shouldBeDeferred(Function *Caller, InlineCost IC, int &TotalSecondaryCost, |
250 | function_ref<InlineCost(CallBase &CB)> GetInlineCost) { |
251 | // For now we only handle local or inline functions. |
252 | if (!Caller->hasLocalLinkage() && !Caller->hasLinkOnceODRLinkage()) |
253 | return false; |
254 | // If the cost of inlining CB is non-positive, it is not going to prevent the |
255 | // caller from being inlined into its callers and hence we don't need to |
256 | // defer. |
257 | if (IC.getCost() <= 0) |
258 | return false; |
259 | // Try to detect the case where the current inlining candidate caller (call |
260 | // it B) is a static or linkonce-ODR function and is an inlining candidate |
261 | // elsewhere, and the current candidate callee (call it C) is large enough |
262 | // that inlining it into B would make B too big to inline later. In these |
263 | // circumstances it may be best not to inline C into B, but to inline B into |
264 | // its callers. |
265 | // |
266 | // This only applies to static and linkonce-ODR functions because those are |
267 | // expected to be available for inlining in the translation units where they |
268 | // are used. Thus we will always have the opportunity to make local inlining |
269 | // decisions. Importantly the linkonce-ODR linkage covers inline functions |
270 | // and templates in C++. |
271 | // |
272 | // FIXME: All of this logic should be sunk into getInlineCost. It relies on |
273 | // the internal implementation of the inline cost metrics rather than |
274 | // treating them as truly abstract units etc. |
275 | TotalSecondaryCost = 0; |
276 | // The candidate cost to be imposed upon the current function. |
277 | int CandidateCost = IC.getCost() - 1; |
278 | // If the caller has local linkage and can be inlined to all its callers, we |
279 | // can apply a huge negative bonus to TotalSecondaryCost. |
280 | bool ApplyLastCallBonus = Caller->hasLocalLinkage() && !Caller->hasOneUse(); |
281 | // This bool tracks what happens if we DO inline C into B. |
282 | bool InliningPreventsSomeOuterInline = false; |
283 | unsigned NumCallerUsers = 0; |
284 | for (User *U : Caller->users()) { |
285 | CallBase *CS2 = dyn_cast<CallBase>(Val: U); |
286 | |
287 | // If this isn't a call to Caller (it could be some other sort |
288 | // of reference) skip it. Such references will prevent the caller |
289 | // from being removed. |
290 | if (!CS2 || CS2->getCalledFunction() != Caller) { |
291 | ApplyLastCallBonus = false; |
292 | continue; |
293 | } |
294 | |
295 | InlineCost IC2 = GetInlineCost(*CS2); |
296 | ++NumCallerCallersAnalyzed; |
297 | if (!IC2) { |
298 | ApplyLastCallBonus = false; |
299 | continue; |
300 | } |
301 | if (IC2.isAlways()) |
302 | continue; |
303 | |
304 | // See if inlining of the original callsite would erase the cost delta of |
305 | // this callsite. We subtract off the penalty for the call instruction, |
306 | // which we would be deleting. |
307 | if (IC2.getCostDelta() <= CandidateCost) { |
308 | InliningPreventsSomeOuterInline = true; |
309 | TotalSecondaryCost += IC2.getCost(); |
310 | NumCallerUsers++; |
311 | } |
312 | } |
313 | |
314 | if (!InliningPreventsSomeOuterInline) |
315 | return false; |
316 | |
317 | // If all outer calls to Caller would get inlined, the cost for the last |
318 | // one is set very low by getInlineCost, in anticipation that Caller will |
319 | // be removed entirely. We did not account for this above unless there |
320 | // is only one caller of Caller. |
321 | if (ApplyLastCallBonus) |
322 | TotalSecondaryCost -= InlineConstants::LastCallToStaticBonus; |
323 | |
324 | // If InlineDeferralScale is negative, then ignore the cost of primary |
325 | // inlining -- IC.getCost() multiplied by the number of callers to Caller. |
326 | if (InlineDeferralScale < 0) |
327 | return TotalSecondaryCost < IC.getCost(); |
328 | |
329 | int TotalCost = TotalSecondaryCost + IC.getCost() * NumCallerUsers; |
330 | int Allowance = IC.getCost() * InlineDeferralScale; |
331 | return TotalCost < Allowance; |
332 | } |
333 | |
334 | namespace llvm { |
335 | static raw_ostream &operator<<(raw_ostream &R, const ore::NV &Arg) { |
336 | return R << Arg.Val; |
337 | } |
338 | |
339 | template <class RemarkT> |
340 | RemarkT &operator<<(RemarkT &&R, const InlineCost &IC) { |
341 | using namespace ore; |
342 | if (IC.isAlways()) { |
343 | R << "(cost=always)" ; |
344 | } else if (IC.isNever()) { |
345 | R << "(cost=never)" ; |
346 | } else { |
347 | R << "(cost=" << ore::NV("Cost" , IC.getCost()) |
348 | << ", threshold=" << ore::NV("Threshold" , IC.getThreshold()) << ")" ; |
349 | } |
350 | if (const char *Reason = IC.getReason()) |
351 | R << ": " << ore::NV("Reason" , Reason); |
352 | return R; |
353 | } |
354 | } // namespace llvm |
355 | |
356 | std::string llvm::inlineCostStr(const InlineCost &IC) { |
357 | std::string Buffer; |
358 | raw_string_ostream (Buffer); |
359 | Remark << IC; |
360 | return Remark.str(); |
361 | } |
362 | |
363 | void llvm::(CallBase &CB, StringRef Message) { |
364 | if (!InlineRemarkAttribute) |
365 | return; |
366 | |
367 | Attribute Attr = Attribute::get(Context&: CB.getContext(), Kind: "inline-remark" , Val: Message); |
368 | CB.addFnAttr(Attr); |
369 | } |
370 | |
371 | /// Return the cost only if the inliner should attempt to inline at the given |
372 | /// CallSite. If we return the cost, we will emit an optimisation remark later |
373 | /// using that cost, so we won't do so from this function. Return std::nullopt |
374 | /// if inlining should not be attempted. |
375 | std::optional<InlineCost> |
376 | llvm::(CallBase &CB, |
377 | function_ref<InlineCost(CallBase &CB)> GetInlineCost, |
378 | OptimizationRemarkEmitter &ORE, bool EnableDeferral) { |
379 | using namespace ore; |
380 | |
381 | InlineCost IC = GetInlineCost(CB); |
382 | Instruction *Call = &CB; |
383 | Function *Callee = CB.getCalledFunction(); |
384 | Function *Caller = CB.getCaller(); |
385 | |
386 | if (IC.isAlways()) { |
387 | LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC) |
388 | << ", Call: " << CB << "\n" ); |
389 | return IC; |
390 | } |
391 | |
392 | if (!IC) { |
393 | LLVM_DEBUG(dbgs() << " NOT Inlining " << inlineCostStr(IC) |
394 | << ", Call: " << CB << "\n" ); |
395 | if (IC.isNever()) { |
396 | ORE.emit(RemarkBuilder: [&]() { |
397 | return OptimizationRemarkMissed(DEBUG_TYPE, "NeverInline" , Call) |
398 | << "'" << NV("Callee" , Callee) << "' not inlined into '" |
399 | << NV("Caller" , Caller) |
400 | << "' because it should never be inlined " << IC; |
401 | }); |
402 | } else { |
403 | ORE.emit(RemarkBuilder: [&]() { |
404 | return OptimizationRemarkMissed(DEBUG_TYPE, "TooCostly" , Call) |
405 | << "'" << NV("Callee" , Callee) << "' not inlined into '" |
406 | << NV("Caller" , Caller) << "' because too costly to inline " |
407 | << IC; |
408 | }); |
409 | } |
410 | setInlineRemark(CB, Message: inlineCostStr(IC)); |
411 | return std::nullopt; |
412 | } |
413 | |
414 | int TotalSecondaryCost = 0; |
415 | if (EnableDeferral && |
416 | shouldBeDeferred(Caller, IC, TotalSecondaryCost, GetInlineCost)) { |
417 | LLVM_DEBUG(dbgs() << " NOT Inlining: " << CB |
418 | << " Cost = " << IC.getCost() |
419 | << ", outer Cost = " << TotalSecondaryCost << '\n'); |
420 | ORE.emit(RemarkBuilder: [&]() { |
421 | return OptimizationRemarkMissed(DEBUG_TYPE, "IncreaseCostInOtherContexts" , |
422 | Call) |
423 | << "Not inlining. Cost of inlining '" << NV("Callee" , Callee) |
424 | << "' increases the cost of inlining '" << NV("Caller" , Caller) |
425 | << "' in other contexts" ; |
426 | }); |
427 | setInlineRemark(CB, Message: "deferred" ); |
428 | return std::nullopt; |
429 | } |
430 | |
431 | LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC) << ", Call: " << CB |
432 | << '\n'); |
433 | return IC; |
434 | } |
435 | |
436 | std::string llvm::formatCallSiteLocation(DebugLoc DLoc, |
437 | const CallSiteFormat &Format) { |
438 | std::string Buffer; |
439 | raw_string_ostream CallSiteLoc(Buffer); |
440 | bool First = true; |
441 | for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) { |
442 | if (!First) |
443 | CallSiteLoc << " @ " ; |
444 | // Note that negative line offset is actually possible, but we use |
445 | // unsigned int to match line offset representation in remarks so |
446 | // it's directly consumable by relay advisor. |
447 | uint32_t Offset = |
448 | DIL->getLine() - DIL->getScope()->getSubprogram()->getLine(); |
449 | uint32_t Discriminator = DIL->getBaseDiscriminator(); |
450 | StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName(); |
451 | if (Name.empty()) |
452 | Name = DIL->getScope()->getSubprogram()->getName(); |
453 | CallSiteLoc << Name.str() << ":" << llvm::utostr(X: Offset); |
454 | if (Format.outputColumn()) |
455 | CallSiteLoc << ":" << llvm::utostr(X: DIL->getColumn()); |
456 | if (Format.outputDiscriminator() && Discriminator) |
457 | CallSiteLoc << "." << llvm::utostr(X: Discriminator); |
458 | First = false; |
459 | } |
460 | |
461 | return CallSiteLoc.str(); |
462 | } |
463 | |
464 | void llvm::(OptimizationRemark &, DebugLoc DLoc) { |
465 | if (!DLoc) { |
466 | return; |
467 | } |
468 | |
469 | bool First = true; |
470 | Remark << " at callsite " ; |
471 | for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) { |
472 | if (!First) |
473 | Remark << " @ " ; |
474 | unsigned int Offset = DIL->getLine(); |
475 | Offset -= DIL->getScope()->getSubprogram()->getLine(); |
476 | unsigned int Discriminator = DIL->getBaseDiscriminator(); |
477 | StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName(); |
478 | if (Name.empty()) |
479 | Name = DIL->getScope()->getSubprogram()->getName(); |
480 | Remark << Name << ":" << ore::NV("Line" , Offset) << ":" |
481 | << ore::NV("Column" , DIL->getColumn()); |
482 | if (Discriminator) |
483 | Remark << "." << ore::NV("Disc" , Discriminator); |
484 | First = false; |
485 | } |
486 | |
487 | Remark << ";" ; |
488 | } |
489 | |
490 | void llvm::( |
491 | OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block, |
492 | const Function &Callee, const Function &Caller, bool AlwaysInline, |
493 | function_ref<void(OptimizationRemark &)> , |
494 | const char *PassName) { |
495 | ORE.emit(RemarkBuilder: [&]() { |
496 | StringRef = AlwaysInline ? "AlwaysInline" : "Inlined" ; |
497 | OptimizationRemark (PassName ? PassName : DEBUG_TYPE, RemarkName, |
498 | DLoc, Block); |
499 | Remark << "'" << ore::NV("Callee" , &Callee) << "' inlined into '" |
500 | << ore::NV("Caller" , &Caller) << "'" ; |
501 | if (ExtraContext) |
502 | ExtraContext(Remark); |
503 | addLocationToRemarks(Remark, DLoc); |
504 | return Remark; |
505 | }); |
506 | } |
507 | |
508 | void llvm::( |
509 | OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block, |
510 | const Function &Callee, const Function &Caller, const InlineCost &IC, |
511 | bool ForProfileContext, const char *PassName) { |
512 | llvm::emitInlinedInto( |
513 | ORE, DLoc, Block, Callee, Caller, AlwaysInline: IC.isAlways(), |
514 | ExtraContext: [&](OptimizationRemark &) { |
515 | if (ForProfileContext) |
516 | Remark << " to match profiling context" ; |
517 | Remark << " with " << IC; |
518 | }, |
519 | PassName); |
520 | } |
521 | |
522 | InlineAdvisor::InlineAdvisor(Module &M, FunctionAnalysisManager &FAM, |
523 | std::optional<InlineContext> IC) |
524 | : M(M), FAM(FAM), IC(IC), |
525 | AnnotatedInlinePassName((IC && AnnotateInlinePhase) |
526 | ? llvm::AnnotateInlinePassName(IC: *IC) |
527 | : DEBUG_TYPE) { |
528 | if (InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No) { |
529 | ImportedFunctionsStats = |
530 | std::make_unique<ImportedFunctionsInliningStatistics>(); |
531 | ImportedFunctionsStats->setModuleInfo(M); |
532 | } |
533 | } |
534 | |
535 | InlineAdvisor::~InlineAdvisor() { |
536 | if (ImportedFunctionsStats) { |
537 | assert(InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No); |
538 | ImportedFunctionsStats->dump(Verbose: InlinerFunctionImportStats == |
539 | InlinerFunctionImportStatsOpts::Verbose); |
540 | } |
541 | } |
542 | |
543 | std::unique_ptr<InlineAdvice> InlineAdvisor::getMandatoryAdvice(CallBase &CB, |
544 | bool Advice) { |
545 | return std::make_unique<MandatoryInlineAdvice>(args: this, args&: CB, args&: getCallerORE(CB), |
546 | args&: Advice); |
547 | } |
548 | |
549 | static inline const char *getLTOPhase(ThinOrFullLTOPhase LTOPhase) { |
550 | switch (LTOPhase) { |
551 | case (ThinOrFullLTOPhase::None): |
552 | return "main" ; |
553 | case (ThinOrFullLTOPhase::ThinLTOPreLink): |
554 | case (ThinOrFullLTOPhase::FullLTOPreLink): |
555 | return "prelink" ; |
556 | case (ThinOrFullLTOPhase::ThinLTOPostLink): |
557 | case (ThinOrFullLTOPhase::FullLTOPostLink): |
558 | return "postlink" ; |
559 | } |
560 | llvm_unreachable("unreachable" ); |
561 | } |
562 | |
563 | static inline const char *getInlineAdvisorContext(InlinePass IP) { |
564 | switch (IP) { |
565 | case (InlinePass::AlwaysInliner): |
566 | return "always-inline" ; |
567 | case (InlinePass::CGSCCInliner): |
568 | return "cgscc-inline" ; |
569 | case (InlinePass::EarlyInliner): |
570 | return "early-inline" ; |
571 | case (InlinePass::MLInliner): |
572 | return "ml-inline" ; |
573 | case (InlinePass::ModuleInliner): |
574 | return "module-inline" ; |
575 | case (InlinePass::ReplayCGSCCInliner): |
576 | return "replay-cgscc-inline" ; |
577 | case (InlinePass::ReplaySampleProfileInliner): |
578 | return "replay-sample-profile-inline" ; |
579 | case (InlinePass::SampleProfileInliner): |
580 | return "sample-profile-inline" ; |
581 | } |
582 | |
583 | llvm_unreachable("unreachable" ); |
584 | } |
585 | |
586 | std::string llvm::AnnotateInlinePassName(InlineContext IC) { |
587 | return std::string(getLTOPhase(LTOPhase: IC.LTOPhase)) + "-" + |
588 | std::string(getInlineAdvisorContext(IP: IC.Pass)); |
589 | } |
590 | |
591 | InlineAdvisor::MandatoryInliningKind |
592 | InlineAdvisor::getMandatoryKind(CallBase &CB, FunctionAnalysisManager &FAM, |
593 | OptimizationRemarkEmitter &ORE) { |
594 | auto &Callee = *CB.getCalledFunction(); |
595 | |
596 | auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & { |
597 | return FAM.getResult<TargetLibraryAnalysis>(IR&: F); |
598 | }; |
599 | |
600 | auto &TIR = FAM.getResult<TargetIRAnalysis>(IR&: Callee); |
601 | |
602 | auto TrivialDecision = |
603 | llvm::getAttributeBasedInliningDecision(Call&: CB, Callee: &Callee, CalleeTTI&: TIR, GetTLI); |
604 | |
605 | if (TrivialDecision) { |
606 | if (TrivialDecision->isSuccess()) |
607 | return MandatoryInliningKind::Always; |
608 | else |
609 | return MandatoryInliningKind::Never; |
610 | } |
611 | return MandatoryInliningKind::NotMandatory; |
612 | } |
613 | |
614 | std::unique_ptr<InlineAdvice> InlineAdvisor::getAdvice(CallBase &CB, |
615 | bool MandatoryOnly) { |
616 | if (!MandatoryOnly) |
617 | return getAdviceImpl(CB); |
618 | bool Advice = CB.getCaller() != CB.getCalledFunction() && |
619 | MandatoryInliningKind::Always == |
620 | getMandatoryKind(CB, FAM, ORE&: getCallerORE(CB)); |
621 | return getMandatoryAdvice(CB, Advice); |
622 | } |
623 | |
624 | OptimizationRemarkEmitter &InlineAdvisor::getCallerORE(CallBase &CB) { |
625 | return FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: *CB.getCaller()); |
626 | } |
627 | |
628 | PreservedAnalyses |
629 | InlineAdvisorAnalysisPrinterPass::run(Module &M, ModuleAnalysisManager &MAM) { |
630 | const auto *IA = MAM.getCachedResult<InlineAdvisorAnalysis>(IR&: M); |
631 | if (!IA) |
632 | OS << "No Inline Advisor\n" ; |
633 | else |
634 | IA->getAdvisor()->print(OS); |
635 | return PreservedAnalyses::all(); |
636 | } |
637 | |
638 | PreservedAnalyses InlineAdvisorAnalysisPrinterPass::run( |
639 | LazyCallGraph::SCC &InitialC, CGSCCAnalysisManager &AM, LazyCallGraph &CG, |
640 | CGSCCUpdateResult &UR) { |
641 | const auto &MAMProxy = |
642 | AM.getResult<ModuleAnalysisManagerCGSCCProxy>(IR&: InitialC, ExtraArgs&: CG); |
643 | |
644 | if (InitialC.size() == 0) { |
645 | OS << "SCC is empty!\n" ; |
646 | return PreservedAnalyses::all(); |
647 | } |
648 | Module &M = *InitialC.begin()->getFunction().getParent(); |
649 | const auto *IA = MAMProxy.getCachedResult<InlineAdvisorAnalysis>(IR&: M); |
650 | if (!IA) |
651 | OS << "No Inline Advisor\n" ; |
652 | else |
653 | IA->getAdvisor()->print(OS); |
654 | return PreservedAnalyses::all(); |
655 | } |
656 | |