1//===- InlineAdvisor.cpp - analysis pass implementation -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements InlineAdvisorAnalysis and DefaultInlineAdvisor, and
10// related types.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Analysis/InlineAdvisor.h"
15#include "llvm/ADT/Statistic.h"
16#include "llvm/ADT/StringExtras.h"
17#include "llvm/Analysis/AssumptionCache.h"
18#include "llvm/Analysis/InlineCost.h"
19#include "llvm/Analysis/OptimizationRemarkEmitter.h"
20#include "llvm/Analysis/ProfileSummaryInfo.h"
21#include "llvm/Analysis/ReplayInlineAdvisor.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Analysis/TargetTransformInfo.h"
24#include "llvm/Analysis/Utils/ImportedFunctionsInliningStatistics.h"
25#include "llvm/IR/DebugInfoMetadata.h"
26#include "llvm/IR/PassManager.h"
27#include "llvm/Support/CommandLine.h"
28#include "llvm/Support/raw_ostream.h"
29
30using namespace llvm;
31#define DEBUG_TYPE "inline"
32#ifdef LLVM_HAVE_TF_AOT_INLINERSIZEMODEL
33#define LLVM_HAVE_TF_AOT
34#endif
35
36// This weirdly named statistic tracks the number of times that, when attempting
37// to inline a function A into B, we analyze the callers of B in order to see
38// if those would be more profitable and blocked inline steps.
39STATISTIC(NumCallerCallersAnalyzed, "Number of caller-callers analyzed");
40
41/// Flag to add inline messages as callsite attributes 'inline-remark'.
42static cl::opt<bool>
43 InlineRemarkAttribute("inline-remark-attribute", cl::init(Val: false),
44 cl::Hidden,
45 cl::desc("Enable adding inline-remark attribute to"
46 " callsites processed by inliner but decided"
47 " to be not inlined"));
48
49static cl::opt<bool> EnableInlineDeferral("inline-deferral", cl::init(Val: false),
50 cl::Hidden,
51 cl::desc("Enable deferred inlining"));
52
53// An integer used to limit the cost of inline deferral. The default negative
54// number tells shouldBeDeferred to only take the secondary cost into account.
55static cl::opt<int>
56 InlineDeferralScale("inline-deferral-scale",
57 cl::desc("Scale to limit the cost of inline deferral"),
58 cl::init(Val: 2), cl::Hidden);
59
60static cl::opt<bool>
61 AnnotateInlinePhase("annotate-inline-phase", cl::Hidden, cl::init(Val: false),
62 cl::desc("If true, annotate inline advisor remarks "
63 "with LTO and pass information."));
64
65namespace llvm {
66extern cl::opt<InlinerFunctionImportStatsOpts> InlinerFunctionImportStats;
67}
68
69namespace {
70using namespace llvm::ore;
71class MandatoryInlineAdvice : public InlineAdvice {
72public:
73 MandatoryInlineAdvice(InlineAdvisor *Advisor, CallBase &CB,
74 OptimizationRemarkEmitter &ORE,
75 bool IsInliningMandatory)
76 : InlineAdvice(Advisor, CB, ORE, IsInliningMandatory) {}
77
78private:
79 void recordInliningWithCalleeDeletedImpl() override { recordInliningImpl(); }
80
81 void recordInliningImpl() override {
82 if (IsInliningRecommended)
83 emitInlinedInto(ORE, DLoc, Block, Callee: *Callee, Caller: *Caller, IsMandatory: IsInliningRecommended,
84 ExtraContext: [&](OptimizationRemark &Remark) {
85 Remark << ": always inline attribute";
86 });
87 }
88
89 void recordUnsuccessfulInliningImpl(const InlineResult &Result) override {
90 if (IsInliningRecommended)
91 ORE.emit(RemarkBuilder: [&]() {
92 return OptimizationRemarkMissed(Advisor->getAnnotatedInlinePassName(),
93 "NotInlined", DLoc, Block)
94 << "'" << NV("Callee", Callee) << "' is not AlwaysInline into '"
95 << NV("Caller", Caller)
96 << "': " << NV("Reason", Result.getFailureReason());
97 });
98 }
99
100 void recordUnattemptedInliningImpl() override {
101 assert(!IsInliningRecommended && "Expected to attempt inlining");
102 }
103};
104} // namespace
105
106void DefaultInlineAdvice::recordUnsuccessfulInliningImpl(
107 const InlineResult &Result) {
108 using namespace ore;
109 llvm::setInlineRemark(CB&: *OriginalCB, Message: std::string(Result.getFailureReason()) +
110 "; " + inlineCostStr(IC: *OIC));
111 ORE.emit(RemarkBuilder: [&]() {
112 return OptimizationRemarkMissed(Advisor->getAnnotatedInlinePassName(),
113 "NotInlined", DLoc, Block)
114 << "'" << NV("Callee", Callee) << "' is not inlined into '"
115 << NV("Caller", Caller)
116 << "': " << NV("Reason", Result.getFailureReason());
117 });
118}
119
120void DefaultInlineAdvice::recordInliningWithCalleeDeletedImpl() {
121 if (EmitRemarks)
122 emitInlinedIntoBasedOnCost(ORE, DLoc, Block, Callee: *Callee, Caller: *Caller, IC: *OIC,
123 /* ForProfileContext= */ false,
124 PassName: Advisor->getAnnotatedInlinePassName());
125}
126
127void DefaultInlineAdvice::recordInliningImpl() {
128 if (EmitRemarks)
129 emitInlinedIntoBasedOnCost(ORE, DLoc, Block, Callee: *Callee, Caller: *Caller, IC: *OIC,
130 /* ForProfileContext= */ false,
131 PassName: Advisor->getAnnotatedInlinePassName());
132}
133
134std::optional<llvm::InlineCost> static getDefaultInlineAdvice(
135 CallBase &CB, FunctionAnalysisManager &FAM, const InlineParams &Params) {
136 Function &Caller = *CB.getCaller();
137 ProfileSummaryInfo *PSI =
138 FAM.getResult<ModuleAnalysisManagerFunctionProxy>(IR&: Caller)
139 .getCachedResult<ProfileSummaryAnalysis>(
140 IR&: *CB.getParent()->getParent()->getParent());
141
142 auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: Caller);
143 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
144 return FAM.getResult<AssumptionAnalysis>(IR&: F);
145 };
146 auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & {
147 return FAM.getResult<BlockFrequencyAnalysis>(IR&: F);
148 };
149 auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
150 return FAM.getResult<TargetLibraryAnalysis>(IR&: F);
151 };
152
153 auto GetInlineCost = [&](CallBase &CB) {
154 Function &Callee = *CB.getCalledFunction();
155 auto &CalleeTTI = FAM.getResult<TargetIRAnalysis>(IR&: Callee);
156 bool RemarksEnabled =
157 Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled(
158 DEBUG_TYPE);
159 return getInlineCost(Call&: CB, Params, CalleeTTI, GetAssumptionCache, GetTLI,
160 GetBFI, PSI, ORE: RemarksEnabled ? &ORE : nullptr);
161 };
162 return llvm::shouldInline(
163 CB, GetInlineCost, ORE,
164 EnableDeferral: Params.EnableDeferral.value_or(u&: EnableInlineDeferral));
165}
166
167std::unique_ptr<InlineAdvice>
168DefaultInlineAdvisor::getAdviceImpl(CallBase &CB) {
169 auto OIC = getDefaultInlineAdvice(CB, FAM, Params);
170 return std::make_unique<DefaultInlineAdvice>(
171 args: this, args&: CB, args&: OIC,
172 args&: FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: *CB.getCaller()));
173}
174
175InlineAdvice::InlineAdvice(InlineAdvisor *Advisor, CallBase &CB,
176 OptimizationRemarkEmitter &ORE,
177 bool IsInliningRecommended)
178 : Advisor(Advisor), Caller(CB.getCaller()), Callee(CB.getCalledFunction()),
179 DLoc(CB.getDebugLoc()), Block(CB.getParent()), ORE(ORE),
180 IsInliningRecommended(IsInliningRecommended) {}
181
182void InlineAdvice::recordInlineStatsIfNeeded() {
183 if (Advisor->ImportedFunctionsStats)
184 Advisor->ImportedFunctionsStats->recordInline(Caller: *Caller, Callee: *Callee);
185}
186
187void InlineAdvice::recordInlining() {
188 markRecorded();
189 recordInlineStatsIfNeeded();
190 recordInliningImpl();
191}
192
193void InlineAdvice::recordInliningWithCalleeDeleted() {
194 markRecorded();
195 recordInlineStatsIfNeeded();
196 recordInliningWithCalleeDeletedImpl();
197}
198
199AnalysisKey InlineAdvisorAnalysis::Key;
200AnalysisKey PluginInlineAdvisorAnalysis::Key;
201bool PluginInlineAdvisorAnalysis::HasBeenRegistered = false;
202
203bool InlineAdvisorAnalysis::Result::tryCreate(
204 InlineParams Params, InliningAdvisorMode Mode,
205 const ReplayInlinerSettings &ReplaySettings, InlineContext IC) {
206 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
207 if (PluginInlineAdvisorAnalysis::HasBeenRegistered) {
208 auto &DA = MAM.getResult<PluginInlineAdvisorAnalysis>(IR&: M);
209 Advisor.reset(p: DA.Factory(M, FAM, Params, IC));
210 return !!Advisor;
211 }
212 auto GetDefaultAdvice = [&FAM, Params](CallBase &CB) {
213 auto OIC = getDefaultInlineAdvice(CB, FAM, Params);
214 return OIC.has_value();
215 };
216 switch (Mode) {
217 case InliningAdvisorMode::Default:
218 LLVM_DEBUG(dbgs() << "Using default inliner heuristic.\n");
219 Advisor.reset(p: new DefaultInlineAdvisor(M, FAM, Params, IC));
220 // Restrict replay to default advisor, ML advisors are stateful so
221 // replay will need augmentations to interleave with them correctly.
222 if (!ReplaySettings.ReplayFile.empty()) {
223 Advisor = llvm::getReplayInlineAdvisor(M, FAM, Context&: M.getContext(),
224 OriginalAdvisor: std::move(Advisor), ReplaySettings,
225 /* EmitRemarks =*/true, IC);
226 }
227 break;
228 case InliningAdvisorMode::Development:
229#ifdef LLVM_HAVE_TFLITE
230 LLVM_DEBUG(dbgs() << "Using development-mode inliner policy.\n");
231 Advisor = llvm::getDevelopmentModeAdvisor(M, MAM, GetDefaultAdvice);
232#endif
233 break;
234 case InliningAdvisorMode::Release:
235 LLVM_DEBUG(dbgs() << "Using release-mode inliner policy.\n");
236 Advisor = llvm::getReleaseModeAdvisor(M, MAM, GetDefaultAdvice);
237 break;
238 }
239
240 return !!Advisor;
241}
242
243/// Return true if inlining of CB can block the caller from being
244/// inlined which is proved to be more beneficial. \p IC is the
245/// estimated inline cost associated with callsite \p CB.
246/// \p TotalSecondaryCost will be set to the estimated cost of inlining the
247/// caller if \p CB is suppressed for inlining.
248static bool
249shouldBeDeferred(Function *Caller, InlineCost IC, int &TotalSecondaryCost,
250 function_ref<InlineCost(CallBase &CB)> GetInlineCost) {
251 // For now we only handle local or inline functions.
252 if (!Caller->hasLocalLinkage() && !Caller->hasLinkOnceODRLinkage())
253 return false;
254 // If the cost of inlining CB is non-positive, it is not going to prevent the
255 // caller from being inlined into its callers and hence we don't need to
256 // defer.
257 if (IC.getCost() <= 0)
258 return false;
259 // Try to detect the case where the current inlining candidate caller (call
260 // it B) is a static or linkonce-ODR function and is an inlining candidate
261 // elsewhere, and the current candidate callee (call it C) is large enough
262 // that inlining it into B would make B too big to inline later. In these
263 // circumstances it may be best not to inline C into B, but to inline B into
264 // its callers.
265 //
266 // This only applies to static and linkonce-ODR functions because those are
267 // expected to be available for inlining in the translation units where they
268 // are used. Thus we will always have the opportunity to make local inlining
269 // decisions. Importantly the linkonce-ODR linkage covers inline functions
270 // and templates in C++.
271 //
272 // FIXME: All of this logic should be sunk into getInlineCost. It relies on
273 // the internal implementation of the inline cost metrics rather than
274 // treating them as truly abstract units etc.
275 TotalSecondaryCost = 0;
276 // The candidate cost to be imposed upon the current function.
277 int CandidateCost = IC.getCost() - 1;
278 // If the caller has local linkage and can be inlined to all its callers, we
279 // can apply a huge negative bonus to TotalSecondaryCost.
280 bool ApplyLastCallBonus = Caller->hasLocalLinkage() && !Caller->hasOneUse();
281 // This bool tracks what happens if we DO inline C into B.
282 bool InliningPreventsSomeOuterInline = false;
283 unsigned NumCallerUsers = 0;
284 for (User *U : Caller->users()) {
285 CallBase *CS2 = dyn_cast<CallBase>(Val: U);
286
287 // If this isn't a call to Caller (it could be some other sort
288 // of reference) skip it. Such references will prevent the caller
289 // from being removed.
290 if (!CS2 || CS2->getCalledFunction() != Caller) {
291 ApplyLastCallBonus = false;
292 continue;
293 }
294
295 InlineCost IC2 = GetInlineCost(*CS2);
296 ++NumCallerCallersAnalyzed;
297 if (!IC2) {
298 ApplyLastCallBonus = false;
299 continue;
300 }
301 if (IC2.isAlways())
302 continue;
303
304 // See if inlining of the original callsite would erase the cost delta of
305 // this callsite. We subtract off the penalty for the call instruction,
306 // which we would be deleting.
307 if (IC2.getCostDelta() <= CandidateCost) {
308 InliningPreventsSomeOuterInline = true;
309 TotalSecondaryCost += IC2.getCost();
310 NumCallerUsers++;
311 }
312 }
313
314 if (!InliningPreventsSomeOuterInline)
315 return false;
316
317 // If all outer calls to Caller would get inlined, the cost for the last
318 // one is set very low by getInlineCost, in anticipation that Caller will
319 // be removed entirely. We did not account for this above unless there
320 // is only one caller of Caller.
321 if (ApplyLastCallBonus)
322 TotalSecondaryCost -= InlineConstants::LastCallToStaticBonus;
323
324 // If InlineDeferralScale is negative, then ignore the cost of primary
325 // inlining -- IC.getCost() multiplied by the number of callers to Caller.
326 if (InlineDeferralScale < 0)
327 return TotalSecondaryCost < IC.getCost();
328
329 int TotalCost = TotalSecondaryCost + IC.getCost() * NumCallerUsers;
330 int Allowance = IC.getCost() * InlineDeferralScale;
331 return TotalCost < Allowance;
332}
333
334namespace llvm {
335static raw_ostream &operator<<(raw_ostream &R, const ore::NV &Arg) {
336 return R << Arg.Val;
337}
338
339template <class RemarkT>
340RemarkT &operator<<(RemarkT &&R, const InlineCost &IC) {
341 using namespace ore;
342 if (IC.isAlways()) {
343 R << "(cost=always)";
344 } else if (IC.isNever()) {
345 R << "(cost=never)";
346 } else {
347 R << "(cost=" << ore::NV("Cost", IC.getCost())
348 << ", threshold=" << ore::NV("Threshold", IC.getThreshold()) << ")";
349 }
350 if (const char *Reason = IC.getReason())
351 R << ": " << ore::NV("Reason", Reason);
352 return R;
353}
354} // namespace llvm
355
356std::string llvm::inlineCostStr(const InlineCost &IC) {
357 std::string Buffer;
358 raw_string_ostream Remark(Buffer);
359 Remark << IC;
360 return Remark.str();
361}
362
363void llvm::setInlineRemark(CallBase &CB, StringRef Message) {
364 if (!InlineRemarkAttribute)
365 return;
366
367 Attribute Attr = Attribute::get(Context&: CB.getContext(), Kind: "inline-remark", Val: Message);
368 CB.addFnAttr(Attr);
369}
370
371/// Return the cost only if the inliner should attempt to inline at the given
372/// CallSite. If we return the cost, we will emit an optimisation remark later
373/// using that cost, so we won't do so from this function. Return std::nullopt
374/// if inlining should not be attempted.
375std::optional<InlineCost>
376llvm::shouldInline(CallBase &CB,
377 function_ref<InlineCost(CallBase &CB)> GetInlineCost,
378 OptimizationRemarkEmitter &ORE, bool EnableDeferral) {
379 using namespace ore;
380
381 InlineCost IC = GetInlineCost(CB);
382 Instruction *Call = &CB;
383 Function *Callee = CB.getCalledFunction();
384 Function *Caller = CB.getCaller();
385
386 if (IC.isAlways()) {
387 LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC)
388 << ", Call: " << CB << "\n");
389 return IC;
390 }
391
392 if (!IC) {
393 LLVM_DEBUG(dbgs() << " NOT Inlining " << inlineCostStr(IC)
394 << ", Call: " << CB << "\n");
395 if (IC.isNever()) {
396 ORE.emit(RemarkBuilder: [&]() {
397 return OptimizationRemarkMissed(DEBUG_TYPE, "NeverInline", Call)
398 << "'" << NV("Callee", Callee) << "' not inlined into '"
399 << NV("Caller", Caller)
400 << "' because it should never be inlined " << IC;
401 });
402 } else {
403 ORE.emit(RemarkBuilder: [&]() {
404 return OptimizationRemarkMissed(DEBUG_TYPE, "TooCostly", Call)
405 << "'" << NV("Callee", Callee) << "' not inlined into '"
406 << NV("Caller", Caller) << "' because too costly to inline "
407 << IC;
408 });
409 }
410 setInlineRemark(CB, Message: inlineCostStr(IC));
411 return std::nullopt;
412 }
413
414 int TotalSecondaryCost = 0;
415 if (EnableDeferral &&
416 shouldBeDeferred(Caller, IC, TotalSecondaryCost, GetInlineCost)) {
417 LLVM_DEBUG(dbgs() << " NOT Inlining: " << CB
418 << " Cost = " << IC.getCost()
419 << ", outer Cost = " << TotalSecondaryCost << '\n');
420 ORE.emit(RemarkBuilder: [&]() {
421 return OptimizationRemarkMissed(DEBUG_TYPE, "IncreaseCostInOtherContexts",
422 Call)
423 << "Not inlining. Cost of inlining '" << NV("Callee", Callee)
424 << "' increases the cost of inlining '" << NV("Caller", Caller)
425 << "' in other contexts";
426 });
427 setInlineRemark(CB, Message: "deferred");
428 return std::nullopt;
429 }
430
431 LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC) << ", Call: " << CB
432 << '\n');
433 return IC;
434}
435
436std::string llvm::formatCallSiteLocation(DebugLoc DLoc,
437 const CallSiteFormat &Format) {
438 std::string Buffer;
439 raw_string_ostream CallSiteLoc(Buffer);
440 bool First = true;
441 for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) {
442 if (!First)
443 CallSiteLoc << " @ ";
444 // Note that negative line offset is actually possible, but we use
445 // unsigned int to match line offset representation in remarks so
446 // it's directly consumable by relay advisor.
447 uint32_t Offset =
448 DIL->getLine() - DIL->getScope()->getSubprogram()->getLine();
449 uint32_t Discriminator = DIL->getBaseDiscriminator();
450 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
451 if (Name.empty())
452 Name = DIL->getScope()->getSubprogram()->getName();
453 CallSiteLoc << Name.str() << ":" << llvm::utostr(X: Offset);
454 if (Format.outputColumn())
455 CallSiteLoc << ":" << llvm::utostr(X: DIL->getColumn());
456 if (Format.outputDiscriminator() && Discriminator)
457 CallSiteLoc << "." << llvm::utostr(X: Discriminator);
458 First = false;
459 }
460
461 return CallSiteLoc.str();
462}
463
464void llvm::addLocationToRemarks(OptimizationRemark &Remark, DebugLoc DLoc) {
465 if (!DLoc) {
466 return;
467 }
468
469 bool First = true;
470 Remark << " at callsite ";
471 for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) {
472 if (!First)
473 Remark << " @ ";
474 unsigned int Offset = DIL->getLine();
475 Offset -= DIL->getScope()->getSubprogram()->getLine();
476 unsigned int Discriminator = DIL->getBaseDiscriminator();
477 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
478 if (Name.empty())
479 Name = DIL->getScope()->getSubprogram()->getName();
480 Remark << Name << ":" << ore::NV("Line", Offset) << ":"
481 << ore::NV("Column", DIL->getColumn());
482 if (Discriminator)
483 Remark << "." << ore::NV("Disc", Discriminator);
484 First = false;
485 }
486
487 Remark << ";";
488}
489
490void llvm::emitInlinedInto(
491 OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block,
492 const Function &Callee, const Function &Caller, bool AlwaysInline,
493 function_ref<void(OptimizationRemark &)> ExtraContext,
494 const char *PassName) {
495 ORE.emit(RemarkBuilder: [&]() {
496 StringRef RemarkName = AlwaysInline ? "AlwaysInline" : "Inlined";
497 OptimizationRemark Remark(PassName ? PassName : DEBUG_TYPE, RemarkName,
498 DLoc, Block);
499 Remark << "'" << ore::NV("Callee", &Callee) << "' inlined into '"
500 << ore::NV("Caller", &Caller) << "'";
501 if (ExtraContext)
502 ExtraContext(Remark);
503 addLocationToRemarks(Remark, DLoc);
504 return Remark;
505 });
506}
507
508void llvm::emitInlinedIntoBasedOnCost(
509 OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block,
510 const Function &Callee, const Function &Caller, const InlineCost &IC,
511 bool ForProfileContext, const char *PassName) {
512 llvm::emitInlinedInto(
513 ORE, DLoc, Block, Callee, Caller, AlwaysInline: IC.isAlways(),
514 ExtraContext: [&](OptimizationRemark &Remark) {
515 if (ForProfileContext)
516 Remark << " to match profiling context";
517 Remark << " with " << IC;
518 },
519 PassName);
520}
521
522InlineAdvisor::InlineAdvisor(Module &M, FunctionAnalysisManager &FAM,
523 std::optional<InlineContext> IC)
524 : M(M), FAM(FAM), IC(IC),
525 AnnotatedInlinePassName((IC && AnnotateInlinePhase)
526 ? llvm::AnnotateInlinePassName(IC: *IC)
527 : DEBUG_TYPE) {
528 if (InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No) {
529 ImportedFunctionsStats =
530 std::make_unique<ImportedFunctionsInliningStatistics>();
531 ImportedFunctionsStats->setModuleInfo(M);
532 }
533}
534
535InlineAdvisor::~InlineAdvisor() {
536 if (ImportedFunctionsStats) {
537 assert(InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No);
538 ImportedFunctionsStats->dump(Verbose: InlinerFunctionImportStats ==
539 InlinerFunctionImportStatsOpts::Verbose);
540 }
541}
542
543std::unique_ptr<InlineAdvice> InlineAdvisor::getMandatoryAdvice(CallBase &CB,
544 bool Advice) {
545 return std::make_unique<MandatoryInlineAdvice>(args: this, args&: CB, args&: getCallerORE(CB),
546 args&: Advice);
547}
548
549static inline const char *getLTOPhase(ThinOrFullLTOPhase LTOPhase) {
550 switch (LTOPhase) {
551 case (ThinOrFullLTOPhase::None):
552 return "main";
553 case (ThinOrFullLTOPhase::ThinLTOPreLink):
554 case (ThinOrFullLTOPhase::FullLTOPreLink):
555 return "prelink";
556 case (ThinOrFullLTOPhase::ThinLTOPostLink):
557 case (ThinOrFullLTOPhase::FullLTOPostLink):
558 return "postlink";
559 }
560 llvm_unreachable("unreachable");
561}
562
563static inline const char *getInlineAdvisorContext(InlinePass IP) {
564 switch (IP) {
565 case (InlinePass::AlwaysInliner):
566 return "always-inline";
567 case (InlinePass::CGSCCInliner):
568 return "cgscc-inline";
569 case (InlinePass::EarlyInliner):
570 return "early-inline";
571 case (InlinePass::MLInliner):
572 return "ml-inline";
573 case (InlinePass::ModuleInliner):
574 return "module-inline";
575 case (InlinePass::ReplayCGSCCInliner):
576 return "replay-cgscc-inline";
577 case (InlinePass::ReplaySampleProfileInliner):
578 return "replay-sample-profile-inline";
579 case (InlinePass::SampleProfileInliner):
580 return "sample-profile-inline";
581 }
582
583 llvm_unreachable("unreachable");
584}
585
586std::string llvm::AnnotateInlinePassName(InlineContext IC) {
587 return std::string(getLTOPhase(LTOPhase: IC.LTOPhase)) + "-" +
588 std::string(getInlineAdvisorContext(IP: IC.Pass));
589}
590
591InlineAdvisor::MandatoryInliningKind
592InlineAdvisor::getMandatoryKind(CallBase &CB, FunctionAnalysisManager &FAM,
593 OptimizationRemarkEmitter &ORE) {
594 auto &Callee = *CB.getCalledFunction();
595
596 auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
597 return FAM.getResult<TargetLibraryAnalysis>(IR&: F);
598 };
599
600 auto &TIR = FAM.getResult<TargetIRAnalysis>(IR&: Callee);
601
602 auto TrivialDecision =
603 llvm::getAttributeBasedInliningDecision(Call&: CB, Callee: &Callee, CalleeTTI&: TIR, GetTLI);
604
605 if (TrivialDecision) {
606 if (TrivialDecision->isSuccess())
607 return MandatoryInliningKind::Always;
608 else
609 return MandatoryInliningKind::Never;
610 }
611 return MandatoryInliningKind::NotMandatory;
612}
613
614std::unique_ptr<InlineAdvice> InlineAdvisor::getAdvice(CallBase &CB,
615 bool MandatoryOnly) {
616 if (!MandatoryOnly)
617 return getAdviceImpl(CB);
618 bool Advice = CB.getCaller() != CB.getCalledFunction() &&
619 MandatoryInliningKind::Always ==
620 getMandatoryKind(CB, FAM, ORE&: getCallerORE(CB));
621 return getMandatoryAdvice(CB, Advice);
622}
623
624OptimizationRemarkEmitter &InlineAdvisor::getCallerORE(CallBase &CB) {
625 return FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: *CB.getCaller());
626}
627
628PreservedAnalyses
629InlineAdvisorAnalysisPrinterPass::run(Module &M, ModuleAnalysisManager &MAM) {
630 const auto *IA = MAM.getCachedResult<InlineAdvisorAnalysis>(IR&: M);
631 if (!IA)
632 OS << "No Inline Advisor\n";
633 else
634 IA->getAdvisor()->print(OS);
635 return PreservedAnalyses::all();
636}
637
638PreservedAnalyses InlineAdvisorAnalysisPrinterPass::run(
639 LazyCallGraph::SCC &InitialC, CGSCCAnalysisManager &AM, LazyCallGraph &CG,
640 CGSCCUpdateResult &UR) {
641 const auto &MAMProxy =
642 AM.getResult<ModuleAnalysisManagerCGSCCProxy>(IR&: InitialC, ExtraArgs&: CG);
643
644 if (InitialC.size() == 0) {
645 OS << "SCC is empty!\n";
646 return PreservedAnalyses::all();
647 }
648 Module &M = *InitialC.begin()->getFunction().getParent();
649 const auto *IA = MAMProxy.getCachedResult<InlineAdvisorAnalysis>(IR&: M);
650 if (!IA)
651 OS << "No Inline Advisor\n";
652 else
653 IA->getAdvisor()->print(OS);
654 return PreservedAnalyses::all();
655}
656

source code of llvm/lib/Analysis/InlineAdvisor.cpp