1 | //===- InlineAdvisor.h - Inlining decision making abstraction -*- C++ ---*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | #ifndef LLVM_ANALYSIS_INLINEADVISOR_H |
10 | #define LLVM_ANALYSIS_INLINEADVISOR_H |
11 | |
12 | #include "llvm/Analysis/CGSCCPassManager.h" |
13 | #include "llvm/Analysis/InlineCost.h" |
14 | #include "llvm/Analysis/LazyCallGraph.h" |
15 | #include "llvm/Config/llvm-config.h" |
16 | #include "llvm/IR/PassManager.h" |
17 | #include <memory> |
18 | |
19 | namespace llvm { |
20 | class BasicBlock; |
21 | class CallBase; |
22 | class Function; |
23 | class Module; |
24 | class ; |
25 | class ImportedFunctionsInliningStatistics; |
26 | class ; |
27 | struct ReplayInlinerSettings; |
28 | |
29 | /// There are 4 scenarios we can use the InlineAdvisor: |
30 | /// - Default - use manual heuristics. |
31 | /// |
32 | /// - Release mode, the expected mode for production, day to day deployments. |
33 | /// In this mode, when building the compiler, we also compile a pre-trained ML |
34 | /// model to native code, and link it as a static library. This mode has low |
35 | /// overhead and no additional dependencies for the compiler runtime. |
36 | /// |
37 | /// - Development mode, for training new models. |
38 | /// In this mode, we trade off runtime performance for flexibility. This mode |
39 | /// requires the TFLite library, and evaluates models dynamically. This mode |
40 | /// also permits generating training logs, for offline training. |
41 | /// |
42 | /// - Dynamically load an advisor via a plugin (PluginInlineAdvisorAnalysis) |
43 | enum class InliningAdvisorMode : int { Default, Release, Development }; |
44 | |
45 | // Each entry represents an inline driver. |
46 | enum class InlinePass : int { |
47 | AlwaysInliner, |
48 | CGSCCInliner, |
49 | EarlyInliner, |
50 | ModuleInliner, |
51 | MLInliner, |
52 | ReplayCGSCCInliner, |
53 | ReplaySampleProfileInliner, |
54 | SampleProfileInliner, |
55 | }; |
56 | |
57 | /// Provides context on when an inline advisor is constructed in the pipeline |
58 | /// (e.g., link phase, inline driver). |
59 | struct InlineContext { |
60 | ThinOrFullLTOPhase LTOPhase; |
61 | |
62 | InlinePass Pass; |
63 | }; |
64 | |
65 | std::string AnnotateInlinePassName(InlineContext IC); |
66 | |
67 | class InlineAdvisor; |
68 | /// Capture state between an inlining decision having had been made, and |
69 | /// its impact being observable. When collecting model training data, this |
70 | /// allows recording features/decisions/partial reward data sets. |
71 | /// |
72 | /// Derivations of this type are expected to be tightly coupled with their |
73 | /// InliningAdvisors. The base type implements the minimal contractual |
74 | /// obligations. |
75 | class InlineAdvice { |
76 | public: |
77 | (InlineAdvisor *Advisor, CallBase &CB, |
78 | OptimizationRemarkEmitter &ORE, bool IsInliningRecommended); |
79 | |
80 | InlineAdvice(InlineAdvice &&) = delete; |
81 | InlineAdvice(const InlineAdvice &) = delete; |
82 | virtual ~InlineAdvice() { |
83 | assert(Recorded && "InlineAdvice should have been informed of the " |
84 | "inliner's decision in all cases" ); |
85 | } |
86 | |
87 | /// Exactly one of the record* APIs must be called. Implementers may extend |
88 | /// behavior by implementing the corresponding record*Impl. |
89 | /// |
90 | /// Call after inlining succeeded, and did not result in deleting the callee. |
91 | void recordInlining(); |
92 | |
93 | /// Call after inlining succeeded, and results in the callee being |
94 | /// delete-able, meaning, it has no more users, and will be cleaned up |
95 | /// subsequently. |
96 | void recordInliningWithCalleeDeleted(); |
97 | |
98 | /// Call after the decision for a call site was to not inline. |
99 | void recordUnsuccessfulInlining(const InlineResult &Result) { |
100 | markRecorded(); |
101 | recordUnsuccessfulInliningImpl(Result); |
102 | } |
103 | |
104 | /// Call to indicate inlining was not attempted. |
105 | void recordUnattemptedInlining() { |
106 | markRecorded(); |
107 | recordUnattemptedInliningImpl(); |
108 | } |
109 | |
110 | /// Get the inlining recommendation. |
111 | bool isInliningRecommended() const { return IsInliningRecommended; } |
112 | const DebugLoc &getOriginalCallSiteDebugLoc() const { return DLoc; } |
113 | const BasicBlock *getOriginalCallSiteBasicBlock() const { return Block; } |
114 | |
115 | protected: |
116 | virtual void recordInliningImpl() {} |
117 | virtual void recordInliningWithCalleeDeletedImpl() {} |
118 | virtual void recordUnsuccessfulInliningImpl(const InlineResult &Result) {} |
119 | virtual void recordUnattemptedInliningImpl() {} |
120 | |
121 | InlineAdvisor *const Advisor; |
122 | /// Caller and Callee are pre-inlining. |
123 | Function *const Caller; |
124 | Function *const Callee; |
125 | |
126 | // Capture the context of CB before inlining, as a successful inlining may |
127 | // change that context, and we want to report success or failure in the |
128 | // original context. |
129 | const DebugLoc DLoc; |
130 | const BasicBlock *const Block; |
131 | OptimizationRemarkEmitter &ORE; |
132 | const bool IsInliningRecommended; |
133 | |
134 | private: |
135 | void markRecorded() { |
136 | assert(!Recorded && "Recording should happen exactly once" ); |
137 | Recorded = true; |
138 | } |
139 | void recordInlineStatsIfNeeded(); |
140 | |
141 | bool Recorded = false; |
142 | }; |
143 | |
144 | class DefaultInlineAdvice : public InlineAdvice { |
145 | public: |
146 | (InlineAdvisor *Advisor, CallBase &CB, |
147 | std::optional<InlineCost> OIC, |
148 | OptimizationRemarkEmitter &ORE, bool = true) |
149 | : InlineAdvice(Advisor, CB, ORE, OIC.has_value()), OriginalCB(&CB), |
150 | OIC(OIC), EmitRemarks(EmitRemarks) {} |
151 | |
152 | private: |
153 | void recordUnsuccessfulInliningImpl(const InlineResult &Result) override; |
154 | void recordInliningWithCalleeDeletedImpl() override; |
155 | void recordInliningImpl() override; |
156 | |
157 | private: |
158 | CallBase *const OriginalCB; |
159 | std::optional<InlineCost> OIC; |
160 | bool ; |
161 | }; |
162 | |
163 | /// Interface for deciding whether to inline a call site or not. |
164 | class InlineAdvisor { |
165 | public: |
166 | InlineAdvisor(InlineAdvisor &&) = delete; |
167 | virtual ~InlineAdvisor(); |
168 | |
169 | /// Get an InlineAdvice containing a recommendation on whether to |
170 | /// inline or not. \p CB is assumed to be a direct call. \p FAM is assumed to |
171 | /// be up-to-date wrt previous inlining decisions. \p MandatoryOnly indicates |
172 | /// only mandatory (always-inline) call sites should be recommended - this |
173 | /// allows the InlineAdvisor track such inlininings. |
174 | /// Returns: |
175 | /// - An InlineAdvice with the inlining recommendation. |
176 | /// - Null when no recommendation is made (https://reviews.llvm.org/D110658). |
177 | /// TODO: Consider removing the Null return scenario by incorporating the |
178 | /// SampleProfile inliner into an InlineAdvisor |
179 | std::unique_ptr<InlineAdvice> getAdvice(CallBase &CB, |
180 | bool MandatoryOnly = false); |
181 | |
182 | /// This must be called when the Inliner pass is entered, to allow the |
183 | /// InlineAdvisor update internal state, as result of function passes run |
184 | /// between Inliner pass runs (for the same module). |
185 | virtual void onPassEntry(LazyCallGraph::SCC *SCC = nullptr) {} |
186 | |
187 | /// This must be called when the Inliner pass is exited, as function passes |
188 | /// may be run subsequently. This allows an implementation of InlineAdvisor |
189 | /// to prepare for a partial update, based on the optional SCC. |
190 | virtual void onPassExit(LazyCallGraph::SCC *SCC = nullptr) {} |
191 | |
192 | /// Support for printer pass |
193 | virtual void print(raw_ostream &OS) const { |
194 | OS << "Unimplemented InlineAdvisor print\n" ; |
195 | } |
196 | |
197 | /// NOTE pass name is annotated only when inline advisor constructor provides InlineContext. |
198 | const char *getAnnotatedInlinePassName() const { |
199 | return AnnotatedInlinePassName.c_str(); |
200 | } |
201 | |
202 | protected: |
203 | InlineAdvisor(Module &M, FunctionAnalysisManager &FAM, |
204 | std::optional<InlineContext> IC = std::nullopt); |
205 | virtual std::unique_ptr<InlineAdvice> getAdviceImpl(CallBase &CB) = 0; |
206 | virtual std::unique_ptr<InlineAdvice> getMandatoryAdvice(CallBase &CB, |
207 | bool Advice); |
208 | |
209 | Module &M; |
210 | FunctionAnalysisManager &FAM; |
211 | const std::optional<InlineContext> IC; |
212 | const std::string AnnotatedInlinePassName; |
213 | std::unique_ptr<ImportedFunctionsInliningStatistics> ImportedFunctionsStats; |
214 | |
215 | enum class MandatoryInliningKind { NotMandatory, Always, Never }; |
216 | |
217 | static MandatoryInliningKind getMandatoryKind(CallBase &CB, |
218 | FunctionAnalysisManager &FAM, |
219 | OptimizationRemarkEmitter &ORE); |
220 | |
221 | OptimizationRemarkEmitter &getCallerORE(CallBase &CB); |
222 | |
223 | private: |
224 | friend class InlineAdvice; |
225 | }; |
226 | |
227 | /// The default (manual heuristics) implementation of the InlineAdvisor. This |
228 | /// implementation does not need to keep state between inliner pass runs, and is |
229 | /// reusable as-is for inliner pass test scenarios, as well as for regular use. |
230 | class DefaultInlineAdvisor : public InlineAdvisor { |
231 | public: |
232 | DefaultInlineAdvisor(Module &M, FunctionAnalysisManager &FAM, |
233 | InlineParams Params, InlineContext IC) |
234 | : InlineAdvisor(M, FAM, IC), Params(Params) {} |
235 | |
236 | private: |
237 | std::unique_ptr<InlineAdvice> getAdviceImpl(CallBase &CB) override; |
238 | |
239 | InlineParams Params; |
240 | }; |
241 | |
242 | /// Used for dynamically registering InlineAdvisors as plugins |
243 | /// |
244 | /// An advisor plugin adds a new advisor at runtime by registering an instance |
245 | /// of PluginInlineAdvisorAnalysis in the current ModuleAnalysisManager. |
246 | /// For example, the following code dynamically registers a |
247 | /// DefaultInlineAdvisor: |
248 | /// |
249 | /// namespace { |
250 | /// |
251 | /// InlineAdvisor *defaultAdvisorFactory(Module &M, FunctionAnalysisManager |
252 | /// &FAM, |
253 | /// InlineParams Params, InlineContext IC) |
254 | /// { |
255 | /// return new DefaultInlineAdvisor(M, FAM, Params, IC); |
256 | /// } |
257 | /// |
258 | /// struct DefaultDynamicAdvisor : PassInfoMixin<DefaultDynamicAdvisor> { |
259 | /// PreservedAnalyses run(Module &, ModuleAnalysisManager &MAM) { |
260 | /// PluginInlineAdvisorAnalysis PA(defaultAdvisorFactory); |
261 | /// MAM.registerPass([&] { return PA; }); |
262 | /// return PreservedAnalyses::all(); |
263 | /// } |
264 | /// }; |
265 | /// |
266 | /// } // namespace |
267 | /// |
268 | /// extern "C" LLVM_ATTRIBUTE_WEAK ::llvm::PassPluginLibraryInfo |
269 | /// llvmGetPassPluginInfo() { |
270 | /// return {LLVM_PLUGIN_API_VERSION, "DynamicDefaultAdvisor", |
271 | /// LLVM_VERSION_STRING, |
272 | /// [](PassBuilder &PB) { |
273 | /// PB.registerPipelineStartEPCallback( |
274 | /// [](ModulePassManager &MPM, OptimizationLevel Level) { |
275 | /// MPM.addPass(DefaultDynamicAdvisor()); |
276 | /// }); |
277 | /// }}; |
278 | /// } |
279 | /// |
280 | /// A plugin must implement an AdvisorFactory and register it with a |
281 | /// PluginInlineAdvisorAnlysis to the provided ModuleanAlysisManager. |
282 | /// |
283 | /// If such a plugin has been registered |
284 | /// InlineAdvisorAnalysis::Result::tryCreate will return the dynamically loaded |
285 | /// advisor. |
286 | /// |
287 | class PluginInlineAdvisorAnalysis |
288 | : public AnalysisInfoMixin<PluginInlineAdvisorAnalysis> { |
289 | public: |
290 | static AnalysisKey Key; |
291 | static bool HasBeenRegistered; |
292 | |
293 | typedef InlineAdvisor *(*AdvisorFactory)(Module &M, |
294 | FunctionAnalysisManager &FAM, |
295 | InlineParams Params, |
296 | InlineContext IC); |
297 | |
298 | PluginInlineAdvisorAnalysis(AdvisorFactory Factory) : Factory(Factory) { |
299 | HasBeenRegistered = true; |
300 | assert(Factory != nullptr && |
301 | "The plugin advisor factory should not be a null pointer." ); |
302 | } |
303 | |
304 | struct Result { |
305 | AdvisorFactory Factory; |
306 | }; |
307 | |
308 | Result run(Module &M, ModuleAnalysisManager &MAM) { return {.Factory: Factory}; } |
309 | Result getResult() { return {.Factory: Factory}; } |
310 | |
311 | private: |
312 | AdvisorFactory Factory; |
313 | }; |
314 | |
315 | /// The InlineAdvisorAnalysis is a module pass because the InlineAdvisor |
316 | /// needs to capture state right before inlining commences over a module. |
317 | class InlineAdvisorAnalysis : public AnalysisInfoMixin<InlineAdvisorAnalysis> { |
318 | public: |
319 | static AnalysisKey Key; |
320 | InlineAdvisorAnalysis() = default; |
321 | struct Result { |
322 | Result(Module &M, ModuleAnalysisManager &MAM) : M(M), MAM(MAM) {} |
323 | bool invalidate(Module &, const PreservedAnalyses &PA, |
324 | ModuleAnalysisManager::Invalidator &) { |
325 | // Check whether the analysis has been explicitly invalidated. Otherwise, |
326 | // it's stateless and remains preserved. |
327 | auto PAC = PA.getChecker<InlineAdvisorAnalysis>(); |
328 | return !PAC.preservedWhenStateless(); |
329 | } |
330 | bool tryCreate(InlineParams Params, InliningAdvisorMode Mode, |
331 | const ReplayInlinerSettings &ReplaySettings, |
332 | InlineContext IC); |
333 | InlineAdvisor *getAdvisor() const { return Advisor.get(); } |
334 | |
335 | private: |
336 | Module &M; |
337 | ModuleAnalysisManager &MAM; |
338 | std::unique_ptr<InlineAdvisor> Advisor; |
339 | }; |
340 | |
341 | Result run(Module &M, ModuleAnalysisManager &MAM) { return Result(M, MAM); } |
342 | }; |
343 | |
344 | /// Printer pass for the InlineAdvisorAnalysis results. |
345 | class InlineAdvisorAnalysisPrinterPass |
346 | : public PassInfoMixin<InlineAdvisorAnalysisPrinterPass> { |
347 | raw_ostream &OS; |
348 | |
349 | public: |
350 | explicit InlineAdvisorAnalysisPrinterPass(raw_ostream &OS) : OS(OS) {} |
351 | |
352 | PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); |
353 | |
354 | PreservedAnalyses run(LazyCallGraph::SCC &InitialC, CGSCCAnalysisManager &AM, |
355 | LazyCallGraph &CG, CGSCCUpdateResult &UR); |
356 | static bool isRequired() { return true; } |
357 | }; |
358 | |
359 | std::unique_ptr<InlineAdvisor> |
360 | getReleaseModeAdvisor(Module &M, ModuleAnalysisManager &MAM, |
361 | std::function<bool(CallBase &)> GetDefaultAdvice); |
362 | |
363 | std::unique_ptr<InlineAdvisor> |
364 | getDevelopmentModeAdvisor(Module &M, ModuleAnalysisManager &MAM, |
365 | std::function<bool(CallBase &)> GetDefaultAdvice); |
366 | |
367 | // Default (manual policy) decision making helper APIs. Shared with the legacy |
368 | // pass manager inliner. |
369 | |
370 | /// Return the cost only if the inliner should attempt to inline at the given |
371 | /// CallSite. If we return the cost, we will emit an optimisation remark later |
372 | /// using that cost, so we won't do so from this function. Return std::nullopt |
373 | /// if inlining should not be attempted. |
374 | std::optional<InlineCost> |
375 | (CallBase &CB, function_ref<InlineCost(CallBase &CB)> GetInlineCost, |
376 | OptimizationRemarkEmitter &ORE, bool EnableDeferral = true); |
377 | |
378 | /// Emit ORE message. |
379 | void (OptimizationRemarkEmitter &ORE, DebugLoc DLoc, |
380 | const BasicBlock *Block, const Function &Callee, |
381 | const Function &Caller, bool IsMandatory, |
382 | function_ref<void(OptimizationRemark &)> = {}, |
383 | const char *PassName = nullptr); |
384 | |
385 | /// Emit ORE message based in cost (default heuristic). |
386 | void (OptimizationRemarkEmitter &ORE, DebugLoc DLoc, |
387 | const BasicBlock *Block, const Function &Callee, |
388 | const Function &Caller, const InlineCost &IC, |
389 | bool ForProfileContext = false, |
390 | const char *PassName = nullptr); |
391 | |
392 | /// Add location info to ORE message. |
393 | void (OptimizationRemark &, DebugLoc DLoc); |
394 | |
395 | /// Set the inline-remark attribute. |
396 | void (CallBase &CB, StringRef Message); |
397 | |
398 | /// Utility for extracting the inline cost message to a string. |
399 | std::string inlineCostStr(const InlineCost &IC); |
400 | } // namespace llvm |
401 | #endif // LLVM_ANALYSIS_INLINEADVISOR_H |
402 | |