1//===- AlwaysInliner.cpp - Code to inline always_inline functions ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements a custom inliner that handles only functions that
10// are marked as "always inline".
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/IPO/AlwaysInliner.h"
15#include "llvm/ADT/SetVector.h"
16#include "llvm/Analysis/AliasAnalysis.h"
17#include "llvm/Analysis/AssumptionCache.h"
18#include "llvm/Analysis/InlineCost.h"
19#include "llvm/Analysis/OptimizationRemarkEmitter.h"
20#include "llvm/Analysis/ProfileSummaryInfo.h"
21#include "llvm/IR/Module.h"
22#include "llvm/InitializePasses.h"
23#include "llvm/Transforms/IPO/Inliner.h"
24#include "llvm/Transforms/Utils/Cloning.h"
25#include "llvm/Transforms/Utils/ModuleUtils.h"
26
27using namespace llvm;
28
29#define DEBUG_TYPE "inline"
30
31namespace {
32
33bool AlwaysInlineImpl(
34 Module &M, bool InsertLifetime, ProfileSummaryInfo &PSI,
35 function_ref<AssumptionCache &(Function &)> GetAssumptionCache,
36 function_ref<AAResults &(Function &)> GetAAR,
37 function_ref<BlockFrequencyInfo &(Function &)> GetBFI) {
38 SmallSetVector<CallBase *, 16> Calls;
39 bool Changed = false;
40 SmallVector<Function *, 16> InlinedFunctions;
41 for (Function &F : M) {
42 // When callee coroutine function is inlined into caller coroutine function
43 // before coro-split pass,
44 // coro-early pass can not handle this quiet well.
45 // So we won't inline the coroutine function if it have not been unsplited
46 if (F.isPresplitCoroutine())
47 continue;
48
49 if (!F.isDeclaration() && isInlineViable(Callee&: F).isSuccess()) {
50 Calls.clear();
51
52 for (User *U : F.users())
53 if (auto *CB = dyn_cast<CallBase>(Val: U))
54 if (CB->getCalledFunction() == &F &&
55 CB->hasFnAttr(Attribute::AlwaysInline) &&
56 !CB->getAttributes().hasFnAttr(Attribute::NoInline))
57 Calls.insert(X: CB);
58
59 for (CallBase *CB : Calls) {
60 Function *Caller = CB->getCaller();
61 OptimizationRemarkEmitter ORE(Caller);
62 DebugLoc DLoc = CB->getDebugLoc();
63 BasicBlock *Block = CB->getParent();
64
65 InlineFunctionInfo IFI(GetAssumptionCache, &PSI,
66 GetBFI ? &GetBFI(*Caller) : nullptr,
67 GetBFI ? &GetBFI(F) : nullptr);
68
69 InlineResult Res = InlineFunction(CB&: *CB, IFI, /*MergeAttributes=*/true,
70 CalleeAAR: &GetAAR(F), InsertLifetime);
71 if (!Res.isSuccess()) {
72 ORE.emit(RemarkBuilder: [&]() {
73 return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc,
74 Block)
75 << "'" << ore::NV("Callee", &F) << "' is not inlined into '"
76 << ore::NV("Caller", Caller)
77 << "': " << ore::NV("Reason", Res.getFailureReason());
78 });
79 continue;
80 }
81
82 emitInlinedIntoBasedOnCost(
83 ORE, DLoc, Block, Callee: F, Caller: *Caller,
84 IC: InlineCost::getAlways(Reason: "always inline attribute"),
85 /*ForProfileContext=*/false, DEBUG_TYPE);
86
87 Changed = true;
88 }
89
90 if (F.hasFnAttribute(Attribute::AlwaysInline)) {
91 // Remember to try and delete this function afterward. This both avoids
92 // re-walking the rest of the module and avoids dealing with any
93 // iterator invalidation issues while deleting functions.
94 InlinedFunctions.push_back(Elt: &F);
95 }
96 }
97 }
98
99 // Remove any live functions.
100 erase_if(C&: InlinedFunctions, P: [&](Function *F) {
101 F->removeDeadConstantUsers();
102 return !F->isDefTriviallyDead();
103 });
104
105 // Delete the non-comdat ones from the module and also from our vector.
106 auto NonComdatBegin = partition(
107 Range&: InlinedFunctions, P: [&](Function *F) { return F->hasComdat(); });
108 for (Function *F : make_range(x: NonComdatBegin, y: InlinedFunctions.end())) {
109 M.getFunctionList().erase(IT: F);
110 Changed = true;
111 }
112 InlinedFunctions.erase(CS: NonComdatBegin, CE: InlinedFunctions.end());
113
114 if (!InlinedFunctions.empty()) {
115 // Now we just have the comdat functions. Filter out the ones whose comdats
116 // are not actually dead.
117 filterDeadComdatFunctions(DeadComdatFunctions&: InlinedFunctions);
118 // The remaining functions are actually dead.
119 for (Function *F : InlinedFunctions) {
120 M.getFunctionList().erase(IT: F);
121 Changed = true;
122 }
123 }
124
125 return Changed;
126}
127
128struct AlwaysInlinerLegacyPass : public ModulePass {
129 bool InsertLifetime;
130
131 AlwaysInlinerLegacyPass()
132 : AlwaysInlinerLegacyPass(/*InsertLifetime*/ true) {}
133
134 AlwaysInlinerLegacyPass(bool InsertLifetime)
135 : ModulePass(ID), InsertLifetime(InsertLifetime) {
136 initializeAlwaysInlinerLegacyPassPass(*PassRegistry::getPassRegistry());
137 }
138
139 /// Main run interface method. We override here to avoid calling skipSCC().
140 bool runOnModule(Module &M) override {
141
142 auto &PSI = getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
143 auto GetAAR = [&](Function &F) -> AAResults & {
144 return getAnalysis<AAResultsWrapperPass>(F).getAAResults();
145 };
146 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
147 return getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
148 };
149
150 return AlwaysInlineImpl(M, InsertLifetime, PSI, GetAssumptionCache, GetAAR,
151 /*GetBFI*/ nullptr);
152 }
153
154 static char ID; // Pass identification, replacement for typeid
155
156 void getAnalysisUsage(AnalysisUsage &AU) const override {
157 AU.addRequired<AssumptionCacheTracker>();
158 AU.addRequired<AAResultsWrapperPass>();
159 AU.addRequired<ProfileSummaryInfoWrapperPass>();
160 }
161};
162
163} // namespace
164
165char AlwaysInlinerLegacyPass::ID = 0;
166INITIALIZE_PASS_BEGIN(AlwaysInlinerLegacyPass, "always-inline",
167 "Inliner for always_inline functions", false, false)
168INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
169INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
170INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
171INITIALIZE_PASS_END(AlwaysInlinerLegacyPass, "always-inline",
172 "Inliner for always_inline functions", false, false)
173
174Pass *llvm::createAlwaysInlinerLegacyPass(bool InsertLifetime) {
175 return new AlwaysInlinerLegacyPass(InsertLifetime);
176}
177
178PreservedAnalyses AlwaysInlinerPass::run(Module &M,
179 ModuleAnalysisManager &MAM) {
180 FunctionAnalysisManager &FAM =
181 MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
182 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
183 return FAM.getResult<AssumptionAnalysis>(IR&: F);
184 };
185 auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & {
186 return FAM.getResult<BlockFrequencyAnalysis>(IR&: F);
187 };
188 auto GetAAR = [&](Function &F) -> AAResults & {
189 return FAM.getResult<AAManager>(IR&: F);
190 };
191 auto &PSI = MAM.getResult<ProfileSummaryAnalysis>(IR&: M);
192
193 bool Changed = AlwaysInlineImpl(M, InsertLifetime, PSI, GetAssumptionCache,
194 GetAAR, GetBFI);
195
196 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
197}
198

source code of llvm/lib/Transforms/IPO/AlwaysInliner.cpp