1//===- PreISelIntrinsicLowering.cpp - Pre-ISel intrinsic lowering pass ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass implements IR lowering for the llvm.memcpy, llvm.memmove,
10// llvm.memset, llvm.load.relative and llvm.objc.* intrinsics.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/CodeGen/PreISelIntrinsicLowering.h"
15#include "llvm/Analysis/ObjCARCInstKind.h"
16#include "llvm/Analysis/ObjCARCUtil.h"
17#include "llvm/Analysis/TargetTransformInfo.h"
18#include "llvm/CodeGen/Passes.h"
19#include "llvm/CodeGen/TargetLowering.h"
20#include "llvm/CodeGen/TargetPassConfig.h"
21#include "llvm/IR/Function.h"
22#include "llvm/IR/IRBuilder.h"
23#include "llvm/IR/Instructions.h"
24#include "llvm/IR/IntrinsicInst.h"
25#include "llvm/IR/Module.h"
26#include "llvm/IR/Type.h"
27#include "llvm/InitializePasses.h"
28#include "llvm/Pass.h"
29#include "llvm/Support/Casting.h"
30#include "llvm/Target/TargetMachine.h"
31#include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
32
33using namespace llvm;
34
35/// Threshold to leave statically sized memory intrinsic calls. Calls of known
36/// size larger than this will be expanded by the pass. Calls of unknown or
37/// lower size will be left for expansion in codegen.
38static cl::opt<int64_t> MemIntrinsicExpandSizeThresholdOpt(
39 "mem-intrinsic-expand-size",
40 cl::desc("Set minimum mem intrinsic size to expand in IR"), cl::init(Val: -1),
41 cl::Hidden);
42
43namespace {
44
45struct PreISelIntrinsicLowering {
46 const TargetMachine &TM;
47 const function_ref<TargetTransformInfo &(Function &)> LookupTTI;
48
49 /// If this is true, assume it's preferably to leave memory intrinsic calls
50 /// for replacement with a library call later. Otherwise this depends on
51 /// TargetLoweringInfo availability of the corresponding function.
52 const bool UseMemIntrinsicLibFunc;
53
54 explicit PreISelIntrinsicLowering(
55 const TargetMachine &TM_,
56 function_ref<TargetTransformInfo &(Function &)> LookupTTI_,
57 bool UseMemIntrinsicLibFunc_ = true)
58 : TM(TM_), LookupTTI(LookupTTI_),
59 UseMemIntrinsicLibFunc(UseMemIntrinsicLibFunc_) {}
60
61 static bool shouldExpandMemIntrinsicWithSize(Value *Size,
62 const TargetTransformInfo &TTI);
63 bool expandMemIntrinsicUses(Function &F) const;
64 bool lowerIntrinsics(Module &M) const;
65};
66
67} // namespace
68
69static bool lowerLoadRelative(Function &F) {
70 if (F.use_empty())
71 return false;
72
73 bool Changed = false;
74 Type *Int32Ty = Type::getInt32Ty(C&: F.getContext());
75
76 for (Use &U : llvm::make_early_inc_range(Range: F.uses())) {
77 auto CI = dyn_cast<CallInst>(Val: U.getUser());
78 if (!CI || CI->getCalledOperand() != &F)
79 continue;
80
81 IRBuilder<> B(CI);
82 Value *OffsetPtr =
83 B.CreatePtrAdd(Ptr: CI->getArgOperand(i: 0), Offset: CI->getArgOperand(i: 1));
84 Value *OffsetI32 = B.CreateAlignedLoad(Ty: Int32Ty, Ptr: OffsetPtr, Align: Align(4));
85
86 Value *ResultPtr = B.CreatePtrAdd(Ptr: CI->getArgOperand(i: 0), Offset: OffsetI32);
87
88 CI->replaceAllUsesWith(V: ResultPtr);
89 CI->eraseFromParent();
90 Changed = true;
91 }
92
93 return Changed;
94}
95
96// ObjCARC has knowledge about whether an obj-c runtime function needs to be
97// always tail-called or never tail-called.
98static CallInst::TailCallKind getOverridingTailCallKind(const Function &F) {
99 objcarc::ARCInstKind Kind = objcarc::GetFunctionClass(F: &F);
100 if (objcarc::IsAlwaysTail(Class: Kind))
101 return CallInst::TCK_Tail;
102 else if (objcarc::IsNeverTail(Class: Kind))
103 return CallInst::TCK_NoTail;
104 return CallInst::TCK_None;
105}
106
107static bool lowerObjCCall(Function &F, const char *NewFn,
108 bool setNonLazyBind = false) {
109 assert(IntrinsicInst::mayLowerToFunctionCall(F.getIntrinsicID()) &&
110 "Pre-ISel intrinsics do lower into regular function calls");
111 if (F.use_empty())
112 return false;
113
114 // If we haven't already looked up this function, check to see if the
115 // program already contains a function with this name.
116 Module *M = F.getParent();
117 FunctionCallee FCache = M->getOrInsertFunction(Name: NewFn, T: F.getFunctionType());
118
119 if (Function *Fn = dyn_cast<Function>(Val: FCache.getCallee())) {
120 Fn->setLinkage(F.getLinkage());
121 if (setNonLazyBind && !Fn->isWeakForLinker()) {
122 // If we have Native ARC, set nonlazybind attribute for these APIs for
123 // performance.
124 Fn->addFnAttr(Attribute::NonLazyBind);
125 }
126 }
127
128 CallInst::TailCallKind OverridingTCK = getOverridingTailCallKind(F);
129
130 for (Use &U : llvm::make_early_inc_range(Range: F.uses())) {
131 auto *CB = cast<CallBase>(Val: U.getUser());
132
133 if (CB->getCalledFunction() != &F) {
134 objcarc::ARCInstKind Kind = objcarc::getAttachedARCFunctionKind(CB);
135 (void)Kind;
136 assert((Kind == objcarc::ARCInstKind::RetainRV ||
137 Kind == objcarc::ARCInstKind::UnsafeClaimRV) &&
138 "use expected to be the argument of operand bundle "
139 "\"clang.arc.attachedcall\"");
140 U.set(FCache.getCallee());
141 continue;
142 }
143
144 auto *CI = cast<CallInst>(Val: CB);
145 assert(CI->getCalledFunction() && "Cannot lower an indirect call!");
146
147 IRBuilder<> Builder(CI->getParent(), CI->getIterator());
148 SmallVector<Value *, 8> Args(CI->args());
149 SmallVector<llvm::OperandBundleDef, 1> BundleList;
150 CI->getOperandBundlesAsDefs(Defs&: BundleList);
151 CallInst *NewCI = Builder.CreateCall(Callee: FCache, Args, OpBundles: BundleList);
152 NewCI->setName(CI->getName());
153
154 // Try to set the most appropriate TailCallKind based on both the current
155 // attributes and the ones that we could get from ObjCARC's special
156 // knowledge of the runtime functions.
157 //
158 // std::max respects both requirements of notail and tail here:
159 // * notail on either the call or from ObjCARC becomes notail
160 // * tail on either side is stronger than none, but not notail
161 CallInst::TailCallKind TCK = CI->getTailCallKind();
162 NewCI->setTailCallKind(std::max(a: TCK, b: OverridingTCK));
163
164 // Transfer the 'returned' attribute from the intrinsic to the call site.
165 // By applying this only to intrinsic call sites, we avoid applying it to
166 // non-ARC explicit calls to things like objc_retain which have not been
167 // auto-upgraded to use the intrinsics.
168 unsigned Index;
169 if (F.getAttributes().hasAttrSomewhere(Attribute::Kind: Returned, Index: &Index) &&
170 Index)
171 NewCI->addParamAttr(Index - AttributeList::FirstArgIndex,
172 Attribute::Returned);
173
174 if (!CI->use_empty())
175 CI->replaceAllUsesWith(V: NewCI);
176 CI->eraseFromParent();
177 }
178
179 return true;
180}
181
182// TODO: Should refine based on estimated number of accesses (e.g. does it
183// require splitting based on alignment)
184bool PreISelIntrinsicLowering::shouldExpandMemIntrinsicWithSize(
185 Value *Size, const TargetTransformInfo &TTI) {
186 ConstantInt *CI = dyn_cast<ConstantInt>(Val: Size);
187 if (!CI)
188 return true;
189 uint64_t Threshold = MemIntrinsicExpandSizeThresholdOpt.getNumOccurrences()
190 ? MemIntrinsicExpandSizeThresholdOpt
191 : TTI.getMaxMemIntrinsicInlineSizeThreshold();
192 uint64_t SizeVal = CI->getZExtValue();
193
194 // Treat a threshold of 0 as a special case to force expansion of all
195 // intrinsics, including size 0.
196 return SizeVal > Threshold || Threshold == 0;
197}
198
199static bool canEmitLibcall(const TargetMachine &TM, Function *F,
200 RTLIB::Libcall LC) {
201 // TODO: Should this consider the address space of the memcpy?
202 const TargetLowering *TLI = TM.getSubtargetImpl(*F)->getTargetLowering();
203 return TLI->getLibcallName(Call: LC) != nullptr;
204}
205
206// TODO: Handle atomic memcpy and memcpy.inline
207// TODO: Pass ScalarEvolution
208bool PreISelIntrinsicLowering::expandMemIntrinsicUses(Function &F) const {
209 Intrinsic::ID ID = F.getIntrinsicID();
210 bool Changed = false;
211
212 for (User *U : llvm::make_early_inc_range(Range: F.users())) {
213 Instruction *Inst = cast<Instruction>(Val: U);
214
215 switch (ID) {
216 case Intrinsic::memcpy: {
217 auto *Memcpy = cast<MemCpyInst>(Val: Inst);
218 Function *ParentFunc = Memcpy->getFunction();
219 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
220 if (shouldExpandMemIntrinsicWithSize(Size: Memcpy->getLength(), TTI)) {
221 if (UseMemIntrinsicLibFunc &&
222 canEmitLibcall(TM, F: ParentFunc, LC: RTLIB::MEMCPY))
223 break;
224
225 // TODO: For optsize, emit the loop into a separate function
226 expandMemCpyAsLoop(MemCpy: Memcpy, TTI);
227 Changed = true;
228 Memcpy->eraseFromParent();
229 }
230
231 break;
232 }
233 case Intrinsic::memmove: {
234 auto *Memmove = cast<MemMoveInst>(Val: Inst);
235 Function *ParentFunc = Memmove->getFunction();
236 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
237 if (shouldExpandMemIntrinsicWithSize(Size: Memmove->getLength(), TTI)) {
238 if (UseMemIntrinsicLibFunc &&
239 canEmitLibcall(TM, F: ParentFunc, LC: RTLIB::MEMMOVE))
240 break;
241
242 if (expandMemMoveAsLoop(MemMove: Memmove, TTI)) {
243 Changed = true;
244 Memmove->eraseFromParent();
245 }
246 }
247
248 break;
249 }
250 case Intrinsic::memset: {
251 auto *Memset = cast<MemSetInst>(Val: Inst);
252 Function *ParentFunc = Memset->getFunction();
253 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
254 if (shouldExpandMemIntrinsicWithSize(Size: Memset->getLength(), TTI)) {
255 if (UseMemIntrinsicLibFunc &&
256 canEmitLibcall(TM, F: ParentFunc, LC: RTLIB::MEMSET))
257 break;
258
259 expandMemSetAsLoop(MemSet: Memset);
260 Changed = true;
261 Memset->eraseFromParent();
262 }
263
264 break;
265 }
266 default:
267 llvm_unreachable("unhandled intrinsic");
268 }
269 }
270
271 return Changed;
272}
273
274bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const {
275 bool Changed = false;
276 for (Function &F : M) {
277 switch (F.getIntrinsicID()) {
278 default:
279 break;
280 case Intrinsic::memcpy:
281 case Intrinsic::memmove:
282 case Intrinsic::memset:
283 Changed |= expandMemIntrinsicUses(F);
284 break;
285 case Intrinsic::load_relative:
286 Changed |= lowerLoadRelative(F);
287 break;
288 case Intrinsic::objc_autorelease:
289 Changed |= lowerObjCCall(F, NewFn: "objc_autorelease");
290 break;
291 case Intrinsic::objc_autoreleasePoolPop:
292 Changed |= lowerObjCCall(F, NewFn: "objc_autoreleasePoolPop");
293 break;
294 case Intrinsic::objc_autoreleasePoolPush:
295 Changed |= lowerObjCCall(F, NewFn: "objc_autoreleasePoolPush");
296 break;
297 case Intrinsic::objc_autoreleaseReturnValue:
298 Changed |= lowerObjCCall(F, NewFn: "objc_autoreleaseReturnValue");
299 break;
300 case Intrinsic::objc_copyWeak:
301 Changed |= lowerObjCCall(F, NewFn: "objc_copyWeak");
302 break;
303 case Intrinsic::objc_destroyWeak:
304 Changed |= lowerObjCCall(F, NewFn: "objc_destroyWeak");
305 break;
306 case Intrinsic::objc_initWeak:
307 Changed |= lowerObjCCall(F, NewFn: "objc_initWeak");
308 break;
309 case Intrinsic::objc_loadWeak:
310 Changed |= lowerObjCCall(F, NewFn: "objc_loadWeak");
311 break;
312 case Intrinsic::objc_loadWeakRetained:
313 Changed |= lowerObjCCall(F, NewFn: "objc_loadWeakRetained");
314 break;
315 case Intrinsic::objc_moveWeak:
316 Changed |= lowerObjCCall(F, NewFn: "objc_moveWeak");
317 break;
318 case Intrinsic::objc_release:
319 Changed |= lowerObjCCall(F, NewFn: "objc_release", setNonLazyBind: true);
320 break;
321 case Intrinsic::objc_retain:
322 Changed |= lowerObjCCall(F, NewFn: "objc_retain", setNonLazyBind: true);
323 break;
324 case Intrinsic::objc_retainAutorelease:
325 Changed |= lowerObjCCall(F, NewFn: "objc_retainAutorelease");
326 break;
327 case Intrinsic::objc_retainAutoreleaseReturnValue:
328 Changed |= lowerObjCCall(F, NewFn: "objc_retainAutoreleaseReturnValue");
329 break;
330 case Intrinsic::objc_retainAutoreleasedReturnValue:
331 Changed |= lowerObjCCall(F, NewFn: "objc_retainAutoreleasedReturnValue");
332 break;
333 case Intrinsic::objc_retainBlock:
334 Changed |= lowerObjCCall(F, NewFn: "objc_retainBlock");
335 break;
336 case Intrinsic::objc_storeStrong:
337 Changed |= lowerObjCCall(F, NewFn: "objc_storeStrong");
338 break;
339 case Intrinsic::objc_storeWeak:
340 Changed |= lowerObjCCall(F, NewFn: "objc_storeWeak");
341 break;
342 case Intrinsic::objc_unsafeClaimAutoreleasedReturnValue:
343 Changed |= lowerObjCCall(F, NewFn: "objc_unsafeClaimAutoreleasedReturnValue");
344 break;
345 case Intrinsic::objc_retainedObject:
346 Changed |= lowerObjCCall(F, NewFn: "objc_retainedObject");
347 break;
348 case Intrinsic::objc_unretainedObject:
349 Changed |= lowerObjCCall(F, NewFn: "objc_unretainedObject");
350 break;
351 case Intrinsic::objc_unretainedPointer:
352 Changed |= lowerObjCCall(F, NewFn: "objc_unretainedPointer");
353 break;
354 case Intrinsic::objc_retain_autorelease:
355 Changed |= lowerObjCCall(F, NewFn: "objc_retain_autorelease");
356 break;
357 case Intrinsic::objc_sync_enter:
358 Changed |= lowerObjCCall(F, NewFn: "objc_sync_enter");
359 break;
360 case Intrinsic::objc_sync_exit:
361 Changed |= lowerObjCCall(F, NewFn: "objc_sync_exit");
362 break;
363 }
364 }
365 return Changed;
366}
367
368namespace {
369
370class PreISelIntrinsicLoweringLegacyPass : public ModulePass {
371public:
372 static char ID;
373
374 PreISelIntrinsicLoweringLegacyPass() : ModulePass(ID) {}
375
376 void getAnalysisUsage(AnalysisUsage &AU) const override {
377 AU.addRequired<TargetTransformInfoWrapperPass>();
378 AU.addRequired<TargetPassConfig>();
379 }
380
381 bool runOnModule(Module &M) override {
382 auto LookupTTI = [this](Function &F) -> TargetTransformInfo & {
383 return this->getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
384 };
385
386 const auto &TM = getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
387 PreISelIntrinsicLowering Lowering(TM, LookupTTI);
388 return Lowering.lowerIntrinsics(M);
389 }
390};
391
392} // end anonymous namespace
393
394char PreISelIntrinsicLoweringLegacyPass::ID;
395
396INITIALIZE_PASS_BEGIN(PreISelIntrinsicLoweringLegacyPass,
397 "pre-isel-intrinsic-lowering",
398 "Pre-ISel Intrinsic Lowering", false, false)
399INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
400INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
401INITIALIZE_PASS_END(PreISelIntrinsicLoweringLegacyPass,
402 "pre-isel-intrinsic-lowering",
403 "Pre-ISel Intrinsic Lowering", false, false)
404
405ModulePass *llvm::createPreISelIntrinsicLoweringPass() {
406 return new PreISelIntrinsicLoweringLegacyPass();
407}
408
409PreservedAnalyses PreISelIntrinsicLoweringPass::run(Module &M,
410 ModuleAnalysisManager &AM) {
411 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
412
413 auto LookupTTI = [&FAM](Function &F) -> TargetTransformInfo & {
414 return FAM.getResult<TargetIRAnalysis>(IR&: F);
415 };
416
417 PreISelIntrinsicLowering Lowering(TM, LookupTTI);
418 if (!Lowering.lowerIntrinsics(M))
419 return PreservedAnalyses::all();
420 else
421 return PreservedAnalyses::none();
422}
423

source code of llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp