1//===-- IntrinsicLowering.cpp - Intrinsic Lowering default implementation -===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the IntrinsicLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/IntrinsicLowering.h"
14#include "llvm/ADT/SmallVector.h"
15#include "llvm/IR/Constants.h"
16#include "llvm/IR/DataLayout.h"
17#include "llvm/IR/DerivedTypes.h"
18#include "llvm/IR/IRBuilder.h"
19#include "llvm/IR/Module.h"
20#include "llvm/IR/Type.h"
21#include "llvm/Support/ErrorHandling.h"
22#include "llvm/Support/raw_ostream.h"
23using namespace llvm;
24
25/// This function is used when we want to lower an intrinsic call to a call of
26/// an external function. This handles hard cases such as when there was already
27/// a prototype for the external function, but that prototype doesn't match the
28/// arguments we expect to pass in.
29template <class ArgIt>
30static CallInst *ReplaceCallWith(const char *NewFn, CallInst *CI,
31 ArgIt ArgBegin, ArgIt ArgEnd,
32 Type *RetTy) {
33 // If we haven't already looked up this function, check to see if the
34 // program already contains a function with this name.
35 Module *M = CI->getModule();
36 // Get or insert the definition now.
37 std::vector<Type *> ParamTys;
38 for (ArgIt I = ArgBegin; I != ArgEnd; ++I)
39 ParamTys.push_back((*I)->getType());
40 FunctionCallee FCache =
41 M->getOrInsertFunction(Name: NewFn, T: FunctionType::get(Result: RetTy, Params: ParamTys, isVarArg: false));
42
43 IRBuilder<> Builder(CI->getParent(), CI->getIterator());
44 SmallVector<Value *, 8> Args(ArgBegin, ArgEnd);
45 CallInst *NewCI = Builder.CreateCall(Callee: FCache, Args);
46 NewCI->setName(CI->getName());
47 if (!CI->use_empty())
48 CI->replaceAllUsesWith(V: NewCI);
49 return NewCI;
50}
51
52/// Emit the code to lower bswap of V before the specified instruction IP.
53static Value *LowerBSWAP(LLVMContext &Context, Value *V, Instruction *IP) {
54 assert(V->getType()->isIntOrIntVectorTy() && "Can't bswap a non-integer type!");
55
56 unsigned BitSize = V->getType()->getScalarSizeInBits();
57
58 IRBuilder<> Builder(IP);
59
60 switch(BitSize) {
61 default: llvm_unreachable("Unhandled type size of value to byteswap!");
62 case 16: {
63 Value *Tmp1 = Builder.CreateShl(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 8),
64 Name: "bswap.2");
65 Value *Tmp2 = Builder.CreateLShr(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 8),
66 Name: "bswap.1");
67 V = Builder.CreateOr(LHS: Tmp1, RHS: Tmp2, Name: "bswap.i16");
68 break;
69 }
70 case 32: {
71 Value *Tmp4 = Builder.CreateShl(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 24),
72 Name: "bswap.4");
73 Value *Tmp3 = Builder.CreateShl(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 8),
74 Name: "bswap.3");
75 Value *Tmp2 = Builder.CreateLShr(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 8),
76 Name: "bswap.2");
77 Value *Tmp1 = Builder.CreateLShr(LHS: V,RHS: ConstantInt::get(Ty: V->getType(), V: 24),
78 Name: "bswap.1");
79 Tmp3 = Builder.CreateAnd(LHS: Tmp3,
80 RHS: ConstantInt::get(Ty: V->getType(), V: 0xFF0000),
81 Name: "bswap.and3");
82 Tmp2 = Builder.CreateAnd(LHS: Tmp2,
83 RHS: ConstantInt::get(Ty: V->getType(), V: 0xFF00),
84 Name: "bswap.and2");
85 Tmp4 = Builder.CreateOr(LHS: Tmp4, RHS: Tmp3, Name: "bswap.or1");
86 Tmp2 = Builder.CreateOr(LHS: Tmp2, RHS: Tmp1, Name: "bswap.or2");
87 V = Builder.CreateOr(LHS: Tmp4, RHS: Tmp2, Name: "bswap.i32");
88 break;
89 }
90 case 64: {
91 Value *Tmp8 = Builder.CreateShl(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 56),
92 Name: "bswap.8");
93 Value *Tmp7 = Builder.CreateShl(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 40),
94 Name: "bswap.7");
95 Value *Tmp6 = Builder.CreateShl(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 24),
96 Name: "bswap.6");
97 Value *Tmp5 = Builder.CreateShl(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 8),
98 Name: "bswap.5");
99 Value* Tmp4 = Builder.CreateLShr(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 8),
100 Name: "bswap.4");
101 Value* Tmp3 = Builder.CreateLShr(LHS: V,
102 RHS: ConstantInt::get(Ty: V->getType(), V: 24),
103 Name: "bswap.3");
104 Value* Tmp2 = Builder.CreateLShr(LHS: V,
105 RHS: ConstantInt::get(Ty: V->getType(), V: 40),
106 Name: "bswap.2");
107 Value* Tmp1 = Builder.CreateLShr(LHS: V,
108 RHS: ConstantInt::get(Ty: V->getType(), V: 56),
109 Name: "bswap.1");
110 Tmp7 = Builder.CreateAnd(LHS: Tmp7,
111 RHS: ConstantInt::get(Ty: V->getType(),
112 V: 0xFF000000000000ULL),
113 Name: "bswap.and7");
114 Tmp6 = Builder.CreateAnd(LHS: Tmp6,
115 RHS: ConstantInt::get(Ty: V->getType(),
116 V: 0xFF0000000000ULL),
117 Name: "bswap.and6");
118 Tmp5 = Builder.CreateAnd(LHS: Tmp5,
119 RHS: ConstantInt::get(Ty: V->getType(),
120 V: 0xFF00000000ULL),
121 Name: "bswap.and5");
122 Tmp4 = Builder.CreateAnd(LHS: Tmp4,
123 RHS: ConstantInt::get(Ty: V->getType(),
124 V: 0xFF000000ULL),
125 Name: "bswap.and4");
126 Tmp3 = Builder.CreateAnd(LHS: Tmp3,
127 RHS: ConstantInt::get(Ty: V->getType(),
128 V: 0xFF0000ULL),
129 Name: "bswap.and3");
130 Tmp2 = Builder.CreateAnd(LHS: Tmp2,
131 RHS: ConstantInt::get(Ty: V->getType(),
132 V: 0xFF00ULL),
133 Name: "bswap.and2");
134 Tmp8 = Builder.CreateOr(LHS: Tmp8, RHS: Tmp7, Name: "bswap.or1");
135 Tmp6 = Builder.CreateOr(LHS: Tmp6, RHS: Tmp5, Name: "bswap.or2");
136 Tmp4 = Builder.CreateOr(LHS: Tmp4, RHS: Tmp3, Name: "bswap.or3");
137 Tmp2 = Builder.CreateOr(LHS: Tmp2, RHS: Tmp1, Name: "bswap.or4");
138 Tmp8 = Builder.CreateOr(LHS: Tmp8, RHS: Tmp6, Name: "bswap.or5");
139 Tmp4 = Builder.CreateOr(LHS: Tmp4, RHS: Tmp2, Name: "bswap.or6");
140 V = Builder.CreateOr(LHS: Tmp8, RHS: Tmp4, Name: "bswap.i64");
141 break;
142 }
143 }
144 return V;
145}
146
147/// Emit the code to lower ctpop of V before the specified instruction IP.
148static Value *LowerCTPOP(LLVMContext &Context, Value *V, Instruction *IP) {
149 assert(V->getType()->isIntegerTy() && "Can't ctpop a non-integer type!");
150
151 static const uint64_t MaskValues[6] = {
152 0x5555555555555555ULL, 0x3333333333333333ULL,
153 0x0F0F0F0F0F0F0F0FULL, 0x00FF00FF00FF00FFULL,
154 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL
155 };
156
157 IRBuilder<> Builder(IP);
158
159 unsigned BitSize = V->getType()->getPrimitiveSizeInBits();
160 unsigned WordSize = (BitSize + 63) / 64;
161 Value *Count = ConstantInt::get(Ty: V->getType(), V: 0);
162
163 for (unsigned n = 0; n < WordSize; ++n) {
164 Value *PartValue = V;
165 for (unsigned i = 1, ct = 0; i < (BitSize>64 ? 64 : BitSize);
166 i <<= 1, ++ct) {
167 Value *MaskCst = ConstantInt::get(Ty: V->getType(), V: MaskValues[ct]);
168 Value *LHS = Builder.CreateAnd(LHS: PartValue, RHS: MaskCst, Name: "cppop.and1");
169 Value *VShift = Builder.CreateLShr(LHS: PartValue,
170 RHS: ConstantInt::get(Ty: V->getType(), V: i),
171 Name: "ctpop.sh");
172 Value *RHS = Builder.CreateAnd(LHS: VShift, RHS: MaskCst, Name: "cppop.and2");
173 PartValue = Builder.CreateAdd(LHS, RHS, Name: "ctpop.step");
174 }
175 Count = Builder.CreateAdd(LHS: PartValue, RHS: Count, Name: "ctpop.part");
176 if (BitSize > 64) {
177 V = Builder.CreateLShr(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 64),
178 Name: "ctpop.part.sh");
179 BitSize -= 64;
180 }
181 }
182
183 return Count;
184}
185
186/// Emit the code to lower ctlz of V before the specified instruction IP.
187static Value *LowerCTLZ(LLVMContext &Context, Value *V, Instruction *IP) {
188
189 IRBuilder<> Builder(IP);
190
191 unsigned BitSize = V->getType()->getPrimitiveSizeInBits();
192 for (unsigned i = 1; i < BitSize; i <<= 1) {
193 Value *ShVal = ConstantInt::get(Ty: V->getType(), V: i);
194 ShVal = Builder.CreateLShr(LHS: V, RHS: ShVal, Name: "ctlz.sh");
195 V = Builder.CreateOr(LHS: V, RHS: ShVal, Name: "ctlz.step");
196 }
197
198 V = Builder.CreateNot(V);
199 return LowerCTPOP(Context, V, IP);
200}
201
202static void ReplaceFPIntrinsicWithCall(CallInst *CI, const char *Fname,
203 const char *Dname,
204 const char *LDname) {
205 switch (CI->getArgOperand(i: 0)->getType()->getTypeID()) {
206 default: llvm_unreachable("Invalid type in intrinsic");
207 case Type::FloatTyID:
208 ReplaceCallWith(NewFn: Fname, CI, ArgBegin: CI->arg_begin(), ArgEnd: CI->arg_end(),
209 RetTy: Type::getFloatTy(C&: CI->getContext()));
210 break;
211 case Type::DoubleTyID:
212 ReplaceCallWith(NewFn: Dname, CI, ArgBegin: CI->arg_begin(), ArgEnd: CI->arg_end(),
213 RetTy: Type::getDoubleTy(C&: CI->getContext()));
214 break;
215 case Type::X86_FP80TyID:
216 case Type::FP128TyID:
217 case Type::PPC_FP128TyID:
218 ReplaceCallWith(NewFn: LDname, CI, ArgBegin: CI->arg_begin(), ArgEnd: CI->arg_end(),
219 RetTy: CI->getArgOperand(i: 0)->getType());
220 break;
221 }
222}
223
224void IntrinsicLowering::LowerIntrinsicCall(CallInst *CI) {
225 IRBuilder<> Builder(CI);
226 LLVMContext &Context = CI->getContext();
227
228 const Function *Callee = CI->getCalledFunction();
229 assert(Callee && "Cannot lower an indirect call!");
230
231 switch (Callee->getIntrinsicID()) {
232 case Intrinsic::not_intrinsic:
233 report_fatal_error(reason: "Cannot lower a call to a non-intrinsic function '"+
234 Callee->getName() + "'!");
235 default:
236 report_fatal_error(reason: "Code generator does not support intrinsic function '"+
237 Callee->getName()+"'!");
238
239 case Intrinsic::expect: {
240 // Just replace __builtin_expect(exp, c) with EXP.
241 Value *V = CI->getArgOperand(i: 0);
242 CI->replaceAllUsesWith(V);
243 break;
244 }
245
246 case Intrinsic::ctpop:
247 CI->replaceAllUsesWith(V: LowerCTPOP(Context, V: CI->getArgOperand(i: 0), IP: CI));
248 break;
249
250 case Intrinsic::bswap:
251 CI->replaceAllUsesWith(V: LowerBSWAP(Context, V: CI->getArgOperand(i: 0), IP: CI));
252 break;
253
254 case Intrinsic::ctlz:
255 CI->replaceAllUsesWith(V: LowerCTLZ(Context, V: CI->getArgOperand(i: 0), IP: CI));
256 break;
257
258 case Intrinsic::cttz: {
259 // cttz(x) -> ctpop(~X & (X-1))
260 Value *Src = CI->getArgOperand(i: 0);
261 Value *NotSrc = Builder.CreateNot(V: Src);
262 NotSrc->setName(Src->getName() + ".not");
263 Value *SrcM1 = ConstantInt::get(Ty: Src->getType(), V: 1);
264 SrcM1 = Builder.CreateSub(LHS: Src, RHS: SrcM1);
265 Src = LowerCTPOP(Context, V: Builder.CreateAnd(LHS: NotSrc, RHS: SrcM1), IP: CI);
266 CI->replaceAllUsesWith(V: Src);
267 break;
268 }
269
270 case Intrinsic::stacksave:
271 case Intrinsic::stackrestore: {
272 if (!Warned)
273 errs() << "WARNING: this target does not support the llvm.stack"
274 << (Callee->getIntrinsicID() == Intrinsic::stacksave ?
275 "save" : "restore") << " intrinsic.\n";
276 Warned = true;
277 if (Callee->getIntrinsicID() == Intrinsic::stacksave)
278 CI->replaceAllUsesWith(V: Constant::getNullValue(Ty: CI->getType()));
279 break;
280 }
281
282 case Intrinsic::get_dynamic_area_offset:
283 errs() << "WARNING: this target does not support the custom llvm.get."
284 "dynamic.area.offset. It is being lowered to a constant 0\n";
285 // Just lower it to a constant 0 because for most targets
286 // @llvm.get.dynamic.area.offset is lowered to zero.
287 CI->replaceAllUsesWith(V: ConstantInt::get(Ty: CI->getType(), V: 0));
288 break;
289 case Intrinsic::returnaddress:
290 case Intrinsic::frameaddress:
291 errs() << "WARNING: this target does not support the llvm."
292 << (Callee->getIntrinsicID() == Intrinsic::returnaddress ?
293 "return" : "frame") << "address intrinsic.\n";
294 CI->replaceAllUsesWith(
295 V: ConstantPointerNull::get(T: cast<PointerType>(Val: CI->getType())));
296 break;
297 case Intrinsic::addressofreturnaddress:
298 errs() << "WARNING: this target does not support the "
299 "llvm.addressofreturnaddress intrinsic.\n";
300 CI->replaceAllUsesWith(
301 V: ConstantPointerNull::get(T: cast<PointerType>(Val: CI->getType())));
302 break;
303
304 case Intrinsic::prefetch:
305 break; // Simply strip out prefetches on unsupported architectures
306
307 case Intrinsic::pcmarker:
308 break; // Simply strip out pcmarker on unsupported architectures
309 case Intrinsic::readcyclecounter: {
310 errs() << "WARNING: this target does not support the llvm.readcyclecoun"
311 << "ter intrinsic. It is being lowered to a constant 0\n";
312 CI->replaceAllUsesWith(V: ConstantInt::get(Ty: Type::getInt64Ty(C&: Context), V: 0));
313 break;
314 }
315 case Intrinsic::readsteadycounter: {
316 errs() << "WARNING: this target does not support the llvm.readsteadycounter"
317 << " intrinsic. It is being lowered to a constant 0\n";
318 CI->replaceAllUsesWith(V: ConstantInt::get(Ty: Type::getInt64Ty(C&: Context), V: 0));
319 break;
320 }
321
322 case Intrinsic::dbg_declare:
323 case Intrinsic::dbg_label:
324 break; // Simply strip out debugging intrinsics
325
326 case Intrinsic::eh_typeid_for:
327 // Return something different to eh_selector.
328 CI->replaceAllUsesWith(V: ConstantInt::get(Ty: CI->getType(), V: 1));
329 break;
330
331 case Intrinsic::annotation:
332 case Intrinsic::ptr_annotation:
333 // Just drop the annotation, but forward the value
334 CI->replaceAllUsesWith(V: CI->getOperand(i_nocapture: 0));
335 break;
336
337 case Intrinsic::assume:
338 case Intrinsic::experimental_noalias_scope_decl:
339 case Intrinsic::var_annotation:
340 break; // Strip out these intrinsics
341
342 case Intrinsic::memcpy: {
343 Type *IntPtr = DL.getIntPtrType(C&: Context);
344 Value *Size = Builder.CreateIntCast(V: CI->getArgOperand(i: 2), DestTy: IntPtr,
345 /* isSigned */ false);
346 Value *Ops[3];
347 Ops[0] = CI->getArgOperand(i: 0);
348 Ops[1] = CI->getArgOperand(i: 1);
349 Ops[2] = Size;
350 ReplaceCallWith(NewFn: "memcpy", CI, ArgBegin: Ops, ArgEnd: Ops+3, RetTy: CI->getArgOperand(i: 0)->getType());
351 break;
352 }
353 case Intrinsic::memmove: {
354 Type *IntPtr = DL.getIntPtrType(C&: Context);
355 Value *Size = Builder.CreateIntCast(V: CI->getArgOperand(i: 2), DestTy: IntPtr,
356 /* isSigned */ false);
357 Value *Ops[3];
358 Ops[0] = CI->getArgOperand(i: 0);
359 Ops[1] = CI->getArgOperand(i: 1);
360 Ops[2] = Size;
361 ReplaceCallWith(NewFn: "memmove", CI, ArgBegin: Ops, ArgEnd: Ops+3, RetTy: CI->getArgOperand(i: 0)->getType());
362 break;
363 }
364 case Intrinsic::memset: {
365 Value *Op0 = CI->getArgOperand(i: 0);
366 Type *IntPtr = DL.getIntPtrType(Op0->getType());
367 Value *Size = Builder.CreateIntCast(V: CI->getArgOperand(i: 2), DestTy: IntPtr,
368 /* isSigned */ false);
369 Value *Ops[3];
370 Ops[0] = Op0;
371 // Extend the amount to i32.
372 Ops[1] = Builder.CreateIntCast(V: CI->getArgOperand(i: 1),
373 DestTy: Type::getInt32Ty(C&: Context),
374 /* isSigned */ false);
375 Ops[2] = Size;
376 ReplaceCallWith(NewFn: "memset", CI, ArgBegin: Ops, ArgEnd: Ops+3, RetTy: CI->getArgOperand(i: 0)->getType());
377 break;
378 }
379 case Intrinsic::sqrt: {
380 ReplaceFPIntrinsicWithCall(CI, Fname: "sqrtf", Dname: "sqrt", LDname: "sqrtl");
381 break;
382 }
383 case Intrinsic::log: {
384 ReplaceFPIntrinsicWithCall(CI, Fname: "logf", Dname: "log", LDname: "logl");
385 break;
386 }
387 case Intrinsic::log2: {
388 ReplaceFPIntrinsicWithCall(CI, Fname: "log2f", Dname: "log2", LDname: "log2l");
389 break;
390 }
391 case Intrinsic::log10: {
392 ReplaceFPIntrinsicWithCall(CI, Fname: "log10f", Dname: "log10", LDname: "log10l");
393 break;
394 }
395 case Intrinsic::exp: {
396 ReplaceFPIntrinsicWithCall(CI, Fname: "expf", Dname: "exp", LDname: "expl");
397 break;
398 }
399 case Intrinsic::exp2: {
400 ReplaceFPIntrinsicWithCall(CI, Fname: "exp2f", Dname: "exp2", LDname: "exp2l");
401 break;
402 }
403 case Intrinsic::pow: {
404 ReplaceFPIntrinsicWithCall(CI, Fname: "powf", Dname: "pow", LDname: "powl");
405 break;
406 }
407 case Intrinsic::sin: {
408 ReplaceFPIntrinsicWithCall(CI, Fname: "sinf", Dname: "sin", LDname: "sinl");
409 break;
410 }
411 case Intrinsic::cos: {
412 ReplaceFPIntrinsicWithCall(CI, Fname: "cosf", Dname: "cos", LDname: "cosl");
413 break;
414 }
415 case Intrinsic::floor: {
416 ReplaceFPIntrinsicWithCall(CI, Fname: "floorf", Dname: "floor", LDname: "floorl");
417 break;
418 }
419 case Intrinsic::ceil: {
420 ReplaceFPIntrinsicWithCall(CI, Fname: "ceilf", Dname: "ceil", LDname: "ceill");
421 break;
422 }
423 case Intrinsic::trunc: {
424 ReplaceFPIntrinsicWithCall(CI, Fname: "truncf", Dname: "trunc", LDname: "truncl");
425 break;
426 }
427 case Intrinsic::round: {
428 ReplaceFPIntrinsicWithCall(CI, Fname: "roundf", Dname: "round", LDname: "roundl");
429 break;
430 }
431 case Intrinsic::roundeven: {
432 ReplaceFPIntrinsicWithCall(CI, Fname: "roundevenf", Dname: "roundeven", LDname: "roundevenl");
433 break;
434 }
435 case Intrinsic::copysign: {
436 ReplaceFPIntrinsicWithCall(CI, Fname: "copysignf", Dname: "copysign", LDname: "copysignl");
437 break;
438 }
439 case Intrinsic::get_rounding:
440 // Lower to "round to the nearest"
441 if (!CI->getType()->isVoidTy())
442 CI->replaceAllUsesWith(V: ConstantInt::get(Ty: CI->getType(), V: 1));
443 break;
444 case Intrinsic::invariant_start:
445 case Intrinsic::lifetime_start:
446 // Discard region information.
447 CI->replaceAllUsesWith(V: UndefValue::get(T: CI->getType()));
448 break;
449 case Intrinsic::invariant_end:
450 case Intrinsic::lifetime_end:
451 // Discard region information.
452 break;
453 }
454
455 assert(CI->use_empty() &&
456 "Lowering should have eliminated any uses of the intrinsic call!");
457 CI->eraseFromParent();
458}
459
460bool IntrinsicLowering::LowerToByteSwap(CallInst *CI) {
461 // Verify this is a simple bswap.
462 if (CI->arg_size() != 1 || CI->getType() != CI->getArgOperand(i: 0)->getType() ||
463 !CI->getType()->isIntegerTy())
464 return false;
465
466 IntegerType *Ty = dyn_cast<IntegerType>(Val: CI->getType());
467 if (!Ty)
468 return false;
469
470 // Okay, we can do this xform, do so now.
471 Module *M = CI->getModule();
472 Function *Int = Intrinsic::getDeclaration(M, Intrinsic::id: bswap, Tys: Ty);
473
474 Value *Op = CI->getArgOperand(i: 0);
475 Op = CallInst::Create(Func: Int, Args: Op, NameStr: CI->getName(), InsertBefore: CI);
476
477 CI->replaceAllUsesWith(V: Op);
478 CI->eraseFromParent();
479 return true;
480}
481

source code of llvm/lib/CodeGen/IntrinsicLowering.cpp