1 | //===- llvm/Analysis/TargetTransformInfo.cpp ------------------------------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | |
9 | #include "llvm/Analysis/TargetTransformInfo.h" |
10 | #include "llvm/Analysis/CFG.h" |
11 | #include "llvm/Analysis/LoopIterator.h" |
12 | #include "llvm/Analysis/TargetTransformInfoImpl.h" |
13 | #include "llvm/IR/CFG.h" |
14 | #include "llvm/IR/Dominators.h" |
15 | #include "llvm/IR/Instruction.h" |
16 | #include "llvm/IR/Instructions.h" |
17 | #include "llvm/IR/IntrinsicInst.h" |
18 | #include "llvm/IR/Module.h" |
19 | #include "llvm/IR/Operator.h" |
20 | #include "llvm/IR/PatternMatch.h" |
21 | #include "llvm/InitializePasses.h" |
22 | #include "llvm/Support/CommandLine.h" |
23 | #include <optional> |
24 | #include <utility> |
25 | |
26 | using namespace llvm; |
27 | using namespace PatternMatch; |
28 | |
29 | #define DEBUG_TYPE "tti" |
30 | |
31 | static cl::opt<bool> EnableReduxCost("costmodel-reduxcost" , cl::init(Val: false), |
32 | cl::Hidden, |
33 | cl::desc("Recognize reduction patterns." )); |
34 | |
35 | static cl::opt<unsigned> CacheLineSize( |
36 | "cache-line-size" , cl::init(Val: 0), cl::Hidden, |
37 | cl::desc("Use this to override the target cache line size when " |
38 | "specified by the user." )); |
39 | |
40 | static cl::opt<unsigned> MinPageSize( |
41 | "min-page-size" , cl::init(Val: 0), cl::Hidden, |
42 | cl::desc("Use this to override the target's minimum page size." )); |
43 | |
44 | static cl::opt<unsigned> PredictableBranchThreshold( |
45 | "predictable-branch-threshold" , cl::init(Val: 99), cl::Hidden, |
46 | cl::desc( |
47 | "Use this to override the target's predictable branch threshold (%)." )); |
48 | |
49 | namespace { |
50 | /// No-op implementation of the TTI interface using the utility base |
51 | /// classes. |
52 | /// |
53 | /// This is used when no target specific information is available. |
54 | struct NoTTIImpl : TargetTransformInfoImplCRTPBase<NoTTIImpl> { |
55 | explicit NoTTIImpl(const DataLayout &DL) |
56 | : TargetTransformInfoImplCRTPBase<NoTTIImpl>(DL) {} |
57 | }; |
58 | } // namespace |
59 | |
60 | bool HardwareLoopInfo::canAnalyze(LoopInfo &LI) { |
61 | // If the loop has irreducible control flow, it can not be converted to |
62 | // Hardware loop. |
63 | LoopBlocksRPO RPOT(L); |
64 | RPOT.perform(LI: &LI); |
65 | if (containsIrreducibleCFG<const BasicBlock *>(RPOTraversal&: RPOT, LI)) |
66 | return false; |
67 | return true; |
68 | } |
69 | |
70 | IntrinsicCostAttributes::IntrinsicCostAttributes( |
71 | Intrinsic::ID Id, const CallBase &CI, InstructionCost ScalarizationCost, |
72 | bool TypeBasedOnly) |
73 | : II(dyn_cast<IntrinsicInst>(Val: &CI)), RetTy(CI.getType()), IID(Id), |
74 | ScalarizationCost(ScalarizationCost) { |
75 | |
76 | if (const auto *FPMO = dyn_cast<FPMathOperator>(Val: &CI)) |
77 | FMF = FPMO->getFastMathFlags(); |
78 | |
79 | if (!TypeBasedOnly) |
80 | Arguments.insert(I: Arguments.begin(), From: CI.arg_begin(), To: CI.arg_end()); |
81 | FunctionType *FTy = CI.getCalledFunction()->getFunctionType(); |
82 | ParamTys.insert(I: ParamTys.begin(), From: FTy->param_begin(), To: FTy->param_end()); |
83 | } |
84 | |
85 | IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *RTy, |
86 | ArrayRef<Type *> Tys, |
87 | FastMathFlags Flags, |
88 | const IntrinsicInst *I, |
89 | InstructionCost ScalarCost) |
90 | : II(I), RetTy(RTy), IID(Id), FMF(Flags), ScalarizationCost(ScalarCost) { |
91 | ParamTys.insert(I: ParamTys.begin(), From: Tys.begin(), To: Tys.end()); |
92 | } |
93 | |
94 | IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *Ty, |
95 | ArrayRef<const Value *> Args) |
96 | : RetTy(Ty), IID(Id) { |
97 | |
98 | Arguments.insert(I: Arguments.begin(), From: Args.begin(), To: Args.end()); |
99 | ParamTys.reserve(N: Arguments.size()); |
100 | for (unsigned Idx = 0, Size = Arguments.size(); Idx != Size; ++Idx) |
101 | ParamTys.push_back(Elt: Arguments[Idx]->getType()); |
102 | } |
103 | |
104 | IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *RTy, |
105 | ArrayRef<const Value *> Args, |
106 | ArrayRef<Type *> Tys, |
107 | FastMathFlags Flags, |
108 | const IntrinsicInst *I, |
109 | InstructionCost ScalarCost) |
110 | : II(I), RetTy(RTy), IID(Id), FMF(Flags), ScalarizationCost(ScalarCost) { |
111 | ParamTys.insert(I: ParamTys.begin(), From: Tys.begin(), To: Tys.end()); |
112 | Arguments.insert(I: Arguments.begin(), From: Args.begin(), To: Args.end()); |
113 | } |
114 | |
115 | HardwareLoopInfo::HardwareLoopInfo(Loop *L) : L(L) { |
116 | // Match default options: |
117 | // - hardware-loop-counter-bitwidth = 32 |
118 | // - hardware-loop-decrement = 1 |
119 | CountType = Type::getInt32Ty(C&: L->getHeader()->getContext()); |
120 | LoopDecrement = ConstantInt::get(Ty: CountType, V: 1); |
121 | } |
122 | |
123 | bool HardwareLoopInfo::isHardwareLoopCandidate(ScalarEvolution &SE, |
124 | LoopInfo &LI, DominatorTree &DT, |
125 | bool ForceNestedLoop, |
126 | bool ForceHardwareLoopPHI) { |
127 | SmallVector<BasicBlock *, 4> ExitingBlocks; |
128 | L->getExitingBlocks(ExitingBlocks); |
129 | |
130 | for (BasicBlock *BB : ExitingBlocks) { |
131 | // If we pass the updated counter back through a phi, we need to know |
132 | // which latch the updated value will be coming from. |
133 | if (!L->isLoopLatch(BB)) { |
134 | if (ForceHardwareLoopPHI || CounterInReg) |
135 | continue; |
136 | } |
137 | |
138 | const SCEV *EC = SE.getExitCount(L, ExitingBlock: BB); |
139 | if (isa<SCEVCouldNotCompute>(Val: EC)) |
140 | continue; |
141 | if (const SCEVConstant *ConstEC = dyn_cast<SCEVConstant>(Val: EC)) { |
142 | if (ConstEC->getValue()->isZero()) |
143 | continue; |
144 | } else if (!SE.isLoopInvariant(S: EC, L)) |
145 | continue; |
146 | |
147 | if (SE.getTypeSizeInBits(Ty: EC->getType()) > CountType->getBitWidth()) |
148 | continue; |
149 | |
150 | // If this exiting block is contained in a nested loop, it is not eligible |
151 | // for insertion of the branch-and-decrement since the inner loop would |
152 | // end up messing up the value in the CTR. |
153 | if (!IsNestingLegal && LI.getLoopFor(BB) != L && !ForceNestedLoop) |
154 | continue; |
155 | |
156 | // We now have a loop-invariant count of loop iterations (which is not the |
157 | // constant zero) for which we know that this loop will not exit via this |
158 | // existing block. |
159 | |
160 | // We need to make sure that this block will run on every loop iteration. |
161 | // For this to be true, we must dominate all blocks with backedges. Such |
162 | // blocks are in-loop predecessors to the header block. |
163 | bool NotAlways = false; |
164 | for (BasicBlock *Pred : predecessors(BB: L->getHeader())) { |
165 | if (!L->contains(BB: Pred)) |
166 | continue; |
167 | |
168 | if (!DT.dominates(A: BB, B: Pred)) { |
169 | NotAlways = true; |
170 | break; |
171 | } |
172 | } |
173 | |
174 | if (NotAlways) |
175 | continue; |
176 | |
177 | // Make sure this blocks ends with a conditional branch. |
178 | Instruction *TI = BB->getTerminator(); |
179 | if (!TI) |
180 | continue; |
181 | |
182 | if (BranchInst *BI = dyn_cast<BranchInst>(Val: TI)) { |
183 | if (!BI->isConditional()) |
184 | continue; |
185 | |
186 | ExitBranch = BI; |
187 | } else |
188 | continue; |
189 | |
190 | // Note that this block may not be the loop latch block, even if the loop |
191 | // has a latch block. |
192 | ExitBlock = BB; |
193 | ExitCount = EC; |
194 | break; |
195 | } |
196 | |
197 | if (!ExitBlock) |
198 | return false; |
199 | return true; |
200 | } |
201 | |
202 | TargetTransformInfo::TargetTransformInfo(const DataLayout &DL) |
203 | : TTIImpl(new Model<NoTTIImpl>(NoTTIImpl(DL))) {} |
204 | |
205 | TargetTransformInfo::~TargetTransformInfo() = default; |
206 | |
207 | TargetTransformInfo::TargetTransformInfo(TargetTransformInfo &&Arg) |
208 | : TTIImpl(std::move(Arg.TTIImpl)) {} |
209 | |
210 | TargetTransformInfo &TargetTransformInfo::operator=(TargetTransformInfo &&RHS) { |
211 | TTIImpl = std::move(RHS.TTIImpl); |
212 | return *this; |
213 | } |
214 | |
215 | unsigned TargetTransformInfo::getInliningThresholdMultiplier() const { |
216 | return TTIImpl->getInliningThresholdMultiplier(); |
217 | } |
218 | |
219 | unsigned |
220 | TargetTransformInfo::getInliningCostBenefitAnalysisSavingsMultiplier() const { |
221 | return TTIImpl->getInliningCostBenefitAnalysisSavingsMultiplier(); |
222 | } |
223 | |
224 | unsigned |
225 | TargetTransformInfo::getInliningCostBenefitAnalysisProfitableMultiplier() |
226 | const { |
227 | return TTIImpl->getInliningCostBenefitAnalysisProfitableMultiplier(); |
228 | } |
229 | |
230 | unsigned |
231 | TargetTransformInfo::adjustInliningThreshold(const CallBase *CB) const { |
232 | return TTIImpl->adjustInliningThreshold(CB); |
233 | } |
234 | |
235 | unsigned TargetTransformInfo::getCallerAllocaCost(const CallBase *CB, |
236 | const AllocaInst *AI) const { |
237 | return TTIImpl->getCallerAllocaCost(CB, AI); |
238 | } |
239 | |
240 | int TargetTransformInfo::getInlinerVectorBonusPercent() const { |
241 | return TTIImpl->getInlinerVectorBonusPercent(); |
242 | } |
243 | |
244 | InstructionCost TargetTransformInfo::getGEPCost( |
245 | Type *PointeeType, const Value *Ptr, ArrayRef<const Value *> Operands, |
246 | Type *AccessType, TTI::TargetCostKind CostKind) const { |
247 | return TTIImpl->getGEPCost(PointeeType, Ptr, Operands, AccessType, CostKind); |
248 | } |
249 | |
250 | InstructionCost TargetTransformInfo::getPointersChainCost( |
251 | ArrayRef<const Value *> Ptrs, const Value *Base, |
252 | const TTI::PointersChainInfo &Info, Type *AccessTy, |
253 | TTI::TargetCostKind CostKind) const { |
254 | assert((Base || !Info.isSameBase()) && |
255 | "If pointers have same base address it has to be provided." ); |
256 | return TTIImpl->getPointersChainCost(Ptrs, Base, Info, AccessTy, CostKind); |
257 | } |
258 | |
259 | unsigned TargetTransformInfo::getEstimatedNumberOfCaseClusters( |
260 | const SwitchInst &SI, unsigned &JTSize, ProfileSummaryInfo *PSI, |
261 | BlockFrequencyInfo *BFI) const { |
262 | return TTIImpl->getEstimatedNumberOfCaseClusters(SI, JTSize, PSI, BFI); |
263 | } |
264 | |
265 | InstructionCost |
266 | TargetTransformInfo::getInstructionCost(const User *U, |
267 | ArrayRef<const Value *> Operands, |
268 | enum TargetCostKind CostKind) const { |
269 | InstructionCost Cost = TTIImpl->getInstructionCost(U, Operands, CostKind); |
270 | assert((CostKind == TTI::TCK_RecipThroughput || Cost >= 0) && |
271 | "TTI should not produce negative costs!" ); |
272 | return Cost; |
273 | } |
274 | |
275 | BranchProbability TargetTransformInfo::getPredictableBranchThreshold() const { |
276 | return PredictableBranchThreshold.getNumOccurrences() > 0 |
277 | ? BranchProbability(PredictableBranchThreshold, 100) |
278 | : TTIImpl->getPredictableBranchThreshold(); |
279 | } |
280 | |
281 | bool TargetTransformInfo::hasBranchDivergence(const Function *F) const { |
282 | return TTIImpl->hasBranchDivergence(F); |
283 | } |
284 | |
285 | bool TargetTransformInfo::isSourceOfDivergence(const Value *V) const { |
286 | return TTIImpl->isSourceOfDivergence(V); |
287 | } |
288 | |
289 | bool llvm::TargetTransformInfo::isAlwaysUniform(const Value *V) const { |
290 | return TTIImpl->isAlwaysUniform(V); |
291 | } |
292 | |
293 | bool llvm::TargetTransformInfo::isValidAddrSpaceCast(unsigned FromAS, |
294 | unsigned ToAS) const { |
295 | return TTIImpl->isValidAddrSpaceCast(FromAS, ToAS); |
296 | } |
297 | |
298 | bool llvm::TargetTransformInfo::addrspacesMayAlias(unsigned FromAS, |
299 | unsigned ToAS) const { |
300 | return TTIImpl->addrspacesMayAlias(AS0: FromAS, AS1: ToAS); |
301 | } |
302 | |
303 | unsigned TargetTransformInfo::getFlatAddressSpace() const { |
304 | return TTIImpl->getFlatAddressSpace(); |
305 | } |
306 | |
307 | bool TargetTransformInfo::collectFlatAddressOperands( |
308 | SmallVectorImpl<int> &OpIndexes, Intrinsic::ID IID) const { |
309 | return TTIImpl->collectFlatAddressOperands(OpIndexes, IID); |
310 | } |
311 | |
312 | bool TargetTransformInfo::isNoopAddrSpaceCast(unsigned FromAS, |
313 | unsigned ToAS) const { |
314 | return TTIImpl->isNoopAddrSpaceCast(FromAS, ToAS); |
315 | } |
316 | |
317 | bool TargetTransformInfo::canHaveNonUndefGlobalInitializerInAddressSpace( |
318 | unsigned AS) const { |
319 | return TTIImpl->canHaveNonUndefGlobalInitializerInAddressSpace(AS); |
320 | } |
321 | |
322 | unsigned TargetTransformInfo::getAssumedAddrSpace(const Value *V) const { |
323 | return TTIImpl->getAssumedAddrSpace(V); |
324 | } |
325 | |
326 | bool TargetTransformInfo::isSingleThreaded() const { |
327 | return TTIImpl->isSingleThreaded(); |
328 | } |
329 | |
330 | std::pair<const Value *, unsigned> |
331 | TargetTransformInfo::getPredicatedAddrSpace(const Value *V) const { |
332 | return TTIImpl->getPredicatedAddrSpace(V); |
333 | } |
334 | |
335 | Value *TargetTransformInfo::rewriteIntrinsicWithAddressSpace( |
336 | IntrinsicInst *II, Value *OldV, Value *NewV) const { |
337 | return TTIImpl->rewriteIntrinsicWithAddressSpace(II, OldV, NewV); |
338 | } |
339 | |
340 | bool TargetTransformInfo::isLoweredToCall(const Function *F) const { |
341 | return TTIImpl->isLoweredToCall(F); |
342 | } |
343 | |
344 | bool TargetTransformInfo::isHardwareLoopProfitable( |
345 | Loop *L, ScalarEvolution &SE, AssumptionCache &AC, |
346 | TargetLibraryInfo *LibInfo, HardwareLoopInfo &HWLoopInfo) const { |
347 | return TTIImpl->isHardwareLoopProfitable(L, SE, AC, LibInfo, HWLoopInfo); |
348 | } |
349 | |
350 | bool TargetTransformInfo::preferPredicateOverEpilogue( |
351 | TailFoldingInfo *TFI) const { |
352 | return TTIImpl->preferPredicateOverEpilogue(TFI); |
353 | } |
354 | |
355 | TailFoldingStyle TargetTransformInfo::getPreferredTailFoldingStyle( |
356 | bool IVUpdateMayOverflow) const { |
357 | return TTIImpl->getPreferredTailFoldingStyle(IVUpdateMayOverflow); |
358 | } |
359 | |
360 | std::optional<Instruction *> |
361 | TargetTransformInfo::instCombineIntrinsic(InstCombiner &IC, |
362 | IntrinsicInst &II) const { |
363 | return TTIImpl->instCombineIntrinsic(IC, II); |
364 | } |
365 | |
366 | std::optional<Value *> TargetTransformInfo::simplifyDemandedUseBitsIntrinsic( |
367 | InstCombiner &IC, IntrinsicInst &II, APInt DemandedMask, KnownBits &Known, |
368 | bool &KnownBitsComputed) const { |
369 | return TTIImpl->simplifyDemandedUseBitsIntrinsic(IC, II, DemandedMask, Known, |
370 | KnownBitsComputed); |
371 | } |
372 | |
373 | std::optional<Value *> TargetTransformInfo::simplifyDemandedVectorEltsIntrinsic( |
374 | InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts, |
375 | APInt &UndefElts2, APInt &UndefElts3, |
376 | std::function<void(Instruction *, unsigned, APInt, APInt &)> |
377 | SimplifyAndSetOp) const { |
378 | return TTIImpl->simplifyDemandedVectorEltsIntrinsic( |
379 | IC, II, DemandedElts, UndefElts, UndefElts2, UndefElts3, |
380 | SimplifyAndSetOp); |
381 | } |
382 | |
383 | void TargetTransformInfo::( |
384 | Loop *L, ScalarEvolution &SE, UnrollingPreferences &UP, |
385 | OptimizationRemarkEmitter *ORE) const { |
386 | return TTIImpl->getUnrollingPreferences(L, SE, UP, ORE); |
387 | } |
388 | |
389 | void TargetTransformInfo::getPeelingPreferences(Loop *L, ScalarEvolution &SE, |
390 | PeelingPreferences &PP) const { |
391 | return TTIImpl->getPeelingPreferences(L, SE, PP); |
392 | } |
393 | |
394 | bool TargetTransformInfo::isLegalAddImmediate(int64_t Imm) const { |
395 | return TTIImpl->isLegalAddImmediate(Imm); |
396 | } |
397 | |
398 | bool TargetTransformInfo::isLegalICmpImmediate(int64_t Imm) const { |
399 | return TTIImpl->isLegalICmpImmediate(Imm); |
400 | } |
401 | |
402 | bool TargetTransformInfo::isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV, |
403 | int64_t BaseOffset, |
404 | bool HasBaseReg, int64_t Scale, |
405 | unsigned AddrSpace, |
406 | Instruction *I) const { |
407 | return TTIImpl->isLegalAddressingMode(Ty, BaseGV, BaseOffset, HasBaseReg, |
408 | Scale, AddrSpace, I); |
409 | } |
410 | |
411 | bool TargetTransformInfo::isLSRCostLess(const LSRCost &C1, |
412 | const LSRCost &C2) const { |
413 | return TTIImpl->isLSRCostLess(C1, C2); |
414 | } |
415 | |
416 | bool TargetTransformInfo::isNumRegsMajorCostOfLSR() const { |
417 | return TTIImpl->isNumRegsMajorCostOfLSR(); |
418 | } |
419 | |
420 | bool TargetTransformInfo::shouldFoldTerminatingConditionAfterLSR() const { |
421 | return TTIImpl->shouldFoldTerminatingConditionAfterLSR(); |
422 | } |
423 | |
424 | bool TargetTransformInfo::isProfitableLSRChainElement(Instruction *I) const { |
425 | return TTIImpl->isProfitableLSRChainElement(I); |
426 | } |
427 | |
428 | bool TargetTransformInfo::canMacroFuseCmp() const { |
429 | return TTIImpl->canMacroFuseCmp(); |
430 | } |
431 | |
432 | bool TargetTransformInfo::canSaveCmp(Loop *L, BranchInst **BI, |
433 | ScalarEvolution *SE, LoopInfo *LI, |
434 | DominatorTree *DT, AssumptionCache *AC, |
435 | TargetLibraryInfo *LibInfo) const { |
436 | return TTIImpl->canSaveCmp(L, BI, SE, LI, DT, AC, LibInfo); |
437 | } |
438 | |
439 | TTI::AddressingModeKind |
440 | TargetTransformInfo::getPreferredAddressingMode(const Loop *L, |
441 | ScalarEvolution *SE) const { |
442 | return TTIImpl->getPreferredAddressingMode(L, SE); |
443 | } |
444 | |
445 | bool TargetTransformInfo::isLegalMaskedStore(Type *DataType, |
446 | Align Alignment) const { |
447 | return TTIImpl->isLegalMaskedStore(DataType, Alignment); |
448 | } |
449 | |
450 | bool TargetTransformInfo::isLegalMaskedLoad(Type *DataType, |
451 | Align Alignment) const { |
452 | return TTIImpl->isLegalMaskedLoad(DataType, Alignment); |
453 | } |
454 | |
455 | bool TargetTransformInfo::isLegalNTStore(Type *DataType, |
456 | Align Alignment) const { |
457 | return TTIImpl->isLegalNTStore(DataType, Alignment); |
458 | } |
459 | |
460 | bool TargetTransformInfo::isLegalNTLoad(Type *DataType, Align Alignment) const { |
461 | return TTIImpl->isLegalNTLoad(DataType, Alignment); |
462 | } |
463 | |
464 | bool TargetTransformInfo::isLegalBroadcastLoad(Type *ElementTy, |
465 | ElementCount NumElements) const { |
466 | return TTIImpl->isLegalBroadcastLoad(ElementTy, NumElements); |
467 | } |
468 | |
469 | bool TargetTransformInfo::isLegalMaskedGather(Type *DataType, |
470 | Align Alignment) const { |
471 | return TTIImpl->isLegalMaskedGather(DataType, Alignment); |
472 | } |
473 | |
474 | bool TargetTransformInfo::isLegalAltInstr( |
475 | VectorType *VecTy, unsigned Opcode0, unsigned Opcode1, |
476 | const SmallBitVector &OpcodeMask) const { |
477 | return TTIImpl->isLegalAltInstr(VecTy, Opcode0, Opcode1, OpcodeMask); |
478 | } |
479 | |
480 | bool TargetTransformInfo::isLegalMaskedScatter(Type *DataType, |
481 | Align Alignment) const { |
482 | return TTIImpl->isLegalMaskedScatter(DataType, Alignment); |
483 | } |
484 | |
485 | bool TargetTransformInfo::forceScalarizeMaskedGather(VectorType *DataType, |
486 | Align Alignment) const { |
487 | return TTIImpl->forceScalarizeMaskedGather(DataType, Alignment); |
488 | } |
489 | |
490 | bool TargetTransformInfo::forceScalarizeMaskedScatter(VectorType *DataType, |
491 | Align Alignment) const { |
492 | return TTIImpl->forceScalarizeMaskedScatter(DataType, Alignment); |
493 | } |
494 | |
495 | bool TargetTransformInfo::isLegalMaskedCompressStore(Type *DataType) const { |
496 | return TTIImpl->isLegalMaskedCompressStore(DataType); |
497 | } |
498 | |
499 | bool TargetTransformInfo::isLegalMaskedExpandLoad(Type *DataType) const { |
500 | return TTIImpl->isLegalMaskedExpandLoad(DataType); |
501 | } |
502 | |
503 | bool TargetTransformInfo::isLegalStridedLoadStore(Type *DataType, |
504 | Align Alignment) const { |
505 | return TTIImpl->isLegalStridedLoadStore(DataType, Alignment); |
506 | } |
507 | |
508 | bool TargetTransformInfo::enableOrderedReductions() const { |
509 | return TTIImpl->enableOrderedReductions(); |
510 | } |
511 | |
512 | bool TargetTransformInfo::hasDivRemOp(Type *DataType, bool IsSigned) const { |
513 | return TTIImpl->hasDivRemOp(DataType, IsSigned); |
514 | } |
515 | |
516 | bool TargetTransformInfo::hasVolatileVariant(Instruction *I, |
517 | unsigned AddrSpace) const { |
518 | return TTIImpl->hasVolatileVariant(I, AddrSpace); |
519 | } |
520 | |
521 | bool TargetTransformInfo::prefersVectorizedAddressing() const { |
522 | return TTIImpl->prefersVectorizedAddressing(); |
523 | } |
524 | |
525 | InstructionCost TargetTransformInfo::getScalingFactorCost( |
526 | Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, |
527 | int64_t Scale, unsigned AddrSpace) const { |
528 | InstructionCost Cost = TTIImpl->getScalingFactorCost( |
529 | Ty, BaseGV, BaseOffset, HasBaseReg, Scale, AddrSpace); |
530 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
531 | return Cost; |
532 | } |
533 | |
534 | bool TargetTransformInfo::LSRWithInstrQueries() const { |
535 | return TTIImpl->LSRWithInstrQueries(); |
536 | } |
537 | |
538 | bool TargetTransformInfo::isTruncateFree(Type *Ty1, Type *Ty2) const { |
539 | return TTIImpl->isTruncateFree(Ty1, Ty2); |
540 | } |
541 | |
542 | bool TargetTransformInfo::isProfitableToHoist(Instruction *I) const { |
543 | return TTIImpl->isProfitableToHoist(I); |
544 | } |
545 | |
546 | bool TargetTransformInfo::useAA() const { return TTIImpl->useAA(); } |
547 | |
548 | bool TargetTransformInfo::isTypeLegal(Type *Ty) const { |
549 | return TTIImpl->isTypeLegal(Ty); |
550 | } |
551 | |
552 | unsigned TargetTransformInfo::getRegUsageForType(Type *Ty) const { |
553 | return TTIImpl->getRegUsageForType(Ty); |
554 | } |
555 | |
556 | bool TargetTransformInfo::shouldBuildLookupTables() const { |
557 | return TTIImpl->shouldBuildLookupTables(); |
558 | } |
559 | |
560 | bool TargetTransformInfo::shouldBuildLookupTablesForConstant( |
561 | Constant *C) const { |
562 | return TTIImpl->shouldBuildLookupTablesForConstant(C); |
563 | } |
564 | |
565 | bool TargetTransformInfo::shouldBuildRelLookupTables() const { |
566 | return TTIImpl->shouldBuildRelLookupTables(); |
567 | } |
568 | |
569 | bool TargetTransformInfo::useColdCCForColdCall(Function &F) const { |
570 | return TTIImpl->useColdCCForColdCall(F); |
571 | } |
572 | |
573 | InstructionCost TargetTransformInfo::getScalarizationOverhead( |
574 | VectorType *Ty, const APInt &DemandedElts, bool Insert, bool , |
575 | TTI::TargetCostKind CostKind) const { |
576 | return TTIImpl->getScalarizationOverhead(Ty, DemandedElts, Insert, Extract, |
577 | CostKind); |
578 | } |
579 | |
580 | InstructionCost TargetTransformInfo::getOperandsScalarizationOverhead( |
581 | ArrayRef<const Value *> Args, ArrayRef<Type *> Tys, |
582 | TTI::TargetCostKind CostKind) const { |
583 | return TTIImpl->getOperandsScalarizationOverhead(Args, Tys, CostKind); |
584 | } |
585 | |
586 | bool TargetTransformInfo::supportsEfficientVectorElementLoadStore() const { |
587 | return TTIImpl->supportsEfficientVectorElementLoadStore(); |
588 | } |
589 | |
590 | bool TargetTransformInfo::supportsTailCalls() const { |
591 | return TTIImpl->supportsTailCalls(); |
592 | } |
593 | |
594 | bool TargetTransformInfo::supportsTailCallFor(const CallBase *CB) const { |
595 | return TTIImpl->supportsTailCallFor(CB); |
596 | } |
597 | |
598 | bool TargetTransformInfo::enableAggressiveInterleaving( |
599 | bool LoopHasReductions) const { |
600 | return TTIImpl->enableAggressiveInterleaving(LoopHasReductions); |
601 | } |
602 | |
603 | TargetTransformInfo::MemCmpExpansionOptions |
604 | TargetTransformInfo::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const { |
605 | return TTIImpl->enableMemCmpExpansion(OptSize, IsZeroCmp); |
606 | } |
607 | |
608 | bool TargetTransformInfo::enableSelectOptimize() const { |
609 | return TTIImpl->enableSelectOptimize(); |
610 | } |
611 | |
612 | bool TargetTransformInfo::shouldTreatInstructionLikeSelect( |
613 | const Instruction *I) const { |
614 | return TTIImpl->shouldTreatInstructionLikeSelect(I); |
615 | } |
616 | |
617 | bool TargetTransformInfo::enableInterleavedAccessVectorization() const { |
618 | return TTIImpl->enableInterleavedAccessVectorization(); |
619 | } |
620 | |
621 | bool TargetTransformInfo::enableMaskedInterleavedAccessVectorization() const { |
622 | return TTIImpl->enableMaskedInterleavedAccessVectorization(); |
623 | } |
624 | |
625 | bool TargetTransformInfo::isFPVectorizationPotentiallyUnsafe() const { |
626 | return TTIImpl->isFPVectorizationPotentiallyUnsafe(); |
627 | } |
628 | |
629 | bool |
630 | TargetTransformInfo::allowsMisalignedMemoryAccesses(LLVMContext &Context, |
631 | unsigned BitWidth, |
632 | unsigned AddressSpace, |
633 | Align Alignment, |
634 | unsigned *Fast) const { |
635 | return TTIImpl->allowsMisalignedMemoryAccesses(Context, BitWidth, |
636 | AddressSpace, Alignment, Fast); |
637 | } |
638 | |
639 | TargetTransformInfo::PopcntSupportKind |
640 | TargetTransformInfo::getPopcntSupport(unsigned IntTyWidthInBit) const { |
641 | return TTIImpl->getPopcntSupport(IntTyWidthInBit); |
642 | } |
643 | |
644 | bool TargetTransformInfo::haveFastSqrt(Type *Ty) const { |
645 | return TTIImpl->haveFastSqrt(Ty); |
646 | } |
647 | |
648 | bool TargetTransformInfo::isExpensiveToSpeculativelyExecute( |
649 | const Instruction *I) const { |
650 | return TTIImpl->isExpensiveToSpeculativelyExecute(I); |
651 | } |
652 | |
653 | bool TargetTransformInfo::isFCmpOrdCheaperThanFCmpZero(Type *Ty) const { |
654 | return TTIImpl->isFCmpOrdCheaperThanFCmpZero(Ty); |
655 | } |
656 | |
657 | InstructionCost TargetTransformInfo::getFPOpCost(Type *Ty) const { |
658 | InstructionCost Cost = TTIImpl->getFPOpCost(Ty); |
659 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
660 | return Cost; |
661 | } |
662 | |
663 | InstructionCost TargetTransformInfo::getIntImmCodeSizeCost(unsigned Opcode, |
664 | unsigned Idx, |
665 | const APInt &Imm, |
666 | Type *Ty) const { |
667 | InstructionCost Cost = TTIImpl->getIntImmCodeSizeCost(Opc: Opcode, Idx, Imm, Ty); |
668 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
669 | return Cost; |
670 | } |
671 | |
672 | InstructionCost |
673 | TargetTransformInfo::getIntImmCost(const APInt &Imm, Type *Ty, |
674 | TTI::TargetCostKind CostKind) const { |
675 | InstructionCost Cost = TTIImpl->getIntImmCost(Imm, Ty, CostKind); |
676 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
677 | return Cost; |
678 | } |
679 | |
680 | InstructionCost TargetTransformInfo::getIntImmCostInst( |
681 | unsigned Opcode, unsigned Idx, const APInt &Imm, Type *Ty, |
682 | TTI::TargetCostKind CostKind, Instruction *Inst) const { |
683 | InstructionCost Cost = |
684 | TTIImpl->getIntImmCostInst(Opc: Opcode, Idx, Imm, Ty, CostKind, Inst); |
685 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
686 | return Cost; |
687 | } |
688 | |
689 | InstructionCost |
690 | TargetTransformInfo::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx, |
691 | const APInt &Imm, Type *Ty, |
692 | TTI::TargetCostKind CostKind) const { |
693 | InstructionCost Cost = |
694 | TTIImpl->getIntImmCostIntrin(IID, Idx, Imm, Ty, CostKind); |
695 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
696 | return Cost; |
697 | } |
698 | |
699 | bool TargetTransformInfo::preferToKeepConstantsAttached( |
700 | const Instruction &Inst, const Function &Fn) const { |
701 | return TTIImpl->preferToKeepConstantsAttached(Inst, Fn); |
702 | } |
703 | |
704 | unsigned TargetTransformInfo::getNumberOfRegisters(unsigned ClassID) const { |
705 | return TTIImpl->getNumberOfRegisters(ClassID); |
706 | } |
707 | |
708 | unsigned TargetTransformInfo::getRegisterClassForType(bool Vector, |
709 | Type *Ty) const { |
710 | return TTIImpl->getRegisterClassForType(Vector, Ty); |
711 | } |
712 | |
713 | const char *TargetTransformInfo::getRegisterClassName(unsigned ClassID) const { |
714 | return TTIImpl->getRegisterClassName(ClassID); |
715 | } |
716 | |
717 | TypeSize TargetTransformInfo::getRegisterBitWidth( |
718 | TargetTransformInfo::RegisterKind K) const { |
719 | return TTIImpl->getRegisterBitWidth(K); |
720 | } |
721 | |
722 | unsigned TargetTransformInfo::getMinVectorRegisterBitWidth() const { |
723 | return TTIImpl->getMinVectorRegisterBitWidth(); |
724 | } |
725 | |
726 | std::optional<unsigned> TargetTransformInfo::getMaxVScale() const { |
727 | return TTIImpl->getMaxVScale(); |
728 | } |
729 | |
730 | std::optional<unsigned> TargetTransformInfo::getVScaleForTuning() const { |
731 | return TTIImpl->getVScaleForTuning(); |
732 | } |
733 | |
734 | bool TargetTransformInfo::isVScaleKnownToBeAPowerOfTwo() const { |
735 | return TTIImpl->isVScaleKnownToBeAPowerOfTwo(); |
736 | } |
737 | |
738 | bool TargetTransformInfo::shouldMaximizeVectorBandwidth( |
739 | TargetTransformInfo::RegisterKind K) const { |
740 | return TTIImpl->shouldMaximizeVectorBandwidth(K); |
741 | } |
742 | |
743 | ElementCount TargetTransformInfo::getMinimumVF(unsigned ElemWidth, |
744 | bool IsScalable) const { |
745 | return TTIImpl->getMinimumVF(ElemWidth, IsScalable); |
746 | } |
747 | |
748 | unsigned TargetTransformInfo::getMaximumVF(unsigned ElemWidth, |
749 | unsigned Opcode) const { |
750 | return TTIImpl->getMaximumVF(ElemWidth, Opcode); |
751 | } |
752 | |
753 | unsigned TargetTransformInfo::getStoreMinimumVF(unsigned VF, Type *ScalarMemTy, |
754 | Type *ScalarValTy) const { |
755 | return TTIImpl->getStoreMinimumVF(VF, ScalarMemTy, ScalarValTy); |
756 | } |
757 | |
758 | bool TargetTransformInfo::shouldConsiderAddressTypePromotion( |
759 | const Instruction &I, bool &) const { |
760 | return TTIImpl->shouldConsiderAddressTypePromotion( |
761 | I, AllowPromotionWithoutCommonHeader); |
762 | } |
763 | |
764 | unsigned TargetTransformInfo::getCacheLineSize() const { |
765 | return CacheLineSize.getNumOccurrences() > 0 ? CacheLineSize |
766 | : TTIImpl->getCacheLineSize(); |
767 | } |
768 | |
769 | std::optional<unsigned> |
770 | TargetTransformInfo::getCacheSize(CacheLevel Level) const { |
771 | return TTIImpl->getCacheSize(Level); |
772 | } |
773 | |
774 | std::optional<unsigned> |
775 | TargetTransformInfo::getCacheAssociativity(CacheLevel Level) const { |
776 | return TTIImpl->getCacheAssociativity(Level); |
777 | } |
778 | |
779 | std::optional<unsigned> TargetTransformInfo::getMinPageSize() const { |
780 | return MinPageSize.getNumOccurrences() > 0 ? MinPageSize |
781 | : TTIImpl->getMinPageSize(); |
782 | } |
783 | |
784 | unsigned TargetTransformInfo::getPrefetchDistance() const { |
785 | return TTIImpl->getPrefetchDistance(); |
786 | } |
787 | |
788 | unsigned TargetTransformInfo::getMinPrefetchStride( |
789 | unsigned NumMemAccesses, unsigned NumStridedMemAccesses, |
790 | unsigned NumPrefetches, bool HasCall) const { |
791 | return TTIImpl->getMinPrefetchStride(NumMemAccesses, NumStridedMemAccesses, |
792 | NumPrefetches, HasCall); |
793 | } |
794 | |
795 | unsigned TargetTransformInfo::getMaxPrefetchIterationsAhead() const { |
796 | return TTIImpl->getMaxPrefetchIterationsAhead(); |
797 | } |
798 | |
799 | bool TargetTransformInfo::enableWritePrefetching() const { |
800 | return TTIImpl->enableWritePrefetching(); |
801 | } |
802 | |
803 | bool TargetTransformInfo::shouldPrefetchAddressSpace(unsigned AS) const { |
804 | return TTIImpl->shouldPrefetchAddressSpace(AS); |
805 | } |
806 | |
807 | unsigned TargetTransformInfo::getMaxInterleaveFactor(ElementCount VF) const { |
808 | return TTIImpl->getMaxInterleaveFactor(VF); |
809 | } |
810 | |
811 | TargetTransformInfo::OperandValueInfo |
812 | TargetTransformInfo::getOperandInfo(const Value *V) { |
813 | OperandValueKind OpInfo = OK_AnyValue; |
814 | OperandValueProperties OpProps = OP_None; |
815 | |
816 | if (isa<ConstantInt>(Val: V) || isa<ConstantFP>(Val: V)) { |
817 | if (const auto *CI = dyn_cast<ConstantInt>(Val: V)) { |
818 | if (CI->getValue().isPowerOf2()) |
819 | OpProps = OP_PowerOf2; |
820 | else if (CI->getValue().isNegatedPowerOf2()) |
821 | OpProps = OP_NegatedPowerOf2; |
822 | } |
823 | return {.Kind: OK_UniformConstantValue, .Properties: OpProps}; |
824 | } |
825 | |
826 | // A broadcast shuffle creates a uniform value. |
827 | // TODO: Add support for non-zero index broadcasts. |
828 | // TODO: Add support for different source vector width. |
829 | if (const auto *ShuffleInst = dyn_cast<ShuffleVectorInst>(Val: V)) |
830 | if (ShuffleInst->isZeroEltSplat()) |
831 | OpInfo = OK_UniformValue; |
832 | |
833 | const Value *Splat = getSplatValue(V); |
834 | |
835 | // Check for a splat of a constant or for a non uniform vector of constants |
836 | // and check if the constant(s) are all powers of two. |
837 | if (isa<ConstantVector>(Val: V) || isa<ConstantDataVector>(Val: V)) { |
838 | OpInfo = OK_NonUniformConstantValue; |
839 | if (Splat) { |
840 | OpInfo = OK_UniformConstantValue; |
841 | if (auto *CI = dyn_cast<ConstantInt>(Val: Splat)) { |
842 | if (CI->getValue().isPowerOf2()) |
843 | OpProps = OP_PowerOf2; |
844 | else if (CI->getValue().isNegatedPowerOf2()) |
845 | OpProps = OP_NegatedPowerOf2; |
846 | } |
847 | } else if (const auto *CDS = dyn_cast<ConstantDataSequential>(Val: V)) { |
848 | bool AllPow2 = true, AllNegPow2 = true; |
849 | for (unsigned I = 0, E = CDS->getNumElements(); I != E; ++I) { |
850 | if (auto *CI = dyn_cast<ConstantInt>(Val: CDS->getElementAsConstant(i: I))) { |
851 | AllPow2 &= CI->getValue().isPowerOf2(); |
852 | AllNegPow2 &= CI->getValue().isNegatedPowerOf2(); |
853 | if (AllPow2 || AllNegPow2) |
854 | continue; |
855 | } |
856 | AllPow2 = AllNegPow2 = false; |
857 | break; |
858 | } |
859 | OpProps = AllPow2 ? OP_PowerOf2 : OpProps; |
860 | OpProps = AllNegPow2 ? OP_NegatedPowerOf2 : OpProps; |
861 | } |
862 | } |
863 | |
864 | // Check for a splat of a uniform value. This is not loop aware, so return |
865 | // true only for the obviously uniform cases (argument, globalvalue) |
866 | if (Splat && (isa<Argument>(Val: Splat) || isa<GlobalValue>(Val: Splat))) |
867 | OpInfo = OK_UniformValue; |
868 | |
869 | return {.Kind: OpInfo, .Properties: OpProps}; |
870 | } |
871 | |
872 | InstructionCost TargetTransformInfo::getArithmeticInstrCost( |
873 | unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, |
874 | OperandValueInfo Op1Info, OperandValueInfo Op2Info, |
875 | ArrayRef<const Value *> Args, const Instruction *CxtI) const { |
876 | InstructionCost Cost = |
877 | TTIImpl->getArithmeticInstrCost(Opcode, Ty, CostKind, |
878 | Opd1Info: Op1Info, Opd2Info: Op2Info, |
879 | Args, CxtI); |
880 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
881 | return Cost; |
882 | } |
883 | |
884 | InstructionCost TargetTransformInfo::getAltInstrCost( |
885 | VectorType *VecTy, unsigned Opcode0, unsigned Opcode1, |
886 | const SmallBitVector &OpcodeMask, TTI::TargetCostKind CostKind) const { |
887 | InstructionCost Cost = |
888 | TTIImpl->getAltInstrCost(VecTy, Opcode0, Opcode1, OpcodeMask, CostKind); |
889 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
890 | return Cost; |
891 | } |
892 | |
893 | InstructionCost TargetTransformInfo::getShuffleCost( |
894 | ShuffleKind Kind, VectorType *Ty, ArrayRef<int> Mask, |
895 | TTI::TargetCostKind CostKind, int Index, VectorType *SubTp, |
896 | ArrayRef<const Value *> Args) const { |
897 | InstructionCost Cost = |
898 | TTIImpl->getShuffleCost(Kind, Tp: Ty, Mask, CostKind, Index, SubTp, Args); |
899 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
900 | return Cost; |
901 | } |
902 | |
903 | TTI::CastContextHint |
904 | TargetTransformInfo::getCastContextHint(const Instruction *I) { |
905 | if (!I) |
906 | return CastContextHint::None; |
907 | |
908 | auto getLoadStoreKind = [](const Value *V, unsigned LdStOp, unsigned MaskedOp, |
909 | unsigned GatScatOp) { |
910 | const Instruction *I = dyn_cast<Instruction>(Val: V); |
911 | if (!I) |
912 | return CastContextHint::None; |
913 | |
914 | if (I->getOpcode() == LdStOp) |
915 | return CastContextHint::Normal; |
916 | |
917 | if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I)) { |
918 | if (II->getIntrinsicID() == MaskedOp) |
919 | return TTI::CastContextHint::Masked; |
920 | if (II->getIntrinsicID() == GatScatOp) |
921 | return TTI::CastContextHint::GatherScatter; |
922 | } |
923 | |
924 | return TTI::CastContextHint::None; |
925 | }; |
926 | |
927 | switch (I->getOpcode()) { |
928 | case Instruction::ZExt: |
929 | case Instruction::SExt: |
930 | case Instruction::FPExt: |
931 | return getLoadStoreKind(I->getOperand(i: 0), Instruction::Load, |
932 | Intrinsic::masked_load, Intrinsic::masked_gather); |
933 | case Instruction::Trunc: |
934 | case Instruction::FPTrunc: |
935 | if (I->hasOneUse()) |
936 | return getLoadStoreKind(*I->user_begin(), Instruction::Store, |
937 | Intrinsic::masked_store, |
938 | Intrinsic::masked_scatter); |
939 | break; |
940 | default: |
941 | return CastContextHint::None; |
942 | } |
943 | |
944 | return TTI::CastContextHint::None; |
945 | } |
946 | |
947 | InstructionCost TargetTransformInfo::getCastInstrCost( |
948 | unsigned Opcode, Type *Dst, Type *Src, CastContextHint CCH, |
949 | TTI::TargetCostKind CostKind, const Instruction *I) const { |
950 | assert((I == nullptr || I->getOpcode() == Opcode) && |
951 | "Opcode should reflect passed instruction." ); |
952 | InstructionCost Cost = |
953 | TTIImpl->getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I); |
954 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
955 | return Cost; |
956 | } |
957 | |
958 | InstructionCost TargetTransformInfo::( |
959 | unsigned Opcode, Type *Dst, VectorType *VecTy, unsigned Index) const { |
960 | InstructionCost Cost = |
961 | TTIImpl->getExtractWithExtendCost(Opcode, Dst, VecTy, Index); |
962 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
963 | return Cost; |
964 | } |
965 | |
966 | InstructionCost TargetTransformInfo::getCFInstrCost( |
967 | unsigned Opcode, TTI::TargetCostKind CostKind, const Instruction *I) const { |
968 | assert((I == nullptr || I->getOpcode() == Opcode) && |
969 | "Opcode should reflect passed instruction." ); |
970 | InstructionCost Cost = TTIImpl->getCFInstrCost(Opcode, CostKind, I); |
971 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
972 | return Cost; |
973 | } |
974 | |
975 | InstructionCost TargetTransformInfo::getCmpSelInstrCost( |
976 | unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred, |
977 | TTI::TargetCostKind CostKind, const Instruction *I) const { |
978 | assert((I == nullptr || I->getOpcode() == Opcode) && |
979 | "Opcode should reflect passed instruction." ); |
980 | InstructionCost Cost = |
981 | TTIImpl->getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, I); |
982 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
983 | return Cost; |
984 | } |
985 | |
986 | InstructionCost TargetTransformInfo::getVectorInstrCost( |
987 | unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index, |
988 | Value *Op0, Value *Op1) const { |
989 | // FIXME: Assert that Opcode is either InsertElement or ExtractElement. |
990 | // This is mentioned in the interface description and respected by all |
991 | // callers, but never asserted upon. |
992 | InstructionCost Cost = |
993 | TTIImpl->getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1); |
994 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
995 | return Cost; |
996 | } |
997 | |
998 | InstructionCost |
999 | TargetTransformInfo::getVectorInstrCost(const Instruction &I, Type *Val, |
1000 | TTI::TargetCostKind CostKind, |
1001 | unsigned Index) const { |
1002 | // FIXME: Assert that Opcode is either InsertElement or ExtractElement. |
1003 | // This is mentioned in the interface description and respected by all |
1004 | // callers, but never asserted upon. |
1005 | InstructionCost Cost = TTIImpl->getVectorInstrCost(I, Val, CostKind, Index); |
1006 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
1007 | return Cost; |
1008 | } |
1009 | |
1010 | InstructionCost TargetTransformInfo::getReplicationShuffleCost( |
1011 | Type *EltTy, int ReplicationFactor, int VF, const APInt &DemandedDstElts, |
1012 | TTI::TargetCostKind CostKind) { |
1013 | InstructionCost Cost = TTIImpl->getReplicationShuffleCost( |
1014 | EltTy, ReplicationFactor, VF, DemandedDstElts, CostKind); |
1015 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
1016 | return Cost; |
1017 | } |
1018 | |
1019 | InstructionCost TargetTransformInfo::getMemoryOpCost( |
1020 | unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, |
1021 | TTI::TargetCostKind CostKind, TTI::OperandValueInfo OpInfo, |
1022 | const Instruction *I) const { |
1023 | assert((I == nullptr || I->getOpcode() == Opcode) && |
1024 | "Opcode should reflect passed instruction." ); |
1025 | InstructionCost Cost = TTIImpl->getMemoryOpCost( |
1026 | Opcode, Src, Alignment, AddressSpace, CostKind, OpInfo, I); |
1027 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
1028 | return Cost; |
1029 | } |
1030 | |
1031 | InstructionCost TargetTransformInfo::getMaskedMemoryOpCost( |
1032 | unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, |
1033 | TTI::TargetCostKind CostKind) const { |
1034 | InstructionCost Cost = TTIImpl->getMaskedMemoryOpCost(Opcode, Src, Alignment, |
1035 | AddressSpace, CostKind); |
1036 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
1037 | return Cost; |
1038 | } |
1039 | |
1040 | InstructionCost TargetTransformInfo::getGatherScatterOpCost( |
1041 | unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask, |
1042 | Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const { |
1043 | InstructionCost Cost = TTIImpl->getGatherScatterOpCost( |
1044 | Opcode, DataTy, Ptr, VariableMask, Alignment, CostKind, I); |
1045 | assert((!Cost.isValid() || Cost >= 0) && |
1046 | "TTI should not produce negative costs!" ); |
1047 | return Cost; |
1048 | } |
1049 | |
1050 | InstructionCost TargetTransformInfo::getStridedMemoryOpCost( |
1051 | unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask, |
1052 | Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const { |
1053 | InstructionCost Cost = TTIImpl->getStridedMemoryOpCost( |
1054 | Opcode, DataTy, Ptr, VariableMask, Alignment, CostKind, I); |
1055 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
1056 | return Cost; |
1057 | } |
1058 | |
1059 | InstructionCost TargetTransformInfo::getInterleavedMemoryOpCost( |
1060 | unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices, |
1061 | Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, |
1062 | bool UseMaskForCond, bool UseMaskForGaps) const { |
1063 | InstructionCost Cost = TTIImpl->getInterleavedMemoryOpCost( |
1064 | Opcode, VecTy, Factor, Indices, Alignment, AddressSpace, CostKind, |
1065 | UseMaskForCond, UseMaskForGaps); |
1066 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
1067 | return Cost; |
1068 | } |
1069 | |
1070 | InstructionCost |
1071 | TargetTransformInfo::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, |
1072 | TTI::TargetCostKind CostKind) const { |
1073 | InstructionCost Cost = TTIImpl->getIntrinsicInstrCost(ICA, CostKind); |
1074 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
1075 | return Cost; |
1076 | } |
1077 | |
1078 | InstructionCost |
1079 | TargetTransformInfo::getCallInstrCost(Function *F, Type *RetTy, |
1080 | ArrayRef<Type *> Tys, |
1081 | TTI::TargetCostKind CostKind) const { |
1082 | InstructionCost Cost = TTIImpl->getCallInstrCost(F, RetTy, Tys, CostKind); |
1083 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
1084 | return Cost; |
1085 | } |
1086 | |
1087 | unsigned TargetTransformInfo::getNumberOfParts(Type *Tp) const { |
1088 | return TTIImpl->getNumberOfParts(Tp); |
1089 | } |
1090 | |
1091 | InstructionCost |
1092 | TargetTransformInfo::getAddressComputationCost(Type *Tp, ScalarEvolution *SE, |
1093 | const SCEV *Ptr) const { |
1094 | InstructionCost Cost = TTIImpl->getAddressComputationCost(Ty: Tp, SE, Ptr); |
1095 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
1096 | return Cost; |
1097 | } |
1098 | |
1099 | InstructionCost TargetTransformInfo::getMemcpyCost(const Instruction *I) const { |
1100 | InstructionCost Cost = TTIImpl->getMemcpyCost(I); |
1101 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
1102 | return Cost; |
1103 | } |
1104 | |
1105 | uint64_t TargetTransformInfo::getMaxMemIntrinsicInlineSizeThreshold() const { |
1106 | return TTIImpl->getMaxMemIntrinsicInlineSizeThreshold(); |
1107 | } |
1108 | |
1109 | InstructionCost TargetTransformInfo::getArithmeticReductionCost( |
1110 | unsigned Opcode, VectorType *Ty, std::optional<FastMathFlags> FMF, |
1111 | TTI::TargetCostKind CostKind) const { |
1112 | InstructionCost Cost = |
1113 | TTIImpl->getArithmeticReductionCost(Opcode, Ty, FMF, CostKind); |
1114 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
1115 | return Cost; |
1116 | } |
1117 | |
1118 | InstructionCost TargetTransformInfo::getMinMaxReductionCost( |
1119 | Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF, |
1120 | TTI::TargetCostKind CostKind) const { |
1121 | InstructionCost Cost = |
1122 | TTIImpl->getMinMaxReductionCost(IID, Ty, FMF, CostKind); |
1123 | assert(Cost >= 0 && "TTI should not produce negative costs!" ); |
1124 | return Cost; |
1125 | } |
1126 | |
1127 | InstructionCost TargetTransformInfo::getExtendedReductionCost( |
1128 | unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *Ty, |
1129 | FastMathFlags FMF, TTI::TargetCostKind CostKind) const { |
1130 | return TTIImpl->getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty, FMF, |
1131 | CostKind); |
1132 | } |
1133 | |
1134 | InstructionCost TargetTransformInfo::getMulAccReductionCost( |
1135 | bool IsUnsigned, Type *ResTy, VectorType *Ty, |
1136 | TTI::TargetCostKind CostKind) const { |
1137 | return TTIImpl->getMulAccReductionCost(IsUnsigned, ResTy, Ty, CostKind); |
1138 | } |
1139 | |
1140 | InstructionCost |
1141 | TargetTransformInfo::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const { |
1142 | return TTIImpl->getCostOfKeepingLiveOverCall(Tys); |
1143 | } |
1144 | |
1145 | bool TargetTransformInfo::getTgtMemIntrinsic(IntrinsicInst *Inst, |
1146 | MemIntrinsicInfo &Info) const { |
1147 | return TTIImpl->getTgtMemIntrinsic(Inst, Info); |
1148 | } |
1149 | |
1150 | unsigned TargetTransformInfo::getAtomicMemIntrinsicMaxElementSize() const { |
1151 | return TTIImpl->getAtomicMemIntrinsicMaxElementSize(); |
1152 | } |
1153 | |
1154 | Value *TargetTransformInfo::getOrCreateResultFromMemIntrinsic( |
1155 | IntrinsicInst *Inst, Type *ExpectedType) const { |
1156 | return TTIImpl->getOrCreateResultFromMemIntrinsic(Inst, ExpectedType); |
1157 | } |
1158 | |
1159 | Type *TargetTransformInfo::getMemcpyLoopLoweringType( |
1160 | LLVMContext &Context, Value *Length, unsigned SrcAddrSpace, |
1161 | unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign, |
1162 | std::optional<uint32_t> AtomicElementSize) const { |
1163 | return TTIImpl->getMemcpyLoopLoweringType(Context, Length, SrcAddrSpace, |
1164 | DestAddrSpace, SrcAlign, DestAlign, |
1165 | AtomicElementSize); |
1166 | } |
1167 | |
1168 | void TargetTransformInfo::getMemcpyLoopResidualLoweringType( |
1169 | SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context, |
1170 | unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace, |
1171 | unsigned SrcAlign, unsigned DestAlign, |
1172 | std::optional<uint32_t> AtomicCpySize) const { |
1173 | TTIImpl->getMemcpyLoopResidualLoweringType( |
1174 | OpsOut, Context, RemainingBytes, SrcAddrSpace, DestAddrSpace, SrcAlign, |
1175 | DestAlign, AtomicCpySize); |
1176 | } |
1177 | |
1178 | bool TargetTransformInfo::areInlineCompatible(const Function *Caller, |
1179 | const Function *Callee) const { |
1180 | return TTIImpl->areInlineCompatible(Caller, Callee); |
1181 | } |
1182 | |
1183 | unsigned |
1184 | TargetTransformInfo::getInlineCallPenalty(const Function *F, |
1185 | const CallBase &Call, |
1186 | unsigned DefaultCallPenalty) const { |
1187 | return TTIImpl->getInlineCallPenalty(F, Call, DefaultCallPenalty); |
1188 | } |
1189 | |
1190 | bool TargetTransformInfo::areTypesABICompatible( |
1191 | const Function *Caller, const Function *Callee, |
1192 | const ArrayRef<Type *> &Types) const { |
1193 | return TTIImpl->areTypesABICompatible(Caller, Callee, Types); |
1194 | } |
1195 | |
1196 | bool TargetTransformInfo::isIndexedLoadLegal(MemIndexedMode Mode, |
1197 | Type *Ty) const { |
1198 | return TTIImpl->isIndexedLoadLegal(Mode, Ty); |
1199 | } |
1200 | |
1201 | bool TargetTransformInfo::isIndexedStoreLegal(MemIndexedMode Mode, |
1202 | Type *Ty) const { |
1203 | return TTIImpl->isIndexedStoreLegal(Mode, Ty); |
1204 | } |
1205 | |
1206 | unsigned TargetTransformInfo::getLoadStoreVecRegBitWidth(unsigned AS) const { |
1207 | return TTIImpl->getLoadStoreVecRegBitWidth(AddrSpace: AS); |
1208 | } |
1209 | |
1210 | bool TargetTransformInfo::isLegalToVectorizeLoad(LoadInst *LI) const { |
1211 | return TTIImpl->isLegalToVectorizeLoad(LI); |
1212 | } |
1213 | |
1214 | bool TargetTransformInfo::isLegalToVectorizeStore(StoreInst *SI) const { |
1215 | return TTIImpl->isLegalToVectorizeStore(SI); |
1216 | } |
1217 | |
1218 | bool TargetTransformInfo::isLegalToVectorizeLoadChain( |
1219 | unsigned ChainSizeInBytes, Align Alignment, unsigned AddrSpace) const { |
1220 | return TTIImpl->isLegalToVectorizeLoadChain(ChainSizeInBytes, Alignment, |
1221 | AddrSpace); |
1222 | } |
1223 | |
1224 | bool TargetTransformInfo::isLegalToVectorizeStoreChain( |
1225 | unsigned ChainSizeInBytes, Align Alignment, unsigned AddrSpace) const { |
1226 | return TTIImpl->isLegalToVectorizeStoreChain(ChainSizeInBytes, Alignment, |
1227 | AddrSpace); |
1228 | } |
1229 | |
1230 | bool TargetTransformInfo::isLegalToVectorizeReduction( |
1231 | const RecurrenceDescriptor &RdxDesc, ElementCount VF) const { |
1232 | return TTIImpl->isLegalToVectorizeReduction(RdxDesc, VF); |
1233 | } |
1234 | |
1235 | bool TargetTransformInfo::isElementTypeLegalForScalableVector(Type *Ty) const { |
1236 | return TTIImpl->isElementTypeLegalForScalableVector(Ty); |
1237 | } |
1238 | |
1239 | unsigned TargetTransformInfo::getLoadVectorFactor(unsigned VF, |
1240 | unsigned LoadSize, |
1241 | unsigned ChainSizeInBytes, |
1242 | VectorType *VecTy) const { |
1243 | return TTIImpl->getLoadVectorFactor(VF, LoadSize, ChainSizeInBytes, VecTy); |
1244 | } |
1245 | |
1246 | unsigned TargetTransformInfo::getStoreVectorFactor(unsigned VF, |
1247 | unsigned StoreSize, |
1248 | unsigned ChainSizeInBytes, |
1249 | VectorType *VecTy) const { |
1250 | return TTIImpl->getStoreVectorFactor(VF, StoreSize, ChainSizeInBytes, VecTy); |
1251 | } |
1252 | |
1253 | bool TargetTransformInfo::preferInLoopReduction(unsigned Opcode, Type *Ty, |
1254 | ReductionFlags Flags) const { |
1255 | return TTIImpl->preferInLoopReduction(Opcode, Ty, Flags); |
1256 | } |
1257 | |
1258 | bool TargetTransformInfo::preferPredicatedReductionSelect( |
1259 | unsigned Opcode, Type *Ty, ReductionFlags Flags) const { |
1260 | return TTIImpl->preferPredicatedReductionSelect(Opcode, Ty, Flags); |
1261 | } |
1262 | |
1263 | bool TargetTransformInfo::preferEpilogueVectorization() const { |
1264 | return TTIImpl->preferEpilogueVectorization(); |
1265 | } |
1266 | |
1267 | TargetTransformInfo::VPLegalization |
1268 | TargetTransformInfo::getVPLegalizationStrategy(const VPIntrinsic &VPI) const { |
1269 | return TTIImpl->getVPLegalizationStrategy(PI: VPI); |
1270 | } |
1271 | |
1272 | bool TargetTransformInfo::hasArmWideBranch(bool Thumb) const { |
1273 | return TTIImpl->hasArmWideBranch(Thumb); |
1274 | } |
1275 | |
1276 | unsigned TargetTransformInfo::getMaxNumArgs() const { |
1277 | return TTIImpl->getMaxNumArgs(); |
1278 | } |
1279 | |
1280 | bool TargetTransformInfo::shouldExpandReduction(const IntrinsicInst *II) const { |
1281 | return TTIImpl->shouldExpandReduction(II); |
1282 | } |
1283 | |
1284 | unsigned TargetTransformInfo::getGISelRematGlobalCost() const { |
1285 | return TTIImpl->getGISelRematGlobalCost(); |
1286 | } |
1287 | |
1288 | unsigned TargetTransformInfo::getMinTripCountTailFoldingThreshold() const { |
1289 | return TTIImpl->getMinTripCountTailFoldingThreshold(); |
1290 | } |
1291 | |
1292 | bool TargetTransformInfo::supportsScalableVectors() const { |
1293 | return TTIImpl->supportsScalableVectors(); |
1294 | } |
1295 | |
1296 | bool TargetTransformInfo::enableScalableVectorization() const { |
1297 | return TTIImpl->enableScalableVectorization(); |
1298 | } |
1299 | |
1300 | bool TargetTransformInfo::hasActiveVectorLength(unsigned Opcode, Type *DataType, |
1301 | Align Alignment) const { |
1302 | return TTIImpl->hasActiveVectorLength(Opcode, DataType, Alignment); |
1303 | } |
1304 | |
1305 | TargetTransformInfo::Concept::~Concept() = default; |
1306 | |
1307 | TargetIRAnalysis::TargetIRAnalysis() : TTICallback(&getDefaultTTI) {} |
1308 | |
1309 | TargetIRAnalysis::TargetIRAnalysis( |
1310 | std::function<Result(const Function &)> TTICallback) |
1311 | : TTICallback(std::move(TTICallback)) {} |
1312 | |
1313 | TargetIRAnalysis::Result TargetIRAnalysis::run(const Function &F, |
1314 | FunctionAnalysisManager &) { |
1315 | return TTICallback(F); |
1316 | } |
1317 | |
1318 | AnalysisKey TargetIRAnalysis::Key; |
1319 | |
1320 | TargetIRAnalysis::Result TargetIRAnalysis::getDefaultTTI(const Function &F) { |
1321 | return Result(F.getParent()->getDataLayout()); |
1322 | } |
1323 | |
1324 | // Register the basic pass. |
1325 | INITIALIZE_PASS(TargetTransformInfoWrapperPass, "tti" , |
1326 | "Target Transform Information" , false, true) |
1327 | char TargetTransformInfoWrapperPass::ID = 0; |
1328 | |
1329 | void TargetTransformInfoWrapperPass::anchor() {} |
1330 | |
1331 | TargetTransformInfoWrapperPass::TargetTransformInfoWrapperPass() |
1332 | : ImmutablePass(ID) { |
1333 | initializeTargetTransformInfoWrapperPassPass( |
1334 | Registry&: *PassRegistry::getPassRegistry()); |
1335 | } |
1336 | |
1337 | TargetTransformInfoWrapperPass::TargetTransformInfoWrapperPass( |
1338 | TargetIRAnalysis TIRA) |
1339 | : ImmutablePass(ID), TIRA(std::move(TIRA)) { |
1340 | initializeTargetTransformInfoWrapperPassPass( |
1341 | Registry&: *PassRegistry::getPassRegistry()); |
1342 | } |
1343 | |
1344 | TargetTransformInfo &TargetTransformInfoWrapperPass::getTTI(const Function &F) { |
1345 | FunctionAnalysisManager DummyFAM; |
1346 | TTI = TIRA.run(F, DummyFAM); |
1347 | return *TTI; |
1348 | } |
1349 | |
1350 | ImmutablePass * |
1351 | llvm::createTargetTransformInfoWrapperPass(TargetIRAnalysis TIRA) { |
1352 | return new TargetTransformInfoWrapperPass(std::move(TIRA)); |
1353 | } |
1354 | |