1 | //===- InterleavedAccessPass.cpp ------------------------------------------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file implements the Interleaved Access pass, which identifies |
10 | // interleaved memory accesses and transforms them into target specific |
11 | // intrinsics. |
12 | // |
13 | // An interleaved load reads data from memory into several vectors, with |
14 | // DE-interleaving the data on a factor. An interleaved store writes several |
15 | // vectors to memory with RE-interleaving the data on a factor. |
16 | // |
17 | // As interleaved accesses are difficult to identified in CodeGen (mainly |
18 | // because the VECTOR_SHUFFLE DAG node is quite different from the shufflevector |
19 | // IR), we identify and transform them to intrinsics in this pass so the |
20 | // intrinsics can be easily matched into target specific instructions later in |
21 | // CodeGen. |
22 | // |
23 | // E.g. An interleaved load (Factor = 2): |
24 | // %wide.vec = load <8 x i32>, <8 x i32>* %ptr |
25 | // %v0 = shuffle <8 x i32> %wide.vec, <8 x i32> poison, <0, 2, 4, 6> |
26 | // %v1 = shuffle <8 x i32> %wide.vec, <8 x i32> poison, <1, 3, 5, 7> |
27 | // |
28 | // It could be transformed into a ld2 intrinsic in AArch64 backend or a vld2 |
29 | // intrinsic in ARM backend. |
30 | // |
31 | // In X86, this can be further optimized into a set of target |
32 | // specific loads followed by an optimized sequence of shuffles. |
33 | // |
34 | // E.g. An interleaved store (Factor = 3): |
35 | // %i.vec = shuffle <8 x i32> %v0, <8 x i32> %v1, |
36 | // <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11> |
37 | // store <12 x i32> %i.vec, <12 x i32>* %ptr |
38 | // |
39 | // It could be transformed into a st3 intrinsic in AArch64 backend or a vst3 |
40 | // intrinsic in ARM backend. |
41 | // |
42 | // Similarly, a set of interleaved stores can be transformed into an optimized |
43 | // sequence of shuffles followed by a set of target specific stores for X86. |
44 | // |
45 | //===----------------------------------------------------------------------===// |
46 | |
47 | #include "llvm/ADT/ArrayRef.h" |
48 | #include "llvm/ADT/DenseMap.h" |
49 | #include "llvm/ADT/SetVector.h" |
50 | #include "llvm/ADT/SmallVector.h" |
51 | #include "llvm/CodeGen/InterleavedAccess.h" |
52 | #include "llvm/CodeGen/TargetLowering.h" |
53 | #include "llvm/CodeGen/TargetPassConfig.h" |
54 | #include "llvm/CodeGen/TargetSubtargetInfo.h" |
55 | #include "llvm/IR/Constants.h" |
56 | #include "llvm/IR/Dominators.h" |
57 | #include "llvm/IR/Function.h" |
58 | #include "llvm/IR/IRBuilder.h" |
59 | #include "llvm/IR/InstIterator.h" |
60 | #include "llvm/IR/Instruction.h" |
61 | #include "llvm/IR/Instructions.h" |
62 | #include "llvm/IR/IntrinsicInst.h" |
63 | #include "llvm/InitializePasses.h" |
64 | #include "llvm/Pass.h" |
65 | #include "llvm/Support/Casting.h" |
66 | #include "llvm/Support/CommandLine.h" |
67 | #include "llvm/Support/Debug.h" |
68 | #include "llvm/Support/MathExtras.h" |
69 | #include "llvm/Support/raw_ostream.h" |
70 | #include "llvm/Target/TargetMachine.h" |
71 | #include "llvm/Transforms/Utils/Local.h" |
72 | #include <cassert> |
73 | #include <utility> |
74 | |
75 | using namespace llvm; |
76 | |
77 | #define DEBUG_TYPE "interleaved-access" |
78 | |
79 | static cl::opt<bool> LowerInterleavedAccesses( |
80 | "lower-interleaved-accesses" , |
81 | cl::desc("Enable lowering interleaved accesses to intrinsics" ), |
82 | cl::init(Val: true), cl::Hidden); |
83 | |
84 | namespace { |
85 | |
86 | class InterleavedAccessImpl { |
87 | friend class InterleavedAccess; |
88 | |
89 | public: |
90 | InterleavedAccessImpl() = default; |
91 | InterleavedAccessImpl(DominatorTree *DT, const TargetLowering *TLI) |
92 | : DT(DT), TLI(TLI), MaxFactor(TLI->getMaxSupportedInterleaveFactor()) {} |
93 | bool runOnFunction(Function &F); |
94 | |
95 | private: |
96 | DominatorTree *DT = nullptr; |
97 | const TargetLowering *TLI = nullptr; |
98 | |
99 | /// The maximum supported interleave factor. |
100 | unsigned MaxFactor = 0u; |
101 | |
102 | /// Transform an interleaved load into target specific intrinsics. |
103 | bool lowerInterleavedLoad(LoadInst *LI, |
104 | SmallVector<Instruction *, 32> &DeadInsts); |
105 | |
106 | /// Transform an interleaved store into target specific intrinsics. |
107 | bool lowerInterleavedStore(StoreInst *SI, |
108 | SmallVector<Instruction *, 32> &DeadInsts); |
109 | |
110 | /// Transform a load and a deinterleave intrinsic into target specific |
111 | /// instructions. |
112 | bool lowerDeinterleaveIntrinsic(IntrinsicInst *II, |
113 | SmallVector<Instruction *, 32> &DeadInsts); |
114 | |
115 | /// Transform an interleave intrinsic and a store into target specific |
116 | /// instructions. |
117 | bool lowerInterleaveIntrinsic(IntrinsicInst *II, |
118 | SmallVector<Instruction *, 32> &DeadInsts); |
119 | |
120 | /// Returns true if the uses of an interleaved load by the |
121 | /// extractelement instructions in \p Extracts can be replaced by uses of the |
122 | /// shufflevector instructions in \p Shuffles instead. If so, the necessary |
123 | /// replacements are also performed. |
124 | bool tryReplaceExtracts(ArrayRef<ExtractElementInst *> , |
125 | ArrayRef<ShuffleVectorInst *> Shuffles); |
126 | |
127 | /// Given a number of shuffles of the form shuffle(binop(x,y)), convert them |
128 | /// to binop(shuffle(x), shuffle(y)) to allow the formation of an |
129 | /// interleaving load. Any newly created shuffles that operate on \p LI will |
130 | /// be added to \p Shuffles. Returns true, if any changes to the IR have been |
131 | /// made. |
132 | bool replaceBinOpShuffles(ArrayRef<ShuffleVectorInst *> BinOpShuffles, |
133 | SmallVectorImpl<ShuffleVectorInst *> &Shuffles, |
134 | LoadInst *LI); |
135 | }; |
136 | |
137 | class InterleavedAccess : public FunctionPass { |
138 | InterleavedAccessImpl Impl; |
139 | |
140 | public: |
141 | static char ID; |
142 | |
143 | InterleavedAccess() : FunctionPass(ID) { |
144 | initializeInterleavedAccessPass(*PassRegistry::getPassRegistry()); |
145 | } |
146 | |
147 | StringRef getPassName() const override { return "Interleaved Access Pass" ; } |
148 | |
149 | bool runOnFunction(Function &F) override; |
150 | |
151 | void getAnalysisUsage(AnalysisUsage &AU) const override { |
152 | AU.addRequired<DominatorTreeWrapperPass>(); |
153 | AU.setPreservesCFG(); |
154 | } |
155 | }; |
156 | |
157 | } // end anonymous namespace. |
158 | |
159 | PreservedAnalyses InterleavedAccessPass::run(Function &F, |
160 | FunctionAnalysisManager &FAM) { |
161 | auto *DT = &FAM.getResult<DominatorTreeAnalysis>(IR&: F); |
162 | auto *TLI = TM->getSubtargetImpl(F)->getTargetLowering(); |
163 | InterleavedAccessImpl Impl(DT, TLI); |
164 | bool Changed = Impl.runOnFunction(F); |
165 | |
166 | if (!Changed) |
167 | return PreservedAnalyses::all(); |
168 | |
169 | PreservedAnalyses PA; |
170 | PA.preserveSet<CFGAnalyses>(); |
171 | return PA; |
172 | } |
173 | |
174 | char InterleavedAccess::ID = 0; |
175 | |
176 | bool InterleavedAccess::runOnFunction(Function &F) { |
177 | auto *TPC = getAnalysisIfAvailable<TargetPassConfig>(); |
178 | if (!TPC || !LowerInterleavedAccesses) |
179 | return false; |
180 | |
181 | LLVM_DEBUG(dbgs() << "*** " << getPassName() << ": " << F.getName() << "\n" ); |
182 | |
183 | Impl.DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree(); |
184 | auto &TM = TPC->getTM<TargetMachine>(); |
185 | Impl.TLI = TM.getSubtargetImpl(F)->getTargetLowering(); |
186 | Impl.MaxFactor = Impl.TLI->getMaxSupportedInterleaveFactor(); |
187 | |
188 | return Impl.runOnFunction(F); |
189 | } |
190 | |
191 | INITIALIZE_PASS_BEGIN(InterleavedAccess, DEBUG_TYPE, |
192 | "Lower interleaved memory accesses to target specific intrinsics" , false, |
193 | false) |
194 | INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) |
195 | INITIALIZE_PASS_END(InterleavedAccess, DEBUG_TYPE, |
196 | "Lower interleaved memory accesses to target specific intrinsics" , false, |
197 | false) |
198 | |
199 | FunctionPass *llvm::createInterleavedAccessPass() { |
200 | return new InterleavedAccess(); |
201 | } |
202 | |
203 | /// Check if the mask is a DE-interleave mask of the given factor |
204 | /// \p Factor like: |
205 | /// <Index, Index+Factor, ..., Index+(NumElts-1)*Factor> |
206 | static bool isDeInterleaveMaskOfFactor(ArrayRef<int> Mask, unsigned Factor, |
207 | unsigned &Index) { |
208 | // Check all potential start indices from 0 to (Factor - 1). |
209 | for (Index = 0; Index < Factor; Index++) { |
210 | unsigned i = 0; |
211 | |
212 | // Check that elements are in ascending order by Factor. Ignore undef |
213 | // elements. |
214 | for (; i < Mask.size(); i++) |
215 | if (Mask[i] >= 0 && static_cast<unsigned>(Mask[i]) != Index + i * Factor) |
216 | break; |
217 | |
218 | if (i == Mask.size()) |
219 | return true; |
220 | } |
221 | |
222 | return false; |
223 | } |
224 | |
225 | /// Check if the mask is a DE-interleave mask for an interleaved load. |
226 | /// |
227 | /// E.g. DE-interleave masks (Factor = 2) could be: |
228 | /// <0, 2, 4, 6> (mask of index 0 to extract even elements) |
229 | /// <1, 3, 5, 7> (mask of index 1 to extract odd elements) |
230 | static bool isDeInterleaveMask(ArrayRef<int> Mask, unsigned &Factor, |
231 | unsigned &Index, unsigned MaxFactor, |
232 | unsigned NumLoadElements) { |
233 | if (Mask.size() < 2) |
234 | return false; |
235 | |
236 | // Check potential Factors. |
237 | for (Factor = 2; Factor <= MaxFactor; Factor++) { |
238 | // Make sure we don't produce a load wider than the input load. |
239 | if (Mask.size() * Factor > NumLoadElements) |
240 | return false; |
241 | if (isDeInterleaveMaskOfFactor(Mask, Factor, Index)) |
242 | return true; |
243 | } |
244 | |
245 | return false; |
246 | } |
247 | |
248 | /// Check if the mask can be used in an interleaved store. |
249 | // |
250 | /// It checks for a more general pattern than the RE-interleave mask. |
251 | /// I.e. <x, y, ... z, x+1, y+1, ...z+1, x+2, y+2, ...z+2, ...> |
252 | /// E.g. For a Factor of 2 (LaneLen=4): <4, 32, 5, 33, 6, 34, 7, 35> |
253 | /// E.g. For a Factor of 3 (LaneLen=4): <4, 32, 16, 5, 33, 17, 6, 34, 18, 7, 35, 19> |
254 | /// E.g. For a Factor of 4 (LaneLen=2): <8, 2, 12, 4, 9, 3, 13, 5> |
255 | /// |
256 | /// The particular case of an RE-interleave mask is: |
257 | /// I.e. <0, LaneLen, ... , LaneLen*(Factor - 1), 1, LaneLen + 1, ...> |
258 | /// E.g. For a Factor of 2 (LaneLen=4): <0, 4, 1, 5, 2, 6, 3, 7> |
259 | static bool isReInterleaveMask(ShuffleVectorInst *SVI, unsigned &Factor, |
260 | unsigned MaxFactor) { |
261 | unsigned NumElts = SVI->getShuffleMask().size(); |
262 | if (NumElts < 4) |
263 | return false; |
264 | |
265 | // Check potential Factors. |
266 | for (Factor = 2; Factor <= MaxFactor; Factor++) { |
267 | if (SVI->isInterleave(Factor)) |
268 | return true; |
269 | } |
270 | |
271 | return false; |
272 | } |
273 | |
274 | bool InterleavedAccessImpl::lowerInterleavedLoad( |
275 | LoadInst *LI, SmallVector<Instruction *, 32> &DeadInsts) { |
276 | if (!LI->isSimple() || isa<ScalableVectorType>(Val: LI->getType())) |
277 | return false; |
278 | |
279 | // Check if all users of this load are shufflevectors. If we encounter any |
280 | // users that are extractelement instructions or binary operators, we save |
281 | // them to later check if they can be modified to extract from one of the |
282 | // shufflevectors instead of the load. |
283 | |
284 | SmallVector<ShuffleVectorInst *, 4> Shuffles; |
285 | SmallVector<ExtractElementInst *, 4> ; |
286 | // BinOpShuffles need to be handled a single time in case both operands of the |
287 | // binop are the same load. |
288 | SmallSetVector<ShuffleVectorInst *, 4> BinOpShuffles; |
289 | |
290 | for (auto *User : LI->users()) { |
291 | auto * = dyn_cast<ExtractElementInst>(Val: User); |
292 | if (Extract && isa<ConstantInt>(Val: Extract->getIndexOperand())) { |
293 | Extracts.push_back(Elt: Extract); |
294 | continue; |
295 | } |
296 | if (auto *BI = dyn_cast<BinaryOperator>(Val: User)) { |
297 | if (!BI->user_empty() && all_of(Range: BI->users(), P: [](auto *U) { |
298 | auto *SVI = dyn_cast<ShuffleVectorInst>(U); |
299 | return SVI && isa<UndefValue>(SVI->getOperand(1)); |
300 | })) { |
301 | for (auto *SVI : BI->users()) |
302 | BinOpShuffles.insert(X: cast<ShuffleVectorInst>(Val: SVI)); |
303 | continue; |
304 | } |
305 | } |
306 | auto *SVI = dyn_cast<ShuffleVectorInst>(Val: User); |
307 | if (!SVI || !isa<UndefValue>(Val: SVI->getOperand(i_nocapture: 1))) |
308 | return false; |
309 | |
310 | Shuffles.push_back(Elt: SVI); |
311 | } |
312 | |
313 | if (Shuffles.empty() && BinOpShuffles.empty()) |
314 | return false; |
315 | |
316 | unsigned Factor, Index; |
317 | |
318 | unsigned NumLoadElements = |
319 | cast<FixedVectorType>(Val: LI->getType())->getNumElements(); |
320 | auto *FirstSVI = Shuffles.size() > 0 ? Shuffles[0] : BinOpShuffles[0]; |
321 | // Check if the first shufflevector is DE-interleave shuffle. |
322 | if (!isDeInterleaveMask(Mask: FirstSVI->getShuffleMask(), Factor, Index, MaxFactor, |
323 | NumLoadElements)) |
324 | return false; |
325 | |
326 | // Holds the corresponding index for each DE-interleave shuffle. |
327 | SmallVector<unsigned, 4> Indices; |
328 | |
329 | Type *VecTy = FirstSVI->getType(); |
330 | |
331 | // Check if other shufflevectors are also DE-interleaved of the same type |
332 | // and factor as the first shufflevector. |
333 | for (auto *Shuffle : Shuffles) { |
334 | if (Shuffle->getType() != VecTy) |
335 | return false; |
336 | if (!isDeInterleaveMaskOfFactor(Mask: Shuffle->getShuffleMask(), Factor, |
337 | Index)) |
338 | return false; |
339 | |
340 | assert(Shuffle->getShuffleMask().size() <= NumLoadElements); |
341 | Indices.push_back(Elt: Index); |
342 | } |
343 | for (auto *Shuffle : BinOpShuffles) { |
344 | if (Shuffle->getType() != VecTy) |
345 | return false; |
346 | if (!isDeInterleaveMaskOfFactor(Mask: Shuffle->getShuffleMask(), Factor, |
347 | Index)) |
348 | return false; |
349 | |
350 | assert(Shuffle->getShuffleMask().size() <= NumLoadElements); |
351 | |
352 | if (cast<Instruction>(Val: Shuffle->getOperand(i_nocapture: 0))->getOperand(i: 0) == LI) |
353 | Indices.push_back(Elt: Index); |
354 | if (cast<Instruction>(Val: Shuffle->getOperand(i_nocapture: 0))->getOperand(i: 1) == LI) |
355 | Indices.push_back(Elt: Index); |
356 | } |
357 | |
358 | // Try and modify users of the load that are extractelement instructions to |
359 | // use the shufflevector instructions instead of the load. |
360 | if (!tryReplaceExtracts(Extracts, Shuffles)) |
361 | return false; |
362 | |
363 | bool BinOpShuffleChanged = |
364 | replaceBinOpShuffles(BinOpShuffles: BinOpShuffles.getArrayRef(), Shuffles, LI); |
365 | |
366 | LLVM_DEBUG(dbgs() << "IA: Found an interleaved load: " << *LI << "\n" ); |
367 | |
368 | // Try to create target specific intrinsics to replace the load and shuffles. |
369 | if (!TLI->lowerInterleavedLoad(LI, Shuffles, Indices, Factor)) { |
370 | // If Extracts is not empty, tryReplaceExtracts made changes earlier. |
371 | return !Extracts.empty() || BinOpShuffleChanged; |
372 | } |
373 | |
374 | append_range(C&: DeadInsts, R&: Shuffles); |
375 | |
376 | DeadInsts.push_back(Elt: LI); |
377 | return true; |
378 | } |
379 | |
380 | bool InterleavedAccessImpl::replaceBinOpShuffles( |
381 | ArrayRef<ShuffleVectorInst *> BinOpShuffles, |
382 | SmallVectorImpl<ShuffleVectorInst *> &Shuffles, LoadInst *LI) { |
383 | for (auto *SVI : BinOpShuffles) { |
384 | BinaryOperator *BI = cast<BinaryOperator>(Val: SVI->getOperand(i_nocapture: 0)); |
385 | Type *BIOp0Ty = BI->getOperand(i_nocapture: 0)->getType(); |
386 | ArrayRef<int> Mask = SVI->getShuffleMask(); |
387 | assert(all_of(Mask, [&](int Idx) { |
388 | return Idx < (int)cast<FixedVectorType>(BIOp0Ty)->getNumElements(); |
389 | })); |
390 | |
391 | auto *NewSVI1 = |
392 | new ShuffleVectorInst(BI->getOperand(i_nocapture: 0), PoisonValue::get(T: BIOp0Ty), |
393 | Mask, SVI->getName(), SVI); |
394 | auto *NewSVI2 = new ShuffleVectorInst( |
395 | BI->getOperand(i_nocapture: 1), PoisonValue::get(T: BI->getOperand(i_nocapture: 1)->getType()), Mask, |
396 | SVI->getName(), SVI); |
397 | BinaryOperator *NewBI = BinaryOperator::CreateWithCopiedFlags( |
398 | Opc: BI->getOpcode(), V1: NewSVI1, V2: NewSVI2, CopyO: BI, Name: BI->getName(), InsertBefore: SVI); |
399 | SVI->replaceAllUsesWith(V: NewBI); |
400 | LLVM_DEBUG(dbgs() << " Replaced: " << *BI << "\n And : " << *SVI |
401 | << "\n With : " << *NewSVI1 << "\n And : " |
402 | << *NewSVI2 << "\n And : " << *NewBI << "\n" ); |
403 | RecursivelyDeleteTriviallyDeadInstructions(V: SVI); |
404 | if (NewSVI1->getOperand(i_nocapture: 0) == LI) |
405 | Shuffles.push_back(Elt: NewSVI1); |
406 | if (NewSVI2->getOperand(i_nocapture: 0) == LI) |
407 | Shuffles.push_back(Elt: NewSVI2); |
408 | } |
409 | |
410 | return !BinOpShuffles.empty(); |
411 | } |
412 | |
413 | bool InterleavedAccessImpl::( |
414 | ArrayRef<ExtractElementInst *> , |
415 | ArrayRef<ShuffleVectorInst *> Shuffles) { |
416 | // If there aren't any extractelement instructions to modify, there's nothing |
417 | // to do. |
418 | if (Extracts.empty()) |
419 | return true; |
420 | |
421 | // Maps extractelement instructions to vector-index pairs. The extractlement |
422 | // instructions will be modified to use the new vector and index operands. |
423 | DenseMap<ExtractElementInst *, std::pair<Value *, int>> ReplacementMap; |
424 | |
425 | for (auto * : Extracts) { |
426 | // The vector index that is extracted. |
427 | auto *IndexOperand = cast<ConstantInt>(Val: Extract->getIndexOperand()); |
428 | auto Index = IndexOperand->getSExtValue(); |
429 | |
430 | // Look for a suitable shufflevector instruction. The goal is to modify the |
431 | // extractelement instruction (which uses an interleaved load) to use one |
432 | // of the shufflevector instructions instead of the load. |
433 | for (auto *Shuffle : Shuffles) { |
434 | // If the shufflevector instruction doesn't dominate the extract, we |
435 | // can't create a use of it. |
436 | if (!DT->dominates(Def: Shuffle, User: Extract)) |
437 | continue; |
438 | |
439 | // Inspect the indices of the shufflevector instruction. If the shuffle |
440 | // selects the same index that is extracted, we can modify the |
441 | // extractelement instruction. |
442 | SmallVector<int, 4> Indices; |
443 | Shuffle->getShuffleMask(Result&: Indices); |
444 | for (unsigned I = 0; I < Indices.size(); ++I) |
445 | if (Indices[I] == Index) { |
446 | assert(Extract->getOperand(0) == Shuffle->getOperand(0) && |
447 | "Vector operations do not match" ); |
448 | ReplacementMap[Extract] = std::make_pair(x&: Shuffle, y&: I); |
449 | break; |
450 | } |
451 | |
452 | // If we found a suitable shufflevector instruction, stop looking. |
453 | if (ReplacementMap.count(Val: Extract)) |
454 | break; |
455 | } |
456 | |
457 | // If we did not find a suitable shufflevector instruction, the |
458 | // extractelement instruction cannot be modified, so we must give up. |
459 | if (!ReplacementMap.count(Val: Extract)) |
460 | return false; |
461 | } |
462 | |
463 | // Finally, perform the replacements. |
464 | IRBuilder<> Builder(Extracts[0]->getContext()); |
465 | for (auto &Replacement : ReplacementMap) { |
466 | auto * = Replacement.first; |
467 | auto *Vector = Replacement.second.first; |
468 | auto Index = Replacement.second.second; |
469 | Builder.SetInsertPoint(Extract); |
470 | Extract->replaceAllUsesWith(V: Builder.CreateExtractElement(Vec: Vector, Idx: Index)); |
471 | Extract->eraseFromParent(); |
472 | } |
473 | |
474 | return true; |
475 | } |
476 | |
477 | bool InterleavedAccessImpl::lowerInterleavedStore( |
478 | StoreInst *SI, SmallVector<Instruction *, 32> &DeadInsts) { |
479 | if (!SI->isSimple()) |
480 | return false; |
481 | |
482 | auto *SVI = dyn_cast<ShuffleVectorInst>(Val: SI->getValueOperand()); |
483 | if (!SVI || !SVI->hasOneUse() || isa<ScalableVectorType>(Val: SVI->getType())) |
484 | return false; |
485 | |
486 | // Check if the shufflevector is RE-interleave shuffle. |
487 | unsigned Factor; |
488 | if (!isReInterleaveMask(SVI, Factor, MaxFactor)) |
489 | return false; |
490 | |
491 | LLVM_DEBUG(dbgs() << "IA: Found an interleaved store: " << *SI << "\n" ); |
492 | |
493 | // Try to create target specific intrinsics to replace the store and shuffle. |
494 | if (!TLI->lowerInterleavedStore(SI, SVI, Factor)) |
495 | return false; |
496 | |
497 | // Already have a new target specific interleaved store. Erase the old store. |
498 | DeadInsts.push_back(Elt: SI); |
499 | DeadInsts.push_back(Elt: SVI); |
500 | return true; |
501 | } |
502 | |
503 | bool InterleavedAccessImpl::lowerDeinterleaveIntrinsic( |
504 | IntrinsicInst *DI, SmallVector<Instruction *, 32> &DeadInsts) { |
505 | LoadInst *LI = dyn_cast<LoadInst>(Val: DI->getOperand(i_nocapture: 0)); |
506 | |
507 | if (!LI || !LI->hasOneUse() || !LI->isSimple()) |
508 | return false; |
509 | |
510 | LLVM_DEBUG(dbgs() << "IA: Found a deinterleave intrinsic: " << *DI << "\n" ); |
511 | |
512 | // Try and match this with target specific intrinsics. |
513 | if (!TLI->lowerDeinterleaveIntrinsicToLoad(DI, LI)) |
514 | return false; |
515 | |
516 | // We now have a target-specific load, so delete the old one. |
517 | DeadInsts.push_back(Elt: DI); |
518 | DeadInsts.push_back(Elt: LI); |
519 | return true; |
520 | } |
521 | |
522 | bool InterleavedAccessImpl::lowerInterleaveIntrinsic( |
523 | IntrinsicInst *II, SmallVector<Instruction *, 32> &DeadInsts) { |
524 | if (!II->hasOneUse()) |
525 | return false; |
526 | |
527 | StoreInst *SI = dyn_cast<StoreInst>(Val: *(II->users().begin())); |
528 | |
529 | if (!SI || !SI->isSimple()) |
530 | return false; |
531 | |
532 | LLVM_DEBUG(dbgs() << "IA: Found an interleave intrinsic: " << *II << "\n" ); |
533 | |
534 | // Try and match this with target specific intrinsics. |
535 | if (!TLI->lowerInterleaveIntrinsicToStore(II, SI)) |
536 | return false; |
537 | |
538 | // We now have a target-specific store, so delete the old one. |
539 | DeadInsts.push_back(Elt: SI); |
540 | DeadInsts.push_back(Elt: II); |
541 | return true; |
542 | } |
543 | |
544 | bool InterleavedAccessImpl::runOnFunction(Function &F) { |
545 | // Holds dead instructions that will be erased later. |
546 | SmallVector<Instruction *, 32> DeadInsts; |
547 | bool Changed = false; |
548 | |
549 | for (auto &I : instructions(F)) { |
550 | if (auto *LI = dyn_cast<LoadInst>(Val: &I)) |
551 | Changed |= lowerInterleavedLoad(LI, DeadInsts); |
552 | |
553 | if (auto *SI = dyn_cast<StoreInst>(Val: &I)) |
554 | Changed |= lowerInterleavedStore(SI, DeadInsts); |
555 | |
556 | if (auto *II = dyn_cast<IntrinsicInst>(Val: &I)) { |
557 | // At present, we only have intrinsics to represent (de)interleaving |
558 | // with a factor of 2. |
559 | if (II->getIntrinsicID() == Intrinsic::experimental_vector_deinterleave2) |
560 | Changed |= lowerDeinterleaveIntrinsic(DI: II, DeadInsts); |
561 | if (II->getIntrinsicID() == Intrinsic::experimental_vector_interleave2) |
562 | Changed |= lowerInterleaveIntrinsic(II, DeadInsts); |
563 | } |
564 | } |
565 | |
566 | for (auto *I : DeadInsts) |
567 | I->eraseFromParent(); |
568 | |
569 | return Changed; |
570 | } |
571 | |