1 | //===-- lib/CodeGen/MachineInstrBundle.cpp --------------------------------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | |
9 | #include "llvm/CodeGen/MachineInstrBundle.h" |
10 | #include "llvm/ADT/SmallSet.h" |
11 | #include "llvm/ADT/SmallVector.h" |
12 | #include "llvm/CodeGen/MachineFunctionPass.h" |
13 | #include "llvm/CodeGen/MachineInstrBuilder.h" |
14 | #include "llvm/CodeGen/Passes.h" |
15 | #include "llvm/CodeGen/TargetInstrInfo.h" |
16 | #include "llvm/CodeGen/TargetRegisterInfo.h" |
17 | #include "llvm/CodeGen/TargetSubtargetInfo.h" |
18 | #include "llvm/InitializePasses.h" |
19 | #include "llvm/Pass.h" |
20 | #include "llvm/PassRegistry.h" |
21 | #include <utility> |
22 | using namespace llvm; |
23 | |
24 | namespace { |
25 | class UnpackMachineBundles : public MachineFunctionPass { |
26 | public: |
27 | static char ID; // Pass identification |
28 | UnpackMachineBundles( |
29 | std::function<bool(const MachineFunction &)> Ftor = nullptr) |
30 | : MachineFunctionPass(ID), PredicateFtor(std::move(Ftor)) { |
31 | initializeUnpackMachineBundlesPass(*PassRegistry::getPassRegistry()); |
32 | } |
33 | |
34 | bool runOnMachineFunction(MachineFunction &MF) override; |
35 | |
36 | private: |
37 | std::function<bool(const MachineFunction &)> PredicateFtor; |
38 | }; |
39 | } // end anonymous namespace |
40 | |
41 | char UnpackMachineBundles::ID = 0; |
42 | char &llvm::UnpackMachineBundlesID = UnpackMachineBundles::ID; |
43 | INITIALIZE_PASS(UnpackMachineBundles, "unpack-mi-bundles" , |
44 | "Unpack machine instruction bundles" , false, false) |
45 | |
46 | bool UnpackMachineBundles::runOnMachineFunction(MachineFunction &MF) { |
47 | if (PredicateFtor && !PredicateFtor(MF)) |
48 | return false; |
49 | |
50 | bool Changed = false; |
51 | for (MachineBasicBlock &MBB : MF) { |
52 | for (MachineBasicBlock::instr_iterator MII = MBB.instr_begin(), |
53 | MIE = MBB.instr_end(); MII != MIE; ) { |
54 | MachineInstr *MI = &*MII; |
55 | |
56 | // Remove BUNDLE instruction and the InsideBundle flags from bundled |
57 | // instructions. |
58 | if (MI->isBundle()) { |
59 | while (++MII != MIE && MII->isBundledWithPred()) { |
60 | MII->unbundleFromPred(); |
61 | for (MachineOperand &MO : MII->operands()) { |
62 | if (MO.isReg() && MO.isInternalRead()) |
63 | MO.setIsInternalRead(false); |
64 | } |
65 | } |
66 | MI->eraseFromParent(); |
67 | |
68 | Changed = true; |
69 | continue; |
70 | } |
71 | |
72 | ++MII; |
73 | } |
74 | } |
75 | |
76 | return Changed; |
77 | } |
78 | |
79 | FunctionPass * |
80 | llvm::createUnpackMachineBundles( |
81 | std::function<bool(const MachineFunction &)> Ftor) { |
82 | return new UnpackMachineBundles(std::move(Ftor)); |
83 | } |
84 | |
85 | namespace { |
86 | class FinalizeMachineBundles : public MachineFunctionPass { |
87 | public: |
88 | static char ID; // Pass identification |
89 | FinalizeMachineBundles() : MachineFunctionPass(ID) { |
90 | initializeFinalizeMachineBundlesPass(*PassRegistry::getPassRegistry()); |
91 | } |
92 | |
93 | bool runOnMachineFunction(MachineFunction &MF) override; |
94 | }; |
95 | } // end anonymous namespace |
96 | |
97 | char FinalizeMachineBundles::ID = 0; |
98 | char &llvm::FinalizeMachineBundlesID = FinalizeMachineBundles::ID; |
99 | INITIALIZE_PASS(FinalizeMachineBundles, "finalize-mi-bundles" , |
100 | "Finalize machine instruction bundles" , false, false) |
101 | |
102 | bool FinalizeMachineBundles::runOnMachineFunction(MachineFunction &MF) { |
103 | return llvm::finalizeBundles(MF); |
104 | } |
105 | |
106 | /// Return the first found DebugLoc that has a DILocation, given a range of |
107 | /// instructions. The search range is from FirstMI to LastMI (exclusive). If no |
108 | /// DILocation is found, then an empty location is returned. |
109 | static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, |
110 | MachineBasicBlock::instr_iterator LastMI) { |
111 | for (auto MII = FirstMI; MII != LastMI; ++MII) |
112 | if (MII->getDebugLoc()) |
113 | return MII->getDebugLoc(); |
114 | return DebugLoc(); |
115 | } |
116 | |
117 | /// finalizeBundle - Finalize a machine instruction bundle which includes |
118 | /// a sequence of instructions starting from FirstMI to LastMI (exclusive). |
119 | /// This routine adds a BUNDLE instruction to represent the bundle, it adds |
120 | /// IsInternalRead markers to MachineOperands which are defined inside the |
121 | /// bundle, and it copies externally visible defs and uses to the BUNDLE |
122 | /// instruction. |
123 | void llvm::finalizeBundle(MachineBasicBlock &MBB, |
124 | MachineBasicBlock::instr_iterator FirstMI, |
125 | MachineBasicBlock::instr_iterator LastMI) { |
126 | assert(FirstMI != LastMI && "Empty bundle?" ); |
127 | MIBundleBuilder Bundle(MBB, FirstMI, LastMI); |
128 | |
129 | MachineFunction &MF = *MBB.getParent(); |
130 | const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo(); |
131 | const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); |
132 | |
133 | MachineInstrBuilder MIB = |
134 | BuildMI(MF, MIMD: getDebugLoc(FirstMI, LastMI), MCID: TII->get(Opcode: TargetOpcode::BUNDLE)); |
135 | Bundle.prepend(MI: MIB); |
136 | |
137 | SmallVector<Register, 32> LocalDefs; |
138 | SmallSet<Register, 32> LocalDefSet; |
139 | SmallSet<Register, 8> DeadDefSet; |
140 | SmallSet<Register, 16> KilledDefSet; |
141 | SmallVector<Register, 8> ExternUses; |
142 | SmallSet<Register, 8> ExternUseSet; |
143 | SmallSet<Register, 8> KilledUseSet; |
144 | SmallSet<Register, 8> UndefUseSet; |
145 | SmallVector<MachineOperand*, 4> Defs; |
146 | for (auto MII = FirstMI; MII != LastMI; ++MII) { |
147 | // Debug instructions have no effects to track. |
148 | if (MII->isDebugInstr()) |
149 | continue; |
150 | |
151 | for (MachineOperand &MO : MII->operands()) { |
152 | if (!MO.isReg()) |
153 | continue; |
154 | if (MO.isDef()) { |
155 | Defs.push_back(Elt: &MO); |
156 | continue; |
157 | } |
158 | |
159 | Register Reg = MO.getReg(); |
160 | if (!Reg) |
161 | continue; |
162 | |
163 | if (LocalDefSet.count(V: Reg)) { |
164 | MO.setIsInternalRead(); |
165 | if (MO.isKill()) |
166 | // Internal def is now killed. |
167 | KilledDefSet.insert(V: Reg); |
168 | } else { |
169 | if (ExternUseSet.insert(V: Reg).second) { |
170 | ExternUses.push_back(Elt: Reg); |
171 | if (MO.isUndef()) |
172 | UndefUseSet.insert(V: Reg); |
173 | } |
174 | if (MO.isKill()) |
175 | // External def is now killed. |
176 | KilledUseSet.insert(V: Reg); |
177 | } |
178 | } |
179 | |
180 | for (unsigned i = 0, e = Defs.size(); i != e; ++i) { |
181 | MachineOperand &MO = *Defs[i]; |
182 | Register Reg = MO.getReg(); |
183 | if (!Reg) |
184 | continue; |
185 | |
186 | if (LocalDefSet.insert(V: Reg).second) { |
187 | LocalDefs.push_back(Elt: Reg); |
188 | if (MO.isDead()) { |
189 | DeadDefSet.insert(V: Reg); |
190 | } |
191 | } else { |
192 | // Re-defined inside the bundle, it's no longer killed. |
193 | KilledDefSet.erase(V: Reg); |
194 | if (!MO.isDead()) |
195 | // Previously defined but dead. |
196 | DeadDefSet.erase(V: Reg); |
197 | } |
198 | |
199 | if (!MO.isDead() && Reg.isPhysical()) { |
200 | for (MCPhysReg SubReg : TRI->subregs(Reg)) { |
201 | if (LocalDefSet.insert(V: SubReg).second) |
202 | LocalDefs.push_back(Elt: SubReg); |
203 | } |
204 | } |
205 | } |
206 | |
207 | Defs.clear(); |
208 | } |
209 | |
210 | SmallSet<Register, 32> Added; |
211 | for (Register Reg : LocalDefs) { |
212 | if (Added.insert(V: Reg).second) { |
213 | // If it's not live beyond end of the bundle, mark it dead. |
214 | bool isDead = DeadDefSet.count(V: Reg) || KilledDefSet.count(V: Reg); |
215 | MIB.addReg(RegNo: Reg, flags: getDefRegState(B: true) | getDeadRegState(B: isDead) | |
216 | getImplRegState(B: true)); |
217 | } |
218 | } |
219 | |
220 | for (Register Reg : ExternUses) { |
221 | bool isKill = KilledUseSet.count(V: Reg); |
222 | bool isUndef = UndefUseSet.count(V: Reg); |
223 | MIB.addReg(RegNo: Reg, flags: getKillRegState(B: isKill) | getUndefRegState(B: isUndef) | |
224 | getImplRegState(B: true)); |
225 | } |
226 | |
227 | // Set FrameSetup/FrameDestroy for the bundle. If any of the instructions got |
228 | // the property, then also set it on the bundle. |
229 | for (auto MII = FirstMI; MII != LastMI; ++MII) { |
230 | if (MII->getFlag(Flag: MachineInstr::FrameSetup)) |
231 | MIB.setMIFlag(MachineInstr::FrameSetup); |
232 | if (MII->getFlag(Flag: MachineInstr::FrameDestroy)) |
233 | MIB.setMIFlag(MachineInstr::FrameDestroy); |
234 | } |
235 | } |
236 | |
237 | /// finalizeBundle - Same functionality as the previous finalizeBundle except |
238 | /// the last instruction in the bundle is not provided as an input. This is |
239 | /// used in cases where bundles are pre-determined by marking instructions |
240 | /// with 'InsideBundle' marker. It returns the MBB instruction iterator that |
241 | /// points to the end of the bundle. |
242 | MachineBasicBlock::instr_iterator |
243 | llvm::finalizeBundle(MachineBasicBlock &MBB, |
244 | MachineBasicBlock::instr_iterator FirstMI) { |
245 | MachineBasicBlock::instr_iterator E = MBB.instr_end(); |
246 | MachineBasicBlock::instr_iterator LastMI = std::next(x: FirstMI); |
247 | while (LastMI != E && LastMI->isInsideBundle()) |
248 | ++LastMI; |
249 | finalizeBundle(MBB, FirstMI, LastMI); |
250 | return LastMI; |
251 | } |
252 | |
253 | /// finalizeBundles - Finalize instruction bundles in the specified |
254 | /// MachineFunction. Return true if any bundles are finalized. |
255 | bool llvm::finalizeBundles(MachineFunction &MF) { |
256 | bool Changed = false; |
257 | for (MachineBasicBlock &MBB : MF) { |
258 | MachineBasicBlock::instr_iterator MII = MBB.instr_begin(); |
259 | MachineBasicBlock::instr_iterator MIE = MBB.instr_end(); |
260 | if (MII == MIE) |
261 | continue; |
262 | assert(!MII->isInsideBundle() && |
263 | "First instr cannot be inside bundle before finalization!" ); |
264 | |
265 | for (++MII; MII != MIE; ) { |
266 | if (!MII->isInsideBundle()) |
267 | ++MII; |
268 | else { |
269 | MII = finalizeBundle(MBB, FirstMI: std::prev(x: MII)); |
270 | Changed = true; |
271 | } |
272 | } |
273 | } |
274 | |
275 | return Changed; |
276 | } |
277 | |
278 | VirtRegInfo llvm::AnalyzeVirtRegInBundle( |
279 | MachineInstr &MI, Register Reg, |
280 | SmallVectorImpl<std::pair<MachineInstr *, unsigned>> *Ops) { |
281 | VirtRegInfo RI = {.Reads: false, .Writes: false, .Tied: false}; |
282 | for (MIBundleOperands O(MI); O.isValid(); ++O) { |
283 | MachineOperand &MO = *O; |
284 | if (!MO.isReg() || MO.getReg() != Reg) |
285 | continue; |
286 | |
287 | // Remember each (MI, OpNo) that refers to Reg. |
288 | if (Ops) |
289 | Ops->push_back(Elt: std::make_pair(x: MO.getParent(), y: O.getOperandNo())); |
290 | |
291 | // Both defs and uses can read virtual registers. |
292 | if (MO.readsReg()) { |
293 | RI.Reads = true; |
294 | if (MO.isDef()) |
295 | RI.Tied = true; |
296 | } |
297 | |
298 | // Only defs can write. |
299 | if (MO.isDef()) |
300 | RI.Writes = true; |
301 | else if (!RI.Tied && |
302 | MO.getParent()->isRegTiedToDefOperand(UseOpIdx: O.getOperandNo())) |
303 | RI.Tied = true; |
304 | } |
305 | return RI; |
306 | } |
307 | |
308 | std::pair<LaneBitmask, LaneBitmask> |
309 | llvm::AnalyzeVirtRegLanesInBundle(const MachineInstr &MI, Register Reg, |
310 | const MachineRegisterInfo &MRI, |
311 | const TargetRegisterInfo &TRI) { |
312 | |
313 | LaneBitmask UseMask, DefMask; |
314 | |
315 | for (ConstMIBundleOperands O(MI); O.isValid(); ++O) { |
316 | const MachineOperand &MO = *O; |
317 | if (!MO.isReg() || MO.getReg() != Reg) |
318 | continue; |
319 | |
320 | unsigned SubReg = MO.getSubReg(); |
321 | if (SubReg == 0 && MO.isUse() && !MO.isUndef()) |
322 | UseMask |= MRI.getMaxLaneMaskForVReg(Reg); |
323 | |
324 | LaneBitmask SubRegMask = TRI.getSubRegIndexLaneMask(SubIdx: SubReg); |
325 | if (MO.isDef()) { |
326 | if (!MO.isUndef()) |
327 | UseMask |= ~SubRegMask; |
328 | DefMask |= SubRegMask; |
329 | } else if (!MO.isUndef()) |
330 | UseMask |= SubRegMask; |
331 | } |
332 | |
333 | return {UseMask, DefMask}; |
334 | } |
335 | |
336 | PhysRegInfo llvm::AnalyzePhysRegInBundle(const MachineInstr &MI, Register Reg, |
337 | const TargetRegisterInfo *TRI) { |
338 | bool AllDefsDead = true; |
339 | PhysRegInfo PRI = {.Clobbered: false, .Defined: false, .FullyDefined: false, .Read: false, .FullyRead: false, .DeadDef: false, .PartialDeadDef: false, .Killed: false}; |
340 | |
341 | assert(Reg.isPhysical() && "analyzePhysReg not given a physical register!" ); |
342 | for (ConstMIBundleOperands O(MI); O.isValid(); ++O) { |
343 | const MachineOperand &MO = *O; |
344 | |
345 | if (MO.isRegMask() && MO.clobbersPhysReg(PhysReg: Reg)) { |
346 | PRI.Clobbered = true; |
347 | continue; |
348 | } |
349 | |
350 | if (!MO.isReg()) |
351 | continue; |
352 | |
353 | Register MOReg = MO.getReg(); |
354 | if (!MOReg || !MOReg.isPhysical()) |
355 | continue; |
356 | |
357 | if (!TRI->regsOverlap(RegA: MOReg, RegB: Reg)) |
358 | continue; |
359 | |
360 | bool Covered = TRI->isSuperRegisterEq(RegA: Reg, RegB: MOReg); |
361 | if (MO.readsReg()) { |
362 | PRI.Read = true; |
363 | if (Covered) { |
364 | PRI.FullyRead = true; |
365 | if (MO.isKill()) |
366 | PRI.Killed = true; |
367 | } |
368 | } else if (MO.isDef()) { |
369 | PRI.Defined = true; |
370 | if (Covered) |
371 | PRI.FullyDefined = true; |
372 | if (!MO.isDead()) |
373 | AllDefsDead = false; |
374 | } |
375 | } |
376 | |
377 | if (AllDefsDead) { |
378 | if (PRI.FullyDefined || PRI.Clobbered) |
379 | PRI.DeadDef = true; |
380 | else if (PRI.Defined) |
381 | PRI.PartialDeadDef = true; |
382 | } |
383 | |
384 | return PRI; |
385 | } |
386 | |