1 | //==-- llvm/CodeGen/ExecutionDomainFix.h - Execution Domain Fix -*- C++ -*--==// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | /// \file Execution Domain Fix pass. |
10 | /// |
11 | /// Some X86 SSE instructions like mov, and, or, xor are available in different |
12 | /// variants for different operand types. These variant instructions are |
13 | /// equivalent, but on Nehalem and newer cpus there is extra latency |
14 | /// transferring data between integer and floating point domains. ARM cores |
15 | /// have similar issues when they are configured with both VFP and NEON |
16 | /// pipelines. |
17 | /// |
18 | /// This pass changes the variant instructions to minimize domain crossings. |
19 | // |
20 | //===----------------------------------------------------------------------===// |
21 | |
22 | #ifndef LLVM_CODEGEN_EXECUTIONDOMAINFIX_H |
23 | #define LLVM_CODEGEN_EXECUTIONDOMAINFIX_H |
24 | |
25 | #include "llvm/ADT/SmallVector.h" |
26 | #include "llvm/CodeGen/LoopTraversal.h" |
27 | #include "llvm/CodeGen/MachineFunctionPass.h" |
28 | #include "llvm/CodeGen/ReachingDefAnalysis.h" |
29 | #include "llvm/CodeGen/TargetRegisterInfo.h" |
30 | |
31 | namespace llvm { |
32 | |
33 | class MachineInstr; |
34 | class TargetInstrInfo; |
35 | |
36 | /// A DomainValue is a bit like LiveIntervals' ValNo, but it also keeps track |
37 | /// of execution domains. |
38 | /// |
39 | /// An open DomainValue represents a set of instructions that can still switch |
40 | /// execution domain. Multiple registers may refer to the same open |
41 | /// DomainValue - they will eventually be collapsed to the same execution |
42 | /// domain. |
43 | /// |
44 | /// A collapsed DomainValue represents a single register that has been forced |
45 | /// into one of more execution domains. There is a separate collapsed |
46 | /// DomainValue for each register, but it may contain multiple execution |
47 | /// domains. A register value is initially created in a single execution |
48 | /// domain, but if we were forced to pay the penalty of a domain crossing, we |
49 | /// keep track of the fact that the register is now available in multiple |
50 | /// domains. |
51 | struct DomainValue { |
52 | /// Basic reference counting. |
53 | unsigned Refs = 0; |
54 | |
55 | /// Bitmask of available domains. For an open DomainValue, it is the still |
56 | /// possible domains for collapsing. For a collapsed DomainValue it is the |
57 | /// domains where the register is available for free. |
58 | unsigned AvailableDomains; |
59 | |
60 | /// Pointer to the next DomainValue in a chain. When two DomainValues are |
61 | /// merged, Victim.Next is set to point to Victor, so old DomainValue |
62 | /// references can be updated by following the chain. |
63 | DomainValue *Next; |
64 | |
65 | /// Twiddleable instructions using or defining these registers. |
66 | SmallVector<MachineInstr *, 8> Instrs; |
67 | |
68 | DomainValue() { clear(); } |
69 | |
70 | /// A collapsed DomainValue has no instructions to twiddle - it simply keeps |
71 | /// track of the domains where the registers are already available. |
72 | bool isCollapsed() const { return Instrs.empty(); } |
73 | |
74 | /// Is domain available? |
75 | bool hasDomain(unsigned domain) const { |
76 | assert(domain < |
77 | static_cast<unsigned>(std::numeric_limits<unsigned>::digits) && |
78 | "undefined behavior" ); |
79 | return AvailableDomains & (1u << domain); |
80 | } |
81 | |
82 | /// Mark domain as available. |
83 | void addDomain(unsigned domain) { |
84 | assert(domain < |
85 | static_cast<unsigned>(std::numeric_limits<unsigned>::digits) && |
86 | "undefined behavior" ); |
87 | AvailableDomains |= 1u << domain; |
88 | } |
89 | |
90 | // Restrict to a single domain available. |
91 | void setSingleDomain(unsigned domain) { |
92 | assert(domain < |
93 | static_cast<unsigned>(std::numeric_limits<unsigned>::digits) && |
94 | "undefined behavior" ); |
95 | AvailableDomains = 1u << domain; |
96 | } |
97 | |
98 | /// Return bitmask of domains that are available and in mask. |
99 | unsigned getCommonDomains(unsigned mask) const { |
100 | return AvailableDomains & mask; |
101 | } |
102 | |
103 | /// First domain available. |
104 | unsigned getFirstDomain() const { |
105 | return llvm::countr_zero(Val: AvailableDomains); |
106 | } |
107 | |
108 | /// Clear this DomainValue and point to next which has all its data. |
109 | void clear() { |
110 | AvailableDomains = 0; |
111 | Next = nullptr; |
112 | Instrs.clear(); |
113 | } |
114 | }; |
115 | |
116 | class ExecutionDomainFix : public MachineFunctionPass { |
117 | SpecificBumpPtrAllocator<DomainValue> Allocator; |
118 | SmallVector<DomainValue *, 16> Avail; |
119 | |
120 | const TargetRegisterClass *const RC; |
121 | MachineFunction *MF = nullptr; |
122 | const TargetInstrInfo *TII = nullptr; |
123 | const TargetRegisterInfo *TRI = nullptr; |
124 | std::vector<SmallVector<int, 1>> AliasMap; |
125 | const unsigned NumRegs; |
126 | /// Value currently in each register, or NULL when no value is being tracked. |
127 | /// This counts as a DomainValue reference. |
128 | using LiveRegsDVInfo = std::vector<DomainValue *>; |
129 | LiveRegsDVInfo LiveRegs; |
130 | /// Keeps domain information for all registers. Note that this |
131 | /// is different from the usual definition notion of liveness. The CPU |
132 | /// doesn't care whether or not we consider a register killed. |
133 | using OutRegsInfoMap = SmallVector<LiveRegsDVInfo, 4>; |
134 | OutRegsInfoMap MBBOutRegsInfos; |
135 | |
136 | ReachingDefAnalysis *RDA = nullptr; |
137 | |
138 | public: |
139 | ExecutionDomainFix(char &PassID, const TargetRegisterClass &RC) |
140 | : MachineFunctionPass(PassID), RC(&RC), NumRegs(RC.getNumRegs()) {} |
141 | |
142 | void getAnalysisUsage(AnalysisUsage &AU) const override { |
143 | AU.setPreservesAll(); |
144 | AU.addRequired<ReachingDefAnalysis>(); |
145 | MachineFunctionPass::getAnalysisUsage(AU); |
146 | } |
147 | |
148 | bool runOnMachineFunction(MachineFunction &MF) override; |
149 | |
150 | MachineFunctionProperties getRequiredProperties() const override { |
151 | return MachineFunctionProperties().set( |
152 | MachineFunctionProperties::Property::NoVRegs); |
153 | } |
154 | |
155 | private: |
156 | /// Translate TRI register number to a list of indices into our smaller tables |
157 | /// of interesting registers. |
158 | iterator_range<SmallVectorImpl<int>::const_iterator> |
159 | regIndices(unsigned Reg) const; |
160 | |
161 | /// DomainValue allocation. |
162 | DomainValue *alloc(int domain = -1); |
163 | |
164 | /// Add reference to DV. |
165 | DomainValue *retain(DomainValue *DV) { |
166 | if (DV) |
167 | ++DV->Refs; |
168 | return DV; |
169 | } |
170 | |
171 | /// Release a reference to DV. When the last reference is released, |
172 | /// collapse if needed. |
173 | void release(DomainValue *); |
174 | |
175 | /// Follow the chain of dead DomainValues until a live DomainValue is reached. |
176 | /// Update the referenced pointer when necessary. |
177 | DomainValue *resolve(DomainValue *&); |
178 | |
179 | /// Set LiveRegs[rx] = dv, updating reference counts. |
180 | void setLiveReg(int rx, DomainValue *DV); |
181 | |
182 | /// Kill register rx, recycle or collapse any DomainValue. |
183 | void kill(int rx); |
184 | |
185 | /// Force register rx into domain. |
186 | void force(int rx, unsigned domain); |
187 | |
188 | /// Collapse open DomainValue into given domain. If there are multiple |
189 | /// registers using dv, they each get a unique collapsed DomainValue. |
190 | void collapse(DomainValue *dv, unsigned domain); |
191 | |
192 | /// All instructions and registers in B are moved to A, and B is released. |
193 | bool merge(DomainValue *A, DomainValue *B); |
194 | |
195 | /// Set up LiveRegs by merging predecessor live-out values. |
196 | void enterBasicBlock(const LoopTraversal::TraversedMBBInfo &TraversedMBB); |
197 | |
198 | /// Update live-out values. |
199 | void leaveBasicBlock(const LoopTraversal::TraversedMBBInfo &TraversedMBB); |
200 | |
201 | /// Process he given basic block. |
202 | void processBasicBlock(const LoopTraversal::TraversedMBBInfo &TraversedMBB); |
203 | |
204 | /// Visit given insturcion. |
205 | bool visitInstr(MachineInstr *); |
206 | |
207 | /// Update def-ages for registers defined by MI. |
208 | /// If Kill is set, also kill off DomainValues clobbered by the defs. |
209 | void processDefs(MachineInstr *, bool Kill); |
210 | |
211 | /// A soft instruction can be changed to work in other domains given by mask. |
212 | void visitSoftInstr(MachineInstr *, unsigned mask); |
213 | |
214 | /// A hard instruction only works in one domain. All input registers will be |
215 | /// forced into that domain. |
216 | void visitHardInstr(MachineInstr *, unsigned domain); |
217 | }; |
218 | |
219 | } // namespace llvm |
220 | |
221 | #endif // LLVM_CODEGEN_EXECUTIONDOMAINFIX_H |
222 | |