1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Intel I/OAT DMA Linux driver |
4 | * Copyright(c) 2007 - 2009 Intel Corporation. |
5 | */ |
6 | |
7 | #include <linux/kernel.h> |
8 | #include <linux/pci.h> |
9 | #include <linux/smp.h> |
10 | #include <linux/interrupt.h> |
11 | #include <linux/dca.h> |
12 | |
13 | /* either a kernel change is needed, or we need something like this in kernel */ |
14 | #ifndef CONFIG_SMP |
15 | #include <asm/smp.h> |
16 | #undef cpu_physical_id |
17 | #define cpu_physical_id(cpu) (cpuid_ebx(1) >> 24) |
18 | #endif |
19 | |
20 | #include "dma.h" |
21 | #include "registers.h" |
22 | |
23 | /* |
24 | * Bit 7 of a tag map entry is the "valid" bit, if it is set then bits 0:6 |
25 | * contain the bit number of the APIC ID to map into the DCA tag. If the valid |
26 | * bit is not set, then the value must be 0 or 1 and defines the bit in the tag. |
27 | */ |
28 | #define DCA_TAG_MAP_VALID 0x80 |
29 | |
30 | #define DCA3_TAG_MAP_BIT_TO_INV 0x80 |
31 | #define DCA3_TAG_MAP_BIT_TO_SEL 0x40 |
32 | #define DCA3_TAG_MAP_LITERAL_VAL 0x1 |
33 | |
34 | #define DCA_TAG_MAP_MASK 0xDF |
35 | |
36 | /* expected tag map bytes for I/OAT ver.2 */ |
37 | #define DCA2_TAG_MAP_BYTE0 0x80 |
38 | #define DCA2_TAG_MAP_BYTE1 0x0 |
39 | #define DCA2_TAG_MAP_BYTE2 0x81 |
40 | #define DCA2_TAG_MAP_BYTE3 0x82 |
41 | #define DCA2_TAG_MAP_BYTE4 0x82 |
42 | |
43 | /* |
44 | * "Legacy" DCA systems do not implement the DCA register set in the |
45 | * I/OAT device. Software needs direct support for their tag mappings. |
46 | */ |
47 | |
48 | #define APICID_BIT(x) (DCA_TAG_MAP_VALID | (x)) |
49 | #define IOAT_TAG_MAP_LEN 8 |
50 | |
51 | /* pack PCI B/D/F into a u16 */ |
52 | static inline u16 dcaid_from_pcidev(struct pci_dev *pci) |
53 | { |
54 | return pci_dev_id(dev: pci); |
55 | } |
56 | |
57 | static int dca_enabled_in_bios(struct pci_dev *pdev) |
58 | { |
59 | /* CPUID level 9 returns DCA configuration */ |
60 | /* Bit 0 indicates DCA enabled by the BIOS */ |
61 | unsigned long cpuid_level_9; |
62 | int res; |
63 | |
64 | cpuid_level_9 = cpuid_eax(op: 9); |
65 | res = test_bit(0, &cpuid_level_9); |
66 | if (!res) |
67 | dev_dbg(&pdev->dev, "DCA is disabled in BIOS\n" ); |
68 | |
69 | return res; |
70 | } |
71 | |
72 | int system_has_dca_enabled(struct pci_dev *pdev) |
73 | { |
74 | if (boot_cpu_has(X86_FEATURE_DCA)) |
75 | return dca_enabled_in_bios(pdev); |
76 | |
77 | dev_dbg(&pdev->dev, "boot cpu doesn't have X86_FEATURE_DCA\n" ); |
78 | return 0; |
79 | } |
80 | |
81 | struct ioat_dca_slot { |
82 | struct pci_dev *pdev; /* requester device */ |
83 | u16 rid; /* requester id, as used by IOAT */ |
84 | }; |
85 | |
86 | #define IOAT_DCA_MAX_REQ 6 |
87 | #define IOAT3_DCA_MAX_REQ 2 |
88 | |
89 | struct ioat_dca_priv { |
90 | void __iomem *iobase; |
91 | void __iomem *dca_base; |
92 | int max_requesters; |
93 | int requester_count; |
94 | u8 tag_map[IOAT_TAG_MAP_LEN]; |
95 | struct ioat_dca_slot req_slots[]; |
96 | }; |
97 | |
98 | static int ioat_dca_dev_managed(struct dca_provider *dca, |
99 | struct device *dev) |
100 | { |
101 | struct ioat_dca_priv *ioatdca = dca_priv(dca); |
102 | struct pci_dev *pdev; |
103 | int i; |
104 | |
105 | pdev = to_pci_dev(dev); |
106 | for (i = 0; i < ioatdca->max_requesters; i++) { |
107 | if (ioatdca->req_slots[i].pdev == pdev) |
108 | return 1; |
109 | } |
110 | return 0; |
111 | } |
112 | |
113 | static int ioat_dca_add_requester(struct dca_provider *dca, struct device *dev) |
114 | { |
115 | struct ioat_dca_priv *ioatdca = dca_priv(dca); |
116 | struct pci_dev *pdev; |
117 | int i; |
118 | u16 id; |
119 | u16 global_req_table; |
120 | |
121 | /* This implementation only supports PCI-Express */ |
122 | if (!dev_is_pci(dev)) |
123 | return -ENODEV; |
124 | pdev = to_pci_dev(dev); |
125 | id = dcaid_from_pcidev(pci: pdev); |
126 | |
127 | if (ioatdca->requester_count == ioatdca->max_requesters) |
128 | return -ENODEV; |
129 | |
130 | for (i = 0; i < ioatdca->max_requesters; i++) { |
131 | if (ioatdca->req_slots[i].pdev == NULL) { |
132 | /* found an empty slot */ |
133 | ioatdca->requester_count++; |
134 | ioatdca->req_slots[i].pdev = pdev; |
135 | ioatdca->req_slots[i].rid = id; |
136 | global_req_table = |
137 | readw(addr: ioatdca->dca_base + IOAT3_DCA_GREQID_OFFSET); |
138 | writel(val: id | IOAT_DCA_GREQID_VALID, |
139 | addr: ioatdca->iobase + global_req_table + (i * 4)); |
140 | return i; |
141 | } |
142 | } |
143 | /* Error, ioatdma->requester_count is out of whack */ |
144 | return -EFAULT; |
145 | } |
146 | |
147 | static int ioat_dca_remove_requester(struct dca_provider *dca, |
148 | struct device *dev) |
149 | { |
150 | struct ioat_dca_priv *ioatdca = dca_priv(dca); |
151 | struct pci_dev *pdev; |
152 | int i; |
153 | u16 global_req_table; |
154 | |
155 | /* This implementation only supports PCI-Express */ |
156 | if (!dev_is_pci(dev)) |
157 | return -ENODEV; |
158 | pdev = to_pci_dev(dev); |
159 | |
160 | for (i = 0; i < ioatdca->max_requesters; i++) { |
161 | if (ioatdca->req_slots[i].pdev == pdev) { |
162 | global_req_table = |
163 | readw(addr: ioatdca->dca_base + IOAT3_DCA_GREQID_OFFSET); |
164 | writel(val: 0, addr: ioatdca->iobase + global_req_table + (i * 4)); |
165 | ioatdca->req_slots[i].pdev = NULL; |
166 | ioatdca->req_slots[i].rid = 0; |
167 | ioatdca->requester_count--; |
168 | return i; |
169 | } |
170 | } |
171 | return -ENODEV; |
172 | } |
173 | |
174 | static u8 ioat_dca_get_tag(struct dca_provider *dca, |
175 | struct device *dev, |
176 | int cpu) |
177 | { |
178 | u8 tag; |
179 | |
180 | struct ioat_dca_priv *ioatdca = dca_priv(dca); |
181 | int i, apic_id, bit, value; |
182 | u8 entry; |
183 | |
184 | tag = 0; |
185 | apic_id = cpu_physical_id(cpu); |
186 | |
187 | for (i = 0; i < IOAT_TAG_MAP_LEN; i++) { |
188 | entry = ioatdca->tag_map[i]; |
189 | if (entry & DCA3_TAG_MAP_BIT_TO_SEL) { |
190 | bit = entry & |
191 | ~(DCA3_TAG_MAP_BIT_TO_SEL | DCA3_TAG_MAP_BIT_TO_INV); |
192 | value = (apic_id & (1 << bit)) ? 1 : 0; |
193 | } else if (entry & DCA3_TAG_MAP_BIT_TO_INV) { |
194 | bit = entry & ~DCA3_TAG_MAP_BIT_TO_INV; |
195 | value = (apic_id & (1 << bit)) ? 0 : 1; |
196 | } else { |
197 | value = (entry & DCA3_TAG_MAP_LITERAL_VAL) ? 1 : 0; |
198 | } |
199 | tag |= (value << i); |
200 | } |
201 | |
202 | return tag; |
203 | } |
204 | |
205 | static const struct dca_ops ioat_dca_ops = { |
206 | .add_requester = ioat_dca_add_requester, |
207 | .remove_requester = ioat_dca_remove_requester, |
208 | .get_tag = ioat_dca_get_tag, |
209 | .dev_managed = ioat_dca_dev_managed, |
210 | }; |
211 | |
212 | static int ioat_dca_count_dca_slots(void *iobase, u16 dca_offset) |
213 | { |
214 | int slots = 0; |
215 | u32 req; |
216 | u16 global_req_table; |
217 | |
218 | global_req_table = readw(addr: iobase + dca_offset + IOAT3_DCA_GREQID_OFFSET); |
219 | if (global_req_table == 0) |
220 | return 0; |
221 | |
222 | do { |
223 | req = readl(addr: iobase + global_req_table + (slots * sizeof(u32))); |
224 | slots++; |
225 | } while ((req & IOAT_DCA_GREQID_LASTID) == 0); |
226 | |
227 | return slots; |
228 | } |
229 | |
230 | static inline int dca3_tag_map_invalid(u8 *tag_map) |
231 | { |
232 | /* |
233 | * If the tag map is not programmed by the BIOS the default is: |
234 | * 0x80 0x80 0x80 0x80 0x80 0x00 0x00 0x00 |
235 | * |
236 | * This an invalid map and will result in only 2 possible tags |
237 | * 0x1F and 0x00. 0x00 is an invalid DCA tag so we know that |
238 | * this entire definition is invalid. |
239 | */ |
240 | return ((tag_map[0] == DCA_TAG_MAP_VALID) && |
241 | (tag_map[1] == DCA_TAG_MAP_VALID) && |
242 | (tag_map[2] == DCA_TAG_MAP_VALID) && |
243 | (tag_map[3] == DCA_TAG_MAP_VALID) && |
244 | (tag_map[4] == DCA_TAG_MAP_VALID)); |
245 | } |
246 | |
247 | struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase) |
248 | { |
249 | struct dca_provider *dca; |
250 | struct ioat_dca_priv *ioatdca; |
251 | int slots; |
252 | int i; |
253 | int err; |
254 | u16 dca_offset; |
255 | u16 csi_fsb_control; |
256 | u16 pcie_control; |
257 | u8 bit; |
258 | |
259 | union { |
260 | u64 full; |
261 | struct { |
262 | u32 low; |
263 | u32 high; |
264 | }; |
265 | } tag_map; |
266 | |
267 | if (!system_has_dca_enabled(pdev)) |
268 | return NULL; |
269 | |
270 | dca_offset = readw(addr: iobase + IOAT_DCAOFFSET_OFFSET); |
271 | if (dca_offset == 0) |
272 | return NULL; |
273 | |
274 | slots = ioat_dca_count_dca_slots(iobase, dca_offset); |
275 | if (slots == 0) |
276 | return NULL; |
277 | |
278 | dca = alloc_dca_provider(ops: &ioat_dca_ops, |
279 | struct_size(ioatdca, req_slots, slots)); |
280 | if (!dca) |
281 | return NULL; |
282 | |
283 | ioatdca = dca_priv(dca); |
284 | ioatdca->iobase = iobase; |
285 | ioatdca->dca_base = iobase + dca_offset; |
286 | ioatdca->max_requesters = slots; |
287 | |
288 | /* some bios might not know to turn these on */ |
289 | csi_fsb_control = readw(addr: ioatdca->dca_base + IOAT3_CSI_CONTROL_OFFSET); |
290 | if ((csi_fsb_control & IOAT3_CSI_CONTROL_PREFETCH) == 0) { |
291 | csi_fsb_control |= IOAT3_CSI_CONTROL_PREFETCH; |
292 | writew(val: csi_fsb_control, |
293 | addr: ioatdca->dca_base + IOAT3_CSI_CONTROL_OFFSET); |
294 | } |
295 | pcie_control = readw(addr: ioatdca->dca_base + IOAT3_PCI_CONTROL_OFFSET); |
296 | if ((pcie_control & IOAT3_PCI_CONTROL_MEMWR) == 0) { |
297 | pcie_control |= IOAT3_PCI_CONTROL_MEMWR; |
298 | writew(val: pcie_control, |
299 | addr: ioatdca->dca_base + IOAT3_PCI_CONTROL_OFFSET); |
300 | } |
301 | |
302 | |
303 | /* TODO version, compatibility and configuration checks */ |
304 | |
305 | /* copy out the APIC to DCA tag map */ |
306 | tag_map.low = |
307 | readl(addr: ioatdca->dca_base + IOAT3_APICID_TAG_MAP_OFFSET_LOW); |
308 | tag_map.high = |
309 | readl(addr: ioatdca->dca_base + IOAT3_APICID_TAG_MAP_OFFSET_HIGH); |
310 | for (i = 0; i < 8; i++) { |
311 | bit = tag_map.full >> (8 * i); |
312 | ioatdca->tag_map[i] = bit & DCA_TAG_MAP_MASK; |
313 | } |
314 | |
315 | if (dca3_tag_map_invalid(tag_map: ioatdca->tag_map)) { |
316 | add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK); |
317 | pr_warn_once("%s %s: APICID_TAG_MAP set incorrectly by BIOS, disabling DCA\n" , |
318 | dev_driver_string(&pdev->dev), |
319 | dev_name(&pdev->dev)); |
320 | free_dca_provider(dca); |
321 | return NULL; |
322 | } |
323 | |
324 | err = register_dca_provider(dca, dev: &pdev->dev); |
325 | if (err) { |
326 | free_dca_provider(dca); |
327 | return NULL; |
328 | } |
329 | |
330 | return dca; |
331 | } |
332 | |