1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Intel I/OAT DMA Linux driver |
4 | * Copyright(c) 2004 - 2015 Intel Corporation. |
5 | */ |
6 | |
7 | #include <linux/init.h> |
8 | #include <linux/module.h> |
9 | #include <linux/slab.h> |
10 | #include <linux/pci.h> |
11 | #include <linux/interrupt.h> |
12 | #include <linux/dmaengine.h> |
13 | #include <linux/delay.h> |
14 | #include <linux/dma-mapping.h> |
15 | #include <linux/workqueue.h> |
16 | #include <linux/prefetch.h> |
17 | #include <linux/dca.h> |
18 | #include <linux/sizes.h> |
19 | #include "dma.h" |
20 | #include "registers.h" |
21 | #include "hw.h" |
22 | |
23 | #include "../dmaengine.h" |
24 | |
25 | MODULE_VERSION(IOAT_DMA_VERSION); |
26 | MODULE_LICENSE("Dual BSD/GPL" ); |
27 | MODULE_AUTHOR("Intel Corporation" ); |
28 | |
29 | static const struct pci_device_id ioat_pci_tbl[] = { |
30 | /* I/OAT v3 platforms */ |
31 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG0) }, |
32 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG1) }, |
33 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG2) }, |
34 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG3) }, |
35 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG4) }, |
36 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG5) }, |
37 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG6) }, |
38 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG7) }, |
39 | |
40 | /* I/OAT v3.2 platforms */ |
41 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF0) }, |
42 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF1) }, |
43 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF2) }, |
44 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF3) }, |
45 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF4) }, |
46 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF5) }, |
47 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF6) }, |
48 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF7) }, |
49 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF8) }, |
50 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF9) }, |
51 | |
52 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB0) }, |
53 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB1) }, |
54 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB2) }, |
55 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB3) }, |
56 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB4) }, |
57 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB5) }, |
58 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB6) }, |
59 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB7) }, |
60 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB8) }, |
61 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB9) }, |
62 | |
63 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB0) }, |
64 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB1) }, |
65 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB2) }, |
66 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB3) }, |
67 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB4) }, |
68 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB5) }, |
69 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB6) }, |
70 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB7) }, |
71 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB8) }, |
72 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB9) }, |
73 | |
74 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW0) }, |
75 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW1) }, |
76 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW2) }, |
77 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW3) }, |
78 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW4) }, |
79 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW5) }, |
80 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW6) }, |
81 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW7) }, |
82 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW8) }, |
83 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW9) }, |
84 | |
85 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX0) }, |
86 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX1) }, |
87 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX2) }, |
88 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX3) }, |
89 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX4) }, |
90 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX5) }, |
91 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX6) }, |
92 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX7) }, |
93 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX8) }, |
94 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX9) }, |
95 | |
96 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SKX) }, |
97 | |
98 | /* I/OAT v3.3 platforms */ |
99 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD0) }, |
100 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD1) }, |
101 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD2) }, |
102 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD3) }, |
103 | |
104 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE0) }, |
105 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE1) }, |
106 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE2) }, |
107 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE3) }, |
108 | |
109 | /* I/OAT v3.4 platforms */ |
110 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_ICX) }, |
111 | |
112 | { 0, } |
113 | }; |
114 | MODULE_DEVICE_TABLE(pci, ioat_pci_tbl); |
115 | |
116 | static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id); |
117 | static void ioat_remove(struct pci_dev *pdev); |
118 | static void |
119 | ioat_init_channel(struct ioatdma_device *ioat_dma, |
120 | struct ioatdma_chan *ioat_chan, int idx); |
121 | static void ioat_intr_quirk(struct ioatdma_device *ioat_dma); |
122 | static void ioat_enumerate_channels(struct ioatdma_device *ioat_dma); |
123 | static int ioat3_dma_self_test(struct ioatdma_device *ioat_dma); |
124 | |
125 | static int ioat_dca_enabled = 1; |
126 | module_param(ioat_dca_enabled, int, 0644); |
127 | MODULE_PARM_DESC(ioat_dca_enabled, "control support of dca service (default: 1)" ); |
128 | int ioat_pending_level = 7; |
129 | module_param(ioat_pending_level, int, 0644); |
130 | MODULE_PARM_DESC(ioat_pending_level, |
131 | "high-water mark for pushing ioat descriptors (default: 7)" ); |
132 | static char ioat_interrupt_style[32] = "msix" ; |
133 | module_param_string(ioat_interrupt_style, ioat_interrupt_style, |
134 | sizeof(ioat_interrupt_style), 0644); |
135 | MODULE_PARM_DESC(ioat_interrupt_style, |
136 | "set ioat interrupt style: msix (default), msi, intx" ); |
137 | |
138 | struct kmem_cache *ioat_cache; |
139 | struct kmem_cache *ioat_sed_cache; |
140 | |
141 | static bool is_jf_ioat(struct pci_dev *pdev) |
142 | { |
143 | switch (pdev->device) { |
144 | case PCI_DEVICE_ID_INTEL_IOAT_JSF0: |
145 | case PCI_DEVICE_ID_INTEL_IOAT_JSF1: |
146 | case PCI_DEVICE_ID_INTEL_IOAT_JSF2: |
147 | case PCI_DEVICE_ID_INTEL_IOAT_JSF3: |
148 | case PCI_DEVICE_ID_INTEL_IOAT_JSF4: |
149 | case PCI_DEVICE_ID_INTEL_IOAT_JSF5: |
150 | case PCI_DEVICE_ID_INTEL_IOAT_JSF6: |
151 | case PCI_DEVICE_ID_INTEL_IOAT_JSF7: |
152 | case PCI_DEVICE_ID_INTEL_IOAT_JSF8: |
153 | case PCI_DEVICE_ID_INTEL_IOAT_JSF9: |
154 | return true; |
155 | default: |
156 | return false; |
157 | } |
158 | } |
159 | |
160 | static bool is_snb_ioat(struct pci_dev *pdev) |
161 | { |
162 | switch (pdev->device) { |
163 | case PCI_DEVICE_ID_INTEL_IOAT_SNB0: |
164 | case PCI_DEVICE_ID_INTEL_IOAT_SNB1: |
165 | case PCI_DEVICE_ID_INTEL_IOAT_SNB2: |
166 | case PCI_DEVICE_ID_INTEL_IOAT_SNB3: |
167 | case PCI_DEVICE_ID_INTEL_IOAT_SNB4: |
168 | case PCI_DEVICE_ID_INTEL_IOAT_SNB5: |
169 | case PCI_DEVICE_ID_INTEL_IOAT_SNB6: |
170 | case PCI_DEVICE_ID_INTEL_IOAT_SNB7: |
171 | case PCI_DEVICE_ID_INTEL_IOAT_SNB8: |
172 | case PCI_DEVICE_ID_INTEL_IOAT_SNB9: |
173 | return true; |
174 | default: |
175 | return false; |
176 | } |
177 | } |
178 | |
179 | static bool is_ivb_ioat(struct pci_dev *pdev) |
180 | { |
181 | switch (pdev->device) { |
182 | case PCI_DEVICE_ID_INTEL_IOAT_IVB0: |
183 | case PCI_DEVICE_ID_INTEL_IOAT_IVB1: |
184 | case PCI_DEVICE_ID_INTEL_IOAT_IVB2: |
185 | case PCI_DEVICE_ID_INTEL_IOAT_IVB3: |
186 | case PCI_DEVICE_ID_INTEL_IOAT_IVB4: |
187 | case PCI_DEVICE_ID_INTEL_IOAT_IVB5: |
188 | case PCI_DEVICE_ID_INTEL_IOAT_IVB6: |
189 | case PCI_DEVICE_ID_INTEL_IOAT_IVB7: |
190 | case PCI_DEVICE_ID_INTEL_IOAT_IVB8: |
191 | case PCI_DEVICE_ID_INTEL_IOAT_IVB9: |
192 | return true; |
193 | default: |
194 | return false; |
195 | } |
196 | |
197 | } |
198 | |
199 | static bool is_hsw_ioat(struct pci_dev *pdev) |
200 | { |
201 | switch (pdev->device) { |
202 | case PCI_DEVICE_ID_INTEL_IOAT_HSW0: |
203 | case PCI_DEVICE_ID_INTEL_IOAT_HSW1: |
204 | case PCI_DEVICE_ID_INTEL_IOAT_HSW2: |
205 | case PCI_DEVICE_ID_INTEL_IOAT_HSW3: |
206 | case PCI_DEVICE_ID_INTEL_IOAT_HSW4: |
207 | case PCI_DEVICE_ID_INTEL_IOAT_HSW5: |
208 | case PCI_DEVICE_ID_INTEL_IOAT_HSW6: |
209 | case PCI_DEVICE_ID_INTEL_IOAT_HSW7: |
210 | case PCI_DEVICE_ID_INTEL_IOAT_HSW8: |
211 | case PCI_DEVICE_ID_INTEL_IOAT_HSW9: |
212 | return true; |
213 | default: |
214 | return false; |
215 | } |
216 | |
217 | } |
218 | |
219 | static bool is_bdx_ioat(struct pci_dev *pdev) |
220 | { |
221 | switch (pdev->device) { |
222 | case PCI_DEVICE_ID_INTEL_IOAT_BDX0: |
223 | case PCI_DEVICE_ID_INTEL_IOAT_BDX1: |
224 | case PCI_DEVICE_ID_INTEL_IOAT_BDX2: |
225 | case PCI_DEVICE_ID_INTEL_IOAT_BDX3: |
226 | case PCI_DEVICE_ID_INTEL_IOAT_BDX4: |
227 | case PCI_DEVICE_ID_INTEL_IOAT_BDX5: |
228 | case PCI_DEVICE_ID_INTEL_IOAT_BDX6: |
229 | case PCI_DEVICE_ID_INTEL_IOAT_BDX7: |
230 | case PCI_DEVICE_ID_INTEL_IOAT_BDX8: |
231 | case PCI_DEVICE_ID_INTEL_IOAT_BDX9: |
232 | return true; |
233 | default: |
234 | return false; |
235 | } |
236 | } |
237 | |
238 | static inline bool is_skx_ioat(struct pci_dev *pdev) |
239 | { |
240 | return (pdev->device == PCI_DEVICE_ID_INTEL_IOAT_SKX) ? true : false; |
241 | } |
242 | |
243 | static bool is_xeon_cb32(struct pci_dev *pdev) |
244 | { |
245 | return is_jf_ioat(pdev) || is_snb_ioat(pdev) || is_ivb_ioat(pdev) || |
246 | is_hsw_ioat(pdev) || is_bdx_ioat(pdev) || is_skx_ioat(pdev); |
247 | } |
248 | |
249 | bool is_bwd_ioat(struct pci_dev *pdev) |
250 | { |
251 | switch (pdev->device) { |
252 | case PCI_DEVICE_ID_INTEL_IOAT_BWD0: |
253 | case PCI_DEVICE_ID_INTEL_IOAT_BWD1: |
254 | case PCI_DEVICE_ID_INTEL_IOAT_BWD2: |
255 | case PCI_DEVICE_ID_INTEL_IOAT_BWD3: |
256 | /* even though not Atom, BDX-DE has same DMA silicon */ |
257 | case PCI_DEVICE_ID_INTEL_IOAT_BDXDE0: |
258 | case PCI_DEVICE_ID_INTEL_IOAT_BDXDE1: |
259 | case PCI_DEVICE_ID_INTEL_IOAT_BDXDE2: |
260 | case PCI_DEVICE_ID_INTEL_IOAT_BDXDE3: |
261 | return true; |
262 | default: |
263 | return false; |
264 | } |
265 | } |
266 | |
267 | static bool is_bwd_noraid(struct pci_dev *pdev) |
268 | { |
269 | switch (pdev->device) { |
270 | case PCI_DEVICE_ID_INTEL_IOAT_BWD2: |
271 | case PCI_DEVICE_ID_INTEL_IOAT_BWD3: |
272 | case PCI_DEVICE_ID_INTEL_IOAT_BDXDE0: |
273 | case PCI_DEVICE_ID_INTEL_IOAT_BDXDE1: |
274 | case PCI_DEVICE_ID_INTEL_IOAT_BDXDE2: |
275 | case PCI_DEVICE_ID_INTEL_IOAT_BDXDE3: |
276 | return true; |
277 | default: |
278 | return false; |
279 | } |
280 | |
281 | } |
282 | |
283 | /* |
284 | * Perform a IOAT transaction to verify the HW works. |
285 | */ |
286 | #define IOAT_TEST_SIZE 2000 |
287 | |
288 | static void ioat_dma_test_callback(void *dma_async_param) |
289 | { |
290 | struct completion *cmp = dma_async_param; |
291 | |
292 | complete(cmp); |
293 | } |
294 | |
295 | /** |
296 | * ioat_dma_self_test - Perform a IOAT transaction to verify the HW works. |
297 | * @ioat_dma: dma device to be tested |
298 | */ |
299 | static int ioat_dma_self_test(struct ioatdma_device *ioat_dma) |
300 | { |
301 | int i; |
302 | u8 *src; |
303 | u8 *dest; |
304 | struct dma_device *dma = &ioat_dma->dma_dev; |
305 | struct device *dev = &ioat_dma->pdev->dev; |
306 | struct dma_chan *dma_chan; |
307 | struct dma_async_tx_descriptor *tx; |
308 | dma_addr_t dma_dest, dma_src; |
309 | dma_cookie_t cookie; |
310 | int err = 0; |
311 | struct completion cmp; |
312 | unsigned long tmo; |
313 | unsigned long flags; |
314 | |
315 | src = kzalloc(IOAT_TEST_SIZE, GFP_KERNEL); |
316 | if (!src) |
317 | return -ENOMEM; |
318 | dest = kzalloc(IOAT_TEST_SIZE, GFP_KERNEL); |
319 | if (!dest) { |
320 | kfree(objp: src); |
321 | return -ENOMEM; |
322 | } |
323 | |
324 | /* Fill in src buffer */ |
325 | for (i = 0; i < IOAT_TEST_SIZE; i++) |
326 | src[i] = (u8)i; |
327 | |
328 | /* Start copy, using first DMA channel */ |
329 | dma_chan = container_of(dma->channels.next, struct dma_chan, |
330 | device_node); |
331 | if (dma->device_alloc_chan_resources(dma_chan) < 1) { |
332 | dev_err(dev, "selftest cannot allocate chan resource\n" ); |
333 | err = -ENODEV; |
334 | goto out; |
335 | } |
336 | |
337 | dma_src = dma_map_single(dev, src, IOAT_TEST_SIZE, DMA_TO_DEVICE); |
338 | if (dma_mapping_error(dev, dma_addr: dma_src)) { |
339 | dev_err(dev, "mapping src buffer failed\n" ); |
340 | err = -ENOMEM; |
341 | goto free_resources; |
342 | } |
343 | dma_dest = dma_map_single(dev, dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE); |
344 | if (dma_mapping_error(dev, dma_addr: dma_dest)) { |
345 | dev_err(dev, "mapping dest buffer failed\n" ); |
346 | err = -ENOMEM; |
347 | goto unmap_src; |
348 | } |
349 | flags = DMA_PREP_INTERRUPT; |
350 | tx = ioat_dma->dma_dev.device_prep_dma_memcpy(dma_chan, dma_dest, |
351 | dma_src, IOAT_TEST_SIZE, |
352 | flags); |
353 | if (!tx) { |
354 | dev_err(dev, "Self-test prep failed, disabling\n" ); |
355 | err = -ENODEV; |
356 | goto unmap_dma; |
357 | } |
358 | |
359 | async_tx_ack(tx); |
360 | init_completion(x: &cmp); |
361 | tx->callback = ioat_dma_test_callback; |
362 | tx->callback_param = &cmp; |
363 | cookie = tx->tx_submit(tx); |
364 | if (cookie < 0) { |
365 | dev_err(dev, "Self-test setup failed, disabling\n" ); |
366 | err = -ENODEV; |
367 | goto unmap_dma; |
368 | } |
369 | dma->device_issue_pending(dma_chan); |
370 | |
371 | tmo = wait_for_completion_timeout(x: &cmp, timeout: msecs_to_jiffies(m: 3000)); |
372 | |
373 | if (tmo == 0 || |
374 | dma->device_tx_status(dma_chan, cookie, NULL) |
375 | != DMA_COMPLETE) { |
376 | dev_err(dev, "Self-test copy timed out, disabling\n" ); |
377 | err = -ENODEV; |
378 | goto unmap_dma; |
379 | } |
380 | if (memcmp(p: src, q: dest, IOAT_TEST_SIZE)) { |
381 | dev_err(dev, "Self-test copy failed compare, disabling\n" ); |
382 | err = -ENODEV; |
383 | goto unmap_dma; |
384 | } |
385 | |
386 | unmap_dma: |
387 | dma_unmap_single(dev, dma_dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE); |
388 | unmap_src: |
389 | dma_unmap_single(dev, dma_src, IOAT_TEST_SIZE, DMA_TO_DEVICE); |
390 | free_resources: |
391 | dma->device_free_chan_resources(dma_chan); |
392 | out: |
393 | kfree(objp: src); |
394 | kfree(objp: dest); |
395 | return err; |
396 | } |
397 | |
398 | /** |
399 | * ioat_dma_setup_interrupts - setup interrupt handler |
400 | * @ioat_dma: ioat dma device |
401 | */ |
402 | int ioat_dma_setup_interrupts(struct ioatdma_device *ioat_dma) |
403 | { |
404 | struct ioatdma_chan *ioat_chan; |
405 | struct pci_dev *pdev = ioat_dma->pdev; |
406 | struct device *dev = &pdev->dev; |
407 | struct msix_entry *msix; |
408 | int i, j, msixcnt; |
409 | int err = -EINVAL; |
410 | u8 intrctrl = 0; |
411 | |
412 | if (!strcmp(ioat_interrupt_style, "msix" )) |
413 | goto msix; |
414 | if (!strcmp(ioat_interrupt_style, "msi" )) |
415 | goto msi; |
416 | if (!strcmp(ioat_interrupt_style, "intx" )) |
417 | goto intx; |
418 | dev_err(dev, "invalid ioat_interrupt_style %s\n" , ioat_interrupt_style); |
419 | goto err_no_irq; |
420 | |
421 | msix: |
422 | /* The number of MSI-X vectors should equal the number of channels */ |
423 | msixcnt = ioat_dma->chancnt; |
424 | for (i = 0; i < msixcnt; i++) |
425 | ioat_dma->msix_entries[i].entry = i; |
426 | |
427 | err = pci_enable_msix_exact(dev: pdev, entries: ioat_dma->msix_entries, nvec: msixcnt); |
428 | if (err) |
429 | goto msi; |
430 | |
431 | for (i = 0; i < msixcnt; i++) { |
432 | msix = &ioat_dma->msix_entries[i]; |
433 | ioat_chan = ioat_chan_by_index(ioat_dma, index: i); |
434 | err = devm_request_irq(dev, irq: msix->vector, |
435 | handler: ioat_dma_do_interrupt_msix, irqflags: 0, |
436 | devname: "ioat-msix" , dev_id: ioat_chan); |
437 | if (err) { |
438 | for (j = 0; j < i; j++) { |
439 | msix = &ioat_dma->msix_entries[j]; |
440 | ioat_chan = ioat_chan_by_index(ioat_dma, index: j); |
441 | devm_free_irq(dev, irq: msix->vector, dev_id: ioat_chan); |
442 | } |
443 | goto msi; |
444 | } |
445 | } |
446 | intrctrl |= IOAT_INTRCTRL_MSIX_VECTOR_CONTROL; |
447 | ioat_dma->irq_mode = IOAT_MSIX; |
448 | goto done; |
449 | |
450 | msi: |
451 | err = pci_enable_msi(dev: pdev); |
452 | if (err) |
453 | goto intx; |
454 | |
455 | err = devm_request_irq(dev, irq: pdev->irq, handler: ioat_dma_do_interrupt, irqflags: 0, |
456 | devname: "ioat-msi" , dev_id: ioat_dma); |
457 | if (err) { |
458 | pci_disable_msi(dev: pdev); |
459 | goto intx; |
460 | } |
461 | ioat_dma->irq_mode = IOAT_MSI; |
462 | goto done; |
463 | |
464 | intx: |
465 | err = devm_request_irq(dev, irq: pdev->irq, handler: ioat_dma_do_interrupt, |
466 | IRQF_SHARED, devname: "ioat-intx" , dev_id: ioat_dma); |
467 | if (err) |
468 | goto err_no_irq; |
469 | |
470 | ioat_dma->irq_mode = IOAT_INTX; |
471 | done: |
472 | if (is_bwd_ioat(pdev)) |
473 | ioat_intr_quirk(ioat_dma); |
474 | intrctrl |= IOAT_INTRCTRL_MASTER_INT_EN; |
475 | writeb(val: intrctrl, addr: ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET); |
476 | return 0; |
477 | |
478 | err_no_irq: |
479 | /* Disable all interrupt generation */ |
480 | writeb(val: 0, addr: ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET); |
481 | ioat_dma->irq_mode = IOAT_NOIRQ; |
482 | dev_err(dev, "no usable interrupts\n" ); |
483 | return err; |
484 | } |
485 | |
486 | static void ioat_disable_interrupts(struct ioatdma_device *ioat_dma) |
487 | { |
488 | /* Disable all interrupt generation */ |
489 | writeb(val: 0, addr: ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET); |
490 | } |
491 | |
492 | static int ioat_probe(struct ioatdma_device *ioat_dma) |
493 | { |
494 | int err = -ENODEV; |
495 | struct dma_device *dma = &ioat_dma->dma_dev; |
496 | struct pci_dev *pdev = ioat_dma->pdev; |
497 | struct device *dev = &pdev->dev; |
498 | |
499 | ioat_dma->completion_pool = dma_pool_create(name: "completion_pool" , dev, |
500 | size: sizeof(u64), |
501 | SMP_CACHE_BYTES, |
502 | SMP_CACHE_BYTES); |
503 | |
504 | if (!ioat_dma->completion_pool) { |
505 | err = -ENOMEM; |
506 | goto err_out; |
507 | } |
508 | |
509 | ioat_enumerate_channels(ioat_dma); |
510 | |
511 | dma_cap_set(DMA_MEMCPY, dma->cap_mask); |
512 | dma->dev = &pdev->dev; |
513 | |
514 | if (!ioat_dma->chancnt) { |
515 | dev_err(dev, "channel enumeration error\n" ); |
516 | goto err_setup_interrupts; |
517 | } |
518 | |
519 | err = ioat_dma_setup_interrupts(ioat_dma); |
520 | if (err) |
521 | goto err_setup_interrupts; |
522 | |
523 | err = ioat3_dma_self_test(ioat_dma); |
524 | if (err) |
525 | goto err_self_test; |
526 | |
527 | return 0; |
528 | |
529 | err_self_test: |
530 | ioat_disable_interrupts(ioat_dma); |
531 | err_setup_interrupts: |
532 | dma_pool_destroy(pool: ioat_dma->completion_pool); |
533 | err_out: |
534 | return err; |
535 | } |
536 | |
537 | static int ioat_register(struct ioatdma_device *ioat_dma) |
538 | { |
539 | int err = dma_async_device_register(device: &ioat_dma->dma_dev); |
540 | |
541 | if (err) { |
542 | ioat_disable_interrupts(ioat_dma); |
543 | dma_pool_destroy(pool: ioat_dma->completion_pool); |
544 | } |
545 | |
546 | return err; |
547 | } |
548 | |
549 | static void ioat_dma_remove(struct ioatdma_device *ioat_dma) |
550 | { |
551 | struct dma_device *dma = &ioat_dma->dma_dev; |
552 | |
553 | ioat_disable_interrupts(ioat_dma); |
554 | |
555 | ioat_kobject_del(ioat_dma); |
556 | |
557 | dma_async_device_unregister(device: dma); |
558 | } |
559 | |
560 | /** |
561 | * ioat_enumerate_channels - find and initialize the device's channels |
562 | * @ioat_dma: the ioat dma device to be enumerated |
563 | */ |
564 | static void ioat_enumerate_channels(struct ioatdma_device *ioat_dma) |
565 | { |
566 | struct ioatdma_chan *ioat_chan; |
567 | struct device *dev = &ioat_dma->pdev->dev; |
568 | struct dma_device *dma = &ioat_dma->dma_dev; |
569 | u8 xfercap_log; |
570 | int chancnt; |
571 | int i; |
572 | |
573 | INIT_LIST_HEAD(list: &dma->channels); |
574 | chancnt = readb(addr: ioat_dma->reg_base + IOAT_CHANCNT_OFFSET); |
575 | chancnt &= 0x1f; /* bits [4:0] valid */ |
576 | if (chancnt > ARRAY_SIZE(ioat_dma->idx)) { |
577 | dev_warn(dev, "(%d) exceeds max supported channels (%zu)\n" , |
578 | chancnt, ARRAY_SIZE(ioat_dma->idx)); |
579 | chancnt = ARRAY_SIZE(ioat_dma->idx); |
580 | } |
581 | xfercap_log = readb(addr: ioat_dma->reg_base + IOAT_XFERCAP_OFFSET); |
582 | xfercap_log &= 0x1f; /* bits [4:0] valid */ |
583 | if (xfercap_log == 0) |
584 | return; |
585 | dev_dbg(dev, "%s: xfercap = %d\n" , __func__, 1 << xfercap_log); |
586 | |
587 | for (i = 0; i < chancnt; i++) { |
588 | ioat_chan = kzalloc(size: sizeof(*ioat_chan), GFP_KERNEL); |
589 | if (!ioat_chan) |
590 | break; |
591 | |
592 | ioat_init_channel(ioat_dma, ioat_chan, idx: i); |
593 | ioat_chan->xfercap_log = xfercap_log; |
594 | spin_lock_init(&ioat_chan->prep_lock); |
595 | if (ioat_reset_hw(ioat_chan)) { |
596 | i = 0; |
597 | break; |
598 | } |
599 | } |
600 | ioat_dma->chancnt = i; |
601 | } |
602 | |
603 | /** |
604 | * ioat_free_chan_resources - release all the descriptors |
605 | * @c: the channel to be cleaned |
606 | */ |
607 | static void ioat_free_chan_resources(struct dma_chan *c) |
608 | { |
609 | struct ioatdma_chan *ioat_chan = to_ioat_chan(c); |
610 | struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma; |
611 | struct ioat_ring_ent *desc; |
612 | const int total_descs = 1 << ioat_chan->alloc_order; |
613 | int descs; |
614 | int i; |
615 | |
616 | /* Before freeing channel resources first check |
617 | * if they have been previously allocated for this channel. |
618 | */ |
619 | if (!ioat_chan->ring) |
620 | return; |
621 | |
622 | ioat_stop(ioat_chan); |
623 | |
624 | if (!test_bit(IOAT_CHAN_DOWN, &ioat_chan->state)) { |
625 | ioat_reset_hw(ioat_chan); |
626 | |
627 | /* Put LTR to idle */ |
628 | if (ioat_dma->version >= IOAT_VER_3_4) |
629 | writeb(IOAT_CHAN_LTR_SWSEL_IDLE, |
630 | addr: ioat_chan->reg_base + |
631 | IOAT_CHAN_LTR_SWSEL_OFFSET); |
632 | } |
633 | |
634 | spin_lock_bh(lock: &ioat_chan->cleanup_lock); |
635 | spin_lock_bh(lock: &ioat_chan->prep_lock); |
636 | descs = ioat_ring_space(ioat_chan); |
637 | dev_dbg(to_dev(ioat_chan), "freeing %d idle descriptors\n" , descs); |
638 | for (i = 0; i < descs; i++) { |
639 | desc = ioat_get_ring_ent(ioat_chan, idx: ioat_chan->head + i); |
640 | ioat_free_ring_ent(desc, chan: c); |
641 | } |
642 | |
643 | if (descs < total_descs) |
644 | dev_err(to_dev(ioat_chan), "Freeing %d in use descriptors!\n" , |
645 | total_descs - descs); |
646 | |
647 | for (i = 0; i < total_descs - descs; i++) { |
648 | desc = ioat_get_ring_ent(ioat_chan, idx: ioat_chan->tail + i); |
649 | dump_desc_dbg(ioat_chan, desc); |
650 | ioat_free_ring_ent(desc, chan: c); |
651 | } |
652 | |
653 | for (i = 0; i < ioat_chan->desc_chunks; i++) { |
654 | dma_free_coherent(to_dev(ioat_chan), IOAT_CHUNK_SIZE, |
655 | cpu_addr: ioat_chan->descs[i].virt, |
656 | dma_handle: ioat_chan->descs[i].hw); |
657 | ioat_chan->descs[i].virt = NULL; |
658 | ioat_chan->descs[i].hw = 0; |
659 | } |
660 | ioat_chan->desc_chunks = 0; |
661 | |
662 | kfree(objp: ioat_chan->ring); |
663 | ioat_chan->ring = NULL; |
664 | ioat_chan->alloc_order = 0; |
665 | dma_pool_free(pool: ioat_dma->completion_pool, vaddr: ioat_chan->completion, |
666 | addr: ioat_chan->completion_dma); |
667 | spin_unlock_bh(lock: &ioat_chan->prep_lock); |
668 | spin_unlock_bh(lock: &ioat_chan->cleanup_lock); |
669 | |
670 | ioat_chan->last_completion = 0; |
671 | ioat_chan->completion_dma = 0; |
672 | ioat_chan->dmacount = 0; |
673 | } |
674 | |
675 | /* ioat_alloc_chan_resources - allocate/initialize ioat descriptor ring |
676 | * @chan: channel to be initialized |
677 | */ |
678 | static int ioat_alloc_chan_resources(struct dma_chan *c) |
679 | { |
680 | struct ioatdma_chan *ioat_chan = to_ioat_chan(c); |
681 | struct ioat_ring_ent **ring; |
682 | u64 status; |
683 | int order; |
684 | int i = 0; |
685 | u32 chanerr; |
686 | |
687 | /* have we already been set up? */ |
688 | if (ioat_chan->ring) |
689 | return 1 << ioat_chan->alloc_order; |
690 | |
691 | /* Setup register to interrupt and write completion status on error */ |
692 | writew(IOAT_CHANCTRL_RUN, addr: ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET); |
693 | |
694 | /* allocate a completion writeback area */ |
695 | /* doing 2 32bit writes to mmio since 1 64b write doesn't work */ |
696 | ioat_chan->completion = |
697 | dma_pool_zalloc(pool: ioat_chan->ioat_dma->completion_pool, |
698 | GFP_NOWAIT, handle: &ioat_chan->completion_dma); |
699 | if (!ioat_chan->completion) |
700 | return -ENOMEM; |
701 | |
702 | writel(val: ((u64)ioat_chan->completion_dma) & 0x00000000FFFFFFFF, |
703 | addr: ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW); |
704 | writel(val: ((u64)ioat_chan->completion_dma) >> 32, |
705 | addr: ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH); |
706 | |
707 | order = IOAT_MAX_ORDER; |
708 | ring = ioat_alloc_ring(c, order, GFP_NOWAIT); |
709 | if (!ring) |
710 | return -ENOMEM; |
711 | |
712 | spin_lock_bh(lock: &ioat_chan->cleanup_lock); |
713 | spin_lock_bh(lock: &ioat_chan->prep_lock); |
714 | ioat_chan->ring = ring; |
715 | ioat_chan->head = 0; |
716 | ioat_chan->issued = 0; |
717 | ioat_chan->tail = 0; |
718 | ioat_chan->alloc_order = order; |
719 | set_bit(IOAT_RUN, addr: &ioat_chan->state); |
720 | spin_unlock_bh(lock: &ioat_chan->prep_lock); |
721 | spin_unlock_bh(lock: &ioat_chan->cleanup_lock); |
722 | |
723 | /* Setting up LTR values for 3.4 or later */ |
724 | if (ioat_chan->ioat_dma->version >= IOAT_VER_3_4) { |
725 | u32 lat_val; |
726 | |
727 | lat_val = IOAT_CHAN_LTR_ACTIVE_SNVAL | |
728 | IOAT_CHAN_LTR_ACTIVE_SNLATSCALE | |
729 | IOAT_CHAN_LTR_ACTIVE_SNREQMNT; |
730 | writel(val: lat_val, addr: ioat_chan->reg_base + |
731 | IOAT_CHAN_LTR_ACTIVE_OFFSET); |
732 | |
733 | lat_val = IOAT_CHAN_LTR_IDLE_SNVAL | |
734 | IOAT_CHAN_LTR_IDLE_SNLATSCALE | |
735 | IOAT_CHAN_LTR_IDLE_SNREQMNT; |
736 | writel(val: lat_val, addr: ioat_chan->reg_base + |
737 | IOAT_CHAN_LTR_IDLE_OFFSET); |
738 | |
739 | /* Select to active */ |
740 | writeb(IOAT_CHAN_LTR_SWSEL_ACTIVE, |
741 | addr: ioat_chan->reg_base + |
742 | IOAT_CHAN_LTR_SWSEL_OFFSET); |
743 | } |
744 | |
745 | ioat_start_null_desc(ioat_chan); |
746 | |
747 | /* check that we got off the ground */ |
748 | do { |
749 | udelay(1); |
750 | status = ioat_chansts(ioat_chan); |
751 | } while (i++ < 20 && !is_ioat_active(status) && !is_ioat_idle(status)); |
752 | |
753 | if (is_ioat_active(status) || is_ioat_idle(status)) |
754 | return 1 << ioat_chan->alloc_order; |
755 | |
756 | chanerr = readl(addr: ioat_chan->reg_base + IOAT_CHANERR_OFFSET); |
757 | |
758 | dev_WARN(to_dev(ioat_chan), |
759 | "failed to start channel chanerr: %#x\n" , chanerr); |
760 | ioat_free_chan_resources(c); |
761 | return -EFAULT; |
762 | } |
763 | |
764 | /* common channel initialization */ |
765 | static void |
766 | ioat_init_channel(struct ioatdma_device *ioat_dma, |
767 | struct ioatdma_chan *ioat_chan, int idx) |
768 | { |
769 | struct dma_device *dma = &ioat_dma->dma_dev; |
770 | |
771 | ioat_chan->ioat_dma = ioat_dma; |
772 | ioat_chan->reg_base = ioat_dma->reg_base + (0x80 * (idx + 1)); |
773 | spin_lock_init(&ioat_chan->cleanup_lock); |
774 | ioat_chan->dma_chan.device = dma; |
775 | dma_cookie_init(chan: &ioat_chan->dma_chan); |
776 | list_add_tail(new: &ioat_chan->dma_chan.device_node, head: &dma->channels); |
777 | ioat_dma->idx[idx] = ioat_chan; |
778 | timer_setup(&ioat_chan->timer, ioat_timer_event, 0); |
779 | tasklet_setup(t: &ioat_chan->cleanup_task, callback: ioat_cleanup_event); |
780 | } |
781 | |
782 | #define IOAT_NUM_SRC_TEST 6 /* must be <= 8 */ |
783 | static int ioat_xor_val_self_test(struct ioatdma_device *ioat_dma) |
784 | { |
785 | int i, src_idx; |
786 | struct page *dest; |
787 | struct page *xor_srcs[IOAT_NUM_SRC_TEST]; |
788 | struct page *xor_val_srcs[IOAT_NUM_SRC_TEST + 1]; |
789 | dma_addr_t dma_srcs[IOAT_NUM_SRC_TEST + 1]; |
790 | dma_addr_t dest_dma; |
791 | struct dma_async_tx_descriptor *tx; |
792 | struct dma_chan *dma_chan; |
793 | dma_cookie_t cookie; |
794 | u8 cmp_byte = 0; |
795 | u32 cmp_word; |
796 | u32 xor_val_result; |
797 | int err = 0; |
798 | struct completion cmp; |
799 | unsigned long tmo; |
800 | struct device *dev = &ioat_dma->pdev->dev; |
801 | struct dma_device *dma = &ioat_dma->dma_dev; |
802 | u8 op = 0; |
803 | |
804 | dev_dbg(dev, "%s\n" , __func__); |
805 | |
806 | if (!dma_has_cap(DMA_XOR, dma->cap_mask)) |
807 | return 0; |
808 | |
809 | for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++) { |
810 | xor_srcs[src_idx] = alloc_page(GFP_KERNEL); |
811 | if (!xor_srcs[src_idx]) { |
812 | while (src_idx--) |
813 | __free_page(xor_srcs[src_idx]); |
814 | return -ENOMEM; |
815 | } |
816 | } |
817 | |
818 | dest = alloc_page(GFP_KERNEL); |
819 | if (!dest) { |
820 | while (src_idx--) |
821 | __free_page(xor_srcs[src_idx]); |
822 | return -ENOMEM; |
823 | } |
824 | |
825 | /* Fill in src buffers */ |
826 | for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++) { |
827 | u8 *ptr = page_address(xor_srcs[src_idx]); |
828 | |
829 | for (i = 0; i < PAGE_SIZE; i++) |
830 | ptr[i] = (1 << src_idx); |
831 | } |
832 | |
833 | for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++) |
834 | cmp_byte ^= (u8) (1 << src_idx); |
835 | |
836 | cmp_word = (cmp_byte << 24) | (cmp_byte << 16) | |
837 | (cmp_byte << 8) | cmp_byte; |
838 | |
839 | memset(page_address(dest), 0, PAGE_SIZE); |
840 | |
841 | dma_chan = container_of(dma->channels.next, struct dma_chan, |
842 | device_node); |
843 | if (dma->device_alloc_chan_resources(dma_chan) < 1) { |
844 | err = -ENODEV; |
845 | goto out; |
846 | } |
847 | |
848 | /* test xor */ |
849 | op = IOAT_OP_XOR; |
850 | |
851 | dest_dma = dma_map_page(dev, dest, 0, PAGE_SIZE, DMA_FROM_DEVICE); |
852 | if (dma_mapping_error(dev, dma_addr: dest_dma)) { |
853 | err = -ENOMEM; |
854 | goto free_resources; |
855 | } |
856 | |
857 | for (i = 0; i < IOAT_NUM_SRC_TEST; i++) { |
858 | dma_srcs[i] = dma_map_page(dev, xor_srcs[i], 0, PAGE_SIZE, |
859 | DMA_TO_DEVICE); |
860 | if (dma_mapping_error(dev, dma_addr: dma_srcs[i])) { |
861 | err = -ENOMEM; |
862 | goto dma_unmap; |
863 | } |
864 | } |
865 | tx = dma->device_prep_dma_xor(dma_chan, dest_dma, dma_srcs, |
866 | IOAT_NUM_SRC_TEST, PAGE_SIZE, |
867 | DMA_PREP_INTERRUPT); |
868 | |
869 | if (!tx) { |
870 | dev_err(dev, "Self-test xor prep failed\n" ); |
871 | err = -ENODEV; |
872 | goto dma_unmap; |
873 | } |
874 | |
875 | async_tx_ack(tx); |
876 | init_completion(x: &cmp); |
877 | tx->callback = ioat_dma_test_callback; |
878 | tx->callback_param = &cmp; |
879 | cookie = tx->tx_submit(tx); |
880 | if (cookie < 0) { |
881 | dev_err(dev, "Self-test xor setup failed\n" ); |
882 | err = -ENODEV; |
883 | goto dma_unmap; |
884 | } |
885 | dma->device_issue_pending(dma_chan); |
886 | |
887 | tmo = wait_for_completion_timeout(x: &cmp, timeout: msecs_to_jiffies(m: 3000)); |
888 | |
889 | if (tmo == 0 || |
890 | dma->device_tx_status(dma_chan, cookie, NULL) != DMA_COMPLETE) { |
891 | dev_err(dev, "Self-test xor timed out\n" ); |
892 | err = -ENODEV; |
893 | goto dma_unmap; |
894 | } |
895 | |
896 | for (i = 0; i < IOAT_NUM_SRC_TEST; i++) |
897 | dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, DMA_TO_DEVICE); |
898 | |
899 | dma_sync_single_for_cpu(dev, addr: dest_dma, PAGE_SIZE, dir: DMA_FROM_DEVICE); |
900 | for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) { |
901 | u32 *ptr = page_address(dest); |
902 | |
903 | if (ptr[i] != cmp_word) { |
904 | dev_err(dev, "Self-test xor failed compare\n" ); |
905 | err = -ENODEV; |
906 | goto free_resources; |
907 | } |
908 | } |
909 | dma_sync_single_for_device(dev, addr: dest_dma, PAGE_SIZE, dir: DMA_FROM_DEVICE); |
910 | |
911 | dma_unmap_page(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE); |
912 | |
913 | /* skip validate if the capability is not present */ |
914 | if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask)) |
915 | goto free_resources; |
916 | |
917 | op = IOAT_OP_XOR_VAL; |
918 | |
919 | /* validate the sources with the destintation page */ |
920 | for (i = 0; i < IOAT_NUM_SRC_TEST; i++) |
921 | xor_val_srcs[i] = xor_srcs[i]; |
922 | xor_val_srcs[i] = dest; |
923 | |
924 | xor_val_result = 1; |
925 | |
926 | for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) { |
927 | dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE, |
928 | DMA_TO_DEVICE); |
929 | if (dma_mapping_error(dev, dma_addr: dma_srcs[i])) { |
930 | err = -ENOMEM; |
931 | goto dma_unmap; |
932 | } |
933 | } |
934 | tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs, |
935 | IOAT_NUM_SRC_TEST + 1, PAGE_SIZE, |
936 | &xor_val_result, DMA_PREP_INTERRUPT); |
937 | if (!tx) { |
938 | dev_err(dev, "Self-test zero prep failed\n" ); |
939 | err = -ENODEV; |
940 | goto dma_unmap; |
941 | } |
942 | |
943 | async_tx_ack(tx); |
944 | init_completion(x: &cmp); |
945 | tx->callback = ioat_dma_test_callback; |
946 | tx->callback_param = &cmp; |
947 | cookie = tx->tx_submit(tx); |
948 | if (cookie < 0) { |
949 | dev_err(dev, "Self-test zero setup failed\n" ); |
950 | err = -ENODEV; |
951 | goto dma_unmap; |
952 | } |
953 | dma->device_issue_pending(dma_chan); |
954 | |
955 | tmo = wait_for_completion_timeout(x: &cmp, timeout: msecs_to_jiffies(m: 3000)); |
956 | |
957 | if (tmo == 0 || |
958 | dma->device_tx_status(dma_chan, cookie, NULL) != DMA_COMPLETE) { |
959 | dev_err(dev, "Self-test validate timed out\n" ); |
960 | err = -ENODEV; |
961 | goto dma_unmap; |
962 | } |
963 | |
964 | for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) |
965 | dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, DMA_TO_DEVICE); |
966 | |
967 | if (xor_val_result != 0) { |
968 | dev_err(dev, "Self-test validate failed compare\n" ); |
969 | err = -ENODEV; |
970 | goto free_resources; |
971 | } |
972 | |
973 | memset(page_address(dest), 0, PAGE_SIZE); |
974 | |
975 | /* test for non-zero parity sum */ |
976 | op = IOAT_OP_XOR_VAL; |
977 | |
978 | xor_val_result = 0; |
979 | for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) { |
980 | dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE, |
981 | DMA_TO_DEVICE); |
982 | if (dma_mapping_error(dev, dma_addr: dma_srcs[i])) { |
983 | err = -ENOMEM; |
984 | goto dma_unmap; |
985 | } |
986 | } |
987 | tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs, |
988 | IOAT_NUM_SRC_TEST + 1, PAGE_SIZE, |
989 | &xor_val_result, DMA_PREP_INTERRUPT); |
990 | if (!tx) { |
991 | dev_err(dev, "Self-test 2nd zero prep failed\n" ); |
992 | err = -ENODEV; |
993 | goto dma_unmap; |
994 | } |
995 | |
996 | async_tx_ack(tx); |
997 | init_completion(x: &cmp); |
998 | tx->callback = ioat_dma_test_callback; |
999 | tx->callback_param = &cmp; |
1000 | cookie = tx->tx_submit(tx); |
1001 | if (cookie < 0) { |
1002 | dev_err(dev, "Self-test 2nd zero setup failed\n" ); |
1003 | err = -ENODEV; |
1004 | goto dma_unmap; |
1005 | } |
1006 | dma->device_issue_pending(dma_chan); |
1007 | |
1008 | tmo = wait_for_completion_timeout(x: &cmp, timeout: msecs_to_jiffies(m: 3000)); |
1009 | |
1010 | if (tmo == 0 || |
1011 | dma->device_tx_status(dma_chan, cookie, NULL) != DMA_COMPLETE) { |
1012 | dev_err(dev, "Self-test 2nd validate timed out\n" ); |
1013 | err = -ENODEV; |
1014 | goto dma_unmap; |
1015 | } |
1016 | |
1017 | if (xor_val_result != SUM_CHECK_P_RESULT) { |
1018 | dev_err(dev, "Self-test validate failed compare\n" ); |
1019 | err = -ENODEV; |
1020 | goto dma_unmap; |
1021 | } |
1022 | |
1023 | for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) |
1024 | dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, DMA_TO_DEVICE); |
1025 | |
1026 | goto free_resources; |
1027 | dma_unmap: |
1028 | if (op == IOAT_OP_XOR) { |
1029 | while (--i >= 0) |
1030 | dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, |
1031 | DMA_TO_DEVICE); |
1032 | dma_unmap_page(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE); |
1033 | } else if (op == IOAT_OP_XOR_VAL) { |
1034 | while (--i >= 0) |
1035 | dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, |
1036 | DMA_TO_DEVICE); |
1037 | } |
1038 | free_resources: |
1039 | dma->device_free_chan_resources(dma_chan); |
1040 | out: |
1041 | src_idx = IOAT_NUM_SRC_TEST; |
1042 | while (src_idx--) |
1043 | __free_page(xor_srcs[src_idx]); |
1044 | __free_page(dest); |
1045 | return err; |
1046 | } |
1047 | |
1048 | static int ioat3_dma_self_test(struct ioatdma_device *ioat_dma) |
1049 | { |
1050 | int rc; |
1051 | |
1052 | rc = ioat_dma_self_test(ioat_dma); |
1053 | if (rc) |
1054 | return rc; |
1055 | |
1056 | rc = ioat_xor_val_self_test(ioat_dma); |
1057 | |
1058 | return rc; |
1059 | } |
1060 | |
1061 | static void ioat_intr_quirk(struct ioatdma_device *ioat_dma) |
1062 | { |
1063 | struct dma_device *dma; |
1064 | struct dma_chan *c; |
1065 | struct ioatdma_chan *ioat_chan; |
1066 | u32 errmask; |
1067 | |
1068 | dma = &ioat_dma->dma_dev; |
1069 | |
1070 | /* |
1071 | * if we have descriptor write back error status, we mask the |
1072 | * error interrupts |
1073 | */ |
1074 | if (ioat_dma->cap & IOAT_CAP_DWBES) { |
1075 | list_for_each_entry(c, &dma->channels, device_node) { |
1076 | ioat_chan = to_ioat_chan(c); |
1077 | errmask = readl(addr: ioat_chan->reg_base + |
1078 | IOAT_CHANERR_MASK_OFFSET); |
1079 | errmask |= IOAT_CHANERR_XOR_P_OR_CRC_ERR | |
1080 | IOAT_CHANERR_XOR_Q_ERR; |
1081 | writel(val: errmask, addr: ioat_chan->reg_base + |
1082 | IOAT_CHANERR_MASK_OFFSET); |
1083 | } |
1084 | } |
1085 | } |
1086 | |
1087 | static int ioat3_dma_probe(struct ioatdma_device *ioat_dma, int dca) |
1088 | { |
1089 | struct pci_dev *pdev = ioat_dma->pdev; |
1090 | int dca_en = system_has_dca_enabled(pdev); |
1091 | struct dma_device *dma; |
1092 | struct dma_chan *c; |
1093 | struct ioatdma_chan *ioat_chan; |
1094 | int err; |
1095 | u16 val16; |
1096 | |
1097 | dma = &ioat_dma->dma_dev; |
1098 | dma->device_prep_dma_memcpy = ioat_dma_prep_memcpy_lock; |
1099 | dma->device_issue_pending = ioat_issue_pending; |
1100 | dma->device_alloc_chan_resources = ioat_alloc_chan_resources; |
1101 | dma->device_free_chan_resources = ioat_free_chan_resources; |
1102 | |
1103 | dma_cap_set(DMA_INTERRUPT, dma->cap_mask); |
1104 | dma->device_prep_dma_interrupt = ioat_prep_interrupt_lock; |
1105 | |
1106 | ioat_dma->cap = readl(addr: ioat_dma->reg_base + IOAT_DMA_CAP_OFFSET); |
1107 | |
1108 | if (is_xeon_cb32(pdev) || is_bwd_noraid(pdev)) |
1109 | ioat_dma->cap &= |
1110 | ~(IOAT_CAP_XOR | IOAT_CAP_PQ | IOAT_CAP_RAID16SS); |
1111 | |
1112 | /* dca is incompatible with raid operations */ |
1113 | if (dca_en && (ioat_dma->cap & (IOAT_CAP_XOR|IOAT_CAP_PQ))) |
1114 | ioat_dma->cap &= ~(IOAT_CAP_XOR|IOAT_CAP_PQ); |
1115 | |
1116 | if (ioat_dma->cap & IOAT_CAP_XOR) { |
1117 | dma->max_xor = 8; |
1118 | |
1119 | dma_cap_set(DMA_XOR, dma->cap_mask); |
1120 | dma->device_prep_dma_xor = ioat_prep_xor; |
1121 | |
1122 | dma_cap_set(DMA_XOR_VAL, dma->cap_mask); |
1123 | dma->device_prep_dma_xor_val = ioat_prep_xor_val; |
1124 | } |
1125 | |
1126 | if (ioat_dma->cap & IOAT_CAP_PQ) { |
1127 | |
1128 | dma->device_prep_dma_pq = ioat_prep_pq; |
1129 | dma->device_prep_dma_pq_val = ioat_prep_pq_val; |
1130 | dma_cap_set(DMA_PQ, dma->cap_mask); |
1131 | dma_cap_set(DMA_PQ_VAL, dma->cap_mask); |
1132 | |
1133 | if (ioat_dma->cap & IOAT_CAP_RAID16SS) |
1134 | dma_set_maxpq(dma, maxpq: 16, has_pq_continue: 0); |
1135 | else |
1136 | dma_set_maxpq(dma, maxpq: 8, has_pq_continue: 0); |
1137 | |
1138 | if (!(ioat_dma->cap & IOAT_CAP_XOR)) { |
1139 | dma->device_prep_dma_xor = ioat_prep_pqxor; |
1140 | dma->device_prep_dma_xor_val = ioat_prep_pqxor_val; |
1141 | dma_cap_set(DMA_XOR, dma->cap_mask); |
1142 | dma_cap_set(DMA_XOR_VAL, dma->cap_mask); |
1143 | |
1144 | if (ioat_dma->cap & IOAT_CAP_RAID16SS) |
1145 | dma->max_xor = 16; |
1146 | else |
1147 | dma->max_xor = 8; |
1148 | } |
1149 | } |
1150 | |
1151 | dma->device_tx_status = ioat_tx_status; |
1152 | |
1153 | /* starting with CB3.3 super extended descriptors are supported */ |
1154 | if (ioat_dma->cap & IOAT_CAP_RAID16SS) { |
1155 | char pool_name[14]; |
1156 | int i; |
1157 | |
1158 | for (i = 0; i < MAX_SED_POOLS; i++) { |
1159 | snprintf(buf: pool_name, size: 14, fmt: "ioat_hw%d_sed" , i); |
1160 | |
1161 | /* allocate SED DMA pool */ |
1162 | ioat_dma->sed_hw_pool[i] = dmam_pool_create(name: pool_name, |
1163 | dev: &pdev->dev, |
1164 | SED_SIZE * (i + 1), align: 64, allocation: 0); |
1165 | if (!ioat_dma->sed_hw_pool[i]) |
1166 | return -ENOMEM; |
1167 | |
1168 | } |
1169 | } |
1170 | |
1171 | if (!(ioat_dma->cap & (IOAT_CAP_XOR | IOAT_CAP_PQ))) |
1172 | dma_cap_set(DMA_PRIVATE, dma->cap_mask); |
1173 | |
1174 | err = ioat_probe(ioat_dma); |
1175 | if (err) |
1176 | return err; |
1177 | |
1178 | list_for_each_entry(c, &dma->channels, device_node) { |
1179 | ioat_chan = to_ioat_chan(c); |
1180 | writel(IOAT_DMA_DCA_ANY_CPU, |
1181 | addr: ioat_chan->reg_base + IOAT_DCACTRL_OFFSET); |
1182 | } |
1183 | |
1184 | err = ioat_register(ioat_dma); |
1185 | if (err) |
1186 | return err; |
1187 | |
1188 | ioat_kobject_add(ioat_dma, type: &ioat_ktype); |
1189 | |
1190 | if (dca) |
1191 | ioat_dma->dca = ioat_dca_init(pdev, iobase: ioat_dma->reg_base); |
1192 | |
1193 | /* disable relaxed ordering */ |
1194 | err = pcie_capability_read_word(dev: pdev, PCI_EXP_DEVCTL, val: &val16); |
1195 | if (err) |
1196 | return pcibios_err_to_errno(err); |
1197 | |
1198 | /* clear relaxed ordering enable */ |
1199 | val16 &= ~PCI_EXP_DEVCTL_RELAX_EN; |
1200 | err = pcie_capability_write_word(dev: pdev, PCI_EXP_DEVCTL, val: val16); |
1201 | if (err) |
1202 | return pcibios_err_to_errno(err); |
1203 | |
1204 | if (ioat_dma->cap & IOAT_CAP_DPS) |
1205 | writeb(val: ioat_pending_level + 1, |
1206 | addr: ioat_dma->reg_base + IOAT_PREFETCH_LIMIT_OFFSET); |
1207 | |
1208 | return 0; |
1209 | } |
1210 | |
1211 | static void ioat_shutdown(struct pci_dev *pdev) |
1212 | { |
1213 | struct ioatdma_device *ioat_dma = pci_get_drvdata(pdev); |
1214 | struct ioatdma_chan *ioat_chan; |
1215 | int i; |
1216 | |
1217 | if (!ioat_dma) |
1218 | return; |
1219 | |
1220 | for (i = 0; i < IOAT_MAX_CHANS; i++) { |
1221 | ioat_chan = ioat_dma->idx[i]; |
1222 | if (!ioat_chan) |
1223 | continue; |
1224 | |
1225 | spin_lock_bh(lock: &ioat_chan->prep_lock); |
1226 | set_bit(IOAT_CHAN_DOWN, addr: &ioat_chan->state); |
1227 | spin_unlock_bh(lock: &ioat_chan->prep_lock); |
1228 | /* |
1229 | * Synchronization rule for del_timer_sync(): |
1230 | * - The caller must not hold locks which would prevent |
1231 | * completion of the timer's handler. |
1232 | * So prep_lock cannot be held before calling it. |
1233 | */ |
1234 | del_timer_sync(timer: &ioat_chan->timer); |
1235 | |
1236 | /* this should quiesce then reset */ |
1237 | ioat_reset_hw(ioat_chan); |
1238 | } |
1239 | |
1240 | ioat_disable_interrupts(ioat_dma); |
1241 | } |
1242 | |
1243 | static void ioat_resume(struct ioatdma_device *ioat_dma) |
1244 | { |
1245 | struct ioatdma_chan *ioat_chan; |
1246 | u32 chanerr; |
1247 | int i; |
1248 | |
1249 | for (i = 0; i < IOAT_MAX_CHANS; i++) { |
1250 | ioat_chan = ioat_dma->idx[i]; |
1251 | if (!ioat_chan) |
1252 | continue; |
1253 | |
1254 | spin_lock_bh(lock: &ioat_chan->prep_lock); |
1255 | clear_bit(IOAT_CHAN_DOWN, addr: &ioat_chan->state); |
1256 | spin_unlock_bh(lock: &ioat_chan->prep_lock); |
1257 | |
1258 | chanerr = readl(addr: ioat_chan->reg_base + IOAT_CHANERR_OFFSET); |
1259 | writel(val: chanerr, addr: ioat_chan->reg_base + IOAT_CHANERR_OFFSET); |
1260 | |
1261 | /* no need to reset as shutdown already did that */ |
1262 | } |
1263 | } |
1264 | |
1265 | #define DRV_NAME "ioatdma" |
1266 | |
1267 | static pci_ers_result_t ioat_pcie_error_detected(struct pci_dev *pdev, |
1268 | pci_channel_state_t error) |
1269 | { |
1270 | dev_dbg(&pdev->dev, "%s: PCIe AER error %d\n" , DRV_NAME, error); |
1271 | |
1272 | /* quiesce and block I/O */ |
1273 | ioat_shutdown(pdev); |
1274 | |
1275 | return PCI_ERS_RESULT_NEED_RESET; |
1276 | } |
1277 | |
1278 | static pci_ers_result_t ioat_pcie_error_slot_reset(struct pci_dev *pdev) |
1279 | { |
1280 | pci_ers_result_t result = PCI_ERS_RESULT_RECOVERED; |
1281 | |
1282 | dev_dbg(&pdev->dev, "%s post reset handling\n" , DRV_NAME); |
1283 | |
1284 | if (pci_enable_device_mem(dev: pdev) < 0) { |
1285 | dev_err(&pdev->dev, |
1286 | "Failed to enable PCIe device after reset.\n" ); |
1287 | result = PCI_ERS_RESULT_DISCONNECT; |
1288 | } else { |
1289 | pci_set_master(dev: pdev); |
1290 | pci_restore_state(dev: pdev); |
1291 | pci_save_state(dev: pdev); |
1292 | pci_wake_from_d3(dev: pdev, enable: false); |
1293 | } |
1294 | |
1295 | return result; |
1296 | } |
1297 | |
1298 | static void ioat_pcie_error_resume(struct pci_dev *pdev) |
1299 | { |
1300 | struct ioatdma_device *ioat_dma = pci_get_drvdata(pdev); |
1301 | |
1302 | dev_dbg(&pdev->dev, "%s: AER handling resuming\n" , DRV_NAME); |
1303 | |
1304 | /* initialize and bring everything back */ |
1305 | ioat_resume(ioat_dma); |
1306 | } |
1307 | |
1308 | static const struct pci_error_handlers ioat_err_handler = { |
1309 | .error_detected = ioat_pcie_error_detected, |
1310 | .slot_reset = ioat_pcie_error_slot_reset, |
1311 | .resume = ioat_pcie_error_resume, |
1312 | }; |
1313 | |
1314 | static struct pci_driver ioat_pci_driver = { |
1315 | .name = DRV_NAME, |
1316 | .id_table = ioat_pci_tbl, |
1317 | .probe = ioat_pci_probe, |
1318 | .remove = ioat_remove, |
1319 | .shutdown = ioat_shutdown, |
1320 | .err_handler = &ioat_err_handler, |
1321 | }; |
1322 | |
1323 | static void release_ioatdma(struct dma_device *device) |
1324 | { |
1325 | struct ioatdma_device *d = to_ioatdma_device(device); |
1326 | int i; |
1327 | |
1328 | for (i = 0; i < IOAT_MAX_CHANS; i++) |
1329 | kfree(objp: d->idx[i]); |
1330 | |
1331 | dma_pool_destroy(pool: d->completion_pool); |
1332 | kfree(objp: d); |
1333 | } |
1334 | |
1335 | static struct ioatdma_device * |
1336 | alloc_ioatdma(struct pci_dev *pdev, void __iomem *iobase) |
1337 | { |
1338 | struct ioatdma_device *d = kzalloc(size: sizeof(*d), GFP_KERNEL); |
1339 | |
1340 | if (!d) |
1341 | return NULL; |
1342 | d->pdev = pdev; |
1343 | d->reg_base = iobase; |
1344 | d->dma_dev.device_release = release_ioatdma; |
1345 | return d; |
1346 | } |
1347 | |
1348 | static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) |
1349 | { |
1350 | void __iomem * const *iomap; |
1351 | struct device *dev = &pdev->dev; |
1352 | struct ioatdma_device *device; |
1353 | int err; |
1354 | |
1355 | err = pcim_enable_device(pdev); |
1356 | if (err) |
1357 | return err; |
1358 | |
1359 | err = pcim_iomap_regions(pdev, mask: 1 << IOAT_MMIO_BAR, DRV_NAME); |
1360 | if (err) |
1361 | return err; |
1362 | iomap = pcim_iomap_table(pdev); |
1363 | if (!iomap) |
1364 | return -ENOMEM; |
1365 | |
1366 | err = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(64)); |
1367 | if (err) |
1368 | return err; |
1369 | |
1370 | device = alloc_ioatdma(pdev, iobase: iomap[IOAT_MMIO_BAR]); |
1371 | if (!device) |
1372 | return -ENOMEM; |
1373 | pci_set_master(dev: pdev); |
1374 | pci_set_drvdata(pdev, data: device); |
1375 | |
1376 | device->version = readb(addr: device->reg_base + IOAT_VER_OFFSET); |
1377 | if (device->version >= IOAT_VER_3_4) |
1378 | ioat_dca_enabled = 0; |
1379 | if (device->version >= IOAT_VER_3_0) { |
1380 | if (is_skx_ioat(pdev)) |
1381 | device->version = IOAT_VER_3_2; |
1382 | err = ioat3_dma_probe(ioat_dma: device, dca: ioat_dca_enabled); |
1383 | } else |
1384 | return -ENODEV; |
1385 | |
1386 | if (err) { |
1387 | dev_err(dev, "Intel(R) I/OAT DMA Engine init failed\n" ); |
1388 | return -ENODEV; |
1389 | } |
1390 | |
1391 | return 0; |
1392 | } |
1393 | |
1394 | static void ioat_remove(struct pci_dev *pdev) |
1395 | { |
1396 | struct ioatdma_device *device = pci_get_drvdata(pdev); |
1397 | |
1398 | if (!device) |
1399 | return; |
1400 | |
1401 | ioat_shutdown(pdev); |
1402 | |
1403 | dev_err(&pdev->dev, "Removing dma and dca services\n" ); |
1404 | if (device->dca) { |
1405 | unregister_dca_provider(dca: device->dca, dev: &pdev->dev); |
1406 | free_dca_provider(dca: device->dca); |
1407 | device->dca = NULL; |
1408 | } |
1409 | |
1410 | ioat_dma_remove(ioat_dma: device); |
1411 | } |
1412 | |
1413 | static int __init ioat_init_module(void) |
1414 | { |
1415 | int err = -ENOMEM; |
1416 | |
1417 | pr_info("%s: Intel(R) QuickData Technology Driver %s\n" , |
1418 | DRV_NAME, IOAT_DMA_VERSION); |
1419 | |
1420 | ioat_cache = kmem_cache_create(name: "ioat" , size: sizeof(struct ioat_ring_ent), |
1421 | align: 0, SLAB_HWCACHE_ALIGN, NULL); |
1422 | if (!ioat_cache) |
1423 | return -ENOMEM; |
1424 | |
1425 | ioat_sed_cache = KMEM_CACHE(ioat_sed_ent, 0); |
1426 | if (!ioat_sed_cache) |
1427 | goto err_ioat_cache; |
1428 | |
1429 | err = pci_register_driver(&ioat_pci_driver); |
1430 | if (err) |
1431 | goto err_ioat3_cache; |
1432 | |
1433 | return 0; |
1434 | |
1435 | err_ioat3_cache: |
1436 | kmem_cache_destroy(s: ioat_sed_cache); |
1437 | |
1438 | err_ioat_cache: |
1439 | kmem_cache_destroy(s: ioat_cache); |
1440 | |
1441 | return err; |
1442 | } |
1443 | module_init(ioat_init_module); |
1444 | |
1445 | static void __exit ioat_exit_module(void) |
1446 | { |
1447 | pci_unregister_driver(dev: &ioat_pci_driver); |
1448 | kmem_cache_destroy(s: ioat_cache); |
1449 | } |
1450 | module_exit(ioat_exit_module); |
1451 | |