1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Intel I/OAT DMA Linux driver |
4 | * Copyright(c) 2004 - 2015 Intel Corporation. |
5 | */ |
6 | |
7 | /* |
8 | * This driver supports an Intel I/OAT DMA engine, which does asynchronous |
9 | * copy operations. |
10 | */ |
11 | |
12 | #include <linux/init.h> |
13 | #include <linux/module.h> |
14 | #include <linux/slab.h> |
15 | #include <linux/pci.h> |
16 | #include <linux/interrupt.h> |
17 | #include <linux/dmaengine.h> |
18 | #include <linux/delay.h> |
19 | #include <linux/dma-mapping.h> |
20 | #include <linux/workqueue.h> |
21 | #include <linux/prefetch.h> |
22 | #include <linux/sizes.h> |
23 | #include "dma.h" |
24 | #include "registers.h" |
25 | #include "hw.h" |
26 | |
27 | #include "../dmaengine.h" |
28 | |
29 | static int completion_timeout = 200; |
30 | module_param(completion_timeout, int, 0644); |
31 | MODULE_PARM_DESC(completion_timeout, |
32 | "set ioat completion timeout [msec] (default 200 [msec])" ); |
33 | static int idle_timeout = 2000; |
34 | module_param(idle_timeout, int, 0644); |
35 | MODULE_PARM_DESC(idle_timeout, |
36 | "set ioat idle timeout [msec] (default 2000 [msec])" ); |
37 | |
38 | #define IDLE_TIMEOUT msecs_to_jiffies(idle_timeout) |
39 | #define COMPLETION_TIMEOUT msecs_to_jiffies(completion_timeout) |
40 | |
41 | static char *chanerr_str[] = { |
42 | "DMA Transfer Source Address Error" , |
43 | "DMA Transfer Destination Address Error" , |
44 | "Next Descriptor Address Error" , |
45 | "Descriptor Error" , |
46 | "Chan Address Value Error" , |
47 | "CHANCMD Error" , |
48 | "Chipset Uncorrectable Data Integrity Error" , |
49 | "DMA Uncorrectable Data Integrity Error" , |
50 | "Read Data Error" , |
51 | "Write Data Error" , |
52 | "Descriptor Control Error" , |
53 | "Descriptor Transfer Size Error" , |
54 | "Completion Address Error" , |
55 | "Interrupt Configuration Error" , |
56 | "Super extended descriptor Address Error" , |
57 | "Unaffiliated Error" , |
58 | "CRC or XOR P Error" , |
59 | "XOR Q Error" , |
60 | "Descriptor Count Error" , |
61 | "DIF All F detect Error" , |
62 | "Guard Tag verification Error" , |
63 | "Application Tag verification Error" , |
64 | "Reference Tag verification Error" , |
65 | "Bundle Bit Error" , |
66 | "Result DIF All F detect Error" , |
67 | "Result Guard Tag verification Error" , |
68 | "Result Application Tag verification Error" , |
69 | "Result Reference Tag verification Error" , |
70 | }; |
71 | |
72 | static void ioat_eh(struct ioatdma_chan *ioat_chan); |
73 | |
74 | static void ioat_print_chanerrs(struct ioatdma_chan *ioat_chan, u32 chanerr) |
75 | { |
76 | int i; |
77 | |
78 | for (i = 0; i < ARRAY_SIZE(chanerr_str); i++) { |
79 | if ((chanerr >> i) & 1) { |
80 | dev_err(to_dev(ioat_chan), "Err(%d): %s\n" , |
81 | i, chanerr_str[i]); |
82 | } |
83 | } |
84 | } |
85 | |
86 | /** |
87 | * ioat_dma_do_interrupt - handler used for single vector interrupt mode |
88 | * @irq: interrupt id |
89 | * @data: interrupt data |
90 | */ |
91 | irqreturn_t ioat_dma_do_interrupt(int irq, void *data) |
92 | { |
93 | struct ioatdma_device *instance = data; |
94 | struct ioatdma_chan *ioat_chan; |
95 | unsigned long attnstatus; |
96 | int bit; |
97 | u8 intrctrl; |
98 | |
99 | intrctrl = readb(addr: instance->reg_base + IOAT_INTRCTRL_OFFSET); |
100 | |
101 | if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN)) |
102 | return IRQ_NONE; |
103 | |
104 | if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) { |
105 | writeb(val: intrctrl, addr: instance->reg_base + IOAT_INTRCTRL_OFFSET); |
106 | return IRQ_NONE; |
107 | } |
108 | |
109 | attnstatus = readl(addr: instance->reg_base + IOAT_ATTNSTATUS_OFFSET); |
110 | for_each_set_bit(bit, &attnstatus, BITS_PER_LONG) { |
111 | ioat_chan = ioat_chan_by_index(ioat_dma: instance, index: bit); |
112 | if (test_bit(IOAT_RUN, &ioat_chan->state)) |
113 | tasklet_schedule(t: &ioat_chan->cleanup_task); |
114 | } |
115 | |
116 | writeb(val: intrctrl, addr: instance->reg_base + IOAT_INTRCTRL_OFFSET); |
117 | return IRQ_HANDLED; |
118 | } |
119 | |
120 | /** |
121 | * ioat_dma_do_interrupt_msix - handler used for vector-per-channel interrupt mode |
122 | * @irq: interrupt id |
123 | * @data: interrupt data |
124 | */ |
125 | irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data) |
126 | { |
127 | struct ioatdma_chan *ioat_chan = data; |
128 | |
129 | if (test_bit(IOAT_RUN, &ioat_chan->state)) |
130 | tasklet_schedule(t: &ioat_chan->cleanup_task); |
131 | |
132 | return IRQ_HANDLED; |
133 | } |
134 | |
135 | void ioat_stop(struct ioatdma_chan *ioat_chan) |
136 | { |
137 | struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma; |
138 | struct pci_dev *pdev = ioat_dma->pdev; |
139 | int chan_id = chan_num(ioat_chan); |
140 | struct msix_entry *msix; |
141 | |
142 | /* 1/ stop irq from firing tasklets |
143 | * 2/ stop the tasklet from re-arming irqs |
144 | */ |
145 | clear_bit(IOAT_RUN, addr: &ioat_chan->state); |
146 | |
147 | /* flush inflight interrupts */ |
148 | switch (ioat_dma->irq_mode) { |
149 | case IOAT_MSIX: |
150 | msix = &ioat_dma->msix_entries[chan_id]; |
151 | synchronize_irq(irq: msix->vector); |
152 | break; |
153 | case IOAT_MSI: |
154 | case IOAT_INTX: |
155 | synchronize_irq(irq: pdev->irq); |
156 | break; |
157 | default: |
158 | break; |
159 | } |
160 | |
161 | /* flush inflight timers */ |
162 | del_timer_sync(timer: &ioat_chan->timer); |
163 | |
164 | /* flush inflight tasklet runs */ |
165 | tasklet_kill(t: &ioat_chan->cleanup_task); |
166 | |
167 | /* final cleanup now that everything is quiesced and can't re-arm */ |
168 | ioat_cleanup_event(t: &ioat_chan->cleanup_task); |
169 | } |
170 | |
171 | static void __ioat_issue_pending(struct ioatdma_chan *ioat_chan) |
172 | { |
173 | ioat_chan->dmacount += ioat_ring_pending(ioat_chan); |
174 | ioat_chan->issued = ioat_chan->head; |
175 | writew(val: ioat_chan->dmacount, |
176 | addr: ioat_chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET); |
177 | dev_dbg(to_dev(ioat_chan), |
178 | "%s: head: %#x tail: %#x issued: %#x count: %#x\n" , |
179 | __func__, ioat_chan->head, ioat_chan->tail, |
180 | ioat_chan->issued, ioat_chan->dmacount); |
181 | } |
182 | |
183 | void ioat_issue_pending(struct dma_chan *c) |
184 | { |
185 | struct ioatdma_chan *ioat_chan = to_ioat_chan(c); |
186 | |
187 | if (ioat_ring_pending(ioat_chan)) { |
188 | spin_lock_bh(lock: &ioat_chan->prep_lock); |
189 | __ioat_issue_pending(ioat_chan); |
190 | spin_unlock_bh(lock: &ioat_chan->prep_lock); |
191 | } |
192 | } |
193 | |
194 | /** |
195 | * ioat_update_pending - log pending descriptors |
196 | * @ioat_chan: ioat+ channel |
197 | * |
198 | * Check if the number of unsubmitted descriptors has exceeded the |
199 | * watermark. Called with prep_lock held |
200 | */ |
201 | static void ioat_update_pending(struct ioatdma_chan *ioat_chan) |
202 | { |
203 | if (ioat_ring_pending(ioat_chan) > ioat_pending_level) |
204 | __ioat_issue_pending(ioat_chan); |
205 | } |
206 | |
207 | static void __ioat_start_null_desc(struct ioatdma_chan *ioat_chan) |
208 | { |
209 | struct ioat_ring_ent *desc; |
210 | struct ioat_dma_descriptor *hw; |
211 | |
212 | if (ioat_ring_space(ioat_chan) < 1) { |
213 | dev_err(to_dev(ioat_chan), |
214 | "Unable to start null desc - ring full\n" ); |
215 | return; |
216 | } |
217 | |
218 | dev_dbg(to_dev(ioat_chan), |
219 | "%s: head: %#x tail: %#x issued: %#x\n" , |
220 | __func__, ioat_chan->head, ioat_chan->tail, ioat_chan->issued); |
221 | desc = ioat_get_ring_ent(ioat_chan, idx: ioat_chan->head); |
222 | |
223 | hw = desc->hw; |
224 | hw->ctl = 0; |
225 | hw->ctl_f.null = 1; |
226 | hw->ctl_f.int_en = 1; |
227 | hw->ctl_f.compl_write = 1; |
228 | /* set size to non-zero value (channel returns error when size is 0) */ |
229 | hw->size = NULL_DESC_BUFFER_SIZE; |
230 | hw->src_addr = 0; |
231 | hw->dst_addr = 0; |
232 | async_tx_ack(tx: &desc->txd); |
233 | ioat_set_chainaddr(ioat_chan, addr: desc->txd.phys); |
234 | dump_desc_dbg(ioat_chan, desc); |
235 | /* make sure descriptors are written before we submit */ |
236 | wmb(); |
237 | ioat_chan->head += 1; |
238 | __ioat_issue_pending(ioat_chan); |
239 | } |
240 | |
241 | void ioat_start_null_desc(struct ioatdma_chan *ioat_chan) |
242 | { |
243 | spin_lock_bh(lock: &ioat_chan->prep_lock); |
244 | if (!test_bit(IOAT_CHAN_DOWN, &ioat_chan->state)) |
245 | __ioat_start_null_desc(ioat_chan); |
246 | spin_unlock_bh(lock: &ioat_chan->prep_lock); |
247 | } |
248 | |
249 | static void __ioat_restart_chan(struct ioatdma_chan *ioat_chan) |
250 | { |
251 | /* set the tail to be re-issued */ |
252 | ioat_chan->issued = ioat_chan->tail; |
253 | ioat_chan->dmacount = 0; |
254 | mod_timer(timer: &ioat_chan->timer, expires: jiffies + COMPLETION_TIMEOUT); |
255 | |
256 | dev_dbg(to_dev(ioat_chan), |
257 | "%s: head: %#x tail: %#x issued: %#x count: %#x\n" , |
258 | __func__, ioat_chan->head, ioat_chan->tail, |
259 | ioat_chan->issued, ioat_chan->dmacount); |
260 | |
261 | if (ioat_ring_pending(ioat_chan)) { |
262 | struct ioat_ring_ent *desc; |
263 | |
264 | desc = ioat_get_ring_ent(ioat_chan, idx: ioat_chan->tail); |
265 | ioat_set_chainaddr(ioat_chan, addr: desc->txd.phys); |
266 | __ioat_issue_pending(ioat_chan); |
267 | } else |
268 | __ioat_start_null_desc(ioat_chan); |
269 | } |
270 | |
271 | static int ioat_quiesce(struct ioatdma_chan *ioat_chan, unsigned long tmo) |
272 | { |
273 | unsigned long end = jiffies + tmo; |
274 | int err = 0; |
275 | u32 status; |
276 | |
277 | status = ioat_chansts(ioat_chan); |
278 | if (is_ioat_active(status) || is_ioat_idle(status)) |
279 | ioat_suspend(ioat_chan); |
280 | while (is_ioat_active(status) || is_ioat_idle(status)) { |
281 | if (tmo && time_after(jiffies, end)) { |
282 | err = -ETIMEDOUT; |
283 | break; |
284 | } |
285 | status = ioat_chansts(ioat_chan); |
286 | cpu_relax(); |
287 | } |
288 | |
289 | return err; |
290 | } |
291 | |
292 | static int ioat_reset_sync(struct ioatdma_chan *ioat_chan, unsigned long tmo) |
293 | { |
294 | unsigned long end = jiffies + tmo; |
295 | int err = 0; |
296 | |
297 | ioat_reset(ioat_chan); |
298 | while (ioat_reset_pending(ioat_chan)) { |
299 | if (end && time_after(jiffies, end)) { |
300 | err = -ETIMEDOUT; |
301 | break; |
302 | } |
303 | cpu_relax(); |
304 | } |
305 | |
306 | return err; |
307 | } |
308 | |
309 | static dma_cookie_t ioat_tx_submit_unlock(struct dma_async_tx_descriptor *tx) |
310 | __releases(&ioat_chan->prep_lock) |
311 | { |
312 | struct dma_chan *c = tx->chan; |
313 | struct ioatdma_chan *ioat_chan = to_ioat_chan(c); |
314 | dma_cookie_t cookie; |
315 | |
316 | cookie = dma_cookie_assign(tx); |
317 | dev_dbg(to_dev(ioat_chan), "%s: cookie: %d\n" , __func__, cookie); |
318 | |
319 | if (!test_and_set_bit(IOAT_CHAN_ACTIVE, addr: &ioat_chan->state)) |
320 | mod_timer(timer: &ioat_chan->timer, expires: jiffies + COMPLETION_TIMEOUT); |
321 | |
322 | /* make descriptor updates visible before advancing ioat->head, |
323 | * this is purposefully not smp_wmb() since we are also |
324 | * publishing the descriptor updates to a dma device |
325 | */ |
326 | wmb(); |
327 | |
328 | ioat_chan->head += ioat_chan->produce; |
329 | |
330 | ioat_update_pending(ioat_chan); |
331 | spin_unlock_bh(lock: &ioat_chan->prep_lock); |
332 | |
333 | return cookie; |
334 | } |
335 | |
336 | static struct ioat_ring_ent * |
337 | ioat_alloc_ring_ent(struct dma_chan *chan, int idx, gfp_t flags) |
338 | { |
339 | struct ioat_dma_descriptor *hw; |
340 | struct ioat_ring_ent *desc; |
341 | struct ioatdma_chan *ioat_chan = to_ioat_chan(c: chan); |
342 | int chunk; |
343 | dma_addr_t phys; |
344 | u8 *pos; |
345 | off_t offs; |
346 | |
347 | chunk = idx / IOAT_DESCS_PER_CHUNK; |
348 | idx &= (IOAT_DESCS_PER_CHUNK - 1); |
349 | offs = idx * IOAT_DESC_SZ; |
350 | pos = (u8 *)ioat_chan->descs[chunk].virt + offs; |
351 | phys = ioat_chan->descs[chunk].hw + offs; |
352 | hw = (struct ioat_dma_descriptor *)pos; |
353 | memset(hw, 0, sizeof(*hw)); |
354 | |
355 | desc = kmem_cache_zalloc(k: ioat_cache, flags); |
356 | if (!desc) |
357 | return NULL; |
358 | |
359 | dma_async_tx_descriptor_init(tx: &desc->txd, chan); |
360 | desc->txd.tx_submit = ioat_tx_submit_unlock; |
361 | desc->hw = hw; |
362 | desc->txd.phys = phys; |
363 | return desc; |
364 | } |
365 | |
366 | void ioat_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan) |
367 | { |
368 | kmem_cache_free(s: ioat_cache, objp: desc); |
369 | } |
370 | |
371 | struct ioat_ring_ent ** |
372 | ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags) |
373 | { |
374 | struct ioatdma_chan *ioat_chan = to_ioat_chan(c); |
375 | struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma; |
376 | struct ioat_ring_ent **ring; |
377 | int total_descs = 1 << order; |
378 | int i, chunks; |
379 | |
380 | /* allocate the array to hold the software ring */ |
381 | ring = kcalloc(n: total_descs, size: sizeof(*ring), flags); |
382 | if (!ring) |
383 | return NULL; |
384 | |
385 | chunks = (total_descs * IOAT_DESC_SZ) / IOAT_CHUNK_SIZE; |
386 | ioat_chan->desc_chunks = chunks; |
387 | |
388 | for (i = 0; i < chunks; i++) { |
389 | struct ioat_descs *descs = &ioat_chan->descs[i]; |
390 | |
391 | descs->virt = dma_alloc_coherent(to_dev(ioat_chan), |
392 | IOAT_CHUNK_SIZE, dma_handle: &descs->hw, gfp: flags); |
393 | if (!descs->virt) { |
394 | int idx; |
395 | |
396 | for (idx = 0; idx < i; idx++) { |
397 | descs = &ioat_chan->descs[idx]; |
398 | dma_free_coherent(to_dev(ioat_chan), |
399 | IOAT_CHUNK_SIZE, |
400 | cpu_addr: descs->virt, dma_handle: descs->hw); |
401 | descs->virt = NULL; |
402 | descs->hw = 0; |
403 | } |
404 | |
405 | ioat_chan->desc_chunks = 0; |
406 | kfree(objp: ring); |
407 | return NULL; |
408 | } |
409 | } |
410 | |
411 | for (i = 0; i < total_descs; i++) { |
412 | ring[i] = ioat_alloc_ring_ent(chan: c, idx: i, flags); |
413 | if (!ring[i]) { |
414 | int idx; |
415 | |
416 | while (i--) |
417 | ioat_free_ring_ent(desc: ring[i], chan: c); |
418 | |
419 | for (idx = 0; idx < ioat_chan->desc_chunks; idx++) { |
420 | dma_free_coherent(to_dev(ioat_chan), |
421 | IOAT_CHUNK_SIZE, |
422 | cpu_addr: ioat_chan->descs[idx].virt, |
423 | dma_handle: ioat_chan->descs[idx].hw); |
424 | ioat_chan->descs[idx].virt = NULL; |
425 | ioat_chan->descs[idx].hw = 0; |
426 | } |
427 | |
428 | ioat_chan->desc_chunks = 0; |
429 | kfree(objp: ring); |
430 | return NULL; |
431 | } |
432 | set_desc_id(ring[i], i); |
433 | } |
434 | |
435 | /* link descs */ |
436 | for (i = 0; i < total_descs-1; i++) { |
437 | struct ioat_ring_ent *next = ring[i+1]; |
438 | struct ioat_dma_descriptor *hw = ring[i]->hw; |
439 | |
440 | hw->next = next->txd.phys; |
441 | } |
442 | ring[i]->hw->next = ring[0]->txd.phys; |
443 | |
444 | /* setup descriptor pre-fetching for v3.4 */ |
445 | if (ioat_dma->cap & IOAT_CAP_DPS) { |
446 | u16 drsctl = IOAT_CHAN_DRSZ_2MB | IOAT_CHAN_DRS_EN; |
447 | |
448 | if (chunks == 1) |
449 | drsctl |= IOAT_CHAN_DRS_AUTOWRAP; |
450 | |
451 | writew(val: drsctl, addr: ioat_chan->reg_base + IOAT_CHAN_DRSCTL_OFFSET); |
452 | |
453 | } |
454 | |
455 | return ring; |
456 | } |
457 | |
458 | /** |
459 | * ioat_check_space_lock - verify space and grab ring producer lock |
460 | * @ioat_chan: ioat,3 channel (ring) to operate on |
461 | * @num_descs: allocation length |
462 | */ |
463 | int ioat_check_space_lock(struct ioatdma_chan *ioat_chan, int num_descs) |
464 | __acquires(&ioat_chan->prep_lock) |
465 | { |
466 | spin_lock_bh(lock: &ioat_chan->prep_lock); |
467 | /* never allow the last descriptor to be consumed, we need at |
468 | * least one free at all times to allow for on-the-fly ring |
469 | * resizing. |
470 | */ |
471 | if (likely(ioat_ring_space(ioat_chan) > num_descs)) { |
472 | dev_dbg(to_dev(ioat_chan), "%s: num_descs: %d (%x:%x:%x)\n" , |
473 | __func__, num_descs, ioat_chan->head, |
474 | ioat_chan->tail, ioat_chan->issued); |
475 | ioat_chan->produce = num_descs; |
476 | return 0; /* with ioat->prep_lock held */ |
477 | } |
478 | spin_unlock_bh(lock: &ioat_chan->prep_lock); |
479 | |
480 | dev_dbg_ratelimited(to_dev(ioat_chan), |
481 | "%s: ring full! num_descs: %d (%x:%x:%x)\n" , |
482 | __func__, num_descs, ioat_chan->head, |
483 | ioat_chan->tail, ioat_chan->issued); |
484 | |
485 | /* progress reclaim in the allocation failure case we may be |
486 | * called under bh_disabled so we need to trigger the timer |
487 | * event directly |
488 | */ |
489 | if (time_is_before_jiffies(ioat_chan->timer.expires) |
490 | && timer_pending(timer: &ioat_chan->timer)) { |
491 | mod_timer(timer: &ioat_chan->timer, expires: jiffies + COMPLETION_TIMEOUT); |
492 | ioat_timer_event(t: &ioat_chan->timer); |
493 | } |
494 | |
495 | return -ENOMEM; |
496 | } |
497 | |
498 | static bool desc_has_ext(struct ioat_ring_ent *desc) |
499 | { |
500 | struct ioat_dma_descriptor *hw = desc->hw; |
501 | |
502 | if (hw->ctl_f.op == IOAT_OP_XOR || |
503 | hw->ctl_f.op == IOAT_OP_XOR_VAL) { |
504 | struct ioat_xor_descriptor *xor = desc->xor; |
505 | |
506 | if (src_cnt_to_sw(xor->ctl_f.src_cnt) > 5) |
507 | return true; |
508 | } else if (hw->ctl_f.op == IOAT_OP_PQ || |
509 | hw->ctl_f.op == IOAT_OP_PQ_VAL) { |
510 | struct ioat_pq_descriptor *pq = desc->pq; |
511 | |
512 | if (src_cnt_to_sw(pq->ctl_f.src_cnt) > 3) |
513 | return true; |
514 | } |
515 | |
516 | return false; |
517 | } |
518 | |
519 | static void |
520 | ioat_free_sed(struct ioatdma_device *ioat_dma, struct ioat_sed_ent *sed) |
521 | { |
522 | if (!sed) |
523 | return; |
524 | |
525 | dma_pool_free(pool: ioat_dma->sed_hw_pool[sed->hw_pool], vaddr: sed->hw, addr: sed->dma); |
526 | kmem_cache_free(s: ioat_sed_cache, objp: sed); |
527 | } |
528 | |
529 | static u64 ioat_get_current_completion(struct ioatdma_chan *ioat_chan) |
530 | { |
531 | u64 phys_complete; |
532 | u64 completion; |
533 | |
534 | completion = *ioat_chan->completion; |
535 | phys_complete = ioat_chansts_to_addr(status: completion); |
536 | |
537 | dev_dbg(to_dev(ioat_chan), "%s: phys_complete: %#llx\n" , __func__, |
538 | (unsigned long long) phys_complete); |
539 | |
540 | return phys_complete; |
541 | } |
542 | |
543 | static bool ioat_cleanup_preamble(struct ioatdma_chan *ioat_chan, |
544 | u64 *phys_complete) |
545 | { |
546 | *phys_complete = ioat_get_current_completion(ioat_chan); |
547 | if (*phys_complete == ioat_chan->last_completion) |
548 | return false; |
549 | |
550 | clear_bit(IOAT_COMPLETION_ACK, addr: &ioat_chan->state); |
551 | mod_timer(timer: &ioat_chan->timer, expires: jiffies + COMPLETION_TIMEOUT); |
552 | |
553 | return true; |
554 | } |
555 | |
556 | static void |
557 | desc_get_errstat(struct ioatdma_chan *ioat_chan, struct ioat_ring_ent *desc) |
558 | { |
559 | struct ioat_dma_descriptor *hw = desc->hw; |
560 | |
561 | switch (hw->ctl_f.op) { |
562 | case IOAT_OP_PQ_VAL: |
563 | case IOAT_OP_PQ_VAL_16S: |
564 | { |
565 | struct ioat_pq_descriptor *pq = desc->pq; |
566 | |
567 | /* check if there's error written */ |
568 | if (!pq->dwbes_f.wbes) |
569 | return; |
570 | |
571 | /* need to set a chanerr var for checking to clear later */ |
572 | |
573 | if (pq->dwbes_f.p_val_err) |
574 | *desc->result |= SUM_CHECK_P_RESULT; |
575 | |
576 | if (pq->dwbes_f.q_val_err) |
577 | *desc->result |= SUM_CHECK_Q_RESULT; |
578 | |
579 | return; |
580 | } |
581 | default: |
582 | return; |
583 | } |
584 | } |
585 | |
586 | /** |
587 | * __ioat_cleanup - reclaim used descriptors |
588 | * @ioat_chan: channel (ring) to clean |
589 | * @phys_complete: zeroed (or not) completion address (from status) |
590 | */ |
591 | static void __ioat_cleanup(struct ioatdma_chan *ioat_chan, dma_addr_t phys_complete) |
592 | { |
593 | struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma; |
594 | struct ioat_ring_ent *desc; |
595 | bool seen_current = false; |
596 | int idx = ioat_chan->tail, i; |
597 | u16 active; |
598 | |
599 | dev_dbg(to_dev(ioat_chan), "%s: head: %#x tail: %#x issued: %#x\n" , |
600 | __func__, ioat_chan->head, ioat_chan->tail, ioat_chan->issued); |
601 | |
602 | /* |
603 | * At restart of the channel, the completion address and the |
604 | * channel status will be 0 due to starting a new chain. Since |
605 | * it's new chain and the first descriptor "fails", there is |
606 | * nothing to clean up. We do not want to reap the entire submitted |
607 | * chain due to this 0 address value and then BUG. |
608 | */ |
609 | if (!phys_complete) |
610 | return; |
611 | |
612 | active = ioat_ring_active(ioat_chan); |
613 | for (i = 0; i < active && !seen_current; i++) { |
614 | struct dma_async_tx_descriptor *tx; |
615 | |
616 | prefetch(ioat_get_ring_ent(ioat_chan, idx + i + 1)); |
617 | desc = ioat_get_ring_ent(ioat_chan, idx: idx + i); |
618 | dump_desc_dbg(ioat_chan, desc); |
619 | |
620 | /* set err stat if we are using dwbes */ |
621 | if (ioat_dma->cap & IOAT_CAP_DWBES) |
622 | desc_get_errstat(ioat_chan, desc); |
623 | |
624 | tx = &desc->txd; |
625 | if (tx->cookie) { |
626 | dma_cookie_complete(tx); |
627 | dma_descriptor_unmap(tx); |
628 | dmaengine_desc_get_callback_invoke(tx, NULL); |
629 | tx->callback = NULL; |
630 | tx->callback_result = NULL; |
631 | } |
632 | |
633 | if (tx->phys == phys_complete) |
634 | seen_current = true; |
635 | |
636 | /* skip extended descriptors */ |
637 | if (desc_has_ext(desc)) { |
638 | BUG_ON(i + 1 >= active); |
639 | i++; |
640 | } |
641 | |
642 | /* cleanup super extended descriptors */ |
643 | if (desc->sed) { |
644 | ioat_free_sed(ioat_dma, sed: desc->sed); |
645 | desc->sed = NULL; |
646 | } |
647 | } |
648 | |
649 | /* finish all descriptor reads before incrementing tail */ |
650 | smp_mb(); |
651 | ioat_chan->tail = idx + i; |
652 | /* no active descs have written a completion? */ |
653 | BUG_ON(active && !seen_current); |
654 | ioat_chan->last_completion = phys_complete; |
655 | |
656 | if (active - i == 0) { |
657 | dev_dbg(to_dev(ioat_chan), "%s: cancel completion timeout\n" , |
658 | __func__); |
659 | mod_timer_pending(timer: &ioat_chan->timer, expires: jiffies + IDLE_TIMEOUT); |
660 | } |
661 | |
662 | /* microsecond delay by sysfs variable per pending descriptor */ |
663 | if (ioat_chan->intr_coalesce != ioat_chan->prev_intr_coalesce) { |
664 | writew(min((ioat_chan->intr_coalesce * (active - i)), |
665 | IOAT_INTRDELAY_MASK), |
666 | addr: ioat_chan->ioat_dma->reg_base + IOAT_INTRDELAY_OFFSET); |
667 | ioat_chan->prev_intr_coalesce = ioat_chan->intr_coalesce; |
668 | } |
669 | } |
670 | |
671 | static void ioat_cleanup(struct ioatdma_chan *ioat_chan) |
672 | { |
673 | u64 phys_complete; |
674 | |
675 | spin_lock_bh(lock: &ioat_chan->cleanup_lock); |
676 | |
677 | if (ioat_cleanup_preamble(ioat_chan, phys_complete: &phys_complete)) |
678 | __ioat_cleanup(ioat_chan, phys_complete); |
679 | |
680 | if (is_ioat_halted(status: *ioat_chan->completion)) { |
681 | u32 chanerr = readl(addr: ioat_chan->reg_base + IOAT_CHANERR_OFFSET); |
682 | |
683 | if (chanerr & |
684 | (IOAT_CHANERR_HANDLE_MASK | IOAT_CHANERR_RECOVER_MASK)) { |
685 | mod_timer_pending(timer: &ioat_chan->timer, expires: jiffies + IDLE_TIMEOUT); |
686 | ioat_eh(ioat_chan); |
687 | } |
688 | } |
689 | |
690 | spin_unlock_bh(lock: &ioat_chan->cleanup_lock); |
691 | } |
692 | |
693 | void ioat_cleanup_event(struct tasklet_struct *t) |
694 | { |
695 | struct ioatdma_chan *ioat_chan = from_tasklet(ioat_chan, t, cleanup_task); |
696 | |
697 | ioat_cleanup(ioat_chan); |
698 | if (!test_bit(IOAT_RUN, &ioat_chan->state)) |
699 | return; |
700 | writew(IOAT_CHANCTRL_RUN, addr: ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET); |
701 | } |
702 | |
703 | static void ioat_restart_channel(struct ioatdma_chan *ioat_chan) |
704 | { |
705 | u64 phys_complete; |
706 | |
707 | /* set the completion address register again */ |
708 | writel(lower_32_bits(ioat_chan->completion_dma), |
709 | addr: ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW); |
710 | writel(upper_32_bits(ioat_chan->completion_dma), |
711 | addr: ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH); |
712 | |
713 | ioat_quiesce(ioat_chan, tmo: 0); |
714 | if (ioat_cleanup_preamble(ioat_chan, phys_complete: &phys_complete)) |
715 | __ioat_cleanup(ioat_chan, phys_complete); |
716 | |
717 | __ioat_restart_chan(ioat_chan); |
718 | } |
719 | |
720 | |
721 | static void ioat_abort_descs(struct ioatdma_chan *ioat_chan) |
722 | { |
723 | struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma; |
724 | struct ioat_ring_ent *desc; |
725 | u16 active; |
726 | int idx = ioat_chan->tail, i; |
727 | |
728 | /* |
729 | * We assume that the failed descriptor has been processed. |
730 | * Now we are just returning all the remaining submitted |
731 | * descriptors to abort. |
732 | */ |
733 | active = ioat_ring_active(ioat_chan); |
734 | |
735 | /* we skip the failed descriptor that tail points to */ |
736 | for (i = 1; i < active; i++) { |
737 | struct dma_async_tx_descriptor *tx; |
738 | |
739 | prefetch(ioat_get_ring_ent(ioat_chan, idx + i + 1)); |
740 | desc = ioat_get_ring_ent(ioat_chan, idx: idx + i); |
741 | |
742 | tx = &desc->txd; |
743 | if (tx->cookie) { |
744 | struct dmaengine_result res; |
745 | |
746 | dma_cookie_complete(tx); |
747 | dma_descriptor_unmap(tx); |
748 | res.result = DMA_TRANS_ABORTED; |
749 | dmaengine_desc_get_callback_invoke(tx, result: &res); |
750 | tx->callback = NULL; |
751 | tx->callback_result = NULL; |
752 | } |
753 | |
754 | /* skip extended descriptors */ |
755 | if (desc_has_ext(desc)) { |
756 | WARN_ON(i + 1 >= active); |
757 | i++; |
758 | } |
759 | |
760 | /* cleanup super extended descriptors */ |
761 | if (desc->sed) { |
762 | ioat_free_sed(ioat_dma, sed: desc->sed); |
763 | desc->sed = NULL; |
764 | } |
765 | } |
766 | |
767 | smp_mb(); /* finish all descriptor reads before incrementing tail */ |
768 | ioat_chan->tail = idx + active; |
769 | |
770 | desc = ioat_get_ring_ent(ioat_chan, idx: ioat_chan->tail); |
771 | ioat_chan->last_completion = *ioat_chan->completion = desc->txd.phys; |
772 | } |
773 | |
774 | static void ioat_eh(struct ioatdma_chan *ioat_chan) |
775 | { |
776 | struct pci_dev *pdev = to_pdev(ioat_chan); |
777 | struct ioat_dma_descriptor *hw; |
778 | struct dma_async_tx_descriptor *tx; |
779 | u64 phys_complete; |
780 | struct ioat_ring_ent *desc; |
781 | u32 err_handled = 0; |
782 | u32 chanerr_int; |
783 | u32 chanerr; |
784 | bool abort = false; |
785 | struct dmaengine_result res; |
786 | |
787 | /* cleanup so tail points to descriptor that caused the error */ |
788 | if (ioat_cleanup_preamble(ioat_chan, phys_complete: &phys_complete)) |
789 | __ioat_cleanup(ioat_chan, phys_complete); |
790 | |
791 | chanerr = readl(addr: ioat_chan->reg_base + IOAT_CHANERR_OFFSET); |
792 | pci_read_config_dword(dev: pdev, IOAT_PCI_CHANERR_INT_OFFSET, val: &chanerr_int); |
793 | |
794 | dev_dbg(to_dev(ioat_chan), "%s: error = %x:%x\n" , |
795 | __func__, chanerr, chanerr_int); |
796 | |
797 | desc = ioat_get_ring_ent(ioat_chan, idx: ioat_chan->tail); |
798 | hw = desc->hw; |
799 | dump_desc_dbg(ioat_chan, desc); |
800 | |
801 | switch (hw->ctl_f.op) { |
802 | case IOAT_OP_XOR_VAL: |
803 | if (chanerr & IOAT_CHANERR_XOR_P_OR_CRC_ERR) { |
804 | *desc->result |= SUM_CHECK_P_RESULT; |
805 | err_handled |= IOAT_CHANERR_XOR_P_OR_CRC_ERR; |
806 | } |
807 | break; |
808 | case IOAT_OP_PQ_VAL: |
809 | case IOAT_OP_PQ_VAL_16S: |
810 | if (chanerr & IOAT_CHANERR_XOR_P_OR_CRC_ERR) { |
811 | *desc->result |= SUM_CHECK_P_RESULT; |
812 | err_handled |= IOAT_CHANERR_XOR_P_OR_CRC_ERR; |
813 | } |
814 | if (chanerr & IOAT_CHANERR_XOR_Q_ERR) { |
815 | *desc->result |= SUM_CHECK_Q_RESULT; |
816 | err_handled |= IOAT_CHANERR_XOR_Q_ERR; |
817 | } |
818 | break; |
819 | } |
820 | |
821 | if (chanerr & IOAT_CHANERR_RECOVER_MASK) { |
822 | if (chanerr & IOAT_CHANERR_READ_DATA_ERR) { |
823 | res.result = DMA_TRANS_READ_FAILED; |
824 | err_handled |= IOAT_CHANERR_READ_DATA_ERR; |
825 | } else if (chanerr & IOAT_CHANERR_WRITE_DATA_ERR) { |
826 | res.result = DMA_TRANS_WRITE_FAILED; |
827 | err_handled |= IOAT_CHANERR_WRITE_DATA_ERR; |
828 | } |
829 | |
830 | abort = true; |
831 | } else |
832 | res.result = DMA_TRANS_NOERROR; |
833 | |
834 | /* fault on unhandled error or spurious halt */ |
835 | if (chanerr ^ err_handled || chanerr == 0) { |
836 | dev_err(to_dev(ioat_chan), "%s: fatal error (%x:%x)\n" , |
837 | __func__, chanerr, err_handled); |
838 | dev_err(to_dev(ioat_chan), "Errors handled:\n" ); |
839 | ioat_print_chanerrs(ioat_chan, chanerr: err_handled); |
840 | dev_err(to_dev(ioat_chan), "Errors not handled:\n" ); |
841 | ioat_print_chanerrs(ioat_chan, chanerr: (chanerr & ~err_handled)); |
842 | |
843 | BUG(); |
844 | } |
845 | |
846 | /* cleanup the faulty descriptor since we are continuing */ |
847 | tx = &desc->txd; |
848 | if (tx->cookie) { |
849 | dma_cookie_complete(tx); |
850 | dma_descriptor_unmap(tx); |
851 | dmaengine_desc_get_callback_invoke(tx, result: &res); |
852 | tx->callback = NULL; |
853 | tx->callback_result = NULL; |
854 | } |
855 | |
856 | /* mark faulting descriptor as complete */ |
857 | *ioat_chan->completion = desc->txd.phys; |
858 | |
859 | spin_lock_bh(lock: &ioat_chan->prep_lock); |
860 | /* we need abort all descriptors */ |
861 | if (abort) { |
862 | ioat_abort_descs(ioat_chan); |
863 | /* clean up the channel, we could be in weird state */ |
864 | ioat_reset_hw(ioat_chan); |
865 | } |
866 | |
867 | writel(val: chanerr, addr: ioat_chan->reg_base + IOAT_CHANERR_OFFSET); |
868 | pci_write_config_dword(dev: pdev, IOAT_PCI_CHANERR_INT_OFFSET, val: chanerr_int); |
869 | |
870 | ioat_restart_channel(ioat_chan); |
871 | spin_unlock_bh(lock: &ioat_chan->prep_lock); |
872 | } |
873 | |
874 | static void check_active(struct ioatdma_chan *ioat_chan) |
875 | { |
876 | if (ioat_ring_active(ioat_chan)) { |
877 | mod_timer(timer: &ioat_chan->timer, expires: jiffies + COMPLETION_TIMEOUT); |
878 | return; |
879 | } |
880 | |
881 | if (test_and_clear_bit(IOAT_CHAN_ACTIVE, addr: &ioat_chan->state)) |
882 | mod_timer_pending(timer: &ioat_chan->timer, expires: jiffies + IDLE_TIMEOUT); |
883 | } |
884 | |
885 | static void ioat_reboot_chan(struct ioatdma_chan *ioat_chan) |
886 | { |
887 | spin_lock_bh(lock: &ioat_chan->prep_lock); |
888 | set_bit(IOAT_CHAN_DOWN, addr: &ioat_chan->state); |
889 | spin_unlock_bh(lock: &ioat_chan->prep_lock); |
890 | |
891 | ioat_abort_descs(ioat_chan); |
892 | dev_warn(to_dev(ioat_chan), "Reset channel...\n" ); |
893 | ioat_reset_hw(ioat_chan); |
894 | dev_warn(to_dev(ioat_chan), "Restart channel...\n" ); |
895 | ioat_restart_channel(ioat_chan); |
896 | |
897 | spin_lock_bh(lock: &ioat_chan->prep_lock); |
898 | clear_bit(IOAT_CHAN_DOWN, addr: &ioat_chan->state); |
899 | spin_unlock_bh(lock: &ioat_chan->prep_lock); |
900 | } |
901 | |
902 | void ioat_timer_event(struct timer_list *t) |
903 | { |
904 | struct ioatdma_chan *ioat_chan = from_timer(ioat_chan, t, timer); |
905 | dma_addr_t phys_complete; |
906 | u64 status; |
907 | |
908 | status = ioat_chansts(ioat_chan); |
909 | |
910 | /* when halted due to errors check for channel |
911 | * programming errors before advancing the completion state |
912 | */ |
913 | if (is_ioat_halted(status)) { |
914 | u32 chanerr; |
915 | |
916 | chanerr = readl(addr: ioat_chan->reg_base + IOAT_CHANERR_OFFSET); |
917 | dev_err(to_dev(ioat_chan), "%s: Channel halted (%x)\n" , |
918 | __func__, chanerr); |
919 | dev_err(to_dev(ioat_chan), "Errors:\n" ); |
920 | ioat_print_chanerrs(ioat_chan, chanerr); |
921 | |
922 | if (test_bit(IOAT_RUN, &ioat_chan->state)) { |
923 | spin_lock_bh(lock: &ioat_chan->cleanup_lock); |
924 | ioat_reboot_chan(ioat_chan); |
925 | spin_unlock_bh(lock: &ioat_chan->cleanup_lock); |
926 | } |
927 | |
928 | return; |
929 | } |
930 | |
931 | spin_lock_bh(lock: &ioat_chan->cleanup_lock); |
932 | |
933 | /* handle the no-actives case */ |
934 | if (!ioat_ring_active(ioat_chan)) { |
935 | spin_lock_bh(lock: &ioat_chan->prep_lock); |
936 | check_active(ioat_chan); |
937 | spin_unlock_bh(lock: &ioat_chan->prep_lock); |
938 | goto unlock_out; |
939 | } |
940 | |
941 | /* handle the missed cleanup case */ |
942 | if (ioat_cleanup_preamble(ioat_chan, phys_complete: &phys_complete)) { |
943 | /* timer restarted in ioat_cleanup_preamble |
944 | * and IOAT_COMPLETION_ACK cleared |
945 | */ |
946 | __ioat_cleanup(ioat_chan, phys_complete); |
947 | goto unlock_out; |
948 | } |
949 | |
950 | /* if we haven't made progress and we have already |
951 | * acknowledged a pending completion once, then be more |
952 | * forceful with a restart |
953 | */ |
954 | if (test_bit(IOAT_COMPLETION_ACK, &ioat_chan->state)) { |
955 | u32 chanerr; |
956 | |
957 | chanerr = readl(addr: ioat_chan->reg_base + IOAT_CHANERR_OFFSET); |
958 | dev_err(to_dev(ioat_chan), "CHANSTS: %#Lx CHANERR: %#x\n" , |
959 | status, chanerr); |
960 | dev_err(to_dev(ioat_chan), "Errors:\n" ); |
961 | ioat_print_chanerrs(ioat_chan, chanerr); |
962 | |
963 | dev_dbg(to_dev(ioat_chan), "Active descriptors: %d\n" , |
964 | ioat_ring_active(ioat_chan)); |
965 | |
966 | ioat_reboot_chan(ioat_chan); |
967 | |
968 | goto unlock_out; |
969 | } |
970 | |
971 | /* handle missed issue pending case */ |
972 | if (ioat_ring_pending(ioat_chan)) { |
973 | dev_warn(to_dev(ioat_chan), |
974 | "Completion timeout with pending descriptors\n" ); |
975 | spin_lock_bh(lock: &ioat_chan->prep_lock); |
976 | __ioat_issue_pending(ioat_chan); |
977 | spin_unlock_bh(lock: &ioat_chan->prep_lock); |
978 | } |
979 | |
980 | set_bit(IOAT_COMPLETION_ACK, addr: &ioat_chan->state); |
981 | mod_timer(timer: &ioat_chan->timer, expires: jiffies + COMPLETION_TIMEOUT); |
982 | unlock_out: |
983 | spin_unlock_bh(lock: &ioat_chan->cleanup_lock); |
984 | } |
985 | |
986 | enum dma_status |
987 | ioat_tx_status(struct dma_chan *c, dma_cookie_t cookie, |
988 | struct dma_tx_state *txstate) |
989 | { |
990 | struct ioatdma_chan *ioat_chan = to_ioat_chan(c); |
991 | enum dma_status ret; |
992 | |
993 | ret = dma_cookie_status(chan: c, cookie, state: txstate); |
994 | if (ret == DMA_COMPLETE) |
995 | return ret; |
996 | |
997 | ioat_cleanup(ioat_chan); |
998 | |
999 | return dma_cookie_status(chan: c, cookie, state: txstate); |
1000 | } |
1001 | |
1002 | int ioat_reset_hw(struct ioatdma_chan *ioat_chan) |
1003 | { |
1004 | /* throw away whatever the channel was doing and get it |
1005 | * initialized, with ioat3 specific workarounds |
1006 | */ |
1007 | struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma; |
1008 | struct pci_dev *pdev = ioat_dma->pdev; |
1009 | u32 chanerr; |
1010 | u16 dev_id; |
1011 | int err; |
1012 | |
1013 | ioat_quiesce(ioat_chan, tmo: msecs_to_jiffies(m: 100)); |
1014 | |
1015 | chanerr = readl(addr: ioat_chan->reg_base + IOAT_CHANERR_OFFSET); |
1016 | writel(val: chanerr, addr: ioat_chan->reg_base + IOAT_CHANERR_OFFSET); |
1017 | |
1018 | if (ioat_dma->version < IOAT_VER_3_3) { |
1019 | /* clear any pending errors */ |
1020 | err = pci_read_config_dword(dev: pdev, |
1021 | IOAT_PCI_CHANERR_INT_OFFSET, val: &chanerr); |
1022 | if (err) { |
1023 | dev_err(&pdev->dev, |
1024 | "channel error register unreachable\n" ); |
1025 | return err; |
1026 | } |
1027 | pci_write_config_dword(dev: pdev, |
1028 | IOAT_PCI_CHANERR_INT_OFFSET, val: chanerr); |
1029 | |
1030 | /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit |
1031 | * (workaround for spurious config parity error after restart) |
1032 | */ |
1033 | pci_read_config_word(dev: pdev, IOAT_PCI_DEVICE_ID_OFFSET, val: &dev_id); |
1034 | if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0) { |
1035 | pci_write_config_dword(dev: pdev, |
1036 | IOAT_PCI_DMAUNCERRSTS_OFFSET, |
1037 | val: 0x10); |
1038 | } |
1039 | } |
1040 | |
1041 | if (is_bwd_ioat(pdev) && (ioat_dma->irq_mode == IOAT_MSIX)) { |
1042 | ioat_dma->msixtba0 = readq(addr: ioat_dma->reg_base + 0x1000); |
1043 | ioat_dma->msixdata0 = readq(addr: ioat_dma->reg_base + 0x1008); |
1044 | ioat_dma->msixpba = readq(addr: ioat_dma->reg_base + 0x1800); |
1045 | } |
1046 | |
1047 | |
1048 | err = ioat_reset_sync(ioat_chan, tmo: msecs_to_jiffies(m: 200)); |
1049 | if (!err) { |
1050 | if (is_bwd_ioat(pdev) && (ioat_dma->irq_mode == IOAT_MSIX)) { |
1051 | writeq(val: ioat_dma->msixtba0, addr: ioat_dma->reg_base + 0x1000); |
1052 | writeq(val: ioat_dma->msixdata0, addr: ioat_dma->reg_base + 0x1008); |
1053 | writeq(val: ioat_dma->msixpba, addr: ioat_dma->reg_base + 0x1800); |
1054 | } |
1055 | } |
1056 | |
1057 | if (err) |
1058 | dev_err(&pdev->dev, "Failed to reset: %d\n" , err); |
1059 | |
1060 | return err; |
1061 | } |
1062 | |