1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Qualcomm Technologies HIDMA DMA engine low level code |
4 | * |
5 | * Copyright (c) 2015-2016, The Linux Foundation. All rights reserved. |
6 | */ |
7 | |
8 | #include <linux/dmaengine.h> |
9 | #include <linux/slab.h> |
10 | #include <linux/interrupt.h> |
11 | #include <linux/mm.h> |
12 | #include <linux/highmem.h> |
13 | #include <linux/dma-mapping.h> |
14 | #include <linux/delay.h> |
15 | #include <linux/atomic.h> |
16 | #include <linux/iopoll.h> |
17 | #include <linux/kfifo.h> |
18 | #include <linux/bitops.h> |
19 | |
20 | #include "hidma.h" |
21 | |
22 | #define HIDMA_EVRE_SIZE 16 /* each EVRE is 16 bytes */ |
23 | |
24 | #define HIDMA_TRCA_CTRLSTS_REG 0x000 |
25 | #define HIDMA_TRCA_RING_LOW_REG 0x008 |
26 | #define HIDMA_TRCA_RING_HIGH_REG 0x00C |
27 | #define HIDMA_TRCA_RING_LEN_REG 0x010 |
28 | #define HIDMA_TRCA_DOORBELL_REG 0x400 |
29 | |
30 | #define HIDMA_EVCA_CTRLSTS_REG 0x000 |
31 | #define HIDMA_EVCA_INTCTRL_REG 0x004 |
32 | #define HIDMA_EVCA_RING_LOW_REG 0x008 |
33 | #define HIDMA_EVCA_RING_HIGH_REG 0x00C |
34 | #define HIDMA_EVCA_RING_LEN_REG 0x010 |
35 | #define HIDMA_EVCA_WRITE_PTR_REG 0x020 |
36 | #define HIDMA_EVCA_DOORBELL_REG 0x400 |
37 | |
38 | #define HIDMA_EVCA_IRQ_STAT_REG 0x100 |
39 | #define HIDMA_EVCA_IRQ_CLR_REG 0x108 |
40 | #define HIDMA_EVCA_IRQ_EN_REG 0x110 |
41 | |
42 | #define HIDMA_EVRE_CFG_IDX 0 |
43 | |
44 | #define HIDMA_EVRE_ERRINFO_BIT_POS 24 |
45 | #define HIDMA_EVRE_CODE_BIT_POS 28 |
46 | |
47 | #define HIDMA_EVRE_ERRINFO_MASK GENMASK(3, 0) |
48 | #define HIDMA_EVRE_CODE_MASK GENMASK(3, 0) |
49 | |
50 | #define HIDMA_CH_CONTROL_MASK GENMASK(7, 0) |
51 | #define HIDMA_CH_STATE_MASK GENMASK(7, 0) |
52 | #define HIDMA_CH_STATE_BIT_POS 0x8 |
53 | |
54 | #define HIDMA_IRQ_EV_CH_EOB_IRQ_BIT_POS 0 |
55 | #define HIDMA_IRQ_EV_CH_WR_RESP_BIT_POS 1 |
56 | #define HIDMA_IRQ_TR_CH_TRE_RD_RSP_ER_BIT_POS 9 |
57 | #define HIDMA_IRQ_TR_CH_DATA_RD_ER_BIT_POS 10 |
58 | #define HIDMA_IRQ_TR_CH_DATA_WR_ER_BIT_POS 11 |
59 | #define HIDMA_IRQ_TR_CH_INVALID_TRE_BIT_POS 14 |
60 | |
61 | #define ENABLE_IRQS (BIT(HIDMA_IRQ_EV_CH_EOB_IRQ_BIT_POS) | \ |
62 | BIT(HIDMA_IRQ_EV_CH_WR_RESP_BIT_POS) | \ |
63 | BIT(HIDMA_IRQ_TR_CH_TRE_RD_RSP_ER_BIT_POS) | \ |
64 | BIT(HIDMA_IRQ_TR_CH_DATA_RD_ER_BIT_POS) | \ |
65 | BIT(HIDMA_IRQ_TR_CH_DATA_WR_ER_BIT_POS) | \ |
66 | BIT(HIDMA_IRQ_TR_CH_INVALID_TRE_BIT_POS)) |
67 | |
68 | #define HIDMA_INCREMENT_ITERATOR(iter, size, ring_size) \ |
69 | do { \ |
70 | iter += size; \ |
71 | if (iter >= ring_size) \ |
72 | iter -= ring_size; \ |
73 | } while (0) |
74 | |
75 | #define HIDMA_CH_STATE(val) \ |
76 | ((val >> HIDMA_CH_STATE_BIT_POS) & HIDMA_CH_STATE_MASK) |
77 | |
78 | #define HIDMA_ERR_INT_MASK \ |
79 | (BIT(HIDMA_IRQ_TR_CH_INVALID_TRE_BIT_POS) | \ |
80 | BIT(HIDMA_IRQ_TR_CH_TRE_RD_RSP_ER_BIT_POS) | \ |
81 | BIT(HIDMA_IRQ_EV_CH_WR_RESP_BIT_POS) | \ |
82 | BIT(HIDMA_IRQ_TR_CH_DATA_RD_ER_BIT_POS) | \ |
83 | BIT(HIDMA_IRQ_TR_CH_DATA_WR_ER_BIT_POS)) |
84 | |
85 | enum ch_command { |
86 | HIDMA_CH_DISABLE = 0, |
87 | HIDMA_CH_ENABLE = 1, |
88 | HIDMA_CH_SUSPEND = 2, |
89 | HIDMA_CH_RESET = 9, |
90 | }; |
91 | |
92 | enum ch_state { |
93 | HIDMA_CH_DISABLED = 0, |
94 | HIDMA_CH_ENABLED = 1, |
95 | HIDMA_CH_RUNNING = 2, |
96 | HIDMA_CH_SUSPENDED = 3, |
97 | HIDMA_CH_STOPPED = 4, |
98 | }; |
99 | |
100 | enum err_code { |
101 | HIDMA_EVRE_STATUS_COMPLETE = 1, |
102 | HIDMA_EVRE_STATUS_ERROR = 4, |
103 | }; |
104 | |
105 | static int hidma_is_chan_enabled(int state) |
106 | { |
107 | switch (state) { |
108 | case HIDMA_CH_ENABLED: |
109 | case HIDMA_CH_RUNNING: |
110 | return true; |
111 | default: |
112 | return false; |
113 | } |
114 | } |
115 | |
116 | void hidma_ll_free(struct hidma_lldev *lldev, u32 tre_ch) |
117 | { |
118 | struct hidma_tre *tre; |
119 | |
120 | if (tre_ch >= lldev->nr_tres) { |
121 | dev_err(lldev->dev, "invalid TRE number in free:%d" , tre_ch); |
122 | return; |
123 | } |
124 | |
125 | tre = &lldev->trepool[tre_ch]; |
126 | if (atomic_read(v: &tre->allocated) != true) { |
127 | dev_err(lldev->dev, "trying to free an unused TRE:%d" , tre_ch); |
128 | return; |
129 | } |
130 | |
131 | atomic_set(v: &tre->allocated, i: 0); |
132 | } |
133 | |
134 | int hidma_ll_request(struct hidma_lldev *lldev, u32 sig, const char *dev_name, |
135 | void (*callback)(void *data), void *data, u32 *tre_ch) |
136 | { |
137 | unsigned int i; |
138 | struct hidma_tre *tre; |
139 | u32 *tre_local; |
140 | |
141 | if (!tre_ch || !lldev) |
142 | return -EINVAL; |
143 | |
144 | /* need to have at least one empty spot in the queue */ |
145 | for (i = 0; i < lldev->nr_tres - 1; i++) { |
146 | if (atomic_add_unless(v: &lldev->trepool[i].allocated, a: 1, u: 1)) |
147 | break; |
148 | } |
149 | |
150 | if (i == (lldev->nr_tres - 1)) |
151 | return -ENOMEM; |
152 | |
153 | tre = &lldev->trepool[i]; |
154 | tre->dma_sig = sig; |
155 | tre->dev_name = dev_name; |
156 | tre->callback = callback; |
157 | tre->data = data; |
158 | tre->idx = i; |
159 | tre->status = 0; |
160 | tre->queued = 0; |
161 | tre->err_code = 0; |
162 | tre->err_info = 0; |
163 | tre->lldev = lldev; |
164 | tre_local = &tre->tre_local[0]; |
165 | tre_local[HIDMA_TRE_CFG_IDX] = (lldev->chidx & 0xFF) << 8; |
166 | tre_local[HIDMA_TRE_CFG_IDX] |= BIT(16); /* set IEOB */ |
167 | *tre_ch = i; |
168 | if (callback) |
169 | callback(data); |
170 | return 0; |
171 | } |
172 | |
173 | /* |
174 | * Multiple TREs may be queued and waiting in the pending queue. |
175 | */ |
176 | static void hidma_ll_tre_complete(struct tasklet_struct *t) |
177 | { |
178 | struct hidma_lldev *lldev = from_tasklet(lldev, t, task); |
179 | struct hidma_tre *tre; |
180 | |
181 | while (kfifo_out(&lldev->handoff_fifo, &tre, 1)) { |
182 | /* call the user if it has been read by the hardware */ |
183 | if (tre->callback) |
184 | tre->callback(tre->data); |
185 | } |
186 | } |
187 | |
188 | static int hidma_post_completed(struct hidma_lldev *lldev, u8 err_info, |
189 | u8 err_code) |
190 | { |
191 | struct hidma_tre *tre; |
192 | unsigned long flags; |
193 | u32 tre_iterator; |
194 | |
195 | spin_lock_irqsave(&lldev->lock, flags); |
196 | |
197 | tre_iterator = lldev->tre_processed_off; |
198 | tre = lldev->pending_tre_list[tre_iterator / HIDMA_TRE_SIZE]; |
199 | if (!tre) { |
200 | spin_unlock_irqrestore(lock: &lldev->lock, flags); |
201 | dev_warn(lldev->dev, "tre_index [%d] and tre out of sync\n" , |
202 | tre_iterator / HIDMA_TRE_SIZE); |
203 | return -EINVAL; |
204 | } |
205 | lldev->pending_tre_list[tre->tre_index] = NULL; |
206 | |
207 | /* |
208 | * Keep track of pending TREs that SW is expecting to receive |
209 | * from HW. We got one now. Decrement our counter. |
210 | */ |
211 | if (atomic_dec_return(v: &lldev->pending_tre_count) < 0) { |
212 | dev_warn(lldev->dev, "tre count mismatch on completion" ); |
213 | atomic_set(v: &lldev->pending_tre_count, i: 0); |
214 | } |
215 | |
216 | HIDMA_INCREMENT_ITERATOR(tre_iterator, HIDMA_TRE_SIZE, |
217 | lldev->tre_ring_size); |
218 | lldev->tre_processed_off = tre_iterator; |
219 | spin_unlock_irqrestore(lock: &lldev->lock, flags); |
220 | |
221 | tre->err_info = err_info; |
222 | tre->err_code = err_code; |
223 | tre->queued = 0; |
224 | |
225 | kfifo_put(&lldev->handoff_fifo, tre); |
226 | tasklet_schedule(t: &lldev->task); |
227 | |
228 | return 0; |
229 | } |
230 | |
231 | /* |
232 | * Called to handle the interrupt for the channel. |
233 | * Return a positive number if TRE or EVRE were consumed on this run. |
234 | * Return a positive number if there are pending TREs or EVREs. |
235 | * Return 0 if there is nothing to consume or no pending TREs/EVREs found. |
236 | */ |
237 | static int hidma_handle_tre_completion(struct hidma_lldev *lldev) |
238 | { |
239 | u32 evre_ring_size = lldev->evre_ring_size; |
240 | u32 err_info, err_code, evre_write_off; |
241 | u32 evre_iterator; |
242 | u32 num_completed = 0; |
243 | |
244 | evre_write_off = readl_relaxed(lldev->evca + HIDMA_EVCA_WRITE_PTR_REG); |
245 | evre_iterator = lldev->evre_processed_off; |
246 | |
247 | if ((evre_write_off > evre_ring_size) || |
248 | (evre_write_off % HIDMA_EVRE_SIZE)) { |
249 | dev_err(lldev->dev, "HW reports invalid EVRE write offset\n" ); |
250 | return 0; |
251 | } |
252 | |
253 | /* |
254 | * By the time control reaches here the number of EVREs and TREs |
255 | * may not match. Only consume the ones that hardware told us. |
256 | */ |
257 | while ((evre_iterator != evre_write_off)) { |
258 | u32 *current_evre = lldev->evre_ring + evre_iterator; |
259 | u32 cfg; |
260 | |
261 | cfg = current_evre[HIDMA_EVRE_CFG_IDX]; |
262 | err_info = cfg >> HIDMA_EVRE_ERRINFO_BIT_POS; |
263 | err_info &= HIDMA_EVRE_ERRINFO_MASK; |
264 | err_code = |
265 | (cfg >> HIDMA_EVRE_CODE_BIT_POS) & HIDMA_EVRE_CODE_MASK; |
266 | |
267 | if (hidma_post_completed(lldev, err_info, err_code)) |
268 | break; |
269 | |
270 | HIDMA_INCREMENT_ITERATOR(evre_iterator, HIDMA_EVRE_SIZE, |
271 | evre_ring_size); |
272 | |
273 | /* |
274 | * Read the new event descriptor written by the HW. |
275 | * As we are processing the delivered events, other events |
276 | * get queued to the SW for processing. |
277 | */ |
278 | evre_write_off = |
279 | readl_relaxed(lldev->evca + HIDMA_EVCA_WRITE_PTR_REG); |
280 | num_completed++; |
281 | |
282 | /* |
283 | * An error interrupt might have arrived while we are processing |
284 | * the completed interrupt. |
285 | */ |
286 | if (!hidma_ll_isenabled(llhndl: lldev)) |
287 | break; |
288 | } |
289 | |
290 | if (num_completed) { |
291 | u32 evre_read_off = (lldev->evre_processed_off + |
292 | HIDMA_EVRE_SIZE * num_completed); |
293 | evre_read_off = evre_read_off % evre_ring_size; |
294 | writel(val: evre_read_off, addr: lldev->evca + HIDMA_EVCA_DOORBELL_REG); |
295 | |
296 | /* record the last processed tre offset */ |
297 | lldev->evre_processed_off = evre_read_off; |
298 | } |
299 | |
300 | return num_completed; |
301 | } |
302 | |
303 | void hidma_cleanup_pending_tre(struct hidma_lldev *lldev, u8 err_info, |
304 | u8 err_code) |
305 | { |
306 | while (atomic_read(v: &lldev->pending_tre_count)) { |
307 | if (hidma_post_completed(lldev, err_info, err_code)) |
308 | break; |
309 | } |
310 | } |
311 | |
312 | static int hidma_ll_reset(struct hidma_lldev *lldev) |
313 | { |
314 | u32 val; |
315 | int ret; |
316 | |
317 | val = readl(addr: lldev->trca + HIDMA_TRCA_CTRLSTS_REG); |
318 | val &= ~(HIDMA_CH_CONTROL_MASK << 16); |
319 | val |= HIDMA_CH_RESET << 16; |
320 | writel(val, addr: lldev->trca + HIDMA_TRCA_CTRLSTS_REG); |
321 | |
322 | /* |
323 | * Delay 10ms after reset to allow DMA logic to quiesce. |
324 | * Do a polled read up to 1ms and 10ms maximum. |
325 | */ |
326 | ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val, |
327 | HIDMA_CH_STATE(val) == HIDMA_CH_DISABLED, |
328 | 1000, 10000); |
329 | if (ret) { |
330 | dev_err(lldev->dev, "transfer channel did not reset\n" ); |
331 | return ret; |
332 | } |
333 | |
334 | val = readl(addr: lldev->evca + HIDMA_EVCA_CTRLSTS_REG); |
335 | val &= ~(HIDMA_CH_CONTROL_MASK << 16); |
336 | val |= HIDMA_CH_RESET << 16; |
337 | writel(val, addr: lldev->evca + HIDMA_EVCA_CTRLSTS_REG); |
338 | |
339 | /* |
340 | * Delay 10ms after reset to allow DMA logic to quiesce. |
341 | * Do a polled read up to 1ms and 10ms maximum. |
342 | */ |
343 | ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val, |
344 | HIDMA_CH_STATE(val) == HIDMA_CH_DISABLED, |
345 | 1000, 10000); |
346 | if (ret) |
347 | return ret; |
348 | |
349 | lldev->trch_state = HIDMA_CH_DISABLED; |
350 | lldev->evch_state = HIDMA_CH_DISABLED; |
351 | return 0; |
352 | } |
353 | |
354 | /* |
355 | * The interrupt handler for HIDMA will try to consume as many pending |
356 | * EVRE from the event queue as possible. Each EVRE has an associated |
357 | * TRE that holds the user interface parameters. EVRE reports the |
358 | * result of the transaction. Hardware guarantees ordering between EVREs |
359 | * and TREs. We use last processed offset to figure out which TRE is |
360 | * associated with which EVRE. If two TREs are consumed by HW, the EVREs |
361 | * are in order in the event ring. |
362 | * |
363 | * This handler will do a one pass for consuming EVREs. Other EVREs may |
364 | * be delivered while we are working. It will try to consume incoming |
365 | * EVREs one more time and return. |
366 | * |
367 | * For unprocessed EVREs, hardware will trigger another interrupt until |
368 | * all the interrupt bits are cleared. |
369 | * |
370 | * Hardware guarantees that by the time interrupt is observed, all data |
371 | * transactions in flight are delivered to their respective places and |
372 | * are visible to the CPU. |
373 | * |
374 | * On demand paging for IOMMU is only supported for PCIe via PRI |
375 | * (Page Request Interface) not for HIDMA. All other hardware instances |
376 | * including HIDMA work on pinned DMA addresses. |
377 | * |
378 | * HIDMA is not aware of IOMMU presence since it follows the DMA API. All |
379 | * IOMMU latency will be built into the data movement time. By the time |
380 | * interrupt happens, IOMMU lookups + data movement has already taken place. |
381 | * |
382 | * While the first read in a typical PCI endpoint ISR flushes all outstanding |
383 | * requests traditionally to the destination, this concept does not apply |
384 | * here for this HW. |
385 | */ |
386 | static void hidma_ll_int_handler_internal(struct hidma_lldev *lldev, int cause) |
387 | { |
388 | unsigned long irqflags; |
389 | |
390 | if (cause & HIDMA_ERR_INT_MASK) { |
391 | dev_err(lldev->dev, "error 0x%x, disabling...\n" , |
392 | cause); |
393 | |
394 | /* Clear out pending interrupts */ |
395 | writel(val: cause, addr: lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); |
396 | |
397 | /* No further submissions. */ |
398 | hidma_ll_disable(lldev); |
399 | |
400 | /* Driver completes the txn and intimates the client.*/ |
401 | hidma_cleanup_pending_tre(lldev, err_info: 0xFF, |
402 | err_code: HIDMA_EVRE_STATUS_ERROR); |
403 | |
404 | return; |
405 | } |
406 | |
407 | spin_lock_irqsave(&lldev->lock, irqflags); |
408 | writel_relaxed(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); |
409 | spin_unlock_irqrestore(lock: &lldev->lock, flags: irqflags); |
410 | |
411 | /* |
412 | * Fine tuned for this HW... |
413 | * |
414 | * This ISR has been designed for this particular hardware. Relaxed |
415 | * read and write accessors are used for performance reasons due to |
416 | * interrupt delivery guarantees. Do not copy this code blindly and |
417 | * expect that to work. |
418 | * |
419 | * Try to consume as many EVREs as possible. |
420 | */ |
421 | hidma_handle_tre_completion(lldev); |
422 | } |
423 | |
424 | irqreturn_t hidma_ll_inthandler(int chirq, void *arg) |
425 | { |
426 | struct hidma_lldev *lldev = arg; |
427 | u32 status; |
428 | u32 enable; |
429 | u32 cause; |
430 | |
431 | status = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG); |
432 | enable = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_EN_REG); |
433 | cause = status & enable; |
434 | |
435 | while (cause) { |
436 | hidma_ll_int_handler_internal(lldev, cause); |
437 | |
438 | /* |
439 | * Another interrupt might have arrived while we are |
440 | * processing this one. Read the new cause. |
441 | */ |
442 | status = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG); |
443 | enable = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_EN_REG); |
444 | cause = status & enable; |
445 | } |
446 | |
447 | return IRQ_HANDLED; |
448 | } |
449 | |
450 | irqreturn_t hidma_ll_inthandler_msi(int chirq, void *arg, int cause) |
451 | { |
452 | struct hidma_lldev *lldev = arg; |
453 | |
454 | hidma_ll_int_handler_internal(lldev, cause); |
455 | return IRQ_HANDLED; |
456 | } |
457 | |
458 | int hidma_ll_enable(struct hidma_lldev *lldev) |
459 | { |
460 | u32 val; |
461 | int ret; |
462 | |
463 | val = readl(addr: lldev->evca + HIDMA_EVCA_CTRLSTS_REG); |
464 | val &= ~(HIDMA_CH_CONTROL_MASK << 16); |
465 | val |= HIDMA_CH_ENABLE << 16; |
466 | writel(val, addr: lldev->evca + HIDMA_EVCA_CTRLSTS_REG); |
467 | |
468 | ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val, |
469 | hidma_is_chan_enabled(HIDMA_CH_STATE(val)), |
470 | 1000, 10000); |
471 | if (ret) { |
472 | dev_err(lldev->dev, "event channel did not get enabled\n" ); |
473 | return ret; |
474 | } |
475 | |
476 | val = readl(addr: lldev->trca + HIDMA_TRCA_CTRLSTS_REG); |
477 | val &= ~(HIDMA_CH_CONTROL_MASK << 16); |
478 | val |= HIDMA_CH_ENABLE << 16; |
479 | writel(val, addr: lldev->trca + HIDMA_TRCA_CTRLSTS_REG); |
480 | |
481 | ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val, |
482 | hidma_is_chan_enabled(HIDMA_CH_STATE(val)), |
483 | 1000, 10000); |
484 | if (ret) { |
485 | dev_err(lldev->dev, "transfer channel did not get enabled\n" ); |
486 | return ret; |
487 | } |
488 | |
489 | lldev->trch_state = HIDMA_CH_ENABLED; |
490 | lldev->evch_state = HIDMA_CH_ENABLED; |
491 | |
492 | /* enable irqs */ |
493 | writel(ENABLE_IRQS, addr: lldev->evca + HIDMA_EVCA_IRQ_EN_REG); |
494 | |
495 | return 0; |
496 | } |
497 | |
498 | void hidma_ll_start(struct hidma_lldev *lldev) |
499 | { |
500 | unsigned long irqflags; |
501 | |
502 | spin_lock_irqsave(&lldev->lock, irqflags); |
503 | writel(val: lldev->tre_write_offset, addr: lldev->trca + HIDMA_TRCA_DOORBELL_REG); |
504 | spin_unlock_irqrestore(lock: &lldev->lock, flags: irqflags); |
505 | } |
506 | |
507 | bool hidma_ll_isenabled(struct hidma_lldev *lldev) |
508 | { |
509 | u32 val; |
510 | |
511 | val = readl(addr: lldev->trca + HIDMA_TRCA_CTRLSTS_REG); |
512 | lldev->trch_state = HIDMA_CH_STATE(val); |
513 | val = readl(addr: lldev->evca + HIDMA_EVCA_CTRLSTS_REG); |
514 | lldev->evch_state = HIDMA_CH_STATE(val); |
515 | |
516 | /* both channels have to be enabled before calling this function */ |
517 | if (hidma_is_chan_enabled(state: lldev->trch_state) && |
518 | hidma_is_chan_enabled(state: lldev->evch_state)) |
519 | return true; |
520 | |
521 | return false; |
522 | } |
523 | |
524 | void hidma_ll_queue_request(struct hidma_lldev *lldev, u32 tre_ch) |
525 | { |
526 | struct hidma_tre *tre; |
527 | unsigned long flags; |
528 | |
529 | tre = &lldev->trepool[tre_ch]; |
530 | |
531 | /* copy the TRE into its location in the TRE ring */ |
532 | spin_lock_irqsave(&lldev->lock, flags); |
533 | tre->tre_index = lldev->tre_write_offset / HIDMA_TRE_SIZE; |
534 | lldev->pending_tre_list[tre->tre_index] = tre; |
535 | memcpy(lldev->tre_ring + lldev->tre_write_offset, |
536 | &tre->tre_local[0], HIDMA_TRE_SIZE); |
537 | tre->err_code = 0; |
538 | tre->err_info = 0; |
539 | tre->queued = 1; |
540 | atomic_inc(v: &lldev->pending_tre_count); |
541 | lldev->tre_write_offset = (lldev->tre_write_offset + HIDMA_TRE_SIZE) |
542 | % lldev->tre_ring_size; |
543 | spin_unlock_irqrestore(lock: &lldev->lock, flags); |
544 | } |
545 | |
546 | /* |
547 | * Note that even though we stop this channel if there is a pending transaction |
548 | * in flight it will complete and follow the callback. This request will |
549 | * prevent further requests to be made. |
550 | */ |
551 | int hidma_ll_disable(struct hidma_lldev *lldev) |
552 | { |
553 | u32 val; |
554 | int ret; |
555 | |
556 | /* The channel needs to be in working state */ |
557 | if (!hidma_ll_isenabled(lldev)) |
558 | return 0; |
559 | |
560 | val = readl(addr: lldev->trca + HIDMA_TRCA_CTRLSTS_REG); |
561 | val &= ~(HIDMA_CH_CONTROL_MASK << 16); |
562 | val |= HIDMA_CH_SUSPEND << 16; |
563 | writel(val, addr: lldev->trca + HIDMA_TRCA_CTRLSTS_REG); |
564 | |
565 | /* |
566 | * Start the wait right after the suspend is confirmed. |
567 | * Do a polled read up to 1ms and 10ms maximum. |
568 | */ |
569 | ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val, |
570 | HIDMA_CH_STATE(val) == HIDMA_CH_SUSPENDED, |
571 | 1000, 10000); |
572 | if (ret) |
573 | return ret; |
574 | |
575 | val = readl(addr: lldev->evca + HIDMA_EVCA_CTRLSTS_REG); |
576 | val &= ~(HIDMA_CH_CONTROL_MASK << 16); |
577 | val |= HIDMA_CH_SUSPEND << 16; |
578 | writel(val, addr: lldev->evca + HIDMA_EVCA_CTRLSTS_REG); |
579 | |
580 | /* |
581 | * Start the wait right after the suspend is confirmed |
582 | * Delay up to 10ms after reset to allow DMA logic to quiesce. |
583 | */ |
584 | ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val, |
585 | HIDMA_CH_STATE(val) == HIDMA_CH_SUSPENDED, |
586 | 1000, 10000); |
587 | if (ret) |
588 | return ret; |
589 | |
590 | lldev->trch_state = HIDMA_CH_SUSPENDED; |
591 | lldev->evch_state = HIDMA_CH_SUSPENDED; |
592 | |
593 | /* disable interrupts */ |
594 | writel(val: 0, addr: lldev->evca + HIDMA_EVCA_IRQ_EN_REG); |
595 | return 0; |
596 | } |
597 | |
598 | void hidma_ll_set_transfer_params(struct hidma_lldev *lldev, u32 tre_ch, |
599 | dma_addr_t src, dma_addr_t dest, u32 len, |
600 | u32 flags, u32 txntype) |
601 | { |
602 | struct hidma_tre *tre; |
603 | u32 *tre_local; |
604 | |
605 | if (tre_ch >= lldev->nr_tres) { |
606 | dev_err(lldev->dev, "invalid TRE number in transfer params:%d" , |
607 | tre_ch); |
608 | return; |
609 | } |
610 | |
611 | tre = &lldev->trepool[tre_ch]; |
612 | if (atomic_read(v: &tre->allocated) != true) { |
613 | dev_err(lldev->dev, "trying to set params on an unused TRE:%d" , |
614 | tre_ch); |
615 | return; |
616 | } |
617 | |
618 | tre_local = &tre->tre_local[0]; |
619 | tre_local[HIDMA_TRE_CFG_IDX] &= ~GENMASK(7, 0); |
620 | tre_local[HIDMA_TRE_CFG_IDX] |= txntype; |
621 | tre_local[HIDMA_TRE_LEN_IDX] = len; |
622 | tre_local[HIDMA_TRE_SRC_LOW_IDX] = lower_32_bits(src); |
623 | tre_local[HIDMA_TRE_SRC_HI_IDX] = upper_32_bits(src); |
624 | tre_local[HIDMA_TRE_DEST_LOW_IDX] = lower_32_bits(dest); |
625 | tre_local[HIDMA_TRE_DEST_HI_IDX] = upper_32_bits(dest); |
626 | tre->int_flags = flags; |
627 | } |
628 | |
629 | /* |
630 | * Called during initialization and after an error condition |
631 | * to restore hardware state. |
632 | */ |
633 | int hidma_ll_setup(struct hidma_lldev *lldev) |
634 | { |
635 | int rc; |
636 | u64 addr; |
637 | u32 val; |
638 | u32 nr_tres = lldev->nr_tres; |
639 | |
640 | atomic_set(v: &lldev->pending_tre_count, i: 0); |
641 | lldev->tre_processed_off = 0; |
642 | lldev->evre_processed_off = 0; |
643 | lldev->tre_write_offset = 0; |
644 | |
645 | /* disable interrupts */ |
646 | writel(val: 0, addr: lldev->evca + HIDMA_EVCA_IRQ_EN_REG); |
647 | |
648 | /* clear all pending interrupts */ |
649 | val = readl(addr: lldev->evca + HIDMA_EVCA_IRQ_STAT_REG); |
650 | writel(val, addr: lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); |
651 | |
652 | rc = hidma_ll_reset(lldev); |
653 | if (rc) |
654 | return rc; |
655 | |
656 | /* |
657 | * Clear all pending interrupts again. |
658 | * Otherwise, we observe reset complete interrupts. |
659 | */ |
660 | val = readl(addr: lldev->evca + HIDMA_EVCA_IRQ_STAT_REG); |
661 | writel(val, addr: lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); |
662 | |
663 | /* disable interrupts again after reset */ |
664 | writel(val: 0, addr: lldev->evca + HIDMA_EVCA_IRQ_EN_REG); |
665 | |
666 | addr = lldev->tre_dma; |
667 | writel(lower_32_bits(addr), addr: lldev->trca + HIDMA_TRCA_RING_LOW_REG); |
668 | writel(upper_32_bits(addr), addr: lldev->trca + HIDMA_TRCA_RING_HIGH_REG); |
669 | writel(val: lldev->tre_ring_size, addr: lldev->trca + HIDMA_TRCA_RING_LEN_REG); |
670 | |
671 | addr = lldev->evre_dma; |
672 | writel(lower_32_bits(addr), addr: lldev->evca + HIDMA_EVCA_RING_LOW_REG); |
673 | writel(upper_32_bits(addr), addr: lldev->evca + HIDMA_EVCA_RING_HIGH_REG); |
674 | writel(HIDMA_EVRE_SIZE * nr_tres, |
675 | addr: lldev->evca + HIDMA_EVCA_RING_LEN_REG); |
676 | |
677 | /* configure interrupts */ |
678 | hidma_ll_setup_irq(lldev, msi: lldev->msi_support); |
679 | |
680 | rc = hidma_ll_enable(lldev); |
681 | if (rc) |
682 | return rc; |
683 | |
684 | return rc; |
685 | } |
686 | |
687 | void hidma_ll_setup_irq(struct hidma_lldev *lldev, bool msi) |
688 | { |
689 | u32 val; |
690 | |
691 | lldev->msi_support = msi; |
692 | |
693 | /* disable interrupts again after reset */ |
694 | writel(val: 0, addr: lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); |
695 | writel(val: 0, addr: lldev->evca + HIDMA_EVCA_IRQ_EN_REG); |
696 | |
697 | /* support IRQ by default */ |
698 | val = readl(addr: lldev->evca + HIDMA_EVCA_INTCTRL_REG); |
699 | val &= ~0xF; |
700 | if (!lldev->msi_support) |
701 | val = val | 0x1; |
702 | writel(val, addr: lldev->evca + HIDMA_EVCA_INTCTRL_REG); |
703 | |
704 | /* clear all pending interrupts and enable them */ |
705 | writel(ENABLE_IRQS, addr: lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); |
706 | writel(ENABLE_IRQS, addr: lldev->evca + HIDMA_EVCA_IRQ_EN_REG); |
707 | } |
708 | |
709 | struct hidma_lldev *hidma_ll_init(struct device *dev, u32 nr_tres, |
710 | void __iomem *trca, void __iomem *evca, |
711 | u8 chidx) |
712 | { |
713 | u32 required_bytes; |
714 | struct hidma_lldev *lldev; |
715 | int rc; |
716 | size_t sz; |
717 | |
718 | if (!trca || !evca || !dev || !nr_tres) |
719 | return NULL; |
720 | |
721 | /* need at least four TREs */ |
722 | if (nr_tres < 4) |
723 | return NULL; |
724 | |
725 | /* need an extra space */ |
726 | nr_tres += 1; |
727 | |
728 | lldev = devm_kzalloc(dev, size: sizeof(struct hidma_lldev), GFP_KERNEL); |
729 | if (!lldev) |
730 | return NULL; |
731 | |
732 | lldev->evca = evca; |
733 | lldev->trca = trca; |
734 | lldev->dev = dev; |
735 | sz = sizeof(struct hidma_tre); |
736 | lldev->trepool = devm_kcalloc(dev: lldev->dev, n: nr_tres, size: sz, GFP_KERNEL); |
737 | if (!lldev->trepool) |
738 | return NULL; |
739 | |
740 | required_bytes = sizeof(lldev->pending_tre_list[0]); |
741 | lldev->pending_tre_list = devm_kcalloc(dev, n: nr_tres, size: required_bytes, |
742 | GFP_KERNEL); |
743 | if (!lldev->pending_tre_list) |
744 | return NULL; |
745 | |
746 | sz = (HIDMA_TRE_SIZE + 1) * nr_tres; |
747 | lldev->tre_ring = dmam_alloc_coherent(dev, size: sz, dma_handle: &lldev->tre_dma, |
748 | GFP_KERNEL); |
749 | if (!lldev->tre_ring) |
750 | return NULL; |
751 | |
752 | lldev->tre_ring_size = HIDMA_TRE_SIZE * nr_tres; |
753 | lldev->nr_tres = nr_tres; |
754 | |
755 | /* the TRE ring has to be TRE_SIZE aligned */ |
756 | if (!IS_ALIGNED(lldev->tre_dma, HIDMA_TRE_SIZE)) { |
757 | u8 tre_ring_shift; |
758 | |
759 | tre_ring_shift = lldev->tre_dma % HIDMA_TRE_SIZE; |
760 | tre_ring_shift = HIDMA_TRE_SIZE - tre_ring_shift; |
761 | lldev->tre_dma += tre_ring_shift; |
762 | lldev->tre_ring += tre_ring_shift; |
763 | } |
764 | |
765 | sz = (HIDMA_EVRE_SIZE + 1) * nr_tres; |
766 | lldev->evre_ring = dmam_alloc_coherent(dev, size: sz, dma_handle: &lldev->evre_dma, |
767 | GFP_KERNEL); |
768 | if (!lldev->evre_ring) |
769 | return NULL; |
770 | |
771 | lldev->evre_ring_size = HIDMA_EVRE_SIZE * nr_tres; |
772 | |
773 | /* the EVRE ring has to be EVRE_SIZE aligned */ |
774 | if (!IS_ALIGNED(lldev->evre_dma, HIDMA_EVRE_SIZE)) { |
775 | u8 evre_ring_shift; |
776 | |
777 | evre_ring_shift = lldev->evre_dma % HIDMA_EVRE_SIZE; |
778 | evre_ring_shift = HIDMA_EVRE_SIZE - evre_ring_shift; |
779 | lldev->evre_dma += evre_ring_shift; |
780 | lldev->evre_ring += evre_ring_shift; |
781 | } |
782 | lldev->nr_tres = nr_tres; |
783 | lldev->chidx = chidx; |
784 | |
785 | sz = nr_tres * sizeof(struct hidma_tre *); |
786 | rc = kfifo_alloc(&lldev->handoff_fifo, sz, GFP_KERNEL); |
787 | if (rc) |
788 | return NULL; |
789 | |
790 | rc = hidma_ll_setup(lldev); |
791 | if (rc) |
792 | return NULL; |
793 | |
794 | spin_lock_init(&lldev->lock); |
795 | tasklet_setup(t: &lldev->task, callback: hidma_ll_tre_complete); |
796 | lldev->initialized = 1; |
797 | writel(ENABLE_IRQS, addr: lldev->evca + HIDMA_EVCA_IRQ_EN_REG); |
798 | return lldev; |
799 | } |
800 | |
801 | int hidma_ll_uninit(struct hidma_lldev *lldev) |
802 | { |
803 | u32 required_bytes; |
804 | int rc = 0; |
805 | u32 val; |
806 | |
807 | if (!lldev) |
808 | return -ENODEV; |
809 | |
810 | if (!lldev->initialized) |
811 | return 0; |
812 | |
813 | lldev->initialized = 0; |
814 | |
815 | required_bytes = sizeof(struct hidma_tre) * lldev->nr_tres; |
816 | tasklet_kill(t: &lldev->task); |
817 | memset(lldev->trepool, 0, required_bytes); |
818 | lldev->trepool = NULL; |
819 | atomic_set(v: &lldev->pending_tre_count, i: 0); |
820 | lldev->tre_write_offset = 0; |
821 | |
822 | rc = hidma_ll_reset(lldev); |
823 | |
824 | /* |
825 | * Clear all pending interrupts again. |
826 | * Otherwise, we observe reset complete interrupts. |
827 | */ |
828 | val = readl(addr: lldev->evca + HIDMA_EVCA_IRQ_STAT_REG); |
829 | writel(val, addr: lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); |
830 | writel(val: 0, addr: lldev->evca + HIDMA_EVCA_IRQ_EN_REG); |
831 | return rc; |
832 | } |
833 | |
834 | enum dma_status hidma_ll_status(struct hidma_lldev *lldev, u32 tre_ch) |
835 | { |
836 | enum dma_status ret = DMA_ERROR; |
837 | struct hidma_tre *tre; |
838 | unsigned long flags; |
839 | u8 err_code; |
840 | |
841 | spin_lock_irqsave(&lldev->lock, flags); |
842 | |
843 | tre = &lldev->trepool[tre_ch]; |
844 | err_code = tre->err_code; |
845 | |
846 | if (err_code & HIDMA_EVRE_STATUS_COMPLETE) |
847 | ret = DMA_COMPLETE; |
848 | else if (err_code & HIDMA_EVRE_STATUS_ERROR) |
849 | ret = DMA_ERROR; |
850 | else |
851 | ret = DMA_IN_PROGRESS; |
852 | spin_unlock_irqrestore(lock: &lldev->lock, flags); |
853 | |
854 | return ret; |
855 | } |
856 | |