1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * AMD Passthrough DMA device driver |
4 | * -- Based on the CCP driver |
5 | * |
6 | * Copyright (C) 2016,2021 Advanced Micro Devices, Inc. |
7 | * |
8 | * Author: Sanjay R Mehta <sanju.mehta@amd.com> |
9 | * Author: Gary R Hook <gary.hook@amd.com> |
10 | */ |
11 | |
12 | #include "ptdma.h" |
13 | #include "../dmaengine.h" |
14 | #include "../virt-dma.h" |
15 | |
16 | static inline struct pt_dma_chan *to_pt_chan(struct dma_chan *dma_chan) |
17 | { |
18 | return container_of(dma_chan, struct pt_dma_chan, vc.chan); |
19 | } |
20 | |
21 | static inline struct pt_dma_desc *to_pt_desc(struct virt_dma_desc *vd) |
22 | { |
23 | return container_of(vd, struct pt_dma_desc, vd); |
24 | } |
25 | |
26 | static void pt_free_chan_resources(struct dma_chan *dma_chan) |
27 | { |
28 | struct pt_dma_chan *chan = to_pt_chan(dma_chan); |
29 | |
30 | vchan_free_chan_resources(vc: &chan->vc); |
31 | } |
32 | |
33 | static void pt_synchronize(struct dma_chan *dma_chan) |
34 | { |
35 | struct pt_dma_chan *chan = to_pt_chan(dma_chan); |
36 | |
37 | vchan_synchronize(vc: &chan->vc); |
38 | } |
39 | |
40 | static void pt_do_cleanup(struct virt_dma_desc *vd) |
41 | { |
42 | struct pt_dma_desc *desc = to_pt_desc(vd); |
43 | struct pt_device *pt = desc->pt; |
44 | |
45 | kmem_cache_free(s: pt->dma_desc_cache, objp: desc); |
46 | } |
47 | |
48 | static int pt_dma_start_desc(struct pt_dma_desc *desc) |
49 | { |
50 | struct pt_passthru_engine *pt_engine; |
51 | struct pt_device *pt; |
52 | struct pt_cmd *pt_cmd; |
53 | struct pt_cmd_queue *cmd_q; |
54 | |
55 | desc->issued_to_hw = 1; |
56 | |
57 | pt_cmd = &desc->pt_cmd; |
58 | pt = pt_cmd->pt; |
59 | cmd_q = &pt->cmd_q; |
60 | pt_engine = &pt_cmd->passthru; |
61 | |
62 | pt->tdata.cmd = pt_cmd; |
63 | |
64 | /* Execute the command */ |
65 | pt_cmd->ret = pt_core_perform_passthru(cmd_q, pt_engine); |
66 | |
67 | return 0; |
68 | } |
69 | |
70 | static struct pt_dma_desc *pt_next_dma_desc(struct pt_dma_chan *chan) |
71 | { |
72 | /* Get the next DMA descriptor on the active list */ |
73 | struct virt_dma_desc *vd = vchan_next_desc(vc: &chan->vc); |
74 | |
75 | return vd ? to_pt_desc(vd) : NULL; |
76 | } |
77 | |
78 | static struct pt_dma_desc *pt_handle_active_desc(struct pt_dma_chan *chan, |
79 | struct pt_dma_desc *desc) |
80 | { |
81 | struct dma_async_tx_descriptor *tx_desc; |
82 | struct virt_dma_desc *vd; |
83 | unsigned long flags; |
84 | |
85 | /* Loop over descriptors until one is found with commands */ |
86 | do { |
87 | if (desc) { |
88 | if (!desc->issued_to_hw) { |
89 | /* No errors, keep going */ |
90 | if (desc->status != DMA_ERROR) |
91 | return desc; |
92 | } |
93 | |
94 | tx_desc = &desc->vd.tx; |
95 | vd = &desc->vd; |
96 | } else { |
97 | tx_desc = NULL; |
98 | } |
99 | |
100 | spin_lock_irqsave(&chan->vc.lock, flags); |
101 | |
102 | if (desc) { |
103 | if (desc->status != DMA_COMPLETE) { |
104 | if (desc->status != DMA_ERROR) |
105 | desc->status = DMA_COMPLETE; |
106 | |
107 | dma_cookie_complete(tx: tx_desc); |
108 | dma_descriptor_unmap(tx: tx_desc); |
109 | list_del(entry: &desc->vd.node); |
110 | } else { |
111 | /* Don't handle it twice */ |
112 | tx_desc = NULL; |
113 | } |
114 | } |
115 | |
116 | desc = pt_next_dma_desc(chan); |
117 | |
118 | spin_unlock_irqrestore(lock: &chan->vc.lock, flags); |
119 | |
120 | if (tx_desc) { |
121 | dmaengine_desc_get_callback_invoke(tx: tx_desc, NULL); |
122 | dma_run_dependencies(tx: tx_desc); |
123 | vchan_vdesc_fini(vd); |
124 | } |
125 | } while (desc); |
126 | |
127 | return NULL; |
128 | } |
129 | |
130 | static void pt_cmd_callback(void *data, int err) |
131 | { |
132 | struct pt_dma_desc *desc = data; |
133 | struct dma_chan *dma_chan; |
134 | struct pt_dma_chan *chan; |
135 | int ret; |
136 | |
137 | if (err == -EINPROGRESS) |
138 | return; |
139 | |
140 | dma_chan = desc->vd.tx.chan; |
141 | chan = to_pt_chan(dma_chan); |
142 | |
143 | if (err) |
144 | desc->status = DMA_ERROR; |
145 | |
146 | while (true) { |
147 | /* Check for DMA descriptor completion */ |
148 | desc = pt_handle_active_desc(chan, desc); |
149 | |
150 | /* Don't submit cmd if no descriptor or DMA is paused */ |
151 | if (!desc) |
152 | break; |
153 | |
154 | ret = pt_dma_start_desc(desc); |
155 | if (!ret) |
156 | break; |
157 | |
158 | desc->status = DMA_ERROR; |
159 | } |
160 | } |
161 | |
162 | static struct pt_dma_desc *pt_alloc_dma_desc(struct pt_dma_chan *chan, |
163 | unsigned long flags) |
164 | { |
165 | struct pt_dma_desc *desc; |
166 | |
167 | desc = kmem_cache_zalloc(k: chan->pt->dma_desc_cache, GFP_NOWAIT); |
168 | if (!desc) |
169 | return NULL; |
170 | |
171 | vchan_tx_prep(vc: &chan->vc, vd: &desc->vd, tx_flags: flags); |
172 | |
173 | desc->pt = chan->pt; |
174 | desc->pt->cmd_q.int_en = !!(flags & DMA_PREP_INTERRUPT); |
175 | desc->issued_to_hw = 0; |
176 | desc->status = DMA_IN_PROGRESS; |
177 | |
178 | return desc; |
179 | } |
180 | |
181 | static struct pt_dma_desc *pt_create_desc(struct dma_chan *dma_chan, |
182 | dma_addr_t dst, |
183 | dma_addr_t src, |
184 | unsigned int len, |
185 | unsigned long flags) |
186 | { |
187 | struct pt_dma_chan *chan = to_pt_chan(dma_chan); |
188 | struct pt_passthru_engine *pt_engine; |
189 | struct pt_dma_desc *desc; |
190 | struct pt_cmd *pt_cmd; |
191 | |
192 | desc = pt_alloc_dma_desc(chan, flags); |
193 | if (!desc) |
194 | return NULL; |
195 | |
196 | pt_cmd = &desc->pt_cmd; |
197 | pt_cmd->pt = chan->pt; |
198 | pt_engine = &pt_cmd->passthru; |
199 | pt_cmd->engine = PT_ENGINE_PASSTHRU; |
200 | pt_engine->src_dma = src; |
201 | pt_engine->dst_dma = dst; |
202 | pt_engine->src_len = len; |
203 | pt_cmd->pt_cmd_callback = pt_cmd_callback; |
204 | pt_cmd->data = desc; |
205 | |
206 | desc->len = len; |
207 | |
208 | return desc; |
209 | } |
210 | |
211 | static struct dma_async_tx_descriptor * |
212 | pt_prep_dma_memcpy(struct dma_chan *dma_chan, dma_addr_t dst, |
213 | dma_addr_t src, size_t len, unsigned long flags) |
214 | { |
215 | struct pt_dma_desc *desc; |
216 | |
217 | desc = pt_create_desc(dma_chan, dst, src, len, flags); |
218 | if (!desc) |
219 | return NULL; |
220 | |
221 | return &desc->vd.tx; |
222 | } |
223 | |
224 | static struct dma_async_tx_descriptor * |
225 | pt_prep_dma_interrupt(struct dma_chan *dma_chan, unsigned long flags) |
226 | { |
227 | struct pt_dma_chan *chan = to_pt_chan(dma_chan); |
228 | struct pt_dma_desc *desc; |
229 | |
230 | desc = pt_alloc_dma_desc(chan, flags); |
231 | if (!desc) |
232 | return NULL; |
233 | |
234 | return &desc->vd.tx; |
235 | } |
236 | |
237 | static void pt_issue_pending(struct dma_chan *dma_chan) |
238 | { |
239 | struct pt_dma_chan *chan = to_pt_chan(dma_chan); |
240 | struct pt_dma_desc *desc; |
241 | unsigned long flags; |
242 | bool engine_is_idle = true; |
243 | |
244 | spin_lock_irqsave(&chan->vc.lock, flags); |
245 | |
246 | desc = pt_next_dma_desc(chan); |
247 | if (desc) |
248 | engine_is_idle = false; |
249 | |
250 | vchan_issue_pending(vc: &chan->vc); |
251 | |
252 | desc = pt_next_dma_desc(chan); |
253 | |
254 | spin_unlock_irqrestore(lock: &chan->vc.lock, flags); |
255 | |
256 | /* If there was nothing active, start processing */ |
257 | if (engine_is_idle && desc) |
258 | pt_cmd_callback(data: desc, err: 0); |
259 | } |
260 | |
261 | static enum dma_status |
262 | pt_tx_status(struct dma_chan *c, dma_cookie_t cookie, |
263 | struct dma_tx_state *txstate) |
264 | { |
265 | struct pt_device *pt = to_pt_chan(dma_chan: c)->pt; |
266 | struct pt_cmd_queue *cmd_q = &pt->cmd_q; |
267 | |
268 | pt_check_status_trans(pt, cmd_q); |
269 | return dma_cookie_status(chan: c, cookie, state: txstate); |
270 | } |
271 | |
272 | static int pt_pause(struct dma_chan *dma_chan) |
273 | { |
274 | struct pt_dma_chan *chan = to_pt_chan(dma_chan); |
275 | unsigned long flags; |
276 | |
277 | spin_lock_irqsave(&chan->vc.lock, flags); |
278 | pt_stop_queue(cmd_q: &chan->pt->cmd_q); |
279 | spin_unlock_irqrestore(lock: &chan->vc.lock, flags); |
280 | |
281 | return 0; |
282 | } |
283 | |
284 | static int pt_resume(struct dma_chan *dma_chan) |
285 | { |
286 | struct pt_dma_chan *chan = to_pt_chan(dma_chan); |
287 | struct pt_dma_desc *desc = NULL; |
288 | unsigned long flags; |
289 | |
290 | spin_lock_irqsave(&chan->vc.lock, flags); |
291 | pt_start_queue(cmd_q: &chan->pt->cmd_q); |
292 | desc = pt_next_dma_desc(chan); |
293 | spin_unlock_irqrestore(lock: &chan->vc.lock, flags); |
294 | |
295 | /* If there was something active, re-start */ |
296 | if (desc) |
297 | pt_cmd_callback(data: desc, err: 0); |
298 | |
299 | return 0; |
300 | } |
301 | |
302 | static int pt_terminate_all(struct dma_chan *dma_chan) |
303 | { |
304 | struct pt_dma_chan *chan = to_pt_chan(dma_chan); |
305 | unsigned long flags; |
306 | struct pt_cmd_queue *cmd_q = &chan->pt->cmd_q; |
307 | LIST_HEAD(head); |
308 | |
309 | iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010); |
310 | spin_lock_irqsave(&chan->vc.lock, flags); |
311 | vchan_get_all_descriptors(vc: &chan->vc, head: &head); |
312 | spin_unlock_irqrestore(lock: &chan->vc.lock, flags); |
313 | |
314 | vchan_dma_desc_free_list(vc: &chan->vc, head: &head); |
315 | vchan_free_chan_resources(vc: &chan->vc); |
316 | |
317 | return 0; |
318 | } |
319 | |
320 | int pt_dmaengine_register(struct pt_device *pt) |
321 | { |
322 | struct pt_dma_chan *chan; |
323 | struct dma_device *dma_dev = &pt->dma_dev; |
324 | char *cmd_cache_name; |
325 | char *desc_cache_name; |
326 | int ret; |
327 | |
328 | pt->pt_dma_chan = devm_kzalloc(dev: pt->dev, size: sizeof(*pt->pt_dma_chan), |
329 | GFP_KERNEL); |
330 | if (!pt->pt_dma_chan) |
331 | return -ENOMEM; |
332 | |
333 | cmd_cache_name = devm_kasprintf(dev: pt->dev, GFP_KERNEL, |
334 | fmt: "%s-dmaengine-cmd-cache" , |
335 | dev_name(dev: pt->dev)); |
336 | if (!cmd_cache_name) |
337 | return -ENOMEM; |
338 | |
339 | desc_cache_name = devm_kasprintf(dev: pt->dev, GFP_KERNEL, |
340 | fmt: "%s-dmaengine-desc-cache" , |
341 | dev_name(dev: pt->dev)); |
342 | if (!desc_cache_name) { |
343 | ret = -ENOMEM; |
344 | goto err_cache; |
345 | } |
346 | |
347 | pt->dma_desc_cache = kmem_cache_create(name: desc_cache_name, |
348 | size: sizeof(struct pt_dma_desc), align: 0, |
349 | SLAB_HWCACHE_ALIGN, NULL); |
350 | if (!pt->dma_desc_cache) { |
351 | ret = -ENOMEM; |
352 | goto err_cache; |
353 | } |
354 | |
355 | dma_dev->dev = pt->dev; |
356 | dma_dev->src_addr_widths = DMA_SLAVE_BUSWIDTH_64_BYTES; |
357 | dma_dev->dst_addr_widths = DMA_SLAVE_BUSWIDTH_64_BYTES; |
358 | dma_dev->directions = DMA_MEM_TO_MEM; |
359 | dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; |
360 | dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); |
361 | dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask); |
362 | |
363 | /* |
364 | * PTDMA is intended to be used with the AMD NTB devices, hence |
365 | * marking it as DMA_PRIVATE. |
366 | */ |
367 | dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask); |
368 | |
369 | INIT_LIST_HEAD(list: &dma_dev->channels); |
370 | |
371 | chan = pt->pt_dma_chan; |
372 | chan->pt = pt; |
373 | |
374 | /* Set base and prep routines */ |
375 | dma_dev->device_free_chan_resources = pt_free_chan_resources; |
376 | dma_dev->device_prep_dma_memcpy = pt_prep_dma_memcpy; |
377 | dma_dev->device_prep_dma_interrupt = pt_prep_dma_interrupt; |
378 | dma_dev->device_issue_pending = pt_issue_pending; |
379 | dma_dev->device_tx_status = pt_tx_status; |
380 | dma_dev->device_pause = pt_pause; |
381 | dma_dev->device_resume = pt_resume; |
382 | dma_dev->device_terminate_all = pt_terminate_all; |
383 | dma_dev->device_synchronize = pt_synchronize; |
384 | |
385 | chan->vc.desc_free = pt_do_cleanup; |
386 | vchan_init(vc: &chan->vc, dmadev: dma_dev); |
387 | |
388 | ret = dma_async_device_register(device: dma_dev); |
389 | if (ret) |
390 | goto err_reg; |
391 | |
392 | return 0; |
393 | |
394 | err_reg: |
395 | kmem_cache_destroy(s: pt->dma_desc_cache); |
396 | |
397 | err_cache: |
398 | kmem_cache_destroy(s: pt->dma_cmd_cache); |
399 | |
400 | return ret; |
401 | } |
402 | |
403 | void pt_dmaengine_unregister(struct pt_device *pt) |
404 | { |
405 | struct dma_device *dma_dev = &pt->dma_dev; |
406 | |
407 | dma_async_device_unregister(device: dma_dev); |
408 | |
409 | kmem_cache_destroy(s: pt->dma_desc_cache); |
410 | kmem_cache_destroy(s: pt->dma_cmd_cache); |
411 | } |
412 | |