1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * AMD Passthrough DMA device driver
4 * -- Based on the CCP driver
5 *
6 * Copyright (C) 2016,2021 Advanced Micro Devices, Inc.
7 *
8 * Author: Sanjay R Mehta <sanju.mehta@amd.com>
9 * Author: Gary R Hook <gary.hook@amd.com>
10 */
11
12#include "ptdma.h"
13#include "../dmaengine.h"
14#include "../virt-dma.h"
15
16static inline struct pt_dma_chan *to_pt_chan(struct dma_chan *dma_chan)
17{
18 return container_of(dma_chan, struct pt_dma_chan, vc.chan);
19}
20
21static inline struct pt_dma_desc *to_pt_desc(struct virt_dma_desc *vd)
22{
23 return container_of(vd, struct pt_dma_desc, vd);
24}
25
26static void pt_free_chan_resources(struct dma_chan *dma_chan)
27{
28 struct pt_dma_chan *chan = to_pt_chan(dma_chan);
29
30 vchan_free_chan_resources(vc: &chan->vc);
31}
32
33static void pt_synchronize(struct dma_chan *dma_chan)
34{
35 struct pt_dma_chan *chan = to_pt_chan(dma_chan);
36
37 vchan_synchronize(vc: &chan->vc);
38}
39
40static void pt_do_cleanup(struct virt_dma_desc *vd)
41{
42 struct pt_dma_desc *desc = to_pt_desc(vd);
43 struct pt_device *pt = desc->pt;
44
45 kmem_cache_free(s: pt->dma_desc_cache, objp: desc);
46}
47
48static int pt_dma_start_desc(struct pt_dma_desc *desc)
49{
50 struct pt_passthru_engine *pt_engine;
51 struct pt_device *pt;
52 struct pt_cmd *pt_cmd;
53 struct pt_cmd_queue *cmd_q;
54
55 desc->issued_to_hw = 1;
56
57 pt_cmd = &desc->pt_cmd;
58 pt = pt_cmd->pt;
59 cmd_q = &pt->cmd_q;
60 pt_engine = &pt_cmd->passthru;
61
62 pt->tdata.cmd = pt_cmd;
63
64 /* Execute the command */
65 pt_cmd->ret = pt_core_perform_passthru(cmd_q, pt_engine);
66
67 return 0;
68}
69
70static struct pt_dma_desc *pt_next_dma_desc(struct pt_dma_chan *chan)
71{
72 /* Get the next DMA descriptor on the active list */
73 struct virt_dma_desc *vd = vchan_next_desc(vc: &chan->vc);
74
75 return vd ? to_pt_desc(vd) : NULL;
76}
77
78static struct pt_dma_desc *pt_handle_active_desc(struct pt_dma_chan *chan,
79 struct pt_dma_desc *desc)
80{
81 struct dma_async_tx_descriptor *tx_desc;
82 struct virt_dma_desc *vd;
83 unsigned long flags;
84
85 /* Loop over descriptors until one is found with commands */
86 do {
87 if (desc) {
88 if (!desc->issued_to_hw) {
89 /* No errors, keep going */
90 if (desc->status != DMA_ERROR)
91 return desc;
92 }
93
94 tx_desc = &desc->vd.tx;
95 vd = &desc->vd;
96 } else {
97 tx_desc = NULL;
98 }
99
100 spin_lock_irqsave(&chan->vc.lock, flags);
101
102 if (desc) {
103 if (desc->status != DMA_COMPLETE) {
104 if (desc->status != DMA_ERROR)
105 desc->status = DMA_COMPLETE;
106
107 dma_cookie_complete(tx: tx_desc);
108 dma_descriptor_unmap(tx: tx_desc);
109 list_del(entry: &desc->vd.node);
110 } else {
111 /* Don't handle it twice */
112 tx_desc = NULL;
113 }
114 }
115
116 desc = pt_next_dma_desc(chan);
117
118 spin_unlock_irqrestore(lock: &chan->vc.lock, flags);
119
120 if (tx_desc) {
121 dmaengine_desc_get_callback_invoke(tx: tx_desc, NULL);
122 dma_run_dependencies(tx: tx_desc);
123 vchan_vdesc_fini(vd);
124 }
125 } while (desc);
126
127 return NULL;
128}
129
130static void pt_cmd_callback(void *data, int err)
131{
132 struct pt_dma_desc *desc = data;
133 struct dma_chan *dma_chan;
134 struct pt_dma_chan *chan;
135 int ret;
136
137 if (err == -EINPROGRESS)
138 return;
139
140 dma_chan = desc->vd.tx.chan;
141 chan = to_pt_chan(dma_chan);
142
143 if (err)
144 desc->status = DMA_ERROR;
145
146 while (true) {
147 /* Check for DMA descriptor completion */
148 desc = pt_handle_active_desc(chan, desc);
149
150 /* Don't submit cmd if no descriptor or DMA is paused */
151 if (!desc)
152 break;
153
154 ret = pt_dma_start_desc(desc);
155 if (!ret)
156 break;
157
158 desc->status = DMA_ERROR;
159 }
160}
161
162static struct pt_dma_desc *pt_alloc_dma_desc(struct pt_dma_chan *chan,
163 unsigned long flags)
164{
165 struct pt_dma_desc *desc;
166
167 desc = kmem_cache_zalloc(k: chan->pt->dma_desc_cache, GFP_NOWAIT);
168 if (!desc)
169 return NULL;
170
171 vchan_tx_prep(vc: &chan->vc, vd: &desc->vd, tx_flags: flags);
172
173 desc->pt = chan->pt;
174 desc->pt->cmd_q.int_en = !!(flags & DMA_PREP_INTERRUPT);
175 desc->issued_to_hw = 0;
176 desc->status = DMA_IN_PROGRESS;
177
178 return desc;
179}
180
181static struct pt_dma_desc *pt_create_desc(struct dma_chan *dma_chan,
182 dma_addr_t dst,
183 dma_addr_t src,
184 unsigned int len,
185 unsigned long flags)
186{
187 struct pt_dma_chan *chan = to_pt_chan(dma_chan);
188 struct pt_passthru_engine *pt_engine;
189 struct pt_dma_desc *desc;
190 struct pt_cmd *pt_cmd;
191
192 desc = pt_alloc_dma_desc(chan, flags);
193 if (!desc)
194 return NULL;
195
196 pt_cmd = &desc->pt_cmd;
197 pt_cmd->pt = chan->pt;
198 pt_engine = &pt_cmd->passthru;
199 pt_cmd->engine = PT_ENGINE_PASSTHRU;
200 pt_engine->src_dma = src;
201 pt_engine->dst_dma = dst;
202 pt_engine->src_len = len;
203 pt_cmd->pt_cmd_callback = pt_cmd_callback;
204 pt_cmd->data = desc;
205
206 desc->len = len;
207
208 return desc;
209}
210
211static struct dma_async_tx_descriptor *
212pt_prep_dma_memcpy(struct dma_chan *dma_chan, dma_addr_t dst,
213 dma_addr_t src, size_t len, unsigned long flags)
214{
215 struct pt_dma_desc *desc;
216
217 desc = pt_create_desc(dma_chan, dst, src, len, flags);
218 if (!desc)
219 return NULL;
220
221 return &desc->vd.tx;
222}
223
224static struct dma_async_tx_descriptor *
225pt_prep_dma_interrupt(struct dma_chan *dma_chan, unsigned long flags)
226{
227 struct pt_dma_chan *chan = to_pt_chan(dma_chan);
228 struct pt_dma_desc *desc;
229
230 desc = pt_alloc_dma_desc(chan, flags);
231 if (!desc)
232 return NULL;
233
234 return &desc->vd.tx;
235}
236
237static void pt_issue_pending(struct dma_chan *dma_chan)
238{
239 struct pt_dma_chan *chan = to_pt_chan(dma_chan);
240 struct pt_dma_desc *desc;
241 unsigned long flags;
242 bool engine_is_idle = true;
243
244 spin_lock_irqsave(&chan->vc.lock, flags);
245
246 desc = pt_next_dma_desc(chan);
247 if (desc)
248 engine_is_idle = false;
249
250 vchan_issue_pending(vc: &chan->vc);
251
252 desc = pt_next_dma_desc(chan);
253
254 spin_unlock_irqrestore(lock: &chan->vc.lock, flags);
255
256 /* If there was nothing active, start processing */
257 if (engine_is_idle && desc)
258 pt_cmd_callback(data: desc, err: 0);
259}
260
261static enum dma_status
262pt_tx_status(struct dma_chan *c, dma_cookie_t cookie,
263 struct dma_tx_state *txstate)
264{
265 struct pt_device *pt = to_pt_chan(dma_chan: c)->pt;
266 struct pt_cmd_queue *cmd_q = &pt->cmd_q;
267
268 pt_check_status_trans(pt, cmd_q);
269 return dma_cookie_status(chan: c, cookie, state: txstate);
270}
271
272static int pt_pause(struct dma_chan *dma_chan)
273{
274 struct pt_dma_chan *chan = to_pt_chan(dma_chan);
275 unsigned long flags;
276
277 spin_lock_irqsave(&chan->vc.lock, flags);
278 pt_stop_queue(cmd_q: &chan->pt->cmd_q);
279 spin_unlock_irqrestore(lock: &chan->vc.lock, flags);
280
281 return 0;
282}
283
284static int pt_resume(struct dma_chan *dma_chan)
285{
286 struct pt_dma_chan *chan = to_pt_chan(dma_chan);
287 struct pt_dma_desc *desc = NULL;
288 unsigned long flags;
289
290 spin_lock_irqsave(&chan->vc.lock, flags);
291 pt_start_queue(cmd_q: &chan->pt->cmd_q);
292 desc = pt_next_dma_desc(chan);
293 spin_unlock_irqrestore(lock: &chan->vc.lock, flags);
294
295 /* If there was something active, re-start */
296 if (desc)
297 pt_cmd_callback(data: desc, err: 0);
298
299 return 0;
300}
301
302static int pt_terminate_all(struct dma_chan *dma_chan)
303{
304 struct pt_dma_chan *chan = to_pt_chan(dma_chan);
305 unsigned long flags;
306 struct pt_cmd_queue *cmd_q = &chan->pt->cmd_q;
307 LIST_HEAD(head);
308
309 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010);
310 spin_lock_irqsave(&chan->vc.lock, flags);
311 vchan_get_all_descriptors(vc: &chan->vc, head: &head);
312 spin_unlock_irqrestore(lock: &chan->vc.lock, flags);
313
314 vchan_dma_desc_free_list(vc: &chan->vc, head: &head);
315 vchan_free_chan_resources(vc: &chan->vc);
316
317 return 0;
318}
319
320int pt_dmaengine_register(struct pt_device *pt)
321{
322 struct pt_dma_chan *chan;
323 struct dma_device *dma_dev = &pt->dma_dev;
324 char *cmd_cache_name;
325 char *desc_cache_name;
326 int ret;
327
328 pt->pt_dma_chan = devm_kzalloc(dev: pt->dev, size: sizeof(*pt->pt_dma_chan),
329 GFP_KERNEL);
330 if (!pt->pt_dma_chan)
331 return -ENOMEM;
332
333 cmd_cache_name = devm_kasprintf(dev: pt->dev, GFP_KERNEL,
334 fmt: "%s-dmaengine-cmd-cache",
335 dev_name(dev: pt->dev));
336 if (!cmd_cache_name)
337 return -ENOMEM;
338
339 desc_cache_name = devm_kasprintf(dev: pt->dev, GFP_KERNEL,
340 fmt: "%s-dmaengine-desc-cache",
341 dev_name(dev: pt->dev));
342 if (!desc_cache_name) {
343 ret = -ENOMEM;
344 goto err_cache;
345 }
346
347 pt->dma_desc_cache = kmem_cache_create(name: desc_cache_name,
348 size: sizeof(struct pt_dma_desc), align: 0,
349 SLAB_HWCACHE_ALIGN, NULL);
350 if (!pt->dma_desc_cache) {
351 ret = -ENOMEM;
352 goto err_cache;
353 }
354
355 dma_dev->dev = pt->dev;
356 dma_dev->src_addr_widths = DMA_SLAVE_BUSWIDTH_64_BYTES;
357 dma_dev->dst_addr_widths = DMA_SLAVE_BUSWIDTH_64_BYTES;
358 dma_dev->directions = DMA_MEM_TO_MEM;
359 dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
360 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
361 dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask);
362
363 /*
364 * PTDMA is intended to be used with the AMD NTB devices, hence
365 * marking it as DMA_PRIVATE.
366 */
367 dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask);
368
369 INIT_LIST_HEAD(list: &dma_dev->channels);
370
371 chan = pt->pt_dma_chan;
372 chan->pt = pt;
373
374 /* Set base and prep routines */
375 dma_dev->device_free_chan_resources = pt_free_chan_resources;
376 dma_dev->device_prep_dma_memcpy = pt_prep_dma_memcpy;
377 dma_dev->device_prep_dma_interrupt = pt_prep_dma_interrupt;
378 dma_dev->device_issue_pending = pt_issue_pending;
379 dma_dev->device_tx_status = pt_tx_status;
380 dma_dev->device_pause = pt_pause;
381 dma_dev->device_resume = pt_resume;
382 dma_dev->device_terminate_all = pt_terminate_all;
383 dma_dev->device_synchronize = pt_synchronize;
384
385 chan->vc.desc_free = pt_do_cleanup;
386 vchan_init(vc: &chan->vc, dmadev: dma_dev);
387
388 ret = dma_async_device_register(device: dma_dev);
389 if (ret)
390 goto err_reg;
391
392 return 0;
393
394err_reg:
395 kmem_cache_destroy(s: pt->dma_desc_cache);
396
397err_cache:
398 kmem_cache_destroy(s: pt->dma_cmd_cache);
399
400 return ret;
401}
402
403void pt_dmaengine_unregister(struct pt_device *pt)
404{
405 struct dma_device *dma_dev = &pt->dma_dev;
406
407 dma_async_device_unregister(device: dma_dev);
408
409 kmem_cache_destroy(s: pt->dma_desc_cache);
410 kmem_cache_destroy(s: pt->dma_cmd_cache);
411}
412

source code of linux/drivers/dma/ptdma/ptdma-dmaengine.c