1// SPDX-License-Identifier: GPL-2.0
2/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
3#include <linux/init.h>
4#include <linux/kernel.h>
5#include <linux/module.h>
6#include <linux/pci.h>
7#include <linux/device.h>
8#include <linux/io-64-nonatomic-lo-hi.h>
9#include <linux/dmaengine.h>
10#include <uapi/linux/idxd.h>
11#include "../dmaengine.h"
12#include "registers.h"
13#include "idxd.h"
14
15static inline struct idxd_wq *to_idxd_wq(struct dma_chan *c)
16{
17 struct idxd_dma_chan *idxd_chan;
18
19 idxd_chan = container_of(c, struct idxd_dma_chan, chan);
20 return idxd_chan->wq;
21}
22
23void idxd_dma_complete_txd(struct idxd_desc *desc,
24 enum idxd_complete_type comp_type,
25 bool free_desc, void *ctx, u32 *status)
26{
27 struct idxd_device *idxd = desc->wq->idxd;
28 struct dma_async_tx_descriptor *tx;
29 struct dmaengine_result res;
30 int complete = 1;
31
32 if (desc->completion->status == DSA_COMP_SUCCESS) {
33 res.result = DMA_TRANS_NOERROR;
34 } else if (desc->completion->status) {
35 if (idxd->request_int_handles && comp_type != IDXD_COMPLETE_ABORT &&
36 desc->completion->status == DSA_COMP_INT_HANDLE_INVAL &&
37 idxd_queue_int_handle_resubmit(desc))
38 return;
39 res.result = DMA_TRANS_WRITE_FAILED;
40 } else if (comp_type == IDXD_COMPLETE_ABORT) {
41 res.result = DMA_TRANS_ABORTED;
42 } else {
43 complete = 0;
44 }
45
46 tx = &desc->txd;
47 if (complete && tx->cookie) {
48 dma_cookie_complete(tx);
49 dma_descriptor_unmap(tx);
50 dmaengine_desc_get_callback_invoke(tx, result: &res);
51 tx->callback = NULL;
52 tx->callback_result = NULL;
53 }
54
55 if (free_desc)
56 idxd_free_desc(wq: desc->wq, desc);
57}
58
59static void op_flag_setup(unsigned long flags, u32 *desc_flags)
60{
61 *desc_flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR;
62 if (flags & DMA_PREP_INTERRUPT)
63 *desc_flags |= IDXD_OP_FLAG_RCI;
64}
65
66static inline void idxd_prep_desc_common(struct idxd_wq *wq,
67 struct dsa_hw_desc *hw, char opcode,
68 u64 addr_f1, u64 addr_f2, u64 len,
69 u64 compl, u32 flags)
70{
71 hw->flags = flags;
72 hw->opcode = opcode;
73 hw->src_addr = addr_f1;
74 hw->dst_addr = addr_f2;
75 hw->xfer_size = len;
76 /*
77 * For dedicated WQ, this field is ignored and HW will use the WQCFG.priv
78 * field instead. This field should be set to 0 for kernel descriptors
79 * since kernel DMA on VT-d supports "user" privilege only.
80 */
81 hw->priv = 0;
82 hw->completion_addr = compl;
83}
84
85static struct dma_async_tx_descriptor *
86idxd_dma_prep_interrupt(struct dma_chan *c, unsigned long flags)
87{
88 struct idxd_wq *wq = to_idxd_wq(c);
89 u32 desc_flags;
90 struct idxd_desc *desc;
91
92 if (wq->state != IDXD_WQ_ENABLED)
93 return NULL;
94
95 op_flag_setup(flags, desc_flags: &desc_flags);
96 desc = idxd_alloc_desc(wq, optype: IDXD_OP_BLOCK);
97 if (IS_ERR(ptr: desc))
98 return NULL;
99
100 idxd_prep_desc_common(wq, hw: desc->hw, opcode: DSA_OPCODE_NOOP,
101 addr_f1: 0, addr_f2: 0, len: 0, compl: desc->compl_dma, flags: desc_flags);
102 desc->txd.flags = flags;
103 return &desc->txd;
104}
105
106static struct dma_async_tx_descriptor *
107idxd_dma_submit_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
108 dma_addr_t dma_src, size_t len, unsigned long flags)
109{
110 struct idxd_wq *wq = to_idxd_wq(c);
111 u32 desc_flags;
112 struct idxd_device *idxd = wq->idxd;
113 struct idxd_desc *desc;
114
115 if (wq->state != IDXD_WQ_ENABLED)
116 return NULL;
117
118 if (len > idxd->max_xfer_bytes)
119 return NULL;
120
121 op_flag_setup(flags, desc_flags: &desc_flags);
122 desc = idxd_alloc_desc(wq, optype: IDXD_OP_BLOCK);
123 if (IS_ERR(ptr: desc))
124 return NULL;
125
126 idxd_prep_desc_common(wq, hw: desc->hw, opcode: DSA_OPCODE_MEMMOVE,
127 addr_f1: dma_src, addr_f2: dma_dest, len, compl: desc->compl_dma,
128 flags: desc_flags);
129
130 desc->txd.flags = flags;
131
132 return &desc->txd;
133}
134
135static int idxd_dma_alloc_chan_resources(struct dma_chan *chan)
136{
137 struct idxd_wq *wq = to_idxd_wq(c: chan);
138 struct device *dev = &wq->idxd->pdev->dev;
139
140 idxd_wq_get(wq);
141 dev_dbg(dev, "%s: client_count: %d\n", __func__,
142 idxd_wq_refcount(wq));
143 return 0;
144}
145
146static void idxd_dma_free_chan_resources(struct dma_chan *chan)
147{
148 struct idxd_wq *wq = to_idxd_wq(c: chan);
149 struct device *dev = &wq->idxd->pdev->dev;
150
151 idxd_wq_put(wq);
152 dev_dbg(dev, "%s: client_count: %d\n", __func__,
153 idxd_wq_refcount(wq));
154}
155
156static enum dma_status idxd_dma_tx_status(struct dma_chan *dma_chan,
157 dma_cookie_t cookie,
158 struct dma_tx_state *txstate)
159{
160 return DMA_OUT_OF_ORDER;
161}
162
163/*
164 * issue_pending() does not need to do anything since tx_submit() does the job
165 * already.
166 */
167static void idxd_dma_issue_pending(struct dma_chan *dma_chan)
168{
169}
170
171static dma_cookie_t idxd_dma_tx_submit(struct dma_async_tx_descriptor *tx)
172{
173 struct dma_chan *c = tx->chan;
174 struct idxd_wq *wq = to_idxd_wq(c);
175 dma_cookie_t cookie;
176 int rc;
177 struct idxd_desc *desc = container_of(tx, struct idxd_desc, txd);
178
179 cookie = dma_cookie_assign(tx);
180
181 rc = idxd_submit_desc(wq, desc);
182 if (rc < 0) {
183 idxd_free_desc(wq, desc);
184 return rc;
185 }
186
187 return cookie;
188}
189
190static void idxd_dma_release(struct dma_device *device)
191{
192 struct idxd_dma_dev *idxd_dma = container_of(device, struct idxd_dma_dev, dma);
193
194 kfree(objp: idxd_dma);
195}
196
197int idxd_register_dma_device(struct idxd_device *idxd)
198{
199 struct idxd_dma_dev *idxd_dma;
200 struct dma_device *dma;
201 struct device *dev = &idxd->pdev->dev;
202 int rc;
203
204 idxd_dma = kzalloc_node(size: sizeof(*idxd_dma), GFP_KERNEL, node: dev_to_node(dev));
205 if (!idxd_dma)
206 return -ENOMEM;
207
208 dma = &idxd_dma->dma;
209 INIT_LIST_HEAD(list: &dma->channels);
210 dma->dev = dev;
211
212 dma_cap_set(DMA_INTERRUPT, dma->cap_mask);
213 dma_cap_set(DMA_PRIVATE, dma->cap_mask);
214 dma_cap_set(DMA_COMPLETION_NO_ORDER, dma->cap_mask);
215 dma->device_release = idxd_dma_release;
216
217 dma->device_prep_dma_interrupt = idxd_dma_prep_interrupt;
218 if (idxd->hw.opcap.bits[0] & IDXD_OPCAP_MEMMOVE) {
219 dma_cap_set(DMA_MEMCPY, dma->cap_mask);
220 dma->device_prep_dma_memcpy = idxd_dma_submit_memcpy;
221 }
222
223 dma->device_tx_status = idxd_dma_tx_status;
224 dma->device_issue_pending = idxd_dma_issue_pending;
225 dma->device_alloc_chan_resources = idxd_dma_alloc_chan_resources;
226 dma->device_free_chan_resources = idxd_dma_free_chan_resources;
227
228 rc = dma_async_device_register(device: dma);
229 if (rc < 0) {
230 kfree(objp: idxd_dma);
231 return rc;
232 }
233
234 idxd_dma->idxd = idxd;
235 /*
236 * This pointer is protected by the refs taken by the dma_chan. It will remain valid
237 * as long as there are outstanding channels.
238 */
239 idxd->idxd_dma = idxd_dma;
240 return 0;
241}
242
243void idxd_unregister_dma_device(struct idxd_device *idxd)
244{
245 dma_async_device_unregister(device: &idxd->idxd_dma->dma);
246}
247
248static int idxd_register_dma_channel(struct idxd_wq *wq)
249{
250 struct idxd_device *idxd = wq->idxd;
251 struct dma_device *dma = &idxd->idxd_dma->dma;
252 struct device *dev = &idxd->pdev->dev;
253 struct idxd_dma_chan *idxd_chan;
254 struct dma_chan *chan;
255 int rc, i;
256
257 idxd_chan = kzalloc_node(size: sizeof(*idxd_chan), GFP_KERNEL, node: dev_to_node(dev));
258 if (!idxd_chan)
259 return -ENOMEM;
260
261 chan = &idxd_chan->chan;
262 chan->device = dma;
263 list_add_tail(new: &chan->device_node, head: &dma->channels);
264
265 for (i = 0; i < wq->num_descs; i++) {
266 struct idxd_desc *desc = wq->descs[i];
267
268 dma_async_tx_descriptor_init(tx: &desc->txd, chan);
269 desc->txd.tx_submit = idxd_dma_tx_submit;
270 }
271
272 rc = dma_async_device_channel_register(device: dma, chan);
273 if (rc < 0) {
274 kfree(objp: idxd_chan);
275 return rc;
276 }
277
278 wq->idxd_chan = idxd_chan;
279 idxd_chan->wq = wq;
280 get_device(wq_confdev(wq));
281
282 return 0;
283}
284
285static void idxd_unregister_dma_channel(struct idxd_wq *wq)
286{
287 struct idxd_dma_chan *idxd_chan = wq->idxd_chan;
288 struct dma_chan *chan = &idxd_chan->chan;
289 struct idxd_dma_dev *idxd_dma = wq->idxd->idxd_dma;
290
291 dma_async_device_channel_unregister(device: &idxd_dma->dma, chan);
292 list_del(entry: &chan->device_node);
293 kfree(objp: wq->idxd_chan);
294 wq->idxd_chan = NULL;
295 put_device(wq_confdev(wq));
296}
297
298static int idxd_dmaengine_drv_probe(struct idxd_dev *idxd_dev)
299{
300 struct device *dev = &idxd_dev->conf_dev;
301 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
302 struct idxd_device *idxd = wq->idxd;
303 int rc;
304
305 if (idxd->state != IDXD_DEV_ENABLED)
306 return -ENXIO;
307
308 mutex_lock(&wq->wq_lock);
309 if (!idxd_wq_driver_name_match(wq, dev)) {
310 idxd->cmd_status = IDXD_SCMD_WQ_NO_DRV_NAME;
311 rc = -ENODEV;
312 goto err;
313 }
314
315 wq->type = IDXD_WQT_KERNEL;
316
317 rc = idxd_drv_enable_wq(wq);
318 if (rc < 0) {
319 dev_dbg(dev, "Enable wq %d failed: %d\n", wq->id, rc);
320 rc = -ENXIO;
321 goto err;
322 }
323
324 rc = idxd_register_dma_channel(wq);
325 if (rc < 0) {
326 idxd->cmd_status = IDXD_SCMD_DMA_CHAN_ERR;
327 dev_dbg(dev, "Failed to register dma channel\n");
328 goto err_dma;
329 }
330
331 idxd->cmd_status = 0;
332 mutex_unlock(lock: &wq->wq_lock);
333 return 0;
334
335err_dma:
336 idxd_drv_disable_wq(wq);
337err:
338 wq->type = IDXD_WQT_NONE;
339 mutex_unlock(lock: &wq->wq_lock);
340 return rc;
341}
342
343static void idxd_dmaengine_drv_remove(struct idxd_dev *idxd_dev)
344{
345 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
346
347 mutex_lock(&wq->wq_lock);
348 __idxd_wq_quiesce(wq);
349 idxd_unregister_dma_channel(wq);
350 idxd_drv_disable_wq(wq);
351 mutex_unlock(lock: &wq->wq_lock);
352}
353
354static enum idxd_dev_type dev_types[] = {
355 IDXD_DEV_WQ,
356 IDXD_DEV_NONE,
357};
358
359struct idxd_device_driver idxd_dmaengine_drv = {
360 .probe = idxd_dmaengine_drv_probe,
361 .remove = idxd_dmaengine_drv_remove,
362 .desc_complete = idxd_dma_complete_txd,
363 .name = "dmaengine",
364 .type = dev_types,
365};
366EXPORT_SYMBOL_GPL(idxd_dmaengine_drv);
367

source code of linux/drivers/dma/idxd/dma.c