1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Xen grant DMA-mapping layer - contains special DMA-mapping routines
4 * for providing grant references as DMA addresses to be used by frontends
5 * (e.g. virtio) in Xen guests
6 *
7 * Copyright (c) 2021, Juergen Gross <jgross@suse.com>
8 */
9
10#include <linux/module.h>
11#include <linux/dma-map-ops.h>
12#include <linux/of.h>
13#include <linux/pci.h>
14#include <linux/pfn.h>
15#include <linux/xarray.h>
16#include <linux/virtio_anchor.h>
17#include <linux/virtio.h>
18#include <xen/xen.h>
19#include <xen/xen-ops.h>
20#include <xen/grant_table.h>
21
22struct xen_grant_dma_data {
23 /* The ID of backend domain */
24 domid_t backend_domid;
25 /* Is device behaving sane? */
26 bool broken;
27};
28
29static DEFINE_XARRAY_FLAGS(xen_grant_dma_devices, XA_FLAGS_LOCK_IRQ);
30
31#define XEN_GRANT_DMA_ADDR_OFF (1ULL << 63)
32
33static inline dma_addr_t grant_to_dma(grant_ref_t grant)
34{
35 return XEN_GRANT_DMA_ADDR_OFF | ((dma_addr_t)grant << XEN_PAGE_SHIFT);
36}
37
38static inline grant_ref_t dma_to_grant(dma_addr_t dma)
39{
40 return (grant_ref_t)((dma & ~XEN_GRANT_DMA_ADDR_OFF) >> XEN_PAGE_SHIFT);
41}
42
43static struct xen_grant_dma_data *find_xen_grant_dma_data(struct device *dev)
44{
45 struct xen_grant_dma_data *data;
46 unsigned long flags;
47
48 xa_lock_irqsave(&xen_grant_dma_devices, flags);
49 data = xa_load(&xen_grant_dma_devices, index: (unsigned long)dev);
50 xa_unlock_irqrestore(&xen_grant_dma_devices, flags);
51
52 return data;
53}
54
55static int store_xen_grant_dma_data(struct device *dev,
56 struct xen_grant_dma_data *data)
57{
58 unsigned long flags;
59 int ret;
60
61 xa_lock_irqsave(&xen_grant_dma_devices, flags);
62 ret = xa_err(entry: __xa_store(&xen_grant_dma_devices, index: (unsigned long)dev, entry: data,
63 GFP_ATOMIC));
64 xa_unlock_irqrestore(&xen_grant_dma_devices, flags);
65
66 return ret;
67}
68
69/*
70 * DMA ops for Xen frontends (e.g. virtio).
71 *
72 * Used to act as a kind of software IOMMU for Xen guests by using grants as
73 * DMA addresses.
74 * Such a DMA address is formed by using the grant reference as a frame
75 * number and setting the highest address bit (this bit is for the backend
76 * to be able to distinguish it from e.g. a mmio address).
77 */
78static void *xen_grant_dma_alloc(struct device *dev, size_t size,
79 dma_addr_t *dma_handle, gfp_t gfp,
80 unsigned long attrs)
81{
82 struct xen_grant_dma_data *data;
83 unsigned int i, n_pages = XEN_PFN_UP(size);
84 unsigned long pfn;
85 grant_ref_t grant;
86 void *ret;
87
88 data = find_xen_grant_dma_data(dev);
89 if (!data)
90 return NULL;
91
92 if (unlikely(data->broken))
93 return NULL;
94
95 ret = alloc_pages_exact(size: n_pages * XEN_PAGE_SIZE, gfp_mask: gfp);
96 if (!ret)
97 return NULL;
98
99 pfn = virt_to_pfn(v: ret);
100
101 if (gnttab_alloc_grant_reference_seq(count: n_pages, first: &grant)) {
102 free_pages_exact(virt: ret, size: n_pages * XEN_PAGE_SIZE);
103 return NULL;
104 }
105
106 for (i = 0; i < n_pages; i++) {
107 gnttab_grant_foreign_access_ref(ref: grant + i, domid: data->backend_domid,
108 frame: pfn_to_gfn(pfn: pfn + i), readonly: 0);
109 }
110
111 *dma_handle = grant_to_dma(grant);
112
113 return ret;
114}
115
116static void xen_grant_dma_free(struct device *dev, size_t size, void *vaddr,
117 dma_addr_t dma_handle, unsigned long attrs)
118{
119 struct xen_grant_dma_data *data;
120 unsigned int i, n_pages = XEN_PFN_UP(size);
121 grant_ref_t grant;
122
123 data = find_xen_grant_dma_data(dev);
124 if (!data)
125 return;
126
127 if (unlikely(data->broken))
128 return;
129
130 grant = dma_to_grant(dma: dma_handle);
131
132 for (i = 0; i < n_pages; i++) {
133 if (unlikely(!gnttab_end_foreign_access_ref(grant + i))) {
134 dev_alert(dev, "Grant still in use by backend domain, disabled for further use\n");
135 data->broken = true;
136 return;
137 }
138 }
139
140 gnttab_free_grant_reference_seq(head: grant, count: n_pages);
141
142 free_pages_exact(virt: vaddr, size: n_pages * XEN_PAGE_SIZE);
143}
144
145static struct page *xen_grant_dma_alloc_pages(struct device *dev, size_t size,
146 dma_addr_t *dma_handle,
147 enum dma_data_direction dir,
148 gfp_t gfp)
149{
150 void *vaddr;
151
152 vaddr = xen_grant_dma_alloc(dev, size, dma_handle, gfp, attrs: 0);
153 if (!vaddr)
154 return NULL;
155
156 return virt_to_page(vaddr);
157}
158
159static void xen_grant_dma_free_pages(struct device *dev, size_t size,
160 struct page *vaddr, dma_addr_t dma_handle,
161 enum dma_data_direction dir)
162{
163 xen_grant_dma_free(dev, size, page_to_virt(vaddr), dma_handle, attrs: 0);
164}
165
166static dma_addr_t xen_grant_dma_map_page(struct device *dev, struct page *page,
167 unsigned long offset, size_t size,
168 enum dma_data_direction dir,
169 unsigned long attrs)
170{
171 struct xen_grant_dma_data *data;
172 unsigned long dma_offset = xen_offset_in_page(offset),
173 pfn_offset = XEN_PFN_DOWN(offset);
174 unsigned int i, n_pages = XEN_PFN_UP(dma_offset + size);
175 grant_ref_t grant;
176 dma_addr_t dma_handle;
177
178 if (WARN_ON(dir == DMA_NONE))
179 return DMA_MAPPING_ERROR;
180
181 data = find_xen_grant_dma_data(dev);
182 if (!data)
183 return DMA_MAPPING_ERROR;
184
185 if (unlikely(data->broken))
186 return DMA_MAPPING_ERROR;
187
188 if (gnttab_alloc_grant_reference_seq(count: n_pages, first: &grant))
189 return DMA_MAPPING_ERROR;
190
191 for (i = 0; i < n_pages; i++) {
192 gnttab_grant_foreign_access_ref(ref: grant + i, domid: data->backend_domid,
193 frame: pfn_to_gfn(page_to_xen_pfn(page) + i + pfn_offset),
194 readonly: dir == DMA_TO_DEVICE);
195 }
196
197 dma_handle = grant_to_dma(grant) + dma_offset;
198
199 return dma_handle;
200}
201
202static void xen_grant_dma_unmap_page(struct device *dev, dma_addr_t dma_handle,
203 size_t size, enum dma_data_direction dir,
204 unsigned long attrs)
205{
206 struct xen_grant_dma_data *data;
207 unsigned long dma_offset = xen_offset_in_page(dma_handle);
208 unsigned int i, n_pages = XEN_PFN_UP(dma_offset + size);
209 grant_ref_t grant;
210
211 if (WARN_ON(dir == DMA_NONE))
212 return;
213
214 data = find_xen_grant_dma_data(dev);
215 if (!data)
216 return;
217
218 if (unlikely(data->broken))
219 return;
220
221 grant = dma_to_grant(dma: dma_handle);
222
223 for (i = 0; i < n_pages; i++) {
224 if (unlikely(!gnttab_end_foreign_access_ref(grant + i))) {
225 dev_alert(dev, "Grant still in use by backend domain, disabled for further use\n");
226 data->broken = true;
227 return;
228 }
229 }
230
231 gnttab_free_grant_reference_seq(head: grant, count: n_pages);
232}
233
234static void xen_grant_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
235 int nents, enum dma_data_direction dir,
236 unsigned long attrs)
237{
238 struct scatterlist *s;
239 unsigned int i;
240
241 if (WARN_ON(dir == DMA_NONE))
242 return;
243
244 for_each_sg(sg, s, nents, i)
245 xen_grant_dma_unmap_page(dev, dma_handle: s->dma_address, sg_dma_len(s), dir,
246 attrs);
247}
248
249static int xen_grant_dma_map_sg(struct device *dev, struct scatterlist *sg,
250 int nents, enum dma_data_direction dir,
251 unsigned long attrs)
252{
253 struct scatterlist *s;
254 unsigned int i;
255
256 if (WARN_ON(dir == DMA_NONE))
257 return -EINVAL;
258
259 for_each_sg(sg, s, nents, i) {
260 s->dma_address = xen_grant_dma_map_page(dev, page: sg_page(sg: s), offset: s->offset,
261 size: s->length, dir, attrs);
262 if (s->dma_address == DMA_MAPPING_ERROR)
263 goto out;
264
265 sg_dma_len(s) = s->length;
266 }
267
268 return nents;
269
270out:
271 xen_grant_dma_unmap_sg(dev, sg, nents: i, dir, attrs: attrs | DMA_ATTR_SKIP_CPU_SYNC);
272 sg_dma_len(sg) = 0;
273
274 return -EIO;
275}
276
277static int xen_grant_dma_supported(struct device *dev, u64 mask)
278{
279 return mask == DMA_BIT_MASK(64);
280}
281
282static const struct dma_map_ops xen_grant_dma_ops = {
283 .alloc = xen_grant_dma_alloc,
284 .free = xen_grant_dma_free,
285 .alloc_pages = xen_grant_dma_alloc_pages,
286 .free_pages = xen_grant_dma_free_pages,
287 .mmap = dma_common_mmap,
288 .get_sgtable = dma_common_get_sgtable,
289 .map_page = xen_grant_dma_map_page,
290 .unmap_page = xen_grant_dma_unmap_page,
291 .map_sg = xen_grant_dma_map_sg,
292 .unmap_sg = xen_grant_dma_unmap_sg,
293 .dma_supported = xen_grant_dma_supported,
294};
295
296static struct device_node *xen_dt_get_node(struct device *dev)
297{
298 if (dev_is_pci(dev)) {
299 struct pci_dev *pdev = to_pci_dev(dev);
300 struct pci_bus *bus = pdev->bus;
301
302 /* Walk up to the root bus to look for PCI Host controller */
303 while (!pci_is_root_bus(pbus: bus))
304 bus = bus->parent;
305
306 if (!bus->bridge->parent)
307 return NULL;
308 return of_node_get(node: bus->bridge->parent->of_node);
309 }
310
311 return of_node_get(node: dev->of_node);
312}
313
314static int xen_dt_grant_init_backend_domid(struct device *dev,
315 struct device_node *np,
316 domid_t *backend_domid)
317{
318 struct of_phandle_args iommu_spec = { .args_count = 1 };
319
320 if (dev_is_pci(dev)) {
321 struct pci_dev *pdev = to_pci_dev(dev);
322 u32 rid = PCI_DEVID(pdev->bus->number, pdev->devfn);
323
324 if (of_map_id(np, id: rid, map_name: "iommu-map", map_mask_name: "iommu-map-mask", target: &iommu_spec.np,
325 id_out: iommu_spec.args)) {
326 dev_dbg(dev, "Cannot translate ID\n");
327 return -ESRCH;
328 }
329 } else {
330 if (of_parse_phandle_with_args(np, list_name: "iommus", cells_name: "#iommu-cells",
331 index: 0, out_args: &iommu_spec)) {
332 dev_dbg(dev, "Cannot parse iommus property\n");
333 return -ESRCH;
334 }
335 }
336
337 if (!of_device_is_compatible(device: iommu_spec.np, "xen,grant-dma") ||
338 iommu_spec.args_count != 1) {
339 dev_dbg(dev, "Incompatible IOMMU node\n");
340 of_node_put(node: iommu_spec.np);
341 return -ESRCH;
342 }
343
344 of_node_put(node: iommu_spec.np);
345
346 /*
347 * The endpoint ID here means the ID of the domain where the
348 * corresponding backend is running
349 */
350 *backend_domid = iommu_spec.args[0];
351
352 return 0;
353}
354
355static int xen_grant_init_backend_domid(struct device *dev,
356 domid_t *backend_domid)
357{
358 struct device_node *np;
359 int ret = -ENODEV;
360
361 np = xen_dt_get_node(dev);
362 if (np) {
363 ret = xen_dt_grant_init_backend_domid(dev, np, backend_domid);
364 of_node_put(node: np);
365 } else if (IS_ENABLED(CONFIG_XEN_VIRTIO_FORCE_GRANT) || xen_pv_domain()) {
366 dev_info(dev, "Using dom0 as backend\n");
367 *backend_domid = 0;
368 ret = 0;
369 }
370
371 return ret;
372}
373
374static void xen_grant_setup_dma_ops(struct device *dev, domid_t backend_domid)
375{
376 struct xen_grant_dma_data *data;
377
378 data = find_xen_grant_dma_data(dev);
379 if (data) {
380 dev_err(dev, "Xen grant DMA data is already created\n");
381 return;
382 }
383
384 data = devm_kzalloc(dev, size: sizeof(*data), GFP_KERNEL);
385 if (!data)
386 goto err;
387
388 data->backend_domid = backend_domid;
389
390 if (store_xen_grant_dma_data(dev, data)) {
391 dev_err(dev, "Cannot store Xen grant DMA data\n");
392 goto err;
393 }
394
395 dev->dma_ops = &xen_grant_dma_ops;
396
397 return;
398
399err:
400 devm_kfree(dev, p: data);
401 dev_err(dev, "Cannot set up Xen grant DMA ops, retain platform DMA ops\n");
402}
403
404bool xen_virtio_restricted_mem_acc(struct virtio_device *dev)
405{
406 domid_t backend_domid;
407
408 if (!xen_grant_init_backend_domid(dev: dev->dev.parent, backend_domid: &backend_domid)) {
409 xen_grant_setup_dma_ops(dev: dev->dev.parent, backend_domid);
410 return true;
411 }
412
413 return false;
414}
415
416MODULE_DESCRIPTION("Xen grant DMA-mapping layer");
417MODULE_AUTHOR("Juergen Gross <jgross@suse.com>");
418MODULE_LICENSE("GPL");
419

source code of linux/drivers/xen/grant-dma-ops.c