1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Xen grant DMA-mapping layer - contains special DMA-mapping routines |
4 | * for providing grant references as DMA addresses to be used by frontends |
5 | * (e.g. virtio) in Xen guests |
6 | * |
7 | * Copyright (c) 2021, Juergen Gross <jgross@suse.com> |
8 | */ |
9 | |
10 | #include <linux/module.h> |
11 | #include <linux/dma-map-ops.h> |
12 | #include <linux/of.h> |
13 | #include <linux/pci.h> |
14 | #include <linux/pfn.h> |
15 | #include <linux/xarray.h> |
16 | #include <linux/virtio_anchor.h> |
17 | #include <linux/virtio.h> |
18 | #include <xen/xen.h> |
19 | #include <xen/xen-ops.h> |
20 | #include <xen/grant_table.h> |
21 | |
22 | struct xen_grant_dma_data { |
23 | /* The ID of backend domain */ |
24 | domid_t backend_domid; |
25 | /* Is device behaving sane? */ |
26 | bool broken; |
27 | }; |
28 | |
29 | static DEFINE_XARRAY_FLAGS(xen_grant_dma_devices, XA_FLAGS_LOCK_IRQ); |
30 | |
31 | #define XEN_GRANT_DMA_ADDR_OFF (1ULL << 63) |
32 | |
33 | static inline dma_addr_t grant_to_dma(grant_ref_t grant) |
34 | { |
35 | return XEN_GRANT_DMA_ADDR_OFF | ((dma_addr_t)grant << XEN_PAGE_SHIFT); |
36 | } |
37 | |
38 | static inline grant_ref_t dma_to_grant(dma_addr_t dma) |
39 | { |
40 | return (grant_ref_t)((dma & ~XEN_GRANT_DMA_ADDR_OFF) >> XEN_PAGE_SHIFT); |
41 | } |
42 | |
43 | static struct xen_grant_dma_data *find_xen_grant_dma_data(struct device *dev) |
44 | { |
45 | struct xen_grant_dma_data *data; |
46 | unsigned long flags; |
47 | |
48 | xa_lock_irqsave(&xen_grant_dma_devices, flags); |
49 | data = xa_load(&xen_grant_dma_devices, index: (unsigned long)dev); |
50 | xa_unlock_irqrestore(&xen_grant_dma_devices, flags); |
51 | |
52 | return data; |
53 | } |
54 | |
55 | static int store_xen_grant_dma_data(struct device *dev, |
56 | struct xen_grant_dma_data *data) |
57 | { |
58 | unsigned long flags; |
59 | int ret; |
60 | |
61 | xa_lock_irqsave(&xen_grant_dma_devices, flags); |
62 | ret = xa_err(entry: __xa_store(&xen_grant_dma_devices, index: (unsigned long)dev, entry: data, |
63 | GFP_ATOMIC)); |
64 | xa_unlock_irqrestore(&xen_grant_dma_devices, flags); |
65 | |
66 | return ret; |
67 | } |
68 | |
69 | /* |
70 | * DMA ops for Xen frontends (e.g. virtio). |
71 | * |
72 | * Used to act as a kind of software IOMMU for Xen guests by using grants as |
73 | * DMA addresses. |
74 | * Such a DMA address is formed by using the grant reference as a frame |
75 | * number and setting the highest address bit (this bit is for the backend |
76 | * to be able to distinguish it from e.g. a mmio address). |
77 | */ |
78 | static void *xen_grant_dma_alloc(struct device *dev, size_t size, |
79 | dma_addr_t *dma_handle, gfp_t gfp, |
80 | unsigned long attrs) |
81 | { |
82 | struct xen_grant_dma_data *data; |
83 | unsigned int i, n_pages = XEN_PFN_UP(size); |
84 | unsigned long pfn; |
85 | grant_ref_t grant; |
86 | void *ret; |
87 | |
88 | data = find_xen_grant_dma_data(dev); |
89 | if (!data) |
90 | return NULL; |
91 | |
92 | if (unlikely(data->broken)) |
93 | return NULL; |
94 | |
95 | ret = alloc_pages_exact(size: n_pages * XEN_PAGE_SIZE, gfp_mask: gfp); |
96 | if (!ret) |
97 | return NULL; |
98 | |
99 | pfn = virt_to_pfn(v: ret); |
100 | |
101 | if (gnttab_alloc_grant_reference_seq(count: n_pages, first: &grant)) { |
102 | free_pages_exact(virt: ret, size: n_pages * XEN_PAGE_SIZE); |
103 | return NULL; |
104 | } |
105 | |
106 | for (i = 0; i < n_pages; i++) { |
107 | gnttab_grant_foreign_access_ref(ref: grant + i, domid: data->backend_domid, |
108 | frame: pfn_to_gfn(pfn: pfn + i), readonly: 0); |
109 | } |
110 | |
111 | *dma_handle = grant_to_dma(grant); |
112 | |
113 | return ret; |
114 | } |
115 | |
116 | static void xen_grant_dma_free(struct device *dev, size_t size, void *vaddr, |
117 | dma_addr_t dma_handle, unsigned long attrs) |
118 | { |
119 | struct xen_grant_dma_data *data; |
120 | unsigned int i, n_pages = XEN_PFN_UP(size); |
121 | grant_ref_t grant; |
122 | |
123 | data = find_xen_grant_dma_data(dev); |
124 | if (!data) |
125 | return; |
126 | |
127 | if (unlikely(data->broken)) |
128 | return; |
129 | |
130 | grant = dma_to_grant(dma: dma_handle); |
131 | |
132 | for (i = 0; i < n_pages; i++) { |
133 | if (unlikely(!gnttab_end_foreign_access_ref(grant + i))) { |
134 | dev_alert(dev, "Grant still in use by backend domain, disabled for further use\n" ); |
135 | data->broken = true; |
136 | return; |
137 | } |
138 | } |
139 | |
140 | gnttab_free_grant_reference_seq(head: grant, count: n_pages); |
141 | |
142 | free_pages_exact(virt: vaddr, size: n_pages * XEN_PAGE_SIZE); |
143 | } |
144 | |
145 | static struct page *xen_grant_dma_alloc_pages(struct device *dev, size_t size, |
146 | dma_addr_t *dma_handle, |
147 | enum dma_data_direction dir, |
148 | gfp_t gfp) |
149 | { |
150 | void *vaddr; |
151 | |
152 | vaddr = xen_grant_dma_alloc(dev, size, dma_handle, gfp, attrs: 0); |
153 | if (!vaddr) |
154 | return NULL; |
155 | |
156 | return virt_to_page(vaddr); |
157 | } |
158 | |
159 | static void xen_grant_dma_free_pages(struct device *dev, size_t size, |
160 | struct page *vaddr, dma_addr_t dma_handle, |
161 | enum dma_data_direction dir) |
162 | { |
163 | xen_grant_dma_free(dev, size, page_to_virt(vaddr), dma_handle, attrs: 0); |
164 | } |
165 | |
166 | static dma_addr_t xen_grant_dma_map_page(struct device *dev, struct page *page, |
167 | unsigned long offset, size_t size, |
168 | enum dma_data_direction dir, |
169 | unsigned long attrs) |
170 | { |
171 | struct xen_grant_dma_data *data; |
172 | unsigned long dma_offset = xen_offset_in_page(offset), |
173 | pfn_offset = XEN_PFN_DOWN(offset); |
174 | unsigned int i, n_pages = XEN_PFN_UP(dma_offset + size); |
175 | grant_ref_t grant; |
176 | dma_addr_t dma_handle; |
177 | |
178 | if (WARN_ON(dir == DMA_NONE)) |
179 | return DMA_MAPPING_ERROR; |
180 | |
181 | data = find_xen_grant_dma_data(dev); |
182 | if (!data) |
183 | return DMA_MAPPING_ERROR; |
184 | |
185 | if (unlikely(data->broken)) |
186 | return DMA_MAPPING_ERROR; |
187 | |
188 | if (gnttab_alloc_grant_reference_seq(count: n_pages, first: &grant)) |
189 | return DMA_MAPPING_ERROR; |
190 | |
191 | for (i = 0; i < n_pages; i++) { |
192 | gnttab_grant_foreign_access_ref(ref: grant + i, domid: data->backend_domid, |
193 | frame: pfn_to_gfn(page_to_xen_pfn(page) + i + pfn_offset), |
194 | readonly: dir == DMA_TO_DEVICE); |
195 | } |
196 | |
197 | dma_handle = grant_to_dma(grant) + dma_offset; |
198 | |
199 | return dma_handle; |
200 | } |
201 | |
202 | static void xen_grant_dma_unmap_page(struct device *dev, dma_addr_t dma_handle, |
203 | size_t size, enum dma_data_direction dir, |
204 | unsigned long attrs) |
205 | { |
206 | struct xen_grant_dma_data *data; |
207 | unsigned long dma_offset = xen_offset_in_page(dma_handle); |
208 | unsigned int i, n_pages = XEN_PFN_UP(dma_offset + size); |
209 | grant_ref_t grant; |
210 | |
211 | if (WARN_ON(dir == DMA_NONE)) |
212 | return; |
213 | |
214 | data = find_xen_grant_dma_data(dev); |
215 | if (!data) |
216 | return; |
217 | |
218 | if (unlikely(data->broken)) |
219 | return; |
220 | |
221 | grant = dma_to_grant(dma: dma_handle); |
222 | |
223 | for (i = 0; i < n_pages; i++) { |
224 | if (unlikely(!gnttab_end_foreign_access_ref(grant + i))) { |
225 | dev_alert(dev, "Grant still in use by backend domain, disabled for further use\n" ); |
226 | data->broken = true; |
227 | return; |
228 | } |
229 | } |
230 | |
231 | gnttab_free_grant_reference_seq(head: grant, count: n_pages); |
232 | } |
233 | |
234 | static void xen_grant_dma_unmap_sg(struct device *dev, struct scatterlist *sg, |
235 | int nents, enum dma_data_direction dir, |
236 | unsigned long attrs) |
237 | { |
238 | struct scatterlist *s; |
239 | unsigned int i; |
240 | |
241 | if (WARN_ON(dir == DMA_NONE)) |
242 | return; |
243 | |
244 | for_each_sg(sg, s, nents, i) |
245 | xen_grant_dma_unmap_page(dev, dma_handle: s->dma_address, sg_dma_len(s), dir, |
246 | attrs); |
247 | } |
248 | |
249 | static int xen_grant_dma_map_sg(struct device *dev, struct scatterlist *sg, |
250 | int nents, enum dma_data_direction dir, |
251 | unsigned long attrs) |
252 | { |
253 | struct scatterlist *s; |
254 | unsigned int i; |
255 | |
256 | if (WARN_ON(dir == DMA_NONE)) |
257 | return -EINVAL; |
258 | |
259 | for_each_sg(sg, s, nents, i) { |
260 | s->dma_address = xen_grant_dma_map_page(dev, page: sg_page(sg: s), offset: s->offset, |
261 | size: s->length, dir, attrs); |
262 | if (s->dma_address == DMA_MAPPING_ERROR) |
263 | goto out; |
264 | |
265 | sg_dma_len(s) = s->length; |
266 | } |
267 | |
268 | return nents; |
269 | |
270 | out: |
271 | xen_grant_dma_unmap_sg(dev, sg, nents: i, dir, attrs: attrs | DMA_ATTR_SKIP_CPU_SYNC); |
272 | sg_dma_len(sg) = 0; |
273 | |
274 | return -EIO; |
275 | } |
276 | |
277 | static int xen_grant_dma_supported(struct device *dev, u64 mask) |
278 | { |
279 | return mask == DMA_BIT_MASK(64); |
280 | } |
281 | |
282 | static const struct dma_map_ops xen_grant_dma_ops = { |
283 | .alloc = xen_grant_dma_alloc, |
284 | .free = xen_grant_dma_free, |
285 | .alloc_pages = xen_grant_dma_alloc_pages, |
286 | .free_pages = xen_grant_dma_free_pages, |
287 | .mmap = dma_common_mmap, |
288 | .get_sgtable = dma_common_get_sgtable, |
289 | .map_page = xen_grant_dma_map_page, |
290 | .unmap_page = xen_grant_dma_unmap_page, |
291 | .map_sg = xen_grant_dma_map_sg, |
292 | .unmap_sg = xen_grant_dma_unmap_sg, |
293 | .dma_supported = xen_grant_dma_supported, |
294 | }; |
295 | |
296 | static struct device_node *xen_dt_get_node(struct device *dev) |
297 | { |
298 | if (dev_is_pci(dev)) { |
299 | struct pci_dev *pdev = to_pci_dev(dev); |
300 | struct pci_bus *bus = pdev->bus; |
301 | |
302 | /* Walk up to the root bus to look for PCI Host controller */ |
303 | while (!pci_is_root_bus(pbus: bus)) |
304 | bus = bus->parent; |
305 | |
306 | if (!bus->bridge->parent) |
307 | return NULL; |
308 | return of_node_get(node: bus->bridge->parent->of_node); |
309 | } |
310 | |
311 | return of_node_get(node: dev->of_node); |
312 | } |
313 | |
314 | static int xen_dt_grant_init_backend_domid(struct device *dev, |
315 | struct device_node *np, |
316 | domid_t *backend_domid) |
317 | { |
318 | struct of_phandle_args iommu_spec = { .args_count = 1 }; |
319 | |
320 | if (dev_is_pci(dev)) { |
321 | struct pci_dev *pdev = to_pci_dev(dev); |
322 | u32 rid = PCI_DEVID(pdev->bus->number, pdev->devfn); |
323 | |
324 | if (of_map_id(np, id: rid, map_name: "iommu-map" , map_mask_name: "iommu-map-mask" , target: &iommu_spec.np, |
325 | id_out: iommu_spec.args)) { |
326 | dev_dbg(dev, "Cannot translate ID\n" ); |
327 | return -ESRCH; |
328 | } |
329 | } else { |
330 | if (of_parse_phandle_with_args(np, list_name: "iommus" , cells_name: "#iommu-cells" , |
331 | index: 0, out_args: &iommu_spec)) { |
332 | dev_dbg(dev, "Cannot parse iommus property\n" ); |
333 | return -ESRCH; |
334 | } |
335 | } |
336 | |
337 | if (!of_device_is_compatible(device: iommu_spec.np, "xen,grant-dma" ) || |
338 | iommu_spec.args_count != 1) { |
339 | dev_dbg(dev, "Incompatible IOMMU node\n" ); |
340 | of_node_put(node: iommu_spec.np); |
341 | return -ESRCH; |
342 | } |
343 | |
344 | of_node_put(node: iommu_spec.np); |
345 | |
346 | /* |
347 | * The endpoint ID here means the ID of the domain where the |
348 | * corresponding backend is running |
349 | */ |
350 | *backend_domid = iommu_spec.args[0]; |
351 | |
352 | return 0; |
353 | } |
354 | |
355 | static int xen_grant_init_backend_domid(struct device *dev, |
356 | domid_t *backend_domid) |
357 | { |
358 | struct device_node *np; |
359 | int ret = -ENODEV; |
360 | |
361 | np = xen_dt_get_node(dev); |
362 | if (np) { |
363 | ret = xen_dt_grant_init_backend_domid(dev, np, backend_domid); |
364 | of_node_put(node: np); |
365 | } else if (IS_ENABLED(CONFIG_XEN_VIRTIO_FORCE_GRANT) || xen_pv_domain()) { |
366 | dev_info(dev, "Using dom0 as backend\n" ); |
367 | *backend_domid = 0; |
368 | ret = 0; |
369 | } |
370 | |
371 | return ret; |
372 | } |
373 | |
374 | static void xen_grant_setup_dma_ops(struct device *dev, domid_t backend_domid) |
375 | { |
376 | struct xen_grant_dma_data *data; |
377 | |
378 | data = find_xen_grant_dma_data(dev); |
379 | if (data) { |
380 | dev_err(dev, "Xen grant DMA data is already created\n" ); |
381 | return; |
382 | } |
383 | |
384 | data = devm_kzalloc(dev, size: sizeof(*data), GFP_KERNEL); |
385 | if (!data) |
386 | goto err; |
387 | |
388 | data->backend_domid = backend_domid; |
389 | |
390 | if (store_xen_grant_dma_data(dev, data)) { |
391 | dev_err(dev, "Cannot store Xen grant DMA data\n" ); |
392 | goto err; |
393 | } |
394 | |
395 | dev->dma_ops = &xen_grant_dma_ops; |
396 | |
397 | return; |
398 | |
399 | err: |
400 | devm_kfree(dev, p: data); |
401 | dev_err(dev, "Cannot set up Xen grant DMA ops, retain platform DMA ops\n" ); |
402 | } |
403 | |
404 | bool xen_virtio_restricted_mem_acc(struct virtio_device *dev) |
405 | { |
406 | domid_t backend_domid; |
407 | |
408 | if (!xen_grant_init_backend_domid(dev: dev->dev.parent, backend_domid: &backend_domid)) { |
409 | xen_grant_setup_dma_ops(dev: dev->dev.parent, backend_domid); |
410 | return true; |
411 | } |
412 | |
413 | return false; |
414 | } |
415 | |
416 | MODULE_DESCRIPTION("Xen grant DMA-mapping layer" ); |
417 | MODULE_AUTHOR("Juergen Gross <jgross@suse.com>" ); |
418 | MODULE_LICENSE("GPL" ); |
419 | |