1// SPDX-License-Identifier: GPL-2.0 OR MIT
2
3/*
4 * Xen frontend/backend page directory based shared buffer
5 * helper module.
6 *
7 * Copyright (C) 2018 EPAM Systems Inc.
8 *
9 * Author: Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>
10 */
11
12#include <linux/module.h>
13#include <linux/errno.h>
14#include <linux/mm.h>
15
16#include <asm/xen/hypervisor.h>
17#include <xen/balloon.h>
18#include <xen/xen.h>
19#include <xen/xenbus.h>
20#include <xen/interface/io/ring.h>
21
22#include <xen/xen-front-pgdir-shbuf.h>
23
24/**
25 * This structure represents the structure of a shared page
26 * that contains grant references to the pages of the shared
27 * buffer. This structure is common to many Xen para-virtualized
28 * protocols at include/xen/interface/io/
29 */
30struct xen_page_directory {
31 grant_ref_t gref_dir_next_page;
32#define XEN_GREF_LIST_END 0
33 grant_ref_t gref[]; /* Variable length */
34};
35
36/**
37 * Shared buffer ops which are differently implemented
38 * depending on the allocation mode, e.g. if the buffer
39 * is allocated by the corresponding backend or frontend.
40 * Some of the operations.
41 */
42struct xen_front_pgdir_shbuf_ops {
43 /*
44 * Calculate number of grefs required to handle this buffer,
45 * e.g. if grefs are required for page directory only or the buffer
46 * pages as well.
47 */
48 void (*calc_num_grefs)(struct xen_front_pgdir_shbuf *buf);
49
50 /* Fill page directory according to para-virtual display protocol. */
51 void (*fill_page_dir)(struct xen_front_pgdir_shbuf *buf);
52
53 /* Claim grant references for the pages of the buffer. */
54 int (*grant_refs_for_buffer)(struct xen_front_pgdir_shbuf *buf,
55 grant_ref_t *priv_gref_head, int gref_idx);
56
57 /* Map grant references of the buffer. */
58 int (*map)(struct xen_front_pgdir_shbuf *buf);
59
60 /* Unmap grant references of the buffer. */
61 int (*unmap)(struct xen_front_pgdir_shbuf *buf);
62};
63
64/**
65 * Get granted reference to the very first page of the
66 * page directory. Usually this is passed to the backend,
67 * so it can find/fill the grant references to the buffer's
68 * pages.
69 *
70 * \param buf shared buffer which page directory is of interest.
71 * \return granted reference to the very first page of the
72 * page directory.
73 */
74grant_ref_t
75xen_front_pgdir_shbuf_get_dir_start(struct xen_front_pgdir_shbuf *buf)
76{
77 if (!buf->grefs)
78 return INVALID_GRANT_REF;
79
80 return buf->grefs[0];
81}
82EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_get_dir_start);
83
84/**
85 * Map granted references of the shared buffer.
86 *
87 * Depending on the shared buffer mode of allocation
88 * (be_alloc flag) this can either do nothing (for buffers
89 * shared by the frontend itself) or map the provided granted
90 * references onto the backing storage (buf->pages).
91 *
92 * \param buf shared buffer which grants to be mapped.
93 * \return zero on success or a negative number on failure.
94 */
95int xen_front_pgdir_shbuf_map(struct xen_front_pgdir_shbuf *buf)
96{
97 if (buf->ops && buf->ops->map)
98 return buf->ops->map(buf);
99
100 /* No need to map own grant references. */
101 return 0;
102}
103EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_map);
104
105/**
106 * Unmap granted references of the shared buffer.
107 *
108 * Depending on the shared buffer mode of allocation
109 * (be_alloc flag) this can either do nothing (for buffers
110 * shared by the frontend itself) or unmap the provided granted
111 * references.
112 *
113 * \param buf shared buffer which grants to be unmapped.
114 * \return zero on success or a negative number on failure.
115 */
116int xen_front_pgdir_shbuf_unmap(struct xen_front_pgdir_shbuf *buf)
117{
118 if (buf->ops && buf->ops->unmap)
119 return buf->ops->unmap(buf);
120
121 /* No need to unmap own grant references. */
122 return 0;
123}
124EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_unmap);
125
126/**
127 * Free all the resources of the shared buffer.
128 *
129 * \param buf shared buffer which resources to be freed.
130 */
131void xen_front_pgdir_shbuf_free(struct xen_front_pgdir_shbuf *buf)
132{
133 if (buf->grefs) {
134 int i;
135
136 for (i = 0; i < buf->num_grefs; i++)
137 if (buf->grefs[i] != INVALID_GRANT_REF)
138 gnttab_end_foreign_access(ref: buf->grefs[i], NULL);
139 }
140 kfree(objp: buf->grefs);
141 kfree(objp: buf->directory);
142}
143EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_free);
144
145/*
146 * Number of grefs a page can hold with respect to the
147 * struct xen_page_directory header.
148 */
149#define XEN_NUM_GREFS_PER_PAGE ((PAGE_SIZE - \
150 offsetof(struct xen_page_directory, \
151 gref)) / sizeof(grant_ref_t))
152
153/**
154 * Get the number of pages the page directory consumes itself.
155 *
156 * \param buf shared buffer.
157 */
158static int get_num_pages_dir(struct xen_front_pgdir_shbuf *buf)
159{
160 return DIV_ROUND_UP(buf->num_pages, XEN_NUM_GREFS_PER_PAGE);
161}
162
163/**
164 * Calculate the number of grant references needed to share the buffer
165 * and its pages when backend allocates the buffer.
166 *
167 * \param buf shared buffer.
168 */
169static void backend_calc_num_grefs(struct xen_front_pgdir_shbuf *buf)
170{
171 /* Only for pages the page directory consumes itself. */
172 buf->num_grefs = get_num_pages_dir(buf);
173}
174
175/**
176 * Calculate the number of grant references needed to share the buffer
177 * and its pages when frontend allocates the buffer.
178 *
179 * \param buf shared buffer.
180 */
181static void guest_calc_num_grefs(struct xen_front_pgdir_shbuf *buf)
182{
183 /*
184 * Number of pages the page directory consumes itself
185 * plus grefs for the buffer pages.
186 */
187 buf->num_grefs = get_num_pages_dir(buf) + buf->num_pages;
188}
189
190#define xen_page_to_vaddr(page) \
191 ((uintptr_t)pfn_to_kaddr(page_to_xen_pfn(page)))
192
193/**
194 * Unmap the buffer previously mapped with grant references
195 * provided by the backend.
196 *
197 * \param buf shared buffer.
198 * \return zero on success or a negative number on failure.
199 */
200static int backend_unmap(struct xen_front_pgdir_shbuf *buf)
201{
202 struct gnttab_unmap_grant_ref *unmap_ops;
203 int i, ret;
204
205 if (!buf->pages || !buf->backend_map_handles || !buf->grefs)
206 return 0;
207
208 unmap_ops = kcalloc(n: buf->num_pages, size: sizeof(*unmap_ops),
209 GFP_KERNEL);
210 if (!unmap_ops)
211 return -ENOMEM;
212
213 for (i = 0; i < buf->num_pages; i++) {
214 phys_addr_t addr;
215
216 addr = xen_page_to_vaddr(buf->pages[i]);
217 gnttab_set_unmap_op(unmap: &unmap_ops[i], addr, GNTMAP_host_map,
218 handle: buf->backend_map_handles[i]);
219 }
220
221 ret = gnttab_unmap_refs(unmap_ops, NULL, pages: buf->pages,
222 count: buf->num_pages);
223
224 for (i = 0; i < buf->num_pages; i++) {
225 if (unlikely(unmap_ops[i].status != GNTST_okay))
226 dev_err(&buf->xb_dev->dev,
227 "Failed to unmap page %d: %d\n",
228 i, unmap_ops[i].status);
229 }
230
231 if (ret)
232 dev_err(&buf->xb_dev->dev,
233 "Failed to unmap grant references, ret %d", ret);
234
235 kfree(objp: unmap_ops);
236 kfree(objp: buf->backend_map_handles);
237 buf->backend_map_handles = NULL;
238 return ret;
239}
240
241/**
242 * Map the buffer with grant references provided by the backend.
243 *
244 * \param buf shared buffer.
245 * \return zero on success or a negative number on failure.
246 */
247static int backend_map(struct xen_front_pgdir_shbuf *buf)
248{
249 struct gnttab_map_grant_ref *map_ops = NULL;
250 unsigned char *ptr;
251 int ret, cur_gref, cur_dir_page, cur_page, grefs_left;
252
253 map_ops = kcalloc(n: buf->num_pages, size: sizeof(*map_ops), GFP_KERNEL);
254 if (!map_ops)
255 return -ENOMEM;
256
257 buf->backend_map_handles = kcalloc(n: buf->num_pages,
258 size: sizeof(*buf->backend_map_handles),
259 GFP_KERNEL);
260 if (!buf->backend_map_handles) {
261 kfree(objp: map_ops);
262 return -ENOMEM;
263 }
264
265 /*
266 * Read page directory to get grefs from the backend: for external
267 * buffer we only allocate buf->grefs for the page directory,
268 * so buf->num_grefs has number of pages in the page directory itself.
269 */
270 ptr = buf->directory;
271 grefs_left = buf->num_pages;
272 cur_page = 0;
273 for (cur_dir_page = 0; cur_dir_page < buf->num_grefs; cur_dir_page++) {
274 struct xen_page_directory *page_dir =
275 (struct xen_page_directory *)ptr;
276 int to_copy = XEN_NUM_GREFS_PER_PAGE;
277
278 if (to_copy > grefs_left)
279 to_copy = grefs_left;
280
281 for (cur_gref = 0; cur_gref < to_copy; cur_gref++) {
282 phys_addr_t addr;
283
284 addr = xen_page_to_vaddr(buf->pages[cur_page]);
285 gnttab_set_map_op(map: &map_ops[cur_page], addr,
286 GNTMAP_host_map,
287 ref: page_dir->gref[cur_gref],
288 domid: buf->xb_dev->otherend_id);
289 cur_page++;
290 }
291
292 grefs_left -= to_copy;
293 ptr += PAGE_SIZE;
294 }
295 ret = gnttab_map_refs(map_ops, NULL, pages: buf->pages, count: buf->num_pages);
296
297 /* Save handles even if error, so we can unmap. */
298 for (cur_page = 0; cur_page < buf->num_pages; cur_page++) {
299 if (likely(map_ops[cur_page].status == GNTST_okay)) {
300 buf->backend_map_handles[cur_page] =
301 map_ops[cur_page].handle;
302 } else {
303 buf->backend_map_handles[cur_page] =
304 INVALID_GRANT_HANDLE;
305 if (!ret)
306 ret = -ENXIO;
307 dev_err(&buf->xb_dev->dev,
308 "Failed to map page %d: %d\n",
309 cur_page, map_ops[cur_page].status);
310 }
311 }
312
313 if (ret) {
314 dev_err(&buf->xb_dev->dev,
315 "Failed to map grant references, ret %d", ret);
316 backend_unmap(buf);
317 }
318
319 kfree(objp: map_ops);
320 return ret;
321}
322
323/**
324 * Fill page directory with grant references to the pages of the
325 * page directory itself.
326 *
327 * The grant references to the buffer pages are provided by the
328 * backend in this case.
329 *
330 * \param buf shared buffer.
331 */
332static void backend_fill_page_dir(struct xen_front_pgdir_shbuf *buf)
333{
334 struct xen_page_directory *page_dir;
335 unsigned char *ptr;
336 int i, num_pages_dir;
337
338 ptr = buf->directory;
339 num_pages_dir = get_num_pages_dir(buf);
340
341 /* Fill only grefs for the page directory itself. */
342 for (i = 0; i < num_pages_dir - 1; i++) {
343 page_dir = (struct xen_page_directory *)ptr;
344
345 page_dir->gref_dir_next_page = buf->grefs[i + 1];
346 ptr += PAGE_SIZE;
347 }
348 /* Last page must say there is no more pages. */
349 page_dir = (struct xen_page_directory *)ptr;
350 page_dir->gref_dir_next_page = XEN_GREF_LIST_END;
351}
352
353/**
354 * Fill page directory with grant references to the pages of the
355 * page directory and the buffer we share with the backend.
356 *
357 * \param buf shared buffer.
358 */
359static void guest_fill_page_dir(struct xen_front_pgdir_shbuf *buf)
360{
361 unsigned char *ptr;
362 int cur_gref, grefs_left, to_copy, i, num_pages_dir;
363
364 ptr = buf->directory;
365 num_pages_dir = get_num_pages_dir(buf);
366
367 /*
368 * While copying, skip grefs at start, they are for pages
369 * granted for the page directory itself.
370 */
371 cur_gref = num_pages_dir;
372 grefs_left = buf->num_pages;
373 for (i = 0; i < num_pages_dir; i++) {
374 struct xen_page_directory *page_dir =
375 (struct xen_page_directory *)ptr;
376
377 if (grefs_left <= XEN_NUM_GREFS_PER_PAGE) {
378 to_copy = grefs_left;
379 page_dir->gref_dir_next_page = XEN_GREF_LIST_END;
380 } else {
381 to_copy = XEN_NUM_GREFS_PER_PAGE;
382 page_dir->gref_dir_next_page = buf->grefs[i + 1];
383 }
384 memcpy(&page_dir->gref, &buf->grefs[cur_gref],
385 to_copy * sizeof(grant_ref_t));
386 ptr += PAGE_SIZE;
387 grefs_left -= to_copy;
388 cur_gref += to_copy;
389 }
390}
391
392/**
393 * Grant references to the frontend's buffer pages.
394 *
395 * These will be shared with the backend, so it can
396 * access the buffer's data.
397 *
398 * \param buf shared buffer.
399 * \return zero on success or a negative number on failure.
400 */
401static int guest_grant_refs_for_buffer(struct xen_front_pgdir_shbuf *buf,
402 grant_ref_t *priv_gref_head,
403 int gref_idx)
404{
405 int i, cur_ref, otherend_id;
406
407 otherend_id = buf->xb_dev->otherend_id;
408 for (i = 0; i < buf->num_pages; i++) {
409 cur_ref = gnttab_claim_grant_reference(pprivate_head: priv_gref_head);
410 if (cur_ref < 0)
411 return cur_ref;
412
413 gnttab_grant_foreign_access_ref(ref: cur_ref, domid: otherend_id,
414 frame: xen_page_to_gfn(page: buf->pages[i]),
415 readonly: 0);
416 buf->grefs[gref_idx++] = cur_ref;
417 }
418 return 0;
419}
420
421/**
422 * Grant all the references needed to share the buffer.
423 *
424 * Grant references to the page directory pages and, if
425 * needed, also to the pages of the shared buffer data.
426 *
427 * \param buf shared buffer.
428 * \return zero on success or a negative number on failure.
429 */
430static int grant_references(struct xen_front_pgdir_shbuf *buf)
431{
432 grant_ref_t priv_gref_head;
433 int ret, i, j, cur_ref;
434 int otherend_id, num_pages_dir;
435
436 ret = gnttab_alloc_grant_references(count: buf->num_grefs, pprivate_head: &priv_gref_head);
437 if (ret < 0) {
438 dev_err(&buf->xb_dev->dev,
439 "Cannot allocate grant references\n");
440 return ret;
441 }
442
443 otherend_id = buf->xb_dev->otherend_id;
444 j = 0;
445 num_pages_dir = get_num_pages_dir(buf);
446 for (i = 0; i < num_pages_dir; i++) {
447 unsigned long frame;
448
449 cur_ref = gnttab_claim_grant_reference(pprivate_head: &priv_gref_head);
450 if (cur_ref < 0)
451 return cur_ref;
452
453 frame = xen_page_to_gfn(virt_to_page(buf->directory +
454 PAGE_SIZE * i));
455 gnttab_grant_foreign_access_ref(ref: cur_ref, domid: otherend_id, frame, readonly: 0);
456 buf->grefs[j++] = cur_ref;
457 }
458
459 if (buf->ops->grant_refs_for_buffer) {
460 ret = buf->ops->grant_refs_for_buffer(buf, &priv_gref_head, j);
461 if (ret)
462 return ret;
463 }
464
465 gnttab_free_grant_references(head: priv_gref_head);
466 return 0;
467}
468
469/**
470 * Allocate all required structures to mange shared buffer.
471 *
472 * \param buf shared buffer.
473 * \return zero on success or a negative number on failure.
474 */
475static int alloc_storage(struct xen_front_pgdir_shbuf *buf)
476{
477 buf->grefs = kcalloc(n: buf->num_grefs, size: sizeof(*buf->grefs), GFP_KERNEL);
478 if (!buf->grefs)
479 return -ENOMEM;
480
481 buf->directory = kcalloc(n: get_num_pages_dir(buf), PAGE_SIZE, GFP_KERNEL);
482 if (!buf->directory)
483 return -ENOMEM;
484
485 return 0;
486}
487
488/*
489 * For backend allocated buffers we don't need grant_refs_for_buffer
490 * as those grant references are allocated at backend side.
491 */
492static const struct xen_front_pgdir_shbuf_ops backend_ops = {
493 .calc_num_grefs = backend_calc_num_grefs,
494 .fill_page_dir = backend_fill_page_dir,
495 .map = backend_map,
496 .unmap = backend_unmap
497};
498
499/*
500 * For locally granted references we do not need to map/unmap
501 * the references.
502 */
503static const struct xen_front_pgdir_shbuf_ops local_ops = {
504 .calc_num_grefs = guest_calc_num_grefs,
505 .fill_page_dir = guest_fill_page_dir,
506 .grant_refs_for_buffer = guest_grant_refs_for_buffer,
507};
508
509/**
510 * Allocate a new instance of a shared buffer.
511 *
512 * \param cfg configuration to be used while allocating a new shared buffer.
513 * \return zero on success or a negative number on failure.
514 */
515int xen_front_pgdir_shbuf_alloc(struct xen_front_pgdir_shbuf_cfg *cfg)
516{
517 struct xen_front_pgdir_shbuf *buf = cfg->pgdir;
518 int ret;
519
520 if (cfg->be_alloc)
521 buf->ops = &backend_ops;
522 else
523 buf->ops = &local_ops;
524 buf->xb_dev = cfg->xb_dev;
525 buf->num_pages = cfg->num_pages;
526 buf->pages = cfg->pages;
527
528 buf->ops->calc_num_grefs(buf);
529
530 ret = alloc_storage(buf);
531 if (ret)
532 goto fail;
533
534 ret = grant_references(buf);
535 if (ret)
536 goto fail;
537
538 buf->ops->fill_page_dir(buf);
539
540 return 0;
541
542fail:
543 xen_front_pgdir_shbuf_free(buf);
544 return ret;
545}
546EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_alloc);
547
548MODULE_DESCRIPTION("Xen frontend/backend page directory based "
549 "shared buffer handling");
550MODULE_AUTHOR("Oleksandr Andrushchenko");
551MODULE_LICENSE("GPL");
552

source code of linux/drivers/xen/xen-front-pgdir-shbuf.c