1 | /****************************************************************************** |
2 | * gntalloc.c |
3 | * |
4 | * Device for creating grant references (in user-space) that may be shared |
5 | * with other domains. |
6 | * |
7 | * This program is distributed in the hope that it will be useful, |
8 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
9 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
10 | * GNU General Public License for more details. |
11 | * |
12 | * You should have received a copy of the GNU General Public License |
13 | * along with this program; if not, write to the Free Software |
14 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA |
15 | */ |
16 | |
17 | /* |
18 | * This driver exists to allow userspace programs in Linux to allocate kernel |
19 | * memory that will later be shared with another domain. Without this device, |
20 | * Linux userspace programs cannot create grant references. |
21 | * |
22 | * How this stuff works: |
23 | * X -> granting a page to Y |
24 | * Y -> mapping the grant from X |
25 | * |
26 | * 1. X uses the gntalloc device to allocate a page of kernel memory, P. |
27 | * 2. X creates an entry in the grant table that says domid(Y) can access P. |
28 | * This is done without a hypercall unless the grant table needs expansion. |
29 | * 3. X gives the grant reference identifier, GREF, to Y. |
30 | * 4. Y maps the page, either directly into kernel memory for use in a backend |
31 | * driver, or via a the gntdev device to map into the address space of an |
32 | * application running in Y. This is the first point at which Xen does any |
33 | * tracking of the page. |
34 | * 5. A program in X mmap()s a segment of the gntalloc device that corresponds |
35 | * to the shared page, and can now communicate with Y over the shared page. |
36 | * |
37 | * |
38 | * NOTE TO USERSPACE LIBRARIES: |
39 | * The grant allocation and mmap()ing are, naturally, two separate operations. |
40 | * You set up the sharing by calling the create ioctl() and then the mmap(). |
41 | * Teardown requires munmap() and either close() or ioctl(). |
42 | * |
43 | * WARNING: Since Xen does not allow a guest to forcibly end the use of a grant |
44 | * reference, this device can be used to consume kernel memory by leaving grant |
45 | * references mapped by another domain when an application exits. Therefore, |
46 | * there is a global limit on the number of pages that can be allocated. When |
47 | * all references to the page are unmapped, it will be freed during the next |
48 | * grant operation. |
49 | */ |
50 | |
51 | #define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt |
52 | |
53 | #include <linux/atomic.h> |
54 | #include <linux/module.h> |
55 | #include <linux/miscdevice.h> |
56 | #include <linux/kernel.h> |
57 | #include <linux/init.h> |
58 | #include <linux/slab.h> |
59 | #include <linux/fs.h> |
60 | #include <linux/device.h> |
61 | #include <linux/mm.h> |
62 | #include <linux/uaccess.h> |
63 | #include <linux/types.h> |
64 | #include <linux/list.h> |
65 | #include <linux/highmem.h> |
66 | |
67 | #include <xen/xen.h> |
68 | #include <xen/page.h> |
69 | #include <xen/grant_table.h> |
70 | #include <xen/gntalloc.h> |
71 | #include <xen/events.h> |
72 | |
73 | static int limit = 1024; |
74 | module_param(limit, int, 0644); |
75 | MODULE_PARM_DESC(limit, "Maximum number of grants that may be allocated by " |
76 | "the gntalloc device" ); |
77 | |
78 | static LIST_HEAD(gref_list); |
79 | static DEFINE_MUTEX(gref_mutex); |
80 | static int gref_size; |
81 | |
82 | struct notify_info { |
83 | uint16_t pgoff:12; /* Bits 0-11: Offset of the byte to clear */ |
84 | uint16_t flags:2; /* Bits 12-13: Unmap notification flags */ |
85 | int event; /* Port (event channel) to notify */ |
86 | }; |
87 | |
88 | /* Metadata on a grant reference. */ |
89 | struct gntalloc_gref { |
90 | struct list_head next_gref; /* list entry gref_list */ |
91 | struct list_head next_file; /* list entry file->list, if open */ |
92 | struct page *page; /* The shared page */ |
93 | uint64_t file_index; /* File offset for mmap() */ |
94 | unsigned int users; /* Use count - when zero, waiting on Xen */ |
95 | grant_ref_t gref_id; /* The grant reference number */ |
96 | struct notify_info notify; /* Unmap notification */ |
97 | }; |
98 | |
99 | struct gntalloc_file_private_data { |
100 | struct list_head list; |
101 | uint64_t index; |
102 | }; |
103 | |
104 | struct gntalloc_vma_private_data { |
105 | struct gntalloc_gref *gref; |
106 | int users; |
107 | int count; |
108 | }; |
109 | |
110 | static void __del_gref(struct gntalloc_gref *gref); |
111 | |
112 | static void do_cleanup(void) |
113 | { |
114 | struct gntalloc_gref *gref, *n; |
115 | list_for_each_entry_safe(gref, n, &gref_list, next_gref) { |
116 | if (!gref->users) |
117 | __del_gref(gref); |
118 | } |
119 | } |
120 | |
121 | static int add_grefs(struct ioctl_gntalloc_alloc_gref *op, |
122 | uint32_t *gref_ids, struct gntalloc_file_private_data *priv) |
123 | { |
124 | int i, rc, readonly; |
125 | LIST_HEAD(queue_gref); |
126 | LIST_HEAD(queue_file); |
127 | struct gntalloc_gref *gref, *next; |
128 | |
129 | readonly = !(op->flags & GNTALLOC_FLAG_WRITABLE); |
130 | for (i = 0; i < op->count; i++) { |
131 | gref = kzalloc(sizeof(*gref), GFP_KERNEL); |
132 | if (!gref) { |
133 | rc = -ENOMEM; |
134 | goto undo; |
135 | } |
136 | list_add_tail(&gref->next_gref, &queue_gref); |
137 | list_add_tail(&gref->next_file, &queue_file); |
138 | gref->users = 1; |
139 | gref->file_index = op->index + i * PAGE_SIZE; |
140 | gref->page = alloc_page(GFP_KERNEL|__GFP_ZERO); |
141 | if (!gref->page) { |
142 | rc = -ENOMEM; |
143 | goto undo; |
144 | } |
145 | |
146 | /* Grant foreign access to the page. */ |
147 | rc = gnttab_grant_foreign_access(op->domid, |
148 | xen_page_to_gfn(gref->page), |
149 | readonly); |
150 | if (rc < 0) |
151 | goto undo; |
152 | gref_ids[i] = gref->gref_id = rc; |
153 | } |
154 | |
155 | /* Add to gref lists. */ |
156 | mutex_lock(&gref_mutex); |
157 | list_splice_tail(&queue_gref, &gref_list); |
158 | list_splice_tail(&queue_file, &priv->list); |
159 | mutex_unlock(&gref_mutex); |
160 | |
161 | return 0; |
162 | |
163 | undo: |
164 | mutex_lock(&gref_mutex); |
165 | gref_size -= (op->count - i); |
166 | |
167 | list_for_each_entry_safe(gref, next, &queue_file, next_file) { |
168 | list_del(&gref->next_file); |
169 | __del_gref(gref); |
170 | } |
171 | |
172 | /* It's possible for the target domain to map the just-allocated grant |
173 | * references by blindly guessing their IDs; if this is done, then |
174 | * __del_gref will leave them in the queue_gref list. They need to be |
175 | * added to the global list so that we can free them when they are no |
176 | * longer referenced. |
177 | */ |
178 | if (unlikely(!list_empty(&queue_gref))) |
179 | list_splice_tail(&queue_gref, &gref_list); |
180 | mutex_unlock(&gref_mutex); |
181 | return rc; |
182 | } |
183 | |
184 | static void __del_gref(struct gntalloc_gref *gref) |
185 | { |
186 | if (gref->notify.flags & UNMAP_NOTIFY_CLEAR_BYTE) { |
187 | uint8_t *tmp = kmap(gref->page); |
188 | tmp[gref->notify.pgoff] = 0; |
189 | kunmap(gref->page); |
190 | } |
191 | if (gref->notify.flags & UNMAP_NOTIFY_SEND_EVENT) { |
192 | notify_remote_via_evtchn(gref->notify.event); |
193 | evtchn_put(gref->notify.event); |
194 | } |
195 | |
196 | gref->notify.flags = 0; |
197 | |
198 | if (gref->gref_id) { |
199 | if (gnttab_query_foreign_access(gref->gref_id)) |
200 | return; |
201 | |
202 | if (!gnttab_end_foreign_access_ref(gref->gref_id, 0)) |
203 | return; |
204 | |
205 | gnttab_free_grant_reference(gref->gref_id); |
206 | } |
207 | |
208 | gref_size--; |
209 | list_del(&gref->next_gref); |
210 | |
211 | if (gref->page) |
212 | __free_page(gref->page); |
213 | |
214 | kfree(gref); |
215 | } |
216 | |
217 | /* finds contiguous grant references in a file, returns the first */ |
218 | static struct gntalloc_gref *find_grefs(struct gntalloc_file_private_data *priv, |
219 | uint64_t index, uint32_t count) |
220 | { |
221 | struct gntalloc_gref *rv = NULL, *gref; |
222 | list_for_each_entry(gref, &priv->list, next_file) { |
223 | if (gref->file_index == index && !rv) |
224 | rv = gref; |
225 | if (rv) { |
226 | if (gref->file_index != index) |
227 | return NULL; |
228 | index += PAGE_SIZE; |
229 | count--; |
230 | if (count == 0) |
231 | return rv; |
232 | } |
233 | } |
234 | return NULL; |
235 | } |
236 | |
237 | /* |
238 | * ------------------------------------- |
239 | * File operations. |
240 | * ------------------------------------- |
241 | */ |
242 | static int gntalloc_open(struct inode *inode, struct file *filp) |
243 | { |
244 | struct gntalloc_file_private_data *priv; |
245 | |
246 | priv = kzalloc(sizeof(*priv), GFP_KERNEL); |
247 | if (!priv) |
248 | goto out_nomem; |
249 | INIT_LIST_HEAD(&priv->list); |
250 | |
251 | filp->private_data = priv; |
252 | |
253 | pr_debug("%s: priv %p\n" , __func__, priv); |
254 | |
255 | return 0; |
256 | |
257 | out_nomem: |
258 | return -ENOMEM; |
259 | } |
260 | |
261 | static int gntalloc_release(struct inode *inode, struct file *filp) |
262 | { |
263 | struct gntalloc_file_private_data *priv = filp->private_data; |
264 | struct gntalloc_gref *gref; |
265 | |
266 | pr_debug("%s: priv %p\n" , __func__, priv); |
267 | |
268 | mutex_lock(&gref_mutex); |
269 | while (!list_empty(&priv->list)) { |
270 | gref = list_entry(priv->list.next, |
271 | struct gntalloc_gref, next_file); |
272 | list_del(&gref->next_file); |
273 | gref->users--; |
274 | if (gref->users == 0) |
275 | __del_gref(gref); |
276 | } |
277 | kfree(priv); |
278 | mutex_unlock(&gref_mutex); |
279 | |
280 | return 0; |
281 | } |
282 | |
283 | static long gntalloc_ioctl_alloc(struct gntalloc_file_private_data *priv, |
284 | struct ioctl_gntalloc_alloc_gref __user *arg) |
285 | { |
286 | int rc = 0; |
287 | struct ioctl_gntalloc_alloc_gref op; |
288 | uint32_t *gref_ids; |
289 | |
290 | pr_debug("%s: priv %p\n" , __func__, priv); |
291 | |
292 | if (copy_from_user(&op, arg, sizeof(op))) { |
293 | rc = -EFAULT; |
294 | goto out; |
295 | } |
296 | |
297 | gref_ids = kcalloc(op.count, sizeof(gref_ids[0]), GFP_KERNEL); |
298 | if (!gref_ids) { |
299 | rc = -ENOMEM; |
300 | goto out; |
301 | } |
302 | |
303 | mutex_lock(&gref_mutex); |
304 | /* Clean up pages that were at zero (local) users but were still mapped |
305 | * by remote domains. Since those pages count towards the limit that we |
306 | * are about to enforce, removing them here is a good idea. |
307 | */ |
308 | do_cleanup(); |
309 | if (gref_size + op.count > limit) { |
310 | mutex_unlock(&gref_mutex); |
311 | rc = -ENOSPC; |
312 | goto out_free; |
313 | } |
314 | gref_size += op.count; |
315 | op.index = priv->index; |
316 | priv->index += op.count * PAGE_SIZE; |
317 | mutex_unlock(&gref_mutex); |
318 | |
319 | rc = add_grefs(&op, gref_ids, priv); |
320 | if (rc < 0) |
321 | goto out_free; |
322 | |
323 | /* Once we finish add_grefs, it is unsafe to touch the new reference, |
324 | * since it is possible for a concurrent ioctl to remove it (by guessing |
325 | * its index). If the userspace application doesn't provide valid memory |
326 | * to write the IDs to, then it will need to close the file in order to |
327 | * release - which it will do by segfaulting when it tries to access the |
328 | * IDs to close them. |
329 | */ |
330 | if (copy_to_user(arg, &op, sizeof(op))) { |
331 | rc = -EFAULT; |
332 | goto out_free; |
333 | } |
334 | if (copy_to_user(arg->gref_ids, gref_ids, |
335 | sizeof(gref_ids[0]) * op.count)) { |
336 | rc = -EFAULT; |
337 | goto out_free; |
338 | } |
339 | |
340 | out_free: |
341 | kfree(gref_ids); |
342 | out: |
343 | return rc; |
344 | } |
345 | |
346 | static long gntalloc_ioctl_dealloc(struct gntalloc_file_private_data *priv, |
347 | void __user *arg) |
348 | { |
349 | int i, rc = 0; |
350 | struct ioctl_gntalloc_dealloc_gref op; |
351 | struct gntalloc_gref *gref, *n; |
352 | |
353 | pr_debug("%s: priv %p\n" , __func__, priv); |
354 | |
355 | if (copy_from_user(&op, arg, sizeof(op))) { |
356 | rc = -EFAULT; |
357 | goto dealloc_grant_out; |
358 | } |
359 | |
360 | mutex_lock(&gref_mutex); |
361 | gref = find_grefs(priv, op.index, op.count); |
362 | if (gref) { |
363 | /* Remove from the file list only, and decrease reference count. |
364 | * The later call to do_cleanup() will remove from gref_list and |
365 | * free the memory if the pages aren't mapped anywhere. |
366 | */ |
367 | for (i = 0; i < op.count; i++) { |
368 | n = list_entry(gref->next_file.next, |
369 | struct gntalloc_gref, next_file); |
370 | list_del(&gref->next_file); |
371 | gref->users--; |
372 | gref = n; |
373 | } |
374 | } else { |
375 | rc = -EINVAL; |
376 | } |
377 | |
378 | do_cleanup(); |
379 | |
380 | mutex_unlock(&gref_mutex); |
381 | dealloc_grant_out: |
382 | return rc; |
383 | } |
384 | |
385 | static long gntalloc_ioctl_unmap_notify(struct gntalloc_file_private_data *priv, |
386 | void __user *arg) |
387 | { |
388 | struct ioctl_gntalloc_unmap_notify op; |
389 | struct gntalloc_gref *gref; |
390 | uint64_t index; |
391 | int pgoff; |
392 | int rc; |
393 | |
394 | if (copy_from_user(&op, arg, sizeof(op))) |
395 | return -EFAULT; |
396 | |
397 | index = op.index & ~(PAGE_SIZE - 1); |
398 | pgoff = op.index & (PAGE_SIZE - 1); |
399 | |
400 | mutex_lock(&gref_mutex); |
401 | |
402 | gref = find_grefs(priv, index, 1); |
403 | if (!gref) { |
404 | rc = -ENOENT; |
405 | goto unlock_out; |
406 | } |
407 | |
408 | if (op.action & ~(UNMAP_NOTIFY_CLEAR_BYTE|UNMAP_NOTIFY_SEND_EVENT)) { |
409 | rc = -EINVAL; |
410 | goto unlock_out; |
411 | } |
412 | |
413 | /* We need to grab a reference to the event channel we are going to use |
414 | * to send the notify before releasing the reference we may already have |
415 | * (if someone has called this ioctl twice). This is required so that |
416 | * it is possible to change the clear_byte part of the notification |
417 | * without disturbing the event channel part, which may now be the last |
418 | * reference to that event channel. |
419 | */ |
420 | if (op.action & UNMAP_NOTIFY_SEND_EVENT) { |
421 | if (evtchn_get(op.event_channel_port)) { |
422 | rc = -EINVAL; |
423 | goto unlock_out; |
424 | } |
425 | } |
426 | |
427 | if (gref->notify.flags & UNMAP_NOTIFY_SEND_EVENT) |
428 | evtchn_put(gref->notify.event); |
429 | |
430 | gref->notify.flags = op.action; |
431 | gref->notify.pgoff = pgoff; |
432 | gref->notify.event = op.event_channel_port; |
433 | rc = 0; |
434 | |
435 | unlock_out: |
436 | mutex_unlock(&gref_mutex); |
437 | return rc; |
438 | } |
439 | |
440 | static long gntalloc_ioctl(struct file *filp, unsigned int cmd, |
441 | unsigned long arg) |
442 | { |
443 | struct gntalloc_file_private_data *priv = filp->private_data; |
444 | |
445 | switch (cmd) { |
446 | case IOCTL_GNTALLOC_ALLOC_GREF: |
447 | return gntalloc_ioctl_alloc(priv, (void __user *)arg); |
448 | |
449 | case IOCTL_GNTALLOC_DEALLOC_GREF: |
450 | return gntalloc_ioctl_dealloc(priv, (void __user *)arg); |
451 | |
452 | case IOCTL_GNTALLOC_SET_UNMAP_NOTIFY: |
453 | return gntalloc_ioctl_unmap_notify(priv, (void __user *)arg); |
454 | |
455 | default: |
456 | return -ENOIOCTLCMD; |
457 | } |
458 | |
459 | return 0; |
460 | } |
461 | |
462 | static void gntalloc_vma_open(struct vm_area_struct *vma) |
463 | { |
464 | struct gntalloc_vma_private_data *priv = vma->vm_private_data; |
465 | |
466 | if (!priv) |
467 | return; |
468 | |
469 | mutex_lock(&gref_mutex); |
470 | priv->users++; |
471 | mutex_unlock(&gref_mutex); |
472 | } |
473 | |
474 | static void gntalloc_vma_close(struct vm_area_struct *vma) |
475 | { |
476 | struct gntalloc_vma_private_data *priv = vma->vm_private_data; |
477 | struct gntalloc_gref *gref, *next; |
478 | int i; |
479 | |
480 | if (!priv) |
481 | return; |
482 | |
483 | mutex_lock(&gref_mutex); |
484 | priv->users--; |
485 | if (priv->users == 0) { |
486 | gref = priv->gref; |
487 | for (i = 0; i < priv->count; i++) { |
488 | gref->users--; |
489 | next = list_entry(gref->next_gref.next, |
490 | struct gntalloc_gref, next_gref); |
491 | if (gref->users == 0) |
492 | __del_gref(gref); |
493 | gref = next; |
494 | } |
495 | kfree(priv); |
496 | } |
497 | mutex_unlock(&gref_mutex); |
498 | } |
499 | |
500 | static const struct vm_operations_struct gntalloc_vmops = { |
501 | .open = gntalloc_vma_open, |
502 | .close = gntalloc_vma_close, |
503 | }; |
504 | |
505 | static int gntalloc_mmap(struct file *filp, struct vm_area_struct *vma) |
506 | { |
507 | struct gntalloc_file_private_data *priv = filp->private_data; |
508 | struct gntalloc_vma_private_data *vm_priv; |
509 | struct gntalloc_gref *gref; |
510 | int count = vma_pages(vma); |
511 | int rv, i; |
512 | |
513 | if (!(vma->vm_flags & VM_SHARED)) { |
514 | pr_err("%s: Mapping must be shared\n" , __func__); |
515 | return -EINVAL; |
516 | } |
517 | |
518 | vm_priv = kmalloc(sizeof(*vm_priv), GFP_KERNEL); |
519 | if (!vm_priv) |
520 | return -ENOMEM; |
521 | |
522 | mutex_lock(&gref_mutex); |
523 | |
524 | pr_debug("%s: priv %p,%p, page %lu+%d\n" , __func__, |
525 | priv, vm_priv, vma->vm_pgoff, count); |
526 | |
527 | gref = find_grefs(priv, vma->vm_pgoff << PAGE_SHIFT, count); |
528 | if (gref == NULL) { |
529 | rv = -ENOENT; |
530 | pr_debug("%s: Could not find grant reference" , |
531 | __func__); |
532 | kfree(vm_priv); |
533 | goto out_unlock; |
534 | } |
535 | |
536 | vm_priv->gref = gref; |
537 | vm_priv->users = 1; |
538 | vm_priv->count = count; |
539 | |
540 | vma->vm_private_data = vm_priv; |
541 | |
542 | vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP; |
543 | |
544 | vma->vm_ops = &gntalloc_vmops; |
545 | |
546 | for (i = 0; i < count; i++) { |
547 | gref->users++; |
548 | rv = vm_insert_page(vma, vma->vm_start + i * PAGE_SIZE, |
549 | gref->page); |
550 | if (rv) |
551 | goto out_unlock; |
552 | |
553 | gref = list_entry(gref->next_file.next, |
554 | struct gntalloc_gref, next_file); |
555 | } |
556 | rv = 0; |
557 | |
558 | out_unlock: |
559 | mutex_unlock(&gref_mutex); |
560 | return rv; |
561 | } |
562 | |
563 | static const struct file_operations gntalloc_fops = { |
564 | .owner = THIS_MODULE, |
565 | .open = gntalloc_open, |
566 | .release = gntalloc_release, |
567 | .unlocked_ioctl = gntalloc_ioctl, |
568 | .mmap = gntalloc_mmap |
569 | }; |
570 | |
571 | /* |
572 | * ------------------------------------- |
573 | * Module creation/destruction. |
574 | * ------------------------------------- |
575 | */ |
576 | static struct miscdevice gntalloc_miscdev = { |
577 | .minor = MISC_DYNAMIC_MINOR, |
578 | .name = "xen/gntalloc" , |
579 | .fops = &gntalloc_fops, |
580 | }; |
581 | |
582 | static int __init gntalloc_init(void) |
583 | { |
584 | int err; |
585 | |
586 | if (!xen_domain()) |
587 | return -ENODEV; |
588 | |
589 | err = misc_register(&gntalloc_miscdev); |
590 | if (err != 0) { |
591 | pr_err("Could not register misc gntalloc device\n" ); |
592 | return err; |
593 | } |
594 | |
595 | pr_debug("Created grant allocation device at %d,%d\n" , |
596 | MISC_MAJOR, gntalloc_miscdev.minor); |
597 | |
598 | return 0; |
599 | } |
600 | |
601 | static void __exit gntalloc_exit(void) |
602 | { |
603 | misc_deregister(&gntalloc_miscdev); |
604 | } |
605 | |
606 | module_init(gntalloc_init); |
607 | module_exit(gntalloc_exit); |
608 | |
609 | MODULE_LICENSE("GPL" ); |
610 | MODULE_AUTHOR("Carter Weatherly <carter.weatherly@jhuapl.edu>, " |
611 | "Daniel De Graaf <dgdegra@tycho.nsa.gov>" ); |
612 | MODULE_DESCRIPTION("User-space grant reference allocator driver" ); |
613 | |