1 | /****************************************************************************** |
2 | * gntalloc.c |
3 | * |
4 | * Device for creating grant references (in user-space) that may be shared |
5 | * with other domains. |
6 | * |
7 | * This program is distributed in the hope that it will be useful, |
8 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
9 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
10 | * GNU General Public License for more details. |
11 | * |
12 | * You should have received a copy of the GNU General Public License |
13 | * along with this program; if not, write to the Free Software |
14 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA |
15 | */ |
16 | |
17 | /* |
18 | * This driver exists to allow userspace programs in Linux to allocate kernel |
19 | * memory that will later be shared with another domain. Without this device, |
20 | * Linux userspace programs cannot create grant references. |
21 | * |
22 | * How this stuff works: |
23 | * X -> granting a page to Y |
24 | * Y -> mapping the grant from X |
25 | * |
26 | * 1. X uses the gntalloc device to allocate a page of kernel memory, P. |
27 | * 2. X creates an entry in the grant table that says domid(Y) can access P. |
28 | * This is done without a hypercall unless the grant table needs expansion. |
29 | * 3. X gives the grant reference identifier, GREF, to Y. |
30 | * 4. Y maps the page, either directly into kernel memory for use in a backend |
31 | * driver, or via a the gntdev device to map into the address space of an |
32 | * application running in Y. This is the first point at which Xen does any |
33 | * tracking of the page. |
34 | * 5. A program in X mmap()s a segment of the gntalloc device that corresponds |
35 | * to the shared page, and can now communicate with Y over the shared page. |
36 | * |
37 | * |
38 | * NOTE TO USERSPACE LIBRARIES: |
39 | * The grant allocation and mmap()ing are, naturally, two separate operations. |
40 | * You set up the sharing by calling the create ioctl() and then the mmap(). |
41 | * Teardown requires munmap() and either close() or ioctl(). |
42 | * |
43 | * WARNING: Since Xen does not allow a guest to forcibly end the use of a grant |
44 | * reference, this device can be used to consume kernel memory by leaving grant |
45 | * references mapped by another domain when an application exits. Therefore, |
46 | * there is a global limit on the number of pages that can be allocated. When |
47 | * all references to the page are unmapped, it will be freed during the next |
48 | * grant operation. |
49 | */ |
50 | |
51 | #define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt |
52 | |
53 | #include <linux/atomic.h> |
54 | #include <linux/module.h> |
55 | #include <linux/miscdevice.h> |
56 | #include <linux/kernel.h> |
57 | #include <linux/init.h> |
58 | #include <linux/slab.h> |
59 | #include <linux/fs.h> |
60 | #include <linux/device.h> |
61 | #include <linux/mm.h> |
62 | #include <linux/uaccess.h> |
63 | #include <linux/types.h> |
64 | #include <linux/list.h> |
65 | #include <linux/highmem.h> |
66 | |
67 | #include <xen/xen.h> |
68 | #include <xen/page.h> |
69 | #include <xen/grant_table.h> |
70 | #include <xen/gntalloc.h> |
71 | #include <xen/events.h> |
72 | |
73 | static int limit = 1024; |
74 | module_param(limit, int, 0644); |
75 | MODULE_PARM_DESC(limit, "Maximum number of grants that may be allocated by " |
76 | "the gntalloc device" ); |
77 | |
78 | static LIST_HEAD(gref_list); |
79 | static DEFINE_MUTEX(gref_mutex); |
80 | static int gref_size; |
81 | |
82 | struct notify_info { |
83 | uint16_t pgoff:12; /* Bits 0-11: Offset of the byte to clear */ |
84 | uint16_t flags:2; /* Bits 12-13: Unmap notification flags */ |
85 | int event; /* Port (event channel) to notify */ |
86 | }; |
87 | |
88 | /* Metadata on a grant reference. */ |
89 | struct gntalloc_gref { |
90 | struct list_head next_gref; /* list entry gref_list */ |
91 | struct list_head next_file; /* list entry file->list, if open */ |
92 | struct page *page; /* The shared page */ |
93 | uint64_t file_index; /* File offset for mmap() */ |
94 | unsigned int users; /* Use count - when zero, waiting on Xen */ |
95 | grant_ref_t gref_id; /* The grant reference number */ |
96 | struct notify_info notify; /* Unmap notification */ |
97 | }; |
98 | |
99 | struct gntalloc_file_private_data { |
100 | struct list_head list; |
101 | uint64_t index; |
102 | }; |
103 | |
104 | struct gntalloc_vma_private_data { |
105 | struct gntalloc_gref *gref; |
106 | int users; |
107 | int count; |
108 | }; |
109 | |
110 | static void __del_gref(struct gntalloc_gref *gref); |
111 | |
112 | static void do_cleanup(void) |
113 | { |
114 | struct gntalloc_gref *gref, *n; |
115 | list_for_each_entry_safe(gref, n, &gref_list, next_gref) { |
116 | if (!gref->users) |
117 | __del_gref(gref); |
118 | } |
119 | } |
120 | |
121 | static int add_grefs(struct ioctl_gntalloc_alloc_gref *op, |
122 | uint32_t *gref_ids, struct gntalloc_file_private_data *priv) |
123 | { |
124 | int i, rc, readonly; |
125 | LIST_HEAD(queue_gref); |
126 | LIST_HEAD(queue_file); |
127 | struct gntalloc_gref *gref, *next; |
128 | |
129 | readonly = !(op->flags & GNTALLOC_FLAG_WRITABLE); |
130 | for (i = 0; i < op->count; i++) { |
131 | gref = kzalloc(size: sizeof(*gref), GFP_KERNEL); |
132 | if (!gref) { |
133 | rc = -ENOMEM; |
134 | goto undo; |
135 | } |
136 | list_add_tail(new: &gref->next_gref, head: &queue_gref); |
137 | list_add_tail(new: &gref->next_file, head: &queue_file); |
138 | gref->users = 1; |
139 | gref->file_index = op->index + i * PAGE_SIZE; |
140 | gref->page = alloc_page(GFP_KERNEL|__GFP_ZERO); |
141 | if (!gref->page) { |
142 | rc = -ENOMEM; |
143 | goto undo; |
144 | } |
145 | |
146 | /* Grant foreign access to the page. */ |
147 | rc = gnttab_grant_foreign_access(domid: op->domid, |
148 | frame: xen_page_to_gfn(page: gref->page), |
149 | readonly); |
150 | if (rc < 0) |
151 | goto undo; |
152 | gref_ids[i] = gref->gref_id = rc; |
153 | } |
154 | |
155 | /* Add to gref lists. */ |
156 | mutex_lock(&gref_mutex); |
157 | list_splice_tail(list: &queue_gref, head: &gref_list); |
158 | list_splice_tail(list: &queue_file, head: &priv->list); |
159 | mutex_unlock(lock: &gref_mutex); |
160 | |
161 | return 0; |
162 | |
163 | undo: |
164 | mutex_lock(&gref_mutex); |
165 | gref_size -= (op->count - i); |
166 | |
167 | list_for_each_entry_safe(gref, next, &queue_file, next_file) { |
168 | list_del(entry: &gref->next_file); |
169 | __del_gref(gref); |
170 | } |
171 | |
172 | mutex_unlock(lock: &gref_mutex); |
173 | return rc; |
174 | } |
175 | |
176 | static void __del_gref(struct gntalloc_gref *gref) |
177 | { |
178 | if (gref->notify.flags & UNMAP_NOTIFY_CLEAR_BYTE) { |
179 | uint8_t *tmp = kmap_local_page(page: gref->page); |
180 | tmp[gref->notify.pgoff] = 0; |
181 | kunmap_local(tmp); |
182 | } |
183 | if (gref->notify.flags & UNMAP_NOTIFY_SEND_EVENT) { |
184 | notify_remote_via_evtchn(port: gref->notify.event); |
185 | evtchn_put(evtchn: gref->notify.event); |
186 | } |
187 | |
188 | gref->notify.flags = 0; |
189 | |
190 | if (gref->gref_id) { |
191 | if (gref->page) |
192 | gnttab_end_foreign_access(ref: gref->gref_id, page: gref->page); |
193 | else |
194 | gnttab_free_grant_reference(ref: gref->gref_id); |
195 | } |
196 | |
197 | gref_size--; |
198 | list_del(entry: &gref->next_gref); |
199 | |
200 | kfree(objp: gref); |
201 | } |
202 | |
203 | /* finds contiguous grant references in a file, returns the first */ |
204 | static struct gntalloc_gref *find_grefs(struct gntalloc_file_private_data *priv, |
205 | uint64_t index, uint32_t count) |
206 | { |
207 | struct gntalloc_gref *rv = NULL, *gref; |
208 | list_for_each_entry(gref, &priv->list, next_file) { |
209 | if (gref->file_index == index && !rv) |
210 | rv = gref; |
211 | if (rv) { |
212 | if (gref->file_index != index) |
213 | return NULL; |
214 | index += PAGE_SIZE; |
215 | count--; |
216 | if (count == 0) |
217 | return rv; |
218 | } |
219 | } |
220 | return NULL; |
221 | } |
222 | |
223 | /* |
224 | * ------------------------------------- |
225 | * File operations. |
226 | * ------------------------------------- |
227 | */ |
228 | static int gntalloc_open(struct inode *inode, struct file *filp) |
229 | { |
230 | struct gntalloc_file_private_data *priv; |
231 | |
232 | priv = kzalloc(size: sizeof(*priv), GFP_KERNEL); |
233 | if (!priv) |
234 | goto out_nomem; |
235 | INIT_LIST_HEAD(list: &priv->list); |
236 | |
237 | filp->private_data = priv; |
238 | |
239 | pr_debug("%s: priv %p\n" , __func__, priv); |
240 | |
241 | return 0; |
242 | |
243 | out_nomem: |
244 | return -ENOMEM; |
245 | } |
246 | |
247 | static int gntalloc_release(struct inode *inode, struct file *filp) |
248 | { |
249 | struct gntalloc_file_private_data *priv = filp->private_data; |
250 | struct gntalloc_gref *gref; |
251 | |
252 | pr_debug("%s: priv %p\n" , __func__, priv); |
253 | |
254 | mutex_lock(&gref_mutex); |
255 | while (!list_empty(head: &priv->list)) { |
256 | gref = list_entry(priv->list.next, |
257 | struct gntalloc_gref, next_file); |
258 | list_del(entry: &gref->next_file); |
259 | gref->users--; |
260 | if (gref->users == 0) |
261 | __del_gref(gref); |
262 | } |
263 | kfree(objp: priv); |
264 | mutex_unlock(lock: &gref_mutex); |
265 | |
266 | return 0; |
267 | } |
268 | |
269 | static long gntalloc_ioctl_alloc(struct gntalloc_file_private_data *priv, |
270 | struct ioctl_gntalloc_alloc_gref __user *arg) |
271 | { |
272 | int rc = 0; |
273 | struct ioctl_gntalloc_alloc_gref op; |
274 | uint32_t *gref_ids; |
275 | |
276 | pr_debug("%s: priv %p\n" , __func__, priv); |
277 | |
278 | if (copy_from_user(to: &op, from: arg, n: sizeof(op))) { |
279 | rc = -EFAULT; |
280 | goto out; |
281 | } |
282 | |
283 | gref_ids = kcalloc(n: op.count, size: sizeof(gref_ids[0]), GFP_KERNEL); |
284 | if (!gref_ids) { |
285 | rc = -ENOMEM; |
286 | goto out; |
287 | } |
288 | |
289 | mutex_lock(&gref_mutex); |
290 | /* Clean up pages that were at zero (local) users but were still mapped |
291 | * by remote domains. Since those pages count towards the limit that we |
292 | * are about to enforce, removing them here is a good idea. |
293 | */ |
294 | do_cleanup(); |
295 | if (gref_size + op.count > limit) { |
296 | mutex_unlock(lock: &gref_mutex); |
297 | rc = -ENOSPC; |
298 | goto out_free; |
299 | } |
300 | gref_size += op.count; |
301 | op.index = priv->index; |
302 | priv->index += op.count * PAGE_SIZE; |
303 | mutex_unlock(lock: &gref_mutex); |
304 | |
305 | rc = add_grefs(op: &op, gref_ids, priv); |
306 | if (rc < 0) |
307 | goto out_free; |
308 | |
309 | /* Once we finish add_grefs, it is unsafe to touch the new reference, |
310 | * since it is possible for a concurrent ioctl to remove it (by guessing |
311 | * its index). If the userspace application doesn't provide valid memory |
312 | * to write the IDs to, then it will need to close the file in order to |
313 | * release - which it will do by segfaulting when it tries to access the |
314 | * IDs to close them. |
315 | */ |
316 | if (copy_to_user(to: arg, from: &op, n: sizeof(op))) { |
317 | rc = -EFAULT; |
318 | goto out_free; |
319 | } |
320 | if (copy_to_user(to: arg->gref_ids, from: gref_ids, |
321 | n: sizeof(gref_ids[0]) * op.count)) { |
322 | rc = -EFAULT; |
323 | goto out_free; |
324 | } |
325 | |
326 | out_free: |
327 | kfree(objp: gref_ids); |
328 | out: |
329 | return rc; |
330 | } |
331 | |
332 | static long gntalloc_ioctl_dealloc(struct gntalloc_file_private_data *priv, |
333 | void __user *arg) |
334 | { |
335 | int i, rc = 0; |
336 | struct ioctl_gntalloc_dealloc_gref op; |
337 | struct gntalloc_gref *gref, *n; |
338 | |
339 | pr_debug("%s: priv %p\n" , __func__, priv); |
340 | |
341 | if (copy_from_user(to: &op, from: arg, n: sizeof(op))) { |
342 | rc = -EFAULT; |
343 | goto dealloc_grant_out; |
344 | } |
345 | |
346 | mutex_lock(&gref_mutex); |
347 | gref = find_grefs(priv, index: op.index, count: op.count); |
348 | if (gref) { |
349 | /* Remove from the file list only, and decrease reference count. |
350 | * The later call to do_cleanup() will remove from gref_list and |
351 | * free the memory if the pages aren't mapped anywhere. |
352 | */ |
353 | for (i = 0; i < op.count; i++) { |
354 | n = list_entry(gref->next_file.next, |
355 | struct gntalloc_gref, next_file); |
356 | list_del(entry: &gref->next_file); |
357 | gref->users--; |
358 | gref = n; |
359 | } |
360 | } else { |
361 | rc = -EINVAL; |
362 | } |
363 | |
364 | do_cleanup(); |
365 | |
366 | mutex_unlock(lock: &gref_mutex); |
367 | dealloc_grant_out: |
368 | return rc; |
369 | } |
370 | |
371 | static long gntalloc_ioctl_unmap_notify(struct gntalloc_file_private_data *priv, |
372 | void __user *arg) |
373 | { |
374 | struct ioctl_gntalloc_unmap_notify op; |
375 | struct gntalloc_gref *gref; |
376 | uint64_t index; |
377 | int pgoff; |
378 | int rc; |
379 | |
380 | if (copy_from_user(to: &op, from: arg, n: sizeof(op))) |
381 | return -EFAULT; |
382 | |
383 | index = op.index & ~(PAGE_SIZE - 1); |
384 | pgoff = op.index & (PAGE_SIZE - 1); |
385 | |
386 | mutex_lock(&gref_mutex); |
387 | |
388 | gref = find_grefs(priv, index, count: 1); |
389 | if (!gref) { |
390 | rc = -ENOENT; |
391 | goto unlock_out; |
392 | } |
393 | |
394 | if (op.action & ~(UNMAP_NOTIFY_CLEAR_BYTE|UNMAP_NOTIFY_SEND_EVENT)) { |
395 | rc = -EINVAL; |
396 | goto unlock_out; |
397 | } |
398 | |
399 | /* We need to grab a reference to the event channel we are going to use |
400 | * to send the notify before releasing the reference we may already have |
401 | * (if someone has called this ioctl twice). This is required so that |
402 | * it is possible to change the clear_byte part of the notification |
403 | * without disturbing the event channel part, which may now be the last |
404 | * reference to that event channel. |
405 | */ |
406 | if (op.action & UNMAP_NOTIFY_SEND_EVENT) { |
407 | if (evtchn_get(evtchn: op.event_channel_port)) { |
408 | rc = -EINVAL; |
409 | goto unlock_out; |
410 | } |
411 | } |
412 | |
413 | if (gref->notify.flags & UNMAP_NOTIFY_SEND_EVENT) |
414 | evtchn_put(evtchn: gref->notify.event); |
415 | |
416 | gref->notify.flags = op.action; |
417 | gref->notify.pgoff = pgoff; |
418 | gref->notify.event = op.event_channel_port; |
419 | rc = 0; |
420 | |
421 | unlock_out: |
422 | mutex_unlock(lock: &gref_mutex); |
423 | return rc; |
424 | } |
425 | |
426 | static long gntalloc_ioctl(struct file *filp, unsigned int cmd, |
427 | unsigned long arg) |
428 | { |
429 | struct gntalloc_file_private_data *priv = filp->private_data; |
430 | |
431 | switch (cmd) { |
432 | case IOCTL_GNTALLOC_ALLOC_GREF: |
433 | return gntalloc_ioctl_alloc(priv, arg: (void __user *)arg); |
434 | |
435 | case IOCTL_GNTALLOC_DEALLOC_GREF: |
436 | return gntalloc_ioctl_dealloc(priv, arg: (void __user *)arg); |
437 | |
438 | case IOCTL_GNTALLOC_SET_UNMAP_NOTIFY: |
439 | return gntalloc_ioctl_unmap_notify(priv, arg: (void __user *)arg); |
440 | |
441 | default: |
442 | return -ENOIOCTLCMD; |
443 | } |
444 | |
445 | return 0; |
446 | } |
447 | |
448 | static void gntalloc_vma_open(struct vm_area_struct *vma) |
449 | { |
450 | struct gntalloc_vma_private_data *priv = vma->vm_private_data; |
451 | |
452 | if (!priv) |
453 | return; |
454 | |
455 | mutex_lock(&gref_mutex); |
456 | priv->users++; |
457 | mutex_unlock(lock: &gref_mutex); |
458 | } |
459 | |
460 | static void gntalloc_vma_close(struct vm_area_struct *vma) |
461 | { |
462 | struct gntalloc_vma_private_data *priv = vma->vm_private_data; |
463 | struct gntalloc_gref *gref, *next; |
464 | int i; |
465 | |
466 | if (!priv) |
467 | return; |
468 | |
469 | mutex_lock(&gref_mutex); |
470 | priv->users--; |
471 | if (priv->users == 0) { |
472 | gref = priv->gref; |
473 | for (i = 0; i < priv->count; i++) { |
474 | gref->users--; |
475 | next = list_entry(gref->next_gref.next, |
476 | struct gntalloc_gref, next_gref); |
477 | if (gref->users == 0) |
478 | __del_gref(gref); |
479 | gref = next; |
480 | } |
481 | kfree(objp: priv); |
482 | } |
483 | mutex_unlock(lock: &gref_mutex); |
484 | } |
485 | |
486 | static const struct vm_operations_struct gntalloc_vmops = { |
487 | .open = gntalloc_vma_open, |
488 | .close = gntalloc_vma_close, |
489 | }; |
490 | |
491 | static int gntalloc_mmap(struct file *filp, struct vm_area_struct *vma) |
492 | { |
493 | struct gntalloc_file_private_data *priv = filp->private_data; |
494 | struct gntalloc_vma_private_data *vm_priv; |
495 | struct gntalloc_gref *gref; |
496 | int count = vma_pages(vma); |
497 | int rv, i; |
498 | |
499 | if (!(vma->vm_flags & VM_SHARED)) { |
500 | pr_err("%s: Mapping must be shared\n" , __func__); |
501 | return -EINVAL; |
502 | } |
503 | |
504 | vm_priv = kmalloc(size: sizeof(*vm_priv), GFP_KERNEL); |
505 | if (!vm_priv) |
506 | return -ENOMEM; |
507 | |
508 | mutex_lock(&gref_mutex); |
509 | |
510 | pr_debug("%s: priv %p,%p, page %lu+%d\n" , __func__, |
511 | priv, vm_priv, vma->vm_pgoff, count); |
512 | |
513 | gref = find_grefs(priv, index: vma->vm_pgoff << PAGE_SHIFT, count); |
514 | if (gref == NULL) { |
515 | rv = -ENOENT; |
516 | pr_debug("%s: Could not find grant reference" , |
517 | __func__); |
518 | kfree(objp: vm_priv); |
519 | goto out_unlock; |
520 | } |
521 | |
522 | vm_priv->gref = gref; |
523 | vm_priv->users = 1; |
524 | vm_priv->count = count; |
525 | |
526 | vma->vm_private_data = vm_priv; |
527 | |
528 | vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP); |
529 | |
530 | vma->vm_ops = &gntalloc_vmops; |
531 | |
532 | for (i = 0; i < count; i++) { |
533 | gref->users++; |
534 | rv = vm_insert_page(vma, addr: vma->vm_start + i * PAGE_SIZE, |
535 | gref->page); |
536 | if (rv) |
537 | goto out_unlock; |
538 | |
539 | gref = list_entry(gref->next_file.next, |
540 | struct gntalloc_gref, next_file); |
541 | } |
542 | rv = 0; |
543 | |
544 | out_unlock: |
545 | mutex_unlock(lock: &gref_mutex); |
546 | return rv; |
547 | } |
548 | |
549 | static const struct file_operations gntalloc_fops = { |
550 | .owner = THIS_MODULE, |
551 | .open = gntalloc_open, |
552 | .release = gntalloc_release, |
553 | .unlocked_ioctl = gntalloc_ioctl, |
554 | .mmap = gntalloc_mmap |
555 | }; |
556 | |
557 | /* |
558 | * ------------------------------------- |
559 | * Module creation/destruction. |
560 | * ------------------------------------- |
561 | */ |
562 | static struct miscdevice gntalloc_miscdev = { |
563 | .minor = MISC_DYNAMIC_MINOR, |
564 | .name = "xen/gntalloc" , |
565 | .fops = &gntalloc_fops, |
566 | }; |
567 | |
568 | static int __init gntalloc_init(void) |
569 | { |
570 | int err; |
571 | |
572 | if (!xen_domain()) |
573 | return -ENODEV; |
574 | |
575 | err = misc_register(misc: &gntalloc_miscdev); |
576 | if (err != 0) { |
577 | pr_err("Could not register misc gntalloc device\n" ); |
578 | return err; |
579 | } |
580 | |
581 | pr_debug("Created grant allocation device at %d,%d\n" , |
582 | MISC_MAJOR, gntalloc_miscdev.minor); |
583 | |
584 | return 0; |
585 | } |
586 | |
587 | static void __exit gntalloc_exit(void) |
588 | { |
589 | misc_deregister(misc: &gntalloc_miscdev); |
590 | } |
591 | |
592 | module_init(gntalloc_init); |
593 | module_exit(gntalloc_exit); |
594 | |
595 | MODULE_LICENSE("GPL" ); |
596 | MODULE_AUTHOR("Carter Weatherly <carter.weatherly@jhuapl.edu>, " |
597 | "Daniel De Graaf <dgdegra@tycho.nsa.gov>" ); |
598 | MODULE_DESCRIPTION("User-space grant reference allocator driver" ); |
599 | |