1 | /* |
2 | * Copyright 2013 Red Hat Inc. |
3 | * |
4 | * This program is free software; you can redistribute it and/or modify |
5 | * it under the terms of the GNU General Public License as published by |
6 | * the Free Software Foundation; either version 2 of the License, or |
7 | * (at your option) any later version. |
8 | * |
9 | * This program is distributed in the hope that it will be useful, |
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
12 | * GNU General Public License for more details. |
13 | * |
14 | * Authors: Jérôme Glisse <jglisse@redhat.com> |
15 | */ |
16 | /* |
17 | * Refer to include/linux/hmm.h for information about heterogeneous memory |
18 | * management or HMM for short. |
19 | */ |
20 | #include <linux/mm.h> |
21 | #include <linux/hmm.h> |
22 | #include <linux/init.h> |
23 | #include <linux/rmap.h> |
24 | #include <linux/swap.h> |
25 | #include <linux/slab.h> |
26 | #include <linux/sched.h> |
27 | #include <linux/mmzone.h> |
28 | #include <linux/pagemap.h> |
29 | #include <linux/swapops.h> |
30 | #include <linux/hugetlb.h> |
31 | #include <linux/memremap.h> |
32 | #include <linux/jump_label.h> |
33 | #include <linux/mmu_notifier.h> |
34 | #include <linux/memory_hotplug.h> |
35 | |
36 | #define PA_SECTION_SIZE (1UL << PA_SECTION_SHIFT) |
37 | |
38 | #if IS_ENABLED(CONFIG_HMM_MIRROR) |
39 | static const struct mmu_notifier_ops hmm_mmu_notifier_ops; |
40 | |
41 | /* |
42 | * struct hmm - HMM per mm struct |
43 | * |
44 | * @mm: mm struct this HMM struct is bound to |
45 | * @lock: lock protecting ranges list |
46 | * @ranges: list of range being snapshotted |
47 | * @mirrors: list of mirrors for this mm |
48 | * @mmu_notifier: mmu notifier to track updates to CPU page table |
49 | * @mirrors_sem: read/write semaphore protecting the mirrors list |
50 | */ |
51 | struct hmm { |
52 | struct mm_struct *mm; |
53 | spinlock_t lock; |
54 | struct list_head ranges; |
55 | struct list_head mirrors; |
56 | struct mmu_notifier mmu_notifier; |
57 | struct rw_semaphore mirrors_sem; |
58 | }; |
59 | |
60 | /* |
61 | * hmm_register - register HMM against an mm (HMM internal) |
62 | * |
63 | * @mm: mm struct to attach to |
64 | * |
65 | * This is not intended to be used directly by device drivers. It allocates an |
66 | * HMM struct if mm does not have one, and initializes it. |
67 | */ |
68 | static struct hmm *hmm_register(struct mm_struct *mm) |
69 | { |
70 | struct hmm *hmm = READ_ONCE(mm->hmm); |
71 | bool cleanup = false; |
72 | |
73 | /* |
74 | * The hmm struct can only be freed once the mm_struct goes away, |
75 | * hence we should always have pre-allocated an new hmm struct |
76 | * above. |
77 | */ |
78 | if (hmm) |
79 | return hmm; |
80 | |
81 | hmm = kmalloc(sizeof(*hmm), GFP_KERNEL); |
82 | if (!hmm) |
83 | return NULL; |
84 | INIT_LIST_HEAD(&hmm->mirrors); |
85 | init_rwsem(&hmm->mirrors_sem); |
86 | hmm->mmu_notifier.ops = NULL; |
87 | INIT_LIST_HEAD(&hmm->ranges); |
88 | spin_lock_init(&hmm->lock); |
89 | hmm->mm = mm; |
90 | |
91 | spin_lock(&mm->page_table_lock); |
92 | if (!mm->hmm) |
93 | mm->hmm = hmm; |
94 | else |
95 | cleanup = true; |
96 | spin_unlock(&mm->page_table_lock); |
97 | |
98 | if (cleanup) |
99 | goto error; |
100 | |
101 | /* |
102 | * We should only get here if hold the mmap_sem in write mode ie on |
103 | * registration of first mirror through hmm_mirror_register() |
104 | */ |
105 | hmm->mmu_notifier.ops = &hmm_mmu_notifier_ops; |
106 | if (__mmu_notifier_register(&hmm->mmu_notifier, mm)) |
107 | goto error_mm; |
108 | |
109 | return mm->hmm; |
110 | |
111 | error_mm: |
112 | spin_lock(&mm->page_table_lock); |
113 | if (mm->hmm == hmm) |
114 | mm->hmm = NULL; |
115 | spin_unlock(&mm->page_table_lock); |
116 | error: |
117 | kfree(hmm); |
118 | return NULL; |
119 | } |
120 | |
121 | void hmm_mm_destroy(struct mm_struct *mm) |
122 | { |
123 | kfree(mm->hmm); |
124 | } |
125 | |
126 | static int hmm_invalidate_range(struct hmm *hmm, bool device, |
127 | const struct hmm_update *update) |
128 | { |
129 | struct hmm_mirror *mirror; |
130 | struct hmm_range *range; |
131 | |
132 | spin_lock(&hmm->lock); |
133 | list_for_each_entry(range, &hmm->ranges, list) { |
134 | unsigned long addr, idx, npages; |
135 | |
136 | if (update->end < range->start || update->start >= range->end) |
137 | continue; |
138 | |
139 | range->valid = false; |
140 | addr = max(update->start, range->start); |
141 | idx = (addr - range->start) >> PAGE_SHIFT; |
142 | npages = (min(range->end, update->end) - addr) >> PAGE_SHIFT; |
143 | memset(&range->pfns[idx], 0, sizeof(*range->pfns) * npages); |
144 | } |
145 | spin_unlock(&hmm->lock); |
146 | |
147 | if (!device) |
148 | return 0; |
149 | |
150 | down_read(&hmm->mirrors_sem); |
151 | list_for_each_entry(mirror, &hmm->mirrors, list) { |
152 | int ret; |
153 | |
154 | ret = mirror->ops->sync_cpu_device_pagetables(mirror, update); |
155 | if (!update->blockable && ret == -EAGAIN) { |
156 | up_read(&hmm->mirrors_sem); |
157 | return -EAGAIN; |
158 | } |
159 | } |
160 | up_read(&hmm->mirrors_sem); |
161 | |
162 | return 0; |
163 | } |
164 | |
165 | static void hmm_release(struct mmu_notifier *mn, struct mm_struct *mm) |
166 | { |
167 | struct hmm_mirror *mirror; |
168 | struct hmm *hmm = mm->hmm; |
169 | |
170 | down_write(&hmm->mirrors_sem); |
171 | mirror = list_first_entry_or_null(&hmm->mirrors, struct hmm_mirror, |
172 | list); |
173 | while (mirror) { |
174 | list_del_init(&mirror->list); |
175 | if (mirror->ops->release) { |
176 | /* |
177 | * Drop mirrors_sem so callback can wait on any pending |
178 | * work that might itself trigger mmu_notifier callback |
179 | * and thus would deadlock with us. |
180 | */ |
181 | up_write(&hmm->mirrors_sem); |
182 | mirror->ops->release(mirror); |
183 | down_write(&hmm->mirrors_sem); |
184 | } |
185 | mirror = list_first_entry_or_null(&hmm->mirrors, |
186 | struct hmm_mirror, list); |
187 | } |
188 | up_write(&hmm->mirrors_sem); |
189 | } |
190 | |
191 | static int hmm_invalidate_range_start(struct mmu_notifier *mn, |
192 | const struct mmu_notifier_range *range) |
193 | { |
194 | struct hmm_update update; |
195 | struct hmm *hmm = range->mm->hmm; |
196 | |
197 | VM_BUG_ON(!hmm); |
198 | |
199 | update.start = range->start; |
200 | update.end = range->end; |
201 | update.event = HMM_UPDATE_INVALIDATE; |
202 | update.blockable = range->blockable; |
203 | return hmm_invalidate_range(hmm, true, &update); |
204 | } |
205 | |
206 | static void hmm_invalidate_range_end(struct mmu_notifier *mn, |
207 | const struct mmu_notifier_range *range) |
208 | { |
209 | struct hmm_update update; |
210 | struct hmm *hmm = range->mm->hmm; |
211 | |
212 | VM_BUG_ON(!hmm); |
213 | |
214 | update.start = range->start; |
215 | update.end = range->end; |
216 | update.event = HMM_UPDATE_INVALIDATE; |
217 | update.blockable = true; |
218 | hmm_invalidate_range(hmm, false, &update); |
219 | } |
220 | |
221 | static const struct mmu_notifier_ops hmm_mmu_notifier_ops = { |
222 | .release = hmm_release, |
223 | .invalidate_range_start = hmm_invalidate_range_start, |
224 | .invalidate_range_end = hmm_invalidate_range_end, |
225 | }; |
226 | |
227 | /* |
228 | * hmm_mirror_register() - register a mirror against an mm |
229 | * |
230 | * @mirror: new mirror struct to register |
231 | * @mm: mm to register against |
232 | * |
233 | * To start mirroring a process address space, the device driver must register |
234 | * an HMM mirror struct. |
235 | * |
236 | * THE mm->mmap_sem MUST BE HELD IN WRITE MODE ! |
237 | */ |
238 | int hmm_mirror_register(struct hmm_mirror *mirror, struct mm_struct *mm) |
239 | { |
240 | /* Sanity check */ |
241 | if (!mm || !mirror || !mirror->ops) |
242 | return -EINVAL; |
243 | |
244 | again: |
245 | mirror->hmm = hmm_register(mm); |
246 | if (!mirror->hmm) |
247 | return -ENOMEM; |
248 | |
249 | down_write(&mirror->hmm->mirrors_sem); |
250 | if (mirror->hmm->mm == NULL) { |
251 | /* |
252 | * A racing hmm_mirror_unregister() is about to destroy the hmm |
253 | * struct. Try again to allocate a new one. |
254 | */ |
255 | up_write(&mirror->hmm->mirrors_sem); |
256 | mirror->hmm = NULL; |
257 | goto again; |
258 | } else { |
259 | list_add(&mirror->list, &mirror->hmm->mirrors); |
260 | up_write(&mirror->hmm->mirrors_sem); |
261 | } |
262 | |
263 | return 0; |
264 | } |
265 | EXPORT_SYMBOL(hmm_mirror_register); |
266 | |
267 | /* |
268 | * hmm_mirror_unregister() - unregister a mirror |
269 | * |
270 | * @mirror: new mirror struct to register |
271 | * |
272 | * Stop mirroring a process address space, and cleanup. |
273 | */ |
274 | void hmm_mirror_unregister(struct hmm_mirror *mirror) |
275 | { |
276 | bool should_unregister = false; |
277 | struct mm_struct *mm; |
278 | struct hmm *hmm; |
279 | |
280 | if (mirror->hmm == NULL) |
281 | return; |
282 | |
283 | hmm = mirror->hmm; |
284 | down_write(&hmm->mirrors_sem); |
285 | list_del_init(&mirror->list); |
286 | should_unregister = list_empty(&hmm->mirrors); |
287 | mirror->hmm = NULL; |
288 | mm = hmm->mm; |
289 | hmm->mm = NULL; |
290 | up_write(&hmm->mirrors_sem); |
291 | |
292 | if (!should_unregister || mm == NULL) |
293 | return; |
294 | |
295 | mmu_notifier_unregister_no_release(&hmm->mmu_notifier, mm); |
296 | |
297 | spin_lock(&mm->page_table_lock); |
298 | if (mm->hmm == hmm) |
299 | mm->hmm = NULL; |
300 | spin_unlock(&mm->page_table_lock); |
301 | |
302 | kfree(hmm); |
303 | } |
304 | EXPORT_SYMBOL(hmm_mirror_unregister); |
305 | |
306 | struct hmm_vma_walk { |
307 | struct hmm_range *range; |
308 | unsigned long last; |
309 | bool fault; |
310 | bool block; |
311 | }; |
312 | |
313 | static int hmm_vma_do_fault(struct mm_walk *walk, unsigned long addr, |
314 | bool write_fault, uint64_t *pfn) |
315 | { |
316 | unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_REMOTE; |
317 | struct hmm_vma_walk *hmm_vma_walk = walk->private; |
318 | struct hmm_range *range = hmm_vma_walk->range; |
319 | struct vm_area_struct *vma = walk->vma; |
320 | vm_fault_t ret; |
321 | |
322 | flags |= hmm_vma_walk->block ? 0 : FAULT_FLAG_ALLOW_RETRY; |
323 | flags |= write_fault ? FAULT_FLAG_WRITE : 0; |
324 | ret = handle_mm_fault(vma, addr, flags); |
325 | if (ret & VM_FAULT_RETRY) |
326 | return -EBUSY; |
327 | if (ret & VM_FAULT_ERROR) { |
328 | *pfn = range->values[HMM_PFN_ERROR]; |
329 | return -EFAULT; |
330 | } |
331 | |
332 | return -EAGAIN; |
333 | } |
334 | |
335 | static int hmm_pfns_bad(unsigned long addr, |
336 | unsigned long end, |
337 | struct mm_walk *walk) |
338 | { |
339 | struct hmm_vma_walk *hmm_vma_walk = walk->private; |
340 | struct hmm_range *range = hmm_vma_walk->range; |
341 | uint64_t *pfns = range->pfns; |
342 | unsigned long i; |
343 | |
344 | i = (addr - range->start) >> PAGE_SHIFT; |
345 | for (; addr < end; addr += PAGE_SIZE, i++) |
346 | pfns[i] = range->values[HMM_PFN_ERROR]; |
347 | |
348 | return 0; |
349 | } |
350 | |
351 | /* |
352 | * hmm_vma_walk_hole() - handle a range lacking valid pmd or pte(s) |
353 | * @start: range virtual start address (inclusive) |
354 | * @end: range virtual end address (exclusive) |
355 | * @fault: should we fault or not ? |
356 | * @write_fault: write fault ? |
357 | * @walk: mm_walk structure |
358 | * Returns: 0 on success, -EAGAIN after page fault, or page fault error |
359 | * |
360 | * This function will be called whenever pmd_none() or pte_none() returns true, |
361 | * or whenever there is no page directory covering the virtual address range. |
362 | */ |
363 | static int hmm_vma_walk_hole_(unsigned long addr, unsigned long end, |
364 | bool fault, bool write_fault, |
365 | struct mm_walk *walk) |
366 | { |
367 | struct hmm_vma_walk *hmm_vma_walk = walk->private; |
368 | struct hmm_range *range = hmm_vma_walk->range; |
369 | uint64_t *pfns = range->pfns; |
370 | unsigned long i; |
371 | |
372 | hmm_vma_walk->last = addr; |
373 | i = (addr - range->start) >> PAGE_SHIFT; |
374 | for (; addr < end; addr += PAGE_SIZE, i++) { |
375 | pfns[i] = range->values[HMM_PFN_NONE]; |
376 | if (fault || write_fault) { |
377 | int ret; |
378 | |
379 | ret = hmm_vma_do_fault(walk, addr, write_fault, |
380 | &pfns[i]); |
381 | if (ret != -EAGAIN) |
382 | return ret; |
383 | } |
384 | } |
385 | |
386 | return (fault || write_fault) ? -EAGAIN : 0; |
387 | } |
388 | |
389 | static inline void hmm_pte_need_fault(const struct hmm_vma_walk *hmm_vma_walk, |
390 | uint64_t pfns, uint64_t cpu_flags, |
391 | bool *fault, bool *write_fault) |
392 | { |
393 | struct hmm_range *range = hmm_vma_walk->range; |
394 | |
395 | *fault = *write_fault = false; |
396 | if (!hmm_vma_walk->fault) |
397 | return; |
398 | |
399 | /* We aren't ask to do anything ... */ |
400 | if (!(pfns & range->flags[HMM_PFN_VALID])) |
401 | return; |
402 | /* If this is device memory than only fault if explicitly requested */ |
403 | if ((cpu_flags & range->flags[HMM_PFN_DEVICE_PRIVATE])) { |
404 | /* Do we fault on device memory ? */ |
405 | if (pfns & range->flags[HMM_PFN_DEVICE_PRIVATE]) { |
406 | *write_fault = pfns & range->flags[HMM_PFN_WRITE]; |
407 | *fault = true; |
408 | } |
409 | return; |
410 | } |
411 | |
412 | /* If CPU page table is not valid then we need to fault */ |
413 | *fault = !(cpu_flags & range->flags[HMM_PFN_VALID]); |
414 | /* Need to write fault ? */ |
415 | if ((pfns & range->flags[HMM_PFN_WRITE]) && |
416 | !(cpu_flags & range->flags[HMM_PFN_WRITE])) { |
417 | *write_fault = true; |
418 | *fault = true; |
419 | } |
420 | } |
421 | |
422 | static void hmm_range_need_fault(const struct hmm_vma_walk *hmm_vma_walk, |
423 | const uint64_t *pfns, unsigned long npages, |
424 | uint64_t cpu_flags, bool *fault, |
425 | bool *write_fault) |
426 | { |
427 | unsigned long i; |
428 | |
429 | if (!hmm_vma_walk->fault) { |
430 | *fault = *write_fault = false; |
431 | return; |
432 | } |
433 | |
434 | for (i = 0; i < npages; ++i) { |
435 | hmm_pte_need_fault(hmm_vma_walk, pfns[i], cpu_flags, |
436 | fault, write_fault); |
437 | if ((*fault) || (*write_fault)) |
438 | return; |
439 | } |
440 | } |
441 | |
442 | static int hmm_vma_walk_hole(unsigned long addr, unsigned long end, |
443 | struct mm_walk *walk) |
444 | { |
445 | struct hmm_vma_walk *hmm_vma_walk = walk->private; |
446 | struct hmm_range *range = hmm_vma_walk->range; |
447 | bool fault, write_fault; |
448 | unsigned long i, npages; |
449 | uint64_t *pfns; |
450 | |
451 | i = (addr - range->start) >> PAGE_SHIFT; |
452 | npages = (end - addr) >> PAGE_SHIFT; |
453 | pfns = &range->pfns[i]; |
454 | hmm_range_need_fault(hmm_vma_walk, pfns, npages, |
455 | 0, &fault, &write_fault); |
456 | return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk); |
457 | } |
458 | |
459 | static inline uint64_t pmd_to_hmm_pfn_flags(struct hmm_range *range, pmd_t pmd) |
460 | { |
461 | if (pmd_protnone(pmd)) |
462 | return 0; |
463 | return pmd_write(pmd) ? range->flags[HMM_PFN_VALID] | |
464 | range->flags[HMM_PFN_WRITE] : |
465 | range->flags[HMM_PFN_VALID]; |
466 | } |
467 | |
468 | static int hmm_vma_handle_pmd(struct mm_walk *walk, |
469 | unsigned long addr, |
470 | unsigned long end, |
471 | uint64_t *pfns, |
472 | pmd_t pmd) |
473 | { |
474 | struct hmm_vma_walk *hmm_vma_walk = walk->private; |
475 | struct hmm_range *range = hmm_vma_walk->range; |
476 | unsigned long pfn, npages, i; |
477 | bool fault, write_fault; |
478 | uint64_t cpu_flags; |
479 | |
480 | npages = (end - addr) >> PAGE_SHIFT; |
481 | cpu_flags = pmd_to_hmm_pfn_flags(range, pmd); |
482 | hmm_range_need_fault(hmm_vma_walk, pfns, npages, cpu_flags, |
483 | &fault, &write_fault); |
484 | |
485 | if (pmd_protnone(pmd) || fault || write_fault) |
486 | return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk); |
487 | |
488 | pfn = pmd_pfn(pmd) + pte_index(addr); |
489 | for (i = 0; addr < end; addr += PAGE_SIZE, i++, pfn++) |
490 | pfns[i] = hmm_pfn_from_pfn(range, pfn) | cpu_flags; |
491 | hmm_vma_walk->last = end; |
492 | return 0; |
493 | } |
494 | |
495 | static inline uint64_t pte_to_hmm_pfn_flags(struct hmm_range *range, pte_t pte) |
496 | { |
497 | if (pte_none(pte) || !pte_present(pte)) |
498 | return 0; |
499 | return pte_write(pte) ? range->flags[HMM_PFN_VALID] | |
500 | range->flags[HMM_PFN_WRITE] : |
501 | range->flags[HMM_PFN_VALID]; |
502 | } |
503 | |
504 | static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr, |
505 | unsigned long end, pmd_t *pmdp, pte_t *ptep, |
506 | uint64_t *pfn) |
507 | { |
508 | struct hmm_vma_walk *hmm_vma_walk = walk->private; |
509 | struct hmm_range *range = hmm_vma_walk->range; |
510 | struct vm_area_struct *vma = walk->vma; |
511 | bool fault, write_fault; |
512 | uint64_t cpu_flags; |
513 | pte_t pte = *ptep; |
514 | uint64_t orig_pfn = *pfn; |
515 | |
516 | *pfn = range->values[HMM_PFN_NONE]; |
517 | cpu_flags = pte_to_hmm_pfn_flags(range, pte); |
518 | hmm_pte_need_fault(hmm_vma_walk, orig_pfn, cpu_flags, |
519 | &fault, &write_fault); |
520 | |
521 | if (pte_none(pte)) { |
522 | if (fault || write_fault) |
523 | goto fault; |
524 | return 0; |
525 | } |
526 | |
527 | if (!pte_present(pte)) { |
528 | swp_entry_t entry = pte_to_swp_entry(pte); |
529 | |
530 | if (!non_swap_entry(entry)) { |
531 | if (fault || write_fault) |
532 | goto fault; |
533 | return 0; |
534 | } |
535 | |
536 | /* |
537 | * This is a special swap entry, ignore migration, use |
538 | * device and report anything else as error. |
539 | */ |
540 | if (is_device_private_entry(entry)) { |
541 | cpu_flags = range->flags[HMM_PFN_VALID] | |
542 | range->flags[HMM_PFN_DEVICE_PRIVATE]; |
543 | cpu_flags |= is_write_device_private_entry(entry) ? |
544 | range->flags[HMM_PFN_WRITE] : 0; |
545 | hmm_pte_need_fault(hmm_vma_walk, orig_pfn, cpu_flags, |
546 | &fault, &write_fault); |
547 | if (fault || write_fault) |
548 | goto fault; |
549 | *pfn = hmm_pfn_from_pfn(range, swp_offset(entry)); |
550 | *pfn |= cpu_flags; |
551 | return 0; |
552 | } |
553 | |
554 | if (is_migration_entry(entry)) { |
555 | if (fault || write_fault) { |
556 | pte_unmap(ptep); |
557 | hmm_vma_walk->last = addr; |
558 | migration_entry_wait(vma->vm_mm, |
559 | pmdp, addr); |
560 | return -EAGAIN; |
561 | } |
562 | return 0; |
563 | } |
564 | |
565 | /* Report error for everything else */ |
566 | *pfn = range->values[HMM_PFN_ERROR]; |
567 | return -EFAULT; |
568 | } |
569 | |
570 | if (fault || write_fault) |
571 | goto fault; |
572 | |
573 | *pfn = hmm_pfn_from_pfn(range, pte_pfn(pte)) | cpu_flags; |
574 | return 0; |
575 | |
576 | fault: |
577 | pte_unmap(ptep); |
578 | /* Fault any virtual address we were asked to fault */ |
579 | return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk); |
580 | } |
581 | |
582 | static int hmm_vma_walk_pmd(pmd_t *pmdp, |
583 | unsigned long start, |
584 | unsigned long end, |
585 | struct mm_walk *walk) |
586 | { |
587 | struct hmm_vma_walk *hmm_vma_walk = walk->private; |
588 | struct hmm_range *range = hmm_vma_walk->range; |
589 | struct vm_area_struct *vma = walk->vma; |
590 | uint64_t *pfns = range->pfns; |
591 | unsigned long addr = start, i; |
592 | pte_t *ptep; |
593 | pmd_t pmd; |
594 | |
595 | |
596 | again: |
597 | pmd = READ_ONCE(*pmdp); |
598 | if (pmd_none(pmd)) |
599 | return hmm_vma_walk_hole(start, end, walk); |
600 | |
601 | if (pmd_huge(pmd) && (range->vma->vm_flags & VM_HUGETLB)) |
602 | return hmm_pfns_bad(start, end, walk); |
603 | |
604 | if (thp_migration_supported() && is_pmd_migration_entry(pmd)) { |
605 | bool fault, write_fault; |
606 | unsigned long npages; |
607 | uint64_t *pfns; |
608 | |
609 | i = (addr - range->start) >> PAGE_SHIFT; |
610 | npages = (end - addr) >> PAGE_SHIFT; |
611 | pfns = &range->pfns[i]; |
612 | |
613 | hmm_range_need_fault(hmm_vma_walk, pfns, npages, |
614 | 0, &fault, &write_fault); |
615 | if (fault || write_fault) { |
616 | hmm_vma_walk->last = addr; |
617 | pmd_migration_entry_wait(vma->vm_mm, pmdp); |
618 | return -EAGAIN; |
619 | } |
620 | return 0; |
621 | } else if (!pmd_present(pmd)) |
622 | return hmm_pfns_bad(start, end, walk); |
623 | |
624 | if (pmd_devmap(pmd) || pmd_trans_huge(pmd)) { |
625 | /* |
626 | * No need to take pmd_lock here, even if some other threads |
627 | * is splitting the huge pmd we will get that event through |
628 | * mmu_notifier callback. |
629 | * |
630 | * So just read pmd value and check again its a transparent |
631 | * huge or device mapping one and compute corresponding pfn |
632 | * values. |
633 | */ |
634 | pmd = pmd_read_atomic(pmdp); |
635 | barrier(); |
636 | if (!pmd_devmap(pmd) && !pmd_trans_huge(pmd)) |
637 | goto again; |
638 | |
639 | i = (addr - range->start) >> PAGE_SHIFT; |
640 | return hmm_vma_handle_pmd(walk, addr, end, &pfns[i], pmd); |
641 | } |
642 | |
643 | /* |
644 | * We have handled all the valid case above ie either none, migration, |
645 | * huge or transparent huge. At this point either it is a valid pmd |
646 | * entry pointing to pte directory or it is a bad pmd that will not |
647 | * recover. |
648 | */ |
649 | if (pmd_bad(pmd)) |
650 | return hmm_pfns_bad(start, end, walk); |
651 | |
652 | ptep = pte_offset_map(pmdp, addr); |
653 | i = (addr - range->start) >> PAGE_SHIFT; |
654 | for (; addr < end; addr += PAGE_SIZE, ptep++, i++) { |
655 | int r; |
656 | |
657 | r = hmm_vma_handle_pte(walk, addr, end, pmdp, ptep, &pfns[i]); |
658 | if (r) { |
659 | /* hmm_vma_handle_pte() did unmap pte directory */ |
660 | hmm_vma_walk->last = addr; |
661 | return r; |
662 | } |
663 | } |
664 | pte_unmap(ptep - 1); |
665 | |
666 | hmm_vma_walk->last = addr; |
667 | return 0; |
668 | } |
669 | |
670 | static void hmm_pfns_clear(struct hmm_range *range, |
671 | uint64_t *pfns, |
672 | unsigned long addr, |
673 | unsigned long end) |
674 | { |
675 | for (; addr < end; addr += PAGE_SIZE, pfns++) |
676 | *pfns = range->values[HMM_PFN_NONE]; |
677 | } |
678 | |
679 | static void hmm_pfns_special(struct hmm_range *range) |
680 | { |
681 | unsigned long addr = range->start, i = 0; |
682 | |
683 | for (; addr < range->end; addr += PAGE_SIZE, i++) |
684 | range->pfns[i] = range->values[HMM_PFN_SPECIAL]; |
685 | } |
686 | |
687 | /* |
688 | * hmm_vma_get_pfns() - snapshot CPU page table for a range of virtual addresses |
689 | * @range: range being snapshotted |
690 | * Returns: -EINVAL if invalid argument, -ENOMEM out of memory, -EPERM invalid |
691 | * vma permission, 0 success |
692 | * |
693 | * This snapshots the CPU page table for a range of virtual addresses. Snapshot |
694 | * validity is tracked by range struct. See hmm_vma_range_done() for further |
695 | * information. |
696 | * |
697 | * The range struct is initialized here. It tracks the CPU page table, but only |
698 | * if the function returns success (0), in which case the caller must then call |
699 | * hmm_vma_range_done() to stop CPU page table update tracking on this range. |
700 | * |
701 | * NOT CALLING hmm_vma_range_done() IF FUNCTION RETURNS 0 WILL LEAD TO SERIOUS |
702 | * MEMORY CORRUPTION ! YOU HAVE BEEN WARNED ! |
703 | */ |
704 | int hmm_vma_get_pfns(struct hmm_range *range) |
705 | { |
706 | struct vm_area_struct *vma = range->vma; |
707 | struct hmm_vma_walk hmm_vma_walk; |
708 | struct mm_walk mm_walk; |
709 | struct hmm *hmm; |
710 | |
711 | /* Sanity check, this really should not happen ! */ |
712 | if (range->start < vma->vm_start || range->start >= vma->vm_end) |
713 | return -EINVAL; |
714 | if (range->end < vma->vm_start || range->end > vma->vm_end) |
715 | return -EINVAL; |
716 | |
717 | hmm = hmm_register(vma->vm_mm); |
718 | if (!hmm) |
719 | return -ENOMEM; |
720 | /* Caller must have registered a mirror, via hmm_mirror_register() ! */ |
721 | if (!hmm->mmu_notifier.ops) |
722 | return -EINVAL; |
723 | |
724 | /* FIXME support hugetlb fs */ |
725 | if (is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_SPECIAL) || |
726 | vma_is_dax(vma)) { |
727 | hmm_pfns_special(range); |
728 | return -EINVAL; |
729 | } |
730 | |
731 | if (!(vma->vm_flags & VM_READ)) { |
732 | /* |
733 | * If vma do not allow read access, then assume that it does |
734 | * not allow write access, either. Architecture that allow |
735 | * write without read access are not supported by HMM, because |
736 | * operations such has atomic access would not work. |
737 | */ |
738 | hmm_pfns_clear(range, range->pfns, range->start, range->end); |
739 | return -EPERM; |
740 | } |
741 | |
742 | /* Initialize range to track CPU page table update */ |
743 | spin_lock(&hmm->lock); |
744 | range->valid = true; |
745 | list_add_rcu(&range->list, &hmm->ranges); |
746 | spin_unlock(&hmm->lock); |
747 | |
748 | hmm_vma_walk.fault = false; |
749 | hmm_vma_walk.range = range; |
750 | mm_walk.private = &hmm_vma_walk; |
751 | |
752 | mm_walk.vma = vma; |
753 | mm_walk.mm = vma->vm_mm; |
754 | mm_walk.pte_entry = NULL; |
755 | mm_walk.test_walk = NULL; |
756 | mm_walk.hugetlb_entry = NULL; |
757 | mm_walk.pmd_entry = hmm_vma_walk_pmd; |
758 | mm_walk.pte_hole = hmm_vma_walk_hole; |
759 | |
760 | walk_page_range(range->start, range->end, &mm_walk); |
761 | return 0; |
762 | } |
763 | EXPORT_SYMBOL(hmm_vma_get_pfns); |
764 | |
765 | /* |
766 | * hmm_vma_range_done() - stop tracking change to CPU page table over a range |
767 | * @range: range being tracked |
768 | * Returns: false if range data has been invalidated, true otherwise |
769 | * |
770 | * Range struct is used to track updates to the CPU page table after a call to |
771 | * either hmm_vma_get_pfns() or hmm_vma_fault(). Once the device driver is done |
772 | * using the data, or wants to lock updates to the data it got from those |
773 | * functions, it must call the hmm_vma_range_done() function, which will then |
774 | * stop tracking CPU page table updates. |
775 | * |
776 | * Note that device driver must still implement general CPU page table update |
777 | * tracking either by using hmm_mirror (see hmm_mirror_register()) or by using |
778 | * the mmu_notifier API directly. |
779 | * |
780 | * CPU page table update tracking done through hmm_range is only temporary and |
781 | * to be used while trying to duplicate CPU page table contents for a range of |
782 | * virtual addresses. |
783 | * |
784 | * There are two ways to use this : |
785 | * again: |
786 | * hmm_vma_get_pfns(range); or hmm_vma_fault(...); |
787 | * trans = device_build_page_table_update_transaction(pfns); |
788 | * device_page_table_lock(); |
789 | * if (!hmm_vma_range_done(range)) { |
790 | * device_page_table_unlock(); |
791 | * goto again; |
792 | * } |
793 | * device_commit_transaction(trans); |
794 | * device_page_table_unlock(); |
795 | * |
796 | * Or: |
797 | * hmm_vma_get_pfns(range); or hmm_vma_fault(...); |
798 | * device_page_table_lock(); |
799 | * hmm_vma_range_done(range); |
800 | * device_update_page_table(range->pfns); |
801 | * device_page_table_unlock(); |
802 | */ |
803 | bool hmm_vma_range_done(struct hmm_range *range) |
804 | { |
805 | unsigned long npages = (range->end - range->start) >> PAGE_SHIFT; |
806 | struct hmm *hmm; |
807 | |
808 | if (range->end <= range->start) { |
809 | BUG(); |
810 | return false; |
811 | } |
812 | |
813 | hmm = hmm_register(range->vma->vm_mm); |
814 | if (!hmm) { |
815 | memset(range->pfns, 0, sizeof(*range->pfns) * npages); |
816 | return false; |
817 | } |
818 | |
819 | spin_lock(&hmm->lock); |
820 | list_del_rcu(&range->list); |
821 | spin_unlock(&hmm->lock); |
822 | |
823 | return range->valid; |
824 | } |
825 | EXPORT_SYMBOL(hmm_vma_range_done); |
826 | |
827 | /* |
828 | * hmm_vma_fault() - try to fault some address in a virtual address range |
829 | * @range: range being faulted |
830 | * @block: allow blocking on fault (if true it sleeps and do not drop mmap_sem) |
831 | * Returns: 0 success, error otherwise (-EAGAIN means mmap_sem have been drop) |
832 | * |
833 | * This is similar to a regular CPU page fault except that it will not trigger |
834 | * any memory migration if the memory being faulted is not accessible by CPUs. |
835 | * |
836 | * On error, for one virtual address in the range, the function will mark the |
837 | * corresponding HMM pfn entry with an error flag. |
838 | * |
839 | * Expected use pattern: |
840 | * retry: |
841 | * down_read(&mm->mmap_sem); |
842 | * // Find vma and address device wants to fault, initialize hmm_pfn_t |
843 | * // array accordingly |
844 | * ret = hmm_vma_fault(range, write, block); |
845 | * switch (ret) { |
846 | * case -EAGAIN: |
847 | * hmm_vma_range_done(range); |
848 | * // You might want to rate limit or yield to play nicely, you may |
849 | * // also commit any valid pfn in the array assuming that you are |
850 | * // getting true from hmm_vma_range_monitor_end() |
851 | * goto retry; |
852 | * case 0: |
853 | * break; |
854 | * case -ENOMEM: |
855 | * case -EINVAL: |
856 | * case -EPERM: |
857 | * default: |
858 | * // Handle error ! |
859 | * up_read(&mm->mmap_sem) |
860 | * return; |
861 | * } |
862 | * // Take device driver lock that serialize device page table update |
863 | * driver_lock_device_page_table_update(); |
864 | * hmm_vma_range_done(range); |
865 | * // Commit pfns we got from hmm_vma_fault() |
866 | * driver_unlock_device_page_table_update(); |
867 | * up_read(&mm->mmap_sem) |
868 | * |
869 | * YOU MUST CALL hmm_vma_range_done() AFTER THIS FUNCTION RETURN SUCCESS (0) |
870 | * BEFORE FREEING THE range struct OR YOU WILL HAVE SERIOUS MEMORY CORRUPTION ! |
871 | * |
872 | * YOU HAVE BEEN WARNED ! |
873 | */ |
874 | int hmm_vma_fault(struct hmm_range *range, bool block) |
875 | { |
876 | struct vm_area_struct *vma = range->vma; |
877 | unsigned long start = range->start; |
878 | struct hmm_vma_walk hmm_vma_walk; |
879 | struct mm_walk mm_walk; |
880 | struct hmm *hmm; |
881 | int ret; |
882 | |
883 | /* Sanity check, this really should not happen ! */ |
884 | if (range->start < vma->vm_start || range->start >= vma->vm_end) |
885 | return -EINVAL; |
886 | if (range->end < vma->vm_start || range->end > vma->vm_end) |
887 | return -EINVAL; |
888 | |
889 | hmm = hmm_register(vma->vm_mm); |
890 | if (!hmm) { |
891 | hmm_pfns_clear(range, range->pfns, range->start, range->end); |
892 | return -ENOMEM; |
893 | } |
894 | /* Caller must have registered a mirror using hmm_mirror_register() */ |
895 | if (!hmm->mmu_notifier.ops) |
896 | return -EINVAL; |
897 | |
898 | /* FIXME support hugetlb fs */ |
899 | if (is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_SPECIAL) || |
900 | vma_is_dax(vma)) { |
901 | hmm_pfns_special(range); |
902 | return -EINVAL; |
903 | } |
904 | |
905 | if (!(vma->vm_flags & VM_READ)) { |
906 | /* |
907 | * If vma do not allow read access, then assume that it does |
908 | * not allow write access, either. Architecture that allow |
909 | * write without read access are not supported by HMM, because |
910 | * operations such has atomic access would not work. |
911 | */ |
912 | hmm_pfns_clear(range, range->pfns, range->start, range->end); |
913 | return -EPERM; |
914 | } |
915 | |
916 | /* Initialize range to track CPU page table update */ |
917 | spin_lock(&hmm->lock); |
918 | range->valid = true; |
919 | list_add_rcu(&range->list, &hmm->ranges); |
920 | spin_unlock(&hmm->lock); |
921 | |
922 | hmm_vma_walk.fault = true; |
923 | hmm_vma_walk.block = block; |
924 | hmm_vma_walk.range = range; |
925 | mm_walk.private = &hmm_vma_walk; |
926 | hmm_vma_walk.last = range->start; |
927 | |
928 | mm_walk.vma = vma; |
929 | mm_walk.mm = vma->vm_mm; |
930 | mm_walk.pte_entry = NULL; |
931 | mm_walk.test_walk = NULL; |
932 | mm_walk.hugetlb_entry = NULL; |
933 | mm_walk.pmd_entry = hmm_vma_walk_pmd; |
934 | mm_walk.pte_hole = hmm_vma_walk_hole; |
935 | |
936 | do { |
937 | ret = walk_page_range(start, range->end, &mm_walk); |
938 | start = hmm_vma_walk.last; |
939 | } while (ret == -EAGAIN); |
940 | |
941 | if (ret) { |
942 | unsigned long i; |
943 | |
944 | i = (hmm_vma_walk.last - range->start) >> PAGE_SHIFT; |
945 | hmm_pfns_clear(range, &range->pfns[i], hmm_vma_walk.last, |
946 | range->end); |
947 | hmm_vma_range_done(range); |
948 | } |
949 | return ret; |
950 | } |
951 | EXPORT_SYMBOL(hmm_vma_fault); |
952 | #endif /* IS_ENABLED(CONFIG_HMM_MIRROR) */ |
953 | |
954 | |
955 | #if IS_ENABLED(CONFIG_DEVICE_PRIVATE) || IS_ENABLED(CONFIG_DEVICE_PUBLIC) |
956 | struct page *hmm_vma_alloc_locked_page(struct vm_area_struct *vma, |
957 | unsigned long addr) |
958 | { |
959 | struct page *page; |
960 | |
961 | page = alloc_page_vma(GFP_HIGHUSER, vma, addr); |
962 | if (!page) |
963 | return NULL; |
964 | lock_page(page); |
965 | return page; |
966 | } |
967 | EXPORT_SYMBOL(hmm_vma_alloc_locked_page); |
968 | |
969 | |
970 | static void hmm_devmem_ref_release(struct percpu_ref *ref) |
971 | { |
972 | struct hmm_devmem *devmem; |
973 | |
974 | devmem = container_of(ref, struct hmm_devmem, ref); |
975 | complete(&devmem->completion); |
976 | } |
977 | |
978 | static void hmm_devmem_ref_exit(void *data) |
979 | { |
980 | struct percpu_ref *ref = data; |
981 | struct hmm_devmem *devmem; |
982 | |
983 | devmem = container_of(ref, struct hmm_devmem, ref); |
984 | wait_for_completion(&devmem->completion); |
985 | percpu_ref_exit(ref); |
986 | } |
987 | |
988 | static void hmm_devmem_ref_kill(struct percpu_ref *ref) |
989 | { |
990 | percpu_ref_kill(ref); |
991 | } |
992 | |
993 | static vm_fault_t hmm_devmem_fault(struct vm_area_struct *vma, |
994 | unsigned long addr, |
995 | const struct page *page, |
996 | unsigned int flags, |
997 | pmd_t *pmdp) |
998 | { |
999 | struct hmm_devmem *devmem = page->pgmap->data; |
1000 | |
1001 | return devmem->ops->fault(devmem, vma, addr, page, flags, pmdp); |
1002 | } |
1003 | |
1004 | static void hmm_devmem_free(struct page *page, void *data) |
1005 | { |
1006 | struct hmm_devmem *devmem = data; |
1007 | |
1008 | page->mapping = NULL; |
1009 | |
1010 | devmem->ops->free(devmem, page); |
1011 | } |
1012 | |
1013 | /* |
1014 | * hmm_devmem_add() - hotplug ZONE_DEVICE memory for device memory |
1015 | * |
1016 | * @ops: memory event device driver callback (see struct hmm_devmem_ops) |
1017 | * @device: device struct to bind the resource too |
1018 | * @size: size in bytes of the device memory to add |
1019 | * Returns: pointer to new hmm_devmem struct ERR_PTR otherwise |
1020 | * |
1021 | * This function first finds an empty range of physical address big enough to |
1022 | * contain the new resource, and then hotplugs it as ZONE_DEVICE memory, which |
1023 | * in turn allocates struct pages. It does not do anything beyond that; all |
1024 | * events affecting the memory will go through the various callbacks provided |
1025 | * by hmm_devmem_ops struct. |
1026 | * |
1027 | * Device driver should call this function during device initialization and |
1028 | * is then responsible of memory management. HMM only provides helpers. |
1029 | */ |
1030 | struct hmm_devmem *hmm_devmem_add(const struct hmm_devmem_ops *ops, |
1031 | struct device *device, |
1032 | unsigned long size) |
1033 | { |
1034 | struct hmm_devmem *devmem; |
1035 | resource_size_t addr; |
1036 | void *result; |
1037 | int ret; |
1038 | |
1039 | dev_pagemap_get_ops(); |
1040 | |
1041 | devmem = devm_kzalloc(device, sizeof(*devmem), GFP_KERNEL); |
1042 | if (!devmem) |
1043 | return ERR_PTR(-ENOMEM); |
1044 | |
1045 | init_completion(&devmem->completion); |
1046 | devmem->pfn_first = -1UL; |
1047 | devmem->pfn_last = -1UL; |
1048 | devmem->resource = NULL; |
1049 | devmem->device = device; |
1050 | devmem->ops = ops; |
1051 | |
1052 | ret = percpu_ref_init(&devmem->ref, &hmm_devmem_ref_release, |
1053 | 0, GFP_KERNEL); |
1054 | if (ret) |
1055 | return ERR_PTR(ret); |
1056 | |
1057 | ret = devm_add_action_or_reset(device, hmm_devmem_ref_exit, &devmem->ref); |
1058 | if (ret) |
1059 | return ERR_PTR(ret); |
1060 | |
1061 | size = ALIGN(size, PA_SECTION_SIZE); |
1062 | addr = min((unsigned long)iomem_resource.end, |
1063 | (1UL << MAX_PHYSMEM_BITS) - 1); |
1064 | addr = addr - size + 1UL; |
1065 | |
1066 | /* |
1067 | * FIXME add a new helper to quickly walk resource tree and find free |
1068 | * range |
1069 | * |
1070 | * FIXME what about ioport_resource resource ? |
1071 | */ |
1072 | for (; addr > size && addr >= iomem_resource.start; addr -= size) { |
1073 | ret = region_intersects(addr, size, 0, IORES_DESC_NONE); |
1074 | if (ret != REGION_DISJOINT) |
1075 | continue; |
1076 | |
1077 | devmem->resource = devm_request_mem_region(device, addr, size, |
1078 | dev_name(device)); |
1079 | if (!devmem->resource) |
1080 | return ERR_PTR(-ENOMEM); |
1081 | break; |
1082 | } |
1083 | if (!devmem->resource) |
1084 | return ERR_PTR(-ERANGE); |
1085 | |
1086 | devmem->resource->desc = IORES_DESC_DEVICE_PRIVATE_MEMORY; |
1087 | devmem->pfn_first = devmem->resource->start >> PAGE_SHIFT; |
1088 | devmem->pfn_last = devmem->pfn_first + |
1089 | (resource_size(devmem->resource) >> PAGE_SHIFT); |
1090 | devmem->page_fault = hmm_devmem_fault; |
1091 | |
1092 | devmem->pagemap.type = MEMORY_DEVICE_PRIVATE; |
1093 | devmem->pagemap.res = *devmem->resource; |
1094 | devmem->pagemap.page_free = hmm_devmem_free; |
1095 | devmem->pagemap.altmap_valid = false; |
1096 | devmem->pagemap.ref = &devmem->ref; |
1097 | devmem->pagemap.data = devmem; |
1098 | devmem->pagemap.kill = hmm_devmem_ref_kill; |
1099 | |
1100 | result = devm_memremap_pages(devmem->device, &devmem->pagemap); |
1101 | if (IS_ERR(result)) |
1102 | return result; |
1103 | return devmem; |
1104 | } |
1105 | EXPORT_SYMBOL_GPL(hmm_devmem_add); |
1106 | |
1107 | struct hmm_devmem *hmm_devmem_add_resource(const struct hmm_devmem_ops *ops, |
1108 | struct device *device, |
1109 | struct resource *res) |
1110 | { |
1111 | struct hmm_devmem *devmem; |
1112 | void *result; |
1113 | int ret; |
1114 | |
1115 | if (res->desc != IORES_DESC_DEVICE_PUBLIC_MEMORY) |
1116 | return ERR_PTR(-EINVAL); |
1117 | |
1118 | dev_pagemap_get_ops(); |
1119 | |
1120 | devmem = devm_kzalloc(device, sizeof(*devmem), GFP_KERNEL); |
1121 | if (!devmem) |
1122 | return ERR_PTR(-ENOMEM); |
1123 | |
1124 | init_completion(&devmem->completion); |
1125 | devmem->pfn_first = -1UL; |
1126 | devmem->pfn_last = -1UL; |
1127 | devmem->resource = res; |
1128 | devmem->device = device; |
1129 | devmem->ops = ops; |
1130 | |
1131 | ret = percpu_ref_init(&devmem->ref, &hmm_devmem_ref_release, |
1132 | 0, GFP_KERNEL); |
1133 | if (ret) |
1134 | return ERR_PTR(ret); |
1135 | |
1136 | ret = devm_add_action_or_reset(device, hmm_devmem_ref_exit, |
1137 | &devmem->ref); |
1138 | if (ret) |
1139 | return ERR_PTR(ret); |
1140 | |
1141 | devmem->pfn_first = devmem->resource->start >> PAGE_SHIFT; |
1142 | devmem->pfn_last = devmem->pfn_first + |
1143 | (resource_size(devmem->resource) >> PAGE_SHIFT); |
1144 | devmem->page_fault = hmm_devmem_fault; |
1145 | |
1146 | devmem->pagemap.type = MEMORY_DEVICE_PUBLIC; |
1147 | devmem->pagemap.res = *devmem->resource; |
1148 | devmem->pagemap.page_free = hmm_devmem_free; |
1149 | devmem->pagemap.altmap_valid = false; |
1150 | devmem->pagemap.ref = &devmem->ref; |
1151 | devmem->pagemap.data = devmem; |
1152 | devmem->pagemap.kill = hmm_devmem_ref_kill; |
1153 | |
1154 | result = devm_memremap_pages(devmem->device, &devmem->pagemap); |
1155 | if (IS_ERR(result)) |
1156 | return result; |
1157 | return devmem; |
1158 | } |
1159 | EXPORT_SYMBOL_GPL(hmm_devmem_add_resource); |
1160 | |
1161 | /* |
1162 | * A device driver that wants to handle multiple devices memory through a |
1163 | * single fake device can use hmm_device to do so. This is purely a helper |
1164 | * and it is not needed to make use of any HMM functionality. |
1165 | */ |
1166 | #define HMM_DEVICE_MAX 256 |
1167 | |
1168 | static DECLARE_BITMAP(hmm_device_mask, HMM_DEVICE_MAX); |
1169 | static DEFINE_SPINLOCK(hmm_device_lock); |
1170 | static struct class *hmm_device_class; |
1171 | static dev_t hmm_device_devt; |
1172 | |
1173 | static void hmm_device_release(struct device *device) |
1174 | { |
1175 | struct hmm_device *hmm_device; |
1176 | |
1177 | hmm_device = container_of(device, struct hmm_device, device); |
1178 | spin_lock(&hmm_device_lock); |
1179 | clear_bit(hmm_device->minor, hmm_device_mask); |
1180 | spin_unlock(&hmm_device_lock); |
1181 | |
1182 | kfree(hmm_device); |
1183 | } |
1184 | |
1185 | struct hmm_device *hmm_device_new(void *drvdata) |
1186 | { |
1187 | struct hmm_device *hmm_device; |
1188 | |
1189 | hmm_device = kzalloc(sizeof(*hmm_device), GFP_KERNEL); |
1190 | if (!hmm_device) |
1191 | return ERR_PTR(-ENOMEM); |
1192 | |
1193 | spin_lock(&hmm_device_lock); |
1194 | hmm_device->minor = find_first_zero_bit(hmm_device_mask, HMM_DEVICE_MAX); |
1195 | if (hmm_device->minor >= HMM_DEVICE_MAX) { |
1196 | spin_unlock(&hmm_device_lock); |
1197 | kfree(hmm_device); |
1198 | return ERR_PTR(-EBUSY); |
1199 | } |
1200 | set_bit(hmm_device->minor, hmm_device_mask); |
1201 | spin_unlock(&hmm_device_lock); |
1202 | |
1203 | dev_set_name(&hmm_device->device, "hmm_device%d" , hmm_device->minor); |
1204 | hmm_device->device.devt = MKDEV(MAJOR(hmm_device_devt), |
1205 | hmm_device->minor); |
1206 | hmm_device->device.release = hmm_device_release; |
1207 | dev_set_drvdata(&hmm_device->device, drvdata); |
1208 | hmm_device->device.class = hmm_device_class; |
1209 | device_initialize(&hmm_device->device); |
1210 | |
1211 | return hmm_device; |
1212 | } |
1213 | EXPORT_SYMBOL(hmm_device_new); |
1214 | |
1215 | void hmm_device_put(struct hmm_device *hmm_device) |
1216 | { |
1217 | put_device(&hmm_device->device); |
1218 | } |
1219 | EXPORT_SYMBOL(hmm_device_put); |
1220 | |
1221 | static int __init hmm_init(void) |
1222 | { |
1223 | int ret; |
1224 | |
1225 | ret = alloc_chrdev_region(&hmm_device_devt, 0, |
1226 | HMM_DEVICE_MAX, |
1227 | "hmm_device" ); |
1228 | if (ret) |
1229 | return ret; |
1230 | |
1231 | hmm_device_class = class_create(THIS_MODULE, "hmm_device" ); |
1232 | if (IS_ERR(hmm_device_class)) { |
1233 | unregister_chrdev_region(hmm_device_devt, HMM_DEVICE_MAX); |
1234 | return PTR_ERR(hmm_device_class); |
1235 | } |
1236 | return 0; |
1237 | } |
1238 | |
1239 | device_initcall(hmm_init); |
1240 | #endif /* CONFIG_DEVICE_PRIVATE || CONFIG_DEVICE_PUBLIC */ |
1241 | |