1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _LINUX_MMU_NOTIFIER_H
3#define _LINUX_MMU_NOTIFIER_H
4
5#include <linux/types.h>
6#include <linux/list.h>
7#include <linux/spinlock.h>
8#include <linux/mm_types.h>
9#include <linux/srcu.h>
10
11struct mmu_notifier;
12struct mmu_notifier_ops;
13
14/* mmu_notifier_ops flags */
15#define MMU_INVALIDATE_DOES_NOT_BLOCK (0x01)
16
17#ifdef CONFIG_MMU_NOTIFIER
18
19/*
20 * The mmu notifier_mm structure is allocated and installed in
21 * mm->mmu_notifier_mm inside the mm_take_all_locks() protected
22 * critical section and it's released only when mm_count reaches zero
23 * in mmdrop().
24 */
25struct mmu_notifier_mm {
26 /* all mmu notifiers registerd in this mm are queued in this list */
27 struct hlist_head list;
28 /* to serialize the list modifications and hlist_unhashed */
29 spinlock_t lock;
30};
31
32struct mmu_notifier_ops {
33 /*
34 * Flags to specify behavior of callbacks for this MMU notifier.
35 * Used to determine which context an operation may be called.
36 *
37 * MMU_INVALIDATE_DOES_NOT_BLOCK: invalidate_range_* callbacks do not
38 * block
39 */
40 int flags;
41
42 /*
43 * Called either by mmu_notifier_unregister or when the mm is
44 * being destroyed by exit_mmap, always before all pages are
45 * freed. This can run concurrently with other mmu notifier
46 * methods (the ones invoked outside the mm context) and it
47 * should tear down all secondary mmu mappings and freeze the
48 * secondary mmu. If this method isn't implemented you've to
49 * be sure that nothing could possibly write to the pages
50 * through the secondary mmu by the time the last thread with
51 * tsk->mm == mm exits.
52 *
53 * As side note: the pages freed after ->release returns could
54 * be immediately reallocated by the gart at an alias physical
55 * address with a different cache model, so if ->release isn't
56 * implemented because all _software_ driven memory accesses
57 * through the secondary mmu are terminated by the time the
58 * last thread of this mm quits, you've also to be sure that
59 * speculative _hardware_ operations can't allocate dirty
60 * cachelines in the cpu that could not be snooped and made
61 * coherent with the other read and write operations happening
62 * through the gart alias address, so leading to memory
63 * corruption.
64 */
65 void (*release)(struct mmu_notifier *mn,
66 struct mm_struct *mm);
67
68 /*
69 * clear_flush_young is called after the VM is
70 * test-and-clearing the young/accessed bitflag in the
71 * pte. This way the VM will provide proper aging to the
72 * accesses to the page through the secondary MMUs and not
73 * only to the ones through the Linux pte.
74 * Start-end is necessary in case the secondary MMU is mapping the page
75 * at a smaller granularity than the primary MMU.
76 */
77 int (*clear_flush_young)(struct mmu_notifier *mn,
78 struct mm_struct *mm,
79 unsigned long start,
80 unsigned long end);
81
82 /*
83 * clear_young is a lightweight version of clear_flush_young. Like the
84 * latter, it is supposed to test-and-clear the young/accessed bitflag
85 * in the secondary pte, but it may omit flushing the secondary tlb.
86 */
87 int (*clear_young)(struct mmu_notifier *mn,
88 struct mm_struct *mm,
89 unsigned long start,
90 unsigned long end);
91
92 /*
93 * test_young is called to check the young/accessed bitflag in
94 * the secondary pte. This is used to know if the page is
95 * frequently used without actually clearing the flag or tearing
96 * down the secondary mapping on the page.
97 */
98 int (*test_young)(struct mmu_notifier *mn,
99 struct mm_struct *mm,
100 unsigned long address);
101
102 /*
103 * change_pte is called in cases that pte mapping to page is changed:
104 * for example, when ksm remaps pte to point to a new shared page.
105 */
106 void (*change_pte)(struct mmu_notifier *mn,
107 struct mm_struct *mm,
108 unsigned long address,
109 pte_t pte);
110
111 /*
112 * invalidate_range_start() and invalidate_range_end() must be
113 * paired and are called only when the mmap_sem and/or the
114 * locks protecting the reverse maps are held. If the subsystem
115 * can't guarantee that no additional references are taken to
116 * the pages in the range, it has to implement the
117 * invalidate_range() notifier to remove any references taken
118 * after invalidate_range_start().
119 *
120 * Invalidation of multiple concurrent ranges may be
121 * optionally permitted by the driver. Either way the
122 * establishment of sptes is forbidden in the range passed to
123 * invalidate_range_begin/end for the whole duration of the
124 * invalidate_range_begin/end critical section.
125 *
126 * invalidate_range_start() is called when all pages in the
127 * range are still mapped and have at least a refcount of one.
128 *
129 * invalidate_range_end() is called when all pages in the
130 * range have been unmapped and the pages have been freed by
131 * the VM.
132 *
133 * The VM will remove the page table entries and potentially
134 * the page between invalidate_range_start() and
135 * invalidate_range_end(). If the page must not be freed
136 * because of pending I/O or other circumstances then the
137 * invalidate_range_start() callback (or the initial mapping
138 * by the driver) must make sure that the refcount is kept
139 * elevated.
140 *
141 * If the driver increases the refcount when the pages are
142 * initially mapped into an address space then either
143 * invalidate_range_start() or invalidate_range_end() may
144 * decrease the refcount. If the refcount is decreased on
145 * invalidate_range_start() then the VM can free pages as page
146 * table entries are removed. If the refcount is only
147 * droppped on invalidate_range_end() then the driver itself
148 * will drop the last refcount but it must take care to flush
149 * any secondary tlb before doing the final free on the
150 * page. Pages will no longer be referenced by the linux
151 * address space but may still be referenced by sptes until
152 * the last refcount is dropped.
153 *
154 * If both of these callbacks cannot block, and invalidate_range
155 * cannot block, mmu_notifier_ops.flags should have
156 * MMU_INVALIDATE_DOES_NOT_BLOCK set.
157 */
158 void (*invalidate_range_start)(struct mmu_notifier *mn,
159 struct mm_struct *mm,
160 unsigned long start, unsigned long end);
161 void (*invalidate_range_end)(struct mmu_notifier *mn,
162 struct mm_struct *mm,
163 unsigned long start, unsigned long end);
164
165 /*
166 * invalidate_range() is either called between
167 * invalidate_range_start() and invalidate_range_end() when the
168 * VM has to free pages that where unmapped, but before the
169 * pages are actually freed, or outside of _start()/_end() when
170 * a (remote) TLB is necessary.
171 *
172 * If invalidate_range() is used to manage a non-CPU TLB with
173 * shared page-tables, it not necessary to implement the
174 * invalidate_range_start()/end() notifiers, as
175 * invalidate_range() alread catches the points in time when an
176 * external TLB range needs to be flushed. For more in depth
177 * discussion on this see Documentation/vm/mmu_notifier.rst
178 *
179 * Note that this function might be called with just a sub-range
180 * of what was passed to invalidate_range_start()/end(), if
181 * called between those functions.
182 *
183 * If this callback cannot block, and invalidate_range_{start,end}
184 * cannot block, mmu_notifier_ops.flags should have
185 * MMU_INVALIDATE_DOES_NOT_BLOCK set.
186 */
187 void (*invalidate_range)(struct mmu_notifier *mn, struct mm_struct *mm,
188 unsigned long start, unsigned long end);
189};
190
191/*
192 * The notifier chains are protected by mmap_sem and/or the reverse map
193 * semaphores. Notifier chains are only changed when all reverse maps and
194 * the mmap_sem locks are taken.
195 *
196 * Therefore notifier chains can only be traversed when either
197 *
198 * 1. mmap_sem is held.
199 * 2. One of the reverse map locks is held (i_mmap_rwsem or anon_vma->rwsem).
200 * 3. No other concurrent thread can access the list (release)
201 */
202struct mmu_notifier {
203 struct hlist_node hlist;
204 const struct mmu_notifier_ops *ops;
205};
206
207static inline int mm_has_notifiers(struct mm_struct *mm)
208{
209 return unlikely(mm->mmu_notifier_mm);
210}
211
212extern int mmu_notifier_register(struct mmu_notifier *mn,
213 struct mm_struct *mm);
214extern int __mmu_notifier_register(struct mmu_notifier *mn,
215 struct mm_struct *mm);
216extern void mmu_notifier_unregister(struct mmu_notifier *mn,
217 struct mm_struct *mm);
218extern void mmu_notifier_unregister_no_release(struct mmu_notifier *mn,
219 struct mm_struct *mm);
220extern void __mmu_notifier_mm_destroy(struct mm_struct *mm);
221extern void __mmu_notifier_release(struct mm_struct *mm);
222extern int __mmu_notifier_clear_flush_young(struct mm_struct *mm,
223 unsigned long start,
224 unsigned long end);
225extern int __mmu_notifier_clear_young(struct mm_struct *mm,
226 unsigned long start,
227 unsigned long end);
228extern int __mmu_notifier_test_young(struct mm_struct *mm,
229 unsigned long address);
230extern void __mmu_notifier_change_pte(struct mm_struct *mm,
231 unsigned long address, pte_t pte);
232extern void __mmu_notifier_invalidate_range_start(struct mm_struct *mm,
233 unsigned long start, unsigned long end);
234extern void __mmu_notifier_invalidate_range_end(struct mm_struct *mm,
235 unsigned long start, unsigned long end,
236 bool only_end);
237extern void __mmu_notifier_invalidate_range(struct mm_struct *mm,
238 unsigned long start, unsigned long end);
239extern bool mm_has_blockable_invalidate_notifiers(struct mm_struct *mm);
240
241static inline void mmu_notifier_release(struct mm_struct *mm)
242{
243 if (mm_has_notifiers(mm))
244 __mmu_notifier_release(mm);
245}
246
247static inline int mmu_notifier_clear_flush_young(struct mm_struct *mm,
248 unsigned long start,
249 unsigned long end)
250{
251 if (mm_has_notifiers(mm))
252 return __mmu_notifier_clear_flush_young(mm, start, end);
253 return 0;
254}
255
256static inline int mmu_notifier_clear_young(struct mm_struct *mm,
257 unsigned long start,
258 unsigned long end)
259{
260 if (mm_has_notifiers(mm))
261 return __mmu_notifier_clear_young(mm, start, end);
262 return 0;
263}
264
265static inline int mmu_notifier_test_young(struct mm_struct *mm,
266 unsigned long address)
267{
268 if (mm_has_notifiers(mm))
269 return __mmu_notifier_test_young(mm, address);
270 return 0;
271}
272
273static inline void mmu_notifier_change_pte(struct mm_struct *mm,
274 unsigned long address, pte_t pte)
275{
276 if (mm_has_notifiers(mm))
277 __mmu_notifier_change_pte(mm, address, pte);
278}
279
280static inline void mmu_notifier_invalidate_range_start(struct mm_struct *mm,
281 unsigned long start, unsigned long end)
282{
283 if (mm_has_notifiers(mm))
284 __mmu_notifier_invalidate_range_start(mm, start, end);
285}
286
287static inline void mmu_notifier_invalidate_range_end(struct mm_struct *mm,
288 unsigned long start, unsigned long end)
289{
290 if (mm_has_notifiers(mm))
291 __mmu_notifier_invalidate_range_end(mm, start, end, false);
292}
293
294static inline void mmu_notifier_invalidate_range_only_end(struct mm_struct *mm,
295 unsigned long start, unsigned long end)
296{
297 if (mm_has_notifiers(mm))
298 __mmu_notifier_invalidate_range_end(mm, start, end, true);
299}
300
301static inline void mmu_notifier_invalidate_range(struct mm_struct *mm,
302 unsigned long start, unsigned long end)
303{
304 if (mm_has_notifiers(mm))
305 __mmu_notifier_invalidate_range(mm, start, end);
306}
307
308static inline void mmu_notifier_mm_init(struct mm_struct *mm)
309{
310 mm->mmu_notifier_mm = NULL;
311}
312
313static inline void mmu_notifier_mm_destroy(struct mm_struct *mm)
314{
315 if (mm_has_notifiers(mm))
316 __mmu_notifier_mm_destroy(mm);
317}
318
319#define ptep_clear_flush_young_notify(__vma, __address, __ptep) \
320({ \
321 int __young; \
322 struct vm_area_struct *___vma = __vma; \
323 unsigned long ___address = __address; \
324 __young = ptep_clear_flush_young(___vma, ___address, __ptep); \
325 __young |= mmu_notifier_clear_flush_young(___vma->vm_mm, \
326 ___address, \
327 ___address + \
328 PAGE_SIZE); \
329 __young; \
330})
331
332#define pmdp_clear_flush_young_notify(__vma, __address, __pmdp) \
333({ \
334 int __young; \
335 struct vm_area_struct *___vma = __vma; \
336 unsigned long ___address = __address; \
337 __young = pmdp_clear_flush_young(___vma, ___address, __pmdp); \
338 __young |= mmu_notifier_clear_flush_young(___vma->vm_mm, \
339 ___address, \
340 ___address + \
341 PMD_SIZE); \
342 __young; \
343})
344
345#define ptep_clear_young_notify(__vma, __address, __ptep) \
346({ \
347 int __young; \
348 struct vm_area_struct *___vma = __vma; \
349 unsigned long ___address = __address; \
350 __young = ptep_test_and_clear_young(___vma, ___address, __ptep);\
351 __young |= mmu_notifier_clear_young(___vma->vm_mm, ___address, \
352 ___address + PAGE_SIZE); \
353 __young; \
354})
355
356#define pmdp_clear_young_notify(__vma, __address, __pmdp) \
357({ \
358 int __young; \
359 struct vm_area_struct *___vma = __vma; \
360 unsigned long ___address = __address; \
361 __young = pmdp_test_and_clear_young(___vma, ___address, __pmdp);\
362 __young |= mmu_notifier_clear_young(___vma->vm_mm, ___address, \
363 ___address + PMD_SIZE); \
364 __young; \
365})
366
367#define ptep_clear_flush_notify(__vma, __address, __ptep) \
368({ \
369 unsigned long ___addr = __address & PAGE_MASK; \
370 struct mm_struct *___mm = (__vma)->vm_mm; \
371 pte_t ___pte; \
372 \
373 ___pte = ptep_clear_flush(__vma, __address, __ptep); \
374 mmu_notifier_invalidate_range(___mm, ___addr, \
375 ___addr + PAGE_SIZE); \
376 \
377 ___pte; \
378})
379
380#define pmdp_huge_clear_flush_notify(__vma, __haddr, __pmd) \
381({ \
382 unsigned long ___haddr = __haddr & HPAGE_PMD_MASK; \
383 struct mm_struct *___mm = (__vma)->vm_mm; \
384 pmd_t ___pmd; \
385 \
386 ___pmd = pmdp_huge_clear_flush(__vma, __haddr, __pmd); \
387 mmu_notifier_invalidate_range(___mm, ___haddr, \
388 ___haddr + HPAGE_PMD_SIZE); \
389 \
390 ___pmd; \
391})
392
393#define pudp_huge_clear_flush_notify(__vma, __haddr, __pud) \
394({ \
395 unsigned long ___haddr = __haddr & HPAGE_PUD_MASK; \
396 struct mm_struct *___mm = (__vma)->vm_mm; \
397 pud_t ___pud; \
398 \
399 ___pud = pudp_huge_clear_flush(__vma, __haddr, __pud); \
400 mmu_notifier_invalidate_range(___mm, ___haddr, \
401 ___haddr + HPAGE_PUD_SIZE); \
402 \
403 ___pud; \
404})
405
406/*
407 * set_pte_at_notify() sets the pte _after_ running the notifier.
408 * This is safe to start by updating the secondary MMUs, because the primary MMU
409 * pte invalidate must have already happened with a ptep_clear_flush() before
410 * set_pte_at_notify() has been invoked. Updating the secondary MMUs first is
411 * required when we change both the protection of the mapping from read-only to
412 * read-write and the pfn (like during copy on write page faults). Otherwise the
413 * old page would remain mapped readonly in the secondary MMUs after the new
414 * page is already writable by some CPU through the primary MMU.
415 */
416#define set_pte_at_notify(__mm, __address, __ptep, __pte) \
417({ \
418 struct mm_struct *___mm = __mm; \
419 unsigned long ___address = __address; \
420 pte_t ___pte = __pte; \
421 \
422 mmu_notifier_change_pte(___mm, ___address, ___pte); \
423 set_pte_at(___mm, ___address, __ptep, ___pte); \
424})
425
426extern void mmu_notifier_call_srcu(struct rcu_head *rcu,
427 void (*func)(struct rcu_head *rcu));
428extern void mmu_notifier_synchronize(void);
429
430#else /* CONFIG_MMU_NOTIFIER */
431
432static inline int mm_has_notifiers(struct mm_struct *mm)
433{
434 return 0;
435}
436
437static inline void mmu_notifier_release(struct mm_struct *mm)
438{
439}
440
441static inline int mmu_notifier_clear_flush_young(struct mm_struct *mm,
442 unsigned long start,
443 unsigned long end)
444{
445 return 0;
446}
447
448static inline int mmu_notifier_test_young(struct mm_struct *mm,
449 unsigned long address)
450{
451 return 0;
452}
453
454static inline void mmu_notifier_change_pte(struct mm_struct *mm,
455 unsigned long address, pte_t pte)
456{
457}
458
459static inline void mmu_notifier_invalidate_range_start(struct mm_struct *mm,
460 unsigned long start, unsigned long end)
461{
462}
463
464static inline void mmu_notifier_invalidate_range_end(struct mm_struct *mm,
465 unsigned long start, unsigned long end)
466{
467}
468
469static inline void mmu_notifier_invalidate_range_only_end(struct mm_struct *mm,
470 unsigned long start, unsigned long end)
471{
472}
473
474static inline void mmu_notifier_invalidate_range(struct mm_struct *mm,
475 unsigned long start, unsigned long end)
476{
477}
478
479static inline bool mm_has_blockable_invalidate_notifiers(struct mm_struct *mm)
480{
481 return false;
482}
483
484static inline void mmu_notifier_mm_init(struct mm_struct *mm)
485{
486}
487
488static inline void mmu_notifier_mm_destroy(struct mm_struct *mm)
489{
490}
491
492#define ptep_clear_flush_young_notify ptep_clear_flush_young
493#define pmdp_clear_flush_young_notify pmdp_clear_flush_young
494#define ptep_clear_young_notify ptep_test_and_clear_young
495#define pmdp_clear_young_notify pmdp_test_and_clear_young
496#define ptep_clear_flush_notify ptep_clear_flush
497#define pmdp_huge_clear_flush_notify pmdp_huge_clear_flush
498#define pudp_huge_clear_flush_notify pudp_huge_clear_flush
499#define set_pte_at_notify set_pte_at
500
501#endif /* CONFIG_MMU_NOTIFIER */
502
503#endif /* _LINUX_MMU_NOTIFIER_H */
504