1 | // SPDX-License-Identifier: GPL-2.0 |
2 | |
3 | /* |
4 | * Copyright (c) 2021, Google LLC. |
5 | * Pasha Tatashin <pasha.tatashin@soleen.com> |
6 | */ |
7 | #include <linux/kstrtox.h> |
8 | #include <linux/mm.h> |
9 | #include <linux/page_table_check.h> |
10 | |
11 | #undef pr_fmt |
12 | #define pr_fmt(fmt) "page_table_check: " fmt |
13 | |
14 | struct page_table_check { |
15 | atomic_t anon_map_count; |
16 | atomic_t file_map_count; |
17 | }; |
18 | |
19 | static bool __page_table_check_enabled __initdata = |
20 | IS_ENABLED(CONFIG_PAGE_TABLE_CHECK_ENFORCED); |
21 | |
22 | DEFINE_STATIC_KEY_TRUE(page_table_check_disabled); |
23 | EXPORT_SYMBOL(page_table_check_disabled); |
24 | |
25 | static int __init early_page_table_check_param(char *buf) |
26 | { |
27 | return kstrtobool(s: buf, res: &__page_table_check_enabled); |
28 | } |
29 | |
30 | early_param("page_table_check" , early_page_table_check_param); |
31 | |
32 | static bool __init need_page_table_check(void) |
33 | { |
34 | return __page_table_check_enabled; |
35 | } |
36 | |
37 | static void __init init_page_table_check(void) |
38 | { |
39 | if (!__page_table_check_enabled) |
40 | return; |
41 | static_branch_disable(&page_table_check_disabled); |
42 | } |
43 | |
44 | struct page_ext_operations page_table_check_ops = { |
45 | .size = sizeof(struct page_table_check), |
46 | .need = need_page_table_check, |
47 | .init = init_page_table_check, |
48 | .need_shared_flags = false, |
49 | }; |
50 | |
51 | static struct page_table_check *get_page_table_check(struct page_ext *page_ext) |
52 | { |
53 | BUG_ON(!page_ext); |
54 | return page_ext_data(page_ext, ops: &page_table_check_ops); |
55 | } |
56 | |
57 | /* |
58 | * An entry is removed from the page table, decrement the counters for that page |
59 | * verify that it is of correct type and counters do not become negative. |
60 | */ |
61 | static void page_table_check_clear(unsigned long pfn, unsigned long pgcnt) |
62 | { |
63 | struct page_ext *page_ext; |
64 | struct page *page; |
65 | unsigned long i; |
66 | bool anon; |
67 | |
68 | if (!pfn_valid(pfn)) |
69 | return; |
70 | |
71 | page = pfn_to_page(pfn); |
72 | page_ext = page_ext_get(page); |
73 | |
74 | BUG_ON(PageSlab(page)); |
75 | anon = PageAnon(page); |
76 | |
77 | for (i = 0; i < pgcnt; i++) { |
78 | struct page_table_check *ptc = get_page_table_check(page_ext); |
79 | |
80 | if (anon) { |
81 | BUG_ON(atomic_read(&ptc->file_map_count)); |
82 | BUG_ON(atomic_dec_return(&ptc->anon_map_count) < 0); |
83 | } else { |
84 | BUG_ON(atomic_read(&ptc->anon_map_count)); |
85 | BUG_ON(atomic_dec_return(&ptc->file_map_count) < 0); |
86 | } |
87 | page_ext = page_ext_next(curr: page_ext); |
88 | } |
89 | page_ext_put(page_ext); |
90 | } |
91 | |
92 | /* |
93 | * A new entry is added to the page table, increment the counters for that page |
94 | * verify that it is of correct type and is not being mapped with a different |
95 | * type to a different process. |
96 | */ |
97 | static void page_table_check_set(unsigned long pfn, unsigned long pgcnt, |
98 | bool rw) |
99 | { |
100 | struct page_ext *page_ext; |
101 | struct page *page; |
102 | unsigned long i; |
103 | bool anon; |
104 | |
105 | if (!pfn_valid(pfn)) |
106 | return; |
107 | |
108 | page = pfn_to_page(pfn); |
109 | page_ext = page_ext_get(page); |
110 | |
111 | BUG_ON(PageSlab(page)); |
112 | anon = PageAnon(page); |
113 | |
114 | for (i = 0; i < pgcnt; i++) { |
115 | struct page_table_check *ptc = get_page_table_check(page_ext); |
116 | |
117 | if (anon) { |
118 | BUG_ON(atomic_read(&ptc->file_map_count)); |
119 | BUG_ON(atomic_inc_return(&ptc->anon_map_count) > 1 && rw); |
120 | } else { |
121 | BUG_ON(atomic_read(&ptc->anon_map_count)); |
122 | BUG_ON(atomic_inc_return(&ptc->file_map_count) < 0); |
123 | } |
124 | page_ext = page_ext_next(curr: page_ext); |
125 | } |
126 | page_ext_put(page_ext); |
127 | } |
128 | |
129 | /* |
130 | * page is on free list, or is being allocated, verify that counters are zeroes |
131 | * crash if they are not. |
132 | */ |
133 | void __page_table_check_zero(struct page *page, unsigned int order) |
134 | { |
135 | struct page_ext *page_ext; |
136 | unsigned long i; |
137 | |
138 | BUG_ON(PageSlab(page)); |
139 | |
140 | page_ext = page_ext_get(page); |
141 | BUG_ON(!page_ext); |
142 | for (i = 0; i < (1ul << order); i++) { |
143 | struct page_table_check *ptc = get_page_table_check(page_ext); |
144 | |
145 | BUG_ON(atomic_read(&ptc->anon_map_count)); |
146 | BUG_ON(atomic_read(&ptc->file_map_count)); |
147 | page_ext = page_ext_next(curr: page_ext); |
148 | } |
149 | page_ext_put(page_ext); |
150 | } |
151 | |
152 | void __page_table_check_pte_clear(struct mm_struct *mm, pte_t pte) |
153 | { |
154 | if (&init_mm == mm) |
155 | return; |
156 | |
157 | if (pte_user_accessible_page(pte)) { |
158 | page_table_check_clear(pfn: pte_pfn(pte), PAGE_SIZE >> PAGE_SHIFT); |
159 | } |
160 | } |
161 | EXPORT_SYMBOL(__page_table_check_pte_clear); |
162 | |
163 | void __page_table_check_pmd_clear(struct mm_struct *mm, pmd_t pmd) |
164 | { |
165 | if (&init_mm == mm) |
166 | return; |
167 | |
168 | if (pmd_user_accessible_page(pmd)) { |
169 | page_table_check_clear(pfn: pmd_pfn(pmd), PMD_SIZE >> PAGE_SHIFT); |
170 | } |
171 | } |
172 | EXPORT_SYMBOL(__page_table_check_pmd_clear); |
173 | |
174 | void __page_table_check_pud_clear(struct mm_struct *mm, pud_t pud) |
175 | { |
176 | if (&init_mm == mm) |
177 | return; |
178 | |
179 | if (pud_user_accessible_page(pud)) { |
180 | page_table_check_clear(pfn: pud_pfn(pud), PUD_SIZE >> PAGE_SHIFT); |
181 | } |
182 | } |
183 | EXPORT_SYMBOL(__page_table_check_pud_clear); |
184 | |
185 | void __page_table_check_ptes_set(struct mm_struct *mm, pte_t *ptep, pte_t pte, |
186 | unsigned int nr) |
187 | { |
188 | unsigned int i; |
189 | |
190 | if (&init_mm == mm) |
191 | return; |
192 | |
193 | for (i = 0; i < nr; i++) |
194 | __page_table_check_pte_clear(mm, ptep_get(ptep: ptep + i)); |
195 | if (pte_user_accessible_page(pte)) |
196 | page_table_check_set(pfn: pte_pfn(pte), pgcnt: nr, rw: pte_write(pte)); |
197 | } |
198 | EXPORT_SYMBOL(__page_table_check_ptes_set); |
199 | |
200 | void __page_table_check_pmd_set(struct mm_struct *mm, pmd_t *pmdp, pmd_t pmd) |
201 | { |
202 | if (&init_mm == mm) |
203 | return; |
204 | |
205 | __page_table_check_pmd_clear(mm, *pmdp); |
206 | if (pmd_user_accessible_page(pmd)) { |
207 | page_table_check_set(pfn: pmd_pfn(pmd), PMD_SIZE >> PAGE_SHIFT, |
208 | pmd_write(pmd)); |
209 | } |
210 | } |
211 | EXPORT_SYMBOL(__page_table_check_pmd_set); |
212 | |
213 | void __page_table_check_pud_set(struct mm_struct *mm, pud_t *pudp, pud_t pud) |
214 | { |
215 | if (&init_mm == mm) |
216 | return; |
217 | |
218 | __page_table_check_pud_clear(mm, *pudp); |
219 | if (pud_user_accessible_page(pud)) { |
220 | page_table_check_set(pfn: pud_pfn(pud), PUD_SIZE >> PAGE_SHIFT, |
221 | pud_write(pud)); |
222 | } |
223 | } |
224 | EXPORT_SYMBOL(__page_table_check_pud_set); |
225 | |
226 | void __page_table_check_pte_clear_range(struct mm_struct *mm, |
227 | unsigned long addr, |
228 | pmd_t pmd) |
229 | { |
230 | if (&init_mm == mm) |
231 | return; |
232 | |
233 | if (!pmd_bad(pmd) && !pmd_leaf(pte: pmd)) { |
234 | pte_t *ptep = pte_offset_map(pmd: &pmd, addr); |
235 | unsigned long i; |
236 | |
237 | if (WARN_ON(!ptep)) |
238 | return; |
239 | for (i = 0; i < PTRS_PER_PTE; i++) { |
240 | __page_table_check_pte_clear(mm, ptep_get(ptep)); |
241 | addr += PAGE_SIZE; |
242 | ptep++; |
243 | } |
244 | pte_unmap(pte: ptep - PTRS_PER_PTE); |
245 | } |
246 | } |
247 | |