1#include <linux/gfp.h>
2#include <linux/highmem.h>
3#include <linux/kernel.h>
4#include <linux/mmdebug.h>
5#include <linux/mm_types.h>
6#include <linux/pagemap.h>
7#include <linux/rcupdate.h>
8#include <linux/smp.h>
9#include <linux/swap.h>
10
11#include <asm/pgalloc.h>
12#include <asm/tlb.h>
13
14#ifdef HAVE_GENERIC_MMU_GATHER
15
16static bool tlb_next_batch(struct mmu_gather *tlb)
17{
18 struct mmu_gather_batch *batch;
19
20 batch = tlb->active;
21 if (batch->next) {
22 tlb->active = batch->next;
23 return true;
24 }
25
26 if (tlb->batch_count == MAX_GATHER_BATCH_COUNT)
27 return false;
28
29 batch = (void *)__get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0);
30 if (!batch)
31 return false;
32
33 tlb->batch_count++;
34 batch->next = NULL;
35 batch->nr = 0;
36 batch->max = MAX_GATHER_BATCH;
37
38 tlb->active->next = batch;
39 tlb->active = batch;
40
41 return true;
42}
43
44void arch_tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
45 unsigned long start, unsigned long end)
46{
47 tlb->mm = mm;
48
49 /* Is it from 0 to ~0? */
50 tlb->fullmm = !(start | (end+1));
51 tlb->need_flush_all = 0;
52 tlb->local.next = NULL;
53 tlb->local.nr = 0;
54 tlb->local.max = ARRAY_SIZE(tlb->__pages);
55 tlb->active = &tlb->local;
56 tlb->batch_count = 0;
57
58#ifdef CONFIG_HAVE_RCU_TABLE_FREE
59 tlb->batch = NULL;
60#endif
61 tlb->page_size = 0;
62
63 __tlb_reset_range(tlb);
64}
65
66void tlb_flush_mmu_free(struct mmu_gather *tlb)
67{
68 struct mmu_gather_batch *batch;
69
70#ifdef CONFIG_HAVE_RCU_TABLE_FREE
71 tlb_table_flush(tlb);
72#endif
73 for (batch = &tlb->local; batch && batch->nr; batch = batch->next) {
74 free_pages_and_swap_cache(batch->pages, batch->nr);
75 batch->nr = 0;
76 }
77 tlb->active = &tlb->local;
78}
79
80void tlb_flush_mmu(struct mmu_gather *tlb)
81{
82 tlb_flush_mmu_tlbonly(tlb);
83 tlb_flush_mmu_free(tlb);
84}
85
86/* tlb_finish_mmu
87 * Called at the end of the shootdown operation to free up any resources
88 * that were required.
89 */
90void arch_tlb_finish_mmu(struct mmu_gather *tlb,
91 unsigned long start, unsigned long end, bool force)
92{
93 struct mmu_gather_batch *batch, *next;
94
95 if (force) {
96 __tlb_reset_range(tlb);
97 __tlb_adjust_range(tlb, start, end - start);
98 }
99
100 tlb_flush_mmu(tlb);
101
102 /* keep the page table cache within bounds */
103 check_pgt_cache();
104
105 for (batch = tlb->local.next; batch; batch = next) {
106 next = batch->next;
107 free_pages((unsigned long)batch, 0);
108 }
109 tlb->local.next = NULL;
110}
111
112/* __tlb_remove_page
113 * Must perform the equivalent to __free_pte(pte_get_and_clear(ptep)), while
114 * handling the additional races in SMP caused by other CPUs caching valid
115 * mappings in their TLBs. Returns the number of free page slots left.
116 * When out of page slots we must call tlb_flush_mmu().
117 *returns true if the caller should flush.
118 */
119bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_size)
120{
121 struct mmu_gather_batch *batch;
122
123 VM_BUG_ON(!tlb->end);
124 VM_WARN_ON(tlb->page_size != page_size);
125
126 batch = tlb->active;
127 /*
128 * Add the page and check if we are full. If so
129 * force a flush.
130 */
131 batch->pages[batch->nr++] = page;
132 if (batch->nr == batch->max) {
133 if (!tlb_next_batch(tlb))
134 return true;
135 batch = tlb->active;
136 }
137 VM_BUG_ON_PAGE(batch->nr > batch->max, page);
138
139 return false;
140}
141
142#endif /* HAVE_GENERIC_MMU_GATHER */
143
144#ifdef CONFIG_HAVE_RCU_TABLE_FREE
145
146/*
147 * See the comment near struct mmu_table_batch.
148 */
149
150/*
151 * If we want tlb_remove_table() to imply TLB invalidates.
152 */
153static inline void tlb_table_invalidate(struct mmu_gather *tlb)
154{
155#ifdef CONFIG_HAVE_RCU_TABLE_INVALIDATE
156 /*
157 * Invalidate page-table caches used by hardware walkers. Then we still
158 * need to RCU-sched wait while freeing the pages because software
159 * walkers can still be in-flight.
160 */
161 tlb_flush_mmu_tlbonly(tlb);
162#endif
163}
164
165static void tlb_remove_table_smp_sync(void *arg)
166{
167 /* Simply deliver the interrupt */
168}
169
170static void tlb_remove_table_one(void *table)
171{
172 /*
173 * This isn't an RCU grace period and hence the page-tables cannot be
174 * assumed to be actually RCU-freed.
175 *
176 * It is however sufficient for software page-table walkers that rely on
177 * IRQ disabling. See the comment near struct mmu_table_batch.
178 */
179 smp_call_function(tlb_remove_table_smp_sync, NULL, 1);
180 __tlb_remove_table(table);
181}
182
183static void tlb_remove_table_rcu(struct rcu_head *head)
184{
185 struct mmu_table_batch *batch;
186 int i;
187
188 batch = container_of(head, struct mmu_table_batch, rcu);
189
190 for (i = 0; i < batch->nr; i++)
191 __tlb_remove_table(batch->tables[i]);
192
193 free_page((unsigned long)batch);
194}
195
196void tlb_table_flush(struct mmu_gather *tlb)
197{
198 struct mmu_table_batch **batch = &tlb->batch;
199
200 if (*batch) {
201 tlb_table_invalidate(tlb);
202 call_rcu(&(*batch)->rcu, tlb_remove_table_rcu);
203 *batch = NULL;
204 }
205}
206
207void tlb_remove_table(struct mmu_gather *tlb, void *table)
208{
209 struct mmu_table_batch **batch = &tlb->batch;
210
211 if (*batch == NULL) {
212 *batch = (struct mmu_table_batch *)__get_free_page(GFP_NOWAIT | __GFP_NOWARN);
213 if (*batch == NULL) {
214 tlb_table_invalidate(tlb);
215 tlb_remove_table_one(table);
216 return;
217 }
218 (*batch)->nr = 0;
219 }
220
221 (*batch)->tables[(*batch)->nr++] = table;
222 if ((*batch)->nr == MAX_TABLE_BATCH)
223 tlb_table_flush(tlb);
224}
225
226#endif /* CONFIG_HAVE_RCU_TABLE_FREE */
227
228/**
229 * tlb_gather_mmu - initialize an mmu_gather structure for page-table tear-down
230 * @tlb: the mmu_gather structure to initialize
231 * @mm: the mm_struct of the target address space
232 * @start: start of the region that will be removed from the page-table
233 * @end: end of the region that will be removed from the page-table
234 *
235 * Called to initialize an (on-stack) mmu_gather structure for page-table
236 * tear-down from @mm. The @start and @end are set to 0 and -1
237 * respectively when @mm is without users and we're going to destroy
238 * the full address space (exit/execve).
239 */
240void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
241 unsigned long start, unsigned long end)
242{
243 arch_tlb_gather_mmu(tlb, mm, start, end);
244 inc_tlb_flush_pending(tlb->mm);
245}
246
247void tlb_finish_mmu(struct mmu_gather *tlb,
248 unsigned long start, unsigned long end)
249{
250 /*
251 * If there are parallel threads are doing PTE changes on same range
252 * under non-exclusive lock(e.g., mmap_sem read-side) but defer TLB
253 * flush by batching, a thread has stable TLB entry can fail to flush
254 * the TLB by observing pte_none|!pte_dirty, for example so flush TLB
255 * forcefully if we detect parallel PTE batching threads.
256 */
257 bool force = mm_tlb_flush_nested(tlb->mm);
258
259 arch_tlb_finish_mmu(tlb, start, end, force);
260 dec_tlb_flush_pending(tlb->mm);
261}
262