1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * mm/balloon_compaction.c |
4 | * |
5 | * Common interface for making balloon pages movable by compaction. |
6 | * |
7 | * Copyright (C) 2012, Red Hat, Inc. Rafael Aquini <aquini@redhat.com> |
8 | */ |
9 | #include <linux/mm.h> |
10 | #include <linux/slab.h> |
11 | #include <linux/export.h> |
12 | #include <linux/balloon_compaction.h> |
13 | |
14 | static void balloon_page_enqueue_one(struct balloon_dev_info *b_dev_info, |
15 | struct page *page) |
16 | { |
17 | /* |
18 | * Block others from accessing the 'page' when we get around to |
19 | * establishing additional references. We should be the only one |
20 | * holding a reference to the 'page' at this point. If we are not, then |
21 | * memory corruption is possible and we should stop execution. |
22 | */ |
23 | BUG_ON(!trylock_page(page)); |
24 | balloon_page_insert(balloon: b_dev_info, page); |
25 | unlock_page(page); |
26 | __count_vm_event(item: BALLOON_INFLATE); |
27 | } |
28 | |
29 | /** |
30 | * balloon_page_list_enqueue() - inserts a list of pages into the balloon page |
31 | * list. |
32 | * @b_dev_info: balloon device descriptor where we will insert a new page to |
33 | * @pages: pages to enqueue - allocated using balloon_page_alloc. |
34 | * |
35 | * Driver must call this function to properly enqueue balloon pages before |
36 | * definitively removing them from the guest system. |
37 | * |
38 | * Return: number of pages that were enqueued. |
39 | */ |
40 | size_t balloon_page_list_enqueue(struct balloon_dev_info *b_dev_info, |
41 | struct list_head *pages) |
42 | { |
43 | struct page *page, *tmp; |
44 | unsigned long flags; |
45 | size_t n_pages = 0; |
46 | |
47 | spin_lock_irqsave(&b_dev_info->pages_lock, flags); |
48 | list_for_each_entry_safe(page, tmp, pages, lru) { |
49 | list_del(entry: &page->lru); |
50 | balloon_page_enqueue_one(b_dev_info, page); |
51 | n_pages++; |
52 | } |
53 | spin_unlock_irqrestore(lock: &b_dev_info->pages_lock, flags); |
54 | return n_pages; |
55 | } |
56 | EXPORT_SYMBOL_GPL(balloon_page_list_enqueue); |
57 | |
58 | /** |
59 | * balloon_page_list_dequeue() - removes pages from balloon's page list and |
60 | * returns a list of the pages. |
61 | * @b_dev_info: balloon device descriptor where we will grab a page from. |
62 | * @pages: pointer to the list of pages that would be returned to the caller. |
63 | * @n_req_pages: number of requested pages. |
64 | * |
65 | * Driver must call this function to properly de-allocate a previous enlisted |
66 | * balloon pages before definitively releasing it back to the guest system. |
67 | * This function tries to remove @n_req_pages from the ballooned pages and |
68 | * return them to the caller in the @pages list. |
69 | * |
70 | * Note that this function may fail to dequeue some pages even if the balloon |
71 | * isn't empty - since the page list can be temporarily empty due to compaction |
72 | * of isolated pages. |
73 | * |
74 | * Return: number of pages that were added to the @pages list. |
75 | */ |
76 | size_t balloon_page_list_dequeue(struct balloon_dev_info *b_dev_info, |
77 | struct list_head *pages, size_t n_req_pages) |
78 | { |
79 | struct page *page, *tmp; |
80 | unsigned long flags; |
81 | size_t n_pages = 0; |
82 | |
83 | spin_lock_irqsave(&b_dev_info->pages_lock, flags); |
84 | list_for_each_entry_safe(page, tmp, &b_dev_info->pages, lru) { |
85 | if (n_pages == n_req_pages) |
86 | break; |
87 | |
88 | /* |
89 | * Block others from accessing the 'page' while we get around to |
90 | * establishing additional references and preparing the 'page' |
91 | * to be released by the balloon driver. |
92 | */ |
93 | if (!trylock_page(page)) |
94 | continue; |
95 | |
96 | if (IS_ENABLED(CONFIG_BALLOON_COMPACTION) && |
97 | PageIsolated(page)) { |
98 | /* raced with isolation */ |
99 | unlock_page(page); |
100 | continue; |
101 | } |
102 | balloon_page_delete(page); |
103 | __count_vm_event(item: BALLOON_DEFLATE); |
104 | list_add(new: &page->lru, head: pages); |
105 | unlock_page(page); |
106 | n_pages++; |
107 | } |
108 | spin_unlock_irqrestore(lock: &b_dev_info->pages_lock, flags); |
109 | |
110 | return n_pages; |
111 | } |
112 | EXPORT_SYMBOL_GPL(balloon_page_list_dequeue); |
113 | |
114 | /* |
115 | * balloon_page_alloc - allocates a new page for insertion into the balloon |
116 | * page list. |
117 | * |
118 | * Driver must call this function to properly allocate a new balloon page. |
119 | * Driver must call balloon_page_enqueue before definitively removing the page |
120 | * from the guest system. |
121 | * |
122 | * Return: struct page for the allocated page or NULL on allocation failure. |
123 | */ |
124 | struct page *balloon_page_alloc(void) |
125 | { |
126 | struct page *page = alloc_page(balloon_mapping_gfp_mask() | |
127 | __GFP_NOMEMALLOC | __GFP_NORETRY | |
128 | __GFP_NOWARN); |
129 | return page; |
130 | } |
131 | EXPORT_SYMBOL_GPL(balloon_page_alloc); |
132 | |
133 | /* |
134 | * balloon_page_enqueue - inserts a new page into the balloon page list. |
135 | * |
136 | * @b_dev_info: balloon device descriptor where we will insert a new page |
137 | * @page: new page to enqueue - allocated using balloon_page_alloc. |
138 | * |
139 | * Drivers must call this function to properly enqueue a new allocated balloon |
140 | * page before definitively removing the page from the guest system. |
141 | * |
142 | * Drivers must not call balloon_page_enqueue on pages that have been pushed to |
143 | * a list with balloon_page_push before removing them with balloon_page_pop. To |
144 | * enqueue a list of pages, use balloon_page_list_enqueue instead. |
145 | */ |
146 | void balloon_page_enqueue(struct balloon_dev_info *b_dev_info, |
147 | struct page *page) |
148 | { |
149 | unsigned long flags; |
150 | |
151 | spin_lock_irqsave(&b_dev_info->pages_lock, flags); |
152 | balloon_page_enqueue_one(b_dev_info, page); |
153 | spin_unlock_irqrestore(lock: &b_dev_info->pages_lock, flags); |
154 | } |
155 | EXPORT_SYMBOL_GPL(balloon_page_enqueue); |
156 | |
157 | /* |
158 | * balloon_page_dequeue - removes a page from balloon's page list and returns |
159 | * its address to allow the driver to release the page. |
160 | * @b_dev_info: balloon device descriptor where we will grab a page from. |
161 | * |
162 | * Driver must call this function to properly dequeue a previously enqueued page |
163 | * before definitively releasing it back to the guest system. |
164 | * |
165 | * Caller must perform its own accounting to ensure that this |
166 | * function is called only if some pages are actually enqueued. |
167 | * |
168 | * Note that this function may fail to dequeue some pages even if there are |
169 | * some enqueued pages - since the page list can be temporarily empty due to |
170 | * the compaction of isolated pages. |
171 | * |
172 | * TODO: remove the caller accounting requirements, and allow caller to wait |
173 | * until all pages can be dequeued. |
174 | * |
175 | * Return: struct page for the dequeued page, or NULL if no page was dequeued. |
176 | */ |
177 | struct page *balloon_page_dequeue(struct balloon_dev_info *b_dev_info) |
178 | { |
179 | unsigned long flags; |
180 | LIST_HEAD(pages); |
181 | int n_pages; |
182 | |
183 | n_pages = balloon_page_list_dequeue(b_dev_info, &pages, 1); |
184 | |
185 | if (n_pages != 1) { |
186 | /* |
187 | * If we are unable to dequeue a balloon page because the page |
188 | * list is empty and there are no isolated pages, then something |
189 | * went out of track and some balloon pages are lost. |
190 | * BUG() here, otherwise the balloon driver may get stuck in |
191 | * an infinite loop while attempting to release all its pages. |
192 | */ |
193 | spin_lock_irqsave(&b_dev_info->pages_lock, flags); |
194 | if (unlikely(list_empty(&b_dev_info->pages) && |
195 | !b_dev_info->isolated_pages)) |
196 | BUG(); |
197 | spin_unlock_irqrestore(lock: &b_dev_info->pages_lock, flags); |
198 | return NULL; |
199 | } |
200 | return list_first_entry(&pages, struct page, lru); |
201 | } |
202 | EXPORT_SYMBOL_GPL(balloon_page_dequeue); |
203 | |
204 | #ifdef CONFIG_BALLOON_COMPACTION |
205 | |
206 | static bool balloon_page_isolate(struct page *page, isolate_mode_t mode) |
207 | |
208 | { |
209 | struct balloon_dev_info *b_dev_info = balloon_page_device(page); |
210 | unsigned long flags; |
211 | |
212 | spin_lock_irqsave(&b_dev_info->pages_lock, flags); |
213 | list_del(entry: &page->lru); |
214 | b_dev_info->isolated_pages++; |
215 | spin_unlock_irqrestore(lock: &b_dev_info->pages_lock, flags); |
216 | |
217 | return true; |
218 | } |
219 | |
220 | static void balloon_page_putback(struct page *page) |
221 | { |
222 | struct balloon_dev_info *b_dev_info = balloon_page_device(page); |
223 | unsigned long flags; |
224 | |
225 | spin_lock_irqsave(&b_dev_info->pages_lock, flags); |
226 | list_add(new: &page->lru, head: &b_dev_info->pages); |
227 | b_dev_info->isolated_pages--; |
228 | spin_unlock_irqrestore(lock: &b_dev_info->pages_lock, flags); |
229 | } |
230 | |
231 | /* move_to_new_page() counterpart for a ballooned page */ |
232 | static int balloon_page_migrate(struct page *newpage, struct page *page, |
233 | enum migrate_mode mode) |
234 | { |
235 | struct balloon_dev_info *balloon = balloon_page_device(page); |
236 | |
237 | /* |
238 | * We can not easily support the no copy case here so ignore it as it |
239 | * is unlikely to be used with balloon pages. See include/linux/hmm.h |
240 | * for a user of the MIGRATE_SYNC_NO_COPY mode. |
241 | */ |
242 | if (mode == MIGRATE_SYNC_NO_COPY) |
243 | return -EINVAL; |
244 | |
245 | VM_BUG_ON_PAGE(!PageLocked(page), page); |
246 | VM_BUG_ON_PAGE(!PageLocked(newpage), newpage); |
247 | |
248 | return balloon->migratepage(balloon, newpage, page, mode); |
249 | } |
250 | |
251 | const struct movable_operations balloon_mops = { |
252 | .migrate_page = balloon_page_migrate, |
253 | .isolate_page = balloon_page_isolate, |
254 | .putback_page = balloon_page_putback, |
255 | }; |
256 | EXPORT_SYMBOL_GPL(balloon_mops); |
257 | |
258 | #endif /* CONFIG_BALLOON_COMPACTION */ |
259 | |