1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | /* |
3 | * include/linux/writeback.h |
4 | */ |
5 | #ifndef WRITEBACK_H |
6 | #define WRITEBACK_H |
7 | |
8 | #include <linux/sched.h> |
9 | #include <linux/workqueue.h> |
10 | #include <linux/fs.h> |
11 | #include <linux/flex_proportions.h> |
12 | #include <linux/backing-dev-defs.h> |
13 | #include <linux/blk_types.h> |
14 | |
15 | struct bio; |
16 | |
17 | DECLARE_PER_CPU(int, dirty_throttle_leaks); |
18 | |
19 | /* |
20 | * The 1/4 region under the global dirty thresh is for smooth dirty throttling: |
21 | * |
22 | * (thresh - thresh/DIRTY_FULL_SCOPE, thresh) |
23 | * |
24 | * Further beyond, all dirtier tasks will enter a loop waiting (possibly long |
25 | * time) for the dirty pages to drop, unless written enough pages. |
26 | * |
27 | * The global dirty threshold is normally equal to the global dirty limit, |
28 | * except when the system suddenly allocates a lot of anonymous memory and |
29 | * knocks down the global dirty threshold quickly, in which case the global |
30 | * dirty limit will follow down slowly to prevent livelocking all dirtier tasks. |
31 | */ |
32 | #define DIRTY_SCOPE 8 |
33 | #define DIRTY_FULL_SCOPE (DIRTY_SCOPE / 2) |
34 | |
35 | struct backing_dev_info; |
36 | |
37 | /* |
38 | * fs/fs-writeback.c |
39 | */ |
40 | enum writeback_sync_modes { |
41 | WB_SYNC_NONE, /* Don't wait on anything */ |
42 | WB_SYNC_ALL, /* Wait on every mapping */ |
43 | }; |
44 | |
45 | /* |
46 | * A control structure which tells the writeback code what to do. These are |
47 | * always on the stack, and hence need no locking. They are always initialised |
48 | * in a manner such that unspecified fields are set to zero. |
49 | */ |
50 | struct writeback_control { |
51 | long nr_to_write; /* Write this many pages, and decrement |
52 | this for each page written */ |
53 | long pages_skipped; /* Pages which were not written */ |
54 | |
55 | /* |
56 | * For a_ops->writepages(): if start or end are non-zero then this is |
57 | * a hint that the filesystem need only write out the pages inside that |
58 | * byterange. The byte at `end' is included in the writeout request. |
59 | */ |
60 | loff_t range_start; |
61 | loff_t range_end; |
62 | |
63 | enum writeback_sync_modes sync_mode; |
64 | |
65 | unsigned for_kupdate:1; /* A kupdate writeback */ |
66 | unsigned for_background:1; /* A background writeback */ |
67 | unsigned tagged_writepages:1; /* tag-and-write to avoid livelock */ |
68 | unsigned for_reclaim:1; /* Invoked from the page allocator */ |
69 | unsigned range_cyclic:1; /* range_start is cyclic */ |
70 | unsigned for_sync:1; /* sync(2) WB_SYNC_ALL writeback */ |
71 | #ifdef CONFIG_CGROUP_WRITEBACK |
72 | struct bdi_writeback *wb; /* wb this writeback is issued under */ |
73 | struct inode *inode; /* inode being written out */ |
74 | |
75 | /* foreign inode detection, see wbc_detach_inode() */ |
76 | int wb_id; /* current wb id */ |
77 | int wb_lcand_id; /* last foreign candidate wb id */ |
78 | int wb_tcand_id; /* this foreign candidate wb id */ |
79 | size_t wb_bytes; /* bytes written by current wb */ |
80 | size_t wb_lcand_bytes; /* bytes written by last candidate */ |
81 | size_t wb_tcand_bytes; /* bytes written by this candidate */ |
82 | #endif |
83 | }; |
84 | |
85 | static inline int wbc_to_write_flags(struct writeback_control *wbc) |
86 | { |
87 | if (wbc->sync_mode == WB_SYNC_ALL) |
88 | return REQ_SYNC; |
89 | else if (wbc->for_kupdate || wbc->for_background) |
90 | return REQ_BACKGROUND; |
91 | |
92 | return 0; |
93 | } |
94 | |
95 | /* |
96 | * A wb_domain represents a domain that wb's (bdi_writeback's) belong to |
97 | * and are measured against each other in. There always is one global |
98 | * domain, global_wb_domain, that every wb in the system is a member of. |
99 | * This allows measuring the relative bandwidth of each wb to distribute |
100 | * dirtyable memory accordingly. |
101 | */ |
102 | struct wb_domain { |
103 | spinlock_t lock; |
104 | |
105 | /* |
106 | * Scale the writeback cache size proportional to the relative |
107 | * writeout speed. |
108 | * |
109 | * We do this by keeping a floating proportion between BDIs, based |
110 | * on page writeback completions [end_page_writeback()]. Those |
111 | * devices that write out pages fastest will get the larger share, |
112 | * while the slower will get a smaller share. |
113 | * |
114 | * We use page writeout completions because we are interested in |
115 | * getting rid of dirty pages. Having them written out is the |
116 | * primary goal. |
117 | * |
118 | * We introduce a concept of time, a period over which we measure |
119 | * these events, because demand can/will vary over time. The length |
120 | * of this period itself is measured in page writeback completions. |
121 | */ |
122 | struct fprop_global completions; |
123 | struct timer_list period_timer; /* timer for aging of completions */ |
124 | unsigned long period_time; |
125 | |
126 | /* |
127 | * The dirtyable memory and dirty threshold could be suddenly |
128 | * knocked down by a large amount (eg. on the startup of KVM in a |
129 | * swapless system). This may throw the system into deep dirty |
130 | * exceeded state and throttle heavy/light dirtiers alike. To |
131 | * retain good responsiveness, maintain global_dirty_limit for |
132 | * tracking slowly down to the knocked down dirty threshold. |
133 | * |
134 | * Both fields are protected by ->lock. |
135 | */ |
136 | unsigned long dirty_limit_tstamp; |
137 | unsigned long dirty_limit; |
138 | }; |
139 | |
140 | /** |
141 | * wb_domain_size_changed - memory available to a wb_domain has changed |
142 | * @dom: wb_domain of interest |
143 | * |
144 | * This function should be called when the amount of memory available to |
145 | * @dom has changed. It resets @dom's dirty limit parameters to prevent |
146 | * the past values which don't match the current configuration from skewing |
147 | * dirty throttling. Without this, when memory size of a wb_domain is |
148 | * greatly reduced, the dirty throttling logic may allow too many pages to |
149 | * be dirtied leading to consecutive unnecessary OOMs and may get stuck in |
150 | * that situation. |
151 | */ |
152 | static inline void wb_domain_size_changed(struct wb_domain *dom) |
153 | { |
154 | spin_lock(&dom->lock); |
155 | dom->dirty_limit_tstamp = jiffies; |
156 | dom->dirty_limit = 0; |
157 | spin_unlock(&dom->lock); |
158 | } |
159 | |
160 | /* |
161 | * fs/fs-writeback.c |
162 | */ |
163 | struct bdi_writeback; |
164 | void writeback_inodes_sb(struct super_block *, enum wb_reason reason); |
165 | void writeback_inodes_sb_nr(struct super_block *, unsigned long nr, |
166 | enum wb_reason reason); |
167 | void try_to_writeback_inodes_sb(struct super_block *sb, enum wb_reason reason); |
168 | void sync_inodes_sb(struct super_block *); |
169 | void wakeup_flusher_threads(enum wb_reason reason); |
170 | void wakeup_flusher_threads_bdi(struct backing_dev_info *bdi, |
171 | enum wb_reason reason); |
172 | void inode_wait_for_writeback(struct inode *inode); |
173 | |
174 | /* writeback.h requires fs.h; it, too, is not included from here. */ |
175 | static inline void wait_on_inode(struct inode *inode) |
176 | { |
177 | might_sleep(); |
178 | wait_on_bit(&inode->i_state, __I_NEW, TASK_UNINTERRUPTIBLE); |
179 | } |
180 | |
181 | #ifdef CONFIG_CGROUP_WRITEBACK |
182 | |
183 | #include <linux/cgroup.h> |
184 | #include <linux/bio.h> |
185 | |
186 | void __inode_attach_wb(struct inode *inode, struct page *page); |
187 | void wbc_attach_and_unlock_inode(struct writeback_control *wbc, |
188 | struct inode *inode) |
189 | __releases(&inode->i_lock); |
190 | void wbc_detach_inode(struct writeback_control *wbc); |
191 | void wbc_account_io(struct writeback_control *wbc, struct page *page, |
192 | size_t bytes); |
193 | void cgroup_writeback_umount(void); |
194 | |
195 | /** |
196 | * inode_attach_wb - associate an inode with its wb |
197 | * @inode: inode of interest |
198 | * @page: page being dirtied (may be NULL) |
199 | * |
200 | * If @inode doesn't have its wb, associate it with the wb matching the |
201 | * memcg of @page or, if @page is NULL, %current. May be called w/ or w/o |
202 | * @inode->i_lock. |
203 | */ |
204 | static inline void inode_attach_wb(struct inode *inode, struct page *page) |
205 | { |
206 | if (!inode->i_wb) |
207 | __inode_attach_wb(inode, page); |
208 | } |
209 | |
210 | /** |
211 | * inode_detach_wb - disassociate an inode from its wb |
212 | * @inode: inode of interest |
213 | * |
214 | * @inode is being freed. Detach from its wb. |
215 | */ |
216 | static inline void inode_detach_wb(struct inode *inode) |
217 | { |
218 | if (inode->i_wb) { |
219 | WARN_ON_ONCE(!(inode->i_state & I_CLEAR)); |
220 | wb_put(inode->i_wb); |
221 | inode->i_wb = NULL; |
222 | } |
223 | } |
224 | |
225 | /** |
226 | * wbc_attach_fdatawrite_inode - associate wbc and inode for fdatawrite |
227 | * @wbc: writeback_control of interest |
228 | * @inode: target inode |
229 | * |
230 | * This function is to be used by __filemap_fdatawrite_range(), which is an |
231 | * alternative entry point into writeback code, and first ensures @inode is |
232 | * associated with a bdi_writeback and attaches it to @wbc. |
233 | */ |
234 | static inline void wbc_attach_fdatawrite_inode(struct writeback_control *wbc, |
235 | struct inode *inode) |
236 | { |
237 | spin_lock(&inode->i_lock); |
238 | inode_attach_wb(inode, NULL); |
239 | wbc_attach_and_unlock_inode(wbc, inode); |
240 | } |
241 | |
242 | /** |
243 | * wbc_init_bio - writeback specific initializtion of bio |
244 | * @wbc: writeback_control for the writeback in progress |
245 | * @bio: bio to be initialized |
246 | * |
247 | * @bio is a part of the writeback in progress controlled by @wbc. Perform |
248 | * writeback specific initialization. This is used to apply the cgroup |
249 | * writeback context. Must be called after the bio has been associated with |
250 | * a device. |
251 | */ |
252 | static inline void wbc_init_bio(struct writeback_control *wbc, struct bio *bio) |
253 | { |
254 | /* |
255 | * pageout() path doesn't attach @wbc to the inode being written |
256 | * out. This is intentional as we don't want the function to block |
257 | * behind a slow cgroup. Ultimately, we want pageout() to kick off |
258 | * regular writeback instead of writing things out itself. |
259 | */ |
260 | if (wbc->wb) |
261 | bio_associate_blkg_from_css(bio, wbc->wb->blkcg_css); |
262 | } |
263 | |
264 | #else /* CONFIG_CGROUP_WRITEBACK */ |
265 | |
266 | static inline void inode_attach_wb(struct inode *inode, struct page *page) |
267 | { |
268 | } |
269 | |
270 | static inline void inode_detach_wb(struct inode *inode) |
271 | { |
272 | } |
273 | |
274 | static inline void wbc_attach_and_unlock_inode(struct writeback_control *wbc, |
275 | struct inode *inode) |
276 | __releases(&inode->i_lock) |
277 | { |
278 | spin_unlock(&inode->i_lock); |
279 | } |
280 | |
281 | static inline void wbc_attach_fdatawrite_inode(struct writeback_control *wbc, |
282 | struct inode *inode) |
283 | { |
284 | } |
285 | |
286 | static inline void wbc_detach_inode(struct writeback_control *wbc) |
287 | { |
288 | } |
289 | |
290 | static inline void wbc_init_bio(struct writeback_control *wbc, struct bio *bio) |
291 | { |
292 | } |
293 | |
294 | static inline void wbc_account_io(struct writeback_control *wbc, |
295 | struct page *page, size_t bytes) |
296 | { |
297 | } |
298 | |
299 | static inline void cgroup_writeback_umount(void) |
300 | { |
301 | } |
302 | |
303 | #endif /* CONFIG_CGROUP_WRITEBACK */ |
304 | |
305 | /* |
306 | * mm/page-writeback.c |
307 | */ |
308 | #ifdef CONFIG_BLOCK |
309 | void laptop_io_completion(struct backing_dev_info *info); |
310 | void laptop_sync_completion(void); |
311 | void laptop_mode_sync(struct work_struct *work); |
312 | void laptop_mode_timer_fn(struct timer_list *t); |
313 | #else |
314 | static inline void laptop_sync_completion(void) { } |
315 | #endif |
316 | bool node_dirty_ok(struct pglist_data *pgdat); |
317 | int wb_domain_init(struct wb_domain *dom, gfp_t gfp); |
318 | #ifdef CONFIG_CGROUP_WRITEBACK |
319 | void wb_domain_exit(struct wb_domain *dom); |
320 | #endif |
321 | |
322 | extern struct wb_domain global_wb_domain; |
323 | |
324 | /* These are exported to sysctl. */ |
325 | extern int dirty_background_ratio; |
326 | extern unsigned long dirty_background_bytes; |
327 | extern int vm_dirty_ratio; |
328 | extern unsigned long vm_dirty_bytes; |
329 | extern unsigned int dirty_writeback_interval; |
330 | extern unsigned int dirty_expire_interval; |
331 | extern unsigned int dirtytime_expire_interval; |
332 | extern int vm_highmem_is_dirtyable; |
333 | extern int block_dump; |
334 | extern int laptop_mode; |
335 | |
336 | extern int dirty_background_ratio_handler(struct ctl_table *table, int write, |
337 | void __user *buffer, size_t *lenp, |
338 | loff_t *ppos); |
339 | extern int dirty_background_bytes_handler(struct ctl_table *table, int write, |
340 | void __user *buffer, size_t *lenp, |
341 | loff_t *ppos); |
342 | extern int dirty_ratio_handler(struct ctl_table *table, int write, |
343 | void __user *buffer, size_t *lenp, |
344 | loff_t *ppos); |
345 | extern int dirty_bytes_handler(struct ctl_table *table, int write, |
346 | void __user *buffer, size_t *lenp, |
347 | loff_t *ppos); |
348 | int dirtytime_interval_handler(struct ctl_table *table, int write, |
349 | void __user *buffer, size_t *lenp, loff_t *ppos); |
350 | |
351 | struct ctl_table; |
352 | int dirty_writeback_centisecs_handler(struct ctl_table *, int, |
353 | void __user *, size_t *, loff_t *); |
354 | |
355 | void global_dirty_limits(unsigned long *pbackground, unsigned long *pdirty); |
356 | unsigned long wb_calc_thresh(struct bdi_writeback *wb, unsigned long thresh); |
357 | |
358 | void wb_update_bandwidth(struct bdi_writeback *wb, unsigned long start_time); |
359 | void balance_dirty_pages_ratelimited(struct address_space *mapping); |
360 | bool wb_over_bg_thresh(struct bdi_writeback *wb); |
361 | |
362 | typedef int (*writepage_t)(struct page *page, struct writeback_control *wbc, |
363 | void *data); |
364 | |
365 | int generic_writepages(struct address_space *mapping, |
366 | struct writeback_control *wbc); |
367 | void tag_pages_for_writeback(struct address_space *mapping, |
368 | pgoff_t start, pgoff_t end); |
369 | int write_cache_pages(struct address_space *mapping, |
370 | struct writeback_control *wbc, writepage_t writepage, |
371 | void *data); |
372 | int do_writepages(struct address_space *mapping, struct writeback_control *wbc); |
373 | void writeback_set_ratelimit(void); |
374 | void tag_pages_for_writeback(struct address_space *mapping, |
375 | pgoff_t start, pgoff_t end); |
376 | |
377 | void account_page_redirty(struct page *page); |
378 | |
379 | void sb_mark_inode_writeback(struct inode *inode); |
380 | void sb_clear_inode_writeback(struct inode *inode); |
381 | |
382 | #endif /* WRITEBACK_H */ |
383 | |