1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * DAMON-based page reclamation |
4 | * |
5 | * Author: SeongJae Park <sj@kernel.org> |
6 | */ |
7 | |
8 | #define pr_fmt(fmt) "damon-reclaim: " fmt |
9 | |
10 | #include <linux/damon.h> |
11 | #include <linux/kstrtox.h> |
12 | #include <linux/module.h> |
13 | |
14 | #include "modules-common.h" |
15 | |
16 | #ifdef MODULE_PARAM_PREFIX |
17 | #undef MODULE_PARAM_PREFIX |
18 | #endif |
19 | #define MODULE_PARAM_PREFIX "damon_reclaim." |
20 | |
21 | /* |
22 | * Enable or disable DAMON_RECLAIM. |
23 | * |
24 | * You can enable DAMON_RCLAIM by setting the value of this parameter as ``Y``. |
25 | * Setting it as ``N`` disables DAMON_RECLAIM. Note that DAMON_RECLAIM could |
26 | * do no real monitoring and reclamation due to the watermarks-based activation |
27 | * condition. Refer to below descriptions for the watermarks parameter for |
28 | * this. |
29 | */ |
30 | static bool enabled __read_mostly; |
31 | |
32 | /* |
33 | * Make DAMON_RECLAIM reads the input parameters again, except ``enabled``. |
34 | * |
35 | * Input parameters that updated while DAMON_RECLAIM is running are not applied |
36 | * by default. Once this parameter is set as ``Y``, DAMON_RECLAIM reads values |
37 | * of parametrs except ``enabled`` again. Once the re-reading is done, this |
38 | * parameter is set as ``N``. If invalid parameters are found while the |
39 | * re-reading, DAMON_RECLAIM will be disabled. |
40 | */ |
41 | static bool commit_inputs __read_mostly; |
42 | module_param(commit_inputs, bool, 0600); |
43 | |
44 | /* |
45 | * Time threshold for cold memory regions identification in microseconds. |
46 | * |
47 | * If a memory region is not accessed for this or longer time, DAMON_RECLAIM |
48 | * identifies the region as cold, and reclaims. 120 seconds by default. |
49 | */ |
50 | static unsigned long min_age __read_mostly = 120000000; |
51 | module_param(min_age, ulong, 0600); |
52 | |
53 | static struct damos_quota damon_reclaim_quota = { |
54 | /* use up to 10 ms time, reclaim up to 128 MiB per 1 sec by default */ |
55 | .ms = 10, |
56 | .sz = 128 * 1024 * 1024, |
57 | .reset_interval = 1000, |
58 | /* Within the quota, page out older regions first. */ |
59 | .weight_sz = 0, |
60 | .weight_nr_accesses = 0, |
61 | .weight_age = 1 |
62 | }; |
63 | DEFINE_DAMON_MODULES_DAMOS_QUOTAS(damon_reclaim_quota); |
64 | |
65 | /* |
66 | * Desired level of memory pressure-stall time in microseconds. |
67 | * |
68 | * While keeping the caps that set by other quotas, DAMON_RECLAIM automatically |
69 | * increases and decreases the effective level of the quota aiming this level of |
70 | * memory pressure is incurred. System-wide ``some`` memory PSI in microseconds |
71 | * per quota reset interval (``quota_reset_interval_ms``) is collected and |
72 | * compared to this value to see if the aim is satisfied. Value zero means |
73 | * disabling this auto-tuning feature. |
74 | * |
75 | * Disabled by default. |
76 | */ |
77 | static unsigned long quota_mem_pressure_us __read_mostly; |
78 | module_param(quota_mem_pressure_us, ulong, 0600); |
79 | |
80 | /* |
81 | * User-specifiable feedback for auto-tuning of the effective quota. |
82 | * |
83 | * While keeping the caps that set by other quotas, DAMON_RECLAIM automatically |
84 | * increases and decreases the effective level of the quota aiming receiving this |
85 | * feedback of value ``10,000`` from the user. DAMON_RECLAIM assumes the feedback |
86 | * value and the quota are positively proportional. Value zero means disabling |
87 | * this auto-tuning feature. |
88 | * |
89 | * Disabled by default. |
90 | * |
91 | */ |
92 | static unsigned long quota_autotune_feedback __read_mostly; |
93 | module_param(quota_autotune_feedback, ulong, 0600); |
94 | |
95 | static struct damos_watermarks damon_reclaim_wmarks = { |
96 | .metric = DAMOS_WMARK_FREE_MEM_RATE, |
97 | .interval = 5000000, /* 5 seconds */ |
98 | .high = 500, /* 50 percent */ |
99 | .mid = 400, /* 40 percent */ |
100 | .low = 200, /* 20 percent */ |
101 | }; |
102 | DEFINE_DAMON_MODULES_WMARKS_PARAMS(damon_reclaim_wmarks); |
103 | |
104 | static struct damon_attrs damon_reclaim_mon_attrs = { |
105 | .sample_interval = 5000, /* 5 ms */ |
106 | .aggr_interval = 100000, /* 100 ms */ |
107 | .ops_update_interval = 0, |
108 | .min_nr_regions = 10, |
109 | .max_nr_regions = 1000, |
110 | }; |
111 | DEFINE_DAMON_MODULES_MON_ATTRS_PARAMS(damon_reclaim_mon_attrs); |
112 | |
113 | /* |
114 | * Start of the target memory region in physical address. |
115 | * |
116 | * The start physical address of memory region that DAMON_RECLAIM will do work |
117 | * against. By default, biggest System RAM is used as the region. |
118 | */ |
119 | static unsigned long monitor_region_start __read_mostly; |
120 | module_param(monitor_region_start, ulong, 0600); |
121 | |
122 | /* |
123 | * End of the target memory region in physical address. |
124 | * |
125 | * The end physical address of memory region that DAMON_RECLAIM will do work |
126 | * against. By default, biggest System RAM is used as the region. |
127 | */ |
128 | static unsigned long monitor_region_end __read_mostly; |
129 | module_param(monitor_region_end, ulong, 0600); |
130 | |
131 | /* |
132 | * Skip anonymous pages reclamation. |
133 | * |
134 | * If this parameter is set as ``Y``, DAMON_RECLAIM does not reclaim anonymous |
135 | * pages. By default, ``N``. |
136 | */ |
137 | static bool skip_anon __read_mostly; |
138 | module_param(skip_anon, bool, 0600); |
139 | |
140 | /* |
141 | * PID of the DAMON thread |
142 | * |
143 | * If DAMON_RECLAIM is enabled, this becomes the PID of the worker thread. |
144 | * Else, -1. |
145 | */ |
146 | static int kdamond_pid __read_mostly = -1; |
147 | module_param(kdamond_pid, int, 0400); |
148 | |
149 | static struct damos_stat damon_reclaim_stat; |
150 | DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_reclaim_stat, |
151 | reclaim_tried_regions, reclaimed_regions, quota_exceeds); |
152 | |
153 | static struct damon_ctx *ctx; |
154 | static struct damon_target *target; |
155 | |
156 | static struct damos *damon_reclaim_new_scheme(void) |
157 | { |
158 | struct damos_access_pattern pattern = { |
159 | /* Find regions having PAGE_SIZE or larger size */ |
160 | .min_sz_region = PAGE_SIZE, |
161 | .max_sz_region = ULONG_MAX, |
162 | /* and not accessed at all */ |
163 | .min_nr_accesses = 0, |
164 | .max_nr_accesses = 0, |
165 | /* for min_age or more micro-seconds */ |
166 | .min_age_region = min_age / |
167 | damon_reclaim_mon_attrs.aggr_interval, |
168 | .max_age_region = UINT_MAX, |
169 | }; |
170 | |
171 | return damon_new_scheme( |
172 | pattern: &pattern, |
173 | /* page out those, as soon as found */ |
174 | action: DAMOS_PAGEOUT, |
175 | /* for each aggregation interval */ |
176 | apply_interval_us: 0, |
177 | /* under the quota. */ |
178 | quota: &damon_reclaim_quota, |
179 | /* (De)activate this according to the watermarks. */ |
180 | wmarks: &damon_reclaim_wmarks); |
181 | } |
182 | |
183 | static void damon_reclaim_copy_quota_status(struct damos_quota *dst, |
184 | struct damos_quota *src) |
185 | { |
186 | dst->total_charged_sz = src->total_charged_sz; |
187 | dst->total_charged_ns = src->total_charged_ns; |
188 | dst->charged_sz = src->charged_sz; |
189 | dst->charged_from = src->charged_from; |
190 | dst->charge_target_from = src->charge_target_from; |
191 | dst->charge_addr_from = src->charge_addr_from; |
192 | dst->esz_bp = src->esz_bp; |
193 | } |
194 | |
195 | static int damon_reclaim_apply_parameters(void) |
196 | { |
197 | struct damos *scheme, *old_scheme; |
198 | struct damos_quota_goal *goal; |
199 | struct damos_filter *filter; |
200 | int err = 0; |
201 | |
202 | err = damon_set_attrs(ctx, attrs: &damon_reclaim_mon_attrs); |
203 | if (err) |
204 | return err; |
205 | |
206 | /* Will be freed by next 'damon_set_schemes()' below */ |
207 | scheme = damon_reclaim_new_scheme(); |
208 | if (!scheme) |
209 | return -ENOMEM; |
210 | if (!list_empty(head: &ctx->schemes)) { |
211 | damon_for_each_scheme(old_scheme, ctx) |
212 | damon_reclaim_copy_quota_status(dst: &scheme->quota, |
213 | src: &old_scheme->quota); |
214 | } |
215 | |
216 | if (quota_mem_pressure_us) { |
217 | goal = damos_new_quota_goal(metric: DAMOS_QUOTA_SOME_MEM_PSI_US, |
218 | target_value: quota_mem_pressure_us); |
219 | if (!goal) { |
220 | damon_destroy_scheme(s: scheme); |
221 | return -ENOMEM; |
222 | } |
223 | damos_add_quota_goal(q: &scheme->quota, g: goal); |
224 | } |
225 | |
226 | if (quota_autotune_feedback) { |
227 | goal = damos_new_quota_goal(metric: DAMOS_QUOTA_USER_INPUT, target_value: 10000); |
228 | if (!goal) { |
229 | damon_destroy_scheme(s: scheme); |
230 | return -ENOMEM; |
231 | } |
232 | goal->current_value = quota_autotune_feedback; |
233 | damos_add_quota_goal(q: &scheme->quota, g: goal); |
234 | } |
235 | |
236 | if (skip_anon) { |
237 | filter = damos_new_filter(type: DAMOS_FILTER_TYPE_ANON, matching: true); |
238 | if (!filter) { |
239 | /* Will be freed by next 'damon_set_schemes()' below */ |
240 | damon_destroy_scheme(s: scheme); |
241 | return -ENOMEM; |
242 | } |
243 | damos_add_filter(s: scheme, f: filter); |
244 | } |
245 | damon_set_schemes(ctx, schemes: &scheme, nr_schemes: 1); |
246 | |
247 | return damon_set_region_biggest_system_ram_default(t: target, |
248 | start: &monitor_region_start, |
249 | end: &monitor_region_end); |
250 | } |
251 | |
252 | static int damon_reclaim_turn(bool on) |
253 | { |
254 | int err; |
255 | |
256 | if (!on) { |
257 | err = damon_stop(ctxs: &ctx, nr_ctxs: 1); |
258 | if (!err) |
259 | kdamond_pid = -1; |
260 | return err; |
261 | } |
262 | |
263 | err = damon_reclaim_apply_parameters(); |
264 | if (err) |
265 | return err; |
266 | |
267 | err = damon_start(ctxs: &ctx, nr_ctxs: 1, exclusive: true); |
268 | if (err) |
269 | return err; |
270 | kdamond_pid = ctx->kdamond->pid; |
271 | return 0; |
272 | } |
273 | |
274 | static int damon_reclaim_enabled_store(const char *val, |
275 | const struct kernel_param *kp) |
276 | { |
277 | bool is_enabled = enabled; |
278 | bool enable; |
279 | int err; |
280 | |
281 | err = kstrtobool(s: val, res: &enable); |
282 | if (err) |
283 | return err; |
284 | |
285 | if (is_enabled == enable) |
286 | return 0; |
287 | |
288 | /* Called before init function. The function will handle this. */ |
289 | if (!ctx) |
290 | goto set_param_out; |
291 | |
292 | err = damon_reclaim_turn(on: enable); |
293 | if (err) |
294 | return err; |
295 | |
296 | set_param_out: |
297 | enabled = enable; |
298 | return err; |
299 | } |
300 | |
301 | static const struct kernel_param_ops enabled_param_ops = { |
302 | .set = damon_reclaim_enabled_store, |
303 | .get = param_get_bool, |
304 | }; |
305 | |
306 | module_param_cb(enabled, &enabled_param_ops, &enabled, 0600); |
307 | MODULE_PARM_DESC(enabled, |
308 | "Enable or disable DAMON_RECLAIM (default: disabled)" ); |
309 | |
310 | static int damon_reclaim_handle_commit_inputs(void) |
311 | { |
312 | int err; |
313 | |
314 | if (!commit_inputs) |
315 | return 0; |
316 | |
317 | err = damon_reclaim_apply_parameters(); |
318 | commit_inputs = false; |
319 | return err; |
320 | } |
321 | |
322 | static int damon_reclaim_after_aggregation(struct damon_ctx *c) |
323 | { |
324 | struct damos *s; |
325 | |
326 | /* update the stats parameter */ |
327 | damon_for_each_scheme(s, c) |
328 | damon_reclaim_stat = s->stat; |
329 | |
330 | return damon_reclaim_handle_commit_inputs(); |
331 | } |
332 | |
333 | static int damon_reclaim_after_wmarks_check(struct damon_ctx *c) |
334 | { |
335 | return damon_reclaim_handle_commit_inputs(); |
336 | } |
337 | |
338 | static int __init damon_reclaim_init(void) |
339 | { |
340 | int err = damon_modules_new_paddr_ctx_target(ctxp: &ctx, targetp: &target); |
341 | |
342 | if (err) |
343 | return err; |
344 | |
345 | ctx->callback.after_wmarks_check = damon_reclaim_after_wmarks_check; |
346 | ctx->callback.after_aggregation = damon_reclaim_after_aggregation; |
347 | |
348 | /* 'enabled' has set before this function, probably via command line */ |
349 | if (enabled) |
350 | err = damon_reclaim_turn(on: true); |
351 | |
352 | return err; |
353 | } |
354 | |
355 | module_init(damon_reclaim_init); |
356 | |