1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * DAMON-based LRU-lists Sorting |
4 | * |
5 | * Author: SeongJae Park <sj@kernel.org> |
6 | */ |
7 | |
8 | #define pr_fmt(fmt) "damon-lru-sort: " fmt |
9 | |
10 | #include <linux/damon.h> |
11 | #include <linux/kstrtox.h> |
12 | #include <linux/module.h> |
13 | |
14 | #include "modules-common.h" |
15 | |
16 | #ifdef MODULE_PARAM_PREFIX |
17 | #undef MODULE_PARAM_PREFIX |
18 | #endif |
19 | #define MODULE_PARAM_PREFIX "damon_lru_sort." |
20 | |
21 | /* |
22 | * Enable or disable DAMON_LRU_SORT. |
23 | * |
24 | * You can enable DAMON_LRU_SORT by setting the value of this parameter as |
25 | * ``Y``. Setting it as ``N`` disables DAMON_LRU_SORT. Note that |
26 | * DAMON_LRU_SORT could do no real monitoring and LRU-lists sorting due to the |
27 | * watermarks-based activation condition. Refer to below descriptions for the |
28 | * watermarks parameter for this. |
29 | */ |
30 | static bool enabled __read_mostly; |
31 | |
32 | /* |
33 | * Make DAMON_LRU_SORT reads the input parameters again, except ``enabled``. |
34 | * |
35 | * Input parameters that updated while DAMON_LRU_SORT is running are not |
36 | * applied by default. Once this parameter is set as ``Y``, DAMON_LRU_SORT |
37 | * reads values of parametrs except ``enabled`` again. Once the re-reading is |
38 | * done, this parameter is set as ``N``. If invalid parameters are found while |
39 | * the re-reading, DAMON_LRU_SORT will be disabled. |
40 | */ |
41 | static bool commit_inputs __read_mostly; |
42 | module_param(commit_inputs, bool, 0600); |
43 | |
44 | /* |
45 | * Access frequency threshold for hot memory regions identification in permil. |
46 | * |
47 | * If a memory region is accessed in frequency of this or higher, |
48 | * DAMON_LRU_SORT identifies the region as hot, and mark it as accessed on the |
49 | * LRU list, so that it could not be reclaimed under memory pressure. 50% by |
50 | * default. |
51 | */ |
52 | static unsigned long hot_thres_access_freq = 500; |
53 | module_param(hot_thres_access_freq, ulong, 0600); |
54 | |
55 | /* |
56 | * Time threshold for cold memory regions identification in microseconds. |
57 | * |
58 | * If a memory region is not accessed for this or longer time, DAMON_LRU_SORT |
59 | * identifies the region as cold, and mark it as unaccessed on the LRU list, so |
60 | * that it could be reclaimed first under memory pressure. 120 seconds by |
61 | * default. |
62 | */ |
63 | static unsigned long cold_min_age __read_mostly = 120000000; |
64 | module_param(cold_min_age, ulong, 0600); |
65 | |
66 | static struct damos_quota damon_lru_sort_quota = { |
67 | /* Use up to 10 ms per 1 sec, by default */ |
68 | .ms = 10, |
69 | .sz = 0, |
70 | .reset_interval = 1000, |
71 | /* Within the quota, mark hotter regions accessed first. */ |
72 | .weight_sz = 0, |
73 | .weight_nr_accesses = 1, |
74 | .weight_age = 0, |
75 | }; |
76 | DEFINE_DAMON_MODULES_DAMOS_TIME_QUOTA(damon_lru_sort_quota); |
77 | |
78 | static struct damos_watermarks damon_lru_sort_wmarks = { |
79 | .metric = DAMOS_WMARK_FREE_MEM_RATE, |
80 | .interval = 5000000, /* 5 seconds */ |
81 | .high = 200, /* 20 percent */ |
82 | .mid = 150, /* 15 percent */ |
83 | .low = 50, /* 5 percent */ |
84 | }; |
85 | DEFINE_DAMON_MODULES_WMARKS_PARAMS(damon_lru_sort_wmarks); |
86 | |
87 | static struct damon_attrs damon_lru_sort_mon_attrs = { |
88 | .sample_interval = 5000, /* 5 ms */ |
89 | .aggr_interval = 100000, /* 100 ms */ |
90 | .ops_update_interval = 0, |
91 | .min_nr_regions = 10, |
92 | .max_nr_regions = 1000, |
93 | }; |
94 | DEFINE_DAMON_MODULES_MON_ATTRS_PARAMS(damon_lru_sort_mon_attrs); |
95 | |
96 | /* |
97 | * Start of the target memory region in physical address. |
98 | * |
99 | * The start physical address of memory region that DAMON_LRU_SORT will do work |
100 | * against. By default, biggest System RAM is used as the region. |
101 | */ |
102 | static unsigned long monitor_region_start __read_mostly; |
103 | module_param(monitor_region_start, ulong, 0600); |
104 | |
105 | /* |
106 | * End of the target memory region in physical address. |
107 | * |
108 | * The end physical address of memory region that DAMON_LRU_SORT will do work |
109 | * against. By default, biggest System RAM is used as the region. |
110 | */ |
111 | static unsigned long monitor_region_end __read_mostly; |
112 | module_param(monitor_region_end, ulong, 0600); |
113 | |
114 | /* |
115 | * PID of the DAMON thread |
116 | * |
117 | * If DAMON_LRU_SORT is enabled, this becomes the PID of the worker thread. |
118 | * Else, -1. |
119 | */ |
120 | static int kdamond_pid __read_mostly = -1; |
121 | module_param(kdamond_pid, int, 0400); |
122 | |
123 | static struct damos_stat damon_lru_sort_hot_stat; |
124 | DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_lru_sort_hot_stat, |
125 | lru_sort_tried_hot_regions, lru_sorted_hot_regions, |
126 | hot_quota_exceeds); |
127 | |
128 | static struct damos_stat damon_lru_sort_cold_stat; |
129 | DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_lru_sort_cold_stat, |
130 | lru_sort_tried_cold_regions, lru_sorted_cold_regions, |
131 | cold_quota_exceeds); |
132 | |
133 | static struct damos_access_pattern damon_lru_sort_stub_pattern = { |
134 | /* Find regions having PAGE_SIZE or larger size */ |
135 | .min_sz_region = PAGE_SIZE, |
136 | .max_sz_region = ULONG_MAX, |
137 | /* no matter its access frequency */ |
138 | .min_nr_accesses = 0, |
139 | .max_nr_accesses = UINT_MAX, |
140 | /* no matter its age */ |
141 | .min_age_region = 0, |
142 | .max_age_region = UINT_MAX, |
143 | }; |
144 | |
145 | static struct damon_ctx *ctx; |
146 | static struct damon_target *target; |
147 | |
148 | static struct damos *damon_lru_sort_new_scheme( |
149 | struct damos_access_pattern *pattern, enum damos_action action) |
150 | { |
151 | struct damos_quota quota = damon_lru_sort_quota; |
152 | |
153 | /* Use half of total quota for hot/cold pages sorting */ |
154 | quota.ms = quota.ms / 2; |
155 | |
156 | return damon_new_scheme( |
157 | /* find the pattern, and */ |
158 | pattern, |
159 | /* (de)prioritize on LRU-lists */ |
160 | action, |
161 | /* for each aggregation interval */ |
162 | apply_interval_us: 0, |
163 | /* under the quota. */ |
164 | quota: "a, |
165 | /* (De)activate this according to the watermarks. */ |
166 | wmarks: &damon_lru_sort_wmarks); |
167 | } |
168 | |
169 | /* Create a DAMON-based operation scheme for hot memory regions */ |
170 | static struct damos *damon_lru_sort_new_hot_scheme(unsigned int hot_thres) |
171 | { |
172 | struct damos_access_pattern pattern = damon_lru_sort_stub_pattern; |
173 | |
174 | pattern.min_nr_accesses = hot_thres; |
175 | return damon_lru_sort_new_scheme(pattern: &pattern, action: DAMOS_LRU_PRIO); |
176 | } |
177 | |
178 | /* Create a DAMON-based operation scheme for cold memory regions */ |
179 | static struct damos *damon_lru_sort_new_cold_scheme(unsigned int cold_thres) |
180 | { |
181 | struct damos_access_pattern pattern = damon_lru_sort_stub_pattern; |
182 | |
183 | pattern.max_nr_accesses = 0; |
184 | pattern.min_age_region = cold_thres; |
185 | return damon_lru_sort_new_scheme(pattern: &pattern, action: DAMOS_LRU_DEPRIO); |
186 | } |
187 | |
188 | static void damon_lru_sort_copy_quota_status(struct damos_quota *dst, |
189 | struct damos_quota *src) |
190 | { |
191 | dst->total_charged_sz = src->total_charged_sz; |
192 | dst->total_charged_ns = src->total_charged_ns; |
193 | dst->charged_sz = src->charged_sz; |
194 | dst->charged_from = src->charged_from; |
195 | dst->charge_target_from = src->charge_target_from; |
196 | dst->charge_addr_from = src->charge_addr_from; |
197 | } |
198 | |
199 | static int damon_lru_sort_apply_parameters(void) |
200 | { |
201 | struct damos *scheme, *hot_scheme, *cold_scheme; |
202 | struct damos *old_hot_scheme = NULL, *old_cold_scheme = NULL; |
203 | unsigned int hot_thres, cold_thres; |
204 | int err = 0; |
205 | |
206 | err = damon_set_attrs(ctx, attrs: &damon_lru_sort_mon_attrs); |
207 | if (err) |
208 | return err; |
209 | |
210 | damon_for_each_scheme(scheme, ctx) { |
211 | if (!old_hot_scheme) { |
212 | old_hot_scheme = scheme; |
213 | continue; |
214 | } |
215 | old_cold_scheme = scheme; |
216 | } |
217 | |
218 | hot_thres = damon_max_nr_accesses(attrs: &damon_lru_sort_mon_attrs) * |
219 | hot_thres_access_freq / 1000; |
220 | hot_scheme = damon_lru_sort_new_hot_scheme(hot_thres); |
221 | if (!hot_scheme) |
222 | return -ENOMEM; |
223 | if (old_hot_scheme) |
224 | damon_lru_sort_copy_quota_status(dst: &hot_scheme->quota, |
225 | src: &old_hot_scheme->quota); |
226 | |
227 | cold_thres = cold_min_age / damon_lru_sort_mon_attrs.aggr_interval; |
228 | cold_scheme = damon_lru_sort_new_cold_scheme(cold_thres); |
229 | if (!cold_scheme) { |
230 | damon_destroy_scheme(s: hot_scheme); |
231 | return -ENOMEM; |
232 | } |
233 | if (old_cold_scheme) |
234 | damon_lru_sort_copy_quota_status(dst: &cold_scheme->quota, |
235 | src: &old_cold_scheme->quota); |
236 | |
237 | damon_set_schemes(ctx, schemes: &hot_scheme, nr_schemes: 1); |
238 | damon_add_scheme(ctx, s: cold_scheme); |
239 | |
240 | return damon_set_region_biggest_system_ram_default(t: target, |
241 | start: &monitor_region_start, |
242 | end: &monitor_region_end); |
243 | } |
244 | |
245 | static int damon_lru_sort_turn(bool on) |
246 | { |
247 | int err; |
248 | |
249 | if (!on) { |
250 | err = damon_stop(ctxs: &ctx, nr_ctxs: 1); |
251 | if (!err) |
252 | kdamond_pid = -1; |
253 | return err; |
254 | } |
255 | |
256 | err = damon_lru_sort_apply_parameters(); |
257 | if (err) |
258 | return err; |
259 | |
260 | err = damon_start(ctxs: &ctx, nr_ctxs: 1, exclusive: true); |
261 | if (err) |
262 | return err; |
263 | kdamond_pid = ctx->kdamond->pid; |
264 | return 0; |
265 | } |
266 | |
267 | static int damon_lru_sort_enabled_store(const char *val, |
268 | const struct kernel_param *kp) |
269 | { |
270 | bool is_enabled = enabled; |
271 | bool enable; |
272 | int err; |
273 | |
274 | err = kstrtobool(s: val, res: &enable); |
275 | if (err) |
276 | return err; |
277 | |
278 | if (is_enabled == enable) |
279 | return 0; |
280 | |
281 | /* Called before init function. The function will handle this. */ |
282 | if (!ctx) |
283 | goto set_param_out; |
284 | |
285 | err = damon_lru_sort_turn(on: enable); |
286 | if (err) |
287 | return err; |
288 | |
289 | set_param_out: |
290 | enabled = enable; |
291 | return err; |
292 | } |
293 | |
294 | static const struct kernel_param_ops enabled_param_ops = { |
295 | .set = damon_lru_sort_enabled_store, |
296 | .get = param_get_bool, |
297 | }; |
298 | |
299 | module_param_cb(enabled, &enabled_param_ops, &enabled, 0600); |
300 | MODULE_PARM_DESC(enabled, |
301 | "Enable or disable DAMON_LRU_SORT (default: disabled)" ); |
302 | |
303 | static int damon_lru_sort_handle_commit_inputs(void) |
304 | { |
305 | int err; |
306 | |
307 | if (!commit_inputs) |
308 | return 0; |
309 | |
310 | err = damon_lru_sort_apply_parameters(); |
311 | commit_inputs = false; |
312 | return err; |
313 | } |
314 | |
315 | static int damon_lru_sort_after_aggregation(struct damon_ctx *c) |
316 | { |
317 | struct damos *s; |
318 | |
319 | /* update the stats parameter */ |
320 | damon_for_each_scheme(s, c) { |
321 | if (s->action == DAMOS_LRU_PRIO) |
322 | damon_lru_sort_hot_stat = s->stat; |
323 | else if (s->action == DAMOS_LRU_DEPRIO) |
324 | damon_lru_sort_cold_stat = s->stat; |
325 | } |
326 | |
327 | return damon_lru_sort_handle_commit_inputs(); |
328 | } |
329 | |
330 | static int damon_lru_sort_after_wmarks_check(struct damon_ctx *c) |
331 | { |
332 | return damon_lru_sort_handle_commit_inputs(); |
333 | } |
334 | |
335 | static int __init damon_lru_sort_init(void) |
336 | { |
337 | int err = damon_modules_new_paddr_ctx_target(ctxp: &ctx, targetp: &target); |
338 | |
339 | if (err) |
340 | return err; |
341 | |
342 | ctx->callback.after_wmarks_check = damon_lru_sort_after_wmarks_check; |
343 | ctx->callback.after_aggregation = damon_lru_sort_after_aggregation; |
344 | |
345 | /* 'enabled' has set before this function, probably via command line */ |
346 | if (enabled) |
347 | err = damon_lru_sort_turn(on: true); |
348 | |
349 | return err; |
350 | } |
351 | |
352 | module_init(damon_lru_sort_init); |
353 | |