1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Housekeeping management. Manage the targets for routine code that can run on |
4 | * any CPU: unbound workqueues, timers, kthreads and any offloadable work. |
5 | * |
6 | * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker |
7 | * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker |
8 | * |
9 | */ |
10 | |
11 | enum hk_flags { |
12 | HK_FLAG_TIMER = BIT(HK_TYPE_TIMER), |
13 | HK_FLAG_RCU = BIT(HK_TYPE_RCU), |
14 | HK_FLAG_MISC = BIT(HK_TYPE_MISC), |
15 | HK_FLAG_SCHED = BIT(HK_TYPE_SCHED), |
16 | HK_FLAG_TICK = BIT(HK_TYPE_TICK), |
17 | HK_FLAG_DOMAIN = BIT(HK_TYPE_DOMAIN), |
18 | HK_FLAG_WQ = BIT(HK_TYPE_WQ), |
19 | HK_FLAG_MANAGED_IRQ = BIT(HK_TYPE_MANAGED_IRQ), |
20 | HK_FLAG_KTHREAD = BIT(HK_TYPE_KTHREAD), |
21 | }; |
22 | |
23 | DEFINE_STATIC_KEY_FALSE(housekeeping_overridden); |
24 | EXPORT_SYMBOL_GPL(housekeeping_overridden); |
25 | |
26 | struct housekeeping { |
27 | cpumask_var_t cpumasks[HK_TYPE_MAX]; |
28 | unsigned long flags; |
29 | }; |
30 | |
31 | static struct housekeeping housekeeping; |
32 | |
33 | bool housekeeping_enabled(enum hk_type type) |
34 | { |
35 | return !!(housekeeping.flags & BIT(type)); |
36 | } |
37 | EXPORT_SYMBOL_GPL(housekeeping_enabled); |
38 | |
39 | int housekeeping_any_cpu(enum hk_type type) |
40 | { |
41 | int cpu; |
42 | |
43 | if (static_branch_unlikely(&housekeeping_overridden)) { |
44 | if (housekeeping.flags & BIT(type)) { |
45 | cpu = sched_numa_find_closest(cpus: housekeeping.cpumasks[type], smp_processor_id()); |
46 | if (cpu < nr_cpu_ids) |
47 | return cpu; |
48 | |
49 | return cpumask_any_and(housekeeping.cpumasks[type], cpu_online_mask); |
50 | } |
51 | } |
52 | return smp_processor_id(); |
53 | } |
54 | EXPORT_SYMBOL_GPL(housekeeping_any_cpu); |
55 | |
56 | const struct cpumask *housekeeping_cpumask(enum hk_type type) |
57 | { |
58 | if (static_branch_unlikely(&housekeeping_overridden)) |
59 | if (housekeeping.flags & BIT(type)) |
60 | return housekeeping.cpumasks[type]; |
61 | return cpu_possible_mask; |
62 | } |
63 | EXPORT_SYMBOL_GPL(housekeeping_cpumask); |
64 | |
65 | void housekeeping_affine(struct task_struct *t, enum hk_type type) |
66 | { |
67 | if (static_branch_unlikely(&housekeeping_overridden)) |
68 | if (housekeeping.flags & BIT(type)) |
69 | set_cpus_allowed_ptr(p: t, new_mask: housekeeping.cpumasks[type]); |
70 | } |
71 | EXPORT_SYMBOL_GPL(housekeeping_affine); |
72 | |
73 | bool housekeeping_test_cpu(int cpu, enum hk_type type) |
74 | { |
75 | if (static_branch_unlikely(&housekeeping_overridden)) |
76 | if (housekeeping.flags & BIT(type)) |
77 | return cpumask_test_cpu(cpu, cpumask: housekeeping.cpumasks[type]); |
78 | return true; |
79 | } |
80 | EXPORT_SYMBOL_GPL(housekeeping_test_cpu); |
81 | |
82 | void __init housekeeping_init(void) |
83 | { |
84 | enum hk_type type; |
85 | |
86 | if (!housekeeping.flags) |
87 | return; |
88 | |
89 | static_branch_enable(&housekeeping_overridden); |
90 | |
91 | if (housekeeping.flags & HK_FLAG_TICK) |
92 | sched_tick_offload_init(); |
93 | |
94 | for_each_set_bit(type, &housekeeping.flags, HK_TYPE_MAX) { |
95 | /* We need at least one CPU to handle housekeeping work */ |
96 | WARN_ON_ONCE(cpumask_empty(housekeeping.cpumasks[type])); |
97 | } |
98 | } |
99 | |
100 | static void __init housekeeping_setup_type(enum hk_type type, |
101 | cpumask_var_t housekeeping_staging) |
102 | { |
103 | |
104 | alloc_bootmem_cpumask_var(mask: &housekeeping.cpumasks[type]); |
105 | cpumask_copy(dstp: housekeeping.cpumasks[type], |
106 | srcp: housekeeping_staging); |
107 | } |
108 | |
109 | static int __init housekeeping_setup(char *str, unsigned long flags) |
110 | { |
111 | cpumask_var_t non_housekeeping_mask, housekeeping_staging; |
112 | int err = 0; |
113 | |
114 | if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK)) { |
115 | if (!IS_ENABLED(CONFIG_NO_HZ_FULL)) { |
116 | pr_warn("Housekeeping: nohz unsupported." |
117 | " Build with CONFIG_NO_HZ_FULL\n" ); |
118 | return 0; |
119 | } |
120 | } |
121 | |
122 | alloc_bootmem_cpumask_var(mask: &non_housekeeping_mask); |
123 | if (cpulist_parse(buf: str, dstp: non_housekeeping_mask) < 0) { |
124 | pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n" ); |
125 | goto free_non_housekeeping_mask; |
126 | } |
127 | |
128 | alloc_bootmem_cpumask_var(mask: &housekeeping_staging); |
129 | cpumask_andnot(dstp: housekeeping_staging, |
130 | cpu_possible_mask, src2p: non_housekeeping_mask); |
131 | |
132 | if (!cpumask_intersects(cpu_present_mask, src2p: housekeeping_staging)) { |
133 | __cpumask_set_cpu(smp_processor_id(), dstp: housekeeping_staging); |
134 | __cpumask_clear_cpu(smp_processor_id(), dstp: non_housekeeping_mask); |
135 | if (!housekeeping.flags) { |
136 | pr_warn("Housekeeping: must include one present CPU, " |
137 | "using boot CPU:%d\n" , smp_processor_id()); |
138 | } |
139 | } |
140 | |
141 | if (!housekeeping.flags) { |
142 | /* First setup call ("nohz_full=" or "isolcpus=") */ |
143 | enum hk_type type; |
144 | |
145 | for_each_set_bit(type, &flags, HK_TYPE_MAX) |
146 | housekeeping_setup_type(type, housekeeping_staging); |
147 | } else { |
148 | /* Second setup call ("nohz_full=" after "isolcpus=" or the reverse) */ |
149 | enum hk_type type; |
150 | unsigned long iter_flags = flags & housekeeping.flags; |
151 | |
152 | for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) { |
153 | if (!cpumask_equal(src1p: housekeeping_staging, |
154 | src2p: housekeeping.cpumasks[type])) { |
155 | pr_warn("Housekeeping: nohz_full= must match isolcpus=\n" ); |
156 | goto free_housekeeping_staging; |
157 | } |
158 | } |
159 | |
160 | iter_flags = flags & ~housekeeping.flags; |
161 | |
162 | for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) |
163 | housekeeping_setup_type(type, housekeeping_staging); |
164 | } |
165 | |
166 | if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK)) |
167 | tick_nohz_full_setup(cpumask: non_housekeeping_mask); |
168 | |
169 | housekeeping.flags |= flags; |
170 | err = 1; |
171 | |
172 | free_housekeeping_staging: |
173 | free_bootmem_cpumask_var(mask: housekeeping_staging); |
174 | free_non_housekeeping_mask: |
175 | free_bootmem_cpumask_var(mask: non_housekeeping_mask); |
176 | |
177 | return err; |
178 | } |
179 | |
180 | static int __init housekeeping_nohz_full_setup(char *str) |
181 | { |
182 | unsigned long flags; |
183 | |
184 | flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU | |
185 | HK_FLAG_MISC | HK_FLAG_KTHREAD; |
186 | |
187 | return housekeeping_setup(str, flags); |
188 | } |
189 | __setup("nohz_full=" , housekeeping_nohz_full_setup); |
190 | |
191 | static int __init housekeeping_isolcpus_setup(char *str) |
192 | { |
193 | unsigned long flags = 0; |
194 | bool illegal = false; |
195 | char *par; |
196 | int len; |
197 | |
198 | while (isalpha(*str)) { |
199 | if (!strncmp(str, "nohz," , 5)) { |
200 | str += 5; |
201 | flags |= HK_FLAG_TICK; |
202 | continue; |
203 | } |
204 | |
205 | if (!strncmp(str, "domain," , 7)) { |
206 | str += 7; |
207 | flags |= HK_FLAG_DOMAIN; |
208 | continue; |
209 | } |
210 | |
211 | if (!strncmp(str, "managed_irq," , 12)) { |
212 | str += 12; |
213 | flags |= HK_FLAG_MANAGED_IRQ; |
214 | continue; |
215 | } |
216 | |
217 | /* |
218 | * Skip unknown sub-parameter and validate that it is not |
219 | * containing an invalid character. |
220 | */ |
221 | for (par = str, len = 0; *str && *str != ','; str++, len++) { |
222 | if (!isalpha(*str) && *str != '_') |
223 | illegal = true; |
224 | } |
225 | |
226 | if (illegal) { |
227 | pr_warn("isolcpus: Invalid flag %.*s\n" , len, par); |
228 | return 0; |
229 | } |
230 | |
231 | pr_info("isolcpus: Skipped unknown flag %.*s\n" , len, par); |
232 | str++; |
233 | } |
234 | |
235 | /* Default behaviour for isolcpus without flags */ |
236 | if (!flags) |
237 | flags |= HK_FLAG_DOMAIN; |
238 | |
239 | return housekeeping_setup(str, flags); |
240 | } |
241 | __setup("isolcpus=" , housekeeping_isolcpus_setup); |
242 | |