1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * cpuidle-powernv - idle state cpuidle driver. |
4 | * Adapted from drivers/cpuidle/cpuidle-pseries |
5 | * |
6 | */ |
7 | |
8 | #include <linux/kernel.h> |
9 | #include <linux/module.h> |
10 | #include <linux/init.h> |
11 | #include <linux/moduleparam.h> |
12 | #include <linux/cpuidle.h> |
13 | #include <linux/cpu.h> |
14 | #include <linux/notifier.h> |
15 | #include <linux/clockchips.h> |
16 | #include <linux/of.h> |
17 | #include <linux/slab.h> |
18 | |
19 | #include <asm/machdep.h> |
20 | #include <asm/firmware.h> |
21 | #include <asm/opal.h> |
22 | #include <asm/runlatch.h> |
23 | #include <asm/cpuidle.h> |
24 | |
25 | /* |
26 | * Expose only those Hardware idle states via the cpuidle framework |
27 | * that have latency value below POWERNV_THRESHOLD_LATENCY_NS. |
28 | */ |
29 | #define POWERNV_THRESHOLD_LATENCY_NS 200000 |
30 | |
31 | static struct cpuidle_driver powernv_idle_driver = { |
32 | .name = "powernv_idle" , |
33 | .owner = THIS_MODULE, |
34 | }; |
35 | |
36 | static int max_idle_state __read_mostly; |
37 | static struct cpuidle_state *cpuidle_state_table __read_mostly; |
38 | |
39 | struct stop_psscr_table { |
40 | u64 val; |
41 | u64 mask; |
42 | }; |
43 | |
44 | static struct stop_psscr_table stop_psscr_table[CPUIDLE_STATE_MAX] __read_mostly; |
45 | |
46 | static u64 default_snooze_timeout __read_mostly; |
47 | static bool snooze_timeout_en __read_mostly; |
48 | |
49 | static u64 get_snooze_timeout(struct cpuidle_device *dev, |
50 | struct cpuidle_driver *drv, |
51 | int index) |
52 | { |
53 | int i; |
54 | |
55 | if (unlikely(!snooze_timeout_en)) |
56 | return default_snooze_timeout; |
57 | |
58 | for (i = index + 1; i < drv->state_count; i++) { |
59 | if (dev->states_usage[i].disable) |
60 | continue; |
61 | |
62 | return drv->states[i].target_residency * tb_ticks_per_usec; |
63 | } |
64 | |
65 | return default_snooze_timeout; |
66 | } |
67 | |
68 | static int snooze_loop(struct cpuidle_device *dev, |
69 | struct cpuidle_driver *drv, |
70 | int index) |
71 | { |
72 | u64 snooze_exit_time; |
73 | |
74 | set_thread_flag(TIF_POLLING_NRFLAG); |
75 | |
76 | local_irq_enable(); |
77 | |
78 | snooze_exit_time = get_tb() + get_snooze_timeout(dev, drv, index); |
79 | dev->poll_time_limit = false; |
80 | ppc64_runlatch_off(); |
81 | HMT_very_low(); |
82 | while (!need_resched()) { |
83 | if (likely(snooze_timeout_en) && get_tb() > snooze_exit_time) { |
84 | /* |
85 | * Task has not woken up but we are exiting the polling |
86 | * loop anyway. Require a barrier after polling is |
87 | * cleared to order subsequent test of need_resched(). |
88 | */ |
89 | clear_thread_flag(TIF_POLLING_NRFLAG); |
90 | dev->poll_time_limit = true; |
91 | smp_mb(); |
92 | break; |
93 | } |
94 | } |
95 | |
96 | HMT_medium(); |
97 | ppc64_runlatch_on(); |
98 | clear_thread_flag(TIF_POLLING_NRFLAG); |
99 | |
100 | local_irq_disable(); |
101 | |
102 | return index; |
103 | } |
104 | |
105 | static int nap_loop(struct cpuidle_device *dev, |
106 | struct cpuidle_driver *drv, |
107 | int index) |
108 | { |
109 | power7_idle_type(PNV_THREAD_NAP); |
110 | |
111 | return index; |
112 | } |
113 | |
114 | /* Register for fastsleep only in oneshot mode of broadcast */ |
115 | #ifdef CONFIG_TICK_ONESHOT |
116 | static int fastsleep_loop(struct cpuidle_device *dev, |
117 | struct cpuidle_driver *drv, |
118 | int index) |
119 | { |
120 | unsigned long old_lpcr = mfspr(SPRN_LPCR); |
121 | unsigned long new_lpcr; |
122 | |
123 | if (unlikely(system_state < SYSTEM_RUNNING)) |
124 | return index; |
125 | |
126 | new_lpcr = old_lpcr; |
127 | /* Do not exit powersave upon decrementer as we've setup the timer |
128 | * offload. |
129 | */ |
130 | new_lpcr &= ~LPCR_PECE1; |
131 | |
132 | mtspr(SPRN_LPCR, new_lpcr); |
133 | |
134 | power7_idle_type(PNV_THREAD_SLEEP); |
135 | |
136 | mtspr(SPRN_LPCR, old_lpcr); |
137 | |
138 | return index; |
139 | } |
140 | #endif |
141 | |
142 | static int stop_loop(struct cpuidle_device *dev, |
143 | struct cpuidle_driver *drv, |
144 | int index) |
145 | { |
146 | arch300_idle_type(stop_psscr_table[index].val, |
147 | stop_psscr_table[index].mask); |
148 | return index; |
149 | } |
150 | |
151 | /* |
152 | * States for dedicated partition case. |
153 | */ |
154 | static struct cpuidle_state powernv_states[CPUIDLE_STATE_MAX] = { |
155 | { /* Snooze */ |
156 | .name = "snooze" , |
157 | .desc = "snooze" , |
158 | .exit_latency = 0, |
159 | .target_residency = 0, |
160 | .enter = snooze_loop, |
161 | .flags = CPUIDLE_FLAG_POLLING }, |
162 | }; |
163 | |
164 | static int powernv_cpuidle_cpu_online(unsigned int cpu) |
165 | { |
166 | struct cpuidle_device *dev = per_cpu(cpuidle_devices, cpu); |
167 | |
168 | if (dev && cpuidle_get_driver()) { |
169 | cpuidle_pause_and_lock(); |
170 | cpuidle_enable_device(dev); |
171 | cpuidle_resume_and_unlock(); |
172 | } |
173 | return 0; |
174 | } |
175 | |
176 | static int powernv_cpuidle_cpu_dead(unsigned int cpu) |
177 | { |
178 | struct cpuidle_device *dev = per_cpu(cpuidle_devices, cpu); |
179 | |
180 | if (dev && cpuidle_get_driver()) { |
181 | cpuidle_pause_and_lock(); |
182 | cpuidle_disable_device(dev); |
183 | cpuidle_resume_and_unlock(); |
184 | } |
185 | return 0; |
186 | } |
187 | |
188 | /* |
189 | * powernv_cpuidle_driver_init() |
190 | */ |
191 | static int powernv_cpuidle_driver_init(void) |
192 | { |
193 | int idle_state; |
194 | struct cpuidle_driver *drv = &powernv_idle_driver; |
195 | |
196 | drv->state_count = 0; |
197 | |
198 | for (idle_state = 0; idle_state < max_idle_state; ++idle_state) { |
199 | /* Is the state not enabled? */ |
200 | if (cpuidle_state_table[idle_state].enter == NULL) |
201 | continue; |
202 | |
203 | drv->states[drv->state_count] = /* structure copy */ |
204 | cpuidle_state_table[idle_state]; |
205 | |
206 | drv->state_count += 1; |
207 | } |
208 | |
209 | /* |
210 | * On the PowerNV platform cpu_present may be less than cpu_possible in |
211 | * cases when firmware detects the CPU, but it is not available to the |
212 | * OS. If CONFIG_HOTPLUG_CPU=n, then such CPUs are not hotplugable at |
213 | * run time and hence cpu_devices are not created for those CPUs by the |
214 | * generic topology_init(). |
215 | * |
216 | * drv->cpumask defaults to cpu_possible_mask in |
217 | * __cpuidle_driver_init(). This breaks cpuidle on PowerNV where |
218 | * cpu_devices are not created for CPUs in cpu_possible_mask that |
219 | * cannot be hot-added later at run time. |
220 | * |
221 | * Trying cpuidle_register_device() on a CPU without a cpu_device is |
222 | * incorrect, so pass a correct CPU mask to the generic cpuidle driver. |
223 | */ |
224 | |
225 | drv->cpumask = (struct cpumask *)cpu_present_mask; |
226 | |
227 | return 0; |
228 | } |
229 | |
230 | static inline void add_powernv_state(int index, const char *name, |
231 | unsigned int flags, |
232 | int (*idle_fn)(struct cpuidle_device *, |
233 | struct cpuidle_driver *, |
234 | int), |
235 | unsigned int target_residency, |
236 | unsigned int exit_latency, |
237 | u64 psscr_val, u64 psscr_mask) |
238 | { |
239 | strscpy(p: powernv_states[index].name, q: name, CPUIDLE_NAME_LEN); |
240 | strscpy(p: powernv_states[index].desc, q: name, CPUIDLE_NAME_LEN); |
241 | powernv_states[index].flags = flags; |
242 | powernv_states[index].target_residency = target_residency; |
243 | powernv_states[index].exit_latency = exit_latency; |
244 | powernv_states[index].enter = idle_fn; |
245 | /* For power8 and below psscr_* will be 0 */ |
246 | stop_psscr_table[index].val = psscr_val; |
247 | stop_psscr_table[index].mask = psscr_mask; |
248 | } |
249 | |
250 | extern u32 pnv_get_supported_cpuidle_states(void); |
251 | static int powernv_add_idle_states(void) |
252 | { |
253 | int nr_idle_states = 1; /* Snooze */ |
254 | int dt_idle_states; |
255 | u32 has_stop_states = 0; |
256 | int i; |
257 | u32 supported_flags = pnv_get_supported_cpuidle_states(); |
258 | |
259 | |
260 | /* Currently we have snooze statically defined */ |
261 | if (nr_pnv_idle_states <= 0) { |
262 | pr_warn("cpuidle-powernv : Only Snooze is available\n" ); |
263 | goto out; |
264 | } |
265 | |
266 | /* TODO: Count only states which are eligible for cpuidle */ |
267 | dt_idle_states = nr_pnv_idle_states; |
268 | |
269 | /* |
270 | * Since snooze is used as first idle state, max idle states allowed is |
271 | * CPUIDLE_STATE_MAX -1 |
272 | */ |
273 | if (nr_pnv_idle_states > CPUIDLE_STATE_MAX - 1) { |
274 | pr_warn("cpuidle-powernv: discovered idle states more than allowed" ); |
275 | dt_idle_states = CPUIDLE_STATE_MAX - 1; |
276 | } |
277 | |
278 | /* |
279 | * If the idle states use stop instruction, probe for psscr values |
280 | * and psscr mask which are necessary to specify required stop level. |
281 | */ |
282 | has_stop_states = (pnv_idle_states[0].flags & |
283 | (OPAL_PM_STOP_INST_FAST | OPAL_PM_STOP_INST_DEEP)); |
284 | |
285 | for (i = 0; i < dt_idle_states; i++) { |
286 | unsigned int exit_latency, target_residency; |
287 | bool stops_timebase = false; |
288 | struct pnv_idle_states_t *state = &pnv_idle_states[i]; |
289 | |
290 | /* |
291 | * Skip the platform idle state whose flag isn't in |
292 | * the supported_cpuidle_states flag mask. |
293 | */ |
294 | if ((state->flags & supported_flags) != state->flags) |
295 | continue; |
296 | /* |
297 | * If an idle state has exit latency beyond |
298 | * POWERNV_THRESHOLD_LATENCY_NS then don't use it |
299 | * in cpu-idle. |
300 | */ |
301 | if (state->latency_ns > POWERNV_THRESHOLD_LATENCY_NS) |
302 | continue; |
303 | /* |
304 | * Firmware passes residency and latency values in ns. |
305 | * cpuidle expects it in us. |
306 | */ |
307 | exit_latency = DIV_ROUND_UP(state->latency_ns, 1000); |
308 | target_residency = DIV_ROUND_UP(state->residency_ns, 1000); |
309 | |
310 | if (has_stop_states && !(state->valid)) |
311 | continue; |
312 | |
313 | if (state->flags & OPAL_PM_TIMEBASE_STOP) |
314 | stops_timebase = true; |
315 | |
316 | if (state->flags & OPAL_PM_NAP_ENABLED) { |
317 | /* Add NAP state */ |
318 | add_powernv_state(index: nr_idle_states, name: "Nap" , |
319 | CPUIDLE_FLAG_NONE, idle_fn: nap_loop, |
320 | target_residency, exit_latency, psscr_val: 0, psscr_mask: 0); |
321 | } else if (has_stop_states && !stops_timebase) { |
322 | add_powernv_state(index: nr_idle_states, name: state->name, |
323 | CPUIDLE_FLAG_NONE, idle_fn: stop_loop, |
324 | target_residency, exit_latency, |
325 | psscr_val: state->psscr_val, |
326 | psscr_mask: state->psscr_mask); |
327 | } |
328 | |
329 | /* |
330 | * All cpuidle states with CPUIDLE_FLAG_TIMER_STOP set must come |
331 | * within this config dependency check. |
332 | */ |
333 | #ifdef CONFIG_TICK_ONESHOT |
334 | else if (state->flags & OPAL_PM_SLEEP_ENABLED || |
335 | state->flags & OPAL_PM_SLEEP_ENABLED_ER1) { |
336 | /* Add FASTSLEEP state */ |
337 | add_powernv_state(index: nr_idle_states, name: "FastSleep" , |
338 | CPUIDLE_FLAG_TIMER_STOP, |
339 | idle_fn: fastsleep_loop, |
340 | target_residency, exit_latency, psscr_val: 0, psscr_mask: 0); |
341 | } else if (has_stop_states && stops_timebase) { |
342 | add_powernv_state(index: nr_idle_states, name: state->name, |
343 | CPUIDLE_FLAG_TIMER_STOP, idle_fn: stop_loop, |
344 | target_residency, exit_latency, |
345 | psscr_val: state->psscr_val, |
346 | psscr_mask: state->psscr_mask); |
347 | } |
348 | #endif |
349 | else |
350 | continue; |
351 | nr_idle_states++; |
352 | } |
353 | out: |
354 | return nr_idle_states; |
355 | } |
356 | |
357 | /* |
358 | * powernv_idle_probe() |
359 | * Choose state table for shared versus dedicated partition |
360 | */ |
361 | static int powernv_idle_probe(void) |
362 | { |
363 | if (cpuidle_disable != IDLE_NO_OVERRIDE) |
364 | return -ENODEV; |
365 | |
366 | if (firmware_has_feature(FW_FEATURE_OPAL)) { |
367 | cpuidle_state_table = powernv_states; |
368 | /* Device tree can indicate more idle states */ |
369 | max_idle_state = powernv_add_idle_states(); |
370 | default_snooze_timeout = TICK_USEC * tb_ticks_per_usec; |
371 | if (max_idle_state > 1) |
372 | snooze_timeout_en = true; |
373 | } else |
374 | return -ENODEV; |
375 | |
376 | return 0; |
377 | } |
378 | |
379 | static int __init powernv_processor_idle_init(void) |
380 | { |
381 | int retval; |
382 | |
383 | retval = powernv_idle_probe(); |
384 | if (retval) |
385 | return retval; |
386 | |
387 | powernv_cpuidle_driver_init(); |
388 | retval = cpuidle_register(drv: &powernv_idle_driver, NULL); |
389 | if (retval) { |
390 | printk(KERN_DEBUG "Registration of powernv driver failed.\n" ); |
391 | return retval; |
392 | } |
393 | |
394 | retval = cpuhp_setup_state_nocalls(state: CPUHP_AP_ONLINE_DYN, |
395 | name: "cpuidle/powernv:online" , |
396 | startup: powernv_cpuidle_cpu_online, NULL); |
397 | WARN_ON(retval < 0); |
398 | retval = cpuhp_setup_state_nocalls(state: CPUHP_CPUIDLE_DEAD, |
399 | name: "cpuidle/powernv:dead" , NULL, |
400 | teardown: powernv_cpuidle_cpu_dead); |
401 | WARN_ON(retval < 0); |
402 | printk(KERN_DEBUG "powernv_idle_driver registered\n" ); |
403 | return 0; |
404 | } |
405 | |
406 | device_initcall(powernv_processor_idle_init); |
407 | |