1 | /* |
2 | * cpuidle.c - core cpuidle infrastructure |
3 | * |
4 | * (C) 2006-2007 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com> |
5 | * Shaohua Li <shaohua.li@intel.com> |
6 | * Adam Belay <abelay@novell.com> |
7 | * |
8 | * This code is licenced under the GPL. |
9 | */ |
10 | |
11 | #include "linux/percpu-defs.h" |
12 | #include <linux/clockchips.h> |
13 | #include <linux/kernel.h> |
14 | #include <linux/mutex.h> |
15 | #include <linux/sched.h> |
16 | #include <linux/sched/clock.h> |
17 | #include <linux/sched/idle.h> |
18 | #include <linux/notifier.h> |
19 | #include <linux/pm_qos.h> |
20 | #include <linux/cpu.h> |
21 | #include <linux/cpuidle.h> |
22 | #include <linux/ktime.h> |
23 | #include <linux/hrtimer.h> |
24 | #include <linux/module.h> |
25 | #include <linux/suspend.h> |
26 | #include <linux/tick.h> |
27 | #include <linux/mmu_context.h> |
28 | #include <linux/context_tracking.h> |
29 | #include <trace/events/power.h> |
30 | |
31 | #include "cpuidle.h" |
32 | |
33 | DEFINE_PER_CPU(struct cpuidle_device *, cpuidle_devices); |
34 | DEFINE_PER_CPU(struct cpuidle_device, cpuidle_dev); |
35 | |
36 | DEFINE_MUTEX(cpuidle_lock); |
37 | LIST_HEAD(cpuidle_detected_devices); |
38 | |
39 | static int enabled_devices; |
40 | static int off __read_mostly; |
41 | static int initialized __read_mostly; |
42 | |
43 | int cpuidle_disabled(void) |
44 | { |
45 | return off; |
46 | } |
47 | void disable_cpuidle(void) |
48 | { |
49 | off = 1; |
50 | } |
51 | |
52 | bool cpuidle_not_available(struct cpuidle_driver *drv, |
53 | struct cpuidle_device *dev) |
54 | { |
55 | return off || !initialized || !drv || !dev || !dev->enabled; |
56 | } |
57 | |
58 | /** |
59 | * cpuidle_play_dead - cpu off-lining |
60 | * |
61 | * Returns in case of an error or no driver |
62 | */ |
63 | int cpuidle_play_dead(void) |
64 | { |
65 | struct cpuidle_device *dev = __this_cpu_read(cpuidle_devices); |
66 | struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev); |
67 | int i; |
68 | |
69 | if (!drv) |
70 | return -ENODEV; |
71 | |
72 | /* Find lowest-power state that supports long-term idle */ |
73 | for (i = drv->state_count - 1; i >= 0; i--) |
74 | if (drv->states[i].enter_dead) |
75 | return drv->states[i].enter_dead(dev, i); |
76 | |
77 | return -ENODEV; |
78 | } |
79 | |
80 | static int find_deepest_state(struct cpuidle_driver *drv, |
81 | struct cpuidle_device *dev, |
82 | u64 max_latency_ns, |
83 | unsigned int forbidden_flags, |
84 | bool s2idle) |
85 | { |
86 | u64 latency_req = 0; |
87 | int i, ret = 0; |
88 | |
89 | for (i = 1; i < drv->state_count; i++) { |
90 | struct cpuidle_state *s = &drv->states[i]; |
91 | |
92 | if (dev->states_usage[i].disable || |
93 | s->exit_latency_ns <= latency_req || |
94 | s->exit_latency_ns > max_latency_ns || |
95 | (s->flags & forbidden_flags) || |
96 | (s2idle && !s->enter_s2idle)) |
97 | continue; |
98 | |
99 | latency_req = s->exit_latency_ns; |
100 | ret = i; |
101 | } |
102 | return ret; |
103 | } |
104 | |
105 | /** |
106 | * cpuidle_use_deepest_state - Set/unset governor override mode. |
107 | * @latency_limit_ns: Idle state exit latency limit (or no override if 0). |
108 | * |
109 | * If @latency_limit_ns is nonzero, set the current CPU to use the deepest idle |
110 | * state with exit latency within @latency_limit_ns (override governors going |
111 | * forward), or do not override governors if it is zero. |
112 | */ |
113 | void cpuidle_use_deepest_state(u64 latency_limit_ns) |
114 | { |
115 | struct cpuidle_device *dev; |
116 | |
117 | preempt_disable(); |
118 | dev = cpuidle_get_device(); |
119 | if (dev) |
120 | dev->forced_idle_latency_limit_ns = latency_limit_ns; |
121 | preempt_enable(); |
122 | } |
123 | |
124 | /** |
125 | * cpuidle_find_deepest_state - Find the deepest available idle state. |
126 | * @drv: cpuidle driver for the given CPU. |
127 | * @dev: cpuidle device for the given CPU. |
128 | * @latency_limit_ns: Idle state exit latency limit |
129 | * |
130 | * Return: the index of the deepest available idle state. |
131 | */ |
132 | int cpuidle_find_deepest_state(struct cpuidle_driver *drv, |
133 | struct cpuidle_device *dev, |
134 | u64 latency_limit_ns) |
135 | { |
136 | return find_deepest_state(drv, dev, max_latency_ns: latency_limit_ns, forbidden_flags: 0, s2idle: false); |
137 | } |
138 | |
139 | #ifdef CONFIG_SUSPEND |
140 | static noinstr void enter_s2idle_proper(struct cpuidle_driver *drv, |
141 | struct cpuidle_device *dev, int index) |
142 | { |
143 | struct cpuidle_state *target_state = &drv->states[index]; |
144 | ktime_t time_start, time_end; |
145 | |
146 | instrumentation_begin(); |
147 | |
148 | time_start = ns_to_ktime(ns: local_clock_noinstr()); |
149 | |
150 | tick_freeze(); |
151 | /* |
152 | * The state used here cannot be a "coupled" one, because the "coupled" |
153 | * cpuidle mechanism enables interrupts and doing that with timekeeping |
154 | * suspended is generally unsafe. |
155 | */ |
156 | stop_critical_timings(); |
157 | if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE)) { |
158 | ct_cpuidle_enter(); |
159 | /* Annotate away the indirect call */ |
160 | instrumentation_begin(); |
161 | } |
162 | target_state->enter_s2idle(dev, drv, index); |
163 | if (WARN_ON_ONCE(!irqs_disabled())) |
164 | raw_local_irq_disable(); |
165 | if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE)) { |
166 | instrumentation_end(); |
167 | ct_cpuidle_exit(); |
168 | } |
169 | tick_unfreeze(); |
170 | start_critical_timings(); |
171 | |
172 | time_end = ns_to_ktime(ns: local_clock_noinstr()); |
173 | |
174 | dev->states_usage[index].s2idle_time += ktime_us_delta(later: time_end, earlier: time_start); |
175 | dev->states_usage[index].s2idle_usage++; |
176 | instrumentation_end(); |
177 | } |
178 | |
179 | /** |
180 | * cpuidle_enter_s2idle - Enter an idle state suitable for suspend-to-idle. |
181 | * @drv: cpuidle driver for the given CPU. |
182 | * @dev: cpuidle device for the given CPU. |
183 | * |
184 | * If there are states with the ->enter_s2idle callback, find the deepest of |
185 | * them and enter it with frozen tick. |
186 | */ |
187 | int cpuidle_enter_s2idle(struct cpuidle_driver *drv, struct cpuidle_device *dev) |
188 | { |
189 | int index; |
190 | |
191 | /* |
192 | * Find the deepest state with ->enter_s2idle present, which guarantees |
193 | * that interrupts won't be enabled when it exits and allows the tick to |
194 | * be frozen safely. |
195 | */ |
196 | index = find_deepest_state(drv, dev, U64_MAX, forbidden_flags: 0, s2idle: true); |
197 | if (index > 0) { |
198 | enter_s2idle_proper(drv, dev, index); |
199 | local_irq_enable(); |
200 | } |
201 | return index; |
202 | } |
203 | #endif /* CONFIG_SUSPEND */ |
204 | |
205 | /** |
206 | * cpuidle_enter_state - enter the state and update stats |
207 | * @dev: cpuidle device for this cpu |
208 | * @drv: cpuidle driver for this cpu |
209 | * @index: index into the states table in @drv of the state to enter |
210 | */ |
211 | noinstr int cpuidle_enter_state(struct cpuidle_device *dev, |
212 | struct cpuidle_driver *drv, |
213 | int index) |
214 | { |
215 | int entered_state; |
216 | |
217 | struct cpuidle_state *target_state = &drv->states[index]; |
218 | bool broadcast = !!(target_state->flags & CPUIDLE_FLAG_TIMER_STOP); |
219 | ktime_t time_start, time_end; |
220 | |
221 | instrumentation_begin(); |
222 | |
223 | /* |
224 | * Tell the time framework to switch to a broadcast timer because our |
225 | * local timer will be shut down. If a local timer is used from another |
226 | * CPU as a broadcast timer, this call may fail if it is not available. |
227 | */ |
228 | if (broadcast && tick_broadcast_enter()) { |
229 | index = find_deepest_state(drv, dev, max_latency_ns: target_state->exit_latency_ns, |
230 | CPUIDLE_FLAG_TIMER_STOP, s2idle: false); |
231 | if (index < 0) { |
232 | default_idle_call(); |
233 | return -EBUSY; |
234 | } |
235 | target_state = &drv->states[index]; |
236 | broadcast = false; |
237 | } |
238 | |
239 | if (target_state->flags & CPUIDLE_FLAG_TLB_FLUSHED) |
240 | leave_mm(cpu: dev->cpu); |
241 | |
242 | /* Take note of the planned idle state. */ |
243 | sched_idle_set_state(idle_state: target_state); |
244 | |
245 | trace_cpu_idle(state: index, cpu_id: dev->cpu); |
246 | time_start = ns_to_ktime(ns: local_clock_noinstr()); |
247 | |
248 | stop_critical_timings(); |
249 | if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE)) { |
250 | ct_cpuidle_enter(); |
251 | /* Annotate away the indirect call */ |
252 | instrumentation_begin(); |
253 | } |
254 | |
255 | /* |
256 | * NOTE!! |
257 | * |
258 | * For cpuidle_state::enter() methods that do *NOT* set |
259 | * CPUIDLE_FLAG_RCU_IDLE RCU will be disabled here and these functions |
260 | * must be marked either noinstr or __cpuidle. |
261 | * |
262 | * For cpuidle_state::enter() methods that *DO* set |
263 | * CPUIDLE_FLAG_RCU_IDLE this isn't required, but they must mark the |
264 | * function calling ct_cpuidle_enter() as noinstr/__cpuidle and all |
265 | * functions called within the RCU-idle region. |
266 | */ |
267 | entered_state = target_state->enter(dev, drv, index); |
268 | |
269 | if (WARN_ONCE(!irqs_disabled(), "%ps leaked IRQ state" , target_state->enter)) |
270 | raw_local_irq_disable(); |
271 | |
272 | if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE)) { |
273 | instrumentation_end(); |
274 | ct_cpuidle_exit(); |
275 | } |
276 | start_critical_timings(); |
277 | |
278 | sched_clock_idle_wakeup_event(); |
279 | time_end = ns_to_ktime(ns: local_clock_noinstr()); |
280 | trace_cpu_idle(PWR_EVENT_EXIT, cpu_id: dev->cpu); |
281 | |
282 | /* The cpu is no longer idle or about to enter idle. */ |
283 | sched_idle_set_state(NULL); |
284 | |
285 | if (broadcast) |
286 | tick_broadcast_exit(); |
287 | |
288 | if (!cpuidle_state_is_coupled(drv, state: index)) |
289 | local_irq_enable(); |
290 | |
291 | if (entered_state >= 0) { |
292 | s64 diff, delay = drv->states[entered_state].exit_latency_ns; |
293 | int i; |
294 | |
295 | /* |
296 | * Update cpuidle counters |
297 | * This can be moved to within driver enter routine, |
298 | * but that results in multiple copies of same code. |
299 | */ |
300 | diff = ktime_sub(time_end, time_start); |
301 | |
302 | dev->last_residency_ns = diff; |
303 | dev->states_usage[entered_state].time_ns += diff; |
304 | dev->states_usage[entered_state].usage++; |
305 | |
306 | if (diff < drv->states[entered_state].target_residency_ns) { |
307 | for (i = entered_state - 1; i >= 0; i--) { |
308 | if (dev->states_usage[i].disable) |
309 | continue; |
310 | |
311 | /* Shallower states are enabled, so update. */ |
312 | dev->states_usage[entered_state].above++; |
313 | trace_cpu_idle_miss(cpu_id: dev->cpu, state: entered_state, below: false); |
314 | break; |
315 | } |
316 | } else if (diff > delay) { |
317 | for (i = entered_state + 1; i < drv->state_count; i++) { |
318 | if (dev->states_usage[i].disable) |
319 | continue; |
320 | |
321 | /* |
322 | * Update if a deeper state would have been a |
323 | * better match for the observed idle duration. |
324 | */ |
325 | if (diff - delay >= drv->states[i].target_residency_ns) { |
326 | dev->states_usage[entered_state].below++; |
327 | trace_cpu_idle_miss(cpu_id: dev->cpu, state: entered_state, below: true); |
328 | } |
329 | |
330 | break; |
331 | } |
332 | } |
333 | } else { |
334 | dev->last_residency_ns = 0; |
335 | dev->states_usage[index].rejected++; |
336 | } |
337 | |
338 | instrumentation_end(); |
339 | |
340 | return entered_state; |
341 | } |
342 | |
343 | /** |
344 | * cpuidle_select - ask the cpuidle framework to choose an idle state |
345 | * |
346 | * @drv: the cpuidle driver |
347 | * @dev: the cpuidle device |
348 | * @stop_tick: indication on whether or not to stop the tick |
349 | * |
350 | * Returns the index of the idle state. The return value must not be negative. |
351 | * |
352 | * The memory location pointed to by @stop_tick is expected to be written the |
353 | * 'false' boolean value if the scheduler tick should not be stopped before |
354 | * entering the returned state. |
355 | */ |
356 | int cpuidle_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, |
357 | bool *stop_tick) |
358 | { |
359 | return cpuidle_curr_governor->select(drv, dev, stop_tick); |
360 | } |
361 | |
362 | /** |
363 | * cpuidle_enter - enter into the specified idle state |
364 | * |
365 | * @drv: the cpuidle driver tied with the cpu |
366 | * @dev: the cpuidle device |
367 | * @index: the index in the idle state table |
368 | * |
369 | * Returns the index in the idle state, < 0 in case of error. |
370 | * The error code depends on the backend driver |
371 | */ |
372 | int cpuidle_enter(struct cpuidle_driver *drv, struct cpuidle_device *dev, |
373 | int index) |
374 | { |
375 | int ret = 0; |
376 | |
377 | /* |
378 | * Store the next hrtimer, which becomes either next tick or the next |
379 | * timer event, whatever expires first. Additionally, to make this data |
380 | * useful for consumers outside cpuidle, we rely on that the governor's |
381 | * ->select() callback have decided, whether to stop the tick or not. |
382 | */ |
383 | WRITE_ONCE(dev->next_hrtimer, tick_nohz_get_next_hrtimer()); |
384 | |
385 | if (cpuidle_state_is_coupled(drv, state: index)) |
386 | ret = cpuidle_enter_state_coupled(dev, drv, next_state: index); |
387 | else |
388 | ret = cpuidle_enter_state(dev, drv, index); |
389 | |
390 | WRITE_ONCE(dev->next_hrtimer, 0); |
391 | return ret; |
392 | } |
393 | |
394 | /** |
395 | * cpuidle_reflect - tell the underlying governor what was the state |
396 | * we were in |
397 | * |
398 | * @dev : the cpuidle device |
399 | * @index: the index in the idle state table |
400 | * |
401 | */ |
402 | void cpuidle_reflect(struct cpuidle_device *dev, int index) |
403 | { |
404 | if (cpuidle_curr_governor->reflect && index >= 0) |
405 | cpuidle_curr_governor->reflect(dev, index); |
406 | } |
407 | |
408 | /* |
409 | * Min polling interval of 10usec is a guess. It is assuming that |
410 | * for most users, the time for a single ping-pong workload like |
411 | * perf bench pipe would generally complete within 10usec but |
412 | * this is hardware dependant. Actual time can be estimated with |
413 | * |
414 | * perf bench sched pipe -l 10000 |
415 | * |
416 | * Run multiple times to avoid cpufreq effects. |
417 | */ |
418 | #define CPUIDLE_POLL_MIN 10000 |
419 | #define CPUIDLE_POLL_MAX (TICK_NSEC / 16) |
420 | |
421 | /** |
422 | * cpuidle_poll_time - return amount of time to poll for, |
423 | * governors can override dev->poll_limit_ns if necessary |
424 | * |
425 | * @drv: the cpuidle driver tied with the cpu |
426 | * @dev: the cpuidle device |
427 | * |
428 | */ |
429 | __cpuidle u64 cpuidle_poll_time(struct cpuidle_driver *drv, |
430 | struct cpuidle_device *dev) |
431 | { |
432 | int i; |
433 | u64 limit_ns; |
434 | |
435 | BUILD_BUG_ON(CPUIDLE_POLL_MIN > CPUIDLE_POLL_MAX); |
436 | |
437 | if (dev->poll_limit_ns) |
438 | return dev->poll_limit_ns; |
439 | |
440 | limit_ns = CPUIDLE_POLL_MAX; |
441 | for (i = 1; i < drv->state_count; i++) { |
442 | u64 state_limit; |
443 | |
444 | if (dev->states_usage[i].disable) |
445 | continue; |
446 | |
447 | state_limit = drv->states[i].target_residency_ns; |
448 | if (state_limit < CPUIDLE_POLL_MIN) |
449 | continue; |
450 | |
451 | limit_ns = min_t(u64, state_limit, CPUIDLE_POLL_MAX); |
452 | break; |
453 | } |
454 | |
455 | dev->poll_limit_ns = limit_ns; |
456 | |
457 | return dev->poll_limit_ns; |
458 | } |
459 | |
460 | /** |
461 | * cpuidle_install_idle_handler - installs the cpuidle idle loop handler |
462 | */ |
463 | void cpuidle_install_idle_handler(void) |
464 | { |
465 | if (enabled_devices) { |
466 | /* Make sure all changes finished before we switch to new idle */ |
467 | smp_wmb(); |
468 | initialized = 1; |
469 | } |
470 | } |
471 | |
472 | /** |
473 | * cpuidle_uninstall_idle_handler - uninstalls the cpuidle idle loop handler |
474 | */ |
475 | void cpuidle_uninstall_idle_handler(void) |
476 | { |
477 | if (enabled_devices) { |
478 | initialized = 0; |
479 | wake_up_all_idle_cpus(); |
480 | } |
481 | |
482 | /* |
483 | * Make sure external observers (such as the scheduler) |
484 | * are done looking at pointed idle states. |
485 | */ |
486 | synchronize_rcu(); |
487 | } |
488 | |
489 | /** |
490 | * cpuidle_pause_and_lock - temporarily disables CPUIDLE |
491 | */ |
492 | void cpuidle_pause_and_lock(void) |
493 | { |
494 | mutex_lock(&cpuidle_lock); |
495 | cpuidle_uninstall_idle_handler(); |
496 | } |
497 | |
498 | EXPORT_SYMBOL_GPL(cpuidle_pause_and_lock); |
499 | |
500 | /** |
501 | * cpuidle_resume_and_unlock - resumes CPUIDLE operation |
502 | */ |
503 | void cpuidle_resume_and_unlock(void) |
504 | { |
505 | cpuidle_install_idle_handler(); |
506 | mutex_unlock(lock: &cpuidle_lock); |
507 | } |
508 | |
509 | EXPORT_SYMBOL_GPL(cpuidle_resume_and_unlock); |
510 | |
511 | /* Currently used in suspend/resume path to suspend cpuidle */ |
512 | void cpuidle_pause(void) |
513 | { |
514 | mutex_lock(&cpuidle_lock); |
515 | cpuidle_uninstall_idle_handler(); |
516 | mutex_unlock(lock: &cpuidle_lock); |
517 | } |
518 | |
519 | /* Currently used in suspend/resume path to resume cpuidle */ |
520 | void cpuidle_resume(void) |
521 | { |
522 | mutex_lock(&cpuidle_lock); |
523 | cpuidle_install_idle_handler(); |
524 | mutex_unlock(lock: &cpuidle_lock); |
525 | } |
526 | |
527 | /** |
528 | * cpuidle_enable_device - enables idle PM for a CPU |
529 | * @dev: the CPU |
530 | * |
531 | * This function must be called between cpuidle_pause_and_lock and |
532 | * cpuidle_resume_and_unlock when used externally. |
533 | */ |
534 | int cpuidle_enable_device(struct cpuidle_device *dev) |
535 | { |
536 | int ret; |
537 | struct cpuidle_driver *drv; |
538 | |
539 | if (!dev) |
540 | return -EINVAL; |
541 | |
542 | if (dev->enabled) |
543 | return 0; |
544 | |
545 | if (!cpuidle_curr_governor) |
546 | return -EIO; |
547 | |
548 | drv = cpuidle_get_cpu_driver(dev); |
549 | |
550 | if (!drv) |
551 | return -EIO; |
552 | |
553 | if (!dev->registered) |
554 | return -EINVAL; |
555 | |
556 | ret = cpuidle_add_device_sysfs(device: dev); |
557 | if (ret) |
558 | return ret; |
559 | |
560 | if (cpuidle_curr_governor->enable) { |
561 | ret = cpuidle_curr_governor->enable(drv, dev); |
562 | if (ret) |
563 | goto fail_sysfs; |
564 | } |
565 | |
566 | smp_wmb(); |
567 | |
568 | dev->enabled = 1; |
569 | |
570 | enabled_devices++; |
571 | return 0; |
572 | |
573 | fail_sysfs: |
574 | cpuidle_remove_device_sysfs(device: dev); |
575 | |
576 | return ret; |
577 | } |
578 | |
579 | EXPORT_SYMBOL_GPL(cpuidle_enable_device); |
580 | |
581 | /** |
582 | * cpuidle_disable_device - disables idle PM for a CPU |
583 | * @dev: the CPU |
584 | * |
585 | * This function must be called between cpuidle_pause_and_lock and |
586 | * cpuidle_resume_and_unlock when used externally. |
587 | */ |
588 | void cpuidle_disable_device(struct cpuidle_device *dev) |
589 | { |
590 | struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev); |
591 | |
592 | if (!dev || !dev->enabled) |
593 | return; |
594 | |
595 | if (!drv || !cpuidle_curr_governor) |
596 | return; |
597 | |
598 | dev->enabled = 0; |
599 | |
600 | if (cpuidle_curr_governor->disable) |
601 | cpuidle_curr_governor->disable(drv, dev); |
602 | |
603 | cpuidle_remove_device_sysfs(device: dev); |
604 | enabled_devices--; |
605 | } |
606 | |
607 | EXPORT_SYMBOL_GPL(cpuidle_disable_device); |
608 | |
609 | static void __cpuidle_unregister_device(struct cpuidle_device *dev) |
610 | { |
611 | struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev); |
612 | |
613 | list_del(entry: &dev->device_list); |
614 | per_cpu(cpuidle_devices, dev->cpu) = NULL; |
615 | module_put(module: drv->owner); |
616 | |
617 | dev->registered = 0; |
618 | } |
619 | |
620 | static void __cpuidle_device_init(struct cpuidle_device *dev) |
621 | { |
622 | memset(dev->states_usage, 0, sizeof(dev->states_usage)); |
623 | dev->last_residency_ns = 0; |
624 | dev->next_hrtimer = 0; |
625 | } |
626 | |
627 | /** |
628 | * __cpuidle_register_device - internal register function called before register |
629 | * and enable routines |
630 | * @dev: the cpu |
631 | * |
632 | * cpuidle_lock mutex must be held before this is called |
633 | */ |
634 | static int __cpuidle_register_device(struct cpuidle_device *dev) |
635 | { |
636 | struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev); |
637 | int i, ret; |
638 | |
639 | if (!try_module_get(module: drv->owner)) |
640 | return -EINVAL; |
641 | |
642 | for (i = 0; i < drv->state_count; i++) { |
643 | if (drv->states[i].flags & CPUIDLE_FLAG_UNUSABLE) |
644 | dev->states_usage[i].disable |= CPUIDLE_STATE_DISABLED_BY_DRIVER; |
645 | |
646 | if (drv->states[i].flags & CPUIDLE_FLAG_OFF) |
647 | dev->states_usage[i].disable |= CPUIDLE_STATE_DISABLED_BY_USER; |
648 | } |
649 | |
650 | per_cpu(cpuidle_devices, dev->cpu) = dev; |
651 | list_add(new: &dev->device_list, head: &cpuidle_detected_devices); |
652 | |
653 | ret = cpuidle_coupled_register_device(dev); |
654 | if (ret) |
655 | __cpuidle_unregister_device(dev); |
656 | else |
657 | dev->registered = 1; |
658 | |
659 | return ret; |
660 | } |
661 | |
662 | /** |
663 | * cpuidle_register_device - registers a CPU's idle PM feature |
664 | * @dev: the cpu |
665 | */ |
666 | int cpuidle_register_device(struct cpuidle_device *dev) |
667 | { |
668 | int ret = -EBUSY; |
669 | |
670 | if (!dev) |
671 | return -EINVAL; |
672 | |
673 | mutex_lock(&cpuidle_lock); |
674 | |
675 | if (dev->registered) |
676 | goto out_unlock; |
677 | |
678 | __cpuidle_device_init(dev); |
679 | |
680 | ret = __cpuidle_register_device(dev); |
681 | if (ret) |
682 | goto out_unlock; |
683 | |
684 | ret = cpuidle_add_sysfs(dev); |
685 | if (ret) |
686 | goto out_unregister; |
687 | |
688 | ret = cpuidle_enable_device(dev); |
689 | if (ret) |
690 | goto out_sysfs; |
691 | |
692 | cpuidle_install_idle_handler(); |
693 | |
694 | out_unlock: |
695 | mutex_unlock(lock: &cpuidle_lock); |
696 | |
697 | return ret; |
698 | |
699 | out_sysfs: |
700 | cpuidle_remove_sysfs(dev); |
701 | out_unregister: |
702 | __cpuidle_unregister_device(dev); |
703 | goto out_unlock; |
704 | } |
705 | |
706 | EXPORT_SYMBOL_GPL(cpuidle_register_device); |
707 | |
708 | /** |
709 | * cpuidle_unregister_device - unregisters a CPU's idle PM feature |
710 | * @dev: the cpu |
711 | */ |
712 | void cpuidle_unregister_device(struct cpuidle_device *dev) |
713 | { |
714 | if (!dev || dev->registered == 0) |
715 | return; |
716 | |
717 | cpuidle_pause_and_lock(); |
718 | |
719 | cpuidle_disable_device(dev); |
720 | |
721 | cpuidle_remove_sysfs(dev); |
722 | |
723 | __cpuidle_unregister_device(dev); |
724 | |
725 | cpuidle_coupled_unregister_device(dev); |
726 | |
727 | cpuidle_resume_and_unlock(); |
728 | } |
729 | |
730 | EXPORT_SYMBOL_GPL(cpuidle_unregister_device); |
731 | |
732 | /** |
733 | * cpuidle_unregister: unregister a driver and the devices. This function |
734 | * can be used only if the driver has been previously registered through |
735 | * the cpuidle_register function. |
736 | * |
737 | * @drv: a valid pointer to a struct cpuidle_driver |
738 | */ |
739 | void cpuidle_unregister(struct cpuidle_driver *drv) |
740 | { |
741 | int cpu; |
742 | struct cpuidle_device *device; |
743 | |
744 | for_each_cpu(cpu, drv->cpumask) { |
745 | device = &per_cpu(cpuidle_dev, cpu); |
746 | cpuidle_unregister_device(device); |
747 | } |
748 | |
749 | cpuidle_unregister_driver(drv); |
750 | } |
751 | EXPORT_SYMBOL_GPL(cpuidle_unregister); |
752 | |
753 | /** |
754 | * cpuidle_register: registers the driver and the cpu devices with the |
755 | * coupled_cpus passed as parameter. This function is used for all common |
756 | * initialization pattern there are in the arch specific drivers. The |
757 | * devices is globally defined in this file. |
758 | * |
759 | * @drv : a valid pointer to a struct cpuidle_driver |
760 | * @coupled_cpus: a cpumask for the coupled states |
761 | * |
762 | * Returns 0 on success, < 0 otherwise |
763 | */ |
764 | int cpuidle_register(struct cpuidle_driver *drv, |
765 | const struct cpumask *const coupled_cpus) |
766 | { |
767 | int ret, cpu; |
768 | struct cpuidle_device *device; |
769 | |
770 | ret = cpuidle_register_driver(drv); |
771 | if (ret) { |
772 | pr_err("failed to register cpuidle driver\n" ); |
773 | return ret; |
774 | } |
775 | |
776 | for_each_cpu(cpu, drv->cpumask) { |
777 | device = &per_cpu(cpuidle_dev, cpu); |
778 | device->cpu = cpu; |
779 | |
780 | #ifdef CONFIG_ARCH_NEEDS_CPU_IDLE_COUPLED |
781 | /* |
782 | * On multiplatform for ARM, the coupled idle states could be |
783 | * enabled in the kernel even if the cpuidle driver does not |
784 | * use it. Note, coupled_cpus is a struct copy. |
785 | */ |
786 | if (coupled_cpus) |
787 | device->coupled_cpus = *coupled_cpus; |
788 | #endif |
789 | ret = cpuidle_register_device(device); |
790 | if (!ret) |
791 | continue; |
792 | |
793 | pr_err("Failed to register cpuidle device for cpu%d\n" , cpu); |
794 | |
795 | cpuidle_unregister(drv); |
796 | break; |
797 | } |
798 | |
799 | return ret; |
800 | } |
801 | EXPORT_SYMBOL_GPL(cpuidle_register); |
802 | |
803 | /** |
804 | * cpuidle_init - core initializer |
805 | */ |
806 | static int __init cpuidle_init(void) |
807 | { |
808 | if (cpuidle_disabled()) |
809 | return -ENODEV; |
810 | |
811 | return cpuidle_add_interface(); |
812 | } |
813 | |
814 | module_param(off, int, 0444); |
815 | module_param_string(governor, param_governor, CPUIDLE_NAME_LEN, 0444); |
816 | core_initcall(cpuidle_init); |
817 | |