1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * drivers/base/power/domain_governor.c - Governors for device PM domains. |
4 | * |
5 | * Copyright (C) 2011 Rafael J. Wysocki <rjw@sisk.pl>, Renesas Electronics Corp. |
6 | */ |
7 | #include <linux/kernel.h> |
8 | #include <linux/pm_domain.h> |
9 | #include <linux/pm_qos.h> |
10 | #include <linux/hrtimer.h> |
11 | #include <linux/cpuidle.h> |
12 | #include <linux/cpumask.h> |
13 | #include <linux/ktime.h> |
14 | |
15 | static int dev_update_qos_constraint(struct device *dev, void *data) |
16 | { |
17 | s64 *constraint_ns_p = data; |
18 | s64 constraint_ns; |
19 | |
20 | if (dev->power.subsys_data && dev->power.subsys_data->domain_data) { |
21 | struct gpd_timing_data *td = dev_gpd_data(dev)->td; |
22 | |
23 | /* |
24 | * Only take suspend-time QoS constraints of devices into |
25 | * account, because constraints updated after the device has |
26 | * been suspended are not guaranteed to be taken into account |
27 | * anyway. In order for them to take effect, the device has to |
28 | * be resumed and suspended again. |
29 | */ |
30 | constraint_ns = td ? td->effective_constraint_ns : |
31 | PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS; |
32 | } else { |
33 | /* |
34 | * The child is not in a domain and there's no info on its |
35 | * suspend/resume latencies, so assume them to be negligible and |
36 | * take its current PM QoS constraint (that's the only thing |
37 | * known at this point anyway). |
38 | */ |
39 | constraint_ns = dev_pm_qos_read_value(dev, type: DEV_PM_QOS_RESUME_LATENCY); |
40 | constraint_ns *= NSEC_PER_USEC; |
41 | } |
42 | |
43 | if (constraint_ns < *constraint_ns_p) |
44 | *constraint_ns_p = constraint_ns; |
45 | |
46 | return 0; |
47 | } |
48 | |
49 | /** |
50 | * default_suspend_ok - Default PM domain governor routine to suspend devices. |
51 | * @dev: Device to check. |
52 | * |
53 | * Returns: true if OK to suspend, false if not OK to suspend |
54 | */ |
55 | static bool default_suspend_ok(struct device *dev) |
56 | { |
57 | struct gpd_timing_data *td = dev_gpd_data(dev)->td; |
58 | unsigned long flags; |
59 | s64 constraint_ns; |
60 | |
61 | dev_dbg(dev, "%s()\n" , __func__); |
62 | |
63 | spin_lock_irqsave(&dev->power.lock, flags); |
64 | |
65 | if (!td->constraint_changed) { |
66 | bool ret = td->cached_suspend_ok; |
67 | |
68 | spin_unlock_irqrestore(lock: &dev->power.lock, flags); |
69 | return ret; |
70 | } |
71 | td->constraint_changed = false; |
72 | td->cached_suspend_ok = false; |
73 | td->effective_constraint_ns = 0; |
74 | constraint_ns = __dev_pm_qos_resume_latency(dev); |
75 | |
76 | spin_unlock_irqrestore(lock: &dev->power.lock, flags); |
77 | |
78 | if (constraint_ns == 0) |
79 | return false; |
80 | |
81 | constraint_ns *= NSEC_PER_USEC; |
82 | /* |
83 | * We can walk the children without any additional locking, because |
84 | * they all have been suspended at this point and their |
85 | * effective_constraint_ns fields won't be modified in parallel with us. |
86 | */ |
87 | if (!dev->power.ignore_children) |
88 | device_for_each_child(dev, data: &constraint_ns, |
89 | fn: dev_update_qos_constraint); |
90 | |
91 | if (constraint_ns == PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS) { |
92 | /* "No restriction", so the device is allowed to suspend. */ |
93 | td->effective_constraint_ns = PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS; |
94 | td->cached_suspend_ok = true; |
95 | } else if (constraint_ns == 0) { |
96 | /* |
97 | * This triggers if one of the children that don't belong to a |
98 | * domain has a zero PM QoS constraint and it's better not to |
99 | * suspend then. effective_constraint_ns is zero already and |
100 | * cached_suspend_ok is false, so bail out. |
101 | */ |
102 | return false; |
103 | } else { |
104 | constraint_ns -= td->suspend_latency_ns + |
105 | td->resume_latency_ns; |
106 | /* |
107 | * effective_constraint_ns is zero already and cached_suspend_ok |
108 | * is false, so if the computed value is not positive, return |
109 | * right away. |
110 | */ |
111 | if (constraint_ns <= 0) |
112 | return false; |
113 | |
114 | td->effective_constraint_ns = constraint_ns; |
115 | td->cached_suspend_ok = true; |
116 | } |
117 | |
118 | /* |
119 | * The children have been suspended already, so we don't need to take |
120 | * their suspend latencies into account here. |
121 | */ |
122 | return td->cached_suspend_ok; |
123 | } |
124 | |
125 | static void update_domain_next_wakeup(struct generic_pm_domain *genpd, ktime_t now) |
126 | { |
127 | ktime_t domain_wakeup = KTIME_MAX; |
128 | ktime_t next_wakeup; |
129 | struct pm_domain_data *pdd; |
130 | struct gpd_link *link; |
131 | |
132 | if (!(genpd->flags & GENPD_FLAG_MIN_RESIDENCY)) |
133 | return; |
134 | |
135 | /* |
136 | * Devices that have a predictable wakeup pattern, may specify |
137 | * their next wakeup. Let's find the next wakeup from all the |
138 | * devices attached to this domain and from all the sub-domains. |
139 | * It is possible that component's a next wakeup may have become |
140 | * stale when we read that here. We will ignore to ensure the domain |
141 | * is able to enter its optimal idle state. |
142 | */ |
143 | list_for_each_entry(pdd, &genpd->dev_list, list_node) { |
144 | next_wakeup = to_gpd_data(pdd)->td->next_wakeup; |
145 | if (next_wakeup != KTIME_MAX && !ktime_before(cmp1: next_wakeup, cmp2: now)) |
146 | if (ktime_before(cmp1: next_wakeup, cmp2: domain_wakeup)) |
147 | domain_wakeup = next_wakeup; |
148 | } |
149 | |
150 | list_for_each_entry(link, &genpd->parent_links, parent_node) { |
151 | struct genpd_governor_data *cgd = link->child->gd; |
152 | |
153 | next_wakeup = cgd ? cgd->next_wakeup : KTIME_MAX; |
154 | if (next_wakeup != KTIME_MAX && !ktime_before(cmp1: next_wakeup, cmp2: now)) |
155 | if (ktime_before(cmp1: next_wakeup, cmp2: domain_wakeup)) |
156 | domain_wakeup = next_wakeup; |
157 | } |
158 | |
159 | genpd->gd->next_wakeup = domain_wakeup; |
160 | } |
161 | |
162 | static bool next_wakeup_allows_state(struct generic_pm_domain *genpd, |
163 | unsigned int state, ktime_t now) |
164 | { |
165 | ktime_t domain_wakeup = genpd->gd->next_wakeup; |
166 | s64 idle_time_ns, min_sleep_ns; |
167 | |
168 | min_sleep_ns = genpd->states[state].power_off_latency_ns + |
169 | genpd->states[state].residency_ns; |
170 | |
171 | idle_time_ns = ktime_to_ns(ktime_sub(domain_wakeup, now)); |
172 | |
173 | return idle_time_ns >= min_sleep_ns; |
174 | } |
175 | |
176 | static bool __default_power_down_ok(struct dev_pm_domain *pd, |
177 | unsigned int state) |
178 | { |
179 | struct generic_pm_domain *genpd = pd_to_genpd(pd); |
180 | struct gpd_link *link; |
181 | struct pm_domain_data *pdd; |
182 | s64 min_off_time_ns; |
183 | s64 off_on_time_ns; |
184 | |
185 | off_on_time_ns = genpd->states[state].power_off_latency_ns + |
186 | genpd->states[state].power_on_latency_ns; |
187 | |
188 | min_off_time_ns = -1; |
189 | /* |
190 | * Check if subdomains can be off for enough time. |
191 | * |
192 | * All subdomains have been powered off already at this point. |
193 | */ |
194 | list_for_each_entry(link, &genpd->parent_links, parent_node) { |
195 | struct genpd_governor_data *cgd = link->child->gd; |
196 | |
197 | s64 sd_max_off_ns = cgd ? cgd->max_off_time_ns : -1; |
198 | |
199 | if (sd_max_off_ns < 0) |
200 | continue; |
201 | |
202 | /* |
203 | * Check if the subdomain is allowed to be off long enough for |
204 | * the current domain to turn off and on (that's how much time |
205 | * it will have to wait worst case). |
206 | */ |
207 | if (sd_max_off_ns <= off_on_time_ns) |
208 | return false; |
209 | |
210 | if (min_off_time_ns > sd_max_off_ns || min_off_time_ns < 0) |
211 | min_off_time_ns = sd_max_off_ns; |
212 | } |
213 | |
214 | /* |
215 | * Check if the devices in the domain can be off enough time. |
216 | */ |
217 | list_for_each_entry(pdd, &genpd->dev_list, list_node) { |
218 | struct gpd_timing_data *td; |
219 | s64 constraint_ns; |
220 | |
221 | /* |
222 | * Check if the device is allowed to be off long enough for the |
223 | * domain to turn off and on (that's how much time it will |
224 | * have to wait worst case). |
225 | */ |
226 | td = to_gpd_data(pdd)->td; |
227 | constraint_ns = td->effective_constraint_ns; |
228 | /* |
229 | * Zero means "no suspend at all" and this runs only when all |
230 | * devices in the domain are suspended, so it must be positive. |
231 | */ |
232 | if (constraint_ns == PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS) |
233 | continue; |
234 | |
235 | if (constraint_ns <= off_on_time_ns) |
236 | return false; |
237 | |
238 | if (min_off_time_ns > constraint_ns || min_off_time_ns < 0) |
239 | min_off_time_ns = constraint_ns; |
240 | } |
241 | |
242 | /* |
243 | * If the computed minimum device off time is negative, there are no |
244 | * latency constraints, so the domain can spend arbitrary time in the |
245 | * "off" state. |
246 | */ |
247 | if (min_off_time_ns < 0) |
248 | return true; |
249 | |
250 | /* |
251 | * The difference between the computed minimum subdomain or device off |
252 | * time and the time needed to turn the domain on is the maximum |
253 | * theoretical time this domain can spend in the "off" state. |
254 | */ |
255 | genpd->gd->max_off_time_ns = min_off_time_ns - |
256 | genpd->states[state].power_on_latency_ns; |
257 | return true; |
258 | } |
259 | |
260 | /** |
261 | * _default_power_down_ok - Default generic PM domain power off governor routine. |
262 | * @pd: PM domain to check. |
263 | * @now: current ktime. |
264 | * |
265 | * This routine must be executed under the PM domain's lock. |
266 | * |
267 | * Returns: true if OK to power down, false if not OK to power down |
268 | */ |
269 | static bool _default_power_down_ok(struct dev_pm_domain *pd, ktime_t now) |
270 | { |
271 | struct generic_pm_domain *genpd = pd_to_genpd(pd); |
272 | struct genpd_governor_data *gd = genpd->gd; |
273 | int state_idx = genpd->state_count - 1; |
274 | struct gpd_link *link; |
275 | |
276 | /* |
277 | * Find the next wakeup from devices that can determine their own wakeup |
278 | * to find when the domain would wakeup and do it for every device down |
279 | * the hierarchy. It is not worth while to sleep if the state's residency |
280 | * cannot be met. |
281 | */ |
282 | update_domain_next_wakeup(genpd, now); |
283 | if ((genpd->flags & GENPD_FLAG_MIN_RESIDENCY) && (gd->next_wakeup != KTIME_MAX)) { |
284 | /* Let's find out the deepest domain idle state, the devices prefer */ |
285 | while (state_idx >= 0) { |
286 | if (next_wakeup_allows_state(genpd, state: state_idx, now)) { |
287 | gd->max_off_time_changed = true; |
288 | break; |
289 | } |
290 | state_idx--; |
291 | } |
292 | |
293 | if (state_idx < 0) { |
294 | state_idx = 0; |
295 | gd->cached_power_down_ok = false; |
296 | goto done; |
297 | } |
298 | } |
299 | |
300 | if (!gd->max_off_time_changed) { |
301 | genpd->state_idx = gd->cached_power_down_state_idx; |
302 | return gd->cached_power_down_ok; |
303 | } |
304 | |
305 | /* |
306 | * We have to invalidate the cached results for the parents, so |
307 | * use the observation that default_power_down_ok() is not |
308 | * going to be called for any parent until this instance |
309 | * returns. |
310 | */ |
311 | list_for_each_entry(link, &genpd->child_links, child_node) { |
312 | struct genpd_governor_data *pgd = link->parent->gd; |
313 | |
314 | if (pgd) |
315 | pgd->max_off_time_changed = true; |
316 | } |
317 | |
318 | gd->max_off_time_ns = -1; |
319 | gd->max_off_time_changed = false; |
320 | gd->cached_power_down_ok = true; |
321 | |
322 | /* |
323 | * Find a state to power down to, starting from the state |
324 | * determined by the next wakeup. |
325 | */ |
326 | while (!__default_power_down_ok(pd, state: state_idx)) { |
327 | if (state_idx == 0) { |
328 | gd->cached_power_down_ok = false; |
329 | break; |
330 | } |
331 | state_idx--; |
332 | } |
333 | |
334 | done: |
335 | genpd->state_idx = state_idx; |
336 | gd->cached_power_down_state_idx = genpd->state_idx; |
337 | return gd->cached_power_down_ok; |
338 | } |
339 | |
340 | static bool default_power_down_ok(struct dev_pm_domain *pd) |
341 | { |
342 | return _default_power_down_ok(pd, now: ktime_get()); |
343 | } |
344 | |
345 | #ifdef CONFIG_CPU_IDLE |
346 | static bool cpu_power_down_ok(struct dev_pm_domain *pd) |
347 | { |
348 | struct generic_pm_domain *genpd = pd_to_genpd(pd); |
349 | struct cpuidle_device *dev; |
350 | ktime_t domain_wakeup, next_hrtimer; |
351 | ktime_t now = ktime_get(); |
352 | s64 idle_duration_ns; |
353 | int cpu, i; |
354 | |
355 | /* Validate dev PM QoS constraints. */ |
356 | if (!_default_power_down_ok(pd, now)) |
357 | return false; |
358 | |
359 | if (!(genpd->flags & GENPD_FLAG_CPU_DOMAIN)) |
360 | return true; |
361 | |
362 | /* |
363 | * Find the next wakeup for any of the online CPUs within the PM domain |
364 | * and its subdomains. Note, we only need the genpd->cpus, as it already |
365 | * contains a mask of all CPUs from subdomains. |
366 | */ |
367 | domain_wakeup = ktime_set(KTIME_SEC_MAX, nsecs: 0); |
368 | for_each_cpu_and(cpu, genpd->cpus, cpu_online_mask) { |
369 | dev = per_cpu(cpuidle_devices, cpu); |
370 | if (dev) { |
371 | next_hrtimer = READ_ONCE(dev->next_hrtimer); |
372 | if (ktime_before(cmp1: next_hrtimer, cmp2: domain_wakeup)) |
373 | domain_wakeup = next_hrtimer; |
374 | } |
375 | } |
376 | |
377 | /* The minimum idle duration is from now - until the next wakeup. */ |
378 | idle_duration_ns = ktime_to_ns(ktime_sub(domain_wakeup, now)); |
379 | if (idle_duration_ns <= 0) |
380 | return false; |
381 | |
382 | /* Store the next domain_wakeup to allow consumers to use it. */ |
383 | genpd->gd->next_hrtimer = domain_wakeup; |
384 | |
385 | /* |
386 | * Find the deepest idle state that has its residency value satisfied |
387 | * and by also taking into account the power off latency for the state. |
388 | * Start at the state picked by the dev PM QoS constraint validation. |
389 | */ |
390 | i = genpd->state_idx; |
391 | do { |
392 | if (idle_duration_ns >= (genpd->states[i].residency_ns + |
393 | genpd->states[i].power_off_latency_ns)) { |
394 | genpd->state_idx = i; |
395 | return true; |
396 | } |
397 | } while (--i >= 0); |
398 | |
399 | return false; |
400 | } |
401 | |
402 | struct dev_power_governor pm_domain_cpu_gov = { |
403 | .suspend_ok = default_suspend_ok, |
404 | .power_down_ok = cpu_power_down_ok, |
405 | }; |
406 | #endif |
407 | |
408 | struct dev_power_governor simple_qos_governor = { |
409 | .suspend_ok = default_suspend_ok, |
410 | .power_down_ok = default_power_down_ok, |
411 | }; |
412 | |
413 | /* |
414 | * pm_domain_always_on_gov - A governor implementing an always-on policy |
415 | */ |
416 | struct dev_power_governor pm_domain_always_on_gov = { |
417 | .suspend_ok = default_suspend_ok, |
418 | }; |
419 | |