1// SPDX-License-Identifier: GPL-2.0
2/*
3 * drivers/base/power/domain_governor.c - Governors for device PM domains.
4 *
5 * Copyright (C) 2011 Rafael J. Wysocki <rjw@sisk.pl>, Renesas Electronics Corp.
6 */
7#include <linux/kernel.h>
8#include <linux/pm_domain.h>
9#include <linux/pm_qos.h>
10#include <linux/hrtimer.h>
11#include <linux/cpuidle.h>
12#include <linux/cpumask.h>
13#include <linux/ktime.h>
14
15static int dev_update_qos_constraint(struct device *dev, void *data)
16{
17 s64 *constraint_ns_p = data;
18 s64 constraint_ns;
19
20 if (dev->power.subsys_data && dev->power.subsys_data->domain_data) {
21 struct gpd_timing_data *td = dev_gpd_data(dev)->td;
22
23 /*
24 * Only take suspend-time QoS constraints of devices into
25 * account, because constraints updated after the device has
26 * been suspended are not guaranteed to be taken into account
27 * anyway. In order for them to take effect, the device has to
28 * be resumed and suspended again.
29 */
30 constraint_ns = td ? td->effective_constraint_ns :
31 PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS;
32 } else {
33 /*
34 * The child is not in a domain and there's no info on its
35 * suspend/resume latencies, so assume them to be negligible and
36 * take its current PM QoS constraint (that's the only thing
37 * known at this point anyway).
38 */
39 constraint_ns = dev_pm_qos_read_value(dev, type: DEV_PM_QOS_RESUME_LATENCY);
40 constraint_ns *= NSEC_PER_USEC;
41 }
42
43 if (constraint_ns < *constraint_ns_p)
44 *constraint_ns_p = constraint_ns;
45
46 return 0;
47}
48
49/**
50 * default_suspend_ok - Default PM domain governor routine to suspend devices.
51 * @dev: Device to check.
52 *
53 * Returns: true if OK to suspend, false if not OK to suspend
54 */
55static bool default_suspend_ok(struct device *dev)
56{
57 struct gpd_timing_data *td = dev_gpd_data(dev)->td;
58 unsigned long flags;
59 s64 constraint_ns;
60
61 dev_dbg(dev, "%s()\n", __func__);
62
63 spin_lock_irqsave(&dev->power.lock, flags);
64
65 if (!td->constraint_changed) {
66 bool ret = td->cached_suspend_ok;
67
68 spin_unlock_irqrestore(lock: &dev->power.lock, flags);
69 return ret;
70 }
71 td->constraint_changed = false;
72 td->cached_suspend_ok = false;
73 td->effective_constraint_ns = 0;
74 constraint_ns = __dev_pm_qos_resume_latency(dev);
75
76 spin_unlock_irqrestore(lock: &dev->power.lock, flags);
77
78 if (constraint_ns == 0)
79 return false;
80
81 constraint_ns *= NSEC_PER_USEC;
82 /*
83 * We can walk the children without any additional locking, because
84 * they all have been suspended at this point and their
85 * effective_constraint_ns fields won't be modified in parallel with us.
86 */
87 if (!dev->power.ignore_children)
88 device_for_each_child(dev, data: &constraint_ns,
89 fn: dev_update_qos_constraint);
90
91 if (constraint_ns == PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS) {
92 /* "No restriction", so the device is allowed to suspend. */
93 td->effective_constraint_ns = PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS;
94 td->cached_suspend_ok = true;
95 } else if (constraint_ns == 0) {
96 /*
97 * This triggers if one of the children that don't belong to a
98 * domain has a zero PM QoS constraint and it's better not to
99 * suspend then. effective_constraint_ns is zero already and
100 * cached_suspend_ok is false, so bail out.
101 */
102 return false;
103 } else {
104 constraint_ns -= td->suspend_latency_ns +
105 td->resume_latency_ns;
106 /*
107 * effective_constraint_ns is zero already and cached_suspend_ok
108 * is false, so if the computed value is not positive, return
109 * right away.
110 */
111 if (constraint_ns <= 0)
112 return false;
113
114 td->effective_constraint_ns = constraint_ns;
115 td->cached_suspend_ok = true;
116 }
117
118 /*
119 * The children have been suspended already, so we don't need to take
120 * their suspend latencies into account here.
121 */
122 return td->cached_suspend_ok;
123}
124
125static void update_domain_next_wakeup(struct generic_pm_domain *genpd, ktime_t now)
126{
127 ktime_t domain_wakeup = KTIME_MAX;
128 ktime_t next_wakeup;
129 struct pm_domain_data *pdd;
130 struct gpd_link *link;
131
132 if (!(genpd->flags & GENPD_FLAG_MIN_RESIDENCY))
133 return;
134
135 /*
136 * Devices that have a predictable wakeup pattern, may specify
137 * their next wakeup. Let's find the next wakeup from all the
138 * devices attached to this domain and from all the sub-domains.
139 * It is possible that component's a next wakeup may have become
140 * stale when we read that here. We will ignore to ensure the domain
141 * is able to enter its optimal idle state.
142 */
143 list_for_each_entry(pdd, &genpd->dev_list, list_node) {
144 next_wakeup = to_gpd_data(pdd)->td->next_wakeup;
145 if (next_wakeup != KTIME_MAX && !ktime_before(cmp1: next_wakeup, cmp2: now))
146 if (ktime_before(cmp1: next_wakeup, cmp2: domain_wakeup))
147 domain_wakeup = next_wakeup;
148 }
149
150 list_for_each_entry(link, &genpd->parent_links, parent_node) {
151 struct genpd_governor_data *cgd = link->child->gd;
152
153 next_wakeup = cgd ? cgd->next_wakeup : KTIME_MAX;
154 if (next_wakeup != KTIME_MAX && !ktime_before(cmp1: next_wakeup, cmp2: now))
155 if (ktime_before(cmp1: next_wakeup, cmp2: domain_wakeup))
156 domain_wakeup = next_wakeup;
157 }
158
159 genpd->gd->next_wakeup = domain_wakeup;
160}
161
162static bool next_wakeup_allows_state(struct generic_pm_domain *genpd,
163 unsigned int state, ktime_t now)
164{
165 ktime_t domain_wakeup = genpd->gd->next_wakeup;
166 s64 idle_time_ns, min_sleep_ns;
167
168 min_sleep_ns = genpd->states[state].power_off_latency_ns +
169 genpd->states[state].residency_ns;
170
171 idle_time_ns = ktime_to_ns(ktime_sub(domain_wakeup, now));
172
173 return idle_time_ns >= min_sleep_ns;
174}
175
176static bool __default_power_down_ok(struct dev_pm_domain *pd,
177 unsigned int state)
178{
179 struct generic_pm_domain *genpd = pd_to_genpd(pd);
180 struct gpd_link *link;
181 struct pm_domain_data *pdd;
182 s64 min_off_time_ns;
183 s64 off_on_time_ns;
184
185 off_on_time_ns = genpd->states[state].power_off_latency_ns +
186 genpd->states[state].power_on_latency_ns;
187
188 min_off_time_ns = -1;
189 /*
190 * Check if subdomains can be off for enough time.
191 *
192 * All subdomains have been powered off already at this point.
193 */
194 list_for_each_entry(link, &genpd->parent_links, parent_node) {
195 struct genpd_governor_data *cgd = link->child->gd;
196
197 s64 sd_max_off_ns = cgd ? cgd->max_off_time_ns : -1;
198
199 if (sd_max_off_ns < 0)
200 continue;
201
202 /*
203 * Check if the subdomain is allowed to be off long enough for
204 * the current domain to turn off and on (that's how much time
205 * it will have to wait worst case).
206 */
207 if (sd_max_off_ns <= off_on_time_ns)
208 return false;
209
210 if (min_off_time_ns > sd_max_off_ns || min_off_time_ns < 0)
211 min_off_time_ns = sd_max_off_ns;
212 }
213
214 /*
215 * Check if the devices in the domain can be off enough time.
216 */
217 list_for_each_entry(pdd, &genpd->dev_list, list_node) {
218 struct gpd_timing_data *td;
219 s64 constraint_ns;
220
221 /*
222 * Check if the device is allowed to be off long enough for the
223 * domain to turn off and on (that's how much time it will
224 * have to wait worst case).
225 */
226 td = to_gpd_data(pdd)->td;
227 constraint_ns = td->effective_constraint_ns;
228 /*
229 * Zero means "no suspend at all" and this runs only when all
230 * devices in the domain are suspended, so it must be positive.
231 */
232 if (constraint_ns == PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS)
233 continue;
234
235 if (constraint_ns <= off_on_time_ns)
236 return false;
237
238 if (min_off_time_ns > constraint_ns || min_off_time_ns < 0)
239 min_off_time_ns = constraint_ns;
240 }
241
242 /*
243 * If the computed minimum device off time is negative, there are no
244 * latency constraints, so the domain can spend arbitrary time in the
245 * "off" state.
246 */
247 if (min_off_time_ns < 0)
248 return true;
249
250 /*
251 * The difference between the computed minimum subdomain or device off
252 * time and the time needed to turn the domain on is the maximum
253 * theoretical time this domain can spend in the "off" state.
254 */
255 genpd->gd->max_off_time_ns = min_off_time_ns -
256 genpd->states[state].power_on_latency_ns;
257 return true;
258}
259
260/**
261 * _default_power_down_ok - Default generic PM domain power off governor routine.
262 * @pd: PM domain to check.
263 * @now: current ktime.
264 *
265 * This routine must be executed under the PM domain's lock.
266 *
267 * Returns: true if OK to power down, false if not OK to power down
268 */
269static bool _default_power_down_ok(struct dev_pm_domain *pd, ktime_t now)
270{
271 struct generic_pm_domain *genpd = pd_to_genpd(pd);
272 struct genpd_governor_data *gd = genpd->gd;
273 int state_idx = genpd->state_count - 1;
274 struct gpd_link *link;
275
276 /*
277 * Find the next wakeup from devices that can determine their own wakeup
278 * to find when the domain would wakeup and do it for every device down
279 * the hierarchy. It is not worth while to sleep if the state's residency
280 * cannot be met.
281 */
282 update_domain_next_wakeup(genpd, now);
283 if ((genpd->flags & GENPD_FLAG_MIN_RESIDENCY) && (gd->next_wakeup != KTIME_MAX)) {
284 /* Let's find out the deepest domain idle state, the devices prefer */
285 while (state_idx >= 0) {
286 if (next_wakeup_allows_state(genpd, state: state_idx, now)) {
287 gd->max_off_time_changed = true;
288 break;
289 }
290 state_idx--;
291 }
292
293 if (state_idx < 0) {
294 state_idx = 0;
295 gd->cached_power_down_ok = false;
296 goto done;
297 }
298 }
299
300 if (!gd->max_off_time_changed) {
301 genpd->state_idx = gd->cached_power_down_state_idx;
302 return gd->cached_power_down_ok;
303 }
304
305 /*
306 * We have to invalidate the cached results for the parents, so
307 * use the observation that default_power_down_ok() is not
308 * going to be called for any parent until this instance
309 * returns.
310 */
311 list_for_each_entry(link, &genpd->child_links, child_node) {
312 struct genpd_governor_data *pgd = link->parent->gd;
313
314 if (pgd)
315 pgd->max_off_time_changed = true;
316 }
317
318 gd->max_off_time_ns = -1;
319 gd->max_off_time_changed = false;
320 gd->cached_power_down_ok = true;
321
322 /*
323 * Find a state to power down to, starting from the state
324 * determined by the next wakeup.
325 */
326 while (!__default_power_down_ok(pd, state: state_idx)) {
327 if (state_idx == 0) {
328 gd->cached_power_down_ok = false;
329 break;
330 }
331 state_idx--;
332 }
333
334done:
335 genpd->state_idx = state_idx;
336 gd->cached_power_down_state_idx = genpd->state_idx;
337 return gd->cached_power_down_ok;
338}
339
340static bool default_power_down_ok(struct dev_pm_domain *pd)
341{
342 return _default_power_down_ok(pd, now: ktime_get());
343}
344
345#ifdef CONFIG_CPU_IDLE
346static bool cpu_power_down_ok(struct dev_pm_domain *pd)
347{
348 struct generic_pm_domain *genpd = pd_to_genpd(pd);
349 struct cpuidle_device *dev;
350 ktime_t domain_wakeup, next_hrtimer;
351 ktime_t now = ktime_get();
352 s64 idle_duration_ns;
353 int cpu, i;
354
355 /* Validate dev PM QoS constraints. */
356 if (!_default_power_down_ok(pd, now))
357 return false;
358
359 if (!(genpd->flags & GENPD_FLAG_CPU_DOMAIN))
360 return true;
361
362 /*
363 * Find the next wakeup for any of the online CPUs within the PM domain
364 * and its subdomains. Note, we only need the genpd->cpus, as it already
365 * contains a mask of all CPUs from subdomains.
366 */
367 domain_wakeup = ktime_set(KTIME_SEC_MAX, nsecs: 0);
368 for_each_cpu_and(cpu, genpd->cpus, cpu_online_mask) {
369 dev = per_cpu(cpuidle_devices, cpu);
370 if (dev) {
371 next_hrtimer = READ_ONCE(dev->next_hrtimer);
372 if (ktime_before(cmp1: next_hrtimer, cmp2: domain_wakeup))
373 domain_wakeup = next_hrtimer;
374 }
375 }
376
377 /* The minimum idle duration is from now - until the next wakeup. */
378 idle_duration_ns = ktime_to_ns(ktime_sub(domain_wakeup, now));
379 if (idle_duration_ns <= 0)
380 return false;
381
382 /* Store the next domain_wakeup to allow consumers to use it. */
383 genpd->gd->next_hrtimer = domain_wakeup;
384
385 /*
386 * Find the deepest idle state that has its residency value satisfied
387 * and by also taking into account the power off latency for the state.
388 * Start at the state picked by the dev PM QoS constraint validation.
389 */
390 i = genpd->state_idx;
391 do {
392 if (idle_duration_ns >= (genpd->states[i].residency_ns +
393 genpd->states[i].power_off_latency_ns)) {
394 genpd->state_idx = i;
395 return true;
396 }
397 } while (--i >= 0);
398
399 return false;
400}
401
402struct dev_power_governor pm_domain_cpu_gov = {
403 .suspend_ok = default_suspend_ok,
404 .power_down_ok = cpu_power_down_ok,
405};
406#endif
407
408struct dev_power_governor simple_qos_governor = {
409 .suspend_ok = default_suspend_ok,
410 .power_down_ok = default_power_down_ok,
411};
412
413/*
414 * pm_domain_always_on_gov - A governor implementing an always-on policy
415 */
416struct dev_power_governor pm_domain_always_on_gov = {
417 .suspend_ok = default_suspend_ok,
418};
419

source code of linux/drivers/pmdomain/governor.c