1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* drivers/base/power/domain.c - Common code related to device power domains.
4	*
5	* Copyright (C) 2011 Rafael J. Wysocki <rjw@sisk.pl>, Renesas Electronics Corp.
6	*/
7	#define pr_fmt(fmt) "PM: " fmt
8
9	#include <linux/delay.h>
10	#include <linux/kernel.h>
11	#include <linux/io.h>
12	#include <linux/platform_device.h>
13	#include <linux/pm_opp.h>
14	#include <linux/pm_runtime.h>
15	#include <linux/pm_domain.h>
16	#include <linux/pm_qos.h>
17	#include <linux/pm_clock.h>
18	#include <linux/slab.h>
19	#include <linux/err.h>
20	#include <linux/sched.h>
21	#include <linux/suspend.h>
22	#include <linux/export.h>
23	#include <linux/cpu.h>
24	#include <linux/debugfs.h>
25
26	#include "power.h"
27
28	#define GENPD_RETRY_MAX_MS 250 /* Approximate */
29
30	#define GENPD_DEV_CALLBACK(genpd, type, callback, dev) \
31	({ \
32	type (*__routine)(struct device *__d); \
33	type __ret = (type)0; \
34	\
35	__routine = genpd->dev_ops.callback; \
36	if (__routine) { \
37	__ret = __routine(dev); \
38	} \
39	__ret; \
40	})
41
42	static LIST_HEAD(gpd_list);
43	static DEFINE_MUTEX(gpd_list_lock);
44
45	struct genpd_lock_ops {
46	void (*lock)(struct generic_pm_domain *genpd);
47	void (*lock_nested)(struct generic_pm_domain *genpd, int depth);
48	int (*lock_interruptible)(struct generic_pm_domain *genpd);
49	void (*unlock)(struct generic_pm_domain *genpd);
50	};
51
52	static void genpd_lock_mtx(struct generic_pm_domain *genpd)
53	{
54	mutex_lock(&genpd->mlock);
55	}
56
57	static void genpd_lock_nested_mtx(struct generic_pm_domain *genpd,
58	int depth)
59	{
60	mutex_lock_nested(lock: &genpd->mlock, subclass: depth);
61	}
62
63	static int genpd_lock_interruptible_mtx(struct generic_pm_domain *genpd)
64	{
65	return mutex_lock_interruptible(&genpd->mlock);
66	}
67
68	static void genpd_unlock_mtx(struct generic_pm_domain *genpd)
69	{
70	return mutex_unlock(lock: &genpd->mlock);
71	}
72
73	static const struct genpd_lock_ops genpd_mtx_ops = {
74	.lock = genpd_lock_mtx,
75	.lock_nested = genpd_lock_nested_mtx,
76	.lock_interruptible = genpd_lock_interruptible_mtx,
77	.unlock = genpd_unlock_mtx,
78	};
79
80	static void genpd_lock_spin(struct generic_pm_domain *genpd)
81	__acquires(&genpd->slock)
82	{
83	unsigned long flags;
84
85	spin_lock_irqsave(&genpd->slock, flags);
86	genpd->lock_flags = flags;
87	}
88
89	static void genpd_lock_nested_spin(struct generic_pm_domain *genpd,
90	int depth)
91	__acquires(&genpd->slock)
92	{
93	unsigned long flags;
94
95	spin_lock_irqsave_nested(&genpd->slock, flags, depth);
96	genpd->lock_flags = flags;
97	}
98
99	static int genpd_lock_interruptible_spin(struct generic_pm_domain *genpd)
100	__acquires(&genpd->slock)
101	{
102	unsigned long flags;
103
104	spin_lock_irqsave(&genpd->slock, flags);
105	genpd->lock_flags = flags;
106	return 0;
107	}
108
109	static void genpd_unlock_spin(struct generic_pm_domain *genpd)
110	__releases(&genpd->slock)
111	{
112	spin_unlock_irqrestore(lock: &genpd->slock, flags: genpd->lock_flags);
113	}
114
115	static const struct genpd_lock_ops genpd_spin_ops = {
116	.lock = genpd_lock_spin,
117	.lock_nested = genpd_lock_nested_spin,
118	.lock_interruptible = genpd_lock_interruptible_spin,
119	.unlock = genpd_unlock_spin,
120	};
121
122	#define genpd_lock(p) p->lock_ops->lock(p)
123	#define genpd_lock_nested(p, d) p->lock_ops->lock_nested(p, d)
124	#define genpd_lock_interruptible(p) p->lock_ops->lock_interruptible(p)
125	#define genpd_unlock(p) p->lock_ops->unlock(p)
126
127	#define genpd_status_on(genpd) (genpd->status == GENPD_STATE_ON)
128	#define genpd_is_irq_safe(genpd) (genpd->flags & GENPD_FLAG_IRQ_SAFE)
129	#define genpd_is_always_on(genpd) (genpd->flags & GENPD_FLAG_ALWAYS_ON)
130	#define genpd_is_active_wakeup(genpd) (genpd->flags & GENPD_FLAG_ACTIVE_WAKEUP)
131	#define genpd_is_cpu_domain(genpd) (genpd->flags & GENPD_FLAG_CPU_DOMAIN)
132	#define genpd_is_rpm_always_on(genpd) (genpd->flags & GENPD_FLAG_RPM_ALWAYS_ON)
133	#define genpd_is_opp_table_fw(genpd) (genpd->flags & GENPD_FLAG_OPP_TABLE_FW)
134
135	static inline bool irq_safe_dev_in_sleep_domain(struct device *dev,
136	const struct generic_pm_domain *genpd)
137	{
138	bool ret;
139
140	ret = pm_runtime_is_irq_safe(dev) && !genpd_is_irq_safe(genpd);
141
142	/*
143	* Warn once if an IRQ safe device is attached to a domain, which
144	* callbacks are allowed to sleep. This indicates a suboptimal
145	* configuration for PM, but it doesn't matter for an always on domain.
146	*/
147	if (genpd_is_always_on(genpd) || genpd_is_rpm_always_on(genpd))
148	return ret;
149
150	if (ret)
151	dev_warn_once(dev, "PM domain %s will not be powered off\n",
152	genpd->name);
153
154	return ret;
155	}
156
157	static int genpd_runtime_suspend(struct device *dev);
158
159	/*
160	* Get the generic PM domain for a particular struct device.
161	* This validates the struct device pointer, the PM domain pointer,
162	* and checks that the PM domain pointer is a real generic PM domain.
163	* Any failure results in NULL being returned.
164	*/
165	static struct generic_pm_domain *dev_to_genpd_safe(struct device *dev)
166	{
167	if (IS_ERR_OR_NULL(ptr: dev) || IS_ERR_OR_NULL(ptr: dev->pm_domain))
168	return NULL;
169
170	/* A genpd's always have its ->runtime_suspend() callback assigned. */
171	if (dev->pm_domain->ops.runtime_suspend == genpd_runtime_suspend)
172	return pd_to_genpd(pd: dev->pm_domain);
173
174	return NULL;
175	}
176
177	/*
178	* This should only be used where we are certain that the pm_domain
179	* attached to the device is a genpd domain.
180	*/
181	static struct generic_pm_domain *dev_to_genpd(struct device *dev)
182	{
183	if (IS_ERR_OR_NULL(ptr: dev->pm_domain))
184	return ERR_PTR(error: -EINVAL);
185
186	return pd_to_genpd(pd: dev->pm_domain);
187	}
188
189	static int genpd_stop_dev(const struct generic_pm_domain *genpd,
190	struct device *dev)
191	{
192	return GENPD_DEV_CALLBACK(genpd, int, stop, dev);
193	}
194
195	static int genpd_start_dev(const struct generic_pm_domain *genpd,
196	struct device *dev)
197	{
198	return GENPD_DEV_CALLBACK(genpd, int, start, dev);
199	}
200
201	static bool genpd_sd_counter_dec(struct generic_pm_domain *genpd)
202	{
203	bool ret = false;
204
205	if (!WARN_ON(atomic_read(&genpd->sd_count) == 0))
206	ret = !!atomic_dec_and_test(v: &genpd->sd_count);
207
208	return ret;
209	}
210
211	static void genpd_sd_counter_inc(struct generic_pm_domain *genpd)
212	{
213	atomic_inc(v: &genpd->sd_count);
214	smp_mb__after_atomic();
215	}
216
217	#ifdef CONFIG_DEBUG_FS
218	static struct dentry *genpd_debugfs_dir;
219
220	static void genpd_debug_add(struct generic_pm_domain *genpd);
221
222	static void genpd_debug_remove(struct generic_pm_domain *genpd)
223	{
224	if (!genpd_debugfs_dir)
225	return;
226
227	debugfs_lookup_and_remove(name: genpd->name, parent: genpd_debugfs_dir);
228	}
229
230	static void genpd_update_accounting(struct generic_pm_domain *genpd)
231	{
232	u64 delta, now;
233
234	now = ktime_get_mono_fast_ns();
235	if (now <= genpd->accounting_time)
236	return;
237
238	delta = now - genpd->accounting_time;
239
240	/*
241	* If genpd->status is active, it means we are just
242	* out of off and so update the idle time and vice
243	* versa.
244	*/
245	if (genpd->status == GENPD_STATE_ON)
246	genpd->states[genpd->state_idx].idle_time += delta;
247	else
248	genpd->on_time += delta;
249
250	genpd->accounting_time = now;
251	}
252	#else
253	static inline void genpd_debug_add(struct generic_pm_domain *genpd) {}
254	static inline void genpd_debug_remove(struct generic_pm_domain *genpd) {}
255	static inline void genpd_update_accounting(struct generic_pm_domain *genpd) {}
256	#endif
257
258	static int _genpd_reeval_performance_state(struct generic_pm_domain *genpd,
259	unsigned int state)
260	{
261	struct generic_pm_domain_data *pd_data;
262	struct pm_domain_data *pdd;
263	struct gpd_link *link;
264
265	/* New requested state is same as Max requested state */
266	if (state == genpd->performance_state)
267	return state;
268
269	/* New requested state is higher than Max requested state */
270	if (state > genpd->performance_state)
271	return state;
272
273	/* Traverse all devices within the domain */
274	list_for_each_entry(pdd, &genpd->dev_list, list_node) {
275	pd_data = to_gpd_data(pdd);
276
277	if (pd_data->performance_state > state)
278	state = pd_data->performance_state;
279	}
280
281	/*
282	* Traverse all sub-domains within the domain. This can be
283	* done without any additional locking as the link->performance_state
284	* field is protected by the parent genpd->lock, which is already taken.
285	*
286	* Also note that link->performance_state (subdomain's performance state
287	* requirement to parent domain) is different from
288	* link->child->performance_state (current performance state requirement
289	* of the devices/sub-domains of the subdomain) and so can have a
290	* different value.
291	*
292	* Note that we also take vote from powered-off sub-domains into account
293	* as the same is done for devices right now.
294	*/
295	list_for_each_entry(link, &genpd->parent_links, parent_node) {
296	if (link->performance_state > state)
297	state = link->performance_state;
298	}
299
300	return state;
301	}
302
303	static int genpd_xlate_performance_state(struct generic_pm_domain *genpd,
304	struct generic_pm_domain *parent,
305	unsigned int pstate)
306	{
307	if (!parent->set_performance_state)
308	return pstate;
309
310	return dev_pm_opp_xlate_performance_state(src_table: genpd->opp_table,
311	dst_table: parent->opp_table,
312	pstate);
313	}
314
315	static int _genpd_set_performance_state(struct generic_pm_domain *genpd,
316	unsigned int state, int depth)
317	{
318	struct generic_pm_domain *parent;
319	struct gpd_link *link;
320	int parent_state, ret;
321
322	if (state == genpd->performance_state)
323	return 0;
324
325	/* Propagate to parents of genpd */
326	list_for_each_entry(link, &genpd->child_links, child_node) {
327	parent = link->parent;
328
329	/* Find parent's performance state */
330	ret = genpd_xlate_performance_state(genpd, parent, pstate: state);
331	if (unlikely(ret < 0))
332	goto err;
333
334	parent_state = ret;
335
336	genpd_lock_nested(parent, depth + 1);
337
338	link->prev_performance_state = link->performance_state;
339	link->performance_state = parent_state;
340	parent_state = _genpd_reeval_performance_state(genpd: parent,
341	state: parent_state);
342	ret = _genpd_set_performance_state(genpd: parent, state: parent_state, depth: depth + 1);
343	if (ret)
344	link->performance_state = link->prev_performance_state;
345
346	genpd_unlock(parent);
347
348	if (ret)
349	goto err;
350	}
351
352	if (genpd->set_performance_state) {
353	ret = genpd->set_performance_state(genpd, state);
354	if (ret)
355	goto err;
356	}
357
358	genpd->performance_state = state;
359	return 0;
360
361	err:
362	/* Encountered an error, lets rollback */
363	list_for_each_entry_continue_reverse(link, &genpd->child_links,
364	child_node) {
365	parent = link->parent;
366
367	genpd_lock_nested(parent, depth + 1);
368
369	parent_state = link->prev_performance_state;
370	link->performance_state = parent_state;
371
372	parent_state = _genpd_reeval_performance_state(genpd: parent,
373	state: parent_state);
374	if (_genpd_set_performance_state(genpd: parent, state: parent_state, depth: depth + 1)) {
375	pr_err("%s: Failed to roll back to %d performance state\n",
376	parent->name, parent_state);
377	}
378
379	genpd_unlock(parent);
380	}
381
382	return ret;
383	}
384
385	static int genpd_set_performance_state(struct device *dev, unsigned int state)
386	{
387	struct generic_pm_domain *genpd = dev_to_genpd(dev);
388	struct generic_pm_domain_data *gpd_data = dev_gpd_data(dev);
389	unsigned int prev_state;
390	int ret;
391
392	prev_state = gpd_data->performance_state;
393	if (prev_state == state)
394	return 0;
395
396	gpd_data->performance_state = state;
397	state = _genpd_reeval_performance_state(genpd, state);
398
399	ret = _genpd_set_performance_state(genpd, state, depth: 0);
400	if (ret)
401	gpd_data->performance_state = prev_state;
402
403	return ret;
404	}
405
406	static int genpd_drop_performance_state(struct device *dev)
407	{
408	unsigned int prev_state = dev_gpd_data(dev)->performance_state;
409
410	if (!genpd_set_performance_state(dev, state: 0))
411	return prev_state;
412
413	return 0;
414	}
415
416	static void genpd_restore_performance_state(struct device *dev,
417	unsigned int state)
418	{
419	if (state)
420	genpd_set_performance_state(dev, state);
421	}
422
423	static int genpd_dev_pm_set_performance_state(struct device *dev,
424	unsigned int state)
425	{
426	struct generic_pm_domain *genpd = dev_to_genpd(dev);
427	int ret = 0;
428
429	genpd_lock(genpd);
430	if (pm_runtime_suspended(dev)) {
431	dev_gpd_data(dev)->rpm_pstate = state;
432	} else {
433	ret = genpd_set_performance_state(dev, state);
434	if (!ret)
435	dev_gpd_data(dev)->rpm_pstate = 0;
436	}
437	genpd_unlock(genpd);
438
439	return ret;
440	}
441
442	/**
443	* dev_pm_genpd_set_performance_state- Set performance state of device's power
444	* domain.
445	*
446	* @dev: Device for which the performance-state needs to be set.
447	* @state: Target performance state of the device. This can be set as 0 when the
448	* device doesn't have any performance state constraints left (And so
449	* the device wouldn't participate anymore to find the target
450	* performance state of the genpd).
451	*
452	* It is assumed that the users guarantee that the genpd wouldn't be detached
453	* while this routine is getting called.
454	*
455	* Returns 0 on success and negative error values on failures.
456	*/
457	int dev_pm_genpd_set_performance_state(struct device *dev, unsigned int state)
458	{
459	struct generic_pm_domain *genpd;
460
461	genpd = dev_to_genpd_safe(dev);
462	if (!genpd)
463	return -ENODEV;
464
465	if (WARN_ON(!dev->power.subsys_data ||
466	!dev->power.subsys_data->domain_data))
467	return -EINVAL;
468
469	return genpd_dev_pm_set_performance_state(dev, state);
470	}
471	EXPORT_SYMBOL_GPL(dev_pm_genpd_set_performance_state);
472
473	/**
474	* dev_pm_genpd_set_next_wakeup - Notify PM framework of an impending wakeup.
475	*
476	* @dev: Device to handle
477	* @next: impending interrupt/wakeup for the device
478	*
479	*
480	* Allow devices to inform of the next wakeup. It's assumed that the users
481	* guarantee that the genpd wouldn't be detached while this routine is getting
482	* called. Additionally, it's also assumed that @dev isn't runtime suspended
483	* (RPM_SUSPENDED)."
484	* Although devices are expected to update the next_wakeup after the end of
485	* their usecase as well, it is possible the devices themselves may not know
486	* about that, so stale @next will be ignored when powering off the domain.
487	*/
488	void dev_pm_genpd_set_next_wakeup(struct device *dev, ktime_t next)
489	{
490	struct generic_pm_domain *genpd;
491	struct gpd_timing_data *td;
492
493	genpd = dev_to_genpd_safe(dev);
494	if (!genpd)
495	return;
496
497	td = to_gpd_data(pdd: dev->power.subsys_data->domain_data)->td;
498	if (td)
499	td->next_wakeup = next;
500	}
501	EXPORT_SYMBOL_GPL(dev_pm_genpd_set_next_wakeup);
502
503	/**
504	* dev_pm_genpd_get_next_hrtimer - Return the next_hrtimer for the genpd
505	* @dev: A device that is attached to the genpd.
506	*
507	* This routine should typically be called for a device, at the point of when a
508	* GENPD_NOTIFY_PRE_OFF notification has been sent for it.
509	*
510	* Returns the aggregated value of the genpd's next hrtimer or KTIME_MAX if no
511	* valid value have been set.
512	*/
513	ktime_t dev_pm_genpd_get_next_hrtimer(struct device *dev)
514	{
515	struct generic_pm_domain *genpd;
516
517	genpd = dev_to_genpd_safe(dev);
518	if (!genpd)
519	return KTIME_MAX;
520
521	if (genpd->gd)
522	return genpd->gd->next_hrtimer;
523
524	return KTIME_MAX;
525	}
526	EXPORT_SYMBOL_GPL(dev_pm_genpd_get_next_hrtimer);
527
528	/*
529	* dev_pm_genpd_synced_poweroff - Next power off should be synchronous
530	*
531	* @dev: A device that is attached to the genpd.
532	*
533	* Allows a consumer of the genpd to notify the provider that the next power off
534	* should be synchronous.
535	*
536	* It is assumed that the users guarantee that the genpd wouldn't be detached
537	* while this routine is getting called.
538	*/
539	void dev_pm_genpd_synced_poweroff(struct device *dev)
540	{
541	struct generic_pm_domain *genpd;
542
543	genpd = dev_to_genpd_safe(dev);
544	if (!genpd)
545	return;
546
547	genpd_lock(genpd);
548	genpd->synced_poweroff = true;
549	genpd_unlock(genpd);
550	}
551	EXPORT_SYMBOL_GPL(dev_pm_genpd_synced_poweroff);
552
553	static int _genpd_power_on(struct generic_pm_domain *genpd, bool timed)
554	{
555	unsigned int state_idx = genpd->state_idx;
556	ktime_t time_start;
557	s64 elapsed_ns;
558	int ret;
559
560	/* Notify consumers that we are about to power on. */
561	ret = raw_notifier_call_chain_robust(nh: &genpd->power_notifiers,
562	val_up: GENPD_NOTIFY_PRE_ON,
563	val_down: GENPD_NOTIFY_OFF, NULL);
564	ret = notifier_to_errno(ret);
565	if (ret)
566	return ret;
567
568	if (!genpd->power_on)
569	goto out;
570
571	timed = timed && genpd->gd && !genpd->states[state_idx].fwnode;
572	if (!timed) {
573	ret = genpd->power_on(genpd);
574	if (ret)
575	goto err;
576
577	goto out;
578	}
579
580	time_start = ktime_get();
581	ret = genpd->power_on(genpd);
582	if (ret)
583	goto err;
584
585	elapsed_ns = ktime_to_ns(ktime_sub(ktime_get(), time_start));
586	if (elapsed_ns <= genpd->states[state_idx].power_on_latency_ns)
587	goto out;
588
589	genpd->states[state_idx].power_on_latency_ns = elapsed_ns;
590	genpd->gd->max_off_time_changed = true;
591	pr_debug("%s: Power-%s latency exceeded, new value %lld ns\n",
592	genpd->name, "on", elapsed_ns);
593
594	out:
595	raw_notifier_call_chain(nh: &genpd->power_notifiers, val: GENPD_NOTIFY_ON, NULL);
596	genpd->synced_poweroff = false;
597	return 0;
598	err:
599	raw_notifier_call_chain(nh: &genpd->power_notifiers, val: GENPD_NOTIFY_OFF,
600	NULL);
601	return ret;
602	}
603
604	static int _genpd_power_off(struct generic_pm_domain *genpd, bool timed)
605	{
606	unsigned int state_idx = genpd->state_idx;
607	ktime_t time_start;
608	s64 elapsed_ns;
609	int ret;
610
611	/* Notify consumers that we are about to power off. */
612	ret = raw_notifier_call_chain_robust(nh: &genpd->power_notifiers,
613	val_up: GENPD_NOTIFY_PRE_OFF,
614	val_down: GENPD_NOTIFY_ON, NULL);
615	ret = notifier_to_errno(ret);
616	if (ret)
617	return ret;
618
619	if (!genpd->power_off)
620	goto out;
621
622	timed = timed && genpd->gd && !genpd->states[state_idx].fwnode;
623	if (!timed) {
624	ret = genpd->power_off(genpd);
625	if (ret)
626	goto busy;
627
628	goto out;
629	}
630
631	time_start = ktime_get();
632	ret = genpd->power_off(genpd);
633	if (ret)
634	goto busy;
635
636	elapsed_ns = ktime_to_ns(ktime_sub(ktime_get(), time_start));
637	if (elapsed_ns <= genpd->states[state_idx].power_off_latency_ns)
638	goto out;
639
640	genpd->states[state_idx].power_off_latency_ns = elapsed_ns;
641	genpd->gd->max_off_time_changed = true;
642	pr_debug("%s: Power-%s latency exceeded, new value %lld ns\n",
643	genpd->name, "off", elapsed_ns);
644
645	out:
646	raw_notifier_call_chain(nh: &genpd->power_notifiers, val: GENPD_NOTIFY_OFF,
647	NULL);
648	return 0;
649	busy:
650	raw_notifier_call_chain(nh: &genpd->power_notifiers, val: GENPD_NOTIFY_ON, NULL);
651	return ret;
652	}
653
654	/**
655	* genpd_queue_power_off_work - Queue up the execution of genpd_power_off().
656	* @genpd: PM domain to power off.
657	*
658	* Queue up the execution of genpd_power_off() unless it's already been done
659	* before.
660	*/
661	static void genpd_queue_power_off_work(struct generic_pm_domain *genpd)
662	{
663	queue_work(wq: pm_wq, work: &genpd->power_off_work);
664	}
665
666	/**
667	* genpd_power_off - Remove power from a given PM domain.
668	* @genpd: PM domain to power down.
669	* @one_dev_on: If invoked from genpd's ->runtime_suspend|resume() callback, the
670	* RPM status of the releated device is in an intermediate state, not yet turned
671	* into RPM_SUSPENDED. This means genpd_power_off() must allow one device to not
672	* be RPM_SUSPENDED, while it tries to power off the PM domain.
673	* @depth: nesting count for lockdep.
674	*
675	* If all of the @genpd's devices have been suspended and all of its subdomains
676	* have been powered down, remove power from @genpd.
677	*/
678	static int genpd_power_off(struct generic_pm_domain *genpd, bool one_dev_on,
679	unsigned int depth)
680	{
681	struct pm_domain_data *pdd;
682	struct gpd_link *link;
683	unsigned int not_suspended = 0;
684	int ret;
685
686	/*
687	* Do not try to power off the domain in the following situations:
688	* (1) The domain is already in the "power off" state.
689	* (2) System suspend is in progress.
690	*/
691	if (!genpd_status_on(genpd) || genpd->prepared_count > 0)
692	return 0;
693
694	/*
695	* Abort power off for the PM domain in the following situations:
696	* (1) The domain is configured as always on.
697	* (2) When the domain has a subdomain being powered on.
698	*/
699	if (genpd_is_always_on(genpd) ||
700	genpd_is_rpm_always_on(genpd) ||
701	atomic_read(v: &genpd->sd_count) > 0)
702	return -EBUSY;
703
704	/*
705	* The children must be in their deepest (powered-off) states to allow
706	* the parent to be powered off. Note that, there's no need for
707	* additional locking, as powering on a child, requires the parent's
708	* lock to be acquired first.
709	*/
710	list_for_each_entry(link, &genpd->parent_links, parent_node) {
711	struct generic_pm_domain *child = link->child;
712	if (child->state_idx < child->state_count - 1)
713	return -EBUSY;
714	}
715
716	list_for_each_entry(pdd, &genpd->dev_list, list_node) {
717	/*
718	* Do not allow PM domain to be powered off, when an IRQ safe
719	* device is part of a non-IRQ safe domain.
720	*/
721	if (!pm_runtime_suspended(dev: pdd->dev) ||
722	irq_safe_dev_in_sleep_domain(dev: pdd->dev, genpd))
723	not_suspended++;
724	}
725
726	if (not_suspended > 1 || (not_suspended == 1 && !one_dev_on))
727	return -EBUSY;
728
729	if (genpd->gov && genpd->gov->power_down_ok) {
730	if (!genpd->gov->power_down_ok(&genpd->domain))
731	return -EAGAIN;
732	}
733
734	/* Default to shallowest state. */
735	if (!genpd->gov)
736	genpd->state_idx = 0;
737
738	/* Don't power off, if a child domain is waiting to power on. */
739	if (atomic_read(v: &genpd->sd_count) > 0)
740	return -EBUSY;
741
742	ret = _genpd_power_off(genpd, timed: true);
743	if (ret) {
744	genpd->states[genpd->state_idx].rejected++;
745	return ret;
746	}
747
748	genpd->status = GENPD_STATE_OFF;
749	genpd_update_accounting(genpd);
750	genpd->states[genpd->state_idx].usage++;
751
752	list_for_each_entry(link, &genpd->child_links, child_node) {
753	genpd_sd_counter_dec(genpd: link->parent);
754	genpd_lock_nested(link->parent, depth + 1);
755	genpd_power_off(genpd: link->parent, one_dev_on: false, depth: depth + 1);
756	genpd_unlock(link->parent);
757	}
758
759	return 0;
760	}
761
762	/**
763	* genpd_power_on - Restore power to a given PM domain and its parents.
764	* @genpd: PM domain to power up.
765	* @depth: nesting count for lockdep.
766	*
767	* Restore power to @genpd and all of its parents so that it is possible to
768	* resume a device belonging to it.
769	*/
770	static int genpd_power_on(struct generic_pm_domain *genpd, unsigned int depth)
771	{
772	struct gpd_link *link;
773	int ret = 0;
774
775	if (genpd_status_on(genpd))
776	return 0;
777
778	/*
779	* The list is guaranteed not to change while the loop below is being
780	* executed, unless one of the parents' .power_on() callbacks fiddles
781	* with it.
782	*/
783	list_for_each_entry(link, &genpd->child_links, child_node) {
784	struct generic_pm_domain *parent = link->parent;
785
786	genpd_sd_counter_inc(genpd: parent);
787
788	genpd_lock_nested(parent, depth + 1);
789	ret = genpd_power_on(genpd: parent, depth: depth + 1);
790	genpd_unlock(parent);
791
792	if (ret) {
793	genpd_sd_counter_dec(genpd: parent);
794	goto err;
795	}
796	}
797
798	ret = _genpd_power_on(genpd, timed: true);
799	if (ret)
800	goto err;
801
802	genpd->status = GENPD_STATE_ON;
803	genpd_update_accounting(genpd);
804
805	return 0;
806
807	err:
808	list_for_each_entry_continue_reverse(link,
809	&genpd->child_links,
810	child_node) {
811	genpd_sd_counter_dec(genpd: link->parent);
812	genpd_lock_nested(link->parent, depth + 1);
813	genpd_power_off(genpd: link->parent, one_dev_on: false, depth: depth + 1);
814	genpd_unlock(link->parent);
815	}
816
817	return ret;
818	}
819
820	static int genpd_dev_pm_start(struct device *dev)
821	{
822	struct generic_pm_domain *genpd = dev_to_genpd(dev);
823
824	return genpd_start_dev(genpd, dev);
825	}
826
827	static int genpd_dev_pm_qos_notifier(struct notifier_block *nb,
828	unsigned long val, void *ptr)
829	{
830	struct generic_pm_domain_data *gpd_data;
831	struct device *dev;
832
833	gpd_data = container_of(nb, struct generic_pm_domain_data, nb);
834	dev = gpd_data->base.dev;
835
836	for (;;) {
837	struct generic_pm_domain *genpd = ERR_PTR(error: -ENODATA);
838	struct pm_domain_data *pdd;
839	struct gpd_timing_data *td;
840
841	spin_lock_irq(lock: &dev->power.lock);
842
843	pdd = dev->power.subsys_data ?
844	dev->power.subsys_data->domain_data : NULL;
845	if (pdd) {
846	td = to_gpd_data(pdd)->td;
847	if (td) {
848	td->constraint_changed = true;
849	genpd = dev_to_genpd(dev);
850	}
851	}
852
853	spin_unlock_irq(lock: &dev->power.lock);
854
855	if (!IS_ERR(ptr: genpd)) {
856	genpd_lock(genpd);
857	genpd->gd->max_off_time_changed = true;
858	genpd_unlock(genpd);
859	}
860
861	dev = dev->parent;
862	if (!dev || dev->power.ignore_children)
863	break;
864	}
865
866	return NOTIFY_DONE;
867	}
868
869	/**
870	* genpd_power_off_work_fn - Power off PM domain whose subdomain count is 0.
871	* @work: Work structure used for scheduling the execution of this function.
872	*/
873	static void genpd_power_off_work_fn(struct work_struct *work)
874	{
875	struct generic_pm_domain *genpd;
876
877	genpd = container_of(work, struct generic_pm_domain, power_off_work);
878
879	genpd_lock(genpd);
880	genpd_power_off(genpd, one_dev_on: false, depth: 0);
881	genpd_unlock(genpd);
882	}
883
884	/**
885	* __genpd_runtime_suspend - walk the hierarchy of ->runtime_suspend() callbacks
886	* @dev: Device to handle.
887	*/
888	static int __genpd_runtime_suspend(struct device *dev)
889	{
890	int (*cb)(struct device *__dev);
891
892	if (dev->type && dev->type->pm)
893	cb = dev->type->pm->runtime_suspend;
894	else if (dev->class && dev->class->pm)
895	cb = dev->class->pm->runtime_suspend;
896	else if (dev->bus && dev->bus->pm)
897	cb = dev->bus->pm->runtime_suspend;
898	else
899	cb = NULL;
900
901	if (!cb && dev->driver && dev->driver->pm)
902	cb = dev->driver->pm->runtime_suspend;
903
904	return cb ? cb(dev) : 0;
905	}
906
907	/**
908	* __genpd_runtime_resume - walk the hierarchy of ->runtime_resume() callbacks
909	* @dev: Device to handle.
910	*/
911	static int __genpd_runtime_resume(struct device *dev)
912	{
913	int (*cb)(struct device *__dev);
914
915	if (dev->type && dev->type->pm)
916	cb = dev->type->pm->runtime_resume;
917	else if (dev->class && dev->class->pm)
918	cb = dev->class->pm->runtime_resume;
919	else if (dev->bus && dev->bus->pm)
920	cb = dev->bus->pm->runtime_resume;
921	else
922	cb = NULL;
923
924	if (!cb && dev->driver && dev->driver->pm)
925	cb = dev->driver->pm->runtime_resume;
926
927	return cb ? cb(dev) : 0;
928	}
929
930	/**
931	* genpd_runtime_suspend - Suspend a device belonging to I/O PM domain.
932	* @dev: Device to suspend.
933	*
934	* Carry out a runtime suspend of a device under the assumption that its
935	* pm_domain field points to the domain member of an object of type
936	* struct generic_pm_domain representing a PM domain consisting of I/O devices.
937	*/
938	static int genpd_runtime_suspend(struct device *dev)
939	{
940	struct generic_pm_domain *genpd;
941	bool (*suspend_ok)(struct device *__dev);
942	struct generic_pm_domain_data *gpd_data = dev_gpd_data(dev);
943	struct gpd_timing_data *td = gpd_data->td;
944	bool runtime_pm = pm_runtime_enabled(dev);
945	ktime_t time_start = 0;
946	s64 elapsed_ns;
947	int ret;
948
949	dev_dbg(dev, "%s()\n", __func__);
950
951	genpd = dev_to_genpd(dev);
952	if (IS_ERR(ptr: genpd))
953	return -EINVAL;
954
955	/*
956	* A runtime PM centric subsystem/driver may re-use the runtime PM
957	* callbacks for other purposes than runtime PM. In those scenarios
958	* runtime PM is disabled. Under these circumstances, we shall skip
959	* validating/measuring the PM QoS latency.
960	*/
961	suspend_ok = genpd->gov ? genpd->gov->suspend_ok : NULL;
962	if (runtime_pm && suspend_ok && !suspend_ok(dev))
963	return -EBUSY;
964
965	/* Measure suspend latency. */
966	if (td && runtime_pm)
967	time_start = ktime_get();
968
969	ret = __genpd_runtime_suspend(dev);
970	if (ret)
971	return ret;
972
973	ret = genpd_stop_dev(genpd, dev);
974	if (ret) {
975	__genpd_runtime_resume(dev);
976	return ret;
977	}
978
979	/* Update suspend latency value if the measured time exceeds it. */
980	if (td && runtime_pm) {
981	elapsed_ns = ktime_to_ns(ktime_sub(ktime_get(), time_start));
982	if (elapsed_ns > td->suspend_latency_ns) {
983	td->suspend_latency_ns = elapsed_ns;
984	dev_dbg(dev, "suspend latency exceeded, %lld ns\n",
985	elapsed_ns);
986	genpd->gd->max_off_time_changed = true;
987	td->constraint_changed = true;
988	}
989	}
990
991	/*
992	* If power.irq_safe is set, this routine may be run with
993	* IRQs disabled, so suspend only if the PM domain also is irq_safe.
994	*/
995	if (irq_safe_dev_in_sleep_domain(dev, genpd))
996	return 0;
997
998	genpd_lock(genpd);
999	genpd_power_off(genpd, one_dev_on: true, depth: 0);
1000	gpd_data->rpm_pstate = genpd_drop_performance_state(dev);
1001	genpd_unlock(genpd);
1002
1003	return 0;
1004	}
1005
1006	/**
1007	* genpd_runtime_resume - Resume a device belonging to I/O PM domain.
1008	* @dev: Device to resume.
1009	*
1010	* Carry out a runtime resume of a device under the assumption that its
1011	* pm_domain field points to the domain member of an object of type
1012	* struct generic_pm_domain representing a PM domain consisting of I/O devices.
1013	*/
1014	static int genpd_runtime_resume(struct device *dev)
1015	{
1016	struct generic_pm_domain *genpd;
1017	struct generic_pm_domain_data *gpd_data = dev_gpd_data(dev);
1018	struct gpd_timing_data *td = gpd_data->td;
1019	bool timed = td && pm_runtime_enabled(dev);
1020	ktime_t time_start = 0;
1021	s64 elapsed_ns;
1022	int ret;
1023
1024	dev_dbg(dev, "%s()\n", __func__);
1025
1026	genpd = dev_to_genpd(dev);
1027	if (IS_ERR(ptr: genpd))
1028	return -EINVAL;
1029
1030	/*
1031	* As we don't power off a non IRQ safe domain, which holds
1032	* an IRQ safe device, we don't need to restore power to it.
1033	*/
1034	if (irq_safe_dev_in_sleep_domain(dev, genpd))
1035	goto out;
1036
1037	genpd_lock(genpd);
1038	genpd_restore_performance_state(dev, state: gpd_data->rpm_pstate);
1039	ret = genpd_power_on(genpd, depth: 0);
1040	genpd_unlock(genpd);
1041
1042	if (ret)
1043	return ret;
1044
1045	out:
1046	/* Measure resume latency. */
1047	if (timed)
1048	time_start = ktime_get();
1049
1050	ret = genpd_start_dev(genpd, dev);
1051	if (ret)
1052	goto err_poweroff;
1053
1054	ret = __genpd_runtime_resume(dev);
1055	if (ret)
1056	goto err_stop;
1057
1058	/* Update resume latency value if the measured time exceeds it. */
1059	if (timed) {
1060	elapsed_ns = ktime_to_ns(ktime_sub(ktime_get(), time_start));
1061	if (elapsed_ns > td->resume_latency_ns) {
1062	td->resume_latency_ns = elapsed_ns;
1063	dev_dbg(dev, "resume latency exceeded, %lld ns\n",
1064	elapsed_ns);
1065	genpd->gd->max_off_time_changed = true;
1066	td->constraint_changed = true;
1067	}
1068	}
1069
1070	return 0;
1071
1072	err_stop:
1073	genpd_stop_dev(genpd, dev);
1074	err_poweroff:
1075	if (!pm_runtime_is_irq_safe(dev) || genpd_is_irq_safe(genpd)) {
1076	genpd_lock(genpd);
1077	genpd_power_off(genpd, one_dev_on: true, depth: 0);
1078	gpd_data->rpm_pstate = genpd_drop_performance_state(dev);
1079	genpd_unlock(genpd);
1080	}
1081
1082	return ret;
1083	}
1084
1085	static bool pd_ignore_unused;
1086	static int __init pd_ignore_unused_setup(char *__unused)
1087	{
1088	pd_ignore_unused = true;
1089	return 1;
1090	}
1091	__setup("pd_ignore_unused", pd_ignore_unused_setup);
1092
1093	/**
1094	* genpd_power_off_unused - Power off all PM domains with no devices in use.
1095	*/
1096	static int __init genpd_power_off_unused(void)
1097	{
1098	struct generic_pm_domain *genpd;
1099
1100	if (pd_ignore_unused) {
1101	pr_warn("genpd: Not disabling unused power domains\n");
1102	return 0;
1103	}
1104
1105	mutex_lock(&gpd_list_lock);
1106
1107	list_for_each_entry(genpd, &gpd_list, gpd_list_node)
1108	genpd_queue_power_off_work(genpd);
1109
1110	mutex_unlock(lock: &gpd_list_lock);
1111
1112	return 0;
1113	}
1114	late_initcall(genpd_power_off_unused);
1115
1116	#ifdef CONFIG_PM_SLEEP
1117
1118	/**
1119	* genpd_sync_power_off - Synchronously power off a PM domain and its parents.
1120	* @genpd: PM domain to power off, if possible.
1121	* @use_lock: use the lock.
1122	* @depth: nesting count for lockdep.
1123	*
1124	* Check if the given PM domain can be powered off (during system suspend or
1125	* hibernation) and do that if so. Also, in that case propagate to its parents.
1126	*
1127	* This function is only called in "noirq" and "syscore" stages of system power
1128	* transitions. The "noirq" callbacks may be executed asynchronously, thus in
1129	* these cases the lock must be held.
1130	*/
1131	static void genpd_sync_power_off(struct generic_pm_domain *genpd, bool use_lock,
1132	unsigned int depth)
1133	{
1134	struct gpd_link *link;
1135
1136	if (!genpd_status_on(genpd) || genpd_is_always_on(genpd))
1137	return;
1138
1139	if (genpd->suspended_count != genpd->device_count
1140	|| atomic_read(v: &genpd->sd_count) > 0)
1141	return;
1142
1143	/* Check that the children are in their deepest (powered-off) state. */
1144	list_for_each_entry(link, &genpd->parent_links, parent_node) {
1145	struct generic_pm_domain *child = link->child;
1146	if (child->state_idx < child->state_count - 1)
1147	return;
1148	}
1149
1150	/* Choose the deepest state when suspending */
1151	genpd->state_idx = genpd->state_count - 1;
1152	if (_genpd_power_off(genpd, timed: false))
1153	return;
1154
1155	genpd->status = GENPD_STATE_OFF;
1156
1157	list_for_each_entry(link, &genpd->child_links, child_node) {
1158	genpd_sd_counter_dec(genpd: link->parent);
1159
1160	if (use_lock)
1161	genpd_lock_nested(link->parent, depth + 1);
1162
1163	genpd_sync_power_off(genpd: link->parent, use_lock, depth: depth + 1);
1164
1165	if (use_lock)
1166	genpd_unlock(link->parent);
1167	}
1168	}
1169
1170	/**
1171	* genpd_sync_power_on - Synchronously power on a PM domain and its parents.
1172	* @genpd: PM domain to power on.
1173	* @use_lock: use the lock.
1174	* @depth: nesting count for lockdep.
1175	*
1176	* This function is only called in "noirq" and "syscore" stages of system power
1177	* transitions. The "noirq" callbacks may be executed asynchronously, thus in
1178	* these cases the lock must be held.
1179	*/
1180	static void genpd_sync_power_on(struct generic_pm_domain *genpd, bool use_lock,
1181	unsigned int depth)
1182	{
1183	struct gpd_link *link;
1184
1185	if (genpd_status_on(genpd))
1186	return;
1187
1188	list_for_each_entry(link, &genpd->child_links, child_node) {
1189	genpd_sd_counter_inc(genpd: link->parent);
1190
1191	if (use_lock)
1192	genpd_lock_nested(link->parent, depth + 1);
1193
1194	genpd_sync_power_on(genpd: link->parent, use_lock, depth: depth + 1);
1195
1196	if (use_lock)
1197	genpd_unlock(link->parent);
1198	}
1199
1200	_genpd_power_on(genpd, timed: false);
1201	genpd->status = GENPD_STATE_ON;
1202	}
1203
1204	/**
1205	* genpd_prepare - Start power transition of a device in a PM domain.
1206	* @dev: Device to start the transition of.
1207	*
1208	* Start a power transition of a device (during a system-wide power transition)
1209	* under the assumption that its pm_domain field points to the domain member of
1210	* an object of type struct generic_pm_domain representing a PM domain
1211	* consisting of I/O devices.
1212	*/
1213	static int genpd_prepare(struct device *dev)
1214	{
1215	struct generic_pm_domain *genpd;
1216	int ret;
1217
1218	dev_dbg(dev, "%s()\n", __func__);
1219
1220	genpd = dev_to_genpd(dev);
1221	if (IS_ERR(ptr: genpd))
1222	return -EINVAL;
1223
1224	genpd_lock(genpd);
1225
1226	if (genpd->prepared_count++ == 0)
1227	genpd->suspended_count = 0;
1228
1229	genpd_unlock(genpd);
1230
1231	ret = pm_generic_prepare(dev);
1232	if (ret < 0) {
1233	genpd_lock(genpd);
1234
1235	genpd->prepared_count--;
1236
1237	genpd_unlock(genpd);
1238	}
1239
1240	/* Never return 1, as genpd don't cope with the direct_complete path. */
1241	return ret >= 0 ? 0 : ret;
1242	}
1243
1244	/**
1245	* genpd_finish_suspend - Completion of suspend or hibernation of device in an
1246	* I/O pm domain.
1247	* @dev: Device to suspend.
1248	* @suspend_noirq: Generic suspend_noirq callback.
1249	* @resume_noirq: Generic resume_noirq callback.
1250	*
1251	* Stop the device and remove power from the domain if all devices in it have
1252	* been stopped.
1253	*/
1254	static int genpd_finish_suspend(struct device *dev,
1255	int (*suspend_noirq)(struct device *dev),
1256	int (*resume_noirq)(struct device *dev))
1257	{
1258	struct generic_pm_domain *genpd;
1259	int ret = 0;
1260
1261	genpd = dev_to_genpd(dev);
1262	if (IS_ERR(ptr: genpd))
1263	return -EINVAL;
1264
1265	ret = suspend_noirq(dev);
1266	if (ret)
1267	return ret;
1268
1269	if (device_wakeup_path(dev) && genpd_is_active_wakeup(genpd))
1270	return 0;
1271
1272	if (genpd->dev_ops.stop && genpd->dev_ops.start &&
1273	!pm_runtime_status_suspended(dev)) {
1274	ret = genpd_stop_dev(genpd, dev);
1275	if (ret) {
1276	resume_noirq(dev);
1277	return ret;
1278	}
1279	}
1280
1281	genpd_lock(genpd);
1282	genpd->suspended_count++;
1283	genpd_sync_power_off(genpd, use_lock: true, depth: 0);
1284	genpd_unlock(genpd);
1285
1286	return 0;
1287	}
1288
1289	/**
1290	* genpd_suspend_noirq - Completion of suspend of device in an I/O PM domain.
1291	* @dev: Device to suspend.
1292	*
1293	* Stop the device and remove power from the domain if all devices in it have
1294	* been stopped.
1295	*/
1296	static int genpd_suspend_noirq(struct device *dev)
1297	{
1298	dev_dbg(dev, "%s()\n", __func__);
1299
1300	return genpd_finish_suspend(dev,
1301	suspend_noirq: pm_generic_suspend_noirq,
1302	resume_noirq: pm_generic_resume_noirq);
1303	}
1304
1305	/**
1306	* genpd_finish_resume - Completion of resume of device in an I/O PM domain.
1307	* @dev: Device to resume.
1308	* @resume_noirq: Generic resume_noirq callback.
1309	*
1310	* Restore power to the device's PM domain, if necessary, and start the device.
1311	*/
1312	static int genpd_finish_resume(struct device *dev,
1313	int (*resume_noirq)(struct device *dev))
1314	{
1315	struct generic_pm_domain *genpd;
1316	int ret;
1317
1318	dev_dbg(dev, "%s()\n", __func__);
1319
1320	genpd = dev_to_genpd(dev);
1321	if (IS_ERR(ptr: genpd))
1322	return -EINVAL;
1323
1324	if (device_wakeup_path(dev) && genpd_is_active_wakeup(genpd))
1325	return resume_noirq(dev);
1326
1327	genpd_lock(genpd);
1328	genpd_sync_power_on(genpd, use_lock: true, depth: 0);
1329	genpd->suspended_count--;
1330	genpd_unlock(genpd);
1331
1332	if (genpd->dev_ops.stop && genpd->dev_ops.start &&
1333	!pm_runtime_status_suspended(dev)) {
1334	ret = genpd_start_dev(genpd, dev);
1335	if (ret)
1336	return ret;
1337	}
1338
1339	return pm_generic_resume_noirq(dev);
1340	}
1341
1342	/**
1343	* genpd_resume_noirq - Start of resume of device in an I/O PM domain.
1344	* @dev: Device to resume.
1345	*
1346	* Restore power to the device's PM domain, if necessary, and start the device.
1347	*/
1348	static int genpd_resume_noirq(struct device *dev)
1349	{
1350	dev_dbg(dev, "%s()\n", __func__);
1351
1352	return genpd_finish_resume(dev, resume_noirq: pm_generic_resume_noirq);
1353	}
1354
1355	/**
1356	* genpd_freeze_noirq - Completion of freezing a device in an I/O PM domain.
1357	* @dev: Device to freeze.
1358	*
1359	* Carry out a late freeze of a device under the assumption that its
1360	* pm_domain field points to the domain member of an object of type
1361	* struct generic_pm_domain representing a power domain consisting of I/O
1362	* devices.
1363	*/
1364	static int genpd_freeze_noirq(struct device *dev)
1365	{
1366	dev_dbg(dev, "%s()\n", __func__);
1367
1368	return genpd_finish_suspend(dev,
1369	suspend_noirq: pm_generic_freeze_noirq,
1370	resume_noirq: pm_generic_thaw_noirq);
1371	}
1372
1373	/**
1374	* genpd_thaw_noirq - Early thaw of device in an I/O PM domain.
1375	* @dev: Device to thaw.
1376	*
1377	* Start the device, unless power has been removed from the domain already
1378	* before the system transition.
1379	*/
1380	static int genpd_thaw_noirq(struct device *dev)
1381	{
1382	dev_dbg(dev, "%s()\n", __func__);
1383
1384	return genpd_finish_resume(dev, resume_noirq: pm_generic_thaw_noirq);
1385	}
1386
1387	/**
1388	* genpd_poweroff_noirq - Completion of hibernation of device in an
1389	* I/O PM domain.
1390	* @dev: Device to poweroff.
1391	*
1392	* Stop the device and remove power from the domain if all devices in it have
1393	* been stopped.
1394	*/
1395	static int genpd_poweroff_noirq(struct device *dev)
1396	{
1397	dev_dbg(dev, "%s()\n", __func__);
1398
1399	return genpd_finish_suspend(dev,
1400	suspend_noirq: pm_generic_poweroff_noirq,
1401	resume_noirq: pm_generic_restore_noirq);
1402	}
1403
1404	/**
1405	* genpd_restore_noirq - Start of restore of device in an I/O PM domain.
1406	* @dev: Device to resume.
1407	*
1408	* Make sure the domain will be in the same power state as before the
1409	* hibernation the system is resuming from and start the device if necessary.
1410	*/
1411	static int genpd_restore_noirq(struct device *dev)
1412	{
1413	dev_dbg(dev, "%s()\n", __func__);
1414
1415	return genpd_finish_resume(dev, resume_noirq: pm_generic_restore_noirq);
1416	}
1417
1418	/**
1419	* genpd_complete - Complete power transition of a device in a power domain.
1420	* @dev: Device to complete the transition of.
1421	*
1422	* Complete a power transition of a device (during a system-wide power
1423	* transition) under the assumption that its pm_domain field points to the
1424	* domain member of an object of type struct generic_pm_domain representing
1425	* a power domain consisting of I/O devices.
1426	*/
1427	static void genpd_complete(struct device *dev)
1428	{
1429	struct generic_pm_domain *genpd;
1430
1431	dev_dbg(dev, "%s()\n", __func__);
1432
1433	genpd = dev_to_genpd(dev);
1434	if (IS_ERR(ptr: genpd))
1435	return;
1436
1437	pm_generic_complete(dev);
1438
1439	genpd_lock(genpd);
1440
1441	genpd->prepared_count--;
1442	if (!genpd->prepared_count)
1443	genpd_queue_power_off_work(genpd);
1444
1445	genpd_unlock(genpd);
1446	}
1447
1448	static void genpd_switch_state(struct device *dev, bool suspend)
1449	{
1450	struct generic_pm_domain *genpd;
1451	bool use_lock;
1452
1453	genpd = dev_to_genpd_safe(dev);
1454	if (!genpd)
1455	return;
1456
1457	use_lock = genpd_is_irq_safe(genpd);
1458
1459	if (use_lock)
1460	genpd_lock(genpd);
1461
1462	if (suspend) {
1463	genpd->suspended_count++;
1464	genpd_sync_power_off(genpd, use_lock, depth: 0);
1465	} else {
1466	genpd_sync_power_on(genpd, use_lock, depth: 0);
1467	genpd->suspended_count--;
1468	}
1469
1470	if (use_lock)
1471	genpd_unlock(genpd);
1472	}
1473
1474	/**
1475	* dev_pm_genpd_suspend - Synchronously try to suspend the genpd for @dev
1476	* @dev: The device that is attached to the genpd, that can be suspended.
1477	*
1478	* This routine should typically be called for a device that needs to be
1479	* suspended during the syscore suspend phase. It may also be called during
1480	* suspend-to-idle to suspend a corresponding CPU device that is attached to a
1481	* genpd.
1482	*/
1483	void dev_pm_genpd_suspend(struct device *dev)
1484	{
1485	genpd_switch_state(dev, suspend: true);
1486	}
1487	EXPORT_SYMBOL_GPL(dev_pm_genpd_suspend);
1488
1489	/**
1490	* dev_pm_genpd_resume - Synchronously try to resume the genpd for @dev
1491	* @dev: The device that is attached to the genpd, which needs to be resumed.
1492	*
1493	* This routine should typically be called for a device that needs to be resumed
1494	* during the syscore resume phase. It may also be called during suspend-to-idle
1495	* to resume a corresponding CPU device that is attached to a genpd.
1496	*/
1497	void dev_pm_genpd_resume(struct device *dev)
1498	{
1499	genpd_switch_state(dev, suspend: false);
1500	}
1501	EXPORT_SYMBOL_GPL(dev_pm_genpd_resume);
1502
1503	#else /* !CONFIG_PM_SLEEP */
1504
1505	#define genpd_prepare NULL
1506	#define genpd_suspend_noirq NULL
1507	#define genpd_resume_noirq NULL
1508	#define genpd_freeze_noirq NULL
1509	#define genpd_thaw_noirq NULL
1510	#define genpd_poweroff_noirq NULL
1511	#define genpd_restore_noirq NULL
1512	#define genpd_complete NULL
1513
1514	#endif /* CONFIG_PM_SLEEP */
1515
1516	static struct generic_pm_domain_data *genpd_alloc_dev_data(struct device *dev,
1517	bool has_governor)
1518	{
1519	struct generic_pm_domain_data *gpd_data;
1520	struct gpd_timing_data *td;
1521	int ret;
1522
1523	ret = dev_pm_get_subsys_data(dev);
1524	if (ret)
1525	return ERR_PTR(error: ret);
1526
1527	gpd_data = kzalloc(size: sizeof(*gpd_data), GFP_KERNEL);
1528	if (!gpd_data) {
1529	ret = -ENOMEM;
1530	goto err_put;
1531	}
1532
1533	gpd_data->base.dev = dev;
1534	gpd_data->nb.notifier_call = genpd_dev_pm_qos_notifier;
1535
1536	/* Allocate data used by a governor. */
1537	if (has_governor) {
1538	td = kzalloc(size: sizeof(*td), GFP_KERNEL);
1539	if (!td) {
1540	ret = -ENOMEM;
1541	goto err_free;
1542	}
1543
1544	td->constraint_changed = true;
1545	td->effective_constraint_ns = PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS;
1546	td->next_wakeup = KTIME_MAX;
1547	gpd_data->td = td;
1548	}
1549
1550	spin_lock_irq(lock: &dev->power.lock);
1551
1552	if (dev->power.subsys_data->domain_data)
1553	ret = -EINVAL;
1554	else
1555	dev->power.subsys_data->domain_data = &gpd_data->base;
1556
1557	spin_unlock_irq(lock: &dev->power.lock);
1558
1559	if (ret)
1560	goto err_free;
1561
1562	return gpd_data;
1563
1564	err_free:
1565	kfree(objp: gpd_data->td);
1566	kfree(objp: gpd_data);
1567	err_put:
1568	dev_pm_put_subsys_data(dev);
1569	return ERR_PTR(error: ret);
1570	}
1571
1572	static void genpd_free_dev_data(struct device *dev,
1573	struct generic_pm_domain_data *gpd_data)
1574	{
1575	spin_lock_irq(lock: &dev->power.lock);
1576
1577	dev->power.subsys_data->domain_data = NULL;
1578
1579	spin_unlock_irq(lock: &dev->power.lock);
1580
1581	kfree(objp: gpd_data->td);
1582	kfree(objp: gpd_data);
1583	dev_pm_put_subsys_data(dev);
1584	}
1585
1586	static void genpd_update_cpumask(struct generic_pm_domain *genpd,
1587	int cpu, bool set, unsigned int depth)
1588	{
1589	struct gpd_link *link;
1590
1591	if (!genpd_is_cpu_domain(genpd))
1592	return;
1593
1594	list_for_each_entry(link, &genpd->child_links, child_node) {
1595	struct generic_pm_domain *parent = link->parent;
1596
1597	genpd_lock_nested(parent, depth + 1);
1598	genpd_update_cpumask(genpd: parent, cpu, set, depth: depth + 1);
1599	genpd_unlock(parent);
1600	}
1601
1602	if (set)
1603	cpumask_set_cpu(cpu, dstp: genpd->cpus);
1604	else
1605	cpumask_clear_cpu(cpu, dstp: genpd->cpus);
1606	}
1607
1608	static void genpd_set_cpumask(struct generic_pm_domain *genpd, int cpu)
1609	{
1610	if (cpu >= 0)
1611	genpd_update_cpumask(genpd, cpu, set: true, depth: 0);
1612	}
1613
1614	static void genpd_clear_cpumask(struct generic_pm_domain *genpd, int cpu)
1615	{
1616	if (cpu >= 0)
1617	genpd_update_cpumask(genpd, cpu, set: false, depth: 0);
1618	}
1619
1620	static int genpd_get_cpu(struct generic_pm_domain *genpd, struct device *dev)
1621	{
1622	int cpu;
1623
1624	if (!genpd_is_cpu_domain(genpd))
1625	return -1;
1626
1627	for_each_possible_cpu(cpu) {
1628	if (get_cpu_device(cpu) == dev)
1629	return cpu;
1630	}
1631
1632	return -1;
1633	}
1634
1635	static int genpd_add_device(struct generic_pm_domain *genpd, struct device *dev,
1636	struct device *base_dev)
1637	{
1638	struct genpd_governor_data *gd = genpd->gd;
1639	struct generic_pm_domain_data *gpd_data;
1640	int ret;
1641
1642	dev_dbg(dev, "%s()\n", __func__);
1643
1644	gpd_data = genpd_alloc_dev_data(dev, has_governor: gd);
1645	if (IS_ERR(ptr: gpd_data))
1646	return PTR_ERR(ptr: gpd_data);
1647
1648	gpd_data->cpu = genpd_get_cpu(genpd, dev: base_dev);
1649
1650	ret = genpd->attach_dev ? genpd->attach_dev(genpd, dev) : 0;
1651	if (ret)
1652	goto out;
1653
1654	genpd_lock(genpd);
1655
1656	genpd_set_cpumask(genpd, cpu: gpd_data->cpu);
1657	dev_pm_domain_set(dev, pd: &genpd->domain);
1658
1659	genpd->device_count++;
1660	if (gd)
1661	gd->max_off_time_changed = true;
1662
1663	list_add_tail(new: &gpd_data->base.list_node, head: &genpd->dev_list);
1664
1665	genpd_unlock(genpd);
1666	out:
1667	if (ret)
1668	genpd_free_dev_data(dev, gpd_data);
1669	else
1670	dev_pm_qos_add_notifier(dev, notifier: &gpd_data->nb,
1671	type: DEV_PM_QOS_RESUME_LATENCY);
1672
1673	return ret;
1674	}
1675
1676	/**
1677	* pm_genpd_add_device - Add a device to an I/O PM domain.
1678	* @genpd: PM domain to add the device to.
1679	* @dev: Device to be added.
1680	*/
1681	int pm_genpd_add_device(struct generic_pm_domain *genpd, struct device *dev)
1682	{
1683	int ret;
1684
1685	if (!genpd || !dev)
1686	return -EINVAL;
1687
1688	mutex_lock(&gpd_list_lock);
1689	ret = genpd_add_device(genpd, dev, base_dev: dev);
1690	mutex_unlock(lock: &gpd_list_lock);
1691
1692	return ret;
1693	}
1694	EXPORT_SYMBOL_GPL(pm_genpd_add_device);
1695
1696	static int genpd_remove_device(struct generic_pm_domain *genpd,
1697	struct device *dev)
1698	{
1699	struct generic_pm_domain_data *gpd_data;
1700	struct pm_domain_data *pdd;
1701	int ret = 0;
1702
1703	dev_dbg(dev, "%s()\n", __func__);
1704
1705	pdd = dev->power.subsys_data->domain_data;
1706	gpd_data = to_gpd_data(pdd);
1707	dev_pm_qos_remove_notifier(dev, notifier: &gpd_data->nb,
1708	type: DEV_PM_QOS_RESUME_LATENCY);
1709
1710	genpd_lock(genpd);
1711
1712	if (genpd->prepared_count > 0) {
1713	ret = -EAGAIN;
1714	goto out;
1715	}
1716
1717	genpd->device_count--;
1718	if (genpd->gd)
1719	genpd->gd->max_off_time_changed = true;
1720
1721	genpd_clear_cpumask(genpd, cpu: gpd_data->cpu);
1722	dev_pm_domain_set(dev, NULL);
1723
1724	list_del_init(entry: &pdd->list_node);
1725
1726	genpd_unlock(genpd);
1727
1728	if (genpd->detach_dev)
1729	genpd->detach_dev(genpd, dev);
1730
1731	genpd_free_dev_data(dev, gpd_data);
1732
1733	return 0;
1734
1735	out:
1736	genpd_unlock(genpd);
1737	dev_pm_qos_add_notifier(dev, notifier: &gpd_data->nb, type: DEV_PM_QOS_RESUME_LATENCY);
1738
1739	return ret;
1740	}
1741
1742	/**
1743	* pm_genpd_remove_device - Remove a device from an I/O PM domain.
1744	* @dev: Device to be removed.
1745	*/
1746	int pm_genpd_remove_device(struct device *dev)
1747	{
1748	struct generic_pm_domain *genpd = dev_to_genpd_safe(dev);
1749
1750	if (!genpd)
1751	return -EINVAL;
1752
1753	return genpd_remove_device(genpd, dev);
1754	}
1755	EXPORT_SYMBOL_GPL(pm_genpd_remove_device);
1756
1757	/**
1758	* dev_pm_genpd_add_notifier - Add a genpd power on/off notifier for @dev
1759	*
1760	* @dev: Device that should be associated with the notifier
1761	* @nb: The notifier block to register
1762	*
1763	* Users may call this function to add a genpd power on/off notifier for an
1764	* attached @dev. Only one notifier per device is allowed. The notifier is
1765	* sent when genpd is powering on/off the PM domain.
1766	*
1767	* It is assumed that the user guarantee that the genpd wouldn't be detached
1768	* while this routine is getting called.
1769	*
1770	* Returns 0 on success and negative error values on failures.
1771	*/
1772	int dev_pm_genpd_add_notifier(struct device *dev, struct notifier_block *nb)
1773	{
1774	struct generic_pm_domain *genpd;
1775	struct generic_pm_domain_data *gpd_data;
1776	int ret;
1777
1778	genpd = dev_to_genpd_safe(dev);
1779	if (!genpd)
1780	return -ENODEV;
1781
1782	if (WARN_ON(!dev->power.subsys_data ||
1783	!dev->power.subsys_data->domain_data))
1784	return -EINVAL;
1785
1786	gpd_data = to_gpd_data(pdd: dev->power.subsys_data->domain_data);
1787	if (gpd_data->power_nb)
1788	return -EEXIST;
1789
1790	genpd_lock(genpd);
1791	ret = raw_notifier_chain_register(nh: &genpd->power_notifiers, nb);
1792	genpd_unlock(genpd);
1793
1794	if (ret) {
1795	dev_warn(dev, "failed to add notifier for PM domain %s\n",
1796	genpd->name);
1797	return ret;
1798	}
1799
1800	gpd_data->power_nb = nb;
1801	return 0;
1802	}
1803	EXPORT_SYMBOL_GPL(dev_pm_genpd_add_notifier);
1804
1805	/**
1806	* dev_pm_genpd_remove_notifier - Remove a genpd power on/off notifier for @dev
1807	*
1808	* @dev: Device that is associated with the notifier
1809	*
1810	* Users may call this function to remove a genpd power on/off notifier for an
1811	* attached @dev.
1812	*
1813	* It is assumed that the user guarantee that the genpd wouldn't be detached
1814	* while this routine is getting called.
1815	*
1816	* Returns 0 on success and negative error values on failures.
1817	*/
1818	int dev_pm_genpd_remove_notifier(struct device *dev)
1819	{
1820	struct generic_pm_domain *genpd;
1821	struct generic_pm_domain_data *gpd_data;
1822	int ret;
1823
1824	genpd = dev_to_genpd_safe(dev);
1825	if (!genpd)
1826	return -ENODEV;
1827
1828	if (WARN_ON(!dev->power.subsys_data ||
1829	!dev->power.subsys_data->domain_data))
1830	return -EINVAL;
1831
1832	gpd_data = to_gpd_data(pdd: dev->power.subsys_data->domain_data);
1833	if (!gpd_data->power_nb)
1834	return -ENODEV;
1835
1836	genpd_lock(genpd);
1837	ret = raw_notifier_chain_unregister(nh: &genpd->power_notifiers,
1838	nb: gpd_data->power_nb);
1839	genpd_unlock(genpd);
1840
1841	if (ret) {
1842	dev_warn(dev, "failed to remove notifier for PM domain %s\n",
1843	genpd->name);
1844	return ret;
1845	}
1846
1847	gpd_data->power_nb = NULL;
1848	return 0;
1849	}
1850	EXPORT_SYMBOL_GPL(dev_pm_genpd_remove_notifier);
1851
1852	static int genpd_add_subdomain(struct generic_pm_domain *genpd,
1853	struct generic_pm_domain *subdomain)
1854	{
1855	struct gpd_link *link, *itr;
1856	int ret = 0;
1857
1858	if (IS_ERR_OR_NULL(ptr: genpd) || IS_ERR_OR_NULL(ptr: subdomain)
1859	|| genpd == subdomain)
1860	return -EINVAL;
1861
1862	/*
1863	* If the domain can be powered on/off in an IRQ safe
1864	* context, ensure that the subdomain can also be
1865	* powered on/off in that context.
1866	*/
1867	if (!genpd_is_irq_safe(genpd) && genpd_is_irq_safe(subdomain)) {
1868	WARN(1, "Parent %s of subdomain %s must be IRQ safe\n",
1869	genpd->name, subdomain->name);
1870	return -EINVAL;
1871	}
1872
1873	link = kzalloc(size: sizeof(*link), GFP_KERNEL);
1874	if (!link)
1875	return -ENOMEM;
1876
1877	genpd_lock(subdomain);
1878	genpd_lock_nested(genpd, SINGLE_DEPTH_NESTING);
1879
1880	if (!genpd_status_on(genpd) && genpd_status_on(subdomain)) {
1881	ret = -EINVAL;
1882	goto out;
1883	}
1884
1885	list_for_each_entry(itr, &genpd->parent_links, parent_node) {
1886	if (itr->child == subdomain && itr->parent == genpd) {
1887	ret = -EINVAL;
1888	goto out;
1889	}
1890	}
1891
1892	link->parent = genpd;
1893	list_add_tail(new: &link->parent_node, head: &genpd->parent_links);
1894	link->child = subdomain;
1895	list_add_tail(new: &link->child_node, head: &subdomain->child_links);
1896	if (genpd_status_on(subdomain))
1897	genpd_sd_counter_inc(genpd);
1898
1899	out:
1900	genpd_unlock(genpd);
1901	genpd_unlock(subdomain);
1902	if (ret)
1903	kfree(objp: link);
1904	return ret;
1905	}
1906
1907	/**
1908	* pm_genpd_add_subdomain - Add a subdomain to an I/O PM domain.
1909	* @genpd: Leader PM domain to add the subdomain to.
1910	* @subdomain: Subdomain to be added.
1911	*/
1912	int pm_genpd_add_subdomain(struct generic_pm_domain *genpd,
1913	struct generic_pm_domain *subdomain)
1914	{
1915	int ret;
1916
1917	mutex_lock(&gpd_list_lock);
1918	ret = genpd_add_subdomain(genpd, subdomain);
1919	mutex_unlock(lock: &gpd_list_lock);
1920
1921	return ret;
1922	}
1923	EXPORT_SYMBOL_GPL(pm_genpd_add_subdomain);
1924
1925	/**
1926	* pm_genpd_remove_subdomain - Remove a subdomain from an I/O PM domain.
1927	* @genpd: Leader PM domain to remove the subdomain from.
1928	* @subdomain: Subdomain to be removed.
1929	*/
1930	int pm_genpd_remove_subdomain(struct generic_pm_domain *genpd,
1931	struct generic_pm_domain *subdomain)
1932	{
1933	struct gpd_link *l, *link;
1934	int ret = -EINVAL;
1935
1936	if (IS_ERR_OR_NULL(ptr: genpd) || IS_ERR_OR_NULL(ptr: subdomain))
1937	return -EINVAL;
1938
1939	genpd_lock(subdomain);
1940	genpd_lock_nested(genpd, SINGLE_DEPTH_NESTING);
1941
1942	if (!list_empty(head: &subdomain->parent_links) || subdomain->device_count) {
1943	pr_warn("%s: unable to remove subdomain %s\n",
1944	genpd->name, subdomain->name);
1945	ret = -EBUSY;
1946	goto out;
1947	}
1948
1949	list_for_each_entry_safe(link, l, &genpd->parent_links, parent_node) {
1950	if (link->child != subdomain)
1951	continue;
1952
1953	list_del(entry: &link->parent_node);
1954	list_del(entry: &link->child_node);
1955	kfree(objp: link);
1956	if (genpd_status_on(subdomain))
1957	genpd_sd_counter_dec(genpd);
1958
1959	ret = 0;
1960	break;
1961	}
1962
1963	out:
1964	genpd_unlock(genpd);
1965	genpd_unlock(subdomain);
1966
1967	return ret;
1968	}
1969	EXPORT_SYMBOL_GPL(pm_genpd_remove_subdomain);
1970
1971	static void genpd_free_default_power_state(struct genpd_power_state *states,
1972	unsigned int state_count)
1973	{
1974	kfree(objp: states);
1975	}
1976
1977	static int genpd_set_default_power_state(struct generic_pm_domain *genpd)
1978	{
1979	struct genpd_power_state *state;
1980
1981	state = kzalloc(size: sizeof(*state), GFP_KERNEL);
1982	if (!state)
1983	return -ENOMEM;
1984
1985	genpd->states = state;
1986	genpd->state_count = 1;
1987	genpd->free_states = genpd_free_default_power_state;
1988
1989	return 0;
1990	}
1991
1992	static int genpd_alloc_data(struct generic_pm_domain *genpd)
1993	{
1994	struct genpd_governor_data *gd = NULL;
1995	int ret;
1996
1997	if (genpd_is_cpu_domain(genpd) &&
1998	!zalloc_cpumask_var(mask: &genpd->cpus, GFP_KERNEL))
1999	return -ENOMEM;
2000
2001	if (genpd->gov) {
2002	gd = kzalloc(size: sizeof(*gd), GFP_KERNEL);
2003	if (!gd) {
2004	ret = -ENOMEM;
2005	goto free;
2006	}
2007
2008	gd->max_off_time_ns = -1;
2009	gd->max_off_time_changed = true;
2010	gd->next_wakeup = KTIME_MAX;
2011	gd->next_hrtimer = KTIME_MAX;
2012	}
2013
2014	/* Use only one "off" state if there were no states declared */
2015	if (genpd->state_count == 0) {
2016	ret = genpd_set_default_power_state(genpd);
2017	if (ret)
2018	goto free;
2019	}
2020
2021	genpd->gd = gd;
2022	return 0;
2023
2024	free:
2025	if (genpd_is_cpu_domain(genpd))
2026	free_cpumask_var(mask: genpd->cpus);
2027	kfree(objp: gd);
2028	return ret;
2029	}
2030
2031	static void genpd_free_data(struct generic_pm_domain *genpd)
2032	{
2033	if (genpd_is_cpu_domain(genpd))
2034	free_cpumask_var(mask: genpd->cpus);
2035	if (genpd->free_states)
2036	genpd->free_states(genpd->states, genpd->state_count);
2037	kfree(objp: genpd->gd);
2038	}
2039
2040	static void genpd_lock_init(struct generic_pm_domain *genpd)
2041	{
2042	if (genpd->flags & GENPD_FLAG_IRQ_SAFE) {
2043	spin_lock_init(&genpd->slock);
2044	genpd->lock_ops = &genpd_spin_ops;
2045	} else {
2046	mutex_init(&genpd->mlock);
2047	genpd->lock_ops = &genpd_mtx_ops;
2048	}
2049	}
2050
2051	/**
2052	* pm_genpd_init - Initialize a generic I/O PM domain object.
2053	* @genpd: PM domain object to initialize.
2054	* @gov: PM domain governor to associate with the domain (may be NULL).
2055	* @is_off: Initial value of the domain's power_is_off field.
2056	*
2057	* Returns 0 on successful initialization, else a negative error code.
2058	*/
2059	int pm_genpd_init(struct generic_pm_domain *genpd,
2060	struct dev_power_governor *gov, bool is_off)
2061	{
2062	int ret;
2063
2064	if (IS_ERR_OR_NULL(ptr: genpd))
2065	return -EINVAL;
2066
2067	INIT_LIST_HEAD(list: &genpd->parent_links);
2068	INIT_LIST_HEAD(list: &genpd->child_links);
2069	INIT_LIST_HEAD(list: &genpd->dev_list);
2070	RAW_INIT_NOTIFIER_HEAD(&genpd->power_notifiers);
2071	genpd_lock_init(genpd);
2072	genpd->gov = gov;
2073	INIT_WORK(&genpd->power_off_work, genpd_power_off_work_fn);
2074	atomic_set(v: &genpd->sd_count, i: 0);
2075	genpd->status = is_off ? GENPD_STATE_OFF : GENPD_STATE_ON;
2076	genpd->device_count = 0;
2077	genpd->provider = NULL;
2078	genpd->has_provider = false;
2079	genpd->accounting_time = ktime_get_mono_fast_ns();
2080	genpd->domain.ops.runtime_suspend = genpd_runtime_suspend;
2081	genpd->domain.ops.runtime_resume = genpd_runtime_resume;
2082	genpd->domain.ops.prepare = genpd_prepare;
2083	genpd->domain.ops.suspend_noirq = genpd_suspend_noirq;
2084	genpd->domain.ops.resume_noirq = genpd_resume_noirq;
2085	genpd->domain.ops.freeze_noirq = genpd_freeze_noirq;
2086	genpd->domain.ops.thaw_noirq = genpd_thaw_noirq;
2087	genpd->domain.ops.poweroff_noirq = genpd_poweroff_noirq;
2088	genpd->domain.ops.restore_noirq = genpd_restore_noirq;
2089	genpd->domain.ops.complete = genpd_complete;
2090	genpd->domain.start = genpd_dev_pm_start;
2091	genpd->domain.set_performance_state = genpd_dev_pm_set_performance_state;
2092
2093	if (genpd->flags & GENPD_FLAG_PM_CLK) {
2094	genpd->dev_ops.stop = pm_clk_suspend;
2095	genpd->dev_ops.start = pm_clk_resume;
2096	}
2097
2098	/* The always-on governor works better with the corresponding flag. */
2099	if (gov == &pm_domain_always_on_gov)
2100	genpd->flags |= GENPD_FLAG_RPM_ALWAYS_ON;
2101
2102	/* Always-on domains must be powered on at initialization. */
2103	if ((genpd_is_always_on(genpd) || genpd_is_rpm_always_on(genpd)) &&
2104	!genpd_status_on(genpd)) {
2105	pr_err("always-on PM domain %s is not on\n", genpd->name);
2106	return -EINVAL;
2107	}
2108
2109	/* Multiple states but no governor doesn't make sense. */
2110	if (!gov && genpd->state_count > 1)
2111	pr_warn("%s: no governor for states\n", genpd->name);
2112
2113	ret = genpd_alloc_data(genpd);
2114	if (ret)
2115	return ret;
2116
2117	device_initialize(dev: &genpd->dev);
2118	dev_set_name(dev: &genpd->dev, name: "%s", genpd->name);
2119
2120	mutex_lock(&gpd_list_lock);
2121	list_add(new: &genpd->gpd_list_node, head: &gpd_list);
2122	mutex_unlock(lock: &gpd_list_lock);
2123	genpd_debug_add(genpd);
2124
2125	return 0;
2126	}
2127	EXPORT_SYMBOL_GPL(pm_genpd_init);
2128
2129	static int genpd_remove(struct generic_pm_domain *genpd)
2130	{
2131	struct gpd_link *l, *link;
2132
2133	if (IS_ERR_OR_NULL(ptr: genpd))
2134	return -EINVAL;
2135
2136	genpd_lock(genpd);
2137
2138	if (genpd->has_provider) {
2139	genpd_unlock(genpd);
2140	pr_err("Provider present, unable to remove %s\n", genpd->name);
2141	return -EBUSY;
2142	}
2143
2144	if (!list_empty(head: &genpd->parent_links) || genpd->device_count) {
2145	genpd_unlock(genpd);
2146	pr_err("%s: unable to remove %s\n", __func__, genpd->name);
2147	return -EBUSY;
2148	}
2149
2150	list_for_each_entry_safe(link, l, &genpd->child_links, child_node) {
2151	list_del(entry: &link->parent_node);
2152	list_del(entry: &link->child_node);
2153	kfree(objp: link);
2154	}
2155
2156	list_del(entry: &genpd->gpd_list_node);
2157	genpd_unlock(genpd);
2158	genpd_debug_remove(genpd);
2159	cancel_work_sync(work: &genpd->power_off_work);
2160	genpd_free_data(genpd);
2161
2162	pr_debug("%s: removed %s\n", __func__, genpd->name);
2163
2164	return 0;
2165	}
2166
2167	/**
2168	* pm_genpd_remove - Remove a generic I/O PM domain
2169	* @genpd: Pointer to PM domain that is to be removed.
2170	*
2171	* To remove the PM domain, this function:
2172	* - Removes the PM domain as a subdomain to any parent domains,
2173	* if it was added.
2174	* - Removes the PM domain from the list of registered PM domains.
2175	*
2176	* The PM domain will only be removed, if the associated provider has
2177	* been removed, it is not a parent to any other PM domain and has no
2178	* devices associated with it.
2179	*/
2180	int pm_genpd_remove(struct generic_pm_domain *genpd)
2181	{
2182	int ret;
2183
2184	mutex_lock(&gpd_list_lock);
2185	ret = genpd_remove(genpd);
2186	mutex_unlock(lock: &gpd_list_lock);
2187
2188	return ret;
2189	}
2190	EXPORT_SYMBOL_GPL(pm_genpd_remove);
2191
2192	#ifdef CONFIG_PM_GENERIC_DOMAINS_OF
2193
2194	/*
2195	* Device Tree based PM domain providers.
2196	*
2197	* The code below implements generic device tree based PM domain providers that
2198	* bind device tree nodes with generic PM domains registered in the system.
2199	*
2200	* Any driver that registers generic PM domains and needs to support binding of
2201	* devices to these domains is supposed to register a PM domain provider, which
2202	* maps a PM domain specifier retrieved from the device tree to a PM domain.
2203	*
2204	* Two simple mapping functions have been provided for convenience:
2205	* - genpd_xlate_simple() for 1:1 device tree node to PM domain mapping.
2206	* - genpd_xlate_onecell() for mapping of multiple PM domains per node by
2207	* index.
2208	*/
2209
2210	/**
2211	* struct of_genpd_provider - PM domain provider registration structure
2212	* @link: Entry in global list of PM domain providers
2213	* @node: Pointer to device tree node of PM domain provider
2214	* @xlate: Provider-specific xlate callback mapping a set of specifier cells
2215	* into a PM domain.
2216	* @data: context pointer to be passed into @xlate callback
2217	*/
2218	struct of_genpd_provider {
2219	struct list_head link;
2220	struct device_node *node;
2221	genpd_xlate_t xlate;
2222	void *data;
2223	};
2224
2225	/* List of registered PM domain providers. */
2226	static LIST_HEAD(of_genpd_providers);
2227	/* Mutex to protect the list above. */
2228	static DEFINE_MUTEX(of_genpd_mutex);
2229
2230	/**
2231	* genpd_xlate_simple() - Xlate function for direct node-domain mapping
2232	* @genpdspec: OF phandle args to map into a PM domain
2233	* @data: xlate function private data - pointer to struct generic_pm_domain
2234	*
2235	* This is a generic xlate function that can be used to model PM domains that
2236	* have their own device tree nodes. The private data of xlate function needs
2237	* to be a valid pointer to struct generic_pm_domain.
2238	*/
2239	static struct generic_pm_domain *genpd_xlate_simple(
2240	struct of_phandle_args *genpdspec,
2241	void *data)
2242	{
2243	return data;
2244	}
2245
2246	/**
2247	* genpd_xlate_onecell() - Xlate function using a single index.
2248	* @genpdspec: OF phandle args to map into a PM domain
2249	* @data: xlate function private data - pointer to struct genpd_onecell_data
2250	*
2251	* This is a generic xlate function that can be used to model simple PM domain
2252	* controllers that have one device tree node and provide multiple PM domains.
2253	* A single cell is used as an index into an array of PM domains specified in
2254	* the genpd_onecell_data struct when registering the provider.
2255	*/
2256	static struct generic_pm_domain *genpd_xlate_onecell(
2257	struct of_phandle_args *genpdspec,
2258	void *data)
2259	{
2260	struct genpd_onecell_data *genpd_data = data;
2261	unsigned int idx = genpdspec->args[0];
2262
2263	if (genpdspec->args_count != 1)
2264	return ERR_PTR(error: -EINVAL);
2265
2266	if (idx >= genpd_data->num_domains) {
2267	pr_err("%s: invalid domain index %u\n", __func__, idx);
2268	return ERR_PTR(error: -EINVAL);
2269	}
2270
2271	if (!genpd_data->domains[idx])
2272	return ERR_PTR(error: -ENOENT);
2273
2274	return genpd_data->domains[idx];
2275	}
2276
2277	/**
2278	* genpd_add_provider() - Register a PM domain provider for a node
2279	* @np: Device node pointer associated with the PM domain provider.
2280	* @xlate: Callback for decoding PM domain from phandle arguments.
2281	* @data: Context pointer for @xlate callback.
2282	*/
2283	static int genpd_add_provider(struct device_node *np, genpd_xlate_t xlate,
2284	void *data)
2285	{
2286	struct of_genpd_provider *cp;
2287
2288	cp = kzalloc(size: sizeof(*cp), GFP_KERNEL);
2289	if (!cp)
2290	return -ENOMEM;
2291
2292	cp->node = of_node_get(node: np);
2293	cp->data = data;
2294	cp->xlate = xlate;
2295	fwnode_dev_initialized(fwnode: &np->fwnode, initialized: true);
2296
2297	mutex_lock(&of_genpd_mutex);
2298	list_add(new: &cp->link, head: &of_genpd_providers);
2299	mutex_unlock(lock: &of_genpd_mutex);
2300	pr_debug("Added domain provider from %pOF\n", np);
2301
2302	return 0;
2303	}
2304
2305	static bool genpd_present(const struct generic_pm_domain *genpd)
2306	{
2307	bool ret = false;
2308	const struct generic_pm_domain *gpd;
2309
2310	mutex_lock(&gpd_list_lock);
2311	list_for_each_entry(gpd, &gpd_list, gpd_list_node) {
2312	if (gpd == genpd) {
2313	ret = true;
2314	break;
2315	}
2316	}
2317	mutex_unlock(lock: &gpd_list_lock);
2318
2319	return ret;
2320	}
2321
2322	/**
2323	* of_genpd_add_provider_simple() - Register a simple PM domain provider
2324	* @np: Device node pointer associated with the PM domain provider.
2325	* @genpd: Pointer to PM domain associated with the PM domain provider.
2326	*/
2327	int of_genpd_add_provider_simple(struct device_node *np,
2328	struct generic_pm_domain *genpd)
2329	{
2330	int ret;
2331
2332	if (!np || !genpd)
2333	return -EINVAL;
2334
2335	if (!genpd_present(genpd))
2336	return -EINVAL;
2337
2338	genpd->dev.of_node = np;
2339
2340	/* Parse genpd OPP table */
2341	if (!genpd_is_opp_table_fw(genpd) && genpd->set_performance_state) {
2342	ret = dev_pm_opp_of_add_table(dev: &genpd->dev);
2343	if (ret)
2344	return dev_err_probe(dev: &genpd->dev, err: ret, fmt: "Failed to add OPP table\n");
2345
2346	/*
2347	* Save table for faster processing while setting performance
2348	* state.
2349	*/
2350	genpd->opp_table = dev_pm_opp_get_opp_table(dev: &genpd->dev);
2351	WARN_ON(IS_ERR(genpd->opp_table));
2352	}
2353
2354	ret = genpd_add_provider(np, xlate: genpd_xlate_simple, data: genpd);
2355	if (ret) {
2356	if (!genpd_is_opp_table_fw(genpd) && genpd->set_performance_state) {
2357	dev_pm_opp_put_opp_table(opp_table: genpd->opp_table);
2358	dev_pm_opp_of_remove_table(dev: &genpd->dev);
2359	}
2360
2361	return ret;
2362	}
2363
2364	genpd->provider = &np->fwnode;
2365	genpd->has_provider = true;
2366
2367	return 0;
2368	}
2369	EXPORT_SYMBOL_GPL(of_genpd_add_provider_simple);
2370
2371	/**
2372	* of_genpd_add_provider_onecell() - Register a onecell PM domain provider
2373	* @np: Device node pointer associated with the PM domain provider.
2374	* @data: Pointer to the data associated with the PM domain provider.
2375	*/
2376	int of_genpd_add_provider_onecell(struct device_node *np,
2377	struct genpd_onecell_data *data)
2378	{
2379	struct generic_pm_domain *genpd;
2380	unsigned int i;
2381	int ret = -EINVAL;
2382
2383	if (!np || !data)
2384	return -EINVAL;
2385
2386	if (!data->xlate)
2387	data->xlate = genpd_xlate_onecell;
2388
2389	for (i = 0; i < data->num_domains; i++) {
2390	genpd = data->domains[i];
2391
2392	if (!genpd)
2393	continue;
2394	if (!genpd_present(genpd))
2395	goto error;
2396
2397	genpd->dev.of_node = np;
2398
2399	/* Parse genpd OPP table */
2400	if (!genpd_is_opp_table_fw(genpd) && genpd->set_performance_state) {
2401	ret = dev_pm_opp_of_add_table_indexed(dev: &genpd->dev, index: i);
2402	if (ret) {
2403	dev_err_probe(dev: &genpd->dev, err: ret,
2404	fmt: "Failed to add OPP table for index %d\n", i);
2405	goto error;
2406	}
2407
2408	/*
2409	* Save table for faster processing while setting
2410	* performance state.
2411	*/
2412	genpd->opp_table = dev_pm_opp_get_opp_table(dev: &genpd->dev);
2413	WARN_ON(IS_ERR(genpd->opp_table));
2414	}
2415
2416	genpd->provider = &np->fwnode;
2417	genpd->has_provider = true;
2418	}
2419
2420	ret = genpd_add_provider(np, xlate: data->xlate, data);
2421	if (ret < 0)
2422	goto error;
2423
2424	return 0;
2425
2426	error:
2427	while (i--) {
2428	genpd = data->domains[i];
2429
2430	if (!genpd)
2431	continue;
2432
2433	genpd->provider = NULL;
2434	genpd->has_provider = false;
2435
2436	if (!genpd_is_opp_table_fw(genpd) && genpd->set_performance_state) {
2437	dev_pm_opp_put_opp_table(opp_table: genpd->opp_table);
2438	dev_pm_opp_of_remove_table(dev: &genpd->dev);
2439	}
2440	}
2441
2442	return ret;
2443	}
2444	EXPORT_SYMBOL_GPL(of_genpd_add_provider_onecell);
2445
2446	/**
2447	* of_genpd_del_provider() - Remove a previously registered PM domain provider
2448	* @np: Device node pointer associated with the PM domain provider
2449	*/
2450	void of_genpd_del_provider(struct device_node *np)
2451	{
2452	struct of_genpd_provider *cp, *tmp;
2453	struct generic_pm_domain *gpd;
2454
2455	mutex_lock(&gpd_list_lock);
2456	mutex_lock(&of_genpd_mutex);
2457	list_for_each_entry_safe(cp, tmp, &of_genpd_providers, link) {
2458	if (cp->node == np) {
2459	/*
2460	* For each PM domain associated with the
2461	* provider, set the 'has_provider' to false
2462	* so that the PM domain can be safely removed.
2463	*/
2464	list_for_each_entry(gpd, &gpd_list, gpd_list_node) {
2465	if (gpd->provider == &np->fwnode) {
2466	gpd->has_provider = false;
2467
2468	if (genpd_is_opp_table_fw(gpd) || !gpd->set_performance_state)
2469	continue;
2470
2471	dev_pm_opp_put_opp_table(opp_table: gpd->opp_table);
2472	dev_pm_opp_of_remove_table(dev: &gpd->dev);
2473	}
2474	}
2475
2476	fwnode_dev_initialized(fwnode: &cp->node->fwnode, initialized: false);
2477	list_del(entry: &cp->link);
2478	of_node_put(node: cp->node);
2479	kfree(objp: cp);
2480	break;
2481	}
2482	}
2483	mutex_unlock(lock: &of_genpd_mutex);
2484	mutex_unlock(lock: &gpd_list_lock);
2485	}
2486	EXPORT_SYMBOL_GPL(of_genpd_del_provider);
2487
2488	/**
2489	* genpd_get_from_provider() - Look-up PM domain
2490	* @genpdspec: OF phandle args to use for look-up
2491	*
2492	* Looks for a PM domain provider under the node specified by @genpdspec and if
2493	* found, uses xlate function of the provider to map phandle args to a PM
2494	* domain.
2495	*
2496	* Returns a valid pointer to struct generic_pm_domain on success or ERR_PTR()
2497	* on failure.
2498	*/
2499	static struct generic_pm_domain *genpd_get_from_provider(
2500	struct of_phandle_args *genpdspec)
2501	{
2502	struct generic_pm_domain *genpd = ERR_PTR(error: -ENOENT);
2503	struct of_genpd_provider *provider;
2504
2505	if (!genpdspec)
2506	return ERR_PTR(error: -EINVAL);
2507
2508	mutex_lock(&of_genpd_mutex);
2509
2510	/* Check if we have such a provider in our array */
2511	list_for_each_entry(provider, &of_genpd_providers, link) {
2512	if (provider->node == genpdspec->np)
2513	genpd = provider->xlate(genpdspec, provider->data);
2514	if (!IS_ERR(ptr: genpd))
2515	break;
2516	}
2517
2518	mutex_unlock(lock: &of_genpd_mutex);
2519
2520	return genpd;
2521	}
2522
2523	/**
2524	* of_genpd_add_device() - Add a device to an I/O PM domain
2525	* @genpdspec: OF phandle args to use for look-up PM domain
2526	* @dev: Device to be added.
2527	*
2528	* Looks-up an I/O PM domain based upon phandle args provided and adds
2529	* the device to the PM domain. Returns a negative error code on failure.
2530	*/
2531	int of_genpd_add_device(struct of_phandle_args *genpdspec, struct device *dev)
2532	{
2533	struct generic_pm_domain *genpd;
2534	int ret;
2535
2536	if (!dev)
2537	return -EINVAL;
2538
2539	mutex_lock(&gpd_list_lock);
2540
2541	genpd = genpd_get_from_provider(genpdspec);
2542	if (IS_ERR(ptr: genpd)) {
2543	ret = PTR_ERR(ptr: genpd);
2544	goto out;
2545	}
2546
2547	ret = genpd_add_device(genpd, dev, base_dev: dev);
2548
2549	out:
2550	mutex_unlock(lock: &gpd_list_lock);
2551
2552	return ret;
2553	}
2554	EXPORT_SYMBOL_GPL(of_genpd_add_device);
2555
2556	/**
2557	* of_genpd_add_subdomain - Add a subdomain to an I/O PM domain.
2558	* @parent_spec: OF phandle args to use for parent PM domain look-up
2559	* @subdomain_spec: OF phandle args to use for subdomain look-up
2560	*
2561	* Looks-up a parent PM domain and subdomain based upon phandle args
2562	* provided and adds the subdomain to the parent PM domain. Returns a
2563	* negative error code on failure.
2564	*/
2565	int of_genpd_add_subdomain(struct of_phandle_args *parent_spec,
2566	struct of_phandle_args *subdomain_spec)
2567	{
2568	struct generic_pm_domain *parent, *subdomain;
2569	int ret;
2570
2571	mutex_lock(&gpd_list_lock);
2572
2573	parent = genpd_get_from_provider(genpdspec: parent_spec);
2574	if (IS_ERR(ptr: parent)) {
2575	ret = PTR_ERR(ptr: parent);
2576	goto out;
2577	}
2578
2579	subdomain = genpd_get_from_provider(genpdspec: subdomain_spec);
2580	if (IS_ERR(ptr: subdomain)) {
2581	ret = PTR_ERR(ptr: subdomain);
2582	goto out;
2583	}
2584
2585	ret = genpd_add_subdomain(genpd: parent, subdomain);
2586
2587	out:
2588	mutex_unlock(lock: &gpd_list_lock);
2589
2590	return ret == -ENOENT ? -EPROBE_DEFER : ret;
2591	}
2592	EXPORT_SYMBOL_GPL(of_genpd_add_subdomain);
2593
2594	/**
2595	* of_genpd_remove_subdomain - Remove a subdomain from an I/O PM domain.
2596	* @parent_spec: OF phandle args to use for parent PM domain look-up
2597	* @subdomain_spec: OF phandle args to use for subdomain look-up
2598	*
2599	* Looks-up a parent PM domain and subdomain based upon phandle args
2600	* provided and removes the subdomain from the parent PM domain. Returns a
2601	* negative error code on failure.
2602	*/
2603	int of_genpd_remove_subdomain(struct of_phandle_args *parent_spec,
2604	struct of_phandle_args *subdomain_spec)
2605	{
2606	struct generic_pm_domain *parent, *subdomain;
2607	int ret;
2608
2609	mutex_lock(&gpd_list_lock);
2610
2611	parent = genpd_get_from_provider(genpdspec: parent_spec);
2612	if (IS_ERR(ptr: parent)) {
2613	ret = PTR_ERR(ptr: parent);
2614	goto out;
2615	}
2616
2617	subdomain = genpd_get_from_provider(genpdspec: subdomain_spec);
2618	if (IS_ERR(ptr: subdomain)) {
2619	ret = PTR_ERR(ptr: subdomain);
2620	goto out;
2621	}
2622
2623	ret = pm_genpd_remove_subdomain(parent, subdomain);
2624
2625	out:
2626	mutex_unlock(lock: &gpd_list_lock);
2627
2628	return ret;
2629	}
2630	EXPORT_SYMBOL_GPL(of_genpd_remove_subdomain);
2631
2632	/**
2633	* of_genpd_remove_last - Remove the last PM domain registered for a provider
2634	* @np: Pointer to device node associated with provider
2635	*
2636	* Find the last PM domain that was added by a particular provider and
2637	* remove this PM domain from the list of PM domains. The provider is
2638	* identified by the 'provider' device structure that is passed. The PM
2639	* domain will only be removed, if the provider associated with domain
2640	* has been removed.
2641	*
2642	* Returns a valid pointer to struct generic_pm_domain on success or
2643	* ERR_PTR() on failure.
2644	*/
2645	struct generic_pm_domain *of_genpd_remove_last(struct device_node *np)
2646	{
2647	struct generic_pm_domain *gpd, *tmp, *genpd = ERR_PTR(error: -ENOENT);
2648	int ret;
2649
2650	if (IS_ERR_OR_NULL(ptr: np))
2651	return ERR_PTR(error: -EINVAL);
2652
2653	mutex_lock(&gpd_list_lock);
2654	list_for_each_entry_safe(gpd, tmp, &gpd_list, gpd_list_node) {
2655	if (gpd->provider == &np->fwnode) {
2656	ret = genpd_remove(genpd: gpd);
2657	genpd = ret ? ERR_PTR(error: ret) : gpd;
2658	break;
2659	}
2660	}
2661	mutex_unlock(lock: &gpd_list_lock);
2662
2663	return genpd;
2664	}
2665	EXPORT_SYMBOL_GPL(of_genpd_remove_last);
2666
2667	static void genpd_release_dev(struct device *dev)
2668	{
2669	of_node_put(node: dev->of_node);
2670	kfree(objp: dev);
2671	}
2672
2673	static struct bus_type genpd_bus_type = {
2674	.name = "genpd",
2675	};
2676
2677	/**
2678	* genpd_dev_pm_detach - Detach a device from its PM domain.
2679	* @dev: Device to detach.
2680	* @power_off: Currently not used
2681	*
2682	* Try to locate a corresponding generic PM domain, which the device was
2683	* attached to previously. If such is found, the device is detached from it.
2684	*/
2685	static void genpd_dev_pm_detach(struct device *dev, bool power_off)
2686	{
2687	struct generic_pm_domain *pd;
2688	unsigned int i;
2689	int ret = 0;
2690
2691	pd = dev_to_genpd(dev);
2692	if (IS_ERR(ptr: pd))
2693	return;
2694
2695	dev_dbg(dev, "removing from PM domain %s\n", pd->name);
2696
2697	/* Drop the default performance state */
2698	if (dev_gpd_data(dev)->default_pstate) {
2699	dev_pm_genpd_set_performance_state(dev, 0);
2700	dev_gpd_data(dev)->default_pstate = 0;
2701	}
2702
2703	for (i = 1; i < GENPD_RETRY_MAX_MS; i <<= 1) {
2704	ret = genpd_remove_device(genpd: pd, dev);
2705	if (ret != -EAGAIN)
2706	break;
2707
2708	mdelay(i);
2709	cond_resched();
2710	}
2711
2712	if (ret < 0) {
2713	dev_err(dev, "failed to remove from PM domain %s: %d",
2714	pd->name, ret);
2715	return;
2716	}
2717
2718	/* Check if PM domain can be powered off after removing this device. */
2719	genpd_queue_power_off_work(genpd: pd);
2720
2721	/* Unregister the device if it was created by genpd. */
2722	if (dev->bus == &genpd_bus_type)
2723	device_unregister(dev);
2724	}
2725
2726	static void genpd_dev_pm_sync(struct device *dev)
2727	{
2728	struct generic_pm_domain *pd;
2729
2730	pd = dev_to_genpd(dev);
2731	if (IS_ERR(ptr: pd))
2732	return;
2733
2734	genpd_queue_power_off_work(genpd: pd);
2735	}
2736
2737	static int __genpd_dev_pm_attach(struct device *dev, struct device *base_dev,
2738	unsigned int index, bool power_on)
2739	{
2740	struct of_phandle_args pd_args;
2741	struct generic_pm_domain *pd;
2742	int pstate;
2743	int ret;
2744
2745	ret = of_parse_phandle_with_args(np: dev->of_node, list_name: "power-domains",
2746	cells_name: "#power-domain-cells", index, out_args: &pd_args);
2747	if (ret < 0)
2748	return ret;
2749
2750	mutex_lock(&gpd_list_lock);
2751	pd = genpd_get_from_provider(genpdspec: &pd_args);
2752	of_node_put(node: pd_args.np);
2753	if (IS_ERR(ptr: pd)) {
2754	mutex_unlock(lock: &gpd_list_lock);
2755	dev_dbg(dev, "%s() failed to find PM domain: %ld\n",
2756	__func__, PTR_ERR(pd));
2757	return driver_deferred_probe_check_state(dev: base_dev);
2758	}
2759
2760	dev_dbg(dev, "adding to PM domain %s\n", pd->name);
2761
2762	ret = genpd_add_device(genpd: pd, dev, base_dev);
2763	mutex_unlock(lock: &gpd_list_lock);
2764
2765	if (ret < 0)
2766	return dev_err_probe(dev, err: ret, fmt: "failed to add to PM domain %s\n", pd->name);
2767
2768	dev->pm_domain->detach = genpd_dev_pm_detach;
2769	dev->pm_domain->sync = genpd_dev_pm_sync;
2770
2771	/* Set the default performance state */
2772	pstate = of_get_required_opp_performance_state(np: dev->of_node, index);
2773	if (pstate < 0 && pstate != -ENODEV && pstate != -EOPNOTSUPP) {
2774	ret = pstate;
2775	goto err;
2776	} else if (pstate > 0) {
2777	ret = dev_pm_genpd_set_performance_state(dev, pstate);
2778	if (ret)
2779	goto err;
2780	dev_gpd_data(dev)->default_pstate = pstate;
2781	}
2782
2783	if (power_on) {
2784	genpd_lock(pd);
2785	ret = genpd_power_on(genpd: pd, depth: 0);
2786	genpd_unlock(pd);
2787	}
2788
2789	if (ret) {
2790	/* Drop the default performance state */
2791	if (dev_gpd_data(dev)->default_pstate) {
2792	dev_pm_genpd_set_performance_state(dev, 0);
2793	dev_gpd_data(dev)->default_pstate = 0;
2794	}
2795
2796	genpd_remove_device(genpd: pd, dev);
2797	return -EPROBE_DEFER;
2798	}
2799
2800	return 1;
2801
2802	err:
2803	dev_err(dev, "failed to set required performance state for power-domain %s: %d\n",
2804	pd->name, ret);
2805	genpd_remove_device(genpd: pd, dev);
2806	return ret;
2807	}
2808
2809	/**
2810	* genpd_dev_pm_attach - Attach a device to its PM domain using DT.
2811	* @dev: Device to attach.
2812	*
2813	* Parse device's OF node to find a PM domain specifier. If such is found,
2814	* attaches the device to retrieved pm_domain ops.
2815	*
2816	* Returns 1 on successfully attached PM domain, 0 when the device don't need a
2817	* PM domain or when multiple power-domains exists for it, else a negative error
2818	* code. Note that if a power-domain exists for the device, but it cannot be
2819	* found or turned on, then return -EPROBE_DEFER to ensure that the device is
2820	* not probed and to re-try again later.
2821	*/
2822	int genpd_dev_pm_attach(struct device *dev)
2823	{
2824	if (!dev->of_node)
2825	return 0;
2826
2827	/*
2828	* Devices with multiple PM domains must be attached separately, as we
2829	* can only attach one PM domain per device.
2830	*/
2831	if (of_count_phandle_with_args(np: dev->of_node, list_name: "power-domains",
2832	cells_name: "#power-domain-cells") != 1)
2833	return 0;
2834
2835	return __genpd_dev_pm_attach(dev, base_dev: dev, index: 0, power_on: true);
2836	}
2837	EXPORT_SYMBOL_GPL(genpd_dev_pm_attach);
2838
2839	/**
2840	* genpd_dev_pm_attach_by_id - Associate a device with one of its PM domains.
2841	* @dev: The device used to lookup the PM domain.
2842	* @index: The index of the PM domain.
2843	*
2844	* Parse device's OF node to find a PM domain specifier at the provided @index.
2845	* If such is found, creates a virtual device and attaches it to the retrieved
2846	* pm_domain ops. To deal with detaching of the virtual device, the ->detach()
2847	* callback in the struct dev_pm_domain are assigned to genpd_dev_pm_detach().
2848	*
2849	* Returns the created virtual device if successfully attached PM domain, NULL
2850	* when the device don't need a PM domain, else an ERR_PTR() in case of
2851	* failures. If a power-domain exists for the device, but cannot be found or
2852	* turned on, then ERR_PTR(-EPROBE_DEFER) is returned to ensure that the device
2853	* is not probed and to re-try again later.
2854	*/
2855	struct device *genpd_dev_pm_attach_by_id(struct device *dev,
2856	unsigned int index)
2857	{
2858	struct device *virt_dev;
2859	int num_domains;
2860	int ret;
2861
2862	if (!dev->of_node)
2863	return NULL;
2864
2865	/* Verify that the index is within a valid range. */
2866	num_domains = of_count_phandle_with_args(np: dev->of_node, list_name: "power-domains",
2867	cells_name: "#power-domain-cells");
2868	if (index >= num_domains)
2869	return NULL;
2870
2871	/* Allocate and register device on the genpd bus. */
2872	virt_dev = kzalloc(size: sizeof(*virt_dev), GFP_KERNEL);
2873	if (!virt_dev)
2874	return ERR_PTR(error: -ENOMEM);
2875
2876	dev_set_name(dev: virt_dev, name: "genpd:%u:%s", index, dev_name(dev));
2877	virt_dev->bus = &genpd_bus_type;
2878	virt_dev->release = genpd_release_dev;
2879	virt_dev->of_node = of_node_get(node: dev->of_node);
2880
2881	ret = device_register(dev: virt_dev);
2882	if (ret) {
2883	put_device(dev: virt_dev);
2884	return ERR_PTR(error: ret);
2885	}
2886
2887	/* Try to attach the device to the PM domain at the specified index. */
2888	ret = __genpd_dev_pm_attach(dev: virt_dev, base_dev: dev, index, power_on: false);
2889	if (ret < 1) {
2890	device_unregister(dev: virt_dev);
2891	return ret ? ERR_PTR(error: ret) : NULL;
2892	}
2893
2894	pm_runtime_enable(dev: virt_dev);
2895	genpd_queue_power_off_work(genpd: dev_to_genpd(dev: virt_dev));
2896
2897	return virt_dev;
2898	}
2899	EXPORT_SYMBOL_GPL(genpd_dev_pm_attach_by_id);
2900
2901	/**
2902	* genpd_dev_pm_attach_by_name - Associate a device with one of its PM domains.
2903	* @dev: The device used to lookup the PM domain.
2904	* @name: The name of the PM domain.
2905	*
2906	* Parse device's OF node to find a PM domain specifier using the
2907	* power-domain-names DT property. For further description see
2908	* genpd_dev_pm_attach_by_id().
2909	*/
2910	struct device *genpd_dev_pm_attach_by_name(struct device *dev, const char *name)
2911	{
2912	int index;
2913
2914	if (!dev->of_node)
2915	return NULL;
2916
2917	index = of_property_match_string(np: dev->of_node, propname: "power-domain-names",
2918	string: name);
2919	if (index < 0)
2920	return NULL;
2921
2922	return genpd_dev_pm_attach_by_id(dev, index);
2923	}
2924
2925	static const struct of_device_id idle_state_match[] = {
2926	{ .compatible = "domain-idle-state", },
2927	{ }
2928	};
2929
2930	static int genpd_parse_state(struct genpd_power_state *genpd_state,
2931	struct device_node *state_node)
2932	{
2933	int err;
2934	u32 residency;
2935	u32 entry_latency, exit_latency;
2936
2937	err = of_property_read_u32(np: state_node, propname: "entry-latency-us",
2938	out_value: &entry_latency);
2939	if (err) {
2940	pr_debug(" * %pOF missing entry-latency-us property\n",
2941	state_node);
2942	return -EINVAL;
2943	}
2944
2945	err = of_property_read_u32(np: state_node, propname: "exit-latency-us",
2946	out_value: &exit_latency);
2947	if (err) {
2948	pr_debug(" * %pOF missing exit-latency-us property\n",
2949	state_node);
2950	return -EINVAL;
2951	}
2952
2953	err = of_property_read_u32(np: state_node, propname: "min-residency-us", out_value: &residency);
2954	if (!err)
2955	genpd_state->residency_ns = 1000LL * residency;
2956
2957	genpd_state->power_on_latency_ns = 1000LL * exit_latency;
2958	genpd_state->power_off_latency_ns = 1000LL * entry_latency;
2959	genpd_state->fwnode = &state_node->fwnode;
2960
2961	return 0;
2962	}
2963
2964	static int genpd_iterate_idle_states(struct device_node *dn,
2965	struct genpd_power_state *states)
2966	{
2967	int ret;
2968	struct of_phandle_iterator it;
2969	struct device_node *np;
2970	int i = 0;
2971
2972	ret = of_count_phandle_with_args(np: dn, list_name: "domain-idle-states", NULL);
2973	if (ret <= 0)
2974	return ret == -ENOENT ? 0 : ret;
2975
2976	/* Loop over the phandles until all the requested entry is found */
2977	of_for_each_phandle(&it, ret, dn, "domain-idle-states", NULL, 0) {
2978	np = it.node;
2979	if (!of_match_node(matches: idle_state_match, node: np))
2980	continue;
2981
2982	if (!of_device_is_available(device: np))
2983	continue;
2984
2985	if (states) {
2986	ret = genpd_parse_state(genpd_state: &states[i], state_node: np);
2987	if (ret) {
2988	pr_err("Parsing idle state node %pOF failed with err %d\n",
2989	np, ret);
2990	of_node_put(node: np);
2991	return ret;
2992	}
2993	}
2994	i++;
2995	}
2996
2997	return i;
2998	}
2999
3000	/**
3001	* of_genpd_parse_idle_states: Return array of idle states for the genpd.
3002	*
3003	* @dn: The genpd device node
3004	* @states: The pointer to which the state array will be saved.
3005	* @n: The count of elements in the array returned from this function.
3006	*
3007	* Returns the device states parsed from the OF node. The memory for the states
3008	* is allocated by this function and is the responsibility of the caller to
3009	* free the memory after use. If any or zero compatible domain idle states is
3010	* found it returns 0 and in case of errors, a negative error code is returned.
3011	*/
3012	int of_genpd_parse_idle_states(struct device_node *dn,
3013	struct genpd_power_state **states, int *n)
3014	{
3015	struct genpd_power_state *st;
3016	int ret;
3017
3018	ret = genpd_iterate_idle_states(dn, NULL);
3019	if (ret < 0)
3020	return ret;
3021
3022	if (!ret) {
3023	*states = NULL;
3024	*n = 0;
3025	return 0;
3026	}
3027
3028	st = kcalloc(n: ret, size: sizeof(*st), GFP_KERNEL);
3029	if (!st)
3030	return -ENOMEM;
3031
3032	ret = genpd_iterate_idle_states(dn, states: st);
3033	if (ret <= 0) {
3034	kfree(objp: st);
3035	return ret < 0 ? ret : -EINVAL;
3036	}
3037
3038	*states = st;
3039	*n = ret;
3040
3041	return 0;
3042	}
3043	EXPORT_SYMBOL_GPL(of_genpd_parse_idle_states);
3044
3045	/**
3046	* pm_genpd_opp_to_performance_state - Gets performance state of the genpd from its OPP node.
3047	*
3048	* @genpd_dev: Genpd's device for which the performance-state needs to be found.
3049	* @opp: struct dev_pm_opp of the OPP for which we need to find performance
3050	* state.
3051	*
3052	* Returns performance state encoded in the OPP of the genpd. This calls
3053	* platform specific genpd->opp_to_performance_state() callback to translate
3054	* power domain OPP to performance state.
3055	*
3056	* Returns performance state on success and 0 on failure.
3057	*/
3058	unsigned int pm_genpd_opp_to_performance_state(struct device *genpd_dev,
3059	struct dev_pm_opp *opp)
3060	{
3061	struct generic_pm_domain *genpd = NULL;
3062	int state;
3063
3064	genpd = container_of(genpd_dev, struct generic_pm_domain, dev);
3065
3066	if (unlikely(!genpd->opp_to_performance_state))
3067	return 0;
3068
3069	genpd_lock(genpd);
3070	state = genpd->opp_to_performance_state(genpd, opp);
3071	genpd_unlock(genpd);
3072
3073	return state;
3074	}
3075	EXPORT_SYMBOL_GPL(pm_genpd_opp_to_performance_state);
3076
3077	static int __init genpd_bus_init(void)
3078	{
3079	return bus_register(bus: &genpd_bus_type);
3080	}
3081	core_initcall(genpd_bus_init);
3082
3083	#endif /* CONFIG_PM_GENERIC_DOMAINS_OF */
3084
3085
3086	/*** debugfs support ***/
3087
3088	#ifdef CONFIG_DEBUG_FS
3089	/*
3090	* TODO: This function is a slightly modified version of rtpm_status_show
3091	* from sysfs.c, so generalize it.
3092	*/
3093	static void rtpm_status_str(struct seq_file *s, struct device *dev)
3094	{
3095	static const char * const status_lookup[] = {
3096	[RPM_ACTIVE] = "active",
3097	[RPM_RESUMING] = "resuming",
3098	[RPM_SUSPENDED] = "suspended",
3099	[RPM_SUSPENDING] = "suspending"
3100	};
3101	const char *p = "";
3102
3103	if (dev->power.runtime_error)
3104	p = "error";
3105	else if (dev->power.disable_depth)
3106	p = "unsupported";
3107	else if (dev->power.runtime_status < ARRAY_SIZE(status_lookup))
3108	p = status_lookup[dev->power.runtime_status];
3109	else
3110	WARN_ON(1);
3111
3112	seq_printf(m: s, fmt: "%-25s ", p);
3113	}
3114
3115	static void perf_status_str(struct seq_file *s, struct device *dev)
3116	{
3117	struct generic_pm_domain_data *gpd_data;
3118
3119	gpd_data = to_gpd_data(pdd: dev->power.subsys_data->domain_data);
3120	seq_put_decimal_ull(m: s, delimiter: "", num: gpd_data->performance_state);
3121	}
3122
3123	static int genpd_summary_one(struct seq_file *s,
3124	struct generic_pm_domain *genpd)
3125	{
3126	static const char * const status_lookup[] = {
3127	[GENPD_STATE_ON] = "on",
3128	[GENPD_STATE_OFF] = "off"
3129	};
3130	struct pm_domain_data *pm_data;
3131	const char *kobj_path;
3132	struct gpd_link *link;
3133	char state[16];
3134	int ret;
3135
3136	ret = genpd_lock_interruptible(genpd);
3137	if (ret)
3138	return -ERESTARTSYS;
3139
3140	if (WARN_ON(genpd->status >= ARRAY_SIZE(status_lookup)))
3141	goto exit;
3142	if (!genpd_status_on(genpd))
3143	snprintf(buf: state, size: sizeof(state), fmt: "%s-%u",
3144	status_lookup[genpd->status], genpd->state_idx);
3145	else
3146	snprintf(buf: state, size: sizeof(state), fmt: "%s",
3147	status_lookup[genpd->status]);
3148	seq_printf(m: s, fmt: "%-30s %-50s %u", genpd->name, state, genpd->performance_state);
3149
3150	/*
3151	* Modifications on the list require holding locks on both
3152	* parent and child, so we are safe.
3153	* Also genpd->name is immutable.
3154	*/
3155	list_for_each_entry(link, &genpd->parent_links, parent_node) {
3156	if (list_is_first(list: &link->parent_node, head: &genpd->parent_links))
3157	seq_printf(m: s, fmt: "\n%48s", " ");
3158	seq_printf(m: s, fmt: "%s", link->child->name);
3159	if (!list_is_last(list: &link->parent_node, head: &genpd->parent_links))
3160	seq_puts(m: s, s: ", ");
3161	}
3162
3163	list_for_each_entry(pm_data, &genpd->dev_list, list_node) {
3164	kobj_path = kobject_get_path(kobj: &pm_data->dev->kobj,
3165	genpd_is_irq_safe(genpd) ?
3166	GFP_ATOMIC : GFP_KERNEL);
3167	if (kobj_path == NULL)
3168	continue;
3169
3170	seq_printf(m: s, fmt: "\n %-50s ", kobj_path);
3171	rtpm_status_str(s, dev: pm_data->dev);
3172	perf_status_str(s, dev: pm_data->dev);
3173	kfree(objp: kobj_path);
3174	}
3175
3176	seq_puts(m: s, s: "\n");
3177	exit:
3178	genpd_unlock(genpd);
3179
3180	return 0;
3181	}
3182
3183	static int summary_show(struct seq_file *s, void *data)
3184	{
3185	struct generic_pm_domain *genpd;
3186	int ret = 0;
3187
3188	seq_puts(m: s, s: "domain status children performance\n");
3189	seq_puts(m: s, s: " /device runtime status\n");
3190	seq_puts(m: s, s: "----------------------------------------------------------------------------------------------\n");
3191
3192	ret = mutex_lock_interruptible(&gpd_list_lock);
3193	if (ret)
3194	return -ERESTARTSYS;
3195
3196	list_for_each_entry(genpd, &gpd_list, gpd_list_node) {
3197	ret = genpd_summary_one(s, genpd);
3198	if (ret)
3199	break;
3200	}
3201	mutex_unlock(lock: &gpd_list_lock);
3202
3203	return ret;
3204	}
3205
3206	static int status_show(struct seq_file *s, void *data)
3207	{
3208	static const char * const status_lookup[] = {
3209	[GENPD_STATE_ON] = "on",
3210	[GENPD_STATE_OFF] = "off"
3211	};
3212
3213	struct generic_pm_domain *genpd = s->private;
3214	int ret = 0;
3215
3216	ret = genpd_lock_interruptible(genpd);
3217	if (ret)
3218	return -ERESTARTSYS;
3219
3220	if (WARN_ON_ONCE(genpd->status >= ARRAY_SIZE(status_lookup)))
3221	goto exit;
3222
3223	if (genpd->status == GENPD_STATE_OFF)
3224	seq_printf(m: s, fmt: "%s-%u\n", status_lookup[genpd->status],
3225	genpd->state_idx);
3226	else
3227	seq_printf(m: s, fmt: "%s\n", status_lookup[genpd->status]);
3228	exit:
3229	genpd_unlock(genpd);
3230	return ret;
3231	}
3232
3233	static int sub_domains_show(struct seq_file *s, void *data)
3234	{
3235	struct generic_pm_domain *genpd = s->private;
3236	struct gpd_link *link;
3237	int ret = 0;
3238
3239	ret = genpd_lock_interruptible(genpd);
3240	if (ret)
3241	return -ERESTARTSYS;
3242
3243	list_for_each_entry(link, &genpd->parent_links, parent_node)
3244	seq_printf(m: s, fmt: "%s\n", link->child->name);
3245
3246	genpd_unlock(genpd);
3247	return ret;
3248	}
3249
3250	static int idle_states_show(struct seq_file *s, void *data)
3251	{
3252	struct generic_pm_domain *genpd = s->private;
3253	u64 now, delta, idle_time = 0;
3254	unsigned int i;
3255	int ret = 0;
3256
3257	ret = genpd_lock_interruptible(genpd);
3258	if (ret)
3259	return -ERESTARTSYS;
3260
3261	seq_puts(m: s, s: "State Time Spent(ms) Usage Rejected\n");
3262
3263	for (i = 0; i < genpd->state_count; i++) {
3264	idle_time += genpd->states[i].idle_time;
3265
3266	if (genpd->status == GENPD_STATE_OFF && genpd->state_idx == i) {
3267	now = ktime_get_mono_fast_ns();
3268	if (now > genpd->accounting_time) {
3269	delta = now - genpd->accounting_time;
3270	idle_time += delta;
3271	}
3272	}
3273
3274	do_div(idle_time, NSEC_PER_MSEC);
3275	seq_printf(m: s, fmt: "S%-13i %-14llu %-14llu %llu\n", i, idle_time,
3276	genpd->states[i].usage, genpd->states[i].rejected);
3277	}
3278
3279	genpd_unlock(genpd);
3280	return ret;
3281	}
3282
3283	static int active_time_show(struct seq_file *s, void *data)
3284	{
3285	struct generic_pm_domain *genpd = s->private;
3286	u64 now, on_time, delta = 0;
3287	int ret = 0;
3288
3289	ret = genpd_lock_interruptible(genpd);
3290	if (ret)
3291	return -ERESTARTSYS;
3292
3293	if (genpd->status == GENPD_STATE_ON) {
3294	now = ktime_get_mono_fast_ns();
3295	if (now > genpd->accounting_time)
3296	delta = now - genpd->accounting_time;
3297	}
3298
3299	on_time = genpd->on_time + delta;
3300	do_div(on_time, NSEC_PER_MSEC);
3301	seq_printf(m: s, fmt: "%llu ms\n", on_time);
3302
3303	genpd_unlock(genpd);
3304	return ret;
3305	}
3306
3307	static int total_idle_time_show(struct seq_file *s, void *data)
3308	{
3309	struct generic_pm_domain *genpd = s->private;
3310	u64 now, delta, total = 0;
3311	unsigned int i;
3312	int ret = 0;
3313
3314	ret = genpd_lock_interruptible(genpd);
3315	if (ret)
3316	return -ERESTARTSYS;
3317
3318	for (i = 0; i < genpd->state_count; i++) {
3319	total += genpd->states[i].idle_time;
3320
3321	if (genpd->status == GENPD_STATE_OFF && genpd->state_idx == i) {
3322	now = ktime_get_mono_fast_ns();
3323	if (now > genpd->accounting_time) {
3324	delta = now - genpd->accounting_time;
3325	total += delta;
3326	}
3327	}
3328	}
3329
3330	do_div(total, NSEC_PER_MSEC);
3331	seq_printf(m: s, fmt: "%llu ms\n", total);
3332
3333	genpd_unlock(genpd);
3334	return ret;
3335	}
3336
3337
3338	static int devices_show(struct seq_file *s, void *data)
3339	{
3340	struct generic_pm_domain *genpd = s->private;
3341	struct pm_domain_data *pm_data;
3342	const char *kobj_path;
3343	int ret = 0;
3344
3345	ret = genpd_lock_interruptible(genpd);
3346	if (ret)
3347	return -ERESTARTSYS;
3348
3349	list_for_each_entry(pm_data, &genpd->dev_list, list_node) {
3350	kobj_path = kobject_get_path(kobj: &pm_data->dev->kobj,
3351	genpd_is_irq_safe(genpd) ?
3352	GFP_ATOMIC : GFP_KERNEL);
3353	if (kobj_path == NULL)
3354	continue;
3355
3356	seq_printf(m: s, fmt: "%s\n", kobj_path);
3357	kfree(objp: kobj_path);
3358	}
3359
3360	genpd_unlock(genpd);
3361	return ret;
3362	}
3363
3364	static int perf_state_show(struct seq_file *s, void *data)
3365	{
3366	struct generic_pm_domain *genpd = s->private;
3367
3368	if (genpd_lock_interruptible(genpd))
3369	return -ERESTARTSYS;
3370
3371	seq_printf(m: s, fmt: "%u\n", genpd->performance_state);
3372
3373	genpd_unlock(genpd);
3374	return 0;
3375	}
3376
3377	DEFINE_SHOW_ATTRIBUTE(summary);
3378	DEFINE_SHOW_ATTRIBUTE(status);
3379	DEFINE_SHOW_ATTRIBUTE(sub_domains);
3380	DEFINE_SHOW_ATTRIBUTE(idle_states);
3381	DEFINE_SHOW_ATTRIBUTE(active_time);
3382	DEFINE_SHOW_ATTRIBUTE(total_idle_time);
3383	DEFINE_SHOW_ATTRIBUTE(devices);
3384	DEFINE_SHOW_ATTRIBUTE(perf_state);
3385
3386	static void genpd_debug_add(struct generic_pm_domain *genpd)
3387	{
3388	struct dentry *d;
3389
3390	if (!genpd_debugfs_dir)
3391	return;
3392
3393	d = debugfs_create_dir(name: genpd->name, parent: genpd_debugfs_dir);
3394
3395	debugfs_create_file(name: "current_state", mode: 0444,
3396	parent: d, data: genpd, fops: &status_fops);
3397	debugfs_create_file(name: "sub_domains", mode: 0444,
3398	parent: d, data: genpd, fops: &sub_domains_fops);
3399	debugfs_create_file(name: "idle_states", mode: 0444,
3400	parent: d, data: genpd, fops: &idle_states_fops);
3401	debugfs_create_file(name: "active_time", mode: 0444,
3402	parent: d, data: genpd, fops: &active_time_fops);
3403	debugfs_create_file(name: "total_idle_time", mode: 0444,
3404	parent: d, data: genpd, fops: &total_idle_time_fops);
3405	debugfs_create_file(name: "devices", mode: 0444,
3406	parent: d, data: genpd, fops: &devices_fops);
3407	if (genpd->set_performance_state)
3408	debugfs_create_file(name: "perf_state", mode: 0444,
3409	parent: d, data: genpd, fops: &perf_state_fops);
3410	}
3411
3412	static int __init genpd_debug_init(void)
3413	{
3414	struct generic_pm_domain *genpd;
3415
3416	genpd_debugfs_dir = debugfs_create_dir(name: "pm_genpd", NULL);
3417
3418	debugfs_create_file(name: "pm_genpd_summary", S_IRUGO, parent: genpd_debugfs_dir,
3419	NULL, fops: &summary_fops);
3420
3421	list_for_each_entry(genpd, &gpd_list, gpd_list_node)
3422	genpd_debug_add(genpd);
3423
3424	return 0;
3425	}
3426	late_initcall(genpd_debug_init);
3427
3428	static void __exit genpd_debug_exit(void)
3429	{
3430	debugfs_remove_recursive(dentry: genpd_debugfs_dir);
3431	}
3432	__exitcall(genpd_debug_exit);
3433	#endif /* CONFIG_DEBUG_FS */
3434