1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* linux/drivers/cpufreq/cpufreq.c
4	*
5	* Copyright (C) 2001 Russell King
6	* (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
7	* (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
8	*
9	* Oct 2005 - Ashok Raj <ashok.raj@intel.com>
10	* Added handling for CPU hotplug
11	* Feb 2006 - Jacob Shin <jacob.shin@amd.com>
12	* Fix handling for CPU hotplug -- affected CPUs
13	*/
14
15	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
16
17	#include <linux/cpu.h>
18	#include <linux/cpufreq.h>
19	#include <linux/cpu_cooling.h>
20	#include <linux/delay.h>
21	#include <linux/device.h>
22	#include <linux/init.h>
23	#include <linux/kernel_stat.h>
24	#include <linux/module.h>
25	#include <linux/mutex.h>
26	#include <linux/pm_qos.h>
27	#include <linux/slab.h>
28	#include <linux/suspend.h>
29	#include <linux/syscore_ops.h>
30	#include <linux/tick.h>
31	#include <linux/units.h>
32	#include <trace/events/power.h>
33
34	static LIST_HEAD(cpufreq_policy_list);
35
36	/* Macros to iterate over CPU policies */
37	#define for_each_suitable_policy(__policy, __active) \
38	list_for_each_entry(__policy, &cpufreq_policy_list, policy_list) \
39	if ((__active) == !policy_is_inactive(__policy))
40
41	#define for_each_active_policy(__policy) \
42	for_each_suitable_policy(__policy, true)
43	#define for_each_inactive_policy(__policy) \
44	for_each_suitable_policy(__policy, false)
45
46	/* Iterate over governors */
47	static LIST_HEAD(cpufreq_governor_list);
48	#define for_each_governor(__governor) \
49	list_for_each_entry(__governor, &cpufreq_governor_list, governor_list)
50
51	static char default_governor[CPUFREQ_NAME_LEN];
52
53	/*
54	* The "cpufreq driver" - the arch- or hardware-dependent low
55	* level driver of CPUFreq support, and its spinlock. This lock
56	* also protects the cpufreq_cpu_data array.
57	*/
58	static struct cpufreq_driver *cpufreq_driver;
59	static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
60	static DEFINE_RWLOCK(cpufreq_driver_lock);
61
62	static DEFINE_STATIC_KEY_FALSE(cpufreq_freq_invariance);
63	bool cpufreq_supports_freq_invariance(void)
64	{
65	return static_branch_likely(&cpufreq_freq_invariance);
66	}
67
68	/* Flag to suspend/resume CPUFreq governors */
69	static bool cpufreq_suspended;
70
71	static inline bool has_target(void)
72	{
73	return cpufreq_driver->target_index || cpufreq_driver->target;
74	}
75
76	bool has_target_index(void)
77	{
78	return !!cpufreq_driver->target_index;
79	}
80
81	/* internal prototypes */
82	static unsigned int __cpufreq_get(struct cpufreq_policy *policy);
83	static int cpufreq_init_governor(struct cpufreq_policy *policy);
84	static void cpufreq_exit_governor(struct cpufreq_policy *policy);
85	static void cpufreq_governor_limits(struct cpufreq_policy *policy);
86	static int cpufreq_set_policy(struct cpufreq_policy *policy,
87	struct cpufreq_governor *new_gov,
88	unsigned int new_pol);
89	static bool cpufreq_boost_supported(void);
90
91	/*
92	* Two notifier lists: the "policy" list is involved in the
93	* validation process for a new CPU frequency policy; the
94	* "transition" list for kernel code that needs to handle
95	* changes to devices when the CPU clock speed changes.
96	* The mutex locks both lists.
97	*/
98	static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
99	SRCU_NOTIFIER_HEAD_STATIC(cpufreq_transition_notifier_list);
100
101	static int off __read_mostly;
102	static int cpufreq_disabled(void)
103	{
104	return off;
105	}
106	void disable_cpufreq(void)
107	{
108	off = 1;
109	}
110	static DEFINE_MUTEX(cpufreq_governor_mutex);
111
112	bool have_governor_per_policy(void)
113	{
114	return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
115	}
116	EXPORT_SYMBOL_GPL(have_governor_per_policy);
117
118	static struct kobject *cpufreq_global_kobject;
119
120	struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
121	{
122	if (have_governor_per_policy())
123	return &policy->kobj;
124	else
125	return cpufreq_global_kobject;
126	}
127	EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
128
129	static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
130	{
131	struct kernel_cpustat kcpustat;
132	u64 cur_wall_time;
133	u64 idle_time;
134	u64 busy_time;
135
136	cur_wall_time = jiffies64_to_nsecs(j: get_jiffies_64());
137
138	kcpustat_cpu_fetch(dst: &kcpustat, cpu);
139
140	busy_time = kcpustat.cpustat[CPUTIME_USER];
141	busy_time += kcpustat.cpustat[CPUTIME_SYSTEM];
142	busy_time += kcpustat.cpustat[CPUTIME_IRQ];
143	busy_time += kcpustat.cpustat[CPUTIME_SOFTIRQ];
144	busy_time += kcpustat.cpustat[CPUTIME_STEAL];
145	busy_time += kcpustat.cpustat[CPUTIME_NICE];
146
147	idle_time = cur_wall_time - busy_time;
148	if (wall)
149	*wall = div_u64(dividend: cur_wall_time, NSEC_PER_USEC);
150
151	return div_u64(dividend: idle_time, NSEC_PER_USEC);
152	}
153
154	u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
155	{
156	u64 idle_time = get_cpu_idle_time_us(cpu, last_update_time: io_busy ? wall : NULL);
157
158	if (idle_time == -1ULL)
159	return get_cpu_idle_time_jiffy(cpu, wall);
160	else if (!io_busy)
161	idle_time += get_cpu_iowait_time_us(cpu, last_update_time: wall);
162
163	return idle_time;
164	}
165	EXPORT_SYMBOL_GPL(get_cpu_idle_time);
166
167	/*
168	* This is a generic cpufreq init() routine which can be used by cpufreq
169	* drivers of SMP systems. It will do following:
170	* - validate & show freq table passed
171	* - set policies transition latency
172	* - policy->cpus with all possible CPUs
173	*/
174	void cpufreq_generic_init(struct cpufreq_policy *policy,
175	struct cpufreq_frequency_table *table,
176	unsigned int transition_latency)
177	{
178	policy->freq_table = table;
179	policy->cpuinfo.transition_latency = transition_latency;
180
181	/*
182	* The driver only supports the SMP configuration where all processors
183	* share the clock and voltage and clock.
184	*/
185	cpumask_setall(dstp: policy->cpus);
186	}
187	EXPORT_SYMBOL_GPL(cpufreq_generic_init);
188
189	struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
190	{
191	struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
192
193	return policy && cpumask_test_cpu(cpu, cpumask: policy->cpus) ? policy : NULL;
194	}
195	EXPORT_SYMBOL_GPL(cpufreq_cpu_get_raw);
196
197	unsigned int cpufreq_generic_get(unsigned int cpu)
198	{
199	struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
200
201	if (!policy || IS_ERR(ptr: policy->clk)) {
202	pr_err("%s: No %s associated to cpu: %d\n",
203	__func__, policy ? "clk" : "policy", cpu);
204	return 0;
205	}
206
207	return clk_get_rate(clk: policy->clk) / 1000;
208	}
209	EXPORT_SYMBOL_GPL(cpufreq_generic_get);
210
211	/**
212	* cpufreq_cpu_get - Return policy for a CPU and mark it as busy.
213	* @cpu: CPU to find the policy for.
214	*
215	* Call cpufreq_cpu_get_raw() to obtain a cpufreq policy for @cpu and increment
216	* the kobject reference counter of that policy. Return a valid policy on
217	* success or NULL on failure.
218	*
219	* The policy returned by this function has to be released with the help of
220	* cpufreq_cpu_put() to balance its kobject reference counter properly.
221	*/
222	struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
223	{
224	struct cpufreq_policy *policy = NULL;
225	unsigned long flags;
226
227	if (WARN_ON(cpu >= nr_cpu_ids))
228	return NULL;
229
230	/* get the cpufreq driver */
231	read_lock_irqsave(&cpufreq_driver_lock, flags);
232
233	if (cpufreq_driver) {
234	/* get the CPU */
235	policy = cpufreq_cpu_get_raw(cpu);
236	if (policy)
237	kobject_get(kobj: &policy->kobj);
238	}
239
240	read_unlock_irqrestore(&cpufreq_driver_lock, flags);
241
242	return policy;
243	}
244	EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
245
246	/**
247	* cpufreq_cpu_put - Decrement kobject usage counter for cpufreq policy.
248	* @policy: cpufreq policy returned by cpufreq_cpu_get().
249	*/
250	void cpufreq_cpu_put(struct cpufreq_policy *policy)
251	{
252	kobject_put(kobj: &policy->kobj);
253	}
254	EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
255
256	/**
257	* cpufreq_cpu_release - Unlock a policy and decrement its usage counter.
258	* @policy: cpufreq policy returned by cpufreq_cpu_acquire().
259	*/
260	void cpufreq_cpu_release(struct cpufreq_policy *policy)
261	{
262	if (WARN_ON(!policy))
263	return;
264
265	lockdep_assert_held(&policy->rwsem);
266
267	up_write(sem: &policy->rwsem);
268
269	cpufreq_cpu_put(policy);
270	}
271
272	/**
273	* cpufreq_cpu_acquire - Find policy for a CPU, mark it as busy and lock it.
274	* @cpu: CPU to find the policy for.
275	*
276	* Call cpufreq_cpu_get() to get a reference on the cpufreq policy for @cpu and
277	* if the policy returned by it is not NULL, acquire its rwsem for writing.
278	* Return the policy if it is active or release it and return NULL otherwise.
279	*
280	* The policy returned by this function has to be released with the help of
281	* cpufreq_cpu_release() in order to release its rwsem and balance its usage
282	* counter properly.
283	*/
284	struct cpufreq_policy *cpufreq_cpu_acquire(unsigned int cpu)
285	{
286	struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
287
288	if (!policy)
289	return NULL;
290
291	down_write(sem: &policy->rwsem);
292
293	if (policy_is_inactive(policy)) {
294	cpufreq_cpu_release(policy);
295	return NULL;
296	}
297
298	return policy;
299	}
300
301	/*********************************************************************
302	* EXTERNALLY AFFECTING FREQUENCY CHANGES *
303	*********************************************************************/
304
305	/**
306	* adjust_jiffies - Adjust the system "loops_per_jiffy".
307	* @val: CPUFREQ_PRECHANGE or CPUFREQ_POSTCHANGE.
308	* @ci: Frequency change information.
309	*
310	* This function alters the system "loops_per_jiffy" for the clock
311	* speed change. Note that loops_per_jiffy cannot be updated on SMP
312	* systems as each CPU might be scaled differently. So, use the arch
313	* per-CPU loops_per_jiffy value wherever possible.
314	*/
315	static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
316	{
317	#ifndef CONFIG_SMP
318	static unsigned long l_p_j_ref;
319	static unsigned int l_p_j_ref_freq;
320
321	if (ci->flags & CPUFREQ_CONST_LOOPS)
322	return;
323
324	if (!l_p_j_ref_freq) {
325	l_p_j_ref = loops_per_jiffy;
326	l_p_j_ref_freq = ci->old;
327	pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n",
328	l_p_j_ref, l_p_j_ref_freq);
329	}
330	if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
331	loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
332	ci->new);
333	pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n",
334	loops_per_jiffy, ci->new);
335	}
336	#endif
337	}
338
339	/**
340	* cpufreq_notify_transition - Notify frequency transition and adjust jiffies.
341	* @policy: cpufreq policy to enable fast frequency switching for.
342	* @freqs: contain details of the frequency update.
343	* @state: set to CPUFREQ_PRECHANGE or CPUFREQ_POSTCHANGE.
344	*
345	* This function calls the transition notifiers and adjust_jiffies().
346	*
347	* It is called twice on all CPU frequency changes that have external effects.
348	*/
349	static void cpufreq_notify_transition(struct cpufreq_policy *policy,
350	struct cpufreq_freqs *freqs,
351	unsigned int state)
352	{
353	int cpu;
354
355	BUG_ON(irqs_disabled());
356
357	if (cpufreq_disabled())
358	return;
359
360	freqs->policy = policy;
361	freqs->flags = cpufreq_driver->flags;
362	pr_debug("notification %u of frequency transition to %u kHz\n",
363	state, freqs->new);
364
365	switch (state) {
366	case CPUFREQ_PRECHANGE:
367	/*
368	* Detect if the driver reported a value as "old frequency"
369	* which is not equal to what the cpufreq core thinks is
370	* "old frequency".
371	*/
372	if (policy->cur && policy->cur != freqs->old) {
373	pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
374	freqs->old, policy->cur);
375	freqs->old = policy->cur;
376	}
377
378	srcu_notifier_call_chain(nh: &cpufreq_transition_notifier_list,
379	CPUFREQ_PRECHANGE, v: freqs);
380
381	adjust_jiffies(CPUFREQ_PRECHANGE, ci: freqs);
382	break;
383
384	case CPUFREQ_POSTCHANGE:
385	adjust_jiffies(CPUFREQ_POSTCHANGE, ci: freqs);
386	pr_debug("FREQ: %u - CPUs: %*pbl\n", freqs->new,
387	cpumask_pr_args(policy->cpus));
388
389	for_each_cpu(cpu, policy->cpus)
390	trace_cpu_frequency(frequency: freqs->new, cpu_id: cpu);
391
392	srcu_notifier_call_chain(nh: &cpufreq_transition_notifier_list,
393	CPUFREQ_POSTCHANGE, v: freqs);
394
395	cpufreq_stats_record_transition(policy, new_freq: freqs->new);
396	policy->cur = freqs->new;
397	}
398	}
399
400	/* Do post notifications when there are chances that transition has failed */
401	static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
402	struct cpufreq_freqs *freqs, int transition_failed)
403	{
404	cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
405	if (!transition_failed)
406	return;
407
408	swap(freqs->old, freqs->new);
409	cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
410	cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
411	}
412
413	void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
414	struct cpufreq_freqs *freqs)
415	{
416
417	/*
418	* Catch double invocations of _begin() which lead to self-deadlock.
419	* ASYNC_NOTIFICATION drivers are left out because the cpufreq core
420	* doesn't invoke _begin() on their behalf, and hence the chances of
421	* double invocations are very low. Moreover, there are scenarios
422	* where these checks can emit false-positive warnings in these
423	* drivers; so we avoid that by skipping them altogether.
424	*/
425	WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION)
426	&& current == policy->transition_task);
427
428	wait:
429	wait_event(policy->transition_wait, !policy->transition_ongoing);
430
431	spin_lock(lock: &policy->transition_lock);
432
433	if (unlikely(policy->transition_ongoing)) {
434	spin_unlock(lock: &policy->transition_lock);
435	goto wait;
436	}
437
438	policy->transition_ongoing = true;
439	policy->transition_task = current;
440
441	spin_unlock(lock: &policy->transition_lock);
442
443	cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
444	}
445	EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin);
446
447	void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
448	struct cpufreq_freqs *freqs, int transition_failed)
449	{
450	if (WARN_ON(!policy->transition_ongoing))
451	return;
452
453	cpufreq_notify_post_transition(policy, freqs, transition_failed);
454
455	arch_set_freq_scale(cpus: policy->related_cpus,
456	cur_freq: policy->cur,
457	max_freq: policy->cpuinfo.max_freq);
458
459	spin_lock(lock: &policy->transition_lock);
460	policy->transition_ongoing = false;
461	policy->transition_task = NULL;
462	spin_unlock(lock: &policy->transition_lock);
463
464	wake_up(&policy->transition_wait);
465	}
466	EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end);
467
468	/*
469	* Fast frequency switching status count. Positive means "enabled", negative
470	* means "disabled" and 0 means "not decided yet".
471	*/
472	static int cpufreq_fast_switch_count;
473	static DEFINE_MUTEX(cpufreq_fast_switch_lock);
474
475	static void cpufreq_list_transition_notifiers(void)
476	{
477	struct notifier_block *nb;
478
479	pr_info("Registered transition notifiers:\n");
480
481	mutex_lock(&cpufreq_transition_notifier_list.mutex);
482
483	for (nb = cpufreq_transition_notifier_list.head; nb; nb = nb->next)
484	pr_info("%pS\n", nb->notifier_call);
485
486	mutex_unlock(lock: &cpufreq_transition_notifier_list.mutex);
487	}
488
489	/**
490	* cpufreq_enable_fast_switch - Enable fast frequency switching for policy.
491	* @policy: cpufreq policy to enable fast frequency switching for.
492	*
493	* Try to enable fast frequency switching for @policy.
494	*
495	* The attempt will fail if there is at least one transition notifier registered
496	* at this point, as fast frequency switching is quite fundamentally at odds
497	* with transition notifiers. Thus if successful, it will make registration of
498	* transition notifiers fail going forward.
499	*/
500	void cpufreq_enable_fast_switch(struct cpufreq_policy *policy)
501	{
502	lockdep_assert_held(&policy->rwsem);
503
504	if (!policy->fast_switch_possible)
505	return;
506
507	mutex_lock(&cpufreq_fast_switch_lock);
508	if (cpufreq_fast_switch_count >= 0) {
509	cpufreq_fast_switch_count++;
510	policy->fast_switch_enabled = true;
511	} else {
512	pr_warn("CPU%u: Fast frequency switching not enabled\n",
513	policy->cpu);
514	cpufreq_list_transition_notifiers();
515	}
516	mutex_unlock(lock: &cpufreq_fast_switch_lock);
517	}
518	EXPORT_SYMBOL_GPL(cpufreq_enable_fast_switch);
519
520	/**
521	* cpufreq_disable_fast_switch - Disable fast frequency switching for policy.
522	* @policy: cpufreq policy to disable fast frequency switching for.
523	*/
524	void cpufreq_disable_fast_switch(struct cpufreq_policy *policy)
525	{
526	mutex_lock(&cpufreq_fast_switch_lock);
527	if (policy->fast_switch_enabled) {
528	policy->fast_switch_enabled = false;
529	if (!WARN_ON(cpufreq_fast_switch_count <= 0))
530	cpufreq_fast_switch_count--;
531	}
532	mutex_unlock(lock: &cpufreq_fast_switch_lock);
533	}
534	EXPORT_SYMBOL_GPL(cpufreq_disable_fast_switch);
535
536	static unsigned int __resolve_freq(struct cpufreq_policy *policy,
537	unsigned int target_freq, unsigned int relation)
538	{
539	unsigned int idx;
540
541	target_freq = clamp_val(target_freq, policy->min, policy->max);
542
543	if (!policy->freq_table)
544	return target_freq;
545
546	idx = cpufreq_frequency_table_target(policy, target_freq, relation);
547	policy->cached_resolved_idx = idx;
548	policy->cached_target_freq = target_freq;
549	return policy->freq_table[idx].frequency;
550	}
551
552	/**
553	* cpufreq_driver_resolve_freq - Map a target frequency to a driver-supported
554	* one.
555	* @policy: associated policy to interrogate
556	* @target_freq: target frequency to resolve.
557	*
558	* The target to driver frequency mapping is cached in the policy.
559	*
560	* Return: Lowest driver-supported frequency greater than or equal to the
561	* given target_freq, subject to policy (min/max) and driver limitations.
562	*/
563	unsigned int cpufreq_driver_resolve_freq(struct cpufreq_policy *policy,
564	unsigned int target_freq)
565	{
566	return __resolve_freq(policy, target_freq, CPUFREQ_RELATION_LE);
567	}
568	EXPORT_SYMBOL_GPL(cpufreq_driver_resolve_freq);
569
570	unsigned int cpufreq_policy_transition_delay_us(struct cpufreq_policy *policy)
571	{
572	unsigned int latency;
573
574	if (policy->transition_delay_us)
575	return policy->transition_delay_us;
576
577	latency = policy->cpuinfo.transition_latency / NSEC_PER_USEC;
578	if (latency) {
579	/*
580	* For platforms that can change the frequency very fast (< 10
581	* us), the above formula gives a decent transition delay. But
582	* for platforms where transition_latency is in milliseconds, it
583	* ends up giving unrealistic values.
584	*
585	* Cap the default transition delay to 10 ms, which seems to be
586	* a reasonable amount of time after which we should reevaluate
587	* the frequency.
588	*/
589	return min(latency * LATENCY_MULTIPLIER, (unsigned int)10000);
590	}
591
592	return LATENCY_MULTIPLIER;
593	}
594	EXPORT_SYMBOL_GPL(cpufreq_policy_transition_delay_us);
595
596	/*********************************************************************
597	* SYSFS INTERFACE *
598	*********************************************************************/
599	static ssize_t show_boost(struct kobject *kobj,
600	struct kobj_attribute *attr, char *buf)
601	{
602	return sprintf(buf, fmt: "%d\n", cpufreq_driver->boost_enabled);
603	}
604
605	static ssize_t store_boost(struct kobject *kobj, struct kobj_attribute *attr,
606	const char *buf, size_t count)
607	{
608	int ret, enable;
609
610	ret = sscanf(buf, "%d", &enable);
611	if (ret != 1 || enable < 0 || enable > 1)
612	return -EINVAL;
613
614	if (cpufreq_boost_trigger_state(state: enable)) {
615	pr_err("%s: Cannot %s BOOST!\n",
616	__func__, enable ? "enable" : "disable");
617	return -EINVAL;
618	}
619
620	pr_debug("%s: cpufreq BOOST %s\n",
621	__func__, enable ? "enabled" : "disabled");
622
623	return count;
624	}
625	define_one_global_rw(boost);
626
627	static ssize_t show_local_boost(struct cpufreq_policy *policy, char *buf)
628	{
629	return sysfs_emit(buf, fmt: "%d\n", policy->boost_enabled);
630	}
631
632	static ssize_t store_local_boost(struct cpufreq_policy *policy,
633	const char *buf, size_t count)
634	{
635	int ret, enable;
636
637	ret = kstrtoint(s: buf, base: 10, res: &enable);
638	if (ret || enable < 0 || enable > 1)
639	return -EINVAL;
640
641	if (!cpufreq_driver->boost_enabled)
642	return -EINVAL;
643
644	if (policy->boost_enabled == enable)
645	return count;
646
647	cpus_read_lock();
648	ret = cpufreq_driver->set_boost(policy, enable);
649	cpus_read_unlock();
650
651	if (ret)
652	return ret;
653
654	policy->boost_enabled = enable;
655
656	return count;
657	}
658
659	static struct freq_attr local_boost = __ATTR(boost, 0644, show_local_boost, store_local_boost);
660
661	static struct cpufreq_governor *find_governor(const char *str_governor)
662	{
663	struct cpufreq_governor *t;
664
665	for_each_governor(t)
666	if (!strncasecmp(s1: str_governor, s2: t->name, CPUFREQ_NAME_LEN))
667	return t;
668
669	return NULL;
670	}
671
672	static struct cpufreq_governor *get_governor(const char *str_governor)
673	{
674	struct cpufreq_governor *t;
675
676	mutex_lock(&cpufreq_governor_mutex);
677	t = find_governor(str_governor);
678	if (!t)
679	goto unlock;
680
681	if (!try_module_get(module: t->owner))
682	t = NULL;
683
684	unlock:
685	mutex_unlock(lock: &cpufreq_governor_mutex);
686
687	return t;
688	}
689
690	static unsigned int cpufreq_parse_policy(char *str_governor)
691	{
692	if (!strncasecmp(s1: str_governor, s2: "performance", CPUFREQ_NAME_LEN))
693	return CPUFREQ_POLICY_PERFORMANCE;
694
695	if (!strncasecmp(s1: str_governor, s2: "powersave", CPUFREQ_NAME_LEN))
696	return CPUFREQ_POLICY_POWERSAVE;
697
698	return CPUFREQ_POLICY_UNKNOWN;
699	}
700
701	/**
702	* cpufreq_parse_governor - parse a governor string only for has_target()
703	* @str_governor: Governor name.
704	*/
705	static struct cpufreq_governor *cpufreq_parse_governor(char *str_governor)
706	{
707	struct cpufreq_governor *t;
708
709	t = get_governor(str_governor);
710	if (t)
711	return t;
712
713	if (request_module("cpufreq_%s", str_governor))
714	return NULL;
715
716	return get_governor(str_governor);
717	}
718
719	/*
720	* cpufreq_per_cpu_attr_read() / show_##file_name() -
721	* print out cpufreq information
722	*
723	* Write out information from cpufreq_driver->policy[cpu]; object must be
724	* "unsigned int".
725	*/
726
727	#define show_one(file_name, object) \
728	static ssize_t show_##file_name \
729	(struct cpufreq_policy *policy, char *buf) \
730	{ \
731	return sprintf(buf, "%u\n", policy->object); \
732	}
733
734	show_one(cpuinfo_min_freq, cpuinfo.min_freq);
735	show_one(cpuinfo_max_freq, cpuinfo.max_freq);
736	show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
737	show_one(scaling_min_freq, min);
738	show_one(scaling_max_freq, max);
739
740	__weak unsigned int arch_freq_get_on_cpu(int cpu)
741	{
742	return 0;
743	}
744
745	static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf)
746	{
747	ssize_t ret;
748	unsigned int freq;
749
750	freq = arch_freq_get_on_cpu(cpu: policy->cpu);
751	if (freq)
752	ret = sprintf(buf, fmt: "%u\n", freq);
753	else if (cpufreq_driver->setpolicy && cpufreq_driver->get)
754	ret = sprintf(buf, fmt: "%u\n", cpufreq_driver->get(policy->cpu));
755	else
756	ret = sprintf(buf, fmt: "%u\n", policy->cur);
757	return ret;
758	}
759
760	/*
761	* cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
762	*/
763	#define store_one(file_name, object) \
764	static ssize_t store_##file_name \
765	(struct cpufreq_policy *policy, const char *buf, size_t count) \
766	{ \
767	unsigned long val; \
768	int ret; \
769	\
770	ret = kstrtoul(buf, 0, &val); \
771	if (ret) \
772	return ret; \
773	\
774	ret = freq_qos_update_request(policy->object##_freq_req, val);\
775	return ret >= 0 ? count : ret; \
776	}
777
778	store_one(scaling_min_freq, min);
779	store_one(scaling_max_freq, max);
780
781	/*
782	* show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
783	*/
784	static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
785	char *buf)
786	{
787	unsigned int cur_freq = __cpufreq_get(policy);
788
789	if (cur_freq)
790	return sprintf(buf, fmt: "%u\n", cur_freq);
791
792	return sprintf(buf, fmt: "<unknown>\n");
793	}
794
795	/*
796	* show_scaling_governor - show the current policy for the specified CPU
797	*/
798	static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
799	{
800	if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
801	return sprintf(buf, fmt: "powersave\n");
802	else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
803	return sprintf(buf, fmt: "performance\n");
804	else if (policy->governor)
805	return scnprintf(buf, CPUFREQ_NAME_PLEN, fmt: "%s\n",
806	policy->governor->name);
807	return -EINVAL;
808	}
809
810	/*
811	* store_scaling_governor - store policy for the specified CPU
812	*/
813	static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
814	const char *buf, size_t count)
815	{
816	char str_governor[16];
817	int ret;
818
819	ret = sscanf(buf, "%15s", str_governor);
820	if (ret != 1)
821	return -EINVAL;
822
823	if (cpufreq_driver->setpolicy) {
824	unsigned int new_pol;
825
826	new_pol = cpufreq_parse_policy(str_governor);
827	if (!new_pol)
828	return -EINVAL;
829
830	ret = cpufreq_set_policy(policy, NULL, new_pol);
831	} else {
832	struct cpufreq_governor *new_gov;
833
834	new_gov = cpufreq_parse_governor(str_governor);
835	if (!new_gov)
836	return -EINVAL;
837
838	ret = cpufreq_set_policy(policy, new_gov,
839	CPUFREQ_POLICY_UNKNOWN);
840
841	module_put(module: new_gov->owner);
842	}
843
844	return ret ? ret : count;
845	}
846
847	/*
848	* show_scaling_driver - show the cpufreq driver currently loaded
849	*/
850	static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
851	{
852	return scnprintf(buf, CPUFREQ_NAME_PLEN, fmt: "%s\n", cpufreq_driver->name);
853	}
854
855	/*
856	* show_scaling_available_governors - show the available CPUfreq governors
857	*/
858	static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
859	char *buf)
860	{
861	ssize_t i = 0;
862	struct cpufreq_governor *t;
863
864	if (!has_target()) {
865	i += sprintf(buf, fmt: "performance powersave");
866	goto out;
867	}
868
869	mutex_lock(&cpufreq_governor_mutex);
870	for_each_governor(t) {
871	if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
872	- (CPUFREQ_NAME_LEN + 2)))
873	break;
874	i += scnprintf(buf: &buf[i], CPUFREQ_NAME_PLEN, fmt: "%s ", t->name);
875	}
876	mutex_unlock(lock: &cpufreq_governor_mutex);
877	out:
878	i += sprintf(buf: &buf[i], fmt: "\n");
879	return i;
880	}
881
882	ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
883	{
884	ssize_t i = 0;
885	unsigned int cpu;
886
887	for_each_cpu(cpu, mask) {
888	i += scnprintf(buf: &buf[i], size: (PAGE_SIZE - i - 2), fmt: "%u ", cpu);
889	if (i >= (PAGE_SIZE - 5))
890	break;
891	}
892
893	/* Remove the extra space at the end */
894	i--;
895
896	i += sprintf(buf: &buf[i], fmt: "\n");
897	return i;
898	}
899	EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
900
901	/*
902	* show_related_cpus - show the CPUs affected by each transition even if
903	* hw coordination is in use
904	*/
905	static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
906	{
907	return cpufreq_show_cpus(policy->related_cpus, buf);
908	}
909
910	/*
911	* show_affected_cpus - show the CPUs affected by each transition
912	*/
913	static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
914	{
915	return cpufreq_show_cpus(policy->cpus, buf);
916	}
917
918	static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
919	const char *buf, size_t count)
920	{
921	unsigned int freq = 0;
922	unsigned int ret;
923
924	if (!policy->governor || !policy->governor->store_setspeed)
925	return -EINVAL;
926
927	ret = sscanf(buf, "%u", &freq);
928	if (ret != 1)
929	return -EINVAL;
930
931	policy->governor->store_setspeed(policy, freq);
932
933	return count;
934	}
935
936	static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
937	{
938	if (!policy->governor || !policy->governor->show_setspeed)
939	return sprintf(buf, fmt: "<unsupported>\n");
940
941	return policy->governor->show_setspeed(policy, buf);
942	}
943
944	/*
945	* show_bios_limit - show the current cpufreq HW/BIOS limitation
946	*/
947	static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
948	{
949	unsigned int limit;
950	int ret;
951	ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
952	if (!ret)
953	return sprintf(buf, fmt: "%u\n", limit);
954	return sprintf(buf, fmt: "%u\n", policy->cpuinfo.max_freq);
955	}
956
957	cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
958	cpufreq_freq_attr_ro(cpuinfo_min_freq);
959	cpufreq_freq_attr_ro(cpuinfo_max_freq);
960	cpufreq_freq_attr_ro(cpuinfo_transition_latency);
961	cpufreq_freq_attr_ro(scaling_available_governors);
962	cpufreq_freq_attr_ro(scaling_driver);
963	cpufreq_freq_attr_ro(scaling_cur_freq);
964	cpufreq_freq_attr_ro(bios_limit);
965	cpufreq_freq_attr_ro(related_cpus);
966	cpufreq_freq_attr_ro(affected_cpus);
967	cpufreq_freq_attr_rw(scaling_min_freq);
968	cpufreq_freq_attr_rw(scaling_max_freq);
969	cpufreq_freq_attr_rw(scaling_governor);
970	cpufreq_freq_attr_rw(scaling_setspeed);
971
972	static struct attribute *cpufreq_attrs[] = {
973	&cpuinfo_min_freq.attr,
974	&cpuinfo_max_freq.attr,
975	&cpuinfo_transition_latency.attr,
976	&scaling_min_freq.attr,
977	&scaling_max_freq.attr,
978	&affected_cpus.attr,
979	&related_cpus.attr,
980	&scaling_governor.attr,
981	&scaling_driver.attr,
982	&scaling_available_governors.attr,
983	&scaling_setspeed.attr,
984	NULL
985	};
986	ATTRIBUTE_GROUPS(cpufreq);
987
988	#define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
989	#define to_attr(a) container_of(a, struct freq_attr, attr)
990
991	static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
992	{
993	struct cpufreq_policy *policy = to_policy(kobj);
994	struct freq_attr *fattr = to_attr(attr);
995	ssize_t ret = -EBUSY;
996
997	if (!fattr->show)
998	return -EIO;
999
1000	down_read(sem: &policy->rwsem);
1001	if (likely(!policy_is_inactive(policy)))
1002	ret = fattr->show(policy, buf);
1003	up_read(sem: &policy->rwsem);
1004
1005	return ret;
1006	}
1007
1008	static ssize_t store(struct kobject *kobj, struct attribute *attr,
1009	const char *buf, size_t count)
1010	{
1011	struct cpufreq_policy *policy = to_policy(kobj);
1012	struct freq_attr *fattr = to_attr(attr);
1013	ssize_t ret = -EBUSY;
1014
1015	if (!fattr->store)
1016	return -EIO;
1017
1018	down_write(sem: &policy->rwsem);
1019	if (likely(!policy_is_inactive(policy)))
1020	ret = fattr->store(policy, buf, count);
1021	up_write(sem: &policy->rwsem);
1022
1023	return ret;
1024	}
1025
1026	static void cpufreq_sysfs_release(struct kobject *kobj)
1027	{
1028	struct cpufreq_policy *policy = to_policy(kobj);
1029	pr_debug("last reference is dropped\n");
1030	complete(&policy->kobj_unregister);
1031	}
1032
1033	static const struct sysfs_ops sysfs_ops = {
1034	.show = show,
1035	.store = store,
1036	};
1037
1038	static const struct kobj_type ktype_cpufreq = {
1039	.sysfs_ops = &sysfs_ops,
1040	.default_groups = cpufreq_groups,
1041	.release = cpufreq_sysfs_release,
1042	};
1043
1044	static void add_cpu_dev_symlink(struct cpufreq_policy *policy, unsigned int cpu,
1045	struct device *dev)
1046	{
1047	if (unlikely(!dev))
1048	return;
1049
1050	if (cpumask_test_and_set_cpu(cpu, cpumask: policy->real_cpus))
1051	return;
1052
1053	dev_dbg(dev, "%s: Adding symlink\n", __func__);
1054	if (sysfs_create_link(kobj: &dev->kobj, target: &policy->kobj, name: "cpufreq"))
1055	dev_err(dev, "cpufreq symlink creation failed\n");
1056	}
1057
1058	static void remove_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu,
1059	struct device *dev)
1060	{
1061	dev_dbg(dev, "%s: Removing symlink\n", __func__);
1062	sysfs_remove_link(kobj: &dev->kobj, name: "cpufreq");
1063	cpumask_clear_cpu(cpu, dstp: policy->real_cpus);
1064	}
1065
1066	static int cpufreq_add_dev_interface(struct cpufreq_policy *policy)
1067	{
1068	struct freq_attr **drv_attr;
1069	int ret = 0;
1070
1071	/* set up files for this cpu device */
1072	drv_attr = cpufreq_driver->attr;
1073	while (drv_attr && *drv_attr) {
1074	ret = sysfs_create_file(kobj: &policy->kobj, attr: &((*drv_attr)->attr));
1075	if (ret)
1076	return ret;
1077	drv_attr++;
1078	}
1079	if (cpufreq_driver->get) {
1080	ret = sysfs_create_file(kobj: &policy->kobj, attr: &cpuinfo_cur_freq.attr);
1081	if (ret)
1082	return ret;
1083	}
1084
1085	ret = sysfs_create_file(kobj: &policy->kobj, attr: &scaling_cur_freq.attr);
1086	if (ret)
1087	return ret;
1088
1089	if (cpufreq_driver->bios_limit) {
1090	ret = sysfs_create_file(kobj: &policy->kobj, attr: &bios_limit.attr);
1091	if (ret)
1092	return ret;
1093	}
1094
1095	if (cpufreq_boost_supported()) {
1096	ret = sysfs_create_file(kobj: &policy->kobj, attr: &local_boost.attr);
1097	if (ret)
1098	return ret;
1099	}
1100
1101	return 0;
1102	}
1103
1104	static int cpufreq_init_policy(struct cpufreq_policy *policy)
1105	{
1106	struct cpufreq_governor *gov = NULL;
1107	unsigned int pol = CPUFREQ_POLICY_UNKNOWN;
1108	int ret;
1109
1110	if (has_target()) {
1111	/* Update policy governor to the one used before hotplug. */
1112	gov = get_governor(str_governor: policy->last_governor);
1113	if (gov) {
1114	pr_debug("Restoring governor %s for cpu %d\n",
1115	gov->name, policy->cpu);
1116	} else {
1117	gov = get_governor(str_governor: default_governor);
1118	}
1119
1120	if (!gov) {
1121	gov = cpufreq_default_governor();
1122	__module_get(module: gov->owner);
1123	}
1124
1125	} else {
1126
1127	/* Use the default policy if there is no last_policy. */
1128	if (policy->last_policy) {
1129	pol = policy->last_policy;
1130	} else {
1131	pol = cpufreq_parse_policy(str_governor: default_governor);
1132	/*
1133	* In case the default governor is neither "performance"
1134	* nor "powersave", fall back to the initial policy
1135	* value set by the driver.
1136	*/
1137	if (pol == CPUFREQ_POLICY_UNKNOWN)
1138	pol = policy->policy;
1139	}
1140	if (pol != CPUFREQ_POLICY_PERFORMANCE &&
1141	pol != CPUFREQ_POLICY_POWERSAVE)
1142	return -ENODATA;
1143	}
1144
1145	ret = cpufreq_set_policy(policy, new_gov: gov, new_pol: pol);
1146	if (gov)
1147	module_put(module: gov->owner);
1148
1149	return ret;
1150	}
1151
1152	static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
1153	{
1154	int ret = 0;
1155
1156	/* Has this CPU been taken care of already? */
1157	if (cpumask_test_cpu(cpu, cpumask: policy->cpus))
1158	return 0;
1159
1160	down_write(sem: &policy->rwsem);
1161	if (has_target())
1162	cpufreq_stop_governor(policy);
1163
1164	cpumask_set_cpu(cpu, dstp: policy->cpus);
1165
1166	if (has_target()) {
1167	ret = cpufreq_start_governor(policy);
1168	if (ret)
1169	pr_err("%s: Failed to start governor\n", __func__);
1170	}
1171	up_write(sem: &policy->rwsem);
1172	return ret;
1173	}
1174
1175	void refresh_frequency_limits(struct cpufreq_policy *policy)
1176	{
1177	if (!policy_is_inactive(policy)) {
1178	pr_debug("updating policy for CPU %u\n", policy->cpu);
1179
1180	cpufreq_set_policy(policy, new_gov: policy->governor, new_pol: policy->policy);
1181	}
1182	}
1183	EXPORT_SYMBOL(refresh_frequency_limits);
1184
1185	static void handle_update(struct work_struct *work)
1186	{
1187	struct cpufreq_policy *policy =
1188	container_of(work, struct cpufreq_policy, update);
1189
1190	pr_debug("handle_update for cpu %u called\n", policy->cpu);
1191	down_write(sem: &policy->rwsem);
1192	refresh_frequency_limits(policy);
1193	up_write(sem: &policy->rwsem);
1194	}
1195
1196	static int cpufreq_notifier_min(struct notifier_block *nb, unsigned long freq,
1197	void *data)
1198	{
1199	struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_min);
1200
1201	schedule_work(work: &policy->update);
1202	return 0;
1203	}
1204
1205	static int cpufreq_notifier_max(struct notifier_block *nb, unsigned long freq,
1206	void *data)
1207	{
1208	struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_max);
1209
1210	schedule_work(work: &policy->update);
1211	return 0;
1212	}
1213
1214	static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy)
1215	{
1216	struct kobject *kobj;
1217	struct completion *cmp;
1218
1219	down_write(sem: &policy->rwsem);
1220	cpufreq_stats_free_table(policy);
1221	kobj = &policy->kobj;
1222	cmp = &policy->kobj_unregister;
1223	up_write(sem: &policy->rwsem);
1224	kobject_put(kobj);
1225
1226	/*
1227	* We need to make sure that the underlying kobj is
1228	* actually not referenced anymore by anybody before we
1229	* proceed with unloading.
1230	*/
1231	pr_debug("waiting for dropping of refcount\n");
1232	wait_for_completion(cmp);
1233	pr_debug("wait complete\n");
1234	}
1235
1236	static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu)
1237	{
1238	struct cpufreq_policy *policy;
1239	struct device *dev = get_cpu_device(cpu);
1240	int ret;
1241
1242	if (!dev)
1243	return NULL;
1244
1245	policy = kzalloc(size: sizeof(*policy), GFP_KERNEL);
1246	if (!policy)
1247	return NULL;
1248
1249	if (!alloc_cpumask_var(mask: &policy->cpus, GFP_KERNEL))
1250	goto err_free_policy;
1251
1252	if (!zalloc_cpumask_var(mask: &policy->related_cpus, GFP_KERNEL))
1253	goto err_free_cpumask;
1254
1255	if (!zalloc_cpumask_var(mask: &policy->real_cpus, GFP_KERNEL))
1256	goto err_free_rcpumask;
1257
1258	init_completion(x: &policy->kobj_unregister);
1259	ret = kobject_init_and_add(kobj: &policy->kobj, ktype: &ktype_cpufreq,
1260	parent: cpufreq_global_kobject, fmt: "policy%u", cpu);
1261	if (ret) {
1262	dev_err(dev, "%s: failed to init policy->kobj: %d\n", __func__, ret);
1263	/*
1264	* The entire policy object will be freed below, but the extra
1265	* memory allocated for the kobject name needs to be freed by
1266	* releasing the kobject.
1267	*/
1268	kobject_put(kobj: &policy->kobj);
1269	goto err_free_real_cpus;
1270	}
1271
1272	freq_constraints_init(qos: &policy->constraints);
1273
1274	policy->nb_min.notifier_call = cpufreq_notifier_min;
1275	policy->nb_max.notifier_call = cpufreq_notifier_max;
1276
1277	ret = freq_qos_add_notifier(qos: &policy->constraints, type: FREQ_QOS_MIN,
1278	notifier: &policy->nb_min);
1279	if (ret) {
1280	dev_err(dev, "Failed to register MIN QoS notifier: %d (CPU%u)\n",
1281	ret, cpu);
1282	goto err_kobj_remove;
1283	}
1284
1285	ret = freq_qos_add_notifier(qos: &policy->constraints, type: FREQ_QOS_MAX,
1286	notifier: &policy->nb_max);
1287	if (ret) {
1288	dev_err(dev, "Failed to register MAX QoS notifier: %d (CPU%u)\n",
1289	ret, cpu);
1290	goto err_min_qos_notifier;
1291	}
1292
1293	INIT_LIST_HEAD(list: &policy->policy_list);
1294	init_rwsem(&policy->rwsem);
1295	spin_lock_init(&policy->transition_lock);
1296	init_waitqueue_head(&policy->transition_wait);
1297	INIT_WORK(&policy->update, handle_update);
1298
1299	policy->cpu = cpu;
1300	return policy;
1301
1302	err_min_qos_notifier:
1303	freq_qos_remove_notifier(qos: &policy->constraints, type: FREQ_QOS_MIN,
1304	notifier: &policy->nb_min);
1305	err_kobj_remove:
1306	cpufreq_policy_put_kobj(policy);
1307	err_free_real_cpus:
1308	free_cpumask_var(mask: policy->real_cpus);
1309	err_free_rcpumask:
1310	free_cpumask_var(mask: policy->related_cpus);
1311	err_free_cpumask:
1312	free_cpumask_var(mask: policy->cpus);
1313	err_free_policy:
1314	kfree(objp: policy);
1315
1316	return NULL;
1317	}
1318
1319	static void cpufreq_policy_free(struct cpufreq_policy *policy)
1320	{
1321	unsigned long flags;
1322	int cpu;
1323
1324	/*
1325	* The callers must ensure the policy is inactive by now, to avoid any
1326	* races with show()/store() callbacks.
1327	*/
1328	if (unlikely(!policy_is_inactive(policy)))
1329	pr_warn("%s: Freeing active policy\n", __func__);
1330
1331	/* Remove policy from list */
1332	write_lock_irqsave(&cpufreq_driver_lock, flags);
1333	list_del(entry: &policy->policy_list);
1334
1335	for_each_cpu(cpu, policy->related_cpus)
1336	per_cpu(cpufreq_cpu_data, cpu) = NULL;
1337	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1338
1339	freq_qos_remove_notifier(qos: &policy->constraints, type: FREQ_QOS_MAX,
1340	notifier: &policy->nb_max);
1341	freq_qos_remove_notifier(qos: &policy->constraints, type: FREQ_QOS_MIN,
1342	notifier: &policy->nb_min);
1343
1344	/* Cancel any pending policy->update work before freeing the policy. */
1345	cancel_work_sync(work: &policy->update);
1346
1347	if (policy->max_freq_req) {
1348	/*
1349	* Remove max_freq_req after sending CPUFREQ_REMOVE_POLICY
1350	* notification, since CPUFREQ_CREATE_POLICY notification was
1351	* sent after adding max_freq_req earlier.
1352	*/
1353	blocking_notifier_call_chain(nh: &cpufreq_policy_notifier_list,
1354	CPUFREQ_REMOVE_POLICY, v: policy);
1355	freq_qos_remove_request(req: policy->max_freq_req);
1356	}
1357
1358	freq_qos_remove_request(req: policy->min_freq_req);
1359	kfree(objp: policy->min_freq_req);
1360
1361	cpufreq_policy_put_kobj(policy);
1362	free_cpumask_var(mask: policy->real_cpus);
1363	free_cpumask_var(mask: policy->related_cpus);
1364	free_cpumask_var(mask: policy->cpus);
1365	kfree(objp: policy);
1366	}
1367
1368	static int cpufreq_online(unsigned int cpu)
1369	{
1370	struct cpufreq_policy *policy;
1371	bool new_policy;
1372	unsigned long flags;
1373	unsigned int j;
1374	int ret;
1375
1376	pr_debug("%s: bringing CPU%u online\n", __func__, cpu);
1377
1378	/* Check if this CPU already has a policy to manage it */
1379	policy = per_cpu(cpufreq_cpu_data, cpu);
1380	if (policy) {
1381	WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus));
1382	if (!policy_is_inactive(policy))
1383	return cpufreq_add_policy_cpu(policy, cpu);
1384
1385	/* This is the only online CPU for the policy. Start over. */
1386	new_policy = false;
1387	down_write(sem: &policy->rwsem);
1388	policy->cpu = cpu;
1389	policy->governor = NULL;
1390	} else {
1391	new_policy = true;
1392	policy = cpufreq_policy_alloc(cpu);
1393	if (!policy)
1394	return -ENOMEM;
1395	down_write(sem: &policy->rwsem);
1396	}
1397
1398	if (!new_policy && cpufreq_driver->online) {
1399	/* Recover policy->cpus using related_cpus */
1400	cpumask_copy(dstp: policy->cpus, srcp: policy->related_cpus);
1401
1402	ret = cpufreq_driver->online(policy);
1403	if (ret) {
1404	pr_debug("%s: %d: initialization failed\n", __func__,
1405	__LINE__);
1406	goto out_exit_policy;
1407	}
1408	} else {
1409	cpumask_copy(dstp: policy->cpus, cpumask_of(cpu));
1410
1411	/*
1412	* Call driver. From then on the cpufreq must be able
1413	* to accept all calls to ->verify and ->setpolicy for this CPU.
1414	*/
1415	ret = cpufreq_driver->init(policy);
1416	if (ret) {
1417	pr_debug("%s: %d: initialization failed\n", __func__,
1418	__LINE__);
1419	goto out_free_policy;
1420	}
1421
1422	/*
1423	* The initialization has succeeded and the policy is online.
1424	* If there is a problem with its frequency table, take it
1425	* offline and drop it.
1426	*/
1427	ret = cpufreq_table_validate_and_sort(policy);
1428	if (ret)
1429	goto out_offline_policy;
1430
1431	/* related_cpus should at least include policy->cpus. */
1432	cpumask_copy(dstp: policy->related_cpus, srcp: policy->cpus);
1433	}
1434
1435	/*
1436	* affected cpus must always be the one, which are online. We aren't
1437	* managing offline cpus here.
1438	*/
1439	cpumask_and(dstp: policy->cpus, src1p: policy->cpus, cpu_online_mask);
1440
1441	if (new_policy) {
1442	for_each_cpu(j, policy->related_cpus) {
1443	per_cpu(cpufreq_cpu_data, j) = policy;
1444	add_cpu_dev_symlink(policy, cpu: j, dev: get_cpu_device(cpu: j));
1445	}
1446
1447	policy->min_freq_req = kzalloc(size: 2 * sizeof(*policy->min_freq_req),
1448	GFP_KERNEL);
1449	if (!policy->min_freq_req) {
1450	ret = -ENOMEM;
1451	goto out_destroy_policy;
1452	}
1453
1454	ret = freq_qos_add_request(qos: &policy->constraints,
1455	req: policy->min_freq_req, type: FREQ_QOS_MIN,
1456	FREQ_QOS_MIN_DEFAULT_VALUE);
1457	if (ret < 0) {
1458	/*
1459	* So we don't call freq_qos_remove_request() for an
1460	* uninitialized request.
1461	*/
1462	kfree(objp: policy->min_freq_req);
1463	policy->min_freq_req = NULL;
1464	goto out_destroy_policy;
1465	}
1466
1467	/*
1468	* This must be initialized right here to avoid calling
1469	* freq_qos_remove_request() on uninitialized request in case
1470	* of errors.
1471	*/
1472	policy->max_freq_req = policy->min_freq_req + 1;
1473
1474	ret = freq_qos_add_request(qos: &policy->constraints,
1475	req: policy->max_freq_req, type: FREQ_QOS_MAX,
1476	FREQ_QOS_MAX_DEFAULT_VALUE);
1477	if (ret < 0) {
1478	policy->max_freq_req = NULL;
1479	goto out_destroy_policy;
1480	}
1481
1482	blocking_notifier_call_chain(nh: &cpufreq_policy_notifier_list,
1483	CPUFREQ_CREATE_POLICY, v: policy);
1484	}
1485
1486	if (cpufreq_driver->get && has_target()) {
1487	policy->cur = cpufreq_driver->get(policy->cpu);
1488	if (!policy->cur) {
1489	ret = -EIO;
1490	pr_err("%s: ->get() failed\n", __func__);
1491	goto out_destroy_policy;
1492	}
1493	}
1494
1495	/*
1496	* Sometimes boot loaders set CPU frequency to a value outside of
1497	* frequency table present with cpufreq core. In such cases CPU might be
1498	* unstable if it has to run on that frequency for long duration of time
1499	* and so its better to set it to a frequency which is specified in
1500	* freq-table. This also makes cpufreq stats inconsistent as
1501	* cpufreq-stats would fail to register because current frequency of CPU
1502	* isn't found in freq-table.
1503	*
1504	* Because we don't want this change to effect boot process badly, we go
1505	* for the next freq which is >= policy->cur ('cur' must be set by now,
1506	* otherwise we will end up setting freq to lowest of the table as 'cur'
1507	* is initialized to zero).
1508	*
1509	* We are passing target-freq as "policy->cur - 1" otherwise
1510	* __cpufreq_driver_target() would simply fail, as policy->cur will be
1511	* equal to target-freq.
1512	*/
1513	if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK)
1514	&& has_target()) {
1515	unsigned int old_freq = policy->cur;
1516
1517	/* Are we running at unknown frequency ? */
1518	ret = cpufreq_frequency_table_get_index(policy, freq: old_freq);
1519	if (ret == -EINVAL) {
1520	ret = __cpufreq_driver_target(policy, target_freq: old_freq - 1,
1521	CPUFREQ_RELATION_L);
1522
1523	/*
1524	* Reaching here after boot in a few seconds may not
1525	* mean that system will remain stable at "unknown"
1526	* frequency for longer duration. Hence, a BUG_ON().
1527	*/
1528	BUG_ON(ret);
1529	pr_info("%s: CPU%d: Running at unlisted initial frequency: %u KHz, changing to: %u KHz\n",
1530	__func__, policy->cpu, old_freq, policy->cur);
1531	}
1532	}
1533
1534	if (new_policy) {
1535	ret = cpufreq_add_dev_interface(policy);
1536	if (ret)
1537	goto out_destroy_policy;
1538
1539	cpufreq_stats_create_table(policy);
1540
1541	write_lock_irqsave(&cpufreq_driver_lock, flags);
1542	list_add(new: &policy->policy_list, head: &cpufreq_policy_list);
1543	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1544
1545	/*
1546	* Register with the energy model before
1547	* sugov_eas_rebuild_sd() is called, which will result
1548	* in rebuilding of the sched domains, which should only be done
1549	* once the energy model is properly initialized for the policy
1550	* first.
1551	*
1552	* Also, this should be called before the policy is registered
1553	* with cooling framework.
1554	*/
1555	if (cpufreq_driver->register_em)
1556	cpufreq_driver->register_em(policy);
1557	}
1558
1559	ret = cpufreq_init_policy(policy);
1560	if (ret) {
1561	pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n",
1562	__func__, cpu, ret);
1563	goto out_destroy_policy;
1564	}
1565
1566	up_write(sem: &policy->rwsem);
1567
1568	kobject_uevent(kobj: &policy->kobj, action: KOBJ_ADD);
1569
1570	/* Callback for handling stuff after policy is ready */
1571	if (cpufreq_driver->ready)
1572	cpufreq_driver->ready(policy);
1573
1574	if (cpufreq_thermal_control_enabled(drv: cpufreq_driver))
1575	policy->cdev = of_cpufreq_cooling_register(policy);
1576
1577	pr_debug("initialization complete\n");
1578
1579	return 0;
1580
1581	out_destroy_policy:
1582	for_each_cpu(j, policy->real_cpus)
1583	remove_cpu_dev_symlink(policy, cpu: j, dev: get_cpu_device(cpu: j));
1584
1585	out_offline_policy:
1586	if (cpufreq_driver->offline)
1587	cpufreq_driver->offline(policy);
1588
1589	out_exit_policy:
1590	if (cpufreq_driver->exit)
1591	cpufreq_driver->exit(policy);
1592
1593	out_free_policy:
1594	cpumask_clear(dstp: policy->cpus);
1595	up_write(sem: &policy->rwsem);
1596
1597	cpufreq_policy_free(policy);
1598	return ret;
1599	}
1600
1601	/**
1602	* cpufreq_add_dev - the cpufreq interface for a CPU device.
1603	* @dev: CPU device.
1604	* @sif: Subsystem interface structure pointer (not used)
1605	*/
1606	static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1607	{
1608	struct cpufreq_policy *policy;
1609	unsigned cpu = dev->id;
1610	int ret;
1611
1612	dev_dbg(dev, "%s: adding CPU%u\n", __func__, cpu);
1613
1614	if (cpu_online(cpu)) {
1615	ret = cpufreq_online(cpu);
1616	if (ret)
1617	return ret;
1618	}
1619
1620	/* Create sysfs link on CPU registration */
1621	policy = per_cpu(cpufreq_cpu_data, cpu);
1622	if (policy)
1623	add_cpu_dev_symlink(policy, cpu, dev);
1624
1625	return 0;
1626	}
1627
1628	static void __cpufreq_offline(unsigned int cpu, struct cpufreq_policy *policy)
1629	{
1630	int ret;
1631
1632	if (has_target())
1633	cpufreq_stop_governor(policy);
1634
1635	cpumask_clear_cpu(cpu, dstp: policy->cpus);
1636
1637	if (!policy_is_inactive(policy)) {
1638	/* Nominate a new CPU if necessary. */
1639	if (cpu == policy->cpu)
1640	policy->cpu = cpumask_any(policy->cpus);
1641
1642	/* Start the governor again for the active policy. */
1643	if (has_target()) {
1644	ret = cpufreq_start_governor(policy);
1645	if (ret)
1646	pr_err("%s: Failed to start governor\n", __func__);
1647	}
1648
1649	return;
1650	}
1651
1652	if (has_target())
1653	strscpy(p: policy->last_governor, q: policy->governor->name,
1654	CPUFREQ_NAME_LEN);
1655	else
1656	policy->last_policy = policy->policy;
1657
1658	if (cpufreq_thermal_control_enabled(drv: cpufreq_driver)) {
1659	cpufreq_cooling_unregister(cdev: policy->cdev);
1660	policy->cdev = NULL;
1661	}
1662
1663	if (has_target())
1664	cpufreq_exit_governor(policy);
1665
1666	/*
1667	* Perform the ->offline() during light-weight tear-down, as
1668	* that allows fast recovery when the CPU comes back.
1669	*/
1670	if (cpufreq_driver->offline) {
1671	cpufreq_driver->offline(policy);
1672	} else if (cpufreq_driver->exit) {
1673	cpufreq_driver->exit(policy);
1674	policy->freq_table = NULL;
1675	}
1676	}
1677
1678	static int cpufreq_offline(unsigned int cpu)
1679	{
1680	struct cpufreq_policy *policy;
1681
1682	pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1683
1684	policy = cpufreq_cpu_get_raw(cpu);
1685	if (!policy) {
1686	pr_debug("%s: No cpu_data found\n", __func__);
1687	return 0;
1688	}
1689
1690	down_write(sem: &policy->rwsem);
1691
1692	__cpufreq_offline(cpu, policy);
1693
1694	up_write(sem: &policy->rwsem);
1695	return 0;
1696	}
1697
1698	/*
1699	* cpufreq_remove_dev - remove a CPU device
1700	*
1701	* Removes the cpufreq interface for a CPU device.
1702	*/
1703	static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1704	{
1705	unsigned int cpu = dev->id;
1706	struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1707
1708	if (!policy)
1709	return;
1710
1711	down_write(sem: &policy->rwsem);
1712
1713	if (cpu_online(cpu))
1714	__cpufreq_offline(cpu, policy);
1715
1716	remove_cpu_dev_symlink(policy, cpu, dev);
1717
1718	if (!cpumask_empty(srcp: policy->real_cpus)) {
1719	up_write(sem: &policy->rwsem);
1720	return;
1721	}
1722
1723	/* We did light-weight exit earlier, do full tear down now */
1724	if (cpufreq_driver->offline)
1725	cpufreq_driver->exit(policy);
1726
1727	up_write(sem: &policy->rwsem);
1728
1729	cpufreq_policy_free(policy);
1730	}
1731
1732	/**
1733	* cpufreq_out_of_sync - Fix up actual and saved CPU frequency difference.
1734	* @policy: Policy managing CPUs.
1735	* @new_freq: New CPU frequency.
1736	*
1737	* Adjust to the current frequency first and clean up later by either calling
1738	* cpufreq_update_policy(), or scheduling handle_update().
1739	*/
1740	static void cpufreq_out_of_sync(struct cpufreq_policy *policy,
1741	unsigned int new_freq)
1742	{
1743	struct cpufreq_freqs freqs;
1744
1745	pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n",
1746	policy->cur, new_freq);
1747
1748	freqs.old = policy->cur;
1749	freqs.new = new_freq;
1750
1751	cpufreq_freq_transition_begin(policy, &freqs);
1752	cpufreq_freq_transition_end(policy, &freqs, 0);
1753	}
1754
1755	static unsigned int cpufreq_verify_current_freq(struct cpufreq_policy *policy, bool update)
1756	{
1757	unsigned int new_freq;
1758
1759	new_freq = cpufreq_driver->get(policy->cpu);
1760	if (!new_freq)
1761	return 0;
1762
1763	/*
1764	* If fast frequency switching is used with the given policy, the check
1765	* against policy->cur is pointless, so skip it in that case.
1766	*/
1767	if (policy->fast_switch_enabled || !has_target())
1768	return new_freq;
1769
1770	if (policy->cur != new_freq) {
1771	/*
1772	* For some platforms, the frequency returned by hardware may be
1773	* slightly different from what is provided in the frequency
1774	* table, for example hardware may return 499 MHz instead of 500
1775	* MHz. In such cases it is better to avoid getting into
1776	* unnecessary frequency updates.
1777	*/
1778	if (abs(policy->cur - new_freq) < KHZ_PER_MHZ)
1779	return policy->cur;
1780
1781	cpufreq_out_of_sync(policy, new_freq);
1782	if (update)
1783	schedule_work(work: &policy->update);
1784	}
1785
1786	return new_freq;
1787	}
1788
1789	/**
1790	* cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1791	* @cpu: CPU number
1792	*
1793	* This is the last known freq, without actually getting it from the driver.
1794	* Return value will be same as what is shown in scaling_cur_freq in sysfs.
1795	*/
1796	unsigned int cpufreq_quick_get(unsigned int cpu)
1797	{
1798	struct cpufreq_policy *policy;
1799	unsigned int ret_freq = 0;
1800	unsigned long flags;
1801
1802	read_lock_irqsave(&cpufreq_driver_lock, flags);
1803
1804	if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get) {
1805	ret_freq = cpufreq_driver->get(cpu);
1806	read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1807	return ret_freq;
1808	}
1809
1810	read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1811
1812	policy = cpufreq_cpu_get(cpu);
1813	if (policy) {
1814	ret_freq = policy->cur;
1815	cpufreq_cpu_put(policy);
1816	}
1817
1818	return ret_freq;
1819	}
1820	EXPORT_SYMBOL(cpufreq_quick_get);
1821
1822	/**
1823	* cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1824	* @cpu: CPU number
1825	*
1826	* Just return the max possible frequency for a given CPU.
1827	*/
1828	unsigned int cpufreq_quick_get_max(unsigned int cpu)
1829	{
1830	struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1831	unsigned int ret_freq = 0;
1832
1833	if (policy) {
1834	ret_freq = policy->max;
1835	cpufreq_cpu_put(policy);
1836	}
1837
1838	return ret_freq;
1839	}
1840	EXPORT_SYMBOL(cpufreq_quick_get_max);
1841
1842	/**
1843	* cpufreq_get_hw_max_freq - get the max hardware frequency of the CPU
1844	* @cpu: CPU number
1845	*
1846	* The default return value is the max_freq field of cpuinfo.
1847	*/
1848	__weak unsigned int cpufreq_get_hw_max_freq(unsigned int cpu)
1849	{
1850	struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1851	unsigned int ret_freq = 0;
1852
1853	if (policy) {
1854	ret_freq = policy->cpuinfo.max_freq;
1855	cpufreq_cpu_put(policy);
1856	}
1857
1858	return ret_freq;
1859	}
1860	EXPORT_SYMBOL(cpufreq_get_hw_max_freq);
1861
1862	static unsigned int __cpufreq_get(struct cpufreq_policy *policy)
1863	{
1864	if (unlikely(policy_is_inactive(policy)))
1865	return 0;
1866
1867	return cpufreq_verify_current_freq(policy, update: true);
1868	}
1869
1870	/**
1871	* cpufreq_get - get the current CPU frequency (in kHz)
1872	* @cpu: CPU number
1873	*
1874	* Get the CPU current (static) CPU frequency
1875	*/
1876	unsigned int cpufreq_get(unsigned int cpu)
1877	{
1878	struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1879	unsigned int ret_freq = 0;
1880
1881	if (policy) {
1882	down_read(sem: &policy->rwsem);
1883	if (cpufreq_driver->get)
1884	ret_freq = __cpufreq_get(policy);
1885	up_read(sem: &policy->rwsem);
1886
1887	cpufreq_cpu_put(policy);
1888	}
1889
1890	return ret_freq;
1891	}
1892	EXPORT_SYMBOL(cpufreq_get);
1893
1894	static struct subsys_interface cpufreq_interface = {
1895	.name = "cpufreq",
1896	.subsys = &cpu_subsys,
1897	.add_dev = cpufreq_add_dev,
1898	.remove_dev = cpufreq_remove_dev,
1899	};
1900
1901	/*
1902	* In case platform wants some specific frequency to be configured
1903	* during suspend..
1904	*/
1905	int cpufreq_generic_suspend(struct cpufreq_policy *policy)
1906	{
1907	int ret;
1908
1909	if (!policy->suspend_freq) {
1910	pr_debug("%s: suspend_freq not defined\n", __func__);
1911	return 0;
1912	}
1913
1914	pr_debug("%s: Setting suspend-freq: %u\n", __func__,
1915	policy->suspend_freq);
1916
1917	ret = __cpufreq_driver_target(policy, target_freq: policy->suspend_freq,
1918	CPUFREQ_RELATION_H);
1919	if (ret)
1920	pr_err("%s: unable to set suspend-freq: %u. err: %d\n",
1921	__func__, policy->suspend_freq, ret);
1922
1923	return ret;
1924	}
1925	EXPORT_SYMBOL(cpufreq_generic_suspend);
1926
1927	/**
1928	* cpufreq_suspend() - Suspend CPUFreq governors.
1929	*
1930	* Called during system wide Suspend/Hibernate cycles for suspending governors
1931	* as some platforms can't change frequency after this point in suspend cycle.
1932	* Because some of the devices (like: i2c, regulators, etc) they use for
1933	* changing frequency are suspended quickly after this point.
1934	*/
1935	void cpufreq_suspend(void)
1936	{
1937	struct cpufreq_policy *policy;
1938
1939	if (!cpufreq_driver)
1940	return;
1941
1942	if (!has_target() && !cpufreq_driver->suspend)
1943	goto suspend;
1944
1945	pr_debug("%s: Suspending Governors\n", __func__);
1946
1947	for_each_active_policy(policy) {
1948	if (has_target()) {
1949	down_write(sem: &policy->rwsem);
1950	cpufreq_stop_governor(policy);
1951	up_write(sem: &policy->rwsem);
1952	}
1953
1954	if (cpufreq_driver->suspend && cpufreq_driver->suspend(policy))
1955	pr_err("%s: Failed to suspend driver: %s\n", __func__,
1956	cpufreq_driver->name);
1957	}
1958
1959	suspend:
1960	cpufreq_suspended = true;
1961	}
1962
1963	/**
1964	* cpufreq_resume() - Resume CPUFreq governors.
1965	*
1966	* Called during system wide Suspend/Hibernate cycle for resuming governors that
1967	* are suspended with cpufreq_suspend().
1968	*/
1969	void cpufreq_resume(void)
1970	{
1971	struct cpufreq_policy *policy;
1972	int ret;
1973
1974	if (!cpufreq_driver)
1975	return;
1976
1977	if (unlikely(!cpufreq_suspended))
1978	return;
1979
1980	cpufreq_suspended = false;
1981
1982	if (!has_target() && !cpufreq_driver->resume)
1983	return;
1984
1985	pr_debug("%s: Resuming Governors\n", __func__);
1986
1987	for_each_active_policy(policy) {
1988	if (cpufreq_driver->resume && cpufreq_driver->resume(policy)) {
1989	pr_err("%s: Failed to resume driver: %s\n", __func__,
1990	cpufreq_driver->name);
1991	} else if (has_target()) {
1992	down_write(sem: &policy->rwsem);
1993	ret = cpufreq_start_governor(policy);
1994	up_write(sem: &policy->rwsem);
1995
1996	if (ret)
1997	pr_err("%s: Failed to start governor for CPU%u's policy\n",
1998	__func__, policy->cpu);
1999	}
2000	}
2001	}
2002
2003	/**
2004	* cpufreq_driver_test_flags - Test cpufreq driver's flags against given ones.
2005	* @flags: Flags to test against the current cpufreq driver's flags.
2006	*
2007	* Assumes that the driver is there, so callers must ensure that this is the
2008	* case.
2009	*/
2010	bool cpufreq_driver_test_flags(u16 flags)
2011	{
2012	return !!(cpufreq_driver->flags & flags);
2013	}
2014
2015	/**
2016	* cpufreq_get_current_driver - Return the current driver's name.
2017	*
2018	* Return the name string of the currently registered cpufreq driver or NULL if
2019	* none.
2020	*/
2021	const char *cpufreq_get_current_driver(void)
2022	{
2023	if (cpufreq_driver)
2024	return cpufreq_driver->name;
2025
2026	return NULL;
2027	}
2028	EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
2029
2030	/**
2031	* cpufreq_get_driver_data - Return current driver data.
2032	*
2033	* Return the private data of the currently registered cpufreq driver, or NULL
2034	* if no cpufreq driver has been registered.
2035	*/
2036	void *cpufreq_get_driver_data(void)
2037	{
2038	if (cpufreq_driver)
2039	return cpufreq_driver->driver_data;
2040
2041	return NULL;
2042	}
2043	EXPORT_SYMBOL_GPL(cpufreq_get_driver_data);
2044
2045	/*********************************************************************
2046	* NOTIFIER LISTS INTERFACE *
2047	*********************************************************************/
2048
2049	/**
2050	* cpufreq_register_notifier - Register a notifier with cpufreq.
2051	* @nb: notifier function to register.
2052	* @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER.
2053	*
2054	* Add a notifier to one of two lists: either a list of notifiers that run on
2055	* clock rate changes (once before and once after every transition), or a list
2056	* of notifiers that ron on cpufreq policy changes.
2057	*
2058	* This function may sleep and it has the same return values as
2059	* blocking_notifier_chain_register().
2060	*/
2061	int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
2062	{
2063	int ret;
2064
2065	if (cpufreq_disabled())
2066	return -EINVAL;
2067
2068	switch (list) {
2069	case CPUFREQ_TRANSITION_NOTIFIER:
2070	mutex_lock(&cpufreq_fast_switch_lock);
2071
2072	if (cpufreq_fast_switch_count > 0) {
2073	mutex_unlock(lock: &cpufreq_fast_switch_lock);
2074	return -EBUSY;
2075	}
2076	ret = srcu_notifier_chain_register(
2077	nh: &cpufreq_transition_notifier_list, nb);
2078	if (!ret)
2079	cpufreq_fast_switch_count--;
2080
2081	mutex_unlock(lock: &cpufreq_fast_switch_lock);
2082	break;
2083	case CPUFREQ_POLICY_NOTIFIER:
2084	ret = blocking_notifier_chain_register(
2085	nh: &cpufreq_policy_notifier_list, nb);
2086	break;
2087	default:
2088	ret = -EINVAL;
2089	}
2090
2091	return ret;
2092	}
2093	EXPORT_SYMBOL(cpufreq_register_notifier);
2094
2095	/**
2096	* cpufreq_unregister_notifier - Unregister a notifier from cpufreq.
2097	* @nb: notifier block to be unregistered.
2098	* @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER.
2099	*
2100	* Remove a notifier from one of the cpufreq notifier lists.
2101	*
2102	* This function may sleep and it has the same return values as
2103	* blocking_notifier_chain_unregister().
2104	*/
2105	int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
2106	{
2107	int ret;
2108
2109	if (cpufreq_disabled())
2110	return -EINVAL;
2111
2112	switch (list) {
2113	case CPUFREQ_TRANSITION_NOTIFIER:
2114	mutex_lock(&cpufreq_fast_switch_lock);
2115
2116	ret = srcu_notifier_chain_unregister(
2117	nh: &cpufreq_transition_notifier_list, nb);
2118	if (!ret && !WARN_ON(cpufreq_fast_switch_count >= 0))
2119	cpufreq_fast_switch_count++;
2120
2121	mutex_unlock(lock: &cpufreq_fast_switch_lock);
2122	break;
2123	case CPUFREQ_POLICY_NOTIFIER:
2124	ret = blocking_notifier_chain_unregister(
2125	nh: &cpufreq_policy_notifier_list, nb);
2126	break;
2127	default:
2128	ret = -EINVAL;
2129	}
2130
2131	return ret;
2132	}
2133	EXPORT_SYMBOL(cpufreq_unregister_notifier);
2134
2135
2136	/*********************************************************************
2137	* GOVERNORS *
2138	*********************************************************************/
2139
2140	/**
2141	* cpufreq_driver_fast_switch - Carry out a fast CPU frequency switch.
2142	* @policy: cpufreq policy to switch the frequency for.
2143	* @target_freq: New frequency to set (may be approximate).
2144	*
2145	* Carry out a fast frequency switch without sleeping.
2146	*
2147	* The driver's ->fast_switch() callback invoked by this function must be
2148	* suitable for being called from within RCU-sched read-side critical sections
2149	* and it is expected to select the minimum available frequency greater than or
2150	* equal to @target_freq (CPUFREQ_RELATION_L).
2151	*
2152	* This function must not be called if policy->fast_switch_enabled is unset.
2153	*
2154	* Governors calling this function must guarantee that it will never be invoked
2155	* twice in parallel for the same policy and that it will never be called in
2156	* parallel with either ->target() or ->target_index() for the same policy.
2157	*
2158	* Returns the actual frequency set for the CPU.
2159	*
2160	* If 0 is returned by the driver's ->fast_switch() callback to indicate an
2161	* error condition, the hardware configuration must be preserved.
2162	*/
2163	unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy,
2164	unsigned int target_freq)
2165	{
2166	unsigned int freq;
2167	int cpu;
2168
2169	target_freq = clamp_val(target_freq, policy->min, policy->max);
2170	freq = cpufreq_driver->fast_switch(policy, target_freq);
2171
2172	if (!freq)
2173	return 0;
2174
2175	policy->cur = freq;
2176	arch_set_freq_scale(cpus: policy->related_cpus, cur_freq: freq,
2177	max_freq: policy->cpuinfo.max_freq);
2178	cpufreq_stats_record_transition(policy, new_freq: freq);
2179
2180	if (trace_cpu_frequency_enabled()) {
2181	for_each_cpu(cpu, policy->cpus)
2182	trace_cpu_frequency(frequency: freq, cpu_id: cpu);
2183	}
2184
2185	return freq;
2186	}
2187	EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch);
2188
2189	/**
2190	* cpufreq_driver_adjust_perf - Adjust CPU performance level in one go.
2191	* @cpu: Target CPU.
2192	* @min_perf: Minimum (required) performance level (units of @capacity).
2193	* @target_perf: Target (desired) performance level (units of @capacity).
2194	* @capacity: Capacity of the target CPU.
2195	*
2196	* Carry out a fast performance level switch of @cpu without sleeping.
2197	*
2198	* The driver's ->adjust_perf() callback invoked by this function must be
2199	* suitable for being called from within RCU-sched read-side critical sections
2200	* and it is expected to select a suitable performance level equal to or above
2201	* @min_perf and preferably equal to or below @target_perf.
2202	*
2203	* This function must not be called if policy->fast_switch_enabled is unset.
2204	*
2205	* Governors calling this function must guarantee that it will never be invoked
2206	* twice in parallel for the same CPU and that it will never be called in
2207	* parallel with either ->target() or ->target_index() or ->fast_switch() for
2208	* the same CPU.
2209	*/
2210	void cpufreq_driver_adjust_perf(unsigned int cpu,
2211	unsigned long min_perf,
2212	unsigned long target_perf,
2213	unsigned long capacity)
2214	{
2215	cpufreq_driver->adjust_perf(cpu, min_perf, target_perf, capacity);
2216	}
2217
2218	/**
2219	* cpufreq_driver_has_adjust_perf - Check "direct fast switch" callback.
2220	*
2221	* Return 'true' if the ->adjust_perf callback is present for the
2222	* current driver or 'false' otherwise.
2223	*/
2224	bool cpufreq_driver_has_adjust_perf(void)
2225	{
2226	return !!cpufreq_driver->adjust_perf;
2227	}
2228
2229	/* Must set freqs->new to intermediate frequency */
2230	static int __target_intermediate(struct cpufreq_policy *policy,
2231	struct cpufreq_freqs *freqs, int index)
2232	{
2233	int ret;
2234
2235	freqs->new = cpufreq_driver->get_intermediate(policy, index);
2236
2237	/* We don't need to switch to intermediate freq */
2238	if (!freqs->new)
2239	return 0;
2240
2241	pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n",
2242	__func__, policy->cpu, freqs->old, freqs->new);
2243
2244	cpufreq_freq_transition_begin(policy, freqs);
2245	ret = cpufreq_driver->target_intermediate(policy, index);
2246	cpufreq_freq_transition_end(policy, freqs, ret);
2247
2248	if (ret)
2249	pr_err("%s: Failed to change to intermediate frequency: %d\n",
2250	__func__, ret);
2251
2252	return ret;
2253	}
2254
2255	static int __target_index(struct cpufreq_policy *policy, int index)
2256	{
2257	struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0};
2258	unsigned int restore_freq, intermediate_freq = 0;
2259	unsigned int newfreq = policy->freq_table[index].frequency;
2260	int retval = -EINVAL;
2261	bool notify;
2262
2263	if (newfreq == policy->cur)
2264	return 0;
2265
2266	/* Save last value to restore later on errors */
2267	restore_freq = policy->cur;
2268
2269	notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
2270	if (notify) {
2271	/* Handle switching to intermediate frequency */
2272	if (cpufreq_driver->get_intermediate) {
2273	retval = __target_intermediate(policy, freqs: &freqs, index);
2274	if (retval)
2275	return retval;
2276
2277	intermediate_freq = freqs.new;
2278	/* Set old freq to intermediate */
2279	if (intermediate_freq)
2280	freqs.old = freqs.new;
2281	}
2282
2283	freqs.new = newfreq;
2284	pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
2285	__func__, policy->cpu, freqs.old, freqs.new);
2286
2287	cpufreq_freq_transition_begin(policy, &freqs);
2288	}
2289
2290	retval = cpufreq_driver->target_index(policy, index);
2291	if (retval)
2292	pr_err("%s: Failed to change cpu frequency: %d\n", __func__,
2293	retval);
2294
2295	if (notify) {
2296	cpufreq_freq_transition_end(policy, &freqs, retval);
2297
2298	/*
2299	* Failed after setting to intermediate freq? Driver should have
2300	* reverted back to initial frequency and so should we. Check
2301	* here for intermediate_freq instead of get_intermediate, in
2302	* case we haven't switched to intermediate freq at all.
2303	*/
2304	if (unlikely(retval && intermediate_freq)) {
2305	freqs.old = intermediate_freq;
2306	freqs.new = restore_freq;
2307	cpufreq_freq_transition_begin(policy, &freqs);
2308	cpufreq_freq_transition_end(policy, &freqs, 0);
2309	}
2310	}
2311
2312	return retval;
2313	}
2314
2315	int __cpufreq_driver_target(struct cpufreq_policy *policy,
2316	unsigned int target_freq,
2317	unsigned int relation)
2318	{
2319	unsigned int old_target_freq = target_freq;
2320
2321	if (cpufreq_disabled())
2322	return -ENODEV;
2323
2324	target_freq = __resolve_freq(policy, target_freq, relation);
2325
2326	pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
2327	policy->cpu, target_freq, relation, old_target_freq);
2328
2329	/*
2330	* This might look like a redundant call as we are checking it again
2331	* after finding index. But it is left intentionally for cases where
2332	* exactly same freq is called again and so we can save on few function
2333	* calls.
2334	*/
2335	if (target_freq == policy->cur &&
2336	!(cpufreq_driver->flags & CPUFREQ_NEED_UPDATE_LIMITS))
2337	return 0;
2338
2339	if (cpufreq_driver->target) {
2340	/*
2341	* If the driver hasn't setup a single inefficient frequency,
2342	* it's unlikely it knows how to decode CPUFREQ_RELATION_E.
2343	*/
2344	if (!policy->efficiencies_available)
2345	relation &= ~CPUFREQ_RELATION_E;
2346
2347	return cpufreq_driver->target(policy, target_freq, relation);
2348	}
2349
2350	if (!cpufreq_driver->target_index)
2351	return -EINVAL;
2352
2353	return __target_index(policy, index: policy->cached_resolved_idx);
2354	}
2355	EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
2356
2357	int cpufreq_driver_target(struct cpufreq_policy *policy,
2358	unsigned int target_freq,
2359	unsigned int relation)
2360	{
2361	int ret;
2362
2363	down_write(sem: &policy->rwsem);
2364
2365	ret = __cpufreq_driver_target(policy, target_freq, relation);
2366
2367	up_write(sem: &policy->rwsem);
2368
2369	return ret;
2370	}
2371	EXPORT_SYMBOL_GPL(cpufreq_driver_target);
2372
2373	__weak struct cpufreq_governor *cpufreq_fallback_governor(void)
2374	{
2375	return NULL;
2376	}
2377
2378	static int cpufreq_init_governor(struct cpufreq_policy *policy)
2379	{
2380	int ret;
2381
2382	/* Don't start any governor operations if we are entering suspend */
2383	if (cpufreq_suspended)
2384	return 0;
2385	/*
2386	* Governor might not be initiated here if ACPI _PPC changed
2387	* notification happened, so check it.
2388	*/
2389	if (!policy->governor)
2390	return -EINVAL;
2391
2392	/* Platform doesn't want dynamic frequency switching ? */
2393	if (policy->governor->flags & CPUFREQ_GOV_DYNAMIC_SWITCHING &&
2394	cpufreq_driver->flags & CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING) {
2395	struct cpufreq_governor *gov = cpufreq_fallback_governor();
2396
2397	if (gov) {
2398	pr_warn("Can't use %s governor as dynamic switching is disallowed. Fallback to %s governor\n",
2399	policy->governor->name, gov->name);
2400	policy->governor = gov;
2401	} else {
2402	return -EINVAL;
2403	}
2404	}
2405
2406	if (!try_module_get(module: policy->governor->owner))
2407	return -EINVAL;
2408
2409	pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2410
2411	if (policy->governor->init) {
2412	ret = policy->governor->init(policy);
2413	if (ret) {
2414	module_put(module: policy->governor->owner);
2415	return ret;
2416	}
2417	}
2418
2419	policy->strict_target = !!(policy->governor->flags & CPUFREQ_GOV_STRICT_TARGET);
2420
2421	return 0;
2422	}
2423
2424	static void cpufreq_exit_governor(struct cpufreq_policy *policy)
2425	{
2426	if (cpufreq_suspended || !policy->governor)
2427	return;
2428
2429	pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2430
2431	if (policy->governor->exit)
2432	policy->governor->exit(policy);
2433
2434	module_put(module: policy->governor->owner);
2435	}
2436
2437	int cpufreq_start_governor(struct cpufreq_policy *policy)
2438	{
2439	int ret;
2440
2441	if (cpufreq_suspended)
2442	return 0;
2443
2444	if (!policy->governor)
2445	return -EINVAL;
2446
2447	pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2448
2449	if (cpufreq_driver->get)
2450	cpufreq_verify_current_freq(policy, update: false);
2451
2452	if (policy->governor->start) {
2453	ret = policy->governor->start(policy);
2454	if (ret)
2455	return ret;
2456	}
2457
2458	if (policy->governor->limits)
2459	policy->governor->limits(policy);
2460
2461	return 0;
2462	}
2463
2464	void cpufreq_stop_governor(struct cpufreq_policy *policy)
2465	{
2466	if (cpufreq_suspended || !policy->governor)
2467	return;
2468
2469	pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2470
2471	if (policy->governor->stop)
2472	policy->governor->stop(policy);
2473	}
2474
2475	static void cpufreq_governor_limits(struct cpufreq_policy *policy)
2476	{
2477	if (cpufreq_suspended || !policy->governor)
2478	return;
2479
2480	pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2481
2482	if (policy->governor->limits)
2483	policy->governor->limits(policy);
2484	}
2485
2486	int cpufreq_register_governor(struct cpufreq_governor *governor)
2487	{
2488	int err;
2489
2490	if (!governor)
2491	return -EINVAL;
2492
2493	if (cpufreq_disabled())
2494	return -ENODEV;
2495
2496	mutex_lock(&cpufreq_governor_mutex);
2497
2498	err = -EBUSY;
2499	if (!find_governor(str_governor: governor->name)) {
2500	err = 0;
2501	list_add(new: &governor->governor_list, head: &cpufreq_governor_list);
2502	}
2503
2504	mutex_unlock(lock: &cpufreq_governor_mutex);
2505	return err;
2506	}
2507	EXPORT_SYMBOL_GPL(cpufreq_register_governor);
2508
2509	void cpufreq_unregister_governor(struct cpufreq_governor *governor)
2510	{
2511	struct cpufreq_policy *policy;
2512	unsigned long flags;
2513
2514	if (!governor)
2515	return;
2516
2517	if (cpufreq_disabled())
2518	return;
2519
2520	/* clear last_governor for all inactive policies */
2521	read_lock_irqsave(&cpufreq_driver_lock, flags);
2522	for_each_inactive_policy(policy) {
2523	if (!strcmp(policy->last_governor, governor->name)) {
2524	policy->governor = NULL;
2525	strcpy(p: policy->last_governor, q: "\0");
2526	}
2527	}
2528	read_unlock_irqrestore(&cpufreq_driver_lock, flags);
2529
2530	mutex_lock(&cpufreq_governor_mutex);
2531	list_del(entry: &governor->governor_list);
2532	mutex_unlock(lock: &cpufreq_governor_mutex);
2533	}
2534	EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
2535
2536
2537	/*********************************************************************
2538	* POLICY INTERFACE *
2539	*********************************************************************/
2540
2541	/**
2542	* cpufreq_get_policy - get the current cpufreq_policy
2543	* @policy: struct cpufreq_policy into which the current cpufreq_policy
2544	* is written
2545	* @cpu: CPU to find the policy for
2546	*
2547	* Reads the current cpufreq policy.
2548	*/
2549	int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
2550	{
2551	struct cpufreq_policy *cpu_policy;
2552	if (!policy)
2553	return -EINVAL;
2554
2555	cpu_policy = cpufreq_cpu_get(cpu);
2556	if (!cpu_policy)
2557	return -EINVAL;
2558
2559	memcpy(policy, cpu_policy, sizeof(*policy));
2560
2561	cpufreq_cpu_put(cpu_policy);
2562	return 0;
2563	}
2564	EXPORT_SYMBOL(cpufreq_get_policy);
2565
2566	/**
2567	* cpufreq_set_policy - Modify cpufreq policy parameters.
2568	* @policy: Policy object to modify.
2569	* @new_gov: Policy governor pointer.
2570	* @new_pol: Policy value (for drivers with built-in governors).
2571	*
2572	* Invoke the cpufreq driver's ->verify() callback to sanity-check the frequency
2573	* limits to be set for the policy, update @policy with the verified limits
2574	* values and either invoke the driver's ->setpolicy() callback (if present) or
2575	* carry out a governor update for @policy. That is, run the current governor's
2576	* ->limits() callback (if @new_gov points to the same object as the one in
2577	* @policy) or replace the governor for @policy with @new_gov.
2578	*
2579	* The cpuinfo part of @policy is not updated by this function.
2580	*/
2581	static int cpufreq_set_policy(struct cpufreq_policy *policy,
2582	struct cpufreq_governor *new_gov,
2583	unsigned int new_pol)
2584	{
2585	struct cpufreq_policy_data new_data;
2586	struct cpufreq_governor *old_gov;
2587	int ret;
2588
2589	memcpy(&new_data.cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
2590	new_data.freq_table = policy->freq_table;
2591	new_data.cpu = policy->cpu;
2592	/*
2593	* PM QoS framework collects all the requests from users and provide us
2594	* the final aggregated value here.
2595	*/
2596	new_data.min = freq_qos_read_value(qos: &policy->constraints, type: FREQ_QOS_MIN);
2597	new_data.max = freq_qos_read_value(qos: &policy->constraints, type: FREQ_QOS_MAX);
2598
2599	pr_debug("setting new policy for CPU %u: %u - %u kHz\n",
2600	new_data.cpu, new_data.min, new_data.max);
2601
2602	/*
2603	* Verify that the CPU speed can be set within these limits and make sure
2604	* that min <= max.
2605	*/
2606	ret = cpufreq_driver->verify(&new_data);
2607	if (ret)
2608	return ret;
2609
2610	/*
2611	* Resolve policy min/max to available frequencies. It ensures
2612	* no frequency resolution will neither overshoot the requested maximum
2613	* nor undershoot the requested minimum.
2614	*/
2615	policy->min = new_data.min;
2616	policy->max = new_data.max;
2617	policy->min = __resolve_freq(policy, target_freq: policy->min, CPUFREQ_RELATION_L);
2618	policy->max = __resolve_freq(policy, target_freq: policy->max, CPUFREQ_RELATION_H);
2619	trace_cpu_frequency_limits(policy);
2620
2621	policy->cached_target_freq = UINT_MAX;
2622
2623	pr_debug("new min and max freqs are %u - %u kHz\n",
2624	policy->min, policy->max);
2625
2626	if (cpufreq_driver->setpolicy) {
2627	policy->policy = new_pol;
2628	pr_debug("setting range\n");
2629	return cpufreq_driver->setpolicy(policy);
2630	}
2631
2632	if (new_gov == policy->governor) {
2633	pr_debug("governor limits update\n");
2634	cpufreq_governor_limits(policy);
2635	return 0;
2636	}
2637
2638	pr_debug("governor switch\n");
2639
2640	/* save old, working values */
2641	old_gov = policy->governor;
2642	/* end old governor */
2643	if (old_gov) {
2644	cpufreq_stop_governor(policy);
2645	cpufreq_exit_governor(policy);
2646	}
2647
2648	/* start new governor */
2649	policy->governor = new_gov;
2650	ret = cpufreq_init_governor(policy);
2651	if (!ret) {
2652	ret = cpufreq_start_governor(policy);
2653	if (!ret) {
2654	pr_debug("governor change\n");
2655	return 0;
2656	}
2657	cpufreq_exit_governor(policy);
2658	}
2659
2660	/* new governor failed, so re-start old one */
2661	pr_debug("starting governor %s failed\n", policy->governor->name);
2662	if (old_gov) {
2663	policy->governor = old_gov;
2664	if (cpufreq_init_governor(policy))
2665	policy->governor = NULL;
2666	else
2667	cpufreq_start_governor(policy);
2668	}
2669
2670	return ret;
2671	}
2672
2673	/**
2674	* cpufreq_update_policy - Re-evaluate an existing cpufreq policy.
2675	* @cpu: CPU to re-evaluate the policy for.
2676	*
2677	* Update the current frequency for the cpufreq policy of @cpu and use
2678	* cpufreq_set_policy() to re-apply the min and max limits, which triggers the
2679	* evaluation of policy notifiers and the cpufreq driver's ->verify() callback
2680	* for the policy in question, among other things.
2681	*/
2682	void cpufreq_update_policy(unsigned int cpu)
2683	{
2684	struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu);
2685
2686	if (!policy)
2687	return;
2688
2689	/*
2690	* BIOS might change freq behind our back
2691	* -> ask driver for current freq and notify governors about a change
2692	*/
2693	if (cpufreq_driver->get && has_target() &&
2694	(cpufreq_suspended || WARN_ON(!cpufreq_verify_current_freq(policy, false))))
2695	goto unlock;
2696
2697	refresh_frequency_limits(policy);
2698
2699	unlock:
2700	cpufreq_cpu_release(policy);
2701	}
2702	EXPORT_SYMBOL(cpufreq_update_policy);
2703
2704	/**
2705	* cpufreq_update_limits - Update policy limits for a given CPU.
2706	* @cpu: CPU to update the policy limits for.
2707	*
2708	* Invoke the driver's ->update_limits callback if present or call
2709	* cpufreq_update_policy() for @cpu.
2710	*/
2711	void cpufreq_update_limits(unsigned int cpu)
2712	{
2713	if (cpufreq_driver->update_limits)
2714	cpufreq_driver->update_limits(cpu);
2715	else
2716	cpufreq_update_policy(cpu);
2717	}
2718	EXPORT_SYMBOL_GPL(cpufreq_update_limits);
2719
2720	/*********************************************************************
2721	* BOOST *
2722	*********************************************************************/
2723	static int cpufreq_boost_set_sw(struct cpufreq_policy *policy, int state)
2724	{
2725	int ret;
2726
2727	if (!policy->freq_table)
2728	return -ENXIO;
2729
2730	ret = cpufreq_frequency_table_cpuinfo(policy, table: policy->freq_table);
2731	if (ret) {
2732	pr_err("%s: Policy frequency update failed\n", __func__);
2733	return ret;
2734	}
2735
2736	ret = freq_qos_update_request(req: policy->max_freq_req, new_value: policy->max);
2737	if (ret < 0)
2738	return ret;
2739
2740	return 0;
2741	}
2742
2743	int cpufreq_boost_trigger_state(int state)
2744	{
2745	struct cpufreq_policy *policy;
2746	unsigned long flags;
2747	int ret = 0;
2748
2749	if (cpufreq_driver->boost_enabled == state)
2750	return 0;
2751
2752	write_lock_irqsave(&cpufreq_driver_lock, flags);
2753	cpufreq_driver->boost_enabled = state;
2754	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2755
2756	cpus_read_lock();
2757	for_each_active_policy(policy) {
2758	ret = cpufreq_driver->set_boost(policy, state);
2759	if (ret)
2760	goto err_reset_state;
2761
2762	policy->boost_enabled = state;
2763	}
2764	cpus_read_unlock();
2765
2766	return 0;
2767
2768	err_reset_state:
2769	cpus_read_unlock();
2770
2771	write_lock_irqsave(&cpufreq_driver_lock, flags);
2772	cpufreq_driver->boost_enabled = !state;
2773	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2774
2775	pr_err("%s: Cannot %s BOOST\n",
2776	__func__, state ? "enable" : "disable");
2777
2778	return ret;
2779	}
2780
2781	static bool cpufreq_boost_supported(void)
2782	{
2783	return cpufreq_driver->set_boost;
2784	}
2785
2786	static int create_boost_sysfs_file(void)
2787	{
2788	int ret;
2789
2790	ret = sysfs_create_file(kobj: cpufreq_global_kobject, attr: &boost.attr);
2791	if (ret)
2792	pr_err("%s: cannot register global BOOST sysfs file\n",
2793	__func__);
2794
2795	return ret;
2796	}
2797
2798	static void remove_boost_sysfs_file(void)
2799	{
2800	if (cpufreq_boost_supported())
2801	sysfs_remove_file(kobj: cpufreq_global_kobject, attr: &boost.attr);
2802	}
2803
2804	int cpufreq_enable_boost_support(void)
2805	{
2806	if (!cpufreq_driver)
2807	return -EINVAL;
2808
2809	if (cpufreq_boost_supported())
2810	return 0;
2811
2812	cpufreq_driver->set_boost = cpufreq_boost_set_sw;
2813
2814	/* This will get removed on driver unregister */
2815	return create_boost_sysfs_file();
2816	}
2817	EXPORT_SYMBOL_GPL(cpufreq_enable_boost_support);
2818
2819	int cpufreq_boost_enabled(void)
2820	{
2821	return cpufreq_driver->boost_enabled;
2822	}
2823	EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);
2824
2825	/*********************************************************************
2826	* REGISTER / UNREGISTER CPUFREQ DRIVER *
2827	*********************************************************************/
2828	static enum cpuhp_state hp_online;
2829
2830	static int cpuhp_cpufreq_online(unsigned int cpu)
2831	{
2832	cpufreq_online(cpu);
2833
2834	return 0;
2835	}
2836
2837	static int cpuhp_cpufreq_offline(unsigned int cpu)
2838	{
2839	cpufreq_offline(cpu);
2840
2841	return 0;
2842	}
2843
2844	/**
2845	* cpufreq_register_driver - register a CPU Frequency driver
2846	* @driver_data: A struct cpufreq_driver containing the values#
2847	* submitted by the CPU Frequency driver.
2848	*
2849	* Registers a CPU Frequency driver to this core code. This code
2850	* returns zero on success, -EEXIST when another driver got here first
2851	* (and isn't unregistered in the meantime).
2852	*
2853	*/
2854	int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2855	{
2856	unsigned long flags;
2857	int ret;
2858
2859	if (cpufreq_disabled())
2860	return -ENODEV;
2861
2862	/*
2863	* The cpufreq core depends heavily on the availability of device
2864	* structure, make sure they are available before proceeding further.
2865	*/
2866	if (!get_cpu_device(cpu: 0))
2867	return -EPROBE_DEFER;
2868
2869	if (!driver_data || !driver_data->verify || !driver_data->init ||
2870	!(driver_data->setpolicy || driver_data->target_index ||
2871	driver_data->target) ||
2872	(driver_data->setpolicy && (driver_data->target_index ||
2873	driver_data->target)) ||
2874	(!driver_data->get_intermediate != !driver_data->target_intermediate) ||
2875	(!driver_data->online != !driver_data->offline) ||
2876	(driver_data->adjust_perf && !driver_data->fast_switch))
2877	return -EINVAL;
2878
2879	pr_debug("trying to register driver %s\n", driver_data->name);
2880
2881	/* Protect against concurrent CPU online/offline. */
2882	cpus_read_lock();
2883
2884	write_lock_irqsave(&cpufreq_driver_lock, flags);
2885	if (cpufreq_driver) {
2886	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2887	ret = -EEXIST;
2888	goto out;
2889	}
2890	cpufreq_driver = driver_data;
2891	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2892
2893	/*
2894	* Mark support for the scheduler's frequency invariance engine for
2895	* drivers that implement target(), target_index() or fast_switch().
2896	*/
2897	if (!cpufreq_driver->setpolicy) {
2898	static_branch_enable_cpuslocked(&cpufreq_freq_invariance);
2899	pr_debug("supports frequency invariance");
2900	}
2901
2902	if (driver_data->setpolicy)
2903	driver_data->flags |= CPUFREQ_CONST_LOOPS;
2904
2905	if (cpufreq_boost_supported()) {
2906	ret = create_boost_sysfs_file();
2907	if (ret)
2908	goto err_null_driver;
2909	}
2910
2911	ret = subsys_interface_register(sif: &cpufreq_interface);
2912	if (ret)
2913	goto err_boost_unreg;
2914
2915	if (unlikely(list_empty(&cpufreq_policy_list))) {
2916	/* if all ->init() calls failed, unregister */
2917	ret = -ENODEV;
2918	pr_debug("%s: No CPU initialized for driver %s\n", __func__,
2919	driver_data->name);
2920	goto err_if_unreg;
2921	}
2922
2923	ret = cpuhp_setup_state_nocalls_cpuslocked(state: CPUHP_AP_ONLINE_DYN,
2924	name: "cpufreq:online",
2925	startup: cpuhp_cpufreq_online,
2926	teardown: cpuhp_cpufreq_offline);
2927	if (ret < 0)
2928	goto err_if_unreg;
2929	hp_online = ret;
2930	ret = 0;
2931
2932	pr_debug("driver %s up and running\n", driver_data->name);
2933	goto out;
2934
2935	err_if_unreg:
2936	subsys_interface_unregister(sif: &cpufreq_interface);
2937	err_boost_unreg:
2938	remove_boost_sysfs_file();
2939	err_null_driver:
2940	write_lock_irqsave(&cpufreq_driver_lock, flags);
2941	cpufreq_driver = NULL;
2942	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2943	out:
2944	cpus_read_unlock();
2945	return ret;
2946	}
2947	EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2948
2949	/*
2950	* cpufreq_unregister_driver - unregister the current CPUFreq driver
2951	*
2952	* Unregister the current CPUFreq driver. Only call this if you have
2953	* the right to do so, i.e. if you have succeeded in initialising before!
2954	* Returns zero if successful, and -EINVAL if the cpufreq_driver is
2955	* currently not initialised.
2956	*/
2957	void cpufreq_unregister_driver(struct cpufreq_driver *driver)
2958	{
2959	unsigned long flags;
2960
2961	if (WARN_ON(!cpufreq_driver || (driver != cpufreq_driver)))
2962	return;
2963
2964	pr_debug("unregistering driver %s\n", driver->name);
2965
2966	/* Protect against concurrent cpu hotplug */
2967	cpus_read_lock();
2968	subsys_interface_unregister(sif: &cpufreq_interface);
2969	remove_boost_sysfs_file();
2970	static_branch_disable_cpuslocked(&cpufreq_freq_invariance);
2971	cpuhp_remove_state_nocalls_cpuslocked(state: hp_online);
2972
2973	write_lock_irqsave(&cpufreq_driver_lock, flags);
2974
2975	cpufreq_driver = NULL;
2976
2977	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2978	cpus_read_unlock();
2979	}
2980	EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2981
2982	static int __init cpufreq_core_init(void)
2983	{
2984	struct cpufreq_governor *gov = cpufreq_default_governor();
2985	struct device *dev_root;
2986
2987	if (cpufreq_disabled())
2988	return -ENODEV;
2989
2990	dev_root = bus_get_dev_root(bus: &cpu_subsys);
2991	if (dev_root) {
2992	cpufreq_global_kobject = kobject_create_and_add(name: "cpufreq", parent: &dev_root->kobj);
2993	put_device(dev: dev_root);
2994	}
2995	BUG_ON(!cpufreq_global_kobject);
2996
2997	if (!strlen(default_governor))
2998	strscpy(p: default_governor, q: gov->name, CPUFREQ_NAME_LEN);
2999
3000	return 0;
3001	}
3002	module_param(off, int, 0444);
3003	module_param_string(default_governor, default_governor, CPUFREQ_NAME_LEN, 0444);
3004	core_initcall(cpufreq_core_init);
3005