1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Energy Model of devices
4 *
5 * Copyright (c) 2018-2021, Arm ltd.
6 * Written by: Quentin Perret, Arm ltd.
7 * Improvements provided by: Lukasz Luba, Arm ltd.
8 */
9
10#define pr_fmt(fmt) "energy_model: " fmt
11
12#include <linux/cpu.h>
13#include <linux/cpufreq.h>
14#include <linux/cpumask.h>
15#include <linux/debugfs.h>
16#include <linux/energy_model.h>
17#include <linux/sched/topology.h>
18#include <linux/slab.h>
19
20/*
21 * Mutex serializing the registrations of performance domains and letting
22 * callbacks defined by drivers sleep.
23 */
24static DEFINE_MUTEX(em_pd_mutex);
25
26static bool _is_cpu_device(struct device *dev)
27{
28 return (dev->bus == &cpu_subsys);
29}
30
31#ifdef CONFIG_DEBUG_FS
32static struct dentry *rootdir;
33
34static void em_debug_create_ps(struct em_perf_state *ps, struct dentry *pd)
35{
36 struct dentry *d;
37 char name[24];
38
39 snprintf(buf: name, size: sizeof(name), fmt: "ps:%lu", ps->frequency);
40
41 /* Create per-ps directory */
42 d = debugfs_create_dir(name, parent: pd);
43 debugfs_create_ulong(name: "frequency", mode: 0444, parent: d, value: &ps->frequency);
44 debugfs_create_ulong(name: "power", mode: 0444, parent: d, value: &ps->power);
45 debugfs_create_ulong(name: "cost", mode: 0444, parent: d, value: &ps->cost);
46 debugfs_create_ulong(name: "inefficient", mode: 0444, parent: d, value: &ps->flags);
47}
48
49static int em_debug_cpus_show(struct seq_file *s, void *unused)
50{
51 seq_printf(m: s, fmt: "%*pbl\n", cpumask_pr_args(to_cpumask(s->private)));
52
53 return 0;
54}
55DEFINE_SHOW_ATTRIBUTE(em_debug_cpus);
56
57static int em_debug_flags_show(struct seq_file *s, void *unused)
58{
59 struct em_perf_domain *pd = s->private;
60
61 seq_printf(m: s, fmt: "%#lx\n", pd->flags);
62
63 return 0;
64}
65DEFINE_SHOW_ATTRIBUTE(em_debug_flags);
66
67static void em_debug_create_pd(struct device *dev)
68{
69 struct dentry *d;
70 int i;
71
72 /* Create the directory of the performance domain */
73 d = debugfs_create_dir(name: dev_name(dev), parent: rootdir);
74
75 if (_is_cpu_device(dev))
76 debugfs_create_file(name: "cpus", mode: 0444, parent: d, data: dev->em_pd->cpus,
77 fops: &em_debug_cpus_fops);
78
79 debugfs_create_file(name: "flags", mode: 0444, parent: d, data: dev->em_pd,
80 fops: &em_debug_flags_fops);
81
82 /* Create a sub-directory for each performance state */
83 for (i = 0; i < dev->em_pd->nr_perf_states; i++)
84 em_debug_create_ps(ps: &dev->em_pd->table[i], pd: d);
85
86}
87
88static void em_debug_remove_pd(struct device *dev)
89{
90 debugfs_lookup_and_remove(name: dev_name(dev), parent: rootdir);
91}
92
93static int __init em_debug_init(void)
94{
95 /* Create /sys/kernel/debug/energy_model directory */
96 rootdir = debugfs_create_dir(name: "energy_model", NULL);
97
98 return 0;
99}
100fs_initcall(em_debug_init);
101#else /* CONFIG_DEBUG_FS */
102static void em_debug_create_pd(struct device *dev) {}
103static void em_debug_remove_pd(struct device *dev) {}
104#endif
105
106static int em_create_perf_table(struct device *dev, struct em_perf_domain *pd,
107 int nr_states, struct em_data_callback *cb,
108 unsigned long flags)
109{
110 unsigned long power, freq, prev_freq = 0, prev_cost = ULONG_MAX;
111 struct em_perf_state *table;
112 int i, ret;
113 u64 fmax;
114
115 table = kcalloc(n: nr_states, size: sizeof(*table), GFP_KERNEL);
116 if (!table)
117 return -ENOMEM;
118
119 /* Build the list of performance states for this performance domain */
120 for (i = 0, freq = 0; i < nr_states; i++, freq++) {
121 /*
122 * active_power() is a driver callback which ceils 'freq' to
123 * lowest performance state of 'dev' above 'freq' and updates
124 * 'power' and 'freq' accordingly.
125 */
126 ret = cb->active_power(dev, &power, &freq);
127 if (ret) {
128 dev_err(dev, "EM: invalid perf. state: %d\n",
129 ret);
130 goto free_ps_table;
131 }
132
133 /*
134 * We expect the driver callback to increase the frequency for
135 * higher performance states.
136 */
137 if (freq <= prev_freq) {
138 dev_err(dev, "EM: non-increasing freq: %lu\n",
139 freq);
140 goto free_ps_table;
141 }
142
143 /*
144 * The power returned by active_state() is expected to be
145 * positive and be in range.
146 */
147 if (!power || power > EM_MAX_POWER) {
148 dev_err(dev, "EM: invalid power: %lu\n",
149 power);
150 goto free_ps_table;
151 }
152
153 table[i].power = power;
154 table[i].frequency = prev_freq = freq;
155 }
156
157 /* Compute the cost of each performance state. */
158 fmax = (u64) table[nr_states - 1].frequency;
159 for (i = nr_states - 1; i >= 0; i--) {
160 unsigned long power_res, cost;
161
162 if (flags & EM_PERF_DOMAIN_ARTIFICIAL) {
163 ret = cb->get_cost(dev, table[i].frequency, &cost);
164 if (ret || !cost || cost > EM_MAX_POWER) {
165 dev_err(dev, "EM: invalid cost %lu %d\n",
166 cost, ret);
167 goto free_ps_table;
168 }
169 } else {
170 power_res = table[i].power;
171 cost = div64_u64(dividend: fmax * power_res, divisor: table[i].frequency);
172 }
173
174 table[i].cost = cost;
175
176 if (table[i].cost >= prev_cost) {
177 table[i].flags = EM_PERF_STATE_INEFFICIENT;
178 dev_dbg(dev, "EM: OPP:%lu is inefficient\n",
179 table[i].frequency);
180 } else {
181 prev_cost = table[i].cost;
182 }
183 }
184
185 pd->table = table;
186 pd->nr_perf_states = nr_states;
187
188 return 0;
189
190free_ps_table:
191 kfree(objp: table);
192 return -EINVAL;
193}
194
195static int em_create_pd(struct device *dev, int nr_states,
196 struct em_data_callback *cb, cpumask_t *cpus,
197 unsigned long flags)
198{
199 struct em_perf_domain *pd;
200 struct device *cpu_dev;
201 int cpu, ret, num_cpus;
202
203 if (_is_cpu_device(dev)) {
204 num_cpus = cpumask_weight(srcp: cpus);
205
206 /* Prevent max possible energy calculation to not overflow */
207 if (num_cpus > EM_MAX_NUM_CPUS) {
208 dev_err(dev, "EM: too many CPUs, overflow possible\n");
209 return -EINVAL;
210 }
211
212 pd = kzalloc(size: sizeof(*pd) + cpumask_size(), GFP_KERNEL);
213 if (!pd)
214 return -ENOMEM;
215
216 cpumask_copy(em_span_cpus(pd), srcp: cpus);
217 } else {
218 pd = kzalloc(size: sizeof(*pd), GFP_KERNEL);
219 if (!pd)
220 return -ENOMEM;
221 }
222
223 ret = em_create_perf_table(dev, pd, nr_states, cb, flags);
224 if (ret) {
225 kfree(objp: pd);
226 return ret;
227 }
228
229 if (_is_cpu_device(dev))
230 for_each_cpu(cpu, cpus) {
231 cpu_dev = get_cpu_device(cpu);
232 cpu_dev->em_pd = pd;
233 }
234
235 dev->em_pd = pd;
236
237 return 0;
238}
239
240static void em_cpufreq_update_efficiencies(struct device *dev)
241{
242 struct em_perf_domain *pd = dev->em_pd;
243 struct em_perf_state *table;
244 struct cpufreq_policy *policy;
245 int found = 0;
246 int i;
247
248 if (!_is_cpu_device(dev) || !pd)
249 return;
250
251 policy = cpufreq_cpu_get(cpu: cpumask_first(em_span_cpus(pd)));
252 if (!policy) {
253 dev_warn(dev, "EM: Access to CPUFreq policy failed");
254 return;
255 }
256
257 table = pd->table;
258
259 for (i = 0; i < pd->nr_perf_states; i++) {
260 if (!(table[i].flags & EM_PERF_STATE_INEFFICIENT))
261 continue;
262
263 if (!cpufreq_table_set_inefficient(policy, frequency: table[i].frequency))
264 found++;
265 }
266
267 cpufreq_cpu_put(policy);
268
269 if (!found)
270 return;
271
272 /*
273 * Efficiencies have been installed in CPUFreq, inefficient frequencies
274 * will be skipped. The EM can do the same.
275 */
276 pd->flags |= EM_PERF_DOMAIN_SKIP_INEFFICIENCIES;
277}
278
279/**
280 * em_pd_get() - Return the performance domain for a device
281 * @dev : Device to find the performance domain for
282 *
283 * Returns the performance domain to which @dev belongs, or NULL if it doesn't
284 * exist.
285 */
286struct em_perf_domain *em_pd_get(struct device *dev)
287{
288 if (IS_ERR_OR_NULL(ptr: dev))
289 return NULL;
290
291 return dev->em_pd;
292}
293EXPORT_SYMBOL_GPL(em_pd_get);
294
295/**
296 * em_cpu_get() - Return the performance domain for a CPU
297 * @cpu : CPU to find the performance domain for
298 *
299 * Returns the performance domain to which @cpu belongs, or NULL if it doesn't
300 * exist.
301 */
302struct em_perf_domain *em_cpu_get(int cpu)
303{
304 struct device *cpu_dev;
305
306 cpu_dev = get_cpu_device(cpu);
307 if (!cpu_dev)
308 return NULL;
309
310 return em_pd_get(cpu_dev);
311}
312EXPORT_SYMBOL_GPL(em_cpu_get);
313
314/**
315 * em_dev_register_perf_domain() - Register the Energy Model (EM) for a device
316 * @dev : Device for which the EM is to register
317 * @nr_states : Number of performance states to register
318 * @cb : Callback functions providing the data of the Energy Model
319 * @cpus : Pointer to cpumask_t, which in case of a CPU device is
320 * obligatory. It can be taken from i.e. 'policy->cpus'. For other
321 * type of devices this should be set to NULL.
322 * @microwatts : Flag indicating that the power values are in micro-Watts or
323 * in some other scale. It must be set properly.
324 *
325 * Create Energy Model tables for a performance domain using the callbacks
326 * defined in cb.
327 *
328 * The @microwatts is important to set with correct value. Some kernel
329 * sub-systems might rely on this flag and check if all devices in the EM are
330 * using the same scale.
331 *
332 * If multiple clients register the same performance domain, all but the first
333 * registration will be ignored.
334 *
335 * Return 0 on success
336 */
337int em_dev_register_perf_domain(struct device *dev, unsigned int nr_states,
338 struct em_data_callback *cb, cpumask_t *cpus,
339 bool microwatts)
340{
341 unsigned long cap, prev_cap = 0;
342 unsigned long flags = 0;
343 int cpu, ret;
344
345 if (!dev || !nr_states || !cb)
346 return -EINVAL;
347
348 /*
349 * Use a mutex to serialize the registration of performance domains and
350 * let the driver-defined callback functions sleep.
351 */
352 mutex_lock(&em_pd_mutex);
353
354 if (dev->em_pd) {
355 ret = -EEXIST;
356 goto unlock;
357 }
358
359 if (_is_cpu_device(dev)) {
360 if (!cpus) {
361 dev_err(dev, "EM: invalid CPU mask\n");
362 ret = -EINVAL;
363 goto unlock;
364 }
365
366 for_each_cpu(cpu, cpus) {
367 if (em_cpu_get(cpu)) {
368 dev_err(dev, "EM: exists for CPU%d\n", cpu);
369 ret = -EEXIST;
370 goto unlock;
371 }
372 /*
373 * All CPUs of a domain must have the same
374 * micro-architecture since they all share the same
375 * table.
376 */
377 cap = arch_scale_cpu_capacity(cpu);
378 if (prev_cap && prev_cap != cap) {
379 dev_err(dev, "EM: CPUs of %*pbl must have the same capacity\n",
380 cpumask_pr_args(cpus));
381
382 ret = -EINVAL;
383 goto unlock;
384 }
385 prev_cap = cap;
386 }
387 }
388
389 if (microwatts)
390 flags |= EM_PERF_DOMAIN_MICROWATTS;
391 else if (cb->get_cost)
392 flags |= EM_PERF_DOMAIN_ARTIFICIAL;
393
394 ret = em_create_pd(dev, nr_states, cb, cpus, flags);
395 if (ret)
396 goto unlock;
397
398 dev->em_pd->flags |= flags;
399
400 em_cpufreq_update_efficiencies(dev);
401
402 em_debug_create_pd(dev);
403 dev_info(dev, "EM: created perf domain\n");
404
405unlock:
406 mutex_unlock(lock: &em_pd_mutex);
407 return ret;
408}
409EXPORT_SYMBOL_GPL(em_dev_register_perf_domain);
410
411/**
412 * em_dev_unregister_perf_domain() - Unregister Energy Model (EM) for a device
413 * @dev : Device for which the EM is registered
414 *
415 * Unregister the EM for the specified @dev (but not a CPU device).
416 */
417void em_dev_unregister_perf_domain(struct device *dev)
418{
419 if (IS_ERR_OR_NULL(ptr: dev) || !dev->em_pd)
420 return;
421
422 if (_is_cpu_device(dev))
423 return;
424
425 /*
426 * The mutex separates all register/unregister requests and protects
427 * from potential clean-up/setup issues in the debugfs directories.
428 * The debugfs directory name is the same as device's name.
429 */
430 mutex_lock(&em_pd_mutex);
431 em_debug_remove_pd(dev);
432
433 kfree(objp: dev->em_pd->table);
434 kfree(objp: dev->em_pd);
435 dev->em_pd = NULL;
436 mutex_unlock(lock: &em_pd_mutex);
437}
438EXPORT_SYMBOL_GPL(em_dev_unregister_perf_domain);
439

source code of linux/kernel/power/energy_model.c