1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * CPPC (Collaborative Processor Performance Control) driver for |
4 | * interfacing with the CPUfreq layer and governors. See |
5 | * cppc_acpi.c for CPPC specific methods. |
6 | * |
7 | * (C) Copyright 2014, 2015 Linaro Ltd. |
8 | * Author: Ashwin Chaugule <ashwin.chaugule@linaro.org> |
9 | */ |
10 | |
11 | #define pr_fmt(fmt) "CPPC Cpufreq:" fmt |
12 | |
13 | #include <linux/arch_topology.h> |
14 | #include <linux/kernel.h> |
15 | #include <linux/module.h> |
16 | #include <linux/delay.h> |
17 | #include <linux/cpu.h> |
18 | #include <linux/cpufreq.h> |
19 | #include <linux/dmi.h> |
20 | #include <linux/irq_work.h> |
21 | #include <linux/kthread.h> |
22 | #include <linux/time.h> |
23 | #include <linux/vmalloc.h> |
24 | #include <uapi/linux/sched/types.h> |
25 | |
26 | #include <asm/unaligned.h> |
27 | |
28 | #include <acpi/cppc_acpi.h> |
29 | |
30 | /* Minimum struct length needed for the DMI processor entry we want */ |
31 | #define DMI_ENTRY_PROCESSOR_MIN_LENGTH 48 |
32 | |
33 | /* Offset in the DMI processor structure for the max frequency */ |
34 | #define DMI_PROCESSOR_MAX_SPEED 0x14 |
35 | |
36 | /* |
37 | * This list contains information parsed from per CPU ACPI _CPC and _PSD |
38 | * structures: e.g. the highest and lowest supported performance, capabilities, |
39 | * desired performance, level requested etc. Depending on the share_type, not |
40 | * all CPUs will have an entry in the list. |
41 | */ |
42 | static LIST_HEAD(cpu_data_list); |
43 | |
44 | static bool boost_supported; |
45 | |
46 | struct cppc_workaround_oem_info { |
47 | char oem_id[ACPI_OEM_ID_SIZE + 1]; |
48 | char oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1]; |
49 | u32 oem_revision; |
50 | }; |
51 | |
52 | static struct cppc_workaround_oem_info wa_info[] = { |
53 | { |
54 | .oem_id = "HISI " , |
55 | .oem_table_id = "HIP07 " , |
56 | .oem_revision = 0, |
57 | }, { |
58 | .oem_id = "HISI " , |
59 | .oem_table_id = "HIP08 " , |
60 | .oem_revision = 0, |
61 | } |
62 | }; |
63 | |
64 | static struct cpufreq_driver cppc_cpufreq_driver; |
65 | |
66 | static enum { |
67 | FIE_UNSET = -1, |
68 | FIE_ENABLED, |
69 | FIE_DISABLED |
70 | } fie_disabled = FIE_UNSET; |
71 | |
72 | #ifdef CONFIG_ACPI_CPPC_CPUFREQ_FIE |
73 | module_param(fie_disabled, int, 0444); |
74 | MODULE_PARM_DESC(fie_disabled, "Disable Frequency Invariance Engine (FIE)" ); |
75 | |
76 | /* Frequency invariance support */ |
77 | struct cppc_freq_invariance { |
78 | int cpu; |
79 | struct irq_work irq_work; |
80 | struct kthread_work work; |
81 | struct cppc_perf_fb_ctrs prev_perf_fb_ctrs; |
82 | struct cppc_cpudata *cpu_data; |
83 | }; |
84 | |
85 | static DEFINE_PER_CPU(struct cppc_freq_invariance, cppc_freq_inv); |
86 | static struct kthread_worker *kworker_fie; |
87 | |
88 | static unsigned int hisi_cppc_cpufreq_get_rate(unsigned int cpu); |
89 | static int cppc_perf_from_fbctrs(struct cppc_cpudata *cpu_data, |
90 | struct cppc_perf_fb_ctrs *fb_ctrs_t0, |
91 | struct cppc_perf_fb_ctrs *fb_ctrs_t1); |
92 | |
93 | /** |
94 | * cppc_scale_freq_workfn - CPPC arch_freq_scale updater for frequency invariance |
95 | * @work: The work item. |
96 | * |
97 | * The CPPC driver register itself with the topology core to provide its own |
98 | * implementation (cppc_scale_freq_tick()) of topology_scale_freq_tick() which |
99 | * gets called by the scheduler on every tick. |
100 | * |
101 | * Note that the arch specific counters have higher priority than CPPC counters, |
102 | * if available, though the CPPC driver doesn't need to have any special |
103 | * handling for that. |
104 | * |
105 | * On an invocation of cppc_scale_freq_tick(), we schedule an irq work (since we |
106 | * reach here from hard-irq context), which then schedules a normal work item |
107 | * and cppc_scale_freq_workfn() updates the per_cpu arch_freq_scale variable |
108 | * based on the counter updates since the last tick. |
109 | */ |
110 | static void cppc_scale_freq_workfn(struct kthread_work *work) |
111 | { |
112 | struct cppc_freq_invariance *cppc_fi; |
113 | struct cppc_perf_fb_ctrs fb_ctrs = {0}; |
114 | struct cppc_cpudata *cpu_data; |
115 | unsigned long local_freq_scale; |
116 | u64 perf; |
117 | |
118 | cppc_fi = container_of(work, struct cppc_freq_invariance, work); |
119 | cpu_data = cppc_fi->cpu_data; |
120 | |
121 | if (cppc_get_perf_ctrs(cppc_fi->cpu, &fb_ctrs)) { |
122 | pr_warn("%s: failed to read perf counters\n" , __func__); |
123 | return; |
124 | } |
125 | |
126 | perf = cppc_perf_from_fbctrs(cpu_data, &cppc_fi->prev_perf_fb_ctrs, |
127 | &fb_ctrs); |
128 | cppc_fi->prev_perf_fb_ctrs = fb_ctrs; |
129 | |
130 | perf <<= SCHED_CAPACITY_SHIFT; |
131 | local_freq_scale = div64_u64(perf, cpu_data->perf_caps.highest_perf); |
132 | |
133 | /* This can happen due to counter's overflow */ |
134 | if (unlikely(local_freq_scale > 1024)) |
135 | local_freq_scale = 1024; |
136 | |
137 | per_cpu(arch_freq_scale, cppc_fi->cpu) = local_freq_scale; |
138 | } |
139 | |
140 | static void cppc_irq_work(struct irq_work *irq_work) |
141 | { |
142 | struct cppc_freq_invariance *cppc_fi; |
143 | |
144 | cppc_fi = container_of(irq_work, struct cppc_freq_invariance, irq_work); |
145 | kthread_queue_work(kworker_fie, &cppc_fi->work); |
146 | } |
147 | |
148 | static void cppc_scale_freq_tick(void) |
149 | { |
150 | struct cppc_freq_invariance *cppc_fi = &per_cpu(cppc_freq_inv, smp_processor_id()); |
151 | |
152 | /* |
153 | * cppc_get_perf_ctrs() can potentially sleep, call that from the right |
154 | * context. |
155 | */ |
156 | irq_work_queue(&cppc_fi->irq_work); |
157 | } |
158 | |
159 | static struct scale_freq_data cppc_sftd = { |
160 | .source = SCALE_FREQ_SOURCE_CPPC, |
161 | .set_freq_scale = cppc_scale_freq_tick, |
162 | }; |
163 | |
164 | static void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy) |
165 | { |
166 | struct cppc_freq_invariance *cppc_fi; |
167 | int cpu, ret; |
168 | |
169 | if (fie_disabled) |
170 | return; |
171 | |
172 | for_each_cpu(cpu, policy->cpus) { |
173 | cppc_fi = &per_cpu(cppc_freq_inv, cpu); |
174 | cppc_fi->cpu = cpu; |
175 | cppc_fi->cpu_data = policy->driver_data; |
176 | kthread_init_work(&cppc_fi->work, cppc_scale_freq_workfn); |
177 | init_irq_work(&cppc_fi->irq_work, cppc_irq_work); |
178 | |
179 | ret = cppc_get_perf_ctrs(cpu, &cppc_fi->prev_perf_fb_ctrs); |
180 | if (ret) { |
181 | pr_warn("%s: failed to read perf counters for cpu:%d: %d\n" , |
182 | __func__, cpu, ret); |
183 | |
184 | /* |
185 | * Don't abort if the CPU was offline while the driver |
186 | * was getting registered. |
187 | */ |
188 | if (cpu_online(cpu)) |
189 | return; |
190 | } |
191 | } |
192 | |
193 | /* Register for freq-invariance */ |
194 | topology_set_scale_freq_source(&cppc_sftd, policy->cpus); |
195 | } |
196 | |
197 | /* |
198 | * We free all the resources on policy's removal and not on CPU removal as the |
199 | * irq-work are per-cpu and the hotplug core takes care of flushing the pending |
200 | * irq-works (hint: smpcfd_dying_cpu()) on CPU hotplug. Even if the kthread-work |
201 | * fires on another CPU after the concerned CPU is removed, it won't harm. |
202 | * |
203 | * We just need to make sure to remove them all on policy->exit(). |
204 | */ |
205 | static void cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy *policy) |
206 | { |
207 | struct cppc_freq_invariance *cppc_fi; |
208 | int cpu; |
209 | |
210 | if (fie_disabled) |
211 | return; |
212 | |
213 | /* policy->cpus will be empty here, use related_cpus instead */ |
214 | topology_clear_scale_freq_source(SCALE_FREQ_SOURCE_CPPC, policy->related_cpus); |
215 | |
216 | for_each_cpu(cpu, policy->related_cpus) { |
217 | cppc_fi = &per_cpu(cppc_freq_inv, cpu); |
218 | irq_work_sync(&cppc_fi->irq_work); |
219 | kthread_cancel_work_sync(&cppc_fi->work); |
220 | } |
221 | } |
222 | |
223 | static void __init cppc_freq_invariance_init(void) |
224 | { |
225 | struct sched_attr attr = { |
226 | .size = sizeof(struct sched_attr), |
227 | .sched_policy = SCHED_DEADLINE, |
228 | .sched_nice = 0, |
229 | .sched_priority = 0, |
230 | /* |
231 | * Fake (unused) bandwidth; workaround to "fix" |
232 | * priority inheritance. |
233 | */ |
234 | .sched_runtime = 1000000, |
235 | .sched_deadline = 10000000, |
236 | .sched_period = 10000000, |
237 | }; |
238 | int ret; |
239 | |
240 | if (fie_disabled != FIE_ENABLED && fie_disabled != FIE_DISABLED) { |
241 | fie_disabled = FIE_ENABLED; |
242 | if (cppc_perf_ctrs_in_pcc()) { |
243 | pr_info("FIE not enabled on systems with registers in PCC\n" ); |
244 | fie_disabled = FIE_DISABLED; |
245 | } |
246 | } |
247 | |
248 | if (fie_disabled) |
249 | return; |
250 | |
251 | kworker_fie = kthread_create_worker(0, "cppc_fie" ); |
252 | if (IS_ERR(kworker_fie)) { |
253 | pr_warn("%s: failed to create kworker_fie: %ld\n" , __func__, |
254 | PTR_ERR(kworker_fie)); |
255 | fie_disabled = FIE_DISABLED; |
256 | return; |
257 | } |
258 | |
259 | ret = sched_setattr_nocheck(kworker_fie->task, &attr); |
260 | if (ret) { |
261 | pr_warn("%s: failed to set SCHED_DEADLINE: %d\n" , __func__, |
262 | ret); |
263 | kthread_destroy_worker(kworker_fie); |
264 | fie_disabled = FIE_DISABLED; |
265 | } |
266 | } |
267 | |
268 | static void cppc_freq_invariance_exit(void) |
269 | { |
270 | if (fie_disabled) |
271 | return; |
272 | |
273 | kthread_destroy_worker(kworker_fie); |
274 | } |
275 | |
276 | #else |
277 | static inline void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy) |
278 | { |
279 | } |
280 | |
281 | static inline void cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy *policy) |
282 | { |
283 | } |
284 | |
285 | static inline void cppc_freq_invariance_init(void) |
286 | { |
287 | } |
288 | |
289 | static inline void cppc_freq_invariance_exit(void) |
290 | { |
291 | } |
292 | #endif /* CONFIG_ACPI_CPPC_CPUFREQ_FIE */ |
293 | |
294 | /* Callback function used to retrieve the max frequency from DMI */ |
295 | static void cppc_find_dmi_mhz(const struct dmi_header *dm, void *private) |
296 | { |
297 | const u8 *dmi_data = (const u8 *)dm; |
298 | u16 *mhz = (u16 *)private; |
299 | |
300 | if (dm->type == DMI_ENTRY_PROCESSOR && |
301 | dm->length >= DMI_ENTRY_PROCESSOR_MIN_LENGTH) { |
302 | u16 val = (u16)get_unaligned((const u16 *) |
303 | (dmi_data + DMI_PROCESSOR_MAX_SPEED)); |
304 | *mhz = val > *mhz ? val : *mhz; |
305 | } |
306 | } |
307 | |
308 | /* Look up the max frequency in DMI */ |
309 | static u64 cppc_get_dmi_max_khz(void) |
310 | { |
311 | u16 mhz = 0; |
312 | |
313 | dmi_walk(decode: cppc_find_dmi_mhz, private_data: &mhz); |
314 | |
315 | /* |
316 | * Real stupid fallback value, just in case there is no |
317 | * actual value set. |
318 | */ |
319 | mhz = mhz ? mhz : 1; |
320 | |
321 | return (1000 * mhz); |
322 | } |
323 | |
324 | /* |
325 | * If CPPC lowest_freq and nominal_freq registers are exposed then we can |
326 | * use them to convert perf to freq and vice versa. The conversion is |
327 | * extrapolated as an affine function passing by the 2 points: |
328 | * - (Low perf, Low freq) |
329 | * - (Nominal perf, Nominal perf) |
330 | */ |
331 | static unsigned int cppc_cpufreq_perf_to_khz(struct cppc_cpudata *cpu_data, |
332 | unsigned int perf) |
333 | { |
334 | struct cppc_perf_caps *caps = &cpu_data->perf_caps; |
335 | s64 retval, offset = 0; |
336 | static u64 max_khz; |
337 | u64 mul, div; |
338 | |
339 | if (caps->lowest_freq && caps->nominal_freq) { |
340 | mul = caps->nominal_freq - caps->lowest_freq; |
341 | div = caps->nominal_perf - caps->lowest_perf; |
342 | offset = caps->nominal_freq - div64_u64(dividend: caps->nominal_perf * mul, divisor: div); |
343 | } else { |
344 | if (!max_khz) |
345 | max_khz = cppc_get_dmi_max_khz(); |
346 | mul = max_khz; |
347 | div = caps->highest_perf; |
348 | } |
349 | |
350 | retval = offset + div64_u64(dividend: perf * mul, divisor: div); |
351 | if (retval >= 0) |
352 | return retval; |
353 | return 0; |
354 | } |
355 | |
356 | static unsigned int cppc_cpufreq_khz_to_perf(struct cppc_cpudata *cpu_data, |
357 | unsigned int freq) |
358 | { |
359 | struct cppc_perf_caps *caps = &cpu_data->perf_caps; |
360 | s64 retval, offset = 0; |
361 | static u64 max_khz; |
362 | u64 mul, div; |
363 | |
364 | if (caps->lowest_freq && caps->nominal_freq) { |
365 | mul = caps->nominal_perf - caps->lowest_perf; |
366 | div = caps->nominal_freq - caps->lowest_freq; |
367 | offset = caps->nominal_perf - div64_u64(dividend: caps->nominal_freq * mul, divisor: div); |
368 | } else { |
369 | if (!max_khz) |
370 | max_khz = cppc_get_dmi_max_khz(); |
371 | mul = caps->highest_perf; |
372 | div = max_khz; |
373 | } |
374 | |
375 | retval = offset + div64_u64(dividend: freq * mul, divisor: div); |
376 | if (retval >= 0) |
377 | return retval; |
378 | return 0; |
379 | } |
380 | |
381 | static int cppc_cpufreq_set_target(struct cpufreq_policy *policy, |
382 | unsigned int target_freq, |
383 | unsigned int relation) |
384 | |
385 | { |
386 | struct cppc_cpudata *cpu_data = policy->driver_data; |
387 | unsigned int cpu = policy->cpu; |
388 | struct cpufreq_freqs freqs; |
389 | u32 desired_perf; |
390 | int ret = 0; |
391 | |
392 | desired_perf = cppc_cpufreq_khz_to_perf(cpu_data, freq: target_freq); |
393 | /* Return if it is exactly the same perf */ |
394 | if (desired_perf == cpu_data->perf_ctrls.desired_perf) |
395 | return ret; |
396 | |
397 | cpu_data->perf_ctrls.desired_perf = desired_perf; |
398 | freqs.old = policy->cur; |
399 | freqs.new = target_freq; |
400 | |
401 | cpufreq_freq_transition_begin(policy, freqs: &freqs); |
402 | ret = cppc_set_perf(cpu, perf_ctrls: &cpu_data->perf_ctrls); |
403 | cpufreq_freq_transition_end(policy, freqs: &freqs, transition_failed: ret != 0); |
404 | |
405 | if (ret) |
406 | pr_debug("Failed to set target on CPU:%d. ret:%d\n" , |
407 | cpu, ret); |
408 | |
409 | return ret; |
410 | } |
411 | |
412 | static unsigned int cppc_cpufreq_fast_switch(struct cpufreq_policy *policy, |
413 | unsigned int target_freq) |
414 | { |
415 | struct cppc_cpudata *cpu_data = policy->driver_data; |
416 | unsigned int cpu = policy->cpu; |
417 | u32 desired_perf; |
418 | int ret; |
419 | |
420 | desired_perf = cppc_cpufreq_khz_to_perf(cpu_data, freq: target_freq); |
421 | cpu_data->perf_ctrls.desired_perf = desired_perf; |
422 | ret = cppc_set_perf(cpu, perf_ctrls: &cpu_data->perf_ctrls); |
423 | |
424 | if (ret) { |
425 | pr_debug("Failed to set target on CPU:%d. ret:%d\n" , |
426 | cpu, ret); |
427 | return 0; |
428 | } |
429 | |
430 | return target_freq; |
431 | } |
432 | |
433 | static int cppc_verify_policy(struct cpufreq_policy_data *policy) |
434 | { |
435 | cpufreq_verify_within_cpu_limits(policy); |
436 | return 0; |
437 | } |
438 | |
439 | /* |
440 | * The PCC subspace describes the rate at which platform can accept commands |
441 | * on the shared PCC channel (including READs which do not count towards freq |
442 | * transition requests), so ideally we need to use the PCC values as a fallback |
443 | * if we don't have a platform specific transition_delay_us |
444 | */ |
445 | #ifdef CONFIG_ARM64 |
446 | #include <asm/cputype.h> |
447 | |
448 | static unsigned int cppc_cpufreq_get_transition_delay_us(unsigned int cpu) |
449 | { |
450 | unsigned long implementor = read_cpuid_implementor(); |
451 | unsigned long part_num = read_cpuid_part_number(); |
452 | |
453 | switch (implementor) { |
454 | case ARM_CPU_IMP_QCOM: |
455 | switch (part_num) { |
456 | case QCOM_CPU_PART_FALKOR_V1: |
457 | case QCOM_CPU_PART_FALKOR: |
458 | return 10000; |
459 | } |
460 | } |
461 | return cppc_get_transition_latency(cpu) / NSEC_PER_USEC; |
462 | } |
463 | #else |
464 | static unsigned int cppc_cpufreq_get_transition_delay_us(unsigned int cpu) |
465 | { |
466 | return cppc_get_transition_latency(cpu) / NSEC_PER_USEC; |
467 | } |
468 | #endif |
469 | |
470 | #if defined(CONFIG_ARM64) && defined(CONFIG_ENERGY_MODEL) |
471 | |
472 | static DEFINE_PER_CPU(unsigned int, efficiency_class); |
473 | static void cppc_cpufreq_register_em(struct cpufreq_policy *policy); |
474 | |
475 | /* Create an artificial performance state every CPPC_EM_CAP_STEP capacity unit. */ |
476 | #define CPPC_EM_CAP_STEP (20) |
477 | /* Increase the cost value by CPPC_EM_COST_STEP every performance state. */ |
478 | #define CPPC_EM_COST_STEP (1) |
479 | /* Add a cost gap correspnding to the energy of 4 CPUs. */ |
480 | #define CPPC_EM_COST_GAP (4 * SCHED_CAPACITY_SCALE * CPPC_EM_COST_STEP \ |
481 | / CPPC_EM_CAP_STEP) |
482 | |
483 | static unsigned int get_perf_level_count(struct cpufreq_policy *policy) |
484 | { |
485 | struct cppc_perf_caps *perf_caps; |
486 | unsigned int min_cap, max_cap; |
487 | struct cppc_cpudata *cpu_data; |
488 | int cpu = policy->cpu; |
489 | |
490 | cpu_data = policy->driver_data; |
491 | perf_caps = &cpu_data->perf_caps; |
492 | max_cap = arch_scale_cpu_capacity(cpu); |
493 | min_cap = div_u64((u64)max_cap * perf_caps->lowest_perf, |
494 | perf_caps->highest_perf); |
495 | if ((min_cap == 0) || (max_cap < min_cap)) |
496 | return 0; |
497 | return 1 + max_cap / CPPC_EM_CAP_STEP - min_cap / CPPC_EM_CAP_STEP; |
498 | } |
499 | |
500 | /* |
501 | * The cost is defined as: |
502 | * cost = power * max_frequency / frequency |
503 | */ |
504 | static inline unsigned long compute_cost(int cpu, int step) |
505 | { |
506 | return CPPC_EM_COST_GAP * per_cpu(efficiency_class, cpu) + |
507 | step * CPPC_EM_COST_STEP; |
508 | } |
509 | |
510 | static int cppc_get_cpu_power(struct device *cpu_dev, |
511 | unsigned long *power, unsigned long *KHz) |
512 | { |
513 | unsigned long perf_step, perf_prev, perf, perf_check; |
514 | unsigned int min_step, max_step, step, step_check; |
515 | unsigned long prev_freq = *KHz; |
516 | unsigned int min_cap, max_cap; |
517 | struct cpufreq_policy *policy; |
518 | |
519 | struct cppc_perf_caps *perf_caps; |
520 | struct cppc_cpudata *cpu_data; |
521 | |
522 | policy = cpufreq_cpu_get_raw(cpu_dev->id); |
523 | cpu_data = policy->driver_data; |
524 | perf_caps = &cpu_data->perf_caps; |
525 | max_cap = arch_scale_cpu_capacity(cpu_dev->id); |
526 | min_cap = div_u64((u64)max_cap * perf_caps->lowest_perf, |
527 | perf_caps->highest_perf); |
528 | perf_step = div_u64((u64)CPPC_EM_CAP_STEP * perf_caps->highest_perf, |
529 | max_cap); |
530 | min_step = min_cap / CPPC_EM_CAP_STEP; |
531 | max_step = max_cap / CPPC_EM_CAP_STEP; |
532 | |
533 | perf_prev = cppc_cpufreq_khz_to_perf(cpu_data, *KHz); |
534 | step = perf_prev / perf_step; |
535 | |
536 | if (step > max_step) |
537 | return -EINVAL; |
538 | |
539 | if (min_step == max_step) { |
540 | step = max_step; |
541 | perf = perf_caps->highest_perf; |
542 | } else if (step < min_step) { |
543 | step = min_step; |
544 | perf = perf_caps->lowest_perf; |
545 | } else { |
546 | step++; |
547 | if (step == max_step) |
548 | perf = perf_caps->highest_perf; |
549 | else |
550 | perf = step * perf_step; |
551 | } |
552 | |
553 | *KHz = cppc_cpufreq_perf_to_khz(cpu_data, perf); |
554 | perf_check = cppc_cpufreq_khz_to_perf(cpu_data, *KHz); |
555 | step_check = perf_check / perf_step; |
556 | |
557 | /* |
558 | * To avoid bad integer approximation, check that new frequency value |
559 | * increased and that the new frequency will be converted to the |
560 | * desired step value. |
561 | */ |
562 | while ((*KHz == prev_freq) || (step_check != step)) { |
563 | perf++; |
564 | *KHz = cppc_cpufreq_perf_to_khz(cpu_data, perf); |
565 | perf_check = cppc_cpufreq_khz_to_perf(cpu_data, *KHz); |
566 | step_check = perf_check / perf_step; |
567 | } |
568 | |
569 | /* |
570 | * With an artificial EM, only the cost value is used. Still the power |
571 | * is populated such as 0 < power < EM_MAX_POWER. This allows to add |
572 | * more sense to the artificial performance states. |
573 | */ |
574 | *power = compute_cost(cpu_dev->id, step); |
575 | |
576 | return 0; |
577 | } |
578 | |
579 | static int cppc_get_cpu_cost(struct device *cpu_dev, unsigned long KHz, |
580 | unsigned long *cost) |
581 | { |
582 | unsigned long perf_step, perf_prev; |
583 | struct cppc_perf_caps *perf_caps; |
584 | struct cpufreq_policy *policy; |
585 | struct cppc_cpudata *cpu_data; |
586 | unsigned int max_cap; |
587 | int step; |
588 | |
589 | policy = cpufreq_cpu_get_raw(cpu_dev->id); |
590 | cpu_data = policy->driver_data; |
591 | perf_caps = &cpu_data->perf_caps; |
592 | max_cap = arch_scale_cpu_capacity(cpu_dev->id); |
593 | |
594 | perf_prev = cppc_cpufreq_khz_to_perf(cpu_data, KHz); |
595 | perf_step = CPPC_EM_CAP_STEP * perf_caps->highest_perf / max_cap; |
596 | step = perf_prev / perf_step; |
597 | |
598 | *cost = compute_cost(cpu_dev->id, step); |
599 | |
600 | return 0; |
601 | } |
602 | |
603 | static int populate_efficiency_class(void) |
604 | { |
605 | struct acpi_madt_generic_interrupt *gicc; |
606 | DECLARE_BITMAP(used_classes, 256) = {}; |
607 | int class, cpu, index; |
608 | |
609 | for_each_possible_cpu(cpu) { |
610 | gicc = acpi_cpu_get_madt_gicc(cpu); |
611 | class = gicc->efficiency_class; |
612 | bitmap_set(used_classes, class, 1); |
613 | } |
614 | |
615 | if (bitmap_weight(used_classes, 256) <= 1) { |
616 | pr_debug("Efficiency classes are all equal (=%d). " |
617 | "No EM registered" , class); |
618 | return -EINVAL; |
619 | } |
620 | |
621 | /* |
622 | * Squeeze efficiency class values on [0:#efficiency_class-1]. |
623 | * Values are per spec in [0:255]. |
624 | */ |
625 | index = 0; |
626 | for_each_set_bit(class, used_classes, 256) { |
627 | for_each_possible_cpu(cpu) { |
628 | gicc = acpi_cpu_get_madt_gicc(cpu); |
629 | if (gicc->efficiency_class == class) |
630 | per_cpu(efficiency_class, cpu) = index; |
631 | } |
632 | index++; |
633 | } |
634 | cppc_cpufreq_driver.register_em = cppc_cpufreq_register_em; |
635 | |
636 | return 0; |
637 | } |
638 | |
639 | static void cppc_cpufreq_register_em(struct cpufreq_policy *policy) |
640 | { |
641 | struct cppc_cpudata *cpu_data; |
642 | struct em_data_callback em_cb = |
643 | EM_ADV_DATA_CB(cppc_get_cpu_power, cppc_get_cpu_cost); |
644 | |
645 | cpu_data = policy->driver_data; |
646 | em_dev_register_perf_domain(get_cpu_device(policy->cpu), |
647 | get_perf_level_count(policy), &em_cb, |
648 | cpu_data->shared_cpu_map, 0); |
649 | } |
650 | |
651 | #else |
652 | static int populate_efficiency_class(void) |
653 | { |
654 | return 0; |
655 | } |
656 | #endif |
657 | |
658 | static struct cppc_cpudata *cppc_cpufreq_get_cpu_data(unsigned int cpu) |
659 | { |
660 | struct cppc_cpudata *cpu_data; |
661 | int ret; |
662 | |
663 | cpu_data = kzalloc(size: sizeof(struct cppc_cpudata), GFP_KERNEL); |
664 | if (!cpu_data) |
665 | goto out; |
666 | |
667 | if (!zalloc_cpumask_var(mask: &cpu_data->shared_cpu_map, GFP_KERNEL)) |
668 | goto free_cpu; |
669 | |
670 | ret = acpi_get_psd_map(cpu, cpu_data); |
671 | if (ret) { |
672 | pr_debug("Err parsing CPU%d PSD data: ret:%d\n" , cpu, ret); |
673 | goto free_mask; |
674 | } |
675 | |
676 | ret = cppc_get_perf_caps(cpu, caps: &cpu_data->perf_caps); |
677 | if (ret) { |
678 | pr_debug("Err reading CPU%d perf caps: ret:%d\n" , cpu, ret); |
679 | goto free_mask; |
680 | } |
681 | |
682 | /* Convert the lowest and nominal freq from MHz to KHz */ |
683 | cpu_data->perf_caps.lowest_freq *= 1000; |
684 | cpu_data->perf_caps.nominal_freq *= 1000; |
685 | |
686 | list_add(new: &cpu_data->node, head: &cpu_data_list); |
687 | |
688 | return cpu_data; |
689 | |
690 | free_mask: |
691 | free_cpumask_var(mask: cpu_data->shared_cpu_map); |
692 | free_cpu: |
693 | kfree(objp: cpu_data); |
694 | out: |
695 | return NULL; |
696 | } |
697 | |
698 | static void cppc_cpufreq_put_cpu_data(struct cpufreq_policy *policy) |
699 | { |
700 | struct cppc_cpudata *cpu_data = policy->driver_data; |
701 | |
702 | list_del(entry: &cpu_data->node); |
703 | free_cpumask_var(mask: cpu_data->shared_cpu_map); |
704 | kfree(objp: cpu_data); |
705 | policy->driver_data = NULL; |
706 | } |
707 | |
708 | static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy) |
709 | { |
710 | unsigned int cpu = policy->cpu; |
711 | struct cppc_cpudata *cpu_data; |
712 | struct cppc_perf_caps *caps; |
713 | int ret; |
714 | |
715 | cpu_data = cppc_cpufreq_get_cpu_data(cpu); |
716 | if (!cpu_data) { |
717 | pr_err("Error in acquiring _CPC/_PSD data for CPU%d.\n" , cpu); |
718 | return -ENODEV; |
719 | } |
720 | caps = &cpu_data->perf_caps; |
721 | policy->driver_data = cpu_data; |
722 | |
723 | /* |
724 | * Set min to lowest nonlinear perf to avoid any efficiency penalty (see |
725 | * Section 8.4.7.1.1.5 of ACPI 6.1 spec) |
726 | */ |
727 | policy->min = cppc_cpufreq_perf_to_khz(cpu_data, |
728 | perf: caps->lowest_nonlinear_perf); |
729 | policy->max = cppc_cpufreq_perf_to_khz(cpu_data, |
730 | perf: caps->nominal_perf); |
731 | |
732 | /* |
733 | * Set cpuinfo.min_freq to Lowest to make the full range of performance |
734 | * available if userspace wants to use any perf between lowest & lowest |
735 | * nonlinear perf |
736 | */ |
737 | policy->cpuinfo.min_freq = cppc_cpufreq_perf_to_khz(cpu_data, |
738 | perf: caps->lowest_perf); |
739 | policy->cpuinfo.max_freq = cppc_cpufreq_perf_to_khz(cpu_data, |
740 | perf: caps->nominal_perf); |
741 | |
742 | policy->transition_delay_us = cppc_cpufreq_get_transition_delay_us(cpu); |
743 | policy->shared_type = cpu_data->shared_type; |
744 | |
745 | switch (policy->shared_type) { |
746 | case CPUFREQ_SHARED_TYPE_HW: |
747 | case CPUFREQ_SHARED_TYPE_NONE: |
748 | /* Nothing to be done - we'll have a policy for each CPU */ |
749 | break; |
750 | case CPUFREQ_SHARED_TYPE_ANY: |
751 | /* |
752 | * All CPUs in the domain will share a policy and all cpufreq |
753 | * operations will use a single cppc_cpudata structure stored |
754 | * in policy->driver_data. |
755 | */ |
756 | cpumask_copy(dstp: policy->cpus, srcp: cpu_data->shared_cpu_map); |
757 | break; |
758 | default: |
759 | pr_debug("Unsupported CPU co-ord type: %d\n" , |
760 | policy->shared_type); |
761 | ret = -EFAULT; |
762 | goto out; |
763 | } |
764 | |
765 | policy->fast_switch_possible = cppc_allow_fast_switch(); |
766 | policy->dvfs_possible_from_any_cpu = true; |
767 | |
768 | /* |
769 | * If 'highest_perf' is greater than 'nominal_perf', we assume CPU Boost |
770 | * is supported. |
771 | */ |
772 | if (caps->highest_perf > caps->nominal_perf) |
773 | boost_supported = true; |
774 | |
775 | /* Set policy->cur to max now. The governors will adjust later. */ |
776 | policy->cur = cppc_cpufreq_perf_to_khz(cpu_data, perf: caps->highest_perf); |
777 | cpu_data->perf_ctrls.desired_perf = caps->highest_perf; |
778 | |
779 | ret = cppc_set_perf(cpu, perf_ctrls: &cpu_data->perf_ctrls); |
780 | if (ret) { |
781 | pr_debug("Err setting perf value:%d on CPU:%d. ret:%d\n" , |
782 | caps->highest_perf, cpu, ret); |
783 | goto out; |
784 | } |
785 | |
786 | cppc_cpufreq_cpu_fie_init(policy); |
787 | return 0; |
788 | |
789 | out: |
790 | cppc_cpufreq_put_cpu_data(policy); |
791 | return ret; |
792 | } |
793 | |
794 | static int cppc_cpufreq_cpu_exit(struct cpufreq_policy *policy) |
795 | { |
796 | struct cppc_cpudata *cpu_data = policy->driver_data; |
797 | struct cppc_perf_caps *caps = &cpu_data->perf_caps; |
798 | unsigned int cpu = policy->cpu; |
799 | int ret; |
800 | |
801 | cppc_cpufreq_cpu_fie_exit(policy); |
802 | |
803 | cpu_data->perf_ctrls.desired_perf = caps->lowest_perf; |
804 | |
805 | ret = cppc_set_perf(cpu, perf_ctrls: &cpu_data->perf_ctrls); |
806 | if (ret) |
807 | pr_debug("Err setting perf value:%d on CPU:%d. ret:%d\n" , |
808 | caps->lowest_perf, cpu, ret); |
809 | |
810 | cppc_cpufreq_put_cpu_data(policy); |
811 | return 0; |
812 | } |
813 | |
814 | static inline u64 get_delta(u64 t1, u64 t0) |
815 | { |
816 | if (t1 > t0 || t0 > ~(u32)0) |
817 | return t1 - t0; |
818 | |
819 | return (u32)t1 - (u32)t0; |
820 | } |
821 | |
822 | static int cppc_perf_from_fbctrs(struct cppc_cpudata *cpu_data, |
823 | struct cppc_perf_fb_ctrs *fb_ctrs_t0, |
824 | struct cppc_perf_fb_ctrs *fb_ctrs_t1) |
825 | { |
826 | u64 delta_reference, delta_delivered; |
827 | u64 reference_perf; |
828 | |
829 | reference_perf = fb_ctrs_t0->reference_perf; |
830 | |
831 | delta_reference = get_delta(t1: fb_ctrs_t1->reference, |
832 | t0: fb_ctrs_t0->reference); |
833 | delta_delivered = get_delta(t1: fb_ctrs_t1->delivered, |
834 | t0: fb_ctrs_t0->delivered); |
835 | |
836 | /* Check to avoid divide-by zero and invalid delivered_perf */ |
837 | if (!delta_reference || !delta_delivered) |
838 | return cpu_data->perf_ctrls.desired_perf; |
839 | |
840 | return (reference_perf * delta_delivered) / delta_reference; |
841 | } |
842 | |
843 | static unsigned int cppc_cpufreq_get_rate(unsigned int cpu) |
844 | { |
845 | struct cppc_perf_fb_ctrs fb_ctrs_t0 = {0}, fb_ctrs_t1 = {0}; |
846 | struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); |
847 | struct cppc_cpudata *cpu_data = policy->driver_data; |
848 | u64 delivered_perf; |
849 | int ret; |
850 | |
851 | cpufreq_cpu_put(policy); |
852 | |
853 | ret = cppc_get_perf_ctrs(cpu, perf_fb_ctrs: &fb_ctrs_t0); |
854 | if (ret) |
855 | return 0; |
856 | |
857 | udelay(2); /* 2usec delay between sampling */ |
858 | |
859 | ret = cppc_get_perf_ctrs(cpu, perf_fb_ctrs: &fb_ctrs_t1); |
860 | if (ret) |
861 | return 0; |
862 | |
863 | delivered_perf = cppc_perf_from_fbctrs(cpu_data, fb_ctrs_t0: &fb_ctrs_t0, |
864 | fb_ctrs_t1: &fb_ctrs_t1); |
865 | |
866 | return cppc_cpufreq_perf_to_khz(cpu_data, perf: delivered_perf); |
867 | } |
868 | |
869 | static int cppc_cpufreq_set_boost(struct cpufreq_policy *policy, int state) |
870 | { |
871 | struct cppc_cpudata *cpu_data = policy->driver_data; |
872 | struct cppc_perf_caps *caps = &cpu_data->perf_caps; |
873 | int ret; |
874 | |
875 | if (!boost_supported) { |
876 | pr_err("BOOST not supported by CPU or firmware\n" ); |
877 | return -EINVAL; |
878 | } |
879 | |
880 | if (state) |
881 | policy->max = cppc_cpufreq_perf_to_khz(cpu_data, |
882 | perf: caps->highest_perf); |
883 | else |
884 | policy->max = cppc_cpufreq_perf_to_khz(cpu_data, |
885 | perf: caps->nominal_perf); |
886 | policy->cpuinfo.max_freq = policy->max; |
887 | |
888 | ret = freq_qos_update_request(req: policy->max_freq_req, new_value: policy->max); |
889 | if (ret < 0) |
890 | return ret; |
891 | |
892 | return 0; |
893 | } |
894 | |
895 | static ssize_t show_freqdomain_cpus(struct cpufreq_policy *policy, char *buf) |
896 | { |
897 | struct cppc_cpudata *cpu_data = policy->driver_data; |
898 | |
899 | return cpufreq_show_cpus(mask: cpu_data->shared_cpu_map, buf); |
900 | } |
901 | cpufreq_freq_attr_ro(freqdomain_cpus); |
902 | |
903 | static struct freq_attr *cppc_cpufreq_attr[] = { |
904 | &freqdomain_cpus, |
905 | NULL, |
906 | }; |
907 | |
908 | static struct cpufreq_driver cppc_cpufreq_driver = { |
909 | .flags = CPUFREQ_CONST_LOOPS, |
910 | .verify = cppc_verify_policy, |
911 | .target = cppc_cpufreq_set_target, |
912 | .get = cppc_cpufreq_get_rate, |
913 | .fast_switch = cppc_cpufreq_fast_switch, |
914 | .init = cppc_cpufreq_cpu_init, |
915 | .exit = cppc_cpufreq_cpu_exit, |
916 | .set_boost = cppc_cpufreq_set_boost, |
917 | .attr = cppc_cpufreq_attr, |
918 | .name = "cppc_cpufreq" , |
919 | }; |
920 | |
921 | /* |
922 | * HISI platform does not support delivered performance counter and |
923 | * reference performance counter. It can calculate the performance using the |
924 | * platform specific mechanism. We reuse the desired performance register to |
925 | * store the real performance calculated by the platform. |
926 | */ |
927 | static unsigned int hisi_cppc_cpufreq_get_rate(unsigned int cpu) |
928 | { |
929 | struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); |
930 | struct cppc_cpudata *cpu_data = policy->driver_data; |
931 | u64 desired_perf; |
932 | int ret; |
933 | |
934 | cpufreq_cpu_put(policy); |
935 | |
936 | ret = cppc_get_desired_perf(cpunum: cpu, desired_perf: &desired_perf); |
937 | if (ret < 0) |
938 | return -EIO; |
939 | |
940 | return cppc_cpufreq_perf_to_khz(cpu_data, perf: desired_perf); |
941 | } |
942 | |
943 | static void cppc_check_hisi_workaround(void) |
944 | { |
945 | struct acpi_table_header *tbl; |
946 | acpi_status status = AE_OK; |
947 | int i; |
948 | |
949 | status = acpi_get_table(ACPI_SIG_PCCT, instance: 0, out_table: &tbl); |
950 | if (ACPI_FAILURE(status) || !tbl) |
951 | return; |
952 | |
953 | for (i = 0; i < ARRAY_SIZE(wa_info); i++) { |
954 | if (!memcmp(p: wa_info[i].oem_id, q: tbl->oem_id, ACPI_OEM_ID_SIZE) && |
955 | !memcmp(p: wa_info[i].oem_table_id, q: tbl->oem_table_id, ACPI_OEM_TABLE_ID_SIZE) && |
956 | wa_info[i].oem_revision == tbl->oem_revision) { |
957 | /* Overwrite the get() callback */ |
958 | cppc_cpufreq_driver.get = hisi_cppc_cpufreq_get_rate; |
959 | fie_disabled = FIE_DISABLED; |
960 | break; |
961 | } |
962 | } |
963 | |
964 | acpi_put_table(table: tbl); |
965 | } |
966 | |
967 | static int __init cppc_cpufreq_init(void) |
968 | { |
969 | int ret; |
970 | |
971 | if (!acpi_cpc_valid()) |
972 | return -ENODEV; |
973 | |
974 | cppc_check_hisi_workaround(); |
975 | cppc_freq_invariance_init(); |
976 | populate_efficiency_class(); |
977 | |
978 | ret = cpufreq_register_driver(driver_data: &cppc_cpufreq_driver); |
979 | if (ret) |
980 | cppc_freq_invariance_exit(); |
981 | |
982 | return ret; |
983 | } |
984 | |
985 | static inline void free_cpu_data(void) |
986 | { |
987 | struct cppc_cpudata *iter, *tmp; |
988 | |
989 | list_for_each_entry_safe(iter, tmp, &cpu_data_list, node) { |
990 | free_cpumask_var(mask: iter->shared_cpu_map); |
991 | list_del(entry: &iter->node); |
992 | kfree(objp: iter); |
993 | } |
994 | |
995 | } |
996 | |
997 | static void __exit cppc_cpufreq_exit(void) |
998 | { |
999 | cpufreq_unregister_driver(driver_data: &cppc_cpufreq_driver); |
1000 | cppc_freq_invariance_exit(); |
1001 | |
1002 | free_cpu_data(); |
1003 | } |
1004 | |
1005 | module_exit(cppc_cpufreq_exit); |
1006 | MODULE_AUTHOR("Ashwin Chaugule" ); |
1007 | MODULE_DESCRIPTION("CPUFreq driver based on the ACPI CPPC v5.0+ spec" ); |
1008 | MODULE_LICENSE("GPL" ); |
1009 | |
1010 | late_initcall(cppc_cpufreq_init); |
1011 | |
1012 | static const struct acpi_device_id cppc_acpi_ids[] __used = { |
1013 | {ACPI_PROCESSOR_DEVICE_HID, }, |
1014 | {} |
1015 | }; |
1016 | |
1017 | MODULE_DEVICE_TABLE(acpi, cppc_acpi_ids); |
1018 | |