1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (C) 2005 Intel Corporation |
4 | * Venkatesh Pallipadi <venkatesh.pallipadi@intel.com> |
5 | * - Added _PDC for SMP C-states on Intel CPUs |
6 | */ |
7 | |
8 | #include <linux/kernel.h> |
9 | #include <linux/export.h> |
10 | #include <linux/init.h> |
11 | #include <linux/acpi.h> |
12 | #include <linux/cpu.h> |
13 | #include <linux/sched.h> |
14 | |
15 | #include <acpi/processor.h> |
16 | #include <asm/mwait.h> |
17 | #include <asm/special_insns.h> |
18 | |
19 | /* |
20 | * Initialize bm_flags based on the CPU cache properties |
21 | * On SMP it depends on cache configuration |
22 | * - When cache is not shared among all CPUs, we flush cache |
23 | * before entering C3. |
24 | * - When cache is shared among all CPUs, we use bm_check |
25 | * mechanism as in UP case |
26 | * |
27 | * This routine is called only after all the CPUs are online |
28 | */ |
29 | void acpi_processor_power_init_bm_check(struct acpi_processor_flags *flags, |
30 | unsigned int cpu) |
31 | { |
32 | struct cpuinfo_x86 *c = &cpu_data(cpu); |
33 | |
34 | flags->bm_check = 0; |
35 | if (num_online_cpus() == 1) |
36 | flags->bm_check = 1; |
37 | else if (c->x86_vendor == X86_VENDOR_INTEL) { |
38 | /* |
39 | * Today all MP CPUs that support C3 share cache. |
40 | * And caches should not be flushed by software while |
41 | * entering C3 type state. |
42 | */ |
43 | flags->bm_check = 1; |
44 | } |
45 | |
46 | /* |
47 | * On all recent Intel platforms, ARB_DISABLE is a nop. |
48 | * So, set bm_control to zero to indicate that ARB_DISABLE |
49 | * is not required while entering C3 type state on |
50 | * P4, Core and beyond CPUs |
51 | */ |
52 | if (c->x86_vendor == X86_VENDOR_INTEL && |
53 | (c->x86 > 0xf || (c->x86 == 6 && c->x86_model >= 0x0f))) |
54 | flags->bm_control = 0; |
55 | |
56 | if (c->x86_vendor == X86_VENDOR_CENTAUR) { |
57 | if (c->x86 > 6 || (c->x86 == 6 && c->x86_model == 0x0f && |
58 | c->x86_stepping >= 0x0e)) { |
59 | /* |
60 | * For all recent Centaur CPUs, the ucode will make sure that each |
61 | * core can keep cache coherence with each other while entering C3 |
62 | * type state. So, set bm_check to 1 to indicate that the kernel |
63 | * doesn't need to execute a cache flush operation (WBINVD) when |
64 | * entering C3 type state. |
65 | */ |
66 | flags->bm_check = 1; |
67 | /* |
68 | * For all recent Centaur platforms, ARB_DISABLE is a nop. |
69 | * Set bm_control to zero to indicate that ARB_DISABLE is |
70 | * not required while entering C3 type state. |
71 | */ |
72 | flags->bm_control = 0; |
73 | } |
74 | } |
75 | |
76 | if (c->x86_vendor == X86_VENDOR_ZHAOXIN) { |
77 | /* |
78 | * All Zhaoxin CPUs that support C3 share cache. |
79 | * And caches should not be flushed by software while |
80 | * entering C3 type state. |
81 | */ |
82 | flags->bm_check = 1; |
83 | /* |
84 | * On all recent Zhaoxin platforms, ARB_DISABLE is a nop. |
85 | * So, set bm_control to zero to indicate that ARB_DISABLE |
86 | * is not required while entering C3 type state. |
87 | */ |
88 | flags->bm_control = 0; |
89 | } |
90 | if (c->x86_vendor == X86_VENDOR_AMD && c->x86 >= 0x17) { |
91 | /* |
92 | * For all AMD Zen or newer CPUs that support C3, caches |
93 | * should not be flushed by software while entering C3 |
94 | * type state. Set bm->check to 1 so that kernel doesn't |
95 | * need to execute cache flush operation. |
96 | */ |
97 | flags->bm_check = 1; |
98 | /* |
99 | * In current AMD C state implementation ARB_DIS is no longer |
100 | * used. So set bm_control to zero to indicate ARB_DIS is not |
101 | * required while entering C3 type state. |
102 | */ |
103 | flags->bm_control = 0; |
104 | } |
105 | } |
106 | EXPORT_SYMBOL(acpi_processor_power_init_bm_check); |
107 | |
108 | /* The code below handles cstate entry with monitor-mwait pair on Intel*/ |
109 | |
110 | struct cstate_entry { |
111 | struct { |
112 | unsigned int eax; |
113 | unsigned int ecx; |
114 | } states[ACPI_PROCESSOR_MAX_POWER]; |
115 | }; |
116 | static struct cstate_entry __percpu *cpu_cstate_entry; /* per CPU ptr */ |
117 | |
118 | static short mwait_supported[ACPI_PROCESSOR_MAX_POWER]; |
119 | |
120 | #define NATIVE_CSTATE_BEYOND_HALT (2) |
121 | |
122 | static long acpi_processor_ffh_cstate_probe_cpu(void *_cx) |
123 | { |
124 | struct acpi_processor_cx *cx = _cx; |
125 | long retval; |
126 | unsigned int eax, ebx, ecx, edx; |
127 | unsigned int edx_part; |
128 | unsigned int cstate_type; /* C-state type and not ACPI C-state type */ |
129 | unsigned int num_cstate_subtype; |
130 | |
131 | cpuid(CPUID_MWAIT_LEAF, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx); |
132 | |
133 | /* Check whether this particular cx_type (in CST) is supported or not */ |
134 | cstate_type = (((cx->address >> MWAIT_SUBSTATE_SIZE) & |
135 | MWAIT_CSTATE_MASK) + 1) & MWAIT_CSTATE_MASK; |
136 | edx_part = edx >> (cstate_type * MWAIT_SUBSTATE_SIZE); |
137 | num_cstate_subtype = edx_part & MWAIT_SUBSTATE_MASK; |
138 | |
139 | retval = 0; |
140 | /* If the HW does not support any sub-states in this C-state */ |
141 | if (num_cstate_subtype == 0) { |
142 | pr_warn(FW_BUG "ACPI MWAIT C-state 0x%x not supported by HW (0x%x)\n" , |
143 | cx->address, edx_part); |
144 | retval = -1; |
145 | goto out; |
146 | } |
147 | |
148 | /* mwait ecx extensions INTERRUPT_BREAK should be supported for C2/C3 */ |
149 | if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) || |
150 | !(ecx & CPUID5_ECX_INTERRUPT_BREAK)) { |
151 | retval = -1; |
152 | goto out; |
153 | } |
154 | |
155 | if (!mwait_supported[cstate_type]) { |
156 | mwait_supported[cstate_type] = 1; |
157 | printk(KERN_DEBUG |
158 | "Monitor-Mwait will be used to enter C-%d state\n" , |
159 | cx->type); |
160 | } |
161 | snprintf(buf: cx->desc, |
162 | ACPI_CX_DESC_LEN, fmt: "ACPI FFH MWAIT 0x%x" , |
163 | cx->address); |
164 | out: |
165 | return retval; |
166 | } |
167 | |
168 | int acpi_processor_ffh_cstate_probe(unsigned int cpu, |
169 | struct acpi_processor_cx *cx, struct acpi_power_register *reg) |
170 | { |
171 | struct cstate_entry *percpu_entry; |
172 | struct cpuinfo_x86 *c = &cpu_data(cpu); |
173 | long retval; |
174 | |
175 | if (!cpu_cstate_entry || c->cpuid_level < CPUID_MWAIT_LEAF) |
176 | return -1; |
177 | |
178 | if (reg->bit_offset != NATIVE_CSTATE_BEYOND_HALT) |
179 | return -1; |
180 | |
181 | percpu_entry = per_cpu_ptr(cpu_cstate_entry, cpu); |
182 | percpu_entry->states[cx->index].eax = 0; |
183 | percpu_entry->states[cx->index].ecx = 0; |
184 | |
185 | /* Make sure we are running on right CPU */ |
186 | |
187 | retval = call_on_cpu(cpu, fn: acpi_processor_ffh_cstate_probe_cpu, arg: cx, |
188 | direct: false); |
189 | if (retval == 0) { |
190 | /* Use the hint in CST */ |
191 | percpu_entry->states[cx->index].eax = cx->address; |
192 | percpu_entry->states[cx->index].ecx = MWAIT_ECX_INTERRUPT_BREAK; |
193 | } |
194 | |
195 | /* |
196 | * For _CST FFH on Intel, if GAS.access_size bit 1 is cleared, |
197 | * then we should skip checking BM_STS for this C-state. |
198 | * ref: "Intel Processor Vendor-Specific ACPI Interface Specification" |
199 | */ |
200 | if ((c->x86_vendor == X86_VENDOR_INTEL) && !(reg->access_size & 0x2)) |
201 | cx->bm_sts_skip = 1; |
202 | |
203 | return retval; |
204 | } |
205 | EXPORT_SYMBOL_GPL(acpi_processor_ffh_cstate_probe); |
206 | |
207 | void __cpuidle acpi_processor_ffh_cstate_enter(struct acpi_processor_cx *cx) |
208 | { |
209 | unsigned int cpu = smp_processor_id(); |
210 | struct cstate_entry *percpu_entry; |
211 | |
212 | percpu_entry = per_cpu_ptr(cpu_cstate_entry, cpu); |
213 | mwait_idle_with_hints(eax: percpu_entry->states[cx->index].eax, |
214 | ecx: percpu_entry->states[cx->index].ecx); |
215 | } |
216 | EXPORT_SYMBOL_GPL(acpi_processor_ffh_cstate_enter); |
217 | |
218 | static int __init ffh_cstate_init(void) |
219 | { |
220 | struct cpuinfo_x86 *c = &boot_cpu_data; |
221 | |
222 | if (c->x86_vendor != X86_VENDOR_INTEL && |
223 | c->x86_vendor != X86_VENDOR_AMD && |
224 | c->x86_vendor != X86_VENDOR_HYGON) |
225 | return -1; |
226 | |
227 | cpu_cstate_entry = alloc_percpu(struct cstate_entry); |
228 | return 0; |
229 | } |
230 | |
231 | static void __exit ffh_cstate_exit(void) |
232 | { |
233 | free_percpu(pdata: cpu_cstate_entry); |
234 | cpu_cstate_entry = NULL; |
235 | } |
236 | |
237 | arch_initcall(ffh_cstate_init); |
238 | __exitcall(ffh_cstate_exit); |
239 | |