1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | #include <linux/export.h> |
3 | #include <linux/bitops.h> |
4 | #include <linux/elf.h> |
5 | #include <linux/mm.h> |
6 | |
7 | #include <linux/io.h> |
8 | #include <linux/sched.h> |
9 | #include <linux/sched/clock.h> |
10 | #include <linux/random.h> |
11 | #include <linux/topology.h> |
12 | #include <asm/processor.h> |
13 | #include <asm/apic.h> |
14 | #include <asm/cacheinfo.h> |
15 | #include <asm/cpu.h> |
16 | #include <asm/spec-ctrl.h> |
17 | #include <asm/smp.h> |
18 | #include <asm/numa.h> |
19 | #include <asm/pci-direct.h> |
20 | #include <asm/delay.h> |
21 | #include <asm/debugreg.h> |
22 | #include <asm/resctrl.h> |
23 | |
24 | #ifdef CONFIG_X86_64 |
25 | # include <asm/mmconfig.h> |
26 | #endif |
27 | |
28 | #include "cpu.h" |
29 | |
30 | /* |
31 | * nodes_per_socket: Stores the number of nodes per socket. |
32 | * Refer to Fam15h Models 00-0fh BKDG - CPUID Fn8000_001E_ECX |
33 | * Node Identifiers[10:8] |
34 | */ |
35 | static u32 nodes_per_socket = 1; |
36 | |
37 | /* |
38 | * AMD errata checking |
39 | * |
40 | * Errata are defined as arrays of ints using the AMD_LEGACY_ERRATUM() or |
41 | * AMD_OSVW_ERRATUM() macros. The latter is intended for newer errata that |
42 | * have an OSVW id assigned, which it takes as first argument. Both take a |
43 | * variable number of family-specific model-stepping ranges created by |
44 | * AMD_MODEL_RANGE(). |
45 | * |
46 | * Example: |
47 | * |
48 | * const int amd_erratum_319[] = |
49 | * AMD_LEGACY_ERRATUM(AMD_MODEL_RANGE(0x10, 0x2, 0x1, 0x4, 0x2), |
50 | * AMD_MODEL_RANGE(0x10, 0x8, 0x0, 0x8, 0x0), |
51 | * AMD_MODEL_RANGE(0x10, 0x9, 0x0, 0x9, 0x0)); |
52 | */ |
53 | |
54 | #define AMD_LEGACY_ERRATUM(...) { -1, __VA_ARGS__, 0 } |
55 | #define AMD_OSVW_ERRATUM(osvw_id, ...) { osvw_id, __VA_ARGS__, 0 } |
56 | #define AMD_MODEL_RANGE(f, m_start, s_start, m_end, s_end) \ |
57 | ((f << 24) | (m_start << 16) | (s_start << 12) | (m_end << 4) | (s_end)) |
58 | #define AMD_MODEL_RANGE_FAMILY(range) (((range) >> 24) & 0xff) |
59 | #define AMD_MODEL_RANGE_START(range) (((range) >> 12) & 0xfff) |
60 | #define AMD_MODEL_RANGE_END(range) ((range) & 0xfff) |
61 | |
62 | static const int amd_erratum_400[] = |
63 | AMD_OSVW_ERRATUM(1, AMD_MODEL_RANGE(0xf, 0x41, 0x2, 0xff, 0xf), |
64 | AMD_MODEL_RANGE(0x10, 0x2, 0x1, 0xff, 0xf)); |
65 | |
66 | static const int amd_erratum_383[] = |
67 | AMD_OSVW_ERRATUM(3, AMD_MODEL_RANGE(0x10, 0, 0, 0xff, 0xf)); |
68 | |
69 | /* #1054: Instructions Retired Performance Counter May Be Inaccurate */ |
70 | static const int amd_erratum_1054[] = |
71 | AMD_LEGACY_ERRATUM(AMD_MODEL_RANGE(0x17, 0, 0, 0x2f, 0xf)); |
72 | |
73 | static const int amd_zenbleed[] = |
74 | AMD_LEGACY_ERRATUM(AMD_MODEL_RANGE(0x17, 0x30, 0x0, 0x4f, 0xf), |
75 | AMD_MODEL_RANGE(0x17, 0x60, 0x0, 0x7f, 0xf), |
76 | AMD_MODEL_RANGE(0x17, 0x90, 0x0, 0x91, 0xf), |
77 | AMD_MODEL_RANGE(0x17, 0xa0, 0x0, 0xaf, 0xf)); |
78 | |
79 | static const int amd_div0[] = |
80 | AMD_LEGACY_ERRATUM(AMD_MODEL_RANGE(0x17, 0x00, 0x0, 0x2f, 0xf), |
81 | AMD_MODEL_RANGE(0x17, 0x50, 0x0, 0x5f, 0xf)); |
82 | |
83 | static const int amd_erratum_1485[] = |
84 | AMD_LEGACY_ERRATUM(AMD_MODEL_RANGE(0x19, 0x10, 0x0, 0x1f, 0xf), |
85 | AMD_MODEL_RANGE(0x19, 0x60, 0x0, 0xaf, 0xf)); |
86 | |
87 | static bool cpu_has_amd_erratum(struct cpuinfo_x86 *cpu, const int *erratum) |
88 | { |
89 | int osvw_id = *erratum++; |
90 | u32 range; |
91 | u32 ms; |
92 | |
93 | if (osvw_id >= 0 && osvw_id < 65536 && |
94 | cpu_has(cpu, X86_FEATURE_OSVW)) { |
95 | u64 osvw_len; |
96 | |
97 | rdmsrl(MSR_AMD64_OSVW_ID_LENGTH, osvw_len); |
98 | if (osvw_id < osvw_len) { |
99 | u64 osvw_bits; |
100 | |
101 | rdmsrl(MSR_AMD64_OSVW_STATUS + (osvw_id >> 6), |
102 | osvw_bits); |
103 | return osvw_bits & (1ULL << (osvw_id & 0x3f)); |
104 | } |
105 | } |
106 | |
107 | /* OSVW unavailable or ID unknown, match family-model-stepping range */ |
108 | ms = (cpu->x86_model << 4) | cpu->x86_stepping; |
109 | while ((range = *erratum++)) |
110 | if ((cpu->x86 == AMD_MODEL_RANGE_FAMILY(range)) && |
111 | (ms >= AMD_MODEL_RANGE_START(range)) && |
112 | (ms <= AMD_MODEL_RANGE_END(range))) |
113 | return true; |
114 | |
115 | return false; |
116 | } |
117 | |
118 | static inline int rdmsrl_amd_safe(unsigned msr, unsigned long long *p) |
119 | { |
120 | u32 gprs[8] = { 0 }; |
121 | int err; |
122 | |
123 | WARN_ONCE((boot_cpu_data.x86 != 0xf), |
124 | "%s should only be used on K8!\n" , __func__); |
125 | |
126 | gprs[1] = msr; |
127 | gprs[7] = 0x9c5a203a; |
128 | |
129 | err = rdmsr_safe_regs(regs: gprs); |
130 | |
131 | *p = gprs[0] | ((u64)gprs[2] << 32); |
132 | |
133 | return err; |
134 | } |
135 | |
136 | static inline int wrmsrl_amd_safe(unsigned msr, unsigned long long val) |
137 | { |
138 | u32 gprs[8] = { 0 }; |
139 | |
140 | WARN_ONCE((boot_cpu_data.x86 != 0xf), |
141 | "%s should only be used on K8!\n" , __func__); |
142 | |
143 | gprs[0] = (u32)val; |
144 | gprs[1] = msr; |
145 | gprs[2] = val >> 32; |
146 | gprs[7] = 0x9c5a203a; |
147 | |
148 | return wrmsr_safe_regs(regs: gprs); |
149 | } |
150 | |
151 | /* |
152 | * B step AMD K6 before B 9730xxxx have hardware bugs that can cause |
153 | * misexecution of code under Linux. Owners of such processors should |
154 | * contact AMD for precise details and a CPU swap. |
155 | * |
156 | * See http://www.multimania.com/poulot/k6bug.html |
157 | * and section 2.6.2 of "AMD-K6 Processor Revision Guide - Model 6" |
158 | * (Publication # 21266 Issue Date: August 1998) |
159 | * |
160 | * The following test is erm.. interesting. AMD neglected to up |
161 | * the chip setting when fixing the bug but they also tweaked some |
162 | * performance at the same time.. |
163 | */ |
164 | |
165 | #ifdef CONFIG_X86_32 |
166 | extern __visible void vide(void); |
167 | __asm__(".text\n" |
168 | ".globl vide\n" |
169 | ".type vide, @function\n" |
170 | ".align 4\n" |
171 | "vide: ret\n" ); |
172 | #endif |
173 | |
174 | static void init_amd_k5(struct cpuinfo_x86 *c) |
175 | { |
176 | #ifdef CONFIG_X86_32 |
177 | /* |
178 | * General Systems BIOSen alias the cpu frequency registers |
179 | * of the Elan at 0x000df000. Unfortunately, one of the Linux |
180 | * drivers subsequently pokes it, and changes the CPU speed. |
181 | * Workaround : Remove the unneeded alias. |
182 | */ |
183 | #define CBAR (0xfffc) /* Configuration Base Address (32-bit) */ |
184 | #define CBAR_ENB (0x80000000) |
185 | #define CBAR_KEY (0X000000CB) |
186 | if (c->x86_model == 9 || c->x86_model == 10) { |
187 | if (inl(CBAR) & CBAR_ENB) |
188 | outl(0 | CBAR_KEY, CBAR); |
189 | } |
190 | #endif |
191 | } |
192 | |
193 | static void init_amd_k6(struct cpuinfo_x86 *c) |
194 | { |
195 | #ifdef CONFIG_X86_32 |
196 | u32 l, h; |
197 | int mbytes = get_num_physpages() >> (20-PAGE_SHIFT); |
198 | |
199 | if (c->x86_model < 6) { |
200 | /* Based on AMD doc 20734R - June 2000 */ |
201 | if (c->x86_model == 0) { |
202 | clear_cpu_cap(c, X86_FEATURE_APIC); |
203 | set_cpu_cap(c, X86_FEATURE_PGE); |
204 | } |
205 | return; |
206 | } |
207 | |
208 | if (c->x86_model == 6 && c->x86_stepping == 1) { |
209 | const int K6_BUG_LOOP = 1000000; |
210 | int n; |
211 | void (*f_vide)(void); |
212 | u64 d, d2; |
213 | |
214 | pr_info("AMD K6 stepping B detected - " ); |
215 | |
216 | /* |
217 | * It looks like AMD fixed the 2.6.2 bug and improved indirect |
218 | * calls at the same time. |
219 | */ |
220 | |
221 | n = K6_BUG_LOOP; |
222 | f_vide = vide; |
223 | OPTIMIZER_HIDE_VAR(f_vide); |
224 | d = rdtsc(); |
225 | while (n--) |
226 | f_vide(); |
227 | d2 = rdtsc(); |
228 | d = d2-d; |
229 | |
230 | if (d > 20*K6_BUG_LOOP) |
231 | pr_cont("system stability may be impaired when more than 32 MB are used.\n" ); |
232 | else |
233 | pr_cont("probably OK (after B9730xxxx).\n" ); |
234 | } |
235 | |
236 | /* K6 with old style WHCR */ |
237 | if (c->x86_model < 8 || |
238 | (c->x86_model == 8 && c->x86_stepping < 8)) { |
239 | /* We can only write allocate on the low 508Mb */ |
240 | if (mbytes > 508) |
241 | mbytes = 508; |
242 | |
243 | rdmsr(MSR_K6_WHCR, l, h); |
244 | if ((l&0x0000FFFF) == 0) { |
245 | unsigned long flags; |
246 | l = (1<<0)|((mbytes/4)<<1); |
247 | local_irq_save(flags); |
248 | wbinvd(); |
249 | wrmsr(MSR_K6_WHCR, l, h); |
250 | local_irq_restore(flags); |
251 | pr_info("Enabling old style K6 write allocation for %d Mb\n" , |
252 | mbytes); |
253 | } |
254 | return; |
255 | } |
256 | |
257 | if ((c->x86_model == 8 && c->x86_stepping > 7) || |
258 | c->x86_model == 9 || c->x86_model == 13) { |
259 | /* The more serious chips .. */ |
260 | |
261 | if (mbytes > 4092) |
262 | mbytes = 4092; |
263 | |
264 | rdmsr(MSR_K6_WHCR, l, h); |
265 | if ((l&0xFFFF0000) == 0) { |
266 | unsigned long flags; |
267 | l = ((mbytes>>2)<<22)|(1<<16); |
268 | local_irq_save(flags); |
269 | wbinvd(); |
270 | wrmsr(MSR_K6_WHCR, l, h); |
271 | local_irq_restore(flags); |
272 | pr_info("Enabling new style K6 write allocation for %d Mb\n" , |
273 | mbytes); |
274 | } |
275 | |
276 | return; |
277 | } |
278 | |
279 | if (c->x86_model == 10) { |
280 | /* AMD Geode LX is model 10 */ |
281 | /* placeholder for any needed mods */ |
282 | return; |
283 | } |
284 | #endif |
285 | } |
286 | |
287 | static void init_amd_k7(struct cpuinfo_x86 *c) |
288 | { |
289 | #ifdef CONFIG_X86_32 |
290 | u32 l, h; |
291 | |
292 | /* |
293 | * Bit 15 of Athlon specific MSR 15, needs to be 0 |
294 | * to enable SSE on Palomino/Morgan/Barton CPU's. |
295 | * If the BIOS didn't enable it already, enable it here. |
296 | */ |
297 | if (c->x86_model >= 6 && c->x86_model <= 10) { |
298 | if (!cpu_has(c, X86_FEATURE_XMM)) { |
299 | pr_info("Enabling disabled K7/SSE Support.\n" ); |
300 | msr_clear_bit(MSR_K7_HWCR, 15); |
301 | set_cpu_cap(c, X86_FEATURE_XMM); |
302 | } |
303 | } |
304 | |
305 | /* |
306 | * It's been determined by AMD that Athlons since model 8 stepping 1 |
307 | * are more robust with CLK_CTL set to 200xxxxx instead of 600xxxxx |
308 | * As per AMD technical note 27212 0.2 |
309 | */ |
310 | if ((c->x86_model == 8 && c->x86_stepping >= 1) || (c->x86_model > 8)) { |
311 | rdmsr(MSR_K7_CLK_CTL, l, h); |
312 | if ((l & 0xfff00000) != 0x20000000) { |
313 | pr_info("CPU: CLK_CTL MSR was %x. Reprogramming to %x\n" , |
314 | l, ((l & 0x000fffff)|0x20000000)); |
315 | wrmsr(MSR_K7_CLK_CTL, (l & 0x000fffff)|0x20000000, h); |
316 | } |
317 | } |
318 | |
319 | /* calling is from identify_secondary_cpu() ? */ |
320 | if (!c->cpu_index) |
321 | return; |
322 | |
323 | /* |
324 | * Certain Athlons might work (for various values of 'work') in SMP |
325 | * but they are not certified as MP capable. |
326 | */ |
327 | /* Athlon 660/661 is valid. */ |
328 | if ((c->x86_model == 6) && ((c->x86_stepping == 0) || |
329 | (c->x86_stepping == 1))) |
330 | return; |
331 | |
332 | /* Duron 670 is valid */ |
333 | if ((c->x86_model == 7) && (c->x86_stepping == 0)) |
334 | return; |
335 | |
336 | /* |
337 | * Athlon 662, Duron 671, and Athlon >model 7 have capability |
338 | * bit. It's worth noting that the A5 stepping (662) of some |
339 | * Athlon XP's have the MP bit set. |
340 | * See http://www.heise.de/newsticker/data/jow-18.10.01-000 for |
341 | * more. |
342 | */ |
343 | if (((c->x86_model == 6) && (c->x86_stepping >= 2)) || |
344 | ((c->x86_model == 7) && (c->x86_stepping >= 1)) || |
345 | (c->x86_model > 7)) |
346 | if (cpu_has(c, X86_FEATURE_MP)) |
347 | return; |
348 | |
349 | /* If we get here, not a certified SMP capable AMD system. */ |
350 | |
351 | /* |
352 | * Don't taint if we are running SMP kernel on a single non-MP |
353 | * approved Athlon |
354 | */ |
355 | WARN_ONCE(1, "WARNING: This combination of AMD" |
356 | " processors is not suitable for SMP.\n" ); |
357 | add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_NOW_UNRELIABLE); |
358 | #endif |
359 | } |
360 | |
361 | #ifdef CONFIG_NUMA |
362 | /* |
363 | * To workaround broken NUMA config. Read the comment in |
364 | * srat_detect_node(). |
365 | */ |
366 | static int nearby_node(int apicid) |
367 | { |
368 | int i, node; |
369 | |
370 | for (i = apicid - 1; i >= 0; i--) { |
371 | node = __apicid_to_node[i]; |
372 | if (node != NUMA_NO_NODE && node_online(node)) |
373 | return node; |
374 | } |
375 | for (i = apicid + 1; i < MAX_LOCAL_APIC; i++) { |
376 | node = __apicid_to_node[i]; |
377 | if (node != NUMA_NO_NODE && node_online(node)) |
378 | return node; |
379 | } |
380 | return first_node(node_online_map); /* Shouldn't happen */ |
381 | } |
382 | #endif |
383 | |
384 | /* |
385 | * Fix up topo::core_id for pre-F17h systems to be in the |
386 | * [0 .. cores_per_node - 1] range. Not really needed but |
387 | * kept so as not to break existing setups. |
388 | */ |
389 | static void legacy_fixup_core_id(struct cpuinfo_x86 *c) |
390 | { |
391 | u32 cus_per_node; |
392 | |
393 | if (c->x86 >= 0x17) |
394 | return; |
395 | |
396 | cus_per_node = c->x86_max_cores / nodes_per_socket; |
397 | c->topo.core_id %= cus_per_node; |
398 | } |
399 | |
400 | /* |
401 | * Fixup core topology information for |
402 | * (1) AMD multi-node processors |
403 | * Assumption: Number of cores in each internal node is the same. |
404 | * (2) AMD processors supporting compute units |
405 | */ |
406 | static void amd_get_topology(struct cpuinfo_x86 *c) |
407 | { |
408 | /* get information required for multi-node processors */ |
409 | if (boot_cpu_has(X86_FEATURE_TOPOEXT)) { |
410 | int err; |
411 | u32 eax, ebx, ecx, edx; |
412 | |
413 | cpuid(op: 0x8000001e, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx); |
414 | |
415 | c->topo.die_id = ecx & 0xff; |
416 | |
417 | if (c->x86 == 0x15) |
418 | c->topo.cu_id = ebx & 0xff; |
419 | |
420 | if (c->x86 >= 0x17) { |
421 | c->topo.core_id = ebx & 0xff; |
422 | |
423 | if (smp_num_siblings > 1) |
424 | c->x86_max_cores /= smp_num_siblings; |
425 | } |
426 | |
427 | /* |
428 | * In case leaf B is available, use it to derive |
429 | * topology information. |
430 | */ |
431 | err = detect_extended_topology(c); |
432 | if (!err) |
433 | c->x86_coreid_bits = get_count_order(count: c->x86_max_cores); |
434 | |
435 | cacheinfo_amd_init_llc_id(c); |
436 | |
437 | } else if (cpu_has(c, X86_FEATURE_NODEID_MSR)) { |
438 | u64 value; |
439 | |
440 | rdmsrl(MSR_FAM10H_NODE_ID, value); |
441 | c->topo.die_id = value & 7; |
442 | c->topo.llc_id = c->topo.die_id; |
443 | } else |
444 | return; |
445 | |
446 | if (nodes_per_socket > 1) { |
447 | set_cpu_cap(c, X86_FEATURE_AMD_DCM); |
448 | legacy_fixup_core_id(c); |
449 | } |
450 | } |
451 | |
452 | /* |
453 | * On a AMD dual core setup the lower bits of the APIC id distinguish the cores. |
454 | * Assumes number of cores is a power of two. |
455 | */ |
456 | static void amd_detect_cmp(struct cpuinfo_x86 *c) |
457 | { |
458 | unsigned bits; |
459 | |
460 | bits = c->x86_coreid_bits; |
461 | /* Low order bits define the core id (index of core in socket) */ |
462 | c->topo.core_id = c->topo.initial_apicid & ((1 << bits)-1); |
463 | /* Convert the initial APIC ID into the socket ID */ |
464 | c->topo.pkg_id = c->topo.initial_apicid >> bits; |
465 | /* use socket ID also for last level cache */ |
466 | c->topo.llc_id = c->topo.die_id = c->topo.pkg_id; |
467 | } |
468 | |
469 | u32 amd_get_nodes_per_socket(void) |
470 | { |
471 | return nodes_per_socket; |
472 | } |
473 | EXPORT_SYMBOL_GPL(amd_get_nodes_per_socket); |
474 | |
475 | static void srat_detect_node(struct cpuinfo_x86 *c) |
476 | { |
477 | #ifdef CONFIG_NUMA |
478 | int cpu = smp_processor_id(); |
479 | int node; |
480 | unsigned apicid = c->topo.apicid; |
481 | |
482 | node = numa_cpu_node(cpu); |
483 | if (node == NUMA_NO_NODE) |
484 | node = per_cpu_llc_id(cpu); |
485 | |
486 | /* |
487 | * On multi-fabric platform (e.g. Numascale NumaChip) a |
488 | * platform-specific handler needs to be called to fixup some |
489 | * IDs of the CPU. |
490 | */ |
491 | if (x86_cpuinit.fixup_cpu_id) |
492 | x86_cpuinit.fixup_cpu_id(c, node); |
493 | |
494 | if (!node_online(node)) { |
495 | /* |
496 | * Two possibilities here: |
497 | * |
498 | * - The CPU is missing memory and no node was created. In |
499 | * that case try picking one from a nearby CPU. |
500 | * |
501 | * - The APIC IDs differ from the HyperTransport node IDs |
502 | * which the K8 northbridge parsing fills in. Assume |
503 | * they are all increased by a constant offset, but in |
504 | * the same order as the HT nodeids. If that doesn't |
505 | * result in a usable node fall back to the path for the |
506 | * previous case. |
507 | * |
508 | * This workaround operates directly on the mapping between |
509 | * APIC ID and NUMA node, assuming certain relationship |
510 | * between APIC ID, HT node ID and NUMA topology. As going |
511 | * through CPU mapping may alter the outcome, directly |
512 | * access __apicid_to_node[]. |
513 | */ |
514 | int ht_nodeid = c->topo.initial_apicid; |
515 | |
516 | if (__apicid_to_node[ht_nodeid] != NUMA_NO_NODE) |
517 | node = __apicid_to_node[ht_nodeid]; |
518 | /* Pick a nearby node */ |
519 | if (!node_online(node)) |
520 | node = nearby_node(apicid); |
521 | } |
522 | numa_set_node(cpu, node); |
523 | #endif |
524 | } |
525 | |
526 | static void early_init_amd_mc(struct cpuinfo_x86 *c) |
527 | { |
528 | #ifdef CONFIG_SMP |
529 | unsigned bits, ecx; |
530 | |
531 | /* Multi core CPU? */ |
532 | if (c->extended_cpuid_level < 0x80000008) |
533 | return; |
534 | |
535 | ecx = cpuid_ecx(op: 0x80000008); |
536 | |
537 | c->x86_max_cores = (ecx & 0xff) + 1; |
538 | |
539 | /* CPU telling us the core id bits shift? */ |
540 | bits = (ecx >> 12) & 0xF; |
541 | |
542 | /* Otherwise recompute */ |
543 | if (bits == 0) { |
544 | while ((1 << bits) < c->x86_max_cores) |
545 | bits++; |
546 | } |
547 | |
548 | c->x86_coreid_bits = bits; |
549 | #endif |
550 | } |
551 | |
552 | static void bsp_init_amd(struct cpuinfo_x86 *c) |
553 | { |
554 | if (cpu_has(c, X86_FEATURE_CONSTANT_TSC)) { |
555 | |
556 | if (c->x86 > 0x10 || |
557 | (c->x86 == 0x10 && c->x86_model >= 0x2)) { |
558 | u64 val; |
559 | |
560 | rdmsrl(MSR_K7_HWCR, val); |
561 | if (!(val & BIT(24))) |
562 | pr_warn(FW_BUG "TSC doesn't count with P0 frequency!\n" ); |
563 | } |
564 | } |
565 | |
566 | if (c->x86 == 0x15) { |
567 | unsigned long upperbit; |
568 | u32 cpuid, assoc; |
569 | |
570 | cpuid = cpuid_edx(op: 0x80000005); |
571 | assoc = cpuid >> 16 & 0xff; |
572 | upperbit = ((cpuid >> 24) << 10) / assoc; |
573 | |
574 | va_align.mask = (upperbit - 1) & PAGE_MASK; |
575 | va_align.flags = ALIGN_VA_32 | ALIGN_VA_64; |
576 | |
577 | /* A random value per boot for bit slice [12:upper_bit) */ |
578 | va_align.bits = get_random_u32() & va_align.mask; |
579 | } |
580 | |
581 | if (cpu_has(c, X86_FEATURE_MWAITX)) |
582 | use_mwaitx_delay(); |
583 | |
584 | if (boot_cpu_has(X86_FEATURE_TOPOEXT)) { |
585 | u32 ecx; |
586 | |
587 | ecx = cpuid_ecx(op: 0x8000001e); |
588 | __max_die_per_package = nodes_per_socket = ((ecx >> 8) & 7) + 1; |
589 | } else if (boot_cpu_has(X86_FEATURE_NODEID_MSR)) { |
590 | u64 value; |
591 | |
592 | rdmsrl(MSR_FAM10H_NODE_ID, value); |
593 | __max_die_per_package = nodes_per_socket = ((value >> 3) & 7) + 1; |
594 | } |
595 | |
596 | if (!boot_cpu_has(X86_FEATURE_AMD_SSBD) && |
597 | !boot_cpu_has(X86_FEATURE_VIRT_SSBD) && |
598 | c->x86 >= 0x15 && c->x86 <= 0x17) { |
599 | unsigned int bit; |
600 | |
601 | switch (c->x86) { |
602 | case 0x15: bit = 54; break; |
603 | case 0x16: bit = 33; break; |
604 | case 0x17: bit = 10; break; |
605 | default: return; |
606 | } |
607 | /* |
608 | * Try to cache the base value so further operations can |
609 | * avoid RMW. If that faults, do not enable SSBD. |
610 | */ |
611 | if (!rdmsrl_safe(MSR_AMD64_LS_CFG, p: &x86_amd_ls_cfg_base)) { |
612 | setup_force_cpu_cap(X86_FEATURE_LS_CFG_SSBD); |
613 | setup_force_cpu_cap(X86_FEATURE_SSBD); |
614 | x86_amd_ls_cfg_ssbd_mask = 1ULL << bit; |
615 | } |
616 | } |
617 | |
618 | resctrl_cpu_detect(c); |
619 | } |
620 | |
621 | static void early_detect_mem_encrypt(struct cpuinfo_x86 *c) |
622 | { |
623 | u64 msr; |
624 | |
625 | /* |
626 | * BIOS support is required for SME and SEV. |
627 | * For SME: If BIOS has enabled SME then adjust x86_phys_bits by |
628 | * the SME physical address space reduction value. |
629 | * If BIOS has not enabled SME then don't advertise the |
630 | * SME feature (set in scattered.c). |
631 | * If the kernel has not enabled SME via any means then |
632 | * don't advertise the SME feature. |
633 | * For SEV: If BIOS has not enabled SEV then don't advertise the |
634 | * SEV and SEV_ES feature (set in scattered.c). |
635 | * |
636 | * In all cases, since support for SME and SEV requires long mode, |
637 | * don't advertise the feature under CONFIG_X86_32. |
638 | */ |
639 | if (cpu_has(c, X86_FEATURE_SME) || cpu_has(c, X86_FEATURE_SEV)) { |
640 | /* Check if memory encryption is enabled */ |
641 | rdmsrl(MSR_AMD64_SYSCFG, msr); |
642 | if (!(msr & MSR_AMD64_SYSCFG_MEM_ENCRYPT)) |
643 | goto clear_all; |
644 | |
645 | /* |
646 | * Always adjust physical address bits. Even though this |
647 | * will be a value above 32-bits this is still done for |
648 | * CONFIG_X86_32 so that accurate values are reported. |
649 | */ |
650 | c->x86_phys_bits -= (cpuid_ebx(op: 0x8000001f) >> 6) & 0x3f; |
651 | |
652 | if (IS_ENABLED(CONFIG_X86_32)) |
653 | goto clear_all; |
654 | |
655 | if (!sme_me_mask) |
656 | setup_clear_cpu_cap(X86_FEATURE_SME); |
657 | |
658 | rdmsrl(MSR_K7_HWCR, msr); |
659 | if (!(msr & MSR_K7_HWCR_SMMLOCK)) |
660 | goto clear_sev; |
661 | |
662 | return; |
663 | |
664 | clear_all: |
665 | setup_clear_cpu_cap(X86_FEATURE_SME); |
666 | clear_sev: |
667 | setup_clear_cpu_cap(X86_FEATURE_SEV); |
668 | setup_clear_cpu_cap(X86_FEATURE_SEV_ES); |
669 | } |
670 | } |
671 | |
672 | static void early_init_amd(struct cpuinfo_x86 *c) |
673 | { |
674 | u64 value; |
675 | u32 dummy; |
676 | |
677 | early_init_amd_mc(c); |
678 | |
679 | if (c->x86 >= 0xf) |
680 | set_cpu_cap(c, X86_FEATURE_K8); |
681 | |
682 | rdmsr_safe(MSR_AMD64_PATCH_LEVEL, &c->microcode, &dummy); |
683 | |
684 | /* |
685 | * c->x86_power is 8000_0007 edx. Bit 8 is TSC runs at constant rate |
686 | * with P/T states and does not stop in deep C-states |
687 | */ |
688 | if (c->x86_power & (1 << 8)) { |
689 | set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC); |
690 | set_cpu_cap(c, X86_FEATURE_NONSTOP_TSC); |
691 | } |
692 | |
693 | /* Bit 12 of 8000_0007 edx is accumulated power mechanism. */ |
694 | if (c->x86_power & BIT(12)) |
695 | set_cpu_cap(c, X86_FEATURE_ACC_POWER); |
696 | |
697 | /* Bit 14 indicates the Runtime Average Power Limit interface. */ |
698 | if (c->x86_power & BIT(14)) |
699 | set_cpu_cap(c, X86_FEATURE_RAPL); |
700 | |
701 | #ifdef CONFIG_X86_64 |
702 | set_cpu_cap(c, X86_FEATURE_SYSCALL32); |
703 | #else |
704 | /* Set MTRR capability flag if appropriate */ |
705 | if (c->x86 == 5) |
706 | if (c->x86_model == 13 || c->x86_model == 9 || |
707 | (c->x86_model == 8 && c->x86_stepping >= 8)) |
708 | set_cpu_cap(c, X86_FEATURE_K6_MTRR); |
709 | #endif |
710 | #if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_PCI) |
711 | /* |
712 | * ApicID can always be treated as an 8-bit value for AMD APIC versions |
713 | * >= 0x10, but even old K8s came out of reset with version 0x10. So, we |
714 | * can safely set X86_FEATURE_EXTD_APICID unconditionally for families |
715 | * after 16h. |
716 | */ |
717 | if (boot_cpu_has(X86_FEATURE_APIC)) { |
718 | if (c->x86 > 0x16) |
719 | set_cpu_cap(c, X86_FEATURE_EXTD_APICID); |
720 | else if (c->x86 >= 0xf) { |
721 | /* check CPU config space for extended APIC ID */ |
722 | unsigned int val; |
723 | |
724 | val = read_pci_config(bus: 0, slot: 24, func: 0, offset: 0x68); |
725 | if ((val >> 17 & 0x3) == 0x3) |
726 | set_cpu_cap(c, X86_FEATURE_EXTD_APICID); |
727 | } |
728 | } |
729 | #endif |
730 | |
731 | /* |
732 | * This is only needed to tell the kernel whether to use VMCALL |
733 | * and VMMCALL. VMMCALL is never executed except under virt, so |
734 | * we can set it unconditionally. |
735 | */ |
736 | set_cpu_cap(c, X86_FEATURE_VMMCALL); |
737 | |
738 | /* F16h erratum 793, CVE-2013-6885 */ |
739 | if (c->x86 == 0x16 && c->x86_model <= 0xf) |
740 | msr_set_bit(MSR_AMD64_LS_CFG, bit: 15); |
741 | |
742 | /* |
743 | * Check whether the machine is affected by erratum 400. This is |
744 | * used to select the proper idle routine and to enable the check |
745 | * whether the machine is affected in arch_post_acpi_init(), which |
746 | * sets the X86_BUG_AMD_APIC_C1E bug depending on the MSR check. |
747 | */ |
748 | if (cpu_has_amd_erratum(cpu: c, erratum: amd_erratum_400)) |
749 | set_cpu_bug(c, X86_BUG_AMD_E400); |
750 | |
751 | early_detect_mem_encrypt(c); |
752 | |
753 | /* Re-enable TopologyExtensions if switched off by BIOS */ |
754 | if (c->x86 == 0x15 && |
755 | (c->x86_model >= 0x10 && c->x86_model <= 0x6f) && |
756 | !cpu_has(c, X86_FEATURE_TOPOEXT)) { |
757 | |
758 | if (msr_set_bit(msr: 0xc0011005, bit: 54) > 0) { |
759 | rdmsrl(0xc0011005, value); |
760 | if (value & BIT_64(54)) { |
761 | set_cpu_cap(c, X86_FEATURE_TOPOEXT); |
762 | pr_info_once(FW_INFO "CPU: Re-enabling disabled Topology Extensions Support.\n" ); |
763 | } |
764 | } |
765 | } |
766 | |
767 | if (cpu_has(c, X86_FEATURE_TOPOEXT)) |
768 | smp_num_siblings = ((cpuid_ebx(op: 0x8000001e) >> 8) & 0xff) + 1; |
769 | |
770 | if (!cpu_has(c, X86_FEATURE_HYPERVISOR) && !cpu_has(c, X86_FEATURE_IBPB_BRTYPE)) { |
771 | if (c->x86 == 0x17 && boot_cpu_has(X86_FEATURE_AMD_IBPB)) |
772 | setup_force_cpu_cap(X86_FEATURE_IBPB_BRTYPE); |
773 | else if (c->x86 >= 0x19 && !wrmsrl_safe(MSR_IA32_PRED_CMD, PRED_CMD_SBPB)) { |
774 | setup_force_cpu_cap(X86_FEATURE_IBPB_BRTYPE); |
775 | setup_force_cpu_cap(X86_FEATURE_SBPB); |
776 | } |
777 | } |
778 | } |
779 | |
780 | static void init_amd_k8(struct cpuinfo_x86 *c) |
781 | { |
782 | u32 level; |
783 | u64 value; |
784 | |
785 | /* On C+ stepping K8 rep microcode works well for copy/memset */ |
786 | level = cpuid_eax(op: 1); |
787 | if ((level >= 0x0f48 && level < 0x0f50) || level >= 0x0f58) |
788 | set_cpu_cap(c, X86_FEATURE_REP_GOOD); |
789 | |
790 | /* |
791 | * Some BIOSes incorrectly force this feature, but only K8 revision D |
792 | * (model = 0x14) and later actually support it. |
793 | * (AMD Erratum #110, docId: 25759). |
794 | */ |
795 | if (c->x86_model < 0x14 && cpu_has(c, X86_FEATURE_LAHF_LM)) { |
796 | clear_cpu_cap(c, X86_FEATURE_LAHF_LM); |
797 | if (!rdmsrl_amd_safe(msr: 0xc001100d, p: &value)) { |
798 | value &= ~BIT_64(32); |
799 | wrmsrl_amd_safe(msr: 0xc001100d, val: value); |
800 | } |
801 | } |
802 | |
803 | if (!c->x86_model_id[0]) |
804 | strcpy(p: c->x86_model_id, q: "Hammer" ); |
805 | |
806 | #ifdef CONFIG_SMP |
807 | /* |
808 | * Disable TLB flush filter by setting HWCR.FFDIS on K8 |
809 | * bit 6 of msr C001_0015 |
810 | * |
811 | * Errata 63 for SH-B3 steppings |
812 | * Errata 122 for all steppings (F+ have it disabled by default) |
813 | */ |
814 | msr_set_bit(MSR_K7_HWCR, bit: 6); |
815 | #endif |
816 | set_cpu_bug(c, X86_BUG_SWAPGS_FENCE); |
817 | } |
818 | |
819 | static void init_amd_gh(struct cpuinfo_x86 *c) |
820 | { |
821 | #ifdef CONFIG_MMCONF_FAM10H |
822 | /* do this for boot cpu */ |
823 | if (c == &boot_cpu_data) |
824 | check_enable_amd_mmconf_dmi(); |
825 | |
826 | fam10h_check_enable_mmcfg(); |
827 | #endif |
828 | |
829 | /* |
830 | * Disable GART TLB Walk Errors on Fam10h. We do this here because this |
831 | * is always needed when GART is enabled, even in a kernel which has no |
832 | * MCE support built in. BIOS should disable GartTlbWlk Errors already. |
833 | * If it doesn't, we do it here as suggested by the BKDG. |
834 | * |
835 | * Fixes: https://bugzilla.kernel.org/show_bug.cgi?id=33012 |
836 | */ |
837 | msr_set_bit(MSR_AMD64_MCx_MASK(4), bit: 10); |
838 | |
839 | /* |
840 | * On family 10h BIOS may not have properly enabled WC+ support, causing |
841 | * it to be converted to CD memtype. This may result in performance |
842 | * degradation for certain nested-paging guests. Prevent this conversion |
843 | * by clearing bit 24 in MSR_AMD64_BU_CFG2. |
844 | * |
845 | * NOTE: we want to use the _safe accessors so as not to #GP kvm |
846 | * guests on older kvm hosts. |
847 | */ |
848 | msr_clear_bit(MSR_AMD64_BU_CFG2, bit: 24); |
849 | |
850 | if (cpu_has_amd_erratum(cpu: c, erratum: amd_erratum_383)) |
851 | set_cpu_bug(c, X86_BUG_AMD_TLB_MMATCH); |
852 | } |
853 | |
854 | static void init_amd_ln(struct cpuinfo_x86 *c) |
855 | { |
856 | /* |
857 | * Apply erratum 665 fix unconditionally so machines without a BIOS |
858 | * fix work. |
859 | */ |
860 | msr_set_bit(MSR_AMD64_DE_CFG, bit: 31); |
861 | } |
862 | |
863 | static bool rdrand_force; |
864 | |
865 | static int __init rdrand_cmdline(char *str) |
866 | { |
867 | if (!str) |
868 | return -EINVAL; |
869 | |
870 | if (!strcmp(str, "force" )) |
871 | rdrand_force = true; |
872 | else |
873 | return -EINVAL; |
874 | |
875 | return 0; |
876 | } |
877 | early_param("rdrand" , rdrand_cmdline); |
878 | |
879 | static void clear_rdrand_cpuid_bit(struct cpuinfo_x86 *c) |
880 | { |
881 | /* |
882 | * Saving of the MSR used to hide the RDRAND support during |
883 | * suspend/resume is done by arch/x86/power/cpu.c, which is |
884 | * dependent on CONFIG_PM_SLEEP. |
885 | */ |
886 | if (!IS_ENABLED(CONFIG_PM_SLEEP)) |
887 | return; |
888 | |
889 | /* |
890 | * The self-test can clear X86_FEATURE_RDRAND, so check for |
891 | * RDRAND support using the CPUID function directly. |
892 | */ |
893 | if (!(cpuid_ecx(op: 1) & BIT(30)) || rdrand_force) |
894 | return; |
895 | |
896 | msr_clear_bit(MSR_AMD64_CPUID_FN_1, bit: 62); |
897 | |
898 | /* |
899 | * Verify that the CPUID change has occurred in case the kernel is |
900 | * running virtualized and the hypervisor doesn't support the MSR. |
901 | */ |
902 | if (cpuid_ecx(op: 1) & BIT(30)) { |
903 | pr_info_once("BIOS may not properly restore RDRAND after suspend, but hypervisor does not support hiding RDRAND via CPUID.\n" ); |
904 | return; |
905 | } |
906 | |
907 | clear_cpu_cap(c, X86_FEATURE_RDRAND); |
908 | pr_info_once("BIOS may not properly restore RDRAND after suspend, hiding RDRAND via CPUID. Use rdrand=force to reenable.\n" ); |
909 | } |
910 | |
911 | static void init_amd_jg(struct cpuinfo_x86 *c) |
912 | { |
913 | /* |
914 | * Some BIOS implementations do not restore proper RDRAND support |
915 | * across suspend and resume. Check on whether to hide the RDRAND |
916 | * instruction support via CPUID. |
917 | */ |
918 | clear_rdrand_cpuid_bit(c); |
919 | } |
920 | |
921 | static void init_amd_bd(struct cpuinfo_x86 *c) |
922 | { |
923 | u64 value; |
924 | |
925 | /* |
926 | * The way access filter has a performance penalty on some workloads. |
927 | * Disable it on the affected CPUs. |
928 | */ |
929 | if ((c->x86_model >= 0x02) && (c->x86_model < 0x20)) { |
930 | if (!rdmsrl_safe(MSR_F15H_IC_CFG, p: &value) && !(value & 0x1E)) { |
931 | value |= 0x1E; |
932 | wrmsrl_safe(MSR_F15H_IC_CFG, val: value); |
933 | } |
934 | } |
935 | |
936 | /* |
937 | * Some BIOS implementations do not restore proper RDRAND support |
938 | * across suspend and resume. Check on whether to hide the RDRAND |
939 | * instruction support via CPUID. |
940 | */ |
941 | clear_rdrand_cpuid_bit(c); |
942 | } |
943 | |
944 | void init_spectral_chicken(struct cpuinfo_x86 *c) |
945 | { |
946 | #ifdef CONFIG_CPU_UNRET_ENTRY |
947 | u64 value; |
948 | |
949 | /* |
950 | * On Zen2 we offer this chicken (bit) on the altar of Speculation. |
951 | * |
952 | * This suppresses speculation from the middle of a basic block, i.e. it |
953 | * suppresses non-branch predictions. |
954 | * |
955 | * We use STIBP as a heuristic to filter out Zen2 from the rest of F17H |
956 | */ |
957 | if (!cpu_has(c, X86_FEATURE_HYPERVISOR) && cpu_has(c, X86_FEATURE_AMD_STIBP)) { |
958 | if (!rdmsrl_safe(MSR_ZEN2_SPECTRAL_CHICKEN, p: &value)) { |
959 | value |= MSR_ZEN2_SPECTRAL_CHICKEN_BIT; |
960 | wrmsrl_safe(MSR_ZEN2_SPECTRAL_CHICKEN, val: value); |
961 | } |
962 | } |
963 | #endif |
964 | /* |
965 | * Work around Erratum 1386. The XSAVES instruction malfunctions in |
966 | * certain circumstances on Zen1/2 uarch, and not all parts have had |
967 | * updated microcode at the time of writing (March 2023). |
968 | * |
969 | * Affected parts all have no supervisor XSAVE states, meaning that |
970 | * the XSAVEC instruction (which works fine) is equivalent. |
971 | */ |
972 | clear_cpu_cap(c, X86_FEATURE_XSAVES); |
973 | } |
974 | |
975 | static void init_amd_zn(struct cpuinfo_x86 *c) |
976 | { |
977 | set_cpu_cap(c, X86_FEATURE_ZEN); |
978 | |
979 | #ifdef CONFIG_NUMA |
980 | node_reclaim_distance = 32; |
981 | #endif |
982 | |
983 | /* Fix up CPUID bits, but only if not virtualised. */ |
984 | if (!cpu_has(c, X86_FEATURE_HYPERVISOR)) { |
985 | |
986 | /* Erratum 1076: CPB feature bit not being set in CPUID. */ |
987 | if (!cpu_has(c, X86_FEATURE_CPB)) |
988 | set_cpu_cap(c, X86_FEATURE_CPB); |
989 | |
990 | /* |
991 | * Zen3 (Fam19 model < 0x10) parts are not susceptible to |
992 | * Branch Type Confusion, but predate the allocation of the |
993 | * BTC_NO bit. |
994 | */ |
995 | if (c->x86 == 0x19 && !cpu_has(c, X86_FEATURE_BTC_NO)) |
996 | set_cpu_cap(c, X86_FEATURE_BTC_NO); |
997 | } |
998 | } |
999 | |
1000 | static bool cpu_has_zenbleed_microcode(void) |
1001 | { |
1002 | u32 good_rev = 0; |
1003 | |
1004 | switch (boot_cpu_data.x86_model) { |
1005 | case 0x30 ... 0x3f: good_rev = 0x0830107a; break; |
1006 | case 0x60 ... 0x67: good_rev = 0x0860010b; break; |
1007 | case 0x68 ... 0x6f: good_rev = 0x08608105; break; |
1008 | case 0x70 ... 0x7f: good_rev = 0x08701032; break; |
1009 | case 0xa0 ... 0xaf: good_rev = 0x08a00008; break; |
1010 | |
1011 | default: |
1012 | return false; |
1013 | } |
1014 | |
1015 | if (boot_cpu_data.microcode < good_rev) |
1016 | return false; |
1017 | |
1018 | return true; |
1019 | } |
1020 | |
1021 | static void zenbleed_check(struct cpuinfo_x86 *c) |
1022 | { |
1023 | if (!cpu_has_amd_erratum(cpu: c, erratum: amd_zenbleed)) |
1024 | return; |
1025 | |
1026 | if (cpu_has(c, X86_FEATURE_HYPERVISOR)) |
1027 | return; |
1028 | |
1029 | if (!cpu_has(c, X86_FEATURE_AVX)) |
1030 | return; |
1031 | |
1032 | if (!cpu_has_zenbleed_microcode()) { |
1033 | pr_notice_once("Zenbleed: please update your microcode for the most optimal fix\n" ); |
1034 | msr_set_bit(MSR_AMD64_DE_CFG, MSR_AMD64_DE_CFG_ZEN2_FP_BACKUP_FIX_BIT); |
1035 | } else { |
1036 | msr_clear_bit(MSR_AMD64_DE_CFG, MSR_AMD64_DE_CFG_ZEN2_FP_BACKUP_FIX_BIT); |
1037 | } |
1038 | } |
1039 | |
1040 | static void init_amd(struct cpuinfo_x86 *c) |
1041 | { |
1042 | u64 vm_cr; |
1043 | |
1044 | early_init_amd(c); |
1045 | |
1046 | /* |
1047 | * Bit 31 in normal CPUID used for nonstandard 3DNow ID; |
1048 | * 3DNow is IDd by bit 31 in extended CPUID (1*32+31) anyway |
1049 | */ |
1050 | clear_cpu_cap(c, bit: 0*32+31); |
1051 | |
1052 | if (c->x86 >= 0x10) |
1053 | set_cpu_cap(c, X86_FEATURE_REP_GOOD); |
1054 | |
1055 | /* AMD FSRM also implies FSRS */ |
1056 | if (cpu_has(c, X86_FEATURE_FSRM)) |
1057 | set_cpu_cap(c, X86_FEATURE_FSRS); |
1058 | |
1059 | /* get apicid instead of initial apic id from cpuid */ |
1060 | c->topo.apicid = read_apic_id(); |
1061 | |
1062 | /* K6s reports MCEs but don't actually have all the MSRs */ |
1063 | if (c->x86 < 6) |
1064 | clear_cpu_cap(c, X86_FEATURE_MCE); |
1065 | |
1066 | switch (c->x86) { |
1067 | case 4: init_amd_k5(c); break; |
1068 | case 5: init_amd_k6(c); break; |
1069 | case 6: init_amd_k7(c); break; |
1070 | case 0xf: init_amd_k8(c); break; |
1071 | case 0x10: init_amd_gh(c); break; |
1072 | case 0x12: init_amd_ln(c); break; |
1073 | case 0x15: init_amd_bd(c); break; |
1074 | case 0x16: init_amd_jg(c); break; |
1075 | case 0x17: init_spectral_chicken(c); |
1076 | fallthrough; |
1077 | case 0x19: init_amd_zn(c); break; |
1078 | } |
1079 | |
1080 | /* |
1081 | * Enable workaround for FXSAVE leak on CPUs |
1082 | * without a XSaveErPtr feature |
1083 | */ |
1084 | if ((c->x86 >= 6) && (!cpu_has(c, X86_FEATURE_XSAVEERPTR))) |
1085 | set_cpu_bug(c, X86_BUG_FXSAVE_LEAK); |
1086 | |
1087 | cpu_detect_cache_sizes(c); |
1088 | |
1089 | amd_detect_cmp(c); |
1090 | amd_get_topology(c); |
1091 | srat_detect_node(c); |
1092 | |
1093 | init_amd_cacheinfo(c); |
1094 | |
1095 | if (cpu_has(c, X86_FEATURE_SVM)) { |
1096 | rdmsrl(MSR_VM_CR, vm_cr); |
1097 | if (vm_cr & SVM_VM_CR_SVM_DIS_MASK) { |
1098 | pr_notice_once("SVM disabled (by BIOS) in MSR_VM_CR\n" ); |
1099 | clear_cpu_cap(c, X86_FEATURE_SVM); |
1100 | } |
1101 | } |
1102 | |
1103 | if (!cpu_has(c, X86_FEATURE_LFENCE_RDTSC) && cpu_has(c, X86_FEATURE_XMM2)) { |
1104 | /* |
1105 | * Use LFENCE for execution serialization. On families which |
1106 | * don't have that MSR, LFENCE is already serializing. |
1107 | * msr_set_bit() uses the safe accessors, too, even if the MSR |
1108 | * is not present. |
1109 | */ |
1110 | msr_set_bit(MSR_AMD64_DE_CFG, |
1111 | MSR_AMD64_DE_CFG_LFENCE_SERIALIZE_BIT); |
1112 | |
1113 | /* A serializing LFENCE stops RDTSC speculation */ |
1114 | set_cpu_cap(c, X86_FEATURE_LFENCE_RDTSC); |
1115 | } |
1116 | |
1117 | /* |
1118 | * Family 0x12 and above processors have APIC timer |
1119 | * running in deep C states. |
1120 | */ |
1121 | if (c->x86 > 0x11) |
1122 | set_cpu_cap(c, X86_FEATURE_ARAT); |
1123 | |
1124 | /* 3DNow or LM implies PREFETCHW */ |
1125 | if (!cpu_has(c, X86_FEATURE_3DNOWPREFETCH)) |
1126 | if (cpu_has(c, X86_FEATURE_3DNOW) || cpu_has(c, X86_FEATURE_LM)) |
1127 | set_cpu_cap(c, X86_FEATURE_3DNOWPREFETCH); |
1128 | |
1129 | /* AMD CPUs don't reset SS attributes on SYSRET, Xen does. */ |
1130 | if (!cpu_feature_enabled(X86_FEATURE_XENPV)) |
1131 | set_cpu_bug(c, X86_BUG_SYSRET_SS_ATTRS); |
1132 | |
1133 | /* |
1134 | * Turn on the Instructions Retired free counter on machines not |
1135 | * susceptible to erratum #1054 "Instructions Retired Performance |
1136 | * Counter May Be Inaccurate". |
1137 | */ |
1138 | if (cpu_has(c, X86_FEATURE_IRPERF) && |
1139 | !cpu_has_amd_erratum(cpu: c, erratum: amd_erratum_1054)) |
1140 | msr_set_bit(MSR_K7_HWCR, MSR_K7_HWCR_IRPERF_EN_BIT); |
1141 | |
1142 | check_null_seg_clears_base(c); |
1143 | |
1144 | /* |
1145 | * Make sure EFER[AIBRSE - Automatic IBRS Enable] is set. The APs are brought up |
1146 | * using the trampoline code and as part of it, MSR_EFER gets prepared there in |
1147 | * order to be replicated onto them. Regardless, set it here again, if not set, |
1148 | * to protect against any future refactoring/code reorganization which might |
1149 | * miss setting this important bit. |
1150 | */ |
1151 | if (spectre_v2_in_eibrs_mode(mode: spectre_v2_enabled) && |
1152 | cpu_has(c, X86_FEATURE_AUTOIBRS)) |
1153 | WARN_ON_ONCE(msr_set_bit(MSR_EFER, _EFER_AUTOIBRS)); |
1154 | |
1155 | zenbleed_check(c); |
1156 | |
1157 | if (cpu_has_amd_erratum(cpu: c, erratum: amd_div0)) { |
1158 | pr_notice_once("AMD Zen1 DIV0 bug detected. Disable SMT for full protection.\n" ); |
1159 | setup_force_cpu_bug(X86_BUG_DIV0); |
1160 | } |
1161 | |
1162 | if (!cpu_has(c, X86_FEATURE_HYPERVISOR) && |
1163 | cpu_has_amd_erratum(cpu: c, erratum: amd_erratum_1485)) |
1164 | msr_set_bit(MSR_ZEN4_BP_CFG, MSR_ZEN4_BP_CFG_SHARED_BTB_FIX_BIT); |
1165 | } |
1166 | |
1167 | #ifdef CONFIG_X86_32 |
1168 | static unsigned int amd_size_cache(struct cpuinfo_x86 *c, unsigned int size) |
1169 | { |
1170 | /* AMD errata T13 (order #21922) */ |
1171 | if (c->x86 == 6) { |
1172 | /* Duron Rev A0 */ |
1173 | if (c->x86_model == 3 && c->x86_stepping == 0) |
1174 | size = 64; |
1175 | /* Tbird rev A1/A2 */ |
1176 | if (c->x86_model == 4 && |
1177 | (c->x86_stepping == 0 || c->x86_stepping == 1)) |
1178 | size = 256; |
1179 | } |
1180 | return size; |
1181 | } |
1182 | #endif |
1183 | |
1184 | static void cpu_detect_tlb_amd(struct cpuinfo_x86 *c) |
1185 | { |
1186 | u32 ebx, eax, ecx, edx; |
1187 | u16 mask = 0xfff; |
1188 | |
1189 | if (c->x86 < 0xf) |
1190 | return; |
1191 | |
1192 | if (c->extended_cpuid_level < 0x80000006) |
1193 | return; |
1194 | |
1195 | cpuid(op: 0x80000006, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx); |
1196 | |
1197 | tlb_lld_4k[ENTRIES] = (ebx >> 16) & mask; |
1198 | tlb_lli_4k[ENTRIES] = ebx & mask; |
1199 | |
1200 | /* |
1201 | * K8 doesn't have 2M/4M entries in the L2 TLB so read out the L1 TLB |
1202 | * characteristics from the CPUID function 0x80000005 instead. |
1203 | */ |
1204 | if (c->x86 == 0xf) { |
1205 | cpuid(op: 0x80000005, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx); |
1206 | mask = 0xff; |
1207 | } |
1208 | |
1209 | /* Handle DTLB 2M and 4M sizes, fall back to L1 if L2 is disabled */ |
1210 | if (!((eax >> 16) & mask)) |
1211 | tlb_lld_2m[ENTRIES] = (cpuid_eax(op: 0x80000005) >> 16) & 0xff; |
1212 | else |
1213 | tlb_lld_2m[ENTRIES] = (eax >> 16) & mask; |
1214 | |
1215 | /* a 4M entry uses two 2M entries */ |
1216 | tlb_lld_4m[ENTRIES] = tlb_lld_2m[ENTRIES] >> 1; |
1217 | |
1218 | /* Handle ITLB 2M and 4M sizes, fall back to L1 if L2 is disabled */ |
1219 | if (!(eax & mask)) { |
1220 | /* Erratum 658 */ |
1221 | if (c->x86 == 0x15 && c->x86_model <= 0x1f) { |
1222 | tlb_lli_2m[ENTRIES] = 1024; |
1223 | } else { |
1224 | cpuid(op: 0x80000005, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx); |
1225 | tlb_lli_2m[ENTRIES] = eax & 0xff; |
1226 | } |
1227 | } else |
1228 | tlb_lli_2m[ENTRIES] = eax & mask; |
1229 | |
1230 | tlb_lli_4m[ENTRIES] = tlb_lli_2m[ENTRIES] >> 1; |
1231 | } |
1232 | |
1233 | static const struct cpu_dev amd_cpu_dev = { |
1234 | .c_vendor = "AMD" , |
1235 | .c_ident = { "AuthenticAMD" }, |
1236 | #ifdef CONFIG_X86_32 |
1237 | .legacy_models = { |
1238 | { .family = 4, .model_names = |
1239 | { |
1240 | [3] = "486 DX/2" , |
1241 | [7] = "486 DX/2-WB" , |
1242 | [8] = "486 DX/4" , |
1243 | [9] = "486 DX/4-WB" , |
1244 | [14] = "Am5x86-WT" , |
1245 | [15] = "Am5x86-WB" |
1246 | } |
1247 | }, |
1248 | }, |
1249 | .legacy_cache_size = amd_size_cache, |
1250 | #endif |
1251 | .c_early_init = early_init_amd, |
1252 | .c_detect_tlb = cpu_detect_tlb_amd, |
1253 | .c_bsp_init = bsp_init_amd, |
1254 | .c_init = init_amd, |
1255 | .c_x86_vendor = X86_VENDOR_AMD, |
1256 | }; |
1257 | |
1258 | cpu_dev_register(amd_cpu_dev); |
1259 | |
1260 | static DEFINE_PER_CPU_READ_MOSTLY(unsigned long[4], amd_dr_addr_mask); |
1261 | |
1262 | static unsigned int amd_msr_dr_addr_masks[] = { |
1263 | MSR_F16H_DR0_ADDR_MASK, |
1264 | MSR_F16H_DR1_ADDR_MASK, |
1265 | MSR_F16H_DR1_ADDR_MASK + 1, |
1266 | MSR_F16H_DR1_ADDR_MASK + 2 |
1267 | }; |
1268 | |
1269 | void amd_set_dr_addr_mask(unsigned long mask, unsigned int dr) |
1270 | { |
1271 | int cpu = smp_processor_id(); |
1272 | |
1273 | if (!cpu_feature_enabled(X86_FEATURE_BPEXT)) |
1274 | return; |
1275 | |
1276 | if (WARN_ON_ONCE(dr >= ARRAY_SIZE(amd_msr_dr_addr_masks))) |
1277 | return; |
1278 | |
1279 | if (per_cpu(amd_dr_addr_mask, cpu)[dr] == mask) |
1280 | return; |
1281 | |
1282 | wrmsr(amd_msr_dr_addr_masks[dr], mask, 0); |
1283 | per_cpu(amd_dr_addr_mask, cpu)[dr] = mask; |
1284 | } |
1285 | |
1286 | unsigned long amd_get_dr_addr_mask(unsigned int dr) |
1287 | { |
1288 | if (!cpu_feature_enabled(X86_FEATURE_BPEXT)) |
1289 | return 0; |
1290 | |
1291 | if (WARN_ON_ONCE(dr >= ARRAY_SIZE(amd_msr_dr_addr_masks))) |
1292 | return 0; |
1293 | |
1294 | return per_cpu(amd_dr_addr_mask[dr], smp_processor_id()); |
1295 | } |
1296 | EXPORT_SYMBOL_GPL(amd_get_dr_addr_mask); |
1297 | |
1298 | u32 amd_get_highest_perf(void) |
1299 | { |
1300 | struct cpuinfo_x86 *c = &boot_cpu_data; |
1301 | |
1302 | if (c->x86 == 0x17 && ((c->x86_model >= 0x30 && c->x86_model < 0x40) || |
1303 | (c->x86_model >= 0x70 && c->x86_model < 0x80))) |
1304 | return 166; |
1305 | |
1306 | if (c->x86 == 0x19 && ((c->x86_model >= 0x20 && c->x86_model < 0x30) || |
1307 | (c->x86_model >= 0x40 && c->x86_model < 0x70))) |
1308 | return 166; |
1309 | |
1310 | return 255; |
1311 | } |
1312 | EXPORT_SYMBOL_GPL(amd_get_highest_perf); |
1313 | |
1314 | static void zenbleed_check_cpu(void *unused) |
1315 | { |
1316 | struct cpuinfo_x86 *c = &cpu_data(smp_processor_id()); |
1317 | |
1318 | zenbleed_check(c); |
1319 | } |
1320 | |
1321 | void amd_check_microcode(void) |
1322 | { |
1323 | on_each_cpu(func: zenbleed_check_cpu, NULL, wait: 1); |
1324 | } |
1325 | |
1326 | /* |
1327 | * Issue a DIV 0/1 insn to clear any division data from previous DIV |
1328 | * operations. |
1329 | */ |
1330 | void noinstr amd_clear_divider(void) |
1331 | { |
1332 | asm volatile(ALTERNATIVE("" , "div %2\n\t" , X86_BUG_DIV0) |
1333 | :: "a" (0), "d" (0), "r" (1)); |
1334 | } |
1335 | EXPORT_SYMBOL_GPL(amd_clear_divider); |
1336 | |