1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * Copyright (C) 2001 Ben. Herrenschmidt (benh@kernel.crashing.org) |
4 | * |
5 | * Modifications for ppc64: |
6 | * Copyright (C) 2003 Dave Engebretsen <engebret@us.ibm.com> |
7 | * |
8 | * Copyright 2008 Michael Ellerman, IBM Corporation. |
9 | */ |
10 | |
11 | #include <linux/types.h> |
12 | #include <linux/jump_label.h> |
13 | #include <linux/kernel.h> |
14 | #include <linux/string.h> |
15 | #include <linux/init.h> |
16 | #include <linux/sched/mm.h> |
17 | #include <linux/stop_machine.h> |
18 | #include <asm/cputable.h> |
19 | #include <asm/code-patching.h> |
20 | #include <asm/interrupt.h> |
21 | #include <asm/page.h> |
22 | #include <asm/sections.h> |
23 | #include <asm/setup.h> |
24 | #include <asm/security_features.h> |
25 | #include <asm/firmware.h> |
26 | #include <asm/inst.h> |
27 | |
28 | struct fixup_entry { |
29 | unsigned long mask; |
30 | unsigned long value; |
31 | long start_off; |
32 | long end_off; |
33 | long alt_start_off; |
34 | long alt_end_off; |
35 | }; |
36 | |
37 | static u32 *calc_addr(struct fixup_entry *fcur, long offset) |
38 | { |
39 | /* |
40 | * We store the offset to the code as a negative offset from |
41 | * the start of the alt_entry, to support the VDSO. This |
42 | * routine converts that back into an actual address. |
43 | */ |
44 | return (u32 *)((unsigned long)fcur + offset); |
45 | } |
46 | |
47 | static int patch_alt_instruction(u32 *src, u32 *dest, u32 *alt_start, u32 *alt_end) |
48 | { |
49 | int err; |
50 | ppc_inst_t instr; |
51 | |
52 | instr = ppc_inst_read(src); |
53 | |
54 | if (instr_is_relative_branch(ppc_inst_read(src))) { |
55 | u32 *target = (u32 *)branch_target(src); |
56 | |
57 | /* Branch within the section doesn't need translating */ |
58 | if (target < alt_start || target > alt_end) { |
59 | err = translate_branch(&instr, dest, src); |
60 | if (err) |
61 | return 1; |
62 | } |
63 | } |
64 | |
65 | raw_patch_instruction(dest, instr); |
66 | |
67 | return 0; |
68 | } |
69 | |
70 | static int patch_feature_section_mask(unsigned long value, unsigned long mask, |
71 | struct fixup_entry *fcur) |
72 | { |
73 | u32 *start, *end, *alt_start, *alt_end, *src, *dest; |
74 | |
75 | start = calc_addr(fcur, offset: fcur->start_off); |
76 | end = calc_addr(fcur, offset: fcur->end_off); |
77 | alt_start = calc_addr(fcur, offset: fcur->alt_start_off); |
78 | alt_end = calc_addr(fcur, offset: fcur->alt_end_off); |
79 | |
80 | if ((alt_end - alt_start) > (end - start)) |
81 | return 1; |
82 | |
83 | if ((value & fcur->mask & mask) == (fcur->value & mask)) |
84 | return 0; |
85 | |
86 | src = alt_start; |
87 | dest = start; |
88 | |
89 | for (; src < alt_end; src = ppc_inst_next(src, src), |
90 | dest = ppc_inst_next(dest, dest)) { |
91 | if (patch_alt_instruction(src, dest, alt_start, alt_end)) |
92 | return 1; |
93 | } |
94 | |
95 | for (; dest < end; dest++) |
96 | raw_patch_instruction(dest, ppc_inst(PPC_RAW_NOP())); |
97 | |
98 | return 0; |
99 | } |
100 | |
101 | static void do_feature_fixups_mask(unsigned long value, unsigned long mask, |
102 | void *fixup_start, void *fixup_end) |
103 | { |
104 | struct fixup_entry *fcur, *fend; |
105 | |
106 | fcur = fixup_start; |
107 | fend = fixup_end; |
108 | |
109 | for (; fcur < fend; fcur++) { |
110 | if (patch_feature_section_mask(value, mask, fcur)) { |
111 | WARN_ON(1); |
112 | printk("Unable to patch feature section at %p - %p" \ |
113 | " with %p - %p\n" , |
114 | calc_addr(fcur, fcur->start_off), |
115 | calc_addr(fcur, fcur->end_off), |
116 | calc_addr(fcur, fcur->alt_start_off), |
117 | calc_addr(fcur, fcur->alt_end_off)); |
118 | } |
119 | } |
120 | } |
121 | |
122 | void do_feature_fixups(unsigned long value, void *fixup_start, void *fixup_end) |
123 | { |
124 | do_feature_fixups_mask(value, mask: ~0, fixup_start, fixup_end); |
125 | } |
126 | |
127 | #ifdef CONFIG_PPC_BARRIER_NOSPEC |
128 | static bool is_fixup_addr_valid(void *dest, size_t size) |
129 | { |
130 | return system_state < SYSTEM_FREEING_INITMEM || |
131 | !init_section_contains(dest, size); |
132 | } |
133 | |
134 | static int do_patch_fixups(long *start, long *end, unsigned int *instrs, int num) |
135 | { |
136 | int i; |
137 | |
138 | for (i = 0; start < end; start++, i++) { |
139 | int j; |
140 | unsigned int *dest = (void *)start + *start; |
141 | |
142 | if (!is_fixup_addr_valid(dest, sizeof(*instrs) * num)) |
143 | continue; |
144 | |
145 | pr_devel("patching dest %lx\n" , (unsigned long)dest); |
146 | |
147 | for (j = 0; j < num; j++) |
148 | patch_instruction(dest + j, ppc_inst(instrs[j])); |
149 | } |
150 | return i; |
151 | } |
152 | #endif |
153 | |
154 | #ifdef CONFIG_PPC_BOOK3S_64 |
155 | static int do_patch_entry_fixups(long *start, long *end, unsigned int *instrs, |
156 | bool do_fallback, void *fallback) |
157 | { |
158 | int i; |
159 | |
160 | for (i = 0; start < end; start++, i++) { |
161 | unsigned int *dest = (void *)start + *start; |
162 | |
163 | if (!is_fixup_addr_valid(dest, sizeof(*instrs) * 3)) |
164 | continue; |
165 | |
166 | pr_devel("patching dest %lx\n" , (unsigned long)dest); |
167 | |
168 | // See comment in do_entry_flush_fixups() RE order of patching |
169 | if (do_fallback) { |
170 | patch_instruction(dest, ppc_inst(instrs[0])); |
171 | patch_instruction(dest + 2, ppc_inst(instrs[2])); |
172 | patch_branch(dest + 1, (unsigned long)fallback, BRANCH_SET_LINK); |
173 | } else { |
174 | patch_instruction(dest + 1, ppc_inst(instrs[1])); |
175 | patch_instruction(dest + 2, ppc_inst(instrs[2])); |
176 | patch_instruction(dest, ppc_inst(instrs[0])); |
177 | } |
178 | } |
179 | return i; |
180 | } |
181 | |
182 | static void do_stf_entry_barrier_fixups(enum stf_barrier_type types) |
183 | { |
184 | unsigned int instrs[3]; |
185 | long *start, *end; |
186 | int i; |
187 | |
188 | start = PTRRELOC(&__start___stf_entry_barrier_fixup); |
189 | end = PTRRELOC(&__stop___stf_entry_barrier_fixup); |
190 | |
191 | instrs[0] = PPC_RAW_NOP(); |
192 | instrs[1] = PPC_RAW_NOP(); |
193 | instrs[2] = PPC_RAW_NOP(); |
194 | |
195 | i = 0; |
196 | if (types & STF_BARRIER_FALLBACK) { |
197 | instrs[i++] = PPC_RAW_MFLR(_R10); |
198 | instrs[i++] = PPC_RAW_NOP(); /* branch patched below */ |
199 | instrs[i++] = PPC_RAW_MTLR(_R10); |
200 | } else if (types & STF_BARRIER_EIEIO) { |
201 | instrs[i++] = PPC_RAW_EIEIO() | 0x02000000; /* eieio + bit 6 hint */ |
202 | } else if (types & STF_BARRIER_SYNC_ORI) { |
203 | instrs[i++] = PPC_RAW_SYNC(); |
204 | instrs[i++] = PPC_RAW_LD(_R10, _R13, 0); |
205 | instrs[i++] = PPC_RAW_ORI(_R31, _R31, 0); /* speculation barrier */ |
206 | } |
207 | |
208 | i = do_patch_entry_fixups(start, end, instrs, types & STF_BARRIER_FALLBACK, |
209 | &stf_barrier_fallback); |
210 | |
211 | printk(KERN_DEBUG "stf-barrier: patched %d entry locations (%s barrier)\n" , i, |
212 | (types == STF_BARRIER_NONE) ? "no" : |
213 | (types == STF_BARRIER_FALLBACK) ? "fallback" : |
214 | (types == STF_BARRIER_EIEIO) ? "eieio" : |
215 | (types == (STF_BARRIER_SYNC_ORI)) ? "hwsync" |
216 | : "unknown" ); |
217 | } |
218 | |
219 | static void do_stf_exit_barrier_fixups(enum stf_barrier_type types) |
220 | { |
221 | unsigned int instrs[6]; |
222 | long *start, *end; |
223 | int i; |
224 | |
225 | start = PTRRELOC(&__start___stf_exit_barrier_fixup); |
226 | end = PTRRELOC(&__stop___stf_exit_barrier_fixup); |
227 | |
228 | instrs[0] = PPC_RAW_NOP(); |
229 | instrs[1] = PPC_RAW_NOP(); |
230 | instrs[2] = PPC_RAW_NOP(); |
231 | instrs[3] = PPC_RAW_NOP(); |
232 | instrs[4] = PPC_RAW_NOP(); |
233 | instrs[5] = PPC_RAW_NOP(); |
234 | |
235 | i = 0; |
236 | if (types & STF_BARRIER_FALLBACK || types & STF_BARRIER_SYNC_ORI) { |
237 | if (cpu_has_feature(CPU_FTR_HVMODE)) { |
238 | instrs[i++] = PPC_RAW_MTSPR(SPRN_HSPRG1, _R13); |
239 | instrs[i++] = PPC_RAW_MFSPR(_R13, SPRN_HSPRG0); |
240 | } else { |
241 | instrs[i++] = PPC_RAW_MTSPR(SPRN_SPRG2, _R13); |
242 | instrs[i++] = PPC_RAW_MFSPR(_R13, SPRN_SPRG1); |
243 | } |
244 | instrs[i++] = PPC_RAW_SYNC(); |
245 | instrs[i++] = PPC_RAW_LD(_R13, _R13, 0); |
246 | instrs[i++] = PPC_RAW_ORI(_R31, _R31, 0); /* speculation barrier */ |
247 | if (cpu_has_feature(CPU_FTR_HVMODE)) |
248 | instrs[i++] = PPC_RAW_MFSPR(_R13, SPRN_HSPRG1); |
249 | else |
250 | instrs[i++] = PPC_RAW_MFSPR(_R13, SPRN_SPRG2); |
251 | } else if (types & STF_BARRIER_EIEIO) { |
252 | instrs[i++] = PPC_RAW_EIEIO() | 0x02000000; /* eieio + bit 6 hint */ |
253 | } |
254 | |
255 | i = do_patch_fixups(start, end, instrs, ARRAY_SIZE(instrs)); |
256 | |
257 | printk(KERN_DEBUG "stf-barrier: patched %d exit locations (%s barrier)\n" , i, |
258 | (types == STF_BARRIER_NONE) ? "no" : |
259 | (types == STF_BARRIER_FALLBACK) ? "fallback" : |
260 | (types == STF_BARRIER_EIEIO) ? "eieio" : |
261 | (types == (STF_BARRIER_SYNC_ORI)) ? "hwsync" |
262 | : "unknown" ); |
263 | } |
264 | |
265 | static bool stf_exit_reentrant = false; |
266 | static bool rfi_exit_reentrant = false; |
267 | static DEFINE_MUTEX(exit_flush_lock); |
268 | |
269 | static int __do_stf_barrier_fixups(void *data) |
270 | { |
271 | enum stf_barrier_type *types = data; |
272 | |
273 | do_stf_entry_barrier_fixups(*types); |
274 | do_stf_exit_barrier_fixups(*types); |
275 | |
276 | return 0; |
277 | } |
278 | |
279 | void do_stf_barrier_fixups(enum stf_barrier_type types) |
280 | { |
281 | /* |
282 | * The call to the fallback entry flush, and the fallback/sync-ori exit |
283 | * flush can not be safely patched in/out while other CPUs are |
284 | * executing them. So call __do_stf_barrier_fixups() on one CPU while |
285 | * all other CPUs spin in the stop machine core with interrupts hard |
286 | * disabled. |
287 | * |
288 | * The branch to mark interrupt exits non-reentrant is enabled first, |
289 | * then stop_machine runs which will ensure all CPUs are out of the |
290 | * low level interrupt exit code before patching. After the patching, |
291 | * if allowed, then flip the branch to allow fast exits. |
292 | */ |
293 | |
294 | // Prevent static key update races with do_rfi_flush_fixups() |
295 | mutex_lock(&exit_flush_lock); |
296 | static_branch_enable(&interrupt_exit_not_reentrant); |
297 | |
298 | stop_machine(__do_stf_barrier_fixups, &types, NULL); |
299 | |
300 | if ((types & STF_BARRIER_FALLBACK) || (types & STF_BARRIER_SYNC_ORI)) |
301 | stf_exit_reentrant = false; |
302 | else |
303 | stf_exit_reentrant = true; |
304 | |
305 | if (stf_exit_reentrant && rfi_exit_reentrant) |
306 | static_branch_disable(&interrupt_exit_not_reentrant); |
307 | |
308 | mutex_unlock(&exit_flush_lock); |
309 | } |
310 | |
311 | void do_uaccess_flush_fixups(enum l1d_flush_type types) |
312 | { |
313 | unsigned int instrs[4]; |
314 | long *start, *end; |
315 | int i; |
316 | |
317 | start = PTRRELOC(&__start___uaccess_flush_fixup); |
318 | end = PTRRELOC(&__stop___uaccess_flush_fixup); |
319 | |
320 | instrs[0] = PPC_RAW_NOP(); |
321 | instrs[1] = PPC_RAW_NOP(); |
322 | instrs[2] = PPC_RAW_NOP(); |
323 | instrs[3] = PPC_RAW_BLR(); |
324 | |
325 | i = 0; |
326 | if (types == L1D_FLUSH_FALLBACK) { |
327 | instrs[3] = PPC_RAW_NOP(); |
328 | /* fallthrough to fallback flush */ |
329 | } |
330 | |
331 | if (types & L1D_FLUSH_ORI) { |
332 | instrs[i++] = PPC_RAW_ORI(_R31, _R31, 0); /* speculation barrier */ |
333 | instrs[i++] = PPC_RAW_ORI(_R30, _R30, 0); /* L1d flush */ |
334 | } |
335 | |
336 | if (types & L1D_FLUSH_MTTRIG) |
337 | instrs[i++] = PPC_RAW_MTSPR(SPRN_TRIG2, _R0); |
338 | |
339 | i = do_patch_fixups(start, end, instrs, ARRAY_SIZE(instrs)); |
340 | |
341 | printk(KERN_DEBUG "uaccess-flush: patched %d locations (%s flush)\n" , i, |
342 | (types == L1D_FLUSH_NONE) ? "no" : |
343 | (types == L1D_FLUSH_FALLBACK) ? "fallback displacement" : |
344 | (types & L1D_FLUSH_ORI) ? (types & L1D_FLUSH_MTTRIG) |
345 | ? "ori+mttrig type" |
346 | : "ori type" : |
347 | (types & L1D_FLUSH_MTTRIG) ? "mttrig type" |
348 | : "unknown" ); |
349 | } |
350 | |
351 | static int __do_entry_flush_fixups(void *data) |
352 | { |
353 | enum l1d_flush_type types = *(enum l1d_flush_type *)data; |
354 | unsigned int instrs[3]; |
355 | long *start, *end; |
356 | int i; |
357 | |
358 | instrs[0] = PPC_RAW_NOP(); |
359 | instrs[1] = PPC_RAW_NOP(); |
360 | instrs[2] = PPC_RAW_NOP(); |
361 | |
362 | i = 0; |
363 | if (types == L1D_FLUSH_FALLBACK) { |
364 | instrs[i++] = PPC_RAW_MFLR(_R10); |
365 | instrs[i++] = PPC_RAW_NOP(); /* branch patched below */ |
366 | instrs[i++] = PPC_RAW_MTLR(_R10); |
367 | } |
368 | |
369 | if (types & L1D_FLUSH_ORI) { |
370 | instrs[i++] = PPC_RAW_ORI(_R31, _R31, 0); /* speculation barrier */ |
371 | instrs[i++] = PPC_RAW_ORI(_R30, _R30, 0); /* L1d flush */ |
372 | } |
373 | |
374 | if (types & L1D_FLUSH_MTTRIG) |
375 | instrs[i++] = PPC_RAW_MTSPR(SPRN_TRIG2, _R0); |
376 | |
377 | /* |
378 | * If we're patching in or out the fallback flush we need to be careful about the |
379 | * order in which we patch instructions. That's because it's possible we could |
380 | * take a page fault after patching one instruction, so the sequence of |
381 | * instructions must be safe even in a half patched state. |
382 | * |
383 | * To make that work, when patching in the fallback flush we patch in this order: |
384 | * - the mflr (dest) |
385 | * - the mtlr (dest + 2) |
386 | * - the branch (dest + 1) |
387 | * |
388 | * That ensures the sequence is safe to execute at any point. In contrast if we |
389 | * patch the mtlr last, it's possible we could return from the branch and not |
390 | * restore LR, leading to a crash later. |
391 | * |
392 | * When patching out the fallback flush (either with nops or another flush type), |
393 | * we patch in this order: |
394 | * - the branch (dest + 1) |
395 | * - the mtlr (dest + 2) |
396 | * - the mflr (dest) |
397 | * |
398 | * Note we are protected by stop_machine() from other CPUs executing the code in a |
399 | * semi-patched state. |
400 | */ |
401 | |
402 | start = PTRRELOC(&__start___entry_flush_fixup); |
403 | end = PTRRELOC(&__stop___entry_flush_fixup); |
404 | i = do_patch_entry_fixups(start, end, instrs, types == L1D_FLUSH_FALLBACK, |
405 | &entry_flush_fallback); |
406 | |
407 | start = PTRRELOC(&__start___scv_entry_flush_fixup); |
408 | end = PTRRELOC(&__stop___scv_entry_flush_fixup); |
409 | i += do_patch_entry_fixups(start, end, instrs, types == L1D_FLUSH_FALLBACK, |
410 | &scv_entry_flush_fallback); |
411 | |
412 | printk(KERN_DEBUG "entry-flush: patched %d locations (%s flush)\n" , i, |
413 | (types == L1D_FLUSH_NONE) ? "no" : |
414 | (types == L1D_FLUSH_FALLBACK) ? "fallback displacement" : |
415 | (types & L1D_FLUSH_ORI) ? (types & L1D_FLUSH_MTTRIG) |
416 | ? "ori+mttrig type" |
417 | : "ori type" : |
418 | (types & L1D_FLUSH_MTTRIG) ? "mttrig type" |
419 | : "unknown" ); |
420 | |
421 | return 0; |
422 | } |
423 | |
424 | void do_entry_flush_fixups(enum l1d_flush_type types) |
425 | { |
426 | /* |
427 | * The call to the fallback flush can not be safely patched in/out while |
428 | * other CPUs are executing it. So call __do_entry_flush_fixups() on one |
429 | * CPU while all other CPUs spin in the stop machine core with interrupts |
430 | * hard disabled. |
431 | */ |
432 | stop_machine(__do_entry_flush_fixups, &types, NULL); |
433 | } |
434 | |
435 | static int __do_rfi_flush_fixups(void *data) |
436 | { |
437 | enum l1d_flush_type types = *(enum l1d_flush_type *)data; |
438 | unsigned int instrs[3]; |
439 | long *start, *end; |
440 | int i; |
441 | |
442 | start = PTRRELOC(&__start___rfi_flush_fixup); |
443 | end = PTRRELOC(&__stop___rfi_flush_fixup); |
444 | |
445 | instrs[0] = PPC_RAW_NOP(); |
446 | instrs[1] = PPC_RAW_NOP(); |
447 | instrs[2] = PPC_RAW_NOP(); |
448 | |
449 | if (types & L1D_FLUSH_FALLBACK) |
450 | /* b .+16 to fallback flush */ |
451 | instrs[0] = PPC_RAW_BRANCH(16); |
452 | |
453 | i = 0; |
454 | if (types & L1D_FLUSH_ORI) { |
455 | instrs[i++] = PPC_RAW_ORI(_R31, _R31, 0); /* speculation barrier */ |
456 | instrs[i++] = PPC_RAW_ORI(_R30, _R30, 0); /* L1d flush */ |
457 | } |
458 | |
459 | if (types & L1D_FLUSH_MTTRIG) |
460 | instrs[i++] = PPC_RAW_MTSPR(SPRN_TRIG2, _R0); |
461 | |
462 | i = do_patch_fixups(start, end, instrs, ARRAY_SIZE(instrs)); |
463 | |
464 | printk(KERN_DEBUG "rfi-flush: patched %d locations (%s flush)\n" , i, |
465 | (types == L1D_FLUSH_NONE) ? "no" : |
466 | (types == L1D_FLUSH_FALLBACK) ? "fallback displacement" : |
467 | (types & L1D_FLUSH_ORI) ? (types & L1D_FLUSH_MTTRIG) |
468 | ? "ori+mttrig type" |
469 | : "ori type" : |
470 | (types & L1D_FLUSH_MTTRIG) ? "mttrig type" |
471 | : "unknown" ); |
472 | |
473 | return 0; |
474 | } |
475 | |
476 | void do_rfi_flush_fixups(enum l1d_flush_type types) |
477 | { |
478 | /* |
479 | * stop_machine gets all CPUs out of the interrupt exit handler same |
480 | * as do_stf_barrier_fixups. do_rfi_flush_fixups patching can run |
481 | * without stop_machine, so this could be achieved with a broadcast |
482 | * IPI instead, but this matches the stf sequence. |
483 | */ |
484 | |
485 | // Prevent static key update races with do_stf_barrier_fixups() |
486 | mutex_lock(&exit_flush_lock); |
487 | static_branch_enable(&interrupt_exit_not_reentrant); |
488 | |
489 | stop_machine(__do_rfi_flush_fixups, &types, NULL); |
490 | |
491 | if (types & L1D_FLUSH_FALLBACK) |
492 | rfi_exit_reentrant = false; |
493 | else |
494 | rfi_exit_reentrant = true; |
495 | |
496 | if (stf_exit_reentrant && rfi_exit_reentrant) |
497 | static_branch_disable(&interrupt_exit_not_reentrant); |
498 | |
499 | mutex_unlock(&exit_flush_lock); |
500 | } |
501 | |
502 | void do_barrier_nospec_fixups_range(bool enable, void *fixup_start, void *fixup_end) |
503 | { |
504 | unsigned int instr; |
505 | long *start, *end; |
506 | int i; |
507 | |
508 | start = fixup_start; |
509 | end = fixup_end; |
510 | |
511 | instr = PPC_RAW_NOP(); |
512 | |
513 | if (enable) { |
514 | pr_info("barrier-nospec: using ORI speculation barrier\n" ); |
515 | instr = PPC_RAW_ORI(_R31, _R31, 0); /* speculation barrier */ |
516 | } |
517 | |
518 | i = do_patch_fixups(start, end, &instr, 1); |
519 | |
520 | printk(KERN_DEBUG "barrier-nospec: patched %d locations\n" , i); |
521 | } |
522 | |
523 | #endif /* CONFIG_PPC_BOOK3S_64 */ |
524 | |
525 | #ifdef CONFIG_PPC_BARRIER_NOSPEC |
526 | void do_barrier_nospec_fixups(bool enable) |
527 | { |
528 | void *start, *end; |
529 | |
530 | start = PTRRELOC(&__start___barrier_nospec_fixup); |
531 | end = PTRRELOC(&__stop___barrier_nospec_fixup); |
532 | |
533 | do_barrier_nospec_fixups_range(enable, start, end); |
534 | } |
535 | #endif /* CONFIG_PPC_BARRIER_NOSPEC */ |
536 | |
537 | #ifdef CONFIG_PPC_E500 |
538 | void do_barrier_nospec_fixups_range(bool enable, void *fixup_start, void *fixup_end) |
539 | { |
540 | unsigned int instr[2]; |
541 | long *start, *end; |
542 | int i; |
543 | |
544 | start = fixup_start; |
545 | end = fixup_end; |
546 | |
547 | instr[0] = PPC_RAW_NOP(); |
548 | instr[1] = PPC_RAW_NOP(); |
549 | |
550 | if (enable) { |
551 | pr_info("barrier-nospec: using isync; sync as speculation barrier\n" ); |
552 | instr[0] = PPC_RAW_ISYNC(); |
553 | instr[1] = PPC_RAW_SYNC(); |
554 | } |
555 | |
556 | i = do_patch_fixups(start, end, instr, ARRAY_SIZE(instr)); |
557 | |
558 | printk(KERN_DEBUG "barrier-nospec: patched %d locations\n" , i); |
559 | } |
560 | |
561 | static void __init patch_btb_flush_section(long *curr) |
562 | { |
563 | unsigned int *start, *end; |
564 | |
565 | start = (void *)curr + *curr; |
566 | end = (void *)curr + *(curr + 1); |
567 | for (; start < end; start++) { |
568 | pr_devel("patching dest %lx\n" , (unsigned long)start); |
569 | patch_instruction(start, ppc_inst(PPC_RAW_NOP())); |
570 | } |
571 | } |
572 | |
573 | void __init do_btb_flush_fixups(void) |
574 | { |
575 | long *start, *end; |
576 | |
577 | start = PTRRELOC(&__start__btb_flush_fixup); |
578 | end = PTRRELOC(&__stop__btb_flush_fixup); |
579 | |
580 | for (; start < end; start += 2) |
581 | patch_btb_flush_section(start); |
582 | } |
583 | #endif /* CONFIG_PPC_E500 */ |
584 | |
585 | void do_lwsync_fixups(unsigned long value, void *fixup_start, void *fixup_end) |
586 | { |
587 | long *start, *end; |
588 | u32 *dest; |
589 | |
590 | if (!(value & CPU_FTR_LWSYNC)) |
591 | return ; |
592 | |
593 | start = fixup_start; |
594 | end = fixup_end; |
595 | |
596 | for (; start < end; start++) { |
597 | dest = (void *)start + *start; |
598 | raw_patch_instruction(dest, ppc_inst(PPC_INST_LWSYNC)); |
599 | } |
600 | } |
601 | |
602 | static void __init do_final_fixups(void) |
603 | { |
604 | #if defined(CONFIG_PPC64) && defined(CONFIG_RELOCATABLE) |
605 | ppc_inst_t inst; |
606 | u32 *src, *dest, *end; |
607 | |
608 | if (PHYSICAL_START == 0) |
609 | return; |
610 | |
611 | src = (u32 *)(KERNELBASE + PHYSICAL_START); |
612 | dest = (u32 *)KERNELBASE; |
613 | end = (void *)src + (__end_interrupts - _stext); |
614 | |
615 | while (src < end) { |
616 | inst = ppc_inst_read(src); |
617 | raw_patch_instruction(dest, inst); |
618 | src = ppc_inst_next(src, src); |
619 | dest = ppc_inst_next(dest, dest); |
620 | } |
621 | #endif |
622 | } |
623 | |
624 | static unsigned long __initdata saved_cpu_features; |
625 | static unsigned int __initdata saved_mmu_features; |
626 | #ifdef CONFIG_PPC64 |
627 | static unsigned long __initdata saved_firmware_features; |
628 | #endif |
629 | |
630 | void __init apply_feature_fixups(void) |
631 | { |
632 | struct cpu_spec *spec = PTRRELOC(*PTRRELOC(&cur_cpu_spec)); |
633 | |
634 | *PTRRELOC(&saved_cpu_features) = spec->cpu_features; |
635 | *PTRRELOC(&saved_mmu_features) = spec->mmu_features; |
636 | |
637 | /* |
638 | * Apply the CPU-specific and firmware specific fixups to kernel text |
639 | * (nop out sections not relevant to this CPU or this firmware). |
640 | */ |
641 | do_feature_fixups(value: spec->cpu_features, |
642 | fixup_start: PTRRELOC(&__start___ftr_fixup), |
643 | fixup_end: PTRRELOC(&__stop___ftr_fixup)); |
644 | |
645 | do_feature_fixups(value: spec->mmu_features, |
646 | fixup_start: PTRRELOC(&__start___mmu_ftr_fixup), |
647 | fixup_end: PTRRELOC(&__stop___mmu_ftr_fixup)); |
648 | |
649 | do_lwsync_fixups(value: spec->cpu_features, |
650 | fixup_start: PTRRELOC(&__start___lwsync_fixup), |
651 | fixup_end: PTRRELOC(&__stop___lwsync_fixup)); |
652 | |
653 | #ifdef CONFIG_PPC64 |
654 | saved_firmware_features = powerpc_firmware_features; |
655 | do_feature_fixups(powerpc_firmware_features, |
656 | &__start___fw_ftr_fixup, &__stop___fw_ftr_fixup); |
657 | #endif |
658 | do_final_fixups(); |
659 | } |
660 | |
661 | void __init update_mmu_feature_fixups(unsigned long mask) |
662 | { |
663 | saved_mmu_features &= ~mask; |
664 | saved_mmu_features |= cur_cpu_spec->mmu_features & mask; |
665 | |
666 | do_feature_fixups_mask(value: cur_cpu_spec->mmu_features, mask, |
667 | fixup_start: PTRRELOC(&__start___mmu_ftr_fixup), |
668 | fixup_end: PTRRELOC(&__stop___mmu_ftr_fixup)); |
669 | mmu_feature_keys_init(); |
670 | } |
671 | |
672 | void __init setup_feature_keys(void) |
673 | { |
674 | /* |
675 | * Initialise jump label. This causes all the cpu/mmu_has_feature() |
676 | * checks to take on their correct polarity based on the current set of |
677 | * CPU/MMU features. |
678 | */ |
679 | jump_label_init(); |
680 | cpu_feature_keys_init(); |
681 | mmu_feature_keys_init(); |
682 | } |
683 | |
684 | static int __init check_features(void) |
685 | { |
686 | WARN(saved_cpu_features != cur_cpu_spec->cpu_features, |
687 | "CPU features changed after feature patching!\n" ); |
688 | WARN(saved_mmu_features != cur_cpu_spec->mmu_features, |
689 | "MMU features changed after feature patching!\n" ); |
690 | #ifdef CONFIG_PPC64 |
691 | WARN(saved_firmware_features != powerpc_firmware_features, |
692 | "Firmware features changed after feature patching!\n" ); |
693 | #endif |
694 | |
695 | return 0; |
696 | } |
697 | late_initcall(check_features); |
698 | |
699 | #ifdef CONFIG_FTR_FIXUP_SELFTEST |
700 | |
701 | #define check(x) \ |
702 | if (!(x)) printk("feature-fixups: test failed at line %d\n", __LINE__); |
703 | |
704 | static int patch_feature_section(unsigned long value, struct fixup_entry *fcur) |
705 | { |
706 | return patch_feature_section_mask(value, ~0, fcur); |
707 | } |
708 | |
709 | /* This must be after the text it fixes up, vmlinux.lds.S enforces that atm */ |
710 | static struct fixup_entry fixup; |
711 | |
712 | static long __init calc_offset(struct fixup_entry *entry, unsigned int *p) |
713 | { |
714 | return (unsigned long)p - (unsigned long)entry; |
715 | } |
716 | |
717 | static void __init test_basic_patching(void) |
718 | { |
719 | extern unsigned int ftr_fixup_test1[]; |
720 | extern unsigned int end_ftr_fixup_test1[]; |
721 | extern unsigned int ftr_fixup_test1_orig[]; |
722 | extern unsigned int ftr_fixup_test1_expected[]; |
723 | int size = 4 * (end_ftr_fixup_test1 - ftr_fixup_test1); |
724 | |
725 | fixup.value = fixup.mask = 8; |
726 | fixup.start_off = calc_offset(&fixup, ftr_fixup_test1 + 1); |
727 | fixup.end_off = calc_offset(&fixup, ftr_fixup_test1 + 2); |
728 | fixup.alt_start_off = fixup.alt_end_off = 0; |
729 | |
730 | /* Sanity check */ |
731 | check(memcmp(ftr_fixup_test1, ftr_fixup_test1_orig, size) == 0); |
732 | |
733 | /* Check we don't patch if the value matches */ |
734 | patch_feature_section(8, &fixup); |
735 | check(memcmp(ftr_fixup_test1, ftr_fixup_test1_orig, size) == 0); |
736 | |
737 | /* Check we do patch if the value doesn't match */ |
738 | patch_feature_section(0, &fixup); |
739 | check(memcmp(ftr_fixup_test1, ftr_fixup_test1_expected, size) == 0); |
740 | |
741 | /* Check we do patch if the mask doesn't match */ |
742 | memcpy(ftr_fixup_test1, ftr_fixup_test1_orig, size); |
743 | check(memcmp(ftr_fixup_test1, ftr_fixup_test1_orig, size) == 0); |
744 | patch_feature_section(~8, &fixup); |
745 | check(memcmp(ftr_fixup_test1, ftr_fixup_test1_expected, size) == 0); |
746 | } |
747 | |
748 | static void __init test_alternative_patching(void) |
749 | { |
750 | extern unsigned int ftr_fixup_test2[]; |
751 | extern unsigned int end_ftr_fixup_test2[]; |
752 | extern unsigned int ftr_fixup_test2_orig[]; |
753 | extern unsigned int ftr_fixup_test2_alt[]; |
754 | extern unsigned int ftr_fixup_test2_expected[]; |
755 | int size = 4 * (end_ftr_fixup_test2 - ftr_fixup_test2); |
756 | |
757 | fixup.value = fixup.mask = 0xF; |
758 | fixup.start_off = calc_offset(&fixup, ftr_fixup_test2 + 1); |
759 | fixup.end_off = calc_offset(&fixup, ftr_fixup_test2 + 2); |
760 | fixup.alt_start_off = calc_offset(&fixup, ftr_fixup_test2_alt); |
761 | fixup.alt_end_off = calc_offset(&fixup, ftr_fixup_test2_alt + 1); |
762 | |
763 | /* Sanity check */ |
764 | check(memcmp(ftr_fixup_test2, ftr_fixup_test2_orig, size) == 0); |
765 | |
766 | /* Check we don't patch if the value matches */ |
767 | patch_feature_section(0xF, &fixup); |
768 | check(memcmp(ftr_fixup_test2, ftr_fixup_test2_orig, size) == 0); |
769 | |
770 | /* Check we do patch if the value doesn't match */ |
771 | patch_feature_section(0, &fixup); |
772 | check(memcmp(ftr_fixup_test2, ftr_fixup_test2_expected, size) == 0); |
773 | |
774 | /* Check we do patch if the mask doesn't match */ |
775 | memcpy(ftr_fixup_test2, ftr_fixup_test2_orig, size); |
776 | check(memcmp(ftr_fixup_test2, ftr_fixup_test2_orig, size) == 0); |
777 | patch_feature_section(~0xF, &fixup); |
778 | check(memcmp(ftr_fixup_test2, ftr_fixup_test2_expected, size) == 0); |
779 | } |
780 | |
781 | static void __init test_alternative_case_too_big(void) |
782 | { |
783 | extern unsigned int ftr_fixup_test3[]; |
784 | extern unsigned int end_ftr_fixup_test3[]; |
785 | extern unsigned int ftr_fixup_test3_orig[]; |
786 | extern unsigned int ftr_fixup_test3_alt[]; |
787 | int size = 4 * (end_ftr_fixup_test3 - ftr_fixup_test3); |
788 | |
789 | fixup.value = fixup.mask = 0xC; |
790 | fixup.start_off = calc_offset(&fixup, ftr_fixup_test3 + 1); |
791 | fixup.end_off = calc_offset(&fixup, ftr_fixup_test3 + 2); |
792 | fixup.alt_start_off = calc_offset(&fixup, ftr_fixup_test3_alt); |
793 | fixup.alt_end_off = calc_offset(&fixup, ftr_fixup_test3_alt + 2); |
794 | |
795 | /* Sanity check */ |
796 | check(memcmp(ftr_fixup_test3, ftr_fixup_test3_orig, size) == 0); |
797 | |
798 | /* Expect nothing to be patched, and the error returned to us */ |
799 | check(patch_feature_section(0xF, &fixup) == 1); |
800 | check(memcmp(ftr_fixup_test3, ftr_fixup_test3_orig, size) == 0); |
801 | check(patch_feature_section(0, &fixup) == 1); |
802 | check(memcmp(ftr_fixup_test3, ftr_fixup_test3_orig, size) == 0); |
803 | check(patch_feature_section(~0xF, &fixup) == 1); |
804 | check(memcmp(ftr_fixup_test3, ftr_fixup_test3_orig, size) == 0); |
805 | } |
806 | |
807 | static void __init test_alternative_case_too_small(void) |
808 | { |
809 | extern unsigned int ftr_fixup_test4[]; |
810 | extern unsigned int end_ftr_fixup_test4[]; |
811 | extern unsigned int ftr_fixup_test4_orig[]; |
812 | extern unsigned int ftr_fixup_test4_alt[]; |
813 | extern unsigned int ftr_fixup_test4_expected[]; |
814 | int size = 4 * (end_ftr_fixup_test4 - ftr_fixup_test4); |
815 | unsigned long flag; |
816 | |
817 | /* Check a high-bit flag */ |
818 | flag = 1UL << ((sizeof(unsigned long) - 1) * 8); |
819 | fixup.value = fixup.mask = flag; |
820 | fixup.start_off = calc_offset(&fixup, ftr_fixup_test4 + 1); |
821 | fixup.end_off = calc_offset(&fixup, ftr_fixup_test4 + 5); |
822 | fixup.alt_start_off = calc_offset(&fixup, ftr_fixup_test4_alt); |
823 | fixup.alt_end_off = calc_offset(&fixup, ftr_fixup_test4_alt + 2); |
824 | |
825 | /* Sanity check */ |
826 | check(memcmp(ftr_fixup_test4, ftr_fixup_test4_orig, size) == 0); |
827 | |
828 | /* Check we don't patch if the value matches */ |
829 | patch_feature_section(flag, &fixup); |
830 | check(memcmp(ftr_fixup_test4, ftr_fixup_test4_orig, size) == 0); |
831 | |
832 | /* Check we do patch if the value doesn't match */ |
833 | patch_feature_section(0, &fixup); |
834 | check(memcmp(ftr_fixup_test4, ftr_fixup_test4_expected, size) == 0); |
835 | |
836 | /* Check we do patch if the mask doesn't match */ |
837 | memcpy(ftr_fixup_test4, ftr_fixup_test4_orig, size); |
838 | check(memcmp(ftr_fixup_test4, ftr_fixup_test4_orig, size) == 0); |
839 | patch_feature_section(~flag, &fixup); |
840 | check(memcmp(ftr_fixup_test4, ftr_fixup_test4_expected, size) == 0); |
841 | } |
842 | |
843 | static void test_alternative_case_with_branch(void) |
844 | { |
845 | extern unsigned int ftr_fixup_test5[]; |
846 | extern unsigned int end_ftr_fixup_test5[]; |
847 | extern unsigned int ftr_fixup_test5_expected[]; |
848 | int size = 4 * (end_ftr_fixup_test5 - ftr_fixup_test5); |
849 | |
850 | check(memcmp(ftr_fixup_test5, ftr_fixup_test5_expected, size) == 0); |
851 | } |
852 | |
853 | static void __init test_alternative_case_with_external_branch(void) |
854 | { |
855 | extern unsigned int ftr_fixup_test6[]; |
856 | extern unsigned int end_ftr_fixup_test6[]; |
857 | extern unsigned int ftr_fixup_test6_expected[]; |
858 | int size = 4 * (end_ftr_fixup_test6 - ftr_fixup_test6); |
859 | |
860 | check(memcmp(ftr_fixup_test6, ftr_fixup_test6_expected, size) == 0); |
861 | } |
862 | |
863 | static void __init test_alternative_case_with_branch_to_end(void) |
864 | { |
865 | extern unsigned int ftr_fixup_test7[]; |
866 | extern unsigned int end_ftr_fixup_test7[]; |
867 | extern unsigned int ftr_fixup_test7_expected[]; |
868 | int size = 4 * (end_ftr_fixup_test7 - ftr_fixup_test7); |
869 | |
870 | check(memcmp(ftr_fixup_test7, ftr_fixup_test7_expected, size) == 0); |
871 | } |
872 | |
873 | static void __init test_cpu_macros(void) |
874 | { |
875 | extern u8 ftr_fixup_test_FTR_macros[]; |
876 | extern u8 ftr_fixup_test_FTR_macros_expected[]; |
877 | unsigned long size = ftr_fixup_test_FTR_macros_expected - |
878 | ftr_fixup_test_FTR_macros; |
879 | |
880 | /* The fixups have already been done for us during boot */ |
881 | check(memcmp(ftr_fixup_test_FTR_macros, |
882 | ftr_fixup_test_FTR_macros_expected, size) == 0); |
883 | } |
884 | |
885 | static void __init test_fw_macros(void) |
886 | { |
887 | #ifdef CONFIG_PPC64 |
888 | extern u8 ftr_fixup_test_FW_FTR_macros[]; |
889 | extern u8 ftr_fixup_test_FW_FTR_macros_expected[]; |
890 | unsigned long size = ftr_fixup_test_FW_FTR_macros_expected - |
891 | ftr_fixup_test_FW_FTR_macros; |
892 | |
893 | /* The fixups have already been done for us during boot */ |
894 | check(memcmp(ftr_fixup_test_FW_FTR_macros, |
895 | ftr_fixup_test_FW_FTR_macros_expected, size) == 0); |
896 | #endif |
897 | } |
898 | |
899 | static void __init test_lwsync_macros(void) |
900 | { |
901 | extern u8 lwsync_fixup_test[]; |
902 | extern u8 end_lwsync_fixup_test[]; |
903 | extern u8 lwsync_fixup_test_expected_LWSYNC[]; |
904 | extern u8 lwsync_fixup_test_expected_SYNC[]; |
905 | unsigned long size = end_lwsync_fixup_test - |
906 | lwsync_fixup_test; |
907 | |
908 | /* The fixups have already been done for us during boot */ |
909 | if (cur_cpu_spec->cpu_features & CPU_FTR_LWSYNC) { |
910 | check(memcmp(lwsync_fixup_test, |
911 | lwsync_fixup_test_expected_LWSYNC, size) == 0); |
912 | } else { |
913 | check(memcmp(lwsync_fixup_test, |
914 | lwsync_fixup_test_expected_SYNC, size) == 0); |
915 | } |
916 | } |
917 | |
918 | #ifdef CONFIG_PPC64 |
919 | static void __init test_prefix_patching(void) |
920 | { |
921 | extern unsigned int ftr_fixup_prefix1[]; |
922 | extern unsigned int end_ftr_fixup_prefix1[]; |
923 | extern unsigned int ftr_fixup_prefix1_orig[]; |
924 | extern unsigned int ftr_fixup_prefix1_expected[]; |
925 | int size = sizeof(unsigned int) * (end_ftr_fixup_prefix1 - ftr_fixup_prefix1); |
926 | |
927 | fixup.value = fixup.mask = 8; |
928 | fixup.start_off = calc_offset(&fixup, ftr_fixup_prefix1 + 1); |
929 | fixup.end_off = calc_offset(&fixup, ftr_fixup_prefix1 + 3); |
930 | fixup.alt_start_off = fixup.alt_end_off = 0; |
931 | |
932 | /* Sanity check */ |
933 | check(memcmp(ftr_fixup_prefix1, ftr_fixup_prefix1_orig, size) == 0); |
934 | |
935 | patch_feature_section(0, &fixup); |
936 | check(memcmp(ftr_fixup_prefix1, ftr_fixup_prefix1_expected, size) == 0); |
937 | check(memcmp(ftr_fixup_prefix1, ftr_fixup_prefix1_orig, size) != 0); |
938 | } |
939 | |
940 | static void __init test_prefix_alt_patching(void) |
941 | { |
942 | extern unsigned int ftr_fixup_prefix2[]; |
943 | extern unsigned int end_ftr_fixup_prefix2[]; |
944 | extern unsigned int ftr_fixup_prefix2_orig[]; |
945 | extern unsigned int ftr_fixup_prefix2_expected[]; |
946 | extern unsigned int ftr_fixup_prefix2_alt[]; |
947 | int size = sizeof(unsigned int) * (end_ftr_fixup_prefix2 - ftr_fixup_prefix2); |
948 | |
949 | fixup.value = fixup.mask = 8; |
950 | fixup.start_off = calc_offset(&fixup, ftr_fixup_prefix2 + 1); |
951 | fixup.end_off = calc_offset(&fixup, ftr_fixup_prefix2 + 3); |
952 | fixup.alt_start_off = calc_offset(&fixup, ftr_fixup_prefix2_alt); |
953 | fixup.alt_end_off = calc_offset(&fixup, ftr_fixup_prefix2_alt + 2); |
954 | /* Sanity check */ |
955 | check(memcmp(ftr_fixup_prefix2, ftr_fixup_prefix2_orig, size) == 0); |
956 | |
957 | patch_feature_section(0, &fixup); |
958 | check(memcmp(ftr_fixup_prefix2, ftr_fixup_prefix2_expected, size) == 0); |
959 | check(memcmp(ftr_fixup_prefix2, ftr_fixup_prefix2_orig, size) != 0); |
960 | } |
961 | |
962 | static void __init test_prefix_word_alt_patching(void) |
963 | { |
964 | extern unsigned int ftr_fixup_prefix3[]; |
965 | extern unsigned int end_ftr_fixup_prefix3[]; |
966 | extern unsigned int ftr_fixup_prefix3_orig[]; |
967 | extern unsigned int ftr_fixup_prefix3_expected[]; |
968 | extern unsigned int ftr_fixup_prefix3_alt[]; |
969 | int size = sizeof(unsigned int) * (end_ftr_fixup_prefix3 - ftr_fixup_prefix3); |
970 | |
971 | fixup.value = fixup.mask = 8; |
972 | fixup.start_off = calc_offset(&fixup, ftr_fixup_prefix3 + 1); |
973 | fixup.end_off = calc_offset(&fixup, ftr_fixup_prefix3 + 4); |
974 | fixup.alt_start_off = calc_offset(&fixup, ftr_fixup_prefix3_alt); |
975 | fixup.alt_end_off = calc_offset(&fixup, ftr_fixup_prefix3_alt + 3); |
976 | /* Sanity check */ |
977 | check(memcmp(ftr_fixup_prefix3, ftr_fixup_prefix3_orig, size) == 0); |
978 | |
979 | patch_feature_section(0, &fixup); |
980 | check(memcmp(ftr_fixup_prefix3, ftr_fixup_prefix3_expected, size) == 0); |
981 | patch_feature_section(0, &fixup); |
982 | check(memcmp(ftr_fixup_prefix3, ftr_fixup_prefix3_orig, size) != 0); |
983 | } |
984 | #else |
985 | static inline void test_prefix_patching(void) {} |
986 | static inline void test_prefix_alt_patching(void) {} |
987 | static inline void test_prefix_word_alt_patching(void) {} |
988 | #endif /* CONFIG_PPC64 */ |
989 | |
990 | static int __init test_feature_fixups(void) |
991 | { |
992 | printk(KERN_DEBUG "Running feature fixup self-tests ...\n" ); |
993 | |
994 | test_basic_patching(); |
995 | test_alternative_patching(); |
996 | test_alternative_case_too_big(); |
997 | test_alternative_case_too_small(); |
998 | test_alternative_case_with_branch(); |
999 | test_alternative_case_with_external_branch(); |
1000 | test_alternative_case_with_branch_to_end(); |
1001 | test_cpu_macros(); |
1002 | test_fw_macros(); |
1003 | test_lwsync_macros(); |
1004 | test_prefix_patching(); |
1005 | test_prefix_alt_patching(); |
1006 | test_prefix_word_alt_patching(); |
1007 | |
1008 | return 0; |
1009 | } |
1010 | late_initcall(test_feature_fixups); |
1011 | |
1012 | #endif /* CONFIG_FTR_FIXUP_SELFTEST */ |
1013 | |