1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #ifndef _ASM_X86_ALTERNATIVE_H |
3 | #define _ASM_X86_ALTERNATIVE_H |
4 | |
5 | #include <linux/types.h> |
6 | #include <linux/stringify.h> |
7 | #include <asm/asm.h> |
8 | |
9 | #define ALT_FLAGS_SHIFT 16 |
10 | |
11 | #define ALT_FLAG_NOT (1 << 0) |
12 | #define ALT_NOT(feature) ((ALT_FLAG_NOT << ALT_FLAGS_SHIFT) | (feature)) |
13 | |
14 | #ifndef __ASSEMBLY__ |
15 | |
16 | #include <linux/stddef.h> |
17 | |
18 | /* |
19 | * Alternative inline assembly for SMP. |
20 | * |
21 | * The LOCK_PREFIX macro defined here replaces the LOCK and |
22 | * LOCK_PREFIX macros used everywhere in the source tree. |
23 | * |
24 | * SMP alternatives use the same data structures as the other |
25 | * alternatives and the X86_FEATURE_UP flag to indicate the case of a |
26 | * UP system running a SMP kernel. The existing apply_alternatives() |
27 | * works fine for patching a SMP kernel for UP. |
28 | * |
29 | * The SMP alternative tables can be kept after boot and contain both |
30 | * UP and SMP versions of the instructions to allow switching back to |
31 | * SMP at runtime, when hotplugging in a new CPU, which is especially |
32 | * useful in virtualized environments. |
33 | * |
34 | * The very common lock prefix is handled as special case in a |
35 | * separate table which is a pure address list without replacement ptr |
36 | * and size information. That keeps the table sizes small. |
37 | */ |
38 | |
39 | #ifdef CONFIG_SMP |
40 | #define LOCK_PREFIX_HERE \ |
41 | ".pushsection .smp_locks,\"a\"\n" \ |
42 | ".balign 4\n" \ |
43 | ".long 671f - .\n" /* offset */ \ |
44 | ".popsection\n" \ |
45 | "671:" |
46 | |
47 | #define LOCK_PREFIX LOCK_PREFIX_HERE "\n\tlock; " |
48 | |
49 | #else /* ! CONFIG_SMP */ |
50 | #define LOCK_PREFIX_HERE "" |
51 | #define LOCK_PREFIX "" |
52 | #endif |
53 | |
54 | /* |
55 | * objtool annotation to ignore the alternatives and only consider the original |
56 | * instruction(s). |
57 | */ |
58 | #define ANNOTATE_IGNORE_ALTERNATIVE \ |
59 | "999:\n\t" \ |
60 | ".pushsection .discard.ignore_alts\n\t" \ |
61 | ".long 999b\n\t" \ |
62 | ".popsection\n\t" |
63 | |
64 | /* |
65 | * The patching flags are part of the upper bits of the @ft_flags parameter when |
66 | * specifying them. The split is currently like this: |
67 | * |
68 | * [31... flags ...16][15... CPUID feature bit ...0] |
69 | * |
70 | * but since this is all hidden in the macros argument being split, those fields can be |
71 | * extended in the future to fit in a u64 or however the need arises. |
72 | */ |
73 | struct alt_instr { |
74 | s32 instr_offset; /* original instruction */ |
75 | s32 repl_offset; /* offset to replacement instruction */ |
76 | |
77 | union { |
78 | struct { |
79 | u32 cpuid: 16; /* CPUID bit set for replacement */ |
80 | u32 flags: 16; /* patching control flags */ |
81 | }; |
82 | u32 ft_flags; |
83 | }; |
84 | |
85 | u8 instrlen; /* length of original instruction */ |
86 | u8 replacementlen; /* length of new instruction */ |
87 | } __packed; |
88 | |
89 | /* |
90 | * Debug flag that can be tested to see whether alternative |
91 | * instructions were patched in already: |
92 | */ |
93 | extern int alternatives_patched; |
94 | |
95 | extern void alternative_instructions(void); |
96 | extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end); |
97 | extern void apply_retpolines(s32 *start, s32 *end); |
98 | extern void apply_returns(s32 *start, s32 *end); |
99 | extern void apply_seal_endbr(s32 *start, s32 *end); |
100 | extern void apply_fineibt(s32 *start_retpoline, s32 *end_retpoine, |
101 | s32 *start_cfi, s32 *end_cfi); |
102 | |
103 | struct module; |
104 | struct paravirt_patch_site; |
105 | |
106 | struct callthunk_sites { |
107 | s32 *call_start, *call_end; |
108 | struct paravirt_patch_site *pv_start, *pv_end; |
109 | }; |
110 | |
111 | #ifdef CONFIG_CALL_THUNKS |
112 | extern void callthunks_patch_builtin_calls(void); |
113 | extern void callthunks_patch_module_calls(struct callthunk_sites *sites, |
114 | struct module *mod); |
115 | extern void *callthunks_translate_call_dest(void *dest); |
116 | extern int x86_call_depth_emit_accounting(u8 **pprog, void *func); |
117 | #else |
118 | static __always_inline void callthunks_patch_builtin_calls(void) {} |
119 | static __always_inline void |
120 | callthunks_patch_module_calls(struct callthunk_sites *sites, |
121 | struct module *mod) {} |
122 | static __always_inline void *callthunks_translate_call_dest(void *dest) |
123 | { |
124 | return dest; |
125 | } |
126 | static __always_inline int x86_call_depth_emit_accounting(u8 **pprog, |
127 | void *func) |
128 | { |
129 | return 0; |
130 | } |
131 | #endif |
132 | |
133 | #ifdef CONFIG_SMP |
134 | extern void alternatives_smp_module_add(struct module *mod, char *name, |
135 | void *locks, void *locks_end, |
136 | void *text, void *text_end); |
137 | extern void alternatives_smp_module_del(struct module *mod); |
138 | extern void alternatives_enable_smp(void); |
139 | extern int alternatives_text_reserved(void *start, void *end); |
140 | extern bool skip_smp_alternatives; |
141 | #else |
142 | static inline void alternatives_smp_module_add(struct module *mod, char *name, |
143 | void *locks, void *locks_end, |
144 | void *text, void *text_end) {} |
145 | static inline void alternatives_smp_module_del(struct module *mod) {} |
146 | static inline void alternatives_enable_smp(void) {} |
147 | static inline int alternatives_text_reserved(void *start, void *end) |
148 | { |
149 | return 0; |
150 | } |
151 | #endif /* CONFIG_SMP */ |
152 | |
153 | #define b_replacement(num) "664"#num |
154 | #define e_replacement(num) "665"#num |
155 | |
156 | #define alt_end_marker "663" |
157 | #define alt_slen "662b-661b" |
158 | #define alt_total_slen alt_end_marker"b-661b" |
159 | #define alt_rlen(num) e_replacement(num)"f-"b_replacement(num)"f" |
160 | |
161 | #define OLDINSTR(oldinstr, num) \ |
162 | "# ALT: oldnstr\n" \ |
163 | "661:\n\t" oldinstr "\n662:\n" \ |
164 | "# ALT: padding\n" \ |
165 | ".skip -(((" alt_rlen(num) ")-(" alt_slen ")) > 0) * " \ |
166 | "((" alt_rlen(num) ")-(" alt_slen ")),0x90\n" \ |
167 | alt_end_marker ":\n" |
168 | |
169 | /* |
170 | * gas compatible max based on the idea from: |
171 | * http://graphics.stanford.edu/~seander/bithacks.html#IntegerMinOrMax |
172 | * |
173 | * The additional "-" is needed because gas uses a "true" value of -1. |
174 | */ |
175 | #define alt_max_short(a, b) "((" a ") ^ (((" a ") ^ (" b ")) & -(-((" a ") < (" b ")))))" |
176 | |
177 | /* |
178 | * Pad the second replacement alternative with additional NOPs if it is |
179 | * additionally longer than the first replacement alternative. |
180 | */ |
181 | #define OLDINSTR_2(oldinstr, num1, num2) \ |
182 | "# ALT: oldinstr2\n" \ |
183 | "661:\n\t" oldinstr "\n662:\n" \ |
184 | "# ALT: padding2\n" \ |
185 | ".skip -((" alt_max_short(alt_rlen(num1), alt_rlen(num2)) " - (" alt_slen ")) > 0) * " \ |
186 | "(" alt_max_short(alt_rlen(num1), alt_rlen(num2)) " - (" alt_slen ")), 0x90\n" \ |
187 | alt_end_marker ":\n" |
188 | |
189 | #define OLDINSTR_3(oldinsn, n1, n2, n3) \ |
190 | "# ALT: oldinstr3\n" \ |
191 | "661:\n\t" oldinsn "\n662:\n" \ |
192 | "# ALT: padding3\n" \ |
193 | ".skip -((" alt_max_short(alt_max_short(alt_rlen(n1), alt_rlen(n2)), alt_rlen(n3)) \ |
194 | " - (" alt_slen ")) > 0) * " \ |
195 | "(" alt_max_short(alt_max_short(alt_rlen(n1), alt_rlen(n2)), alt_rlen(n3)) \ |
196 | " - (" alt_slen ")), 0x90\n" \ |
197 | alt_end_marker ":\n" |
198 | |
199 | #define ALTINSTR_ENTRY(ft_flags, num) \ |
200 | " .long 661b - .\n" /* label */ \ |
201 | " .long " b_replacement(num)"f - .\n" /* new instruction */ \ |
202 | " .4byte " __stringify(ft_flags) "\n" /* feature + flags */ \ |
203 | " .byte " alt_total_slen "\n" /* source len */ \ |
204 | " .byte " alt_rlen(num) "\n" /* replacement len */ |
205 | |
206 | #define ALTINSTR_REPLACEMENT(newinstr, num) /* replacement */ \ |
207 | "# ALT: replacement " #num "\n" \ |
208 | b_replacement(num)":\n\t" newinstr "\n" e_replacement(num) ":\n" |
209 | |
210 | /* alternative assembly primitive: */ |
211 | #define ALTERNATIVE(oldinstr, newinstr, ft_flags) \ |
212 | OLDINSTR(oldinstr, 1) \ |
213 | ".pushsection .altinstructions,\"a\"\n" \ |
214 | ALTINSTR_ENTRY(ft_flags, 1) \ |
215 | ".popsection\n" \ |
216 | ".pushsection .altinstr_replacement, \"ax\"\n" \ |
217 | ALTINSTR_REPLACEMENT(newinstr, 1) \ |
218 | ".popsection\n" |
219 | |
220 | #define ALTERNATIVE_2(oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2) \ |
221 | OLDINSTR_2(oldinstr, 1, 2) \ |
222 | ".pushsection .altinstructions,\"a\"\n" \ |
223 | ALTINSTR_ENTRY(ft_flags1, 1) \ |
224 | ALTINSTR_ENTRY(ft_flags2, 2) \ |
225 | ".popsection\n" \ |
226 | ".pushsection .altinstr_replacement, \"ax\"\n" \ |
227 | ALTINSTR_REPLACEMENT(newinstr1, 1) \ |
228 | ALTINSTR_REPLACEMENT(newinstr2, 2) \ |
229 | ".popsection\n" |
230 | |
231 | /* If @feature is set, patch in @newinstr_yes, otherwise @newinstr_no. */ |
232 | #define ALTERNATIVE_TERNARY(oldinstr, ft_flags, newinstr_yes, newinstr_no) \ |
233 | ALTERNATIVE_2(oldinstr, newinstr_no, X86_FEATURE_ALWAYS, \ |
234 | newinstr_yes, ft_flags) |
235 | |
236 | #define ALTERNATIVE_3(oldinsn, newinsn1, ft_flags1, newinsn2, ft_flags2, \ |
237 | newinsn3, ft_flags3) \ |
238 | OLDINSTR_3(oldinsn, 1, 2, 3) \ |
239 | ".pushsection .altinstructions,\"a\"\n" \ |
240 | ALTINSTR_ENTRY(ft_flags1, 1) \ |
241 | ALTINSTR_ENTRY(ft_flags2, 2) \ |
242 | ALTINSTR_ENTRY(ft_flags3, 3) \ |
243 | ".popsection\n" \ |
244 | ".pushsection .altinstr_replacement, \"ax\"\n" \ |
245 | ALTINSTR_REPLACEMENT(newinsn1, 1) \ |
246 | ALTINSTR_REPLACEMENT(newinsn2, 2) \ |
247 | ALTINSTR_REPLACEMENT(newinsn3, 3) \ |
248 | ".popsection\n" |
249 | |
250 | /* |
251 | * Alternative instructions for different CPU types or capabilities. |
252 | * |
253 | * This allows to use optimized instructions even on generic binary |
254 | * kernels. |
255 | * |
256 | * length of oldinstr must be longer or equal the length of newinstr |
257 | * It can be padded with nops as needed. |
258 | * |
259 | * For non barrier like inlines please define new variants |
260 | * without volatile and memory clobber. |
261 | */ |
262 | #define alternative(oldinstr, newinstr, ft_flags) \ |
263 | asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, ft_flags) : : : "memory") |
264 | |
265 | #define alternative_2(oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2) \ |
266 | asm_inline volatile(ALTERNATIVE_2(oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2) ::: "memory") |
267 | |
268 | #define alternative_ternary(oldinstr, ft_flags, newinstr_yes, newinstr_no) \ |
269 | asm_inline volatile(ALTERNATIVE_TERNARY(oldinstr, ft_flags, newinstr_yes, newinstr_no) ::: "memory") |
270 | |
271 | /* |
272 | * Alternative inline assembly with input. |
273 | * |
274 | * Peculiarities: |
275 | * No memory clobber here. |
276 | * Argument numbers start with 1. |
277 | * Leaving an unused argument 0 to keep API compatibility. |
278 | */ |
279 | #define alternative_input(oldinstr, newinstr, ft_flags, input...) \ |
280 | asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, ft_flags) \ |
281 | : : "i" (0), ## input) |
282 | |
283 | /* |
284 | * This is similar to alternative_input. But it has two features and |
285 | * respective instructions. |
286 | * |
287 | * If CPU has feature2, newinstr2 is used. |
288 | * Otherwise, if CPU has feature1, newinstr1 is used. |
289 | * Otherwise, oldinstr is used. |
290 | */ |
291 | #define alternative_input_2(oldinstr, newinstr1, ft_flags1, newinstr2, \ |
292 | ft_flags2, input...) \ |
293 | asm_inline volatile(ALTERNATIVE_2(oldinstr, newinstr1, ft_flags1, \ |
294 | newinstr2, ft_flags2) \ |
295 | : : "i" (0), ## input) |
296 | |
297 | /* Like alternative_input, but with a single output argument */ |
298 | #define alternative_io(oldinstr, newinstr, ft_flags, output, input...) \ |
299 | asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, ft_flags) \ |
300 | : output : "i" (0), ## input) |
301 | |
302 | /* Like alternative_io, but for replacing a direct call with another one. */ |
303 | #define alternative_call(oldfunc, newfunc, ft_flags, output, input...) \ |
304 | asm_inline volatile (ALTERNATIVE("call %P[old]", "call %P[new]", ft_flags) \ |
305 | : output : [old] "i" (oldfunc), [new] "i" (newfunc), ## input) |
306 | |
307 | /* |
308 | * Like alternative_call, but there are two features and respective functions. |
309 | * If CPU has feature2, function2 is used. |
310 | * Otherwise, if CPU has feature1, function1 is used. |
311 | * Otherwise, old function is used. |
312 | */ |
313 | #define alternative_call_2(oldfunc, newfunc1, ft_flags1, newfunc2, ft_flags2, \ |
314 | output, input...) \ |
315 | asm_inline volatile (ALTERNATIVE_2("call %P[old]", "call %P[new1]", ft_flags1,\ |
316 | "call %P[new2]", ft_flags2) \ |
317 | : output, ASM_CALL_CONSTRAINT \ |
318 | : [old] "i" (oldfunc), [new1] "i" (newfunc1), \ |
319 | [new2] "i" (newfunc2), ## input) |
320 | |
321 | /* |
322 | * use this macro(s) if you need more than one output parameter |
323 | * in alternative_io |
324 | */ |
325 | #define ASM_OUTPUT2(a...) a |
326 | |
327 | /* |
328 | * use this macro if you need clobbers but no inputs in |
329 | * alternative_{input,io,call}() |
330 | */ |
331 | #define ASM_NO_INPUT_CLOBBER(clbr...) "i" (0) : clbr |
332 | |
333 | #else /* __ASSEMBLY__ */ |
334 | |
335 | #ifdef CONFIG_SMP |
336 | .macro LOCK_PREFIX |
337 | 672: lock |
338 | .pushsection .smp_locks,"a" |
339 | .balign 4 |
340 | .long 672b - . |
341 | .popsection |
342 | .endm |
343 | #else |
344 | .macro LOCK_PREFIX |
345 | .endm |
346 | #endif |
347 | |
348 | /* |
349 | * objtool annotation to ignore the alternatives and only consider the original |
350 | * instruction(s). |
351 | */ |
352 | .macro ANNOTATE_IGNORE_ALTERNATIVE |
353 | .Lannotate_\@: |
354 | .pushsection .discard.ignore_alts |
355 | .long .Lannotate_\@ |
356 | .popsection |
357 | .endm |
358 | |
359 | /* |
360 | * Issue one struct alt_instr descriptor entry (need to put it into |
361 | * the section .altinstructions, see below). This entry contains |
362 | * enough information for the alternatives patching code to patch an |
363 | * instruction. See apply_alternatives(). |
364 | */ |
365 | .macro altinstr_entry orig alt ft_flags orig_len alt_len |
366 | .long \orig - . |
367 | .long \alt - . |
368 | .4byte \ft_flags |
369 | .byte \orig_len |
370 | .byte \alt_len |
371 | .endm |
372 | |
373 | /* |
374 | * Define an alternative between two instructions. If @feature is |
375 | * present, early code in apply_alternatives() replaces @oldinstr with |
376 | * @newinstr. ".skip" directive takes care of proper instruction padding |
377 | * in case @newinstr is longer than @oldinstr. |
378 | */ |
379 | .macro ALTERNATIVE oldinstr, newinstr, ft_flags |
380 | 140: |
381 | \oldinstr |
382 | 141: |
383 | .skip -(((144f-143f)-(141b-140b)) > 0) * ((144f-143f)-(141b-140b)),0x90 |
384 | 142: |
385 | |
386 | .pushsection .altinstructions,"a" |
387 | altinstr_entry 140b,143f,\ft_flags,142b-140b,144f-143f |
388 | .popsection |
389 | |
390 | .pushsection .altinstr_replacement,"ax" |
391 | 143: |
392 | \newinstr |
393 | 144: |
394 | .popsection |
395 | .endm |
396 | |
397 | #define old_len 141b-140b |
398 | #define new_len1 144f-143f |
399 | #define new_len2 145f-144f |
400 | #define new_len3 146f-145f |
401 | |
402 | /* |
403 | * gas compatible max based on the idea from: |
404 | * http://graphics.stanford.edu/~seander/bithacks.html#IntegerMinOrMax |
405 | * |
406 | * The additional "-" is needed because gas uses a "true" value of -1. |
407 | */ |
408 | #define alt_max_2(a, b) ((a) ^ (((a) ^ (b)) & -(-((a) < (b))))) |
409 | #define alt_max_3(a, b, c) (alt_max_2(alt_max_2(a, b), c)) |
410 | |
411 | |
412 | /* |
413 | * Same as ALTERNATIVE macro above but for two alternatives. If CPU |
414 | * has @feature1, it replaces @oldinstr with @newinstr1. If CPU has |
415 | * @feature2, it replaces @oldinstr with @feature2. |
416 | */ |
417 | .macro ALTERNATIVE_2 oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2 |
418 | 140: |
419 | \oldinstr |
420 | 141: |
421 | .skip -((alt_max_2(new_len1, new_len2) - (old_len)) > 0) * \ |
422 | (alt_max_2(new_len1, new_len2) - (old_len)),0x90 |
423 | 142: |
424 | |
425 | .pushsection .altinstructions,"a" |
426 | altinstr_entry 140b,143f,\ft_flags1,142b-140b,144f-143f |
427 | altinstr_entry 140b,144f,\ft_flags2,142b-140b,145f-144f |
428 | .popsection |
429 | |
430 | .pushsection .altinstr_replacement,"ax" |
431 | 143: |
432 | \newinstr1 |
433 | 144: |
434 | \newinstr2 |
435 | 145: |
436 | .popsection |
437 | .endm |
438 | |
439 | .macro ALTERNATIVE_3 oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2, newinstr3, ft_flags3 |
440 | 140: |
441 | \oldinstr |
442 | 141: |
443 | .skip -((alt_max_3(new_len1, new_len2, new_len3) - (old_len)) > 0) * \ |
444 | (alt_max_3(new_len1, new_len2, new_len3) - (old_len)),0x90 |
445 | 142: |
446 | |
447 | .pushsection .altinstructions,"a" |
448 | altinstr_entry 140b,143f,\ft_flags1,142b-140b,144f-143f |
449 | altinstr_entry 140b,144f,\ft_flags2,142b-140b,145f-144f |
450 | altinstr_entry 140b,145f,\ft_flags3,142b-140b,146f-145f |
451 | .popsection |
452 | |
453 | .pushsection .altinstr_replacement,"ax" |
454 | 143: |
455 | \newinstr1 |
456 | 144: |
457 | \newinstr2 |
458 | 145: |
459 | \newinstr3 |
460 | 146: |
461 | .popsection |
462 | .endm |
463 | |
464 | /* If @feature is set, patch in @newinstr_yes, otherwise @newinstr_no. */ |
465 | #define ALTERNATIVE_TERNARY(oldinstr, ft_flags, newinstr_yes, newinstr_no) \ |
466 | ALTERNATIVE_2 oldinstr, newinstr_no, X86_FEATURE_ALWAYS, \ |
467 | newinstr_yes, ft_flags |
468 | |
469 | #endif /* __ASSEMBLY__ */ |
470 | |
471 | #endif /* _ASM_X86_ALTERNATIVE_H */ |
472 | |