1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _ASM_X86_ALTERNATIVE_H
3#define _ASM_X86_ALTERNATIVE_H
4
5#include <linux/types.h>
6#include <linux/stringify.h>
7#include <asm/asm.h>
8
9#define ALT_FLAGS_SHIFT 16
10
11#define ALT_FLAG_NOT (1 << 0)
12#define ALT_NOT(feature) ((ALT_FLAG_NOT << ALT_FLAGS_SHIFT) | (feature))
13
14#ifndef __ASSEMBLY__
15
16#include <linux/stddef.h>
17
18/*
19 * Alternative inline assembly for SMP.
20 *
21 * The LOCK_PREFIX macro defined here replaces the LOCK and
22 * LOCK_PREFIX macros used everywhere in the source tree.
23 *
24 * SMP alternatives use the same data structures as the other
25 * alternatives and the X86_FEATURE_UP flag to indicate the case of a
26 * UP system running a SMP kernel. The existing apply_alternatives()
27 * works fine for patching a SMP kernel for UP.
28 *
29 * The SMP alternative tables can be kept after boot and contain both
30 * UP and SMP versions of the instructions to allow switching back to
31 * SMP at runtime, when hotplugging in a new CPU, which is especially
32 * useful in virtualized environments.
33 *
34 * The very common lock prefix is handled as special case in a
35 * separate table which is a pure address list without replacement ptr
36 * and size information. That keeps the table sizes small.
37 */
38
39#ifdef CONFIG_SMP
40#define LOCK_PREFIX_HERE \
41 ".pushsection .smp_locks,\"a\"\n" \
42 ".balign 4\n" \
43 ".long 671f - .\n" /* offset */ \
44 ".popsection\n" \
45 "671:"
46
47#define LOCK_PREFIX LOCK_PREFIX_HERE "\n\tlock; "
48
49#else /* ! CONFIG_SMP */
50#define LOCK_PREFIX_HERE ""
51#define LOCK_PREFIX ""
52#endif
53
54/*
55 * objtool annotation to ignore the alternatives and only consider the original
56 * instruction(s).
57 */
58#define ANNOTATE_IGNORE_ALTERNATIVE \
59 "999:\n\t" \
60 ".pushsection .discard.ignore_alts\n\t" \
61 ".long 999b\n\t" \
62 ".popsection\n\t"
63
64/*
65 * The patching flags are part of the upper bits of the @ft_flags parameter when
66 * specifying them. The split is currently like this:
67 *
68 * [31... flags ...16][15... CPUID feature bit ...0]
69 *
70 * but since this is all hidden in the macros argument being split, those fields can be
71 * extended in the future to fit in a u64 or however the need arises.
72 */
73struct alt_instr {
74 s32 instr_offset; /* original instruction */
75 s32 repl_offset; /* offset to replacement instruction */
76
77 union {
78 struct {
79 u32 cpuid: 16; /* CPUID bit set for replacement */
80 u32 flags: 16; /* patching control flags */
81 };
82 u32 ft_flags;
83 };
84
85 u8 instrlen; /* length of original instruction */
86 u8 replacementlen; /* length of new instruction */
87} __packed;
88
89/*
90 * Debug flag that can be tested to see whether alternative
91 * instructions were patched in already:
92 */
93extern int alternatives_patched;
94
95extern void alternative_instructions(void);
96extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end);
97extern void apply_retpolines(s32 *start, s32 *end);
98extern void apply_returns(s32 *start, s32 *end);
99extern void apply_seal_endbr(s32 *start, s32 *end);
100extern void apply_fineibt(s32 *start_retpoline, s32 *end_retpoine,
101 s32 *start_cfi, s32 *end_cfi);
102
103struct module;
104struct paravirt_patch_site;
105
106struct callthunk_sites {
107 s32 *call_start, *call_end;
108 struct paravirt_patch_site *pv_start, *pv_end;
109};
110
111#ifdef CONFIG_CALL_THUNKS
112extern void callthunks_patch_builtin_calls(void);
113extern void callthunks_patch_module_calls(struct callthunk_sites *sites,
114 struct module *mod);
115extern void *callthunks_translate_call_dest(void *dest);
116extern int x86_call_depth_emit_accounting(u8 **pprog, void *func);
117#else
118static __always_inline void callthunks_patch_builtin_calls(void) {}
119static __always_inline void
120callthunks_patch_module_calls(struct callthunk_sites *sites,
121 struct module *mod) {}
122static __always_inline void *callthunks_translate_call_dest(void *dest)
123{
124 return dest;
125}
126static __always_inline int x86_call_depth_emit_accounting(u8 **pprog,
127 void *func)
128{
129 return 0;
130}
131#endif
132
133#ifdef CONFIG_SMP
134extern void alternatives_smp_module_add(struct module *mod, char *name,
135 void *locks, void *locks_end,
136 void *text, void *text_end);
137extern void alternatives_smp_module_del(struct module *mod);
138extern void alternatives_enable_smp(void);
139extern int alternatives_text_reserved(void *start, void *end);
140extern bool skip_smp_alternatives;
141#else
142static inline void alternatives_smp_module_add(struct module *mod, char *name,
143 void *locks, void *locks_end,
144 void *text, void *text_end) {}
145static inline void alternatives_smp_module_del(struct module *mod) {}
146static inline void alternatives_enable_smp(void) {}
147static inline int alternatives_text_reserved(void *start, void *end)
148{
149 return 0;
150}
151#endif /* CONFIG_SMP */
152
153#define b_replacement(num) "664"#num
154#define e_replacement(num) "665"#num
155
156#define alt_end_marker "663"
157#define alt_slen "662b-661b"
158#define alt_total_slen alt_end_marker"b-661b"
159#define alt_rlen(num) e_replacement(num)"f-"b_replacement(num)"f"
160
161#define OLDINSTR(oldinstr, num) \
162 "# ALT: oldnstr\n" \
163 "661:\n\t" oldinstr "\n662:\n" \
164 "# ALT: padding\n" \
165 ".skip -(((" alt_rlen(num) ")-(" alt_slen ")) > 0) * " \
166 "((" alt_rlen(num) ")-(" alt_slen ")),0x90\n" \
167 alt_end_marker ":\n"
168
169/*
170 * gas compatible max based on the idea from:
171 * http://graphics.stanford.edu/~seander/bithacks.html#IntegerMinOrMax
172 *
173 * The additional "-" is needed because gas uses a "true" value of -1.
174 */
175#define alt_max_short(a, b) "((" a ") ^ (((" a ") ^ (" b ")) & -(-((" a ") < (" b ")))))"
176
177/*
178 * Pad the second replacement alternative with additional NOPs if it is
179 * additionally longer than the first replacement alternative.
180 */
181#define OLDINSTR_2(oldinstr, num1, num2) \
182 "# ALT: oldinstr2\n" \
183 "661:\n\t" oldinstr "\n662:\n" \
184 "# ALT: padding2\n" \
185 ".skip -((" alt_max_short(alt_rlen(num1), alt_rlen(num2)) " - (" alt_slen ")) > 0) * " \
186 "(" alt_max_short(alt_rlen(num1), alt_rlen(num2)) " - (" alt_slen ")), 0x90\n" \
187 alt_end_marker ":\n"
188
189#define OLDINSTR_3(oldinsn, n1, n2, n3) \
190 "# ALT: oldinstr3\n" \
191 "661:\n\t" oldinsn "\n662:\n" \
192 "# ALT: padding3\n" \
193 ".skip -((" alt_max_short(alt_max_short(alt_rlen(n1), alt_rlen(n2)), alt_rlen(n3)) \
194 " - (" alt_slen ")) > 0) * " \
195 "(" alt_max_short(alt_max_short(alt_rlen(n1), alt_rlen(n2)), alt_rlen(n3)) \
196 " - (" alt_slen ")), 0x90\n" \
197 alt_end_marker ":\n"
198
199#define ALTINSTR_ENTRY(ft_flags, num) \
200 " .long 661b - .\n" /* label */ \
201 " .long " b_replacement(num)"f - .\n" /* new instruction */ \
202 " .4byte " __stringify(ft_flags) "\n" /* feature + flags */ \
203 " .byte " alt_total_slen "\n" /* source len */ \
204 " .byte " alt_rlen(num) "\n" /* replacement len */
205
206#define ALTINSTR_REPLACEMENT(newinstr, num) /* replacement */ \
207 "# ALT: replacement " #num "\n" \
208 b_replacement(num)":\n\t" newinstr "\n" e_replacement(num) ":\n"
209
210/* alternative assembly primitive: */
211#define ALTERNATIVE(oldinstr, newinstr, ft_flags) \
212 OLDINSTR(oldinstr, 1) \
213 ".pushsection .altinstructions,\"a\"\n" \
214 ALTINSTR_ENTRY(ft_flags, 1) \
215 ".popsection\n" \
216 ".pushsection .altinstr_replacement, \"ax\"\n" \
217 ALTINSTR_REPLACEMENT(newinstr, 1) \
218 ".popsection\n"
219
220#define ALTERNATIVE_2(oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2) \
221 OLDINSTR_2(oldinstr, 1, 2) \
222 ".pushsection .altinstructions,\"a\"\n" \
223 ALTINSTR_ENTRY(ft_flags1, 1) \
224 ALTINSTR_ENTRY(ft_flags2, 2) \
225 ".popsection\n" \
226 ".pushsection .altinstr_replacement, \"ax\"\n" \
227 ALTINSTR_REPLACEMENT(newinstr1, 1) \
228 ALTINSTR_REPLACEMENT(newinstr2, 2) \
229 ".popsection\n"
230
231/* If @feature is set, patch in @newinstr_yes, otherwise @newinstr_no. */
232#define ALTERNATIVE_TERNARY(oldinstr, ft_flags, newinstr_yes, newinstr_no) \
233 ALTERNATIVE_2(oldinstr, newinstr_no, X86_FEATURE_ALWAYS, \
234 newinstr_yes, ft_flags)
235
236#define ALTERNATIVE_3(oldinsn, newinsn1, ft_flags1, newinsn2, ft_flags2, \
237 newinsn3, ft_flags3) \
238 OLDINSTR_3(oldinsn, 1, 2, 3) \
239 ".pushsection .altinstructions,\"a\"\n" \
240 ALTINSTR_ENTRY(ft_flags1, 1) \
241 ALTINSTR_ENTRY(ft_flags2, 2) \
242 ALTINSTR_ENTRY(ft_flags3, 3) \
243 ".popsection\n" \
244 ".pushsection .altinstr_replacement, \"ax\"\n" \
245 ALTINSTR_REPLACEMENT(newinsn1, 1) \
246 ALTINSTR_REPLACEMENT(newinsn2, 2) \
247 ALTINSTR_REPLACEMENT(newinsn3, 3) \
248 ".popsection\n"
249
250/*
251 * Alternative instructions for different CPU types or capabilities.
252 *
253 * This allows to use optimized instructions even on generic binary
254 * kernels.
255 *
256 * length of oldinstr must be longer or equal the length of newinstr
257 * It can be padded with nops as needed.
258 *
259 * For non barrier like inlines please define new variants
260 * without volatile and memory clobber.
261 */
262#define alternative(oldinstr, newinstr, ft_flags) \
263 asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, ft_flags) : : : "memory")
264
265#define alternative_2(oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2) \
266 asm_inline volatile(ALTERNATIVE_2(oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2) ::: "memory")
267
268#define alternative_ternary(oldinstr, ft_flags, newinstr_yes, newinstr_no) \
269 asm_inline volatile(ALTERNATIVE_TERNARY(oldinstr, ft_flags, newinstr_yes, newinstr_no) ::: "memory")
270
271/*
272 * Alternative inline assembly with input.
273 *
274 * Peculiarities:
275 * No memory clobber here.
276 * Argument numbers start with 1.
277 * Leaving an unused argument 0 to keep API compatibility.
278 */
279#define alternative_input(oldinstr, newinstr, ft_flags, input...) \
280 asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, ft_flags) \
281 : : "i" (0), ## input)
282
283/*
284 * This is similar to alternative_input. But it has two features and
285 * respective instructions.
286 *
287 * If CPU has feature2, newinstr2 is used.
288 * Otherwise, if CPU has feature1, newinstr1 is used.
289 * Otherwise, oldinstr is used.
290 */
291#define alternative_input_2(oldinstr, newinstr1, ft_flags1, newinstr2, \
292 ft_flags2, input...) \
293 asm_inline volatile(ALTERNATIVE_2(oldinstr, newinstr1, ft_flags1, \
294 newinstr2, ft_flags2) \
295 : : "i" (0), ## input)
296
297/* Like alternative_input, but with a single output argument */
298#define alternative_io(oldinstr, newinstr, ft_flags, output, input...) \
299 asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, ft_flags) \
300 : output : "i" (0), ## input)
301
302/* Like alternative_io, but for replacing a direct call with another one. */
303#define alternative_call(oldfunc, newfunc, ft_flags, output, input...) \
304 asm_inline volatile (ALTERNATIVE("call %P[old]", "call %P[new]", ft_flags) \
305 : output : [old] "i" (oldfunc), [new] "i" (newfunc), ## input)
306
307/*
308 * Like alternative_call, but there are two features and respective functions.
309 * If CPU has feature2, function2 is used.
310 * Otherwise, if CPU has feature1, function1 is used.
311 * Otherwise, old function is used.
312 */
313#define alternative_call_2(oldfunc, newfunc1, ft_flags1, newfunc2, ft_flags2, \
314 output, input...) \
315 asm_inline volatile (ALTERNATIVE_2("call %P[old]", "call %P[new1]", ft_flags1,\
316 "call %P[new2]", ft_flags2) \
317 : output, ASM_CALL_CONSTRAINT \
318 : [old] "i" (oldfunc), [new1] "i" (newfunc1), \
319 [new2] "i" (newfunc2), ## input)
320
321/*
322 * use this macro(s) if you need more than one output parameter
323 * in alternative_io
324 */
325#define ASM_OUTPUT2(a...) a
326
327/*
328 * use this macro if you need clobbers but no inputs in
329 * alternative_{input,io,call}()
330 */
331#define ASM_NO_INPUT_CLOBBER(clbr...) "i" (0) : clbr
332
333#else /* __ASSEMBLY__ */
334
335#ifdef CONFIG_SMP
336 .macro LOCK_PREFIX
337672: lock
338 .pushsection .smp_locks,"a"
339 .balign 4
340 .long 672b - .
341 .popsection
342 .endm
343#else
344 .macro LOCK_PREFIX
345 .endm
346#endif
347
348/*
349 * objtool annotation to ignore the alternatives and only consider the original
350 * instruction(s).
351 */
352.macro ANNOTATE_IGNORE_ALTERNATIVE
353 .Lannotate_\@:
354 .pushsection .discard.ignore_alts
355 .long .Lannotate_\@
356 .popsection
357.endm
358
359/*
360 * Issue one struct alt_instr descriptor entry (need to put it into
361 * the section .altinstructions, see below). This entry contains
362 * enough information for the alternatives patching code to patch an
363 * instruction. See apply_alternatives().
364 */
365.macro altinstr_entry orig alt ft_flags orig_len alt_len
366 .long \orig - .
367 .long \alt - .
368 .4byte \ft_flags
369 .byte \orig_len
370 .byte \alt_len
371.endm
372
373/*
374 * Define an alternative between two instructions. If @feature is
375 * present, early code in apply_alternatives() replaces @oldinstr with
376 * @newinstr. ".skip" directive takes care of proper instruction padding
377 * in case @newinstr is longer than @oldinstr.
378 */
379.macro ALTERNATIVE oldinstr, newinstr, ft_flags
380140:
381 \oldinstr
382141:
383 .skip -(((144f-143f)-(141b-140b)) > 0) * ((144f-143f)-(141b-140b)),0x90
384142:
385
386 .pushsection .altinstructions,"a"
387 altinstr_entry 140b,143f,\ft_flags,142b-140b,144f-143f
388 .popsection
389
390 .pushsection .altinstr_replacement,"ax"
391143:
392 \newinstr
393144:
394 .popsection
395.endm
396
397#define old_len 141b-140b
398#define new_len1 144f-143f
399#define new_len2 145f-144f
400#define new_len3 146f-145f
401
402/*
403 * gas compatible max based on the idea from:
404 * http://graphics.stanford.edu/~seander/bithacks.html#IntegerMinOrMax
405 *
406 * The additional "-" is needed because gas uses a "true" value of -1.
407 */
408#define alt_max_2(a, b) ((a) ^ (((a) ^ (b)) & -(-((a) < (b)))))
409#define alt_max_3(a, b, c) (alt_max_2(alt_max_2(a, b), c))
410
411
412/*
413 * Same as ALTERNATIVE macro above but for two alternatives. If CPU
414 * has @feature1, it replaces @oldinstr with @newinstr1. If CPU has
415 * @feature2, it replaces @oldinstr with @feature2.
416 */
417.macro ALTERNATIVE_2 oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2
418140:
419 \oldinstr
420141:
421 .skip -((alt_max_2(new_len1, new_len2) - (old_len)) > 0) * \
422 (alt_max_2(new_len1, new_len2) - (old_len)),0x90
423142:
424
425 .pushsection .altinstructions,"a"
426 altinstr_entry 140b,143f,\ft_flags1,142b-140b,144f-143f
427 altinstr_entry 140b,144f,\ft_flags2,142b-140b,145f-144f
428 .popsection
429
430 .pushsection .altinstr_replacement,"ax"
431143:
432 \newinstr1
433144:
434 \newinstr2
435145:
436 .popsection
437.endm
438
439.macro ALTERNATIVE_3 oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2, newinstr3, ft_flags3
440140:
441 \oldinstr
442141:
443 .skip -((alt_max_3(new_len1, new_len2, new_len3) - (old_len)) > 0) * \
444 (alt_max_3(new_len1, new_len2, new_len3) - (old_len)),0x90
445142:
446
447 .pushsection .altinstructions,"a"
448 altinstr_entry 140b,143f,\ft_flags1,142b-140b,144f-143f
449 altinstr_entry 140b,144f,\ft_flags2,142b-140b,145f-144f
450 altinstr_entry 140b,145f,\ft_flags3,142b-140b,146f-145f
451 .popsection
452
453 .pushsection .altinstr_replacement,"ax"
454143:
455 \newinstr1
456144:
457 \newinstr2
458145:
459 \newinstr3
460146:
461 .popsection
462.endm
463
464/* If @feature is set, patch in @newinstr_yes, otherwise @newinstr_no. */
465#define ALTERNATIVE_TERNARY(oldinstr, ft_flags, newinstr_yes, newinstr_no) \
466 ALTERNATIVE_2 oldinstr, newinstr_no, X86_FEATURE_ALWAYS, \
467 newinstr_yes, ft_flags
468
469#endif /* __ASSEMBLY__ */
470
471#endif /* _ASM_X86_ALTERNATIVE_H */
472

source code of linux/arch/x86/include/asm/alternative.h