1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #ifndef _ASM_X86_ALTERNATIVE_H |
3 | #define _ASM_X86_ALTERNATIVE_H |
4 | |
5 | #ifndef __ASSEMBLY__ |
6 | |
7 | #include <linux/types.h> |
8 | #include <linux/stddef.h> |
9 | #include <linux/stringify.h> |
10 | #include <asm/asm.h> |
11 | |
12 | /* |
13 | * Alternative inline assembly for SMP. |
14 | * |
15 | * The LOCK_PREFIX macro defined here replaces the LOCK and |
16 | * LOCK_PREFIX macros used everywhere in the source tree. |
17 | * |
18 | * SMP alternatives use the same data structures as the other |
19 | * alternatives and the X86_FEATURE_UP flag to indicate the case of a |
20 | * UP system running a SMP kernel. The existing apply_alternatives() |
21 | * works fine for patching a SMP kernel for UP. |
22 | * |
23 | * The SMP alternative tables can be kept after boot and contain both |
24 | * UP and SMP versions of the instructions to allow switching back to |
25 | * SMP at runtime, when hotplugging in a new CPU, which is especially |
26 | * useful in virtualized environments. |
27 | * |
28 | * The very common lock prefix is handled as special case in a |
29 | * separate table which is a pure address list without replacement ptr |
30 | * and size information. That keeps the table sizes small. |
31 | */ |
32 | |
33 | #ifdef CONFIG_SMP |
34 | #define LOCK_PREFIX_HERE \ |
35 | ".pushsection .smp_locks,\"a\"\n" \ |
36 | ".balign 4\n" \ |
37 | ".long 671f - .\n" /* offset */ \ |
38 | ".popsection\n" \ |
39 | "671:" |
40 | |
41 | #define LOCK_PREFIX LOCK_PREFIX_HERE "\n\tlock; " |
42 | |
43 | #else /* ! CONFIG_SMP */ |
44 | #define LOCK_PREFIX_HERE "" |
45 | #define LOCK_PREFIX "" |
46 | #endif |
47 | |
48 | struct alt_instr { |
49 | s32 instr_offset; /* original instruction */ |
50 | s32 repl_offset; /* offset to replacement instruction */ |
51 | u16 cpuid; /* cpuid bit set for replacement */ |
52 | u8 instrlen; /* length of original instruction */ |
53 | u8 replacementlen; /* length of new instruction */ |
54 | u8 padlen; /* length of build-time padding */ |
55 | } __packed; |
56 | |
57 | /* |
58 | * Debug flag that can be tested to see whether alternative |
59 | * instructions were patched in already: |
60 | */ |
61 | extern int alternatives_patched; |
62 | |
63 | extern void alternative_instructions(void); |
64 | extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end); |
65 | |
66 | struct module; |
67 | |
68 | #ifdef CONFIG_SMP |
69 | extern void alternatives_smp_module_add(struct module *mod, char *name, |
70 | void *locks, void *locks_end, |
71 | void *text, void *text_end); |
72 | extern void alternatives_smp_module_del(struct module *mod); |
73 | extern void alternatives_enable_smp(void); |
74 | extern int alternatives_text_reserved(void *start, void *end); |
75 | extern bool skip_smp_alternatives; |
76 | #else |
77 | static inline void alternatives_smp_module_add(struct module *mod, char *name, |
78 | void *locks, void *locks_end, |
79 | void *text, void *text_end) {} |
80 | static inline void alternatives_smp_module_del(struct module *mod) {} |
81 | static inline void alternatives_enable_smp(void) {} |
82 | static inline int alternatives_text_reserved(void *start, void *end) |
83 | { |
84 | return 0; |
85 | } |
86 | #endif /* CONFIG_SMP */ |
87 | |
88 | #define b_replacement(num) "664"#num |
89 | #define e_replacement(num) "665"#num |
90 | |
91 | #define alt_end_marker "663" |
92 | #define alt_slen "662b-661b" |
93 | #define alt_pad_len alt_end_marker"b-662b" |
94 | #define alt_total_slen alt_end_marker"b-661b" |
95 | #define alt_rlen(num) e_replacement(num)"f-"b_replacement(num)"f" |
96 | |
97 | #define OLDINSTR(oldinstr, num) \ |
98 | "# ALT: oldnstr\n" \ |
99 | "661:\n\t" oldinstr "\n662:\n" \ |
100 | "# ALT: padding\n" \ |
101 | ".skip -(((" alt_rlen(num) ")-(" alt_slen ")) > 0) * " \ |
102 | "((" alt_rlen(num) ")-(" alt_slen ")),0x90\n" \ |
103 | alt_end_marker ":\n" |
104 | |
105 | /* |
106 | * gas compatible max based on the idea from: |
107 | * http://graphics.stanford.edu/~seander/bithacks.html#IntegerMinOrMax |
108 | * |
109 | * The additional "-" is needed because gas uses a "true" value of -1. |
110 | */ |
111 | #define alt_max_short(a, b) "((" a ") ^ (((" a ") ^ (" b ")) & -(-((" a ") < (" b ")))))" |
112 | |
113 | /* |
114 | * Pad the second replacement alternative with additional NOPs if it is |
115 | * additionally longer than the first replacement alternative. |
116 | */ |
117 | #define OLDINSTR_2(oldinstr, num1, num2) \ |
118 | "# ALT: oldinstr2\n" \ |
119 | "661:\n\t" oldinstr "\n662:\n" \ |
120 | "# ALT: padding2\n" \ |
121 | ".skip -((" alt_max_short(alt_rlen(num1), alt_rlen(num2)) " - (" alt_slen ")) > 0) * " \ |
122 | "(" alt_max_short(alt_rlen(num1), alt_rlen(num2)) " - (" alt_slen ")), 0x90\n" \ |
123 | alt_end_marker ":\n" |
124 | |
125 | #define OLDINSTR_3(oldinsn, n1, n2, n3) \ |
126 | "# ALT: oldinstr3\n" \ |
127 | "661:\n\t" oldinsn "\n662:\n" \ |
128 | "# ALT: padding3\n" \ |
129 | ".skip -((" alt_max_short(alt_max_short(alt_rlen(n1), alt_rlen(n2)), alt_rlen(n3)) \ |
130 | " - (" alt_slen ")) > 0) * " \ |
131 | "(" alt_max_short(alt_max_short(alt_rlen(n1), alt_rlen(n2)), alt_rlen(n3)) \ |
132 | " - (" alt_slen ")), 0x90\n" \ |
133 | alt_end_marker ":\n" |
134 | |
135 | #define ALTINSTR_ENTRY(feature, num) \ |
136 | " .long 661b - .\n" /* label */ \ |
137 | " .long " b_replacement(num)"f - .\n" /* new instruction */ \ |
138 | " .word " __stringify(feature) "\n" /* feature bit */ \ |
139 | " .byte " alt_total_slen "\n" /* source len */ \ |
140 | " .byte " alt_rlen(num) "\n" /* replacement len */ \ |
141 | " .byte " alt_pad_len "\n" /* pad len */ |
142 | |
143 | #define ALTINSTR_REPLACEMENT(newinstr, feature, num) /* replacement */ \ |
144 | "# ALT: replacement " #num "\n" \ |
145 | b_replacement(num)":\n\t" newinstr "\n" e_replacement(num) ":\n" |
146 | |
147 | /* alternative assembly primitive: */ |
148 | #define ALTERNATIVE(oldinstr, newinstr, feature) \ |
149 | OLDINSTR(oldinstr, 1) \ |
150 | ".pushsection .altinstructions,\"a\"\n" \ |
151 | ALTINSTR_ENTRY(feature, 1) \ |
152 | ".popsection\n" \ |
153 | ".pushsection .altinstr_replacement, \"ax\"\n" \ |
154 | ALTINSTR_REPLACEMENT(newinstr, feature, 1) \ |
155 | ".popsection\n" |
156 | |
157 | #define ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2)\ |
158 | OLDINSTR_2(oldinstr, 1, 2) \ |
159 | ".pushsection .altinstructions,\"a\"\n" \ |
160 | ALTINSTR_ENTRY(feature1, 1) \ |
161 | ALTINSTR_ENTRY(feature2, 2) \ |
162 | ".popsection\n" \ |
163 | ".pushsection .altinstr_replacement, \"ax\"\n" \ |
164 | ALTINSTR_REPLACEMENT(newinstr1, feature1, 1) \ |
165 | ALTINSTR_REPLACEMENT(newinstr2, feature2, 2) \ |
166 | ".popsection\n" |
167 | |
168 | #define ALTERNATIVE_3(oldinsn, newinsn1, feat1, newinsn2, feat2, newinsn3, feat3) \ |
169 | OLDINSTR_3(oldinsn, 1, 2, 3) \ |
170 | ".pushsection .altinstructions,\"a\"\n" \ |
171 | ALTINSTR_ENTRY(feat1, 1) \ |
172 | ALTINSTR_ENTRY(feat2, 2) \ |
173 | ALTINSTR_ENTRY(feat3, 3) \ |
174 | ".popsection\n" \ |
175 | ".pushsection .altinstr_replacement, \"ax\"\n" \ |
176 | ALTINSTR_REPLACEMENT(newinsn1, feat1, 1) \ |
177 | ALTINSTR_REPLACEMENT(newinsn2, feat2, 2) \ |
178 | ALTINSTR_REPLACEMENT(newinsn3, feat3, 3) \ |
179 | ".popsection\n" |
180 | |
181 | /* |
182 | * Alternative instructions for different CPU types or capabilities. |
183 | * |
184 | * This allows to use optimized instructions even on generic binary |
185 | * kernels. |
186 | * |
187 | * length of oldinstr must be longer or equal the length of newinstr |
188 | * It can be padded with nops as needed. |
189 | * |
190 | * For non barrier like inlines please define new variants |
191 | * without volatile and memory clobber. |
192 | */ |
193 | #define alternative(oldinstr, newinstr, feature) \ |
194 | asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) : : : "memory") |
195 | |
196 | #define alternative_2(oldinstr, newinstr1, feature1, newinstr2, feature2) \ |
197 | asm volatile(ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2) ::: "memory") |
198 | |
199 | /* |
200 | * Alternative inline assembly with input. |
201 | * |
202 | * Peculiarities: |
203 | * No memory clobber here. |
204 | * Argument numbers start with 1. |
205 | * Best is to use constraints that are fixed size (like (%1) ... "r") |
206 | * If you use variable sized constraints like "m" or "g" in the |
207 | * replacement make sure to pad to the worst case length. |
208 | * Leaving an unused argument 0 to keep API compatibility. |
209 | */ |
210 | #define alternative_input(oldinstr, newinstr, feature, input...) \ |
211 | asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) \ |
212 | : : "i" (0), ## input) |
213 | |
214 | /* |
215 | * This is similar to alternative_input. But it has two features and |
216 | * respective instructions. |
217 | * |
218 | * If CPU has feature2, newinstr2 is used. |
219 | * Otherwise, if CPU has feature1, newinstr1 is used. |
220 | * Otherwise, oldinstr is used. |
221 | */ |
222 | #define alternative_input_2(oldinstr, newinstr1, feature1, newinstr2, \ |
223 | feature2, input...) \ |
224 | asm volatile(ALTERNATIVE_2(oldinstr, newinstr1, feature1, \ |
225 | newinstr2, feature2) \ |
226 | : : "i" (0), ## input) |
227 | |
228 | /* Like alternative_input, but with a single output argument */ |
229 | #define alternative_io(oldinstr, newinstr, feature, output, input...) \ |
230 | asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) \ |
231 | : output : "i" (0), ## input) |
232 | |
233 | /* Like alternative_io, but for replacing a direct call with another one. */ |
234 | #define alternative_call(oldfunc, newfunc, feature, output, input...) \ |
235 | asm volatile (ALTERNATIVE("call %P[old]", "call %P[new]", feature) \ |
236 | : output : [old] "i" (oldfunc), [new] "i" (newfunc), ## input) |
237 | |
238 | /* |
239 | * Like alternative_call, but there are two features and respective functions. |
240 | * If CPU has feature2, function2 is used. |
241 | * Otherwise, if CPU has feature1, function1 is used. |
242 | * Otherwise, old function is used. |
243 | */ |
244 | #define alternative_call_2(oldfunc, newfunc1, feature1, newfunc2, feature2, \ |
245 | output, input...) \ |
246 | asm volatile (ALTERNATIVE_2("call %P[old]", "call %P[new1]", feature1,\ |
247 | "call %P[new2]", feature2) \ |
248 | : output, ASM_CALL_CONSTRAINT \ |
249 | : [old] "i" (oldfunc), [new1] "i" (newfunc1), \ |
250 | [new2] "i" (newfunc2), ## input) |
251 | |
252 | /* |
253 | * use this macro(s) if you need more than one output parameter |
254 | * in alternative_io |
255 | */ |
256 | #define ASM_OUTPUT2(a...) a |
257 | |
258 | /* |
259 | * use this macro if you need clobbers but no inputs in |
260 | * alternative_{input,io,call}() |
261 | */ |
262 | #define ASM_NO_INPUT_CLOBBER(clbr...) "i" (0) : clbr |
263 | |
264 | #endif /* __ASSEMBLY__ */ |
265 | |
266 | #endif /* _ASM_X86_ALTERNATIVE_H */ |
267 | |