1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | |
3 | #ifndef _ASM_X86_NOSPEC_BRANCH_H_ |
4 | #define _ASM_X86_NOSPEC_BRANCH_H_ |
5 | |
6 | #include <linux/static_key.h> |
7 | |
8 | #include <asm/alternative.h> |
9 | #include <asm/alternative-asm.h> |
10 | #include <asm/cpufeatures.h> |
11 | #include <asm/msr-index.h> |
12 | |
13 | /* |
14 | * Fill the CPU return stack buffer. |
15 | * |
16 | * Each entry in the RSB, if used for a speculative 'ret', contains an |
17 | * infinite 'pause; lfence; jmp' loop to capture speculative execution. |
18 | * |
19 | * This is required in various cases for retpoline and IBRS-based |
20 | * mitigations for the Spectre variant 2 vulnerability. Sometimes to |
21 | * eliminate potentially bogus entries from the RSB, and sometimes |
22 | * purely to ensure that it doesn't get empty, which on some CPUs would |
23 | * allow predictions from other (unwanted!) sources to be used. |
24 | * |
25 | * We define a CPP macro such that it can be used from both .S files and |
26 | * inline assembly. It's possible to do a .macro and then include that |
27 | * from C via asm(".include <asm/nospec-branch.h>") but let's not go there. |
28 | */ |
29 | |
30 | #define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */ |
31 | #define RSB_FILL_LOOPS 16 /* To avoid underflow */ |
32 | |
33 | /* |
34 | * Google experimented with loop-unrolling and this turned out to be |
35 | * the optimal version — two calls, each with their own speculation |
36 | * trap should their return address end up getting used, in a loop. |
37 | */ |
38 | #define __FILL_RETURN_BUFFER(reg, nr, sp) \ |
39 | mov $(nr/2), reg; \ |
40 | 771: \ |
41 | call 772f; \ |
42 | 773: /* speculation trap */ \ |
43 | pause; \ |
44 | lfence; \ |
45 | jmp 773b; \ |
46 | 772: \ |
47 | call 774f; \ |
48 | 775: /* speculation trap */ \ |
49 | pause; \ |
50 | lfence; \ |
51 | jmp 775b; \ |
52 | 774: \ |
53 | dec reg; \ |
54 | jnz 771b; \ |
55 | add $(BITS_PER_LONG/8) * nr, sp; |
56 | |
57 | #ifdef __ASSEMBLY__ |
58 | |
59 | /* |
60 | * This should be used immediately before a retpoline alternative. It tells |
61 | * objtool where the retpolines are so that it can make sense of the control |
62 | * flow by just reading the original instruction(s) and ignoring the |
63 | * alternatives. |
64 | */ |
65 | .macro ANNOTATE_NOSPEC_ALTERNATIVE |
66 | .Lannotate_\@: |
67 | .pushsection .discard.nospec |
68 | .long .Lannotate_\@ - . |
69 | .popsection |
70 | .endm |
71 | |
72 | /* |
73 | * This should be used immediately before an indirect jump/call. It tells |
74 | * objtool the subsequent indirect jump/call is vouched safe for retpoline |
75 | * builds. |
76 | */ |
77 | .macro ANNOTATE_RETPOLINE_SAFE |
78 | .Lannotate_\@: |
79 | .pushsection .discard.retpoline_safe |
80 | _ASM_PTR .Lannotate_\@ |
81 | .popsection |
82 | .endm |
83 | |
84 | /* |
85 | * These are the bare retpoline primitives for indirect jmp and call. |
86 | * Do not use these directly; they only exist to make the ALTERNATIVE |
87 | * invocation below less ugly. |
88 | */ |
89 | .macro RETPOLINE_JMP reg:req |
90 | call .Ldo_rop_\@ |
91 | .Lspec_trap_\@: |
92 | pause |
93 | lfence |
94 | jmp .Lspec_trap_\@ |
95 | .Ldo_rop_\@: |
96 | mov \reg, (%_ASM_SP) |
97 | ret |
98 | .endm |
99 | |
100 | /* |
101 | * This is a wrapper around RETPOLINE_JMP so the called function in reg |
102 | * returns to the instruction after the macro. |
103 | */ |
104 | .macro RETPOLINE_CALL reg:req |
105 | jmp .Ldo_call_\@ |
106 | .Ldo_retpoline_jmp_\@: |
107 | RETPOLINE_JMP \reg |
108 | .Ldo_call_\@: |
109 | call .Ldo_retpoline_jmp_\@ |
110 | .endm |
111 | |
112 | /* |
113 | * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple |
114 | * indirect jmp/call which may be susceptible to the Spectre variant 2 |
115 | * attack. |
116 | */ |
117 | .macro JMP_NOSPEC reg:req |
118 | #ifdef CONFIG_RETPOLINE |
119 | ANNOTATE_NOSPEC_ALTERNATIVE |
120 | ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *\reg), \ |
121 | __stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE, \ |
122 | __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *\reg), X86_FEATURE_RETPOLINE_AMD |
123 | #else |
124 | jmp *\reg |
125 | #endif |
126 | .endm |
127 | |
128 | .macro CALL_NOSPEC reg:req |
129 | #ifdef CONFIG_RETPOLINE |
130 | ANNOTATE_NOSPEC_ALTERNATIVE |
131 | ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; call *\reg), \ |
132 | __stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\ |
133 | __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *\reg), X86_FEATURE_RETPOLINE_AMD |
134 | #else |
135 | call *\reg |
136 | #endif |
137 | .endm |
138 | |
139 | /* |
140 | * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP |
141 | * monstrosity above, manually. |
142 | */ |
143 | .macro FILL_RETURN_BUFFER reg:req nr:req ftr:req |
144 | #ifdef CONFIG_RETPOLINE |
145 | ANNOTATE_NOSPEC_ALTERNATIVE |
146 | ALTERNATIVE "jmp .Lskip_rsb_\@" , \ |
147 | __stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP)) \ |
148 | \ftr |
149 | .Lskip_rsb_\@: |
150 | #endif |
151 | .endm |
152 | |
153 | #else /* __ASSEMBLY__ */ |
154 | |
155 | #define ANNOTATE_NOSPEC_ALTERNATIVE \ |
156 | "999:\n\t" \ |
157 | ".pushsection .discard.nospec\n\t" \ |
158 | ".long 999b - .\n\t" \ |
159 | ".popsection\n\t" |
160 | |
161 | #define ANNOTATE_RETPOLINE_SAFE \ |
162 | "999:\n\t" \ |
163 | ".pushsection .discard.retpoline_safe\n\t" \ |
164 | _ASM_PTR " 999b\n\t" \ |
165 | ".popsection\n\t" |
166 | |
167 | #ifdef CONFIG_RETPOLINE |
168 | #ifdef CONFIG_X86_64 |
169 | |
170 | /* |
171 | * Inline asm uses the %V modifier which is only in newer GCC |
172 | * which is ensured when CONFIG_RETPOLINE is defined. |
173 | */ |
174 | # define CALL_NOSPEC \ |
175 | ANNOTATE_NOSPEC_ALTERNATIVE \ |
176 | ALTERNATIVE_2( \ |
177 | ANNOTATE_RETPOLINE_SAFE \ |
178 | "call *%[thunk_target]\n", \ |
179 | "call __x86_indirect_thunk_%V[thunk_target]\n", \ |
180 | X86_FEATURE_RETPOLINE, \ |
181 | "lfence;\n" \ |
182 | ANNOTATE_RETPOLINE_SAFE \ |
183 | "call *%[thunk_target]\n", \ |
184 | X86_FEATURE_RETPOLINE_AMD) |
185 | # define THUNK_TARGET(addr) [thunk_target] "r" (addr) |
186 | |
187 | #else /* CONFIG_X86_32 */ |
188 | /* |
189 | * For i386 we use the original ret-equivalent retpoline, because |
190 | * otherwise we'll run out of registers. We don't care about CET |
191 | * here, anyway. |
192 | */ |
193 | # define CALL_NOSPEC \ |
194 | ANNOTATE_NOSPEC_ALTERNATIVE \ |
195 | ALTERNATIVE_2( \ |
196 | ANNOTATE_RETPOLINE_SAFE \ |
197 | "call *%[thunk_target]\n", \ |
198 | " jmp 904f;\n" \ |
199 | " .align 16\n" \ |
200 | "901: call 903f;\n" \ |
201 | "902: pause;\n" \ |
202 | " lfence;\n" \ |
203 | " jmp 902b;\n" \ |
204 | " .align 16\n" \ |
205 | "903: addl $4, %%esp;\n" \ |
206 | " pushl %[thunk_target];\n" \ |
207 | " ret;\n" \ |
208 | " .align 16\n" \ |
209 | "904: call 901b;\n", \ |
210 | X86_FEATURE_RETPOLINE, \ |
211 | "lfence;\n" \ |
212 | ANNOTATE_RETPOLINE_SAFE \ |
213 | "call *%[thunk_target]\n", \ |
214 | X86_FEATURE_RETPOLINE_AMD) |
215 | |
216 | # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) |
217 | #endif |
218 | #else /* No retpoline for C / inline asm */ |
219 | # define CALL_NOSPEC "call *%[thunk_target]\n" |
220 | # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) |
221 | #endif |
222 | |
223 | /* The Spectre V2 mitigation variants */ |
224 | enum spectre_v2_mitigation { |
225 | SPECTRE_V2_NONE, |
226 | SPECTRE_V2_RETPOLINE_GENERIC, |
227 | SPECTRE_V2_RETPOLINE_AMD, |
228 | SPECTRE_V2_IBRS_ENHANCED, |
229 | }; |
230 | |
231 | /* The indirect branch speculation control variants */ |
232 | enum spectre_v2_user_mitigation { |
233 | SPECTRE_V2_USER_NONE, |
234 | SPECTRE_V2_USER_STRICT, |
235 | SPECTRE_V2_USER_STRICT_PREFERRED, |
236 | SPECTRE_V2_USER_PRCTL, |
237 | SPECTRE_V2_USER_SECCOMP, |
238 | }; |
239 | |
240 | /* The Speculative Store Bypass disable variants */ |
241 | enum ssb_mitigation { |
242 | SPEC_STORE_BYPASS_NONE, |
243 | SPEC_STORE_BYPASS_DISABLE, |
244 | SPEC_STORE_BYPASS_PRCTL, |
245 | SPEC_STORE_BYPASS_SECCOMP, |
246 | }; |
247 | |
248 | extern char __indirect_thunk_start[]; |
249 | extern char __indirect_thunk_end[]; |
250 | |
251 | /* |
252 | * On VMEXIT we must ensure that no RSB predictions learned in the guest |
253 | * can be followed in the host, by overwriting the RSB completely. Both |
254 | * retpoline and IBRS mitigations for Spectre v2 need this; only on future |
255 | * CPUs with IBRS_ALL *might* it be avoided. |
256 | */ |
257 | static inline void vmexit_fill_RSB(void) |
258 | { |
259 | #ifdef CONFIG_RETPOLINE |
260 | unsigned long loops; |
261 | |
262 | asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE |
263 | ALTERNATIVE("jmp 910f" , |
264 | __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)), |
265 | X86_FEATURE_RETPOLINE) |
266 | "910:" |
267 | : "=r" (loops), ASM_CALL_CONSTRAINT |
268 | : : "memory" ); |
269 | #endif |
270 | } |
271 | |
272 | static __always_inline |
273 | void alternative_msr_write(unsigned int msr, u64 val, unsigned int feature) |
274 | { |
275 | asm volatile(ALTERNATIVE("" , "wrmsr" , %c[feature]) |
276 | : : "c" (msr), |
277 | "a" ((u32)val), |
278 | "d" ((u32)(val >> 32)), |
279 | [feature] "i" (feature) |
280 | : "memory" ); |
281 | } |
282 | |
283 | static inline void indirect_branch_prediction_barrier(void) |
284 | { |
285 | u64 val = PRED_CMD_IBPB; |
286 | |
287 | alternative_msr_write(MSR_IA32_PRED_CMD, val, X86_FEATURE_USE_IBPB); |
288 | } |
289 | |
290 | /* The Intel SPEC CTRL MSR base value cache */ |
291 | extern u64 x86_spec_ctrl_base; |
292 | |
293 | /* |
294 | * With retpoline, we must use IBRS to restrict branch prediction |
295 | * before calling into firmware. |
296 | * |
297 | * (Implemented as CPP macros due to header hell.) |
298 | */ |
299 | #define firmware_restrict_branch_speculation_start() \ |
300 | do { \ |
301 | u64 val = x86_spec_ctrl_base | SPEC_CTRL_IBRS; \ |
302 | \ |
303 | preempt_disable(); \ |
304 | alternative_msr_write(MSR_IA32_SPEC_CTRL, val, \ |
305 | X86_FEATURE_USE_IBRS_FW); \ |
306 | } while (0) |
307 | |
308 | #define firmware_restrict_branch_speculation_end() \ |
309 | do { \ |
310 | u64 val = x86_spec_ctrl_base; \ |
311 | \ |
312 | alternative_msr_write(MSR_IA32_SPEC_CTRL, val, \ |
313 | X86_FEATURE_USE_IBRS_FW); \ |
314 | preempt_enable(); \ |
315 | } while (0) |
316 | |
317 | DECLARE_STATIC_KEY_FALSE(switch_to_cond_stibp); |
318 | DECLARE_STATIC_KEY_FALSE(switch_mm_cond_ibpb); |
319 | DECLARE_STATIC_KEY_FALSE(switch_mm_always_ibpb); |
320 | |
321 | #endif /* __ASSEMBLY__ */ |
322 | |
323 | /* |
324 | * Below is used in the eBPF JIT compiler and emits the byte sequence |
325 | * for the following assembly: |
326 | * |
327 | * With retpolines configured: |
328 | * |
329 | * callq do_rop |
330 | * spec_trap: |
331 | * pause |
332 | * lfence |
333 | * jmp spec_trap |
334 | * do_rop: |
335 | * mov %rax,(%rsp) for x86_64 |
336 | * mov %edx,(%esp) for x86_32 |
337 | * retq |
338 | * |
339 | * Without retpolines configured: |
340 | * |
341 | * jmp *%rax for x86_64 |
342 | * jmp *%edx for x86_32 |
343 | */ |
344 | #ifdef CONFIG_RETPOLINE |
345 | # ifdef CONFIG_X86_64 |
346 | # define RETPOLINE_RAX_BPF_JIT_SIZE 17 |
347 | # define RETPOLINE_RAX_BPF_JIT() \ |
348 | do { \ |
349 | EMIT1_off32(0xE8, 7); /* callq do_rop */ \ |
350 | /* spec_trap: */ \ |
351 | EMIT2(0xF3, 0x90); /* pause */ \ |
352 | EMIT3(0x0F, 0xAE, 0xE8); /* lfence */ \ |
353 | EMIT2(0xEB, 0xF9); /* jmp spec_trap */ \ |
354 | /* do_rop: */ \ |
355 | EMIT4(0x48, 0x89, 0x04, 0x24); /* mov %rax,(%rsp) */ \ |
356 | EMIT1(0xC3); /* retq */ \ |
357 | } while (0) |
358 | # else /* !CONFIG_X86_64 */ |
359 | # define RETPOLINE_EDX_BPF_JIT() \ |
360 | do { \ |
361 | EMIT1_off32(0xE8, 7); /* call do_rop */ \ |
362 | /* spec_trap: */ \ |
363 | EMIT2(0xF3, 0x90); /* pause */ \ |
364 | EMIT3(0x0F, 0xAE, 0xE8); /* lfence */ \ |
365 | EMIT2(0xEB, 0xF9); /* jmp spec_trap */ \ |
366 | /* do_rop: */ \ |
367 | EMIT3(0x89, 0x14, 0x24); /* mov %edx,(%esp) */ \ |
368 | EMIT1(0xC3); /* ret */ \ |
369 | } while (0) |
370 | # endif |
371 | #else /* !CONFIG_RETPOLINE */ |
372 | # ifdef CONFIG_X86_64 |
373 | # define RETPOLINE_RAX_BPF_JIT_SIZE 2 |
374 | # define RETPOLINE_RAX_BPF_JIT() \ |
375 | EMIT2(0xFF, 0xE0); /* jmp *%rax */ |
376 | # else /* !CONFIG_X86_64 */ |
377 | # define RETPOLINE_EDX_BPF_JIT() \ |
378 | EMIT2(0xFF, 0xE2) /* jmp *%edx */ |
379 | # endif |
380 | #endif |
381 | |
382 | #endif /* _ASM_X86_NOSPEC_BRANCH_H_ */ |
383 | |