1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #ifndef __X86_KERNEL_FPU_XSTATE_H |
3 | #define __X86_KERNEL_FPU_XSTATE_H |
4 | |
5 | #include <asm/cpufeature.h> |
6 | #include <asm/fpu/xstate.h> |
7 | #include <asm/fpu/xcr.h> |
8 | |
9 | #ifdef CONFIG_X86_64 |
10 | DECLARE_PER_CPU(u64, xfd_state); |
11 | #endif |
12 | |
13 | static inline void xstate_init_xcomp_bv(struct xregs_state *xsave, u64 mask) |
14 | { |
15 | /* |
16 | * XRSTORS requires these bits set in xcomp_bv, or it will |
17 | * trigger #GP: |
18 | */ |
19 | if (cpu_feature_enabled(X86_FEATURE_XCOMPACTED)) |
20 | xsave->header.xcomp_bv = mask | XCOMP_BV_COMPACTED_FORMAT; |
21 | } |
22 | |
23 | static inline u64 xstate_get_group_perm(bool guest) |
24 | { |
25 | struct fpu *fpu = ¤t->group_leader->thread.fpu; |
26 | struct fpu_state_perm *perm; |
27 | |
28 | /* Pairs with WRITE_ONCE() in xstate_request_perm() */ |
29 | perm = guest ? &fpu->guest_perm : &fpu->perm; |
30 | return READ_ONCE(perm->__state_perm); |
31 | } |
32 | |
33 | static inline u64 xstate_get_host_group_perm(void) |
34 | { |
35 | return xstate_get_group_perm(guest: false); |
36 | } |
37 | |
38 | enum xstate_copy_mode { |
39 | XSTATE_COPY_FP, |
40 | XSTATE_COPY_FX, |
41 | XSTATE_COPY_XSAVE, |
42 | }; |
43 | |
44 | struct membuf; |
45 | extern void __copy_xstate_to_uabi_buf(struct membuf to, struct fpstate *fpstate, |
46 | u64 xfeatures, u32 pkru_val, |
47 | enum xstate_copy_mode copy_mode); |
48 | extern void copy_xstate_to_uabi_buf(struct membuf to, struct task_struct *tsk, |
49 | enum xstate_copy_mode mode); |
50 | extern int copy_uabi_from_kernel_to_xstate(struct fpstate *fpstate, const void *kbuf, u32 *pkru); |
51 | extern int copy_sigframe_from_user_to_xstate(struct task_struct *tsk, const void __user *ubuf); |
52 | |
53 | |
54 | extern void fpu__init_cpu_xstate(void); |
55 | extern void fpu__init_system_xstate(unsigned int legacy_size); |
56 | |
57 | extern void *get_xsave_addr(struct xregs_state *xsave, int xfeature_nr); |
58 | |
59 | static inline u64 xfeatures_mask_supervisor(void) |
60 | { |
61 | return fpu_kernel_cfg.max_features & XFEATURE_MASK_SUPERVISOR_SUPPORTED; |
62 | } |
63 | |
64 | static inline u64 xfeatures_mask_independent(void) |
65 | { |
66 | if (!cpu_feature_enabled(X86_FEATURE_ARCH_LBR)) |
67 | return XFEATURE_MASK_INDEPENDENT & ~XFEATURE_MASK_LBR; |
68 | |
69 | return XFEATURE_MASK_INDEPENDENT; |
70 | } |
71 | |
72 | /* XSAVE/XRSTOR wrapper functions */ |
73 | |
74 | #ifdef CONFIG_X86_64 |
75 | #define REX_PREFIX "0x48, " |
76 | #else |
77 | #define REX_PREFIX |
78 | #endif |
79 | |
80 | /* These macros all use (%edi)/(%rdi) as the single memory argument. */ |
81 | #define XSAVE ".byte " REX_PREFIX "0x0f,0xae,0x27" |
82 | #define XSAVEOPT ".byte " REX_PREFIX "0x0f,0xae,0x37" |
83 | #define XSAVEC ".byte " REX_PREFIX "0x0f,0xc7,0x27" |
84 | #define XSAVES ".byte " REX_PREFIX "0x0f,0xc7,0x2f" |
85 | #define XRSTOR ".byte " REX_PREFIX "0x0f,0xae,0x2f" |
86 | #define XRSTORS ".byte " REX_PREFIX "0x0f,0xc7,0x1f" |
87 | |
88 | /* |
89 | * After this @err contains 0 on success or the trap number when the |
90 | * operation raises an exception. |
91 | */ |
92 | #define XSTATE_OP(op, st, lmask, hmask, err) \ |
93 | asm volatile("1:" op "\n\t" \ |
94 | "xor %[err], %[err]\n" \ |
95 | "2:\n\t" \ |
96 | _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_FAULT_MCE_SAFE) \ |
97 | : [err] "=a" (err) \ |
98 | : "D" (st), "m" (*st), "a" (lmask), "d" (hmask) \ |
99 | : "memory") |
100 | |
101 | /* |
102 | * If XSAVES is enabled, it replaces XSAVEC because it supports supervisor |
103 | * states in addition to XSAVEC. |
104 | * |
105 | * Otherwise if XSAVEC is enabled, it replaces XSAVEOPT because it supports |
106 | * compacted storage format in addition to XSAVEOPT. |
107 | * |
108 | * Otherwise, if XSAVEOPT is enabled, XSAVEOPT replaces XSAVE because XSAVEOPT |
109 | * supports modified optimization which is not supported by XSAVE. |
110 | * |
111 | * We use XSAVE as a fallback. |
112 | * |
113 | * The 661 label is defined in the ALTERNATIVE* macros as the address of the |
114 | * original instruction which gets replaced. We need to use it here as the |
115 | * address of the instruction where we might get an exception at. |
116 | */ |
117 | #define XSTATE_XSAVE(st, lmask, hmask, err) \ |
118 | asm volatile(ALTERNATIVE_3(XSAVE, \ |
119 | XSAVEOPT, X86_FEATURE_XSAVEOPT, \ |
120 | XSAVEC, X86_FEATURE_XSAVEC, \ |
121 | XSAVES, X86_FEATURE_XSAVES) \ |
122 | "\n" \ |
123 | "xor %[err], %[err]\n" \ |
124 | "3:\n" \ |
125 | _ASM_EXTABLE_TYPE_REG(661b, 3b, EX_TYPE_EFAULT_REG, %[err]) \ |
126 | : [err] "=r" (err) \ |
127 | : "D" (st), "m" (*st), "a" (lmask), "d" (hmask) \ |
128 | : "memory") |
129 | |
130 | /* |
131 | * Use XRSTORS to restore context if it is enabled. XRSTORS supports compact |
132 | * XSAVE area format. |
133 | */ |
134 | #define XSTATE_XRESTORE(st, lmask, hmask) \ |
135 | asm volatile(ALTERNATIVE(XRSTOR, \ |
136 | XRSTORS, X86_FEATURE_XSAVES) \ |
137 | "\n" \ |
138 | "3:\n" \ |
139 | _ASM_EXTABLE_TYPE(661b, 3b, EX_TYPE_FPU_RESTORE) \ |
140 | : \ |
141 | : "D" (st), "m" (*st), "a" (lmask), "d" (hmask) \ |
142 | : "memory") |
143 | |
144 | #if defined(CONFIG_X86_64) && defined(CONFIG_X86_DEBUG_FPU) |
145 | extern void xfd_validate_state(struct fpstate *fpstate, u64 mask, bool rstor); |
146 | #else |
147 | static inline void xfd_validate_state(struct fpstate *fpstate, u64 mask, bool rstor) { } |
148 | #endif |
149 | |
150 | #ifdef CONFIG_X86_64 |
151 | static inline void xfd_set_state(u64 xfd) |
152 | { |
153 | wrmsrl(MSR_IA32_XFD, val: xfd); |
154 | __this_cpu_write(xfd_state, xfd); |
155 | } |
156 | |
157 | static inline void xfd_update_state(struct fpstate *fpstate) |
158 | { |
159 | if (fpu_state_size_dynamic()) { |
160 | u64 xfd = fpstate->xfd; |
161 | |
162 | if (__this_cpu_read(xfd_state) != xfd) |
163 | xfd_set_state(xfd); |
164 | } |
165 | } |
166 | |
167 | extern int __xfd_enable_feature(u64 which, struct fpu_guest *guest_fpu); |
168 | #else |
169 | static inline void xfd_set_state(u64 xfd) { } |
170 | |
171 | static inline void xfd_update_state(struct fpstate *fpstate) { } |
172 | |
173 | static inline int __xfd_enable_feature(u64 which, struct fpu_guest *guest_fpu) { |
174 | return -EPERM; |
175 | } |
176 | #endif |
177 | |
178 | /* |
179 | * Save processor xstate to xsave area. |
180 | * |
181 | * Uses either XSAVE or XSAVEOPT or XSAVES depending on the CPU features |
182 | * and command line options. The choice is permanent until the next reboot. |
183 | */ |
184 | static inline void os_xsave(struct fpstate *fpstate) |
185 | { |
186 | u64 mask = fpstate->xfeatures; |
187 | u32 lmask = mask; |
188 | u32 hmask = mask >> 32; |
189 | int err; |
190 | |
191 | WARN_ON_FPU(!alternatives_patched); |
192 | xfd_validate_state(fpstate, mask, rstor: false); |
193 | |
194 | XSTATE_XSAVE(&fpstate->regs.xsave, lmask, hmask, err); |
195 | |
196 | /* We should never fault when copying to a kernel buffer: */ |
197 | WARN_ON_FPU(err); |
198 | } |
199 | |
200 | /* |
201 | * Restore processor xstate from xsave area. |
202 | * |
203 | * Uses XRSTORS when XSAVES is used, XRSTOR otherwise. |
204 | */ |
205 | static inline void os_xrstor(struct fpstate *fpstate, u64 mask) |
206 | { |
207 | u32 lmask = mask; |
208 | u32 hmask = mask >> 32; |
209 | |
210 | xfd_validate_state(fpstate, mask, rstor: true); |
211 | XSTATE_XRESTORE(&fpstate->regs.xsave, lmask, hmask); |
212 | } |
213 | |
214 | /* Restore of supervisor state. Does not require XFD */ |
215 | static inline void os_xrstor_supervisor(struct fpstate *fpstate) |
216 | { |
217 | u64 mask = xfeatures_mask_supervisor(); |
218 | u32 lmask = mask; |
219 | u32 hmask = mask >> 32; |
220 | |
221 | XSTATE_XRESTORE(&fpstate->regs.xsave, lmask, hmask); |
222 | } |
223 | |
224 | /* |
225 | * XSAVE itself always writes all requested xfeatures. Removing features |
226 | * from the request bitmap reduces the features which are written. |
227 | * Generate a mask of features which must be written to a sigframe. The |
228 | * unset features can be optimized away and not written. |
229 | * |
230 | * This optimization is user-visible. Only use for states where |
231 | * uninitialized sigframe contents are tolerable, like dynamic features. |
232 | * |
233 | * Users of buffers produced with this optimization must check XSTATE_BV |
234 | * to determine which features have been optimized out. |
235 | */ |
236 | static inline u64 xfeatures_need_sigframe_write(void) |
237 | { |
238 | u64 xfeaures_to_write; |
239 | |
240 | /* In-use features must be written: */ |
241 | xfeaures_to_write = xfeatures_in_use(); |
242 | |
243 | /* Also write all non-optimizable sigframe features: */ |
244 | xfeaures_to_write |= XFEATURE_MASK_USER_SUPPORTED & |
245 | ~XFEATURE_MASK_SIGFRAME_INITOPT; |
246 | |
247 | return xfeaures_to_write; |
248 | } |
249 | |
250 | /* |
251 | * Save xstate to user space xsave area. |
252 | * |
253 | * We don't use modified optimization because xrstor/xrstors might track |
254 | * a different application. |
255 | * |
256 | * We don't use compacted format xsave area for backward compatibility for |
257 | * old applications which don't understand the compacted format of the |
258 | * xsave area. |
259 | * |
260 | * The caller has to zero buf::header before calling this because XSAVE* |
261 | * does not touch the reserved fields in the header. |
262 | */ |
263 | static inline int xsave_to_user_sigframe(struct xregs_state __user *buf) |
264 | { |
265 | /* |
266 | * Include the features which are not xsaved/rstored by the kernel |
267 | * internally, e.g. PKRU. That's user space ABI and also required |
268 | * to allow the signal handler to modify PKRU. |
269 | */ |
270 | struct fpstate *fpstate = current->thread.fpu.fpstate; |
271 | u64 mask = fpstate->user_xfeatures; |
272 | u32 lmask; |
273 | u32 hmask; |
274 | int err; |
275 | |
276 | /* Optimize away writing unnecessary xfeatures: */ |
277 | if (fpu_state_size_dynamic()) |
278 | mask &= xfeatures_need_sigframe_write(); |
279 | |
280 | lmask = mask; |
281 | hmask = mask >> 32; |
282 | xfd_validate_state(fpstate, mask, rstor: false); |
283 | |
284 | stac(); |
285 | XSTATE_OP(XSAVE, buf, lmask, hmask, err); |
286 | clac(); |
287 | |
288 | return err; |
289 | } |
290 | |
291 | /* |
292 | * Restore xstate from user space xsave area. |
293 | */ |
294 | static inline int xrstor_from_user_sigframe(struct xregs_state __user *buf, u64 mask) |
295 | { |
296 | struct xregs_state *xstate = ((__force struct xregs_state *)buf); |
297 | u32 lmask = mask; |
298 | u32 hmask = mask >> 32; |
299 | int err; |
300 | |
301 | xfd_validate_state(current->thread.fpu.fpstate, mask, rstor: true); |
302 | |
303 | stac(); |
304 | XSTATE_OP(XRSTOR, xstate, lmask, hmask, err); |
305 | clac(); |
306 | |
307 | return err; |
308 | } |
309 | |
310 | /* |
311 | * Restore xstate from kernel space xsave area, return an error code instead of |
312 | * an exception. |
313 | */ |
314 | static inline int os_xrstor_safe(struct fpstate *fpstate, u64 mask) |
315 | { |
316 | struct xregs_state *xstate = &fpstate->regs.xsave; |
317 | u32 lmask = mask; |
318 | u32 hmask = mask >> 32; |
319 | int err; |
320 | |
321 | /* Ensure that XFD is up to date */ |
322 | xfd_update_state(fpstate); |
323 | |
324 | if (cpu_feature_enabled(X86_FEATURE_XSAVES)) |
325 | XSTATE_OP(XRSTORS, xstate, lmask, hmask, err); |
326 | else |
327 | XSTATE_OP(XRSTOR, xstate, lmask, hmask, err); |
328 | |
329 | return err; |
330 | } |
331 | |
332 | |
333 | #endif |
334 | |