1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * FPU register's regset abstraction, for ptrace, core dumps, etc. |
4 | */ |
5 | #include <linux/sched/task_stack.h> |
6 | #include <linux/vmalloc.h> |
7 | |
8 | #include <asm/fpu/api.h> |
9 | #include <asm/fpu/signal.h> |
10 | #include <asm/fpu/regset.h> |
11 | #include <asm/prctl.h> |
12 | |
13 | #include "context.h" |
14 | #include "internal.h" |
15 | #include "legacy.h" |
16 | #include "xstate.h" |
17 | |
18 | /* |
19 | * The xstateregs_active() routine is the same as the regset_fpregs_active() routine, |
20 | * as the "regset->n" for the xstate regset will be updated based on the feature |
21 | * capabilities supported by the xsave. |
22 | */ |
23 | int regset_fpregs_active(struct task_struct *target, const struct user_regset *regset) |
24 | { |
25 | return regset->n; |
26 | } |
27 | |
28 | int regset_xregset_fpregs_active(struct task_struct *target, const struct user_regset *regset) |
29 | { |
30 | if (boot_cpu_has(X86_FEATURE_FXSR)) |
31 | return regset->n; |
32 | else |
33 | return 0; |
34 | } |
35 | |
36 | /* |
37 | * The regset get() functions are invoked from: |
38 | * |
39 | * - coredump to dump the current task's fpstate. If the current task |
40 | * owns the FPU then the memory state has to be synchronized and the |
41 | * FPU register state preserved. Otherwise fpstate is already in sync. |
42 | * |
43 | * - ptrace to dump fpstate of a stopped task, in which case the registers |
44 | * have already been saved to fpstate on context switch. |
45 | */ |
46 | static void sync_fpstate(struct fpu *fpu) |
47 | { |
48 | if (fpu == ¤t->thread.fpu) |
49 | fpu_sync_fpstate(fpu); |
50 | } |
51 | |
52 | /* |
53 | * Invalidate cached FPU registers before modifying the stopped target |
54 | * task's fpstate. |
55 | * |
56 | * This forces the target task on resume to restore the FPU registers from |
57 | * modified fpstate. Otherwise the task might skip the restore and operate |
58 | * with the cached FPU registers which discards the modifications. |
59 | */ |
60 | static void fpu_force_restore(struct fpu *fpu) |
61 | { |
62 | /* |
63 | * Only stopped child tasks can be used to modify the FPU |
64 | * state in the fpstate buffer: |
65 | */ |
66 | WARN_ON_FPU(fpu == ¤t->thread.fpu); |
67 | |
68 | __fpu_invalidate_fpregs_state(fpu); |
69 | } |
70 | |
71 | int xfpregs_get(struct task_struct *target, const struct user_regset *regset, |
72 | struct membuf to) |
73 | { |
74 | struct fpu *fpu = &target->thread.fpu; |
75 | |
76 | if (!cpu_feature_enabled(X86_FEATURE_FXSR)) |
77 | return -ENODEV; |
78 | |
79 | sync_fpstate(fpu); |
80 | |
81 | if (!use_xsave()) { |
82 | return membuf_write(s: &to, v: &fpu->fpstate->regs.fxsave, |
83 | size: sizeof(fpu->fpstate->regs.fxsave)); |
84 | } |
85 | |
86 | copy_xstate_to_uabi_buf(to, tsk: target, mode: XSTATE_COPY_FX); |
87 | return 0; |
88 | } |
89 | |
90 | int xfpregs_set(struct task_struct *target, const struct user_regset *regset, |
91 | unsigned int pos, unsigned int count, |
92 | const void *kbuf, const void __user *ubuf) |
93 | { |
94 | struct fpu *fpu = &target->thread.fpu; |
95 | struct fxregs_state newstate; |
96 | int ret; |
97 | |
98 | if (!cpu_feature_enabled(X86_FEATURE_FXSR)) |
99 | return -ENODEV; |
100 | |
101 | /* No funny business with partial or oversized writes is permitted. */ |
102 | if (pos != 0 || count != sizeof(newstate)) |
103 | return -EINVAL; |
104 | |
105 | ret = user_regset_copyin(pos: &pos, count: &count, kbuf: &kbuf, ubuf: &ubuf, data: &newstate, start_pos: 0, end_pos: -1); |
106 | if (ret) |
107 | return ret; |
108 | |
109 | /* Do not allow an invalid MXCSR value. */ |
110 | if (newstate.mxcsr & ~mxcsr_feature_mask) |
111 | return -EINVAL; |
112 | |
113 | fpu_force_restore(fpu); |
114 | |
115 | /* Copy the state */ |
116 | memcpy(&fpu->fpstate->regs.fxsave, &newstate, sizeof(newstate)); |
117 | |
118 | /* Clear xmm8..15 for 32-bit callers */ |
119 | BUILD_BUG_ON(sizeof(fpu->__fpstate.regs.fxsave.xmm_space) != 16 * 16); |
120 | if (in_ia32_syscall()) |
121 | memset(&fpu->fpstate->regs.fxsave.xmm_space[8*4], 0, 8 * 16); |
122 | |
123 | /* Mark FP and SSE as in use when XSAVE is enabled */ |
124 | if (use_xsave()) |
125 | fpu->fpstate->regs.xsave.header.xfeatures |= XFEATURE_MASK_FPSSE; |
126 | |
127 | return 0; |
128 | } |
129 | |
130 | int xstateregs_get(struct task_struct *target, const struct user_regset *regset, |
131 | struct membuf to) |
132 | { |
133 | if (!cpu_feature_enabled(X86_FEATURE_XSAVE)) |
134 | return -ENODEV; |
135 | |
136 | sync_fpstate(fpu: &target->thread.fpu); |
137 | |
138 | copy_xstate_to_uabi_buf(to, tsk: target, mode: XSTATE_COPY_XSAVE); |
139 | return 0; |
140 | } |
141 | |
142 | int xstateregs_set(struct task_struct *target, const struct user_regset *regset, |
143 | unsigned int pos, unsigned int count, |
144 | const void *kbuf, const void __user *ubuf) |
145 | { |
146 | struct fpu *fpu = &target->thread.fpu; |
147 | struct xregs_state *tmpbuf = NULL; |
148 | int ret; |
149 | |
150 | if (!cpu_feature_enabled(X86_FEATURE_XSAVE)) |
151 | return -ENODEV; |
152 | |
153 | /* |
154 | * A whole standard-format XSAVE buffer is needed: |
155 | */ |
156 | if (pos != 0 || count != fpu_user_cfg.max_size) |
157 | return -EFAULT; |
158 | |
159 | if (!kbuf) { |
160 | tmpbuf = vmalloc(size: count); |
161 | if (!tmpbuf) |
162 | return -ENOMEM; |
163 | |
164 | if (copy_from_user(to: tmpbuf, from: ubuf, n: count)) { |
165 | ret = -EFAULT; |
166 | goto out; |
167 | } |
168 | } |
169 | |
170 | fpu_force_restore(fpu); |
171 | ret = copy_uabi_from_kernel_to_xstate(fpstate: fpu->fpstate, kbuf: kbuf ?: tmpbuf, pkru: &target->thread.pkru); |
172 | |
173 | out: |
174 | vfree(addr: tmpbuf); |
175 | return ret; |
176 | } |
177 | |
178 | #ifdef CONFIG_X86_USER_SHADOW_STACK |
179 | int ssp_active(struct task_struct *target, const struct user_regset *regset) |
180 | { |
181 | if (target->thread.features & ARCH_SHSTK_SHSTK) |
182 | return regset->n; |
183 | |
184 | return 0; |
185 | } |
186 | |
187 | int ssp_get(struct task_struct *target, const struct user_regset *regset, |
188 | struct membuf to) |
189 | { |
190 | struct fpu *fpu = &target->thread.fpu; |
191 | struct cet_user_state *cetregs; |
192 | |
193 | if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK)) |
194 | return -ENODEV; |
195 | |
196 | sync_fpstate(fpu); |
197 | cetregs = get_xsave_addr(xsave: &fpu->fpstate->regs.xsave, xfeature_nr: XFEATURE_CET_USER); |
198 | if (WARN_ON(!cetregs)) { |
199 | /* |
200 | * This shouldn't ever be NULL because shadow stack was |
201 | * verified to be enabled above. This means |
202 | * MSR_IA32_U_CET.CET_SHSTK_EN should be 1 and so |
203 | * XFEATURE_CET_USER should not be in the init state. |
204 | */ |
205 | return -ENODEV; |
206 | } |
207 | |
208 | return membuf_write(s: &to, v: (unsigned long *)&cetregs->user_ssp, |
209 | size: sizeof(cetregs->user_ssp)); |
210 | } |
211 | |
212 | int ssp_set(struct task_struct *target, const struct user_regset *regset, |
213 | unsigned int pos, unsigned int count, |
214 | const void *kbuf, const void __user *ubuf) |
215 | { |
216 | struct fpu *fpu = &target->thread.fpu; |
217 | struct xregs_state *xsave = &fpu->fpstate->regs.xsave; |
218 | struct cet_user_state *cetregs; |
219 | unsigned long user_ssp; |
220 | int r; |
221 | |
222 | if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK) || |
223 | !ssp_active(target, regset)) |
224 | return -ENODEV; |
225 | |
226 | if (pos != 0 || count != sizeof(user_ssp)) |
227 | return -EINVAL; |
228 | |
229 | r = user_regset_copyin(pos: &pos, count: &count, kbuf: &kbuf, ubuf: &ubuf, data: &user_ssp, start_pos: 0, end_pos: -1); |
230 | if (r) |
231 | return r; |
232 | |
233 | /* |
234 | * Some kernel instructions (IRET, etc) can cause exceptions in the case |
235 | * of disallowed CET register values. Just prevent invalid values. |
236 | */ |
237 | if (user_ssp >= TASK_SIZE_MAX || !IS_ALIGNED(user_ssp, 8)) |
238 | return -EINVAL; |
239 | |
240 | fpu_force_restore(fpu); |
241 | |
242 | cetregs = get_xsave_addr(xsave, xfeature_nr: XFEATURE_CET_USER); |
243 | if (WARN_ON(!cetregs)) { |
244 | /* |
245 | * This shouldn't ever be NULL because shadow stack was |
246 | * verified to be enabled above. This means |
247 | * MSR_IA32_U_CET.CET_SHSTK_EN should be 1 and so |
248 | * XFEATURE_CET_USER should not be in the init state. |
249 | */ |
250 | return -ENODEV; |
251 | } |
252 | |
253 | cetregs->user_ssp = user_ssp; |
254 | return 0; |
255 | } |
256 | #endif /* CONFIG_X86_USER_SHADOW_STACK */ |
257 | |
258 | #if defined CONFIG_X86_32 || defined CONFIG_IA32_EMULATION |
259 | |
260 | /* |
261 | * FPU tag word conversions. |
262 | */ |
263 | |
264 | static inline unsigned short twd_i387_to_fxsr(unsigned short twd) |
265 | { |
266 | unsigned int tmp; /* to avoid 16 bit prefixes in the code */ |
267 | |
268 | /* Transform each pair of bits into 01 (valid) or 00 (empty) */ |
269 | tmp = ~twd; |
270 | tmp = (tmp | (tmp>>1)) & 0x5555; /* 0V0V0V0V0V0V0V0V */ |
271 | /* and move the valid bits to the lower byte. */ |
272 | tmp = (tmp | (tmp >> 1)) & 0x3333; /* 00VV00VV00VV00VV */ |
273 | tmp = (tmp | (tmp >> 2)) & 0x0f0f; /* 0000VVVV0000VVVV */ |
274 | tmp = (tmp | (tmp >> 4)) & 0x00ff; /* 00000000VVVVVVVV */ |
275 | |
276 | return tmp; |
277 | } |
278 | |
279 | #define FPREG_ADDR(f, n) ((void *)&(f)->st_space + (n) * 16) |
280 | #define FP_EXP_TAG_VALID 0 |
281 | #define FP_EXP_TAG_ZERO 1 |
282 | #define FP_EXP_TAG_SPECIAL 2 |
283 | #define FP_EXP_TAG_EMPTY 3 |
284 | |
285 | static inline u32 twd_fxsr_to_i387(struct fxregs_state *fxsave) |
286 | { |
287 | struct _fpxreg *st; |
288 | u32 tos = (fxsave->swd >> 11) & 7; |
289 | u32 twd = (unsigned long) fxsave->twd; |
290 | u32 tag; |
291 | u32 ret = 0xffff0000u; |
292 | int i; |
293 | |
294 | for (i = 0; i < 8; i++, twd >>= 1) { |
295 | if (twd & 0x1) { |
296 | st = FPREG_ADDR(fxsave, (i - tos) & 7); |
297 | |
298 | switch (st->exponent & 0x7fff) { |
299 | case 0x7fff: |
300 | tag = FP_EXP_TAG_SPECIAL; |
301 | break; |
302 | case 0x0000: |
303 | if (!st->significand[0] && |
304 | !st->significand[1] && |
305 | !st->significand[2] && |
306 | !st->significand[3]) |
307 | tag = FP_EXP_TAG_ZERO; |
308 | else |
309 | tag = FP_EXP_TAG_SPECIAL; |
310 | break; |
311 | default: |
312 | if (st->significand[3] & 0x8000) |
313 | tag = FP_EXP_TAG_VALID; |
314 | else |
315 | tag = FP_EXP_TAG_SPECIAL; |
316 | break; |
317 | } |
318 | } else { |
319 | tag = FP_EXP_TAG_EMPTY; |
320 | } |
321 | ret |= tag << (2 * i); |
322 | } |
323 | return ret; |
324 | } |
325 | |
326 | /* |
327 | * FXSR floating point environment conversions. |
328 | */ |
329 | |
330 | static void __convert_from_fxsr(struct user_i387_ia32_struct *env, |
331 | struct task_struct *tsk, |
332 | struct fxregs_state *fxsave) |
333 | { |
334 | struct _fpreg *to = (struct _fpreg *) &env->st_space[0]; |
335 | struct _fpxreg *from = (struct _fpxreg *) &fxsave->st_space[0]; |
336 | int i; |
337 | |
338 | env->cwd = fxsave->cwd | 0xffff0000u; |
339 | env->swd = fxsave->swd | 0xffff0000u; |
340 | env->twd = twd_fxsr_to_i387(fxsave); |
341 | |
342 | #ifdef CONFIG_X86_64 |
343 | env->fip = fxsave->rip; |
344 | env->foo = fxsave->rdp; |
345 | /* |
346 | * should be actually ds/cs at fpu exception time, but |
347 | * that information is not available in 64bit mode. |
348 | */ |
349 | env->fcs = task_pt_regs(tsk)->cs; |
350 | if (tsk == current) { |
351 | savesegment(ds, env->fos); |
352 | } else { |
353 | env->fos = tsk->thread.ds; |
354 | } |
355 | env->fos |= 0xffff0000; |
356 | #else |
357 | env->fip = fxsave->fip; |
358 | env->fcs = (u16) fxsave->fcs | ((u32) fxsave->fop << 16); |
359 | env->foo = fxsave->foo; |
360 | env->fos = fxsave->fos; |
361 | #endif |
362 | |
363 | for (i = 0; i < 8; ++i) |
364 | memcpy(&to[i], &from[i], sizeof(to[0])); |
365 | } |
366 | |
367 | void |
368 | convert_from_fxsr(struct user_i387_ia32_struct *env, struct task_struct *tsk) |
369 | { |
370 | __convert_from_fxsr(env, tsk, fxsave: &tsk->thread.fpu.fpstate->regs.fxsave); |
371 | } |
372 | |
373 | void convert_to_fxsr(struct fxregs_state *fxsave, |
374 | const struct user_i387_ia32_struct *env) |
375 | |
376 | { |
377 | struct _fpreg *from = (struct _fpreg *) &env->st_space[0]; |
378 | struct _fpxreg *to = (struct _fpxreg *) &fxsave->st_space[0]; |
379 | int i; |
380 | |
381 | fxsave->cwd = env->cwd; |
382 | fxsave->swd = env->swd; |
383 | fxsave->twd = twd_i387_to_fxsr(twd: env->twd); |
384 | fxsave->fop = (u16) ((u32) env->fcs >> 16); |
385 | #ifdef CONFIG_X86_64 |
386 | fxsave->rip = env->fip; |
387 | fxsave->rdp = env->foo; |
388 | /* cs and ds ignored */ |
389 | #else |
390 | fxsave->fip = env->fip; |
391 | fxsave->fcs = (env->fcs & 0xffff); |
392 | fxsave->foo = env->foo; |
393 | fxsave->fos = env->fos; |
394 | #endif |
395 | |
396 | for (i = 0; i < 8; ++i) |
397 | memcpy(&to[i], &from[i], sizeof(from[0])); |
398 | } |
399 | |
400 | int fpregs_get(struct task_struct *target, const struct user_regset *regset, |
401 | struct membuf to) |
402 | { |
403 | struct fpu *fpu = &target->thread.fpu; |
404 | struct user_i387_ia32_struct env; |
405 | struct fxregs_state fxsave, *fx; |
406 | |
407 | sync_fpstate(fpu); |
408 | |
409 | if (!cpu_feature_enabled(X86_FEATURE_FPU)) |
410 | return fpregs_soft_get(target, regset, to); |
411 | |
412 | if (!cpu_feature_enabled(X86_FEATURE_FXSR)) { |
413 | return membuf_write(s: &to, v: &fpu->fpstate->regs.fsave, |
414 | size: sizeof(struct fregs_state)); |
415 | } |
416 | |
417 | if (use_xsave()) { |
418 | struct membuf mb = { .p = &fxsave, .left = sizeof(fxsave) }; |
419 | |
420 | /* Handle init state optimized xstate correctly */ |
421 | copy_xstate_to_uabi_buf(to: mb, tsk: target, mode: XSTATE_COPY_FP); |
422 | fx = &fxsave; |
423 | } else { |
424 | fx = &fpu->fpstate->regs.fxsave; |
425 | } |
426 | |
427 | __convert_from_fxsr(env: &env, tsk: target, fxsave: fx); |
428 | return membuf_write(s: &to, v: &env, size: sizeof(env)); |
429 | } |
430 | |
431 | int fpregs_set(struct task_struct *target, const struct user_regset *regset, |
432 | unsigned int pos, unsigned int count, |
433 | const void *kbuf, const void __user *ubuf) |
434 | { |
435 | struct fpu *fpu = &target->thread.fpu; |
436 | struct user_i387_ia32_struct env; |
437 | int ret; |
438 | |
439 | /* No funny business with partial or oversized writes is permitted. */ |
440 | if (pos != 0 || count != sizeof(struct user_i387_ia32_struct)) |
441 | return -EINVAL; |
442 | |
443 | if (!cpu_feature_enabled(X86_FEATURE_FPU)) |
444 | return fpregs_soft_set(target, regset, pos, count, kbuf, ubuf); |
445 | |
446 | ret = user_regset_copyin(pos: &pos, count: &count, kbuf: &kbuf, ubuf: &ubuf, data: &env, start_pos: 0, end_pos: -1); |
447 | if (ret) |
448 | return ret; |
449 | |
450 | fpu_force_restore(fpu); |
451 | |
452 | if (cpu_feature_enabled(X86_FEATURE_FXSR)) |
453 | convert_to_fxsr(fxsave: &fpu->fpstate->regs.fxsave, env: &env); |
454 | else |
455 | memcpy(&fpu->fpstate->regs.fsave, &env, sizeof(env)); |
456 | |
457 | /* |
458 | * Update the header bit in the xsave header, indicating the |
459 | * presence of FP. |
460 | */ |
461 | if (cpu_feature_enabled(X86_FEATURE_XSAVE)) |
462 | fpu->fpstate->regs.xsave.header.xfeatures |= XFEATURE_MASK_FP; |
463 | |
464 | return 0; |
465 | } |
466 | |
467 | #endif /* CONFIG_X86_32 || CONFIG_IA32_EMULATION */ |
468 | |