1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #ifndef __X86_KERNEL_FPU_CONTEXT_H |
3 | #define __X86_KERNEL_FPU_CONTEXT_H |
4 | |
5 | #include <asm/fpu/xstate.h> |
6 | #include <asm/trace/fpu.h> |
7 | |
8 | /* Functions related to FPU context tracking */ |
9 | |
10 | /* |
11 | * The in-register FPU state for an FPU context on a CPU is assumed to be |
12 | * valid if the fpu->last_cpu matches the CPU, and the fpu_fpregs_owner_ctx |
13 | * matches the FPU. |
14 | * |
15 | * If the FPU register state is valid, the kernel can skip restoring the |
16 | * FPU state from memory. |
17 | * |
18 | * Any code that clobbers the FPU registers or updates the in-memory |
19 | * FPU state for a task MUST let the rest of the kernel know that the |
20 | * FPU registers are no longer valid for this task. |
21 | * |
22 | * Invalidate a resource you control: CPU if using the CPU for something else |
23 | * (with preemption disabled), FPU for the current task, or a task that |
24 | * is prevented from running by the current task. |
25 | */ |
26 | static inline void __cpu_invalidate_fpregs_state(void) |
27 | { |
28 | __this_cpu_write(fpu_fpregs_owner_ctx, NULL); |
29 | } |
30 | |
31 | static inline void __fpu_invalidate_fpregs_state(struct fpu *fpu) |
32 | { |
33 | fpu->last_cpu = -1; |
34 | } |
35 | |
36 | static inline int fpregs_state_valid(struct fpu *fpu, unsigned int cpu) |
37 | { |
38 | return fpu == this_cpu_read(fpu_fpregs_owner_ctx) && cpu == fpu->last_cpu; |
39 | } |
40 | |
41 | static inline void fpregs_deactivate(struct fpu *fpu) |
42 | { |
43 | __this_cpu_write(fpu_fpregs_owner_ctx, NULL); |
44 | trace_x86_fpu_regs_deactivated(fpu); |
45 | } |
46 | |
47 | static inline void fpregs_activate(struct fpu *fpu) |
48 | { |
49 | __this_cpu_write(fpu_fpregs_owner_ctx, fpu); |
50 | trace_x86_fpu_regs_activated(fpu); |
51 | } |
52 | |
53 | /* Internal helper for switch_fpu_return() and signal frame setup */ |
54 | static inline void fpregs_restore_userregs(void) |
55 | { |
56 | struct fpu *fpu = ¤t->thread.fpu; |
57 | int cpu = smp_processor_id(); |
58 | |
59 | if (WARN_ON_ONCE(current->flags & (PF_KTHREAD | PF_USER_WORKER))) |
60 | return; |
61 | |
62 | if (!fpregs_state_valid(fpu, cpu)) { |
63 | /* |
64 | * This restores _all_ xstate which has not been |
65 | * established yet. |
66 | * |
67 | * If PKRU is enabled, then the PKRU value is already |
68 | * correct because it was either set in switch_to() or in |
69 | * flush_thread(). So it is excluded because it might be |
70 | * not up to date in current->thread.fpu.xsave state. |
71 | * |
72 | * XFD state is handled in restore_fpregs_from_fpstate(). |
73 | */ |
74 | restore_fpregs_from_fpstate(fpstate: fpu->fpstate, XFEATURE_MASK_FPSTATE); |
75 | |
76 | fpregs_activate(fpu); |
77 | fpu->last_cpu = cpu; |
78 | } |
79 | clear_thread_flag(TIF_NEED_FPU_LOAD); |
80 | } |
81 | |
82 | #endif |
83 | |