1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | /* |
3 | * FPU data structures: |
4 | */ |
5 | #ifndef _ASM_X86_FPU_H |
6 | #define _ASM_X86_FPU_H |
7 | |
8 | #include <asm/page_types.h> |
9 | |
10 | /* |
11 | * The legacy x87 FPU state format, as saved by FSAVE and |
12 | * restored by the FRSTOR instructions: |
13 | */ |
14 | struct fregs_state { |
15 | u32 cwd; /* FPU Control Word */ |
16 | u32 swd; /* FPU Status Word */ |
17 | u32 twd; /* FPU Tag Word */ |
18 | u32 fip; /* FPU IP Offset */ |
19 | u32 fcs; /* FPU IP Selector */ |
20 | u32 foo; /* FPU Operand Pointer Offset */ |
21 | u32 fos; /* FPU Operand Pointer Selector */ |
22 | |
23 | /* 8*10 bytes for each FP-reg = 80 bytes: */ |
24 | u32 st_space[20]; |
25 | |
26 | /* Software status information [not touched by FSAVE]: */ |
27 | u32 status; |
28 | }; |
29 | |
30 | /* |
31 | * The legacy fx SSE/MMX FPU state format, as saved by FXSAVE and |
32 | * restored by the FXRSTOR instructions. It's similar to the FSAVE |
33 | * format, but differs in some areas, plus has extensions at |
34 | * the end for the XMM registers. |
35 | */ |
36 | struct fxregs_state { |
37 | u16 cwd; /* Control Word */ |
38 | u16 swd; /* Status Word */ |
39 | u16 twd; /* Tag Word */ |
40 | u16 fop; /* Last Instruction Opcode */ |
41 | union { |
42 | struct { |
43 | u64 rip; /* Instruction Pointer */ |
44 | u64 rdp; /* Data Pointer */ |
45 | }; |
46 | struct { |
47 | u32 fip; /* FPU IP Offset */ |
48 | u32 fcs; /* FPU IP Selector */ |
49 | u32 foo; /* FPU Operand Offset */ |
50 | u32 fos; /* FPU Operand Selector */ |
51 | }; |
52 | }; |
53 | u32 mxcsr; /* MXCSR Register State */ |
54 | u32 mxcsr_mask; /* MXCSR Mask */ |
55 | |
56 | /* 8*16 bytes for each FP-reg = 128 bytes: */ |
57 | u32 st_space[32]; |
58 | |
59 | /* 16*16 bytes for each XMM-reg = 256 bytes: */ |
60 | u32 xmm_space[64]; |
61 | |
62 | u32 padding[12]; |
63 | |
64 | union { |
65 | u32 padding1[12]; |
66 | u32 sw_reserved[12]; |
67 | }; |
68 | |
69 | } __attribute__((aligned(16))); |
70 | |
71 | /* Default value for fxregs_state.mxcsr: */ |
72 | #define MXCSR_DEFAULT 0x1f80 |
73 | |
74 | /* Copy both mxcsr & mxcsr_flags with a single u64 memcpy: */ |
75 | #define MXCSR_AND_FLAGS_SIZE sizeof(u64) |
76 | |
77 | /* |
78 | * Software based FPU emulation state. This is arbitrary really, |
79 | * it matches the x87 format to make it easier to understand: |
80 | */ |
81 | struct swregs_state { |
82 | u32 cwd; |
83 | u32 swd; |
84 | u32 twd; |
85 | u32 fip; |
86 | u32 fcs; |
87 | u32 foo; |
88 | u32 fos; |
89 | /* 8*10 bytes for each FP-reg = 80 bytes: */ |
90 | u32 st_space[20]; |
91 | u8 ftop; |
92 | u8 changed; |
93 | u8 lookahead; |
94 | u8 no_update; |
95 | u8 rm; |
96 | u8 alimit; |
97 | struct math_emu_info *info; |
98 | u32 entry_eip; |
99 | }; |
100 | |
101 | /* |
102 | * List of XSAVE features Linux knows about: |
103 | */ |
104 | enum xfeature { |
105 | XFEATURE_FP, |
106 | XFEATURE_SSE, |
107 | /* |
108 | * Values above here are "legacy states". |
109 | * Those below are "extended states". |
110 | */ |
111 | XFEATURE_YMM, |
112 | XFEATURE_BNDREGS, |
113 | XFEATURE_BNDCSR, |
114 | XFEATURE_OPMASK, |
115 | XFEATURE_ZMM_Hi256, |
116 | XFEATURE_Hi16_ZMM, |
117 | XFEATURE_PT_UNIMPLEMENTED_SO_FAR, |
118 | XFEATURE_PKRU, |
119 | XFEATURE_PASID, |
120 | XFEATURE_CET_USER, |
121 | XFEATURE_CET_KERNEL_UNUSED, |
122 | XFEATURE_RSRVD_COMP_13, |
123 | XFEATURE_RSRVD_COMP_14, |
124 | XFEATURE_LBR, |
125 | XFEATURE_RSRVD_COMP_16, |
126 | XFEATURE_XTILE_CFG, |
127 | XFEATURE_XTILE_DATA, |
128 | |
129 | XFEATURE_MAX, |
130 | }; |
131 | |
132 | #define XFEATURE_MASK_FP (1 << XFEATURE_FP) |
133 | #define XFEATURE_MASK_SSE (1 << XFEATURE_SSE) |
134 | #define XFEATURE_MASK_YMM (1 << XFEATURE_YMM) |
135 | #define XFEATURE_MASK_BNDREGS (1 << XFEATURE_BNDREGS) |
136 | #define XFEATURE_MASK_BNDCSR (1 << XFEATURE_BNDCSR) |
137 | #define XFEATURE_MASK_OPMASK (1 << XFEATURE_OPMASK) |
138 | #define XFEATURE_MASK_ZMM_Hi256 (1 << XFEATURE_ZMM_Hi256) |
139 | #define XFEATURE_MASK_Hi16_ZMM (1 << XFEATURE_Hi16_ZMM) |
140 | #define XFEATURE_MASK_PT (1 << XFEATURE_PT_UNIMPLEMENTED_SO_FAR) |
141 | #define XFEATURE_MASK_PKRU (1 << XFEATURE_PKRU) |
142 | #define XFEATURE_MASK_PASID (1 << XFEATURE_PASID) |
143 | #define XFEATURE_MASK_CET_USER (1 << XFEATURE_CET_USER) |
144 | #define XFEATURE_MASK_CET_KERNEL (1 << XFEATURE_CET_KERNEL_UNUSED) |
145 | #define XFEATURE_MASK_LBR (1 << XFEATURE_LBR) |
146 | #define XFEATURE_MASK_XTILE_CFG (1 << XFEATURE_XTILE_CFG) |
147 | #define XFEATURE_MASK_XTILE_DATA (1 << XFEATURE_XTILE_DATA) |
148 | |
149 | #define XFEATURE_MASK_FPSSE (XFEATURE_MASK_FP | XFEATURE_MASK_SSE) |
150 | #define XFEATURE_MASK_AVX512 (XFEATURE_MASK_OPMASK \ |
151 | | XFEATURE_MASK_ZMM_Hi256 \ |
152 | | XFEATURE_MASK_Hi16_ZMM) |
153 | |
154 | #ifdef CONFIG_X86_64 |
155 | # define XFEATURE_MASK_XTILE (XFEATURE_MASK_XTILE_DATA \ |
156 | | XFEATURE_MASK_XTILE_CFG) |
157 | #else |
158 | # define XFEATURE_MASK_XTILE (0) |
159 | #endif |
160 | |
161 | #define FIRST_EXTENDED_XFEATURE XFEATURE_YMM |
162 | |
163 | struct reg_128_bit { |
164 | u8 regbytes[128/8]; |
165 | }; |
166 | struct reg_256_bit { |
167 | u8 regbytes[256/8]; |
168 | }; |
169 | struct reg_512_bit { |
170 | u8 regbytes[512/8]; |
171 | }; |
172 | struct reg_1024_byte { |
173 | u8 regbytes[1024]; |
174 | }; |
175 | |
176 | /* |
177 | * State component 2: |
178 | * |
179 | * There are 16x 256-bit AVX registers named YMM0-YMM15. |
180 | * The low 128 bits are aliased to the 16 SSE registers (XMM0-XMM15) |
181 | * and are stored in 'struct fxregs_state::xmm_space[]' in the |
182 | * "legacy" area. |
183 | * |
184 | * The high 128 bits are stored here. |
185 | */ |
186 | struct ymmh_struct { |
187 | struct reg_128_bit hi_ymm[16]; |
188 | } __packed; |
189 | |
190 | /* Intel MPX support: */ |
191 | |
192 | struct mpx_bndreg { |
193 | u64 lower_bound; |
194 | u64 upper_bound; |
195 | } __packed; |
196 | /* |
197 | * State component 3 is used for the 4 128-bit bounds registers |
198 | */ |
199 | struct mpx_bndreg_state { |
200 | struct mpx_bndreg bndreg[4]; |
201 | } __packed; |
202 | |
203 | /* |
204 | * State component 4 is used for the 64-bit user-mode MPX |
205 | * configuration register BNDCFGU and the 64-bit MPX status |
206 | * register BNDSTATUS. We call the pair "BNDCSR". |
207 | */ |
208 | struct mpx_bndcsr { |
209 | u64 bndcfgu; |
210 | u64 bndstatus; |
211 | } __packed; |
212 | |
213 | /* |
214 | * The BNDCSR state is padded out to be 64-bytes in size. |
215 | */ |
216 | struct mpx_bndcsr_state { |
217 | union { |
218 | struct mpx_bndcsr bndcsr; |
219 | u8 pad_to_64_bytes[64]; |
220 | }; |
221 | } __packed; |
222 | |
223 | /* AVX-512 Components: */ |
224 | |
225 | /* |
226 | * State component 5 is used for the 8 64-bit opmask registers |
227 | * k0-k7 (opmask state). |
228 | */ |
229 | struct avx_512_opmask_state { |
230 | u64 opmask_reg[8]; |
231 | } __packed; |
232 | |
233 | /* |
234 | * State component 6 is used for the upper 256 bits of the |
235 | * registers ZMM0-ZMM15. These 16 256-bit values are denoted |
236 | * ZMM0_H-ZMM15_H (ZMM_Hi256 state). |
237 | */ |
238 | struct avx_512_zmm_uppers_state { |
239 | struct reg_256_bit zmm_upper[16]; |
240 | } __packed; |
241 | |
242 | /* |
243 | * State component 7 is used for the 16 512-bit registers |
244 | * ZMM16-ZMM31 (Hi16_ZMM state). |
245 | */ |
246 | struct avx_512_hi16_state { |
247 | struct reg_512_bit hi16_zmm[16]; |
248 | } __packed; |
249 | |
250 | /* |
251 | * State component 9: 32-bit PKRU register. The state is |
252 | * 8 bytes long but only 4 bytes is used currently. |
253 | */ |
254 | struct pkru_state { |
255 | u32 pkru; |
256 | u32 pad; |
257 | } __packed; |
258 | |
259 | /* |
260 | * State component 11 is Control-flow Enforcement user states |
261 | */ |
262 | struct cet_user_state { |
263 | /* user control-flow settings */ |
264 | u64 user_cet; |
265 | /* user shadow stack pointer */ |
266 | u64 user_ssp; |
267 | }; |
268 | |
269 | /* |
270 | * State component 15: Architectural LBR configuration state. |
271 | * The size of Arch LBR state depends on the number of LBRs (lbr_depth). |
272 | */ |
273 | |
274 | struct lbr_entry { |
275 | u64 from; |
276 | u64 to; |
277 | u64 info; |
278 | }; |
279 | |
280 | struct arch_lbr_state { |
281 | u64 lbr_ctl; |
282 | u64 lbr_depth; |
283 | u64 ler_from; |
284 | u64 ler_to; |
285 | u64 ler_info; |
286 | struct lbr_entry entries[]; |
287 | }; |
288 | |
289 | /* |
290 | * State component 17: 64-byte tile configuration register. |
291 | */ |
292 | struct xtile_cfg { |
293 | u64 tcfg[8]; |
294 | } __packed; |
295 | |
296 | /* |
297 | * State component 18: 1KB tile data register. |
298 | * Each register represents 16 64-byte rows of the matrix |
299 | * data. But the number of registers depends on the actual |
300 | * implementation. |
301 | */ |
302 | struct xtile_data { |
303 | struct reg_1024_byte tmm; |
304 | } __packed; |
305 | |
306 | /* |
307 | * State component 10 is supervisor state used for context-switching the |
308 | * PASID state. |
309 | */ |
310 | struct ia32_pasid_state { |
311 | u64 pasid; |
312 | } __packed; |
313 | |
314 | struct { |
315 | u64 ; |
316 | u64 ; |
317 | u64 [6]; |
318 | } __attribute__((packed)); |
319 | |
320 | /* |
321 | * xstate_header.xcomp_bv[63] indicates that the extended_state_area |
322 | * is in compacted format. |
323 | */ |
324 | #define XCOMP_BV_COMPACTED_FORMAT ((u64)1 << 63) |
325 | |
326 | /* |
327 | * This is our most modern FPU state format, as saved by the XSAVE |
328 | * and restored by the XRSTOR instructions. |
329 | * |
330 | * It consists of a legacy fxregs portion, an xstate header and |
331 | * subsequent areas as defined by the xstate header. Not all CPUs |
332 | * support all the extensions, so the size of the extended area |
333 | * can vary quite a bit between CPUs. |
334 | */ |
335 | struct xregs_state { |
336 | struct fxregs_state i387; |
337 | struct xstate_header ; |
338 | u8 extended_state_area[]; |
339 | } __attribute__ ((packed, aligned (64))); |
340 | |
341 | /* |
342 | * This is a union of all the possible FPU state formats |
343 | * put together, so that we can pick the right one runtime. |
344 | * |
345 | * The size of the structure is determined by the largest |
346 | * member - which is the xsave area. The padding is there |
347 | * to ensure that statically-allocated task_structs (just |
348 | * the init_task today) have enough space. |
349 | */ |
350 | union fpregs_state { |
351 | struct fregs_state fsave; |
352 | struct fxregs_state fxsave; |
353 | struct swregs_state soft; |
354 | struct xregs_state xsave; |
355 | u8 __padding[PAGE_SIZE]; |
356 | }; |
357 | |
358 | struct fpstate { |
359 | /* @kernel_size: The size of the kernel register image */ |
360 | unsigned int size; |
361 | |
362 | /* @user_size: The size in non-compacted UABI format */ |
363 | unsigned int user_size; |
364 | |
365 | /* @xfeatures: xfeatures for which the storage is sized */ |
366 | u64 xfeatures; |
367 | |
368 | /* @user_xfeatures: xfeatures valid in UABI buffers */ |
369 | u64 user_xfeatures; |
370 | |
371 | /* @xfd: xfeatures disabled to trap userspace use. */ |
372 | u64 xfd; |
373 | |
374 | /* @is_valloc: Indicator for dynamically allocated state */ |
375 | unsigned int is_valloc : 1; |
376 | |
377 | /* @is_guest: Indicator for guest state (KVM) */ |
378 | unsigned int is_guest : 1; |
379 | |
380 | /* |
381 | * @is_confidential: Indicator for KVM confidential mode. |
382 | * The FPU registers are restored by the |
383 | * vmentry firmware from encrypted guest |
384 | * memory. On vmexit the FPU registers are |
385 | * saved by firmware to encrypted guest memory |
386 | * and the registers are scrubbed before |
387 | * returning to the host. So there is no |
388 | * content which is worth saving and restoring. |
389 | * The fpstate has to be there so that |
390 | * preemption and softirq FPU usage works |
391 | * without special casing. |
392 | */ |
393 | unsigned int is_confidential : 1; |
394 | |
395 | /* @in_use: State is in use */ |
396 | unsigned int in_use : 1; |
397 | |
398 | /* @regs: The register state union for all supported formats */ |
399 | union fpregs_state regs; |
400 | |
401 | /* @regs is dynamically sized! Don't add anything after @regs! */ |
402 | } __aligned(64); |
403 | |
404 | #define FPU_GUEST_PERM_LOCKED BIT_ULL(63) |
405 | |
406 | struct fpu_state_perm { |
407 | /* |
408 | * @__state_perm: |
409 | * |
410 | * This bitmap indicates the permission for state components, which |
411 | * are available to a thread group. The permission prctl() sets the |
412 | * enabled state bits in thread_group_leader()->thread.fpu. |
413 | * |
414 | * All run time operations use the per thread information in the |
415 | * currently active fpu.fpstate which contains the xfeature masks |
416 | * and sizes for kernel and user space. |
417 | * |
418 | * This master permission field is only to be used when |
419 | * task.fpu.fpstate based checks fail to validate whether the task |
420 | * is allowed to expand its xfeatures set which requires to |
421 | * allocate a larger sized fpstate buffer. |
422 | * |
423 | * Do not access this field directly. Use the provided helper |
424 | * function. Unlocked access is possible for quick checks. |
425 | */ |
426 | u64 __state_perm; |
427 | |
428 | /* |
429 | * @__state_size: |
430 | * |
431 | * The size required for @__state_perm. Only valid to access |
432 | * with sighand locked. |
433 | */ |
434 | unsigned int __state_size; |
435 | |
436 | /* |
437 | * @__user_state_size: |
438 | * |
439 | * The size required for @__state_perm user part. Only valid to |
440 | * access with sighand locked. |
441 | */ |
442 | unsigned int __user_state_size; |
443 | }; |
444 | |
445 | /* |
446 | * Highest level per task FPU state data structure that |
447 | * contains the FPU register state plus various FPU |
448 | * state fields: |
449 | */ |
450 | struct fpu { |
451 | /* |
452 | * @last_cpu: |
453 | * |
454 | * Records the last CPU on which this context was loaded into |
455 | * FPU registers. (In the lazy-restore case we might be |
456 | * able to reuse FPU registers across multiple context switches |
457 | * this way, if no intermediate task used the FPU.) |
458 | * |
459 | * A value of -1 is used to indicate that the FPU state in context |
460 | * memory is newer than the FPU state in registers, and that the |
461 | * FPU state should be reloaded next time the task is run. |
462 | */ |
463 | unsigned int last_cpu; |
464 | |
465 | /* |
466 | * @avx512_timestamp: |
467 | * |
468 | * Records the timestamp of AVX512 use during last context switch. |
469 | */ |
470 | unsigned long avx512_timestamp; |
471 | |
472 | /* |
473 | * @fpstate: |
474 | * |
475 | * Pointer to the active struct fpstate. Initialized to |
476 | * point at @__fpstate below. |
477 | */ |
478 | struct fpstate *fpstate; |
479 | |
480 | /* |
481 | * @__task_fpstate: |
482 | * |
483 | * Pointer to an inactive struct fpstate. Initialized to NULL. Is |
484 | * used only for KVM support to swap out the regular task fpstate. |
485 | */ |
486 | struct fpstate *__task_fpstate; |
487 | |
488 | /* |
489 | * @perm: |
490 | * |
491 | * Permission related information |
492 | */ |
493 | struct fpu_state_perm perm; |
494 | |
495 | /* |
496 | * @guest_perm: |
497 | * |
498 | * Permission related information for guest pseudo FPUs |
499 | */ |
500 | struct fpu_state_perm guest_perm; |
501 | |
502 | /* |
503 | * @__fpstate: |
504 | * |
505 | * Initial in-memory storage for FPU registers which are saved in |
506 | * context switch and when the kernel uses the FPU. The registers |
507 | * are restored from this storage on return to user space if they |
508 | * are not longer containing the tasks FPU register state. |
509 | */ |
510 | struct fpstate __fpstate; |
511 | /* |
512 | * WARNING: '__fpstate' is dynamically-sized. Do not put |
513 | * anything after it here. |
514 | */ |
515 | }; |
516 | |
517 | /* |
518 | * Guest pseudo FPU container |
519 | */ |
520 | struct fpu_guest { |
521 | /* |
522 | * @xfeatures: xfeature bitmap of features which are |
523 | * currently enabled for the guest vCPU. |
524 | */ |
525 | u64 xfeatures; |
526 | |
527 | /* |
528 | * @perm: xfeature bitmap of features which are |
529 | * permitted to be enabled for the guest |
530 | * vCPU. |
531 | */ |
532 | u64 perm; |
533 | |
534 | /* |
535 | * @xfd_err: Save the guest value. |
536 | */ |
537 | u64 xfd_err; |
538 | |
539 | /* |
540 | * @uabi_size: Size required for save/restore |
541 | */ |
542 | unsigned int uabi_size; |
543 | |
544 | /* |
545 | * @fpstate: Pointer to the allocated guest fpstate |
546 | */ |
547 | struct fpstate *fpstate; |
548 | }; |
549 | |
550 | /* |
551 | * FPU state configuration data. Initialized at boot time. Read only after init. |
552 | */ |
553 | struct fpu_state_config { |
554 | /* |
555 | * @max_size: |
556 | * |
557 | * The maximum size of the register state buffer. Includes all |
558 | * supported features except independent managed features. |
559 | */ |
560 | unsigned int max_size; |
561 | |
562 | /* |
563 | * @default_size: |
564 | * |
565 | * The default size of the register state buffer. Includes all |
566 | * supported features except independent managed features and |
567 | * features which have to be requested by user space before usage. |
568 | */ |
569 | unsigned int default_size; |
570 | |
571 | /* |
572 | * @max_features: |
573 | * |
574 | * The maximum supported features bitmap. Does not include |
575 | * independent managed features. |
576 | */ |
577 | u64 max_features; |
578 | |
579 | /* |
580 | * @default_features: |
581 | * |
582 | * The default supported features bitmap. Does not include |
583 | * independent managed features and features which have to |
584 | * be requested by user space before usage. |
585 | */ |
586 | u64 default_features; |
587 | /* |
588 | * @legacy_features: |
589 | * |
590 | * Features which can be reported back to user space |
591 | * even without XSAVE support, i.e. legacy features FP + SSE |
592 | */ |
593 | u64 legacy_features; |
594 | }; |
595 | |
596 | /* FPU state configuration information */ |
597 | extern struct fpu_state_config fpu_kernel_cfg, fpu_user_cfg; |
598 | |
599 | #endif /* _ASM_X86_FPU_H */ |
600 | |