1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #ifndef _ASM_X86_SEGMENT_H |
3 | #define _ASM_X86_SEGMENT_H |
4 | |
5 | #include <linux/const.h> |
6 | #include <asm/alternative.h> |
7 | |
8 | /* |
9 | * Constructor for a conventional segment GDT (or LDT) entry. |
10 | * This is a macro so it can be used in initializers. |
11 | */ |
12 | #define GDT_ENTRY(flags, base, limit) \ |
13 | ((((base) & _AC(0xff000000,ULL)) << (56-24)) | \ |
14 | (((flags) & _AC(0x0000f0ff,ULL)) << 40) | \ |
15 | (((limit) & _AC(0x000f0000,ULL)) << (48-16)) | \ |
16 | (((base) & _AC(0x00ffffff,ULL)) << 16) | \ |
17 | (((limit) & _AC(0x0000ffff,ULL)))) |
18 | |
19 | /* Simple and small GDT entries for booting only: */ |
20 | |
21 | #define GDT_ENTRY_BOOT_CS 2 |
22 | #define GDT_ENTRY_BOOT_DS 3 |
23 | #define GDT_ENTRY_BOOT_TSS 4 |
24 | #define __BOOT_CS (GDT_ENTRY_BOOT_CS*8) |
25 | #define __BOOT_DS (GDT_ENTRY_BOOT_DS*8) |
26 | #define __BOOT_TSS (GDT_ENTRY_BOOT_TSS*8) |
27 | |
28 | /* |
29 | * Bottom two bits of selector give the ring |
30 | * privilege level |
31 | */ |
32 | #define SEGMENT_RPL_MASK 0x3 |
33 | |
34 | /* User mode is privilege level 3: */ |
35 | #define USER_RPL 0x3 |
36 | |
37 | /* Bit 2 is Table Indicator (TI): selects between LDT or GDT */ |
38 | #define SEGMENT_TI_MASK 0x4 |
39 | /* LDT segment has TI set ... */ |
40 | #define SEGMENT_LDT 0x4 |
41 | /* ... GDT has it cleared */ |
42 | #define SEGMENT_GDT 0x0 |
43 | |
44 | #define GDT_ENTRY_INVALID_SEG 0 |
45 | |
46 | #ifdef CONFIG_X86_32 |
47 | /* |
48 | * The layout of the per-CPU GDT under Linux: |
49 | * |
50 | * 0 - null <=== cacheline #1 |
51 | * 1 - reserved |
52 | * 2 - reserved |
53 | * 3 - reserved |
54 | * |
55 | * 4 - unused <=== cacheline #2 |
56 | * 5 - unused |
57 | * |
58 | * ------- start of TLS (Thread-Local Storage) segments: |
59 | * |
60 | * 6 - TLS segment #1 [ glibc's TLS segment ] |
61 | * 7 - TLS segment #2 [ Wine's %fs Win32 segment ] |
62 | * 8 - TLS segment #3 <=== cacheline #3 |
63 | * 9 - reserved |
64 | * 10 - reserved |
65 | * 11 - reserved |
66 | * |
67 | * ------- start of kernel segments: |
68 | * |
69 | * 12 - kernel code segment <=== cacheline #4 |
70 | * 13 - kernel data segment |
71 | * 14 - default user CS |
72 | * 15 - default user DS |
73 | * 16 - TSS <=== cacheline #5 |
74 | * 17 - LDT |
75 | * 18 - PNPBIOS support (16->32 gate) |
76 | * 19 - PNPBIOS support |
77 | * 20 - PNPBIOS support <=== cacheline #6 |
78 | * 21 - PNPBIOS support |
79 | * 22 - PNPBIOS support |
80 | * 23 - APM BIOS support |
81 | * 24 - APM BIOS support <=== cacheline #7 |
82 | * 25 - APM BIOS support |
83 | * |
84 | * 26 - ESPFIX small SS |
85 | * 27 - per-cpu [ offset to per-cpu data area ] |
86 | * 28 - stack_canary-20 [ for stack protector ] <=== cacheline #8 |
87 | * 29 - unused |
88 | * 30 - unused |
89 | * 31 - TSS for double fault handler |
90 | */ |
91 | #define GDT_ENTRY_TLS_MIN 6 |
92 | #define GDT_ENTRY_TLS_MAX (GDT_ENTRY_TLS_MIN + GDT_ENTRY_TLS_ENTRIES - 1) |
93 | |
94 | #define GDT_ENTRY_KERNEL_CS 12 |
95 | #define GDT_ENTRY_KERNEL_DS 13 |
96 | #define GDT_ENTRY_DEFAULT_USER_CS 14 |
97 | #define GDT_ENTRY_DEFAULT_USER_DS 15 |
98 | #define GDT_ENTRY_TSS 16 |
99 | #define GDT_ENTRY_LDT 17 |
100 | #define GDT_ENTRY_PNPBIOS_CS32 18 |
101 | #define GDT_ENTRY_PNPBIOS_CS16 19 |
102 | #define GDT_ENTRY_PNPBIOS_DS 20 |
103 | #define GDT_ENTRY_PNPBIOS_TS1 21 |
104 | #define GDT_ENTRY_PNPBIOS_TS2 22 |
105 | #define GDT_ENTRY_APMBIOS_BASE 23 |
106 | |
107 | #define GDT_ENTRY_ESPFIX_SS 26 |
108 | #define GDT_ENTRY_PERCPU 27 |
109 | #define GDT_ENTRY_STACK_CANARY 28 |
110 | |
111 | #define GDT_ENTRY_DOUBLEFAULT_TSS 31 |
112 | |
113 | /* |
114 | * Number of entries in the GDT table: |
115 | */ |
116 | #define GDT_ENTRIES 32 |
117 | |
118 | /* |
119 | * Segment selector values corresponding to the above entries: |
120 | */ |
121 | |
122 | #define __KERNEL_CS (GDT_ENTRY_KERNEL_CS*8) |
123 | #define __KERNEL_DS (GDT_ENTRY_KERNEL_DS*8) |
124 | #define __USER_DS (GDT_ENTRY_DEFAULT_USER_DS*8 + 3) |
125 | #define __USER_CS (GDT_ENTRY_DEFAULT_USER_CS*8 + 3) |
126 | #define __ESPFIX_SS (GDT_ENTRY_ESPFIX_SS*8) |
127 | |
128 | /* segment for calling fn: */ |
129 | #define PNP_CS32 (GDT_ENTRY_PNPBIOS_CS32*8) |
130 | /* code segment for BIOS: */ |
131 | #define PNP_CS16 (GDT_ENTRY_PNPBIOS_CS16*8) |
132 | |
133 | /* "Is this PNP code selector (PNP_CS32 or PNP_CS16)?" */ |
134 | #define SEGMENT_IS_PNP_CODE(x) (((x) & 0xf4) == PNP_CS32) |
135 | |
136 | /* data segment for BIOS: */ |
137 | #define PNP_DS (GDT_ENTRY_PNPBIOS_DS*8) |
138 | /* transfer data segment: */ |
139 | #define PNP_TS1 (GDT_ENTRY_PNPBIOS_TS1*8) |
140 | /* another data segment: */ |
141 | #define PNP_TS2 (GDT_ENTRY_PNPBIOS_TS2*8) |
142 | |
143 | #ifdef CONFIG_SMP |
144 | # define __KERNEL_PERCPU (GDT_ENTRY_PERCPU*8) |
145 | #else |
146 | # define __KERNEL_PERCPU 0 |
147 | #endif |
148 | |
149 | #ifdef CONFIG_STACKPROTECTOR |
150 | # define __KERNEL_STACK_CANARY (GDT_ENTRY_STACK_CANARY*8) |
151 | #else |
152 | # define __KERNEL_STACK_CANARY 0 |
153 | #endif |
154 | |
155 | #else /* 64-bit: */ |
156 | |
157 | #include <asm/cache.h> |
158 | |
159 | #define GDT_ENTRY_KERNEL32_CS 1 |
160 | #define GDT_ENTRY_KERNEL_CS 2 |
161 | #define GDT_ENTRY_KERNEL_DS 3 |
162 | |
163 | /* |
164 | * We cannot use the same code segment descriptor for user and kernel mode, |
165 | * not even in long flat mode, because of different DPL. |
166 | * |
167 | * GDT layout to get 64-bit SYSCALL/SYSRET support right. SYSRET hardcodes |
168 | * selectors: |
169 | * |
170 | * if returning to 32-bit userspace: cs = STAR.SYSRET_CS, |
171 | * if returning to 64-bit userspace: cs = STAR.SYSRET_CS+16, |
172 | * |
173 | * ss = STAR.SYSRET_CS+8 (in either case) |
174 | * |
175 | * thus USER_DS should be between 32-bit and 64-bit code selectors: |
176 | */ |
177 | #define GDT_ENTRY_DEFAULT_USER32_CS 4 |
178 | #define GDT_ENTRY_DEFAULT_USER_DS 5 |
179 | #define GDT_ENTRY_DEFAULT_USER_CS 6 |
180 | |
181 | /* Needs two entries */ |
182 | #define GDT_ENTRY_TSS 8 |
183 | /* Needs two entries */ |
184 | #define GDT_ENTRY_LDT 10 |
185 | |
186 | #define GDT_ENTRY_TLS_MIN 12 |
187 | #define GDT_ENTRY_TLS_MAX 14 |
188 | |
189 | #define GDT_ENTRY_CPUNODE 15 |
190 | |
191 | /* |
192 | * Number of entries in the GDT table: |
193 | */ |
194 | #define GDT_ENTRIES 16 |
195 | |
196 | /* |
197 | * Segment selector values corresponding to the above entries: |
198 | * |
199 | * Note, selectors also need to have a correct RPL, |
200 | * expressed with the +3 value for user-space selectors: |
201 | */ |
202 | #define __KERNEL32_CS (GDT_ENTRY_KERNEL32_CS*8) |
203 | #define __KERNEL_CS (GDT_ENTRY_KERNEL_CS*8) |
204 | #define __KERNEL_DS (GDT_ENTRY_KERNEL_DS*8) |
205 | #define __USER32_CS (GDT_ENTRY_DEFAULT_USER32_CS*8 + 3) |
206 | #define __USER_DS (GDT_ENTRY_DEFAULT_USER_DS*8 + 3) |
207 | #define __USER32_DS __USER_DS |
208 | #define __USER_CS (GDT_ENTRY_DEFAULT_USER_CS*8 + 3) |
209 | #define __CPUNODE_SEG (GDT_ENTRY_CPUNODE*8 + 3) |
210 | |
211 | #endif |
212 | |
213 | #ifndef CONFIG_PARAVIRT_XXL |
214 | # define get_kernel_rpl() 0 |
215 | #endif |
216 | |
217 | #define IDT_ENTRIES 256 |
218 | #define NUM_EXCEPTION_VECTORS 32 |
219 | |
220 | /* Bitmask of exception vectors which push an error code on the stack: */ |
221 | #define EXCEPTION_ERRCODE_MASK 0x00027d00 |
222 | |
223 | #define GDT_SIZE (GDT_ENTRIES*8) |
224 | #define GDT_ENTRY_TLS_ENTRIES 3 |
225 | #define TLS_SIZE (GDT_ENTRY_TLS_ENTRIES* 8) |
226 | |
227 | #ifdef CONFIG_X86_64 |
228 | |
229 | /* Bit size and mask of CPU number stored in the per CPU data (and TSC_AUX) */ |
230 | #define VDSO_CPUNODE_BITS 12 |
231 | #define VDSO_CPUNODE_MASK 0xfff |
232 | |
233 | #ifndef __ASSEMBLY__ |
234 | |
235 | /* Helper functions to store/load CPU and node numbers */ |
236 | |
237 | static inline unsigned long vdso_encode_cpunode(int cpu, unsigned long node) |
238 | { |
239 | return (node << VDSO_CPUNODE_BITS) | cpu; |
240 | } |
241 | |
242 | static inline void vdso_read_cpunode(unsigned *cpu, unsigned *node) |
243 | { |
244 | unsigned int p; |
245 | |
246 | /* |
247 | * Load CPU and node number from the GDT. LSL is faster than RDTSCP |
248 | * and works on all CPUs. This is volatile so that it orders |
249 | * correctly with respect to barrier() and to keep GCC from cleverly |
250 | * hoisting it out of the calling function. |
251 | * |
252 | * If RDPID is available, use it. |
253 | */ |
254 | alternative_io ("lsl %[seg],%[p]" , |
255 | ".byte 0xf3,0x0f,0xc7,0xf8" , /* RDPID %eax/rax */ |
256 | X86_FEATURE_RDPID, |
257 | [p] "=a" (p), [seg] "r" (__CPUNODE_SEG)); |
258 | |
259 | if (cpu) |
260 | *cpu = (p & VDSO_CPUNODE_MASK); |
261 | if (node) |
262 | *node = (p >> VDSO_CPUNODE_BITS); |
263 | } |
264 | |
265 | #endif /* !__ASSEMBLY__ */ |
266 | #endif /* CONFIG_X86_64 */ |
267 | |
268 | #ifdef __KERNEL__ |
269 | |
270 | /* |
271 | * early_idt_handler_array is an array of entry points referenced in the |
272 | * early IDT. For simplicity, it's a real array with one entry point |
273 | * every nine bytes. That leaves room for an optional 'push $0' if the |
274 | * vector has no error code (two bytes), a 'push $vector_number' (two |
275 | * bytes), and a jump to the common entry code (up to five bytes). |
276 | */ |
277 | #define EARLY_IDT_HANDLER_SIZE 9 |
278 | |
279 | /* |
280 | * xen_early_idt_handler_array is for Xen pv guests: for each entry in |
281 | * early_idt_handler_array it contains a prequel in the form of |
282 | * pop %rcx; pop %r11; jmp early_idt_handler_array[i]; summing up to |
283 | * max 8 bytes. |
284 | */ |
285 | #define XEN_EARLY_IDT_HANDLER_SIZE 8 |
286 | |
287 | #ifndef __ASSEMBLY__ |
288 | |
289 | extern const char early_idt_handler_array[NUM_EXCEPTION_VECTORS][EARLY_IDT_HANDLER_SIZE]; |
290 | extern void early_ignore_irq(void); |
291 | |
292 | #if defined(CONFIG_X86_64) && defined(CONFIG_XEN_PV) |
293 | extern const char xen_early_idt_handler_array[NUM_EXCEPTION_VECTORS][XEN_EARLY_IDT_HANDLER_SIZE]; |
294 | #endif |
295 | |
296 | /* |
297 | * Load a segment. Fall back on loading the zero segment if something goes |
298 | * wrong. This variant assumes that loading zero fully clears the segment. |
299 | * This is always the case on Intel CPUs and, even on 64-bit AMD CPUs, any |
300 | * failure to fully clear the cached descriptor is only observable for |
301 | * FS and GS. |
302 | */ |
303 | #define __loadsegment_simple(seg, value) \ |
304 | do { \ |
305 | unsigned short __val = (value); \ |
306 | \ |
307 | asm volatile(" \n" \ |
308 | "1: movl %k0,%%" #seg " \n" \ |
309 | \ |
310 | ".section .fixup,\"ax\" \n" \ |
311 | "2: xorl %k0,%k0 \n" \ |
312 | " jmp 1b \n" \ |
313 | ".previous \n" \ |
314 | \ |
315 | _ASM_EXTABLE(1b, 2b) \ |
316 | \ |
317 | : "+r" (__val) : : "memory"); \ |
318 | } while (0) |
319 | |
320 | #define __loadsegment_ss(value) __loadsegment_simple(ss, (value)) |
321 | #define __loadsegment_ds(value) __loadsegment_simple(ds, (value)) |
322 | #define __loadsegment_es(value) __loadsegment_simple(es, (value)) |
323 | |
324 | #ifdef CONFIG_X86_32 |
325 | |
326 | /* |
327 | * On 32-bit systems, the hidden parts of FS and GS are unobservable if |
328 | * the selector is NULL, so there's no funny business here. |
329 | */ |
330 | #define __loadsegment_fs(value) __loadsegment_simple(fs, (value)) |
331 | #define __loadsegment_gs(value) __loadsegment_simple(gs, (value)) |
332 | |
333 | #else |
334 | |
335 | static inline void __loadsegment_fs(unsigned short value) |
336 | { |
337 | asm volatile(" \n" |
338 | "1: movw %0, %%fs \n" |
339 | "2: \n" |
340 | |
341 | _ASM_EXTABLE_HANDLE(1b, 2b, ex_handler_clear_fs) |
342 | |
343 | : : "rm" (value) : "memory" ); |
344 | } |
345 | |
346 | /* __loadsegment_gs is intentionally undefined. Use load_gs_index instead. */ |
347 | |
348 | #endif |
349 | |
350 | #define loadsegment(seg, value) __loadsegment_ ## seg (value) |
351 | |
352 | /* |
353 | * Save a segment register away: |
354 | */ |
355 | #define savesegment(seg, value) \ |
356 | asm("mov %%" #seg ",%0":"=r" (value) : : "memory") |
357 | |
358 | /* |
359 | * x86-32 user GS accessors: |
360 | */ |
361 | #ifdef CONFIG_X86_32 |
362 | # ifdef CONFIG_X86_32_LAZY_GS |
363 | # define get_user_gs(regs) (u16)({ unsigned long v; savesegment(gs, v); v; }) |
364 | # define set_user_gs(regs, v) loadsegment(gs, (unsigned long)(v)) |
365 | # define task_user_gs(tsk) ((tsk)->thread.gs) |
366 | # define lazy_save_gs(v) savesegment(gs, (v)) |
367 | # define lazy_load_gs(v) loadsegment(gs, (v)) |
368 | # else /* X86_32_LAZY_GS */ |
369 | # define get_user_gs(regs) (u16)((regs)->gs) |
370 | # define set_user_gs(regs, v) do { (regs)->gs = (v); } while (0) |
371 | # define task_user_gs(tsk) (task_pt_regs(tsk)->gs) |
372 | # define lazy_save_gs(v) do { } while (0) |
373 | # define lazy_load_gs(v) do { } while (0) |
374 | # endif /* X86_32_LAZY_GS */ |
375 | #endif /* X86_32 */ |
376 | |
377 | #endif /* !__ASSEMBLY__ */ |
378 | #endif /* __KERNEL__ */ |
379 | |
380 | #endif /* _ASM_X86_SEGMENT_H */ |
381 | |