1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #ifndef _ASM_X86_IDTENTRY_H |
3 | #define _ASM_X86_IDTENTRY_H |
4 | |
5 | /* Interrupts/Exceptions */ |
6 | #include <asm/trapnr.h> |
7 | |
8 | #define IDT_ALIGN (8 * (1 + HAS_KERNEL_IBT)) |
9 | |
10 | #ifndef __ASSEMBLY__ |
11 | #include <linux/entry-common.h> |
12 | #include <linux/hardirq.h> |
13 | |
14 | #include <asm/irq_stack.h> |
15 | |
16 | typedef void (*idtentry_t)(struct pt_regs *regs); |
17 | |
18 | /** |
19 | * DECLARE_IDTENTRY - Declare functions for simple IDT entry points |
20 | * No error code pushed by hardware |
21 | * @vector: Vector number (ignored for C) |
22 | * @func: Function name of the entry point |
23 | * |
24 | * Declares four functions: |
25 | * - The ASM entry point: asm_##func |
26 | * - The XEN PV trap entry point: xen_##func (maybe unused) |
27 | * - The C handler called from the FRED event dispatcher (maybe unused) |
28 | * - The C handler called from the ASM entry point |
29 | * |
30 | * Note: This is the C variant of DECLARE_IDTENTRY(). As the name says it |
31 | * declares the entry points for usage in C code. There is an ASM variant |
32 | * as well which is used to emit the entry stubs in entry_32/64.S. |
33 | */ |
34 | #define DECLARE_IDTENTRY(vector, func) \ |
35 | asmlinkage void asm_##func(void); \ |
36 | asmlinkage void xen_asm_##func(void); \ |
37 | void fred_##func(struct pt_regs *regs); \ |
38 | __visible void func(struct pt_regs *regs) |
39 | |
40 | /** |
41 | * DEFINE_IDTENTRY - Emit code for simple IDT entry points |
42 | * @func: Function name of the entry point |
43 | * |
44 | * @func is called from ASM entry code with interrupts disabled. |
45 | * |
46 | * The macro is written so it acts as function definition. Append the |
47 | * body with a pair of curly brackets. |
48 | * |
49 | * irqentry_enter() contains common code which has to be invoked before |
50 | * arbitrary code in the body. irqentry_exit() contains common code |
51 | * which has to run before returning to the low level assembly code. |
52 | */ |
53 | #define DEFINE_IDTENTRY(func) \ |
54 | static __always_inline void __##func(struct pt_regs *regs); \ |
55 | \ |
56 | __visible noinstr void func(struct pt_regs *regs) \ |
57 | { \ |
58 | irqentry_state_t state = irqentry_enter(regs); \ |
59 | \ |
60 | instrumentation_begin(); \ |
61 | __##func (regs); \ |
62 | instrumentation_end(); \ |
63 | irqentry_exit(regs, state); \ |
64 | } \ |
65 | \ |
66 | static __always_inline void __##func(struct pt_regs *regs) |
67 | |
68 | /* Special case for 32bit IRET 'trap' */ |
69 | #define DECLARE_IDTENTRY_SW DECLARE_IDTENTRY |
70 | #define DEFINE_IDTENTRY_SW DEFINE_IDTENTRY |
71 | |
72 | /** |
73 | * DECLARE_IDTENTRY_ERRORCODE - Declare functions for simple IDT entry points |
74 | * Error code pushed by hardware |
75 | * @vector: Vector number (ignored for C) |
76 | * @func: Function name of the entry point |
77 | * |
78 | * Declares three functions: |
79 | * - The ASM entry point: asm_##func |
80 | * - The XEN PV trap entry point: xen_##func (maybe unused) |
81 | * - The C handler called from the ASM entry point |
82 | * |
83 | * Same as DECLARE_IDTENTRY, but has an extra error_code argument for the |
84 | * C-handler. |
85 | */ |
86 | #define DECLARE_IDTENTRY_ERRORCODE(vector, func) \ |
87 | asmlinkage void asm_##func(void); \ |
88 | asmlinkage void xen_asm_##func(void); \ |
89 | __visible void func(struct pt_regs *regs, unsigned long error_code) |
90 | |
91 | /** |
92 | * DEFINE_IDTENTRY_ERRORCODE - Emit code for simple IDT entry points |
93 | * Error code pushed by hardware |
94 | * @func: Function name of the entry point |
95 | * |
96 | * Same as DEFINE_IDTENTRY, but has an extra error_code argument |
97 | */ |
98 | #define DEFINE_IDTENTRY_ERRORCODE(func) \ |
99 | static __always_inline void __##func(struct pt_regs *regs, \ |
100 | unsigned long error_code); \ |
101 | \ |
102 | __visible noinstr void func(struct pt_regs *regs, \ |
103 | unsigned long error_code) \ |
104 | { \ |
105 | irqentry_state_t state = irqentry_enter(regs); \ |
106 | \ |
107 | instrumentation_begin(); \ |
108 | __##func (regs, error_code); \ |
109 | instrumentation_end(); \ |
110 | irqentry_exit(regs, state); \ |
111 | } \ |
112 | \ |
113 | static __always_inline void __##func(struct pt_regs *regs, \ |
114 | unsigned long error_code) |
115 | |
116 | /** |
117 | * DECLARE_IDTENTRY_RAW - Declare functions for raw IDT entry points |
118 | * No error code pushed by hardware |
119 | * @vector: Vector number (ignored for C) |
120 | * @func: Function name of the entry point |
121 | * |
122 | * Maps to DECLARE_IDTENTRY(). |
123 | */ |
124 | #define DECLARE_IDTENTRY_RAW(vector, func) \ |
125 | DECLARE_IDTENTRY(vector, func) |
126 | |
127 | /** |
128 | * DEFINE_IDTENTRY_RAW - Emit code for raw IDT entry points |
129 | * @func: Function name of the entry point |
130 | * |
131 | * @func is called from ASM entry code with interrupts disabled. |
132 | * |
133 | * The macro is written so it acts as function definition. Append the |
134 | * body with a pair of curly brackets. |
135 | * |
136 | * Contrary to DEFINE_IDTENTRY() this does not invoke the |
137 | * idtentry_enter/exit() helpers before and after the body invocation. This |
138 | * needs to be done in the body itself if applicable. Use if extra work |
139 | * is required before the enter/exit() helpers are invoked. |
140 | */ |
141 | #define DEFINE_IDTENTRY_RAW(func) \ |
142 | __visible noinstr void func(struct pt_regs *regs) |
143 | |
144 | /** |
145 | * DEFINE_FREDENTRY_RAW - Emit code for raw FRED entry points |
146 | * @func: Function name of the entry point |
147 | * |
148 | * @func is called from the FRED event dispatcher with interrupts disabled. |
149 | * |
150 | * See @DEFINE_IDTENTRY_RAW for further details. |
151 | */ |
152 | #define DEFINE_FREDENTRY_RAW(func) \ |
153 | noinstr void fred_##func(struct pt_regs *regs) |
154 | |
155 | /** |
156 | * DECLARE_IDTENTRY_RAW_ERRORCODE - Declare functions for raw IDT entry points |
157 | * Error code pushed by hardware |
158 | * @vector: Vector number (ignored for C) |
159 | * @func: Function name of the entry point |
160 | * |
161 | * Maps to DECLARE_IDTENTRY_ERRORCODE() |
162 | */ |
163 | #define DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func) \ |
164 | DECLARE_IDTENTRY_ERRORCODE(vector, func) |
165 | |
166 | /** |
167 | * DEFINE_IDTENTRY_RAW_ERRORCODE - Emit code for raw IDT entry points |
168 | * @func: Function name of the entry point |
169 | * |
170 | * @func is called from ASM entry code with interrupts disabled. |
171 | * |
172 | * The macro is written so it acts as function definition. Append the |
173 | * body with a pair of curly brackets. |
174 | * |
175 | * Contrary to DEFINE_IDTENTRY_ERRORCODE() this does not invoke the |
176 | * irqentry_enter/exit() helpers before and after the body invocation. This |
177 | * needs to be done in the body itself if applicable. Use if extra work |
178 | * is required before the enter/exit() helpers are invoked. |
179 | */ |
180 | #define DEFINE_IDTENTRY_RAW_ERRORCODE(func) \ |
181 | __visible noinstr void func(struct pt_regs *regs, unsigned long error_code) |
182 | |
183 | /** |
184 | * DECLARE_IDTENTRY_IRQ - Declare functions for device interrupt IDT entry |
185 | * points (common/spurious) |
186 | * @vector: Vector number (ignored for C) |
187 | * @func: Function name of the entry point |
188 | * |
189 | * Maps to DECLARE_IDTENTRY_ERRORCODE() |
190 | */ |
191 | #define DECLARE_IDTENTRY_IRQ(vector, func) \ |
192 | DECLARE_IDTENTRY_ERRORCODE(vector, func) |
193 | |
194 | /** |
195 | * DEFINE_IDTENTRY_IRQ - Emit code for device interrupt IDT entry points |
196 | * @func: Function name of the entry point |
197 | * |
198 | * The vector number is pushed by the low level entry stub and handed |
199 | * to the function as error_code argument which needs to be truncated |
200 | * to an u8 because the push is sign extending. |
201 | * |
202 | * irq_enter/exit_rcu() are invoked before the function body and the |
203 | * KVM L1D flush request is set. Stack switching to the interrupt stack |
204 | * has to be done in the function body if necessary. |
205 | */ |
206 | #define DEFINE_IDTENTRY_IRQ(func) \ |
207 | static void __##func(struct pt_regs *regs, u32 vector); \ |
208 | \ |
209 | __visible noinstr void func(struct pt_regs *regs, \ |
210 | unsigned long error_code) \ |
211 | { \ |
212 | irqentry_state_t state = irqentry_enter(regs); \ |
213 | u32 vector = (u32)(u8)error_code; \ |
214 | \ |
215 | instrumentation_begin(); \ |
216 | kvm_set_cpu_l1tf_flush_l1d(); \ |
217 | run_irq_on_irqstack_cond(__##func, regs, vector); \ |
218 | instrumentation_end(); \ |
219 | irqentry_exit(regs, state); \ |
220 | } \ |
221 | \ |
222 | static noinline void __##func(struct pt_regs *regs, u32 vector) |
223 | |
224 | /** |
225 | * DECLARE_IDTENTRY_SYSVEC - Declare functions for system vector entry points |
226 | * @vector: Vector number (ignored for C) |
227 | * @func: Function name of the entry point |
228 | * |
229 | * Declares three functions: |
230 | * - The ASM entry point: asm_##func |
231 | * - The XEN PV trap entry point: xen_##func (maybe unused) |
232 | * - The C handler called from the ASM entry point |
233 | * |
234 | * Maps to DECLARE_IDTENTRY(). |
235 | */ |
236 | #define DECLARE_IDTENTRY_SYSVEC(vector, func) \ |
237 | DECLARE_IDTENTRY(vector, func) |
238 | |
239 | /** |
240 | * DEFINE_IDTENTRY_SYSVEC - Emit code for system vector IDT entry points |
241 | * @func: Function name of the entry point |
242 | * |
243 | * irqentry_enter/exit() and irq_enter/exit_rcu() are invoked before the |
244 | * function body. KVM L1D flush request is set. |
245 | * |
246 | * Runs the function on the interrupt stack if the entry hit kernel mode |
247 | */ |
248 | #define DEFINE_IDTENTRY_SYSVEC(func) \ |
249 | static void __##func(struct pt_regs *regs); \ |
250 | \ |
251 | static __always_inline void instr_##func(struct pt_regs *regs) \ |
252 | { \ |
253 | kvm_set_cpu_l1tf_flush_l1d(); \ |
254 | run_sysvec_on_irqstack_cond(__##func, regs); \ |
255 | } \ |
256 | \ |
257 | __visible noinstr void func(struct pt_regs *regs) \ |
258 | { \ |
259 | irqentry_state_t state = irqentry_enter(regs); \ |
260 | \ |
261 | instrumentation_begin(); \ |
262 | instr_##func (regs); \ |
263 | instrumentation_end(); \ |
264 | irqentry_exit(regs, state); \ |
265 | } \ |
266 | \ |
267 | void fred_##func(struct pt_regs *regs) \ |
268 | { \ |
269 | instr_##func (regs); \ |
270 | } \ |
271 | \ |
272 | static noinline void __##func(struct pt_regs *regs) |
273 | |
274 | /** |
275 | * DEFINE_IDTENTRY_SYSVEC_SIMPLE - Emit code for simple system vector IDT |
276 | * entry points |
277 | * @func: Function name of the entry point |
278 | * |
279 | * Runs the function on the interrupted stack. No switch to IRQ stack and |
280 | * only the minimal __irq_enter/exit() handling. |
281 | * |
282 | * Only use for 'empty' vectors like reschedule IPI and KVM posted |
283 | * interrupt vectors. |
284 | */ |
285 | #define DEFINE_IDTENTRY_SYSVEC_SIMPLE(func) \ |
286 | static __always_inline void __##func(struct pt_regs *regs); \ |
287 | \ |
288 | static __always_inline void instr_##func(struct pt_regs *regs) \ |
289 | { \ |
290 | __irq_enter_raw(); \ |
291 | kvm_set_cpu_l1tf_flush_l1d(); \ |
292 | __##func (regs); \ |
293 | __irq_exit_raw(); \ |
294 | } \ |
295 | \ |
296 | __visible noinstr void func(struct pt_regs *regs) \ |
297 | { \ |
298 | irqentry_state_t state = irqentry_enter(regs); \ |
299 | \ |
300 | instrumentation_begin(); \ |
301 | instr_##func (regs); \ |
302 | instrumentation_end(); \ |
303 | irqentry_exit(regs, state); \ |
304 | } \ |
305 | \ |
306 | void fred_##func(struct pt_regs *regs) \ |
307 | { \ |
308 | instr_##func (regs); \ |
309 | } \ |
310 | \ |
311 | static __always_inline void __##func(struct pt_regs *regs) |
312 | |
313 | /** |
314 | * DECLARE_IDTENTRY_XENCB - Declare functions for XEN HV callback entry point |
315 | * @vector: Vector number (ignored for C) |
316 | * @func: Function name of the entry point |
317 | * |
318 | * Declares three functions: |
319 | * - The ASM entry point: asm_##func |
320 | * - The XEN PV trap entry point: xen_##func (maybe unused) |
321 | * - The C handler called from the ASM entry point |
322 | * |
323 | * Maps to DECLARE_IDTENTRY(). Distinct entry point to handle the 32/64-bit |
324 | * difference |
325 | */ |
326 | #define DECLARE_IDTENTRY_XENCB(vector, func) \ |
327 | DECLARE_IDTENTRY(vector, func) |
328 | |
329 | #ifdef CONFIG_X86_64 |
330 | /** |
331 | * DECLARE_IDTENTRY_IST - Declare functions for IST handling IDT entry points |
332 | * @vector: Vector number (ignored for C) |
333 | * @func: Function name of the entry point |
334 | * |
335 | * Maps to DECLARE_IDTENTRY_RAW, but declares also the NOIST C handler |
336 | * which is called from the ASM entry point on user mode entry |
337 | */ |
338 | #define DECLARE_IDTENTRY_IST(vector, func) \ |
339 | DECLARE_IDTENTRY_RAW(vector, func); \ |
340 | __visible void noist_##func(struct pt_regs *regs) |
341 | |
342 | /** |
343 | * DECLARE_IDTENTRY_VC - Declare functions for the VC entry point |
344 | * @vector: Vector number (ignored for C) |
345 | * @func: Function name of the entry point |
346 | * |
347 | * Maps to DECLARE_IDTENTRY_RAW_ERRORCODE, but declares also the |
348 | * safe_stack C handler. |
349 | */ |
350 | #define DECLARE_IDTENTRY_VC(vector, func) \ |
351 | DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func); \ |
352 | __visible noinstr void kernel_##func(struct pt_regs *regs, unsigned long error_code); \ |
353 | __visible noinstr void user_##func(struct pt_regs *regs, unsigned long error_code) |
354 | |
355 | /** |
356 | * DEFINE_IDTENTRY_IST - Emit code for IST entry points |
357 | * @func: Function name of the entry point |
358 | * |
359 | * Maps to DEFINE_IDTENTRY_RAW |
360 | */ |
361 | #define DEFINE_IDTENTRY_IST(func) \ |
362 | DEFINE_IDTENTRY_RAW(func) |
363 | |
364 | /** |
365 | * DEFINE_IDTENTRY_NOIST - Emit code for NOIST entry points which |
366 | * belong to a IST entry point (MCE, DB) |
367 | * @func: Function name of the entry point. Must be the same as |
368 | * the function name of the corresponding IST variant |
369 | * |
370 | * Maps to DEFINE_IDTENTRY_RAW(). |
371 | */ |
372 | #define DEFINE_IDTENTRY_NOIST(func) \ |
373 | DEFINE_IDTENTRY_RAW(noist_##func) |
374 | |
375 | /** |
376 | * DECLARE_IDTENTRY_DF - Declare functions for double fault |
377 | * @vector: Vector number (ignored for C) |
378 | * @func: Function name of the entry point |
379 | * |
380 | * Maps to DECLARE_IDTENTRY_RAW_ERRORCODE |
381 | */ |
382 | #define DECLARE_IDTENTRY_DF(vector, func) \ |
383 | DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func) |
384 | |
385 | /** |
386 | * DEFINE_IDTENTRY_DF - Emit code for double fault |
387 | * @func: Function name of the entry point |
388 | * |
389 | * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE |
390 | */ |
391 | #define DEFINE_IDTENTRY_DF(func) \ |
392 | DEFINE_IDTENTRY_RAW_ERRORCODE(func) |
393 | |
394 | /** |
395 | * DEFINE_IDTENTRY_VC_KERNEL - Emit code for VMM communication handler |
396 | when raised from kernel mode |
397 | * @func: Function name of the entry point |
398 | * |
399 | * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE |
400 | */ |
401 | #define DEFINE_IDTENTRY_VC_KERNEL(func) \ |
402 | DEFINE_IDTENTRY_RAW_ERRORCODE(kernel_##func) |
403 | |
404 | /** |
405 | * DEFINE_IDTENTRY_VC_USER - Emit code for VMM communication handler |
406 | when raised from user mode |
407 | * @func: Function name of the entry point |
408 | * |
409 | * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE |
410 | */ |
411 | #define DEFINE_IDTENTRY_VC_USER(func) \ |
412 | DEFINE_IDTENTRY_RAW_ERRORCODE(user_##func) |
413 | |
414 | #else /* CONFIG_X86_64 */ |
415 | |
416 | /** |
417 | * DECLARE_IDTENTRY_DF - Declare functions for double fault 32bit variant |
418 | * @vector: Vector number (ignored for C) |
419 | * @func: Function name of the entry point |
420 | * |
421 | * Declares two functions: |
422 | * - The ASM entry point: asm_##func |
423 | * - The C handler called from the C shim |
424 | */ |
425 | #define DECLARE_IDTENTRY_DF(vector, func) \ |
426 | asmlinkage void asm_##func(void); \ |
427 | __visible void func(struct pt_regs *regs, \ |
428 | unsigned long error_code, \ |
429 | unsigned long address) |
430 | |
431 | /** |
432 | * DEFINE_IDTENTRY_DF - Emit code for double fault on 32bit |
433 | * @func: Function name of the entry point |
434 | * |
435 | * This is called through the doublefault shim which already provides |
436 | * cr2 in the address argument. |
437 | */ |
438 | #define DEFINE_IDTENTRY_DF(func) \ |
439 | __visible noinstr void func(struct pt_regs *regs, \ |
440 | unsigned long error_code, \ |
441 | unsigned long address) |
442 | |
443 | #endif /* !CONFIG_X86_64 */ |
444 | |
445 | /* C-Code mapping */ |
446 | #define DECLARE_IDTENTRY_NMI DECLARE_IDTENTRY_RAW |
447 | #define DEFINE_IDTENTRY_NMI DEFINE_IDTENTRY_RAW |
448 | #define DEFINE_FREDENTRY_NMI DEFINE_FREDENTRY_RAW |
449 | |
450 | #ifdef CONFIG_X86_64 |
451 | #define DECLARE_IDTENTRY_MCE DECLARE_IDTENTRY_IST |
452 | #define DEFINE_IDTENTRY_MCE DEFINE_IDTENTRY_IST |
453 | #define DEFINE_IDTENTRY_MCE_USER DEFINE_IDTENTRY_NOIST |
454 | #define DEFINE_FREDENTRY_MCE DEFINE_FREDENTRY_RAW |
455 | |
456 | #define DECLARE_IDTENTRY_DEBUG DECLARE_IDTENTRY_IST |
457 | #define DEFINE_IDTENTRY_DEBUG DEFINE_IDTENTRY_IST |
458 | #define DEFINE_IDTENTRY_DEBUG_USER DEFINE_IDTENTRY_NOIST |
459 | #define DEFINE_FREDENTRY_DEBUG DEFINE_FREDENTRY_RAW |
460 | #endif |
461 | |
462 | void idt_install_sysvec(unsigned int n, const void *function); |
463 | |
464 | #ifdef CONFIG_X86_FRED |
465 | void fred_install_sysvec(unsigned int vector, const idtentry_t function); |
466 | #else |
467 | static inline void fred_install_sysvec(unsigned int vector, const idtentry_t function) { } |
468 | #endif |
469 | |
470 | #define sysvec_install(vector, function) { \ |
471 | if (cpu_feature_enabled(X86_FEATURE_FRED)) \ |
472 | fred_install_sysvec(vector, function); \ |
473 | else \ |
474 | idt_install_sysvec(vector, asm_##function); \ |
475 | } |
476 | |
477 | #else /* !__ASSEMBLY__ */ |
478 | |
479 | /* |
480 | * The ASM variants for DECLARE_IDTENTRY*() which emit the ASM entry stubs. |
481 | */ |
482 | #define DECLARE_IDTENTRY(vector, func) \ |
483 | idtentry vector asm_##func func has_error_code=0 |
484 | |
485 | #define DECLARE_IDTENTRY_ERRORCODE(vector, func) \ |
486 | idtentry vector asm_##func func has_error_code=1 |
487 | |
488 | /* Special case for 32bit IRET 'trap'. Do not emit ASM code */ |
489 | #define DECLARE_IDTENTRY_SW(vector, func) |
490 | |
491 | #define DECLARE_IDTENTRY_RAW(vector, func) \ |
492 | DECLARE_IDTENTRY(vector, func) |
493 | |
494 | #define DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func) \ |
495 | DECLARE_IDTENTRY_ERRORCODE(vector, func) |
496 | |
497 | /* Entries for common/spurious (device) interrupts */ |
498 | #define DECLARE_IDTENTRY_IRQ(vector, func) \ |
499 | idtentry_irq vector func |
500 | |
501 | /* System vector entries */ |
502 | #define DECLARE_IDTENTRY_SYSVEC(vector, func) \ |
503 | DECLARE_IDTENTRY(vector, func) |
504 | |
505 | #ifdef CONFIG_X86_64 |
506 | # define DECLARE_IDTENTRY_MCE(vector, func) \ |
507 | idtentry_mce_db vector asm_##func func |
508 | |
509 | # define DECLARE_IDTENTRY_DEBUG(vector, func) \ |
510 | idtentry_mce_db vector asm_##func func |
511 | |
512 | # define DECLARE_IDTENTRY_DF(vector, func) \ |
513 | idtentry_df vector asm_##func func |
514 | |
515 | # define DECLARE_IDTENTRY_XENCB(vector, func) \ |
516 | DECLARE_IDTENTRY(vector, func) |
517 | |
518 | # define DECLARE_IDTENTRY_VC(vector, func) \ |
519 | idtentry_vc vector asm_##func func |
520 | |
521 | #else |
522 | # define DECLARE_IDTENTRY_MCE(vector, func) \ |
523 | DECLARE_IDTENTRY(vector, func) |
524 | |
525 | /* No ASM emitted for DF as this goes through a C shim */ |
526 | # define DECLARE_IDTENTRY_DF(vector, func) |
527 | |
528 | /* No ASM emitted for XEN hypervisor callback */ |
529 | # define DECLARE_IDTENTRY_XENCB(vector, func) |
530 | |
531 | #endif |
532 | |
533 | /* No ASM code emitted for NMI */ |
534 | #define DECLARE_IDTENTRY_NMI(vector, func) |
535 | |
536 | /* |
537 | * ASM code to emit the common vector entry stubs where each stub is |
538 | * packed into IDT_ALIGN bytes. |
539 | * |
540 | * Note, that the 'pushq imm8' is emitted via '.byte 0x6a, vector' because |
541 | * GCC treats the local vector variable as unsigned int and would expand |
542 | * all vectors above 0x7F to a 5 byte push. The original code did an |
543 | * adjustment of the vector number to be in the signed byte range to avoid |
544 | * this. While clever it's mindboggling counterintuitive and requires the |
545 | * odd conversion back to a real vector number in the C entry points. Using |
546 | * .byte achieves the same thing and the only fixup needed in the C entry |
547 | * point is to mask off the bits above bit 7 because the push is sign |
548 | * extending. |
549 | */ |
550 | .align IDT_ALIGN |
551 | SYM_CODE_START(irq_entries_start) |
552 | vector=FIRST_EXTERNAL_VECTOR |
553 | .rept NR_EXTERNAL_VECTORS |
554 | UNWIND_HINT_IRET_REGS |
555 | 0 : |
556 | ENDBR |
557 | .byte 0x6a, vector |
558 | jmp asm_common_interrupt |
559 | /* Ensure that the above is IDT_ALIGN bytes max */ |
560 | .fill 0b + IDT_ALIGN - ., 1, 0xcc |
561 | vector = vector+1 |
562 | .endr |
563 | SYM_CODE_END(irq_entries_start) |
564 | |
565 | #ifdef CONFIG_X86_LOCAL_APIC |
566 | .align IDT_ALIGN |
567 | SYM_CODE_START(spurious_entries_start) |
568 | vector=FIRST_SYSTEM_VECTOR |
569 | .rept NR_SYSTEM_VECTORS |
570 | UNWIND_HINT_IRET_REGS |
571 | 0 : |
572 | ENDBR |
573 | .byte 0x6a, vector |
574 | jmp asm_spurious_interrupt |
575 | /* Ensure that the above is IDT_ALIGN bytes max */ |
576 | .fill 0b + IDT_ALIGN - ., 1, 0xcc |
577 | vector = vector+1 |
578 | .endr |
579 | SYM_CODE_END(spurious_entries_start) |
580 | #endif |
581 | |
582 | #endif /* __ASSEMBLY__ */ |
583 | |
584 | /* |
585 | * The actual entry points. Note that DECLARE_IDTENTRY*() serves two |
586 | * purposes: |
587 | * - provide the function declarations when included from C-Code |
588 | * - emit the ASM stubs when included from entry_32/64.S |
589 | * |
590 | * This avoids duplicate defines and ensures that everything is consistent. |
591 | */ |
592 | |
593 | /* |
594 | * Dummy trap number so the low level ASM macro vector number checks do not |
595 | * match which results in emitting plain IDTENTRY stubs without bells and |
596 | * whistles. |
597 | */ |
598 | #define X86_TRAP_OTHER 0xFFFF |
599 | |
600 | /* Simple exception entry points. No hardware error code */ |
601 | DECLARE_IDTENTRY(X86_TRAP_DE, exc_divide_error); |
602 | DECLARE_IDTENTRY(X86_TRAP_OF, exc_overflow); |
603 | DECLARE_IDTENTRY(X86_TRAP_BR, exc_bounds); |
604 | DECLARE_IDTENTRY(X86_TRAP_NM, exc_device_not_available); |
605 | DECLARE_IDTENTRY(X86_TRAP_OLD_MF, exc_coproc_segment_overrun); |
606 | DECLARE_IDTENTRY(X86_TRAP_SPURIOUS, exc_spurious_interrupt_bug); |
607 | DECLARE_IDTENTRY(X86_TRAP_MF, exc_coprocessor_error); |
608 | DECLARE_IDTENTRY(X86_TRAP_XF, exc_simd_coprocessor_error); |
609 | |
610 | /* 32bit software IRET trap. Do not emit ASM code */ |
611 | DECLARE_IDTENTRY_SW(X86_TRAP_IRET, iret_error); |
612 | |
613 | /* Simple exception entries with error code pushed by hardware */ |
614 | DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_TS, exc_invalid_tss); |
615 | DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_NP, exc_segment_not_present); |
616 | DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_SS, exc_stack_segment); |
617 | DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_GP, exc_general_protection); |
618 | DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_AC, exc_alignment_check); |
619 | |
620 | /* Raw exception entries which need extra work */ |
621 | DECLARE_IDTENTRY_RAW(X86_TRAP_UD, exc_invalid_op); |
622 | DECLARE_IDTENTRY_RAW(X86_TRAP_BP, exc_int3); |
623 | DECLARE_IDTENTRY_RAW_ERRORCODE(X86_TRAP_PF, exc_page_fault); |
624 | |
625 | #if defined(CONFIG_IA32_EMULATION) |
626 | DECLARE_IDTENTRY_RAW(IA32_SYSCALL_VECTOR, int80_emulation); |
627 | #endif |
628 | |
629 | #ifdef CONFIG_X86_MCE |
630 | #ifdef CONFIG_X86_64 |
631 | DECLARE_IDTENTRY_MCE(X86_TRAP_MC, exc_machine_check); |
632 | #else |
633 | DECLARE_IDTENTRY_RAW(X86_TRAP_MC, exc_machine_check); |
634 | #endif |
635 | #ifdef CONFIG_XEN_PV |
636 | DECLARE_IDTENTRY_RAW(X86_TRAP_MC, xenpv_exc_machine_check); |
637 | #endif |
638 | #endif |
639 | |
640 | /* NMI */ |
641 | |
642 | #if IS_ENABLED(CONFIG_KVM_INTEL) |
643 | /* |
644 | * Special entry point for VMX which invokes this on the kernel stack, even for |
645 | * 64-bit, i.e. without using an IST. asm_exc_nmi() requires an IST to work |
646 | * correctly vs. the NMI 'executing' marker. Used for 32-bit kernels as well |
647 | * to avoid more ifdeffery. |
648 | */ |
649 | DECLARE_IDTENTRY(X86_TRAP_NMI, exc_nmi_kvm_vmx); |
650 | #endif |
651 | |
652 | DECLARE_IDTENTRY_NMI(X86_TRAP_NMI, exc_nmi); |
653 | #ifdef CONFIG_XEN_PV |
654 | DECLARE_IDTENTRY_RAW(X86_TRAP_NMI, xenpv_exc_nmi); |
655 | #endif |
656 | |
657 | /* #DB */ |
658 | #ifdef CONFIG_X86_64 |
659 | DECLARE_IDTENTRY_DEBUG(X86_TRAP_DB, exc_debug); |
660 | #else |
661 | DECLARE_IDTENTRY_RAW(X86_TRAP_DB, exc_debug); |
662 | #endif |
663 | #ifdef CONFIG_XEN_PV |
664 | DECLARE_IDTENTRY_RAW(X86_TRAP_DB, xenpv_exc_debug); |
665 | #endif |
666 | |
667 | /* #DF */ |
668 | DECLARE_IDTENTRY_DF(X86_TRAP_DF, exc_double_fault); |
669 | #ifdef CONFIG_XEN_PV |
670 | DECLARE_IDTENTRY_RAW_ERRORCODE(X86_TRAP_DF, xenpv_exc_double_fault); |
671 | #endif |
672 | |
673 | /* #CP */ |
674 | #ifdef CONFIG_X86_CET |
675 | DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_CP, exc_control_protection); |
676 | #endif |
677 | |
678 | /* #VC */ |
679 | #ifdef CONFIG_AMD_MEM_ENCRYPT |
680 | DECLARE_IDTENTRY_VC(X86_TRAP_VC, exc_vmm_communication); |
681 | #endif |
682 | |
683 | #ifdef CONFIG_XEN_PV |
684 | DECLARE_IDTENTRY_XENCB(X86_TRAP_OTHER, exc_xen_hypervisor_callback); |
685 | DECLARE_IDTENTRY_RAW(X86_TRAP_OTHER, exc_xen_unknown_trap); |
686 | #endif |
687 | |
688 | #ifdef CONFIG_INTEL_TDX_GUEST |
689 | DECLARE_IDTENTRY(X86_TRAP_VE, exc_virtualization_exception); |
690 | #endif |
691 | |
692 | /* Device interrupts common/spurious */ |
693 | DECLARE_IDTENTRY_IRQ(X86_TRAP_OTHER, common_interrupt); |
694 | #ifdef CONFIG_X86_LOCAL_APIC |
695 | DECLARE_IDTENTRY_IRQ(X86_TRAP_OTHER, spurious_interrupt); |
696 | #endif |
697 | |
698 | /* System vector entry points */ |
699 | #ifdef CONFIG_X86_LOCAL_APIC |
700 | DECLARE_IDTENTRY_SYSVEC(ERROR_APIC_VECTOR, sysvec_error_interrupt); |
701 | DECLARE_IDTENTRY_SYSVEC(SPURIOUS_APIC_VECTOR, sysvec_spurious_apic_interrupt); |
702 | DECLARE_IDTENTRY_SYSVEC(LOCAL_TIMER_VECTOR, sysvec_apic_timer_interrupt); |
703 | DECLARE_IDTENTRY_SYSVEC(X86_PLATFORM_IPI_VECTOR, sysvec_x86_platform_ipi); |
704 | #endif |
705 | |
706 | #ifdef CONFIG_SMP |
707 | DECLARE_IDTENTRY(RESCHEDULE_VECTOR, sysvec_reschedule_ipi); |
708 | DECLARE_IDTENTRY_SYSVEC(REBOOT_VECTOR, sysvec_reboot); |
709 | DECLARE_IDTENTRY_SYSVEC(CALL_FUNCTION_SINGLE_VECTOR, sysvec_call_function_single); |
710 | DECLARE_IDTENTRY_SYSVEC(CALL_FUNCTION_VECTOR, sysvec_call_function); |
711 | #else |
712 | # define fred_sysvec_reschedule_ipi NULL |
713 | # define fred_sysvec_reboot NULL |
714 | # define fred_sysvec_call_function_single NULL |
715 | # define fred_sysvec_call_function NULL |
716 | #endif |
717 | |
718 | #ifdef CONFIG_X86_LOCAL_APIC |
719 | # ifdef CONFIG_X86_MCE_THRESHOLD |
720 | DECLARE_IDTENTRY_SYSVEC(THRESHOLD_APIC_VECTOR, sysvec_threshold); |
721 | # else |
722 | # define fred_sysvec_threshold NULL |
723 | # endif |
724 | |
725 | # ifdef CONFIG_X86_MCE_AMD |
726 | DECLARE_IDTENTRY_SYSVEC(DEFERRED_ERROR_VECTOR, sysvec_deferred_error); |
727 | # else |
728 | # define fred_sysvec_deferred_error NULL |
729 | # endif |
730 | |
731 | # ifdef CONFIG_X86_THERMAL_VECTOR |
732 | DECLARE_IDTENTRY_SYSVEC(THERMAL_APIC_VECTOR, sysvec_thermal); |
733 | # else |
734 | # define fred_sysvec_thermal NULL |
735 | # endif |
736 | |
737 | # ifdef CONFIG_IRQ_WORK |
738 | DECLARE_IDTENTRY_SYSVEC(IRQ_WORK_VECTOR, sysvec_irq_work); |
739 | # else |
740 | # define fred_sysvec_irq_work NULL |
741 | # endif |
742 | #endif |
743 | |
744 | #if IS_ENABLED(CONFIG_KVM) |
745 | DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_VECTOR, sysvec_kvm_posted_intr_ipi); |
746 | DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_WAKEUP_VECTOR, sysvec_kvm_posted_intr_wakeup_ipi); |
747 | DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_NESTED_VECTOR, sysvec_kvm_posted_intr_nested_ipi); |
748 | #else |
749 | # define fred_sysvec_kvm_posted_intr_ipi NULL |
750 | # define fred_sysvec_kvm_posted_intr_wakeup_ipi NULL |
751 | # define fred_sysvec_kvm_posted_intr_nested_ipi NULL |
752 | #endif |
753 | |
754 | #if IS_ENABLED(CONFIG_HYPERV) |
755 | DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR, sysvec_hyperv_callback); |
756 | DECLARE_IDTENTRY_SYSVEC(HYPERV_REENLIGHTENMENT_VECTOR, sysvec_hyperv_reenlightenment); |
757 | DECLARE_IDTENTRY_SYSVEC(HYPERV_STIMER0_VECTOR, sysvec_hyperv_stimer0); |
758 | #endif |
759 | |
760 | #if IS_ENABLED(CONFIG_ACRN_GUEST) |
761 | DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR, sysvec_acrn_hv_callback); |
762 | #endif |
763 | |
764 | #ifdef CONFIG_XEN_PVHVM |
765 | DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR, sysvec_xen_hvm_callback); |
766 | #endif |
767 | |
768 | #ifdef CONFIG_KVM_GUEST |
769 | DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR, sysvec_kvm_asyncpf_interrupt); |
770 | #endif |
771 | |
772 | #undef X86_TRAP_OTHER |
773 | |
774 | #endif |
775 | |