1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | /* |
3 | * The FRED specific kernel/user entry functions which are invoked from |
4 | * assembly code and dispatch to the associated handlers. |
5 | */ |
6 | #include <linux/kernel.h> |
7 | #include <linux/kdebug.h> |
8 | #include <linux/nospec.h> |
9 | |
10 | #include <asm/desc.h> |
11 | #include <asm/fred.h> |
12 | #include <asm/idtentry.h> |
13 | #include <asm/syscall.h> |
14 | #include <asm/trapnr.h> |
15 | #include <asm/traps.h> |
16 | |
17 | /* FRED EVENT_TYPE_OTHER vector numbers */ |
18 | #define FRED_SYSCALL 1 |
19 | #define FRED_SYSENTER 2 |
20 | |
21 | static noinstr void fred_bad_type(struct pt_regs *regs, unsigned long error_code) |
22 | { |
23 | irqentry_state_t irq_state = irqentry_nmi_enter(regs); |
24 | |
25 | instrumentation_begin(); |
26 | |
27 | /* Panic on events from a high stack level */ |
28 | if (regs->fred_cs.sl > 0) { |
29 | pr_emerg("PANIC: invalid or fatal FRED event; event type %u " |
30 | "vector %u error 0x%lx aux 0x%lx at %04x:%016lx\n" , |
31 | regs->fred_ss.type, regs->fred_ss.vector, error_code, |
32 | fred_event_data(regs), regs->cs, regs->ip); |
33 | die("invalid or fatal FRED event" , regs, error_code); |
34 | panic(fmt: "invalid or fatal FRED event" ); |
35 | } else { |
36 | unsigned long flags = oops_begin(); |
37 | int sig = SIGKILL; |
38 | |
39 | pr_alert("BUG: invalid or fatal FRED event; event type %u " |
40 | "vector %u error 0x%lx aux 0x%lx at %04x:%016lx\n" , |
41 | regs->fred_ss.type, regs->fred_ss.vector, error_code, |
42 | fred_event_data(regs), regs->cs, regs->ip); |
43 | |
44 | if (__die("Invalid or fatal FRED event" , regs, error_code)) |
45 | sig = 0; |
46 | |
47 | oops_end(flags, regs, signr: sig); |
48 | } |
49 | |
50 | instrumentation_end(); |
51 | irqentry_nmi_exit(regs, irq_state); |
52 | } |
53 | |
54 | static noinstr void fred_intx(struct pt_regs *regs) |
55 | { |
56 | switch (regs->fred_ss.vector) { |
57 | /* Opcode 0xcd, 0x3, NOT INT3 (opcode 0xcc) */ |
58 | case X86_TRAP_BP: |
59 | return exc_int3(regs); |
60 | |
61 | /* Opcode 0xcd, 0x4, NOT INTO (opcode 0xce) */ |
62 | case X86_TRAP_OF: |
63 | return exc_overflow(regs); |
64 | |
65 | #ifdef CONFIG_IA32_EMULATION |
66 | /* INT80 */ |
67 | case IA32_SYSCALL_VECTOR: |
68 | if (ia32_enabled()) |
69 | return fred_int80_emulation(regs); |
70 | fallthrough; |
71 | #endif |
72 | |
73 | default: |
74 | return exc_general_protection(regs, error_code: 0); |
75 | } |
76 | } |
77 | |
78 | static __always_inline void fred_other(struct pt_regs *regs) |
79 | { |
80 | /* The compiler can fold these conditions into a single test */ |
81 | if (likely(regs->fred_ss.vector == FRED_SYSCALL && regs->fred_ss.lm)) { |
82 | regs->orig_ax = regs->ax; |
83 | regs->ax = -ENOSYS; |
84 | do_syscall_64(regs, nr: regs->orig_ax); |
85 | return; |
86 | } else if (ia32_enabled() && |
87 | likely(regs->fred_ss.vector == FRED_SYSENTER && !regs->fred_ss.lm)) { |
88 | regs->orig_ax = regs->ax; |
89 | regs->ax = -ENOSYS; |
90 | do_fast_syscall_32(regs); |
91 | return; |
92 | } else { |
93 | exc_invalid_op(regs); |
94 | return; |
95 | } |
96 | } |
97 | |
98 | #define SYSVEC(_vector, _function) [_vector - FIRST_SYSTEM_VECTOR] = fred_sysvec_##_function |
99 | |
100 | static idtentry_t sysvec_table[NR_SYSTEM_VECTORS] __ro_after_init = { |
101 | SYSVEC(ERROR_APIC_VECTOR, error_interrupt), |
102 | SYSVEC(SPURIOUS_APIC_VECTOR, spurious_apic_interrupt), |
103 | SYSVEC(LOCAL_TIMER_VECTOR, apic_timer_interrupt), |
104 | SYSVEC(X86_PLATFORM_IPI_VECTOR, x86_platform_ipi), |
105 | |
106 | SYSVEC(RESCHEDULE_VECTOR, reschedule_ipi), |
107 | SYSVEC(CALL_FUNCTION_SINGLE_VECTOR, call_function_single), |
108 | SYSVEC(CALL_FUNCTION_VECTOR, call_function), |
109 | SYSVEC(REBOOT_VECTOR, reboot), |
110 | |
111 | SYSVEC(THRESHOLD_APIC_VECTOR, threshold), |
112 | SYSVEC(DEFERRED_ERROR_VECTOR, deferred_error), |
113 | SYSVEC(THERMAL_APIC_VECTOR, thermal), |
114 | |
115 | SYSVEC(IRQ_WORK_VECTOR, irq_work), |
116 | |
117 | SYSVEC(POSTED_INTR_VECTOR, kvm_posted_intr_ipi), |
118 | SYSVEC(POSTED_INTR_WAKEUP_VECTOR, kvm_posted_intr_wakeup_ipi), |
119 | SYSVEC(POSTED_INTR_NESTED_VECTOR, kvm_posted_intr_nested_ipi), |
120 | }; |
121 | |
122 | static bool fred_setup_done __initdata; |
123 | |
124 | void __init fred_install_sysvec(unsigned int sysvec, idtentry_t handler) |
125 | { |
126 | if (WARN_ON_ONCE(sysvec < FIRST_SYSTEM_VECTOR)) |
127 | return; |
128 | |
129 | if (WARN_ON_ONCE(fred_setup_done)) |
130 | return; |
131 | |
132 | if (!WARN_ON_ONCE(sysvec_table[sysvec - FIRST_SYSTEM_VECTOR])) |
133 | sysvec_table[sysvec - FIRST_SYSTEM_VECTOR] = handler; |
134 | } |
135 | |
136 | static noinstr void fred_handle_spurious_interrupt(struct pt_regs *regs) |
137 | { |
138 | spurious_interrupt(regs, error_code: regs->fred_ss.vector); |
139 | } |
140 | |
141 | void __init fred_complete_exception_setup(void) |
142 | { |
143 | unsigned int vector; |
144 | |
145 | for (vector = 0; vector < FIRST_EXTERNAL_VECTOR; vector++) |
146 | set_bit(nr: vector, addr: system_vectors); |
147 | |
148 | for (vector = 0; vector < NR_SYSTEM_VECTORS; vector++) { |
149 | if (sysvec_table[vector]) |
150 | set_bit(nr: vector + FIRST_SYSTEM_VECTOR, addr: system_vectors); |
151 | else |
152 | sysvec_table[vector] = fred_handle_spurious_interrupt; |
153 | } |
154 | fred_setup_done = true; |
155 | } |
156 | |
157 | static noinstr void fred_extint(struct pt_regs *regs) |
158 | { |
159 | unsigned int vector = regs->fred_ss.vector; |
160 | unsigned int index = array_index_nospec(vector - FIRST_SYSTEM_VECTOR, |
161 | NR_SYSTEM_VECTORS); |
162 | |
163 | if (WARN_ON_ONCE(vector < FIRST_EXTERNAL_VECTOR)) |
164 | return; |
165 | |
166 | if (likely(vector >= FIRST_SYSTEM_VECTOR)) { |
167 | irqentry_state_t state = irqentry_enter(regs); |
168 | |
169 | instrumentation_begin(); |
170 | sysvec_table[index](regs); |
171 | instrumentation_end(); |
172 | irqentry_exit(regs, state); |
173 | } else { |
174 | common_interrupt(regs, error_code: vector); |
175 | } |
176 | } |
177 | |
178 | static noinstr void fred_hwexc(struct pt_regs *regs, unsigned long error_code) |
179 | { |
180 | /* Optimize for #PF. That's the only exception which matters performance wise */ |
181 | if (likely(regs->fred_ss.vector == X86_TRAP_PF)) |
182 | return exc_page_fault(regs, error_code); |
183 | |
184 | switch (regs->fred_ss.vector) { |
185 | case X86_TRAP_DE: return exc_divide_error(regs); |
186 | case X86_TRAP_DB: return fred_exc_debug(regs); |
187 | case X86_TRAP_BR: return exc_bounds(regs); |
188 | case X86_TRAP_UD: return exc_invalid_op(regs); |
189 | case X86_TRAP_NM: return exc_device_not_available(regs); |
190 | case X86_TRAP_DF: return exc_double_fault(regs, error_code); |
191 | case X86_TRAP_TS: return exc_invalid_tss(regs, error_code); |
192 | case X86_TRAP_NP: return exc_segment_not_present(regs, error_code); |
193 | case X86_TRAP_SS: return exc_stack_segment(regs, error_code); |
194 | case X86_TRAP_GP: return exc_general_protection(regs, error_code); |
195 | case X86_TRAP_MF: return exc_coprocessor_error(regs); |
196 | case X86_TRAP_AC: return exc_alignment_check(regs, error_code); |
197 | case X86_TRAP_XF: return exc_simd_coprocessor_error(regs); |
198 | |
199 | #ifdef CONFIG_X86_MCE |
200 | case X86_TRAP_MC: return fred_exc_machine_check(regs); |
201 | #endif |
202 | #ifdef CONFIG_INTEL_TDX_GUEST |
203 | case X86_TRAP_VE: return exc_virtualization_exception(regs); |
204 | #endif |
205 | #ifdef CONFIG_X86_CET |
206 | case X86_TRAP_CP: return exc_control_protection(regs, error_code); |
207 | #endif |
208 | default: return fred_bad_type(regs, error_code); |
209 | } |
210 | |
211 | } |
212 | |
213 | static noinstr void fred_swexc(struct pt_regs *regs, unsigned long error_code) |
214 | { |
215 | switch (regs->fred_ss.vector) { |
216 | case X86_TRAP_BP: return exc_int3(regs); |
217 | case X86_TRAP_OF: return exc_overflow(regs); |
218 | default: return fred_bad_type(regs, error_code); |
219 | } |
220 | } |
221 | |
222 | __visible noinstr void fred_entry_from_user(struct pt_regs *regs) |
223 | { |
224 | unsigned long error_code = regs->orig_ax; |
225 | |
226 | /* Invalidate orig_ax so that syscall_get_nr() works correctly */ |
227 | regs->orig_ax = -1; |
228 | |
229 | switch (regs->fred_ss.type) { |
230 | case EVENT_TYPE_EXTINT: |
231 | return fred_extint(regs); |
232 | case EVENT_TYPE_NMI: |
233 | if (likely(regs->fred_ss.vector == X86_TRAP_NMI)) |
234 | return fred_exc_nmi(regs); |
235 | break; |
236 | case EVENT_TYPE_HWEXC: |
237 | return fred_hwexc(regs, error_code); |
238 | case EVENT_TYPE_SWINT: |
239 | return fred_intx(regs); |
240 | case EVENT_TYPE_PRIV_SWEXC: |
241 | if (likely(regs->fred_ss.vector == X86_TRAP_DB)) |
242 | return fred_exc_debug(regs); |
243 | break; |
244 | case EVENT_TYPE_SWEXC: |
245 | return fred_swexc(regs, error_code); |
246 | case EVENT_TYPE_OTHER: |
247 | return fred_other(regs); |
248 | default: break; |
249 | } |
250 | |
251 | return fred_bad_type(regs, error_code); |
252 | } |
253 | |
254 | __visible noinstr void fred_entry_from_kernel(struct pt_regs *regs) |
255 | { |
256 | unsigned long error_code = regs->orig_ax; |
257 | |
258 | /* Invalidate orig_ax so that syscall_get_nr() works correctly */ |
259 | regs->orig_ax = -1; |
260 | |
261 | switch (regs->fred_ss.type) { |
262 | case EVENT_TYPE_EXTINT: |
263 | return fred_extint(regs); |
264 | case EVENT_TYPE_NMI: |
265 | if (likely(regs->fred_ss.vector == X86_TRAP_NMI)) |
266 | return fred_exc_nmi(regs); |
267 | break; |
268 | case EVENT_TYPE_HWEXC: |
269 | return fred_hwexc(regs, error_code); |
270 | case EVENT_TYPE_PRIV_SWEXC: |
271 | if (likely(regs->fred_ss.vector == X86_TRAP_DB)) |
272 | return fred_exc_debug(regs); |
273 | break; |
274 | case EVENT_TYPE_SWEXC: |
275 | return fred_swexc(regs, error_code); |
276 | default: break; |
277 | } |
278 | |
279 | return fred_bad_type(regs, error_code); |
280 | } |
281 | |
282 | #if IS_ENABLED(CONFIG_KVM_INTEL) |
283 | __visible noinstr void __fred_entry_from_kvm(struct pt_regs *regs) |
284 | { |
285 | switch (regs->fred_ss.type) { |
286 | case EVENT_TYPE_EXTINT: |
287 | return fred_extint(regs); |
288 | case EVENT_TYPE_NMI: |
289 | return fred_exc_nmi(regs); |
290 | default: |
291 | WARN_ON_ONCE(1); |
292 | } |
293 | } |
294 | #endif |
295 | |