1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
2 | #include <linux/export.h> |
3 | #include <asm/asm-offsets.h> |
4 | #include <asm/cache.h> |
5 | #include <asm/code-patching-asm.h> |
6 | #include <asm/exception-64s.h> |
7 | #include <asm/kvm_asm.h> |
8 | #include <asm/kvm_book3s_asm.h> |
9 | #include <asm/mmu.h> |
10 | #include <asm/ppc_asm.h> |
11 | #include <asm/ptrace.h> |
12 | #include <asm/reg.h> |
13 | #include <asm/ultravisor-api.h> |
14 | |
15 | /* |
16 | * These are branched to from interrupt handlers in exception-64s.S which set |
17 | * IKVM_REAL or IKVM_VIRT, if HSTATE_IN_GUEST was found to be non-zero. |
18 | */ |
19 | |
20 | /* |
21 | * This is a hcall, so register convention is as |
22 | * Documentation/arch/powerpc/papr_hcalls.rst. |
23 | * |
24 | * This may also be a syscall from PR-KVM userspace that is to be |
25 | * reflected to the PR guest kernel, so registers may be set up for |
26 | * a system call rather than hcall. We don't currently clobber |
27 | * anything here, but the 0xc00 handler has already clobbered CTR |
28 | * and CR0, so PR-KVM can not support a guest kernel that preserves |
29 | * those registers across its system calls. |
30 | * |
31 | * The state of registers is as kvmppc_interrupt, except CFAR is not |
32 | * saved, R13 is not in SCRATCH0, and R10 does not contain the trap. |
33 | */ |
34 | .global kvmppc_hcall |
35 | .balign IFETCH_ALIGN_BYTES |
36 | kvmppc_hcall: |
37 | #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE |
38 | lbz r10,HSTATE_IN_GUEST(r13) |
39 | cmpwi r10,KVM_GUEST_MODE_HV_P9 |
40 | beq kvmppc_p9_exit_hcall |
41 | #endif |
42 | ld r10,PACA_EXGEN+EX_R13(r13) |
43 | SET_SCRATCH0(r10) |
44 | li r10,0xc00 |
45 | /* Now we look like kvmppc_interrupt */ |
46 | li r11,PACA_EXGEN |
47 | b .Lgot_save_area |
48 | |
49 | /* |
50 | * KVM interrupt entry occurs after GEN_INT_ENTRY runs, and follows that |
51 | * call convention: |
52 | * |
53 | * guest R9-R13, CTR, CFAR, PPR saved in PACA EX_xxx save area |
54 | * guest (H)DAR, (H)DSISR are also in the save area for relevant interrupts |
55 | * guest R13 also saved in SCRATCH0 |
56 | * R13 = PACA |
57 | * R11 = (H)SRR0 |
58 | * R12 = (H)SRR1 |
59 | * R9 = guest CR |
60 | * PPR is set to medium |
61 | * |
62 | * With the addition for KVM: |
63 | * R10 = trap vector |
64 | */ |
65 | .global kvmppc_interrupt |
66 | .balign IFETCH_ALIGN_BYTES |
67 | kvmppc_interrupt: |
68 | #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE |
69 | std r10,HSTATE_SCRATCH0(r13) |
70 | lbz r10,HSTATE_IN_GUEST(r13) |
71 | cmpwi r10,KVM_GUEST_MODE_HV_P9 |
72 | beq kvmppc_p9_exit_interrupt |
73 | ld r10,HSTATE_SCRATCH0(r13) |
74 | #endif |
75 | li r11,PACA_EXGEN |
76 | cmpdi r10,0x200 |
77 | bgt+ .Lgot_save_area |
78 | li r11,PACA_EXMC |
79 | beq .Lgot_save_area |
80 | li r11,PACA_EXNMI |
81 | .Lgot_save_area: |
82 | add r11,r11,r13 |
83 | BEGIN_FTR_SECTION |
84 | ld r12,EX_CFAR(r11) |
85 | std r12,HSTATE_CFAR(r13) |
86 | END_FTR_SECTION_IFSET(CPU_FTR_CFAR) |
87 | ld r12,EX_CTR(r11) |
88 | mtctr r12 |
89 | BEGIN_FTR_SECTION |
90 | ld r12,EX_PPR(r11) |
91 | std r12,HSTATE_PPR(r13) |
92 | END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) |
93 | ld r12,EX_R12(r11) |
94 | std r12,HSTATE_SCRATCH0(r13) |
95 | sldi r12,r9,32 |
96 | or r12,r12,r10 |
97 | ld r9,EX_R9(r11) |
98 | ld r10,EX_R10(r11) |
99 | ld r11,EX_R11(r11) |
100 | |
101 | /* |
102 | * Hcalls and other interrupts come here after normalising register |
103 | * contents and save locations: |
104 | * |
105 | * R12 = (guest CR << 32) | interrupt vector |
106 | * R13 = PACA |
107 | * guest R12 saved in shadow HSTATE_SCRATCH0 |
108 | * guest R13 saved in SPRN_SCRATCH0 |
109 | */ |
110 | std r9,HSTATE_SCRATCH2(r13) |
111 | lbz r9,HSTATE_IN_GUEST(r13) |
112 | cmpwi r9,KVM_GUEST_MODE_SKIP |
113 | beq- .Lmaybe_skip |
114 | .Lno_skip: |
115 | #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE |
116 | #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE |
117 | cmpwi r9,KVM_GUEST_MODE_GUEST |
118 | beq kvmppc_interrupt_pr |
119 | #endif |
120 | b kvmppc_interrupt_hv |
121 | #else |
122 | b kvmppc_interrupt_pr |
123 | #endif |
124 | |
125 | /* |
126 | * "Skip" interrupts are part of a trick KVM uses a with hash guests to load |
127 | * the faulting instruction in guest memory from the hypervisor without |
128 | * walking page tables. |
129 | * |
130 | * When the guest takes a fault that requires the hypervisor to load the |
131 | * instruction (e.g., MMIO emulation), KVM is running in real-mode with HV=1 |
132 | * and the guest MMU context loaded. It sets KVM_GUEST_MODE_SKIP, and sets |
133 | * MSR[DR]=1 while leaving MSR[IR]=0, so it continues to fetch HV instructions |
134 | * but loads and stores will access the guest context. This is used to load |
135 | * the faulting instruction using the faulting guest effective address. |
136 | * |
137 | * However the guest context may not be able to translate, or it may cause a |
138 | * machine check or other issue, which results in a fault in the host |
139 | * (even with KVM-HV). |
140 | * |
141 | * These faults come here because KVM_GUEST_MODE_SKIP was set, so if they |
142 | * are (or are likely) caused by that load, the instruction is skipped by |
143 | * just returning with the PC advanced +4, where it is noticed the load did |
144 | * not execute and it goes to the slow path which walks the page tables to |
145 | * read guest memory. |
146 | */ |
147 | .Lmaybe_skip: |
148 | cmpwi r12,BOOK3S_INTERRUPT_MACHINE_CHECK |
149 | beq 1f |
150 | cmpwi r12,BOOK3S_INTERRUPT_DATA_STORAGE |
151 | beq 1f |
152 | cmpwi r12,BOOK3S_INTERRUPT_DATA_SEGMENT |
153 | beq 1f |
154 | #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE |
155 | /* HSRR interrupts get 2 added to interrupt number */ |
156 | cmpwi r12,BOOK3S_INTERRUPT_H_DATA_STORAGE | 0x2 |
157 | beq 2f |
158 | #endif |
159 | b .Lno_skip |
160 | 1: mfspr r9,SPRN_SRR0 |
161 | addi r9,r9,4 |
162 | mtspr SPRN_SRR0,r9 |
163 | ld r12,HSTATE_SCRATCH0(r13) |
164 | ld r9,HSTATE_SCRATCH2(r13) |
165 | GET_SCRATCH0(r13) |
166 | RFI_TO_KERNEL |
167 | #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE |
168 | 2: mfspr r9,SPRN_HSRR0 |
169 | addi r9,r9,4 |
170 | mtspr SPRN_HSRR0,r9 |
171 | ld r12,HSTATE_SCRATCH0(r13) |
172 | ld r9,HSTATE_SCRATCH2(r13) |
173 | GET_SCRATCH0(r13) |
174 | HRFI_TO_KERNEL |
175 | #endif |
176 | |
177 | #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE |
178 | |
179 | /* Stack frame offsets for kvmppc_p9_enter_guest */ |
180 | #define SFS (144 + STACK_FRAME_MIN_SIZE) |
181 | #define STACK_SLOT_NVGPRS (SFS - 144) /* 18 gprs */ |
182 | |
183 | /* |
184 | * void kvmppc_p9_enter_guest(struct vcpu *vcpu); |
185 | * |
186 | * Enter the guest on a ISAv3.0 or later system. |
187 | */ |
188 | .balign IFETCH_ALIGN_BYTES |
189 | _GLOBAL(kvmppc_p9_enter_guest) |
190 | EXPORT_SYMBOL_GPL(kvmppc_p9_enter_guest) |
191 | mflr r0 |
192 | std r0,PPC_LR_STKOFF(r1) |
193 | stdu r1,-SFS(r1) |
194 | |
195 | std r1,HSTATE_HOST_R1(r13) |
196 | |
197 | mfcr r4 |
198 | stw r4,SFS+8(r1) |
199 | |
200 | reg = 14 |
201 | .rept 18 |
202 | std reg,STACK_SLOT_NVGPRS + ((reg - 14) * 8)(r1) |
203 | reg = reg + 1 |
204 | .endr |
205 | |
206 | ld r4,VCPU_LR(r3) |
207 | mtlr r4 |
208 | ld r4,VCPU_CTR(r3) |
209 | mtctr r4 |
210 | ld r4,VCPU_XER(r3) |
211 | mtspr SPRN_XER,r4 |
212 | |
213 | ld r1,VCPU_CR(r3) |
214 | |
215 | BEGIN_FTR_SECTION |
216 | ld r4,VCPU_CFAR(r3) |
217 | mtspr SPRN_CFAR,r4 |
218 | END_FTR_SECTION_IFSET(CPU_FTR_CFAR) |
219 | BEGIN_FTR_SECTION |
220 | ld r4,VCPU_PPR(r3) |
221 | mtspr SPRN_PPR,r4 |
222 | END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) |
223 | |
224 | reg = 4 |
225 | .rept 28 |
226 | ld reg,__VCPU_GPR(reg)(r3) |
227 | reg = reg + 1 |
228 | .endr |
229 | |
230 | ld r4,VCPU_KVM(r3) |
231 | lbz r4,KVM_SECURE_GUEST(r4) |
232 | cmpdi r4,0 |
233 | ld r4,VCPU_GPR(R4)(r3) |
234 | bne .Lret_to_ultra |
235 | |
236 | mtcr r1 |
237 | |
238 | ld r0,VCPU_GPR(R0)(r3) |
239 | ld r1,VCPU_GPR(R1)(r3) |
240 | ld r2,VCPU_GPR(R2)(r3) |
241 | ld r3,VCPU_GPR(R3)(r3) |
242 | |
243 | HRFI_TO_GUEST |
244 | b . |
245 | |
246 | /* |
247 | * Use UV_RETURN ultracall to return control back to the Ultravisor |
248 | * after processing an hypercall or interrupt that was forwarded |
249 | * (a.k.a. reflected) to the Hypervisor. |
250 | * |
251 | * All registers have already been reloaded except the ucall requires: |
252 | * R0 = hcall result |
253 | * R2 = SRR1, so UV can detect a synthesized interrupt (if any) |
254 | * R3 = UV_RETURN |
255 | */ |
256 | .Lret_to_ultra: |
257 | mtcr r1 |
258 | ld r1,VCPU_GPR(R1)(r3) |
259 | |
260 | ld r0,VCPU_GPR(R3)(r3) |
261 | mfspr r2,SPRN_SRR1 |
262 | LOAD_REG_IMMEDIATE(r3, UV_RETURN) |
263 | sc 2 |
264 | |
265 | /* |
266 | * kvmppc_p9_exit_hcall and kvmppc_p9_exit_interrupt are branched to from |
267 | * above if the interrupt was taken for a guest that was entered via |
268 | * kvmppc_p9_enter_guest(). |
269 | * |
270 | * The exit code recovers the host stack and vcpu pointer, saves all guest GPRs |
271 | * and CR, LR, XER as well as guest MSR and NIA into the VCPU, then re- |
272 | * establishes the host stack and registers to return from the |
273 | * kvmppc_p9_enter_guest() function, which saves CTR and other guest registers |
274 | * (SPRs and FP, VEC, etc). |
275 | */ |
276 | .balign IFETCH_ALIGN_BYTES |
277 | kvmppc_p9_exit_hcall: |
278 | mfspr r11,SPRN_SRR0 |
279 | mfspr r12,SPRN_SRR1 |
280 | li r10,0xc00 |
281 | std r10,HSTATE_SCRATCH0(r13) |
282 | |
283 | .balign IFETCH_ALIGN_BYTES |
284 | kvmppc_p9_exit_interrupt: |
285 | /* |
286 | * If set to KVM_GUEST_MODE_HV_P9 but we're still in the |
287 | * hypervisor, that means we can't return from the entry stack. |
288 | */ |
289 | rldicl. r10,r12,64-MSR_HV_LG,63 |
290 | bne- kvmppc_p9_bad_interrupt |
291 | |
292 | std r1,HSTATE_SCRATCH1(r13) |
293 | std r3,HSTATE_SCRATCH2(r13) |
294 | ld r1,HSTATE_HOST_R1(r13) |
295 | ld r3,HSTATE_KVM_VCPU(r13) |
296 | |
297 | std r9,VCPU_CR(r3) |
298 | |
299 | 1: |
300 | std r11,VCPU_PC(r3) |
301 | std r12,VCPU_MSR(r3) |
302 | |
303 | reg = 14 |
304 | .rept 18 |
305 | std reg,__VCPU_GPR(reg)(r3) |
306 | reg = reg + 1 |
307 | .endr |
308 | |
309 | /* r1, r3, r9-r13 are saved to vcpu by C code */ |
310 | std r0,VCPU_GPR(R0)(r3) |
311 | std r2,VCPU_GPR(R2)(r3) |
312 | reg = 4 |
313 | .rept 5 |
314 | std reg,__VCPU_GPR(reg)(r3) |
315 | reg = reg + 1 |
316 | .endr |
317 | |
318 | LOAD_PACA_TOC() |
319 | |
320 | mflr r4 |
321 | std r4,VCPU_LR(r3) |
322 | mfspr r4,SPRN_XER |
323 | std r4,VCPU_XER(r3) |
324 | |
325 | reg = 14 |
326 | .rept 18 |
327 | ld reg,STACK_SLOT_NVGPRS + ((reg - 14) * 8)(r1) |
328 | reg = reg + 1 |
329 | .endr |
330 | |
331 | lwz r4,SFS+8(r1) |
332 | mtcr r4 |
333 | |
334 | /* |
335 | * Flush the link stack here, before executing the first blr on the |
336 | * way out of the guest. |
337 | * |
338 | * The link stack won't match coming out of the guest anyway so the |
339 | * only cost is the flush itself. The call clobbers r0. |
340 | */ |
341 | 1: nop |
342 | patch_site 1b patch__call_kvm_flush_link_stack_p9 |
343 | |
344 | addi r1,r1,SFS |
345 | ld r0,PPC_LR_STKOFF(r1) |
346 | mtlr r0 |
347 | blr |
348 | |
349 | /* |
350 | * Took an interrupt somewhere right before HRFID to guest, so registers are |
351 | * in a bad way. Return things hopefully enough to run host virtual code and |
352 | * run the Linux interrupt handler (SRESET or MCE) to print something useful. |
353 | * |
354 | * We could be really clever and save all host registers in known locations |
355 | * before setting HSTATE_IN_GUEST, then restoring them all here, and setting |
356 | * return address to a fixup that sets them up again. But that's a lot of |
357 | * effort for a small bit of code. Lots of other things to do first. |
358 | */ |
359 | kvmppc_p9_bad_interrupt: |
360 | BEGIN_MMU_FTR_SECTION |
361 | /* |
362 | * Hash host doesn't try to recover MMU (requires host SLB reload) |
363 | */ |
364 | b . |
365 | END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_RADIX) |
366 | /* |
367 | * Clean up guest registers to give host a chance to run. |
368 | */ |
369 | li r10,0 |
370 | mtspr SPRN_AMR,r10 |
371 | mtspr SPRN_IAMR,r10 |
372 | mtspr SPRN_CIABR,r10 |
373 | mtspr SPRN_DAWRX0,r10 |
374 | BEGIN_FTR_SECTION |
375 | mtspr SPRN_DAWRX1,r10 |
376 | END_FTR_SECTION_IFSET(CPU_FTR_DAWR1) |
377 | |
378 | /* |
379 | * Switch to host MMU mode (don't have the real host PID but we aren't |
380 | * going back to userspace). |
381 | */ |
382 | hwsync |
383 | isync |
384 | |
385 | mtspr SPRN_PID,r10 |
386 | |
387 | ld r10, HSTATE_KVM_VCPU(r13) |
388 | ld r10, VCPU_KVM(r10) |
389 | lwz r10, KVM_HOST_LPID(r10) |
390 | mtspr SPRN_LPID,r10 |
391 | |
392 | ld r10, HSTATE_KVM_VCPU(r13) |
393 | ld r10, VCPU_KVM(r10) |
394 | ld r10, KVM_HOST_LPCR(r10) |
395 | mtspr SPRN_LPCR,r10 |
396 | |
397 | isync |
398 | |
399 | /* |
400 | * Set GUEST_MODE_NONE so the handler won't branch to KVM, and clear |
401 | * MSR_RI in r12 ([H]SRR1) so the handler won't try to return. |
402 | */ |
403 | li r10,KVM_GUEST_MODE_NONE |
404 | stb r10,HSTATE_IN_GUEST(r13) |
405 | li r10,MSR_RI |
406 | andc r12,r12,r10 |
407 | |
408 | /* |
409 | * Go back to interrupt handler. MCE and SRESET have their specific |
410 | * PACA save area so they should be used directly. They set up their |
411 | * own stack. The other handlers all use EXGEN. They will use the |
412 | * guest r1 if it looks like a kernel stack, so just load the |
413 | * emergency stack and go to program check for all other interrupts. |
414 | */ |
415 | ld r10,HSTATE_SCRATCH0(r13) |
416 | cmpwi r10,BOOK3S_INTERRUPT_MACHINE_CHECK |
417 | beq .Lcall_machine_check_common |
418 | |
419 | cmpwi r10,BOOK3S_INTERRUPT_SYSTEM_RESET |
420 | beq .Lcall_system_reset_common |
421 | |
422 | b . |
423 | |
424 | .Lcall_machine_check_common: |
425 | b machine_check_common |
426 | |
427 | .Lcall_system_reset_common: |
428 | b system_reset_common |
429 | #endif |
430 | |