1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
2 | /* |
3 | * |
4 | * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> |
5 | * |
6 | * Derived from book3s_rmhandlers.S and other files, which are: |
7 | * |
8 | * Copyright SUSE Linux Products GmbH 2009 |
9 | * |
10 | * Authors: Alexander Graf <agraf@suse.de> |
11 | */ |
12 | |
13 | #include <linux/export.h> |
14 | #include <linux/linkage.h> |
15 | #include <linux/objtool.h> |
16 | #include <asm/ppc_asm.h> |
17 | #include <asm/code-patching-asm.h> |
18 | #include <asm/kvm_asm.h> |
19 | #include <asm/reg.h> |
20 | #include <asm/mmu.h> |
21 | #include <asm/page.h> |
22 | #include <asm/ptrace.h> |
23 | #include <asm/hvcall.h> |
24 | #include <asm/asm-offsets.h> |
25 | #include <asm/exception-64s.h> |
26 | #include <asm/kvm_book3s_asm.h> |
27 | #include <asm/book3s/64/mmu-hash.h> |
28 | #include <asm/tm.h> |
29 | #include <asm/opal.h> |
30 | #include <asm/thread_info.h> |
31 | #include <asm/asm-compat.h> |
32 | #include <asm/feature-fixups.h> |
33 | #include <asm/cpuidle.h> |
34 | |
35 | /* Values in HSTATE_NAPPING(r13) */ |
36 | #define NAPPING_CEDE 1 |
37 | #define NAPPING_NOVCPU 2 |
38 | #define NAPPING_UNSPLIT 3 |
39 | |
40 | /* Stack frame offsets for kvmppc_hv_entry */ |
41 | #define SFS 160 |
42 | #define STACK_SLOT_TRAP (SFS-4) |
43 | #define STACK_SLOT_TID (SFS-16) |
44 | #define STACK_SLOT_PSSCR (SFS-24) |
45 | #define STACK_SLOT_PID (SFS-32) |
46 | #define STACK_SLOT_IAMR (SFS-40) |
47 | #define STACK_SLOT_CIABR (SFS-48) |
48 | #define STACK_SLOT_DAWR0 (SFS-56) |
49 | #define STACK_SLOT_DAWRX0 (SFS-64) |
50 | #define STACK_SLOT_HFSCR (SFS-72) |
51 | #define STACK_SLOT_AMR (SFS-80) |
52 | #define STACK_SLOT_UAMOR (SFS-88) |
53 | #define STACK_SLOT_FSCR (SFS-96) |
54 | |
55 | /* |
56 | * Use the last LPID (all implemented LPID bits = 1) for partition switching. |
57 | * This is reserved in the LPID allocator. POWER7 only implements 0x3ff, but |
58 | * we write 0xfff into the LPID SPR anyway, which seems to work and just |
59 | * ignores the top bits. |
60 | */ |
61 | #define LPID_RSVD 0xfff |
62 | |
63 | /* |
64 | * Call kvmppc_hv_entry in real mode. |
65 | * Must be called with interrupts hard-disabled. |
66 | * |
67 | * Input Registers: |
68 | * |
69 | * LR = return address to continue at after eventually re-enabling MMU |
70 | */ |
71 | _GLOBAL_TOC(kvmppc_hv_entry_trampoline) |
72 | mflr r0 |
73 | std r0, PPC_LR_STKOFF(r1) |
74 | stdu r1, -112(r1) |
75 | mfmsr r10 |
76 | std r10, HSTATE_HOST_MSR(r13) |
77 | LOAD_REG_ADDR(r5, kvmppc_call_hv_entry) |
78 | li r0,MSR_RI |
79 | andc r0,r10,r0 |
80 | li r6,MSR_IR | MSR_DR |
81 | andc r6,r10,r6 |
82 | mtmsrd r0,1 /* clear RI in MSR */ |
83 | mtsrr0 r5 |
84 | mtsrr1 r6 |
85 | RFI_TO_KERNEL |
86 | |
87 | kvmppc_call_hv_entry: |
88 | ld r4, HSTATE_KVM_VCPU(r13) |
89 | bl kvmppc_hv_entry |
90 | |
91 | /* Back from guest - restore host state and return to caller */ |
92 | |
93 | BEGIN_FTR_SECTION |
94 | /* Restore host DABR and DABRX */ |
95 | ld r5,HSTATE_DABR(r13) |
96 | li r6,7 |
97 | mtspr SPRN_DABR,r5 |
98 | mtspr SPRN_DABRX,r6 |
99 | END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) |
100 | |
101 | /* Restore SPRG3 */ |
102 | ld r3,PACA_SPRG_VDSO(r13) |
103 | mtspr SPRN_SPRG_VDSO_WRITE,r3 |
104 | |
105 | /* Reload the host's PMU registers */ |
106 | bl kvmhv_load_host_pmu |
107 | |
108 | /* |
109 | * Reload DEC. HDEC interrupts were disabled when |
110 | * we reloaded the host's LPCR value. |
111 | */ |
112 | ld r3, HSTATE_DECEXP(r13) |
113 | mftb r4 |
114 | subf r4, r4, r3 |
115 | mtspr SPRN_DEC, r4 |
116 | |
117 | /* hwthread_req may have got set by cede or no vcpu, so clear it */ |
118 | li r0, 0 |
119 | stb r0, HSTATE_HWTHREAD_REQ(r13) |
120 | |
121 | /* |
122 | * For external interrupts we need to call the Linux |
123 | * handler to process the interrupt. We do that by jumping |
124 | * to absolute address 0x500 for external interrupts. |
125 | * The [h]rfid at the end of the handler will return to |
126 | * the book3s_hv_interrupts.S code. For other interrupts |
127 | * we do the rfid to get back to the book3s_hv_interrupts.S |
128 | * code here. |
129 | */ |
130 | ld r8, 112+PPC_LR_STKOFF(r1) |
131 | addi r1, r1, 112 |
132 | ld r7, HSTATE_HOST_MSR(r13) |
133 | |
134 | /* Return the trap number on this thread as the return value */ |
135 | mr r3, r12 |
136 | |
137 | /* RFI into the highmem handler */ |
138 | mfmsr r6 |
139 | li r0, MSR_RI |
140 | andc r6, r6, r0 |
141 | mtmsrd r6, 1 /* Clear RI in MSR */ |
142 | mtsrr0 r8 |
143 | mtsrr1 r7 |
144 | RFI_TO_KERNEL |
145 | |
146 | kvmppc_primary_no_guest: |
147 | /* We handle this much like a ceded vcpu */ |
148 | /* put the HDEC into the DEC, since HDEC interrupts don't wake us */ |
149 | /* HDEC may be larger than DEC for arch >= v3.00, but since the */ |
150 | /* HDEC value came from DEC in the first place, it will fit */ |
151 | mfspr r3, SPRN_HDEC |
152 | mtspr SPRN_DEC, r3 |
153 | /* |
154 | * Make sure the primary has finished the MMU switch. |
155 | * We should never get here on a secondary thread, but |
156 | * check it for robustness' sake. |
157 | */ |
158 | ld r5, HSTATE_KVM_VCORE(r13) |
159 | 65: lbz r0, VCORE_IN_GUEST(r5) |
160 | cmpwi r0, 0 |
161 | beq 65b |
162 | /* Set LPCR. */ |
163 | ld r8,VCORE_LPCR(r5) |
164 | mtspr SPRN_LPCR,r8 |
165 | isync |
166 | /* set our bit in napping_threads */ |
167 | ld r5, HSTATE_KVM_VCORE(r13) |
168 | lbz r7, HSTATE_PTID(r13) |
169 | li r0, 1 |
170 | sld r0, r0, r7 |
171 | addi r6, r5, VCORE_NAPPING_THREADS |
172 | 1: lwarx r3, 0, r6 |
173 | or r3, r3, r0 |
174 | stwcx. r3, 0, r6 |
175 | bne 1b |
176 | /* order napping_threads update vs testing entry_exit_map */ |
177 | isync |
178 | li r12, 0 |
179 | lwz r7, VCORE_ENTRY_EXIT(r5) |
180 | cmpwi r7, 0x100 |
181 | bge kvm_novcpu_exit /* another thread already exiting */ |
182 | li r3, NAPPING_NOVCPU |
183 | stb r3, HSTATE_NAPPING(r13) |
184 | |
185 | li r3, 0 /* Don't wake on privileged (OS) doorbell */ |
186 | b kvm_do_nap |
187 | |
188 | /* |
189 | * kvm_novcpu_wakeup |
190 | * Entered from kvm_start_guest if kvm_hstate.napping is set |
191 | * to NAPPING_NOVCPU |
192 | * r2 = kernel TOC |
193 | * r13 = paca |
194 | */ |
195 | kvm_novcpu_wakeup: |
196 | ld r1, HSTATE_HOST_R1(r13) |
197 | ld r5, HSTATE_KVM_VCORE(r13) |
198 | li r0, 0 |
199 | stb r0, HSTATE_NAPPING(r13) |
200 | |
201 | /* check the wake reason */ |
202 | bl kvmppc_check_wake_reason |
203 | |
204 | /* |
205 | * Restore volatile registers since we could have called |
206 | * a C routine in kvmppc_check_wake_reason. |
207 | * r5 = VCORE |
208 | */ |
209 | ld r5, HSTATE_KVM_VCORE(r13) |
210 | |
211 | /* see if any other thread is already exiting */ |
212 | lwz r0, VCORE_ENTRY_EXIT(r5) |
213 | cmpwi r0, 0x100 |
214 | bge kvm_novcpu_exit |
215 | |
216 | /* clear our bit in napping_threads */ |
217 | lbz r7, HSTATE_PTID(r13) |
218 | li r0, 1 |
219 | sld r0, r0, r7 |
220 | addi r6, r5, VCORE_NAPPING_THREADS |
221 | 4: lwarx r7, 0, r6 |
222 | andc r7, r7, r0 |
223 | stwcx. r7, 0, r6 |
224 | bne 4b |
225 | |
226 | /* See if the wake reason means we need to exit */ |
227 | cmpdi r3, 0 |
228 | bge kvm_novcpu_exit |
229 | |
230 | /* See if our timeslice has expired (HDEC is negative) */ |
231 | mfspr r0, SPRN_HDEC |
232 | extsw r0, r0 |
233 | li r12, BOOK3S_INTERRUPT_HV_DECREMENTER |
234 | cmpdi r0, 0 |
235 | blt kvm_novcpu_exit |
236 | |
237 | /* Got an IPI but other vcpus aren't yet exiting, must be a latecomer */ |
238 | ld r4, HSTATE_KVM_VCPU(r13) |
239 | cmpdi r4, 0 |
240 | beq kvmppc_primary_no_guest |
241 | |
242 | #ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING |
243 | addi r3, r4, VCPU_TB_RMENTRY |
244 | bl kvmhv_start_timing |
245 | #endif |
246 | b kvmppc_got_guest |
247 | |
248 | kvm_novcpu_exit: |
249 | #ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING |
250 | ld r4, HSTATE_KVM_VCPU(r13) |
251 | cmpdi r4, 0 |
252 | beq 13f |
253 | addi r3, r4, VCPU_TB_RMEXIT |
254 | bl kvmhv_accumulate_time |
255 | #endif |
256 | 13: mr r3, r12 |
257 | stw r12, STACK_SLOT_TRAP(r1) |
258 | bl kvmhv_commence_exit |
259 | nop |
260 | b kvmhv_switch_to_host |
261 | |
262 | /* |
263 | * We come in here when wakened from Linux offline idle code. |
264 | * Relocation is off |
265 | * r3 contains the SRR1 wakeup value, SRR1 is trashed. |
266 | */ |
267 | _GLOBAL(idle_kvm_start_guest) |
268 | mfcr r5 |
269 | mflr r0 |
270 | std r5, 8(r1) // Save CR in caller's frame |
271 | std r0, 16(r1) // Save LR in caller's frame |
272 | // Create frame on emergency stack |
273 | ld r4, PACAEMERGSP(r13) |
274 | stdu r1, -SWITCH_FRAME_SIZE(r4) |
275 | // Switch to new frame on emergency stack |
276 | mr r1, r4 |
277 | std r3, 32(r1) // Save SRR1 wakeup value |
278 | SAVE_NVGPRS(r1) |
279 | |
280 | /* |
281 | * Could avoid this and pass it through in r3. For now, |
282 | * code expects it to be in SRR1. |
283 | */ |
284 | mtspr SPRN_SRR1,r3 |
285 | |
286 | li r0,0 |
287 | stb r0,PACA_FTRACE_ENABLED(r13) |
288 | |
289 | li r0,KVM_HWTHREAD_IN_KVM |
290 | stb r0,HSTATE_HWTHREAD_STATE(r13) |
291 | |
292 | /* kvm cede / napping does not come through here */ |
293 | lbz r0,HSTATE_NAPPING(r13) |
294 | twnei r0,0 |
295 | |
296 | b 1f |
297 | |
298 | kvm_unsplit_wakeup: |
299 | li r0, 0 |
300 | stb r0, HSTATE_NAPPING(r13) |
301 | |
302 | 1: |
303 | |
304 | /* |
305 | * We weren't napping due to cede, so this must be a secondary |
306 | * thread being woken up to run a guest, or being woken up due |
307 | * to a stray IPI. (Or due to some machine check or hypervisor |
308 | * maintenance interrupt while the core is in KVM.) |
309 | */ |
310 | |
311 | /* Check the wake reason in SRR1 to see why we got here */ |
312 | bl kvmppc_check_wake_reason |
313 | /* |
314 | * kvmppc_check_wake_reason could invoke a C routine, but we |
315 | * have no volatile registers to restore when we return. |
316 | */ |
317 | |
318 | cmpdi r3, 0 |
319 | bge kvm_no_guest |
320 | |
321 | /* get vcore pointer, NULL if we have nothing to run */ |
322 | ld r5,HSTATE_KVM_VCORE(r13) |
323 | cmpdi r5,0 |
324 | /* if we have no vcore to run, go back to sleep */ |
325 | beq kvm_no_guest |
326 | |
327 | kvm_secondary_got_guest: |
328 | |
329 | // About to go to guest, clear saved SRR1 |
330 | li r0, 0 |
331 | std r0, 32(r1) |
332 | |
333 | /* Set HSTATE_DSCR(r13) to something sensible */ |
334 | ld r6, PACA_DSCR_DEFAULT(r13) |
335 | std r6, HSTATE_DSCR(r13) |
336 | |
337 | /* On thread 0 of a subcore, set HDEC to max */ |
338 | lbz r4, HSTATE_PTID(r13) |
339 | cmpwi r4, 0 |
340 | bne 63f |
341 | lis r6,0x7fff /* MAX_INT@h */ |
342 | mtspr SPRN_HDEC, r6 |
343 | /* and set per-LPAR registers, if doing dynamic micro-threading */ |
344 | ld r6, HSTATE_SPLIT_MODE(r13) |
345 | cmpdi r6, 0 |
346 | beq 63f |
347 | ld r0, KVM_SPLIT_RPR(r6) |
348 | mtspr SPRN_RPR, r0 |
349 | ld r0, KVM_SPLIT_PMMAR(r6) |
350 | mtspr SPRN_PMMAR, r0 |
351 | ld r0, KVM_SPLIT_LDBAR(r6) |
352 | mtspr SPRN_LDBAR, r0 |
353 | isync |
354 | 63: |
355 | /* Order load of vcpu after load of vcore */ |
356 | lwsync |
357 | ld r4, HSTATE_KVM_VCPU(r13) |
358 | bl kvmppc_hv_entry |
359 | |
360 | /* Back from the guest, go back to nap */ |
361 | /* Clear our vcpu and vcore pointers so we don't come back in early */ |
362 | li r0, 0 |
363 | std r0, HSTATE_KVM_VCPU(r13) |
364 | /* |
365 | * Once we clear HSTATE_KVM_VCORE(r13), the code in |
366 | * kvmppc_run_core() is going to assume that all our vcpu |
367 | * state is visible in memory. This lwsync makes sure |
368 | * that that is true. |
369 | */ |
370 | lwsync |
371 | std r0, HSTATE_KVM_VCORE(r13) |
372 | |
373 | /* |
374 | * All secondaries exiting guest will fall through this path. |
375 | * Before proceeding, just check for HMI interrupt and |
376 | * invoke opal hmi handler. By now we are sure that the |
377 | * primary thread on this core/subcore has already made partition |
378 | * switch/TB resync and we are good to call opal hmi handler. |
379 | */ |
380 | cmpwi r12, BOOK3S_INTERRUPT_HMI |
381 | bne kvm_no_guest |
382 | |
383 | li r3,0 /* NULL argument */ |
384 | bl CFUNC(hmi_exception_realmode) |
385 | /* |
386 | * At this point we have finished executing in the guest. |
387 | * We need to wait for hwthread_req to become zero, since |
388 | * we may not turn on the MMU while hwthread_req is non-zero. |
389 | * While waiting we also need to check if we get given a vcpu to run. |
390 | */ |
391 | kvm_no_guest: |
392 | lbz r3, HSTATE_HWTHREAD_REQ(r13) |
393 | cmpwi r3, 0 |
394 | bne 53f |
395 | HMT_MEDIUM |
396 | li r0, KVM_HWTHREAD_IN_KERNEL |
397 | stb r0, HSTATE_HWTHREAD_STATE(r13) |
398 | /* need to recheck hwthread_req after a barrier, to avoid race */ |
399 | sync |
400 | lbz r3, HSTATE_HWTHREAD_REQ(r13) |
401 | cmpwi r3, 0 |
402 | bne 54f |
403 | |
404 | /* |
405 | * Jump to idle_return_gpr_loss, which returns to the |
406 | * idle_kvm_start_guest caller. |
407 | */ |
408 | li r3, LPCR_PECE0 |
409 | mfspr r4, SPRN_LPCR |
410 | rlwimi r4, r3, 0, LPCR_PECE0 | LPCR_PECE1 |
411 | mtspr SPRN_LPCR, r4 |
412 | // Return SRR1 wakeup value, or 0 if we went into the guest |
413 | ld r3, 32(r1) |
414 | REST_NVGPRS(r1) |
415 | ld r1, 0(r1) // Switch back to caller stack |
416 | ld r0, 16(r1) // Reload LR |
417 | ld r5, 8(r1) // Reload CR |
418 | mtlr r0 |
419 | mtcr r5 |
420 | blr |
421 | |
422 | 53: |
423 | HMT_LOW |
424 | ld r5, HSTATE_KVM_VCORE(r13) |
425 | cmpdi r5, 0 |
426 | bne 60f |
427 | ld r3, HSTATE_SPLIT_MODE(r13) |
428 | cmpdi r3, 0 |
429 | beq kvm_no_guest |
430 | lbz r0, KVM_SPLIT_DO_NAP(r3) |
431 | cmpwi r0, 0 |
432 | beq kvm_no_guest |
433 | HMT_MEDIUM |
434 | b kvm_unsplit_nap |
435 | 60: HMT_MEDIUM |
436 | b kvm_secondary_got_guest |
437 | |
438 | 54: li r0, KVM_HWTHREAD_IN_KVM |
439 | stb r0, HSTATE_HWTHREAD_STATE(r13) |
440 | b kvm_no_guest |
441 | |
442 | /* |
443 | * Here the primary thread is trying to return the core to |
444 | * whole-core mode, so we need to nap. |
445 | */ |
446 | kvm_unsplit_nap: |
447 | /* |
448 | * When secondaries are napping in kvm_unsplit_nap() with |
449 | * hwthread_req = 1, HMI goes ignored even though subcores are |
450 | * already exited the guest. Hence HMI keeps waking up secondaries |
451 | * from nap in a loop and secondaries always go back to nap since |
452 | * no vcore is assigned to them. This makes impossible for primary |
453 | * thread to get hold of secondary threads resulting into a soft |
454 | * lockup in KVM path. |
455 | * |
456 | * Let us check if HMI is pending and handle it before we go to nap. |
457 | */ |
458 | cmpwi r12, BOOK3S_INTERRUPT_HMI |
459 | bne 55f |
460 | li r3, 0 /* NULL argument */ |
461 | bl CFUNC(hmi_exception_realmode) |
462 | 55: |
463 | /* |
464 | * Ensure that secondary doesn't nap when it has |
465 | * its vcore pointer set. |
466 | */ |
467 | sync /* matches smp_mb() before setting split_info.do_nap */ |
468 | ld r0, HSTATE_KVM_VCORE(r13) |
469 | cmpdi r0, 0 |
470 | bne kvm_no_guest |
471 | /* clear any pending message */ |
472 | BEGIN_FTR_SECTION |
473 | lis r6, (PPC_DBELL_SERVER << (63-36))@h |
474 | PPC_MSGCLR(6) |
475 | END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) |
476 | /* Set kvm_split_mode.napped[tid] = 1 */ |
477 | ld r3, HSTATE_SPLIT_MODE(r13) |
478 | li r0, 1 |
479 | lhz r4, PACAPACAINDEX(r13) |
480 | clrldi r4, r4, 61 /* micro-threading => P8 => 8 threads/core */ |
481 | addi r4, r4, KVM_SPLIT_NAPPED |
482 | stbx r0, r3, r4 |
483 | /* Check the do_nap flag again after setting napped[] */ |
484 | sync |
485 | lbz r0, KVM_SPLIT_DO_NAP(r3) |
486 | cmpwi r0, 0 |
487 | beq 57f |
488 | li r3, NAPPING_UNSPLIT |
489 | stb r3, HSTATE_NAPPING(r13) |
490 | li r3, (LPCR_PECEDH | LPCR_PECE0) >> 4 |
491 | mfspr r5, SPRN_LPCR |
492 | rlwimi r5, r3, 4, (LPCR_PECEDP | LPCR_PECEDH | LPCR_PECE0 | LPCR_PECE1) |
493 | b kvm_nap_sequence |
494 | |
495 | 57: li r0, 0 |
496 | stbx r0, r3, r4 |
497 | b kvm_no_guest |
498 | |
499 | /****************************************************************************** |
500 | * * |
501 | * Entry code * |
502 | * * |
503 | *****************************************************************************/ |
504 | |
505 | SYM_CODE_START_LOCAL(kvmppc_hv_entry) |
506 | |
507 | /* Required state: |
508 | * |
509 | * R4 = vcpu pointer (or NULL) |
510 | * MSR = ~IR|DR |
511 | * R13 = PACA |
512 | * R1 = host R1 |
513 | * R2 = TOC |
514 | * all other volatile GPRS = free |
515 | * Does not preserve non-volatile GPRs or CR fields |
516 | */ |
517 | mflr r0 |
518 | std r0, PPC_LR_STKOFF(r1) |
519 | stdu r1, -SFS(r1) |
520 | |
521 | /* Save R1 in the PACA */ |
522 | std r1, HSTATE_HOST_R1(r13) |
523 | |
524 | li r6, KVM_GUEST_MODE_HOST_HV |
525 | stb r6, HSTATE_IN_GUEST(r13) |
526 | |
527 | #ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING |
528 | /* Store initial timestamp */ |
529 | cmpdi r4, 0 |
530 | beq 1f |
531 | addi r3, r4, VCPU_TB_RMENTRY |
532 | bl kvmhv_start_timing |
533 | 1: |
534 | #endif |
535 | |
536 | ld r5, HSTATE_KVM_VCORE(r13) |
537 | ld r9, VCORE_KVM(r5) /* pointer to struct kvm */ |
538 | |
539 | /* |
540 | * POWER7/POWER8 host -> guest partition switch code. |
541 | * We don't have to lock against concurrent tlbies, |
542 | * but we do have to coordinate across hardware threads. |
543 | */ |
544 | /* Set bit in entry map iff exit map is zero. */ |
545 | li r7, 1 |
546 | lbz r6, HSTATE_PTID(r13) |
547 | sld r7, r7, r6 |
548 | addi r8, r5, VCORE_ENTRY_EXIT |
549 | 21: lwarx r3, 0, r8 |
550 | cmpwi r3, 0x100 /* any threads starting to exit? */ |
551 | bge secondary_too_late /* if so we're too late to the party */ |
552 | or r3, r3, r7 |
553 | stwcx. r3, 0, r8 |
554 | bne 21b |
555 | |
556 | /* Primary thread switches to guest partition. */ |
557 | cmpwi r6,0 |
558 | bne 10f |
559 | |
560 | lwz r7,KVM_LPID(r9) |
561 | ld r6,KVM_SDR1(r9) |
562 | li r0,LPID_RSVD /* switch to reserved LPID */ |
563 | mtspr SPRN_LPID,r0 |
564 | ptesync |
565 | mtspr SPRN_SDR1,r6 /* switch to partition page table */ |
566 | mtspr SPRN_LPID,r7 |
567 | isync |
568 | |
569 | /* See if we need to flush the TLB. */ |
570 | mr r3, r9 /* kvm pointer */ |
571 | lhz r4, PACAPACAINDEX(r13) /* physical cpu number */ |
572 | li r5, 0 /* nested vcpu pointer */ |
573 | bl kvmppc_check_need_tlb_flush |
574 | nop |
575 | ld r5, HSTATE_KVM_VCORE(r13) |
576 | |
577 | /* Add timebase offset onto timebase */ |
578 | 22: ld r8,VCORE_TB_OFFSET(r5) |
579 | cmpdi r8,0 |
580 | beq 37f |
581 | std r8, VCORE_TB_OFFSET_APPL(r5) |
582 | mftb r6 /* current host timebase */ |
583 | add r8,r8,r6 |
584 | mtspr SPRN_TBU40,r8 /* update upper 40 bits */ |
585 | mftb r7 /* check if lower 24 bits overflowed */ |
586 | clrldi r6,r6,40 |
587 | clrldi r7,r7,40 |
588 | cmpld r7,r6 |
589 | bge 37f |
590 | addis r8,r8,0x100 /* if so, increment upper 40 bits */ |
591 | mtspr SPRN_TBU40,r8 |
592 | |
593 | /* Load guest PCR value to select appropriate compat mode */ |
594 | 37: ld r7, VCORE_PCR(r5) |
595 | LOAD_REG_IMMEDIATE(r6, PCR_MASK) |
596 | cmpld r7, r6 |
597 | beq 38f |
598 | or r7, r7, r6 |
599 | mtspr SPRN_PCR, r7 |
600 | 38: |
601 | |
602 | BEGIN_FTR_SECTION |
603 | /* DPDES and VTB are shared between threads */ |
604 | ld r8, VCORE_DPDES(r5) |
605 | ld r7, VCORE_VTB(r5) |
606 | mtspr SPRN_DPDES, r8 |
607 | mtspr SPRN_VTB, r7 |
608 | END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) |
609 | |
610 | /* Mark the subcore state as inside guest */ |
611 | bl kvmppc_subcore_enter_guest |
612 | nop |
613 | ld r5, HSTATE_KVM_VCORE(r13) |
614 | ld r4, HSTATE_KVM_VCPU(r13) |
615 | li r0,1 |
616 | stb r0,VCORE_IN_GUEST(r5) /* signal secondaries to continue */ |
617 | |
618 | /* Do we have a guest vcpu to run? */ |
619 | 10: cmpdi r4, 0 |
620 | beq kvmppc_primary_no_guest |
621 | kvmppc_got_guest: |
622 | /* Increment yield count if they have a VPA */ |
623 | ld r3, VCPU_VPA(r4) |
624 | cmpdi r3, 0 |
625 | beq 25f |
626 | li r6, LPPACA_YIELDCOUNT |
627 | LWZX_BE r5, r3, r6 |
628 | addi r5, r5, 1 |
629 | STWX_BE r5, r3, r6 |
630 | li r6, 1 |
631 | stb r6, VCPU_VPA_DIRTY(r4) |
632 | 25: |
633 | |
634 | /* Save purr/spurr */ |
635 | mfspr r5,SPRN_PURR |
636 | mfspr r6,SPRN_SPURR |
637 | std r5,HSTATE_PURR(r13) |
638 | std r6,HSTATE_SPURR(r13) |
639 | ld r7,VCPU_PURR(r4) |
640 | ld r8,VCPU_SPURR(r4) |
641 | mtspr SPRN_PURR,r7 |
642 | mtspr SPRN_SPURR,r8 |
643 | |
644 | /* Save host values of some registers */ |
645 | BEGIN_FTR_SECTION |
646 | mfspr r5, SPRN_CIABR |
647 | mfspr r6, SPRN_DAWR0 |
648 | mfspr r7, SPRN_DAWRX0 |
649 | mfspr r8, SPRN_IAMR |
650 | std r5, STACK_SLOT_CIABR(r1) |
651 | std r6, STACK_SLOT_DAWR0(r1) |
652 | std r7, STACK_SLOT_DAWRX0(r1) |
653 | std r8, STACK_SLOT_IAMR(r1) |
654 | mfspr r5, SPRN_FSCR |
655 | std r5, STACK_SLOT_FSCR(r1) |
656 | END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) |
657 | |
658 | mfspr r5, SPRN_AMR |
659 | std r5, STACK_SLOT_AMR(r1) |
660 | mfspr r6, SPRN_UAMOR |
661 | std r6, STACK_SLOT_UAMOR(r1) |
662 | |
663 | BEGIN_FTR_SECTION |
664 | /* Set partition DABR */ |
665 | /* Do this before re-enabling PMU to avoid P7 DABR corruption bug */ |
666 | lwz r5,VCPU_DABRX(r4) |
667 | ld r6,VCPU_DABR(r4) |
668 | mtspr SPRN_DABRX,r5 |
669 | mtspr SPRN_DABR,r6 |
670 | isync |
671 | END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) |
672 | |
673 | #ifdef CONFIG_PPC_TRANSACTIONAL_MEM |
674 | BEGIN_FTR_SECTION |
675 | b 91f |
676 | END_FTR_SECTION_IFCLR(CPU_FTR_TM) |
677 | /* |
678 | * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR) |
679 | */ |
680 | mr r3, r4 |
681 | ld r4, VCPU_MSR(r3) |
682 | li r5, 0 /* don't preserve non-vol regs */ |
683 | bl kvmppc_restore_tm_hv |
684 | nop |
685 | ld r4, HSTATE_KVM_VCPU(r13) |
686 | 91: |
687 | #endif |
688 | |
689 | /* Load guest PMU registers; r4 = vcpu pointer here */ |
690 | mr r3, r4 |
691 | bl kvmhv_load_guest_pmu |
692 | |
693 | /* Load up FP, VMX and VSX registers */ |
694 | ld r4, HSTATE_KVM_VCPU(r13) |
695 | bl kvmppc_load_fp |
696 | |
697 | ld r14, VCPU_GPR(R14)(r4) |
698 | ld r15, VCPU_GPR(R15)(r4) |
699 | ld r16, VCPU_GPR(R16)(r4) |
700 | ld r17, VCPU_GPR(R17)(r4) |
701 | ld r18, VCPU_GPR(R18)(r4) |
702 | ld r19, VCPU_GPR(R19)(r4) |
703 | ld r20, VCPU_GPR(R20)(r4) |
704 | ld r21, VCPU_GPR(R21)(r4) |
705 | ld r22, VCPU_GPR(R22)(r4) |
706 | ld r23, VCPU_GPR(R23)(r4) |
707 | ld r24, VCPU_GPR(R24)(r4) |
708 | ld r25, VCPU_GPR(R25)(r4) |
709 | ld r26, VCPU_GPR(R26)(r4) |
710 | ld r27, VCPU_GPR(R27)(r4) |
711 | ld r28, VCPU_GPR(R28)(r4) |
712 | ld r29, VCPU_GPR(R29)(r4) |
713 | ld r30, VCPU_GPR(R30)(r4) |
714 | ld r31, VCPU_GPR(R31)(r4) |
715 | |
716 | /* Switch DSCR to guest value */ |
717 | ld r5, VCPU_DSCR(r4) |
718 | mtspr SPRN_DSCR, r5 |
719 | |
720 | BEGIN_FTR_SECTION |
721 | /* Skip next section on POWER7 */ |
722 | b 8f |
723 | END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) |
724 | /* Load up POWER8-specific registers */ |
725 | ld r5, VCPU_IAMR(r4) |
726 | lwz r6, VCPU_PSPB(r4) |
727 | ld r7, VCPU_FSCR(r4) |
728 | mtspr SPRN_IAMR, r5 |
729 | mtspr SPRN_PSPB, r6 |
730 | mtspr SPRN_FSCR, r7 |
731 | /* |
732 | * Handle broken DAWR case by not writing it. This means we |
733 | * can still store the DAWR register for migration. |
734 | */ |
735 | LOAD_REG_ADDR(r5, dawr_force_enable) |
736 | lbz r5, 0(r5) |
737 | cmpdi r5, 0 |
738 | beq 1f |
739 | ld r5, VCPU_DAWR0(r4) |
740 | ld r6, VCPU_DAWRX0(r4) |
741 | mtspr SPRN_DAWR0, r5 |
742 | mtspr SPRN_DAWRX0, r6 |
743 | 1: |
744 | ld r7, VCPU_CIABR(r4) |
745 | ld r8, VCPU_TAR(r4) |
746 | mtspr SPRN_CIABR, r7 |
747 | mtspr SPRN_TAR, r8 |
748 | ld r5, VCPU_IC(r4) |
749 | ld r8, VCPU_EBBHR(r4) |
750 | mtspr SPRN_IC, r5 |
751 | mtspr SPRN_EBBHR, r8 |
752 | ld r5, VCPU_EBBRR(r4) |
753 | ld r6, VCPU_BESCR(r4) |
754 | lwz r7, VCPU_GUEST_PID(r4) |
755 | ld r8, VCPU_WORT(r4) |
756 | mtspr SPRN_EBBRR, r5 |
757 | mtspr SPRN_BESCR, r6 |
758 | mtspr SPRN_PID, r7 |
759 | mtspr SPRN_WORT, r8 |
760 | /* POWER8-only registers */ |
761 | ld r5, VCPU_TCSCR(r4) |
762 | ld r6, VCPU_ACOP(r4) |
763 | ld r7, VCPU_CSIGR(r4) |
764 | ld r8, VCPU_TACR(r4) |
765 | mtspr SPRN_TCSCR, r5 |
766 | mtspr SPRN_ACOP, r6 |
767 | mtspr SPRN_CSIGR, r7 |
768 | mtspr SPRN_TACR, r8 |
769 | nop |
770 | 8: |
771 | |
772 | ld r5, VCPU_SPRG0(r4) |
773 | ld r6, VCPU_SPRG1(r4) |
774 | ld r7, VCPU_SPRG2(r4) |
775 | ld r8, VCPU_SPRG3(r4) |
776 | mtspr SPRN_SPRG0, r5 |
777 | mtspr SPRN_SPRG1, r6 |
778 | mtspr SPRN_SPRG2, r7 |
779 | mtspr SPRN_SPRG3, r8 |
780 | |
781 | /* Load up DAR and DSISR */ |
782 | ld r5, VCPU_DAR(r4) |
783 | lwz r6, VCPU_DSISR(r4) |
784 | mtspr SPRN_DAR, r5 |
785 | mtspr SPRN_DSISR, r6 |
786 | |
787 | /* Restore AMR and UAMOR, set AMOR to all 1s */ |
788 | ld r5,VCPU_AMR(r4) |
789 | ld r6,VCPU_UAMOR(r4) |
790 | mtspr SPRN_AMR,r5 |
791 | mtspr SPRN_UAMOR,r6 |
792 | |
793 | /* Restore state of CTRL run bit; the host currently has it set to 1 */ |
794 | lwz r5,VCPU_CTRL(r4) |
795 | andi. r5,r5,1 |
796 | bne 4f |
797 | li r6,0 |
798 | mtspr SPRN_CTRLT,r6 |
799 | 4: |
800 | /* Secondary threads wait for primary to have done partition switch */ |
801 | ld r5, HSTATE_KVM_VCORE(r13) |
802 | lbz r6, HSTATE_PTID(r13) |
803 | cmpwi r6, 0 |
804 | beq 21f |
805 | lbz r0, VCORE_IN_GUEST(r5) |
806 | cmpwi r0, 0 |
807 | bne 21f |
808 | HMT_LOW |
809 | 20: lwz r3, VCORE_ENTRY_EXIT(r5) |
810 | cmpwi r3, 0x100 |
811 | bge no_switch_exit |
812 | lbz r0, VCORE_IN_GUEST(r5) |
813 | cmpwi r0, 0 |
814 | beq 20b |
815 | HMT_MEDIUM |
816 | 21: |
817 | /* Set LPCR. */ |
818 | ld r8,VCORE_LPCR(r5) |
819 | mtspr SPRN_LPCR,r8 |
820 | isync |
821 | |
822 | /* |
823 | * Set the decrementer to the guest decrementer. |
824 | */ |
825 | ld r8,VCPU_DEC_EXPIRES(r4) |
826 | mftb r7 |
827 | subf r3,r7,r8 |
828 | mtspr SPRN_DEC,r3 |
829 | |
830 | /* Check if HDEC expires soon */ |
831 | mfspr r3, SPRN_HDEC |
832 | extsw r3, r3 |
833 | cmpdi r3, 512 /* 1 microsecond */ |
834 | blt hdec_soon |
835 | |
836 | /* Clear out and reload the SLB */ |
837 | li r6, 0 |
838 | slbmte r6, r6 |
839 | PPC_SLBIA(6) |
840 | ptesync |
841 | |
842 | /* Load up guest SLB entries (N.B. slb_max will be 0 for radix) */ |
843 | lwz r5,VCPU_SLB_MAX(r4) |
844 | cmpwi r5,0 |
845 | beq 9f |
846 | mtctr r5 |
847 | addi r6,r4,VCPU_SLB |
848 | 1: ld r8,VCPU_SLB_E(r6) |
849 | ld r9,VCPU_SLB_V(r6) |
850 | slbmte r9,r8 |
851 | addi r6,r6,VCPU_SLB_SIZE |
852 | bdnz 1b |
853 | 9: |
854 | |
855 | deliver_guest_interrupt: /* r4 = vcpu, r13 = paca */ |
856 | /* Check if we can deliver an external or decrementer interrupt now */ |
857 | ld r0, VCPU_PENDING_EXC(r4) |
858 | cmpdi r0, 0 |
859 | beq 71f |
860 | mr r3, r4 |
861 | bl CFUNC(kvmppc_guest_entry_inject_int) |
862 | ld r4, HSTATE_KVM_VCPU(r13) |
863 | 71: |
864 | ld r6, VCPU_SRR0(r4) |
865 | ld r7, VCPU_SRR1(r4) |
866 | mtspr SPRN_SRR0, r6 |
867 | mtspr SPRN_SRR1, r7 |
868 | |
869 | ld r10, VCPU_PC(r4) |
870 | ld r11, VCPU_MSR(r4) |
871 | /* r11 = vcpu->arch.msr & ~MSR_HV */ |
872 | rldicl r11, r11, 63 - MSR_HV_LG, 1 |
873 | rotldi r11, r11, 1 + MSR_HV_LG |
874 | ori r11, r11, MSR_ME |
875 | |
876 | ld r6, VCPU_CTR(r4) |
877 | ld r7, VCPU_XER(r4) |
878 | mtctr r6 |
879 | mtxer r7 |
880 | |
881 | /* |
882 | * Required state: |
883 | * R4 = vcpu |
884 | * R10: value for HSRR0 |
885 | * R11: value for HSRR1 |
886 | * R13 = PACA |
887 | */ |
888 | fast_guest_return: |
889 | li r0,0 |
890 | stb r0,VCPU_CEDED(r4) /* cancel cede */ |
891 | mtspr SPRN_HSRR0,r10 |
892 | mtspr SPRN_HSRR1,r11 |
893 | |
894 | /* Activate guest mode, so faults get handled by KVM */ |
895 | li r9, KVM_GUEST_MODE_GUEST_HV |
896 | stb r9, HSTATE_IN_GUEST(r13) |
897 | |
898 | #ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING |
899 | /* Accumulate timing */ |
900 | addi r3, r4, VCPU_TB_GUEST |
901 | bl kvmhv_accumulate_time |
902 | #endif |
903 | |
904 | /* Enter guest */ |
905 | |
906 | BEGIN_FTR_SECTION |
907 | ld r5, VCPU_CFAR(r4) |
908 | mtspr SPRN_CFAR, r5 |
909 | END_FTR_SECTION_IFSET(CPU_FTR_CFAR) |
910 | BEGIN_FTR_SECTION |
911 | ld r0, VCPU_PPR(r4) |
912 | END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) |
913 | |
914 | ld r5, VCPU_LR(r4) |
915 | mtlr r5 |
916 | |
917 | ld r1, VCPU_GPR(R1)(r4) |
918 | ld r5, VCPU_GPR(R5)(r4) |
919 | ld r8, VCPU_GPR(R8)(r4) |
920 | ld r9, VCPU_GPR(R9)(r4) |
921 | ld r10, VCPU_GPR(R10)(r4) |
922 | ld r11, VCPU_GPR(R11)(r4) |
923 | ld r12, VCPU_GPR(R12)(r4) |
924 | ld r13, VCPU_GPR(R13)(r4) |
925 | |
926 | BEGIN_FTR_SECTION |
927 | mtspr SPRN_PPR, r0 |
928 | END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) |
929 | |
930 | ld r6, VCPU_GPR(R6)(r4) |
931 | ld r7, VCPU_GPR(R7)(r4) |
932 | |
933 | ld r0, VCPU_CR(r4) |
934 | mtcr r0 |
935 | |
936 | ld r0, VCPU_GPR(R0)(r4) |
937 | ld r2, VCPU_GPR(R2)(r4) |
938 | ld r3, VCPU_GPR(R3)(r4) |
939 | ld r4, VCPU_GPR(R4)(r4) |
940 | HRFI_TO_GUEST |
941 | b . |
942 | SYM_CODE_END(kvmppc_hv_entry) |
943 | |
944 | secondary_too_late: |
945 | li r12, 0 |
946 | stw r12, STACK_SLOT_TRAP(r1) |
947 | cmpdi r4, 0 |
948 | beq 11f |
949 | stw r12, VCPU_TRAP(r4) |
950 | #ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING |
951 | addi r3, r4, VCPU_TB_RMEXIT |
952 | bl kvmhv_accumulate_time |
953 | #endif |
954 | 11: b kvmhv_switch_to_host |
955 | |
956 | no_switch_exit: |
957 | HMT_MEDIUM |
958 | li r12, 0 |
959 | b 12f |
960 | hdec_soon: |
961 | li r12, BOOK3S_INTERRUPT_HV_DECREMENTER |
962 | 12: stw r12, VCPU_TRAP(r4) |
963 | mr r9, r4 |
964 | #ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING |
965 | addi r3, r4, VCPU_TB_RMEXIT |
966 | bl kvmhv_accumulate_time |
967 | #endif |
968 | b guest_bypass |
969 | |
970 | /****************************************************************************** |
971 | * * |
972 | * Exit code * |
973 | * * |
974 | *****************************************************************************/ |
975 | |
976 | /* |
977 | * We come here from the first-level interrupt handlers. |
978 | */ |
979 | .globl kvmppc_interrupt_hv |
980 | kvmppc_interrupt_hv: |
981 | /* |
982 | * Register contents: |
983 | * R9 = HSTATE_IN_GUEST |
984 | * R12 = (guest CR << 32) | interrupt vector |
985 | * R13 = PACA |
986 | * guest R12 saved in shadow VCPU SCRATCH0 |
987 | * guest R13 saved in SPRN_SCRATCH0 |
988 | * guest R9 saved in HSTATE_SCRATCH2 |
989 | */ |
990 | /* We're now back in the host but in guest MMU context */ |
991 | cmpwi r9,KVM_GUEST_MODE_HOST_HV |
992 | beq kvmppc_bad_host_intr |
993 | li r9, KVM_GUEST_MODE_HOST_HV |
994 | stb r9, HSTATE_IN_GUEST(r13) |
995 | |
996 | ld r9, HSTATE_KVM_VCPU(r13) |
997 | |
998 | /* Save registers */ |
999 | |
1000 | std r0, VCPU_GPR(R0)(r9) |
1001 | std r1, VCPU_GPR(R1)(r9) |
1002 | std r2, VCPU_GPR(R2)(r9) |
1003 | std r3, VCPU_GPR(R3)(r9) |
1004 | std r4, VCPU_GPR(R4)(r9) |
1005 | std r5, VCPU_GPR(R5)(r9) |
1006 | std r6, VCPU_GPR(R6)(r9) |
1007 | std r7, VCPU_GPR(R7)(r9) |
1008 | std r8, VCPU_GPR(R8)(r9) |
1009 | ld r0, HSTATE_SCRATCH2(r13) |
1010 | std r0, VCPU_GPR(R9)(r9) |
1011 | std r10, VCPU_GPR(R10)(r9) |
1012 | std r11, VCPU_GPR(R11)(r9) |
1013 | ld r3, HSTATE_SCRATCH0(r13) |
1014 | std r3, VCPU_GPR(R12)(r9) |
1015 | /* CR is in the high half of r12 */ |
1016 | srdi r4, r12, 32 |
1017 | std r4, VCPU_CR(r9) |
1018 | BEGIN_FTR_SECTION |
1019 | ld r3, HSTATE_CFAR(r13) |
1020 | std r3, VCPU_CFAR(r9) |
1021 | END_FTR_SECTION_IFSET(CPU_FTR_CFAR) |
1022 | BEGIN_FTR_SECTION |
1023 | ld r4, HSTATE_PPR(r13) |
1024 | std r4, VCPU_PPR(r9) |
1025 | END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) |
1026 | |
1027 | /* Restore R1/R2 so we can handle faults */ |
1028 | ld r1, HSTATE_HOST_R1(r13) |
1029 | LOAD_PACA_TOC() |
1030 | |
1031 | mfspr r10, SPRN_SRR0 |
1032 | mfspr r11, SPRN_SRR1 |
1033 | std r10, VCPU_SRR0(r9) |
1034 | std r11, VCPU_SRR1(r9) |
1035 | /* trap is in the low half of r12, clear CR from the high half */ |
1036 | clrldi r12, r12, 32 |
1037 | andi. r0, r12, 2 /* need to read HSRR0/1? */ |
1038 | beq 1f |
1039 | mfspr r10, SPRN_HSRR0 |
1040 | mfspr r11, SPRN_HSRR1 |
1041 | clrrdi r12, r12, 2 |
1042 | 1: std r10, VCPU_PC(r9) |
1043 | std r11, VCPU_MSR(r9) |
1044 | |
1045 | GET_SCRATCH0(r3) |
1046 | mflr r4 |
1047 | std r3, VCPU_GPR(R13)(r9) |
1048 | std r4, VCPU_LR(r9) |
1049 | |
1050 | stw r12,VCPU_TRAP(r9) |
1051 | |
1052 | /* |
1053 | * Now that we have saved away SRR0/1 and HSRR0/1, |
1054 | * interrupts are recoverable in principle, so set MSR_RI. |
1055 | * This becomes important for relocation-on interrupts from |
1056 | * the guest, which we can get in radix mode on POWER9. |
1057 | */ |
1058 | li r0, MSR_RI |
1059 | mtmsrd r0, 1 |
1060 | |
1061 | #ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING |
1062 | addi r3, r9, VCPU_TB_RMINTR |
1063 | mr r4, r9 |
1064 | bl kvmhv_accumulate_time |
1065 | ld r5, VCPU_GPR(R5)(r9) |
1066 | ld r6, VCPU_GPR(R6)(r9) |
1067 | ld r7, VCPU_GPR(R7)(r9) |
1068 | ld r8, VCPU_GPR(R8)(r9) |
1069 | #endif |
1070 | |
1071 | /* Save HEIR (HV emulation assist reg) in emul_inst |
1072 | if this is an HEI (HV emulation interrupt, e40) */ |
1073 | li r3,KVM_INST_FETCH_FAILED |
1074 | std r3,VCPU_LAST_INST(r9) |
1075 | cmpwi r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST |
1076 | bne 11f |
1077 | mfspr r3,SPRN_HEIR |
1078 | 11: std r3,VCPU_HEIR(r9) |
1079 | |
1080 | /* these are volatile across C function calls */ |
1081 | mfctr r3 |
1082 | mfxer r4 |
1083 | std r3, VCPU_CTR(r9) |
1084 | std r4, VCPU_XER(r9) |
1085 | |
1086 | /* Save more register state */ |
1087 | mfdar r3 |
1088 | mfdsisr r4 |
1089 | std r3, VCPU_DAR(r9) |
1090 | stw r4, VCPU_DSISR(r9) |
1091 | |
1092 | /* If this is a page table miss then see if it's theirs or ours */ |
1093 | cmpwi r12, BOOK3S_INTERRUPT_H_DATA_STORAGE |
1094 | beq kvmppc_hdsi |
1095 | std r3, VCPU_FAULT_DAR(r9) |
1096 | stw r4, VCPU_FAULT_DSISR(r9) |
1097 | cmpwi r12, BOOK3S_INTERRUPT_H_INST_STORAGE |
1098 | beq kvmppc_hisi |
1099 | |
1100 | /* See if this is a leftover HDEC interrupt */ |
1101 | cmpwi r12,BOOK3S_INTERRUPT_HV_DECREMENTER |
1102 | bne 2f |
1103 | mfspr r3,SPRN_HDEC |
1104 | extsw r3, r3 |
1105 | cmpdi r3,0 |
1106 | mr r4,r9 |
1107 | bge fast_guest_return |
1108 | 2: |
1109 | /* See if this is an hcall we can handle in real mode */ |
1110 | cmpwi r12,BOOK3S_INTERRUPT_SYSCALL |
1111 | beq hcall_try_real_mode |
1112 | |
1113 | /* Hypervisor doorbell - exit only if host IPI flag set */ |
1114 | cmpwi r12, BOOK3S_INTERRUPT_H_DOORBELL |
1115 | bne 3f |
1116 | lbz r0, HSTATE_HOST_IPI(r13) |
1117 | cmpwi r0, 0 |
1118 | beq maybe_reenter_guest |
1119 | b guest_exit_cont |
1120 | 3: |
1121 | /* If it's a hypervisor facility unavailable interrupt, save HFSCR */ |
1122 | cmpwi r12, BOOK3S_INTERRUPT_H_FAC_UNAVAIL |
1123 | bne 14f |
1124 | mfspr r3, SPRN_HFSCR |
1125 | std r3, VCPU_HFSCR(r9) |
1126 | b guest_exit_cont |
1127 | 14: |
1128 | /* External interrupt ? */ |
1129 | cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL |
1130 | beq kvmppc_guest_external |
1131 | /* See if it is a machine check */ |
1132 | cmpwi r12, BOOK3S_INTERRUPT_MACHINE_CHECK |
1133 | beq machine_check_realmode |
1134 | /* Or a hypervisor maintenance interrupt */ |
1135 | cmpwi r12, BOOK3S_INTERRUPT_HMI |
1136 | beq hmi_realmode |
1137 | |
1138 | guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */ |
1139 | |
1140 | #ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING |
1141 | addi r3, r9, VCPU_TB_RMEXIT |
1142 | mr r4, r9 |
1143 | bl kvmhv_accumulate_time |
1144 | #endif |
1145 | |
1146 | /* |
1147 | * Possibly flush the link stack here, before we do a blr in |
1148 | * kvmhv_switch_to_host. |
1149 | */ |
1150 | 1: nop |
1151 | patch_site 1b patch__call_kvm_flush_link_stack |
1152 | |
1153 | /* For hash guest, read the guest SLB and save it away */ |
1154 | li r5, 0 |
1155 | lwz r0,VCPU_SLB_NR(r9) /* number of entries in SLB */ |
1156 | mtctr r0 |
1157 | li r6,0 |
1158 | addi r7,r9,VCPU_SLB |
1159 | 1: slbmfee r8,r6 |
1160 | andis. r0,r8,SLB_ESID_V@h |
1161 | beq 2f |
1162 | add r8,r8,r6 /* put index in */ |
1163 | slbmfev r3,r6 |
1164 | std r8,VCPU_SLB_E(r7) |
1165 | std r3,VCPU_SLB_V(r7) |
1166 | addi r7,r7,VCPU_SLB_SIZE |
1167 | addi r5,r5,1 |
1168 | 2: addi r6,r6,1 |
1169 | bdnz 1b |
1170 | /* Finally clear out the SLB */ |
1171 | li r0,0 |
1172 | slbmte r0,r0 |
1173 | PPC_SLBIA(6) |
1174 | ptesync |
1175 | stw r5,VCPU_SLB_MAX(r9) |
1176 | |
1177 | /* load host SLB entries */ |
1178 | ld r8,PACA_SLBSHADOWPTR(r13) |
1179 | |
1180 | .rept SLB_NUM_BOLTED |
1181 | li r3, SLBSHADOW_SAVEAREA |
1182 | LDX_BE r5, r8, r3 |
1183 | addi r3, r3, 8 |
1184 | LDX_BE r6, r8, r3 |
1185 | andis. r7,r5,SLB_ESID_V@h |
1186 | beq 1f |
1187 | slbmte r6,r5 |
1188 | 1: addi r8,r8,16 |
1189 | .endr |
1190 | |
1191 | guest_bypass: |
1192 | stw r12, STACK_SLOT_TRAP(r1) |
1193 | |
1194 | /* Save DEC */ |
1195 | /* Do this before kvmhv_commence_exit so we know TB is guest TB */ |
1196 | ld r3, HSTATE_KVM_VCORE(r13) |
1197 | mfspr r5,SPRN_DEC |
1198 | mftb r6 |
1199 | extsw r5,r5 |
1200 | 16: add r5,r5,r6 |
1201 | std r5,VCPU_DEC_EXPIRES(r9) |
1202 | |
1203 | /* Increment exit count, poke other threads to exit */ |
1204 | mr r3, r12 |
1205 | bl kvmhv_commence_exit |
1206 | nop |
1207 | ld r9, HSTATE_KVM_VCPU(r13) |
1208 | |
1209 | /* Stop others sending VCPU interrupts to this physical CPU */ |
1210 | li r0, -1 |
1211 | stw r0, VCPU_CPU(r9) |
1212 | stw r0, VCPU_THREAD_CPU(r9) |
1213 | |
1214 | /* Save guest CTRL register, set runlatch to 1 if it was clear */ |
1215 | mfspr r6,SPRN_CTRLF |
1216 | stw r6,VCPU_CTRL(r9) |
1217 | andi. r0,r6,1 |
1218 | bne 4f |
1219 | li r6,1 |
1220 | mtspr SPRN_CTRLT,r6 |
1221 | 4: |
1222 | /* |
1223 | * Save the guest PURR/SPURR |
1224 | */ |
1225 | mfspr r5,SPRN_PURR |
1226 | mfspr r6,SPRN_SPURR |
1227 | ld r7,VCPU_PURR(r9) |
1228 | ld r8,VCPU_SPURR(r9) |
1229 | std r5,VCPU_PURR(r9) |
1230 | std r6,VCPU_SPURR(r9) |
1231 | subf r5,r7,r5 |
1232 | subf r6,r8,r6 |
1233 | |
1234 | /* |
1235 | * Restore host PURR/SPURR and add guest times |
1236 | * so that the time in the guest gets accounted. |
1237 | */ |
1238 | ld r3,HSTATE_PURR(r13) |
1239 | ld r4,HSTATE_SPURR(r13) |
1240 | add r3,r3,r5 |
1241 | add r4,r4,r6 |
1242 | mtspr SPRN_PURR,r3 |
1243 | mtspr SPRN_SPURR,r4 |
1244 | |
1245 | BEGIN_FTR_SECTION |
1246 | b 8f |
1247 | END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) |
1248 | /* Save POWER8-specific registers */ |
1249 | mfspr r5, SPRN_IAMR |
1250 | mfspr r6, SPRN_PSPB |
1251 | mfspr r7, SPRN_FSCR |
1252 | std r5, VCPU_IAMR(r9) |
1253 | stw r6, VCPU_PSPB(r9) |
1254 | std r7, VCPU_FSCR(r9) |
1255 | mfspr r5, SPRN_IC |
1256 | mfspr r7, SPRN_TAR |
1257 | std r5, VCPU_IC(r9) |
1258 | std r7, VCPU_TAR(r9) |
1259 | mfspr r8, SPRN_EBBHR |
1260 | std r8, VCPU_EBBHR(r9) |
1261 | mfspr r5, SPRN_EBBRR |
1262 | mfspr r6, SPRN_BESCR |
1263 | mfspr r7, SPRN_PID |
1264 | mfspr r8, SPRN_WORT |
1265 | std r5, VCPU_EBBRR(r9) |
1266 | std r6, VCPU_BESCR(r9) |
1267 | stw r7, VCPU_GUEST_PID(r9) |
1268 | std r8, VCPU_WORT(r9) |
1269 | mfspr r5, SPRN_TCSCR |
1270 | mfspr r6, SPRN_ACOP |
1271 | mfspr r7, SPRN_CSIGR |
1272 | mfspr r8, SPRN_TACR |
1273 | std r5, VCPU_TCSCR(r9) |
1274 | std r6, VCPU_ACOP(r9) |
1275 | std r7, VCPU_CSIGR(r9) |
1276 | std r8, VCPU_TACR(r9) |
1277 | BEGIN_FTR_SECTION |
1278 | ld r5, STACK_SLOT_FSCR(r1) |
1279 | mtspr SPRN_FSCR, r5 |
1280 | END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) |
1281 | /* |
1282 | * Restore various registers to 0, where non-zero values |
1283 | * set by the guest could disrupt the host. |
1284 | */ |
1285 | li r0, 0 |
1286 | mtspr SPRN_PSPB, r0 |
1287 | mtspr SPRN_WORT, r0 |
1288 | mtspr SPRN_TCSCR, r0 |
1289 | /* Set MMCRS to 1<<31 to freeze and disable the SPMC counters */ |
1290 | li r0, 1 |
1291 | sldi r0, r0, 31 |
1292 | mtspr SPRN_MMCRS, r0 |
1293 | |
1294 | /* Save and restore AMR, IAMR and UAMOR before turning on the MMU */ |
1295 | ld r8, STACK_SLOT_IAMR(r1) |
1296 | mtspr SPRN_IAMR, r8 |
1297 | |
1298 | 8: /* Power7 jumps back in here */ |
1299 | mfspr r5,SPRN_AMR |
1300 | mfspr r6,SPRN_UAMOR |
1301 | std r5,VCPU_AMR(r9) |
1302 | std r6,VCPU_UAMOR(r9) |
1303 | ld r5,STACK_SLOT_AMR(r1) |
1304 | ld r6,STACK_SLOT_UAMOR(r1) |
1305 | mtspr SPRN_AMR, r5 |
1306 | mtspr SPRN_UAMOR, r6 |
1307 | |
1308 | /* Switch DSCR back to host value */ |
1309 | mfspr r8, SPRN_DSCR |
1310 | ld r7, HSTATE_DSCR(r13) |
1311 | std r8, VCPU_DSCR(r9) |
1312 | mtspr SPRN_DSCR, r7 |
1313 | |
1314 | /* Save non-volatile GPRs */ |
1315 | std r14, VCPU_GPR(R14)(r9) |
1316 | std r15, VCPU_GPR(R15)(r9) |
1317 | std r16, VCPU_GPR(R16)(r9) |
1318 | std r17, VCPU_GPR(R17)(r9) |
1319 | std r18, VCPU_GPR(R18)(r9) |
1320 | std r19, VCPU_GPR(R19)(r9) |
1321 | std r20, VCPU_GPR(R20)(r9) |
1322 | std r21, VCPU_GPR(R21)(r9) |
1323 | std r22, VCPU_GPR(R22)(r9) |
1324 | std r23, VCPU_GPR(R23)(r9) |
1325 | std r24, VCPU_GPR(R24)(r9) |
1326 | std r25, VCPU_GPR(R25)(r9) |
1327 | std r26, VCPU_GPR(R26)(r9) |
1328 | std r27, VCPU_GPR(R27)(r9) |
1329 | std r28, VCPU_GPR(R28)(r9) |
1330 | std r29, VCPU_GPR(R29)(r9) |
1331 | std r30, VCPU_GPR(R30)(r9) |
1332 | std r31, VCPU_GPR(R31)(r9) |
1333 | |
1334 | /* Save SPRGs */ |
1335 | mfspr r3, SPRN_SPRG0 |
1336 | mfspr r4, SPRN_SPRG1 |
1337 | mfspr r5, SPRN_SPRG2 |
1338 | mfspr r6, SPRN_SPRG3 |
1339 | std r3, VCPU_SPRG0(r9) |
1340 | std r4, VCPU_SPRG1(r9) |
1341 | std r5, VCPU_SPRG2(r9) |
1342 | std r6, VCPU_SPRG3(r9) |
1343 | |
1344 | /* save FP state */ |
1345 | mr r3, r9 |
1346 | bl kvmppc_save_fp |
1347 | |
1348 | #ifdef CONFIG_PPC_TRANSACTIONAL_MEM |
1349 | BEGIN_FTR_SECTION |
1350 | b 91f |
1351 | END_FTR_SECTION_IFCLR(CPU_FTR_TM) |
1352 | /* |
1353 | * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR) |
1354 | */ |
1355 | mr r3, r9 |
1356 | ld r4, VCPU_MSR(r3) |
1357 | li r5, 0 /* don't preserve non-vol regs */ |
1358 | bl kvmppc_save_tm_hv |
1359 | nop |
1360 | ld r9, HSTATE_KVM_VCPU(r13) |
1361 | 91: |
1362 | #endif |
1363 | |
1364 | /* Increment yield count if they have a VPA */ |
1365 | ld r8, VCPU_VPA(r9) /* do they have a VPA? */ |
1366 | cmpdi r8, 0 |
1367 | beq 25f |
1368 | li r4, LPPACA_YIELDCOUNT |
1369 | LWZX_BE r3, r8, r4 |
1370 | addi r3, r3, 1 |
1371 | STWX_BE r3, r8, r4 |
1372 | li r3, 1 |
1373 | stb r3, VCPU_VPA_DIRTY(r9) |
1374 | 25: |
1375 | /* Save PMU registers if requested */ |
1376 | /* r8 and cr0.eq are live here */ |
1377 | mr r3, r9 |
1378 | li r4, 1 |
1379 | beq 21f /* if no VPA, save PMU stuff anyway */ |
1380 | lbz r4, LPPACA_PMCINUSE(r8) |
1381 | 21: bl kvmhv_save_guest_pmu |
1382 | ld r9, HSTATE_KVM_VCPU(r13) |
1383 | |
1384 | /* Restore host values of some registers */ |
1385 | BEGIN_FTR_SECTION |
1386 | ld r5, STACK_SLOT_CIABR(r1) |
1387 | ld r6, STACK_SLOT_DAWR0(r1) |
1388 | ld r7, STACK_SLOT_DAWRX0(r1) |
1389 | mtspr SPRN_CIABR, r5 |
1390 | /* |
1391 | * If the DAWR doesn't work, it's ok to write these here as |
1392 | * this value should always be zero |
1393 | */ |
1394 | mtspr SPRN_DAWR0, r6 |
1395 | mtspr SPRN_DAWRX0, r7 |
1396 | END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) |
1397 | |
1398 | /* |
1399 | * POWER7/POWER8 guest -> host partition switch code. |
1400 | * We don't have to lock against tlbies but we do |
1401 | * have to coordinate the hardware threads. |
1402 | * Here STACK_SLOT_TRAP(r1) contains the trap number. |
1403 | */ |
1404 | kvmhv_switch_to_host: |
1405 | /* Secondary threads wait for primary to do partition switch */ |
1406 | ld r5,HSTATE_KVM_VCORE(r13) |
1407 | ld r4,VCORE_KVM(r5) /* pointer to struct kvm */ |
1408 | lbz r3,HSTATE_PTID(r13) |
1409 | cmpwi r3,0 |
1410 | beq 15f |
1411 | HMT_LOW |
1412 | 13: lbz r3,VCORE_IN_GUEST(r5) |
1413 | cmpwi r3,0 |
1414 | bne 13b |
1415 | HMT_MEDIUM |
1416 | b 16f |
1417 | |
1418 | /* Primary thread waits for all the secondaries to exit guest */ |
1419 | 15: lwz r3,VCORE_ENTRY_EXIT(r5) |
1420 | rlwinm r0,r3,32-8,0xff |
1421 | clrldi r3,r3,56 |
1422 | cmpw r3,r0 |
1423 | bne 15b |
1424 | isync |
1425 | |
1426 | /* Did we actually switch to the guest at all? */ |
1427 | lbz r6, VCORE_IN_GUEST(r5) |
1428 | cmpwi r6, 0 |
1429 | beq 19f |
1430 | |
1431 | /* Primary thread switches back to host partition */ |
1432 | lwz r7,KVM_HOST_LPID(r4) |
1433 | ld r6,KVM_HOST_SDR1(r4) |
1434 | li r8,LPID_RSVD /* switch to reserved LPID */ |
1435 | mtspr SPRN_LPID,r8 |
1436 | ptesync |
1437 | mtspr SPRN_SDR1,r6 /* switch to host page table */ |
1438 | mtspr SPRN_LPID,r7 |
1439 | isync |
1440 | |
1441 | BEGIN_FTR_SECTION |
1442 | /* DPDES and VTB are shared between threads */ |
1443 | mfspr r7, SPRN_DPDES |
1444 | mfspr r8, SPRN_VTB |
1445 | std r7, VCORE_DPDES(r5) |
1446 | std r8, VCORE_VTB(r5) |
1447 | /* clear DPDES so we don't get guest doorbells in the host */ |
1448 | li r8, 0 |
1449 | mtspr SPRN_DPDES, r8 |
1450 | END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) |
1451 | |
1452 | /* Subtract timebase offset from timebase */ |
1453 | ld r8, VCORE_TB_OFFSET_APPL(r5) |
1454 | cmpdi r8,0 |
1455 | beq 17f |
1456 | li r0, 0 |
1457 | std r0, VCORE_TB_OFFSET_APPL(r5) |
1458 | mftb r6 /* current guest timebase */ |
1459 | subf r8,r8,r6 |
1460 | mtspr SPRN_TBU40,r8 /* update upper 40 bits */ |
1461 | mftb r7 /* check if lower 24 bits overflowed */ |
1462 | clrldi r6,r6,40 |
1463 | clrldi r7,r7,40 |
1464 | cmpld r7,r6 |
1465 | bge 17f |
1466 | addis r8,r8,0x100 /* if so, increment upper 40 bits */ |
1467 | mtspr SPRN_TBU40,r8 |
1468 | |
1469 | 17: |
1470 | /* |
1471 | * If this is an HMI, we called kvmppc_realmode_hmi_handler |
1472 | * above, which may or may not have already called |
1473 | * kvmppc_subcore_exit_guest. Fortunately, all that |
1474 | * kvmppc_subcore_exit_guest does is clear a flag, so calling |
1475 | * it again here is benign even if kvmppc_realmode_hmi_handler |
1476 | * has already called it. |
1477 | */ |
1478 | bl kvmppc_subcore_exit_guest |
1479 | nop |
1480 | 30: ld r5,HSTATE_KVM_VCORE(r13) |
1481 | ld r4,VCORE_KVM(r5) /* pointer to struct kvm */ |
1482 | |
1483 | /* Reset PCR */ |
1484 | ld r0, VCORE_PCR(r5) |
1485 | LOAD_REG_IMMEDIATE(r6, PCR_MASK) |
1486 | cmpld r0, r6 |
1487 | beq 18f |
1488 | mtspr SPRN_PCR, r6 |
1489 | 18: |
1490 | /* Signal secondary CPUs to continue */ |
1491 | li r0, 0 |
1492 | stb r0,VCORE_IN_GUEST(r5) |
1493 | 19: lis r8,0x7fff /* MAX_INT@h */ |
1494 | mtspr SPRN_HDEC,r8 |
1495 | |
1496 | 16: ld r8,KVM_HOST_LPCR(r4) |
1497 | mtspr SPRN_LPCR,r8 |
1498 | isync |
1499 | |
1500 | #ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING |
1501 | /* Finish timing, if we have a vcpu */ |
1502 | ld r4, HSTATE_KVM_VCPU(r13) |
1503 | cmpdi r4, 0 |
1504 | li r3, 0 |
1505 | beq 2f |
1506 | bl kvmhv_accumulate_time |
1507 | 2: |
1508 | #endif |
1509 | /* Unset guest mode */ |
1510 | li r0, KVM_GUEST_MODE_NONE |
1511 | stb r0, HSTATE_IN_GUEST(r13) |
1512 | |
1513 | lwz r12, STACK_SLOT_TRAP(r1) /* return trap # in r12 */ |
1514 | ld r0, SFS+PPC_LR_STKOFF(r1) |
1515 | addi r1, r1, SFS |
1516 | mtlr r0 |
1517 | blr |
1518 | |
1519 | .balign 32 |
1520 | .global kvm_flush_link_stack |
1521 | kvm_flush_link_stack: |
1522 | /* Save LR into r0 */ |
1523 | mflr r0 |
1524 | |
1525 | /* Flush the link stack. On Power8 it's up to 32 entries in size. */ |
1526 | .rept 32 |
1527 | ANNOTATE_INTRA_FUNCTION_CALL |
1528 | bl .+4 |
1529 | .endr |
1530 | |
1531 | /* And on Power9 it's up to 64. */ |
1532 | BEGIN_FTR_SECTION |
1533 | .rept 32 |
1534 | ANNOTATE_INTRA_FUNCTION_CALL |
1535 | bl .+4 |
1536 | .endr |
1537 | END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) |
1538 | |
1539 | /* Restore LR */ |
1540 | mtlr r0 |
1541 | blr |
1542 | |
1543 | kvmppc_guest_external: |
1544 | /* External interrupt, first check for host_ipi. If this is |
1545 | * set, we know the host wants us out so let's do it now |
1546 | */ |
1547 | bl CFUNC(kvmppc_read_intr) |
1548 | |
1549 | /* |
1550 | * Restore the active volatile registers after returning from |
1551 | * a C function. |
1552 | */ |
1553 | ld r9, HSTATE_KVM_VCPU(r13) |
1554 | li r12, BOOK3S_INTERRUPT_EXTERNAL |
1555 | |
1556 | /* |
1557 | * kvmppc_read_intr return codes: |
1558 | * |
1559 | * Exit to host (r3 > 0) |
1560 | * 1 An interrupt is pending that needs to be handled by the host |
1561 | * Exit guest and return to host by branching to guest_exit_cont |
1562 | * |
1563 | * 2 Passthrough that needs completion in the host |
1564 | * Exit guest and return to host by branching to guest_exit_cont |
1565 | * However, we also set r12 to BOOK3S_INTERRUPT_HV_RM_HARD |
1566 | * to indicate to the host to complete handling the interrupt |
1567 | * |
1568 | * Before returning to guest, we check if any CPU is heading out |
1569 | * to the host and if so, we head out also. If no CPUs are heading |
1570 | * check return values <= 0. |
1571 | * |
1572 | * Return to guest (r3 <= 0) |
1573 | * 0 No external interrupt is pending |
1574 | * -1 A guest wakeup IPI (which has now been cleared) |
1575 | * In either case, we return to guest to deliver any pending |
1576 | * guest interrupts. |
1577 | * |
1578 | * -2 A PCI passthrough external interrupt was handled |
1579 | * (interrupt was delivered directly to guest) |
1580 | * Return to guest to deliver any pending guest interrupts. |
1581 | */ |
1582 | |
1583 | cmpdi r3, 1 |
1584 | ble 1f |
1585 | |
1586 | /* Return code = 2 */ |
1587 | li r12, BOOK3S_INTERRUPT_HV_RM_HARD |
1588 | stw r12, VCPU_TRAP(r9) |
1589 | b guest_exit_cont |
1590 | |
1591 | 1: /* Return code <= 1 */ |
1592 | cmpdi r3, 0 |
1593 | bgt guest_exit_cont |
1594 | |
1595 | /* Return code <= 0 */ |
1596 | maybe_reenter_guest: |
1597 | ld r5, HSTATE_KVM_VCORE(r13) |
1598 | lwz r0, VCORE_ENTRY_EXIT(r5) |
1599 | cmpwi r0, 0x100 |
1600 | mr r4, r9 |
1601 | blt deliver_guest_interrupt |
1602 | b guest_exit_cont |
1603 | |
1604 | /* |
1605 | * Check whether an HDSI is an HPTE not found fault or something else. |
1606 | * If it is an HPTE not found fault that is due to the guest accessing |
1607 | * a page that they have mapped but which we have paged out, then |
1608 | * we continue on with the guest exit path. In all other cases, |
1609 | * reflect the HDSI to the guest as a DSI. |
1610 | */ |
1611 | kvmppc_hdsi: |
1612 | mfspr r4, SPRN_HDAR |
1613 | mfspr r6, SPRN_HDSISR |
1614 | /* HPTE not found fault or protection fault? */ |
1615 | andis. r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h |
1616 | beq 1f /* if not, send it to the guest */ |
1617 | andi. r0, r11, MSR_DR /* data relocation enabled? */ |
1618 | beq 3f |
1619 | clrrdi r0, r4, 28 |
1620 | PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */ |
1621 | li r0, BOOK3S_INTERRUPT_DATA_SEGMENT |
1622 | bne 7f /* if no SLB entry found */ |
1623 | 4: std r4, VCPU_FAULT_DAR(r9) |
1624 | stw r6, VCPU_FAULT_DSISR(r9) |
1625 | |
1626 | /* Search the hash table. */ |
1627 | mr r3, r9 /* vcpu pointer */ |
1628 | li r7, 1 /* data fault */ |
1629 | bl CFUNC(kvmppc_hpte_hv_fault) |
1630 | ld r9, HSTATE_KVM_VCPU(r13) |
1631 | ld r10, VCPU_PC(r9) |
1632 | ld r11, VCPU_MSR(r9) |
1633 | li r12, BOOK3S_INTERRUPT_H_DATA_STORAGE |
1634 | cmpdi r3, 0 /* retry the instruction */ |
1635 | beq 6f |
1636 | cmpdi r3, -1 /* handle in kernel mode */ |
1637 | beq guest_exit_cont |
1638 | cmpdi r3, -2 /* MMIO emulation; need instr word */ |
1639 | beq 2f |
1640 | |
1641 | /* Synthesize a DSI (or DSegI) for the guest */ |
1642 | ld r4, VCPU_FAULT_DAR(r9) |
1643 | mr r6, r3 |
1644 | 1: li r0, BOOK3S_INTERRUPT_DATA_STORAGE |
1645 | mtspr SPRN_DSISR, r6 |
1646 | 7: mtspr SPRN_DAR, r4 |
1647 | mtspr SPRN_SRR0, r10 |
1648 | mtspr SPRN_SRR1, r11 |
1649 | mr r10, r0 |
1650 | bl kvmppc_msr_interrupt |
1651 | fast_interrupt_c_return: |
1652 | 6: ld r7, VCPU_CTR(r9) |
1653 | ld r8, VCPU_XER(r9) |
1654 | mtctr r7 |
1655 | mtxer r8 |
1656 | mr r4, r9 |
1657 | b fast_guest_return |
1658 | |
1659 | 3: ld r5, VCPU_KVM(r9) /* not relocated, use VRMA */ |
1660 | ld r5, KVM_VRMA_SLB_V(r5) |
1661 | b 4b |
1662 | |
1663 | /* If this is for emulated MMIO, load the instruction word */ |
1664 | 2: li r8, KVM_INST_FETCH_FAILED /* In case lwz faults */ |
1665 | |
1666 | /* Set guest mode to 'jump over instruction' so if lwz faults |
1667 | * we'll just continue at the next IP. */ |
1668 | li r0, KVM_GUEST_MODE_SKIP |
1669 | stb r0, HSTATE_IN_GUEST(r13) |
1670 | |
1671 | /* Do the access with MSR:DR enabled */ |
1672 | mfmsr r3 |
1673 | ori r4, r3, MSR_DR /* Enable paging for data */ |
1674 | mtmsrd r4 |
1675 | lwz r8, 0(r10) |
1676 | mtmsrd r3 |
1677 | |
1678 | /* Store the result */ |
1679 | std r8, VCPU_LAST_INST(r9) |
1680 | |
1681 | /* Unset guest mode. */ |
1682 | li r0, KVM_GUEST_MODE_HOST_HV |
1683 | stb r0, HSTATE_IN_GUEST(r13) |
1684 | b guest_exit_cont |
1685 | |
1686 | /* |
1687 | * Similarly for an HISI, reflect it to the guest as an ISI unless |
1688 | * it is an HPTE not found fault for a page that we have paged out. |
1689 | */ |
1690 | kvmppc_hisi: |
1691 | andis. r0, r11, SRR1_ISI_NOPT@h |
1692 | beq 1f |
1693 | andi. r0, r11, MSR_IR /* instruction relocation enabled? */ |
1694 | beq 3f |
1695 | clrrdi r0, r10, 28 |
1696 | PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */ |
1697 | li r0, BOOK3S_INTERRUPT_INST_SEGMENT |
1698 | bne 7f /* if no SLB entry found */ |
1699 | 4: |
1700 | /* Search the hash table. */ |
1701 | mr r3, r9 /* vcpu pointer */ |
1702 | mr r4, r10 |
1703 | mr r6, r11 |
1704 | li r7, 0 /* instruction fault */ |
1705 | bl CFUNC(kvmppc_hpte_hv_fault) |
1706 | ld r9, HSTATE_KVM_VCPU(r13) |
1707 | ld r10, VCPU_PC(r9) |
1708 | ld r11, VCPU_MSR(r9) |
1709 | li r12, BOOK3S_INTERRUPT_H_INST_STORAGE |
1710 | cmpdi r3, 0 /* retry the instruction */ |
1711 | beq fast_interrupt_c_return |
1712 | cmpdi r3, -1 /* handle in kernel mode */ |
1713 | beq guest_exit_cont |
1714 | |
1715 | /* Synthesize an ISI (or ISegI) for the guest */ |
1716 | mr r11, r3 |
1717 | 1: li r0, BOOK3S_INTERRUPT_INST_STORAGE |
1718 | 7: mtspr SPRN_SRR0, r10 |
1719 | mtspr SPRN_SRR1, r11 |
1720 | mr r10, r0 |
1721 | bl kvmppc_msr_interrupt |
1722 | b fast_interrupt_c_return |
1723 | |
1724 | 3: ld r6, VCPU_KVM(r9) /* not relocated, use VRMA */ |
1725 | ld r5, KVM_VRMA_SLB_V(r6) |
1726 | b 4b |
1727 | |
1728 | /* |
1729 | * Try to handle an hcall in real mode. |
1730 | * Returns to the guest if we handle it, or continues on up to |
1731 | * the kernel if we can't (i.e. if we don't have a handler for |
1732 | * it, or if the handler returns H_TOO_HARD). |
1733 | * |
1734 | * r5 - r8 contain hcall args, |
1735 | * r9 = vcpu, r10 = pc, r11 = msr, r12 = trap, r13 = paca |
1736 | */ |
1737 | hcall_try_real_mode: |
1738 | ld r3,VCPU_GPR(R3)(r9) |
1739 | andi. r0,r11,MSR_PR |
1740 | /* sc 1 from userspace - reflect to guest syscall */ |
1741 | bne sc_1_fast_return |
1742 | clrrdi r3,r3,2 |
1743 | cmpldi r3,hcall_real_table_end - hcall_real_table |
1744 | bge guest_exit_cont |
1745 | /* See if this hcall is enabled for in-kernel handling */ |
1746 | ld r4, VCPU_KVM(r9) |
1747 | srdi r0, r3, 8 /* r0 = (r3 / 4) >> 6 */ |
1748 | sldi r0, r0, 3 /* index into kvm->arch.enabled_hcalls[] */ |
1749 | add r4, r4, r0 |
1750 | ld r0, KVM_ENABLED_HCALLS(r4) |
1751 | rlwinm r4, r3, 32-2, 0x3f /* r4 = (r3 / 4) & 0x3f */ |
1752 | srd r0, r0, r4 |
1753 | andi. r0, r0, 1 |
1754 | beq guest_exit_cont |
1755 | /* Get pointer to handler, if any, and call it */ |
1756 | LOAD_REG_ADDR(r4, hcall_real_table) |
1757 | lwax r3,r3,r4 |
1758 | cmpwi r3,0 |
1759 | beq guest_exit_cont |
1760 | add r12,r3,r4 |
1761 | mtctr r12 |
1762 | mr r3,r9 /* get vcpu pointer */ |
1763 | ld r4,VCPU_GPR(R4)(r9) |
1764 | bctrl |
1765 | cmpdi r3,H_TOO_HARD |
1766 | beq hcall_real_fallback |
1767 | ld r4,HSTATE_KVM_VCPU(r13) |
1768 | std r3,VCPU_GPR(R3)(r4) |
1769 | ld r10,VCPU_PC(r4) |
1770 | ld r11,VCPU_MSR(r4) |
1771 | b fast_guest_return |
1772 | |
1773 | sc_1_fast_return: |
1774 | mtspr SPRN_SRR0,r10 |
1775 | mtspr SPRN_SRR1,r11 |
1776 | li r10, BOOK3S_INTERRUPT_SYSCALL |
1777 | bl kvmppc_msr_interrupt |
1778 | mr r4,r9 |
1779 | b fast_guest_return |
1780 | |
1781 | /* We've attempted a real mode hcall, but it's punted it back |
1782 | * to userspace. We need to restore some clobbered volatiles |
1783 | * before resuming the pass-it-to-qemu path */ |
1784 | hcall_real_fallback: |
1785 | li r12,BOOK3S_INTERRUPT_SYSCALL |
1786 | ld r9, HSTATE_KVM_VCPU(r13) |
1787 | |
1788 | b guest_exit_cont |
1789 | |
1790 | .globl hcall_real_table |
1791 | hcall_real_table: |
1792 | .long 0 /* 0 - unused */ |
1793 | .long DOTSYM(kvmppc_h_remove) - hcall_real_table |
1794 | .long DOTSYM(kvmppc_h_enter) - hcall_real_table |
1795 | .long DOTSYM(kvmppc_h_read) - hcall_real_table |
1796 | .long DOTSYM(kvmppc_h_clear_mod) - hcall_real_table |
1797 | .long DOTSYM(kvmppc_h_clear_ref) - hcall_real_table |
1798 | .long DOTSYM(kvmppc_h_protect) - hcall_real_table |
1799 | .long 0 /* 0x1c */ |
1800 | .long 0 /* 0x20 */ |
1801 | .long 0 /* 0x24 - H_SET_SPRG0 */ |
1802 | .long DOTSYM(kvmppc_h_set_dabr) - hcall_real_table |
1803 | .long DOTSYM(kvmppc_rm_h_page_init) - hcall_real_table |
1804 | .long 0 /* 0x30 */ |
1805 | .long 0 /* 0x34 */ |
1806 | .long 0 /* 0x38 */ |
1807 | .long 0 /* 0x3c */ |
1808 | .long 0 /* 0x40 */ |
1809 | .long 0 /* 0x44 */ |
1810 | .long 0 /* 0x48 */ |
1811 | .long 0 /* 0x4c */ |
1812 | .long 0 /* 0x50 */ |
1813 | .long 0 /* 0x54 */ |
1814 | .long 0 /* 0x58 */ |
1815 | .long 0 /* 0x5c */ |
1816 | .long 0 /* 0x60 */ |
1817 | #ifdef CONFIG_KVM_XICS |
1818 | .long DOTSYM(xics_rm_h_eoi) - hcall_real_table |
1819 | .long DOTSYM(xics_rm_h_cppr) - hcall_real_table |
1820 | .long DOTSYM(xics_rm_h_ipi) - hcall_real_table |
1821 | .long 0 /* 0x70 - H_IPOLL */ |
1822 | .long DOTSYM(xics_rm_h_xirr) - hcall_real_table |
1823 | #else |
1824 | .long 0 /* 0x64 - H_EOI */ |
1825 | .long 0 /* 0x68 - H_CPPR */ |
1826 | .long 0 /* 0x6c - H_IPI */ |
1827 | .long 0 /* 0x70 - H_IPOLL */ |
1828 | .long 0 /* 0x74 - H_XIRR */ |
1829 | #endif |
1830 | .long 0 /* 0x78 */ |
1831 | .long 0 /* 0x7c */ |
1832 | .long 0 /* 0x80 */ |
1833 | .long 0 /* 0x84 */ |
1834 | .long 0 /* 0x88 */ |
1835 | .long 0 /* 0x8c */ |
1836 | .long 0 /* 0x90 */ |
1837 | .long 0 /* 0x94 */ |
1838 | .long 0 /* 0x98 */ |
1839 | .long 0 /* 0x9c */ |
1840 | .long 0 /* 0xa0 */ |
1841 | .long 0 /* 0xa4 */ |
1842 | .long 0 /* 0xa8 */ |
1843 | .long 0 /* 0xac */ |
1844 | .long 0 /* 0xb0 */ |
1845 | .long 0 /* 0xb4 */ |
1846 | .long 0 /* 0xb8 */ |
1847 | .long 0 /* 0xbc */ |
1848 | .long 0 /* 0xc0 */ |
1849 | .long 0 /* 0xc4 */ |
1850 | .long 0 /* 0xc8 */ |
1851 | .long 0 /* 0xcc */ |
1852 | .long 0 /* 0xd0 */ |
1853 | .long 0 /* 0xd4 */ |
1854 | .long 0 /* 0xd8 */ |
1855 | .long 0 /* 0xdc */ |
1856 | .long DOTSYM(kvmppc_h_cede) - hcall_real_table |
1857 | .long DOTSYM(kvmppc_rm_h_confer) - hcall_real_table |
1858 | .long 0 /* 0xe8 */ |
1859 | .long 0 /* 0xec */ |
1860 | .long 0 /* 0xf0 */ |
1861 | .long 0 /* 0xf4 */ |
1862 | .long 0 /* 0xf8 */ |
1863 | .long 0 /* 0xfc */ |
1864 | .long 0 /* 0x100 */ |
1865 | .long 0 /* 0x104 */ |
1866 | .long 0 /* 0x108 */ |
1867 | .long 0 /* 0x10c */ |
1868 | .long 0 /* 0x110 */ |
1869 | .long 0 /* 0x114 */ |
1870 | .long 0 /* 0x118 */ |
1871 | .long 0 /* 0x11c */ |
1872 | .long 0 /* 0x120 */ |
1873 | .long DOTSYM(kvmppc_h_bulk_remove) - hcall_real_table |
1874 | .long 0 /* 0x128 */ |
1875 | .long 0 /* 0x12c */ |
1876 | .long 0 /* 0x130 */ |
1877 | .long DOTSYM(kvmppc_h_set_xdabr) - hcall_real_table |
1878 | .long 0 /* 0x138 */ |
1879 | .long 0 /* 0x13c */ |
1880 | .long 0 /* 0x140 */ |
1881 | .long 0 /* 0x144 */ |
1882 | .long 0 /* 0x148 */ |
1883 | .long 0 /* 0x14c */ |
1884 | .long 0 /* 0x150 */ |
1885 | .long 0 /* 0x154 */ |
1886 | .long 0 /* 0x158 */ |
1887 | .long 0 /* 0x15c */ |
1888 | .long 0 /* 0x160 */ |
1889 | .long 0 /* 0x164 */ |
1890 | .long 0 /* 0x168 */ |
1891 | .long 0 /* 0x16c */ |
1892 | .long 0 /* 0x170 */ |
1893 | .long 0 /* 0x174 */ |
1894 | .long 0 /* 0x178 */ |
1895 | .long 0 /* 0x17c */ |
1896 | .long 0 /* 0x180 */ |
1897 | .long 0 /* 0x184 */ |
1898 | .long 0 /* 0x188 */ |
1899 | .long 0 /* 0x18c */ |
1900 | .long 0 /* 0x190 */ |
1901 | .long 0 /* 0x194 */ |
1902 | .long 0 /* 0x198 */ |
1903 | .long 0 /* 0x19c */ |
1904 | .long 0 /* 0x1a0 */ |
1905 | .long 0 /* 0x1a4 */ |
1906 | .long 0 /* 0x1a8 */ |
1907 | .long 0 /* 0x1ac */ |
1908 | .long 0 /* 0x1b0 */ |
1909 | .long 0 /* 0x1b4 */ |
1910 | .long 0 /* 0x1b8 */ |
1911 | .long 0 /* 0x1bc */ |
1912 | .long 0 /* 0x1c0 */ |
1913 | .long 0 /* 0x1c4 */ |
1914 | .long 0 /* 0x1c8 */ |
1915 | .long 0 /* 0x1cc */ |
1916 | .long 0 /* 0x1d0 */ |
1917 | .long 0 /* 0x1d4 */ |
1918 | .long 0 /* 0x1d8 */ |
1919 | .long 0 /* 0x1dc */ |
1920 | .long 0 /* 0x1e0 */ |
1921 | .long 0 /* 0x1e4 */ |
1922 | .long 0 /* 0x1e8 */ |
1923 | .long 0 /* 0x1ec */ |
1924 | .long 0 /* 0x1f0 */ |
1925 | .long 0 /* 0x1f4 */ |
1926 | .long 0 /* 0x1f8 */ |
1927 | .long 0 /* 0x1fc */ |
1928 | .long 0 /* 0x200 */ |
1929 | .long 0 /* 0x204 */ |
1930 | .long 0 /* 0x208 */ |
1931 | .long 0 /* 0x20c */ |
1932 | .long 0 /* 0x210 */ |
1933 | .long 0 /* 0x214 */ |
1934 | .long 0 /* 0x218 */ |
1935 | .long 0 /* 0x21c */ |
1936 | .long 0 /* 0x220 */ |
1937 | .long 0 /* 0x224 */ |
1938 | .long 0 /* 0x228 */ |
1939 | .long 0 /* 0x22c */ |
1940 | .long 0 /* 0x230 */ |
1941 | .long 0 /* 0x234 */ |
1942 | .long 0 /* 0x238 */ |
1943 | .long 0 /* 0x23c */ |
1944 | .long 0 /* 0x240 */ |
1945 | .long 0 /* 0x244 */ |
1946 | .long 0 /* 0x248 */ |
1947 | .long 0 /* 0x24c */ |
1948 | .long 0 /* 0x250 */ |
1949 | .long 0 /* 0x254 */ |
1950 | .long 0 /* 0x258 */ |
1951 | .long 0 /* 0x25c */ |
1952 | .long 0 /* 0x260 */ |
1953 | .long 0 /* 0x264 */ |
1954 | .long 0 /* 0x268 */ |
1955 | .long 0 /* 0x26c */ |
1956 | .long 0 /* 0x270 */ |
1957 | .long 0 /* 0x274 */ |
1958 | .long 0 /* 0x278 */ |
1959 | .long 0 /* 0x27c */ |
1960 | .long 0 /* 0x280 */ |
1961 | .long 0 /* 0x284 */ |
1962 | .long 0 /* 0x288 */ |
1963 | .long 0 /* 0x28c */ |
1964 | .long 0 /* 0x290 */ |
1965 | .long 0 /* 0x294 */ |
1966 | .long 0 /* 0x298 */ |
1967 | .long 0 /* 0x29c */ |
1968 | .long 0 /* 0x2a0 */ |
1969 | .long 0 /* 0x2a4 */ |
1970 | .long 0 /* 0x2a8 */ |
1971 | .long 0 /* 0x2ac */ |
1972 | .long 0 /* 0x2b0 */ |
1973 | .long 0 /* 0x2b4 */ |
1974 | .long 0 /* 0x2b8 */ |
1975 | .long 0 /* 0x2bc */ |
1976 | .long 0 /* 0x2c0 */ |
1977 | .long 0 /* 0x2c4 */ |
1978 | .long 0 /* 0x2c8 */ |
1979 | .long 0 /* 0x2cc */ |
1980 | .long 0 /* 0x2d0 */ |
1981 | .long 0 /* 0x2d4 */ |
1982 | .long 0 /* 0x2d8 */ |
1983 | .long 0 /* 0x2dc */ |
1984 | .long 0 /* 0x2e0 */ |
1985 | .long 0 /* 0x2e4 */ |
1986 | .long 0 /* 0x2e8 */ |
1987 | .long 0 /* 0x2ec */ |
1988 | .long 0 /* 0x2f0 */ |
1989 | .long 0 /* 0x2f4 */ |
1990 | .long 0 /* 0x2f8 */ |
1991 | #ifdef CONFIG_KVM_XICS |
1992 | .long DOTSYM(xics_rm_h_xirr_x) - hcall_real_table |
1993 | #else |
1994 | .long 0 /* 0x2fc - H_XIRR_X*/ |
1995 | #endif |
1996 | .long DOTSYM(kvmppc_rm_h_random) - hcall_real_table |
1997 | .globl hcall_real_table_end |
1998 | hcall_real_table_end: |
1999 | |
2000 | _GLOBAL_TOC(kvmppc_h_set_xdabr) |
2001 | EXPORT_SYMBOL_GPL(kvmppc_h_set_xdabr) |
2002 | andi. r0, r5, DABRX_USER | DABRX_KERNEL |
2003 | beq 6f |
2004 | li r0, DABRX_USER | DABRX_KERNEL | DABRX_BTI |
2005 | andc. r0, r5, r0 |
2006 | beq 3f |
2007 | 6: li r3, H_PARAMETER |
2008 | blr |
2009 | |
2010 | _GLOBAL_TOC(kvmppc_h_set_dabr) |
2011 | EXPORT_SYMBOL_GPL(kvmppc_h_set_dabr) |
2012 | li r5, DABRX_USER | DABRX_KERNEL |
2013 | 3: |
2014 | BEGIN_FTR_SECTION |
2015 | b 2f |
2016 | END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) |
2017 | std r4,VCPU_DABR(r3) |
2018 | stw r5, VCPU_DABRX(r3) |
2019 | mtspr SPRN_DABRX, r5 |
2020 | /* Work around P7 bug where DABR can get corrupted on mtspr */ |
2021 | 1: mtspr SPRN_DABR,r4 |
2022 | mfspr r5, SPRN_DABR |
2023 | cmpd r4, r5 |
2024 | bne 1b |
2025 | isync |
2026 | li r3,0 |
2027 | blr |
2028 | |
2029 | 2: |
2030 | LOAD_REG_ADDR(r11, dawr_force_enable) |
2031 | lbz r11, 0(r11) |
2032 | cmpdi r11, 0 |
2033 | bne 3f |
2034 | li r3, H_HARDWARE |
2035 | blr |
2036 | 3: |
2037 | /* Emulate H_SET_DABR/X on P8 for the sake of compat mode guests */ |
2038 | rlwimi r5, r4, 5, DAWRX_DR | DAWRX_DW |
2039 | rlwimi r5, r4, 2, DAWRX_WT |
2040 | clrrdi r4, r4, 3 |
2041 | std r4, VCPU_DAWR0(r3) |
2042 | std r5, VCPU_DAWRX0(r3) |
2043 | /* |
2044 | * If came in through the real mode hcall handler then it is necessary |
2045 | * to write the registers since the return path won't. Otherwise it is |
2046 | * sufficient to store then in the vcpu struct as they will be loaded |
2047 | * next time the vcpu is run. |
2048 | */ |
2049 | mfmsr r6 |
2050 | andi. r6, r6, MSR_DR /* in real mode? */ |
2051 | bne 4f |
2052 | mtspr SPRN_DAWR0, r4 |
2053 | mtspr SPRN_DAWRX0, r5 |
2054 | 4: li r3, 0 |
2055 | blr |
2056 | |
2057 | _GLOBAL(kvmppc_h_cede) /* r3 = vcpu pointer, r11 = msr, r13 = paca */ |
2058 | ori r11,r11,MSR_EE |
2059 | std r11,VCPU_MSR(r3) |
2060 | li r0,1 |
2061 | stb r0,VCPU_CEDED(r3) |
2062 | sync /* order setting ceded vs. testing prodded */ |
2063 | lbz r5,VCPU_PRODDED(r3) |
2064 | cmpwi r5,0 |
2065 | bne kvm_cede_prodded |
2066 | li r12,0 /* set trap to 0 to say hcall is handled */ |
2067 | stw r12,VCPU_TRAP(r3) |
2068 | li r0,H_SUCCESS |
2069 | std r0,VCPU_GPR(R3)(r3) |
2070 | |
2071 | /* |
2072 | * Set our bit in the bitmask of napping threads unless all the |
2073 | * other threads are already napping, in which case we send this |
2074 | * up to the host. |
2075 | */ |
2076 | ld r5,HSTATE_KVM_VCORE(r13) |
2077 | lbz r6,HSTATE_PTID(r13) |
2078 | lwz r8,VCORE_ENTRY_EXIT(r5) |
2079 | clrldi r8,r8,56 |
2080 | li r0,1 |
2081 | sld r0,r0,r6 |
2082 | addi r6,r5,VCORE_NAPPING_THREADS |
2083 | 31: lwarx r4,0,r6 |
2084 | or r4,r4,r0 |
2085 | cmpw r4,r8 |
2086 | beq kvm_cede_exit |
2087 | stwcx. r4,0,r6 |
2088 | bne 31b |
2089 | /* order napping_threads update vs testing entry_exit_map */ |
2090 | isync |
2091 | li r0,NAPPING_CEDE |
2092 | stb r0,HSTATE_NAPPING(r13) |
2093 | lwz r7,VCORE_ENTRY_EXIT(r5) |
2094 | cmpwi r7,0x100 |
2095 | bge 33f /* another thread already exiting */ |
2096 | |
2097 | /* |
2098 | * Although not specifically required by the architecture, POWER7 |
2099 | * preserves the following registers in nap mode, even if an SMT mode |
2100 | * switch occurs: SLB entries, PURR, SPURR, AMOR, UAMOR, AMR, SPRG0-3, |
2101 | * DAR, DSISR, DABR, DABRX, DSCR, PMCx, MMCRx, SIAR, SDAR. |
2102 | */ |
2103 | /* Save non-volatile GPRs */ |
2104 | std r14, VCPU_GPR(R14)(r3) |
2105 | std r15, VCPU_GPR(R15)(r3) |
2106 | std r16, VCPU_GPR(R16)(r3) |
2107 | std r17, VCPU_GPR(R17)(r3) |
2108 | std r18, VCPU_GPR(R18)(r3) |
2109 | std r19, VCPU_GPR(R19)(r3) |
2110 | std r20, VCPU_GPR(R20)(r3) |
2111 | std r21, VCPU_GPR(R21)(r3) |
2112 | std r22, VCPU_GPR(R22)(r3) |
2113 | std r23, VCPU_GPR(R23)(r3) |
2114 | std r24, VCPU_GPR(R24)(r3) |
2115 | std r25, VCPU_GPR(R25)(r3) |
2116 | std r26, VCPU_GPR(R26)(r3) |
2117 | std r27, VCPU_GPR(R27)(r3) |
2118 | std r28, VCPU_GPR(R28)(r3) |
2119 | std r29, VCPU_GPR(R29)(r3) |
2120 | std r30, VCPU_GPR(R30)(r3) |
2121 | std r31, VCPU_GPR(R31)(r3) |
2122 | |
2123 | /* save FP state */ |
2124 | bl kvmppc_save_fp |
2125 | |
2126 | #ifdef CONFIG_PPC_TRANSACTIONAL_MEM |
2127 | BEGIN_FTR_SECTION |
2128 | b 91f |
2129 | END_FTR_SECTION_IFCLR(CPU_FTR_TM) |
2130 | /* |
2131 | * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR) |
2132 | */ |
2133 | ld r3, HSTATE_KVM_VCPU(r13) |
2134 | ld r4, VCPU_MSR(r3) |
2135 | li r5, 0 /* don't preserve non-vol regs */ |
2136 | bl kvmppc_save_tm_hv |
2137 | nop |
2138 | 91: |
2139 | #endif |
2140 | |
2141 | /* |
2142 | * Set DEC to the smaller of DEC and HDEC, so that we wake |
2143 | * no later than the end of our timeslice (HDEC interrupts |
2144 | * don't wake us from nap). |
2145 | */ |
2146 | mfspr r3, SPRN_DEC |
2147 | mfspr r4, SPRN_HDEC |
2148 | mftb r5 |
2149 | extsw r3, r3 |
2150 | extsw r4, r4 |
2151 | cmpd r3, r4 |
2152 | ble 67f |
2153 | mtspr SPRN_DEC, r4 |
2154 | 67: |
2155 | /* save expiry time of guest decrementer */ |
2156 | add r3, r3, r5 |
2157 | ld r4, HSTATE_KVM_VCPU(r13) |
2158 | std r3, VCPU_DEC_EXPIRES(r4) |
2159 | |
2160 | #ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING |
2161 | ld r4, HSTATE_KVM_VCPU(r13) |
2162 | addi r3, r4, VCPU_TB_CEDE |
2163 | bl kvmhv_accumulate_time |
2164 | #endif |
2165 | |
2166 | lis r3, LPCR_PECEDP@h /* Do wake on privileged doorbell */ |
2167 | |
2168 | /* Go back to host stack */ |
2169 | ld r1, HSTATE_HOST_R1(r13) |
2170 | |
2171 | /* |
2172 | * Take a nap until a decrementer or external or doobell interrupt |
2173 | * occurs, with PECE1 and PECE0 set in LPCR. |
2174 | * On POWER8, set PECEDH, and if we are ceding, also set PECEDP. |
2175 | * Also clear the runlatch bit before napping. |
2176 | */ |
2177 | kvm_do_nap: |
2178 | li r0,0 |
2179 | mtspr SPRN_CTRLT, r0 |
2180 | |
2181 | li r0,1 |
2182 | stb r0,HSTATE_HWTHREAD_REQ(r13) |
2183 | mfspr r5,SPRN_LPCR |
2184 | ori r5,r5,LPCR_PECE0 | LPCR_PECE1 |
2185 | BEGIN_FTR_SECTION |
2186 | ori r5, r5, LPCR_PECEDH |
2187 | rlwimi r5, r3, 0, LPCR_PECEDP |
2188 | END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) |
2189 | |
2190 | kvm_nap_sequence: /* desired LPCR value in r5 */ |
2191 | li r3, PNV_THREAD_NAP |
2192 | mtspr SPRN_LPCR,r5 |
2193 | isync |
2194 | |
2195 | bl isa206_idle_insn_mayloss |
2196 | |
2197 | li r0,1 |
2198 | mtspr SPRN_CTRLT, r0 |
2199 | |
2200 | mtspr SPRN_SRR1, r3 |
2201 | |
2202 | li r0, 0 |
2203 | stb r0, PACA_FTRACE_ENABLED(r13) |
2204 | |
2205 | li r0, KVM_HWTHREAD_IN_KVM |
2206 | stb r0, HSTATE_HWTHREAD_STATE(r13) |
2207 | |
2208 | lbz r0, HSTATE_NAPPING(r13) |
2209 | cmpwi r0, NAPPING_CEDE |
2210 | beq kvm_end_cede |
2211 | cmpwi r0, NAPPING_NOVCPU |
2212 | beq kvm_novcpu_wakeup |
2213 | cmpwi r0, NAPPING_UNSPLIT |
2214 | beq kvm_unsplit_wakeup |
2215 | twi 31,0,0 /* Nap state must not be zero */ |
2216 | |
2217 | 33: mr r4, r3 |
2218 | li r3, 0 |
2219 | li r12, 0 |
2220 | b 34f |
2221 | |
2222 | kvm_end_cede: |
2223 | /* Woken by external or decrementer interrupt */ |
2224 | |
2225 | /* get vcpu pointer */ |
2226 | ld r4, HSTATE_KVM_VCPU(r13) |
2227 | |
2228 | #ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING |
2229 | addi r3, r4, VCPU_TB_RMINTR |
2230 | bl kvmhv_accumulate_time |
2231 | #endif |
2232 | |
2233 | #ifdef CONFIG_PPC_TRANSACTIONAL_MEM |
2234 | BEGIN_FTR_SECTION |
2235 | b 91f |
2236 | END_FTR_SECTION_IFCLR(CPU_FTR_TM) |
2237 | /* |
2238 | * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR) |
2239 | */ |
2240 | mr r3, r4 |
2241 | ld r4, VCPU_MSR(r3) |
2242 | li r5, 0 /* don't preserve non-vol regs */ |
2243 | bl kvmppc_restore_tm_hv |
2244 | nop |
2245 | ld r4, HSTATE_KVM_VCPU(r13) |
2246 | 91: |
2247 | #endif |
2248 | |
2249 | /* load up FP state */ |
2250 | bl kvmppc_load_fp |
2251 | |
2252 | /* Restore guest decrementer */ |
2253 | ld r3, VCPU_DEC_EXPIRES(r4) |
2254 | mftb r7 |
2255 | subf r3, r7, r3 |
2256 | mtspr SPRN_DEC, r3 |
2257 | |
2258 | /* Load NV GPRS */ |
2259 | ld r14, VCPU_GPR(R14)(r4) |
2260 | ld r15, VCPU_GPR(R15)(r4) |
2261 | ld r16, VCPU_GPR(R16)(r4) |
2262 | ld r17, VCPU_GPR(R17)(r4) |
2263 | ld r18, VCPU_GPR(R18)(r4) |
2264 | ld r19, VCPU_GPR(R19)(r4) |
2265 | ld r20, VCPU_GPR(R20)(r4) |
2266 | ld r21, VCPU_GPR(R21)(r4) |
2267 | ld r22, VCPU_GPR(R22)(r4) |
2268 | ld r23, VCPU_GPR(R23)(r4) |
2269 | ld r24, VCPU_GPR(R24)(r4) |
2270 | ld r25, VCPU_GPR(R25)(r4) |
2271 | ld r26, VCPU_GPR(R26)(r4) |
2272 | ld r27, VCPU_GPR(R27)(r4) |
2273 | ld r28, VCPU_GPR(R28)(r4) |
2274 | ld r29, VCPU_GPR(R29)(r4) |
2275 | ld r30, VCPU_GPR(R30)(r4) |
2276 | ld r31, VCPU_GPR(R31)(r4) |
2277 | |
2278 | /* Check the wake reason in SRR1 to see why we got here */ |
2279 | bl kvmppc_check_wake_reason |
2280 | |
2281 | /* |
2282 | * Restore volatile registers since we could have called a |
2283 | * C routine in kvmppc_check_wake_reason |
2284 | * r4 = VCPU |
2285 | * r3 tells us whether we need to return to host or not |
2286 | * WARNING: it gets checked further down: |
2287 | * should not modify r3 until this check is done. |
2288 | */ |
2289 | ld r4, HSTATE_KVM_VCPU(r13) |
2290 | |
2291 | /* clear our bit in vcore->napping_threads */ |
2292 | 34: ld r5,HSTATE_KVM_VCORE(r13) |
2293 | lbz r7,HSTATE_PTID(r13) |
2294 | li r0,1 |
2295 | sld r0,r0,r7 |
2296 | addi r6,r5,VCORE_NAPPING_THREADS |
2297 | 32: lwarx r7,0,r6 |
2298 | andc r7,r7,r0 |
2299 | stwcx. r7,0,r6 |
2300 | bne 32b |
2301 | li r0,0 |
2302 | stb r0,HSTATE_NAPPING(r13) |
2303 | |
2304 | /* See if the wake reason saved in r3 means we need to exit */ |
2305 | stw r12, VCPU_TRAP(r4) |
2306 | mr r9, r4 |
2307 | cmpdi r3, 0 |
2308 | bgt guest_exit_cont |
2309 | b maybe_reenter_guest |
2310 | |
2311 | /* cede when already previously prodded case */ |
2312 | kvm_cede_prodded: |
2313 | li r0,0 |
2314 | stb r0,VCPU_PRODDED(r3) |
2315 | sync /* order testing prodded vs. clearing ceded */ |
2316 | stb r0,VCPU_CEDED(r3) |
2317 | li r3,H_SUCCESS |
2318 | blr |
2319 | |
2320 | /* we've ceded but we want to give control to the host */ |
2321 | kvm_cede_exit: |
2322 | ld r9, HSTATE_KVM_VCPU(r13) |
2323 | b guest_exit_cont |
2324 | |
2325 | /* Try to do machine check recovery in real mode */ |
2326 | machine_check_realmode: |
2327 | mr r3, r9 /* get vcpu pointer */ |
2328 | bl kvmppc_realmode_machine_check |
2329 | nop |
2330 | /* all machine checks go to virtual mode for further handling */ |
2331 | ld r9, HSTATE_KVM_VCPU(r13) |
2332 | li r12, BOOK3S_INTERRUPT_MACHINE_CHECK |
2333 | b guest_exit_cont |
2334 | |
2335 | /* |
2336 | * Call C code to handle a HMI in real mode. |
2337 | * Only the primary thread does the call, secondary threads are handled |
2338 | * by calling hmi_exception_realmode() after kvmppc_hv_entry returns. |
2339 | * r9 points to the vcpu on entry |
2340 | */ |
2341 | hmi_realmode: |
2342 | lbz r0, HSTATE_PTID(r13) |
2343 | cmpwi r0, 0 |
2344 | bne guest_exit_cont |
2345 | bl CFUNC(kvmppc_realmode_hmi_handler) |
2346 | ld r9, HSTATE_KVM_VCPU(r13) |
2347 | li r12, BOOK3S_INTERRUPT_HMI |
2348 | b guest_exit_cont |
2349 | |
2350 | /* |
2351 | * Check the reason we woke from nap, and take appropriate action. |
2352 | * Returns (in r3): |
2353 | * 0 if nothing needs to be done |
2354 | * 1 if something happened that needs to be handled by the host |
2355 | * -1 if there was a guest wakeup (IPI or msgsnd) |
2356 | * -2 if we handled a PCI passthrough interrupt (returned by |
2357 | * kvmppc_read_intr only) |
2358 | * |
2359 | * Also sets r12 to the interrupt vector for any interrupt that needs |
2360 | * to be handled now by the host (0x500 for external interrupt), or zero. |
2361 | * Modifies all volatile registers (since it may call a C function). |
2362 | * This routine calls kvmppc_read_intr, a C function, if an external |
2363 | * interrupt is pending. |
2364 | */ |
2365 | SYM_FUNC_START_LOCAL(kvmppc_check_wake_reason) |
2366 | mfspr r6, SPRN_SRR1 |
2367 | BEGIN_FTR_SECTION |
2368 | rlwinm r6, r6, 45-31, 0xf /* extract wake reason field (P8) */ |
2369 | FTR_SECTION_ELSE |
2370 | rlwinm r6, r6, 45-31, 0xe /* P7 wake reason field is 3 bits */ |
2371 | ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S) |
2372 | cmpwi r6, 8 /* was it an external interrupt? */ |
2373 | beq 7f /* if so, see what it was */ |
2374 | li r3, 0 |
2375 | li r12, 0 |
2376 | cmpwi r6, 6 /* was it the decrementer? */ |
2377 | beq 0f |
2378 | BEGIN_FTR_SECTION |
2379 | cmpwi r6, 5 /* privileged doorbell? */ |
2380 | beq 0f |
2381 | cmpwi r6, 3 /* hypervisor doorbell? */ |
2382 | beq 3f |
2383 | END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) |
2384 | cmpwi r6, 0xa /* Hypervisor maintenance ? */ |
2385 | beq 4f |
2386 | li r3, 1 /* anything else, return 1 */ |
2387 | 0: blr |
2388 | |
2389 | /* hypervisor doorbell */ |
2390 | 3: li r12, BOOK3S_INTERRUPT_H_DOORBELL |
2391 | |
2392 | /* |
2393 | * Clear the doorbell as we will invoke the handler |
2394 | * explicitly in the guest exit path. |
2395 | */ |
2396 | lis r6, (PPC_DBELL_SERVER << (63-36))@h |
2397 | PPC_MSGCLR(6) |
2398 | /* see if it's a host IPI */ |
2399 | li r3, 1 |
2400 | lbz r0, HSTATE_HOST_IPI(r13) |
2401 | cmpwi r0, 0 |
2402 | bnelr |
2403 | /* if not, return -1 */ |
2404 | li r3, -1 |
2405 | blr |
2406 | |
2407 | /* Woken up due to Hypervisor maintenance interrupt */ |
2408 | 4: li r12, BOOK3S_INTERRUPT_HMI |
2409 | li r3, 1 |
2410 | blr |
2411 | |
2412 | /* external interrupt - create a stack frame so we can call C */ |
2413 | 7: mflr r0 |
2414 | std r0, PPC_LR_STKOFF(r1) |
2415 | stdu r1, -PPC_MIN_STKFRM(r1) |
2416 | bl CFUNC(kvmppc_read_intr) |
2417 | nop |
2418 | li r12, BOOK3S_INTERRUPT_EXTERNAL |
2419 | cmpdi r3, 1 |
2420 | ble 1f |
2421 | |
2422 | /* |
2423 | * Return code of 2 means PCI passthrough interrupt, but |
2424 | * we need to return back to host to complete handling the |
2425 | * interrupt. Trap reason is expected in r12 by guest |
2426 | * exit code. |
2427 | */ |
2428 | li r12, BOOK3S_INTERRUPT_HV_RM_HARD |
2429 | 1: |
2430 | ld r0, PPC_MIN_STKFRM+PPC_LR_STKOFF(r1) |
2431 | addi r1, r1, PPC_MIN_STKFRM |
2432 | mtlr r0 |
2433 | blr |
2434 | SYM_FUNC_END(kvmppc_check_wake_reason) |
2435 | |
2436 | /* |
2437 | * Save away FP, VMX and VSX registers. |
2438 | * r3 = vcpu pointer |
2439 | * N.B. r30 and r31 are volatile across this function, |
2440 | * thus it is not callable from C. |
2441 | */ |
2442 | SYM_FUNC_START_LOCAL(kvmppc_save_fp) |
2443 | mflr r30 |
2444 | mr r31,r3 |
2445 | mfmsr r5 |
2446 | ori r8,r5,MSR_FP |
2447 | #ifdef CONFIG_ALTIVEC |
2448 | BEGIN_FTR_SECTION |
2449 | oris r8,r8,MSR_VEC@h |
2450 | END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) |
2451 | #endif |
2452 | #ifdef CONFIG_VSX |
2453 | BEGIN_FTR_SECTION |
2454 | oris r8,r8,MSR_VSX@h |
2455 | END_FTR_SECTION_IFSET(CPU_FTR_VSX) |
2456 | #endif |
2457 | mtmsrd r8 |
2458 | addi r3,r3,VCPU_FPRS |
2459 | bl store_fp_state |
2460 | #ifdef CONFIG_ALTIVEC |
2461 | BEGIN_FTR_SECTION |
2462 | addi r3,r31,VCPU_VRS |
2463 | bl store_vr_state |
2464 | END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) |
2465 | #endif |
2466 | mfspr r6,SPRN_VRSAVE |
2467 | stw r6,VCPU_VRSAVE(r31) |
2468 | mtlr r30 |
2469 | blr |
2470 | SYM_FUNC_END(kvmppc_save_fp) |
2471 | |
2472 | /* |
2473 | * Load up FP, VMX and VSX registers |
2474 | * r4 = vcpu pointer |
2475 | * N.B. r30 and r31 are volatile across this function, |
2476 | * thus it is not callable from C. |
2477 | */ |
2478 | SYM_FUNC_START_LOCAL(kvmppc_load_fp) |
2479 | mflr r30 |
2480 | mr r31,r4 |
2481 | mfmsr r9 |
2482 | ori r8,r9,MSR_FP |
2483 | #ifdef CONFIG_ALTIVEC |
2484 | BEGIN_FTR_SECTION |
2485 | oris r8,r8,MSR_VEC@h |
2486 | END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) |
2487 | #endif |
2488 | #ifdef CONFIG_VSX |
2489 | BEGIN_FTR_SECTION |
2490 | oris r8,r8,MSR_VSX@h |
2491 | END_FTR_SECTION_IFSET(CPU_FTR_VSX) |
2492 | #endif |
2493 | mtmsrd r8 |
2494 | addi r3,r4,VCPU_FPRS |
2495 | bl load_fp_state |
2496 | #ifdef CONFIG_ALTIVEC |
2497 | BEGIN_FTR_SECTION |
2498 | addi r3,r31,VCPU_VRS |
2499 | bl load_vr_state |
2500 | END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) |
2501 | #endif |
2502 | lwz r7,VCPU_VRSAVE(r31) |
2503 | mtspr SPRN_VRSAVE,r7 |
2504 | mtlr r30 |
2505 | mr r4,r31 |
2506 | blr |
2507 | SYM_FUNC_END(kvmppc_load_fp) |
2508 | |
2509 | #ifdef CONFIG_PPC_TRANSACTIONAL_MEM |
2510 | /* |
2511 | * Save transactional state and TM-related registers. |
2512 | * Called with r3 pointing to the vcpu struct and r4 containing |
2513 | * the guest MSR value. |
2514 | * r5 is non-zero iff non-volatile register state needs to be maintained. |
2515 | * If r5 == 0, this can modify all checkpointed registers, but |
2516 | * restores r1 and r2 before exit. |
2517 | */ |
2518 | _GLOBAL_TOC(kvmppc_save_tm_hv) |
2519 | EXPORT_SYMBOL_GPL(kvmppc_save_tm_hv) |
2520 | /* See if we need to handle fake suspend mode */ |
2521 | BEGIN_FTR_SECTION |
2522 | b __kvmppc_save_tm |
2523 | END_FTR_SECTION_IFCLR(CPU_FTR_P9_TM_HV_ASSIST) |
2524 | |
2525 | lbz r0, HSTATE_FAKE_SUSPEND(r13) /* Were we fake suspended? */ |
2526 | cmpwi r0, 0 |
2527 | beq __kvmppc_save_tm |
2528 | |
2529 | /* The following code handles the fake_suspend = 1 case */ |
2530 | mflr r0 |
2531 | std r0, PPC_LR_STKOFF(r1) |
2532 | stdu r1, -TM_FRAME_SIZE(r1) |
2533 | |
2534 | /* Turn on TM. */ |
2535 | mfmsr r8 |
2536 | li r0, 1 |
2537 | rldimi r8, r0, MSR_TM_LG, 63-MSR_TM_LG |
2538 | mtmsrd r8 |
2539 | |
2540 | rldicl. r8, r8, 64 - MSR_TS_S_LG, 62 /* Did we actually hrfid? */ |
2541 | beq 4f |
2542 | BEGIN_FTR_SECTION |
2543 | bl pnv_power9_force_smt4_catch |
2544 | END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_XER_SO_BUG) |
2545 | nop |
2546 | |
2547 | /* |
2548 | * It's possible that treclaim. may modify registers, if we have lost |
2549 | * track of fake-suspend state in the guest due to it using rfscv. |
2550 | * Save and restore registers in case this occurs. |
2551 | */ |
2552 | mfspr r3, SPRN_DSCR |
2553 | mfspr r4, SPRN_XER |
2554 | mfspr r5, SPRN_AMR |
2555 | /* SPRN_TAR would need to be saved here if the kernel ever used it */ |
2556 | mfcr r12 |
2557 | SAVE_NVGPRS(r1) |
2558 | SAVE_GPR(2, r1) |
2559 | SAVE_GPR(3, r1) |
2560 | SAVE_GPR(4, r1) |
2561 | SAVE_GPR(5, r1) |
2562 | stw r12, 8(r1) |
2563 | std r1, HSTATE_HOST_R1(r13) |
2564 | |
2565 | /* We have to treclaim here because that's the only way to do S->N */ |
2566 | li r3, TM_CAUSE_KVM_RESCHED |
2567 | TRECLAIM(R3) |
2568 | |
2569 | GET_PACA(r13) |
2570 | ld r1, HSTATE_HOST_R1(r13) |
2571 | REST_GPR(2, r1) |
2572 | REST_GPR(3, r1) |
2573 | REST_GPR(4, r1) |
2574 | REST_GPR(5, r1) |
2575 | lwz r12, 8(r1) |
2576 | REST_NVGPRS(r1) |
2577 | mtspr SPRN_DSCR, r3 |
2578 | mtspr SPRN_XER, r4 |
2579 | mtspr SPRN_AMR, r5 |
2580 | mtcr r12 |
2581 | HMT_MEDIUM |
2582 | |
2583 | /* |
2584 | * We were in fake suspend, so we are not going to save the |
2585 | * register state as the guest checkpointed state (since |
2586 | * we already have it), therefore we can now use any volatile GPR. |
2587 | * In fact treclaim in fake suspend state doesn't modify |
2588 | * any registers. |
2589 | */ |
2590 | |
2591 | BEGIN_FTR_SECTION |
2592 | bl pnv_power9_force_smt4_release |
2593 | END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_XER_SO_BUG) |
2594 | nop |
2595 | |
2596 | 4: |
2597 | mfspr r3, SPRN_PSSCR |
2598 | /* PSSCR_FAKE_SUSPEND is a write-only bit, but clear it anyway */ |
2599 | li r0, PSSCR_FAKE_SUSPEND |
2600 | andc r3, r3, r0 |
2601 | mtspr SPRN_PSSCR, r3 |
2602 | |
2603 | /* Don't save TEXASR, use value from last exit in real suspend state */ |
2604 | ld r9, HSTATE_KVM_VCPU(r13) |
2605 | mfspr r5, SPRN_TFHAR |
2606 | mfspr r6, SPRN_TFIAR |
2607 | std r5, VCPU_TFHAR(r9) |
2608 | std r6, VCPU_TFIAR(r9) |
2609 | |
2610 | addi r1, r1, TM_FRAME_SIZE |
2611 | ld r0, PPC_LR_STKOFF(r1) |
2612 | mtlr r0 |
2613 | blr |
2614 | |
2615 | /* |
2616 | * Restore transactional state and TM-related registers. |
2617 | * Called with r3 pointing to the vcpu struct |
2618 | * and r4 containing the guest MSR value. |
2619 | * r5 is non-zero iff non-volatile register state needs to be maintained. |
2620 | * This potentially modifies all checkpointed registers. |
2621 | * It restores r1 and r2 from the PACA. |
2622 | */ |
2623 | _GLOBAL_TOC(kvmppc_restore_tm_hv) |
2624 | EXPORT_SYMBOL_GPL(kvmppc_restore_tm_hv) |
2625 | /* |
2626 | * If we are doing TM emulation for the guest on a POWER9 DD2, |
2627 | * then we don't actually do a trechkpt -- we either set up |
2628 | * fake-suspend mode, or emulate a TM rollback. |
2629 | */ |
2630 | BEGIN_FTR_SECTION |
2631 | b __kvmppc_restore_tm |
2632 | END_FTR_SECTION_IFCLR(CPU_FTR_P9_TM_HV_ASSIST) |
2633 | mflr r0 |
2634 | std r0, PPC_LR_STKOFF(r1) |
2635 | |
2636 | li r0, 0 |
2637 | stb r0, HSTATE_FAKE_SUSPEND(r13) |
2638 | |
2639 | /* Turn on TM so we can restore TM SPRs */ |
2640 | mfmsr r5 |
2641 | li r0, 1 |
2642 | rldimi r5, r0, MSR_TM_LG, 63-MSR_TM_LG |
2643 | mtmsrd r5 |
2644 | |
2645 | /* |
2646 | * The user may change these outside of a transaction, so they must |
2647 | * always be context switched. |
2648 | */ |
2649 | ld r5, VCPU_TFHAR(r3) |
2650 | ld r6, VCPU_TFIAR(r3) |
2651 | ld r7, VCPU_TEXASR(r3) |
2652 | mtspr SPRN_TFHAR, r5 |
2653 | mtspr SPRN_TFIAR, r6 |
2654 | mtspr SPRN_TEXASR, r7 |
2655 | |
2656 | rldicl. r5, r4, 64 - MSR_TS_S_LG, 62 |
2657 | beqlr /* TM not active in guest */ |
2658 | |
2659 | /* Make sure the failure summary is set */ |
2660 | oris r7, r7, (TEXASR_FS)@h |
2661 | mtspr SPRN_TEXASR, r7 |
2662 | |
2663 | cmpwi r5, 1 /* check for suspended state */ |
2664 | bgt 10f |
2665 | stb r5, HSTATE_FAKE_SUSPEND(r13) |
2666 | b 9f /* and return */ |
2667 | 10: stdu r1, -PPC_MIN_STKFRM(r1) |
2668 | /* guest is in transactional state, so simulate rollback */ |
2669 | bl kvmhv_emulate_tm_rollback |
2670 | nop |
2671 | addi r1, r1, PPC_MIN_STKFRM |
2672 | 9: ld r0, PPC_LR_STKOFF(r1) |
2673 | mtlr r0 |
2674 | blr |
2675 | #endif /* CONFIG_PPC_TRANSACTIONAL_MEM */ |
2676 | |
2677 | /* |
2678 | * We come here if we get any exception or interrupt while we are |
2679 | * executing host real mode code while in guest MMU context. |
2680 | * r12 is (CR << 32) | vector |
2681 | * r13 points to our PACA |
2682 | * r12 is saved in HSTATE_SCRATCH0(r13) |
2683 | * r9 is saved in HSTATE_SCRATCH2(r13) |
2684 | * r13 is saved in HSPRG1 |
2685 | * cfar is saved in HSTATE_CFAR(r13) |
2686 | * ppr is saved in HSTATE_PPR(r13) |
2687 | */ |
2688 | kvmppc_bad_host_intr: |
2689 | /* |
2690 | * Switch to the emergency stack, but start half-way down in |
2691 | * case we were already on it. |
2692 | */ |
2693 | mr r9, r1 |
2694 | std r1, PACAR1(r13) |
2695 | ld r1, PACAEMERGSP(r13) |
2696 | subi r1, r1, THREAD_SIZE/2 + INT_FRAME_SIZE |
2697 | std r9, 0(r1) |
2698 | std r0, GPR0(r1) |
2699 | std r9, GPR1(r1) |
2700 | std r2, GPR2(r1) |
2701 | SAVE_GPRS(3, 8, r1) |
2702 | srdi r0, r12, 32 |
2703 | clrldi r12, r12, 32 |
2704 | std r0, _CCR(r1) |
2705 | std r12, _TRAP(r1) |
2706 | andi. r0, r12, 2 |
2707 | beq 1f |
2708 | mfspr r3, SPRN_HSRR0 |
2709 | mfspr r4, SPRN_HSRR1 |
2710 | mfspr r5, SPRN_HDAR |
2711 | mfspr r6, SPRN_HDSISR |
2712 | b 2f |
2713 | 1: mfspr r3, SPRN_SRR0 |
2714 | mfspr r4, SPRN_SRR1 |
2715 | mfspr r5, SPRN_DAR |
2716 | mfspr r6, SPRN_DSISR |
2717 | 2: std r3, _NIP(r1) |
2718 | std r4, _MSR(r1) |
2719 | std r5, _DAR(r1) |
2720 | std r6, _DSISR(r1) |
2721 | ld r9, HSTATE_SCRATCH2(r13) |
2722 | ld r12, HSTATE_SCRATCH0(r13) |
2723 | GET_SCRATCH0(r0) |
2724 | SAVE_GPRS(9, 12, r1) |
2725 | std r0, GPR13(r1) |
2726 | SAVE_NVGPRS(r1) |
2727 | ld r5, HSTATE_CFAR(r13) |
2728 | std r5, ORIG_GPR3(r1) |
2729 | mflr r3 |
2730 | mfctr r4 |
2731 | mfxer r5 |
2732 | lbz r6, PACAIRQSOFTMASK(r13) |
2733 | std r3, _LINK(r1) |
2734 | std r4, _CTR(r1) |
2735 | std r5, _XER(r1) |
2736 | std r6, SOFTE(r1) |
2737 | LOAD_PACA_TOC() |
2738 | LOAD_REG_IMMEDIATE(3, STACK_FRAME_REGS_MARKER) |
2739 | std r3, STACK_INT_FRAME_MARKER(r1) |
2740 | |
2741 | /* |
2742 | * XXX On POWER7 and POWER8, we just spin here since we don't |
2743 | * know what the other threads are doing (and we don't want to |
2744 | * coordinate with them) - but at least we now have register state |
2745 | * in memory that we might be able to look at from another CPU. |
2746 | */ |
2747 | b . |
2748 | |
2749 | /* |
2750 | * This mimics the MSR transition on IRQ delivery. The new guest MSR is taken |
2751 | * from VCPU_INTR_MSR and is modified based on the required TM state changes. |
2752 | * r11 has the guest MSR value (in/out) |
2753 | * r9 has a vcpu pointer (in) |
2754 | * r0 is used as a scratch register |
2755 | */ |
2756 | SYM_FUNC_START_LOCAL(kvmppc_msr_interrupt) |
2757 | rldicl r0, r11, 64 - MSR_TS_S_LG, 62 |
2758 | cmpwi r0, 2 /* Check if we are in transactional state.. */ |
2759 | ld r11, VCPU_INTR_MSR(r9) |
2760 | bne 1f |
2761 | /* ... if transactional, change to suspended */ |
2762 | li r0, 1 |
2763 | 1: rldimi r11, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG |
2764 | blr |
2765 | SYM_FUNC_END(kvmppc_msr_interrupt) |
2766 | |
2767 | /* |
2768 | * void kvmhv_load_guest_pmu(struct kvm_vcpu *vcpu) |
2769 | * |
2770 | * Load up guest PMU state. R3 points to the vcpu struct. |
2771 | */ |
2772 | SYM_FUNC_START_LOCAL(kvmhv_load_guest_pmu) |
2773 | mr r4, r3 |
2774 | mflr r0 |
2775 | li r3, 1 |
2776 | sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */ |
2777 | mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */ |
2778 | isync |
2779 | BEGIN_FTR_SECTION |
2780 | ld r3, VCPU_MMCR(r4) |
2781 | andi. r5, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO |
2782 | cmpwi r5, MMCR0_PMAO |
2783 | beql kvmppc_fix_pmao |
2784 | END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG) |
2785 | lwz r3, VCPU_PMC(r4) /* always load up guest PMU registers */ |
2786 | lwz r5, VCPU_PMC + 4(r4) /* to prevent information leak */ |
2787 | lwz r6, VCPU_PMC + 8(r4) |
2788 | lwz r7, VCPU_PMC + 12(r4) |
2789 | lwz r8, VCPU_PMC + 16(r4) |
2790 | lwz r9, VCPU_PMC + 20(r4) |
2791 | mtspr SPRN_PMC1, r3 |
2792 | mtspr SPRN_PMC2, r5 |
2793 | mtspr SPRN_PMC3, r6 |
2794 | mtspr SPRN_PMC4, r7 |
2795 | mtspr SPRN_PMC5, r8 |
2796 | mtspr SPRN_PMC6, r9 |
2797 | ld r3, VCPU_MMCR(r4) |
2798 | ld r5, VCPU_MMCR + 8(r4) |
2799 | ld r6, VCPU_MMCRA(r4) |
2800 | ld r7, VCPU_SIAR(r4) |
2801 | ld r8, VCPU_SDAR(r4) |
2802 | mtspr SPRN_MMCR1, r5 |
2803 | mtspr SPRN_MMCRA, r6 |
2804 | mtspr SPRN_SIAR, r7 |
2805 | mtspr SPRN_SDAR, r8 |
2806 | BEGIN_FTR_SECTION |
2807 | ld r5, VCPU_MMCR + 16(r4) |
2808 | ld r6, VCPU_SIER(r4) |
2809 | mtspr SPRN_MMCR2, r5 |
2810 | mtspr SPRN_SIER, r6 |
2811 | lwz r7, VCPU_PMC + 24(r4) |
2812 | lwz r8, VCPU_PMC + 28(r4) |
2813 | ld r9, VCPU_MMCRS(r4) |
2814 | mtspr SPRN_SPMC1, r7 |
2815 | mtspr SPRN_SPMC2, r8 |
2816 | mtspr SPRN_MMCRS, r9 |
2817 | END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) |
2818 | mtspr SPRN_MMCR0, r3 |
2819 | isync |
2820 | mtlr r0 |
2821 | blr |
2822 | SYM_FUNC_END(kvmhv_load_guest_pmu) |
2823 | |
2824 | /* |
2825 | * void kvmhv_load_host_pmu(void) |
2826 | * |
2827 | * Reload host PMU state saved in the PACA by kvmhv_save_host_pmu. |
2828 | */ |
2829 | SYM_FUNC_START_LOCAL(kvmhv_load_host_pmu) |
2830 | mflr r0 |
2831 | lbz r4, PACA_PMCINUSE(r13) /* is the host using the PMU? */ |
2832 | cmpwi r4, 0 |
2833 | beq 23f /* skip if not */ |
2834 | BEGIN_FTR_SECTION |
2835 | ld r3, HSTATE_MMCR0(r13) |
2836 | andi. r4, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO |
2837 | cmpwi r4, MMCR0_PMAO |
2838 | beql kvmppc_fix_pmao |
2839 | END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG) |
2840 | lwz r3, HSTATE_PMC1(r13) |
2841 | lwz r4, HSTATE_PMC2(r13) |
2842 | lwz r5, HSTATE_PMC3(r13) |
2843 | lwz r6, HSTATE_PMC4(r13) |
2844 | lwz r8, HSTATE_PMC5(r13) |
2845 | lwz r9, HSTATE_PMC6(r13) |
2846 | mtspr SPRN_PMC1, r3 |
2847 | mtspr SPRN_PMC2, r4 |
2848 | mtspr SPRN_PMC3, r5 |
2849 | mtspr SPRN_PMC4, r6 |
2850 | mtspr SPRN_PMC5, r8 |
2851 | mtspr SPRN_PMC6, r9 |
2852 | ld r3, HSTATE_MMCR0(r13) |
2853 | ld r4, HSTATE_MMCR1(r13) |
2854 | ld r5, HSTATE_MMCRA(r13) |
2855 | ld r6, HSTATE_SIAR(r13) |
2856 | ld r7, HSTATE_SDAR(r13) |
2857 | mtspr SPRN_MMCR1, r4 |
2858 | mtspr SPRN_MMCRA, r5 |
2859 | mtspr SPRN_SIAR, r6 |
2860 | mtspr SPRN_SDAR, r7 |
2861 | BEGIN_FTR_SECTION |
2862 | ld r8, HSTATE_MMCR2(r13) |
2863 | ld r9, HSTATE_SIER(r13) |
2864 | mtspr SPRN_MMCR2, r8 |
2865 | mtspr SPRN_SIER, r9 |
2866 | END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) |
2867 | mtspr SPRN_MMCR0, r3 |
2868 | isync |
2869 | mtlr r0 |
2870 | 23: blr |
2871 | SYM_FUNC_END(kvmhv_load_host_pmu) |
2872 | |
2873 | /* |
2874 | * void kvmhv_save_guest_pmu(struct kvm_vcpu *vcpu, bool pmu_in_use) |
2875 | * |
2876 | * Save guest PMU state into the vcpu struct. |
2877 | * r3 = vcpu, r4 = full save flag (PMU in use flag set in VPA) |
2878 | */ |
2879 | SYM_FUNC_START_LOCAL(kvmhv_save_guest_pmu) |
2880 | mr r9, r3 |
2881 | mr r8, r4 |
2882 | BEGIN_FTR_SECTION |
2883 | /* |
2884 | * POWER8 seems to have a hardware bug where setting |
2885 | * MMCR0[PMAE] along with MMCR0[PMC1CE] and/or MMCR0[PMCjCE] |
2886 | * when some counters are already negative doesn't seem |
2887 | * to cause a performance monitor alert (and hence interrupt). |
2888 | * The effect of this is that when saving the PMU state, |
2889 | * if there is no PMU alert pending when we read MMCR0 |
2890 | * before freezing the counters, but one becomes pending |
2891 | * before we read the counters, we lose it. |
2892 | * To work around this, we need a way to freeze the counters |
2893 | * before reading MMCR0. Normally, freezing the counters |
2894 | * is done by writing MMCR0 (to set MMCR0[FC]) which |
2895 | * unavoidably writes MMCR0[PMA0] as well. On POWER8, |
2896 | * we can also freeze the counters using MMCR2, by writing |
2897 | * 1s to all the counter freeze condition bits (there are |
2898 | * 9 bits each for 6 counters). |
2899 | */ |
2900 | li r3, -1 /* set all freeze bits */ |
2901 | clrrdi r3, r3, 10 |
2902 | mfspr r10, SPRN_MMCR2 |
2903 | mtspr SPRN_MMCR2, r3 |
2904 | isync |
2905 | END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) |
2906 | li r3, 1 |
2907 | sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */ |
2908 | mfspr r4, SPRN_MMCR0 /* save MMCR0 */ |
2909 | mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */ |
2910 | mfspr r6, SPRN_MMCRA |
2911 | /* Clear MMCRA in order to disable SDAR updates */ |
2912 | li r7, 0 |
2913 | mtspr SPRN_MMCRA, r7 |
2914 | isync |
2915 | cmpwi r8, 0 /* did they ask for PMU stuff to be saved? */ |
2916 | bne 21f |
2917 | std r3, VCPU_MMCR(r9) /* if not, set saved MMCR0 to FC */ |
2918 | b 22f |
2919 | 21: mfspr r5, SPRN_MMCR1 |
2920 | mfspr r7, SPRN_SIAR |
2921 | mfspr r8, SPRN_SDAR |
2922 | std r4, VCPU_MMCR(r9) |
2923 | std r5, VCPU_MMCR + 8(r9) |
2924 | std r6, VCPU_MMCRA(r9) |
2925 | BEGIN_FTR_SECTION |
2926 | std r10, VCPU_MMCR + 16(r9) |
2927 | END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) |
2928 | std r7, VCPU_SIAR(r9) |
2929 | std r8, VCPU_SDAR(r9) |
2930 | mfspr r3, SPRN_PMC1 |
2931 | mfspr r4, SPRN_PMC2 |
2932 | mfspr r5, SPRN_PMC3 |
2933 | mfspr r6, SPRN_PMC4 |
2934 | mfspr r7, SPRN_PMC5 |
2935 | mfspr r8, SPRN_PMC6 |
2936 | stw r3, VCPU_PMC(r9) |
2937 | stw r4, VCPU_PMC + 4(r9) |
2938 | stw r5, VCPU_PMC + 8(r9) |
2939 | stw r6, VCPU_PMC + 12(r9) |
2940 | stw r7, VCPU_PMC + 16(r9) |
2941 | stw r8, VCPU_PMC + 20(r9) |
2942 | BEGIN_FTR_SECTION |
2943 | mfspr r5, SPRN_SIER |
2944 | std r5, VCPU_SIER(r9) |
2945 | mfspr r6, SPRN_SPMC1 |
2946 | mfspr r7, SPRN_SPMC2 |
2947 | mfspr r8, SPRN_MMCRS |
2948 | stw r6, VCPU_PMC + 24(r9) |
2949 | stw r7, VCPU_PMC + 28(r9) |
2950 | std r8, VCPU_MMCRS(r9) |
2951 | lis r4, 0x8000 |
2952 | mtspr SPRN_MMCRS, r4 |
2953 | END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) |
2954 | 22: blr |
2955 | SYM_FUNC_END(kvmhv_save_guest_pmu) |
2956 | |
2957 | /* |
2958 | * This works around a hardware bug on POWER8E processors, where |
2959 | * writing a 1 to the MMCR0[PMAO] bit doesn't generate a |
2960 | * performance monitor interrupt. Instead, when we need to have |
2961 | * an interrupt pending, we have to arrange for a counter to overflow. |
2962 | */ |
2963 | kvmppc_fix_pmao: |
2964 | li r3, 0 |
2965 | mtspr SPRN_MMCR2, r3 |
2966 | lis r3, (MMCR0_PMXE | MMCR0_FCECE)@h |
2967 | ori r3, r3, MMCR0_PMCjCE | MMCR0_C56RUN |
2968 | mtspr SPRN_MMCR0, r3 |
2969 | lis r3, 0x7fff |
2970 | ori r3, r3, 0xffff |
2971 | mtspr SPRN_PMC6, r3 |
2972 | isync |
2973 | blr |
2974 | |
2975 | #ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING |
2976 | /* |
2977 | * Start timing an activity |
2978 | * r3 = pointer to time accumulation struct, r4 = vcpu |
2979 | */ |
2980 | kvmhv_start_timing: |
2981 | ld r5, HSTATE_KVM_VCORE(r13) |
2982 | ld r6, VCORE_TB_OFFSET_APPL(r5) |
2983 | mftb r5 |
2984 | subf r5, r6, r5 /* subtract current timebase offset */ |
2985 | std r3, VCPU_CUR_ACTIVITY(r4) |
2986 | std r5, VCPU_ACTIVITY_START(r4) |
2987 | blr |
2988 | |
2989 | /* |
2990 | * Accumulate time to one activity and start another. |
2991 | * r3 = pointer to new time accumulation struct, r4 = vcpu |
2992 | */ |
2993 | kvmhv_accumulate_time: |
2994 | ld r5, HSTATE_KVM_VCORE(r13) |
2995 | ld r8, VCORE_TB_OFFSET_APPL(r5) |
2996 | ld r5, VCPU_CUR_ACTIVITY(r4) |
2997 | ld r6, VCPU_ACTIVITY_START(r4) |
2998 | std r3, VCPU_CUR_ACTIVITY(r4) |
2999 | mftb r7 |
3000 | subf r7, r8, r7 /* subtract current timebase offset */ |
3001 | std r7, VCPU_ACTIVITY_START(r4) |
3002 | cmpdi r5, 0 |
3003 | beqlr |
3004 | subf r3, r6, r7 |
3005 | ld r8, TAS_SEQCOUNT(r5) |
3006 | cmpdi r8, 0 |
3007 | addi r8, r8, 1 |
3008 | std r8, TAS_SEQCOUNT(r5) |
3009 | lwsync |
3010 | ld r7, TAS_TOTAL(r5) |
3011 | add r7, r7, r3 |
3012 | std r7, TAS_TOTAL(r5) |
3013 | ld r6, TAS_MIN(r5) |
3014 | ld r7, TAS_MAX(r5) |
3015 | beq 3f |
3016 | cmpd r3, r6 |
3017 | bge 1f |
3018 | 3: std r3, TAS_MIN(r5) |
3019 | 1: cmpd r3, r7 |
3020 | ble 2f |
3021 | std r3, TAS_MAX(r5) |
3022 | 2: lwsync |
3023 | addi r8, r8, 1 |
3024 | std r8, TAS_SEQCOUNT(r5) |
3025 | blr |
3026 | #endif |
3027 | |