1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
2 | /* |
3 | * Kernel-based Virtual Machine driver for Linux |
4 | * |
5 | * This header defines architecture specific interfaces, x86 version |
6 | */ |
7 | |
8 | #ifndef _ASM_X86_KVM_HOST_H |
9 | #define _ASM_X86_KVM_HOST_H |
10 | |
11 | #include <linux/types.h> |
12 | #include <linux/mm.h> |
13 | #include <linux/mmu_notifier.h> |
14 | #include <linux/tracepoint.h> |
15 | #include <linux/cpumask.h> |
16 | #include <linux/irq_work.h> |
17 | #include <linux/irq.h> |
18 | #include <linux/workqueue.h> |
19 | |
20 | #include <linux/kvm.h> |
21 | #include <linux/kvm_para.h> |
22 | #include <linux/kvm_types.h> |
23 | #include <linux/perf_event.h> |
24 | #include <linux/pvclock_gtod.h> |
25 | #include <linux/clocksource.h> |
26 | #include <linux/irqbypass.h> |
27 | #include <linux/hyperv.h> |
28 | #include <linux/kfifo.h> |
29 | |
30 | #include <asm/apic.h> |
31 | #include <asm/pvclock-abi.h> |
32 | #include <asm/desc.h> |
33 | #include <asm/mtrr.h> |
34 | #include <asm/msr-index.h> |
35 | #include <asm/asm.h> |
36 | #include <asm/kvm_page_track.h> |
37 | #include <asm/kvm_vcpu_regs.h> |
38 | #include <asm/hyperv-tlfs.h> |
39 | |
40 | #define __KVM_HAVE_ARCH_VCPU_DEBUGFS |
41 | |
42 | /* |
43 | * CONFIG_KVM_MAX_NR_VCPUS is defined iff CONFIG_KVM!=n, provide a dummy max if |
44 | * KVM is disabled (arbitrarily use the default from CONFIG_KVM_MAX_NR_VCPUS). |
45 | */ |
46 | #ifdef CONFIG_KVM_MAX_NR_VCPUS |
47 | #define KVM_MAX_VCPUS CONFIG_KVM_MAX_NR_VCPUS |
48 | #else |
49 | #define KVM_MAX_VCPUS 1024 |
50 | #endif |
51 | |
52 | /* |
53 | * In x86, the VCPU ID corresponds to the APIC ID, and APIC IDs |
54 | * might be larger than the actual number of VCPUs because the |
55 | * APIC ID encodes CPU topology information. |
56 | * |
57 | * In the worst case, we'll need less than one extra bit for the |
58 | * Core ID, and less than one extra bit for the Package (Die) ID, |
59 | * so ratio of 4 should be enough. |
60 | */ |
61 | #define KVM_VCPU_ID_RATIO 4 |
62 | #define KVM_MAX_VCPU_IDS (KVM_MAX_VCPUS * KVM_VCPU_ID_RATIO) |
63 | |
64 | /* memory slots that are not exposed to userspace */ |
65 | #define KVM_INTERNAL_MEM_SLOTS 3 |
66 | |
67 | #define KVM_HALT_POLL_NS_DEFAULT 200000 |
68 | |
69 | #define KVM_IRQCHIP_NUM_PINS KVM_IOAPIC_NUM_PINS |
70 | |
71 | #define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \ |
72 | KVM_DIRTY_LOG_INITIALLY_SET) |
73 | |
74 | #define KVM_BUS_LOCK_DETECTION_VALID_MODE (KVM_BUS_LOCK_DETECTION_OFF | \ |
75 | KVM_BUS_LOCK_DETECTION_EXIT) |
76 | |
77 | #define KVM_X86_NOTIFY_VMEXIT_VALID_BITS (KVM_X86_NOTIFY_VMEXIT_ENABLED | \ |
78 | KVM_X86_NOTIFY_VMEXIT_USER) |
79 | |
80 | /* x86-specific vcpu->requests bit members */ |
81 | #define KVM_REQ_MIGRATE_TIMER KVM_ARCH_REQ(0) |
82 | #define KVM_REQ_REPORT_TPR_ACCESS KVM_ARCH_REQ(1) |
83 | #define KVM_REQ_TRIPLE_FAULT KVM_ARCH_REQ(2) |
84 | #define KVM_REQ_MMU_SYNC KVM_ARCH_REQ(3) |
85 | #define KVM_REQ_CLOCK_UPDATE KVM_ARCH_REQ(4) |
86 | #define KVM_REQ_LOAD_MMU_PGD KVM_ARCH_REQ(5) |
87 | #define KVM_REQ_EVENT KVM_ARCH_REQ(6) |
88 | #define KVM_REQ_APF_HALT KVM_ARCH_REQ(7) |
89 | #define KVM_REQ_STEAL_UPDATE KVM_ARCH_REQ(8) |
90 | #define KVM_REQ_NMI KVM_ARCH_REQ(9) |
91 | #define KVM_REQ_PMU KVM_ARCH_REQ(10) |
92 | #define KVM_REQ_PMI KVM_ARCH_REQ(11) |
93 | #ifdef CONFIG_KVM_SMM |
94 | #define KVM_REQ_SMI KVM_ARCH_REQ(12) |
95 | #endif |
96 | #define KVM_REQ_MASTERCLOCK_UPDATE KVM_ARCH_REQ(13) |
97 | #define KVM_REQ_MCLOCK_INPROGRESS \ |
98 | KVM_ARCH_REQ_FLAGS(14, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) |
99 | #define KVM_REQ_SCAN_IOAPIC \ |
100 | KVM_ARCH_REQ_FLAGS(15, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) |
101 | #define KVM_REQ_GLOBAL_CLOCK_UPDATE KVM_ARCH_REQ(16) |
102 | #define KVM_REQ_APIC_PAGE_RELOAD \ |
103 | KVM_ARCH_REQ_FLAGS(17, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) |
104 | #define KVM_REQ_HV_CRASH KVM_ARCH_REQ(18) |
105 | #define KVM_REQ_IOAPIC_EOI_EXIT KVM_ARCH_REQ(19) |
106 | #define KVM_REQ_HV_RESET KVM_ARCH_REQ(20) |
107 | #define KVM_REQ_HV_EXIT KVM_ARCH_REQ(21) |
108 | #define KVM_REQ_HV_STIMER KVM_ARCH_REQ(22) |
109 | #define KVM_REQ_LOAD_EOI_EXITMAP KVM_ARCH_REQ(23) |
110 | #define KVM_REQ_GET_NESTED_STATE_PAGES KVM_ARCH_REQ(24) |
111 | #define KVM_REQ_APICV_UPDATE \ |
112 | KVM_ARCH_REQ_FLAGS(25, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) |
113 | #define KVM_REQ_TLB_FLUSH_CURRENT KVM_ARCH_REQ(26) |
114 | #define KVM_REQ_TLB_FLUSH_GUEST \ |
115 | KVM_ARCH_REQ_FLAGS(27, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) |
116 | #define KVM_REQ_APF_READY KVM_ARCH_REQ(28) |
117 | #define KVM_REQ_MSR_FILTER_CHANGED KVM_ARCH_REQ(29) |
118 | #define KVM_REQ_UPDATE_CPU_DIRTY_LOGGING \ |
119 | KVM_ARCH_REQ_FLAGS(30, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) |
120 | #define KVM_REQ_MMU_FREE_OBSOLETE_ROOTS \ |
121 | KVM_ARCH_REQ_FLAGS(31, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) |
122 | #define KVM_REQ_HV_TLB_FLUSH \ |
123 | KVM_ARCH_REQ_FLAGS(32, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) |
124 | |
125 | #define CR0_RESERVED_BITS \ |
126 | (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \ |
127 | | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \ |
128 | | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG)) |
129 | |
130 | #define CR4_RESERVED_BITS \ |
131 | (~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\ |
132 | | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE \ |
133 | | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR | X86_CR4_PCIDE \ |
134 | | X86_CR4_OSXSAVE | X86_CR4_SMEP | X86_CR4_FSGSBASE \ |
135 | | X86_CR4_OSXMMEXCPT | X86_CR4_LA57 | X86_CR4_VMXE \ |
136 | | X86_CR4_SMAP | X86_CR4_PKE | X86_CR4_UMIP)) |
137 | |
138 | #define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR) |
139 | |
140 | |
141 | |
142 | #define INVALID_PAGE (~(hpa_t)0) |
143 | #define VALID_PAGE(x) ((x) != INVALID_PAGE) |
144 | |
145 | /* KVM Hugepage definitions for x86 */ |
146 | #define KVM_MAX_HUGEPAGE_LEVEL PG_LEVEL_1G |
147 | #define KVM_NR_PAGE_SIZES (KVM_MAX_HUGEPAGE_LEVEL - PG_LEVEL_4K + 1) |
148 | #define KVM_HPAGE_GFN_SHIFT(x) (((x) - 1) * 9) |
149 | #define KVM_HPAGE_SHIFT(x) (PAGE_SHIFT + KVM_HPAGE_GFN_SHIFT(x)) |
150 | #define KVM_HPAGE_SIZE(x) (1UL << KVM_HPAGE_SHIFT(x)) |
151 | #define KVM_HPAGE_MASK(x) (~(KVM_HPAGE_SIZE(x) - 1)) |
152 | #define KVM_PAGES_PER_HPAGE(x) (KVM_HPAGE_SIZE(x) / PAGE_SIZE) |
153 | |
154 | #define KVM_MEMSLOT_PAGES_TO_MMU_PAGES_RATIO 50 |
155 | #define KVM_MIN_ALLOC_MMU_PAGES 64UL |
156 | #define KVM_MMU_HASH_SHIFT 12 |
157 | #define KVM_NUM_MMU_PAGES (1 << KVM_MMU_HASH_SHIFT) |
158 | #define KVM_MIN_FREE_MMU_PAGES 5 |
159 | #define KVM_REFILL_PAGES 25 |
160 | #define KVM_MAX_CPUID_ENTRIES 256 |
161 | #define KVM_NR_FIXED_MTRR_REGION 88 |
162 | #define KVM_NR_VAR_MTRR 8 |
163 | |
164 | #define ASYNC_PF_PER_VCPU 64 |
165 | |
166 | enum kvm_reg { |
167 | VCPU_REGS_RAX = __VCPU_REGS_RAX, |
168 | VCPU_REGS_RCX = __VCPU_REGS_RCX, |
169 | VCPU_REGS_RDX = __VCPU_REGS_RDX, |
170 | VCPU_REGS_RBX = __VCPU_REGS_RBX, |
171 | VCPU_REGS_RSP = __VCPU_REGS_RSP, |
172 | VCPU_REGS_RBP = __VCPU_REGS_RBP, |
173 | VCPU_REGS_RSI = __VCPU_REGS_RSI, |
174 | VCPU_REGS_RDI = __VCPU_REGS_RDI, |
175 | #ifdef CONFIG_X86_64 |
176 | VCPU_REGS_R8 = __VCPU_REGS_R8, |
177 | VCPU_REGS_R9 = __VCPU_REGS_R9, |
178 | VCPU_REGS_R10 = __VCPU_REGS_R10, |
179 | VCPU_REGS_R11 = __VCPU_REGS_R11, |
180 | VCPU_REGS_R12 = __VCPU_REGS_R12, |
181 | VCPU_REGS_R13 = __VCPU_REGS_R13, |
182 | VCPU_REGS_R14 = __VCPU_REGS_R14, |
183 | VCPU_REGS_R15 = __VCPU_REGS_R15, |
184 | #endif |
185 | VCPU_REGS_RIP, |
186 | NR_VCPU_REGS, |
187 | |
188 | VCPU_EXREG_PDPTR = NR_VCPU_REGS, |
189 | VCPU_EXREG_CR0, |
190 | VCPU_EXREG_CR3, |
191 | VCPU_EXREG_CR4, |
192 | VCPU_EXREG_RFLAGS, |
193 | VCPU_EXREG_SEGMENTS, |
194 | VCPU_EXREG_EXIT_INFO_1, |
195 | VCPU_EXREG_EXIT_INFO_2, |
196 | }; |
197 | |
198 | enum { |
199 | VCPU_SREG_ES, |
200 | VCPU_SREG_CS, |
201 | VCPU_SREG_SS, |
202 | VCPU_SREG_DS, |
203 | VCPU_SREG_FS, |
204 | VCPU_SREG_GS, |
205 | VCPU_SREG_TR, |
206 | VCPU_SREG_LDTR, |
207 | }; |
208 | |
209 | enum exit_fastpath_completion { |
210 | EXIT_FASTPATH_NONE, |
211 | EXIT_FASTPATH_REENTER_GUEST, |
212 | EXIT_FASTPATH_EXIT_HANDLED, |
213 | }; |
214 | typedef enum exit_fastpath_completion fastpath_t; |
215 | |
216 | struct x86_emulate_ctxt; |
217 | struct x86_exception; |
218 | union kvm_smram; |
219 | enum x86_intercept; |
220 | enum x86_intercept_stage; |
221 | |
222 | #define KVM_NR_DB_REGS 4 |
223 | |
224 | #define DR6_BUS_LOCK (1 << 11) |
225 | #define DR6_BD (1 << 13) |
226 | #define DR6_BS (1 << 14) |
227 | #define DR6_BT (1 << 15) |
228 | #define DR6_RTM (1 << 16) |
229 | /* |
230 | * DR6_ACTIVE_LOW combines fixed-1 and active-low bits. |
231 | * We can regard all the bits in DR6_FIXED_1 as active_low bits; |
232 | * they will never be 0 for now, but when they are defined |
233 | * in the future it will require no code change. |
234 | * |
235 | * DR6_ACTIVE_LOW is also used as the init/reset value for DR6. |
236 | */ |
237 | #define DR6_ACTIVE_LOW 0xffff0ff0 |
238 | #define DR6_VOLATILE 0x0001e80f |
239 | #define DR6_FIXED_1 (DR6_ACTIVE_LOW & ~DR6_VOLATILE) |
240 | |
241 | #define DR7_BP_EN_MASK 0x000000ff |
242 | #define DR7_GE (1 << 9) |
243 | #define DR7_GD (1 << 13) |
244 | #define DR7_FIXED_1 0x00000400 |
245 | #define DR7_VOLATILE 0xffff2bff |
246 | |
247 | #define KVM_GUESTDBG_VALID_MASK \ |
248 | (KVM_GUESTDBG_ENABLE | \ |
249 | KVM_GUESTDBG_SINGLESTEP | \ |
250 | KVM_GUESTDBG_USE_HW_BP | \ |
251 | KVM_GUESTDBG_USE_SW_BP | \ |
252 | KVM_GUESTDBG_INJECT_BP | \ |
253 | KVM_GUESTDBG_INJECT_DB | \ |
254 | KVM_GUESTDBG_BLOCKIRQ) |
255 | |
256 | |
257 | #define PFERR_PRESENT_BIT 0 |
258 | #define PFERR_WRITE_BIT 1 |
259 | #define PFERR_USER_BIT 2 |
260 | #define PFERR_RSVD_BIT 3 |
261 | #define PFERR_FETCH_BIT 4 |
262 | #define PFERR_PK_BIT 5 |
263 | #define PFERR_SGX_BIT 15 |
264 | #define PFERR_GUEST_FINAL_BIT 32 |
265 | #define PFERR_GUEST_PAGE_BIT 33 |
266 | #define PFERR_IMPLICIT_ACCESS_BIT 48 |
267 | |
268 | #define PFERR_PRESENT_MASK BIT(PFERR_PRESENT_BIT) |
269 | #define PFERR_WRITE_MASK BIT(PFERR_WRITE_BIT) |
270 | #define PFERR_USER_MASK BIT(PFERR_USER_BIT) |
271 | #define PFERR_RSVD_MASK BIT(PFERR_RSVD_BIT) |
272 | #define PFERR_FETCH_MASK BIT(PFERR_FETCH_BIT) |
273 | #define PFERR_PK_MASK BIT(PFERR_PK_BIT) |
274 | #define PFERR_SGX_MASK BIT(PFERR_SGX_BIT) |
275 | #define PFERR_GUEST_FINAL_MASK BIT_ULL(PFERR_GUEST_FINAL_BIT) |
276 | #define PFERR_GUEST_PAGE_MASK BIT_ULL(PFERR_GUEST_PAGE_BIT) |
277 | #define PFERR_IMPLICIT_ACCESS BIT_ULL(PFERR_IMPLICIT_ACCESS_BIT) |
278 | |
279 | #define PFERR_NESTED_GUEST_PAGE (PFERR_GUEST_PAGE_MASK | \ |
280 | PFERR_WRITE_MASK | \ |
281 | PFERR_PRESENT_MASK) |
282 | |
283 | /* apic attention bits */ |
284 | #define KVM_APIC_CHECK_VAPIC 0 |
285 | /* |
286 | * The following bit is set with PV-EOI, unset on EOI. |
287 | * We detect PV-EOI changes by guest by comparing |
288 | * this bit with PV-EOI in guest memory. |
289 | * See the implementation in apic_update_pv_eoi. |
290 | */ |
291 | #define KVM_APIC_PV_EOI_PENDING 1 |
292 | |
293 | struct kvm_kernel_irq_routing_entry; |
294 | |
295 | /* |
296 | * kvm_mmu_page_role tracks the properties of a shadow page (where shadow page |
297 | * also includes TDP pages) to determine whether or not a page can be used in |
298 | * the given MMU context. This is a subset of the overall kvm_cpu_role to |
299 | * minimize the size of kvm_memory_slot.arch.gfn_write_track, i.e. allows |
300 | * allocating 2 bytes per gfn instead of 4 bytes per gfn. |
301 | * |
302 | * Upper-level shadow pages having gptes are tracked for write-protection via |
303 | * gfn_write_track. As above, gfn_write_track is a 16 bit counter, so KVM must |
304 | * not create more than 2^16-1 upper-level shadow pages at a single gfn, |
305 | * otherwise gfn_write_track will overflow and explosions will ensue. |
306 | * |
307 | * A unique shadow page (SP) for a gfn is created if and only if an existing SP |
308 | * cannot be reused. The ability to reuse a SP is tracked by its role, which |
309 | * incorporates various mode bits and properties of the SP. Roughly speaking, |
310 | * the number of unique SPs that can theoretically be created is 2^n, where n |
311 | * is the number of bits that are used to compute the role. |
312 | * |
313 | * But, even though there are 19 bits in the mask below, not all combinations |
314 | * of modes and flags are possible: |
315 | * |
316 | * - invalid shadow pages are not accounted, so the bits are effectively 18 |
317 | * |
318 | * - quadrant will only be used if has_4_byte_gpte=1 (non-PAE paging); |
319 | * execonly and ad_disabled are only used for nested EPT which has |
320 | * has_4_byte_gpte=0. Therefore, 2 bits are always unused. |
321 | * |
322 | * - the 4 bits of level are effectively limited to the values 2/3/4/5, |
323 | * as 4k SPs are not tracked (allowed to go unsync). In addition non-PAE |
324 | * paging has exactly one upper level, making level completely redundant |
325 | * when has_4_byte_gpte=1. |
326 | * |
327 | * - on top of this, smep_andnot_wp and smap_andnot_wp are only set if |
328 | * cr0_wp=0, therefore these three bits only give rise to 5 possibilities. |
329 | * |
330 | * Therefore, the maximum number of possible upper-level shadow pages for a |
331 | * single gfn is a bit less than 2^13. |
332 | */ |
333 | union kvm_mmu_page_role { |
334 | u32 word; |
335 | struct { |
336 | unsigned level:4; |
337 | unsigned has_4_byte_gpte:1; |
338 | unsigned quadrant:2; |
339 | unsigned direct:1; |
340 | unsigned access:3; |
341 | unsigned invalid:1; |
342 | unsigned efer_nx:1; |
343 | unsigned cr0_wp:1; |
344 | unsigned smep_andnot_wp:1; |
345 | unsigned smap_andnot_wp:1; |
346 | unsigned ad_disabled:1; |
347 | unsigned guest_mode:1; |
348 | unsigned passthrough:1; |
349 | unsigned :5; |
350 | |
351 | /* |
352 | * This is left at the top of the word so that |
353 | * kvm_memslots_for_spte_role can extract it with a |
354 | * simple shift. While there is room, give it a whole |
355 | * byte so it is also faster to load it from memory. |
356 | */ |
357 | unsigned smm:8; |
358 | }; |
359 | }; |
360 | |
361 | /* |
362 | * kvm_mmu_extended_role complements kvm_mmu_page_role, tracking properties |
363 | * relevant to the current MMU configuration. When loading CR0, CR4, or EFER, |
364 | * including on nested transitions, if nothing in the full role changes then |
365 | * MMU re-configuration can be skipped. @valid bit is set on first usage so we |
366 | * don't treat all-zero structure as valid data. |
367 | * |
368 | * The properties that are tracked in the extended role but not the page role |
369 | * are for things that either (a) do not affect the validity of the shadow page |
370 | * or (b) are indirectly reflected in the shadow page's role. For example, |
371 | * CR4.PKE only affects permission checks for software walks of the guest page |
372 | * tables (because KVM doesn't support Protection Keys with shadow paging), and |
373 | * CR0.PG, CR4.PAE, and CR4.PSE are indirectly reflected in role.level. |
374 | * |
375 | * Note, SMEP and SMAP are not redundant with sm*p_andnot_wp in the page role. |
376 | * If CR0.WP=1, KVM can reuse shadow pages for the guest regardless of SMEP and |
377 | * SMAP, but the MMU's permission checks for software walks need to be SMEP and |
378 | * SMAP aware regardless of CR0.WP. |
379 | */ |
380 | union kvm_mmu_extended_role { |
381 | u32 word; |
382 | struct { |
383 | unsigned int valid:1; |
384 | unsigned int execonly:1; |
385 | unsigned int cr4_pse:1; |
386 | unsigned int cr4_pke:1; |
387 | unsigned int cr4_smap:1; |
388 | unsigned int cr4_smep:1; |
389 | unsigned int cr4_la57:1; |
390 | unsigned int efer_lma:1; |
391 | }; |
392 | }; |
393 | |
394 | union kvm_cpu_role { |
395 | u64 as_u64; |
396 | struct { |
397 | union kvm_mmu_page_role base; |
398 | union kvm_mmu_extended_role ext; |
399 | }; |
400 | }; |
401 | |
402 | struct kvm_rmap_head { |
403 | unsigned long val; |
404 | }; |
405 | |
406 | struct kvm_pio_request { |
407 | unsigned long linear_rip; |
408 | unsigned long count; |
409 | int in; |
410 | int port; |
411 | int size; |
412 | }; |
413 | |
414 | #define PT64_ROOT_MAX_LEVEL 5 |
415 | |
416 | struct rsvd_bits_validate { |
417 | u64 rsvd_bits_mask[2][PT64_ROOT_MAX_LEVEL]; |
418 | u64 bad_mt_xwr; |
419 | }; |
420 | |
421 | struct kvm_mmu_root_info { |
422 | gpa_t pgd; |
423 | hpa_t hpa; |
424 | }; |
425 | |
426 | #define KVM_MMU_ROOT_INFO_INVALID \ |
427 | ((struct kvm_mmu_root_info) { .pgd = INVALID_PAGE, .hpa = INVALID_PAGE }) |
428 | |
429 | #define KVM_MMU_NUM_PREV_ROOTS 3 |
430 | |
431 | #define KVM_MMU_ROOT_CURRENT BIT(0) |
432 | #define KVM_MMU_ROOT_PREVIOUS(i) BIT(1+i) |
433 | #define KVM_MMU_ROOTS_ALL (BIT(1 + KVM_MMU_NUM_PREV_ROOTS) - 1) |
434 | |
435 | #define KVM_HAVE_MMU_RWLOCK |
436 | |
437 | struct kvm_mmu_page; |
438 | struct kvm_page_fault; |
439 | |
440 | /* |
441 | * x86 supports 4 paging modes (5-level 64-bit, 4-level 64-bit, 3-level 32-bit, |
442 | * and 2-level 32-bit). The kvm_mmu structure abstracts the details of the |
443 | * current mmu mode. |
444 | */ |
445 | struct kvm_mmu { |
446 | unsigned long (*get_guest_pgd)(struct kvm_vcpu *vcpu); |
447 | u64 (*get_pdptr)(struct kvm_vcpu *vcpu, int index); |
448 | int (*page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault); |
449 | void (*inject_page_fault)(struct kvm_vcpu *vcpu, |
450 | struct x86_exception *fault); |
451 | gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, |
452 | gpa_t gva_or_gpa, u64 access, |
453 | struct x86_exception *exception); |
454 | int (*sync_spte)(struct kvm_vcpu *vcpu, |
455 | struct kvm_mmu_page *sp, int i); |
456 | struct kvm_mmu_root_info root; |
457 | union kvm_cpu_role cpu_role; |
458 | union kvm_mmu_page_role root_role; |
459 | |
460 | /* |
461 | * The pkru_mask indicates if protection key checks are needed. It |
462 | * consists of 16 domains indexed by page fault error code bits [4:1], |
463 | * with PFEC.RSVD replaced by ACC_USER_MASK from the page tables. |
464 | * Each domain has 2 bits which are ANDed with AD and WD from PKRU. |
465 | */ |
466 | u32 pkru_mask; |
467 | |
468 | struct kvm_mmu_root_info prev_roots[KVM_MMU_NUM_PREV_ROOTS]; |
469 | |
470 | /* |
471 | * Bitmap; bit set = permission fault |
472 | * Byte index: page fault error code [4:1] |
473 | * Bit index: pte permissions in ACC_* format |
474 | */ |
475 | u8 permissions[16]; |
476 | |
477 | u64 *pae_root; |
478 | u64 *pml4_root; |
479 | u64 *pml5_root; |
480 | |
481 | /* |
482 | * check zero bits on shadow page table entries, these |
483 | * bits include not only hardware reserved bits but also |
484 | * the bits spte never used. |
485 | */ |
486 | struct rsvd_bits_validate shadow_zero_check; |
487 | |
488 | struct rsvd_bits_validate guest_rsvd_check; |
489 | |
490 | u64 pdptrs[4]; /* pae */ |
491 | }; |
492 | |
493 | enum pmc_type { |
494 | KVM_PMC_GP = 0, |
495 | KVM_PMC_FIXED, |
496 | }; |
497 | |
498 | struct kvm_pmc { |
499 | enum pmc_type type; |
500 | u8 idx; |
501 | bool is_paused; |
502 | bool intr; |
503 | u64 counter; |
504 | u64 prev_counter; |
505 | u64 eventsel; |
506 | struct perf_event *perf_event; |
507 | struct kvm_vcpu *vcpu; |
508 | /* |
509 | * only for creating or reusing perf_event, |
510 | * eventsel value for general purpose counters, |
511 | * ctrl value for fixed counters. |
512 | */ |
513 | u64 current_config; |
514 | }; |
515 | |
516 | /* More counters may conflict with other existing Architectural MSRs */ |
517 | #define KVM_INTEL_PMC_MAX_GENERIC 8 |
518 | #define MSR_ARCH_PERFMON_PERFCTR_MAX (MSR_ARCH_PERFMON_PERFCTR0 + KVM_INTEL_PMC_MAX_GENERIC - 1) |
519 | #define MSR_ARCH_PERFMON_EVENTSEL_MAX (MSR_ARCH_PERFMON_EVENTSEL0 + KVM_INTEL_PMC_MAX_GENERIC - 1) |
520 | #define KVM_PMC_MAX_FIXED 3 |
521 | #define MSR_ARCH_PERFMON_FIXED_CTR_MAX (MSR_ARCH_PERFMON_FIXED_CTR0 + KVM_PMC_MAX_FIXED - 1) |
522 | #define KVM_AMD_PMC_MAX_GENERIC 6 |
523 | struct kvm_pmu { |
524 | u8 version; |
525 | unsigned nr_arch_gp_counters; |
526 | unsigned nr_arch_fixed_counters; |
527 | unsigned available_event_types; |
528 | u64 fixed_ctr_ctrl; |
529 | u64 fixed_ctr_ctrl_mask; |
530 | u64 global_ctrl; |
531 | u64 global_status; |
532 | u64 counter_bitmask[2]; |
533 | u64 global_ctrl_mask; |
534 | u64 global_status_mask; |
535 | u64 reserved_bits; |
536 | u64 raw_event_mask; |
537 | struct kvm_pmc gp_counters[KVM_INTEL_PMC_MAX_GENERIC]; |
538 | struct kvm_pmc fixed_counters[KVM_PMC_MAX_FIXED]; |
539 | |
540 | /* |
541 | * Overlay the bitmap with a 64-bit atomic so that all bits can be |
542 | * set in a single access, e.g. to reprogram all counters when the PMU |
543 | * filter changes. |
544 | */ |
545 | union { |
546 | DECLARE_BITMAP(reprogram_pmi, X86_PMC_IDX_MAX); |
547 | atomic64_t __reprogram_pmi; |
548 | }; |
549 | DECLARE_BITMAP(all_valid_pmc_idx, X86_PMC_IDX_MAX); |
550 | DECLARE_BITMAP(pmc_in_use, X86_PMC_IDX_MAX); |
551 | |
552 | u64 ds_area; |
553 | u64 pebs_enable; |
554 | u64 pebs_enable_mask; |
555 | u64 pebs_data_cfg; |
556 | u64 pebs_data_cfg_mask; |
557 | |
558 | /* |
559 | * If a guest counter is cross-mapped to host counter with different |
560 | * index, its PEBS capability will be temporarily disabled. |
561 | * |
562 | * The user should make sure that this mask is updated |
563 | * after disabling interrupts and before perf_guest_get_msrs(); |
564 | */ |
565 | u64 host_cross_mapped_mask; |
566 | |
567 | /* |
568 | * The gate to release perf_events not marked in |
569 | * pmc_in_use only once in a vcpu time slice. |
570 | */ |
571 | bool need_cleanup; |
572 | |
573 | /* |
574 | * The total number of programmed perf_events and it helps to avoid |
575 | * redundant check before cleanup if guest don't use vPMU at all. |
576 | */ |
577 | u8 event_count; |
578 | }; |
579 | |
580 | struct kvm_pmu_ops; |
581 | |
582 | enum { |
583 | KVM_DEBUGREG_BP_ENABLED = 1, |
584 | KVM_DEBUGREG_WONT_EXIT = 2, |
585 | }; |
586 | |
587 | struct kvm_mtrr_range { |
588 | u64 base; |
589 | u64 mask; |
590 | struct list_head node; |
591 | }; |
592 | |
593 | struct kvm_mtrr { |
594 | struct kvm_mtrr_range var_ranges[KVM_NR_VAR_MTRR]; |
595 | mtrr_type fixed_ranges[KVM_NR_FIXED_MTRR_REGION]; |
596 | u64 deftype; |
597 | |
598 | struct list_head head; |
599 | }; |
600 | |
601 | /* Hyper-V SynIC timer */ |
602 | struct kvm_vcpu_hv_stimer { |
603 | struct hrtimer timer; |
604 | int index; |
605 | union hv_stimer_config config; |
606 | u64 count; |
607 | u64 exp_time; |
608 | struct hv_message msg; |
609 | bool msg_pending; |
610 | }; |
611 | |
612 | /* Hyper-V synthetic interrupt controller (SynIC)*/ |
613 | struct kvm_vcpu_hv_synic { |
614 | u64 version; |
615 | u64 control; |
616 | u64 msg_page; |
617 | u64 evt_page; |
618 | atomic64_t sint[HV_SYNIC_SINT_COUNT]; |
619 | atomic_t sint_to_gsi[HV_SYNIC_SINT_COUNT]; |
620 | DECLARE_BITMAP(auto_eoi_bitmap, 256); |
621 | DECLARE_BITMAP(vec_bitmap, 256); |
622 | bool active; |
623 | bool dont_zero_synic_pages; |
624 | }; |
625 | |
626 | /* The maximum number of entries on the TLB flush fifo. */ |
627 | #define KVM_HV_TLB_FLUSH_FIFO_SIZE (16) |
628 | /* |
629 | * Note: the following 'magic' entry is made up by KVM to avoid putting |
630 | * anything besides GVA on the TLB flush fifo. It is theoretically possible |
631 | * to observe a request to flush 4095 PFNs starting from 0xfffffffffffff000 |
632 | * which will look identical. KVM's action to 'flush everything' instead of |
633 | * flushing these particular addresses is, however, fully legitimate as |
634 | * flushing more than requested is always OK. |
635 | */ |
636 | #define KVM_HV_TLB_FLUSHALL_ENTRY ((u64)-1) |
637 | |
638 | enum hv_tlb_flush_fifos { |
639 | HV_L1_TLB_FLUSH_FIFO, |
640 | HV_L2_TLB_FLUSH_FIFO, |
641 | HV_NR_TLB_FLUSH_FIFOS, |
642 | }; |
643 | |
644 | struct kvm_vcpu_hv_tlb_flush_fifo { |
645 | spinlock_t write_lock; |
646 | DECLARE_KFIFO(entries, u64, KVM_HV_TLB_FLUSH_FIFO_SIZE); |
647 | }; |
648 | |
649 | /* Hyper-V per vcpu emulation context */ |
650 | struct kvm_vcpu_hv { |
651 | struct kvm_vcpu *vcpu; |
652 | u32 vp_index; |
653 | u64 hv_vapic; |
654 | s64 runtime_offset; |
655 | struct kvm_vcpu_hv_synic synic; |
656 | struct kvm_hyperv_exit exit; |
657 | struct kvm_vcpu_hv_stimer stimer[HV_SYNIC_STIMER_COUNT]; |
658 | DECLARE_BITMAP(stimer_pending_bitmap, HV_SYNIC_STIMER_COUNT); |
659 | bool enforce_cpuid; |
660 | struct { |
661 | u32 features_eax; /* HYPERV_CPUID_FEATURES.EAX */ |
662 | u32 features_ebx; /* HYPERV_CPUID_FEATURES.EBX */ |
663 | u32 features_edx; /* HYPERV_CPUID_FEATURES.EDX */ |
664 | u32 enlightenments_eax; /* HYPERV_CPUID_ENLIGHTMENT_INFO.EAX */ |
665 | u32 enlightenments_ebx; /* HYPERV_CPUID_ENLIGHTMENT_INFO.EBX */ |
666 | u32 syndbg_cap_eax; /* HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES.EAX */ |
667 | u32 nested_eax; /* HYPERV_CPUID_NESTED_FEATURES.EAX */ |
668 | u32 nested_ebx; /* HYPERV_CPUID_NESTED_FEATURES.EBX */ |
669 | } cpuid_cache; |
670 | |
671 | struct kvm_vcpu_hv_tlb_flush_fifo tlb_flush_fifo[HV_NR_TLB_FLUSH_FIFOS]; |
672 | |
673 | /* Preallocated buffer for handling hypercalls passing sparse vCPU set */ |
674 | u64 sparse_banks[HV_MAX_SPARSE_VCPU_BANKS]; |
675 | |
676 | struct hv_vp_assist_page vp_assist_page; |
677 | |
678 | struct { |
679 | u64 pa_page_gpa; |
680 | u64 vm_id; |
681 | u32 vp_id; |
682 | } nested; |
683 | }; |
684 | |
685 | struct kvm_hypervisor_cpuid { |
686 | u32 base; |
687 | u32 limit; |
688 | }; |
689 | |
690 | #ifdef CONFIG_KVM_XEN |
691 | /* Xen HVM per vcpu emulation context */ |
692 | struct kvm_vcpu_xen { |
693 | u64 hypercall_rip; |
694 | u32 current_runstate; |
695 | u8 upcall_vector; |
696 | struct gfn_to_pfn_cache vcpu_info_cache; |
697 | struct gfn_to_pfn_cache vcpu_time_info_cache; |
698 | struct gfn_to_pfn_cache runstate_cache; |
699 | struct gfn_to_pfn_cache runstate2_cache; |
700 | u64 last_steal; |
701 | u64 runstate_entry_time; |
702 | u64 runstate_times[4]; |
703 | unsigned long evtchn_pending_sel; |
704 | u32 vcpu_id; /* The Xen / ACPI vCPU ID */ |
705 | u32 timer_virq; |
706 | u64 timer_expires; /* In guest epoch */ |
707 | atomic_t timer_pending; |
708 | struct hrtimer timer; |
709 | int poll_evtchn; |
710 | struct timer_list poll_timer; |
711 | struct kvm_hypervisor_cpuid cpuid; |
712 | }; |
713 | #endif |
714 | |
715 | struct kvm_queued_exception { |
716 | bool pending; |
717 | bool injected; |
718 | bool has_error_code; |
719 | u8 vector; |
720 | u32 error_code; |
721 | unsigned long payload; |
722 | bool has_payload; |
723 | }; |
724 | |
725 | struct kvm_vcpu_arch { |
726 | /* |
727 | * rip and regs accesses must go through |
728 | * kvm_{register,rip}_{read,write} functions. |
729 | */ |
730 | unsigned long regs[NR_VCPU_REGS]; |
731 | u32 regs_avail; |
732 | u32 regs_dirty; |
733 | |
734 | unsigned long cr0; |
735 | unsigned long cr0_guest_owned_bits; |
736 | unsigned long cr2; |
737 | unsigned long cr3; |
738 | unsigned long cr4; |
739 | unsigned long cr4_guest_owned_bits; |
740 | unsigned long cr4_guest_rsvd_bits; |
741 | unsigned long cr8; |
742 | u32 host_pkru; |
743 | u32 pkru; |
744 | u32 hflags; |
745 | u64 efer; |
746 | u64 apic_base; |
747 | struct kvm_lapic *apic; /* kernel irqchip context */ |
748 | bool load_eoi_exitmap_pending; |
749 | DECLARE_BITMAP(ioapic_handled_vectors, 256); |
750 | unsigned long apic_attention; |
751 | int32_t apic_arb_prio; |
752 | int mp_state; |
753 | u64 ia32_misc_enable_msr; |
754 | u64 smbase; |
755 | u64 smi_count; |
756 | bool at_instruction_boundary; |
757 | bool tpr_access_reporting; |
758 | bool xfd_no_write_intercept; |
759 | u64 ia32_xss; |
760 | u64 microcode_version; |
761 | u64 arch_capabilities; |
762 | u64 perf_capabilities; |
763 | |
764 | /* |
765 | * Paging state of the vcpu |
766 | * |
767 | * If the vcpu runs in guest mode with two level paging this still saves |
768 | * the paging mode of the l1 guest. This context is always used to |
769 | * handle faults. |
770 | */ |
771 | struct kvm_mmu *mmu; |
772 | |
773 | /* Non-nested MMU for L1 */ |
774 | struct kvm_mmu root_mmu; |
775 | |
776 | /* L1 MMU when running nested */ |
777 | struct kvm_mmu guest_mmu; |
778 | |
779 | /* |
780 | * Paging state of an L2 guest (used for nested npt) |
781 | * |
782 | * This context will save all necessary information to walk page tables |
783 | * of an L2 guest. This context is only initialized for page table |
784 | * walking and not for faulting since we never handle l2 page faults on |
785 | * the host. |
786 | */ |
787 | struct kvm_mmu nested_mmu; |
788 | |
789 | /* |
790 | * Pointer to the mmu context currently used for |
791 | * gva_to_gpa translations. |
792 | */ |
793 | struct kvm_mmu *walk_mmu; |
794 | |
795 | struct kvm_mmu_memory_cache mmu_pte_list_desc_cache; |
796 | struct kvm_mmu_memory_cache mmu_shadow_page_cache; |
797 | struct kvm_mmu_memory_cache mmu_shadowed_info_cache; |
798 | struct kvm_mmu_memory_cache ; |
799 | |
800 | /* |
801 | * QEMU userspace and the guest each have their own FPU state. |
802 | * In vcpu_run, we switch between the user and guest FPU contexts. |
803 | * While running a VCPU, the VCPU thread will have the guest FPU |
804 | * context. |
805 | * |
806 | * Note that while the PKRU state lives inside the fpu registers, |
807 | * it is switched out separately at VMENTER and VMEXIT time. The |
808 | * "guest_fpstate" state here contains the guest FPU context, with the |
809 | * host PRKU bits. |
810 | */ |
811 | struct fpu_guest guest_fpu; |
812 | |
813 | u64 xcr0; |
814 | u64 guest_supported_xcr0; |
815 | |
816 | struct kvm_pio_request pio; |
817 | void *pio_data; |
818 | void *sev_pio_data; |
819 | unsigned sev_pio_count; |
820 | |
821 | u8 event_exit_inst_len; |
822 | |
823 | bool exception_from_userspace; |
824 | |
825 | /* Exceptions to be injected to the guest. */ |
826 | struct kvm_queued_exception exception; |
827 | /* Exception VM-Exits to be synthesized to L1. */ |
828 | struct kvm_queued_exception exception_vmexit; |
829 | |
830 | struct kvm_queued_interrupt { |
831 | bool injected; |
832 | bool soft; |
833 | u8 nr; |
834 | } interrupt; |
835 | |
836 | int halt_request; /* real mode on Intel only */ |
837 | |
838 | int cpuid_nent; |
839 | struct kvm_cpuid_entry2 *cpuid_entries; |
840 | struct kvm_hypervisor_cpuid kvm_cpuid; |
841 | |
842 | /* |
843 | * FIXME: Drop this macro and use KVM_NR_GOVERNED_FEATURES directly |
844 | * when "struct kvm_vcpu_arch" is no longer defined in an |
845 | * arch/x86/include/asm header. The max is mostly arbitrary, i.e. |
846 | * can be increased as necessary. |
847 | */ |
848 | #define KVM_MAX_NR_GOVERNED_FEATURES BITS_PER_LONG |
849 | |
850 | /* |
851 | * Track whether or not the guest is allowed to use features that are |
852 | * governed by KVM, where "governed" means KVM needs to manage state |
853 | * and/or explicitly enable the feature in hardware. Typically, but |
854 | * not always, governed features can be used by the guest if and only |
855 | * if both KVM and userspace want to expose the feature to the guest. |
856 | */ |
857 | struct { |
858 | DECLARE_BITMAP(enabled, KVM_MAX_NR_GOVERNED_FEATURES); |
859 | } governed_features; |
860 | |
861 | u64 reserved_gpa_bits; |
862 | int maxphyaddr; |
863 | |
864 | /* emulate context */ |
865 | |
866 | struct x86_emulate_ctxt *emulate_ctxt; |
867 | bool emulate_regs_need_sync_to_vcpu; |
868 | bool emulate_regs_need_sync_from_vcpu; |
869 | int (*complete_userspace_io)(struct kvm_vcpu *vcpu); |
870 | |
871 | gpa_t time; |
872 | struct pvclock_vcpu_time_info hv_clock; |
873 | unsigned int hw_tsc_khz; |
874 | struct gfn_to_pfn_cache pv_time; |
875 | /* set guest stopped flag in pvclock flags field */ |
876 | bool pvclock_set_guest_stopped_request; |
877 | |
878 | struct { |
879 | u8 preempted; |
880 | u64 msr_val; |
881 | u64 last_steal; |
882 | struct gfn_to_hva_cache cache; |
883 | } st; |
884 | |
885 | u64 l1_tsc_offset; |
886 | u64 tsc_offset; /* current tsc offset */ |
887 | u64 last_guest_tsc; |
888 | u64 last_host_tsc; |
889 | u64 tsc_offset_adjustment; |
890 | u64 this_tsc_nsec; |
891 | u64 this_tsc_write; |
892 | u64 this_tsc_generation; |
893 | bool tsc_catchup; |
894 | bool tsc_always_catchup; |
895 | s8 virtual_tsc_shift; |
896 | u32 virtual_tsc_mult; |
897 | u32 virtual_tsc_khz; |
898 | s64 ia32_tsc_adjust_msr; |
899 | u64 msr_ia32_power_ctl; |
900 | u64 l1_tsc_scaling_ratio; |
901 | u64 tsc_scaling_ratio; /* current scaling ratio */ |
902 | |
903 | atomic_t nmi_queued; /* unprocessed asynchronous NMIs */ |
904 | /* Number of NMIs pending injection, not including hardware vNMIs. */ |
905 | unsigned int nmi_pending; |
906 | bool nmi_injected; /* Trying to inject an NMI this entry */ |
907 | bool smi_pending; /* SMI queued after currently running handler */ |
908 | u8 handling_intr_from_guest; |
909 | |
910 | struct kvm_mtrr mtrr_state; |
911 | u64 pat; |
912 | |
913 | unsigned switch_db_regs; |
914 | unsigned long db[KVM_NR_DB_REGS]; |
915 | unsigned long dr6; |
916 | unsigned long dr7; |
917 | unsigned long eff_db[KVM_NR_DB_REGS]; |
918 | unsigned long guest_debug_dr7; |
919 | u64 msr_platform_info; |
920 | u64 msr_misc_features_enables; |
921 | |
922 | u64 mcg_cap; |
923 | u64 mcg_status; |
924 | u64 mcg_ctl; |
925 | u64 mcg_ext_ctl; |
926 | u64 *mce_banks; |
927 | u64 *mci_ctl2_banks; |
928 | |
929 | /* Cache MMIO info */ |
930 | u64 mmio_gva; |
931 | unsigned mmio_access; |
932 | gfn_t mmio_gfn; |
933 | u64 mmio_gen; |
934 | |
935 | struct kvm_pmu pmu; |
936 | |
937 | /* used for guest single stepping over the given code position */ |
938 | unsigned long singlestep_rip; |
939 | |
940 | bool hyperv_enabled; |
941 | struct kvm_vcpu_hv *hyperv; |
942 | #ifdef CONFIG_KVM_XEN |
943 | struct kvm_vcpu_xen xen; |
944 | #endif |
945 | cpumask_var_t wbinvd_dirty_mask; |
946 | |
947 | unsigned long last_retry_eip; |
948 | unsigned long last_retry_addr; |
949 | |
950 | struct { |
951 | bool halted; |
952 | gfn_t gfns[ASYNC_PF_PER_VCPU]; |
953 | struct gfn_to_hva_cache data; |
954 | u64 msr_en_val; /* MSR_KVM_ASYNC_PF_EN */ |
955 | u64 msr_int_val; /* MSR_KVM_ASYNC_PF_INT */ |
956 | u16 vec; |
957 | u32 id; |
958 | bool send_user_only; |
959 | u32 host_apf_flags; |
960 | bool delivery_as_pf_vmexit; |
961 | bool ; |
962 | } apf; |
963 | |
964 | /* OSVW MSRs (AMD only) */ |
965 | struct { |
966 | u64 length; |
967 | u64 status; |
968 | } osvw; |
969 | |
970 | struct { |
971 | u64 msr_val; |
972 | struct gfn_to_hva_cache data; |
973 | } pv_eoi; |
974 | |
975 | u64 msr_kvm_poll_control; |
976 | |
977 | /* set at EPT violation at this point */ |
978 | unsigned long exit_qualification; |
979 | |
980 | /* pv related host specific info */ |
981 | struct { |
982 | bool pv_unhalted; |
983 | } pv; |
984 | |
985 | int pending_ioapic_eoi; |
986 | int pending_external_vector; |
987 | |
988 | /* be preempted when it's in kernel-mode(cpl=0) */ |
989 | bool preempted_in_kernel; |
990 | |
991 | /* Flush the L1 Data cache for L1TF mitigation on VMENTER */ |
992 | bool l1tf_flush_l1d; |
993 | |
994 | /* Host CPU on which VM-entry was most recently attempted */ |
995 | int last_vmentry_cpu; |
996 | |
997 | /* AMD MSRC001_0015 Hardware Configuration */ |
998 | u64 msr_hwcr; |
999 | |
1000 | /* pv related cpuid info */ |
1001 | struct { |
1002 | /* |
1003 | * value of the eax register in the KVM_CPUID_FEATURES CPUID |
1004 | * leaf. |
1005 | */ |
1006 | u32 features; |
1007 | |
1008 | /* |
1009 | * indicates whether pv emulation should be disabled if features |
1010 | * are not present in the guest's cpuid |
1011 | */ |
1012 | bool enforce; |
1013 | } pv_cpuid; |
1014 | |
1015 | /* Protected Guests */ |
1016 | bool guest_state_protected; |
1017 | |
1018 | /* |
1019 | * Set when PDPTS were loaded directly by the userspace without |
1020 | * reading the guest memory |
1021 | */ |
1022 | bool pdptrs_from_userspace; |
1023 | |
1024 | #if IS_ENABLED(CONFIG_HYPERV) |
1025 | hpa_t hv_root_tdp; |
1026 | #endif |
1027 | }; |
1028 | |
1029 | struct kvm_lpage_info { |
1030 | int disallow_lpage; |
1031 | }; |
1032 | |
1033 | struct kvm_arch_memory_slot { |
1034 | struct kvm_rmap_head *rmap[KVM_NR_PAGE_SIZES]; |
1035 | struct kvm_lpage_info *lpage_info[KVM_NR_PAGE_SIZES - 1]; |
1036 | unsigned short *gfn_write_track; |
1037 | }; |
1038 | |
1039 | /* |
1040 | * Track the mode of the optimized logical map, as the rules for decoding the |
1041 | * destination vary per mode. Enabling the optimized logical map requires all |
1042 | * software-enabled local APIs to be in the same mode, each addressable APIC to |
1043 | * be mapped to only one MDA, and each MDA to map to at most one APIC. |
1044 | */ |
1045 | enum kvm_apic_logical_mode { |
1046 | /* All local APICs are software disabled. */ |
1047 | KVM_APIC_MODE_SW_DISABLED, |
1048 | /* All software enabled local APICs in xAPIC cluster addressing mode. */ |
1049 | KVM_APIC_MODE_XAPIC_CLUSTER, |
1050 | /* All software enabled local APICs in xAPIC flat addressing mode. */ |
1051 | KVM_APIC_MODE_XAPIC_FLAT, |
1052 | /* All software enabled local APICs in x2APIC mode. */ |
1053 | KVM_APIC_MODE_X2APIC, |
1054 | /* |
1055 | * Optimized map disabled, e.g. not all local APICs in the same logical |
1056 | * mode, same logical ID assigned to multiple APICs, etc. |
1057 | */ |
1058 | KVM_APIC_MODE_MAP_DISABLED, |
1059 | }; |
1060 | |
1061 | struct kvm_apic_map { |
1062 | struct rcu_head rcu; |
1063 | enum kvm_apic_logical_mode logical_mode; |
1064 | u32 max_apic_id; |
1065 | union { |
1066 | struct kvm_lapic *xapic_flat_map[8]; |
1067 | struct kvm_lapic *xapic_cluster_map[16][4]; |
1068 | }; |
1069 | struct kvm_lapic *phys_map[]; |
1070 | }; |
1071 | |
1072 | /* Hyper-V synthetic debugger (SynDbg)*/ |
1073 | struct kvm_hv_syndbg { |
1074 | struct { |
1075 | u64 control; |
1076 | u64 status; |
1077 | u64 send_page; |
1078 | u64 recv_page; |
1079 | u64 pending_page; |
1080 | } control; |
1081 | u64 options; |
1082 | }; |
1083 | |
1084 | /* Current state of Hyper-V TSC page clocksource */ |
1085 | enum hv_tsc_page_status { |
1086 | /* TSC page was not set up or disabled */ |
1087 | HV_TSC_PAGE_UNSET = 0, |
1088 | /* TSC page MSR was written by the guest, update pending */ |
1089 | HV_TSC_PAGE_GUEST_CHANGED, |
1090 | /* TSC page update was triggered from the host side */ |
1091 | HV_TSC_PAGE_HOST_CHANGED, |
1092 | /* TSC page was properly set up and is currently active */ |
1093 | HV_TSC_PAGE_SET, |
1094 | /* TSC page was set up with an inaccessible GPA */ |
1095 | HV_TSC_PAGE_BROKEN, |
1096 | }; |
1097 | |
1098 | /* Hyper-V emulation context */ |
1099 | struct kvm_hv { |
1100 | struct mutex hv_lock; |
1101 | u64 hv_guest_os_id; |
1102 | u64 hv_hypercall; |
1103 | u64 hv_tsc_page; |
1104 | enum hv_tsc_page_status hv_tsc_page_status; |
1105 | |
1106 | /* Hyper-v based guest crash (NT kernel bugcheck) parameters */ |
1107 | u64 hv_crash_param[HV_X64_MSR_CRASH_PARAMS]; |
1108 | u64 hv_crash_ctl; |
1109 | |
1110 | struct ms_hyperv_tsc_page tsc_ref; |
1111 | |
1112 | struct idr conn_to_evt; |
1113 | |
1114 | u64 hv_reenlightenment_control; |
1115 | u64 hv_tsc_emulation_control; |
1116 | u64 hv_tsc_emulation_status; |
1117 | u64 hv_invtsc_control; |
1118 | |
1119 | /* How many vCPUs have VP index != vCPU index */ |
1120 | atomic_t num_mismatched_vp_indexes; |
1121 | |
1122 | /* |
1123 | * How many SynICs use 'AutoEOI' feature |
1124 | * (protected by arch.apicv_update_lock) |
1125 | */ |
1126 | unsigned int synic_auto_eoi_used; |
1127 | |
1128 | struct hv_partition_assist_pg *hv_pa_pg; |
1129 | struct kvm_hv_syndbg hv_syndbg; |
1130 | }; |
1131 | |
1132 | struct msr_bitmap_range { |
1133 | u32 flags; |
1134 | u32 nmsrs; |
1135 | u32 base; |
1136 | unsigned long *bitmap; |
1137 | }; |
1138 | |
1139 | /* Xen emulation context */ |
1140 | struct kvm_xen { |
1141 | struct mutex xen_lock; |
1142 | u32 xen_version; |
1143 | bool long_mode; |
1144 | bool runstate_update_flag; |
1145 | u8 upcall_vector; |
1146 | struct gfn_to_pfn_cache shinfo_cache; |
1147 | struct idr evtchn_ports; |
1148 | unsigned long poll_mask[BITS_TO_LONGS(KVM_MAX_VCPUS)]; |
1149 | }; |
1150 | |
1151 | enum kvm_irqchip_mode { |
1152 | KVM_IRQCHIP_NONE, |
1153 | KVM_IRQCHIP_KERNEL, /* created with KVM_CREATE_IRQCHIP */ |
1154 | KVM_IRQCHIP_SPLIT, /* created with KVM_CAP_SPLIT_IRQCHIP */ |
1155 | }; |
1156 | |
1157 | struct kvm_x86_msr_filter { |
1158 | u8 count; |
1159 | bool default_allow:1; |
1160 | struct msr_bitmap_range ranges[16]; |
1161 | }; |
1162 | |
1163 | struct kvm_x86_pmu_event_filter { |
1164 | __u32 action; |
1165 | __u32 nevents; |
1166 | __u32 fixed_counter_bitmap; |
1167 | __u32 flags; |
1168 | __u32 nr_includes; |
1169 | __u32 nr_excludes; |
1170 | __u64 *includes; |
1171 | __u64 *excludes; |
1172 | __u64 events[]; |
1173 | }; |
1174 | |
1175 | enum kvm_apicv_inhibit { |
1176 | |
1177 | /********************************************************************/ |
1178 | /* INHIBITs that are relevant to both Intel's APICv and AMD's AVIC. */ |
1179 | /********************************************************************/ |
1180 | |
1181 | /* |
1182 | * APIC acceleration is disabled by a module parameter |
1183 | * and/or not supported in hardware. |
1184 | */ |
1185 | APICV_INHIBIT_REASON_DISABLE, |
1186 | |
1187 | /* |
1188 | * APIC acceleration is inhibited because AutoEOI feature is |
1189 | * being used by a HyperV guest. |
1190 | */ |
1191 | APICV_INHIBIT_REASON_HYPERV, |
1192 | |
1193 | /* |
1194 | * APIC acceleration is inhibited because the userspace didn't yet |
1195 | * enable the kernel/split irqchip. |
1196 | */ |
1197 | APICV_INHIBIT_REASON_ABSENT, |
1198 | |
1199 | /* APIC acceleration is inhibited because KVM_GUESTDBG_BLOCKIRQ |
1200 | * (out of band, debug measure of blocking all interrupts on this vCPU) |
1201 | * was enabled, to avoid AVIC/APICv bypassing it. |
1202 | */ |
1203 | APICV_INHIBIT_REASON_BLOCKIRQ, |
1204 | |
1205 | /* |
1206 | * APICv is disabled because not all vCPUs have a 1:1 mapping between |
1207 | * APIC ID and vCPU, _and_ KVM is not applying its x2APIC hotplug hack. |
1208 | */ |
1209 | APICV_INHIBIT_REASON_PHYSICAL_ID_ALIASED, |
1210 | |
1211 | /* |
1212 | * For simplicity, the APIC acceleration is inhibited |
1213 | * first time either APIC ID or APIC base are changed by the guest |
1214 | * from their reset values. |
1215 | */ |
1216 | APICV_INHIBIT_REASON_APIC_ID_MODIFIED, |
1217 | APICV_INHIBIT_REASON_APIC_BASE_MODIFIED, |
1218 | |
1219 | /******************************************************/ |
1220 | /* INHIBITs that are relevant only to the AMD's AVIC. */ |
1221 | /******************************************************/ |
1222 | |
1223 | /* |
1224 | * AVIC is inhibited on a vCPU because it runs a nested guest. |
1225 | * |
1226 | * This is needed because unlike APICv, the peers of this vCPU |
1227 | * cannot use the doorbell mechanism to signal interrupts via AVIC when |
1228 | * a vCPU runs nested. |
1229 | */ |
1230 | APICV_INHIBIT_REASON_NESTED, |
1231 | |
1232 | /* |
1233 | * On SVM, the wait for the IRQ window is implemented with pending vIRQ, |
1234 | * which cannot be injected when the AVIC is enabled, thus AVIC |
1235 | * is inhibited while KVM waits for IRQ window. |
1236 | */ |
1237 | APICV_INHIBIT_REASON_IRQWIN, |
1238 | |
1239 | /* |
1240 | * PIT (i8254) 're-inject' mode, relies on EOI intercept, |
1241 | * which AVIC doesn't support for edge triggered interrupts. |
1242 | */ |
1243 | APICV_INHIBIT_REASON_PIT_REINJ, |
1244 | |
1245 | /* |
1246 | * AVIC is disabled because SEV doesn't support it. |
1247 | */ |
1248 | APICV_INHIBIT_REASON_SEV, |
1249 | |
1250 | /* |
1251 | * AVIC is disabled because not all vCPUs with a valid LDR have a 1:1 |
1252 | * mapping between logical ID and vCPU. |
1253 | */ |
1254 | APICV_INHIBIT_REASON_LOGICAL_ID_ALIASED, |
1255 | }; |
1256 | |
1257 | struct kvm_arch { |
1258 | unsigned long n_used_mmu_pages; |
1259 | unsigned long n_requested_mmu_pages; |
1260 | unsigned long n_max_mmu_pages; |
1261 | unsigned int indirect_shadow_pages; |
1262 | u8 mmu_valid_gen; |
1263 | struct hlist_head mmu_page_hash[KVM_NUM_MMU_PAGES]; |
1264 | struct list_head active_mmu_pages; |
1265 | struct list_head zapped_obsolete_pages; |
1266 | /* |
1267 | * A list of kvm_mmu_page structs that, if zapped, could possibly be |
1268 | * replaced by an NX huge page. A shadow page is on this list if its |
1269 | * existence disallows an NX huge page (nx_huge_page_disallowed is set) |
1270 | * and there are no other conditions that prevent a huge page, e.g. |
1271 | * the backing host page is huge, dirtly logging is not enabled for its |
1272 | * memslot, etc... Note, zapping shadow pages on this list doesn't |
1273 | * guarantee an NX huge page will be created in its stead, e.g. if the |
1274 | * guest attempts to execute from the region then KVM obviously can't |
1275 | * create an NX huge page (without hanging the guest). |
1276 | */ |
1277 | struct list_head possible_nx_huge_pages; |
1278 | #ifdef CONFIG_KVM_EXTERNAL_WRITE_TRACKING |
1279 | struct kvm_page_track_notifier_head track_notifier_head; |
1280 | #endif |
1281 | /* |
1282 | * Protects marking pages unsync during page faults, as TDP MMU page |
1283 | * faults only take mmu_lock for read. For simplicity, the unsync |
1284 | * pages lock is always taken when marking pages unsync regardless of |
1285 | * whether mmu_lock is held for read or write. |
1286 | */ |
1287 | spinlock_t mmu_unsync_pages_lock; |
1288 | |
1289 | struct iommu_domain *iommu_domain; |
1290 | bool iommu_noncoherent; |
1291 | #define __KVM_HAVE_ARCH_NONCOHERENT_DMA |
1292 | atomic_t noncoherent_dma_count; |
1293 | #define __KVM_HAVE_ARCH_ASSIGNED_DEVICE |
1294 | atomic_t assigned_device_count; |
1295 | struct kvm_pic *vpic; |
1296 | struct kvm_ioapic *vioapic; |
1297 | struct kvm_pit *vpit; |
1298 | atomic_t vapics_in_nmi_mode; |
1299 | struct mutex apic_map_lock; |
1300 | struct kvm_apic_map __rcu *apic_map; |
1301 | atomic_t apic_map_dirty; |
1302 | |
1303 | bool apic_access_memslot_enabled; |
1304 | bool apic_access_memslot_inhibited; |
1305 | |
1306 | /* Protects apicv_inhibit_reasons */ |
1307 | struct rw_semaphore apicv_update_lock; |
1308 | unsigned long apicv_inhibit_reasons; |
1309 | |
1310 | gpa_t wall_clock; |
1311 | |
1312 | bool mwait_in_guest; |
1313 | bool hlt_in_guest; |
1314 | bool pause_in_guest; |
1315 | bool cstate_in_guest; |
1316 | |
1317 | unsigned long irq_sources_bitmap; |
1318 | s64 kvmclock_offset; |
1319 | |
1320 | /* |
1321 | * This also protects nr_vcpus_matched_tsc which is read from a |
1322 | * preemption-disabled region, so it must be a raw spinlock. |
1323 | */ |
1324 | raw_spinlock_t tsc_write_lock; |
1325 | u64 last_tsc_nsec; |
1326 | u64 last_tsc_write; |
1327 | u32 last_tsc_khz; |
1328 | u64 last_tsc_offset; |
1329 | u64 cur_tsc_nsec; |
1330 | u64 cur_tsc_write; |
1331 | u64 cur_tsc_offset; |
1332 | u64 cur_tsc_generation; |
1333 | int nr_vcpus_matched_tsc; |
1334 | |
1335 | u32 default_tsc_khz; |
1336 | bool user_set_tsc; |
1337 | |
1338 | seqcount_raw_spinlock_t pvclock_sc; |
1339 | bool use_master_clock; |
1340 | u64 master_kernel_ns; |
1341 | u64 master_cycle_now; |
1342 | struct delayed_work kvmclock_update_work; |
1343 | struct delayed_work kvmclock_sync_work; |
1344 | |
1345 | struct kvm_xen_hvm_config xen_hvm_config; |
1346 | |
1347 | /* reads protected by irq_srcu, writes by irq_lock */ |
1348 | struct hlist_head mask_notifier_list; |
1349 | |
1350 | struct kvm_hv hyperv; |
1351 | struct kvm_xen xen; |
1352 | |
1353 | bool backwards_tsc_observed; |
1354 | bool boot_vcpu_runs_old_kvmclock; |
1355 | u32 bsp_vcpu_id; |
1356 | |
1357 | u64 disabled_quirks; |
1358 | |
1359 | enum kvm_irqchip_mode irqchip_mode; |
1360 | u8 nr_reserved_ioapic_pins; |
1361 | |
1362 | bool disabled_lapic_found; |
1363 | |
1364 | bool x2apic_format; |
1365 | bool x2apic_broadcast_quirk_disabled; |
1366 | |
1367 | bool guest_can_read_msr_platform_info; |
1368 | bool exception_payload_enabled; |
1369 | |
1370 | bool triple_fault_event; |
1371 | |
1372 | bool bus_lock_detection_enabled; |
1373 | bool enable_pmu; |
1374 | |
1375 | u32 notify_window; |
1376 | u32 notify_vmexit_flags; |
1377 | /* |
1378 | * If exit_on_emulation_error is set, and the in-kernel instruction |
1379 | * emulator fails to emulate an instruction, allow userspace |
1380 | * the opportunity to look at it. |
1381 | */ |
1382 | bool exit_on_emulation_error; |
1383 | |
1384 | /* Deflect RDMSR and WRMSR to user space when they trigger a #GP */ |
1385 | u32 user_space_msr_mask; |
1386 | struct kvm_x86_msr_filter __rcu *msr_filter; |
1387 | |
1388 | u32 hypercall_exit_enabled; |
1389 | |
1390 | /* Guest can access the SGX PROVISIONKEY. */ |
1391 | bool sgx_provisioning_allowed; |
1392 | |
1393 | struct kvm_x86_pmu_event_filter __rcu *pmu_event_filter; |
1394 | struct task_struct *nx_huge_page_recovery_thread; |
1395 | |
1396 | #ifdef CONFIG_X86_64 |
1397 | /* The number of TDP MMU pages across all roots. */ |
1398 | atomic64_t tdp_mmu_pages; |
1399 | |
1400 | /* |
1401 | * List of struct kvm_mmu_pages being used as roots. |
1402 | * All struct kvm_mmu_pages in the list should have |
1403 | * tdp_mmu_page set. |
1404 | * |
1405 | * For reads, this list is protected by: |
1406 | * the MMU lock in read mode + RCU or |
1407 | * the MMU lock in write mode |
1408 | * |
1409 | * For writes, this list is protected by: |
1410 | * the MMU lock in read mode + the tdp_mmu_pages_lock or |
1411 | * the MMU lock in write mode |
1412 | * |
1413 | * Roots will remain in the list until their tdp_mmu_root_count |
1414 | * drops to zero, at which point the thread that decremented the |
1415 | * count to zero should removed the root from the list and clean |
1416 | * it up, freeing the root after an RCU grace period. |
1417 | */ |
1418 | struct list_head tdp_mmu_roots; |
1419 | |
1420 | /* |
1421 | * Protects accesses to the following fields when the MMU lock |
1422 | * is held in read mode: |
1423 | * - tdp_mmu_roots (above) |
1424 | * - the link field of kvm_mmu_page structs used by the TDP MMU |
1425 | * - possible_nx_huge_pages; |
1426 | * - the possible_nx_huge_page_link field of kvm_mmu_page structs used |
1427 | * by the TDP MMU |
1428 | * It is acceptable, but not necessary, to acquire this lock when |
1429 | * the thread holds the MMU lock in write mode. |
1430 | */ |
1431 | spinlock_t tdp_mmu_pages_lock; |
1432 | #endif /* CONFIG_X86_64 */ |
1433 | |
1434 | /* |
1435 | * If set, at least one shadow root has been allocated. This flag |
1436 | * is used as one input when determining whether certain memslot |
1437 | * related allocations are necessary. |
1438 | */ |
1439 | bool shadow_root_allocated; |
1440 | |
1441 | #if IS_ENABLED(CONFIG_HYPERV) |
1442 | hpa_t hv_root_tdp; |
1443 | spinlock_t hv_root_tdp_lock; |
1444 | #endif |
1445 | /* |
1446 | * VM-scope maximum vCPU ID. Used to determine the size of structures |
1447 | * that increase along with the maximum vCPU ID, in which case, using |
1448 | * the global KVM_MAX_VCPU_IDS may lead to significant memory waste. |
1449 | */ |
1450 | u32 max_vcpu_ids; |
1451 | |
1452 | bool disable_nx_huge_pages; |
1453 | |
1454 | /* |
1455 | * Memory caches used to allocate shadow pages when performing eager |
1456 | * page splitting. No need for a shadowed_info_cache since eager page |
1457 | * splitting only allocates direct shadow pages. |
1458 | * |
1459 | * Protected by kvm->slots_lock. |
1460 | */ |
1461 | struct kvm_mmu_memory_cache split_shadow_page_cache; |
1462 | struct kvm_mmu_memory_cache ; |
1463 | |
1464 | /* |
1465 | * Memory cache used to allocate pte_list_desc structs while splitting |
1466 | * huge pages. In the worst case, to split one huge page, 512 |
1467 | * pte_list_desc structs are needed to add each lower level leaf sptep |
1468 | * to the rmap plus 1 to extend the parent_ptes rmap of the lower level |
1469 | * page table. |
1470 | * |
1471 | * Protected by kvm->slots_lock. |
1472 | */ |
1473 | #define SPLIT_DESC_CACHE_MIN_NR_OBJECTS (SPTE_ENT_PER_PAGE + 1) |
1474 | struct kvm_mmu_memory_cache split_desc_cache; |
1475 | }; |
1476 | |
1477 | struct kvm_vm_stat { |
1478 | struct kvm_vm_stat_generic generic; |
1479 | u64 mmu_shadow_zapped; |
1480 | u64 mmu_pte_write; |
1481 | u64 mmu_pde_zapped; |
1482 | u64 mmu_flooded; |
1483 | u64 mmu_recycled; |
1484 | u64 mmu_cache_miss; |
1485 | u64 mmu_unsync; |
1486 | union { |
1487 | struct { |
1488 | atomic64_t pages_4k; |
1489 | atomic64_t pages_2m; |
1490 | atomic64_t pages_1g; |
1491 | }; |
1492 | atomic64_t pages[KVM_NR_PAGE_SIZES]; |
1493 | }; |
1494 | u64 nx_lpage_splits; |
1495 | u64 max_mmu_page_hash_collisions; |
1496 | u64 max_mmu_rmap_size; |
1497 | }; |
1498 | |
1499 | struct kvm_vcpu_stat { |
1500 | struct kvm_vcpu_stat_generic generic; |
1501 | u64 pf_taken; |
1502 | u64 pf_fixed; |
1503 | u64 pf_emulate; |
1504 | u64 pf_spurious; |
1505 | u64 pf_fast; |
1506 | u64 pf_mmio_spte_created; |
1507 | u64 pf_guest; |
1508 | u64 tlb_flush; |
1509 | u64 invlpg; |
1510 | |
1511 | u64 exits; |
1512 | u64 io_exits; |
1513 | u64 mmio_exits; |
1514 | u64 signal_exits; |
1515 | u64 irq_window_exits; |
1516 | u64 nmi_window_exits; |
1517 | u64 l1d_flush; |
1518 | u64 halt_exits; |
1519 | u64 request_irq_exits; |
1520 | u64 irq_exits; |
1521 | u64 host_state_reload; |
1522 | u64 fpu_reload; |
1523 | u64 insn_emulation; |
1524 | u64 insn_emulation_fail; |
1525 | u64 hypercalls; |
1526 | u64 irq_injections; |
1527 | u64 nmi_injections; |
1528 | u64 req_event; |
1529 | u64 nested_run; |
1530 | u64 directed_yield_attempted; |
1531 | u64 directed_yield_successful; |
1532 | u64 preemption_reported; |
1533 | u64 preemption_other; |
1534 | u64 guest_mode; |
1535 | u64 notify_window_exits; |
1536 | }; |
1537 | |
1538 | struct x86_instruction_info; |
1539 | |
1540 | struct msr_data { |
1541 | bool host_initiated; |
1542 | u32 index; |
1543 | u64 data; |
1544 | }; |
1545 | |
1546 | struct kvm_lapic_irq { |
1547 | u32 vector; |
1548 | u16 delivery_mode; |
1549 | u16 dest_mode; |
1550 | bool level; |
1551 | u16 trig_mode; |
1552 | u32 shorthand; |
1553 | u32 dest_id; |
1554 | bool msi_redir_hint; |
1555 | }; |
1556 | |
1557 | static inline u16 kvm_lapic_irq_dest_mode(bool dest_mode_logical) |
1558 | { |
1559 | return dest_mode_logical ? APIC_DEST_LOGICAL : APIC_DEST_PHYSICAL; |
1560 | } |
1561 | |
1562 | struct kvm_x86_ops { |
1563 | const char *name; |
1564 | |
1565 | int (*check_processor_compatibility)(void); |
1566 | |
1567 | int (*hardware_enable)(void); |
1568 | void (*hardware_disable)(void); |
1569 | void (*hardware_unsetup)(void); |
1570 | bool (*has_emulated_msr)(struct kvm *kvm, u32 index); |
1571 | void (*vcpu_after_set_cpuid)(struct kvm_vcpu *vcpu); |
1572 | |
1573 | unsigned int vm_size; |
1574 | int (*vm_init)(struct kvm *kvm); |
1575 | void (*vm_destroy)(struct kvm *kvm); |
1576 | |
1577 | /* Create, but do not attach this VCPU */ |
1578 | int (*vcpu_precreate)(struct kvm *kvm); |
1579 | int (*vcpu_create)(struct kvm_vcpu *vcpu); |
1580 | void (*vcpu_free)(struct kvm_vcpu *vcpu); |
1581 | void (*vcpu_reset)(struct kvm_vcpu *vcpu, bool init_event); |
1582 | |
1583 | void (*prepare_switch_to_guest)(struct kvm_vcpu *vcpu); |
1584 | void (*vcpu_load)(struct kvm_vcpu *vcpu, int cpu); |
1585 | void (*vcpu_put)(struct kvm_vcpu *vcpu); |
1586 | |
1587 | void (*update_exception_bitmap)(struct kvm_vcpu *vcpu); |
1588 | int (*get_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr); |
1589 | int (*set_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr); |
1590 | u64 (*get_segment_base)(struct kvm_vcpu *vcpu, int seg); |
1591 | void (*get_segment)(struct kvm_vcpu *vcpu, |
1592 | struct kvm_segment *var, int seg); |
1593 | int (*get_cpl)(struct kvm_vcpu *vcpu); |
1594 | void (*set_segment)(struct kvm_vcpu *vcpu, |
1595 | struct kvm_segment *var, int seg); |
1596 | void (*get_cs_db_l_bits)(struct kvm_vcpu *vcpu, int *db, int *l); |
1597 | bool (*is_valid_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0); |
1598 | void (*set_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0); |
1599 | void (*post_set_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3); |
1600 | bool (*is_valid_cr4)(struct kvm_vcpu *vcpu, unsigned long cr4); |
1601 | void (*set_cr4)(struct kvm_vcpu *vcpu, unsigned long cr4); |
1602 | int (*set_efer)(struct kvm_vcpu *vcpu, u64 efer); |
1603 | void (*get_idt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt); |
1604 | void (*set_idt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt); |
1605 | void (*get_gdt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt); |
1606 | void (*set_gdt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt); |
1607 | void (*sync_dirty_debug_regs)(struct kvm_vcpu *vcpu); |
1608 | void (*set_dr7)(struct kvm_vcpu *vcpu, unsigned long value); |
1609 | void (*cache_reg)(struct kvm_vcpu *vcpu, enum kvm_reg reg); |
1610 | unsigned long (*get_rflags)(struct kvm_vcpu *vcpu); |
1611 | void (*set_rflags)(struct kvm_vcpu *vcpu, unsigned long rflags); |
1612 | bool (*get_if_flag)(struct kvm_vcpu *vcpu); |
1613 | |
1614 | void (*flush_tlb_all)(struct kvm_vcpu *vcpu); |
1615 | void (*flush_tlb_current)(struct kvm_vcpu *vcpu); |
1616 | int (*flush_remote_tlbs)(struct kvm *kvm); |
1617 | int (*flush_remote_tlbs_range)(struct kvm *kvm, gfn_t gfn, |
1618 | gfn_t nr_pages); |
1619 | |
1620 | /* |
1621 | * Flush any TLB entries associated with the given GVA. |
1622 | * Does not need to flush GPA->HPA mappings. |
1623 | * Can potentially get non-canonical addresses through INVLPGs, which |
1624 | * the implementation may choose to ignore if appropriate. |
1625 | */ |
1626 | void (*flush_tlb_gva)(struct kvm_vcpu *vcpu, gva_t addr); |
1627 | |
1628 | /* |
1629 | * Flush any TLB entries created by the guest. Like tlb_flush_gva(), |
1630 | * does not need to flush GPA->HPA mappings. |
1631 | */ |
1632 | void (*flush_tlb_guest)(struct kvm_vcpu *vcpu); |
1633 | |
1634 | int (*vcpu_pre_run)(struct kvm_vcpu *vcpu); |
1635 | enum exit_fastpath_completion (*vcpu_run)(struct kvm_vcpu *vcpu); |
1636 | int (*handle_exit)(struct kvm_vcpu *vcpu, |
1637 | enum exit_fastpath_completion exit_fastpath); |
1638 | int (*skip_emulated_instruction)(struct kvm_vcpu *vcpu); |
1639 | void (*update_emulated_instruction)(struct kvm_vcpu *vcpu); |
1640 | void (*set_interrupt_shadow)(struct kvm_vcpu *vcpu, int mask); |
1641 | u32 (*get_interrupt_shadow)(struct kvm_vcpu *vcpu); |
1642 | void (*patch_hypercall)(struct kvm_vcpu *vcpu, |
1643 | unsigned char *hypercall_addr); |
1644 | void (*inject_irq)(struct kvm_vcpu *vcpu, bool reinjected); |
1645 | void (*inject_nmi)(struct kvm_vcpu *vcpu); |
1646 | void (*inject_exception)(struct kvm_vcpu *vcpu); |
1647 | void (*cancel_injection)(struct kvm_vcpu *vcpu); |
1648 | int (*interrupt_allowed)(struct kvm_vcpu *vcpu, bool for_injection); |
1649 | int (*nmi_allowed)(struct kvm_vcpu *vcpu, bool for_injection); |
1650 | bool (*get_nmi_mask)(struct kvm_vcpu *vcpu); |
1651 | void (*set_nmi_mask)(struct kvm_vcpu *vcpu, bool masked); |
1652 | /* Whether or not a virtual NMI is pending in hardware. */ |
1653 | bool (*is_vnmi_pending)(struct kvm_vcpu *vcpu); |
1654 | /* |
1655 | * Attempt to pend a virtual NMI in harware. Returns %true on success |
1656 | * to allow using static_call_ret0 as the fallback. |
1657 | */ |
1658 | bool (*set_vnmi_pending)(struct kvm_vcpu *vcpu); |
1659 | void (*enable_nmi_window)(struct kvm_vcpu *vcpu); |
1660 | void (*enable_irq_window)(struct kvm_vcpu *vcpu); |
1661 | void (*update_cr8_intercept)(struct kvm_vcpu *vcpu, int tpr, int irr); |
1662 | bool (*check_apicv_inhibit_reasons)(enum kvm_apicv_inhibit reason); |
1663 | const unsigned long required_apicv_inhibits; |
1664 | bool allow_apicv_in_x2apic_without_x2apic_virtualization; |
1665 | void (*refresh_apicv_exec_ctrl)(struct kvm_vcpu *vcpu); |
1666 | void (*hwapic_irr_update)(struct kvm_vcpu *vcpu, int max_irr); |
1667 | void (*hwapic_isr_update)(int isr); |
1668 | bool (*guest_apic_has_interrupt)(struct kvm_vcpu *vcpu); |
1669 | void (*load_eoi_exitmap)(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap); |
1670 | void (*set_virtual_apic_mode)(struct kvm_vcpu *vcpu); |
1671 | void (*set_apic_access_page_addr)(struct kvm_vcpu *vcpu); |
1672 | void (*deliver_interrupt)(struct kvm_lapic *apic, int delivery_mode, |
1673 | int trig_mode, int vector); |
1674 | int (*sync_pir_to_irr)(struct kvm_vcpu *vcpu); |
1675 | int (*set_tss_addr)(struct kvm *kvm, unsigned int addr); |
1676 | int (*set_identity_map_addr)(struct kvm *kvm, u64 ident_addr); |
1677 | u8 (*get_mt_mask)(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio); |
1678 | |
1679 | void (*load_mmu_pgd)(struct kvm_vcpu *vcpu, hpa_t root_hpa, |
1680 | int root_level); |
1681 | |
1682 | bool (*has_wbinvd_exit)(void); |
1683 | |
1684 | u64 (*get_l2_tsc_offset)(struct kvm_vcpu *vcpu); |
1685 | u64 (*get_l2_tsc_multiplier)(struct kvm_vcpu *vcpu); |
1686 | void (*write_tsc_offset)(struct kvm_vcpu *vcpu); |
1687 | void (*write_tsc_multiplier)(struct kvm_vcpu *vcpu); |
1688 | |
1689 | /* |
1690 | * Retrieve somewhat arbitrary exit information. Intended to |
1691 | * be used only from within tracepoints or error paths. |
1692 | */ |
1693 | void (*get_exit_info)(struct kvm_vcpu *vcpu, u32 *reason, |
1694 | u64 *info1, u64 *info2, |
1695 | u32 *exit_int_info, u32 *exit_int_info_err_code); |
1696 | |
1697 | int (*check_intercept)(struct kvm_vcpu *vcpu, |
1698 | struct x86_instruction_info *info, |
1699 | enum x86_intercept_stage stage, |
1700 | struct x86_exception *exception); |
1701 | void (*handle_exit_irqoff)(struct kvm_vcpu *vcpu); |
1702 | |
1703 | void (*request_immediate_exit)(struct kvm_vcpu *vcpu); |
1704 | |
1705 | void (*sched_in)(struct kvm_vcpu *vcpu, int cpu); |
1706 | |
1707 | /* |
1708 | * Size of the CPU's dirty log buffer, i.e. VMX's PML buffer. A zero |
1709 | * value indicates CPU dirty logging is unsupported or disabled. |
1710 | */ |
1711 | int cpu_dirty_log_size; |
1712 | void (*update_cpu_dirty_logging)(struct kvm_vcpu *vcpu); |
1713 | |
1714 | const struct kvm_x86_nested_ops *nested_ops; |
1715 | |
1716 | void (*vcpu_blocking)(struct kvm_vcpu *vcpu); |
1717 | void (*vcpu_unblocking)(struct kvm_vcpu *vcpu); |
1718 | |
1719 | int (*pi_update_irte)(struct kvm *kvm, unsigned int host_irq, |
1720 | uint32_t guest_irq, bool set); |
1721 | void (*pi_start_assignment)(struct kvm *kvm); |
1722 | void (*apicv_pre_state_restore)(struct kvm_vcpu *vcpu); |
1723 | void (*apicv_post_state_restore)(struct kvm_vcpu *vcpu); |
1724 | bool (*dy_apicv_has_pending_interrupt)(struct kvm_vcpu *vcpu); |
1725 | |
1726 | int (*set_hv_timer)(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc, |
1727 | bool *expired); |
1728 | void (*cancel_hv_timer)(struct kvm_vcpu *vcpu); |
1729 | |
1730 | void (*setup_mce)(struct kvm_vcpu *vcpu); |
1731 | |
1732 | #ifdef CONFIG_KVM_SMM |
1733 | int (*smi_allowed)(struct kvm_vcpu *vcpu, bool for_injection); |
1734 | int (*enter_smm)(struct kvm_vcpu *vcpu, union kvm_smram *smram); |
1735 | int (*leave_smm)(struct kvm_vcpu *vcpu, const union kvm_smram *smram); |
1736 | void (*enable_smi_window)(struct kvm_vcpu *vcpu); |
1737 | #endif |
1738 | |
1739 | int (*mem_enc_ioctl)(struct kvm *kvm, void __user *argp); |
1740 | int (*mem_enc_register_region)(struct kvm *kvm, struct kvm_enc_region *argp); |
1741 | int (*mem_enc_unregister_region)(struct kvm *kvm, struct kvm_enc_region *argp); |
1742 | int (*vm_copy_enc_context_from)(struct kvm *kvm, unsigned int source_fd); |
1743 | int (*vm_move_enc_context_from)(struct kvm *kvm, unsigned int source_fd); |
1744 | void (*guest_memory_reclaimed)(struct kvm *kvm); |
1745 | |
1746 | int (*get_msr_feature)(struct kvm_msr_entry *entry); |
1747 | |
1748 | int (*check_emulate_instruction)(struct kvm_vcpu *vcpu, int emul_type, |
1749 | void *insn, int insn_len); |
1750 | |
1751 | bool (*apic_init_signal_blocked)(struct kvm_vcpu *vcpu); |
1752 | int (*enable_l2_tlb_flush)(struct kvm_vcpu *vcpu); |
1753 | |
1754 | void (*migrate_timers)(struct kvm_vcpu *vcpu); |
1755 | void (*msr_filter_changed)(struct kvm_vcpu *vcpu); |
1756 | int (*complete_emulated_msr)(struct kvm_vcpu *vcpu, int err); |
1757 | |
1758 | void (*vcpu_deliver_sipi_vector)(struct kvm_vcpu *vcpu, u8 vector); |
1759 | |
1760 | /* |
1761 | * Returns vCPU specific APICv inhibit reasons |
1762 | */ |
1763 | unsigned long (*vcpu_get_apicv_inhibit_reasons)(struct kvm_vcpu *vcpu); |
1764 | }; |
1765 | |
1766 | struct kvm_x86_nested_ops { |
1767 | void (*leave_nested)(struct kvm_vcpu *vcpu); |
1768 | bool (*is_exception_vmexit)(struct kvm_vcpu *vcpu, u8 vector, |
1769 | u32 error_code); |
1770 | int (*check_events)(struct kvm_vcpu *vcpu); |
1771 | bool (*has_events)(struct kvm_vcpu *vcpu); |
1772 | void (*triple_fault)(struct kvm_vcpu *vcpu); |
1773 | int (*get_state)(struct kvm_vcpu *vcpu, |
1774 | struct kvm_nested_state __user *user_kvm_nested_state, |
1775 | unsigned user_data_size); |
1776 | int (*set_state)(struct kvm_vcpu *vcpu, |
1777 | struct kvm_nested_state __user *user_kvm_nested_state, |
1778 | struct kvm_nested_state *kvm_state); |
1779 | bool (*get_nested_state_pages)(struct kvm_vcpu *vcpu); |
1780 | int (*write_log_dirty)(struct kvm_vcpu *vcpu, gpa_t l2_gpa); |
1781 | |
1782 | int (*enable_evmcs)(struct kvm_vcpu *vcpu, |
1783 | uint16_t *vmcs_version); |
1784 | uint16_t (*get_evmcs_version)(struct kvm_vcpu *vcpu); |
1785 | void (*hv_inject_synthetic_vmexit_post_tlb_flush)(struct kvm_vcpu *vcpu); |
1786 | }; |
1787 | |
1788 | struct kvm_x86_init_ops { |
1789 | int (*hardware_setup)(void); |
1790 | unsigned int (*handle_intel_pt_intr)(void); |
1791 | |
1792 | struct kvm_x86_ops *runtime_ops; |
1793 | struct kvm_pmu_ops *pmu_ops; |
1794 | }; |
1795 | |
1796 | struct kvm_arch_async_pf { |
1797 | u32 token; |
1798 | gfn_t gfn; |
1799 | unsigned long cr3; |
1800 | bool direct_map; |
1801 | }; |
1802 | |
1803 | extern u32 __read_mostly kvm_nr_uret_msrs; |
1804 | extern u64 __read_mostly host_efer; |
1805 | extern bool __read_mostly allow_smaller_maxphyaddr; |
1806 | extern bool __read_mostly enable_apicv; |
1807 | extern struct kvm_x86_ops kvm_x86_ops; |
1808 | |
1809 | #define KVM_X86_OP(func) \ |
1810 | DECLARE_STATIC_CALL(kvm_x86_##func, *(((struct kvm_x86_ops *)0)->func)); |
1811 | #define KVM_X86_OP_OPTIONAL KVM_X86_OP |
1812 | #define KVM_X86_OP_OPTIONAL_RET0 KVM_X86_OP |
1813 | #include <asm/kvm-x86-ops.h> |
1814 | |
1815 | int kvm_x86_vendor_init(struct kvm_x86_init_ops *ops); |
1816 | void kvm_x86_vendor_exit(void); |
1817 | |
1818 | #define __KVM_HAVE_ARCH_VM_ALLOC |
1819 | static inline struct kvm *kvm_arch_alloc_vm(void) |
1820 | { |
1821 | return __vmalloc(size: kvm_x86_ops.vm_size, GFP_KERNEL_ACCOUNT | __GFP_ZERO); |
1822 | } |
1823 | |
1824 | #define __KVM_HAVE_ARCH_VM_FREE |
1825 | void kvm_arch_free_vm(struct kvm *kvm); |
1826 | |
1827 | #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS |
1828 | static inline int kvm_arch_flush_remote_tlbs(struct kvm *kvm) |
1829 | { |
1830 | if (kvm_x86_ops.flush_remote_tlbs && |
1831 | !static_call(kvm_x86_flush_remote_tlbs)(kvm)) |
1832 | return 0; |
1833 | else |
1834 | return -ENOTSUPP; |
1835 | } |
1836 | |
1837 | #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS_RANGE |
1838 | |
1839 | #define kvm_arch_pmi_in_guest(vcpu) \ |
1840 | ((vcpu) && (vcpu)->arch.handling_intr_from_guest) |
1841 | |
1842 | void __init kvm_mmu_x86_module_init(void); |
1843 | int kvm_mmu_vendor_module_init(void); |
1844 | void kvm_mmu_vendor_module_exit(void); |
1845 | |
1846 | void kvm_mmu_destroy(struct kvm_vcpu *vcpu); |
1847 | int kvm_mmu_create(struct kvm_vcpu *vcpu); |
1848 | void kvm_mmu_init_vm(struct kvm *kvm); |
1849 | void kvm_mmu_uninit_vm(struct kvm *kvm); |
1850 | |
1851 | void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu); |
1852 | void kvm_mmu_reset_context(struct kvm_vcpu *vcpu); |
1853 | void kvm_mmu_slot_remove_write_access(struct kvm *kvm, |
1854 | const struct kvm_memory_slot *memslot, |
1855 | int start_level); |
1856 | void kvm_mmu_slot_try_split_huge_pages(struct kvm *kvm, |
1857 | const struct kvm_memory_slot *memslot, |
1858 | int target_level); |
1859 | void kvm_mmu_try_split_huge_pages(struct kvm *kvm, |
1860 | const struct kvm_memory_slot *memslot, |
1861 | u64 start, u64 end, |
1862 | int target_level); |
1863 | void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm, |
1864 | const struct kvm_memory_slot *memslot); |
1865 | void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm, |
1866 | const struct kvm_memory_slot *memslot); |
1867 | void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen); |
1868 | void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned long kvm_nr_mmu_pages); |
1869 | |
1870 | int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3); |
1871 | |
1872 | int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa, |
1873 | const void *val, int bytes); |
1874 | |
1875 | struct kvm_irq_mask_notifier { |
1876 | void (*func)(struct kvm_irq_mask_notifier *kimn, bool masked); |
1877 | int irq; |
1878 | struct hlist_node link; |
1879 | }; |
1880 | |
1881 | void kvm_register_irq_mask_notifier(struct kvm *kvm, int irq, |
1882 | struct kvm_irq_mask_notifier *kimn); |
1883 | void kvm_unregister_irq_mask_notifier(struct kvm *kvm, int irq, |
1884 | struct kvm_irq_mask_notifier *kimn); |
1885 | void kvm_fire_mask_notifiers(struct kvm *kvm, unsigned irqchip, unsigned pin, |
1886 | bool mask); |
1887 | |
1888 | extern bool tdp_enabled; |
1889 | |
1890 | u64 vcpu_tsc_khz(struct kvm_vcpu *vcpu); |
1891 | |
1892 | /* |
1893 | * EMULTYPE_NO_DECODE - Set when re-emulating an instruction (after completing |
1894 | * userspace I/O) to indicate that the emulation context |
1895 | * should be reused as is, i.e. skip initialization of |
1896 | * emulation context, instruction fetch and decode. |
1897 | * |
1898 | * EMULTYPE_TRAP_UD - Set when emulating an intercepted #UD from hardware. |
1899 | * Indicates that only select instructions (tagged with |
1900 | * EmulateOnUD) should be emulated (to minimize the emulator |
1901 | * attack surface). See also EMULTYPE_TRAP_UD_FORCED. |
1902 | * |
1903 | * EMULTYPE_SKIP - Set when emulating solely to skip an instruction, i.e. to |
1904 | * decode the instruction length. For use *only* by |
1905 | * kvm_x86_ops.skip_emulated_instruction() implementations if |
1906 | * EMULTYPE_COMPLETE_USER_EXIT is not set. |
1907 | * |
1908 | * EMULTYPE_ALLOW_RETRY_PF - Set when the emulator should resume the guest to |
1909 | * retry native execution under certain conditions, |
1910 | * Can only be set in conjunction with EMULTYPE_PF. |
1911 | * |
1912 | * EMULTYPE_TRAP_UD_FORCED - Set when emulating an intercepted #UD that was |
1913 | * triggered by KVM's magic "force emulation" prefix, |
1914 | * which is opt in via module param (off by default). |
1915 | * Bypasses EmulateOnUD restriction despite emulating |
1916 | * due to an intercepted #UD (see EMULTYPE_TRAP_UD). |
1917 | * Used to test the full emulator from userspace. |
1918 | * |
1919 | * EMULTYPE_VMWARE_GP - Set when emulating an intercepted #GP for VMware |
1920 | * backdoor emulation, which is opt in via module param. |
1921 | * VMware backdoor emulation handles select instructions |
1922 | * and reinjects the #GP for all other cases. |
1923 | * |
1924 | * EMULTYPE_PF - Set when emulating MMIO by way of an intercepted #PF, in which |
1925 | * case the CR2/GPA value pass on the stack is valid. |
1926 | * |
1927 | * EMULTYPE_COMPLETE_USER_EXIT - Set when the emulator should update interruptibility |
1928 | * state and inject single-step #DBs after skipping |
1929 | * an instruction (after completing userspace I/O). |
1930 | * |
1931 | * EMULTYPE_WRITE_PF_TO_SP - Set when emulating an intercepted page fault that |
1932 | * is attempting to write a gfn that contains one or |
1933 | * more of the PTEs used to translate the write itself, |
1934 | * and the owning page table is being shadowed by KVM. |
1935 | * If emulation of the faulting instruction fails and |
1936 | * this flag is set, KVM will exit to userspace instead |
1937 | * of retrying emulation as KVM cannot make forward |
1938 | * progress. |
1939 | * |
1940 | * If emulation fails for a write to guest page tables, |
1941 | * KVM unprotects (zaps) the shadow page for the target |
1942 | * gfn and resumes the guest to retry the non-emulatable |
1943 | * instruction (on hardware). Unprotecting the gfn |
1944 | * doesn't allow forward progress for a self-changing |
1945 | * access because doing so also zaps the translation for |
1946 | * the gfn, i.e. retrying the instruction will hit a |
1947 | * !PRESENT fault, which results in a new shadow page |
1948 | * and sends KVM back to square one. |
1949 | */ |
1950 | #define EMULTYPE_NO_DECODE (1 << 0) |
1951 | #define EMULTYPE_TRAP_UD (1 << 1) |
1952 | #define EMULTYPE_SKIP (1 << 2) |
1953 | #define EMULTYPE_ALLOW_RETRY_PF (1 << 3) |
1954 | #define EMULTYPE_TRAP_UD_FORCED (1 << 4) |
1955 | #define EMULTYPE_VMWARE_GP (1 << 5) |
1956 | #define EMULTYPE_PF (1 << 6) |
1957 | #define EMULTYPE_COMPLETE_USER_EXIT (1 << 7) |
1958 | #define EMULTYPE_WRITE_PF_TO_SP (1 << 8) |
1959 | |
1960 | int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type); |
1961 | int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu, |
1962 | void *insn, int insn_len); |
1963 | void __kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu, |
1964 | u64 *data, u8 ndata); |
1965 | void kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu); |
1966 | |
1967 | void kvm_enable_efer_bits(u64); |
1968 | bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer); |
1969 | int __kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data, bool host_initiated); |
1970 | int kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data); |
1971 | int kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data); |
1972 | int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu); |
1973 | int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu); |
1974 | int kvm_emulate_as_nop(struct kvm_vcpu *vcpu); |
1975 | int kvm_emulate_invd(struct kvm_vcpu *vcpu); |
1976 | int kvm_emulate_mwait(struct kvm_vcpu *vcpu); |
1977 | int kvm_handle_invalid_op(struct kvm_vcpu *vcpu); |
1978 | int kvm_emulate_monitor(struct kvm_vcpu *vcpu); |
1979 | |
1980 | int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in); |
1981 | int kvm_emulate_cpuid(struct kvm_vcpu *vcpu); |
1982 | int kvm_emulate_halt(struct kvm_vcpu *vcpu); |
1983 | int kvm_emulate_halt_noskip(struct kvm_vcpu *vcpu); |
1984 | int kvm_emulate_ap_reset_hold(struct kvm_vcpu *vcpu); |
1985 | int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu); |
1986 | |
1987 | void kvm_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg); |
1988 | void kvm_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg); |
1989 | int kvm_load_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector, int seg); |
1990 | void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector); |
1991 | |
1992 | int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index, |
1993 | int reason, bool has_error_code, u32 error_code); |
1994 | |
1995 | void kvm_post_set_cr0(struct kvm_vcpu *vcpu, unsigned long old_cr0, unsigned long cr0); |
1996 | void kvm_post_set_cr4(struct kvm_vcpu *vcpu, unsigned long old_cr4, unsigned long cr4); |
1997 | int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0); |
1998 | int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3); |
1999 | int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4); |
2000 | int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8); |
2001 | int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val); |
2002 | void kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val); |
2003 | unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu); |
2004 | void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw); |
2005 | int kvm_emulate_xsetbv(struct kvm_vcpu *vcpu); |
2006 | |
2007 | int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr); |
2008 | int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr); |
2009 | |
2010 | unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu); |
2011 | void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags); |
2012 | int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu); |
2013 | |
2014 | void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr); |
2015 | void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code); |
2016 | void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr, unsigned long payload); |
2017 | void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr); |
2018 | void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code); |
2019 | void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault); |
2020 | void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu, |
2021 | struct x86_exception *fault); |
2022 | bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl); |
2023 | bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr); |
2024 | |
2025 | static inline int __kvm_irq_line_state(unsigned long *irq_state, |
2026 | int irq_source_id, int level) |
2027 | { |
2028 | /* Logical OR for level trig interrupt */ |
2029 | if (level) |
2030 | __set_bit(irq_source_id, irq_state); |
2031 | else |
2032 | __clear_bit(irq_source_id, irq_state); |
2033 | |
2034 | return !!(*irq_state); |
2035 | } |
2036 | |
2037 | int kvm_pic_set_irq(struct kvm_pic *pic, int irq, int irq_source_id, int level); |
2038 | void kvm_pic_clear_all(struct kvm_pic *pic, int irq_source_id); |
2039 | |
2040 | void kvm_inject_nmi(struct kvm_vcpu *vcpu); |
2041 | int kvm_get_nr_pending_nmis(struct kvm_vcpu *vcpu); |
2042 | |
2043 | void kvm_update_dr7(struct kvm_vcpu *vcpu); |
2044 | |
2045 | int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn); |
2046 | void kvm_mmu_free_roots(struct kvm *kvm, struct kvm_mmu *mmu, |
2047 | ulong roots_to_free); |
2048 | void kvm_mmu_free_guest_mode_roots(struct kvm *kvm, struct kvm_mmu *mmu); |
2049 | gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva, |
2050 | struct x86_exception *exception); |
2051 | gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva, |
2052 | struct x86_exception *exception); |
2053 | gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva, |
2054 | struct x86_exception *exception); |
2055 | |
2056 | bool kvm_apicv_activated(struct kvm *kvm); |
2057 | bool kvm_vcpu_apicv_activated(struct kvm_vcpu *vcpu); |
2058 | void __kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu); |
2059 | void __kvm_set_or_clear_apicv_inhibit(struct kvm *kvm, |
2060 | enum kvm_apicv_inhibit reason, bool set); |
2061 | void kvm_set_or_clear_apicv_inhibit(struct kvm *kvm, |
2062 | enum kvm_apicv_inhibit reason, bool set); |
2063 | |
2064 | static inline void kvm_set_apicv_inhibit(struct kvm *kvm, |
2065 | enum kvm_apicv_inhibit reason) |
2066 | { |
2067 | kvm_set_or_clear_apicv_inhibit(kvm, reason, set: true); |
2068 | } |
2069 | |
2070 | static inline void kvm_clear_apicv_inhibit(struct kvm *kvm, |
2071 | enum kvm_apicv_inhibit reason) |
2072 | { |
2073 | kvm_set_or_clear_apicv_inhibit(kvm, reason, set: false); |
2074 | } |
2075 | |
2076 | int kvm_emulate_hypercall(struct kvm_vcpu *vcpu); |
2077 | |
2078 | int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code, |
2079 | void *insn, int insn_len); |
2080 | void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva); |
2081 | void kvm_mmu_invalidate_addr(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, |
2082 | u64 addr, unsigned long roots); |
2083 | void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid); |
2084 | void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd); |
2085 | |
2086 | void kvm_configure_mmu(bool enable_tdp, int tdp_forced_root_level, |
2087 | int tdp_max_root_level, int tdp_huge_page_level); |
2088 | |
2089 | static inline u16 kvm_read_ldt(void) |
2090 | { |
2091 | u16 ldt; |
2092 | asm("sldt %0" : "=g" (ldt)); |
2093 | return ldt; |
2094 | } |
2095 | |
2096 | static inline void kvm_load_ldt(u16 sel) |
2097 | { |
2098 | asm("lldt %0" : : "rm" (sel)); |
2099 | } |
2100 | |
2101 | #ifdef CONFIG_X86_64 |
2102 | static inline unsigned long read_msr(unsigned long msr) |
2103 | { |
2104 | u64 value; |
2105 | |
2106 | rdmsrl(msr, value); |
2107 | return value; |
2108 | } |
2109 | #endif |
2110 | |
2111 | static inline void kvm_inject_gp(struct kvm_vcpu *vcpu, u32 error_code) |
2112 | { |
2113 | kvm_queue_exception_e(vcpu, GP_VECTOR, error_code); |
2114 | } |
2115 | |
2116 | #define TSS_IOPB_BASE_OFFSET 0x66 |
2117 | #define TSS_BASE_SIZE 0x68 |
2118 | #define TSS_IOPB_SIZE (65536 / 8) |
2119 | #define TSS_REDIRECTION_SIZE (256 / 8) |
2120 | #define RMODE_TSS_SIZE \ |
2121 | (TSS_BASE_SIZE + TSS_REDIRECTION_SIZE + TSS_IOPB_SIZE + 1) |
2122 | |
2123 | enum { |
2124 | TASK_SWITCH_CALL = 0, |
2125 | TASK_SWITCH_IRET = 1, |
2126 | TASK_SWITCH_JMP = 2, |
2127 | TASK_SWITCH_GATE = 3, |
2128 | }; |
2129 | |
2130 | #define HF_GUEST_MASK (1 << 0) /* VCPU is in guest-mode */ |
2131 | |
2132 | #ifdef CONFIG_KVM_SMM |
2133 | #define HF_SMM_MASK (1 << 1) |
2134 | #define HF_SMM_INSIDE_NMI_MASK (1 << 2) |
2135 | |
2136 | # define __KVM_VCPU_MULTIPLE_ADDRESS_SPACE |
2137 | # define KVM_ADDRESS_SPACE_NUM 2 |
2138 | # define kvm_arch_vcpu_memslots_id(vcpu) ((vcpu)->arch.hflags & HF_SMM_MASK ? 1 : 0) |
2139 | # define kvm_memslots_for_spte_role(kvm, role) __kvm_memslots(kvm, (role).smm) |
2140 | #else |
2141 | # define kvm_memslots_for_spte_role(kvm, role) __kvm_memslots(kvm, 0) |
2142 | #endif |
2143 | |
2144 | #define KVM_ARCH_WANT_MMU_NOTIFIER |
2145 | |
2146 | int kvm_cpu_has_injectable_intr(struct kvm_vcpu *v); |
2147 | int kvm_cpu_has_interrupt(struct kvm_vcpu *vcpu); |
2148 | int kvm_cpu_has_extint(struct kvm_vcpu *v); |
2149 | int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu); |
2150 | int kvm_cpu_get_interrupt(struct kvm_vcpu *v); |
2151 | void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event); |
2152 | |
2153 | int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low, |
2154 | unsigned long ipi_bitmap_high, u32 min, |
2155 | unsigned long icr, int op_64_bit); |
2156 | |
2157 | int kvm_add_user_return_msr(u32 msr); |
2158 | int kvm_find_user_return_msr(u32 msr); |
2159 | int kvm_set_user_return_msr(unsigned index, u64 val, u64 mask); |
2160 | |
2161 | static inline bool kvm_is_supported_user_return_msr(u32 msr) |
2162 | { |
2163 | return kvm_find_user_return_msr(msr) >= 0; |
2164 | } |
2165 | |
2166 | u64 kvm_scale_tsc(u64 tsc, u64 ratio); |
2167 | u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc); |
2168 | u64 kvm_calc_nested_tsc_offset(u64 l1_offset, u64 l2_offset, u64 l2_multiplier); |
2169 | u64 kvm_calc_nested_tsc_multiplier(u64 l1_multiplier, u64 l2_multiplier); |
2170 | |
2171 | unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu); |
2172 | bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip); |
2173 | |
2174 | void kvm_make_scan_ioapic_request(struct kvm *kvm); |
2175 | void kvm_make_scan_ioapic_request_mask(struct kvm *kvm, |
2176 | unsigned long *vcpu_bitmap); |
2177 | |
2178 | bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, |
2179 | struct kvm_async_pf *work); |
2180 | void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, |
2181 | struct kvm_async_pf *work); |
2182 | void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, |
2183 | struct kvm_async_pf *work); |
2184 | void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu); |
2185 | bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu); |
2186 | extern bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn); |
2187 | |
2188 | int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu); |
2189 | int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err); |
2190 | void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu); |
2191 | |
2192 | void __user *__x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, |
2193 | u32 size); |
2194 | bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu); |
2195 | bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu); |
2196 | |
2197 | bool kvm_intr_is_single_vcpu(struct kvm *kvm, struct kvm_lapic_irq *irq, |
2198 | struct kvm_vcpu **dest_vcpu); |
2199 | |
2200 | void kvm_set_msi_irq(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, |
2201 | struct kvm_lapic_irq *irq); |
2202 | |
2203 | static inline bool kvm_irq_is_postable(struct kvm_lapic_irq *irq) |
2204 | { |
2205 | /* We can only post Fixed and LowPrio IRQs */ |
2206 | return (irq->delivery_mode == APIC_DM_FIXED || |
2207 | irq->delivery_mode == APIC_DM_LOWEST); |
2208 | } |
2209 | |
2210 | static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) |
2211 | { |
2212 | static_call_cond(kvm_x86_vcpu_blocking)(vcpu); |
2213 | } |
2214 | |
2215 | static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) |
2216 | { |
2217 | static_call_cond(kvm_x86_vcpu_unblocking)(vcpu); |
2218 | } |
2219 | |
2220 | static inline int kvm_cpu_get_apicid(int mps_cpu) |
2221 | { |
2222 | #ifdef CONFIG_X86_LOCAL_APIC |
2223 | return default_cpu_present_to_apicid(mps_cpu); |
2224 | #else |
2225 | WARN_ON_ONCE(1); |
2226 | return BAD_APICID; |
2227 | #endif |
2228 | } |
2229 | |
2230 | int memslot_rmap_alloc(struct kvm_memory_slot *slot, unsigned long npages); |
2231 | |
2232 | #define KVM_CLOCK_VALID_FLAGS \ |
2233 | (KVM_CLOCK_TSC_STABLE | KVM_CLOCK_REALTIME | KVM_CLOCK_HOST_TSC) |
2234 | |
2235 | #define KVM_X86_VALID_QUIRKS \ |
2236 | (KVM_X86_QUIRK_LINT0_REENABLED | \ |
2237 | KVM_X86_QUIRK_CD_NW_CLEARED | \ |
2238 | KVM_X86_QUIRK_LAPIC_MMIO_HOLE | \ |
2239 | KVM_X86_QUIRK_OUT_7E_INC_RIP | \ |
2240 | KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT | \ |
2241 | KVM_X86_QUIRK_FIX_HYPERCALL_INSN | \ |
2242 | KVM_X86_QUIRK_MWAIT_NEVER_UD_FAULTS) |
2243 | |
2244 | /* |
2245 | * KVM previously used a u32 field in kvm_run to indicate the hypercall was |
2246 | * initiated from long mode. KVM now sets bit 0 to indicate long mode, but the |
2247 | * remaining 31 lower bits must be 0 to preserve ABI. |
2248 | */ |
2249 | #define KVM_EXIT_HYPERCALL_MBZ GENMASK_ULL(31, 1) |
2250 | |
2251 | #endif /* _ASM_X86_KVM_HOST_H */ |
2252 | |