1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * KVM L1 hypervisor optimizations on Hyper-V. |
4 | */ |
5 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
6 | |
7 | #include <linux/kvm_host.h> |
8 | #include <asm/mshyperv.h> |
9 | |
10 | #include "hyperv.h" |
11 | #include "kvm_onhyperv.h" |
12 | |
13 | struct kvm_hv_tlb_range { |
14 | u64 start_gfn; |
15 | u64 pages; |
16 | }; |
17 | |
18 | static int kvm_fill_hv_flush_list_func(struct hv_guest_mapping_flush_list *flush, |
19 | void *data) |
20 | { |
21 | struct kvm_hv_tlb_range *range = data; |
22 | |
23 | return hyperv_fill_flush_guest_mapping_list(flush, start_gfn: range->start_gfn, |
24 | end_gfn: range->pages); |
25 | } |
26 | |
27 | static inline int hv_remote_flush_root_tdp(hpa_t root_tdp, |
28 | struct kvm_hv_tlb_range *range) |
29 | { |
30 | if (range) |
31 | return hyperv_flush_guest_mapping_range(as: root_tdp, |
32 | fill_func: kvm_fill_hv_flush_list_func, data: (void *)range); |
33 | else |
34 | return hyperv_flush_guest_mapping(as: root_tdp); |
35 | } |
36 | |
37 | static int __hv_flush_remote_tlbs_range(struct kvm *kvm, |
38 | struct kvm_hv_tlb_range *range) |
39 | { |
40 | struct kvm_arch *kvm_arch = &kvm->arch; |
41 | struct kvm_vcpu *vcpu; |
42 | int ret = 0, nr_unique_valid_roots; |
43 | unsigned long i; |
44 | hpa_t root; |
45 | |
46 | spin_lock(lock: &kvm_arch->hv_root_tdp_lock); |
47 | |
48 | if (!VALID_PAGE(kvm_arch->hv_root_tdp)) { |
49 | nr_unique_valid_roots = 0; |
50 | |
51 | /* |
52 | * Flush all valid roots, and see if all vCPUs have converged |
53 | * on a common root, in which case future flushes can skip the |
54 | * loop and flush the common root. |
55 | */ |
56 | kvm_for_each_vcpu(i, vcpu, kvm) { |
57 | root = vcpu->arch.hv_root_tdp; |
58 | if (!VALID_PAGE(root) || root == kvm_arch->hv_root_tdp) |
59 | continue; |
60 | |
61 | /* |
62 | * Set the tracked root to the first valid root. Keep |
63 | * this root for the entirety of the loop even if more |
64 | * roots are encountered as a low effort optimization |
65 | * to avoid flushing the same (first) root again. |
66 | */ |
67 | if (++nr_unique_valid_roots == 1) |
68 | kvm_arch->hv_root_tdp = root; |
69 | |
70 | if (!ret) |
71 | ret = hv_remote_flush_root_tdp(root_tdp: root, range); |
72 | |
73 | /* |
74 | * Stop processing roots if a failure occurred and |
75 | * multiple valid roots have already been detected. |
76 | */ |
77 | if (ret && nr_unique_valid_roots > 1) |
78 | break; |
79 | } |
80 | |
81 | /* |
82 | * The optimized flush of a single root can't be used if there |
83 | * are multiple valid roots (obviously). |
84 | */ |
85 | if (nr_unique_valid_roots > 1) |
86 | kvm_arch->hv_root_tdp = INVALID_PAGE; |
87 | } else { |
88 | ret = hv_remote_flush_root_tdp(root_tdp: kvm_arch->hv_root_tdp, range); |
89 | } |
90 | |
91 | spin_unlock(lock: &kvm_arch->hv_root_tdp_lock); |
92 | return ret; |
93 | } |
94 | |
95 | int hv_flush_remote_tlbs_range(struct kvm *kvm, gfn_t start_gfn, gfn_t nr_pages) |
96 | { |
97 | struct kvm_hv_tlb_range range = { |
98 | .start_gfn = start_gfn, |
99 | .pages = nr_pages, |
100 | }; |
101 | |
102 | return __hv_flush_remote_tlbs_range(kvm, range: &range); |
103 | } |
104 | EXPORT_SYMBOL_GPL(hv_flush_remote_tlbs_range); |
105 | |
106 | int hv_flush_remote_tlbs(struct kvm *kvm) |
107 | { |
108 | return __hv_flush_remote_tlbs_range(kvm, NULL); |
109 | } |
110 | EXPORT_SYMBOL_GPL(hv_flush_remote_tlbs); |
111 | |
112 | void hv_track_root_tdp(struct kvm_vcpu *vcpu, hpa_t root_tdp) |
113 | { |
114 | struct kvm_arch *kvm_arch = &vcpu->kvm->arch; |
115 | |
116 | if (kvm_x86_ops.flush_remote_tlbs == hv_flush_remote_tlbs) { |
117 | spin_lock(lock: &kvm_arch->hv_root_tdp_lock); |
118 | vcpu->arch.hv_root_tdp = root_tdp; |
119 | if (root_tdp != kvm_arch->hv_root_tdp) |
120 | kvm_arch->hv_root_tdp = INVALID_PAGE; |
121 | spin_unlock(lock: &kvm_arch->hv_root_tdp_lock); |
122 | } |
123 | } |
124 | EXPORT_SYMBOL_GPL(hv_track_root_tdp); |
125 | |