1 | // SPDX-License-Identifier: GPL-2.0 |
2 | |
3 | /* |
4 | * Hyper-V specific APIC code. |
5 | * |
6 | * Copyright (C) 2018, Microsoft, Inc. |
7 | * |
8 | * Author : K. Y. Srinivasan <kys@microsoft.com> |
9 | * |
10 | * This program is free software; you can redistribute it and/or modify it |
11 | * under the terms of the GNU General Public License version 2 as published |
12 | * by the Free Software Foundation. |
13 | * |
14 | * This program is distributed in the hope that it will be useful, but |
15 | * WITHOUT ANY WARRANTY; without even the implied warranty of |
16 | * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or |
17 | * NON INFRINGEMENT. See the GNU General Public License for more |
18 | * details. |
19 | * |
20 | */ |
21 | |
22 | #include <linux/types.h> |
23 | #include <linux/vmalloc.h> |
24 | #include <linux/mm.h> |
25 | #include <linux/clockchips.h> |
26 | #include <linux/hyperv.h> |
27 | #include <linux/slab.h> |
28 | #include <linux/cpuhotplug.h> |
29 | #include <asm/hypervisor.h> |
30 | #include <asm/mshyperv.h> |
31 | #include <asm/apic.h> |
32 | |
33 | #include <asm/trace/hyperv.h> |
34 | |
35 | static struct apic orig_apic; |
36 | |
37 | static u64 hv_apic_icr_read(void) |
38 | { |
39 | u64 reg_val; |
40 | |
41 | rdmsrl(HV_X64_MSR_ICR, reg_val); |
42 | return reg_val; |
43 | } |
44 | |
45 | static void hv_apic_icr_write(u32 low, u32 id) |
46 | { |
47 | u64 reg_val; |
48 | |
49 | reg_val = SET_XAPIC_DEST_FIELD(id); |
50 | reg_val = reg_val << 32; |
51 | reg_val |= low; |
52 | |
53 | wrmsrl(HV_X64_MSR_ICR, val: reg_val); |
54 | } |
55 | |
56 | static u32 hv_apic_read(u32 reg) |
57 | { |
58 | u32 reg_val, hi; |
59 | |
60 | switch (reg) { |
61 | case APIC_EOI: |
62 | rdmsr(HV_X64_MSR_EOI, reg_val, hi); |
63 | (void)hi; |
64 | return reg_val; |
65 | case APIC_TASKPRI: |
66 | rdmsr(HV_X64_MSR_TPR, reg_val, hi); |
67 | (void)hi; |
68 | return reg_val; |
69 | |
70 | default: |
71 | return native_apic_mem_read(reg); |
72 | } |
73 | } |
74 | |
75 | static void hv_apic_write(u32 reg, u32 val) |
76 | { |
77 | switch (reg) { |
78 | case APIC_EOI: |
79 | wrmsr(HV_X64_MSR_EOI, val, 0); |
80 | break; |
81 | case APIC_TASKPRI: |
82 | wrmsr(HV_X64_MSR_TPR, val, 0); |
83 | break; |
84 | default: |
85 | native_apic_mem_write(reg, v: val); |
86 | } |
87 | } |
88 | |
89 | static void hv_apic_eoi_write(void) |
90 | { |
91 | struct hv_vp_assist_page *hvp = hv_vp_assist_page[smp_processor_id()]; |
92 | |
93 | if (hvp && (xchg(&hvp->apic_assist, 0) & 0x1)) |
94 | return; |
95 | |
96 | wrmsr(HV_X64_MSR_EOI, APIC_EOI_ACK, 0); |
97 | } |
98 | |
99 | static bool cpu_is_self(int cpu) |
100 | { |
101 | return cpu == smp_processor_id(); |
102 | } |
103 | |
104 | /* |
105 | * IPI implementation on Hyper-V. |
106 | */ |
107 | static bool __send_ipi_mask_ex(const struct cpumask *mask, int vector, |
108 | bool exclude_self) |
109 | { |
110 | struct hv_send_ipi_ex *ipi_arg; |
111 | unsigned long flags; |
112 | int nr_bank = 0; |
113 | u64 status = HV_STATUS_INVALID_PARAMETER; |
114 | |
115 | if (!(ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED)) |
116 | return false; |
117 | |
118 | local_irq_save(flags); |
119 | ipi_arg = *this_cpu_ptr(hyperv_pcpu_input_arg); |
120 | |
121 | if (unlikely(!ipi_arg)) |
122 | goto ipi_mask_ex_done; |
123 | |
124 | ipi_arg->vector = vector; |
125 | ipi_arg->reserved = 0; |
126 | ipi_arg->vp_set.valid_bank_mask = 0; |
127 | |
128 | /* |
129 | * Use HV_GENERIC_SET_ALL and avoid converting cpumask to VP_SET |
130 | * when the IPI is sent to all currently present CPUs. |
131 | */ |
132 | if (!cpumask_equal(src1p: mask, cpu_present_mask) || exclude_self) { |
133 | ipi_arg->vp_set.format = HV_GENERIC_SET_SPARSE_4K; |
134 | |
135 | nr_bank = cpumask_to_vpset_skip(vpset: &ipi_arg->vp_set, cpus: mask, |
136 | func: exclude_self ? cpu_is_self : NULL); |
137 | |
138 | /* |
139 | * 'nr_bank <= 0' means some CPUs in cpumask can't be |
140 | * represented in VP_SET. Return an error and fall back to |
141 | * native (architectural) method of sending IPIs. |
142 | */ |
143 | if (nr_bank <= 0) |
144 | goto ipi_mask_ex_done; |
145 | } else { |
146 | ipi_arg->vp_set.format = HV_GENERIC_SET_ALL; |
147 | } |
148 | |
149 | status = hv_do_rep_hypercall(HVCALL_SEND_IPI_EX, rep_count: 0, varhead_size: nr_bank, |
150 | input: ipi_arg, NULL); |
151 | |
152 | ipi_mask_ex_done: |
153 | local_irq_restore(flags); |
154 | return hv_result_success(status); |
155 | } |
156 | |
157 | static bool __send_ipi_mask(const struct cpumask *mask, int vector, |
158 | bool exclude_self) |
159 | { |
160 | int cur_cpu, vcpu, this_cpu = smp_processor_id(); |
161 | struct hv_send_ipi ipi_arg; |
162 | u64 status; |
163 | unsigned int weight; |
164 | |
165 | trace_hyperv_send_ipi_mask(cpus: mask, vector); |
166 | |
167 | weight = cpumask_weight(srcp: mask); |
168 | |
169 | /* |
170 | * Do nothing if |
171 | * 1. the mask is empty |
172 | * 2. the mask only contains self when exclude_self is true |
173 | */ |
174 | if (weight == 0 || |
175 | (exclude_self && weight == 1 && cpumask_test_cpu(cpu: this_cpu, cpumask: mask))) |
176 | return true; |
177 | |
178 | /* A fully enlightened TDX VM uses GHCI rather than hv_hypercall_pg. */ |
179 | if (!hv_hypercall_pg) { |
180 | if (ms_hyperv.paravisor_present || !hv_isolation_type_tdx()) |
181 | return false; |
182 | } |
183 | |
184 | if (vector < HV_IPI_LOW_VECTOR || vector > HV_IPI_HIGH_VECTOR) |
185 | return false; |
186 | |
187 | /* |
188 | * From the supplied CPU set we need to figure out if we can get away |
189 | * with cheaper HVCALL_SEND_IPI hypercall. This is possible when the |
190 | * highest VP number in the set is < 64. As VP numbers are usually in |
191 | * ascending order and match Linux CPU ids, here is an optimization: |
192 | * we check the VP number for the highest bit in the supplied set first |
193 | * so we can quickly find out if using HVCALL_SEND_IPI_EX hypercall is |
194 | * a must. We will also check all VP numbers when walking the supplied |
195 | * CPU set to remain correct in all cases. |
196 | */ |
197 | if (hv_cpu_number_to_vp_number(cpu_number: cpumask_last(srcp: mask)) >= 64) |
198 | goto do_ex_hypercall; |
199 | |
200 | ipi_arg.vector = vector; |
201 | ipi_arg.cpu_mask = 0; |
202 | |
203 | for_each_cpu(cur_cpu, mask) { |
204 | if (exclude_self && cur_cpu == this_cpu) |
205 | continue; |
206 | vcpu = hv_cpu_number_to_vp_number(cpu_number: cur_cpu); |
207 | if (vcpu == VP_INVAL) |
208 | return false; |
209 | |
210 | /* |
211 | * This particular version of the IPI hypercall can |
212 | * only target up to 64 CPUs. |
213 | */ |
214 | if (vcpu >= 64) |
215 | goto do_ex_hypercall; |
216 | |
217 | __set_bit(vcpu, (unsigned long *)&ipi_arg.cpu_mask); |
218 | } |
219 | |
220 | status = hv_do_fast_hypercall16(HVCALL_SEND_IPI, input1: ipi_arg.vector, |
221 | input2: ipi_arg.cpu_mask); |
222 | return hv_result_success(status); |
223 | |
224 | do_ex_hypercall: |
225 | return __send_ipi_mask_ex(mask, vector, exclude_self); |
226 | } |
227 | |
228 | static bool __send_ipi_one(int cpu, int vector) |
229 | { |
230 | int vp = hv_cpu_number_to_vp_number(cpu_number: cpu); |
231 | u64 status; |
232 | |
233 | trace_hyperv_send_ipi_one(cpu, vector); |
234 | |
235 | if (vp == VP_INVAL) |
236 | return false; |
237 | |
238 | /* A fully enlightened TDX VM uses GHCI rather than hv_hypercall_pg. */ |
239 | if (!hv_hypercall_pg) { |
240 | if (ms_hyperv.paravisor_present || !hv_isolation_type_tdx()) |
241 | return false; |
242 | } |
243 | |
244 | if (vector < HV_IPI_LOW_VECTOR || vector > HV_IPI_HIGH_VECTOR) |
245 | return false; |
246 | |
247 | if (vp >= 64) |
248 | return __send_ipi_mask_ex(cpumask_of(cpu), vector, exclude_self: false); |
249 | |
250 | status = hv_do_fast_hypercall16(HVCALL_SEND_IPI, input1: vector, BIT_ULL(vp)); |
251 | return hv_result_success(status); |
252 | } |
253 | |
254 | static void hv_send_ipi(int cpu, int vector) |
255 | { |
256 | if (!__send_ipi_one(cpu, vector)) |
257 | orig_apic.send_IPI(cpu, vector); |
258 | } |
259 | |
260 | static void hv_send_ipi_mask(const struct cpumask *mask, int vector) |
261 | { |
262 | if (!__send_ipi_mask(mask, vector, exclude_self: false)) |
263 | orig_apic.send_IPI_mask(mask, vector); |
264 | } |
265 | |
266 | static void hv_send_ipi_mask_allbutself(const struct cpumask *mask, int vector) |
267 | { |
268 | if (!__send_ipi_mask(mask, vector, exclude_self: true)) |
269 | orig_apic.send_IPI_mask_allbutself(mask, vector); |
270 | } |
271 | |
272 | static void hv_send_ipi_allbutself(int vector) |
273 | { |
274 | hv_send_ipi_mask_allbutself(cpu_online_mask, vector); |
275 | } |
276 | |
277 | static void hv_send_ipi_all(int vector) |
278 | { |
279 | if (!__send_ipi_mask(cpu_online_mask, vector, exclude_self: false)) |
280 | orig_apic.send_IPI_all(vector); |
281 | } |
282 | |
283 | static void hv_send_ipi_self(int vector) |
284 | { |
285 | if (!__send_ipi_one(smp_processor_id(), vector)) |
286 | orig_apic.send_IPI_self(vector); |
287 | } |
288 | |
289 | void __init hv_apic_init(void) |
290 | { |
291 | if (ms_hyperv.hints & HV_X64_CLUSTER_IPI_RECOMMENDED) { |
292 | pr_info("Hyper-V: Using IPI hypercalls\n" ); |
293 | /* |
294 | * Set the IPI entry points. |
295 | */ |
296 | orig_apic = *apic; |
297 | |
298 | apic_update_callback(send_IPI, hv_send_ipi); |
299 | apic_update_callback(send_IPI_mask, hv_send_ipi_mask); |
300 | apic_update_callback(send_IPI_mask_allbutself, hv_send_ipi_mask_allbutself); |
301 | apic_update_callback(send_IPI_allbutself, hv_send_ipi_allbutself); |
302 | apic_update_callback(send_IPI_all, hv_send_ipi_all); |
303 | apic_update_callback(send_IPI_self, hv_send_ipi_self); |
304 | } |
305 | |
306 | if (ms_hyperv.hints & HV_X64_APIC_ACCESS_RECOMMENDED) { |
307 | pr_info("Hyper-V: Using enlightened APIC (%s mode)" , |
308 | x2apic_enabled() ? "x2apic" : "xapic" ); |
309 | /* |
310 | * When in x2apic mode, don't use the Hyper-V specific APIC |
311 | * accessors since the field layout in the ICR register is |
312 | * different in x2apic mode. Furthermore, the architectural |
313 | * x2apic MSRs function just as well as the Hyper-V |
314 | * synthetic APIC MSRs, so there's no benefit in having |
315 | * separate Hyper-V accessors for x2apic mode. The only |
316 | * exception is hv_apic_eoi_write, because it benefits from |
317 | * lazy EOI when available, but the same accessor works for |
318 | * both xapic and x2apic because the field layout is the same. |
319 | */ |
320 | apic_update_callback(eoi, hv_apic_eoi_write); |
321 | if (!x2apic_enabled()) { |
322 | apic_update_callback(read, hv_apic_read); |
323 | apic_update_callback(write, hv_apic_write); |
324 | apic_update_callback(icr_write, hv_apic_icr_write); |
325 | apic_update_callback(icr_read, hv_apic_icr_read); |
326 | } |
327 | } |
328 | } |
329 | |