1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #ifndef _ASM_X86_MSHYPER_H |
3 | #define _ASM_X86_MSHYPER_H |
4 | |
5 | #include <linux/types.h> |
6 | #include <linux/atomic.h> |
7 | #include <linux/nmi.h> |
8 | #include <asm/io.h> |
9 | #include <asm/hyperv-tlfs.h> |
10 | #include <asm/nospec-branch.h> |
11 | |
12 | #define VP_INVAL U32_MAX |
13 | |
14 | struct ms_hyperv_info { |
15 | u32 features; |
16 | u32 misc_features; |
17 | u32 hints; |
18 | u32 nested_features; |
19 | u32 max_vp_index; |
20 | u32 max_lp_index; |
21 | }; |
22 | |
23 | extern struct ms_hyperv_info ms_hyperv; |
24 | |
25 | |
26 | typedef int (*hyperv_fill_flush_list_func)( |
27 | struct hv_guest_mapping_flush_list *flush, |
28 | void *data); |
29 | |
30 | /* |
31 | * Generate the guest ID. |
32 | */ |
33 | |
34 | static inline __u64 generate_guest_id(__u64 d_info1, __u64 kernel_version, |
35 | __u64 d_info2) |
36 | { |
37 | __u64 guest_id = 0; |
38 | |
39 | guest_id = (((__u64)HV_LINUX_VENDOR_ID) << 48); |
40 | guest_id |= (d_info1 << 48); |
41 | guest_id |= (kernel_version << 16); |
42 | guest_id |= d_info2; |
43 | |
44 | return guest_id; |
45 | } |
46 | |
47 | |
48 | /* Free the message slot and signal end-of-message if required */ |
49 | static inline void vmbus_signal_eom(struct hv_message *msg, u32 old_msg_type) |
50 | { |
51 | /* |
52 | * On crash we're reading some other CPU's message page and we need |
53 | * to be careful: this other CPU may already had cleared the header |
54 | * and the host may already had delivered some other message there. |
55 | * In case we blindly write msg->header.message_type we're going |
56 | * to lose it. We can still lose a message of the same type but |
57 | * we count on the fact that there can only be one |
58 | * CHANNELMSG_UNLOAD_RESPONSE and we don't care about other messages |
59 | * on crash. |
60 | */ |
61 | if (cmpxchg(&msg->header.message_type, old_msg_type, |
62 | HVMSG_NONE) != old_msg_type) |
63 | return; |
64 | |
65 | /* |
66 | * Make sure the write to MessageType (ie set to |
67 | * HVMSG_NONE) happens before we read the |
68 | * MessagePending and EOMing. Otherwise, the EOMing |
69 | * will not deliver any more messages since there is |
70 | * no empty slot |
71 | */ |
72 | mb(); |
73 | |
74 | if (msg->header.message_flags.msg_pending) { |
75 | /* |
76 | * This will cause message queue rescan to |
77 | * possibly deliver another msg from the |
78 | * hypervisor |
79 | */ |
80 | wrmsrl(HV_X64_MSR_EOM, 0); |
81 | } |
82 | } |
83 | |
84 | #define hv_init_timer(timer, tick) \ |
85 | wrmsrl(HV_X64_MSR_STIMER0_COUNT + (2*timer), tick) |
86 | #define hv_init_timer_config(timer, val) \ |
87 | wrmsrl(HV_X64_MSR_STIMER0_CONFIG + (2*timer), val) |
88 | |
89 | #define hv_get_simp(val) rdmsrl(HV_X64_MSR_SIMP, val) |
90 | #define hv_set_simp(val) wrmsrl(HV_X64_MSR_SIMP, val) |
91 | |
92 | #define hv_get_siefp(val) rdmsrl(HV_X64_MSR_SIEFP, val) |
93 | #define hv_set_siefp(val) wrmsrl(HV_X64_MSR_SIEFP, val) |
94 | |
95 | #define hv_get_synic_state(val) rdmsrl(HV_X64_MSR_SCONTROL, val) |
96 | #define hv_set_synic_state(val) wrmsrl(HV_X64_MSR_SCONTROL, val) |
97 | |
98 | #define hv_get_vp_index(index) rdmsrl(HV_X64_MSR_VP_INDEX, index) |
99 | |
100 | #define hv_get_synint_state(int_num, val) \ |
101 | rdmsrl(HV_X64_MSR_SINT0 + int_num, val) |
102 | #define hv_set_synint_state(int_num, val) \ |
103 | wrmsrl(HV_X64_MSR_SINT0 + int_num, val) |
104 | |
105 | #define hv_get_crash_ctl(val) \ |
106 | rdmsrl(HV_X64_MSR_CRASH_CTL, val) |
107 | |
108 | void hyperv_callback_vector(void); |
109 | void hyperv_reenlightenment_vector(void); |
110 | #ifdef CONFIG_TRACING |
111 | #define trace_hyperv_callback_vector hyperv_callback_vector |
112 | #endif |
113 | void hyperv_vector_handler(struct pt_regs *regs); |
114 | void hv_setup_vmbus_irq(void (*handler)(void)); |
115 | void hv_remove_vmbus_irq(void); |
116 | |
117 | void hv_setup_kexec_handler(void (*handler)(void)); |
118 | void hv_remove_kexec_handler(void); |
119 | void hv_setup_crash_handler(void (*handler)(struct pt_regs *regs)); |
120 | void hv_remove_crash_handler(void); |
121 | |
122 | /* |
123 | * Routines for stimer0 Direct Mode handling. |
124 | * On x86/x64, there are no percpu actions to take. |
125 | */ |
126 | void hv_stimer0_vector_handler(struct pt_regs *regs); |
127 | void hv_stimer0_callback_vector(void); |
128 | int hv_setup_stimer0_irq(int *irq, int *vector, void (*handler)(void)); |
129 | void hv_remove_stimer0_irq(int irq); |
130 | |
131 | static inline void hv_enable_stimer0_percpu_irq(int irq) {} |
132 | static inline void hv_disable_stimer0_percpu_irq(int irq) {} |
133 | |
134 | |
135 | #if IS_ENABLED(CONFIG_HYPERV) |
136 | extern struct clocksource *hyperv_cs; |
137 | extern void *hv_hypercall_pg; |
138 | extern void __percpu **hyperv_pcpu_input_arg; |
139 | |
140 | static inline u64 hv_do_hypercall(u64 control, void *input, void *output) |
141 | { |
142 | u64 input_address = input ? virt_to_phys(input) : 0; |
143 | u64 output_address = output ? virt_to_phys(output) : 0; |
144 | u64 hv_status; |
145 | |
146 | #ifdef CONFIG_X86_64 |
147 | if (!hv_hypercall_pg) |
148 | return U64_MAX; |
149 | |
150 | __asm__ __volatile__("mov %4, %%r8\n" |
151 | CALL_NOSPEC |
152 | : "=a" (hv_status), ASM_CALL_CONSTRAINT, |
153 | "+c" (control), "+d" (input_address) |
154 | : "r" (output_address), |
155 | THUNK_TARGET(hv_hypercall_pg) |
156 | : "cc" , "memory" , "r8" , "r9" , "r10" , "r11" ); |
157 | #else |
158 | u32 input_address_hi = upper_32_bits(input_address); |
159 | u32 input_address_lo = lower_32_bits(input_address); |
160 | u32 output_address_hi = upper_32_bits(output_address); |
161 | u32 output_address_lo = lower_32_bits(output_address); |
162 | |
163 | if (!hv_hypercall_pg) |
164 | return U64_MAX; |
165 | |
166 | __asm__ __volatile__(CALL_NOSPEC |
167 | : "=A" (hv_status), |
168 | "+c" (input_address_lo), ASM_CALL_CONSTRAINT |
169 | : "A" (control), |
170 | "b" (input_address_hi), |
171 | "D" (output_address_hi), "S" (output_address_lo), |
172 | THUNK_TARGET(hv_hypercall_pg) |
173 | : "cc" , "memory" ); |
174 | #endif /* !x86_64 */ |
175 | return hv_status; |
176 | } |
177 | |
178 | /* Fast hypercall with 8 bytes of input and no output */ |
179 | static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1) |
180 | { |
181 | u64 hv_status, control = (u64)code | HV_HYPERCALL_FAST_BIT; |
182 | |
183 | #ifdef CONFIG_X86_64 |
184 | { |
185 | __asm__ __volatile__(CALL_NOSPEC |
186 | : "=a" (hv_status), ASM_CALL_CONSTRAINT, |
187 | "+c" (control), "+d" (input1) |
188 | : THUNK_TARGET(hv_hypercall_pg) |
189 | : "cc" , "r8" , "r9" , "r10" , "r11" ); |
190 | } |
191 | #else |
192 | { |
193 | u32 input1_hi = upper_32_bits(input1); |
194 | u32 input1_lo = lower_32_bits(input1); |
195 | |
196 | __asm__ __volatile__ (CALL_NOSPEC |
197 | : "=A" (hv_status), |
198 | "+c" (input1_lo), |
199 | ASM_CALL_CONSTRAINT |
200 | : "A" (control), |
201 | "b" (input1_hi), |
202 | THUNK_TARGET(hv_hypercall_pg) |
203 | : "cc" , "edi" , "esi" ); |
204 | } |
205 | #endif |
206 | return hv_status; |
207 | } |
208 | |
209 | /* Fast hypercall with 16 bytes of input */ |
210 | static inline u64 hv_do_fast_hypercall16(u16 code, u64 input1, u64 input2) |
211 | { |
212 | u64 hv_status, control = (u64)code | HV_HYPERCALL_FAST_BIT; |
213 | |
214 | #ifdef CONFIG_X86_64 |
215 | { |
216 | __asm__ __volatile__("mov %4, %%r8\n" |
217 | CALL_NOSPEC |
218 | : "=a" (hv_status), ASM_CALL_CONSTRAINT, |
219 | "+c" (control), "+d" (input1) |
220 | : "r" (input2), |
221 | THUNK_TARGET(hv_hypercall_pg) |
222 | : "cc" , "r8" , "r9" , "r10" , "r11" ); |
223 | } |
224 | #else |
225 | { |
226 | u32 input1_hi = upper_32_bits(input1); |
227 | u32 input1_lo = lower_32_bits(input1); |
228 | u32 input2_hi = upper_32_bits(input2); |
229 | u32 input2_lo = lower_32_bits(input2); |
230 | |
231 | __asm__ __volatile__ (CALL_NOSPEC |
232 | : "=A" (hv_status), |
233 | "+c" (input1_lo), ASM_CALL_CONSTRAINT |
234 | : "A" (control), "b" (input1_hi), |
235 | "D" (input2_hi), "S" (input2_lo), |
236 | THUNK_TARGET(hv_hypercall_pg) |
237 | : "cc" ); |
238 | } |
239 | #endif |
240 | return hv_status; |
241 | } |
242 | |
243 | /* |
244 | * Rep hypercalls. Callers of this functions are supposed to ensure that |
245 | * rep_count and varhead_size comply with Hyper-V hypercall definition. |
246 | */ |
247 | static inline u64 hv_do_rep_hypercall(u16 code, u16 rep_count, u16 varhead_size, |
248 | void *input, void *output) |
249 | { |
250 | u64 control = code; |
251 | u64 status; |
252 | u16 rep_comp; |
253 | |
254 | control |= (u64)varhead_size << HV_HYPERCALL_VARHEAD_OFFSET; |
255 | control |= (u64)rep_count << HV_HYPERCALL_REP_COMP_OFFSET; |
256 | |
257 | do { |
258 | status = hv_do_hypercall(control, input, output); |
259 | if ((status & HV_HYPERCALL_RESULT_MASK) != HV_STATUS_SUCCESS) |
260 | return status; |
261 | |
262 | /* Bits 32-43 of status have 'Reps completed' data. */ |
263 | rep_comp = (status & HV_HYPERCALL_REP_COMP_MASK) >> |
264 | HV_HYPERCALL_REP_COMP_OFFSET; |
265 | |
266 | control &= ~HV_HYPERCALL_REP_START_MASK; |
267 | control |= (u64)rep_comp << HV_HYPERCALL_REP_START_OFFSET; |
268 | |
269 | touch_nmi_watchdog(); |
270 | } while (rep_comp < rep_count); |
271 | |
272 | return status; |
273 | } |
274 | |
275 | /* |
276 | * Hypervisor's notion of virtual processor ID is different from |
277 | * Linux' notion of CPU ID. This information can only be retrieved |
278 | * in the context of the calling CPU. Setup a map for easy access |
279 | * to this information. |
280 | */ |
281 | extern u32 *hv_vp_index; |
282 | extern u32 hv_max_vp_index; |
283 | extern struct hv_vp_assist_page **hv_vp_assist_page; |
284 | |
285 | static inline struct hv_vp_assist_page *hv_get_vp_assist_page(unsigned int cpu) |
286 | { |
287 | if (!hv_vp_assist_page) |
288 | return NULL; |
289 | |
290 | return hv_vp_assist_page[cpu]; |
291 | } |
292 | |
293 | /** |
294 | * hv_cpu_number_to_vp_number() - Map CPU to VP. |
295 | * @cpu_number: CPU number in Linux terms |
296 | * |
297 | * This function returns the mapping between the Linux processor |
298 | * number and the hypervisor's virtual processor number, useful |
299 | * in making hypercalls and such that talk about specific |
300 | * processors. |
301 | * |
302 | * Return: Virtual processor number in Hyper-V terms |
303 | */ |
304 | static inline int hv_cpu_number_to_vp_number(int cpu_number) |
305 | { |
306 | return hv_vp_index[cpu_number]; |
307 | } |
308 | |
309 | static inline int cpumask_to_vpset(struct hv_vpset *vpset, |
310 | const struct cpumask *cpus) |
311 | { |
312 | int cpu, vcpu, vcpu_bank, vcpu_offset, nr_bank = 1; |
313 | |
314 | /* valid_bank_mask can represent up to 64 banks */ |
315 | if (hv_max_vp_index / 64 >= 64) |
316 | return 0; |
317 | |
318 | /* |
319 | * Clear all banks up to the maximum possible bank as hv_tlb_flush_ex |
320 | * structs are not cleared between calls, we risk flushing unneeded |
321 | * vCPUs otherwise. |
322 | */ |
323 | for (vcpu_bank = 0; vcpu_bank <= hv_max_vp_index / 64; vcpu_bank++) |
324 | vpset->bank_contents[vcpu_bank] = 0; |
325 | |
326 | /* |
327 | * Some banks may end up being empty but this is acceptable. |
328 | */ |
329 | for_each_cpu(cpu, cpus) { |
330 | vcpu = hv_cpu_number_to_vp_number(cpu); |
331 | if (vcpu == VP_INVAL) |
332 | return -1; |
333 | vcpu_bank = vcpu / 64; |
334 | vcpu_offset = vcpu % 64; |
335 | __set_bit(vcpu_offset, (unsigned long *) |
336 | &vpset->bank_contents[vcpu_bank]); |
337 | if (vcpu_bank >= nr_bank) |
338 | nr_bank = vcpu_bank + 1; |
339 | } |
340 | vpset->valid_bank_mask = GENMASK_ULL(nr_bank - 1, 0); |
341 | return nr_bank; |
342 | } |
343 | |
344 | void __init hyperv_init(void); |
345 | void hyperv_setup_mmu_ops(void); |
346 | void hyperv_report_panic(struct pt_regs *regs, long err); |
347 | void hyperv_report_panic_msg(phys_addr_t pa, size_t size); |
348 | bool hv_is_hyperv_initialized(void); |
349 | void hyperv_cleanup(void); |
350 | |
351 | void hyperv_reenlightenment_intr(struct pt_regs *regs); |
352 | void set_hv_tscchange_cb(void (*cb)(void)); |
353 | void clear_hv_tscchange_cb(void); |
354 | void hyperv_stop_tsc_emulation(void); |
355 | int hyperv_flush_guest_mapping(u64 as); |
356 | int hyperv_flush_guest_mapping_range(u64 as, |
357 | hyperv_fill_flush_list_func fill_func, void *data); |
358 | int hyperv_fill_flush_guest_mapping_list( |
359 | struct hv_guest_mapping_flush_list *flush, |
360 | u64 start_gfn, u64 end_gfn); |
361 | |
362 | #ifdef CONFIG_X86_64 |
363 | void hv_apic_init(void); |
364 | void __init hv_init_spinlocks(void); |
365 | bool hv_vcpu_is_preempted(int vcpu); |
366 | #else |
367 | static inline void hv_apic_init(void) {} |
368 | #endif |
369 | |
370 | #else /* CONFIG_HYPERV */ |
371 | static inline void hyperv_init(void) {} |
372 | static inline bool hv_is_hyperv_initialized(void) { return false; } |
373 | static inline void hyperv_cleanup(void) {} |
374 | static inline void hyperv_setup_mmu_ops(void) {} |
375 | static inline void set_hv_tscchange_cb(void (*cb)(void)) {} |
376 | static inline void clear_hv_tscchange_cb(void) {} |
377 | static inline void hyperv_stop_tsc_emulation(void) {}; |
378 | static inline struct hv_vp_assist_page *hv_get_vp_assist_page(unsigned int cpu) |
379 | { |
380 | return NULL; |
381 | } |
382 | static inline int hyperv_flush_guest_mapping(u64 as) { return -1; } |
383 | static inline int hyperv_flush_guest_mapping_range(u64 as, |
384 | hyperv_fill_flush_list_func fill_func, void *data) |
385 | { |
386 | return -1; |
387 | } |
388 | #endif /* CONFIG_HYPERV */ |
389 | |
390 | #ifdef CONFIG_HYPERV_TSCPAGE |
391 | struct ms_hyperv_tsc_page *hv_get_tsc_page(void); |
392 | static inline u64 hv_read_tsc_page_tsc(const struct ms_hyperv_tsc_page *tsc_pg, |
393 | u64 *cur_tsc) |
394 | { |
395 | u64 scale, offset; |
396 | u32 sequence; |
397 | |
398 | /* |
399 | * The protocol for reading Hyper-V TSC page is specified in Hypervisor |
400 | * Top-Level Functional Specification ver. 3.0 and above. To get the |
401 | * reference time we must do the following: |
402 | * - READ ReferenceTscSequence |
403 | * A special '0' value indicates the time source is unreliable and we |
404 | * need to use something else. The currently published specification |
405 | * versions (up to 4.0b) contain a mistake and wrongly claim '-1' |
406 | * instead of '0' as the special value, see commit c35b82ef0294. |
407 | * - ReferenceTime = |
408 | * ((RDTSC() * ReferenceTscScale) >> 64) + ReferenceTscOffset |
409 | * - READ ReferenceTscSequence again. In case its value has changed |
410 | * since our first reading we need to discard ReferenceTime and repeat |
411 | * the whole sequence as the hypervisor was updating the page in |
412 | * between. |
413 | */ |
414 | do { |
415 | sequence = READ_ONCE(tsc_pg->tsc_sequence); |
416 | if (!sequence) |
417 | return U64_MAX; |
418 | /* |
419 | * Make sure we read sequence before we read other values from |
420 | * TSC page. |
421 | */ |
422 | smp_rmb(); |
423 | |
424 | scale = READ_ONCE(tsc_pg->tsc_scale); |
425 | offset = READ_ONCE(tsc_pg->tsc_offset); |
426 | *cur_tsc = rdtsc_ordered(); |
427 | |
428 | /* |
429 | * Make sure we read sequence after we read all other values |
430 | * from TSC page. |
431 | */ |
432 | smp_rmb(); |
433 | |
434 | } while (READ_ONCE(tsc_pg->tsc_sequence) != sequence); |
435 | |
436 | return mul_u64_u64_shr(*cur_tsc, scale, 64) + offset; |
437 | } |
438 | |
439 | static inline u64 hv_read_tsc_page(const struct ms_hyperv_tsc_page *tsc_pg) |
440 | { |
441 | u64 cur_tsc; |
442 | |
443 | return hv_read_tsc_page_tsc(tsc_pg, &cur_tsc); |
444 | } |
445 | |
446 | #else |
447 | static inline struct ms_hyperv_tsc_page *hv_get_tsc_page(void) |
448 | { |
449 | return NULL; |
450 | } |
451 | |
452 | static inline u64 hv_read_tsc_page_tsc(const struct ms_hyperv_tsc_page *tsc_pg, |
453 | u64 *cur_tsc) |
454 | { |
455 | BUG(); |
456 | return U64_MAX; |
457 | } |
458 | #endif |
459 | #endif |
460 | |