1 | // SPDX-License-Identifier: GPL-2.0 |
2 | |
3 | #ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG |
4 | #include <linux/memblock.h> |
5 | #endif |
6 | #include <linux/console.h> |
7 | #include <linux/cpu.h> |
8 | #include <linux/kexec.h> |
9 | #include <linux/slab.h> |
10 | #include <linux/panic_notifier.h> |
11 | |
12 | #include <xen/xen.h> |
13 | #include <xen/features.h> |
14 | #include <xen/interface/sched.h> |
15 | #include <xen/interface/version.h> |
16 | #include <xen/page.h> |
17 | |
18 | #include <asm/xen/hypercall.h> |
19 | #include <asm/xen/hypervisor.h> |
20 | #include <asm/cpu.h> |
21 | #include <asm/e820/api.h> |
22 | #include <asm/setup.h> |
23 | |
24 | #include "xen-ops.h" |
25 | #include "smp.h" |
26 | #include "pmu.h" |
27 | |
28 | EXPORT_SYMBOL_GPL(hypercall_page); |
29 | |
30 | /* |
31 | * Pointer to the xen_vcpu_info structure or |
32 | * &HYPERVISOR_shared_info->vcpu_info[cpu]. See xen_hvm_init_shared_info |
33 | * and xen_vcpu_setup for details. By default it points to share_info->vcpu_info |
34 | * but during boot it is switched to point to xen_vcpu_info. |
35 | * The pointer is used in xen_evtchn_do_upcall to acknowledge pending events. |
36 | */ |
37 | DEFINE_PER_CPU(struct vcpu_info *, xen_vcpu); |
38 | DEFINE_PER_CPU(struct vcpu_info, xen_vcpu_info); |
39 | |
40 | /* Linux <-> Xen vCPU id mapping */ |
41 | DEFINE_PER_CPU(uint32_t, xen_vcpu_id); |
42 | EXPORT_PER_CPU_SYMBOL(xen_vcpu_id); |
43 | |
44 | unsigned long *machine_to_phys_mapping = (void *)MACH2PHYS_VIRT_START; |
45 | EXPORT_SYMBOL(machine_to_phys_mapping); |
46 | unsigned long machine_to_phys_nr; |
47 | EXPORT_SYMBOL(machine_to_phys_nr); |
48 | |
49 | struct start_info *xen_start_info; |
50 | EXPORT_SYMBOL_GPL(xen_start_info); |
51 | |
52 | struct shared_info xen_dummy_shared_info; |
53 | |
54 | __read_mostly bool xen_have_vector_callback = true; |
55 | EXPORT_SYMBOL_GPL(xen_have_vector_callback); |
56 | |
57 | /* |
58 | * NB: These need to live in .data or alike because they're used by |
59 | * xen_prepare_pvh() which runs before clearing the bss. |
60 | */ |
61 | enum xen_domain_type __ro_after_init xen_domain_type = XEN_NATIVE; |
62 | EXPORT_SYMBOL_GPL(xen_domain_type); |
63 | uint32_t __ro_after_init xen_start_flags; |
64 | EXPORT_SYMBOL(xen_start_flags); |
65 | |
66 | /* |
67 | * Point at some empty memory to start with. We map the real shared_info |
68 | * page as soon as fixmap is up and running. |
69 | */ |
70 | struct shared_info *HYPERVISOR_shared_info = &xen_dummy_shared_info; |
71 | |
72 | static int xen_cpu_up_online(unsigned int cpu) |
73 | { |
74 | xen_init_lock_cpu(cpu); |
75 | return 0; |
76 | } |
77 | |
78 | int xen_cpuhp_setup(int (*cpu_up_prepare_cb)(unsigned int), |
79 | int (*cpu_dead_cb)(unsigned int)) |
80 | { |
81 | int rc; |
82 | |
83 | rc = cpuhp_setup_state_nocalls(state: CPUHP_XEN_PREPARE, |
84 | name: "x86/xen/guest:prepare" , |
85 | startup: cpu_up_prepare_cb, teardown: cpu_dead_cb); |
86 | if (rc >= 0) { |
87 | rc = cpuhp_setup_state_nocalls(state: CPUHP_AP_ONLINE_DYN, |
88 | name: "x86/xen/guest:online" , |
89 | startup: xen_cpu_up_online, NULL); |
90 | if (rc < 0) |
91 | cpuhp_remove_state_nocalls(state: CPUHP_XEN_PREPARE); |
92 | } |
93 | |
94 | return rc >= 0 ? 0 : rc; |
95 | } |
96 | |
97 | static void xen_vcpu_setup_restore(int cpu) |
98 | { |
99 | /* Any per_cpu(xen_vcpu) is stale, so reset it */ |
100 | xen_vcpu_info_reset(cpu); |
101 | |
102 | /* |
103 | * For PVH and PVHVM, setup online VCPUs only. The rest will |
104 | * be handled by hotplug. |
105 | */ |
106 | if (xen_pv_domain() || |
107 | (xen_hvm_domain() && cpu_online(cpu))) |
108 | xen_vcpu_setup(cpu); |
109 | } |
110 | |
111 | /* |
112 | * On restore, set the vcpu placement up again. |
113 | * If it fails, then we're in a bad state, since |
114 | * we can't back out from using it... |
115 | */ |
116 | void xen_vcpu_restore(void) |
117 | { |
118 | int cpu; |
119 | |
120 | for_each_possible_cpu(cpu) { |
121 | bool other_cpu = (cpu != smp_processor_id()); |
122 | bool is_up; |
123 | |
124 | if (xen_vcpu_nr(cpu) == XEN_VCPU_ID_INVALID) |
125 | continue; |
126 | |
127 | /* Only Xen 4.5 and higher support this. */ |
128 | is_up = HYPERVISOR_vcpu_op(VCPUOP_is_up, |
129 | vcpuid: xen_vcpu_nr(cpu), NULL) > 0; |
130 | |
131 | if (other_cpu && is_up && |
132 | HYPERVISOR_vcpu_op(VCPUOP_down, vcpuid: xen_vcpu_nr(cpu), NULL)) |
133 | BUG(); |
134 | |
135 | if (xen_pv_domain() || xen_feature(XENFEAT_hvm_safe_pvclock)) |
136 | xen_setup_runstate_info(cpu); |
137 | |
138 | xen_vcpu_setup_restore(cpu); |
139 | |
140 | if (other_cpu && is_up && |
141 | HYPERVISOR_vcpu_op(VCPUOP_up, vcpuid: xen_vcpu_nr(cpu), NULL)) |
142 | BUG(); |
143 | } |
144 | } |
145 | |
146 | void xen_vcpu_info_reset(int cpu) |
147 | { |
148 | if (xen_vcpu_nr(cpu) < MAX_VIRT_CPUS) { |
149 | per_cpu(xen_vcpu, cpu) = |
150 | &HYPERVISOR_shared_info->vcpu_info[xen_vcpu_nr(cpu)]; |
151 | } else { |
152 | /* Set to NULL so that if somebody accesses it we get an OOPS */ |
153 | per_cpu(xen_vcpu, cpu) = NULL; |
154 | } |
155 | } |
156 | |
157 | void xen_vcpu_setup(int cpu) |
158 | { |
159 | struct vcpu_register_vcpu_info info; |
160 | int err; |
161 | struct vcpu_info *vcpup; |
162 | |
163 | BUG_ON(HYPERVISOR_shared_info == &xen_dummy_shared_info); |
164 | |
165 | /* |
166 | * This path is called on PVHVM at bootup (xen_hvm_smp_prepare_boot_cpu) |
167 | * and at restore (xen_vcpu_restore). Also called for hotplugged |
168 | * VCPUs (cpu_init -> xen_hvm_cpu_prepare_hvm). |
169 | * However, the hypercall can only be done once (see below) so if a VCPU |
170 | * is offlined and comes back online then let's not redo the hypercall. |
171 | * |
172 | * For PV it is called during restore (xen_vcpu_restore) and bootup |
173 | * (xen_setup_vcpu_info_placement). The hotplug mechanism does not |
174 | * use this function. |
175 | */ |
176 | if (xen_hvm_domain()) { |
177 | if (per_cpu(xen_vcpu, cpu) == &per_cpu(xen_vcpu_info, cpu)) |
178 | return; |
179 | } |
180 | |
181 | vcpup = &per_cpu(xen_vcpu_info, cpu); |
182 | info.mfn = arbitrary_virt_to_mfn(vaddr: vcpup); |
183 | info.offset = offset_in_page(vcpup); |
184 | |
185 | /* |
186 | * N.B. This hypercall can _only_ be called once per CPU. |
187 | * Subsequent calls will error out with -EINVAL. This is due to |
188 | * the fact that hypervisor has no unregister variant and this |
189 | * hypercall does not allow to over-write info.mfn and |
190 | * info.offset. |
191 | */ |
192 | err = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_info, vcpuid: xen_vcpu_nr(cpu), |
193 | extra_args: &info); |
194 | if (err) |
195 | panic(fmt: "register_vcpu_info failed: cpu=%d err=%d\n" , cpu, err); |
196 | |
197 | per_cpu(xen_vcpu, cpu) = vcpup; |
198 | } |
199 | |
200 | void __init xen_banner(void) |
201 | { |
202 | unsigned version = HYPERVISOR_xen_version(XENVER_version, NULL); |
203 | struct xen_extraversion ; |
204 | |
205 | HYPERVISOR_xen_version(XENVER_extraversion, arg: &extra); |
206 | |
207 | pr_info("Booting kernel on %s\n" , pv_info.name); |
208 | pr_info("Xen version: %u.%u%s%s\n" , |
209 | version >> 16, version & 0xffff, extra.extraversion, |
210 | xen_feature(XENFEAT_mmu_pt_update_preserve_ad) |
211 | ? " (preserve-AD)" : "" ); |
212 | } |
213 | |
214 | /* Check if running on Xen version (major, minor) or later */ |
215 | bool xen_running_on_version_or_later(unsigned int major, unsigned int minor) |
216 | { |
217 | unsigned int version; |
218 | |
219 | if (!xen_domain()) |
220 | return false; |
221 | |
222 | version = HYPERVISOR_xen_version(XENVER_version, NULL); |
223 | if ((((version >> 16) == major) && ((version & 0xffff) >= minor)) || |
224 | ((version >> 16) > major)) |
225 | return true; |
226 | return false; |
227 | } |
228 | |
229 | void __init xen_add_preferred_consoles(void) |
230 | { |
231 | add_preferred_console(name: "xenboot" , idx: 0, NULL); |
232 | if (!boot_params.screen_info.orig_video_isVGA) |
233 | add_preferred_console(name: "tty" , idx: 0, NULL); |
234 | add_preferred_console(name: "hvc" , idx: 0, NULL); |
235 | if (boot_params.screen_info.orig_video_isVGA) |
236 | add_preferred_console(name: "tty" , idx: 0, NULL); |
237 | } |
238 | |
239 | void xen_reboot(int reason) |
240 | { |
241 | struct sched_shutdown r = { .reason = reason }; |
242 | int cpu; |
243 | |
244 | for_each_online_cpu(cpu) |
245 | xen_pmu_finish(cpu); |
246 | |
247 | if (HYPERVISOR_sched_op(SCHEDOP_shutdown, arg: &r)) |
248 | BUG(); |
249 | } |
250 | |
251 | static int reboot_reason = SHUTDOWN_reboot; |
252 | static bool xen_legacy_crash; |
253 | void xen_emergency_restart(void) |
254 | { |
255 | xen_reboot(reason: reboot_reason); |
256 | } |
257 | |
258 | static int |
259 | xen_panic_event(struct notifier_block *this, unsigned long event, void *ptr) |
260 | { |
261 | if (!kexec_crash_loaded()) { |
262 | if (xen_legacy_crash) |
263 | xen_reboot(SHUTDOWN_crash); |
264 | |
265 | reboot_reason = SHUTDOWN_crash; |
266 | |
267 | /* |
268 | * If panic_timeout==0 then we are supposed to wait forever. |
269 | * However, to preserve original dom0 behavior we have to drop |
270 | * into hypervisor. (domU behavior is controlled by its |
271 | * config file) |
272 | */ |
273 | if (panic_timeout == 0) |
274 | panic_timeout = -1; |
275 | } |
276 | return NOTIFY_DONE; |
277 | } |
278 | |
279 | static int __init parse_xen_legacy_crash(char *arg) |
280 | { |
281 | xen_legacy_crash = true; |
282 | return 0; |
283 | } |
284 | early_param("xen_legacy_crash" , parse_xen_legacy_crash); |
285 | |
286 | static struct notifier_block xen_panic_block = { |
287 | .notifier_call = xen_panic_event, |
288 | .priority = INT_MIN |
289 | }; |
290 | |
291 | int xen_panic_handler_init(void) |
292 | { |
293 | atomic_notifier_chain_register(nh: &panic_notifier_list, nb: &xen_panic_block); |
294 | return 0; |
295 | } |
296 | |
297 | void xen_pin_vcpu(int cpu) |
298 | { |
299 | static bool disable_pinning; |
300 | struct sched_pin_override pin_override; |
301 | int ret; |
302 | |
303 | if (disable_pinning) |
304 | return; |
305 | |
306 | pin_override.pcpu = cpu; |
307 | ret = HYPERVISOR_sched_op(SCHEDOP_pin_override, arg: &pin_override); |
308 | |
309 | /* Ignore errors when removing override. */ |
310 | if (cpu < 0) |
311 | return; |
312 | |
313 | switch (ret) { |
314 | case -ENOSYS: |
315 | pr_warn("Unable to pin on physical cpu %d. In case of problems consider vcpu pinning.\n" , |
316 | cpu); |
317 | disable_pinning = true; |
318 | break; |
319 | case -EPERM: |
320 | WARN(1, "Trying to pin vcpu without having privilege to do so\n" ); |
321 | disable_pinning = true; |
322 | break; |
323 | case -EINVAL: |
324 | case -EBUSY: |
325 | pr_warn("Physical cpu %d not available for pinning. Check Xen cpu configuration.\n" , |
326 | cpu); |
327 | break; |
328 | case 0: |
329 | break; |
330 | default: |
331 | WARN(1, "rc %d while trying to pin vcpu\n" , ret); |
332 | disable_pinning = true; |
333 | } |
334 | } |
335 | |
336 | #ifdef CONFIG_HOTPLUG_CPU |
337 | void xen_arch_register_cpu(int num) |
338 | { |
339 | arch_register_cpu(cpu: num); |
340 | } |
341 | EXPORT_SYMBOL(xen_arch_register_cpu); |
342 | |
343 | void xen_arch_unregister_cpu(int num) |
344 | { |
345 | arch_unregister_cpu(cpu: num); |
346 | } |
347 | EXPORT_SYMBOL(xen_arch_unregister_cpu); |
348 | #endif |
349 | |