1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * KVM paravirt_ops implementation |
4 | * |
5 | * Copyright (C) 2007, Red Hat, Inc., Ingo Molnar <mingo@redhat.com> |
6 | * Copyright IBM Corporation, 2007 |
7 | * Authors: Anthony Liguori <aliguori@us.ibm.com> |
8 | */ |
9 | |
10 | #define pr_fmt(fmt) "kvm-guest: " fmt |
11 | |
12 | #include <linux/context_tracking.h> |
13 | #include <linux/init.h> |
14 | #include <linux/irq.h> |
15 | #include <linux/kernel.h> |
16 | #include <linux/kvm_para.h> |
17 | #include <linux/cpu.h> |
18 | #include <linux/mm.h> |
19 | #include <linux/highmem.h> |
20 | #include <linux/hardirq.h> |
21 | #include <linux/notifier.h> |
22 | #include <linux/reboot.h> |
23 | #include <linux/hash.h> |
24 | #include <linux/sched.h> |
25 | #include <linux/slab.h> |
26 | #include <linux/kprobes.h> |
27 | #include <linux/nmi.h> |
28 | #include <linux/swait.h> |
29 | #include <linux/syscore_ops.h> |
30 | #include <linux/cc_platform.h> |
31 | #include <linux/efi.h> |
32 | #include <asm/timer.h> |
33 | #include <asm/cpu.h> |
34 | #include <asm/traps.h> |
35 | #include <asm/desc.h> |
36 | #include <asm/tlbflush.h> |
37 | #include <asm/apic.h> |
38 | #include <asm/apicdef.h> |
39 | #include <asm/hypervisor.h> |
40 | #include <asm/tlb.h> |
41 | #include <asm/cpuidle_haltpoll.h> |
42 | #include <asm/ptrace.h> |
43 | #include <asm/reboot.h> |
44 | #include <asm/svm.h> |
45 | #include <asm/e820/api.h> |
46 | |
47 | DEFINE_STATIC_KEY_FALSE(kvm_async_pf_enabled); |
48 | |
49 | static int kvmapf = 1; |
50 | |
51 | static int __init parse_no_kvmapf(char *arg) |
52 | { |
53 | kvmapf = 0; |
54 | return 0; |
55 | } |
56 | |
57 | early_param("no-kvmapf" , parse_no_kvmapf); |
58 | |
59 | static int steal_acc = 1; |
60 | static int __init parse_no_stealacc(char *arg) |
61 | { |
62 | steal_acc = 0; |
63 | return 0; |
64 | } |
65 | |
66 | early_param("no-steal-acc" , parse_no_stealacc); |
67 | |
68 | static DEFINE_PER_CPU_READ_MOSTLY(bool, async_pf_enabled); |
69 | static DEFINE_PER_CPU_DECRYPTED(struct kvm_vcpu_pv_apf_data, apf_reason) __aligned(64); |
70 | DEFINE_PER_CPU_DECRYPTED(struct kvm_steal_time, steal_time) __aligned(64) __visible; |
71 | static int has_steal_clock = 0; |
72 | |
73 | static int has_guest_poll = 0; |
74 | /* |
75 | * No need for any "IO delay" on KVM |
76 | */ |
77 | static void kvm_io_delay(void) |
78 | { |
79 | } |
80 | |
81 | #define KVM_TASK_SLEEP_HASHBITS 8 |
82 | #define KVM_TASK_SLEEP_HASHSIZE (1<<KVM_TASK_SLEEP_HASHBITS) |
83 | |
84 | struct kvm_task_sleep_node { |
85 | struct hlist_node link; |
86 | struct swait_queue_head wq; |
87 | u32 token; |
88 | int cpu; |
89 | }; |
90 | |
91 | static struct kvm_task_sleep_head { |
92 | raw_spinlock_t lock; |
93 | struct hlist_head list; |
94 | } async_pf_sleepers[KVM_TASK_SLEEP_HASHSIZE]; |
95 | |
96 | static struct kvm_task_sleep_node *_find_apf_task(struct kvm_task_sleep_head *b, |
97 | u32 token) |
98 | { |
99 | struct hlist_node *p; |
100 | |
101 | hlist_for_each(p, &b->list) { |
102 | struct kvm_task_sleep_node *n = |
103 | hlist_entry(p, typeof(*n), link); |
104 | if (n->token == token) |
105 | return n; |
106 | } |
107 | |
108 | return NULL; |
109 | } |
110 | |
111 | static bool kvm_async_pf_queue_task(u32 token, struct kvm_task_sleep_node *n) |
112 | { |
113 | u32 key = hash_32(val: token, KVM_TASK_SLEEP_HASHBITS); |
114 | struct kvm_task_sleep_head *b = &async_pf_sleepers[key]; |
115 | struct kvm_task_sleep_node *e; |
116 | |
117 | raw_spin_lock(&b->lock); |
118 | e = _find_apf_task(b, token); |
119 | if (e) { |
120 | /* dummy entry exist -> wake up was delivered ahead of PF */ |
121 | hlist_del(n: &e->link); |
122 | raw_spin_unlock(&b->lock); |
123 | kfree(objp: e); |
124 | return false; |
125 | } |
126 | |
127 | n->token = token; |
128 | n->cpu = smp_processor_id(); |
129 | init_swait_queue_head(&n->wq); |
130 | hlist_add_head(n: &n->link, h: &b->list); |
131 | raw_spin_unlock(&b->lock); |
132 | return true; |
133 | } |
134 | |
135 | /* |
136 | * kvm_async_pf_task_wait_schedule - Wait for pagefault to be handled |
137 | * @token: Token to identify the sleep node entry |
138 | * |
139 | * Invoked from the async pagefault handling code or from the VM exit page |
140 | * fault handler. In both cases RCU is watching. |
141 | */ |
142 | void kvm_async_pf_task_wait_schedule(u32 token) |
143 | { |
144 | struct kvm_task_sleep_node n; |
145 | DECLARE_SWAITQUEUE(wait); |
146 | |
147 | lockdep_assert_irqs_disabled(); |
148 | |
149 | if (!kvm_async_pf_queue_task(token, n: &n)) |
150 | return; |
151 | |
152 | for (;;) { |
153 | prepare_to_swait_exclusive(q: &n.wq, wait: &wait, TASK_UNINTERRUPTIBLE); |
154 | if (hlist_unhashed(h: &n.link)) |
155 | break; |
156 | |
157 | local_irq_enable(); |
158 | schedule(); |
159 | local_irq_disable(); |
160 | } |
161 | finish_swait(q: &n.wq, wait: &wait); |
162 | } |
163 | EXPORT_SYMBOL_GPL(kvm_async_pf_task_wait_schedule); |
164 | |
165 | static void apf_task_wake_one(struct kvm_task_sleep_node *n) |
166 | { |
167 | hlist_del_init(n: &n->link); |
168 | if (swq_has_sleeper(wq: &n->wq)) |
169 | swake_up_one(q: &n->wq); |
170 | } |
171 | |
172 | static void apf_task_wake_all(void) |
173 | { |
174 | int i; |
175 | |
176 | for (i = 0; i < KVM_TASK_SLEEP_HASHSIZE; i++) { |
177 | struct kvm_task_sleep_head *b = &async_pf_sleepers[i]; |
178 | struct kvm_task_sleep_node *n; |
179 | struct hlist_node *p, *next; |
180 | |
181 | raw_spin_lock(&b->lock); |
182 | hlist_for_each_safe(p, next, &b->list) { |
183 | n = hlist_entry(p, typeof(*n), link); |
184 | if (n->cpu == smp_processor_id()) |
185 | apf_task_wake_one(n); |
186 | } |
187 | raw_spin_unlock(&b->lock); |
188 | } |
189 | } |
190 | |
191 | void kvm_async_pf_task_wake(u32 token) |
192 | { |
193 | u32 key = hash_32(val: token, KVM_TASK_SLEEP_HASHBITS); |
194 | struct kvm_task_sleep_head *b = &async_pf_sleepers[key]; |
195 | struct kvm_task_sleep_node *n, *dummy = NULL; |
196 | |
197 | if (token == ~0) { |
198 | apf_task_wake_all(); |
199 | return; |
200 | } |
201 | |
202 | again: |
203 | raw_spin_lock(&b->lock); |
204 | n = _find_apf_task(b, token); |
205 | if (!n) { |
206 | /* |
207 | * Async #PF not yet handled, add a dummy entry for the token. |
208 | * Allocating the token must be down outside of the raw lock |
209 | * as the allocator is preemptible on PREEMPT_RT kernels. |
210 | */ |
211 | if (!dummy) { |
212 | raw_spin_unlock(&b->lock); |
213 | dummy = kzalloc(size: sizeof(*dummy), GFP_ATOMIC); |
214 | |
215 | /* |
216 | * Continue looping on allocation failure, eventually |
217 | * the async #PF will be handled and allocating a new |
218 | * node will be unnecessary. |
219 | */ |
220 | if (!dummy) |
221 | cpu_relax(); |
222 | |
223 | /* |
224 | * Recheck for async #PF completion before enqueueing |
225 | * the dummy token to avoid duplicate list entries. |
226 | */ |
227 | goto again; |
228 | } |
229 | dummy->token = token; |
230 | dummy->cpu = smp_processor_id(); |
231 | init_swait_queue_head(&dummy->wq); |
232 | hlist_add_head(n: &dummy->link, h: &b->list); |
233 | dummy = NULL; |
234 | } else { |
235 | apf_task_wake_one(n); |
236 | } |
237 | raw_spin_unlock(&b->lock); |
238 | |
239 | /* A dummy token might be allocated and ultimately not used. */ |
240 | kfree(objp: dummy); |
241 | } |
242 | EXPORT_SYMBOL_GPL(kvm_async_pf_task_wake); |
243 | |
244 | noinstr u32 kvm_read_and_reset_apf_flags(void) |
245 | { |
246 | u32 flags = 0; |
247 | |
248 | if (__this_cpu_read(async_pf_enabled)) { |
249 | flags = __this_cpu_read(apf_reason.flags); |
250 | __this_cpu_write(apf_reason.flags, 0); |
251 | } |
252 | |
253 | return flags; |
254 | } |
255 | EXPORT_SYMBOL_GPL(kvm_read_and_reset_apf_flags); |
256 | |
257 | noinstr bool __kvm_handle_async_pf(struct pt_regs *regs, u32 token) |
258 | { |
259 | u32 flags = kvm_read_and_reset_apf_flags(); |
260 | irqentry_state_t state; |
261 | |
262 | if (!flags) |
263 | return false; |
264 | |
265 | state = irqentry_enter(regs); |
266 | instrumentation_begin(); |
267 | |
268 | /* |
269 | * If the host managed to inject an async #PF into an interrupt |
270 | * disabled region, then die hard as this is not going to end well |
271 | * and the host side is seriously broken. |
272 | */ |
273 | if (unlikely(!(regs->flags & X86_EFLAGS_IF))) |
274 | panic(fmt: "Host injected async #PF in interrupt disabled region\n" ); |
275 | |
276 | if (flags & KVM_PV_REASON_PAGE_NOT_PRESENT) { |
277 | if (unlikely(!(user_mode(regs)))) |
278 | panic(fmt: "Host injected async #PF in kernel mode\n" ); |
279 | /* Page is swapped out by the host. */ |
280 | kvm_async_pf_task_wait_schedule(token); |
281 | } else { |
282 | WARN_ONCE(1, "Unexpected async PF flags: %x\n" , flags); |
283 | } |
284 | |
285 | instrumentation_end(); |
286 | irqentry_exit(regs, state); |
287 | return true; |
288 | } |
289 | |
290 | DEFINE_IDTENTRY_SYSVEC(sysvec_kvm_asyncpf_interrupt) |
291 | { |
292 | struct pt_regs *old_regs = set_irq_regs(regs); |
293 | u32 token; |
294 | |
295 | apic_eoi(); |
296 | |
297 | inc_irq_stat(irq_hv_callback_count); |
298 | |
299 | if (__this_cpu_read(async_pf_enabled)) { |
300 | token = __this_cpu_read(apf_reason.token); |
301 | kvm_async_pf_task_wake(token); |
302 | __this_cpu_write(apf_reason.token, 0); |
303 | wrmsrl(MSR_KVM_ASYNC_PF_ACK, val: 1); |
304 | } |
305 | |
306 | set_irq_regs(old_regs); |
307 | } |
308 | |
309 | static void __init paravirt_ops_setup(void) |
310 | { |
311 | pv_info.name = "KVM" ; |
312 | |
313 | if (kvm_para_has_feature(KVM_FEATURE_NOP_IO_DELAY)) |
314 | pv_ops.cpu.io_delay = kvm_io_delay; |
315 | |
316 | #ifdef CONFIG_X86_IO_APIC |
317 | no_timer_check = 1; |
318 | #endif |
319 | } |
320 | |
321 | static void kvm_register_steal_time(void) |
322 | { |
323 | int cpu = smp_processor_id(); |
324 | struct kvm_steal_time *st = &per_cpu(steal_time, cpu); |
325 | |
326 | if (!has_steal_clock) |
327 | return; |
328 | |
329 | wrmsrl(MSR_KVM_STEAL_TIME, val: (slow_virt_to_phys(address: st) | KVM_MSR_ENABLED)); |
330 | pr_debug("stealtime: cpu %d, msr %llx\n" , cpu, |
331 | (unsigned long long) slow_virt_to_phys(st)); |
332 | } |
333 | |
334 | static DEFINE_PER_CPU_DECRYPTED(unsigned long, kvm_apic_eoi) = KVM_PV_EOI_DISABLED; |
335 | |
336 | static notrace __maybe_unused void kvm_guest_apic_eoi_write(void) |
337 | { |
338 | /** |
339 | * This relies on __test_and_clear_bit to modify the memory |
340 | * in a way that is atomic with respect to the local CPU. |
341 | * The hypervisor only accesses this memory from the local CPU so |
342 | * there's no need for lock or memory barriers. |
343 | * An optimization barrier is implied in apic write. |
344 | */ |
345 | if (__test_and_clear_bit(KVM_PV_EOI_BIT, this_cpu_ptr(&kvm_apic_eoi))) |
346 | return; |
347 | apic_native_eoi(); |
348 | } |
349 | |
350 | static void kvm_guest_cpu_init(void) |
351 | { |
352 | if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF_INT) && kvmapf) { |
353 | u64 pa; |
354 | |
355 | WARN_ON_ONCE(!static_branch_likely(&kvm_async_pf_enabled)); |
356 | |
357 | pa = slow_virt_to_phys(this_cpu_ptr(&apf_reason)); |
358 | pa |= KVM_ASYNC_PF_ENABLED | KVM_ASYNC_PF_DELIVERY_AS_INT; |
359 | |
360 | if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF_VMEXIT)) |
361 | pa |= KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT; |
362 | |
363 | wrmsrl(MSR_KVM_ASYNC_PF_INT, HYPERVISOR_CALLBACK_VECTOR); |
364 | |
365 | wrmsrl(MSR_KVM_ASYNC_PF_EN, val: pa); |
366 | __this_cpu_write(async_pf_enabled, true); |
367 | pr_debug("setup async PF for cpu %d\n" , smp_processor_id()); |
368 | } |
369 | |
370 | if (kvm_para_has_feature(KVM_FEATURE_PV_EOI)) { |
371 | unsigned long pa; |
372 | |
373 | /* Size alignment is implied but just to make it explicit. */ |
374 | BUILD_BUG_ON(__alignof__(kvm_apic_eoi) < 4); |
375 | __this_cpu_write(kvm_apic_eoi, 0); |
376 | pa = slow_virt_to_phys(this_cpu_ptr(&kvm_apic_eoi)) |
377 | | KVM_MSR_ENABLED; |
378 | wrmsrl(MSR_KVM_PV_EOI_EN, val: pa); |
379 | } |
380 | |
381 | if (has_steal_clock) |
382 | kvm_register_steal_time(); |
383 | } |
384 | |
385 | static void kvm_pv_disable_apf(void) |
386 | { |
387 | if (!__this_cpu_read(async_pf_enabled)) |
388 | return; |
389 | |
390 | wrmsrl(MSR_KVM_ASYNC_PF_EN, val: 0); |
391 | __this_cpu_write(async_pf_enabled, false); |
392 | |
393 | pr_debug("disable async PF for cpu %d\n" , smp_processor_id()); |
394 | } |
395 | |
396 | static void kvm_disable_steal_time(void) |
397 | { |
398 | if (!has_steal_clock) |
399 | return; |
400 | |
401 | wrmsr(MSR_KVM_STEAL_TIME, 0, 0); |
402 | } |
403 | |
404 | static u64 kvm_steal_clock(int cpu) |
405 | { |
406 | u64 steal; |
407 | struct kvm_steal_time *src; |
408 | int version; |
409 | |
410 | src = &per_cpu(steal_time, cpu); |
411 | do { |
412 | version = src->version; |
413 | virt_rmb(); |
414 | steal = src->steal; |
415 | virt_rmb(); |
416 | } while ((version & 1) || (version != src->version)); |
417 | |
418 | return steal; |
419 | } |
420 | |
421 | static inline void __set_percpu_decrypted(void *ptr, unsigned long size) |
422 | { |
423 | early_set_memory_decrypted(vaddr: (unsigned long) ptr, size); |
424 | } |
425 | |
426 | /* |
427 | * Iterate through all possible CPUs and map the memory region pointed |
428 | * by apf_reason, steal_time and kvm_apic_eoi as decrypted at once. |
429 | * |
430 | * Note: we iterate through all possible CPUs to ensure that CPUs |
431 | * hotplugged will have their per-cpu variable already mapped as |
432 | * decrypted. |
433 | */ |
434 | static void __init sev_map_percpu_data(void) |
435 | { |
436 | int cpu; |
437 | |
438 | if (cc_vendor != CC_VENDOR_AMD || |
439 | !cc_platform_has(attr: CC_ATTR_GUEST_MEM_ENCRYPT)) |
440 | return; |
441 | |
442 | for_each_possible_cpu(cpu) { |
443 | __set_percpu_decrypted(ptr: &per_cpu(apf_reason, cpu), size: sizeof(apf_reason)); |
444 | __set_percpu_decrypted(ptr: &per_cpu(steal_time, cpu), size: sizeof(steal_time)); |
445 | __set_percpu_decrypted(ptr: &per_cpu(kvm_apic_eoi, cpu), size: sizeof(kvm_apic_eoi)); |
446 | } |
447 | } |
448 | |
449 | static void kvm_guest_cpu_offline(bool shutdown) |
450 | { |
451 | kvm_disable_steal_time(); |
452 | if (kvm_para_has_feature(KVM_FEATURE_PV_EOI)) |
453 | wrmsrl(MSR_KVM_PV_EOI_EN, val: 0); |
454 | if (kvm_para_has_feature(KVM_FEATURE_MIGRATION_CONTROL)) |
455 | wrmsrl(MSR_KVM_MIGRATION_CONTROL, val: 0); |
456 | kvm_pv_disable_apf(); |
457 | if (!shutdown) |
458 | apf_task_wake_all(); |
459 | kvmclock_disable(); |
460 | } |
461 | |
462 | static int kvm_cpu_online(unsigned int cpu) |
463 | { |
464 | unsigned long flags; |
465 | |
466 | local_irq_save(flags); |
467 | kvm_guest_cpu_init(); |
468 | local_irq_restore(flags); |
469 | return 0; |
470 | } |
471 | |
472 | #ifdef CONFIG_SMP |
473 | |
474 | static DEFINE_PER_CPU(cpumask_var_t, __pv_cpu_mask); |
475 | |
476 | static bool pv_tlb_flush_supported(void) |
477 | { |
478 | return (kvm_para_has_feature(KVM_FEATURE_PV_TLB_FLUSH) && |
479 | !kvm_para_has_hint(KVM_HINTS_REALTIME) && |
480 | kvm_para_has_feature(KVM_FEATURE_STEAL_TIME) && |
481 | !boot_cpu_has(X86_FEATURE_MWAIT) && |
482 | (num_possible_cpus() != 1)); |
483 | } |
484 | |
485 | static bool pv_ipi_supported(void) |
486 | { |
487 | return (kvm_para_has_feature(KVM_FEATURE_PV_SEND_IPI) && |
488 | (num_possible_cpus() != 1)); |
489 | } |
490 | |
491 | static bool pv_sched_yield_supported(void) |
492 | { |
493 | return (kvm_para_has_feature(KVM_FEATURE_PV_SCHED_YIELD) && |
494 | !kvm_para_has_hint(KVM_HINTS_REALTIME) && |
495 | kvm_para_has_feature(KVM_FEATURE_STEAL_TIME) && |
496 | !boot_cpu_has(X86_FEATURE_MWAIT) && |
497 | (num_possible_cpus() != 1)); |
498 | } |
499 | |
500 | #define KVM_IPI_CLUSTER_SIZE (2 * BITS_PER_LONG) |
501 | |
502 | static void __send_ipi_mask(const struct cpumask *mask, int vector) |
503 | { |
504 | unsigned long flags; |
505 | int cpu, min = 0, max = 0; |
506 | #ifdef CONFIG_X86_64 |
507 | __uint128_t ipi_bitmap = 0; |
508 | #else |
509 | u64 ipi_bitmap = 0; |
510 | #endif |
511 | u32 apic_id, icr; |
512 | long ret; |
513 | |
514 | if (cpumask_empty(srcp: mask)) |
515 | return; |
516 | |
517 | local_irq_save(flags); |
518 | |
519 | switch (vector) { |
520 | default: |
521 | icr = APIC_DM_FIXED | vector; |
522 | break; |
523 | case NMI_VECTOR: |
524 | icr = APIC_DM_NMI; |
525 | break; |
526 | } |
527 | |
528 | for_each_cpu(cpu, mask) { |
529 | apic_id = per_cpu(x86_cpu_to_apicid, cpu); |
530 | if (!ipi_bitmap) { |
531 | min = max = apic_id; |
532 | } else if (apic_id < min && max - apic_id < KVM_IPI_CLUSTER_SIZE) { |
533 | ipi_bitmap <<= min - apic_id; |
534 | min = apic_id; |
535 | } else if (apic_id > min && apic_id < min + KVM_IPI_CLUSTER_SIZE) { |
536 | max = apic_id < max ? max : apic_id; |
537 | } else { |
538 | ret = kvm_hypercall4(KVM_HC_SEND_IPI, p1: (unsigned long)ipi_bitmap, |
539 | p2: (unsigned long)(ipi_bitmap >> BITS_PER_LONG), p3: min, p4: icr); |
540 | WARN_ONCE(ret < 0, "kvm-guest: failed to send PV IPI: %ld" , |
541 | ret); |
542 | min = max = apic_id; |
543 | ipi_bitmap = 0; |
544 | } |
545 | __set_bit(apic_id - min, (unsigned long *)&ipi_bitmap); |
546 | } |
547 | |
548 | if (ipi_bitmap) { |
549 | ret = kvm_hypercall4(KVM_HC_SEND_IPI, p1: (unsigned long)ipi_bitmap, |
550 | p2: (unsigned long)(ipi_bitmap >> BITS_PER_LONG), p3: min, p4: icr); |
551 | WARN_ONCE(ret < 0, "kvm-guest: failed to send PV IPI: %ld" , |
552 | ret); |
553 | } |
554 | |
555 | local_irq_restore(flags); |
556 | } |
557 | |
558 | static void kvm_send_ipi_mask(const struct cpumask *mask, int vector) |
559 | { |
560 | __send_ipi_mask(mask, vector); |
561 | } |
562 | |
563 | static void kvm_send_ipi_mask_allbutself(const struct cpumask *mask, int vector) |
564 | { |
565 | unsigned int this_cpu = smp_processor_id(); |
566 | struct cpumask *new_mask = this_cpu_cpumask_var_ptr(__pv_cpu_mask); |
567 | const struct cpumask *local_mask; |
568 | |
569 | cpumask_copy(dstp: new_mask, srcp: mask); |
570 | cpumask_clear_cpu(cpu: this_cpu, dstp: new_mask); |
571 | local_mask = new_mask; |
572 | __send_ipi_mask(mask: local_mask, vector); |
573 | } |
574 | |
575 | static int __init setup_efi_kvm_sev_migration(void) |
576 | { |
577 | efi_char16_t efi_sev_live_migration_enabled[] = L"SevLiveMigrationEnabled" ; |
578 | efi_guid_t efi_variable_guid = AMD_SEV_MEM_ENCRYPT_GUID; |
579 | efi_status_t status; |
580 | unsigned long size; |
581 | bool enabled; |
582 | |
583 | if (!cc_platform_has(attr: CC_ATTR_GUEST_MEM_ENCRYPT) || |
584 | !kvm_para_has_feature(KVM_FEATURE_MIGRATION_CONTROL)) |
585 | return 0; |
586 | |
587 | if (!efi_enabled(EFI_BOOT)) |
588 | return 0; |
589 | |
590 | if (!efi_enabled(EFI_RUNTIME_SERVICES)) { |
591 | pr_info("%s : EFI runtime services are not enabled\n" , __func__); |
592 | return 0; |
593 | } |
594 | |
595 | size = sizeof(enabled); |
596 | |
597 | /* Get variable contents into buffer */ |
598 | status = efi.get_variable(efi_sev_live_migration_enabled, |
599 | &efi_variable_guid, NULL, &size, &enabled); |
600 | |
601 | if (status == EFI_NOT_FOUND) { |
602 | pr_info("%s : EFI live migration variable not found\n" , __func__); |
603 | return 0; |
604 | } |
605 | |
606 | if (status != EFI_SUCCESS) { |
607 | pr_info("%s : EFI variable retrieval failed\n" , __func__); |
608 | return 0; |
609 | } |
610 | |
611 | if (enabled == 0) { |
612 | pr_info("%s: live migration disabled in EFI\n" , __func__); |
613 | return 0; |
614 | } |
615 | |
616 | pr_info("%s : live migration enabled in EFI\n" , __func__); |
617 | wrmsrl(MSR_KVM_MIGRATION_CONTROL, KVM_MIGRATION_READY); |
618 | |
619 | return 1; |
620 | } |
621 | |
622 | late_initcall(setup_efi_kvm_sev_migration); |
623 | |
624 | /* |
625 | * Set the IPI entry points |
626 | */ |
627 | static __init void kvm_setup_pv_ipi(void) |
628 | { |
629 | apic_update_callback(send_IPI_mask, kvm_send_ipi_mask); |
630 | apic_update_callback(send_IPI_mask_allbutself, kvm_send_ipi_mask_allbutself); |
631 | pr_info("setup PV IPIs\n" ); |
632 | } |
633 | |
634 | static void kvm_smp_send_call_func_ipi(const struct cpumask *mask) |
635 | { |
636 | int cpu; |
637 | |
638 | native_send_call_func_ipi(mask); |
639 | |
640 | /* Make sure other vCPUs get a chance to run if they need to. */ |
641 | for_each_cpu(cpu, mask) { |
642 | if (!idle_cpu(cpu) && vcpu_is_preempted(cpu)) { |
643 | kvm_hypercall1(KVM_HC_SCHED_YIELD, per_cpu(x86_cpu_to_apicid, cpu)); |
644 | break; |
645 | } |
646 | } |
647 | } |
648 | |
649 | static void kvm_flush_tlb_multi(const struct cpumask *cpumask, |
650 | const struct flush_tlb_info *info) |
651 | { |
652 | u8 state; |
653 | int cpu; |
654 | struct kvm_steal_time *src; |
655 | struct cpumask *flushmask = this_cpu_cpumask_var_ptr(__pv_cpu_mask); |
656 | |
657 | cpumask_copy(dstp: flushmask, srcp: cpumask); |
658 | /* |
659 | * We have to call flush only on online vCPUs. And |
660 | * queue flush_on_enter for pre-empted vCPUs |
661 | */ |
662 | for_each_cpu(cpu, flushmask) { |
663 | /* |
664 | * The local vCPU is never preempted, so we do not explicitly |
665 | * skip check for local vCPU - it will never be cleared from |
666 | * flushmask. |
667 | */ |
668 | src = &per_cpu(steal_time, cpu); |
669 | state = READ_ONCE(src->preempted); |
670 | if ((state & KVM_VCPU_PREEMPTED)) { |
671 | if (try_cmpxchg(&src->preempted, &state, |
672 | state | KVM_VCPU_FLUSH_TLB)) |
673 | __cpumask_clear_cpu(cpu, dstp: flushmask); |
674 | } |
675 | } |
676 | |
677 | native_flush_tlb_multi(cpumask: flushmask, info); |
678 | } |
679 | |
680 | static __init int kvm_alloc_cpumask(void) |
681 | { |
682 | int cpu; |
683 | |
684 | if (!kvm_para_available() || nopv) |
685 | return 0; |
686 | |
687 | if (pv_tlb_flush_supported() || pv_ipi_supported()) |
688 | for_each_possible_cpu(cpu) { |
689 | zalloc_cpumask_var_node(per_cpu_ptr(&__pv_cpu_mask, cpu), |
690 | GFP_KERNEL, cpu_to_node(cpu)); |
691 | } |
692 | |
693 | return 0; |
694 | } |
695 | arch_initcall(kvm_alloc_cpumask); |
696 | |
697 | static void __init kvm_smp_prepare_boot_cpu(void) |
698 | { |
699 | /* |
700 | * Map the per-cpu variables as decrypted before kvm_guest_cpu_init() |
701 | * shares the guest physical address with the hypervisor. |
702 | */ |
703 | sev_map_percpu_data(); |
704 | |
705 | kvm_guest_cpu_init(); |
706 | native_smp_prepare_boot_cpu(); |
707 | kvm_spinlock_init(); |
708 | } |
709 | |
710 | static int kvm_cpu_down_prepare(unsigned int cpu) |
711 | { |
712 | unsigned long flags; |
713 | |
714 | local_irq_save(flags); |
715 | kvm_guest_cpu_offline(shutdown: false); |
716 | local_irq_restore(flags); |
717 | return 0; |
718 | } |
719 | |
720 | #endif |
721 | |
722 | static int kvm_suspend(void) |
723 | { |
724 | u64 val = 0; |
725 | |
726 | kvm_guest_cpu_offline(shutdown: false); |
727 | |
728 | #ifdef CONFIG_ARCH_CPUIDLE_HALTPOLL |
729 | if (kvm_para_has_feature(KVM_FEATURE_POLL_CONTROL)) |
730 | rdmsrl(MSR_KVM_POLL_CONTROL, val); |
731 | has_guest_poll = !(val & 1); |
732 | #endif |
733 | return 0; |
734 | } |
735 | |
736 | static void kvm_resume(void) |
737 | { |
738 | kvm_cpu_online(raw_smp_processor_id()); |
739 | |
740 | #ifdef CONFIG_ARCH_CPUIDLE_HALTPOLL |
741 | if (kvm_para_has_feature(KVM_FEATURE_POLL_CONTROL) && has_guest_poll) |
742 | wrmsrl(MSR_KVM_POLL_CONTROL, val: 0); |
743 | #endif |
744 | } |
745 | |
746 | static struct syscore_ops kvm_syscore_ops = { |
747 | .suspend = kvm_suspend, |
748 | .resume = kvm_resume, |
749 | }; |
750 | |
751 | static void kvm_pv_guest_cpu_reboot(void *unused) |
752 | { |
753 | kvm_guest_cpu_offline(shutdown: true); |
754 | } |
755 | |
756 | static int kvm_pv_reboot_notify(struct notifier_block *nb, |
757 | unsigned long code, void *unused) |
758 | { |
759 | if (code == SYS_RESTART) |
760 | on_each_cpu(func: kvm_pv_guest_cpu_reboot, NULL, wait: 1); |
761 | return NOTIFY_DONE; |
762 | } |
763 | |
764 | static struct notifier_block kvm_pv_reboot_nb = { |
765 | .notifier_call = kvm_pv_reboot_notify, |
766 | }; |
767 | |
768 | /* |
769 | * After a PV feature is registered, the host will keep writing to the |
770 | * registered memory location. If the guest happens to shutdown, this memory |
771 | * won't be valid. In cases like kexec, in which you install a new kernel, this |
772 | * means a random memory location will be kept being written. |
773 | */ |
774 | #ifdef CONFIG_CRASH_DUMP |
775 | static void kvm_crash_shutdown(struct pt_regs *regs) |
776 | { |
777 | kvm_guest_cpu_offline(shutdown: true); |
778 | native_machine_crash_shutdown(regs); |
779 | } |
780 | #endif |
781 | |
782 | #if defined(CONFIG_X86_32) || !defined(CONFIG_SMP) |
783 | bool __kvm_vcpu_is_preempted(long cpu); |
784 | |
785 | __visible bool __kvm_vcpu_is_preempted(long cpu) |
786 | { |
787 | struct kvm_steal_time *src = &per_cpu(steal_time, cpu); |
788 | |
789 | return !!(src->preempted & KVM_VCPU_PREEMPTED); |
790 | } |
791 | PV_CALLEE_SAVE_REGS_THUNK(__kvm_vcpu_is_preempted); |
792 | |
793 | #else |
794 | |
795 | #include <asm/asm-offsets.h> |
796 | |
797 | extern bool __raw_callee_save___kvm_vcpu_is_preempted(long); |
798 | |
799 | /* |
800 | * Hand-optimize version for x86-64 to avoid 8 64-bit register saving and |
801 | * restoring to/from the stack. |
802 | */ |
803 | #define PV_VCPU_PREEMPTED_ASM \ |
804 | "movq __per_cpu_offset(,%rdi,8), %rax\n\t" \ |
805 | "cmpb $0, " __stringify(KVM_STEAL_TIME_preempted) "+steal_time(%rax)\n\t" \ |
806 | "setne %al\n\t" |
807 | |
808 | DEFINE_ASM_FUNC(__raw_callee_save___kvm_vcpu_is_preempted, |
809 | PV_VCPU_PREEMPTED_ASM, .text); |
810 | #endif |
811 | |
812 | static void __init kvm_guest_init(void) |
813 | { |
814 | int i; |
815 | |
816 | paravirt_ops_setup(); |
817 | register_reboot_notifier(&kvm_pv_reboot_nb); |
818 | for (i = 0; i < KVM_TASK_SLEEP_HASHSIZE; i++) |
819 | raw_spin_lock_init(&async_pf_sleepers[i].lock); |
820 | |
821 | if (kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) { |
822 | has_steal_clock = 1; |
823 | static_call_update(pv_steal_clock, kvm_steal_clock); |
824 | |
825 | pv_ops.lock.vcpu_is_preempted = |
826 | PV_CALLEE_SAVE(__kvm_vcpu_is_preempted); |
827 | } |
828 | |
829 | if (kvm_para_has_feature(KVM_FEATURE_PV_EOI)) |
830 | apic_update_callback(eoi, kvm_guest_apic_eoi_write); |
831 | |
832 | if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF_INT) && kvmapf) { |
833 | static_branch_enable(&kvm_async_pf_enabled); |
834 | sysvec_install(HYPERVISOR_CALLBACK_VECTOR, sysvec_kvm_asyncpf_interrupt); |
835 | } |
836 | |
837 | #ifdef CONFIG_SMP |
838 | if (pv_tlb_flush_supported()) { |
839 | pv_ops.mmu.flush_tlb_multi = kvm_flush_tlb_multi; |
840 | pv_ops.mmu.tlb_remove_table = tlb_remove_table; |
841 | pr_info("KVM setup pv remote TLB flush\n" ); |
842 | } |
843 | |
844 | smp_ops.smp_prepare_boot_cpu = kvm_smp_prepare_boot_cpu; |
845 | if (pv_sched_yield_supported()) { |
846 | smp_ops.send_call_func_ipi = kvm_smp_send_call_func_ipi; |
847 | pr_info("setup PV sched yield\n" ); |
848 | } |
849 | if (cpuhp_setup_state_nocalls(state: CPUHP_AP_ONLINE_DYN, name: "x86/kvm:online" , |
850 | startup: kvm_cpu_online, teardown: kvm_cpu_down_prepare) < 0) |
851 | pr_err("failed to install cpu hotplug callbacks\n" ); |
852 | #else |
853 | sev_map_percpu_data(); |
854 | kvm_guest_cpu_init(); |
855 | #endif |
856 | |
857 | #ifdef CONFIG_CRASH_DUMP |
858 | machine_ops.crash_shutdown = kvm_crash_shutdown; |
859 | #endif |
860 | |
861 | register_syscore_ops(ops: &kvm_syscore_ops); |
862 | |
863 | /* |
864 | * Hard lockup detection is enabled by default. Disable it, as guests |
865 | * can get false positives too easily, for example if the host is |
866 | * overcommitted. |
867 | */ |
868 | hardlockup_detector_disable(); |
869 | } |
870 | |
871 | static noinline uint32_t __kvm_cpuid_base(void) |
872 | { |
873 | if (boot_cpu_data.cpuid_level < 0) |
874 | return 0; /* So we don't blow up on old processors */ |
875 | |
876 | if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) |
877 | return hypervisor_cpuid_base(KVM_SIGNATURE, leaves: 0); |
878 | |
879 | return 0; |
880 | } |
881 | |
882 | static inline uint32_t kvm_cpuid_base(void) |
883 | { |
884 | static int kvm_cpuid_base = -1; |
885 | |
886 | if (kvm_cpuid_base == -1) |
887 | kvm_cpuid_base = __kvm_cpuid_base(); |
888 | |
889 | return kvm_cpuid_base; |
890 | } |
891 | |
892 | bool kvm_para_available(void) |
893 | { |
894 | return kvm_cpuid_base() != 0; |
895 | } |
896 | EXPORT_SYMBOL_GPL(kvm_para_available); |
897 | |
898 | unsigned int kvm_arch_para_features(void) |
899 | { |
900 | return cpuid_eax(op: kvm_cpuid_base() | KVM_CPUID_FEATURES); |
901 | } |
902 | |
903 | unsigned int kvm_arch_para_hints(void) |
904 | { |
905 | return cpuid_edx(op: kvm_cpuid_base() | KVM_CPUID_FEATURES); |
906 | } |
907 | EXPORT_SYMBOL_GPL(kvm_arch_para_hints); |
908 | |
909 | static uint32_t __init kvm_detect(void) |
910 | { |
911 | return kvm_cpuid_base(); |
912 | } |
913 | |
914 | static void __init kvm_apic_init(void) |
915 | { |
916 | #ifdef CONFIG_SMP |
917 | if (pv_ipi_supported()) |
918 | kvm_setup_pv_ipi(); |
919 | #endif |
920 | } |
921 | |
922 | static bool __init kvm_msi_ext_dest_id(void) |
923 | { |
924 | return kvm_para_has_feature(KVM_FEATURE_MSI_EXT_DEST_ID); |
925 | } |
926 | |
927 | static void kvm_sev_hc_page_enc_status(unsigned long pfn, int npages, bool enc) |
928 | { |
929 | kvm_sev_hypercall3(KVM_HC_MAP_GPA_RANGE, p1: pfn << PAGE_SHIFT, p2: npages, |
930 | KVM_MAP_GPA_RANGE_ENC_STAT(enc) | KVM_MAP_GPA_RANGE_PAGE_SZ_4K); |
931 | } |
932 | |
933 | static void __init kvm_init_platform(void) |
934 | { |
935 | if (cc_platform_has(attr: CC_ATTR_GUEST_MEM_ENCRYPT) && |
936 | kvm_para_has_feature(KVM_FEATURE_MIGRATION_CONTROL)) { |
937 | unsigned long nr_pages; |
938 | int i; |
939 | |
940 | pv_ops.mmu.notify_page_enc_status_changed = |
941 | kvm_sev_hc_page_enc_status; |
942 | |
943 | /* |
944 | * Reset the host's shared pages list related to kernel |
945 | * specific page encryption status settings before we load a |
946 | * new kernel by kexec. Reset the page encryption status |
947 | * during early boot instead of just before kexec to avoid SMP |
948 | * races during kvm_pv_guest_cpu_reboot(). |
949 | * NOTE: We cannot reset the complete shared pages list |
950 | * here as we need to retain the UEFI/OVMF firmware |
951 | * specific settings. |
952 | */ |
953 | |
954 | for (i = 0; i < e820_table->nr_entries; i++) { |
955 | struct e820_entry *entry = &e820_table->entries[i]; |
956 | |
957 | if (entry->type != E820_TYPE_RAM) |
958 | continue; |
959 | |
960 | nr_pages = DIV_ROUND_UP(entry->size, PAGE_SIZE); |
961 | |
962 | kvm_sev_hypercall3(KVM_HC_MAP_GPA_RANGE, p1: entry->addr, |
963 | p2: nr_pages, |
964 | KVM_MAP_GPA_RANGE_ENCRYPTED | KVM_MAP_GPA_RANGE_PAGE_SZ_4K); |
965 | } |
966 | |
967 | /* |
968 | * Ensure that _bss_decrypted section is marked as decrypted in the |
969 | * shared pages list. |
970 | */ |
971 | early_set_mem_enc_dec_hypercall(vaddr: (unsigned long)__start_bss_decrypted, |
972 | size: __end_bss_decrypted - __start_bss_decrypted, enc: 0); |
973 | |
974 | /* |
975 | * If not booted using EFI, enable Live migration support. |
976 | */ |
977 | if (!efi_enabled(EFI_BOOT)) |
978 | wrmsrl(MSR_KVM_MIGRATION_CONTROL, |
979 | KVM_MIGRATION_READY); |
980 | } |
981 | kvmclock_init(); |
982 | x86_platform.apic_post_init = kvm_apic_init; |
983 | } |
984 | |
985 | #if defined(CONFIG_AMD_MEM_ENCRYPT) |
986 | static void kvm_sev_es_hcall_prepare(struct ghcb *ghcb, struct pt_regs *regs) |
987 | { |
988 | /* RAX and CPL are already in the GHCB */ |
989 | ghcb_set_rbx(ghcb, value: regs->bx); |
990 | ghcb_set_rcx(ghcb, value: regs->cx); |
991 | ghcb_set_rdx(ghcb, value: regs->dx); |
992 | ghcb_set_rsi(ghcb, value: regs->si); |
993 | } |
994 | |
995 | static bool kvm_sev_es_hcall_finish(struct ghcb *ghcb, struct pt_regs *regs) |
996 | { |
997 | /* No checking of the return state needed */ |
998 | return true; |
999 | } |
1000 | #endif |
1001 | |
1002 | const __initconst struct hypervisor_x86 x86_hyper_kvm = { |
1003 | .name = "KVM" , |
1004 | .detect = kvm_detect, |
1005 | .type = X86_HYPER_KVM, |
1006 | .init.guest_late_init = kvm_guest_init, |
1007 | .init.x2apic_available = kvm_para_available, |
1008 | .init.msi_ext_dest_id = kvm_msi_ext_dest_id, |
1009 | .init.init_platform = kvm_init_platform, |
1010 | #if defined(CONFIG_AMD_MEM_ENCRYPT) |
1011 | .runtime.sev_es_hcall_prepare = kvm_sev_es_hcall_prepare, |
1012 | .runtime.sev_es_hcall_finish = kvm_sev_es_hcall_finish, |
1013 | #endif |
1014 | }; |
1015 | |
1016 | static __init int activate_jump_labels(void) |
1017 | { |
1018 | if (has_steal_clock) { |
1019 | static_key_slow_inc(key: ¶virt_steal_enabled); |
1020 | if (steal_acc) |
1021 | static_key_slow_inc(key: ¶virt_steal_rq_enabled); |
1022 | } |
1023 | |
1024 | return 0; |
1025 | } |
1026 | arch_initcall(activate_jump_labels); |
1027 | |
1028 | #ifdef CONFIG_PARAVIRT_SPINLOCKS |
1029 | |
1030 | /* Kick a cpu by its apicid. Used to wake up a halted vcpu */ |
1031 | static void kvm_kick_cpu(int cpu) |
1032 | { |
1033 | unsigned long flags = 0; |
1034 | u32 apicid; |
1035 | |
1036 | apicid = per_cpu(x86_cpu_to_apicid, cpu); |
1037 | kvm_hypercall2(KVM_HC_KICK_CPU, p1: flags, p2: apicid); |
1038 | } |
1039 | |
1040 | #include <asm/qspinlock.h> |
1041 | |
1042 | static void kvm_wait(u8 *ptr, u8 val) |
1043 | { |
1044 | if (in_nmi()) |
1045 | return; |
1046 | |
1047 | /* |
1048 | * halt until it's our turn and kicked. Note that we do safe halt |
1049 | * for irq enabled case to avoid hang when lock info is overwritten |
1050 | * in irq spinlock slowpath and no spurious interrupt occur to save us. |
1051 | */ |
1052 | if (irqs_disabled()) { |
1053 | if (READ_ONCE(*ptr) == val) |
1054 | halt(); |
1055 | } else { |
1056 | local_irq_disable(); |
1057 | |
1058 | /* safe_halt() will enable IRQ */ |
1059 | if (READ_ONCE(*ptr) == val) |
1060 | safe_halt(); |
1061 | else |
1062 | local_irq_enable(); |
1063 | } |
1064 | } |
1065 | |
1066 | /* |
1067 | * Setup pv_lock_ops to exploit KVM_FEATURE_PV_UNHALT if present. |
1068 | */ |
1069 | void __init kvm_spinlock_init(void) |
1070 | { |
1071 | /* |
1072 | * In case host doesn't support KVM_FEATURE_PV_UNHALT there is still an |
1073 | * advantage of keeping virt_spin_lock_key enabled: virt_spin_lock() is |
1074 | * preferred over native qspinlock when vCPU is preempted. |
1075 | */ |
1076 | if (!kvm_para_has_feature(KVM_FEATURE_PV_UNHALT)) { |
1077 | pr_info("PV spinlocks disabled, no host support\n" ); |
1078 | return; |
1079 | } |
1080 | |
1081 | /* |
1082 | * Disable PV spinlocks and use native qspinlock when dedicated pCPUs |
1083 | * are available. |
1084 | */ |
1085 | if (kvm_para_has_hint(KVM_HINTS_REALTIME)) { |
1086 | pr_info("PV spinlocks disabled with KVM_HINTS_REALTIME hints\n" ); |
1087 | goto out; |
1088 | } |
1089 | |
1090 | if (num_possible_cpus() == 1) { |
1091 | pr_info("PV spinlocks disabled, single CPU\n" ); |
1092 | goto out; |
1093 | } |
1094 | |
1095 | if (nopvspin) { |
1096 | pr_info("PV spinlocks disabled, forced by \"nopvspin\" parameter\n" ); |
1097 | goto out; |
1098 | } |
1099 | |
1100 | pr_info("PV spinlocks enabled\n" ); |
1101 | |
1102 | __pv_init_lock_hash(); |
1103 | pv_ops.lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; |
1104 | pv_ops.lock.queued_spin_unlock = |
1105 | PV_CALLEE_SAVE(__pv_queued_spin_unlock); |
1106 | pv_ops.lock.wait = kvm_wait; |
1107 | pv_ops.lock.kick = kvm_kick_cpu; |
1108 | |
1109 | /* |
1110 | * When PV spinlock is enabled which is preferred over |
1111 | * virt_spin_lock(), virt_spin_lock_key's value is meaningless. |
1112 | * Just disable it anyway. |
1113 | */ |
1114 | out: |
1115 | static_branch_disable(&virt_spin_lock_key); |
1116 | } |
1117 | |
1118 | #endif /* CONFIG_PARAVIRT_SPINLOCKS */ |
1119 | |
1120 | #ifdef CONFIG_ARCH_CPUIDLE_HALTPOLL |
1121 | |
1122 | static void kvm_disable_host_haltpoll(void *i) |
1123 | { |
1124 | wrmsrl(MSR_KVM_POLL_CONTROL, val: 0); |
1125 | } |
1126 | |
1127 | static void kvm_enable_host_haltpoll(void *i) |
1128 | { |
1129 | wrmsrl(MSR_KVM_POLL_CONTROL, val: 1); |
1130 | } |
1131 | |
1132 | void arch_haltpoll_enable(unsigned int cpu) |
1133 | { |
1134 | if (!kvm_para_has_feature(KVM_FEATURE_POLL_CONTROL)) { |
1135 | pr_err_once("host does not support poll control\n" ); |
1136 | pr_err_once("host upgrade recommended\n" ); |
1137 | return; |
1138 | } |
1139 | |
1140 | /* Enable guest halt poll disables host halt poll */ |
1141 | smp_call_function_single(cpuid: cpu, func: kvm_disable_host_haltpoll, NULL, wait: 1); |
1142 | } |
1143 | EXPORT_SYMBOL_GPL(arch_haltpoll_enable); |
1144 | |
1145 | void arch_haltpoll_disable(unsigned int cpu) |
1146 | { |
1147 | if (!kvm_para_has_feature(KVM_FEATURE_POLL_CONTROL)) |
1148 | return; |
1149 | |
1150 | /* Disable guest halt poll enables host halt poll */ |
1151 | smp_call_function_single(cpuid: cpu, func: kvm_enable_host_haltpoll, NULL, wait: 1); |
1152 | } |
1153 | EXPORT_SYMBOL_GPL(arch_haltpoll_disable); |
1154 | #endif |
1155 | |