1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* paravirtual clock -- common code used by kvm/xen |
3 | |
4 | */ |
5 | |
6 | #include <linux/clocksource.h> |
7 | #include <linux/kernel.h> |
8 | #include <linux/percpu.h> |
9 | #include <linux/notifier.h> |
10 | #include <linux/sched.h> |
11 | #include <linux/gfp.h> |
12 | #include <linux/memblock.h> |
13 | #include <linux/nmi.h> |
14 | |
15 | #include <asm/fixmap.h> |
16 | #include <asm/pvclock.h> |
17 | #include <asm/vgtod.h> |
18 | |
19 | static u8 valid_flags __read_mostly = 0; |
20 | static struct pvclock_vsyscall_time_info *pvti_cpu0_va __read_mostly; |
21 | |
22 | void pvclock_set_flags(u8 flags) |
23 | { |
24 | valid_flags = flags; |
25 | } |
26 | |
27 | unsigned long pvclock_tsc_khz(struct pvclock_vcpu_time_info *src) |
28 | { |
29 | u64 pv_tsc_khz = 1000000ULL << 32; |
30 | |
31 | do_div(pv_tsc_khz, src->tsc_to_system_mul); |
32 | if (src->tsc_shift < 0) |
33 | pv_tsc_khz <<= -src->tsc_shift; |
34 | else |
35 | pv_tsc_khz >>= src->tsc_shift; |
36 | return pv_tsc_khz; |
37 | } |
38 | |
39 | void pvclock_touch_watchdogs(void) |
40 | { |
41 | touch_softlockup_watchdog_sync(); |
42 | clocksource_touch_watchdog(); |
43 | rcu_cpu_stall_reset(); |
44 | reset_hung_task_detector(); |
45 | } |
46 | |
47 | static atomic64_t last_value = ATOMIC64_INIT(0); |
48 | |
49 | void pvclock_resume(void) |
50 | { |
51 | atomic64_set(v: &last_value, i: 0); |
52 | } |
53 | |
54 | u8 pvclock_read_flags(struct pvclock_vcpu_time_info *src) |
55 | { |
56 | unsigned version; |
57 | u8 flags; |
58 | |
59 | do { |
60 | version = pvclock_read_begin(src); |
61 | flags = src->flags; |
62 | } while (pvclock_read_retry(src, version)); |
63 | |
64 | return flags & valid_flags; |
65 | } |
66 | |
67 | static __always_inline |
68 | u64 __pvclock_clocksource_read(struct pvclock_vcpu_time_info *src, bool dowd) |
69 | { |
70 | unsigned version; |
71 | u64 ret; |
72 | u64 last; |
73 | u8 flags; |
74 | |
75 | do { |
76 | version = pvclock_read_begin(src); |
77 | ret = __pvclock_read_cycles(src, tsc: rdtsc_ordered()); |
78 | flags = src->flags; |
79 | } while (pvclock_read_retry(src, version)); |
80 | |
81 | if (dowd && unlikely((flags & PVCLOCK_GUEST_STOPPED) != 0)) { |
82 | src->flags &= ~PVCLOCK_GUEST_STOPPED; |
83 | pvclock_touch_watchdogs(); |
84 | } |
85 | |
86 | if ((valid_flags & PVCLOCK_TSC_STABLE_BIT) && |
87 | (flags & PVCLOCK_TSC_STABLE_BIT)) |
88 | return ret; |
89 | |
90 | /* |
91 | * Assumption here is that last_value, a global accumulator, always goes |
92 | * forward. If we are less than that, we should not be much smaller. |
93 | * We assume there is an error margin we're inside, and then the correction |
94 | * does not sacrifice accuracy. |
95 | * |
96 | * For reads: global may have changed between test and return, |
97 | * but this means someone else updated poked the clock at a later time. |
98 | * We just need to make sure we are not seeing a backwards event. |
99 | * |
100 | * For updates: last_value = ret is not enough, since two vcpus could be |
101 | * updating at the same time, and one of them could be slightly behind, |
102 | * making the assumption that last_value always go forward fail to hold. |
103 | */ |
104 | last = raw_atomic64_read(v: &last_value); |
105 | do { |
106 | if (ret <= last) |
107 | return last; |
108 | } while (!raw_atomic64_try_cmpxchg(v: &last_value, old: &last, new: ret)); |
109 | |
110 | return ret; |
111 | } |
112 | |
113 | u64 pvclock_clocksource_read(struct pvclock_vcpu_time_info *src) |
114 | { |
115 | return __pvclock_clocksource_read(src, dowd: true); |
116 | } |
117 | |
118 | noinstr u64 pvclock_clocksource_read_nowd(struct pvclock_vcpu_time_info *src) |
119 | { |
120 | return __pvclock_clocksource_read(src, dowd: false); |
121 | } |
122 | |
123 | void pvclock_read_wallclock(struct pvclock_wall_clock *wall_clock, |
124 | struct pvclock_vcpu_time_info *vcpu_time, |
125 | struct timespec64 *ts) |
126 | { |
127 | u32 version; |
128 | u64 delta; |
129 | struct timespec64 now; |
130 | |
131 | /* get wallclock at system boot */ |
132 | do { |
133 | version = wall_clock->version; |
134 | rmb(); /* fetch version before time */ |
135 | /* |
136 | * Note: wall_clock->sec is a u32 value, so it can |
137 | * only store dates between 1970 and 2106. To allow |
138 | * times beyond that, we need to create a new hypercall |
139 | * interface with an extended pvclock_wall_clock structure |
140 | * like ARM has. |
141 | */ |
142 | now.tv_sec = wall_clock->sec; |
143 | now.tv_nsec = wall_clock->nsec; |
144 | rmb(); /* fetch time before checking version */ |
145 | } while ((wall_clock->version & 1) || (version != wall_clock->version)); |
146 | |
147 | delta = pvclock_clocksource_read(src: vcpu_time); /* time since system boot */ |
148 | delta += now.tv_sec * NSEC_PER_SEC + now.tv_nsec; |
149 | |
150 | now.tv_nsec = do_div(delta, NSEC_PER_SEC); |
151 | now.tv_sec = delta; |
152 | |
153 | set_normalized_timespec64(ts, sec: now.tv_sec, nsec: now.tv_nsec); |
154 | } |
155 | |
156 | void pvclock_set_pvti_cpu0_va(struct pvclock_vsyscall_time_info *pvti) |
157 | { |
158 | WARN_ON(vclock_was_used(VDSO_CLOCKMODE_PVCLOCK)); |
159 | pvti_cpu0_va = pvti; |
160 | } |
161 | |
162 | struct pvclock_vsyscall_time_info *pvclock_get_pvti_cpu0_va(void) |
163 | { |
164 | return pvti_cpu0_va; |
165 | } |
166 | EXPORT_SYMBOL_GPL(pvclock_get_pvti_cpu0_va); |
167 | |