1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Xen stolen ticks accounting. |
4 | */ |
5 | #include <linux/kernel.h> |
6 | #include <linux/kernel_stat.h> |
7 | #include <linux/math64.h> |
8 | #include <linux/gfp.h> |
9 | #include <linux/slab.h> |
10 | #include <linux/static_call.h> |
11 | |
12 | #include <asm/paravirt.h> |
13 | #include <asm/xen/hypervisor.h> |
14 | #include <asm/xen/hypercall.h> |
15 | |
16 | #include <xen/events.h> |
17 | #include <xen/features.h> |
18 | #include <xen/interface/xen.h> |
19 | #include <xen/interface/vcpu.h> |
20 | #include <xen/xen-ops.h> |
21 | |
22 | /* runstate info updated by Xen */ |
23 | static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate); |
24 | |
25 | static DEFINE_PER_CPU(u64[4], old_runstate_time); |
26 | |
27 | /* return an consistent snapshot of 64-bit time/counter value */ |
28 | static u64 get64(const u64 *p) |
29 | { |
30 | u64 ret; |
31 | |
32 | if (BITS_PER_LONG < 64) { |
33 | u32 *p32 = (u32 *)p; |
34 | u32 h, l, h2; |
35 | |
36 | /* |
37 | * Read high then low, and then make sure high is |
38 | * still the same; this will only loop if low wraps |
39 | * and carries into high. |
40 | * XXX some clean way to make this endian-proof? |
41 | */ |
42 | do { |
43 | h = READ_ONCE(p32[1]); |
44 | l = READ_ONCE(p32[0]); |
45 | h2 = READ_ONCE(p32[1]); |
46 | } while(h2 != h); |
47 | |
48 | ret = (((u64)h) << 32) | l; |
49 | } else |
50 | ret = READ_ONCE(*p); |
51 | |
52 | return ret; |
53 | } |
54 | |
55 | static void xen_get_runstate_snapshot_cpu_delta( |
56 | struct vcpu_runstate_info *res, unsigned int cpu) |
57 | { |
58 | u64 state_time; |
59 | struct vcpu_runstate_info *state; |
60 | |
61 | BUG_ON(preemptible()); |
62 | |
63 | state = per_cpu_ptr(&xen_runstate, cpu); |
64 | |
65 | do { |
66 | state_time = get64(p: &state->state_entry_time); |
67 | rmb(); /* Hypervisor might update data. */ |
68 | *res = __READ_ONCE(*state); |
69 | rmb(); /* Hypervisor might update data. */ |
70 | } while (get64(p: &state->state_entry_time) != state_time || |
71 | (state_time & XEN_RUNSTATE_UPDATE)); |
72 | } |
73 | |
74 | static void xen_get_runstate_snapshot_cpu(struct vcpu_runstate_info *res, |
75 | unsigned int cpu) |
76 | { |
77 | int i; |
78 | |
79 | xen_get_runstate_snapshot_cpu_delta(res, cpu); |
80 | |
81 | for (i = 0; i < 4; i++) |
82 | res->time[i] += per_cpu(old_runstate_time, cpu)[i]; |
83 | } |
84 | |
85 | void xen_manage_runstate_time(int action) |
86 | { |
87 | static struct vcpu_runstate_info *runstate_delta; |
88 | struct vcpu_runstate_info state; |
89 | int cpu, i; |
90 | |
91 | switch (action) { |
92 | case -1: /* backup runstate time before suspend */ |
93 | if (unlikely(runstate_delta)) |
94 | pr_warn_once("%s: memory leak as runstate_delta is not NULL\n" , |
95 | __func__); |
96 | |
97 | runstate_delta = kmalloc_array(num_possible_cpus(), |
98 | size: sizeof(*runstate_delta), |
99 | GFP_ATOMIC); |
100 | if (unlikely(!runstate_delta)) { |
101 | pr_warn("%s: failed to allocate runstate_delta\n" , |
102 | __func__); |
103 | return; |
104 | } |
105 | |
106 | for_each_possible_cpu(cpu) { |
107 | xen_get_runstate_snapshot_cpu_delta(res: &state, cpu); |
108 | memcpy(runstate_delta[cpu].time, state.time, |
109 | sizeof(runstate_delta[cpu].time)); |
110 | } |
111 | |
112 | break; |
113 | |
114 | case 0: /* backup runstate time after resume */ |
115 | if (unlikely(!runstate_delta)) { |
116 | pr_warn("%s: cannot accumulate runstate time as runstate_delta is NULL\n" , |
117 | __func__); |
118 | return; |
119 | } |
120 | |
121 | for_each_possible_cpu(cpu) { |
122 | for (i = 0; i < 4; i++) |
123 | per_cpu(old_runstate_time, cpu)[i] += |
124 | runstate_delta[cpu].time[i]; |
125 | } |
126 | |
127 | break; |
128 | |
129 | default: /* do not accumulate runstate time for checkpointing */ |
130 | break; |
131 | } |
132 | |
133 | if (action != -1 && runstate_delta) { |
134 | kfree(objp: runstate_delta); |
135 | runstate_delta = NULL; |
136 | } |
137 | } |
138 | |
139 | /* |
140 | * Runstate accounting |
141 | */ |
142 | void xen_get_runstate_snapshot(struct vcpu_runstate_info *res) |
143 | { |
144 | xen_get_runstate_snapshot_cpu(res, smp_processor_id()); |
145 | } |
146 | |
147 | /* return true when a vcpu could run but has no real cpu to run on */ |
148 | bool xen_vcpu_stolen(int vcpu) |
149 | { |
150 | return per_cpu(xen_runstate, vcpu).state == RUNSTATE_runnable; |
151 | } |
152 | |
153 | u64 xen_steal_clock(int cpu) |
154 | { |
155 | struct vcpu_runstate_info state; |
156 | |
157 | xen_get_runstate_snapshot_cpu(res: &state, cpu); |
158 | return state.time[RUNSTATE_runnable] + state.time[RUNSTATE_offline]; |
159 | } |
160 | |
161 | void xen_setup_runstate_info(int cpu) |
162 | { |
163 | struct vcpu_register_runstate_memory_area area; |
164 | |
165 | area.addr.v = &per_cpu(xen_runstate, cpu); |
166 | |
167 | if (HYPERVISOR_vcpu_op(VCPUOP_register_runstate_memory_area, |
168 | vcpuid: xen_vcpu_nr(cpu), extra_args: &area)) |
169 | BUG(); |
170 | } |
171 | |
172 | void __init xen_time_setup_guest(void) |
173 | { |
174 | bool xen_runstate_remote; |
175 | |
176 | xen_runstate_remote = !HYPERVISOR_vm_assist(VMASST_CMD_enable, |
177 | VMASST_TYPE_runstate_update_flag); |
178 | |
179 | static_call_update(pv_steal_clock, xen_steal_clock); |
180 | |
181 | static_key_slow_inc(key: ¶virt_steal_enabled); |
182 | if (xen_runstate_remote) |
183 | static_key_slow_inc(key: ¶virt_steal_rq_enabled); |
184 | } |
185 | |