1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Xen hypercall batching. |
4 | * |
5 | * Xen allows multiple hypercalls to be issued at once, using the |
6 | * multicall interface. This allows the cost of trapping into the |
7 | * hypervisor to be amortized over several calls. |
8 | * |
9 | * This file implements a simple interface for multicalls. There's a |
10 | * per-cpu buffer of outstanding multicalls. When you want to queue a |
11 | * multicall for issuing, you can allocate a multicall slot for the |
12 | * call and its arguments, along with storage for space which is |
13 | * pointed to by the arguments (for passing pointers to structures, |
14 | * etc). When the multicall is actually issued, all the space for the |
15 | * commands and allocated memory is freed for reuse. |
16 | * |
17 | * Multicalls are flushed whenever any of the buffers get full, or |
18 | * when explicitly requested. There's no way to get per-multicall |
19 | * return results back. It will BUG if any of the multicalls fail. |
20 | * |
21 | * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007 |
22 | */ |
23 | #include <linux/percpu.h> |
24 | #include <linux/hardirq.h> |
25 | #include <linux/debugfs.h> |
26 | |
27 | #include <asm/xen/hypercall.h> |
28 | |
29 | #include "multicalls.h" |
30 | #include "debugfs.h" |
31 | |
32 | #define MC_BATCH 32 |
33 | |
34 | #define MC_DEBUG 0 |
35 | |
36 | #define MC_ARGS (MC_BATCH * 16) |
37 | |
38 | |
39 | struct mc_buffer { |
40 | unsigned mcidx, argidx, cbidx; |
41 | struct multicall_entry entries[MC_BATCH]; |
42 | #if MC_DEBUG |
43 | struct multicall_entry debug[MC_BATCH]; |
44 | void *caller[MC_BATCH]; |
45 | #endif |
46 | unsigned char args[MC_ARGS]; |
47 | struct callback { |
48 | void (*fn)(void *); |
49 | void *data; |
50 | } callbacks[MC_BATCH]; |
51 | }; |
52 | |
53 | static DEFINE_PER_CPU(struct mc_buffer, mc_buffer); |
54 | DEFINE_PER_CPU(unsigned long, xen_mc_irq_flags); |
55 | |
56 | void xen_mc_flush(void) |
57 | { |
58 | struct mc_buffer *b = this_cpu_ptr(&mc_buffer); |
59 | struct multicall_entry *mc; |
60 | int ret = 0; |
61 | unsigned long flags; |
62 | int i; |
63 | |
64 | BUG_ON(preemptible()); |
65 | |
66 | /* Disable interrupts in case someone comes in and queues |
67 | something in the middle */ |
68 | local_irq_save(flags); |
69 | |
70 | trace_xen_mc_flush(mcidx: b->mcidx, argidx: b->argidx, cbidx: b->cbidx); |
71 | |
72 | #if MC_DEBUG |
73 | memcpy(b->debug, b->entries, |
74 | b->mcidx * sizeof(struct multicall_entry)); |
75 | #endif |
76 | |
77 | switch (b->mcidx) { |
78 | case 0: |
79 | /* no-op */ |
80 | BUG_ON(b->argidx != 0); |
81 | break; |
82 | |
83 | case 1: |
84 | /* Singleton multicall - bypass multicall machinery |
85 | and just do the call directly. */ |
86 | mc = &b->entries[0]; |
87 | |
88 | mc->result = xen_single_call(call: mc->op, a1: mc->args[0], a2: mc->args[1], |
89 | a3: mc->args[2], a4: mc->args[3], |
90 | a5: mc->args[4]); |
91 | ret = mc->result < 0; |
92 | break; |
93 | |
94 | default: |
95 | if (HYPERVISOR_multicall(call_list: b->entries, nr_calls: b->mcidx) != 0) |
96 | BUG(); |
97 | for (i = 0; i < b->mcidx; i++) |
98 | if (b->entries[i].result < 0) |
99 | ret++; |
100 | } |
101 | |
102 | if (WARN_ON(ret)) { |
103 | pr_err("%d of %d multicall(s) failed: cpu %d\n" , |
104 | ret, b->mcidx, smp_processor_id()); |
105 | for (i = 0; i < b->mcidx; i++) { |
106 | if (b->entries[i].result < 0) { |
107 | #if MC_DEBUG |
108 | pr_err(" call %2d: op=%lu arg=[%lx] result=%ld\t%pS\n" , |
109 | i + 1, |
110 | b->debug[i].op, |
111 | b->debug[i].args[0], |
112 | b->entries[i].result, |
113 | b->caller[i]); |
114 | #else |
115 | pr_err(" call %2d: op=%lu arg=[%lx] result=%ld\n" , |
116 | i + 1, |
117 | b->entries[i].op, |
118 | b->entries[i].args[0], |
119 | b->entries[i].result); |
120 | #endif |
121 | } |
122 | } |
123 | } |
124 | |
125 | b->mcidx = 0; |
126 | b->argidx = 0; |
127 | |
128 | for (i = 0; i < b->cbidx; i++) { |
129 | struct callback *cb = &b->callbacks[i]; |
130 | |
131 | (*cb->fn)(cb->data); |
132 | } |
133 | b->cbidx = 0; |
134 | |
135 | local_irq_restore(flags); |
136 | } |
137 | |
138 | struct multicall_space __xen_mc_entry(size_t args) |
139 | { |
140 | struct mc_buffer *b = this_cpu_ptr(&mc_buffer); |
141 | struct multicall_space ret; |
142 | unsigned argidx = roundup(b->argidx, sizeof(u64)); |
143 | |
144 | trace_xen_mc_entry_alloc(args); |
145 | |
146 | BUG_ON(preemptible()); |
147 | BUG_ON(b->argidx >= MC_ARGS); |
148 | |
149 | if (unlikely(b->mcidx == MC_BATCH || |
150 | (argidx + args) >= MC_ARGS)) { |
151 | trace_xen_mc_flush_reason(reason: (b->mcidx == MC_BATCH) ? |
152 | XEN_MC_FL_BATCH : XEN_MC_FL_ARGS); |
153 | xen_mc_flush(); |
154 | argidx = roundup(b->argidx, sizeof(u64)); |
155 | } |
156 | |
157 | ret.mc = &b->entries[b->mcidx]; |
158 | #if MC_DEBUG |
159 | b->caller[b->mcidx] = __builtin_return_address(0); |
160 | #endif |
161 | b->mcidx++; |
162 | ret.args = &b->args[argidx]; |
163 | b->argidx = argidx + args; |
164 | |
165 | BUG_ON(b->argidx >= MC_ARGS); |
166 | return ret; |
167 | } |
168 | |
169 | struct multicall_space xen_mc_extend_args(unsigned long op, size_t size) |
170 | { |
171 | struct mc_buffer *b = this_cpu_ptr(&mc_buffer); |
172 | struct multicall_space ret = { NULL, NULL }; |
173 | |
174 | BUG_ON(preemptible()); |
175 | BUG_ON(b->argidx >= MC_ARGS); |
176 | |
177 | if (unlikely(b->mcidx == 0 || |
178 | b->entries[b->mcidx - 1].op != op)) { |
179 | trace_xen_mc_extend_args(op, args: size, res: XEN_MC_XE_BAD_OP); |
180 | goto out; |
181 | } |
182 | |
183 | if (unlikely((b->argidx + size) >= MC_ARGS)) { |
184 | trace_xen_mc_extend_args(op, args: size, res: XEN_MC_XE_NO_SPACE); |
185 | goto out; |
186 | } |
187 | |
188 | ret.mc = &b->entries[b->mcidx - 1]; |
189 | ret.args = &b->args[b->argidx]; |
190 | b->argidx += size; |
191 | |
192 | BUG_ON(b->argidx >= MC_ARGS); |
193 | |
194 | trace_xen_mc_extend_args(op, args: size, res: XEN_MC_XE_OK); |
195 | out: |
196 | return ret; |
197 | } |
198 | |
199 | void xen_mc_callback(void (*fn)(void *), void *data) |
200 | { |
201 | struct mc_buffer *b = this_cpu_ptr(&mc_buffer); |
202 | struct callback *cb; |
203 | |
204 | if (b->cbidx == MC_BATCH) { |
205 | trace_xen_mc_flush_reason(reason: XEN_MC_FL_CALLBACK); |
206 | xen_mc_flush(); |
207 | } |
208 | |
209 | trace_xen_mc_callback(fn, data); |
210 | |
211 | cb = &b->callbacks[b->cbidx++]; |
212 | cb->fn = fn; |
213 | cb->data = data; |
214 | } |
215 | |