1 | // SPDX-License-Identifier: GPL-2.0 |
2 | #include <linux/smp.h> |
3 | #include <linux/cpu.h> |
4 | #include <linux/slab.h> |
5 | #include <linux/cpumask.h> |
6 | #include <linux/percpu.h> |
7 | |
8 | #include <xen/events.h> |
9 | |
10 | #include <xen/hvc-console.h> |
11 | #include "xen-ops.h" |
12 | #include "smp.h" |
13 | |
14 | static DEFINE_PER_CPU(struct xen_common_irq, xen_resched_irq) = { .irq = -1 }; |
15 | static DEFINE_PER_CPU(struct xen_common_irq, xen_callfunc_irq) = { .irq = -1 }; |
16 | static DEFINE_PER_CPU(struct xen_common_irq, xen_callfuncsingle_irq) = { .irq = -1 }; |
17 | static DEFINE_PER_CPU(struct xen_common_irq, xen_debug_irq) = { .irq = -1 }; |
18 | |
19 | static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id); |
20 | static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id); |
21 | |
22 | /* |
23 | * Reschedule call back. |
24 | */ |
25 | static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id) |
26 | { |
27 | inc_irq_stat(irq_resched_count); |
28 | scheduler_ipi(); |
29 | |
30 | return IRQ_HANDLED; |
31 | } |
32 | |
33 | void xen_smp_intr_free(unsigned int cpu) |
34 | { |
35 | kfree(per_cpu(xen_resched_irq, cpu).name); |
36 | per_cpu(xen_resched_irq, cpu).name = NULL; |
37 | if (per_cpu(xen_resched_irq, cpu).irq >= 0) { |
38 | unbind_from_irqhandler(per_cpu(xen_resched_irq, cpu).irq, NULL); |
39 | per_cpu(xen_resched_irq, cpu).irq = -1; |
40 | } |
41 | kfree(per_cpu(xen_callfunc_irq, cpu).name); |
42 | per_cpu(xen_callfunc_irq, cpu).name = NULL; |
43 | if (per_cpu(xen_callfunc_irq, cpu).irq >= 0) { |
44 | unbind_from_irqhandler(per_cpu(xen_callfunc_irq, cpu).irq, NULL); |
45 | per_cpu(xen_callfunc_irq, cpu).irq = -1; |
46 | } |
47 | kfree(per_cpu(xen_debug_irq, cpu).name); |
48 | per_cpu(xen_debug_irq, cpu).name = NULL; |
49 | if (per_cpu(xen_debug_irq, cpu).irq >= 0) { |
50 | unbind_from_irqhandler(per_cpu(xen_debug_irq, cpu).irq, NULL); |
51 | per_cpu(xen_debug_irq, cpu).irq = -1; |
52 | } |
53 | kfree(per_cpu(xen_callfuncsingle_irq, cpu).name); |
54 | per_cpu(xen_callfuncsingle_irq, cpu).name = NULL; |
55 | if (per_cpu(xen_callfuncsingle_irq, cpu).irq >= 0) { |
56 | unbind_from_irqhandler(per_cpu(xen_callfuncsingle_irq, cpu).irq, |
57 | NULL); |
58 | per_cpu(xen_callfuncsingle_irq, cpu).irq = -1; |
59 | } |
60 | } |
61 | |
62 | int xen_smp_intr_init(unsigned int cpu) |
63 | { |
64 | int rc; |
65 | char *resched_name, *callfunc_name, *debug_name; |
66 | |
67 | resched_name = kasprintf(GFP_KERNEL, fmt: "resched%d" , cpu); |
68 | if (!resched_name) |
69 | goto fail_mem; |
70 | per_cpu(xen_resched_irq, cpu).name = resched_name; |
71 | rc = bind_ipi_to_irqhandler(ipi: XEN_RESCHEDULE_VECTOR, |
72 | cpu, |
73 | handler: xen_reschedule_interrupt, |
74 | IRQF_PERCPU|IRQF_NOBALANCING, |
75 | devname: resched_name, |
76 | NULL); |
77 | if (rc < 0) |
78 | goto fail; |
79 | per_cpu(xen_resched_irq, cpu).irq = rc; |
80 | |
81 | callfunc_name = kasprintf(GFP_KERNEL, fmt: "callfunc%d" , cpu); |
82 | if (!callfunc_name) |
83 | goto fail_mem; |
84 | per_cpu(xen_callfunc_irq, cpu).name = callfunc_name; |
85 | rc = bind_ipi_to_irqhandler(ipi: XEN_CALL_FUNCTION_VECTOR, |
86 | cpu, |
87 | handler: xen_call_function_interrupt, |
88 | IRQF_PERCPU|IRQF_NOBALANCING, |
89 | devname: callfunc_name, |
90 | NULL); |
91 | if (rc < 0) |
92 | goto fail; |
93 | per_cpu(xen_callfunc_irq, cpu).irq = rc; |
94 | |
95 | if (!xen_fifo_events) { |
96 | debug_name = kasprintf(GFP_KERNEL, fmt: "debug%d" , cpu); |
97 | if (!debug_name) |
98 | goto fail_mem; |
99 | |
100 | per_cpu(xen_debug_irq, cpu).name = debug_name; |
101 | rc = bind_virq_to_irqhandler(VIRQ_DEBUG, cpu, |
102 | handler: xen_debug_interrupt, |
103 | IRQF_PERCPU | IRQF_NOBALANCING, |
104 | devname: debug_name, NULL); |
105 | if (rc < 0) |
106 | goto fail; |
107 | per_cpu(xen_debug_irq, cpu).irq = rc; |
108 | } |
109 | |
110 | callfunc_name = kasprintf(GFP_KERNEL, fmt: "callfuncsingle%d" , cpu); |
111 | if (!callfunc_name) |
112 | goto fail_mem; |
113 | |
114 | per_cpu(xen_callfuncsingle_irq, cpu).name = callfunc_name; |
115 | rc = bind_ipi_to_irqhandler(ipi: XEN_CALL_FUNCTION_SINGLE_VECTOR, |
116 | cpu, |
117 | handler: xen_call_function_single_interrupt, |
118 | IRQF_PERCPU|IRQF_NOBALANCING, |
119 | devname: callfunc_name, |
120 | NULL); |
121 | if (rc < 0) |
122 | goto fail; |
123 | per_cpu(xen_callfuncsingle_irq, cpu).irq = rc; |
124 | |
125 | return 0; |
126 | |
127 | fail_mem: |
128 | rc = -ENOMEM; |
129 | fail: |
130 | xen_smp_intr_free(cpu); |
131 | return rc; |
132 | } |
133 | |
134 | void __init xen_smp_cpus_done(unsigned int max_cpus) |
135 | { |
136 | if (xen_hvm_domain()) |
137 | native_smp_cpus_done(max_cpus); |
138 | } |
139 | |
140 | void xen_smp_send_reschedule(int cpu) |
141 | { |
142 | xen_send_IPI_one(cpu, vector: XEN_RESCHEDULE_VECTOR); |
143 | } |
144 | |
145 | static void __xen_send_IPI_mask(const struct cpumask *mask, |
146 | int vector) |
147 | { |
148 | unsigned cpu; |
149 | |
150 | for_each_cpu_and(cpu, mask, cpu_online_mask) |
151 | xen_send_IPI_one(cpu, vector); |
152 | } |
153 | |
154 | void xen_smp_send_call_function_ipi(const struct cpumask *mask) |
155 | { |
156 | int cpu; |
157 | |
158 | __xen_send_IPI_mask(mask, vector: XEN_CALL_FUNCTION_VECTOR); |
159 | |
160 | /* Make sure other vcpus get a chance to run if they need to. */ |
161 | for_each_cpu(cpu, mask) { |
162 | if (xen_vcpu_stolen(vcpu: cpu)) { |
163 | HYPERVISOR_sched_op(SCHEDOP_yield, NULL); |
164 | break; |
165 | } |
166 | } |
167 | } |
168 | |
169 | void xen_smp_send_call_function_single_ipi(int cpu) |
170 | { |
171 | __xen_send_IPI_mask(cpumask_of(cpu), |
172 | vector: XEN_CALL_FUNCTION_SINGLE_VECTOR); |
173 | } |
174 | |
175 | static inline int xen_map_vector(int vector) |
176 | { |
177 | int xen_vector; |
178 | |
179 | switch (vector) { |
180 | case RESCHEDULE_VECTOR: |
181 | xen_vector = XEN_RESCHEDULE_VECTOR; |
182 | break; |
183 | case CALL_FUNCTION_VECTOR: |
184 | xen_vector = XEN_CALL_FUNCTION_VECTOR; |
185 | break; |
186 | case CALL_FUNCTION_SINGLE_VECTOR: |
187 | xen_vector = XEN_CALL_FUNCTION_SINGLE_VECTOR; |
188 | break; |
189 | case IRQ_WORK_VECTOR: |
190 | xen_vector = XEN_IRQ_WORK_VECTOR; |
191 | break; |
192 | #ifdef CONFIG_X86_64 |
193 | case NMI_VECTOR: |
194 | case APIC_DM_NMI: /* Some use that instead of NMI_VECTOR */ |
195 | xen_vector = XEN_NMI_VECTOR; |
196 | break; |
197 | #endif |
198 | default: |
199 | xen_vector = -1; |
200 | printk(KERN_ERR "xen: vector 0x%x is not implemented\n" , |
201 | vector); |
202 | } |
203 | |
204 | return xen_vector; |
205 | } |
206 | |
207 | void xen_send_IPI_mask(const struct cpumask *mask, |
208 | int vector) |
209 | { |
210 | int xen_vector = xen_map_vector(vector); |
211 | |
212 | if (xen_vector >= 0) |
213 | __xen_send_IPI_mask(mask, vector: xen_vector); |
214 | } |
215 | |
216 | void xen_send_IPI_all(int vector) |
217 | { |
218 | int xen_vector = xen_map_vector(vector); |
219 | |
220 | if (xen_vector >= 0) |
221 | __xen_send_IPI_mask(cpu_online_mask, vector: xen_vector); |
222 | } |
223 | |
224 | void xen_send_IPI_self(int vector) |
225 | { |
226 | int xen_vector = xen_map_vector(vector); |
227 | |
228 | if (xen_vector >= 0) |
229 | xen_send_IPI_one(smp_processor_id(), vector: xen_vector); |
230 | } |
231 | |
232 | void xen_send_IPI_mask_allbutself(const struct cpumask *mask, |
233 | int vector) |
234 | { |
235 | unsigned cpu; |
236 | unsigned int this_cpu = smp_processor_id(); |
237 | int xen_vector = xen_map_vector(vector); |
238 | |
239 | if (!(num_online_cpus() > 1) || (xen_vector < 0)) |
240 | return; |
241 | |
242 | for_each_cpu_and(cpu, mask, cpu_online_mask) { |
243 | if (this_cpu == cpu) |
244 | continue; |
245 | |
246 | xen_send_IPI_one(cpu, vector: xen_vector); |
247 | } |
248 | } |
249 | |
250 | void xen_send_IPI_allbutself(int vector) |
251 | { |
252 | xen_send_IPI_mask_allbutself(cpu_online_mask, vector); |
253 | } |
254 | |
255 | static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id) |
256 | { |
257 | generic_smp_call_function_interrupt(); |
258 | inc_irq_stat(irq_call_count); |
259 | |
260 | return IRQ_HANDLED; |
261 | } |
262 | |
263 | static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id) |
264 | { |
265 | generic_smp_call_function_single_interrupt(); |
266 | inc_irq_stat(irq_call_count); |
267 | |
268 | return IRQ_HANDLED; |
269 | } |
270 | |