1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright (C) 2015 Imagination Technologies Ltd |
4 | * Author: Qais Yousef <qais.yousef@imgtec.com> |
5 | * |
6 | * This file contains driver APIs to the IPI subsystem. |
7 | */ |
8 | |
9 | #define pr_fmt(fmt) "genirq/ipi: " fmt |
10 | |
11 | #include <linux/irqdomain.h> |
12 | #include <linux/irq.h> |
13 | |
14 | /** |
15 | * irq_reserve_ipi() - Setup an IPI to destination cpumask |
16 | * @domain: IPI domain |
17 | * @dest: cpumask of CPUs which can receive the IPI |
18 | * |
19 | * Allocate a virq that can be used to send IPI to any CPU in dest mask. |
20 | * |
21 | * Return: Linux IRQ number on success or error code on failure |
22 | */ |
23 | int irq_reserve_ipi(struct irq_domain *domain, |
24 | const struct cpumask *dest) |
25 | { |
26 | unsigned int nr_irqs, offset; |
27 | struct irq_data *data; |
28 | int virq, i; |
29 | |
30 | if (!domain ||!irq_domain_is_ipi(domain)) { |
31 | pr_warn("Reservation on a non IPI domain\n" ); |
32 | return -EINVAL; |
33 | } |
34 | |
35 | if (!cpumask_subset(src1p: dest, cpu_possible_mask)) { |
36 | pr_warn("Reservation is not in possible_cpu_mask\n" ); |
37 | return -EINVAL; |
38 | } |
39 | |
40 | nr_irqs = cpumask_weight(srcp: dest); |
41 | if (!nr_irqs) { |
42 | pr_warn("Reservation for empty destination mask\n" ); |
43 | return -EINVAL; |
44 | } |
45 | |
46 | if (irq_domain_is_ipi_single(domain)) { |
47 | /* |
48 | * If the underlying implementation uses a single HW irq on |
49 | * all cpus then we only need a single Linux irq number for |
50 | * it. We have no restrictions vs. the destination mask. The |
51 | * underlying implementation can deal with holes nicely. |
52 | */ |
53 | nr_irqs = 1; |
54 | offset = 0; |
55 | } else { |
56 | unsigned int next; |
57 | |
58 | /* |
59 | * The IPI requires a separate HW irq on each CPU. We require |
60 | * that the destination mask is consecutive. If an |
61 | * implementation needs to support holes, it can reserve |
62 | * several IPI ranges. |
63 | */ |
64 | offset = cpumask_first(srcp: dest); |
65 | /* |
66 | * Find a hole and if found look for another set bit after the |
67 | * hole. For now we don't support this scenario. |
68 | */ |
69 | next = cpumask_next_zero(n: offset, srcp: dest); |
70 | if (next < nr_cpu_ids) |
71 | next = cpumask_next(n: next, srcp: dest); |
72 | if (next < nr_cpu_ids) { |
73 | pr_warn("Destination mask has holes\n" ); |
74 | return -EINVAL; |
75 | } |
76 | } |
77 | |
78 | virq = irq_domain_alloc_descs(virq: -1, nr_irqs, hwirq: 0, NUMA_NO_NODE, NULL); |
79 | if (virq <= 0) { |
80 | pr_warn("Can't reserve IPI, failed to alloc descs\n" ); |
81 | return -ENOMEM; |
82 | } |
83 | |
84 | virq = __irq_domain_alloc_irqs(domain, irq_base: virq, nr_irqs, NUMA_NO_NODE, |
85 | arg: (void *) dest, realloc: true, NULL); |
86 | |
87 | if (virq <= 0) { |
88 | pr_warn("Can't reserve IPI, failed to alloc hw irqs\n" ); |
89 | goto free_descs; |
90 | } |
91 | |
92 | for (i = 0; i < nr_irqs; i++) { |
93 | data = irq_get_irq_data(irq: virq + i); |
94 | cpumask_copy(dstp: data->common->affinity, srcp: dest); |
95 | data->common->ipi_offset = offset; |
96 | irq_set_status_flags(irq: virq + i, set: IRQ_NO_BALANCING); |
97 | } |
98 | return virq; |
99 | |
100 | free_descs: |
101 | irq_free_descs(irq: virq, cnt: nr_irqs); |
102 | return -EBUSY; |
103 | } |
104 | |
105 | /** |
106 | * irq_destroy_ipi() - unreserve an IPI that was previously allocated |
107 | * @irq: Linux IRQ number to be destroyed |
108 | * @dest: cpumask of CPUs which should have the IPI removed |
109 | * |
110 | * The IPIs allocated with irq_reserve_ipi() are returned to the system |
111 | * destroying all virqs associated with them. |
112 | * |
113 | * Return: %0 on success or error code on failure. |
114 | */ |
115 | int irq_destroy_ipi(unsigned int irq, const struct cpumask *dest) |
116 | { |
117 | struct irq_data *data = irq_get_irq_data(irq); |
118 | const struct cpumask *ipimask; |
119 | struct irq_domain *domain; |
120 | unsigned int nr_irqs; |
121 | |
122 | if (!irq || !data) |
123 | return -EINVAL; |
124 | |
125 | domain = data->domain; |
126 | if (WARN_ON(domain == NULL)) |
127 | return -EINVAL; |
128 | |
129 | if (!irq_domain_is_ipi(domain)) { |
130 | pr_warn("Trying to destroy a non IPI domain!\n" ); |
131 | return -EINVAL; |
132 | } |
133 | |
134 | ipimask = irq_data_get_affinity_mask(d: data); |
135 | if (!ipimask || WARN_ON(!cpumask_subset(dest, ipimask))) |
136 | /* |
137 | * Must be destroying a subset of CPUs to which this IPI |
138 | * was set up to target |
139 | */ |
140 | return -EINVAL; |
141 | |
142 | if (irq_domain_is_ipi_per_cpu(domain)) { |
143 | irq = irq + cpumask_first(srcp: dest) - data->common->ipi_offset; |
144 | nr_irqs = cpumask_weight(srcp: dest); |
145 | } else { |
146 | nr_irqs = 1; |
147 | } |
148 | |
149 | irq_domain_free_irqs(virq: irq, nr_irqs); |
150 | return 0; |
151 | } |
152 | |
153 | /** |
154 | * ipi_get_hwirq - Get the hwirq associated with an IPI to a CPU |
155 | * @irq: Linux IRQ number |
156 | * @cpu: the target CPU |
157 | * |
158 | * When dealing with coprocessors IPI, we need to inform the coprocessor of |
159 | * the hwirq it needs to use to receive and send IPIs. |
160 | * |
161 | * Return: hwirq value on success or INVALID_HWIRQ on failure. |
162 | */ |
163 | irq_hw_number_t ipi_get_hwirq(unsigned int irq, unsigned int cpu) |
164 | { |
165 | struct irq_data *data = irq_get_irq_data(irq); |
166 | const struct cpumask *ipimask; |
167 | |
168 | if (!data || cpu >= nr_cpu_ids) |
169 | return INVALID_HWIRQ; |
170 | |
171 | ipimask = irq_data_get_affinity_mask(d: data); |
172 | if (!ipimask || !cpumask_test_cpu(cpu, cpumask: ipimask)) |
173 | return INVALID_HWIRQ; |
174 | |
175 | /* |
176 | * Get the real hardware irq number if the underlying implementation |
177 | * uses a separate irq per cpu. If the underlying implementation uses |
178 | * a single hardware irq for all cpus then the IPI send mechanism |
179 | * needs to take care of the cpu destinations. |
180 | */ |
181 | if (irq_domain_is_ipi_per_cpu(domain: data->domain)) |
182 | data = irq_get_irq_data(irq: irq + cpu - data->common->ipi_offset); |
183 | |
184 | return data ? irqd_to_hwirq(d: data) : INVALID_HWIRQ; |
185 | } |
186 | EXPORT_SYMBOL_GPL(ipi_get_hwirq); |
187 | |
188 | static int ipi_send_verify(struct irq_chip *chip, struct irq_data *data, |
189 | const struct cpumask *dest, unsigned int cpu) |
190 | { |
191 | const struct cpumask *ipimask; |
192 | |
193 | if (!chip || !data) |
194 | return -EINVAL; |
195 | |
196 | if (!chip->ipi_send_single && !chip->ipi_send_mask) |
197 | return -EINVAL; |
198 | |
199 | if (cpu >= nr_cpu_ids) |
200 | return -EINVAL; |
201 | |
202 | ipimask = irq_data_get_affinity_mask(d: data); |
203 | if (!ipimask) |
204 | return -EINVAL; |
205 | |
206 | if (dest) { |
207 | if (!cpumask_subset(src1p: dest, src2p: ipimask)) |
208 | return -EINVAL; |
209 | } else { |
210 | if (!cpumask_test_cpu(cpu, cpumask: ipimask)) |
211 | return -EINVAL; |
212 | } |
213 | return 0; |
214 | } |
215 | |
216 | /** |
217 | * __ipi_send_single - send an IPI to a target Linux SMP CPU |
218 | * @desc: pointer to irq_desc of the IRQ |
219 | * @cpu: destination CPU, must in the destination mask passed to |
220 | * irq_reserve_ipi() |
221 | * |
222 | * This function is for architecture or core code to speed up IPI sending. Not |
223 | * usable from driver code. |
224 | * |
225 | * Return: %0 on success or negative error number on failure. |
226 | */ |
227 | int __ipi_send_single(struct irq_desc *desc, unsigned int cpu) |
228 | { |
229 | struct irq_data *data = irq_desc_get_irq_data(desc); |
230 | struct irq_chip *chip = irq_data_get_irq_chip(d: data); |
231 | |
232 | #ifdef DEBUG |
233 | /* |
234 | * Minimise the overhead by omitting the checks for Linux SMP IPIs. |
235 | * Since the callers should be arch or core code which is generally |
236 | * trusted, only check for errors when debugging. |
237 | */ |
238 | if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu))) |
239 | return -EINVAL; |
240 | #endif |
241 | if (!chip->ipi_send_single) { |
242 | chip->ipi_send_mask(data, cpumask_of(cpu)); |
243 | return 0; |
244 | } |
245 | |
246 | /* FIXME: Store this information in irqdata flags */ |
247 | if (irq_domain_is_ipi_per_cpu(domain: data->domain) && |
248 | cpu != data->common->ipi_offset) { |
249 | /* use the correct data for that cpu */ |
250 | unsigned irq = data->irq + cpu - data->common->ipi_offset; |
251 | |
252 | data = irq_get_irq_data(irq); |
253 | } |
254 | chip->ipi_send_single(data, cpu); |
255 | return 0; |
256 | } |
257 | |
258 | /** |
259 | * __ipi_send_mask - send an IPI to target Linux SMP CPU(s) |
260 | * @desc: pointer to irq_desc of the IRQ |
261 | * @dest: dest CPU(s), must be a subset of the mask passed to |
262 | * irq_reserve_ipi() |
263 | * |
264 | * This function is for architecture or core code to speed up IPI sending. Not |
265 | * usable from driver code. |
266 | * |
267 | * Return: %0 on success or negative error number on failure. |
268 | */ |
269 | int __ipi_send_mask(struct irq_desc *desc, const struct cpumask *dest) |
270 | { |
271 | struct irq_data *data = irq_desc_get_irq_data(desc); |
272 | struct irq_chip *chip = irq_data_get_irq_chip(d: data); |
273 | unsigned int cpu; |
274 | |
275 | #ifdef DEBUG |
276 | /* |
277 | * Minimise the overhead by omitting the checks for Linux SMP IPIs. |
278 | * Since the callers should be arch or core code which is generally |
279 | * trusted, only check for errors when debugging. |
280 | */ |
281 | if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0))) |
282 | return -EINVAL; |
283 | #endif |
284 | if (chip->ipi_send_mask) { |
285 | chip->ipi_send_mask(data, dest); |
286 | return 0; |
287 | } |
288 | |
289 | if (irq_domain_is_ipi_per_cpu(domain: data->domain)) { |
290 | unsigned int base = data->irq; |
291 | |
292 | for_each_cpu(cpu, dest) { |
293 | unsigned irq = base + cpu - data->common->ipi_offset; |
294 | |
295 | data = irq_get_irq_data(irq); |
296 | chip->ipi_send_single(data, cpu); |
297 | } |
298 | } else { |
299 | for_each_cpu(cpu, dest) |
300 | chip->ipi_send_single(data, cpu); |
301 | } |
302 | return 0; |
303 | } |
304 | |
305 | /** |
306 | * ipi_send_single - Send an IPI to a single CPU |
307 | * @virq: Linux IRQ number from irq_reserve_ipi() |
308 | * @cpu: destination CPU, must in the destination mask passed to |
309 | * irq_reserve_ipi() |
310 | * |
311 | * Return: %0 on success or negative error number on failure. |
312 | */ |
313 | int ipi_send_single(unsigned int virq, unsigned int cpu) |
314 | { |
315 | struct irq_desc *desc = irq_to_desc(irq: virq); |
316 | struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL; |
317 | struct irq_chip *chip = data ? irq_data_get_irq_chip(d: data) : NULL; |
318 | |
319 | if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu))) |
320 | return -EINVAL; |
321 | |
322 | return __ipi_send_single(desc, cpu); |
323 | } |
324 | EXPORT_SYMBOL_GPL(ipi_send_single); |
325 | |
326 | /** |
327 | * ipi_send_mask - Send an IPI to target CPU(s) |
328 | * @virq: Linux IRQ number from irq_reserve_ipi() |
329 | * @dest: dest CPU(s), must be a subset of the mask passed to |
330 | * irq_reserve_ipi() |
331 | * |
332 | * Return: %0 on success or negative error number on failure. |
333 | */ |
334 | int ipi_send_mask(unsigned int virq, const struct cpumask *dest) |
335 | { |
336 | struct irq_desc *desc = irq_to_desc(irq: virq); |
337 | struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL; |
338 | struct irq_chip *chip = data ? irq_data_get_irq_chip(d: data) : NULL; |
339 | |
340 | if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0))) |
341 | return -EINVAL; |
342 | |
343 | return __ipi_send_mask(desc, dest); |
344 | } |
345 | EXPORT_SYMBOL_GPL(ipi_send_mask); |
346 | |