1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * Multiplex several virtual IPIs over a single HW IPI. |
4 | * |
5 | * Copyright The Asahi Linux Contributors |
6 | * Copyright (c) 2022 Ventana Micro Systems Inc. |
7 | */ |
8 | |
9 | #define pr_fmt(fmt) "ipi-mux: " fmt |
10 | #include <linux/cpu.h> |
11 | #include <linux/init.h> |
12 | #include <linux/irq.h> |
13 | #include <linux/irqchip.h> |
14 | #include <linux/irqchip/chained_irq.h> |
15 | #include <linux/irqdomain.h> |
16 | #include <linux/jump_label.h> |
17 | #include <linux/percpu.h> |
18 | #include <linux/smp.h> |
19 | |
20 | struct ipi_mux_cpu { |
21 | atomic_t enable; |
22 | atomic_t bits; |
23 | }; |
24 | |
25 | static struct ipi_mux_cpu __percpu *ipi_mux_pcpu; |
26 | static struct irq_domain *ipi_mux_domain; |
27 | static void (*ipi_mux_send)(unsigned int cpu); |
28 | |
29 | static void ipi_mux_mask(struct irq_data *d) |
30 | { |
31 | struct ipi_mux_cpu *icpu = this_cpu_ptr(ipi_mux_pcpu); |
32 | |
33 | atomic_andnot(BIT(irqd_to_hwirq(d)), v: &icpu->enable); |
34 | } |
35 | |
36 | static void ipi_mux_unmask(struct irq_data *d) |
37 | { |
38 | struct ipi_mux_cpu *icpu = this_cpu_ptr(ipi_mux_pcpu); |
39 | u32 ibit = BIT(irqd_to_hwirq(d)); |
40 | |
41 | atomic_or(i: ibit, v: &icpu->enable); |
42 | |
43 | /* |
44 | * The atomic_or() above must complete before the atomic_read() |
45 | * below to avoid racing ipi_mux_send_mask(). |
46 | */ |
47 | smp_mb__after_atomic(); |
48 | |
49 | /* If a pending IPI was unmasked, raise a parent IPI immediately. */ |
50 | if (atomic_read(v: &icpu->bits) & ibit) |
51 | ipi_mux_send(smp_processor_id()); |
52 | } |
53 | |
54 | static void ipi_mux_send_mask(struct irq_data *d, const struct cpumask *mask) |
55 | { |
56 | struct ipi_mux_cpu *icpu = this_cpu_ptr(ipi_mux_pcpu); |
57 | u32 ibit = BIT(irqd_to_hwirq(d)); |
58 | unsigned long pending; |
59 | int cpu; |
60 | |
61 | for_each_cpu(cpu, mask) { |
62 | icpu = per_cpu_ptr(ipi_mux_pcpu, cpu); |
63 | |
64 | /* |
65 | * This sequence is the mirror of the one in ipi_mux_unmask(); |
66 | * see the comment there. Additionally, release semantics |
67 | * ensure that the vIPI flag set is ordered after any shared |
68 | * memory accesses that precede it. This therefore also pairs |
69 | * with the atomic_fetch_andnot in ipi_mux_process(). |
70 | */ |
71 | pending = atomic_fetch_or_release(i: ibit, v: &icpu->bits); |
72 | |
73 | /* |
74 | * The atomic_fetch_or_release() above must complete |
75 | * before the atomic_read() below to avoid racing with |
76 | * ipi_mux_unmask(). |
77 | */ |
78 | smp_mb__after_atomic(); |
79 | |
80 | /* |
81 | * The flag writes must complete before the physical IPI is |
82 | * issued to another CPU. This is implied by the control |
83 | * dependency on the result of atomic_read() below, which is |
84 | * itself already ordered after the vIPI flag write. |
85 | */ |
86 | if (!(pending & ibit) && (atomic_read(v: &icpu->enable) & ibit)) |
87 | ipi_mux_send(cpu); |
88 | } |
89 | } |
90 | |
91 | static const struct irq_chip ipi_mux_chip = { |
92 | .name = "IPI Mux" , |
93 | .irq_mask = ipi_mux_mask, |
94 | .irq_unmask = ipi_mux_unmask, |
95 | .ipi_send_mask = ipi_mux_send_mask, |
96 | }; |
97 | |
98 | static int ipi_mux_domain_alloc(struct irq_domain *d, unsigned int virq, |
99 | unsigned int nr_irqs, void *arg) |
100 | { |
101 | int i; |
102 | |
103 | for (i = 0; i < nr_irqs; i++) { |
104 | irq_set_percpu_devid(irq: virq + i); |
105 | irq_domain_set_info(domain: d, virq: virq + i, hwirq: i, chip: &ipi_mux_chip, NULL, |
106 | handler: handle_percpu_devid_irq, NULL, NULL); |
107 | } |
108 | |
109 | return 0; |
110 | } |
111 | |
112 | static const struct irq_domain_ops ipi_mux_domain_ops = { |
113 | .alloc = ipi_mux_domain_alloc, |
114 | .free = irq_domain_free_irqs_top, |
115 | }; |
116 | |
117 | /** |
118 | * ipi_mux_process - Process multiplexed virtual IPIs |
119 | */ |
120 | void ipi_mux_process(void) |
121 | { |
122 | struct ipi_mux_cpu *icpu = this_cpu_ptr(ipi_mux_pcpu); |
123 | irq_hw_number_t hwirq; |
124 | unsigned long ipis; |
125 | unsigned int en; |
126 | |
127 | /* |
128 | * Reading enable mask does not need to be ordered as long as |
129 | * this function is called from interrupt handler because only |
130 | * the CPU itself can change it's own enable mask. |
131 | */ |
132 | en = atomic_read(v: &icpu->enable); |
133 | |
134 | /* |
135 | * Clear the IPIs we are about to handle. This pairs with the |
136 | * atomic_fetch_or_release() in ipi_mux_send_mask(). |
137 | */ |
138 | ipis = atomic_fetch_andnot(i: en, v: &icpu->bits) & en; |
139 | |
140 | for_each_set_bit(hwirq, &ipis, BITS_PER_TYPE(int)) |
141 | generic_handle_domain_irq(domain: ipi_mux_domain, hwirq); |
142 | } |
143 | |
144 | /** |
145 | * ipi_mux_create - Create virtual IPIs multiplexed on top of a single |
146 | * parent IPI. |
147 | * @nr_ipi: number of virtual IPIs to create. This should |
148 | * be <= BITS_PER_TYPE(int) |
149 | * @mux_send: callback to trigger parent IPI for a particular CPU |
150 | * |
151 | * Returns first virq of the newly created virtual IPIs upon success |
152 | * or <=0 upon failure |
153 | */ |
154 | int ipi_mux_create(unsigned int nr_ipi, void (*mux_send)(unsigned int cpu)) |
155 | { |
156 | struct fwnode_handle *fwnode; |
157 | struct irq_domain *domain; |
158 | int rc; |
159 | |
160 | if (ipi_mux_domain) |
161 | return -EEXIST; |
162 | |
163 | if (BITS_PER_TYPE(int) < nr_ipi || !mux_send) |
164 | return -EINVAL; |
165 | |
166 | ipi_mux_pcpu = alloc_percpu(typeof(*ipi_mux_pcpu)); |
167 | if (!ipi_mux_pcpu) |
168 | return -ENOMEM; |
169 | |
170 | fwnode = irq_domain_alloc_named_fwnode(name: "IPI-Mux" ); |
171 | if (!fwnode) { |
172 | pr_err("unable to create IPI Mux fwnode\n" ); |
173 | rc = -ENOMEM; |
174 | goto fail_free_cpu; |
175 | } |
176 | |
177 | domain = irq_domain_create_linear(fwnode, size: nr_ipi, |
178 | ops: &ipi_mux_domain_ops, NULL); |
179 | if (!domain) { |
180 | pr_err("unable to add IPI Mux domain\n" ); |
181 | rc = -ENOMEM; |
182 | goto fail_free_fwnode; |
183 | } |
184 | |
185 | domain->flags |= IRQ_DOMAIN_FLAG_IPI_SINGLE; |
186 | irq_domain_update_bus_token(domain, bus_token: DOMAIN_BUS_IPI); |
187 | |
188 | rc = irq_domain_alloc_irqs(domain, nr_irqs: nr_ipi, NUMA_NO_NODE, NULL); |
189 | if (rc <= 0) { |
190 | pr_err("unable to alloc IRQs from IPI Mux domain\n" ); |
191 | goto fail_free_domain; |
192 | } |
193 | |
194 | ipi_mux_domain = domain; |
195 | ipi_mux_send = mux_send; |
196 | |
197 | return rc; |
198 | |
199 | fail_free_domain: |
200 | irq_domain_remove(host: domain); |
201 | fail_free_fwnode: |
202 | irq_domain_free_fwnode(fwnode); |
203 | fail_free_cpu: |
204 | free_percpu(pdata: ipi_mux_pcpu); |
205 | return rc; |
206 | } |
207 | |