1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Support for adapter interruptions |
4 | * |
5 | * Copyright IBM Corp. 1999, 2007 |
6 | * Author(s): Ingo Adlung <adlung@de.ibm.com> |
7 | * Cornelia Huck <cornelia.huck@de.ibm.com> |
8 | * Arnd Bergmann <arndb@de.ibm.com> |
9 | * Peter Oberparleiter <peter.oberparleiter@de.ibm.com> |
10 | */ |
11 | |
12 | #include <linux/init.h> |
13 | #include <linux/irq.h> |
14 | #include <linux/kernel_stat.h> |
15 | #include <linux/module.h> |
16 | #include <linux/mutex.h> |
17 | #include <linux/rculist.h> |
18 | #include <linux/slab.h> |
19 | #include <linux/dmapool.h> |
20 | |
21 | #include <asm/airq.h> |
22 | #include <asm/isc.h> |
23 | #include <asm/cio.h> |
24 | |
25 | #include "cio.h" |
26 | #include "cio_debug.h" |
27 | #include "ioasm.h" |
28 | |
29 | static DEFINE_SPINLOCK(airq_lists_lock); |
30 | static struct hlist_head airq_lists[MAX_ISC+1]; |
31 | |
32 | static struct dma_pool *airq_iv_cache; |
33 | |
34 | /** |
35 | * register_adapter_interrupt() - register adapter interrupt handler |
36 | * @airq: pointer to adapter interrupt descriptor |
37 | * |
38 | * Returns 0 on success, or -EINVAL. |
39 | */ |
40 | int register_adapter_interrupt(struct airq_struct *airq) |
41 | { |
42 | char dbf_txt[32]; |
43 | |
44 | if (!airq->handler || airq->isc > MAX_ISC) |
45 | return -EINVAL; |
46 | if (!airq->lsi_ptr) { |
47 | airq->lsi_ptr = cio_dma_zalloc(1); |
48 | if (!airq->lsi_ptr) |
49 | return -ENOMEM; |
50 | airq->flags |= AIRQ_PTR_ALLOCATED; |
51 | } |
52 | snprintf(buf: dbf_txt, size: sizeof(dbf_txt), fmt: "rairq:%p" , airq); |
53 | CIO_TRACE_EVENT(4, dbf_txt); |
54 | isc_register(airq->isc); |
55 | spin_lock(lock: &airq_lists_lock); |
56 | hlist_add_head_rcu(n: &airq->list, h: &airq_lists[airq->isc]); |
57 | spin_unlock(lock: &airq_lists_lock); |
58 | return 0; |
59 | } |
60 | EXPORT_SYMBOL(register_adapter_interrupt); |
61 | |
62 | /** |
63 | * unregister_adapter_interrupt - unregister adapter interrupt handler |
64 | * @airq: pointer to adapter interrupt descriptor |
65 | */ |
66 | void unregister_adapter_interrupt(struct airq_struct *airq) |
67 | { |
68 | char dbf_txt[32]; |
69 | |
70 | if (hlist_unhashed(h: &airq->list)) |
71 | return; |
72 | snprintf(buf: dbf_txt, size: sizeof(dbf_txt), fmt: "urairq:%p" , airq); |
73 | CIO_TRACE_EVENT(4, dbf_txt); |
74 | spin_lock(lock: &airq_lists_lock); |
75 | hlist_del_rcu(n: &airq->list); |
76 | spin_unlock(lock: &airq_lists_lock); |
77 | synchronize_rcu(); |
78 | isc_unregister(airq->isc); |
79 | if (airq->flags & AIRQ_PTR_ALLOCATED) { |
80 | cio_dma_free(airq->lsi_ptr, 1); |
81 | airq->lsi_ptr = NULL; |
82 | airq->flags &= ~AIRQ_PTR_ALLOCATED; |
83 | } |
84 | } |
85 | EXPORT_SYMBOL(unregister_adapter_interrupt); |
86 | |
87 | static irqreturn_t do_airq_interrupt(int irq, void *dummy) |
88 | { |
89 | struct tpi_info *tpi_info; |
90 | struct airq_struct *airq; |
91 | struct hlist_head *head; |
92 | |
93 | set_cpu_flag(CIF_NOHZ_DELAY); |
94 | tpi_info = &get_irq_regs()->tpi_info; |
95 | trace_s390_cio_adapter_int(tpi_info); |
96 | head = &airq_lists[tpi_info->isc]; |
97 | rcu_read_lock(); |
98 | hlist_for_each_entry_rcu(airq, head, list) |
99 | if (*airq->lsi_ptr != 0) |
100 | airq->handler(airq, tpi_info); |
101 | rcu_read_unlock(); |
102 | |
103 | return IRQ_HANDLED; |
104 | } |
105 | |
106 | void __init init_airq_interrupts(void) |
107 | { |
108 | irq_set_chip_and_handler(irq: THIN_INTERRUPT, |
109 | chip: &dummy_irq_chip, handle: handle_percpu_irq); |
110 | if (request_irq(irq: THIN_INTERRUPT, handler: do_airq_interrupt, flags: 0, name: "AIO" , NULL)) |
111 | panic(fmt: "Failed to register AIO interrupt\n" ); |
112 | } |
113 | |
114 | static inline unsigned long iv_size(unsigned long bits) |
115 | { |
116 | return BITS_TO_LONGS(bits) * sizeof(unsigned long); |
117 | } |
118 | |
119 | /** |
120 | * airq_iv_create - create an interrupt vector |
121 | * @bits: number of bits in the interrupt vector |
122 | * @flags: allocation flags |
123 | * @vec: pointer to pinned guest memory if AIRQ_IV_GUESTVEC |
124 | * |
125 | * Returns a pointer to an interrupt vector structure |
126 | */ |
127 | struct airq_iv *airq_iv_create(unsigned long bits, unsigned long flags, |
128 | unsigned long *vec) |
129 | { |
130 | struct airq_iv *iv; |
131 | unsigned long size; |
132 | |
133 | iv = kzalloc(sizeof(*iv), GFP_KERNEL); |
134 | if (!iv) |
135 | goto out; |
136 | iv->bits = bits; |
137 | iv->flags = flags; |
138 | size = iv_size(bits); |
139 | |
140 | if (flags & AIRQ_IV_CACHELINE) { |
141 | if ((cache_line_size() * BITS_PER_BYTE) < bits |
142 | || !airq_iv_cache) |
143 | goto out_free; |
144 | |
145 | iv->vector = dma_pool_zalloc(pool: airq_iv_cache, GFP_KERNEL, |
146 | handle: &iv->vector_dma); |
147 | if (!iv->vector) |
148 | goto out_free; |
149 | } else if (flags & AIRQ_IV_GUESTVEC) { |
150 | iv->vector = vec; |
151 | } else { |
152 | iv->vector = cio_dma_zalloc(size); |
153 | if (!iv->vector) |
154 | goto out_free; |
155 | } |
156 | if (flags & AIRQ_IV_ALLOC) { |
157 | iv->avail = kmalloc(size, GFP_KERNEL); |
158 | if (!iv->avail) |
159 | goto out_free; |
160 | memset(iv->avail, 0xff, size); |
161 | iv->end = 0; |
162 | } else |
163 | iv->end = bits; |
164 | if (flags & AIRQ_IV_BITLOCK) { |
165 | iv->bitlock = kzalloc(size, GFP_KERNEL); |
166 | if (!iv->bitlock) |
167 | goto out_free; |
168 | } |
169 | if (flags & AIRQ_IV_PTR) { |
170 | size = bits * sizeof(unsigned long); |
171 | iv->ptr = kzalloc(size, GFP_KERNEL); |
172 | if (!iv->ptr) |
173 | goto out_free; |
174 | } |
175 | if (flags & AIRQ_IV_DATA) { |
176 | size = bits * sizeof(unsigned int); |
177 | iv->data = kzalloc(size, GFP_KERNEL); |
178 | if (!iv->data) |
179 | goto out_free; |
180 | } |
181 | spin_lock_init(&iv->lock); |
182 | return iv; |
183 | |
184 | out_free: |
185 | kfree(objp: iv->ptr); |
186 | kfree(objp: iv->bitlock); |
187 | kfree(objp: iv->avail); |
188 | if (iv->flags & AIRQ_IV_CACHELINE && iv->vector) |
189 | dma_pool_free(pool: airq_iv_cache, vaddr: iv->vector, addr: iv->vector_dma); |
190 | else if (!(iv->flags & AIRQ_IV_GUESTVEC)) |
191 | cio_dma_free(iv->vector, size); |
192 | kfree(objp: iv); |
193 | out: |
194 | return NULL; |
195 | } |
196 | EXPORT_SYMBOL(airq_iv_create); |
197 | |
198 | /** |
199 | * airq_iv_release - release an interrupt vector |
200 | * @iv: pointer to interrupt vector structure |
201 | */ |
202 | void airq_iv_release(struct airq_iv *iv) |
203 | { |
204 | kfree(objp: iv->data); |
205 | kfree(objp: iv->ptr); |
206 | kfree(objp: iv->bitlock); |
207 | if (iv->flags & AIRQ_IV_CACHELINE) |
208 | dma_pool_free(pool: airq_iv_cache, vaddr: iv->vector, addr: iv->vector_dma); |
209 | else if (!(iv->flags & AIRQ_IV_GUESTVEC)) |
210 | cio_dma_free(iv->vector, iv_size(bits: iv->bits)); |
211 | kfree(objp: iv->avail); |
212 | kfree(objp: iv); |
213 | } |
214 | EXPORT_SYMBOL(airq_iv_release); |
215 | |
216 | /** |
217 | * airq_iv_alloc - allocate irq bits from an interrupt vector |
218 | * @iv: pointer to an interrupt vector structure |
219 | * @num: number of consecutive irq bits to allocate |
220 | * |
221 | * Returns the bit number of the first irq in the allocated block of irqs, |
222 | * or -1UL if no bit is available or the AIRQ_IV_ALLOC flag has not been |
223 | * specified |
224 | */ |
225 | unsigned long airq_iv_alloc(struct airq_iv *iv, unsigned long num) |
226 | { |
227 | unsigned long bit, i, flags; |
228 | |
229 | if (!iv->avail || num == 0) |
230 | return -1UL; |
231 | spin_lock_irqsave(&iv->lock, flags); |
232 | bit = find_first_bit_inv(iv->avail, iv->bits); |
233 | while (bit + num <= iv->bits) { |
234 | for (i = 1; i < num; i++) |
235 | if (!test_bit_inv(bit + i, iv->avail)) |
236 | break; |
237 | if (i >= num) { |
238 | /* Found a suitable block of irqs */ |
239 | for (i = 0; i < num; i++) |
240 | clear_bit_inv(bit + i, iv->avail); |
241 | if (bit + num >= iv->end) |
242 | iv->end = bit + num + 1; |
243 | break; |
244 | } |
245 | bit = find_next_bit_inv(iv->avail, iv->bits, bit + i + 1); |
246 | } |
247 | if (bit + num > iv->bits) |
248 | bit = -1UL; |
249 | spin_unlock_irqrestore(lock: &iv->lock, flags); |
250 | return bit; |
251 | } |
252 | EXPORT_SYMBOL(airq_iv_alloc); |
253 | |
254 | /** |
255 | * airq_iv_free - free irq bits of an interrupt vector |
256 | * @iv: pointer to interrupt vector structure |
257 | * @bit: number of the first irq bit to free |
258 | * @num: number of consecutive irq bits to free |
259 | */ |
260 | void airq_iv_free(struct airq_iv *iv, unsigned long bit, unsigned long num) |
261 | { |
262 | unsigned long i, flags; |
263 | |
264 | if (!iv->avail || num == 0) |
265 | return; |
266 | spin_lock_irqsave(&iv->lock, flags); |
267 | for (i = 0; i < num; i++) { |
268 | /* Clear (possibly left over) interrupt bit */ |
269 | clear_bit_inv(bit + i, iv->vector); |
270 | /* Make the bit positions available again */ |
271 | set_bit_inv(bit + i, iv->avail); |
272 | } |
273 | if (bit + num >= iv->end) { |
274 | /* Find new end of bit-field */ |
275 | while (iv->end > 0 && !test_bit_inv(iv->end - 1, iv->avail)) |
276 | iv->end--; |
277 | } |
278 | spin_unlock_irqrestore(lock: &iv->lock, flags); |
279 | } |
280 | EXPORT_SYMBOL(airq_iv_free); |
281 | |
282 | /** |
283 | * airq_iv_scan - scan interrupt vector for non-zero bits |
284 | * @iv: pointer to interrupt vector structure |
285 | * @start: bit number to start the search |
286 | * @end: bit number to end the search |
287 | * |
288 | * Returns the bit number of the next non-zero interrupt bit, or |
289 | * -1UL if the scan completed without finding any more any non-zero bits. |
290 | */ |
291 | unsigned long airq_iv_scan(struct airq_iv *iv, unsigned long start, |
292 | unsigned long end) |
293 | { |
294 | unsigned long bit; |
295 | |
296 | /* Find non-zero bit starting from 'ivs->next'. */ |
297 | bit = find_next_bit_inv(iv->vector, end, start); |
298 | if (bit >= end) |
299 | return -1UL; |
300 | clear_bit_inv(bit, iv->vector); |
301 | return bit; |
302 | } |
303 | EXPORT_SYMBOL(airq_iv_scan); |
304 | |
305 | int __init airq_init(void) |
306 | { |
307 | airq_iv_cache = dma_pool_create(name: "airq_iv_cache" , dev: cio_get_dma_css_dev(), |
308 | cache_line_size(), |
309 | cache_line_size(), PAGE_SIZE); |
310 | if (!airq_iv_cache) |
311 | return -ENOMEM; |
312 | return 0; |
313 | } |
314 | |