1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Support for adapter interruptions
4 *
5 * Copyright IBM Corp. 1999, 2007
6 * Author(s): Ingo Adlung <adlung@de.ibm.com>
7 * Cornelia Huck <cornelia.huck@de.ibm.com>
8 * Arnd Bergmann <arndb@de.ibm.com>
9 * Peter Oberparleiter <peter.oberparleiter@de.ibm.com>
10 */
11
12#include <linux/init.h>
13#include <linux/irq.h>
14#include <linux/kernel_stat.h>
15#include <linux/module.h>
16#include <linux/mutex.h>
17#include <linux/rculist.h>
18#include <linux/slab.h>
19#include <linux/dmapool.h>
20
21#include <asm/airq.h>
22#include <asm/isc.h>
23#include <asm/cio.h>
24
25#include "cio.h"
26#include "cio_debug.h"
27#include "ioasm.h"
28
29static DEFINE_SPINLOCK(airq_lists_lock);
30static struct hlist_head airq_lists[MAX_ISC+1];
31
32static struct dma_pool *airq_iv_cache;
33
34/**
35 * register_adapter_interrupt() - register adapter interrupt handler
36 * @airq: pointer to adapter interrupt descriptor
37 *
38 * Returns 0 on success, or -EINVAL.
39 */
40int register_adapter_interrupt(struct airq_struct *airq)
41{
42 char dbf_txt[32];
43
44 if (!airq->handler || airq->isc > MAX_ISC)
45 return -EINVAL;
46 if (!airq->lsi_ptr) {
47 airq->lsi_ptr = cio_dma_zalloc(1);
48 if (!airq->lsi_ptr)
49 return -ENOMEM;
50 airq->flags |= AIRQ_PTR_ALLOCATED;
51 }
52 snprintf(buf: dbf_txt, size: sizeof(dbf_txt), fmt: "rairq:%p", airq);
53 CIO_TRACE_EVENT(4, dbf_txt);
54 isc_register(airq->isc);
55 spin_lock(lock: &airq_lists_lock);
56 hlist_add_head_rcu(n: &airq->list, h: &airq_lists[airq->isc]);
57 spin_unlock(lock: &airq_lists_lock);
58 return 0;
59}
60EXPORT_SYMBOL(register_adapter_interrupt);
61
62/**
63 * unregister_adapter_interrupt - unregister adapter interrupt handler
64 * @airq: pointer to adapter interrupt descriptor
65 */
66void unregister_adapter_interrupt(struct airq_struct *airq)
67{
68 char dbf_txt[32];
69
70 if (hlist_unhashed(h: &airq->list))
71 return;
72 snprintf(buf: dbf_txt, size: sizeof(dbf_txt), fmt: "urairq:%p", airq);
73 CIO_TRACE_EVENT(4, dbf_txt);
74 spin_lock(lock: &airq_lists_lock);
75 hlist_del_rcu(n: &airq->list);
76 spin_unlock(lock: &airq_lists_lock);
77 synchronize_rcu();
78 isc_unregister(airq->isc);
79 if (airq->flags & AIRQ_PTR_ALLOCATED) {
80 cio_dma_free(airq->lsi_ptr, 1);
81 airq->lsi_ptr = NULL;
82 airq->flags &= ~AIRQ_PTR_ALLOCATED;
83 }
84}
85EXPORT_SYMBOL(unregister_adapter_interrupt);
86
87static irqreturn_t do_airq_interrupt(int irq, void *dummy)
88{
89 struct tpi_info *tpi_info;
90 struct airq_struct *airq;
91 struct hlist_head *head;
92
93 set_cpu_flag(CIF_NOHZ_DELAY);
94 tpi_info = &get_irq_regs()->tpi_info;
95 trace_s390_cio_adapter_int(tpi_info);
96 head = &airq_lists[tpi_info->isc];
97 rcu_read_lock();
98 hlist_for_each_entry_rcu(airq, head, list)
99 if (*airq->lsi_ptr != 0)
100 airq->handler(airq, tpi_info);
101 rcu_read_unlock();
102
103 return IRQ_HANDLED;
104}
105
106void __init init_airq_interrupts(void)
107{
108 irq_set_chip_and_handler(irq: THIN_INTERRUPT,
109 chip: &dummy_irq_chip, handle: handle_percpu_irq);
110 if (request_irq(irq: THIN_INTERRUPT, handler: do_airq_interrupt, flags: 0, name: "AIO", NULL))
111 panic(fmt: "Failed to register AIO interrupt\n");
112}
113
114static inline unsigned long iv_size(unsigned long bits)
115{
116 return BITS_TO_LONGS(bits) * sizeof(unsigned long);
117}
118
119/**
120 * airq_iv_create - create an interrupt vector
121 * @bits: number of bits in the interrupt vector
122 * @flags: allocation flags
123 * @vec: pointer to pinned guest memory if AIRQ_IV_GUESTVEC
124 *
125 * Returns a pointer to an interrupt vector structure
126 */
127struct airq_iv *airq_iv_create(unsigned long bits, unsigned long flags,
128 unsigned long *vec)
129{
130 struct airq_iv *iv;
131 unsigned long size;
132
133 iv = kzalloc(sizeof(*iv), GFP_KERNEL);
134 if (!iv)
135 goto out;
136 iv->bits = bits;
137 iv->flags = flags;
138 size = iv_size(bits);
139
140 if (flags & AIRQ_IV_CACHELINE) {
141 if ((cache_line_size() * BITS_PER_BYTE) < bits
142 || !airq_iv_cache)
143 goto out_free;
144
145 iv->vector = dma_pool_zalloc(pool: airq_iv_cache, GFP_KERNEL,
146 handle: &iv->vector_dma);
147 if (!iv->vector)
148 goto out_free;
149 } else if (flags & AIRQ_IV_GUESTVEC) {
150 iv->vector = vec;
151 } else {
152 iv->vector = cio_dma_zalloc(size);
153 if (!iv->vector)
154 goto out_free;
155 }
156 if (flags & AIRQ_IV_ALLOC) {
157 iv->avail = kmalloc(size, GFP_KERNEL);
158 if (!iv->avail)
159 goto out_free;
160 memset(iv->avail, 0xff, size);
161 iv->end = 0;
162 } else
163 iv->end = bits;
164 if (flags & AIRQ_IV_BITLOCK) {
165 iv->bitlock = kzalloc(size, GFP_KERNEL);
166 if (!iv->bitlock)
167 goto out_free;
168 }
169 if (flags & AIRQ_IV_PTR) {
170 size = bits * sizeof(unsigned long);
171 iv->ptr = kzalloc(size, GFP_KERNEL);
172 if (!iv->ptr)
173 goto out_free;
174 }
175 if (flags & AIRQ_IV_DATA) {
176 size = bits * sizeof(unsigned int);
177 iv->data = kzalloc(size, GFP_KERNEL);
178 if (!iv->data)
179 goto out_free;
180 }
181 spin_lock_init(&iv->lock);
182 return iv;
183
184out_free:
185 kfree(objp: iv->ptr);
186 kfree(objp: iv->bitlock);
187 kfree(objp: iv->avail);
188 if (iv->flags & AIRQ_IV_CACHELINE && iv->vector)
189 dma_pool_free(pool: airq_iv_cache, vaddr: iv->vector, addr: iv->vector_dma);
190 else if (!(iv->flags & AIRQ_IV_GUESTVEC))
191 cio_dma_free(iv->vector, size);
192 kfree(objp: iv);
193out:
194 return NULL;
195}
196EXPORT_SYMBOL(airq_iv_create);
197
198/**
199 * airq_iv_release - release an interrupt vector
200 * @iv: pointer to interrupt vector structure
201 */
202void airq_iv_release(struct airq_iv *iv)
203{
204 kfree(objp: iv->data);
205 kfree(objp: iv->ptr);
206 kfree(objp: iv->bitlock);
207 if (iv->flags & AIRQ_IV_CACHELINE)
208 dma_pool_free(pool: airq_iv_cache, vaddr: iv->vector, addr: iv->vector_dma);
209 else if (!(iv->flags & AIRQ_IV_GUESTVEC))
210 cio_dma_free(iv->vector, iv_size(bits: iv->bits));
211 kfree(objp: iv->avail);
212 kfree(objp: iv);
213}
214EXPORT_SYMBOL(airq_iv_release);
215
216/**
217 * airq_iv_alloc - allocate irq bits from an interrupt vector
218 * @iv: pointer to an interrupt vector structure
219 * @num: number of consecutive irq bits to allocate
220 *
221 * Returns the bit number of the first irq in the allocated block of irqs,
222 * or -1UL if no bit is available or the AIRQ_IV_ALLOC flag has not been
223 * specified
224 */
225unsigned long airq_iv_alloc(struct airq_iv *iv, unsigned long num)
226{
227 unsigned long bit, i, flags;
228
229 if (!iv->avail || num == 0)
230 return -1UL;
231 spin_lock_irqsave(&iv->lock, flags);
232 bit = find_first_bit_inv(iv->avail, iv->bits);
233 while (bit + num <= iv->bits) {
234 for (i = 1; i < num; i++)
235 if (!test_bit_inv(bit + i, iv->avail))
236 break;
237 if (i >= num) {
238 /* Found a suitable block of irqs */
239 for (i = 0; i < num; i++)
240 clear_bit_inv(bit + i, iv->avail);
241 if (bit + num >= iv->end)
242 iv->end = bit + num + 1;
243 break;
244 }
245 bit = find_next_bit_inv(iv->avail, iv->bits, bit + i + 1);
246 }
247 if (bit + num > iv->bits)
248 bit = -1UL;
249 spin_unlock_irqrestore(lock: &iv->lock, flags);
250 return bit;
251}
252EXPORT_SYMBOL(airq_iv_alloc);
253
254/**
255 * airq_iv_free - free irq bits of an interrupt vector
256 * @iv: pointer to interrupt vector structure
257 * @bit: number of the first irq bit to free
258 * @num: number of consecutive irq bits to free
259 */
260void airq_iv_free(struct airq_iv *iv, unsigned long bit, unsigned long num)
261{
262 unsigned long i, flags;
263
264 if (!iv->avail || num == 0)
265 return;
266 spin_lock_irqsave(&iv->lock, flags);
267 for (i = 0; i < num; i++) {
268 /* Clear (possibly left over) interrupt bit */
269 clear_bit_inv(bit + i, iv->vector);
270 /* Make the bit positions available again */
271 set_bit_inv(bit + i, iv->avail);
272 }
273 if (bit + num >= iv->end) {
274 /* Find new end of bit-field */
275 while (iv->end > 0 && !test_bit_inv(iv->end - 1, iv->avail))
276 iv->end--;
277 }
278 spin_unlock_irqrestore(lock: &iv->lock, flags);
279}
280EXPORT_SYMBOL(airq_iv_free);
281
282/**
283 * airq_iv_scan - scan interrupt vector for non-zero bits
284 * @iv: pointer to interrupt vector structure
285 * @start: bit number to start the search
286 * @end: bit number to end the search
287 *
288 * Returns the bit number of the next non-zero interrupt bit, or
289 * -1UL if the scan completed without finding any more any non-zero bits.
290 */
291unsigned long airq_iv_scan(struct airq_iv *iv, unsigned long start,
292 unsigned long end)
293{
294 unsigned long bit;
295
296 /* Find non-zero bit starting from 'ivs->next'. */
297 bit = find_next_bit_inv(iv->vector, end, start);
298 if (bit >= end)
299 return -1UL;
300 clear_bit_inv(bit, iv->vector);
301 return bit;
302}
303EXPORT_SYMBOL(airq_iv_scan);
304
305int __init airq_init(void)
306{
307 airq_iv_cache = dma_pool_create(name: "airq_iv_cache", dev: cio_get_dma_css_dev(),
308 cache_line_size(),
309 cache_line_size(), PAGE_SIZE);
310 if (!airq_iv_cache)
311 return -ENOMEM;
312 return 0;
313}
314

source code of linux/drivers/s390/cio/airq.c