1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright (C) 2017 - Cambridge Greys Ltd |
4 | * Copyright (C) 2011 - 2014 Cisco Systems Inc |
5 | * Copyright (C) 2000 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com) |
6 | * Derived (i.e. mostly copied) from arch/i386/kernel/irq.c: |
7 | * Copyright (C) 1992, 1998 Linus Torvalds, Ingo Molnar |
8 | */ |
9 | |
10 | #include <linux/cpumask.h> |
11 | #include <linux/hardirq.h> |
12 | #include <linux/interrupt.h> |
13 | #include <linux/kernel_stat.h> |
14 | #include <linux/module.h> |
15 | #include <linux/sched.h> |
16 | #include <linux/seq_file.h> |
17 | #include <linux/slab.h> |
18 | #include <as-layout.h> |
19 | #include <kern_util.h> |
20 | #include <os.h> |
21 | #include <irq_user.h> |
22 | #include <irq_kern.h> |
23 | #include <linux/time-internal.h> |
24 | |
25 | |
26 | /* When epoll triggers we do not know why it did so |
27 | * we can also have different IRQs for read and write. |
28 | * This is why we keep a small irq_reg array for each fd - |
29 | * one entry per IRQ type |
30 | */ |
31 | struct irq_reg { |
32 | void *id; |
33 | int irq; |
34 | /* it's cheaper to store this than to query it */ |
35 | int events; |
36 | bool active; |
37 | bool pending; |
38 | bool wakeup; |
39 | #ifdef CONFIG_UML_TIME_TRAVEL_SUPPORT |
40 | bool pending_on_resume; |
41 | void (*timetravel_handler)(int, int, void *, |
42 | struct time_travel_event *); |
43 | struct time_travel_event event; |
44 | #endif |
45 | }; |
46 | |
47 | struct irq_entry { |
48 | struct list_head list; |
49 | int fd; |
50 | struct irq_reg reg[NUM_IRQ_TYPES]; |
51 | bool suspended; |
52 | bool sigio_workaround; |
53 | }; |
54 | |
55 | static DEFINE_SPINLOCK(irq_lock); |
56 | static LIST_HEAD(active_fds); |
57 | static DECLARE_BITMAP(irqs_allocated, UM_LAST_SIGNAL_IRQ); |
58 | static bool irqs_suspended; |
59 | |
60 | static void irq_io_loop(struct irq_reg *irq, struct uml_pt_regs *regs) |
61 | { |
62 | /* |
63 | * irq->active guards against reentry |
64 | * irq->pending accumulates pending requests |
65 | * if pending is raised the irq_handler is re-run |
66 | * until pending is cleared |
67 | */ |
68 | if (irq->active) { |
69 | irq->active = false; |
70 | |
71 | do { |
72 | irq->pending = false; |
73 | do_IRQ(irq->irq, regs); |
74 | } while (irq->pending); |
75 | |
76 | irq->active = true; |
77 | } else { |
78 | irq->pending = true; |
79 | } |
80 | } |
81 | |
82 | #ifdef CONFIG_UML_TIME_TRAVEL_SUPPORT |
83 | static void irq_event_handler(struct time_travel_event *ev) |
84 | { |
85 | struct irq_reg *reg = container_of(ev, struct irq_reg, event); |
86 | |
87 | /* do nothing if suspended - just to cause a wakeup */ |
88 | if (irqs_suspended) |
89 | return; |
90 | |
91 | generic_handle_irq(reg->irq); |
92 | } |
93 | |
94 | static bool irq_do_timetravel_handler(struct irq_entry *entry, |
95 | enum um_irq_type t) |
96 | { |
97 | struct irq_reg *reg = &entry->reg[t]; |
98 | |
99 | if (!reg->timetravel_handler) |
100 | return false; |
101 | |
102 | /* |
103 | * Handle all messages - we might get multiple even while |
104 | * interrupts are already suspended, due to suspend order |
105 | * etc. Note that time_travel_add_irq_event() will not add |
106 | * an event twice, if it's pending already "first wins". |
107 | */ |
108 | reg->timetravel_handler(reg->irq, entry->fd, reg->id, ®->event); |
109 | |
110 | if (!reg->event.pending) |
111 | return false; |
112 | |
113 | if (irqs_suspended) |
114 | reg->pending_on_resume = true; |
115 | return true; |
116 | } |
117 | #else |
118 | static bool irq_do_timetravel_handler(struct irq_entry *entry, |
119 | enum um_irq_type t) |
120 | { |
121 | return false; |
122 | } |
123 | #endif |
124 | |
125 | static void sigio_reg_handler(int idx, struct irq_entry *entry, enum um_irq_type t, |
126 | struct uml_pt_regs *regs, |
127 | bool timetravel_handlers_only) |
128 | { |
129 | struct irq_reg *reg = &entry->reg[t]; |
130 | |
131 | if (!reg->events) |
132 | return; |
133 | |
134 | if (os_epoll_triggered(idx, reg->events) <= 0) |
135 | return; |
136 | |
137 | if (irq_do_timetravel_handler(entry, t: t)) |
138 | return; |
139 | |
140 | /* |
141 | * If we're called to only run time-travel handlers then don't |
142 | * actually proceed but mark sigio as pending (if applicable). |
143 | * For suspend/resume, timetravel_handlers_only may be true |
144 | * despite time-travel not being configured and used. |
145 | */ |
146 | if (timetravel_handlers_only) { |
147 | #ifdef CONFIG_UML_TIME_TRAVEL_SUPPORT |
148 | mark_sigio_pending(); |
149 | #endif |
150 | return; |
151 | } |
152 | |
153 | irq_io_loop(irq: reg, regs); |
154 | } |
155 | |
156 | static void _sigio_handler(struct uml_pt_regs *regs, |
157 | bool timetravel_handlers_only) |
158 | { |
159 | struct irq_entry *irq_entry; |
160 | int n, i; |
161 | |
162 | if (timetravel_handlers_only && !um_irq_timetravel_handler_used()) |
163 | return; |
164 | |
165 | while (1) { |
166 | /* This is now lockless - epoll keeps back-referencesto the irqs |
167 | * which have trigger it so there is no need to walk the irq |
168 | * list and lock it every time. We avoid locking by turning off |
169 | * IO for a specific fd by executing os_del_epoll_fd(fd) before |
170 | * we do any changes to the actual data structures |
171 | */ |
172 | n = os_waiting_for_events_epoll(); |
173 | |
174 | if (n <= 0) { |
175 | if (n == -EINTR) |
176 | continue; |
177 | else |
178 | break; |
179 | } |
180 | |
181 | for (i = 0; i < n ; i++) { |
182 | enum um_irq_type t; |
183 | |
184 | irq_entry = os_epoll_get_data_pointer(i); |
185 | |
186 | for (t = 0; t < NUM_IRQ_TYPES; t++) |
187 | sigio_reg_handler(idx: i, entry: irq_entry, t: t, regs, |
188 | timetravel_handlers_only); |
189 | } |
190 | } |
191 | |
192 | if (!timetravel_handlers_only) |
193 | free_irqs(); |
194 | } |
195 | |
196 | void sigio_handler(int sig, struct siginfo *unused_si, struct uml_pt_regs *regs) |
197 | { |
198 | _sigio_handler(regs, timetravel_handlers_only: irqs_suspended); |
199 | } |
200 | |
201 | static struct irq_entry *get_irq_entry_by_fd(int fd) |
202 | { |
203 | struct irq_entry *walk; |
204 | |
205 | lockdep_assert_held(&irq_lock); |
206 | |
207 | list_for_each_entry(walk, &active_fds, list) { |
208 | if (walk->fd == fd) |
209 | return walk; |
210 | } |
211 | |
212 | return NULL; |
213 | } |
214 | |
215 | static void free_irq_entry(struct irq_entry *to_free, bool remove) |
216 | { |
217 | if (!to_free) |
218 | return; |
219 | |
220 | if (remove) |
221 | os_del_epoll_fd(to_free->fd); |
222 | list_del(entry: &to_free->list); |
223 | kfree(objp: to_free); |
224 | } |
225 | |
226 | static bool update_irq_entry(struct irq_entry *entry) |
227 | { |
228 | enum um_irq_type i; |
229 | int events = 0; |
230 | |
231 | for (i = 0; i < NUM_IRQ_TYPES; i++) |
232 | events |= entry->reg[i].events; |
233 | |
234 | if (events) { |
235 | /* will modify (instead of add) if needed */ |
236 | os_add_epoll_fd(events, entry->fd, entry); |
237 | return true; |
238 | } |
239 | |
240 | os_del_epoll_fd(entry->fd); |
241 | return false; |
242 | } |
243 | |
244 | static void update_or_free_irq_entry(struct irq_entry *entry) |
245 | { |
246 | if (!update_irq_entry(entry)) |
247 | free_irq_entry(to_free: entry, remove: false); |
248 | } |
249 | |
250 | static int activate_fd(int irq, int fd, enum um_irq_type type, void *dev_id, |
251 | void (*timetravel_handler)(int, int, void *, |
252 | struct time_travel_event *)) |
253 | { |
254 | struct irq_entry *irq_entry; |
255 | int err, events = os_event_mask(type); |
256 | unsigned long flags; |
257 | |
258 | err = os_set_fd_async(fd); |
259 | if (err < 0) |
260 | goto out; |
261 | |
262 | spin_lock_irqsave(&irq_lock, flags); |
263 | irq_entry = get_irq_entry_by_fd(fd); |
264 | if (irq_entry) { |
265 | /* cannot register the same FD twice with the same type */ |
266 | if (WARN_ON(irq_entry->reg[type].events)) { |
267 | err = -EALREADY; |
268 | goto out_unlock; |
269 | } |
270 | |
271 | /* temporarily disable to avoid IRQ-side locking */ |
272 | os_del_epoll_fd(fd); |
273 | } else { |
274 | irq_entry = kzalloc(size: sizeof(*irq_entry), GFP_ATOMIC); |
275 | if (!irq_entry) { |
276 | err = -ENOMEM; |
277 | goto out_unlock; |
278 | } |
279 | irq_entry->fd = fd; |
280 | list_add_tail(new: &irq_entry->list, head: &active_fds); |
281 | maybe_sigio_broken(fd); |
282 | } |
283 | |
284 | irq_entry->reg[type].id = dev_id; |
285 | irq_entry->reg[type].irq = irq; |
286 | irq_entry->reg[type].active = true; |
287 | irq_entry->reg[type].events = events; |
288 | |
289 | #ifdef CONFIG_UML_TIME_TRAVEL_SUPPORT |
290 | if (um_irq_timetravel_handler_used()) { |
291 | irq_entry->reg[type].timetravel_handler = timetravel_handler; |
292 | irq_entry->reg[type].event.fn = irq_event_handler; |
293 | } |
294 | #endif |
295 | |
296 | WARN_ON(!update_irq_entry(irq_entry)); |
297 | spin_unlock_irqrestore(lock: &irq_lock, flags); |
298 | |
299 | return 0; |
300 | out_unlock: |
301 | spin_unlock_irqrestore(lock: &irq_lock, flags); |
302 | out: |
303 | return err; |
304 | } |
305 | |
306 | /* |
307 | * Remove the entry or entries for a specific FD, if you |
308 | * don't want to remove all the possible entries then use |
309 | * um_free_irq() or deactivate_fd() instead. |
310 | */ |
311 | void free_irq_by_fd(int fd) |
312 | { |
313 | struct irq_entry *to_free; |
314 | unsigned long flags; |
315 | |
316 | spin_lock_irqsave(&irq_lock, flags); |
317 | to_free = get_irq_entry_by_fd(fd); |
318 | free_irq_entry(to_free, remove: true); |
319 | spin_unlock_irqrestore(lock: &irq_lock, flags); |
320 | } |
321 | EXPORT_SYMBOL(free_irq_by_fd); |
322 | |
323 | static void free_irq_by_irq_and_dev(unsigned int irq, void *dev) |
324 | { |
325 | struct irq_entry *entry; |
326 | unsigned long flags; |
327 | |
328 | spin_lock_irqsave(&irq_lock, flags); |
329 | list_for_each_entry(entry, &active_fds, list) { |
330 | enum um_irq_type i; |
331 | |
332 | for (i = 0; i < NUM_IRQ_TYPES; i++) { |
333 | struct irq_reg *reg = &entry->reg[i]; |
334 | |
335 | if (!reg->events) |
336 | continue; |
337 | if (reg->irq != irq) |
338 | continue; |
339 | if (reg->id != dev) |
340 | continue; |
341 | |
342 | os_del_epoll_fd(entry->fd); |
343 | reg->events = 0; |
344 | update_or_free_irq_entry(entry); |
345 | goto out; |
346 | } |
347 | } |
348 | out: |
349 | spin_unlock_irqrestore(lock: &irq_lock, flags); |
350 | } |
351 | |
352 | void deactivate_fd(int fd, int irqnum) |
353 | { |
354 | struct irq_entry *entry; |
355 | unsigned long flags; |
356 | enum um_irq_type i; |
357 | |
358 | os_del_epoll_fd(fd); |
359 | |
360 | spin_lock_irqsave(&irq_lock, flags); |
361 | entry = get_irq_entry_by_fd(fd); |
362 | if (!entry) |
363 | goto out; |
364 | |
365 | for (i = 0; i < NUM_IRQ_TYPES; i++) { |
366 | if (!entry->reg[i].events) |
367 | continue; |
368 | if (entry->reg[i].irq == irqnum) |
369 | entry->reg[i].events = 0; |
370 | } |
371 | |
372 | update_or_free_irq_entry(entry); |
373 | out: |
374 | spin_unlock_irqrestore(lock: &irq_lock, flags); |
375 | |
376 | ignore_sigio_fd(fd); |
377 | } |
378 | EXPORT_SYMBOL(deactivate_fd); |
379 | |
380 | /* |
381 | * Called just before shutdown in order to provide a clean exec |
382 | * environment in case the system is rebooting. No locking because |
383 | * that would cause a pointless shutdown hang if something hadn't |
384 | * released the lock. |
385 | */ |
386 | int deactivate_all_fds(void) |
387 | { |
388 | struct irq_entry *entry; |
389 | |
390 | /* Stop IO. The IRQ loop has no lock so this is our |
391 | * only way of making sure we are safe to dispose |
392 | * of all IRQ handlers |
393 | */ |
394 | os_set_ioignore(); |
395 | |
396 | /* we can no longer call kfree() here so just deactivate */ |
397 | list_for_each_entry(entry, &active_fds, list) |
398 | os_del_epoll_fd(entry->fd); |
399 | os_close_epoll_fd(); |
400 | return 0; |
401 | } |
402 | |
403 | /* |
404 | * do_IRQ handles all normal device IRQs (the special |
405 | * SMP cross-CPU interrupts have their own specific |
406 | * handlers). |
407 | */ |
408 | unsigned int do_IRQ(int irq, struct uml_pt_regs *regs) |
409 | { |
410 | struct pt_regs *old_regs = set_irq_regs((struct pt_regs *)regs); |
411 | irq_enter(); |
412 | generic_handle_irq(irq); |
413 | irq_exit(); |
414 | set_irq_regs(old_regs); |
415 | return 1; |
416 | } |
417 | |
418 | void um_free_irq(int irq, void *dev) |
419 | { |
420 | if (WARN(irq < 0 || irq > UM_LAST_SIGNAL_IRQ, |
421 | "freeing invalid irq %d" , irq)) |
422 | return; |
423 | |
424 | free_irq_by_irq_and_dev(irq, dev); |
425 | free_irq(irq, dev); |
426 | clear_bit(nr: irq, addr: irqs_allocated); |
427 | } |
428 | EXPORT_SYMBOL(um_free_irq); |
429 | |
430 | static int |
431 | _um_request_irq(int irq, int fd, enum um_irq_type type, |
432 | irq_handler_t handler, unsigned long irqflags, |
433 | const char *devname, void *dev_id, |
434 | void (*timetravel_handler)(int, int, void *, |
435 | struct time_travel_event *)) |
436 | { |
437 | int err; |
438 | |
439 | if (irq == UM_IRQ_ALLOC) { |
440 | int i; |
441 | |
442 | for (i = UM_FIRST_DYN_IRQ; i < NR_IRQS; i++) { |
443 | if (!test_and_set_bit(nr: i, addr: irqs_allocated)) { |
444 | irq = i; |
445 | break; |
446 | } |
447 | } |
448 | } |
449 | |
450 | if (irq < 0) |
451 | return -ENOSPC; |
452 | |
453 | if (fd != -1) { |
454 | err = activate_fd(irq, fd, type: type, dev_id, timetravel_handler); |
455 | if (err) |
456 | goto error; |
457 | } |
458 | |
459 | err = request_irq(irq, handler, flags: irqflags, name: devname, dev: dev_id); |
460 | if (err < 0) |
461 | goto error; |
462 | |
463 | return irq; |
464 | error: |
465 | clear_bit(nr: irq, addr: irqs_allocated); |
466 | return err; |
467 | } |
468 | |
469 | int um_request_irq(int irq, int fd, enum um_irq_type type, |
470 | irq_handler_t handler, unsigned long irqflags, |
471 | const char *devname, void *dev_id) |
472 | { |
473 | return _um_request_irq(irq, fd, type: type, handler, irqflags, |
474 | devname, dev_id, NULL); |
475 | } |
476 | EXPORT_SYMBOL(um_request_irq); |
477 | |
478 | #ifdef CONFIG_UML_TIME_TRAVEL_SUPPORT |
479 | int um_request_irq_tt(int irq, int fd, enum um_irq_type type, |
480 | irq_handler_t handler, unsigned long irqflags, |
481 | const char *devname, void *dev_id, |
482 | void (*timetravel_handler)(int, int, void *, |
483 | struct time_travel_event *)) |
484 | { |
485 | return _um_request_irq(irq, fd, type, handler, irqflags, |
486 | devname, dev_id, timetravel_handler); |
487 | } |
488 | EXPORT_SYMBOL(um_request_irq_tt); |
489 | |
490 | void sigio_run_timetravel_handlers(void) |
491 | { |
492 | _sigio_handler(NULL, true); |
493 | } |
494 | #endif |
495 | |
496 | #ifdef CONFIG_PM_SLEEP |
497 | void um_irqs_suspend(void) |
498 | { |
499 | struct irq_entry *entry; |
500 | unsigned long flags; |
501 | |
502 | irqs_suspended = true; |
503 | |
504 | spin_lock_irqsave(&irq_lock, flags); |
505 | list_for_each_entry(entry, &active_fds, list) { |
506 | enum um_irq_type t; |
507 | bool clear = true; |
508 | |
509 | for (t = 0; t < NUM_IRQ_TYPES; t++) { |
510 | if (!entry->reg[t].events) |
511 | continue; |
512 | |
513 | /* |
514 | * For the SIGIO_WRITE_IRQ, which is used to handle the |
515 | * SIGIO workaround thread, we need special handling: |
516 | * enable wake for it itself, but below we tell it about |
517 | * any FDs that should be suspended. |
518 | */ |
519 | if (entry->reg[t].wakeup || |
520 | entry->reg[t].irq == SIGIO_WRITE_IRQ |
521 | #ifdef CONFIG_UML_TIME_TRAVEL_SUPPORT |
522 | || entry->reg[t].timetravel_handler |
523 | #endif |
524 | ) { |
525 | clear = false; |
526 | break; |
527 | } |
528 | } |
529 | |
530 | if (clear) { |
531 | entry->suspended = true; |
532 | os_clear_fd_async(entry->fd); |
533 | entry->sigio_workaround = |
534 | !__ignore_sigio_fd(entry->fd); |
535 | } |
536 | } |
537 | spin_unlock_irqrestore(lock: &irq_lock, flags); |
538 | } |
539 | |
540 | void um_irqs_resume(void) |
541 | { |
542 | struct irq_entry *entry; |
543 | unsigned long flags; |
544 | |
545 | |
546 | local_irq_save(flags); |
547 | #ifdef CONFIG_UML_TIME_TRAVEL_SUPPORT |
548 | /* |
549 | * We don't need to lock anything here since we're in resume |
550 | * and nothing else is running, but have disabled IRQs so we |
551 | * don't try anything else with the interrupt list from there. |
552 | */ |
553 | list_for_each_entry(entry, &active_fds, list) { |
554 | enum um_irq_type t; |
555 | |
556 | for (t = 0; t < NUM_IRQ_TYPES; t++) { |
557 | struct irq_reg *reg = &entry->reg[t]; |
558 | |
559 | if (reg->pending_on_resume) { |
560 | irq_enter(); |
561 | generic_handle_irq(reg->irq); |
562 | irq_exit(); |
563 | reg->pending_on_resume = false; |
564 | } |
565 | } |
566 | } |
567 | #endif |
568 | |
569 | spin_lock(lock: &irq_lock); |
570 | list_for_each_entry(entry, &active_fds, list) { |
571 | if (entry->suspended) { |
572 | int err = os_set_fd_async(entry->fd); |
573 | |
574 | WARN(err < 0, "os_set_fd_async returned %d\n" , err); |
575 | entry->suspended = false; |
576 | |
577 | if (entry->sigio_workaround) { |
578 | err = __add_sigio_fd(entry->fd); |
579 | WARN(err < 0, "add_sigio_returned %d\n" , err); |
580 | } |
581 | } |
582 | } |
583 | spin_unlock_irqrestore(lock: &irq_lock, flags); |
584 | |
585 | irqs_suspended = false; |
586 | send_sigio_to_self(); |
587 | } |
588 | |
589 | static int normal_irq_set_wake(struct irq_data *d, unsigned int on) |
590 | { |
591 | struct irq_entry *entry; |
592 | unsigned long flags; |
593 | |
594 | spin_lock_irqsave(&irq_lock, flags); |
595 | list_for_each_entry(entry, &active_fds, list) { |
596 | enum um_irq_type t; |
597 | |
598 | for (t = 0; t < NUM_IRQ_TYPES; t++) { |
599 | if (!entry->reg[t].events) |
600 | continue; |
601 | |
602 | if (entry->reg[t].irq != d->irq) |
603 | continue; |
604 | entry->reg[t].wakeup = on; |
605 | goto unlock; |
606 | } |
607 | } |
608 | unlock: |
609 | spin_unlock_irqrestore(lock: &irq_lock, flags); |
610 | return 0; |
611 | } |
612 | #else |
613 | #define normal_irq_set_wake NULL |
614 | #endif |
615 | |
616 | /* |
617 | * irq_chip must define at least enable/disable and ack when |
618 | * the edge handler is used. |
619 | */ |
620 | static void dummy(struct irq_data *d) |
621 | { |
622 | } |
623 | |
624 | /* This is used for everything other than the timer. */ |
625 | static struct irq_chip normal_irq_type = { |
626 | .name = "SIGIO" , |
627 | .irq_disable = dummy, |
628 | .irq_enable = dummy, |
629 | .irq_ack = dummy, |
630 | .irq_mask = dummy, |
631 | .irq_unmask = dummy, |
632 | .irq_set_wake = normal_irq_set_wake, |
633 | }; |
634 | |
635 | static struct irq_chip alarm_irq_type = { |
636 | .name = "SIGALRM" , |
637 | .irq_disable = dummy, |
638 | .irq_enable = dummy, |
639 | .irq_ack = dummy, |
640 | .irq_mask = dummy, |
641 | .irq_unmask = dummy, |
642 | }; |
643 | |
644 | void __init init_IRQ(void) |
645 | { |
646 | int i; |
647 | |
648 | irq_set_chip_and_handler(TIMER_IRQ, &alarm_irq_type, handle_edge_irq); |
649 | |
650 | for (i = 1; i < UM_LAST_SIGNAL_IRQ; i++) |
651 | irq_set_chip_and_handler(i, &normal_irq_type, handle_edge_irq); |
652 | /* Initialize EPOLL Loop */ |
653 | os_setup_epoll(); |
654 | } |
655 | |
656 | /* |
657 | * IRQ stack entry and exit: |
658 | * |
659 | * Unlike i386, UML doesn't receive IRQs on the normal kernel stack |
660 | * and switch over to the IRQ stack after some preparation. We use |
661 | * sigaltstack to receive signals on a separate stack from the start. |
662 | * These two functions make sure the rest of the kernel won't be too |
663 | * upset by being on a different stack. The IRQ stack has a |
664 | * thread_info structure at the bottom so that current et al continue |
665 | * to work. |
666 | * |
667 | * to_irq_stack copies the current task's thread_info to the IRQ stack |
668 | * thread_info and sets the tasks's stack to point to the IRQ stack. |
669 | * |
670 | * from_irq_stack copies the thread_info struct back (flags may have |
671 | * been modified) and resets the task's stack pointer. |
672 | * |
673 | * Tricky bits - |
674 | * |
675 | * What happens when two signals race each other? UML doesn't block |
676 | * signals with sigprocmask, SA_DEFER, or sa_mask, so a second signal |
677 | * could arrive while a previous one is still setting up the |
678 | * thread_info. |
679 | * |
680 | * There are three cases - |
681 | * The first interrupt on the stack - sets up the thread_info and |
682 | * handles the interrupt |
683 | * A nested interrupt interrupting the copying of the thread_info - |
684 | * can't handle the interrupt, as the stack is in an unknown state |
685 | * A nested interrupt not interrupting the copying of the |
686 | * thread_info - doesn't do any setup, just handles the interrupt |
687 | * |
688 | * The first job is to figure out whether we interrupted stack setup. |
689 | * This is done by xchging the signal mask with thread_info->pending. |
690 | * If the value that comes back is zero, then there is no setup in |
691 | * progress, and the interrupt can be handled. If the value is |
692 | * non-zero, then there is stack setup in progress. In order to have |
693 | * the interrupt handled, we leave our signal in the mask, and it will |
694 | * be handled by the upper handler after it has set up the stack. |
695 | * |
696 | * Next is to figure out whether we are the outer handler or a nested |
697 | * one. As part of setting up the stack, thread_info->real_thread is |
698 | * set to non-NULL (and is reset to NULL on exit). This is the |
699 | * nesting indicator. If it is non-NULL, then the stack is already |
700 | * set up and the handler can run. |
701 | */ |
702 | |
703 | static unsigned long pending_mask; |
704 | |
705 | unsigned long to_irq_stack(unsigned long *mask_out) |
706 | { |
707 | struct thread_info *ti; |
708 | unsigned long mask, old; |
709 | int nested; |
710 | |
711 | mask = xchg(&pending_mask, *mask_out); |
712 | if (mask != 0) { |
713 | /* |
714 | * If any interrupts come in at this point, we want to |
715 | * make sure that their bits aren't lost by our |
716 | * putting our bit in. So, this loop accumulates bits |
717 | * until xchg returns the same value that we put in. |
718 | * When that happens, there were no new interrupts, |
719 | * and pending_mask contains a bit for each interrupt |
720 | * that came in. |
721 | */ |
722 | old = *mask_out; |
723 | do { |
724 | old |= mask; |
725 | mask = xchg(&pending_mask, old); |
726 | } while (mask != old); |
727 | return 1; |
728 | } |
729 | |
730 | ti = current_thread_info(); |
731 | nested = (ti->real_thread != NULL); |
732 | if (!nested) { |
733 | struct task_struct *task; |
734 | struct thread_info *tti; |
735 | |
736 | task = cpu_tasks[ti->cpu].task; |
737 | tti = task_thread_info(task); |
738 | |
739 | *ti = *tti; |
740 | ti->real_thread = tti; |
741 | task->stack = ti; |
742 | } |
743 | |
744 | mask = xchg(&pending_mask, 0); |
745 | *mask_out |= mask | nested; |
746 | return 0; |
747 | } |
748 | |
749 | unsigned long from_irq_stack(int nested) |
750 | { |
751 | struct thread_info *ti, *to; |
752 | unsigned long mask; |
753 | |
754 | ti = current_thread_info(); |
755 | |
756 | pending_mask = 1; |
757 | |
758 | to = ti->real_thread; |
759 | current->stack = to; |
760 | ti->real_thread = NULL; |
761 | *to = *ti; |
762 | |
763 | mask = xchg(&pending_mask, 0); |
764 | return mask & ~1; |
765 | } |
766 | |
767 | |