1/*
2 * linux/kernel/softirq.c
3 *
4 * Copyright (C) 1992 Linus Torvalds
5 *
6 * Distribute under GPLv2.
7 *
8 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
9 */
10
11#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12
13#include <linux/export.h>
14#include <linux/kernel_stat.h>
15#include <linux/interrupt.h>
16#include <linux/init.h>
17#include <linux/mm.h>
18#include <linux/notifier.h>
19#include <linux/percpu.h>
20#include <linux/cpu.h>
21#include <linux/freezer.h>
22#include <linux/kthread.h>
23#include <linux/rcupdate.h>
24#include <linux/ftrace.h>
25#include <linux/smp.h>
26#include <linux/smpboot.h>
27#include <linux/tick.h>
28#include <linux/irq.h>
29
30#define CREATE_TRACE_POINTS
31#include <trace/events/irq.h>
32
33/*
34 - No shared variables, all the data are CPU local.
35 - If a softirq needs serialization, let it serialize itself
36 by its own spinlocks.
37 - Even if softirq is serialized, only local cpu is marked for
38 execution. Hence, we get something sort of weak cpu binding.
39 Though it is still not clear, will it result in better locality
40 or will not.
41
42 Examples:
43 - NET RX softirq. It is multithreaded and does not require
44 any global serialization.
45 - NET TX softirq. It kicks software netdevice queues, hence
46 it is logically serialized per device, but this serialization
47 is invisible to common code.
48 - Tasklets: serialized wrt itself.
49 */
50
51#ifndef __ARCH_IRQ_STAT
52DEFINE_PER_CPU_ALIGNED(irq_cpustat_t, irq_stat);
53EXPORT_PER_CPU_SYMBOL(irq_stat);
54#endif
55
56static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
57
58DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
59
60const char * const softirq_to_name[NR_SOFTIRQS] = {
61 "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "IRQ_POLL",
62 "TASKLET", "SCHED", "HRTIMER", "RCU"
63};
64
65/*
66 * we cannot loop indefinitely here to avoid userspace starvation,
67 * but we also don't want to introduce a worst case 1/HZ latency
68 * to the pending events, so lets the scheduler to balance
69 * the softirq load for us.
70 */
71static void wakeup_softirqd(void)
72{
73 /* Interrupts are disabled: no need to stop preemption */
74 struct task_struct *tsk = __this_cpu_read(ksoftirqd);
75
76 if (tsk && tsk->state != TASK_RUNNING)
77 wake_up_process(tsk);
78}
79
80/*
81 * If ksoftirqd is scheduled, we do not want to process pending softirqs
82 * right now. Let ksoftirqd handle this at its own rate, to get fairness,
83 * unless we're doing some of the synchronous softirqs.
84 */
85#define SOFTIRQ_NOW_MASK ((1 << HI_SOFTIRQ) | (1 << TASKLET_SOFTIRQ))
86static bool ksoftirqd_running(unsigned long pending)
87{
88 struct task_struct *tsk = __this_cpu_read(ksoftirqd);
89
90 if (pending & SOFTIRQ_NOW_MASK)
91 return false;
92 return tsk && (tsk->state == TASK_RUNNING) &&
93 !__kthread_should_park(tsk);
94}
95
96/*
97 * preempt_count and SOFTIRQ_OFFSET usage:
98 * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
99 * softirq processing.
100 * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
101 * on local_bh_disable or local_bh_enable.
102 * This lets us distinguish between whether we are currently processing
103 * softirq and whether we just have bh disabled.
104 */
105
106/*
107 * This one is for softirq.c-internal use,
108 * where hardirqs are disabled legitimately:
109 */
110#ifdef CONFIG_TRACE_IRQFLAGS
111void __local_bh_disable_ip(unsigned long ip, unsigned int cnt)
112{
113 unsigned long flags;
114
115 WARN_ON_ONCE(in_irq());
116
117 raw_local_irq_save(flags);
118 /*
119 * The preempt tracer hooks into preempt_count_add and will break
120 * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
121 * is set and before current->softirq_enabled is cleared.
122 * We must manually increment preempt_count here and manually
123 * call the trace_preempt_off later.
124 */
125 __preempt_count_add(cnt);
126 /*
127 * Were softirqs turned off above:
128 */
129 if (softirq_count() == (cnt & SOFTIRQ_MASK))
130 trace_softirqs_off(ip);
131 raw_local_irq_restore(flags);
132
133 if (preempt_count() == cnt) {
134#ifdef CONFIG_DEBUG_PREEMPT
135 current->preempt_disable_ip = get_lock_parent_ip();
136#endif
137 trace_preempt_off(CALLER_ADDR0, get_lock_parent_ip());
138 }
139}
140EXPORT_SYMBOL(__local_bh_disable_ip);
141#endif /* CONFIG_TRACE_IRQFLAGS */
142
143static void __local_bh_enable(unsigned int cnt)
144{
145 lockdep_assert_irqs_disabled();
146
147 if (preempt_count() == cnt)
148 trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip());
149
150 if (softirq_count() == (cnt & SOFTIRQ_MASK))
151 trace_softirqs_on(_RET_IP_);
152
153 __preempt_count_sub(cnt);
154}
155
156/*
157 * Special-case - softirqs can safely be enabled by __do_softirq(),
158 * without processing still-pending softirqs:
159 */
160void _local_bh_enable(void)
161{
162 WARN_ON_ONCE(in_irq());
163 __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
164}
165EXPORT_SYMBOL(_local_bh_enable);
166
167void __local_bh_enable_ip(unsigned long ip, unsigned int cnt)
168{
169 WARN_ON_ONCE(in_irq());
170 lockdep_assert_irqs_enabled();
171#ifdef CONFIG_TRACE_IRQFLAGS
172 local_irq_disable();
173#endif
174 /*
175 * Are softirqs going to be turned on now:
176 */
177 if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
178 trace_softirqs_on(ip);
179 /*
180 * Keep preemption disabled until we are done with
181 * softirq processing:
182 */
183 preempt_count_sub(cnt - 1);
184
185 if (unlikely(!in_interrupt() && local_softirq_pending())) {
186 /*
187 * Run softirq if any pending. And do it in its own stack
188 * as we may be calling this deep in a task call stack already.
189 */
190 do_softirq();
191 }
192
193 preempt_count_dec();
194#ifdef CONFIG_TRACE_IRQFLAGS
195 local_irq_enable();
196#endif
197 preempt_check_resched();
198}
199EXPORT_SYMBOL(__local_bh_enable_ip);
200
201/*
202 * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
203 * but break the loop if need_resched() is set or after 2 ms.
204 * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in
205 * certain cases, such as stop_machine(), jiffies may cease to
206 * increment and so we need the MAX_SOFTIRQ_RESTART limit as
207 * well to make sure we eventually return from this method.
208 *
209 * These limits have been established via experimentation.
210 * The two things to balance is latency against fairness -
211 * we want to handle softirqs as soon as possible, but they
212 * should not be able to lock up the box.
213 */
214#define MAX_SOFTIRQ_TIME msecs_to_jiffies(2)
215#define MAX_SOFTIRQ_RESTART 10
216
217#ifdef CONFIG_TRACE_IRQFLAGS
218/*
219 * When we run softirqs from irq_exit() and thus on the hardirq stack we need
220 * to keep the lockdep irq context tracking as tight as possible in order to
221 * not miss-qualify lock contexts and miss possible deadlocks.
222 */
223
224static inline bool lockdep_softirq_start(void)
225{
226 bool in_hardirq = false;
227
228 if (trace_hardirq_context(current)) {
229 in_hardirq = true;
230 trace_hardirq_exit();
231 }
232
233 lockdep_softirq_enter();
234
235 return in_hardirq;
236}
237
238static inline void lockdep_softirq_end(bool in_hardirq)
239{
240 lockdep_softirq_exit();
241
242 if (in_hardirq)
243 trace_hardirq_enter();
244}
245#else
246static inline bool lockdep_softirq_start(void) { return false; }
247static inline void lockdep_softirq_end(bool in_hardirq) { }
248#endif
249
250asmlinkage __visible void __softirq_entry __do_softirq(void)
251{
252 unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
253 unsigned long old_flags = current->flags;
254 int max_restart = MAX_SOFTIRQ_RESTART;
255 struct softirq_action *h;
256 bool in_hardirq;
257 __u32 pending;
258 int softirq_bit;
259
260 /*
261 * Mask out PF_MEMALLOC as the current task context is borrowed for the
262 * softirq. A softirq handled, such as network RX, might set PF_MEMALLOC
263 * again if the socket is related to swapping.
264 */
265 current->flags &= ~PF_MEMALLOC;
266
267 pending = local_softirq_pending();
268 account_irq_enter_time(current);
269
270 __local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET);
271 in_hardirq = lockdep_softirq_start();
272
273restart:
274 /* Reset the pending bitmask before enabling irqs */
275 set_softirq_pending(0);
276
277 local_irq_enable();
278
279 h = softirq_vec;
280
281 while ((softirq_bit = ffs(pending))) {
282 unsigned int vec_nr;
283 int prev_count;
284
285 h += softirq_bit - 1;
286
287 vec_nr = h - softirq_vec;
288 prev_count = preempt_count();
289
290 kstat_incr_softirqs_this_cpu(vec_nr);
291
292 trace_softirq_entry(vec_nr);
293 h->action(h);
294 trace_softirq_exit(vec_nr);
295 if (unlikely(prev_count != preempt_count())) {
296 pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n",
297 vec_nr, softirq_to_name[vec_nr], h->action,
298 prev_count, preempt_count());
299 preempt_count_set(prev_count);
300 }
301 h++;
302 pending >>= softirq_bit;
303 }
304
305 if (__this_cpu_read(ksoftirqd) == current)
306 rcu_softirq_qs();
307 local_irq_disable();
308
309 pending = local_softirq_pending();
310 if (pending) {
311 if (time_before(jiffies, end) && !need_resched() &&
312 --max_restart)
313 goto restart;
314
315 wakeup_softirqd();
316 }
317
318 lockdep_softirq_end(in_hardirq);
319 account_irq_exit_time(current);
320 __local_bh_enable(SOFTIRQ_OFFSET);
321 WARN_ON_ONCE(in_interrupt());
322 current_restore_flags(old_flags, PF_MEMALLOC);
323}
324
325asmlinkage __visible void do_softirq(void)
326{
327 __u32 pending;
328 unsigned long flags;
329
330 if (in_interrupt())
331 return;
332
333 local_irq_save(flags);
334
335 pending = local_softirq_pending();
336
337 if (pending && !ksoftirqd_running(pending))
338 do_softirq_own_stack();
339
340 local_irq_restore(flags);
341}
342
343/*
344 * Enter an interrupt context.
345 */
346void irq_enter(void)
347{
348 rcu_irq_enter();
349 if (is_idle_task(current) && !in_interrupt()) {
350 /*
351 * Prevent raise_softirq from needlessly waking up ksoftirqd
352 * here, as softirq will be serviced on return from interrupt.
353 */
354 local_bh_disable();
355 tick_irq_enter();
356 _local_bh_enable();
357 }
358
359 __irq_enter();
360}
361
362static inline void invoke_softirq(void)
363{
364 if (ksoftirqd_running(local_softirq_pending()))
365 return;
366
367 if (!force_irqthreads) {
368#ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
369 /*
370 * We can safely execute softirq on the current stack if
371 * it is the irq stack, because it should be near empty
372 * at this stage.
373 */
374 __do_softirq();
375#else
376 /*
377 * Otherwise, irq_exit() is called on the task stack that can
378 * be potentially deep already. So call softirq in its own stack
379 * to prevent from any overrun.
380 */
381 do_softirq_own_stack();
382#endif
383 } else {
384 wakeup_softirqd();
385 }
386}
387
388static inline void tick_irq_exit(void)
389{
390#ifdef CONFIG_NO_HZ_COMMON
391 int cpu = smp_processor_id();
392
393 /* Make sure that timer wheel updates are propagated */
394 if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
395 if (!in_irq())
396 tick_nohz_irq_exit();
397 }
398#endif
399}
400
401/*
402 * Exit an interrupt context. Process softirqs if needed and possible:
403 */
404void irq_exit(void)
405{
406#ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
407 local_irq_disable();
408#else
409 lockdep_assert_irqs_disabled();
410#endif
411 account_irq_exit_time(current);
412 preempt_count_sub(HARDIRQ_OFFSET);
413 if (!in_interrupt() && local_softirq_pending())
414 invoke_softirq();
415
416 tick_irq_exit();
417 rcu_irq_exit();
418 trace_hardirq_exit(); /* must be last! */
419}
420
421/*
422 * This function must run with irqs disabled!
423 */
424inline void raise_softirq_irqoff(unsigned int nr)
425{
426 __raise_softirq_irqoff(nr);
427
428 /*
429 * If we're in an interrupt or softirq, we're done
430 * (this also catches softirq-disabled code). We will
431 * actually run the softirq once we return from
432 * the irq or softirq.
433 *
434 * Otherwise we wake up ksoftirqd to make sure we
435 * schedule the softirq soon.
436 */
437 if (!in_interrupt())
438 wakeup_softirqd();
439}
440
441void raise_softirq(unsigned int nr)
442{
443 unsigned long flags;
444
445 local_irq_save(flags);
446 raise_softirq_irqoff(nr);
447 local_irq_restore(flags);
448}
449
450void __raise_softirq_irqoff(unsigned int nr)
451{
452 trace_softirq_raise(nr);
453 or_softirq_pending(1UL << nr);
454}
455
456void open_softirq(int nr, void (*action)(struct softirq_action *))
457{
458 softirq_vec[nr].action = action;
459}
460
461/*
462 * Tasklets
463 */
464struct tasklet_head {
465 struct tasklet_struct *head;
466 struct tasklet_struct **tail;
467};
468
469static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
470static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
471
472static void __tasklet_schedule_common(struct tasklet_struct *t,
473 struct tasklet_head __percpu *headp,
474 unsigned int softirq_nr)
475{
476 struct tasklet_head *head;
477 unsigned long flags;
478
479 local_irq_save(flags);
480 head = this_cpu_ptr(headp);
481 t->next = NULL;
482 *head->tail = t;
483 head->tail = &(t->next);
484 raise_softirq_irqoff(softirq_nr);
485 local_irq_restore(flags);
486}
487
488void __tasklet_schedule(struct tasklet_struct *t)
489{
490 __tasklet_schedule_common(t, &tasklet_vec,
491 TASKLET_SOFTIRQ);
492}
493EXPORT_SYMBOL(__tasklet_schedule);
494
495void __tasklet_hi_schedule(struct tasklet_struct *t)
496{
497 __tasklet_schedule_common(t, &tasklet_hi_vec,
498 HI_SOFTIRQ);
499}
500EXPORT_SYMBOL(__tasklet_hi_schedule);
501
502static void tasklet_action_common(struct softirq_action *a,
503 struct tasklet_head *tl_head,
504 unsigned int softirq_nr)
505{
506 struct tasklet_struct *list;
507
508 local_irq_disable();
509 list = tl_head->head;
510 tl_head->head = NULL;
511 tl_head->tail = &tl_head->head;
512 local_irq_enable();
513
514 while (list) {
515 struct tasklet_struct *t = list;
516
517 list = list->next;
518
519 if (tasklet_trylock(t)) {
520 if (!atomic_read(&t->count)) {
521 if (!test_and_clear_bit(TASKLET_STATE_SCHED,
522 &t->state))
523 BUG();
524 t->func(t->data);
525 tasklet_unlock(t);
526 continue;
527 }
528 tasklet_unlock(t);
529 }
530
531 local_irq_disable();
532 t->next = NULL;
533 *tl_head->tail = t;
534 tl_head->tail = &t->next;
535 __raise_softirq_irqoff(softirq_nr);
536 local_irq_enable();
537 }
538}
539
540static __latent_entropy void tasklet_action(struct softirq_action *a)
541{
542 tasklet_action_common(a, this_cpu_ptr(&tasklet_vec), TASKLET_SOFTIRQ);
543}
544
545static __latent_entropy void tasklet_hi_action(struct softirq_action *a)
546{
547 tasklet_action_common(a, this_cpu_ptr(&tasklet_hi_vec), HI_SOFTIRQ);
548}
549
550void tasklet_init(struct tasklet_struct *t,
551 void (*func)(unsigned long), unsigned long data)
552{
553 t->next = NULL;
554 t->state = 0;
555 atomic_set(&t->count, 0);
556 t->func = func;
557 t->data = data;
558}
559EXPORT_SYMBOL(tasklet_init);
560
561void tasklet_kill(struct tasklet_struct *t)
562{
563 if (in_interrupt())
564 pr_notice("Attempt to kill tasklet from interrupt\n");
565
566 while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
567 do {
568 yield();
569 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
570 }
571 tasklet_unlock_wait(t);
572 clear_bit(TASKLET_STATE_SCHED, &t->state);
573}
574EXPORT_SYMBOL(tasklet_kill);
575
576/*
577 * tasklet_hrtimer
578 */
579
580/*
581 * The trampoline is called when the hrtimer expires. It schedules a tasklet
582 * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
583 * hrtimer callback, but from softirq context.
584 */
585static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
586{
587 struct tasklet_hrtimer *ttimer =
588 container_of(timer, struct tasklet_hrtimer, timer);
589
590 tasklet_hi_schedule(&ttimer->tasklet);
591 return HRTIMER_NORESTART;
592}
593
594/*
595 * Helper function which calls the hrtimer callback from
596 * tasklet/softirq context
597 */
598static void __tasklet_hrtimer_trampoline(unsigned long data)
599{
600 struct tasklet_hrtimer *ttimer = (void *)data;
601 enum hrtimer_restart restart;
602
603 restart = ttimer->function(&ttimer->timer);
604 if (restart != HRTIMER_NORESTART)
605 hrtimer_restart(&ttimer->timer);
606}
607
608/**
609 * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
610 * @ttimer: tasklet_hrtimer which is initialized
611 * @function: hrtimer callback function which gets called from softirq context
612 * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
613 * @mode: hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
614 */
615void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
616 enum hrtimer_restart (*function)(struct hrtimer *),
617 clockid_t which_clock, enum hrtimer_mode mode)
618{
619 hrtimer_init(&ttimer->timer, which_clock, mode);
620 ttimer->timer.function = __hrtimer_tasklet_trampoline;
621 tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
622 (unsigned long)ttimer);
623 ttimer->function = function;
624}
625EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
626
627void __init softirq_init(void)
628{
629 int cpu;
630
631 for_each_possible_cpu(cpu) {
632 per_cpu(tasklet_vec, cpu).tail =
633 &per_cpu(tasklet_vec, cpu).head;
634 per_cpu(tasklet_hi_vec, cpu).tail =
635 &per_cpu(tasklet_hi_vec, cpu).head;
636 }
637
638 open_softirq(TASKLET_SOFTIRQ, tasklet_action);
639 open_softirq(HI_SOFTIRQ, tasklet_hi_action);
640}
641
642static int ksoftirqd_should_run(unsigned int cpu)
643{
644 return local_softirq_pending();
645}
646
647static void run_ksoftirqd(unsigned int cpu)
648{
649 local_irq_disable();
650 if (local_softirq_pending()) {
651 /*
652 * We can safely run softirq on inline stack, as we are not deep
653 * in the task stack here.
654 */
655 __do_softirq();
656 local_irq_enable();
657 cond_resched();
658 return;
659 }
660 local_irq_enable();
661}
662
663#ifdef CONFIG_HOTPLUG_CPU
664/*
665 * tasklet_kill_immediate is called to remove a tasklet which can already be
666 * scheduled for execution on @cpu.
667 *
668 * Unlike tasklet_kill, this function removes the tasklet
669 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
670 *
671 * When this function is called, @cpu must be in the CPU_DEAD state.
672 */
673void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
674{
675 struct tasklet_struct **i;
676
677 BUG_ON(cpu_online(cpu));
678 BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
679
680 if (!test_bit(TASKLET_STATE_SCHED, &t->state))
681 return;
682
683 /* CPU is dead, so no lock needed. */
684 for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
685 if (*i == t) {
686 *i = t->next;
687 /* If this was the tail element, move the tail ptr */
688 if (*i == NULL)
689 per_cpu(tasklet_vec, cpu).tail = i;
690 return;
691 }
692 }
693 BUG();
694}
695
696static int takeover_tasklets(unsigned int cpu)
697{
698 /* CPU is dead, so no lock needed. */
699 local_irq_disable();
700
701 /* Find end, append list for that CPU. */
702 if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
703 *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
704 this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
705 per_cpu(tasklet_vec, cpu).head = NULL;
706 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
707 }
708 raise_softirq_irqoff(TASKLET_SOFTIRQ);
709
710 if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
711 *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
712 __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
713 per_cpu(tasklet_hi_vec, cpu).head = NULL;
714 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
715 }
716 raise_softirq_irqoff(HI_SOFTIRQ);
717
718 local_irq_enable();
719 return 0;
720}
721#else
722#define takeover_tasklets NULL
723#endif /* CONFIG_HOTPLUG_CPU */
724
725static struct smp_hotplug_thread softirq_threads = {
726 .store = &ksoftirqd,
727 .thread_should_run = ksoftirqd_should_run,
728 .thread_fn = run_ksoftirqd,
729 .thread_comm = "ksoftirqd/%u",
730};
731
732static __init int spawn_ksoftirqd(void)
733{
734 cpuhp_setup_state_nocalls(CPUHP_SOFTIRQ_DEAD, "softirq:dead", NULL,
735 takeover_tasklets);
736 BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
737
738 return 0;
739}
740early_initcall(spawn_ksoftirqd);
741
742/*
743 * [ These __weak aliases are kept in a separate compilation unit, so that
744 * GCC does not inline them incorrectly. ]
745 */
746
747int __init __weak early_irq_init(void)
748{
749 return 0;
750}
751
752int __init __weak arch_probe_nr_irqs(void)
753{
754 return NR_IRQS_LEGACY;
755}
756
757int __init __weak arch_early_irq_init(void)
758{
759 return 0;
760}
761
762unsigned int __weak arch_dynirq_lower_bound(unsigned int from)
763{
764 return from;
765}
766