1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright (C) 1992, 1998 Linus Torvalds, Ingo Molnar |
4 | * |
5 | * This file contains the lowest level x86_64-specific interrupt |
6 | * entry and irq statistics code. All the remaining irq logic is |
7 | * done by the generic kernel/irq/ code and in the |
8 | * x86_64-specific irq controller code. (e.g. i8259.c and |
9 | * io_apic.c.) |
10 | */ |
11 | |
12 | #include <linux/kernel_stat.h> |
13 | #include <linux/interrupt.h> |
14 | #include <linux/irq.h> |
15 | #include <linux/seq_file.h> |
16 | #include <linux/delay.h> |
17 | #include <linux/ftrace.h> |
18 | #include <linux/uaccess.h> |
19 | #include <linux/smp.h> |
20 | #include <linux/sched/task_stack.h> |
21 | |
22 | #include <asm/cpu_entry_area.h> |
23 | #include <asm/softirq_stack.h> |
24 | #include <asm/irq_stack.h> |
25 | #include <asm/io_apic.h> |
26 | #include <asm/apic.h> |
27 | |
28 | DEFINE_PER_CPU_PAGE_ALIGNED(struct irq_stack, irq_stack_backing_store) __visible; |
29 | DECLARE_INIT_PER_CPU(irq_stack_backing_store); |
30 | |
31 | #ifdef CONFIG_VMAP_STACK |
32 | /* |
33 | * VMAP the backing store with guard pages |
34 | */ |
35 | static int map_irq_stack(unsigned int cpu) |
36 | { |
37 | char *stack = (char *)per_cpu_ptr(&irq_stack_backing_store, cpu); |
38 | struct page *pages[IRQ_STACK_SIZE / PAGE_SIZE]; |
39 | void *va; |
40 | int i; |
41 | |
42 | for (i = 0; i < IRQ_STACK_SIZE / PAGE_SIZE; i++) { |
43 | phys_addr_t pa = per_cpu_ptr_to_phys(addr: stack + (i << PAGE_SHIFT)); |
44 | |
45 | pages[i] = pfn_to_page(pa >> PAGE_SHIFT); |
46 | } |
47 | |
48 | va = vmap(pages, IRQ_STACK_SIZE / PAGE_SIZE, VM_MAP, PAGE_KERNEL); |
49 | if (!va) |
50 | return -ENOMEM; |
51 | |
52 | /* Store actual TOS to avoid adjustment in the hotpath */ |
53 | per_cpu(pcpu_hot.hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8; |
54 | return 0; |
55 | } |
56 | #else |
57 | /* |
58 | * If VMAP stacks are disabled due to KASAN, just use the per cpu |
59 | * backing store without guard pages. |
60 | */ |
61 | static int map_irq_stack(unsigned int cpu) |
62 | { |
63 | void *va = per_cpu_ptr(&irq_stack_backing_store, cpu); |
64 | |
65 | /* Store actual TOS to avoid adjustment in the hotpath */ |
66 | per_cpu(pcpu_hot.hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8; |
67 | return 0; |
68 | } |
69 | #endif |
70 | |
71 | int irq_init_percpu_irqstack(unsigned int cpu) |
72 | { |
73 | if (per_cpu(pcpu_hot.hardirq_stack_ptr, cpu)) |
74 | return 0; |
75 | return map_irq_stack(cpu); |
76 | } |
77 | |