1 | /* |
2 | * Copyright (C) 2009 Wind River Systems Inc |
3 | * Implemented by fredrik.markstrom@gmail.com and ivarholmqvist@gmail.com |
4 | * |
5 | * based on arch/mips/mm/fault.c which is: |
6 | * |
7 | * Copyright (C) 1995-2000 Ralf Baechle |
8 | * |
9 | * This file is subject to the terms and conditions of the GNU General Public |
10 | * License. See the file "COPYING" in the main directory of this archive |
11 | * for more details. |
12 | */ |
13 | |
14 | #include <linux/signal.h> |
15 | #include <linux/sched.h> |
16 | #include <linux/sched/debug.h> |
17 | #include <linux/interrupt.h> |
18 | #include <linux/kernel.h> |
19 | #include <linux/errno.h> |
20 | #include <linux/string.h> |
21 | #include <linux/types.h> |
22 | #include <linux/ptrace.h> |
23 | #include <linux/mman.h> |
24 | #include <linux/mm.h> |
25 | #include <linux/extable.h> |
26 | #include <linux/uaccess.h> |
27 | #include <linux/perf_event.h> |
28 | |
29 | #include <asm/mmu_context.h> |
30 | #include <asm/traps.h> |
31 | |
32 | #define EXC_SUPERV_INSN_ACCESS 9 /* Supervisor only instruction address */ |
33 | #define EXC_SUPERV_DATA_ACCESS 11 /* Supervisor only data address */ |
34 | #define EXC_X_PROTECTION_FAULT 13 /* TLB permission violation (x) */ |
35 | #define EXC_R_PROTECTION_FAULT 14 /* TLB permission violation (r) */ |
36 | #define EXC_W_PROTECTION_FAULT 15 /* TLB permission violation (w) */ |
37 | |
38 | /* |
39 | * This routine handles page faults. It determines the address, |
40 | * and the problem, and then passes it off to one of the appropriate |
41 | * routines. |
42 | */ |
43 | asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long cause, |
44 | unsigned long address) |
45 | { |
46 | struct vm_area_struct *vma = NULL; |
47 | struct task_struct *tsk = current; |
48 | struct mm_struct *mm = tsk->mm; |
49 | int code = SEGV_MAPERR; |
50 | vm_fault_t fault; |
51 | unsigned int flags = FAULT_FLAG_DEFAULT; |
52 | |
53 | cause >>= 2; |
54 | |
55 | /* Restart the instruction */ |
56 | regs->ea -= 4; |
57 | |
58 | /* |
59 | * We fault-in kernel-space virtual memory on-demand. The |
60 | * 'reference' page table is init_mm.pgd. |
61 | * |
62 | * NOTE! We MUST NOT take any locks for this case. We may |
63 | * be in an interrupt or a critical region, and should |
64 | * only copy the information from the master page table, |
65 | * nothing more. |
66 | */ |
67 | if (unlikely(address >= VMALLOC_START && address <= VMALLOC_END)) { |
68 | if (user_mode(regs)) |
69 | goto bad_area_nosemaphore; |
70 | else |
71 | goto vmalloc_fault; |
72 | } |
73 | |
74 | if (unlikely(address >= TASK_SIZE)) |
75 | goto bad_area_nosemaphore; |
76 | |
77 | /* |
78 | * If we're in an interrupt or have no user |
79 | * context, we must not take the fault.. |
80 | */ |
81 | if (faulthandler_disabled() || !mm) |
82 | goto bad_area_nosemaphore; |
83 | |
84 | if (user_mode(regs)) |
85 | flags |= FAULT_FLAG_USER; |
86 | |
87 | perf_sw_event(event_id: PERF_COUNT_SW_PAGE_FAULTS, nr: 1, regs, addr: address); |
88 | |
89 | retry: |
90 | vma = lock_mm_and_find_vma(mm, address, regs); |
91 | if (!vma) |
92 | goto bad_area_nosemaphore; |
93 | /* |
94 | * Ok, we have a good vm_area for this memory access, so |
95 | * we can handle it.. |
96 | */ |
97 | code = SEGV_ACCERR; |
98 | |
99 | switch (cause) { |
100 | case EXC_SUPERV_INSN_ACCESS: |
101 | goto bad_area; |
102 | case EXC_SUPERV_DATA_ACCESS: |
103 | goto bad_area; |
104 | case EXC_X_PROTECTION_FAULT: |
105 | if (!(vma->vm_flags & VM_EXEC)) |
106 | goto bad_area; |
107 | break; |
108 | case EXC_R_PROTECTION_FAULT: |
109 | if (!(vma->vm_flags & VM_READ)) |
110 | goto bad_area; |
111 | break; |
112 | case EXC_W_PROTECTION_FAULT: |
113 | if (!(vma->vm_flags & VM_WRITE)) |
114 | goto bad_area; |
115 | flags = FAULT_FLAG_WRITE; |
116 | break; |
117 | } |
118 | |
119 | /* |
120 | * If for any reason at all we couldn't handle the fault, |
121 | * make sure we exit gracefully rather than endlessly redo |
122 | * the fault. |
123 | */ |
124 | fault = handle_mm_fault(vma, address, flags, regs); |
125 | |
126 | if (fault_signal_pending(fault_flags: fault, regs)) { |
127 | if (!user_mode(regs)) |
128 | goto no_context; |
129 | return; |
130 | } |
131 | |
132 | /* The fault is fully completed (including releasing mmap lock) */ |
133 | if (fault & VM_FAULT_COMPLETED) |
134 | return; |
135 | |
136 | if (unlikely(fault & VM_FAULT_ERROR)) { |
137 | if (fault & VM_FAULT_OOM) |
138 | goto out_of_memory; |
139 | else if (fault & VM_FAULT_SIGSEGV) |
140 | goto bad_area; |
141 | else if (fault & VM_FAULT_SIGBUS) |
142 | goto do_sigbus; |
143 | BUG(); |
144 | } |
145 | |
146 | if (fault & VM_FAULT_RETRY) { |
147 | flags |= FAULT_FLAG_TRIED; |
148 | |
149 | /* |
150 | * No need to mmap_read_unlock(mm) as we would |
151 | * have already released it in __lock_page_or_retry |
152 | * in mm/filemap.c. |
153 | */ |
154 | |
155 | goto retry; |
156 | } |
157 | |
158 | mmap_read_unlock(mm); |
159 | return; |
160 | |
161 | /* |
162 | * Something tried to access memory that isn't in our memory map.. |
163 | * Fix it, but check if it's kernel or user first.. |
164 | */ |
165 | bad_area: |
166 | mmap_read_unlock(mm); |
167 | |
168 | bad_area_nosemaphore: |
169 | /* User mode accesses just cause a SIGSEGV */ |
170 | if (user_mode(regs)) { |
171 | if (unhandled_signal(current, SIGSEGV) && printk_ratelimit()) { |
172 | pr_info("%s: unhandled page fault (%d) at 0x%08lx, " |
173 | "cause %ld\n" , current->comm, SIGSEGV, address, cause); |
174 | show_regs(regs); |
175 | } |
176 | _exception(SIGSEGV, regs, code, address); |
177 | return; |
178 | } |
179 | |
180 | no_context: |
181 | /* Are we prepared to handle this kernel fault? */ |
182 | if (fixup_exception(regs)) |
183 | return; |
184 | |
185 | /* |
186 | * Oops. The kernel tried to access some bad page. We'll have to |
187 | * terminate things with extreme prejudice. |
188 | */ |
189 | bust_spinlocks(yes: 1); |
190 | |
191 | pr_alert("Unable to handle kernel %s at virtual address %08lx" , |
192 | address < PAGE_SIZE ? "NULL pointer dereference" : |
193 | "paging request" , address); |
194 | pr_alert("ea = %08lx, ra = %08lx, cause = %ld\n" , regs->ea, regs->ra, |
195 | cause); |
196 | panic(fmt: "Oops" ); |
197 | return; |
198 | |
199 | /* |
200 | * We ran out of memory, or some other thing happened to us that made |
201 | * us unable to handle the page fault gracefully. |
202 | */ |
203 | out_of_memory: |
204 | mmap_read_unlock(mm); |
205 | if (!user_mode(regs)) |
206 | goto no_context; |
207 | pagefault_out_of_memory(); |
208 | return; |
209 | |
210 | do_sigbus: |
211 | mmap_read_unlock(mm); |
212 | |
213 | /* Kernel mode? Handle exceptions or die */ |
214 | if (!user_mode(regs)) |
215 | goto no_context; |
216 | |
217 | _exception(SIGBUS, regs, BUS_ADRERR, address); |
218 | return; |
219 | |
220 | vmalloc_fault: |
221 | { |
222 | /* |
223 | * Synchronize this task's top level page-table |
224 | * with the 'reference' page table. |
225 | * |
226 | * Do _not_ use "tsk" here. We might be inside |
227 | * an interrupt in the middle of a task switch.. |
228 | */ |
229 | int offset = pgd_index(address); |
230 | pgd_t *pgd, *pgd_k; |
231 | p4d_t *p4d, *p4d_k; |
232 | pud_t *pud, *pud_k; |
233 | pmd_t *pmd, *pmd_k; |
234 | pte_t *pte_k; |
235 | |
236 | pgd = pgd_current + offset; |
237 | pgd_k = init_mm.pgd + offset; |
238 | |
239 | if (!pgd_present(pgd: *pgd_k)) |
240 | goto no_context; |
241 | set_pgd(pgd, *pgd_k); |
242 | |
243 | p4d = p4d_offset(pgd, address); |
244 | p4d_k = p4d_offset(pgd: pgd_k, address); |
245 | if (!p4d_present(p4d: *p4d_k)) |
246 | goto no_context; |
247 | pud = pud_offset(p4d, address); |
248 | pud_k = pud_offset(p4d: p4d_k, address); |
249 | if (!pud_present(pud: *pud_k)) |
250 | goto no_context; |
251 | pmd = pmd_offset(pud, address); |
252 | pmd_k = pmd_offset(pud: pud_k, address); |
253 | if (!pmd_present(pmd: *pmd_k)) |
254 | goto no_context; |
255 | set_pmd(pmdp: pmd, pmd: *pmd_k); |
256 | |
257 | pte_k = pte_offset_kernel(pmd: pmd_k, address); |
258 | if (!pte_present(a: *pte_k)) |
259 | goto no_context; |
260 | |
261 | flush_tlb_kernel_page(address); |
262 | return; |
263 | } |
264 | } |
265 | |