1 | /* |
2 | * Nios2 TLB handling |
3 | * |
4 | * Copyright (C) 2009, Wind River Systems Inc |
5 | * Implemented by fredrik.markstrom@gmail.com and ivarholmqvist@gmail.com |
6 | * |
7 | * This file is subject to the terms and conditions of the GNU General Public |
8 | * License. See the file "COPYING" in the main directory of this archive |
9 | * for more details. |
10 | */ |
11 | |
12 | #include <linux/init.h> |
13 | #include <linux/sched.h> |
14 | #include <linux/mm.h> |
15 | #include <linux/pagemap.h> |
16 | |
17 | #include <asm/tlb.h> |
18 | #include <asm/mmu_context.h> |
19 | #include <asm/cpuinfo.h> |
20 | |
21 | #define TLB_INDEX_MASK \ |
22 | ((((1UL << (cpuinfo.tlb_ptr_sz - cpuinfo.tlb_num_ways_log2))) - 1) \ |
23 | << PAGE_SHIFT) |
24 | |
25 | static void get_misc_and_pid(unsigned long *misc, unsigned long *pid) |
26 | { |
27 | *misc = RDCTL(CTL_TLBMISC); |
28 | *misc &= (TLBMISC_PID | TLBMISC_WAY); |
29 | *pid = *misc & TLBMISC_PID; |
30 | } |
31 | |
32 | /* |
33 | * This provides a PTEADDR value for addr that will cause a TLB miss |
34 | * (fast TLB miss). TLB invalidation replaces entries with this value. |
35 | */ |
36 | static unsigned long pteaddr_invalid(unsigned long addr) |
37 | { |
38 | return ((addr | 0xC0000000UL) >> PAGE_SHIFT) << 2; |
39 | } |
40 | |
41 | /* |
42 | * This one is only used for pages with the global bit set so we don't care |
43 | * much about the ASID. |
44 | */ |
45 | static void replace_tlb_one_pid(unsigned long addr, unsigned long mmu_pid, unsigned long tlbacc) |
46 | { |
47 | unsigned int way; |
48 | unsigned long org_misc, pid_misc; |
49 | |
50 | /* remember pid/way until we return. */ |
51 | get_misc_and_pid(misc: &org_misc, pid: &pid_misc); |
52 | |
53 | WRCTL(CTL_PTEADDR, (addr >> PAGE_SHIFT) << 2); |
54 | |
55 | for (way = 0; way < cpuinfo.tlb_num_ways; way++) { |
56 | unsigned long pteaddr; |
57 | unsigned long tlbmisc; |
58 | unsigned long pid; |
59 | |
60 | tlbmisc = TLBMISC_RD | (way << TLBMISC_WAY_SHIFT); |
61 | WRCTL(CTL_TLBMISC, tlbmisc); |
62 | |
63 | pteaddr = RDCTL(CTL_PTEADDR); |
64 | if (((pteaddr >> 2) & 0xfffff) != (addr >> PAGE_SHIFT)) |
65 | continue; |
66 | |
67 | tlbmisc = RDCTL(CTL_TLBMISC); |
68 | pid = (tlbmisc >> TLBMISC_PID_SHIFT) & TLBMISC_PID_MASK; |
69 | if (pid != mmu_pid) |
70 | continue; |
71 | |
72 | tlbmisc = (mmu_pid << TLBMISC_PID_SHIFT) | TLBMISC_WE | |
73 | (way << TLBMISC_WAY_SHIFT); |
74 | WRCTL(CTL_TLBMISC, tlbmisc); |
75 | if (tlbacc == 0) |
76 | WRCTL(CTL_PTEADDR, pteaddr_invalid(addr)); |
77 | WRCTL(CTL_TLBACC, tlbacc); |
78 | /* |
79 | * There should be only a single entry that maps a |
80 | * particular {address,pid} so break after a match. |
81 | */ |
82 | break; |
83 | } |
84 | |
85 | WRCTL(CTL_TLBMISC, org_misc); |
86 | } |
87 | |
88 | static void flush_tlb_one_pid(unsigned long addr, unsigned long mmu_pid) |
89 | { |
90 | pr_debug("Flush tlb-entry for vaddr=%#lx\n" , addr); |
91 | |
92 | replace_tlb_one_pid(addr, mmu_pid, tlbacc: 0); |
93 | } |
94 | |
95 | static void reload_tlb_one_pid(unsigned long addr, unsigned long mmu_pid, pte_t pte) |
96 | { |
97 | pr_debug("Reload tlb-entry for vaddr=%#lx\n" , addr); |
98 | |
99 | replace_tlb_one_pid(addr, mmu_pid, tlbacc: pte_val(pte)); |
100 | } |
101 | |
102 | void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, |
103 | unsigned long end) |
104 | { |
105 | unsigned long mmu_pid = get_pid_from_context(&vma->vm_mm->context); |
106 | |
107 | while (start < end) { |
108 | flush_tlb_one_pid(addr: start, mmu_pid); |
109 | start += PAGE_SIZE; |
110 | } |
111 | } |
112 | |
113 | void reload_tlb_page(struct vm_area_struct *vma, unsigned long addr, pte_t pte) |
114 | { |
115 | unsigned long mmu_pid = get_pid_from_context(&vma->vm_mm->context); |
116 | |
117 | reload_tlb_one_pid(addr, mmu_pid, pte); |
118 | } |
119 | |
120 | /* |
121 | * This one is only used for pages with the global bit set so we don't care |
122 | * much about the ASID. |
123 | */ |
124 | static void flush_tlb_one(unsigned long addr) |
125 | { |
126 | unsigned int way; |
127 | unsigned long org_misc, pid_misc; |
128 | |
129 | pr_debug("Flush tlb-entry for vaddr=%#lx\n" , addr); |
130 | |
131 | /* remember pid/way until we return. */ |
132 | get_misc_and_pid(misc: &org_misc, pid: &pid_misc); |
133 | |
134 | WRCTL(CTL_PTEADDR, (addr >> PAGE_SHIFT) << 2); |
135 | |
136 | for (way = 0; way < cpuinfo.tlb_num_ways; way++) { |
137 | unsigned long pteaddr; |
138 | unsigned long tlbmisc; |
139 | |
140 | tlbmisc = TLBMISC_RD | (way << TLBMISC_WAY_SHIFT); |
141 | WRCTL(CTL_TLBMISC, tlbmisc); |
142 | |
143 | pteaddr = RDCTL(CTL_PTEADDR); |
144 | if (((pteaddr >> 2) & 0xfffff) != (addr >> PAGE_SHIFT)) |
145 | continue; |
146 | |
147 | pr_debug("Flush entry by writing way=%dl pid=%ld\n" , |
148 | way, (pid_misc >> TLBMISC_PID_SHIFT)); |
149 | |
150 | tlbmisc = TLBMISC_WE | (way << TLBMISC_WAY_SHIFT); |
151 | WRCTL(CTL_TLBMISC, tlbmisc); |
152 | WRCTL(CTL_PTEADDR, pteaddr_invalid(addr)); |
153 | WRCTL(CTL_TLBACC, 0); |
154 | } |
155 | |
156 | WRCTL(CTL_TLBMISC, org_misc); |
157 | } |
158 | |
159 | void flush_tlb_kernel_range(unsigned long start, unsigned long end) |
160 | { |
161 | while (start < end) { |
162 | flush_tlb_one(addr: start); |
163 | start += PAGE_SIZE; |
164 | } |
165 | } |
166 | |
167 | void dump_tlb_line(unsigned long line) |
168 | { |
169 | unsigned int way; |
170 | unsigned long org_misc; |
171 | |
172 | pr_debug("dump tlb-entries for line=%#lx (addr %08lx)\n" , line, |
173 | line << (PAGE_SHIFT + cpuinfo.tlb_num_ways_log2)); |
174 | |
175 | /* remember pid/way until we return */ |
176 | org_misc = (RDCTL(CTL_TLBMISC) & (TLBMISC_PID | TLBMISC_WAY)); |
177 | |
178 | WRCTL(CTL_PTEADDR, line << 2); |
179 | |
180 | for (way = 0; way < cpuinfo.tlb_num_ways; way++) { |
181 | unsigned long pteaddr; |
182 | unsigned long tlbmisc; |
183 | unsigned long tlbacc; |
184 | |
185 | WRCTL(CTL_TLBMISC, TLBMISC_RD | (way << TLBMISC_WAY_SHIFT)); |
186 | pteaddr = RDCTL(CTL_PTEADDR); |
187 | tlbmisc = RDCTL(CTL_TLBMISC); |
188 | tlbacc = RDCTL(CTL_TLBACC); |
189 | |
190 | if ((tlbacc << PAGE_SHIFT) != 0) { |
191 | pr_debug("-- way:%02x vpn:0x%08lx phys:0x%08lx pid:0x%02lx flags:%c%c%c%c%c\n" , |
192 | way, |
193 | (pteaddr << (PAGE_SHIFT-2)), |
194 | (tlbacc << PAGE_SHIFT), |
195 | ((tlbmisc >> TLBMISC_PID_SHIFT) & |
196 | TLBMISC_PID_MASK), |
197 | (tlbacc & _PAGE_READ ? 'r' : '-'), |
198 | (tlbacc & _PAGE_WRITE ? 'w' : '-'), |
199 | (tlbacc & _PAGE_EXEC ? 'x' : '-'), |
200 | (tlbacc & _PAGE_GLOBAL ? 'g' : '-'), |
201 | (tlbacc & _PAGE_CACHED ? 'c' : '-')); |
202 | } |
203 | } |
204 | |
205 | WRCTL(CTL_TLBMISC, org_misc); |
206 | } |
207 | |
208 | void dump_tlb(void) |
209 | { |
210 | unsigned int i; |
211 | |
212 | for (i = 0; i < cpuinfo.tlb_num_lines; i++) |
213 | dump_tlb_line(i); |
214 | } |
215 | |
216 | void flush_tlb_pid(unsigned long mmu_pid) |
217 | { |
218 | unsigned long addr = 0; |
219 | unsigned int line; |
220 | unsigned int way; |
221 | unsigned long org_misc, pid_misc; |
222 | |
223 | /* remember pid/way until we return */ |
224 | get_misc_and_pid(misc: &org_misc, pid: &pid_misc); |
225 | |
226 | for (line = 0; line < cpuinfo.tlb_num_lines; line++) { |
227 | WRCTL(CTL_PTEADDR, pteaddr_invalid(addr)); |
228 | |
229 | for (way = 0; way < cpuinfo.tlb_num_ways; way++) { |
230 | unsigned long tlbmisc; |
231 | unsigned long pid; |
232 | |
233 | tlbmisc = TLBMISC_RD | (way << TLBMISC_WAY_SHIFT); |
234 | WRCTL(CTL_TLBMISC, tlbmisc); |
235 | tlbmisc = RDCTL(CTL_TLBMISC); |
236 | pid = (tlbmisc >> TLBMISC_PID_SHIFT) & TLBMISC_PID_MASK; |
237 | if (pid != mmu_pid) |
238 | continue; |
239 | |
240 | tlbmisc = TLBMISC_WE | (way << TLBMISC_WAY_SHIFT); |
241 | WRCTL(CTL_TLBMISC, tlbmisc); |
242 | WRCTL(CTL_TLBACC, 0); |
243 | } |
244 | |
245 | addr += PAGE_SIZE; |
246 | } |
247 | |
248 | WRCTL(CTL_TLBMISC, org_misc); |
249 | } |
250 | |
251 | /* |
252 | * All entries common to a mm share an asid. To effectively flush these |
253 | * entries, we just bump the asid. |
254 | */ |
255 | void flush_tlb_mm(struct mm_struct *mm) |
256 | { |
257 | if (current->mm == mm) { |
258 | unsigned long mmu_pid = get_pid_from_context(&mm->context); |
259 | flush_tlb_pid(mmu_pid); |
260 | } else { |
261 | memset(&mm->context, 0, sizeof(mm_context_t)); |
262 | } |
263 | } |
264 | |
265 | void flush_tlb_all(void) |
266 | { |
267 | unsigned long addr = 0; |
268 | unsigned int line; |
269 | unsigned int way; |
270 | unsigned long org_misc, pid_misc; |
271 | |
272 | /* remember pid/way until we return */ |
273 | get_misc_and_pid(misc: &org_misc, pid: &pid_misc); |
274 | |
275 | /* Start at way 0, way is auto-incremented after each TLBACC write */ |
276 | WRCTL(CTL_TLBMISC, TLBMISC_WE); |
277 | |
278 | /* Map each TLB entry to physcal address 0 with no-access and a |
279 | bad ptbase */ |
280 | for (line = 0; line < cpuinfo.tlb_num_lines; line++) { |
281 | WRCTL(CTL_PTEADDR, pteaddr_invalid(addr)); |
282 | for (way = 0; way < cpuinfo.tlb_num_ways; way++) |
283 | WRCTL(CTL_TLBACC, 0); |
284 | |
285 | addr += PAGE_SIZE; |
286 | } |
287 | |
288 | /* restore pid/way */ |
289 | WRCTL(CTL_TLBMISC, org_misc); |
290 | } |
291 | |
292 | void set_mmu_pid(unsigned long pid) |
293 | { |
294 | unsigned long tlbmisc; |
295 | |
296 | tlbmisc = RDCTL(CTL_TLBMISC); |
297 | tlbmisc = (tlbmisc & TLBMISC_WAY); |
298 | tlbmisc |= (pid & TLBMISC_PID_MASK) << TLBMISC_PID_SHIFT; |
299 | WRCTL(CTL_TLBMISC, tlbmisc); |
300 | } |
301 | |