1 | /* |
2 | * This file is subject to the terms and conditions of the GNU General Public |
3 | * License. See the file "COPYING" in the main directory of this archive |
4 | * for more details. |
5 | * |
6 | * Copyright (C) 1996 David S. Miller (davem@davemloft.net) |
7 | * Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002 Ralf Baechle (ralf@gnu.org) |
8 | * Copyright (C) 1999, 2000 Silicon Graphics, Inc. |
9 | */ |
10 | #include <linux/cpu_pm.h> |
11 | #include <linux/hardirq.h> |
12 | #include <linux/init.h> |
13 | #include <linux/highmem.h> |
14 | #include <linux/kernel.h> |
15 | #include <linux/linkage.h> |
16 | #include <linux/preempt.h> |
17 | #include <linux/sched.h> |
18 | #include <linux/smp.h> |
19 | #include <linux/mm.h> |
20 | #include <linux/export.h> |
21 | #include <linux/bitops.h> |
22 | #include <linux/dma-map-ops.h> /* for dma_default_coherent */ |
23 | |
24 | #include <asm/bcache.h> |
25 | #include <asm/bootinfo.h> |
26 | #include <asm/cache.h> |
27 | #include <asm/cacheops.h> |
28 | #include <asm/cpu.h> |
29 | #include <asm/cpu-features.h> |
30 | #include <asm/cpu-type.h> |
31 | #include <asm/io.h> |
32 | #include <asm/page.h> |
33 | #include <asm/r4kcache.h> |
34 | #include <asm/sections.h> |
35 | #include <asm/mmu_context.h> |
36 | #include <asm/cacheflush.h> /* for run_uncached() */ |
37 | #include <asm/traps.h> |
38 | #include <asm/mips-cps.h> |
39 | |
40 | /* |
41 | * Bits describing what cache ops an SMP callback function may perform. |
42 | * |
43 | * R4K_HIT - Virtual user or kernel address based cache operations. The |
44 | * active_mm must be checked before using user addresses, falling |
45 | * back to kmap. |
46 | * R4K_INDEX - Index based cache operations. |
47 | */ |
48 | |
49 | #define R4K_HIT BIT(0) |
50 | #define R4K_INDEX BIT(1) |
51 | |
52 | /** |
53 | * r4k_op_needs_ipi() - Decide if a cache op needs to be done on every core. |
54 | * @type: Type of cache operations (R4K_HIT or R4K_INDEX). |
55 | * |
56 | * Decides whether a cache op needs to be performed on every core in the system. |
57 | * This may change depending on the @type of cache operation, as well as the set |
58 | * of online CPUs, so preemption should be disabled by the caller to prevent CPU |
59 | * hotplug from changing the result. |
60 | * |
61 | * Returns: 1 if the cache operation @type should be done on every core in |
62 | * the system. |
63 | * 0 if the cache operation @type is globalized and only needs to |
64 | * be performed on a simple CPU. |
65 | */ |
66 | static inline bool r4k_op_needs_ipi(unsigned int type) |
67 | { |
68 | /* The MIPS Coherence Manager (CM) globalizes address-based cache ops */ |
69 | if (type == R4K_HIT && mips_cm_present()) |
70 | return false; |
71 | |
72 | /* |
73 | * Hardware doesn't globalize the required cache ops, so SMP calls may |
74 | * be needed, but only if there are foreign CPUs (non-siblings with |
75 | * separate caches). |
76 | */ |
77 | /* cpu_foreign_map[] undeclared when !CONFIG_SMP */ |
78 | #ifdef CONFIG_SMP |
79 | return !cpumask_empty(&cpu_foreign_map[0]); |
80 | #else |
81 | return false; |
82 | #endif |
83 | } |
84 | |
85 | /* |
86 | * Special Variant of smp_call_function for use by cache functions: |
87 | * |
88 | * o No return value |
89 | * o collapses to normal function call on UP kernels |
90 | * o collapses to normal function call on systems with a single shared |
91 | * primary cache. |
92 | * o doesn't disable interrupts on the local CPU |
93 | */ |
94 | static inline void r4k_on_each_cpu(unsigned int type, |
95 | void (*func)(void *info), void *info) |
96 | { |
97 | preempt_disable(); |
98 | if (r4k_op_needs_ipi(type)) |
99 | smp_call_function_many(mask: &cpu_foreign_map[smp_processor_id()], |
100 | func, info, wait: 1); |
101 | func(info); |
102 | preempt_enable(); |
103 | } |
104 | |
105 | /* |
106 | * Must die. |
107 | */ |
108 | static unsigned long icache_size __read_mostly; |
109 | static unsigned long dcache_size __read_mostly; |
110 | static unsigned long vcache_size __read_mostly; |
111 | static unsigned long scache_size __read_mostly; |
112 | |
113 | #define cpu_is_r4600_v1_x() ((read_c0_prid() & 0xfffffff0) == 0x00002010) |
114 | #define cpu_is_r4600_v2_x() ((read_c0_prid() & 0xfffffff0) == 0x00002020) |
115 | |
116 | #define R4600_HIT_CACHEOP_WAR_IMPL \ |
117 | do { \ |
118 | if (IS_ENABLED(CONFIG_WAR_R4600_V2_HIT_CACHEOP) && \ |
119 | cpu_is_r4600_v2_x()) \ |
120 | *(volatile unsigned long *)CKSEG1; \ |
121 | if (IS_ENABLED(CONFIG_WAR_R4600_V1_HIT_CACHEOP)) \ |
122 | __asm__ __volatile__("nop;nop;nop;nop"); \ |
123 | } while (0) |
124 | |
125 | static void (*r4k_blast_dcache_page)(unsigned long addr); |
126 | |
127 | static inline void r4k_blast_dcache_page_dc32(unsigned long addr) |
128 | { |
129 | R4600_HIT_CACHEOP_WAR_IMPL; |
130 | blast_dcache32_page(addr); |
131 | } |
132 | |
133 | static inline void r4k_blast_dcache_page_dc64(unsigned long addr) |
134 | { |
135 | blast_dcache64_page(addr); |
136 | } |
137 | |
138 | static inline void r4k_blast_dcache_page_dc128(unsigned long addr) |
139 | { |
140 | blast_dcache128_page(addr); |
141 | } |
142 | |
143 | static void r4k_blast_dcache_page_setup(void) |
144 | { |
145 | unsigned long dc_lsize = cpu_dcache_line_size(); |
146 | |
147 | switch (dc_lsize) { |
148 | case 0: |
149 | r4k_blast_dcache_page = (void *)cache_noop; |
150 | break; |
151 | case 16: |
152 | r4k_blast_dcache_page = blast_dcache16_page; |
153 | break; |
154 | case 32: |
155 | r4k_blast_dcache_page = r4k_blast_dcache_page_dc32; |
156 | break; |
157 | case 64: |
158 | r4k_blast_dcache_page = r4k_blast_dcache_page_dc64; |
159 | break; |
160 | case 128: |
161 | r4k_blast_dcache_page = r4k_blast_dcache_page_dc128; |
162 | break; |
163 | default: |
164 | break; |
165 | } |
166 | } |
167 | |
168 | #ifndef CONFIG_EVA |
169 | #define r4k_blast_dcache_user_page r4k_blast_dcache_page |
170 | #else |
171 | |
172 | static void (*r4k_blast_dcache_user_page)(unsigned long addr); |
173 | |
174 | static void r4k_blast_dcache_user_page_setup(void) |
175 | { |
176 | unsigned long dc_lsize = cpu_dcache_line_size(); |
177 | |
178 | if (dc_lsize == 0) |
179 | r4k_blast_dcache_user_page = (void *)cache_noop; |
180 | else if (dc_lsize == 16) |
181 | r4k_blast_dcache_user_page = blast_dcache16_user_page; |
182 | else if (dc_lsize == 32) |
183 | r4k_blast_dcache_user_page = blast_dcache32_user_page; |
184 | else if (dc_lsize == 64) |
185 | r4k_blast_dcache_user_page = blast_dcache64_user_page; |
186 | } |
187 | |
188 | #endif |
189 | |
190 | void (* r4k_blast_dcache)(void); |
191 | EXPORT_SYMBOL(r4k_blast_dcache); |
192 | |
193 | static void r4k_blast_dcache_setup(void) |
194 | { |
195 | unsigned long dc_lsize = cpu_dcache_line_size(); |
196 | |
197 | if (dc_lsize == 0) |
198 | r4k_blast_dcache = (void *)cache_noop; |
199 | else if (dc_lsize == 16) |
200 | r4k_blast_dcache = blast_dcache16; |
201 | else if (dc_lsize == 32) |
202 | r4k_blast_dcache = blast_dcache32; |
203 | else if (dc_lsize == 64) |
204 | r4k_blast_dcache = blast_dcache64; |
205 | else if (dc_lsize == 128) |
206 | r4k_blast_dcache = blast_dcache128; |
207 | } |
208 | |
209 | /* force code alignment (used for CONFIG_WAR_TX49XX_ICACHE_INDEX_INV) */ |
210 | #define JUMP_TO_ALIGN(order) \ |
211 | __asm__ __volatile__( \ |
212 | "b\t1f\n\t" \ |
213 | ".align\t" #order "\n\t" \ |
214 | "1:\n\t" \ |
215 | ) |
216 | #define CACHE32_UNROLL32_ALIGN JUMP_TO_ALIGN(10) /* 32 * 32 = 1024 */ |
217 | #define CACHE32_UNROLL32_ALIGN2 JUMP_TO_ALIGN(11) |
218 | |
219 | static inline void blast_r4600_v1_icache32(void) |
220 | { |
221 | unsigned long flags; |
222 | |
223 | local_irq_save(flags); |
224 | blast_icache32(); |
225 | local_irq_restore(flags); |
226 | } |
227 | |
228 | static inline void tx49_blast_icache32(void) |
229 | { |
230 | unsigned long start = INDEX_BASE; |
231 | unsigned long end = start + current_cpu_data.icache.waysize; |
232 | unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit; |
233 | unsigned long ws_end = current_cpu_data.icache.ways << |
234 | current_cpu_data.icache.waybit; |
235 | unsigned long ws, addr; |
236 | |
237 | CACHE32_UNROLL32_ALIGN2; |
238 | /* I'm in even chunk. blast odd chunks */ |
239 | for (ws = 0; ws < ws_end; ws += ws_inc) |
240 | for (addr = start + 0x400; addr < end; addr += 0x400 * 2) |
241 | cache_unroll(32, kernel_cache, Index_Invalidate_I, |
242 | addr | ws, 32); |
243 | CACHE32_UNROLL32_ALIGN; |
244 | /* I'm in odd chunk. blast even chunks */ |
245 | for (ws = 0; ws < ws_end; ws += ws_inc) |
246 | for (addr = start; addr < end; addr += 0x400 * 2) |
247 | cache_unroll(32, kernel_cache, Index_Invalidate_I, |
248 | addr | ws, 32); |
249 | } |
250 | |
251 | static void (* r4k_blast_icache_page)(unsigned long addr); |
252 | |
253 | static void r4k_blast_icache_page_setup(void) |
254 | { |
255 | unsigned long ic_lsize = cpu_icache_line_size(); |
256 | |
257 | if (ic_lsize == 0) |
258 | r4k_blast_icache_page = (void *)cache_noop; |
259 | else if (ic_lsize == 16) |
260 | r4k_blast_icache_page = blast_icache16_page; |
261 | else if (ic_lsize == 32 && current_cpu_type() == CPU_LOONGSON2EF) |
262 | r4k_blast_icache_page = loongson2_blast_icache32_page; |
263 | else if (ic_lsize == 32) |
264 | r4k_blast_icache_page = blast_icache32_page; |
265 | else if (ic_lsize == 64) |
266 | r4k_blast_icache_page = blast_icache64_page; |
267 | else if (ic_lsize == 128) |
268 | r4k_blast_icache_page = blast_icache128_page; |
269 | } |
270 | |
271 | #ifndef CONFIG_EVA |
272 | #define r4k_blast_icache_user_page r4k_blast_icache_page |
273 | #else |
274 | |
275 | static void (*r4k_blast_icache_user_page)(unsigned long addr); |
276 | |
277 | static void r4k_blast_icache_user_page_setup(void) |
278 | { |
279 | unsigned long ic_lsize = cpu_icache_line_size(); |
280 | |
281 | if (ic_lsize == 0) |
282 | r4k_blast_icache_user_page = (void *)cache_noop; |
283 | else if (ic_lsize == 16) |
284 | r4k_blast_icache_user_page = blast_icache16_user_page; |
285 | else if (ic_lsize == 32) |
286 | r4k_blast_icache_user_page = blast_icache32_user_page; |
287 | else if (ic_lsize == 64) |
288 | r4k_blast_icache_user_page = blast_icache64_user_page; |
289 | } |
290 | |
291 | #endif |
292 | |
293 | void (* r4k_blast_icache)(void); |
294 | EXPORT_SYMBOL(r4k_blast_icache); |
295 | |
296 | static void r4k_blast_icache_setup(void) |
297 | { |
298 | unsigned long ic_lsize = cpu_icache_line_size(); |
299 | |
300 | if (ic_lsize == 0) |
301 | r4k_blast_icache = (void *)cache_noop; |
302 | else if (ic_lsize == 16) |
303 | r4k_blast_icache = blast_icache16; |
304 | else if (ic_lsize == 32) { |
305 | if (IS_ENABLED(CONFIG_WAR_R4600_V1_INDEX_ICACHEOP) && |
306 | cpu_is_r4600_v1_x()) |
307 | r4k_blast_icache = blast_r4600_v1_icache32; |
308 | else if (IS_ENABLED(CONFIG_WAR_TX49XX_ICACHE_INDEX_INV)) |
309 | r4k_blast_icache = tx49_blast_icache32; |
310 | else if (current_cpu_type() == CPU_LOONGSON2EF) |
311 | r4k_blast_icache = loongson2_blast_icache32; |
312 | else |
313 | r4k_blast_icache = blast_icache32; |
314 | } else if (ic_lsize == 64) |
315 | r4k_blast_icache = blast_icache64; |
316 | else if (ic_lsize == 128) |
317 | r4k_blast_icache = blast_icache128; |
318 | } |
319 | |
320 | static void (* r4k_blast_scache_page)(unsigned long addr); |
321 | |
322 | static void r4k_blast_scache_page_setup(void) |
323 | { |
324 | unsigned long sc_lsize = cpu_scache_line_size(); |
325 | |
326 | if (scache_size == 0) |
327 | r4k_blast_scache_page = (void *)cache_noop; |
328 | else if (sc_lsize == 16) |
329 | r4k_blast_scache_page = blast_scache16_page; |
330 | else if (sc_lsize == 32) |
331 | r4k_blast_scache_page = blast_scache32_page; |
332 | else if (sc_lsize == 64) |
333 | r4k_blast_scache_page = blast_scache64_page; |
334 | else if (sc_lsize == 128) |
335 | r4k_blast_scache_page = blast_scache128_page; |
336 | } |
337 | |
338 | static void (* r4k_blast_scache)(void); |
339 | |
340 | static void r4k_blast_scache_setup(void) |
341 | { |
342 | unsigned long sc_lsize = cpu_scache_line_size(); |
343 | |
344 | if (scache_size == 0) |
345 | r4k_blast_scache = (void *)cache_noop; |
346 | else if (sc_lsize == 16) |
347 | r4k_blast_scache = blast_scache16; |
348 | else if (sc_lsize == 32) |
349 | r4k_blast_scache = blast_scache32; |
350 | else if (sc_lsize == 64) |
351 | r4k_blast_scache = blast_scache64; |
352 | else if (sc_lsize == 128) |
353 | r4k_blast_scache = blast_scache128; |
354 | } |
355 | |
356 | static void (*r4k_blast_scache_node)(long node); |
357 | |
358 | static void r4k_blast_scache_node_setup(void) |
359 | { |
360 | unsigned long sc_lsize = cpu_scache_line_size(); |
361 | |
362 | if (current_cpu_type() != CPU_LOONGSON64) |
363 | r4k_blast_scache_node = (void *)cache_noop; |
364 | else if (sc_lsize == 16) |
365 | r4k_blast_scache_node = blast_scache16_node; |
366 | else if (sc_lsize == 32) |
367 | r4k_blast_scache_node = blast_scache32_node; |
368 | else if (sc_lsize == 64) |
369 | r4k_blast_scache_node = blast_scache64_node; |
370 | else if (sc_lsize == 128) |
371 | r4k_blast_scache_node = blast_scache128_node; |
372 | } |
373 | |
374 | static inline void local_r4k___flush_cache_all(void * args) |
375 | { |
376 | switch (current_cpu_type()) { |
377 | case CPU_LOONGSON2EF: |
378 | case CPU_R4000SC: |
379 | case CPU_R4000MC: |
380 | case CPU_R4400SC: |
381 | case CPU_R4400MC: |
382 | case CPU_R10000: |
383 | case CPU_R12000: |
384 | case CPU_R14000: |
385 | case CPU_R16000: |
386 | /* |
387 | * These caches are inclusive caches, that is, if something |
388 | * is not cached in the S-cache, we know it also won't be |
389 | * in one of the primary caches. |
390 | */ |
391 | r4k_blast_scache(); |
392 | break; |
393 | |
394 | case CPU_LOONGSON64: |
395 | /* Use get_ebase_cpunum() for both NUMA=y/n */ |
396 | r4k_blast_scache_node(get_ebase_cpunum() >> 2); |
397 | break; |
398 | |
399 | case CPU_BMIPS5000: |
400 | r4k_blast_scache(); |
401 | __sync(); |
402 | break; |
403 | |
404 | default: |
405 | r4k_blast_dcache(); |
406 | r4k_blast_icache(); |
407 | break; |
408 | } |
409 | } |
410 | |
411 | static void r4k___flush_cache_all(void) |
412 | { |
413 | r4k_on_each_cpu(R4K_INDEX, func: local_r4k___flush_cache_all, NULL); |
414 | } |
415 | |
416 | /** |
417 | * has_valid_asid() - Determine if an mm already has an ASID. |
418 | * @mm: Memory map. |
419 | * @type: R4K_HIT or R4K_INDEX, type of cache op. |
420 | * |
421 | * Determines whether @mm already has an ASID on any of the CPUs which cache ops |
422 | * of type @type within an r4k_on_each_cpu() call will affect. If |
423 | * r4k_on_each_cpu() does an SMP call to a single VPE in each core, then the |
424 | * scope of the operation is confined to sibling CPUs, otherwise all online CPUs |
425 | * will need to be checked. |
426 | * |
427 | * Must be called in non-preemptive context. |
428 | * |
429 | * Returns: 1 if the CPUs affected by @type cache ops have an ASID for @mm. |
430 | * 0 otherwise. |
431 | */ |
432 | static inline int has_valid_asid(const struct mm_struct *mm, unsigned int type) |
433 | { |
434 | unsigned int i; |
435 | const cpumask_t *mask = cpu_present_mask; |
436 | |
437 | if (cpu_has_mmid) |
438 | return cpu_context(0, mm) != 0; |
439 | |
440 | /* cpu_sibling_map[] undeclared when !CONFIG_SMP */ |
441 | #ifdef CONFIG_SMP |
442 | /* |
443 | * If r4k_on_each_cpu does SMP calls, it does them to a single VPE in |
444 | * each foreign core, so we only need to worry about siblings. |
445 | * Otherwise we need to worry about all present CPUs. |
446 | */ |
447 | if (r4k_op_needs_ipi(type)) |
448 | mask = &cpu_sibling_map[smp_processor_id()]; |
449 | #endif |
450 | for_each_cpu(i, mask) |
451 | if (cpu_context(i, mm)) |
452 | return 1; |
453 | return 0; |
454 | } |
455 | |
456 | static void r4k__flush_cache_vmap(void) |
457 | { |
458 | r4k_blast_dcache(); |
459 | } |
460 | |
461 | static void r4k__flush_cache_vunmap(void) |
462 | { |
463 | r4k_blast_dcache(); |
464 | } |
465 | |
466 | /* |
467 | * Note: flush_tlb_range() assumes flush_cache_range() sufficiently flushes |
468 | * whole caches when vma is executable. |
469 | */ |
470 | static inline void local_r4k_flush_cache_range(void * args) |
471 | { |
472 | struct vm_area_struct *vma = args; |
473 | int exec = vma->vm_flags & VM_EXEC; |
474 | |
475 | if (!has_valid_asid(mm: vma->vm_mm, R4K_INDEX)) |
476 | return; |
477 | |
478 | /* |
479 | * If dcache can alias, we must blast it since mapping is changing. |
480 | * If executable, we must ensure any dirty lines are written back far |
481 | * enough to be visible to icache. |
482 | */ |
483 | if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) |
484 | r4k_blast_dcache(); |
485 | /* If executable, blast stale lines from icache */ |
486 | if (exec) |
487 | r4k_blast_icache(); |
488 | } |
489 | |
490 | static void r4k_flush_cache_range(struct vm_area_struct *vma, |
491 | unsigned long start, unsigned long end) |
492 | { |
493 | int exec = vma->vm_flags & VM_EXEC; |
494 | |
495 | if (cpu_has_dc_aliases || exec) |
496 | r4k_on_each_cpu(R4K_INDEX, func: local_r4k_flush_cache_range, info: vma); |
497 | } |
498 | |
499 | static inline void local_r4k_flush_cache_mm(void * args) |
500 | { |
501 | struct mm_struct *mm = args; |
502 | |
503 | if (!has_valid_asid(mm, R4K_INDEX)) |
504 | return; |
505 | |
506 | /* |
507 | * Kludge alert. For obscure reasons R4000SC and R4400SC go nuts if we |
508 | * only flush the primary caches but R1x000 behave sane ... |
509 | * R4000SC and R4400SC indexed S-cache ops also invalidate primary |
510 | * caches, so we can bail out early. |
511 | */ |
512 | if (current_cpu_type() == CPU_R4000SC || |
513 | current_cpu_type() == CPU_R4000MC || |
514 | current_cpu_type() == CPU_R4400SC || |
515 | current_cpu_type() == CPU_R4400MC) { |
516 | r4k_blast_scache(); |
517 | return; |
518 | } |
519 | |
520 | r4k_blast_dcache(); |
521 | } |
522 | |
523 | static void r4k_flush_cache_mm(struct mm_struct *mm) |
524 | { |
525 | if (!cpu_has_dc_aliases) |
526 | return; |
527 | |
528 | r4k_on_each_cpu(R4K_INDEX, func: local_r4k_flush_cache_mm, info: mm); |
529 | } |
530 | |
531 | struct flush_cache_page_args { |
532 | struct vm_area_struct *vma; |
533 | unsigned long addr; |
534 | unsigned long pfn; |
535 | }; |
536 | |
537 | static inline void local_r4k_flush_cache_page(void *args) |
538 | { |
539 | struct flush_cache_page_args *fcp_args = args; |
540 | struct vm_area_struct *vma = fcp_args->vma; |
541 | unsigned long addr = fcp_args->addr; |
542 | struct page *page = pfn_to_page(fcp_args->pfn); |
543 | int exec = vma->vm_flags & VM_EXEC; |
544 | struct mm_struct *mm = vma->vm_mm; |
545 | int map_coherent = 0; |
546 | pmd_t *pmdp; |
547 | pte_t *ptep; |
548 | void *vaddr; |
549 | |
550 | /* |
551 | * If owns no valid ASID yet, cannot possibly have gotten |
552 | * this page into the cache. |
553 | */ |
554 | if (!has_valid_asid(mm, R4K_HIT)) |
555 | return; |
556 | |
557 | addr &= PAGE_MASK; |
558 | pmdp = pmd_off(mm, va: addr); |
559 | ptep = pte_offset_kernel(pmd: pmdp, address: addr); |
560 | |
561 | /* |
562 | * If the page isn't marked valid, the page cannot possibly be |
563 | * in the cache. |
564 | */ |
565 | if (!(pte_present(a: *ptep))) |
566 | return; |
567 | |
568 | if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID)) |
569 | vaddr = NULL; |
570 | else { |
571 | struct folio *folio = page_folio(page); |
572 | /* |
573 | * Use kmap_coherent or kmap_atomic to do flushes for |
574 | * another ASID than the current one. |
575 | */ |
576 | map_coherent = (cpu_has_dc_aliases && |
577 | folio_mapped(folio) && |
578 | !folio_test_dcache_dirty(folio)); |
579 | if (map_coherent) |
580 | vaddr = kmap_coherent(page, addr); |
581 | else |
582 | vaddr = kmap_atomic(page); |
583 | addr = (unsigned long)vaddr; |
584 | } |
585 | |
586 | if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) { |
587 | vaddr ? r4k_blast_dcache_page(addr) : |
588 | r4k_blast_dcache_user_page(addr); |
589 | if (exec && !cpu_icache_snoops_remote_store) |
590 | r4k_blast_scache_page(addr); |
591 | } |
592 | if (exec) { |
593 | if (vaddr && cpu_has_vtag_icache && mm == current->active_mm) { |
594 | drop_mmu_context(mm); |
595 | } else |
596 | vaddr ? r4k_blast_icache_page(addr) : |
597 | r4k_blast_icache_user_page(addr); |
598 | } |
599 | |
600 | if (vaddr) { |
601 | if (map_coherent) |
602 | kunmap_coherent(); |
603 | else |
604 | kunmap_atomic(vaddr); |
605 | } |
606 | } |
607 | |
608 | static void r4k_flush_cache_page(struct vm_area_struct *vma, |
609 | unsigned long addr, unsigned long pfn) |
610 | { |
611 | struct flush_cache_page_args args; |
612 | |
613 | args.vma = vma; |
614 | args.addr = addr; |
615 | args.pfn = pfn; |
616 | |
617 | r4k_on_each_cpu(R4K_HIT, func: local_r4k_flush_cache_page, info: &args); |
618 | } |
619 | |
620 | static inline void local_r4k_flush_data_cache_page(void * addr) |
621 | { |
622 | r4k_blast_dcache_page((unsigned long) addr); |
623 | } |
624 | |
625 | static void r4k_flush_data_cache_page(unsigned long addr) |
626 | { |
627 | if (in_atomic()) |
628 | local_r4k_flush_data_cache_page(addr: (void *)addr); |
629 | else |
630 | r4k_on_each_cpu(R4K_HIT, func: local_r4k_flush_data_cache_page, |
631 | info: (void *) addr); |
632 | } |
633 | |
634 | struct flush_icache_range_args { |
635 | unsigned long start; |
636 | unsigned long end; |
637 | unsigned int type; |
638 | bool user; |
639 | }; |
640 | |
641 | static inline void __local_r4k_flush_icache_range(unsigned long start, |
642 | unsigned long end, |
643 | unsigned int type, |
644 | bool user) |
645 | { |
646 | if (!cpu_has_ic_fills_f_dc) { |
647 | if (type == R4K_INDEX || |
648 | (type & R4K_INDEX && end - start >= dcache_size)) { |
649 | r4k_blast_dcache(); |
650 | } else { |
651 | R4600_HIT_CACHEOP_WAR_IMPL; |
652 | if (user) |
653 | protected_blast_dcache_range(start, end); |
654 | else |
655 | blast_dcache_range(start, end); |
656 | } |
657 | } |
658 | |
659 | if (type == R4K_INDEX || |
660 | (type & R4K_INDEX && end - start > icache_size)) |
661 | r4k_blast_icache(); |
662 | else { |
663 | switch (boot_cpu_type()) { |
664 | case CPU_LOONGSON2EF: |
665 | protected_loongson2_blast_icache_range(start, end); |
666 | break; |
667 | |
668 | default: |
669 | if (user) |
670 | protected_blast_icache_range(start, end); |
671 | else |
672 | blast_icache_range(start, end); |
673 | break; |
674 | } |
675 | } |
676 | } |
677 | |
678 | static inline void local_r4k_flush_icache_range(unsigned long start, |
679 | unsigned long end) |
680 | { |
681 | __local_r4k_flush_icache_range(start, end, R4K_HIT | R4K_INDEX, user: false); |
682 | } |
683 | |
684 | static inline void local_r4k_flush_icache_user_range(unsigned long start, |
685 | unsigned long end) |
686 | { |
687 | __local_r4k_flush_icache_range(start, end, R4K_HIT | R4K_INDEX, user: true); |
688 | } |
689 | |
690 | static inline void local_r4k_flush_icache_range_ipi(void *args) |
691 | { |
692 | struct flush_icache_range_args *fir_args = args; |
693 | unsigned long start = fir_args->start; |
694 | unsigned long end = fir_args->end; |
695 | unsigned int type = fir_args->type; |
696 | bool user = fir_args->user; |
697 | |
698 | __local_r4k_flush_icache_range(start, end, type, user); |
699 | } |
700 | |
701 | static void __r4k_flush_icache_range(unsigned long start, unsigned long end, |
702 | bool user) |
703 | { |
704 | struct flush_icache_range_args args; |
705 | unsigned long size, cache_size; |
706 | |
707 | args.start = start; |
708 | args.end = end; |
709 | args.type = R4K_HIT | R4K_INDEX; |
710 | args.user = user; |
711 | |
712 | /* |
713 | * Indexed cache ops require an SMP call. |
714 | * Consider if that can or should be avoided. |
715 | */ |
716 | preempt_disable(); |
717 | if (r4k_op_needs_ipi(R4K_INDEX) && !r4k_op_needs_ipi(R4K_HIT)) { |
718 | /* |
719 | * If address-based cache ops don't require an SMP call, then |
720 | * use them exclusively for small flushes. |
721 | */ |
722 | size = end - start; |
723 | cache_size = icache_size; |
724 | if (!cpu_has_ic_fills_f_dc) { |
725 | size *= 2; |
726 | cache_size += dcache_size; |
727 | } |
728 | if (size <= cache_size) |
729 | args.type &= ~R4K_INDEX; |
730 | } |
731 | r4k_on_each_cpu(type: args.type, func: local_r4k_flush_icache_range_ipi, info: &args); |
732 | preempt_enable(); |
733 | instruction_hazard(); |
734 | } |
735 | |
736 | static void r4k_flush_icache_range(unsigned long start, unsigned long end) |
737 | { |
738 | return __r4k_flush_icache_range(start, end, user: false); |
739 | } |
740 | |
741 | static void r4k_flush_icache_user_range(unsigned long start, unsigned long end) |
742 | { |
743 | return __r4k_flush_icache_range(start, end, user: true); |
744 | } |
745 | |
746 | #ifdef CONFIG_DMA_NONCOHERENT |
747 | |
748 | static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size) |
749 | { |
750 | /* Catch bad driver code */ |
751 | if (WARN_ON(size == 0)) |
752 | return; |
753 | |
754 | preempt_disable(); |
755 | if (cpu_has_inclusive_pcaches) { |
756 | if (size >= scache_size) { |
757 | if (current_cpu_type() != CPU_LOONGSON64) |
758 | r4k_blast_scache(); |
759 | else |
760 | r4k_blast_scache_node(pa_to_nid(addr)); |
761 | } else { |
762 | blast_scache_range(addr, addr + size); |
763 | } |
764 | preempt_enable(); |
765 | __sync(); |
766 | return; |
767 | } |
768 | |
769 | /* |
770 | * Either no secondary cache or the available caches don't have the |
771 | * subset property so we have to flush the primary caches |
772 | * explicitly. |
773 | * If we would need IPI to perform an INDEX-type operation, then |
774 | * we have to use the HIT-type alternative as IPI cannot be used |
775 | * here due to interrupts possibly being disabled. |
776 | */ |
777 | if (!r4k_op_needs_ipi(R4K_INDEX) && size >= dcache_size) { |
778 | r4k_blast_dcache(); |
779 | } else { |
780 | R4600_HIT_CACHEOP_WAR_IMPL; |
781 | blast_dcache_range(addr, addr + size); |
782 | } |
783 | preempt_enable(); |
784 | |
785 | bc_wback_inv(addr, size); |
786 | __sync(); |
787 | } |
788 | |
789 | static void prefetch_cache_inv(unsigned long addr, unsigned long size) |
790 | { |
791 | unsigned int linesz = cpu_scache_line_size(); |
792 | unsigned long addr0 = addr, addr1; |
793 | |
794 | addr0 &= ~(linesz - 1); |
795 | addr1 = (addr0 + size - 1) & ~(linesz - 1); |
796 | |
797 | protected_writeback_scache_line(addr0); |
798 | if (likely(addr1 != addr0)) |
799 | protected_writeback_scache_line(addr1); |
800 | else |
801 | return; |
802 | |
803 | addr0 += linesz; |
804 | if (likely(addr1 != addr0)) |
805 | protected_writeback_scache_line(addr0); |
806 | else |
807 | return; |
808 | |
809 | addr1 -= linesz; |
810 | if (likely(addr1 > addr0)) |
811 | protected_writeback_scache_line(addr0); |
812 | } |
813 | |
814 | static void r4k_dma_cache_inv(unsigned long addr, unsigned long size) |
815 | { |
816 | /* Catch bad driver code */ |
817 | if (WARN_ON(size == 0)) |
818 | return; |
819 | |
820 | preempt_disable(); |
821 | |
822 | if (current_cpu_type() == CPU_BMIPS5000) |
823 | prefetch_cache_inv(addr, size); |
824 | |
825 | if (cpu_has_inclusive_pcaches) { |
826 | if (size >= scache_size) { |
827 | if (current_cpu_type() != CPU_LOONGSON64) |
828 | r4k_blast_scache(); |
829 | else |
830 | r4k_blast_scache_node(pa_to_nid(addr)); |
831 | } else { |
832 | /* |
833 | * There is no clearly documented alignment requirement |
834 | * for the cache instruction on MIPS processors and |
835 | * some processors, among them the RM5200 and RM7000 |
836 | * QED processors will throw an address error for cache |
837 | * hit ops with insufficient alignment. Solved by |
838 | * aligning the address to cache line size. |
839 | */ |
840 | blast_inv_scache_range(addr, addr + size); |
841 | } |
842 | preempt_enable(); |
843 | __sync(); |
844 | return; |
845 | } |
846 | |
847 | if (!r4k_op_needs_ipi(R4K_INDEX) && size >= dcache_size) { |
848 | r4k_blast_dcache(); |
849 | } else { |
850 | R4600_HIT_CACHEOP_WAR_IMPL; |
851 | blast_inv_dcache_range(addr, addr + size); |
852 | } |
853 | preempt_enable(); |
854 | |
855 | bc_inv(addr, size); |
856 | __sync(); |
857 | } |
858 | #endif /* CONFIG_DMA_NONCOHERENT */ |
859 | |
860 | static void r4k_flush_icache_all(void) |
861 | { |
862 | if (cpu_has_vtag_icache) |
863 | r4k_blast_icache(); |
864 | } |
865 | |
866 | struct flush_kernel_vmap_range_args { |
867 | unsigned long vaddr; |
868 | int size; |
869 | }; |
870 | |
871 | static inline void local_r4k_flush_kernel_vmap_range_index(void *args) |
872 | { |
873 | /* |
874 | * Aliases only affect the primary caches so don't bother with |
875 | * S-caches or T-caches. |
876 | */ |
877 | r4k_blast_dcache(); |
878 | } |
879 | |
880 | static inline void local_r4k_flush_kernel_vmap_range(void *args) |
881 | { |
882 | struct flush_kernel_vmap_range_args *vmra = args; |
883 | unsigned long vaddr = vmra->vaddr; |
884 | int size = vmra->size; |
885 | |
886 | /* |
887 | * Aliases only affect the primary caches so don't bother with |
888 | * S-caches or T-caches. |
889 | */ |
890 | R4600_HIT_CACHEOP_WAR_IMPL; |
891 | blast_dcache_range(vaddr, vaddr + size); |
892 | } |
893 | |
894 | static void r4k_flush_kernel_vmap_range(unsigned long vaddr, int size) |
895 | { |
896 | struct flush_kernel_vmap_range_args args; |
897 | |
898 | args.vaddr = (unsigned long) vaddr; |
899 | args.size = size; |
900 | |
901 | if (size >= dcache_size) |
902 | r4k_on_each_cpu(R4K_INDEX, |
903 | func: local_r4k_flush_kernel_vmap_range_index, NULL); |
904 | else |
905 | r4k_on_each_cpu(R4K_HIT, func: local_r4k_flush_kernel_vmap_range, |
906 | info: &args); |
907 | } |
908 | |
909 | static inline void rm7k_erratum31(void) |
910 | { |
911 | const unsigned long ic_lsize = 32; |
912 | unsigned long addr; |
913 | |
914 | /* RM7000 erratum #31. The icache is screwed at startup. */ |
915 | write_c0_taglo(0); |
916 | write_c0_taghi(0); |
917 | |
918 | for (addr = INDEX_BASE; addr <= INDEX_BASE + 4096; addr += ic_lsize) { |
919 | __asm__ __volatile__ ( |
920 | ".set push\n\t" |
921 | ".set noreorder\n\t" |
922 | ".set mips3\n\t" |
923 | "cache\t%1, 0(%0)\n\t" |
924 | "cache\t%1, 0x1000(%0)\n\t" |
925 | "cache\t%1, 0x2000(%0)\n\t" |
926 | "cache\t%1, 0x3000(%0)\n\t" |
927 | "cache\t%2, 0(%0)\n\t" |
928 | "cache\t%2, 0x1000(%0)\n\t" |
929 | "cache\t%2, 0x2000(%0)\n\t" |
930 | "cache\t%2, 0x3000(%0)\n\t" |
931 | "cache\t%1, 0(%0)\n\t" |
932 | "cache\t%1, 0x1000(%0)\n\t" |
933 | "cache\t%1, 0x2000(%0)\n\t" |
934 | "cache\t%1, 0x3000(%0)\n\t" |
935 | ".set pop\n" |
936 | : |
937 | : "r" (addr), "i" (Index_Store_Tag_I), "i" (Fill_I)); |
938 | } |
939 | } |
940 | |
941 | static inline int alias_74k_erratum(struct cpuinfo_mips *c) |
942 | { |
943 | unsigned int imp = c->processor_id & PRID_IMP_MASK; |
944 | unsigned int rev = c->processor_id & PRID_REV_MASK; |
945 | int present = 0; |
946 | |
947 | /* |
948 | * Early versions of the 74K do not update the cache tags on a |
949 | * vtag miss/ptag hit which can occur in the case of KSEG0/KUSEG |
950 | * aliases. In this case it is better to treat the cache as always |
951 | * having aliases. Also disable the synonym tag update feature |
952 | * where available. In this case no opportunistic tag update will |
953 | * happen where a load causes a virtual address miss but a physical |
954 | * address hit during a D-cache look-up. |
955 | */ |
956 | switch (imp) { |
957 | case PRID_IMP_74K: |
958 | if (rev <= PRID_REV_ENCODE_332(2, 4, 0)) |
959 | present = 1; |
960 | if (rev == PRID_REV_ENCODE_332(2, 4, 0)) |
961 | write_c0_config6(read_c0_config6() | MTI_CONF6_SYND); |
962 | break; |
963 | case PRID_IMP_1074K: |
964 | if (rev <= PRID_REV_ENCODE_332(1, 1, 0)) { |
965 | present = 1; |
966 | write_c0_config6(read_c0_config6() | MTI_CONF6_SYND); |
967 | } |
968 | break; |
969 | default: |
970 | BUG(); |
971 | } |
972 | |
973 | return present; |
974 | } |
975 | |
976 | static void b5k_instruction_hazard(void) |
977 | { |
978 | __sync(); |
979 | __sync(); |
980 | __asm__ __volatile__( |
981 | " nop; nop; nop; nop; nop; nop; nop; nop\n" |
982 | " nop; nop; nop; nop; nop; nop; nop; nop\n" |
983 | " nop; nop; nop; nop; nop; nop; nop; nop\n" |
984 | " nop; nop; nop; nop; nop; nop; nop; nop\n" |
985 | : : : "memory" ); |
986 | } |
987 | |
988 | static char *way_string[] = { NULL, "direct mapped" , "2-way" , |
989 | "3-way" , "4-way" , "5-way" , "6-way" , "7-way" , "8-way" , |
990 | "9-way" , "10-way" , "11-way" , "12-way" , |
991 | "13-way" , "14-way" , "15-way" , "16-way" , |
992 | }; |
993 | |
994 | static void probe_pcache(void) |
995 | { |
996 | struct cpuinfo_mips *c = ¤t_cpu_data; |
997 | unsigned int config = read_c0_config(); |
998 | unsigned int prid = read_c0_prid(); |
999 | int has_74k_erratum = 0; |
1000 | unsigned long config1; |
1001 | unsigned int lsize; |
1002 | |
1003 | switch (current_cpu_type()) { |
1004 | case CPU_R4600: /* QED style two way caches? */ |
1005 | case CPU_R4700: |
1006 | case CPU_R5000: |
1007 | case CPU_NEVADA: |
1008 | icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); |
1009 | c->icache.linesz = 16 << ((config & CONF_IB) >> 5); |
1010 | c->icache.ways = 2; |
1011 | c->icache.waybit = __ffs(icache_size/2); |
1012 | |
1013 | dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); |
1014 | c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); |
1015 | c->dcache.ways = 2; |
1016 | c->dcache.waybit= __ffs(dcache_size/2); |
1017 | |
1018 | c->options |= MIPS_CPU_CACHE_CDEX_P; |
1019 | break; |
1020 | |
1021 | case CPU_R5500: |
1022 | icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); |
1023 | c->icache.linesz = 16 << ((config & CONF_IB) >> 5); |
1024 | c->icache.ways = 2; |
1025 | c->icache.waybit= 0; |
1026 | |
1027 | dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); |
1028 | c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); |
1029 | c->dcache.ways = 2; |
1030 | c->dcache.waybit = 0; |
1031 | |
1032 | c->options |= MIPS_CPU_CACHE_CDEX_P | MIPS_CPU_PREFETCH; |
1033 | break; |
1034 | |
1035 | case CPU_TX49XX: |
1036 | icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); |
1037 | c->icache.linesz = 16 << ((config & CONF_IB) >> 5); |
1038 | c->icache.ways = 4; |
1039 | c->icache.waybit= 0; |
1040 | |
1041 | dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); |
1042 | c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); |
1043 | c->dcache.ways = 4; |
1044 | c->dcache.waybit = 0; |
1045 | |
1046 | c->options |= MIPS_CPU_CACHE_CDEX_P; |
1047 | c->options |= MIPS_CPU_PREFETCH; |
1048 | break; |
1049 | |
1050 | case CPU_R4000PC: |
1051 | case CPU_R4000SC: |
1052 | case CPU_R4000MC: |
1053 | case CPU_R4400PC: |
1054 | case CPU_R4400SC: |
1055 | case CPU_R4400MC: |
1056 | case CPU_R4300: |
1057 | icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); |
1058 | c->icache.linesz = 16 << ((config & CONF_IB) >> 5); |
1059 | c->icache.ways = 1; |
1060 | c->icache.waybit = 0; /* doesn't matter */ |
1061 | |
1062 | dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); |
1063 | c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); |
1064 | c->dcache.ways = 1; |
1065 | c->dcache.waybit = 0; /* does not matter */ |
1066 | |
1067 | c->options |= MIPS_CPU_CACHE_CDEX_P; |
1068 | break; |
1069 | |
1070 | case CPU_R10000: |
1071 | case CPU_R12000: |
1072 | case CPU_R14000: |
1073 | case CPU_R16000: |
1074 | icache_size = 1 << (12 + ((config & R10K_CONF_IC) >> 29)); |
1075 | c->icache.linesz = 64; |
1076 | c->icache.ways = 2; |
1077 | c->icache.waybit = 0; |
1078 | |
1079 | dcache_size = 1 << (12 + ((config & R10K_CONF_DC) >> 26)); |
1080 | c->dcache.linesz = 32; |
1081 | c->dcache.ways = 2; |
1082 | c->dcache.waybit = 0; |
1083 | |
1084 | c->options |= MIPS_CPU_PREFETCH; |
1085 | break; |
1086 | |
1087 | case CPU_RM7000: |
1088 | rm7k_erratum31(); |
1089 | |
1090 | icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); |
1091 | c->icache.linesz = 16 << ((config & CONF_IB) >> 5); |
1092 | c->icache.ways = 4; |
1093 | c->icache.waybit = __ffs(icache_size / c->icache.ways); |
1094 | |
1095 | dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); |
1096 | c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); |
1097 | c->dcache.ways = 4; |
1098 | c->dcache.waybit = __ffs(dcache_size / c->dcache.ways); |
1099 | |
1100 | c->options |= MIPS_CPU_CACHE_CDEX_P; |
1101 | c->options |= MIPS_CPU_PREFETCH; |
1102 | break; |
1103 | |
1104 | case CPU_LOONGSON2EF: |
1105 | icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); |
1106 | c->icache.linesz = 16 << ((config & CONF_IB) >> 5); |
1107 | if (prid & 0x3) |
1108 | c->icache.ways = 4; |
1109 | else |
1110 | c->icache.ways = 2; |
1111 | c->icache.waybit = 0; |
1112 | |
1113 | dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); |
1114 | c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); |
1115 | if (prid & 0x3) |
1116 | c->dcache.ways = 4; |
1117 | else |
1118 | c->dcache.ways = 2; |
1119 | c->dcache.waybit = 0; |
1120 | break; |
1121 | |
1122 | case CPU_LOONGSON64: |
1123 | config1 = read_c0_config1(); |
1124 | lsize = (config1 >> 19) & 7; |
1125 | if (lsize) |
1126 | c->icache.linesz = 2 << lsize; |
1127 | else |
1128 | c->icache.linesz = 0; |
1129 | c->icache.sets = 64 << ((config1 >> 22) & 7); |
1130 | c->icache.ways = 1 + ((config1 >> 16) & 7); |
1131 | icache_size = c->icache.sets * |
1132 | c->icache.ways * |
1133 | c->icache.linesz; |
1134 | c->icache.waybit = 0; |
1135 | |
1136 | lsize = (config1 >> 10) & 7; |
1137 | if (lsize) |
1138 | c->dcache.linesz = 2 << lsize; |
1139 | else |
1140 | c->dcache.linesz = 0; |
1141 | c->dcache.sets = 64 << ((config1 >> 13) & 7); |
1142 | c->dcache.ways = 1 + ((config1 >> 7) & 7); |
1143 | dcache_size = c->dcache.sets * |
1144 | c->dcache.ways * |
1145 | c->dcache.linesz; |
1146 | c->dcache.waybit = 0; |
1147 | if ((c->processor_id & (PRID_IMP_MASK | PRID_REV_MASK)) >= |
1148 | (PRID_IMP_LOONGSON_64C | PRID_REV_LOONGSON3A_R2_0) || |
1149 | (c->processor_id & PRID_IMP_MASK) == PRID_IMP_LOONGSON_64R) |
1150 | c->options |= MIPS_CPU_PREFETCH; |
1151 | break; |
1152 | |
1153 | case CPU_CAVIUM_OCTEON3: |
1154 | /* For now lie about the number of ways. */ |
1155 | c->icache.linesz = 128; |
1156 | c->icache.sets = 16; |
1157 | c->icache.ways = 8; |
1158 | c->icache.flags |= MIPS_CACHE_VTAG; |
1159 | icache_size = c->icache.sets * c->icache.ways * c->icache.linesz; |
1160 | |
1161 | c->dcache.linesz = 128; |
1162 | c->dcache.ways = 8; |
1163 | c->dcache.sets = 8; |
1164 | dcache_size = c->dcache.sets * c->dcache.ways * c->dcache.linesz; |
1165 | c->options |= MIPS_CPU_PREFETCH; |
1166 | break; |
1167 | |
1168 | default: |
1169 | if (!(config & MIPS_CONF_M)) |
1170 | panic(fmt: "Don't know how to probe P-caches on this cpu." ); |
1171 | |
1172 | /* |
1173 | * So we seem to be a MIPS32 or MIPS64 CPU |
1174 | * So let's probe the I-cache ... |
1175 | */ |
1176 | config1 = read_c0_config1(); |
1177 | |
1178 | lsize = (config1 >> 19) & 7; |
1179 | |
1180 | /* IL == 7 is reserved */ |
1181 | if (lsize == 7) |
1182 | panic(fmt: "Invalid icache line size" ); |
1183 | |
1184 | c->icache.linesz = lsize ? 2 << lsize : 0; |
1185 | |
1186 | c->icache.sets = 32 << (((config1 >> 22) + 1) & 7); |
1187 | c->icache.ways = 1 + ((config1 >> 16) & 7); |
1188 | |
1189 | icache_size = c->icache.sets * |
1190 | c->icache.ways * |
1191 | c->icache.linesz; |
1192 | c->icache.waybit = __ffs(icache_size/c->icache.ways); |
1193 | |
1194 | if (config & MIPS_CONF_VI) |
1195 | c->icache.flags |= MIPS_CACHE_VTAG; |
1196 | |
1197 | /* |
1198 | * Now probe the MIPS32 / MIPS64 data cache. |
1199 | */ |
1200 | c->dcache.flags = 0; |
1201 | |
1202 | lsize = (config1 >> 10) & 7; |
1203 | |
1204 | /* DL == 7 is reserved */ |
1205 | if (lsize == 7) |
1206 | panic(fmt: "Invalid dcache line size" ); |
1207 | |
1208 | c->dcache.linesz = lsize ? 2 << lsize : 0; |
1209 | |
1210 | c->dcache.sets = 32 << (((config1 >> 13) + 1) & 7); |
1211 | c->dcache.ways = 1 + ((config1 >> 7) & 7); |
1212 | |
1213 | dcache_size = c->dcache.sets * |
1214 | c->dcache.ways * |
1215 | c->dcache.linesz; |
1216 | c->dcache.waybit = __ffs(dcache_size/c->dcache.ways); |
1217 | |
1218 | c->options |= MIPS_CPU_PREFETCH; |
1219 | break; |
1220 | } |
1221 | |
1222 | /* |
1223 | * Processor configuration sanity check for the R4000SC erratum |
1224 | * #5. With page sizes larger than 32kB there is no possibility |
1225 | * to get a VCE exception anymore so we don't care about this |
1226 | * misconfiguration. The case is rather theoretical anyway; |
1227 | * presumably no vendor is shipping his hardware in the "bad" |
1228 | * configuration. |
1229 | */ |
1230 | if ((prid & PRID_IMP_MASK) == PRID_IMP_R4000 && |
1231 | (prid & PRID_REV_MASK) < PRID_REV_R4400 && |
1232 | !(config & CONF_SC) && c->icache.linesz != 16 && |
1233 | PAGE_SIZE <= 0x8000) |
1234 | panic(fmt: "Improper R4000SC processor configuration detected" ); |
1235 | |
1236 | /* compute a couple of other cache variables */ |
1237 | c->icache.waysize = icache_size / c->icache.ways; |
1238 | c->dcache.waysize = dcache_size / c->dcache.ways; |
1239 | |
1240 | c->icache.sets = c->icache.linesz ? |
1241 | icache_size / (c->icache.linesz * c->icache.ways) : 0; |
1242 | c->dcache.sets = c->dcache.linesz ? |
1243 | dcache_size / (c->dcache.linesz * c->dcache.ways) : 0; |
1244 | |
1245 | /* |
1246 | * R1x000 P-caches are odd in a positive way. They're 32kB 2-way |
1247 | * virtually indexed so normally would suffer from aliases. So |
1248 | * normally they'd suffer from aliases but magic in the hardware deals |
1249 | * with that for us so we don't need to take care ourselves. |
1250 | */ |
1251 | switch (current_cpu_type()) { |
1252 | case CPU_20KC: |
1253 | case CPU_25KF: |
1254 | case CPU_I6400: |
1255 | case CPU_I6500: |
1256 | case CPU_SB1: |
1257 | case CPU_SB1A: |
1258 | c->dcache.flags |= MIPS_CACHE_PINDEX; |
1259 | break; |
1260 | |
1261 | case CPU_R10000: |
1262 | case CPU_R12000: |
1263 | case CPU_R14000: |
1264 | case CPU_R16000: |
1265 | break; |
1266 | |
1267 | case CPU_74K: |
1268 | case CPU_1074K: |
1269 | has_74k_erratum = alias_74k_erratum(c); |
1270 | fallthrough; |
1271 | case CPU_M14KC: |
1272 | case CPU_M14KEC: |
1273 | case CPU_24K: |
1274 | case CPU_34K: |
1275 | case CPU_1004K: |
1276 | case CPU_INTERAPTIV: |
1277 | case CPU_P5600: |
1278 | case CPU_PROAPTIV: |
1279 | case CPU_M5150: |
1280 | case CPU_QEMU_GENERIC: |
1281 | case CPU_P6600: |
1282 | case CPU_M6250: |
1283 | if (!(read_c0_config7() & MIPS_CONF7_IAR) && |
1284 | (c->icache.waysize > PAGE_SIZE)) |
1285 | c->icache.flags |= MIPS_CACHE_ALIASES; |
1286 | if (!has_74k_erratum && (read_c0_config7() & MIPS_CONF7_AR)) { |
1287 | /* |
1288 | * Effectively physically indexed dcache, |
1289 | * thus no virtual aliases. |
1290 | */ |
1291 | c->dcache.flags |= MIPS_CACHE_PINDEX; |
1292 | break; |
1293 | } |
1294 | fallthrough; |
1295 | default: |
1296 | if (has_74k_erratum || c->dcache.waysize > PAGE_SIZE) |
1297 | c->dcache.flags |= MIPS_CACHE_ALIASES; |
1298 | } |
1299 | |
1300 | /* Physically indexed caches don't suffer from virtual aliasing */ |
1301 | if (c->dcache.flags & MIPS_CACHE_PINDEX) |
1302 | c->dcache.flags &= ~MIPS_CACHE_ALIASES; |
1303 | |
1304 | /* |
1305 | * In systems with CM the icache fills from L2 or closer caches, and |
1306 | * thus sees remote stores without needing to write them back any |
1307 | * further than that. |
1308 | */ |
1309 | if (mips_cm_present()) |
1310 | c->icache.flags |= MIPS_IC_SNOOPS_REMOTE; |
1311 | |
1312 | switch (current_cpu_type()) { |
1313 | case CPU_20KC: |
1314 | /* |
1315 | * Some older 20Kc chips doesn't have the 'VI' bit in |
1316 | * the config register. |
1317 | */ |
1318 | c->icache.flags |= MIPS_CACHE_VTAG; |
1319 | break; |
1320 | |
1321 | case CPU_ALCHEMY: |
1322 | case CPU_I6400: |
1323 | case CPU_I6500: |
1324 | c->icache.flags |= MIPS_CACHE_IC_F_DC; |
1325 | break; |
1326 | |
1327 | case CPU_BMIPS5000: |
1328 | c->icache.flags |= MIPS_CACHE_IC_F_DC; |
1329 | /* Cache aliases are handled in hardware; allow HIGHMEM */ |
1330 | c->dcache.flags &= ~MIPS_CACHE_ALIASES; |
1331 | break; |
1332 | |
1333 | case CPU_LOONGSON2EF: |
1334 | /* |
1335 | * LOONGSON2 has 4 way icache, but when using indexed cache op, |
1336 | * one op will act on all 4 ways |
1337 | */ |
1338 | c->icache.ways = 1; |
1339 | } |
1340 | |
1341 | pr_info("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n" , |
1342 | icache_size >> 10, |
1343 | c->icache.flags & MIPS_CACHE_VTAG ? "VIVT" : "VIPT" , |
1344 | way_string[c->icache.ways], c->icache.linesz); |
1345 | |
1346 | pr_info("Primary data cache %ldkB, %s, %s, %s, linesize %d bytes\n" , |
1347 | dcache_size >> 10, way_string[c->dcache.ways], |
1348 | (c->dcache.flags & MIPS_CACHE_PINDEX) ? "PIPT" : "VIPT" , |
1349 | (c->dcache.flags & MIPS_CACHE_ALIASES) ? |
1350 | "cache aliases" : "no aliases" , |
1351 | c->dcache.linesz); |
1352 | } |
1353 | |
1354 | static void probe_vcache(void) |
1355 | { |
1356 | struct cpuinfo_mips *c = ¤t_cpu_data; |
1357 | unsigned int config2, lsize; |
1358 | |
1359 | if (current_cpu_type() != CPU_LOONGSON64) |
1360 | return; |
1361 | |
1362 | config2 = read_c0_config2(); |
1363 | if ((lsize = ((config2 >> 20) & 15))) |
1364 | c->vcache.linesz = 2 << lsize; |
1365 | else |
1366 | c->vcache.linesz = lsize; |
1367 | |
1368 | c->vcache.sets = 64 << ((config2 >> 24) & 15); |
1369 | c->vcache.ways = 1 + ((config2 >> 16) & 15); |
1370 | |
1371 | vcache_size = c->vcache.sets * c->vcache.ways * c->vcache.linesz; |
1372 | |
1373 | c->vcache.waybit = 0; |
1374 | c->vcache.waysize = vcache_size / c->vcache.ways; |
1375 | |
1376 | pr_info("Unified victim cache %ldkB %s, linesize %d bytes.\n" , |
1377 | vcache_size >> 10, way_string[c->vcache.ways], c->vcache.linesz); |
1378 | } |
1379 | |
1380 | /* |
1381 | * If you even _breathe_ on this function, look at the gcc output and make sure |
1382 | * it does not pop things on and off the stack for the cache sizing loop that |
1383 | * executes in KSEG1 space or else you will crash and burn badly. You have |
1384 | * been warned. |
1385 | */ |
1386 | static int probe_scache(void) |
1387 | { |
1388 | unsigned long flags, addr, begin, end, pow2; |
1389 | unsigned int config = read_c0_config(); |
1390 | struct cpuinfo_mips *c = ¤t_cpu_data; |
1391 | |
1392 | if (config & CONF_SC) |
1393 | return 0; |
1394 | |
1395 | begin = (unsigned long) &_stext; |
1396 | begin &= ~((4 * 1024 * 1024) - 1); |
1397 | end = begin + (4 * 1024 * 1024); |
1398 | |
1399 | /* |
1400 | * This is such a bitch, you'd think they would make it easy to do |
1401 | * this. Away you daemons of stupidity! |
1402 | */ |
1403 | local_irq_save(flags); |
1404 | |
1405 | /* Fill each size-multiple cache line with a valid tag. */ |
1406 | pow2 = (64 * 1024); |
1407 | for (addr = begin; addr < end; addr = (begin + pow2)) { |
1408 | unsigned long *p = (unsigned long *) addr; |
1409 | __asm__ __volatile__("nop" : : "r" (*p)); /* whee... */ |
1410 | pow2 <<= 1; |
1411 | } |
1412 | |
1413 | /* Load first line with zero (therefore invalid) tag. */ |
1414 | write_c0_taglo(0); |
1415 | write_c0_taghi(0); |
1416 | __asm__ __volatile__("nop; nop; nop; nop;" ); /* avoid the hazard */ |
1417 | cache_op(Index_Store_Tag_I, begin); |
1418 | cache_op(Index_Store_Tag_D, begin); |
1419 | cache_op(Index_Store_Tag_SD, begin); |
1420 | |
1421 | /* Now search for the wrap around point. */ |
1422 | pow2 = (128 * 1024); |
1423 | for (addr = begin + (128 * 1024); addr < end; addr = begin + pow2) { |
1424 | cache_op(Index_Load_Tag_SD, addr); |
1425 | __asm__ __volatile__("nop; nop; nop; nop;" ); /* hazard... */ |
1426 | if (!read_c0_taglo()) |
1427 | break; |
1428 | pow2 <<= 1; |
1429 | } |
1430 | local_irq_restore(flags); |
1431 | addr -= begin; |
1432 | |
1433 | scache_size = addr; |
1434 | c->scache.linesz = 16 << ((config & R4K_CONF_SB) >> 22); |
1435 | c->scache.ways = 1; |
1436 | c->scache.waybit = 0; /* does not matter */ |
1437 | |
1438 | return 1; |
1439 | } |
1440 | |
1441 | static void loongson2_sc_init(void) |
1442 | { |
1443 | struct cpuinfo_mips *c = ¤t_cpu_data; |
1444 | |
1445 | scache_size = 512*1024; |
1446 | c->scache.linesz = 32; |
1447 | c->scache.ways = 4; |
1448 | c->scache.waybit = 0; |
1449 | c->scache.waysize = scache_size / (c->scache.ways); |
1450 | c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways); |
1451 | pr_info("Unified secondary cache %ldkB %s, linesize %d bytes.\n" , |
1452 | scache_size >> 10, way_string[c->scache.ways], c->scache.linesz); |
1453 | |
1454 | c->options |= MIPS_CPU_INCLUSIVE_CACHES; |
1455 | } |
1456 | |
1457 | static void loongson3_sc_init(void) |
1458 | { |
1459 | struct cpuinfo_mips *c = ¤t_cpu_data; |
1460 | unsigned int config2, lsize; |
1461 | |
1462 | config2 = read_c0_config2(); |
1463 | lsize = (config2 >> 4) & 15; |
1464 | if (lsize) |
1465 | c->scache.linesz = 2 << lsize; |
1466 | else |
1467 | c->scache.linesz = 0; |
1468 | c->scache.sets = 64 << ((config2 >> 8) & 15); |
1469 | c->scache.ways = 1 + (config2 & 15); |
1470 | |
1471 | /* Loongson-3 has 4-Scache banks, while Loongson-2K have only 2 banks */ |
1472 | if ((c->processor_id & PRID_IMP_MASK) == PRID_IMP_LOONGSON_64R) |
1473 | c->scache.sets *= 2; |
1474 | else |
1475 | c->scache.sets *= 4; |
1476 | |
1477 | scache_size = c->scache.sets * c->scache.ways * c->scache.linesz; |
1478 | |
1479 | c->scache.waybit = 0; |
1480 | c->scache.waysize = scache_size / c->scache.ways; |
1481 | pr_info("Unified secondary cache %ldkB %s, linesize %d bytes.\n" , |
1482 | scache_size >> 10, way_string[c->scache.ways], c->scache.linesz); |
1483 | if (scache_size) |
1484 | c->options |= MIPS_CPU_INCLUSIVE_CACHES; |
1485 | return; |
1486 | } |
1487 | |
1488 | static void setup_scache(void) |
1489 | { |
1490 | struct cpuinfo_mips *c = ¤t_cpu_data; |
1491 | unsigned int config = read_c0_config(); |
1492 | int sc_present = 0; |
1493 | |
1494 | /* |
1495 | * Do the probing thing on R4000SC and R4400SC processors. Other |
1496 | * processors don't have a S-cache that would be relevant to the |
1497 | * Linux memory management. |
1498 | */ |
1499 | switch (current_cpu_type()) { |
1500 | case CPU_R4000SC: |
1501 | case CPU_R4000MC: |
1502 | case CPU_R4400SC: |
1503 | case CPU_R4400MC: |
1504 | sc_present = run_uncached(probe_scache); |
1505 | if (sc_present) |
1506 | c->options |= MIPS_CPU_CACHE_CDEX_S; |
1507 | break; |
1508 | |
1509 | case CPU_R10000: |
1510 | case CPU_R12000: |
1511 | case CPU_R14000: |
1512 | case CPU_R16000: |
1513 | scache_size = 0x80000 << ((config & R10K_CONF_SS) >> 16); |
1514 | c->scache.linesz = 64 << ((config >> 13) & 1); |
1515 | c->scache.ways = 2; |
1516 | c->scache.waybit= 0; |
1517 | sc_present = 1; |
1518 | break; |
1519 | |
1520 | case CPU_R5000: |
1521 | case CPU_NEVADA: |
1522 | #ifdef CONFIG_R5000_CPU_SCACHE |
1523 | r5k_sc_init(); |
1524 | #endif |
1525 | return; |
1526 | |
1527 | case CPU_RM7000: |
1528 | #ifdef CONFIG_RM7000_CPU_SCACHE |
1529 | rm7k_sc_init(); |
1530 | #endif |
1531 | return; |
1532 | |
1533 | case CPU_LOONGSON2EF: |
1534 | loongson2_sc_init(); |
1535 | return; |
1536 | |
1537 | case CPU_LOONGSON64: |
1538 | loongson3_sc_init(); |
1539 | return; |
1540 | |
1541 | case CPU_CAVIUM_OCTEON3: |
1542 | /* don't need to worry about L2, fully coherent */ |
1543 | return; |
1544 | |
1545 | default: |
1546 | if (c->isa_level & (MIPS_CPU_ISA_M32R1 | MIPS_CPU_ISA_M64R1 | |
1547 | MIPS_CPU_ISA_M32R2 | MIPS_CPU_ISA_M64R2 | |
1548 | MIPS_CPU_ISA_M32R5 | MIPS_CPU_ISA_M64R5 | |
1549 | MIPS_CPU_ISA_M32R6 | MIPS_CPU_ISA_M64R6)) { |
1550 | #ifdef CONFIG_MIPS_CPU_SCACHE |
1551 | if (mips_sc_init ()) { |
1552 | scache_size = c->scache.ways * c->scache.sets * c->scache.linesz; |
1553 | printk("MIPS secondary cache %ldkB, %s, linesize %d bytes.\n" , |
1554 | scache_size >> 10, |
1555 | way_string[c->scache.ways], c->scache.linesz); |
1556 | |
1557 | if (current_cpu_type() == CPU_BMIPS5000) |
1558 | c->options |= MIPS_CPU_INCLUSIVE_CACHES; |
1559 | } |
1560 | |
1561 | #else |
1562 | if (!(c->scache.flags & MIPS_CACHE_NOT_PRESENT)) |
1563 | panic(fmt: "Dunno how to handle MIPS32 / MIPS64 second level cache" ); |
1564 | #endif |
1565 | return; |
1566 | } |
1567 | sc_present = 0; |
1568 | } |
1569 | |
1570 | if (!sc_present) |
1571 | return; |
1572 | |
1573 | /* compute a couple of other cache variables */ |
1574 | c->scache.waysize = scache_size / c->scache.ways; |
1575 | |
1576 | c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways); |
1577 | |
1578 | printk("Unified secondary cache %ldkB %s, linesize %d bytes.\n" , |
1579 | scache_size >> 10, way_string[c->scache.ways], c->scache.linesz); |
1580 | |
1581 | c->options |= MIPS_CPU_INCLUSIVE_CACHES; |
1582 | } |
1583 | |
1584 | void au1x00_fixup_config_od(void) |
1585 | { |
1586 | /* |
1587 | * c0_config.od (bit 19) was write only (and read as 0) |
1588 | * on the early revisions of Alchemy SOCs. It disables the bus |
1589 | * transaction overlapping and needs to be set to fix various errata. |
1590 | */ |
1591 | switch (read_c0_prid()) { |
1592 | case 0x00030100: /* Au1000 DA */ |
1593 | case 0x00030201: /* Au1000 HA */ |
1594 | case 0x00030202: /* Au1000 HB */ |
1595 | case 0x01030200: /* Au1500 AB */ |
1596 | /* |
1597 | * Au1100 errata actually keeps silence about this bit, so we set it |
1598 | * just in case for those revisions that require it to be set according |
1599 | * to the (now gone) cpu table. |
1600 | */ |
1601 | case 0x02030200: /* Au1100 AB */ |
1602 | case 0x02030201: /* Au1100 BA */ |
1603 | case 0x02030202: /* Au1100 BC */ |
1604 | set_c0_config(1 << 19); |
1605 | break; |
1606 | } |
1607 | } |
1608 | |
1609 | /* CP0 hazard avoidance. */ |
1610 | #define NXP_BARRIER() \ |
1611 | __asm__ __volatile__( \ |
1612 | ".set noreorder\n\t" \ |
1613 | "nop; nop; nop; nop; nop; nop;\n\t" \ |
1614 | ".set reorder\n\t") |
1615 | |
1616 | static void nxp_pr4450_fixup_config(void) |
1617 | { |
1618 | unsigned long config0; |
1619 | |
1620 | config0 = read_c0_config(); |
1621 | |
1622 | /* clear all three cache coherency fields */ |
1623 | config0 &= ~(0x7 | (7 << 25) | (7 << 28)); |
1624 | config0 |= (((_page_cachable_default >> _CACHE_SHIFT) << 0) | |
1625 | ((_page_cachable_default >> _CACHE_SHIFT) << 25) | |
1626 | ((_page_cachable_default >> _CACHE_SHIFT) << 28)); |
1627 | write_c0_config(config0); |
1628 | NXP_BARRIER(); |
1629 | } |
1630 | |
1631 | static int cca = -1; |
1632 | |
1633 | static int __init cca_setup(char *str) |
1634 | { |
1635 | get_option(str: &str, pint: &cca); |
1636 | |
1637 | return 0; |
1638 | } |
1639 | |
1640 | early_param("cca" , cca_setup); |
1641 | |
1642 | static void coherency_setup(void) |
1643 | { |
1644 | if (cca < 0 || cca > 7) |
1645 | cca = read_c0_config() & CONF_CM_CMASK; |
1646 | _page_cachable_default = cca << _CACHE_SHIFT; |
1647 | |
1648 | pr_debug("Using cache attribute %d\n" , cca); |
1649 | change_c0_config(CONF_CM_CMASK, cca); |
1650 | |
1651 | /* |
1652 | * c0_status.cu=0 specifies that updates by the sc instruction use |
1653 | * the coherency mode specified by the TLB; 1 means cacheable |
1654 | * coherent update on write will be used. Not all processors have |
1655 | * this bit and; some wire it to zero, others like Toshiba had the |
1656 | * silly idea of putting something else there ... |
1657 | */ |
1658 | switch (current_cpu_type()) { |
1659 | case CPU_R4000PC: |
1660 | case CPU_R4000SC: |
1661 | case CPU_R4000MC: |
1662 | case CPU_R4400PC: |
1663 | case CPU_R4400SC: |
1664 | case CPU_R4400MC: |
1665 | clear_c0_config(CONF_CU); |
1666 | break; |
1667 | /* |
1668 | * We need to catch the early Alchemy SOCs with |
1669 | * the write-only co_config.od bit and set it back to one on: |
1670 | * Au1000 rev DA, HA, HB; Au1100 AB, BA, BC, Au1500 AB |
1671 | */ |
1672 | case CPU_ALCHEMY: |
1673 | au1x00_fixup_config_od(); |
1674 | break; |
1675 | |
1676 | case PRID_IMP_PR4450: |
1677 | nxp_pr4450_fixup_config(); |
1678 | break; |
1679 | } |
1680 | } |
1681 | |
1682 | static void r4k_cache_error_setup(void) |
1683 | { |
1684 | extern char __weak except_vec2_generic; |
1685 | extern char __weak except_vec2_sb1; |
1686 | |
1687 | switch (current_cpu_type()) { |
1688 | case CPU_SB1: |
1689 | case CPU_SB1A: |
1690 | set_uncached_handler(0x100, &except_vec2_sb1, 0x80); |
1691 | break; |
1692 | |
1693 | default: |
1694 | set_uncached_handler(0x100, &except_vec2_generic, 0x80); |
1695 | break; |
1696 | } |
1697 | } |
1698 | |
1699 | void r4k_cache_init(void) |
1700 | { |
1701 | extern void build_clear_page(void); |
1702 | extern void build_copy_page(void); |
1703 | struct cpuinfo_mips *c = ¤t_cpu_data; |
1704 | |
1705 | probe_pcache(); |
1706 | probe_vcache(); |
1707 | setup_scache(); |
1708 | |
1709 | r4k_blast_dcache_page_setup(); |
1710 | r4k_blast_dcache_setup(); |
1711 | r4k_blast_icache_page_setup(); |
1712 | r4k_blast_icache_setup(); |
1713 | r4k_blast_scache_page_setup(); |
1714 | r4k_blast_scache_setup(); |
1715 | r4k_blast_scache_node_setup(); |
1716 | #ifdef CONFIG_EVA |
1717 | r4k_blast_dcache_user_page_setup(); |
1718 | r4k_blast_icache_user_page_setup(); |
1719 | #endif |
1720 | |
1721 | /* |
1722 | * Some MIPS32 and MIPS64 processors have physically indexed caches. |
1723 | * This code supports virtually indexed processors and will be |
1724 | * unnecessarily inefficient on physically indexed processors. |
1725 | */ |
1726 | if (c->dcache.linesz && cpu_has_dc_aliases) |
1727 | shm_align_mask = max_t( unsigned long, |
1728 | c->dcache.sets * c->dcache.linesz - 1, |
1729 | PAGE_SIZE - 1); |
1730 | else |
1731 | shm_align_mask = PAGE_SIZE-1; |
1732 | |
1733 | __flush_cache_vmap = r4k__flush_cache_vmap; |
1734 | __flush_cache_vunmap = r4k__flush_cache_vunmap; |
1735 | |
1736 | flush_cache_all = cache_noop; |
1737 | __flush_cache_all = r4k___flush_cache_all; |
1738 | flush_cache_mm = r4k_flush_cache_mm; |
1739 | flush_cache_page = r4k_flush_cache_page; |
1740 | flush_cache_range = r4k_flush_cache_range; |
1741 | |
1742 | __flush_kernel_vmap_range = r4k_flush_kernel_vmap_range; |
1743 | |
1744 | flush_icache_all = r4k_flush_icache_all; |
1745 | flush_data_cache_page = r4k_flush_data_cache_page; |
1746 | flush_icache_range = r4k_flush_icache_range; |
1747 | local_flush_icache_range = local_r4k_flush_icache_range; |
1748 | __flush_icache_user_range = r4k_flush_icache_user_range; |
1749 | __local_flush_icache_user_range = local_r4k_flush_icache_user_range; |
1750 | |
1751 | #ifdef CONFIG_DMA_NONCOHERENT |
1752 | _dma_cache_wback_inv = r4k_dma_cache_wback_inv; |
1753 | _dma_cache_wback = r4k_dma_cache_wback_inv; |
1754 | _dma_cache_inv = r4k_dma_cache_inv; |
1755 | #endif /* CONFIG_DMA_NONCOHERENT */ |
1756 | |
1757 | build_clear_page(); |
1758 | build_copy_page(); |
1759 | |
1760 | /* |
1761 | * We want to run CMP kernels on core with and without coherent |
1762 | * caches. Therefore, do not use CONFIG_MIPS_CMP to decide whether |
1763 | * or not to flush caches. |
1764 | */ |
1765 | local_r4k___flush_cache_all(NULL); |
1766 | |
1767 | coherency_setup(); |
1768 | board_cache_error_setup = r4k_cache_error_setup; |
1769 | |
1770 | /* |
1771 | * Per-CPU overrides |
1772 | */ |
1773 | switch (current_cpu_type()) { |
1774 | case CPU_BMIPS4350: |
1775 | case CPU_BMIPS4380: |
1776 | /* No IPI is needed because all CPUs share the same D$ */ |
1777 | flush_data_cache_page = r4k_blast_dcache_page; |
1778 | break; |
1779 | case CPU_BMIPS5000: |
1780 | /* We lose our superpowers if L2 is disabled */ |
1781 | if (c->scache.flags & MIPS_CACHE_NOT_PRESENT) |
1782 | break; |
1783 | |
1784 | /* I$ fills from D$ just by emptying the write buffers */ |
1785 | flush_cache_page = (void *)b5k_instruction_hazard; |
1786 | flush_cache_range = (void *)b5k_instruction_hazard; |
1787 | flush_data_cache_page = (void *)b5k_instruction_hazard; |
1788 | flush_icache_range = (void *)b5k_instruction_hazard; |
1789 | local_flush_icache_range = (void *)b5k_instruction_hazard; |
1790 | |
1791 | |
1792 | /* Optimization: an L2 flush implicitly flushes the L1 */ |
1793 | current_cpu_data.options |= MIPS_CPU_INCLUSIVE_CACHES; |
1794 | break; |
1795 | case CPU_LOONGSON64: |
1796 | /* Loongson-3 maintains cache coherency by hardware */ |
1797 | __flush_cache_all = cache_noop; |
1798 | __flush_cache_vmap = cache_noop; |
1799 | __flush_cache_vunmap = cache_noop; |
1800 | __flush_kernel_vmap_range = (void *)cache_noop; |
1801 | flush_cache_mm = (void *)cache_noop; |
1802 | flush_cache_page = (void *)cache_noop; |
1803 | flush_cache_range = (void *)cache_noop; |
1804 | flush_icache_all = (void *)cache_noop; |
1805 | flush_data_cache_page = (void *)cache_noop; |
1806 | break; |
1807 | } |
1808 | } |
1809 | |
1810 | static int r4k_cache_pm_notifier(struct notifier_block *self, unsigned long cmd, |
1811 | void *v) |
1812 | { |
1813 | switch (cmd) { |
1814 | case CPU_PM_ENTER_FAILED: |
1815 | case CPU_PM_EXIT: |
1816 | coherency_setup(); |
1817 | break; |
1818 | } |
1819 | |
1820 | return NOTIFY_OK; |
1821 | } |
1822 | |
1823 | static struct notifier_block r4k_cache_pm_notifier_block = { |
1824 | .notifier_call = r4k_cache_pm_notifier, |
1825 | }; |
1826 | |
1827 | static int __init r4k_cache_init_pm(void) |
1828 | { |
1829 | return cpu_pm_register_notifier(nb: &r4k_cache_pm_notifier_block); |
1830 | } |
1831 | arch_initcall(r4k_cache_init_pm); |
1832 | |