1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * arch/sh/mm/cache.c |
4 | * |
5 | * Copyright (C) 1999, 2000, 2002 Niibe Yutaka |
6 | * Copyright (C) 2002 - 2010 Paul Mundt |
7 | */ |
8 | #include <linux/mm.h> |
9 | #include <linux/init.h> |
10 | #include <linux/mutex.h> |
11 | #include <linux/fs.h> |
12 | #include <linux/smp.h> |
13 | #include <linux/highmem.h> |
14 | #include <linux/module.h> |
15 | #include <asm/mmu_context.h> |
16 | #include <asm/cacheflush.h> |
17 | |
18 | void (*local_flush_cache_all)(void *args) = cache_noop; |
19 | void (*local_flush_cache_mm)(void *args) = cache_noop; |
20 | void (*local_flush_cache_dup_mm)(void *args) = cache_noop; |
21 | void (*local_flush_cache_page)(void *args) = cache_noop; |
22 | void (*local_flush_cache_range)(void *args) = cache_noop; |
23 | void (*local_flush_dcache_folio)(void *args) = cache_noop; |
24 | void (*local_flush_icache_range)(void *args) = cache_noop; |
25 | void (*local_flush_icache_folio)(void *args) = cache_noop; |
26 | void (*local_flush_cache_sigtramp)(void *args) = cache_noop; |
27 | |
28 | void (*__flush_wback_region)(void *start, int size); |
29 | EXPORT_SYMBOL(__flush_wback_region); |
30 | void (*__flush_purge_region)(void *start, int size); |
31 | EXPORT_SYMBOL(__flush_purge_region); |
32 | void (*__flush_invalidate_region)(void *start, int size); |
33 | EXPORT_SYMBOL(__flush_invalidate_region); |
34 | |
35 | static inline void noop__flush_region(void *start, int size) |
36 | { |
37 | } |
38 | |
39 | static inline void cacheop_on_each_cpu(void (*func) (void *info), void *info, |
40 | int wait) |
41 | { |
42 | preempt_disable(); |
43 | |
44 | /* Needing IPI for cross-core flush is SHX3-specific. */ |
45 | #ifdef CONFIG_CPU_SHX3 |
46 | /* |
47 | * It's possible that this gets called early on when IRQs are |
48 | * still disabled due to ioremapping by the boot CPU, so don't |
49 | * even attempt IPIs unless there are other CPUs online. |
50 | */ |
51 | if (num_online_cpus() > 1) |
52 | smp_call_function(func, info, wait); |
53 | #endif |
54 | |
55 | func(info); |
56 | |
57 | preempt_enable(); |
58 | } |
59 | |
60 | void copy_to_user_page(struct vm_area_struct *vma, struct page *page, |
61 | unsigned long vaddr, void *dst, const void *src, |
62 | unsigned long len) |
63 | { |
64 | struct folio *folio = page_folio(page); |
65 | |
66 | if (boot_cpu_data.dcache.n_aliases && folio_mapped(folio) && |
67 | test_bit(PG_dcache_clean, &folio->flags)) { |
68 | void *vto = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK); |
69 | memcpy(vto, src, len); |
70 | kunmap_coherent(vto); |
71 | } else { |
72 | memcpy(dst, src, len); |
73 | if (boot_cpu_data.dcache.n_aliases) |
74 | clear_bit(PG_dcache_clean, &folio->flags); |
75 | } |
76 | |
77 | if (vma->vm_flags & VM_EXEC) |
78 | flush_cache_page(vma, vaddr, page_to_pfn(page)); |
79 | } |
80 | |
81 | void copy_from_user_page(struct vm_area_struct *vma, struct page *page, |
82 | unsigned long vaddr, void *dst, const void *src, |
83 | unsigned long len) |
84 | { |
85 | struct folio *folio = page_folio(page); |
86 | |
87 | if (boot_cpu_data.dcache.n_aliases && page_mapcount(page) && |
88 | test_bit(PG_dcache_clean, &folio->flags)) { |
89 | void *vfrom = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK); |
90 | memcpy(dst, vfrom, len); |
91 | kunmap_coherent(vfrom); |
92 | } else { |
93 | memcpy(dst, src, len); |
94 | if (boot_cpu_data.dcache.n_aliases) |
95 | clear_bit(PG_dcache_clean, &folio->flags); |
96 | } |
97 | } |
98 | |
99 | void copy_user_highpage(struct page *to, struct page *from, |
100 | unsigned long vaddr, struct vm_area_struct *vma) |
101 | { |
102 | struct folio *src = page_folio(from); |
103 | void *vfrom, *vto; |
104 | |
105 | vto = kmap_atomic(page: to); |
106 | |
107 | if (boot_cpu_data.dcache.n_aliases && folio_mapped(src) && |
108 | test_bit(PG_dcache_clean, &src->flags)) { |
109 | vfrom = kmap_coherent(from, vaddr); |
110 | copy_page(to: vto, from: vfrom); |
111 | kunmap_coherent(vfrom); |
112 | } else { |
113 | vfrom = kmap_atomic(page: from); |
114 | copy_page(to: vto, from: vfrom); |
115 | kunmap_atomic(vfrom); |
116 | } |
117 | |
118 | if (pages_do_alias((unsigned long)vto, vaddr & PAGE_MASK) || |
119 | (vma->vm_flags & VM_EXEC)) |
120 | __flush_purge_region(vto, PAGE_SIZE); |
121 | |
122 | kunmap_atomic(vto); |
123 | /* Make sure this page is cleared on other CPU's too before using it */ |
124 | smp_wmb(); |
125 | } |
126 | EXPORT_SYMBOL(copy_user_highpage); |
127 | |
128 | void clear_user_highpage(struct page *page, unsigned long vaddr) |
129 | { |
130 | void *kaddr = kmap_atomic(page); |
131 | |
132 | clear_page(page: kaddr); |
133 | |
134 | if (pages_do_alias((unsigned long)kaddr, vaddr & PAGE_MASK)) |
135 | __flush_purge_region(kaddr, PAGE_SIZE); |
136 | |
137 | kunmap_atomic(kaddr); |
138 | } |
139 | EXPORT_SYMBOL(clear_user_highpage); |
140 | |
141 | void __update_cache(struct vm_area_struct *vma, |
142 | unsigned long address, pte_t pte) |
143 | { |
144 | unsigned long pfn = pte_pfn(pte); |
145 | |
146 | if (!boot_cpu_data.dcache.n_aliases) |
147 | return; |
148 | |
149 | if (pfn_valid(pfn)) { |
150 | struct folio *folio = page_folio(pfn_to_page(pfn)); |
151 | int dirty = !test_and_set_bit(nr: PG_dcache_clean, addr: &folio->flags); |
152 | if (dirty) |
153 | __flush_purge_region(folio_address(folio), |
154 | folio_size(folio)); |
155 | } |
156 | } |
157 | |
158 | void __flush_anon_page(struct page *page, unsigned long vmaddr) |
159 | { |
160 | struct folio *folio = page_folio(page); |
161 | unsigned long addr = (unsigned long) page_address(page); |
162 | |
163 | if (pages_do_alias(addr, vmaddr)) { |
164 | if (boot_cpu_data.dcache.n_aliases && folio_mapped(folio) && |
165 | test_bit(PG_dcache_clean, &folio->flags)) { |
166 | void *kaddr; |
167 | |
168 | kaddr = kmap_coherent(page, vmaddr); |
169 | /* XXX.. For now kunmap_coherent() does a purge */ |
170 | /* __flush_purge_region((void *)kaddr, PAGE_SIZE); */ |
171 | kunmap_coherent(kaddr); |
172 | } else |
173 | __flush_purge_region(folio_address(folio), |
174 | folio_size(folio)); |
175 | } |
176 | } |
177 | |
178 | void flush_cache_all(void) |
179 | { |
180 | cacheop_on_each_cpu(func: local_flush_cache_all, NULL, wait: 1); |
181 | } |
182 | EXPORT_SYMBOL(flush_cache_all); |
183 | |
184 | void flush_cache_mm(struct mm_struct *mm) |
185 | { |
186 | if (boot_cpu_data.dcache.n_aliases == 0) |
187 | return; |
188 | |
189 | cacheop_on_each_cpu(func: local_flush_cache_mm, info: mm, wait: 1); |
190 | } |
191 | |
192 | void flush_cache_dup_mm(struct mm_struct *mm) |
193 | { |
194 | if (boot_cpu_data.dcache.n_aliases == 0) |
195 | return; |
196 | |
197 | cacheop_on_each_cpu(func: local_flush_cache_dup_mm, info: mm, wait: 1); |
198 | } |
199 | |
200 | void flush_cache_page(struct vm_area_struct *vma, unsigned long addr, |
201 | unsigned long pfn) |
202 | { |
203 | struct flusher_data data; |
204 | |
205 | data.vma = vma; |
206 | data.addr1 = addr; |
207 | data.addr2 = pfn; |
208 | |
209 | cacheop_on_each_cpu(func: local_flush_cache_page, info: (void *)&data, wait: 1); |
210 | } |
211 | |
212 | void flush_cache_range(struct vm_area_struct *vma, unsigned long start, |
213 | unsigned long end) |
214 | { |
215 | struct flusher_data data; |
216 | |
217 | data.vma = vma; |
218 | data.addr1 = start; |
219 | data.addr2 = end; |
220 | |
221 | cacheop_on_each_cpu(func: local_flush_cache_range, info: (void *)&data, wait: 1); |
222 | } |
223 | EXPORT_SYMBOL(flush_cache_range); |
224 | |
225 | void flush_dcache_folio(struct folio *folio) |
226 | { |
227 | cacheop_on_each_cpu(func: local_flush_dcache_folio, info: folio, wait: 1); |
228 | } |
229 | EXPORT_SYMBOL(flush_dcache_folio); |
230 | |
231 | void flush_icache_range(unsigned long start, unsigned long end) |
232 | { |
233 | struct flusher_data data; |
234 | |
235 | data.vma = NULL; |
236 | data.addr1 = start; |
237 | data.addr2 = end; |
238 | |
239 | cacheop_on_each_cpu(func: local_flush_icache_range, info: (void *)&data, wait: 1); |
240 | } |
241 | EXPORT_SYMBOL(flush_icache_range); |
242 | |
243 | void flush_icache_pages(struct vm_area_struct *vma, struct page *page, |
244 | unsigned int nr) |
245 | { |
246 | /* Nothing uses the VMA, so just pass the folio along */ |
247 | cacheop_on_each_cpu(func: local_flush_icache_folio, page_folio(page), wait: 1); |
248 | } |
249 | |
250 | void flush_cache_sigtramp(unsigned long address) |
251 | { |
252 | cacheop_on_each_cpu(func: local_flush_cache_sigtramp, info: (void *)address, wait: 1); |
253 | } |
254 | |
255 | static void compute_alias(struct cache_info *c) |
256 | { |
257 | #ifdef CONFIG_MMU |
258 | c->alias_mask = ((c->sets - 1) << c->entry_shift) & ~(PAGE_SIZE - 1); |
259 | #else |
260 | c->alias_mask = 0; |
261 | #endif |
262 | c->n_aliases = c->alias_mask ? (c->alias_mask >> PAGE_SHIFT) + 1 : 0; |
263 | } |
264 | |
265 | static void __init emit_cache_params(void) |
266 | { |
267 | printk(KERN_NOTICE "I-cache : n_ways=%d n_sets=%d way_incr=%d\n" , |
268 | boot_cpu_data.icache.ways, |
269 | boot_cpu_data.icache.sets, |
270 | boot_cpu_data.icache.way_incr); |
271 | printk(KERN_NOTICE "I-cache : entry_mask=0x%08x alias_mask=0x%08x n_aliases=%d\n" , |
272 | boot_cpu_data.icache.entry_mask, |
273 | boot_cpu_data.icache.alias_mask, |
274 | boot_cpu_data.icache.n_aliases); |
275 | printk(KERN_NOTICE "D-cache : n_ways=%d n_sets=%d way_incr=%d\n" , |
276 | boot_cpu_data.dcache.ways, |
277 | boot_cpu_data.dcache.sets, |
278 | boot_cpu_data.dcache.way_incr); |
279 | printk(KERN_NOTICE "D-cache : entry_mask=0x%08x alias_mask=0x%08x n_aliases=%d\n" , |
280 | boot_cpu_data.dcache.entry_mask, |
281 | boot_cpu_data.dcache.alias_mask, |
282 | boot_cpu_data.dcache.n_aliases); |
283 | |
284 | /* |
285 | * Emit Secondary Cache parameters if the CPU has a probed L2. |
286 | */ |
287 | if (boot_cpu_data.flags & CPU_HAS_L2_CACHE) { |
288 | printk(KERN_NOTICE "S-cache : n_ways=%d n_sets=%d way_incr=%d\n" , |
289 | boot_cpu_data.scache.ways, |
290 | boot_cpu_data.scache.sets, |
291 | boot_cpu_data.scache.way_incr); |
292 | printk(KERN_NOTICE "S-cache : entry_mask=0x%08x alias_mask=0x%08x n_aliases=%d\n" , |
293 | boot_cpu_data.scache.entry_mask, |
294 | boot_cpu_data.scache.alias_mask, |
295 | boot_cpu_data.scache.n_aliases); |
296 | } |
297 | } |
298 | |
299 | void __init cpu_cache_init(void) |
300 | { |
301 | unsigned int cache_disabled = 0; |
302 | |
303 | #ifdef SH_CCR |
304 | cache_disabled = !(__raw_readl(SH_CCR) & CCR_CACHE_ENABLE); |
305 | #endif |
306 | |
307 | compute_alias(c: &boot_cpu_data.icache); |
308 | compute_alias(c: &boot_cpu_data.dcache); |
309 | compute_alias(c: &boot_cpu_data.scache); |
310 | |
311 | __flush_wback_region = noop__flush_region; |
312 | __flush_purge_region = noop__flush_region; |
313 | __flush_invalidate_region = noop__flush_region; |
314 | |
315 | /* |
316 | * No flushing is necessary in the disabled cache case so we can |
317 | * just keep the noop functions in local_flush_..() and __flush_..() |
318 | */ |
319 | if (unlikely(cache_disabled)) |
320 | goto skip; |
321 | |
322 | if (boot_cpu_data.type == CPU_J2) { |
323 | extern void __weak j2_cache_init(void); |
324 | |
325 | j2_cache_init(); |
326 | } else if (boot_cpu_data.family == CPU_FAMILY_SH2) { |
327 | extern void __weak sh2_cache_init(void); |
328 | |
329 | sh2_cache_init(); |
330 | } |
331 | |
332 | if (boot_cpu_data.family == CPU_FAMILY_SH2A) { |
333 | extern void __weak sh2a_cache_init(void); |
334 | |
335 | sh2a_cache_init(); |
336 | } |
337 | |
338 | if (boot_cpu_data.family == CPU_FAMILY_SH3) { |
339 | extern void __weak sh3_cache_init(void); |
340 | |
341 | sh3_cache_init(); |
342 | |
343 | if ((boot_cpu_data.type == CPU_SH7705) && |
344 | (boot_cpu_data.dcache.sets == 512)) { |
345 | extern void __weak sh7705_cache_init(void); |
346 | |
347 | sh7705_cache_init(); |
348 | } |
349 | } |
350 | |
351 | if ((boot_cpu_data.family == CPU_FAMILY_SH4) || |
352 | (boot_cpu_data.family == CPU_FAMILY_SH4A) || |
353 | (boot_cpu_data.family == CPU_FAMILY_SH4AL_DSP)) { |
354 | extern void __weak sh4_cache_init(void); |
355 | |
356 | sh4_cache_init(); |
357 | |
358 | if ((boot_cpu_data.type == CPU_SH7786) || |
359 | (boot_cpu_data.type == CPU_SHX3)) { |
360 | extern void __weak shx3_cache_init(void); |
361 | |
362 | shx3_cache_init(); |
363 | } |
364 | } |
365 | |
366 | skip: |
367 | emit_cache_params(); |
368 | } |
369 | |