1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | |
3 | #include <linux/highmem.h> |
4 | #include <linux/kprobes.h> |
5 | |
6 | /** |
7 | * flush_coherent_icache() - if a CPU has a coherent icache, flush it |
8 | * Return true if the cache was flushed, false otherwise |
9 | */ |
10 | static inline bool flush_coherent_icache(void) |
11 | { |
12 | /* |
13 | * For a snooping icache, we still need a dummy icbi to purge all the |
14 | * prefetched instructions from the ifetch buffers. We also need a sync |
15 | * before the icbi to order the actual stores to memory that might |
16 | * have modified instructions with the icbi. |
17 | */ |
18 | if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) { |
19 | mb(); /* sync */ |
20 | icbi((void *)PAGE_OFFSET); |
21 | mb(); /* sync */ |
22 | isync(); |
23 | return true; |
24 | } |
25 | |
26 | return false; |
27 | } |
28 | |
29 | /** |
30 | * invalidate_icache_range() - Flush the icache by issuing icbi across an address range |
31 | * @start: the start address |
32 | * @stop: the stop address (exclusive) |
33 | */ |
34 | static void invalidate_icache_range(unsigned long start, unsigned long stop) |
35 | { |
36 | unsigned long shift = l1_icache_shift(); |
37 | unsigned long bytes = l1_icache_bytes(); |
38 | char *addr = (char *)(start & ~(bytes - 1)); |
39 | unsigned long size = stop - (unsigned long)addr + (bytes - 1); |
40 | unsigned long i; |
41 | |
42 | for (i = 0; i < size >> shift; i++, addr += bytes) |
43 | icbi(addr); |
44 | |
45 | mb(); /* sync */ |
46 | isync(); |
47 | } |
48 | |
49 | /** |
50 | * flush_icache_range: Write any modified data cache blocks out to memory |
51 | * and invalidate the corresponding blocks in the instruction cache |
52 | * |
53 | * Generic code will call this after writing memory, before executing from it. |
54 | * |
55 | * @start: the start address |
56 | * @stop: the stop address (exclusive) |
57 | */ |
58 | void flush_icache_range(unsigned long start, unsigned long stop) |
59 | { |
60 | if (flush_coherent_icache()) |
61 | return; |
62 | |
63 | clean_dcache_range(start, stop); |
64 | |
65 | if (IS_ENABLED(CONFIG_44x)) { |
66 | /* |
67 | * Flash invalidate on 44x because we are passed kmapped |
68 | * addresses and this doesn't work for userspace pages due to |
69 | * the virtually tagged icache. |
70 | */ |
71 | iccci((void *)start); |
72 | mb(); /* sync */ |
73 | isync(); |
74 | } else |
75 | invalidate_icache_range(start, stop); |
76 | } |
77 | EXPORT_SYMBOL(flush_icache_range); |
78 | |
79 | #ifdef CONFIG_HIGHMEM |
80 | /** |
81 | * flush_dcache_icache_phys() - Flush a page by it's physical address |
82 | * @physaddr: the physical address of the page |
83 | */ |
84 | static void flush_dcache_icache_phys(unsigned long physaddr) |
85 | { |
86 | unsigned long bytes = l1_dcache_bytes(); |
87 | unsigned long nb = PAGE_SIZE / bytes; |
88 | unsigned long addr = physaddr & PAGE_MASK; |
89 | unsigned long msr, msr0; |
90 | unsigned long loop1 = addr, loop2 = addr; |
91 | |
92 | msr0 = mfmsr(); |
93 | msr = msr0 & ~MSR_DR; |
94 | /* |
95 | * This must remain as ASM to prevent potential memory accesses |
96 | * while the data MMU is disabled |
97 | */ |
98 | asm volatile( |
99 | " mtctr %2;\n" |
100 | " mtmsr %3;\n" |
101 | " isync;\n" |
102 | "0: dcbst 0, %0;\n" |
103 | " addi %0, %0, %4;\n" |
104 | " bdnz 0b;\n" |
105 | " sync;\n" |
106 | " mtctr %2;\n" |
107 | "1: icbi 0, %1;\n" |
108 | " addi %1, %1, %4;\n" |
109 | " bdnz 1b;\n" |
110 | " sync;\n" |
111 | " mtmsr %5;\n" |
112 | " isync;\n" |
113 | : "+&r" (loop1), "+&r" (loop2) |
114 | : "r" (nb), "r" (msr), "i" (bytes), "r" (msr0) |
115 | : "ctr" , "memory" ); |
116 | } |
117 | NOKPROBE_SYMBOL(flush_dcache_icache_phys) |
118 | #else |
119 | static void flush_dcache_icache_phys(unsigned long physaddr) |
120 | { |
121 | } |
122 | #endif |
123 | |
124 | /** |
125 | * __flush_dcache_icache(): Flush a particular page from the data cache to RAM. |
126 | * Note: this is necessary because the instruction cache does *not* |
127 | * snoop from the data cache. |
128 | * |
129 | * @p: the address of the page to flush |
130 | */ |
131 | static void __flush_dcache_icache(void *p) |
132 | { |
133 | unsigned long addr = (unsigned long)p & PAGE_MASK; |
134 | |
135 | clean_dcache_range(addr, addr + PAGE_SIZE); |
136 | |
137 | /* |
138 | * We don't flush the icache on 44x. Those have a virtual icache and we |
139 | * don't have access to the virtual address here (it's not the page |
140 | * vaddr but where it's mapped in user space). The flushing of the |
141 | * icache on these is handled elsewhere, when a change in the address |
142 | * space occurs, before returning to user space. |
143 | */ |
144 | |
145 | if (mmu_has_feature(MMU_FTR_TYPE_44x)) |
146 | return; |
147 | |
148 | invalidate_icache_range(start: addr, stop: addr + PAGE_SIZE); |
149 | } |
150 | |
151 | void flush_dcache_icache_folio(struct folio *folio) |
152 | { |
153 | unsigned int i, nr = folio_nr_pages(folio); |
154 | |
155 | if (flush_coherent_icache()) |
156 | return; |
157 | |
158 | if (!folio_test_highmem(folio)) { |
159 | void *addr = folio_address(folio); |
160 | for (i = 0; i < nr; i++) |
161 | __flush_dcache_icache(p: addr + i * PAGE_SIZE); |
162 | } else if (IS_ENABLED(CONFIG_BOOKE) || sizeof(phys_addr_t) > sizeof(void *)) { |
163 | for (i = 0; i < nr; i++) { |
164 | void *start = kmap_local_folio(folio, offset: i * PAGE_SIZE); |
165 | |
166 | __flush_dcache_icache(p: start); |
167 | kunmap_local(start); |
168 | } |
169 | } else { |
170 | unsigned long pfn = folio_pfn(folio); |
171 | for (i = 0; i < nr; i++) |
172 | flush_dcache_icache_phys(physaddr: (pfn + i) * PAGE_SIZE); |
173 | } |
174 | } |
175 | EXPORT_SYMBOL(flush_dcache_icache_folio); |
176 | |
177 | void clear_user_page(void *page, unsigned long vaddr, struct page *pg) |
178 | { |
179 | clear_page(page); |
180 | |
181 | /* |
182 | * We shouldn't have to do this, but some versions of glibc |
183 | * require it (ld.so assumes zero filled pages are icache clean) |
184 | * - Anton |
185 | */ |
186 | flush_dcache_page(page: pg); |
187 | } |
188 | EXPORT_SYMBOL(clear_user_page); |
189 | |
190 | void copy_user_page(void *vto, void *vfrom, unsigned long vaddr, |
191 | struct page *pg) |
192 | { |
193 | copy_page(to: vto, from: vfrom); |
194 | |
195 | /* |
196 | * We should be able to use the following optimisation, however |
197 | * there are two problems. |
198 | * Firstly a bug in some versions of binutils meant PLT sections |
199 | * were not marked executable. |
200 | * Secondly the first word in the GOT section is blrl, used |
201 | * to establish the GOT address. Until recently the GOT was |
202 | * not marked executable. |
203 | * - Anton |
204 | */ |
205 | #if 0 |
206 | if (!vma->vm_file && ((vma->vm_flags & VM_EXEC) == 0)) |
207 | return; |
208 | #endif |
209 | |
210 | flush_dcache_page(page: pg); |
211 | } |
212 | |
213 | void flush_icache_user_page(struct vm_area_struct *vma, struct page *page, |
214 | unsigned long addr, int len) |
215 | { |
216 | void *maddr; |
217 | |
218 | maddr = kmap_local_page(page) + (addr & ~PAGE_MASK); |
219 | flush_icache_range((unsigned long)maddr, (unsigned long)maddr + len); |
220 | kunmap_local(maddr); |
221 | } |
222 | |