1 | /* |
2 | * This file is subject to the terms and conditions of the GNU General Public |
3 | * License. See the file "COPYING" in the main directory of this archive |
4 | * for more details. |
5 | * |
6 | * Copyright (C) 1998, 1999, 2000 by Ralf Baechle |
7 | * Copyright (C) 1999, 2000 Silicon Graphics, Inc. |
8 | * Copyright (C) 2007 by Maciej W. Rozycki |
9 | * Copyright (C) 2011, 2012 MIPS Technologies, Inc. |
10 | */ |
11 | #include <linux/export.h> |
12 | #include <asm/asm.h> |
13 | #include <asm/asm-offsets.h> |
14 | #include <asm/regdef.h> |
15 | |
16 | #if LONGSIZE == 4 |
17 | #define LONG_S_L swl |
18 | #define LONG_S_R swr |
19 | #else |
20 | #define LONG_S_L sdl |
21 | #define LONG_S_R sdr |
22 | #endif |
23 | |
24 | #ifdef CONFIG_CPU_MICROMIPS |
25 | #define STORSIZE (LONGSIZE * 2) |
26 | #define STORMASK (STORSIZE - 1) |
27 | #define FILL64RG t8 |
28 | #define FILLPTRG t7 |
29 | #undef LONG_S |
30 | #define LONG_S LONG_SP |
31 | #else |
32 | #define STORSIZE LONGSIZE |
33 | #define STORMASK LONGMASK |
34 | #define FILL64RG a1 |
35 | #define FILLPTRG t0 |
36 | #endif |
37 | |
38 | #define LEGACY_MODE 1 |
39 | #define EVA_MODE 2 |
40 | |
41 | /* |
42 | * No need to protect it with EVA #ifdefery. The generated block of code |
43 | * will never be assembled if EVA is not enabled. |
44 | */ |
45 | #define __EVAFY(insn, reg, addr) __BUILD_EVA_INSN(insn##e, reg, addr) |
46 | #define ___BUILD_EVA_INSN(insn, reg, addr) __EVAFY(insn, reg, addr) |
47 | |
48 | #define EX(insn,reg,addr,handler) \ |
49 | .if \mode == LEGACY_MODE; \ |
50 | 9: insn reg, addr; \ |
51 | .else; \ |
52 | 9: ___BUILD_EVA_INSN(insn, reg, addr); \ |
53 | .endif; \ |
54 | .section __ex_table,"a"; \ |
55 | PTR_WD 9b, handler; \ |
56 | .previous |
57 | |
58 | .macro f_fill64 dst, offset, val, fixup, mode |
59 | EX(LONG_S, \val, (\offset + 0 * STORSIZE)(\dst), \fixup) |
60 | EX(LONG_S, \val, (\offset + 1 * STORSIZE)(\dst), \fixup) |
61 | EX(LONG_S, \val, (\offset + 2 * STORSIZE)(\dst), \fixup) |
62 | EX(LONG_S, \val, (\offset + 3 * STORSIZE)(\dst), \fixup) |
63 | #if ((defined(CONFIG_CPU_MICROMIPS) && (LONGSIZE == 4)) || !defined(CONFIG_CPU_MICROMIPS)) |
64 | EX(LONG_S, \val, (\offset + 4 * STORSIZE)(\dst), \fixup) |
65 | EX(LONG_S, \val, (\offset + 5 * STORSIZE)(\dst), \fixup) |
66 | EX(LONG_S, \val, (\offset + 6 * STORSIZE)(\dst), \fixup) |
67 | EX(LONG_S, \val, (\offset + 7 * STORSIZE)(\dst), \fixup) |
68 | #endif |
69 | #if (!defined(CONFIG_CPU_MICROMIPS) && (LONGSIZE == 4)) |
70 | EX(LONG_S, \val, (\offset + 8 * STORSIZE)(\dst), \fixup) |
71 | EX(LONG_S, \val, (\offset + 9 * STORSIZE)(\dst), \fixup) |
72 | EX(LONG_S, \val, (\offset + 10 * STORSIZE)(\dst), \fixup) |
73 | EX(LONG_S, \val, (\offset + 11 * STORSIZE)(\dst), \fixup) |
74 | EX(LONG_S, \val, (\offset + 12 * STORSIZE)(\dst), \fixup) |
75 | EX(LONG_S, \val, (\offset + 13 * STORSIZE)(\dst), \fixup) |
76 | EX(LONG_S, \val, (\offset + 14 * STORSIZE)(\dst), \fixup) |
77 | EX(LONG_S, \val, (\offset + 15 * STORSIZE)(\dst), \fixup) |
78 | #endif |
79 | .endm |
80 | |
81 | .align 5 |
82 | |
83 | /* |
84 | * Macro to generate the __bzero{,_user} symbol |
85 | * Arguments: |
86 | * mode: LEGACY_MODE or EVA_MODE |
87 | */ |
88 | .macro __BUILD_BZERO mode |
89 | /* Initialize __memset if this is the first time we call this macro */ |
90 | .ifnotdef __memset |
91 | .set __memset, 1 |
92 | .hidden __memset /* Make sure it does not leak */ |
93 | .endif |
94 | |
95 | sltiu t0, a2, STORSIZE /* very small region? */ |
96 | .set noreorder |
97 | bnez t0, .Lsmall_memset\@ |
98 | andi t0, a0, STORMASK /* aligned? */ |
99 | .set reorder |
100 | |
101 | #ifdef CONFIG_CPU_MICROMIPS |
102 | move t8, a1 /* used by 'swp' instruction */ |
103 | move t9, a1 |
104 | #endif |
105 | .set noreorder |
106 | #ifndef CONFIG_CPU_DADDI_WORKAROUNDS |
107 | beqz t0, 1f |
108 | PTR_SUBU t0, STORSIZE /* alignment in bytes */ |
109 | #else |
110 | .set noat |
111 | li AT, STORSIZE |
112 | beqz t0, 1f |
113 | PTR_SUBU t0, AT /* alignment in bytes */ |
114 | .set at |
115 | #endif |
116 | .set reorder |
117 | |
118 | #ifndef CONFIG_CPU_NO_LOAD_STORE_LR |
119 | R10KCBARRIER(0(ra)) |
120 | #ifdef __MIPSEB__ |
121 | EX(LONG_S_L, a1, (a0), .Lfirst_fixup\@) /* make word/dword aligned */ |
122 | #else |
123 | EX(LONG_S_R, a1, (a0), .Lfirst_fixup\@) /* make word/dword aligned */ |
124 | #endif |
125 | PTR_SUBU a0, t0 /* long align ptr */ |
126 | PTR_ADDU a2, t0 /* correct size */ |
127 | |
128 | #else /* CONFIG_CPU_NO_LOAD_STORE_LR */ |
129 | #define STORE_BYTE(N) \ |
130 | EX(sb, a1, N(a0), .Lbyte_fixup\@); \ |
131 | .set noreorder; \ |
132 | beqz t0, 0f; \ |
133 | PTR_ADDU t0, 1; \ |
134 | .set reorder; |
135 | |
136 | PTR_ADDU a2, t0 /* correct size */ |
137 | PTR_ADDU t0, 1 |
138 | STORE_BYTE(0) |
139 | STORE_BYTE(1) |
140 | #if LONGSIZE == 4 |
141 | EX(sb, a1, 2(a0), .Lbyte_fixup\@) |
142 | #else |
143 | STORE_BYTE(2) |
144 | STORE_BYTE(3) |
145 | STORE_BYTE(4) |
146 | STORE_BYTE(5) |
147 | EX(sb, a1, 6(a0), .Lbyte_fixup\@) |
148 | #endif |
149 | 0: |
150 | ori a0, STORMASK |
151 | xori a0, STORMASK |
152 | PTR_ADDIU a0, STORSIZE |
153 | #endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ |
154 | 1: ori t1, a2, 0x3f /* # of full blocks */ |
155 | xori t1, 0x3f |
156 | andi t0, a2, 0x40-STORSIZE |
157 | beqz t1, .Lmemset_partial\@ /* no block to fill */ |
158 | |
159 | PTR_ADDU t1, a0 /* end address */ |
160 | 1: PTR_ADDIU a0, 64 |
161 | R10KCBARRIER(0(ra)) |
162 | f_fill64 a0, -64, FILL64RG, .Lfwd_fixup\@, \mode |
163 | bne t1, a0, 1b |
164 | |
165 | .Lmemset_partial\@: |
166 | R10KCBARRIER(0(ra)) |
167 | PTR_LA t1, 2f /* where to start */ |
168 | #ifdef CONFIG_CPU_MICROMIPS |
169 | LONG_SRL t7, t0, 1 |
170 | #endif |
171 | #if LONGSIZE == 4 |
172 | PTR_SUBU t1, FILLPTRG |
173 | #else |
174 | .set noat |
175 | LONG_SRL AT, FILLPTRG, 1 |
176 | PTR_SUBU t1, AT |
177 | .set at |
178 | #endif |
179 | PTR_ADDU a0, t0 /* dest ptr */ |
180 | jr t1 |
181 | |
182 | /* ... but first do longs ... */ |
183 | f_fill64 a0, -64, FILL64RG, .Lpartial_fixup\@, \mode |
184 | 2: andi a2, STORMASK /* At most one long to go */ |
185 | |
186 | .set noreorder |
187 | beqz a2, 1f |
188 | #ifndef CONFIG_CPU_NO_LOAD_STORE_LR |
189 | PTR_ADDU a0, a2 /* What's left */ |
190 | .set reorder |
191 | R10KCBARRIER(0(ra)) |
192 | #ifdef __MIPSEB__ |
193 | EX(LONG_S_R, a1, -1(a0), .Llast_fixup\@) |
194 | #else |
195 | EX(LONG_S_L, a1, -1(a0), .Llast_fixup\@) |
196 | #endif |
197 | #else /* CONFIG_CPU_NO_LOAD_STORE_LR */ |
198 | PTR_SUBU t0, $0, a2 |
199 | .set reorder |
200 | move a2, zero /* No remaining longs */ |
201 | PTR_ADDIU t0, 1 |
202 | STORE_BYTE(0) |
203 | STORE_BYTE(1) |
204 | #if LONGSIZE == 4 |
205 | EX(sb, a1, 2(a0), .Lbyte_fixup\@) |
206 | #else |
207 | STORE_BYTE(2) |
208 | STORE_BYTE(3) |
209 | STORE_BYTE(4) |
210 | STORE_BYTE(5) |
211 | EX(sb, a1, 6(a0), .Lbyte_fixup\@) |
212 | #endif |
213 | 0: |
214 | #endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ |
215 | 1: move a2, zero |
216 | jr ra |
217 | |
218 | .Lsmall_memset\@: |
219 | PTR_ADDU t1, a0, a2 |
220 | beqz a2, 2f |
221 | |
222 | 1: PTR_ADDIU a0, 1 /* fill bytewise */ |
223 | R10KCBARRIER(0(ra)) |
224 | .set noreorder |
225 | bne t1, a0, 1b |
226 | EX(sb, a1, -1(a0), .Lsmall_fixup\@) |
227 | .set reorder |
228 | |
229 | 2: move a2, zero |
230 | jr ra /* done */ |
231 | .if __memset == 1 |
232 | END(memset) |
233 | .set __memset, 0 |
234 | .hidden __memset |
235 | .endif |
236 | |
237 | #ifdef CONFIG_CPU_NO_LOAD_STORE_LR |
238 | .Lbyte_fixup\@: |
239 | /* |
240 | * unset_bytes = (#bytes - (#unaligned bytes)) - (-#unaligned bytes remaining + 1) + 1 |
241 | * a2 = a2 - t0 + 1 |
242 | */ |
243 | PTR_SUBU a2, t0 |
244 | PTR_ADDIU a2, 1 |
245 | jr ra |
246 | #endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ |
247 | |
248 | .Lfirst_fixup\@: |
249 | /* unset_bytes already in a2 */ |
250 | jr ra |
251 | |
252 | .Lfwd_fixup\@: |
253 | /* |
254 | * unset_bytes = partial_start_addr + #bytes - fault_addr |
255 | * a2 = t1 + (a2 & 3f) - $28->task->BUADDR |
256 | */ |
257 | PTR_L t0, TI_TASK($28) |
258 | andi a2, 0x3f |
259 | LONG_L t0, THREAD_BUADDR(t0) |
260 | LONG_ADDU a2, t1 |
261 | LONG_SUBU a2, t0 |
262 | jr ra |
263 | |
264 | .Lpartial_fixup\@: |
265 | /* |
266 | * unset_bytes = partial_end_addr + #bytes - fault_addr |
267 | * a2 = a0 + (a2 & STORMASK) - $28->task->BUADDR |
268 | */ |
269 | PTR_L t0, TI_TASK($28) |
270 | andi a2, STORMASK |
271 | LONG_L t0, THREAD_BUADDR(t0) |
272 | LONG_ADDU a2, a0 |
273 | LONG_SUBU a2, t0 |
274 | jr ra |
275 | |
276 | .Llast_fixup\@: |
277 | /* unset_bytes already in a2 */ |
278 | jr ra |
279 | |
280 | .Lsmall_fixup\@: |
281 | /* |
282 | * unset_bytes = end_addr - current_addr + 1 |
283 | * a2 = t1 - a0 + 1 |
284 | */ |
285 | PTR_SUBU a2, t1, a0 |
286 | PTR_ADDIU a2, 1 |
287 | jr ra |
288 | |
289 | .endm |
290 | |
291 | /* |
292 | * memset(void *s, int c, size_t n) |
293 | * |
294 | * a0: start of area to clear |
295 | * a1: char to fill with |
296 | * a2: size of area to clear |
297 | */ |
298 | |
299 | LEAF(memset) |
300 | EXPORT_SYMBOL(memset) |
301 | move v0, a0 /* result */ |
302 | beqz a1, 1f |
303 | |
304 | andi a1, 0xff /* spread fillword */ |
305 | LONG_SLL t1, a1, 8 |
306 | or a1, t1 |
307 | LONG_SLL t1, a1, 16 |
308 | #if LONGSIZE == 8 |
309 | or a1, t1 |
310 | LONG_SLL t1, a1, 32 |
311 | #endif |
312 | or a1, t1 |
313 | 1: |
314 | #ifndef CONFIG_EVA |
315 | FEXPORT(__bzero) |
316 | EXPORT_SYMBOL(__bzero) |
317 | #endif |
318 | __BUILD_BZERO LEGACY_MODE |
319 | |
320 | #ifdef CONFIG_EVA |
321 | LEAF(__bzero) |
322 | EXPORT_SYMBOL(__bzero) |
323 | __BUILD_BZERO EVA_MODE |
324 | END(__bzero) |
325 | #endif |
326 | |