1 | /* |
2 | * Copyright (C) 2009 Michal Simek <monstr@monstr.eu> |
3 | * Copyright (C) 2009 PetaLogix |
4 | * Copyright (C) 2007 LynuxWorks, Inc. |
5 | * |
6 | * This file is subject to the terms and conditions of the GNU General Public |
7 | * License. See the file "COPYING" in the main directory of this archive |
8 | * for more details. |
9 | */ |
10 | |
11 | #include <linux/errno.h> |
12 | #include <linux/linkage.h> |
13 | #include <asm/page.h> |
14 | |
15 | /* Loop unrolling for __copy_tofrom_user */ |
16 | #define COPY(offset) \ |
17 | 1: lwi r4 , r6, 0x0000 + offset; \ |
18 | 2: lwi r19, r6, 0x0004 + offset; \ |
19 | 3: lwi r20, r6, 0x0008 + offset; \ |
20 | 4: lwi r21, r6, 0x000C + offset; \ |
21 | 5: lwi r22, r6, 0x0010 + offset; \ |
22 | 6: lwi r23, r6, 0x0014 + offset; \ |
23 | 7: lwi r24, r6, 0x0018 + offset; \ |
24 | 8: lwi r25, r6, 0x001C + offset; \ |
25 | 9: swi r4 , r5, 0x0000 + offset; \ |
26 | 10: swi r19, r5, 0x0004 + offset; \ |
27 | 11: swi r20, r5, 0x0008 + offset; \ |
28 | 12: swi r21, r5, 0x000C + offset; \ |
29 | 13: swi r22, r5, 0x0010 + offset; \ |
30 | 14: swi r23, r5, 0x0014 + offset; \ |
31 | 15: swi r24, r5, 0x0018 + offset; \ |
32 | 16: swi r25, r5, 0x001C + offset; \ |
33 | .section __ex_table,"a"; \ |
34 | .word 1b, 33f; \ |
35 | .word 2b, 33f; \ |
36 | .word 3b, 33f; \ |
37 | .word 4b, 33f; \ |
38 | .word 5b, 33f; \ |
39 | .word 6b, 33f; \ |
40 | .word 7b, 33f; \ |
41 | .word 8b, 33f; \ |
42 | .word 9b, 33f; \ |
43 | .word 10b, 33f; \ |
44 | .word 11b, 33f; \ |
45 | .word 12b, 33f; \ |
46 | .word 13b, 33f; \ |
47 | .word 14b, 33f; \ |
48 | .word 15b, 33f; \ |
49 | .word 16b, 33f; \ |
50 | .text |
51 | |
52 | #define COPY_80(offset) \ |
53 | COPY(0x00 + offset);\ |
54 | COPY(0x20 + offset);\ |
55 | COPY(0x40 + offset);\ |
56 | COPY(0x60 + offset); |
57 | |
58 | /* |
59 | * int __copy_tofrom_user(char *to, char *from, int len) |
60 | * Return: |
61 | * 0 on success |
62 | * number of not copied bytes on error |
63 | */ |
64 | .text |
65 | .globl __copy_tofrom_user; |
66 | .type __copy_tofrom_user, @function |
67 | .align 4; |
68 | __copy_tofrom_user: |
69 | /* |
70 | * r5 - to |
71 | * r6 - from |
72 | * r7, r3 - count |
73 | * r4 - tempval |
74 | */ |
75 | beqid r7, 0f /* zero size is not likely */ |
76 | or r3, r5, r6 /* find if is any to/from unaligned */ |
77 | or r3, r3, r7 /* find if count is unaligned */ |
78 | andi r3, r3, 0x3 /* mask last 3 bits */ |
79 | bneid r3, bu1 /* if r3 is not zero then byte copying */ |
80 | or r3, r0, r0 |
81 | |
82 | rsubi r3, r7, PAGE_SIZE /* detect PAGE_SIZE */ |
83 | beqid r3, page; |
84 | or r3, r0, r0 |
85 | |
86 | w1: lw r4, r6, r3 /* at least one 4 byte copy */ |
87 | w2: sw r4, r5, r3 |
88 | addik r7, r7, -4 |
89 | bneid r7, w1 |
90 | addik r3, r3, 4 |
91 | addik r3, r7, 0 |
92 | rtsd r15, 8 |
93 | nop |
94 | |
95 | .section __ex_table,"a" |
96 | .word w1, 0f; |
97 | .word w2, 0f; |
98 | .text |
99 | |
100 | .align 4 /* Alignment is important to keep icache happy */ |
101 | page: /* Create room on stack and save registers for storing values */ |
102 | addik r1, r1, -40 |
103 | swi r5, r1, 0 |
104 | swi r6, r1, 4 |
105 | swi r7, r1, 8 |
106 | swi r19, r1, 12 |
107 | swi r20, r1, 16 |
108 | swi r21, r1, 20 |
109 | swi r22, r1, 24 |
110 | swi r23, r1, 28 |
111 | swi r24, r1, 32 |
112 | swi r25, r1, 36 |
113 | loop: /* r4, r19, r20, r21, r22, r23, r24, r25 are used for storing values */ |
114 | /* Loop unrolling to get performance boost */ |
115 | COPY_80(0x000); |
116 | COPY_80(0x080); |
117 | COPY_80(0x100); |
118 | COPY_80(0x180); |
119 | /* copy loop */ |
120 | addik r6, r6, 0x200 |
121 | addik r7, r7, -0x200 |
122 | bneid r7, loop |
123 | addik r5, r5, 0x200 |
124 | |
125 | /* Restore register content */ |
126 | lwi r5, r1, 0 |
127 | lwi r6, r1, 4 |
128 | lwi r7, r1, 8 |
129 | lwi r19, r1, 12 |
130 | lwi r20, r1, 16 |
131 | lwi r21, r1, 20 |
132 | lwi r22, r1, 24 |
133 | lwi r23, r1, 28 |
134 | lwi r24, r1, 32 |
135 | lwi r25, r1, 36 |
136 | addik r1, r1, 40 |
137 | /* return back */ |
138 | addik r3, r0, 0 |
139 | rtsd r15, 8 |
140 | nop |
141 | |
142 | /* Fault case - return temp count */ |
143 | 33: |
144 | addik r3, r7, 0 |
145 | /* Restore register content */ |
146 | lwi r5, r1, 0 |
147 | lwi r6, r1, 4 |
148 | lwi r7, r1, 8 |
149 | lwi r19, r1, 12 |
150 | lwi r20, r1, 16 |
151 | lwi r21, r1, 20 |
152 | lwi r22, r1, 24 |
153 | lwi r23, r1, 28 |
154 | lwi r24, r1, 32 |
155 | lwi r25, r1, 36 |
156 | addik r1, r1, 40 |
157 | /* return back */ |
158 | rtsd r15, 8 |
159 | nop |
160 | |
161 | .align 4 /* Alignment is important to keep icache happy */ |
162 | bu1: lbu r4,r6,r3 |
163 | bu2: sb r4,r5,r3 |
164 | addik r7,r7,-1 |
165 | bneid r7,bu1 |
166 | addik r3,r3,1 /* delay slot */ |
167 | 0: |
168 | addik r3,r7,0 |
169 | rtsd r15,8 |
170 | nop |
171 | .size __copy_tofrom_user, . - __copy_tofrom_user |
172 | |
173 | .section __ex_table,"a" |
174 | .word bu1, 0b; |
175 | .word bu2, 0b; |
176 | .text |
177 | |