1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * User address space access functions. |
4 | * The non inlined parts of asm-i386/uaccess.h are here. |
5 | * |
6 | * Copyright 1997 Andi Kleen <ak@muc.de> |
7 | * Copyright 1997 Linus Torvalds |
8 | */ |
9 | #include <linux/export.h> |
10 | #include <linux/uaccess.h> |
11 | #include <asm/asm.h> |
12 | |
13 | #ifdef CONFIG_X86_INTEL_USERCOPY |
14 | /* |
15 | * Alignment at which movsl is preferred for bulk memory copies. |
16 | */ |
17 | struct movsl_mask movsl_mask __read_mostly; |
18 | #endif |
19 | |
20 | static inline int __movsl_is_ok(unsigned long a1, unsigned long a2, unsigned long n) |
21 | { |
22 | #ifdef CONFIG_X86_INTEL_USERCOPY |
23 | if (n >= 64 && ((a1 ^ a2) & movsl_mask.mask)) |
24 | return 0; |
25 | #endif |
26 | return 1; |
27 | } |
28 | #define movsl_is_ok(a1, a2, n) \ |
29 | __movsl_is_ok((unsigned long)(a1), (unsigned long)(a2), (n)) |
30 | |
31 | /* |
32 | * Zero Userspace |
33 | */ |
34 | |
35 | #define __do_clear_user(addr,size) \ |
36 | do { \ |
37 | int __d0; \ |
38 | might_fault(); \ |
39 | __asm__ __volatile__( \ |
40 | ASM_STAC "\n" \ |
41 | "0: rep; stosl\n" \ |
42 | " movl %2,%0\n" \ |
43 | "1: rep; stosb\n" \ |
44 | "2: " ASM_CLAC "\n" \ |
45 | _ASM_EXTABLE_TYPE_REG(0b, 2b, EX_TYPE_UCOPY_LEN4, %2) \ |
46 | _ASM_EXTABLE_UA(1b, 2b) \ |
47 | : "=&c"(size), "=&D" (__d0) \ |
48 | : "r"(size & 3), "0"(size / 4), "1"(addr), "a"(0)); \ |
49 | } while (0) |
50 | |
51 | /** |
52 | * clear_user - Zero a block of memory in user space. |
53 | * @to: Destination address, in user space. |
54 | * @n: Number of bytes to zero. |
55 | * |
56 | * Zero a block of memory in user space. |
57 | * |
58 | * Return: number of bytes that could not be cleared. |
59 | * On success, this will be zero. |
60 | */ |
61 | unsigned long |
62 | clear_user(void __user *to, unsigned long n) |
63 | { |
64 | might_fault(); |
65 | if (access_ok(to, n)) |
66 | __do_clear_user(to, n); |
67 | return n; |
68 | } |
69 | EXPORT_SYMBOL(clear_user); |
70 | |
71 | /** |
72 | * __clear_user - Zero a block of memory in user space, with less checking. |
73 | * @to: Destination address, in user space. |
74 | * @n: Number of bytes to zero. |
75 | * |
76 | * Zero a block of memory in user space. Caller must check |
77 | * the specified block with access_ok() before calling this function. |
78 | * |
79 | * Return: number of bytes that could not be cleared. |
80 | * On success, this will be zero. |
81 | */ |
82 | unsigned long |
83 | __clear_user(void __user *to, unsigned long n) |
84 | { |
85 | __do_clear_user(to, n); |
86 | return n; |
87 | } |
88 | EXPORT_SYMBOL(__clear_user); |
89 | |
90 | #ifdef CONFIG_X86_INTEL_USERCOPY |
91 | static unsigned long |
92 | __copy_user_intel(void __user *to, const void *from, unsigned long size) |
93 | { |
94 | int d0, d1; |
95 | __asm__ __volatile__( |
96 | " .align 2,0x90\n" |
97 | "1: movl 32(%4), %%eax\n" |
98 | " cmpl $67, %0\n" |
99 | " jbe 3f\n" |
100 | "2: movl 64(%4), %%eax\n" |
101 | " .align 2,0x90\n" |
102 | "3: movl 0(%4), %%eax\n" |
103 | "4: movl 4(%4), %%edx\n" |
104 | "5: movl %%eax, 0(%3)\n" |
105 | "6: movl %%edx, 4(%3)\n" |
106 | "7: movl 8(%4), %%eax\n" |
107 | "8: movl 12(%4),%%edx\n" |
108 | "9: movl %%eax, 8(%3)\n" |
109 | "10: movl %%edx, 12(%3)\n" |
110 | "11: movl 16(%4), %%eax\n" |
111 | "12: movl 20(%4), %%edx\n" |
112 | "13: movl %%eax, 16(%3)\n" |
113 | "14: movl %%edx, 20(%3)\n" |
114 | "15: movl 24(%4), %%eax\n" |
115 | "16: movl 28(%4), %%edx\n" |
116 | "17: movl %%eax, 24(%3)\n" |
117 | "18: movl %%edx, 28(%3)\n" |
118 | "19: movl 32(%4), %%eax\n" |
119 | "20: movl 36(%4), %%edx\n" |
120 | "21: movl %%eax, 32(%3)\n" |
121 | "22: movl %%edx, 36(%3)\n" |
122 | "23: movl 40(%4), %%eax\n" |
123 | "24: movl 44(%4), %%edx\n" |
124 | "25: movl %%eax, 40(%3)\n" |
125 | "26: movl %%edx, 44(%3)\n" |
126 | "27: movl 48(%4), %%eax\n" |
127 | "28: movl 52(%4), %%edx\n" |
128 | "29: movl %%eax, 48(%3)\n" |
129 | "30: movl %%edx, 52(%3)\n" |
130 | "31: movl 56(%4), %%eax\n" |
131 | "32: movl 60(%4), %%edx\n" |
132 | "33: movl %%eax, 56(%3)\n" |
133 | "34: movl %%edx, 60(%3)\n" |
134 | " addl $-64, %0\n" |
135 | " addl $64, %4\n" |
136 | " addl $64, %3\n" |
137 | " cmpl $63, %0\n" |
138 | " ja 1b\n" |
139 | "35: movl %0, %%eax\n" |
140 | " shrl $2, %0\n" |
141 | " andl $3, %%eax\n" |
142 | " cld\n" |
143 | "99: rep; movsl\n" |
144 | "36: movl %%eax, %0\n" |
145 | "37: rep; movsb\n" |
146 | "100:\n" |
147 | _ASM_EXTABLE_UA(1b, 100b) |
148 | _ASM_EXTABLE_UA(2b, 100b) |
149 | _ASM_EXTABLE_UA(3b, 100b) |
150 | _ASM_EXTABLE_UA(4b, 100b) |
151 | _ASM_EXTABLE_UA(5b, 100b) |
152 | _ASM_EXTABLE_UA(6b, 100b) |
153 | _ASM_EXTABLE_UA(7b, 100b) |
154 | _ASM_EXTABLE_UA(8b, 100b) |
155 | _ASM_EXTABLE_UA(9b, 100b) |
156 | _ASM_EXTABLE_UA(10b, 100b) |
157 | _ASM_EXTABLE_UA(11b, 100b) |
158 | _ASM_EXTABLE_UA(12b, 100b) |
159 | _ASM_EXTABLE_UA(13b, 100b) |
160 | _ASM_EXTABLE_UA(14b, 100b) |
161 | _ASM_EXTABLE_UA(15b, 100b) |
162 | _ASM_EXTABLE_UA(16b, 100b) |
163 | _ASM_EXTABLE_UA(17b, 100b) |
164 | _ASM_EXTABLE_UA(18b, 100b) |
165 | _ASM_EXTABLE_UA(19b, 100b) |
166 | _ASM_EXTABLE_UA(20b, 100b) |
167 | _ASM_EXTABLE_UA(21b, 100b) |
168 | _ASM_EXTABLE_UA(22b, 100b) |
169 | _ASM_EXTABLE_UA(23b, 100b) |
170 | _ASM_EXTABLE_UA(24b, 100b) |
171 | _ASM_EXTABLE_UA(25b, 100b) |
172 | _ASM_EXTABLE_UA(26b, 100b) |
173 | _ASM_EXTABLE_UA(27b, 100b) |
174 | _ASM_EXTABLE_UA(28b, 100b) |
175 | _ASM_EXTABLE_UA(29b, 100b) |
176 | _ASM_EXTABLE_UA(30b, 100b) |
177 | _ASM_EXTABLE_UA(31b, 100b) |
178 | _ASM_EXTABLE_UA(32b, 100b) |
179 | _ASM_EXTABLE_UA(33b, 100b) |
180 | _ASM_EXTABLE_UA(34b, 100b) |
181 | _ASM_EXTABLE_UA(35b, 100b) |
182 | _ASM_EXTABLE_UA(36b, 100b) |
183 | _ASM_EXTABLE_UA(37b, 100b) |
184 | _ASM_EXTABLE_TYPE_REG(99b, 100b, EX_TYPE_UCOPY_LEN4, %%eax) |
185 | : "=&c" (size), "=&D" (d0), "=&S" (d1) |
186 | : "1" (to), "2" (from), "0" (size) |
187 | : "eax" , "edx" , "memory" ); |
188 | return size; |
189 | } |
190 | |
191 | static unsigned long __copy_user_intel_nocache(void *to, |
192 | const void __user *from, unsigned long size) |
193 | { |
194 | int d0, d1; |
195 | |
196 | __asm__ __volatile__( |
197 | " .align 2,0x90\n" |
198 | "0: movl 32(%4), %%eax\n" |
199 | " cmpl $67, %0\n" |
200 | " jbe 2f\n" |
201 | "1: movl 64(%4), %%eax\n" |
202 | " .align 2,0x90\n" |
203 | "2: movl 0(%4), %%eax\n" |
204 | "21: movl 4(%4), %%edx\n" |
205 | " movnti %%eax, 0(%3)\n" |
206 | " movnti %%edx, 4(%3)\n" |
207 | "3: movl 8(%4), %%eax\n" |
208 | "31: movl 12(%4),%%edx\n" |
209 | " movnti %%eax, 8(%3)\n" |
210 | " movnti %%edx, 12(%3)\n" |
211 | "4: movl 16(%4), %%eax\n" |
212 | "41: movl 20(%4), %%edx\n" |
213 | " movnti %%eax, 16(%3)\n" |
214 | " movnti %%edx, 20(%3)\n" |
215 | "10: movl 24(%4), %%eax\n" |
216 | "51: movl 28(%4), %%edx\n" |
217 | " movnti %%eax, 24(%3)\n" |
218 | " movnti %%edx, 28(%3)\n" |
219 | "11: movl 32(%4), %%eax\n" |
220 | "61: movl 36(%4), %%edx\n" |
221 | " movnti %%eax, 32(%3)\n" |
222 | " movnti %%edx, 36(%3)\n" |
223 | "12: movl 40(%4), %%eax\n" |
224 | "71: movl 44(%4), %%edx\n" |
225 | " movnti %%eax, 40(%3)\n" |
226 | " movnti %%edx, 44(%3)\n" |
227 | "13: movl 48(%4), %%eax\n" |
228 | "81: movl 52(%4), %%edx\n" |
229 | " movnti %%eax, 48(%3)\n" |
230 | " movnti %%edx, 52(%3)\n" |
231 | "14: movl 56(%4), %%eax\n" |
232 | "91: movl 60(%4), %%edx\n" |
233 | " movnti %%eax, 56(%3)\n" |
234 | " movnti %%edx, 60(%3)\n" |
235 | " addl $-64, %0\n" |
236 | " addl $64, %4\n" |
237 | " addl $64, %3\n" |
238 | " cmpl $63, %0\n" |
239 | " ja 0b\n" |
240 | " sfence \n" |
241 | "5: movl %0, %%eax\n" |
242 | " shrl $2, %0\n" |
243 | " andl $3, %%eax\n" |
244 | " cld\n" |
245 | "6: rep; movsl\n" |
246 | " movl %%eax,%0\n" |
247 | "7: rep; movsb\n" |
248 | "8:\n" |
249 | _ASM_EXTABLE_UA(0b, 8b) |
250 | _ASM_EXTABLE_UA(1b, 8b) |
251 | _ASM_EXTABLE_UA(2b, 8b) |
252 | _ASM_EXTABLE_UA(21b, 8b) |
253 | _ASM_EXTABLE_UA(3b, 8b) |
254 | _ASM_EXTABLE_UA(31b, 8b) |
255 | _ASM_EXTABLE_UA(4b, 8b) |
256 | _ASM_EXTABLE_UA(41b, 8b) |
257 | _ASM_EXTABLE_UA(10b, 8b) |
258 | _ASM_EXTABLE_UA(51b, 8b) |
259 | _ASM_EXTABLE_UA(11b, 8b) |
260 | _ASM_EXTABLE_UA(61b, 8b) |
261 | _ASM_EXTABLE_UA(12b, 8b) |
262 | _ASM_EXTABLE_UA(71b, 8b) |
263 | _ASM_EXTABLE_UA(13b, 8b) |
264 | _ASM_EXTABLE_UA(81b, 8b) |
265 | _ASM_EXTABLE_UA(14b, 8b) |
266 | _ASM_EXTABLE_UA(91b, 8b) |
267 | _ASM_EXTABLE_TYPE_REG(6b, 8b, EX_TYPE_UCOPY_LEN4, %%eax) |
268 | _ASM_EXTABLE_UA(7b, 8b) |
269 | : "=&c" (size), "=&D" (d0), "=&S" (d1) |
270 | : "1" (to), "2" (from), "0" (size) |
271 | : "eax" , "edx" , "memory" ); |
272 | return size; |
273 | } |
274 | |
275 | #else |
276 | |
277 | /* |
278 | * Leave these declared but undefined. They should not be any references to |
279 | * them |
280 | */ |
281 | unsigned long __copy_user_intel(void __user *to, const void *from, |
282 | unsigned long size); |
283 | #endif /* CONFIG_X86_INTEL_USERCOPY */ |
284 | |
285 | /* Generic arbitrary sized copy. */ |
286 | #define __copy_user(to, from, size) \ |
287 | do { \ |
288 | int __d0, __d1, __d2; \ |
289 | __asm__ __volatile__( \ |
290 | " cmp $7,%0\n" \ |
291 | " jbe 1f\n" \ |
292 | " movl %1,%0\n" \ |
293 | " negl %0\n" \ |
294 | " andl $7,%0\n" \ |
295 | " subl %0,%3\n" \ |
296 | "4: rep; movsb\n" \ |
297 | " movl %3,%0\n" \ |
298 | " shrl $2,%0\n" \ |
299 | " andl $3,%3\n" \ |
300 | " .align 2,0x90\n" \ |
301 | "0: rep; movsl\n" \ |
302 | " movl %3,%0\n" \ |
303 | "1: rep; movsb\n" \ |
304 | "2:\n" \ |
305 | _ASM_EXTABLE_TYPE_REG(4b, 2b, EX_TYPE_UCOPY_LEN1, %3) \ |
306 | _ASM_EXTABLE_TYPE_REG(0b, 2b, EX_TYPE_UCOPY_LEN4, %3) \ |
307 | _ASM_EXTABLE_UA(1b, 2b) \ |
308 | : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2) \ |
309 | : "3"(size), "0"(size), "1"(to), "2"(from) \ |
310 | : "memory"); \ |
311 | } while (0) |
312 | |
313 | unsigned long __copy_user_ll(void *to, const void *from, unsigned long n) |
314 | { |
315 | __uaccess_begin_nospec(); |
316 | if (movsl_is_ok(to, from, n)) |
317 | __copy_user(to, from, n); |
318 | else |
319 | n = __copy_user_intel(to, from, size: n); |
320 | __uaccess_end(); |
321 | return n; |
322 | } |
323 | EXPORT_SYMBOL(__copy_user_ll); |
324 | |
325 | unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *from, |
326 | unsigned long n) |
327 | { |
328 | __uaccess_begin_nospec(); |
329 | #ifdef CONFIG_X86_INTEL_USERCOPY |
330 | if (n > 64 && static_cpu_has(X86_FEATURE_XMM2)) |
331 | n = __copy_user_intel_nocache(to, from, n); |
332 | else |
333 | __copy_user(to, from, n); |
334 | #else |
335 | __copy_user(to, from, n); |
336 | #endif |
337 | __uaccess_end(); |
338 | return n; |
339 | } |
340 | EXPORT_SYMBOL(__copy_from_user_ll_nocache_nozero); |
341 | |