1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Access kernel or user memory without faulting. |
4 | */ |
5 | #include <linux/export.h> |
6 | #include <linux/mm.h> |
7 | #include <linux/uaccess.h> |
8 | #include <asm/tlb.h> |
9 | |
10 | bool __weak copy_from_kernel_nofault_allowed(const void *unsafe_src, |
11 | size_t size) |
12 | { |
13 | return true; |
14 | } |
15 | |
16 | #define copy_from_kernel_nofault_loop(dst, src, len, type, err_label) \ |
17 | while (len >= sizeof(type)) { \ |
18 | __get_kernel_nofault(dst, src, type, err_label); \ |
19 | dst += sizeof(type); \ |
20 | src += sizeof(type); \ |
21 | len -= sizeof(type); \ |
22 | } |
23 | |
24 | long copy_from_kernel_nofault(void *dst, const void *src, size_t size) |
25 | { |
26 | unsigned long align = 0; |
27 | |
28 | if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) |
29 | align = (unsigned long)dst | (unsigned long)src; |
30 | |
31 | if (!copy_from_kernel_nofault_allowed(unsafe_src: src, size)) |
32 | return -ERANGE; |
33 | |
34 | pagefault_disable(); |
35 | if (!(align & 7)) |
36 | copy_from_kernel_nofault_loop(dst, src, size, u64, Efault); |
37 | if (!(align & 3)) |
38 | copy_from_kernel_nofault_loop(dst, src, size, u32, Efault); |
39 | if (!(align & 1)) |
40 | copy_from_kernel_nofault_loop(dst, src, size, u16, Efault); |
41 | copy_from_kernel_nofault_loop(dst, src, size, u8, Efault); |
42 | pagefault_enable(); |
43 | return 0; |
44 | Efault: |
45 | pagefault_enable(); |
46 | return -EFAULT; |
47 | } |
48 | EXPORT_SYMBOL_GPL(copy_from_kernel_nofault); |
49 | |
50 | #define copy_to_kernel_nofault_loop(dst, src, len, type, err_label) \ |
51 | while (len >= sizeof(type)) { \ |
52 | __put_kernel_nofault(dst, src, type, err_label); \ |
53 | dst += sizeof(type); \ |
54 | src += sizeof(type); \ |
55 | len -= sizeof(type); \ |
56 | } |
57 | |
58 | long copy_to_kernel_nofault(void *dst, const void *src, size_t size) |
59 | { |
60 | unsigned long align = 0; |
61 | |
62 | if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) |
63 | align = (unsigned long)dst | (unsigned long)src; |
64 | |
65 | pagefault_disable(); |
66 | if (!(align & 7)) |
67 | copy_to_kernel_nofault_loop(dst, src, size, u64, Efault); |
68 | if (!(align & 3)) |
69 | copy_to_kernel_nofault_loop(dst, src, size, u32, Efault); |
70 | if (!(align & 1)) |
71 | copy_to_kernel_nofault_loop(dst, src, size, u16, Efault); |
72 | copy_to_kernel_nofault_loop(dst, src, size, u8, Efault); |
73 | pagefault_enable(); |
74 | return 0; |
75 | Efault: |
76 | pagefault_enable(); |
77 | return -EFAULT; |
78 | } |
79 | |
80 | long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count) |
81 | { |
82 | const void *src = unsafe_addr; |
83 | |
84 | if (unlikely(count <= 0)) |
85 | return 0; |
86 | if (!copy_from_kernel_nofault_allowed(unsafe_src: unsafe_addr, size: count)) |
87 | return -ERANGE; |
88 | |
89 | pagefault_disable(); |
90 | do { |
91 | __get_kernel_nofault(dst, src, u8, Efault); |
92 | dst++; |
93 | src++; |
94 | } while (dst[-1] && src - unsafe_addr < count); |
95 | pagefault_enable(); |
96 | |
97 | dst[-1] = '\0'; |
98 | return src - unsafe_addr; |
99 | Efault: |
100 | pagefault_enable(); |
101 | dst[0] = '\0'; |
102 | return -EFAULT; |
103 | } |
104 | |
105 | /** |
106 | * copy_from_user_nofault(): safely attempt to read from a user-space location |
107 | * @dst: pointer to the buffer that shall take the data |
108 | * @src: address to read from. This must be a user address. |
109 | * @size: size of the data chunk |
110 | * |
111 | * Safely read from user address @src to the buffer at @dst. If a kernel fault |
112 | * happens, handle that and return -EFAULT. |
113 | */ |
114 | long copy_from_user_nofault(void *dst, const void __user *src, size_t size) |
115 | { |
116 | long ret = -EFAULT; |
117 | |
118 | if (!__access_ok(ptr: src, size)) |
119 | return ret; |
120 | |
121 | if (!nmi_uaccess_okay()) |
122 | return ret; |
123 | |
124 | pagefault_disable(); |
125 | ret = __copy_from_user_inatomic(to: dst, from: src, n: size); |
126 | pagefault_enable(); |
127 | |
128 | if (ret) |
129 | return -EFAULT; |
130 | return 0; |
131 | } |
132 | EXPORT_SYMBOL_GPL(copy_from_user_nofault); |
133 | |
134 | /** |
135 | * copy_to_user_nofault(): safely attempt to write to a user-space location |
136 | * @dst: address to write to |
137 | * @src: pointer to the data that shall be written |
138 | * @size: size of the data chunk |
139 | * |
140 | * Safely write to address @dst from the buffer at @src. If a kernel fault |
141 | * happens, handle that and return -EFAULT. |
142 | */ |
143 | long copy_to_user_nofault(void __user *dst, const void *src, size_t size) |
144 | { |
145 | long ret = -EFAULT; |
146 | |
147 | if (access_ok(dst, size)) { |
148 | pagefault_disable(); |
149 | ret = __copy_to_user_inatomic(to: dst, from: src, n: size); |
150 | pagefault_enable(); |
151 | } |
152 | |
153 | if (ret) |
154 | return -EFAULT; |
155 | return 0; |
156 | } |
157 | EXPORT_SYMBOL_GPL(copy_to_user_nofault); |
158 | |
159 | /** |
160 | * strncpy_from_user_nofault: - Copy a NUL terminated string from unsafe user |
161 | * address. |
162 | * @dst: Destination address, in kernel space. This buffer must be at |
163 | * least @count bytes long. |
164 | * @unsafe_addr: Unsafe user address. |
165 | * @count: Maximum number of bytes to copy, including the trailing NUL. |
166 | * |
167 | * Copies a NUL-terminated string from unsafe user address to kernel buffer. |
168 | * |
169 | * On success, returns the length of the string INCLUDING the trailing NUL. |
170 | * |
171 | * If access fails, returns -EFAULT (some data may have been copied |
172 | * and the trailing NUL added). |
173 | * |
174 | * If @count is smaller than the length of the string, copies @count-1 bytes, |
175 | * sets the last byte of @dst buffer to NUL and returns @count. |
176 | */ |
177 | long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr, |
178 | long count) |
179 | { |
180 | long ret; |
181 | |
182 | if (unlikely(count <= 0)) |
183 | return 0; |
184 | |
185 | pagefault_disable(); |
186 | ret = strncpy_from_user(dst, src: unsafe_addr, count); |
187 | pagefault_enable(); |
188 | |
189 | if (ret >= count) { |
190 | ret = count; |
191 | dst[ret - 1] = '\0'; |
192 | } else if (ret > 0) { |
193 | ret++; |
194 | } |
195 | |
196 | return ret; |
197 | } |
198 | |
199 | /** |
200 | * strnlen_user_nofault: - Get the size of a user string INCLUDING final NUL. |
201 | * @unsafe_addr: The string to measure. |
202 | * @count: Maximum count (including NUL) |
203 | * |
204 | * Get the size of a NUL-terminated string in user space without pagefault. |
205 | * |
206 | * Returns the size of the string INCLUDING the terminating NUL. |
207 | * |
208 | * If the string is too long, returns a number larger than @count. User |
209 | * has to check the return value against "> count". |
210 | * On exception (or invalid count), returns 0. |
211 | * |
212 | * Unlike strnlen_user, this can be used from IRQ handler etc. because |
213 | * it disables pagefaults. |
214 | */ |
215 | long strnlen_user_nofault(const void __user *unsafe_addr, long count) |
216 | { |
217 | int ret; |
218 | |
219 | pagefault_disable(); |
220 | ret = strnlen_user(str: unsafe_addr, n: count); |
221 | pagefault_enable(); |
222 | |
223 | return ret; |
224 | } |
225 | |
226 | void __copy_overflow(int size, unsigned long count) |
227 | { |
228 | WARN(1, "Buffer overflow detected (%d < %lu)!\n" , size, count); |
229 | } |
230 | EXPORT_SYMBOL(__copy_overflow); |
231 | |