1// SPDX-License-Identifier: GPL-2.0
2/*
3 * linux/fs/read_write.c
4 *
5 * Copyright (C) 1991, 1992 Linus Torvalds
6 */
7
8#include <linux/slab.h>
9#include <linux/stat.h>
10#include <linux/sched/xacct.h>
11#include <linux/fcntl.h>
12#include <linux/file.h>
13#include <linux/uio.h>
14#include <linux/fsnotify.h>
15#include <linux/security.h>
16#include <linux/export.h>
17#include <linux/syscalls.h>
18#include <linux/pagemap.h>
19#include <linux/splice.h>
20#include <linux/compat.h>
21#include <linux/mount.h>
22#include <linux/fs.h>
23#include "internal.h"
24
25#include <linux/uaccess.h>
26#include <asm/unistd.h>
27
28const struct file_operations generic_ro_fops = {
29 .llseek = generic_file_llseek,
30 .read_iter = generic_file_read_iter,
31 .mmap = generic_file_readonly_mmap,
32 .splice_read = generic_file_splice_read,
33};
34
35EXPORT_SYMBOL(generic_ro_fops);
36
37static inline bool unsigned_offsets(struct file *file)
38{
39 return file->f_mode & FMODE_UNSIGNED_OFFSET;
40}
41
42/**
43 * vfs_setpos - update the file offset for lseek
44 * @file: file structure in question
45 * @offset: file offset to seek to
46 * @maxsize: maximum file size
47 *
48 * This is a low-level filesystem helper for updating the file offset to
49 * the value specified by @offset if the given offset is valid and it is
50 * not equal to the current file offset.
51 *
52 * Return the specified offset on success and -EINVAL on invalid offset.
53 */
54loff_t vfs_setpos(struct file *file, loff_t offset, loff_t maxsize)
55{
56 if (offset < 0 && !unsigned_offsets(file))
57 return -EINVAL;
58 if (offset > maxsize)
59 return -EINVAL;
60
61 if (offset != file->f_pos) {
62 file->f_pos = offset;
63 file->f_version = 0;
64 }
65 return offset;
66}
67EXPORT_SYMBOL(vfs_setpos);
68
69/**
70 * generic_file_llseek_size - generic llseek implementation for regular files
71 * @file: file structure to seek on
72 * @offset: file offset to seek to
73 * @whence: type of seek
74 * @size: max size of this file in file system
75 * @eof: offset used for SEEK_END position
76 *
77 * This is a variant of generic_file_llseek that allows passing in a custom
78 * maximum file size and a custom EOF position, for e.g. hashed directories
79 *
80 * Synchronization:
81 * SEEK_SET and SEEK_END are unsynchronized (but atomic on 64bit platforms)
82 * SEEK_CUR is synchronized against other SEEK_CURs, but not read/writes.
83 * read/writes behave like SEEK_SET against seeks.
84 */
85loff_t
86generic_file_llseek_size(struct file *file, loff_t offset, int whence,
87 loff_t maxsize, loff_t eof)
88{
89 switch (whence) {
90 case SEEK_END:
91 offset += eof;
92 break;
93 case SEEK_CUR:
94 /*
95 * Here we special-case the lseek(fd, 0, SEEK_CUR)
96 * position-querying operation. Avoid rewriting the "same"
97 * f_pos value back to the file because a concurrent read(),
98 * write() or lseek() might have altered it
99 */
100 if (offset == 0)
101 return file->f_pos;
102 /*
103 * f_lock protects against read/modify/write race with other
104 * SEEK_CURs. Note that parallel writes and reads behave
105 * like SEEK_SET.
106 */
107 spin_lock(&file->f_lock);
108 offset = vfs_setpos(file, file->f_pos + offset, maxsize);
109 spin_unlock(&file->f_lock);
110 return offset;
111 case SEEK_DATA:
112 /*
113 * In the generic case the entire file is data, so as long as
114 * offset isn't at the end of the file then the offset is data.
115 */
116 if ((unsigned long long)offset >= eof)
117 return -ENXIO;
118 break;
119 case SEEK_HOLE:
120 /*
121 * There is a virtual hole at the end of the file, so as long as
122 * offset isn't i_size or larger, return i_size.
123 */
124 if ((unsigned long long)offset >= eof)
125 return -ENXIO;
126 offset = eof;
127 break;
128 }
129
130 return vfs_setpos(file, offset, maxsize);
131}
132EXPORT_SYMBOL(generic_file_llseek_size);
133
134/**
135 * generic_file_llseek - generic llseek implementation for regular files
136 * @file: file structure to seek on
137 * @offset: file offset to seek to
138 * @whence: type of seek
139 *
140 * This is a generic implemenation of ->llseek useable for all normal local
141 * filesystems. It just updates the file offset to the value specified by
142 * @offset and @whence.
143 */
144loff_t generic_file_llseek(struct file *file, loff_t offset, int whence)
145{
146 struct inode *inode = file->f_mapping->host;
147
148 return generic_file_llseek_size(file, offset, whence,
149 inode->i_sb->s_maxbytes,
150 i_size_read(inode));
151}
152EXPORT_SYMBOL(generic_file_llseek);
153
154/**
155 * fixed_size_llseek - llseek implementation for fixed-sized devices
156 * @file: file structure to seek on
157 * @offset: file offset to seek to
158 * @whence: type of seek
159 * @size: size of the file
160 *
161 */
162loff_t fixed_size_llseek(struct file *file, loff_t offset, int whence, loff_t size)
163{
164 switch (whence) {
165 case SEEK_SET: case SEEK_CUR: case SEEK_END:
166 return generic_file_llseek_size(file, offset, whence,
167 size, size);
168 default:
169 return -EINVAL;
170 }
171}
172EXPORT_SYMBOL(fixed_size_llseek);
173
174/**
175 * no_seek_end_llseek - llseek implementation for fixed-sized devices
176 * @file: file structure to seek on
177 * @offset: file offset to seek to
178 * @whence: type of seek
179 *
180 */
181loff_t no_seek_end_llseek(struct file *file, loff_t offset, int whence)
182{
183 switch (whence) {
184 case SEEK_SET: case SEEK_CUR:
185 return generic_file_llseek_size(file, offset, whence,
186 OFFSET_MAX, 0);
187 default:
188 return -EINVAL;
189 }
190}
191EXPORT_SYMBOL(no_seek_end_llseek);
192
193/**
194 * no_seek_end_llseek_size - llseek implementation for fixed-sized devices
195 * @file: file structure to seek on
196 * @offset: file offset to seek to
197 * @whence: type of seek
198 * @size: maximal offset allowed
199 *
200 */
201loff_t no_seek_end_llseek_size(struct file *file, loff_t offset, int whence, loff_t size)
202{
203 switch (whence) {
204 case SEEK_SET: case SEEK_CUR:
205 return generic_file_llseek_size(file, offset, whence,
206 size, 0);
207 default:
208 return -EINVAL;
209 }
210}
211EXPORT_SYMBOL(no_seek_end_llseek_size);
212
213/**
214 * noop_llseek - No Operation Performed llseek implementation
215 * @file: file structure to seek on
216 * @offset: file offset to seek to
217 * @whence: type of seek
218 *
219 * This is an implementation of ->llseek useable for the rare special case when
220 * userspace expects the seek to succeed but the (device) file is actually not
221 * able to perform the seek. In this case you use noop_llseek() instead of
222 * falling back to the default implementation of ->llseek.
223 */
224loff_t noop_llseek(struct file *file, loff_t offset, int whence)
225{
226 return file->f_pos;
227}
228EXPORT_SYMBOL(noop_llseek);
229
230loff_t no_llseek(struct file *file, loff_t offset, int whence)
231{
232 return -ESPIPE;
233}
234EXPORT_SYMBOL(no_llseek);
235
236loff_t default_llseek(struct file *file, loff_t offset, int whence)
237{
238 struct inode *inode = file_inode(file);
239 loff_t retval;
240
241 inode_lock(inode);
242 switch (whence) {
243 case SEEK_END:
244 offset += i_size_read(inode);
245 break;
246 case SEEK_CUR:
247 if (offset == 0) {
248 retval = file->f_pos;
249 goto out;
250 }
251 offset += file->f_pos;
252 break;
253 case SEEK_DATA:
254 /*
255 * In the generic case the entire file is data, so as
256 * long as offset isn't at the end of the file then the
257 * offset is data.
258 */
259 if (offset >= inode->i_size) {
260 retval = -ENXIO;
261 goto out;
262 }
263 break;
264 case SEEK_HOLE:
265 /*
266 * There is a virtual hole at the end of the file, so
267 * as long as offset isn't i_size or larger, return
268 * i_size.
269 */
270 if (offset >= inode->i_size) {
271 retval = -ENXIO;
272 goto out;
273 }
274 offset = inode->i_size;
275 break;
276 }
277 retval = -EINVAL;
278 if (offset >= 0 || unsigned_offsets(file)) {
279 if (offset != file->f_pos) {
280 file->f_pos = offset;
281 file->f_version = 0;
282 }
283 retval = offset;
284 }
285out:
286 inode_unlock(inode);
287 return retval;
288}
289EXPORT_SYMBOL(default_llseek);
290
291loff_t vfs_llseek(struct file *file, loff_t offset, int whence)
292{
293 loff_t (*fn)(struct file *, loff_t, int);
294
295 fn = no_llseek;
296 if (file->f_mode & FMODE_LSEEK) {
297 if (file->f_op->llseek)
298 fn = file->f_op->llseek;
299 }
300 return fn(file, offset, whence);
301}
302EXPORT_SYMBOL(vfs_llseek);
303
304off_t ksys_lseek(unsigned int fd, off_t offset, unsigned int whence)
305{
306 off_t retval;
307 struct fd f = fdget_pos(fd);
308 if (!f.file)
309 return -EBADF;
310
311 retval = -EINVAL;
312 if (whence <= SEEK_MAX) {
313 loff_t res = vfs_llseek(f.file, offset, whence);
314 retval = res;
315 if (res != (loff_t)retval)
316 retval = -EOVERFLOW; /* LFS: should only happen on 32 bit platforms */
317 }
318 fdput_pos(f);
319 return retval;
320}
321
322SYSCALL_DEFINE3(lseek, unsigned int, fd, off_t, offset, unsigned int, whence)
323{
324 return ksys_lseek(fd, offset, whence);
325}
326
327#ifdef CONFIG_COMPAT
328COMPAT_SYSCALL_DEFINE3(lseek, unsigned int, fd, compat_off_t, offset, unsigned int, whence)
329{
330 return ksys_lseek(fd, offset, whence);
331}
332#endif
333
334#if !defined(CONFIG_64BIT) || defined(CONFIG_COMPAT)
335SYSCALL_DEFINE5(llseek, unsigned int, fd, unsigned long, offset_high,
336 unsigned long, offset_low, loff_t __user *, result,
337 unsigned int, whence)
338{
339 int retval;
340 struct fd f = fdget_pos(fd);
341 loff_t offset;
342
343 if (!f.file)
344 return -EBADF;
345
346 retval = -EINVAL;
347 if (whence > SEEK_MAX)
348 goto out_putf;
349
350 offset = vfs_llseek(f.file, ((loff_t) offset_high << 32) | offset_low,
351 whence);
352
353 retval = (int)offset;
354 if (offset >= 0) {
355 retval = -EFAULT;
356 if (!copy_to_user(result, &offset, sizeof(offset)))
357 retval = 0;
358 }
359out_putf:
360 fdput_pos(f);
361 return retval;
362}
363#endif
364
365int rw_verify_area(int read_write, struct file *file, const loff_t *ppos, size_t count)
366{
367 struct inode *inode;
368 loff_t pos;
369 int retval = -EINVAL;
370
371 inode = file_inode(file);
372 if (unlikely((ssize_t) count < 0))
373 return retval;
374 pos = *ppos;
375 if (unlikely(pos < 0)) {
376 if (!unsigned_offsets(file))
377 return retval;
378 if (count >= -pos) /* both values are in 0..LLONG_MAX */
379 return -EOVERFLOW;
380 } else if (unlikely((loff_t) (pos + count) < 0)) {
381 if (!unsigned_offsets(file))
382 return retval;
383 }
384
385 if (unlikely(inode->i_flctx && mandatory_lock(inode))) {
386 retval = locks_mandatory_area(inode, file, pos, pos + count - 1,
387 read_write == READ ? F_RDLCK : F_WRLCK);
388 if (retval < 0)
389 return retval;
390 }
391 return security_file_permission(file,
392 read_write == READ ? MAY_READ : MAY_WRITE);
393}
394
395static ssize_t new_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos)
396{
397 struct iovec iov = { .iov_base = buf, .iov_len = len };
398 struct kiocb kiocb;
399 struct iov_iter iter;
400 ssize_t ret;
401
402 init_sync_kiocb(&kiocb, filp);
403 kiocb.ki_pos = *ppos;
404 iov_iter_init(&iter, READ, &iov, 1, len);
405
406 ret = call_read_iter(filp, &kiocb, &iter);
407 BUG_ON(ret == -EIOCBQUEUED);
408 *ppos = kiocb.ki_pos;
409 return ret;
410}
411
412ssize_t __vfs_read(struct file *file, char __user *buf, size_t count,
413 loff_t *pos)
414{
415 if (file->f_op->read)
416 return file->f_op->read(file, buf, count, pos);
417 else if (file->f_op->read_iter)
418 return new_sync_read(file, buf, count, pos);
419 else
420 return -EINVAL;
421}
422
423ssize_t kernel_read(struct file *file, void *buf, size_t count, loff_t *pos)
424{
425 mm_segment_t old_fs;
426 ssize_t result;
427
428 old_fs = get_fs();
429 set_fs(KERNEL_DS);
430 /* The cast to a user pointer is valid due to the set_fs() */
431 result = vfs_read(file, (void __user *)buf, count, pos);
432 set_fs(old_fs);
433 return result;
434}
435EXPORT_SYMBOL(kernel_read);
436
437ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos)
438{
439 ssize_t ret;
440
441 if (!(file->f_mode & FMODE_READ))
442 return -EBADF;
443 if (!(file->f_mode & FMODE_CAN_READ))
444 return -EINVAL;
445 if (unlikely(!access_ok(buf, count)))
446 return -EFAULT;
447
448 ret = rw_verify_area(READ, file, pos, count);
449 if (!ret) {
450 if (count > MAX_RW_COUNT)
451 count = MAX_RW_COUNT;
452 ret = __vfs_read(file, buf, count, pos);
453 if (ret > 0) {
454 fsnotify_access(file);
455 add_rchar(current, ret);
456 }
457 inc_syscr(current);
458 }
459
460 return ret;
461}
462
463static ssize_t new_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos)
464{
465 struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = len };
466 struct kiocb kiocb;
467 struct iov_iter iter;
468 ssize_t ret;
469
470 init_sync_kiocb(&kiocb, filp);
471 kiocb.ki_pos = *ppos;
472 iov_iter_init(&iter, WRITE, &iov, 1, len);
473
474 ret = call_write_iter(filp, &kiocb, &iter);
475 BUG_ON(ret == -EIOCBQUEUED);
476 if (ret > 0)
477 *ppos = kiocb.ki_pos;
478 return ret;
479}
480
481static ssize_t __vfs_write(struct file *file, const char __user *p,
482 size_t count, loff_t *pos)
483{
484 if (file->f_op->write)
485 return file->f_op->write(file, p, count, pos);
486 else if (file->f_op->write_iter)
487 return new_sync_write(file, p, count, pos);
488 else
489 return -EINVAL;
490}
491
492ssize_t __kernel_write(struct file *file, const void *buf, size_t count, loff_t *pos)
493{
494 mm_segment_t old_fs;
495 const char __user *p;
496 ssize_t ret;
497
498 if (!(file->f_mode & FMODE_CAN_WRITE))
499 return -EINVAL;
500
501 old_fs = get_fs();
502 set_fs(KERNEL_DS);
503 p = (__force const char __user *)buf;
504 if (count > MAX_RW_COUNT)
505 count = MAX_RW_COUNT;
506 ret = __vfs_write(file, p, count, pos);
507 set_fs(old_fs);
508 if (ret > 0) {
509 fsnotify_modify(file);
510 add_wchar(current, ret);
511 }
512 inc_syscw(current);
513 return ret;
514}
515EXPORT_SYMBOL(__kernel_write);
516
517ssize_t kernel_write(struct file *file, const void *buf, size_t count,
518 loff_t *pos)
519{
520 mm_segment_t old_fs;
521 ssize_t res;
522
523 old_fs = get_fs();
524 set_fs(KERNEL_DS);
525 /* The cast to a user pointer is valid due to the set_fs() */
526 res = vfs_write(file, (__force const char __user *)buf, count, pos);
527 set_fs(old_fs);
528
529 return res;
530}
531EXPORT_SYMBOL(kernel_write);
532
533ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos)
534{
535 ssize_t ret;
536
537 if (!(file->f_mode & FMODE_WRITE))
538 return -EBADF;
539 if (!(file->f_mode & FMODE_CAN_WRITE))
540 return -EINVAL;
541 if (unlikely(!access_ok(buf, count)))
542 return -EFAULT;
543
544 ret = rw_verify_area(WRITE, file, pos, count);
545 if (!ret) {
546 if (count > MAX_RW_COUNT)
547 count = MAX_RW_COUNT;
548 file_start_write(file);
549 ret = __vfs_write(file, buf, count, pos);
550 if (ret > 0) {
551 fsnotify_modify(file);
552 add_wchar(current, ret);
553 }
554 inc_syscw(current);
555 file_end_write(file);
556 }
557
558 return ret;
559}
560
561static inline loff_t file_pos_read(struct file *file)
562{
563 return file->f_pos;
564}
565
566static inline void file_pos_write(struct file *file, loff_t pos)
567{
568 file->f_pos = pos;
569}
570
571ssize_t ksys_read(unsigned int fd, char __user *buf, size_t count)
572{
573 struct fd f = fdget_pos(fd);
574 ssize_t ret = -EBADF;
575
576 if (f.file) {
577 loff_t pos = file_pos_read(f.file);
578 ret = vfs_read(f.file, buf, count, &pos);
579 if (ret >= 0)
580 file_pos_write(f.file, pos);
581 fdput_pos(f);
582 }
583 return ret;
584}
585
586SYSCALL_DEFINE3(read, unsigned int, fd, char __user *, buf, size_t, count)
587{
588 return ksys_read(fd, buf, count);
589}
590
591ssize_t ksys_write(unsigned int fd, const char __user *buf, size_t count)
592{
593 struct fd f = fdget_pos(fd);
594 ssize_t ret = -EBADF;
595
596 if (f.file) {
597 loff_t pos = file_pos_read(f.file);
598 ret = vfs_write(f.file, buf, count, &pos);
599 if (ret >= 0)
600 file_pos_write(f.file, pos);
601 fdput_pos(f);
602 }
603
604 return ret;
605}
606
607SYSCALL_DEFINE3(write, unsigned int, fd, const char __user *, buf,
608 size_t, count)
609{
610 return ksys_write(fd, buf, count);
611}
612
613ssize_t ksys_pread64(unsigned int fd, char __user *buf, size_t count,
614 loff_t pos)
615{
616 struct fd f;
617 ssize_t ret = -EBADF;
618
619 if (pos < 0)
620 return -EINVAL;
621
622 f = fdget(fd);
623 if (f.file) {
624 ret = -ESPIPE;
625 if (f.file->f_mode & FMODE_PREAD)
626 ret = vfs_read(f.file, buf, count, &pos);
627 fdput(f);
628 }
629
630 return ret;
631}
632
633SYSCALL_DEFINE4(pread64, unsigned int, fd, char __user *, buf,
634 size_t, count, loff_t, pos)
635{
636 return ksys_pread64(fd, buf, count, pos);
637}
638
639ssize_t ksys_pwrite64(unsigned int fd, const char __user *buf,
640 size_t count, loff_t pos)
641{
642 struct fd f;
643 ssize_t ret = -EBADF;
644
645 if (pos < 0)
646 return -EINVAL;
647
648 f = fdget(fd);
649 if (f.file) {
650 ret = -ESPIPE;
651 if (f.file->f_mode & FMODE_PWRITE)
652 ret = vfs_write(f.file, buf, count, &pos);
653 fdput(f);
654 }
655
656 return ret;
657}
658
659SYSCALL_DEFINE4(pwrite64, unsigned int, fd, const char __user *, buf,
660 size_t, count, loff_t, pos)
661{
662 return ksys_pwrite64(fd, buf, count, pos);
663}
664
665static ssize_t do_iter_readv_writev(struct file *filp, struct iov_iter *iter,
666 loff_t *ppos, int type, rwf_t flags)
667{
668 struct kiocb kiocb;
669 ssize_t ret;
670
671 init_sync_kiocb(&kiocb, filp);
672 ret = kiocb_set_rw_flags(&kiocb, flags);
673 if (ret)
674 return ret;
675 kiocb.ki_pos = *ppos;
676
677 if (type == READ)
678 ret = call_read_iter(filp, &kiocb, iter);
679 else
680 ret = call_write_iter(filp, &kiocb, iter);
681 BUG_ON(ret == -EIOCBQUEUED);
682 *ppos = kiocb.ki_pos;
683 return ret;
684}
685
686/* Do it by hand, with file-ops */
687static ssize_t do_loop_readv_writev(struct file *filp, struct iov_iter *iter,
688 loff_t *ppos, int type, rwf_t flags)
689{
690 ssize_t ret = 0;
691
692 if (flags & ~RWF_HIPRI)
693 return -EOPNOTSUPP;
694
695 while (iov_iter_count(iter)) {
696 struct iovec iovec = iov_iter_iovec(iter);
697 ssize_t nr;
698
699 if (type == READ) {
700 nr = filp->f_op->read(filp, iovec.iov_base,
701 iovec.iov_len, ppos);
702 } else {
703 nr = filp->f_op->write(filp, iovec.iov_base,
704 iovec.iov_len, ppos);
705 }
706
707 if (nr < 0) {
708 if (!ret)
709 ret = nr;
710 break;
711 }
712 ret += nr;
713 if (nr != iovec.iov_len)
714 break;
715 iov_iter_advance(iter, nr);
716 }
717
718 return ret;
719}
720
721/**
722 * rw_copy_check_uvector() - Copy an array of &struct iovec from userspace
723 * into the kernel and check that it is valid.
724 *
725 * @type: One of %CHECK_IOVEC_ONLY, %READ, or %WRITE.
726 * @uvector: Pointer to the userspace array.
727 * @nr_segs: Number of elements in userspace array.
728 * @fast_segs: Number of elements in @fast_pointer.
729 * @fast_pointer: Pointer to (usually small on-stack) kernel array.
730 * @ret_pointer: (output parameter) Pointer to a variable that will point to
731 * either @fast_pointer, a newly allocated kernel array, or NULL,
732 * depending on which array was used.
733 *
734 * This function copies an array of &struct iovec of @nr_segs from
735 * userspace into the kernel and checks that each element is valid (e.g.
736 * it does not point to a kernel address or cause overflow by being too
737 * large, etc.).
738 *
739 * As an optimization, the caller may provide a pointer to a small
740 * on-stack array in @fast_pointer, typically %UIO_FASTIOV elements long
741 * (the size of this array, or 0 if unused, should be given in @fast_segs).
742 *
743 * @ret_pointer will always point to the array that was used, so the
744 * caller must take care not to call kfree() on it e.g. in case the
745 * @fast_pointer array was used and it was allocated on the stack.
746 *
747 * Return: The total number of bytes covered by the iovec array on success
748 * or a negative error code on error.
749 */
750ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector,
751 unsigned long nr_segs, unsigned long fast_segs,
752 struct iovec *fast_pointer,
753 struct iovec **ret_pointer)
754{
755 unsigned long seg;
756 ssize_t ret;
757 struct iovec *iov = fast_pointer;
758
759 /*
760 * SuS says "The readv() function *may* fail if the iovcnt argument
761 * was less than or equal to 0, or greater than {IOV_MAX}. Linux has
762 * traditionally returned zero for zero segments, so...
763 */
764 if (nr_segs == 0) {
765 ret = 0;
766 goto out;
767 }
768
769 /*
770 * First get the "struct iovec" from user memory and
771 * verify all the pointers
772 */
773 if (nr_segs > UIO_MAXIOV) {
774 ret = -EINVAL;
775 goto out;
776 }
777 if (nr_segs > fast_segs) {
778 iov = kmalloc_array(nr_segs, sizeof(struct iovec), GFP_KERNEL);
779 if (iov == NULL) {
780 ret = -ENOMEM;
781 goto out;
782 }
783 }
784 if (copy_from_user(iov, uvector, nr_segs*sizeof(*uvector))) {
785 ret = -EFAULT;
786 goto out;
787 }
788
789 /*
790 * According to the Single Unix Specification we should return EINVAL
791 * if an element length is < 0 when cast to ssize_t or if the
792 * total length would overflow the ssize_t return value of the
793 * system call.
794 *
795 * Linux caps all read/write calls to MAX_RW_COUNT, and avoids the
796 * overflow case.
797 */
798 ret = 0;
799 for (seg = 0; seg < nr_segs; seg++) {
800 void __user *buf = iov[seg].iov_base;
801 ssize_t len = (ssize_t)iov[seg].iov_len;
802
803 /* see if we we're about to use an invalid len or if
804 * it's about to overflow ssize_t */
805 if (len < 0) {
806 ret = -EINVAL;
807 goto out;
808 }
809 if (type >= 0
810 && unlikely(!access_ok(buf, len))) {
811 ret = -EFAULT;
812 goto out;
813 }
814 if (len > MAX_RW_COUNT - ret) {
815 len = MAX_RW_COUNT - ret;
816 iov[seg].iov_len = len;
817 }
818 ret += len;
819 }
820out:
821 *ret_pointer = iov;
822 return ret;
823}
824
825#ifdef CONFIG_COMPAT
826ssize_t compat_rw_copy_check_uvector(int type,
827 const struct compat_iovec __user *uvector, unsigned long nr_segs,
828 unsigned long fast_segs, struct iovec *fast_pointer,
829 struct iovec **ret_pointer)
830{
831 compat_ssize_t tot_len;
832 struct iovec *iov = *ret_pointer = fast_pointer;
833 ssize_t ret = 0;
834 int seg;
835
836 /*
837 * SuS says "The readv() function *may* fail if the iovcnt argument
838 * was less than or equal to 0, or greater than {IOV_MAX}. Linux has
839 * traditionally returned zero for zero segments, so...
840 */
841 if (nr_segs == 0)
842 goto out;
843
844 ret = -EINVAL;
845 if (nr_segs > UIO_MAXIOV)
846 goto out;
847 if (nr_segs > fast_segs) {
848 ret = -ENOMEM;
849 iov = kmalloc_array(nr_segs, sizeof(struct iovec), GFP_KERNEL);
850 if (iov == NULL)
851 goto out;
852 }
853 *ret_pointer = iov;
854
855 ret = -EFAULT;
856 if (!access_ok(uvector, nr_segs*sizeof(*uvector)))
857 goto out;
858
859 /*
860 * Single unix specification:
861 * We should -EINVAL if an element length is not >= 0 and fitting an
862 * ssize_t.
863 *
864 * In Linux, the total length is limited to MAX_RW_COUNT, there is
865 * no overflow possibility.
866 */
867 tot_len = 0;
868 ret = -EINVAL;
869 for (seg = 0; seg < nr_segs; seg++) {
870 compat_uptr_t buf;
871 compat_ssize_t len;
872
873 if (__get_user(len, &uvector->iov_len) ||
874 __get_user(buf, &uvector->iov_base)) {
875 ret = -EFAULT;
876 goto out;
877 }
878 if (len < 0) /* size_t not fitting in compat_ssize_t .. */
879 goto out;
880 if (type >= 0 &&
881 !access_ok(compat_ptr(buf), len)) {
882 ret = -EFAULT;
883 goto out;
884 }
885 if (len > MAX_RW_COUNT - tot_len)
886 len = MAX_RW_COUNT - tot_len;
887 tot_len += len;
888 iov->iov_base = compat_ptr(buf);
889 iov->iov_len = (compat_size_t) len;
890 uvector++;
891 iov++;
892 }
893 ret = tot_len;
894
895out:
896 return ret;
897}
898#endif
899
900static ssize_t do_iter_read(struct file *file, struct iov_iter *iter,
901 loff_t *pos, rwf_t flags)
902{
903 size_t tot_len;
904 ssize_t ret = 0;
905
906 if (!(file->f_mode & FMODE_READ))
907 return -EBADF;
908 if (!(file->f_mode & FMODE_CAN_READ))
909 return -EINVAL;
910
911 tot_len = iov_iter_count(iter);
912 if (!tot_len)
913 goto out;
914 ret = rw_verify_area(READ, file, pos, tot_len);
915 if (ret < 0)
916 return ret;
917
918 if (file->f_op->read_iter)
919 ret = do_iter_readv_writev(file, iter, pos, READ, flags);
920 else
921 ret = do_loop_readv_writev(file, iter, pos, READ, flags);
922out:
923 if (ret >= 0)
924 fsnotify_access(file);
925 return ret;
926}
927
928ssize_t vfs_iter_read(struct file *file, struct iov_iter *iter, loff_t *ppos,
929 rwf_t flags)
930{
931 if (!file->f_op->read_iter)
932 return -EINVAL;
933 return do_iter_read(file, iter, ppos, flags);
934}
935EXPORT_SYMBOL(vfs_iter_read);
936
937static ssize_t do_iter_write(struct file *file, struct iov_iter *iter,
938 loff_t *pos, rwf_t flags)
939{
940 size_t tot_len;
941 ssize_t ret = 0;
942
943 if (!(file->f_mode & FMODE_WRITE))
944 return -EBADF;
945 if (!(file->f_mode & FMODE_CAN_WRITE))
946 return -EINVAL;
947
948 tot_len = iov_iter_count(iter);
949 if (!tot_len)
950 return 0;
951 ret = rw_verify_area(WRITE, file, pos, tot_len);
952 if (ret < 0)
953 return ret;
954
955 if (file->f_op->write_iter)
956 ret = do_iter_readv_writev(file, iter, pos, WRITE, flags);
957 else
958 ret = do_loop_readv_writev(file, iter, pos, WRITE, flags);
959 if (ret > 0)
960 fsnotify_modify(file);
961 return ret;
962}
963
964ssize_t vfs_iter_write(struct file *file, struct iov_iter *iter, loff_t *ppos,
965 rwf_t flags)
966{
967 if (!file->f_op->write_iter)
968 return -EINVAL;
969 return do_iter_write(file, iter, ppos, flags);
970}
971EXPORT_SYMBOL(vfs_iter_write);
972
973ssize_t vfs_readv(struct file *file, const struct iovec __user *vec,
974 unsigned long vlen, loff_t *pos, rwf_t flags)
975{
976 struct iovec iovstack[UIO_FASTIOV];
977 struct iovec *iov = iovstack;
978 struct iov_iter iter;
979 ssize_t ret;
980
981 ret = import_iovec(READ, vec, vlen, ARRAY_SIZE(iovstack), &iov, &iter);
982 if (ret >= 0) {
983 ret = do_iter_read(file, &iter, pos, flags);
984 kfree(iov);
985 }
986
987 return ret;
988}
989
990static ssize_t vfs_writev(struct file *file, const struct iovec __user *vec,
991 unsigned long vlen, loff_t *pos, rwf_t flags)
992{
993 struct iovec iovstack[UIO_FASTIOV];
994 struct iovec *iov = iovstack;
995 struct iov_iter iter;
996 ssize_t ret;
997
998 ret = import_iovec(WRITE, vec, vlen, ARRAY_SIZE(iovstack), &iov, &iter);
999 if (ret >= 0) {
1000 file_start_write(file);
1001 ret = do_iter_write(file, &iter, pos, flags);
1002 file_end_write(file);
1003 kfree(iov);
1004 }
1005 return ret;
1006}
1007
1008static ssize_t do_readv(unsigned long fd, const struct iovec __user *vec,
1009 unsigned long vlen, rwf_t flags)
1010{
1011 struct fd f = fdget_pos(fd);
1012 ssize_t ret = -EBADF;
1013
1014 if (f.file) {
1015 loff_t pos = file_pos_read(f.file);
1016 ret = vfs_readv(f.file, vec, vlen, &pos, flags);
1017 if (ret >= 0)
1018 file_pos_write(f.file, pos);
1019 fdput_pos(f);
1020 }
1021
1022 if (ret > 0)
1023 add_rchar(current, ret);
1024 inc_syscr(current);
1025 return ret;
1026}
1027
1028static ssize_t do_writev(unsigned long fd, const struct iovec __user *vec,
1029 unsigned long vlen, rwf_t flags)
1030{
1031 struct fd f = fdget_pos(fd);
1032 ssize_t ret = -EBADF;
1033
1034 if (f.file) {
1035 loff_t pos = file_pos_read(f.file);
1036 ret = vfs_writev(f.file, vec, vlen, &pos, flags);
1037 if (ret >= 0)
1038 file_pos_write(f.file, pos);
1039 fdput_pos(f);
1040 }
1041
1042 if (ret > 0)
1043 add_wchar(current, ret);
1044 inc_syscw(current);
1045 return ret;
1046}
1047
1048static inline loff_t pos_from_hilo(unsigned long high, unsigned long low)
1049{
1050#define HALF_LONG_BITS (BITS_PER_LONG / 2)
1051 return (((loff_t)high << HALF_LONG_BITS) << HALF_LONG_BITS) | low;
1052}
1053
1054static ssize_t do_preadv(unsigned long fd, const struct iovec __user *vec,
1055 unsigned long vlen, loff_t pos, rwf_t flags)
1056{
1057 struct fd f;
1058 ssize_t ret = -EBADF;
1059
1060 if (pos < 0)
1061 return -EINVAL;
1062
1063 f = fdget(fd);
1064 if (f.file) {
1065 ret = -ESPIPE;
1066 if (f.file->f_mode & FMODE_PREAD)
1067 ret = vfs_readv(f.file, vec, vlen, &pos, flags);
1068 fdput(f);
1069 }
1070
1071 if (ret > 0)
1072 add_rchar(current, ret);
1073 inc_syscr(current);
1074 return ret;
1075}
1076
1077static ssize_t do_pwritev(unsigned long fd, const struct iovec __user *vec,
1078 unsigned long vlen, loff_t pos, rwf_t flags)
1079{
1080 struct fd f;
1081 ssize_t ret = -EBADF;
1082
1083 if (pos < 0)
1084 return -EINVAL;
1085
1086 f = fdget(fd);
1087 if (f.file) {
1088 ret = -ESPIPE;
1089 if (f.file->f_mode & FMODE_PWRITE)
1090 ret = vfs_writev(f.file, vec, vlen, &pos, flags);
1091 fdput(f);
1092 }
1093
1094 if (ret > 0)
1095 add_wchar(current, ret);
1096 inc_syscw(current);
1097 return ret;
1098}
1099
1100SYSCALL_DEFINE3(readv, unsigned long, fd, const struct iovec __user *, vec,
1101 unsigned long, vlen)
1102{
1103 return do_readv(fd, vec, vlen, 0);
1104}
1105
1106SYSCALL_DEFINE3(writev, unsigned long, fd, const struct iovec __user *, vec,
1107 unsigned long, vlen)
1108{
1109 return do_writev(fd, vec, vlen, 0);
1110}
1111
1112SYSCALL_DEFINE5(preadv, unsigned long, fd, const struct iovec __user *, vec,
1113 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
1114{
1115 loff_t pos = pos_from_hilo(pos_h, pos_l);
1116
1117 return do_preadv(fd, vec, vlen, pos, 0);
1118}
1119
1120SYSCALL_DEFINE6(preadv2, unsigned long, fd, const struct iovec __user *, vec,
1121 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h,
1122 rwf_t, flags)
1123{
1124 loff_t pos = pos_from_hilo(pos_h, pos_l);
1125
1126 if (pos == -1)
1127 return do_readv(fd, vec, vlen, flags);
1128
1129 return do_preadv(fd, vec, vlen, pos, flags);
1130}
1131
1132SYSCALL_DEFINE5(pwritev, unsigned long, fd, const struct iovec __user *, vec,
1133 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
1134{
1135 loff_t pos = pos_from_hilo(pos_h, pos_l);
1136
1137 return do_pwritev(fd, vec, vlen, pos, 0);
1138}
1139
1140SYSCALL_DEFINE6(pwritev2, unsigned long, fd, const struct iovec __user *, vec,
1141 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h,
1142 rwf_t, flags)
1143{
1144 loff_t pos = pos_from_hilo(pos_h, pos_l);
1145
1146 if (pos == -1)
1147 return do_writev(fd, vec, vlen, flags);
1148
1149 return do_pwritev(fd, vec, vlen, pos, flags);
1150}
1151
1152#ifdef CONFIG_COMPAT
1153static size_t compat_readv(struct file *file,
1154 const struct compat_iovec __user *vec,
1155 unsigned long vlen, loff_t *pos, rwf_t flags)
1156{
1157 struct iovec iovstack[UIO_FASTIOV];
1158 struct iovec *iov = iovstack;
1159 struct iov_iter iter;
1160 ssize_t ret;
1161
1162 ret = compat_import_iovec(READ, vec, vlen, UIO_FASTIOV, &iov, &iter);
1163 if (ret >= 0) {
1164 ret = do_iter_read(file, &iter, pos, flags);
1165 kfree(iov);
1166 }
1167 if (ret > 0)
1168 add_rchar(current, ret);
1169 inc_syscr(current);
1170 return ret;
1171}
1172
1173static size_t do_compat_readv(compat_ulong_t fd,
1174 const struct compat_iovec __user *vec,
1175 compat_ulong_t vlen, rwf_t flags)
1176{
1177 struct fd f = fdget_pos(fd);
1178 ssize_t ret;
1179 loff_t pos;
1180
1181 if (!f.file)
1182 return -EBADF;
1183 pos = f.file->f_pos;
1184 ret = compat_readv(f.file, vec, vlen, &pos, flags);
1185 if (ret >= 0)
1186 f.file->f_pos = pos;
1187 fdput_pos(f);
1188 return ret;
1189
1190}
1191
1192COMPAT_SYSCALL_DEFINE3(readv, compat_ulong_t, fd,
1193 const struct compat_iovec __user *,vec,
1194 compat_ulong_t, vlen)
1195{
1196 return do_compat_readv(fd, vec, vlen, 0);
1197}
1198
1199static long do_compat_preadv64(unsigned long fd,
1200 const struct compat_iovec __user *vec,
1201 unsigned long vlen, loff_t pos, rwf_t flags)
1202{
1203 struct fd f;
1204 ssize_t ret;
1205
1206 if (pos < 0)
1207 return -EINVAL;
1208 f = fdget(fd);
1209 if (!f.file)
1210 return -EBADF;
1211 ret = -ESPIPE;
1212 if (f.file->f_mode & FMODE_PREAD)
1213 ret = compat_readv(f.file, vec, vlen, &pos, flags);
1214 fdput(f);
1215 return ret;
1216}
1217
1218#ifdef __ARCH_WANT_COMPAT_SYS_PREADV64
1219COMPAT_SYSCALL_DEFINE4(preadv64, unsigned long, fd,
1220 const struct compat_iovec __user *,vec,
1221 unsigned long, vlen, loff_t, pos)
1222{
1223 return do_compat_preadv64(fd, vec, vlen, pos, 0);
1224}
1225#endif
1226
1227COMPAT_SYSCALL_DEFINE5(preadv, compat_ulong_t, fd,
1228 const struct compat_iovec __user *,vec,
1229 compat_ulong_t, vlen, u32, pos_low, u32, pos_high)
1230{
1231 loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1232
1233 return do_compat_preadv64(fd, vec, vlen, pos, 0);
1234}
1235
1236#ifdef __ARCH_WANT_COMPAT_SYS_PREADV64V2
1237COMPAT_SYSCALL_DEFINE5(preadv64v2, unsigned long, fd,
1238 const struct compat_iovec __user *,vec,
1239 unsigned long, vlen, loff_t, pos, rwf_t, flags)
1240{
1241 if (pos == -1)
1242 return do_compat_readv(fd, vec, vlen, flags);
1243
1244 return do_compat_preadv64(fd, vec, vlen, pos, flags);
1245}
1246#endif
1247
1248COMPAT_SYSCALL_DEFINE6(preadv2, compat_ulong_t, fd,
1249 const struct compat_iovec __user *,vec,
1250 compat_ulong_t, vlen, u32, pos_low, u32, pos_high,
1251 rwf_t, flags)
1252{
1253 loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1254
1255 if (pos == -1)
1256 return do_compat_readv(fd, vec, vlen, flags);
1257
1258 return do_compat_preadv64(fd, vec, vlen, pos, flags);
1259}
1260
1261static size_t compat_writev(struct file *file,
1262 const struct compat_iovec __user *vec,
1263 unsigned long vlen, loff_t *pos, rwf_t flags)
1264{
1265 struct iovec iovstack[UIO_FASTIOV];
1266 struct iovec *iov = iovstack;
1267 struct iov_iter iter;
1268 ssize_t ret;
1269
1270 ret = compat_import_iovec(WRITE, vec, vlen, UIO_FASTIOV, &iov, &iter);
1271 if (ret >= 0) {
1272 file_start_write(file);
1273 ret = do_iter_write(file, &iter, pos, flags);
1274 file_end_write(file);
1275 kfree(iov);
1276 }
1277 if (ret > 0)
1278 add_wchar(current, ret);
1279 inc_syscw(current);
1280 return ret;
1281}
1282
1283static size_t do_compat_writev(compat_ulong_t fd,
1284 const struct compat_iovec __user* vec,
1285 compat_ulong_t vlen, rwf_t flags)
1286{
1287 struct fd f = fdget_pos(fd);
1288 ssize_t ret;
1289 loff_t pos;
1290
1291 if (!f.file)
1292 return -EBADF;
1293 pos = f.file->f_pos;
1294 ret = compat_writev(f.file, vec, vlen, &pos, flags);
1295 if (ret >= 0)
1296 f.file->f_pos = pos;
1297 fdput_pos(f);
1298 return ret;
1299}
1300
1301COMPAT_SYSCALL_DEFINE3(writev, compat_ulong_t, fd,
1302 const struct compat_iovec __user *, vec,
1303 compat_ulong_t, vlen)
1304{
1305 return do_compat_writev(fd, vec, vlen, 0);
1306}
1307
1308static long do_compat_pwritev64(unsigned long fd,
1309 const struct compat_iovec __user *vec,
1310 unsigned long vlen, loff_t pos, rwf_t flags)
1311{
1312 struct fd f;
1313 ssize_t ret;
1314
1315 if (pos < 0)
1316 return -EINVAL;
1317 f = fdget(fd);
1318 if (!f.file)
1319 return -EBADF;
1320 ret = -ESPIPE;
1321 if (f.file->f_mode & FMODE_PWRITE)
1322 ret = compat_writev(f.file, vec, vlen, &pos, flags);
1323 fdput(f);
1324 return ret;
1325}
1326
1327#ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64
1328COMPAT_SYSCALL_DEFINE4(pwritev64, unsigned long, fd,
1329 const struct compat_iovec __user *,vec,
1330 unsigned long, vlen, loff_t, pos)
1331{
1332 return do_compat_pwritev64(fd, vec, vlen, pos, 0);
1333}
1334#endif
1335
1336COMPAT_SYSCALL_DEFINE5(pwritev, compat_ulong_t, fd,
1337 const struct compat_iovec __user *,vec,
1338 compat_ulong_t, vlen, u32, pos_low, u32, pos_high)
1339{
1340 loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1341
1342 return do_compat_pwritev64(fd, vec, vlen, pos, 0);
1343}
1344
1345#ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64V2
1346COMPAT_SYSCALL_DEFINE5(pwritev64v2, unsigned long, fd,
1347 const struct compat_iovec __user *,vec,
1348 unsigned long, vlen, loff_t, pos, rwf_t, flags)
1349{
1350 if (pos == -1)
1351 return do_compat_writev(fd, vec, vlen, flags);
1352
1353 return do_compat_pwritev64(fd, vec, vlen, pos, flags);
1354}
1355#endif
1356
1357COMPAT_SYSCALL_DEFINE6(pwritev2, compat_ulong_t, fd,
1358 const struct compat_iovec __user *,vec,
1359 compat_ulong_t, vlen, u32, pos_low, u32, pos_high, rwf_t, flags)
1360{
1361 loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1362
1363 if (pos == -1)
1364 return do_compat_writev(fd, vec, vlen, flags);
1365
1366 return do_compat_pwritev64(fd, vec, vlen, pos, flags);
1367}
1368
1369#endif
1370
1371static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
1372 size_t count, loff_t max)
1373{
1374 struct fd in, out;
1375 struct inode *in_inode, *out_inode;
1376 loff_t pos;
1377 loff_t out_pos;
1378 ssize_t retval;
1379 int fl;
1380
1381 /*
1382 * Get input file, and verify that it is ok..
1383 */
1384 retval = -EBADF;
1385 in = fdget(in_fd);
1386 if (!in.file)
1387 goto out;
1388 if (!(in.file->f_mode & FMODE_READ))
1389 goto fput_in;
1390 retval = -ESPIPE;
1391 if (!ppos) {
1392 pos = in.file->f_pos;
1393 } else {
1394 pos = *ppos;
1395 if (!(in.file->f_mode & FMODE_PREAD))
1396 goto fput_in;
1397 }
1398 retval = rw_verify_area(READ, in.file, &pos, count);
1399 if (retval < 0)
1400 goto fput_in;
1401 if (count > MAX_RW_COUNT)
1402 count = MAX_RW_COUNT;
1403
1404 /*
1405 * Get output file, and verify that it is ok..
1406 */
1407 retval = -EBADF;
1408 out = fdget(out_fd);
1409 if (!out.file)
1410 goto fput_in;
1411 if (!(out.file->f_mode & FMODE_WRITE))
1412 goto fput_out;
1413 in_inode = file_inode(in.file);
1414 out_inode = file_inode(out.file);
1415 out_pos = out.file->f_pos;
1416 retval = rw_verify_area(WRITE, out.file, &out_pos, count);
1417 if (retval < 0)
1418 goto fput_out;
1419
1420 if (!max)
1421 max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes);
1422
1423 if (unlikely(pos + count > max)) {
1424 retval = -EOVERFLOW;
1425 if (pos >= max)
1426 goto fput_out;
1427 count = max - pos;
1428 }
1429
1430 fl = 0;
1431#if 0
1432 /*
1433 * We need to debate whether we can enable this or not. The
1434 * man page documents EAGAIN return for the output at least,
1435 * and the application is arguably buggy if it doesn't expect
1436 * EAGAIN on a non-blocking file descriptor.
1437 */
1438 if (in.file->f_flags & O_NONBLOCK)
1439 fl = SPLICE_F_NONBLOCK;
1440#endif
1441 file_start_write(out.file);
1442 retval = do_splice_direct(in.file, &pos, out.file, &out_pos, count, fl);
1443 file_end_write(out.file);
1444
1445 if (retval > 0) {
1446 add_rchar(current, retval);
1447 add_wchar(current, retval);
1448 fsnotify_access(in.file);
1449 fsnotify_modify(out.file);
1450 out.file->f_pos = out_pos;
1451 if (ppos)
1452 *ppos = pos;
1453 else
1454 in.file->f_pos = pos;
1455 }
1456
1457 inc_syscr(current);
1458 inc_syscw(current);
1459 if (pos > max)
1460 retval = -EOVERFLOW;
1461
1462fput_out:
1463 fdput(out);
1464fput_in:
1465 fdput(in);
1466out:
1467 return retval;
1468}
1469
1470SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd, off_t __user *, offset, size_t, count)
1471{
1472 loff_t pos;
1473 off_t off;
1474 ssize_t ret;
1475
1476 if (offset) {
1477 if (unlikely(get_user(off, offset)))
1478 return -EFAULT;
1479 pos = off;
1480 ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
1481 if (unlikely(put_user(pos, offset)))
1482 return -EFAULT;
1483 return ret;
1484 }
1485
1486 return do_sendfile(out_fd, in_fd, NULL, count, 0);
1487}
1488
1489SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd, loff_t __user *, offset, size_t, count)
1490{
1491 loff_t pos;
1492 ssize_t ret;
1493
1494 if (offset) {
1495 if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
1496 return -EFAULT;
1497 ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
1498 if (unlikely(put_user(pos, offset)))
1499 return -EFAULT;
1500 return ret;
1501 }
1502
1503 return do_sendfile(out_fd, in_fd, NULL, count, 0);
1504}
1505
1506#ifdef CONFIG_COMPAT
1507COMPAT_SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd,
1508 compat_off_t __user *, offset, compat_size_t, count)
1509{
1510 loff_t pos;
1511 off_t off;
1512 ssize_t ret;
1513
1514 if (offset) {
1515 if (unlikely(get_user(off, offset)))
1516 return -EFAULT;
1517 pos = off;
1518 ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
1519 if (unlikely(put_user(pos, offset)))
1520 return -EFAULT;
1521 return ret;
1522 }
1523
1524 return do_sendfile(out_fd, in_fd, NULL, count, 0);
1525}
1526
1527COMPAT_SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd,
1528 compat_loff_t __user *, offset, compat_size_t, count)
1529{
1530 loff_t pos;
1531 ssize_t ret;
1532
1533 if (offset) {
1534 if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
1535 return -EFAULT;
1536 ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
1537 if (unlikely(put_user(pos, offset)))
1538 return -EFAULT;
1539 return ret;
1540 }
1541
1542 return do_sendfile(out_fd, in_fd, NULL, count, 0);
1543}
1544#endif
1545
1546/*
1547 * copy_file_range() differs from regular file read and write in that it
1548 * specifically allows return partial success. When it does so is up to
1549 * the copy_file_range method.
1550 */
1551ssize_t vfs_copy_file_range(struct file *file_in, loff_t pos_in,
1552 struct file *file_out, loff_t pos_out,
1553 size_t len, unsigned int flags)
1554{
1555 struct inode *inode_in = file_inode(file_in);
1556 struct inode *inode_out = file_inode(file_out);
1557 ssize_t ret;
1558
1559 if (flags != 0)
1560 return -EINVAL;
1561
1562 if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode))
1563 return -EISDIR;
1564 if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode))
1565 return -EINVAL;
1566
1567 ret = rw_verify_area(READ, file_in, &pos_in, len);
1568 if (unlikely(ret))
1569 return ret;
1570
1571 ret = rw_verify_area(WRITE, file_out, &pos_out, len);
1572 if (unlikely(ret))
1573 return ret;
1574
1575 if (!(file_in->f_mode & FMODE_READ) ||
1576 !(file_out->f_mode & FMODE_WRITE) ||
1577 (file_out->f_flags & O_APPEND))
1578 return -EBADF;
1579
1580 /* this could be relaxed once a method supports cross-fs copies */
1581 if (inode_in->i_sb != inode_out->i_sb)
1582 return -EXDEV;
1583
1584 if (len == 0)
1585 return 0;
1586
1587 file_start_write(file_out);
1588
1589 /*
1590 * Try cloning first, this is supported by more file systems, and
1591 * more efficient if both clone and copy are supported (e.g. NFS).
1592 */
1593 if (file_in->f_op->remap_file_range) {
1594 loff_t cloned;
1595
1596 cloned = file_in->f_op->remap_file_range(file_in, pos_in,
1597 file_out, pos_out,
1598 min_t(loff_t, MAX_RW_COUNT, len),
1599 REMAP_FILE_CAN_SHORTEN);
1600 if (cloned > 0) {
1601 ret = cloned;
1602 goto done;
1603 }
1604 }
1605
1606 if (file_out->f_op->copy_file_range) {
1607 ret = file_out->f_op->copy_file_range(file_in, pos_in, file_out,
1608 pos_out, len, flags);
1609 if (ret != -EOPNOTSUPP)
1610 goto done;
1611 }
1612
1613 ret = do_splice_direct(file_in, &pos_in, file_out, &pos_out,
1614 len > MAX_RW_COUNT ? MAX_RW_COUNT : len, 0);
1615
1616done:
1617 if (ret > 0) {
1618 fsnotify_access(file_in);
1619 add_rchar(current, ret);
1620 fsnotify_modify(file_out);
1621 add_wchar(current, ret);
1622 }
1623
1624 inc_syscr(current);
1625 inc_syscw(current);
1626
1627 file_end_write(file_out);
1628
1629 return ret;
1630}
1631EXPORT_SYMBOL(vfs_copy_file_range);
1632
1633SYSCALL_DEFINE6(copy_file_range, int, fd_in, loff_t __user *, off_in,
1634 int, fd_out, loff_t __user *, off_out,
1635 size_t, len, unsigned int, flags)
1636{
1637 loff_t pos_in;
1638 loff_t pos_out;
1639 struct fd f_in;
1640 struct fd f_out;
1641 ssize_t ret = -EBADF;
1642
1643 f_in = fdget(fd_in);
1644 if (!f_in.file)
1645 goto out2;
1646
1647 f_out = fdget(fd_out);
1648 if (!f_out.file)
1649 goto out1;
1650
1651 ret = -EFAULT;
1652 if (off_in) {
1653 if (copy_from_user(&pos_in, off_in, sizeof(loff_t)))
1654 goto out;
1655 } else {
1656 pos_in = f_in.file->f_pos;
1657 }
1658
1659 if (off_out) {
1660 if (copy_from_user(&pos_out, off_out, sizeof(loff_t)))
1661 goto out;
1662 } else {
1663 pos_out = f_out.file->f_pos;
1664 }
1665
1666 ret = vfs_copy_file_range(f_in.file, pos_in, f_out.file, pos_out, len,
1667 flags);
1668 if (ret > 0) {
1669 pos_in += ret;
1670 pos_out += ret;
1671
1672 if (off_in) {
1673 if (copy_to_user(off_in, &pos_in, sizeof(loff_t)))
1674 ret = -EFAULT;
1675 } else {
1676 f_in.file->f_pos = pos_in;
1677 }
1678
1679 if (off_out) {
1680 if (copy_to_user(off_out, &pos_out, sizeof(loff_t)))
1681 ret = -EFAULT;
1682 } else {
1683 f_out.file->f_pos = pos_out;
1684 }
1685 }
1686
1687out:
1688 fdput(f_out);
1689out1:
1690 fdput(f_in);
1691out2:
1692 return ret;
1693}
1694
1695static int remap_verify_area(struct file *file, loff_t pos, loff_t len,
1696 bool write)
1697{
1698 struct inode *inode = file_inode(file);
1699
1700 if (unlikely(pos < 0 || len < 0))
1701 return -EINVAL;
1702
1703 if (unlikely((loff_t) (pos + len) < 0))
1704 return -EINVAL;
1705
1706 if (unlikely(inode->i_flctx && mandatory_lock(inode))) {
1707 loff_t end = len ? pos + len - 1 : OFFSET_MAX;
1708 int retval;
1709
1710 retval = locks_mandatory_area(inode, file, pos, end,
1711 write ? F_WRLCK : F_RDLCK);
1712 if (retval < 0)
1713 return retval;
1714 }
1715
1716 return security_file_permission(file, write ? MAY_WRITE : MAY_READ);
1717}
1718/*
1719 * Ensure that we don't remap a partial EOF block in the middle of something
1720 * else. Assume that the offsets have already been checked for block
1721 * alignment.
1722 *
1723 * For deduplication we always scale down to the previous block because we
1724 * can't meaningfully compare post-EOF contents.
1725 *
1726 * For clone we only link a partial EOF block above the destination file's EOF.
1727 *
1728 * Shorten the request if possible.
1729 */
1730static int generic_remap_check_len(struct inode *inode_in,
1731 struct inode *inode_out,
1732 loff_t pos_out,
1733 loff_t *len,
1734 unsigned int remap_flags)
1735{
1736 u64 blkmask = i_blocksize(inode_in) - 1;
1737 loff_t new_len = *len;
1738
1739 if ((*len & blkmask) == 0)
1740 return 0;
1741
1742 if ((remap_flags & REMAP_FILE_DEDUP) ||
1743 pos_out + *len < i_size_read(inode_out))
1744 new_len &= ~blkmask;
1745
1746 if (new_len == *len)
1747 return 0;
1748
1749 if (remap_flags & REMAP_FILE_CAN_SHORTEN) {
1750 *len = new_len;
1751 return 0;
1752 }
1753
1754 return (remap_flags & REMAP_FILE_DEDUP) ? -EBADE : -EINVAL;
1755}
1756
1757/*
1758 * Read a page's worth of file data into the page cache. Return the page
1759 * locked.
1760 */
1761static struct page *vfs_dedupe_get_page(struct inode *inode, loff_t offset)
1762{
1763 struct page *page;
1764
1765 page = read_mapping_page(inode->i_mapping, offset >> PAGE_SHIFT, NULL);
1766 if (IS_ERR(page))
1767 return page;
1768 if (!PageUptodate(page)) {
1769 put_page(page);
1770 return ERR_PTR(-EIO);
1771 }
1772 lock_page(page);
1773 return page;
1774}
1775
1776/*
1777 * Compare extents of two files to see if they are the same.
1778 * Caller must have locked both inodes to prevent write races.
1779 */
1780static int vfs_dedupe_file_range_compare(struct inode *src, loff_t srcoff,
1781 struct inode *dest, loff_t destoff,
1782 loff_t len, bool *is_same)
1783{
1784 loff_t src_poff;
1785 loff_t dest_poff;
1786 void *src_addr;
1787 void *dest_addr;
1788 struct page *src_page;
1789 struct page *dest_page;
1790 loff_t cmp_len;
1791 bool same;
1792 int error;
1793
1794 error = -EINVAL;
1795 same = true;
1796 while (len) {
1797 src_poff = srcoff & (PAGE_SIZE - 1);
1798 dest_poff = destoff & (PAGE_SIZE - 1);
1799 cmp_len = min(PAGE_SIZE - src_poff,
1800 PAGE_SIZE - dest_poff);
1801 cmp_len = min(cmp_len, len);
1802 if (cmp_len <= 0)
1803 goto out_error;
1804
1805 src_page = vfs_dedupe_get_page(src, srcoff);
1806 if (IS_ERR(src_page)) {
1807 error = PTR_ERR(src_page);
1808 goto out_error;
1809 }
1810 dest_page = vfs_dedupe_get_page(dest, destoff);
1811 if (IS_ERR(dest_page)) {
1812 error = PTR_ERR(dest_page);
1813 unlock_page(src_page);
1814 put_page(src_page);
1815 goto out_error;
1816 }
1817 src_addr = kmap_atomic(src_page);
1818 dest_addr = kmap_atomic(dest_page);
1819
1820 flush_dcache_page(src_page);
1821 flush_dcache_page(dest_page);
1822
1823 if (memcmp(src_addr + src_poff, dest_addr + dest_poff, cmp_len))
1824 same = false;
1825
1826 kunmap_atomic(dest_addr);
1827 kunmap_atomic(src_addr);
1828 unlock_page(dest_page);
1829 unlock_page(src_page);
1830 put_page(dest_page);
1831 put_page(src_page);
1832
1833 if (!same)
1834 break;
1835
1836 srcoff += cmp_len;
1837 destoff += cmp_len;
1838 len -= cmp_len;
1839 }
1840
1841 *is_same = same;
1842 return 0;
1843
1844out_error:
1845 return error;
1846}
1847
1848/*
1849 * Check that the two inodes are eligible for cloning, the ranges make
1850 * sense, and then flush all dirty data. Caller must ensure that the
1851 * inodes have been locked against any other modifications.
1852 *
1853 * If there's an error, then the usual negative error code is returned.
1854 * Otherwise returns 0 with *len set to the request length.
1855 */
1856int generic_remap_file_range_prep(struct file *file_in, loff_t pos_in,
1857 struct file *file_out, loff_t pos_out,
1858 loff_t *len, unsigned int remap_flags)
1859{
1860 struct inode *inode_in = file_inode(file_in);
1861 struct inode *inode_out = file_inode(file_out);
1862 bool same_inode = (inode_in == inode_out);
1863 int ret;
1864
1865 /* Don't touch certain kinds of inodes */
1866 if (IS_IMMUTABLE(inode_out))
1867 return -EPERM;
1868
1869 if (IS_SWAPFILE(inode_in) || IS_SWAPFILE(inode_out))
1870 return -ETXTBSY;
1871
1872 /* Don't reflink dirs, pipes, sockets... */
1873 if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode))
1874 return -EISDIR;
1875 if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode))
1876 return -EINVAL;
1877
1878 /* Zero length dedupe exits immediately; reflink goes to EOF. */
1879 if (*len == 0) {
1880 loff_t isize = i_size_read(inode_in);
1881
1882 if ((remap_flags & REMAP_FILE_DEDUP) || pos_in == isize)
1883 return 0;
1884 if (pos_in > isize)
1885 return -EINVAL;
1886 *len = isize - pos_in;
1887 if (*len == 0)
1888 return 0;
1889 }
1890
1891 /* Check that we don't violate system file offset limits. */
1892 ret = generic_remap_checks(file_in, pos_in, file_out, pos_out, len,
1893 remap_flags);
1894 if (ret)
1895 return ret;
1896
1897 /* Wait for the completion of any pending IOs on both files */
1898 inode_dio_wait(inode_in);
1899 if (!same_inode)
1900 inode_dio_wait(inode_out);
1901
1902 ret = filemap_write_and_wait_range(inode_in->i_mapping,
1903 pos_in, pos_in + *len - 1);
1904 if (ret)
1905 return ret;
1906
1907 ret = filemap_write_and_wait_range(inode_out->i_mapping,
1908 pos_out, pos_out + *len - 1);
1909 if (ret)
1910 return ret;
1911
1912 /*
1913 * Check that the extents are the same.
1914 */
1915 if (remap_flags & REMAP_FILE_DEDUP) {
1916 bool is_same = false;
1917
1918 ret = vfs_dedupe_file_range_compare(inode_in, pos_in,
1919 inode_out, pos_out, *len, &is_same);
1920 if (ret)
1921 return ret;
1922 if (!is_same)
1923 return -EBADE;
1924 }
1925
1926 ret = generic_remap_check_len(inode_in, inode_out, pos_out, len,
1927 remap_flags);
1928 if (ret)
1929 return ret;
1930
1931 /* If can't alter the file contents, we're done. */
1932 if (!(remap_flags & REMAP_FILE_DEDUP)) {
1933 /* Update the timestamps, since we can alter file contents. */
1934 if (!(file_out->f_mode & FMODE_NOCMTIME)) {
1935 ret = file_update_time(file_out);
1936 if (ret)
1937 return ret;
1938 }
1939
1940 /*
1941 * Clear the security bits if the process is not being run by
1942 * root. This keeps people from modifying setuid and setgid
1943 * binaries.
1944 */
1945 ret = file_remove_privs(file_out);
1946 if (ret)
1947 return ret;
1948 }
1949
1950 return 0;
1951}
1952EXPORT_SYMBOL(generic_remap_file_range_prep);
1953
1954loff_t do_clone_file_range(struct file *file_in, loff_t pos_in,
1955 struct file *file_out, loff_t pos_out,
1956 loff_t len, unsigned int remap_flags)
1957{
1958 struct inode *inode_in = file_inode(file_in);
1959 struct inode *inode_out = file_inode(file_out);
1960 loff_t ret;
1961
1962 WARN_ON_ONCE(remap_flags & REMAP_FILE_DEDUP);
1963
1964 if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode))
1965 return -EISDIR;
1966 if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode))
1967 return -EINVAL;
1968
1969 /*
1970 * FICLONE/FICLONERANGE ioctls enforce that src and dest files are on
1971 * the same mount. Practically, they only need to be on the same file
1972 * system.
1973 */
1974 if (inode_in->i_sb != inode_out->i_sb)
1975 return -EXDEV;
1976
1977 if (!(file_in->f_mode & FMODE_READ) ||
1978 !(file_out->f_mode & FMODE_WRITE) ||
1979 (file_out->f_flags & O_APPEND))
1980 return -EBADF;
1981
1982 if (!file_in->f_op->remap_file_range)
1983 return -EOPNOTSUPP;
1984
1985 ret = remap_verify_area(file_in, pos_in, len, false);
1986 if (ret)
1987 return ret;
1988
1989 ret = remap_verify_area(file_out, pos_out, len, true);
1990 if (ret)
1991 return ret;
1992
1993 ret = file_in->f_op->remap_file_range(file_in, pos_in,
1994 file_out, pos_out, len, remap_flags);
1995 if (ret < 0)
1996 return ret;
1997
1998 fsnotify_access(file_in);
1999 fsnotify_modify(file_out);
2000 return ret;
2001}
2002EXPORT_SYMBOL(do_clone_file_range);
2003
2004loff_t vfs_clone_file_range(struct file *file_in, loff_t pos_in,
2005 struct file *file_out, loff_t pos_out,
2006 loff_t len, unsigned int remap_flags)
2007{
2008 loff_t ret;
2009
2010 file_start_write(file_out);
2011 ret = do_clone_file_range(file_in, pos_in, file_out, pos_out, len,
2012 remap_flags);
2013 file_end_write(file_out);
2014
2015 return ret;
2016}
2017EXPORT_SYMBOL(vfs_clone_file_range);
2018
2019/* Check whether we are allowed to dedupe the destination file */
2020static bool allow_file_dedupe(struct file *file)
2021{
2022 if (capable(CAP_SYS_ADMIN))
2023 return true;
2024 if (file->f_mode & FMODE_WRITE)
2025 return true;
2026 if (uid_eq(current_fsuid(), file_inode(file)->i_uid))
2027 return true;
2028 if (!inode_permission(file_inode(file), MAY_WRITE))
2029 return true;
2030 return false;
2031}
2032
2033loff_t vfs_dedupe_file_range_one(struct file *src_file, loff_t src_pos,
2034 struct file *dst_file, loff_t dst_pos,
2035 loff_t len, unsigned int remap_flags)
2036{
2037 loff_t ret;
2038
2039 WARN_ON_ONCE(remap_flags & ~(REMAP_FILE_DEDUP |
2040 REMAP_FILE_CAN_SHORTEN));
2041
2042 ret = mnt_want_write_file(dst_file);
2043 if (ret)
2044 return ret;
2045
2046 ret = remap_verify_area(dst_file, dst_pos, len, true);
2047 if (ret < 0)
2048 goto out_drop_write;
2049
2050 ret = -EPERM;
2051 if (!allow_file_dedupe(dst_file))
2052 goto out_drop_write;
2053
2054 ret = -EXDEV;
2055 if (src_file->f_path.mnt != dst_file->f_path.mnt)
2056 goto out_drop_write;
2057
2058 ret = -EISDIR;
2059 if (S_ISDIR(file_inode(dst_file)->i_mode))
2060 goto out_drop_write;
2061
2062 ret = -EINVAL;
2063 if (!dst_file->f_op->remap_file_range)
2064 goto out_drop_write;
2065
2066 if (len == 0) {
2067 ret = 0;
2068 goto out_drop_write;
2069 }
2070
2071 ret = dst_file->f_op->remap_file_range(src_file, src_pos, dst_file,
2072 dst_pos, len, remap_flags | REMAP_FILE_DEDUP);
2073out_drop_write:
2074 mnt_drop_write_file(dst_file);
2075
2076 return ret;
2077}
2078EXPORT_SYMBOL(vfs_dedupe_file_range_one);
2079
2080int vfs_dedupe_file_range(struct file *file, struct file_dedupe_range *same)
2081{
2082 struct file_dedupe_range_info *info;
2083 struct inode *src = file_inode(file);
2084 u64 off;
2085 u64 len;
2086 int i;
2087 int ret;
2088 u16 count = same->dest_count;
2089 loff_t deduped;
2090
2091 if (!(file->f_mode & FMODE_READ))
2092 return -EINVAL;
2093
2094 if (same->reserved1 || same->reserved2)
2095 return -EINVAL;
2096
2097 off = same->src_offset;
2098 len = same->src_length;
2099
2100 if (S_ISDIR(src->i_mode))
2101 return -EISDIR;
2102
2103 if (!S_ISREG(src->i_mode))
2104 return -EINVAL;
2105
2106 if (!file->f_op->remap_file_range)
2107 return -EOPNOTSUPP;
2108
2109 ret = remap_verify_area(file, off, len, false);
2110 if (ret < 0)
2111 return ret;
2112 ret = 0;
2113
2114 if (off + len > i_size_read(src))
2115 return -EINVAL;
2116
2117 /* Arbitrary 1G limit on a single dedupe request, can be raised. */
2118 len = min_t(u64, len, 1 << 30);
2119
2120 /* pre-format output fields to sane values */
2121 for (i = 0; i < count; i++) {
2122 same->info[i].bytes_deduped = 0ULL;
2123 same->info[i].status = FILE_DEDUPE_RANGE_SAME;
2124 }
2125
2126 for (i = 0, info = same->info; i < count; i++, info++) {
2127 struct fd dst_fd = fdget(info->dest_fd);
2128 struct file *dst_file = dst_fd.file;
2129
2130 if (!dst_file) {
2131 info->status = -EBADF;
2132 goto next_loop;
2133 }
2134
2135 if (info->reserved) {
2136 info->status = -EINVAL;
2137 goto next_fdput;
2138 }
2139
2140 deduped = vfs_dedupe_file_range_one(file, off, dst_file,
2141 info->dest_offset, len,
2142 REMAP_FILE_CAN_SHORTEN);
2143 if (deduped == -EBADE)
2144 info->status = FILE_DEDUPE_RANGE_DIFFERS;
2145 else if (deduped < 0)
2146 info->status = deduped;
2147 else
2148 info->bytes_deduped = len;
2149
2150next_fdput:
2151 fdput(dst_fd);
2152next_loop:
2153 if (fatal_signal_pending(current))
2154 break;
2155 }
2156 return ret;
2157}
2158EXPORT_SYMBOL(vfs_dedupe_file_range);
2159