1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * I/O and data path helper functionality. |
4 | * |
5 | * Borrowed from NFS Copyright (c) 2016 Trond Myklebust |
6 | */ |
7 | |
8 | #include <linux/kernel.h> |
9 | #include <linux/netfs.h> |
10 | #include "internal.h" |
11 | |
12 | /* |
13 | * inode_dio_wait_interruptible - wait for outstanding DIO requests to finish |
14 | * @inode: inode to wait for |
15 | * |
16 | * Waits for all pending direct I/O requests to finish so that we can |
17 | * proceed with a truncate or equivalent operation. |
18 | * |
19 | * Must be called under a lock that serializes taking new references |
20 | * to i_dio_count, usually by inode->i_mutex. |
21 | */ |
22 | static int inode_dio_wait_interruptible(struct inode *inode) |
23 | { |
24 | if (!atomic_read(v: &inode->i_dio_count)) |
25 | return 0; |
26 | |
27 | wait_queue_head_t *wq = bit_waitqueue(word: &inode->i_state, __I_DIO_WAKEUP); |
28 | DEFINE_WAIT_BIT(q, &inode->i_state, __I_DIO_WAKEUP); |
29 | |
30 | for (;;) { |
31 | prepare_to_wait(wq_head: wq, wq_entry: &q.wq_entry, TASK_INTERRUPTIBLE); |
32 | if (!atomic_read(v: &inode->i_dio_count)) |
33 | break; |
34 | if (signal_pending(current)) |
35 | break; |
36 | schedule(); |
37 | } |
38 | finish_wait(wq_head: wq, wq_entry: &q.wq_entry); |
39 | |
40 | return atomic_read(v: &inode->i_dio_count) ? -ERESTARTSYS : 0; |
41 | } |
42 | |
43 | /* Call with exclusively locked inode->i_rwsem */ |
44 | static int netfs_block_o_direct(struct netfs_inode *ictx) |
45 | { |
46 | if (!test_bit(NETFS_ICTX_ODIRECT, &ictx->flags)) |
47 | return 0; |
48 | clear_bit(NETFS_ICTX_ODIRECT, addr: &ictx->flags); |
49 | return inode_dio_wait_interruptible(inode: &ictx->inode); |
50 | } |
51 | |
52 | /** |
53 | * netfs_start_io_read - declare the file is being used for buffered reads |
54 | * @inode: file inode |
55 | * |
56 | * Declare that a buffered read operation is about to start, and ensure |
57 | * that we block all direct I/O. |
58 | * On exit, the function ensures that the NETFS_ICTX_ODIRECT flag is unset, |
59 | * and holds a shared lock on inode->i_rwsem to ensure that the flag |
60 | * cannot be changed. |
61 | * In practice, this means that buffered read operations are allowed to |
62 | * execute in parallel, thanks to the shared lock, whereas direct I/O |
63 | * operations need to wait to grab an exclusive lock in order to set |
64 | * NETFS_ICTX_ODIRECT. |
65 | * Note that buffered writes and truncates both take a write lock on |
66 | * inode->i_rwsem, meaning that those are serialised w.r.t. the reads. |
67 | */ |
68 | int netfs_start_io_read(struct inode *inode) |
69 | __acquires(inode->i_rwsem) |
70 | { |
71 | struct netfs_inode *ictx = netfs_inode(inode); |
72 | |
73 | /* Be an optimist! */ |
74 | if (down_read_interruptible(sem: &inode->i_rwsem) < 0) |
75 | return -ERESTARTSYS; |
76 | if (test_bit(NETFS_ICTX_ODIRECT, &ictx->flags) == 0) |
77 | return 0; |
78 | up_read(sem: &inode->i_rwsem); |
79 | |
80 | /* Slow path.... */ |
81 | if (down_write_killable(sem: &inode->i_rwsem) < 0) |
82 | return -ERESTARTSYS; |
83 | if (netfs_block_o_direct(ictx) < 0) { |
84 | up_write(sem: &inode->i_rwsem); |
85 | return -ERESTARTSYS; |
86 | } |
87 | downgrade_write(sem: &inode->i_rwsem); |
88 | return 0; |
89 | } |
90 | EXPORT_SYMBOL(netfs_start_io_read); |
91 | |
92 | /** |
93 | * netfs_end_io_read - declare that the buffered read operation is done |
94 | * @inode: file inode |
95 | * |
96 | * Declare that a buffered read operation is done, and release the shared |
97 | * lock on inode->i_rwsem. |
98 | */ |
99 | void netfs_end_io_read(struct inode *inode) |
100 | __releases(inode->i_rwsem) |
101 | { |
102 | up_read(sem: &inode->i_rwsem); |
103 | } |
104 | EXPORT_SYMBOL(netfs_end_io_read); |
105 | |
106 | /** |
107 | * netfs_start_io_write - declare the file is being used for buffered writes |
108 | * @inode: file inode |
109 | * |
110 | * Declare that a buffered read operation is about to start, and ensure |
111 | * that we block all direct I/O. |
112 | */ |
113 | int netfs_start_io_write(struct inode *inode) |
114 | __acquires(inode->i_rwsem) |
115 | { |
116 | struct netfs_inode *ictx = netfs_inode(inode); |
117 | |
118 | if (down_write_killable(sem: &inode->i_rwsem) < 0) |
119 | return -ERESTARTSYS; |
120 | if (netfs_block_o_direct(ictx) < 0) { |
121 | up_write(sem: &inode->i_rwsem); |
122 | return -ERESTARTSYS; |
123 | } |
124 | return 0; |
125 | } |
126 | EXPORT_SYMBOL(netfs_start_io_write); |
127 | |
128 | /** |
129 | * netfs_end_io_write - declare that the buffered write operation is done |
130 | * @inode: file inode |
131 | * |
132 | * Declare that a buffered write operation is done, and release the |
133 | * lock on inode->i_rwsem. |
134 | */ |
135 | void netfs_end_io_write(struct inode *inode) |
136 | __releases(inode->i_rwsem) |
137 | { |
138 | up_write(sem: &inode->i_rwsem); |
139 | } |
140 | EXPORT_SYMBOL(netfs_end_io_write); |
141 | |
142 | /* Call with exclusively locked inode->i_rwsem */ |
143 | static int netfs_block_buffered(struct inode *inode) |
144 | { |
145 | struct netfs_inode *ictx = netfs_inode(inode); |
146 | int ret; |
147 | |
148 | if (!test_bit(NETFS_ICTX_ODIRECT, &ictx->flags)) { |
149 | set_bit(NETFS_ICTX_ODIRECT, addr: &ictx->flags); |
150 | if (inode->i_mapping->nrpages != 0) { |
151 | unmap_mapping_range(mapping: inode->i_mapping, holebegin: 0, holelen: 0, even_cows: 0); |
152 | ret = filemap_fdatawait(mapping: inode->i_mapping); |
153 | if (ret < 0) { |
154 | clear_bit(NETFS_ICTX_ODIRECT, addr: &ictx->flags); |
155 | return ret; |
156 | } |
157 | } |
158 | } |
159 | return 0; |
160 | } |
161 | |
162 | /** |
163 | * netfs_start_io_direct - declare the file is being used for direct i/o |
164 | * @inode: file inode |
165 | * |
166 | * Declare that a direct I/O operation is about to start, and ensure |
167 | * that we block all buffered I/O. |
168 | * On exit, the function ensures that the NETFS_ICTX_ODIRECT flag is set, |
169 | * and holds a shared lock on inode->i_rwsem to ensure that the flag |
170 | * cannot be changed. |
171 | * In practice, this means that direct I/O operations are allowed to |
172 | * execute in parallel, thanks to the shared lock, whereas buffered I/O |
173 | * operations need to wait to grab an exclusive lock in order to clear |
174 | * NETFS_ICTX_ODIRECT. |
175 | * Note that buffered writes and truncates both take a write lock on |
176 | * inode->i_rwsem, meaning that those are serialised w.r.t. O_DIRECT. |
177 | */ |
178 | int netfs_start_io_direct(struct inode *inode) |
179 | __acquires(inode->i_rwsem) |
180 | { |
181 | struct netfs_inode *ictx = netfs_inode(inode); |
182 | int ret; |
183 | |
184 | /* Be an optimist! */ |
185 | if (down_read_interruptible(sem: &inode->i_rwsem) < 0) |
186 | return -ERESTARTSYS; |
187 | if (test_bit(NETFS_ICTX_ODIRECT, &ictx->flags) != 0) |
188 | return 0; |
189 | up_read(sem: &inode->i_rwsem); |
190 | |
191 | /* Slow path.... */ |
192 | if (down_write_killable(sem: &inode->i_rwsem) < 0) |
193 | return -ERESTARTSYS; |
194 | ret = netfs_block_buffered(inode); |
195 | if (ret < 0) { |
196 | up_write(sem: &inode->i_rwsem); |
197 | return ret; |
198 | } |
199 | downgrade_write(sem: &inode->i_rwsem); |
200 | return 0; |
201 | } |
202 | EXPORT_SYMBOL(netfs_start_io_direct); |
203 | |
204 | /** |
205 | * netfs_end_io_direct - declare that the direct i/o operation is done |
206 | * @inode: file inode |
207 | * |
208 | * Declare that a direct I/O operation is done, and release the shared |
209 | * lock on inode->i_rwsem. |
210 | */ |
211 | void netfs_end_io_direct(struct inode *inode) |
212 | __releases(inode->i_rwsem) |
213 | { |
214 | up_read(sem: &inode->i_rwsem); |
215 | } |
216 | EXPORT_SYMBOL(netfs_end_io_direct); |
217 | |