1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | |
3 | /* |
4 | * The implementation of the wait_bit*() and related waiting APIs: |
5 | */ |
6 | |
7 | #define WAIT_TABLE_BITS 8 |
8 | #define WAIT_TABLE_SIZE (1 << WAIT_TABLE_BITS) |
9 | |
10 | static wait_queue_head_t bit_wait_table[WAIT_TABLE_SIZE] __cacheline_aligned; |
11 | |
12 | wait_queue_head_t *bit_waitqueue(void *word, int bit) |
13 | { |
14 | const int shift = BITS_PER_LONG == 32 ? 5 : 6; |
15 | unsigned long val = (unsigned long)word << shift | bit; |
16 | |
17 | return bit_wait_table + hash_long(val, WAIT_TABLE_BITS); |
18 | } |
19 | EXPORT_SYMBOL(bit_waitqueue); |
20 | |
21 | int wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *arg) |
22 | { |
23 | struct wait_bit_key *key = arg; |
24 | struct wait_bit_queue_entry *wait_bit = container_of(wq_entry, struct wait_bit_queue_entry, wq_entry); |
25 | |
26 | if (wait_bit->key.flags != key->flags || |
27 | wait_bit->key.bit_nr != key->bit_nr || |
28 | test_bit(key->bit_nr, key->flags)) |
29 | return 0; |
30 | |
31 | return autoremove_wake_function(wq_entry, mode, sync, key); |
32 | } |
33 | EXPORT_SYMBOL(wake_bit_function); |
34 | |
35 | /* |
36 | * To allow interruptible waiting and asynchronous (i.e. nonblocking) |
37 | * waiting, the actions of __wait_on_bit() and __wait_on_bit_lock() are |
38 | * permitted return codes. Nonzero return codes halt waiting and return. |
39 | */ |
40 | int __sched |
41 | __wait_on_bit(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry, |
42 | wait_bit_action_f *action, unsigned mode) |
43 | { |
44 | int ret = 0; |
45 | |
46 | do { |
47 | prepare_to_wait(wq_head, wq_entry: &wbq_entry->wq_entry, state: mode); |
48 | if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) |
49 | ret = (*action)(&wbq_entry->key, mode); |
50 | } while (test_bit_acquire(wbq_entry->key.bit_nr, wbq_entry->key.flags) && !ret); |
51 | |
52 | finish_wait(wq_head, wq_entry: &wbq_entry->wq_entry); |
53 | |
54 | return ret; |
55 | } |
56 | EXPORT_SYMBOL(__wait_on_bit); |
57 | |
58 | int __sched out_of_line_wait_on_bit(void *word, int bit, |
59 | wait_bit_action_f *action, unsigned mode) |
60 | { |
61 | struct wait_queue_head *wq_head = bit_waitqueue(word, bit); |
62 | DEFINE_WAIT_BIT(wq_entry, word, bit); |
63 | |
64 | return __wait_on_bit(wq_head, &wq_entry, action, mode); |
65 | } |
66 | EXPORT_SYMBOL(out_of_line_wait_on_bit); |
67 | |
68 | int __sched out_of_line_wait_on_bit_timeout( |
69 | void *word, int bit, wait_bit_action_f *action, |
70 | unsigned mode, unsigned long timeout) |
71 | { |
72 | struct wait_queue_head *wq_head = bit_waitqueue(word, bit); |
73 | DEFINE_WAIT_BIT(wq_entry, word, bit); |
74 | |
75 | wq_entry.key.timeout = jiffies + timeout; |
76 | |
77 | return __wait_on_bit(wq_head, &wq_entry, action, mode); |
78 | } |
79 | EXPORT_SYMBOL_GPL(out_of_line_wait_on_bit_timeout); |
80 | |
81 | int __sched |
82 | __wait_on_bit_lock(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry, |
83 | wait_bit_action_f *action, unsigned mode) |
84 | { |
85 | int ret = 0; |
86 | |
87 | for (;;) { |
88 | prepare_to_wait_exclusive(wq_head, wq_entry: &wbq_entry->wq_entry, state: mode); |
89 | if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) { |
90 | ret = action(&wbq_entry->key, mode); |
91 | /* |
92 | * See the comment in prepare_to_wait_event(). |
93 | * finish_wait() does not necessarily takes wwq_head->lock, |
94 | * but test_and_set_bit() implies mb() which pairs with |
95 | * smp_mb__after_atomic() before wake_up_page(). |
96 | */ |
97 | if (ret) |
98 | finish_wait(wq_head, wq_entry: &wbq_entry->wq_entry); |
99 | } |
100 | if (!test_and_set_bit(nr: wbq_entry->key.bit_nr, addr: wbq_entry->key.flags)) { |
101 | if (!ret) |
102 | finish_wait(wq_head, wq_entry: &wbq_entry->wq_entry); |
103 | return 0; |
104 | } else if (ret) { |
105 | return ret; |
106 | } |
107 | } |
108 | } |
109 | EXPORT_SYMBOL(__wait_on_bit_lock); |
110 | |
111 | int __sched out_of_line_wait_on_bit_lock(void *word, int bit, |
112 | wait_bit_action_f *action, unsigned mode) |
113 | { |
114 | struct wait_queue_head *wq_head = bit_waitqueue(word, bit); |
115 | DEFINE_WAIT_BIT(wq_entry, word, bit); |
116 | |
117 | return __wait_on_bit_lock(wq_head, &wq_entry, action, mode); |
118 | } |
119 | EXPORT_SYMBOL(out_of_line_wait_on_bit_lock); |
120 | |
121 | void __wake_up_bit(struct wait_queue_head *wq_head, void *word, int bit) |
122 | { |
123 | struct wait_bit_key key = __WAIT_BIT_KEY_INITIALIZER(word, bit); |
124 | |
125 | if (waitqueue_active(wq_head)) |
126 | __wake_up(wq_head, TASK_NORMAL, nr: 1, key: &key); |
127 | } |
128 | EXPORT_SYMBOL(__wake_up_bit); |
129 | |
130 | /** |
131 | * wake_up_bit - wake up a waiter on a bit |
132 | * @word: the word being waited on, a kernel virtual address |
133 | * @bit: the bit of the word being waited on |
134 | * |
135 | * There is a standard hashed waitqueue table for generic use. This |
136 | * is the part of the hashtable's accessor API that wakes up waiters |
137 | * on a bit. For instance, if one were to have waiters on a bitflag, |
138 | * one would call wake_up_bit() after clearing the bit. |
139 | * |
140 | * In order for this to function properly, as it uses waitqueue_active() |
141 | * internally, some kind of memory barrier must be done prior to calling |
142 | * this. Typically, this will be smp_mb__after_atomic(), but in some |
143 | * cases where bitflags are manipulated non-atomically under a lock, one |
144 | * may need to use a less regular barrier, such fs/inode.c's smp_mb(), |
145 | * because spin_unlock() does not guarantee a memory barrier. |
146 | */ |
147 | void wake_up_bit(void *word, int bit) |
148 | { |
149 | __wake_up_bit(bit_waitqueue(word, bit), word, bit); |
150 | } |
151 | EXPORT_SYMBOL(wake_up_bit); |
152 | |
153 | wait_queue_head_t *__var_waitqueue(void *p) |
154 | { |
155 | return bit_wait_table + hash_ptr(ptr: p, WAIT_TABLE_BITS); |
156 | } |
157 | EXPORT_SYMBOL(__var_waitqueue); |
158 | |
159 | static int |
160 | var_wake_function(struct wait_queue_entry *wq_entry, unsigned int mode, |
161 | int sync, void *arg) |
162 | { |
163 | struct wait_bit_key *key = arg; |
164 | struct wait_bit_queue_entry *wbq_entry = |
165 | container_of(wq_entry, struct wait_bit_queue_entry, wq_entry); |
166 | |
167 | if (wbq_entry->key.flags != key->flags || |
168 | wbq_entry->key.bit_nr != key->bit_nr) |
169 | return 0; |
170 | |
171 | return autoremove_wake_function(wq_entry, mode, sync, key); |
172 | } |
173 | |
174 | void init_wait_var_entry(struct wait_bit_queue_entry *wbq_entry, void *var, int flags) |
175 | { |
176 | *wbq_entry = (struct wait_bit_queue_entry){ |
177 | .key = { |
178 | .flags = (var), |
179 | .bit_nr = -1, |
180 | }, |
181 | .wq_entry = { |
182 | .flags = flags, |
183 | .private = current, |
184 | .func = var_wake_function, |
185 | .entry = LIST_HEAD_INIT(wbq_entry->wq_entry.entry), |
186 | }, |
187 | }; |
188 | } |
189 | EXPORT_SYMBOL(init_wait_var_entry); |
190 | |
191 | void wake_up_var(void *var) |
192 | { |
193 | __wake_up_bit(__var_waitqueue(var), var, -1); |
194 | } |
195 | EXPORT_SYMBOL(wake_up_var); |
196 | |
197 | __sched int bit_wait(struct wait_bit_key *word, int mode) |
198 | { |
199 | schedule(); |
200 | if (signal_pending_state(state: mode, current)) |
201 | return -EINTR; |
202 | |
203 | return 0; |
204 | } |
205 | EXPORT_SYMBOL(bit_wait); |
206 | |
207 | __sched int bit_wait_io(struct wait_bit_key *word, int mode) |
208 | { |
209 | io_schedule(); |
210 | if (signal_pending_state(state: mode, current)) |
211 | return -EINTR; |
212 | |
213 | return 0; |
214 | } |
215 | EXPORT_SYMBOL(bit_wait_io); |
216 | |
217 | __sched int bit_wait_timeout(struct wait_bit_key *word, int mode) |
218 | { |
219 | unsigned long now = READ_ONCE(jiffies); |
220 | |
221 | if (time_after_eq(now, word->timeout)) |
222 | return -EAGAIN; |
223 | schedule_timeout(timeout: word->timeout - now); |
224 | if (signal_pending_state(state: mode, current)) |
225 | return -EINTR; |
226 | |
227 | return 0; |
228 | } |
229 | EXPORT_SYMBOL_GPL(bit_wait_timeout); |
230 | |
231 | __sched int bit_wait_io_timeout(struct wait_bit_key *word, int mode) |
232 | { |
233 | unsigned long now = READ_ONCE(jiffies); |
234 | |
235 | if (time_after_eq(now, word->timeout)) |
236 | return -EAGAIN; |
237 | io_schedule_timeout(timeout: word->timeout - now); |
238 | if (signal_pending_state(state: mode, current)) |
239 | return -EINTR; |
240 | |
241 | return 0; |
242 | } |
243 | EXPORT_SYMBOL_GPL(bit_wait_io_timeout); |
244 | |
245 | void __init wait_bit_init(void) |
246 | { |
247 | int i; |
248 | |
249 | for (i = 0; i < WAIT_TABLE_SIZE; i++) |
250 | init_waitqueue_head(bit_wait_table + i); |
251 | } |
252 | |