1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* netfs cookie management |
3 | * |
4 | * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. |
5 | * Written by David Howells (dhowells@redhat.com) |
6 | * |
7 | * See Documentation/filesystems/caching/netfs-api.rst for more information on |
8 | * the netfs API. |
9 | */ |
10 | |
11 | #define FSCACHE_DEBUG_LEVEL COOKIE |
12 | #include <linux/module.h> |
13 | #include <linux/slab.h> |
14 | #include "internal.h" |
15 | |
16 | struct kmem_cache *fscache_cookie_jar; |
17 | |
18 | static void fscache_cookie_lru_timed_out(struct timer_list *timer); |
19 | static void fscache_cookie_lru_worker(struct work_struct *work); |
20 | static void fscache_cookie_worker(struct work_struct *work); |
21 | static void fscache_unhash_cookie(struct fscache_cookie *cookie); |
22 | static void fscache_perform_invalidation(struct fscache_cookie *cookie); |
23 | |
24 | #define fscache_cookie_hash_shift 15 |
25 | static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift]; |
26 | static LIST_HEAD(fscache_cookies); |
27 | static DEFINE_RWLOCK(fscache_cookies_lock); |
28 | static LIST_HEAD(fscache_cookie_lru); |
29 | static DEFINE_SPINLOCK(fscache_cookie_lru_lock); |
30 | DEFINE_TIMER(fscache_cookie_lru_timer, fscache_cookie_lru_timed_out); |
31 | static DECLARE_WORK(fscache_cookie_lru_work, fscache_cookie_lru_worker); |
32 | static const char fscache_cookie_states[FSCACHE_COOKIE_STATE__NR] = "-LCAIFUWRD" ; |
33 | static unsigned int fscache_lru_cookie_timeout = 10 * HZ; |
34 | |
35 | void fscache_print_cookie(struct fscache_cookie *cookie, char prefix) |
36 | { |
37 | const u8 *k; |
38 | |
39 | pr_err("%c-cookie c=%08x [fl=%lx na=%u nA=%u s=%c]\n" , |
40 | prefix, |
41 | cookie->debug_id, |
42 | cookie->flags, |
43 | atomic_read(&cookie->n_active), |
44 | atomic_read(&cookie->n_accesses), |
45 | fscache_cookie_states[cookie->state]); |
46 | pr_err("%c-cookie V=%08x [%s]\n" , |
47 | prefix, |
48 | cookie->volume->debug_id, |
49 | cookie->volume->key); |
50 | |
51 | k = (cookie->key_len <= sizeof(cookie->inline_key)) ? |
52 | cookie->inline_key : cookie->key; |
53 | pr_err("%c-key=[%u] '%*phN'\n" , prefix, cookie->key_len, cookie->key_len, k); |
54 | } |
55 | |
56 | static void fscache_free_cookie(struct fscache_cookie *cookie) |
57 | { |
58 | if (WARN_ON_ONCE(!list_empty(&cookie->commit_link))) { |
59 | spin_lock(lock: &fscache_cookie_lru_lock); |
60 | list_del_init(entry: &cookie->commit_link); |
61 | spin_unlock(lock: &fscache_cookie_lru_lock); |
62 | fscache_stat_d(stat: &fscache_n_cookies_lru); |
63 | fscache_stat(stat: &fscache_n_cookies_lru_removed); |
64 | } |
65 | |
66 | if (WARN_ON_ONCE(test_bit(FSCACHE_COOKIE_IS_HASHED, &cookie->flags))) { |
67 | fscache_print_cookie(cookie, prefix: 'F'); |
68 | return; |
69 | } |
70 | |
71 | write_lock(&fscache_cookies_lock); |
72 | list_del(entry: &cookie->proc_link); |
73 | write_unlock(&fscache_cookies_lock); |
74 | if (cookie->aux_len > sizeof(cookie->inline_aux)) |
75 | kfree(objp: cookie->aux); |
76 | if (cookie->key_len > sizeof(cookie->inline_key)) |
77 | kfree(objp: cookie->key); |
78 | fscache_stat_d(stat: &fscache_n_cookies); |
79 | kmem_cache_free(s: fscache_cookie_jar, objp: cookie); |
80 | } |
81 | |
82 | static void __fscache_queue_cookie(struct fscache_cookie *cookie) |
83 | { |
84 | if (!queue_work(wq: fscache_wq, work: &cookie->work)) |
85 | fscache_put_cookie(cookie, where: fscache_cookie_put_over_queued); |
86 | } |
87 | |
88 | static void fscache_queue_cookie(struct fscache_cookie *cookie, |
89 | enum fscache_cookie_trace where) |
90 | { |
91 | fscache_get_cookie(cookie, where); |
92 | __fscache_queue_cookie(cookie); |
93 | } |
94 | |
95 | /* |
96 | * Initialise the access gate on a cookie by setting a flag to prevent the |
97 | * state machine from being queued when the access counter transitions to 0. |
98 | * We're only interested in this when we withdraw caching services from the |
99 | * cookie. |
100 | */ |
101 | static void fscache_init_access_gate(struct fscache_cookie *cookie) |
102 | { |
103 | int n_accesses; |
104 | |
105 | n_accesses = atomic_read(v: &cookie->n_accesses); |
106 | trace_fscache_access(cookie_debug_id: cookie->debug_id, ref: refcount_read(r: &cookie->ref), |
107 | n_accesses, why: fscache_access_cache_pin); |
108 | set_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, addr: &cookie->flags); |
109 | } |
110 | |
111 | /** |
112 | * fscache_end_cookie_access - Unpin a cache at the end of an access. |
113 | * @cookie: A data file cookie |
114 | * @why: An indication of the circumstances of the access for tracing |
115 | * |
116 | * Unpin a cache cookie after we've accessed it and bring a deferred |
117 | * relinquishment or withdrawal state into effect. |
118 | * |
119 | * The @why indicator is provided for tracing purposes. |
120 | */ |
121 | void fscache_end_cookie_access(struct fscache_cookie *cookie, |
122 | enum fscache_access_trace why) |
123 | { |
124 | int n_accesses; |
125 | |
126 | smp_mb__before_atomic(); |
127 | n_accesses = atomic_dec_return(v: &cookie->n_accesses); |
128 | trace_fscache_access(cookie_debug_id: cookie->debug_id, ref: refcount_read(r: &cookie->ref), |
129 | n_accesses, why); |
130 | if (n_accesses == 0 && |
131 | !test_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags)) |
132 | fscache_queue_cookie(cookie, where: fscache_cookie_get_end_access); |
133 | } |
134 | EXPORT_SYMBOL(fscache_end_cookie_access); |
135 | |
136 | /* |
137 | * Pin the cache behind a cookie so that we can access it. |
138 | */ |
139 | static void __fscache_begin_cookie_access(struct fscache_cookie *cookie, |
140 | enum fscache_access_trace why) |
141 | { |
142 | int n_accesses; |
143 | |
144 | n_accesses = atomic_inc_return(v: &cookie->n_accesses); |
145 | smp_mb__after_atomic(); /* (Future) read state after is-caching. |
146 | * Reread n_accesses after is-caching |
147 | */ |
148 | trace_fscache_access(cookie_debug_id: cookie->debug_id, ref: refcount_read(r: &cookie->ref), |
149 | n_accesses, why); |
150 | } |
151 | |
152 | /** |
153 | * fscache_begin_cookie_access - Pin a cache so data can be accessed |
154 | * @cookie: A data file cookie |
155 | * @why: An indication of the circumstances of the access for tracing |
156 | * |
157 | * Attempt to pin the cache to prevent it from going away whilst we're |
158 | * accessing data and returns true if successful. This works as follows: |
159 | * |
160 | * (1) If the cookie is not being cached (ie. FSCACHE_COOKIE_IS_CACHING is not |
161 | * set), we return false to indicate access was not permitted. |
162 | * |
163 | * (2) If the cookie is being cached, we increment its n_accesses count and |
164 | * then recheck the IS_CACHING flag, ending the access if it got cleared. |
165 | * |
166 | * (3) When we end the access, we decrement the cookie's n_accesses and wake |
167 | * up the any waiters if it reaches 0. |
168 | * |
169 | * (4) Whilst the cookie is actively being cached, its n_accesses is kept |
170 | * artificially incremented to prevent wakeups from happening. |
171 | * |
172 | * (5) When the cache is taken offline or if the cookie is culled, the flag is |
173 | * cleared to prevent new accesses, the cookie's n_accesses is decremented |
174 | * and we wait for it to become 0. |
175 | * |
176 | * The @why indicator are merely provided for tracing purposes. |
177 | */ |
178 | bool fscache_begin_cookie_access(struct fscache_cookie *cookie, |
179 | enum fscache_access_trace why) |
180 | { |
181 | if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags)) |
182 | return false; |
183 | __fscache_begin_cookie_access(cookie, why); |
184 | if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags) || |
185 | !fscache_cache_is_live(cache: cookie->volume->cache)) { |
186 | fscache_end_cookie_access(cookie, fscache_access_unlive); |
187 | return false; |
188 | } |
189 | return true; |
190 | } |
191 | |
192 | static inline void wake_up_cookie_state(struct fscache_cookie *cookie) |
193 | { |
194 | /* Use a barrier to ensure that waiters see the state variable |
195 | * change, as spin_unlock doesn't guarantee a barrier. |
196 | * |
197 | * See comments over wake_up_bit() and waitqueue_active(). |
198 | */ |
199 | smp_mb(); |
200 | wake_up_var(var: &cookie->state); |
201 | } |
202 | |
203 | /* |
204 | * Change the state a cookie is at and wake up anyone waiting for that. Impose |
205 | * an ordering between the stuff stored in the cookie and the state member. |
206 | * Paired with fscache_cookie_state(). |
207 | */ |
208 | static void __fscache_set_cookie_state(struct fscache_cookie *cookie, |
209 | enum fscache_cookie_state state) |
210 | { |
211 | smp_store_release(&cookie->state, state); |
212 | } |
213 | |
214 | static void fscache_set_cookie_state(struct fscache_cookie *cookie, |
215 | enum fscache_cookie_state state) |
216 | { |
217 | spin_lock(lock: &cookie->lock); |
218 | __fscache_set_cookie_state(cookie, state); |
219 | spin_unlock(lock: &cookie->lock); |
220 | wake_up_cookie_state(cookie); |
221 | } |
222 | |
223 | /** |
224 | * fscache_cookie_lookup_negative - Note negative lookup |
225 | * @cookie: The cookie that was being looked up |
226 | * |
227 | * Note that some part of the metadata path in the cache doesn't exist and so |
228 | * we can release any waiting readers in the certain knowledge that there's |
229 | * nothing for them to actually read. |
230 | * |
231 | * This function uses no locking and must only be called from the state machine. |
232 | */ |
233 | void fscache_cookie_lookup_negative(struct fscache_cookie *cookie) |
234 | { |
235 | set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, addr: &cookie->flags); |
236 | fscache_set_cookie_state(cookie, state: FSCACHE_COOKIE_STATE_CREATING); |
237 | } |
238 | EXPORT_SYMBOL(fscache_cookie_lookup_negative); |
239 | |
240 | /** |
241 | * fscache_resume_after_invalidation - Allow I/O to resume after invalidation |
242 | * @cookie: The cookie that was invalidated |
243 | * |
244 | * Tell fscache that invalidation is sufficiently complete that I/O can be |
245 | * allowed again. |
246 | */ |
247 | void fscache_resume_after_invalidation(struct fscache_cookie *cookie) |
248 | { |
249 | fscache_set_cookie_state(cookie, state: FSCACHE_COOKIE_STATE_ACTIVE); |
250 | } |
251 | EXPORT_SYMBOL(fscache_resume_after_invalidation); |
252 | |
253 | /** |
254 | * fscache_caching_failed - Report that a failure stopped caching on a cookie |
255 | * @cookie: The cookie that was affected |
256 | * |
257 | * Tell fscache that caching on a cookie needs to be stopped due to some sort |
258 | * of failure. |
259 | * |
260 | * This function uses no locking and must only be called from the state machine. |
261 | */ |
262 | void fscache_caching_failed(struct fscache_cookie *cookie) |
263 | { |
264 | clear_bit(FSCACHE_COOKIE_IS_CACHING, addr: &cookie->flags); |
265 | fscache_set_cookie_state(cookie, state: FSCACHE_COOKIE_STATE_FAILED); |
266 | trace_fscache_cookie(cookie_debug_id: cookie->debug_id, ref: refcount_read(r: &cookie->ref), |
267 | where: fscache_cookie_failed); |
268 | } |
269 | EXPORT_SYMBOL(fscache_caching_failed); |
270 | |
271 | /* |
272 | * Set the index key in a cookie. The cookie struct has space for a 16-byte |
273 | * key plus length and hash, but if that's not big enough, it's instead a |
274 | * pointer to a buffer containing 3 bytes of hash, 1 byte of length and then |
275 | * the key data. |
276 | */ |
277 | static int fscache_set_key(struct fscache_cookie *cookie, |
278 | const void *index_key, size_t index_key_len) |
279 | { |
280 | void *buf; |
281 | size_t buf_size; |
282 | |
283 | buf_size = round_up(index_key_len, sizeof(__le32)); |
284 | |
285 | if (index_key_len > sizeof(cookie->inline_key)) { |
286 | buf = kzalloc(size: buf_size, GFP_KERNEL); |
287 | if (!buf) |
288 | return -ENOMEM; |
289 | cookie->key = buf; |
290 | } else { |
291 | buf = cookie->inline_key; |
292 | } |
293 | |
294 | memcpy(buf, index_key, index_key_len); |
295 | cookie->key_hash = fscache_hash(salt: cookie->volume->key_hash, |
296 | data: buf, len: buf_size); |
297 | return 0; |
298 | } |
299 | |
300 | static bool fscache_cookie_same(const struct fscache_cookie *a, |
301 | const struct fscache_cookie *b) |
302 | { |
303 | const void *ka, *kb; |
304 | |
305 | if (a->key_hash != b->key_hash || |
306 | a->volume != b->volume || |
307 | a->key_len != b->key_len) |
308 | return false; |
309 | |
310 | if (a->key_len <= sizeof(a->inline_key)) { |
311 | ka = &a->inline_key; |
312 | kb = &b->inline_key; |
313 | } else { |
314 | ka = a->key; |
315 | kb = b->key; |
316 | } |
317 | return memcmp(p: ka, q: kb, size: a->key_len) == 0; |
318 | } |
319 | |
320 | static atomic_t fscache_cookie_debug_id = ATOMIC_INIT(1); |
321 | |
322 | /* |
323 | * Allocate a cookie. |
324 | */ |
325 | static struct fscache_cookie *fscache_alloc_cookie( |
326 | struct fscache_volume *volume, |
327 | u8 advice, |
328 | const void *index_key, size_t index_key_len, |
329 | const void *aux_data, size_t aux_data_len, |
330 | loff_t object_size) |
331 | { |
332 | struct fscache_cookie *cookie; |
333 | |
334 | /* allocate and initialise a cookie */ |
335 | cookie = kmem_cache_zalloc(k: fscache_cookie_jar, GFP_KERNEL); |
336 | if (!cookie) |
337 | return NULL; |
338 | fscache_stat(stat: &fscache_n_cookies); |
339 | |
340 | cookie->volume = volume; |
341 | cookie->advice = advice; |
342 | cookie->key_len = index_key_len; |
343 | cookie->aux_len = aux_data_len; |
344 | cookie->object_size = object_size; |
345 | if (object_size == 0) |
346 | __set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); |
347 | |
348 | if (fscache_set_key(cookie, index_key, index_key_len) < 0) |
349 | goto nomem; |
350 | |
351 | if (cookie->aux_len <= sizeof(cookie->inline_aux)) { |
352 | memcpy(cookie->inline_aux, aux_data, cookie->aux_len); |
353 | } else { |
354 | cookie->aux = kmemdup(p: aux_data, size: cookie->aux_len, GFP_KERNEL); |
355 | if (!cookie->aux) |
356 | goto nomem; |
357 | } |
358 | |
359 | refcount_set(r: &cookie->ref, n: 1); |
360 | cookie->debug_id = atomic_inc_return(v: &fscache_cookie_debug_id); |
361 | spin_lock_init(&cookie->lock); |
362 | INIT_LIST_HEAD(list: &cookie->commit_link); |
363 | INIT_WORK(&cookie->work, fscache_cookie_worker); |
364 | __fscache_set_cookie_state(cookie, state: FSCACHE_COOKIE_STATE_QUIESCENT); |
365 | |
366 | write_lock(&fscache_cookies_lock); |
367 | list_add_tail(new: &cookie->proc_link, head: &fscache_cookies); |
368 | write_unlock(&fscache_cookies_lock); |
369 | fscache_see_cookie(cookie, where: fscache_cookie_new_acquire); |
370 | return cookie; |
371 | |
372 | nomem: |
373 | fscache_free_cookie(cookie); |
374 | return NULL; |
375 | } |
376 | |
377 | static inline bool fscache_cookie_is_dropped(struct fscache_cookie *cookie) |
378 | { |
379 | return READ_ONCE(cookie->state) == FSCACHE_COOKIE_STATE_DROPPED; |
380 | } |
381 | |
382 | static void fscache_wait_on_collision(struct fscache_cookie *candidate, |
383 | struct fscache_cookie *wait_for) |
384 | { |
385 | enum fscache_cookie_state *statep = &wait_for->state; |
386 | |
387 | wait_var_event_timeout(statep, fscache_cookie_is_dropped(wait_for), |
388 | 20 * HZ); |
389 | if (!fscache_cookie_is_dropped(cookie: wait_for)) { |
390 | pr_notice("Potential collision c=%08x old: c=%08x" , |
391 | candidate->debug_id, wait_for->debug_id); |
392 | wait_var_event(statep, fscache_cookie_is_dropped(wait_for)); |
393 | } |
394 | } |
395 | |
396 | /* |
397 | * Attempt to insert the new cookie into the hash. If there's a collision, we |
398 | * wait for the old cookie to complete if it's being relinquished and an error |
399 | * otherwise. |
400 | */ |
401 | static bool fscache_hash_cookie(struct fscache_cookie *candidate) |
402 | { |
403 | struct fscache_cookie *cursor, *wait_for = NULL; |
404 | struct hlist_bl_head *h; |
405 | struct hlist_bl_node *p; |
406 | unsigned int bucket; |
407 | |
408 | bucket = candidate->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1); |
409 | h = &fscache_cookie_hash[bucket]; |
410 | |
411 | hlist_bl_lock(b: h); |
412 | hlist_bl_for_each_entry(cursor, p, h, hash_link) { |
413 | if (fscache_cookie_same(a: candidate, b: cursor)) { |
414 | if (!test_bit(FSCACHE_COOKIE_RELINQUISHED, &cursor->flags)) |
415 | goto collision; |
416 | wait_for = fscache_get_cookie(cookie: cursor, |
417 | where: fscache_cookie_get_hash_collision); |
418 | break; |
419 | } |
420 | } |
421 | |
422 | fscache_get_volume(volume: candidate->volume, where: fscache_volume_get_cookie); |
423 | atomic_inc(v: &candidate->volume->n_cookies); |
424 | hlist_bl_add_head(n: &candidate->hash_link, h); |
425 | set_bit(FSCACHE_COOKIE_IS_HASHED, addr: &candidate->flags); |
426 | hlist_bl_unlock(b: h); |
427 | |
428 | if (wait_for) { |
429 | fscache_wait_on_collision(candidate, wait_for); |
430 | fscache_put_cookie(cookie: wait_for, where: fscache_cookie_put_hash_collision); |
431 | } |
432 | return true; |
433 | |
434 | collision: |
435 | trace_fscache_cookie(cookie_debug_id: cursor->debug_id, ref: refcount_read(r: &cursor->ref), |
436 | where: fscache_cookie_collision); |
437 | pr_err("Duplicate cookie detected\n" ); |
438 | fscache_print_cookie(cookie: cursor, prefix: 'O'); |
439 | fscache_print_cookie(cookie: candidate, prefix: 'N'); |
440 | hlist_bl_unlock(b: h); |
441 | return false; |
442 | } |
443 | |
444 | /* |
445 | * Request a cookie to represent a data storage object within a volume. |
446 | * |
447 | * We never let on to the netfs about errors. We may set a negative cookie |
448 | * pointer, but that's okay |
449 | */ |
450 | struct fscache_cookie *__fscache_acquire_cookie( |
451 | struct fscache_volume *volume, |
452 | u8 advice, |
453 | const void *index_key, size_t index_key_len, |
454 | const void *aux_data, size_t aux_data_len, |
455 | loff_t object_size) |
456 | { |
457 | struct fscache_cookie *cookie; |
458 | |
459 | _enter("V=%x" , volume->debug_id); |
460 | |
461 | if (!index_key || !index_key_len || index_key_len > 255 || aux_data_len > 255) |
462 | return NULL; |
463 | if (!aux_data || !aux_data_len) { |
464 | aux_data = NULL; |
465 | aux_data_len = 0; |
466 | } |
467 | |
468 | fscache_stat(stat: &fscache_n_acquires); |
469 | |
470 | cookie = fscache_alloc_cookie(volume, advice, |
471 | index_key, index_key_len, |
472 | aux_data, aux_data_len, |
473 | object_size); |
474 | if (!cookie) { |
475 | fscache_stat(stat: &fscache_n_acquires_oom); |
476 | return NULL; |
477 | } |
478 | |
479 | if (!fscache_hash_cookie(candidate: cookie)) { |
480 | fscache_see_cookie(cookie, where: fscache_cookie_discard); |
481 | fscache_free_cookie(cookie); |
482 | return NULL; |
483 | } |
484 | |
485 | trace_fscache_acquire(cookie); |
486 | fscache_stat(stat: &fscache_n_acquires_ok); |
487 | _leave(" = c=%08x" , cookie->debug_id); |
488 | return cookie; |
489 | } |
490 | EXPORT_SYMBOL(__fscache_acquire_cookie); |
491 | |
492 | /* |
493 | * Prepare a cache object to be written to. |
494 | */ |
495 | static void fscache_prepare_to_write(struct fscache_cookie *cookie) |
496 | { |
497 | cookie->volume->cache->ops->prepare_to_write(cookie); |
498 | } |
499 | |
500 | /* |
501 | * Look up a cookie in the cache. |
502 | */ |
503 | static void fscache_perform_lookup(struct fscache_cookie *cookie) |
504 | { |
505 | enum fscache_access_trace trace = fscache_access_lookup_cookie_end_failed; |
506 | bool need_withdraw = false; |
507 | |
508 | _enter("" ); |
509 | |
510 | if (!cookie->volume->cache_priv) { |
511 | fscache_create_volume(volume: cookie->volume, wait: true); |
512 | if (!cookie->volume->cache_priv) { |
513 | fscache_set_cookie_state(cookie, state: FSCACHE_COOKIE_STATE_QUIESCENT); |
514 | goto out; |
515 | } |
516 | } |
517 | |
518 | if (!cookie->volume->cache->ops->lookup_cookie(cookie)) { |
519 | if (cookie->state != FSCACHE_COOKIE_STATE_FAILED) |
520 | fscache_set_cookie_state(cookie, state: FSCACHE_COOKIE_STATE_QUIESCENT); |
521 | need_withdraw = true; |
522 | _leave(" [fail]" ); |
523 | goto out; |
524 | } |
525 | |
526 | fscache_see_cookie(cookie, where: fscache_cookie_see_active); |
527 | spin_lock(lock: &cookie->lock); |
528 | if (test_and_clear_bit(FSCACHE_COOKIE_DO_INVALIDATE, addr: &cookie->flags)) |
529 | __fscache_set_cookie_state(cookie, |
530 | state: FSCACHE_COOKIE_STATE_INVALIDATING); |
531 | else |
532 | __fscache_set_cookie_state(cookie, state: FSCACHE_COOKIE_STATE_ACTIVE); |
533 | spin_unlock(lock: &cookie->lock); |
534 | wake_up_cookie_state(cookie); |
535 | trace = fscache_access_lookup_cookie_end; |
536 | |
537 | out: |
538 | fscache_end_cookie_access(cookie, trace); |
539 | if (need_withdraw) |
540 | fscache_withdraw_cookie(cookie); |
541 | fscache_end_volume_access(volume: cookie->volume, cookie, why: trace); |
542 | } |
543 | |
544 | /* |
545 | * Begin the process of looking up a cookie. We offload the actual process to |
546 | * a worker thread. |
547 | */ |
548 | static bool fscache_begin_lookup(struct fscache_cookie *cookie, bool will_modify) |
549 | { |
550 | if (will_modify) { |
551 | set_bit(FSCACHE_COOKIE_LOCAL_WRITE, addr: &cookie->flags); |
552 | set_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, addr: &cookie->flags); |
553 | } |
554 | if (!fscache_begin_volume_access(volume: cookie->volume, cookie, |
555 | why: fscache_access_lookup_cookie)) |
556 | return false; |
557 | |
558 | __fscache_begin_cookie_access(cookie, why: fscache_access_lookup_cookie); |
559 | __fscache_set_cookie_state(cookie, state: FSCACHE_COOKIE_STATE_LOOKING_UP); |
560 | set_bit(FSCACHE_COOKIE_IS_CACHING, addr: &cookie->flags); |
561 | set_bit(FSCACHE_COOKIE_HAS_BEEN_CACHED, addr: &cookie->flags); |
562 | return true; |
563 | } |
564 | |
565 | /* |
566 | * Start using the cookie for I/O. This prevents the backing object from being |
567 | * reaped by VM pressure. |
568 | */ |
569 | void __fscache_use_cookie(struct fscache_cookie *cookie, bool will_modify) |
570 | { |
571 | enum fscache_cookie_state state; |
572 | bool queue = false; |
573 | int n_active; |
574 | |
575 | _enter("c=%08x" , cookie->debug_id); |
576 | |
577 | if (WARN(test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags), |
578 | "Trying to use relinquished cookie\n" )) |
579 | return; |
580 | |
581 | spin_lock(lock: &cookie->lock); |
582 | |
583 | n_active = atomic_inc_return(v: &cookie->n_active); |
584 | trace_fscache_active(cookie_debug_id: cookie->debug_id, ref: refcount_read(r: &cookie->ref), |
585 | n_active, n_accesses: atomic_read(v: &cookie->n_accesses), |
586 | why: will_modify ? |
587 | fscache_active_use_modify : fscache_active_use); |
588 | |
589 | again: |
590 | state = fscache_cookie_state(cookie); |
591 | switch (state) { |
592 | case FSCACHE_COOKIE_STATE_QUIESCENT: |
593 | queue = fscache_begin_lookup(cookie, will_modify); |
594 | break; |
595 | |
596 | case FSCACHE_COOKIE_STATE_LOOKING_UP: |
597 | case FSCACHE_COOKIE_STATE_CREATING: |
598 | if (will_modify) |
599 | set_bit(FSCACHE_COOKIE_LOCAL_WRITE, addr: &cookie->flags); |
600 | break; |
601 | case FSCACHE_COOKIE_STATE_ACTIVE: |
602 | case FSCACHE_COOKIE_STATE_INVALIDATING: |
603 | if (will_modify && |
604 | !test_and_set_bit(FSCACHE_COOKIE_LOCAL_WRITE, addr: &cookie->flags)) { |
605 | set_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, addr: &cookie->flags); |
606 | queue = true; |
607 | } |
608 | /* |
609 | * We could race with cookie_lru which may set LRU_DISCARD bit |
610 | * but has yet to run the cookie state machine. If this happens |
611 | * and another thread tries to use the cookie, clear LRU_DISCARD |
612 | * so we don't end up withdrawing the cookie while in use. |
613 | */ |
614 | if (test_and_clear_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, addr: &cookie->flags)) |
615 | fscache_see_cookie(cookie, where: fscache_cookie_see_lru_discard_clear); |
616 | break; |
617 | |
618 | case FSCACHE_COOKIE_STATE_FAILED: |
619 | case FSCACHE_COOKIE_STATE_WITHDRAWING: |
620 | break; |
621 | |
622 | case FSCACHE_COOKIE_STATE_LRU_DISCARDING: |
623 | spin_unlock(lock: &cookie->lock); |
624 | wait_var_event(&cookie->state, |
625 | fscache_cookie_state(cookie) != |
626 | FSCACHE_COOKIE_STATE_LRU_DISCARDING); |
627 | spin_lock(lock: &cookie->lock); |
628 | goto again; |
629 | |
630 | case FSCACHE_COOKIE_STATE_DROPPED: |
631 | case FSCACHE_COOKIE_STATE_RELINQUISHING: |
632 | WARN(1, "Can't use cookie in state %u\n" , state); |
633 | break; |
634 | } |
635 | |
636 | spin_unlock(lock: &cookie->lock); |
637 | if (queue) |
638 | fscache_queue_cookie(cookie, where: fscache_cookie_get_use_work); |
639 | _leave("" ); |
640 | } |
641 | EXPORT_SYMBOL(__fscache_use_cookie); |
642 | |
643 | static void fscache_unuse_cookie_locked(struct fscache_cookie *cookie) |
644 | { |
645 | clear_bit(FSCACHE_COOKIE_DISABLED, addr: &cookie->flags); |
646 | if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags)) |
647 | return; |
648 | |
649 | cookie->unused_at = jiffies; |
650 | spin_lock(lock: &fscache_cookie_lru_lock); |
651 | if (list_empty(head: &cookie->commit_link)) { |
652 | fscache_get_cookie(cookie, where: fscache_cookie_get_lru); |
653 | fscache_stat(stat: &fscache_n_cookies_lru); |
654 | } |
655 | list_move_tail(list: &cookie->commit_link, head: &fscache_cookie_lru); |
656 | |
657 | spin_unlock(lock: &fscache_cookie_lru_lock); |
658 | timer_reduce(timer: &fscache_cookie_lru_timer, |
659 | expires: jiffies + fscache_lru_cookie_timeout); |
660 | } |
661 | |
662 | /* |
663 | * Stop using the cookie for I/O. |
664 | */ |
665 | void __fscache_unuse_cookie(struct fscache_cookie *cookie, |
666 | const void *aux_data, const loff_t *object_size) |
667 | { |
668 | unsigned int debug_id = cookie->debug_id; |
669 | unsigned int r = refcount_read(r: &cookie->ref); |
670 | unsigned int a = atomic_read(v: &cookie->n_accesses); |
671 | unsigned int c; |
672 | |
673 | if (aux_data || object_size) |
674 | __fscache_update_cookie(cookie, aux_data, object_size); |
675 | |
676 | /* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */ |
677 | c = atomic_fetch_add_unless(v: &cookie->n_active, a: -1, u: 1); |
678 | if (c != 1) { |
679 | trace_fscache_active(cookie_debug_id: debug_id, ref: r, n_active: c - 1, n_accesses: a, why: fscache_active_unuse); |
680 | return; |
681 | } |
682 | |
683 | spin_lock(lock: &cookie->lock); |
684 | r = refcount_read(r: &cookie->ref); |
685 | a = atomic_read(v: &cookie->n_accesses); |
686 | c = atomic_dec_return(v: &cookie->n_active); |
687 | trace_fscache_active(cookie_debug_id: debug_id, ref: r, n_active: c, n_accesses: a, why: fscache_active_unuse); |
688 | if (c == 0) |
689 | fscache_unuse_cookie_locked(cookie); |
690 | spin_unlock(lock: &cookie->lock); |
691 | } |
692 | EXPORT_SYMBOL(__fscache_unuse_cookie); |
693 | |
694 | /* |
695 | * Perform work upon the cookie, such as committing its cache state, |
696 | * relinquishing it or withdrawing the backing cache. We're protected from the |
697 | * cache going away under us as object withdrawal must come through this |
698 | * non-reentrant work item. |
699 | */ |
700 | static void fscache_cookie_state_machine(struct fscache_cookie *cookie) |
701 | { |
702 | enum fscache_cookie_state state; |
703 | bool wake = false; |
704 | |
705 | _enter("c=%x" , cookie->debug_id); |
706 | |
707 | again: |
708 | spin_lock(lock: &cookie->lock); |
709 | again_locked: |
710 | state = cookie->state; |
711 | switch (state) { |
712 | case FSCACHE_COOKIE_STATE_QUIESCENT: |
713 | /* The QUIESCENT state is jumped to the LOOKING_UP state by |
714 | * fscache_use_cookie(). |
715 | */ |
716 | |
717 | if (atomic_read(v: &cookie->n_accesses) == 0 && |
718 | test_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags)) { |
719 | __fscache_set_cookie_state(cookie, |
720 | state: FSCACHE_COOKIE_STATE_RELINQUISHING); |
721 | wake = true; |
722 | goto again_locked; |
723 | } |
724 | break; |
725 | |
726 | case FSCACHE_COOKIE_STATE_LOOKING_UP: |
727 | spin_unlock(lock: &cookie->lock); |
728 | fscache_init_access_gate(cookie); |
729 | fscache_perform_lookup(cookie); |
730 | goto again; |
731 | |
732 | case FSCACHE_COOKIE_STATE_INVALIDATING: |
733 | spin_unlock(lock: &cookie->lock); |
734 | fscache_perform_invalidation(cookie); |
735 | goto again; |
736 | |
737 | case FSCACHE_COOKIE_STATE_ACTIVE: |
738 | if (test_and_clear_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, addr: &cookie->flags)) { |
739 | spin_unlock(lock: &cookie->lock); |
740 | fscache_prepare_to_write(cookie); |
741 | spin_lock(lock: &cookie->lock); |
742 | } |
743 | if (test_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags)) { |
744 | __fscache_set_cookie_state(cookie, |
745 | state: FSCACHE_COOKIE_STATE_LRU_DISCARDING); |
746 | wake = true; |
747 | goto again_locked; |
748 | } |
749 | fallthrough; |
750 | |
751 | case FSCACHE_COOKIE_STATE_FAILED: |
752 | if (test_and_clear_bit(FSCACHE_COOKIE_DO_INVALIDATE, addr: &cookie->flags)) |
753 | fscache_end_cookie_access(cookie, fscache_access_invalidate_cookie_end); |
754 | |
755 | if (atomic_read(v: &cookie->n_accesses) != 0) |
756 | break; |
757 | if (test_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags)) { |
758 | __fscache_set_cookie_state(cookie, |
759 | state: FSCACHE_COOKIE_STATE_RELINQUISHING); |
760 | wake = true; |
761 | goto again_locked; |
762 | } |
763 | if (test_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags)) { |
764 | __fscache_set_cookie_state(cookie, |
765 | state: FSCACHE_COOKIE_STATE_WITHDRAWING); |
766 | wake = true; |
767 | goto again_locked; |
768 | } |
769 | break; |
770 | |
771 | case FSCACHE_COOKIE_STATE_LRU_DISCARDING: |
772 | case FSCACHE_COOKIE_STATE_RELINQUISHING: |
773 | case FSCACHE_COOKIE_STATE_WITHDRAWING: |
774 | if (cookie->cache_priv) { |
775 | spin_unlock(lock: &cookie->lock); |
776 | cookie->volume->cache->ops->withdraw_cookie(cookie); |
777 | spin_lock(lock: &cookie->lock); |
778 | } |
779 | |
780 | if (test_and_clear_bit(FSCACHE_COOKIE_DO_INVALIDATE, addr: &cookie->flags)) |
781 | fscache_end_cookie_access(cookie, fscache_access_invalidate_cookie_end); |
782 | |
783 | switch (state) { |
784 | case FSCACHE_COOKIE_STATE_RELINQUISHING: |
785 | fscache_see_cookie(cookie, where: fscache_cookie_see_relinquish); |
786 | fscache_unhash_cookie(cookie); |
787 | __fscache_set_cookie_state(cookie, |
788 | state: FSCACHE_COOKIE_STATE_DROPPED); |
789 | wake = true; |
790 | goto out; |
791 | case FSCACHE_COOKIE_STATE_LRU_DISCARDING: |
792 | fscache_see_cookie(cookie, where: fscache_cookie_see_lru_discard); |
793 | break; |
794 | case FSCACHE_COOKIE_STATE_WITHDRAWING: |
795 | fscache_see_cookie(cookie, where: fscache_cookie_see_withdraw); |
796 | break; |
797 | default: |
798 | BUG(); |
799 | } |
800 | |
801 | clear_bit(FSCACHE_COOKIE_NEEDS_UPDATE, addr: &cookie->flags); |
802 | clear_bit(FSCACHE_COOKIE_DO_WITHDRAW, addr: &cookie->flags); |
803 | clear_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, addr: &cookie->flags); |
804 | clear_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, addr: &cookie->flags); |
805 | set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, addr: &cookie->flags); |
806 | __fscache_set_cookie_state(cookie, state: FSCACHE_COOKIE_STATE_QUIESCENT); |
807 | wake = true; |
808 | goto again_locked; |
809 | |
810 | case FSCACHE_COOKIE_STATE_DROPPED: |
811 | break; |
812 | |
813 | default: |
814 | WARN_ONCE(1, "Cookie %x in unexpected state %u\n" , |
815 | cookie->debug_id, state); |
816 | break; |
817 | } |
818 | |
819 | out: |
820 | spin_unlock(lock: &cookie->lock); |
821 | if (wake) |
822 | wake_up_cookie_state(cookie); |
823 | _leave("" ); |
824 | } |
825 | |
826 | static void fscache_cookie_worker(struct work_struct *work) |
827 | { |
828 | struct fscache_cookie *cookie = container_of(work, struct fscache_cookie, work); |
829 | |
830 | fscache_see_cookie(cookie, where: fscache_cookie_see_work); |
831 | fscache_cookie_state_machine(cookie); |
832 | fscache_put_cookie(cookie, where: fscache_cookie_put_work); |
833 | } |
834 | |
835 | /* |
836 | * Wait for the object to become inactive. The cookie's work item will be |
837 | * scheduled when someone transitions n_accesses to 0 - but if someone's |
838 | * already done that, schedule it anyway. |
839 | */ |
840 | static void __fscache_withdraw_cookie(struct fscache_cookie *cookie) |
841 | { |
842 | int n_accesses; |
843 | bool unpinned; |
844 | |
845 | unpinned = test_and_clear_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, addr: &cookie->flags); |
846 | |
847 | /* Need to read the access count after unpinning */ |
848 | n_accesses = atomic_read(v: &cookie->n_accesses); |
849 | if (unpinned) |
850 | trace_fscache_access(cookie_debug_id: cookie->debug_id, ref: refcount_read(r: &cookie->ref), |
851 | n_accesses, why: fscache_access_cache_unpin); |
852 | if (n_accesses == 0) |
853 | fscache_queue_cookie(cookie, where: fscache_cookie_get_end_access); |
854 | } |
855 | |
856 | static void fscache_cookie_lru_do_one(struct fscache_cookie *cookie) |
857 | { |
858 | fscache_see_cookie(cookie, where: fscache_cookie_see_lru_do_one); |
859 | |
860 | spin_lock(lock: &cookie->lock); |
861 | if (cookie->state != FSCACHE_COOKIE_STATE_ACTIVE || |
862 | time_before(jiffies, cookie->unused_at + fscache_lru_cookie_timeout) || |
863 | atomic_read(v: &cookie->n_active) > 0) { |
864 | spin_unlock(lock: &cookie->lock); |
865 | fscache_stat(stat: &fscache_n_cookies_lru_removed); |
866 | } else { |
867 | set_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, addr: &cookie->flags); |
868 | spin_unlock(lock: &cookie->lock); |
869 | fscache_stat(stat: &fscache_n_cookies_lru_expired); |
870 | _debug("lru c=%x" , cookie->debug_id); |
871 | __fscache_withdraw_cookie(cookie); |
872 | } |
873 | |
874 | fscache_put_cookie(cookie, where: fscache_cookie_put_lru); |
875 | } |
876 | |
877 | static void fscache_cookie_lru_worker(struct work_struct *work) |
878 | { |
879 | struct fscache_cookie *cookie; |
880 | unsigned long unused_at; |
881 | |
882 | spin_lock(lock: &fscache_cookie_lru_lock); |
883 | |
884 | while (!list_empty(head: &fscache_cookie_lru)) { |
885 | cookie = list_first_entry(&fscache_cookie_lru, |
886 | struct fscache_cookie, commit_link); |
887 | unused_at = cookie->unused_at + fscache_lru_cookie_timeout; |
888 | if (time_before(jiffies, unused_at)) { |
889 | timer_reduce(timer: &fscache_cookie_lru_timer, expires: unused_at); |
890 | break; |
891 | } |
892 | |
893 | list_del_init(entry: &cookie->commit_link); |
894 | fscache_stat_d(stat: &fscache_n_cookies_lru); |
895 | spin_unlock(lock: &fscache_cookie_lru_lock); |
896 | fscache_cookie_lru_do_one(cookie); |
897 | spin_lock(lock: &fscache_cookie_lru_lock); |
898 | } |
899 | |
900 | spin_unlock(lock: &fscache_cookie_lru_lock); |
901 | } |
902 | |
903 | static void fscache_cookie_lru_timed_out(struct timer_list *timer) |
904 | { |
905 | queue_work(wq: fscache_wq, work: &fscache_cookie_lru_work); |
906 | } |
907 | |
908 | static void fscache_cookie_drop_from_lru(struct fscache_cookie *cookie) |
909 | { |
910 | bool need_put = false; |
911 | |
912 | if (!list_empty(head: &cookie->commit_link)) { |
913 | spin_lock(lock: &fscache_cookie_lru_lock); |
914 | if (!list_empty(head: &cookie->commit_link)) { |
915 | list_del_init(entry: &cookie->commit_link); |
916 | fscache_stat_d(stat: &fscache_n_cookies_lru); |
917 | fscache_stat(stat: &fscache_n_cookies_lru_dropped); |
918 | need_put = true; |
919 | } |
920 | spin_unlock(lock: &fscache_cookie_lru_lock); |
921 | if (need_put) |
922 | fscache_put_cookie(cookie, where: fscache_cookie_put_lru); |
923 | } |
924 | } |
925 | |
926 | /* |
927 | * Remove a cookie from the hash table. |
928 | */ |
929 | static void fscache_unhash_cookie(struct fscache_cookie *cookie) |
930 | { |
931 | struct hlist_bl_head *h; |
932 | unsigned int bucket; |
933 | |
934 | bucket = cookie->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1); |
935 | h = &fscache_cookie_hash[bucket]; |
936 | |
937 | hlist_bl_lock(b: h); |
938 | hlist_bl_del(n: &cookie->hash_link); |
939 | clear_bit(FSCACHE_COOKIE_IS_HASHED, addr: &cookie->flags); |
940 | hlist_bl_unlock(b: h); |
941 | fscache_stat(stat: &fscache_n_relinquishes_dropped); |
942 | } |
943 | |
944 | static void fscache_drop_withdraw_cookie(struct fscache_cookie *cookie) |
945 | { |
946 | fscache_cookie_drop_from_lru(cookie); |
947 | __fscache_withdraw_cookie(cookie); |
948 | } |
949 | |
950 | /** |
951 | * fscache_withdraw_cookie - Mark a cookie for withdrawal |
952 | * @cookie: The cookie to be withdrawn. |
953 | * |
954 | * Allow the cache backend to withdraw the backing for a cookie for its own |
955 | * reasons, even if that cookie is in active use. |
956 | */ |
957 | void fscache_withdraw_cookie(struct fscache_cookie *cookie) |
958 | { |
959 | set_bit(FSCACHE_COOKIE_DO_WITHDRAW, addr: &cookie->flags); |
960 | fscache_drop_withdraw_cookie(cookie); |
961 | } |
962 | EXPORT_SYMBOL(fscache_withdraw_cookie); |
963 | |
964 | /* |
965 | * Allow the netfs to release a cookie back to the cache. |
966 | * - the object will be marked as recyclable on disk if retire is true |
967 | */ |
968 | void __fscache_relinquish_cookie(struct fscache_cookie *cookie, bool retire) |
969 | { |
970 | fscache_stat(stat: &fscache_n_relinquishes); |
971 | if (retire) |
972 | fscache_stat(stat: &fscache_n_relinquishes_retire); |
973 | |
974 | _enter("c=%08x{%d},%d" , |
975 | cookie->debug_id, atomic_read(&cookie->n_active), retire); |
976 | |
977 | if (WARN(test_and_set_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags), |
978 | "Cookie c=%x already relinquished\n" , cookie->debug_id)) |
979 | return; |
980 | |
981 | if (retire) |
982 | set_bit(FSCACHE_COOKIE_RETIRED, addr: &cookie->flags); |
983 | trace_fscache_relinquish(cookie, retire); |
984 | |
985 | ASSERTCMP(atomic_read(&cookie->n_active), ==, 0); |
986 | ASSERTCMP(atomic_read(&cookie->volume->n_cookies), >, 0); |
987 | atomic_dec(v: &cookie->volume->n_cookies); |
988 | |
989 | if (test_bit(FSCACHE_COOKIE_HAS_BEEN_CACHED, &cookie->flags)) { |
990 | set_bit(FSCACHE_COOKIE_DO_RELINQUISH, addr: &cookie->flags); |
991 | fscache_drop_withdraw_cookie(cookie); |
992 | } else { |
993 | fscache_set_cookie_state(cookie, state: FSCACHE_COOKIE_STATE_DROPPED); |
994 | fscache_unhash_cookie(cookie); |
995 | } |
996 | fscache_put_cookie(cookie, where: fscache_cookie_put_relinquish); |
997 | } |
998 | EXPORT_SYMBOL(__fscache_relinquish_cookie); |
999 | |
1000 | /* |
1001 | * Drop a reference to a cookie. |
1002 | */ |
1003 | void fscache_put_cookie(struct fscache_cookie *cookie, |
1004 | enum fscache_cookie_trace where) |
1005 | { |
1006 | struct fscache_volume *volume = cookie->volume; |
1007 | unsigned int cookie_debug_id = cookie->debug_id; |
1008 | bool zero; |
1009 | int ref; |
1010 | |
1011 | zero = __refcount_dec_and_test(r: &cookie->ref, oldp: &ref); |
1012 | trace_fscache_cookie(cookie_debug_id, ref: ref - 1, where); |
1013 | if (zero) { |
1014 | fscache_free_cookie(cookie); |
1015 | fscache_put_volume(volume, where: fscache_volume_put_cookie); |
1016 | } |
1017 | } |
1018 | EXPORT_SYMBOL(fscache_put_cookie); |
1019 | |
1020 | /* |
1021 | * Get a reference to a cookie. |
1022 | */ |
1023 | struct fscache_cookie *fscache_get_cookie(struct fscache_cookie *cookie, |
1024 | enum fscache_cookie_trace where) |
1025 | { |
1026 | int ref; |
1027 | |
1028 | __refcount_inc(r: &cookie->ref, oldp: &ref); |
1029 | trace_fscache_cookie(cookie_debug_id: cookie->debug_id, ref: ref + 1, where); |
1030 | return cookie; |
1031 | } |
1032 | EXPORT_SYMBOL(fscache_get_cookie); |
1033 | |
1034 | /* |
1035 | * Ask the cache to effect invalidation of a cookie. |
1036 | */ |
1037 | static void fscache_perform_invalidation(struct fscache_cookie *cookie) |
1038 | { |
1039 | if (!cookie->volume->cache->ops->invalidate_cookie(cookie)) |
1040 | fscache_caching_failed(cookie); |
1041 | fscache_end_cookie_access(cookie, fscache_access_invalidate_cookie_end); |
1042 | } |
1043 | |
1044 | /* |
1045 | * Invalidate an object. |
1046 | */ |
1047 | void __fscache_invalidate(struct fscache_cookie *cookie, |
1048 | const void *aux_data, loff_t new_size, |
1049 | unsigned int flags) |
1050 | { |
1051 | bool is_caching; |
1052 | |
1053 | _enter("c=%x" , cookie->debug_id); |
1054 | |
1055 | fscache_stat(stat: &fscache_n_invalidates); |
1056 | |
1057 | if (WARN(test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags), |
1058 | "Trying to invalidate relinquished cookie\n" )) |
1059 | return; |
1060 | |
1061 | if ((flags & FSCACHE_INVAL_DIO_WRITE) && |
1062 | test_and_set_bit(FSCACHE_COOKIE_DISABLED, addr: &cookie->flags)) |
1063 | return; |
1064 | |
1065 | spin_lock(lock: &cookie->lock); |
1066 | set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, addr: &cookie->flags); |
1067 | fscache_update_aux(cookie, aux_data, object_size: &new_size); |
1068 | cookie->inval_counter++; |
1069 | trace_fscache_invalidate(cookie, new_size); |
1070 | |
1071 | switch (cookie->state) { |
1072 | case FSCACHE_COOKIE_STATE_INVALIDATING: /* is_still_valid will catch it */ |
1073 | default: |
1074 | spin_unlock(lock: &cookie->lock); |
1075 | _leave(" [no %u]" , cookie->state); |
1076 | return; |
1077 | |
1078 | case FSCACHE_COOKIE_STATE_LOOKING_UP: |
1079 | if (!test_and_set_bit(FSCACHE_COOKIE_DO_INVALIDATE, addr: &cookie->flags)) |
1080 | __fscache_begin_cookie_access(cookie, why: fscache_access_invalidate_cookie); |
1081 | fallthrough; |
1082 | case FSCACHE_COOKIE_STATE_CREATING: |
1083 | spin_unlock(lock: &cookie->lock); |
1084 | _leave(" [look %x]" , cookie->inval_counter); |
1085 | return; |
1086 | |
1087 | case FSCACHE_COOKIE_STATE_ACTIVE: |
1088 | is_caching = fscache_begin_cookie_access( |
1089 | cookie, why: fscache_access_invalidate_cookie); |
1090 | if (is_caching) |
1091 | __fscache_set_cookie_state(cookie, state: FSCACHE_COOKIE_STATE_INVALIDATING); |
1092 | spin_unlock(lock: &cookie->lock); |
1093 | wake_up_cookie_state(cookie); |
1094 | |
1095 | if (is_caching) |
1096 | fscache_queue_cookie(cookie, where: fscache_cookie_get_inval_work); |
1097 | _leave(" [inv]" ); |
1098 | return; |
1099 | } |
1100 | } |
1101 | EXPORT_SYMBOL(__fscache_invalidate); |
1102 | |
1103 | #ifdef CONFIG_PROC_FS |
1104 | /* |
1105 | * Generate a list of extant cookies in /proc/fs/fscache/cookies |
1106 | */ |
1107 | static int fscache_cookies_seq_show(struct seq_file *m, void *v) |
1108 | { |
1109 | struct fscache_cookie *cookie; |
1110 | unsigned int keylen = 0, auxlen = 0; |
1111 | u8 *p; |
1112 | |
1113 | if (v == &fscache_cookies) { |
1114 | seq_puts(m, |
1115 | s: "COOKIE VOLUME REF ACT ACC S FL DEF \n" |
1116 | "======== ======== === === === = == ================\n" |
1117 | ); |
1118 | return 0; |
1119 | } |
1120 | |
1121 | cookie = list_entry(v, struct fscache_cookie, proc_link); |
1122 | |
1123 | seq_printf(m, |
1124 | fmt: "%08x %08x %3d %3d %3d %c %02lx" , |
1125 | cookie->debug_id, |
1126 | cookie->volume->debug_id, |
1127 | refcount_read(r: &cookie->ref), |
1128 | atomic_read(v: &cookie->n_active), |
1129 | atomic_read(v: &cookie->n_accesses), |
1130 | fscache_cookie_states[cookie->state], |
1131 | cookie->flags); |
1132 | |
1133 | keylen = cookie->key_len; |
1134 | auxlen = cookie->aux_len; |
1135 | |
1136 | if (keylen > 0 || auxlen > 0) { |
1137 | seq_puts(m, s: " " ); |
1138 | p = keylen <= sizeof(cookie->inline_key) ? |
1139 | cookie->inline_key : cookie->key; |
1140 | for (; keylen > 0; keylen--) |
1141 | seq_printf(m, fmt: "%02x" , *p++); |
1142 | if (auxlen > 0) { |
1143 | seq_puts(m, s: ", " ); |
1144 | p = auxlen <= sizeof(cookie->inline_aux) ? |
1145 | cookie->inline_aux : cookie->aux; |
1146 | for (; auxlen > 0; auxlen--) |
1147 | seq_printf(m, fmt: "%02x" , *p++); |
1148 | } |
1149 | } |
1150 | |
1151 | seq_puts(m, s: "\n" ); |
1152 | return 0; |
1153 | } |
1154 | |
1155 | static void *fscache_cookies_seq_start(struct seq_file *m, loff_t *_pos) |
1156 | __acquires(fscache_cookies_lock) |
1157 | { |
1158 | read_lock(&fscache_cookies_lock); |
1159 | return seq_list_start_head(head: &fscache_cookies, pos: *_pos); |
1160 | } |
1161 | |
1162 | static void *fscache_cookies_seq_next(struct seq_file *m, void *v, loff_t *_pos) |
1163 | { |
1164 | return seq_list_next(v, head: &fscache_cookies, ppos: _pos); |
1165 | } |
1166 | |
1167 | static void fscache_cookies_seq_stop(struct seq_file *m, void *v) |
1168 | __releases(rcu) |
1169 | { |
1170 | read_unlock(&fscache_cookies_lock); |
1171 | } |
1172 | |
1173 | |
1174 | const struct seq_operations fscache_cookies_seq_ops = { |
1175 | .start = fscache_cookies_seq_start, |
1176 | .next = fscache_cookies_seq_next, |
1177 | .stop = fscache_cookies_seq_stop, |
1178 | .show = fscache_cookies_seq_show, |
1179 | }; |
1180 | #endif |
1181 | |