1 | // SPDX-License-Identifier: GPL-2.0 |
2 | |
3 | #include "bcachefs.h" |
4 | #include "bkey_methods.h" |
5 | #include "nocow_locking.h" |
6 | #include "util.h" |
7 | |
8 | #include <linux/closure.h> |
9 | |
10 | bool bch2_bucket_nocow_is_locked(struct bucket_nocow_lock_table *t, struct bpos bucket) |
11 | { |
12 | u64 dev_bucket = bucket_to_u64(bucket); |
13 | struct nocow_lock_bucket *l = bucket_nocow_lock(t, dev_bucket); |
14 | unsigned i; |
15 | |
16 | for (i = 0; i < ARRAY_SIZE(l->b); i++) |
17 | if (l->b[i] == dev_bucket && atomic_read(v: &l->l[i])) |
18 | return true; |
19 | return false; |
20 | } |
21 | |
22 | #define sign(v) (v < 0 ? -1 : v > 0 ? 1 : 0) |
23 | |
24 | void bch2_bucket_nocow_unlock(struct bucket_nocow_lock_table *t, struct bpos bucket, int flags) |
25 | { |
26 | u64 dev_bucket = bucket_to_u64(bucket); |
27 | struct nocow_lock_bucket *l = bucket_nocow_lock(t, dev_bucket); |
28 | int lock_val = flags ? 1 : -1; |
29 | unsigned i; |
30 | |
31 | for (i = 0; i < ARRAY_SIZE(l->b); i++) |
32 | if (l->b[i] == dev_bucket) { |
33 | int v = atomic_sub_return(i: lock_val, v: &l->l[i]); |
34 | |
35 | BUG_ON(v && sign(v) != lock_val); |
36 | if (!v) |
37 | closure_wake_up(list: &l->wait); |
38 | return; |
39 | } |
40 | |
41 | BUG(); |
42 | } |
43 | |
44 | bool __bch2_bucket_nocow_trylock(struct nocow_lock_bucket *l, |
45 | u64 dev_bucket, int flags) |
46 | { |
47 | int v, lock_val = flags ? 1 : -1; |
48 | unsigned i; |
49 | |
50 | spin_lock(lock: &l->lock); |
51 | |
52 | for (i = 0; i < ARRAY_SIZE(l->b); i++) |
53 | if (l->b[i] == dev_bucket) |
54 | goto got_entry; |
55 | |
56 | for (i = 0; i < ARRAY_SIZE(l->b); i++) |
57 | if (!atomic_read(v: &l->l[i])) { |
58 | l->b[i] = dev_bucket; |
59 | goto take_lock; |
60 | } |
61 | fail: |
62 | spin_unlock(lock: &l->lock); |
63 | return false; |
64 | got_entry: |
65 | v = atomic_read(v: &l->l[i]); |
66 | if (lock_val > 0 ? v < 0 : v > 0) |
67 | goto fail; |
68 | take_lock: |
69 | v = atomic_read(v: &l->l[i]); |
70 | /* Overflow? */ |
71 | if (v && sign(v + lock_val) != sign(v)) |
72 | goto fail; |
73 | |
74 | atomic_add(i: lock_val, v: &l->l[i]); |
75 | spin_unlock(lock: &l->lock); |
76 | return true; |
77 | } |
78 | |
79 | void __bch2_bucket_nocow_lock(struct bucket_nocow_lock_table *t, |
80 | struct nocow_lock_bucket *l, |
81 | u64 dev_bucket, int flags) |
82 | { |
83 | if (!__bch2_bucket_nocow_trylock(l, dev_bucket, flags)) { |
84 | struct bch_fs *c = container_of(t, struct bch_fs, nocow_locks); |
85 | u64 start_time = local_clock(); |
86 | |
87 | __closure_wait_event(&l->wait, __bch2_bucket_nocow_trylock(l, dev_bucket, flags)); |
88 | bch2_time_stats_update(stats: &c->times[BCH_TIME_nocow_lock_contended], start: start_time); |
89 | } |
90 | } |
91 | |
92 | void bch2_nocow_locks_to_text(struct printbuf *out, struct bucket_nocow_lock_table *t) |
93 | |
94 | { |
95 | unsigned i, nr_zero = 0; |
96 | struct nocow_lock_bucket *l; |
97 | |
98 | for (l = t->l; l < t->l + ARRAY_SIZE(t->l); l++) { |
99 | unsigned v = 0; |
100 | |
101 | for (i = 0; i < ARRAY_SIZE(l->l); i++) |
102 | v |= atomic_read(v: &l->l[i]); |
103 | |
104 | if (!v) { |
105 | nr_zero++; |
106 | continue; |
107 | } |
108 | |
109 | if (nr_zero) |
110 | prt_printf(out, "(%u empty entries)\n" , nr_zero); |
111 | nr_zero = 0; |
112 | |
113 | for (i = 0; i < ARRAY_SIZE(l->l); i++) { |
114 | int v = atomic_read(v: &l->l[i]); |
115 | if (v) { |
116 | bch2_bpos_to_text(out, u64_to_bucket(bucket: l->b[i])); |
117 | prt_printf(out, ": %s %u " , v < 0 ? "copy" : "update" , abs(v)); |
118 | } |
119 | } |
120 | prt_newline(out); |
121 | } |
122 | |
123 | if (nr_zero) |
124 | prt_printf(out, "(%u empty entries)\n" , nr_zero); |
125 | } |
126 | |
127 | void bch2_fs_nocow_locking_exit(struct bch_fs *c) |
128 | { |
129 | struct bucket_nocow_lock_table *t = &c->nocow_locks; |
130 | |
131 | for (struct nocow_lock_bucket *l = t->l; l < t->l + ARRAY_SIZE(t->l); l++) |
132 | for (unsigned j = 0; j < ARRAY_SIZE(l->l); j++) |
133 | BUG_ON(atomic_read(&l->l[j])); |
134 | } |
135 | |
136 | int bch2_fs_nocow_locking_init(struct bch_fs *c) |
137 | { |
138 | struct bucket_nocow_lock_table *t = &c->nocow_locks; |
139 | |
140 | for (struct nocow_lock_bucket *l = t->l; l < t->l + ARRAY_SIZE(t->l); l++) |
141 | spin_lock_init(&l->lock); |
142 | |
143 | return 0; |
144 | } |
145 | |