1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #ifndef _BCACHEFS_ALLOC_FOREGROUND_H |
3 | #define _BCACHEFS_ALLOC_FOREGROUND_H |
4 | |
5 | #include "bcachefs.h" |
6 | #include "alloc_types.h" |
7 | #include "extents.h" |
8 | #include "sb-members.h" |
9 | |
10 | #include <linux/hash.h> |
11 | |
12 | struct bkey; |
13 | struct bch_dev; |
14 | struct bch_fs; |
15 | struct bch_devs_List; |
16 | |
17 | extern const char * const bch2_watermarks[]; |
18 | |
19 | void bch2_reset_alloc_cursors(struct bch_fs *); |
20 | |
21 | struct dev_alloc_list { |
22 | unsigned nr; |
23 | u8 devs[BCH_SB_MEMBERS_MAX]; |
24 | }; |
25 | |
26 | struct dev_alloc_list bch2_dev_alloc_list(struct bch_fs *, |
27 | struct dev_stripe_state *, |
28 | struct bch_devs_mask *); |
29 | void bch2_dev_stripe_increment(struct bch_dev *, struct dev_stripe_state *); |
30 | |
31 | long bch2_bucket_alloc_new_fs(struct bch_dev *); |
32 | |
33 | struct open_bucket *bch2_bucket_alloc(struct bch_fs *, struct bch_dev *, |
34 | enum bch_watermark, struct closure *); |
35 | |
36 | static inline void ob_push(struct bch_fs *c, struct open_buckets *obs, |
37 | struct open_bucket *ob) |
38 | { |
39 | BUG_ON(obs->nr >= ARRAY_SIZE(obs->v)); |
40 | |
41 | obs->v[obs->nr++] = ob - c->open_buckets; |
42 | } |
43 | |
44 | #define open_bucket_for_each(_c, _obs, _ob, _i) \ |
45 | for ((_i) = 0; \ |
46 | (_i) < (_obs)->nr && \ |
47 | ((_ob) = (_c)->open_buckets + (_obs)->v[_i], true); \ |
48 | (_i)++) |
49 | |
50 | static inline struct open_bucket *ec_open_bucket(struct bch_fs *c, |
51 | struct open_buckets *obs) |
52 | { |
53 | struct open_bucket *ob; |
54 | unsigned i; |
55 | |
56 | open_bucket_for_each(c, obs, ob, i) |
57 | if (ob->ec) |
58 | return ob; |
59 | |
60 | return NULL; |
61 | } |
62 | |
63 | void bch2_open_bucket_write_error(struct bch_fs *, |
64 | struct open_buckets *, unsigned); |
65 | |
66 | void __bch2_open_bucket_put(struct bch_fs *, struct open_bucket *); |
67 | |
68 | static inline void bch2_open_bucket_put(struct bch_fs *c, struct open_bucket *ob) |
69 | { |
70 | if (atomic_dec_and_test(v: &ob->pin)) |
71 | __bch2_open_bucket_put(c, ob); |
72 | } |
73 | |
74 | static inline void bch2_open_buckets_put(struct bch_fs *c, |
75 | struct open_buckets *ptrs) |
76 | { |
77 | struct open_bucket *ob; |
78 | unsigned i; |
79 | |
80 | open_bucket_for_each(c, ptrs, ob, i) |
81 | bch2_open_bucket_put(c, ob); |
82 | ptrs->nr = 0; |
83 | } |
84 | |
85 | static inline void bch2_alloc_sectors_done_inlined(struct bch_fs *c, struct write_point *wp) |
86 | { |
87 | struct open_buckets ptrs = { .nr = 0 }, keep = { .nr = 0 }; |
88 | struct open_bucket *ob; |
89 | unsigned i; |
90 | |
91 | open_bucket_for_each(c, &wp->ptrs, ob, i) |
92 | ob_push(c, obs: !ob->sectors_free ? &ptrs : &keep, ob); |
93 | wp->ptrs = keep; |
94 | |
95 | mutex_unlock(lock: &wp->lock); |
96 | |
97 | bch2_open_buckets_put(c, ptrs: &ptrs); |
98 | } |
99 | |
100 | static inline void bch2_open_bucket_get(struct bch_fs *c, |
101 | struct write_point *wp, |
102 | struct open_buckets *ptrs) |
103 | { |
104 | struct open_bucket *ob; |
105 | unsigned i; |
106 | |
107 | open_bucket_for_each(c, &wp->ptrs, ob, i) { |
108 | ob->data_type = wp->data_type; |
109 | atomic_inc(v: &ob->pin); |
110 | ob_push(c, obs: ptrs, ob); |
111 | } |
112 | } |
113 | |
114 | static inline open_bucket_idx_t *open_bucket_hashslot(struct bch_fs *c, |
115 | unsigned dev, u64 bucket) |
116 | { |
117 | return c->open_buckets_hash + |
118 | (jhash_3words(a: dev, b: bucket, c: bucket >> 32, initval: 0) & |
119 | (OPEN_BUCKETS_COUNT - 1)); |
120 | } |
121 | |
122 | static inline bool bch2_bucket_is_open(struct bch_fs *c, unsigned dev, u64 bucket) |
123 | { |
124 | open_bucket_idx_t slot = *open_bucket_hashslot(c, dev, bucket); |
125 | |
126 | while (slot) { |
127 | struct open_bucket *ob = &c->open_buckets[slot]; |
128 | |
129 | if (ob->dev == dev && ob->bucket == bucket) |
130 | return true; |
131 | |
132 | slot = ob->hash; |
133 | } |
134 | |
135 | return false; |
136 | } |
137 | |
138 | static inline bool bch2_bucket_is_open_safe(struct bch_fs *c, unsigned dev, u64 bucket) |
139 | { |
140 | bool ret; |
141 | |
142 | if (bch2_bucket_is_open(c, dev, bucket)) |
143 | return true; |
144 | |
145 | spin_lock(lock: &c->freelist_lock); |
146 | ret = bch2_bucket_is_open(c, dev, bucket); |
147 | spin_unlock(lock: &c->freelist_lock); |
148 | |
149 | return ret; |
150 | } |
151 | |
152 | int bch2_bucket_alloc_set_trans(struct btree_trans *, struct open_buckets *, |
153 | struct dev_stripe_state *, struct bch_devs_mask *, |
154 | unsigned, unsigned *, bool *, unsigned, |
155 | enum bch_data_type, enum bch_watermark, |
156 | struct closure *); |
157 | |
158 | int bch2_alloc_sectors_start_trans(struct btree_trans *, |
159 | unsigned, unsigned, |
160 | struct write_point_specifier, |
161 | struct bch_devs_list *, |
162 | unsigned, unsigned, |
163 | enum bch_watermark, |
164 | unsigned, |
165 | struct closure *, |
166 | struct write_point **); |
167 | |
168 | struct bch_extent_ptr bch2_ob_ptr(struct bch_fs *, struct open_bucket *); |
169 | |
170 | /* |
171 | * Append pointers to the space we just allocated to @k, and mark @sectors space |
172 | * as allocated out of @ob |
173 | */ |
174 | static inline void |
175 | bch2_alloc_sectors_append_ptrs_inlined(struct bch_fs *c, struct write_point *wp, |
176 | struct bkey_i *k, unsigned sectors, |
177 | bool cached) |
178 | { |
179 | struct open_bucket *ob; |
180 | unsigned i; |
181 | |
182 | BUG_ON(sectors > wp->sectors_free); |
183 | wp->sectors_free -= sectors; |
184 | wp->sectors_allocated += sectors; |
185 | |
186 | open_bucket_for_each(c, &wp->ptrs, ob, i) { |
187 | struct bch_dev *ca = bch_dev_bkey_exists(c, idx: ob->dev); |
188 | struct bch_extent_ptr ptr = bch2_ob_ptr(c, ob); |
189 | |
190 | ptr.cached = cached || |
191 | (!ca->mi.durability && |
192 | wp->data_type == BCH_DATA_user); |
193 | |
194 | bch2_bkey_append_ptr(k, ptr); |
195 | |
196 | BUG_ON(sectors > ob->sectors_free); |
197 | ob->sectors_free -= sectors; |
198 | } |
199 | } |
200 | |
201 | void bch2_alloc_sectors_append_ptrs(struct bch_fs *, struct write_point *, |
202 | struct bkey_i *, unsigned, bool); |
203 | void bch2_alloc_sectors_done(struct bch_fs *, struct write_point *); |
204 | |
205 | void bch2_open_buckets_stop(struct bch_fs *c, struct bch_dev *, bool); |
206 | |
207 | static inline struct write_point_specifier writepoint_hashed(unsigned long v) |
208 | { |
209 | return (struct write_point_specifier) { .v = v | 1 }; |
210 | } |
211 | |
212 | static inline struct write_point_specifier writepoint_ptr(struct write_point *wp) |
213 | { |
214 | return (struct write_point_specifier) { .v = (unsigned long) wp }; |
215 | } |
216 | |
217 | void bch2_fs_allocator_foreground_init(struct bch_fs *); |
218 | |
219 | void bch2_open_buckets_to_text(struct printbuf *, struct bch_fs *); |
220 | void bch2_open_buckets_partial_to_text(struct printbuf *, struct bch_fs *); |
221 | |
222 | void bch2_write_points_to_text(struct printbuf *, struct bch_fs *); |
223 | |
224 | #endif /* _BCACHEFS_ALLOC_FOREGROUND_H */ |
225 | |