1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #ifndef _BCACHEFS_ALLOC_BACKGROUND_H |
3 | #define _BCACHEFS_ALLOC_BACKGROUND_H |
4 | |
5 | #include "bcachefs.h" |
6 | #include "alloc_types.h" |
7 | #include "buckets.h" |
8 | #include "debug.h" |
9 | #include "super.h" |
10 | |
11 | enum bkey_invalid_flags; |
12 | |
13 | /* How out of date a pointer gen is allowed to be: */ |
14 | #define BUCKET_GC_GEN_MAX 96U |
15 | |
16 | static inline bool bch2_dev_bucket_exists(struct bch_fs *c, struct bpos pos) |
17 | { |
18 | struct bch_dev *ca; |
19 | |
20 | if (!bch2_dev_exists2(c, dev: pos.inode)) |
21 | return false; |
22 | |
23 | ca = bch_dev_bkey_exists(c, idx: pos.inode); |
24 | return pos.offset >= ca->mi.first_bucket && |
25 | pos.offset < ca->mi.nbuckets; |
26 | } |
27 | |
28 | static inline u64 bucket_to_u64(struct bpos bucket) |
29 | { |
30 | return (bucket.inode << 48) | bucket.offset; |
31 | } |
32 | |
33 | static inline struct bpos u64_to_bucket(u64 bucket) |
34 | { |
35 | return POS(bucket >> 48, bucket & ~(~0ULL << 48)); |
36 | } |
37 | |
38 | static inline u8 alloc_gc_gen(struct bch_alloc_v4 a) |
39 | { |
40 | return a.gen - a.oldest_gen; |
41 | } |
42 | |
43 | static inline enum bch_data_type __alloc_data_type(u32 dirty_sectors, |
44 | u32 cached_sectors, |
45 | u32 stripe, |
46 | struct bch_alloc_v4 a, |
47 | enum bch_data_type data_type) |
48 | { |
49 | if (stripe) |
50 | return data_type == BCH_DATA_parity ? data_type : BCH_DATA_stripe; |
51 | if (dirty_sectors) |
52 | return data_type; |
53 | if (cached_sectors) |
54 | return BCH_DATA_cached; |
55 | if (BCH_ALLOC_V4_NEED_DISCARD(k: &a)) |
56 | return BCH_DATA_need_discard; |
57 | if (alloc_gc_gen(a) >= BUCKET_GC_GEN_MAX) |
58 | return BCH_DATA_need_gc_gens; |
59 | return BCH_DATA_free; |
60 | } |
61 | |
62 | static inline enum bch_data_type alloc_data_type(struct bch_alloc_v4 a, |
63 | enum bch_data_type data_type) |
64 | { |
65 | return __alloc_data_type(dirty_sectors: a.dirty_sectors, cached_sectors: a.cached_sectors, |
66 | stripe: a.stripe, a, data_type); |
67 | } |
68 | |
69 | static inline enum bch_data_type bucket_data_type(enum bch_data_type data_type) |
70 | { |
71 | return data_type == BCH_DATA_stripe ? BCH_DATA_user : data_type; |
72 | } |
73 | |
74 | static inline unsigned bch2_bucket_sectors(struct bch_alloc_v4 a) |
75 | { |
76 | return a.dirty_sectors + a.cached_sectors; |
77 | } |
78 | |
79 | static inline unsigned bch2_bucket_sectors_dirty(struct bch_alloc_v4 a) |
80 | { |
81 | return a.dirty_sectors; |
82 | } |
83 | |
84 | static inline unsigned bch2_bucket_sectors_fragmented(struct bch_dev *ca, |
85 | struct bch_alloc_v4 a) |
86 | { |
87 | int d = bch2_bucket_sectors_dirty(a); |
88 | |
89 | return d ? max(0, ca->mi.bucket_size - d) : 0; |
90 | } |
91 | |
92 | static inline u64 alloc_lru_idx_read(struct bch_alloc_v4 a) |
93 | { |
94 | return a.data_type == BCH_DATA_cached ? a.io_time[READ] : 0; |
95 | } |
96 | |
97 | #define DATA_TYPES_MOVABLE \ |
98 | ((1U << BCH_DATA_btree)| \ |
99 | (1U << BCH_DATA_user)| \ |
100 | (1U << BCH_DATA_stripe)) |
101 | |
102 | static inline bool data_type_movable(enum bch_data_type type) |
103 | { |
104 | return (1U << type) & DATA_TYPES_MOVABLE; |
105 | } |
106 | |
107 | static inline u64 alloc_lru_idx_fragmentation(struct bch_alloc_v4 a, |
108 | struct bch_dev *ca) |
109 | { |
110 | if (!data_type_movable(type: a.data_type) || |
111 | !bch2_bucket_sectors_fragmented(ca, a)) |
112 | return 0; |
113 | |
114 | u64 d = bch2_bucket_sectors_dirty(a); |
115 | return div_u64(dividend: d * (1ULL << 31), divisor: ca->mi.bucket_size); |
116 | } |
117 | |
118 | static inline u64 alloc_freespace_genbits(struct bch_alloc_v4 a) |
119 | { |
120 | return ((u64) alloc_gc_gen(a) >> 4) << 56; |
121 | } |
122 | |
123 | static inline struct bpos alloc_freespace_pos(struct bpos pos, struct bch_alloc_v4 a) |
124 | { |
125 | pos.offset |= alloc_freespace_genbits(a); |
126 | return pos; |
127 | } |
128 | |
129 | static inline unsigned alloc_v4_u64s(const struct bch_alloc_v4 *a) |
130 | { |
131 | unsigned ret = (BCH_ALLOC_V4_BACKPOINTERS_START(k: a) ?: |
132 | BCH_ALLOC_V4_U64s_V0) + |
133 | BCH_ALLOC_V4_NR_BACKPOINTERS(k: a) * |
134 | (sizeof(struct bch_backpointer) / sizeof(u64)); |
135 | |
136 | BUG_ON(ret > U8_MAX - BKEY_U64s); |
137 | return ret; |
138 | } |
139 | |
140 | static inline void set_alloc_v4_u64s(struct bkey_i_alloc_v4 *a) |
141 | { |
142 | set_bkey_val_u64s(k: &a->k, val_u64s: alloc_v4_u64s(a: &a->v)); |
143 | } |
144 | |
145 | struct bkey_i_alloc_v4 * |
146 | bch2_trans_start_alloc_update(struct btree_trans *, struct btree_iter *, struct bpos); |
147 | |
148 | void __bch2_alloc_to_v4(struct bkey_s_c, struct bch_alloc_v4 *); |
149 | |
150 | static inline const struct bch_alloc_v4 *bch2_alloc_to_v4(struct bkey_s_c k, struct bch_alloc_v4 *convert) |
151 | { |
152 | const struct bch_alloc_v4 *ret; |
153 | |
154 | if (unlikely(k.k->type != KEY_TYPE_alloc_v4)) |
155 | goto slowpath; |
156 | |
157 | ret = bkey_s_c_to_alloc_v4(k).v; |
158 | if (BCH_ALLOC_V4_BACKPOINTERS_START(k: ret) != BCH_ALLOC_V4_U64s) |
159 | goto slowpath; |
160 | |
161 | return ret; |
162 | slowpath: |
163 | __bch2_alloc_to_v4(k, convert); |
164 | return convert; |
165 | } |
166 | |
167 | struct bkey_i_alloc_v4 *bch2_alloc_to_v4_mut(struct btree_trans *, struct bkey_s_c); |
168 | |
169 | int bch2_bucket_io_time_reset(struct btree_trans *, unsigned, size_t, int); |
170 | |
171 | int bch2_alloc_v1_invalid(struct bch_fs *, struct bkey_s_c, |
172 | enum bkey_invalid_flags, struct printbuf *); |
173 | int bch2_alloc_v2_invalid(struct bch_fs *, struct bkey_s_c, |
174 | enum bkey_invalid_flags, struct printbuf *); |
175 | int bch2_alloc_v3_invalid(struct bch_fs *, struct bkey_s_c, |
176 | enum bkey_invalid_flags, struct printbuf *); |
177 | int bch2_alloc_v4_invalid(struct bch_fs *, struct bkey_s_c, |
178 | enum bkey_invalid_flags, struct printbuf *); |
179 | void bch2_alloc_v4_swab(struct bkey_s); |
180 | void bch2_alloc_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c); |
181 | |
182 | #define bch2_bkey_ops_alloc ((struct bkey_ops) { \ |
183 | .key_invalid = bch2_alloc_v1_invalid, \ |
184 | .val_to_text = bch2_alloc_to_text, \ |
185 | .trigger = bch2_trigger_alloc, \ |
186 | .min_val_size = 8, \ |
187 | }) |
188 | |
189 | #define bch2_bkey_ops_alloc_v2 ((struct bkey_ops) { \ |
190 | .key_invalid = bch2_alloc_v2_invalid, \ |
191 | .val_to_text = bch2_alloc_to_text, \ |
192 | .trigger = bch2_trigger_alloc, \ |
193 | .min_val_size = 8, \ |
194 | }) |
195 | |
196 | #define bch2_bkey_ops_alloc_v3 ((struct bkey_ops) { \ |
197 | .key_invalid = bch2_alloc_v3_invalid, \ |
198 | .val_to_text = bch2_alloc_to_text, \ |
199 | .trigger = bch2_trigger_alloc, \ |
200 | .min_val_size = 16, \ |
201 | }) |
202 | |
203 | #define bch2_bkey_ops_alloc_v4 ((struct bkey_ops) { \ |
204 | .key_invalid = bch2_alloc_v4_invalid, \ |
205 | .val_to_text = bch2_alloc_to_text, \ |
206 | .swab = bch2_alloc_v4_swab, \ |
207 | .trigger = bch2_trigger_alloc, \ |
208 | .min_val_size = 48, \ |
209 | }) |
210 | |
211 | int bch2_bucket_gens_invalid(struct bch_fs *, struct bkey_s_c, |
212 | enum bkey_invalid_flags, struct printbuf *); |
213 | void bch2_bucket_gens_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c); |
214 | |
215 | #define bch2_bkey_ops_bucket_gens ((struct bkey_ops) { \ |
216 | .key_invalid = bch2_bucket_gens_invalid, \ |
217 | .val_to_text = bch2_bucket_gens_to_text, \ |
218 | }) |
219 | |
220 | int bch2_bucket_gens_init(struct bch_fs *); |
221 | |
222 | static inline bool bkey_is_alloc(const struct bkey *k) |
223 | { |
224 | return k->type == KEY_TYPE_alloc || |
225 | k->type == KEY_TYPE_alloc_v2 || |
226 | k->type == KEY_TYPE_alloc_v3; |
227 | } |
228 | |
229 | int bch2_alloc_read(struct bch_fs *); |
230 | |
231 | int bch2_trigger_alloc(struct btree_trans *, enum btree_id, unsigned, |
232 | struct bkey_s_c, struct bkey_s, unsigned); |
233 | int bch2_check_alloc_info(struct bch_fs *); |
234 | int bch2_check_alloc_to_lru_refs(struct bch_fs *); |
235 | void bch2_do_discards(struct bch_fs *); |
236 | |
237 | static inline u64 should_invalidate_buckets(struct bch_dev *ca, |
238 | struct bch_dev_usage u) |
239 | { |
240 | u64 want_free = ca->mi.nbuckets >> 7; |
241 | u64 free = max_t(s64, 0, |
242 | u.d[BCH_DATA_free].buckets |
243 | + u.d[BCH_DATA_need_discard].buckets |
244 | - bch2_dev_buckets_reserved(ca, BCH_WATERMARK_stripe)); |
245 | |
246 | return clamp_t(s64, want_free - free, 0, u.d[BCH_DATA_cached].buckets); |
247 | } |
248 | |
249 | void bch2_do_invalidates(struct bch_fs *); |
250 | |
251 | static inline struct bch_backpointer *alloc_v4_backpointers(struct bch_alloc_v4 *a) |
252 | { |
253 | return (void *) ((u64 *) &a->v + |
254 | (BCH_ALLOC_V4_BACKPOINTERS_START(k: a) ?: |
255 | BCH_ALLOC_V4_U64s_V0)); |
256 | } |
257 | |
258 | static inline const struct bch_backpointer *alloc_v4_backpointers_c(const struct bch_alloc_v4 *a) |
259 | { |
260 | return (void *) ((u64 *) &a->v + BCH_ALLOC_V4_BACKPOINTERS_START(k: a)); |
261 | } |
262 | |
263 | int bch2_dev_freespace_init(struct bch_fs *, struct bch_dev *, u64, u64); |
264 | int bch2_fs_freespace_init(struct bch_fs *); |
265 | |
266 | void bch2_recalc_capacity(struct bch_fs *); |
267 | u64 bch2_min_rw_member_capacity(struct bch_fs *); |
268 | |
269 | void bch2_dev_allocator_remove(struct bch_fs *, struct bch_dev *); |
270 | void bch2_dev_allocator_add(struct bch_fs *, struct bch_dev *); |
271 | |
272 | void bch2_fs_allocator_background_exit(struct bch_fs *); |
273 | void bch2_fs_allocator_background_init(struct bch_fs *); |
274 | |
275 | #endif /* _BCACHEFS_ALLOC_BACKGROUND_H */ |
276 | |