1 | // SPDX-License-Identifier: GPL-2.0 |
2 | #include "bcachefs.h" |
3 | #include "alloc_background.h" |
4 | #include "alloc_foreground.h" |
5 | #include "btree_io.h" |
6 | #include "btree_update_interior.h" |
7 | #include "btree_write_buffer.h" |
8 | #include "buckets.h" |
9 | #include "checksum.h" |
10 | #include "disk_groups.h" |
11 | #include "error.h" |
12 | #include "journal.h" |
13 | #include "journal_io.h" |
14 | #include "journal_reclaim.h" |
15 | #include "journal_seq_blacklist.h" |
16 | #include "replicas.h" |
17 | #include "sb-clean.h" |
18 | #include "trace.h" |
19 | |
20 | void bch2_journal_ptrs_to_text(struct printbuf *out, struct bch_fs *c, |
21 | struct journal_replay *j) |
22 | { |
23 | darray_for_each(j->ptrs, i) { |
24 | struct bch_dev *ca = bch_dev_bkey_exists(c, idx: i->dev); |
25 | u64 offset; |
26 | |
27 | div64_u64_rem(dividend: i->sector, divisor: ca->mi.bucket_size, remainder: &offset); |
28 | |
29 | if (i != j->ptrs.data) |
30 | prt_printf(out, " " ); |
31 | prt_printf(out, "%u:%u:%u (sector %llu)" , |
32 | i->dev, i->bucket, i->bucket_offset, i->sector); |
33 | } |
34 | } |
35 | |
36 | static void bch2_journal_replay_to_text(struct printbuf *out, struct bch_fs *c, |
37 | struct journal_replay *j) |
38 | { |
39 | prt_printf(out, "seq %llu " , le64_to_cpu(j->j.seq)); |
40 | |
41 | bch2_journal_ptrs_to_text(out, c, j); |
42 | |
43 | for_each_jset_entry_type(entry, &j->j, BCH_JSET_ENTRY_datetime) { |
44 | struct jset_entry_datetime *datetime = |
45 | container_of(entry, struct jset_entry_datetime, entry); |
46 | bch2_prt_datetime(out, le64_to_cpu(datetime->seconds)); |
47 | break; |
48 | } |
49 | } |
50 | |
51 | static struct nonce journal_nonce(const struct jset *jset) |
52 | { |
53 | return (struct nonce) {{ |
54 | [0] = 0, |
55 | [1] = ((__le32 *) &jset->seq)[0], |
56 | [2] = ((__le32 *) &jset->seq)[1], |
57 | [3] = BCH_NONCE_JOURNAL, |
58 | }}; |
59 | } |
60 | |
61 | static bool jset_csum_good(struct bch_fs *c, struct jset *j, struct bch_csum *csum) |
62 | { |
63 | if (!bch2_checksum_type_valid(c, type: JSET_CSUM_TYPE(k: j))) { |
64 | *csum = (struct bch_csum) {}; |
65 | return false; |
66 | } |
67 | |
68 | *csum = csum_vstruct(c, JSET_CSUM_TYPE(j), journal_nonce(j), j); |
69 | return !bch2_crc_cmp(l: j->csum, r: *csum); |
70 | } |
71 | |
72 | static inline u32 journal_entry_radix_idx(struct bch_fs *c, u64 seq) |
73 | { |
74 | return (seq - c->journal_entries_base_seq) & (~0U >> 1); |
75 | } |
76 | |
77 | static void __journal_replay_free(struct bch_fs *c, |
78 | struct journal_replay *i) |
79 | { |
80 | struct journal_replay **p = |
81 | genradix_ptr(&c->journal_entries, |
82 | journal_entry_radix_idx(c, le64_to_cpu(i->j.seq))); |
83 | |
84 | BUG_ON(*p != i); |
85 | *p = NULL; |
86 | kvfree(addr: i); |
87 | } |
88 | |
89 | static void journal_replay_free(struct bch_fs *c, struct journal_replay *i, bool blacklisted) |
90 | { |
91 | if (blacklisted) |
92 | i->ignore_blacklisted = true; |
93 | else |
94 | i->ignore_not_dirty = true; |
95 | |
96 | if (!c->opts.read_entire_journal) |
97 | __journal_replay_free(c, i); |
98 | } |
99 | |
100 | struct journal_list { |
101 | struct closure cl; |
102 | u64 last_seq; |
103 | struct mutex lock; |
104 | int ret; |
105 | }; |
106 | |
107 | #define JOURNAL_ENTRY_ADD_OK 0 |
108 | #define JOURNAL_ENTRY_ADD_OUT_OF_RANGE 5 |
109 | |
110 | /* |
111 | * Given a journal entry we just read, add it to the list of journal entries to |
112 | * be replayed: |
113 | */ |
114 | static int journal_entry_add(struct bch_fs *c, struct bch_dev *ca, |
115 | struct journal_ptr entry_ptr, |
116 | struct journal_list *jlist, struct jset *j) |
117 | { |
118 | struct genradix_iter iter; |
119 | struct journal_replay **_i, *i, *dup; |
120 | size_t bytes = vstruct_bytes(j); |
121 | u64 last_seq = !JSET_NO_FLUSH(k: j) ? le64_to_cpu(j->last_seq) : 0; |
122 | struct printbuf buf = PRINTBUF; |
123 | int ret = JOURNAL_ENTRY_ADD_OK; |
124 | |
125 | /* Is this entry older than the range we need? */ |
126 | if (!c->opts.read_entire_journal && |
127 | le64_to_cpu(j->seq) < jlist->last_seq) |
128 | return JOURNAL_ENTRY_ADD_OUT_OF_RANGE; |
129 | |
130 | /* |
131 | * genradixes are indexed by a ulong, not a u64, so we can't index them |
132 | * by sequence number directly: Assume instead that they will all fall |
133 | * within the range of +-2billion of the filrst one we find. |
134 | */ |
135 | if (!c->journal_entries_base_seq) |
136 | c->journal_entries_base_seq = max_t(s64, 1, le64_to_cpu(j->seq) - S32_MAX); |
137 | |
138 | /* Drop entries we don't need anymore */ |
139 | if (last_seq > jlist->last_seq && !c->opts.read_entire_journal) { |
140 | genradix_for_each_from(&c->journal_entries, iter, _i, |
141 | journal_entry_radix_idx(c, jlist->last_seq)) { |
142 | i = *_i; |
143 | |
144 | if (journal_replay_ignore(i)) |
145 | continue; |
146 | |
147 | if (le64_to_cpu(i->j.seq) >= last_seq) |
148 | break; |
149 | |
150 | journal_replay_free(c, i, blacklisted: false); |
151 | } |
152 | } |
153 | |
154 | jlist->last_seq = max(jlist->last_seq, last_seq); |
155 | |
156 | _i = genradix_ptr_alloc(&c->journal_entries, |
157 | journal_entry_radix_idx(c, le64_to_cpu(j->seq)), |
158 | GFP_KERNEL); |
159 | if (!_i) |
160 | return -BCH_ERR_ENOMEM_journal_entry_add; |
161 | |
162 | /* |
163 | * Duplicate journal entries? If so we want the one that didn't have a |
164 | * checksum error: |
165 | */ |
166 | dup = *_i; |
167 | if (dup) { |
168 | bool identical = bytes == vstruct_bytes(&dup->j) && |
169 | !memcmp(p: j, q: &dup->j, size: bytes); |
170 | bool not_identical = !identical && |
171 | entry_ptr.csum_good && |
172 | dup->csum_good; |
173 | |
174 | bool same_device = false; |
175 | darray_for_each(dup->ptrs, ptr) |
176 | if (ptr->dev == ca->dev_idx) |
177 | same_device = true; |
178 | |
179 | ret = darray_push(&dup->ptrs, entry_ptr); |
180 | if (ret) |
181 | goto out; |
182 | |
183 | bch2_journal_replay_to_text(out: &buf, c, j: dup); |
184 | |
185 | fsck_err_on(same_device, |
186 | c, journal_entry_dup_same_device, |
187 | "duplicate journal entry on same device\n %s" , |
188 | buf.buf); |
189 | |
190 | fsck_err_on(not_identical, |
191 | c, journal_entry_replicas_data_mismatch, |
192 | "found duplicate but non identical journal entries\n %s" , |
193 | buf.buf); |
194 | |
195 | if (entry_ptr.csum_good && !identical) |
196 | goto replace; |
197 | |
198 | goto out; |
199 | } |
200 | replace: |
201 | i = kvmalloc(offsetof(struct journal_replay, j) + bytes, GFP_KERNEL); |
202 | if (!i) |
203 | return -BCH_ERR_ENOMEM_journal_entry_add; |
204 | |
205 | darray_init(&i->ptrs); |
206 | i->csum_good = entry_ptr.csum_good; |
207 | i->ignore_blacklisted = false; |
208 | i->ignore_not_dirty = false; |
209 | unsafe_memcpy(&i->j, j, bytes, "embedded variable length struct" ); |
210 | |
211 | if (dup) { |
212 | /* The first ptr should represent the jset we kept: */ |
213 | darray_for_each(dup->ptrs, ptr) |
214 | darray_push(&i->ptrs, *ptr); |
215 | __journal_replay_free(c, i: dup); |
216 | } else { |
217 | darray_push(&i->ptrs, entry_ptr); |
218 | } |
219 | |
220 | *_i = i; |
221 | out: |
222 | fsck_err: |
223 | printbuf_exit(&buf); |
224 | return ret; |
225 | } |
226 | |
227 | /* this fills in a range with empty jset_entries: */ |
228 | static void journal_entry_null_range(void *start, void *end) |
229 | { |
230 | struct jset_entry *entry; |
231 | |
232 | for (entry = start; entry != end; entry = vstruct_next(entry)) |
233 | memset(entry, 0, sizeof(*entry)); |
234 | } |
235 | |
236 | #define JOURNAL_ENTRY_REREAD 5 |
237 | #define JOURNAL_ENTRY_NONE 6 |
238 | #define JOURNAL_ENTRY_BAD 7 |
239 | |
240 | static void journal_entry_err_msg(struct printbuf *out, |
241 | u32 version, |
242 | struct jset *jset, |
243 | struct jset_entry *entry) |
244 | { |
245 | prt_str(out, str: "invalid journal entry, version=" ); |
246 | bch2_version_to_text(out, version); |
247 | |
248 | if (entry) { |
249 | prt_str(out, str: " type=" ); |
250 | bch2_prt_jset_entry_type(out, entry->type); |
251 | } |
252 | |
253 | if (!jset) { |
254 | prt_printf(out, " in superblock" ); |
255 | } else { |
256 | |
257 | prt_printf(out, " seq=%llu" , le64_to_cpu(jset->seq)); |
258 | |
259 | if (entry) |
260 | prt_printf(out, " offset=%zi/%u" , |
261 | (u64 *) entry - jset->_data, |
262 | le32_to_cpu(jset->u64s)); |
263 | } |
264 | |
265 | prt_str(out, str: ": " ); |
266 | } |
267 | |
268 | #define journal_entry_err(c, version, jset, entry, _err, msg, ...) \ |
269 | ({ \ |
270 | struct printbuf _buf = PRINTBUF; \ |
271 | \ |
272 | journal_entry_err_msg(&_buf, version, jset, entry); \ |
273 | prt_printf(&_buf, msg, ##__VA_ARGS__); \ |
274 | \ |
275 | switch (flags & BKEY_INVALID_WRITE) { \ |
276 | case READ: \ |
277 | mustfix_fsck_err(c, _err, "%s", _buf.buf); \ |
278 | break; \ |
279 | case WRITE: \ |
280 | bch2_sb_error_count(c, BCH_FSCK_ERR_##_err); \ |
281 | bch_err(c, "corrupt metadata before write: %s\n", _buf.buf);\ |
282 | if (bch2_fs_inconsistent(c)) { \ |
283 | ret = -BCH_ERR_fsck_errors_not_fixed; \ |
284 | goto fsck_err; \ |
285 | } \ |
286 | break; \ |
287 | } \ |
288 | \ |
289 | printbuf_exit(&_buf); \ |
290 | true; \ |
291 | }) |
292 | |
293 | #define journal_entry_err_on(cond, ...) \ |
294 | ((cond) ? journal_entry_err(__VA_ARGS__) : false) |
295 | |
296 | #define FSCK_DELETED_KEY 5 |
297 | |
298 | static int journal_validate_key(struct bch_fs *c, |
299 | struct jset *jset, |
300 | struct jset_entry *entry, |
301 | unsigned level, enum btree_id btree_id, |
302 | struct bkey_i *k, |
303 | unsigned version, int big_endian, |
304 | enum bkey_invalid_flags flags) |
305 | { |
306 | int write = flags & BKEY_INVALID_WRITE; |
307 | void *next = vstruct_next(entry); |
308 | struct printbuf buf = PRINTBUF; |
309 | int ret = 0; |
310 | |
311 | if (journal_entry_err_on(!k->k.u64s, |
312 | c, version, jset, entry, |
313 | journal_entry_bkey_u64s_0, |
314 | "k->u64s 0" )) { |
315 | entry->u64s = cpu_to_le16((u64 *) k - entry->_data); |
316 | journal_entry_null_range(vstruct_next(entry), end: next); |
317 | return FSCK_DELETED_KEY; |
318 | } |
319 | |
320 | if (journal_entry_err_on((void *) bkey_next(k) > |
321 | (void *) vstruct_next(entry), |
322 | c, version, jset, entry, |
323 | journal_entry_bkey_past_end, |
324 | "extends past end of journal entry" )) { |
325 | entry->u64s = cpu_to_le16((u64 *) k - entry->_data); |
326 | journal_entry_null_range(vstruct_next(entry), end: next); |
327 | return FSCK_DELETED_KEY; |
328 | } |
329 | |
330 | if (journal_entry_err_on(k->k.format != KEY_FORMAT_CURRENT, |
331 | c, version, jset, entry, |
332 | journal_entry_bkey_bad_format, |
333 | "bad format %u" , k->k.format)) { |
334 | le16_add_cpu(var: &entry->u64s, val: -((u16) k->k.u64s)); |
335 | memmove(k, bkey_next(k), next - (void *) bkey_next(k)); |
336 | journal_entry_null_range(vstruct_next(entry), end: next); |
337 | return FSCK_DELETED_KEY; |
338 | } |
339 | |
340 | if (!write) |
341 | bch2_bkey_compat(level, btree_id, version, big_endian, |
342 | write, NULL, k: bkey_to_packed(k)); |
343 | |
344 | if (bch2_bkey_invalid(c, bkey_i_to_s_c(k), |
345 | __btree_node_type(level, id: btree_id), write, &buf)) { |
346 | printbuf_reset(buf: &buf); |
347 | journal_entry_err_msg(out: &buf, version, jset, entry); |
348 | prt_newline(&buf); |
349 | printbuf_indent_add(&buf, 2); |
350 | |
351 | bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(k)); |
352 | prt_newline(&buf); |
353 | bch2_bkey_invalid(c, bkey_i_to_s_c(k), |
354 | __btree_node_type(level, id: btree_id), write, &buf); |
355 | |
356 | mustfix_fsck_err(c, journal_entry_bkey_invalid, |
357 | "%s" , buf.buf); |
358 | |
359 | le16_add_cpu(var: &entry->u64s, val: -((u16) k->k.u64s)); |
360 | memmove(k, bkey_next(k), next - (void *) bkey_next(k)); |
361 | journal_entry_null_range(vstruct_next(entry), end: next); |
362 | |
363 | printbuf_exit(&buf); |
364 | return FSCK_DELETED_KEY; |
365 | } |
366 | |
367 | if (write) |
368 | bch2_bkey_compat(level, btree_id, version, big_endian, |
369 | write, NULL, k: bkey_to_packed(k)); |
370 | fsck_err: |
371 | printbuf_exit(&buf); |
372 | return ret; |
373 | } |
374 | |
375 | static int journal_entry_btree_keys_validate(struct bch_fs *c, |
376 | struct jset *jset, |
377 | struct jset_entry *entry, |
378 | unsigned version, int big_endian, |
379 | enum bkey_invalid_flags flags) |
380 | { |
381 | struct bkey_i *k = entry->start; |
382 | |
383 | while (k != vstruct_last(entry)) { |
384 | int ret = journal_validate_key(c, jset, entry, |
385 | level: entry->level, |
386 | btree_id: entry->btree_id, |
387 | k, version, big_endian, |
388 | flags: flags|BKEY_INVALID_JOURNAL); |
389 | if (ret == FSCK_DELETED_KEY) |
390 | continue; |
391 | |
392 | k = bkey_next(k); |
393 | } |
394 | |
395 | return 0; |
396 | } |
397 | |
398 | static void journal_entry_btree_keys_to_text(struct printbuf *out, struct bch_fs *c, |
399 | struct jset_entry *entry) |
400 | { |
401 | bool first = true; |
402 | |
403 | jset_entry_for_each_key(entry, k) { |
404 | if (!first) { |
405 | prt_newline(out); |
406 | bch2_prt_jset_entry_type(out, entry->type); |
407 | prt_str(out, str: ": " ); |
408 | } |
409 | prt_printf(out, "btree=%s l=%u " , bch2_btree_id_str(entry->btree_id), entry->level); |
410 | bch2_bkey_val_to_text(out, c, bkey_i_to_s_c(k)); |
411 | first = false; |
412 | } |
413 | } |
414 | |
415 | static int journal_entry_btree_root_validate(struct bch_fs *c, |
416 | struct jset *jset, |
417 | struct jset_entry *entry, |
418 | unsigned version, int big_endian, |
419 | enum bkey_invalid_flags flags) |
420 | { |
421 | struct bkey_i *k = entry->start; |
422 | int ret = 0; |
423 | |
424 | if (journal_entry_err_on(!entry->u64s || |
425 | le16_to_cpu(entry->u64s) != k->k.u64s, |
426 | c, version, jset, entry, |
427 | journal_entry_btree_root_bad_size, |
428 | "invalid btree root journal entry: wrong number of keys" )) { |
429 | void *next = vstruct_next(entry); |
430 | /* |
431 | * we don't want to null out this jset_entry, |
432 | * just the contents, so that later we can tell |
433 | * we were _supposed_ to have a btree root |
434 | */ |
435 | entry->u64s = 0; |
436 | journal_entry_null_range(vstruct_next(entry), end: next); |
437 | return 0; |
438 | } |
439 | |
440 | ret = journal_validate_key(c, jset, entry, level: 1, btree_id: entry->btree_id, k, |
441 | version, big_endian, flags); |
442 | if (ret == FSCK_DELETED_KEY) |
443 | ret = 0; |
444 | fsck_err: |
445 | return ret; |
446 | } |
447 | |
448 | static void journal_entry_btree_root_to_text(struct printbuf *out, struct bch_fs *c, |
449 | struct jset_entry *entry) |
450 | { |
451 | journal_entry_btree_keys_to_text(out, c, entry); |
452 | } |
453 | |
454 | static int journal_entry_prio_ptrs_validate(struct bch_fs *c, |
455 | struct jset *jset, |
456 | struct jset_entry *entry, |
457 | unsigned version, int big_endian, |
458 | enum bkey_invalid_flags flags) |
459 | { |
460 | /* obsolete, don't care: */ |
461 | return 0; |
462 | } |
463 | |
464 | static void journal_entry_prio_ptrs_to_text(struct printbuf *out, struct bch_fs *c, |
465 | struct jset_entry *entry) |
466 | { |
467 | } |
468 | |
469 | static int journal_entry_blacklist_validate(struct bch_fs *c, |
470 | struct jset *jset, |
471 | struct jset_entry *entry, |
472 | unsigned version, int big_endian, |
473 | enum bkey_invalid_flags flags) |
474 | { |
475 | int ret = 0; |
476 | |
477 | if (journal_entry_err_on(le16_to_cpu(entry->u64s) != 1, |
478 | c, version, jset, entry, |
479 | journal_entry_blacklist_bad_size, |
480 | "invalid journal seq blacklist entry: bad size" )) { |
481 | journal_entry_null_range(start: entry, vstruct_next(entry)); |
482 | } |
483 | fsck_err: |
484 | return ret; |
485 | } |
486 | |
487 | static void journal_entry_blacklist_to_text(struct printbuf *out, struct bch_fs *c, |
488 | struct jset_entry *entry) |
489 | { |
490 | struct jset_entry_blacklist *bl = |
491 | container_of(entry, struct jset_entry_blacklist, entry); |
492 | |
493 | prt_printf(out, "seq=%llu" , le64_to_cpu(bl->seq)); |
494 | } |
495 | |
496 | static int journal_entry_blacklist_v2_validate(struct bch_fs *c, |
497 | struct jset *jset, |
498 | struct jset_entry *entry, |
499 | unsigned version, int big_endian, |
500 | enum bkey_invalid_flags flags) |
501 | { |
502 | struct jset_entry_blacklist_v2 *bl_entry; |
503 | int ret = 0; |
504 | |
505 | if (journal_entry_err_on(le16_to_cpu(entry->u64s) != 2, |
506 | c, version, jset, entry, |
507 | journal_entry_blacklist_v2_bad_size, |
508 | "invalid journal seq blacklist entry: bad size" )) { |
509 | journal_entry_null_range(start: entry, vstruct_next(entry)); |
510 | goto out; |
511 | } |
512 | |
513 | bl_entry = container_of(entry, struct jset_entry_blacklist_v2, entry); |
514 | |
515 | if (journal_entry_err_on(le64_to_cpu(bl_entry->start) > |
516 | le64_to_cpu(bl_entry->end), |
517 | c, version, jset, entry, |
518 | journal_entry_blacklist_v2_start_past_end, |
519 | "invalid journal seq blacklist entry: start > end" )) { |
520 | journal_entry_null_range(start: entry, vstruct_next(entry)); |
521 | } |
522 | out: |
523 | fsck_err: |
524 | return ret; |
525 | } |
526 | |
527 | static void journal_entry_blacklist_v2_to_text(struct printbuf *out, struct bch_fs *c, |
528 | struct jset_entry *entry) |
529 | { |
530 | struct jset_entry_blacklist_v2 *bl = |
531 | container_of(entry, struct jset_entry_blacklist_v2, entry); |
532 | |
533 | prt_printf(out, "start=%llu end=%llu" , |
534 | le64_to_cpu(bl->start), |
535 | le64_to_cpu(bl->end)); |
536 | } |
537 | |
538 | static int journal_entry_usage_validate(struct bch_fs *c, |
539 | struct jset *jset, |
540 | struct jset_entry *entry, |
541 | unsigned version, int big_endian, |
542 | enum bkey_invalid_flags flags) |
543 | { |
544 | struct jset_entry_usage *u = |
545 | container_of(entry, struct jset_entry_usage, entry); |
546 | unsigned bytes = jset_u64s(le16_to_cpu(entry->u64s)) * sizeof(u64); |
547 | int ret = 0; |
548 | |
549 | if (journal_entry_err_on(bytes < sizeof(*u), |
550 | c, version, jset, entry, |
551 | journal_entry_usage_bad_size, |
552 | "invalid journal entry usage: bad size" )) { |
553 | journal_entry_null_range(start: entry, vstruct_next(entry)); |
554 | return ret; |
555 | } |
556 | |
557 | fsck_err: |
558 | return ret; |
559 | } |
560 | |
561 | static void journal_entry_usage_to_text(struct printbuf *out, struct bch_fs *c, |
562 | struct jset_entry *entry) |
563 | { |
564 | struct jset_entry_usage *u = |
565 | container_of(entry, struct jset_entry_usage, entry); |
566 | |
567 | prt_str(out, str: "type=" ); |
568 | bch2_prt_fs_usage_type(out, u->entry.btree_id); |
569 | prt_printf(out, " v=%llu" , le64_to_cpu(u->v)); |
570 | } |
571 | |
572 | static int journal_entry_data_usage_validate(struct bch_fs *c, |
573 | struct jset *jset, |
574 | struct jset_entry *entry, |
575 | unsigned version, int big_endian, |
576 | enum bkey_invalid_flags flags) |
577 | { |
578 | struct jset_entry_data_usage *u = |
579 | container_of(entry, struct jset_entry_data_usage, entry); |
580 | unsigned bytes = jset_u64s(le16_to_cpu(entry->u64s)) * sizeof(u64); |
581 | struct printbuf err = PRINTBUF; |
582 | int ret = 0; |
583 | |
584 | if (journal_entry_err_on(bytes < sizeof(*u) || |
585 | bytes < sizeof(*u) + u->r.nr_devs, |
586 | c, version, jset, entry, |
587 | journal_entry_data_usage_bad_size, |
588 | "invalid journal entry usage: bad size" )) { |
589 | journal_entry_null_range(start: entry, vstruct_next(entry)); |
590 | goto out; |
591 | } |
592 | |
593 | if (journal_entry_err_on(bch2_replicas_entry_validate(&u->r, c->disk_sb.sb, &err), |
594 | c, version, jset, entry, |
595 | journal_entry_data_usage_bad_size, |
596 | "invalid journal entry usage: %s" , err.buf)) { |
597 | journal_entry_null_range(start: entry, vstruct_next(entry)); |
598 | goto out; |
599 | } |
600 | out: |
601 | fsck_err: |
602 | printbuf_exit(&err); |
603 | return ret; |
604 | } |
605 | |
606 | static void journal_entry_data_usage_to_text(struct printbuf *out, struct bch_fs *c, |
607 | struct jset_entry *entry) |
608 | { |
609 | struct jset_entry_data_usage *u = |
610 | container_of(entry, struct jset_entry_data_usage, entry); |
611 | |
612 | bch2_replicas_entry_to_text(out, &u->r); |
613 | prt_printf(out, "=%llu" , le64_to_cpu(u->v)); |
614 | } |
615 | |
616 | static int journal_entry_clock_validate(struct bch_fs *c, |
617 | struct jset *jset, |
618 | struct jset_entry *entry, |
619 | unsigned version, int big_endian, |
620 | enum bkey_invalid_flags flags) |
621 | { |
622 | struct jset_entry_clock *clock = |
623 | container_of(entry, struct jset_entry_clock, entry); |
624 | unsigned bytes = jset_u64s(le16_to_cpu(entry->u64s)) * sizeof(u64); |
625 | int ret = 0; |
626 | |
627 | if (journal_entry_err_on(bytes != sizeof(*clock), |
628 | c, version, jset, entry, |
629 | journal_entry_clock_bad_size, |
630 | "bad size" )) { |
631 | journal_entry_null_range(start: entry, vstruct_next(entry)); |
632 | return ret; |
633 | } |
634 | |
635 | if (journal_entry_err_on(clock->rw > 1, |
636 | c, version, jset, entry, |
637 | journal_entry_clock_bad_rw, |
638 | "bad rw" )) { |
639 | journal_entry_null_range(start: entry, vstruct_next(entry)); |
640 | return ret; |
641 | } |
642 | |
643 | fsck_err: |
644 | return ret; |
645 | } |
646 | |
647 | static void journal_entry_clock_to_text(struct printbuf *out, struct bch_fs *c, |
648 | struct jset_entry *entry) |
649 | { |
650 | struct jset_entry_clock *clock = |
651 | container_of(entry, struct jset_entry_clock, entry); |
652 | |
653 | prt_printf(out, "%s=%llu" , clock->rw ? "write" : "read" , le64_to_cpu(clock->time)); |
654 | } |
655 | |
656 | static int journal_entry_dev_usage_validate(struct bch_fs *c, |
657 | struct jset *jset, |
658 | struct jset_entry *entry, |
659 | unsigned version, int big_endian, |
660 | enum bkey_invalid_flags flags) |
661 | { |
662 | struct jset_entry_dev_usage *u = |
663 | container_of(entry, struct jset_entry_dev_usage, entry); |
664 | unsigned bytes = jset_u64s(le16_to_cpu(entry->u64s)) * sizeof(u64); |
665 | unsigned expected = sizeof(*u); |
666 | unsigned dev; |
667 | int ret = 0; |
668 | |
669 | if (journal_entry_err_on(bytes < expected, |
670 | c, version, jset, entry, |
671 | journal_entry_dev_usage_bad_size, |
672 | "bad size (%u < %u)" , |
673 | bytes, expected)) { |
674 | journal_entry_null_range(start: entry, vstruct_next(entry)); |
675 | return ret; |
676 | } |
677 | |
678 | dev = le32_to_cpu(u->dev); |
679 | |
680 | if (journal_entry_err_on(!bch2_dev_exists2(c, dev), |
681 | c, version, jset, entry, |
682 | journal_entry_dev_usage_bad_dev, |
683 | "bad dev" )) { |
684 | journal_entry_null_range(start: entry, vstruct_next(entry)); |
685 | return ret; |
686 | } |
687 | |
688 | if (journal_entry_err_on(u->pad, |
689 | c, version, jset, entry, |
690 | journal_entry_dev_usage_bad_pad, |
691 | "bad pad" )) { |
692 | journal_entry_null_range(start: entry, vstruct_next(entry)); |
693 | return ret; |
694 | } |
695 | |
696 | fsck_err: |
697 | return ret; |
698 | } |
699 | |
700 | static void journal_entry_dev_usage_to_text(struct printbuf *out, struct bch_fs *c, |
701 | struct jset_entry *entry) |
702 | { |
703 | struct jset_entry_dev_usage *u = |
704 | container_of(entry, struct jset_entry_dev_usage, entry); |
705 | unsigned i, nr_types = jset_entry_dev_usage_nr_types(u); |
706 | |
707 | prt_printf(out, "dev=%u" , le32_to_cpu(u->dev)); |
708 | |
709 | for (i = 0; i < nr_types; i++) { |
710 | bch2_prt_data_type(out, i); |
711 | prt_printf(out, ": buckets=%llu sectors=%llu fragmented=%llu" , |
712 | le64_to_cpu(u->d[i].buckets), |
713 | le64_to_cpu(u->d[i].sectors), |
714 | le64_to_cpu(u->d[i].fragmented)); |
715 | } |
716 | } |
717 | |
718 | static int journal_entry_log_validate(struct bch_fs *c, |
719 | struct jset *jset, |
720 | struct jset_entry *entry, |
721 | unsigned version, int big_endian, |
722 | enum bkey_invalid_flags flags) |
723 | { |
724 | return 0; |
725 | } |
726 | |
727 | static void journal_entry_log_to_text(struct printbuf *out, struct bch_fs *c, |
728 | struct jset_entry *entry) |
729 | { |
730 | struct jset_entry_log *l = container_of(entry, struct jset_entry_log, entry); |
731 | unsigned bytes = vstruct_bytes(entry) - offsetof(struct jset_entry_log, d); |
732 | |
733 | prt_printf(out, "%.*s" , bytes, l->d); |
734 | } |
735 | |
736 | static int journal_entry_overwrite_validate(struct bch_fs *c, |
737 | struct jset *jset, |
738 | struct jset_entry *entry, |
739 | unsigned version, int big_endian, |
740 | enum bkey_invalid_flags flags) |
741 | { |
742 | return journal_entry_btree_keys_validate(c, jset, entry, |
743 | version, big_endian, READ); |
744 | } |
745 | |
746 | static void journal_entry_overwrite_to_text(struct printbuf *out, struct bch_fs *c, |
747 | struct jset_entry *entry) |
748 | { |
749 | journal_entry_btree_keys_to_text(out, c, entry); |
750 | } |
751 | |
752 | static int journal_entry_write_buffer_keys_validate(struct bch_fs *c, |
753 | struct jset *jset, |
754 | struct jset_entry *entry, |
755 | unsigned version, int big_endian, |
756 | enum bkey_invalid_flags flags) |
757 | { |
758 | return journal_entry_btree_keys_validate(c, jset, entry, |
759 | version, big_endian, READ); |
760 | } |
761 | |
762 | static void journal_entry_write_buffer_keys_to_text(struct printbuf *out, struct bch_fs *c, |
763 | struct jset_entry *entry) |
764 | { |
765 | journal_entry_btree_keys_to_text(out, c, entry); |
766 | } |
767 | |
768 | static int journal_entry_datetime_validate(struct bch_fs *c, |
769 | struct jset *jset, |
770 | struct jset_entry *entry, |
771 | unsigned version, int big_endian, |
772 | enum bkey_invalid_flags flags) |
773 | { |
774 | unsigned bytes = vstruct_bytes(entry); |
775 | unsigned expected = 16; |
776 | int ret = 0; |
777 | |
778 | if (journal_entry_err_on(vstruct_bytes(entry) < expected, |
779 | c, version, jset, entry, |
780 | journal_entry_dev_usage_bad_size, |
781 | "bad size (%u < %u)" , |
782 | bytes, expected)) { |
783 | journal_entry_null_range(start: entry, vstruct_next(entry)); |
784 | return ret; |
785 | } |
786 | fsck_err: |
787 | return ret; |
788 | } |
789 | |
790 | static void journal_entry_datetime_to_text(struct printbuf *out, struct bch_fs *c, |
791 | struct jset_entry *entry) |
792 | { |
793 | struct jset_entry_datetime *datetime = |
794 | container_of(entry, struct jset_entry_datetime, entry); |
795 | |
796 | bch2_prt_datetime(out, le64_to_cpu(datetime->seconds)); |
797 | } |
798 | |
799 | struct jset_entry_ops { |
800 | int (*validate)(struct bch_fs *, struct jset *, |
801 | struct jset_entry *, unsigned, int, |
802 | enum bkey_invalid_flags); |
803 | void (*to_text)(struct printbuf *, struct bch_fs *, struct jset_entry *); |
804 | }; |
805 | |
806 | static const struct jset_entry_ops bch2_jset_entry_ops[] = { |
807 | #define x(f, nr) \ |
808 | [BCH_JSET_ENTRY_##f] = (struct jset_entry_ops) { \ |
809 | .validate = journal_entry_##f##_validate, \ |
810 | .to_text = journal_entry_##f##_to_text, \ |
811 | }, |
812 | BCH_JSET_ENTRY_TYPES() |
813 | #undef x |
814 | }; |
815 | |
816 | int bch2_journal_entry_validate(struct bch_fs *c, |
817 | struct jset *jset, |
818 | struct jset_entry *entry, |
819 | unsigned version, int big_endian, |
820 | enum bkey_invalid_flags flags) |
821 | { |
822 | return entry->type < BCH_JSET_ENTRY_NR |
823 | ? bch2_jset_entry_ops[entry->type].validate(c, jset, entry, |
824 | version, big_endian, flags) |
825 | : 0; |
826 | } |
827 | |
828 | void bch2_journal_entry_to_text(struct printbuf *out, struct bch_fs *c, |
829 | struct jset_entry *entry) |
830 | { |
831 | bch2_prt_jset_entry_type(out, entry->type); |
832 | |
833 | if (entry->type < BCH_JSET_ENTRY_NR) { |
834 | prt_str(out, str: ": " ); |
835 | bch2_jset_entry_ops[entry->type].to_text(out, c, entry); |
836 | } |
837 | } |
838 | |
839 | static int jset_validate_entries(struct bch_fs *c, struct jset *jset, |
840 | enum bkey_invalid_flags flags) |
841 | { |
842 | unsigned version = le32_to_cpu(jset->version); |
843 | int ret = 0; |
844 | |
845 | vstruct_for_each(jset, entry) { |
846 | if (journal_entry_err_on(vstruct_next(entry) > vstruct_last(jset), |
847 | c, version, jset, entry, |
848 | journal_entry_past_jset_end, |
849 | "journal entry extends past end of jset" )) { |
850 | jset->u64s = cpu_to_le32((u64 *) entry - jset->_data); |
851 | break; |
852 | } |
853 | |
854 | ret = bch2_journal_entry_validate(c, jset, entry, |
855 | version, big_endian: JSET_BIG_ENDIAN(k: jset), flags); |
856 | if (ret) |
857 | break; |
858 | } |
859 | fsck_err: |
860 | return ret; |
861 | } |
862 | |
863 | static int jset_validate(struct bch_fs *c, |
864 | struct bch_dev *ca, |
865 | struct jset *jset, u64 sector, |
866 | enum bkey_invalid_flags flags) |
867 | { |
868 | unsigned version; |
869 | int ret = 0; |
870 | |
871 | if (le64_to_cpu(jset->magic) != jset_magic(c)) |
872 | return JOURNAL_ENTRY_NONE; |
873 | |
874 | version = le32_to_cpu(jset->version); |
875 | if (journal_entry_err_on(!bch2_version_compatible(version), |
876 | c, version, jset, NULL, |
877 | jset_unsupported_version, |
878 | "%s sector %llu seq %llu: incompatible journal entry version %u.%u" , |
879 | ca ? ca->name : c->name, |
880 | sector, le64_to_cpu(jset->seq), |
881 | BCH_VERSION_MAJOR(version), |
882 | BCH_VERSION_MINOR(version))) { |
883 | /* don't try to continue: */ |
884 | return -EINVAL; |
885 | } |
886 | |
887 | if (journal_entry_err_on(!bch2_checksum_type_valid(c, JSET_CSUM_TYPE(jset)), |
888 | c, version, jset, NULL, |
889 | jset_unknown_csum, |
890 | "%s sector %llu seq %llu: journal entry with unknown csum type %llu" , |
891 | ca ? ca->name : c->name, |
892 | sector, le64_to_cpu(jset->seq), |
893 | JSET_CSUM_TYPE(jset))) |
894 | ret = JOURNAL_ENTRY_BAD; |
895 | |
896 | /* last_seq is ignored when JSET_NO_FLUSH is true */ |
897 | if (journal_entry_err_on(!JSET_NO_FLUSH(jset) && |
898 | le64_to_cpu(jset->last_seq) > le64_to_cpu(jset->seq), |
899 | c, version, jset, NULL, |
900 | jset_last_seq_newer_than_seq, |
901 | "invalid journal entry: last_seq > seq (%llu > %llu)" , |
902 | le64_to_cpu(jset->last_seq), |
903 | le64_to_cpu(jset->seq))) { |
904 | jset->last_seq = jset->seq; |
905 | return JOURNAL_ENTRY_BAD; |
906 | } |
907 | |
908 | ret = jset_validate_entries(c, jset, flags); |
909 | fsck_err: |
910 | return ret; |
911 | } |
912 | |
913 | static int jset_validate_early(struct bch_fs *c, |
914 | struct bch_dev *ca, |
915 | struct jset *jset, u64 sector, |
916 | unsigned bucket_sectors_left, |
917 | unsigned sectors_read) |
918 | { |
919 | size_t bytes = vstruct_bytes(jset); |
920 | unsigned version; |
921 | enum bkey_invalid_flags flags = BKEY_INVALID_JOURNAL; |
922 | int ret = 0; |
923 | |
924 | if (le64_to_cpu(jset->magic) != jset_magic(c)) |
925 | return JOURNAL_ENTRY_NONE; |
926 | |
927 | version = le32_to_cpu(jset->version); |
928 | if (journal_entry_err_on(!bch2_version_compatible(version), |
929 | c, version, jset, NULL, |
930 | jset_unsupported_version, |
931 | "%s sector %llu seq %llu: unknown journal entry version %u.%u" , |
932 | ca ? ca->name : c->name, |
933 | sector, le64_to_cpu(jset->seq), |
934 | BCH_VERSION_MAJOR(version), |
935 | BCH_VERSION_MINOR(version))) { |
936 | /* don't try to continue: */ |
937 | return -EINVAL; |
938 | } |
939 | |
940 | if (bytes > (sectors_read << 9) && |
941 | sectors_read < bucket_sectors_left) |
942 | return JOURNAL_ENTRY_REREAD; |
943 | |
944 | if (journal_entry_err_on(bytes > bucket_sectors_left << 9, |
945 | c, version, jset, NULL, |
946 | jset_past_bucket_end, |
947 | "%s sector %llu seq %llu: journal entry too big (%zu bytes)" , |
948 | ca ? ca->name : c->name, |
949 | sector, le64_to_cpu(jset->seq), bytes)) |
950 | le32_add_cpu(var: &jset->u64s, |
951 | val: -((bytes - (bucket_sectors_left << 9)) / 8)); |
952 | fsck_err: |
953 | return ret; |
954 | } |
955 | |
956 | struct journal_read_buf { |
957 | void *data; |
958 | size_t size; |
959 | }; |
960 | |
961 | static int journal_read_buf_realloc(struct journal_read_buf *b, |
962 | size_t new_size) |
963 | { |
964 | void *n; |
965 | |
966 | /* the bios are sized for this many pages, max: */ |
967 | if (new_size > JOURNAL_ENTRY_SIZE_MAX) |
968 | return -BCH_ERR_ENOMEM_journal_read_buf_realloc; |
969 | |
970 | new_size = roundup_pow_of_two(new_size); |
971 | n = kvmalloc(size: new_size, GFP_KERNEL); |
972 | if (!n) |
973 | return -BCH_ERR_ENOMEM_journal_read_buf_realloc; |
974 | |
975 | kvfree(addr: b->data); |
976 | b->data = n; |
977 | b->size = new_size; |
978 | return 0; |
979 | } |
980 | |
981 | static int journal_read_bucket(struct bch_dev *ca, |
982 | struct journal_read_buf *buf, |
983 | struct journal_list *jlist, |
984 | unsigned bucket) |
985 | { |
986 | struct bch_fs *c = ca->fs; |
987 | struct journal_device *ja = &ca->journal; |
988 | struct jset *j = NULL; |
989 | unsigned sectors, sectors_read = 0; |
990 | u64 offset = bucket_to_sector(ca, b: ja->buckets[bucket]), |
991 | end = offset + ca->mi.bucket_size; |
992 | bool saw_bad = false, csum_good; |
993 | struct printbuf err = PRINTBUF; |
994 | int ret = 0; |
995 | |
996 | pr_debug("reading %u" , bucket); |
997 | |
998 | while (offset < end) { |
999 | if (!sectors_read) { |
1000 | struct bio *bio; |
1001 | unsigned nr_bvecs; |
1002 | reread: |
1003 | sectors_read = min_t(unsigned, |
1004 | end - offset, buf->size >> 9); |
1005 | nr_bvecs = buf_pages(p: buf->data, len: sectors_read << 9); |
1006 | |
1007 | bio = bio_kmalloc(nr_vecs: nr_bvecs, GFP_KERNEL); |
1008 | bio_init(bio, bdev: ca->disk_sb.bdev, table: bio->bi_inline_vecs, max_vecs: nr_bvecs, opf: REQ_OP_READ); |
1009 | |
1010 | bio->bi_iter.bi_sector = offset; |
1011 | bch2_bio_map(bio, base: buf->data, sectors_read << 9); |
1012 | |
1013 | ret = submit_bio_wait(bio); |
1014 | kfree(objp: bio); |
1015 | |
1016 | if (bch2_dev_io_err_on(ret, ca, BCH_MEMBER_ERROR_read, |
1017 | "journal read error: sector %llu" , |
1018 | offset) || |
1019 | bch2_meta_read_fault("journal" )) { |
1020 | /* |
1021 | * We don't error out of the recovery process |
1022 | * here, since the relevant journal entry may be |
1023 | * found on a different device, and missing or |
1024 | * no journal entries will be handled later |
1025 | */ |
1026 | goto out; |
1027 | } |
1028 | |
1029 | j = buf->data; |
1030 | } |
1031 | |
1032 | ret = jset_validate_early(c, ca, jset: j, sector: offset, |
1033 | bucket_sectors_left: end - offset, sectors_read); |
1034 | switch (ret) { |
1035 | case 0: |
1036 | sectors = vstruct_sectors(j, c->block_bits); |
1037 | break; |
1038 | case JOURNAL_ENTRY_REREAD: |
1039 | if (vstruct_bytes(j) > buf->size) { |
1040 | ret = journal_read_buf_realloc(b: buf, |
1041 | vstruct_bytes(j)); |
1042 | if (ret) |
1043 | goto err; |
1044 | } |
1045 | goto reread; |
1046 | case JOURNAL_ENTRY_NONE: |
1047 | if (!saw_bad) |
1048 | goto out; |
1049 | /* |
1050 | * On checksum error we don't really trust the size |
1051 | * field of the journal entry we read, so try reading |
1052 | * again at next block boundary: |
1053 | */ |
1054 | sectors = block_sectors(c); |
1055 | goto next_block; |
1056 | default: |
1057 | goto err; |
1058 | } |
1059 | |
1060 | /* |
1061 | * This happens sometimes if we don't have discards on - |
1062 | * when we've partially overwritten a bucket with new |
1063 | * journal entries. We don't need the rest of the |
1064 | * bucket: |
1065 | */ |
1066 | if (le64_to_cpu(j->seq) < ja->bucket_seq[bucket]) |
1067 | goto out; |
1068 | |
1069 | ja->bucket_seq[bucket] = le64_to_cpu(j->seq); |
1070 | |
1071 | enum bch_csum_type csum_type = JSET_CSUM_TYPE(k: j); |
1072 | struct bch_csum csum; |
1073 | csum_good = jset_csum_good(c, j, csum: &csum); |
1074 | |
1075 | if (bch2_dev_io_err_on(!csum_good, ca, BCH_MEMBER_ERROR_checksum, |
1076 | "%s" , |
1077 | (printbuf_reset(&err), |
1078 | prt_str(&err, "journal " ), |
1079 | bch2_csum_err_msg(&err, csum_type, j->csum, csum), |
1080 | err.buf))) |
1081 | saw_bad = true; |
1082 | |
1083 | ret = bch2_encrypt(c, JSET_CSUM_TYPE(k: j), journal_nonce(jset: j), |
1084 | data: j->encrypted_start, |
1085 | vstruct_end(j) - (void *) j->encrypted_start); |
1086 | bch2_fs_fatal_err_on(ret, c, "decrypting journal entry: %s" , bch2_err_str(ret)); |
1087 | |
1088 | mutex_lock(&jlist->lock); |
1089 | ret = journal_entry_add(c, ca, entry_ptr: (struct journal_ptr) { |
1090 | .csum_good = csum_good, |
1091 | .dev = ca->dev_idx, |
1092 | .bucket = bucket, |
1093 | .bucket_offset = offset - |
1094 | bucket_to_sector(ca, b: ja->buckets[bucket]), |
1095 | .sector = offset, |
1096 | }, jlist, j); |
1097 | mutex_unlock(lock: &jlist->lock); |
1098 | |
1099 | switch (ret) { |
1100 | case JOURNAL_ENTRY_ADD_OK: |
1101 | break; |
1102 | case JOURNAL_ENTRY_ADD_OUT_OF_RANGE: |
1103 | break; |
1104 | default: |
1105 | goto err; |
1106 | } |
1107 | next_block: |
1108 | pr_debug("next" ); |
1109 | offset += sectors; |
1110 | sectors_read -= sectors; |
1111 | j = ((void *) j) + (sectors << 9); |
1112 | } |
1113 | |
1114 | out: |
1115 | ret = 0; |
1116 | err: |
1117 | printbuf_exit(&err); |
1118 | return ret; |
1119 | } |
1120 | |
1121 | static CLOSURE_CALLBACK(bch2_journal_read_device) |
1122 | { |
1123 | closure_type(ja, struct journal_device, read); |
1124 | struct bch_dev *ca = container_of(ja, struct bch_dev, journal); |
1125 | struct bch_fs *c = ca->fs; |
1126 | struct journal_list *jlist = |
1127 | container_of(cl->parent, struct journal_list, cl); |
1128 | struct journal_replay *r, **_r; |
1129 | struct genradix_iter iter; |
1130 | struct journal_read_buf buf = { NULL, 0 }; |
1131 | unsigned i; |
1132 | int ret = 0; |
1133 | |
1134 | if (!ja->nr) |
1135 | goto out; |
1136 | |
1137 | ret = journal_read_buf_realloc(b: &buf, PAGE_SIZE); |
1138 | if (ret) |
1139 | goto err; |
1140 | |
1141 | pr_debug("%u journal buckets" , ja->nr); |
1142 | |
1143 | for (i = 0; i < ja->nr; i++) { |
1144 | ret = journal_read_bucket(ca, buf: &buf, jlist, bucket: i); |
1145 | if (ret) |
1146 | goto err; |
1147 | } |
1148 | |
1149 | ja->sectors_free = ca->mi.bucket_size; |
1150 | |
1151 | mutex_lock(&jlist->lock); |
1152 | genradix_for_each_reverse(&c->journal_entries, iter, _r) { |
1153 | r = *_r; |
1154 | |
1155 | if (!r) |
1156 | continue; |
1157 | |
1158 | darray_for_each(r->ptrs, i) |
1159 | if (i->dev == ca->dev_idx) { |
1160 | unsigned wrote = bucket_remainder(ca, s: i->sector) + |
1161 | vstruct_sectors(&r->j, c->block_bits); |
1162 | |
1163 | ja->cur_idx = i->bucket; |
1164 | ja->sectors_free = ca->mi.bucket_size - wrote; |
1165 | goto found; |
1166 | } |
1167 | } |
1168 | found: |
1169 | mutex_unlock(lock: &jlist->lock); |
1170 | |
1171 | if (ja->bucket_seq[ja->cur_idx] && |
1172 | ja->sectors_free == ca->mi.bucket_size) { |
1173 | #if 0 |
1174 | /* |
1175 | * Debug code for ZNS support, where we (probably) want to be |
1176 | * correlated where we stopped in the journal to the zone write |
1177 | * points: |
1178 | */ |
1179 | bch_err(c, "ja->sectors_free == ca->mi.bucket_size" ); |
1180 | bch_err(c, "cur_idx %u/%u" , ja->cur_idx, ja->nr); |
1181 | for (i = 0; i < 3; i++) { |
1182 | unsigned idx = (ja->cur_idx + ja->nr - 1 + i) % ja->nr; |
1183 | |
1184 | bch_err(c, "bucket_seq[%u] = %llu" , idx, ja->bucket_seq[idx]); |
1185 | } |
1186 | #endif |
1187 | ja->sectors_free = 0; |
1188 | } |
1189 | |
1190 | /* |
1191 | * Set dirty_idx to indicate the entire journal is full and needs to be |
1192 | * reclaimed - journal reclaim will immediately reclaim whatever isn't |
1193 | * pinned when it first runs: |
1194 | */ |
1195 | ja->discard_idx = ja->dirty_idx_ondisk = |
1196 | ja->dirty_idx = (ja->cur_idx + 1) % ja->nr; |
1197 | out: |
1198 | bch_verbose(c, "journal read done on device %s, ret %i" , ca->name, ret); |
1199 | kvfree(addr: buf.data); |
1200 | percpu_ref_put(ref: &ca->io_ref); |
1201 | closure_return(cl); |
1202 | return; |
1203 | err: |
1204 | mutex_lock(&jlist->lock); |
1205 | jlist->ret = ret; |
1206 | mutex_unlock(lock: &jlist->lock); |
1207 | goto out; |
1208 | } |
1209 | |
1210 | int bch2_journal_read(struct bch_fs *c, |
1211 | u64 *last_seq, |
1212 | u64 *blacklist_seq, |
1213 | u64 *start_seq) |
1214 | { |
1215 | struct journal_list jlist; |
1216 | struct journal_replay *i, **_i, *prev = NULL; |
1217 | struct genradix_iter radix_iter; |
1218 | struct printbuf buf = PRINTBUF; |
1219 | bool degraded = false, last_write_torn = false; |
1220 | u64 seq; |
1221 | int ret = 0; |
1222 | |
1223 | closure_init_stack(cl: &jlist.cl); |
1224 | mutex_init(&jlist.lock); |
1225 | jlist.last_seq = 0; |
1226 | jlist.ret = 0; |
1227 | |
1228 | for_each_member_device(c, ca) { |
1229 | if (!c->opts.fsck && |
1230 | !(bch2_dev_has_data(c, ca) & (1 << BCH_DATA_journal))) |
1231 | continue; |
1232 | |
1233 | if ((ca->mi.state == BCH_MEMBER_STATE_rw || |
1234 | ca->mi.state == BCH_MEMBER_STATE_ro) && |
1235 | percpu_ref_tryget(ref: &ca->io_ref)) |
1236 | closure_call(cl: &ca->journal.read, |
1237 | fn: bch2_journal_read_device, |
1238 | wq: system_unbound_wq, |
1239 | parent: &jlist.cl); |
1240 | else |
1241 | degraded = true; |
1242 | } |
1243 | |
1244 | closure_sync(cl: &jlist.cl); |
1245 | |
1246 | if (jlist.ret) |
1247 | return jlist.ret; |
1248 | |
1249 | *last_seq = 0; |
1250 | *start_seq = 0; |
1251 | *blacklist_seq = 0; |
1252 | |
1253 | /* |
1254 | * Find most recent flush entry, and ignore newer non flush entries - |
1255 | * those entries will be blacklisted: |
1256 | */ |
1257 | genradix_for_each_reverse(&c->journal_entries, radix_iter, _i) { |
1258 | enum bkey_invalid_flags flags = BKEY_INVALID_JOURNAL; |
1259 | |
1260 | i = *_i; |
1261 | |
1262 | if (journal_replay_ignore(i)) |
1263 | continue; |
1264 | |
1265 | if (!*start_seq) |
1266 | *blacklist_seq = *start_seq = le64_to_cpu(i->j.seq) + 1; |
1267 | |
1268 | if (JSET_NO_FLUSH(k: &i->j)) { |
1269 | i->ignore_blacklisted = true; |
1270 | continue; |
1271 | } |
1272 | |
1273 | if (!last_write_torn && !i->csum_good) { |
1274 | last_write_torn = true; |
1275 | i->ignore_blacklisted = true; |
1276 | continue; |
1277 | } |
1278 | |
1279 | if (journal_entry_err_on(le64_to_cpu(i->j.last_seq) > le64_to_cpu(i->j.seq), |
1280 | c, le32_to_cpu(i->j.version), &i->j, NULL, |
1281 | jset_last_seq_newer_than_seq, |
1282 | "invalid journal entry: last_seq > seq (%llu > %llu)" , |
1283 | le64_to_cpu(i->j.last_seq), |
1284 | le64_to_cpu(i->j.seq))) |
1285 | i->j.last_seq = i->j.seq; |
1286 | |
1287 | *last_seq = le64_to_cpu(i->j.last_seq); |
1288 | *blacklist_seq = le64_to_cpu(i->j.seq) + 1; |
1289 | break; |
1290 | } |
1291 | |
1292 | if (!*start_seq) { |
1293 | bch_info(c, "journal read done, but no entries found" ); |
1294 | return 0; |
1295 | } |
1296 | |
1297 | if (!*last_seq) { |
1298 | fsck_err(c, dirty_but_no_journal_entries_post_drop_nonflushes, |
1299 | "journal read done, but no entries found after dropping non-flushes" ); |
1300 | return 0; |
1301 | } |
1302 | |
1303 | bch_info(c, "journal read done, replaying entries %llu-%llu" , |
1304 | *last_seq, *blacklist_seq - 1); |
1305 | |
1306 | if (*start_seq != *blacklist_seq) |
1307 | bch_info(c, "dropped unflushed entries %llu-%llu" , |
1308 | *blacklist_seq, *start_seq - 1); |
1309 | |
1310 | /* Drop blacklisted entries and entries older than last_seq: */ |
1311 | genradix_for_each(&c->journal_entries, radix_iter, _i) { |
1312 | i = *_i; |
1313 | |
1314 | if (journal_replay_ignore(i)) |
1315 | continue; |
1316 | |
1317 | seq = le64_to_cpu(i->j.seq); |
1318 | if (seq < *last_seq) { |
1319 | journal_replay_free(c, i, blacklisted: false); |
1320 | continue; |
1321 | } |
1322 | |
1323 | if (bch2_journal_seq_is_blacklisted(c, seq, true)) { |
1324 | fsck_err_on(!JSET_NO_FLUSH(&i->j), c, |
1325 | jset_seq_blacklisted, |
1326 | "found blacklisted journal entry %llu" , seq); |
1327 | i->ignore_blacklisted = true; |
1328 | } |
1329 | } |
1330 | |
1331 | /* Check for missing entries: */ |
1332 | seq = *last_seq; |
1333 | genradix_for_each(&c->journal_entries, radix_iter, _i) { |
1334 | i = *_i; |
1335 | |
1336 | if (journal_replay_ignore(i)) |
1337 | continue; |
1338 | |
1339 | BUG_ON(seq > le64_to_cpu(i->j.seq)); |
1340 | |
1341 | while (seq < le64_to_cpu(i->j.seq)) { |
1342 | u64 missing_start, missing_end; |
1343 | struct printbuf buf1 = PRINTBUF, buf2 = PRINTBUF; |
1344 | |
1345 | while (seq < le64_to_cpu(i->j.seq) && |
1346 | bch2_journal_seq_is_blacklisted(c, seq, false)) |
1347 | seq++; |
1348 | |
1349 | if (seq == le64_to_cpu(i->j.seq)) |
1350 | break; |
1351 | |
1352 | missing_start = seq; |
1353 | |
1354 | while (seq < le64_to_cpu(i->j.seq) && |
1355 | !bch2_journal_seq_is_blacklisted(c, seq, false)) |
1356 | seq++; |
1357 | |
1358 | if (prev) { |
1359 | bch2_journal_ptrs_to_text(out: &buf1, c, j: prev); |
1360 | prt_printf(&buf1, " size %zu" , vstruct_sectors(&prev->j, c->block_bits)); |
1361 | } else |
1362 | prt_printf(&buf1, "(none)" ); |
1363 | bch2_journal_ptrs_to_text(out: &buf2, c, j: i); |
1364 | |
1365 | missing_end = seq - 1; |
1366 | fsck_err(c, journal_entries_missing, |
1367 | "journal entries %llu-%llu missing! (replaying %llu-%llu)\n" |
1368 | " prev at %s\n" |
1369 | " next at %s" , |
1370 | missing_start, missing_end, |
1371 | *last_seq, *blacklist_seq - 1, |
1372 | buf1.buf, buf2.buf); |
1373 | |
1374 | printbuf_exit(&buf1); |
1375 | printbuf_exit(&buf2); |
1376 | } |
1377 | |
1378 | prev = i; |
1379 | seq++; |
1380 | } |
1381 | |
1382 | genradix_for_each(&c->journal_entries, radix_iter, _i) { |
1383 | struct bch_replicas_padded replicas = { |
1384 | .e.data_type = BCH_DATA_journal, |
1385 | .e.nr_required = 1, |
1386 | }; |
1387 | |
1388 | i = *_i; |
1389 | if (journal_replay_ignore(i)) |
1390 | continue; |
1391 | |
1392 | darray_for_each(i->ptrs, ptr) { |
1393 | struct bch_dev *ca = bch_dev_bkey_exists(c, idx: ptr->dev); |
1394 | |
1395 | if (!ptr->csum_good) |
1396 | bch_err_dev_offset(ca, ptr->sector, |
1397 | "invalid journal checksum, seq %llu%s" , |
1398 | le64_to_cpu(i->j.seq), |
1399 | i->csum_good ? " (had good copy on another device)" : "" ); |
1400 | } |
1401 | |
1402 | ret = jset_validate(c, |
1403 | ca: bch_dev_bkey_exists(c, idx: i->ptrs.data[0].dev), |
1404 | jset: &i->j, |
1405 | sector: i->ptrs.data[0].sector, |
1406 | READ); |
1407 | if (ret) |
1408 | goto err; |
1409 | |
1410 | darray_for_each(i->ptrs, ptr) |
1411 | replicas.e.devs[replicas.e.nr_devs++] = ptr->dev; |
1412 | |
1413 | bch2_replicas_entry_sort(&replicas.e); |
1414 | |
1415 | printbuf_reset(buf: &buf); |
1416 | bch2_replicas_entry_to_text(&buf, &replicas.e); |
1417 | |
1418 | if (!degraded && |
1419 | !bch2_replicas_marked(c, &replicas.e) && |
1420 | (le64_to_cpu(i->j.seq) == *last_seq || |
1421 | fsck_err(c, journal_entry_replicas_not_marked, |
1422 | "superblock not marked as containing replicas for journal entry %llu\n %s" , |
1423 | le64_to_cpu(i->j.seq), buf.buf))) { |
1424 | ret = bch2_mark_replicas(c, &replicas.e); |
1425 | if (ret) |
1426 | goto err; |
1427 | } |
1428 | } |
1429 | err: |
1430 | fsck_err: |
1431 | printbuf_exit(&buf); |
1432 | return ret; |
1433 | } |
1434 | |
1435 | /* journal write: */ |
1436 | |
1437 | static void __journal_write_alloc(struct journal *j, |
1438 | struct journal_buf *w, |
1439 | struct dev_alloc_list *devs_sorted, |
1440 | unsigned sectors, |
1441 | unsigned *replicas, |
1442 | unsigned replicas_want) |
1443 | { |
1444 | struct bch_fs *c = container_of(j, struct bch_fs, journal); |
1445 | struct journal_device *ja; |
1446 | struct bch_dev *ca; |
1447 | unsigned i; |
1448 | |
1449 | if (*replicas >= replicas_want) |
1450 | return; |
1451 | |
1452 | for (i = 0; i < devs_sorted->nr; i++) { |
1453 | ca = rcu_dereference(c->devs[devs_sorted->devs[i]]); |
1454 | if (!ca) |
1455 | continue; |
1456 | |
1457 | ja = &ca->journal; |
1458 | |
1459 | /* |
1460 | * Check that we can use this device, and aren't already using |
1461 | * it: |
1462 | */ |
1463 | if (!ca->mi.durability || |
1464 | ca->mi.state != BCH_MEMBER_STATE_rw || |
1465 | !ja->nr || |
1466 | bch2_bkey_has_device_c(bkey_i_to_s_c(k: &w->key), ca->dev_idx) || |
1467 | sectors > ja->sectors_free) |
1468 | continue; |
1469 | |
1470 | bch2_dev_stripe_increment(ca, &j->wp.stripe); |
1471 | |
1472 | bch2_bkey_append_ptr(k: &w->key, |
1473 | ptr: (struct bch_extent_ptr) { |
1474 | .offset = bucket_to_sector(ca, |
1475 | b: ja->buckets[ja->cur_idx]) + |
1476 | ca->mi.bucket_size - |
1477 | ja->sectors_free, |
1478 | .dev = ca->dev_idx, |
1479 | }); |
1480 | |
1481 | ja->sectors_free -= sectors; |
1482 | ja->bucket_seq[ja->cur_idx] = le64_to_cpu(w->data->seq); |
1483 | |
1484 | *replicas += ca->mi.durability; |
1485 | |
1486 | if (*replicas >= replicas_want) |
1487 | break; |
1488 | } |
1489 | } |
1490 | |
1491 | /** |
1492 | * journal_write_alloc - decide where to write next journal entry |
1493 | * |
1494 | * @j: journal object |
1495 | * @w: journal buf (entry to be written) |
1496 | * |
1497 | * Returns: 0 on success, or -EROFS on failure |
1498 | */ |
1499 | static int journal_write_alloc(struct journal *j, struct journal_buf *w) |
1500 | { |
1501 | struct bch_fs *c = container_of(j, struct bch_fs, journal); |
1502 | struct bch_devs_mask devs; |
1503 | struct journal_device *ja; |
1504 | struct bch_dev *ca; |
1505 | struct dev_alloc_list devs_sorted; |
1506 | unsigned sectors = vstruct_sectors(w->data, c->block_bits); |
1507 | unsigned target = c->opts.metadata_target ?: |
1508 | c->opts.foreground_target; |
1509 | unsigned i, replicas = 0, replicas_want = |
1510 | READ_ONCE(c->opts.metadata_replicas); |
1511 | unsigned replicas_need = min_t(unsigned, replicas_want, |
1512 | READ_ONCE(c->opts.metadata_replicas_required)); |
1513 | |
1514 | rcu_read_lock(); |
1515 | retry: |
1516 | devs = target_rw_devs(c, data_type: BCH_DATA_journal, target); |
1517 | |
1518 | devs_sorted = bch2_dev_alloc_list(c, &j->wp.stripe, &devs); |
1519 | |
1520 | __journal_write_alloc(j, w, devs_sorted: &devs_sorted, |
1521 | sectors, replicas: &replicas, replicas_want); |
1522 | |
1523 | if (replicas >= replicas_want) |
1524 | goto done; |
1525 | |
1526 | for (i = 0; i < devs_sorted.nr; i++) { |
1527 | ca = rcu_dereference(c->devs[devs_sorted.devs[i]]); |
1528 | if (!ca) |
1529 | continue; |
1530 | |
1531 | ja = &ca->journal; |
1532 | |
1533 | if (sectors > ja->sectors_free && |
1534 | sectors <= ca->mi.bucket_size && |
1535 | bch2_journal_dev_buckets_available(j, ja, |
1536 | journal_space_discarded)) { |
1537 | ja->cur_idx = (ja->cur_idx + 1) % ja->nr; |
1538 | ja->sectors_free = ca->mi.bucket_size; |
1539 | |
1540 | /* |
1541 | * ja->bucket_seq[ja->cur_idx] must always have |
1542 | * something sensible: |
1543 | */ |
1544 | ja->bucket_seq[ja->cur_idx] = le64_to_cpu(w->data->seq); |
1545 | } |
1546 | } |
1547 | |
1548 | __journal_write_alloc(j, w, devs_sorted: &devs_sorted, |
1549 | sectors, replicas: &replicas, replicas_want); |
1550 | |
1551 | if (replicas < replicas_want && target) { |
1552 | /* Retry from all devices: */ |
1553 | target = 0; |
1554 | goto retry; |
1555 | } |
1556 | done: |
1557 | rcu_read_unlock(); |
1558 | |
1559 | BUG_ON(bkey_val_u64s(&w->key.k) > BCH_REPLICAS_MAX); |
1560 | |
1561 | return replicas >= replicas_need ? 0 : -EROFS; |
1562 | } |
1563 | |
1564 | static void journal_buf_realloc(struct journal *j, struct journal_buf *buf) |
1565 | { |
1566 | struct bch_fs *c = container_of(j, struct bch_fs, journal); |
1567 | |
1568 | /* we aren't holding j->lock: */ |
1569 | unsigned new_size = READ_ONCE(j->buf_size_want); |
1570 | void *new_buf; |
1571 | |
1572 | if (buf->buf_size >= new_size) |
1573 | return; |
1574 | |
1575 | size_t btree_write_buffer_size = new_size / 64; |
1576 | |
1577 | if (bch2_btree_write_buffer_resize(c, btree_write_buffer_size)) |
1578 | return; |
1579 | |
1580 | new_buf = kvmalloc(size: new_size, GFP_NOFS|__GFP_NOWARN); |
1581 | if (!new_buf) |
1582 | return; |
1583 | |
1584 | memcpy(new_buf, buf->data, buf->buf_size); |
1585 | |
1586 | spin_lock(lock: &j->lock); |
1587 | swap(buf->data, new_buf); |
1588 | swap(buf->buf_size, new_size); |
1589 | spin_unlock(lock: &j->lock); |
1590 | |
1591 | kvfree(addr: new_buf); |
1592 | } |
1593 | |
1594 | static inline struct journal_buf *journal_last_unwritten_buf(struct journal *j) |
1595 | { |
1596 | return j->buf + (journal_last_unwritten_seq(j) & JOURNAL_BUF_MASK); |
1597 | } |
1598 | |
1599 | static CLOSURE_CALLBACK(journal_write_done) |
1600 | { |
1601 | closure_type(w, struct journal_buf, io); |
1602 | struct journal *j = container_of(w, struct journal, buf[w->idx]); |
1603 | struct bch_fs *c = container_of(j, struct bch_fs, journal); |
1604 | struct bch_replicas_padded replicas; |
1605 | union journal_res_state old, new; |
1606 | u64 v, seq = le64_to_cpu(w->data->seq); |
1607 | int err = 0; |
1608 | |
1609 | bch2_time_stats_update(stats: !JSET_NO_FLUSH(k: w->data) |
1610 | ? j->flush_write_time |
1611 | : j->noflush_write_time, start: j->write_start_time); |
1612 | |
1613 | if (!w->devs_written.nr) { |
1614 | bch_err(c, "unable to write journal to sufficient devices" ); |
1615 | err = -EIO; |
1616 | } else { |
1617 | bch2_devlist_to_replicas(&replicas.e, BCH_DATA_journal, |
1618 | w->devs_written); |
1619 | if (bch2_mark_replicas(c, &replicas.e)) |
1620 | err = -EIO; |
1621 | } |
1622 | |
1623 | if (err) |
1624 | bch2_fatal_error(c); |
1625 | |
1626 | closure_debug_destroy(cl); |
1627 | |
1628 | spin_lock(lock: &j->lock); |
1629 | if (seq >= j->pin.front) |
1630 | journal_seq_pin(j, seq)->devs = w->devs_written; |
1631 | if (err && (!j->err_seq || seq < j->err_seq)) |
1632 | j->err_seq = seq; |
1633 | w->write_done = true; |
1634 | |
1635 | bool completed = false; |
1636 | |
1637 | for (seq = journal_last_unwritten_seq(j); |
1638 | seq <= journal_cur_seq(j); |
1639 | seq++) { |
1640 | w = j->buf + (seq & JOURNAL_BUF_MASK); |
1641 | if (!w->write_done) |
1642 | break; |
1643 | |
1644 | if (!j->err_seq && !JSET_NO_FLUSH(k: w->data)) { |
1645 | j->flushed_seq_ondisk = seq; |
1646 | j->last_seq_ondisk = w->last_seq; |
1647 | |
1648 | bch2_do_discards(c); |
1649 | closure_wake_up(list: &c->freelist_wait); |
1650 | bch2_reset_alloc_cursors(c); |
1651 | } |
1652 | |
1653 | j->seq_ondisk = seq; |
1654 | |
1655 | /* |
1656 | * Updating last_seq_ondisk may let bch2_journal_reclaim_work() discard |
1657 | * more buckets: |
1658 | * |
1659 | * Must come before signaling write completion, for |
1660 | * bch2_fs_journal_stop(): |
1661 | */ |
1662 | if (j->watermark != BCH_WATERMARK_stripe) |
1663 | journal_reclaim_kick(j: &c->journal); |
1664 | |
1665 | v = atomic64_read(v: &j->reservations.counter); |
1666 | do { |
1667 | old.v = new.v = v; |
1668 | BUG_ON(journal_state_count(new, new.unwritten_idx)); |
1669 | BUG_ON(new.unwritten_idx != (seq & JOURNAL_BUF_MASK)); |
1670 | |
1671 | new.unwritten_idx++; |
1672 | } while ((v = atomic64_cmpxchg(v: &j->reservations.counter, old: old.v, new: new.v)) != old.v); |
1673 | |
1674 | closure_wake_up(list: &w->wait); |
1675 | completed = true; |
1676 | } |
1677 | |
1678 | if (completed) { |
1679 | bch2_journal_reclaim_fast(j); |
1680 | bch2_journal_space_available(j); |
1681 | |
1682 | track_event_change(stats: &c->times[BCH_TIME_blocked_journal_max_in_flight], v: false); |
1683 | |
1684 | journal_wake(j); |
1685 | } |
1686 | |
1687 | if (journal_last_unwritten_seq(j) == journal_cur_seq(j) && |
1688 | new.cur_entry_offset < JOURNAL_ENTRY_CLOSED_VAL) { |
1689 | struct journal_buf *buf = journal_cur_buf(j); |
1690 | long delta = buf->expires - jiffies; |
1691 | |
1692 | /* |
1693 | * We don't close a journal entry to write it while there's |
1694 | * previous entries still in flight - the current journal entry |
1695 | * might want to be written now: |
1696 | */ |
1697 | mod_delayed_work(wq: j->wq, dwork: &j->write_work, max(0L, delta)); |
1698 | } |
1699 | |
1700 | spin_unlock(lock: &j->lock); |
1701 | } |
1702 | |
1703 | static void journal_write_endio(struct bio *bio) |
1704 | { |
1705 | struct journal_bio *jbio = container_of(bio, struct journal_bio, bio); |
1706 | struct bch_dev *ca = jbio->ca; |
1707 | struct journal *j = &ca->fs->journal; |
1708 | struct journal_buf *w = j->buf + jbio->buf_idx; |
1709 | |
1710 | if (bch2_dev_io_err_on(bio->bi_status, ca, BCH_MEMBER_ERROR_write, |
1711 | "error writing journal entry %llu: %s" , |
1712 | le64_to_cpu(w->data->seq), |
1713 | bch2_blk_status_to_str(bio->bi_status)) || |
1714 | bch2_meta_write_fault("journal" )) { |
1715 | unsigned long flags; |
1716 | |
1717 | spin_lock_irqsave(&j->err_lock, flags); |
1718 | bch2_dev_list_drop_dev(devs: &w->devs_written, dev: ca->dev_idx); |
1719 | spin_unlock_irqrestore(lock: &j->err_lock, flags); |
1720 | } |
1721 | |
1722 | closure_put(cl: &w->io); |
1723 | percpu_ref_put(ref: &ca->io_ref); |
1724 | } |
1725 | |
1726 | static CLOSURE_CALLBACK(journal_write_submit) |
1727 | { |
1728 | closure_type(w, struct journal_buf, io); |
1729 | struct journal *j = container_of(w, struct journal, buf[w->idx]); |
1730 | struct bch_fs *c = container_of(j, struct bch_fs, journal); |
1731 | unsigned sectors = vstruct_sectors(w->data, c->block_bits); |
1732 | |
1733 | extent_for_each_ptr(bkey_i_to_s_extent(&w->key), ptr) { |
1734 | struct bch_dev *ca = bch_dev_bkey_exists(c, idx: ptr->dev); |
1735 | struct journal_device *ja = &ca->journal; |
1736 | |
1737 | if (!percpu_ref_tryget(ref: &ca->io_ref)) { |
1738 | /* XXX: fix this */ |
1739 | bch_err(c, "missing device for journal write\n" ); |
1740 | continue; |
1741 | } |
1742 | |
1743 | this_cpu_add(ca->io_done->sectors[WRITE][BCH_DATA_journal], |
1744 | sectors); |
1745 | |
1746 | struct bio *bio = &ja->bio[w->idx]->bio; |
1747 | bio_reset(bio, bdev: ca->disk_sb.bdev, opf: REQ_OP_WRITE|REQ_SYNC|REQ_META); |
1748 | bio->bi_iter.bi_sector = ptr->offset; |
1749 | bio->bi_end_io = journal_write_endio; |
1750 | bio->bi_private = ca; |
1751 | |
1752 | BUG_ON(bio->bi_iter.bi_sector == ca->prev_journal_sector); |
1753 | ca->prev_journal_sector = bio->bi_iter.bi_sector; |
1754 | |
1755 | if (!JSET_NO_FLUSH(k: w->data)) |
1756 | bio->bi_opf |= REQ_FUA; |
1757 | if (!JSET_NO_FLUSH(k: w->data) && !w->separate_flush) |
1758 | bio->bi_opf |= REQ_PREFLUSH; |
1759 | |
1760 | bch2_bio_map(bio, base: w->data, sectors << 9); |
1761 | |
1762 | trace_and_count(c, journal_write, bio); |
1763 | closure_bio_submit(bio, cl); |
1764 | |
1765 | ja->bucket_seq[ja->cur_idx] = le64_to_cpu(w->data->seq); |
1766 | } |
1767 | |
1768 | continue_at(cl, journal_write_done, j->wq); |
1769 | } |
1770 | |
1771 | static CLOSURE_CALLBACK(journal_write_preflush) |
1772 | { |
1773 | closure_type(w, struct journal_buf, io); |
1774 | struct journal *j = container_of(w, struct journal, buf[w->idx]); |
1775 | struct bch_fs *c = container_of(j, struct bch_fs, journal); |
1776 | |
1777 | if (j->seq_ondisk + 1 != le64_to_cpu(w->data->seq)) { |
1778 | spin_lock(lock: &j->lock); |
1779 | closure_wait(list: &j->async_wait, cl); |
1780 | spin_unlock(lock: &j->lock); |
1781 | |
1782 | continue_at(cl, journal_write_preflush, j->wq); |
1783 | return; |
1784 | } |
1785 | |
1786 | if (w->separate_flush) { |
1787 | for_each_rw_member(c, ca) { |
1788 | percpu_ref_get(ref: &ca->io_ref); |
1789 | |
1790 | struct journal_device *ja = &ca->journal; |
1791 | struct bio *bio = &ja->bio[w->idx]->bio; |
1792 | bio_reset(bio, bdev: ca->disk_sb.bdev, |
1793 | opf: REQ_OP_WRITE|REQ_SYNC|REQ_META|REQ_PREFLUSH); |
1794 | bio->bi_end_io = journal_write_endio; |
1795 | bio->bi_private = ca; |
1796 | closure_bio_submit(bio, cl); |
1797 | } |
1798 | |
1799 | continue_at(cl, journal_write_submit, j->wq); |
1800 | } else { |
1801 | /* |
1802 | * no need to punt to another work item if we're not waiting on |
1803 | * preflushes |
1804 | */ |
1805 | journal_write_submit(ws: &cl->work); |
1806 | } |
1807 | } |
1808 | |
1809 | static int bch2_journal_write_prep(struct journal *j, struct journal_buf *w) |
1810 | { |
1811 | struct bch_fs *c = container_of(j, struct bch_fs, journal); |
1812 | struct jset_entry *start, *end; |
1813 | struct jset *jset = w->data; |
1814 | struct journal_keys_to_wb wb = { NULL }; |
1815 | unsigned sectors, bytes, u64s; |
1816 | unsigned long btree_roots_have = 0; |
1817 | bool validate_before_checksum = false; |
1818 | u64 seq = le64_to_cpu(jset->seq); |
1819 | int ret; |
1820 | |
1821 | /* |
1822 | * Simple compaction, dropping empty jset_entries (from journal |
1823 | * reservations that weren't fully used) and merging jset_entries that |
1824 | * can be. |
1825 | * |
1826 | * If we wanted to be really fancy here, we could sort all the keys in |
1827 | * the jset and drop keys that were overwritten - probably not worth it: |
1828 | */ |
1829 | vstruct_for_each(jset, i) { |
1830 | unsigned u64s = le16_to_cpu(i->u64s); |
1831 | |
1832 | /* Empty entry: */ |
1833 | if (!u64s) |
1834 | continue; |
1835 | |
1836 | /* |
1837 | * New btree roots are set by journalling them; when the journal |
1838 | * entry gets written we have to propagate them to |
1839 | * c->btree_roots |
1840 | * |
1841 | * But, every journal entry we write has to contain all the |
1842 | * btree roots (at least for now); so after we copy btree roots |
1843 | * to c->btree_roots we have to get any missing btree roots and |
1844 | * add them to this journal entry: |
1845 | */ |
1846 | switch (i->type) { |
1847 | case BCH_JSET_ENTRY_btree_root: |
1848 | bch2_journal_entry_to_btree_root(c, i); |
1849 | __set_bit(i->btree_id, &btree_roots_have); |
1850 | break; |
1851 | case BCH_JSET_ENTRY_write_buffer_keys: |
1852 | EBUG_ON(!w->need_flush_to_write_buffer); |
1853 | |
1854 | if (!wb.wb) |
1855 | bch2_journal_keys_to_write_buffer_start(c, &wb, seq); |
1856 | |
1857 | jset_entry_for_each_key(i, k) { |
1858 | ret = bch2_journal_key_to_wb(c, dst: &wb, btree: i->btree_id, k); |
1859 | if (ret) { |
1860 | bch2_fs_fatal_error(c, "flushing journal keys to btree write buffer: %s" , |
1861 | bch2_err_str(ret)); |
1862 | bch2_journal_keys_to_write_buffer_end(c, &wb); |
1863 | return ret; |
1864 | } |
1865 | } |
1866 | i->type = BCH_JSET_ENTRY_btree_keys; |
1867 | break; |
1868 | } |
1869 | } |
1870 | |
1871 | if (wb.wb) |
1872 | bch2_journal_keys_to_write_buffer_end(c, &wb); |
1873 | |
1874 | spin_lock(lock: &c->journal.lock); |
1875 | w->need_flush_to_write_buffer = false; |
1876 | spin_unlock(lock: &c->journal.lock); |
1877 | |
1878 | start = end = vstruct_last(jset); |
1879 | |
1880 | end = bch2_btree_roots_to_journal_entries(c, end, btree_roots_have); |
1881 | |
1882 | struct jset_entry_datetime *d = |
1883 | container_of(jset_entry_init(&end, sizeof(*d)), struct jset_entry_datetime, entry); |
1884 | d->entry.type = BCH_JSET_ENTRY_datetime; |
1885 | d->seconds = cpu_to_le64(ktime_get_real_seconds()); |
1886 | |
1887 | bch2_journal_super_entries_add_common(c, &end, seq); |
1888 | u64s = (u64 *) end - (u64 *) start; |
1889 | |
1890 | WARN_ON(u64s > j->entry_u64s_reserved); |
1891 | |
1892 | le32_add_cpu(var: &jset->u64s, val: u64s); |
1893 | |
1894 | sectors = vstruct_sectors(jset, c->block_bits); |
1895 | bytes = vstruct_bytes(jset); |
1896 | |
1897 | if (sectors > w->sectors) { |
1898 | bch2_fs_fatal_error(c, ": journal write overran available space, %zu > %u (extra %u reserved %u/%u)" , |
1899 | vstruct_bytes(jset), w->sectors << 9, |
1900 | u64s, w->u64s_reserved, j->entry_u64s_reserved); |
1901 | return -EINVAL; |
1902 | } |
1903 | |
1904 | jset->magic = cpu_to_le64(jset_magic(c)); |
1905 | jset->version = cpu_to_le32(c->sb.version); |
1906 | |
1907 | SET_JSET_BIG_ENDIAN(k: jset, CPU_BIG_ENDIAN); |
1908 | SET_JSET_CSUM_TYPE(k: jset, v: bch2_meta_checksum_type(c)); |
1909 | |
1910 | if (!JSET_NO_FLUSH(k: jset) && journal_entry_empty(j: jset)) |
1911 | j->last_empty_seq = seq; |
1912 | |
1913 | if (bch2_csum_type_is_encryption(type: JSET_CSUM_TYPE(k: jset))) |
1914 | validate_before_checksum = true; |
1915 | |
1916 | if (le32_to_cpu(jset->version) < bcachefs_metadata_version_current) |
1917 | validate_before_checksum = true; |
1918 | |
1919 | if (validate_before_checksum && |
1920 | (ret = jset_validate(c, NULL, jset, sector: 0, WRITE))) |
1921 | return ret; |
1922 | |
1923 | ret = bch2_encrypt(c, JSET_CSUM_TYPE(k: jset), journal_nonce(jset), |
1924 | data: jset->encrypted_start, |
1925 | vstruct_end(jset) - (void *) jset->encrypted_start); |
1926 | if (bch2_fs_fatal_err_on(ret, c, "decrypting journal entry: %s" , bch2_err_str(ret))) |
1927 | return ret; |
1928 | |
1929 | jset->csum = csum_vstruct(c, JSET_CSUM_TYPE(jset), |
1930 | journal_nonce(jset), jset); |
1931 | |
1932 | if (!validate_before_checksum && |
1933 | (ret = jset_validate(c, NULL, jset, sector: 0, WRITE))) |
1934 | return ret; |
1935 | |
1936 | memset((void *) jset + bytes, 0, (sectors << 9) - bytes); |
1937 | return 0; |
1938 | } |
1939 | |
1940 | static int bch2_journal_write_pick_flush(struct journal *j, struct journal_buf *w) |
1941 | { |
1942 | struct bch_fs *c = container_of(j, struct bch_fs, journal); |
1943 | int error = bch2_journal_error(j); |
1944 | |
1945 | /* |
1946 | * If the journal is in an error state - we did an emergency shutdown - |
1947 | * we prefer to continue doing journal writes. We just mark them as |
1948 | * noflush so they'll never be used, but they'll still be visible by the |
1949 | * list_journal tool - this helps in debugging. |
1950 | * |
1951 | * There's a caveat: the first journal write after marking the |
1952 | * superblock dirty must always be a flush write, because on startup |
1953 | * from a clean shutdown we didn't necessarily read the journal and the |
1954 | * new journal write might overwrite whatever was in the journal |
1955 | * previously - we can't leave the journal without any flush writes in |
1956 | * it. |
1957 | * |
1958 | * So if we're in an error state, and we're still starting up, we don't |
1959 | * write anything at all. |
1960 | */ |
1961 | if (error && test_bit(JOURNAL_NEED_FLUSH_WRITE, &j->flags)) |
1962 | return -EIO; |
1963 | |
1964 | if (error || |
1965 | w->noflush || |
1966 | (!w->must_flush && |
1967 | (jiffies - j->last_flush_write) < msecs_to_jiffies(m: c->opts.journal_flush_delay) && |
1968 | test_bit(JOURNAL_MAY_SKIP_FLUSH, &j->flags))) { |
1969 | w->noflush = true; |
1970 | SET_JSET_NO_FLUSH(k: w->data, v: true); |
1971 | w->data->last_seq = 0; |
1972 | w->last_seq = 0; |
1973 | |
1974 | j->nr_noflush_writes++; |
1975 | } else { |
1976 | w->must_flush = true; |
1977 | j->last_flush_write = jiffies; |
1978 | j->nr_flush_writes++; |
1979 | clear_bit(nr: JOURNAL_NEED_FLUSH_WRITE, addr: &j->flags); |
1980 | } |
1981 | |
1982 | return 0; |
1983 | } |
1984 | |
1985 | CLOSURE_CALLBACK(bch2_journal_write) |
1986 | { |
1987 | closure_type(w, struct journal_buf, io); |
1988 | struct journal *j = container_of(w, struct journal, buf[w->idx]); |
1989 | struct bch_fs *c = container_of(j, struct bch_fs, journal); |
1990 | struct bch_replicas_padded replicas; |
1991 | struct printbuf journal_debug_buf = PRINTBUF; |
1992 | unsigned nr_rw_members = 0; |
1993 | int ret; |
1994 | |
1995 | for_each_rw_member(c, ca) |
1996 | nr_rw_members++; |
1997 | |
1998 | BUG_ON(BCH_SB_CLEAN(c->disk_sb.sb)); |
1999 | BUG_ON(!w->write_started); |
2000 | BUG_ON(w->write_allocated); |
2001 | BUG_ON(w->write_done); |
2002 | |
2003 | j->write_start_time = local_clock(); |
2004 | |
2005 | spin_lock(lock: &j->lock); |
2006 | if (nr_rw_members > 1) |
2007 | w->separate_flush = true; |
2008 | |
2009 | ret = bch2_journal_write_pick_flush(j, w); |
2010 | spin_unlock(lock: &j->lock); |
2011 | if (ret) |
2012 | goto err; |
2013 | |
2014 | mutex_lock(&j->buf_lock); |
2015 | journal_buf_realloc(j, buf: w); |
2016 | |
2017 | ret = bch2_journal_write_prep(j, w); |
2018 | mutex_unlock(lock: &j->buf_lock); |
2019 | if (ret) |
2020 | goto err; |
2021 | |
2022 | j->entry_bytes_written += vstruct_bytes(w->data); |
2023 | |
2024 | while (1) { |
2025 | spin_lock(lock: &j->lock); |
2026 | ret = journal_write_alloc(j, w); |
2027 | if (!ret || !j->can_discard) |
2028 | break; |
2029 | |
2030 | spin_unlock(lock: &j->lock); |
2031 | bch2_journal_do_discards(j); |
2032 | } |
2033 | |
2034 | if (ret) { |
2035 | __bch2_journal_debug_to_text(&journal_debug_buf, j); |
2036 | spin_unlock(lock: &j->lock); |
2037 | bch_err(c, "Unable to allocate journal write:\n%s" , |
2038 | journal_debug_buf.buf); |
2039 | printbuf_exit(&journal_debug_buf); |
2040 | goto err; |
2041 | } |
2042 | |
2043 | /* |
2044 | * write is allocated, no longer need to account for it in |
2045 | * bch2_journal_space_available(): |
2046 | */ |
2047 | w->sectors = 0; |
2048 | w->write_allocated = true; |
2049 | |
2050 | /* |
2051 | * journal entry has been compacted and allocated, recalculate space |
2052 | * available: |
2053 | */ |
2054 | bch2_journal_space_available(j); |
2055 | bch2_journal_do_writes(j); |
2056 | spin_unlock(lock: &j->lock); |
2057 | |
2058 | w->devs_written = bch2_bkey_devs(k: bkey_i_to_s_c(k: &w->key)); |
2059 | |
2060 | if (c->opts.nochanges) |
2061 | goto no_io; |
2062 | |
2063 | /* |
2064 | * Mark journal replicas before we submit the write to guarantee |
2065 | * recovery will find the journal entries after a crash. |
2066 | */ |
2067 | bch2_devlist_to_replicas(&replicas.e, BCH_DATA_journal, |
2068 | w->devs_written); |
2069 | ret = bch2_mark_replicas(c, &replicas.e); |
2070 | if (ret) |
2071 | goto err; |
2072 | |
2073 | if (!JSET_NO_FLUSH(k: w->data)) |
2074 | continue_at(cl, journal_write_preflush, j->wq); |
2075 | else |
2076 | continue_at(cl, journal_write_submit, j->wq); |
2077 | return; |
2078 | no_io: |
2079 | continue_at(cl, journal_write_done, j->wq); |
2080 | return; |
2081 | err: |
2082 | bch2_fatal_error(c); |
2083 | continue_at(cl, journal_write_done, j->wq); |
2084 | } |
2085 | |