1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * NILFS B-tree.
4 *
5 * Copyright (C) 2005-2008 Nippon Telegraph and Telephone Corporation.
6 *
7 * Written by Koji Sato.
8 */
9
10#include <linux/slab.h>
11#include <linux/string.h>
12#include <linux/errno.h>
13#include <linux/pagevec.h>
14#include "nilfs.h"
15#include "page.h"
16#include "btnode.h"
17#include "btree.h"
18#include "alloc.h"
19#include "dat.h"
20
21static void __nilfs_btree_init(struct nilfs_bmap *bmap);
22
23static struct nilfs_btree_path *nilfs_btree_alloc_path(void)
24{
25 struct nilfs_btree_path *path;
26 int level = NILFS_BTREE_LEVEL_DATA;
27
28 path = kmem_cache_alloc(cachep: nilfs_btree_path_cache, GFP_NOFS);
29 if (path == NULL)
30 goto out;
31
32 for (; level < NILFS_BTREE_LEVEL_MAX; level++) {
33 path[level].bp_bh = NULL;
34 path[level].bp_sib_bh = NULL;
35 path[level].bp_index = 0;
36 path[level].bp_oldreq.bpr_ptr = NILFS_BMAP_INVALID_PTR;
37 path[level].bp_newreq.bpr_ptr = NILFS_BMAP_INVALID_PTR;
38 path[level].bp_op = NULL;
39 }
40
41out:
42 return path;
43}
44
45static void nilfs_btree_free_path(struct nilfs_btree_path *path)
46{
47 int level = NILFS_BTREE_LEVEL_DATA;
48
49 for (; level < NILFS_BTREE_LEVEL_MAX; level++)
50 brelse(bh: path[level].bp_bh);
51
52 kmem_cache_free(s: nilfs_btree_path_cache, objp: path);
53}
54
55/*
56 * B-tree node operations
57 */
58static int nilfs_btree_get_new_block(const struct nilfs_bmap *btree,
59 __u64 ptr, struct buffer_head **bhp)
60{
61 struct inode *btnc_inode = NILFS_BMAP_I(bmap: btree)->i_assoc_inode;
62 struct address_space *btnc = btnc_inode->i_mapping;
63 struct buffer_head *bh;
64
65 bh = nilfs_btnode_create_block(btnc, blocknr: ptr);
66 if (!bh)
67 return -ENOMEM;
68
69 set_buffer_nilfs_volatile(bh);
70 *bhp = bh;
71 return 0;
72}
73
74static int nilfs_btree_node_get_flags(const struct nilfs_btree_node *node)
75{
76 return node->bn_flags;
77}
78
79static void
80nilfs_btree_node_set_flags(struct nilfs_btree_node *node, int flags)
81{
82 node->bn_flags = flags;
83}
84
85static int nilfs_btree_node_root(const struct nilfs_btree_node *node)
86{
87 return nilfs_btree_node_get_flags(node) & NILFS_BTREE_NODE_ROOT;
88}
89
90static int nilfs_btree_node_get_level(const struct nilfs_btree_node *node)
91{
92 return node->bn_level;
93}
94
95static void
96nilfs_btree_node_set_level(struct nilfs_btree_node *node, int level)
97{
98 node->bn_level = level;
99}
100
101static int nilfs_btree_node_get_nchildren(const struct nilfs_btree_node *node)
102{
103 return le16_to_cpu(node->bn_nchildren);
104}
105
106static void
107nilfs_btree_node_set_nchildren(struct nilfs_btree_node *node, int nchildren)
108{
109 node->bn_nchildren = cpu_to_le16(nchildren);
110}
111
112static int nilfs_btree_node_size(const struct nilfs_bmap *btree)
113{
114 return i_blocksize(node: btree->b_inode);
115}
116
117static int nilfs_btree_nchildren_per_block(const struct nilfs_bmap *btree)
118{
119 return btree->b_nchildren_per_block;
120}
121
122static __le64 *
123nilfs_btree_node_dkeys(const struct nilfs_btree_node *node)
124{
125 return (__le64 *)((char *)(node + 1) +
126 (nilfs_btree_node_root(node) ?
127 0 : NILFS_BTREE_NODE_EXTRA_PAD_SIZE));
128}
129
130static __le64 *
131nilfs_btree_node_dptrs(const struct nilfs_btree_node *node, int ncmax)
132{
133 return (__le64 *)(nilfs_btree_node_dkeys(node) + ncmax);
134}
135
136static __u64
137nilfs_btree_node_get_key(const struct nilfs_btree_node *node, int index)
138{
139 return le64_to_cpu(*(nilfs_btree_node_dkeys(node) + index));
140}
141
142static void
143nilfs_btree_node_set_key(struct nilfs_btree_node *node, int index, __u64 key)
144{
145 *(nilfs_btree_node_dkeys(node) + index) = cpu_to_le64(key);
146}
147
148static __u64
149nilfs_btree_node_get_ptr(const struct nilfs_btree_node *node, int index,
150 int ncmax)
151{
152 return le64_to_cpu(*(nilfs_btree_node_dptrs(node, ncmax) + index));
153}
154
155static void
156nilfs_btree_node_set_ptr(struct nilfs_btree_node *node, int index, __u64 ptr,
157 int ncmax)
158{
159 *(nilfs_btree_node_dptrs(node, ncmax) + index) = cpu_to_le64(ptr);
160}
161
162static void nilfs_btree_node_init(struct nilfs_btree_node *node, int flags,
163 int level, int nchildren, int ncmax,
164 const __u64 *keys, const __u64 *ptrs)
165{
166 __le64 *dkeys;
167 __le64 *dptrs;
168 int i;
169
170 nilfs_btree_node_set_flags(node, flags);
171 nilfs_btree_node_set_level(node, level);
172 nilfs_btree_node_set_nchildren(node, nchildren);
173
174 dkeys = nilfs_btree_node_dkeys(node);
175 dptrs = nilfs_btree_node_dptrs(node, ncmax);
176 for (i = 0; i < nchildren; i++) {
177 dkeys[i] = cpu_to_le64(keys[i]);
178 dptrs[i] = cpu_to_le64(ptrs[i]);
179 }
180}
181
182/* Assume the buffer heads corresponding to left and right are locked. */
183static void nilfs_btree_node_move_left(struct nilfs_btree_node *left,
184 struct nilfs_btree_node *right,
185 int n, int lncmax, int rncmax)
186{
187 __le64 *ldkeys, *rdkeys;
188 __le64 *ldptrs, *rdptrs;
189 int lnchildren, rnchildren;
190
191 ldkeys = nilfs_btree_node_dkeys(node: left);
192 ldptrs = nilfs_btree_node_dptrs(node: left, ncmax: lncmax);
193 lnchildren = nilfs_btree_node_get_nchildren(node: left);
194
195 rdkeys = nilfs_btree_node_dkeys(node: right);
196 rdptrs = nilfs_btree_node_dptrs(node: right, ncmax: rncmax);
197 rnchildren = nilfs_btree_node_get_nchildren(node: right);
198
199 memcpy(ldkeys + lnchildren, rdkeys, n * sizeof(*rdkeys));
200 memcpy(ldptrs + lnchildren, rdptrs, n * sizeof(*rdptrs));
201 memmove(rdkeys, rdkeys + n, (rnchildren - n) * sizeof(*rdkeys));
202 memmove(rdptrs, rdptrs + n, (rnchildren - n) * sizeof(*rdptrs));
203
204 lnchildren += n;
205 rnchildren -= n;
206 nilfs_btree_node_set_nchildren(node: left, nchildren: lnchildren);
207 nilfs_btree_node_set_nchildren(node: right, nchildren: rnchildren);
208}
209
210/* Assume that the buffer heads corresponding to left and right are locked. */
211static void nilfs_btree_node_move_right(struct nilfs_btree_node *left,
212 struct nilfs_btree_node *right,
213 int n, int lncmax, int rncmax)
214{
215 __le64 *ldkeys, *rdkeys;
216 __le64 *ldptrs, *rdptrs;
217 int lnchildren, rnchildren;
218
219 ldkeys = nilfs_btree_node_dkeys(node: left);
220 ldptrs = nilfs_btree_node_dptrs(node: left, ncmax: lncmax);
221 lnchildren = nilfs_btree_node_get_nchildren(node: left);
222
223 rdkeys = nilfs_btree_node_dkeys(node: right);
224 rdptrs = nilfs_btree_node_dptrs(node: right, ncmax: rncmax);
225 rnchildren = nilfs_btree_node_get_nchildren(node: right);
226
227 memmove(rdkeys + n, rdkeys, rnchildren * sizeof(*rdkeys));
228 memmove(rdptrs + n, rdptrs, rnchildren * sizeof(*rdptrs));
229 memcpy(rdkeys, ldkeys + lnchildren - n, n * sizeof(*rdkeys));
230 memcpy(rdptrs, ldptrs + lnchildren - n, n * sizeof(*rdptrs));
231
232 lnchildren -= n;
233 rnchildren += n;
234 nilfs_btree_node_set_nchildren(node: left, nchildren: lnchildren);
235 nilfs_btree_node_set_nchildren(node: right, nchildren: rnchildren);
236}
237
238/* Assume that the buffer head corresponding to node is locked. */
239static void nilfs_btree_node_insert(struct nilfs_btree_node *node, int index,
240 __u64 key, __u64 ptr, int ncmax)
241{
242 __le64 *dkeys;
243 __le64 *dptrs;
244 int nchildren;
245
246 dkeys = nilfs_btree_node_dkeys(node);
247 dptrs = nilfs_btree_node_dptrs(node, ncmax);
248 nchildren = nilfs_btree_node_get_nchildren(node);
249 if (index < nchildren) {
250 memmove(dkeys + index + 1, dkeys + index,
251 (nchildren - index) * sizeof(*dkeys));
252 memmove(dptrs + index + 1, dptrs + index,
253 (nchildren - index) * sizeof(*dptrs));
254 }
255 dkeys[index] = cpu_to_le64(key);
256 dptrs[index] = cpu_to_le64(ptr);
257 nchildren++;
258 nilfs_btree_node_set_nchildren(node, nchildren);
259}
260
261/* Assume that the buffer head corresponding to node is locked. */
262static void nilfs_btree_node_delete(struct nilfs_btree_node *node, int index,
263 __u64 *keyp, __u64 *ptrp, int ncmax)
264{
265 __u64 key;
266 __u64 ptr;
267 __le64 *dkeys;
268 __le64 *dptrs;
269 int nchildren;
270
271 dkeys = nilfs_btree_node_dkeys(node);
272 dptrs = nilfs_btree_node_dptrs(node, ncmax);
273 key = le64_to_cpu(dkeys[index]);
274 ptr = le64_to_cpu(dptrs[index]);
275 nchildren = nilfs_btree_node_get_nchildren(node);
276 if (keyp != NULL)
277 *keyp = key;
278 if (ptrp != NULL)
279 *ptrp = ptr;
280
281 if (index < nchildren - 1) {
282 memmove(dkeys + index, dkeys + index + 1,
283 (nchildren - index - 1) * sizeof(*dkeys));
284 memmove(dptrs + index, dptrs + index + 1,
285 (nchildren - index - 1) * sizeof(*dptrs));
286 }
287 nchildren--;
288 nilfs_btree_node_set_nchildren(node, nchildren);
289}
290
291static int nilfs_btree_node_lookup(const struct nilfs_btree_node *node,
292 __u64 key, int *indexp)
293{
294 __u64 nkey;
295 int index, low, high, s;
296
297 /* binary search */
298 low = 0;
299 high = nilfs_btree_node_get_nchildren(node) - 1;
300 index = 0;
301 s = 0;
302 while (low <= high) {
303 index = (low + high) / 2;
304 nkey = nilfs_btree_node_get_key(node, index);
305 if (nkey == key) {
306 s = 0;
307 goto out;
308 } else if (nkey < key) {
309 low = index + 1;
310 s = -1;
311 } else {
312 high = index - 1;
313 s = 1;
314 }
315 }
316
317 /* adjust index */
318 if (nilfs_btree_node_get_level(node) > NILFS_BTREE_LEVEL_NODE_MIN) {
319 if (s > 0 && index > 0)
320 index--;
321 } else if (s < 0)
322 index++;
323
324 out:
325 *indexp = index;
326
327 return s == 0;
328}
329
330/**
331 * nilfs_btree_node_broken - verify consistency of btree node
332 * @node: btree node block to be examined
333 * @size: node size (in bytes)
334 * @inode: host inode of btree
335 * @blocknr: block number
336 *
337 * Return Value: If node is broken, 1 is returned. Otherwise, 0 is returned.
338 */
339static int nilfs_btree_node_broken(const struct nilfs_btree_node *node,
340 size_t size, struct inode *inode,
341 sector_t blocknr)
342{
343 int level, flags, nchildren;
344 int ret = 0;
345
346 level = nilfs_btree_node_get_level(node);
347 flags = nilfs_btree_node_get_flags(node);
348 nchildren = nilfs_btree_node_get_nchildren(node);
349
350 if (unlikely(level < NILFS_BTREE_LEVEL_NODE_MIN ||
351 level >= NILFS_BTREE_LEVEL_MAX ||
352 (flags & NILFS_BTREE_NODE_ROOT) ||
353 nchildren < 0 ||
354 nchildren > NILFS_BTREE_NODE_NCHILDREN_MAX(size))) {
355 nilfs_crit(inode->i_sb,
356 "bad btree node (ino=%lu, blocknr=%llu): level = %d, flags = 0x%x, nchildren = %d",
357 inode->i_ino, (unsigned long long)blocknr, level,
358 flags, nchildren);
359 ret = 1;
360 }
361 return ret;
362}
363
364/**
365 * nilfs_btree_root_broken - verify consistency of btree root node
366 * @node: btree root node to be examined
367 * @inode: host inode of btree
368 *
369 * Return Value: If node is broken, 1 is returned. Otherwise, 0 is returned.
370 */
371static int nilfs_btree_root_broken(const struct nilfs_btree_node *node,
372 struct inode *inode)
373{
374 int level, flags, nchildren;
375 int ret = 0;
376
377 level = nilfs_btree_node_get_level(node);
378 flags = nilfs_btree_node_get_flags(node);
379 nchildren = nilfs_btree_node_get_nchildren(node);
380
381 if (unlikely(level < NILFS_BTREE_LEVEL_NODE_MIN ||
382 level >= NILFS_BTREE_LEVEL_MAX ||
383 nchildren < 0 ||
384 nchildren > NILFS_BTREE_ROOT_NCHILDREN_MAX)) {
385 nilfs_crit(inode->i_sb,
386 "bad btree root (ino=%lu): level = %d, flags = 0x%x, nchildren = %d",
387 inode->i_ino, level, flags, nchildren);
388 ret = 1;
389 }
390 return ret;
391}
392
393int nilfs_btree_broken_node_block(struct buffer_head *bh)
394{
395 struct inode *inode;
396 int ret;
397
398 if (buffer_nilfs_checked(bh))
399 return 0;
400
401 inode = bh->b_folio->mapping->host;
402 ret = nilfs_btree_node_broken(node: (struct nilfs_btree_node *)bh->b_data,
403 size: bh->b_size, inode, blocknr: bh->b_blocknr);
404 if (likely(!ret))
405 set_buffer_nilfs_checked(bh);
406 return ret;
407}
408
409static struct nilfs_btree_node *
410nilfs_btree_get_root(const struct nilfs_bmap *btree)
411{
412 return (struct nilfs_btree_node *)btree->b_u.u_data;
413}
414
415static struct nilfs_btree_node *
416nilfs_btree_get_nonroot_node(const struct nilfs_btree_path *path, int level)
417{
418 return (struct nilfs_btree_node *)path[level].bp_bh->b_data;
419}
420
421static struct nilfs_btree_node *
422nilfs_btree_get_sib_node(const struct nilfs_btree_path *path, int level)
423{
424 return (struct nilfs_btree_node *)path[level].bp_sib_bh->b_data;
425}
426
427static int nilfs_btree_height(const struct nilfs_bmap *btree)
428{
429 return nilfs_btree_node_get_level(node: nilfs_btree_get_root(btree)) + 1;
430}
431
432static struct nilfs_btree_node *
433nilfs_btree_get_node(const struct nilfs_bmap *btree,
434 const struct nilfs_btree_path *path,
435 int level, int *ncmaxp)
436{
437 struct nilfs_btree_node *node;
438
439 if (level == nilfs_btree_height(btree) - 1) {
440 node = nilfs_btree_get_root(btree);
441 *ncmaxp = NILFS_BTREE_ROOT_NCHILDREN_MAX;
442 } else {
443 node = nilfs_btree_get_nonroot_node(path, level);
444 *ncmaxp = nilfs_btree_nchildren_per_block(btree);
445 }
446 return node;
447}
448
449static int nilfs_btree_bad_node(const struct nilfs_bmap *btree,
450 struct nilfs_btree_node *node, int level)
451{
452 if (unlikely(nilfs_btree_node_get_level(node) != level)) {
453 dump_stack();
454 nilfs_crit(btree->b_inode->i_sb,
455 "btree level mismatch (ino=%lu): %d != %d",
456 btree->b_inode->i_ino,
457 nilfs_btree_node_get_level(node), level);
458 return 1;
459 }
460 return 0;
461}
462
463struct nilfs_btree_readahead_info {
464 struct nilfs_btree_node *node; /* parent node */
465 int max_ra_blocks; /* max nof blocks to read ahead */
466 int index; /* current index on the parent node */
467 int ncmax; /* nof children in the parent node */
468};
469
470static int __nilfs_btree_get_block(const struct nilfs_bmap *btree, __u64 ptr,
471 struct buffer_head **bhp,
472 const struct nilfs_btree_readahead_info *ra)
473{
474 struct inode *btnc_inode = NILFS_BMAP_I(bmap: btree)->i_assoc_inode;
475 struct address_space *btnc = btnc_inode->i_mapping;
476 struct buffer_head *bh, *ra_bh;
477 sector_t submit_ptr = 0;
478 int ret;
479
480 ret = nilfs_btnode_submit_block(btnc, ptr, 0, REQ_OP_READ, &bh,
481 &submit_ptr);
482 if (ret) {
483 if (likely(ret == -EEXIST))
484 goto out_check;
485 if (ret == -ENOENT) {
486 /*
487 * Block address translation failed due to invalid
488 * value of 'ptr'. In this case, return internal code
489 * -EINVAL (broken bmap) to notify bmap layer of fatal
490 * metadata corruption.
491 */
492 ret = -EINVAL;
493 }
494 return ret;
495 }
496
497 if (ra) {
498 int i, n;
499 __u64 ptr2;
500
501 /* read ahead sibling nodes */
502 for (n = ra->max_ra_blocks, i = ra->index + 1;
503 n > 0 && i < ra->ncmax; n--, i++) {
504 ptr2 = nilfs_btree_node_get_ptr(node: ra->node, index: i, ncmax: ra->ncmax);
505
506 ret = nilfs_btnode_submit_block(btnc, ptr2, 0,
507 REQ_OP_READ | REQ_RAHEAD,
508 &ra_bh, &submit_ptr);
509 if (likely(!ret || ret == -EEXIST))
510 brelse(bh: ra_bh);
511 else if (ret != -EBUSY)
512 break;
513 if (!buffer_locked(bh))
514 goto out_no_wait;
515 }
516 }
517
518 wait_on_buffer(bh);
519
520 out_no_wait:
521 if (!buffer_uptodate(bh)) {
522 nilfs_err(btree->b_inode->i_sb,
523 "I/O error reading b-tree node block (ino=%lu, blocknr=%llu)",
524 btree->b_inode->i_ino, (unsigned long long)ptr);
525 brelse(bh);
526 return -EIO;
527 }
528
529 out_check:
530 if (nilfs_btree_broken_node_block(bh)) {
531 clear_buffer_uptodate(bh);
532 brelse(bh);
533 return -EINVAL;
534 }
535
536 *bhp = bh;
537 return 0;
538}
539
540static int nilfs_btree_get_block(const struct nilfs_bmap *btree, __u64 ptr,
541 struct buffer_head **bhp)
542{
543 return __nilfs_btree_get_block(btree, ptr, bhp, NULL);
544}
545
546static int nilfs_btree_do_lookup(const struct nilfs_bmap *btree,
547 struct nilfs_btree_path *path,
548 __u64 key, __u64 *ptrp, int minlevel,
549 int readahead)
550{
551 struct nilfs_btree_node *node;
552 struct nilfs_btree_readahead_info p, *ra;
553 __u64 ptr;
554 int level, index, found, ncmax, ret;
555
556 node = nilfs_btree_get_root(btree);
557 level = nilfs_btree_node_get_level(node);
558 if (level < minlevel || nilfs_btree_node_get_nchildren(node) <= 0)
559 return -ENOENT;
560
561 found = nilfs_btree_node_lookup(node, key, indexp: &index);
562 ptr = nilfs_btree_node_get_ptr(node, index,
563 NILFS_BTREE_ROOT_NCHILDREN_MAX);
564 path[level].bp_bh = NULL;
565 path[level].bp_index = index;
566
567 ncmax = nilfs_btree_nchildren_per_block(btree);
568
569 while (--level >= minlevel) {
570 ra = NULL;
571 if (level == NILFS_BTREE_LEVEL_NODE_MIN && readahead) {
572 p.node = nilfs_btree_get_node(btree, path, level: level + 1,
573 ncmaxp: &p.ncmax);
574 p.index = index;
575 p.max_ra_blocks = 7;
576 ra = &p;
577 }
578 ret = __nilfs_btree_get_block(btree, ptr, bhp: &path[level].bp_bh,
579 ra);
580 if (ret < 0)
581 return ret;
582
583 node = nilfs_btree_get_nonroot_node(path, level);
584 if (nilfs_btree_bad_node(btree, node, level))
585 return -EINVAL;
586 if (!found)
587 found = nilfs_btree_node_lookup(node, key, indexp: &index);
588 else
589 index = 0;
590 if (index < ncmax) {
591 ptr = nilfs_btree_node_get_ptr(node, index, ncmax);
592 } else {
593 WARN_ON(found || level != NILFS_BTREE_LEVEL_NODE_MIN);
594 /* insert */
595 ptr = NILFS_BMAP_INVALID_PTR;
596 }
597 path[level].bp_index = index;
598 }
599 if (!found)
600 return -ENOENT;
601
602 if (ptrp != NULL)
603 *ptrp = ptr;
604
605 return 0;
606}
607
608static int nilfs_btree_do_lookup_last(const struct nilfs_bmap *btree,
609 struct nilfs_btree_path *path,
610 __u64 *keyp, __u64 *ptrp)
611{
612 struct nilfs_btree_node *node;
613 __u64 ptr;
614 int index, level, ncmax, ret;
615
616 node = nilfs_btree_get_root(btree);
617 index = nilfs_btree_node_get_nchildren(node) - 1;
618 if (index < 0)
619 return -ENOENT;
620 level = nilfs_btree_node_get_level(node);
621 ptr = nilfs_btree_node_get_ptr(node, index,
622 NILFS_BTREE_ROOT_NCHILDREN_MAX);
623 path[level].bp_bh = NULL;
624 path[level].bp_index = index;
625 ncmax = nilfs_btree_nchildren_per_block(btree);
626
627 for (level--; level > 0; level--) {
628 ret = nilfs_btree_get_block(btree, ptr, bhp: &path[level].bp_bh);
629 if (ret < 0)
630 return ret;
631 node = nilfs_btree_get_nonroot_node(path, level);
632 if (nilfs_btree_bad_node(btree, node, level))
633 return -EINVAL;
634 index = nilfs_btree_node_get_nchildren(node) - 1;
635 ptr = nilfs_btree_node_get_ptr(node, index, ncmax);
636 path[level].bp_index = index;
637 }
638
639 if (keyp != NULL)
640 *keyp = nilfs_btree_node_get_key(node, index);
641 if (ptrp != NULL)
642 *ptrp = ptr;
643
644 return 0;
645}
646
647/**
648 * nilfs_btree_get_next_key - get next valid key from btree path array
649 * @btree: bmap struct of btree
650 * @path: array of nilfs_btree_path struct
651 * @minlevel: start level
652 * @nextkey: place to store the next valid key
653 *
654 * Return Value: If a next key was found, 0 is returned. Otherwise,
655 * -ENOENT is returned.
656 */
657static int nilfs_btree_get_next_key(const struct nilfs_bmap *btree,
658 const struct nilfs_btree_path *path,
659 int minlevel, __u64 *nextkey)
660{
661 struct nilfs_btree_node *node;
662 int maxlevel = nilfs_btree_height(btree) - 1;
663 int index, next_adj, level;
664
665 /* Next index is already set to bp_index for leaf nodes. */
666 next_adj = 0;
667 for (level = minlevel; level <= maxlevel; level++) {
668 if (level == maxlevel)
669 node = nilfs_btree_get_root(btree);
670 else
671 node = nilfs_btree_get_nonroot_node(path, level);
672
673 index = path[level].bp_index + next_adj;
674 if (index < nilfs_btree_node_get_nchildren(node)) {
675 /* Next key is in this node */
676 *nextkey = nilfs_btree_node_get_key(node, index);
677 return 0;
678 }
679 /* For non-leaf nodes, next index is stored at bp_index + 1. */
680 next_adj = 1;
681 }
682 return -ENOENT;
683}
684
685static int nilfs_btree_lookup(const struct nilfs_bmap *btree,
686 __u64 key, int level, __u64 *ptrp)
687{
688 struct nilfs_btree_path *path;
689 int ret;
690
691 path = nilfs_btree_alloc_path();
692 if (path == NULL)
693 return -ENOMEM;
694
695 ret = nilfs_btree_do_lookup(btree, path, key, ptrp, minlevel: level, readahead: 0);
696
697 nilfs_btree_free_path(path);
698
699 return ret;
700}
701
702static int nilfs_btree_lookup_contig(const struct nilfs_bmap *btree,
703 __u64 key, __u64 *ptrp,
704 unsigned int maxblocks)
705{
706 struct nilfs_btree_path *path;
707 struct nilfs_btree_node *node;
708 struct inode *dat = NULL;
709 __u64 ptr, ptr2;
710 sector_t blocknr;
711 int level = NILFS_BTREE_LEVEL_NODE_MIN;
712 int ret, cnt, index, maxlevel, ncmax;
713 struct nilfs_btree_readahead_info p;
714
715 path = nilfs_btree_alloc_path();
716 if (path == NULL)
717 return -ENOMEM;
718
719 ret = nilfs_btree_do_lookup(btree, path, key, ptrp: &ptr, minlevel: level, readahead: 1);
720 if (ret < 0)
721 goto out;
722
723 if (NILFS_BMAP_USE_VBN(btree)) {
724 dat = nilfs_bmap_get_dat(btree);
725 ret = nilfs_dat_translate(dat, ptr, &blocknr);
726 if (ret < 0)
727 goto out;
728 ptr = blocknr;
729 }
730 cnt = 1;
731 if (cnt == maxblocks)
732 goto end;
733
734 maxlevel = nilfs_btree_height(btree) - 1;
735 node = nilfs_btree_get_node(btree, path, level, ncmaxp: &ncmax);
736 index = path[level].bp_index + 1;
737 for (;;) {
738 while (index < nilfs_btree_node_get_nchildren(node)) {
739 if (nilfs_btree_node_get_key(node, index) !=
740 key + cnt)
741 goto end;
742 ptr2 = nilfs_btree_node_get_ptr(node, index, ncmax);
743 if (dat) {
744 ret = nilfs_dat_translate(dat, ptr2, &blocknr);
745 if (ret < 0)
746 goto out;
747 ptr2 = blocknr;
748 }
749 if (ptr2 != ptr + cnt || ++cnt == maxblocks)
750 goto end;
751 index++;
752 }
753 if (level == maxlevel)
754 break;
755
756 /* look-up right sibling node */
757 p.node = nilfs_btree_get_node(btree, path, level: level + 1, ncmaxp: &p.ncmax);
758 p.index = path[level + 1].bp_index + 1;
759 p.max_ra_blocks = 7;
760 if (p.index >= nilfs_btree_node_get_nchildren(node: p.node) ||
761 nilfs_btree_node_get_key(node: p.node, index: p.index) != key + cnt)
762 break;
763 ptr2 = nilfs_btree_node_get_ptr(node: p.node, index: p.index, ncmax: p.ncmax);
764 path[level + 1].bp_index = p.index;
765
766 brelse(bh: path[level].bp_bh);
767 path[level].bp_bh = NULL;
768
769 ret = __nilfs_btree_get_block(btree, ptr: ptr2, bhp: &path[level].bp_bh,
770 ra: &p);
771 if (ret < 0)
772 goto out;
773 node = nilfs_btree_get_nonroot_node(path, level);
774 ncmax = nilfs_btree_nchildren_per_block(btree);
775 index = 0;
776 path[level].bp_index = index;
777 }
778 end:
779 *ptrp = ptr;
780 ret = cnt;
781 out:
782 nilfs_btree_free_path(path);
783 return ret;
784}
785
786static void nilfs_btree_promote_key(struct nilfs_bmap *btree,
787 struct nilfs_btree_path *path,
788 int level, __u64 key)
789{
790 if (level < nilfs_btree_height(btree) - 1) {
791 do {
792 nilfs_btree_node_set_key(
793 node: nilfs_btree_get_nonroot_node(path, level),
794 index: path[level].bp_index, key);
795 if (!buffer_dirty(bh: path[level].bp_bh))
796 mark_buffer_dirty(bh: path[level].bp_bh);
797 } while ((path[level].bp_index == 0) &&
798 (++level < nilfs_btree_height(btree) - 1));
799 }
800
801 /* root */
802 if (level == nilfs_btree_height(btree) - 1) {
803 nilfs_btree_node_set_key(node: nilfs_btree_get_root(btree),
804 index: path[level].bp_index, key);
805 }
806}
807
808static void nilfs_btree_do_insert(struct nilfs_bmap *btree,
809 struct nilfs_btree_path *path,
810 int level, __u64 *keyp, __u64 *ptrp)
811{
812 struct nilfs_btree_node *node;
813 int ncblk;
814
815 if (level < nilfs_btree_height(btree) - 1) {
816 node = nilfs_btree_get_nonroot_node(path, level);
817 ncblk = nilfs_btree_nchildren_per_block(btree);
818 nilfs_btree_node_insert(node, index: path[level].bp_index,
819 key: *keyp, ptr: *ptrp, ncmax: ncblk);
820 if (!buffer_dirty(bh: path[level].bp_bh))
821 mark_buffer_dirty(bh: path[level].bp_bh);
822
823 if (path[level].bp_index == 0)
824 nilfs_btree_promote_key(btree, path, level: level + 1,
825 key: nilfs_btree_node_get_key(node,
826 index: 0));
827 } else {
828 node = nilfs_btree_get_root(btree);
829 nilfs_btree_node_insert(node, index: path[level].bp_index,
830 key: *keyp, ptr: *ptrp,
831 NILFS_BTREE_ROOT_NCHILDREN_MAX);
832 }
833}
834
835static void nilfs_btree_carry_left(struct nilfs_bmap *btree,
836 struct nilfs_btree_path *path,
837 int level, __u64 *keyp, __u64 *ptrp)
838{
839 struct nilfs_btree_node *node, *left;
840 int nchildren, lnchildren, n, move, ncblk;
841
842 node = nilfs_btree_get_nonroot_node(path, level);
843 left = nilfs_btree_get_sib_node(path, level);
844 nchildren = nilfs_btree_node_get_nchildren(node);
845 lnchildren = nilfs_btree_node_get_nchildren(node: left);
846 ncblk = nilfs_btree_nchildren_per_block(btree);
847 move = 0;
848
849 n = (nchildren + lnchildren + 1) / 2 - lnchildren;
850 if (n > path[level].bp_index) {
851 /* move insert point */
852 n--;
853 move = 1;
854 }
855
856 nilfs_btree_node_move_left(left, right: node, n, lncmax: ncblk, rncmax: ncblk);
857
858 if (!buffer_dirty(bh: path[level].bp_bh))
859 mark_buffer_dirty(bh: path[level].bp_bh);
860 if (!buffer_dirty(bh: path[level].bp_sib_bh))
861 mark_buffer_dirty(bh: path[level].bp_sib_bh);
862
863 nilfs_btree_promote_key(btree, path, level: level + 1,
864 key: nilfs_btree_node_get_key(node, index: 0));
865
866 if (move) {
867 brelse(bh: path[level].bp_bh);
868 path[level].bp_bh = path[level].bp_sib_bh;
869 path[level].bp_sib_bh = NULL;
870 path[level].bp_index += lnchildren;
871 path[level + 1].bp_index--;
872 } else {
873 brelse(bh: path[level].bp_sib_bh);
874 path[level].bp_sib_bh = NULL;
875 path[level].bp_index -= n;
876 }
877
878 nilfs_btree_do_insert(btree, path, level, keyp, ptrp);
879}
880
881static void nilfs_btree_carry_right(struct nilfs_bmap *btree,
882 struct nilfs_btree_path *path,
883 int level, __u64 *keyp, __u64 *ptrp)
884{
885 struct nilfs_btree_node *node, *right;
886 int nchildren, rnchildren, n, move, ncblk;
887
888 node = nilfs_btree_get_nonroot_node(path, level);
889 right = nilfs_btree_get_sib_node(path, level);
890 nchildren = nilfs_btree_node_get_nchildren(node);
891 rnchildren = nilfs_btree_node_get_nchildren(node: right);
892 ncblk = nilfs_btree_nchildren_per_block(btree);
893 move = 0;
894
895 n = (nchildren + rnchildren + 1) / 2 - rnchildren;
896 if (n > nchildren - path[level].bp_index) {
897 /* move insert point */
898 n--;
899 move = 1;
900 }
901
902 nilfs_btree_node_move_right(left: node, right, n, lncmax: ncblk, rncmax: ncblk);
903
904 if (!buffer_dirty(bh: path[level].bp_bh))
905 mark_buffer_dirty(bh: path[level].bp_bh);
906 if (!buffer_dirty(bh: path[level].bp_sib_bh))
907 mark_buffer_dirty(bh: path[level].bp_sib_bh);
908
909 path[level + 1].bp_index++;
910 nilfs_btree_promote_key(btree, path, level: level + 1,
911 key: nilfs_btree_node_get_key(node: right, index: 0));
912 path[level + 1].bp_index--;
913
914 if (move) {
915 brelse(bh: path[level].bp_bh);
916 path[level].bp_bh = path[level].bp_sib_bh;
917 path[level].bp_sib_bh = NULL;
918 path[level].bp_index -= nilfs_btree_node_get_nchildren(node);
919 path[level + 1].bp_index++;
920 } else {
921 brelse(bh: path[level].bp_sib_bh);
922 path[level].bp_sib_bh = NULL;
923 }
924
925 nilfs_btree_do_insert(btree, path, level, keyp, ptrp);
926}
927
928static void nilfs_btree_split(struct nilfs_bmap *btree,
929 struct nilfs_btree_path *path,
930 int level, __u64 *keyp, __u64 *ptrp)
931{
932 struct nilfs_btree_node *node, *right;
933 int nchildren, n, move, ncblk;
934
935 node = nilfs_btree_get_nonroot_node(path, level);
936 right = nilfs_btree_get_sib_node(path, level);
937 nchildren = nilfs_btree_node_get_nchildren(node);
938 ncblk = nilfs_btree_nchildren_per_block(btree);
939 move = 0;
940
941 n = (nchildren + 1) / 2;
942 if (n > nchildren - path[level].bp_index) {
943 n--;
944 move = 1;
945 }
946
947 nilfs_btree_node_move_right(left: node, right, n, lncmax: ncblk, rncmax: ncblk);
948
949 if (!buffer_dirty(bh: path[level].bp_bh))
950 mark_buffer_dirty(bh: path[level].bp_bh);
951 if (!buffer_dirty(bh: path[level].bp_sib_bh))
952 mark_buffer_dirty(bh: path[level].bp_sib_bh);
953
954 if (move) {
955 path[level].bp_index -= nilfs_btree_node_get_nchildren(node);
956 nilfs_btree_node_insert(node: right, index: path[level].bp_index,
957 key: *keyp, ptr: *ptrp, ncmax: ncblk);
958
959 *keyp = nilfs_btree_node_get_key(node: right, index: 0);
960 *ptrp = path[level].bp_newreq.bpr_ptr;
961
962 brelse(bh: path[level].bp_bh);
963 path[level].bp_bh = path[level].bp_sib_bh;
964 path[level].bp_sib_bh = NULL;
965 } else {
966 nilfs_btree_do_insert(btree, path, level, keyp, ptrp);
967
968 *keyp = nilfs_btree_node_get_key(node: right, index: 0);
969 *ptrp = path[level].bp_newreq.bpr_ptr;
970
971 brelse(bh: path[level].bp_sib_bh);
972 path[level].bp_sib_bh = NULL;
973 }
974
975 path[level + 1].bp_index++;
976}
977
978static void nilfs_btree_grow(struct nilfs_bmap *btree,
979 struct nilfs_btree_path *path,
980 int level, __u64 *keyp, __u64 *ptrp)
981{
982 struct nilfs_btree_node *root, *child;
983 int n, ncblk;
984
985 root = nilfs_btree_get_root(btree);
986 child = nilfs_btree_get_sib_node(path, level);
987 ncblk = nilfs_btree_nchildren_per_block(btree);
988
989 n = nilfs_btree_node_get_nchildren(node: root);
990
991 nilfs_btree_node_move_right(left: root, right: child, n,
992 NILFS_BTREE_ROOT_NCHILDREN_MAX, rncmax: ncblk);
993 nilfs_btree_node_set_level(node: root, level: level + 1);
994
995 if (!buffer_dirty(bh: path[level].bp_sib_bh))
996 mark_buffer_dirty(bh: path[level].bp_sib_bh);
997
998 path[level].bp_bh = path[level].bp_sib_bh;
999 path[level].bp_sib_bh = NULL;
1000
1001 nilfs_btree_do_insert(btree, path, level, keyp, ptrp);
1002
1003 *keyp = nilfs_btree_node_get_key(node: child, index: 0);
1004 *ptrp = path[level].bp_newreq.bpr_ptr;
1005}
1006
1007static __u64 nilfs_btree_find_near(const struct nilfs_bmap *btree,
1008 const struct nilfs_btree_path *path)
1009{
1010 struct nilfs_btree_node *node;
1011 int level, ncmax;
1012
1013 if (path == NULL)
1014 return NILFS_BMAP_INVALID_PTR;
1015
1016 /* left sibling */
1017 level = NILFS_BTREE_LEVEL_NODE_MIN;
1018 if (path[level].bp_index > 0) {
1019 node = nilfs_btree_get_node(btree, path, level, ncmaxp: &ncmax);
1020 return nilfs_btree_node_get_ptr(node,
1021 index: path[level].bp_index - 1,
1022 ncmax);
1023 }
1024
1025 /* parent */
1026 level = NILFS_BTREE_LEVEL_NODE_MIN + 1;
1027 if (level <= nilfs_btree_height(btree) - 1) {
1028 node = nilfs_btree_get_node(btree, path, level, ncmaxp: &ncmax);
1029 return nilfs_btree_node_get_ptr(node, index: path[level].bp_index,
1030 ncmax);
1031 }
1032
1033 return NILFS_BMAP_INVALID_PTR;
1034}
1035
1036static __u64 nilfs_btree_find_target_v(const struct nilfs_bmap *btree,
1037 const struct nilfs_btree_path *path,
1038 __u64 key)
1039{
1040 __u64 ptr;
1041
1042 ptr = nilfs_bmap_find_target_seq(btree, key);
1043 if (ptr != NILFS_BMAP_INVALID_PTR)
1044 /* sequential access */
1045 return ptr;
1046
1047 ptr = nilfs_btree_find_near(btree, path);
1048 if (ptr != NILFS_BMAP_INVALID_PTR)
1049 /* near */
1050 return ptr;
1051
1052 /* block group */
1053 return nilfs_bmap_find_target_in_group(btree);
1054}
1055
1056static int nilfs_btree_prepare_insert(struct nilfs_bmap *btree,
1057 struct nilfs_btree_path *path,
1058 int *levelp, __u64 key, __u64 ptr,
1059 struct nilfs_bmap_stats *stats)
1060{
1061 struct buffer_head *bh;
1062 struct nilfs_btree_node *node, *parent, *sib;
1063 __u64 sibptr;
1064 int pindex, level, ncmax, ncblk, ret;
1065 struct inode *dat = NULL;
1066
1067 stats->bs_nblocks = 0;
1068 level = NILFS_BTREE_LEVEL_DATA;
1069
1070 /* allocate a new ptr for data block */
1071 if (NILFS_BMAP_USE_VBN(btree)) {
1072 path[level].bp_newreq.bpr_ptr =
1073 nilfs_btree_find_target_v(btree, path, key);
1074 dat = nilfs_bmap_get_dat(btree);
1075 }
1076
1077 ret = nilfs_bmap_prepare_alloc_ptr(bmap: btree, req: &path[level].bp_newreq, dat);
1078 if (ret < 0)
1079 goto err_out_data;
1080
1081 ncblk = nilfs_btree_nchildren_per_block(btree);
1082
1083 for (level = NILFS_BTREE_LEVEL_NODE_MIN;
1084 level < nilfs_btree_height(btree) - 1;
1085 level++) {
1086 node = nilfs_btree_get_nonroot_node(path, level);
1087 if (nilfs_btree_node_get_nchildren(node) < ncblk) {
1088 path[level].bp_op = nilfs_btree_do_insert;
1089 stats->bs_nblocks++;
1090 goto out;
1091 }
1092
1093 parent = nilfs_btree_get_node(btree, path, level: level + 1, ncmaxp: &ncmax);
1094 pindex = path[level + 1].bp_index;
1095
1096 /* left sibling */
1097 if (pindex > 0) {
1098 sibptr = nilfs_btree_node_get_ptr(node: parent, index: pindex - 1,
1099 ncmax);
1100 ret = nilfs_btree_get_block(btree, ptr: sibptr, bhp: &bh);
1101 if (ret < 0)
1102 goto err_out_child_node;
1103 sib = (struct nilfs_btree_node *)bh->b_data;
1104 if (nilfs_btree_node_get_nchildren(node: sib) < ncblk) {
1105 path[level].bp_sib_bh = bh;
1106 path[level].bp_op = nilfs_btree_carry_left;
1107 stats->bs_nblocks++;
1108 goto out;
1109 } else {
1110 brelse(bh);
1111 }
1112 }
1113
1114 /* right sibling */
1115 if (pindex < nilfs_btree_node_get_nchildren(node: parent) - 1) {
1116 sibptr = nilfs_btree_node_get_ptr(node: parent, index: pindex + 1,
1117 ncmax);
1118 ret = nilfs_btree_get_block(btree, ptr: sibptr, bhp: &bh);
1119 if (ret < 0)
1120 goto err_out_child_node;
1121 sib = (struct nilfs_btree_node *)bh->b_data;
1122 if (nilfs_btree_node_get_nchildren(node: sib) < ncblk) {
1123 path[level].bp_sib_bh = bh;
1124 path[level].bp_op = nilfs_btree_carry_right;
1125 stats->bs_nblocks++;
1126 goto out;
1127 } else {
1128 brelse(bh);
1129 }
1130 }
1131
1132 /* split */
1133 path[level].bp_newreq.bpr_ptr =
1134 path[level - 1].bp_newreq.bpr_ptr + 1;
1135 ret = nilfs_bmap_prepare_alloc_ptr(bmap: btree,
1136 req: &path[level].bp_newreq, dat);
1137 if (ret < 0)
1138 goto err_out_child_node;
1139 ret = nilfs_btree_get_new_block(btree,
1140 ptr: path[level].bp_newreq.bpr_ptr,
1141 bhp: &bh);
1142 if (ret < 0)
1143 goto err_out_curr_node;
1144
1145 stats->bs_nblocks++;
1146
1147 sib = (struct nilfs_btree_node *)bh->b_data;
1148 nilfs_btree_node_init(node: sib, flags: 0, level, nchildren: 0, ncmax: ncblk, NULL, NULL);
1149 path[level].bp_sib_bh = bh;
1150 path[level].bp_op = nilfs_btree_split;
1151 }
1152
1153 /* root */
1154 node = nilfs_btree_get_root(btree);
1155 if (nilfs_btree_node_get_nchildren(node) <
1156 NILFS_BTREE_ROOT_NCHILDREN_MAX) {
1157 path[level].bp_op = nilfs_btree_do_insert;
1158 stats->bs_nblocks++;
1159 goto out;
1160 }
1161
1162 /* grow */
1163 path[level].bp_newreq.bpr_ptr = path[level - 1].bp_newreq.bpr_ptr + 1;
1164 ret = nilfs_bmap_prepare_alloc_ptr(bmap: btree, req: &path[level].bp_newreq, dat);
1165 if (ret < 0)
1166 goto err_out_child_node;
1167 ret = nilfs_btree_get_new_block(btree, ptr: path[level].bp_newreq.bpr_ptr,
1168 bhp: &bh);
1169 if (ret < 0)
1170 goto err_out_curr_node;
1171
1172 nilfs_btree_node_init(node: (struct nilfs_btree_node *)bh->b_data,
1173 flags: 0, level, nchildren: 0, ncmax: ncblk, NULL, NULL);
1174 path[level].bp_sib_bh = bh;
1175 path[level].bp_op = nilfs_btree_grow;
1176
1177 level++;
1178 path[level].bp_op = nilfs_btree_do_insert;
1179
1180 /* a newly-created node block and a data block are added */
1181 stats->bs_nblocks += 2;
1182
1183 /* success */
1184 out:
1185 *levelp = level;
1186 return ret;
1187
1188 /* error */
1189 err_out_curr_node:
1190 nilfs_bmap_abort_alloc_ptr(bmap: btree, req: &path[level].bp_newreq, dat);
1191 err_out_child_node:
1192 for (level--; level > NILFS_BTREE_LEVEL_DATA; level--) {
1193 nilfs_btnode_delete(path[level].bp_sib_bh);
1194 nilfs_bmap_abort_alloc_ptr(bmap: btree, req: &path[level].bp_newreq, dat);
1195
1196 }
1197
1198 nilfs_bmap_abort_alloc_ptr(bmap: btree, req: &path[level].bp_newreq, dat);
1199 err_out_data:
1200 *levelp = level;
1201 stats->bs_nblocks = 0;
1202 return ret;
1203}
1204
1205static void nilfs_btree_commit_insert(struct nilfs_bmap *btree,
1206 struct nilfs_btree_path *path,
1207 int maxlevel, __u64 key, __u64 ptr)
1208{
1209 struct inode *dat = NULL;
1210 int level;
1211
1212 set_buffer_nilfs_volatile((struct buffer_head *)((unsigned long)ptr));
1213 ptr = path[NILFS_BTREE_LEVEL_DATA].bp_newreq.bpr_ptr;
1214 if (NILFS_BMAP_USE_VBN(btree)) {
1215 nilfs_bmap_set_target_v(bmap: btree, key, ptr);
1216 dat = nilfs_bmap_get_dat(btree);
1217 }
1218
1219 for (level = NILFS_BTREE_LEVEL_NODE_MIN; level <= maxlevel; level++) {
1220 nilfs_bmap_commit_alloc_ptr(bmap: btree,
1221 req: &path[level - 1].bp_newreq, dat);
1222 path[level].bp_op(btree, path, level, &key, &ptr);
1223 }
1224
1225 if (!nilfs_bmap_dirty(bmap: btree))
1226 nilfs_bmap_set_dirty(bmap: btree);
1227}
1228
1229static int nilfs_btree_insert(struct nilfs_bmap *btree, __u64 key, __u64 ptr)
1230{
1231 struct nilfs_btree_path *path;
1232 struct nilfs_bmap_stats stats;
1233 int level, ret;
1234
1235 path = nilfs_btree_alloc_path();
1236 if (path == NULL)
1237 return -ENOMEM;
1238
1239 ret = nilfs_btree_do_lookup(btree, path, key, NULL,
1240 NILFS_BTREE_LEVEL_NODE_MIN, readahead: 0);
1241 if (ret != -ENOENT) {
1242 if (ret == 0)
1243 ret = -EEXIST;
1244 goto out;
1245 }
1246
1247 ret = nilfs_btree_prepare_insert(btree, path, levelp: &level, key, ptr, stats: &stats);
1248 if (ret < 0)
1249 goto out;
1250 nilfs_btree_commit_insert(btree, path, maxlevel: level, key, ptr);
1251 nilfs_inode_add_blocks(inode: btree->b_inode, n: stats.bs_nblocks);
1252
1253 out:
1254 nilfs_btree_free_path(path);
1255 return ret;
1256}
1257
1258static void nilfs_btree_do_delete(struct nilfs_bmap *btree,
1259 struct nilfs_btree_path *path,
1260 int level, __u64 *keyp, __u64 *ptrp)
1261{
1262 struct nilfs_btree_node *node;
1263 int ncblk;
1264
1265 if (level < nilfs_btree_height(btree) - 1) {
1266 node = nilfs_btree_get_nonroot_node(path, level);
1267 ncblk = nilfs_btree_nchildren_per_block(btree);
1268 nilfs_btree_node_delete(node, index: path[level].bp_index,
1269 keyp, ptrp, ncmax: ncblk);
1270 if (!buffer_dirty(bh: path[level].bp_bh))
1271 mark_buffer_dirty(bh: path[level].bp_bh);
1272 if (path[level].bp_index == 0)
1273 nilfs_btree_promote_key(btree, path, level: level + 1,
1274 key: nilfs_btree_node_get_key(node, index: 0));
1275 } else {
1276 node = nilfs_btree_get_root(btree);
1277 nilfs_btree_node_delete(node, index: path[level].bp_index,
1278 keyp, ptrp,
1279 NILFS_BTREE_ROOT_NCHILDREN_MAX);
1280 }
1281}
1282
1283static void nilfs_btree_borrow_left(struct nilfs_bmap *btree,
1284 struct nilfs_btree_path *path,
1285 int level, __u64 *keyp, __u64 *ptrp)
1286{
1287 struct nilfs_btree_node *node, *left;
1288 int nchildren, lnchildren, n, ncblk;
1289
1290 nilfs_btree_do_delete(btree, path, level, keyp, ptrp);
1291
1292 node = nilfs_btree_get_nonroot_node(path, level);
1293 left = nilfs_btree_get_sib_node(path, level);
1294 nchildren = nilfs_btree_node_get_nchildren(node);
1295 lnchildren = nilfs_btree_node_get_nchildren(node: left);
1296 ncblk = nilfs_btree_nchildren_per_block(btree);
1297
1298 n = (nchildren + lnchildren) / 2 - nchildren;
1299
1300 nilfs_btree_node_move_right(left, right: node, n, lncmax: ncblk, rncmax: ncblk);
1301
1302 if (!buffer_dirty(bh: path[level].bp_bh))
1303 mark_buffer_dirty(bh: path[level].bp_bh);
1304 if (!buffer_dirty(bh: path[level].bp_sib_bh))
1305 mark_buffer_dirty(bh: path[level].bp_sib_bh);
1306
1307 nilfs_btree_promote_key(btree, path, level: level + 1,
1308 key: nilfs_btree_node_get_key(node, index: 0));
1309
1310 brelse(bh: path[level].bp_sib_bh);
1311 path[level].bp_sib_bh = NULL;
1312 path[level].bp_index += n;
1313}
1314
1315static void nilfs_btree_borrow_right(struct nilfs_bmap *btree,
1316 struct nilfs_btree_path *path,
1317 int level, __u64 *keyp, __u64 *ptrp)
1318{
1319 struct nilfs_btree_node *node, *right;
1320 int nchildren, rnchildren, n, ncblk;
1321
1322 nilfs_btree_do_delete(btree, path, level, keyp, ptrp);
1323
1324 node = nilfs_btree_get_nonroot_node(path, level);
1325 right = nilfs_btree_get_sib_node(path, level);
1326 nchildren = nilfs_btree_node_get_nchildren(node);
1327 rnchildren = nilfs_btree_node_get_nchildren(node: right);
1328 ncblk = nilfs_btree_nchildren_per_block(btree);
1329
1330 n = (nchildren + rnchildren) / 2 - nchildren;
1331
1332 nilfs_btree_node_move_left(left: node, right, n, lncmax: ncblk, rncmax: ncblk);
1333
1334 if (!buffer_dirty(bh: path[level].bp_bh))
1335 mark_buffer_dirty(bh: path[level].bp_bh);
1336 if (!buffer_dirty(bh: path[level].bp_sib_bh))
1337 mark_buffer_dirty(bh: path[level].bp_sib_bh);
1338
1339 path[level + 1].bp_index++;
1340 nilfs_btree_promote_key(btree, path, level: level + 1,
1341 key: nilfs_btree_node_get_key(node: right, index: 0));
1342 path[level + 1].bp_index--;
1343
1344 brelse(bh: path[level].bp_sib_bh);
1345 path[level].bp_sib_bh = NULL;
1346}
1347
1348static void nilfs_btree_concat_left(struct nilfs_bmap *btree,
1349 struct nilfs_btree_path *path,
1350 int level, __u64 *keyp, __u64 *ptrp)
1351{
1352 struct nilfs_btree_node *node, *left;
1353 int n, ncblk;
1354
1355 nilfs_btree_do_delete(btree, path, level, keyp, ptrp);
1356
1357 node = nilfs_btree_get_nonroot_node(path, level);
1358 left = nilfs_btree_get_sib_node(path, level);
1359 ncblk = nilfs_btree_nchildren_per_block(btree);
1360
1361 n = nilfs_btree_node_get_nchildren(node);
1362
1363 nilfs_btree_node_move_left(left, right: node, n, lncmax: ncblk, rncmax: ncblk);
1364
1365 if (!buffer_dirty(bh: path[level].bp_sib_bh))
1366 mark_buffer_dirty(bh: path[level].bp_sib_bh);
1367
1368 nilfs_btnode_delete(path[level].bp_bh);
1369 path[level].bp_bh = path[level].bp_sib_bh;
1370 path[level].bp_sib_bh = NULL;
1371 path[level].bp_index += nilfs_btree_node_get_nchildren(node: left);
1372}
1373
1374static void nilfs_btree_concat_right(struct nilfs_bmap *btree,
1375 struct nilfs_btree_path *path,
1376 int level, __u64 *keyp, __u64 *ptrp)
1377{
1378 struct nilfs_btree_node *node, *right;
1379 int n, ncblk;
1380
1381 nilfs_btree_do_delete(btree, path, level, keyp, ptrp);
1382
1383 node = nilfs_btree_get_nonroot_node(path, level);
1384 right = nilfs_btree_get_sib_node(path, level);
1385 ncblk = nilfs_btree_nchildren_per_block(btree);
1386
1387 n = nilfs_btree_node_get_nchildren(node: right);
1388
1389 nilfs_btree_node_move_left(left: node, right, n, lncmax: ncblk, rncmax: ncblk);
1390
1391 if (!buffer_dirty(bh: path[level].bp_bh))
1392 mark_buffer_dirty(bh: path[level].bp_bh);
1393
1394 nilfs_btnode_delete(path[level].bp_sib_bh);
1395 path[level].bp_sib_bh = NULL;
1396 path[level + 1].bp_index++;
1397}
1398
1399static void nilfs_btree_shrink(struct nilfs_bmap *btree,
1400 struct nilfs_btree_path *path,
1401 int level, __u64 *keyp, __u64 *ptrp)
1402{
1403 struct nilfs_btree_node *root, *child;
1404 int n, ncblk;
1405
1406 nilfs_btree_do_delete(btree, path, level, keyp, ptrp);
1407
1408 root = nilfs_btree_get_root(btree);
1409 child = nilfs_btree_get_nonroot_node(path, level);
1410 ncblk = nilfs_btree_nchildren_per_block(btree);
1411
1412 nilfs_btree_node_delete(node: root, index: 0, NULL, NULL,
1413 NILFS_BTREE_ROOT_NCHILDREN_MAX);
1414 nilfs_btree_node_set_level(node: root, level);
1415 n = nilfs_btree_node_get_nchildren(node: child);
1416 nilfs_btree_node_move_left(left: root, right: child, n,
1417 NILFS_BTREE_ROOT_NCHILDREN_MAX, rncmax: ncblk);
1418
1419 nilfs_btnode_delete(path[level].bp_bh);
1420 path[level].bp_bh = NULL;
1421}
1422
1423static void nilfs_btree_nop(struct nilfs_bmap *btree,
1424 struct nilfs_btree_path *path,
1425 int level, __u64 *keyp, __u64 *ptrp)
1426{
1427}
1428
1429static int nilfs_btree_prepare_delete(struct nilfs_bmap *btree,
1430 struct nilfs_btree_path *path,
1431 int *levelp,
1432 struct nilfs_bmap_stats *stats,
1433 struct inode *dat)
1434{
1435 struct buffer_head *bh;
1436 struct nilfs_btree_node *node, *parent, *sib;
1437 __u64 sibptr;
1438 int pindex, dindex, level, ncmin, ncmax, ncblk, ret;
1439
1440 ret = 0;
1441 stats->bs_nblocks = 0;
1442 ncmin = NILFS_BTREE_NODE_NCHILDREN_MIN(nilfs_btree_node_size(btree));
1443 ncblk = nilfs_btree_nchildren_per_block(btree);
1444
1445 for (level = NILFS_BTREE_LEVEL_NODE_MIN, dindex = path[level].bp_index;
1446 level < nilfs_btree_height(btree) - 1;
1447 level++) {
1448 node = nilfs_btree_get_nonroot_node(path, level);
1449 path[level].bp_oldreq.bpr_ptr =
1450 nilfs_btree_node_get_ptr(node, index: dindex, ncmax: ncblk);
1451 ret = nilfs_bmap_prepare_end_ptr(bmap: btree,
1452 req: &path[level].bp_oldreq, dat);
1453 if (ret < 0)
1454 goto err_out_child_node;
1455
1456 if (nilfs_btree_node_get_nchildren(node) > ncmin) {
1457 path[level].bp_op = nilfs_btree_do_delete;
1458 stats->bs_nblocks++;
1459 goto out;
1460 }
1461
1462 parent = nilfs_btree_get_node(btree, path, level: level + 1, ncmaxp: &ncmax);
1463 pindex = path[level + 1].bp_index;
1464 dindex = pindex;
1465
1466 if (pindex > 0) {
1467 /* left sibling */
1468 sibptr = nilfs_btree_node_get_ptr(node: parent, index: pindex - 1,
1469 ncmax);
1470 ret = nilfs_btree_get_block(btree, ptr: sibptr, bhp: &bh);
1471 if (ret < 0)
1472 goto err_out_curr_node;
1473 sib = (struct nilfs_btree_node *)bh->b_data;
1474 if (nilfs_btree_node_get_nchildren(node: sib) > ncmin) {
1475 path[level].bp_sib_bh = bh;
1476 path[level].bp_op = nilfs_btree_borrow_left;
1477 stats->bs_nblocks++;
1478 goto out;
1479 } else {
1480 path[level].bp_sib_bh = bh;
1481 path[level].bp_op = nilfs_btree_concat_left;
1482 stats->bs_nblocks++;
1483 /* continue; */
1484 }
1485 } else if (pindex <
1486 nilfs_btree_node_get_nchildren(node: parent) - 1) {
1487 /* right sibling */
1488 sibptr = nilfs_btree_node_get_ptr(node: parent, index: pindex + 1,
1489 ncmax);
1490 ret = nilfs_btree_get_block(btree, ptr: sibptr, bhp: &bh);
1491 if (ret < 0)
1492 goto err_out_curr_node;
1493 sib = (struct nilfs_btree_node *)bh->b_data;
1494 if (nilfs_btree_node_get_nchildren(node: sib) > ncmin) {
1495 path[level].bp_sib_bh = bh;
1496 path[level].bp_op = nilfs_btree_borrow_right;
1497 stats->bs_nblocks++;
1498 goto out;
1499 } else {
1500 path[level].bp_sib_bh = bh;
1501 path[level].bp_op = nilfs_btree_concat_right;
1502 stats->bs_nblocks++;
1503 /*
1504 * When merging right sibling node
1505 * into the current node, pointer to
1506 * the right sibling node must be
1507 * terminated instead. The adjustment
1508 * below is required for that.
1509 */
1510 dindex = pindex + 1;
1511 /* continue; */
1512 }
1513 } else {
1514 /* no siblings */
1515 /* the only child of the root node */
1516 WARN_ON(level != nilfs_btree_height(btree) - 2);
1517 if (nilfs_btree_node_get_nchildren(node) - 1 <=
1518 NILFS_BTREE_ROOT_NCHILDREN_MAX) {
1519 path[level].bp_op = nilfs_btree_shrink;
1520 stats->bs_nblocks += 2;
1521 level++;
1522 path[level].bp_op = nilfs_btree_nop;
1523 goto shrink_root_child;
1524 } else {
1525 path[level].bp_op = nilfs_btree_do_delete;
1526 stats->bs_nblocks++;
1527 goto out;
1528 }
1529 }
1530 }
1531
1532 /* child of the root node is deleted */
1533 path[level].bp_op = nilfs_btree_do_delete;
1534 stats->bs_nblocks++;
1535
1536shrink_root_child:
1537 node = nilfs_btree_get_root(btree);
1538 path[level].bp_oldreq.bpr_ptr =
1539 nilfs_btree_node_get_ptr(node, index: dindex,
1540 NILFS_BTREE_ROOT_NCHILDREN_MAX);
1541
1542 ret = nilfs_bmap_prepare_end_ptr(bmap: btree, req: &path[level].bp_oldreq, dat);
1543 if (ret < 0)
1544 goto err_out_child_node;
1545
1546 /* success */
1547 out:
1548 *levelp = level;
1549 return ret;
1550
1551 /* error */
1552 err_out_curr_node:
1553 nilfs_bmap_abort_end_ptr(bmap: btree, req: &path[level].bp_oldreq, dat);
1554 err_out_child_node:
1555 for (level--; level >= NILFS_BTREE_LEVEL_NODE_MIN; level--) {
1556 brelse(bh: path[level].bp_sib_bh);
1557 nilfs_bmap_abort_end_ptr(bmap: btree, req: &path[level].bp_oldreq, dat);
1558 }
1559 *levelp = level;
1560 stats->bs_nblocks = 0;
1561 return ret;
1562}
1563
1564static void nilfs_btree_commit_delete(struct nilfs_bmap *btree,
1565 struct nilfs_btree_path *path,
1566 int maxlevel, struct inode *dat)
1567{
1568 int level;
1569
1570 for (level = NILFS_BTREE_LEVEL_NODE_MIN; level <= maxlevel; level++) {
1571 nilfs_bmap_commit_end_ptr(bmap: btree, req: &path[level].bp_oldreq, dat);
1572 path[level].bp_op(btree, path, level, NULL, NULL);
1573 }
1574
1575 if (!nilfs_bmap_dirty(bmap: btree))
1576 nilfs_bmap_set_dirty(bmap: btree);
1577}
1578
1579static int nilfs_btree_delete(struct nilfs_bmap *btree, __u64 key)
1580
1581{
1582 struct nilfs_btree_path *path;
1583 struct nilfs_bmap_stats stats;
1584 struct inode *dat;
1585 int level, ret;
1586
1587 path = nilfs_btree_alloc_path();
1588 if (path == NULL)
1589 return -ENOMEM;
1590
1591 ret = nilfs_btree_do_lookup(btree, path, key, NULL,
1592 NILFS_BTREE_LEVEL_NODE_MIN, readahead: 0);
1593 if (ret < 0)
1594 goto out;
1595
1596
1597 dat = NILFS_BMAP_USE_VBN(btree) ? nilfs_bmap_get_dat(btree) : NULL;
1598
1599 ret = nilfs_btree_prepare_delete(btree, path, levelp: &level, stats: &stats, dat);
1600 if (ret < 0)
1601 goto out;
1602 nilfs_btree_commit_delete(btree, path, maxlevel: level, dat);
1603 nilfs_inode_sub_blocks(inode: btree->b_inode, n: stats.bs_nblocks);
1604
1605out:
1606 nilfs_btree_free_path(path);
1607 return ret;
1608}
1609
1610static int nilfs_btree_seek_key(const struct nilfs_bmap *btree, __u64 start,
1611 __u64 *keyp)
1612{
1613 struct nilfs_btree_path *path;
1614 const int minlevel = NILFS_BTREE_LEVEL_NODE_MIN;
1615 int ret;
1616
1617 path = nilfs_btree_alloc_path();
1618 if (!path)
1619 return -ENOMEM;
1620
1621 ret = nilfs_btree_do_lookup(btree, path, key: start, NULL, minlevel, readahead: 0);
1622 if (!ret)
1623 *keyp = start;
1624 else if (ret == -ENOENT)
1625 ret = nilfs_btree_get_next_key(btree, path, minlevel, nextkey: keyp);
1626
1627 nilfs_btree_free_path(path);
1628 return ret;
1629}
1630
1631static int nilfs_btree_last_key(const struct nilfs_bmap *btree, __u64 *keyp)
1632{
1633 struct nilfs_btree_path *path;
1634 int ret;
1635
1636 path = nilfs_btree_alloc_path();
1637 if (path == NULL)
1638 return -ENOMEM;
1639
1640 ret = nilfs_btree_do_lookup_last(btree, path, keyp, NULL);
1641
1642 nilfs_btree_free_path(path);
1643
1644 return ret;
1645}
1646
1647static int nilfs_btree_check_delete(struct nilfs_bmap *btree, __u64 key)
1648{
1649 struct buffer_head *bh;
1650 struct nilfs_btree_node *root, *node;
1651 __u64 maxkey, nextmaxkey;
1652 __u64 ptr;
1653 int nchildren, ret;
1654
1655 root = nilfs_btree_get_root(btree);
1656 switch (nilfs_btree_height(btree)) {
1657 case 2:
1658 bh = NULL;
1659 node = root;
1660 break;
1661 case 3:
1662 nchildren = nilfs_btree_node_get_nchildren(node: root);
1663 if (nchildren > 1)
1664 return 0;
1665 ptr = nilfs_btree_node_get_ptr(node: root, index: nchildren - 1,
1666 NILFS_BTREE_ROOT_NCHILDREN_MAX);
1667 ret = nilfs_btree_get_block(btree, ptr, bhp: &bh);
1668 if (ret < 0)
1669 return ret;
1670 node = (struct nilfs_btree_node *)bh->b_data;
1671 break;
1672 default:
1673 return 0;
1674 }
1675
1676 nchildren = nilfs_btree_node_get_nchildren(node);
1677 maxkey = nilfs_btree_node_get_key(node, index: nchildren - 1);
1678 nextmaxkey = (nchildren > 1) ?
1679 nilfs_btree_node_get_key(node, index: nchildren - 2) : 0;
1680 brelse(bh);
1681
1682 return (maxkey == key) && (nextmaxkey < NILFS_BMAP_LARGE_LOW);
1683}
1684
1685static int nilfs_btree_gather_data(struct nilfs_bmap *btree,
1686 __u64 *keys, __u64 *ptrs, int nitems)
1687{
1688 struct buffer_head *bh;
1689 struct nilfs_btree_node *node, *root;
1690 __le64 *dkeys;
1691 __le64 *dptrs;
1692 __u64 ptr;
1693 int nchildren, ncmax, i, ret;
1694
1695 root = nilfs_btree_get_root(btree);
1696 switch (nilfs_btree_height(btree)) {
1697 case 2:
1698 bh = NULL;
1699 node = root;
1700 ncmax = NILFS_BTREE_ROOT_NCHILDREN_MAX;
1701 break;
1702 case 3:
1703 nchildren = nilfs_btree_node_get_nchildren(node: root);
1704 WARN_ON(nchildren > 1);
1705 ptr = nilfs_btree_node_get_ptr(node: root, index: nchildren - 1,
1706 NILFS_BTREE_ROOT_NCHILDREN_MAX);
1707 ret = nilfs_btree_get_block(btree, ptr, bhp: &bh);
1708 if (ret < 0)
1709 return ret;
1710 node = (struct nilfs_btree_node *)bh->b_data;
1711 ncmax = nilfs_btree_nchildren_per_block(btree);
1712 break;
1713 default:
1714 node = NULL;
1715 return -EINVAL;
1716 }
1717
1718 nchildren = nilfs_btree_node_get_nchildren(node);
1719 if (nchildren < nitems)
1720 nitems = nchildren;
1721 dkeys = nilfs_btree_node_dkeys(node);
1722 dptrs = nilfs_btree_node_dptrs(node, ncmax);
1723 for (i = 0; i < nitems; i++) {
1724 keys[i] = le64_to_cpu(dkeys[i]);
1725 ptrs[i] = le64_to_cpu(dptrs[i]);
1726 }
1727
1728 brelse(bh);
1729
1730 return nitems;
1731}
1732
1733static int
1734nilfs_btree_prepare_convert_and_insert(struct nilfs_bmap *btree, __u64 key,
1735 union nilfs_bmap_ptr_req *dreq,
1736 union nilfs_bmap_ptr_req *nreq,
1737 struct buffer_head **bhp,
1738 struct nilfs_bmap_stats *stats)
1739{
1740 struct buffer_head *bh;
1741 struct inode *dat = NULL;
1742 int ret;
1743
1744 stats->bs_nblocks = 0;
1745
1746 /* for data */
1747 /* cannot find near ptr */
1748 if (NILFS_BMAP_USE_VBN(btree)) {
1749 dreq->bpr_ptr = nilfs_btree_find_target_v(btree, NULL, key);
1750 dat = nilfs_bmap_get_dat(btree);
1751 }
1752
1753 ret = nilfs_attach_btree_node_cache(inode: &NILFS_BMAP_I(bmap: btree)->vfs_inode);
1754 if (ret < 0)
1755 return ret;
1756
1757 ret = nilfs_bmap_prepare_alloc_ptr(bmap: btree, req: dreq, dat);
1758 if (ret < 0)
1759 return ret;
1760
1761 *bhp = NULL;
1762 stats->bs_nblocks++;
1763 if (nreq != NULL) {
1764 nreq->bpr_ptr = dreq->bpr_ptr + 1;
1765 ret = nilfs_bmap_prepare_alloc_ptr(bmap: btree, req: nreq, dat);
1766 if (ret < 0)
1767 goto err_out_dreq;
1768
1769 ret = nilfs_btree_get_new_block(btree, ptr: nreq->bpr_ptr, bhp: &bh);
1770 if (ret < 0)
1771 goto err_out_nreq;
1772
1773 *bhp = bh;
1774 stats->bs_nblocks++;
1775 }
1776
1777 /* success */
1778 return 0;
1779
1780 /* error */
1781 err_out_nreq:
1782 nilfs_bmap_abort_alloc_ptr(bmap: btree, req: nreq, dat);
1783 err_out_dreq:
1784 nilfs_bmap_abort_alloc_ptr(bmap: btree, req: dreq, dat);
1785 stats->bs_nblocks = 0;
1786 return ret;
1787
1788}
1789
1790static void
1791nilfs_btree_commit_convert_and_insert(struct nilfs_bmap *btree,
1792 __u64 key, __u64 ptr,
1793 const __u64 *keys, const __u64 *ptrs,
1794 int n,
1795 union nilfs_bmap_ptr_req *dreq,
1796 union nilfs_bmap_ptr_req *nreq,
1797 struct buffer_head *bh)
1798{
1799 struct nilfs_btree_node *node;
1800 struct inode *dat;
1801 __u64 tmpptr;
1802 int ncblk;
1803
1804 /* free resources */
1805 if (btree->b_ops->bop_clear != NULL)
1806 btree->b_ops->bop_clear(btree);
1807
1808 /* ptr must be a pointer to a buffer head. */
1809 set_buffer_nilfs_volatile((struct buffer_head *)((unsigned long)ptr));
1810
1811 /* convert and insert */
1812 dat = NILFS_BMAP_USE_VBN(btree) ? nilfs_bmap_get_dat(btree) : NULL;
1813 __nilfs_btree_init(bmap: btree);
1814 if (nreq != NULL) {
1815 nilfs_bmap_commit_alloc_ptr(bmap: btree, req: dreq, dat);
1816 nilfs_bmap_commit_alloc_ptr(bmap: btree, req: nreq, dat);
1817
1818 /* create child node at level 1 */
1819 node = (struct nilfs_btree_node *)bh->b_data;
1820 ncblk = nilfs_btree_nchildren_per_block(btree);
1821 nilfs_btree_node_init(node, flags: 0, level: 1, nchildren: n, ncmax: ncblk, keys, ptrs);
1822 nilfs_btree_node_insert(node, index: n, key, ptr: dreq->bpr_ptr, ncmax: ncblk);
1823 if (!buffer_dirty(bh))
1824 mark_buffer_dirty(bh);
1825 if (!nilfs_bmap_dirty(bmap: btree))
1826 nilfs_bmap_set_dirty(bmap: btree);
1827
1828 brelse(bh);
1829
1830 /* create root node at level 2 */
1831 node = nilfs_btree_get_root(btree);
1832 tmpptr = nreq->bpr_ptr;
1833 nilfs_btree_node_init(node, NILFS_BTREE_NODE_ROOT, level: 2, nchildren: 1,
1834 NILFS_BTREE_ROOT_NCHILDREN_MAX,
1835 keys: &keys[0], ptrs: &tmpptr);
1836 } else {
1837 nilfs_bmap_commit_alloc_ptr(bmap: btree, req: dreq, dat);
1838
1839 /* create root node at level 1 */
1840 node = nilfs_btree_get_root(btree);
1841 nilfs_btree_node_init(node, NILFS_BTREE_NODE_ROOT, level: 1, nchildren: n,
1842 NILFS_BTREE_ROOT_NCHILDREN_MAX,
1843 keys, ptrs);
1844 nilfs_btree_node_insert(node, index: n, key, ptr: dreq->bpr_ptr,
1845 NILFS_BTREE_ROOT_NCHILDREN_MAX);
1846 if (!nilfs_bmap_dirty(bmap: btree))
1847 nilfs_bmap_set_dirty(bmap: btree);
1848 }
1849
1850 if (NILFS_BMAP_USE_VBN(btree))
1851 nilfs_bmap_set_target_v(bmap: btree, key, ptr: dreq->bpr_ptr);
1852}
1853
1854/**
1855 * nilfs_btree_convert_and_insert -
1856 * @bmap:
1857 * @key:
1858 * @ptr:
1859 * @keys:
1860 * @ptrs:
1861 * @n:
1862 */
1863int nilfs_btree_convert_and_insert(struct nilfs_bmap *btree,
1864 __u64 key, __u64 ptr,
1865 const __u64 *keys, const __u64 *ptrs, int n)
1866{
1867 struct buffer_head *bh = NULL;
1868 union nilfs_bmap_ptr_req dreq, nreq, *di, *ni;
1869 struct nilfs_bmap_stats stats;
1870 int ret;
1871
1872 if (n + 1 <= NILFS_BTREE_ROOT_NCHILDREN_MAX) {
1873 di = &dreq;
1874 ni = NULL;
1875 } else if ((n + 1) <= NILFS_BTREE_NODE_NCHILDREN_MAX(
1876 nilfs_btree_node_size(btree))) {
1877 di = &dreq;
1878 ni = &nreq;
1879 } else {
1880 di = NULL;
1881 ni = NULL;
1882 BUG();
1883 }
1884
1885 ret = nilfs_btree_prepare_convert_and_insert(btree, key, dreq: di, nreq: ni, bhp: &bh,
1886 stats: &stats);
1887 if (ret < 0)
1888 return ret;
1889 nilfs_btree_commit_convert_and_insert(btree, key, ptr, keys, ptrs, n,
1890 dreq: di, nreq: ni, bh);
1891 nilfs_inode_add_blocks(inode: btree->b_inode, n: stats.bs_nblocks);
1892 return 0;
1893}
1894
1895static int nilfs_btree_propagate_p(struct nilfs_bmap *btree,
1896 struct nilfs_btree_path *path,
1897 int level,
1898 struct buffer_head *bh)
1899{
1900 while ((++level < nilfs_btree_height(btree) - 1) &&
1901 !buffer_dirty(bh: path[level].bp_bh))
1902 mark_buffer_dirty(bh: path[level].bp_bh);
1903
1904 return 0;
1905}
1906
1907static int nilfs_btree_prepare_update_v(struct nilfs_bmap *btree,
1908 struct nilfs_btree_path *path,
1909 int level, struct inode *dat)
1910{
1911 struct nilfs_btree_node *parent;
1912 int ncmax, ret;
1913
1914 parent = nilfs_btree_get_node(btree, path, level: level + 1, ncmaxp: &ncmax);
1915 path[level].bp_oldreq.bpr_ptr =
1916 nilfs_btree_node_get_ptr(node: parent, index: path[level + 1].bp_index,
1917 ncmax);
1918 path[level].bp_newreq.bpr_ptr = path[level].bp_oldreq.bpr_ptr + 1;
1919 ret = nilfs_dat_prepare_update(dat, &path[level].bp_oldreq.bpr_req,
1920 &path[level].bp_newreq.bpr_req);
1921 if (ret < 0)
1922 return ret;
1923
1924 if (buffer_nilfs_node(bh: path[level].bp_bh)) {
1925 path[level].bp_ctxt.oldkey = path[level].bp_oldreq.bpr_ptr;
1926 path[level].bp_ctxt.newkey = path[level].bp_newreq.bpr_ptr;
1927 path[level].bp_ctxt.bh = path[level].bp_bh;
1928 ret = nilfs_btnode_prepare_change_key(
1929 NILFS_BMAP_I(bmap: btree)->i_assoc_inode->i_mapping,
1930 &path[level].bp_ctxt);
1931 if (ret < 0) {
1932 nilfs_dat_abort_update(dat,
1933 &path[level].bp_oldreq.bpr_req,
1934 &path[level].bp_newreq.bpr_req);
1935 return ret;
1936 }
1937 }
1938
1939 return 0;
1940}
1941
1942static void nilfs_btree_commit_update_v(struct nilfs_bmap *btree,
1943 struct nilfs_btree_path *path,
1944 int level, struct inode *dat)
1945{
1946 struct nilfs_btree_node *parent;
1947 int ncmax;
1948
1949 nilfs_dat_commit_update(dat, &path[level].bp_oldreq.bpr_req,
1950 &path[level].bp_newreq.bpr_req,
1951 btree->b_ptr_type == NILFS_BMAP_PTR_VS);
1952
1953 if (buffer_nilfs_node(bh: path[level].bp_bh)) {
1954 nilfs_btnode_commit_change_key(
1955 NILFS_BMAP_I(bmap: btree)->i_assoc_inode->i_mapping,
1956 &path[level].bp_ctxt);
1957 path[level].bp_bh = path[level].bp_ctxt.bh;
1958 }
1959 set_buffer_nilfs_volatile(path[level].bp_bh);
1960
1961 parent = nilfs_btree_get_node(btree, path, level: level + 1, ncmaxp: &ncmax);
1962 nilfs_btree_node_set_ptr(node: parent, index: path[level + 1].bp_index,
1963 ptr: path[level].bp_newreq.bpr_ptr, ncmax);
1964}
1965
1966static void nilfs_btree_abort_update_v(struct nilfs_bmap *btree,
1967 struct nilfs_btree_path *path,
1968 int level, struct inode *dat)
1969{
1970 nilfs_dat_abort_update(dat, &path[level].bp_oldreq.bpr_req,
1971 &path[level].bp_newreq.bpr_req);
1972 if (buffer_nilfs_node(bh: path[level].bp_bh))
1973 nilfs_btnode_abort_change_key(
1974 NILFS_BMAP_I(bmap: btree)->i_assoc_inode->i_mapping,
1975 &path[level].bp_ctxt);
1976}
1977
1978static int nilfs_btree_prepare_propagate_v(struct nilfs_bmap *btree,
1979 struct nilfs_btree_path *path,
1980 int minlevel, int *maxlevelp,
1981 struct inode *dat)
1982{
1983 int level, ret;
1984
1985 level = minlevel;
1986 if (!buffer_nilfs_volatile(bh: path[level].bp_bh)) {
1987 ret = nilfs_btree_prepare_update_v(btree, path, level, dat);
1988 if (ret < 0)
1989 return ret;
1990 }
1991 while ((++level < nilfs_btree_height(btree) - 1) &&
1992 !buffer_dirty(bh: path[level].bp_bh)) {
1993
1994 WARN_ON(buffer_nilfs_volatile(path[level].bp_bh));
1995 ret = nilfs_btree_prepare_update_v(btree, path, level, dat);
1996 if (ret < 0)
1997 goto out;
1998 }
1999
2000 /* success */
2001 *maxlevelp = level - 1;
2002 return 0;
2003
2004 /* error */
2005 out:
2006 while (--level > minlevel)
2007 nilfs_btree_abort_update_v(btree, path, level, dat);
2008 if (!buffer_nilfs_volatile(bh: path[level].bp_bh))
2009 nilfs_btree_abort_update_v(btree, path, level, dat);
2010 return ret;
2011}
2012
2013static void nilfs_btree_commit_propagate_v(struct nilfs_bmap *btree,
2014 struct nilfs_btree_path *path,
2015 int minlevel, int maxlevel,
2016 struct buffer_head *bh,
2017 struct inode *dat)
2018{
2019 int level;
2020
2021 if (!buffer_nilfs_volatile(bh: path[minlevel].bp_bh))
2022 nilfs_btree_commit_update_v(btree, path, level: minlevel, dat);
2023
2024 for (level = minlevel + 1; level <= maxlevel; level++)
2025 nilfs_btree_commit_update_v(btree, path, level, dat);
2026}
2027
2028static int nilfs_btree_propagate_v(struct nilfs_bmap *btree,
2029 struct nilfs_btree_path *path,
2030 int level, struct buffer_head *bh)
2031{
2032 int maxlevel = 0, ret;
2033 struct nilfs_btree_node *parent;
2034 struct inode *dat = nilfs_bmap_get_dat(btree);
2035 __u64 ptr;
2036 int ncmax;
2037
2038 get_bh(bh);
2039 path[level].bp_bh = bh;
2040 ret = nilfs_btree_prepare_propagate_v(btree, path, minlevel: level, maxlevelp: &maxlevel,
2041 dat);
2042 if (ret < 0)
2043 goto out;
2044
2045 if (buffer_nilfs_volatile(bh: path[level].bp_bh)) {
2046 parent = nilfs_btree_get_node(btree, path, level: level + 1, ncmaxp: &ncmax);
2047 ptr = nilfs_btree_node_get_ptr(node: parent,
2048 index: path[level + 1].bp_index,
2049 ncmax);
2050 ret = nilfs_dat_mark_dirty(dat, ptr);
2051 if (ret < 0)
2052 goto out;
2053 }
2054
2055 nilfs_btree_commit_propagate_v(btree, path, minlevel: level, maxlevel, bh, dat);
2056
2057 out:
2058 brelse(bh: path[level].bp_bh);
2059 path[level].bp_bh = NULL;
2060 return ret;
2061}
2062
2063static int nilfs_btree_propagate(struct nilfs_bmap *btree,
2064 struct buffer_head *bh)
2065{
2066 struct nilfs_btree_path *path;
2067 struct nilfs_btree_node *node;
2068 __u64 key;
2069 int level, ret;
2070
2071 WARN_ON(!buffer_dirty(bh));
2072
2073 path = nilfs_btree_alloc_path();
2074 if (path == NULL)
2075 return -ENOMEM;
2076
2077 if (buffer_nilfs_node(bh)) {
2078 node = (struct nilfs_btree_node *)bh->b_data;
2079 key = nilfs_btree_node_get_key(node, index: 0);
2080 level = nilfs_btree_node_get_level(node);
2081 } else {
2082 key = nilfs_bmap_data_get_key(btree, bh);
2083 level = NILFS_BTREE_LEVEL_DATA;
2084 }
2085
2086 ret = nilfs_btree_do_lookup(btree, path, key, NULL, minlevel: level + 1, readahead: 0);
2087 if (ret < 0) {
2088 if (unlikely(ret == -ENOENT))
2089 nilfs_crit(btree->b_inode->i_sb,
2090 "writing node/leaf block does not appear in b-tree (ino=%lu) at key=%llu, level=%d",
2091 btree->b_inode->i_ino,
2092 (unsigned long long)key, level);
2093 goto out;
2094 }
2095
2096 ret = NILFS_BMAP_USE_VBN(btree) ?
2097 nilfs_btree_propagate_v(btree, path, level, bh) :
2098 nilfs_btree_propagate_p(btree, path, level, bh);
2099
2100 out:
2101 nilfs_btree_free_path(path);
2102
2103 return ret;
2104}
2105
2106static int nilfs_btree_propagate_gc(struct nilfs_bmap *btree,
2107 struct buffer_head *bh)
2108{
2109 return nilfs_dat_mark_dirty(nilfs_bmap_get_dat(btree), bh->b_blocknr);
2110}
2111
2112static void nilfs_btree_add_dirty_buffer(struct nilfs_bmap *btree,
2113 struct list_head *lists,
2114 struct buffer_head *bh)
2115{
2116 struct list_head *head;
2117 struct buffer_head *cbh;
2118 struct nilfs_btree_node *node, *cnode;
2119 __u64 key, ckey;
2120 int level;
2121
2122 get_bh(bh);
2123 node = (struct nilfs_btree_node *)bh->b_data;
2124 key = nilfs_btree_node_get_key(node, index: 0);
2125 level = nilfs_btree_node_get_level(node);
2126 if (level < NILFS_BTREE_LEVEL_NODE_MIN ||
2127 level >= NILFS_BTREE_LEVEL_MAX) {
2128 dump_stack();
2129 nilfs_warn(btree->b_inode->i_sb,
2130 "invalid btree level: %d (key=%llu, ino=%lu, blocknr=%llu)",
2131 level, (unsigned long long)key,
2132 btree->b_inode->i_ino,
2133 (unsigned long long)bh->b_blocknr);
2134 return;
2135 }
2136
2137 list_for_each(head, &lists[level]) {
2138 cbh = list_entry(head, struct buffer_head, b_assoc_buffers);
2139 cnode = (struct nilfs_btree_node *)cbh->b_data;
2140 ckey = nilfs_btree_node_get_key(node: cnode, index: 0);
2141 if (key < ckey)
2142 break;
2143 }
2144 list_add_tail(new: &bh->b_assoc_buffers, head);
2145}
2146
2147static void nilfs_btree_lookup_dirty_buffers(struct nilfs_bmap *btree,
2148 struct list_head *listp)
2149{
2150 struct inode *btnc_inode = NILFS_BMAP_I(bmap: btree)->i_assoc_inode;
2151 struct address_space *btcache = btnc_inode->i_mapping;
2152 struct list_head lists[NILFS_BTREE_LEVEL_MAX];
2153 struct folio_batch fbatch;
2154 struct buffer_head *bh, *head;
2155 pgoff_t index = 0;
2156 int level, i;
2157
2158 for (level = NILFS_BTREE_LEVEL_NODE_MIN;
2159 level < NILFS_BTREE_LEVEL_MAX;
2160 level++)
2161 INIT_LIST_HEAD(list: &lists[level]);
2162
2163 folio_batch_init(fbatch: &fbatch);
2164
2165 while (filemap_get_folios_tag(mapping: btcache, start: &index, end: (pgoff_t)-1,
2166 PAGECACHE_TAG_DIRTY, fbatch: &fbatch)) {
2167 for (i = 0; i < folio_batch_count(fbatch: &fbatch); i++) {
2168 bh = head = folio_buffers(fbatch.folios[i]);
2169 do {
2170 if (buffer_dirty(bh))
2171 nilfs_btree_add_dirty_buffer(btree,
2172 lists, bh);
2173 } while ((bh = bh->b_this_page) != head);
2174 }
2175 folio_batch_release(fbatch: &fbatch);
2176 cond_resched();
2177 }
2178
2179 for (level = NILFS_BTREE_LEVEL_NODE_MIN;
2180 level < NILFS_BTREE_LEVEL_MAX;
2181 level++)
2182 list_splice_tail(list: &lists[level], head: listp);
2183}
2184
2185static int nilfs_btree_assign_p(struct nilfs_bmap *btree,
2186 struct nilfs_btree_path *path,
2187 int level,
2188 struct buffer_head **bh,
2189 sector_t blocknr,
2190 union nilfs_binfo *binfo)
2191{
2192 struct nilfs_btree_node *parent;
2193 __u64 key;
2194 __u64 ptr;
2195 int ncmax, ret;
2196
2197 parent = nilfs_btree_get_node(btree, path, level: level + 1, ncmaxp: &ncmax);
2198 ptr = nilfs_btree_node_get_ptr(node: parent, index: path[level + 1].bp_index,
2199 ncmax);
2200 if (buffer_nilfs_node(bh: *bh)) {
2201 path[level].bp_ctxt.oldkey = ptr;
2202 path[level].bp_ctxt.newkey = blocknr;
2203 path[level].bp_ctxt.bh = *bh;
2204 ret = nilfs_btnode_prepare_change_key(
2205 NILFS_BMAP_I(bmap: btree)->i_assoc_inode->i_mapping,
2206 &path[level].bp_ctxt);
2207 if (ret < 0)
2208 return ret;
2209 nilfs_btnode_commit_change_key(
2210 NILFS_BMAP_I(bmap: btree)->i_assoc_inode->i_mapping,
2211 &path[level].bp_ctxt);
2212 *bh = path[level].bp_ctxt.bh;
2213 }
2214
2215 nilfs_btree_node_set_ptr(node: parent, index: path[level + 1].bp_index, ptr: blocknr,
2216 ncmax);
2217
2218 key = nilfs_btree_node_get_key(node: parent, index: path[level + 1].bp_index);
2219 /* on-disk format */
2220 binfo->bi_dat.bi_blkoff = cpu_to_le64(key);
2221 binfo->bi_dat.bi_level = level;
2222 memset(binfo->bi_dat.bi_pad, 0, sizeof(binfo->bi_dat.bi_pad));
2223
2224 return 0;
2225}
2226
2227static int nilfs_btree_assign_v(struct nilfs_bmap *btree,
2228 struct nilfs_btree_path *path,
2229 int level,
2230 struct buffer_head **bh,
2231 sector_t blocknr,
2232 union nilfs_binfo *binfo)
2233{
2234 struct nilfs_btree_node *parent;
2235 struct inode *dat = nilfs_bmap_get_dat(btree);
2236 __u64 key;
2237 __u64 ptr;
2238 union nilfs_bmap_ptr_req req;
2239 int ncmax, ret;
2240
2241 parent = nilfs_btree_get_node(btree, path, level: level + 1, ncmaxp: &ncmax);
2242 ptr = nilfs_btree_node_get_ptr(node: parent, index: path[level + 1].bp_index,
2243 ncmax);
2244 req.bpr_ptr = ptr;
2245 ret = nilfs_dat_prepare_start(dat, &req.bpr_req);
2246 if (ret < 0)
2247 return ret;
2248 nilfs_dat_commit_start(dat, &req.bpr_req, blocknr);
2249
2250 key = nilfs_btree_node_get_key(node: parent, index: path[level + 1].bp_index);
2251 /* on-disk format */
2252 binfo->bi_v.bi_vblocknr = cpu_to_le64(ptr);
2253 binfo->bi_v.bi_blkoff = cpu_to_le64(key);
2254
2255 return 0;
2256}
2257
2258static int nilfs_btree_assign(struct nilfs_bmap *btree,
2259 struct buffer_head **bh,
2260 sector_t blocknr,
2261 union nilfs_binfo *binfo)
2262{
2263 struct nilfs_btree_path *path;
2264 struct nilfs_btree_node *node;
2265 __u64 key;
2266 int level, ret;
2267
2268 path = nilfs_btree_alloc_path();
2269 if (path == NULL)
2270 return -ENOMEM;
2271
2272 if (buffer_nilfs_node(bh: *bh)) {
2273 node = (struct nilfs_btree_node *)(*bh)->b_data;
2274 key = nilfs_btree_node_get_key(node, index: 0);
2275 level = nilfs_btree_node_get_level(node);
2276 } else {
2277 key = nilfs_bmap_data_get_key(btree, *bh);
2278 level = NILFS_BTREE_LEVEL_DATA;
2279 }
2280
2281 ret = nilfs_btree_do_lookup(btree, path, key, NULL, minlevel: level + 1, readahead: 0);
2282 if (ret < 0) {
2283 WARN_ON(ret == -ENOENT);
2284 goto out;
2285 }
2286
2287 ret = NILFS_BMAP_USE_VBN(btree) ?
2288 nilfs_btree_assign_v(btree, path, level, bh, blocknr, binfo) :
2289 nilfs_btree_assign_p(btree, path, level, bh, blocknr, binfo);
2290
2291 out:
2292 nilfs_btree_free_path(path);
2293
2294 return ret;
2295}
2296
2297static int nilfs_btree_assign_gc(struct nilfs_bmap *btree,
2298 struct buffer_head **bh,
2299 sector_t blocknr,
2300 union nilfs_binfo *binfo)
2301{
2302 struct nilfs_btree_node *node;
2303 __u64 key;
2304 int ret;
2305
2306 ret = nilfs_dat_move(nilfs_bmap_get_dat(btree), (*bh)->b_blocknr,
2307 blocknr);
2308 if (ret < 0)
2309 return ret;
2310
2311 if (buffer_nilfs_node(bh: *bh)) {
2312 node = (struct nilfs_btree_node *)(*bh)->b_data;
2313 key = nilfs_btree_node_get_key(node, index: 0);
2314 } else
2315 key = nilfs_bmap_data_get_key(btree, *bh);
2316
2317 /* on-disk format */
2318 binfo->bi_v.bi_vblocknr = cpu_to_le64((*bh)->b_blocknr);
2319 binfo->bi_v.bi_blkoff = cpu_to_le64(key);
2320
2321 return 0;
2322}
2323
2324static int nilfs_btree_mark(struct nilfs_bmap *btree, __u64 key, int level)
2325{
2326 struct buffer_head *bh;
2327 struct nilfs_btree_path *path;
2328 __u64 ptr;
2329 int ret;
2330
2331 path = nilfs_btree_alloc_path();
2332 if (path == NULL)
2333 return -ENOMEM;
2334
2335 ret = nilfs_btree_do_lookup(btree, path, key, ptrp: &ptr, minlevel: level + 1, readahead: 0);
2336 if (ret < 0) {
2337 WARN_ON(ret == -ENOENT);
2338 goto out;
2339 }
2340 ret = nilfs_btree_get_block(btree, ptr, bhp: &bh);
2341 if (ret < 0) {
2342 WARN_ON(ret == -ENOENT);
2343 goto out;
2344 }
2345
2346 if (!buffer_dirty(bh))
2347 mark_buffer_dirty(bh);
2348 brelse(bh);
2349 if (!nilfs_bmap_dirty(bmap: btree))
2350 nilfs_bmap_set_dirty(bmap: btree);
2351
2352 out:
2353 nilfs_btree_free_path(path);
2354 return ret;
2355}
2356
2357static const struct nilfs_bmap_operations nilfs_btree_ops = {
2358 .bop_lookup = nilfs_btree_lookup,
2359 .bop_lookup_contig = nilfs_btree_lookup_contig,
2360 .bop_insert = nilfs_btree_insert,
2361 .bop_delete = nilfs_btree_delete,
2362 .bop_clear = NULL,
2363
2364 .bop_propagate = nilfs_btree_propagate,
2365
2366 .bop_lookup_dirty_buffers = nilfs_btree_lookup_dirty_buffers,
2367
2368 .bop_assign = nilfs_btree_assign,
2369 .bop_mark = nilfs_btree_mark,
2370
2371 .bop_seek_key = nilfs_btree_seek_key,
2372 .bop_last_key = nilfs_btree_last_key,
2373
2374 .bop_check_insert = NULL,
2375 .bop_check_delete = nilfs_btree_check_delete,
2376 .bop_gather_data = nilfs_btree_gather_data,
2377};
2378
2379static const struct nilfs_bmap_operations nilfs_btree_ops_gc = {
2380 .bop_lookup = NULL,
2381 .bop_lookup_contig = NULL,
2382 .bop_insert = NULL,
2383 .bop_delete = NULL,
2384 .bop_clear = NULL,
2385
2386 .bop_propagate = nilfs_btree_propagate_gc,
2387
2388 .bop_lookup_dirty_buffers = nilfs_btree_lookup_dirty_buffers,
2389
2390 .bop_assign = nilfs_btree_assign_gc,
2391 .bop_mark = NULL,
2392
2393 .bop_seek_key = NULL,
2394 .bop_last_key = NULL,
2395
2396 .bop_check_insert = NULL,
2397 .bop_check_delete = NULL,
2398 .bop_gather_data = NULL,
2399};
2400
2401static void __nilfs_btree_init(struct nilfs_bmap *bmap)
2402{
2403 bmap->b_ops = &nilfs_btree_ops;
2404 bmap->b_nchildren_per_block =
2405 NILFS_BTREE_NODE_NCHILDREN_MAX(nilfs_btree_node_size(bmap));
2406}
2407
2408int nilfs_btree_init(struct nilfs_bmap *bmap)
2409{
2410 int ret = 0;
2411
2412 __nilfs_btree_init(bmap);
2413
2414 if (nilfs_btree_root_broken(node: nilfs_btree_get_root(btree: bmap), inode: bmap->b_inode))
2415 ret = -EIO;
2416 else
2417 ret = nilfs_attach_btree_node_cache(
2418 inode: &NILFS_BMAP_I(bmap)->vfs_inode);
2419
2420 return ret;
2421}
2422
2423void nilfs_btree_init_gc(struct nilfs_bmap *bmap)
2424{
2425 bmap->b_ops = &nilfs_btree_ops_gc;
2426 bmap->b_nchildren_per_block =
2427 NILFS_BTREE_NODE_NCHILDREN_MAX(nilfs_btree_node_size(bmap));
2428}
2429

source code of linux/fs/nilfs2/btree.c