1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright (c) 2016-present, Facebook, Inc. |
4 | * All rights reserved. |
5 | * |
6 | */ |
7 | |
8 | #include <linux/bio.h> |
9 | #include <linux/bitmap.h> |
10 | #include <linux/err.h> |
11 | #include <linux/init.h> |
12 | #include <linux/kernel.h> |
13 | #include <linux/mm.h> |
14 | #include <linux/sched/mm.h> |
15 | #include <linux/pagemap.h> |
16 | #include <linux/refcount.h> |
17 | #include <linux/sched.h> |
18 | #include <linux/slab.h> |
19 | #include <linux/zstd.h> |
20 | #include "misc.h" |
21 | #include "compression.h" |
22 | #include "ctree.h" |
23 | |
24 | #define ZSTD_BTRFS_MAX_WINDOWLOG 17 |
25 | #define ZSTD_BTRFS_MAX_INPUT (1 << ZSTD_BTRFS_MAX_WINDOWLOG) |
26 | #define ZSTD_BTRFS_DEFAULT_LEVEL 3 |
27 | #define ZSTD_BTRFS_MAX_LEVEL 15 |
28 | /* 307s to avoid pathologically clashing with transaction commit */ |
29 | #define ZSTD_BTRFS_RECLAIM_JIFFIES (307 * HZ) |
30 | |
31 | static zstd_parameters zstd_get_btrfs_parameters(unsigned int level, |
32 | size_t src_len) |
33 | { |
34 | zstd_parameters params = zstd_get_params(level, estimated_src_size: src_len); |
35 | |
36 | if (params.cParams.windowLog > ZSTD_BTRFS_MAX_WINDOWLOG) |
37 | params.cParams.windowLog = ZSTD_BTRFS_MAX_WINDOWLOG; |
38 | WARN_ON(src_len > ZSTD_BTRFS_MAX_INPUT); |
39 | return params; |
40 | } |
41 | |
42 | struct workspace { |
43 | void *mem; |
44 | size_t size; |
45 | char *buf; |
46 | unsigned int level; |
47 | unsigned int req_level; |
48 | unsigned long last_used; /* jiffies */ |
49 | struct list_head list; |
50 | struct list_head lru_list; |
51 | zstd_in_buffer in_buf; |
52 | zstd_out_buffer out_buf; |
53 | }; |
54 | |
55 | /* |
56 | * Zstd Workspace Management |
57 | * |
58 | * Zstd workspaces have different memory requirements depending on the level. |
59 | * The zstd workspaces are managed by having individual lists for each level |
60 | * and a global lru. Forward progress is maintained by protecting a max level |
61 | * workspace. |
62 | * |
63 | * Getting a workspace is done by using the bitmap to identify the levels that |
64 | * have available workspaces and scans up. This lets us recycle higher level |
65 | * workspaces because of the monotonic memory guarantee. A workspace's |
66 | * last_used is only updated if it is being used by the corresponding memory |
67 | * level. Putting a workspace involves adding it back to the appropriate places |
68 | * and adding it back to the lru if necessary. |
69 | * |
70 | * A timer is used to reclaim workspaces if they have not been used for |
71 | * ZSTD_BTRFS_RECLAIM_JIFFIES. This helps keep only active workspaces around. |
72 | * The upper bound is provided by the workqueue limit which is 2 (percpu limit). |
73 | */ |
74 | |
75 | struct zstd_workspace_manager { |
76 | const struct btrfs_compress_op *ops; |
77 | spinlock_t lock; |
78 | struct list_head lru_list; |
79 | struct list_head idle_ws[ZSTD_BTRFS_MAX_LEVEL]; |
80 | unsigned long active_map; |
81 | wait_queue_head_t wait; |
82 | struct timer_list timer; |
83 | }; |
84 | |
85 | static struct zstd_workspace_manager wsm; |
86 | |
87 | static size_t zstd_ws_mem_sizes[ZSTD_BTRFS_MAX_LEVEL]; |
88 | |
89 | static inline struct workspace *list_to_workspace(struct list_head *list) |
90 | { |
91 | return container_of(list, struct workspace, list); |
92 | } |
93 | |
94 | void zstd_free_workspace(struct list_head *ws); |
95 | struct list_head *zstd_alloc_workspace(unsigned int level); |
96 | |
97 | /* |
98 | * Timer callback to free unused workspaces. |
99 | * |
100 | * @t: timer |
101 | * |
102 | * This scans the lru_list and attempts to reclaim any workspace that hasn't |
103 | * been used for ZSTD_BTRFS_RECLAIM_JIFFIES. |
104 | * |
105 | * The context is softirq and does not need the _bh locking primitives. |
106 | */ |
107 | static void zstd_reclaim_timer_fn(struct timer_list *timer) |
108 | { |
109 | unsigned long reclaim_threshold = jiffies - ZSTD_BTRFS_RECLAIM_JIFFIES; |
110 | struct list_head *pos, *next; |
111 | |
112 | spin_lock(lock: &wsm.lock); |
113 | |
114 | if (list_empty(head: &wsm.lru_list)) { |
115 | spin_unlock(lock: &wsm.lock); |
116 | return; |
117 | } |
118 | |
119 | list_for_each_prev_safe(pos, next, &wsm.lru_list) { |
120 | struct workspace *victim = container_of(pos, struct workspace, |
121 | lru_list); |
122 | unsigned int level; |
123 | |
124 | if (time_after(victim->last_used, reclaim_threshold)) |
125 | break; |
126 | |
127 | /* workspace is in use */ |
128 | if (victim->req_level) |
129 | continue; |
130 | |
131 | level = victim->level; |
132 | list_del(entry: &victim->lru_list); |
133 | list_del(entry: &victim->list); |
134 | zstd_free_workspace(ws: &victim->list); |
135 | |
136 | if (list_empty(head: &wsm.idle_ws[level - 1])) |
137 | clear_bit(nr: level - 1, addr: &wsm.active_map); |
138 | |
139 | } |
140 | |
141 | if (!list_empty(head: &wsm.lru_list)) |
142 | mod_timer(timer: &wsm.timer, expires: jiffies + ZSTD_BTRFS_RECLAIM_JIFFIES); |
143 | |
144 | spin_unlock(lock: &wsm.lock); |
145 | } |
146 | |
147 | /* |
148 | * Calculate monotonic memory bounds. |
149 | * |
150 | * It is possible based on the level configurations that a higher level |
151 | * workspace uses less memory than a lower level workspace. In order to reuse |
152 | * workspaces, this must be made a monotonic relationship. This precomputes |
153 | * the required memory for each level and enforces the monotonicity between |
154 | * level and memory required. |
155 | */ |
156 | static void zstd_calc_ws_mem_sizes(void) |
157 | { |
158 | size_t max_size = 0; |
159 | unsigned int level; |
160 | |
161 | for (level = 1; level <= ZSTD_BTRFS_MAX_LEVEL; level++) { |
162 | zstd_parameters params = |
163 | zstd_get_btrfs_parameters(level, ZSTD_BTRFS_MAX_INPUT); |
164 | size_t level_size = |
165 | max_t(size_t, |
166 | zstd_cstream_workspace_bound(¶ms.cParams), |
167 | zstd_dstream_workspace_bound(ZSTD_BTRFS_MAX_INPUT)); |
168 | |
169 | max_size = max_t(size_t, max_size, level_size); |
170 | zstd_ws_mem_sizes[level - 1] = max_size; |
171 | } |
172 | } |
173 | |
174 | void zstd_init_workspace_manager(void) |
175 | { |
176 | struct list_head *ws; |
177 | int i; |
178 | |
179 | zstd_calc_ws_mem_sizes(); |
180 | |
181 | wsm.ops = &btrfs_zstd_compress; |
182 | spin_lock_init(&wsm.lock); |
183 | init_waitqueue_head(&wsm.wait); |
184 | timer_setup(&wsm.timer, zstd_reclaim_timer_fn, 0); |
185 | |
186 | INIT_LIST_HEAD(list: &wsm.lru_list); |
187 | for (i = 0; i < ZSTD_BTRFS_MAX_LEVEL; i++) |
188 | INIT_LIST_HEAD(list: &wsm.idle_ws[i]); |
189 | |
190 | ws = zstd_alloc_workspace(ZSTD_BTRFS_MAX_LEVEL); |
191 | if (IS_ERR(ptr: ws)) { |
192 | pr_warn( |
193 | "BTRFS: cannot preallocate zstd compression workspace\n" ); |
194 | } else { |
195 | set_bit(ZSTD_BTRFS_MAX_LEVEL - 1, addr: &wsm.active_map); |
196 | list_add(new: ws, head: &wsm.idle_ws[ZSTD_BTRFS_MAX_LEVEL - 1]); |
197 | } |
198 | } |
199 | |
200 | void zstd_cleanup_workspace_manager(void) |
201 | { |
202 | struct workspace *workspace; |
203 | int i; |
204 | |
205 | spin_lock_bh(lock: &wsm.lock); |
206 | for (i = 0; i < ZSTD_BTRFS_MAX_LEVEL; i++) { |
207 | while (!list_empty(head: &wsm.idle_ws[i])) { |
208 | workspace = container_of(wsm.idle_ws[i].next, |
209 | struct workspace, list); |
210 | list_del(entry: &workspace->list); |
211 | list_del(entry: &workspace->lru_list); |
212 | zstd_free_workspace(ws: &workspace->list); |
213 | } |
214 | } |
215 | spin_unlock_bh(lock: &wsm.lock); |
216 | |
217 | del_timer_sync(timer: &wsm.timer); |
218 | } |
219 | |
220 | /* |
221 | * Find workspace for given level. |
222 | * |
223 | * @level: compression level |
224 | * |
225 | * This iterates over the set bits in the active_map beginning at the requested |
226 | * compression level. This lets us utilize already allocated workspaces before |
227 | * allocating a new one. If the workspace is of a larger size, it is used, but |
228 | * the place in the lru_list and last_used times are not updated. This is to |
229 | * offer the opportunity to reclaim the workspace in favor of allocating an |
230 | * appropriately sized one in the future. |
231 | */ |
232 | static struct list_head *zstd_find_workspace(unsigned int level) |
233 | { |
234 | struct list_head *ws; |
235 | struct workspace *workspace; |
236 | int i = level - 1; |
237 | |
238 | spin_lock_bh(lock: &wsm.lock); |
239 | for_each_set_bit_from(i, &wsm.active_map, ZSTD_BTRFS_MAX_LEVEL) { |
240 | if (!list_empty(head: &wsm.idle_ws[i])) { |
241 | ws = wsm.idle_ws[i].next; |
242 | workspace = list_to_workspace(list: ws); |
243 | list_del_init(entry: ws); |
244 | /* keep its place if it's a lower level using this */ |
245 | workspace->req_level = level; |
246 | if (level == workspace->level) |
247 | list_del(entry: &workspace->lru_list); |
248 | if (list_empty(head: &wsm.idle_ws[i])) |
249 | clear_bit(nr: i, addr: &wsm.active_map); |
250 | spin_unlock_bh(lock: &wsm.lock); |
251 | return ws; |
252 | } |
253 | } |
254 | spin_unlock_bh(lock: &wsm.lock); |
255 | |
256 | return NULL; |
257 | } |
258 | |
259 | /* |
260 | * Zstd get_workspace for level. |
261 | * |
262 | * @level: compression level |
263 | * |
264 | * If @level is 0, then any compression level can be used. Therefore, we begin |
265 | * scanning from 1. We first scan through possible workspaces and then after |
266 | * attempt to allocate a new workspace. If we fail to allocate one due to |
267 | * memory pressure, go to sleep waiting for the max level workspace to free up. |
268 | */ |
269 | struct list_head *zstd_get_workspace(unsigned int level) |
270 | { |
271 | struct list_head *ws; |
272 | unsigned int nofs_flag; |
273 | |
274 | /* level == 0 means we can use any workspace */ |
275 | if (!level) |
276 | level = 1; |
277 | |
278 | again: |
279 | ws = zstd_find_workspace(level); |
280 | if (ws) |
281 | return ws; |
282 | |
283 | nofs_flag = memalloc_nofs_save(); |
284 | ws = zstd_alloc_workspace(level); |
285 | memalloc_nofs_restore(flags: nofs_flag); |
286 | |
287 | if (IS_ERR(ptr: ws)) { |
288 | DEFINE_WAIT(wait); |
289 | |
290 | prepare_to_wait(wq_head: &wsm.wait, wq_entry: &wait, TASK_UNINTERRUPTIBLE); |
291 | schedule(); |
292 | finish_wait(wq_head: &wsm.wait, wq_entry: &wait); |
293 | |
294 | goto again; |
295 | } |
296 | |
297 | return ws; |
298 | } |
299 | |
300 | /* |
301 | * Zstd put_workspace. |
302 | * |
303 | * @ws: list_head for the workspace |
304 | * |
305 | * When putting back a workspace, we only need to update the LRU if we are of |
306 | * the requested compression level. Here is where we continue to protect the |
307 | * max level workspace or update last_used accordingly. If the reclaim timer |
308 | * isn't set, it is also set here. Only the max level workspace tries and wakes |
309 | * up waiting workspaces. |
310 | */ |
311 | void zstd_put_workspace(struct list_head *ws) |
312 | { |
313 | struct workspace *workspace = list_to_workspace(list: ws); |
314 | |
315 | spin_lock_bh(lock: &wsm.lock); |
316 | |
317 | /* A node is only taken off the lru if we are the corresponding level */ |
318 | if (workspace->req_level == workspace->level) { |
319 | /* Hide a max level workspace from reclaim */ |
320 | if (list_empty(head: &wsm.idle_ws[ZSTD_BTRFS_MAX_LEVEL - 1])) { |
321 | INIT_LIST_HEAD(list: &workspace->lru_list); |
322 | } else { |
323 | workspace->last_used = jiffies; |
324 | list_add(new: &workspace->lru_list, head: &wsm.lru_list); |
325 | if (!timer_pending(timer: &wsm.timer)) |
326 | mod_timer(timer: &wsm.timer, |
327 | expires: jiffies + ZSTD_BTRFS_RECLAIM_JIFFIES); |
328 | } |
329 | } |
330 | |
331 | set_bit(nr: workspace->level - 1, addr: &wsm.active_map); |
332 | list_add(new: &workspace->list, head: &wsm.idle_ws[workspace->level - 1]); |
333 | workspace->req_level = 0; |
334 | |
335 | spin_unlock_bh(lock: &wsm.lock); |
336 | |
337 | if (workspace->level == ZSTD_BTRFS_MAX_LEVEL) |
338 | cond_wake_up(wq: &wsm.wait); |
339 | } |
340 | |
341 | void zstd_free_workspace(struct list_head *ws) |
342 | { |
343 | struct workspace *workspace = list_entry(ws, struct workspace, list); |
344 | |
345 | kvfree(addr: workspace->mem); |
346 | kfree(objp: workspace->buf); |
347 | kfree(objp: workspace); |
348 | } |
349 | |
350 | struct list_head *zstd_alloc_workspace(unsigned int level) |
351 | { |
352 | struct workspace *workspace; |
353 | |
354 | workspace = kzalloc(size: sizeof(*workspace), GFP_KERNEL); |
355 | if (!workspace) |
356 | return ERR_PTR(error: -ENOMEM); |
357 | |
358 | workspace->size = zstd_ws_mem_sizes[level - 1]; |
359 | workspace->level = level; |
360 | workspace->req_level = level; |
361 | workspace->last_used = jiffies; |
362 | workspace->mem = kvmalloc(size: workspace->size, GFP_KERNEL | __GFP_NOWARN); |
363 | workspace->buf = kmalloc(PAGE_SIZE, GFP_KERNEL); |
364 | if (!workspace->mem || !workspace->buf) |
365 | goto fail; |
366 | |
367 | INIT_LIST_HEAD(list: &workspace->list); |
368 | INIT_LIST_HEAD(list: &workspace->lru_list); |
369 | |
370 | return &workspace->list; |
371 | fail: |
372 | zstd_free_workspace(ws: &workspace->list); |
373 | return ERR_PTR(error: -ENOMEM); |
374 | } |
375 | |
376 | int zstd_compress_pages(struct list_head *ws, struct address_space *mapping, |
377 | u64 start, struct page **pages, unsigned long *out_pages, |
378 | unsigned long *total_in, unsigned long *total_out) |
379 | { |
380 | struct workspace *workspace = list_entry(ws, struct workspace, list); |
381 | zstd_cstream *stream; |
382 | int ret = 0; |
383 | int nr_pages = 0; |
384 | struct page *in_page = NULL; /* The current page to read */ |
385 | struct page *out_page = NULL; /* The current page to write to */ |
386 | unsigned long tot_in = 0; |
387 | unsigned long tot_out = 0; |
388 | unsigned long len = *total_out; |
389 | const unsigned long nr_dest_pages = *out_pages; |
390 | unsigned long max_out = nr_dest_pages * PAGE_SIZE; |
391 | zstd_parameters params = zstd_get_btrfs_parameters(level: workspace->req_level, |
392 | src_len: len); |
393 | |
394 | *out_pages = 0; |
395 | *total_out = 0; |
396 | *total_in = 0; |
397 | |
398 | /* Initialize the stream */ |
399 | stream = zstd_init_cstream(parameters: ¶ms, pledged_src_size: len, workspace: workspace->mem, |
400 | workspace_size: workspace->size); |
401 | if (!stream) { |
402 | pr_warn("BTRFS: zstd_init_cstream failed\n" ); |
403 | ret = -EIO; |
404 | goto out; |
405 | } |
406 | |
407 | /* map in the first page of input data */ |
408 | in_page = find_get_page(mapping, offset: start >> PAGE_SHIFT); |
409 | workspace->in_buf.src = kmap_local_page(page: in_page); |
410 | workspace->in_buf.pos = 0; |
411 | workspace->in_buf.size = min_t(size_t, len, PAGE_SIZE); |
412 | |
413 | |
414 | /* Allocate and map in the output buffer */ |
415 | out_page = alloc_page(GFP_NOFS); |
416 | if (out_page == NULL) { |
417 | ret = -ENOMEM; |
418 | goto out; |
419 | } |
420 | pages[nr_pages++] = out_page; |
421 | workspace->out_buf.dst = page_address(out_page); |
422 | workspace->out_buf.pos = 0; |
423 | workspace->out_buf.size = min_t(size_t, max_out, PAGE_SIZE); |
424 | |
425 | while (1) { |
426 | size_t ret2; |
427 | |
428 | ret2 = zstd_compress_stream(cstream: stream, output: &workspace->out_buf, |
429 | input: &workspace->in_buf); |
430 | if (zstd_is_error(code: ret2)) { |
431 | pr_debug("BTRFS: zstd_compress_stream returned %d\n" , |
432 | zstd_get_error_code(ret2)); |
433 | ret = -EIO; |
434 | goto out; |
435 | } |
436 | |
437 | /* Check to see if we are making it bigger */ |
438 | if (tot_in + workspace->in_buf.pos > 8192 && |
439 | tot_in + workspace->in_buf.pos < |
440 | tot_out + workspace->out_buf.pos) { |
441 | ret = -E2BIG; |
442 | goto out; |
443 | } |
444 | |
445 | /* We've reached the end of our output range */ |
446 | if (workspace->out_buf.pos >= max_out) { |
447 | tot_out += workspace->out_buf.pos; |
448 | ret = -E2BIG; |
449 | goto out; |
450 | } |
451 | |
452 | /* Check if we need more output space */ |
453 | if (workspace->out_buf.pos == workspace->out_buf.size) { |
454 | tot_out += PAGE_SIZE; |
455 | max_out -= PAGE_SIZE; |
456 | if (nr_pages == nr_dest_pages) { |
457 | ret = -E2BIG; |
458 | goto out; |
459 | } |
460 | out_page = alloc_page(GFP_NOFS); |
461 | if (out_page == NULL) { |
462 | ret = -ENOMEM; |
463 | goto out; |
464 | } |
465 | pages[nr_pages++] = out_page; |
466 | workspace->out_buf.dst = page_address(out_page); |
467 | workspace->out_buf.pos = 0; |
468 | workspace->out_buf.size = min_t(size_t, max_out, |
469 | PAGE_SIZE); |
470 | } |
471 | |
472 | /* We've reached the end of the input */ |
473 | if (workspace->in_buf.pos >= len) { |
474 | tot_in += workspace->in_buf.pos; |
475 | break; |
476 | } |
477 | |
478 | /* Check if we need more input */ |
479 | if (workspace->in_buf.pos == workspace->in_buf.size) { |
480 | tot_in += PAGE_SIZE; |
481 | kunmap_local(workspace->in_buf.src); |
482 | put_page(page: in_page); |
483 | start += PAGE_SIZE; |
484 | len -= PAGE_SIZE; |
485 | in_page = find_get_page(mapping, offset: start >> PAGE_SHIFT); |
486 | workspace->in_buf.src = kmap_local_page(page: in_page); |
487 | workspace->in_buf.pos = 0; |
488 | workspace->in_buf.size = min_t(size_t, len, PAGE_SIZE); |
489 | } |
490 | } |
491 | while (1) { |
492 | size_t ret2; |
493 | |
494 | ret2 = zstd_end_stream(cstream: stream, output: &workspace->out_buf); |
495 | if (zstd_is_error(code: ret2)) { |
496 | pr_debug("BTRFS: zstd_end_stream returned %d\n" , |
497 | zstd_get_error_code(ret2)); |
498 | ret = -EIO; |
499 | goto out; |
500 | } |
501 | if (ret2 == 0) { |
502 | tot_out += workspace->out_buf.pos; |
503 | break; |
504 | } |
505 | if (workspace->out_buf.pos >= max_out) { |
506 | tot_out += workspace->out_buf.pos; |
507 | ret = -E2BIG; |
508 | goto out; |
509 | } |
510 | |
511 | tot_out += PAGE_SIZE; |
512 | max_out -= PAGE_SIZE; |
513 | if (nr_pages == nr_dest_pages) { |
514 | ret = -E2BIG; |
515 | goto out; |
516 | } |
517 | out_page = alloc_page(GFP_NOFS); |
518 | if (out_page == NULL) { |
519 | ret = -ENOMEM; |
520 | goto out; |
521 | } |
522 | pages[nr_pages++] = out_page; |
523 | workspace->out_buf.dst = page_address(out_page); |
524 | workspace->out_buf.pos = 0; |
525 | workspace->out_buf.size = min_t(size_t, max_out, PAGE_SIZE); |
526 | } |
527 | |
528 | if (tot_out >= tot_in) { |
529 | ret = -E2BIG; |
530 | goto out; |
531 | } |
532 | |
533 | ret = 0; |
534 | *total_in = tot_in; |
535 | *total_out = tot_out; |
536 | out: |
537 | *out_pages = nr_pages; |
538 | if (workspace->in_buf.src) { |
539 | kunmap_local(workspace->in_buf.src); |
540 | put_page(page: in_page); |
541 | } |
542 | return ret; |
543 | } |
544 | |
545 | int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb) |
546 | { |
547 | struct workspace *workspace = list_entry(ws, struct workspace, list); |
548 | struct page **pages_in = cb->compressed_pages; |
549 | size_t srclen = cb->compressed_len; |
550 | zstd_dstream *stream; |
551 | int ret = 0; |
552 | unsigned long page_in_index = 0; |
553 | unsigned long total_pages_in = DIV_ROUND_UP(srclen, PAGE_SIZE); |
554 | unsigned long buf_start; |
555 | unsigned long total_out = 0; |
556 | |
557 | stream = zstd_init_dstream( |
558 | ZSTD_BTRFS_MAX_INPUT, workspace: workspace->mem, workspace_size: workspace->size); |
559 | if (!stream) { |
560 | pr_debug("BTRFS: zstd_init_dstream failed\n" ); |
561 | ret = -EIO; |
562 | goto done; |
563 | } |
564 | |
565 | workspace->in_buf.src = kmap_local_page(page: pages_in[page_in_index]); |
566 | workspace->in_buf.pos = 0; |
567 | workspace->in_buf.size = min_t(size_t, srclen, PAGE_SIZE); |
568 | |
569 | workspace->out_buf.dst = workspace->buf; |
570 | workspace->out_buf.pos = 0; |
571 | workspace->out_buf.size = PAGE_SIZE; |
572 | |
573 | while (1) { |
574 | size_t ret2; |
575 | |
576 | ret2 = zstd_decompress_stream(dstream: stream, output: &workspace->out_buf, |
577 | input: &workspace->in_buf); |
578 | if (zstd_is_error(code: ret2)) { |
579 | pr_debug("BTRFS: zstd_decompress_stream returned %d\n" , |
580 | zstd_get_error_code(ret2)); |
581 | ret = -EIO; |
582 | goto done; |
583 | } |
584 | buf_start = total_out; |
585 | total_out += workspace->out_buf.pos; |
586 | workspace->out_buf.pos = 0; |
587 | |
588 | ret = btrfs_decompress_buf2page(buf: workspace->out_buf.dst, |
589 | buf_len: total_out - buf_start, cb, decompressed: buf_start); |
590 | if (ret == 0) |
591 | break; |
592 | |
593 | if (workspace->in_buf.pos >= srclen) |
594 | break; |
595 | |
596 | /* Check if we've hit the end of a frame */ |
597 | if (ret2 == 0) |
598 | break; |
599 | |
600 | if (workspace->in_buf.pos == workspace->in_buf.size) { |
601 | kunmap_local(workspace->in_buf.src); |
602 | page_in_index++; |
603 | if (page_in_index >= total_pages_in) { |
604 | workspace->in_buf.src = NULL; |
605 | ret = -EIO; |
606 | goto done; |
607 | } |
608 | srclen -= PAGE_SIZE; |
609 | workspace->in_buf.src = kmap_local_page(page: pages_in[page_in_index]); |
610 | workspace->in_buf.pos = 0; |
611 | workspace->in_buf.size = min_t(size_t, srclen, PAGE_SIZE); |
612 | } |
613 | } |
614 | ret = 0; |
615 | done: |
616 | if (workspace->in_buf.src) |
617 | kunmap_local(workspace->in_buf.src); |
618 | return ret; |
619 | } |
620 | |
621 | int zstd_decompress(struct list_head *ws, const u8 *data_in, |
622 | struct page *dest_page, unsigned long start_byte, size_t srclen, |
623 | size_t destlen) |
624 | { |
625 | struct workspace *workspace = list_entry(ws, struct workspace, list); |
626 | zstd_dstream *stream; |
627 | int ret = 0; |
628 | size_t ret2; |
629 | unsigned long total_out = 0; |
630 | unsigned long pg_offset = 0; |
631 | |
632 | stream = zstd_init_dstream( |
633 | ZSTD_BTRFS_MAX_INPUT, workspace: workspace->mem, workspace_size: workspace->size); |
634 | if (!stream) { |
635 | pr_warn("BTRFS: zstd_init_dstream failed\n" ); |
636 | ret = -EIO; |
637 | goto finish; |
638 | } |
639 | |
640 | destlen = min_t(size_t, destlen, PAGE_SIZE); |
641 | |
642 | workspace->in_buf.src = data_in; |
643 | workspace->in_buf.pos = 0; |
644 | workspace->in_buf.size = srclen; |
645 | |
646 | workspace->out_buf.dst = workspace->buf; |
647 | workspace->out_buf.pos = 0; |
648 | workspace->out_buf.size = PAGE_SIZE; |
649 | |
650 | ret2 = 1; |
651 | while (pg_offset < destlen |
652 | && workspace->in_buf.pos < workspace->in_buf.size) { |
653 | unsigned long buf_start; |
654 | unsigned long buf_offset; |
655 | unsigned long bytes; |
656 | |
657 | /* Check if the frame is over and we still need more input */ |
658 | if (ret2 == 0) { |
659 | pr_debug("BTRFS: zstd_decompress_stream ended early\n" ); |
660 | ret = -EIO; |
661 | goto finish; |
662 | } |
663 | ret2 = zstd_decompress_stream(dstream: stream, output: &workspace->out_buf, |
664 | input: &workspace->in_buf); |
665 | if (zstd_is_error(code: ret2)) { |
666 | pr_debug("BTRFS: zstd_decompress_stream returned %d\n" , |
667 | zstd_get_error_code(ret2)); |
668 | ret = -EIO; |
669 | goto finish; |
670 | } |
671 | |
672 | buf_start = total_out; |
673 | total_out += workspace->out_buf.pos; |
674 | workspace->out_buf.pos = 0; |
675 | |
676 | if (total_out <= start_byte) |
677 | continue; |
678 | |
679 | if (total_out > start_byte && buf_start < start_byte) |
680 | buf_offset = start_byte - buf_start; |
681 | else |
682 | buf_offset = 0; |
683 | |
684 | bytes = min_t(unsigned long, destlen - pg_offset, |
685 | workspace->out_buf.size - buf_offset); |
686 | |
687 | memcpy_to_page(page: dest_page, offset: pg_offset, |
688 | from: workspace->out_buf.dst + buf_offset, len: bytes); |
689 | |
690 | pg_offset += bytes; |
691 | } |
692 | ret = 0; |
693 | finish: |
694 | if (pg_offset < destlen) { |
695 | memzero_page(page: dest_page, offset: pg_offset, len: destlen - pg_offset); |
696 | } |
697 | return ret; |
698 | } |
699 | |
700 | const struct btrfs_compress_op btrfs_zstd_compress = { |
701 | /* ZSTD uses own workspace manager */ |
702 | .workspace_manager = NULL, |
703 | .max_level = ZSTD_BTRFS_MAX_LEVEL, |
704 | .default_level = ZSTD_BTRFS_DEFAULT_LEVEL, |
705 | }; |
706 | |