1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright (C) 2008 Oracle. All rights reserved. |
4 | * |
5 | * Based on jffs2 zlib code: |
6 | * Copyright © 2001-2007 Red Hat, Inc. |
7 | * Created by David Woodhouse <dwmw2@infradead.org> |
8 | */ |
9 | |
10 | #include <linux/kernel.h> |
11 | #include <linux/slab.h> |
12 | #include <linux/zlib.h> |
13 | #include <linux/zutil.h> |
14 | #include <linux/mm.h> |
15 | #include <linux/init.h> |
16 | #include <linux/err.h> |
17 | #include <linux/sched.h> |
18 | #include <linux/pagemap.h> |
19 | #include <linux/bio.h> |
20 | #include <linux/refcount.h> |
21 | #include "compression.h" |
22 | |
23 | /* workspace buffer size for s390 zlib hardware support */ |
24 | #define ZLIB_DFLTCC_BUF_SIZE (4 * PAGE_SIZE) |
25 | |
26 | struct workspace { |
27 | z_stream strm; |
28 | char *buf; |
29 | unsigned int buf_size; |
30 | struct list_head list; |
31 | int level; |
32 | }; |
33 | |
34 | static struct workspace_manager wsm; |
35 | |
36 | struct list_head *zlib_get_workspace(unsigned int level) |
37 | { |
38 | struct list_head *ws = btrfs_get_workspace(type: BTRFS_COMPRESS_ZLIB, level); |
39 | struct workspace *workspace = list_entry(ws, struct workspace, list); |
40 | |
41 | workspace->level = level; |
42 | |
43 | return ws; |
44 | } |
45 | |
46 | void zlib_free_workspace(struct list_head *ws) |
47 | { |
48 | struct workspace *workspace = list_entry(ws, struct workspace, list); |
49 | |
50 | kvfree(addr: workspace->strm.workspace); |
51 | kfree(objp: workspace->buf); |
52 | kfree(objp: workspace); |
53 | } |
54 | |
55 | struct list_head *zlib_alloc_workspace(unsigned int level) |
56 | { |
57 | struct workspace *workspace; |
58 | int workspacesize; |
59 | |
60 | workspace = kzalloc(size: sizeof(*workspace), GFP_KERNEL); |
61 | if (!workspace) |
62 | return ERR_PTR(error: -ENOMEM); |
63 | |
64 | workspacesize = max(zlib_deflate_workspacesize(MAX_WBITS, MAX_MEM_LEVEL), |
65 | zlib_inflate_workspacesize()); |
66 | workspace->strm.workspace = kvzalloc(size: workspacesize, GFP_KERNEL | __GFP_NOWARN); |
67 | workspace->level = level; |
68 | workspace->buf = NULL; |
69 | /* |
70 | * In case of s390 zlib hardware support, allocate lager workspace |
71 | * buffer. If allocator fails, fall back to a single page buffer. |
72 | */ |
73 | if (zlib_deflate_dfltcc_enabled()) { |
74 | workspace->buf = kmalloc(ZLIB_DFLTCC_BUF_SIZE, |
75 | __GFP_NOMEMALLOC | __GFP_NORETRY | |
76 | __GFP_NOWARN | GFP_NOIO); |
77 | workspace->buf_size = ZLIB_DFLTCC_BUF_SIZE; |
78 | } |
79 | if (!workspace->buf) { |
80 | workspace->buf = kmalloc(PAGE_SIZE, GFP_KERNEL); |
81 | workspace->buf_size = PAGE_SIZE; |
82 | } |
83 | if (!workspace->strm.workspace || !workspace->buf) |
84 | goto fail; |
85 | |
86 | INIT_LIST_HEAD(list: &workspace->list); |
87 | |
88 | return &workspace->list; |
89 | fail: |
90 | zlib_free_workspace(ws: &workspace->list); |
91 | return ERR_PTR(error: -ENOMEM); |
92 | } |
93 | |
94 | int zlib_compress_pages(struct list_head *ws, struct address_space *mapping, |
95 | u64 start, struct page **pages, unsigned long *out_pages, |
96 | unsigned long *total_in, unsigned long *total_out) |
97 | { |
98 | struct workspace *workspace = list_entry(ws, struct workspace, list); |
99 | int ret; |
100 | char *data_in = NULL; |
101 | char *cpage_out; |
102 | int nr_pages = 0; |
103 | struct page *in_page = NULL; |
104 | struct page *out_page = NULL; |
105 | unsigned long bytes_left; |
106 | unsigned int in_buf_pages; |
107 | unsigned long len = *total_out; |
108 | unsigned long nr_dest_pages = *out_pages; |
109 | const unsigned long max_out = nr_dest_pages * PAGE_SIZE; |
110 | |
111 | *out_pages = 0; |
112 | *total_out = 0; |
113 | *total_in = 0; |
114 | |
115 | if (Z_OK != zlib_deflateInit(&workspace->strm, workspace->level)) { |
116 | pr_warn("BTRFS: deflateInit failed\n" ); |
117 | ret = -EIO; |
118 | goto out; |
119 | } |
120 | |
121 | workspace->strm.total_in = 0; |
122 | workspace->strm.total_out = 0; |
123 | |
124 | out_page = alloc_page(GFP_NOFS); |
125 | if (out_page == NULL) { |
126 | ret = -ENOMEM; |
127 | goto out; |
128 | } |
129 | cpage_out = page_address(out_page); |
130 | pages[0] = out_page; |
131 | nr_pages = 1; |
132 | |
133 | workspace->strm.next_in = workspace->buf; |
134 | workspace->strm.avail_in = 0; |
135 | workspace->strm.next_out = cpage_out; |
136 | workspace->strm.avail_out = PAGE_SIZE; |
137 | |
138 | while (workspace->strm.total_in < len) { |
139 | /* |
140 | * Get next input pages and copy the contents to |
141 | * the workspace buffer if required. |
142 | */ |
143 | if (workspace->strm.avail_in == 0) { |
144 | bytes_left = len - workspace->strm.total_in; |
145 | in_buf_pages = min(DIV_ROUND_UP(bytes_left, PAGE_SIZE), |
146 | workspace->buf_size / PAGE_SIZE); |
147 | if (in_buf_pages > 1) { |
148 | int i; |
149 | |
150 | for (i = 0; i < in_buf_pages; i++) { |
151 | if (data_in) { |
152 | kunmap_local(data_in); |
153 | put_page(page: in_page); |
154 | } |
155 | in_page = find_get_page(mapping, |
156 | offset: start >> PAGE_SHIFT); |
157 | data_in = kmap_local_page(page: in_page); |
158 | copy_page(to: workspace->buf + i * PAGE_SIZE, |
159 | from: data_in); |
160 | start += PAGE_SIZE; |
161 | } |
162 | workspace->strm.next_in = workspace->buf; |
163 | } else { |
164 | if (data_in) { |
165 | kunmap_local(data_in); |
166 | put_page(page: in_page); |
167 | } |
168 | in_page = find_get_page(mapping, |
169 | offset: start >> PAGE_SHIFT); |
170 | data_in = kmap_local_page(page: in_page); |
171 | start += PAGE_SIZE; |
172 | workspace->strm.next_in = data_in; |
173 | } |
174 | workspace->strm.avail_in = min(bytes_left, |
175 | (unsigned long) workspace->buf_size); |
176 | } |
177 | |
178 | ret = zlib_deflate(strm: &workspace->strm, Z_SYNC_FLUSH); |
179 | if (ret != Z_OK) { |
180 | pr_debug("BTRFS: deflate in loop returned %d\n" , |
181 | ret); |
182 | zlib_deflateEnd(strm: &workspace->strm); |
183 | ret = -EIO; |
184 | goto out; |
185 | } |
186 | |
187 | /* we're making it bigger, give up */ |
188 | if (workspace->strm.total_in > 8192 && |
189 | workspace->strm.total_in < |
190 | workspace->strm.total_out) { |
191 | ret = -E2BIG; |
192 | goto out; |
193 | } |
194 | /* we need another page for writing out. Test this |
195 | * before the total_in so we will pull in a new page for |
196 | * the stream end if required |
197 | */ |
198 | if (workspace->strm.avail_out == 0) { |
199 | if (nr_pages == nr_dest_pages) { |
200 | ret = -E2BIG; |
201 | goto out; |
202 | } |
203 | out_page = alloc_page(GFP_NOFS); |
204 | if (out_page == NULL) { |
205 | ret = -ENOMEM; |
206 | goto out; |
207 | } |
208 | cpage_out = page_address(out_page); |
209 | pages[nr_pages] = out_page; |
210 | nr_pages++; |
211 | workspace->strm.avail_out = PAGE_SIZE; |
212 | workspace->strm.next_out = cpage_out; |
213 | } |
214 | /* we're all done */ |
215 | if (workspace->strm.total_in >= len) |
216 | break; |
217 | if (workspace->strm.total_out > max_out) |
218 | break; |
219 | } |
220 | workspace->strm.avail_in = 0; |
221 | /* |
222 | * Call deflate with Z_FINISH flush parameter providing more output |
223 | * space but no more input data, until it returns with Z_STREAM_END. |
224 | */ |
225 | while (ret != Z_STREAM_END) { |
226 | ret = zlib_deflate(strm: &workspace->strm, Z_FINISH); |
227 | if (ret == Z_STREAM_END) |
228 | break; |
229 | if (ret != Z_OK && ret != Z_BUF_ERROR) { |
230 | zlib_deflateEnd(strm: &workspace->strm); |
231 | ret = -EIO; |
232 | goto out; |
233 | } else if (workspace->strm.avail_out == 0) { |
234 | /* get another page for the stream end */ |
235 | if (nr_pages == nr_dest_pages) { |
236 | ret = -E2BIG; |
237 | goto out; |
238 | } |
239 | out_page = alloc_page(GFP_NOFS); |
240 | if (out_page == NULL) { |
241 | ret = -ENOMEM; |
242 | goto out; |
243 | } |
244 | cpage_out = page_address(out_page); |
245 | pages[nr_pages] = out_page; |
246 | nr_pages++; |
247 | workspace->strm.avail_out = PAGE_SIZE; |
248 | workspace->strm.next_out = cpage_out; |
249 | } |
250 | } |
251 | zlib_deflateEnd(strm: &workspace->strm); |
252 | |
253 | if (workspace->strm.total_out >= workspace->strm.total_in) { |
254 | ret = -E2BIG; |
255 | goto out; |
256 | } |
257 | |
258 | ret = 0; |
259 | *total_out = workspace->strm.total_out; |
260 | *total_in = workspace->strm.total_in; |
261 | out: |
262 | *out_pages = nr_pages; |
263 | if (data_in) { |
264 | kunmap_local(data_in); |
265 | put_page(page: in_page); |
266 | } |
267 | |
268 | return ret; |
269 | } |
270 | |
271 | int zlib_decompress_bio(struct list_head *ws, struct compressed_bio *cb) |
272 | { |
273 | struct workspace *workspace = list_entry(ws, struct workspace, list); |
274 | int ret = 0, ret2; |
275 | int wbits = MAX_WBITS; |
276 | char *data_in; |
277 | size_t total_out = 0; |
278 | unsigned long page_in_index = 0; |
279 | size_t srclen = cb->compressed_len; |
280 | unsigned long total_pages_in = DIV_ROUND_UP(srclen, PAGE_SIZE); |
281 | unsigned long buf_start; |
282 | struct page **pages_in = cb->compressed_pages; |
283 | |
284 | data_in = kmap_local_page(page: pages_in[page_in_index]); |
285 | workspace->strm.next_in = data_in; |
286 | workspace->strm.avail_in = min_t(size_t, srclen, PAGE_SIZE); |
287 | workspace->strm.total_in = 0; |
288 | |
289 | workspace->strm.total_out = 0; |
290 | workspace->strm.next_out = workspace->buf; |
291 | workspace->strm.avail_out = workspace->buf_size; |
292 | |
293 | /* If it's deflate, and it's got no preset dictionary, then |
294 | we can tell zlib to skip the adler32 check. */ |
295 | if (srclen > 2 && !(data_in[1] & PRESET_DICT) && |
296 | ((data_in[0] & 0x0f) == Z_DEFLATED) && |
297 | !(((data_in[0]<<8) + data_in[1]) % 31)) { |
298 | |
299 | wbits = -((data_in[0] >> 4) + 8); |
300 | workspace->strm.next_in += 2; |
301 | workspace->strm.avail_in -= 2; |
302 | } |
303 | |
304 | if (Z_OK != zlib_inflateInit2(strm: &workspace->strm, windowBits: wbits)) { |
305 | pr_warn("BTRFS: inflateInit failed\n" ); |
306 | kunmap_local(data_in); |
307 | return -EIO; |
308 | } |
309 | while (workspace->strm.total_in < srclen) { |
310 | ret = zlib_inflate(strm: &workspace->strm, Z_NO_FLUSH); |
311 | if (ret != Z_OK && ret != Z_STREAM_END) |
312 | break; |
313 | |
314 | buf_start = total_out; |
315 | total_out = workspace->strm.total_out; |
316 | |
317 | /* we didn't make progress in this inflate call, we're done */ |
318 | if (buf_start == total_out) |
319 | break; |
320 | |
321 | ret2 = btrfs_decompress_buf2page(buf: workspace->buf, |
322 | buf_len: total_out - buf_start, cb, decompressed: buf_start); |
323 | if (ret2 == 0) { |
324 | ret = 0; |
325 | goto done; |
326 | } |
327 | |
328 | workspace->strm.next_out = workspace->buf; |
329 | workspace->strm.avail_out = workspace->buf_size; |
330 | |
331 | if (workspace->strm.avail_in == 0) { |
332 | unsigned long tmp; |
333 | kunmap_local(data_in); |
334 | page_in_index++; |
335 | if (page_in_index >= total_pages_in) { |
336 | data_in = NULL; |
337 | break; |
338 | } |
339 | data_in = kmap_local_page(page: pages_in[page_in_index]); |
340 | workspace->strm.next_in = data_in; |
341 | tmp = srclen - workspace->strm.total_in; |
342 | workspace->strm.avail_in = min(tmp, PAGE_SIZE); |
343 | } |
344 | } |
345 | if (ret != Z_STREAM_END) |
346 | ret = -EIO; |
347 | else |
348 | ret = 0; |
349 | done: |
350 | zlib_inflateEnd(strm: &workspace->strm); |
351 | if (data_in) |
352 | kunmap_local(data_in); |
353 | return ret; |
354 | } |
355 | |
356 | int zlib_decompress(struct list_head *ws, const u8 *data_in, |
357 | struct page *dest_page, unsigned long start_byte, size_t srclen, |
358 | size_t destlen) |
359 | { |
360 | struct workspace *workspace = list_entry(ws, struct workspace, list); |
361 | int ret = 0; |
362 | int wbits = MAX_WBITS; |
363 | unsigned long bytes_left; |
364 | unsigned long total_out = 0; |
365 | unsigned long pg_offset = 0; |
366 | |
367 | destlen = min_t(unsigned long, destlen, PAGE_SIZE); |
368 | bytes_left = destlen; |
369 | |
370 | workspace->strm.next_in = data_in; |
371 | workspace->strm.avail_in = srclen; |
372 | workspace->strm.total_in = 0; |
373 | |
374 | workspace->strm.next_out = workspace->buf; |
375 | workspace->strm.avail_out = workspace->buf_size; |
376 | workspace->strm.total_out = 0; |
377 | /* If it's deflate, and it's got no preset dictionary, then |
378 | we can tell zlib to skip the adler32 check. */ |
379 | if (srclen > 2 && !(data_in[1] & PRESET_DICT) && |
380 | ((data_in[0] & 0x0f) == Z_DEFLATED) && |
381 | !(((data_in[0]<<8) + data_in[1]) % 31)) { |
382 | |
383 | wbits = -((data_in[0] >> 4) + 8); |
384 | workspace->strm.next_in += 2; |
385 | workspace->strm.avail_in -= 2; |
386 | } |
387 | |
388 | if (Z_OK != zlib_inflateInit2(strm: &workspace->strm, windowBits: wbits)) { |
389 | pr_warn("BTRFS: inflateInit failed\n" ); |
390 | return -EIO; |
391 | } |
392 | |
393 | while (bytes_left > 0) { |
394 | unsigned long buf_start; |
395 | unsigned long buf_offset; |
396 | unsigned long bytes; |
397 | |
398 | ret = zlib_inflate(strm: &workspace->strm, Z_NO_FLUSH); |
399 | if (ret != Z_OK && ret != Z_STREAM_END) |
400 | break; |
401 | |
402 | buf_start = total_out; |
403 | total_out = workspace->strm.total_out; |
404 | |
405 | if (total_out == buf_start) { |
406 | ret = -EIO; |
407 | break; |
408 | } |
409 | |
410 | if (total_out <= start_byte) |
411 | goto next; |
412 | |
413 | if (total_out > start_byte && buf_start < start_byte) |
414 | buf_offset = start_byte - buf_start; |
415 | else |
416 | buf_offset = 0; |
417 | |
418 | bytes = min(PAGE_SIZE - pg_offset, |
419 | PAGE_SIZE - (buf_offset % PAGE_SIZE)); |
420 | bytes = min(bytes, bytes_left); |
421 | |
422 | memcpy_to_page(page: dest_page, offset: pg_offset, |
423 | from: workspace->buf + buf_offset, len: bytes); |
424 | |
425 | pg_offset += bytes; |
426 | bytes_left -= bytes; |
427 | next: |
428 | workspace->strm.next_out = workspace->buf; |
429 | workspace->strm.avail_out = workspace->buf_size; |
430 | } |
431 | |
432 | if (ret != Z_STREAM_END && bytes_left != 0) |
433 | ret = -EIO; |
434 | else |
435 | ret = 0; |
436 | |
437 | zlib_inflateEnd(strm: &workspace->strm); |
438 | |
439 | /* |
440 | * this should only happen if zlib returned fewer bytes than we |
441 | * expected. btrfs_get_block is responsible for zeroing from the |
442 | * end of the inline extent (destlen) to the end of the page |
443 | */ |
444 | if (pg_offset < destlen) { |
445 | memzero_page(page: dest_page, offset: pg_offset, len: destlen - pg_offset); |
446 | } |
447 | return ret; |
448 | } |
449 | |
450 | const struct btrfs_compress_op btrfs_zlib_compress = { |
451 | .workspace_manager = &wsm, |
452 | .max_level = 9, |
453 | .default_level = BTRFS_ZLIB_DEFAULT_LEVEL, |
454 | }; |
455 | |