1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * Copyright (C) 2015 Google, Inc. |
4 | * |
5 | * Author: Sami Tolvanen <samitolvanen@google.com> |
6 | */ |
7 | |
8 | #include "dm-verity-fec.h" |
9 | #include <linux/math64.h> |
10 | |
11 | #define DM_MSG_PREFIX "verity-fec" |
12 | |
13 | /* |
14 | * If error correction has been configured, returns true. |
15 | */ |
16 | bool verity_fec_is_enabled(struct dm_verity *v) |
17 | { |
18 | return v->fec && v->fec->dev; |
19 | } |
20 | |
21 | /* |
22 | * Return a pointer to dm_verity_fec_io after dm_verity_io and its variable |
23 | * length fields. |
24 | */ |
25 | static inline struct dm_verity_fec_io *fec_io(struct dm_verity_io *io) |
26 | { |
27 | return (struct dm_verity_fec_io *) verity_io_digest_end(v: io->v, io); |
28 | } |
29 | |
30 | /* |
31 | * Return an interleaved offset for a byte in RS block. |
32 | */ |
33 | static inline u64 fec_interleave(struct dm_verity *v, u64 offset) |
34 | { |
35 | u32 mod; |
36 | |
37 | mod = do_div(offset, v->fec->rsn); |
38 | return offset + mod * (v->fec->rounds << v->data_dev_block_bits); |
39 | } |
40 | |
41 | /* |
42 | * Decode an RS block using Reed-Solomon. |
43 | */ |
44 | static int fec_decode_rs8(struct dm_verity *v, struct dm_verity_fec_io *fio, |
45 | u8 *data, u8 *fec, int neras) |
46 | { |
47 | int i; |
48 | uint16_t par[DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN]; |
49 | |
50 | for (i = 0; i < v->fec->roots; i++) |
51 | par[i] = fec[i]; |
52 | |
53 | return decode_rs8(rs: fio->rs, data, par, len: v->fec->rsn, NULL, no_eras: neras, |
54 | eras_pos: fio->erasures, invmsk: 0, NULL); |
55 | } |
56 | |
57 | /* |
58 | * Read error-correcting codes for the requested RS block. Returns a pointer |
59 | * to the data block. Caller is responsible for releasing buf. |
60 | */ |
61 | static u8 *fec_read_parity(struct dm_verity *v, u64 rsb, int index, |
62 | unsigned int *offset, struct dm_buffer **buf) |
63 | { |
64 | u64 position, block, rem; |
65 | u8 *res; |
66 | |
67 | position = (index + rsb) * v->fec->roots; |
68 | block = div64_u64_rem(dividend: position, divisor: v->fec->io_size, remainder: &rem); |
69 | *offset = (unsigned int)rem; |
70 | |
71 | res = dm_bufio_read(c: v->fec->bufio, block, bp: buf); |
72 | if (IS_ERR(ptr: res)) { |
73 | DMERR("%s: FEC %llu: parity read failed (block %llu): %ld" , |
74 | v->data_dev->name, (unsigned long long)rsb, |
75 | (unsigned long long)block, PTR_ERR(res)); |
76 | *buf = NULL; |
77 | } |
78 | |
79 | return res; |
80 | } |
81 | |
82 | /* Loop over each preallocated buffer slot. */ |
83 | #define fec_for_each_prealloc_buffer(__i) \ |
84 | for (__i = 0; __i < DM_VERITY_FEC_BUF_PREALLOC; __i++) |
85 | |
86 | /* Loop over each extra buffer slot. */ |
87 | #define (io, __i) \ |
88 | for (__i = DM_VERITY_FEC_BUF_PREALLOC; __i < DM_VERITY_FEC_BUF_MAX; __i++) |
89 | |
90 | /* Loop over each allocated buffer. */ |
91 | #define fec_for_each_buffer(io, __i) \ |
92 | for (__i = 0; __i < (io)->nbufs; __i++) |
93 | |
94 | /* Loop over each RS block in each allocated buffer. */ |
95 | #define fec_for_each_buffer_rs_block(io, __i, __j) \ |
96 | fec_for_each_buffer(io, __i) \ |
97 | for (__j = 0; __j < 1 << DM_VERITY_FEC_BUF_RS_BITS; __j++) |
98 | |
99 | /* |
100 | * Return a pointer to the current RS block when called inside |
101 | * fec_for_each_buffer_rs_block. |
102 | */ |
103 | static inline u8 *fec_buffer_rs_block(struct dm_verity *v, |
104 | struct dm_verity_fec_io *fio, |
105 | unsigned int i, unsigned int j) |
106 | { |
107 | return &fio->bufs[i][j * v->fec->rsn]; |
108 | } |
109 | |
110 | /* |
111 | * Return an index to the current RS block when called inside |
112 | * fec_for_each_buffer_rs_block. |
113 | */ |
114 | static inline unsigned int fec_buffer_rs_index(unsigned int i, unsigned int j) |
115 | { |
116 | return (i << DM_VERITY_FEC_BUF_RS_BITS) + j; |
117 | } |
118 | |
119 | /* |
120 | * Decode all RS blocks from buffers and copy corrected bytes into fio->output |
121 | * starting from block_offset. |
122 | */ |
123 | static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio, |
124 | u64 rsb, int byte_index, unsigned int block_offset, |
125 | int neras) |
126 | { |
127 | int r, corrected = 0, res; |
128 | struct dm_buffer *buf; |
129 | unsigned int n, i, offset; |
130 | u8 *par, *block; |
131 | |
132 | par = fec_read_parity(v, rsb, index: block_offset, offset: &offset, buf: &buf); |
133 | if (IS_ERR(ptr: par)) |
134 | return PTR_ERR(ptr: par); |
135 | |
136 | /* |
137 | * Decode the RS blocks we have in bufs. Each RS block results in |
138 | * one corrected target byte and consumes fec->roots parity bytes. |
139 | */ |
140 | fec_for_each_buffer_rs_block(fio, n, i) { |
141 | block = fec_buffer_rs_block(v, fio, i: n, j: i); |
142 | res = fec_decode_rs8(v, fio, data: block, fec: &par[offset], neras); |
143 | if (res < 0) { |
144 | r = res; |
145 | goto error; |
146 | } |
147 | |
148 | corrected += res; |
149 | fio->output[block_offset] = block[byte_index]; |
150 | |
151 | block_offset++; |
152 | if (block_offset >= 1 << v->data_dev_block_bits) |
153 | goto done; |
154 | |
155 | /* read the next block when we run out of parity bytes */ |
156 | offset += v->fec->roots; |
157 | if (offset >= v->fec->io_size) { |
158 | dm_bufio_release(b: buf); |
159 | |
160 | par = fec_read_parity(v, rsb, index: block_offset, offset: &offset, buf: &buf); |
161 | if (IS_ERR(ptr: par)) |
162 | return PTR_ERR(ptr: par); |
163 | } |
164 | } |
165 | done: |
166 | r = corrected; |
167 | error: |
168 | dm_bufio_release(b: buf); |
169 | |
170 | if (r < 0 && neras) |
171 | DMERR_LIMIT("%s: FEC %llu: failed to correct: %d" , |
172 | v->data_dev->name, (unsigned long long)rsb, r); |
173 | else if (r > 0) |
174 | DMWARN_LIMIT("%s: FEC %llu: corrected %d errors" , |
175 | v->data_dev->name, (unsigned long long)rsb, r); |
176 | |
177 | return r; |
178 | } |
179 | |
180 | /* |
181 | * Locate data block erasures using verity hashes. |
182 | */ |
183 | static int fec_is_erasure(struct dm_verity *v, struct dm_verity_io *io, |
184 | u8 *want_digest, u8 *data) |
185 | { |
186 | if (unlikely(verity_hash(v, verity_io_hash_req(v, io), |
187 | data, 1 << v->data_dev_block_bits, |
188 | verity_io_real_digest(v, io)))) |
189 | return 0; |
190 | |
191 | return memcmp(p: verity_io_real_digest(v, io), q: want_digest, |
192 | size: v->digest_size) != 0; |
193 | } |
194 | |
195 | /* |
196 | * Read data blocks that are part of the RS block and deinterleave as much as |
197 | * fits into buffers. Check for erasure locations if @neras is non-NULL. |
198 | */ |
199 | static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io, |
200 | u64 rsb, u64 target, unsigned int block_offset, |
201 | int *neras) |
202 | { |
203 | bool is_zero; |
204 | int i, j, target_index = -1; |
205 | struct dm_buffer *buf; |
206 | struct dm_bufio_client *bufio; |
207 | struct dm_verity_fec_io *fio = fec_io(io); |
208 | u64 block, ileaved; |
209 | u8 *bbuf, *rs_block; |
210 | u8 want_digest[HASH_MAX_DIGESTSIZE]; |
211 | unsigned int n, k; |
212 | |
213 | if (neras) |
214 | *neras = 0; |
215 | |
216 | if (WARN_ON(v->digest_size > sizeof(want_digest))) |
217 | return -EINVAL; |
218 | |
219 | /* |
220 | * read each of the rsn data blocks that are part of the RS block, and |
221 | * interleave contents to available bufs |
222 | */ |
223 | for (i = 0; i < v->fec->rsn; i++) { |
224 | ileaved = fec_interleave(v, offset: rsb * v->fec->rsn + i); |
225 | |
226 | /* |
227 | * target is the data block we want to correct, target_index is |
228 | * the index of this block within the rsn RS blocks |
229 | */ |
230 | if (ileaved == target) |
231 | target_index = i; |
232 | |
233 | block = ileaved >> v->data_dev_block_bits; |
234 | bufio = v->fec->data_bufio; |
235 | |
236 | if (block >= v->data_blocks) { |
237 | block -= v->data_blocks; |
238 | |
239 | /* |
240 | * blocks outside the area were assumed to contain |
241 | * zeros when encoding data was generated |
242 | */ |
243 | if (unlikely(block >= v->fec->hash_blocks)) |
244 | continue; |
245 | |
246 | block += v->hash_start; |
247 | bufio = v->bufio; |
248 | } |
249 | |
250 | bbuf = dm_bufio_read(c: bufio, block, bp: &buf); |
251 | if (IS_ERR(ptr: bbuf)) { |
252 | DMWARN_LIMIT("%s: FEC %llu: read failed (%llu): %ld" , |
253 | v->data_dev->name, |
254 | (unsigned long long)rsb, |
255 | (unsigned long long)block, PTR_ERR(bbuf)); |
256 | |
257 | /* assume the block is corrupted */ |
258 | if (neras && *neras <= v->fec->roots) |
259 | fio->erasures[(*neras)++] = i; |
260 | |
261 | continue; |
262 | } |
263 | |
264 | /* locate erasures if the block is on the data device */ |
265 | if (bufio == v->fec->data_bufio && |
266 | verity_hash_for_block(v, io, block, digest: want_digest, |
267 | is_zero: &is_zero) == 0) { |
268 | /* skip known zero blocks entirely */ |
269 | if (is_zero) |
270 | goto done; |
271 | |
272 | /* |
273 | * skip if we have already found the theoretical |
274 | * maximum number (i.e. fec->roots) of erasures |
275 | */ |
276 | if (neras && *neras <= v->fec->roots && |
277 | fec_is_erasure(v, io, want_digest, data: bbuf)) |
278 | fio->erasures[(*neras)++] = i; |
279 | } |
280 | |
281 | /* |
282 | * deinterleave and copy the bytes that fit into bufs, |
283 | * starting from block_offset |
284 | */ |
285 | fec_for_each_buffer_rs_block(fio, n, j) { |
286 | k = fec_buffer_rs_index(i: n, j) + block_offset; |
287 | |
288 | if (k >= 1 << v->data_dev_block_bits) |
289 | goto done; |
290 | |
291 | rs_block = fec_buffer_rs_block(v, fio, i: n, j); |
292 | rs_block[i] = bbuf[k]; |
293 | } |
294 | done: |
295 | dm_bufio_release(b: buf); |
296 | } |
297 | |
298 | return target_index; |
299 | } |
300 | |
301 | /* |
302 | * Allocate RS control structure and FEC buffers from preallocated mempools, |
303 | * and attempt to allocate as many extra buffers as available. |
304 | */ |
305 | static int fec_alloc_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio) |
306 | { |
307 | unsigned int n; |
308 | |
309 | if (!fio->rs) |
310 | fio->rs = mempool_alloc(pool: &v->fec->rs_pool, GFP_NOIO); |
311 | |
312 | fec_for_each_prealloc_buffer(n) { |
313 | if (fio->bufs[n]) |
314 | continue; |
315 | |
316 | fio->bufs[n] = mempool_alloc(pool: &v->fec->prealloc_pool, GFP_NOWAIT); |
317 | if (unlikely(!fio->bufs[n])) { |
318 | DMERR("failed to allocate FEC buffer" ); |
319 | return -ENOMEM; |
320 | } |
321 | } |
322 | |
323 | /* try to allocate the maximum number of buffers */ |
324 | fec_for_each_extra_buffer(fio, n) { |
325 | if (fio->bufs[n]) |
326 | continue; |
327 | |
328 | fio->bufs[n] = mempool_alloc(pool: &v->fec->extra_pool, GFP_NOWAIT); |
329 | /* we can manage with even one buffer if necessary */ |
330 | if (unlikely(!fio->bufs[n])) |
331 | break; |
332 | } |
333 | fio->nbufs = n; |
334 | |
335 | if (!fio->output) |
336 | fio->output = mempool_alloc(pool: &v->fec->output_pool, GFP_NOIO); |
337 | |
338 | return 0; |
339 | } |
340 | |
341 | /* |
342 | * Initialize buffers and clear erasures. fec_read_bufs() assumes buffers are |
343 | * zeroed before deinterleaving. |
344 | */ |
345 | static void fec_init_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio) |
346 | { |
347 | unsigned int n; |
348 | |
349 | fec_for_each_buffer(fio, n) |
350 | memset(fio->bufs[n], 0, v->fec->rsn << DM_VERITY_FEC_BUF_RS_BITS); |
351 | |
352 | memset(fio->erasures, 0, sizeof(fio->erasures)); |
353 | } |
354 | |
355 | /* |
356 | * Decode all RS blocks in a single data block and return the target block |
357 | * (indicated by @offset) in fio->output. If @use_erasures is non-zero, uses |
358 | * hashes to locate erasures. |
359 | */ |
360 | static int fec_decode_rsb(struct dm_verity *v, struct dm_verity_io *io, |
361 | struct dm_verity_fec_io *fio, u64 rsb, u64 offset, |
362 | bool use_erasures) |
363 | { |
364 | int r, neras = 0; |
365 | unsigned int pos; |
366 | |
367 | r = fec_alloc_bufs(v, fio); |
368 | if (unlikely(r < 0)) |
369 | return r; |
370 | |
371 | for (pos = 0; pos < 1 << v->data_dev_block_bits; ) { |
372 | fec_init_bufs(v, fio); |
373 | |
374 | r = fec_read_bufs(v, io, rsb, target: offset, block_offset: pos, |
375 | neras: use_erasures ? &neras : NULL); |
376 | if (unlikely(r < 0)) |
377 | return r; |
378 | |
379 | r = fec_decode_bufs(v, fio, rsb, byte_index: r, block_offset: pos, neras); |
380 | if (r < 0) |
381 | return r; |
382 | |
383 | pos += fio->nbufs << DM_VERITY_FEC_BUF_RS_BITS; |
384 | } |
385 | |
386 | /* Always re-validate the corrected block against the expected hash */ |
387 | r = verity_hash(v, req: verity_io_hash_req(v, io), data: fio->output, |
388 | len: 1 << v->data_dev_block_bits, |
389 | digest: verity_io_real_digest(v, io)); |
390 | if (unlikely(r < 0)) |
391 | return r; |
392 | |
393 | if (memcmp(p: verity_io_real_digest(v, io), q: verity_io_want_digest(v, io), |
394 | size: v->digest_size)) { |
395 | DMERR_LIMIT("%s: FEC %llu: failed to correct (%d erasures)" , |
396 | v->data_dev->name, (unsigned long long)rsb, neras); |
397 | return -EILSEQ; |
398 | } |
399 | |
400 | return 0; |
401 | } |
402 | |
403 | static int fec_bv_copy(struct dm_verity *v, struct dm_verity_io *io, u8 *data, |
404 | size_t len) |
405 | { |
406 | struct dm_verity_fec_io *fio = fec_io(io); |
407 | |
408 | memcpy(data, &fio->output[fio->output_pos], len); |
409 | fio->output_pos += len; |
410 | |
411 | return 0; |
412 | } |
413 | |
414 | /* |
415 | * Correct errors in a block. Copies corrected block to dest if non-NULL, |
416 | * otherwise to a bio_vec starting from iter. |
417 | */ |
418 | int verity_fec_decode(struct dm_verity *v, struct dm_verity_io *io, |
419 | enum verity_block_type type, sector_t block, u8 *dest, |
420 | struct bvec_iter *iter) |
421 | { |
422 | int r; |
423 | struct dm_verity_fec_io *fio = fec_io(io); |
424 | u64 offset, res, rsb; |
425 | |
426 | if (!verity_fec_is_enabled(v)) |
427 | return -EOPNOTSUPP; |
428 | |
429 | if (fio->level >= DM_VERITY_FEC_MAX_RECURSION) { |
430 | DMWARN_LIMIT("%s: FEC: recursion too deep" , v->data_dev->name); |
431 | return -EIO; |
432 | } |
433 | |
434 | fio->level++; |
435 | |
436 | if (type == DM_VERITY_BLOCK_TYPE_METADATA) |
437 | block = block - v->hash_start + v->data_blocks; |
438 | |
439 | /* |
440 | * For RS(M, N), the continuous FEC data is divided into blocks of N |
441 | * bytes. Since block size may not be divisible by N, the last block |
442 | * is zero padded when decoding. |
443 | * |
444 | * Each byte of the block is covered by a different RS(M, N) code, |
445 | * and each code is interleaved over N blocks to make it less likely |
446 | * that bursty corruption will leave us in unrecoverable state. |
447 | */ |
448 | |
449 | offset = block << v->data_dev_block_bits; |
450 | res = div64_u64(dividend: offset, divisor: v->fec->rounds << v->data_dev_block_bits); |
451 | |
452 | /* |
453 | * The base RS block we can feed to the interleaver to find out all |
454 | * blocks required for decoding. |
455 | */ |
456 | rsb = offset - res * (v->fec->rounds << v->data_dev_block_bits); |
457 | |
458 | /* |
459 | * Locating erasures is slow, so attempt to recover the block without |
460 | * them first. Do a second attempt with erasures if the corruption is |
461 | * bad enough. |
462 | */ |
463 | r = fec_decode_rsb(v, io, fio, rsb, offset, use_erasures: false); |
464 | if (r < 0) { |
465 | r = fec_decode_rsb(v, io, fio, rsb, offset, use_erasures: true); |
466 | if (r < 0) |
467 | goto done; |
468 | } |
469 | |
470 | if (dest) |
471 | memcpy(dest, fio->output, 1 << v->data_dev_block_bits); |
472 | else if (iter) { |
473 | fio->output_pos = 0; |
474 | r = verity_for_bv_block(v, io, iter, process: fec_bv_copy); |
475 | } |
476 | |
477 | done: |
478 | fio->level--; |
479 | return r; |
480 | } |
481 | |
482 | /* |
483 | * Clean up per-bio data. |
484 | */ |
485 | void verity_fec_finish_io(struct dm_verity_io *io) |
486 | { |
487 | unsigned int n; |
488 | struct dm_verity_fec *f = io->v->fec; |
489 | struct dm_verity_fec_io *fio = fec_io(io); |
490 | |
491 | if (!verity_fec_is_enabled(v: io->v)) |
492 | return; |
493 | |
494 | mempool_free(element: fio->rs, pool: &f->rs_pool); |
495 | |
496 | fec_for_each_prealloc_buffer(n) |
497 | mempool_free(element: fio->bufs[n], pool: &f->prealloc_pool); |
498 | |
499 | fec_for_each_extra_buffer(fio, n) |
500 | mempool_free(element: fio->bufs[n], pool: &f->extra_pool); |
501 | |
502 | mempool_free(element: fio->output, pool: &f->output_pool); |
503 | } |
504 | |
505 | /* |
506 | * Initialize per-bio data. |
507 | */ |
508 | void verity_fec_init_io(struct dm_verity_io *io) |
509 | { |
510 | struct dm_verity_fec_io *fio = fec_io(io); |
511 | |
512 | if (!verity_fec_is_enabled(v: io->v)) |
513 | return; |
514 | |
515 | fio->rs = NULL; |
516 | memset(fio->bufs, 0, sizeof(fio->bufs)); |
517 | fio->nbufs = 0; |
518 | fio->output = NULL; |
519 | fio->level = 0; |
520 | } |
521 | |
522 | /* |
523 | * Append feature arguments and values to the status table. |
524 | */ |
525 | unsigned int verity_fec_status_table(struct dm_verity *v, unsigned int sz, |
526 | char *result, unsigned int maxlen) |
527 | { |
528 | if (!verity_fec_is_enabled(v)) |
529 | return sz; |
530 | |
531 | DMEMIT(" " DM_VERITY_OPT_FEC_DEV " %s " |
532 | DM_VERITY_OPT_FEC_BLOCKS " %llu " |
533 | DM_VERITY_OPT_FEC_START " %llu " |
534 | DM_VERITY_OPT_FEC_ROOTS " %d" , |
535 | v->fec->dev->name, |
536 | (unsigned long long)v->fec->blocks, |
537 | (unsigned long long)v->fec->start, |
538 | v->fec->roots); |
539 | |
540 | return sz; |
541 | } |
542 | |
543 | void verity_fec_dtr(struct dm_verity *v) |
544 | { |
545 | struct dm_verity_fec *f = v->fec; |
546 | |
547 | if (!verity_fec_is_enabled(v)) |
548 | goto out; |
549 | |
550 | mempool_exit(pool: &f->rs_pool); |
551 | mempool_exit(pool: &f->prealloc_pool); |
552 | mempool_exit(pool: &f->extra_pool); |
553 | mempool_exit(pool: &f->output_pool); |
554 | kmem_cache_destroy(s: f->cache); |
555 | |
556 | if (f->data_bufio) |
557 | dm_bufio_client_destroy(c: f->data_bufio); |
558 | if (f->bufio) |
559 | dm_bufio_client_destroy(c: f->bufio); |
560 | |
561 | if (f->dev) |
562 | dm_put_device(ti: v->ti, d: f->dev); |
563 | out: |
564 | kfree(objp: f); |
565 | v->fec = NULL; |
566 | } |
567 | |
568 | static void *fec_rs_alloc(gfp_t gfp_mask, void *pool_data) |
569 | { |
570 | struct dm_verity *v = pool_data; |
571 | |
572 | return init_rs_gfp(symsize: 8, gfpoly: 0x11d, fcr: 0, prim: 1, nroots: v->fec->roots, gfp: gfp_mask); |
573 | } |
574 | |
575 | static void fec_rs_free(void *element, void *pool_data) |
576 | { |
577 | struct rs_control *rs = element; |
578 | |
579 | if (rs) |
580 | free_rs(rs); |
581 | } |
582 | |
583 | bool verity_is_fec_opt_arg(const char *arg_name) |
584 | { |
585 | return (!strcasecmp(s1: arg_name, DM_VERITY_OPT_FEC_DEV) || |
586 | !strcasecmp(s1: arg_name, DM_VERITY_OPT_FEC_BLOCKS) || |
587 | !strcasecmp(s1: arg_name, DM_VERITY_OPT_FEC_START) || |
588 | !strcasecmp(s1: arg_name, DM_VERITY_OPT_FEC_ROOTS)); |
589 | } |
590 | |
591 | int verity_fec_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v, |
592 | unsigned int *argc, const char *arg_name) |
593 | { |
594 | int r; |
595 | struct dm_target *ti = v->ti; |
596 | const char *arg_value; |
597 | unsigned long long num_ll; |
598 | unsigned char num_c; |
599 | char dummy; |
600 | |
601 | if (!*argc) { |
602 | ti->error = "FEC feature arguments require a value" ; |
603 | return -EINVAL; |
604 | } |
605 | |
606 | arg_value = dm_shift_arg(as); |
607 | (*argc)--; |
608 | |
609 | if (!strcasecmp(s1: arg_name, DM_VERITY_OPT_FEC_DEV)) { |
610 | r = dm_get_device(ti, path: arg_value, BLK_OPEN_READ, result: &v->fec->dev); |
611 | if (r) { |
612 | ti->error = "FEC device lookup failed" ; |
613 | return r; |
614 | } |
615 | |
616 | } else if (!strcasecmp(s1: arg_name, DM_VERITY_OPT_FEC_BLOCKS)) { |
617 | if (sscanf(arg_value, "%llu%c" , &num_ll, &dummy) != 1 || |
618 | ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) |
619 | >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) { |
620 | ti->error = "Invalid " DM_VERITY_OPT_FEC_BLOCKS; |
621 | return -EINVAL; |
622 | } |
623 | v->fec->blocks = num_ll; |
624 | |
625 | } else if (!strcasecmp(s1: arg_name, DM_VERITY_OPT_FEC_START)) { |
626 | if (sscanf(arg_value, "%llu%c" , &num_ll, &dummy) != 1 || |
627 | ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) >> |
628 | (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) { |
629 | ti->error = "Invalid " DM_VERITY_OPT_FEC_START; |
630 | return -EINVAL; |
631 | } |
632 | v->fec->start = num_ll; |
633 | |
634 | } else if (!strcasecmp(s1: arg_name, DM_VERITY_OPT_FEC_ROOTS)) { |
635 | if (sscanf(arg_value, "%hhu%c" , &num_c, &dummy) != 1 || !num_c || |
636 | num_c < (DM_VERITY_FEC_RSM - DM_VERITY_FEC_MAX_RSN) || |
637 | num_c > (DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN)) { |
638 | ti->error = "Invalid " DM_VERITY_OPT_FEC_ROOTS; |
639 | return -EINVAL; |
640 | } |
641 | v->fec->roots = num_c; |
642 | |
643 | } else { |
644 | ti->error = "Unrecognized verity FEC feature request" ; |
645 | return -EINVAL; |
646 | } |
647 | |
648 | return 0; |
649 | } |
650 | |
651 | /* |
652 | * Allocate dm_verity_fec for v->fec. Must be called before verity_fec_ctr. |
653 | */ |
654 | int verity_fec_ctr_alloc(struct dm_verity *v) |
655 | { |
656 | struct dm_verity_fec *f; |
657 | |
658 | f = kzalloc(size: sizeof(struct dm_verity_fec), GFP_KERNEL); |
659 | if (!f) { |
660 | v->ti->error = "Cannot allocate FEC structure" ; |
661 | return -ENOMEM; |
662 | } |
663 | v->fec = f; |
664 | |
665 | return 0; |
666 | } |
667 | |
668 | /* |
669 | * Validate arguments and preallocate memory. Must be called after arguments |
670 | * have been parsed using verity_fec_parse_opt_args. |
671 | */ |
672 | int verity_fec_ctr(struct dm_verity *v) |
673 | { |
674 | struct dm_verity_fec *f = v->fec; |
675 | struct dm_target *ti = v->ti; |
676 | u64 hash_blocks, fec_blocks; |
677 | int ret; |
678 | |
679 | if (!verity_fec_is_enabled(v)) { |
680 | verity_fec_dtr(v); |
681 | return 0; |
682 | } |
683 | |
684 | /* |
685 | * FEC is computed over data blocks, possible metadata, and |
686 | * hash blocks. In other words, FEC covers total of fec_blocks |
687 | * blocks consisting of the following: |
688 | * |
689 | * data blocks | hash blocks | metadata (optional) |
690 | * |
691 | * We allow metadata after hash blocks to support a use case |
692 | * where all data is stored on the same device and FEC covers |
693 | * the entire area. |
694 | * |
695 | * If metadata is included, we require it to be available on the |
696 | * hash device after the hash blocks. |
697 | */ |
698 | |
699 | hash_blocks = v->hash_blocks - v->hash_start; |
700 | |
701 | /* |
702 | * Require matching block sizes for data and hash devices for |
703 | * simplicity. |
704 | */ |
705 | if (v->data_dev_block_bits != v->hash_dev_block_bits) { |
706 | ti->error = "Block sizes must match to use FEC" ; |
707 | return -EINVAL; |
708 | } |
709 | |
710 | if (!f->roots) { |
711 | ti->error = "Missing " DM_VERITY_OPT_FEC_ROOTS; |
712 | return -EINVAL; |
713 | } |
714 | f->rsn = DM_VERITY_FEC_RSM - f->roots; |
715 | |
716 | if (!f->blocks) { |
717 | ti->error = "Missing " DM_VERITY_OPT_FEC_BLOCKS; |
718 | return -EINVAL; |
719 | } |
720 | |
721 | f->rounds = f->blocks; |
722 | if (sector_div(f->rounds, f->rsn)) |
723 | f->rounds++; |
724 | |
725 | /* |
726 | * Due to optional metadata, f->blocks can be larger than |
727 | * data_blocks and hash_blocks combined. |
728 | */ |
729 | if (f->blocks < v->data_blocks + hash_blocks || !f->rounds) { |
730 | ti->error = "Invalid " DM_VERITY_OPT_FEC_BLOCKS; |
731 | return -EINVAL; |
732 | } |
733 | |
734 | /* |
735 | * Metadata is accessed through the hash device, so we require |
736 | * it to be large enough. |
737 | */ |
738 | f->hash_blocks = f->blocks - v->data_blocks; |
739 | if (dm_bufio_get_device_size(c: v->bufio) < f->hash_blocks) { |
740 | ti->error = "Hash device is too small for " |
741 | DM_VERITY_OPT_FEC_BLOCKS; |
742 | return -E2BIG; |
743 | } |
744 | |
745 | if ((f->roots << SECTOR_SHIFT) & ((1 << v->data_dev_block_bits) - 1)) |
746 | f->io_size = 1 << v->data_dev_block_bits; |
747 | else |
748 | f->io_size = v->fec->roots << SECTOR_SHIFT; |
749 | |
750 | f->bufio = dm_bufio_client_create(bdev: f->dev->bdev, |
751 | block_size: f->io_size, |
752 | reserved_buffers: 1, aux_size: 0, NULL, NULL, flags: 0); |
753 | if (IS_ERR(ptr: f->bufio)) { |
754 | ti->error = "Cannot initialize FEC bufio client" ; |
755 | return PTR_ERR(ptr: f->bufio); |
756 | } |
757 | |
758 | dm_bufio_set_sector_offset(c: f->bufio, start: f->start << (v->data_dev_block_bits - SECTOR_SHIFT)); |
759 | |
760 | fec_blocks = div64_u64(dividend: f->rounds * f->roots, divisor: v->fec->roots << SECTOR_SHIFT); |
761 | if (dm_bufio_get_device_size(c: f->bufio) < fec_blocks) { |
762 | ti->error = "FEC device is too small" ; |
763 | return -E2BIG; |
764 | } |
765 | |
766 | f->data_bufio = dm_bufio_client_create(bdev: v->data_dev->bdev, |
767 | block_size: 1 << v->data_dev_block_bits, |
768 | reserved_buffers: 1, aux_size: 0, NULL, NULL, flags: 0); |
769 | if (IS_ERR(ptr: f->data_bufio)) { |
770 | ti->error = "Cannot initialize FEC data bufio client" ; |
771 | return PTR_ERR(ptr: f->data_bufio); |
772 | } |
773 | |
774 | if (dm_bufio_get_device_size(c: f->data_bufio) < v->data_blocks) { |
775 | ti->error = "Data device is too small" ; |
776 | return -E2BIG; |
777 | } |
778 | |
779 | /* Preallocate an rs_control structure for each worker thread */ |
780 | ret = mempool_init(pool: &f->rs_pool, min_nr: num_online_cpus(), alloc_fn: fec_rs_alloc, |
781 | free_fn: fec_rs_free, pool_data: (void *) v); |
782 | if (ret) { |
783 | ti->error = "Cannot allocate RS pool" ; |
784 | return ret; |
785 | } |
786 | |
787 | f->cache = kmem_cache_create(name: "dm_verity_fec_buffers" , |
788 | size: f->rsn << DM_VERITY_FEC_BUF_RS_BITS, |
789 | align: 0, flags: 0, NULL); |
790 | if (!f->cache) { |
791 | ti->error = "Cannot create FEC buffer cache" ; |
792 | return -ENOMEM; |
793 | } |
794 | |
795 | /* Preallocate DM_VERITY_FEC_BUF_PREALLOC buffers for each thread */ |
796 | ret = mempool_init_slab_pool(pool: &f->prealloc_pool, min_nr: num_online_cpus() * |
797 | DM_VERITY_FEC_BUF_PREALLOC, |
798 | kc: f->cache); |
799 | if (ret) { |
800 | ti->error = "Cannot allocate FEC buffer prealloc pool" ; |
801 | return ret; |
802 | } |
803 | |
804 | ret = mempool_init_slab_pool(pool: &f->extra_pool, min_nr: 0, kc: f->cache); |
805 | if (ret) { |
806 | ti->error = "Cannot allocate FEC buffer extra pool" ; |
807 | return ret; |
808 | } |
809 | |
810 | /* Preallocate an output buffer for each thread */ |
811 | ret = mempool_init_kmalloc_pool(pool: &f->output_pool, min_nr: num_online_cpus(), |
812 | size: 1 << v->data_dev_block_bits); |
813 | if (ret) { |
814 | ti->error = "Cannot allocate FEC output pool" ; |
815 | return ret; |
816 | } |
817 | |
818 | /* Reserve space for our per-bio data */ |
819 | ti->per_io_data_size += sizeof(struct dm_verity_fec_io); |
820 | |
821 | return 0; |
822 | } |
823 | |