1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * Copyright (C) 2015 Google, Inc. |
4 | * |
5 | * Author: Sami Tolvanen <samitolvanen@google.com> |
6 | */ |
7 | |
8 | #include "dm-verity-fec.h" |
9 | #include <linux/math64.h> |
10 | |
11 | #define DM_MSG_PREFIX "verity-fec" |
12 | |
13 | /* |
14 | * If error correction has been configured, returns true. |
15 | */ |
16 | bool verity_fec_is_enabled(struct dm_verity *v) |
17 | { |
18 | return v->fec && v->fec->dev; |
19 | } |
20 | |
21 | /* |
22 | * Return a pointer to dm_verity_fec_io after dm_verity_io and its variable |
23 | * length fields. |
24 | */ |
25 | static inline struct dm_verity_fec_io *fec_io(struct dm_verity_io *io) |
26 | { |
27 | return (struct dm_verity_fec_io *) |
28 | ((char *)io + io->v->ti->per_io_data_size - sizeof(struct dm_verity_fec_io)); |
29 | } |
30 | |
31 | /* |
32 | * Return an interleaved offset for a byte in RS block. |
33 | */ |
34 | static inline u64 fec_interleave(struct dm_verity *v, u64 offset) |
35 | { |
36 | u32 mod; |
37 | |
38 | mod = do_div(offset, v->fec->rsn); |
39 | return offset + mod * (v->fec->rounds << v->data_dev_block_bits); |
40 | } |
41 | |
42 | /* |
43 | * Decode an RS block using Reed-Solomon. |
44 | */ |
45 | static int fec_decode_rs8(struct dm_verity *v, struct dm_verity_fec_io *fio, |
46 | u8 *data, u8 *fec, int neras) |
47 | { |
48 | int i; |
49 | uint16_t par[DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN]; |
50 | |
51 | for (i = 0; i < v->fec->roots; i++) |
52 | par[i] = fec[i]; |
53 | |
54 | return decode_rs8(rs: fio->rs, data, par, len: v->fec->rsn, NULL, no_eras: neras, |
55 | eras_pos: fio->erasures, invmsk: 0, NULL); |
56 | } |
57 | |
58 | /* |
59 | * Read error-correcting codes for the requested RS block. Returns a pointer |
60 | * to the data block. Caller is responsible for releasing buf. |
61 | */ |
62 | static u8 *fec_read_parity(struct dm_verity *v, u64 rsb, int index, |
63 | unsigned int *offset, struct dm_buffer **buf, |
64 | unsigned short ioprio) |
65 | { |
66 | u64 position, block, rem; |
67 | u8 *res; |
68 | |
69 | position = (index + rsb) * v->fec->roots; |
70 | block = div64_u64_rem(dividend: position, divisor: v->fec->io_size, remainder: &rem); |
71 | *offset = (unsigned int)rem; |
72 | |
73 | res = dm_bufio_read_with_ioprio(c: v->fec->bufio, block, bp: buf, ioprio); |
74 | if (IS_ERR(ptr: res)) { |
75 | DMERR("%s: FEC %llu: parity read failed (block %llu): %ld" , |
76 | v->data_dev->name, (unsigned long long)rsb, |
77 | (unsigned long long)block, PTR_ERR(res)); |
78 | *buf = NULL; |
79 | } |
80 | |
81 | return res; |
82 | } |
83 | |
84 | /* Loop over each preallocated buffer slot. */ |
85 | #define fec_for_each_prealloc_buffer(__i) \ |
86 | for (__i = 0; __i < DM_VERITY_FEC_BUF_PREALLOC; __i++) |
87 | |
88 | /* Loop over each extra buffer slot. */ |
89 | #define (io, __i) \ |
90 | for (__i = DM_VERITY_FEC_BUF_PREALLOC; __i < DM_VERITY_FEC_BUF_MAX; __i++) |
91 | |
92 | /* Loop over each allocated buffer. */ |
93 | #define fec_for_each_buffer(io, __i) \ |
94 | for (__i = 0; __i < (io)->nbufs; __i++) |
95 | |
96 | /* Loop over each RS block in each allocated buffer. */ |
97 | #define fec_for_each_buffer_rs_block(io, __i, __j) \ |
98 | fec_for_each_buffer(io, __i) \ |
99 | for (__j = 0; __j < 1 << DM_VERITY_FEC_BUF_RS_BITS; __j++) |
100 | |
101 | /* |
102 | * Return a pointer to the current RS block when called inside |
103 | * fec_for_each_buffer_rs_block. |
104 | */ |
105 | static inline u8 *fec_buffer_rs_block(struct dm_verity *v, |
106 | struct dm_verity_fec_io *fio, |
107 | unsigned int i, unsigned int j) |
108 | { |
109 | return &fio->bufs[i][j * v->fec->rsn]; |
110 | } |
111 | |
112 | /* |
113 | * Return an index to the current RS block when called inside |
114 | * fec_for_each_buffer_rs_block. |
115 | */ |
116 | static inline unsigned int fec_buffer_rs_index(unsigned int i, unsigned int j) |
117 | { |
118 | return (i << DM_VERITY_FEC_BUF_RS_BITS) + j; |
119 | } |
120 | |
121 | /* |
122 | * Decode all RS blocks from buffers and copy corrected bytes into fio->output |
123 | * starting from block_offset. |
124 | */ |
125 | static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_io *io, |
126 | struct dm_verity_fec_io *fio, u64 rsb, int byte_index, |
127 | unsigned int block_offset, int neras) |
128 | { |
129 | int r, corrected = 0, res; |
130 | struct dm_buffer *buf; |
131 | unsigned int n, i, offset; |
132 | u8 *par, *block; |
133 | struct bio *bio = dm_bio_from_per_bio_data(data: io, data_size: v->ti->per_io_data_size); |
134 | |
135 | par = fec_read_parity(v, rsb, index: block_offset, offset: &offset, buf: &buf, bio_prio(bio)); |
136 | if (IS_ERR(ptr: par)) |
137 | return PTR_ERR(ptr: par); |
138 | |
139 | /* |
140 | * Decode the RS blocks we have in bufs. Each RS block results in |
141 | * one corrected target byte and consumes fec->roots parity bytes. |
142 | */ |
143 | fec_for_each_buffer_rs_block(fio, n, i) { |
144 | block = fec_buffer_rs_block(v, fio, i: n, j: i); |
145 | res = fec_decode_rs8(v, fio, data: block, fec: &par[offset], neras); |
146 | if (res < 0) { |
147 | r = res; |
148 | goto error; |
149 | } |
150 | |
151 | corrected += res; |
152 | fio->output[block_offset] = block[byte_index]; |
153 | |
154 | block_offset++; |
155 | if (block_offset >= 1 << v->data_dev_block_bits) |
156 | goto done; |
157 | |
158 | /* read the next block when we run out of parity bytes */ |
159 | offset += v->fec->roots; |
160 | if (offset >= v->fec->io_size) { |
161 | dm_bufio_release(b: buf); |
162 | |
163 | par = fec_read_parity(v, rsb, index: block_offset, offset: &offset, buf: &buf, bio_prio(bio)); |
164 | if (IS_ERR(ptr: par)) |
165 | return PTR_ERR(ptr: par); |
166 | } |
167 | } |
168 | done: |
169 | r = corrected; |
170 | error: |
171 | dm_bufio_release(b: buf); |
172 | |
173 | if (r < 0 && neras) |
174 | DMERR_LIMIT("%s: FEC %llu: failed to correct: %d" , |
175 | v->data_dev->name, (unsigned long long)rsb, r); |
176 | else if (r > 0) |
177 | DMWARN_LIMIT("%s: FEC %llu: corrected %d errors" , |
178 | v->data_dev->name, (unsigned long long)rsb, r); |
179 | |
180 | return r; |
181 | } |
182 | |
183 | /* |
184 | * Locate data block erasures using verity hashes. |
185 | */ |
186 | static int fec_is_erasure(struct dm_verity *v, struct dm_verity_io *io, |
187 | u8 *want_digest, u8 *data) |
188 | { |
189 | if (unlikely(verity_hash(v, verity_io_hash_req(v, io), |
190 | data, 1 << v->data_dev_block_bits, |
191 | verity_io_real_digest(v, io), true))) |
192 | return 0; |
193 | |
194 | return memcmp(p: verity_io_real_digest(v, io), q: want_digest, |
195 | size: v->digest_size) != 0; |
196 | } |
197 | |
198 | /* |
199 | * Read data blocks that are part of the RS block and deinterleave as much as |
200 | * fits into buffers. Check for erasure locations if @neras is non-NULL. |
201 | */ |
202 | static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io, |
203 | u64 rsb, u64 target, unsigned int block_offset, |
204 | int *neras) |
205 | { |
206 | bool is_zero; |
207 | int i, j, target_index = -1; |
208 | struct dm_buffer *buf; |
209 | struct dm_bufio_client *bufio; |
210 | struct dm_verity_fec_io *fio = fec_io(io); |
211 | u64 block, ileaved; |
212 | u8 *bbuf, *rs_block; |
213 | u8 want_digest[HASH_MAX_DIGESTSIZE]; |
214 | unsigned int n, k; |
215 | struct bio *bio = dm_bio_from_per_bio_data(data: io, data_size: v->ti->per_io_data_size); |
216 | |
217 | if (neras) |
218 | *neras = 0; |
219 | |
220 | if (WARN_ON(v->digest_size > sizeof(want_digest))) |
221 | return -EINVAL; |
222 | |
223 | /* |
224 | * read each of the rsn data blocks that are part of the RS block, and |
225 | * interleave contents to available bufs |
226 | */ |
227 | for (i = 0; i < v->fec->rsn; i++) { |
228 | ileaved = fec_interleave(v, offset: rsb * v->fec->rsn + i); |
229 | |
230 | /* |
231 | * target is the data block we want to correct, target_index is |
232 | * the index of this block within the rsn RS blocks |
233 | */ |
234 | if (ileaved == target) |
235 | target_index = i; |
236 | |
237 | block = ileaved >> v->data_dev_block_bits; |
238 | bufio = v->fec->data_bufio; |
239 | |
240 | if (block >= v->data_blocks) { |
241 | block -= v->data_blocks; |
242 | |
243 | /* |
244 | * blocks outside the area were assumed to contain |
245 | * zeros when encoding data was generated |
246 | */ |
247 | if (unlikely(block >= v->fec->hash_blocks)) |
248 | continue; |
249 | |
250 | block += v->hash_start; |
251 | bufio = v->bufio; |
252 | } |
253 | |
254 | bbuf = dm_bufio_read_with_ioprio(c: bufio, block, bp: &buf, bio_prio(bio)); |
255 | if (IS_ERR(ptr: bbuf)) { |
256 | DMWARN_LIMIT("%s: FEC %llu: read failed (%llu): %ld" , |
257 | v->data_dev->name, |
258 | (unsigned long long)rsb, |
259 | (unsigned long long)block, PTR_ERR(bbuf)); |
260 | |
261 | /* assume the block is corrupted */ |
262 | if (neras && *neras <= v->fec->roots) |
263 | fio->erasures[(*neras)++] = i; |
264 | |
265 | continue; |
266 | } |
267 | |
268 | /* locate erasures if the block is on the data device */ |
269 | if (bufio == v->fec->data_bufio && |
270 | verity_hash_for_block(v, io, block, digest: want_digest, |
271 | is_zero: &is_zero) == 0) { |
272 | /* skip known zero blocks entirely */ |
273 | if (is_zero) |
274 | goto done; |
275 | |
276 | /* |
277 | * skip if we have already found the theoretical |
278 | * maximum number (i.e. fec->roots) of erasures |
279 | */ |
280 | if (neras && *neras <= v->fec->roots && |
281 | fec_is_erasure(v, io, want_digest, data: bbuf)) |
282 | fio->erasures[(*neras)++] = i; |
283 | } |
284 | |
285 | /* |
286 | * deinterleave and copy the bytes that fit into bufs, |
287 | * starting from block_offset |
288 | */ |
289 | fec_for_each_buffer_rs_block(fio, n, j) { |
290 | k = fec_buffer_rs_index(i: n, j) + block_offset; |
291 | |
292 | if (k >= 1 << v->data_dev_block_bits) |
293 | goto done; |
294 | |
295 | rs_block = fec_buffer_rs_block(v, fio, i: n, j); |
296 | rs_block[i] = bbuf[k]; |
297 | } |
298 | done: |
299 | dm_bufio_release(b: buf); |
300 | } |
301 | |
302 | return target_index; |
303 | } |
304 | |
305 | /* |
306 | * Allocate RS control structure and FEC buffers from preallocated mempools, |
307 | * and attempt to allocate as many extra buffers as available. |
308 | */ |
309 | static int fec_alloc_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio) |
310 | { |
311 | unsigned int n; |
312 | |
313 | if (!fio->rs) |
314 | fio->rs = mempool_alloc(pool: &v->fec->rs_pool, GFP_NOIO); |
315 | |
316 | fec_for_each_prealloc_buffer(n) { |
317 | if (fio->bufs[n]) |
318 | continue; |
319 | |
320 | fio->bufs[n] = mempool_alloc(pool: &v->fec->prealloc_pool, GFP_NOWAIT); |
321 | if (unlikely(!fio->bufs[n])) { |
322 | DMERR("failed to allocate FEC buffer" ); |
323 | return -ENOMEM; |
324 | } |
325 | } |
326 | |
327 | /* try to allocate the maximum number of buffers */ |
328 | fec_for_each_extra_buffer(fio, n) { |
329 | if (fio->bufs[n]) |
330 | continue; |
331 | |
332 | fio->bufs[n] = mempool_alloc(pool: &v->fec->extra_pool, GFP_NOWAIT); |
333 | /* we can manage with even one buffer if necessary */ |
334 | if (unlikely(!fio->bufs[n])) |
335 | break; |
336 | } |
337 | fio->nbufs = n; |
338 | |
339 | if (!fio->output) |
340 | fio->output = mempool_alloc(pool: &v->fec->output_pool, GFP_NOIO); |
341 | |
342 | return 0; |
343 | } |
344 | |
345 | /* |
346 | * Initialize buffers and clear erasures. fec_read_bufs() assumes buffers are |
347 | * zeroed before deinterleaving. |
348 | */ |
349 | static void fec_init_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio) |
350 | { |
351 | unsigned int n; |
352 | |
353 | fec_for_each_buffer(fio, n) |
354 | memset(fio->bufs[n], 0, v->fec->rsn << DM_VERITY_FEC_BUF_RS_BITS); |
355 | |
356 | memset(fio->erasures, 0, sizeof(fio->erasures)); |
357 | } |
358 | |
359 | /* |
360 | * Decode all RS blocks in a single data block and return the target block |
361 | * (indicated by @offset) in fio->output. If @use_erasures is non-zero, uses |
362 | * hashes to locate erasures. |
363 | */ |
364 | static int fec_decode_rsb(struct dm_verity *v, struct dm_verity_io *io, |
365 | struct dm_verity_fec_io *fio, u64 rsb, u64 offset, |
366 | bool use_erasures) |
367 | { |
368 | int r, neras = 0; |
369 | unsigned int pos; |
370 | |
371 | r = fec_alloc_bufs(v, fio); |
372 | if (unlikely(r < 0)) |
373 | return r; |
374 | |
375 | for (pos = 0; pos < 1 << v->data_dev_block_bits; ) { |
376 | fec_init_bufs(v, fio); |
377 | |
378 | r = fec_read_bufs(v, io, rsb, target: offset, block_offset: pos, |
379 | neras: use_erasures ? &neras : NULL); |
380 | if (unlikely(r < 0)) |
381 | return r; |
382 | |
383 | r = fec_decode_bufs(v, io, fio, rsb, byte_index: r, block_offset: pos, neras); |
384 | if (r < 0) |
385 | return r; |
386 | |
387 | pos += fio->nbufs << DM_VERITY_FEC_BUF_RS_BITS; |
388 | } |
389 | |
390 | /* Always re-validate the corrected block against the expected hash */ |
391 | r = verity_hash(v, req: verity_io_hash_req(v, io), data: fio->output, |
392 | len: 1 << v->data_dev_block_bits, |
393 | digest: verity_io_real_digest(v, io), may_sleep: true); |
394 | if (unlikely(r < 0)) |
395 | return r; |
396 | |
397 | if (memcmp(p: verity_io_real_digest(v, io), q: verity_io_want_digest(v, io), |
398 | size: v->digest_size)) { |
399 | DMERR_LIMIT("%s: FEC %llu: failed to correct (%d erasures)" , |
400 | v->data_dev->name, (unsigned long long)rsb, neras); |
401 | return -EILSEQ; |
402 | } |
403 | |
404 | return 0; |
405 | } |
406 | |
407 | static int fec_bv_copy(struct dm_verity *v, struct dm_verity_io *io, u8 *data, |
408 | size_t len) |
409 | { |
410 | struct dm_verity_fec_io *fio = fec_io(io); |
411 | |
412 | memcpy(data, &fio->output[fio->output_pos], len); |
413 | fio->output_pos += len; |
414 | |
415 | return 0; |
416 | } |
417 | |
418 | /* |
419 | * Correct errors in a block. Copies corrected block to dest if non-NULL, |
420 | * otherwise to a bio_vec starting from iter. |
421 | */ |
422 | int verity_fec_decode(struct dm_verity *v, struct dm_verity_io *io, |
423 | enum verity_block_type type, sector_t block, u8 *dest, |
424 | struct bvec_iter *iter) |
425 | { |
426 | int r; |
427 | struct dm_verity_fec_io *fio = fec_io(io); |
428 | u64 offset, res, rsb; |
429 | |
430 | if (!verity_fec_is_enabled(v)) |
431 | return -EOPNOTSUPP; |
432 | |
433 | if (fio->level >= DM_VERITY_FEC_MAX_RECURSION) { |
434 | DMWARN_LIMIT("%s: FEC: recursion too deep" , v->data_dev->name); |
435 | return -EIO; |
436 | } |
437 | |
438 | fio->level++; |
439 | |
440 | if (type == DM_VERITY_BLOCK_TYPE_METADATA) |
441 | block = block - v->hash_start + v->data_blocks; |
442 | |
443 | /* |
444 | * For RS(M, N), the continuous FEC data is divided into blocks of N |
445 | * bytes. Since block size may not be divisible by N, the last block |
446 | * is zero padded when decoding. |
447 | * |
448 | * Each byte of the block is covered by a different RS(M, N) code, |
449 | * and each code is interleaved over N blocks to make it less likely |
450 | * that bursty corruption will leave us in unrecoverable state. |
451 | */ |
452 | |
453 | offset = block << v->data_dev_block_bits; |
454 | res = div64_u64(dividend: offset, divisor: v->fec->rounds << v->data_dev_block_bits); |
455 | |
456 | /* |
457 | * The base RS block we can feed to the interleaver to find out all |
458 | * blocks required for decoding. |
459 | */ |
460 | rsb = offset - res * (v->fec->rounds << v->data_dev_block_bits); |
461 | |
462 | /* |
463 | * Locating erasures is slow, so attempt to recover the block without |
464 | * them first. Do a second attempt with erasures if the corruption is |
465 | * bad enough. |
466 | */ |
467 | r = fec_decode_rsb(v, io, fio, rsb, offset, use_erasures: false); |
468 | if (r < 0) { |
469 | r = fec_decode_rsb(v, io, fio, rsb, offset, use_erasures: true); |
470 | if (r < 0) |
471 | goto done; |
472 | } |
473 | |
474 | if (dest) |
475 | memcpy(dest, fio->output, 1 << v->data_dev_block_bits); |
476 | else if (iter) { |
477 | fio->output_pos = 0; |
478 | r = verity_for_bv_block(v, io, iter, process: fec_bv_copy); |
479 | } |
480 | |
481 | done: |
482 | fio->level--; |
483 | return r; |
484 | } |
485 | |
486 | /* |
487 | * Clean up per-bio data. |
488 | */ |
489 | void verity_fec_finish_io(struct dm_verity_io *io) |
490 | { |
491 | unsigned int n; |
492 | struct dm_verity_fec *f = io->v->fec; |
493 | struct dm_verity_fec_io *fio = fec_io(io); |
494 | |
495 | if (!verity_fec_is_enabled(v: io->v)) |
496 | return; |
497 | |
498 | mempool_free(element: fio->rs, pool: &f->rs_pool); |
499 | |
500 | fec_for_each_prealloc_buffer(n) |
501 | mempool_free(element: fio->bufs[n], pool: &f->prealloc_pool); |
502 | |
503 | fec_for_each_extra_buffer(fio, n) |
504 | mempool_free(element: fio->bufs[n], pool: &f->extra_pool); |
505 | |
506 | mempool_free(element: fio->output, pool: &f->output_pool); |
507 | } |
508 | |
509 | /* |
510 | * Initialize per-bio data. |
511 | */ |
512 | void verity_fec_init_io(struct dm_verity_io *io) |
513 | { |
514 | struct dm_verity_fec_io *fio = fec_io(io); |
515 | |
516 | if (!verity_fec_is_enabled(v: io->v)) |
517 | return; |
518 | |
519 | fio->rs = NULL; |
520 | memset(fio->bufs, 0, sizeof(fio->bufs)); |
521 | fio->nbufs = 0; |
522 | fio->output = NULL; |
523 | fio->level = 0; |
524 | } |
525 | |
526 | /* |
527 | * Append feature arguments and values to the status table. |
528 | */ |
529 | unsigned int verity_fec_status_table(struct dm_verity *v, unsigned int sz, |
530 | char *result, unsigned int maxlen) |
531 | { |
532 | if (!verity_fec_is_enabled(v)) |
533 | return sz; |
534 | |
535 | DMEMIT(" " DM_VERITY_OPT_FEC_DEV " %s " |
536 | DM_VERITY_OPT_FEC_BLOCKS " %llu " |
537 | DM_VERITY_OPT_FEC_START " %llu " |
538 | DM_VERITY_OPT_FEC_ROOTS " %d" , |
539 | v->fec->dev->name, |
540 | (unsigned long long)v->fec->blocks, |
541 | (unsigned long long)v->fec->start, |
542 | v->fec->roots); |
543 | |
544 | return sz; |
545 | } |
546 | |
547 | void verity_fec_dtr(struct dm_verity *v) |
548 | { |
549 | struct dm_verity_fec *f = v->fec; |
550 | |
551 | if (!verity_fec_is_enabled(v)) |
552 | goto out; |
553 | |
554 | mempool_exit(pool: &f->rs_pool); |
555 | mempool_exit(pool: &f->prealloc_pool); |
556 | mempool_exit(pool: &f->extra_pool); |
557 | mempool_exit(pool: &f->output_pool); |
558 | kmem_cache_destroy(s: f->cache); |
559 | |
560 | if (f->data_bufio) |
561 | dm_bufio_client_destroy(c: f->data_bufio); |
562 | if (f->bufio) |
563 | dm_bufio_client_destroy(c: f->bufio); |
564 | |
565 | if (f->dev) |
566 | dm_put_device(ti: v->ti, d: f->dev); |
567 | out: |
568 | kfree(objp: f); |
569 | v->fec = NULL; |
570 | } |
571 | |
572 | static void *fec_rs_alloc(gfp_t gfp_mask, void *pool_data) |
573 | { |
574 | struct dm_verity *v = pool_data; |
575 | |
576 | return init_rs_gfp(symsize: 8, gfpoly: 0x11d, fcr: 0, prim: 1, nroots: v->fec->roots, gfp: gfp_mask); |
577 | } |
578 | |
579 | static void fec_rs_free(void *element, void *pool_data) |
580 | { |
581 | struct rs_control *rs = element; |
582 | |
583 | if (rs) |
584 | free_rs(rs); |
585 | } |
586 | |
587 | bool verity_is_fec_opt_arg(const char *arg_name) |
588 | { |
589 | return (!strcasecmp(s1: arg_name, DM_VERITY_OPT_FEC_DEV) || |
590 | !strcasecmp(s1: arg_name, DM_VERITY_OPT_FEC_BLOCKS) || |
591 | !strcasecmp(s1: arg_name, DM_VERITY_OPT_FEC_START) || |
592 | !strcasecmp(s1: arg_name, DM_VERITY_OPT_FEC_ROOTS)); |
593 | } |
594 | |
595 | int verity_fec_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v, |
596 | unsigned int *argc, const char *arg_name) |
597 | { |
598 | int r; |
599 | struct dm_target *ti = v->ti; |
600 | const char *arg_value; |
601 | unsigned long long num_ll; |
602 | unsigned char num_c; |
603 | char dummy; |
604 | |
605 | if (!*argc) { |
606 | ti->error = "FEC feature arguments require a value" ; |
607 | return -EINVAL; |
608 | } |
609 | |
610 | arg_value = dm_shift_arg(as); |
611 | (*argc)--; |
612 | |
613 | if (!strcasecmp(s1: arg_name, DM_VERITY_OPT_FEC_DEV)) { |
614 | r = dm_get_device(ti, path: arg_value, BLK_OPEN_READ, result: &v->fec->dev); |
615 | if (r) { |
616 | ti->error = "FEC device lookup failed" ; |
617 | return r; |
618 | } |
619 | |
620 | } else if (!strcasecmp(s1: arg_name, DM_VERITY_OPT_FEC_BLOCKS)) { |
621 | if (sscanf(arg_value, "%llu%c" , &num_ll, &dummy) != 1 || |
622 | ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) |
623 | >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) { |
624 | ti->error = "Invalid " DM_VERITY_OPT_FEC_BLOCKS; |
625 | return -EINVAL; |
626 | } |
627 | v->fec->blocks = num_ll; |
628 | |
629 | } else if (!strcasecmp(s1: arg_name, DM_VERITY_OPT_FEC_START)) { |
630 | if (sscanf(arg_value, "%llu%c" , &num_ll, &dummy) != 1 || |
631 | ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) >> |
632 | (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) { |
633 | ti->error = "Invalid " DM_VERITY_OPT_FEC_START; |
634 | return -EINVAL; |
635 | } |
636 | v->fec->start = num_ll; |
637 | |
638 | } else if (!strcasecmp(s1: arg_name, DM_VERITY_OPT_FEC_ROOTS)) { |
639 | if (sscanf(arg_value, "%hhu%c" , &num_c, &dummy) != 1 || !num_c || |
640 | num_c < (DM_VERITY_FEC_RSM - DM_VERITY_FEC_MAX_RSN) || |
641 | num_c > (DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN)) { |
642 | ti->error = "Invalid " DM_VERITY_OPT_FEC_ROOTS; |
643 | return -EINVAL; |
644 | } |
645 | v->fec->roots = num_c; |
646 | |
647 | } else { |
648 | ti->error = "Unrecognized verity FEC feature request" ; |
649 | return -EINVAL; |
650 | } |
651 | |
652 | return 0; |
653 | } |
654 | |
655 | /* |
656 | * Allocate dm_verity_fec for v->fec. Must be called before verity_fec_ctr. |
657 | */ |
658 | int verity_fec_ctr_alloc(struct dm_verity *v) |
659 | { |
660 | struct dm_verity_fec *f; |
661 | |
662 | f = kzalloc(size: sizeof(struct dm_verity_fec), GFP_KERNEL); |
663 | if (!f) { |
664 | v->ti->error = "Cannot allocate FEC structure" ; |
665 | return -ENOMEM; |
666 | } |
667 | v->fec = f; |
668 | |
669 | return 0; |
670 | } |
671 | |
672 | /* |
673 | * Validate arguments and preallocate memory. Must be called after arguments |
674 | * have been parsed using verity_fec_parse_opt_args. |
675 | */ |
676 | int verity_fec_ctr(struct dm_verity *v) |
677 | { |
678 | struct dm_verity_fec *f = v->fec; |
679 | struct dm_target *ti = v->ti; |
680 | u64 hash_blocks, fec_blocks; |
681 | int ret; |
682 | |
683 | if (!verity_fec_is_enabled(v)) { |
684 | verity_fec_dtr(v); |
685 | return 0; |
686 | } |
687 | |
688 | /* |
689 | * FEC is computed over data blocks, possible metadata, and |
690 | * hash blocks. In other words, FEC covers total of fec_blocks |
691 | * blocks consisting of the following: |
692 | * |
693 | * data blocks | hash blocks | metadata (optional) |
694 | * |
695 | * We allow metadata after hash blocks to support a use case |
696 | * where all data is stored on the same device and FEC covers |
697 | * the entire area. |
698 | * |
699 | * If metadata is included, we require it to be available on the |
700 | * hash device after the hash blocks. |
701 | */ |
702 | |
703 | hash_blocks = v->hash_blocks - v->hash_start; |
704 | |
705 | /* |
706 | * Require matching block sizes for data and hash devices for |
707 | * simplicity. |
708 | */ |
709 | if (v->data_dev_block_bits != v->hash_dev_block_bits) { |
710 | ti->error = "Block sizes must match to use FEC" ; |
711 | return -EINVAL; |
712 | } |
713 | |
714 | if (!f->roots) { |
715 | ti->error = "Missing " DM_VERITY_OPT_FEC_ROOTS; |
716 | return -EINVAL; |
717 | } |
718 | f->rsn = DM_VERITY_FEC_RSM - f->roots; |
719 | |
720 | if (!f->blocks) { |
721 | ti->error = "Missing " DM_VERITY_OPT_FEC_BLOCKS; |
722 | return -EINVAL; |
723 | } |
724 | |
725 | f->rounds = f->blocks; |
726 | if (sector_div(f->rounds, f->rsn)) |
727 | f->rounds++; |
728 | |
729 | /* |
730 | * Due to optional metadata, f->blocks can be larger than |
731 | * data_blocks and hash_blocks combined. |
732 | */ |
733 | if (f->blocks < v->data_blocks + hash_blocks || !f->rounds) { |
734 | ti->error = "Invalid " DM_VERITY_OPT_FEC_BLOCKS; |
735 | return -EINVAL; |
736 | } |
737 | |
738 | /* |
739 | * Metadata is accessed through the hash device, so we require |
740 | * it to be large enough. |
741 | */ |
742 | f->hash_blocks = f->blocks - v->data_blocks; |
743 | if (dm_bufio_get_device_size(c: v->bufio) < f->hash_blocks) { |
744 | ti->error = "Hash device is too small for " |
745 | DM_VERITY_OPT_FEC_BLOCKS; |
746 | return -E2BIG; |
747 | } |
748 | |
749 | if ((f->roots << SECTOR_SHIFT) & ((1 << v->data_dev_block_bits) - 1)) |
750 | f->io_size = 1 << v->data_dev_block_bits; |
751 | else |
752 | f->io_size = v->fec->roots << SECTOR_SHIFT; |
753 | |
754 | f->bufio = dm_bufio_client_create(bdev: f->dev->bdev, |
755 | block_size: f->io_size, |
756 | reserved_buffers: 1, aux_size: 0, NULL, NULL, flags: 0); |
757 | if (IS_ERR(ptr: f->bufio)) { |
758 | ti->error = "Cannot initialize FEC bufio client" ; |
759 | return PTR_ERR(ptr: f->bufio); |
760 | } |
761 | |
762 | dm_bufio_set_sector_offset(c: f->bufio, start: f->start << (v->data_dev_block_bits - SECTOR_SHIFT)); |
763 | |
764 | fec_blocks = div64_u64(dividend: f->rounds * f->roots, divisor: v->fec->roots << SECTOR_SHIFT); |
765 | if (dm_bufio_get_device_size(c: f->bufio) < fec_blocks) { |
766 | ti->error = "FEC device is too small" ; |
767 | return -E2BIG; |
768 | } |
769 | |
770 | f->data_bufio = dm_bufio_client_create(bdev: v->data_dev->bdev, |
771 | block_size: 1 << v->data_dev_block_bits, |
772 | reserved_buffers: 1, aux_size: 0, NULL, NULL, flags: 0); |
773 | if (IS_ERR(ptr: f->data_bufio)) { |
774 | ti->error = "Cannot initialize FEC data bufio client" ; |
775 | return PTR_ERR(ptr: f->data_bufio); |
776 | } |
777 | |
778 | if (dm_bufio_get_device_size(c: f->data_bufio) < v->data_blocks) { |
779 | ti->error = "Data device is too small" ; |
780 | return -E2BIG; |
781 | } |
782 | |
783 | /* Preallocate an rs_control structure for each worker thread */ |
784 | ret = mempool_init(pool: &f->rs_pool, min_nr: num_online_cpus(), alloc_fn: fec_rs_alloc, |
785 | free_fn: fec_rs_free, pool_data: (void *) v); |
786 | if (ret) { |
787 | ti->error = "Cannot allocate RS pool" ; |
788 | return ret; |
789 | } |
790 | |
791 | f->cache = kmem_cache_create(name: "dm_verity_fec_buffers" , |
792 | size: f->rsn << DM_VERITY_FEC_BUF_RS_BITS, |
793 | align: 0, flags: 0, NULL); |
794 | if (!f->cache) { |
795 | ti->error = "Cannot create FEC buffer cache" ; |
796 | return -ENOMEM; |
797 | } |
798 | |
799 | /* Preallocate DM_VERITY_FEC_BUF_PREALLOC buffers for each thread */ |
800 | ret = mempool_init_slab_pool(pool: &f->prealloc_pool, min_nr: num_online_cpus() * |
801 | DM_VERITY_FEC_BUF_PREALLOC, |
802 | kc: f->cache); |
803 | if (ret) { |
804 | ti->error = "Cannot allocate FEC buffer prealloc pool" ; |
805 | return ret; |
806 | } |
807 | |
808 | ret = mempool_init_slab_pool(pool: &f->extra_pool, min_nr: 0, kc: f->cache); |
809 | if (ret) { |
810 | ti->error = "Cannot allocate FEC buffer extra pool" ; |
811 | return ret; |
812 | } |
813 | |
814 | /* Preallocate an output buffer for each thread */ |
815 | ret = mempool_init_kmalloc_pool(pool: &f->output_pool, min_nr: num_online_cpus(), |
816 | size: 1 << v->data_dev_block_bits); |
817 | if (ret) { |
818 | ti->error = "Cannot allocate FEC output pool" ; |
819 | return ret; |
820 | } |
821 | |
822 | /* Reserve space for our per-bio data */ |
823 | ti->per_io_data_size += sizeof(struct dm_verity_fec_io); |
824 | |
825 | return 0; |
826 | } |
827 | |