1/*
2 * Copyright (C) 2012 Red Hat, Inc.
3 *
4 * Author: Mikulas Patocka <mpatocka@redhat.com>
5 *
6 * Based on Chromium dm-verity driver (C) 2011 The Chromium OS Authors
7 *
8 * This file is released under the GPLv2.
9 *
10 * In the file "/sys/module/dm_verity/parameters/prefetch_cluster" you can set
11 * default prefetch value. Data are read in "prefetch_cluster" chunks from the
12 * hash device. Setting this greatly improves performance when data and hash
13 * are on the same disk on different partitions on devices with poor random
14 * access behavior.
15 */
16
17#include "dm-verity.h"
18#include "dm-verity-fec.h"
19
20#include <linux/module.h>
21#include <linux/reboot.h>
22
23#define DM_MSG_PREFIX "verity"
24
25#define DM_VERITY_ENV_LENGTH 42
26#define DM_VERITY_ENV_VAR_NAME "DM_VERITY_ERR_BLOCK_NR"
27
28#define DM_VERITY_DEFAULT_PREFETCH_SIZE 262144
29
30#define DM_VERITY_MAX_CORRUPTED_ERRS 100
31
32#define DM_VERITY_OPT_LOGGING "ignore_corruption"
33#define DM_VERITY_OPT_RESTART "restart_on_corruption"
34#define DM_VERITY_OPT_IGN_ZEROES "ignore_zero_blocks"
35#define DM_VERITY_OPT_AT_MOST_ONCE "check_at_most_once"
36
37#define DM_VERITY_OPTS_MAX (2 + DM_VERITY_OPTS_FEC)
38
39static unsigned dm_verity_prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE;
40
41module_param_named(prefetch_cluster, dm_verity_prefetch_cluster, uint, S_IRUGO | S_IWUSR);
42
43struct dm_verity_prefetch_work {
44 struct work_struct work;
45 struct dm_verity *v;
46 sector_t block;
47 unsigned n_blocks;
48};
49
50/*
51 * Auxiliary structure appended to each dm-bufio buffer. If the value
52 * hash_verified is nonzero, hash of the block has been verified.
53 *
54 * The variable hash_verified is set to 0 when allocating the buffer, then
55 * it can be changed to 1 and it is never reset to 0 again.
56 *
57 * There is no lock around this value, a race condition can at worst cause
58 * that multiple processes verify the hash of the same buffer simultaneously
59 * and write 1 to hash_verified simultaneously.
60 * This condition is harmless, so we don't need locking.
61 */
62struct buffer_aux {
63 int hash_verified;
64};
65
66/*
67 * Initialize struct buffer_aux for a freshly created buffer.
68 */
69static void dm_bufio_alloc_callback(struct dm_buffer *buf)
70{
71 struct buffer_aux *aux = dm_bufio_get_aux_data(buf);
72
73 aux->hash_verified = 0;
74}
75
76/*
77 * Translate input sector number to the sector number on the target device.
78 */
79static sector_t verity_map_sector(struct dm_verity *v, sector_t bi_sector)
80{
81 return v->data_start + dm_target_offset(v->ti, bi_sector);
82}
83
84/*
85 * Return hash position of a specified block at a specified tree level
86 * (0 is the lowest level).
87 * The lowest "hash_per_block_bits"-bits of the result denote hash position
88 * inside a hash block. The remaining bits denote location of the hash block.
89 */
90static sector_t verity_position_at_level(struct dm_verity *v, sector_t block,
91 int level)
92{
93 return block >> (level * v->hash_per_block_bits);
94}
95
96static int verity_hash_update(struct dm_verity *v, struct ahash_request *req,
97 const u8 *data, size_t len,
98 struct crypto_wait *wait)
99{
100 struct scatterlist sg;
101
102 if (likely(!is_vmalloc_addr(data))) {
103 sg_init_one(&sg, data, len);
104 ahash_request_set_crypt(req, &sg, NULL, len);
105 return crypto_wait_req(crypto_ahash_update(req), wait);
106 } else {
107 do {
108 int r;
109 size_t this_step = min_t(size_t, len, PAGE_SIZE - offset_in_page(data));
110 flush_kernel_vmap_range((void *)data, this_step);
111 sg_init_table(&sg, 1);
112 sg_set_page(&sg, vmalloc_to_page(data), this_step, offset_in_page(data));
113 ahash_request_set_crypt(req, &sg, NULL, this_step);
114 r = crypto_wait_req(crypto_ahash_update(req), wait);
115 if (unlikely(r))
116 return r;
117 data += this_step;
118 len -= this_step;
119 } while (len);
120 return 0;
121 }
122}
123
124/*
125 * Wrapper for crypto_ahash_init, which handles verity salting.
126 */
127static int verity_hash_init(struct dm_verity *v, struct ahash_request *req,
128 struct crypto_wait *wait)
129{
130 int r;
131
132 ahash_request_set_tfm(req, v->tfm);
133 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP |
134 CRYPTO_TFM_REQ_MAY_BACKLOG,
135 crypto_req_done, (void *)wait);
136 crypto_init_wait(wait);
137
138 r = crypto_wait_req(crypto_ahash_init(req), wait);
139
140 if (unlikely(r < 0)) {
141 DMERR("crypto_ahash_init failed: %d", r);
142 return r;
143 }
144
145 if (likely(v->salt_size && (v->version >= 1)))
146 r = verity_hash_update(v, req, v->salt, v->salt_size, wait);
147
148 return r;
149}
150
151static int verity_hash_final(struct dm_verity *v, struct ahash_request *req,
152 u8 *digest, struct crypto_wait *wait)
153{
154 int r;
155
156 if (unlikely(v->salt_size && (!v->version))) {
157 r = verity_hash_update(v, req, v->salt, v->salt_size, wait);
158
159 if (r < 0) {
160 DMERR("verity_hash_final failed updating salt: %d", r);
161 goto out;
162 }
163 }
164
165 ahash_request_set_crypt(req, NULL, digest, 0);
166 r = crypto_wait_req(crypto_ahash_final(req), wait);
167out:
168 return r;
169}
170
171int verity_hash(struct dm_verity *v, struct ahash_request *req,
172 const u8 *data, size_t len, u8 *digest)
173{
174 int r;
175 struct crypto_wait wait;
176
177 r = verity_hash_init(v, req, &wait);
178 if (unlikely(r < 0))
179 goto out;
180
181 r = verity_hash_update(v, req, data, len, &wait);
182 if (unlikely(r < 0))
183 goto out;
184
185 r = verity_hash_final(v, req, digest, &wait);
186
187out:
188 return r;
189}
190
191static void verity_hash_at_level(struct dm_verity *v, sector_t block, int level,
192 sector_t *hash_block, unsigned *offset)
193{
194 sector_t position = verity_position_at_level(v, block, level);
195 unsigned idx;
196
197 *hash_block = v->hash_level_block[level] + (position >> v->hash_per_block_bits);
198
199 if (!offset)
200 return;
201
202 idx = position & ((1 << v->hash_per_block_bits) - 1);
203 if (!v->version)
204 *offset = idx * v->digest_size;
205 else
206 *offset = idx << (v->hash_dev_block_bits - v->hash_per_block_bits);
207}
208
209/*
210 * Handle verification errors.
211 */
212static int verity_handle_err(struct dm_verity *v, enum verity_block_type type,
213 unsigned long long block)
214{
215 char verity_env[DM_VERITY_ENV_LENGTH];
216 char *envp[] = { verity_env, NULL };
217 const char *type_str = "";
218 struct mapped_device *md = dm_table_get_md(v->ti->table);
219
220 /* Corruption should be visible in device status in all modes */
221 v->hash_failed = 1;
222
223 if (v->corrupted_errs >= DM_VERITY_MAX_CORRUPTED_ERRS)
224 goto out;
225
226 v->corrupted_errs++;
227
228 switch (type) {
229 case DM_VERITY_BLOCK_TYPE_DATA:
230 type_str = "data";
231 break;
232 case DM_VERITY_BLOCK_TYPE_METADATA:
233 type_str = "metadata";
234 break;
235 default:
236 BUG();
237 }
238
239 DMERR("%s: %s block %llu is corrupted", v->data_dev->name, type_str,
240 block);
241
242 if (v->corrupted_errs == DM_VERITY_MAX_CORRUPTED_ERRS)
243 DMERR("%s: reached maximum errors", v->data_dev->name);
244
245 snprintf(verity_env, DM_VERITY_ENV_LENGTH, "%s=%d,%llu",
246 DM_VERITY_ENV_VAR_NAME, type, block);
247
248 kobject_uevent_env(&disk_to_dev(dm_disk(md))->kobj, KOBJ_CHANGE, envp);
249
250out:
251 if (v->mode == DM_VERITY_MODE_LOGGING)
252 return 0;
253
254 if (v->mode == DM_VERITY_MODE_RESTART)
255 kernel_restart("dm-verity device corrupted");
256
257 return 1;
258}
259
260/*
261 * Verify hash of a metadata block pertaining to the specified data block
262 * ("block" argument) at a specified level ("level" argument).
263 *
264 * On successful return, verity_io_want_digest(v, io) contains the hash value
265 * for a lower tree level or for the data block (if we're at the lowest level).
266 *
267 * If "skip_unverified" is true, unverified buffer is skipped and 1 is returned.
268 * If "skip_unverified" is false, unverified buffer is hashed and verified
269 * against current value of verity_io_want_digest(v, io).
270 */
271static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io,
272 sector_t block, int level, bool skip_unverified,
273 u8 *want_digest)
274{
275 struct dm_buffer *buf;
276 struct buffer_aux *aux;
277 u8 *data;
278 int r;
279 sector_t hash_block;
280 unsigned offset;
281
282 verity_hash_at_level(v, block, level, &hash_block, &offset);
283
284 data = dm_bufio_read(v->bufio, hash_block, &buf);
285 if (IS_ERR(data))
286 return PTR_ERR(data);
287
288 aux = dm_bufio_get_aux_data(buf);
289
290 if (!aux->hash_verified) {
291 if (skip_unverified) {
292 r = 1;
293 goto release_ret_r;
294 }
295
296 r = verity_hash(v, verity_io_hash_req(v, io),
297 data, 1 << v->hash_dev_block_bits,
298 verity_io_real_digest(v, io));
299 if (unlikely(r < 0))
300 goto release_ret_r;
301
302 if (likely(memcmp(verity_io_real_digest(v, io), want_digest,
303 v->digest_size) == 0))
304 aux->hash_verified = 1;
305 else if (verity_fec_decode(v, io,
306 DM_VERITY_BLOCK_TYPE_METADATA,
307 hash_block, data, NULL) == 0)
308 aux->hash_verified = 1;
309 else if (verity_handle_err(v,
310 DM_VERITY_BLOCK_TYPE_METADATA,
311 hash_block)) {
312 r = -EIO;
313 goto release_ret_r;
314 }
315 }
316
317 data += offset;
318 memcpy(want_digest, data, v->digest_size);
319 r = 0;
320
321release_ret_r:
322 dm_bufio_release(buf);
323 return r;
324}
325
326/*
327 * Find a hash for a given block, write it to digest and verify the integrity
328 * of the hash tree if necessary.
329 */
330int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io,
331 sector_t block, u8 *digest, bool *is_zero)
332{
333 int r = 0, i;
334
335 if (likely(v->levels)) {
336 /*
337 * First, we try to get the requested hash for
338 * the current block. If the hash block itself is
339 * verified, zero is returned. If it isn't, this
340 * function returns 1 and we fall back to whole
341 * chain verification.
342 */
343 r = verity_verify_level(v, io, block, 0, true, digest);
344 if (likely(r <= 0))
345 goto out;
346 }
347
348 memcpy(digest, v->root_digest, v->digest_size);
349
350 for (i = v->levels - 1; i >= 0; i--) {
351 r = verity_verify_level(v, io, block, i, false, digest);
352 if (unlikely(r))
353 goto out;
354 }
355out:
356 if (!r && v->zero_digest)
357 *is_zero = !memcmp(v->zero_digest, digest, v->digest_size);
358 else
359 *is_zero = false;
360
361 return r;
362}
363
364/*
365 * Calculates the digest for the given bio
366 */
367static int verity_for_io_block(struct dm_verity *v, struct dm_verity_io *io,
368 struct bvec_iter *iter, struct crypto_wait *wait)
369{
370 unsigned int todo = 1 << v->data_dev_block_bits;
371 struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size);
372 struct scatterlist sg;
373 struct ahash_request *req = verity_io_hash_req(v, io);
374
375 do {
376 int r;
377 unsigned int len;
378 struct bio_vec bv = bio_iter_iovec(bio, *iter);
379
380 sg_init_table(&sg, 1);
381
382 len = bv.bv_len;
383
384 if (likely(len >= todo))
385 len = todo;
386 /*
387 * Operating on a single page at a time looks suboptimal
388 * until you consider the typical block size is 4,096B.
389 * Going through this loops twice should be very rare.
390 */
391 sg_set_page(&sg, bv.bv_page, len, bv.bv_offset);
392 ahash_request_set_crypt(req, &sg, NULL, len);
393 r = crypto_wait_req(crypto_ahash_update(req), wait);
394
395 if (unlikely(r < 0)) {
396 DMERR("verity_for_io_block crypto op failed: %d", r);
397 return r;
398 }
399
400 bio_advance_iter(bio, iter, len);
401 todo -= len;
402 } while (todo);
403
404 return 0;
405}
406
407/*
408 * Calls function process for 1 << v->data_dev_block_bits bytes in the bio_vec
409 * starting from iter.
410 */
411int verity_for_bv_block(struct dm_verity *v, struct dm_verity_io *io,
412 struct bvec_iter *iter,
413 int (*process)(struct dm_verity *v,
414 struct dm_verity_io *io, u8 *data,
415 size_t len))
416{
417 unsigned todo = 1 << v->data_dev_block_bits;
418 struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size);
419
420 do {
421 int r;
422 u8 *page;
423 unsigned len;
424 struct bio_vec bv = bio_iter_iovec(bio, *iter);
425
426 page = kmap_atomic(bv.bv_page);
427 len = bv.bv_len;
428
429 if (likely(len >= todo))
430 len = todo;
431
432 r = process(v, io, page + bv.bv_offset, len);
433 kunmap_atomic(page);
434
435 if (r < 0)
436 return r;
437
438 bio_advance_iter(bio, iter, len);
439 todo -= len;
440 } while (todo);
441
442 return 0;
443}
444
445static int verity_bv_zero(struct dm_verity *v, struct dm_verity_io *io,
446 u8 *data, size_t len)
447{
448 memset(data, 0, len);
449 return 0;
450}
451
452/*
453 * Moves the bio iter one data block forward.
454 */
455static inline void verity_bv_skip_block(struct dm_verity *v,
456 struct dm_verity_io *io,
457 struct bvec_iter *iter)
458{
459 struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size);
460
461 bio_advance_iter(bio, iter, 1 << v->data_dev_block_bits);
462}
463
464/*
465 * Verify one "dm_verity_io" structure.
466 */
467static int verity_verify_io(struct dm_verity_io *io)
468{
469 bool is_zero;
470 struct dm_verity *v = io->v;
471 struct bvec_iter start;
472 unsigned b;
473 struct crypto_wait wait;
474
475 for (b = 0; b < io->n_blocks; b++) {
476 int r;
477 sector_t cur_block = io->block + b;
478 struct ahash_request *req = verity_io_hash_req(v, io);
479
480 if (v->validated_blocks &&
481 likely(test_bit(cur_block, v->validated_blocks))) {
482 verity_bv_skip_block(v, io, &io->iter);
483 continue;
484 }
485
486 r = verity_hash_for_block(v, io, cur_block,
487 verity_io_want_digest(v, io),
488 &is_zero);
489 if (unlikely(r < 0))
490 return r;
491
492 if (is_zero) {
493 /*
494 * If we expect a zero block, don't validate, just
495 * return zeros.
496 */
497 r = verity_for_bv_block(v, io, &io->iter,
498 verity_bv_zero);
499 if (unlikely(r < 0))
500 return r;
501
502 continue;
503 }
504
505 r = verity_hash_init(v, req, &wait);
506 if (unlikely(r < 0))
507 return r;
508
509 start = io->iter;
510 r = verity_for_io_block(v, io, &io->iter, &wait);
511 if (unlikely(r < 0))
512 return r;
513
514 r = verity_hash_final(v, req, verity_io_real_digest(v, io),
515 &wait);
516 if (unlikely(r < 0))
517 return r;
518
519 if (likely(memcmp(verity_io_real_digest(v, io),
520 verity_io_want_digest(v, io), v->digest_size) == 0)) {
521 if (v->validated_blocks)
522 set_bit(cur_block, v->validated_blocks);
523 continue;
524 }
525 else if (verity_fec_decode(v, io, DM_VERITY_BLOCK_TYPE_DATA,
526 cur_block, NULL, &start) == 0)
527 continue;
528 else if (verity_handle_err(v, DM_VERITY_BLOCK_TYPE_DATA,
529 cur_block))
530 return -EIO;
531 }
532
533 return 0;
534}
535
536/*
537 * End one "io" structure with a given error.
538 */
539static void verity_finish_io(struct dm_verity_io *io, blk_status_t status)
540{
541 struct dm_verity *v = io->v;
542 struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size);
543
544 bio->bi_end_io = io->orig_bi_end_io;
545 bio->bi_status = status;
546
547 verity_fec_finish_io(io);
548
549 bio_endio(bio);
550}
551
552static void verity_work(struct work_struct *w)
553{
554 struct dm_verity_io *io = container_of(w, struct dm_verity_io, work);
555
556 verity_finish_io(io, errno_to_blk_status(verity_verify_io(io)));
557}
558
559static void verity_end_io(struct bio *bio)
560{
561 struct dm_verity_io *io = bio->bi_private;
562
563 if (bio->bi_status && !verity_fec_is_enabled(io->v)) {
564 verity_finish_io(io, bio->bi_status);
565 return;
566 }
567
568 INIT_WORK(&io->work, verity_work);
569 queue_work(io->v->verify_wq, &io->work);
570}
571
572/*
573 * Prefetch buffers for the specified io.
574 * The root buffer is not prefetched, it is assumed that it will be cached
575 * all the time.
576 */
577static void verity_prefetch_io(struct work_struct *work)
578{
579 struct dm_verity_prefetch_work *pw =
580 container_of(work, struct dm_verity_prefetch_work, work);
581 struct dm_verity *v = pw->v;
582 int i;
583
584 for (i = v->levels - 2; i >= 0; i--) {
585 sector_t hash_block_start;
586 sector_t hash_block_end;
587 verity_hash_at_level(v, pw->block, i, &hash_block_start, NULL);
588 verity_hash_at_level(v, pw->block + pw->n_blocks - 1, i, &hash_block_end, NULL);
589 if (!i) {
590 unsigned cluster = READ_ONCE(dm_verity_prefetch_cluster);
591
592 cluster >>= v->data_dev_block_bits;
593 if (unlikely(!cluster))
594 goto no_prefetch_cluster;
595
596 if (unlikely(cluster & (cluster - 1)))
597 cluster = 1 << __fls(cluster);
598
599 hash_block_start &= ~(sector_t)(cluster - 1);
600 hash_block_end |= cluster - 1;
601 if (unlikely(hash_block_end >= v->hash_blocks))
602 hash_block_end = v->hash_blocks - 1;
603 }
604no_prefetch_cluster:
605 dm_bufio_prefetch(v->bufio, hash_block_start,
606 hash_block_end - hash_block_start + 1);
607 }
608
609 kfree(pw);
610}
611
612static void verity_submit_prefetch(struct dm_verity *v, struct dm_verity_io *io)
613{
614 struct dm_verity_prefetch_work *pw;
615
616 pw = kmalloc(sizeof(struct dm_verity_prefetch_work),
617 GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
618
619 if (!pw)
620 return;
621
622 INIT_WORK(&pw->work, verity_prefetch_io);
623 pw->v = v;
624 pw->block = io->block;
625 pw->n_blocks = io->n_blocks;
626 queue_work(v->verify_wq, &pw->work);
627}
628
629/*
630 * Bio map function. It allocates dm_verity_io structure and bio vector and
631 * fills them. Then it issues prefetches and the I/O.
632 */
633static int verity_map(struct dm_target *ti, struct bio *bio)
634{
635 struct dm_verity *v = ti->private;
636 struct dm_verity_io *io;
637
638 bio_set_dev(bio, v->data_dev->bdev);
639 bio->bi_iter.bi_sector = verity_map_sector(v, bio->bi_iter.bi_sector);
640
641 if (((unsigned)bio->bi_iter.bi_sector | bio_sectors(bio)) &
642 ((1 << (v->data_dev_block_bits - SECTOR_SHIFT)) - 1)) {
643 DMERR_LIMIT("unaligned io");
644 return DM_MAPIO_KILL;
645 }
646
647 if (bio_end_sector(bio) >>
648 (v->data_dev_block_bits - SECTOR_SHIFT) > v->data_blocks) {
649 DMERR_LIMIT("io out of range");
650 return DM_MAPIO_KILL;
651 }
652
653 if (bio_data_dir(bio) == WRITE)
654 return DM_MAPIO_KILL;
655
656 io = dm_per_bio_data(bio, ti->per_io_data_size);
657 io->v = v;
658 io->orig_bi_end_io = bio->bi_end_io;
659 io->block = bio->bi_iter.bi_sector >> (v->data_dev_block_bits - SECTOR_SHIFT);
660 io->n_blocks = bio->bi_iter.bi_size >> v->data_dev_block_bits;
661
662 bio->bi_end_io = verity_end_io;
663 bio->bi_private = io;
664 io->iter = bio->bi_iter;
665
666 verity_fec_init_io(io);
667
668 verity_submit_prefetch(v, io);
669
670 generic_make_request(bio);
671
672 return DM_MAPIO_SUBMITTED;
673}
674
675/*
676 * Status: V (valid) or C (corruption found)
677 */
678static void verity_status(struct dm_target *ti, status_type_t type,
679 unsigned status_flags, char *result, unsigned maxlen)
680{
681 struct dm_verity *v = ti->private;
682 unsigned args = 0;
683 unsigned sz = 0;
684 unsigned x;
685
686 switch (type) {
687 case STATUSTYPE_INFO:
688 DMEMIT("%c", v->hash_failed ? 'C' : 'V');
689 break;
690 case STATUSTYPE_TABLE:
691 DMEMIT("%u %s %s %u %u %llu %llu %s ",
692 v->version,
693 v->data_dev->name,
694 v->hash_dev->name,
695 1 << v->data_dev_block_bits,
696 1 << v->hash_dev_block_bits,
697 (unsigned long long)v->data_blocks,
698 (unsigned long long)v->hash_start,
699 v->alg_name
700 );
701 for (x = 0; x < v->digest_size; x++)
702 DMEMIT("%02x", v->root_digest[x]);
703 DMEMIT(" ");
704 if (!v->salt_size)
705 DMEMIT("-");
706 else
707 for (x = 0; x < v->salt_size; x++)
708 DMEMIT("%02x", v->salt[x]);
709 if (v->mode != DM_VERITY_MODE_EIO)
710 args++;
711 if (verity_fec_is_enabled(v))
712 args += DM_VERITY_OPTS_FEC;
713 if (v->zero_digest)
714 args++;
715 if (v->validated_blocks)
716 args++;
717 if (!args)
718 return;
719 DMEMIT(" %u", args);
720 if (v->mode != DM_VERITY_MODE_EIO) {
721 DMEMIT(" ");
722 switch (v->mode) {
723 case DM_VERITY_MODE_LOGGING:
724 DMEMIT(DM_VERITY_OPT_LOGGING);
725 break;
726 case DM_VERITY_MODE_RESTART:
727 DMEMIT(DM_VERITY_OPT_RESTART);
728 break;
729 default:
730 BUG();
731 }
732 }
733 if (v->zero_digest)
734 DMEMIT(" " DM_VERITY_OPT_IGN_ZEROES);
735 if (v->validated_blocks)
736 DMEMIT(" " DM_VERITY_OPT_AT_MOST_ONCE);
737 sz = verity_fec_status_table(v, sz, result, maxlen);
738 break;
739 }
740}
741
742static int verity_prepare_ioctl(struct dm_target *ti, struct block_device **bdev)
743{
744 struct dm_verity *v = ti->private;
745
746 *bdev = v->data_dev->bdev;
747
748 if (v->data_start ||
749 ti->len != i_size_read(v->data_dev->bdev->bd_inode) >> SECTOR_SHIFT)
750 return 1;
751 return 0;
752}
753
754static int verity_iterate_devices(struct dm_target *ti,
755 iterate_devices_callout_fn fn, void *data)
756{
757 struct dm_verity *v = ti->private;
758
759 return fn(ti, v->data_dev, v->data_start, ti->len, data);
760}
761
762static void verity_io_hints(struct dm_target *ti, struct queue_limits *limits)
763{
764 struct dm_verity *v = ti->private;
765
766 if (limits->logical_block_size < 1 << v->data_dev_block_bits)
767 limits->logical_block_size = 1 << v->data_dev_block_bits;
768
769 if (limits->physical_block_size < 1 << v->data_dev_block_bits)
770 limits->physical_block_size = 1 << v->data_dev_block_bits;
771
772 blk_limits_io_min(limits, limits->logical_block_size);
773}
774
775static void verity_dtr(struct dm_target *ti)
776{
777 struct dm_verity *v = ti->private;
778
779 if (v->verify_wq)
780 destroy_workqueue(v->verify_wq);
781
782 if (v->bufio)
783 dm_bufio_client_destroy(v->bufio);
784
785 kvfree(v->validated_blocks);
786 kfree(v->salt);
787 kfree(v->root_digest);
788 kfree(v->zero_digest);
789
790 if (v->tfm)
791 crypto_free_ahash(v->tfm);
792
793 kfree(v->alg_name);
794
795 if (v->hash_dev)
796 dm_put_device(ti, v->hash_dev);
797
798 if (v->data_dev)
799 dm_put_device(ti, v->data_dev);
800
801 verity_fec_dtr(v);
802
803 kfree(v);
804}
805
806static int verity_alloc_most_once(struct dm_verity *v)
807{
808 struct dm_target *ti = v->ti;
809
810 /* the bitset can only handle INT_MAX blocks */
811 if (v->data_blocks > INT_MAX) {
812 ti->error = "device too large to use check_at_most_once";
813 return -E2BIG;
814 }
815
816 v->validated_blocks = kvcalloc(BITS_TO_LONGS(v->data_blocks),
817 sizeof(unsigned long),
818 GFP_KERNEL);
819 if (!v->validated_blocks) {
820 ti->error = "failed to allocate bitset for check_at_most_once";
821 return -ENOMEM;
822 }
823
824 return 0;
825}
826
827static int verity_alloc_zero_digest(struct dm_verity *v)
828{
829 int r = -ENOMEM;
830 struct ahash_request *req;
831 u8 *zero_data;
832
833 v->zero_digest = kmalloc(v->digest_size, GFP_KERNEL);
834
835 if (!v->zero_digest)
836 return r;
837
838 req = kmalloc(v->ahash_reqsize, GFP_KERNEL);
839
840 if (!req)
841 return r; /* verity_dtr will free zero_digest */
842
843 zero_data = kzalloc(1 << v->data_dev_block_bits, GFP_KERNEL);
844
845 if (!zero_data)
846 goto out;
847
848 r = verity_hash(v, req, zero_data, 1 << v->data_dev_block_bits,
849 v->zero_digest);
850
851out:
852 kfree(req);
853 kfree(zero_data);
854
855 return r;
856}
857
858static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v)
859{
860 int r;
861 unsigned argc;
862 struct dm_target *ti = v->ti;
863 const char *arg_name;
864
865 static const struct dm_arg _args[] = {
866 {0, DM_VERITY_OPTS_MAX, "Invalid number of feature args"},
867 };
868
869 r = dm_read_arg_group(_args, as, &argc, &ti->error);
870 if (r)
871 return -EINVAL;
872
873 if (!argc)
874 return 0;
875
876 do {
877 arg_name = dm_shift_arg(as);
878 argc--;
879
880 if (!strcasecmp(arg_name, DM_VERITY_OPT_LOGGING)) {
881 v->mode = DM_VERITY_MODE_LOGGING;
882 continue;
883
884 } else if (!strcasecmp(arg_name, DM_VERITY_OPT_RESTART)) {
885 v->mode = DM_VERITY_MODE_RESTART;
886 continue;
887
888 } else if (!strcasecmp(arg_name, DM_VERITY_OPT_IGN_ZEROES)) {
889 r = verity_alloc_zero_digest(v);
890 if (r) {
891 ti->error = "Cannot allocate zero digest";
892 return r;
893 }
894 continue;
895
896 } else if (!strcasecmp(arg_name, DM_VERITY_OPT_AT_MOST_ONCE)) {
897 r = verity_alloc_most_once(v);
898 if (r)
899 return r;
900 continue;
901
902 } else if (verity_is_fec_opt_arg(arg_name)) {
903 r = verity_fec_parse_opt_args(as, v, &argc, arg_name);
904 if (r)
905 return r;
906 continue;
907 }
908
909 ti->error = "Unrecognized verity feature request";
910 return -EINVAL;
911 } while (argc && !r);
912
913 return r;
914}
915
916/*
917 * Target parameters:
918 * <version> The current format is version 1.
919 * Vsn 0 is compatible with original Chromium OS releases.
920 * <data device>
921 * <hash device>
922 * <data block size>
923 * <hash block size>
924 * <the number of data blocks>
925 * <hash start block>
926 * <algorithm>
927 * <digest>
928 * <salt> Hex string or "-" if no salt.
929 */
930static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv)
931{
932 struct dm_verity *v;
933 struct dm_arg_set as;
934 unsigned int num;
935 unsigned long long num_ll;
936 int r;
937 int i;
938 sector_t hash_position;
939 char dummy;
940
941 v = kzalloc(sizeof(struct dm_verity), GFP_KERNEL);
942 if (!v) {
943 ti->error = "Cannot allocate verity structure";
944 return -ENOMEM;
945 }
946 ti->private = v;
947 v->ti = ti;
948
949 r = verity_fec_ctr_alloc(v);
950 if (r)
951 goto bad;
952
953 if ((dm_table_get_mode(ti->table) & ~FMODE_READ)) {
954 ti->error = "Device must be readonly";
955 r = -EINVAL;
956 goto bad;
957 }
958
959 if (argc < 10) {
960 ti->error = "Not enough arguments";
961 r = -EINVAL;
962 goto bad;
963 }
964
965 if (sscanf(argv[0], "%u%c", &num, &dummy) != 1 ||
966 num > 1) {
967 ti->error = "Invalid version";
968 r = -EINVAL;
969 goto bad;
970 }
971 v->version = num;
972
973 r = dm_get_device(ti, argv[1], FMODE_READ, &v->data_dev);
974 if (r) {
975 ti->error = "Data device lookup failed";
976 goto bad;
977 }
978
979 r = dm_get_device(ti, argv[2], FMODE_READ, &v->hash_dev);
980 if (r) {
981 ti->error = "Hash device lookup failed";
982 goto bad;
983 }
984
985 if (sscanf(argv[3], "%u%c", &num, &dummy) != 1 ||
986 !num || (num & (num - 1)) ||
987 num < bdev_logical_block_size(v->data_dev->bdev) ||
988 num > PAGE_SIZE) {
989 ti->error = "Invalid data device block size";
990 r = -EINVAL;
991 goto bad;
992 }
993 v->data_dev_block_bits = __ffs(num);
994
995 if (sscanf(argv[4], "%u%c", &num, &dummy) != 1 ||
996 !num || (num & (num - 1)) ||
997 num < bdev_logical_block_size(v->hash_dev->bdev) ||
998 num > INT_MAX) {
999 ti->error = "Invalid hash device block size";
1000 r = -EINVAL;
1001 goto bad;
1002 }
1003 v->hash_dev_block_bits = __ffs(num);
1004
1005 if (sscanf(argv[5], "%llu%c", &num_ll, &dummy) != 1 ||
1006 (sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT))
1007 >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll) {
1008 ti->error = "Invalid data blocks";
1009 r = -EINVAL;
1010 goto bad;
1011 }
1012 v->data_blocks = num_ll;
1013
1014 if (ti->len > (v->data_blocks << (v->data_dev_block_bits - SECTOR_SHIFT))) {
1015 ti->error = "Data device is too small";
1016 r = -EINVAL;
1017 goto bad;
1018 }
1019
1020 if (sscanf(argv[6], "%llu%c", &num_ll, &dummy) != 1 ||
1021 (sector_t)(num_ll << (v->hash_dev_block_bits - SECTOR_SHIFT))
1022 >> (v->hash_dev_block_bits - SECTOR_SHIFT) != num_ll) {
1023 ti->error = "Invalid hash start";
1024 r = -EINVAL;
1025 goto bad;
1026 }
1027 v->hash_start = num_ll;
1028
1029 v->alg_name = kstrdup(argv[7], GFP_KERNEL);
1030 if (!v->alg_name) {
1031 ti->error = "Cannot allocate algorithm name";
1032 r = -ENOMEM;
1033 goto bad;
1034 }
1035
1036 v->tfm = crypto_alloc_ahash(v->alg_name, 0, 0);
1037 if (IS_ERR(v->tfm)) {
1038 ti->error = "Cannot initialize hash function";
1039 r = PTR_ERR(v->tfm);
1040 v->tfm = NULL;
1041 goto bad;
1042 }
1043
1044 /*
1045 * dm-verity performance can vary greatly depending on which hash
1046 * algorithm implementation is used. Help people debug performance
1047 * problems by logging the ->cra_driver_name.
1048 */
1049 DMINFO("%s using implementation \"%s\"", v->alg_name,
1050 crypto_hash_alg_common(v->tfm)->base.cra_driver_name);
1051
1052 v->digest_size = crypto_ahash_digestsize(v->tfm);
1053 if ((1 << v->hash_dev_block_bits) < v->digest_size * 2) {
1054 ti->error = "Digest size too big";
1055 r = -EINVAL;
1056 goto bad;
1057 }
1058 v->ahash_reqsize = sizeof(struct ahash_request) +
1059 crypto_ahash_reqsize(v->tfm);
1060
1061 v->root_digest = kmalloc(v->digest_size, GFP_KERNEL);
1062 if (!v->root_digest) {
1063 ti->error = "Cannot allocate root digest";
1064 r = -ENOMEM;
1065 goto bad;
1066 }
1067 if (strlen(argv[8]) != v->digest_size * 2 ||
1068 hex2bin(v->root_digest, argv[8], v->digest_size)) {
1069 ti->error = "Invalid root digest";
1070 r = -EINVAL;
1071 goto bad;
1072 }
1073
1074 if (strcmp(argv[9], "-")) {
1075 v->salt_size = strlen(argv[9]) / 2;
1076 v->salt = kmalloc(v->salt_size, GFP_KERNEL);
1077 if (!v->salt) {
1078 ti->error = "Cannot allocate salt";
1079 r = -ENOMEM;
1080 goto bad;
1081 }
1082 if (strlen(argv[9]) != v->salt_size * 2 ||
1083 hex2bin(v->salt, argv[9], v->salt_size)) {
1084 ti->error = "Invalid salt";
1085 r = -EINVAL;
1086 goto bad;
1087 }
1088 }
1089
1090 argv += 10;
1091 argc -= 10;
1092
1093 /* Optional parameters */
1094 if (argc) {
1095 as.argc = argc;
1096 as.argv = argv;
1097
1098 r = verity_parse_opt_args(&as, v);
1099 if (r < 0)
1100 goto bad;
1101 }
1102
1103 v->hash_per_block_bits =
1104 __fls((1 << v->hash_dev_block_bits) / v->digest_size);
1105
1106 v->levels = 0;
1107 if (v->data_blocks)
1108 while (v->hash_per_block_bits * v->levels < 64 &&
1109 (unsigned long long)(v->data_blocks - 1) >>
1110 (v->hash_per_block_bits * v->levels))
1111 v->levels++;
1112
1113 if (v->levels > DM_VERITY_MAX_LEVELS) {
1114 ti->error = "Too many tree levels";
1115 r = -E2BIG;
1116 goto bad;
1117 }
1118
1119 hash_position = v->hash_start;
1120 for (i = v->levels - 1; i >= 0; i--) {
1121 sector_t s;
1122 v->hash_level_block[i] = hash_position;
1123 s = (v->data_blocks + ((sector_t)1 << ((i + 1) * v->hash_per_block_bits)) - 1)
1124 >> ((i + 1) * v->hash_per_block_bits);
1125 if (hash_position + s < hash_position) {
1126 ti->error = "Hash device offset overflow";
1127 r = -E2BIG;
1128 goto bad;
1129 }
1130 hash_position += s;
1131 }
1132 v->hash_blocks = hash_position;
1133
1134 v->bufio = dm_bufio_client_create(v->hash_dev->bdev,
1135 1 << v->hash_dev_block_bits, 1, sizeof(struct buffer_aux),
1136 dm_bufio_alloc_callback, NULL);
1137 if (IS_ERR(v->bufio)) {
1138 ti->error = "Cannot initialize dm-bufio";
1139 r = PTR_ERR(v->bufio);
1140 v->bufio = NULL;
1141 goto bad;
1142 }
1143
1144 if (dm_bufio_get_device_size(v->bufio) < v->hash_blocks) {
1145 ti->error = "Hash device is too small";
1146 r = -E2BIG;
1147 goto bad;
1148 }
1149
1150 /* WQ_UNBOUND greatly improves performance when running on ramdisk */
1151 v->verify_wq = alloc_workqueue("kverityd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | WQ_UNBOUND, num_online_cpus());
1152 if (!v->verify_wq) {
1153 ti->error = "Cannot allocate workqueue";
1154 r = -ENOMEM;
1155 goto bad;
1156 }
1157
1158 ti->per_io_data_size = sizeof(struct dm_verity_io) +
1159 v->ahash_reqsize + v->digest_size * 2;
1160
1161 r = verity_fec_ctr(v);
1162 if (r)
1163 goto bad;
1164
1165 ti->per_io_data_size = roundup(ti->per_io_data_size,
1166 __alignof__(struct dm_verity_io));
1167
1168 return 0;
1169
1170bad:
1171 verity_dtr(ti);
1172
1173 return r;
1174}
1175
1176static struct target_type verity_target = {
1177 .name = "verity",
1178 .version = {1, 4, 0},
1179 .module = THIS_MODULE,
1180 .ctr = verity_ctr,
1181 .dtr = verity_dtr,
1182 .map = verity_map,
1183 .status = verity_status,
1184 .prepare_ioctl = verity_prepare_ioctl,
1185 .iterate_devices = verity_iterate_devices,
1186 .io_hints = verity_io_hints,
1187};
1188
1189static int __init dm_verity_init(void)
1190{
1191 int r;
1192
1193 r = dm_register_target(&verity_target);
1194 if (r < 0)
1195 DMERR("register failed %d", r);
1196
1197 return r;
1198}
1199
1200static void __exit dm_verity_exit(void)
1201{
1202 dm_unregister_target(&verity_target);
1203}
1204
1205module_init(dm_verity_init);
1206module_exit(dm_verity_exit);
1207
1208MODULE_AUTHOR("Mikulas Patocka <mpatocka@redhat.com>");
1209MODULE_AUTHOR("Mandeep Baines <msb@chromium.org>");
1210MODULE_AUTHOR("Will Drewry <wad@chromium.org>");
1211MODULE_DESCRIPTION(DM_NAME " target for transparent disk integrity checking");
1212MODULE_LICENSE("GPL");
1213