1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Copyright (C) 2003 Jana Saout <jana@saout.de>
4	* Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org>
5	* Copyright (C) 2006-2020 Red Hat, Inc. All rights reserved.
6	* Copyright (C) 2013-2020 Milan Broz <gmazyland@gmail.com>
7	*
8	* This file is released under the GPL.
9	*/
10
11	#include <linux/completion.h>
12	#include <linux/err.h>
13	#include <linux/module.h>
14	#include <linux/init.h>
15	#include <linux/kernel.h>
16	#include <linux/key.h>
17	#include <linux/bio.h>
18	#include <linux/blkdev.h>
19	#include <linux/blk-integrity.h>
20	#include <linux/mempool.h>
21	#include <linux/slab.h>
22	#include <linux/crypto.h>
23	#include <linux/workqueue.h>
24	#include <linux/kthread.h>
25	#include <linux/backing-dev.h>
26	#include <linux/atomic.h>
27	#include <linux/scatterlist.h>
28	#include <linux/rbtree.h>
29	#include <linux/ctype.h>
30	#include <asm/page.h>
31	#include <asm/unaligned.h>
32	#include <crypto/hash.h>
33	#include <crypto/md5.h>
34	#include <crypto/skcipher.h>
35	#include <crypto/aead.h>
36	#include <crypto/authenc.h>
37	#include <crypto/utils.h>
38	#include <linux/rtnetlink.h> /* for struct rtattr and RTA macros only */
39	#include <linux/key-type.h>
40	#include <keys/user-type.h>
41	#include <keys/encrypted-type.h>
42	#include <keys/trusted-type.h>
43
44	#include <linux/device-mapper.h>
45
46	#include "dm-audit.h"
47
48	#define DM_MSG_PREFIX "crypt"
49
50	/*
51	* context holding the current state of a multi-part conversion
52	*/
53	struct convert_context {
54	struct completion restart;
55	struct bio *bio_in;
56	struct bio *bio_out;
57	struct bvec_iter iter_in;
58	struct bvec_iter iter_out;
59	u64 cc_sector;
60	atomic_t cc_pending;
61	union {
62	struct skcipher_request *req;
63	struct aead_request *req_aead;
64	} r;
65
66	};
67
68	/*
69	* per bio private data
70	*/
71	struct dm_crypt_io {
72	struct crypt_config *cc;
73	struct bio *base_bio;
74	u8 *integrity_metadata;
75	bool integrity_metadata_from_pool:1;
76	bool in_tasklet:1;
77
78	struct work_struct work;
79	struct tasklet_struct tasklet;
80
81	struct convert_context ctx;
82
83	atomic_t io_pending;
84	blk_status_t error;
85	sector_t sector;
86
87	struct rb_node rb_node;
88	} CRYPTO_MINALIGN_ATTR;
89
90	struct dm_crypt_request {
91	struct convert_context *ctx;
92	struct scatterlist sg_in[4];
93	struct scatterlist sg_out[4];
94	u64 iv_sector;
95	};
96
97	struct crypt_config;
98
99	struct crypt_iv_operations {
100	int (*ctr)(struct crypt_config *cc, struct dm_target *ti,
101	const char *opts);
102	void (*dtr)(struct crypt_config *cc);
103	int (*init)(struct crypt_config *cc);
104	int (*wipe)(struct crypt_config *cc);
105	int (*generator)(struct crypt_config *cc, u8 *iv,
106	struct dm_crypt_request *dmreq);
107	int (*post)(struct crypt_config *cc, u8 *iv,
108	struct dm_crypt_request *dmreq);
109	};
110
111	struct iv_benbi_private {
112	int shift;
113	};
114
115	#define LMK_SEED_SIZE 64 /* hash + 0 */
116	struct iv_lmk_private {
117	struct crypto_shash *hash_tfm;
118	u8 *seed;
119	};
120
121	#define TCW_WHITENING_SIZE 16
122	struct iv_tcw_private {
123	struct crypto_shash *crc32_tfm;
124	u8 *iv_seed;
125	u8 *whitening;
126	};
127
128	#define ELEPHANT_MAX_KEY_SIZE 32
129	struct iv_elephant_private {
130	struct crypto_skcipher *tfm;
131	};
132
133	/*
134	* Crypt: maps a linear range of a block device
135	* and encrypts / decrypts at the same time.
136	*/
137	enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID,
138	DM_CRYPT_SAME_CPU, DM_CRYPT_NO_OFFLOAD,
139	DM_CRYPT_NO_READ_WORKQUEUE, DM_CRYPT_NO_WRITE_WORKQUEUE,
140	DM_CRYPT_WRITE_INLINE };
141
142	enum cipher_flags {
143	CRYPT_MODE_INTEGRITY_AEAD, /* Use authenticated mode for cipher */
144	CRYPT_IV_LARGE_SECTORS, /* Calculate IV from sector_size, not 512B sectors */
145	CRYPT_ENCRYPT_PREPROCESS, /* Must preprocess data for encryption (elephant) */
146	};
147
148	/*
149	* The fields in here must be read only after initialization.
150	*/
151	struct crypt_config {
152	struct dm_dev *dev;
153	sector_t start;
154
155	struct percpu_counter n_allocated_pages;
156
157	struct workqueue_struct *io_queue;
158	struct workqueue_struct *crypt_queue;
159
160	spinlock_t write_thread_lock;
161	struct task_struct *write_thread;
162	struct rb_root write_tree;
163
164	char *cipher_string;
165	char *cipher_auth;
166	char *key_string;
167
168	const struct crypt_iv_operations *iv_gen_ops;
169	union {
170	struct iv_benbi_private benbi;
171	struct iv_lmk_private lmk;
172	struct iv_tcw_private tcw;
173	struct iv_elephant_private elephant;
174	} iv_gen_private;
175	u64 iv_offset;
176	unsigned int iv_size;
177	unsigned short sector_size;
178	unsigned char sector_shift;
179
180	union {
181	struct crypto_skcipher **tfms;
182	struct crypto_aead **tfms_aead;
183	} cipher_tfm;
184	unsigned int tfms_count;
185	unsigned long cipher_flags;
186
187	/*
188	* Layout of each crypto request:
189	*
190	* struct skcipher_request
191	* context
192	* padding
193	* struct dm_crypt_request
194	* padding
195	* IV
196	*
197	* The padding is added so that dm_crypt_request and the IV are
198	* correctly aligned.
199	*/
200	unsigned int dmreq_start;
201
202	unsigned int per_bio_data_size;
203
204	unsigned long flags;
205	unsigned int key_size;
206	unsigned int key_parts; /* independent parts in key buffer */
207	unsigned int key_extra_size; /* additional keys length */
208	unsigned int key_mac_size; /* MAC key size for authenc(...) */
209
210	unsigned int integrity_tag_size;
211	unsigned int integrity_iv_size;
212	unsigned int on_disk_tag_size;
213
214	/*
215	* pool for per bio private data, crypto requests,
216	* encryption requeusts/buffer pages and integrity tags
217	*/
218	unsigned int tag_pool_max_sectors;
219	mempool_t tag_pool;
220	mempool_t req_pool;
221	mempool_t page_pool;
222
223	struct bio_set bs;
224	struct mutex bio_alloc_lock;
225
226	u8 *authenc_key; /* space for keys in authenc() format (if used) */
227	u8 key[] __counted_by(key_size);
228	};
229
230	#define MIN_IOS 64
231	#define MAX_TAG_SIZE 480
232	#define POOL_ENTRY_SIZE 512
233
234	static DEFINE_SPINLOCK(dm_crypt_clients_lock);
235	static unsigned int dm_crypt_clients_n;
236	static volatile unsigned long dm_crypt_pages_per_client;
237	#define DM_CRYPT_MEMORY_PERCENT 2
238	#define DM_CRYPT_MIN_PAGES_PER_CLIENT (BIO_MAX_VECS * 16)
239
240	static void crypt_endio(struct bio *clone);
241	static void kcryptd_queue_crypt(struct dm_crypt_io *io);
242	static struct scatterlist *crypt_get_sg_data(struct crypt_config *cc,
243	struct scatterlist *sg);
244
245	static bool crypt_integrity_aead(struct crypt_config *cc);
246
247	/*
248	* Use this to access cipher attributes that are independent of the key.
249	*/
250	static struct crypto_skcipher *any_tfm(struct crypt_config *cc)
251	{
252	return cc->cipher_tfm.tfms[0];
253	}
254
255	static struct crypto_aead *any_tfm_aead(struct crypt_config *cc)
256	{
257	return cc->cipher_tfm.tfms_aead[0];
258	}
259
260	/*
261	* Different IV generation algorithms:
262	*
263	* plain: the initial vector is the 32-bit little-endian version of the sector
264	* number, padded with zeros if necessary.
265	*
266	* plain64: the initial vector is the 64-bit little-endian version of the sector
267	* number, padded with zeros if necessary.
268	*
269	* plain64be: the initial vector is the 64-bit big-endian version of the sector
270	* number, padded with zeros if necessary.
271	*
272	* essiv: "encrypted sector|salt initial vector", the sector number is
273	* encrypted with the bulk cipher using a salt as key. The salt
274	* should be derived from the bulk cipher's key via hashing.
275	*
276	* benbi: the 64-bit "big-endian 'narrow block'-count", starting at 1
277	* (needed for LRW-32-AES and possible other narrow block modes)
278	*
279	* null: the initial vector is always zero. Provides compatibility with
280	* obsolete loop_fish2 devices. Do not use for new devices.
281	*
282	* lmk: Compatible implementation of the block chaining mode used
283	* by the Loop-AES block device encryption system
284	* designed by Jari Ruusu. See http://loop-aes.sourceforge.net/
285	* It operates on full 512 byte sectors and uses CBC
286	* with an IV derived from the sector number, the data and
287	* optionally extra IV seed.
288	* This means that after decryption the first block
289	* of sector must be tweaked according to decrypted data.
290	* Loop-AES can use three encryption schemes:
291	* version 1: is plain aes-cbc mode
292	* version 2: uses 64 multikey scheme with lmk IV generator
293	* version 3: the same as version 2 with additional IV seed
294	* (it uses 65 keys, last key is used as IV seed)
295	*
296	* tcw: Compatible implementation of the block chaining mode used
297	* by the TrueCrypt device encryption system (prior to version 4.1).
298	* For more info see: https://gitlab.com/cryptsetup/cryptsetup/wikis/TrueCryptOnDiskFormat
299	* It operates on full 512 byte sectors and uses CBC
300	* with an IV derived from initial key and the sector number.
301	* In addition, whitening value is applied on every sector, whitening
302	* is calculated from initial key, sector number and mixed using CRC32.
303	* Note that this encryption scheme is vulnerable to watermarking attacks
304	* and should be used for old compatible containers access only.
305	*
306	* eboiv: Encrypted byte-offset IV (used in Bitlocker in CBC mode)
307	* The IV is encrypted little-endian byte-offset (with the same key
308	* and cipher as the volume).
309	*
310	* elephant: The extended version of eboiv with additional Elephant diffuser
311	* used with Bitlocker CBC mode.
312	* This mode was used in older Windows systems
313	* https://download.microsoft.com/download/0/2/3/0238acaf-d3bf-4a6d-b3d6-0a0be4bbb36e/bitlockercipher200608.pdf
314	*/
315
316	static int crypt_iv_plain_gen(struct crypt_config *cc, u8 *iv,
317	struct dm_crypt_request *dmreq)
318	{
319	memset(iv, 0, cc->iv_size);
320	*(__le32 *)iv = cpu_to_le32(dmreq->iv_sector & 0xffffffff);
321
322	return 0;
323	}
324
325	static int crypt_iv_plain64_gen(struct crypt_config *cc, u8 *iv,
326	struct dm_crypt_request *dmreq)
327	{
328	memset(iv, 0, cc->iv_size);
329	*(__le64 *)iv = cpu_to_le64(dmreq->iv_sector);
330
331	return 0;
332	}
333
334	static int crypt_iv_plain64be_gen(struct crypt_config *cc, u8 *iv,
335	struct dm_crypt_request *dmreq)
336	{
337	memset(iv, 0, cc->iv_size);
338	/* iv_size is at least of size u64; usually it is 16 bytes */
339	*(__be64 *)&iv[cc->iv_size - sizeof(u64)] = cpu_to_be64(dmreq->iv_sector);
340
341	return 0;
342	}
343
344	static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv,
345	struct dm_crypt_request *dmreq)
346	{
347	/*
348	* ESSIV encryption of the IV is now handled by the crypto API,
349	* so just pass the plain sector number here.
350	*/
351	memset(iv, 0, cc->iv_size);
352	*(__le64 *)iv = cpu_to_le64(dmreq->iv_sector);
353
354	return 0;
355	}
356
357	static int crypt_iv_benbi_ctr(struct crypt_config *cc, struct dm_target *ti,
358	const char *opts)
359	{
360	unsigned int bs;
361	int log;
362
363	if (crypt_integrity_aead(cc))
364	bs = crypto_aead_blocksize(tfm: any_tfm_aead(cc));
365	else
366	bs = crypto_skcipher_blocksize(tfm: any_tfm(cc));
367	log = ilog2(bs);
368
369	/*
370	* We need to calculate how far we must shift the sector count
371	* to get the cipher block count, we use this shift in _gen.
372	*/
373	if (1 << log != bs) {
374	ti->error = "cypher blocksize is not a power of 2";
375	return -EINVAL;
376	}
377
378	if (log > 9) {
379	ti->error = "cypher blocksize is > 512";
380	return -EINVAL;
381	}
382
383	cc->iv_gen_private.benbi.shift = 9 - log;
384
385	return 0;
386	}
387
388	static void crypt_iv_benbi_dtr(struct crypt_config *cc)
389	{
390	}
391
392	static int crypt_iv_benbi_gen(struct crypt_config *cc, u8 *iv,
393	struct dm_crypt_request *dmreq)
394	{
395	__be64 val;
396
397	memset(iv, 0, cc->iv_size - sizeof(u64)); /* rest is cleared below */
398
399	val = cpu_to_be64(((u64)dmreq->iv_sector << cc->iv_gen_private.benbi.shift) + 1);
400	put_unaligned(val, (__be64 *)(iv + cc->iv_size - sizeof(u64)));
401
402	return 0;
403	}
404
405	static int crypt_iv_null_gen(struct crypt_config *cc, u8 *iv,
406	struct dm_crypt_request *dmreq)
407	{
408	memset(iv, 0, cc->iv_size);
409
410	return 0;
411	}
412
413	static void crypt_iv_lmk_dtr(struct crypt_config *cc)
414	{
415	struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
416
417	if (lmk->hash_tfm && !IS_ERR(ptr: lmk->hash_tfm))
418	crypto_free_shash(tfm: lmk->hash_tfm);
419	lmk->hash_tfm = NULL;
420
421	kfree_sensitive(objp: lmk->seed);
422	lmk->seed = NULL;
423	}
424
425	static int crypt_iv_lmk_ctr(struct crypt_config *cc, struct dm_target *ti,
426	const char *opts)
427	{
428	struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
429
430	if (cc->sector_size != (1 << SECTOR_SHIFT)) {
431	ti->error = "Unsupported sector size for LMK";
432	return -EINVAL;
433	}
434
435	lmk->hash_tfm = crypto_alloc_shash(alg_name: "md5", type: 0,
436	CRYPTO_ALG_ALLOCATES_MEMORY);
437	if (IS_ERR(ptr: lmk->hash_tfm)) {
438	ti->error = "Error initializing LMK hash";
439	return PTR_ERR(ptr: lmk->hash_tfm);
440	}
441
442	/* No seed in LMK version 2 */
443	if (cc->key_parts == cc->tfms_count) {
444	lmk->seed = NULL;
445	return 0;
446	}
447
448	lmk->seed = kzalloc(LMK_SEED_SIZE, GFP_KERNEL);
449	if (!lmk->seed) {
450	crypt_iv_lmk_dtr(cc);
451	ti->error = "Error kmallocing seed storage in LMK";
452	return -ENOMEM;
453	}
454
455	return 0;
456	}
457
458	static int crypt_iv_lmk_init(struct crypt_config *cc)
459	{
460	struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
461	int subkey_size = cc->key_size / cc->key_parts;
462
463	/* LMK seed is on the position of LMK_KEYS + 1 key */
464	if (lmk->seed)
465	memcpy(lmk->seed, cc->key + (cc->tfms_count * subkey_size),
466	crypto_shash_digestsize(lmk->hash_tfm));
467
468	return 0;
469	}
470
471	static int crypt_iv_lmk_wipe(struct crypt_config *cc)
472	{
473	struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
474
475	if (lmk->seed)
476	memset(lmk->seed, 0, LMK_SEED_SIZE);
477
478	return 0;
479	}
480
481	static int crypt_iv_lmk_one(struct crypt_config *cc, u8 *iv,
482	struct dm_crypt_request *dmreq,
483	u8 *data)
484	{
485	struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
486	SHASH_DESC_ON_STACK(desc, lmk->hash_tfm);
487	struct md5_state md5state;
488	__le32 buf[4];
489	int i, r;
490
491	desc->tfm = lmk->hash_tfm;
492
493	r = crypto_shash_init(desc);
494	if (r)
495	return r;
496
497	if (lmk->seed) {
498	r = crypto_shash_update(desc, data: lmk->seed, LMK_SEED_SIZE);
499	if (r)
500	return r;
501	}
502
503	/* Sector is always 512B, block size 16, add data of blocks 1-31 */
504	r = crypto_shash_update(desc, data: data + 16, len: 16 * 31);
505	if (r)
506	return r;
507
508	/* Sector is cropped to 56 bits here */
509	buf[0] = cpu_to_le32(dmreq->iv_sector & 0xFFFFFFFF);
510	buf[1] = cpu_to_le32((((u64)dmreq->iv_sector >> 32) & 0x00FFFFFF) | 0x80000000);
511	buf[2] = cpu_to_le32(4024);
512	buf[3] = 0;
513	r = crypto_shash_update(desc, data: (u8 *)buf, len: sizeof(buf));
514	if (r)
515	return r;
516
517	/* No MD5 padding here */
518	r = crypto_shash_export(desc, out: &md5state);
519	if (r)
520	return r;
521
522	for (i = 0; i < MD5_HASH_WORDS; i++)
523	__cpu_to_le32s(&md5state.hash[i]);
524	memcpy(iv, &md5state.hash, cc->iv_size);
525
526	return 0;
527	}
528
529	static int crypt_iv_lmk_gen(struct crypt_config *cc, u8 *iv,
530	struct dm_crypt_request *dmreq)
531	{
532	struct scatterlist *sg;
533	u8 *src;
534	int r = 0;
535
536	if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {
537	sg = crypt_get_sg_data(cc, sg: dmreq->sg_in);
538	src = kmap_local_page(page: sg_page(sg));
539	r = crypt_iv_lmk_one(cc, iv, dmreq, data: src + sg->offset);
540	kunmap_local(src);
541	} else
542	memset(iv, 0, cc->iv_size);
543
544	return r;
545	}
546
547	static int crypt_iv_lmk_post(struct crypt_config *cc, u8 *iv,
548	struct dm_crypt_request *dmreq)
549	{
550	struct scatterlist *sg;
551	u8 *dst;
552	int r;
553
554	if (bio_data_dir(dmreq->ctx->bio_in) == WRITE)
555	return 0;
556
557	sg = crypt_get_sg_data(cc, sg: dmreq->sg_out);
558	dst = kmap_local_page(page: sg_page(sg));
559	r = crypt_iv_lmk_one(cc, iv, dmreq, data: dst + sg->offset);
560
561	/* Tweak the first block of plaintext sector */
562	if (!r)
563	crypto_xor(dst: dst + sg->offset, src: iv, size: cc->iv_size);
564
565	kunmap_local(dst);
566	return r;
567	}
568
569	static void crypt_iv_tcw_dtr(struct crypt_config *cc)
570	{
571	struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
572
573	kfree_sensitive(objp: tcw->iv_seed);
574	tcw->iv_seed = NULL;
575	kfree_sensitive(objp: tcw->whitening);
576	tcw->whitening = NULL;
577
578	if (tcw->crc32_tfm && !IS_ERR(ptr: tcw->crc32_tfm))
579	crypto_free_shash(tfm: tcw->crc32_tfm);
580	tcw->crc32_tfm = NULL;
581	}
582
583	static int crypt_iv_tcw_ctr(struct crypt_config *cc, struct dm_target *ti,
584	const char *opts)
585	{
586	struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
587
588	if (cc->sector_size != (1 << SECTOR_SHIFT)) {
589	ti->error = "Unsupported sector size for TCW";
590	return -EINVAL;
591	}
592
593	if (cc->key_size <= (cc->iv_size + TCW_WHITENING_SIZE)) {
594	ti->error = "Wrong key size for TCW";
595	return -EINVAL;
596	}
597
598	tcw->crc32_tfm = crypto_alloc_shash(alg_name: "crc32", type: 0,
599	CRYPTO_ALG_ALLOCATES_MEMORY);
600	if (IS_ERR(ptr: tcw->crc32_tfm)) {
601	ti->error = "Error initializing CRC32 in TCW";
602	return PTR_ERR(ptr: tcw->crc32_tfm);
603	}
604
605	tcw->iv_seed = kzalloc(size: cc->iv_size, GFP_KERNEL);
606	tcw->whitening = kzalloc(TCW_WHITENING_SIZE, GFP_KERNEL);
607	if (!tcw->iv_seed || !tcw->whitening) {
608	crypt_iv_tcw_dtr(cc);
609	ti->error = "Error allocating seed storage in TCW";
610	return -ENOMEM;
611	}
612
613	return 0;
614	}
615
616	static int crypt_iv_tcw_init(struct crypt_config *cc)
617	{
618	struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
619	int key_offset = cc->key_size - cc->iv_size - TCW_WHITENING_SIZE;
620
621	memcpy(tcw->iv_seed, &cc->key[key_offset], cc->iv_size);
622	memcpy(tcw->whitening, &cc->key[key_offset + cc->iv_size],
623	TCW_WHITENING_SIZE);
624
625	return 0;
626	}
627
628	static int crypt_iv_tcw_wipe(struct crypt_config *cc)
629	{
630	struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
631
632	memset(tcw->iv_seed, 0, cc->iv_size);
633	memset(tcw->whitening, 0, TCW_WHITENING_SIZE);
634
635	return 0;
636	}
637
638	static int crypt_iv_tcw_whitening(struct crypt_config *cc,
639	struct dm_crypt_request *dmreq,
640	u8 *data)
641	{
642	struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
643	__le64 sector = cpu_to_le64(dmreq->iv_sector);
644	u8 buf[TCW_WHITENING_SIZE];
645	SHASH_DESC_ON_STACK(desc, tcw->crc32_tfm);
646	int i, r;
647
648	/* xor whitening with sector number */
649	crypto_xor_cpy(dst: buf, src1: tcw->whitening, src2: (u8 *)&sector, size: 8);
650	crypto_xor_cpy(dst: &buf[8], src1: tcw->whitening + 8, src2: (u8 *)&sector, size: 8);
651
652	/* calculate crc32 for every 32bit part and xor it */
653	desc->tfm = tcw->crc32_tfm;
654	for (i = 0; i < 4; i++) {
655	r = crypto_shash_digest(desc, data: &buf[i * 4], len: 4, out: &buf[i * 4]);
656	if (r)
657	goto out;
658	}
659	crypto_xor(dst: &buf[0], src: &buf[12], size: 4);
660	crypto_xor(dst: &buf[4], src: &buf[8], size: 4);
661
662	/* apply whitening (8 bytes) to whole sector */
663	for (i = 0; i < ((1 << SECTOR_SHIFT) / 8); i++)
664	crypto_xor(dst: data + i * 8, src: buf, size: 8);
665	out:
666	memzero_explicit(s: buf, count: sizeof(buf));
667	return r;
668	}
669
670	static int crypt_iv_tcw_gen(struct crypt_config *cc, u8 *iv,
671	struct dm_crypt_request *dmreq)
672	{
673	struct scatterlist *sg;
674	struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
675	__le64 sector = cpu_to_le64(dmreq->iv_sector);
676	u8 *src;
677	int r = 0;
678
679	/* Remove whitening from ciphertext */
680	if (bio_data_dir(dmreq->ctx->bio_in) != WRITE) {
681	sg = crypt_get_sg_data(cc, sg: dmreq->sg_in);
682	src = kmap_local_page(page: sg_page(sg));
683	r = crypt_iv_tcw_whitening(cc, dmreq, data: src + sg->offset);
684	kunmap_local(src);
685	}
686
687	/* Calculate IV */
688	crypto_xor_cpy(dst: iv, src1: tcw->iv_seed, src2: (u8 *)&sector, size: 8);
689	if (cc->iv_size > 8)
690	crypto_xor_cpy(dst: &iv[8], src1: tcw->iv_seed + 8, src2: (u8 *)&sector,
691	size: cc->iv_size - 8);
692
693	return r;
694	}
695
696	static int crypt_iv_tcw_post(struct crypt_config *cc, u8 *iv,
697	struct dm_crypt_request *dmreq)
698	{
699	struct scatterlist *sg;
700	u8 *dst;
701	int r;
702
703	if (bio_data_dir(dmreq->ctx->bio_in) != WRITE)
704	return 0;
705
706	/* Apply whitening on ciphertext */
707	sg = crypt_get_sg_data(cc, sg: dmreq->sg_out);
708	dst = kmap_local_page(page: sg_page(sg));
709	r = crypt_iv_tcw_whitening(cc, dmreq, data: dst + sg->offset);
710	kunmap_local(dst);
711
712	return r;
713	}
714
715	static int crypt_iv_random_gen(struct crypt_config *cc, u8 *iv,
716	struct dm_crypt_request *dmreq)
717	{
718	/* Used only for writes, there must be an additional space to store IV */
719	get_random_bytes(buf: iv, len: cc->iv_size);
720	return 0;
721	}
722
723	static int crypt_iv_eboiv_ctr(struct crypt_config *cc, struct dm_target *ti,
724	const char *opts)
725	{
726	if (crypt_integrity_aead(cc)) {
727	ti->error = "AEAD transforms not supported for EBOIV";
728	return -EINVAL;
729	}
730
731	if (crypto_skcipher_blocksize(tfm: any_tfm(cc)) != cc->iv_size) {
732	ti->error = "Block size of EBOIV cipher does not match IV size of block cipher";
733	return -EINVAL;
734	}
735
736	return 0;
737	}
738
739	static int crypt_iv_eboiv_gen(struct crypt_config *cc, u8 *iv,
740	struct dm_crypt_request *dmreq)
741	{
742	struct crypto_skcipher *tfm = any_tfm(cc);
743	struct skcipher_request *req;
744	struct scatterlist src, dst;
745	DECLARE_CRYPTO_WAIT(wait);
746	unsigned int reqsize;
747	int err;
748	u8 *buf;
749
750	reqsize = sizeof(*req) + crypto_skcipher_reqsize(tfm);
751	reqsize = ALIGN(reqsize, __alignof__(__le64));
752
753	req = kmalloc(size: reqsize + cc->iv_size, GFP_NOIO);
754	if (!req)
755	return -ENOMEM;
756
757	skcipher_request_set_tfm(req, tfm);
758
759	buf = (u8 *)req + reqsize;
760	memset(buf, 0, cc->iv_size);
761	*(__le64 *)buf = cpu_to_le64(dmreq->iv_sector * cc->sector_size);
762
763	sg_init_one(&src, page_address(ZERO_PAGE(0)), cc->iv_size);
764	sg_init_one(&dst, iv, cc->iv_size);
765	skcipher_request_set_crypt(req, src: &src, dst: &dst, cryptlen: cc->iv_size, iv: buf);
766	skcipher_request_set_callback(req, flags: 0, compl: crypto_req_done, data: &wait);
767	err = crypto_wait_req(err: crypto_skcipher_encrypt(req), wait: &wait);
768	kfree_sensitive(objp: req);
769
770	return err;
771	}
772
773	static void crypt_iv_elephant_dtr(struct crypt_config *cc)
774	{
775	struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant;
776
777	crypto_free_skcipher(tfm: elephant->tfm);
778	elephant->tfm = NULL;
779	}
780
781	static int crypt_iv_elephant_ctr(struct crypt_config *cc, struct dm_target *ti,
782	const char *opts)
783	{
784	struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant;
785	int r;
786
787	elephant->tfm = crypto_alloc_skcipher(alg_name: "ecb(aes)", type: 0,
788	CRYPTO_ALG_ALLOCATES_MEMORY);
789	if (IS_ERR(ptr: elephant->tfm)) {
790	r = PTR_ERR(ptr: elephant->tfm);
791	elephant->tfm = NULL;
792	return r;
793	}
794
795	r = crypt_iv_eboiv_ctr(cc, ti, NULL);
796	if (r)
797	crypt_iv_elephant_dtr(cc);
798	return r;
799	}
800
801	static void diffuser_disk_to_cpu(u32 *d, size_t n)
802	{
803	#ifndef __LITTLE_ENDIAN
804	int i;
805
806	for (i = 0; i < n; i++)
807	d[i] = le32_to_cpu((__le32)d[i]);
808	#endif
809	}
810
811	static void diffuser_cpu_to_disk(__le32 *d, size_t n)
812	{
813	#ifndef __LITTLE_ENDIAN
814	int i;
815
816	for (i = 0; i < n; i++)
817	d[i] = cpu_to_le32((u32)d[i]);
818	#endif
819	}
820
821	static void diffuser_a_decrypt(u32 *d, size_t n)
822	{
823	int i, i1, i2, i3;
824
825	for (i = 0; i < 5; i++) {
826	i1 = 0;
827	i2 = n - 2;
828	i3 = n - 5;
829
830	while (i1 < (n - 1)) {
831	d[i1] += d[i2] ^ (d[i3] << 9 | d[i3] >> 23);
832	i1++; i2++; i3++;
833
834	if (i3 >= n)
835	i3 -= n;
836
837	d[i1] += d[i2] ^ d[i3];
838	i1++; i2++; i3++;
839
840	if (i2 >= n)
841	i2 -= n;
842
843	d[i1] += d[i2] ^ (d[i3] << 13 | d[i3] >> 19);
844	i1++; i2++; i3++;
845
846	d[i1] += d[i2] ^ d[i3];
847	i1++; i2++; i3++;
848	}
849	}
850	}
851
852	static void diffuser_a_encrypt(u32 *d, size_t n)
853	{
854	int i, i1, i2, i3;
855
856	for (i = 0; i < 5; i++) {
857	i1 = n - 1;
858	i2 = n - 2 - 1;
859	i3 = n - 5 - 1;
860
861	while (i1 > 0) {
862	d[i1] -= d[i2] ^ d[i3];
863	i1--; i2--; i3--;
864
865	d[i1] -= d[i2] ^ (d[i3] << 13 | d[i3] >> 19);
866	i1--; i2--; i3--;
867
868	if (i2 < 0)
869	i2 += n;
870
871	d[i1] -= d[i2] ^ d[i3];
872	i1--; i2--; i3--;
873
874	if (i3 < 0)
875	i3 += n;
876
877	d[i1] -= d[i2] ^ (d[i3] << 9 | d[i3] >> 23);
878	i1--; i2--; i3--;
879	}
880	}
881	}
882
883	static void diffuser_b_decrypt(u32 *d, size_t n)
884	{
885	int i, i1, i2, i3;
886
887	for (i = 0; i < 3; i++) {
888	i1 = 0;
889	i2 = 2;
890	i3 = 5;
891
892	while (i1 < (n - 1)) {
893	d[i1] += d[i2] ^ d[i3];
894	i1++; i2++; i3++;
895
896	d[i1] += d[i2] ^ (d[i3] << 10 | d[i3] >> 22);
897	i1++; i2++; i3++;
898
899	if (i2 >= n)
900	i2 -= n;
901
902	d[i1] += d[i2] ^ d[i3];
903	i1++; i2++; i3++;
904
905	if (i3 >= n)
906	i3 -= n;
907
908	d[i1] += d[i2] ^ (d[i3] << 25 | d[i3] >> 7);
909	i1++; i2++; i3++;
910	}
911	}
912	}
913
914	static void diffuser_b_encrypt(u32 *d, size_t n)
915	{
916	int i, i1, i2, i3;
917
918	for (i = 0; i < 3; i++) {
919	i1 = n - 1;
920	i2 = 2 - 1;
921	i3 = 5 - 1;
922
923	while (i1 > 0) {
924	d[i1] -= d[i2] ^ (d[i3] << 25 | d[i3] >> 7);
925	i1--; i2--; i3--;
926
927	if (i3 < 0)
928	i3 += n;
929
930	d[i1] -= d[i2] ^ d[i3];
931	i1--; i2--; i3--;
932
933	if (i2 < 0)
934	i2 += n;
935
936	d[i1] -= d[i2] ^ (d[i3] << 10 | d[i3] >> 22);
937	i1--; i2--; i3--;
938
939	d[i1] -= d[i2] ^ d[i3];
940	i1--; i2--; i3--;
941	}
942	}
943	}
944
945	static int crypt_iv_elephant(struct crypt_config *cc, struct dm_crypt_request *dmreq)
946	{
947	struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant;
948	u8 *es, *ks, *data, *data2, *data_offset;
949	struct skcipher_request *req;
950	struct scatterlist *sg, *sg2, src, dst;
951	DECLARE_CRYPTO_WAIT(wait);
952	int i, r;
953
954	req = skcipher_request_alloc(tfm: elephant->tfm, GFP_NOIO);
955	es = kzalloc(size: 16, GFP_NOIO); /* Key for AES */
956	ks = kzalloc(size: 32, GFP_NOIO); /* Elephant sector key */
957
958	if (!req || !es || !ks) {
959	r = -ENOMEM;
960	goto out;
961	}
962
963	*(__le64 *)es = cpu_to_le64(dmreq->iv_sector * cc->sector_size);
964
965	/* E(Ks, e(s)) */
966	sg_init_one(&src, es, 16);
967	sg_init_one(&dst, ks, 16);
968	skcipher_request_set_crypt(req, src: &src, dst: &dst, cryptlen: 16, NULL);
969	skcipher_request_set_callback(req, flags: 0, compl: crypto_req_done, data: &wait);
970	r = crypto_wait_req(err: crypto_skcipher_encrypt(req), wait: &wait);
971	if (r)
972	goto out;
973
974	/* E(Ks, e'(s)) */
975	es[15] = 0x80;
976	sg_init_one(&dst, &ks[16], 16);
977	r = crypto_wait_req(err: crypto_skcipher_encrypt(req), wait: &wait);
978	if (r)
979	goto out;
980
981	sg = crypt_get_sg_data(cc, sg: dmreq->sg_out);
982	data = kmap_local_page(page: sg_page(sg));
983	data_offset = data + sg->offset;
984
985	/* Cannot modify original bio, copy to sg_out and apply Elephant to it */
986	if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {
987	sg2 = crypt_get_sg_data(cc, sg: dmreq->sg_in);
988	data2 = kmap_local_page(page: sg_page(sg: sg2));
989	memcpy(data_offset, data2 + sg2->offset, cc->sector_size);
990	kunmap_local(data2);
991	}
992
993	if (bio_data_dir(dmreq->ctx->bio_in) != WRITE) {
994	diffuser_disk_to_cpu(d: (u32 *)data_offset, n: cc->sector_size / sizeof(u32));
995	diffuser_b_decrypt(d: (u32 *)data_offset, n: cc->sector_size / sizeof(u32));
996	diffuser_a_decrypt(d: (u32 *)data_offset, n: cc->sector_size / sizeof(u32));
997	diffuser_cpu_to_disk(d: (__le32 *)data_offset, n: cc->sector_size / sizeof(u32));
998	}
999
1000	for (i = 0; i < (cc->sector_size / 32); i++)
1001	crypto_xor(dst: data_offset + i * 32, src: ks, size: 32);
1002
1003	if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {
1004	diffuser_disk_to_cpu(d: (u32 *)data_offset, n: cc->sector_size / sizeof(u32));
1005	diffuser_a_encrypt(d: (u32 *)data_offset, n: cc->sector_size / sizeof(u32));
1006	diffuser_b_encrypt(d: (u32 *)data_offset, n: cc->sector_size / sizeof(u32));
1007	diffuser_cpu_to_disk(d: (__le32 *)data_offset, n: cc->sector_size / sizeof(u32));
1008	}
1009
1010	kunmap_local(data);
1011	out:
1012	kfree_sensitive(objp: ks);
1013	kfree_sensitive(objp: es);
1014	skcipher_request_free(req);
1015	return r;
1016	}
1017
1018	static int crypt_iv_elephant_gen(struct crypt_config *cc, u8 *iv,
1019	struct dm_crypt_request *dmreq)
1020	{
1021	int r;
1022
1023	if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {
1024	r = crypt_iv_elephant(cc, dmreq);
1025	if (r)
1026	return r;
1027	}
1028
1029	return crypt_iv_eboiv_gen(cc, iv, dmreq);
1030	}
1031
1032	static int crypt_iv_elephant_post(struct crypt_config *cc, u8 *iv,
1033	struct dm_crypt_request *dmreq)
1034	{
1035	if (bio_data_dir(dmreq->ctx->bio_in) != WRITE)
1036	return crypt_iv_elephant(cc, dmreq);
1037
1038	return 0;
1039	}
1040
1041	static int crypt_iv_elephant_init(struct crypt_config *cc)
1042	{
1043	struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant;
1044	int key_offset = cc->key_size - cc->key_extra_size;
1045
1046	return crypto_skcipher_setkey(tfm: elephant->tfm, key: &cc->key[key_offset], keylen: cc->key_extra_size);
1047	}
1048
1049	static int crypt_iv_elephant_wipe(struct crypt_config *cc)
1050	{
1051	struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant;
1052	u8 key[ELEPHANT_MAX_KEY_SIZE];
1053
1054	memset(key, 0, cc->key_extra_size);
1055	return crypto_skcipher_setkey(tfm: elephant->tfm, key, keylen: cc->key_extra_size);
1056	}
1057
1058	static const struct crypt_iv_operations crypt_iv_plain_ops = {
1059	.generator = crypt_iv_plain_gen
1060	};
1061
1062	static const struct crypt_iv_operations crypt_iv_plain64_ops = {
1063	.generator = crypt_iv_plain64_gen
1064	};
1065
1066	static const struct crypt_iv_operations crypt_iv_plain64be_ops = {
1067	.generator = crypt_iv_plain64be_gen
1068	};
1069
1070	static const struct crypt_iv_operations crypt_iv_essiv_ops = {
1071	.generator = crypt_iv_essiv_gen
1072	};
1073
1074	static const struct crypt_iv_operations crypt_iv_benbi_ops = {
1075	.ctr = crypt_iv_benbi_ctr,
1076	.dtr = crypt_iv_benbi_dtr,
1077	.generator = crypt_iv_benbi_gen
1078	};
1079
1080	static const struct crypt_iv_operations crypt_iv_null_ops = {
1081	.generator = crypt_iv_null_gen
1082	};
1083
1084	static const struct crypt_iv_operations crypt_iv_lmk_ops = {
1085	.ctr = crypt_iv_lmk_ctr,
1086	.dtr = crypt_iv_lmk_dtr,
1087	.init = crypt_iv_lmk_init,
1088	.wipe = crypt_iv_lmk_wipe,
1089	.generator = crypt_iv_lmk_gen,
1090	.post = crypt_iv_lmk_post
1091	};
1092
1093	static const struct crypt_iv_operations crypt_iv_tcw_ops = {
1094	.ctr = crypt_iv_tcw_ctr,
1095	.dtr = crypt_iv_tcw_dtr,
1096	.init = crypt_iv_tcw_init,
1097	.wipe = crypt_iv_tcw_wipe,
1098	.generator = crypt_iv_tcw_gen,
1099	.post = crypt_iv_tcw_post
1100	};
1101
1102	static const struct crypt_iv_operations crypt_iv_random_ops = {
1103	.generator = crypt_iv_random_gen
1104	};
1105
1106	static const struct crypt_iv_operations crypt_iv_eboiv_ops = {
1107	.ctr = crypt_iv_eboiv_ctr,
1108	.generator = crypt_iv_eboiv_gen
1109	};
1110
1111	static const struct crypt_iv_operations crypt_iv_elephant_ops = {
1112	.ctr = crypt_iv_elephant_ctr,
1113	.dtr = crypt_iv_elephant_dtr,
1114	.init = crypt_iv_elephant_init,
1115	.wipe = crypt_iv_elephant_wipe,
1116	.generator = crypt_iv_elephant_gen,
1117	.post = crypt_iv_elephant_post
1118	};
1119
1120	/*
1121	* Integrity extensions
1122	*/
1123	static bool crypt_integrity_aead(struct crypt_config *cc)
1124	{
1125	return test_bit(CRYPT_MODE_INTEGRITY_AEAD, &cc->cipher_flags);
1126	}
1127
1128	static bool crypt_integrity_hmac(struct crypt_config *cc)
1129	{
1130	return crypt_integrity_aead(cc) && cc->key_mac_size;
1131	}
1132
1133	/* Get sg containing data */
1134	static struct scatterlist *crypt_get_sg_data(struct crypt_config *cc,
1135	struct scatterlist *sg)
1136	{
1137	if (unlikely(crypt_integrity_aead(cc)))
1138	return &sg[2];
1139
1140	return sg;
1141	}
1142
1143	static int dm_crypt_integrity_io_alloc(struct dm_crypt_io *io, struct bio *bio)
1144	{
1145	struct bio_integrity_payload *bip;
1146	unsigned int tag_len;
1147	int ret;
1148
1149	if (!bio_sectors(bio) || !io->cc->on_disk_tag_size)
1150	return 0;
1151
1152	bip = bio_integrity_alloc(bio, GFP_NOIO, 1);
1153	if (IS_ERR(ptr: bip))
1154	return PTR_ERR(ptr: bip);
1155
1156	tag_len = io->cc->on_disk_tag_size * (bio_sectors(bio) >> io->cc->sector_shift);
1157
1158	bip->bip_iter.bi_sector = io->cc->start + io->sector;
1159
1160	ret = bio_integrity_add_page(bio, virt_to_page(io->integrity_metadata),
1161	tag_len, offset_in_page(io->integrity_metadata));
1162	if (unlikely(ret != tag_len))
1163	return -ENOMEM;
1164
1165	return 0;
1166	}
1167
1168	static int crypt_integrity_ctr(struct crypt_config *cc, struct dm_target *ti)
1169	{
1170	#ifdef CONFIG_BLK_DEV_INTEGRITY
1171	struct blk_integrity *bi = blk_get_integrity(disk: cc->dev->bdev->bd_disk);
1172	struct mapped_device *md = dm_table_get_md(t: ti->table);
1173
1174	/* From now we require underlying device with our integrity profile */
1175	if (!bi || strcasecmp(s1: bi->profile->name, s2: "DM-DIF-EXT-TAG")) {
1176	ti->error = "Integrity profile not supported.";
1177	return -EINVAL;
1178	}
1179
1180	if (bi->tag_size != cc->on_disk_tag_size ||
1181	bi->tuple_size != cc->on_disk_tag_size) {
1182	ti->error = "Integrity profile tag size mismatch.";
1183	return -EINVAL;
1184	}
1185	if (1 << bi->interval_exp != cc->sector_size) {
1186	ti->error = "Integrity profile sector size mismatch.";
1187	return -EINVAL;
1188	}
1189
1190	if (crypt_integrity_aead(cc)) {
1191	cc->integrity_tag_size = cc->on_disk_tag_size - cc->integrity_iv_size;
1192	DMDEBUG("%s: Integrity AEAD, tag size %u, IV size %u.", dm_device_name(md),
1193	cc->integrity_tag_size, cc->integrity_iv_size);
1194
1195	if (crypto_aead_setauthsize(tfm: any_tfm_aead(cc), authsize: cc->integrity_tag_size)) {
1196	ti->error = "Integrity AEAD auth tag size is not supported.";
1197	return -EINVAL;
1198	}
1199	} else if (cc->integrity_iv_size)
1200	DMDEBUG("%s: Additional per-sector space %u bytes for IV.", dm_device_name(md),
1201	cc->integrity_iv_size);
1202
1203	if ((cc->integrity_tag_size + cc->integrity_iv_size) != bi->tag_size) {
1204	ti->error = "Not enough space for integrity tag in the profile.";
1205	return -EINVAL;
1206	}
1207
1208	return 0;
1209	#else
1210	ti->error = "Integrity profile not supported.";
1211	return -EINVAL;
1212	#endif
1213	}
1214
1215	static void crypt_convert_init(struct crypt_config *cc,
1216	struct convert_context *ctx,
1217	struct bio *bio_out, struct bio *bio_in,
1218	sector_t sector)
1219	{
1220	ctx->bio_in = bio_in;
1221	ctx->bio_out = bio_out;
1222	if (bio_in)
1223	ctx->iter_in = bio_in->bi_iter;
1224	if (bio_out)
1225	ctx->iter_out = bio_out->bi_iter;
1226	ctx->cc_sector = sector + cc->iv_offset;
1227	init_completion(x: &ctx->restart);
1228	}
1229
1230	static struct dm_crypt_request *dmreq_of_req(struct crypt_config *cc,
1231	void *req)
1232	{
1233	return (struct dm_crypt_request *)((char *)req + cc->dmreq_start);
1234	}
1235
1236	static void *req_of_dmreq(struct crypt_config *cc, struct dm_crypt_request *dmreq)
1237	{
1238	return (void *)((char *)dmreq - cc->dmreq_start);
1239	}
1240
1241	static u8 *iv_of_dmreq(struct crypt_config *cc,
1242	struct dm_crypt_request *dmreq)
1243	{
1244	if (crypt_integrity_aead(cc))
1245	return (u8 *)ALIGN((unsigned long)(dmreq + 1),
1246	crypto_aead_alignmask(any_tfm_aead(cc)) + 1);
1247	else
1248	return (u8 *)ALIGN((unsigned long)(dmreq + 1),
1249	crypto_skcipher_alignmask(any_tfm(cc)) + 1);
1250	}
1251
1252	static u8 *org_iv_of_dmreq(struct crypt_config *cc,
1253	struct dm_crypt_request *dmreq)
1254	{
1255	return iv_of_dmreq(cc, dmreq) + cc->iv_size;
1256	}
1257
1258	static __le64 *org_sector_of_dmreq(struct crypt_config *cc,
1259	struct dm_crypt_request *dmreq)
1260	{
1261	u8 *ptr = iv_of_dmreq(cc, dmreq) + cc->iv_size + cc->iv_size;
1262
1263	return (__le64 *) ptr;
1264	}
1265
1266	static unsigned int *org_tag_of_dmreq(struct crypt_config *cc,
1267	struct dm_crypt_request *dmreq)
1268	{
1269	u8 *ptr = iv_of_dmreq(cc, dmreq) + cc->iv_size +
1270	cc->iv_size + sizeof(uint64_t);
1271
1272	return (unsigned int *)ptr;
1273	}
1274
1275	static void *tag_from_dmreq(struct crypt_config *cc,
1276	struct dm_crypt_request *dmreq)
1277	{
1278	struct convert_context *ctx = dmreq->ctx;
1279	struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx);
1280
1281	return &io->integrity_metadata[*org_tag_of_dmreq(cc, dmreq) *
1282	cc->on_disk_tag_size];
1283	}
1284
1285	static void *iv_tag_from_dmreq(struct crypt_config *cc,
1286	struct dm_crypt_request *dmreq)
1287	{
1288	return tag_from_dmreq(cc, dmreq) + cc->integrity_tag_size;
1289	}
1290
1291	static int crypt_convert_block_aead(struct crypt_config *cc,
1292	struct convert_context *ctx,
1293	struct aead_request *req,
1294	unsigned int tag_offset)
1295	{
1296	struct bio_vec bv_in = bio_iter_iovec(ctx->bio_in, ctx->iter_in);
1297	struct bio_vec bv_out = bio_iter_iovec(ctx->bio_out, ctx->iter_out);
1298	struct dm_crypt_request *dmreq;
1299	u8 *iv, *org_iv, *tag_iv, *tag;
1300	__le64 *sector;
1301	int r = 0;
1302
1303	BUG_ON(cc->integrity_iv_size && cc->integrity_iv_size != cc->iv_size);
1304
1305	/* Reject unexpected unaligned bio. */
1306	if (unlikely(bv_in.bv_len & (cc->sector_size - 1)))
1307	return -EIO;
1308
1309	dmreq = dmreq_of_req(cc, req);
1310	dmreq->iv_sector = ctx->cc_sector;
1311	if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags))
1312	dmreq->iv_sector >>= cc->sector_shift;
1313	dmreq->ctx = ctx;
1314
1315	*org_tag_of_dmreq(cc, dmreq) = tag_offset;
1316
1317	sector = org_sector_of_dmreq(cc, dmreq);
1318	*sector = cpu_to_le64(ctx->cc_sector - cc->iv_offset);
1319
1320	iv = iv_of_dmreq(cc, dmreq);
1321	org_iv = org_iv_of_dmreq(cc, dmreq);
1322	tag = tag_from_dmreq(cc, dmreq);
1323	tag_iv = iv_tag_from_dmreq(cc, dmreq);
1324
1325	/* AEAD request:
1326	* |----- AAD -------|------ DATA -------|-- AUTH TAG --|
1327	* | (authenticated) | (auth+encryption) | |
1328	* | sector_LE | IV | sector in/out | tag in/out |
1329	*/
1330	sg_init_table(dmreq->sg_in, 4);
1331	sg_set_buf(sg: &dmreq->sg_in[0], buf: sector, buflen: sizeof(uint64_t));
1332	sg_set_buf(sg: &dmreq->sg_in[1], buf: org_iv, buflen: cc->iv_size);
1333	sg_set_page(sg: &dmreq->sg_in[2], page: bv_in.bv_page, len: cc->sector_size, offset: bv_in.bv_offset);
1334	sg_set_buf(sg: &dmreq->sg_in[3], buf: tag, buflen: cc->integrity_tag_size);
1335
1336	sg_init_table(dmreq->sg_out, 4);
1337	sg_set_buf(sg: &dmreq->sg_out[0], buf: sector, buflen: sizeof(uint64_t));
1338	sg_set_buf(sg: &dmreq->sg_out[1], buf: org_iv, buflen: cc->iv_size);
1339	sg_set_page(sg: &dmreq->sg_out[2], page: bv_out.bv_page, len: cc->sector_size, offset: bv_out.bv_offset);
1340	sg_set_buf(sg: &dmreq->sg_out[3], buf: tag, buflen: cc->integrity_tag_size);
1341
1342	if (cc->iv_gen_ops) {
1343	/* For READs use IV stored in integrity metadata */
1344	if (cc->integrity_iv_size && bio_data_dir(ctx->bio_in) != WRITE) {
1345	memcpy(org_iv, tag_iv, cc->iv_size);
1346	} else {
1347	r = cc->iv_gen_ops->generator(cc, org_iv, dmreq);
1348	if (r < 0)
1349	return r;
1350	/* Store generated IV in integrity metadata */
1351	if (cc->integrity_iv_size)
1352	memcpy(tag_iv, org_iv, cc->iv_size);
1353	}
1354	/* Working copy of IV, to be modified in crypto API */
1355	memcpy(iv, org_iv, cc->iv_size);
1356	}
1357
1358	aead_request_set_ad(req, assoclen: sizeof(uint64_t) + cc->iv_size);
1359	if (bio_data_dir(ctx->bio_in) == WRITE) {
1360	aead_request_set_crypt(req, src: dmreq->sg_in, dst: dmreq->sg_out,
1361	cryptlen: cc->sector_size, iv);
1362	r = crypto_aead_encrypt(req);
1363	if (cc->integrity_tag_size + cc->integrity_iv_size != cc->on_disk_tag_size)
1364	memset(tag + cc->integrity_tag_size + cc->integrity_iv_size, 0,
1365	cc->on_disk_tag_size - (cc->integrity_tag_size + cc->integrity_iv_size));
1366	} else {
1367	aead_request_set_crypt(req, src: dmreq->sg_in, dst: dmreq->sg_out,
1368	cryptlen: cc->sector_size + cc->integrity_tag_size, iv);
1369	r = crypto_aead_decrypt(req);
1370	}
1371
1372	if (r == -EBADMSG) {
1373	sector_t s = le64_to_cpu(*sector);
1374
1375	DMERR_LIMIT("%pg: INTEGRITY AEAD ERROR, sector %llu",
1376	ctx->bio_in->bi_bdev, s);
1377	dm_audit_log_bio(DM_MSG_PREFIX, op: "integrity-aead",
1378	bio: ctx->bio_in, sector: s, result: 0);
1379	}
1380
1381	if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post)
1382	r = cc->iv_gen_ops->post(cc, org_iv, dmreq);
1383
1384	bio_advance_iter(bio: ctx->bio_in, iter: &ctx->iter_in, bytes: cc->sector_size);
1385	bio_advance_iter(bio: ctx->bio_out, iter: &ctx->iter_out, bytes: cc->sector_size);
1386
1387	return r;
1388	}
1389
1390	static int crypt_convert_block_skcipher(struct crypt_config *cc,
1391	struct convert_context *ctx,
1392	struct skcipher_request *req,
1393	unsigned int tag_offset)
1394	{
1395	struct bio_vec bv_in = bio_iter_iovec(ctx->bio_in, ctx->iter_in);
1396	struct bio_vec bv_out = bio_iter_iovec(ctx->bio_out, ctx->iter_out);
1397	struct scatterlist *sg_in, *sg_out;
1398	struct dm_crypt_request *dmreq;
1399	u8 *iv, *org_iv, *tag_iv;
1400	__le64 *sector;
1401	int r = 0;
1402
1403	/* Reject unexpected unaligned bio. */
1404	if (unlikely(bv_in.bv_len & (cc->sector_size - 1)))
1405	return -EIO;
1406
1407	dmreq = dmreq_of_req(cc, req);
1408	dmreq->iv_sector = ctx->cc_sector;
1409	if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags))
1410	dmreq->iv_sector >>= cc->sector_shift;
1411	dmreq->ctx = ctx;
1412
1413	*org_tag_of_dmreq(cc, dmreq) = tag_offset;
1414
1415	iv = iv_of_dmreq(cc, dmreq);
1416	org_iv = org_iv_of_dmreq(cc, dmreq);
1417	tag_iv = iv_tag_from_dmreq(cc, dmreq);
1418
1419	sector = org_sector_of_dmreq(cc, dmreq);
1420	*sector = cpu_to_le64(ctx->cc_sector - cc->iv_offset);
1421
1422	/* For skcipher we use only the first sg item */
1423	sg_in = &dmreq->sg_in[0];
1424	sg_out = &dmreq->sg_out[0];
1425
1426	sg_init_table(sg_in, 1);
1427	sg_set_page(sg: sg_in, page: bv_in.bv_page, len: cc->sector_size, offset: bv_in.bv_offset);
1428
1429	sg_init_table(sg_out, 1);
1430	sg_set_page(sg: sg_out, page: bv_out.bv_page, len: cc->sector_size, offset: bv_out.bv_offset);
1431
1432	if (cc->iv_gen_ops) {
1433	/* For READs use IV stored in integrity metadata */
1434	if (cc->integrity_iv_size && bio_data_dir(ctx->bio_in) != WRITE) {
1435	memcpy(org_iv, tag_iv, cc->integrity_iv_size);
1436	} else {
1437	r = cc->iv_gen_ops->generator(cc, org_iv, dmreq);
1438	if (r < 0)
1439	return r;
1440	/* Data can be already preprocessed in generator */
1441	if (test_bit(CRYPT_ENCRYPT_PREPROCESS, &cc->cipher_flags))
1442	sg_in = sg_out;
1443	/* Store generated IV in integrity metadata */
1444	if (cc->integrity_iv_size)
1445	memcpy(tag_iv, org_iv, cc->integrity_iv_size);
1446	}
1447	/* Working copy of IV, to be modified in crypto API */
1448	memcpy(iv, org_iv, cc->iv_size);
1449	}
1450
1451	skcipher_request_set_crypt(req, src: sg_in, dst: sg_out, cryptlen: cc->sector_size, iv);
1452
1453	if (bio_data_dir(ctx->bio_in) == WRITE)
1454	r = crypto_skcipher_encrypt(req);
1455	else
1456	r = crypto_skcipher_decrypt(req);
1457
1458	if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post)
1459	r = cc->iv_gen_ops->post(cc, org_iv, dmreq);
1460
1461	bio_advance_iter(bio: ctx->bio_in, iter: &ctx->iter_in, bytes: cc->sector_size);
1462	bio_advance_iter(bio: ctx->bio_out, iter: &ctx->iter_out, bytes: cc->sector_size);
1463
1464	return r;
1465	}
1466
1467	static void kcryptd_async_done(void *async_req, int error);
1468
1469	static int crypt_alloc_req_skcipher(struct crypt_config *cc,
1470	struct convert_context *ctx)
1471	{
1472	unsigned int key_index = ctx->cc_sector & (cc->tfms_count - 1);
1473
1474	if (!ctx->r.req) {
1475	ctx->r.req = mempool_alloc(pool: &cc->req_pool, in_interrupt() ? GFP_ATOMIC : GFP_NOIO);
1476	if (!ctx->r.req)
1477	return -ENOMEM;
1478	}
1479
1480	skcipher_request_set_tfm(req: ctx->r.req, tfm: cc->cipher_tfm.tfms[key_index]);
1481
1482	/*
1483	* Use REQ_MAY_BACKLOG so a cipher driver internally backlogs
1484	* requests if driver request queue is full.
1485	*/
1486	skcipher_request_set_callback(req: ctx->r.req,
1487	CRYPTO_TFM_REQ_MAY_BACKLOG,
1488	compl: kcryptd_async_done, data: dmreq_of_req(cc, req: ctx->r.req));
1489
1490	return 0;
1491	}
1492
1493	static int crypt_alloc_req_aead(struct crypt_config *cc,
1494	struct convert_context *ctx)
1495	{
1496	if (!ctx->r.req_aead) {
1497	ctx->r.req_aead = mempool_alloc(pool: &cc->req_pool, in_interrupt() ? GFP_ATOMIC : GFP_NOIO);
1498	if (!ctx->r.req_aead)
1499	return -ENOMEM;
1500	}
1501
1502	aead_request_set_tfm(req: ctx->r.req_aead, tfm: cc->cipher_tfm.tfms_aead[0]);
1503
1504	/*
1505	* Use REQ_MAY_BACKLOG so a cipher driver internally backlogs
1506	* requests if driver request queue is full.
1507	*/
1508	aead_request_set_callback(req: ctx->r.req_aead,
1509	CRYPTO_TFM_REQ_MAY_BACKLOG,
1510	compl: kcryptd_async_done, data: dmreq_of_req(cc, req: ctx->r.req_aead));
1511
1512	return 0;
1513	}
1514
1515	static int crypt_alloc_req(struct crypt_config *cc,
1516	struct convert_context *ctx)
1517	{
1518	if (crypt_integrity_aead(cc))
1519	return crypt_alloc_req_aead(cc, ctx);
1520	else
1521	return crypt_alloc_req_skcipher(cc, ctx);
1522	}
1523
1524	static void crypt_free_req_skcipher(struct crypt_config *cc,
1525	struct skcipher_request *req, struct bio *base_bio)
1526	{
1527	struct dm_crypt_io *io = dm_per_bio_data(bio: base_bio, data_size: cc->per_bio_data_size);
1528
1529	if ((struct skcipher_request *)(io + 1) != req)
1530	mempool_free(element: req, pool: &cc->req_pool);
1531	}
1532
1533	static void crypt_free_req_aead(struct crypt_config *cc,
1534	struct aead_request *req, struct bio *base_bio)
1535	{
1536	struct dm_crypt_io *io = dm_per_bio_data(bio: base_bio, data_size: cc->per_bio_data_size);
1537
1538	if ((struct aead_request *)(io + 1) != req)
1539	mempool_free(element: req, pool: &cc->req_pool);
1540	}
1541
1542	static void crypt_free_req(struct crypt_config *cc, void *req, struct bio *base_bio)
1543	{
1544	if (crypt_integrity_aead(cc))
1545	crypt_free_req_aead(cc, req, base_bio);
1546	else
1547	crypt_free_req_skcipher(cc, req, base_bio);
1548	}
1549
1550	/*
1551	* Encrypt / decrypt data from one bio to another one (can be the same one)
1552	*/
1553	static blk_status_t crypt_convert(struct crypt_config *cc,
1554	struct convert_context *ctx, bool atomic, bool reset_pending)
1555	{
1556	unsigned int tag_offset = 0;
1557	unsigned int sector_step = cc->sector_size >> SECTOR_SHIFT;
1558	int r;
1559
1560	/*
1561	* if reset_pending is set we are dealing with the bio for the first time,
1562	* else we're continuing to work on the previous bio, so don't mess with
1563	* the cc_pending counter
1564	*/
1565	if (reset_pending)
1566	atomic_set(v: &ctx->cc_pending, i: 1);
1567
1568	while (ctx->iter_in.bi_size && ctx->iter_out.bi_size) {
1569
1570	r = crypt_alloc_req(cc, ctx);
1571	if (r) {
1572	complete(&ctx->restart);
1573	return BLK_STS_DEV_RESOURCE;
1574	}
1575
1576	atomic_inc(v: &ctx->cc_pending);
1577
1578	if (crypt_integrity_aead(cc))
1579	r = crypt_convert_block_aead(cc, ctx, req: ctx->r.req_aead, tag_offset);
1580	else
1581	r = crypt_convert_block_skcipher(cc, ctx, req: ctx->r.req, tag_offset);
1582
1583	switch (r) {
1584	/*
1585	* The request was queued by a crypto driver
1586	* but the driver request queue is full, let's wait.
1587	*/
1588	case -EBUSY:
1589	if (in_interrupt()) {
1590	if (try_wait_for_completion(x: &ctx->restart)) {
1591	/*
1592	* we don't have to block to wait for completion,
1593	* so proceed
1594	*/
1595	} else {
1596	/*
1597	* we can't wait for completion without blocking
1598	* exit and continue processing in a workqueue
1599	*/
1600	ctx->r.req = NULL;
1601	ctx->cc_sector += sector_step;
1602	tag_offset++;
1603	return BLK_STS_DEV_RESOURCE;
1604	}
1605	} else {
1606	wait_for_completion(&ctx->restart);
1607	}
1608	reinit_completion(x: &ctx->restart);
1609	fallthrough;
1610	/*
1611	* The request is queued and processed asynchronously,
1612	* completion function kcryptd_async_done() will be called.
1613	*/
1614	case -EINPROGRESS:
1615	ctx->r.req = NULL;
1616	ctx->cc_sector += sector_step;
1617	tag_offset++;
1618	continue;
1619	/*
1620	* The request was already processed (synchronously).
1621	*/
1622	case 0:
1623	atomic_dec(v: &ctx->cc_pending);
1624	ctx->cc_sector += sector_step;
1625	tag_offset++;
1626	if (!atomic)
1627	cond_resched();
1628	continue;
1629	/*
1630	* There was a data integrity error.
1631	*/
1632	case -EBADMSG:
1633	atomic_dec(v: &ctx->cc_pending);
1634	return BLK_STS_PROTECTION;
1635	/*
1636	* There was an error while processing the request.
1637	*/
1638	default:
1639	atomic_dec(v: &ctx->cc_pending);
1640	return BLK_STS_IOERR;
1641	}
1642	}
1643
1644	return 0;
1645	}
1646
1647	static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone);
1648
1649	/*
1650	* Generate a new unfragmented bio with the given size
1651	* This should never violate the device limitations (but only because
1652	* max_segment_size is being constrained to PAGE_SIZE).
1653	*
1654	* This function may be called concurrently. If we allocate from the mempool
1655	* concurrently, there is a possibility of deadlock. For example, if we have
1656	* mempool of 256 pages, two processes, each wanting 256, pages allocate from
1657	* the mempool concurrently, it may deadlock in a situation where both processes
1658	* have allocated 128 pages and the mempool is exhausted.
1659	*
1660	* In order to avoid this scenario we allocate the pages under a mutex.
1661	*
1662	* In order to not degrade performance with excessive locking, we try
1663	* non-blocking allocations without a mutex first but on failure we fallback
1664	* to blocking allocations with a mutex.
1665	*
1666	* In order to reduce allocation overhead, we try to allocate compound pages in
1667	* the first pass. If they are not available, we fall back to the mempool.
1668	*/
1669	static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned int size)
1670	{
1671	struct crypt_config *cc = io->cc;
1672	struct bio *clone;
1673	unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
1674	gfp_t gfp_mask = GFP_NOWAIT | __GFP_HIGHMEM;
1675	unsigned int remaining_size;
1676	unsigned int order = MAX_ORDER - 1;
1677
1678	retry:
1679	if (unlikely(gfp_mask & __GFP_DIRECT_RECLAIM))
1680	mutex_lock(&cc->bio_alloc_lock);
1681
1682	clone = bio_alloc_bioset(bdev: cc->dev->bdev, nr_vecs: nr_iovecs, opf: io->base_bio->bi_opf,
1683	GFP_NOIO, bs: &cc->bs);
1684	clone->bi_private = io;
1685	clone->bi_end_io = crypt_endio;
1686
1687	remaining_size = size;
1688
1689	while (remaining_size) {
1690	struct page *pages;
1691	unsigned size_to_add;
1692	unsigned remaining_order = __fls(word: (remaining_size + PAGE_SIZE - 1) >> PAGE_SHIFT);
1693	order = min(order, remaining_order);
1694
1695	while (order > 0) {
1696	if (unlikely(percpu_counter_read_positive(&cc->n_allocated_pages) +
1697	(1 << order) > dm_crypt_pages_per_client))
1698	goto decrease_order;
1699	pages = alloc_pages(gfp: gfp_mask
1700	| __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN | __GFP_COMP,
1701	order);
1702	if (likely(pages != NULL)) {
1703	percpu_counter_add(fbc: &cc->n_allocated_pages, amount: 1 << order);
1704	goto have_pages;
1705	}
1706	decrease_order:
1707	order--;
1708	}
1709
1710	pages = mempool_alloc(pool: &cc->page_pool, gfp_mask);
1711	if (!pages) {
1712	crypt_free_buffer_pages(cc, clone);
1713	bio_put(clone);
1714	gfp_mask |= __GFP_DIRECT_RECLAIM;
1715	order = 0;
1716	goto retry;
1717	}
1718
1719	have_pages:
1720	size_to_add = min((unsigned)PAGE_SIZE << order, remaining_size);
1721	__bio_add_page(bio: clone, page: pages, len: size_to_add, off: 0);
1722	remaining_size -= size_to_add;
1723	}
1724
1725	/* Allocate space for integrity tags */
1726	if (dm_crypt_integrity_io_alloc(io, bio: clone)) {
1727	crypt_free_buffer_pages(cc, clone);
1728	bio_put(clone);
1729	clone = NULL;
1730	}
1731
1732	if (unlikely(gfp_mask & __GFP_DIRECT_RECLAIM))
1733	mutex_unlock(lock: &cc->bio_alloc_lock);
1734
1735	return clone;
1736	}
1737
1738	static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone)
1739	{
1740	struct folio_iter fi;
1741
1742	if (clone->bi_vcnt > 0) { /* bio_for_each_folio_all crashes with an empty bio */
1743	bio_for_each_folio_all(fi, clone) {
1744	if (folio_test_large(folio: fi.folio)) {
1745	percpu_counter_sub(fbc: &cc->n_allocated_pages,
1746	amount: 1 << folio_order(folio: fi.folio));
1747	folio_put(folio: fi.folio);
1748	} else {
1749	mempool_free(element: &fi.folio->page, pool: &cc->page_pool);
1750	}
1751	}
1752	}
1753	}
1754
1755	static void crypt_io_init(struct dm_crypt_io *io, struct crypt_config *cc,
1756	struct bio *bio, sector_t sector)
1757	{
1758	io->cc = cc;
1759	io->base_bio = bio;
1760	io->sector = sector;
1761	io->error = 0;
1762	io->ctx.r.req = NULL;
1763	io->integrity_metadata = NULL;
1764	io->integrity_metadata_from_pool = false;
1765	io->in_tasklet = false;
1766	atomic_set(v: &io->io_pending, i: 0);
1767	}
1768
1769	static void crypt_inc_pending(struct dm_crypt_io *io)
1770	{
1771	atomic_inc(v: &io->io_pending);
1772	}
1773
1774	static void kcryptd_io_bio_endio(struct work_struct *work)
1775	{
1776	struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
1777
1778	bio_endio(io->base_bio);
1779	}
1780
1781	/*
1782	* One of the bios was finished. Check for completion of
1783	* the whole request and correctly clean up the buffer.
1784	*/
1785	static void crypt_dec_pending(struct dm_crypt_io *io)
1786	{
1787	struct crypt_config *cc = io->cc;
1788	struct bio *base_bio = io->base_bio;
1789	blk_status_t error = io->error;
1790
1791	if (!atomic_dec_and_test(v: &io->io_pending))
1792	return;
1793
1794	if (io->ctx.r.req)
1795	crypt_free_req(cc, req: io->ctx.r.req, base_bio);
1796
1797	if (unlikely(io->integrity_metadata_from_pool))
1798	mempool_free(element: io->integrity_metadata, pool: &io->cc->tag_pool);
1799	else
1800	kfree(objp: io->integrity_metadata);
1801
1802	base_bio->bi_status = error;
1803
1804	/*
1805	* If we are running this function from our tasklet,
1806	* we can't call bio_endio() here, because it will call
1807	* clone_endio() from dm.c, which in turn will
1808	* free the current struct dm_crypt_io structure with
1809	* our tasklet. In this case we need to delay bio_endio()
1810	* execution to after the tasklet is done and dequeued.
1811	*/
1812	if (io->in_tasklet) {
1813	INIT_WORK(&io->work, kcryptd_io_bio_endio);
1814	queue_work(wq: cc->io_queue, work: &io->work);
1815	return;
1816	}
1817
1818	bio_endio(base_bio);
1819	}
1820
1821	/*
1822	* kcryptd/kcryptd_io:
1823	*
1824	* Needed because it would be very unwise to do decryption in an
1825	* interrupt context.
1826	*
1827	* kcryptd performs the actual encryption or decryption.
1828	*
1829	* kcryptd_io performs the IO submission.
1830	*
1831	* They must be separated as otherwise the final stages could be
1832	* starved by new requests which can block in the first stages due
1833	* to memory allocation.
1834	*
1835	* The work is done per CPU global for all dm-crypt instances.
1836	* They should not depend on each other and do not block.
1837	*/
1838	static void crypt_endio(struct bio *clone)
1839	{
1840	struct dm_crypt_io *io = clone->bi_private;
1841	struct crypt_config *cc = io->cc;
1842	unsigned int rw = bio_data_dir(clone);
1843	blk_status_t error;
1844
1845	/*
1846	* free the processed pages
1847	*/
1848	if (rw == WRITE)
1849	crypt_free_buffer_pages(cc, clone);
1850
1851	error = clone->bi_status;
1852	bio_put(clone);
1853
1854	if (rw == READ && !error) {
1855	kcryptd_queue_crypt(io);
1856	return;
1857	}
1858
1859	if (unlikely(error))
1860	io->error = error;
1861
1862	crypt_dec_pending(io);
1863	}
1864
1865	#define CRYPT_MAP_READ_GFP GFP_NOWAIT
1866
1867	static int kcryptd_io_read(struct dm_crypt_io *io, gfp_t gfp)
1868	{
1869	struct crypt_config *cc = io->cc;
1870	struct bio *clone;
1871
1872	/*
1873	* We need the original biovec array in order to decrypt the whole bio
1874	* data *afterwards* -- thanks to immutable biovecs we don't need to
1875	* worry about the block layer modifying the biovec array; so leverage
1876	* bio_alloc_clone().
1877	*/
1878	clone = bio_alloc_clone(bdev: cc->dev->bdev, bio_src: io->base_bio, gfp, bs: &cc->bs);
1879	if (!clone)
1880	return 1;
1881	clone->bi_private = io;
1882	clone->bi_end_io = crypt_endio;
1883
1884	crypt_inc_pending(io);
1885
1886	clone->bi_iter.bi_sector = cc->start + io->sector;
1887
1888	if (dm_crypt_integrity_io_alloc(io, bio: clone)) {
1889	crypt_dec_pending(io);
1890	bio_put(clone);
1891	return 1;
1892	}
1893
1894	dm_submit_bio_remap(clone: io->base_bio, tgt_clone: clone);
1895	return 0;
1896	}
1897
1898	static void kcryptd_io_read_work(struct work_struct *work)
1899	{
1900	struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
1901
1902	crypt_inc_pending(io);
1903	if (kcryptd_io_read(io, GFP_NOIO))
1904	io->error = BLK_STS_RESOURCE;
1905	crypt_dec_pending(io);
1906	}
1907
1908	static void kcryptd_queue_read(struct dm_crypt_io *io)
1909	{
1910	struct crypt_config *cc = io->cc;
1911
1912	INIT_WORK(&io->work, kcryptd_io_read_work);
1913	queue_work(wq: cc->io_queue, work: &io->work);
1914	}
1915
1916	static void kcryptd_io_write(struct dm_crypt_io *io)
1917	{
1918	struct bio *clone = io->ctx.bio_out;
1919
1920	dm_submit_bio_remap(clone: io->base_bio, tgt_clone: clone);
1921	}
1922
1923	#define crypt_io_from_node(node) rb_entry((node), struct dm_crypt_io, rb_node)
1924
1925	static int dmcrypt_write(void *data)
1926	{
1927	struct crypt_config *cc = data;
1928	struct dm_crypt_io *io;
1929
1930	while (1) {
1931	struct rb_root write_tree;
1932	struct blk_plug plug;
1933
1934	spin_lock_irq(lock: &cc->write_thread_lock);
1935	continue_locked:
1936
1937	if (!RB_EMPTY_ROOT(&cc->write_tree))
1938	goto pop_from_list;
1939
1940	set_current_state(TASK_INTERRUPTIBLE);
1941
1942	spin_unlock_irq(lock: &cc->write_thread_lock);
1943
1944	if (unlikely(kthread_should_stop())) {
1945	set_current_state(TASK_RUNNING);
1946	break;
1947	}
1948
1949	schedule();
1950
1951	set_current_state(TASK_RUNNING);
1952	spin_lock_irq(lock: &cc->write_thread_lock);
1953	goto continue_locked;
1954
1955	pop_from_list:
1956	write_tree = cc->write_tree;
1957	cc->write_tree = RB_ROOT;
1958	spin_unlock_irq(lock: &cc->write_thread_lock);
1959
1960	BUG_ON(rb_parent(write_tree.rb_node));
1961
1962	/*
1963	* Note: we cannot walk the tree here with rb_next because
1964	* the structures may be freed when kcryptd_io_write is called.
1965	*/
1966	blk_start_plug(&plug);
1967	do {
1968	io = crypt_io_from_node(rb_first(&write_tree));
1969	rb_erase(&io->rb_node, &write_tree);
1970	kcryptd_io_write(io);
1971	cond_resched();
1972	} while (!RB_EMPTY_ROOT(&write_tree));
1973	blk_finish_plug(&plug);
1974	}
1975	return 0;
1976	}
1977
1978	static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io, int async)
1979	{
1980	struct bio *clone = io->ctx.bio_out;
1981	struct crypt_config *cc = io->cc;
1982	unsigned long flags;
1983	sector_t sector;
1984	struct rb_node **rbp, *parent;
1985
1986	if (unlikely(io->error)) {
1987	crypt_free_buffer_pages(cc, clone);
1988	bio_put(clone);
1989	crypt_dec_pending(io);
1990	return;
1991	}
1992
1993	/* crypt_convert should have filled the clone bio */
1994	BUG_ON(io->ctx.iter_out.bi_size);
1995
1996	clone->bi_iter.bi_sector = cc->start + io->sector;
1997
1998	if ((likely(!async) && test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags)) ||
1999	test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags)) {
2000	dm_submit_bio_remap(clone: io->base_bio, tgt_clone: clone);
2001	return;
2002	}
2003
2004	spin_lock_irqsave(&cc->write_thread_lock, flags);
2005	if (RB_EMPTY_ROOT(&cc->write_tree))
2006	wake_up_process(tsk: cc->write_thread);
2007	rbp = &cc->write_tree.rb_node;
2008	parent = NULL;
2009	sector = io->sector;
2010	while (*rbp) {
2011	parent = *rbp;
2012	if (sector < crypt_io_from_node(parent)->sector)
2013	rbp = &(*rbp)->rb_left;
2014	else
2015	rbp = &(*rbp)->rb_right;
2016	}
2017	rb_link_node(node: &io->rb_node, parent, rb_link: rbp);
2018	rb_insert_color(&io->rb_node, &cc->write_tree);
2019	spin_unlock_irqrestore(lock: &cc->write_thread_lock, flags);
2020	}
2021
2022	static bool kcryptd_crypt_write_inline(struct crypt_config *cc,
2023	struct convert_context *ctx)
2024
2025	{
2026	if (!test_bit(DM_CRYPT_WRITE_INLINE, &cc->flags))
2027	return false;
2028
2029	/*
2030	* Note: zone append writes (REQ_OP_ZONE_APPEND) do not have ordering
2031	* constraints so they do not need to be issued inline by
2032	* kcryptd_crypt_write_convert().
2033	*/
2034	switch (bio_op(bio: ctx->bio_in)) {
2035	case REQ_OP_WRITE:
2036	case REQ_OP_WRITE_ZEROES:
2037	return true;
2038	default:
2039	return false;
2040	}
2041	}
2042
2043	static void kcryptd_crypt_write_continue(struct work_struct *work)
2044	{
2045	struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
2046	struct crypt_config *cc = io->cc;
2047	struct convert_context *ctx = &io->ctx;
2048	int crypt_finished;
2049	sector_t sector = io->sector;
2050	blk_status_t r;
2051
2052	wait_for_completion(&ctx->restart);
2053	reinit_completion(x: &ctx->restart);
2054
2055	r = crypt_convert(cc, ctx: &io->ctx, atomic: true, reset_pending: false);
2056	if (r)
2057	io->error = r;
2058	crypt_finished = atomic_dec_and_test(v: &ctx->cc_pending);
2059	if (!crypt_finished && kcryptd_crypt_write_inline(cc, ctx)) {
2060	/* Wait for completion signaled by kcryptd_async_done() */
2061	wait_for_completion(&ctx->restart);
2062	crypt_finished = 1;
2063	}
2064
2065	/* Encryption was already finished, submit io now */
2066	if (crypt_finished) {
2067	kcryptd_crypt_write_io_submit(io, async: 0);
2068	io->sector = sector;
2069	}
2070
2071	crypt_dec_pending(io);
2072	}
2073
2074	static void kcryptd_crypt_write_convert(struct dm_crypt_io *io)
2075	{
2076	struct crypt_config *cc = io->cc;
2077	struct convert_context *ctx = &io->ctx;
2078	struct bio *clone;
2079	int crypt_finished;
2080	sector_t sector = io->sector;
2081	blk_status_t r;
2082
2083	/*
2084	* Prevent io from disappearing until this function completes.
2085	*/
2086	crypt_inc_pending(io);
2087	crypt_convert_init(cc, ctx, NULL, bio_in: io->base_bio, sector);
2088
2089	clone = crypt_alloc_buffer(io, size: io->base_bio->bi_iter.bi_size);
2090	if (unlikely(!clone)) {
2091	io->error = BLK_STS_IOERR;
2092	goto dec;
2093	}
2094
2095	io->ctx.bio_out = clone;
2096	io->ctx.iter_out = clone->bi_iter;
2097
2098	sector += bio_sectors(clone);
2099
2100	crypt_inc_pending(io);
2101	r = crypt_convert(cc, ctx,
2102	test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags), reset_pending: true);
2103	/*
2104	* Crypto API backlogged the request, because its queue was full
2105	* and we're in softirq context, so continue from a workqueue
2106	* (TODO: is it actually possible to be in softirq in the write path?)
2107	*/
2108	if (r == BLK_STS_DEV_RESOURCE) {
2109	INIT_WORK(&io->work, kcryptd_crypt_write_continue);
2110	queue_work(wq: cc->crypt_queue, work: &io->work);
2111	return;
2112	}
2113	if (r)
2114	io->error = r;
2115	crypt_finished = atomic_dec_and_test(v: &ctx->cc_pending);
2116	if (!crypt_finished && kcryptd_crypt_write_inline(cc, ctx)) {
2117	/* Wait for completion signaled by kcryptd_async_done() */
2118	wait_for_completion(&ctx->restart);
2119	crypt_finished = 1;
2120	}
2121
2122	/* Encryption was already finished, submit io now */
2123	if (crypt_finished) {
2124	kcryptd_crypt_write_io_submit(io, async: 0);
2125	io->sector = sector;
2126	}
2127
2128	dec:
2129	crypt_dec_pending(io);
2130	}
2131
2132	static void kcryptd_crypt_read_done(struct dm_crypt_io *io)
2133	{
2134	crypt_dec_pending(io);
2135	}
2136
2137	static void kcryptd_crypt_read_continue(struct work_struct *work)
2138	{
2139	struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
2140	struct crypt_config *cc = io->cc;
2141	blk_status_t r;
2142
2143	wait_for_completion(&io->ctx.restart);
2144	reinit_completion(x: &io->ctx.restart);
2145
2146	r = crypt_convert(cc, ctx: &io->ctx, atomic: true, reset_pending: false);
2147	if (r)
2148	io->error = r;
2149
2150	if (atomic_dec_and_test(v: &io->ctx.cc_pending))
2151	kcryptd_crypt_read_done(io);
2152
2153	crypt_dec_pending(io);
2154	}
2155
2156	static void kcryptd_crypt_read_convert(struct dm_crypt_io *io)
2157	{
2158	struct crypt_config *cc = io->cc;
2159	blk_status_t r;
2160
2161	crypt_inc_pending(io);
2162
2163	crypt_convert_init(cc, ctx: &io->ctx, bio_out: io->base_bio, bio_in: io->base_bio,
2164	sector: io->sector);
2165
2166	r = crypt_convert(cc, ctx: &io->ctx,
2167	test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags), reset_pending: true);
2168	/*
2169	* Crypto API backlogged the request, because its queue was full
2170	* and we're in softirq context, so continue from a workqueue
2171	*/
2172	if (r == BLK_STS_DEV_RESOURCE) {
2173	INIT_WORK(&io->work, kcryptd_crypt_read_continue);
2174	queue_work(wq: cc->crypt_queue, work: &io->work);
2175	return;
2176	}
2177	if (r)
2178	io->error = r;
2179
2180	if (atomic_dec_and_test(v: &io->ctx.cc_pending))
2181	kcryptd_crypt_read_done(io);
2182
2183	crypt_dec_pending(io);
2184	}
2185
2186	static void kcryptd_async_done(void *data, int error)
2187	{
2188	struct dm_crypt_request *dmreq = data;
2189	struct convert_context *ctx = dmreq->ctx;
2190	struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx);
2191	struct crypt_config *cc = io->cc;
2192
2193	/*
2194	* A request from crypto driver backlog is going to be processed now,
2195	* finish the completion and continue in crypt_convert().
2196	* (Callback will be called for the second time for this request.)
2197	*/
2198	if (error == -EINPROGRESS) {
2199	complete(&ctx->restart);
2200	return;
2201	}
2202
2203	if (!error && cc->iv_gen_ops && cc->iv_gen_ops->post)
2204	error = cc->iv_gen_ops->post(cc, org_iv_of_dmreq(cc, dmreq), dmreq);
2205
2206	if (error == -EBADMSG) {
2207	sector_t s = le64_to_cpu(*org_sector_of_dmreq(cc, dmreq));
2208
2209	DMERR_LIMIT("%pg: INTEGRITY AEAD ERROR, sector %llu",
2210	ctx->bio_in->bi_bdev, s);
2211	dm_audit_log_bio(DM_MSG_PREFIX, op: "integrity-aead",
2212	bio: ctx->bio_in, sector: s, result: 0);
2213	io->error = BLK_STS_PROTECTION;
2214	} else if (error < 0)
2215	io->error = BLK_STS_IOERR;
2216
2217	crypt_free_req(cc, req: req_of_dmreq(cc, dmreq), base_bio: io->base_bio);
2218
2219	if (!atomic_dec_and_test(v: &ctx->cc_pending))
2220	return;
2221
2222	/*
2223	* The request is fully completed: for inline writes, let
2224	* kcryptd_crypt_write_convert() do the IO submission.
2225	*/
2226	if (bio_data_dir(io->base_bio) == READ) {
2227	kcryptd_crypt_read_done(io);
2228	return;
2229	}
2230
2231	if (kcryptd_crypt_write_inline(cc, ctx)) {
2232	complete(&ctx->restart);
2233	return;
2234	}
2235
2236	kcryptd_crypt_write_io_submit(io, async: 1);
2237	}
2238
2239	static void kcryptd_crypt(struct work_struct *work)
2240	{
2241	struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
2242
2243	if (bio_data_dir(io->base_bio) == READ)
2244	kcryptd_crypt_read_convert(io);
2245	else
2246	kcryptd_crypt_write_convert(io);
2247	}
2248
2249	static void kcryptd_crypt_tasklet(unsigned long work)
2250	{
2251	kcryptd_crypt(work: (struct work_struct *)work);
2252	}
2253
2254	static void kcryptd_queue_crypt(struct dm_crypt_io *io)
2255	{
2256	struct crypt_config *cc = io->cc;
2257
2258	if ((bio_data_dir(io->base_bio) == READ && test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags)) ||
2259	(bio_data_dir(io->base_bio) == WRITE && test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags))) {
2260	/*
2261	* in_hardirq(): Crypto API's skcipher_walk_first() refuses to work in hard IRQ context.
2262	* irqs_disabled(): the kernel may run some IO completion from the idle thread, but
2263	* it is being executed with irqs disabled.
2264	*/
2265	if (in_hardirq() || irqs_disabled()) {
2266	io->in_tasklet = true;
2267	tasklet_init(t: &io->tasklet, func: kcryptd_crypt_tasklet, data: (unsigned long)&io->work);
2268	tasklet_schedule(t: &io->tasklet);
2269	return;
2270	}
2271
2272	kcryptd_crypt(work: &io->work);
2273	return;
2274	}
2275
2276	INIT_WORK(&io->work, kcryptd_crypt);
2277	queue_work(wq: cc->crypt_queue, work: &io->work);
2278	}
2279
2280	static void crypt_free_tfms_aead(struct crypt_config *cc)
2281	{
2282	if (!cc->cipher_tfm.tfms_aead)
2283	return;
2284
2285	if (cc->cipher_tfm.tfms_aead[0] && !IS_ERR(ptr: cc->cipher_tfm.tfms_aead[0])) {
2286	crypto_free_aead(tfm: cc->cipher_tfm.tfms_aead[0]);
2287	cc->cipher_tfm.tfms_aead[0] = NULL;
2288	}
2289
2290	kfree(objp: cc->cipher_tfm.tfms_aead);
2291	cc->cipher_tfm.tfms_aead = NULL;
2292	}
2293
2294	static void crypt_free_tfms_skcipher(struct crypt_config *cc)
2295	{
2296	unsigned int i;
2297
2298	if (!cc->cipher_tfm.tfms)
2299	return;
2300
2301	for (i = 0; i < cc->tfms_count; i++)
2302	if (cc->cipher_tfm.tfms[i] && !IS_ERR(ptr: cc->cipher_tfm.tfms[i])) {
2303	crypto_free_skcipher(tfm: cc->cipher_tfm.tfms[i]);
2304	cc->cipher_tfm.tfms[i] = NULL;
2305	}
2306
2307	kfree(objp: cc->cipher_tfm.tfms);
2308	cc->cipher_tfm.tfms = NULL;
2309	}
2310
2311	static void crypt_free_tfms(struct crypt_config *cc)
2312	{
2313	if (crypt_integrity_aead(cc))
2314	crypt_free_tfms_aead(cc);
2315	else
2316	crypt_free_tfms_skcipher(cc);
2317	}
2318
2319	static int crypt_alloc_tfms_skcipher(struct crypt_config *cc, char *ciphermode)
2320	{
2321	unsigned int i;
2322	int err;
2323
2324	cc->cipher_tfm.tfms = kcalloc(n: cc->tfms_count,
2325	size: sizeof(struct crypto_skcipher *),
2326	GFP_KERNEL);
2327	if (!cc->cipher_tfm.tfms)
2328	return -ENOMEM;
2329
2330	for (i = 0; i < cc->tfms_count; i++) {
2331	cc->cipher_tfm.tfms[i] = crypto_alloc_skcipher(alg_name: ciphermode, type: 0,
2332	CRYPTO_ALG_ALLOCATES_MEMORY);
2333	if (IS_ERR(ptr: cc->cipher_tfm.tfms[i])) {
2334	err = PTR_ERR(ptr: cc->cipher_tfm.tfms[i]);
2335	crypt_free_tfms(cc);
2336	return err;
2337	}
2338	}
2339
2340	/*
2341	* dm-crypt performance can vary greatly depending on which crypto
2342	* algorithm implementation is used. Help people debug performance
2343	* problems by logging the ->cra_driver_name.
2344	*/
2345	DMDEBUG_LIMIT("%s using implementation \"%s\"", ciphermode,
2346	crypto_skcipher_alg(any_tfm(cc))->base.cra_driver_name);
2347	return 0;
2348	}
2349
2350	static int crypt_alloc_tfms_aead(struct crypt_config *cc, char *ciphermode)
2351	{
2352	int err;
2353
2354	cc->cipher_tfm.tfms = kmalloc(size: sizeof(struct crypto_aead *), GFP_KERNEL);
2355	if (!cc->cipher_tfm.tfms)
2356	return -ENOMEM;
2357
2358	cc->cipher_tfm.tfms_aead[0] = crypto_alloc_aead(alg_name: ciphermode, type: 0,
2359	CRYPTO_ALG_ALLOCATES_MEMORY);
2360	if (IS_ERR(ptr: cc->cipher_tfm.tfms_aead[0])) {
2361	err = PTR_ERR(ptr: cc->cipher_tfm.tfms_aead[0]);
2362	crypt_free_tfms(cc);
2363	return err;
2364	}
2365
2366	DMDEBUG_LIMIT("%s using implementation \"%s\"", ciphermode,
2367	crypto_aead_alg(any_tfm_aead(cc))->base.cra_driver_name);
2368	return 0;
2369	}
2370
2371	static int crypt_alloc_tfms(struct crypt_config *cc, char *ciphermode)
2372	{
2373	if (crypt_integrity_aead(cc))
2374	return crypt_alloc_tfms_aead(cc, ciphermode);
2375	else
2376	return crypt_alloc_tfms_skcipher(cc, ciphermode);
2377	}
2378
2379	static unsigned int crypt_subkey_size(struct crypt_config *cc)
2380	{
2381	return (cc->key_size - cc->key_extra_size) >> ilog2(cc->tfms_count);
2382	}
2383
2384	static unsigned int crypt_authenckey_size(struct crypt_config *cc)
2385	{
2386	return crypt_subkey_size(cc) + RTA_SPACE(sizeof(struct crypto_authenc_key_param));
2387	}
2388
2389	/*
2390	* If AEAD is composed like authenc(hmac(sha256),xts(aes)),
2391	* the key must be for some reason in special format.
2392	* This funcion converts cc->key to this special format.
2393	*/
2394	static void crypt_copy_authenckey(char *p, const void *key,
2395	unsigned int enckeylen, unsigned int authkeylen)
2396	{
2397	struct crypto_authenc_key_param *param;
2398	struct rtattr *rta;
2399
2400	rta = (struct rtattr *)p;
2401	param = RTA_DATA(rta);
2402	param->enckeylen = cpu_to_be32(enckeylen);
2403	rta->rta_len = RTA_LENGTH(sizeof(*param));
2404	rta->rta_type = CRYPTO_AUTHENC_KEYA_PARAM;
2405	p += RTA_SPACE(sizeof(*param));
2406	memcpy(p, key + enckeylen, authkeylen);
2407	p += authkeylen;
2408	memcpy(p, key, enckeylen);
2409	}
2410
2411	static int crypt_setkey(struct crypt_config *cc)
2412	{
2413	unsigned int subkey_size;
2414	int err = 0, i, r;
2415
2416	/* Ignore extra keys (which are used for IV etc) */
2417	subkey_size = crypt_subkey_size(cc);
2418
2419	if (crypt_integrity_hmac(cc)) {
2420	if (subkey_size < cc->key_mac_size)
2421	return -EINVAL;
2422
2423	crypt_copy_authenckey(p: cc->authenc_key, key: cc->key,
2424	enckeylen: subkey_size - cc->key_mac_size,
2425	authkeylen: cc->key_mac_size);
2426	}
2427
2428	for (i = 0; i < cc->tfms_count; i++) {
2429	if (crypt_integrity_hmac(cc))
2430	r = crypto_aead_setkey(tfm: cc->cipher_tfm.tfms_aead[i],
2431	key: cc->authenc_key, keylen: crypt_authenckey_size(cc));
2432	else if (crypt_integrity_aead(cc))
2433	r = crypto_aead_setkey(tfm: cc->cipher_tfm.tfms_aead[i],
2434	key: cc->key + (i * subkey_size),
2435	keylen: subkey_size);
2436	else
2437	r = crypto_skcipher_setkey(tfm: cc->cipher_tfm.tfms[i],
2438	key: cc->key + (i * subkey_size),
2439	keylen: subkey_size);
2440	if (r)
2441	err = r;
2442	}
2443
2444	if (crypt_integrity_hmac(cc))
2445	memzero_explicit(s: cc->authenc_key, count: crypt_authenckey_size(cc));
2446
2447	return err;
2448	}
2449
2450	#ifdef CONFIG_KEYS
2451
2452	static bool contains_whitespace(const char *str)
2453	{
2454	while (*str)
2455	if (isspace(*str++))
2456	return true;
2457	return false;
2458	}
2459
2460	static int set_key_user(struct crypt_config *cc, struct key *key)
2461	{
2462	const struct user_key_payload *ukp;
2463
2464	ukp = user_key_payload_locked(key);
2465	if (!ukp)
2466	return -EKEYREVOKED;
2467
2468	if (cc->key_size != ukp->datalen)
2469	return -EINVAL;
2470
2471	memcpy(cc->key, ukp->data, cc->key_size);
2472
2473	return 0;
2474	}
2475
2476	static int set_key_encrypted(struct crypt_config *cc, struct key *key)
2477	{
2478	const struct encrypted_key_payload *ekp;
2479
2480	ekp = key->payload.data[0];
2481	if (!ekp)
2482	return -EKEYREVOKED;
2483
2484	if (cc->key_size != ekp->decrypted_datalen)
2485	return -EINVAL;
2486
2487	memcpy(cc->key, ekp->decrypted_data, cc->key_size);
2488
2489	return 0;
2490	}
2491
2492	static int set_key_trusted(struct crypt_config *cc, struct key *key)
2493	{
2494	const struct trusted_key_payload *tkp;
2495
2496	tkp = key->payload.data[0];
2497	if (!tkp)
2498	return -EKEYREVOKED;
2499
2500	if (cc->key_size != tkp->key_len)
2501	return -EINVAL;
2502
2503	memcpy(cc->key, tkp->key, cc->key_size);
2504
2505	return 0;
2506	}
2507
2508	static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string)
2509	{
2510	char *new_key_string, *key_desc;
2511	int ret;
2512	struct key_type *type;
2513	struct key *key;
2514	int (*set_key)(struct crypt_config *cc, struct key *key);
2515
2516	/*
2517	* Reject key_string with whitespace. dm core currently lacks code for
2518	* proper whitespace escaping in arguments on DM_TABLE_STATUS path.
2519	*/
2520	if (contains_whitespace(str: key_string)) {
2521	DMERR("whitespace chars not allowed in key string");
2522	return -EINVAL;
2523	}
2524
2525	/* look for next ':' separating key_type from key_description */
2526	key_desc = strchr(key_string, ':');
2527	if (!key_desc || key_desc == key_string || !strlen(key_desc + 1))
2528	return -EINVAL;
2529
2530	if (!strncmp(key_string, "logon:", key_desc - key_string + 1)) {
2531	type = &key_type_logon;
2532	set_key = set_key_user;
2533	} else if (!strncmp(key_string, "user:", key_desc - key_string + 1)) {
2534	type = &key_type_user;
2535	set_key = set_key_user;
2536	} else if (IS_ENABLED(CONFIG_ENCRYPTED_KEYS) &&
2537	!strncmp(key_string, "encrypted:", key_desc - key_string + 1)) {
2538	type = &key_type_encrypted;
2539	set_key = set_key_encrypted;
2540	} else if (IS_ENABLED(CONFIG_TRUSTED_KEYS) &&
2541	!strncmp(key_string, "trusted:", key_desc - key_string + 1)) {
2542	type = &key_type_trusted;
2543	set_key = set_key_trusted;
2544	} else {
2545	return -EINVAL;
2546	}
2547
2548	new_key_string = kstrdup(s: key_string, GFP_KERNEL);
2549	if (!new_key_string)
2550	return -ENOMEM;
2551
2552	key = request_key(type, description: key_desc + 1, NULL);
2553	if (IS_ERR(ptr: key)) {
2554	kfree_sensitive(objp: new_key_string);
2555	return PTR_ERR(ptr: key);
2556	}
2557
2558	down_read(sem: &key->sem);
2559
2560	ret = set_key(cc, key);
2561	if (ret < 0) {
2562	up_read(sem: &key->sem);
2563	key_put(key);
2564	kfree_sensitive(objp: new_key_string);
2565	return ret;
2566	}
2567
2568	up_read(sem: &key->sem);
2569	key_put(key);
2570
2571	/* clear the flag since following operations may invalidate previously valid key */
2572	clear_bit(nr: DM_CRYPT_KEY_VALID, addr: &cc->flags);
2573
2574	ret = crypt_setkey(cc);
2575
2576	if (!ret) {
2577	set_bit(nr: DM_CRYPT_KEY_VALID, addr: &cc->flags);
2578	kfree_sensitive(objp: cc->key_string);
2579	cc->key_string = new_key_string;
2580	} else
2581	kfree_sensitive(objp: new_key_string);
2582
2583	return ret;
2584	}
2585
2586	static int get_key_size(char **key_string)
2587	{
2588	char *colon, dummy;
2589	int ret;
2590
2591	if (*key_string[0] != ':')
2592	return strlen(*key_string) >> 1;
2593
2594	/* look for next ':' in key string */
2595	colon = strpbrk(*key_string + 1, ":");
2596	if (!colon)
2597	return -EINVAL;
2598
2599	if (sscanf(*key_string + 1, "%u%c", &ret, &dummy) != 2 || dummy != ':')
2600	return -EINVAL;
2601
2602	*key_string = colon;
2603
2604	/* remaining key string should be :<logon|user>:<key_desc> */
2605
2606	return ret;
2607	}
2608
2609	#else
2610
2611	static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string)
2612	{
2613	return -EINVAL;
2614	}
2615
2616	static int get_key_size(char **key_string)
2617	{
2618	return (*key_string[0] == ':') ? -EINVAL : (int)(strlen(*key_string) >> 1);
2619	}
2620
2621	#endif /* CONFIG_KEYS */
2622
2623	static int crypt_set_key(struct crypt_config *cc, char *key)
2624	{
2625	int r = -EINVAL;
2626	int key_string_len = strlen(key);
2627
2628	/* Hyphen (which gives a key_size of zero) means there is no key. */
2629	if (!cc->key_size && strcmp(key, "-"))
2630	goto out;
2631
2632	/* ':' means the key is in kernel keyring, short-circuit normal key processing */
2633	if (key[0] == ':') {
2634	r = crypt_set_keyring_key(cc, key_string: key + 1);
2635	goto out;
2636	}
2637
2638	/* clear the flag since following operations may invalidate previously valid key */
2639	clear_bit(nr: DM_CRYPT_KEY_VALID, addr: &cc->flags);
2640
2641	/* wipe references to any kernel keyring key */
2642	kfree_sensitive(objp: cc->key_string);
2643	cc->key_string = NULL;
2644
2645	/* Decode key from its hex representation. */
2646	if (cc->key_size && hex2bin(dst: cc->key, src: key, count: cc->key_size) < 0)
2647	goto out;
2648
2649	r = crypt_setkey(cc);
2650	if (!r)
2651	set_bit(nr: DM_CRYPT_KEY_VALID, addr: &cc->flags);
2652
2653	out:
2654	/* Hex key string not needed after here, so wipe it. */
2655	memset(key, '0', key_string_len);
2656
2657	return r;
2658	}
2659
2660	static int crypt_wipe_key(struct crypt_config *cc)
2661	{
2662	int r;
2663
2664	clear_bit(nr: DM_CRYPT_KEY_VALID, addr: &cc->flags);
2665	get_random_bytes(buf: &cc->key, len: cc->key_size);
2666
2667	/* Wipe IV private keys */
2668	if (cc->iv_gen_ops && cc->iv_gen_ops->wipe) {
2669	r = cc->iv_gen_ops->wipe(cc);
2670	if (r)
2671	return r;
2672	}
2673
2674	kfree_sensitive(objp: cc->key_string);
2675	cc->key_string = NULL;
2676	r = crypt_setkey(cc);
2677	memset(&cc->key, 0, cc->key_size * sizeof(u8));
2678
2679	return r;
2680	}
2681
2682	static void crypt_calculate_pages_per_client(void)
2683	{
2684	unsigned long pages = (totalram_pages() - totalhigh_pages()) * DM_CRYPT_MEMORY_PERCENT / 100;
2685
2686	if (!dm_crypt_clients_n)
2687	return;
2688
2689	pages /= dm_crypt_clients_n;
2690	if (pages < DM_CRYPT_MIN_PAGES_PER_CLIENT)
2691	pages = DM_CRYPT_MIN_PAGES_PER_CLIENT;
2692	dm_crypt_pages_per_client = pages;
2693	}
2694
2695	static void *crypt_page_alloc(gfp_t gfp_mask, void *pool_data)
2696	{
2697	struct crypt_config *cc = pool_data;
2698	struct page *page;
2699
2700	/*
2701	* Note, percpu_counter_read_positive() may over (and under) estimate
2702	* the current usage by at most (batch - 1) * num_online_cpus() pages,
2703	* but avoids potential spinlock contention of an exact result.
2704	*/
2705	if (unlikely(percpu_counter_read_positive(&cc->n_allocated_pages) >= dm_crypt_pages_per_client) &&
2706	likely(gfp_mask & __GFP_NORETRY))
2707	return NULL;
2708
2709	page = alloc_page(gfp_mask);
2710	if (likely(page != NULL))
2711	percpu_counter_add(fbc: &cc->n_allocated_pages, amount: 1);
2712
2713	return page;
2714	}
2715
2716	static void crypt_page_free(void *page, void *pool_data)
2717	{
2718	struct crypt_config *cc = pool_data;
2719
2720	__free_page(page);
2721	percpu_counter_sub(fbc: &cc->n_allocated_pages, amount: 1);
2722	}
2723
2724	static void crypt_dtr(struct dm_target *ti)
2725	{
2726	struct crypt_config *cc = ti->private;
2727
2728	ti->private = NULL;
2729
2730	if (!cc)
2731	return;
2732
2733	if (cc->write_thread)
2734	kthread_stop(k: cc->write_thread);
2735
2736	if (cc->io_queue)
2737	destroy_workqueue(wq: cc->io_queue);
2738	if (cc->crypt_queue)
2739	destroy_workqueue(wq: cc->crypt_queue);
2740
2741	crypt_free_tfms(cc);
2742
2743	bioset_exit(&cc->bs);
2744
2745	mempool_exit(pool: &cc->page_pool);
2746	mempool_exit(pool: &cc->req_pool);
2747	mempool_exit(pool: &cc->tag_pool);
2748
2749	WARN_ON(percpu_counter_sum(&cc->n_allocated_pages) != 0);
2750	percpu_counter_destroy(fbc: &cc->n_allocated_pages);
2751
2752	if (cc->iv_gen_ops && cc->iv_gen_ops->dtr)
2753	cc->iv_gen_ops->dtr(cc);
2754
2755	if (cc->dev)
2756	dm_put_device(ti, d: cc->dev);
2757
2758	kfree_sensitive(objp: cc->cipher_string);
2759	kfree_sensitive(objp: cc->key_string);
2760	kfree_sensitive(objp: cc->cipher_auth);
2761	kfree_sensitive(objp: cc->authenc_key);
2762
2763	mutex_destroy(lock: &cc->bio_alloc_lock);
2764
2765	/* Must zero key material before freeing */
2766	kfree_sensitive(objp: cc);
2767
2768	spin_lock(lock: &dm_crypt_clients_lock);
2769	WARN_ON(!dm_crypt_clients_n);
2770	dm_crypt_clients_n--;
2771	crypt_calculate_pages_per_client();
2772	spin_unlock(lock: &dm_crypt_clients_lock);
2773
2774	dm_audit_log_dtr(DM_MSG_PREFIX, ti, result: 1);
2775	}
2776
2777	static int crypt_ctr_ivmode(struct dm_target *ti, const char *ivmode)
2778	{
2779	struct crypt_config *cc = ti->private;
2780
2781	if (crypt_integrity_aead(cc))
2782	cc->iv_size = crypto_aead_ivsize(tfm: any_tfm_aead(cc));
2783	else
2784	cc->iv_size = crypto_skcipher_ivsize(tfm: any_tfm(cc));
2785
2786	if (cc->iv_size)
2787	/* at least a 64 bit sector number should fit in our buffer */
2788	cc->iv_size = max(cc->iv_size,
2789	(unsigned int)(sizeof(u64) / sizeof(u8)));
2790	else if (ivmode) {
2791	DMWARN("Selected cipher does not support IVs");
2792	ivmode = NULL;
2793	}
2794
2795	/* Choose ivmode, see comments at iv code. */
2796	if (ivmode == NULL)
2797	cc->iv_gen_ops = NULL;
2798	else if (strcmp(ivmode, "plain") == 0)
2799	cc->iv_gen_ops = &crypt_iv_plain_ops;
2800	else if (strcmp(ivmode, "plain64") == 0)
2801	cc->iv_gen_ops = &crypt_iv_plain64_ops;
2802	else if (strcmp(ivmode, "plain64be") == 0)
2803	cc->iv_gen_ops = &crypt_iv_plain64be_ops;
2804	else if (strcmp(ivmode, "essiv") == 0)
2805	cc->iv_gen_ops = &crypt_iv_essiv_ops;
2806	else if (strcmp(ivmode, "benbi") == 0)
2807	cc->iv_gen_ops = &crypt_iv_benbi_ops;
2808	else if (strcmp(ivmode, "null") == 0)
2809	cc->iv_gen_ops = &crypt_iv_null_ops;
2810	else if (strcmp(ivmode, "eboiv") == 0)
2811	cc->iv_gen_ops = &crypt_iv_eboiv_ops;
2812	else if (strcmp(ivmode, "elephant") == 0) {
2813	cc->iv_gen_ops = &crypt_iv_elephant_ops;
2814	cc->key_parts = 2;
2815	cc->key_extra_size = cc->key_size / 2;
2816	if (cc->key_extra_size > ELEPHANT_MAX_KEY_SIZE)
2817	return -EINVAL;
2818	set_bit(nr: CRYPT_ENCRYPT_PREPROCESS, addr: &cc->cipher_flags);
2819	} else if (strcmp(ivmode, "lmk") == 0) {
2820	cc->iv_gen_ops = &crypt_iv_lmk_ops;
2821	/*
2822	* Version 2 and 3 is recognised according
2823	* to length of provided multi-key string.
2824	* If present (version 3), last key is used as IV seed.
2825	* All keys (including IV seed) are always the same size.
2826	*/
2827	if (cc->key_size % cc->key_parts) {
2828	cc->key_parts++;
2829	cc->key_extra_size = cc->key_size / cc->key_parts;
2830	}
2831	} else if (strcmp(ivmode, "tcw") == 0) {
2832	cc->iv_gen_ops = &crypt_iv_tcw_ops;
2833	cc->key_parts += 2; /* IV + whitening */
2834	cc->key_extra_size = cc->iv_size + TCW_WHITENING_SIZE;
2835	} else if (strcmp(ivmode, "random") == 0) {
2836	cc->iv_gen_ops = &crypt_iv_random_ops;
2837	/* Need storage space in integrity fields. */
2838	cc->integrity_iv_size = cc->iv_size;
2839	} else {
2840	ti->error = "Invalid IV mode";
2841	return -EINVAL;
2842	}
2843
2844	return 0;
2845	}
2846
2847	/*
2848	* Workaround to parse HMAC algorithm from AEAD crypto API spec.
2849	* The HMAC is needed to calculate tag size (HMAC digest size).
2850	* This should be probably done by crypto-api calls (once available...)
2851	*/
2852	static int crypt_ctr_auth_cipher(struct crypt_config *cc, char *cipher_api)
2853	{
2854	char *start, *end, *mac_alg = NULL;
2855	struct crypto_ahash *mac;
2856
2857	if (!strstarts(str: cipher_api, prefix: "authenc("))
2858	return 0;
2859
2860	start = strchr(cipher_api, '(');
2861	end = strchr(cipher_api, ',');
2862	if (!start || !end || ++start > end)
2863	return -EINVAL;
2864
2865	mac_alg = kmemdup_nul(s: start, len: end - start, GFP_KERNEL);
2866	if (!mac_alg)
2867	return -ENOMEM;
2868
2869	mac = crypto_alloc_ahash(alg_name: mac_alg, type: 0, CRYPTO_ALG_ALLOCATES_MEMORY);
2870	kfree(objp: mac_alg);
2871
2872	if (IS_ERR(ptr: mac))
2873	return PTR_ERR(ptr: mac);
2874
2875	cc->key_mac_size = crypto_ahash_digestsize(tfm: mac);
2876	crypto_free_ahash(tfm: mac);
2877
2878	cc->authenc_key = kmalloc(size: crypt_authenckey_size(cc), GFP_KERNEL);
2879	if (!cc->authenc_key)
2880	return -ENOMEM;
2881
2882	return 0;
2883	}
2884
2885	static int crypt_ctr_cipher_new(struct dm_target *ti, char *cipher_in, char *key,
2886	char **ivmode, char **ivopts)
2887	{
2888	struct crypt_config *cc = ti->private;
2889	char *tmp, *cipher_api, buf[CRYPTO_MAX_ALG_NAME];
2890	int ret = -EINVAL;
2891
2892	cc->tfms_count = 1;
2893
2894	/*
2895	* New format (capi: prefix)
2896	* capi:cipher_api_spec-iv:ivopts
2897	*/
2898	tmp = &cipher_in[strlen("capi:")];
2899
2900	/* Separate IV options if present, it can contain another '-' in hash name */
2901	*ivopts = strrchr(tmp, ':');
2902	if (*ivopts) {
2903	**ivopts = '\0';
2904	(*ivopts)++;
2905	}
2906	/* Parse IV mode */
2907	*ivmode = strrchr(tmp, '-');
2908	if (*ivmode) {
2909	**ivmode = '\0';
2910	(*ivmode)++;
2911	}
2912	/* The rest is crypto API spec */
2913	cipher_api = tmp;
2914
2915	/* Alloc AEAD, can be used only in new format. */
2916	if (crypt_integrity_aead(cc)) {
2917	ret = crypt_ctr_auth_cipher(cc, cipher_api);
2918	if (ret < 0) {
2919	ti->error = "Invalid AEAD cipher spec";
2920	return ret;
2921	}
2922	}
2923
2924	if (*ivmode && !strcmp(*ivmode, "lmk"))
2925	cc->tfms_count = 64;
2926
2927	if (*ivmode && !strcmp(*ivmode, "essiv")) {
2928	if (!*ivopts) {
2929	ti->error = "Digest algorithm missing for ESSIV mode";
2930	return -EINVAL;
2931	}
2932	ret = snprintf(buf, CRYPTO_MAX_ALG_NAME, fmt: "essiv(%s,%s)",
2933	cipher_api, *ivopts);
2934	if (ret < 0 || ret >= CRYPTO_MAX_ALG_NAME) {
2935	ti->error = "Cannot allocate cipher string";
2936	return -ENOMEM;
2937	}
2938	cipher_api = buf;
2939	}
2940
2941	cc->key_parts = cc->tfms_count;
2942
2943	/* Allocate cipher */
2944	ret = crypt_alloc_tfms(cc, ciphermode: cipher_api);
2945	if (ret < 0) {
2946	ti->error = "Error allocating crypto tfm";
2947	return ret;
2948	}
2949
2950	if (crypt_integrity_aead(cc))
2951	cc->iv_size = crypto_aead_ivsize(tfm: any_tfm_aead(cc));
2952	else
2953	cc->iv_size = crypto_skcipher_ivsize(tfm: any_tfm(cc));
2954
2955	return 0;
2956	}
2957
2958	static int crypt_ctr_cipher_old(struct dm_target *ti, char *cipher_in, char *key,
2959	char **ivmode, char **ivopts)
2960	{
2961	struct crypt_config *cc = ti->private;
2962	char *tmp, *cipher, *chainmode, *keycount;
2963	char *cipher_api = NULL;
2964	int ret = -EINVAL;
2965	char dummy;
2966
2967	if (strchr(cipher_in, '(') || crypt_integrity_aead(cc)) {
2968	ti->error = "Bad cipher specification";
2969	return -EINVAL;
2970	}
2971
2972	/*
2973	* Legacy dm-crypt cipher specification
2974	* cipher[:keycount]-mode-iv:ivopts
2975	*/
2976	tmp = cipher_in;
2977	keycount = strsep(&tmp, "-");
2978	cipher = strsep(&keycount, ":");
2979
2980	if (!keycount)
2981	cc->tfms_count = 1;
2982	else if (sscanf(keycount, "%u%c", &cc->tfms_count, &dummy) != 1 ||
2983	!is_power_of_2(n: cc->tfms_count)) {
2984	ti->error = "Bad cipher key count specification";
2985	return -EINVAL;
2986	}
2987	cc->key_parts = cc->tfms_count;
2988
2989	chainmode = strsep(&tmp, "-");
2990	*ivmode = strsep(&tmp, ":");
2991	*ivopts = tmp;
2992
2993	/*
2994	* For compatibility with the original dm-crypt mapping format, if
2995	* only the cipher name is supplied, use cbc-plain.
2996	*/
2997	if (!chainmode || (!strcmp(chainmode, "plain") && !*ivmode)) {
2998	chainmode = "cbc";
2999	*ivmode = "plain";
3000	}
3001
3002	if (strcmp(chainmode, "ecb") && !*ivmode) {
3003	ti->error = "IV mechanism required";
3004	return -EINVAL;
3005	}
3006
3007	cipher_api = kmalloc(CRYPTO_MAX_ALG_NAME, GFP_KERNEL);
3008	if (!cipher_api)
3009	goto bad_mem;
3010
3011	if (*ivmode && !strcmp(*ivmode, "essiv")) {
3012	if (!*ivopts) {
3013	ti->error = "Digest algorithm missing for ESSIV mode";
3014	kfree(objp: cipher_api);
3015	return -EINVAL;
3016	}
3017	ret = snprintf(buf: cipher_api, CRYPTO_MAX_ALG_NAME,
3018	fmt: "essiv(%s(%s),%s)", chainmode, cipher, *ivopts);
3019	} else {
3020	ret = snprintf(buf: cipher_api, CRYPTO_MAX_ALG_NAME,
3021	fmt: "%s(%s)", chainmode, cipher);
3022	}
3023	if (ret < 0 || ret >= CRYPTO_MAX_ALG_NAME) {
3024	kfree(objp: cipher_api);
3025	goto bad_mem;
3026	}
3027
3028	/* Allocate cipher */
3029	ret = crypt_alloc_tfms(cc, ciphermode: cipher_api);
3030	if (ret < 0) {
3031	ti->error = "Error allocating crypto tfm";
3032	kfree(objp: cipher_api);
3033	return ret;
3034	}
3035	kfree(objp: cipher_api);
3036
3037	return 0;
3038	bad_mem:
3039	ti->error = "Cannot allocate cipher strings";
3040	return -ENOMEM;
3041	}
3042
3043	static int crypt_ctr_cipher(struct dm_target *ti, char *cipher_in, char *key)
3044	{
3045	struct crypt_config *cc = ti->private;
3046	char *ivmode = NULL, *ivopts = NULL;
3047	int ret;
3048
3049	cc->cipher_string = kstrdup(s: cipher_in, GFP_KERNEL);
3050	if (!cc->cipher_string) {
3051	ti->error = "Cannot allocate cipher strings";
3052	return -ENOMEM;
3053	}
3054
3055	if (strstarts(str: cipher_in, prefix: "capi:"))
3056	ret = crypt_ctr_cipher_new(ti, cipher_in, key, ivmode: &ivmode, ivopts: &ivopts);
3057	else
3058	ret = crypt_ctr_cipher_old(ti, cipher_in, key, ivmode: &ivmode, ivopts: &ivopts);
3059	if (ret)
3060	return ret;
3061
3062	/* Initialize IV */
3063	ret = crypt_ctr_ivmode(ti, ivmode);
3064	if (ret < 0)
3065	return ret;
3066
3067	/* Initialize and set key */
3068	ret = crypt_set_key(cc, key);
3069	if (ret < 0) {
3070	ti->error = "Error decoding and setting key";
3071	return ret;
3072	}
3073
3074	/* Allocate IV */
3075	if (cc->iv_gen_ops && cc->iv_gen_ops->ctr) {
3076	ret = cc->iv_gen_ops->ctr(cc, ti, ivopts);
3077	if (ret < 0) {
3078	ti->error = "Error creating IV";
3079	return ret;
3080	}
3081	}
3082
3083	/* Initialize IV (set keys for ESSIV etc) */
3084	if (cc->iv_gen_ops && cc->iv_gen_ops->init) {
3085	ret = cc->iv_gen_ops->init(cc);
3086	if (ret < 0) {
3087	ti->error = "Error initialising IV";
3088	return ret;
3089	}
3090	}
3091
3092	/* wipe the kernel key payload copy */
3093	if (cc->key_string)
3094	memset(cc->key, 0, cc->key_size * sizeof(u8));
3095
3096	return ret;
3097	}
3098
3099	static int crypt_ctr_optional(struct dm_target *ti, unsigned int argc, char **argv)
3100	{
3101	struct crypt_config *cc = ti->private;
3102	struct dm_arg_set as;
3103	static const struct dm_arg _args[] = {
3104	{0, 8, "Invalid number of feature args"},
3105	};
3106	unsigned int opt_params, val;
3107	const char *opt_string, *sval;
3108	char dummy;
3109	int ret;
3110
3111	/* Optional parameters */
3112	as.argc = argc;
3113	as.argv = argv;
3114
3115	ret = dm_read_arg_group(arg: _args, arg_set: &as, num_args: &opt_params, error: &ti->error);
3116	if (ret)
3117	return ret;
3118
3119	while (opt_params--) {
3120	opt_string = dm_shift_arg(as: &as);
3121	if (!opt_string) {
3122	ti->error = "Not enough feature arguments";
3123	return -EINVAL;
3124	}
3125
3126	if (!strcasecmp(s1: opt_string, s2: "allow_discards"))
3127	ti->num_discard_bios = 1;
3128
3129	else if (!strcasecmp(s1: opt_string, s2: "same_cpu_crypt"))
3130	set_bit(nr: DM_CRYPT_SAME_CPU, addr: &cc->flags);
3131
3132	else if (!strcasecmp(s1: opt_string, s2: "submit_from_crypt_cpus"))
3133	set_bit(nr: DM_CRYPT_NO_OFFLOAD, addr: &cc->flags);
3134	else if (!strcasecmp(s1: opt_string, s2: "no_read_workqueue"))
3135	set_bit(nr: DM_CRYPT_NO_READ_WORKQUEUE, addr: &cc->flags);
3136	else if (!strcasecmp(s1: opt_string, s2: "no_write_workqueue"))
3137	set_bit(nr: DM_CRYPT_NO_WRITE_WORKQUEUE, addr: &cc->flags);
3138	else if (sscanf(opt_string, "integrity:%u:", &val) == 1) {
3139	if (val == 0 || val > MAX_TAG_SIZE) {
3140	ti->error = "Invalid integrity arguments";
3141	return -EINVAL;
3142	}
3143	cc->on_disk_tag_size = val;
3144	sval = strchr(opt_string + strlen("integrity:"), ':') + 1;
3145	if (!strcasecmp(s1: sval, s2: "aead")) {
3146	set_bit(nr: CRYPT_MODE_INTEGRITY_AEAD, addr: &cc->cipher_flags);
3147	} else if (strcasecmp(s1: sval, s2: "none")) {
3148	ti->error = "Unknown integrity profile";
3149	return -EINVAL;
3150	}
3151
3152	cc->cipher_auth = kstrdup(s: sval, GFP_KERNEL);
3153	if (!cc->cipher_auth)
3154	return -ENOMEM;
3155	} else if (sscanf(opt_string, "sector_size:%hu%c", &cc->sector_size, &dummy) == 1) {
3156	if (cc->sector_size < (1 << SECTOR_SHIFT) ||
3157	cc->sector_size > 4096 ||
3158	(cc->sector_size & (cc->sector_size - 1))) {
3159	ti->error = "Invalid feature value for sector_size";
3160	return -EINVAL;
3161	}
3162	if (ti->len & ((cc->sector_size >> SECTOR_SHIFT) - 1)) {
3163	ti->error = "Device size is not multiple of sector_size feature";
3164	return -EINVAL;
3165	}
3166	cc->sector_shift = __ffs(cc->sector_size) - SECTOR_SHIFT;
3167	} else if (!strcasecmp(s1: opt_string, s2: "iv_large_sectors"))
3168	set_bit(nr: CRYPT_IV_LARGE_SECTORS, addr: &cc->cipher_flags);
3169	else {
3170	ti->error = "Invalid feature arguments";
3171	return -EINVAL;
3172	}
3173	}
3174
3175	return 0;
3176	}
3177
3178	#ifdef CONFIG_BLK_DEV_ZONED
3179	static int crypt_report_zones(struct dm_target *ti,
3180	struct dm_report_zones_args *args, unsigned int nr_zones)
3181	{
3182	struct crypt_config *cc = ti->private;
3183
3184	return dm_report_zones(bdev: cc->dev->bdev, start: cc->start,
3185	sector: cc->start + dm_target_offset(ti, args->next_sector),
3186	args, nr_zones);
3187	}
3188	#else
3189	#define crypt_report_zones NULL
3190	#endif
3191
3192	/*
3193	* Construct an encryption mapping:
3194	* <cipher> [<key>|:<key_size>:<user|logon>:<key_description>] <iv_offset> <dev_path> <start>
3195	*/
3196	static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
3197	{
3198	struct crypt_config *cc;
3199	const char *devname = dm_table_device_name(t: ti->table);
3200	int key_size;
3201	unsigned int align_mask;
3202	unsigned long long tmpll;
3203	int ret;
3204	size_t iv_size_padding, additional_req_size;
3205	char dummy;
3206
3207	if (argc < 5) {
3208	ti->error = "Not enough arguments";
3209	return -EINVAL;
3210	}
3211
3212	key_size = get_key_size(key_string: &argv[1]);
3213	if (key_size < 0) {
3214	ti->error = "Cannot parse key size";
3215	return -EINVAL;
3216	}
3217
3218	cc = kzalloc(struct_size(cc, key, key_size), GFP_KERNEL);
3219	if (!cc) {
3220	ti->error = "Cannot allocate encryption context";
3221	return -ENOMEM;
3222	}
3223	cc->key_size = key_size;
3224	cc->sector_size = (1 << SECTOR_SHIFT);
3225	cc->sector_shift = 0;
3226
3227	ti->private = cc;
3228
3229	spin_lock(lock: &dm_crypt_clients_lock);
3230	dm_crypt_clients_n++;
3231	crypt_calculate_pages_per_client();
3232	spin_unlock(lock: &dm_crypt_clients_lock);
3233
3234	ret = percpu_counter_init(&cc->n_allocated_pages, 0, GFP_KERNEL);
3235	if (ret < 0)
3236	goto bad;
3237
3238	/* Optional parameters need to be read before cipher constructor */
3239	if (argc > 5) {
3240	ret = crypt_ctr_optional(ti, argc: argc - 5, argv: &argv[5]);
3241	if (ret)
3242	goto bad;
3243	}
3244
3245	ret = crypt_ctr_cipher(ti, cipher_in: argv[0], key: argv[1]);
3246	if (ret < 0)
3247	goto bad;
3248
3249	if (crypt_integrity_aead(cc)) {
3250	cc->dmreq_start = sizeof(struct aead_request);
3251	cc->dmreq_start += crypto_aead_reqsize(tfm: any_tfm_aead(cc));
3252	align_mask = crypto_aead_alignmask(tfm: any_tfm_aead(cc));
3253	} else {
3254	cc->dmreq_start = sizeof(struct skcipher_request);
3255	cc->dmreq_start += crypto_skcipher_reqsize(tfm: any_tfm(cc));
3256	align_mask = crypto_skcipher_alignmask(tfm: any_tfm(cc));
3257	}
3258	cc->dmreq_start = ALIGN(cc->dmreq_start, __alignof__(struct dm_crypt_request));
3259
3260	if (align_mask < CRYPTO_MINALIGN) {
3261	/* Allocate the padding exactly */
3262	iv_size_padding = -(cc->dmreq_start + sizeof(struct dm_crypt_request))
3263	& align_mask;
3264	} else {
3265	/*
3266	* If the cipher requires greater alignment than kmalloc
3267	* alignment, we don't know the exact position of the
3268	* initialization vector. We must assume worst case.
3269	*/
3270	iv_size_padding = align_mask;
3271	}
3272
3273	/* ...| IV + padding | original IV | original sec. number | bio tag offset | */
3274	additional_req_size = sizeof(struct dm_crypt_request) +
3275	iv_size_padding + cc->iv_size +
3276	cc->iv_size +
3277	sizeof(uint64_t) +
3278	sizeof(unsigned int);
3279
3280	ret = mempool_init_kmalloc_pool(pool: &cc->req_pool, MIN_IOS, size: cc->dmreq_start + additional_req_size);
3281	if (ret) {
3282	ti->error = "Cannot allocate crypt request mempool";
3283	goto bad;
3284	}
3285
3286	cc->per_bio_data_size = ti->per_io_data_size =
3287	ALIGN(sizeof(struct dm_crypt_io) + cc->dmreq_start + additional_req_size,
3288	ARCH_DMA_MINALIGN);
3289
3290	ret = mempool_init(pool: &cc->page_pool, BIO_MAX_VECS, alloc_fn: crypt_page_alloc, free_fn: crypt_page_free, pool_data: cc);
3291	if (ret) {
3292	ti->error = "Cannot allocate page mempool";
3293	goto bad;
3294	}
3295
3296	ret = bioset_init(&cc->bs, MIN_IOS, 0, flags: BIOSET_NEED_BVECS);
3297	if (ret) {
3298	ti->error = "Cannot allocate crypt bioset";
3299	goto bad;
3300	}
3301
3302	mutex_init(&cc->bio_alloc_lock);
3303
3304	ret = -EINVAL;
3305	if ((sscanf(argv[2], "%llu%c", &tmpll, &dummy) != 1) ||
3306	(tmpll & ((cc->sector_size >> SECTOR_SHIFT) - 1))) {
3307	ti->error = "Invalid iv_offset sector";
3308	goto bad;
3309	}
3310	cc->iv_offset = tmpll;
3311
3312	ret = dm_get_device(ti, path: argv[3], mode: dm_table_get_mode(t: ti->table), result: &cc->dev);
3313	if (ret) {
3314	ti->error = "Device lookup failed";
3315	goto bad;
3316	}
3317
3318	ret = -EINVAL;
3319	if (sscanf(argv[4], "%llu%c", &tmpll, &dummy) != 1 || tmpll != (sector_t)tmpll) {
3320	ti->error = "Invalid device sector";
3321	goto bad;
3322	}
3323	cc->start = tmpll;
3324
3325	if (bdev_is_zoned(bdev: cc->dev->bdev)) {
3326	/*
3327	* For zoned block devices, we need to preserve the issuer write
3328	* ordering. To do so, disable write workqueues and force inline
3329	* encryption completion.
3330	*/
3331	set_bit(nr: DM_CRYPT_NO_WRITE_WORKQUEUE, addr: &cc->flags);
3332	set_bit(nr: DM_CRYPT_WRITE_INLINE, addr: &cc->flags);
3333
3334	/*
3335	* All zone append writes to a zone of a zoned block device will
3336	* have the same BIO sector, the start of the zone. When the
3337	* cypher IV mode uses sector values, all data targeting a
3338	* zone will be encrypted using the first sector numbers of the
3339	* zone. This will not result in write errors but will
3340	* cause most reads to fail as reads will use the sector values
3341	* for the actual data locations, resulting in IV mismatch.
3342	* To avoid this problem, ask DM core to emulate zone append
3343	* operations with regular writes.
3344	*/
3345	DMDEBUG("Zone append operations will be emulated");
3346	ti->emulate_zone_append = true;
3347	}
3348
3349	if (crypt_integrity_aead(cc) || cc->integrity_iv_size) {
3350	ret = crypt_integrity_ctr(cc, ti);
3351	if (ret)
3352	goto bad;
3353
3354	cc->tag_pool_max_sectors = POOL_ENTRY_SIZE / cc->on_disk_tag_size;
3355	if (!cc->tag_pool_max_sectors)
3356	cc->tag_pool_max_sectors = 1;
3357
3358	ret = mempool_init_kmalloc_pool(pool: &cc->tag_pool, MIN_IOS,
3359	size: cc->tag_pool_max_sectors * cc->on_disk_tag_size);
3360	if (ret) {
3361	ti->error = "Cannot allocate integrity tags mempool";
3362	goto bad;
3363	}
3364
3365	cc->tag_pool_max_sectors <<= cc->sector_shift;
3366	}
3367
3368	ret = -ENOMEM;
3369	cc->io_queue = alloc_workqueue(fmt: "kcryptd_io/%s", flags: WQ_MEM_RECLAIM, max_active: 1, devname);
3370	if (!cc->io_queue) {
3371	ti->error = "Couldn't create kcryptd io queue";
3372	goto bad;
3373	}
3374
3375	if (test_bit(DM_CRYPT_SAME_CPU, &cc->flags))
3376	cc->crypt_queue = alloc_workqueue(fmt: "kcryptd/%s", flags: WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM,
3377	max_active: 1, devname);
3378	else
3379	cc->crypt_queue = alloc_workqueue(fmt: "kcryptd/%s",
3380	flags: WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | WQ_UNBOUND,
3381	max_active: num_online_cpus(), devname);
3382	if (!cc->crypt_queue) {
3383	ti->error = "Couldn't create kcryptd queue";
3384	goto bad;
3385	}
3386
3387	spin_lock_init(&cc->write_thread_lock);
3388	cc->write_tree = RB_ROOT;
3389
3390	cc->write_thread = kthread_run(dmcrypt_write, cc, "dmcrypt_write/%s", devname);
3391	if (IS_ERR(ptr: cc->write_thread)) {
3392	ret = PTR_ERR(ptr: cc->write_thread);
3393	cc->write_thread = NULL;
3394	ti->error = "Couldn't spawn write thread";
3395	goto bad;
3396	}
3397
3398	ti->num_flush_bios = 1;
3399	ti->limit_swap_bios = true;
3400	ti->accounts_remapped_io = true;
3401
3402	dm_audit_log_ctr(DM_MSG_PREFIX, ti, result: 1);
3403	return 0;
3404
3405	bad:
3406	dm_audit_log_ctr(DM_MSG_PREFIX, ti, result: 0);
3407	crypt_dtr(ti);
3408	return ret;
3409	}
3410
3411	static int crypt_map(struct dm_target *ti, struct bio *bio)
3412	{
3413	struct dm_crypt_io *io;
3414	struct crypt_config *cc = ti->private;
3415
3416	/*
3417	* If bio is REQ_PREFLUSH or REQ_OP_DISCARD, just bypass crypt queues.
3418	* - for REQ_PREFLUSH device-mapper core ensures that no IO is in-flight
3419	* - for REQ_OP_DISCARD caller must use flush if IO ordering matters
3420	*/
3421	if (unlikely(bio->bi_opf & REQ_PREFLUSH ||
3422	bio_op(bio) == REQ_OP_DISCARD)) {
3423	bio_set_dev(bio, bdev: cc->dev->bdev);
3424	if (bio_sectors(bio))
3425	bio->bi_iter.bi_sector = cc->start +
3426	dm_target_offset(ti, bio->bi_iter.bi_sector);
3427	return DM_MAPIO_REMAPPED;
3428	}
3429
3430	/*
3431	* Check if bio is too large, split as needed.
3432	*/
3433	if (unlikely(bio->bi_iter.bi_size > (BIO_MAX_VECS << PAGE_SHIFT)) &&
3434	(bio_data_dir(bio) == WRITE || cc->on_disk_tag_size))
3435	dm_accept_partial_bio(bio, n_sectors: ((BIO_MAX_VECS << PAGE_SHIFT) >> SECTOR_SHIFT));
3436
3437	/*
3438	* Ensure that bio is a multiple of internal sector encryption size
3439	* and is aligned to this size as defined in IO hints.
3440	*/
3441	if (unlikely((bio->bi_iter.bi_sector & ((cc->sector_size >> SECTOR_SHIFT) - 1)) != 0))
3442	return DM_MAPIO_KILL;
3443
3444	if (unlikely(bio->bi_iter.bi_size & (cc->sector_size - 1)))
3445	return DM_MAPIO_KILL;
3446
3447	io = dm_per_bio_data(bio, data_size: cc->per_bio_data_size);
3448	crypt_io_init(io, cc, bio, dm_target_offset(ti, bio->bi_iter.bi_sector));
3449
3450	if (cc->on_disk_tag_size) {
3451	unsigned int tag_len = cc->on_disk_tag_size * (bio_sectors(bio) >> cc->sector_shift);
3452
3453	if (unlikely(tag_len > KMALLOC_MAX_SIZE))
3454	io->integrity_metadata = NULL;
3455	else
3456	io->integrity_metadata = kmalloc(size: tag_len, GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
3457
3458	if (unlikely(!io->integrity_metadata)) {
3459	if (bio_sectors(bio) > cc->tag_pool_max_sectors)
3460	dm_accept_partial_bio(bio, n_sectors: cc->tag_pool_max_sectors);
3461	io->integrity_metadata = mempool_alloc(pool: &cc->tag_pool, GFP_NOIO);
3462	io->integrity_metadata_from_pool = true;
3463	}
3464	}
3465
3466	if (crypt_integrity_aead(cc))
3467	io->ctx.r.req_aead = (struct aead_request *)(io + 1);
3468	else
3469	io->ctx.r.req = (struct skcipher_request *)(io + 1);
3470
3471	if (bio_data_dir(io->base_bio) == READ) {
3472	if (kcryptd_io_read(io, CRYPT_MAP_READ_GFP))
3473	kcryptd_queue_read(io);
3474	} else
3475	kcryptd_queue_crypt(io);
3476
3477	return DM_MAPIO_SUBMITTED;
3478	}
3479
3480	static char hex2asc(unsigned char c)
3481	{
3482	return c + '0' + ((unsigned int)(9 - c) >> 4 & 0x27);
3483	}
3484
3485	static void crypt_status(struct dm_target *ti, status_type_t type,
3486	unsigned int status_flags, char *result, unsigned int maxlen)
3487	{
3488	struct crypt_config *cc = ti->private;
3489	unsigned int i, sz = 0;
3490	int num_feature_args = 0;
3491
3492	switch (type) {
3493	case STATUSTYPE_INFO:
3494	result[0] = '\0';
3495	break;
3496
3497	case STATUSTYPE_TABLE:
3498	DMEMIT("%s ", cc->cipher_string);
3499
3500	if (cc->key_size > 0) {
3501	if (cc->key_string)
3502	DMEMIT(":%u:%s", cc->key_size, cc->key_string);
3503	else {
3504	for (i = 0; i < cc->key_size; i++) {
3505	DMEMIT("%c%c", hex2asc(cc->key[i] >> 4),
3506	hex2asc(cc->key[i] & 0xf));
3507	}
3508	}
3509	} else
3510	DMEMIT("-");
3511
3512	DMEMIT(" %llu %s %llu", (unsigned long long)cc->iv_offset,
3513	cc->dev->name, (unsigned long long)cc->start);
3514
3515	num_feature_args += !!ti->num_discard_bios;
3516	num_feature_args += test_bit(DM_CRYPT_SAME_CPU, &cc->flags);
3517	num_feature_args += test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags);
3518	num_feature_args += test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags);
3519	num_feature_args += test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags);
3520	num_feature_args += cc->sector_size != (1 << SECTOR_SHIFT);
3521	num_feature_args += test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags);
3522	if (cc->on_disk_tag_size)
3523	num_feature_args++;
3524	if (num_feature_args) {
3525	DMEMIT(" %d", num_feature_args);
3526	if (ti->num_discard_bios)
3527	DMEMIT(" allow_discards");
3528	if (test_bit(DM_CRYPT_SAME_CPU, &cc->flags))
3529	DMEMIT(" same_cpu_crypt");
3530	if (test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags))
3531	DMEMIT(" submit_from_crypt_cpus");
3532	if (test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags))
3533	DMEMIT(" no_read_workqueue");
3534	if (test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags))
3535	DMEMIT(" no_write_workqueue");
3536	if (cc->on_disk_tag_size)
3537	DMEMIT(" integrity:%u:%s", cc->on_disk_tag_size, cc->cipher_auth);
3538	if (cc->sector_size != (1 << SECTOR_SHIFT))
3539	DMEMIT(" sector_size:%d", cc->sector_size);
3540	if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags))
3541	DMEMIT(" iv_large_sectors");
3542	}
3543	break;
3544
3545	case STATUSTYPE_IMA:
3546	DMEMIT_TARGET_NAME_VERSION(ti->type);
3547	DMEMIT(",allow_discards=%c", ti->num_discard_bios ? 'y' : 'n');
3548	DMEMIT(",same_cpu_crypt=%c", test_bit(DM_CRYPT_SAME_CPU, &cc->flags) ? 'y' : 'n');
3549	DMEMIT(",submit_from_crypt_cpus=%c", test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags) ?
3550	'y' : 'n');
3551	DMEMIT(",no_read_workqueue=%c", test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags) ?
3552	'y' : 'n');
3553	DMEMIT(",no_write_workqueue=%c", test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags) ?
3554	'y' : 'n');
3555	DMEMIT(",iv_large_sectors=%c", test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags) ?
3556	'y' : 'n');
3557
3558	if (cc->on_disk_tag_size)
3559	DMEMIT(",integrity_tag_size=%u,cipher_auth=%s",
3560	cc->on_disk_tag_size, cc->cipher_auth);
3561	if (cc->sector_size != (1 << SECTOR_SHIFT))
3562	DMEMIT(",sector_size=%d", cc->sector_size);
3563	if (cc->cipher_string)
3564	DMEMIT(",cipher_string=%s", cc->cipher_string);
3565
3566	DMEMIT(",key_size=%u", cc->key_size);
3567	DMEMIT(",key_parts=%u", cc->key_parts);
3568	DMEMIT(",key_extra_size=%u", cc->key_extra_size);
3569	DMEMIT(",key_mac_size=%u", cc->key_mac_size);
3570	DMEMIT(";");
3571	break;
3572	}
3573	}
3574
3575	static void crypt_postsuspend(struct dm_target *ti)
3576	{
3577	struct crypt_config *cc = ti->private;
3578
3579	set_bit(nr: DM_CRYPT_SUSPENDED, addr: &cc->flags);
3580	}
3581
3582	static int crypt_preresume(struct dm_target *ti)
3583	{
3584	struct crypt_config *cc = ti->private;
3585
3586	if (!test_bit(DM_CRYPT_KEY_VALID, &cc->flags)) {
3587	DMERR("aborting resume - crypt key is not set.");
3588	return -EAGAIN;
3589	}
3590
3591	return 0;
3592	}
3593
3594	static void crypt_resume(struct dm_target *ti)
3595	{
3596	struct crypt_config *cc = ti->private;
3597
3598	clear_bit(nr: DM_CRYPT_SUSPENDED, addr: &cc->flags);
3599	}
3600
3601	/* Message interface
3602	* key set <key>
3603	* key wipe
3604	*/
3605	static int crypt_message(struct dm_target *ti, unsigned int argc, char **argv,
3606	char *result, unsigned int maxlen)
3607	{
3608	struct crypt_config *cc = ti->private;
3609	int key_size, ret = -EINVAL;
3610
3611	if (argc < 2)
3612	goto error;
3613
3614	if (!strcasecmp(s1: argv[0], s2: "key")) {
3615	if (!test_bit(DM_CRYPT_SUSPENDED, &cc->flags)) {
3616	DMWARN("not suspended during key manipulation.");
3617	return -EINVAL;
3618	}
3619	if (argc == 3 && !strcasecmp(s1: argv[1], s2: "set")) {
3620	/* The key size may not be changed. */
3621	key_size = get_key_size(key_string: &argv[2]);
3622	if (key_size < 0 || cc->key_size != key_size) {
3623	memset(argv[2], '0', strlen(argv[2]));
3624	return -EINVAL;
3625	}
3626
3627	ret = crypt_set_key(cc, key: argv[2]);
3628	if (ret)
3629	return ret;
3630	if (cc->iv_gen_ops && cc->iv_gen_ops->init)
3631	ret = cc->iv_gen_ops->init(cc);
3632	/* wipe the kernel key payload copy */
3633	if (cc->key_string)
3634	memset(cc->key, 0, cc->key_size * sizeof(u8));
3635	return ret;
3636	}
3637	if (argc == 2 && !strcasecmp(s1: argv[1], s2: "wipe"))
3638	return crypt_wipe_key(cc);
3639	}
3640
3641	error:
3642	DMWARN("unrecognised message received.");
3643	return -EINVAL;
3644	}
3645
3646	static int crypt_iterate_devices(struct dm_target *ti,
3647	iterate_devices_callout_fn fn, void *data)
3648	{
3649	struct crypt_config *cc = ti->private;
3650
3651	return fn(ti, cc->dev, cc->start, ti->len, data);
3652	}
3653
3654	static void crypt_io_hints(struct dm_target *ti, struct queue_limits *limits)
3655	{
3656	struct crypt_config *cc = ti->private;
3657
3658	/*
3659	* Unfortunate constraint that is required to avoid the potential
3660	* for exceeding underlying device's max_segments limits -- due to
3661	* crypt_alloc_buffer() possibly allocating pages for the encryption
3662	* bio that are not as physically contiguous as the original bio.
3663	*/
3664	limits->max_segment_size = PAGE_SIZE;
3665
3666	limits->logical_block_size =
3667	max_t(unsigned int, limits->logical_block_size, cc->sector_size);
3668	limits->physical_block_size =
3669	max_t(unsigned int, limits->physical_block_size, cc->sector_size);
3670	limits->io_min = max_t(unsigned int, limits->io_min, cc->sector_size);
3671	limits->dma_alignment = limits->logical_block_size - 1;
3672	}
3673
3674	static struct target_type crypt_target = {
3675	.name = "crypt",
3676	.version = {1, 24, 0},
3677	.module = THIS_MODULE,
3678	.ctr = crypt_ctr,
3679	.dtr = crypt_dtr,
3680	.features = DM_TARGET_ZONED_HM,
3681	.report_zones = crypt_report_zones,
3682	.map = crypt_map,
3683	.status = crypt_status,
3684	.postsuspend = crypt_postsuspend,
3685	.preresume = crypt_preresume,
3686	.resume = crypt_resume,
3687	.message = crypt_message,
3688	.iterate_devices = crypt_iterate_devices,
3689	.io_hints = crypt_io_hints,
3690	};
3691	module_dm(crypt);
3692
3693	MODULE_AUTHOR("Jana Saout <jana@saout.de>");
3694	MODULE_DESCRIPTION(DM_NAME " target for transparent encryption / decryption");
3695	MODULE_LICENSE("GPL");
3696