1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Inline encryption support for fscrypt |
4 | * |
5 | * Copyright 2019 Google LLC |
6 | */ |
7 | |
8 | /* |
9 | * With "inline encryption", the block layer handles the decryption/encryption |
10 | * as part of the bio, instead of the filesystem doing the crypto itself via |
11 | * crypto API. See Documentation/block/inline-encryption.rst. fscrypt still |
12 | * provides the key and IV to use. |
13 | */ |
14 | |
15 | #include <linux/blk-crypto.h> |
16 | #include <linux/blkdev.h> |
17 | #include <linux/buffer_head.h> |
18 | #include <linux/sched/mm.h> |
19 | #include <linux/slab.h> |
20 | #include <linux/uio.h> |
21 | |
22 | #include "fscrypt_private.h" |
23 | |
24 | static struct block_device **fscrypt_get_devices(struct super_block *sb, |
25 | unsigned int *num_devs) |
26 | { |
27 | struct block_device **devs; |
28 | |
29 | if (sb->s_cop->get_devices) { |
30 | devs = sb->s_cop->get_devices(sb, num_devs); |
31 | if (devs) |
32 | return devs; |
33 | } |
34 | devs = kmalloc(size: sizeof(*devs), GFP_KERNEL); |
35 | if (!devs) |
36 | return ERR_PTR(error: -ENOMEM); |
37 | devs[0] = sb->s_bdev; |
38 | *num_devs = 1; |
39 | return devs; |
40 | } |
41 | |
42 | static unsigned int fscrypt_get_dun_bytes(const struct fscrypt_inode_info *ci) |
43 | { |
44 | const struct super_block *sb = ci->ci_inode->i_sb; |
45 | unsigned int flags = fscrypt_policy_flags(policy: &ci->ci_policy); |
46 | int dun_bits; |
47 | |
48 | if (flags & FSCRYPT_POLICY_FLAG_DIRECT_KEY) |
49 | return offsetofend(union fscrypt_iv, nonce); |
50 | |
51 | if (flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64) |
52 | return sizeof(__le64); |
53 | |
54 | if (flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32) |
55 | return sizeof(__le32); |
56 | |
57 | /* Default case: IVs are just the file data unit index */ |
58 | dun_bits = fscrypt_max_file_dun_bits(sb, du_bits: ci->ci_data_unit_bits); |
59 | return DIV_ROUND_UP(dun_bits, 8); |
60 | } |
61 | |
62 | /* |
63 | * Log a message when starting to use blk-crypto (native) or blk-crypto-fallback |
64 | * for an encryption mode for the first time. This is the blk-crypto |
65 | * counterpart to the message logged when starting to use the crypto API for the |
66 | * first time. A limitation is that these messages don't convey which specific |
67 | * filesystems or files are using each implementation. However, *usually* |
68 | * systems use just one implementation per mode, which makes these messages |
69 | * helpful for debugging problems where the "wrong" implementation is used. |
70 | */ |
71 | static void fscrypt_log_blk_crypto_impl(struct fscrypt_mode *mode, |
72 | struct block_device **devs, |
73 | unsigned int num_devs, |
74 | const struct blk_crypto_config *cfg) |
75 | { |
76 | unsigned int i; |
77 | |
78 | for (i = 0; i < num_devs; i++) { |
79 | if (!IS_ENABLED(CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK) || |
80 | blk_crypto_config_supported_natively(bdev: devs[i], cfg)) { |
81 | if (!xchg(&mode->logged_blk_crypto_native, 1)) |
82 | pr_info("fscrypt: %s using blk-crypto (native)\n" , |
83 | mode->friendly_name); |
84 | } else if (!xchg(&mode->logged_blk_crypto_fallback, 1)) { |
85 | pr_info("fscrypt: %s using blk-crypto-fallback\n" , |
86 | mode->friendly_name); |
87 | } |
88 | } |
89 | } |
90 | |
91 | /* Enable inline encryption for this file if supported. */ |
92 | int fscrypt_select_encryption_impl(struct fscrypt_inode_info *ci) |
93 | { |
94 | const struct inode *inode = ci->ci_inode; |
95 | struct super_block *sb = inode->i_sb; |
96 | struct blk_crypto_config crypto_cfg; |
97 | struct block_device **devs; |
98 | unsigned int num_devs; |
99 | unsigned int i; |
100 | |
101 | /* The file must need contents encryption, not filenames encryption */ |
102 | if (!S_ISREG(inode->i_mode)) |
103 | return 0; |
104 | |
105 | /* The crypto mode must have a blk-crypto counterpart */ |
106 | if (ci->ci_mode->blk_crypto_mode == BLK_ENCRYPTION_MODE_INVALID) |
107 | return 0; |
108 | |
109 | /* The filesystem must be mounted with -o inlinecrypt */ |
110 | if (!(sb->s_flags & SB_INLINECRYPT)) |
111 | return 0; |
112 | |
113 | /* |
114 | * When a page contains multiple logically contiguous filesystem blocks, |
115 | * some filesystem code only calls fscrypt_mergeable_bio() for the first |
116 | * block in the page. This is fine for most of fscrypt's IV generation |
117 | * strategies, where contiguous blocks imply contiguous IVs. But it |
118 | * doesn't work with IV_INO_LBLK_32. For now, simply exclude |
119 | * IV_INO_LBLK_32 with blocksize != PAGE_SIZE from inline encryption. |
120 | */ |
121 | if ((fscrypt_policy_flags(policy: &ci->ci_policy) & |
122 | FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32) && |
123 | sb->s_blocksize != PAGE_SIZE) |
124 | return 0; |
125 | |
126 | /* |
127 | * On all the filesystem's block devices, blk-crypto must support the |
128 | * crypto configuration that the file would use. |
129 | */ |
130 | crypto_cfg.crypto_mode = ci->ci_mode->blk_crypto_mode; |
131 | crypto_cfg.data_unit_size = 1U << ci->ci_data_unit_bits; |
132 | crypto_cfg.dun_bytes = fscrypt_get_dun_bytes(ci); |
133 | |
134 | devs = fscrypt_get_devices(sb, num_devs: &num_devs); |
135 | if (IS_ERR(ptr: devs)) |
136 | return PTR_ERR(ptr: devs); |
137 | |
138 | for (i = 0; i < num_devs; i++) { |
139 | if (!blk_crypto_config_supported(bdev: devs[i], cfg: &crypto_cfg)) |
140 | goto out_free_devs; |
141 | } |
142 | |
143 | fscrypt_log_blk_crypto_impl(mode: ci->ci_mode, devs, num_devs, cfg: &crypto_cfg); |
144 | |
145 | ci->ci_inlinecrypt = true; |
146 | out_free_devs: |
147 | kfree(objp: devs); |
148 | |
149 | return 0; |
150 | } |
151 | |
152 | int fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key, |
153 | const u8 *raw_key, |
154 | const struct fscrypt_inode_info *ci) |
155 | { |
156 | const struct inode *inode = ci->ci_inode; |
157 | struct super_block *sb = inode->i_sb; |
158 | enum blk_crypto_mode_num crypto_mode = ci->ci_mode->blk_crypto_mode; |
159 | struct blk_crypto_key *blk_key; |
160 | struct block_device **devs; |
161 | unsigned int num_devs; |
162 | unsigned int i; |
163 | int err; |
164 | |
165 | blk_key = kmalloc(size: sizeof(*blk_key), GFP_KERNEL); |
166 | if (!blk_key) |
167 | return -ENOMEM; |
168 | |
169 | err = blk_crypto_init_key(blk_key, raw_key, crypto_mode, |
170 | dun_bytes: fscrypt_get_dun_bytes(ci), |
171 | data_unit_size: 1U << ci->ci_data_unit_bits); |
172 | if (err) { |
173 | fscrypt_err(inode, "error %d initializing blk-crypto key" , err); |
174 | goto fail; |
175 | } |
176 | |
177 | /* Start using blk-crypto on all the filesystem's block devices. */ |
178 | devs = fscrypt_get_devices(sb, num_devs: &num_devs); |
179 | if (IS_ERR(ptr: devs)) { |
180 | err = PTR_ERR(ptr: devs); |
181 | goto fail; |
182 | } |
183 | for (i = 0; i < num_devs; i++) { |
184 | err = blk_crypto_start_using_key(bdev: devs[i], key: blk_key); |
185 | if (err) |
186 | break; |
187 | } |
188 | kfree(objp: devs); |
189 | if (err) { |
190 | fscrypt_err(inode, "error %d starting to use blk-crypto" , err); |
191 | goto fail; |
192 | } |
193 | |
194 | /* |
195 | * Pairs with the smp_load_acquire() in fscrypt_is_key_prepared(). |
196 | * I.e., here we publish ->blk_key with a RELEASE barrier so that |
197 | * concurrent tasks can ACQUIRE it. Note that this concurrency is only |
198 | * possible for per-mode keys, not for per-file keys. |
199 | */ |
200 | smp_store_release(&prep_key->blk_key, blk_key); |
201 | return 0; |
202 | |
203 | fail: |
204 | kfree_sensitive(objp: blk_key); |
205 | return err; |
206 | } |
207 | |
208 | void fscrypt_destroy_inline_crypt_key(struct super_block *sb, |
209 | struct fscrypt_prepared_key *prep_key) |
210 | { |
211 | struct blk_crypto_key *blk_key = prep_key->blk_key; |
212 | struct block_device **devs; |
213 | unsigned int num_devs; |
214 | unsigned int i; |
215 | |
216 | if (!blk_key) |
217 | return; |
218 | |
219 | /* Evict the key from all the filesystem's block devices. */ |
220 | devs = fscrypt_get_devices(sb, num_devs: &num_devs); |
221 | if (!IS_ERR(ptr: devs)) { |
222 | for (i = 0; i < num_devs; i++) |
223 | blk_crypto_evict_key(bdev: devs[i], key: blk_key); |
224 | kfree(objp: devs); |
225 | } |
226 | kfree_sensitive(objp: blk_key); |
227 | } |
228 | |
229 | bool __fscrypt_inode_uses_inline_crypto(const struct inode *inode) |
230 | { |
231 | return inode->i_crypt_info->ci_inlinecrypt; |
232 | } |
233 | EXPORT_SYMBOL_GPL(__fscrypt_inode_uses_inline_crypto); |
234 | |
235 | static void fscrypt_generate_dun(const struct fscrypt_inode_info *ci, |
236 | u64 lblk_num, |
237 | u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE]) |
238 | { |
239 | u64 index = lblk_num << ci->ci_data_units_per_block_bits; |
240 | union fscrypt_iv iv; |
241 | int i; |
242 | |
243 | fscrypt_generate_iv(iv: &iv, index, ci); |
244 | |
245 | BUILD_BUG_ON(FSCRYPT_MAX_IV_SIZE > BLK_CRYPTO_MAX_IV_SIZE); |
246 | memset(dun, 0, BLK_CRYPTO_MAX_IV_SIZE); |
247 | for (i = 0; i < ci->ci_mode->ivsize/sizeof(dun[0]); i++) |
248 | dun[i] = le64_to_cpu(iv.dun[i]); |
249 | } |
250 | |
251 | /** |
252 | * fscrypt_set_bio_crypt_ctx() - prepare a file contents bio for inline crypto |
253 | * @bio: a bio which will eventually be submitted to the file |
254 | * @inode: the file's inode |
255 | * @first_lblk: the first file logical block number in the I/O |
256 | * @gfp_mask: memory allocation flags - these must be a waiting mask so that |
257 | * bio_crypt_set_ctx can't fail. |
258 | * |
259 | * If the contents of the file should be encrypted (or decrypted) with inline |
260 | * encryption, then assign the appropriate encryption context to the bio. |
261 | * |
262 | * Normally the bio should be newly allocated (i.e. no pages added yet), as |
263 | * otherwise fscrypt_mergeable_bio() won't work as intended. |
264 | * |
265 | * The encryption context will be freed automatically when the bio is freed. |
266 | */ |
267 | void fscrypt_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, |
268 | u64 first_lblk, gfp_t gfp_mask) |
269 | { |
270 | const struct fscrypt_inode_info *ci; |
271 | u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; |
272 | |
273 | if (!fscrypt_inode_uses_inline_crypto(inode)) |
274 | return; |
275 | ci = inode->i_crypt_info; |
276 | |
277 | fscrypt_generate_dun(ci, lblk_num: first_lblk, dun); |
278 | bio_crypt_set_ctx(bio, key: ci->ci_enc_key.blk_key, dun, gfp_mask); |
279 | } |
280 | EXPORT_SYMBOL_GPL(fscrypt_set_bio_crypt_ctx); |
281 | |
282 | /* Extract the inode and logical block number from a buffer_head. */ |
283 | static bool bh_get_inode_and_lblk_num(const struct buffer_head *bh, |
284 | const struct inode **inode_ret, |
285 | u64 *lblk_num_ret) |
286 | { |
287 | struct page *page = bh->b_page; |
288 | const struct address_space *mapping; |
289 | const struct inode *inode; |
290 | |
291 | /* |
292 | * The ext4 journal (jbd2) can submit a buffer_head it directly created |
293 | * for a non-pagecache page. fscrypt doesn't care about these. |
294 | */ |
295 | mapping = page_mapping(page); |
296 | if (!mapping) |
297 | return false; |
298 | inode = mapping->host; |
299 | |
300 | *inode_ret = inode; |
301 | *lblk_num_ret = ((u64)page->index << (PAGE_SHIFT - inode->i_blkbits)) + |
302 | (bh_offset(bh) >> inode->i_blkbits); |
303 | return true; |
304 | } |
305 | |
306 | /** |
307 | * fscrypt_set_bio_crypt_ctx_bh() - prepare a file contents bio for inline |
308 | * crypto |
309 | * @bio: a bio which will eventually be submitted to the file |
310 | * @first_bh: the first buffer_head for which I/O will be submitted |
311 | * @gfp_mask: memory allocation flags |
312 | * |
313 | * Same as fscrypt_set_bio_crypt_ctx(), except this takes a buffer_head instead |
314 | * of an inode and block number directly. |
315 | */ |
316 | void fscrypt_set_bio_crypt_ctx_bh(struct bio *bio, |
317 | const struct buffer_head *first_bh, |
318 | gfp_t gfp_mask) |
319 | { |
320 | const struct inode *inode; |
321 | u64 first_lblk; |
322 | |
323 | if (bh_get_inode_and_lblk_num(bh: first_bh, inode_ret: &inode, lblk_num_ret: &first_lblk)) |
324 | fscrypt_set_bio_crypt_ctx(bio, inode, first_lblk, gfp_mask); |
325 | } |
326 | EXPORT_SYMBOL_GPL(fscrypt_set_bio_crypt_ctx_bh); |
327 | |
328 | /** |
329 | * fscrypt_mergeable_bio() - test whether data can be added to a bio |
330 | * @bio: the bio being built up |
331 | * @inode: the inode for the next part of the I/O |
332 | * @next_lblk: the next file logical block number in the I/O |
333 | * |
334 | * When building a bio which may contain data which should undergo inline |
335 | * encryption (or decryption) via fscrypt, filesystems should call this function |
336 | * to ensure that the resulting bio contains only contiguous data unit numbers. |
337 | * This will return false if the next part of the I/O cannot be merged with the |
338 | * bio because either the encryption key would be different or the encryption |
339 | * data unit numbers would be discontiguous. |
340 | * |
341 | * fscrypt_set_bio_crypt_ctx() must have already been called on the bio. |
342 | * |
343 | * This function isn't required in cases where crypto-mergeability is ensured in |
344 | * another way, such as I/O targeting only a single file (and thus a single key) |
345 | * combined with fscrypt_limit_io_blocks() to ensure DUN contiguity. |
346 | * |
347 | * Return: true iff the I/O is mergeable |
348 | */ |
349 | bool fscrypt_mergeable_bio(struct bio *bio, const struct inode *inode, |
350 | u64 next_lblk) |
351 | { |
352 | const struct bio_crypt_ctx *bc = bio->bi_crypt_context; |
353 | u64 next_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; |
354 | |
355 | if (!!bc != fscrypt_inode_uses_inline_crypto(inode)) |
356 | return false; |
357 | if (!bc) |
358 | return true; |
359 | |
360 | /* |
361 | * Comparing the key pointers is good enough, as all I/O for each key |
362 | * uses the same pointer. I.e., there's currently no need to support |
363 | * merging requests where the keys are the same but the pointers differ. |
364 | */ |
365 | if (bc->bc_key != inode->i_crypt_info->ci_enc_key.blk_key) |
366 | return false; |
367 | |
368 | fscrypt_generate_dun(ci: inode->i_crypt_info, lblk_num: next_lblk, dun: next_dun); |
369 | return bio_crypt_dun_is_contiguous(bc, bytes: bio->bi_iter.bi_size, next_dun); |
370 | } |
371 | EXPORT_SYMBOL_GPL(fscrypt_mergeable_bio); |
372 | |
373 | /** |
374 | * fscrypt_mergeable_bio_bh() - test whether data can be added to a bio |
375 | * @bio: the bio being built up |
376 | * @next_bh: the next buffer_head for which I/O will be submitted |
377 | * |
378 | * Same as fscrypt_mergeable_bio(), except this takes a buffer_head instead of |
379 | * an inode and block number directly. |
380 | * |
381 | * Return: true iff the I/O is mergeable |
382 | */ |
383 | bool fscrypt_mergeable_bio_bh(struct bio *bio, |
384 | const struct buffer_head *next_bh) |
385 | { |
386 | const struct inode *inode; |
387 | u64 next_lblk; |
388 | |
389 | if (!bh_get_inode_and_lblk_num(bh: next_bh, inode_ret: &inode, lblk_num_ret: &next_lblk)) |
390 | return !bio->bi_crypt_context; |
391 | |
392 | return fscrypt_mergeable_bio(bio, inode, next_lblk); |
393 | } |
394 | EXPORT_SYMBOL_GPL(fscrypt_mergeable_bio_bh); |
395 | |
396 | /** |
397 | * fscrypt_dio_supported() - check whether DIO (direct I/O) is supported on an |
398 | * inode, as far as encryption is concerned |
399 | * @inode: the inode in question |
400 | * |
401 | * Return: %true if there are no encryption constraints that prevent DIO from |
402 | * being supported; %false if DIO is unsupported. (Note that in the |
403 | * %true case, the filesystem might have other, non-encryption-related |
404 | * constraints that prevent DIO from actually being supported. Also, on |
405 | * encrypted files the filesystem is still responsible for only allowing |
406 | * DIO when requests are filesystem-block-aligned.) |
407 | */ |
408 | bool fscrypt_dio_supported(struct inode *inode) |
409 | { |
410 | int err; |
411 | |
412 | /* If the file is unencrypted, no veto from us. */ |
413 | if (!fscrypt_needs_contents_encryption(inode)) |
414 | return true; |
415 | |
416 | /* |
417 | * We only support DIO with inline crypto, not fs-layer crypto. |
418 | * |
419 | * To determine whether the inode is using inline crypto, we have to set |
420 | * up the key if it wasn't already done. This is because in the current |
421 | * design of fscrypt, the decision of whether to use inline crypto or |
422 | * not isn't made until the inode's encryption key is being set up. In |
423 | * the DIO read/write case, the key will always be set up already, since |
424 | * the file will be open. But in the case of statx(), the key might not |
425 | * be set up yet, as the file might not have been opened yet. |
426 | */ |
427 | err = fscrypt_require_key(inode); |
428 | if (err) { |
429 | /* |
430 | * Key unavailable or couldn't be set up. This edge case isn't |
431 | * worth worrying about; just report that DIO is unsupported. |
432 | */ |
433 | return false; |
434 | } |
435 | return fscrypt_inode_uses_inline_crypto(inode); |
436 | } |
437 | EXPORT_SYMBOL_GPL(fscrypt_dio_supported); |
438 | |
439 | /** |
440 | * fscrypt_limit_io_blocks() - limit I/O blocks to avoid discontiguous DUNs |
441 | * @inode: the file on which I/O is being done |
442 | * @lblk: the block at which the I/O is being started from |
443 | * @nr_blocks: the number of blocks we want to submit starting at @lblk |
444 | * |
445 | * Determine the limit to the number of blocks that can be submitted in a bio |
446 | * targeting @lblk without causing a data unit number (DUN) discontiguity. |
447 | * |
448 | * This is normally just @nr_blocks, as normally the DUNs just increment along |
449 | * with the logical blocks. (Or the file is not encrypted.) |
450 | * |
451 | * In rare cases, fscrypt can be using an IV generation method that allows the |
452 | * DUN to wrap around within logically contiguous blocks, and that wraparound |
453 | * will occur. If this happens, a value less than @nr_blocks will be returned |
454 | * so that the wraparound doesn't occur in the middle of a bio, which would |
455 | * cause encryption/decryption to produce wrong results. |
456 | * |
457 | * Return: the actual number of blocks that can be submitted |
458 | */ |
459 | u64 fscrypt_limit_io_blocks(const struct inode *inode, u64 lblk, u64 nr_blocks) |
460 | { |
461 | const struct fscrypt_inode_info *ci; |
462 | u32 dun; |
463 | |
464 | if (!fscrypt_inode_uses_inline_crypto(inode)) |
465 | return nr_blocks; |
466 | |
467 | if (nr_blocks <= 1) |
468 | return nr_blocks; |
469 | |
470 | ci = inode->i_crypt_info; |
471 | if (!(fscrypt_policy_flags(policy: &ci->ci_policy) & |
472 | FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32)) |
473 | return nr_blocks; |
474 | |
475 | /* With IV_INO_LBLK_32, the DUN can wrap around from U32_MAX to 0. */ |
476 | |
477 | dun = ci->ci_hashed_ino + lblk; |
478 | |
479 | return min_t(u64, nr_blocks, (u64)U32_MAX + 1 - dun); |
480 | } |
481 | EXPORT_SYMBOL_GPL(fscrypt_limit_io_blocks); |
482 | |