1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * This contains encryption functions for per-file encryption. |
4 | * |
5 | * Copyright (C) 2015, Google, Inc. |
6 | * Copyright (C) 2015, Motorola Mobility |
7 | * |
8 | * Written by Michael Halcrow, 2014. |
9 | * |
10 | * Filename encryption additions |
11 | * Uday Savagaonkar, 2014 |
12 | * Encryption policy handling additions |
13 | * Ildar Muslukhov, 2014 |
14 | * Add fscrypt_pullback_bio_page() |
15 | * Jaegeuk Kim, 2015. |
16 | * |
17 | * This has not yet undergone a rigorous security audit. |
18 | * |
19 | * The usage of AES-XTS should conform to recommendations in NIST |
20 | * Special Publication 800-38E and IEEE P1619/D16. |
21 | */ |
22 | |
23 | #include <linux/pagemap.h> |
24 | #include <linux/mempool.h> |
25 | #include <linux/module.h> |
26 | #include <linux/scatterlist.h> |
27 | #include <linux/ratelimit.h> |
28 | #include <crypto/skcipher.h> |
29 | #include "fscrypt_private.h" |
30 | |
31 | static unsigned int num_prealloc_crypto_pages = 32; |
32 | |
33 | module_param(num_prealloc_crypto_pages, uint, 0444); |
34 | MODULE_PARM_DESC(num_prealloc_crypto_pages, |
35 | "Number of crypto pages to preallocate" ); |
36 | |
37 | static mempool_t *fscrypt_bounce_page_pool = NULL; |
38 | |
39 | static struct workqueue_struct *fscrypt_read_workqueue; |
40 | static DEFINE_MUTEX(fscrypt_init_mutex); |
41 | |
42 | struct kmem_cache *fscrypt_inode_info_cachep; |
43 | |
44 | void fscrypt_enqueue_decrypt_work(struct work_struct *work) |
45 | { |
46 | queue_work(wq: fscrypt_read_workqueue, work); |
47 | } |
48 | EXPORT_SYMBOL(fscrypt_enqueue_decrypt_work); |
49 | |
50 | struct page *fscrypt_alloc_bounce_page(gfp_t gfp_flags) |
51 | { |
52 | if (WARN_ON_ONCE(!fscrypt_bounce_page_pool)) { |
53 | /* |
54 | * Oops, the filesystem called a function that uses the bounce |
55 | * page pool, but it didn't set needs_bounce_pages. |
56 | */ |
57 | return NULL; |
58 | } |
59 | return mempool_alloc(pool: fscrypt_bounce_page_pool, gfp_mask: gfp_flags); |
60 | } |
61 | |
62 | /** |
63 | * fscrypt_free_bounce_page() - free a ciphertext bounce page |
64 | * @bounce_page: the bounce page to free, or NULL |
65 | * |
66 | * Free a bounce page that was allocated by fscrypt_encrypt_pagecache_blocks(), |
67 | * or by fscrypt_alloc_bounce_page() directly. |
68 | */ |
69 | void fscrypt_free_bounce_page(struct page *bounce_page) |
70 | { |
71 | if (!bounce_page) |
72 | return; |
73 | set_page_private(page: bounce_page, private: (unsigned long)NULL); |
74 | ClearPagePrivate(page: bounce_page); |
75 | mempool_free(element: bounce_page, pool: fscrypt_bounce_page_pool); |
76 | } |
77 | EXPORT_SYMBOL(fscrypt_free_bounce_page); |
78 | |
79 | /* |
80 | * Generate the IV for the given data unit index within the given file. |
81 | * For filenames encryption, index == 0. |
82 | * |
83 | * Keep this in sync with fscrypt_limit_io_blocks(). fscrypt_limit_io_blocks() |
84 | * needs to know about any IV generation methods where the low bits of IV don't |
85 | * simply contain the data unit index (e.g., IV_INO_LBLK_32). |
86 | */ |
87 | void fscrypt_generate_iv(union fscrypt_iv *iv, u64 index, |
88 | const struct fscrypt_inode_info *ci) |
89 | { |
90 | u8 flags = fscrypt_policy_flags(policy: &ci->ci_policy); |
91 | |
92 | memset(iv, 0, ci->ci_mode->ivsize); |
93 | |
94 | if (flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64) { |
95 | WARN_ON_ONCE(index > U32_MAX); |
96 | WARN_ON_ONCE(ci->ci_inode->i_ino > U32_MAX); |
97 | index |= (u64)ci->ci_inode->i_ino << 32; |
98 | } else if (flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32) { |
99 | WARN_ON_ONCE(index > U32_MAX); |
100 | index = (u32)(ci->ci_hashed_ino + index); |
101 | } else if (flags & FSCRYPT_POLICY_FLAG_DIRECT_KEY) { |
102 | memcpy(iv->nonce, ci->ci_nonce, FSCRYPT_FILE_NONCE_SIZE); |
103 | } |
104 | iv->index = cpu_to_le64(index); |
105 | } |
106 | |
107 | /* Encrypt or decrypt a single "data unit" of file contents. */ |
108 | int fscrypt_crypt_data_unit(const struct fscrypt_inode_info *ci, |
109 | fscrypt_direction_t rw, u64 index, |
110 | struct page *src_page, struct page *dest_page, |
111 | unsigned int len, unsigned int offs, |
112 | gfp_t gfp_flags) |
113 | { |
114 | union fscrypt_iv iv; |
115 | struct skcipher_request *req = NULL; |
116 | DECLARE_CRYPTO_WAIT(wait); |
117 | struct scatterlist dst, src; |
118 | struct crypto_skcipher *tfm = ci->ci_enc_key.tfm; |
119 | int res = 0; |
120 | |
121 | if (WARN_ON_ONCE(len <= 0)) |
122 | return -EINVAL; |
123 | if (WARN_ON_ONCE(len % FSCRYPT_CONTENTS_ALIGNMENT != 0)) |
124 | return -EINVAL; |
125 | |
126 | fscrypt_generate_iv(iv: &iv, index, ci); |
127 | |
128 | req = skcipher_request_alloc(tfm, gfp: gfp_flags); |
129 | if (!req) |
130 | return -ENOMEM; |
131 | |
132 | skcipher_request_set_callback( |
133 | req, CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP, |
134 | compl: crypto_req_done, data: &wait); |
135 | |
136 | sg_init_table(&dst, 1); |
137 | sg_set_page(sg: &dst, page: dest_page, len, offset: offs); |
138 | sg_init_table(&src, 1); |
139 | sg_set_page(sg: &src, page: src_page, len, offset: offs); |
140 | skcipher_request_set_crypt(req, src: &src, dst: &dst, cryptlen: len, iv: &iv); |
141 | if (rw == FS_DECRYPT) |
142 | res = crypto_wait_req(err: crypto_skcipher_decrypt(req), wait: &wait); |
143 | else |
144 | res = crypto_wait_req(err: crypto_skcipher_encrypt(req), wait: &wait); |
145 | skcipher_request_free(req); |
146 | if (res) { |
147 | fscrypt_err(ci->ci_inode, |
148 | "%scryption failed for data unit %llu: %d" , |
149 | (rw == FS_DECRYPT ? "De" : "En" ), index, res); |
150 | return res; |
151 | } |
152 | return 0; |
153 | } |
154 | |
155 | /** |
156 | * fscrypt_encrypt_pagecache_blocks() - Encrypt data from a pagecache page |
157 | * @page: the locked pagecache page containing the data to encrypt |
158 | * @len: size of the data to encrypt, in bytes |
159 | * @offs: offset within @page of the data to encrypt, in bytes |
160 | * @gfp_flags: memory allocation flags; see details below |
161 | * |
162 | * This allocates a new bounce page and encrypts the given data into it. The |
163 | * length and offset of the data must be aligned to the file's crypto data unit |
164 | * size. Alignment to the filesystem block size fulfills this requirement, as |
165 | * the filesystem block size is always a multiple of the data unit size. |
166 | * |
167 | * In the bounce page, the ciphertext data will be located at the same offset at |
168 | * which the plaintext data was located in the source page. Any other parts of |
169 | * the bounce page will be left uninitialized. |
170 | * |
171 | * This is for use by the filesystem's ->writepages() method. |
172 | * |
173 | * The bounce page allocation is mempool-backed, so it will always succeed when |
174 | * @gfp_flags includes __GFP_DIRECT_RECLAIM, e.g. when it's GFP_NOFS. However, |
175 | * only the first page of each bio can be allocated this way. To prevent |
176 | * deadlocks, for any additional pages a mask like GFP_NOWAIT must be used. |
177 | * |
178 | * Return: the new encrypted bounce page on success; an ERR_PTR() on failure |
179 | */ |
180 | struct page *fscrypt_encrypt_pagecache_blocks(struct page *page, |
181 | unsigned int len, |
182 | unsigned int offs, |
183 | gfp_t gfp_flags) |
184 | |
185 | { |
186 | const struct inode *inode = page->mapping->host; |
187 | const struct fscrypt_inode_info *ci = inode->i_crypt_info; |
188 | const unsigned int du_bits = ci->ci_data_unit_bits; |
189 | const unsigned int du_size = 1U << du_bits; |
190 | struct page *ciphertext_page; |
191 | u64 index = ((u64)page->index << (PAGE_SHIFT - du_bits)) + |
192 | (offs >> du_bits); |
193 | unsigned int i; |
194 | int err; |
195 | |
196 | if (WARN_ON_ONCE(!PageLocked(page))) |
197 | return ERR_PTR(error: -EINVAL); |
198 | |
199 | if (WARN_ON_ONCE(len <= 0 || !IS_ALIGNED(len | offs, du_size))) |
200 | return ERR_PTR(error: -EINVAL); |
201 | |
202 | ciphertext_page = fscrypt_alloc_bounce_page(gfp_flags); |
203 | if (!ciphertext_page) |
204 | return ERR_PTR(error: -ENOMEM); |
205 | |
206 | for (i = offs; i < offs + len; i += du_size, index++) { |
207 | err = fscrypt_crypt_data_unit(ci, rw: FS_ENCRYPT, index, |
208 | src_page: page, dest_page: ciphertext_page, |
209 | len: du_size, offs: i, gfp_flags); |
210 | if (err) { |
211 | fscrypt_free_bounce_page(ciphertext_page); |
212 | return ERR_PTR(error: err); |
213 | } |
214 | } |
215 | SetPagePrivate(ciphertext_page); |
216 | set_page_private(page: ciphertext_page, private: (unsigned long)page); |
217 | return ciphertext_page; |
218 | } |
219 | EXPORT_SYMBOL(fscrypt_encrypt_pagecache_blocks); |
220 | |
221 | /** |
222 | * fscrypt_encrypt_block_inplace() - Encrypt a filesystem block in-place |
223 | * @inode: The inode to which this block belongs |
224 | * @page: The page containing the block to encrypt |
225 | * @len: Size of block to encrypt. This must be a multiple of |
226 | * FSCRYPT_CONTENTS_ALIGNMENT. |
227 | * @offs: Byte offset within @page at which the block to encrypt begins |
228 | * @lblk_num: Filesystem logical block number of the block, i.e. the 0-based |
229 | * number of the block within the file |
230 | * @gfp_flags: Memory allocation flags |
231 | * |
232 | * Encrypt a possibly-compressed filesystem block that is located in an |
233 | * arbitrary page, not necessarily in the original pagecache page. The @inode |
234 | * and @lblk_num must be specified, as they can't be determined from @page. |
235 | * |
236 | * This is not compatible with fscrypt_operations::supports_subblock_data_units. |
237 | * |
238 | * Return: 0 on success; -errno on failure |
239 | */ |
240 | int fscrypt_encrypt_block_inplace(const struct inode *inode, struct page *page, |
241 | unsigned int len, unsigned int offs, |
242 | u64 lblk_num, gfp_t gfp_flags) |
243 | { |
244 | if (WARN_ON_ONCE(inode->i_sb->s_cop->supports_subblock_data_units)) |
245 | return -EOPNOTSUPP; |
246 | return fscrypt_crypt_data_unit(ci: inode->i_crypt_info, rw: FS_ENCRYPT, |
247 | index: lblk_num, src_page: page, dest_page: page, len, offs, |
248 | gfp_flags); |
249 | } |
250 | EXPORT_SYMBOL(fscrypt_encrypt_block_inplace); |
251 | |
252 | /** |
253 | * fscrypt_decrypt_pagecache_blocks() - Decrypt data from a pagecache folio |
254 | * @folio: the pagecache folio containing the data to decrypt |
255 | * @len: size of the data to decrypt, in bytes |
256 | * @offs: offset within @folio of the data to decrypt, in bytes |
257 | * |
258 | * Decrypt data that has just been read from an encrypted file. The data must |
259 | * be located in a pagecache folio that is still locked and not yet uptodate. |
260 | * The length and offset of the data must be aligned to the file's crypto data |
261 | * unit size. Alignment to the filesystem block size fulfills this requirement, |
262 | * as the filesystem block size is always a multiple of the data unit size. |
263 | * |
264 | * Return: 0 on success; -errno on failure |
265 | */ |
266 | int fscrypt_decrypt_pagecache_blocks(struct folio *folio, size_t len, |
267 | size_t offs) |
268 | { |
269 | const struct inode *inode = folio->mapping->host; |
270 | const struct fscrypt_inode_info *ci = inode->i_crypt_info; |
271 | const unsigned int du_bits = ci->ci_data_unit_bits; |
272 | const unsigned int du_size = 1U << du_bits; |
273 | u64 index = ((u64)folio->index << (PAGE_SHIFT - du_bits)) + |
274 | (offs >> du_bits); |
275 | size_t i; |
276 | int err; |
277 | |
278 | if (WARN_ON_ONCE(!folio_test_locked(folio))) |
279 | return -EINVAL; |
280 | |
281 | if (WARN_ON_ONCE(len <= 0 || !IS_ALIGNED(len | offs, du_size))) |
282 | return -EINVAL; |
283 | |
284 | for (i = offs; i < offs + len; i += du_size, index++) { |
285 | struct page *page = folio_page(folio, i >> PAGE_SHIFT); |
286 | |
287 | err = fscrypt_crypt_data_unit(ci, rw: FS_DECRYPT, index, src_page: page, |
288 | dest_page: page, len: du_size, offs: i & ~PAGE_MASK, |
289 | GFP_NOFS); |
290 | if (err) |
291 | return err; |
292 | } |
293 | return 0; |
294 | } |
295 | EXPORT_SYMBOL(fscrypt_decrypt_pagecache_blocks); |
296 | |
297 | /** |
298 | * fscrypt_decrypt_block_inplace() - Decrypt a filesystem block in-place |
299 | * @inode: The inode to which this block belongs |
300 | * @page: The page containing the block to decrypt |
301 | * @len: Size of block to decrypt. This must be a multiple of |
302 | * FSCRYPT_CONTENTS_ALIGNMENT. |
303 | * @offs: Byte offset within @page at which the block to decrypt begins |
304 | * @lblk_num: Filesystem logical block number of the block, i.e. the 0-based |
305 | * number of the block within the file |
306 | * |
307 | * Decrypt a possibly-compressed filesystem block that is located in an |
308 | * arbitrary page, not necessarily in the original pagecache page. The @inode |
309 | * and @lblk_num must be specified, as they can't be determined from @page. |
310 | * |
311 | * This is not compatible with fscrypt_operations::supports_subblock_data_units. |
312 | * |
313 | * Return: 0 on success; -errno on failure |
314 | */ |
315 | int fscrypt_decrypt_block_inplace(const struct inode *inode, struct page *page, |
316 | unsigned int len, unsigned int offs, |
317 | u64 lblk_num) |
318 | { |
319 | if (WARN_ON_ONCE(inode->i_sb->s_cop->supports_subblock_data_units)) |
320 | return -EOPNOTSUPP; |
321 | return fscrypt_crypt_data_unit(ci: inode->i_crypt_info, rw: FS_DECRYPT, |
322 | index: lblk_num, src_page: page, dest_page: page, len, offs, |
323 | GFP_NOFS); |
324 | } |
325 | EXPORT_SYMBOL(fscrypt_decrypt_block_inplace); |
326 | |
327 | /** |
328 | * fscrypt_initialize() - allocate major buffers for fs encryption. |
329 | * @sb: the filesystem superblock |
330 | * |
331 | * We only call this when we start accessing encrypted files, since it |
332 | * results in memory getting allocated that wouldn't otherwise be used. |
333 | * |
334 | * Return: 0 on success; -errno on failure |
335 | */ |
336 | int fscrypt_initialize(struct super_block *sb) |
337 | { |
338 | int err = 0; |
339 | mempool_t *pool; |
340 | |
341 | /* pairs with smp_store_release() below */ |
342 | if (likely(smp_load_acquire(&fscrypt_bounce_page_pool))) |
343 | return 0; |
344 | |
345 | /* No need to allocate a bounce page pool if this FS won't use it. */ |
346 | if (!sb->s_cop->needs_bounce_pages) |
347 | return 0; |
348 | |
349 | mutex_lock(&fscrypt_init_mutex); |
350 | if (fscrypt_bounce_page_pool) |
351 | goto out_unlock; |
352 | |
353 | err = -ENOMEM; |
354 | pool = mempool_create_page_pool(min_nr: num_prealloc_crypto_pages, order: 0); |
355 | if (!pool) |
356 | goto out_unlock; |
357 | /* pairs with smp_load_acquire() above */ |
358 | smp_store_release(&fscrypt_bounce_page_pool, pool); |
359 | err = 0; |
360 | out_unlock: |
361 | mutex_unlock(lock: &fscrypt_init_mutex); |
362 | return err; |
363 | } |
364 | |
365 | void fscrypt_msg(const struct inode *inode, const char *level, |
366 | const char *fmt, ...) |
367 | { |
368 | static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, |
369 | DEFAULT_RATELIMIT_BURST); |
370 | struct va_format vaf; |
371 | va_list args; |
372 | |
373 | if (!__ratelimit(&rs)) |
374 | return; |
375 | |
376 | va_start(args, fmt); |
377 | vaf.fmt = fmt; |
378 | vaf.va = &args; |
379 | if (inode && inode->i_ino) |
380 | printk("%sfscrypt (%s, inode %lu): %pV\n" , |
381 | level, inode->i_sb->s_id, inode->i_ino, &vaf); |
382 | else if (inode) |
383 | printk("%sfscrypt (%s): %pV\n" , level, inode->i_sb->s_id, &vaf); |
384 | else |
385 | printk("%sfscrypt: %pV\n" , level, &vaf); |
386 | va_end(args); |
387 | } |
388 | |
389 | /** |
390 | * fscrypt_init() - Set up for fs encryption. |
391 | * |
392 | * Return: 0 on success; -errno on failure |
393 | */ |
394 | static int __init fscrypt_init(void) |
395 | { |
396 | int err = -ENOMEM; |
397 | |
398 | /* |
399 | * Use an unbound workqueue to allow bios to be decrypted in parallel |
400 | * even when they happen to complete on the same CPU. This sacrifices |
401 | * locality, but it's worthwhile since decryption is CPU-intensive. |
402 | * |
403 | * Also use a high-priority workqueue to prioritize decryption work, |
404 | * which blocks reads from completing, over regular application tasks. |
405 | */ |
406 | fscrypt_read_workqueue = alloc_workqueue(fmt: "fscrypt_read_queue" , |
407 | flags: WQ_UNBOUND | WQ_HIGHPRI, |
408 | max_active: num_online_cpus()); |
409 | if (!fscrypt_read_workqueue) |
410 | goto fail; |
411 | |
412 | fscrypt_inode_info_cachep = KMEM_CACHE(fscrypt_inode_info, |
413 | SLAB_RECLAIM_ACCOUNT); |
414 | if (!fscrypt_inode_info_cachep) |
415 | goto fail_free_queue; |
416 | |
417 | err = fscrypt_init_keyring(); |
418 | if (err) |
419 | goto fail_free_inode_info; |
420 | |
421 | return 0; |
422 | |
423 | fail_free_inode_info: |
424 | kmem_cache_destroy(s: fscrypt_inode_info_cachep); |
425 | fail_free_queue: |
426 | destroy_workqueue(wq: fscrypt_read_workqueue); |
427 | fail: |
428 | return err; |
429 | } |
430 | late_initcall(fscrypt_init) |
431 | |