1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * bio-integrity.c - bio data integrity extensions |
4 | * |
5 | * Copyright (C) 2007, 2008, 2009 Oracle Corporation |
6 | * Written by: Martin K. Petersen <martin.petersen@oracle.com> |
7 | */ |
8 | |
9 | #include <linux/blk-integrity.h> |
10 | #include <linux/mempool.h> |
11 | #include <linux/export.h> |
12 | #include <linux/bio.h> |
13 | #include <linux/workqueue.h> |
14 | #include <linux/slab.h> |
15 | #include "blk.h" |
16 | |
17 | static struct kmem_cache *bip_slab; |
18 | static struct workqueue_struct *kintegrityd_wq; |
19 | |
20 | void blk_flush_integrity(void) |
21 | { |
22 | flush_workqueue(kintegrityd_wq); |
23 | } |
24 | |
25 | static void __bio_integrity_free(struct bio_set *bs, |
26 | struct bio_integrity_payload *bip) |
27 | { |
28 | if (bs && mempool_initialized(pool: &bs->bio_integrity_pool)) { |
29 | if (bip->bip_vec) |
30 | bvec_free(pool: &bs->bvec_integrity_pool, bv: bip->bip_vec, |
31 | nr_vecs: bip->bip_max_vcnt); |
32 | mempool_free(element: bip, pool: &bs->bio_integrity_pool); |
33 | } else { |
34 | kfree(objp: bip); |
35 | } |
36 | } |
37 | |
38 | /** |
39 | * bio_integrity_alloc - Allocate integrity payload and attach it to bio |
40 | * @bio: bio to attach integrity metadata to |
41 | * @gfp_mask: Memory allocation mask |
42 | * @nr_vecs: Number of integrity metadata scatter-gather elements |
43 | * |
44 | * Description: This function prepares a bio for attaching integrity |
45 | * metadata. nr_vecs specifies the maximum number of pages containing |
46 | * integrity metadata that can be attached. |
47 | */ |
48 | struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio, |
49 | gfp_t gfp_mask, |
50 | unsigned int nr_vecs) |
51 | { |
52 | struct bio_integrity_payload *bip; |
53 | struct bio_set *bs = bio->bi_pool; |
54 | unsigned inline_vecs; |
55 | |
56 | if (WARN_ON_ONCE(bio_has_crypt_ctx(bio))) |
57 | return ERR_PTR(error: -EOPNOTSUPP); |
58 | |
59 | if (!bs || !mempool_initialized(pool: &bs->bio_integrity_pool)) { |
60 | bip = kmalloc(struct_size(bip, bip_inline_vecs, nr_vecs), flags: gfp_mask); |
61 | inline_vecs = nr_vecs; |
62 | } else { |
63 | bip = mempool_alloc(pool: &bs->bio_integrity_pool, gfp_mask); |
64 | inline_vecs = BIO_INLINE_VECS; |
65 | } |
66 | |
67 | if (unlikely(!bip)) |
68 | return ERR_PTR(error: -ENOMEM); |
69 | |
70 | memset(bip, 0, sizeof(*bip)); |
71 | |
72 | if (nr_vecs > inline_vecs) { |
73 | bip->bip_max_vcnt = nr_vecs; |
74 | bip->bip_vec = bvec_alloc(pool: &bs->bvec_integrity_pool, |
75 | nr_vecs: &bip->bip_max_vcnt, gfp_mask); |
76 | if (!bip->bip_vec) |
77 | goto err; |
78 | } else { |
79 | bip->bip_vec = bip->bip_inline_vecs; |
80 | bip->bip_max_vcnt = inline_vecs; |
81 | } |
82 | |
83 | bip->bip_bio = bio; |
84 | bio->bi_integrity = bip; |
85 | bio->bi_opf |= REQ_INTEGRITY; |
86 | |
87 | return bip; |
88 | err: |
89 | __bio_integrity_free(bs, bip); |
90 | return ERR_PTR(error: -ENOMEM); |
91 | } |
92 | EXPORT_SYMBOL(bio_integrity_alloc); |
93 | |
94 | /** |
95 | * bio_integrity_free - Free bio integrity payload |
96 | * @bio: bio containing bip to be freed |
97 | * |
98 | * Description: Used to free the integrity portion of a bio. Usually |
99 | * called from bio_free(). |
100 | */ |
101 | void bio_integrity_free(struct bio *bio) |
102 | { |
103 | struct bio_integrity_payload *bip = bio_integrity(bio); |
104 | struct bio_set *bs = bio->bi_pool; |
105 | |
106 | if (bip->bip_flags & BIP_BLOCK_INTEGRITY) |
107 | kfree(objp: bvec_virt(bvec: bip->bip_vec)); |
108 | |
109 | __bio_integrity_free(bs, bip); |
110 | bio->bi_integrity = NULL; |
111 | bio->bi_opf &= ~REQ_INTEGRITY; |
112 | } |
113 | |
114 | /** |
115 | * bio_integrity_add_page - Attach integrity metadata |
116 | * @bio: bio to update |
117 | * @page: page containing integrity metadata |
118 | * @len: number of bytes of integrity metadata in page |
119 | * @offset: start offset within page |
120 | * |
121 | * Description: Attach a page containing integrity metadata to bio. |
122 | */ |
123 | int bio_integrity_add_page(struct bio *bio, struct page *page, |
124 | unsigned int len, unsigned int offset) |
125 | { |
126 | struct request_queue *q = bdev_get_queue(bdev: bio->bi_bdev); |
127 | struct bio_integrity_payload *bip = bio_integrity(bio); |
128 | |
129 | if (((bip->bip_iter.bi_size + len) >> SECTOR_SHIFT) > |
130 | queue_max_hw_sectors(q)) |
131 | return 0; |
132 | |
133 | if (bip->bip_vcnt > 0) { |
134 | struct bio_vec *bv = &bip->bip_vec[bip->bip_vcnt - 1]; |
135 | bool same_page = false; |
136 | |
137 | if (bvec_try_merge_hw_page(q, bv, page, len, offset, |
138 | same_page: &same_page)) { |
139 | bip->bip_iter.bi_size += len; |
140 | return len; |
141 | } |
142 | |
143 | if (bip->bip_vcnt >= |
144 | min(bip->bip_max_vcnt, queue_max_integrity_segments(q))) |
145 | return 0; |
146 | |
147 | /* |
148 | * If the queue doesn't support SG gaps and adding this segment |
149 | * would create a gap, disallow it. |
150 | */ |
151 | if (bvec_gap_to_prev(lim: &q->limits, bprv: bv, offset)) |
152 | return 0; |
153 | } |
154 | |
155 | bvec_set_page(bv: &bip->bip_vec[bip->bip_vcnt], page, len, offset); |
156 | bip->bip_vcnt++; |
157 | bip->bip_iter.bi_size += len; |
158 | |
159 | return len; |
160 | } |
161 | EXPORT_SYMBOL(bio_integrity_add_page); |
162 | |
163 | /** |
164 | * bio_integrity_process - Process integrity metadata for a bio |
165 | * @bio: bio to generate/verify integrity metadata for |
166 | * @proc_iter: iterator to process |
167 | * @proc_fn: Pointer to the relevant processing function |
168 | */ |
169 | static blk_status_t bio_integrity_process(struct bio *bio, |
170 | struct bvec_iter *proc_iter, integrity_processing_fn *proc_fn) |
171 | { |
172 | struct blk_integrity *bi = blk_get_integrity(disk: bio->bi_bdev->bd_disk); |
173 | struct blk_integrity_iter iter; |
174 | struct bvec_iter bviter; |
175 | struct bio_vec bv; |
176 | struct bio_integrity_payload *bip = bio_integrity(bio); |
177 | blk_status_t ret = BLK_STS_OK; |
178 | |
179 | iter.disk_name = bio->bi_bdev->bd_disk->disk_name; |
180 | iter.interval = 1 << bi->interval_exp; |
181 | iter.tuple_size = bi->tuple_size; |
182 | iter.seed = proc_iter->bi_sector; |
183 | iter.prot_buf = bvec_virt(bvec: bip->bip_vec); |
184 | |
185 | __bio_for_each_segment(bv, bio, bviter, *proc_iter) { |
186 | void *kaddr = bvec_kmap_local(bvec: &bv); |
187 | |
188 | iter.data_buf = kaddr; |
189 | iter.data_size = bv.bv_len; |
190 | ret = proc_fn(&iter); |
191 | kunmap_local(kaddr); |
192 | |
193 | if (ret) |
194 | break; |
195 | |
196 | } |
197 | return ret; |
198 | } |
199 | |
200 | /** |
201 | * bio_integrity_prep - Prepare bio for integrity I/O |
202 | * @bio: bio to prepare |
203 | * |
204 | * Description: Checks if the bio already has an integrity payload attached. |
205 | * If it does, the payload has been generated by another kernel subsystem, |
206 | * and we just pass it through. Otherwise allocates integrity payload. |
207 | * The bio must have data direction, target device and start sector set priot |
208 | * to calling. In the WRITE case, integrity metadata will be generated using |
209 | * the block device's integrity function. In the READ case, the buffer |
210 | * will be prepared for DMA and a suitable end_io handler set up. |
211 | */ |
212 | bool bio_integrity_prep(struct bio *bio) |
213 | { |
214 | struct bio_integrity_payload *bip; |
215 | struct blk_integrity *bi = blk_get_integrity(disk: bio->bi_bdev->bd_disk); |
216 | void *buf; |
217 | unsigned long start, end; |
218 | unsigned int len, nr_pages; |
219 | unsigned int bytes, offset, i; |
220 | |
221 | if (!bi) |
222 | return true; |
223 | |
224 | if (bio_op(bio) != REQ_OP_READ && bio_op(bio) != REQ_OP_WRITE) |
225 | return true; |
226 | |
227 | if (!bio_sectors(bio)) |
228 | return true; |
229 | |
230 | /* Already protected? */ |
231 | if (bio_integrity(bio)) |
232 | return true; |
233 | |
234 | if (bio_data_dir(bio) == READ) { |
235 | if (!bi->profile->verify_fn || |
236 | !(bi->flags & BLK_INTEGRITY_VERIFY)) |
237 | return true; |
238 | } else { |
239 | if (!bi->profile->generate_fn || |
240 | !(bi->flags & BLK_INTEGRITY_GENERATE)) |
241 | return true; |
242 | } |
243 | |
244 | /* Allocate kernel buffer for protection data */ |
245 | len = bio_integrity_bytes(bi, bio_sectors(bio)); |
246 | buf = kmalloc(size: len, GFP_NOIO); |
247 | if (unlikely(buf == NULL)) { |
248 | printk(KERN_ERR "could not allocate integrity buffer\n" ); |
249 | goto err_end_io; |
250 | } |
251 | |
252 | end = (((unsigned long) buf) + len + PAGE_SIZE - 1) >> PAGE_SHIFT; |
253 | start = ((unsigned long) buf) >> PAGE_SHIFT; |
254 | nr_pages = end - start; |
255 | |
256 | /* Allocate bio integrity payload and integrity vectors */ |
257 | bip = bio_integrity_alloc(bio, GFP_NOIO, nr_pages); |
258 | if (IS_ERR(ptr: bip)) { |
259 | printk(KERN_ERR "could not allocate data integrity bioset\n" ); |
260 | kfree(objp: buf); |
261 | goto err_end_io; |
262 | } |
263 | |
264 | bip->bip_flags |= BIP_BLOCK_INTEGRITY; |
265 | bip_set_seed(bip, seed: bio->bi_iter.bi_sector); |
266 | |
267 | if (bi->flags & BLK_INTEGRITY_IP_CHECKSUM) |
268 | bip->bip_flags |= BIP_IP_CHECKSUM; |
269 | |
270 | /* Map it */ |
271 | offset = offset_in_page(buf); |
272 | for (i = 0; i < nr_pages && len > 0; i++) { |
273 | bytes = PAGE_SIZE - offset; |
274 | |
275 | if (bytes > len) |
276 | bytes = len; |
277 | |
278 | if (bio_integrity_add_page(bio, virt_to_page(buf), |
279 | bytes, offset) < bytes) { |
280 | printk(KERN_ERR "could not attach integrity payload\n" ); |
281 | goto err_end_io; |
282 | } |
283 | |
284 | buf += bytes; |
285 | len -= bytes; |
286 | offset = 0; |
287 | } |
288 | |
289 | /* Auto-generate integrity metadata if this is a write */ |
290 | if (bio_data_dir(bio) == WRITE) { |
291 | bio_integrity_process(bio, proc_iter: &bio->bi_iter, |
292 | proc_fn: bi->profile->generate_fn); |
293 | } else { |
294 | bip->bio_iter = bio->bi_iter; |
295 | } |
296 | return true; |
297 | |
298 | err_end_io: |
299 | bio->bi_status = BLK_STS_RESOURCE; |
300 | bio_endio(bio); |
301 | return false; |
302 | } |
303 | EXPORT_SYMBOL(bio_integrity_prep); |
304 | |
305 | /** |
306 | * bio_integrity_verify_fn - Integrity I/O completion worker |
307 | * @work: Work struct stored in bio to be verified |
308 | * |
309 | * Description: This workqueue function is called to complete a READ |
310 | * request. The function verifies the transferred integrity metadata |
311 | * and then calls the original bio end_io function. |
312 | */ |
313 | static void bio_integrity_verify_fn(struct work_struct *work) |
314 | { |
315 | struct bio_integrity_payload *bip = |
316 | container_of(work, struct bio_integrity_payload, bip_work); |
317 | struct bio *bio = bip->bip_bio; |
318 | struct blk_integrity *bi = blk_get_integrity(disk: bio->bi_bdev->bd_disk); |
319 | |
320 | /* |
321 | * At the moment verify is called bio's iterator was advanced |
322 | * during split and completion, we need to rewind iterator to |
323 | * it's original position. |
324 | */ |
325 | bio->bi_status = bio_integrity_process(bio, proc_iter: &bip->bio_iter, |
326 | proc_fn: bi->profile->verify_fn); |
327 | bio_integrity_free(bio); |
328 | bio_endio(bio); |
329 | } |
330 | |
331 | /** |
332 | * __bio_integrity_endio - Integrity I/O completion function |
333 | * @bio: Protected bio |
334 | * |
335 | * Description: Completion for integrity I/O |
336 | * |
337 | * Normally I/O completion is done in interrupt context. However, |
338 | * verifying I/O integrity is a time-consuming task which must be run |
339 | * in process context. This function postpones completion |
340 | * accordingly. |
341 | */ |
342 | bool __bio_integrity_endio(struct bio *bio) |
343 | { |
344 | struct blk_integrity *bi = blk_get_integrity(disk: bio->bi_bdev->bd_disk); |
345 | struct bio_integrity_payload *bip = bio_integrity(bio); |
346 | |
347 | if (bio_op(bio) == REQ_OP_READ && !bio->bi_status && |
348 | (bip->bip_flags & BIP_BLOCK_INTEGRITY) && bi->profile->verify_fn) { |
349 | INIT_WORK(&bip->bip_work, bio_integrity_verify_fn); |
350 | queue_work(wq: kintegrityd_wq, work: &bip->bip_work); |
351 | return false; |
352 | } |
353 | |
354 | bio_integrity_free(bio); |
355 | return true; |
356 | } |
357 | |
358 | /** |
359 | * bio_integrity_advance - Advance integrity vector |
360 | * @bio: bio whose integrity vector to update |
361 | * @bytes_done: number of data bytes that have been completed |
362 | * |
363 | * Description: This function calculates how many integrity bytes the |
364 | * number of completed data bytes correspond to and advances the |
365 | * integrity vector accordingly. |
366 | */ |
367 | void bio_integrity_advance(struct bio *bio, unsigned int bytes_done) |
368 | { |
369 | struct bio_integrity_payload *bip = bio_integrity(bio); |
370 | struct blk_integrity *bi = blk_get_integrity(disk: bio->bi_bdev->bd_disk); |
371 | unsigned bytes = bio_integrity_bytes(bi, sectors: bytes_done >> 9); |
372 | |
373 | bip->bip_iter.bi_sector += bio_integrity_intervals(bi, sectors: bytes_done >> 9); |
374 | bvec_iter_advance(bv: bip->bip_vec, iter: &bip->bip_iter, bytes); |
375 | } |
376 | |
377 | /** |
378 | * bio_integrity_trim - Trim integrity vector |
379 | * @bio: bio whose integrity vector to update |
380 | * |
381 | * Description: Used to trim the integrity vector in a cloned bio. |
382 | */ |
383 | void bio_integrity_trim(struct bio *bio) |
384 | { |
385 | struct bio_integrity_payload *bip = bio_integrity(bio); |
386 | struct blk_integrity *bi = blk_get_integrity(disk: bio->bi_bdev->bd_disk); |
387 | |
388 | bip->bip_iter.bi_size = bio_integrity_bytes(bi, bio_sectors(bio)); |
389 | } |
390 | EXPORT_SYMBOL(bio_integrity_trim); |
391 | |
392 | /** |
393 | * bio_integrity_clone - Callback for cloning bios with integrity metadata |
394 | * @bio: New bio |
395 | * @bio_src: Original bio |
396 | * @gfp_mask: Memory allocation mask |
397 | * |
398 | * Description: Called to allocate a bip when cloning a bio |
399 | */ |
400 | int bio_integrity_clone(struct bio *bio, struct bio *bio_src, |
401 | gfp_t gfp_mask) |
402 | { |
403 | struct bio_integrity_payload *bip_src = bio_integrity(bio: bio_src); |
404 | struct bio_integrity_payload *bip; |
405 | |
406 | BUG_ON(bip_src == NULL); |
407 | |
408 | bip = bio_integrity_alloc(bio, gfp_mask, bip_src->bip_vcnt); |
409 | if (IS_ERR(ptr: bip)) |
410 | return PTR_ERR(ptr: bip); |
411 | |
412 | memcpy(bip->bip_vec, bip_src->bip_vec, |
413 | bip_src->bip_vcnt * sizeof(struct bio_vec)); |
414 | |
415 | bip->bip_vcnt = bip_src->bip_vcnt; |
416 | bip->bip_iter = bip_src->bip_iter; |
417 | bip->bip_flags = bip_src->bip_flags & ~BIP_BLOCK_INTEGRITY; |
418 | |
419 | return 0; |
420 | } |
421 | |
422 | int bioset_integrity_create(struct bio_set *bs, int pool_size) |
423 | { |
424 | if (mempool_initialized(pool: &bs->bio_integrity_pool)) |
425 | return 0; |
426 | |
427 | if (mempool_init_slab_pool(pool: &bs->bio_integrity_pool, |
428 | min_nr: pool_size, kc: bip_slab)) |
429 | return -1; |
430 | |
431 | if (biovec_init_pool(pool: &bs->bvec_integrity_pool, pool_entries: pool_size)) { |
432 | mempool_exit(pool: &bs->bio_integrity_pool); |
433 | return -1; |
434 | } |
435 | |
436 | return 0; |
437 | } |
438 | EXPORT_SYMBOL(bioset_integrity_create); |
439 | |
440 | void bioset_integrity_free(struct bio_set *bs) |
441 | { |
442 | mempool_exit(pool: &bs->bio_integrity_pool); |
443 | mempool_exit(pool: &bs->bvec_integrity_pool); |
444 | } |
445 | |
446 | void __init bio_integrity_init(void) |
447 | { |
448 | /* |
449 | * kintegrityd won't block much but may burn a lot of CPU cycles. |
450 | * Make it highpri CPU intensive wq with max concurrency of 1. |
451 | */ |
452 | kintegrityd_wq = alloc_workqueue(fmt: "kintegrityd" , flags: WQ_MEM_RECLAIM | |
453 | WQ_HIGHPRI | WQ_CPU_INTENSIVE, max_active: 1); |
454 | if (!kintegrityd_wq) |
455 | panic(fmt: "Failed to create kintegrityd\n" ); |
456 | |
457 | bip_slab = kmem_cache_create(name: "bio_integrity_payload" , |
458 | size: sizeof(struct bio_integrity_payload) + |
459 | sizeof(struct bio_vec) * BIO_INLINE_VECS, |
460 | align: 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); |
461 | } |
462 | |