1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. |
3 | * |
4 | * Kernel side components to support tools/testing/selftests/iommu |
5 | */ |
6 | #include <linux/slab.h> |
7 | #include <linux/iommu.h> |
8 | #include <linux/xarray.h> |
9 | #include <linux/file.h> |
10 | #include <linux/anon_inodes.h> |
11 | #include <linux/fault-inject.h> |
12 | #include <linux/platform_device.h> |
13 | #include <uapi/linux/iommufd.h> |
14 | |
15 | #include "../iommu-priv.h" |
16 | #include "io_pagetable.h" |
17 | #include "iommufd_private.h" |
18 | #include "iommufd_test.h" |
19 | |
20 | static DECLARE_FAULT_ATTR(fail_iommufd); |
21 | static struct dentry *dbgfs_root; |
22 | static struct platform_device *selftest_iommu_dev; |
23 | static const struct iommu_ops mock_ops; |
24 | static struct iommu_domain_ops domain_nested_ops; |
25 | |
26 | size_t iommufd_test_memory_limit = 65536; |
27 | |
28 | struct mock_bus_type { |
29 | struct bus_type bus; |
30 | struct notifier_block nb; |
31 | }; |
32 | |
33 | static struct mock_bus_type iommufd_mock_bus_type = { |
34 | .bus = { |
35 | .name = "iommufd_mock" , |
36 | }, |
37 | }; |
38 | |
39 | static DEFINE_IDA(mock_dev_ida); |
40 | |
41 | enum { |
42 | MOCK_DIRTY_TRACK = 1, |
43 | MOCK_IO_PAGE_SIZE = PAGE_SIZE / 2, |
44 | MOCK_HUGE_PAGE_SIZE = 512 * MOCK_IO_PAGE_SIZE, |
45 | |
46 | /* |
47 | * Like a real page table alignment requires the low bits of the address |
48 | * to be zero. xarray also requires the high bit to be zero, so we store |
49 | * the pfns shifted. The upper bits are used for metadata. |
50 | */ |
51 | MOCK_PFN_MASK = ULONG_MAX / MOCK_IO_PAGE_SIZE, |
52 | |
53 | _MOCK_PFN_START = MOCK_PFN_MASK + 1, |
54 | MOCK_PFN_START_IOVA = _MOCK_PFN_START, |
55 | MOCK_PFN_LAST_IOVA = _MOCK_PFN_START, |
56 | MOCK_PFN_DIRTY_IOVA = _MOCK_PFN_START << 1, |
57 | MOCK_PFN_HUGE_IOVA = _MOCK_PFN_START << 2, |
58 | }; |
59 | |
60 | /* |
61 | * Syzkaller has trouble randomizing the correct iova to use since it is linked |
62 | * to the map ioctl's output, and it has no ide about that. So, simplify things. |
63 | * In syzkaller mode the 64 bit IOVA is converted into an nth area and offset |
64 | * value. This has a much smaller randomization space and syzkaller can hit it. |
65 | */ |
66 | static unsigned long __iommufd_test_syz_conv_iova(struct io_pagetable *iopt, |
67 | u64 *iova) |
68 | { |
69 | struct syz_layout { |
70 | __u32 nth_area; |
71 | __u32 offset; |
72 | }; |
73 | struct syz_layout *syz = (void *)iova; |
74 | unsigned int nth = syz->nth_area; |
75 | struct iopt_area *area; |
76 | |
77 | down_read(sem: &iopt->iova_rwsem); |
78 | for (area = iopt_area_iter_first(iopt, start: 0, ULONG_MAX); area; |
79 | area = iopt_area_iter_next(last_node: area, start: 0, ULONG_MAX)) { |
80 | if (nth == 0) { |
81 | up_read(sem: &iopt->iova_rwsem); |
82 | return iopt_area_iova(area) + syz->offset; |
83 | } |
84 | nth--; |
85 | } |
86 | up_read(sem: &iopt->iova_rwsem); |
87 | |
88 | return 0; |
89 | } |
90 | |
91 | static unsigned long iommufd_test_syz_conv_iova(struct iommufd_access *access, |
92 | u64 *iova) |
93 | { |
94 | unsigned long ret; |
95 | |
96 | mutex_lock(&access->ioas_lock); |
97 | if (!access->ioas) { |
98 | mutex_unlock(lock: &access->ioas_lock); |
99 | return 0; |
100 | } |
101 | ret = __iommufd_test_syz_conv_iova(iopt: &access->ioas->iopt, iova); |
102 | mutex_unlock(lock: &access->ioas_lock); |
103 | return ret; |
104 | } |
105 | |
106 | void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd, |
107 | unsigned int ioas_id, u64 *iova, u32 *flags) |
108 | { |
109 | struct iommufd_ioas *ioas; |
110 | |
111 | if (!(*flags & MOCK_FLAGS_ACCESS_SYZ)) |
112 | return; |
113 | *flags &= ~(u32)MOCK_FLAGS_ACCESS_SYZ; |
114 | |
115 | ioas = iommufd_get_ioas(ictx: ucmd->ictx, id: ioas_id); |
116 | if (IS_ERR(ptr: ioas)) |
117 | return; |
118 | *iova = __iommufd_test_syz_conv_iova(iopt: &ioas->iopt, iova); |
119 | iommufd_put_object(ictx: ucmd->ictx, obj: &ioas->obj); |
120 | } |
121 | |
122 | struct mock_iommu_domain { |
123 | unsigned long flags; |
124 | struct iommu_domain domain; |
125 | struct xarray pfns; |
126 | }; |
127 | |
128 | struct mock_iommu_domain_nested { |
129 | struct iommu_domain domain; |
130 | struct mock_iommu_domain *parent; |
131 | u32 iotlb[MOCK_NESTED_DOMAIN_IOTLB_NUM]; |
132 | }; |
133 | |
134 | enum selftest_obj_type { |
135 | TYPE_IDEV, |
136 | }; |
137 | |
138 | struct mock_dev { |
139 | struct device dev; |
140 | unsigned long flags; |
141 | int id; |
142 | }; |
143 | |
144 | struct selftest_obj { |
145 | struct iommufd_object obj; |
146 | enum selftest_obj_type type; |
147 | |
148 | union { |
149 | struct { |
150 | struct iommufd_device *idev; |
151 | struct iommufd_ctx *ictx; |
152 | struct mock_dev *mock_dev; |
153 | } idev; |
154 | }; |
155 | }; |
156 | |
157 | static int mock_domain_nop_attach(struct iommu_domain *domain, |
158 | struct device *dev) |
159 | { |
160 | struct mock_dev *mdev = container_of(dev, struct mock_dev, dev); |
161 | |
162 | if (domain->dirty_ops && (mdev->flags & MOCK_FLAGS_DEVICE_NO_DIRTY)) |
163 | return -EINVAL; |
164 | |
165 | return 0; |
166 | } |
167 | |
168 | static const struct iommu_domain_ops mock_blocking_ops = { |
169 | .attach_dev = mock_domain_nop_attach, |
170 | }; |
171 | |
172 | static struct iommu_domain mock_blocking_domain = { |
173 | .type = IOMMU_DOMAIN_BLOCKED, |
174 | .ops = &mock_blocking_ops, |
175 | }; |
176 | |
177 | static void *mock_domain_hw_info(struct device *dev, u32 *length, u32 *type) |
178 | { |
179 | struct iommu_test_hw_info *info; |
180 | |
181 | info = kzalloc(size: sizeof(*info), GFP_KERNEL); |
182 | if (!info) |
183 | return ERR_PTR(error: -ENOMEM); |
184 | |
185 | info->test_reg = IOMMU_HW_INFO_SELFTEST_REGVAL; |
186 | *length = sizeof(*info); |
187 | *type = IOMMU_HW_INFO_TYPE_SELFTEST; |
188 | |
189 | return info; |
190 | } |
191 | |
192 | static int mock_domain_set_dirty_tracking(struct iommu_domain *domain, |
193 | bool enable) |
194 | { |
195 | struct mock_iommu_domain *mock = |
196 | container_of(domain, struct mock_iommu_domain, domain); |
197 | unsigned long flags = mock->flags; |
198 | |
199 | if (enable && !domain->dirty_ops) |
200 | return -EINVAL; |
201 | |
202 | /* No change? */ |
203 | if (!(enable ^ !!(flags & MOCK_DIRTY_TRACK))) |
204 | return 0; |
205 | |
206 | flags = (enable ? flags | MOCK_DIRTY_TRACK : flags & ~MOCK_DIRTY_TRACK); |
207 | |
208 | mock->flags = flags; |
209 | return 0; |
210 | } |
211 | |
212 | static bool mock_test_and_clear_dirty(struct mock_iommu_domain *mock, |
213 | unsigned long iova, size_t page_size, |
214 | unsigned long flags) |
215 | { |
216 | unsigned long cur, end = iova + page_size - 1; |
217 | bool dirty = false; |
218 | void *ent, *old; |
219 | |
220 | for (cur = iova; cur < end; cur += MOCK_IO_PAGE_SIZE) { |
221 | ent = xa_load(&mock->pfns, index: cur / MOCK_IO_PAGE_SIZE); |
222 | if (!ent || !(xa_to_value(entry: ent) & MOCK_PFN_DIRTY_IOVA)) |
223 | continue; |
224 | |
225 | dirty = true; |
226 | /* Clear dirty */ |
227 | if (!(flags & IOMMU_DIRTY_NO_CLEAR)) { |
228 | unsigned long val; |
229 | |
230 | val = xa_to_value(entry: ent) & ~MOCK_PFN_DIRTY_IOVA; |
231 | old = xa_store(&mock->pfns, index: cur / MOCK_IO_PAGE_SIZE, |
232 | entry: xa_mk_value(v: val), GFP_KERNEL); |
233 | WARN_ON_ONCE(ent != old); |
234 | } |
235 | } |
236 | |
237 | return dirty; |
238 | } |
239 | |
240 | static int mock_domain_read_and_clear_dirty(struct iommu_domain *domain, |
241 | unsigned long iova, size_t size, |
242 | unsigned long flags, |
243 | struct iommu_dirty_bitmap *dirty) |
244 | { |
245 | struct mock_iommu_domain *mock = |
246 | container_of(domain, struct mock_iommu_domain, domain); |
247 | unsigned long end = iova + size; |
248 | void *ent; |
249 | |
250 | if (!(mock->flags & MOCK_DIRTY_TRACK) && dirty->bitmap) |
251 | return -EINVAL; |
252 | |
253 | do { |
254 | unsigned long pgsize = MOCK_IO_PAGE_SIZE; |
255 | unsigned long head; |
256 | |
257 | ent = xa_load(&mock->pfns, index: iova / MOCK_IO_PAGE_SIZE); |
258 | if (!ent) { |
259 | iova += pgsize; |
260 | continue; |
261 | } |
262 | |
263 | if (xa_to_value(entry: ent) & MOCK_PFN_HUGE_IOVA) |
264 | pgsize = MOCK_HUGE_PAGE_SIZE; |
265 | head = iova & ~(pgsize - 1); |
266 | |
267 | /* Clear dirty */ |
268 | if (mock_test_and_clear_dirty(mock, iova: head, page_size: pgsize, flags)) |
269 | iommu_dirty_bitmap_record(dirty, iova: head, length: pgsize); |
270 | iova = head + pgsize; |
271 | } while (iova < end); |
272 | |
273 | return 0; |
274 | } |
275 | |
276 | const struct iommu_dirty_ops dirty_ops = { |
277 | .set_dirty_tracking = mock_domain_set_dirty_tracking, |
278 | .read_and_clear_dirty = mock_domain_read_and_clear_dirty, |
279 | }; |
280 | |
281 | static struct iommu_domain *mock_domain_alloc_paging(struct device *dev) |
282 | { |
283 | struct mock_dev *mdev = container_of(dev, struct mock_dev, dev); |
284 | struct mock_iommu_domain *mock; |
285 | |
286 | mock = kzalloc(size: sizeof(*mock), GFP_KERNEL); |
287 | if (!mock) |
288 | return NULL; |
289 | mock->domain.geometry.aperture_start = MOCK_APERTURE_START; |
290 | mock->domain.geometry.aperture_end = MOCK_APERTURE_LAST; |
291 | mock->domain.pgsize_bitmap = MOCK_IO_PAGE_SIZE; |
292 | if (dev && mdev->flags & MOCK_FLAGS_DEVICE_HUGE_IOVA) |
293 | mock->domain.pgsize_bitmap |= MOCK_HUGE_PAGE_SIZE; |
294 | mock->domain.ops = mock_ops.default_domain_ops; |
295 | mock->domain.type = IOMMU_DOMAIN_UNMANAGED; |
296 | xa_init(xa: &mock->pfns); |
297 | return &mock->domain; |
298 | } |
299 | |
300 | static struct iommu_domain * |
301 | __mock_domain_alloc_nested(struct mock_iommu_domain *mock_parent, |
302 | const struct iommu_hwpt_selftest *user_cfg) |
303 | { |
304 | struct mock_iommu_domain_nested *mock_nested; |
305 | int i; |
306 | |
307 | mock_nested = kzalloc(size: sizeof(*mock_nested), GFP_KERNEL); |
308 | if (!mock_nested) |
309 | return ERR_PTR(error: -ENOMEM); |
310 | mock_nested->parent = mock_parent; |
311 | mock_nested->domain.ops = &domain_nested_ops; |
312 | mock_nested->domain.type = IOMMU_DOMAIN_NESTED; |
313 | for (i = 0; i < MOCK_NESTED_DOMAIN_IOTLB_NUM; i++) |
314 | mock_nested->iotlb[i] = user_cfg->iotlb; |
315 | return &mock_nested->domain; |
316 | } |
317 | |
318 | static struct iommu_domain * |
319 | mock_domain_alloc_user(struct device *dev, u32 flags, |
320 | struct iommu_domain *parent, |
321 | const struct iommu_user_data *user_data) |
322 | { |
323 | struct mock_iommu_domain *mock_parent; |
324 | struct iommu_hwpt_selftest user_cfg; |
325 | int rc; |
326 | |
327 | /* must be mock_domain */ |
328 | if (!parent) { |
329 | struct mock_dev *mdev = container_of(dev, struct mock_dev, dev); |
330 | bool has_dirty_flag = flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING; |
331 | bool no_dirty_ops = mdev->flags & MOCK_FLAGS_DEVICE_NO_DIRTY; |
332 | struct iommu_domain *domain; |
333 | |
334 | if (flags & (~(IOMMU_HWPT_ALLOC_NEST_PARENT | |
335 | IOMMU_HWPT_ALLOC_DIRTY_TRACKING))) |
336 | return ERR_PTR(error: -EOPNOTSUPP); |
337 | if (user_data || (has_dirty_flag && no_dirty_ops)) |
338 | return ERR_PTR(error: -EOPNOTSUPP); |
339 | domain = mock_domain_alloc_paging(dev); |
340 | if (!domain) |
341 | return ERR_PTR(error: -ENOMEM); |
342 | if (has_dirty_flag) |
343 | container_of(domain, struct mock_iommu_domain, domain) |
344 | ->domain.dirty_ops = &dirty_ops; |
345 | return domain; |
346 | } |
347 | |
348 | /* must be mock_domain_nested */ |
349 | if (user_data->type != IOMMU_HWPT_DATA_SELFTEST || flags) |
350 | return ERR_PTR(error: -EOPNOTSUPP); |
351 | if (!parent || parent->ops != mock_ops.default_domain_ops) |
352 | return ERR_PTR(error: -EINVAL); |
353 | |
354 | mock_parent = container_of(parent, struct mock_iommu_domain, domain); |
355 | if (!mock_parent) |
356 | return ERR_PTR(error: -EINVAL); |
357 | |
358 | rc = iommu_copy_struct_from_user(&user_cfg, user_data, |
359 | IOMMU_HWPT_DATA_SELFTEST, iotlb); |
360 | if (rc) |
361 | return ERR_PTR(error: rc); |
362 | |
363 | return __mock_domain_alloc_nested(mock_parent, user_cfg: &user_cfg); |
364 | } |
365 | |
366 | static void mock_domain_free(struct iommu_domain *domain) |
367 | { |
368 | struct mock_iommu_domain *mock = |
369 | container_of(domain, struct mock_iommu_domain, domain); |
370 | |
371 | WARN_ON(!xa_empty(&mock->pfns)); |
372 | kfree(objp: mock); |
373 | } |
374 | |
375 | static int mock_domain_map_pages(struct iommu_domain *domain, |
376 | unsigned long iova, phys_addr_t paddr, |
377 | size_t pgsize, size_t pgcount, int prot, |
378 | gfp_t gfp, size_t *mapped) |
379 | { |
380 | struct mock_iommu_domain *mock = |
381 | container_of(domain, struct mock_iommu_domain, domain); |
382 | unsigned long flags = MOCK_PFN_START_IOVA; |
383 | unsigned long start_iova = iova; |
384 | |
385 | /* |
386 | * xarray does not reliably work with fault injection because it does a |
387 | * retry allocation, so put our own failure point. |
388 | */ |
389 | if (iommufd_should_fail()) |
390 | return -ENOENT; |
391 | |
392 | WARN_ON(iova % MOCK_IO_PAGE_SIZE); |
393 | WARN_ON(pgsize % MOCK_IO_PAGE_SIZE); |
394 | for (; pgcount; pgcount--) { |
395 | size_t cur; |
396 | |
397 | for (cur = 0; cur != pgsize; cur += MOCK_IO_PAGE_SIZE) { |
398 | void *old; |
399 | |
400 | if (pgcount == 1 && cur + MOCK_IO_PAGE_SIZE == pgsize) |
401 | flags = MOCK_PFN_LAST_IOVA; |
402 | if (pgsize != MOCK_IO_PAGE_SIZE) { |
403 | flags |= MOCK_PFN_HUGE_IOVA; |
404 | } |
405 | old = xa_store(&mock->pfns, index: iova / MOCK_IO_PAGE_SIZE, |
406 | entry: xa_mk_value(v: (paddr / MOCK_IO_PAGE_SIZE) | |
407 | flags), |
408 | gfp); |
409 | if (xa_is_err(entry: old)) { |
410 | for (; start_iova != iova; |
411 | start_iova += MOCK_IO_PAGE_SIZE) |
412 | xa_erase(&mock->pfns, |
413 | index: start_iova / |
414 | MOCK_IO_PAGE_SIZE); |
415 | return xa_err(entry: old); |
416 | } |
417 | WARN_ON(old); |
418 | iova += MOCK_IO_PAGE_SIZE; |
419 | paddr += MOCK_IO_PAGE_SIZE; |
420 | *mapped += MOCK_IO_PAGE_SIZE; |
421 | flags = 0; |
422 | } |
423 | } |
424 | return 0; |
425 | } |
426 | |
427 | static size_t mock_domain_unmap_pages(struct iommu_domain *domain, |
428 | unsigned long iova, size_t pgsize, |
429 | size_t pgcount, |
430 | struct iommu_iotlb_gather *iotlb_gather) |
431 | { |
432 | struct mock_iommu_domain *mock = |
433 | container_of(domain, struct mock_iommu_domain, domain); |
434 | bool first = true; |
435 | size_t ret = 0; |
436 | void *ent; |
437 | |
438 | WARN_ON(iova % MOCK_IO_PAGE_SIZE); |
439 | WARN_ON(pgsize % MOCK_IO_PAGE_SIZE); |
440 | |
441 | for (; pgcount; pgcount--) { |
442 | size_t cur; |
443 | |
444 | for (cur = 0; cur != pgsize; cur += MOCK_IO_PAGE_SIZE) { |
445 | ent = xa_erase(&mock->pfns, index: iova / MOCK_IO_PAGE_SIZE); |
446 | |
447 | /* |
448 | * iommufd generates unmaps that must be a strict |
449 | * superset of the map's performend So every |
450 | * starting/ending IOVA should have been an iova passed |
451 | * to map. |
452 | * |
453 | * This simple logic doesn't work when the HUGE_PAGE is |
454 | * turned on since the core code will automatically |
455 | * switch between the two page sizes creating a break in |
456 | * the unmap calls. The break can land in the middle of |
457 | * contiguous IOVA. |
458 | */ |
459 | if (!(domain->pgsize_bitmap & MOCK_HUGE_PAGE_SIZE)) { |
460 | if (first) { |
461 | WARN_ON(ent && !(xa_to_value(ent) & |
462 | MOCK_PFN_START_IOVA)); |
463 | first = false; |
464 | } |
465 | if (pgcount == 1 && |
466 | cur + MOCK_IO_PAGE_SIZE == pgsize) |
467 | WARN_ON(ent && !(xa_to_value(ent) & |
468 | MOCK_PFN_LAST_IOVA)); |
469 | } |
470 | |
471 | iova += MOCK_IO_PAGE_SIZE; |
472 | ret += MOCK_IO_PAGE_SIZE; |
473 | } |
474 | } |
475 | return ret; |
476 | } |
477 | |
478 | static phys_addr_t mock_domain_iova_to_phys(struct iommu_domain *domain, |
479 | dma_addr_t iova) |
480 | { |
481 | struct mock_iommu_domain *mock = |
482 | container_of(domain, struct mock_iommu_domain, domain); |
483 | void *ent; |
484 | |
485 | WARN_ON(iova % MOCK_IO_PAGE_SIZE); |
486 | ent = xa_load(&mock->pfns, index: iova / MOCK_IO_PAGE_SIZE); |
487 | WARN_ON(!ent); |
488 | return (xa_to_value(entry: ent) & MOCK_PFN_MASK) * MOCK_IO_PAGE_SIZE; |
489 | } |
490 | |
491 | static bool mock_domain_capable(struct device *dev, enum iommu_cap cap) |
492 | { |
493 | struct mock_dev *mdev = container_of(dev, struct mock_dev, dev); |
494 | |
495 | switch (cap) { |
496 | case IOMMU_CAP_CACHE_COHERENCY: |
497 | return true; |
498 | case IOMMU_CAP_DIRTY_TRACKING: |
499 | return !(mdev->flags & MOCK_FLAGS_DEVICE_NO_DIRTY); |
500 | default: |
501 | break; |
502 | } |
503 | |
504 | return false; |
505 | } |
506 | |
507 | static struct iommu_device mock_iommu_device = { |
508 | }; |
509 | |
510 | static struct iommu_device *mock_probe_device(struct device *dev) |
511 | { |
512 | if (dev->bus != &iommufd_mock_bus_type.bus) |
513 | return ERR_PTR(error: -ENODEV); |
514 | return &mock_iommu_device; |
515 | } |
516 | |
517 | static const struct iommu_ops mock_ops = { |
518 | /* |
519 | * IOMMU_DOMAIN_BLOCKED cannot be returned from def_domain_type() |
520 | * because it is zero. |
521 | */ |
522 | .default_domain = &mock_blocking_domain, |
523 | .blocked_domain = &mock_blocking_domain, |
524 | .owner = THIS_MODULE, |
525 | .pgsize_bitmap = MOCK_IO_PAGE_SIZE, |
526 | .hw_info = mock_domain_hw_info, |
527 | .domain_alloc_paging = mock_domain_alloc_paging, |
528 | .domain_alloc_user = mock_domain_alloc_user, |
529 | .capable = mock_domain_capable, |
530 | .device_group = generic_device_group, |
531 | .probe_device = mock_probe_device, |
532 | .default_domain_ops = |
533 | &(struct iommu_domain_ops){ |
534 | .free = mock_domain_free, |
535 | .attach_dev = mock_domain_nop_attach, |
536 | .map_pages = mock_domain_map_pages, |
537 | .unmap_pages = mock_domain_unmap_pages, |
538 | .iova_to_phys = mock_domain_iova_to_phys, |
539 | }, |
540 | }; |
541 | |
542 | static void mock_domain_free_nested(struct iommu_domain *domain) |
543 | { |
544 | struct mock_iommu_domain_nested *mock_nested = |
545 | container_of(domain, struct mock_iommu_domain_nested, domain); |
546 | |
547 | kfree(objp: mock_nested); |
548 | } |
549 | |
550 | static int |
551 | mock_domain_cache_invalidate_user(struct iommu_domain *domain, |
552 | struct iommu_user_data_array *array) |
553 | { |
554 | struct mock_iommu_domain_nested *mock_nested = |
555 | container_of(domain, struct mock_iommu_domain_nested, domain); |
556 | struct iommu_hwpt_invalidate_selftest inv; |
557 | u32 processed = 0; |
558 | int i = 0, j; |
559 | int rc = 0; |
560 | |
561 | if (array->type != IOMMU_HWPT_INVALIDATE_DATA_SELFTEST) { |
562 | rc = -EINVAL; |
563 | goto out; |
564 | } |
565 | |
566 | for ( ; i < array->entry_num; i++) { |
567 | rc = iommu_copy_struct_from_user_array(&inv, array, |
568 | IOMMU_HWPT_INVALIDATE_DATA_SELFTEST, |
569 | i, iotlb_id); |
570 | if (rc) |
571 | break; |
572 | |
573 | if (inv.flags & ~IOMMU_TEST_INVALIDATE_FLAG_ALL) { |
574 | rc = -EOPNOTSUPP; |
575 | break; |
576 | } |
577 | |
578 | if (inv.iotlb_id > MOCK_NESTED_DOMAIN_IOTLB_ID_MAX) { |
579 | rc = -EINVAL; |
580 | break; |
581 | } |
582 | |
583 | if (inv.flags & IOMMU_TEST_INVALIDATE_FLAG_ALL) { |
584 | /* Invalidate all mock iotlb entries and ignore iotlb_id */ |
585 | for (j = 0; j < MOCK_NESTED_DOMAIN_IOTLB_NUM; j++) |
586 | mock_nested->iotlb[j] = 0; |
587 | } else { |
588 | mock_nested->iotlb[inv.iotlb_id] = 0; |
589 | } |
590 | |
591 | processed++; |
592 | } |
593 | |
594 | out: |
595 | array->entry_num = processed; |
596 | return rc; |
597 | } |
598 | |
599 | static struct iommu_domain_ops domain_nested_ops = { |
600 | .free = mock_domain_free_nested, |
601 | .attach_dev = mock_domain_nop_attach, |
602 | .cache_invalidate_user = mock_domain_cache_invalidate_user, |
603 | }; |
604 | |
605 | static inline struct iommufd_hw_pagetable * |
606 | __get_md_pagetable(struct iommufd_ucmd *ucmd, u32 mockpt_id, u32 hwpt_type) |
607 | { |
608 | struct iommufd_object *obj; |
609 | |
610 | obj = iommufd_get_object(ictx: ucmd->ictx, id: mockpt_id, type: hwpt_type); |
611 | if (IS_ERR(ptr: obj)) |
612 | return ERR_CAST(ptr: obj); |
613 | return container_of(obj, struct iommufd_hw_pagetable, obj); |
614 | } |
615 | |
616 | static inline struct iommufd_hw_pagetable * |
617 | get_md_pagetable(struct iommufd_ucmd *ucmd, u32 mockpt_id, |
618 | struct mock_iommu_domain **mock) |
619 | { |
620 | struct iommufd_hw_pagetable *hwpt; |
621 | |
622 | hwpt = __get_md_pagetable(ucmd, mockpt_id, hwpt_type: IOMMUFD_OBJ_HWPT_PAGING); |
623 | if (IS_ERR(ptr: hwpt)) |
624 | return hwpt; |
625 | if (hwpt->domain->type != IOMMU_DOMAIN_UNMANAGED || |
626 | hwpt->domain->ops != mock_ops.default_domain_ops) { |
627 | iommufd_put_object(ictx: ucmd->ictx, obj: &hwpt->obj); |
628 | return ERR_PTR(error: -EINVAL); |
629 | } |
630 | *mock = container_of(hwpt->domain, struct mock_iommu_domain, domain); |
631 | return hwpt; |
632 | } |
633 | |
634 | static inline struct iommufd_hw_pagetable * |
635 | get_md_pagetable_nested(struct iommufd_ucmd *ucmd, u32 mockpt_id, |
636 | struct mock_iommu_domain_nested **mock_nested) |
637 | { |
638 | struct iommufd_hw_pagetable *hwpt; |
639 | |
640 | hwpt = __get_md_pagetable(ucmd, mockpt_id, hwpt_type: IOMMUFD_OBJ_HWPT_NESTED); |
641 | if (IS_ERR(ptr: hwpt)) |
642 | return hwpt; |
643 | if (hwpt->domain->type != IOMMU_DOMAIN_NESTED || |
644 | hwpt->domain->ops != &domain_nested_ops) { |
645 | iommufd_put_object(ictx: ucmd->ictx, obj: &hwpt->obj); |
646 | return ERR_PTR(error: -EINVAL); |
647 | } |
648 | *mock_nested = container_of(hwpt->domain, |
649 | struct mock_iommu_domain_nested, domain); |
650 | return hwpt; |
651 | } |
652 | |
653 | static void mock_dev_release(struct device *dev) |
654 | { |
655 | struct mock_dev *mdev = container_of(dev, struct mock_dev, dev); |
656 | |
657 | ida_free(&mock_dev_ida, id: mdev->id); |
658 | kfree(objp: mdev); |
659 | } |
660 | |
661 | static struct mock_dev *mock_dev_create(unsigned long dev_flags) |
662 | { |
663 | struct mock_dev *mdev; |
664 | int rc; |
665 | |
666 | if (dev_flags & |
667 | ~(MOCK_FLAGS_DEVICE_NO_DIRTY | MOCK_FLAGS_DEVICE_HUGE_IOVA)) |
668 | return ERR_PTR(error: -EINVAL); |
669 | |
670 | mdev = kzalloc(size: sizeof(*mdev), GFP_KERNEL); |
671 | if (!mdev) |
672 | return ERR_PTR(error: -ENOMEM); |
673 | |
674 | device_initialize(dev: &mdev->dev); |
675 | mdev->flags = dev_flags; |
676 | mdev->dev.release = mock_dev_release; |
677 | mdev->dev.bus = &iommufd_mock_bus_type.bus; |
678 | |
679 | rc = ida_alloc(ida: &mock_dev_ida, GFP_KERNEL); |
680 | if (rc < 0) |
681 | goto err_put; |
682 | mdev->id = rc; |
683 | |
684 | rc = dev_set_name(dev: &mdev->dev, name: "iommufd_mock%u" , mdev->id); |
685 | if (rc) |
686 | goto err_put; |
687 | |
688 | rc = device_add(dev: &mdev->dev); |
689 | if (rc) |
690 | goto err_put; |
691 | return mdev; |
692 | |
693 | err_put: |
694 | put_device(dev: &mdev->dev); |
695 | return ERR_PTR(error: rc); |
696 | } |
697 | |
698 | static void mock_dev_destroy(struct mock_dev *mdev) |
699 | { |
700 | device_unregister(dev: &mdev->dev); |
701 | } |
702 | |
703 | bool iommufd_selftest_is_mock_dev(struct device *dev) |
704 | { |
705 | return dev->release == mock_dev_release; |
706 | } |
707 | |
708 | /* Create an hw_pagetable with the mock domain so we can test the domain ops */ |
709 | static int iommufd_test_mock_domain(struct iommufd_ucmd *ucmd, |
710 | struct iommu_test_cmd *cmd) |
711 | { |
712 | struct iommufd_device *idev; |
713 | struct selftest_obj *sobj; |
714 | u32 pt_id = cmd->id; |
715 | u32 dev_flags = 0; |
716 | u32 idev_id; |
717 | int rc; |
718 | |
719 | sobj = iommufd_object_alloc(ucmd->ictx, sobj, IOMMUFD_OBJ_SELFTEST); |
720 | if (IS_ERR(ptr: sobj)) |
721 | return PTR_ERR(ptr: sobj); |
722 | |
723 | sobj->idev.ictx = ucmd->ictx; |
724 | sobj->type = TYPE_IDEV; |
725 | |
726 | if (cmd->op == IOMMU_TEST_OP_MOCK_DOMAIN_FLAGS) |
727 | dev_flags = cmd->mock_domain_flags.dev_flags; |
728 | |
729 | sobj->idev.mock_dev = mock_dev_create(dev_flags); |
730 | if (IS_ERR(ptr: sobj->idev.mock_dev)) { |
731 | rc = PTR_ERR(ptr: sobj->idev.mock_dev); |
732 | goto out_sobj; |
733 | } |
734 | |
735 | idev = iommufd_device_bind(ictx: ucmd->ictx, dev: &sobj->idev.mock_dev->dev, |
736 | id: &idev_id); |
737 | if (IS_ERR(ptr: idev)) { |
738 | rc = PTR_ERR(ptr: idev); |
739 | goto out_mdev; |
740 | } |
741 | sobj->idev.idev = idev; |
742 | |
743 | rc = iommufd_device_attach(idev, pt_id: &pt_id); |
744 | if (rc) |
745 | goto out_unbind; |
746 | |
747 | /* Userspace must destroy the device_id to destroy the object */ |
748 | cmd->mock_domain.out_hwpt_id = pt_id; |
749 | cmd->mock_domain.out_stdev_id = sobj->obj.id; |
750 | cmd->mock_domain.out_idev_id = idev_id; |
751 | rc = iommufd_ucmd_respond(ucmd, cmd_len: sizeof(*cmd)); |
752 | if (rc) |
753 | goto out_detach; |
754 | iommufd_object_finalize(ictx: ucmd->ictx, obj: &sobj->obj); |
755 | return 0; |
756 | |
757 | out_detach: |
758 | iommufd_device_detach(idev); |
759 | out_unbind: |
760 | iommufd_device_unbind(idev); |
761 | out_mdev: |
762 | mock_dev_destroy(mdev: sobj->idev.mock_dev); |
763 | out_sobj: |
764 | iommufd_object_abort(ictx: ucmd->ictx, obj: &sobj->obj); |
765 | return rc; |
766 | } |
767 | |
768 | /* Replace the mock domain with a manually allocated hw_pagetable */ |
769 | static int iommufd_test_mock_domain_replace(struct iommufd_ucmd *ucmd, |
770 | unsigned int device_id, u32 pt_id, |
771 | struct iommu_test_cmd *cmd) |
772 | { |
773 | struct iommufd_object *dev_obj; |
774 | struct selftest_obj *sobj; |
775 | int rc; |
776 | |
777 | /* |
778 | * Prefer to use the OBJ_SELFTEST because the destroy_rwsem will ensure |
779 | * it doesn't race with detach, which is not allowed. |
780 | */ |
781 | dev_obj = |
782 | iommufd_get_object(ictx: ucmd->ictx, id: device_id, type: IOMMUFD_OBJ_SELFTEST); |
783 | if (IS_ERR(ptr: dev_obj)) |
784 | return PTR_ERR(ptr: dev_obj); |
785 | |
786 | sobj = container_of(dev_obj, struct selftest_obj, obj); |
787 | if (sobj->type != TYPE_IDEV) { |
788 | rc = -EINVAL; |
789 | goto out_dev_obj; |
790 | } |
791 | |
792 | rc = iommufd_device_replace(idev: sobj->idev.idev, pt_id: &pt_id); |
793 | if (rc) |
794 | goto out_dev_obj; |
795 | |
796 | cmd->mock_domain_replace.pt_id = pt_id; |
797 | rc = iommufd_ucmd_respond(ucmd, cmd_len: sizeof(*cmd)); |
798 | |
799 | out_dev_obj: |
800 | iommufd_put_object(ictx: ucmd->ictx, obj: dev_obj); |
801 | return rc; |
802 | } |
803 | |
804 | /* Add an additional reserved IOVA to the IOAS */ |
805 | static int iommufd_test_add_reserved(struct iommufd_ucmd *ucmd, |
806 | unsigned int mockpt_id, |
807 | unsigned long start, size_t length) |
808 | { |
809 | struct iommufd_ioas *ioas; |
810 | int rc; |
811 | |
812 | ioas = iommufd_get_ioas(ictx: ucmd->ictx, id: mockpt_id); |
813 | if (IS_ERR(ptr: ioas)) |
814 | return PTR_ERR(ptr: ioas); |
815 | down_write(sem: &ioas->iopt.iova_rwsem); |
816 | rc = iopt_reserve_iova(iopt: &ioas->iopt, start, last: start + length - 1, NULL); |
817 | up_write(sem: &ioas->iopt.iova_rwsem); |
818 | iommufd_put_object(ictx: ucmd->ictx, obj: &ioas->obj); |
819 | return rc; |
820 | } |
821 | |
822 | /* Check that every pfn under each iova matches the pfn under a user VA */ |
823 | static int iommufd_test_md_check_pa(struct iommufd_ucmd *ucmd, |
824 | unsigned int mockpt_id, unsigned long iova, |
825 | size_t length, void __user *uptr) |
826 | { |
827 | struct iommufd_hw_pagetable *hwpt; |
828 | struct mock_iommu_domain *mock; |
829 | uintptr_t end; |
830 | int rc; |
831 | |
832 | if (iova % MOCK_IO_PAGE_SIZE || length % MOCK_IO_PAGE_SIZE || |
833 | (uintptr_t)uptr % MOCK_IO_PAGE_SIZE || |
834 | check_add_overflow((uintptr_t)uptr, (uintptr_t)length, &end)) |
835 | return -EINVAL; |
836 | |
837 | hwpt = get_md_pagetable(ucmd, mockpt_id, mock: &mock); |
838 | if (IS_ERR(ptr: hwpt)) |
839 | return PTR_ERR(ptr: hwpt); |
840 | |
841 | for (; length; length -= MOCK_IO_PAGE_SIZE) { |
842 | struct page *pages[1]; |
843 | unsigned long pfn; |
844 | long npages; |
845 | void *ent; |
846 | |
847 | npages = get_user_pages_fast(start: (uintptr_t)uptr & PAGE_MASK, nr_pages: 1, gup_flags: 0, |
848 | pages); |
849 | if (npages < 0) { |
850 | rc = npages; |
851 | goto out_put; |
852 | } |
853 | if (WARN_ON(npages != 1)) { |
854 | rc = -EFAULT; |
855 | goto out_put; |
856 | } |
857 | pfn = page_to_pfn(pages[0]); |
858 | put_page(page: pages[0]); |
859 | |
860 | ent = xa_load(&mock->pfns, index: iova / MOCK_IO_PAGE_SIZE); |
861 | if (!ent || |
862 | (xa_to_value(entry: ent) & MOCK_PFN_MASK) * MOCK_IO_PAGE_SIZE != |
863 | pfn * PAGE_SIZE + ((uintptr_t)uptr % PAGE_SIZE)) { |
864 | rc = -EINVAL; |
865 | goto out_put; |
866 | } |
867 | iova += MOCK_IO_PAGE_SIZE; |
868 | uptr += MOCK_IO_PAGE_SIZE; |
869 | } |
870 | rc = 0; |
871 | |
872 | out_put: |
873 | iommufd_put_object(ictx: ucmd->ictx, obj: &hwpt->obj); |
874 | return rc; |
875 | } |
876 | |
877 | /* Check that the page ref count matches, to look for missing pin/unpins */ |
878 | static int iommufd_test_md_check_refs(struct iommufd_ucmd *ucmd, |
879 | void __user *uptr, size_t length, |
880 | unsigned int refs) |
881 | { |
882 | uintptr_t end; |
883 | |
884 | if (length % PAGE_SIZE || (uintptr_t)uptr % PAGE_SIZE || |
885 | check_add_overflow((uintptr_t)uptr, (uintptr_t)length, &end)) |
886 | return -EINVAL; |
887 | |
888 | for (; length; length -= PAGE_SIZE) { |
889 | struct page *pages[1]; |
890 | long npages; |
891 | |
892 | npages = get_user_pages_fast(start: (uintptr_t)uptr, nr_pages: 1, gup_flags: 0, pages); |
893 | if (npages < 0) |
894 | return npages; |
895 | if (WARN_ON(npages != 1)) |
896 | return -EFAULT; |
897 | if (!PageCompound(page: pages[0])) { |
898 | unsigned int count; |
899 | |
900 | count = page_ref_count(page: pages[0]); |
901 | if (count / GUP_PIN_COUNTING_BIAS != refs) { |
902 | put_page(page: pages[0]); |
903 | return -EIO; |
904 | } |
905 | } |
906 | put_page(page: pages[0]); |
907 | uptr += PAGE_SIZE; |
908 | } |
909 | return 0; |
910 | } |
911 | |
912 | static int iommufd_test_md_check_iotlb(struct iommufd_ucmd *ucmd, |
913 | u32 mockpt_id, unsigned int iotlb_id, |
914 | u32 iotlb) |
915 | { |
916 | struct mock_iommu_domain_nested *mock_nested; |
917 | struct iommufd_hw_pagetable *hwpt; |
918 | int rc = 0; |
919 | |
920 | hwpt = get_md_pagetable_nested(ucmd, mockpt_id, mock_nested: &mock_nested); |
921 | if (IS_ERR(ptr: hwpt)) |
922 | return PTR_ERR(ptr: hwpt); |
923 | |
924 | mock_nested = container_of(hwpt->domain, |
925 | struct mock_iommu_domain_nested, domain); |
926 | |
927 | if (iotlb_id > MOCK_NESTED_DOMAIN_IOTLB_ID_MAX || |
928 | mock_nested->iotlb[iotlb_id] != iotlb) |
929 | rc = -EINVAL; |
930 | iommufd_put_object(ictx: ucmd->ictx, obj: &hwpt->obj); |
931 | return rc; |
932 | } |
933 | |
934 | struct selftest_access { |
935 | struct iommufd_access *access; |
936 | struct file *file; |
937 | struct mutex lock; |
938 | struct list_head items; |
939 | unsigned int next_id; |
940 | bool destroying; |
941 | }; |
942 | |
943 | struct selftest_access_item { |
944 | struct list_head items_elm; |
945 | unsigned long iova; |
946 | size_t length; |
947 | unsigned int id; |
948 | }; |
949 | |
950 | static const struct file_operations iommfd_test_staccess_fops; |
951 | |
952 | static struct selftest_access *iommufd_access_get(int fd) |
953 | { |
954 | struct file *file; |
955 | |
956 | file = fget(fd); |
957 | if (!file) |
958 | return ERR_PTR(error: -EBADFD); |
959 | |
960 | if (file->f_op != &iommfd_test_staccess_fops) { |
961 | fput(file); |
962 | return ERR_PTR(error: -EBADFD); |
963 | } |
964 | return file->private_data; |
965 | } |
966 | |
967 | static void iommufd_test_access_unmap(void *data, unsigned long iova, |
968 | unsigned long length) |
969 | { |
970 | unsigned long iova_last = iova + length - 1; |
971 | struct selftest_access *staccess = data; |
972 | struct selftest_access_item *item; |
973 | struct selftest_access_item *tmp; |
974 | |
975 | mutex_lock(&staccess->lock); |
976 | list_for_each_entry_safe(item, tmp, &staccess->items, items_elm) { |
977 | if (iova > item->iova + item->length - 1 || |
978 | iova_last < item->iova) |
979 | continue; |
980 | list_del(entry: &item->items_elm); |
981 | iommufd_access_unpin_pages(access: staccess->access, iova: item->iova, |
982 | length: item->length); |
983 | kfree(objp: item); |
984 | } |
985 | mutex_unlock(lock: &staccess->lock); |
986 | } |
987 | |
988 | static int iommufd_test_access_item_destroy(struct iommufd_ucmd *ucmd, |
989 | unsigned int access_id, |
990 | unsigned int item_id) |
991 | { |
992 | struct selftest_access_item *item; |
993 | struct selftest_access *staccess; |
994 | |
995 | staccess = iommufd_access_get(fd: access_id); |
996 | if (IS_ERR(ptr: staccess)) |
997 | return PTR_ERR(ptr: staccess); |
998 | |
999 | mutex_lock(&staccess->lock); |
1000 | list_for_each_entry(item, &staccess->items, items_elm) { |
1001 | if (item->id == item_id) { |
1002 | list_del(entry: &item->items_elm); |
1003 | iommufd_access_unpin_pages(access: staccess->access, iova: item->iova, |
1004 | length: item->length); |
1005 | mutex_unlock(lock: &staccess->lock); |
1006 | kfree(objp: item); |
1007 | fput(staccess->file); |
1008 | return 0; |
1009 | } |
1010 | } |
1011 | mutex_unlock(lock: &staccess->lock); |
1012 | fput(staccess->file); |
1013 | return -ENOENT; |
1014 | } |
1015 | |
1016 | static int iommufd_test_staccess_release(struct inode *inode, |
1017 | struct file *filep) |
1018 | { |
1019 | struct selftest_access *staccess = filep->private_data; |
1020 | |
1021 | if (staccess->access) { |
1022 | iommufd_test_access_unmap(data: staccess, iova: 0, ULONG_MAX); |
1023 | iommufd_access_destroy(access: staccess->access); |
1024 | } |
1025 | mutex_destroy(lock: &staccess->lock); |
1026 | kfree(objp: staccess); |
1027 | return 0; |
1028 | } |
1029 | |
1030 | static const struct iommufd_access_ops selftest_access_ops_pin = { |
1031 | .needs_pin_pages = 1, |
1032 | .unmap = iommufd_test_access_unmap, |
1033 | }; |
1034 | |
1035 | static const struct iommufd_access_ops selftest_access_ops = { |
1036 | .unmap = iommufd_test_access_unmap, |
1037 | }; |
1038 | |
1039 | static const struct file_operations iommfd_test_staccess_fops = { |
1040 | .release = iommufd_test_staccess_release, |
1041 | }; |
1042 | |
1043 | static struct selftest_access *iommufd_test_alloc_access(void) |
1044 | { |
1045 | struct selftest_access *staccess; |
1046 | struct file *filep; |
1047 | |
1048 | staccess = kzalloc(size: sizeof(*staccess), GFP_KERNEL_ACCOUNT); |
1049 | if (!staccess) |
1050 | return ERR_PTR(error: -ENOMEM); |
1051 | INIT_LIST_HEAD(list: &staccess->items); |
1052 | mutex_init(&staccess->lock); |
1053 | |
1054 | filep = anon_inode_getfile(name: "[iommufd_test_staccess]" , |
1055 | fops: &iommfd_test_staccess_fops, priv: staccess, |
1056 | O_RDWR); |
1057 | if (IS_ERR(ptr: filep)) { |
1058 | kfree(objp: staccess); |
1059 | return ERR_CAST(ptr: filep); |
1060 | } |
1061 | staccess->file = filep; |
1062 | return staccess; |
1063 | } |
1064 | |
1065 | static int iommufd_test_create_access(struct iommufd_ucmd *ucmd, |
1066 | unsigned int ioas_id, unsigned int flags) |
1067 | { |
1068 | struct iommu_test_cmd *cmd = ucmd->cmd; |
1069 | struct selftest_access *staccess; |
1070 | struct iommufd_access *access; |
1071 | u32 id; |
1072 | int fdno; |
1073 | int rc; |
1074 | |
1075 | if (flags & ~MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES) |
1076 | return -EOPNOTSUPP; |
1077 | |
1078 | staccess = iommufd_test_alloc_access(); |
1079 | if (IS_ERR(ptr: staccess)) |
1080 | return PTR_ERR(ptr: staccess); |
1081 | |
1082 | fdno = get_unused_fd_flags(O_CLOEXEC); |
1083 | if (fdno < 0) { |
1084 | rc = -ENOMEM; |
1085 | goto out_free_staccess; |
1086 | } |
1087 | |
1088 | access = iommufd_access_create( |
1089 | ictx: ucmd->ictx, |
1090 | ops: (flags & MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES) ? |
1091 | &selftest_access_ops_pin : |
1092 | &selftest_access_ops, |
1093 | data: staccess, id: &id); |
1094 | if (IS_ERR(ptr: access)) { |
1095 | rc = PTR_ERR(ptr: access); |
1096 | goto out_put_fdno; |
1097 | } |
1098 | rc = iommufd_access_attach(access, ioas_id); |
1099 | if (rc) |
1100 | goto out_destroy; |
1101 | cmd->create_access.out_access_fd = fdno; |
1102 | rc = iommufd_ucmd_respond(ucmd, cmd_len: sizeof(*cmd)); |
1103 | if (rc) |
1104 | goto out_destroy; |
1105 | |
1106 | staccess->access = access; |
1107 | fd_install(fd: fdno, file: staccess->file); |
1108 | return 0; |
1109 | |
1110 | out_destroy: |
1111 | iommufd_access_destroy(access); |
1112 | out_put_fdno: |
1113 | put_unused_fd(fd: fdno); |
1114 | out_free_staccess: |
1115 | fput(staccess->file); |
1116 | return rc; |
1117 | } |
1118 | |
1119 | static int iommufd_test_access_replace_ioas(struct iommufd_ucmd *ucmd, |
1120 | unsigned int access_id, |
1121 | unsigned int ioas_id) |
1122 | { |
1123 | struct selftest_access *staccess; |
1124 | int rc; |
1125 | |
1126 | staccess = iommufd_access_get(fd: access_id); |
1127 | if (IS_ERR(ptr: staccess)) |
1128 | return PTR_ERR(ptr: staccess); |
1129 | |
1130 | rc = iommufd_access_replace(access: staccess->access, ioas_id); |
1131 | fput(staccess->file); |
1132 | return rc; |
1133 | } |
1134 | |
1135 | /* Check that the pages in a page array match the pages in the user VA */ |
1136 | static int iommufd_test_check_pages(void __user *uptr, struct page **pages, |
1137 | size_t npages) |
1138 | { |
1139 | for (; npages; npages--) { |
1140 | struct page *tmp_pages[1]; |
1141 | long rc; |
1142 | |
1143 | rc = get_user_pages_fast(start: (uintptr_t)uptr, nr_pages: 1, gup_flags: 0, pages: tmp_pages); |
1144 | if (rc < 0) |
1145 | return rc; |
1146 | if (WARN_ON(rc != 1)) |
1147 | return -EFAULT; |
1148 | put_page(page: tmp_pages[0]); |
1149 | if (tmp_pages[0] != *pages) |
1150 | return -EBADE; |
1151 | pages++; |
1152 | uptr += PAGE_SIZE; |
1153 | } |
1154 | return 0; |
1155 | } |
1156 | |
1157 | static int iommufd_test_access_pages(struct iommufd_ucmd *ucmd, |
1158 | unsigned int access_id, unsigned long iova, |
1159 | size_t length, void __user *uptr, |
1160 | u32 flags) |
1161 | { |
1162 | struct iommu_test_cmd *cmd = ucmd->cmd; |
1163 | struct selftest_access_item *item; |
1164 | struct selftest_access *staccess; |
1165 | struct page **pages; |
1166 | size_t npages; |
1167 | int rc; |
1168 | |
1169 | /* Prevent syzkaller from triggering a WARN_ON in kvzalloc() */ |
1170 | if (length > 16*1024*1024) |
1171 | return -ENOMEM; |
1172 | |
1173 | if (flags & ~(MOCK_FLAGS_ACCESS_WRITE | MOCK_FLAGS_ACCESS_SYZ)) |
1174 | return -EOPNOTSUPP; |
1175 | |
1176 | staccess = iommufd_access_get(fd: access_id); |
1177 | if (IS_ERR(ptr: staccess)) |
1178 | return PTR_ERR(ptr: staccess); |
1179 | |
1180 | if (staccess->access->ops != &selftest_access_ops_pin) { |
1181 | rc = -EOPNOTSUPP; |
1182 | goto out_put; |
1183 | } |
1184 | |
1185 | if (flags & MOCK_FLAGS_ACCESS_SYZ) |
1186 | iova = iommufd_test_syz_conv_iova(access: staccess->access, |
1187 | iova: &cmd->access_pages.iova); |
1188 | |
1189 | npages = (ALIGN(iova + length, PAGE_SIZE) - |
1190 | ALIGN_DOWN(iova, PAGE_SIZE)) / |
1191 | PAGE_SIZE; |
1192 | pages = kvcalloc(n: npages, size: sizeof(*pages), GFP_KERNEL_ACCOUNT); |
1193 | if (!pages) { |
1194 | rc = -ENOMEM; |
1195 | goto out_put; |
1196 | } |
1197 | |
1198 | /* |
1199 | * Drivers will need to think very carefully about this locking. The |
1200 | * core code can do multiple unmaps instantaneously after |
1201 | * iommufd_access_pin_pages() and *all* the unmaps must not return until |
1202 | * the range is unpinned. This simple implementation puts a global lock |
1203 | * around the pin, which may not suit drivers that want this to be a |
1204 | * performance path. drivers that get this wrong will trigger WARN_ON |
1205 | * races and cause EDEADLOCK failures to userspace. |
1206 | */ |
1207 | mutex_lock(&staccess->lock); |
1208 | rc = iommufd_access_pin_pages(access: staccess->access, iova, length, out_pages: pages, |
1209 | flags: flags & MOCK_FLAGS_ACCESS_WRITE); |
1210 | if (rc) |
1211 | goto out_unlock; |
1212 | |
1213 | /* For syzkaller allow uptr to be NULL to skip this check */ |
1214 | if (uptr) { |
1215 | rc = iommufd_test_check_pages( |
1216 | uptr: uptr - (iova - ALIGN_DOWN(iova, PAGE_SIZE)), pages, |
1217 | npages); |
1218 | if (rc) |
1219 | goto out_unaccess; |
1220 | } |
1221 | |
1222 | item = kzalloc(size: sizeof(*item), GFP_KERNEL_ACCOUNT); |
1223 | if (!item) { |
1224 | rc = -ENOMEM; |
1225 | goto out_unaccess; |
1226 | } |
1227 | |
1228 | item->iova = iova; |
1229 | item->length = length; |
1230 | item->id = staccess->next_id++; |
1231 | list_add_tail(new: &item->items_elm, head: &staccess->items); |
1232 | |
1233 | cmd->access_pages.out_access_pages_id = item->id; |
1234 | rc = iommufd_ucmd_respond(ucmd, cmd_len: sizeof(*cmd)); |
1235 | if (rc) |
1236 | goto out_free_item; |
1237 | goto out_unlock; |
1238 | |
1239 | out_free_item: |
1240 | list_del(entry: &item->items_elm); |
1241 | kfree(objp: item); |
1242 | out_unaccess: |
1243 | iommufd_access_unpin_pages(access: staccess->access, iova, length); |
1244 | out_unlock: |
1245 | mutex_unlock(lock: &staccess->lock); |
1246 | kvfree(addr: pages); |
1247 | out_put: |
1248 | fput(staccess->file); |
1249 | return rc; |
1250 | } |
1251 | |
1252 | static int iommufd_test_access_rw(struct iommufd_ucmd *ucmd, |
1253 | unsigned int access_id, unsigned long iova, |
1254 | size_t length, void __user *ubuf, |
1255 | unsigned int flags) |
1256 | { |
1257 | struct iommu_test_cmd *cmd = ucmd->cmd; |
1258 | struct selftest_access *staccess; |
1259 | void *tmp; |
1260 | int rc; |
1261 | |
1262 | /* Prevent syzkaller from triggering a WARN_ON in kvzalloc() */ |
1263 | if (length > 16*1024*1024) |
1264 | return -ENOMEM; |
1265 | |
1266 | if (flags & ~(MOCK_ACCESS_RW_WRITE | MOCK_ACCESS_RW_SLOW_PATH | |
1267 | MOCK_FLAGS_ACCESS_SYZ)) |
1268 | return -EOPNOTSUPP; |
1269 | |
1270 | staccess = iommufd_access_get(fd: access_id); |
1271 | if (IS_ERR(ptr: staccess)) |
1272 | return PTR_ERR(ptr: staccess); |
1273 | |
1274 | tmp = kvzalloc(size: length, GFP_KERNEL_ACCOUNT); |
1275 | if (!tmp) { |
1276 | rc = -ENOMEM; |
1277 | goto out_put; |
1278 | } |
1279 | |
1280 | if (flags & MOCK_ACCESS_RW_WRITE) { |
1281 | if (copy_from_user(to: tmp, from: ubuf, n: length)) { |
1282 | rc = -EFAULT; |
1283 | goto out_free; |
1284 | } |
1285 | } |
1286 | |
1287 | if (flags & MOCK_FLAGS_ACCESS_SYZ) |
1288 | iova = iommufd_test_syz_conv_iova(access: staccess->access, |
1289 | iova: &cmd->access_rw.iova); |
1290 | |
1291 | rc = iommufd_access_rw(access: staccess->access, iova, data: tmp, len: length, flags); |
1292 | if (rc) |
1293 | goto out_free; |
1294 | if (!(flags & MOCK_ACCESS_RW_WRITE)) { |
1295 | if (copy_to_user(to: ubuf, from: tmp, n: length)) { |
1296 | rc = -EFAULT; |
1297 | goto out_free; |
1298 | } |
1299 | } |
1300 | |
1301 | out_free: |
1302 | kvfree(addr: tmp); |
1303 | out_put: |
1304 | fput(staccess->file); |
1305 | return rc; |
1306 | } |
1307 | static_assert((unsigned int)MOCK_ACCESS_RW_WRITE == IOMMUFD_ACCESS_RW_WRITE); |
1308 | static_assert((unsigned int)MOCK_ACCESS_RW_SLOW_PATH == |
1309 | __IOMMUFD_ACCESS_RW_SLOW_PATH); |
1310 | |
1311 | static int iommufd_test_dirty(struct iommufd_ucmd *ucmd, unsigned int mockpt_id, |
1312 | unsigned long iova, size_t length, |
1313 | unsigned long page_size, void __user *uptr, |
1314 | u32 flags) |
1315 | { |
1316 | unsigned long bitmap_size, i, max; |
1317 | struct iommu_test_cmd *cmd = ucmd->cmd; |
1318 | struct iommufd_hw_pagetable *hwpt; |
1319 | struct mock_iommu_domain *mock; |
1320 | int rc, count = 0; |
1321 | void *tmp; |
1322 | |
1323 | if (!page_size || !length || iova % page_size || length % page_size || |
1324 | !uptr) |
1325 | return -EINVAL; |
1326 | |
1327 | hwpt = get_md_pagetable(ucmd, mockpt_id, mock: &mock); |
1328 | if (IS_ERR(ptr: hwpt)) |
1329 | return PTR_ERR(ptr: hwpt); |
1330 | |
1331 | if (!(mock->flags & MOCK_DIRTY_TRACK)) { |
1332 | rc = -EINVAL; |
1333 | goto out_put; |
1334 | } |
1335 | |
1336 | max = length / page_size; |
1337 | bitmap_size = max / BITS_PER_BYTE; |
1338 | |
1339 | tmp = kvzalloc(size: bitmap_size, GFP_KERNEL_ACCOUNT); |
1340 | if (!tmp) { |
1341 | rc = -ENOMEM; |
1342 | goto out_put; |
1343 | } |
1344 | |
1345 | if (copy_from_user(to: tmp, from: uptr, n: bitmap_size)) { |
1346 | rc = -EFAULT; |
1347 | goto out_free; |
1348 | } |
1349 | |
1350 | for (i = 0; i < max; i++) { |
1351 | unsigned long cur = iova + i * page_size; |
1352 | void *ent, *old; |
1353 | |
1354 | if (!test_bit(i, (unsigned long *)tmp)) |
1355 | continue; |
1356 | |
1357 | ent = xa_load(&mock->pfns, index: cur / page_size); |
1358 | if (ent) { |
1359 | unsigned long val; |
1360 | |
1361 | val = xa_to_value(entry: ent) | MOCK_PFN_DIRTY_IOVA; |
1362 | old = xa_store(&mock->pfns, index: cur / page_size, |
1363 | entry: xa_mk_value(v: val), GFP_KERNEL); |
1364 | WARN_ON_ONCE(ent != old); |
1365 | count++; |
1366 | } |
1367 | } |
1368 | |
1369 | cmd->dirty.out_nr_dirty = count; |
1370 | rc = iommufd_ucmd_respond(ucmd, cmd_len: sizeof(*cmd)); |
1371 | out_free: |
1372 | kvfree(addr: tmp); |
1373 | out_put: |
1374 | iommufd_put_object(ictx: ucmd->ictx, obj: &hwpt->obj); |
1375 | return rc; |
1376 | } |
1377 | |
1378 | void iommufd_selftest_destroy(struct iommufd_object *obj) |
1379 | { |
1380 | struct selftest_obj *sobj = container_of(obj, struct selftest_obj, obj); |
1381 | |
1382 | switch (sobj->type) { |
1383 | case TYPE_IDEV: |
1384 | iommufd_device_detach(idev: sobj->idev.idev); |
1385 | iommufd_device_unbind(idev: sobj->idev.idev); |
1386 | mock_dev_destroy(mdev: sobj->idev.mock_dev); |
1387 | break; |
1388 | } |
1389 | } |
1390 | |
1391 | int iommufd_test(struct iommufd_ucmd *ucmd) |
1392 | { |
1393 | struct iommu_test_cmd *cmd = ucmd->cmd; |
1394 | |
1395 | switch (cmd->op) { |
1396 | case IOMMU_TEST_OP_ADD_RESERVED: |
1397 | return iommufd_test_add_reserved(ucmd, mockpt_id: cmd->id, |
1398 | start: cmd->add_reserved.start, |
1399 | length: cmd->add_reserved.length); |
1400 | case IOMMU_TEST_OP_MOCK_DOMAIN: |
1401 | case IOMMU_TEST_OP_MOCK_DOMAIN_FLAGS: |
1402 | return iommufd_test_mock_domain(ucmd, cmd); |
1403 | case IOMMU_TEST_OP_MOCK_DOMAIN_REPLACE: |
1404 | return iommufd_test_mock_domain_replace( |
1405 | ucmd, device_id: cmd->id, pt_id: cmd->mock_domain_replace.pt_id, cmd); |
1406 | case IOMMU_TEST_OP_MD_CHECK_MAP: |
1407 | return iommufd_test_md_check_pa( |
1408 | ucmd, mockpt_id: cmd->id, iova: cmd->check_map.iova, |
1409 | length: cmd->check_map.length, |
1410 | u64_to_user_ptr(cmd->check_map.uptr)); |
1411 | case IOMMU_TEST_OP_MD_CHECK_REFS: |
1412 | return iommufd_test_md_check_refs( |
1413 | ucmd, u64_to_user_ptr(cmd->check_refs.uptr), |
1414 | length: cmd->check_refs.length, refs: cmd->check_refs.refs); |
1415 | case IOMMU_TEST_OP_MD_CHECK_IOTLB: |
1416 | return iommufd_test_md_check_iotlb(ucmd, mockpt_id: cmd->id, |
1417 | iotlb_id: cmd->check_iotlb.id, |
1418 | iotlb: cmd->check_iotlb.iotlb); |
1419 | case IOMMU_TEST_OP_CREATE_ACCESS: |
1420 | return iommufd_test_create_access(ucmd, ioas_id: cmd->id, |
1421 | flags: cmd->create_access.flags); |
1422 | case IOMMU_TEST_OP_ACCESS_REPLACE_IOAS: |
1423 | return iommufd_test_access_replace_ioas( |
1424 | ucmd, access_id: cmd->id, ioas_id: cmd->access_replace_ioas.ioas_id); |
1425 | case IOMMU_TEST_OP_ACCESS_PAGES: |
1426 | return iommufd_test_access_pages( |
1427 | ucmd, access_id: cmd->id, iova: cmd->access_pages.iova, |
1428 | length: cmd->access_pages.length, |
1429 | u64_to_user_ptr(cmd->access_pages.uptr), |
1430 | flags: cmd->access_pages.flags); |
1431 | case IOMMU_TEST_OP_ACCESS_RW: |
1432 | return iommufd_test_access_rw( |
1433 | ucmd, access_id: cmd->id, iova: cmd->access_rw.iova, |
1434 | length: cmd->access_rw.length, |
1435 | u64_to_user_ptr(cmd->access_rw.uptr), |
1436 | flags: cmd->access_rw.flags); |
1437 | case IOMMU_TEST_OP_DESTROY_ACCESS_PAGES: |
1438 | return iommufd_test_access_item_destroy( |
1439 | ucmd, access_id: cmd->id, item_id: cmd->destroy_access_pages.access_pages_id); |
1440 | case IOMMU_TEST_OP_SET_TEMP_MEMORY_LIMIT: |
1441 | /* Protect _batch_init(), can not be less than elmsz */ |
1442 | if (cmd->memory_limit.limit < |
1443 | sizeof(unsigned long) + sizeof(u32)) |
1444 | return -EINVAL; |
1445 | iommufd_test_memory_limit = cmd->memory_limit.limit; |
1446 | return 0; |
1447 | case IOMMU_TEST_OP_DIRTY: |
1448 | return iommufd_test_dirty(ucmd, mockpt_id: cmd->id, iova: cmd->dirty.iova, |
1449 | length: cmd->dirty.length, |
1450 | page_size: cmd->dirty.page_size, |
1451 | u64_to_user_ptr(cmd->dirty.uptr), |
1452 | flags: cmd->dirty.flags); |
1453 | default: |
1454 | return -EOPNOTSUPP; |
1455 | } |
1456 | } |
1457 | |
1458 | bool iommufd_should_fail(void) |
1459 | { |
1460 | return should_fail(attr: &fail_iommufd, size: 1); |
1461 | } |
1462 | |
1463 | int __init iommufd_test_init(void) |
1464 | { |
1465 | struct platform_device_info pdevinfo = { |
1466 | .name = "iommufd_selftest_iommu" , |
1467 | }; |
1468 | int rc; |
1469 | |
1470 | dbgfs_root = |
1471 | fault_create_debugfs_attr(name: "fail_iommufd" , NULL, attr: &fail_iommufd); |
1472 | |
1473 | selftest_iommu_dev = platform_device_register_full(pdevinfo: &pdevinfo); |
1474 | if (IS_ERR(ptr: selftest_iommu_dev)) { |
1475 | rc = PTR_ERR(ptr: selftest_iommu_dev); |
1476 | goto err_dbgfs; |
1477 | } |
1478 | |
1479 | rc = bus_register(bus: &iommufd_mock_bus_type.bus); |
1480 | if (rc) |
1481 | goto err_platform; |
1482 | |
1483 | rc = iommu_device_sysfs_add(iommu: &mock_iommu_device, |
1484 | parent: &selftest_iommu_dev->dev, NULL, fmt: "%s" , |
1485 | dev_name(dev: &selftest_iommu_dev->dev)); |
1486 | if (rc) |
1487 | goto err_bus; |
1488 | |
1489 | rc = iommu_device_register_bus(iommu: &mock_iommu_device, ops: &mock_ops, |
1490 | bus: &iommufd_mock_bus_type.bus, |
1491 | nb: &iommufd_mock_bus_type.nb); |
1492 | if (rc) |
1493 | goto err_sysfs; |
1494 | return 0; |
1495 | |
1496 | err_sysfs: |
1497 | iommu_device_sysfs_remove(iommu: &mock_iommu_device); |
1498 | err_bus: |
1499 | bus_unregister(bus: &iommufd_mock_bus_type.bus); |
1500 | err_platform: |
1501 | platform_device_unregister(selftest_iommu_dev); |
1502 | err_dbgfs: |
1503 | debugfs_remove_recursive(dentry: dbgfs_root); |
1504 | return rc; |
1505 | } |
1506 | |
1507 | void iommufd_test_exit(void) |
1508 | { |
1509 | iommu_device_sysfs_remove(iommu: &mock_iommu_device); |
1510 | iommu_device_unregister_bus(iommu: &mock_iommu_device, |
1511 | bus: &iommufd_mock_bus_type.bus, |
1512 | nb: &iommufd_mock_bus_type.nb); |
1513 | bus_unregister(bus: &iommufd_mock_bus_type.bus); |
1514 | platform_device_unregister(selftest_iommu_dev); |
1515 | debugfs_remove_recursive(dentry: dbgfs_root); |
1516 | } |
1517 | |