1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Implementation of the IOMMU SVA API for the ARM SMMUv3 |
4 | */ |
5 | |
6 | #include <linux/mm.h> |
7 | #include <linux/mmu_context.h> |
8 | #include <linux/mmu_notifier.h> |
9 | #include <linux/sched/mm.h> |
10 | #include <linux/slab.h> |
11 | |
12 | #include "arm-smmu-v3.h" |
13 | #include "../../io-pgtable-arm.h" |
14 | |
15 | struct arm_smmu_mmu_notifier { |
16 | struct mmu_notifier mn; |
17 | struct arm_smmu_ctx_desc *cd; |
18 | bool cleared; |
19 | refcount_t refs; |
20 | struct list_head list; |
21 | struct arm_smmu_domain *domain; |
22 | }; |
23 | |
24 | #define mn_to_smmu(mn) container_of(mn, struct arm_smmu_mmu_notifier, mn) |
25 | |
26 | struct arm_smmu_bond { |
27 | struct mm_struct *mm; |
28 | struct arm_smmu_mmu_notifier *smmu_mn; |
29 | struct list_head list; |
30 | }; |
31 | |
32 | #define sva_to_bond(handle) \ |
33 | container_of(handle, struct arm_smmu_bond, sva) |
34 | |
35 | static DEFINE_MUTEX(sva_lock); |
36 | |
37 | /* |
38 | * Write the CD to the CD tables for all masters that this domain is attached |
39 | * to. Note that this is only used to update existing CD entries in the target |
40 | * CD table, for which it's assumed that arm_smmu_write_ctx_desc can't fail. |
41 | */ |
42 | static void arm_smmu_update_ctx_desc_devices(struct arm_smmu_domain *smmu_domain, |
43 | int ssid, |
44 | struct arm_smmu_ctx_desc *cd) |
45 | { |
46 | struct arm_smmu_master *master; |
47 | unsigned long flags; |
48 | |
49 | spin_lock_irqsave(&smmu_domain->devices_lock, flags); |
50 | list_for_each_entry(master, &smmu_domain->devices, domain_head) { |
51 | arm_smmu_write_ctx_desc(smmu_master: master, ssid, cd); |
52 | } |
53 | spin_unlock_irqrestore(lock: &smmu_domain->devices_lock, flags); |
54 | } |
55 | |
56 | /* |
57 | * Check if the CPU ASID is available on the SMMU side. If a private context |
58 | * descriptor is using it, try to replace it. |
59 | */ |
60 | static struct arm_smmu_ctx_desc * |
61 | arm_smmu_share_asid(struct mm_struct *mm, u16 asid) |
62 | { |
63 | int ret; |
64 | u32 new_asid; |
65 | struct arm_smmu_ctx_desc *cd; |
66 | struct arm_smmu_device *smmu; |
67 | struct arm_smmu_domain *smmu_domain; |
68 | |
69 | cd = xa_load(&arm_smmu_asid_xa, index: asid); |
70 | if (!cd) |
71 | return NULL; |
72 | |
73 | if (cd->mm) { |
74 | if (WARN_ON(cd->mm != mm)) |
75 | return ERR_PTR(error: -EINVAL); |
76 | /* All devices bound to this mm use the same cd struct. */ |
77 | refcount_inc(r: &cd->refs); |
78 | return cd; |
79 | } |
80 | |
81 | smmu_domain = container_of(cd, struct arm_smmu_domain, cd); |
82 | smmu = smmu_domain->smmu; |
83 | |
84 | ret = xa_alloc(xa: &arm_smmu_asid_xa, id: &new_asid, entry: cd, |
85 | XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL); |
86 | if (ret) |
87 | return ERR_PTR(error: -ENOSPC); |
88 | /* |
89 | * Race with unmap: TLB invalidations will start targeting the new ASID, |
90 | * which isn't assigned yet. We'll do an invalidate-all on the old ASID |
91 | * later, so it doesn't matter. |
92 | */ |
93 | cd->asid = new_asid; |
94 | /* |
95 | * Update ASID and invalidate CD in all associated masters. There will |
96 | * be some overlap between use of both ASIDs, until we invalidate the |
97 | * TLB. |
98 | */ |
99 | arm_smmu_update_ctx_desc_devices(smmu_domain, IOMMU_NO_PASID, cd); |
100 | |
101 | /* Invalidate TLB entries previously associated with that context */ |
102 | arm_smmu_tlb_inv_asid(smmu, asid); |
103 | |
104 | xa_erase(&arm_smmu_asid_xa, index: asid); |
105 | return NULL; |
106 | } |
107 | |
108 | static struct arm_smmu_ctx_desc *arm_smmu_alloc_shared_cd(struct mm_struct *mm) |
109 | { |
110 | u16 asid; |
111 | int err = 0; |
112 | u64 tcr, par, reg; |
113 | struct arm_smmu_ctx_desc *cd; |
114 | struct arm_smmu_ctx_desc *ret = NULL; |
115 | |
116 | /* Don't free the mm until we release the ASID */ |
117 | mmgrab(mm); |
118 | |
119 | asid = arm64_mm_context_get(mm); |
120 | if (!asid) { |
121 | err = -ESRCH; |
122 | goto out_drop_mm; |
123 | } |
124 | |
125 | cd = kzalloc(size: sizeof(*cd), GFP_KERNEL); |
126 | if (!cd) { |
127 | err = -ENOMEM; |
128 | goto out_put_context; |
129 | } |
130 | |
131 | refcount_set(r: &cd->refs, n: 1); |
132 | |
133 | mutex_lock(&arm_smmu_asid_lock); |
134 | ret = arm_smmu_share_asid(mm, asid); |
135 | if (ret) { |
136 | mutex_unlock(lock: &arm_smmu_asid_lock); |
137 | goto out_free_cd; |
138 | } |
139 | |
140 | err = xa_insert(xa: &arm_smmu_asid_xa, index: asid, entry: cd, GFP_KERNEL); |
141 | mutex_unlock(lock: &arm_smmu_asid_lock); |
142 | |
143 | if (err) |
144 | goto out_free_asid; |
145 | |
146 | tcr = FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, 64ULL - vabits_actual) | |
147 | FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, ARM_LPAE_TCR_RGN_WBWA) | |
148 | FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, ARM_LPAE_TCR_RGN_WBWA) | |
149 | FIELD_PREP(CTXDESC_CD_0_TCR_SH0, ARM_LPAE_TCR_SH_IS) | |
150 | CTXDESC_CD_0_TCR_EPD1 | CTXDESC_CD_0_AA64; |
151 | |
152 | switch (PAGE_SIZE) { |
153 | case SZ_4K: |
154 | tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_TG0, ARM_LPAE_TCR_TG0_4K); |
155 | break; |
156 | case SZ_16K: |
157 | tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_TG0, ARM_LPAE_TCR_TG0_16K); |
158 | break; |
159 | case SZ_64K: |
160 | tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_TG0, ARM_LPAE_TCR_TG0_64K); |
161 | break; |
162 | default: |
163 | WARN_ON(1); |
164 | err = -EINVAL; |
165 | goto out_free_asid; |
166 | } |
167 | |
168 | reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); |
169 | par = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_PARANGE_SHIFT); |
170 | tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_IPS, par); |
171 | |
172 | cd->ttbr = virt_to_phys(address: mm->pgd); |
173 | cd->tcr = tcr; |
174 | /* |
175 | * MAIR value is pretty much constant and global, so we can just get it |
176 | * from the current CPU register |
177 | */ |
178 | cd->mair = read_sysreg(mair_el1); |
179 | cd->asid = asid; |
180 | cd->mm = mm; |
181 | |
182 | return cd; |
183 | |
184 | out_free_asid: |
185 | arm_smmu_free_asid(cd); |
186 | out_free_cd: |
187 | kfree(objp: cd); |
188 | out_put_context: |
189 | arm64_mm_context_put(mm); |
190 | out_drop_mm: |
191 | mmdrop(mm); |
192 | return err < 0 ? ERR_PTR(error: err) : ret; |
193 | } |
194 | |
195 | static void arm_smmu_free_shared_cd(struct arm_smmu_ctx_desc *cd) |
196 | { |
197 | if (arm_smmu_free_asid(cd)) { |
198 | /* Unpin ASID */ |
199 | arm64_mm_context_put(cd->mm); |
200 | mmdrop(mm: cd->mm); |
201 | kfree(objp: cd); |
202 | } |
203 | } |
204 | |
205 | /* |
206 | * Cloned from the MAX_TLBI_OPS in arch/arm64/include/asm/tlbflush.h, this |
207 | * is used as a threshold to replace per-page TLBI commands to issue in the |
208 | * command queue with an address-space TLBI command, when SMMU w/o a range |
209 | * invalidation feature handles too many per-page TLBI commands, which will |
210 | * otherwise result in a soft lockup. |
211 | */ |
212 | #define CMDQ_MAX_TLBI_OPS (1 << (PAGE_SHIFT - 3)) |
213 | |
214 | static void arm_smmu_mm_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn, |
215 | struct mm_struct *mm, |
216 | unsigned long start, |
217 | unsigned long end) |
218 | { |
219 | struct arm_smmu_mmu_notifier *smmu_mn = mn_to_smmu(mn); |
220 | struct arm_smmu_domain *smmu_domain = smmu_mn->domain; |
221 | size_t size; |
222 | |
223 | /* |
224 | * The mm_types defines vm_end as the first byte after the end address, |
225 | * different from IOMMU subsystem using the last address of an address |
226 | * range. So do a simple translation here by calculating size correctly. |
227 | */ |
228 | size = end - start; |
229 | if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_RANGE_INV)) { |
230 | if (size >= CMDQ_MAX_TLBI_OPS * PAGE_SIZE) |
231 | size = 0; |
232 | } else { |
233 | if (size == ULONG_MAX) |
234 | size = 0; |
235 | } |
236 | |
237 | if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_BTM)) { |
238 | if (!size) |
239 | arm_smmu_tlb_inv_asid(smmu: smmu_domain->smmu, |
240 | asid: smmu_mn->cd->asid); |
241 | else |
242 | arm_smmu_tlb_inv_range_asid(iova: start, size, |
243 | asid: smmu_mn->cd->asid, |
244 | PAGE_SIZE, leaf: false, |
245 | smmu_domain); |
246 | } |
247 | |
248 | arm_smmu_atc_inv_domain(smmu_domain, ssid: mm_get_enqcmd_pasid(mm), iova: start, |
249 | size); |
250 | } |
251 | |
252 | static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) |
253 | { |
254 | struct arm_smmu_mmu_notifier *smmu_mn = mn_to_smmu(mn); |
255 | struct arm_smmu_domain *smmu_domain = smmu_mn->domain; |
256 | |
257 | mutex_lock(&sva_lock); |
258 | if (smmu_mn->cleared) { |
259 | mutex_unlock(lock: &sva_lock); |
260 | return; |
261 | } |
262 | |
263 | /* |
264 | * DMA may still be running. Keep the cd valid to avoid C_BAD_CD events, |
265 | * but disable translation. |
266 | */ |
267 | arm_smmu_update_ctx_desc_devices(smmu_domain, ssid: mm_get_enqcmd_pasid(mm), |
268 | cd: &quiet_cd); |
269 | |
270 | arm_smmu_tlb_inv_asid(smmu: smmu_domain->smmu, asid: smmu_mn->cd->asid); |
271 | arm_smmu_atc_inv_domain(smmu_domain, ssid: mm_get_enqcmd_pasid(mm), iova: 0, size: 0); |
272 | |
273 | smmu_mn->cleared = true; |
274 | mutex_unlock(lock: &sva_lock); |
275 | } |
276 | |
277 | static void arm_smmu_mmu_notifier_free(struct mmu_notifier *mn) |
278 | { |
279 | kfree(mn_to_smmu(mn)); |
280 | } |
281 | |
282 | static const struct mmu_notifier_ops arm_smmu_mmu_notifier_ops = { |
283 | .arch_invalidate_secondary_tlbs = arm_smmu_mm_arch_invalidate_secondary_tlbs, |
284 | .release = arm_smmu_mm_release, |
285 | .free_notifier = arm_smmu_mmu_notifier_free, |
286 | }; |
287 | |
288 | /* Allocate or get existing MMU notifier for this {domain, mm} pair */ |
289 | static struct arm_smmu_mmu_notifier * |
290 | arm_smmu_mmu_notifier_get(struct arm_smmu_domain *smmu_domain, |
291 | struct mm_struct *mm) |
292 | { |
293 | int ret; |
294 | struct arm_smmu_ctx_desc *cd; |
295 | struct arm_smmu_mmu_notifier *smmu_mn; |
296 | |
297 | list_for_each_entry(smmu_mn, &smmu_domain->mmu_notifiers, list) { |
298 | if (smmu_mn->mn.mm == mm) { |
299 | refcount_inc(r: &smmu_mn->refs); |
300 | return smmu_mn; |
301 | } |
302 | } |
303 | |
304 | cd = arm_smmu_alloc_shared_cd(mm); |
305 | if (IS_ERR(ptr: cd)) |
306 | return ERR_CAST(ptr: cd); |
307 | |
308 | smmu_mn = kzalloc(size: sizeof(*smmu_mn), GFP_KERNEL); |
309 | if (!smmu_mn) { |
310 | ret = -ENOMEM; |
311 | goto err_free_cd; |
312 | } |
313 | |
314 | refcount_set(r: &smmu_mn->refs, n: 1); |
315 | smmu_mn->cd = cd; |
316 | smmu_mn->domain = smmu_domain; |
317 | smmu_mn->mn.ops = &arm_smmu_mmu_notifier_ops; |
318 | |
319 | ret = mmu_notifier_register(subscription: &smmu_mn->mn, mm); |
320 | if (ret) { |
321 | kfree(objp: smmu_mn); |
322 | goto err_free_cd; |
323 | } |
324 | |
325 | list_add(new: &smmu_mn->list, head: &smmu_domain->mmu_notifiers); |
326 | return smmu_mn; |
327 | |
328 | err_free_cd: |
329 | arm_smmu_free_shared_cd(cd); |
330 | return ERR_PTR(error: ret); |
331 | } |
332 | |
333 | static void arm_smmu_mmu_notifier_put(struct arm_smmu_mmu_notifier *smmu_mn) |
334 | { |
335 | struct mm_struct *mm = smmu_mn->mn.mm; |
336 | struct arm_smmu_ctx_desc *cd = smmu_mn->cd; |
337 | struct arm_smmu_domain *smmu_domain = smmu_mn->domain; |
338 | |
339 | if (!refcount_dec_and_test(r: &smmu_mn->refs)) |
340 | return; |
341 | |
342 | list_del(entry: &smmu_mn->list); |
343 | |
344 | /* |
345 | * If we went through clear(), we've already invalidated, and no |
346 | * new TLB entry can have been formed. |
347 | */ |
348 | if (!smmu_mn->cleared) { |
349 | arm_smmu_tlb_inv_asid(smmu: smmu_domain->smmu, asid: cd->asid); |
350 | arm_smmu_atc_inv_domain(smmu_domain, ssid: mm_get_enqcmd_pasid(mm), iova: 0, |
351 | size: 0); |
352 | } |
353 | |
354 | /* Frees smmu_mn */ |
355 | mmu_notifier_put(subscription: &smmu_mn->mn); |
356 | arm_smmu_free_shared_cd(cd); |
357 | } |
358 | |
359 | static int __arm_smmu_sva_bind(struct device *dev, ioasid_t pasid, |
360 | struct mm_struct *mm) |
361 | { |
362 | int ret; |
363 | struct arm_smmu_bond *bond; |
364 | struct arm_smmu_master *master = dev_iommu_priv_get(dev); |
365 | struct iommu_domain *domain = iommu_get_domain_for_dev(dev); |
366 | struct arm_smmu_domain *smmu_domain; |
367 | |
368 | if (!(domain->type & __IOMMU_DOMAIN_PAGING)) |
369 | return -ENODEV; |
370 | smmu_domain = to_smmu_domain(dom: domain); |
371 | if (smmu_domain->stage != ARM_SMMU_DOMAIN_S1) |
372 | return -ENODEV; |
373 | |
374 | if (!master || !master->sva_enabled) |
375 | return -ENODEV; |
376 | |
377 | bond = kzalloc(size: sizeof(*bond), GFP_KERNEL); |
378 | if (!bond) |
379 | return -ENOMEM; |
380 | |
381 | bond->mm = mm; |
382 | |
383 | bond->smmu_mn = arm_smmu_mmu_notifier_get(smmu_domain, mm); |
384 | if (IS_ERR(ptr: bond->smmu_mn)) { |
385 | ret = PTR_ERR(ptr: bond->smmu_mn); |
386 | goto err_free_bond; |
387 | } |
388 | |
389 | ret = arm_smmu_write_ctx_desc(smmu_master: master, ssid: pasid, cd: bond->smmu_mn->cd); |
390 | if (ret) |
391 | goto err_put_notifier; |
392 | |
393 | list_add(new: &bond->list, head: &master->bonds); |
394 | return 0; |
395 | |
396 | err_put_notifier: |
397 | arm_smmu_mmu_notifier_put(smmu_mn: bond->smmu_mn); |
398 | err_free_bond: |
399 | kfree(objp: bond); |
400 | return ret; |
401 | } |
402 | |
403 | bool arm_smmu_sva_supported(struct arm_smmu_device *smmu) |
404 | { |
405 | unsigned long reg, fld; |
406 | unsigned long oas; |
407 | unsigned long asid_bits; |
408 | u32 feat_mask = ARM_SMMU_FEAT_COHERENCY; |
409 | |
410 | if (vabits_actual == 52) |
411 | feat_mask |= ARM_SMMU_FEAT_VAX; |
412 | |
413 | if ((smmu->features & feat_mask) != feat_mask) |
414 | return false; |
415 | |
416 | if (!(smmu->pgsize_bitmap & PAGE_SIZE)) |
417 | return false; |
418 | |
419 | /* |
420 | * Get the smallest PA size of all CPUs (sanitized by cpufeature). We're |
421 | * not even pretending to support AArch32 here. Abort if the MMU outputs |
422 | * addresses larger than what we support. |
423 | */ |
424 | reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); |
425 | fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_PARANGE_SHIFT); |
426 | oas = id_aa64mmfr0_parange_to_phys_shift(fld); |
427 | if (smmu->oas < oas) |
428 | return false; |
429 | |
430 | /* We can support bigger ASIDs than the CPU, but not smaller */ |
431 | fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_ASIDBITS_SHIFT); |
432 | asid_bits = fld ? 16 : 8; |
433 | if (smmu->asid_bits < asid_bits) |
434 | return false; |
435 | |
436 | /* |
437 | * See max_pinned_asids in arch/arm64/mm/context.c. The following is |
438 | * generally the maximum number of bindable processes. |
439 | */ |
440 | if (arm64_kernel_unmapped_at_el0()) |
441 | asid_bits--; |
442 | dev_dbg(smmu->dev, "%d shared contexts\n" , (1 << asid_bits) - |
443 | num_possible_cpus() - 2); |
444 | |
445 | return true; |
446 | } |
447 | |
448 | bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master) |
449 | { |
450 | /* We're not keeping track of SIDs in fault events */ |
451 | if (master->num_streams != 1) |
452 | return false; |
453 | |
454 | return master->stall_enabled; |
455 | } |
456 | |
457 | bool arm_smmu_master_sva_supported(struct arm_smmu_master *master) |
458 | { |
459 | if (!(master->smmu->features & ARM_SMMU_FEAT_SVA)) |
460 | return false; |
461 | |
462 | /* SSID support is mandatory for the moment */ |
463 | return master->ssid_bits; |
464 | } |
465 | |
466 | bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master) |
467 | { |
468 | bool enabled; |
469 | |
470 | mutex_lock(&sva_lock); |
471 | enabled = master->sva_enabled; |
472 | mutex_unlock(lock: &sva_lock); |
473 | return enabled; |
474 | } |
475 | |
476 | static int arm_smmu_master_sva_enable_iopf(struct arm_smmu_master *master) |
477 | { |
478 | struct device *dev = master->dev; |
479 | |
480 | /* |
481 | * Drivers for devices supporting PRI or stall should enable IOPF first. |
482 | * Others have device-specific fault handlers and don't need IOPF. |
483 | */ |
484 | if (!arm_smmu_master_iopf_supported(master)) |
485 | return 0; |
486 | |
487 | if (!master->iopf_enabled) |
488 | return -EINVAL; |
489 | |
490 | return iopf_queue_add_device(queue: master->smmu->evtq.iopf, dev); |
491 | } |
492 | |
493 | static void arm_smmu_master_sva_disable_iopf(struct arm_smmu_master *master) |
494 | { |
495 | struct device *dev = master->dev; |
496 | |
497 | if (!master->iopf_enabled) |
498 | return; |
499 | |
500 | iopf_queue_remove_device(queue: master->smmu->evtq.iopf, dev); |
501 | } |
502 | |
503 | int arm_smmu_master_enable_sva(struct arm_smmu_master *master) |
504 | { |
505 | int ret; |
506 | |
507 | mutex_lock(&sva_lock); |
508 | ret = arm_smmu_master_sva_enable_iopf(master); |
509 | if (!ret) |
510 | master->sva_enabled = true; |
511 | mutex_unlock(lock: &sva_lock); |
512 | |
513 | return ret; |
514 | } |
515 | |
516 | int arm_smmu_master_disable_sva(struct arm_smmu_master *master) |
517 | { |
518 | mutex_lock(&sva_lock); |
519 | if (!list_empty(head: &master->bonds)) { |
520 | dev_err(master->dev, "cannot disable SVA, device is bound\n" ); |
521 | mutex_unlock(lock: &sva_lock); |
522 | return -EBUSY; |
523 | } |
524 | arm_smmu_master_sva_disable_iopf(master); |
525 | master->sva_enabled = false; |
526 | mutex_unlock(lock: &sva_lock); |
527 | |
528 | return 0; |
529 | } |
530 | |
531 | void arm_smmu_sva_notifier_synchronize(void) |
532 | { |
533 | /* |
534 | * Some MMU notifiers may still be waiting to be freed, using |
535 | * arm_smmu_mmu_notifier_free(). Wait for them. |
536 | */ |
537 | mmu_notifier_synchronize(); |
538 | } |
539 | |
540 | void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain, |
541 | struct device *dev, ioasid_t id) |
542 | { |
543 | struct mm_struct *mm = domain->mm; |
544 | struct arm_smmu_bond *bond = NULL, *t; |
545 | struct arm_smmu_master *master = dev_iommu_priv_get(dev); |
546 | |
547 | mutex_lock(&sva_lock); |
548 | |
549 | arm_smmu_write_ctx_desc(smmu_master: master, ssid: id, NULL); |
550 | |
551 | list_for_each_entry(t, &master->bonds, list) { |
552 | if (t->mm == mm) { |
553 | bond = t; |
554 | break; |
555 | } |
556 | } |
557 | |
558 | if (!WARN_ON(!bond)) { |
559 | list_del(entry: &bond->list); |
560 | arm_smmu_mmu_notifier_put(smmu_mn: bond->smmu_mn); |
561 | kfree(objp: bond); |
562 | } |
563 | mutex_unlock(lock: &sva_lock); |
564 | } |
565 | |
566 | static int arm_smmu_sva_set_dev_pasid(struct iommu_domain *domain, |
567 | struct device *dev, ioasid_t id) |
568 | { |
569 | int ret = 0; |
570 | struct mm_struct *mm = domain->mm; |
571 | |
572 | mutex_lock(&sva_lock); |
573 | ret = __arm_smmu_sva_bind(dev, pasid: id, mm); |
574 | mutex_unlock(lock: &sva_lock); |
575 | |
576 | return ret; |
577 | } |
578 | |
579 | static void arm_smmu_sva_domain_free(struct iommu_domain *domain) |
580 | { |
581 | kfree(objp: domain); |
582 | } |
583 | |
584 | static const struct iommu_domain_ops arm_smmu_sva_domain_ops = { |
585 | .set_dev_pasid = arm_smmu_sva_set_dev_pasid, |
586 | .free = arm_smmu_sva_domain_free |
587 | }; |
588 | |
589 | struct iommu_domain *arm_smmu_sva_domain_alloc(void) |
590 | { |
591 | struct iommu_domain *domain; |
592 | |
593 | domain = kzalloc(size: sizeof(*domain), GFP_KERNEL); |
594 | if (!domain) |
595 | return NULL; |
596 | domain->ops = &arm_smmu_sva_domain_ops; |
597 | |
598 | return domain; |
599 | } |
600 | |