1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (C) 2013 Red Hat |
4 | * Author: Rob Clark <robdclark@gmail.com> |
5 | */ |
6 | |
7 | #include <linux/adreno-smmu-priv.h> |
8 | #include <linux/io-pgtable.h> |
9 | #include "msm_drv.h" |
10 | #include "msm_mmu.h" |
11 | |
12 | struct msm_iommu { |
13 | struct msm_mmu base; |
14 | struct iommu_domain *domain; |
15 | atomic_t pagetables; |
16 | }; |
17 | |
18 | #define to_msm_iommu(x) container_of(x, struct msm_iommu, base) |
19 | |
20 | struct msm_iommu_pagetable { |
21 | struct msm_mmu base; |
22 | struct msm_mmu *parent; |
23 | struct io_pgtable_ops *pgtbl_ops; |
24 | const struct iommu_flush_ops *tlb; |
25 | struct device *iommu_dev; |
26 | unsigned long pgsize_bitmap; /* Bitmap of page sizes in use */ |
27 | phys_addr_t ttbr; |
28 | u32 asid; |
29 | }; |
30 | static struct msm_iommu_pagetable *to_pagetable(struct msm_mmu *mmu) |
31 | { |
32 | return container_of(mmu, struct msm_iommu_pagetable, base); |
33 | } |
34 | |
35 | /* based on iommu_pgsize() in iommu.c: */ |
36 | static size_t calc_pgsize(struct msm_iommu_pagetable *pagetable, |
37 | unsigned long iova, phys_addr_t paddr, |
38 | size_t size, size_t *count) |
39 | { |
40 | unsigned int pgsize_idx, pgsize_idx_next; |
41 | unsigned long pgsizes; |
42 | size_t offset, pgsize, pgsize_next; |
43 | unsigned long addr_merge = paddr | iova; |
44 | |
45 | /* Page sizes supported by the hardware and small enough for @size */ |
46 | pgsizes = pagetable->pgsize_bitmap & GENMASK(__fls(size), 0); |
47 | |
48 | /* Constrain the page sizes further based on the maximum alignment */ |
49 | if (likely(addr_merge)) |
50 | pgsizes &= GENMASK(__ffs(addr_merge), 0); |
51 | |
52 | /* Make sure we have at least one suitable page size */ |
53 | BUG_ON(!pgsizes); |
54 | |
55 | /* Pick the biggest page size remaining */ |
56 | pgsize_idx = __fls(word: pgsizes); |
57 | pgsize = BIT(pgsize_idx); |
58 | if (!count) |
59 | return pgsize; |
60 | |
61 | /* Find the next biggest support page size, if it exists */ |
62 | pgsizes = pagetable->pgsize_bitmap & ~GENMASK(pgsize_idx, 0); |
63 | if (!pgsizes) |
64 | goto out_set_count; |
65 | |
66 | pgsize_idx_next = __ffs(pgsizes); |
67 | pgsize_next = BIT(pgsize_idx_next); |
68 | |
69 | /* |
70 | * There's no point trying a bigger page size unless the virtual |
71 | * and physical addresses are similarly offset within the larger page. |
72 | */ |
73 | if ((iova ^ paddr) & (pgsize_next - 1)) |
74 | goto out_set_count; |
75 | |
76 | /* Calculate the offset to the next page size alignment boundary */ |
77 | offset = pgsize_next - (addr_merge & (pgsize_next - 1)); |
78 | |
79 | /* |
80 | * If size is big enough to accommodate the larger page, reduce |
81 | * the number of smaller pages. |
82 | */ |
83 | if (offset + pgsize_next <= size) |
84 | size = offset; |
85 | |
86 | out_set_count: |
87 | *count = size >> pgsize_idx; |
88 | return pgsize; |
89 | } |
90 | |
91 | static int msm_iommu_pagetable_unmap(struct msm_mmu *mmu, u64 iova, |
92 | size_t size) |
93 | { |
94 | struct msm_iommu_pagetable *pagetable = to_pagetable(mmu); |
95 | struct io_pgtable_ops *ops = pagetable->pgtbl_ops; |
96 | |
97 | while (size) { |
98 | size_t unmapped, pgsize, count; |
99 | |
100 | pgsize = calc_pgsize(pagetable, iova, paddr: iova, size, count: &count); |
101 | |
102 | unmapped = ops->unmap_pages(ops, iova, pgsize, count, NULL); |
103 | if (!unmapped) |
104 | break; |
105 | |
106 | iova += unmapped; |
107 | size -= unmapped; |
108 | } |
109 | |
110 | iommu_flush_iotlb_all(to_msm_iommu(pagetable->parent)->domain); |
111 | |
112 | return (size == 0) ? 0 : -EINVAL; |
113 | } |
114 | |
115 | static int msm_iommu_pagetable_map(struct msm_mmu *mmu, u64 iova, |
116 | struct sg_table *sgt, size_t len, int prot) |
117 | { |
118 | struct msm_iommu_pagetable *pagetable = to_pagetable(mmu); |
119 | struct io_pgtable_ops *ops = pagetable->pgtbl_ops; |
120 | struct scatterlist *sg; |
121 | u64 addr = iova; |
122 | unsigned int i; |
123 | |
124 | for_each_sgtable_sg(sgt, sg, i) { |
125 | size_t size = sg->length; |
126 | phys_addr_t phys = sg_phys(sg); |
127 | |
128 | while (size) { |
129 | size_t pgsize, count, mapped = 0; |
130 | int ret; |
131 | |
132 | pgsize = calc_pgsize(pagetable, iova: addr, paddr: phys, size, count: &count); |
133 | |
134 | ret = ops->map_pages(ops, addr, phys, pgsize, count, |
135 | prot, GFP_KERNEL, &mapped); |
136 | |
137 | /* map_pages could fail after mapping some of the pages, |
138 | * so update the counters before error handling. |
139 | */ |
140 | phys += mapped; |
141 | addr += mapped; |
142 | size -= mapped; |
143 | |
144 | if (ret) { |
145 | msm_iommu_pagetable_unmap(mmu, iova, size: addr - iova); |
146 | return -EINVAL; |
147 | } |
148 | } |
149 | } |
150 | |
151 | return 0; |
152 | } |
153 | |
154 | static void msm_iommu_pagetable_destroy(struct msm_mmu *mmu) |
155 | { |
156 | struct msm_iommu_pagetable *pagetable = to_pagetable(mmu); |
157 | struct msm_iommu *iommu = to_msm_iommu(pagetable->parent); |
158 | struct adreno_smmu_priv *adreno_smmu = |
159 | dev_get_drvdata(dev: pagetable->parent->dev); |
160 | |
161 | /* |
162 | * If this is the last attached pagetable for the parent, |
163 | * disable TTBR0 in the arm-smmu driver |
164 | */ |
165 | if (atomic_dec_return(v: &iommu->pagetables) == 0) |
166 | adreno_smmu->set_ttbr0_cfg(adreno_smmu->cookie, NULL); |
167 | |
168 | free_io_pgtable_ops(ops: pagetable->pgtbl_ops); |
169 | kfree(objp: pagetable); |
170 | } |
171 | |
172 | int msm_iommu_pagetable_params(struct msm_mmu *mmu, |
173 | phys_addr_t *ttbr, int *asid) |
174 | { |
175 | struct msm_iommu_pagetable *pagetable; |
176 | |
177 | if (mmu->type != MSM_MMU_IOMMU_PAGETABLE) |
178 | return -EINVAL; |
179 | |
180 | pagetable = to_pagetable(mmu); |
181 | |
182 | if (ttbr) |
183 | *ttbr = pagetable->ttbr; |
184 | |
185 | if (asid) |
186 | *asid = pagetable->asid; |
187 | |
188 | return 0; |
189 | } |
190 | |
191 | struct iommu_domain_geometry *msm_iommu_get_geometry(struct msm_mmu *mmu) |
192 | { |
193 | struct msm_iommu *iommu = to_msm_iommu(mmu); |
194 | |
195 | return &iommu->domain->geometry; |
196 | } |
197 | |
198 | static const struct msm_mmu_funcs pagetable_funcs = { |
199 | .map = msm_iommu_pagetable_map, |
200 | .unmap = msm_iommu_pagetable_unmap, |
201 | .destroy = msm_iommu_pagetable_destroy, |
202 | }; |
203 | |
204 | static void msm_iommu_tlb_flush_all(void *cookie) |
205 | { |
206 | struct msm_iommu_pagetable *pagetable = cookie; |
207 | struct adreno_smmu_priv *adreno_smmu; |
208 | |
209 | if (!pm_runtime_get_if_in_use(dev: pagetable->iommu_dev)) |
210 | return; |
211 | |
212 | adreno_smmu = dev_get_drvdata(dev: pagetable->parent->dev); |
213 | |
214 | pagetable->tlb->tlb_flush_all((void *)adreno_smmu->cookie); |
215 | |
216 | pm_runtime_put_autosuspend(dev: pagetable->iommu_dev); |
217 | } |
218 | |
219 | static void msm_iommu_tlb_flush_walk(unsigned long iova, size_t size, |
220 | size_t granule, void *cookie) |
221 | { |
222 | struct msm_iommu_pagetable *pagetable = cookie; |
223 | struct adreno_smmu_priv *adreno_smmu; |
224 | |
225 | if (!pm_runtime_get_if_in_use(dev: pagetable->iommu_dev)) |
226 | return; |
227 | |
228 | adreno_smmu = dev_get_drvdata(dev: pagetable->parent->dev); |
229 | |
230 | pagetable->tlb->tlb_flush_walk(iova, size, granule, (void *)adreno_smmu->cookie); |
231 | |
232 | pm_runtime_put_autosuspend(dev: pagetable->iommu_dev); |
233 | } |
234 | |
235 | static void msm_iommu_tlb_add_page(struct iommu_iotlb_gather *gather, |
236 | unsigned long iova, size_t granule, void *cookie) |
237 | { |
238 | } |
239 | |
240 | static const struct iommu_flush_ops tlb_ops = { |
241 | .tlb_flush_all = msm_iommu_tlb_flush_all, |
242 | .tlb_flush_walk = msm_iommu_tlb_flush_walk, |
243 | .tlb_add_page = msm_iommu_tlb_add_page, |
244 | }; |
245 | |
246 | static int msm_fault_handler(struct iommu_domain *domain, struct device *dev, |
247 | unsigned long iova, int flags, void *arg); |
248 | |
249 | struct msm_mmu *msm_iommu_pagetable_create(struct msm_mmu *parent) |
250 | { |
251 | struct adreno_smmu_priv *adreno_smmu = dev_get_drvdata(dev: parent->dev); |
252 | struct msm_iommu *iommu = to_msm_iommu(parent); |
253 | struct msm_iommu_pagetable *pagetable; |
254 | const struct io_pgtable_cfg *ttbr1_cfg = NULL; |
255 | struct io_pgtable_cfg ttbr0_cfg; |
256 | int ret; |
257 | |
258 | /* Get the pagetable configuration from the domain */ |
259 | if (adreno_smmu->cookie) |
260 | ttbr1_cfg = adreno_smmu->get_ttbr1_cfg(adreno_smmu->cookie); |
261 | |
262 | /* |
263 | * If you hit this WARN_ONCE() you are probably missing an entry in |
264 | * qcom_smmu_impl_of_match[] in arm-smmu-qcom.c |
265 | */ |
266 | if (WARN_ONCE(!ttbr1_cfg, "No per-process page tables" )) |
267 | return ERR_PTR(error: -ENODEV); |
268 | |
269 | pagetable = kzalloc(size: sizeof(*pagetable), GFP_KERNEL); |
270 | if (!pagetable) |
271 | return ERR_PTR(error: -ENOMEM); |
272 | |
273 | msm_mmu_init(mmu: &pagetable->base, dev: parent->dev, funcs: &pagetable_funcs, |
274 | type: MSM_MMU_IOMMU_PAGETABLE); |
275 | |
276 | /* Clone the TTBR1 cfg as starting point for TTBR0 cfg: */ |
277 | ttbr0_cfg = *ttbr1_cfg; |
278 | |
279 | /* The incoming cfg will have the TTBR1 quirk enabled */ |
280 | ttbr0_cfg.quirks &= ~IO_PGTABLE_QUIRK_ARM_TTBR1; |
281 | ttbr0_cfg.tlb = &tlb_ops; |
282 | |
283 | pagetable->pgtbl_ops = alloc_io_pgtable_ops(fmt: ARM_64_LPAE_S1, |
284 | cfg: &ttbr0_cfg, cookie: pagetable); |
285 | |
286 | if (!pagetable->pgtbl_ops) { |
287 | kfree(objp: pagetable); |
288 | return ERR_PTR(error: -ENOMEM); |
289 | } |
290 | |
291 | /* |
292 | * If this is the first pagetable that we've allocated, send it back to |
293 | * the arm-smmu driver as a trigger to set up TTBR0 |
294 | */ |
295 | if (atomic_inc_return(v: &iommu->pagetables) == 1) { |
296 | ret = adreno_smmu->set_ttbr0_cfg(adreno_smmu->cookie, &ttbr0_cfg); |
297 | if (ret) { |
298 | free_io_pgtable_ops(ops: pagetable->pgtbl_ops); |
299 | kfree(objp: pagetable); |
300 | return ERR_PTR(error: ret); |
301 | } |
302 | } |
303 | |
304 | /* Needed later for TLB flush */ |
305 | pagetable->parent = parent; |
306 | pagetable->tlb = ttbr1_cfg->tlb; |
307 | pagetable->iommu_dev = ttbr1_cfg->iommu_dev; |
308 | pagetable->pgsize_bitmap = ttbr0_cfg.pgsize_bitmap; |
309 | pagetable->ttbr = ttbr0_cfg.arm_lpae_s1_cfg.ttbr; |
310 | |
311 | /* |
312 | * TODO we would like each set of page tables to have a unique ASID |
313 | * to optimize TLB invalidation. But iommu_flush_iotlb_all() will |
314 | * end up flushing the ASID used for TTBR1 pagetables, which is not |
315 | * what we want. So for now just use the same ASID as TTBR1. |
316 | */ |
317 | pagetable->asid = 0; |
318 | |
319 | return &pagetable->base; |
320 | } |
321 | |
322 | static int msm_fault_handler(struct iommu_domain *domain, struct device *dev, |
323 | unsigned long iova, int flags, void *arg) |
324 | { |
325 | struct msm_iommu *iommu = arg; |
326 | struct msm_mmu *mmu = &iommu->base; |
327 | struct adreno_smmu_priv *adreno_smmu = dev_get_drvdata(dev: iommu->base.dev); |
328 | struct adreno_smmu_fault_info info, *ptr = NULL; |
329 | |
330 | if (adreno_smmu->get_fault_info) { |
331 | adreno_smmu->get_fault_info(adreno_smmu->cookie, &info); |
332 | ptr = &info; |
333 | } |
334 | |
335 | if (iommu->base.handler) |
336 | return iommu->base.handler(iommu->base.arg, iova, flags, ptr); |
337 | |
338 | pr_warn_ratelimited("*** fault: iova=%16lx, flags=%d\n" , iova, flags); |
339 | |
340 | if (mmu->funcs->resume_translation) |
341 | mmu->funcs->resume_translation(mmu); |
342 | |
343 | return 0; |
344 | } |
345 | |
346 | static void msm_iommu_resume_translation(struct msm_mmu *mmu) |
347 | { |
348 | struct adreno_smmu_priv *adreno_smmu = dev_get_drvdata(dev: mmu->dev); |
349 | |
350 | if (adreno_smmu->resume_translation) |
351 | adreno_smmu->resume_translation(adreno_smmu->cookie, true); |
352 | } |
353 | |
354 | static void msm_iommu_detach(struct msm_mmu *mmu) |
355 | { |
356 | struct msm_iommu *iommu = to_msm_iommu(mmu); |
357 | |
358 | iommu_detach_device(domain: iommu->domain, dev: mmu->dev); |
359 | } |
360 | |
361 | static int msm_iommu_map(struct msm_mmu *mmu, uint64_t iova, |
362 | struct sg_table *sgt, size_t len, int prot) |
363 | { |
364 | struct msm_iommu *iommu = to_msm_iommu(mmu); |
365 | size_t ret; |
366 | |
367 | /* The arm-smmu driver expects the addresses to be sign extended */ |
368 | if (iova & BIT_ULL(48)) |
369 | iova |= GENMASK_ULL(63, 49); |
370 | |
371 | ret = iommu_map_sgtable(domain: iommu->domain, iova, sgt, prot); |
372 | WARN_ON(!ret); |
373 | |
374 | return (ret == len) ? 0 : -EINVAL; |
375 | } |
376 | |
377 | static int msm_iommu_unmap(struct msm_mmu *mmu, uint64_t iova, size_t len) |
378 | { |
379 | struct msm_iommu *iommu = to_msm_iommu(mmu); |
380 | |
381 | if (iova & BIT_ULL(48)) |
382 | iova |= GENMASK_ULL(63, 49); |
383 | |
384 | iommu_unmap(domain: iommu->domain, iova, size: len); |
385 | |
386 | return 0; |
387 | } |
388 | |
389 | static void msm_iommu_destroy(struct msm_mmu *mmu) |
390 | { |
391 | struct msm_iommu *iommu = to_msm_iommu(mmu); |
392 | iommu_domain_free(domain: iommu->domain); |
393 | kfree(objp: iommu); |
394 | } |
395 | |
396 | static const struct msm_mmu_funcs funcs = { |
397 | .detach = msm_iommu_detach, |
398 | .map = msm_iommu_map, |
399 | .unmap = msm_iommu_unmap, |
400 | .destroy = msm_iommu_destroy, |
401 | .resume_translation = msm_iommu_resume_translation, |
402 | }; |
403 | |
404 | struct msm_mmu *msm_iommu_new(struct device *dev, unsigned long quirks) |
405 | { |
406 | struct iommu_domain *domain; |
407 | struct msm_iommu *iommu; |
408 | int ret; |
409 | |
410 | domain = iommu_domain_alloc(bus: dev->bus); |
411 | if (!domain) |
412 | return NULL; |
413 | |
414 | iommu_set_pgtable_quirks(domain, quirks); |
415 | |
416 | iommu = kzalloc(size: sizeof(*iommu), GFP_KERNEL); |
417 | if (!iommu) { |
418 | iommu_domain_free(domain); |
419 | return ERR_PTR(error: -ENOMEM); |
420 | } |
421 | |
422 | iommu->domain = domain; |
423 | msm_mmu_init(mmu: &iommu->base, dev, funcs: &funcs, type: MSM_MMU_IOMMU); |
424 | |
425 | atomic_set(v: &iommu->pagetables, i: 0); |
426 | |
427 | ret = iommu_attach_device(domain: iommu->domain, dev); |
428 | if (ret) { |
429 | iommu_domain_free(domain); |
430 | kfree(objp: iommu); |
431 | return ERR_PTR(error: ret); |
432 | } |
433 | |
434 | return &iommu->base; |
435 | } |
436 | |
437 | struct msm_mmu *msm_iommu_gpu_new(struct device *dev, struct msm_gpu *gpu, unsigned long quirks) |
438 | { |
439 | struct adreno_smmu_priv *adreno_smmu = dev_get_drvdata(dev); |
440 | struct msm_iommu *iommu; |
441 | struct msm_mmu *mmu; |
442 | |
443 | mmu = msm_iommu_new(dev, quirks); |
444 | if (IS_ERR_OR_NULL(ptr: mmu)) |
445 | return mmu; |
446 | |
447 | iommu = to_msm_iommu(mmu); |
448 | iommu_set_fault_handler(domain: iommu->domain, handler: msm_fault_handler, token: iommu); |
449 | |
450 | /* Enable stall on iommu fault: */ |
451 | if (adreno_smmu->set_stall) |
452 | adreno_smmu->set_stall(adreno_smmu->cookie, true); |
453 | |
454 | return mmu; |
455 | } |
456 | |