1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2013 Red Hat
4 * Author: Rob Clark <robdclark@gmail.com>
5 */
6
7#include <linux/adreno-smmu-priv.h>
8#include <linux/io-pgtable.h>
9#include "msm_drv.h"
10#include "msm_mmu.h"
11
12struct msm_iommu {
13 struct msm_mmu base;
14 struct iommu_domain *domain;
15 atomic_t pagetables;
16};
17
18#define to_msm_iommu(x) container_of(x, struct msm_iommu, base)
19
20struct msm_iommu_pagetable {
21 struct msm_mmu base;
22 struct msm_mmu *parent;
23 struct io_pgtable_ops *pgtbl_ops;
24 const struct iommu_flush_ops *tlb;
25 struct device *iommu_dev;
26 unsigned long pgsize_bitmap; /* Bitmap of page sizes in use */
27 phys_addr_t ttbr;
28 u32 asid;
29};
30static struct msm_iommu_pagetable *to_pagetable(struct msm_mmu *mmu)
31{
32 return container_of(mmu, struct msm_iommu_pagetable, base);
33}
34
35/* based on iommu_pgsize() in iommu.c: */
36static size_t calc_pgsize(struct msm_iommu_pagetable *pagetable,
37 unsigned long iova, phys_addr_t paddr,
38 size_t size, size_t *count)
39{
40 unsigned int pgsize_idx, pgsize_idx_next;
41 unsigned long pgsizes;
42 size_t offset, pgsize, pgsize_next;
43 unsigned long addr_merge = paddr | iova;
44
45 /* Page sizes supported by the hardware and small enough for @size */
46 pgsizes = pagetable->pgsize_bitmap & GENMASK(__fls(size), 0);
47
48 /* Constrain the page sizes further based on the maximum alignment */
49 if (likely(addr_merge))
50 pgsizes &= GENMASK(__ffs(addr_merge), 0);
51
52 /* Make sure we have at least one suitable page size */
53 BUG_ON(!pgsizes);
54
55 /* Pick the biggest page size remaining */
56 pgsize_idx = __fls(word: pgsizes);
57 pgsize = BIT(pgsize_idx);
58 if (!count)
59 return pgsize;
60
61 /* Find the next biggest support page size, if it exists */
62 pgsizes = pagetable->pgsize_bitmap & ~GENMASK(pgsize_idx, 0);
63 if (!pgsizes)
64 goto out_set_count;
65
66 pgsize_idx_next = __ffs(pgsizes);
67 pgsize_next = BIT(pgsize_idx_next);
68
69 /*
70 * There's no point trying a bigger page size unless the virtual
71 * and physical addresses are similarly offset within the larger page.
72 */
73 if ((iova ^ paddr) & (pgsize_next - 1))
74 goto out_set_count;
75
76 /* Calculate the offset to the next page size alignment boundary */
77 offset = pgsize_next - (addr_merge & (pgsize_next - 1));
78
79 /*
80 * If size is big enough to accommodate the larger page, reduce
81 * the number of smaller pages.
82 */
83 if (offset + pgsize_next <= size)
84 size = offset;
85
86out_set_count:
87 *count = size >> pgsize_idx;
88 return pgsize;
89}
90
91static int msm_iommu_pagetable_unmap(struct msm_mmu *mmu, u64 iova,
92 size_t size)
93{
94 struct msm_iommu_pagetable *pagetable = to_pagetable(mmu);
95 struct io_pgtable_ops *ops = pagetable->pgtbl_ops;
96
97 while (size) {
98 size_t unmapped, pgsize, count;
99
100 pgsize = calc_pgsize(pagetable, iova, paddr: iova, size, count: &count);
101
102 unmapped = ops->unmap_pages(ops, iova, pgsize, count, NULL);
103 if (!unmapped)
104 break;
105
106 iova += unmapped;
107 size -= unmapped;
108 }
109
110 iommu_flush_iotlb_all(to_msm_iommu(pagetable->parent)->domain);
111
112 return (size == 0) ? 0 : -EINVAL;
113}
114
115static int msm_iommu_pagetable_map(struct msm_mmu *mmu, u64 iova,
116 struct sg_table *sgt, size_t len, int prot)
117{
118 struct msm_iommu_pagetable *pagetable = to_pagetable(mmu);
119 struct io_pgtable_ops *ops = pagetable->pgtbl_ops;
120 struct scatterlist *sg;
121 u64 addr = iova;
122 unsigned int i;
123
124 for_each_sgtable_sg(sgt, sg, i) {
125 size_t size = sg->length;
126 phys_addr_t phys = sg_phys(sg);
127
128 while (size) {
129 size_t pgsize, count, mapped = 0;
130 int ret;
131
132 pgsize = calc_pgsize(pagetable, iova: addr, paddr: phys, size, count: &count);
133
134 ret = ops->map_pages(ops, addr, phys, pgsize, count,
135 prot, GFP_KERNEL, &mapped);
136
137 /* map_pages could fail after mapping some of the pages,
138 * so update the counters before error handling.
139 */
140 phys += mapped;
141 addr += mapped;
142 size -= mapped;
143
144 if (ret) {
145 msm_iommu_pagetable_unmap(mmu, iova, size: addr - iova);
146 return -EINVAL;
147 }
148 }
149 }
150
151 return 0;
152}
153
154static void msm_iommu_pagetable_destroy(struct msm_mmu *mmu)
155{
156 struct msm_iommu_pagetable *pagetable = to_pagetable(mmu);
157 struct msm_iommu *iommu = to_msm_iommu(pagetable->parent);
158 struct adreno_smmu_priv *adreno_smmu =
159 dev_get_drvdata(dev: pagetable->parent->dev);
160
161 /*
162 * If this is the last attached pagetable for the parent,
163 * disable TTBR0 in the arm-smmu driver
164 */
165 if (atomic_dec_return(v: &iommu->pagetables) == 0)
166 adreno_smmu->set_ttbr0_cfg(adreno_smmu->cookie, NULL);
167
168 free_io_pgtable_ops(ops: pagetable->pgtbl_ops);
169 kfree(objp: pagetable);
170}
171
172int msm_iommu_pagetable_params(struct msm_mmu *mmu,
173 phys_addr_t *ttbr, int *asid)
174{
175 struct msm_iommu_pagetable *pagetable;
176
177 if (mmu->type != MSM_MMU_IOMMU_PAGETABLE)
178 return -EINVAL;
179
180 pagetable = to_pagetable(mmu);
181
182 if (ttbr)
183 *ttbr = pagetable->ttbr;
184
185 if (asid)
186 *asid = pagetable->asid;
187
188 return 0;
189}
190
191struct iommu_domain_geometry *msm_iommu_get_geometry(struct msm_mmu *mmu)
192{
193 struct msm_iommu *iommu = to_msm_iommu(mmu);
194
195 return &iommu->domain->geometry;
196}
197
198static const struct msm_mmu_funcs pagetable_funcs = {
199 .map = msm_iommu_pagetable_map,
200 .unmap = msm_iommu_pagetable_unmap,
201 .destroy = msm_iommu_pagetable_destroy,
202};
203
204static void msm_iommu_tlb_flush_all(void *cookie)
205{
206 struct msm_iommu_pagetable *pagetable = cookie;
207 struct adreno_smmu_priv *adreno_smmu;
208
209 if (!pm_runtime_get_if_in_use(dev: pagetable->iommu_dev))
210 return;
211
212 adreno_smmu = dev_get_drvdata(dev: pagetable->parent->dev);
213
214 pagetable->tlb->tlb_flush_all((void *)adreno_smmu->cookie);
215
216 pm_runtime_put_autosuspend(dev: pagetable->iommu_dev);
217}
218
219static void msm_iommu_tlb_flush_walk(unsigned long iova, size_t size,
220 size_t granule, void *cookie)
221{
222 struct msm_iommu_pagetable *pagetable = cookie;
223 struct adreno_smmu_priv *adreno_smmu;
224
225 if (!pm_runtime_get_if_in_use(dev: pagetable->iommu_dev))
226 return;
227
228 adreno_smmu = dev_get_drvdata(dev: pagetable->parent->dev);
229
230 pagetable->tlb->tlb_flush_walk(iova, size, granule, (void *)adreno_smmu->cookie);
231
232 pm_runtime_put_autosuspend(dev: pagetable->iommu_dev);
233}
234
235static void msm_iommu_tlb_add_page(struct iommu_iotlb_gather *gather,
236 unsigned long iova, size_t granule, void *cookie)
237{
238}
239
240static const struct iommu_flush_ops tlb_ops = {
241 .tlb_flush_all = msm_iommu_tlb_flush_all,
242 .tlb_flush_walk = msm_iommu_tlb_flush_walk,
243 .tlb_add_page = msm_iommu_tlb_add_page,
244};
245
246static int msm_fault_handler(struct iommu_domain *domain, struct device *dev,
247 unsigned long iova, int flags, void *arg);
248
249struct msm_mmu *msm_iommu_pagetable_create(struct msm_mmu *parent)
250{
251 struct adreno_smmu_priv *adreno_smmu = dev_get_drvdata(dev: parent->dev);
252 struct msm_iommu *iommu = to_msm_iommu(parent);
253 struct msm_iommu_pagetable *pagetable;
254 const struct io_pgtable_cfg *ttbr1_cfg = NULL;
255 struct io_pgtable_cfg ttbr0_cfg;
256 int ret;
257
258 /* Get the pagetable configuration from the domain */
259 if (adreno_smmu->cookie)
260 ttbr1_cfg = adreno_smmu->get_ttbr1_cfg(adreno_smmu->cookie);
261
262 /*
263 * If you hit this WARN_ONCE() you are probably missing an entry in
264 * qcom_smmu_impl_of_match[] in arm-smmu-qcom.c
265 */
266 if (WARN_ONCE(!ttbr1_cfg, "No per-process page tables"))
267 return ERR_PTR(error: -ENODEV);
268
269 pagetable = kzalloc(size: sizeof(*pagetable), GFP_KERNEL);
270 if (!pagetable)
271 return ERR_PTR(error: -ENOMEM);
272
273 msm_mmu_init(mmu: &pagetable->base, dev: parent->dev, funcs: &pagetable_funcs,
274 type: MSM_MMU_IOMMU_PAGETABLE);
275
276 /* Clone the TTBR1 cfg as starting point for TTBR0 cfg: */
277 ttbr0_cfg = *ttbr1_cfg;
278
279 /* The incoming cfg will have the TTBR1 quirk enabled */
280 ttbr0_cfg.quirks &= ~IO_PGTABLE_QUIRK_ARM_TTBR1;
281 ttbr0_cfg.tlb = &tlb_ops;
282
283 pagetable->pgtbl_ops = alloc_io_pgtable_ops(fmt: ARM_64_LPAE_S1,
284 cfg: &ttbr0_cfg, cookie: pagetable);
285
286 if (!pagetable->pgtbl_ops) {
287 kfree(objp: pagetable);
288 return ERR_PTR(error: -ENOMEM);
289 }
290
291 /*
292 * If this is the first pagetable that we've allocated, send it back to
293 * the arm-smmu driver as a trigger to set up TTBR0
294 */
295 if (atomic_inc_return(v: &iommu->pagetables) == 1) {
296 ret = adreno_smmu->set_ttbr0_cfg(adreno_smmu->cookie, &ttbr0_cfg);
297 if (ret) {
298 free_io_pgtable_ops(ops: pagetable->pgtbl_ops);
299 kfree(objp: pagetable);
300 return ERR_PTR(error: ret);
301 }
302 }
303
304 /* Needed later for TLB flush */
305 pagetable->parent = parent;
306 pagetable->tlb = ttbr1_cfg->tlb;
307 pagetable->iommu_dev = ttbr1_cfg->iommu_dev;
308 pagetable->pgsize_bitmap = ttbr0_cfg.pgsize_bitmap;
309 pagetable->ttbr = ttbr0_cfg.arm_lpae_s1_cfg.ttbr;
310
311 /*
312 * TODO we would like each set of page tables to have a unique ASID
313 * to optimize TLB invalidation. But iommu_flush_iotlb_all() will
314 * end up flushing the ASID used for TTBR1 pagetables, which is not
315 * what we want. So for now just use the same ASID as TTBR1.
316 */
317 pagetable->asid = 0;
318
319 return &pagetable->base;
320}
321
322static int msm_fault_handler(struct iommu_domain *domain, struct device *dev,
323 unsigned long iova, int flags, void *arg)
324{
325 struct msm_iommu *iommu = arg;
326 struct msm_mmu *mmu = &iommu->base;
327 struct adreno_smmu_priv *adreno_smmu = dev_get_drvdata(dev: iommu->base.dev);
328 struct adreno_smmu_fault_info info, *ptr = NULL;
329
330 if (adreno_smmu->get_fault_info) {
331 adreno_smmu->get_fault_info(adreno_smmu->cookie, &info);
332 ptr = &info;
333 }
334
335 if (iommu->base.handler)
336 return iommu->base.handler(iommu->base.arg, iova, flags, ptr);
337
338 pr_warn_ratelimited("*** fault: iova=%16lx, flags=%d\n", iova, flags);
339
340 if (mmu->funcs->resume_translation)
341 mmu->funcs->resume_translation(mmu);
342
343 return 0;
344}
345
346static void msm_iommu_resume_translation(struct msm_mmu *mmu)
347{
348 struct adreno_smmu_priv *adreno_smmu = dev_get_drvdata(dev: mmu->dev);
349
350 if (adreno_smmu->resume_translation)
351 adreno_smmu->resume_translation(adreno_smmu->cookie, true);
352}
353
354static void msm_iommu_detach(struct msm_mmu *mmu)
355{
356 struct msm_iommu *iommu = to_msm_iommu(mmu);
357
358 iommu_detach_device(domain: iommu->domain, dev: mmu->dev);
359}
360
361static int msm_iommu_map(struct msm_mmu *mmu, uint64_t iova,
362 struct sg_table *sgt, size_t len, int prot)
363{
364 struct msm_iommu *iommu = to_msm_iommu(mmu);
365 size_t ret;
366
367 /* The arm-smmu driver expects the addresses to be sign extended */
368 if (iova & BIT_ULL(48))
369 iova |= GENMASK_ULL(63, 49);
370
371 ret = iommu_map_sgtable(domain: iommu->domain, iova, sgt, prot);
372 WARN_ON(!ret);
373
374 return (ret == len) ? 0 : -EINVAL;
375}
376
377static int msm_iommu_unmap(struct msm_mmu *mmu, uint64_t iova, size_t len)
378{
379 struct msm_iommu *iommu = to_msm_iommu(mmu);
380
381 if (iova & BIT_ULL(48))
382 iova |= GENMASK_ULL(63, 49);
383
384 iommu_unmap(domain: iommu->domain, iova, size: len);
385
386 return 0;
387}
388
389static void msm_iommu_destroy(struct msm_mmu *mmu)
390{
391 struct msm_iommu *iommu = to_msm_iommu(mmu);
392 iommu_domain_free(domain: iommu->domain);
393 kfree(objp: iommu);
394}
395
396static const struct msm_mmu_funcs funcs = {
397 .detach = msm_iommu_detach,
398 .map = msm_iommu_map,
399 .unmap = msm_iommu_unmap,
400 .destroy = msm_iommu_destroy,
401 .resume_translation = msm_iommu_resume_translation,
402};
403
404struct msm_mmu *msm_iommu_new(struct device *dev, unsigned long quirks)
405{
406 struct iommu_domain *domain;
407 struct msm_iommu *iommu;
408 int ret;
409
410 domain = iommu_domain_alloc(bus: dev->bus);
411 if (!domain)
412 return NULL;
413
414 iommu_set_pgtable_quirks(domain, quirks);
415
416 iommu = kzalloc(size: sizeof(*iommu), GFP_KERNEL);
417 if (!iommu) {
418 iommu_domain_free(domain);
419 return ERR_PTR(error: -ENOMEM);
420 }
421
422 iommu->domain = domain;
423 msm_mmu_init(mmu: &iommu->base, dev, funcs: &funcs, type: MSM_MMU_IOMMU);
424
425 atomic_set(v: &iommu->pagetables, i: 0);
426
427 ret = iommu_attach_device(domain: iommu->domain, dev);
428 if (ret) {
429 iommu_domain_free(domain);
430 kfree(objp: iommu);
431 return ERR_PTR(error: ret);
432 }
433
434 return &iommu->base;
435}
436
437struct msm_mmu *msm_iommu_gpu_new(struct device *dev, struct msm_gpu *gpu, unsigned long quirks)
438{
439 struct adreno_smmu_priv *adreno_smmu = dev_get_drvdata(dev);
440 struct msm_iommu *iommu;
441 struct msm_mmu *mmu;
442
443 mmu = msm_iommu_new(dev, quirks);
444 if (IS_ERR_OR_NULL(ptr: mmu))
445 return mmu;
446
447 iommu = to_msm_iommu(mmu);
448 iommu_set_fault_handler(domain: iommu->domain, handler: msm_fault_handler, token: iommu);
449
450 /* Enable stall on iommu fault: */
451 if (adreno_smmu->set_stall)
452 adreno_smmu->set_stall(adreno_smmu->cookie, true);
453
454 return mmu;
455}
456

source code of linux/drivers/gpu/drm/msm/msm_iommu.c