1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Unisoc IOMMU driver |
4 | * |
5 | * Copyright (C) 2020 Unisoc, Inc. |
6 | * Author: Chunyan Zhang <chunyan.zhang@unisoc.com> |
7 | */ |
8 | |
9 | #include <linux/clk.h> |
10 | #include <linux/device.h> |
11 | #include <linux/dma-mapping.h> |
12 | #include <linux/errno.h> |
13 | #include <linux/iommu.h> |
14 | #include <linux/mfd/syscon.h> |
15 | #include <linux/module.h> |
16 | #include <linux/of_platform.h> |
17 | #include <linux/platform_device.h> |
18 | #include <linux/regmap.h> |
19 | #include <linux/slab.h> |
20 | |
21 | #define SPRD_IOMMU_PAGE_SHIFT 12 |
22 | #define SPRD_IOMMU_PAGE_SIZE SZ_4K |
23 | |
24 | #define SPRD_EX_CFG 0x0 |
25 | #define SPRD_IOMMU_VAOR_BYPASS BIT(4) |
26 | #define SPRD_IOMMU_GATE_EN BIT(1) |
27 | #define SPRD_IOMMU_EN BIT(0) |
28 | #define SPRD_EX_UPDATE 0x4 |
29 | #define SPRD_EX_FIRST_VPN 0x8 |
30 | #define SPRD_EX_VPN_RANGE 0xc |
31 | #define SPRD_EX_FIRST_PPN 0x10 |
32 | #define SPRD_EX_DEFAULT_PPN 0x14 |
33 | |
34 | #define SPRD_IOMMU_VERSION 0x0 |
35 | #define SPRD_VERSION_MASK GENMASK(15, 8) |
36 | #define SPRD_VERSION_SHIFT 0x8 |
37 | #define SPRD_VAU_CFG 0x4 |
38 | #define SPRD_VAU_UPDATE 0x8 |
39 | #define SPRD_VAU_AUTH_CFG 0xc |
40 | #define SPRD_VAU_FIRST_PPN 0x10 |
41 | #define SPRD_VAU_DEFAULT_PPN_RD 0x14 |
42 | #define SPRD_VAU_DEFAULT_PPN_WR 0x18 |
43 | #define SPRD_VAU_FIRST_VPN 0x1c |
44 | #define SPRD_VAU_VPN_RANGE 0x20 |
45 | |
46 | enum sprd_iommu_version { |
47 | SPRD_IOMMU_EX, |
48 | SPRD_IOMMU_VAU, |
49 | }; |
50 | |
51 | /* |
52 | * struct sprd_iommu_device - high-level sprd IOMMU device representation, |
53 | * including hardware information and configuration, also driver data, etc |
54 | * |
55 | * @ver: sprd IOMMU IP version |
56 | * @prot_page_va: protect page base virtual address |
57 | * @prot_page_pa: protect page base physical address, data would be |
58 | * written to here while translation fault |
59 | * @base: mapped base address for accessing registers |
60 | * @dev: pointer to basic device structure |
61 | * @iommu: IOMMU core representation |
62 | * @group: IOMMU group |
63 | * @eb: gate clock which controls IOMMU access |
64 | */ |
65 | struct sprd_iommu_device { |
66 | struct sprd_iommu_domain *dom; |
67 | enum sprd_iommu_version ver; |
68 | u32 *prot_page_va; |
69 | dma_addr_t prot_page_pa; |
70 | void __iomem *base; |
71 | struct device *dev; |
72 | struct iommu_device iommu; |
73 | struct clk *eb; |
74 | }; |
75 | |
76 | struct sprd_iommu_domain { |
77 | spinlock_t pgtlock; /* lock for page table */ |
78 | struct iommu_domain domain; |
79 | u32 *pgt_va; /* page table virtual address base */ |
80 | dma_addr_t pgt_pa; /* page table physical address base */ |
81 | struct sprd_iommu_device *sdev; |
82 | }; |
83 | |
84 | static const struct iommu_ops sprd_iommu_ops; |
85 | |
86 | static struct sprd_iommu_domain *to_sprd_domain(struct iommu_domain *dom) |
87 | { |
88 | return container_of(dom, struct sprd_iommu_domain, domain); |
89 | } |
90 | |
91 | static inline void |
92 | sprd_iommu_write(struct sprd_iommu_device *sdev, unsigned int reg, u32 val) |
93 | { |
94 | writel_relaxed(val, sdev->base + reg); |
95 | } |
96 | |
97 | static inline u32 |
98 | sprd_iommu_read(struct sprd_iommu_device *sdev, unsigned int reg) |
99 | { |
100 | return readl_relaxed(sdev->base + reg); |
101 | } |
102 | |
103 | static inline void |
104 | sprd_iommu_update_bits(struct sprd_iommu_device *sdev, unsigned int reg, |
105 | u32 mask, u32 shift, u32 val) |
106 | { |
107 | u32 t = sprd_iommu_read(sdev, reg); |
108 | |
109 | t = (t & (~(mask << shift))) | ((val & mask) << shift); |
110 | sprd_iommu_write(sdev, reg, val: t); |
111 | } |
112 | |
113 | static inline int |
114 | sprd_iommu_get_version(struct sprd_iommu_device *sdev) |
115 | { |
116 | int ver = (sprd_iommu_read(sdev, SPRD_IOMMU_VERSION) & |
117 | SPRD_VERSION_MASK) >> SPRD_VERSION_SHIFT; |
118 | |
119 | switch (ver) { |
120 | case SPRD_IOMMU_EX: |
121 | case SPRD_IOMMU_VAU: |
122 | return ver; |
123 | default: |
124 | return -EINVAL; |
125 | } |
126 | } |
127 | |
128 | static size_t |
129 | sprd_iommu_pgt_size(struct iommu_domain *domain) |
130 | { |
131 | return ((domain->geometry.aperture_end - |
132 | domain->geometry.aperture_start + 1) >> |
133 | SPRD_IOMMU_PAGE_SHIFT) * sizeof(u32); |
134 | } |
135 | |
136 | static struct iommu_domain *sprd_iommu_domain_alloc_paging(struct device *dev) |
137 | { |
138 | struct sprd_iommu_domain *dom; |
139 | |
140 | dom = kzalloc(size: sizeof(*dom), GFP_KERNEL); |
141 | if (!dom) |
142 | return NULL; |
143 | |
144 | spin_lock_init(&dom->pgtlock); |
145 | |
146 | dom->domain.geometry.aperture_start = 0; |
147 | dom->domain.geometry.aperture_end = SZ_256M - 1; |
148 | dom->domain.geometry.force_aperture = true; |
149 | |
150 | return &dom->domain; |
151 | } |
152 | |
153 | static void sprd_iommu_first_vpn(struct sprd_iommu_domain *dom) |
154 | { |
155 | struct sprd_iommu_device *sdev = dom->sdev; |
156 | u32 val; |
157 | unsigned int reg; |
158 | |
159 | if (sdev->ver == SPRD_IOMMU_EX) |
160 | reg = SPRD_EX_FIRST_VPN; |
161 | else |
162 | reg = SPRD_VAU_FIRST_VPN; |
163 | |
164 | val = dom->domain.geometry.aperture_start >> SPRD_IOMMU_PAGE_SHIFT; |
165 | sprd_iommu_write(sdev, reg, val); |
166 | } |
167 | |
168 | static void sprd_iommu_vpn_range(struct sprd_iommu_domain *dom) |
169 | { |
170 | struct sprd_iommu_device *sdev = dom->sdev; |
171 | u32 val; |
172 | unsigned int reg; |
173 | |
174 | if (sdev->ver == SPRD_IOMMU_EX) |
175 | reg = SPRD_EX_VPN_RANGE; |
176 | else |
177 | reg = SPRD_VAU_VPN_RANGE; |
178 | |
179 | val = (dom->domain.geometry.aperture_end - |
180 | dom->domain.geometry.aperture_start) >> SPRD_IOMMU_PAGE_SHIFT; |
181 | sprd_iommu_write(sdev, reg, val); |
182 | } |
183 | |
184 | static void sprd_iommu_first_ppn(struct sprd_iommu_domain *dom) |
185 | { |
186 | u32 val = dom->pgt_pa >> SPRD_IOMMU_PAGE_SHIFT; |
187 | struct sprd_iommu_device *sdev = dom->sdev; |
188 | unsigned int reg; |
189 | |
190 | if (sdev->ver == SPRD_IOMMU_EX) |
191 | reg = SPRD_EX_FIRST_PPN; |
192 | else |
193 | reg = SPRD_VAU_FIRST_PPN; |
194 | |
195 | sprd_iommu_write(sdev, reg, val); |
196 | } |
197 | |
198 | static void sprd_iommu_default_ppn(struct sprd_iommu_device *sdev) |
199 | { |
200 | u32 val = sdev->prot_page_pa >> SPRD_IOMMU_PAGE_SHIFT; |
201 | |
202 | if (sdev->ver == SPRD_IOMMU_EX) { |
203 | sprd_iommu_write(sdev, SPRD_EX_DEFAULT_PPN, val); |
204 | } else if (sdev->ver == SPRD_IOMMU_VAU) { |
205 | sprd_iommu_write(sdev, SPRD_VAU_DEFAULT_PPN_RD, val); |
206 | sprd_iommu_write(sdev, SPRD_VAU_DEFAULT_PPN_WR, val); |
207 | } |
208 | } |
209 | |
210 | static void sprd_iommu_hw_en(struct sprd_iommu_device *sdev, bool en) |
211 | { |
212 | unsigned int reg_cfg; |
213 | u32 mask, val; |
214 | |
215 | if (sdev->ver == SPRD_IOMMU_EX) |
216 | reg_cfg = SPRD_EX_CFG; |
217 | else |
218 | reg_cfg = SPRD_VAU_CFG; |
219 | |
220 | mask = SPRD_IOMMU_EN | SPRD_IOMMU_GATE_EN; |
221 | val = en ? mask : 0; |
222 | sprd_iommu_update_bits(sdev, reg: reg_cfg, mask, shift: 0, val); |
223 | } |
224 | |
225 | static void sprd_iommu_cleanup(struct sprd_iommu_domain *dom) |
226 | { |
227 | size_t pgt_size; |
228 | |
229 | /* Nothing need to do if the domain hasn't been attached */ |
230 | if (!dom->sdev) |
231 | return; |
232 | |
233 | pgt_size = sprd_iommu_pgt_size(domain: &dom->domain); |
234 | dma_free_coherent(dev: dom->sdev->dev, size: pgt_size, cpu_addr: dom->pgt_va, dma_handle: dom->pgt_pa); |
235 | dom->sdev = NULL; |
236 | sprd_iommu_hw_en(sdev: dom->sdev, en: false); |
237 | } |
238 | |
239 | static void sprd_iommu_domain_free(struct iommu_domain *domain) |
240 | { |
241 | struct sprd_iommu_domain *dom = to_sprd_domain(dom: domain); |
242 | |
243 | sprd_iommu_cleanup(dom); |
244 | kfree(objp: dom); |
245 | } |
246 | |
247 | static int sprd_iommu_attach_device(struct iommu_domain *domain, |
248 | struct device *dev) |
249 | { |
250 | struct sprd_iommu_device *sdev = dev_iommu_priv_get(dev); |
251 | struct sprd_iommu_domain *dom = to_sprd_domain(dom: domain); |
252 | size_t pgt_size = sprd_iommu_pgt_size(domain); |
253 | |
254 | /* The device is attached to this domain */ |
255 | if (sdev->dom == dom) |
256 | return 0; |
257 | |
258 | /* The first time that domain is attaching to a device */ |
259 | if (!dom->pgt_va) { |
260 | dom->pgt_va = dma_alloc_coherent(dev: sdev->dev, size: pgt_size, dma_handle: &dom->pgt_pa, GFP_KERNEL); |
261 | if (!dom->pgt_va) |
262 | return -ENOMEM; |
263 | |
264 | dom->sdev = sdev; |
265 | } |
266 | |
267 | sdev->dom = dom; |
268 | |
269 | /* |
270 | * One sprd IOMMU serves one client device only, disabled it before |
271 | * configure mapping table to avoid access conflict in case other |
272 | * mapping table is stored in. |
273 | */ |
274 | sprd_iommu_hw_en(sdev, en: false); |
275 | sprd_iommu_first_ppn(dom); |
276 | sprd_iommu_first_vpn(dom); |
277 | sprd_iommu_vpn_range(dom); |
278 | sprd_iommu_default_ppn(sdev); |
279 | sprd_iommu_hw_en(sdev, en: true); |
280 | |
281 | return 0; |
282 | } |
283 | |
284 | static int sprd_iommu_map(struct iommu_domain *domain, unsigned long iova, |
285 | phys_addr_t paddr, size_t pgsize, size_t pgcount, |
286 | int prot, gfp_t gfp, size_t *mapped) |
287 | { |
288 | struct sprd_iommu_domain *dom = to_sprd_domain(dom: domain); |
289 | size_t size = pgcount * SPRD_IOMMU_PAGE_SIZE; |
290 | unsigned long flags; |
291 | unsigned int i; |
292 | u32 *pgt_base_iova; |
293 | u32 pabase = (u32)paddr; |
294 | unsigned long start = domain->geometry.aperture_start; |
295 | unsigned long end = domain->geometry.aperture_end; |
296 | |
297 | if (!dom->sdev) { |
298 | pr_err("No sprd_iommu_device attached to the domain\n" ); |
299 | return -EINVAL; |
300 | } |
301 | |
302 | if (iova < start || (iova + size) > (end + 1)) { |
303 | dev_err(dom->sdev->dev, "(iova(0x%lx) + size(%zx)) are not in the range!\n" , |
304 | iova, size); |
305 | return -EINVAL; |
306 | } |
307 | |
308 | pgt_base_iova = dom->pgt_va + ((iova - start) >> SPRD_IOMMU_PAGE_SHIFT); |
309 | |
310 | spin_lock_irqsave(&dom->pgtlock, flags); |
311 | for (i = 0; i < pgcount; i++) { |
312 | pgt_base_iova[i] = pabase >> SPRD_IOMMU_PAGE_SHIFT; |
313 | pabase += SPRD_IOMMU_PAGE_SIZE; |
314 | } |
315 | spin_unlock_irqrestore(lock: &dom->pgtlock, flags); |
316 | |
317 | *mapped = size; |
318 | return 0; |
319 | } |
320 | |
321 | static size_t sprd_iommu_unmap(struct iommu_domain *domain, unsigned long iova, |
322 | size_t pgsize, size_t pgcount, |
323 | struct iommu_iotlb_gather *iotlb_gather) |
324 | { |
325 | struct sprd_iommu_domain *dom = to_sprd_domain(dom: domain); |
326 | unsigned long flags; |
327 | u32 *pgt_base_iova; |
328 | size_t size = pgcount * SPRD_IOMMU_PAGE_SIZE; |
329 | unsigned long start = domain->geometry.aperture_start; |
330 | unsigned long end = domain->geometry.aperture_end; |
331 | |
332 | if (iova < start || (iova + size) > (end + 1)) |
333 | return 0; |
334 | |
335 | pgt_base_iova = dom->pgt_va + ((iova - start) >> SPRD_IOMMU_PAGE_SHIFT); |
336 | |
337 | spin_lock_irqsave(&dom->pgtlock, flags); |
338 | memset(pgt_base_iova, 0, pgcount * sizeof(u32)); |
339 | spin_unlock_irqrestore(lock: &dom->pgtlock, flags); |
340 | |
341 | return size; |
342 | } |
343 | |
344 | static int sprd_iommu_sync_map(struct iommu_domain *domain, |
345 | unsigned long iova, size_t size) |
346 | { |
347 | struct sprd_iommu_domain *dom = to_sprd_domain(dom: domain); |
348 | unsigned int reg; |
349 | |
350 | if (dom->sdev->ver == SPRD_IOMMU_EX) |
351 | reg = SPRD_EX_UPDATE; |
352 | else |
353 | reg = SPRD_VAU_UPDATE; |
354 | |
355 | /* clear IOMMU TLB buffer after page table updated */ |
356 | sprd_iommu_write(sdev: dom->sdev, reg, val: 0xffffffff); |
357 | return 0; |
358 | } |
359 | |
360 | static void sprd_iommu_sync(struct iommu_domain *domain, |
361 | struct iommu_iotlb_gather *iotlb_gather) |
362 | { |
363 | sprd_iommu_sync_map(domain, iova: 0, size: 0); |
364 | } |
365 | |
366 | static phys_addr_t sprd_iommu_iova_to_phys(struct iommu_domain *domain, |
367 | dma_addr_t iova) |
368 | { |
369 | struct sprd_iommu_domain *dom = to_sprd_domain(dom: domain); |
370 | unsigned long flags; |
371 | phys_addr_t pa; |
372 | unsigned long start = domain->geometry.aperture_start; |
373 | unsigned long end = domain->geometry.aperture_end; |
374 | |
375 | if (WARN_ON(iova < start || iova > end)) |
376 | return 0; |
377 | |
378 | spin_lock_irqsave(&dom->pgtlock, flags); |
379 | pa = *(dom->pgt_va + ((iova - start) >> SPRD_IOMMU_PAGE_SHIFT)); |
380 | pa = (pa << SPRD_IOMMU_PAGE_SHIFT) + ((iova - start) & (SPRD_IOMMU_PAGE_SIZE - 1)); |
381 | spin_unlock_irqrestore(lock: &dom->pgtlock, flags); |
382 | |
383 | return pa; |
384 | } |
385 | |
386 | static struct iommu_device *sprd_iommu_probe_device(struct device *dev) |
387 | { |
388 | struct sprd_iommu_device *sdev = dev_iommu_priv_get(dev); |
389 | |
390 | return &sdev->iommu; |
391 | } |
392 | |
393 | static int sprd_iommu_of_xlate(struct device *dev, |
394 | const struct of_phandle_args *args) |
395 | { |
396 | struct platform_device *pdev; |
397 | |
398 | if (!dev_iommu_priv_get(dev)) { |
399 | pdev = of_find_device_by_node(np: args->np); |
400 | dev_iommu_priv_set(dev, priv: platform_get_drvdata(pdev)); |
401 | platform_device_put(pdev); |
402 | } |
403 | |
404 | return 0; |
405 | } |
406 | |
407 | |
408 | static const struct iommu_ops sprd_iommu_ops = { |
409 | .domain_alloc_paging = sprd_iommu_domain_alloc_paging, |
410 | .probe_device = sprd_iommu_probe_device, |
411 | .device_group = generic_single_device_group, |
412 | .of_xlate = sprd_iommu_of_xlate, |
413 | .pgsize_bitmap = SPRD_IOMMU_PAGE_SIZE, |
414 | .owner = THIS_MODULE, |
415 | .default_domain_ops = &(const struct iommu_domain_ops) { |
416 | .attach_dev = sprd_iommu_attach_device, |
417 | .map_pages = sprd_iommu_map, |
418 | .unmap_pages = sprd_iommu_unmap, |
419 | .iotlb_sync_map = sprd_iommu_sync_map, |
420 | .iotlb_sync = sprd_iommu_sync, |
421 | .iova_to_phys = sprd_iommu_iova_to_phys, |
422 | .free = sprd_iommu_domain_free, |
423 | } |
424 | }; |
425 | |
426 | static const struct of_device_id sprd_iommu_of_match[] = { |
427 | { .compatible = "sprd,iommu-v1" }, |
428 | { }, |
429 | }; |
430 | MODULE_DEVICE_TABLE(of, sprd_iommu_of_match); |
431 | |
432 | /* |
433 | * Clock is not required, access to some of IOMMUs is controlled by gate |
434 | * clk, enabled clocks for that kind of IOMMUs before accessing. |
435 | * Return 0 for success or no clocks found. |
436 | */ |
437 | static int sprd_iommu_clk_enable(struct sprd_iommu_device *sdev) |
438 | { |
439 | struct clk *eb; |
440 | |
441 | eb = devm_clk_get_optional(dev: sdev->dev, NULL); |
442 | if (!eb) |
443 | return 0; |
444 | |
445 | if (IS_ERR(ptr: eb)) |
446 | return PTR_ERR(ptr: eb); |
447 | |
448 | sdev->eb = eb; |
449 | return clk_prepare_enable(clk: eb); |
450 | } |
451 | |
452 | static void sprd_iommu_clk_disable(struct sprd_iommu_device *sdev) |
453 | { |
454 | if (sdev->eb) |
455 | clk_disable_unprepare(clk: sdev->eb); |
456 | } |
457 | |
458 | static int sprd_iommu_probe(struct platform_device *pdev) |
459 | { |
460 | struct sprd_iommu_device *sdev; |
461 | struct device *dev = &pdev->dev; |
462 | void __iomem *base; |
463 | int ret; |
464 | |
465 | sdev = devm_kzalloc(dev, size: sizeof(*sdev), GFP_KERNEL); |
466 | if (!sdev) |
467 | return -ENOMEM; |
468 | |
469 | base = devm_platform_ioremap_resource(pdev, index: 0); |
470 | if (IS_ERR(ptr: base)) { |
471 | dev_err(dev, "Failed to get ioremap resource.\n" ); |
472 | return PTR_ERR(ptr: base); |
473 | } |
474 | sdev->base = base; |
475 | |
476 | sdev->prot_page_va = dma_alloc_coherent(dev, SPRD_IOMMU_PAGE_SIZE, |
477 | dma_handle: &sdev->prot_page_pa, GFP_KERNEL); |
478 | if (!sdev->prot_page_va) |
479 | return -ENOMEM; |
480 | |
481 | platform_set_drvdata(pdev, data: sdev); |
482 | sdev->dev = dev; |
483 | |
484 | ret = iommu_device_sysfs_add(iommu: &sdev->iommu, parent: dev, NULL, fmt: dev_name(dev)); |
485 | if (ret) |
486 | goto free_page; |
487 | |
488 | ret = iommu_device_register(iommu: &sdev->iommu, ops: &sprd_iommu_ops, hwdev: dev); |
489 | if (ret) |
490 | goto remove_sysfs; |
491 | |
492 | ret = sprd_iommu_clk_enable(sdev); |
493 | if (ret) |
494 | goto unregister_iommu; |
495 | |
496 | ret = sprd_iommu_get_version(sdev); |
497 | if (ret < 0) { |
498 | dev_err(dev, "IOMMU version(%d) is invalid.\n" , ret); |
499 | goto disable_clk; |
500 | } |
501 | sdev->ver = ret; |
502 | |
503 | return 0; |
504 | |
505 | disable_clk: |
506 | sprd_iommu_clk_disable(sdev); |
507 | unregister_iommu: |
508 | iommu_device_unregister(iommu: &sdev->iommu); |
509 | remove_sysfs: |
510 | iommu_device_sysfs_remove(iommu: &sdev->iommu); |
511 | free_page: |
512 | dma_free_coherent(dev: sdev->dev, SPRD_IOMMU_PAGE_SIZE, cpu_addr: sdev->prot_page_va, dma_handle: sdev->prot_page_pa); |
513 | return ret; |
514 | } |
515 | |
516 | static void sprd_iommu_remove(struct platform_device *pdev) |
517 | { |
518 | struct sprd_iommu_device *sdev = platform_get_drvdata(pdev); |
519 | |
520 | dma_free_coherent(dev: sdev->dev, SPRD_IOMMU_PAGE_SIZE, cpu_addr: sdev->prot_page_va, dma_handle: sdev->prot_page_pa); |
521 | |
522 | platform_set_drvdata(pdev, NULL); |
523 | iommu_device_sysfs_remove(iommu: &sdev->iommu); |
524 | iommu_device_unregister(iommu: &sdev->iommu); |
525 | } |
526 | |
527 | static struct platform_driver sprd_iommu_driver = { |
528 | .driver = { |
529 | .name = "sprd-iommu" , |
530 | .of_match_table = sprd_iommu_of_match, |
531 | .suppress_bind_attrs = true, |
532 | }, |
533 | .probe = sprd_iommu_probe, |
534 | .remove_new = sprd_iommu_remove, |
535 | }; |
536 | module_platform_driver(sprd_iommu_driver); |
537 | |
538 | MODULE_DESCRIPTION("IOMMU driver for Unisoc SoCs" ); |
539 | MODULE_ALIAS("platform:sprd-iommu" ); |
540 | MODULE_LICENSE("GPL" ); |
541 | |