1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright (C) 2018 Intel Corporation. |
4 | * Copyright 2018 Google LLC. |
5 | * |
6 | * Author: Tuukka Toivonen <tuukka.toivonen@intel.com> |
7 | * Author: Sakari Ailus <sakari.ailus@linux.intel.com> |
8 | * Author: Samu Onkalo <samu.onkalo@intel.com> |
9 | * Author: Tomasz Figa <tfiga@chromium.org> |
10 | * |
11 | */ |
12 | |
13 | #include <linux/dma-mapping.h> |
14 | #include <linux/iopoll.h> |
15 | #include <linux/pm_runtime.h> |
16 | #include <linux/slab.h> |
17 | #include <linux/vmalloc.h> |
18 | |
19 | #include <asm/set_memory.h> |
20 | |
21 | #include "ipu3-mmu.h" |
22 | |
23 | #define IPU3_PT_BITS 10 |
24 | #define IPU3_PT_PTES (1UL << IPU3_PT_BITS) |
25 | #define IPU3_PT_SIZE (IPU3_PT_PTES << 2) |
26 | #define IPU3_PT_ORDER (IPU3_PT_SIZE >> PAGE_SHIFT) |
27 | |
28 | #define IPU3_ADDR2PTE(addr) ((addr) >> IPU3_PAGE_SHIFT) |
29 | #define IPU3_PTE2ADDR(pte) ((phys_addr_t)(pte) << IPU3_PAGE_SHIFT) |
30 | |
31 | #define IPU3_L2PT_SHIFT IPU3_PT_BITS |
32 | #define IPU3_L2PT_MASK ((1UL << IPU3_L2PT_SHIFT) - 1) |
33 | |
34 | #define IPU3_L1PT_SHIFT IPU3_PT_BITS |
35 | #define IPU3_L1PT_MASK ((1UL << IPU3_L1PT_SHIFT) - 1) |
36 | |
37 | #define IPU3_MMU_ADDRESS_BITS (IPU3_PAGE_SHIFT + \ |
38 | IPU3_L2PT_SHIFT + \ |
39 | IPU3_L1PT_SHIFT) |
40 | |
41 | #define IMGU_REG_BASE 0x4000 |
42 | #define REG_TLB_INVALIDATE (IMGU_REG_BASE + 0x300) |
43 | #define TLB_INVALIDATE 1 |
44 | #define REG_L1_PHYS (IMGU_REG_BASE + 0x304) /* 27-bit pfn */ |
45 | #define REG_GP_HALT (IMGU_REG_BASE + 0x5dc) |
46 | #define REG_GP_HALTED (IMGU_REG_BASE + 0x5e0) |
47 | |
48 | struct imgu_mmu { |
49 | struct device *dev; |
50 | void __iomem *base; |
51 | /* protect access to l2pts, l1pt */ |
52 | spinlock_t lock; |
53 | |
54 | void *dummy_page; |
55 | u32 dummy_page_pteval; |
56 | |
57 | u32 *dummy_l2pt; |
58 | u32 dummy_l2pt_pteval; |
59 | |
60 | u32 **l2pts; |
61 | u32 *l1pt; |
62 | |
63 | struct imgu_mmu_info geometry; |
64 | }; |
65 | |
66 | static inline struct imgu_mmu *to_imgu_mmu(struct imgu_mmu_info *info) |
67 | { |
68 | return container_of(info, struct imgu_mmu, geometry); |
69 | } |
70 | |
71 | /** |
72 | * imgu_mmu_tlb_invalidate - invalidate translation look-aside buffer |
73 | * @mmu: MMU to perform the invalidate operation on |
74 | * |
75 | * This function invalidates the whole TLB. Must be called when the hardware |
76 | * is powered on. |
77 | */ |
78 | static void imgu_mmu_tlb_invalidate(struct imgu_mmu *mmu) |
79 | { |
80 | writel(TLB_INVALIDATE, addr: mmu->base + REG_TLB_INVALIDATE); |
81 | } |
82 | |
83 | static void call_if_imgu_is_powered(struct imgu_mmu *mmu, |
84 | void (*func)(struct imgu_mmu *mmu)) |
85 | { |
86 | if (!pm_runtime_get_if_in_use(dev: mmu->dev)) |
87 | return; |
88 | |
89 | func(mmu); |
90 | pm_runtime_put(dev: mmu->dev); |
91 | } |
92 | |
93 | /** |
94 | * imgu_mmu_set_halt - set CIO gate halt bit |
95 | * @mmu: MMU to set the CIO gate bit in. |
96 | * @halt: Desired state of the gate bit. |
97 | * |
98 | * This function sets the CIO gate bit that controls whether external memory |
99 | * accesses are allowed. Must be called when the hardware is powered on. |
100 | */ |
101 | static void imgu_mmu_set_halt(struct imgu_mmu *mmu, bool halt) |
102 | { |
103 | int ret; |
104 | u32 val; |
105 | |
106 | writel(val: halt, addr: mmu->base + REG_GP_HALT); |
107 | ret = readl_poll_timeout(mmu->base + REG_GP_HALTED, |
108 | val, (val & 1) == halt, 1000, 100000); |
109 | |
110 | if (ret) |
111 | dev_err(mmu->dev, "failed to %s CIO gate halt\n" , |
112 | halt ? "set" : "clear" ); |
113 | } |
114 | |
115 | /** |
116 | * imgu_mmu_alloc_page_table - allocate a pre-filled page table |
117 | * @pteval: Value to initialize for page table entries with. |
118 | * |
119 | * Return: Pointer to allocated page table or NULL on failure. |
120 | */ |
121 | static u32 *imgu_mmu_alloc_page_table(u32 pteval) |
122 | { |
123 | u32 *pt; |
124 | int pte; |
125 | |
126 | pt = (u32 *)__get_free_page(GFP_KERNEL); |
127 | if (!pt) |
128 | return NULL; |
129 | |
130 | for (pte = 0; pte < IPU3_PT_PTES; pte++) |
131 | pt[pte] = pteval; |
132 | |
133 | set_memory_uc(addr: (unsigned long)pt, IPU3_PT_ORDER); |
134 | |
135 | return pt; |
136 | } |
137 | |
138 | /** |
139 | * imgu_mmu_free_page_table - free page table |
140 | * @pt: Page table to free. |
141 | */ |
142 | static void imgu_mmu_free_page_table(u32 *pt) |
143 | { |
144 | set_memory_wb(addr: (unsigned long)pt, IPU3_PT_ORDER); |
145 | free_page((unsigned long)pt); |
146 | } |
147 | |
148 | /** |
149 | * address_to_pte_idx - split IOVA into L1 and L2 page table indices |
150 | * @iova: IOVA to split. |
151 | * @l1pt_idx: Output for the L1 page table index. |
152 | * @l2pt_idx: Output for the L2 page index. |
153 | */ |
154 | static inline void address_to_pte_idx(unsigned long iova, u32 *l1pt_idx, |
155 | u32 *l2pt_idx) |
156 | { |
157 | iova >>= IPU3_PAGE_SHIFT; |
158 | |
159 | if (l2pt_idx) |
160 | *l2pt_idx = iova & IPU3_L2PT_MASK; |
161 | |
162 | iova >>= IPU3_L2PT_SHIFT; |
163 | |
164 | if (l1pt_idx) |
165 | *l1pt_idx = iova & IPU3_L1PT_MASK; |
166 | } |
167 | |
168 | static u32 *imgu_mmu_get_l2pt(struct imgu_mmu *mmu, u32 l1pt_idx) |
169 | { |
170 | unsigned long flags; |
171 | u32 *l2pt, *new_l2pt; |
172 | u32 pteval; |
173 | |
174 | spin_lock_irqsave(&mmu->lock, flags); |
175 | |
176 | l2pt = mmu->l2pts[l1pt_idx]; |
177 | if (l2pt) { |
178 | spin_unlock_irqrestore(lock: &mmu->lock, flags); |
179 | return l2pt; |
180 | } |
181 | |
182 | spin_unlock_irqrestore(lock: &mmu->lock, flags); |
183 | |
184 | new_l2pt = imgu_mmu_alloc_page_table(pteval: mmu->dummy_page_pteval); |
185 | if (!new_l2pt) |
186 | return NULL; |
187 | |
188 | spin_lock_irqsave(&mmu->lock, flags); |
189 | |
190 | dev_dbg(mmu->dev, "allocated page table %p for l1pt_idx %u\n" , |
191 | new_l2pt, l1pt_idx); |
192 | |
193 | l2pt = mmu->l2pts[l1pt_idx]; |
194 | if (l2pt) { |
195 | spin_unlock_irqrestore(lock: &mmu->lock, flags); |
196 | imgu_mmu_free_page_table(pt: new_l2pt); |
197 | return l2pt; |
198 | } |
199 | |
200 | l2pt = new_l2pt; |
201 | mmu->l2pts[l1pt_idx] = new_l2pt; |
202 | |
203 | pteval = IPU3_ADDR2PTE(virt_to_phys(new_l2pt)); |
204 | mmu->l1pt[l1pt_idx] = pteval; |
205 | |
206 | spin_unlock_irqrestore(lock: &mmu->lock, flags); |
207 | return l2pt; |
208 | } |
209 | |
210 | static int __imgu_mmu_map(struct imgu_mmu *mmu, unsigned long iova, |
211 | phys_addr_t paddr) |
212 | { |
213 | u32 l1pt_idx, l2pt_idx; |
214 | unsigned long flags; |
215 | u32 *l2pt; |
216 | |
217 | if (!mmu) |
218 | return -ENODEV; |
219 | |
220 | address_to_pte_idx(iova, l1pt_idx: &l1pt_idx, l2pt_idx: &l2pt_idx); |
221 | |
222 | l2pt = imgu_mmu_get_l2pt(mmu, l1pt_idx); |
223 | if (!l2pt) |
224 | return -ENOMEM; |
225 | |
226 | spin_lock_irqsave(&mmu->lock, flags); |
227 | |
228 | if (l2pt[l2pt_idx] != mmu->dummy_page_pteval) { |
229 | spin_unlock_irqrestore(lock: &mmu->lock, flags); |
230 | return -EBUSY; |
231 | } |
232 | |
233 | l2pt[l2pt_idx] = IPU3_ADDR2PTE(paddr); |
234 | |
235 | spin_unlock_irqrestore(lock: &mmu->lock, flags); |
236 | |
237 | return 0; |
238 | } |
239 | |
240 | /** |
241 | * imgu_mmu_map - map a buffer to a physical address |
242 | * |
243 | * @info: MMU mappable range |
244 | * @iova: the virtual address |
245 | * @paddr: the physical address |
246 | * @size: length of the mappable area |
247 | * |
248 | * The function has been adapted from iommu_map() in |
249 | * drivers/iommu/iommu.c . |
250 | */ |
251 | int imgu_mmu_map(struct imgu_mmu_info *info, unsigned long iova, |
252 | phys_addr_t paddr, size_t size) |
253 | { |
254 | struct imgu_mmu *mmu = to_imgu_mmu(info); |
255 | int ret = 0; |
256 | |
257 | /* |
258 | * both the virtual address and the physical one, as well as |
259 | * the size of the mapping, must be aligned (at least) to the |
260 | * size of the smallest page supported by the hardware |
261 | */ |
262 | if (!IS_ALIGNED(iova | paddr | size, IPU3_PAGE_SIZE)) { |
263 | dev_err(mmu->dev, "unaligned: iova 0x%lx pa %pa size 0x%zx\n" , |
264 | iova, &paddr, size); |
265 | return -EINVAL; |
266 | } |
267 | |
268 | dev_dbg(mmu->dev, "map: iova 0x%lx pa %pa size 0x%zx\n" , |
269 | iova, &paddr, size); |
270 | |
271 | while (size) { |
272 | dev_dbg(mmu->dev, "mapping: iova 0x%lx pa %pa\n" , iova, &paddr); |
273 | |
274 | ret = __imgu_mmu_map(mmu, iova, paddr); |
275 | if (ret) |
276 | break; |
277 | |
278 | iova += IPU3_PAGE_SIZE; |
279 | paddr += IPU3_PAGE_SIZE; |
280 | size -= IPU3_PAGE_SIZE; |
281 | } |
282 | |
283 | call_if_imgu_is_powered(mmu, func: imgu_mmu_tlb_invalidate); |
284 | |
285 | return ret; |
286 | } |
287 | |
288 | /** |
289 | * imgu_mmu_map_sg - Map a scatterlist |
290 | * |
291 | * @info: MMU mappable range |
292 | * @iova: the virtual address |
293 | * @sg: the scatterlist to map |
294 | * @nents: number of entries in the scatterlist |
295 | * |
296 | * The function has been adapted from default_iommu_map_sg() in |
297 | * drivers/iommu/iommu.c . |
298 | */ |
299 | size_t imgu_mmu_map_sg(struct imgu_mmu_info *info, unsigned long iova, |
300 | struct scatterlist *sg, unsigned int nents) |
301 | { |
302 | struct imgu_mmu *mmu = to_imgu_mmu(info); |
303 | struct scatterlist *s; |
304 | size_t s_length, mapped = 0; |
305 | unsigned int i; |
306 | int ret; |
307 | |
308 | for_each_sg(sg, s, nents, i) { |
309 | phys_addr_t phys = page_to_phys(sg_page(s)) + s->offset; |
310 | |
311 | s_length = s->length; |
312 | |
313 | if (!IS_ALIGNED(s->offset, IPU3_PAGE_SIZE)) |
314 | goto out_err; |
315 | |
316 | /* must be IPU3_PAGE_SIZE aligned to be mapped singlely */ |
317 | if (i == nents - 1 && !IS_ALIGNED(s->length, IPU3_PAGE_SIZE)) |
318 | s_length = PAGE_ALIGN(s->length); |
319 | |
320 | ret = imgu_mmu_map(info, iova: iova + mapped, paddr: phys, size: s_length); |
321 | if (ret) |
322 | goto out_err; |
323 | |
324 | mapped += s_length; |
325 | } |
326 | |
327 | call_if_imgu_is_powered(mmu, func: imgu_mmu_tlb_invalidate); |
328 | |
329 | return mapped; |
330 | |
331 | out_err: |
332 | /* undo mappings already done */ |
333 | imgu_mmu_unmap(info, iova, size: mapped); |
334 | |
335 | return 0; |
336 | } |
337 | |
338 | static size_t __imgu_mmu_unmap(struct imgu_mmu *mmu, |
339 | unsigned long iova, size_t size) |
340 | { |
341 | u32 l1pt_idx, l2pt_idx; |
342 | unsigned long flags; |
343 | size_t unmap = size; |
344 | u32 *l2pt; |
345 | |
346 | if (!mmu) |
347 | return 0; |
348 | |
349 | address_to_pte_idx(iova, l1pt_idx: &l1pt_idx, l2pt_idx: &l2pt_idx); |
350 | |
351 | spin_lock_irqsave(&mmu->lock, flags); |
352 | |
353 | l2pt = mmu->l2pts[l1pt_idx]; |
354 | if (!l2pt) { |
355 | spin_unlock_irqrestore(lock: &mmu->lock, flags); |
356 | return 0; |
357 | } |
358 | |
359 | if (l2pt[l2pt_idx] == mmu->dummy_page_pteval) |
360 | unmap = 0; |
361 | |
362 | l2pt[l2pt_idx] = mmu->dummy_page_pteval; |
363 | |
364 | spin_unlock_irqrestore(lock: &mmu->lock, flags); |
365 | |
366 | return unmap; |
367 | } |
368 | |
369 | /** |
370 | * imgu_mmu_unmap - Unmap a buffer |
371 | * |
372 | * @info: MMU mappable range |
373 | * @iova: the virtual address |
374 | * @size: the length of the buffer |
375 | * |
376 | * The function has been adapted from iommu_unmap() in |
377 | * drivers/iommu/iommu.c . |
378 | */ |
379 | size_t imgu_mmu_unmap(struct imgu_mmu_info *info, unsigned long iova, |
380 | size_t size) |
381 | { |
382 | struct imgu_mmu *mmu = to_imgu_mmu(info); |
383 | size_t unmapped_page, unmapped = 0; |
384 | |
385 | /* |
386 | * The virtual address, as well as the size of the mapping, must be |
387 | * aligned (at least) to the size of the smallest page supported |
388 | * by the hardware |
389 | */ |
390 | if (!IS_ALIGNED(iova | size, IPU3_PAGE_SIZE)) { |
391 | dev_err(mmu->dev, "unaligned: iova 0x%lx size 0x%zx\n" , |
392 | iova, size); |
393 | return -EINVAL; |
394 | } |
395 | |
396 | dev_dbg(mmu->dev, "unmap this: iova 0x%lx size 0x%zx\n" , iova, size); |
397 | |
398 | /* |
399 | * Keep iterating until we either unmap 'size' bytes (or more) |
400 | * or we hit an area that isn't mapped. |
401 | */ |
402 | while (unmapped < size) { |
403 | unmapped_page = __imgu_mmu_unmap(mmu, iova, IPU3_PAGE_SIZE); |
404 | if (!unmapped_page) |
405 | break; |
406 | |
407 | dev_dbg(mmu->dev, "unmapped: iova 0x%lx size 0x%zx\n" , |
408 | iova, unmapped_page); |
409 | |
410 | iova += unmapped_page; |
411 | unmapped += unmapped_page; |
412 | } |
413 | |
414 | call_if_imgu_is_powered(mmu, func: imgu_mmu_tlb_invalidate); |
415 | |
416 | return unmapped; |
417 | } |
418 | |
419 | /** |
420 | * imgu_mmu_init() - initialize IPU3 MMU block |
421 | * |
422 | * @parent: struct device parent |
423 | * @base: IOMEM base of hardware registers. |
424 | * |
425 | * Return: Pointer to IPU3 MMU private data pointer or ERR_PTR() on error. |
426 | */ |
427 | struct imgu_mmu_info *imgu_mmu_init(struct device *parent, void __iomem *base) |
428 | { |
429 | struct imgu_mmu *mmu; |
430 | u32 pteval; |
431 | |
432 | mmu = kzalloc(size: sizeof(*mmu), GFP_KERNEL); |
433 | if (!mmu) |
434 | return ERR_PTR(error: -ENOMEM); |
435 | |
436 | mmu->dev = parent; |
437 | mmu->base = base; |
438 | spin_lock_init(&mmu->lock); |
439 | |
440 | /* Disallow external memory access when having no valid page tables. */ |
441 | imgu_mmu_set_halt(mmu, halt: true); |
442 | |
443 | /* |
444 | * The MMU does not have a "valid" bit, so we have to use a dummy |
445 | * page for invalid entries. |
446 | */ |
447 | mmu->dummy_page = (void *)__get_free_page(GFP_KERNEL); |
448 | if (!mmu->dummy_page) |
449 | goto fail_group; |
450 | pteval = IPU3_ADDR2PTE(virt_to_phys(mmu->dummy_page)); |
451 | mmu->dummy_page_pteval = pteval; |
452 | |
453 | /* |
454 | * Allocate a dummy L2 page table with all entries pointing to |
455 | * the dummy page. |
456 | */ |
457 | mmu->dummy_l2pt = imgu_mmu_alloc_page_table(pteval); |
458 | if (!mmu->dummy_l2pt) |
459 | goto fail_dummy_page; |
460 | pteval = IPU3_ADDR2PTE(virt_to_phys(mmu->dummy_l2pt)); |
461 | mmu->dummy_l2pt_pteval = pteval; |
462 | |
463 | /* |
464 | * Allocate the array of L2PT CPU pointers, initialized to zero, |
465 | * which means the dummy L2PT allocated above. |
466 | */ |
467 | mmu->l2pts = vzalloc(IPU3_PT_PTES * sizeof(*mmu->l2pts)); |
468 | if (!mmu->l2pts) |
469 | goto fail_l2pt; |
470 | |
471 | /* Allocate the L1 page table. */ |
472 | mmu->l1pt = imgu_mmu_alloc_page_table(pteval: mmu->dummy_l2pt_pteval); |
473 | if (!mmu->l1pt) |
474 | goto fail_l2pts; |
475 | |
476 | pteval = IPU3_ADDR2PTE(virt_to_phys(mmu->l1pt)); |
477 | writel(val: pteval, addr: mmu->base + REG_L1_PHYS); |
478 | imgu_mmu_tlb_invalidate(mmu); |
479 | imgu_mmu_set_halt(mmu, halt: false); |
480 | |
481 | mmu->geometry.aperture_start = 0; |
482 | mmu->geometry.aperture_end = DMA_BIT_MASK(IPU3_MMU_ADDRESS_BITS); |
483 | |
484 | return &mmu->geometry; |
485 | |
486 | fail_l2pts: |
487 | vfree(addr: mmu->l2pts); |
488 | fail_l2pt: |
489 | imgu_mmu_free_page_table(pt: mmu->dummy_l2pt); |
490 | fail_dummy_page: |
491 | free_page((unsigned long)mmu->dummy_page); |
492 | fail_group: |
493 | kfree(objp: mmu); |
494 | |
495 | return ERR_PTR(error: -ENOMEM); |
496 | } |
497 | |
498 | /** |
499 | * imgu_mmu_exit() - clean up IPU3 MMU block |
500 | * |
501 | * @info: MMU mappable range |
502 | */ |
503 | void imgu_mmu_exit(struct imgu_mmu_info *info) |
504 | { |
505 | struct imgu_mmu *mmu = to_imgu_mmu(info); |
506 | |
507 | /* We are going to free our page tables, no more memory access. */ |
508 | imgu_mmu_set_halt(mmu, halt: true); |
509 | imgu_mmu_tlb_invalidate(mmu); |
510 | |
511 | imgu_mmu_free_page_table(pt: mmu->l1pt); |
512 | vfree(addr: mmu->l2pts); |
513 | imgu_mmu_free_page_table(pt: mmu->dummy_l2pt); |
514 | free_page((unsigned long)mmu->dummy_page); |
515 | kfree(objp: mmu); |
516 | } |
517 | |
518 | void imgu_mmu_suspend(struct imgu_mmu_info *info) |
519 | { |
520 | struct imgu_mmu *mmu = to_imgu_mmu(info); |
521 | |
522 | imgu_mmu_set_halt(mmu, halt: true); |
523 | } |
524 | |
525 | void imgu_mmu_resume(struct imgu_mmu_info *info) |
526 | { |
527 | struct imgu_mmu *mmu = to_imgu_mmu(info); |
528 | u32 pteval; |
529 | |
530 | imgu_mmu_set_halt(mmu, halt: true); |
531 | |
532 | pteval = IPU3_ADDR2PTE(virt_to_phys(mmu->l1pt)); |
533 | writel(val: pteval, addr: mmu->base + REG_L1_PHYS); |
534 | |
535 | imgu_mmu_tlb_invalidate(mmu); |
536 | imgu_mmu_set_halt(mmu, halt: false); |
537 | } |
538 | |