1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * IOMMU API for Rockchip |
4 | * |
5 | * Module Authors: Simon Xue <xxm@rock-chips.com> |
6 | * Daniel Kurtz <djkurtz@chromium.org> |
7 | */ |
8 | |
9 | #include <linux/clk.h> |
10 | #include <linux/compiler.h> |
11 | #include <linux/delay.h> |
12 | #include <linux/device.h> |
13 | #include <linux/dma-mapping.h> |
14 | #include <linux/errno.h> |
15 | #include <linux/interrupt.h> |
16 | #include <linux/io.h> |
17 | #include <linux/iommu.h> |
18 | #include <linux/iopoll.h> |
19 | #include <linux/list.h> |
20 | #include <linux/mm.h> |
21 | #include <linux/init.h> |
22 | #include <linux/of.h> |
23 | #include <linux/of_platform.h> |
24 | #include <linux/platform_device.h> |
25 | #include <linux/pm_runtime.h> |
26 | #include <linux/slab.h> |
27 | #include <linux/spinlock.h> |
28 | |
29 | /** MMU register offsets */ |
30 | #define RK_MMU_DTE_ADDR 0x00 /* Directory table address */ |
31 | #define RK_MMU_STATUS 0x04 |
32 | #define RK_MMU_COMMAND 0x08 |
33 | #define RK_MMU_PAGE_FAULT_ADDR 0x0C /* IOVA of last page fault */ |
34 | #define RK_MMU_ZAP_ONE_LINE 0x10 /* Shootdown one IOTLB entry */ |
35 | #define RK_MMU_INT_RAWSTAT 0x14 /* IRQ status ignoring mask */ |
36 | #define RK_MMU_INT_CLEAR 0x18 /* Acknowledge and re-arm irq */ |
37 | #define RK_MMU_INT_MASK 0x1C /* IRQ enable */ |
38 | #define RK_MMU_INT_STATUS 0x20 /* IRQ status after masking */ |
39 | #define RK_MMU_AUTO_GATING 0x24 |
40 | |
41 | #define DTE_ADDR_DUMMY 0xCAFEBABE |
42 | |
43 | #define RK_MMU_POLL_PERIOD_US 100 |
44 | #define RK_MMU_FORCE_RESET_TIMEOUT_US 100000 |
45 | #define RK_MMU_POLL_TIMEOUT_US 1000 |
46 | |
47 | /* RK_MMU_STATUS fields */ |
48 | #define RK_MMU_STATUS_PAGING_ENABLED BIT(0) |
49 | #define RK_MMU_STATUS_PAGE_FAULT_ACTIVE BIT(1) |
50 | #define RK_MMU_STATUS_STALL_ACTIVE BIT(2) |
51 | #define RK_MMU_STATUS_IDLE BIT(3) |
52 | #define RK_MMU_STATUS_REPLAY_BUFFER_EMPTY BIT(4) |
53 | #define RK_MMU_STATUS_PAGE_FAULT_IS_WRITE BIT(5) |
54 | #define RK_MMU_STATUS_STALL_NOT_ACTIVE BIT(31) |
55 | |
56 | /* RK_MMU_COMMAND command values */ |
57 | #define RK_MMU_CMD_ENABLE_PAGING 0 /* Enable memory translation */ |
58 | #define RK_MMU_CMD_DISABLE_PAGING 1 /* Disable memory translation */ |
59 | #define RK_MMU_CMD_ENABLE_STALL 2 /* Stall paging to allow other cmds */ |
60 | #define RK_MMU_CMD_DISABLE_STALL 3 /* Stop stall re-enables paging */ |
61 | #define RK_MMU_CMD_ZAP_CACHE 4 /* Shoot down entire IOTLB */ |
62 | #define RK_MMU_CMD_PAGE_FAULT_DONE 5 /* Clear page fault */ |
63 | #define RK_MMU_CMD_FORCE_RESET 6 /* Reset all registers */ |
64 | |
65 | /* RK_MMU_INT_* register fields */ |
66 | #define RK_MMU_IRQ_PAGE_FAULT 0x01 /* page fault */ |
67 | #define RK_MMU_IRQ_BUS_ERROR 0x02 /* bus read error */ |
68 | #define RK_MMU_IRQ_MASK (RK_MMU_IRQ_PAGE_FAULT | RK_MMU_IRQ_BUS_ERROR) |
69 | |
70 | #define NUM_DT_ENTRIES 1024 |
71 | #define NUM_PT_ENTRIES 1024 |
72 | |
73 | #define SPAGE_ORDER 12 |
74 | #define SPAGE_SIZE (1 << SPAGE_ORDER) |
75 | |
76 | /* |
77 | * Support mapping any size that fits in one page table: |
78 | * 4 KiB to 4 MiB |
79 | */ |
80 | #define RK_IOMMU_PGSIZE_BITMAP 0x007ff000 |
81 | |
82 | struct rk_iommu_domain { |
83 | struct list_head iommus; |
84 | u32 *dt; /* page directory table */ |
85 | dma_addr_t dt_dma; |
86 | spinlock_t iommus_lock; /* lock for iommus list */ |
87 | spinlock_t dt_lock; /* lock for modifying page directory table */ |
88 | |
89 | struct iommu_domain domain; |
90 | }; |
91 | |
92 | /* list of clocks required by IOMMU */ |
93 | static const char * const rk_iommu_clocks[] = { |
94 | "aclk" , "iface" , |
95 | }; |
96 | |
97 | struct rk_iommu_ops { |
98 | phys_addr_t (*pt_address)(u32 dte); |
99 | u32 (*mk_dtentries)(dma_addr_t pt_dma); |
100 | u32 (*mk_ptentries)(phys_addr_t page, int prot); |
101 | u64 dma_bit_mask; |
102 | gfp_t gfp_flags; |
103 | }; |
104 | |
105 | struct rk_iommu { |
106 | struct device *dev; |
107 | void __iomem **bases; |
108 | int num_mmu; |
109 | int num_irq; |
110 | struct clk_bulk_data *clocks; |
111 | int num_clocks; |
112 | bool reset_disabled; |
113 | struct iommu_device iommu; |
114 | struct list_head node; /* entry in rk_iommu_domain.iommus */ |
115 | struct iommu_domain *domain; /* domain to which iommu is attached */ |
116 | struct iommu_group *group; |
117 | }; |
118 | |
119 | struct rk_iommudata { |
120 | struct device_link *link; /* runtime PM link from IOMMU to master */ |
121 | struct rk_iommu *iommu; |
122 | }; |
123 | |
124 | static struct device *dma_dev; |
125 | static const struct rk_iommu_ops *rk_ops; |
126 | static struct iommu_domain rk_identity_domain; |
127 | |
128 | static inline void rk_table_flush(struct rk_iommu_domain *dom, dma_addr_t dma, |
129 | unsigned int count) |
130 | { |
131 | size_t size = count * sizeof(u32); /* count of u32 entry */ |
132 | |
133 | dma_sync_single_for_device(dev: dma_dev, addr: dma, size, dir: DMA_TO_DEVICE); |
134 | } |
135 | |
136 | static struct rk_iommu_domain *to_rk_domain(struct iommu_domain *dom) |
137 | { |
138 | return container_of(dom, struct rk_iommu_domain, domain); |
139 | } |
140 | |
141 | /* |
142 | * The Rockchip rk3288 iommu uses a 2-level page table. |
143 | * The first level is the "Directory Table" (DT). |
144 | * The DT consists of 1024 4-byte Directory Table Entries (DTEs), each pointing |
145 | * to a "Page Table". |
146 | * The second level is the 1024 Page Tables (PT). |
147 | * Each PT consists of 1024 4-byte Page Table Entries (PTEs), each pointing to |
148 | * a 4 KB page of physical memory. |
149 | * |
150 | * The DT and each PT fits in a single 4 KB page (4-bytes * 1024 entries). |
151 | * Each iommu device has a MMU_DTE_ADDR register that contains the physical |
152 | * address of the start of the DT page. |
153 | * |
154 | * The structure of the page table is as follows: |
155 | * |
156 | * DT |
157 | * MMU_DTE_ADDR -> +-----+ |
158 | * | | |
159 | * +-----+ PT |
160 | * | DTE | -> +-----+ |
161 | * +-----+ | | Memory |
162 | * | | +-----+ Page |
163 | * | | | PTE | -> +-----+ |
164 | * +-----+ +-----+ | | |
165 | * | | | | |
166 | * | | | | |
167 | * +-----+ | | |
168 | * | | |
169 | * | | |
170 | * +-----+ |
171 | */ |
172 | |
173 | /* |
174 | * Each DTE has a PT address and a valid bit: |
175 | * +---------------------+-----------+-+ |
176 | * | PT address | Reserved |V| |
177 | * +---------------------+-----------+-+ |
178 | * 31:12 - PT address (PTs always starts on a 4 KB boundary) |
179 | * 11: 1 - Reserved |
180 | * 0 - 1 if PT @ PT address is valid |
181 | */ |
182 | #define RK_DTE_PT_ADDRESS_MASK 0xfffff000 |
183 | #define RK_DTE_PT_VALID BIT(0) |
184 | |
185 | static inline phys_addr_t rk_dte_pt_address(u32 dte) |
186 | { |
187 | return (phys_addr_t)dte & RK_DTE_PT_ADDRESS_MASK; |
188 | } |
189 | |
190 | /* |
191 | * In v2: |
192 | * 31:12 - PT address bit 31:0 |
193 | * 11: 8 - PT address bit 35:32 |
194 | * 7: 4 - PT address bit 39:36 |
195 | * 3: 1 - Reserved |
196 | * 0 - 1 if PT @ PT address is valid |
197 | */ |
198 | #define RK_DTE_PT_ADDRESS_MASK_V2 GENMASK_ULL(31, 4) |
199 | #define DTE_HI_MASK1 GENMASK(11, 8) |
200 | #define DTE_HI_MASK2 GENMASK(7, 4) |
201 | #define DTE_HI_SHIFT1 24 /* shift bit 8 to bit 32 */ |
202 | #define DTE_HI_SHIFT2 32 /* shift bit 4 to bit 36 */ |
203 | #define PAGE_DESC_HI_MASK1 GENMASK_ULL(35, 32) |
204 | #define PAGE_DESC_HI_MASK2 GENMASK_ULL(39, 36) |
205 | |
206 | static inline phys_addr_t rk_dte_pt_address_v2(u32 dte) |
207 | { |
208 | u64 dte_v2 = dte; |
209 | |
210 | dte_v2 = ((dte_v2 & DTE_HI_MASK2) << DTE_HI_SHIFT2) | |
211 | ((dte_v2 & DTE_HI_MASK1) << DTE_HI_SHIFT1) | |
212 | (dte_v2 & RK_DTE_PT_ADDRESS_MASK); |
213 | |
214 | return (phys_addr_t)dte_v2; |
215 | } |
216 | |
217 | static inline bool rk_dte_is_pt_valid(u32 dte) |
218 | { |
219 | return dte & RK_DTE_PT_VALID; |
220 | } |
221 | |
222 | static inline u32 rk_mk_dte(dma_addr_t pt_dma) |
223 | { |
224 | return (pt_dma & RK_DTE_PT_ADDRESS_MASK) | RK_DTE_PT_VALID; |
225 | } |
226 | |
227 | static inline u32 rk_mk_dte_v2(dma_addr_t pt_dma) |
228 | { |
229 | pt_dma = (pt_dma & RK_DTE_PT_ADDRESS_MASK) | |
230 | ((pt_dma & PAGE_DESC_HI_MASK1) >> DTE_HI_SHIFT1) | |
231 | (pt_dma & PAGE_DESC_HI_MASK2) >> DTE_HI_SHIFT2; |
232 | |
233 | return (pt_dma & RK_DTE_PT_ADDRESS_MASK_V2) | RK_DTE_PT_VALID; |
234 | } |
235 | |
236 | /* |
237 | * Each PTE has a Page address, some flags and a valid bit: |
238 | * +---------------------+---+-------+-+ |
239 | * | Page address |Rsv| Flags |V| |
240 | * +---------------------+---+-------+-+ |
241 | * 31:12 - Page address (Pages always start on a 4 KB boundary) |
242 | * 11: 9 - Reserved |
243 | * 8: 1 - Flags |
244 | * 8 - Read allocate - allocate cache space on read misses |
245 | * 7 - Read cache - enable cache & prefetch of data |
246 | * 6 - Write buffer - enable delaying writes on their way to memory |
247 | * 5 - Write allocate - allocate cache space on write misses |
248 | * 4 - Write cache - different writes can be merged together |
249 | * 3 - Override cache attributes |
250 | * if 1, bits 4-8 control cache attributes |
251 | * if 0, the system bus defaults are used |
252 | * 2 - Writable |
253 | * 1 - Readable |
254 | * 0 - 1 if Page @ Page address is valid |
255 | */ |
256 | #define RK_PTE_PAGE_ADDRESS_MASK 0xfffff000 |
257 | #define RK_PTE_PAGE_FLAGS_MASK 0x000001fe |
258 | #define RK_PTE_PAGE_WRITABLE BIT(2) |
259 | #define RK_PTE_PAGE_READABLE BIT(1) |
260 | #define RK_PTE_PAGE_VALID BIT(0) |
261 | |
262 | static inline bool rk_pte_is_page_valid(u32 pte) |
263 | { |
264 | return pte & RK_PTE_PAGE_VALID; |
265 | } |
266 | |
267 | /* TODO: set cache flags per prot IOMMU_CACHE */ |
268 | static u32 rk_mk_pte(phys_addr_t page, int prot) |
269 | { |
270 | u32 flags = 0; |
271 | flags |= (prot & IOMMU_READ) ? RK_PTE_PAGE_READABLE : 0; |
272 | flags |= (prot & IOMMU_WRITE) ? RK_PTE_PAGE_WRITABLE : 0; |
273 | page &= RK_PTE_PAGE_ADDRESS_MASK; |
274 | return page | flags | RK_PTE_PAGE_VALID; |
275 | } |
276 | |
277 | /* |
278 | * In v2: |
279 | * 31:12 - Page address bit 31:0 |
280 | * 11: 8 - Page address bit 35:32 |
281 | * 7: 4 - Page address bit 39:36 |
282 | * 3 - Security |
283 | * 2 - Writable |
284 | * 1 - Readable |
285 | * 0 - 1 if Page @ Page address is valid |
286 | */ |
287 | |
288 | static u32 rk_mk_pte_v2(phys_addr_t page, int prot) |
289 | { |
290 | u32 flags = 0; |
291 | |
292 | flags |= (prot & IOMMU_READ) ? RK_PTE_PAGE_READABLE : 0; |
293 | flags |= (prot & IOMMU_WRITE) ? RK_PTE_PAGE_WRITABLE : 0; |
294 | |
295 | return rk_mk_dte_v2(pt_dma: page) | flags; |
296 | } |
297 | |
298 | static u32 rk_mk_pte_invalid(u32 pte) |
299 | { |
300 | return pte & ~RK_PTE_PAGE_VALID; |
301 | } |
302 | |
303 | /* |
304 | * rk3288 iova (IOMMU Virtual Address) format |
305 | * 31 22.21 12.11 0 |
306 | * +-----------+-----------+-------------+ |
307 | * | DTE index | PTE index | Page offset | |
308 | * +-----------+-----------+-------------+ |
309 | * 31:22 - DTE index - index of DTE in DT |
310 | * 21:12 - PTE index - index of PTE in PT @ DTE.pt_address |
311 | * 11: 0 - Page offset - offset into page @ PTE.page_address |
312 | */ |
313 | #define RK_IOVA_DTE_MASK 0xffc00000 |
314 | #define RK_IOVA_DTE_SHIFT 22 |
315 | #define RK_IOVA_PTE_MASK 0x003ff000 |
316 | #define RK_IOVA_PTE_SHIFT 12 |
317 | #define RK_IOVA_PAGE_MASK 0x00000fff |
318 | #define RK_IOVA_PAGE_SHIFT 0 |
319 | |
320 | static u32 rk_iova_dte_index(dma_addr_t iova) |
321 | { |
322 | return (u32)(iova & RK_IOVA_DTE_MASK) >> RK_IOVA_DTE_SHIFT; |
323 | } |
324 | |
325 | static u32 rk_iova_pte_index(dma_addr_t iova) |
326 | { |
327 | return (u32)(iova & RK_IOVA_PTE_MASK) >> RK_IOVA_PTE_SHIFT; |
328 | } |
329 | |
330 | static u32 rk_iova_page_offset(dma_addr_t iova) |
331 | { |
332 | return (u32)(iova & RK_IOVA_PAGE_MASK) >> RK_IOVA_PAGE_SHIFT; |
333 | } |
334 | |
335 | static u32 rk_iommu_read(void __iomem *base, u32 offset) |
336 | { |
337 | return readl(addr: base + offset); |
338 | } |
339 | |
340 | static void rk_iommu_write(void __iomem *base, u32 offset, u32 value) |
341 | { |
342 | writel(val: value, addr: base + offset); |
343 | } |
344 | |
345 | static void rk_iommu_command(struct rk_iommu *iommu, u32 command) |
346 | { |
347 | int i; |
348 | |
349 | for (i = 0; i < iommu->num_mmu; i++) |
350 | writel(val: command, addr: iommu->bases[i] + RK_MMU_COMMAND); |
351 | } |
352 | |
353 | static void rk_iommu_base_command(void __iomem *base, u32 command) |
354 | { |
355 | writel(val: command, addr: base + RK_MMU_COMMAND); |
356 | } |
357 | static void rk_iommu_zap_lines(struct rk_iommu *iommu, dma_addr_t iova_start, |
358 | size_t size) |
359 | { |
360 | int i; |
361 | dma_addr_t iova_end = iova_start + size; |
362 | /* |
363 | * TODO(djkurtz): Figure out when it is more efficient to shootdown the |
364 | * entire iotlb rather than iterate over individual iovas. |
365 | */ |
366 | for (i = 0; i < iommu->num_mmu; i++) { |
367 | dma_addr_t iova; |
368 | |
369 | for (iova = iova_start; iova < iova_end; iova += SPAGE_SIZE) |
370 | rk_iommu_write(base: iommu->bases[i], RK_MMU_ZAP_ONE_LINE, value: iova); |
371 | } |
372 | } |
373 | |
374 | static bool rk_iommu_is_stall_active(struct rk_iommu *iommu) |
375 | { |
376 | bool active = true; |
377 | int i; |
378 | |
379 | for (i = 0; i < iommu->num_mmu; i++) |
380 | active &= !!(rk_iommu_read(base: iommu->bases[i], RK_MMU_STATUS) & |
381 | RK_MMU_STATUS_STALL_ACTIVE); |
382 | |
383 | return active; |
384 | } |
385 | |
386 | static bool rk_iommu_is_paging_enabled(struct rk_iommu *iommu) |
387 | { |
388 | bool enable = true; |
389 | int i; |
390 | |
391 | for (i = 0; i < iommu->num_mmu; i++) |
392 | enable &= !!(rk_iommu_read(base: iommu->bases[i], RK_MMU_STATUS) & |
393 | RK_MMU_STATUS_PAGING_ENABLED); |
394 | |
395 | return enable; |
396 | } |
397 | |
398 | static bool rk_iommu_is_reset_done(struct rk_iommu *iommu) |
399 | { |
400 | bool done = true; |
401 | int i; |
402 | |
403 | for (i = 0; i < iommu->num_mmu; i++) |
404 | done &= rk_iommu_read(base: iommu->bases[i], RK_MMU_DTE_ADDR) == 0; |
405 | |
406 | return done; |
407 | } |
408 | |
409 | static int rk_iommu_enable_stall(struct rk_iommu *iommu) |
410 | { |
411 | int ret, i; |
412 | bool val; |
413 | |
414 | if (rk_iommu_is_stall_active(iommu)) |
415 | return 0; |
416 | |
417 | /* Stall can only be enabled if paging is enabled */ |
418 | if (!rk_iommu_is_paging_enabled(iommu)) |
419 | return 0; |
420 | |
421 | rk_iommu_command(iommu, RK_MMU_CMD_ENABLE_STALL); |
422 | |
423 | ret = readx_poll_timeout(rk_iommu_is_stall_active, iommu, val, |
424 | val, RK_MMU_POLL_PERIOD_US, |
425 | RK_MMU_POLL_TIMEOUT_US); |
426 | if (ret) |
427 | for (i = 0; i < iommu->num_mmu; i++) |
428 | dev_err(iommu->dev, "Enable stall request timed out, status: %#08x\n" , |
429 | rk_iommu_read(iommu->bases[i], RK_MMU_STATUS)); |
430 | |
431 | return ret; |
432 | } |
433 | |
434 | static int rk_iommu_disable_stall(struct rk_iommu *iommu) |
435 | { |
436 | int ret, i; |
437 | bool val; |
438 | |
439 | if (!rk_iommu_is_stall_active(iommu)) |
440 | return 0; |
441 | |
442 | rk_iommu_command(iommu, RK_MMU_CMD_DISABLE_STALL); |
443 | |
444 | ret = readx_poll_timeout(rk_iommu_is_stall_active, iommu, val, |
445 | !val, RK_MMU_POLL_PERIOD_US, |
446 | RK_MMU_POLL_TIMEOUT_US); |
447 | if (ret) |
448 | for (i = 0; i < iommu->num_mmu; i++) |
449 | dev_err(iommu->dev, "Disable stall request timed out, status: %#08x\n" , |
450 | rk_iommu_read(iommu->bases[i], RK_MMU_STATUS)); |
451 | |
452 | return ret; |
453 | } |
454 | |
455 | static int rk_iommu_enable_paging(struct rk_iommu *iommu) |
456 | { |
457 | int ret, i; |
458 | bool val; |
459 | |
460 | if (rk_iommu_is_paging_enabled(iommu)) |
461 | return 0; |
462 | |
463 | rk_iommu_command(iommu, RK_MMU_CMD_ENABLE_PAGING); |
464 | |
465 | ret = readx_poll_timeout(rk_iommu_is_paging_enabled, iommu, val, |
466 | val, RK_MMU_POLL_PERIOD_US, |
467 | RK_MMU_POLL_TIMEOUT_US); |
468 | if (ret) |
469 | for (i = 0; i < iommu->num_mmu; i++) |
470 | dev_err(iommu->dev, "Enable paging request timed out, status: %#08x\n" , |
471 | rk_iommu_read(iommu->bases[i], RK_MMU_STATUS)); |
472 | |
473 | return ret; |
474 | } |
475 | |
476 | static int rk_iommu_disable_paging(struct rk_iommu *iommu) |
477 | { |
478 | int ret, i; |
479 | bool val; |
480 | |
481 | if (!rk_iommu_is_paging_enabled(iommu)) |
482 | return 0; |
483 | |
484 | rk_iommu_command(iommu, RK_MMU_CMD_DISABLE_PAGING); |
485 | |
486 | ret = readx_poll_timeout(rk_iommu_is_paging_enabled, iommu, val, |
487 | !val, RK_MMU_POLL_PERIOD_US, |
488 | RK_MMU_POLL_TIMEOUT_US); |
489 | if (ret) |
490 | for (i = 0; i < iommu->num_mmu; i++) |
491 | dev_err(iommu->dev, "Disable paging request timed out, status: %#08x\n" , |
492 | rk_iommu_read(iommu->bases[i], RK_MMU_STATUS)); |
493 | |
494 | return ret; |
495 | } |
496 | |
497 | static int rk_iommu_force_reset(struct rk_iommu *iommu) |
498 | { |
499 | int ret, i; |
500 | u32 dte_addr; |
501 | bool val; |
502 | |
503 | if (iommu->reset_disabled) |
504 | return 0; |
505 | |
506 | /* |
507 | * Check if register DTE_ADDR is working by writing DTE_ADDR_DUMMY |
508 | * and verifying that upper 5 (v1) or 7 (v2) nybbles are read back. |
509 | */ |
510 | for (i = 0; i < iommu->num_mmu; i++) { |
511 | dte_addr = rk_ops->pt_address(DTE_ADDR_DUMMY); |
512 | rk_iommu_write(base: iommu->bases[i], RK_MMU_DTE_ADDR, value: dte_addr); |
513 | |
514 | if (dte_addr != rk_iommu_read(base: iommu->bases[i], RK_MMU_DTE_ADDR)) { |
515 | dev_err(iommu->dev, "Error during raw reset. MMU_DTE_ADDR is not functioning\n" ); |
516 | return -EFAULT; |
517 | } |
518 | } |
519 | |
520 | rk_iommu_command(iommu, RK_MMU_CMD_FORCE_RESET); |
521 | |
522 | ret = readx_poll_timeout(rk_iommu_is_reset_done, iommu, val, |
523 | val, RK_MMU_FORCE_RESET_TIMEOUT_US, |
524 | RK_MMU_POLL_TIMEOUT_US); |
525 | if (ret) { |
526 | dev_err(iommu->dev, "FORCE_RESET command timed out\n" ); |
527 | return ret; |
528 | } |
529 | |
530 | return 0; |
531 | } |
532 | |
533 | static void log_iova(struct rk_iommu *iommu, int index, dma_addr_t iova) |
534 | { |
535 | void __iomem *base = iommu->bases[index]; |
536 | u32 dte_index, pte_index, page_offset; |
537 | u32 mmu_dte_addr; |
538 | phys_addr_t mmu_dte_addr_phys, dte_addr_phys; |
539 | u32 *dte_addr; |
540 | u32 dte; |
541 | phys_addr_t pte_addr_phys = 0; |
542 | u32 *pte_addr = NULL; |
543 | u32 pte = 0; |
544 | phys_addr_t page_addr_phys = 0; |
545 | u32 page_flags = 0; |
546 | |
547 | dte_index = rk_iova_dte_index(iova); |
548 | pte_index = rk_iova_pte_index(iova); |
549 | page_offset = rk_iova_page_offset(iova); |
550 | |
551 | mmu_dte_addr = rk_iommu_read(base, RK_MMU_DTE_ADDR); |
552 | mmu_dte_addr_phys = rk_ops->pt_address(mmu_dte_addr); |
553 | |
554 | dte_addr_phys = mmu_dte_addr_phys + (4 * dte_index); |
555 | dte_addr = phys_to_virt(address: dte_addr_phys); |
556 | dte = *dte_addr; |
557 | |
558 | if (!rk_dte_is_pt_valid(dte)) |
559 | goto print_it; |
560 | |
561 | pte_addr_phys = rk_ops->pt_address(dte) + (pte_index * 4); |
562 | pte_addr = phys_to_virt(address: pte_addr_phys); |
563 | pte = *pte_addr; |
564 | |
565 | if (!rk_pte_is_page_valid(pte)) |
566 | goto print_it; |
567 | |
568 | page_addr_phys = rk_ops->pt_address(pte) + page_offset; |
569 | page_flags = pte & RK_PTE_PAGE_FLAGS_MASK; |
570 | |
571 | print_it: |
572 | dev_err(iommu->dev, "iova = %pad: dte_index: %#03x pte_index: %#03x page_offset: %#03x\n" , |
573 | &iova, dte_index, pte_index, page_offset); |
574 | dev_err(iommu->dev, "mmu_dte_addr: %pa dte@%pa: %#08x valid: %u pte@%pa: %#08x valid: %u page@%pa flags: %#03x\n" , |
575 | &mmu_dte_addr_phys, &dte_addr_phys, dte, |
576 | rk_dte_is_pt_valid(dte), &pte_addr_phys, pte, |
577 | rk_pte_is_page_valid(pte), &page_addr_phys, page_flags); |
578 | } |
579 | |
580 | static irqreturn_t rk_iommu_irq(int irq, void *dev_id) |
581 | { |
582 | struct rk_iommu *iommu = dev_id; |
583 | u32 status; |
584 | u32 int_status; |
585 | dma_addr_t iova; |
586 | irqreturn_t ret = IRQ_NONE; |
587 | int i, err; |
588 | |
589 | err = pm_runtime_get_if_in_use(dev: iommu->dev); |
590 | if (!err || WARN_ON_ONCE(err < 0)) |
591 | return ret; |
592 | |
593 | if (WARN_ON(clk_bulk_enable(iommu->num_clocks, iommu->clocks))) |
594 | goto out; |
595 | |
596 | for (i = 0; i < iommu->num_mmu; i++) { |
597 | int_status = rk_iommu_read(base: iommu->bases[i], RK_MMU_INT_STATUS); |
598 | if (int_status == 0) |
599 | continue; |
600 | |
601 | ret = IRQ_HANDLED; |
602 | iova = rk_iommu_read(base: iommu->bases[i], RK_MMU_PAGE_FAULT_ADDR); |
603 | |
604 | if (int_status & RK_MMU_IRQ_PAGE_FAULT) { |
605 | int flags; |
606 | |
607 | status = rk_iommu_read(base: iommu->bases[i], RK_MMU_STATUS); |
608 | flags = (status & RK_MMU_STATUS_PAGE_FAULT_IS_WRITE) ? |
609 | IOMMU_FAULT_WRITE : IOMMU_FAULT_READ; |
610 | |
611 | dev_err(iommu->dev, "Page fault at %pad of type %s\n" , |
612 | &iova, |
613 | (flags == IOMMU_FAULT_WRITE) ? "write" : "read" ); |
614 | |
615 | log_iova(iommu, index: i, iova); |
616 | |
617 | /* |
618 | * Report page fault to any installed handlers. |
619 | * Ignore the return code, though, since we always zap cache |
620 | * and clear the page fault anyway. |
621 | */ |
622 | if (iommu->domain != &rk_identity_domain) |
623 | report_iommu_fault(domain: iommu->domain, dev: iommu->dev, iova, |
624 | flags); |
625 | else |
626 | dev_err(iommu->dev, "Page fault while iommu not attached to domain?\n" ); |
627 | |
628 | rk_iommu_base_command(base: iommu->bases[i], RK_MMU_CMD_ZAP_CACHE); |
629 | rk_iommu_base_command(base: iommu->bases[i], RK_MMU_CMD_PAGE_FAULT_DONE); |
630 | } |
631 | |
632 | if (int_status & RK_MMU_IRQ_BUS_ERROR) |
633 | dev_err(iommu->dev, "BUS_ERROR occurred at %pad\n" , &iova); |
634 | |
635 | if (int_status & ~RK_MMU_IRQ_MASK) |
636 | dev_err(iommu->dev, "unexpected int_status: %#08x\n" , |
637 | int_status); |
638 | |
639 | rk_iommu_write(base: iommu->bases[i], RK_MMU_INT_CLEAR, value: int_status); |
640 | } |
641 | |
642 | clk_bulk_disable(num_clks: iommu->num_clocks, clks: iommu->clocks); |
643 | |
644 | out: |
645 | pm_runtime_put(dev: iommu->dev); |
646 | return ret; |
647 | } |
648 | |
649 | static phys_addr_t rk_iommu_iova_to_phys(struct iommu_domain *domain, |
650 | dma_addr_t iova) |
651 | { |
652 | struct rk_iommu_domain *rk_domain = to_rk_domain(dom: domain); |
653 | unsigned long flags; |
654 | phys_addr_t pt_phys, phys = 0; |
655 | u32 dte, pte; |
656 | u32 *page_table; |
657 | |
658 | spin_lock_irqsave(&rk_domain->dt_lock, flags); |
659 | |
660 | dte = rk_domain->dt[rk_iova_dte_index(iova)]; |
661 | if (!rk_dte_is_pt_valid(dte)) |
662 | goto out; |
663 | |
664 | pt_phys = rk_ops->pt_address(dte); |
665 | page_table = (u32 *)phys_to_virt(address: pt_phys); |
666 | pte = page_table[rk_iova_pte_index(iova)]; |
667 | if (!rk_pte_is_page_valid(pte)) |
668 | goto out; |
669 | |
670 | phys = rk_ops->pt_address(pte) + rk_iova_page_offset(iova); |
671 | out: |
672 | spin_unlock_irqrestore(lock: &rk_domain->dt_lock, flags); |
673 | |
674 | return phys; |
675 | } |
676 | |
677 | static void rk_iommu_zap_iova(struct rk_iommu_domain *rk_domain, |
678 | dma_addr_t iova, size_t size) |
679 | { |
680 | struct list_head *pos; |
681 | unsigned long flags; |
682 | |
683 | /* shootdown these iova from all iommus using this domain */ |
684 | spin_lock_irqsave(&rk_domain->iommus_lock, flags); |
685 | list_for_each(pos, &rk_domain->iommus) { |
686 | struct rk_iommu *iommu; |
687 | int ret; |
688 | |
689 | iommu = list_entry(pos, struct rk_iommu, node); |
690 | |
691 | /* Only zap TLBs of IOMMUs that are powered on. */ |
692 | ret = pm_runtime_get_if_in_use(dev: iommu->dev); |
693 | if (WARN_ON_ONCE(ret < 0)) |
694 | continue; |
695 | if (ret) { |
696 | WARN_ON(clk_bulk_enable(iommu->num_clocks, |
697 | iommu->clocks)); |
698 | rk_iommu_zap_lines(iommu, iova_start: iova, size); |
699 | clk_bulk_disable(num_clks: iommu->num_clocks, clks: iommu->clocks); |
700 | pm_runtime_put(dev: iommu->dev); |
701 | } |
702 | } |
703 | spin_unlock_irqrestore(lock: &rk_domain->iommus_lock, flags); |
704 | } |
705 | |
706 | static void rk_iommu_zap_iova_first_last(struct rk_iommu_domain *rk_domain, |
707 | dma_addr_t iova, size_t size) |
708 | { |
709 | rk_iommu_zap_iova(rk_domain, iova, SPAGE_SIZE); |
710 | if (size > SPAGE_SIZE) |
711 | rk_iommu_zap_iova(rk_domain, iova: iova + size - SPAGE_SIZE, |
712 | SPAGE_SIZE); |
713 | } |
714 | |
715 | static u32 *rk_dte_get_page_table(struct rk_iommu_domain *rk_domain, |
716 | dma_addr_t iova) |
717 | { |
718 | u32 *page_table, *dte_addr; |
719 | u32 dte_index, dte; |
720 | phys_addr_t pt_phys; |
721 | dma_addr_t pt_dma; |
722 | |
723 | assert_spin_locked(&rk_domain->dt_lock); |
724 | |
725 | dte_index = rk_iova_dte_index(iova); |
726 | dte_addr = &rk_domain->dt[dte_index]; |
727 | dte = *dte_addr; |
728 | if (rk_dte_is_pt_valid(dte)) |
729 | goto done; |
730 | |
731 | page_table = (u32 *)get_zeroed_page(GFP_ATOMIC | rk_ops->gfp_flags); |
732 | if (!page_table) |
733 | return ERR_PTR(error: -ENOMEM); |
734 | |
735 | pt_dma = dma_map_single(dma_dev, page_table, SPAGE_SIZE, DMA_TO_DEVICE); |
736 | if (dma_mapping_error(dev: dma_dev, dma_addr: pt_dma)) { |
737 | dev_err(dma_dev, "DMA mapping error while allocating page table\n" ); |
738 | free_page((unsigned long)page_table); |
739 | return ERR_PTR(error: -ENOMEM); |
740 | } |
741 | |
742 | dte = rk_ops->mk_dtentries(pt_dma); |
743 | *dte_addr = dte; |
744 | |
745 | rk_table_flush(dom: rk_domain, |
746 | dma: rk_domain->dt_dma + dte_index * sizeof(u32), count: 1); |
747 | done: |
748 | pt_phys = rk_ops->pt_address(dte); |
749 | return (u32 *)phys_to_virt(address: pt_phys); |
750 | } |
751 | |
752 | static size_t rk_iommu_unmap_iova(struct rk_iommu_domain *rk_domain, |
753 | u32 *pte_addr, dma_addr_t pte_dma, |
754 | size_t size) |
755 | { |
756 | unsigned int pte_count; |
757 | unsigned int pte_total = size / SPAGE_SIZE; |
758 | |
759 | assert_spin_locked(&rk_domain->dt_lock); |
760 | |
761 | for (pte_count = 0; pte_count < pte_total; pte_count++) { |
762 | u32 pte = pte_addr[pte_count]; |
763 | if (!rk_pte_is_page_valid(pte)) |
764 | break; |
765 | |
766 | pte_addr[pte_count] = rk_mk_pte_invalid(pte); |
767 | } |
768 | |
769 | rk_table_flush(dom: rk_domain, dma: pte_dma, count: pte_count); |
770 | |
771 | return pte_count * SPAGE_SIZE; |
772 | } |
773 | |
774 | static int rk_iommu_map_iova(struct rk_iommu_domain *rk_domain, u32 *pte_addr, |
775 | dma_addr_t pte_dma, dma_addr_t iova, |
776 | phys_addr_t paddr, size_t size, int prot) |
777 | { |
778 | unsigned int pte_count; |
779 | unsigned int pte_total = size / SPAGE_SIZE; |
780 | phys_addr_t page_phys; |
781 | |
782 | assert_spin_locked(&rk_domain->dt_lock); |
783 | |
784 | for (pte_count = 0; pte_count < pte_total; pte_count++) { |
785 | u32 pte = pte_addr[pte_count]; |
786 | |
787 | if (rk_pte_is_page_valid(pte)) |
788 | goto unwind; |
789 | |
790 | pte_addr[pte_count] = rk_ops->mk_ptentries(paddr, prot); |
791 | |
792 | paddr += SPAGE_SIZE; |
793 | } |
794 | |
795 | rk_table_flush(dom: rk_domain, dma: pte_dma, count: pte_total); |
796 | |
797 | /* |
798 | * Zap the first and last iova to evict from iotlb any previously |
799 | * mapped cachelines holding stale values for its dte and pte. |
800 | * We only zap the first and last iova, since only they could have |
801 | * dte or pte shared with an existing mapping. |
802 | */ |
803 | rk_iommu_zap_iova_first_last(rk_domain, iova, size); |
804 | |
805 | return 0; |
806 | unwind: |
807 | /* Unmap the range of iovas that we just mapped */ |
808 | rk_iommu_unmap_iova(rk_domain, pte_addr, pte_dma, |
809 | size: pte_count * SPAGE_SIZE); |
810 | |
811 | iova += pte_count * SPAGE_SIZE; |
812 | page_phys = rk_ops->pt_address(pte_addr[pte_count]); |
813 | pr_err("iova: %pad already mapped to %pa cannot remap to phys: %pa prot: %#x\n" , |
814 | &iova, &page_phys, &paddr, prot); |
815 | |
816 | return -EADDRINUSE; |
817 | } |
818 | |
819 | static int rk_iommu_map(struct iommu_domain *domain, unsigned long _iova, |
820 | phys_addr_t paddr, size_t size, int prot, gfp_t gfp) |
821 | { |
822 | struct rk_iommu_domain *rk_domain = to_rk_domain(dom: domain); |
823 | unsigned long flags; |
824 | dma_addr_t pte_dma, iova = (dma_addr_t)_iova; |
825 | u32 *page_table, *pte_addr; |
826 | u32 dte_index, pte_index; |
827 | int ret; |
828 | |
829 | spin_lock_irqsave(&rk_domain->dt_lock, flags); |
830 | |
831 | /* |
832 | * pgsize_bitmap specifies iova sizes that fit in one page table |
833 | * (1024 4-KiB pages = 4 MiB). |
834 | * So, size will always be 4096 <= size <= 4194304. |
835 | * Since iommu_map() guarantees that both iova and size will be |
836 | * aligned, we will always only be mapping from a single dte here. |
837 | */ |
838 | page_table = rk_dte_get_page_table(rk_domain, iova); |
839 | if (IS_ERR(ptr: page_table)) { |
840 | spin_unlock_irqrestore(lock: &rk_domain->dt_lock, flags); |
841 | return PTR_ERR(ptr: page_table); |
842 | } |
843 | |
844 | dte_index = rk_domain->dt[rk_iova_dte_index(iova)]; |
845 | pte_index = rk_iova_pte_index(iova); |
846 | pte_addr = &page_table[pte_index]; |
847 | |
848 | pte_dma = rk_ops->pt_address(dte_index) + pte_index * sizeof(u32); |
849 | ret = rk_iommu_map_iova(rk_domain, pte_addr, pte_dma, iova, |
850 | paddr, size, prot); |
851 | |
852 | spin_unlock_irqrestore(lock: &rk_domain->dt_lock, flags); |
853 | |
854 | return ret; |
855 | } |
856 | |
857 | static size_t rk_iommu_unmap(struct iommu_domain *domain, unsigned long _iova, |
858 | size_t size, struct iommu_iotlb_gather *gather) |
859 | { |
860 | struct rk_iommu_domain *rk_domain = to_rk_domain(dom: domain); |
861 | unsigned long flags; |
862 | dma_addr_t pte_dma, iova = (dma_addr_t)_iova; |
863 | phys_addr_t pt_phys; |
864 | u32 dte; |
865 | u32 *pte_addr; |
866 | size_t unmap_size; |
867 | |
868 | spin_lock_irqsave(&rk_domain->dt_lock, flags); |
869 | |
870 | /* |
871 | * pgsize_bitmap specifies iova sizes that fit in one page table |
872 | * (1024 4-KiB pages = 4 MiB). |
873 | * So, size will always be 4096 <= size <= 4194304. |
874 | * Since iommu_unmap() guarantees that both iova and size will be |
875 | * aligned, we will always only be unmapping from a single dte here. |
876 | */ |
877 | dte = rk_domain->dt[rk_iova_dte_index(iova)]; |
878 | /* Just return 0 if iova is unmapped */ |
879 | if (!rk_dte_is_pt_valid(dte)) { |
880 | spin_unlock_irqrestore(lock: &rk_domain->dt_lock, flags); |
881 | return 0; |
882 | } |
883 | |
884 | pt_phys = rk_ops->pt_address(dte); |
885 | pte_addr = (u32 *)phys_to_virt(address: pt_phys) + rk_iova_pte_index(iova); |
886 | pte_dma = pt_phys + rk_iova_pte_index(iova) * sizeof(u32); |
887 | unmap_size = rk_iommu_unmap_iova(rk_domain, pte_addr, pte_dma, size); |
888 | |
889 | spin_unlock_irqrestore(lock: &rk_domain->dt_lock, flags); |
890 | |
891 | /* Shootdown iotlb entries for iova range that was just unmapped */ |
892 | rk_iommu_zap_iova(rk_domain, iova, size: unmap_size); |
893 | |
894 | return unmap_size; |
895 | } |
896 | |
897 | static struct rk_iommu *rk_iommu_from_dev(struct device *dev) |
898 | { |
899 | struct rk_iommudata *data = dev_iommu_priv_get(dev); |
900 | |
901 | return data ? data->iommu : NULL; |
902 | } |
903 | |
904 | /* Must be called with iommu powered on and attached */ |
905 | static void rk_iommu_disable(struct rk_iommu *iommu) |
906 | { |
907 | int i; |
908 | |
909 | /* Ignore error while disabling, just keep going */ |
910 | WARN_ON(clk_bulk_enable(iommu->num_clocks, iommu->clocks)); |
911 | rk_iommu_enable_stall(iommu); |
912 | rk_iommu_disable_paging(iommu); |
913 | for (i = 0; i < iommu->num_mmu; i++) { |
914 | rk_iommu_write(base: iommu->bases[i], RK_MMU_INT_MASK, value: 0); |
915 | rk_iommu_write(base: iommu->bases[i], RK_MMU_DTE_ADDR, value: 0); |
916 | } |
917 | rk_iommu_disable_stall(iommu); |
918 | clk_bulk_disable(num_clks: iommu->num_clocks, clks: iommu->clocks); |
919 | } |
920 | |
921 | /* Must be called with iommu powered on and attached */ |
922 | static int rk_iommu_enable(struct rk_iommu *iommu) |
923 | { |
924 | struct iommu_domain *domain = iommu->domain; |
925 | struct rk_iommu_domain *rk_domain = to_rk_domain(dom: domain); |
926 | int ret, i; |
927 | |
928 | ret = clk_bulk_enable(num_clks: iommu->num_clocks, clks: iommu->clocks); |
929 | if (ret) |
930 | return ret; |
931 | |
932 | ret = rk_iommu_enable_stall(iommu); |
933 | if (ret) |
934 | goto out_disable_clocks; |
935 | |
936 | ret = rk_iommu_force_reset(iommu); |
937 | if (ret) |
938 | goto out_disable_stall; |
939 | |
940 | for (i = 0; i < iommu->num_mmu; i++) { |
941 | rk_iommu_write(base: iommu->bases[i], RK_MMU_DTE_ADDR, |
942 | value: rk_ops->mk_dtentries(rk_domain->dt_dma)); |
943 | rk_iommu_base_command(base: iommu->bases[i], RK_MMU_CMD_ZAP_CACHE); |
944 | rk_iommu_write(base: iommu->bases[i], RK_MMU_INT_MASK, RK_MMU_IRQ_MASK); |
945 | } |
946 | |
947 | ret = rk_iommu_enable_paging(iommu); |
948 | |
949 | out_disable_stall: |
950 | rk_iommu_disable_stall(iommu); |
951 | out_disable_clocks: |
952 | clk_bulk_disable(num_clks: iommu->num_clocks, clks: iommu->clocks); |
953 | return ret; |
954 | } |
955 | |
956 | static int rk_iommu_identity_attach(struct iommu_domain *identity_domain, |
957 | struct device *dev) |
958 | { |
959 | struct rk_iommu *iommu; |
960 | struct rk_iommu_domain *rk_domain; |
961 | unsigned long flags; |
962 | int ret; |
963 | |
964 | /* Allow 'virtual devices' (eg drm) to detach from domain */ |
965 | iommu = rk_iommu_from_dev(dev); |
966 | if (!iommu) |
967 | return -ENODEV; |
968 | |
969 | rk_domain = to_rk_domain(dom: iommu->domain); |
970 | |
971 | dev_dbg(dev, "Detaching from iommu domain\n" ); |
972 | |
973 | if (iommu->domain == identity_domain) |
974 | return 0; |
975 | |
976 | iommu->domain = identity_domain; |
977 | |
978 | spin_lock_irqsave(&rk_domain->iommus_lock, flags); |
979 | list_del_init(entry: &iommu->node); |
980 | spin_unlock_irqrestore(lock: &rk_domain->iommus_lock, flags); |
981 | |
982 | ret = pm_runtime_get_if_in_use(dev: iommu->dev); |
983 | WARN_ON_ONCE(ret < 0); |
984 | if (ret > 0) { |
985 | rk_iommu_disable(iommu); |
986 | pm_runtime_put(dev: iommu->dev); |
987 | } |
988 | |
989 | return 0; |
990 | } |
991 | |
992 | static void rk_iommu_identity_free(struct iommu_domain *domain) |
993 | { |
994 | } |
995 | |
996 | static struct iommu_domain_ops rk_identity_ops = { |
997 | .attach_dev = rk_iommu_identity_attach, |
998 | .free = rk_iommu_identity_free, |
999 | }; |
1000 | |
1001 | static struct iommu_domain rk_identity_domain = { |
1002 | .type = IOMMU_DOMAIN_IDENTITY, |
1003 | .ops = &rk_identity_ops, |
1004 | }; |
1005 | |
1006 | #ifdef CONFIG_ARM |
1007 | static void rk_iommu_set_platform_dma(struct device *dev) |
1008 | { |
1009 | WARN_ON(rk_iommu_identity_attach(&rk_identity_domain, dev)); |
1010 | } |
1011 | #endif |
1012 | |
1013 | static int rk_iommu_attach_device(struct iommu_domain *domain, |
1014 | struct device *dev) |
1015 | { |
1016 | struct rk_iommu *iommu; |
1017 | struct rk_iommu_domain *rk_domain = to_rk_domain(dom: domain); |
1018 | unsigned long flags; |
1019 | int ret; |
1020 | |
1021 | /* |
1022 | * Allow 'virtual devices' (e.g., drm) to attach to domain. |
1023 | * Such a device does not belong to an iommu group. |
1024 | */ |
1025 | iommu = rk_iommu_from_dev(dev); |
1026 | if (!iommu) |
1027 | return 0; |
1028 | |
1029 | dev_dbg(dev, "Attaching to iommu domain\n" ); |
1030 | |
1031 | /* iommu already attached */ |
1032 | if (iommu->domain == domain) |
1033 | return 0; |
1034 | |
1035 | ret = rk_iommu_identity_attach(identity_domain: &rk_identity_domain, dev); |
1036 | if (ret) |
1037 | return ret; |
1038 | |
1039 | iommu->domain = domain; |
1040 | |
1041 | spin_lock_irqsave(&rk_domain->iommus_lock, flags); |
1042 | list_add_tail(new: &iommu->node, head: &rk_domain->iommus); |
1043 | spin_unlock_irqrestore(lock: &rk_domain->iommus_lock, flags); |
1044 | |
1045 | ret = pm_runtime_get_if_in_use(dev: iommu->dev); |
1046 | if (!ret || WARN_ON_ONCE(ret < 0)) |
1047 | return 0; |
1048 | |
1049 | ret = rk_iommu_enable(iommu); |
1050 | if (ret) |
1051 | WARN_ON(rk_iommu_identity_attach(&rk_identity_domain, dev)); |
1052 | |
1053 | pm_runtime_put(dev: iommu->dev); |
1054 | |
1055 | return ret; |
1056 | } |
1057 | |
1058 | static struct iommu_domain *rk_iommu_domain_alloc(unsigned type) |
1059 | { |
1060 | struct rk_iommu_domain *rk_domain; |
1061 | |
1062 | if (type == IOMMU_DOMAIN_IDENTITY) |
1063 | return &rk_identity_domain; |
1064 | |
1065 | if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA) |
1066 | return NULL; |
1067 | |
1068 | if (!dma_dev) |
1069 | return NULL; |
1070 | |
1071 | rk_domain = kzalloc(size: sizeof(*rk_domain), GFP_KERNEL); |
1072 | if (!rk_domain) |
1073 | return NULL; |
1074 | |
1075 | /* |
1076 | * rk32xx iommus use a 2 level pagetable. |
1077 | * Each level1 (dt) and level2 (pt) table has 1024 4-byte entries. |
1078 | * Allocate one 4 KiB page for each table. |
1079 | */ |
1080 | rk_domain->dt = (u32 *)get_zeroed_page(GFP_KERNEL | rk_ops->gfp_flags); |
1081 | if (!rk_domain->dt) |
1082 | goto err_free_domain; |
1083 | |
1084 | rk_domain->dt_dma = dma_map_single(dma_dev, rk_domain->dt, |
1085 | SPAGE_SIZE, DMA_TO_DEVICE); |
1086 | if (dma_mapping_error(dev: dma_dev, dma_addr: rk_domain->dt_dma)) { |
1087 | dev_err(dma_dev, "DMA map error for DT\n" ); |
1088 | goto err_free_dt; |
1089 | } |
1090 | |
1091 | spin_lock_init(&rk_domain->iommus_lock); |
1092 | spin_lock_init(&rk_domain->dt_lock); |
1093 | INIT_LIST_HEAD(list: &rk_domain->iommus); |
1094 | |
1095 | rk_domain->domain.geometry.aperture_start = 0; |
1096 | rk_domain->domain.geometry.aperture_end = DMA_BIT_MASK(32); |
1097 | rk_domain->domain.geometry.force_aperture = true; |
1098 | |
1099 | return &rk_domain->domain; |
1100 | |
1101 | err_free_dt: |
1102 | free_page((unsigned long)rk_domain->dt); |
1103 | err_free_domain: |
1104 | kfree(objp: rk_domain); |
1105 | |
1106 | return NULL; |
1107 | } |
1108 | |
1109 | static void rk_iommu_domain_free(struct iommu_domain *domain) |
1110 | { |
1111 | struct rk_iommu_domain *rk_domain = to_rk_domain(dom: domain); |
1112 | int i; |
1113 | |
1114 | WARN_ON(!list_empty(&rk_domain->iommus)); |
1115 | |
1116 | for (i = 0; i < NUM_DT_ENTRIES; i++) { |
1117 | u32 dte = rk_domain->dt[i]; |
1118 | if (rk_dte_is_pt_valid(dte)) { |
1119 | phys_addr_t pt_phys = rk_ops->pt_address(dte); |
1120 | u32 *page_table = phys_to_virt(address: pt_phys); |
1121 | dma_unmap_single(dma_dev, pt_phys, |
1122 | SPAGE_SIZE, DMA_TO_DEVICE); |
1123 | free_page((unsigned long)page_table); |
1124 | } |
1125 | } |
1126 | |
1127 | dma_unmap_single(dma_dev, rk_domain->dt_dma, |
1128 | SPAGE_SIZE, DMA_TO_DEVICE); |
1129 | free_page((unsigned long)rk_domain->dt); |
1130 | |
1131 | kfree(objp: rk_domain); |
1132 | } |
1133 | |
1134 | static struct iommu_device *rk_iommu_probe_device(struct device *dev) |
1135 | { |
1136 | struct rk_iommudata *data; |
1137 | struct rk_iommu *iommu; |
1138 | |
1139 | data = dev_iommu_priv_get(dev); |
1140 | if (!data) |
1141 | return ERR_PTR(error: -ENODEV); |
1142 | |
1143 | iommu = rk_iommu_from_dev(dev); |
1144 | |
1145 | data->link = device_link_add(consumer: dev, supplier: iommu->dev, |
1146 | DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME); |
1147 | |
1148 | return &iommu->iommu; |
1149 | } |
1150 | |
1151 | static void rk_iommu_release_device(struct device *dev) |
1152 | { |
1153 | struct rk_iommudata *data = dev_iommu_priv_get(dev); |
1154 | |
1155 | device_link_del(link: data->link); |
1156 | } |
1157 | |
1158 | static struct iommu_group *rk_iommu_device_group(struct device *dev) |
1159 | { |
1160 | struct rk_iommu *iommu; |
1161 | |
1162 | iommu = rk_iommu_from_dev(dev); |
1163 | |
1164 | return iommu_group_ref_get(group: iommu->group); |
1165 | } |
1166 | |
1167 | static int rk_iommu_of_xlate(struct device *dev, |
1168 | struct of_phandle_args *args) |
1169 | { |
1170 | struct platform_device *iommu_dev; |
1171 | struct rk_iommudata *data; |
1172 | |
1173 | data = devm_kzalloc(dev: dma_dev, size: sizeof(*data), GFP_KERNEL); |
1174 | if (!data) |
1175 | return -ENOMEM; |
1176 | |
1177 | iommu_dev = of_find_device_by_node(np: args->np); |
1178 | |
1179 | data->iommu = platform_get_drvdata(pdev: iommu_dev); |
1180 | data->iommu->domain = &rk_identity_domain; |
1181 | dev_iommu_priv_set(dev, priv: data); |
1182 | |
1183 | platform_device_put(pdev: iommu_dev); |
1184 | |
1185 | return 0; |
1186 | } |
1187 | |
1188 | static const struct iommu_ops rk_iommu_ops = { |
1189 | .domain_alloc = rk_iommu_domain_alloc, |
1190 | .probe_device = rk_iommu_probe_device, |
1191 | .release_device = rk_iommu_release_device, |
1192 | .device_group = rk_iommu_device_group, |
1193 | #ifdef CONFIG_ARM |
1194 | .set_platform_dma_ops = rk_iommu_set_platform_dma, |
1195 | #endif |
1196 | .pgsize_bitmap = RK_IOMMU_PGSIZE_BITMAP, |
1197 | .of_xlate = rk_iommu_of_xlate, |
1198 | .default_domain_ops = &(const struct iommu_domain_ops) { |
1199 | .attach_dev = rk_iommu_attach_device, |
1200 | .map = rk_iommu_map, |
1201 | .unmap = rk_iommu_unmap, |
1202 | .iova_to_phys = rk_iommu_iova_to_phys, |
1203 | .free = rk_iommu_domain_free, |
1204 | } |
1205 | }; |
1206 | |
1207 | static int rk_iommu_probe(struct platform_device *pdev) |
1208 | { |
1209 | struct device *dev = &pdev->dev; |
1210 | struct rk_iommu *iommu; |
1211 | struct resource *res; |
1212 | const struct rk_iommu_ops *ops; |
1213 | int num_res = pdev->num_resources; |
1214 | int err, i; |
1215 | |
1216 | iommu = devm_kzalloc(dev, size: sizeof(*iommu), GFP_KERNEL); |
1217 | if (!iommu) |
1218 | return -ENOMEM; |
1219 | |
1220 | platform_set_drvdata(pdev, data: iommu); |
1221 | iommu->dev = dev; |
1222 | iommu->num_mmu = 0; |
1223 | |
1224 | ops = of_device_get_match_data(dev); |
1225 | if (!rk_ops) |
1226 | rk_ops = ops; |
1227 | |
1228 | /* |
1229 | * That should not happen unless different versions of the |
1230 | * hardware block are embedded the same SoC |
1231 | */ |
1232 | if (WARN_ON(rk_ops != ops)) |
1233 | return -EINVAL; |
1234 | |
1235 | iommu->bases = devm_kcalloc(dev, n: num_res, size: sizeof(*iommu->bases), |
1236 | GFP_KERNEL); |
1237 | if (!iommu->bases) |
1238 | return -ENOMEM; |
1239 | |
1240 | for (i = 0; i < num_res; i++) { |
1241 | res = platform_get_resource(pdev, IORESOURCE_MEM, i); |
1242 | if (!res) |
1243 | continue; |
1244 | iommu->bases[i] = devm_ioremap_resource(dev: &pdev->dev, res); |
1245 | if (IS_ERR(ptr: iommu->bases[i])) |
1246 | continue; |
1247 | iommu->num_mmu++; |
1248 | } |
1249 | if (iommu->num_mmu == 0) |
1250 | return PTR_ERR(ptr: iommu->bases[0]); |
1251 | |
1252 | iommu->num_irq = platform_irq_count(pdev); |
1253 | if (iommu->num_irq < 0) |
1254 | return iommu->num_irq; |
1255 | |
1256 | iommu->reset_disabled = device_property_read_bool(dev, |
1257 | propname: "rockchip,disable-mmu-reset" ); |
1258 | |
1259 | iommu->num_clocks = ARRAY_SIZE(rk_iommu_clocks); |
1260 | iommu->clocks = devm_kcalloc(dev: iommu->dev, n: iommu->num_clocks, |
1261 | size: sizeof(*iommu->clocks), GFP_KERNEL); |
1262 | if (!iommu->clocks) |
1263 | return -ENOMEM; |
1264 | |
1265 | for (i = 0; i < iommu->num_clocks; ++i) |
1266 | iommu->clocks[i].id = rk_iommu_clocks[i]; |
1267 | |
1268 | /* |
1269 | * iommu clocks should be present for all new devices and devicetrees |
1270 | * but there are older devicetrees without clocks out in the wild. |
1271 | * So clocks as optional for the time being. |
1272 | */ |
1273 | err = devm_clk_bulk_get(dev: iommu->dev, num_clks: iommu->num_clocks, clks: iommu->clocks); |
1274 | if (err == -ENOENT) |
1275 | iommu->num_clocks = 0; |
1276 | else if (err) |
1277 | return err; |
1278 | |
1279 | err = clk_bulk_prepare(num_clks: iommu->num_clocks, clks: iommu->clocks); |
1280 | if (err) |
1281 | return err; |
1282 | |
1283 | iommu->group = iommu_group_alloc(); |
1284 | if (IS_ERR(ptr: iommu->group)) { |
1285 | err = PTR_ERR(ptr: iommu->group); |
1286 | goto err_unprepare_clocks; |
1287 | } |
1288 | |
1289 | err = iommu_device_sysfs_add(iommu: &iommu->iommu, parent: dev, NULL, fmt: dev_name(dev)); |
1290 | if (err) |
1291 | goto err_put_group; |
1292 | |
1293 | err = iommu_device_register(iommu: &iommu->iommu, ops: &rk_iommu_ops, hwdev: dev); |
1294 | if (err) |
1295 | goto err_remove_sysfs; |
1296 | |
1297 | /* |
1298 | * Use the first registered IOMMU device for domain to use with DMA |
1299 | * API, since a domain might not physically correspond to a single |
1300 | * IOMMU device.. |
1301 | */ |
1302 | if (!dma_dev) |
1303 | dma_dev = &pdev->dev; |
1304 | |
1305 | pm_runtime_enable(dev); |
1306 | |
1307 | for (i = 0; i < iommu->num_irq; i++) { |
1308 | int irq = platform_get_irq(pdev, i); |
1309 | |
1310 | if (irq < 0) { |
1311 | err = irq; |
1312 | goto err_pm_disable; |
1313 | } |
1314 | |
1315 | err = devm_request_irq(dev: iommu->dev, irq, handler: rk_iommu_irq, |
1316 | IRQF_SHARED, devname: dev_name(dev), dev_id: iommu); |
1317 | if (err) |
1318 | goto err_pm_disable; |
1319 | } |
1320 | |
1321 | dma_set_mask_and_coherent(dev, mask: rk_ops->dma_bit_mask); |
1322 | |
1323 | return 0; |
1324 | err_pm_disable: |
1325 | pm_runtime_disable(dev); |
1326 | err_remove_sysfs: |
1327 | iommu_device_sysfs_remove(iommu: &iommu->iommu); |
1328 | err_put_group: |
1329 | iommu_group_put(group: iommu->group); |
1330 | err_unprepare_clocks: |
1331 | clk_bulk_unprepare(num_clks: iommu->num_clocks, clks: iommu->clocks); |
1332 | return err; |
1333 | } |
1334 | |
1335 | static void rk_iommu_shutdown(struct platform_device *pdev) |
1336 | { |
1337 | struct rk_iommu *iommu = platform_get_drvdata(pdev); |
1338 | int i; |
1339 | |
1340 | for (i = 0; i < iommu->num_irq; i++) { |
1341 | int irq = platform_get_irq(pdev, i); |
1342 | |
1343 | devm_free_irq(dev: iommu->dev, irq, dev_id: iommu); |
1344 | } |
1345 | |
1346 | pm_runtime_force_suspend(dev: &pdev->dev); |
1347 | } |
1348 | |
1349 | static int __maybe_unused rk_iommu_suspend(struct device *dev) |
1350 | { |
1351 | struct rk_iommu *iommu = dev_get_drvdata(dev); |
1352 | |
1353 | if (iommu->domain == &rk_identity_domain) |
1354 | return 0; |
1355 | |
1356 | rk_iommu_disable(iommu); |
1357 | return 0; |
1358 | } |
1359 | |
1360 | static int __maybe_unused rk_iommu_resume(struct device *dev) |
1361 | { |
1362 | struct rk_iommu *iommu = dev_get_drvdata(dev); |
1363 | |
1364 | if (iommu->domain == &rk_identity_domain) |
1365 | return 0; |
1366 | |
1367 | return rk_iommu_enable(iommu); |
1368 | } |
1369 | |
1370 | static const struct dev_pm_ops rk_iommu_pm_ops = { |
1371 | SET_RUNTIME_PM_OPS(rk_iommu_suspend, rk_iommu_resume, NULL) |
1372 | SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, |
1373 | pm_runtime_force_resume) |
1374 | }; |
1375 | |
1376 | static struct rk_iommu_ops iommu_data_ops_v1 = { |
1377 | .pt_address = &rk_dte_pt_address, |
1378 | .mk_dtentries = &rk_mk_dte, |
1379 | .mk_ptentries = &rk_mk_pte, |
1380 | .dma_bit_mask = DMA_BIT_MASK(32), |
1381 | .gfp_flags = GFP_DMA32, |
1382 | }; |
1383 | |
1384 | static struct rk_iommu_ops iommu_data_ops_v2 = { |
1385 | .pt_address = &rk_dte_pt_address_v2, |
1386 | .mk_dtentries = &rk_mk_dte_v2, |
1387 | .mk_ptentries = &rk_mk_pte_v2, |
1388 | .dma_bit_mask = DMA_BIT_MASK(40), |
1389 | .gfp_flags = 0, |
1390 | }; |
1391 | |
1392 | static const struct of_device_id rk_iommu_dt_ids[] = { |
1393 | { .compatible = "rockchip,iommu" , |
1394 | .data = &iommu_data_ops_v1, |
1395 | }, |
1396 | { .compatible = "rockchip,rk3568-iommu" , |
1397 | .data = &iommu_data_ops_v2, |
1398 | }, |
1399 | { /* sentinel */ } |
1400 | }; |
1401 | |
1402 | static struct platform_driver rk_iommu_driver = { |
1403 | .probe = rk_iommu_probe, |
1404 | .shutdown = rk_iommu_shutdown, |
1405 | .driver = { |
1406 | .name = "rk_iommu" , |
1407 | .of_match_table = rk_iommu_dt_ids, |
1408 | .pm = &rk_iommu_pm_ops, |
1409 | .suppress_bind_attrs = true, |
1410 | }, |
1411 | }; |
1412 | builtin_platform_driver(rk_iommu_driver); |
1413 | |