1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright (C) 2016-2018 Etnaviv Project |
4 | */ |
5 | |
6 | #include <linux/bitops.h> |
7 | #include <linux/dma-mapping.h> |
8 | #include <linux/platform_device.h> |
9 | #include <linux/sizes.h> |
10 | #include <linux/slab.h> |
11 | #include <linux/vmalloc.h> |
12 | |
13 | #include "etnaviv_cmdbuf.h" |
14 | #include "etnaviv_gpu.h" |
15 | #include "etnaviv_mmu.h" |
16 | #include "state.xml.h" |
17 | #include "state_hi.xml.h" |
18 | |
19 | #define MMUv2_PTE_PRESENT BIT(0) |
20 | #define MMUv2_PTE_EXCEPTION BIT(1) |
21 | #define MMUv2_PTE_WRITEABLE BIT(2) |
22 | |
23 | #define MMUv2_MTLB_MASK 0xffc00000 |
24 | #define MMUv2_MTLB_SHIFT 22 |
25 | #define MMUv2_STLB_MASK 0x003ff000 |
26 | #define MMUv2_STLB_SHIFT 12 |
27 | |
28 | #define MMUv2_MAX_STLB_ENTRIES 1024 |
29 | |
30 | struct etnaviv_iommuv2_context { |
31 | struct etnaviv_iommu_context base; |
32 | unsigned short id; |
33 | /* M(aster) TLB aka first level pagetable */ |
34 | u32 *mtlb_cpu; |
35 | dma_addr_t mtlb_dma; |
36 | /* S(lave) TLB aka second level pagetable */ |
37 | u32 *stlb_cpu[MMUv2_MAX_STLB_ENTRIES]; |
38 | dma_addr_t stlb_dma[MMUv2_MAX_STLB_ENTRIES]; |
39 | }; |
40 | |
41 | static struct etnaviv_iommuv2_context * |
42 | to_v2_context(struct etnaviv_iommu_context *context) |
43 | { |
44 | return container_of(context, struct etnaviv_iommuv2_context, base); |
45 | } |
46 | |
47 | static void etnaviv_iommuv2_free(struct etnaviv_iommu_context *context) |
48 | { |
49 | struct etnaviv_iommuv2_context *v2_context = to_v2_context(context); |
50 | int i; |
51 | |
52 | drm_mm_takedown(mm: &context->mm); |
53 | |
54 | for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) { |
55 | if (v2_context->stlb_cpu[i]) |
56 | dma_free_wc(dev: context->global->dev, SZ_4K, |
57 | cpu_addr: v2_context->stlb_cpu[i], |
58 | dma_addr: v2_context->stlb_dma[i]); |
59 | } |
60 | |
61 | dma_free_wc(dev: context->global->dev, SZ_4K, cpu_addr: v2_context->mtlb_cpu, |
62 | dma_addr: v2_context->mtlb_dma); |
63 | |
64 | clear_bit(nr: v2_context->id, addr: context->global->v2.pta_alloc); |
65 | |
66 | vfree(addr: v2_context); |
67 | } |
68 | static int |
69 | etnaviv_iommuv2_ensure_stlb(struct etnaviv_iommuv2_context *v2_context, |
70 | int stlb) |
71 | { |
72 | if (v2_context->stlb_cpu[stlb]) |
73 | return 0; |
74 | |
75 | v2_context->stlb_cpu[stlb] = |
76 | dma_alloc_wc(dev: v2_context->base.global->dev, SZ_4K, |
77 | dma_addr: &v2_context->stlb_dma[stlb], |
78 | GFP_KERNEL); |
79 | |
80 | if (!v2_context->stlb_cpu[stlb]) |
81 | return -ENOMEM; |
82 | |
83 | memset32(s: v2_context->stlb_cpu[stlb], MMUv2_PTE_EXCEPTION, |
84 | SZ_4K / sizeof(u32)); |
85 | |
86 | v2_context->mtlb_cpu[stlb] = |
87 | v2_context->stlb_dma[stlb] | MMUv2_PTE_PRESENT; |
88 | |
89 | return 0; |
90 | } |
91 | |
92 | static int etnaviv_iommuv2_map(struct etnaviv_iommu_context *context, |
93 | unsigned long iova, phys_addr_t paddr, |
94 | size_t size, int prot) |
95 | { |
96 | struct etnaviv_iommuv2_context *v2_context = to_v2_context(context); |
97 | int mtlb_entry, stlb_entry, ret; |
98 | u32 entry = lower_32_bits(paddr) | MMUv2_PTE_PRESENT; |
99 | |
100 | if (size != SZ_4K) |
101 | return -EINVAL; |
102 | |
103 | if (IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT)) |
104 | entry |= (upper_32_bits(paddr) & 0xff) << 4; |
105 | |
106 | if (prot & ETNAVIV_PROT_WRITE) |
107 | entry |= MMUv2_PTE_WRITEABLE; |
108 | |
109 | mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT; |
110 | stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT; |
111 | |
112 | ret = etnaviv_iommuv2_ensure_stlb(v2_context, stlb: mtlb_entry); |
113 | if (ret) |
114 | return ret; |
115 | |
116 | v2_context->stlb_cpu[mtlb_entry][stlb_entry] = entry; |
117 | |
118 | return 0; |
119 | } |
120 | |
121 | static size_t etnaviv_iommuv2_unmap(struct etnaviv_iommu_context *context, |
122 | unsigned long iova, size_t size) |
123 | { |
124 | struct etnaviv_iommuv2_context *etnaviv_domain = to_v2_context(context); |
125 | int mtlb_entry, stlb_entry; |
126 | |
127 | if (size != SZ_4K) |
128 | return -EINVAL; |
129 | |
130 | mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT; |
131 | stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT; |
132 | |
133 | etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] = MMUv2_PTE_EXCEPTION; |
134 | |
135 | return SZ_4K; |
136 | } |
137 | |
138 | static size_t etnaviv_iommuv2_dump_size(struct etnaviv_iommu_context *context) |
139 | { |
140 | struct etnaviv_iommuv2_context *v2_context = to_v2_context(context); |
141 | size_t dump_size = SZ_4K; |
142 | int i; |
143 | |
144 | for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) |
145 | if (v2_context->mtlb_cpu[i] & MMUv2_PTE_PRESENT) |
146 | dump_size += SZ_4K; |
147 | |
148 | return dump_size; |
149 | } |
150 | |
151 | static void etnaviv_iommuv2_dump(struct etnaviv_iommu_context *context, void *buf) |
152 | { |
153 | struct etnaviv_iommuv2_context *v2_context = to_v2_context(context); |
154 | int i; |
155 | |
156 | memcpy(buf, v2_context->mtlb_cpu, SZ_4K); |
157 | buf += SZ_4K; |
158 | for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) |
159 | if (v2_context->mtlb_cpu[i] & MMUv2_PTE_PRESENT) { |
160 | memcpy(buf, v2_context->stlb_cpu[i], SZ_4K); |
161 | buf += SZ_4K; |
162 | } |
163 | } |
164 | |
165 | static void etnaviv_iommuv2_restore_nonsec(struct etnaviv_gpu *gpu, |
166 | struct etnaviv_iommu_context *context) |
167 | { |
168 | struct etnaviv_iommuv2_context *v2_context = to_v2_context(context); |
169 | u16 prefetch; |
170 | |
171 | /* If the MMU is already enabled the state is still there. */ |
172 | if (gpu_read(gpu, VIVS_MMUv2_CONTROL) & VIVS_MMUv2_CONTROL_ENABLE) |
173 | return; |
174 | |
175 | if (gpu->mmu_context) |
176 | etnaviv_iommu_context_put(ctx: gpu->mmu_context); |
177 | gpu->mmu_context = etnaviv_iommu_context_get(ctx: context); |
178 | |
179 | prefetch = etnaviv_buffer_config_mmuv2(gpu, |
180 | mtlb_addr: (u32)v2_context->mtlb_dma, |
181 | safe_addr: (u32)context->global->bad_page_dma); |
182 | etnaviv_gpu_start_fe(gpu, address: (u32)etnaviv_cmdbuf_get_pa(buf: &gpu->buffer), |
183 | prefetch); |
184 | etnaviv_gpu_wait_idle(gpu, timeout_ms: 100); |
185 | |
186 | gpu_write(gpu, VIVS_MMUv2_CONTROL, VIVS_MMUv2_CONTROL_ENABLE); |
187 | } |
188 | |
189 | static void etnaviv_iommuv2_restore_sec(struct etnaviv_gpu *gpu, |
190 | struct etnaviv_iommu_context *context) |
191 | { |
192 | struct etnaviv_iommuv2_context *v2_context = to_v2_context(context); |
193 | u16 prefetch; |
194 | |
195 | /* If the MMU is already enabled the state is still there. */ |
196 | if (gpu_read(gpu, VIVS_MMUv2_SEC_CONTROL) & VIVS_MMUv2_SEC_CONTROL_ENABLE) |
197 | return; |
198 | |
199 | if (gpu->mmu_context) |
200 | etnaviv_iommu_context_put(ctx: gpu->mmu_context); |
201 | gpu->mmu_context = etnaviv_iommu_context_get(ctx: context); |
202 | |
203 | gpu_write(gpu, VIVS_MMUv2_PTA_ADDRESS_LOW, |
204 | lower_32_bits(context->global->v2.pta_dma)); |
205 | gpu_write(gpu, VIVS_MMUv2_PTA_ADDRESS_HIGH, |
206 | upper_32_bits(context->global->v2.pta_dma)); |
207 | gpu_write(gpu, VIVS_MMUv2_PTA_CONTROL, VIVS_MMUv2_PTA_CONTROL_ENABLE); |
208 | |
209 | gpu_write(gpu, VIVS_MMUv2_NONSEC_SAFE_ADDR_LOW, |
210 | lower_32_bits(context->global->bad_page_dma)); |
211 | gpu_write(gpu, VIVS_MMUv2_SEC_SAFE_ADDR_LOW, |
212 | lower_32_bits(context->global->bad_page_dma)); |
213 | gpu_write(gpu, VIVS_MMUv2_SAFE_ADDRESS_CONFIG, |
214 | VIVS_MMUv2_SAFE_ADDRESS_CONFIG_NON_SEC_SAFE_ADDR_HIGH( |
215 | upper_32_bits(context->global->bad_page_dma)) | |
216 | VIVS_MMUv2_SAFE_ADDRESS_CONFIG_SEC_SAFE_ADDR_HIGH( |
217 | upper_32_bits(context->global->bad_page_dma))); |
218 | |
219 | context->global->v2.pta_cpu[v2_context->id] = v2_context->mtlb_dma | |
220 | VIVS_MMUv2_CONFIGURATION_MODE_MODE4_K; |
221 | |
222 | /* trigger a PTA load through the FE */ |
223 | prefetch = etnaviv_buffer_config_pta(gpu, id: v2_context->id); |
224 | etnaviv_gpu_start_fe(gpu, address: (u32)etnaviv_cmdbuf_get_pa(buf: &gpu->buffer), |
225 | prefetch); |
226 | etnaviv_gpu_wait_idle(gpu, timeout_ms: 100); |
227 | |
228 | gpu_write(gpu, VIVS_MMUv2_SEC_CONTROL, VIVS_MMUv2_SEC_CONTROL_ENABLE); |
229 | } |
230 | |
231 | u32 etnaviv_iommuv2_get_mtlb_addr(struct etnaviv_iommu_context *context) |
232 | { |
233 | struct etnaviv_iommuv2_context *v2_context = to_v2_context(context); |
234 | |
235 | return v2_context->mtlb_dma; |
236 | } |
237 | |
238 | unsigned short etnaviv_iommuv2_get_pta_id(struct etnaviv_iommu_context *context) |
239 | { |
240 | struct etnaviv_iommuv2_context *v2_context = to_v2_context(context); |
241 | |
242 | return v2_context->id; |
243 | } |
244 | static void etnaviv_iommuv2_restore(struct etnaviv_gpu *gpu, |
245 | struct etnaviv_iommu_context *context) |
246 | { |
247 | switch (gpu->sec_mode) { |
248 | case ETNA_SEC_NONE: |
249 | etnaviv_iommuv2_restore_nonsec(gpu, context); |
250 | break; |
251 | case ETNA_SEC_KERNEL: |
252 | etnaviv_iommuv2_restore_sec(gpu, context); |
253 | break; |
254 | default: |
255 | WARN(1, "unhandled GPU security mode\n" ); |
256 | break; |
257 | } |
258 | } |
259 | |
260 | const struct etnaviv_iommu_ops etnaviv_iommuv2_ops = { |
261 | .free = etnaviv_iommuv2_free, |
262 | .map = etnaviv_iommuv2_map, |
263 | .unmap = etnaviv_iommuv2_unmap, |
264 | .dump_size = etnaviv_iommuv2_dump_size, |
265 | .dump = etnaviv_iommuv2_dump, |
266 | .restore = etnaviv_iommuv2_restore, |
267 | }; |
268 | |
269 | struct etnaviv_iommu_context * |
270 | etnaviv_iommuv2_context_alloc(struct etnaviv_iommu_global *global) |
271 | { |
272 | struct etnaviv_iommuv2_context *v2_context; |
273 | struct etnaviv_iommu_context *context; |
274 | |
275 | v2_context = vzalloc(size: sizeof(*v2_context)); |
276 | if (!v2_context) |
277 | return NULL; |
278 | |
279 | mutex_lock(&global->lock); |
280 | v2_context->id = find_first_zero_bit(addr: global->v2.pta_alloc, |
281 | ETNAVIV_PTA_ENTRIES); |
282 | if (v2_context->id < ETNAVIV_PTA_ENTRIES) { |
283 | set_bit(nr: v2_context->id, addr: global->v2.pta_alloc); |
284 | } else { |
285 | mutex_unlock(lock: &global->lock); |
286 | goto out_free; |
287 | } |
288 | mutex_unlock(lock: &global->lock); |
289 | |
290 | v2_context->mtlb_cpu = dma_alloc_wc(dev: global->dev, SZ_4K, |
291 | dma_addr: &v2_context->mtlb_dma, GFP_KERNEL); |
292 | if (!v2_context->mtlb_cpu) |
293 | goto out_free_id; |
294 | |
295 | memset32(s: v2_context->mtlb_cpu, MMUv2_PTE_EXCEPTION, |
296 | MMUv2_MAX_STLB_ENTRIES); |
297 | |
298 | global->v2.pta_cpu[v2_context->id] = v2_context->mtlb_dma; |
299 | |
300 | context = &v2_context->base; |
301 | context->global = global; |
302 | kref_init(kref: &context->refcount); |
303 | mutex_init(&context->lock); |
304 | INIT_LIST_HEAD(list: &context->mappings); |
305 | drm_mm_init(mm: &context->mm, SZ_4K, size: (u64)SZ_1G * 4 - SZ_4K); |
306 | |
307 | return context; |
308 | |
309 | out_free_id: |
310 | clear_bit(nr: v2_context->id, addr: global->v2.pta_alloc); |
311 | out_free: |
312 | vfree(addr: v2_context); |
313 | return NULL; |
314 | } |
315 | |