1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * IOMMU API for Graphics Address Relocation Table on Tegra20 |
4 | * |
5 | * Copyright (c) 2010-2012, NVIDIA CORPORATION. All rights reserved. |
6 | * |
7 | * Author: Hiroshi DOYU <hdoyu@nvidia.com> |
8 | */ |
9 | |
10 | #define dev_fmt(fmt) "gart: " fmt |
11 | |
12 | #include <linux/io.h> |
13 | #include <linux/iommu.h> |
14 | #include <linux/moduleparam.h> |
15 | #include <linux/platform_device.h> |
16 | #include <linux/slab.h> |
17 | #include <linux/spinlock.h> |
18 | #include <linux/vmalloc.h> |
19 | |
20 | #include <soc/tegra/mc.h> |
21 | |
22 | #define GART_REG_BASE 0x24 |
23 | #define GART_CONFIG (0x24 - GART_REG_BASE) |
24 | #define GART_ENTRY_ADDR (0x28 - GART_REG_BASE) |
25 | #define GART_ENTRY_DATA (0x2c - GART_REG_BASE) |
26 | |
27 | #define GART_ENTRY_PHYS_ADDR_VALID BIT(31) |
28 | |
29 | #define GART_PAGE_SHIFT 12 |
30 | #define GART_PAGE_SIZE (1 << GART_PAGE_SHIFT) |
31 | #define GART_PAGE_MASK GENMASK(30, GART_PAGE_SHIFT) |
32 | |
33 | /* bitmap of the page sizes currently supported */ |
34 | #define GART_IOMMU_PGSIZES (GART_PAGE_SIZE) |
35 | |
36 | struct gart_device { |
37 | void __iomem *regs; |
38 | u32 *savedata; |
39 | unsigned long iovmm_base; /* offset to vmm_area start */ |
40 | unsigned long iovmm_end; /* offset to vmm_area end */ |
41 | spinlock_t pte_lock; /* for pagetable */ |
42 | spinlock_t dom_lock; /* for active domain */ |
43 | unsigned int active_devices; /* number of active devices */ |
44 | struct iommu_domain *active_domain; /* current active domain */ |
45 | struct iommu_device iommu; /* IOMMU Core handle */ |
46 | struct device *dev; |
47 | }; |
48 | |
49 | static struct gart_device *gart_handle; /* unique for a system */ |
50 | |
51 | static bool gart_debug; |
52 | |
53 | /* |
54 | * Any interaction between any block on PPSB and a block on APB or AHB |
55 | * must have these read-back to ensure the APB/AHB bus transaction is |
56 | * complete before initiating activity on the PPSB block. |
57 | */ |
58 | #define FLUSH_GART_REGS(gart) readl_relaxed((gart)->regs + GART_CONFIG) |
59 | |
60 | #define for_each_gart_pte(gart, iova) \ |
61 | for (iova = gart->iovmm_base; \ |
62 | iova < gart->iovmm_end; \ |
63 | iova += GART_PAGE_SIZE) |
64 | |
65 | static inline void gart_set_pte(struct gart_device *gart, |
66 | unsigned long iova, unsigned long pte) |
67 | { |
68 | writel_relaxed(iova, gart->regs + GART_ENTRY_ADDR); |
69 | writel_relaxed(pte, gart->regs + GART_ENTRY_DATA); |
70 | } |
71 | |
72 | static inline unsigned long gart_read_pte(struct gart_device *gart, |
73 | unsigned long iova) |
74 | { |
75 | unsigned long pte; |
76 | |
77 | writel_relaxed(iova, gart->regs + GART_ENTRY_ADDR); |
78 | pte = readl_relaxed(gart->regs + GART_ENTRY_DATA); |
79 | |
80 | return pte; |
81 | } |
82 | |
83 | static void do_gart_setup(struct gart_device *gart, const u32 *data) |
84 | { |
85 | unsigned long iova; |
86 | |
87 | for_each_gart_pte(gart, iova) |
88 | gart_set_pte(gart, iova, pte: data ? *(data++) : 0); |
89 | |
90 | writel_relaxed(1, gart->regs + GART_CONFIG); |
91 | FLUSH_GART_REGS(gart); |
92 | } |
93 | |
94 | static inline bool gart_iova_range_invalid(struct gart_device *gart, |
95 | unsigned long iova, size_t bytes) |
96 | { |
97 | return unlikely(iova < gart->iovmm_base || bytes != GART_PAGE_SIZE || |
98 | iova + bytes > gart->iovmm_end); |
99 | } |
100 | |
101 | static inline bool gart_pte_valid(struct gart_device *gart, unsigned long iova) |
102 | { |
103 | return !!(gart_read_pte(gart, iova) & GART_ENTRY_PHYS_ADDR_VALID); |
104 | } |
105 | |
106 | static int gart_iommu_attach_dev(struct iommu_domain *domain, |
107 | struct device *dev) |
108 | { |
109 | struct gart_device *gart = gart_handle; |
110 | int ret = 0; |
111 | |
112 | spin_lock(lock: &gart->dom_lock); |
113 | |
114 | if (gart->active_domain && gart->active_domain != domain) { |
115 | ret = -EINVAL; |
116 | } else if (dev_iommu_priv_get(dev) != domain) { |
117 | dev_iommu_priv_set(dev, priv: domain); |
118 | gart->active_domain = domain; |
119 | gart->active_devices++; |
120 | } |
121 | |
122 | spin_unlock(lock: &gart->dom_lock); |
123 | |
124 | return ret; |
125 | } |
126 | |
127 | static void gart_iommu_set_platform_dma(struct device *dev) |
128 | { |
129 | struct iommu_domain *domain = iommu_get_domain_for_dev(dev); |
130 | struct gart_device *gart = gart_handle; |
131 | |
132 | spin_lock(lock: &gart->dom_lock); |
133 | |
134 | if (dev_iommu_priv_get(dev) == domain) { |
135 | dev_iommu_priv_set(dev, NULL); |
136 | |
137 | if (--gart->active_devices == 0) |
138 | gart->active_domain = NULL; |
139 | } |
140 | |
141 | spin_unlock(lock: &gart->dom_lock); |
142 | } |
143 | |
144 | static struct iommu_domain *gart_iommu_domain_alloc(unsigned type) |
145 | { |
146 | struct iommu_domain *domain; |
147 | |
148 | if (type != IOMMU_DOMAIN_UNMANAGED) |
149 | return NULL; |
150 | |
151 | domain = kzalloc(size: sizeof(*domain), GFP_KERNEL); |
152 | if (domain) { |
153 | domain->geometry.aperture_start = gart_handle->iovmm_base; |
154 | domain->geometry.aperture_end = gart_handle->iovmm_end - 1; |
155 | domain->geometry.force_aperture = true; |
156 | } |
157 | |
158 | return domain; |
159 | } |
160 | |
161 | static void gart_iommu_domain_free(struct iommu_domain *domain) |
162 | { |
163 | WARN_ON(gart_handle->active_domain == domain); |
164 | kfree(objp: domain); |
165 | } |
166 | |
167 | static inline int __gart_iommu_map(struct gart_device *gart, unsigned long iova, |
168 | unsigned long pa) |
169 | { |
170 | if (unlikely(gart_debug && gart_pte_valid(gart, iova))) { |
171 | dev_err(gart->dev, "Page entry is in-use\n" ); |
172 | return -EINVAL; |
173 | } |
174 | |
175 | gart_set_pte(gart, iova, GART_ENTRY_PHYS_ADDR_VALID | pa); |
176 | |
177 | return 0; |
178 | } |
179 | |
180 | static int gart_iommu_map(struct iommu_domain *domain, unsigned long iova, |
181 | phys_addr_t pa, size_t bytes, int prot, gfp_t gfp) |
182 | { |
183 | struct gart_device *gart = gart_handle; |
184 | int ret; |
185 | |
186 | if (gart_iova_range_invalid(gart, iova, bytes)) |
187 | return -EINVAL; |
188 | |
189 | spin_lock(lock: &gart->pte_lock); |
190 | ret = __gart_iommu_map(gart, iova, pa: (unsigned long)pa); |
191 | spin_unlock(lock: &gart->pte_lock); |
192 | |
193 | return ret; |
194 | } |
195 | |
196 | static inline int __gart_iommu_unmap(struct gart_device *gart, |
197 | unsigned long iova) |
198 | { |
199 | if (unlikely(gart_debug && !gart_pte_valid(gart, iova))) { |
200 | dev_err(gart->dev, "Page entry is invalid\n" ); |
201 | return -EINVAL; |
202 | } |
203 | |
204 | gart_set_pte(gart, iova, pte: 0); |
205 | |
206 | return 0; |
207 | } |
208 | |
209 | static size_t gart_iommu_unmap(struct iommu_domain *domain, unsigned long iova, |
210 | size_t bytes, struct iommu_iotlb_gather *gather) |
211 | { |
212 | struct gart_device *gart = gart_handle; |
213 | int err; |
214 | |
215 | if (gart_iova_range_invalid(gart, iova, bytes)) |
216 | return 0; |
217 | |
218 | spin_lock(lock: &gart->pte_lock); |
219 | err = __gart_iommu_unmap(gart, iova); |
220 | spin_unlock(lock: &gart->pte_lock); |
221 | |
222 | return err ? 0 : bytes; |
223 | } |
224 | |
225 | static phys_addr_t gart_iommu_iova_to_phys(struct iommu_domain *domain, |
226 | dma_addr_t iova) |
227 | { |
228 | struct gart_device *gart = gart_handle; |
229 | unsigned long pte; |
230 | |
231 | if (gart_iova_range_invalid(gart, iova, GART_PAGE_SIZE)) |
232 | return -EINVAL; |
233 | |
234 | spin_lock(lock: &gart->pte_lock); |
235 | pte = gart_read_pte(gart, iova); |
236 | spin_unlock(lock: &gart->pte_lock); |
237 | |
238 | return pte & GART_PAGE_MASK; |
239 | } |
240 | |
241 | static struct iommu_device *gart_iommu_probe_device(struct device *dev) |
242 | { |
243 | if (!dev_iommu_fwspec_get(dev)) |
244 | return ERR_PTR(error: -ENODEV); |
245 | |
246 | return &gart_handle->iommu; |
247 | } |
248 | |
249 | static int gart_iommu_of_xlate(struct device *dev, |
250 | struct of_phandle_args *args) |
251 | { |
252 | return 0; |
253 | } |
254 | |
255 | static void gart_iommu_sync_map(struct iommu_domain *domain, unsigned long iova, |
256 | size_t size) |
257 | { |
258 | FLUSH_GART_REGS(gart_handle); |
259 | } |
260 | |
261 | static void gart_iommu_sync(struct iommu_domain *domain, |
262 | struct iommu_iotlb_gather *gather) |
263 | { |
264 | size_t length = gather->end - gather->start + 1; |
265 | |
266 | gart_iommu_sync_map(domain, iova: gather->start, size: length); |
267 | } |
268 | |
269 | static const struct iommu_ops gart_iommu_ops = { |
270 | .domain_alloc = gart_iommu_domain_alloc, |
271 | .probe_device = gart_iommu_probe_device, |
272 | .device_group = generic_device_group, |
273 | .set_platform_dma_ops = gart_iommu_set_platform_dma, |
274 | .pgsize_bitmap = GART_IOMMU_PGSIZES, |
275 | .of_xlate = gart_iommu_of_xlate, |
276 | .default_domain_ops = &(const struct iommu_domain_ops) { |
277 | .attach_dev = gart_iommu_attach_dev, |
278 | .map = gart_iommu_map, |
279 | .unmap = gart_iommu_unmap, |
280 | .iova_to_phys = gart_iommu_iova_to_phys, |
281 | .iotlb_sync_map = gart_iommu_sync_map, |
282 | .iotlb_sync = gart_iommu_sync, |
283 | .free = gart_iommu_domain_free, |
284 | } |
285 | }; |
286 | |
287 | int tegra_gart_suspend(struct gart_device *gart) |
288 | { |
289 | u32 *data = gart->savedata; |
290 | unsigned long iova; |
291 | |
292 | /* |
293 | * All GART users shall be suspended at this point. Disable |
294 | * address translation to trap all GART accesses as invalid |
295 | * memory accesses. |
296 | */ |
297 | writel_relaxed(0, gart->regs + GART_CONFIG); |
298 | FLUSH_GART_REGS(gart); |
299 | |
300 | for_each_gart_pte(gart, iova) |
301 | *(data++) = gart_read_pte(gart, iova); |
302 | |
303 | return 0; |
304 | } |
305 | |
306 | int tegra_gart_resume(struct gart_device *gart) |
307 | { |
308 | do_gart_setup(gart, data: gart->savedata); |
309 | |
310 | return 0; |
311 | } |
312 | |
313 | struct gart_device *tegra_gart_probe(struct device *dev, struct tegra_mc *mc) |
314 | { |
315 | struct gart_device *gart; |
316 | struct resource *res; |
317 | int err; |
318 | |
319 | BUILD_BUG_ON(PAGE_SHIFT != GART_PAGE_SHIFT); |
320 | |
321 | /* the GART memory aperture is required */ |
322 | res = platform_get_resource(to_platform_device(dev), IORESOURCE_MEM, 1); |
323 | if (!res) { |
324 | dev_err(dev, "Memory aperture resource unavailable\n" ); |
325 | return ERR_PTR(error: -ENXIO); |
326 | } |
327 | |
328 | gart = kzalloc(size: sizeof(*gart), GFP_KERNEL); |
329 | if (!gart) |
330 | return ERR_PTR(error: -ENOMEM); |
331 | |
332 | gart_handle = gart; |
333 | |
334 | gart->dev = dev; |
335 | gart->regs = mc->regs + GART_REG_BASE; |
336 | gart->iovmm_base = res->start; |
337 | gart->iovmm_end = res->end + 1; |
338 | spin_lock_init(&gart->pte_lock); |
339 | spin_lock_init(&gart->dom_lock); |
340 | |
341 | do_gart_setup(gart, NULL); |
342 | |
343 | err = iommu_device_sysfs_add(iommu: &gart->iommu, parent: dev, NULL, fmt: "gart" ); |
344 | if (err) |
345 | goto free_gart; |
346 | |
347 | err = iommu_device_register(iommu: &gart->iommu, ops: &gart_iommu_ops, hwdev: dev); |
348 | if (err) |
349 | goto remove_sysfs; |
350 | |
351 | gart->savedata = vmalloc(size: resource_size(res) / GART_PAGE_SIZE * |
352 | sizeof(u32)); |
353 | if (!gart->savedata) { |
354 | err = -ENOMEM; |
355 | goto unregister_iommu; |
356 | } |
357 | |
358 | return gart; |
359 | |
360 | unregister_iommu: |
361 | iommu_device_unregister(iommu: &gart->iommu); |
362 | remove_sysfs: |
363 | iommu_device_sysfs_remove(iommu: &gart->iommu); |
364 | free_gart: |
365 | kfree(objp: gart); |
366 | |
367 | return ERR_PTR(error: err); |
368 | } |
369 | |
370 | module_param(gart_debug, bool, 0644); |
371 | MODULE_PARM_DESC(gart_debug, "Enable GART debugging" ); |
372 | |