1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #ifndef __IO_PGTABLE_H |
3 | #define __IO_PGTABLE_H |
4 | |
5 | #include <linux/bitops.h> |
6 | #include <linux/iommu.h> |
7 | |
8 | /* |
9 | * Public API for use by IOMMU drivers |
10 | */ |
11 | enum io_pgtable_fmt { |
12 | ARM_32_LPAE_S1, |
13 | ARM_32_LPAE_S2, |
14 | ARM_64_LPAE_S1, |
15 | ARM_64_LPAE_S2, |
16 | ARM_V7S, |
17 | ARM_MALI_LPAE, |
18 | AMD_IOMMU_V1, |
19 | AMD_IOMMU_V2, |
20 | APPLE_DART, |
21 | APPLE_DART2, |
22 | IO_PGTABLE_NUM_FMTS, |
23 | }; |
24 | |
25 | /** |
26 | * struct iommu_flush_ops - IOMMU callbacks for TLB and page table management. |
27 | * |
28 | * @tlb_flush_all: Synchronously invalidate the entire TLB context. |
29 | * @tlb_flush_walk: Synchronously invalidate all intermediate TLB state |
30 | * (sometimes referred to as the "walk cache") for a virtual |
31 | * address range. |
32 | * @tlb_add_page: Optional callback to queue up leaf TLB invalidation for a |
33 | * single page. IOMMUs that cannot batch TLB invalidation |
34 | * operations efficiently will typically issue them here, but |
35 | * others may decide to update the iommu_iotlb_gather structure |
36 | * and defer the invalidation until iommu_iotlb_sync() instead. |
37 | * |
38 | * Note that these can all be called in atomic context and must therefore |
39 | * not block. |
40 | */ |
41 | struct iommu_flush_ops { |
42 | void (*tlb_flush_all)(void *cookie); |
43 | void (*tlb_flush_walk)(unsigned long iova, size_t size, size_t granule, |
44 | void *cookie); |
45 | void (*tlb_add_page)(struct iommu_iotlb_gather *gather, |
46 | unsigned long iova, size_t granule, void *cookie); |
47 | }; |
48 | |
49 | /** |
50 | * struct io_pgtable_cfg - Configuration data for a set of page tables. |
51 | * |
52 | * @quirks: A bitmap of hardware quirks that require some special |
53 | * action by the low-level page table allocator. |
54 | * @pgsize_bitmap: A bitmap of page sizes supported by this set of page |
55 | * tables. |
56 | * @ias: Input address (iova) size, in bits. |
57 | * @oas: Output address (paddr) size, in bits. |
58 | * @coherent_walk A flag to indicate whether or not page table walks made |
59 | * by the IOMMU are coherent with the CPU caches. |
60 | * @tlb: TLB management callbacks for this set of tables. |
61 | * @iommu_dev: The device representing the DMA configuration for the |
62 | * page table walker. |
63 | */ |
64 | struct io_pgtable_cfg { |
65 | /* |
66 | * IO_PGTABLE_QUIRK_ARM_NS: (ARM formats) Set NS and NSTABLE bits in |
67 | * stage 1 PTEs, for hardware which insists on validating them |
68 | * even in non-secure state where they should normally be ignored. |
69 | * |
70 | * IO_PGTABLE_QUIRK_NO_PERMS: Ignore the IOMMU_READ, IOMMU_WRITE and |
71 | * IOMMU_NOEXEC flags and map everything with full access, for |
72 | * hardware which does not implement the permissions of a given |
73 | * format, and/or requires some format-specific default value. |
74 | * |
75 | * IO_PGTABLE_QUIRK_ARM_MTK_EXT: (ARM v7s format) MediaTek IOMMUs extend |
76 | * to support up to 35 bits PA where the bit32, bit33 and bit34 are |
77 | * encoded in the bit9, bit4 and bit5 of the PTE respectively. |
78 | * |
79 | * IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT: (ARM v7s format) MediaTek IOMMUs |
80 | * extend the translation table base support up to 35 bits PA, the |
81 | * encoding format is same with IO_PGTABLE_QUIRK_ARM_MTK_EXT. |
82 | * |
83 | * IO_PGTABLE_QUIRK_ARM_TTBR1: (ARM LPAE format) Configure the table |
84 | * for use in the upper half of a split address space. |
85 | * |
86 | * IO_PGTABLE_QUIRK_ARM_OUTER_WBWA: Override the outer-cacheability |
87 | * attributes set in the TCR for a non-coherent page-table walker. |
88 | */ |
89 | #define IO_PGTABLE_QUIRK_ARM_NS BIT(0) |
90 | #define IO_PGTABLE_QUIRK_NO_PERMS BIT(1) |
91 | #define IO_PGTABLE_QUIRK_ARM_MTK_EXT BIT(3) |
92 | #define IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT BIT(4) |
93 | #define IO_PGTABLE_QUIRK_ARM_TTBR1 BIT(5) |
94 | #define IO_PGTABLE_QUIRK_ARM_OUTER_WBWA BIT(6) |
95 | unsigned long quirks; |
96 | unsigned long pgsize_bitmap; |
97 | unsigned int ias; |
98 | unsigned int oas; |
99 | bool coherent_walk; |
100 | const struct iommu_flush_ops *tlb; |
101 | struct device *iommu_dev; |
102 | |
103 | /** |
104 | * @alloc: Custom page allocator. |
105 | * |
106 | * Optional hook used to allocate page tables. If this function is NULL, |
107 | * @free must be NULL too. |
108 | * |
109 | * Memory returned should be zeroed and suitable for dma_map_single() and |
110 | * virt_to_phys(). |
111 | * |
112 | * Not all formats support custom page allocators. Before considering |
113 | * passing a non-NULL value, make sure the chosen page format supports |
114 | * this feature. |
115 | */ |
116 | void *(*alloc)(void *cookie, size_t size, gfp_t gfp); |
117 | |
118 | /** |
119 | * @free: Custom page de-allocator. |
120 | * |
121 | * Optional hook used to free page tables allocated with the @alloc |
122 | * hook. Must be non-NULL if @alloc is not NULL, must be NULL |
123 | * otherwise. |
124 | */ |
125 | void (*free)(void *cookie, void *pages, size_t size); |
126 | |
127 | /* Low-level data specific to the table format */ |
128 | union { |
129 | struct { |
130 | u64 ttbr; |
131 | struct { |
132 | u32 ips:3; |
133 | u32 tg:2; |
134 | u32 sh:2; |
135 | u32 orgn:2; |
136 | u32 irgn:2; |
137 | u32 tsz:6; |
138 | } tcr; |
139 | u64 mair; |
140 | } arm_lpae_s1_cfg; |
141 | |
142 | struct { |
143 | u64 vttbr; |
144 | struct { |
145 | u32 ps:3; |
146 | u32 tg:2; |
147 | u32 sh:2; |
148 | u32 orgn:2; |
149 | u32 irgn:2; |
150 | u32 sl:2; |
151 | u32 tsz:6; |
152 | } vtcr; |
153 | } arm_lpae_s2_cfg; |
154 | |
155 | struct { |
156 | u32 ttbr; |
157 | u32 tcr; |
158 | u32 nmrr; |
159 | u32 prrr; |
160 | } arm_v7s_cfg; |
161 | |
162 | struct { |
163 | u64 transtab; |
164 | u64 memattr; |
165 | } arm_mali_lpae_cfg; |
166 | |
167 | struct { |
168 | u64 ttbr[4]; |
169 | u32 n_ttbrs; |
170 | } apple_dart_cfg; |
171 | }; |
172 | }; |
173 | |
174 | /** |
175 | * struct io_pgtable_ops - Page table manipulation API for IOMMU drivers. |
176 | * |
177 | * @map_pages: Map a physically contiguous range of pages of the same size. |
178 | * @unmap_pages: Unmap a range of virtually contiguous pages of the same size. |
179 | * @iova_to_phys: Translate iova to physical address. |
180 | * |
181 | * These functions map directly onto the iommu_ops member functions with |
182 | * the same names. |
183 | */ |
184 | struct io_pgtable_ops { |
185 | int (*map_pages)(struct io_pgtable_ops *ops, unsigned long iova, |
186 | phys_addr_t paddr, size_t pgsize, size_t pgcount, |
187 | int prot, gfp_t gfp, size_t *mapped); |
188 | size_t (*unmap_pages)(struct io_pgtable_ops *ops, unsigned long iova, |
189 | size_t pgsize, size_t pgcount, |
190 | struct iommu_iotlb_gather *gather); |
191 | phys_addr_t (*iova_to_phys)(struct io_pgtable_ops *ops, |
192 | unsigned long iova); |
193 | int (*read_and_clear_dirty)(struct io_pgtable_ops *ops, |
194 | unsigned long iova, size_t size, |
195 | unsigned long flags, |
196 | struct iommu_dirty_bitmap *dirty); |
197 | }; |
198 | |
199 | /** |
200 | * alloc_io_pgtable_ops() - Allocate a page table allocator for use by an IOMMU. |
201 | * |
202 | * @fmt: The page table format. |
203 | * @cfg: The page table configuration. This will be modified to represent |
204 | * the configuration actually provided by the allocator (e.g. the |
205 | * pgsize_bitmap may be restricted). |
206 | * @cookie: An opaque token provided by the IOMMU driver and passed back to |
207 | * the callback routines in cfg->tlb. |
208 | */ |
209 | struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt, |
210 | struct io_pgtable_cfg *cfg, |
211 | void *cookie); |
212 | |
213 | /** |
214 | * free_io_pgtable_ops() - Free an io_pgtable_ops structure. The caller |
215 | * *must* ensure that the page table is no longer |
216 | * live, but the TLB can be dirty. |
217 | * |
218 | * @ops: The ops returned from alloc_io_pgtable_ops. |
219 | */ |
220 | void free_io_pgtable_ops(struct io_pgtable_ops *ops); |
221 | |
222 | |
223 | /* |
224 | * Internal structures for page table allocator implementations. |
225 | */ |
226 | |
227 | /** |
228 | * struct io_pgtable - Internal structure describing a set of page tables. |
229 | * |
230 | * @fmt: The page table format. |
231 | * @cookie: An opaque token provided by the IOMMU driver and passed back to |
232 | * any callback routines. |
233 | * @cfg: A copy of the page table configuration. |
234 | * @ops: The page table operations in use for this set of page tables. |
235 | */ |
236 | struct io_pgtable { |
237 | enum io_pgtable_fmt fmt; |
238 | void *cookie; |
239 | struct io_pgtable_cfg cfg; |
240 | struct io_pgtable_ops ops; |
241 | }; |
242 | |
243 | #define io_pgtable_ops_to_pgtable(x) container_of((x), struct io_pgtable, ops) |
244 | |
245 | static inline void io_pgtable_tlb_flush_all(struct io_pgtable *iop) |
246 | { |
247 | if (iop->cfg.tlb && iop->cfg.tlb->tlb_flush_all) |
248 | iop->cfg.tlb->tlb_flush_all(iop->cookie); |
249 | } |
250 | |
251 | static inline void |
252 | io_pgtable_tlb_flush_walk(struct io_pgtable *iop, unsigned long iova, |
253 | size_t size, size_t granule) |
254 | { |
255 | if (iop->cfg.tlb && iop->cfg.tlb->tlb_flush_walk) |
256 | iop->cfg.tlb->tlb_flush_walk(iova, size, granule, iop->cookie); |
257 | } |
258 | |
259 | static inline void |
260 | io_pgtable_tlb_add_page(struct io_pgtable *iop, |
261 | struct iommu_iotlb_gather * gather, unsigned long iova, |
262 | size_t granule) |
263 | { |
264 | if (iop->cfg.tlb && iop->cfg.tlb->tlb_add_page) |
265 | iop->cfg.tlb->tlb_add_page(gather, iova, granule, iop->cookie); |
266 | } |
267 | |
268 | /** |
269 | * enum io_pgtable_caps - IO page table backend capabilities. |
270 | */ |
271 | enum io_pgtable_caps { |
272 | /** @IO_PGTABLE_CAP_CUSTOM_ALLOCATOR: Backend accepts custom page table allocators. */ |
273 | IO_PGTABLE_CAP_CUSTOM_ALLOCATOR = BIT(0), |
274 | }; |
275 | |
276 | /** |
277 | * struct io_pgtable_init_fns - Alloc/free a set of page tables for a |
278 | * particular format. |
279 | * |
280 | * @alloc: Allocate a set of page tables described by cfg. |
281 | * @free: Free the page tables associated with iop. |
282 | * @caps: Combination of @io_pgtable_caps flags encoding the backend capabilities. |
283 | */ |
284 | struct io_pgtable_init_fns { |
285 | struct io_pgtable *(*alloc)(struct io_pgtable_cfg *cfg, void *cookie); |
286 | void (*free)(struct io_pgtable *iop); |
287 | u32 caps; |
288 | }; |
289 | |
290 | extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns; |
291 | extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns; |
292 | extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns; |
293 | extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns; |
294 | extern struct io_pgtable_init_fns io_pgtable_arm_v7s_init_fns; |
295 | extern struct io_pgtable_init_fns io_pgtable_arm_mali_lpae_init_fns; |
296 | extern struct io_pgtable_init_fns io_pgtable_amd_iommu_v1_init_fns; |
297 | extern struct io_pgtable_init_fns io_pgtable_amd_iommu_v2_init_fns; |
298 | extern struct io_pgtable_init_fns io_pgtable_apple_dart_init_fns; |
299 | |
300 | #endif /* __IO_PGTABLE_H */ |
301 | |