1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef __IO_PGTABLE_H
3#define __IO_PGTABLE_H
4#include <linux/bitops.h>
5
6/*
7 * Public API for use by IOMMU drivers
8 */
9enum io_pgtable_fmt {
10 ARM_32_LPAE_S1,
11 ARM_32_LPAE_S2,
12 ARM_64_LPAE_S1,
13 ARM_64_LPAE_S2,
14 ARM_V7S,
15 IO_PGTABLE_NUM_FMTS,
16};
17
18/**
19 * struct iommu_gather_ops - IOMMU callbacks for TLB and page table management.
20 *
21 * @tlb_flush_all: Synchronously invalidate the entire TLB context.
22 * @tlb_add_flush: Queue up a TLB invalidation for a virtual address range.
23 * @tlb_sync: Ensure any queued TLB invalidation has taken effect, and
24 * any corresponding page table updates are visible to the
25 * IOMMU.
26 *
27 * Note that these can all be called in atomic context and must therefore
28 * not block.
29 */
30struct iommu_gather_ops {
31 void (*tlb_flush_all)(void *cookie);
32 void (*tlb_add_flush)(unsigned long iova, size_t size, size_t granule,
33 bool leaf, void *cookie);
34 void (*tlb_sync)(void *cookie);
35};
36
37/**
38 * struct io_pgtable_cfg - Configuration data for a set of page tables.
39 *
40 * @quirks: A bitmap of hardware quirks that require some special
41 * action by the low-level page table allocator.
42 * @pgsize_bitmap: A bitmap of page sizes supported by this set of page
43 * tables.
44 * @ias: Input address (iova) size, in bits.
45 * @oas: Output address (paddr) size, in bits.
46 * @tlb: TLB management callbacks for this set of tables.
47 * @iommu_dev: The device representing the DMA configuration for the
48 * page table walker.
49 */
50struct io_pgtable_cfg {
51 /*
52 * IO_PGTABLE_QUIRK_ARM_NS: (ARM formats) Set NS and NSTABLE bits in
53 * stage 1 PTEs, for hardware which insists on validating them
54 * even in non-secure state where they should normally be ignored.
55 *
56 * IO_PGTABLE_QUIRK_NO_PERMS: Ignore the IOMMU_READ, IOMMU_WRITE and
57 * IOMMU_NOEXEC flags and map everything with full access, for
58 * hardware which does not implement the permissions of a given
59 * format, and/or requires some format-specific default value.
60 *
61 * IO_PGTABLE_QUIRK_TLBI_ON_MAP: If the format forbids caching invalid
62 * (unmapped) entries but the hardware might do so anyway, perform
63 * TLB maintenance when mapping as well as when unmapping.
64 *
65 * IO_PGTABLE_QUIRK_ARM_MTK_4GB: (ARM v7s format) Set bit 9 in all
66 * PTEs, for Mediatek IOMMUs which treat it as a 33rd address bit
67 * when the SoC is in "4GB mode" and they can only access the high
68 * remap of DRAM (0x1_00000000 to 0x1_ffffffff).
69 *
70 * IO_PGTABLE_QUIRK_NO_DMA: Guarantees that the tables will only ever
71 * be accessed by a fully cache-coherent IOMMU or CPU (e.g. for a
72 * software-emulated IOMMU), such that pagetable updates need not
73 * be treated as explicit DMA data.
74 *
75 * IO_PGTABLE_QUIRK_NON_STRICT: Skip issuing synchronous leaf TLBIs
76 * on unmap, for DMA domains using the flush queue mechanism for
77 * delayed invalidation.
78 */
79 #define IO_PGTABLE_QUIRK_ARM_NS BIT(0)
80 #define IO_PGTABLE_QUIRK_NO_PERMS BIT(1)
81 #define IO_PGTABLE_QUIRK_TLBI_ON_MAP BIT(2)
82 #define IO_PGTABLE_QUIRK_ARM_MTK_4GB BIT(3)
83 #define IO_PGTABLE_QUIRK_NO_DMA BIT(4)
84 #define IO_PGTABLE_QUIRK_NON_STRICT BIT(5)
85 unsigned long quirks;
86 unsigned long pgsize_bitmap;
87 unsigned int ias;
88 unsigned int oas;
89 const struct iommu_gather_ops *tlb;
90 struct device *iommu_dev;
91
92 /* Low-level data specific to the table format */
93 union {
94 struct {
95 u64 ttbr[2];
96 u64 tcr;
97 u64 mair[2];
98 } arm_lpae_s1_cfg;
99
100 struct {
101 u64 vttbr;
102 u64 vtcr;
103 } arm_lpae_s2_cfg;
104
105 struct {
106 u32 ttbr[2];
107 u32 tcr;
108 u32 nmrr;
109 u32 prrr;
110 } arm_v7s_cfg;
111 };
112};
113
114/**
115 * struct io_pgtable_ops - Page table manipulation API for IOMMU drivers.
116 *
117 * @map: Map a physically contiguous memory region.
118 * @unmap: Unmap a physically contiguous memory region.
119 * @iova_to_phys: Translate iova to physical address.
120 *
121 * These functions map directly onto the iommu_ops member functions with
122 * the same names.
123 */
124struct io_pgtable_ops {
125 int (*map)(struct io_pgtable_ops *ops, unsigned long iova,
126 phys_addr_t paddr, size_t size, int prot);
127 size_t (*unmap)(struct io_pgtable_ops *ops, unsigned long iova,
128 size_t size);
129 phys_addr_t (*iova_to_phys)(struct io_pgtable_ops *ops,
130 unsigned long iova);
131};
132
133/**
134 * alloc_io_pgtable_ops() - Allocate a page table allocator for use by an IOMMU.
135 *
136 * @fmt: The page table format.
137 * @cfg: The page table configuration. This will be modified to represent
138 * the configuration actually provided by the allocator (e.g. the
139 * pgsize_bitmap may be restricted).
140 * @cookie: An opaque token provided by the IOMMU driver and passed back to
141 * the callback routines in cfg->tlb.
142 */
143struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt,
144 struct io_pgtable_cfg *cfg,
145 void *cookie);
146
147/**
148 * free_io_pgtable_ops() - Free an io_pgtable_ops structure. The caller
149 * *must* ensure that the page table is no longer
150 * live, but the TLB can be dirty.
151 *
152 * @ops: The ops returned from alloc_io_pgtable_ops.
153 */
154void free_io_pgtable_ops(struct io_pgtable_ops *ops);
155
156
157/*
158 * Internal structures for page table allocator implementations.
159 */
160
161/**
162 * struct io_pgtable - Internal structure describing a set of page tables.
163 *
164 * @fmt: The page table format.
165 * @cookie: An opaque token provided by the IOMMU driver and passed back to
166 * any callback routines.
167 * @cfg: A copy of the page table configuration.
168 * @ops: The page table operations in use for this set of page tables.
169 */
170struct io_pgtable {
171 enum io_pgtable_fmt fmt;
172 void *cookie;
173 struct io_pgtable_cfg cfg;
174 struct io_pgtable_ops ops;
175};
176
177#define io_pgtable_ops_to_pgtable(x) container_of((x), struct io_pgtable, ops)
178
179static inline void io_pgtable_tlb_flush_all(struct io_pgtable *iop)
180{
181 iop->cfg.tlb->tlb_flush_all(iop->cookie);
182}
183
184static inline void io_pgtable_tlb_add_flush(struct io_pgtable *iop,
185 unsigned long iova, size_t size, size_t granule, bool leaf)
186{
187 iop->cfg.tlb->tlb_add_flush(iova, size, granule, leaf, iop->cookie);
188}
189
190static inline void io_pgtable_tlb_sync(struct io_pgtable *iop)
191{
192 iop->cfg.tlb->tlb_sync(iop->cookie);
193}
194
195/**
196 * struct io_pgtable_init_fns - Alloc/free a set of page tables for a
197 * particular format.
198 *
199 * @alloc: Allocate a set of page tables described by cfg.
200 * @free: Free the page tables associated with iop.
201 */
202struct io_pgtable_init_fns {
203 struct io_pgtable *(*alloc)(struct io_pgtable_cfg *cfg, void *cookie);
204 void (*free)(struct io_pgtable *iop);
205};
206
207extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns;
208extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns;
209extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns;
210extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns;
211extern struct io_pgtable_init_fns io_pgtable_arm_v7s_init_fns;
212
213#endif /* __IO_PGTABLE_H */
214