1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* Copyright (C) 2020 Red Hat, Inc. |
3 | * Author: Jason Wang <jasowang@redhat.com> |
4 | * |
5 | * IOTLB implementation for vhost. |
6 | */ |
7 | #include <linux/slab.h> |
8 | #include <linux/vhost_iotlb.h> |
9 | #include <linux/module.h> |
10 | |
11 | #define MOD_VERSION "0.1" |
12 | #define MOD_DESC "VHOST IOTLB" |
13 | #define MOD_AUTHOR "Jason Wang <jasowang@redhat.com>" |
14 | #define MOD_LICENSE "GPL v2" |
15 | |
16 | #define START(map) ((map)->start) |
17 | #define LAST(map) ((map)->last) |
18 | |
19 | INTERVAL_TREE_DEFINE(struct vhost_iotlb_map, |
20 | rb, __u64, __subtree_last, |
21 | START, LAST, static inline, vhost_iotlb_itree); |
22 | |
23 | /** |
24 | * vhost_iotlb_map_free - remove a map node and free it |
25 | * @iotlb: the IOTLB |
26 | * @map: the map that want to be remove and freed |
27 | */ |
28 | void vhost_iotlb_map_free(struct vhost_iotlb *iotlb, |
29 | struct vhost_iotlb_map *map) |
30 | { |
31 | vhost_iotlb_itree_remove(node: map, root: &iotlb->root); |
32 | list_del(entry: &map->link); |
33 | kfree(objp: map); |
34 | iotlb->nmaps--; |
35 | } |
36 | EXPORT_SYMBOL_GPL(vhost_iotlb_map_free); |
37 | |
38 | /** |
39 | * vhost_iotlb_add_range_ctx - add a new range to vhost IOTLB |
40 | * @iotlb: the IOTLB |
41 | * @start: start of the IOVA range |
42 | * @last: last of IOVA range |
43 | * @addr: the address that is mapped to @start |
44 | * @perm: access permission of this range |
45 | * @opaque: the opaque pointer for the new mapping |
46 | * |
47 | * Returns an error last is smaller than start or memory allocation |
48 | * fails |
49 | */ |
50 | int vhost_iotlb_add_range_ctx(struct vhost_iotlb *iotlb, |
51 | u64 start, u64 last, |
52 | u64 addr, unsigned int perm, |
53 | void *opaque) |
54 | { |
55 | struct vhost_iotlb_map *map; |
56 | |
57 | if (last < start) |
58 | return -EFAULT; |
59 | |
60 | /* If the range being mapped is [0, ULONG_MAX], split it into two entries |
61 | * otherwise its size would overflow u64. |
62 | */ |
63 | if (start == 0 && last == ULONG_MAX) { |
64 | u64 mid = last / 2; |
65 | int err = vhost_iotlb_add_range_ctx(iotlb, start, last: mid, addr, |
66 | perm, opaque); |
67 | |
68 | if (err) |
69 | return err; |
70 | |
71 | addr += mid + 1; |
72 | start = mid + 1; |
73 | } |
74 | |
75 | if (iotlb->limit && |
76 | iotlb->nmaps == iotlb->limit && |
77 | iotlb->flags & VHOST_IOTLB_FLAG_RETIRE) { |
78 | map = list_first_entry(&iotlb->list, typeof(*map), link); |
79 | vhost_iotlb_map_free(iotlb, map); |
80 | } |
81 | |
82 | map = kmalloc(size: sizeof(*map), GFP_ATOMIC); |
83 | if (!map) |
84 | return -ENOMEM; |
85 | |
86 | map->start = start; |
87 | map->size = last - start + 1; |
88 | map->last = last; |
89 | map->addr = addr; |
90 | map->perm = perm; |
91 | map->opaque = opaque; |
92 | |
93 | iotlb->nmaps++; |
94 | vhost_iotlb_itree_insert(node: map, root: &iotlb->root); |
95 | |
96 | INIT_LIST_HEAD(list: &map->link); |
97 | list_add_tail(new: &map->link, head: &iotlb->list); |
98 | |
99 | return 0; |
100 | } |
101 | EXPORT_SYMBOL_GPL(vhost_iotlb_add_range_ctx); |
102 | |
103 | int vhost_iotlb_add_range(struct vhost_iotlb *iotlb, |
104 | u64 start, u64 last, |
105 | u64 addr, unsigned int perm) |
106 | { |
107 | return vhost_iotlb_add_range_ctx(iotlb, start, last, |
108 | addr, perm, NULL); |
109 | } |
110 | EXPORT_SYMBOL_GPL(vhost_iotlb_add_range); |
111 | |
112 | /** |
113 | * vhost_iotlb_del_range - delete overlapped ranges from vhost IOTLB |
114 | * @iotlb: the IOTLB |
115 | * @start: start of the IOVA range |
116 | * @last: last of IOVA range |
117 | */ |
118 | void vhost_iotlb_del_range(struct vhost_iotlb *iotlb, u64 start, u64 last) |
119 | { |
120 | struct vhost_iotlb_map *map; |
121 | |
122 | while ((map = vhost_iotlb_itree_iter_first(root: &iotlb->root, |
123 | start, last))) |
124 | vhost_iotlb_map_free(iotlb, map); |
125 | } |
126 | EXPORT_SYMBOL_GPL(vhost_iotlb_del_range); |
127 | |
128 | /** |
129 | * vhost_iotlb_init - initialize a vhost IOTLB |
130 | * @iotlb: the IOTLB that needs to be initialized |
131 | * @limit: maximum number of IOTLB entries |
132 | * @flags: VHOST_IOTLB_FLAG_XXX |
133 | */ |
134 | void vhost_iotlb_init(struct vhost_iotlb *iotlb, unsigned int limit, |
135 | unsigned int flags) |
136 | { |
137 | iotlb->root = RB_ROOT_CACHED; |
138 | iotlb->limit = limit; |
139 | iotlb->nmaps = 0; |
140 | iotlb->flags = flags; |
141 | INIT_LIST_HEAD(list: &iotlb->list); |
142 | } |
143 | EXPORT_SYMBOL_GPL(vhost_iotlb_init); |
144 | |
145 | /** |
146 | * vhost_iotlb_alloc - add a new vhost IOTLB |
147 | * @limit: maximum number of IOTLB entries |
148 | * @flags: VHOST_IOTLB_FLAG_XXX |
149 | * |
150 | * Returns an error is memory allocation fails |
151 | */ |
152 | struct vhost_iotlb *vhost_iotlb_alloc(unsigned int limit, unsigned int flags) |
153 | { |
154 | struct vhost_iotlb *iotlb = kzalloc(size: sizeof(*iotlb), GFP_KERNEL); |
155 | |
156 | if (!iotlb) |
157 | return NULL; |
158 | |
159 | vhost_iotlb_init(iotlb, limit, flags); |
160 | |
161 | return iotlb; |
162 | } |
163 | EXPORT_SYMBOL_GPL(vhost_iotlb_alloc); |
164 | |
165 | /** |
166 | * vhost_iotlb_reset - reset vhost IOTLB (free all IOTLB entries) |
167 | * @iotlb: the IOTLB to be reset |
168 | */ |
169 | void vhost_iotlb_reset(struct vhost_iotlb *iotlb) |
170 | { |
171 | vhost_iotlb_del_range(iotlb, 0ULL, 0ULL - 1); |
172 | } |
173 | EXPORT_SYMBOL_GPL(vhost_iotlb_reset); |
174 | |
175 | /** |
176 | * vhost_iotlb_free - reset and free vhost IOTLB |
177 | * @iotlb: the IOTLB to be freed |
178 | */ |
179 | void vhost_iotlb_free(struct vhost_iotlb *iotlb) |
180 | { |
181 | if (iotlb) { |
182 | vhost_iotlb_reset(iotlb); |
183 | kfree(objp: iotlb); |
184 | } |
185 | } |
186 | EXPORT_SYMBOL_GPL(vhost_iotlb_free); |
187 | |
188 | /** |
189 | * vhost_iotlb_itree_first - return the first overlapped range |
190 | * @iotlb: the IOTLB |
191 | * @start: start of IOVA range |
192 | * @last: last byte in IOVA range |
193 | */ |
194 | struct vhost_iotlb_map * |
195 | vhost_iotlb_itree_first(struct vhost_iotlb *iotlb, u64 start, u64 last) |
196 | { |
197 | return vhost_iotlb_itree_iter_first(root: &iotlb->root, start, last); |
198 | } |
199 | EXPORT_SYMBOL_GPL(vhost_iotlb_itree_first); |
200 | |
201 | /** |
202 | * vhost_iotlb_itree_next - return the next overlapped range |
203 | * @map: the starting map node |
204 | * @start: start of IOVA range |
205 | * @last: last byte IOVA range |
206 | */ |
207 | struct vhost_iotlb_map * |
208 | vhost_iotlb_itree_next(struct vhost_iotlb_map *map, u64 start, u64 last) |
209 | { |
210 | return vhost_iotlb_itree_iter_next(node: map, start, last); |
211 | } |
212 | EXPORT_SYMBOL_GPL(vhost_iotlb_itree_next); |
213 | |
214 | MODULE_VERSION(MOD_VERSION); |
215 | MODULE_DESCRIPTION(MOD_DESC); |
216 | MODULE_AUTHOR(MOD_AUTHOR); |
217 | MODULE_LICENSE(MOD_LICENSE); |
218 | |