1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Copyright (c) 2023 Isovalent */ |
3 | |
4 | #include <linux/bpf.h> |
5 | #include <linux/bpf_mprog.h> |
6 | #include <linux/netdevice.h> |
7 | |
8 | #include <net/tcx.h> |
9 | |
10 | int tcx_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog) |
11 | { |
12 | bool created, ingress = attr->attach_type == BPF_TCX_INGRESS; |
13 | struct net *net = current->nsproxy->net_ns; |
14 | struct bpf_mprog_entry *entry, *entry_new; |
15 | struct bpf_prog *replace_prog = NULL; |
16 | struct net_device *dev; |
17 | int ret; |
18 | |
19 | rtnl_lock(); |
20 | dev = __dev_get_by_index(net, ifindex: attr->target_ifindex); |
21 | if (!dev) { |
22 | ret = -ENODEV; |
23 | goto out; |
24 | } |
25 | if (attr->attach_flags & BPF_F_REPLACE) { |
26 | replace_prog = bpf_prog_get_type(ufd: attr->replace_bpf_fd, |
27 | type: prog->type); |
28 | if (IS_ERR(ptr: replace_prog)) { |
29 | ret = PTR_ERR(ptr: replace_prog); |
30 | replace_prog = NULL; |
31 | goto out; |
32 | } |
33 | } |
34 | entry = tcx_entry_fetch_or_create(dev, ingress, created: &created); |
35 | if (!entry) { |
36 | ret = -ENOMEM; |
37 | goto out; |
38 | } |
39 | ret = bpf_mprog_attach(entry, entry_new: &entry_new, prog_new: prog, NULL, prog_old: replace_prog, |
40 | flags: attr->attach_flags, id_or_fd: attr->relative_fd, |
41 | revision: attr->expected_revision); |
42 | if (!ret) { |
43 | if (entry != entry_new) { |
44 | tcx_entry_update(dev, entry: entry_new, ingress); |
45 | tcx_entry_sync(); |
46 | tcx_skeys_inc(ingress); |
47 | } |
48 | bpf_mprog_commit(entry); |
49 | } else if (created) { |
50 | tcx_entry_free(entry); |
51 | } |
52 | out: |
53 | if (replace_prog) |
54 | bpf_prog_put(prog: replace_prog); |
55 | rtnl_unlock(); |
56 | return ret; |
57 | } |
58 | |
59 | int tcx_prog_detach(const union bpf_attr *attr, struct bpf_prog *prog) |
60 | { |
61 | bool ingress = attr->attach_type == BPF_TCX_INGRESS; |
62 | struct net *net = current->nsproxy->net_ns; |
63 | struct bpf_mprog_entry *entry, *entry_new; |
64 | struct net_device *dev; |
65 | int ret; |
66 | |
67 | rtnl_lock(); |
68 | dev = __dev_get_by_index(net, ifindex: attr->target_ifindex); |
69 | if (!dev) { |
70 | ret = -ENODEV; |
71 | goto out; |
72 | } |
73 | entry = tcx_entry_fetch(dev, ingress); |
74 | if (!entry) { |
75 | ret = -ENOENT; |
76 | goto out; |
77 | } |
78 | ret = bpf_mprog_detach(entry, entry_new: &entry_new, prog, NULL, flags: attr->attach_flags, |
79 | id_or_fd: attr->relative_fd, revision: attr->expected_revision); |
80 | if (!ret) { |
81 | if (!tcx_entry_is_active(entry: entry_new)) |
82 | entry_new = NULL; |
83 | tcx_entry_update(dev, entry: entry_new, ingress); |
84 | tcx_entry_sync(); |
85 | tcx_skeys_dec(ingress); |
86 | bpf_mprog_commit(entry); |
87 | if (!entry_new) |
88 | tcx_entry_free(entry); |
89 | } |
90 | out: |
91 | rtnl_unlock(); |
92 | return ret; |
93 | } |
94 | |
95 | void tcx_uninstall(struct net_device *dev, bool ingress) |
96 | { |
97 | struct bpf_mprog_entry *entry, *entry_new = NULL; |
98 | struct bpf_tuple tuple = {}; |
99 | struct bpf_mprog_fp *fp; |
100 | struct bpf_mprog_cp *cp; |
101 | bool active; |
102 | |
103 | entry = tcx_entry_fetch(dev, ingress); |
104 | if (!entry) |
105 | return; |
106 | active = tcx_entry(entry)->miniq_active; |
107 | if (active) |
108 | bpf_mprog_clear_all(entry, entry_new: &entry_new); |
109 | tcx_entry_update(dev, entry: entry_new, ingress); |
110 | tcx_entry_sync(); |
111 | bpf_mprog_foreach_tuple(entry, fp, cp, tuple) { |
112 | if (tuple.link) |
113 | tcx_link(link: tuple.link)->dev = NULL; |
114 | else |
115 | bpf_prog_put(prog: tuple.prog); |
116 | tcx_skeys_dec(ingress); |
117 | } |
118 | if (!active) |
119 | tcx_entry_free(entry); |
120 | } |
121 | |
122 | int tcx_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr) |
123 | { |
124 | bool ingress = attr->query.attach_type == BPF_TCX_INGRESS; |
125 | struct net *net = current->nsproxy->net_ns; |
126 | struct net_device *dev; |
127 | int ret; |
128 | |
129 | rtnl_lock(); |
130 | dev = __dev_get_by_index(net, ifindex: attr->query.target_ifindex); |
131 | if (!dev) { |
132 | ret = -ENODEV; |
133 | goto out; |
134 | } |
135 | ret = bpf_mprog_query(attr, uattr, entry: tcx_entry_fetch(dev, ingress)); |
136 | out: |
137 | rtnl_unlock(); |
138 | return ret; |
139 | } |
140 | |
141 | static int tcx_link_prog_attach(struct bpf_link *link, u32 flags, u32 id_or_fd, |
142 | u64 revision) |
143 | { |
144 | struct tcx_link *tcx = tcx_link(link); |
145 | bool created, ingress = tcx->location == BPF_TCX_INGRESS; |
146 | struct bpf_mprog_entry *entry, *entry_new; |
147 | struct net_device *dev = tcx->dev; |
148 | int ret; |
149 | |
150 | ASSERT_RTNL(); |
151 | entry = tcx_entry_fetch_or_create(dev, ingress, created: &created); |
152 | if (!entry) |
153 | return -ENOMEM; |
154 | ret = bpf_mprog_attach(entry, entry_new: &entry_new, prog_new: link->prog, link, NULL, flags, |
155 | id_or_fd, revision); |
156 | if (!ret) { |
157 | if (entry != entry_new) { |
158 | tcx_entry_update(dev, entry: entry_new, ingress); |
159 | tcx_entry_sync(); |
160 | tcx_skeys_inc(ingress); |
161 | } |
162 | bpf_mprog_commit(entry); |
163 | } else if (created) { |
164 | tcx_entry_free(entry); |
165 | } |
166 | return ret; |
167 | } |
168 | |
169 | static void tcx_link_release(struct bpf_link *link) |
170 | { |
171 | struct tcx_link *tcx = tcx_link(link); |
172 | bool ingress = tcx->location == BPF_TCX_INGRESS; |
173 | struct bpf_mprog_entry *entry, *entry_new; |
174 | struct net_device *dev; |
175 | int ret = 0; |
176 | |
177 | rtnl_lock(); |
178 | dev = tcx->dev; |
179 | if (!dev) |
180 | goto out; |
181 | entry = tcx_entry_fetch(dev, ingress); |
182 | if (!entry) { |
183 | ret = -ENOENT; |
184 | goto out; |
185 | } |
186 | ret = bpf_mprog_detach(entry, entry_new: &entry_new, prog: link->prog, link, flags: 0, id_or_fd: 0, revision: 0); |
187 | if (!ret) { |
188 | if (!tcx_entry_is_active(entry: entry_new)) |
189 | entry_new = NULL; |
190 | tcx_entry_update(dev, entry: entry_new, ingress); |
191 | tcx_entry_sync(); |
192 | tcx_skeys_dec(ingress); |
193 | bpf_mprog_commit(entry); |
194 | if (!entry_new) |
195 | tcx_entry_free(entry); |
196 | tcx->dev = NULL; |
197 | } |
198 | out: |
199 | WARN_ON_ONCE(ret); |
200 | rtnl_unlock(); |
201 | } |
202 | |
203 | static int tcx_link_update(struct bpf_link *link, struct bpf_prog *nprog, |
204 | struct bpf_prog *oprog) |
205 | { |
206 | struct tcx_link *tcx = tcx_link(link); |
207 | bool ingress = tcx->location == BPF_TCX_INGRESS; |
208 | struct bpf_mprog_entry *entry, *entry_new; |
209 | struct net_device *dev; |
210 | int ret = 0; |
211 | |
212 | rtnl_lock(); |
213 | dev = tcx->dev; |
214 | if (!dev) { |
215 | ret = -ENOLINK; |
216 | goto out; |
217 | } |
218 | if (oprog && link->prog != oprog) { |
219 | ret = -EPERM; |
220 | goto out; |
221 | } |
222 | oprog = link->prog; |
223 | if (oprog == nprog) { |
224 | bpf_prog_put(prog: nprog); |
225 | goto out; |
226 | } |
227 | entry = tcx_entry_fetch(dev, ingress); |
228 | if (!entry) { |
229 | ret = -ENOENT; |
230 | goto out; |
231 | } |
232 | ret = bpf_mprog_attach(entry, entry_new: &entry_new, prog_new: nprog, link, prog_old: oprog, |
233 | BPF_F_REPLACE | BPF_F_ID, |
234 | id_or_fd: link->prog->aux->id, revision: 0); |
235 | if (!ret) { |
236 | WARN_ON_ONCE(entry != entry_new); |
237 | oprog = xchg(&link->prog, nprog); |
238 | bpf_prog_put(prog: oprog); |
239 | bpf_mprog_commit(entry); |
240 | } |
241 | out: |
242 | rtnl_unlock(); |
243 | return ret; |
244 | } |
245 | |
246 | static void tcx_link_dealloc(struct bpf_link *link) |
247 | { |
248 | kfree(objp: tcx_link(link)); |
249 | } |
250 | |
251 | static void tcx_link_fdinfo(const struct bpf_link *link, struct seq_file *seq) |
252 | { |
253 | const struct tcx_link *tcx = tcx_link(link); |
254 | u32 ifindex = 0; |
255 | |
256 | rtnl_lock(); |
257 | if (tcx->dev) |
258 | ifindex = tcx->dev->ifindex; |
259 | rtnl_unlock(); |
260 | |
261 | seq_printf(m: seq, fmt: "ifindex:\t%u\n" , ifindex); |
262 | seq_printf(m: seq, fmt: "attach_type:\t%u (%s)\n" , |
263 | tcx->location, |
264 | tcx->location == BPF_TCX_INGRESS ? "ingress" : "egress" ); |
265 | } |
266 | |
267 | static int tcx_link_fill_info(const struct bpf_link *link, |
268 | struct bpf_link_info *info) |
269 | { |
270 | const struct tcx_link *tcx = tcx_link(link); |
271 | u32 ifindex = 0; |
272 | |
273 | rtnl_lock(); |
274 | if (tcx->dev) |
275 | ifindex = tcx->dev->ifindex; |
276 | rtnl_unlock(); |
277 | |
278 | info->tcx.ifindex = ifindex; |
279 | info->tcx.attach_type = tcx->location; |
280 | return 0; |
281 | } |
282 | |
283 | static int tcx_link_detach(struct bpf_link *link) |
284 | { |
285 | tcx_link_release(link); |
286 | return 0; |
287 | } |
288 | |
289 | static const struct bpf_link_ops tcx_link_lops = { |
290 | .release = tcx_link_release, |
291 | .detach = tcx_link_detach, |
292 | .dealloc = tcx_link_dealloc, |
293 | .update_prog = tcx_link_update, |
294 | .show_fdinfo = tcx_link_fdinfo, |
295 | .fill_link_info = tcx_link_fill_info, |
296 | }; |
297 | |
298 | static int tcx_link_init(struct tcx_link *tcx, |
299 | struct bpf_link_primer *link_primer, |
300 | const union bpf_attr *attr, |
301 | struct net_device *dev, |
302 | struct bpf_prog *prog) |
303 | { |
304 | bpf_link_init(link: &tcx->link, type: BPF_LINK_TYPE_TCX, ops: &tcx_link_lops, prog); |
305 | tcx->location = attr->link_create.attach_type; |
306 | tcx->dev = dev; |
307 | return bpf_link_prime(link: &tcx->link, primer: link_primer); |
308 | } |
309 | |
310 | int tcx_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) |
311 | { |
312 | struct net *net = current->nsproxy->net_ns; |
313 | struct bpf_link_primer link_primer; |
314 | struct net_device *dev; |
315 | struct tcx_link *tcx; |
316 | int ret; |
317 | |
318 | rtnl_lock(); |
319 | dev = __dev_get_by_index(net, ifindex: attr->link_create.target_ifindex); |
320 | if (!dev) { |
321 | ret = -ENODEV; |
322 | goto out; |
323 | } |
324 | tcx = kzalloc(size: sizeof(*tcx), GFP_USER); |
325 | if (!tcx) { |
326 | ret = -ENOMEM; |
327 | goto out; |
328 | } |
329 | ret = tcx_link_init(tcx, link_primer: &link_primer, attr, dev, prog); |
330 | if (ret) { |
331 | kfree(objp: tcx); |
332 | goto out; |
333 | } |
334 | ret = tcx_link_prog_attach(link: &tcx->link, flags: attr->link_create.flags, |
335 | id_or_fd: attr->link_create.tcx.relative_fd, |
336 | revision: attr->link_create.tcx.expected_revision); |
337 | if (ret) { |
338 | tcx->dev = NULL; |
339 | bpf_link_cleanup(primer: &link_primer); |
340 | goto out; |
341 | } |
342 | ret = bpf_link_settle(primer: &link_primer); |
343 | out: |
344 | rtnl_unlock(); |
345 | return ret; |
346 | } |
347 | |