1 | // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) |
2 | /* Copyright (C) 2017-2018 Netronome Systems, Inc. */ |
3 | |
4 | #include <net/pkt_cls.h> |
5 | |
6 | #include "../nfpcore/nfp_cpp.h" |
7 | #include "../nfpcore/nfp_nffw.h" |
8 | #include "../nfpcore/nfp_nsp.h" |
9 | #include "../nfp_app.h" |
10 | #include "../nfp_main.h" |
11 | #include "../nfp_net.h" |
12 | #include "../nfp_port.h" |
13 | #include "fw.h" |
14 | #include "main.h" |
15 | |
16 | const struct rhashtable_params nfp_bpf_maps_neutral_params = { |
17 | .nelem_hint = 4, |
18 | .key_len = sizeof_field(struct bpf_map, id), |
19 | .key_offset = offsetof(struct nfp_bpf_neutral_map, map_id), |
20 | .head_offset = offsetof(struct nfp_bpf_neutral_map, l), |
21 | .automatic_shrinking = true, |
22 | }; |
23 | |
24 | static bool nfp_net_ebpf_capable(struct nfp_net *nn) |
25 | { |
26 | #ifdef __LITTLE_ENDIAN |
27 | struct nfp_app_bpf *bpf = nn->app->priv; |
28 | |
29 | return nn->cap & NFP_NET_CFG_CTRL_BPF && |
30 | bpf->abi_version && |
31 | nn_readb(nn, NFP_NET_CFG_BPF_ABI) == bpf->abi_version; |
32 | #else |
33 | return false; |
34 | #endif |
35 | } |
36 | |
37 | static int |
38 | nfp_bpf_xdp_offload(struct nfp_app *app, struct nfp_net *nn, |
39 | struct bpf_prog *prog, struct netlink_ext_ack *extack) |
40 | { |
41 | bool running, xdp_running; |
42 | |
43 | if (!nfp_net_ebpf_capable(nn)) |
44 | return -EINVAL; |
45 | |
46 | running = nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF; |
47 | xdp_running = running && nn->xdp_hw.prog; |
48 | |
49 | if (!prog && !xdp_running) |
50 | return 0; |
51 | if (prog && running && !xdp_running) |
52 | return -EBUSY; |
53 | |
54 | return nfp_net_bpf_offload(nn, prog, old_prog: running, extack); |
55 | } |
56 | |
57 | static const char *(struct nfp_app *app, struct nfp_net *nn) |
58 | { |
59 | return nfp_net_ebpf_capable(nn) ? "BPF" : "" ; |
60 | } |
61 | |
62 | static int |
63 | nfp_bpf_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id) |
64 | { |
65 | struct nfp_pf *pf = app->pf; |
66 | struct nfp_bpf_vnic *bv; |
67 | int err; |
68 | |
69 | if (!pf->eth_tbl) { |
70 | nfp_err(pf->cpp, "No ETH table\n" ); |
71 | return -EINVAL; |
72 | } |
73 | if (pf->max_data_vnics != pf->eth_tbl->count) { |
74 | nfp_err(pf->cpp, "ETH entries don't match vNICs (%d vs %d)\n" , |
75 | pf->max_data_vnics, pf->eth_tbl->count); |
76 | return -EINVAL; |
77 | } |
78 | |
79 | bv = kzalloc(size: sizeof(*bv), GFP_KERNEL); |
80 | if (!bv) |
81 | return -ENOMEM; |
82 | nn->app_priv = bv; |
83 | |
84 | err = nfp_app_nic_vnic_alloc(app, nn, id); |
85 | if (err) |
86 | goto err_free_priv; |
87 | |
88 | bv->start_off = nn_readw(nn, NFP_NET_CFG_BPF_START); |
89 | bv->tgt_done = nn_readw(nn, NFP_NET_CFG_BPF_DONE); |
90 | |
91 | return 0; |
92 | err_free_priv: |
93 | kfree(objp: nn->app_priv); |
94 | return err; |
95 | } |
96 | |
97 | static void nfp_bpf_vnic_free(struct nfp_app *app, struct nfp_net *nn) |
98 | { |
99 | struct nfp_bpf_vnic *bv = nn->app_priv; |
100 | |
101 | WARN_ON(bv->tc_prog); |
102 | kfree(objp: bv); |
103 | } |
104 | |
105 | static int nfp_bpf_setup_tc_block_cb(enum tc_setup_type type, |
106 | void *type_data, void *cb_priv) |
107 | { |
108 | struct tc_cls_bpf_offload *cls_bpf = type_data; |
109 | struct nfp_net *nn = cb_priv; |
110 | struct bpf_prog *oldprog; |
111 | struct nfp_bpf_vnic *bv; |
112 | int err; |
113 | |
114 | if (type != TC_SETUP_CLSBPF) { |
115 | NL_SET_ERR_MSG_MOD(cls_bpf->common.extack, |
116 | "only offload of BPF classifiers supported" ); |
117 | return -EOPNOTSUPP; |
118 | } |
119 | if (!tc_cls_can_offload_and_chain0(dev: nn->dp.netdev, common: &cls_bpf->common)) |
120 | return -EOPNOTSUPP; |
121 | if (!nfp_net_ebpf_capable(nn)) { |
122 | NL_SET_ERR_MSG_MOD(cls_bpf->common.extack, |
123 | "NFP firmware does not support eBPF offload" ); |
124 | return -EOPNOTSUPP; |
125 | } |
126 | if (cls_bpf->common.protocol != htons(ETH_P_ALL)) { |
127 | NL_SET_ERR_MSG_MOD(cls_bpf->common.extack, |
128 | "only ETH_P_ALL supported as filter protocol" ); |
129 | return -EOPNOTSUPP; |
130 | } |
131 | |
132 | /* Only support TC direct action */ |
133 | if (!cls_bpf->exts_integrated || |
134 | tcf_exts_has_actions(exts: cls_bpf->exts)) { |
135 | NL_SET_ERR_MSG_MOD(cls_bpf->common.extack, |
136 | "only direct action with no legacy actions supported" ); |
137 | return -EOPNOTSUPP; |
138 | } |
139 | |
140 | if (cls_bpf->command != TC_CLSBPF_OFFLOAD) |
141 | return -EOPNOTSUPP; |
142 | |
143 | bv = nn->app_priv; |
144 | oldprog = cls_bpf->oldprog; |
145 | |
146 | /* Don't remove if oldprog doesn't match driver's state */ |
147 | if (bv->tc_prog != oldprog) { |
148 | oldprog = NULL; |
149 | if (!cls_bpf->prog) |
150 | return 0; |
151 | } |
152 | |
153 | err = nfp_net_bpf_offload(nn, prog: cls_bpf->prog, old_prog: oldprog, |
154 | extack: cls_bpf->common.extack); |
155 | if (err) |
156 | return err; |
157 | |
158 | bv->tc_prog = cls_bpf->prog; |
159 | nn->port->tc_offload_cnt = !!bv->tc_prog; |
160 | return 0; |
161 | } |
162 | |
163 | static LIST_HEAD(nfp_bpf_block_cb_list); |
164 | |
165 | static int nfp_bpf_setup_tc(struct nfp_app *app, struct net_device *netdev, |
166 | enum tc_setup_type type, void *type_data) |
167 | { |
168 | struct nfp_net *nn = netdev_priv(dev: netdev); |
169 | |
170 | switch (type) { |
171 | case TC_SETUP_BLOCK: |
172 | return flow_block_cb_setup_simple(f: type_data, |
173 | driver_list: &nfp_bpf_block_cb_list, |
174 | cb: nfp_bpf_setup_tc_block_cb, |
175 | cb_ident: nn, cb_priv: nn, ingress_only: true); |
176 | default: |
177 | return -EOPNOTSUPP; |
178 | } |
179 | } |
180 | |
181 | static int |
182 | nfp_bpf_check_mtu(struct nfp_app *app, struct net_device *netdev, int new_mtu) |
183 | { |
184 | struct nfp_net *nn = netdev_priv(dev: netdev); |
185 | struct nfp_bpf_vnic *bv; |
186 | struct bpf_prog *prog; |
187 | |
188 | if (~nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF) |
189 | return 0; |
190 | |
191 | if (nn->xdp_hw.prog) { |
192 | prog = nn->xdp_hw.prog; |
193 | } else { |
194 | bv = nn->app_priv; |
195 | prog = bv->tc_prog; |
196 | } |
197 | |
198 | if (nfp_bpf_offload_check_mtu(nn, prog, mtu: new_mtu)) { |
199 | nn_info(nn, "BPF offload active, potential packet access beyond hardware packet boundary" ); |
200 | return -EBUSY; |
201 | } |
202 | return 0; |
203 | } |
204 | |
205 | static int |
206 | nfp_bpf_parse_cap_adjust_head(struct nfp_app_bpf *bpf, void __iomem *value, |
207 | u32 length) |
208 | { |
209 | struct nfp_bpf_cap_tlv_adjust_head __iomem *cap = value; |
210 | struct nfp_cpp *cpp = bpf->app->pf->cpp; |
211 | |
212 | if (length < sizeof(*cap)) { |
213 | nfp_err(cpp, "truncated adjust_head TLV: %d\n" , length); |
214 | return -EINVAL; |
215 | } |
216 | |
217 | bpf->adjust_head.flags = readl(addr: &cap->flags); |
218 | bpf->adjust_head.off_min = readl(addr: &cap->off_min); |
219 | bpf->adjust_head.off_max = readl(addr: &cap->off_max); |
220 | bpf->adjust_head.guaranteed_sub = readl(addr: &cap->guaranteed_sub); |
221 | bpf->adjust_head.guaranteed_add = readl(addr: &cap->guaranteed_add); |
222 | |
223 | if (bpf->adjust_head.off_min > bpf->adjust_head.off_max) { |
224 | nfp_err(cpp, "invalid adjust_head TLV: min > max\n" ); |
225 | return -EINVAL; |
226 | } |
227 | if (!FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_min) || |
228 | !FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_max)) { |
229 | nfp_warn(cpp, "disabling adjust_head - driver expects min/max to fit in as immediates\n" ); |
230 | memset(&bpf->adjust_head, 0, sizeof(bpf->adjust_head)); |
231 | return 0; |
232 | } |
233 | |
234 | return 0; |
235 | } |
236 | |
237 | static int |
238 | nfp_bpf_parse_cap_func(struct nfp_app_bpf *bpf, void __iomem *value, u32 length) |
239 | { |
240 | struct nfp_bpf_cap_tlv_func __iomem *cap = value; |
241 | |
242 | if (length < sizeof(*cap)) { |
243 | nfp_err(bpf->app->cpp, "truncated function TLV: %d\n" , length); |
244 | return -EINVAL; |
245 | } |
246 | |
247 | switch (readl(addr: &cap->func_id)) { |
248 | case BPF_FUNC_map_lookup_elem: |
249 | bpf->helpers.map_lookup = readl(addr: &cap->func_addr); |
250 | break; |
251 | case BPF_FUNC_map_update_elem: |
252 | bpf->helpers.map_update = readl(addr: &cap->func_addr); |
253 | break; |
254 | case BPF_FUNC_map_delete_elem: |
255 | bpf->helpers.map_delete = readl(addr: &cap->func_addr); |
256 | break; |
257 | case BPF_FUNC_perf_event_output: |
258 | bpf->helpers.perf_event_output = readl(addr: &cap->func_addr); |
259 | break; |
260 | } |
261 | |
262 | return 0; |
263 | } |
264 | |
265 | static int |
266 | nfp_bpf_parse_cap_maps(struct nfp_app_bpf *bpf, void __iomem *value, u32 length) |
267 | { |
268 | struct nfp_bpf_cap_tlv_maps __iomem *cap = value; |
269 | |
270 | if (length < sizeof(*cap)) { |
271 | nfp_err(bpf->app->cpp, "truncated maps TLV: %d\n" , length); |
272 | return -EINVAL; |
273 | } |
274 | |
275 | bpf->maps.types = readl(addr: &cap->types); |
276 | bpf->maps.max_maps = readl(addr: &cap->max_maps); |
277 | bpf->maps.max_elems = readl(addr: &cap->max_elems); |
278 | bpf->maps.max_key_sz = readl(addr: &cap->max_key_sz); |
279 | bpf->maps.max_val_sz = readl(addr: &cap->max_val_sz); |
280 | bpf->maps.max_elem_sz = readl(addr: &cap->max_elem_sz); |
281 | |
282 | return 0; |
283 | } |
284 | |
285 | static int |
286 | nfp_bpf_parse_cap_random(struct nfp_app_bpf *bpf, void __iomem *value, |
287 | u32 length) |
288 | { |
289 | bpf->pseudo_random = true; |
290 | return 0; |
291 | } |
292 | |
293 | static int |
294 | nfp_bpf_parse_cap_qsel(struct nfp_app_bpf *bpf, void __iomem *value, u32 length) |
295 | { |
296 | bpf->queue_select = true; |
297 | return 0; |
298 | } |
299 | |
300 | static int |
301 | nfp_bpf_parse_cap_adjust_tail(struct nfp_app_bpf *bpf, void __iomem *value, |
302 | u32 length) |
303 | { |
304 | bpf->adjust_tail = true; |
305 | return 0; |
306 | } |
307 | |
308 | static int |
309 | nfp_bpf_parse_cap_cmsg_multi_ent(struct nfp_app_bpf *bpf, void __iomem *value, |
310 | u32 length) |
311 | { |
312 | bpf->cmsg_multi_ent = true; |
313 | return 0; |
314 | } |
315 | |
316 | static int |
317 | nfp_bpf_parse_cap_abi_version(struct nfp_app_bpf *bpf, void __iomem *value, |
318 | u32 length) |
319 | { |
320 | if (length < 4) { |
321 | nfp_err(bpf->app->cpp, "truncated ABI version TLV: %d\n" , |
322 | length); |
323 | return -EINVAL; |
324 | } |
325 | |
326 | bpf->abi_version = readl(addr: value); |
327 | if (bpf->abi_version < 2 || bpf->abi_version > 3) { |
328 | nfp_warn(bpf->app->cpp, "unsupported BPF ABI version: %d\n" , |
329 | bpf->abi_version); |
330 | bpf->abi_version = 0; |
331 | } |
332 | |
333 | return 0; |
334 | } |
335 | |
336 | static int nfp_bpf_parse_capabilities(struct nfp_app *app) |
337 | { |
338 | struct nfp_cpp *cpp = app->pf->cpp; |
339 | struct nfp_cpp_area *area; |
340 | u8 __iomem *mem, *start; |
341 | |
342 | mem = nfp_rtsym_map(rtbl: app->pf->rtbl, name: "_abi_bpf_capabilities" , id: "bpf.cap" , |
343 | min_size: 8, area: &area); |
344 | if (IS_ERR(ptr: mem)) |
345 | return PTR_ERR(ptr: mem) == -ENOENT ? 0 : PTR_ERR(ptr: mem); |
346 | |
347 | start = mem; |
348 | while (mem - start + 8 <= nfp_cpp_area_size(area)) { |
349 | u8 __iomem *value; |
350 | u32 type, length; |
351 | |
352 | type = readl(addr: mem); |
353 | length = readl(addr: mem + 4); |
354 | value = mem + 8; |
355 | |
356 | mem += 8 + length; |
357 | if (mem - start > nfp_cpp_area_size(area)) |
358 | goto err_release_free; |
359 | |
360 | switch (type) { |
361 | case NFP_BPF_CAP_TYPE_FUNC: |
362 | if (nfp_bpf_parse_cap_func(bpf: app->priv, value, length)) |
363 | goto err_release_free; |
364 | break; |
365 | case NFP_BPF_CAP_TYPE_ADJUST_HEAD: |
366 | if (nfp_bpf_parse_cap_adjust_head(bpf: app->priv, value, |
367 | length)) |
368 | goto err_release_free; |
369 | break; |
370 | case NFP_BPF_CAP_TYPE_MAPS: |
371 | if (nfp_bpf_parse_cap_maps(bpf: app->priv, value, length)) |
372 | goto err_release_free; |
373 | break; |
374 | case NFP_BPF_CAP_TYPE_RANDOM: |
375 | if (nfp_bpf_parse_cap_random(bpf: app->priv, value, length)) |
376 | goto err_release_free; |
377 | break; |
378 | case NFP_BPF_CAP_TYPE_QUEUE_SELECT: |
379 | if (nfp_bpf_parse_cap_qsel(bpf: app->priv, value, length)) |
380 | goto err_release_free; |
381 | break; |
382 | case NFP_BPF_CAP_TYPE_ADJUST_TAIL: |
383 | if (nfp_bpf_parse_cap_adjust_tail(bpf: app->priv, value, |
384 | length)) |
385 | goto err_release_free; |
386 | break; |
387 | case NFP_BPF_CAP_TYPE_ABI_VERSION: |
388 | if (nfp_bpf_parse_cap_abi_version(bpf: app->priv, value, |
389 | length)) |
390 | goto err_release_free; |
391 | break; |
392 | case NFP_BPF_CAP_TYPE_CMSG_MULTI_ENT: |
393 | if (nfp_bpf_parse_cap_cmsg_multi_ent(bpf: app->priv, value, |
394 | length)) |
395 | goto err_release_free; |
396 | break; |
397 | default: |
398 | nfp_dbg(cpp, "unknown BPF capability: %d\n" , type); |
399 | break; |
400 | } |
401 | } |
402 | if (mem - start != nfp_cpp_area_size(area)) { |
403 | nfp_err(cpp, "BPF capabilities left after parsing, parsed:%zd total length:%zu\n" , |
404 | mem - start, nfp_cpp_area_size(area)); |
405 | goto err_release_free; |
406 | } |
407 | |
408 | nfp_cpp_area_release_free(area); |
409 | |
410 | return 0; |
411 | |
412 | err_release_free: |
413 | nfp_err(cpp, "invalid BPF capabilities at offset:%zd\n" , mem - start); |
414 | nfp_cpp_area_release_free(area); |
415 | return -EINVAL; |
416 | } |
417 | |
418 | static void nfp_bpf_init_capabilities(struct nfp_app_bpf *bpf) |
419 | { |
420 | bpf->abi_version = 2; /* Original BPF ABI version */ |
421 | } |
422 | |
423 | static int nfp_bpf_ndo_init(struct nfp_app *app, struct net_device *netdev) |
424 | { |
425 | struct nfp_app_bpf *bpf = app->priv; |
426 | |
427 | return bpf_offload_dev_netdev_register(offdev: bpf->bpf_dev, netdev); |
428 | } |
429 | |
430 | static void nfp_bpf_ndo_uninit(struct nfp_app *app, struct net_device *netdev) |
431 | { |
432 | struct nfp_app_bpf *bpf = app->priv; |
433 | |
434 | bpf_offload_dev_netdev_unregister(offdev: bpf->bpf_dev, netdev); |
435 | } |
436 | |
437 | static int nfp_bpf_start(struct nfp_app *app) |
438 | { |
439 | struct nfp_app_bpf *bpf = app->priv; |
440 | |
441 | if (app->ctrl->dp.mtu < nfp_bpf_ctrl_cmsg_min_mtu(bpf)) { |
442 | nfp_err(bpf->app->cpp, |
443 | "ctrl channel MTU below min required %u < %u\n" , |
444 | app->ctrl->dp.mtu, nfp_bpf_ctrl_cmsg_min_mtu(bpf)); |
445 | return -EINVAL; |
446 | } |
447 | |
448 | if (bpf->cmsg_multi_ent) |
449 | bpf->cmsg_cache_cnt = nfp_bpf_ctrl_cmsg_cache_cnt(bpf); |
450 | else |
451 | bpf->cmsg_cache_cnt = 1; |
452 | |
453 | return 0; |
454 | } |
455 | |
456 | static int nfp_bpf_init(struct nfp_app *app) |
457 | { |
458 | struct nfp_app_bpf *bpf; |
459 | int err; |
460 | |
461 | bpf = kzalloc(size: sizeof(*bpf), GFP_KERNEL); |
462 | if (!bpf) |
463 | return -ENOMEM; |
464 | bpf->app = app; |
465 | app->priv = bpf; |
466 | |
467 | INIT_LIST_HEAD(list: &bpf->map_list); |
468 | |
469 | err = nfp_ccm_init(ccm: &bpf->ccm, app); |
470 | if (err) |
471 | goto err_free_bpf; |
472 | |
473 | err = rhashtable_init(ht: &bpf->maps_neutral, params: &nfp_bpf_maps_neutral_params); |
474 | if (err) |
475 | goto err_clean_ccm; |
476 | |
477 | nfp_bpf_init_capabilities(bpf); |
478 | |
479 | err = nfp_bpf_parse_capabilities(app); |
480 | if (err) |
481 | goto err_free_neutral_maps; |
482 | |
483 | if (bpf->abi_version < 3) { |
484 | bpf->cmsg_key_sz = CMSG_MAP_KEY_LW * 4; |
485 | bpf->cmsg_val_sz = CMSG_MAP_VALUE_LW * 4; |
486 | } else { |
487 | bpf->cmsg_key_sz = bpf->maps.max_key_sz; |
488 | bpf->cmsg_val_sz = bpf->maps.max_val_sz; |
489 | app->ctrl_mtu = nfp_bpf_ctrl_cmsg_mtu(bpf); |
490 | } |
491 | |
492 | bpf->bpf_dev = bpf_offload_dev_create(ops: &nfp_bpf_dev_ops, priv: bpf); |
493 | err = PTR_ERR_OR_ZERO(ptr: bpf->bpf_dev); |
494 | if (err) |
495 | goto err_free_neutral_maps; |
496 | |
497 | return 0; |
498 | |
499 | err_free_neutral_maps: |
500 | rhashtable_destroy(ht: &bpf->maps_neutral); |
501 | err_clean_ccm: |
502 | nfp_ccm_clean(ccm: &bpf->ccm); |
503 | err_free_bpf: |
504 | kfree(objp: bpf); |
505 | return err; |
506 | } |
507 | |
508 | static void nfp_bpf_clean(struct nfp_app *app) |
509 | { |
510 | struct nfp_app_bpf *bpf = app->priv; |
511 | |
512 | bpf_offload_dev_destroy(offdev: bpf->bpf_dev); |
513 | nfp_ccm_clean(ccm: &bpf->ccm); |
514 | WARN_ON(!list_empty(&bpf->map_list)); |
515 | WARN_ON(bpf->maps_in_use || bpf->map_elems_in_use); |
516 | rhashtable_free_and_destroy(ht: &bpf->maps_neutral, |
517 | free_fn: nfp_check_rhashtable_empty, NULL); |
518 | kfree(objp: bpf); |
519 | } |
520 | |
521 | const struct nfp_app_type app_bpf = { |
522 | .id = NFP_APP_BPF_NIC, |
523 | .name = "ebpf" , |
524 | |
525 | .ctrl_cap_mask = 0, |
526 | |
527 | .init = nfp_bpf_init, |
528 | .clean = nfp_bpf_clean, |
529 | .start = nfp_bpf_start, |
530 | |
531 | .check_mtu = nfp_bpf_check_mtu, |
532 | |
533 | .extra_cap = nfp_bpf_extra_cap, |
534 | |
535 | .ndo_init = nfp_bpf_ndo_init, |
536 | .ndo_uninit = nfp_bpf_ndo_uninit, |
537 | |
538 | .vnic_alloc = nfp_bpf_vnic_alloc, |
539 | .vnic_free = nfp_bpf_vnic_free, |
540 | |
541 | .ctrl_msg_rx = nfp_bpf_ctrl_msg_rx, |
542 | .ctrl_msg_rx_raw = nfp_bpf_ctrl_msg_rx_raw, |
543 | |
544 | .setup_tc = nfp_bpf_setup_tc, |
545 | .bpf = nfp_ndo_bpf, |
546 | .xdp_offload = nfp_bpf_xdp_offload, |
547 | }; |
548 | |