1 | // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) |
2 | /* Copyright (C) 2016-2018 Netronome Systems, Inc. */ |
3 | |
4 | /* |
5 | * nfp_net_offload.c |
6 | * Netronome network device driver: TC offload functions for PF and VF |
7 | */ |
8 | |
9 | #define pr_fmt(fmt) "NFP net bpf: " fmt |
10 | |
11 | #include <linux/bpf.h> |
12 | #include <linux/kernel.h> |
13 | #include <linux/netdevice.h> |
14 | #include <linux/pci.h> |
15 | #include <linux/jiffies.h> |
16 | #include <linux/timer.h> |
17 | #include <linux/list.h> |
18 | #include <linux/mm.h> |
19 | |
20 | #include <net/pkt_cls.h> |
21 | #include <net/tc_act/tc_gact.h> |
22 | #include <net/tc_act/tc_mirred.h> |
23 | |
24 | #include "main.h" |
25 | #include "../ccm.h" |
26 | #include "../nfp_app.h" |
27 | #include "../nfp_net_ctrl.h" |
28 | #include "../nfp_net.h" |
29 | |
30 | static int |
31 | nfp_map_ptr_record(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog, |
32 | struct bpf_map *map) |
33 | { |
34 | struct nfp_bpf_neutral_map *record; |
35 | int err; |
36 | |
37 | /* Reuse path - other offloaded program is already tracking this map. */ |
38 | record = rhashtable_lookup_fast(ht: &bpf->maps_neutral, key: &map->id, |
39 | params: nfp_bpf_maps_neutral_params); |
40 | if (record) { |
41 | nfp_prog->map_records[nfp_prog->map_records_cnt++] = record; |
42 | record->count++; |
43 | return 0; |
44 | } |
45 | |
46 | /* Grab a single ref to the map for our record. The prog destroy ndo |
47 | * happens after free_used_maps(). |
48 | */ |
49 | bpf_map_inc(map); |
50 | |
51 | record = kmalloc(size: sizeof(*record), GFP_KERNEL); |
52 | if (!record) { |
53 | err = -ENOMEM; |
54 | goto err_map_put; |
55 | } |
56 | |
57 | record->ptr = map; |
58 | record->map_id = map->id; |
59 | record->count = 1; |
60 | |
61 | err = rhashtable_insert_fast(ht: &bpf->maps_neutral, obj: &record->l, |
62 | params: nfp_bpf_maps_neutral_params); |
63 | if (err) |
64 | goto err_free_rec; |
65 | |
66 | nfp_prog->map_records[nfp_prog->map_records_cnt++] = record; |
67 | |
68 | return 0; |
69 | |
70 | err_free_rec: |
71 | kfree(objp: record); |
72 | err_map_put: |
73 | bpf_map_put(map); |
74 | return err; |
75 | } |
76 | |
77 | static void |
78 | nfp_map_ptrs_forget(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog) |
79 | { |
80 | bool freed = false; |
81 | int i; |
82 | |
83 | for (i = 0; i < nfp_prog->map_records_cnt; i++) { |
84 | if (--nfp_prog->map_records[i]->count) { |
85 | nfp_prog->map_records[i] = NULL; |
86 | continue; |
87 | } |
88 | |
89 | WARN_ON(rhashtable_remove_fast(&bpf->maps_neutral, |
90 | &nfp_prog->map_records[i]->l, |
91 | nfp_bpf_maps_neutral_params)); |
92 | freed = true; |
93 | } |
94 | |
95 | if (freed) { |
96 | synchronize_rcu(); |
97 | |
98 | for (i = 0; i < nfp_prog->map_records_cnt; i++) |
99 | if (nfp_prog->map_records[i]) { |
100 | bpf_map_put(map: nfp_prog->map_records[i]->ptr); |
101 | kfree(objp: nfp_prog->map_records[i]); |
102 | } |
103 | } |
104 | |
105 | kfree(objp: nfp_prog->map_records); |
106 | nfp_prog->map_records = NULL; |
107 | nfp_prog->map_records_cnt = 0; |
108 | } |
109 | |
110 | static int |
111 | nfp_map_ptrs_record(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog, |
112 | struct bpf_prog *prog) |
113 | { |
114 | int i, cnt, err = 0; |
115 | |
116 | mutex_lock(&prog->aux->used_maps_mutex); |
117 | |
118 | /* Quickly count the maps we will have to remember */ |
119 | cnt = 0; |
120 | for (i = 0; i < prog->aux->used_map_cnt; i++) |
121 | if (bpf_map_offload_neutral(map: prog->aux->used_maps[i])) |
122 | cnt++; |
123 | if (!cnt) |
124 | goto out; |
125 | |
126 | nfp_prog->map_records = kmalloc_array(n: cnt, |
127 | size: sizeof(nfp_prog->map_records[0]), |
128 | GFP_KERNEL); |
129 | if (!nfp_prog->map_records) { |
130 | err = -ENOMEM; |
131 | goto out; |
132 | } |
133 | |
134 | for (i = 0; i < prog->aux->used_map_cnt; i++) |
135 | if (bpf_map_offload_neutral(map: prog->aux->used_maps[i])) { |
136 | err = nfp_map_ptr_record(bpf, nfp_prog, |
137 | map: prog->aux->used_maps[i]); |
138 | if (err) { |
139 | nfp_map_ptrs_forget(bpf, nfp_prog); |
140 | goto out; |
141 | } |
142 | } |
143 | WARN_ON(cnt != nfp_prog->map_records_cnt); |
144 | |
145 | out: |
146 | mutex_unlock(lock: &prog->aux->used_maps_mutex); |
147 | return err; |
148 | } |
149 | |
150 | static int |
151 | nfp_prog_prepare(struct nfp_prog *nfp_prog, const struct bpf_insn *prog, |
152 | unsigned int cnt) |
153 | { |
154 | struct nfp_insn_meta *meta; |
155 | unsigned int i; |
156 | |
157 | for (i = 0; i < cnt; i++) { |
158 | meta = kzalloc(size: sizeof(*meta), GFP_KERNEL); |
159 | if (!meta) |
160 | return -ENOMEM; |
161 | |
162 | meta->insn = prog[i]; |
163 | meta->n = i; |
164 | if (is_mbpf_alu(meta)) { |
165 | meta->umin_src = U64_MAX; |
166 | meta->umin_dst = U64_MAX; |
167 | } |
168 | |
169 | list_add_tail(new: &meta->l, head: &nfp_prog->insns); |
170 | } |
171 | nfp_prog->n_insns = cnt; |
172 | |
173 | nfp_bpf_jit_prepare(nfp_prog); |
174 | |
175 | return 0; |
176 | } |
177 | |
178 | static void nfp_prog_free(struct nfp_prog *nfp_prog) |
179 | { |
180 | struct nfp_insn_meta *meta, *tmp; |
181 | |
182 | kfree(objp: nfp_prog->subprog); |
183 | |
184 | list_for_each_entry_safe(meta, tmp, &nfp_prog->insns, l) { |
185 | list_del(entry: &meta->l); |
186 | kfree(objp: meta); |
187 | } |
188 | kfree(objp: nfp_prog); |
189 | } |
190 | |
191 | static int nfp_bpf_verifier_prep(struct bpf_prog *prog) |
192 | { |
193 | struct nfp_prog *nfp_prog; |
194 | int ret; |
195 | |
196 | nfp_prog = kzalloc(size: sizeof(*nfp_prog), GFP_KERNEL); |
197 | if (!nfp_prog) |
198 | return -ENOMEM; |
199 | prog->aux->offload->dev_priv = nfp_prog; |
200 | |
201 | INIT_LIST_HEAD(list: &nfp_prog->insns); |
202 | nfp_prog->type = prog->type; |
203 | nfp_prog->bpf = bpf_offload_dev_priv(offdev: prog->aux->offload->offdev); |
204 | |
205 | ret = nfp_prog_prepare(nfp_prog, prog: prog->insnsi, cnt: prog->len); |
206 | if (ret) |
207 | goto err_free; |
208 | |
209 | nfp_prog->verifier_meta = nfp_prog_first_meta(nfp_prog); |
210 | |
211 | return 0; |
212 | |
213 | err_free: |
214 | nfp_prog_free(nfp_prog); |
215 | |
216 | return ret; |
217 | } |
218 | |
219 | static int nfp_bpf_translate(struct bpf_prog *prog) |
220 | { |
221 | struct nfp_net *nn = netdev_priv(dev: prog->aux->offload->netdev); |
222 | struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv; |
223 | unsigned int max_instr; |
224 | int err; |
225 | |
226 | /* We depend on dead code elimination succeeding */ |
227 | if (prog->aux->offload->opt_failed) |
228 | return -EINVAL; |
229 | |
230 | max_instr = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN); |
231 | nfp_prog->__prog_alloc_len = max_instr * sizeof(u64); |
232 | |
233 | nfp_prog->prog = kvmalloc(size: nfp_prog->__prog_alloc_len, GFP_KERNEL); |
234 | if (!nfp_prog->prog) |
235 | return -ENOMEM; |
236 | |
237 | err = nfp_bpf_jit(prog: nfp_prog); |
238 | if (err) |
239 | return err; |
240 | |
241 | prog->aux->offload->jited_len = nfp_prog->prog_len * sizeof(u64); |
242 | prog->aux->offload->jited_image = nfp_prog->prog; |
243 | |
244 | return nfp_map_ptrs_record(bpf: nfp_prog->bpf, nfp_prog, prog); |
245 | } |
246 | |
247 | static void nfp_bpf_destroy(struct bpf_prog *prog) |
248 | { |
249 | struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv; |
250 | |
251 | kvfree(addr: nfp_prog->prog); |
252 | nfp_map_ptrs_forget(bpf: nfp_prog->bpf, nfp_prog); |
253 | nfp_prog_free(nfp_prog); |
254 | } |
255 | |
256 | /* Atomic engine requires values to be in big endian, we need to byte swap |
257 | * the value words used with xadd. |
258 | */ |
259 | static void nfp_map_bpf_byte_swap(struct nfp_bpf_map *nfp_map, void *value) |
260 | { |
261 | u32 *word = value; |
262 | unsigned int i; |
263 | |
264 | for (i = 0; i < DIV_ROUND_UP(nfp_map->offmap->map.value_size, 4); i++) |
265 | if (nfp_map->use_map[i].type == NFP_MAP_USE_ATOMIC_CNT) |
266 | word[i] = (__force u32)cpu_to_be32(word[i]); |
267 | } |
268 | |
269 | /* Mark value as unsafely initialized in case it becomes atomic later |
270 | * and we didn't byte swap something non-byte swap neutral. |
271 | */ |
272 | static void |
273 | nfp_map_bpf_byte_swap_record(struct nfp_bpf_map *nfp_map, void *value) |
274 | { |
275 | u32 *word = value; |
276 | unsigned int i; |
277 | |
278 | for (i = 0; i < DIV_ROUND_UP(nfp_map->offmap->map.value_size, 4); i++) |
279 | if (nfp_map->use_map[i].type == NFP_MAP_UNUSED && |
280 | word[i] != (__force u32)cpu_to_be32(word[i])) |
281 | nfp_map->use_map[i].non_zero_update = 1; |
282 | } |
283 | |
284 | static int |
285 | nfp_bpf_map_lookup_entry(struct bpf_offloaded_map *offmap, |
286 | void *key, void *value) |
287 | { |
288 | int err; |
289 | |
290 | err = nfp_bpf_ctrl_lookup_entry(offmap, key, value); |
291 | if (err) |
292 | return err; |
293 | |
294 | nfp_map_bpf_byte_swap(nfp_map: offmap->dev_priv, value); |
295 | return 0; |
296 | } |
297 | |
298 | static int |
299 | nfp_bpf_map_update_entry(struct bpf_offloaded_map *offmap, |
300 | void *key, void *value, u64 flags) |
301 | { |
302 | nfp_map_bpf_byte_swap(nfp_map: offmap->dev_priv, value); |
303 | nfp_map_bpf_byte_swap_record(nfp_map: offmap->dev_priv, value); |
304 | return nfp_bpf_ctrl_update_entry(offmap, key, value, flags); |
305 | } |
306 | |
307 | static int |
308 | nfp_bpf_map_get_next_key(struct bpf_offloaded_map *offmap, |
309 | void *key, void *next_key) |
310 | { |
311 | if (!key) |
312 | return nfp_bpf_ctrl_getfirst_entry(offmap, next_key); |
313 | return nfp_bpf_ctrl_getnext_entry(offmap, key, next_key); |
314 | } |
315 | |
316 | static int |
317 | nfp_bpf_map_delete_elem(struct bpf_offloaded_map *offmap, void *key) |
318 | { |
319 | if (offmap->map.map_type == BPF_MAP_TYPE_ARRAY) |
320 | return -EINVAL; |
321 | return nfp_bpf_ctrl_del_entry(offmap, key); |
322 | } |
323 | |
324 | static const struct bpf_map_dev_ops nfp_bpf_map_ops = { |
325 | .map_get_next_key = nfp_bpf_map_get_next_key, |
326 | .map_lookup_elem = nfp_bpf_map_lookup_entry, |
327 | .map_update_elem = nfp_bpf_map_update_entry, |
328 | .map_delete_elem = nfp_bpf_map_delete_elem, |
329 | }; |
330 | |
331 | static int |
332 | nfp_bpf_map_alloc(struct nfp_app_bpf *bpf, struct bpf_offloaded_map *offmap) |
333 | { |
334 | struct nfp_bpf_map *nfp_map; |
335 | unsigned int use_map_size; |
336 | long long int res; |
337 | |
338 | if (!bpf->maps.types) |
339 | return -EOPNOTSUPP; |
340 | |
341 | if (offmap->map.map_flags || |
342 | offmap->map.numa_node != NUMA_NO_NODE) { |
343 | pr_info("map flags are not supported\n" ); |
344 | return -EINVAL; |
345 | } |
346 | |
347 | if (!(bpf->maps.types & 1 << offmap->map.map_type)) { |
348 | pr_info("map type not supported\n" ); |
349 | return -EOPNOTSUPP; |
350 | } |
351 | if (bpf->maps.max_maps == bpf->maps_in_use) { |
352 | pr_info("too many maps for a device\n" ); |
353 | return -ENOMEM; |
354 | } |
355 | if (bpf->maps.max_elems - bpf->map_elems_in_use < |
356 | offmap->map.max_entries) { |
357 | pr_info("map with too many elements: %u, left: %u\n" , |
358 | offmap->map.max_entries, |
359 | bpf->maps.max_elems - bpf->map_elems_in_use); |
360 | return -ENOMEM; |
361 | } |
362 | |
363 | if (round_up(offmap->map.key_size, 8) + |
364 | round_up(offmap->map.value_size, 8) > bpf->maps.max_elem_sz) { |
365 | pr_info("map elements too large: %u, FW max element size (key+value): %u\n" , |
366 | round_up(offmap->map.key_size, 8) + |
367 | round_up(offmap->map.value_size, 8), |
368 | bpf->maps.max_elem_sz); |
369 | return -ENOMEM; |
370 | } |
371 | if (offmap->map.key_size > bpf->maps.max_key_sz) { |
372 | pr_info("map key size %u, FW max is %u\n" , |
373 | offmap->map.key_size, bpf->maps.max_key_sz); |
374 | return -ENOMEM; |
375 | } |
376 | if (offmap->map.value_size > bpf->maps.max_val_sz) { |
377 | pr_info("map value size %u, FW max is %u\n" , |
378 | offmap->map.value_size, bpf->maps.max_val_sz); |
379 | return -ENOMEM; |
380 | } |
381 | |
382 | use_map_size = DIV_ROUND_UP(offmap->map.value_size, 4) * |
383 | sizeof_field(struct nfp_bpf_map, use_map[0]); |
384 | |
385 | nfp_map = kzalloc(size: sizeof(*nfp_map) + use_map_size, GFP_USER); |
386 | if (!nfp_map) |
387 | return -ENOMEM; |
388 | |
389 | offmap->dev_priv = nfp_map; |
390 | nfp_map->offmap = offmap; |
391 | nfp_map->bpf = bpf; |
392 | spin_lock_init(&nfp_map->cache_lock); |
393 | |
394 | res = nfp_bpf_ctrl_alloc_map(bpf, map: &offmap->map); |
395 | if (res < 0) { |
396 | kfree(objp: nfp_map); |
397 | return res; |
398 | } |
399 | |
400 | nfp_map->tid = res; |
401 | offmap->dev_ops = &nfp_bpf_map_ops; |
402 | bpf->maps_in_use++; |
403 | bpf->map_elems_in_use += offmap->map.max_entries; |
404 | list_add_tail(new: &nfp_map->l, head: &bpf->map_list); |
405 | |
406 | return 0; |
407 | } |
408 | |
409 | static int |
410 | nfp_bpf_map_free(struct nfp_app_bpf *bpf, struct bpf_offloaded_map *offmap) |
411 | { |
412 | struct nfp_bpf_map *nfp_map = offmap->dev_priv; |
413 | |
414 | nfp_bpf_ctrl_free_map(bpf, nfp_map); |
415 | dev_consume_skb_any(skb: nfp_map->cache); |
416 | WARN_ON_ONCE(nfp_map->cache_blockers); |
417 | list_del_init(entry: &nfp_map->l); |
418 | bpf->map_elems_in_use -= offmap->map.max_entries; |
419 | bpf->maps_in_use--; |
420 | kfree(objp: nfp_map); |
421 | |
422 | return 0; |
423 | } |
424 | |
425 | int nfp_ndo_bpf(struct nfp_app *app, struct nfp_net *nn, struct netdev_bpf *bpf) |
426 | { |
427 | switch (bpf->command) { |
428 | case BPF_OFFLOAD_MAP_ALLOC: |
429 | return nfp_bpf_map_alloc(bpf: app->priv, offmap: bpf->offmap); |
430 | case BPF_OFFLOAD_MAP_FREE: |
431 | return nfp_bpf_map_free(bpf: app->priv, offmap: bpf->offmap); |
432 | default: |
433 | return -EINVAL; |
434 | } |
435 | } |
436 | |
437 | static unsigned long |
438 | nfp_bpf_perf_event_copy(void *dst, const void *src, |
439 | unsigned long off, unsigned long len) |
440 | { |
441 | memcpy(dst, src + off, len); |
442 | return 0; |
443 | } |
444 | |
445 | int nfp_bpf_event_output(struct nfp_app_bpf *bpf, const void *data, |
446 | unsigned int len) |
447 | { |
448 | struct cmsg_bpf_event *cbe = (void *)data; |
449 | struct nfp_bpf_neutral_map *record; |
450 | u32 pkt_size, data_size, map_id; |
451 | u64 map_id_full; |
452 | |
453 | if (len < sizeof(struct cmsg_bpf_event)) |
454 | return -EINVAL; |
455 | |
456 | pkt_size = be32_to_cpu(cbe->pkt_size); |
457 | data_size = be32_to_cpu(cbe->data_size); |
458 | map_id_full = be64_to_cpu(cbe->map_ptr); |
459 | map_id = map_id_full; |
460 | |
461 | if (len < sizeof(struct cmsg_bpf_event) + pkt_size + data_size) |
462 | return -EINVAL; |
463 | if (cbe->hdr.ver != NFP_CCM_ABI_VERSION) |
464 | return -EINVAL; |
465 | |
466 | rcu_read_lock(); |
467 | record = rhashtable_lookup(ht: &bpf->maps_neutral, key: &map_id, |
468 | params: nfp_bpf_maps_neutral_params); |
469 | if (!record || map_id_full > U32_MAX) { |
470 | rcu_read_unlock(); |
471 | cmsg_warn(bpf, "perf event: map id %lld (0x%llx) not recognized, dropping event\n" , |
472 | map_id_full, map_id_full); |
473 | return -EINVAL; |
474 | } |
475 | |
476 | bpf_event_output(map: record->ptr, be32_to_cpu(cbe->cpu_id), |
477 | meta: &cbe->data[round_up(pkt_size, 4)], meta_size: data_size, |
478 | ctx: cbe->data, ctx_size: pkt_size, ctx_copy: nfp_bpf_perf_event_copy); |
479 | rcu_read_unlock(); |
480 | |
481 | return 0; |
482 | } |
483 | |
484 | bool nfp_bpf_offload_check_mtu(struct nfp_net *nn, struct bpf_prog *prog, |
485 | unsigned int mtu) |
486 | { |
487 | unsigned int fw_mtu, pkt_off; |
488 | |
489 | fw_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32; |
490 | pkt_off = min(prog->aux->max_pkt_offset, mtu); |
491 | |
492 | return fw_mtu < pkt_off; |
493 | } |
494 | |
495 | static int |
496 | nfp_net_bpf_load(struct nfp_net *nn, struct bpf_prog *prog, |
497 | struct netlink_ext_ack *extack) |
498 | { |
499 | struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv; |
500 | unsigned int max_stack, max_prog_len; |
501 | dma_addr_t dma_addr; |
502 | void *img; |
503 | int err; |
504 | |
505 | if (nfp_bpf_offload_check_mtu(nn, prog, mtu: nn->dp.netdev->mtu)) { |
506 | NL_SET_ERR_MSG_MOD(extack, "BPF offload not supported with potential packet access beyond HW packet split boundary" ); |
507 | return -EOPNOTSUPP; |
508 | } |
509 | |
510 | max_stack = nn_readb(nn, NFP_NET_CFG_BPF_STACK_SZ) * 64; |
511 | if (nfp_prog->stack_size > max_stack) { |
512 | NL_SET_ERR_MSG_MOD(extack, "stack too large" ); |
513 | return -EOPNOTSUPP; |
514 | } |
515 | |
516 | max_prog_len = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN); |
517 | if (nfp_prog->prog_len > max_prog_len) { |
518 | NL_SET_ERR_MSG_MOD(extack, "program too long" ); |
519 | return -EOPNOTSUPP; |
520 | } |
521 | |
522 | img = nfp_bpf_relo_for_vnic(nfp_prog, bv: nn->app_priv); |
523 | if (IS_ERR(ptr: img)) |
524 | return PTR_ERR(ptr: img); |
525 | |
526 | dma_addr = dma_map_single(nn->dp.dev, img, |
527 | nfp_prog->prog_len * sizeof(u64), |
528 | DMA_TO_DEVICE); |
529 | if (dma_mapping_error(dev: nn->dp.dev, dma_addr)) { |
530 | kfree(objp: img); |
531 | return -ENOMEM; |
532 | } |
533 | |
534 | nn_writew(nn, NFP_NET_CFG_BPF_SIZE, val: nfp_prog->prog_len); |
535 | nn_writeq(nn, NFP_NET_CFG_BPF_ADDR, val: dma_addr); |
536 | |
537 | /* Load up the JITed code */ |
538 | err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_BPF); |
539 | if (err) |
540 | NL_SET_ERR_MSG_MOD(extack, |
541 | "FW command error while loading BPF" ); |
542 | |
543 | dma_unmap_single(nn->dp.dev, dma_addr, nfp_prog->prog_len * sizeof(u64), |
544 | DMA_TO_DEVICE); |
545 | kfree(objp: img); |
546 | |
547 | return err; |
548 | } |
549 | |
550 | static void |
551 | nfp_net_bpf_start(struct nfp_net *nn, struct netlink_ext_ack *extack) |
552 | { |
553 | int err; |
554 | |
555 | /* Enable passing packets through BPF function */ |
556 | nn->dp.ctrl |= NFP_NET_CFG_CTRL_BPF; |
557 | nn_writel(nn, NFP_NET_CFG_CTRL, val: nn->dp.ctrl); |
558 | err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN); |
559 | if (err) |
560 | NL_SET_ERR_MSG_MOD(extack, |
561 | "FW command error while enabling BPF" ); |
562 | } |
563 | |
564 | static int nfp_net_bpf_stop(struct nfp_net *nn) |
565 | { |
566 | if (!(nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF)) |
567 | return 0; |
568 | |
569 | nn->dp.ctrl &= ~NFP_NET_CFG_CTRL_BPF; |
570 | nn_writel(nn, NFP_NET_CFG_CTRL, val: nn->dp.ctrl); |
571 | |
572 | return nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN); |
573 | } |
574 | |
575 | int nfp_net_bpf_offload(struct nfp_net *nn, struct bpf_prog *prog, |
576 | bool old_prog, struct netlink_ext_ack *extack) |
577 | { |
578 | int err; |
579 | |
580 | if (prog && !bpf_offload_dev_match(prog, netdev: nn->dp.netdev)) |
581 | return -EINVAL; |
582 | |
583 | if (prog && old_prog) { |
584 | u8 cap; |
585 | |
586 | cap = nn_readb(nn, NFP_NET_CFG_BPF_CAP); |
587 | if (!(cap & NFP_NET_BPF_CAP_RELO)) { |
588 | NL_SET_ERR_MSG_MOD(extack, |
589 | "FW does not support live reload" ); |
590 | return -EBUSY; |
591 | } |
592 | } |
593 | |
594 | /* Something else is loaded, different program type? */ |
595 | if (!old_prog && nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF) |
596 | return -EBUSY; |
597 | |
598 | if (old_prog && !prog) |
599 | return nfp_net_bpf_stop(nn); |
600 | |
601 | err = nfp_net_bpf_load(nn, prog, extack); |
602 | if (err) |
603 | return err; |
604 | |
605 | if (!old_prog) |
606 | nfp_net_bpf_start(nn, extack); |
607 | |
608 | return 0; |
609 | } |
610 | |
611 | const struct bpf_prog_offload_ops nfp_bpf_dev_ops = { |
612 | .insn_hook = nfp_verify_insn, |
613 | .finalize = nfp_bpf_finalize, |
614 | .replace_insn = nfp_bpf_opt_replace_insn, |
615 | .remove_insns = nfp_bpf_opt_remove_insns, |
616 | .prepare = nfp_bpf_verifier_prep, |
617 | .translate = nfp_bpf_translate, |
618 | .destroy = nfp_bpf_destroy, |
619 | }; |
620 | |