1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /**************************************************************************** |
3 | * Driver for Solarflare network controllers and boards |
4 | * Copyright 2005-2018 Solarflare Communications Inc. |
5 | * Copyright 2019-2020 Xilinx Inc. |
6 | * |
7 | * This program is free software; you can redistribute it and/or modify it |
8 | * under the terms of the GNU General Public License version 2 as published |
9 | * by the Free Software Foundation, incorporated herein by reference. |
10 | */ |
11 | |
12 | #include "mcdi_filters.h" |
13 | #include "mcdi.h" |
14 | #include "nic.h" |
15 | #include "rx_common.h" |
16 | |
17 | /* The maximum size of a shared RSS context */ |
18 | /* TODO: this should really be from the mcdi protocol export */ |
19 | #define 64UL |
20 | |
21 | #define EFX_EF10_FILTER_ID_INVALID 0xffff |
22 | |
23 | /* An arbitrary search limit for the software hash table */ |
24 | #define EFX_EF10_FILTER_SEARCH_LIMIT 200 |
25 | |
26 | static struct efx_filter_spec * |
27 | efx_mcdi_filter_entry_spec(const struct efx_mcdi_filter_table *table, |
28 | unsigned int filter_idx) |
29 | { |
30 | return (struct efx_filter_spec *)(table->entry[filter_idx].spec & |
31 | ~EFX_EF10_FILTER_FLAGS); |
32 | } |
33 | |
34 | static unsigned int |
35 | efx_mcdi_filter_entry_flags(const struct efx_mcdi_filter_table *table, |
36 | unsigned int filter_idx) |
37 | { |
38 | return table->entry[filter_idx].spec & EFX_EF10_FILTER_FLAGS; |
39 | } |
40 | |
41 | static u32 efx_mcdi_filter_get_unsafe_id(u32 filter_id) |
42 | { |
43 | WARN_ON_ONCE(filter_id == EFX_EF10_FILTER_ID_INVALID); |
44 | return filter_id & (EFX_MCDI_FILTER_TBL_ROWS - 1); |
45 | } |
46 | |
47 | static unsigned int efx_mcdi_filter_get_unsafe_pri(u32 filter_id) |
48 | { |
49 | return filter_id / (EFX_MCDI_FILTER_TBL_ROWS * 2); |
50 | } |
51 | |
52 | static u32 efx_mcdi_filter_make_filter_id(unsigned int pri, u16 idx) |
53 | { |
54 | return pri * EFX_MCDI_FILTER_TBL_ROWS * 2 + idx; |
55 | } |
56 | |
57 | /* |
58 | * Decide whether a filter should be exclusive or else should allow |
59 | * delivery to additional recipients. Currently we decide that |
60 | * filters for specific local unicast MAC and IP addresses are |
61 | * exclusive. |
62 | */ |
63 | static bool efx_mcdi_filter_is_exclusive(const struct efx_filter_spec *spec) |
64 | { |
65 | if (spec->match_flags & EFX_FILTER_MATCH_LOC_MAC && |
66 | !is_multicast_ether_addr(addr: spec->loc_mac)) |
67 | return true; |
68 | |
69 | if ((spec->match_flags & |
70 | (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) == |
71 | (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) { |
72 | if (spec->ether_type == htons(ETH_P_IP) && |
73 | !ipv4_is_multicast(addr: spec->loc_host[0])) |
74 | return true; |
75 | if (spec->ether_type == htons(ETH_P_IPV6) && |
76 | ((const u8 *)spec->loc_host)[0] != 0xff) |
77 | return true; |
78 | } |
79 | |
80 | return false; |
81 | } |
82 | |
83 | static void |
84 | efx_mcdi_filter_set_entry(struct efx_mcdi_filter_table *table, |
85 | unsigned int filter_idx, |
86 | const struct efx_filter_spec *spec, |
87 | unsigned int flags) |
88 | { |
89 | table->entry[filter_idx].spec = (unsigned long)spec | flags; |
90 | } |
91 | |
92 | static void |
93 | efx_mcdi_filter_push_prep_set_match_fields(struct efx_nic *efx, |
94 | const struct efx_filter_spec *spec, |
95 | efx_dword_t *inbuf) |
96 | { |
97 | enum efx_encap_type encap_type = efx_filter_get_encap_type(spec); |
98 | u32 match_fields = 0, uc_match, mc_match; |
99 | |
100 | MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP, |
101 | efx_mcdi_filter_is_exclusive(spec) ? |
102 | MC_CMD_FILTER_OP_IN_OP_INSERT : |
103 | MC_CMD_FILTER_OP_IN_OP_SUBSCRIBE); |
104 | |
105 | /* |
106 | * Convert match flags and values. Unlike almost |
107 | * everything else in MCDI, these fields are in |
108 | * network byte order. |
109 | */ |
110 | #define COPY_VALUE(value, mcdi_field) \ |
111 | do { \ |
112 | match_fields |= \ |
113 | 1 << MC_CMD_FILTER_OP_IN_MATCH_ ## \ |
114 | mcdi_field ## _LBN; \ |
115 | BUILD_BUG_ON( \ |
116 | MC_CMD_FILTER_OP_IN_ ## mcdi_field ## _LEN < \ |
117 | sizeof(value)); \ |
118 | memcpy(MCDI_PTR(inbuf, FILTER_OP_IN_ ## mcdi_field), \ |
119 | &value, sizeof(value)); \ |
120 | } while (0) |
121 | #define COPY_FIELD(gen_flag, gen_field, mcdi_field) \ |
122 | if (spec->match_flags & EFX_FILTER_MATCH_ ## gen_flag) { \ |
123 | COPY_VALUE(spec->gen_field, mcdi_field); \ |
124 | } |
125 | /* |
126 | * Handle encap filters first. They will always be mismatch |
127 | * (unknown UC or MC) filters |
128 | */ |
129 | if (encap_type) { |
130 | /* |
131 | * ether_type and outer_ip_proto need to be variables |
132 | * because COPY_VALUE wants to memcpy them |
133 | */ |
134 | __be16 ether_type = |
135 | htons(encap_type & EFX_ENCAP_FLAG_IPV6 ? |
136 | ETH_P_IPV6 : ETH_P_IP); |
137 | u8 vni_type = MC_CMD_FILTER_OP_EXT_IN_VNI_TYPE_GENEVE; |
138 | u8 outer_ip_proto; |
139 | |
140 | switch (encap_type & EFX_ENCAP_TYPES_MASK) { |
141 | case EFX_ENCAP_TYPE_VXLAN: |
142 | vni_type = MC_CMD_FILTER_OP_EXT_IN_VNI_TYPE_VXLAN; |
143 | fallthrough; |
144 | case EFX_ENCAP_TYPE_GENEVE: |
145 | COPY_VALUE(ether_type, ETHER_TYPE); |
146 | outer_ip_proto = IPPROTO_UDP; |
147 | COPY_VALUE(outer_ip_proto, IP_PROTO); |
148 | /* |
149 | * We always need to set the type field, even |
150 | * though we're not matching on the TNI. |
151 | */ |
152 | MCDI_POPULATE_DWORD_1(inbuf, |
153 | FILTER_OP_EXT_IN_VNI_OR_VSID, |
154 | FILTER_OP_EXT_IN_VNI_TYPE, |
155 | vni_type); |
156 | break; |
157 | case EFX_ENCAP_TYPE_NVGRE: |
158 | COPY_VALUE(ether_type, ETHER_TYPE); |
159 | outer_ip_proto = IPPROTO_GRE; |
160 | COPY_VALUE(outer_ip_proto, IP_PROTO); |
161 | break; |
162 | default: |
163 | WARN_ON(1); |
164 | } |
165 | |
166 | uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_UCAST_DST_LBN; |
167 | mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_MCAST_DST_LBN; |
168 | } else { |
169 | uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_UCAST_DST_LBN; |
170 | mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_MCAST_DST_LBN; |
171 | } |
172 | |
173 | if (spec->match_flags & EFX_FILTER_MATCH_LOC_MAC_IG) |
174 | match_fields |= |
175 | is_multicast_ether_addr(addr: spec->loc_mac) ? |
176 | 1 << mc_match : |
177 | 1 << uc_match; |
178 | COPY_FIELD(REM_HOST, rem_host, SRC_IP); |
179 | COPY_FIELD(LOC_HOST, loc_host, DST_IP); |
180 | COPY_FIELD(REM_MAC, rem_mac, SRC_MAC); |
181 | COPY_FIELD(REM_PORT, rem_port, SRC_PORT); |
182 | COPY_FIELD(LOC_MAC, loc_mac, DST_MAC); |
183 | COPY_FIELD(LOC_PORT, loc_port, DST_PORT); |
184 | COPY_FIELD(ETHER_TYPE, ether_type, ETHER_TYPE); |
185 | COPY_FIELD(INNER_VID, inner_vid, INNER_VLAN); |
186 | COPY_FIELD(OUTER_VID, outer_vid, OUTER_VLAN); |
187 | COPY_FIELD(IP_PROTO, ip_proto, IP_PROTO); |
188 | #undef COPY_FIELD |
189 | #undef COPY_VALUE |
190 | MCDI_SET_DWORD(inbuf, FILTER_OP_IN_MATCH_FIELDS, |
191 | match_fields); |
192 | } |
193 | |
194 | static void efx_mcdi_filter_push_prep(struct efx_nic *efx, |
195 | const struct efx_filter_spec *spec, |
196 | efx_dword_t *inbuf, u64 handle, |
197 | struct efx_rss_context *ctx, |
198 | bool replacing) |
199 | { |
200 | u32 flags = spec->flags; |
201 | |
202 | memset(inbuf, 0, MC_CMD_FILTER_OP_EXT_IN_LEN); |
203 | |
204 | /* If RSS filter, caller better have given us an RSS context */ |
205 | if (flags & EFX_FILTER_FLAG_RX_RSS) { |
206 | /* |
207 | * We don't have the ability to return an error, so we'll just |
208 | * log a warning and disable RSS for the filter. |
209 | */ |
210 | if (WARN_ON_ONCE(!ctx)) |
211 | flags &= ~EFX_FILTER_FLAG_RX_RSS; |
212 | else if (WARN_ON_ONCE(ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID)) |
213 | flags &= ~EFX_FILTER_FLAG_RX_RSS; |
214 | } |
215 | |
216 | if (replacing) { |
217 | MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP, |
218 | MC_CMD_FILTER_OP_IN_OP_REPLACE); |
219 | MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE, handle); |
220 | } else { |
221 | efx_mcdi_filter_push_prep_set_match_fields(efx, spec, inbuf); |
222 | } |
223 | |
224 | if (flags & EFX_FILTER_FLAG_VPORT_ID) |
225 | MCDI_SET_DWORD(inbuf, FILTER_OP_IN_PORT_ID, spec->vport_id); |
226 | else |
227 | MCDI_SET_DWORD(inbuf, FILTER_OP_IN_PORT_ID, efx->vport_id); |
228 | MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_DEST, |
229 | spec->dmaq_id == EFX_FILTER_RX_DMAQ_ID_DROP ? |
230 | MC_CMD_FILTER_OP_IN_RX_DEST_DROP : |
231 | MC_CMD_FILTER_OP_IN_RX_DEST_HOST); |
232 | MCDI_SET_DWORD(inbuf, FILTER_OP_IN_TX_DOMAIN, 0); |
233 | MCDI_SET_DWORD(inbuf, FILTER_OP_IN_TX_DEST, |
234 | MC_CMD_FILTER_OP_IN_TX_DEST_DEFAULT); |
235 | MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_QUEUE, |
236 | spec->dmaq_id == EFX_FILTER_RX_DMAQ_ID_DROP ? |
237 | 0 : spec->dmaq_id); |
238 | MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_MODE, |
239 | (flags & EFX_FILTER_FLAG_RX_RSS) ? |
240 | MC_CMD_FILTER_OP_IN_RX_MODE_RSS : |
241 | MC_CMD_FILTER_OP_IN_RX_MODE_SIMPLE); |
242 | if (flags & EFX_FILTER_FLAG_RX_RSS) |
243 | MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_CONTEXT, ctx->context_id); |
244 | } |
245 | |
246 | static int efx_mcdi_filter_push(struct efx_nic *efx, |
247 | const struct efx_filter_spec *spec, u64 *handle, |
248 | struct efx_rss_context *ctx, bool replacing) |
249 | { |
250 | MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_EXT_IN_LEN); |
251 | MCDI_DECLARE_BUF(outbuf, MC_CMD_FILTER_OP_EXT_OUT_LEN); |
252 | size_t outlen; |
253 | int rc; |
254 | |
255 | efx_mcdi_filter_push_prep(efx, spec, inbuf, handle: *handle, ctx, replacing); |
256 | rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FILTER_OP, inbuf, inlen: sizeof(inbuf), |
257 | outbuf, outlen: sizeof(outbuf), outlen_actual: &outlen); |
258 | if (rc && spec->priority != EFX_FILTER_PRI_HINT) |
259 | efx_mcdi_display_error(efx, MC_CMD_FILTER_OP, inlen: sizeof(inbuf), |
260 | outbuf, outlen, rc); |
261 | if (rc == 0) |
262 | *handle = MCDI_QWORD(outbuf, FILTER_OP_OUT_HANDLE); |
263 | if (rc == -ENOSPC) |
264 | rc = -EBUSY; /* to match efx_farch_filter_insert() */ |
265 | return rc; |
266 | } |
267 | |
268 | static u32 efx_mcdi_filter_mcdi_flags_from_spec(const struct efx_filter_spec *spec) |
269 | { |
270 | enum efx_encap_type encap_type = efx_filter_get_encap_type(spec); |
271 | unsigned int match_flags = spec->match_flags; |
272 | unsigned int uc_match, mc_match; |
273 | u32 mcdi_flags = 0; |
274 | |
275 | #define MAP_FILTER_TO_MCDI_FLAG(gen_flag, mcdi_field, encap) { \ |
276 | unsigned int old_match_flags = match_flags; \ |
277 | match_flags &= ~EFX_FILTER_MATCH_ ## gen_flag; \ |
278 | if (match_flags != old_match_flags) \ |
279 | mcdi_flags |= \ |
280 | (1 << ((encap) ? \ |
281 | MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_ ## \ |
282 | mcdi_field ## _LBN : \ |
283 | MC_CMD_FILTER_OP_EXT_IN_MATCH_ ##\ |
284 | mcdi_field ## _LBN)); \ |
285 | } |
286 | /* inner or outer based on encap type */ |
287 | MAP_FILTER_TO_MCDI_FLAG(REM_HOST, SRC_IP, encap_type); |
288 | MAP_FILTER_TO_MCDI_FLAG(LOC_HOST, DST_IP, encap_type); |
289 | MAP_FILTER_TO_MCDI_FLAG(REM_MAC, SRC_MAC, encap_type); |
290 | MAP_FILTER_TO_MCDI_FLAG(REM_PORT, SRC_PORT, encap_type); |
291 | MAP_FILTER_TO_MCDI_FLAG(LOC_MAC, DST_MAC, encap_type); |
292 | MAP_FILTER_TO_MCDI_FLAG(LOC_PORT, DST_PORT, encap_type); |
293 | MAP_FILTER_TO_MCDI_FLAG(ETHER_TYPE, ETHER_TYPE, encap_type); |
294 | MAP_FILTER_TO_MCDI_FLAG(IP_PROTO, IP_PROTO, encap_type); |
295 | /* always outer */ |
296 | MAP_FILTER_TO_MCDI_FLAG(INNER_VID, INNER_VLAN, false); |
297 | MAP_FILTER_TO_MCDI_FLAG(OUTER_VID, OUTER_VLAN, false); |
298 | #undef MAP_FILTER_TO_MCDI_FLAG |
299 | |
300 | /* special handling for encap type, and mismatch */ |
301 | if (encap_type) { |
302 | match_flags &= ~EFX_FILTER_MATCH_ENCAP_TYPE; |
303 | mcdi_flags |= |
304 | (1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_ETHER_TYPE_LBN); |
305 | mcdi_flags |= (1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_IP_PROTO_LBN); |
306 | |
307 | uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_UCAST_DST_LBN; |
308 | mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_MCAST_DST_LBN; |
309 | } else { |
310 | uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_UCAST_DST_LBN; |
311 | mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_MCAST_DST_LBN; |
312 | } |
313 | |
314 | if (match_flags & EFX_FILTER_MATCH_LOC_MAC_IG) { |
315 | match_flags &= ~EFX_FILTER_MATCH_LOC_MAC_IG; |
316 | mcdi_flags |= |
317 | is_multicast_ether_addr(addr: spec->loc_mac) ? |
318 | 1 << mc_match : |
319 | 1 << uc_match; |
320 | } |
321 | |
322 | /* Did we map them all? */ |
323 | WARN_ON_ONCE(match_flags); |
324 | |
325 | return mcdi_flags; |
326 | } |
327 | |
328 | static int efx_mcdi_filter_pri(struct efx_mcdi_filter_table *table, |
329 | const struct efx_filter_spec *spec) |
330 | { |
331 | u32 mcdi_flags = efx_mcdi_filter_mcdi_flags_from_spec(spec); |
332 | unsigned int match_pri; |
333 | |
334 | for (match_pri = 0; |
335 | match_pri < table->rx_match_count; |
336 | match_pri++) |
337 | if (table->rx_match_mcdi_flags[match_pri] == mcdi_flags) |
338 | return match_pri; |
339 | |
340 | return -EPROTONOSUPPORT; |
341 | } |
342 | |
343 | static s32 efx_mcdi_filter_insert_locked(struct efx_nic *efx, |
344 | struct efx_filter_spec *spec, |
345 | bool replace_equal) |
346 | { |
347 | DECLARE_BITMAP(mc_rem_map, EFX_EF10_FILTER_SEARCH_LIMIT); |
348 | struct efx_mcdi_filter_table *table; |
349 | struct efx_filter_spec *saved_spec; |
350 | struct efx_rss_context *ctx = NULL; |
351 | unsigned int match_pri, hash; |
352 | unsigned int priv_flags; |
353 | bool = false; |
354 | bool replacing = false; |
355 | unsigned int depth, i; |
356 | int ins_index = -1; |
357 | DEFINE_WAIT(wait); |
358 | bool is_mc_recip; |
359 | s32 rc; |
360 | |
361 | WARN_ON(!rwsem_is_locked(&efx->filter_sem)); |
362 | table = efx->filter_state; |
363 | down_write(sem: &table->lock); |
364 | |
365 | /* For now, only support RX filters */ |
366 | if ((spec->flags & (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)) != |
367 | EFX_FILTER_FLAG_RX) { |
368 | rc = -EINVAL; |
369 | goto out_unlock; |
370 | } |
371 | |
372 | rc = efx_mcdi_filter_pri(table, spec); |
373 | if (rc < 0) |
374 | goto out_unlock; |
375 | match_pri = rc; |
376 | |
377 | hash = efx_filter_spec_hash(spec); |
378 | is_mc_recip = efx_filter_is_mc_recipient(spec); |
379 | if (is_mc_recip) |
380 | bitmap_zero(dst: mc_rem_map, EFX_EF10_FILTER_SEARCH_LIMIT); |
381 | |
382 | if (spec->flags & EFX_FILTER_FLAG_RX_RSS) { |
383 | mutex_lock(&efx->rss_lock); |
384 | rss_locked = true; |
385 | if (spec->rss_context) |
386 | ctx = efx_find_rss_context_entry(efx, id: spec->rss_context); |
387 | else |
388 | ctx = &efx->rss_context; |
389 | if (!ctx) { |
390 | rc = -ENOENT; |
391 | goto out_unlock; |
392 | } |
393 | if (ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID) { |
394 | rc = -EOPNOTSUPP; |
395 | goto out_unlock; |
396 | } |
397 | } |
398 | |
399 | /* Find any existing filters with the same match tuple or |
400 | * else a free slot to insert at. |
401 | */ |
402 | for (depth = 1; depth < EFX_EF10_FILTER_SEARCH_LIMIT; depth++) { |
403 | i = (hash + depth) & (EFX_MCDI_FILTER_TBL_ROWS - 1); |
404 | saved_spec = efx_mcdi_filter_entry_spec(table, filter_idx: i); |
405 | |
406 | if (!saved_spec) { |
407 | if (ins_index < 0) |
408 | ins_index = i; |
409 | } else if (efx_filter_spec_equal(left: spec, right: saved_spec)) { |
410 | if (spec->priority < saved_spec->priority && |
411 | spec->priority != EFX_FILTER_PRI_AUTO) { |
412 | rc = -EPERM; |
413 | goto out_unlock; |
414 | } |
415 | if (!is_mc_recip) { |
416 | /* This is the only one */ |
417 | if (spec->priority == |
418 | saved_spec->priority && |
419 | !replace_equal) { |
420 | rc = -EEXIST; |
421 | goto out_unlock; |
422 | } |
423 | ins_index = i; |
424 | break; |
425 | } else if (spec->priority > |
426 | saved_spec->priority || |
427 | (spec->priority == |
428 | saved_spec->priority && |
429 | replace_equal)) { |
430 | if (ins_index < 0) |
431 | ins_index = i; |
432 | else |
433 | __set_bit(depth, mc_rem_map); |
434 | } |
435 | } |
436 | } |
437 | |
438 | /* Once we reach the maximum search depth, use the first suitable |
439 | * slot, or return -EBUSY if there was none |
440 | */ |
441 | if (ins_index < 0) { |
442 | rc = -EBUSY; |
443 | goto out_unlock; |
444 | } |
445 | |
446 | /* Create a software table entry if necessary. */ |
447 | saved_spec = efx_mcdi_filter_entry_spec(table, filter_idx: ins_index); |
448 | if (saved_spec) { |
449 | if (spec->priority == EFX_FILTER_PRI_AUTO && |
450 | saved_spec->priority >= EFX_FILTER_PRI_AUTO) { |
451 | /* Just make sure it won't be removed */ |
452 | if (saved_spec->priority > EFX_FILTER_PRI_AUTO) |
453 | saved_spec->flags |= EFX_FILTER_FLAG_RX_OVER_AUTO; |
454 | table->entry[ins_index].spec &= |
455 | ~EFX_EF10_FILTER_FLAG_AUTO_OLD; |
456 | rc = ins_index; |
457 | goto out_unlock; |
458 | } |
459 | replacing = true; |
460 | priv_flags = efx_mcdi_filter_entry_flags(table, filter_idx: ins_index); |
461 | } else { |
462 | saved_spec = kmalloc(size: sizeof(*spec), GFP_ATOMIC); |
463 | if (!saved_spec) { |
464 | rc = -ENOMEM; |
465 | goto out_unlock; |
466 | } |
467 | *saved_spec = *spec; |
468 | priv_flags = 0; |
469 | } |
470 | efx_mcdi_filter_set_entry(table, filter_idx: ins_index, spec: saved_spec, flags: priv_flags); |
471 | |
472 | /* Actually insert the filter on the HW */ |
473 | rc = efx_mcdi_filter_push(efx, spec, handle: &table->entry[ins_index].handle, |
474 | ctx, replacing); |
475 | |
476 | if (rc == -EINVAL && efx->must_realloc_vis) |
477 | /* The MC rebooted under us, causing it to reject our filter |
478 | * insertion as pointing to an invalid VI (spec->dmaq_id). |
479 | */ |
480 | rc = -EAGAIN; |
481 | |
482 | /* Finalise the software table entry */ |
483 | if (rc == 0) { |
484 | if (replacing) { |
485 | /* Update the fields that may differ */ |
486 | if (saved_spec->priority == EFX_FILTER_PRI_AUTO) |
487 | saved_spec->flags |= |
488 | EFX_FILTER_FLAG_RX_OVER_AUTO; |
489 | saved_spec->priority = spec->priority; |
490 | saved_spec->flags &= EFX_FILTER_FLAG_RX_OVER_AUTO; |
491 | saved_spec->flags |= spec->flags; |
492 | saved_spec->rss_context = spec->rss_context; |
493 | saved_spec->dmaq_id = spec->dmaq_id; |
494 | saved_spec->vport_id = spec->vport_id; |
495 | } |
496 | } else if (!replacing) { |
497 | kfree(objp: saved_spec); |
498 | saved_spec = NULL; |
499 | } else { |
500 | /* We failed to replace, so the old filter is still present. |
501 | * Roll back the software table to reflect this. In fact the |
502 | * efx_mcdi_filter_set_entry() call below will do the right |
503 | * thing, so nothing extra is needed here. |
504 | */ |
505 | } |
506 | efx_mcdi_filter_set_entry(table, filter_idx: ins_index, spec: saved_spec, flags: priv_flags); |
507 | |
508 | /* Remove and finalise entries for lower-priority multicast |
509 | * recipients |
510 | */ |
511 | if (is_mc_recip) { |
512 | MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_EXT_IN_LEN); |
513 | unsigned int depth, i; |
514 | |
515 | memset(inbuf, 0, sizeof(inbuf)); |
516 | |
517 | for (depth = 0; depth < EFX_EF10_FILTER_SEARCH_LIMIT; depth++) { |
518 | if (!test_bit(depth, mc_rem_map)) |
519 | continue; |
520 | |
521 | i = (hash + depth) & (EFX_MCDI_FILTER_TBL_ROWS - 1); |
522 | saved_spec = efx_mcdi_filter_entry_spec(table, filter_idx: i); |
523 | priv_flags = efx_mcdi_filter_entry_flags(table, filter_idx: i); |
524 | |
525 | if (rc == 0) { |
526 | MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP, |
527 | MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE); |
528 | MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE, |
529 | table->entry[i].handle); |
530 | rc = efx_mcdi_rpc(efx, MC_CMD_FILTER_OP, |
531 | inbuf, inlen: sizeof(inbuf), |
532 | NULL, outlen: 0, NULL); |
533 | } |
534 | |
535 | if (rc == 0) { |
536 | kfree(objp: saved_spec); |
537 | saved_spec = NULL; |
538 | priv_flags = 0; |
539 | } |
540 | efx_mcdi_filter_set_entry(table, filter_idx: i, spec: saved_spec, |
541 | flags: priv_flags); |
542 | } |
543 | } |
544 | |
545 | /* If successful, return the inserted filter ID */ |
546 | if (rc == 0) |
547 | rc = efx_mcdi_filter_make_filter_id(pri: match_pri, idx: ins_index); |
548 | |
549 | out_unlock: |
550 | if (rss_locked) |
551 | mutex_unlock(lock: &efx->rss_lock); |
552 | up_write(sem: &table->lock); |
553 | return rc; |
554 | } |
555 | |
556 | s32 efx_mcdi_filter_insert(struct efx_nic *efx, struct efx_filter_spec *spec, |
557 | bool replace_equal) |
558 | { |
559 | s32 ret; |
560 | |
561 | down_read(sem: &efx->filter_sem); |
562 | ret = efx_mcdi_filter_insert_locked(efx, spec, replace_equal); |
563 | up_read(sem: &efx->filter_sem); |
564 | |
565 | return ret; |
566 | } |
567 | |
568 | /* |
569 | * Remove a filter. |
570 | * If !by_index, remove by ID |
571 | * If by_index, remove by index |
572 | * Filter ID may come from userland and must be range-checked. |
573 | * Caller must hold efx->filter_sem for read, and efx->filter_state->lock |
574 | * for write. |
575 | */ |
576 | static int efx_mcdi_filter_remove_internal(struct efx_nic *efx, |
577 | unsigned int priority_mask, |
578 | u32 filter_id, bool by_index) |
579 | { |
580 | unsigned int filter_idx = efx_mcdi_filter_get_unsafe_id(filter_id); |
581 | struct efx_mcdi_filter_table *table = efx->filter_state; |
582 | MCDI_DECLARE_BUF(inbuf, |
583 | MC_CMD_FILTER_OP_IN_HANDLE_OFST + |
584 | MC_CMD_FILTER_OP_IN_HANDLE_LEN); |
585 | struct efx_filter_spec *spec; |
586 | DEFINE_WAIT(wait); |
587 | int rc; |
588 | |
589 | spec = efx_mcdi_filter_entry_spec(table, filter_idx); |
590 | if (!spec || |
591 | (!by_index && |
592 | efx_mcdi_filter_pri(table, spec) != |
593 | efx_mcdi_filter_get_unsafe_pri(filter_id))) |
594 | return -ENOENT; |
595 | |
596 | if (spec->flags & EFX_FILTER_FLAG_RX_OVER_AUTO && |
597 | priority_mask == (1U << EFX_FILTER_PRI_AUTO)) { |
598 | /* Just remove flags */ |
599 | spec->flags &= ~EFX_FILTER_FLAG_RX_OVER_AUTO; |
600 | table->entry[filter_idx].spec &= ~EFX_EF10_FILTER_FLAG_AUTO_OLD; |
601 | return 0; |
602 | } |
603 | |
604 | if (!(priority_mask & (1U << spec->priority))) |
605 | return -ENOENT; |
606 | |
607 | if (spec->flags & EFX_FILTER_FLAG_RX_OVER_AUTO) { |
608 | /* Reset to an automatic filter */ |
609 | |
610 | struct efx_filter_spec new_spec = *spec; |
611 | |
612 | new_spec.priority = EFX_FILTER_PRI_AUTO; |
613 | new_spec.flags = (EFX_FILTER_FLAG_RX | |
614 | (efx_rss_active(ctx: &efx->rss_context) ? |
615 | EFX_FILTER_FLAG_RX_RSS : 0)); |
616 | new_spec.dmaq_id = 0; |
617 | new_spec.rss_context = 0; |
618 | rc = efx_mcdi_filter_push(efx, spec: &new_spec, |
619 | handle: &table->entry[filter_idx].handle, |
620 | ctx: &efx->rss_context, |
621 | replacing: true); |
622 | |
623 | if (rc == 0) |
624 | *spec = new_spec; |
625 | } else { |
626 | /* Really remove the filter */ |
627 | |
628 | MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP, |
629 | efx_mcdi_filter_is_exclusive(spec) ? |
630 | MC_CMD_FILTER_OP_IN_OP_REMOVE : |
631 | MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE); |
632 | MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE, |
633 | table->entry[filter_idx].handle); |
634 | rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FILTER_OP, |
635 | inbuf, inlen: sizeof(inbuf), NULL, outlen: 0, NULL); |
636 | |
637 | if ((rc == 0) || (rc == -ENOENT)) { |
638 | /* Filter removed OK or didn't actually exist */ |
639 | kfree(objp: spec); |
640 | efx_mcdi_filter_set_entry(table, filter_idx, NULL, flags: 0); |
641 | } else { |
642 | efx_mcdi_display_error(efx, MC_CMD_FILTER_OP, |
643 | MC_CMD_FILTER_OP_EXT_IN_LEN, |
644 | NULL, outlen: 0, rc); |
645 | } |
646 | } |
647 | |
648 | return rc; |
649 | } |
650 | |
651 | /* Remove filters that weren't renewed. */ |
652 | static void efx_mcdi_filter_remove_old(struct efx_nic *efx) |
653 | { |
654 | struct efx_mcdi_filter_table *table = efx->filter_state; |
655 | int remove_failed = 0; |
656 | int remove_noent = 0; |
657 | int rc; |
658 | int i; |
659 | |
660 | down_write(sem: &table->lock); |
661 | for (i = 0; i < EFX_MCDI_FILTER_TBL_ROWS; i++) { |
662 | if (READ_ONCE(table->entry[i].spec) & |
663 | EFX_EF10_FILTER_FLAG_AUTO_OLD) { |
664 | rc = efx_mcdi_filter_remove_internal(efx, |
665 | priority_mask: 1U << EFX_FILTER_PRI_AUTO, filter_id: i, by_index: true); |
666 | if (rc == -ENOENT) |
667 | remove_noent++; |
668 | else if (rc) |
669 | remove_failed++; |
670 | } |
671 | } |
672 | up_write(sem: &table->lock); |
673 | |
674 | if (remove_failed) |
675 | netif_info(efx, drv, efx->net_dev, |
676 | "%s: failed to remove %d filters\n" , |
677 | __func__, remove_failed); |
678 | if (remove_noent) |
679 | netif_info(efx, drv, efx->net_dev, |
680 | "%s: failed to remove %d non-existent filters\n" , |
681 | __func__, remove_noent); |
682 | } |
683 | |
684 | int efx_mcdi_filter_remove_safe(struct efx_nic *efx, |
685 | enum efx_filter_priority priority, |
686 | u32 filter_id) |
687 | { |
688 | struct efx_mcdi_filter_table *table; |
689 | int rc; |
690 | |
691 | down_read(sem: &efx->filter_sem); |
692 | table = efx->filter_state; |
693 | down_write(sem: &table->lock); |
694 | rc = efx_mcdi_filter_remove_internal(efx, priority_mask: 1U << priority, filter_id, |
695 | by_index: false); |
696 | up_write(sem: &table->lock); |
697 | up_read(sem: &efx->filter_sem); |
698 | return rc; |
699 | } |
700 | |
701 | /* Caller must hold efx->filter_sem for read */ |
702 | static void efx_mcdi_filter_remove_unsafe(struct efx_nic *efx, |
703 | enum efx_filter_priority priority, |
704 | u32 filter_id) |
705 | { |
706 | struct efx_mcdi_filter_table *table = efx->filter_state; |
707 | |
708 | if (filter_id == EFX_EF10_FILTER_ID_INVALID) |
709 | return; |
710 | |
711 | down_write(sem: &table->lock); |
712 | efx_mcdi_filter_remove_internal(efx, priority_mask: 1U << priority, filter_id, |
713 | by_index: true); |
714 | up_write(sem: &table->lock); |
715 | } |
716 | |
717 | int efx_mcdi_filter_get_safe(struct efx_nic *efx, |
718 | enum efx_filter_priority priority, |
719 | u32 filter_id, struct efx_filter_spec *spec) |
720 | { |
721 | unsigned int filter_idx = efx_mcdi_filter_get_unsafe_id(filter_id); |
722 | const struct efx_filter_spec *saved_spec; |
723 | struct efx_mcdi_filter_table *table; |
724 | int rc; |
725 | |
726 | down_read(sem: &efx->filter_sem); |
727 | table = efx->filter_state; |
728 | down_read(sem: &table->lock); |
729 | saved_spec = efx_mcdi_filter_entry_spec(table, filter_idx); |
730 | if (saved_spec && saved_spec->priority == priority && |
731 | efx_mcdi_filter_pri(table, spec: saved_spec) == |
732 | efx_mcdi_filter_get_unsafe_pri(filter_id)) { |
733 | *spec = *saved_spec; |
734 | rc = 0; |
735 | } else { |
736 | rc = -ENOENT; |
737 | } |
738 | up_read(sem: &table->lock); |
739 | up_read(sem: &efx->filter_sem); |
740 | return rc; |
741 | } |
742 | |
743 | static int efx_mcdi_filter_insert_addr_list(struct efx_nic *efx, |
744 | struct efx_mcdi_filter_vlan *vlan, |
745 | bool multicast, bool rollback) |
746 | { |
747 | struct efx_mcdi_filter_table *table = efx->filter_state; |
748 | struct efx_mcdi_dev_addr *addr_list; |
749 | enum efx_filter_flags filter_flags; |
750 | struct efx_filter_spec spec; |
751 | u8 baddr[ETH_ALEN]; |
752 | unsigned int i, j; |
753 | int addr_count; |
754 | u16 *ids; |
755 | int rc; |
756 | |
757 | if (multicast) { |
758 | addr_list = table->dev_mc_list; |
759 | addr_count = table->dev_mc_count; |
760 | ids = vlan->mc; |
761 | } else { |
762 | addr_list = table->dev_uc_list; |
763 | addr_count = table->dev_uc_count; |
764 | ids = vlan->uc; |
765 | } |
766 | |
767 | filter_flags = efx_rss_active(ctx: &efx->rss_context) ? EFX_FILTER_FLAG_RX_RSS : 0; |
768 | |
769 | /* Insert/renew filters */ |
770 | for (i = 0; i < addr_count; i++) { |
771 | EFX_WARN_ON_PARANOID(ids[i] != EFX_EF10_FILTER_ID_INVALID); |
772 | efx_filter_init_rx(spec: &spec, priority: EFX_FILTER_PRI_AUTO, flags: filter_flags, rxq_id: 0); |
773 | efx_filter_set_eth_local(spec: &spec, vid: vlan->vid, addr: addr_list[i].addr); |
774 | rc = efx_mcdi_filter_insert_locked(efx, spec: &spec, replace_equal: true); |
775 | if (rc < 0) { |
776 | if (rollback) { |
777 | netif_info(efx, drv, efx->net_dev, |
778 | "efx_mcdi_filter_insert failed rc=%d\n" , |
779 | rc); |
780 | /* Fall back to promiscuous */ |
781 | for (j = 0; j < i; j++) { |
782 | efx_mcdi_filter_remove_unsafe( |
783 | efx, priority: EFX_FILTER_PRI_AUTO, |
784 | filter_id: ids[j]); |
785 | ids[j] = EFX_EF10_FILTER_ID_INVALID; |
786 | } |
787 | return rc; |
788 | } else { |
789 | /* keep invalid ID, and carry on */ |
790 | } |
791 | } else { |
792 | ids[i] = efx_mcdi_filter_get_unsafe_id(filter_id: rc); |
793 | } |
794 | } |
795 | |
796 | if (multicast && rollback) { |
797 | /* Also need an Ethernet broadcast filter */ |
798 | EFX_WARN_ON_PARANOID(vlan->default_filters[EFX_EF10_BCAST] != |
799 | EFX_EF10_FILTER_ID_INVALID); |
800 | efx_filter_init_rx(spec: &spec, priority: EFX_FILTER_PRI_AUTO, flags: filter_flags, rxq_id: 0); |
801 | eth_broadcast_addr(addr: baddr); |
802 | efx_filter_set_eth_local(spec: &spec, vid: vlan->vid, addr: baddr); |
803 | rc = efx_mcdi_filter_insert_locked(efx, spec: &spec, replace_equal: true); |
804 | if (rc < 0) { |
805 | netif_warn(efx, drv, efx->net_dev, |
806 | "Broadcast filter insert failed rc=%d\n" , rc); |
807 | /* Fall back to promiscuous */ |
808 | for (j = 0; j < i; j++) { |
809 | efx_mcdi_filter_remove_unsafe( |
810 | efx, priority: EFX_FILTER_PRI_AUTO, |
811 | filter_id: ids[j]); |
812 | ids[j] = EFX_EF10_FILTER_ID_INVALID; |
813 | } |
814 | return rc; |
815 | } else { |
816 | vlan->default_filters[EFX_EF10_BCAST] = |
817 | efx_mcdi_filter_get_unsafe_id(filter_id: rc); |
818 | } |
819 | } |
820 | |
821 | return 0; |
822 | } |
823 | |
824 | static int efx_mcdi_filter_insert_def(struct efx_nic *efx, |
825 | struct efx_mcdi_filter_vlan *vlan, |
826 | enum efx_encap_type encap_type, |
827 | bool multicast, bool rollback) |
828 | { |
829 | struct efx_mcdi_filter_table *table = efx->filter_state; |
830 | enum efx_filter_flags filter_flags; |
831 | struct efx_filter_spec spec; |
832 | u8 baddr[ETH_ALEN]; |
833 | int rc; |
834 | u16 *id; |
835 | |
836 | filter_flags = efx_rss_active(ctx: &efx->rss_context) ? EFX_FILTER_FLAG_RX_RSS : 0; |
837 | |
838 | efx_filter_init_rx(spec: &spec, priority: EFX_FILTER_PRI_AUTO, flags: filter_flags, rxq_id: 0); |
839 | |
840 | if (multicast) |
841 | efx_filter_set_mc_def(spec: &spec); |
842 | else |
843 | efx_filter_set_uc_def(spec: &spec); |
844 | |
845 | if (encap_type) { |
846 | if (efx_has_cap(efx, VXLAN_NVGRE)) |
847 | efx_filter_set_encap_type(spec: &spec, encap_type); |
848 | else |
849 | /* |
850 | * don't insert encap filters on non-supporting |
851 | * platforms. ID will be left as INVALID. |
852 | */ |
853 | return 0; |
854 | } |
855 | |
856 | if (vlan->vid != EFX_FILTER_VID_UNSPEC) |
857 | efx_filter_set_eth_local(spec: &spec, vid: vlan->vid, NULL); |
858 | |
859 | rc = efx_mcdi_filter_insert_locked(efx, spec: &spec, replace_equal: true); |
860 | if (rc < 0) { |
861 | const char *um = multicast ? "Multicast" : "Unicast" ; |
862 | const char *encap_name = "" ; |
863 | const char *encap_ipv = "" ; |
864 | |
865 | if ((encap_type & EFX_ENCAP_TYPES_MASK) == |
866 | EFX_ENCAP_TYPE_VXLAN) |
867 | encap_name = "VXLAN " ; |
868 | else if ((encap_type & EFX_ENCAP_TYPES_MASK) == |
869 | EFX_ENCAP_TYPE_NVGRE) |
870 | encap_name = "NVGRE " ; |
871 | else if ((encap_type & EFX_ENCAP_TYPES_MASK) == |
872 | EFX_ENCAP_TYPE_GENEVE) |
873 | encap_name = "GENEVE " ; |
874 | if (encap_type & EFX_ENCAP_FLAG_IPV6) |
875 | encap_ipv = "IPv6 " ; |
876 | else if (encap_type) |
877 | encap_ipv = "IPv4 " ; |
878 | |
879 | /* |
880 | * unprivileged functions can't insert mismatch filters |
881 | * for encapsulated or unicast traffic, so downgrade |
882 | * those warnings to debug. |
883 | */ |
884 | netif_cond_dbg(efx, drv, efx->net_dev, |
885 | rc == -EPERM && (encap_type || !multicast), warn, |
886 | "%s%s%s mismatch filter insert failed rc=%d\n" , |
887 | encap_name, encap_ipv, um, rc); |
888 | } else if (multicast) { |
889 | /* mapping from encap types to default filter IDs (multicast) */ |
890 | static enum efx_mcdi_filter_default_filters map[] = { |
891 | [EFX_ENCAP_TYPE_NONE] = EFX_EF10_MCDEF, |
892 | [EFX_ENCAP_TYPE_VXLAN] = EFX_EF10_VXLAN4_MCDEF, |
893 | [EFX_ENCAP_TYPE_NVGRE] = EFX_EF10_NVGRE4_MCDEF, |
894 | [EFX_ENCAP_TYPE_GENEVE] = EFX_EF10_GENEVE4_MCDEF, |
895 | [EFX_ENCAP_TYPE_VXLAN | EFX_ENCAP_FLAG_IPV6] = |
896 | EFX_EF10_VXLAN6_MCDEF, |
897 | [EFX_ENCAP_TYPE_NVGRE | EFX_ENCAP_FLAG_IPV6] = |
898 | EFX_EF10_NVGRE6_MCDEF, |
899 | [EFX_ENCAP_TYPE_GENEVE | EFX_ENCAP_FLAG_IPV6] = |
900 | EFX_EF10_GENEVE6_MCDEF, |
901 | }; |
902 | |
903 | /* quick bounds check (BCAST result impossible) */ |
904 | BUILD_BUG_ON(EFX_EF10_BCAST != 0); |
905 | if (encap_type >= ARRAY_SIZE(map) || map[encap_type] == 0) { |
906 | WARN_ON(1); |
907 | return -EINVAL; |
908 | } |
909 | /* then follow map */ |
910 | id = &vlan->default_filters[map[encap_type]]; |
911 | |
912 | EFX_WARN_ON_PARANOID(*id != EFX_EF10_FILTER_ID_INVALID); |
913 | *id = efx_mcdi_filter_get_unsafe_id(filter_id: rc); |
914 | if (!table->mc_chaining && !encap_type) { |
915 | /* Also need an Ethernet broadcast filter */ |
916 | efx_filter_init_rx(spec: &spec, priority: EFX_FILTER_PRI_AUTO, |
917 | flags: filter_flags, rxq_id: 0); |
918 | eth_broadcast_addr(addr: baddr); |
919 | efx_filter_set_eth_local(spec: &spec, vid: vlan->vid, addr: baddr); |
920 | rc = efx_mcdi_filter_insert_locked(efx, spec: &spec, replace_equal: true); |
921 | if (rc < 0) { |
922 | netif_warn(efx, drv, efx->net_dev, |
923 | "Broadcast filter insert failed rc=%d\n" , |
924 | rc); |
925 | if (rollback) { |
926 | /* Roll back the mc_def filter */ |
927 | efx_mcdi_filter_remove_unsafe( |
928 | efx, priority: EFX_FILTER_PRI_AUTO, |
929 | filter_id: *id); |
930 | *id = EFX_EF10_FILTER_ID_INVALID; |
931 | return rc; |
932 | } |
933 | } else { |
934 | EFX_WARN_ON_PARANOID( |
935 | vlan->default_filters[EFX_EF10_BCAST] != |
936 | EFX_EF10_FILTER_ID_INVALID); |
937 | vlan->default_filters[EFX_EF10_BCAST] = |
938 | efx_mcdi_filter_get_unsafe_id(filter_id: rc); |
939 | } |
940 | } |
941 | rc = 0; |
942 | } else { |
943 | /* mapping from encap types to default filter IDs (unicast) */ |
944 | static enum efx_mcdi_filter_default_filters map[] = { |
945 | [EFX_ENCAP_TYPE_NONE] = EFX_EF10_UCDEF, |
946 | [EFX_ENCAP_TYPE_VXLAN] = EFX_EF10_VXLAN4_UCDEF, |
947 | [EFX_ENCAP_TYPE_NVGRE] = EFX_EF10_NVGRE4_UCDEF, |
948 | [EFX_ENCAP_TYPE_GENEVE] = EFX_EF10_GENEVE4_UCDEF, |
949 | [EFX_ENCAP_TYPE_VXLAN | EFX_ENCAP_FLAG_IPV6] = |
950 | EFX_EF10_VXLAN6_UCDEF, |
951 | [EFX_ENCAP_TYPE_NVGRE | EFX_ENCAP_FLAG_IPV6] = |
952 | EFX_EF10_NVGRE6_UCDEF, |
953 | [EFX_ENCAP_TYPE_GENEVE | EFX_ENCAP_FLAG_IPV6] = |
954 | EFX_EF10_GENEVE6_UCDEF, |
955 | }; |
956 | |
957 | /* quick bounds check (BCAST result impossible) */ |
958 | BUILD_BUG_ON(EFX_EF10_BCAST != 0); |
959 | if (encap_type >= ARRAY_SIZE(map) || map[encap_type] == 0) { |
960 | WARN_ON(1); |
961 | return -EINVAL; |
962 | } |
963 | /* then follow map */ |
964 | id = &vlan->default_filters[map[encap_type]]; |
965 | EFX_WARN_ON_PARANOID(*id != EFX_EF10_FILTER_ID_INVALID); |
966 | *id = rc; |
967 | rc = 0; |
968 | } |
969 | return rc; |
970 | } |
971 | |
972 | /* |
973 | * Caller must hold efx->filter_sem for read if race against |
974 | * efx_mcdi_filter_table_remove() is possible |
975 | */ |
976 | static void efx_mcdi_filter_vlan_sync_rx_mode(struct efx_nic *efx, |
977 | struct efx_mcdi_filter_vlan *vlan) |
978 | { |
979 | struct efx_mcdi_filter_table *table = efx->filter_state; |
980 | |
981 | /* |
982 | * Do not install unspecified VID if VLAN filtering is enabled. |
983 | * Do not install all specified VIDs if VLAN filtering is disabled. |
984 | */ |
985 | if ((vlan->vid == EFX_FILTER_VID_UNSPEC) == table->vlan_filter) |
986 | return; |
987 | |
988 | /* Insert/renew unicast filters */ |
989 | if (table->uc_promisc) { |
990 | efx_mcdi_filter_insert_def(efx, vlan, encap_type: EFX_ENCAP_TYPE_NONE, |
991 | multicast: false, rollback: false); |
992 | efx_mcdi_filter_insert_addr_list(efx, vlan, multicast: false, rollback: false); |
993 | } else { |
994 | /* |
995 | * If any of the filters failed to insert, fall back to |
996 | * promiscuous mode - add in the uc_def filter. But keep |
997 | * our individual unicast filters. |
998 | */ |
999 | if (efx_mcdi_filter_insert_addr_list(efx, vlan, multicast: false, rollback: false)) |
1000 | efx_mcdi_filter_insert_def(efx, vlan, |
1001 | encap_type: EFX_ENCAP_TYPE_NONE, |
1002 | multicast: false, rollback: false); |
1003 | } |
1004 | efx_mcdi_filter_insert_def(efx, vlan, encap_type: EFX_ENCAP_TYPE_VXLAN, |
1005 | multicast: false, rollback: false); |
1006 | efx_mcdi_filter_insert_def(efx, vlan, encap_type: EFX_ENCAP_TYPE_VXLAN | |
1007 | EFX_ENCAP_FLAG_IPV6, |
1008 | multicast: false, rollback: false); |
1009 | efx_mcdi_filter_insert_def(efx, vlan, encap_type: EFX_ENCAP_TYPE_NVGRE, |
1010 | multicast: false, rollback: false); |
1011 | efx_mcdi_filter_insert_def(efx, vlan, encap_type: EFX_ENCAP_TYPE_NVGRE | |
1012 | EFX_ENCAP_FLAG_IPV6, |
1013 | multicast: false, rollback: false); |
1014 | efx_mcdi_filter_insert_def(efx, vlan, encap_type: EFX_ENCAP_TYPE_GENEVE, |
1015 | multicast: false, rollback: false); |
1016 | efx_mcdi_filter_insert_def(efx, vlan, encap_type: EFX_ENCAP_TYPE_GENEVE | |
1017 | EFX_ENCAP_FLAG_IPV6, |
1018 | multicast: false, rollback: false); |
1019 | |
1020 | /* |
1021 | * Insert/renew multicast filters |
1022 | * |
1023 | * If changing promiscuous state with cascaded multicast filters, remove |
1024 | * old filters first, so that packets are dropped rather than duplicated |
1025 | */ |
1026 | if (table->mc_chaining && table->mc_promisc_last != table->mc_promisc) |
1027 | efx_mcdi_filter_remove_old(efx); |
1028 | if (table->mc_promisc) { |
1029 | if (table->mc_chaining) { |
1030 | /* |
1031 | * If we failed to insert promiscuous filters, rollback |
1032 | * and fall back to individual multicast filters |
1033 | */ |
1034 | if (efx_mcdi_filter_insert_def(efx, vlan, |
1035 | encap_type: EFX_ENCAP_TYPE_NONE, |
1036 | multicast: true, rollback: true)) { |
1037 | /* Changing promisc state, so remove old filters */ |
1038 | efx_mcdi_filter_remove_old(efx); |
1039 | efx_mcdi_filter_insert_addr_list(efx, vlan, |
1040 | multicast: true, rollback: false); |
1041 | } |
1042 | } else { |
1043 | /* |
1044 | * If we failed to insert promiscuous filters, don't |
1045 | * rollback. Regardless, also insert the mc_list, |
1046 | * unless it's incomplete due to overflow |
1047 | */ |
1048 | efx_mcdi_filter_insert_def(efx, vlan, |
1049 | encap_type: EFX_ENCAP_TYPE_NONE, |
1050 | multicast: true, rollback: false); |
1051 | if (!table->mc_overflow) |
1052 | efx_mcdi_filter_insert_addr_list(efx, vlan, |
1053 | multicast: true, rollback: false); |
1054 | } |
1055 | } else { |
1056 | /* |
1057 | * If any filters failed to insert, rollback and fall back to |
1058 | * promiscuous mode - mc_def filter and maybe broadcast. If |
1059 | * that fails, roll back again and insert as many of our |
1060 | * individual multicast filters as we can. |
1061 | */ |
1062 | if (efx_mcdi_filter_insert_addr_list(efx, vlan, multicast: true, rollback: true)) { |
1063 | /* Changing promisc state, so remove old filters */ |
1064 | if (table->mc_chaining) |
1065 | efx_mcdi_filter_remove_old(efx); |
1066 | if (efx_mcdi_filter_insert_def(efx, vlan, |
1067 | encap_type: EFX_ENCAP_TYPE_NONE, |
1068 | multicast: true, rollback: true)) |
1069 | efx_mcdi_filter_insert_addr_list(efx, vlan, |
1070 | multicast: true, rollback: false); |
1071 | } |
1072 | } |
1073 | efx_mcdi_filter_insert_def(efx, vlan, encap_type: EFX_ENCAP_TYPE_VXLAN, |
1074 | multicast: true, rollback: false); |
1075 | efx_mcdi_filter_insert_def(efx, vlan, encap_type: EFX_ENCAP_TYPE_VXLAN | |
1076 | EFX_ENCAP_FLAG_IPV6, |
1077 | multicast: true, rollback: false); |
1078 | efx_mcdi_filter_insert_def(efx, vlan, encap_type: EFX_ENCAP_TYPE_NVGRE, |
1079 | multicast: true, rollback: false); |
1080 | efx_mcdi_filter_insert_def(efx, vlan, encap_type: EFX_ENCAP_TYPE_NVGRE | |
1081 | EFX_ENCAP_FLAG_IPV6, |
1082 | multicast: true, rollback: false); |
1083 | efx_mcdi_filter_insert_def(efx, vlan, encap_type: EFX_ENCAP_TYPE_GENEVE, |
1084 | multicast: true, rollback: false); |
1085 | efx_mcdi_filter_insert_def(efx, vlan, encap_type: EFX_ENCAP_TYPE_GENEVE | |
1086 | EFX_ENCAP_FLAG_IPV6, |
1087 | multicast: true, rollback: false); |
1088 | } |
1089 | |
1090 | int efx_mcdi_filter_clear_rx(struct efx_nic *efx, |
1091 | enum efx_filter_priority priority) |
1092 | { |
1093 | struct efx_mcdi_filter_table *table; |
1094 | unsigned int priority_mask; |
1095 | unsigned int i; |
1096 | int rc; |
1097 | |
1098 | priority_mask = (((1U << (priority + 1)) - 1) & |
1099 | ~(1U << EFX_FILTER_PRI_AUTO)); |
1100 | |
1101 | down_read(sem: &efx->filter_sem); |
1102 | table = efx->filter_state; |
1103 | down_write(sem: &table->lock); |
1104 | for (i = 0; i < EFX_MCDI_FILTER_TBL_ROWS; i++) { |
1105 | rc = efx_mcdi_filter_remove_internal(efx, priority_mask, |
1106 | filter_id: i, by_index: true); |
1107 | if (rc && rc != -ENOENT) |
1108 | break; |
1109 | rc = 0; |
1110 | } |
1111 | |
1112 | up_write(sem: &table->lock); |
1113 | up_read(sem: &efx->filter_sem); |
1114 | return rc; |
1115 | } |
1116 | |
1117 | u32 efx_mcdi_filter_count_rx_used(struct efx_nic *efx, |
1118 | enum efx_filter_priority priority) |
1119 | { |
1120 | struct efx_mcdi_filter_table *table; |
1121 | unsigned int filter_idx; |
1122 | s32 count = 0; |
1123 | |
1124 | down_read(sem: &efx->filter_sem); |
1125 | table = efx->filter_state; |
1126 | down_read(sem: &table->lock); |
1127 | for (filter_idx = 0; filter_idx < EFX_MCDI_FILTER_TBL_ROWS; filter_idx++) { |
1128 | if (table->entry[filter_idx].spec && |
1129 | efx_mcdi_filter_entry_spec(table, filter_idx)->priority == |
1130 | priority) |
1131 | ++count; |
1132 | } |
1133 | up_read(sem: &table->lock); |
1134 | up_read(sem: &efx->filter_sem); |
1135 | return count; |
1136 | } |
1137 | |
1138 | u32 efx_mcdi_filter_get_rx_id_limit(struct efx_nic *efx) |
1139 | { |
1140 | struct efx_mcdi_filter_table *table = efx->filter_state; |
1141 | |
1142 | return table->rx_match_count * EFX_MCDI_FILTER_TBL_ROWS * 2; |
1143 | } |
1144 | |
1145 | s32 efx_mcdi_filter_get_rx_ids(struct efx_nic *efx, |
1146 | enum efx_filter_priority priority, |
1147 | u32 *buf, u32 size) |
1148 | { |
1149 | struct efx_mcdi_filter_table *table; |
1150 | struct efx_filter_spec *spec; |
1151 | unsigned int filter_idx; |
1152 | s32 count = 0; |
1153 | |
1154 | down_read(sem: &efx->filter_sem); |
1155 | table = efx->filter_state; |
1156 | down_read(sem: &table->lock); |
1157 | |
1158 | for (filter_idx = 0; filter_idx < EFX_MCDI_FILTER_TBL_ROWS; filter_idx++) { |
1159 | spec = efx_mcdi_filter_entry_spec(table, filter_idx); |
1160 | if (spec && spec->priority == priority) { |
1161 | if (count == size) { |
1162 | count = -EMSGSIZE; |
1163 | break; |
1164 | } |
1165 | buf[count++] = |
1166 | efx_mcdi_filter_make_filter_id( |
1167 | pri: efx_mcdi_filter_pri(table, spec), |
1168 | idx: filter_idx); |
1169 | } |
1170 | } |
1171 | up_read(sem: &table->lock); |
1172 | up_read(sem: &efx->filter_sem); |
1173 | return count; |
1174 | } |
1175 | |
1176 | static int efx_mcdi_filter_match_flags_from_mcdi(bool encap, u32 mcdi_flags) |
1177 | { |
1178 | int match_flags = 0; |
1179 | |
1180 | #define MAP_FLAG(gen_flag, mcdi_field) do { \ |
1181 | u32 old_mcdi_flags = mcdi_flags; \ |
1182 | mcdi_flags &= ~(1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_ ## \ |
1183 | mcdi_field ## _LBN); \ |
1184 | if (mcdi_flags != old_mcdi_flags) \ |
1185 | match_flags |= EFX_FILTER_MATCH_ ## gen_flag; \ |
1186 | } while (0) |
1187 | |
1188 | if (encap) { |
1189 | /* encap filters must specify encap type */ |
1190 | match_flags |= EFX_FILTER_MATCH_ENCAP_TYPE; |
1191 | /* and imply ethertype and ip proto */ |
1192 | mcdi_flags &= |
1193 | ~(1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_IP_PROTO_LBN); |
1194 | mcdi_flags &= |
1195 | ~(1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_ETHER_TYPE_LBN); |
1196 | /* VLAN tags refer to the outer packet */ |
1197 | MAP_FLAG(INNER_VID, INNER_VLAN); |
1198 | MAP_FLAG(OUTER_VID, OUTER_VLAN); |
1199 | /* everything else refers to the inner packet */ |
1200 | MAP_FLAG(LOC_MAC_IG, IFRM_UNKNOWN_UCAST_DST); |
1201 | MAP_FLAG(LOC_MAC_IG, IFRM_UNKNOWN_MCAST_DST); |
1202 | MAP_FLAG(REM_HOST, IFRM_SRC_IP); |
1203 | MAP_FLAG(LOC_HOST, IFRM_DST_IP); |
1204 | MAP_FLAG(REM_MAC, IFRM_SRC_MAC); |
1205 | MAP_FLAG(REM_PORT, IFRM_SRC_PORT); |
1206 | MAP_FLAG(LOC_MAC, IFRM_DST_MAC); |
1207 | MAP_FLAG(LOC_PORT, IFRM_DST_PORT); |
1208 | MAP_FLAG(ETHER_TYPE, IFRM_ETHER_TYPE); |
1209 | MAP_FLAG(IP_PROTO, IFRM_IP_PROTO); |
1210 | } else { |
1211 | MAP_FLAG(LOC_MAC_IG, UNKNOWN_UCAST_DST); |
1212 | MAP_FLAG(LOC_MAC_IG, UNKNOWN_MCAST_DST); |
1213 | MAP_FLAG(REM_HOST, SRC_IP); |
1214 | MAP_FLAG(LOC_HOST, DST_IP); |
1215 | MAP_FLAG(REM_MAC, SRC_MAC); |
1216 | MAP_FLAG(REM_PORT, SRC_PORT); |
1217 | MAP_FLAG(LOC_MAC, DST_MAC); |
1218 | MAP_FLAG(LOC_PORT, DST_PORT); |
1219 | MAP_FLAG(ETHER_TYPE, ETHER_TYPE); |
1220 | MAP_FLAG(INNER_VID, INNER_VLAN); |
1221 | MAP_FLAG(OUTER_VID, OUTER_VLAN); |
1222 | MAP_FLAG(IP_PROTO, IP_PROTO); |
1223 | } |
1224 | #undef MAP_FLAG |
1225 | |
1226 | /* Did we map them all? */ |
1227 | if (mcdi_flags) |
1228 | return -EINVAL; |
1229 | |
1230 | return match_flags; |
1231 | } |
1232 | |
1233 | bool efx_mcdi_filter_match_supported(struct efx_mcdi_filter_table *table, |
1234 | bool encap, |
1235 | enum efx_filter_match_flags match_flags) |
1236 | { |
1237 | unsigned int match_pri; |
1238 | int mf; |
1239 | |
1240 | for (match_pri = 0; |
1241 | match_pri < table->rx_match_count; |
1242 | match_pri++) { |
1243 | mf = efx_mcdi_filter_match_flags_from_mcdi(encap, |
1244 | mcdi_flags: table->rx_match_mcdi_flags[match_pri]); |
1245 | if (mf == match_flags) |
1246 | return true; |
1247 | } |
1248 | |
1249 | return false; |
1250 | } |
1251 | |
1252 | static int |
1253 | efx_mcdi_filter_table_probe_matches(struct efx_nic *efx, |
1254 | struct efx_mcdi_filter_table *table, |
1255 | bool encap) |
1256 | { |
1257 | MCDI_DECLARE_BUF(inbuf, MC_CMD_GET_PARSER_DISP_INFO_IN_LEN); |
1258 | MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_PARSER_DISP_INFO_OUT_LENMAX); |
1259 | unsigned int pd_match_pri, pd_match_count; |
1260 | size_t outlen; |
1261 | int rc; |
1262 | |
1263 | /* Find out which RX filter types are supported, and their priorities */ |
1264 | MCDI_SET_DWORD(inbuf, GET_PARSER_DISP_INFO_IN_OP, |
1265 | encap ? |
1266 | MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SUPPORTED_ENCAP_RX_MATCHES : |
1267 | MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SUPPORTED_RX_MATCHES); |
1268 | rc = efx_mcdi_rpc(efx, MC_CMD_GET_PARSER_DISP_INFO, |
1269 | inbuf, inlen: sizeof(inbuf), outbuf, outlen: sizeof(outbuf), |
1270 | outlen_actual: &outlen); |
1271 | if (rc) |
1272 | return rc; |
1273 | |
1274 | pd_match_count = MCDI_VAR_ARRAY_LEN( |
1275 | outlen, GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES); |
1276 | |
1277 | for (pd_match_pri = 0; pd_match_pri < pd_match_count; pd_match_pri++) { |
1278 | u32 mcdi_flags = |
1279 | MCDI_ARRAY_DWORD( |
1280 | outbuf, |
1281 | GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES, |
1282 | pd_match_pri); |
1283 | rc = efx_mcdi_filter_match_flags_from_mcdi(encap, mcdi_flags); |
1284 | if (rc < 0) { |
1285 | netif_dbg(efx, probe, efx->net_dev, |
1286 | "%s: fw flags %#x pri %u not supported in driver\n" , |
1287 | __func__, mcdi_flags, pd_match_pri); |
1288 | } else { |
1289 | netif_dbg(efx, probe, efx->net_dev, |
1290 | "%s: fw flags %#x pri %u supported as driver flags %#x pri %u\n" , |
1291 | __func__, mcdi_flags, pd_match_pri, |
1292 | rc, table->rx_match_count); |
1293 | table->rx_match_mcdi_flags[table->rx_match_count] = mcdi_flags; |
1294 | table->rx_match_count++; |
1295 | } |
1296 | } |
1297 | |
1298 | return 0; |
1299 | } |
1300 | |
1301 | int efx_mcdi_filter_table_probe(struct efx_nic *efx, bool multicast_chaining) |
1302 | { |
1303 | struct net_device *net_dev = efx->net_dev; |
1304 | struct efx_mcdi_filter_table *table; |
1305 | int rc; |
1306 | |
1307 | if (!efx_rwsem_assert_write_locked(sem: &efx->filter_sem)) |
1308 | return -EINVAL; |
1309 | |
1310 | if (efx->filter_state) /* already probed */ |
1311 | return 0; |
1312 | |
1313 | table = kzalloc(size: sizeof(*table), GFP_KERNEL); |
1314 | if (!table) |
1315 | return -ENOMEM; |
1316 | |
1317 | table->mc_chaining = multicast_chaining; |
1318 | table->rx_match_count = 0; |
1319 | rc = efx_mcdi_filter_table_probe_matches(efx, table, encap: false); |
1320 | if (rc) |
1321 | goto fail; |
1322 | if (efx_has_cap(efx, VXLAN_NVGRE)) |
1323 | rc = efx_mcdi_filter_table_probe_matches(efx, table, encap: true); |
1324 | if (rc) |
1325 | goto fail; |
1326 | if ((efx_supported_features(efx) & NETIF_F_HW_VLAN_CTAG_FILTER) && |
1327 | !(efx_mcdi_filter_match_supported(table, encap: false, |
1328 | match_flags: (EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC)) && |
1329 | efx_mcdi_filter_match_supported(table, encap: false, |
1330 | match_flags: (EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC_IG)))) { |
1331 | netif_info(efx, probe, net_dev, |
1332 | "VLAN filters are not supported in this firmware variant\n" ); |
1333 | net_dev->features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; |
1334 | efx->fixed_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; |
1335 | net_dev->hw_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; |
1336 | } |
1337 | |
1338 | table->entry = vzalloc(array_size(EFX_MCDI_FILTER_TBL_ROWS, |
1339 | sizeof(*table->entry))); |
1340 | if (!table->entry) { |
1341 | rc = -ENOMEM; |
1342 | goto fail; |
1343 | } |
1344 | |
1345 | table->mc_promisc_last = false; |
1346 | table->vlan_filter = |
1347 | !!(efx->net_dev->features & NETIF_F_HW_VLAN_CTAG_FILTER); |
1348 | INIT_LIST_HEAD(list: &table->vlan_list); |
1349 | init_rwsem(&table->lock); |
1350 | |
1351 | efx->filter_state = table; |
1352 | |
1353 | return 0; |
1354 | fail: |
1355 | kfree(objp: table); |
1356 | return rc; |
1357 | } |
1358 | |
1359 | void efx_mcdi_filter_table_reset_mc_allocations(struct efx_nic *efx) |
1360 | { |
1361 | struct efx_mcdi_filter_table *table = efx->filter_state; |
1362 | |
1363 | if (table) { |
1364 | table->must_restore_filters = true; |
1365 | table->must_restore_rss_contexts = true; |
1366 | } |
1367 | } |
1368 | |
1369 | /* |
1370 | * Caller must hold efx->filter_sem for read if race against |
1371 | * efx_mcdi_filter_table_remove() is possible |
1372 | */ |
1373 | void efx_mcdi_filter_table_restore(struct efx_nic *efx) |
1374 | { |
1375 | struct efx_mcdi_filter_table *table = efx->filter_state; |
1376 | unsigned int invalid_filters = 0, failed = 0; |
1377 | struct efx_mcdi_filter_vlan *vlan; |
1378 | struct efx_filter_spec *spec; |
1379 | struct efx_rss_context *ctx; |
1380 | unsigned int filter_idx; |
1381 | u32 mcdi_flags; |
1382 | int match_pri; |
1383 | int rc, i; |
1384 | |
1385 | WARN_ON(!rwsem_is_locked(&efx->filter_sem)); |
1386 | |
1387 | if (!table || !table->must_restore_filters) |
1388 | return; |
1389 | |
1390 | down_write(sem: &table->lock); |
1391 | mutex_lock(&efx->rss_lock); |
1392 | |
1393 | for (filter_idx = 0; filter_idx < EFX_MCDI_FILTER_TBL_ROWS; filter_idx++) { |
1394 | spec = efx_mcdi_filter_entry_spec(table, filter_idx); |
1395 | if (!spec) |
1396 | continue; |
1397 | |
1398 | mcdi_flags = efx_mcdi_filter_mcdi_flags_from_spec(spec); |
1399 | match_pri = 0; |
1400 | while (match_pri < table->rx_match_count && |
1401 | table->rx_match_mcdi_flags[match_pri] != mcdi_flags) |
1402 | ++match_pri; |
1403 | if (match_pri >= table->rx_match_count) { |
1404 | invalid_filters++; |
1405 | goto not_restored; |
1406 | } |
1407 | if (spec->rss_context) |
1408 | ctx = efx_find_rss_context_entry(efx, id: spec->rss_context); |
1409 | else |
1410 | ctx = &efx->rss_context; |
1411 | if (spec->flags & EFX_FILTER_FLAG_RX_RSS) { |
1412 | if (!ctx) { |
1413 | netif_warn(efx, drv, efx->net_dev, |
1414 | "Warning: unable to restore a filter with nonexistent RSS context %u.\n" , |
1415 | spec->rss_context); |
1416 | invalid_filters++; |
1417 | goto not_restored; |
1418 | } |
1419 | if (ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID) { |
1420 | netif_warn(efx, drv, efx->net_dev, |
1421 | "Warning: unable to restore a filter with RSS context %u as it was not created.\n" , |
1422 | spec->rss_context); |
1423 | invalid_filters++; |
1424 | goto not_restored; |
1425 | } |
1426 | } |
1427 | |
1428 | rc = efx_mcdi_filter_push(efx, spec, |
1429 | handle: &table->entry[filter_idx].handle, |
1430 | ctx, replacing: false); |
1431 | if (rc) |
1432 | failed++; |
1433 | |
1434 | if (rc) { |
1435 | not_restored: |
1436 | list_for_each_entry(vlan, &table->vlan_list, list) |
1437 | for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; ++i) |
1438 | if (vlan->default_filters[i] == filter_idx) |
1439 | vlan->default_filters[i] = |
1440 | EFX_EF10_FILTER_ID_INVALID; |
1441 | |
1442 | kfree(objp: spec); |
1443 | efx_mcdi_filter_set_entry(table, filter_idx, NULL, flags: 0); |
1444 | } |
1445 | } |
1446 | |
1447 | mutex_unlock(lock: &efx->rss_lock); |
1448 | up_write(sem: &table->lock); |
1449 | |
1450 | /* |
1451 | * This can happen validly if the MC's capabilities have changed, so |
1452 | * is not an error. |
1453 | */ |
1454 | if (invalid_filters) |
1455 | netif_dbg(efx, drv, efx->net_dev, |
1456 | "Did not restore %u filters that are now unsupported.\n" , |
1457 | invalid_filters); |
1458 | |
1459 | if (failed) |
1460 | netif_err(efx, hw, efx->net_dev, |
1461 | "unable to restore %u filters\n" , failed); |
1462 | else |
1463 | table->must_restore_filters = false; |
1464 | } |
1465 | |
1466 | void efx_mcdi_filter_table_down(struct efx_nic *efx) |
1467 | { |
1468 | struct efx_mcdi_filter_table *table = efx->filter_state; |
1469 | MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_EXT_IN_LEN); |
1470 | struct efx_filter_spec *spec; |
1471 | unsigned int filter_idx; |
1472 | int rc; |
1473 | |
1474 | if (!table) |
1475 | return; |
1476 | |
1477 | efx_mcdi_filter_cleanup_vlans(efx); |
1478 | |
1479 | for (filter_idx = 0; filter_idx < EFX_MCDI_FILTER_TBL_ROWS; filter_idx++) { |
1480 | spec = efx_mcdi_filter_entry_spec(table, filter_idx); |
1481 | if (!spec) |
1482 | continue; |
1483 | |
1484 | MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP, |
1485 | efx_mcdi_filter_is_exclusive(spec) ? |
1486 | MC_CMD_FILTER_OP_IN_OP_REMOVE : |
1487 | MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE); |
1488 | MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE, |
1489 | table->entry[filter_idx].handle); |
1490 | rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FILTER_OP, inbuf, |
1491 | inlen: sizeof(inbuf), NULL, outlen: 0, NULL); |
1492 | if (rc) |
1493 | netif_info(efx, drv, efx->net_dev, |
1494 | "%s: filter %04x remove failed\n" , |
1495 | __func__, filter_idx); |
1496 | kfree(objp: spec); |
1497 | } |
1498 | } |
1499 | |
1500 | void efx_mcdi_filter_table_remove(struct efx_nic *efx) |
1501 | { |
1502 | struct efx_mcdi_filter_table *table = efx->filter_state; |
1503 | |
1504 | efx_mcdi_filter_table_down(efx); |
1505 | |
1506 | efx->filter_state = NULL; |
1507 | /* |
1508 | * If we were called without locking, then it's not safe to free |
1509 | * the table as others might be using it. So we just WARN, leak |
1510 | * the memory, and potentially get an inconsistent filter table |
1511 | * state. |
1512 | * This should never actually happen. |
1513 | */ |
1514 | if (!efx_rwsem_assert_write_locked(sem: &efx->filter_sem)) |
1515 | return; |
1516 | |
1517 | if (!table) |
1518 | return; |
1519 | |
1520 | vfree(addr: table->entry); |
1521 | kfree(objp: table); |
1522 | } |
1523 | |
1524 | static void efx_mcdi_filter_mark_one_old(struct efx_nic *efx, uint16_t *id) |
1525 | { |
1526 | struct efx_mcdi_filter_table *table = efx->filter_state; |
1527 | unsigned int filter_idx; |
1528 | |
1529 | efx_rwsem_assert_write_locked(sem: &table->lock); |
1530 | |
1531 | if (*id != EFX_EF10_FILTER_ID_INVALID) { |
1532 | filter_idx = efx_mcdi_filter_get_unsafe_id(filter_id: *id); |
1533 | if (!table->entry[filter_idx].spec) |
1534 | netif_dbg(efx, drv, efx->net_dev, |
1535 | "marked null spec old %04x:%04x\n" , *id, |
1536 | filter_idx); |
1537 | table->entry[filter_idx].spec |= EFX_EF10_FILTER_FLAG_AUTO_OLD; |
1538 | *id = EFX_EF10_FILTER_ID_INVALID; |
1539 | } |
1540 | } |
1541 | |
1542 | /* Mark old per-VLAN filters that may need to be removed */ |
1543 | static void _efx_mcdi_filter_vlan_mark_old(struct efx_nic *efx, |
1544 | struct efx_mcdi_filter_vlan *vlan) |
1545 | { |
1546 | struct efx_mcdi_filter_table *table = efx->filter_state; |
1547 | unsigned int i; |
1548 | |
1549 | for (i = 0; i < table->dev_uc_count; i++) |
1550 | efx_mcdi_filter_mark_one_old(efx, id: &vlan->uc[i]); |
1551 | for (i = 0; i < table->dev_mc_count; i++) |
1552 | efx_mcdi_filter_mark_one_old(efx, id: &vlan->mc[i]); |
1553 | for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; i++) |
1554 | efx_mcdi_filter_mark_one_old(efx, id: &vlan->default_filters[i]); |
1555 | } |
1556 | |
1557 | /* |
1558 | * Mark old filters that may need to be removed. |
1559 | * Caller must hold efx->filter_sem for read if race against |
1560 | * efx_mcdi_filter_table_remove() is possible |
1561 | */ |
1562 | static void efx_mcdi_filter_mark_old(struct efx_nic *efx) |
1563 | { |
1564 | struct efx_mcdi_filter_table *table = efx->filter_state; |
1565 | struct efx_mcdi_filter_vlan *vlan; |
1566 | |
1567 | down_write(sem: &table->lock); |
1568 | list_for_each_entry(vlan, &table->vlan_list, list) |
1569 | _efx_mcdi_filter_vlan_mark_old(efx, vlan); |
1570 | up_write(sem: &table->lock); |
1571 | } |
1572 | |
1573 | int efx_mcdi_filter_add_vlan(struct efx_nic *efx, u16 vid) |
1574 | { |
1575 | struct efx_mcdi_filter_table *table = efx->filter_state; |
1576 | struct efx_mcdi_filter_vlan *vlan; |
1577 | unsigned int i; |
1578 | |
1579 | if (!efx_rwsem_assert_write_locked(sem: &efx->filter_sem)) |
1580 | return -EINVAL; |
1581 | |
1582 | vlan = efx_mcdi_filter_find_vlan(efx, vid); |
1583 | if (WARN_ON(vlan)) { |
1584 | netif_err(efx, drv, efx->net_dev, |
1585 | "VLAN %u already added\n" , vid); |
1586 | return -EALREADY; |
1587 | } |
1588 | |
1589 | vlan = kzalloc(size: sizeof(*vlan), GFP_KERNEL); |
1590 | if (!vlan) |
1591 | return -ENOMEM; |
1592 | |
1593 | vlan->vid = vid; |
1594 | |
1595 | for (i = 0; i < ARRAY_SIZE(vlan->uc); i++) |
1596 | vlan->uc[i] = EFX_EF10_FILTER_ID_INVALID; |
1597 | for (i = 0; i < ARRAY_SIZE(vlan->mc); i++) |
1598 | vlan->mc[i] = EFX_EF10_FILTER_ID_INVALID; |
1599 | for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; i++) |
1600 | vlan->default_filters[i] = EFX_EF10_FILTER_ID_INVALID; |
1601 | |
1602 | list_add_tail(new: &vlan->list, head: &table->vlan_list); |
1603 | |
1604 | if (efx_dev_registered(efx)) |
1605 | efx_mcdi_filter_vlan_sync_rx_mode(efx, vlan); |
1606 | |
1607 | return 0; |
1608 | } |
1609 | |
1610 | static void efx_mcdi_filter_del_vlan_internal(struct efx_nic *efx, |
1611 | struct efx_mcdi_filter_vlan *vlan) |
1612 | { |
1613 | unsigned int i; |
1614 | |
1615 | /* See comment in efx_mcdi_filter_table_remove() */ |
1616 | if (!efx_rwsem_assert_write_locked(sem: &efx->filter_sem)) |
1617 | return; |
1618 | |
1619 | list_del(entry: &vlan->list); |
1620 | |
1621 | for (i = 0; i < ARRAY_SIZE(vlan->uc); i++) |
1622 | efx_mcdi_filter_remove_unsafe(efx, priority: EFX_FILTER_PRI_AUTO, |
1623 | filter_id: vlan->uc[i]); |
1624 | for (i = 0; i < ARRAY_SIZE(vlan->mc); i++) |
1625 | efx_mcdi_filter_remove_unsafe(efx, priority: EFX_FILTER_PRI_AUTO, |
1626 | filter_id: vlan->mc[i]); |
1627 | for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; i++) |
1628 | if (vlan->default_filters[i] != EFX_EF10_FILTER_ID_INVALID) |
1629 | efx_mcdi_filter_remove_unsafe(efx, priority: EFX_FILTER_PRI_AUTO, |
1630 | filter_id: vlan->default_filters[i]); |
1631 | |
1632 | kfree(objp: vlan); |
1633 | } |
1634 | |
1635 | void efx_mcdi_filter_del_vlan(struct efx_nic *efx, u16 vid) |
1636 | { |
1637 | struct efx_mcdi_filter_vlan *vlan; |
1638 | |
1639 | /* See comment in efx_mcdi_filter_table_remove() */ |
1640 | if (!efx_rwsem_assert_write_locked(sem: &efx->filter_sem)) |
1641 | return; |
1642 | |
1643 | vlan = efx_mcdi_filter_find_vlan(efx, vid); |
1644 | if (!vlan) { |
1645 | netif_err(efx, drv, efx->net_dev, |
1646 | "VLAN %u not found in filter state\n" , vid); |
1647 | return; |
1648 | } |
1649 | |
1650 | efx_mcdi_filter_del_vlan_internal(efx, vlan); |
1651 | } |
1652 | |
1653 | struct efx_mcdi_filter_vlan *efx_mcdi_filter_find_vlan(struct efx_nic *efx, |
1654 | u16 vid) |
1655 | { |
1656 | struct efx_mcdi_filter_table *table = efx->filter_state; |
1657 | struct efx_mcdi_filter_vlan *vlan; |
1658 | |
1659 | WARN_ON(!rwsem_is_locked(&efx->filter_sem)); |
1660 | |
1661 | list_for_each_entry(vlan, &table->vlan_list, list) { |
1662 | if (vlan->vid == vid) |
1663 | return vlan; |
1664 | } |
1665 | |
1666 | return NULL; |
1667 | } |
1668 | |
1669 | void efx_mcdi_filter_cleanup_vlans(struct efx_nic *efx) |
1670 | { |
1671 | struct efx_mcdi_filter_table *table = efx->filter_state; |
1672 | struct efx_mcdi_filter_vlan *vlan, *next_vlan; |
1673 | |
1674 | /* See comment in efx_mcdi_filter_table_remove() */ |
1675 | if (!efx_rwsem_assert_write_locked(sem: &efx->filter_sem)) |
1676 | return; |
1677 | |
1678 | if (!table) |
1679 | return; |
1680 | |
1681 | list_for_each_entry_safe(vlan, next_vlan, &table->vlan_list, list) |
1682 | efx_mcdi_filter_del_vlan_internal(efx, vlan); |
1683 | } |
1684 | |
1685 | static void efx_mcdi_filter_uc_addr_list(struct efx_nic *efx) |
1686 | { |
1687 | struct efx_mcdi_filter_table *table = efx->filter_state; |
1688 | struct net_device *net_dev = efx->net_dev; |
1689 | struct netdev_hw_addr *uc; |
1690 | unsigned int i; |
1691 | |
1692 | table->uc_promisc = !!(net_dev->flags & IFF_PROMISC); |
1693 | ether_addr_copy(dst: table->dev_uc_list[0].addr, src: net_dev->dev_addr); |
1694 | i = 1; |
1695 | netdev_for_each_uc_addr(uc, net_dev) { |
1696 | if (i >= EFX_EF10_FILTER_DEV_UC_MAX) { |
1697 | table->uc_promisc = true; |
1698 | break; |
1699 | } |
1700 | ether_addr_copy(dst: table->dev_uc_list[i].addr, src: uc->addr); |
1701 | i++; |
1702 | } |
1703 | |
1704 | table->dev_uc_count = i; |
1705 | } |
1706 | |
1707 | static void efx_mcdi_filter_mc_addr_list(struct efx_nic *efx) |
1708 | { |
1709 | struct efx_mcdi_filter_table *table = efx->filter_state; |
1710 | struct net_device *net_dev = efx->net_dev; |
1711 | struct netdev_hw_addr *mc; |
1712 | unsigned int i; |
1713 | |
1714 | table->mc_overflow = false; |
1715 | table->mc_promisc = !!(net_dev->flags & (IFF_PROMISC | IFF_ALLMULTI)); |
1716 | |
1717 | i = 0; |
1718 | netdev_for_each_mc_addr(mc, net_dev) { |
1719 | if (i >= EFX_EF10_FILTER_DEV_MC_MAX) { |
1720 | table->mc_promisc = true; |
1721 | table->mc_overflow = true; |
1722 | break; |
1723 | } |
1724 | ether_addr_copy(dst: table->dev_mc_list[i].addr, src: mc->addr); |
1725 | i++; |
1726 | } |
1727 | |
1728 | table->dev_mc_count = i; |
1729 | } |
1730 | |
1731 | /* |
1732 | * Caller must hold efx->filter_sem for read if race against |
1733 | * efx_mcdi_filter_table_remove() is possible |
1734 | */ |
1735 | void efx_mcdi_filter_sync_rx_mode(struct efx_nic *efx) |
1736 | { |
1737 | struct efx_mcdi_filter_table *table = efx->filter_state; |
1738 | struct net_device *net_dev = efx->net_dev; |
1739 | struct efx_mcdi_filter_vlan *vlan; |
1740 | bool vlan_filter; |
1741 | |
1742 | if (!efx_dev_registered(efx)) |
1743 | return; |
1744 | |
1745 | if (!table) |
1746 | return; |
1747 | |
1748 | efx_mcdi_filter_mark_old(efx); |
1749 | |
1750 | /* |
1751 | * Copy/convert the address lists; add the primary station |
1752 | * address and broadcast address |
1753 | */ |
1754 | netif_addr_lock_bh(dev: net_dev); |
1755 | efx_mcdi_filter_uc_addr_list(efx); |
1756 | efx_mcdi_filter_mc_addr_list(efx); |
1757 | netif_addr_unlock_bh(dev: net_dev); |
1758 | |
1759 | /* |
1760 | * If VLAN filtering changes, all old filters are finally removed. |
1761 | * Do it in advance to avoid conflicts for unicast untagged and |
1762 | * VLAN 0 tagged filters. |
1763 | */ |
1764 | vlan_filter = !!(net_dev->features & NETIF_F_HW_VLAN_CTAG_FILTER); |
1765 | if (table->vlan_filter != vlan_filter) { |
1766 | table->vlan_filter = vlan_filter; |
1767 | efx_mcdi_filter_remove_old(efx); |
1768 | } |
1769 | |
1770 | list_for_each_entry(vlan, &table->vlan_list, list) |
1771 | efx_mcdi_filter_vlan_sync_rx_mode(efx, vlan); |
1772 | |
1773 | efx_mcdi_filter_remove_old(efx); |
1774 | table->mc_promisc_last = table->mc_promisc; |
1775 | } |
1776 | |
1777 | #ifdef CONFIG_RFS_ACCEL |
1778 | |
1779 | bool efx_mcdi_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id, |
1780 | unsigned int filter_idx) |
1781 | { |
1782 | struct efx_filter_spec *spec, saved_spec; |
1783 | struct efx_mcdi_filter_table *table; |
1784 | struct efx_arfs_rule *rule = NULL; |
1785 | bool ret = true, force = false; |
1786 | u16 arfs_id; |
1787 | |
1788 | down_read(sem: &efx->filter_sem); |
1789 | table = efx->filter_state; |
1790 | down_write(sem: &table->lock); |
1791 | spec = efx_mcdi_filter_entry_spec(table, filter_idx); |
1792 | |
1793 | if (!spec || spec->priority != EFX_FILTER_PRI_HINT) |
1794 | goto out_unlock; |
1795 | |
1796 | spin_lock_bh(lock: &efx->rps_hash_lock); |
1797 | if (!efx->rps_hash_table) { |
1798 | /* In the absence of the table, we always return 0 to ARFS. */ |
1799 | arfs_id = 0; |
1800 | } else { |
1801 | rule = efx_rps_hash_find(efx, spec); |
1802 | if (!rule) |
1803 | /* ARFS table doesn't know of this filter, so remove it */ |
1804 | goto expire; |
1805 | arfs_id = rule->arfs_id; |
1806 | ret = efx_rps_check_rule(rule, filter_idx, force: &force); |
1807 | if (force) |
1808 | goto expire; |
1809 | if (!ret) { |
1810 | spin_unlock_bh(lock: &efx->rps_hash_lock); |
1811 | goto out_unlock; |
1812 | } |
1813 | } |
1814 | if (!rps_may_expire_flow(dev: efx->net_dev, rxq_index: spec->dmaq_id, flow_id, filter_id: arfs_id)) |
1815 | ret = false; |
1816 | else if (rule) |
1817 | rule->filter_id = EFX_ARFS_FILTER_ID_REMOVING; |
1818 | expire: |
1819 | saved_spec = *spec; /* remove operation will kfree spec */ |
1820 | spin_unlock_bh(lock: &efx->rps_hash_lock); |
1821 | /* |
1822 | * At this point (since we dropped the lock), another thread might queue |
1823 | * up a fresh insertion request (but the actual insertion will be held |
1824 | * up by our possession of the filter table lock). In that case, it |
1825 | * will set rule->filter_id to EFX_ARFS_FILTER_ID_PENDING, meaning that |
1826 | * the rule is not removed by efx_rps_hash_del() below. |
1827 | */ |
1828 | if (ret) |
1829 | ret = efx_mcdi_filter_remove_internal(efx, priority_mask: 1U << spec->priority, |
1830 | filter_id: filter_idx, by_index: true) == 0; |
1831 | /* |
1832 | * While we can't safely dereference rule (we dropped the lock), we can |
1833 | * still test it for NULL. |
1834 | */ |
1835 | if (ret && rule) { |
1836 | /* Expiring, so remove entry from ARFS table */ |
1837 | spin_lock_bh(lock: &efx->rps_hash_lock); |
1838 | efx_rps_hash_del(efx, spec: &saved_spec); |
1839 | spin_unlock_bh(lock: &efx->rps_hash_lock); |
1840 | } |
1841 | out_unlock: |
1842 | up_write(sem: &table->lock); |
1843 | up_read(sem: &efx->filter_sem); |
1844 | return ret; |
1845 | } |
1846 | |
1847 | #endif /* CONFIG_RFS_ACCEL */ |
1848 | |
1849 | #define (1 << RSS_MODE_HASH_SRC_ADDR_LBN |\ |
1850 | 1 << RSS_MODE_HASH_DST_ADDR_LBN) |
1851 | #define (1 << RSS_MODE_HASH_SRC_PORT_LBN |\ |
1852 | 1 << RSS_MODE_HASH_DST_PORT_LBN) |
1853 | #define (1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_IPV4_EN_LBN |\ |
1854 | 1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_TCPV4_EN_LBN |\ |
1855 | 1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_IPV6_EN_LBN |\ |
1856 | 1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_TCPV6_EN_LBN |\ |
1857 | (RSS_MODE_HASH_ADDRS | RSS_MODE_HASH_PORTS) << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TCP_IPV4_RSS_MODE_LBN |\ |
1858 | RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV4_RSS_MODE_LBN |\ |
1859 | RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_OTHER_IPV4_RSS_MODE_LBN |\ |
1860 | (RSS_MODE_HASH_ADDRS | RSS_MODE_HASH_PORTS) << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TCP_IPV6_RSS_MODE_LBN |\ |
1861 | RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV6_RSS_MODE_LBN |\ |
1862 | RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_OTHER_IPV6_RSS_MODE_LBN) |
1863 | |
1864 | int (struct efx_nic *efx, u32 context, u32 *flags) |
1865 | { |
1866 | /* |
1867 | * Firmware had a bug (sfc bug 61952) where it would not actually |
1868 | * fill in the flags field in the response to MC_CMD_RSS_CONTEXT_GET_FLAGS. |
1869 | * This meant that it would always contain whatever was previously |
1870 | * in the MCDI buffer. Fortunately, all firmware versions with |
1871 | * this bug have the same default flags value for a newly-allocated |
1872 | * RSS context, and the only time we want to get the flags is just |
1873 | * after allocating. Moreover, the response has a 32-bit hole |
1874 | * where the context ID would be in the request, so we can use an |
1875 | * overlength buffer in the request and pre-fill the flags field |
1876 | * with what we believe the default to be. Thus if the firmware |
1877 | * has the bug, it will leave our pre-filled value in the flags |
1878 | * field of the response, and we will get the right answer. |
1879 | * |
1880 | * However, this does mean that this function should NOT be used if |
1881 | * the RSS context flags might not be their defaults - it is ONLY |
1882 | * reliably correct for a newly-allocated RSS context. |
1883 | */ |
1884 | MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_LEN); |
1885 | MCDI_DECLARE_BUF(outbuf, MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_LEN); |
1886 | size_t outlen; |
1887 | int rc; |
1888 | |
1889 | /* Check we have a hole for the context ID */ |
1890 | BUILD_BUG_ON(MC_CMD_RSS_CONTEXT_GET_FLAGS_IN_LEN != MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_FLAGS_OFST); |
1891 | MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_FLAGS_IN_RSS_CONTEXT_ID, context); |
1892 | MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_FLAGS_OUT_FLAGS, |
1893 | RSS_CONTEXT_FLAGS_DEFAULT); |
1894 | rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_GET_FLAGS, inbuf, |
1895 | inlen: sizeof(inbuf), outbuf, outlen: sizeof(outbuf), outlen_actual: &outlen); |
1896 | if (rc == 0) { |
1897 | if (outlen < MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_LEN) |
1898 | rc = -EIO; |
1899 | else |
1900 | *flags = MCDI_DWORD(outbuf, RSS_CONTEXT_GET_FLAGS_OUT_FLAGS); |
1901 | } |
1902 | return rc; |
1903 | } |
1904 | |
1905 | /* |
1906 | * Attempt to enable 4-tuple UDP hashing on the specified RSS context. |
1907 | * If we fail, we just leave the RSS context at its default hash settings, |
1908 | * which is safe but may slightly reduce performance. |
1909 | * Defaults are 4-tuple for TCP and 2-tuple for UDP and other-IP, so we |
1910 | * just need to set the UDP ports flags (for both IP versions). |
1911 | */ |
1912 | void (struct efx_nic *efx, |
1913 | struct efx_rss_context *ctx) |
1914 | { |
1915 | MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN); |
1916 | u32 flags; |
1917 | |
1918 | BUILD_BUG_ON(MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN != 0); |
1919 | |
1920 | if (efx_mcdi_get_rss_context_flags(efx, context: ctx->context_id, flags: &flags) != 0) |
1921 | return; |
1922 | MCDI_SET_DWORD(inbuf, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID, |
1923 | ctx->context_id); |
1924 | flags |= RSS_MODE_HASH_PORTS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV4_RSS_MODE_LBN; |
1925 | flags |= RSS_MODE_HASH_PORTS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV6_RSS_MODE_LBN; |
1926 | MCDI_SET_DWORD(inbuf, RSS_CONTEXT_SET_FLAGS_IN_FLAGS, flags); |
1927 | if (!efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_SET_FLAGS, inbuf, inlen: sizeof(inbuf), |
1928 | NULL, outlen: 0, NULL)) |
1929 | /* Succeeded, so UDP 4-tuple is now enabled */ |
1930 | ctx->rx_hash_udp_4tuple = true; |
1931 | } |
1932 | |
1933 | static int (struct efx_nic *efx, bool exclusive, |
1934 | struct efx_rss_context *ctx, |
1935 | unsigned *context_size) |
1936 | { |
1937 | MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN); |
1938 | MCDI_DECLARE_BUF(outbuf, MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN); |
1939 | size_t outlen; |
1940 | int rc; |
1941 | u32 alloc_type = exclusive ? |
1942 | MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE : |
1943 | MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED; |
1944 | unsigned = exclusive ? |
1945 | efx->rss_spread : |
1946 | min(rounddown_pow_of_two(efx->rss_spread), |
1947 | EFX_EF10_MAX_SHARED_RSS_CONTEXT_SIZE); |
1948 | |
1949 | if (!exclusive && rss_spread == 1) { |
1950 | ctx->context_id = EFX_MCDI_RSS_CONTEXT_INVALID; |
1951 | if (context_size) |
1952 | *context_size = 1; |
1953 | return 0; |
1954 | } |
1955 | |
1956 | if (efx_has_cap(efx, RX_RSS_LIMITED)) |
1957 | return -EOPNOTSUPP; |
1958 | |
1959 | MCDI_SET_DWORD(inbuf, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID, |
1960 | efx->vport_id); |
1961 | MCDI_SET_DWORD(inbuf, RSS_CONTEXT_ALLOC_IN_TYPE, alloc_type); |
1962 | MCDI_SET_DWORD(inbuf, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, rss_spread); |
1963 | |
1964 | rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_ALLOC, inbuf, inlen: sizeof(inbuf), |
1965 | outbuf, outlen: sizeof(outbuf), outlen_actual: &outlen); |
1966 | if (rc != 0) |
1967 | return rc; |
1968 | |
1969 | if (outlen < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) |
1970 | return -EIO; |
1971 | |
1972 | ctx->context_id = MCDI_DWORD(outbuf, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID); |
1973 | |
1974 | if (context_size) |
1975 | *context_size = rss_spread; |
1976 | |
1977 | if (efx_has_cap(efx, ADDITIONAL_RSS_MODES)) |
1978 | efx_mcdi_set_rss_context_flags(efx, ctx); |
1979 | |
1980 | return 0; |
1981 | } |
1982 | |
1983 | static int (struct efx_nic *efx, u32 context) |
1984 | { |
1985 | MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_FREE_IN_LEN); |
1986 | |
1987 | MCDI_SET_DWORD(inbuf, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, |
1988 | context); |
1989 | return efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_FREE, inbuf, inlen: sizeof(inbuf), |
1990 | NULL, outlen: 0, NULL); |
1991 | } |
1992 | |
1993 | static int (struct efx_nic *efx, u32 context, |
1994 | const u32 *rx_indir_table, const u8 *key) |
1995 | { |
1996 | MCDI_DECLARE_BUF(tablebuf, MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN); |
1997 | MCDI_DECLARE_BUF(keybuf, MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN); |
1998 | int i, rc; |
1999 | |
2000 | MCDI_SET_DWORD(tablebuf, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID, |
2001 | context); |
2002 | BUILD_BUG_ON(ARRAY_SIZE(efx->rss_context.rx_indir_table) != |
2003 | MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN); |
2004 | |
2005 | /* This iterates over the length of efx->rss_context.rx_indir_table, but |
2006 | * copies bytes from rx_indir_table. That's because the latter is a |
2007 | * pointer rather than an array, but should have the same length. |
2008 | * The efx->rss_context.rx_hash_key loop below is similar. |
2009 | */ |
2010 | for (i = 0; i < ARRAY_SIZE(efx->rss_context.rx_indir_table); ++i) |
2011 | MCDI_PTR(tablebuf, |
2012 | RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE)[i] = |
2013 | (u8) rx_indir_table[i]; |
2014 | |
2015 | rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_SET_TABLE, inbuf: tablebuf, |
2016 | inlen: sizeof(tablebuf), NULL, outlen: 0, NULL); |
2017 | if (rc != 0) |
2018 | return rc; |
2019 | |
2020 | MCDI_SET_DWORD(keybuf, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID, |
2021 | context); |
2022 | BUILD_BUG_ON(ARRAY_SIZE(efx->rss_context.rx_hash_key) != |
2023 | MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); |
2024 | for (i = 0; i < ARRAY_SIZE(efx->rss_context.rx_hash_key); ++i) |
2025 | MCDI_PTR(keybuf, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY)[i] = key[i]; |
2026 | |
2027 | return efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_SET_KEY, inbuf: keybuf, |
2028 | inlen: sizeof(keybuf), NULL, outlen: 0, NULL); |
2029 | } |
2030 | |
2031 | void efx_mcdi_rx_free_indir_table(struct efx_nic *efx) |
2032 | { |
2033 | int rc; |
2034 | |
2035 | if (efx->rss_context.context_id != EFX_MCDI_RSS_CONTEXT_INVALID) { |
2036 | rc = efx_mcdi_filter_free_rss_context(efx, context: efx->rss_context.context_id); |
2037 | WARN_ON(rc != 0); |
2038 | } |
2039 | efx->rss_context.context_id = EFX_MCDI_RSS_CONTEXT_INVALID; |
2040 | } |
2041 | |
2042 | static int (struct efx_nic *efx, |
2043 | unsigned *context_size) |
2044 | { |
2045 | struct efx_mcdi_filter_table *table = efx->filter_state; |
2046 | int rc = efx_mcdi_filter_alloc_rss_context(efx, exclusive: false, ctx: &efx->rss_context, |
2047 | context_size); |
2048 | |
2049 | if (rc != 0) |
2050 | return rc; |
2051 | |
2052 | table->rx_rss_context_exclusive = false; |
2053 | efx_set_default_rx_indir_table(efx, ctx: &efx->rss_context); |
2054 | return 0; |
2055 | } |
2056 | |
2057 | static int (struct efx_nic *efx, |
2058 | const u32 *rx_indir_table, |
2059 | const u8 *key) |
2060 | { |
2061 | struct efx_mcdi_filter_table *table = efx->filter_state; |
2062 | u32 = efx->rss_context.context_id; |
2063 | int rc; |
2064 | |
2065 | if (efx->rss_context.context_id == EFX_MCDI_RSS_CONTEXT_INVALID || |
2066 | !table->rx_rss_context_exclusive) { |
2067 | rc = efx_mcdi_filter_alloc_rss_context(efx, exclusive: true, ctx: &efx->rss_context, |
2068 | NULL); |
2069 | if (rc == -EOPNOTSUPP) |
2070 | return rc; |
2071 | else if (rc != 0) |
2072 | goto fail1; |
2073 | } |
2074 | |
2075 | rc = efx_mcdi_filter_populate_rss_table(efx, context: efx->rss_context.context_id, |
2076 | rx_indir_table, key); |
2077 | if (rc != 0) |
2078 | goto fail2; |
2079 | |
2080 | if (efx->rss_context.context_id != old_rx_rss_context && |
2081 | old_rx_rss_context != EFX_MCDI_RSS_CONTEXT_INVALID) |
2082 | WARN_ON(efx_mcdi_filter_free_rss_context(efx, old_rx_rss_context) != 0); |
2083 | table->rx_rss_context_exclusive = true; |
2084 | if (rx_indir_table != efx->rss_context.rx_indir_table) |
2085 | memcpy(efx->rss_context.rx_indir_table, rx_indir_table, |
2086 | sizeof(efx->rss_context.rx_indir_table)); |
2087 | if (key != efx->rss_context.rx_hash_key) |
2088 | memcpy(efx->rss_context.rx_hash_key, key, |
2089 | efx->type->rx_hash_key_size); |
2090 | |
2091 | return 0; |
2092 | |
2093 | fail2: |
2094 | if (old_rx_rss_context != efx->rss_context.context_id) { |
2095 | WARN_ON(efx_mcdi_filter_free_rss_context(efx, efx->rss_context.context_id) != 0); |
2096 | efx->rss_context.context_id = old_rx_rss_context; |
2097 | } |
2098 | fail1: |
2099 | netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n" , __func__, rc); |
2100 | return rc; |
2101 | } |
2102 | |
2103 | int (struct efx_nic *efx, |
2104 | struct efx_rss_context *ctx, |
2105 | const u32 *rx_indir_table, |
2106 | const u8 *key) |
2107 | { |
2108 | int rc; |
2109 | |
2110 | WARN_ON(!mutex_is_locked(&efx->rss_lock)); |
2111 | |
2112 | if (ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID) { |
2113 | rc = efx_mcdi_filter_alloc_rss_context(efx, exclusive: true, ctx, NULL); |
2114 | if (rc) |
2115 | return rc; |
2116 | } |
2117 | |
2118 | if (!rx_indir_table) /* Delete this context */ |
2119 | return efx_mcdi_filter_free_rss_context(efx, context: ctx->context_id); |
2120 | |
2121 | rc = efx_mcdi_filter_populate_rss_table(efx, context: ctx->context_id, |
2122 | rx_indir_table, key); |
2123 | if (rc) |
2124 | return rc; |
2125 | |
2126 | memcpy(ctx->rx_indir_table, rx_indir_table, |
2127 | sizeof(efx->rss_context.rx_indir_table)); |
2128 | memcpy(ctx->rx_hash_key, key, efx->type->rx_hash_key_size); |
2129 | |
2130 | return 0; |
2131 | } |
2132 | |
2133 | int (struct efx_nic *efx, |
2134 | struct efx_rss_context *ctx) |
2135 | { |
2136 | MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_GET_TABLE_IN_LEN); |
2137 | MCDI_DECLARE_BUF(tablebuf, MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_LEN); |
2138 | MCDI_DECLARE_BUF(keybuf, MC_CMD_RSS_CONTEXT_GET_KEY_OUT_LEN); |
2139 | size_t outlen; |
2140 | int rc, i; |
2141 | |
2142 | WARN_ON(!mutex_is_locked(&efx->rss_lock)); |
2143 | |
2144 | BUILD_BUG_ON(MC_CMD_RSS_CONTEXT_GET_TABLE_IN_LEN != |
2145 | MC_CMD_RSS_CONTEXT_GET_KEY_IN_LEN); |
2146 | |
2147 | if (ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID) |
2148 | return -ENOENT; |
2149 | |
2150 | MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_TABLE_IN_RSS_CONTEXT_ID, |
2151 | ctx->context_id); |
2152 | BUILD_BUG_ON(ARRAY_SIZE(ctx->rx_indir_table) != |
2153 | MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_INDIRECTION_TABLE_LEN); |
2154 | rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_GET_TABLE, inbuf, inlen: sizeof(inbuf), |
2155 | outbuf: tablebuf, outlen: sizeof(tablebuf), outlen_actual: &outlen); |
2156 | if (rc != 0) |
2157 | return rc; |
2158 | |
2159 | if (WARN_ON(outlen != MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_LEN)) |
2160 | return -EIO; |
2161 | |
2162 | for (i = 0; i < ARRAY_SIZE(ctx->rx_indir_table); i++) |
2163 | ctx->rx_indir_table[i] = MCDI_PTR(tablebuf, |
2164 | RSS_CONTEXT_GET_TABLE_OUT_INDIRECTION_TABLE)[i]; |
2165 | |
2166 | MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_KEY_IN_RSS_CONTEXT_ID, |
2167 | ctx->context_id); |
2168 | BUILD_BUG_ON(ARRAY_SIZE(ctx->rx_hash_key) != |
2169 | MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); |
2170 | rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_GET_KEY, inbuf, inlen: sizeof(inbuf), |
2171 | outbuf: keybuf, outlen: sizeof(keybuf), outlen_actual: &outlen); |
2172 | if (rc != 0) |
2173 | return rc; |
2174 | |
2175 | if (WARN_ON(outlen != MC_CMD_RSS_CONTEXT_GET_KEY_OUT_LEN)) |
2176 | return -EIO; |
2177 | |
2178 | for (i = 0; i < ARRAY_SIZE(ctx->rx_hash_key); ++i) |
2179 | ctx->rx_hash_key[i] = MCDI_PTR( |
2180 | keybuf, RSS_CONTEXT_GET_KEY_OUT_TOEPLITZ_KEY)[i]; |
2181 | |
2182 | return 0; |
2183 | } |
2184 | |
2185 | int (struct efx_nic *efx) |
2186 | { |
2187 | int rc; |
2188 | |
2189 | mutex_lock(&efx->rss_lock); |
2190 | rc = efx_mcdi_rx_pull_rss_context_config(efx, ctx: &efx->rss_context); |
2191 | mutex_unlock(lock: &efx->rss_lock); |
2192 | return rc; |
2193 | } |
2194 | |
2195 | void (struct efx_nic *efx) |
2196 | { |
2197 | struct efx_mcdi_filter_table *table = efx->filter_state; |
2198 | struct efx_rss_context *ctx; |
2199 | int rc; |
2200 | |
2201 | WARN_ON(!mutex_is_locked(&efx->rss_lock)); |
2202 | |
2203 | if (!table->must_restore_rss_contexts) |
2204 | return; |
2205 | |
2206 | list_for_each_entry(ctx, &efx->rss_context.list, list) { |
2207 | /* previous NIC RSS context is gone */ |
2208 | ctx->context_id = EFX_MCDI_RSS_CONTEXT_INVALID; |
2209 | /* so try to allocate a new one */ |
2210 | rc = efx_mcdi_rx_push_rss_context_config(efx, ctx, |
2211 | rx_indir_table: ctx->rx_indir_table, |
2212 | key: ctx->rx_hash_key); |
2213 | if (rc) |
2214 | netif_warn(efx, probe, efx->net_dev, |
2215 | "failed to restore RSS context %u, rc=%d" |
2216 | "; RSS filters may fail to be applied\n" , |
2217 | ctx->user_id, rc); |
2218 | } |
2219 | table->must_restore_rss_contexts = false; |
2220 | } |
2221 | |
2222 | int (struct efx_nic *efx, bool user, |
2223 | const u32 *rx_indir_table, |
2224 | const u8 *key) |
2225 | { |
2226 | int rc; |
2227 | |
2228 | if (efx->rss_spread == 1) |
2229 | return 0; |
2230 | |
2231 | if (!key) |
2232 | key = efx->rss_context.rx_hash_key; |
2233 | |
2234 | rc = efx_mcdi_filter_rx_push_exclusive_rss_config(efx, rx_indir_table, key); |
2235 | |
2236 | if (rc == -ENOBUFS && !user) { |
2237 | unsigned context_size; |
2238 | bool mismatch = false; |
2239 | size_t i; |
2240 | |
2241 | for (i = 0; |
2242 | i < ARRAY_SIZE(efx->rss_context.rx_indir_table) && !mismatch; |
2243 | i++) |
2244 | mismatch = rx_indir_table[i] != |
2245 | ethtool_rxfh_indir_default(index: i, n_rx_rings: efx->rss_spread); |
2246 | |
2247 | rc = efx_mcdi_filter_rx_push_shared_rss_config(efx, context_size: &context_size); |
2248 | if (rc == 0) { |
2249 | if (context_size != efx->rss_spread) |
2250 | netif_warn(efx, probe, efx->net_dev, |
2251 | "Could not allocate an exclusive RSS" |
2252 | " context; allocated a shared one of" |
2253 | " different size." |
2254 | " Wanted %u, got %u.\n" , |
2255 | efx->rss_spread, context_size); |
2256 | else if (mismatch) |
2257 | netif_warn(efx, probe, efx->net_dev, |
2258 | "Could not allocate an exclusive RSS" |
2259 | " context; allocated a shared one but" |
2260 | " could not apply custom" |
2261 | " indirection.\n" ); |
2262 | else |
2263 | netif_info(efx, probe, efx->net_dev, |
2264 | "Could not allocate an exclusive RSS" |
2265 | " context; allocated a shared one.\n" ); |
2266 | } |
2267 | } |
2268 | return rc; |
2269 | } |
2270 | |
2271 | int (struct efx_nic *efx, bool user, |
2272 | const u32 *rx_indir_table |
2273 | __attribute__ ((unused)), |
2274 | const u8 *key |
2275 | __attribute__ ((unused))) |
2276 | { |
2277 | if (user) |
2278 | return -EOPNOTSUPP; |
2279 | if (efx->rss_context.context_id != EFX_MCDI_RSS_CONTEXT_INVALID) |
2280 | return 0; |
2281 | return efx_mcdi_filter_rx_push_shared_rss_config(efx, NULL); |
2282 | } |
2283 | |
2284 | int efx_mcdi_push_default_indir_table(struct efx_nic *efx, |
2285 | unsigned int ) |
2286 | { |
2287 | int rc = 0; |
2288 | |
2289 | if (efx->rss_spread == rss_spread) |
2290 | return 0; |
2291 | |
2292 | efx->rss_spread = rss_spread; |
2293 | if (!efx->filter_state) |
2294 | return 0; |
2295 | |
2296 | efx_mcdi_rx_free_indir_table(efx); |
2297 | if (rss_spread > 1) { |
2298 | efx_set_default_rx_indir_table(efx, ctx: &efx->rss_context); |
2299 | rc = efx->type->rx_push_rss_config(efx, false, |
2300 | efx->rss_context.rx_indir_table, NULL); |
2301 | } |
2302 | return rc; |
2303 | } |
2304 | |