1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * drivers/net/team/team_mode_loadbalance.c - Load-balancing mode for team |
4 | * Copyright (c) 2012 Jiri Pirko <jpirko@redhat.com> |
5 | */ |
6 | |
7 | #include <linux/kernel.h> |
8 | #include <linux/types.h> |
9 | #include <linux/module.h> |
10 | #include <linux/init.h> |
11 | #include <linux/errno.h> |
12 | #include <linux/netdevice.h> |
13 | #include <linux/etherdevice.h> |
14 | #include <linux/filter.h> |
15 | #include <linux/if_team.h> |
16 | |
17 | static rx_handler_result_t lb_receive(struct team *team, struct team_port *port, |
18 | struct sk_buff *skb) |
19 | { |
20 | if (unlikely(skb->protocol == htons(ETH_P_SLOW))) { |
21 | /* LACPDU packets should go to exact delivery */ |
22 | const unsigned char *dest = eth_hdr(skb)->h_dest; |
23 | |
24 | if (is_link_local_ether_addr(addr: dest) && dest[5] == 0x02) |
25 | return RX_HANDLER_EXACT; |
26 | } |
27 | return RX_HANDLER_ANOTHER; |
28 | } |
29 | |
30 | struct lb_priv; |
31 | |
32 | typedef struct team_port *lb_select_tx_port_func_t(struct team *, |
33 | unsigned char); |
34 | |
35 | #define LB_TX_HASHTABLE_SIZE 256 /* hash is a char */ |
36 | |
37 | struct lb_stats { |
38 | u64 tx_bytes; |
39 | }; |
40 | |
41 | struct lb_pcpu_stats { |
42 | struct lb_stats hash_stats[LB_TX_HASHTABLE_SIZE]; |
43 | struct u64_stats_sync syncp; |
44 | }; |
45 | |
46 | struct lb_stats_info { |
47 | struct lb_stats stats; |
48 | struct lb_stats last_stats; |
49 | struct team_option_inst_info *opt_inst_info; |
50 | }; |
51 | |
52 | struct lb_port_mapping { |
53 | struct team_port __rcu *port; |
54 | struct team_option_inst_info *opt_inst_info; |
55 | }; |
56 | |
57 | struct lb_priv_ex { |
58 | struct team *team; |
59 | struct lb_port_mapping tx_hash_to_port_mapping[LB_TX_HASHTABLE_SIZE]; |
60 | struct sock_fprog_kern *orig_fprog; |
61 | struct { |
62 | unsigned int refresh_interval; /* in tenths of second */ |
63 | struct delayed_work refresh_dw; |
64 | struct lb_stats_info info[LB_TX_HASHTABLE_SIZE]; |
65 | } stats; |
66 | }; |
67 | |
68 | struct lb_priv { |
69 | struct bpf_prog __rcu *fp; |
70 | lb_select_tx_port_func_t __rcu *select_tx_port_func; |
71 | struct lb_pcpu_stats __percpu *pcpu_stats; |
72 | struct lb_priv_ex *ex; /* priv extension */ |
73 | }; |
74 | |
75 | static struct lb_priv *get_lb_priv(struct team *team) |
76 | { |
77 | return (struct lb_priv *) &team->mode_priv; |
78 | } |
79 | |
80 | struct lb_port_priv { |
81 | struct lb_stats __percpu *pcpu_stats; |
82 | struct lb_stats_info stats_info; |
83 | }; |
84 | |
85 | static struct lb_port_priv *get_lb_port_priv(struct team_port *port) |
86 | { |
87 | return (struct lb_port_priv *) &port->mode_priv; |
88 | } |
89 | |
90 | #define LB_HTPM_PORT_BY_HASH(lp_priv, hash) \ |
91 | (lb_priv)->ex->tx_hash_to_port_mapping[hash].port |
92 | |
93 | #define LB_HTPM_OPT_INST_INFO_BY_HASH(lp_priv, hash) \ |
94 | (lb_priv)->ex->tx_hash_to_port_mapping[hash].opt_inst_info |
95 | |
96 | static void lb_tx_hash_to_port_mapping_null_port(struct team *team, |
97 | struct team_port *port) |
98 | { |
99 | struct lb_priv *lb_priv = get_lb_priv(team); |
100 | bool changed = false; |
101 | int i; |
102 | |
103 | for (i = 0; i < LB_TX_HASHTABLE_SIZE; i++) { |
104 | struct lb_port_mapping *pm; |
105 | |
106 | pm = &lb_priv->ex->tx_hash_to_port_mapping[i]; |
107 | if (rcu_access_pointer(pm->port) == port) { |
108 | RCU_INIT_POINTER(pm->port, NULL); |
109 | team_option_inst_set_change(opt_inst_info: pm->opt_inst_info); |
110 | changed = true; |
111 | } |
112 | } |
113 | if (changed) |
114 | team_options_change_check(team); |
115 | } |
116 | |
117 | /* Basic tx selection based solely by hash */ |
118 | static struct team_port *lb_hash_select_tx_port(struct team *team, |
119 | unsigned char hash) |
120 | { |
121 | int port_index = team_num_to_port_index(team, num: hash); |
122 | |
123 | return team_get_port_by_index_rcu(team, port_index); |
124 | } |
125 | |
126 | /* Hash to port mapping select tx port */ |
127 | static struct team_port *lb_htpm_select_tx_port(struct team *team, |
128 | unsigned char hash) |
129 | { |
130 | struct lb_priv *lb_priv = get_lb_priv(team); |
131 | struct team_port *port; |
132 | |
133 | port = rcu_dereference_bh(LB_HTPM_PORT_BY_HASH(lb_priv, hash)); |
134 | if (likely(port)) |
135 | return port; |
136 | /* If no valid port in the table, fall back to simple hash */ |
137 | return lb_hash_select_tx_port(team, hash); |
138 | } |
139 | |
140 | struct lb_select_tx_port { |
141 | char *name; |
142 | lb_select_tx_port_func_t *func; |
143 | }; |
144 | |
145 | static const struct lb_select_tx_port lb_select_tx_port_list[] = { |
146 | { |
147 | .name = "hash" , |
148 | .func = lb_hash_select_tx_port, |
149 | }, |
150 | { |
151 | .name = "hash_to_port_mapping" , |
152 | .func = lb_htpm_select_tx_port, |
153 | }, |
154 | }; |
155 | #define LB_SELECT_TX_PORT_LIST_COUNT ARRAY_SIZE(lb_select_tx_port_list) |
156 | |
157 | static char *lb_select_tx_port_get_name(lb_select_tx_port_func_t *func) |
158 | { |
159 | int i; |
160 | |
161 | for (i = 0; i < LB_SELECT_TX_PORT_LIST_COUNT; i++) { |
162 | const struct lb_select_tx_port *item; |
163 | |
164 | item = &lb_select_tx_port_list[i]; |
165 | if (item->func == func) |
166 | return item->name; |
167 | } |
168 | return NULL; |
169 | } |
170 | |
171 | static lb_select_tx_port_func_t *lb_select_tx_port_get_func(const char *name) |
172 | { |
173 | int i; |
174 | |
175 | for (i = 0; i < LB_SELECT_TX_PORT_LIST_COUNT; i++) { |
176 | const struct lb_select_tx_port *item; |
177 | |
178 | item = &lb_select_tx_port_list[i]; |
179 | if (!strcmp(item->name, name)) |
180 | return item->func; |
181 | } |
182 | return NULL; |
183 | } |
184 | |
185 | static unsigned int lb_get_skb_hash(struct lb_priv *lb_priv, |
186 | struct sk_buff *skb) |
187 | { |
188 | struct bpf_prog *fp; |
189 | uint32_t lhash; |
190 | unsigned char *c; |
191 | |
192 | fp = rcu_dereference_bh(lb_priv->fp); |
193 | if (unlikely(!fp)) |
194 | return 0; |
195 | lhash = bpf_prog_run(prog: fp, ctx: skb); |
196 | c = (char *) &lhash; |
197 | return c[0] ^ c[1] ^ c[2] ^ c[3]; |
198 | } |
199 | |
200 | static void lb_update_tx_stats(unsigned int tx_bytes, struct lb_priv *lb_priv, |
201 | struct lb_port_priv *lb_port_priv, |
202 | unsigned char hash) |
203 | { |
204 | struct lb_pcpu_stats *pcpu_stats; |
205 | struct lb_stats *port_stats; |
206 | struct lb_stats *hash_stats; |
207 | |
208 | pcpu_stats = this_cpu_ptr(lb_priv->pcpu_stats); |
209 | port_stats = this_cpu_ptr(lb_port_priv->pcpu_stats); |
210 | hash_stats = &pcpu_stats->hash_stats[hash]; |
211 | u64_stats_update_begin(syncp: &pcpu_stats->syncp); |
212 | port_stats->tx_bytes += tx_bytes; |
213 | hash_stats->tx_bytes += tx_bytes; |
214 | u64_stats_update_end(syncp: &pcpu_stats->syncp); |
215 | } |
216 | |
217 | static bool lb_transmit(struct team *team, struct sk_buff *skb) |
218 | { |
219 | struct lb_priv *lb_priv = get_lb_priv(team); |
220 | lb_select_tx_port_func_t *select_tx_port_func; |
221 | struct team_port *port; |
222 | unsigned char hash; |
223 | unsigned int tx_bytes = skb->len; |
224 | |
225 | hash = lb_get_skb_hash(lb_priv, skb); |
226 | select_tx_port_func = rcu_dereference_bh(lb_priv->select_tx_port_func); |
227 | port = select_tx_port_func(team, hash); |
228 | if (unlikely(!port)) |
229 | goto drop; |
230 | if (team_dev_queue_xmit(team, port, skb)) |
231 | return false; |
232 | lb_update_tx_stats(tx_bytes, lb_priv, lb_port_priv: get_lb_port_priv(port), hash); |
233 | return true; |
234 | |
235 | drop: |
236 | dev_kfree_skb_any(skb); |
237 | return false; |
238 | } |
239 | |
240 | static void lb_bpf_func_get(struct team *team, struct team_gsetter_ctx *ctx) |
241 | { |
242 | struct lb_priv *lb_priv = get_lb_priv(team); |
243 | |
244 | if (!lb_priv->ex->orig_fprog) { |
245 | ctx->data.bin_val.len = 0; |
246 | ctx->data.bin_val.ptr = NULL; |
247 | return; |
248 | } |
249 | ctx->data.bin_val.len = lb_priv->ex->orig_fprog->len * |
250 | sizeof(struct sock_filter); |
251 | ctx->data.bin_val.ptr = lb_priv->ex->orig_fprog->filter; |
252 | } |
253 | |
254 | static int __fprog_create(struct sock_fprog_kern **pfprog, u32 data_len, |
255 | const void *data) |
256 | { |
257 | struct sock_fprog_kern *fprog; |
258 | struct sock_filter *filter = (struct sock_filter *) data; |
259 | |
260 | if (data_len % sizeof(struct sock_filter)) |
261 | return -EINVAL; |
262 | fprog = kmalloc(size: sizeof(*fprog), GFP_KERNEL); |
263 | if (!fprog) |
264 | return -ENOMEM; |
265 | fprog->filter = kmemdup(p: filter, size: data_len, GFP_KERNEL); |
266 | if (!fprog->filter) { |
267 | kfree(objp: fprog); |
268 | return -ENOMEM; |
269 | } |
270 | fprog->len = data_len / sizeof(struct sock_filter); |
271 | *pfprog = fprog; |
272 | return 0; |
273 | } |
274 | |
275 | static void __fprog_destroy(struct sock_fprog_kern *fprog) |
276 | { |
277 | kfree(objp: fprog->filter); |
278 | kfree(objp: fprog); |
279 | } |
280 | |
281 | static int lb_bpf_func_set(struct team *team, struct team_gsetter_ctx *ctx) |
282 | { |
283 | struct lb_priv *lb_priv = get_lb_priv(team); |
284 | struct bpf_prog *fp = NULL; |
285 | struct bpf_prog *orig_fp = NULL; |
286 | struct sock_fprog_kern *fprog = NULL; |
287 | int err; |
288 | |
289 | if (ctx->data.bin_val.len) { |
290 | err = __fprog_create(pfprog: &fprog, data_len: ctx->data.bin_val.len, |
291 | data: ctx->data.bin_val.ptr); |
292 | if (err) |
293 | return err; |
294 | err = bpf_prog_create(pfp: &fp, fprog); |
295 | if (err) { |
296 | __fprog_destroy(fprog); |
297 | return err; |
298 | } |
299 | } |
300 | |
301 | if (lb_priv->ex->orig_fprog) { |
302 | /* Clear old filter data */ |
303 | __fprog_destroy(fprog: lb_priv->ex->orig_fprog); |
304 | orig_fp = rcu_dereference_protected(lb_priv->fp, |
305 | lockdep_is_held(&team->lock)); |
306 | } |
307 | |
308 | rcu_assign_pointer(lb_priv->fp, fp); |
309 | lb_priv->ex->orig_fprog = fprog; |
310 | |
311 | if (orig_fp) { |
312 | synchronize_rcu(); |
313 | bpf_prog_destroy(fp: orig_fp); |
314 | } |
315 | return 0; |
316 | } |
317 | |
318 | static void lb_bpf_func_free(struct team *team) |
319 | { |
320 | struct lb_priv *lb_priv = get_lb_priv(team); |
321 | struct bpf_prog *fp; |
322 | |
323 | if (!lb_priv->ex->orig_fprog) |
324 | return; |
325 | |
326 | __fprog_destroy(fprog: lb_priv->ex->orig_fprog); |
327 | fp = rcu_dereference_protected(lb_priv->fp, |
328 | lockdep_is_held(&team->lock)); |
329 | bpf_prog_destroy(fp); |
330 | } |
331 | |
332 | static void lb_tx_method_get(struct team *team, struct team_gsetter_ctx *ctx) |
333 | { |
334 | struct lb_priv *lb_priv = get_lb_priv(team); |
335 | lb_select_tx_port_func_t *func; |
336 | char *name; |
337 | |
338 | func = rcu_dereference_protected(lb_priv->select_tx_port_func, |
339 | lockdep_is_held(&team->lock)); |
340 | name = lb_select_tx_port_get_name(func); |
341 | BUG_ON(!name); |
342 | ctx->data.str_val = name; |
343 | } |
344 | |
345 | static int lb_tx_method_set(struct team *team, struct team_gsetter_ctx *ctx) |
346 | { |
347 | struct lb_priv *lb_priv = get_lb_priv(team); |
348 | lb_select_tx_port_func_t *func; |
349 | |
350 | func = lb_select_tx_port_get_func(name: ctx->data.str_val); |
351 | if (!func) |
352 | return -EINVAL; |
353 | rcu_assign_pointer(lb_priv->select_tx_port_func, func); |
354 | return 0; |
355 | } |
356 | |
357 | static void lb_tx_hash_to_port_mapping_init(struct team *team, |
358 | struct team_option_inst_info *info) |
359 | { |
360 | struct lb_priv *lb_priv = get_lb_priv(team); |
361 | unsigned char hash = info->array_index; |
362 | |
363 | LB_HTPM_OPT_INST_INFO_BY_HASH(lb_priv, hash) = info; |
364 | } |
365 | |
366 | static void lb_tx_hash_to_port_mapping_get(struct team *team, |
367 | struct team_gsetter_ctx *ctx) |
368 | { |
369 | struct lb_priv *lb_priv = get_lb_priv(team); |
370 | struct team_port *port; |
371 | unsigned char hash = ctx->info->array_index; |
372 | |
373 | port = LB_HTPM_PORT_BY_HASH(lb_priv, hash); |
374 | ctx->data.u32_val = port ? port->dev->ifindex : 0; |
375 | } |
376 | |
377 | static int lb_tx_hash_to_port_mapping_set(struct team *team, |
378 | struct team_gsetter_ctx *ctx) |
379 | { |
380 | struct lb_priv *lb_priv = get_lb_priv(team); |
381 | struct team_port *port; |
382 | unsigned char hash = ctx->info->array_index; |
383 | |
384 | list_for_each_entry(port, &team->port_list, list) { |
385 | if (ctx->data.u32_val == port->dev->ifindex && |
386 | team_port_enabled(port)) { |
387 | rcu_assign_pointer(LB_HTPM_PORT_BY_HASH(lb_priv, hash), |
388 | port); |
389 | return 0; |
390 | } |
391 | } |
392 | return -ENODEV; |
393 | } |
394 | |
395 | static void lb_hash_stats_init(struct team *team, |
396 | struct team_option_inst_info *info) |
397 | { |
398 | struct lb_priv *lb_priv = get_lb_priv(team); |
399 | unsigned char hash = info->array_index; |
400 | |
401 | lb_priv->ex->stats.info[hash].opt_inst_info = info; |
402 | } |
403 | |
404 | static void lb_hash_stats_get(struct team *team, struct team_gsetter_ctx *ctx) |
405 | { |
406 | struct lb_priv *lb_priv = get_lb_priv(team); |
407 | unsigned char hash = ctx->info->array_index; |
408 | |
409 | ctx->data.bin_val.ptr = &lb_priv->ex->stats.info[hash].stats; |
410 | ctx->data.bin_val.len = sizeof(struct lb_stats); |
411 | } |
412 | |
413 | static void lb_port_stats_init(struct team *team, |
414 | struct team_option_inst_info *info) |
415 | { |
416 | struct team_port *port = info->port; |
417 | struct lb_port_priv *lb_port_priv = get_lb_port_priv(port); |
418 | |
419 | lb_port_priv->stats_info.opt_inst_info = info; |
420 | } |
421 | |
422 | static void lb_port_stats_get(struct team *team, struct team_gsetter_ctx *ctx) |
423 | { |
424 | struct team_port *port = ctx->info->port; |
425 | struct lb_port_priv *lb_port_priv = get_lb_port_priv(port); |
426 | |
427 | ctx->data.bin_val.ptr = &lb_port_priv->stats_info.stats; |
428 | ctx->data.bin_val.len = sizeof(struct lb_stats); |
429 | } |
430 | |
431 | static void __lb_stats_info_refresh_prepare(struct lb_stats_info *s_info) |
432 | { |
433 | memcpy(&s_info->last_stats, &s_info->stats, sizeof(struct lb_stats)); |
434 | memset(&s_info->stats, 0, sizeof(struct lb_stats)); |
435 | } |
436 | |
437 | static bool __lb_stats_info_refresh_check(struct lb_stats_info *s_info, |
438 | struct team *team) |
439 | { |
440 | if (memcmp(p: &s_info->last_stats, q: &s_info->stats, |
441 | size: sizeof(struct lb_stats))) { |
442 | team_option_inst_set_change(opt_inst_info: s_info->opt_inst_info); |
443 | return true; |
444 | } |
445 | return false; |
446 | } |
447 | |
448 | static void __lb_one_cpu_stats_add(struct lb_stats *acc_stats, |
449 | struct lb_stats *cpu_stats, |
450 | struct u64_stats_sync *syncp) |
451 | { |
452 | unsigned int start; |
453 | struct lb_stats tmp; |
454 | |
455 | do { |
456 | start = u64_stats_fetch_begin(syncp); |
457 | tmp.tx_bytes = cpu_stats->tx_bytes; |
458 | } while (u64_stats_fetch_retry(syncp, start)); |
459 | acc_stats->tx_bytes += tmp.tx_bytes; |
460 | } |
461 | |
462 | static void lb_stats_refresh(struct work_struct *work) |
463 | { |
464 | struct team *team; |
465 | struct lb_priv *lb_priv; |
466 | struct lb_priv_ex *lb_priv_ex; |
467 | struct lb_pcpu_stats *pcpu_stats; |
468 | struct lb_stats *stats; |
469 | struct lb_stats_info *s_info; |
470 | struct team_port *port; |
471 | bool changed = false; |
472 | int i; |
473 | int j; |
474 | |
475 | lb_priv_ex = container_of(work, struct lb_priv_ex, |
476 | stats.refresh_dw.work); |
477 | |
478 | team = lb_priv_ex->team; |
479 | lb_priv = get_lb_priv(team); |
480 | |
481 | if (!mutex_trylock(lock: &team->lock)) { |
482 | schedule_delayed_work(dwork: &lb_priv_ex->stats.refresh_dw, delay: 0); |
483 | return; |
484 | } |
485 | |
486 | for (j = 0; j < LB_TX_HASHTABLE_SIZE; j++) { |
487 | s_info = &lb_priv->ex->stats.info[j]; |
488 | __lb_stats_info_refresh_prepare(s_info); |
489 | for_each_possible_cpu(i) { |
490 | pcpu_stats = per_cpu_ptr(lb_priv->pcpu_stats, i); |
491 | stats = &pcpu_stats->hash_stats[j]; |
492 | __lb_one_cpu_stats_add(acc_stats: &s_info->stats, cpu_stats: stats, |
493 | syncp: &pcpu_stats->syncp); |
494 | } |
495 | changed |= __lb_stats_info_refresh_check(s_info, team); |
496 | } |
497 | |
498 | list_for_each_entry(port, &team->port_list, list) { |
499 | struct lb_port_priv *lb_port_priv = get_lb_port_priv(port); |
500 | |
501 | s_info = &lb_port_priv->stats_info; |
502 | __lb_stats_info_refresh_prepare(s_info); |
503 | for_each_possible_cpu(i) { |
504 | pcpu_stats = per_cpu_ptr(lb_priv->pcpu_stats, i); |
505 | stats = per_cpu_ptr(lb_port_priv->pcpu_stats, i); |
506 | __lb_one_cpu_stats_add(acc_stats: &s_info->stats, cpu_stats: stats, |
507 | syncp: &pcpu_stats->syncp); |
508 | } |
509 | changed |= __lb_stats_info_refresh_check(s_info, team); |
510 | } |
511 | |
512 | if (changed) |
513 | team_options_change_check(team); |
514 | |
515 | schedule_delayed_work(dwork: &lb_priv_ex->stats.refresh_dw, |
516 | delay: (lb_priv_ex->stats.refresh_interval * HZ) / 10); |
517 | |
518 | mutex_unlock(lock: &team->lock); |
519 | } |
520 | |
521 | static void lb_stats_refresh_interval_get(struct team *team, |
522 | struct team_gsetter_ctx *ctx) |
523 | { |
524 | struct lb_priv *lb_priv = get_lb_priv(team); |
525 | |
526 | ctx->data.u32_val = lb_priv->ex->stats.refresh_interval; |
527 | } |
528 | |
529 | static int lb_stats_refresh_interval_set(struct team *team, |
530 | struct team_gsetter_ctx *ctx) |
531 | { |
532 | struct lb_priv *lb_priv = get_lb_priv(team); |
533 | unsigned int interval; |
534 | |
535 | interval = ctx->data.u32_val; |
536 | if (lb_priv->ex->stats.refresh_interval == interval) |
537 | return 0; |
538 | lb_priv->ex->stats.refresh_interval = interval; |
539 | if (interval) |
540 | schedule_delayed_work(dwork: &lb_priv->ex->stats.refresh_dw, delay: 0); |
541 | else |
542 | cancel_delayed_work(dwork: &lb_priv->ex->stats.refresh_dw); |
543 | return 0; |
544 | } |
545 | |
546 | static const struct team_option lb_options[] = { |
547 | { |
548 | .name = "bpf_hash_func" , |
549 | .type = TEAM_OPTION_TYPE_BINARY, |
550 | .getter = lb_bpf_func_get, |
551 | .setter = lb_bpf_func_set, |
552 | }, |
553 | { |
554 | .name = "lb_tx_method" , |
555 | .type = TEAM_OPTION_TYPE_STRING, |
556 | .getter = lb_tx_method_get, |
557 | .setter = lb_tx_method_set, |
558 | }, |
559 | { |
560 | .name = "lb_tx_hash_to_port_mapping" , |
561 | .array_size = LB_TX_HASHTABLE_SIZE, |
562 | .type = TEAM_OPTION_TYPE_U32, |
563 | .init = lb_tx_hash_to_port_mapping_init, |
564 | .getter = lb_tx_hash_to_port_mapping_get, |
565 | .setter = lb_tx_hash_to_port_mapping_set, |
566 | }, |
567 | { |
568 | .name = "lb_hash_stats" , |
569 | .array_size = LB_TX_HASHTABLE_SIZE, |
570 | .type = TEAM_OPTION_TYPE_BINARY, |
571 | .init = lb_hash_stats_init, |
572 | .getter = lb_hash_stats_get, |
573 | }, |
574 | { |
575 | .name = "lb_port_stats" , |
576 | .per_port = true, |
577 | .type = TEAM_OPTION_TYPE_BINARY, |
578 | .init = lb_port_stats_init, |
579 | .getter = lb_port_stats_get, |
580 | }, |
581 | { |
582 | .name = "lb_stats_refresh_interval" , |
583 | .type = TEAM_OPTION_TYPE_U32, |
584 | .getter = lb_stats_refresh_interval_get, |
585 | .setter = lb_stats_refresh_interval_set, |
586 | }, |
587 | }; |
588 | |
589 | static int lb_init(struct team *team) |
590 | { |
591 | struct lb_priv *lb_priv = get_lb_priv(team); |
592 | lb_select_tx_port_func_t *func; |
593 | int i, err; |
594 | |
595 | /* set default tx port selector */ |
596 | func = lb_select_tx_port_get_func(name: "hash" ); |
597 | BUG_ON(!func); |
598 | rcu_assign_pointer(lb_priv->select_tx_port_func, func); |
599 | |
600 | lb_priv->ex = kzalloc(size: sizeof(*lb_priv->ex), GFP_KERNEL); |
601 | if (!lb_priv->ex) |
602 | return -ENOMEM; |
603 | lb_priv->ex->team = team; |
604 | |
605 | lb_priv->pcpu_stats = alloc_percpu(struct lb_pcpu_stats); |
606 | if (!lb_priv->pcpu_stats) { |
607 | err = -ENOMEM; |
608 | goto err_alloc_pcpu_stats; |
609 | } |
610 | |
611 | for_each_possible_cpu(i) { |
612 | struct lb_pcpu_stats *team_lb_stats; |
613 | team_lb_stats = per_cpu_ptr(lb_priv->pcpu_stats, i); |
614 | u64_stats_init(syncp: &team_lb_stats->syncp); |
615 | } |
616 | |
617 | |
618 | INIT_DELAYED_WORK(&lb_priv->ex->stats.refresh_dw, lb_stats_refresh); |
619 | |
620 | err = team_options_register(team, option: lb_options, ARRAY_SIZE(lb_options)); |
621 | if (err) |
622 | goto err_options_register; |
623 | return 0; |
624 | |
625 | err_options_register: |
626 | free_percpu(pdata: lb_priv->pcpu_stats); |
627 | err_alloc_pcpu_stats: |
628 | kfree(objp: lb_priv->ex); |
629 | return err; |
630 | } |
631 | |
632 | static void lb_exit(struct team *team) |
633 | { |
634 | struct lb_priv *lb_priv = get_lb_priv(team); |
635 | |
636 | team_options_unregister(team, option: lb_options, |
637 | ARRAY_SIZE(lb_options)); |
638 | lb_bpf_func_free(team); |
639 | cancel_delayed_work_sync(dwork: &lb_priv->ex->stats.refresh_dw); |
640 | free_percpu(pdata: lb_priv->pcpu_stats); |
641 | kfree(objp: lb_priv->ex); |
642 | } |
643 | |
644 | static int lb_port_enter(struct team *team, struct team_port *port) |
645 | { |
646 | struct lb_port_priv *lb_port_priv = get_lb_port_priv(port); |
647 | |
648 | lb_port_priv->pcpu_stats = alloc_percpu(struct lb_stats); |
649 | if (!lb_port_priv->pcpu_stats) |
650 | return -ENOMEM; |
651 | return 0; |
652 | } |
653 | |
654 | static void lb_port_leave(struct team *team, struct team_port *port) |
655 | { |
656 | struct lb_port_priv *lb_port_priv = get_lb_port_priv(port); |
657 | |
658 | free_percpu(pdata: lb_port_priv->pcpu_stats); |
659 | } |
660 | |
661 | static void lb_port_disabled(struct team *team, struct team_port *port) |
662 | { |
663 | lb_tx_hash_to_port_mapping_null_port(team, port); |
664 | } |
665 | |
666 | static const struct team_mode_ops lb_mode_ops = { |
667 | .init = lb_init, |
668 | .exit = lb_exit, |
669 | .port_enter = lb_port_enter, |
670 | .port_leave = lb_port_leave, |
671 | .port_disabled = lb_port_disabled, |
672 | .receive = lb_receive, |
673 | .transmit = lb_transmit, |
674 | }; |
675 | |
676 | static const struct team_mode lb_mode = { |
677 | .kind = "loadbalance" , |
678 | .owner = THIS_MODULE, |
679 | .priv_size = sizeof(struct lb_priv), |
680 | .port_priv_size = sizeof(struct lb_port_priv), |
681 | .ops = &lb_mode_ops, |
682 | .lag_tx_type = NETDEV_LAG_TX_TYPE_HASH, |
683 | }; |
684 | |
685 | static int __init lb_init_module(void) |
686 | { |
687 | return team_mode_register(mode: &lb_mode); |
688 | } |
689 | |
690 | static void __exit lb_cleanup_module(void) |
691 | { |
692 | team_mode_unregister(mode: &lb_mode); |
693 | } |
694 | |
695 | module_init(lb_init_module); |
696 | module_exit(lb_cleanup_module); |
697 | |
698 | MODULE_LICENSE("GPL v2" ); |
699 | MODULE_AUTHOR("Jiri Pirko <jpirko@redhat.com>" ); |
700 | MODULE_DESCRIPTION("Load-balancing mode for team" ); |
701 | MODULE_ALIAS_TEAM_MODE("loadbalance" ); |
702 | |