1// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
2/* Copyright (c) 2017-2018 Mellanox Technologies. All rights reserved */
3
4#include <linux/kernel.h>
5#include <linux/slab.h>
6#include <linux/list.h>
7#include <linux/errno.h>
8
9#include "item.h"
10#include "core_acl_flex_keys.h"
11
12/* For the purpose of the driver, define an internal storage scratchpad
13 * that will be used to store key/mask values. For each defined element type
14 * define an internal storage geometry.
15 *
16 * When adding new elements, MLXSW_AFK_ELEMENT_STORAGE_SIZE must be increased
17 * accordingly.
18 */
19static const struct mlxsw_afk_element_info mlxsw_afk_element_infos[] = {
20 MLXSW_AFK_ELEMENT_INFO_U32(SRC_SYS_PORT, 0x00, 16, 16),
21 MLXSW_AFK_ELEMENT_INFO_BUF(DMAC_32_47, 0x04, 2),
22 MLXSW_AFK_ELEMENT_INFO_BUF(DMAC_0_31, 0x06, 4),
23 MLXSW_AFK_ELEMENT_INFO_BUF(SMAC_32_47, 0x0A, 2),
24 MLXSW_AFK_ELEMENT_INFO_BUF(SMAC_0_31, 0x0C, 4),
25 MLXSW_AFK_ELEMENT_INFO_U32(ETHERTYPE, 0x00, 0, 16),
26 MLXSW_AFK_ELEMENT_INFO_U32(IP_PROTO, 0x10, 0, 8),
27 MLXSW_AFK_ELEMENT_INFO_U32(VID, 0x10, 8, 12),
28 MLXSW_AFK_ELEMENT_INFO_U32(PCP, 0x10, 20, 3),
29 MLXSW_AFK_ELEMENT_INFO_U32(TCP_FLAGS, 0x10, 23, 9),
30 MLXSW_AFK_ELEMENT_INFO_U32(DST_L4_PORT, 0x14, 0, 16),
31 MLXSW_AFK_ELEMENT_INFO_U32(SRC_L4_PORT, 0x14, 16, 16),
32 MLXSW_AFK_ELEMENT_INFO_U32(IP_TTL_, 0x18, 0, 8),
33 MLXSW_AFK_ELEMENT_INFO_U32(IP_ECN, 0x18, 9, 2),
34 MLXSW_AFK_ELEMENT_INFO_U32(IP_DSCP, 0x18, 11, 6),
35 MLXSW_AFK_ELEMENT_INFO_U32(VIRT_ROUTER, 0x18, 17, 12),
36 MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_96_127, 0x20, 4),
37 MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_64_95, 0x24, 4),
38 MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_32_63, 0x28, 4),
39 MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_0_31, 0x2C, 4),
40 MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP_96_127, 0x30, 4),
41 MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP_64_95, 0x34, 4),
42 MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP_32_63, 0x38, 4),
43 MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP_0_31, 0x3C, 4),
44 MLXSW_AFK_ELEMENT_INFO_U32(FDB_MISS, 0x40, 0, 1),
45 MLXSW_AFK_ELEMENT_INFO_U32(L4_PORT_RANGE, 0x40, 1, 16),
46 MLXSW_AFK_ELEMENT_INFO_U32(VIRT_ROUTER_0_3, 0x40, 17, 4),
47 MLXSW_AFK_ELEMENT_INFO_U32(VIRT_ROUTER_4_7, 0x40, 21, 4),
48 MLXSW_AFK_ELEMENT_INFO_U32(VIRT_ROUTER_MSB, 0x40, 25, 4),
49};
50
51struct mlxsw_afk {
52 struct list_head key_info_list;
53 unsigned int max_blocks;
54 const struct mlxsw_afk_ops *ops;
55 const struct mlxsw_afk_block *blocks;
56 unsigned int blocks_count;
57};
58
59static bool mlxsw_afk_blocks_check(struct mlxsw_afk *mlxsw_afk)
60{
61 int i;
62 int j;
63
64 for (i = 0; i < mlxsw_afk->blocks_count; i++) {
65 const struct mlxsw_afk_block *block = &mlxsw_afk->blocks[i];
66
67 for (j = 0; j < block->instances_count; j++) {
68 const struct mlxsw_afk_element_info *elinfo;
69 struct mlxsw_afk_element_inst *elinst;
70
71 elinst = &block->instances[j];
72 elinfo = &mlxsw_afk_element_infos[elinst->element];
73 if (elinst->type != elinfo->type ||
74 (!elinst->avoid_size_check &&
75 elinst->item.size.bits !=
76 elinfo->item.size.bits))
77 return false;
78 }
79 }
80 return true;
81}
82
83struct mlxsw_afk *mlxsw_afk_create(unsigned int max_blocks,
84 const struct mlxsw_afk_ops *ops)
85{
86 struct mlxsw_afk *mlxsw_afk;
87
88 mlxsw_afk = kzalloc(size: sizeof(*mlxsw_afk), GFP_KERNEL);
89 if (!mlxsw_afk)
90 return NULL;
91 INIT_LIST_HEAD(list: &mlxsw_afk->key_info_list);
92 mlxsw_afk->max_blocks = max_blocks;
93 mlxsw_afk->ops = ops;
94 mlxsw_afk->blocks = ops->blocks;
95 mlxsw_afk->blocks_count = ops->blocks_count;
96 WARN_ON(!mlxsw_afk_blocks_check(mlxsw_afk));
97 return mlxsw_afk;
98}
99EXPORT_SYMBOL(mlxsw_afk_create);
100
101void mlxsw_afk_destroy(struct mlxsw_afk *mlxsw_afk)
102{
103 WARN_ON(!list_empty(&mlxsw_afk->key_info_list));
104 kfree(objp: mlxsw_afk);
105}
106EXPORT_SYMBOL(mlxsw_afk_destroy);
107
108struct mlxsw_afk_key_info {
109 struct list_head list;
110 unsigned int ref_count;
111 unsigned int blocks_count;
112 int element_to_block[MLXSW_AFK_ELEMENT_MAX]; /* index is element, value
113 * is index inside "blocks"
114 */
115 struct mlxsw_afk_element_usage elusage;
116 const struct mlxsw_afk_block *blocks[];
117};
118
119static bool
120mlxsw_afk_key_info_elements_eq(struct mlxsw_afk_key_info *key_info,
121 struct mlxsw_afk_element_usage *elusage)
122{
123 return memcmp(p: &key_info->elusage, q: elusage, size: sizeof(*elusage)) == 0;
124}
125
126static struct mlxsw_afk_key_info *
127mlxsw_afk_key_info_find(struct mlxsw_afk *mlxsw_afk,
128 struct mlxsw_afk_element_usage *elusage)
129{
130 struct mlxsw_afk_key_info *key_info;
131
132 list_for_each_entry(key_info, &mlxsw_afk->key_info_list, list) {
133 if (mlxsw_afk_key_info_elements_eq(key_info, elusage))
134 return key_info;
135 }
136 return NULL;
137}
138
139struct mlxsw_afk_picker {
140 DECLARE_BITMAP(element, MLXSW_AFK_ELEMENT_MAX);
141 DECLARE_BITMAP(chosen_element, MLXSW_AFK_ELEMENT_MAX);
142 unsigned int total;
143};
144
145static void mlxsw_afk_picker_count_hits(struct mlxsw_afk *mlxsw_afk,
146 struct mlxsw_afk_picker *picker,
147 enum mlxsw_afk_element element)
148{
149 int i;
150 int j;
151
152 for (i = 0; i < mlxsw_afk->blocks_count; i++) {
153 const struct mlxsw_afk_block *block = &mlxsw_afk->blocks[i];
154
155 for (j = 0; j < block->instances_count; j++) {
156 struct mlxsw_afk_element_inst *elinst;
157
158 elinst = &block->instances[j];
159 if (elinst->element == element) {
160 __set_bit(element, picker[i].element);
161 picker[i].total++;
162 }
163 }
164 }
165}
166
167static void mlxsw_afk_picker_subtract_hits(struct mlxsw_afk *mlxsw_afk,
168 struct mlxsw_afk_picker *picker,
169 int block_index)
170{
171 DECLARE_BITMAP(hits_element, MLXSW_AFK_ELEMENT_MAX);
172 int i;
173 int j;
174
175 memcpy(&hits_element, &picker[block_index].element,
176 sizeof(hits_element));
177
178 for (i = 0; i < mlxsw_afk->blocks_count; i++) {
179 for_each_set_bit(j, hits_element, MLXSW_AFK_ELEMENT_MAX) {
180 if (__test_and_clear_bit(j, picker[i].element))
181 picker[i].total--;
182 }
183 }
184}
185
186static int mlxsw_afk_picker_most_hits_get(struct mlxsw_afk *mlxsw_afk,
187 struct mlxsw_afk_picker *picker)
188{
189 int most_index = -EINVAL; /* Should never happen to return this */
190 int most_hits = 0;
191 int i;
192
193 for (i = 0; i < mlxsw_afk->blocks_count; i++) {
194 if (picker[i].total > most_hits) {
195 most_hits = picker[i].total;
196 most_index = i;
197 }
198 }
199 return most_index;
200}
201
202static int mlxsw_afk_picker_key_info_add(struct mlxsw_afk *mlxsw_afk,
203 struct mlxsw_afk_picker *picker,
204 int block_index,
205 struct mlxsw_afk_key_info *key_info)
206{
207 enum mlxsw_afk_element element;
208
209 if (key_info->blocks_count == mlxsw_afk->max_blocks)
210 return -EINVAL;
211
212 for_each_set_bit(element, picker[block_index].chosen_element,
213 MLXSW_AFK_ELEMENT_MAX) {
214 key_info->element_to_block[element] = key_info->blocks_count;
215 mlxsw_afk_element_usage_add(elusage: &key_info->elusage, element);
216 }
217
218 key_info->blocks[key_info->blocks_count] =
219 &mlxsw_afk->blocks[block_index];
220 key_info->blocks_count++;
221 return 0;
222}
223
224static int mlxsw_afk_keys_fill(struct mlxsw_afk *mlxsw_afk,
225 unsigned long *chosen_blocks_bm,
226 struct mlxsw_afk_picker *picker,
227 struct mlxsw_afk_key_info *key_info)
228{
229 int i, err;
230
231 /* First fill only key blocks with high_entropy. */
232 for_each_set_bit(i, chosen_blocks_bm, mlxsw_afk->blocks_count) {
233 if (!mlxsw_afk->blocks[i].high_entropy)
234 continue;
235
236 err = mlxsw_afk_picker_key_info_add(mlxsw_afk, picker, block_index: i,
237 key_info);
238 if (err)
239 return err;
240 __clear_bit(i, chosen_blocks_bm);
241 }
242
243 /* Fill the rest of key blocks. */
244 for_each_set_bit(i, chosen_blocks_bm, mlxsw_afk->blocks_count) {
245 err = mlxsw_afk_picker_key_info_add(mlxsw_afk, picker, block_index: i,
246 key_info);
247 if (err)
248 return err;
249 }
250
251 return 0;
252}
253
254static int mlxsw_afk_picker(struct mlxsw_afk *mlxsw_afk,
255 struct mlxsw_afk_key_info *key_info,
256 struct mlxsw_afk_element_usage *elusage)
257{
258 DECLARE_BITMAP(elusage_chosen, MLXSW_AFK_ELEMENT_MAX) = {0};
259 struct mlxsw_afk_picker *picker;
260 unsigned long *chosen_blocks_bm;
261 enum mlxsw_afk_element element;
262 int err;
263
264 picker = kcalloc(n: mlxsw_afk->blocks_count, size: sizeof(*picker), GFP_KERNEL);
265 if (!picker)
266 return -ENOMEM;
267
268 chosen_blocks_bm = bitmap_zalloc(nbits: mlxsw_afk->blocks_count, GFP_KERNEL);
269 if (!chosen_blocks_bm) {
270 err = -ENOMEM;
271 goto err_bitmap_alloc;
272 }
273
274 /* Since the same elements could be present in multiple blocks,
275 * we must find out optimal block list in order to make the
276 * block count as low as possible.
277 *
278 * First, we count hits. We go over all available blocks and count
279 * how many of requested elements are covered by each.
280 *
281 * Then in loop, we find block with most hits and add it to
282 * output key_info. Then we have to subtract this block hits so
283 * the next iteration will find most suitable block for
284 * the rest of requested elements.
285 */
286
287 mlxsw_afk_element_usage_for_each(element, elusage)
288 mlxsw_afk_picker_count_hits(mlxsw_afk, picker, element);
289
290 do {
291 int block_index;
292
293 block_index = mlxsw_afk_picker_most_hits_get(mlxsw_afk, picker);
294 if (block_index < 0) {
295 err = block_index;
296 goto out;
297 }
298
299 __set_bit(block_index, chosen_blocks_bm);
300
301 bitmap_copy(dst: picker[block_index].chosen_element,
302 src: picker[block_index].element, nbits: MLXSW_AFK_ELEMENT_MAX);
303
304 bitmap_or(dst: elusage_chosen, src1: elusage_chosen,
305 src2: picker[block_index].chosen_element,
306 nbits: MLXSW_AFK_ELEMENT_MAX);
307
308 mlxsw_afk_picker_subtract_hits(mlxsw_afk, picker, block_index);
309
310 } while (!bitmap_equal(src1: elusage_chosen, src2: elusage->usage,
311 nbits: MLXSW_AFK_ELEMENT_MAX));
312
313 err = mlxsw_afk_keys_fill(mlxsw_afk, chosen_blocks_bm, picker,
314 key_info);
315out:
316 bitmap_free(bitmap: chosen_blocks_bm);
317err_bitmap_alloc:
318 kfree(objp: picker);
319 return err;
320}
321
322static struct mlxsw_afk_key_info *
323mlxsw_afk_key_info_create(struct mlxsw_afk *mlxsw_afk,
324 struct mlxsw_afk_element_usage *elusage)
325{
326 struct mlxsw_afk_key_info *key_info;
327 int err;
328
329 key_info = kzalloc(struct_size(key_info, blocks, mlxsw_afk->max_blocks),
330 GFP_KERNEL);
331 if (!key_info)
332 return ERR_PTR(error: -ENOMEM);
333 err = mlxsw_afk_picker(mlxsw_afk, key_info, elusage);
334 if (err)
335 goto err_picker;
336 list_add(new: &key_info->list, head: &mlxsw_afk->key_info_list);
337 key_info->ref_count = 1;
338 return key_info;
339
340err_picker:
341 kfree(objp: key_info);
342 return ERR_PTR(error: err);
343}
344
345static void mlxsw_afk_key_info_destroy(struct mlxsw_afk_key_info *key_info)
346{
347 list_del(entry: &key_info->list);
348 kfree(objp: key_info);
349}
350
351struct mlxsw_afk_key_info *
352mlxsw_afk_key_info_get(struct mlxsw_afk *mlxsw_afk,
353 struct mlxsw_afk_element_usage *elusage)
354{
355 struct mlxsw_afk_key_info *key_info;
356
357 key_info = mlxsw_afk_key_info_find(mlxsw_afk, elusage);
358 if (key_info) {
359 key_info->ref_count++;
360 return key_info;
361 }
362 return mlxsw_afk_key_info_create(mlxsw_afk, elusage);
363}
364EXPORT_SYMBOL(mlxsw_afk_key_info_get);
365
366void mlxsw_afk_key_info_put(struct mlxsw_afk_key_info *key_info)
367{
368 if (--key_info->ref_count)
369 return;
370 mlxsw_afk_key_info_destroy(key_info);
371}
372EXPORT_SYMBOL(mlxsw_afk_key_info_put);
373
374bool mlxsw_afk_key_info_subset(struct mlxsw_afk_key_info *key_info,
375 struct mlxsw_afk_element_usage *elusage)
376{
377 return mlxsw_afk_element_usage_subset(elusage_small: elusage, elusage_big: &key_info->elusage);
378}
379EXPORT_SYMBOL(mlxsw_afk_key_info_subset);
380
381static const struct mlxsw_afk_element_inst *
382mlxsw_afk_block_elinst_get(const struct mlxsw_afk_block *block,
383 enum mlxsw_afk_element element)
384{
385 int i;
386
387 for (i = 0; i < block->instances_count; i++) {
388 struct mlxsw_afk_element_inst *elinst;
389
390 elinst = &block->instances[i];
391 if (elinst->element == element)
392 return elinst;
393 }
394 return NULL;
395}
396
397static const struct mlxsw_afk_element_inst *
398mlxsw_afk_key_info_elinst_get(struct mlxsw_afk_key_info *key_info,
399 enum mlxsw_afk_element element,
400 int *p_block_index)
401{
402 const struct mlxsw_afk_element_inst *elinst;
403 const struct mlxsw_afk_block *block;
404 int block_index;
405
406 if (WARN_ON(!test_bit(element, key_info->elusage.usage)))
407 return NULL;
408 block_index = key_info->element_to_block[element];
409 block = key_info->blocks[block_index];
410
411 elinst = mlxsw_afk_block_elinst_get(block, element);
412 if (WARN_ON(!elinst))
413 return NULL;
414
415 *p_block_index = block_index;
416 return elinst;
417}
418
419u16
420mlxsw_afk_key_info_block_encoding_get(const struct mlxsw_afk_key_info *key_info,
421 int block_index)
422{
423 return key_info->blocks[block_index]->encoding;
424}
425EXPORT_SYMBOL(mlxsw_afk_key_info_block_encoding_get);
426
427unsigned int
428mlxsw_afk_key_info_blocks_count_get(const struct mlxsw_afk_key_info *key_info)
429{
430 return key_info->blocks_count;
431}
432EXPORT_SYMBOL(mlxsw_afk_key_info_blocks_count_get);
433
434void mlxsw_afk_values_add_u32(struct mlxsw_afk_element_values *values,
435 enum mlxsw_afk_element element,
436 u32 key_value, u32 mask_value)
437{
438 const struct mlxsw_afk_element_info *elinfo =
439 &mlxsw_afk_element_infos[element];
440 const struct mlxsw_item *storage_item = &elinfo->item;
441
442 if (!mask_value)
443 return;
444 if (WARN_ON(elinfo->type != MLXSW_AFK_ELEMENT_TYPE_U32))
445 return;
446 __mlxsw_item_set32(buf: values->storage.key, item: storage_item, index: 0, val: key_value);
447 __mlxsw_item_set32(buf: values->storage.mask, item: storage_item, index: 0, val: mask_value);
448 mlxsw_afk_element_usage_add(elusage: &values->elusage, element);
449}
450EXPORT_SYMBOL(mlxsw_afk_values_add_u32);
451
452void mlxsw_afk_values_add_buf(struct mlxsw_afk_element_values *values,
453 enum mlxsw_afk_element element,
454 const char *key_value, const char *mask_value,
455 unsigned int len)
456{
457 const struct mlxsw_afk_element_info *elinfo =
458 &mlxsw_afk_element_infos[element];
459 const struct mlxsw_item *storage_item = &elinfo->item;
460
461 if (!memchr_inv(p: mask_value, c: 0, size: len)) /* If mask is zero */
462 return;
463 if (WARN_ON(elinfo->type != MLXSW_AFK_ELEMENT_TYPE_BUF) ||
464 WARN_ON(elinfo->item.size.bytes != len))
465 return;
466 __mlxsw_item_memcpy_to(buf: values->storage.key, src: key_value,
467 item: storage_item, index: 0);
468 __mlxsw_item_memcpy_to(buf: values->storage.mask, src: mask_value,
469 item: storage_item, index: 0);
470 mlxsw_afk_element_usage_add(elusage: &values->elusage, element);
471}
472EXPORT_SYMBOL(mlxsw_afk_values_add_buf);
473
474static void mlxsw_sp_afk_encode_u32(const struct mlxsw_item *storage_item,
475 const struct mlxsw_item *output_item,
476 char *storage, char *output, int diff)
477{
478 u32 value;
479
480 value = __mlxsw_item_get32(buf: storage, item: storage_item, index: 0);
481 __mlxsw_item_set32(buf: output, item: output_item, index: 0, val: value + diff);
482}
483
484static void mlxsw_sp_afk_encode_buf(const struct mlxsw_item *storage_item,
485 const struct mlxsw_item *output_item,
486 char *storage, char *output)
487{
488 char *storage_data = __mlxsw_item_data(buf: storage, item: storage_item, index: 0);
489 char *output_data = __mlxsw_item_data(buf: output, item: output_item, index: 0);
490 size_t len = output_item->size.bytes;
491
492 memcpy(output_data, storage_data, len);
493}
494
495static void
496mlxsw_sp_afk_encode_one(const struct mlxsw_afk_element_inst *elinst,
497 char *output, char *storage, int u32_diff)
498{
499 const struct mlxsw_item *output_item = &elinst->item;
500 const struct mlxsw_afk_element_info *elinfo;
501 const struct mlxsw_item *storage_item;
502
503 elinfo = &mlxsw_afk_element_infos[elinst->element];
504 storage_item = &elinfo->item;
505 if (elinst->type == MLXSW_AFK_ELEMENT_TYPE_U32)
506 mlxsw_sp_afk_encode_u32(storage_item, output_item,
507 storage, output, diff: u32_diff);
508 else if (elinst->type == MLXSW_AFK_ELEMENT_TYPE_BUF)
509 mlxsw_sp_afk_encode_buf(storage_item, output_item,
510 storage, output);
511}
512
513#define MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE 16
514
515void mlxsw_afk_encode(struct mlxsw_afk *mlxsw_afk,
516 struct mlxsw_afk_key_info *key_info,
517 struct mlxsw_afk_element_values *values,
518 char *key, char *mask)
519{
520 unsigned int blocks_count =
521 mlxsw_afk_key_info_blocks_count_get(key_info);
522 char block_mask[MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE];
523 char block_key[MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE];
524 const struct mlxsw_afk_element_inst *elinst;
525 enum mlxsw_afk_element element;
526 int block_index, i;
527
528 for (i = 0; i < blocks_count; i++) {
529 memset(block_key, 0, MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE);
530 memset(block_mask, 0, MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE);
531
532 mlxsw_afk_element_usage_for_each(element, &values->elusage) {
533 elinst = mlxsw_afk_key_info_elinst_get(key_info,
534 element,
535 p_block_index: &block_index);
536 if (!elinst || block_index != i)
537 continue;
538
539 mlxsw_sp_afk_encode_one(elinst, output: block_key,
540 storage: values->storage.key,
541 u32_diff: elinst->u32_key_diff);
542 mlxsw_sp_afk_encode_one(elinst, output: block_mask,
543 storage: values->storage.mask, u32_diff: 0);
544 }
545
546 mlxsw_afk->ops->encode_block(key, i, block_key);
547 mlxsw_afk->ops->encode_block(mask, i, block_mask);
548 }
549}
550EXPORT_SYMBOL(mlxsw_afk_encode);
551
552void mlxsw_afk_clear(struct mlxsw_afk *mlxsw_afk, char *key,
553 int block_start, int block_end)
554{
555 int i;
556
557 for (i = block_start; i <= block_end; i++)
558 mlxsw_afk->ops->clear_block(key, i);
559}
560EXPORT_SYMBOL(mlxsw_afk_clear);
561

source code of linux/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c