1 | // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) |
2 | /* Copyright (C) 2018 Netronome Systems, Inc. */ |
3 | |
4 | #include <linux/bitops.h> |
5 | #include <linux/kernel.h> |
6 | #include <linux/log2.h> |
7 | |
8 | #include "../nfpcore/nfp_cpp.h" |
9 | #include "../nfpcore/nfp_nffw.h" |
10 | #include "../nfp_app.h" |
11 | #include "../nfp_abi.h" |
12 | #include "../nfp_main.h" |
13 | #include "../nfp_net.h" |
14 | #include "main.h" |
15 | |
16 | #define NFP_NUM_PRIOS_SYM_NAME "_abi_pci_dscp_num_prio_%u" |
17 | #define NFP_NUM_BANDS_SYM_NAME "_abi_pci_dscp_num_band_%u" |
18 | #define NFP_ACT_MASK_SYM_NAME "_abi_nfd_out_q_actions_%u" |
19 | |
20 | #define NFP_RED_SUPPORT_SYM_NAME "_abi_nfd_out_red_offload_%u" |
21 | |
22 | #define NFP_QLVL_SYM_NAME "_abi_nfd_out_q_lvls_%u%s" |
23 | #define NFP_QLVL_STRIDE 16 |
24 | #define NFP_QLVL_BLOG_BYTES 0 |
25 | #define NFP_QLVL_BLOG_PKTS 4 |
26 | #define NFP_QLVL_THRS 8 |
27 | #define NFP_QLVL_ACT 12 |
28 | |
29 | #define NFP_QMSTAT_SYM_NAME "_abi_nfdqm%u_stats%s" |
30 | #define NFP_QMSTAT_STRIDE 32 |
31 | #define NFP_QMSTAT_NON_STO 0 |
32 | #define NFP_QMSTAT_STO 8 |
33 | #define NFP_QMSTAT_DROP 16 |
34 | #define NFP_QMSTAT_ECN 24 |
35 | |
36 | #define NFP_Q_STAT_SYM_NAME "_abi_nfd_rxq_stats%u%s" |
37 | #define NFP_Q_STAT_STRIDE 16 |
38 | #define NFP_Q_STAT_PKTS 0 |
39 | #define NFP_Q_STAT_BYTES 8 |
40 | |
41 | #define NFP_NET_ABM_MBOX_CMD NFP_NET_CFG_MBOX_SIMPLE_CMD |
42 | #define NFP_NET_ABM_MBOX_RET NFP_NET_CFG_MBOX_SIMPLE_RET |
43 | #define NFP_NET_ABM_MBOX_DATALEN NFP_NET_CFG_MBOX_SIMPLE_VAL |
44 | #define NFP_NET_ABM_MBOX_RESERVED (NFP_NET_CFG_MBOX_SIMPLE_VAL + 4) |
45 | #define NFP_NET_ABM_MBOX_DATA (NFP_NET_CFG_MBOX_SIMPLE_VAL + 8) |
46 | |
47 | static int |
48 | nfp_abm_ctrl_stat(struct nfp_abm_link *alink, const struct nfp_rtsym *sym, |
49 | unsigned int stride, unsigned int offset, unsigned int band, |
50 | unsigned int queue, bool is_u64, u64 *res) |
51 | { |
52 | struct nfp_cpp *cpp = alink->abm->app->cpp; |
53 | u64 val, sym_offset; |
54 | unsigned int qid; |
55 | u32 val32; |
56 | int err; |
57 | |
58 | qid = band * NFP_NET_MAX_RX_RINGS + alink->queue_base + queue; |
59 | |
60 | sym_offset = qid * stride + offset; |
61 | if (is_u64) |
62 | err = __nfp_rtsym_readq(cpp, sym, action: 3, token: 0, off: sym_offset, value: &val); |
63 | else |
64 | err = __nfp_rtsym_readl(cpp, sym, action: 3, token: 0, off: sym_offset, value: &val32); |
65 | if (err) { |
66 | nfp_err(cpp, "RED offload reading stat failed on vNIC %d band %d queue %d (+ %d)\n" , |
67 | alink->id, band, queue, alink->queue_base); |
68 | return err; |
69 | } |
70 | |
71 | *res = is_u64 ? val : val32; |
72 | return 0; |
73 | } |
74 | |
75 | int __nfp_abm_ctrl_set_q_lvl(struct nfp_abm *abm, unsigned int id, u32 val) |
76 | { |
77 | struct nfp_cpp *cpp = abm->app->cpp; |
78 | u64 sym_offset; |
79 | int err; |
80 | |
81 | __clear_bit(id, abm->threshold_undef); |
82 | if (abm->thresholds[id] == val) |
83 | return 0; |
84 | |
85 | sym_offset = id * NFP_QLVL_STRIDE + NFP_QLVL_THRS; |
86 | err = __nfp_rtsym_writel(cpp, sym: abm->q_lvls, action: 4, token: 0, off: sym_offset, value: val); |
87 | if (err) { |
88 | nfp_err(cpp, |
89 | "RED offload setting level failed on subqueue %d\n" , |
90 | id); |
91 | return err; |
92 | } |
93 | |
94 | abm->thresholds[id] = val; |
95 | return 0; |
96 | } |
97 | |
98 | int nfp_abm_ctrl_set_q_lvl(struct nfp_abm_link *alink, unsigned int band, |
99 | unsigned int queue, u32 val) |
100 | { |
101 | unsigned int threshold; |
102 | |
103 | threshold = band * NFP_NET_MAX_RX_RINGS + alink->queue_base + queue; |
104 | |
105 | return __nfp_abm_ctrl_set_q_lvl(abm: alink->abm, id: threshold, val); |
106 | } |
107 | |
108 | int __nfp_abm_ctrl_set_q_act(struct nfp_abm *abm, unsigned int id, |
109 | enum nfp_abm_q_action act) |
110 | { |
111 | struct nfp_cpp *cpp = abm->app->cpp; |
112 | u64 sym_offset; |
113 | int err; |
114 | |
115 | if (abm->actions[id] == act) |
116 | return 0; |
117 | |
118 | sym_offset = id * NFP_QLVL_STRIDE + NFP_QLVL_ACT; |
119 | err = __nfp_rtsym_writel(cpp, sym: abm->q_lvls, action: 4, token: 0, off: sym_offset, value: act); |
120 | if (err) { |
121 | nfp_err(cpp, |
122 | "RED offload setting action failed on subqueue %d\n" , |
123 | id); |
124 | return err; |
125 | } |
126 | |
127 | abm->actions[id] = act; |
128 | return 0; |
129 | } |
130 | |
131 | int nfp_abm_ctrl_set_q_act(struct nfp_abm_link *alink, unsigned int band, |
132 | unsigned int queue, enum nfp_abm_q_action act) |
133 | { |
134 | unsigned int qid; |
135 | |
136 | qid = band * NFP_NET_MAX_RX_RINGS + alink->queue_base + queue; |
137 | |
138 | return __nfp_abm_ctrl_set_q_act(abm: alink->abm, id: qid, act); |
139 | } |
140 | |
141 | u64 nfp_abm_ctrl_stat_non_sto(struct nfp_abm_link *alink, unsigned int queue) |
142 | { |
143 | unsigned int band; |
144 | u64 val, sum = 0; |
145 | |
146 | for (band = 0; band < alink->abm->num_bands; band++) { |
147 | if (nfp_abm_ctrl_stat(alink, sym: alink->abm->qm_stats, |
148 | NFP_QMSTAT_STRIDE, NFP_QMSTAT_NON_STO, |
149 | band, queue, is_u64: true, res: &val)) |
150 | return 0; |
151 | sum += val; |
152 | } |
153 | |
154 | return sum; |
155 | } |
156 | |
157 | u64 nfp_abm_ctrl_stat_sto(struct nfp_abm_link *alink, unsigned int queue) |
158 | { |
159 | unsigned int band; |
160 | u64 val, sum = 0; |
161 | |
162 | for (band = 0; band < alink->abm->num_bands; band++) { |
163 | if (nfp_abm_ctrl_stat(alink, sym: alink->abm->qm_stats, |
164 | NFP_QMSTAT_STRIDE, NFP_QMSTAT_STO, |
165 | band, queue, is_u64: true, res: &val)) |
166 | return 0; |
167 | sum += val; |
168 | } |
169 | |
170 | return sum; |
171 | } |
172 | |
173 | static int |
174 | nfp_abm_ctrl_stat_basic(struct nfp_abm_link *alink, unsigned int band, |
175 | unsigned int queue, unsigned int off, u64 *val) |
176 | { |
177 | if (!nfp_abm_has_prio(abm: alink->abm)) { |
178 | if (!band) { |
179 | unsigned int id = alink->queue_base + queue; |
180 | |
181 | *val = nn_readq(nn: alink->vnic, |
182 | NFP_NET_CFG_RXR_STATS(id) + off); |
183 | } else { |
184 | *val = 0; |
185 | } |
186 | |
187 | return 0; |
188 | } else { |
189 | return nfp_abm_ctrl_stat(alink, sym: alink->abm->q_stats, |
190 | NFP_Q_STAT_STRIDE, offset: off, band, queue, |
191 | is_u64: true, res: val); |
192 | } |
193 | } |
194 | |
195 | int nfp_abm_ctrl_read_q_stats(struct nfp_abm_link *alink, unsigned int band, |
196 | unsigned int queue, struct nfp_alink_stats *stats) |
197 | { |
198 | int err; |
199 | |
200 | err = nfp_abm_ctrl_stat_basic(alink, band, queue, NFP_Q_STAT_PKTS, |
201 | val: &stats->tx_pkts); |
202 | if (err) |
203 | return err; |
204 | |
205 | err = nfp_abm_ctrl_stat_basic(alink, band, queue, NFP_Q_STAT_BYTES, |
206 | val: &stats->tx_bytes); |
207 | if (err) |
208 | return err; |
209 | |
210 | err = nfp_abm_ctrl_stat(alink, sym: alink->abm->q_lvls, NFP_QLVL_STRIDE, |
211 | NFP_QLVL_BLOG_BYTES, band, queue, is_u64: false, |
212 | res: &stats->backlog_bytes); |
213 | if (err) |
214 | return err; |
215 | |
216 | err = nfp_abm_ctrl_stat(alink, sym: alink->abm->q_lvls, |
217 | NFP_QLVL_STRIDE, NFP_QLVL_BLOG_PKTS, |
218 | band, queue, is_u64: false, res: &stats->backlog_pkts); |
219 | if (err) |
220 | return err; |
221 | |
222 | err = nfp_abm_ctrl_stat(alink, sym: alink->abm->qm_stats, |
223 | NFP_QMSTAT_STRIDE, NFP_QMSTAT_DROP, |
224 | band, queue, is_u64: true, res: &stats->drops); |
225 | if (err) |
226 | return err; |
227 | |
228 | return nfp_abm_ctrl_stat(alink, sym: alink->abm->qm_stats, |
229 | NFP_QMSTAT_STRIDE, NFP_QMSTAT_ECN, |
230 | band, queue, is_u64: true, res: &stats->overlimits); |
231 | } |
232 | |
233 | int nfp_abm_ctrl_read_q_xstats(struct nfp_abm_link *alink, |
234 | unsigned int band, unsigned int queue, |
235 | struct nfp_alink_xstats *xstats) |
236 | { |
237 | int err; |
238 | |
239 | err = nfp_abm_ctrl_stat(alink, sym: alink->abm->qm_stats, |
240 | NFP_QMSTAT_STRIDE, NFP_QMSTAT_DROP, |
241 | band, queue, is_u64: true, res: &xstats->pdrop); |
242 | if (err) |
243 | return err; |
244 | |
245 | return nfp_abm_ctrl_stat(alink, sym: alink->abm->qm_stats, |
246 | NFP_QMSTAT_STRIDE, NFP_QMSTAT_ECN, |
247 | band, queue, is_u64: true, res: &xstats->ecn_marked); |
248 | } |
249 | |
250 | int nfp_abm_ctrl_qm_enable(struct nfp_abm *abm) |
251 | { |
252 | return nfp_mbox_cmd(pf: abm->app->pf, cmd: NFP_MBOX_PCIE_ABM_ENABLE, |
253 | NULL, in_length: 0, NULL, out_length: 0); |
254 | } |
255 | |
256 | int nfp_abm_ctrl_qm_disable(struct nfp_abm *abm) |
257 | { |
258 | return nfp_mbox_cmd(pf: abm->app->pf, cmd: NFP_MBOX_PCIE_ABM_DISABLE, |
259 | NULL, in_length: 0, NULL, out_length: 0); |
260 | } |
261 | |
262 | int nfp_abm_ctrl_prio_map_update(struct nfp_abm_link *alink, u32 *packed) |
263 | { |
264 | const u32 cmd = NFP_NET_CFG_MBOX_CMD_PCI_DSCP_PRIOMAP_SET; |
265 | struct nfp_net *nn = alink->vnic; |
266 | unsigned int i; |
267 | int err; |
268 | |
269 | err = nfp_net_mbox_lock(nn, data_size: alink->abm->prio_map_len); |
270 | if (err) |
271 | return err; |
272 | |
273 | /* Write data_len and wipe reserved */ |
274 | nn_writeq(nn, off: nn->tlv_caps.mbox_off + NFP_NET_ABM_MBOX_DATALEN, |
275 | val: alink->abm->prio_map_len); |
276 | |
277 | for (i = 0; i < alink->abm->prio_map_len; i += sizeof(u32)) |
278 | nn_writel(nn, off: nn->tlv_caps.mbox_off + NFP_NET_ABM_MBOX_DATA + i, |
279 | val: packed[i / sizeof(u32)]); |
280 | |
281 | err = nfp_net_mbox_reconfig_and_unlock(nn, mbox_cmd: cmd); |
282 | if (err) |
283 | nfp_err(alink->abm->app->cpp, |
284 | "setting DSCP -> VQ map failed with error %d\n" , err); |
285 | return err; |
286 | } |
287 | |
288 | static int nfp_abm_ctrl_prio_check_params(struct nfp_abm_link *alink) |
289 | { |
290 | struct nfp_abm *abm = alink->abm; |
291 | struct nfp_net *nn = alink->vnic; |
292 | unsigned int min_mbox_sz; |
293 | |
294 | if (!nfp_abm_has_prio(abm: alink->abm)) |
295 | return 0; |
296 | |
297 | min_mbox_sz = NFP_NET_ABM_MBOX_DATA + alink->abm->prio_map_len; |
298 | if (nn->tlv_caps.mbox_len < min_mbox_sz) { |
299 | nfp_err(abm->app->pf->cpp, "vNIC mailbox too small for prio offload: %u, need: %u\n" , |
300 | nn->tlv_caps.mbox_len, min_mbox_sz); |
301 | return -EINVAL; |
302 | } |
303 | |
304 | return 0; |
305 | } |
306 | |
307 | int nfp_abm_ctrl_read_params(struct nfp_abm_link *alink) |
308 | { |
309 | alink->queue_base = nn_readl(nn: alink->vnic, NFP_NET_CFG_START_RXQ); |
310 | alink->queue_base /= alink->vnic->stride_rx; |
311 | |
312 | return nfp_abm_ctrl_prio_check_params(alink); |
313 | } |
314 | |
315 | static unsigned int nfp_abm_ctrl_prio_map_size(struct nfp_abm *abm) |
316 | { |
317 | unsigned int size; |
318 | |
319 | size = roundup_pow_of_two(order_base_2(abm->num_bands)); |
320 | size = DIV_ROUND_UP(size * abm->num_prios, BITS_PER_BYTE); |
321 | size = round_up(size, sizeof(u32)); |
322 | |
323 | return size; |
324 | } |
325 | |
326 | static const struct nfp_rtsym * |
327 | nfp_abm_ctrl_find_rtsym(struct nfp_pf *pf, const char *name, unsigned int size) |
328 | { |
329 | const struct nfp_rtsym *sym; |
330 | |
331 | sym = nfp_rtsym_lookup(rtbl: pf->rtbl, name); |
332 | if (!sym) { |
333 | nfp_err(pf->cpp, "Symbol '%s' not found\n" , name); |
334 | return ERR_PTR(error: -ENOENT); |
335 | } |
336 | if (nfp_rtsym_size(rtsym: sym) != size) { |
337 | nfp_err(pf->cpp, |
338 | "Symbol '%s' wrong size: expected %u got %llu\n" , |
339 | name, size, nfp_rtsym_size(sym)); |
340 | return ERR_PTR(error: -EINVAL); |
341 | } |
342 | |
343 | return sym; |
344 | } |
345 | |
346 | static const struct nfp_rtsym * |
347 | nfp_abm_ctrl_find_q_rtsym(struct nfp_abm *abm, const char *name_fmt, |
348 | size_t size) |
349 | { |
350 | char pf_symbol[64]; |
351 | |
352 | size = array3_size(size, abm->num_bands, NFP_NET_MAX_RX_RINGS); |
353 | snprintf(buf: pf_symbol, size: sizeof(pf_symbol), fmt: name_fmt, |
354 | abm->pf_id, nfp_abm_has_prio(abm) ? "_per_band" : "" ); |
355 | |
356 | return nfp_abm_ctrl_find_rtsym(pf: abm->app->pf, name: pf_symbol, size); |
357 | } |
358 | |
359 | int nfp_abm_ctrl_find_addrs(struct nfp_abm *abm) |
360 | { |
361 | struct nfp_pf *pf = abm->app->pf; |
362 | const struct nfp_rtsym *sym; |
363 | int res; |
364 | |
365 | abm->pf_id = nfp_cppcore_pcie_unit(cpp: pf->cpp); |
366 | |
367 | /* Check if Qdisc offloads are supported */ |
368 | res = nfp_pf_rtsym_read_optional(pf, NFP_RED_SUPPORT_SYM_NAME, default_val: 1); |
369 | if (res < 0) |
370 | return res; |
371 | abm->red_support = res; |
372 | |
373 | /* Read count of prios and prio bands */ |
374 | res = nfp_pf_rtsym_read_optional(pf, NFP_NUM_BANDS_SYM_NAME, default_val: 1); |
375 | if (res < 0) |
376 | return res; |
377 | abm->num_bands = res; |
378 | |
379 | res = nfp_pf_rtsym_read_optional(pf, NFP_NUM_PRIOS_SYM_NAME, default_val: 1); |
380 | if (res < 0) |
381 | return res; |
382 | abm->num_prios = res; |
383 | |
384 | /* Read available actions */ |
385 | res = nfp_pf_rtsym_read_optional(pf, NFP_ACT_MASK_SYM_NAME, |
386 | BIT(NFP_ABM_ACT_MARK_DROP)); |
387 | if (res < 0) |
388 | return res; |
389 | abm->action_mask = res; |
390 | |
391 | abm->prio_map_len = nfp_abm_ctrl_prio_map_size(abm); |
392 | abm->dscp_mask = GENMASK(7, 8 - order_base_2(abm->num_prios)); |
393 | |
394 | /* Check values are sane, U16_MAX is arbitrarily chosen as max */ |
395 | if (!is_power_of_2(n: abm->num_bands) || !is_power_of_2(n: abm->num_prios) || |
396 | abm->num_bands > U16_MAX || abm->num_prios > U16_MAX || |
397 | (abm->num_bands == 1) != (abm->num_prios == 1)) { |
398 | nfp_err(pf->cpp, |
399 | "invalid priomap description num bands: %u and num prios: %u\n" , |
400 | abm->num_bands, abm->num_prios); |
401 | return -EINVAL; |
402 | } |
403 | |
404 | /* Find level and stat symbols */ |
405 | if (!abm->red_support) |
406 | return 0; |
407 | |
408 | sym = nfp_abm_ctrl_find_q_rtsym(abm, NFP_QLVL_SYM_NAME, |
409 | NFP_QLVL_STRIDE); |
410 | if (IS_ERR(ptr: sym)) |
411 | return PTR_ERR(ptr: sym); |
412 | abm->q_lvls = sym; |
413 | |
414 | sym = nfp_abm_ctrl_find_q_rtsym(abm, NFP_QMSTAT_SYM_NAME, |
415 | NFP_QMSTAT_STRIDE); |
416 | if (IS_ERR(ptr: sym)) |
417 | return PTR_ERR(ptr: sym); |
418 | abm->qm_stats = sym; |
419 | |
420 | if (nfp_abm_has_prio(abm)) { |
421 | sym = nfp_abm_ctrl_find_q_rtsym(abm, NFP_Q_STAT_SYM_NAME, |
422 | NFP_Q_STAT_STRIDE); |
423 | if (IS_ERR(ptr: sym)) |
424 | return PTR_ERR(ptr: sym); |
425 | abm->q_stats = sym; |
426 | } |
427 | |
428 | return 0; |
429 | } |
430 | |