1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Marvell RVU Ethernet driver |
3 | * |
4 | * Copyright (C) 2020 Marvell. |
5 | * |
6 | */ |
7 | |
8 | #include <linux/interrupt.h> |
9 | #include <linux/pci.h> |
10 | #include <net/page_pool/helpers.h> |
11 | #include <net/tso.h> |
12 | #include <linux/bitfield.h> |
13 | |
14 | #include "otx2_reg.h" |
15 | #include "otx2_common.h" |
16 | #include "otx2_struct.h" |
17 | #include "cn10k.h" |
18 | |
19 | static void otx2_nix_rq_op_stats(struct queue_stats *stats, |
20 | struct otx2_nic *pfvf, int qidx) |
21 | { |
22 | u64 incr = (u64)qidx << 32; |
23 | u64 *ptr; |
24 | |
25 | ptr = (u64 *)otx2_get_regaddr(nic: pfvf, NIX_LF_RQ_OP_OCTS); |
26 | stats->bytes = otx2_atomic64_add(incr, ptr); |
27 | |
28 | ptr = (u64 *)otx2_get_regaddr(nic: pfvf, NIX_LF_RQ_OP_PKTS); |
29 | stats->pkts = otx2_atomic64_add(incr, ptr); |
30 | } |
31 | |
32 | static void otx2_nix_sq_op_stats(struct queue_stats *stats, |
33 | struct otx2_nic *pfvf, int qidx) |
34 | { |
35 | u64 incr = (u64)qidx << 32; |
36 | u64 *ptr; |
37 | |
38 | ptr = (u64 *)otx2_get_regaddr(nic: pfvf, NIX_LF_SQ_OP_OCTS); |
39 | stats->bytes = otx2_atomic64_add(incr, ptr); |
40 | |
41 | ptr = (u64 *)otx2_get_regaddr(nic: pfvf, NIX_LF_SQ_OP_PKTS); |
42 | stats->pkts = otx2_atomic64_add(incr, ptr); |
43 | } |
44 | |
45 | void otx2_update_lmac_stats(struct otx2_nic *pfvf) |
46 | { |
47 | struct msg_req *req; |
48 | |
49 | if (!netif_running(dev: pfvf->netdev)) |
50 | return; |
51 | |
52 | mutex_lock(&pfvf->mbox.lock); |
53 | req = otx2_mbox_alloc_msg_cgx_stats(mbox: &pfvf->mbox); |
54 | if (!req) { |
55 | mutex_unlock(lock: &pfvf->mbox.lock); |
56 | return; |
57 | } |
58 | |
59 | otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
60 | mutex_unlock(lock: &pfvf->mbox.lock); |
61 | } |
62 | |
63 | void otx2_update_lmac_fec_stats(struct otx2_nic *pfvf) |
64 | { |
65 | struct msg_req *req; |
66 | |
67 | if (!netif_running(dev: pfvf->netdev)) |
68 | return; |
69 | mutex_lock(&pfvf->mbox.lock); |
70 | req = otx2_mbox_alloc_msg_cgx_fec_stats(mbox: &pfvf->mbox); |
71 | if (req) |
72 | otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
73 | mutex_unlock(lock: &pfvf->mbox.lock); |
74 | } |
75 | |
76 | int otx2_update_rq_stats(struct otx2_nic *pfvf, int qidx) |
77 | { |
78 | struct otx2_rcv_queue *rq = &pfvf->qset.rq[qidx]; |
79 | |
80 | if (!pfvf->qset.rq) |
81 | return 0; |
82 | |
83 | otx2_nix_rq_op_stats(stats: &rq->stats, pfvf, qidx); |
84 | return 1; |
85 | } |
86 | |
87 | int otx2_update_sq_stats(struct otx2_nic *pfvf, int qidx) |
88 | { |
89 | struct otx2_snd_queue *sq = &pfvf->qset.sq[qidx]; |
90 | |
91 | if (!pfvf->qset.sq) |
92 | return 0; |
93 | |
94 | if (qidx >= pfvf->hw.non_qos_queues) { |
95 | if (!test_bit(qidx - pfvf->hw.non_qos_queues, pfvf->qos.qos_sq_bmap)) |
96 | return 0; |
97 | } |
98 | |
99 | otx2_nix_sq_op_stats(stats: &sq->stats, pfvf, qidx); |
100 | return 1; |
101 | } |
102 | |
103 | void otx2_get_dev_stats(struct otx2_nic *pfvf) |
104 | { |
105 | struct otx2_dev_stats *dev_stats = &pfvf->hw.dev_stats; |
106 | |
107 | dev_stats->rx_bytes = OTX2_GET_RX_STATS(RX_OCTS); |
108 | dev_stats->rx_drops = OTX2_GET_RX_STATS(RX_DROP); |
109 | dev_stats->rx_bcast_frames = OTX2_GET_RX_STATS(RX_BCAST); |
110 | dev_stats->rx_mcast_frames = OTX2_GET_RX_STATS(RX_MCAST); |
111 | dev_stats->rx_ucast_frames = OTX2_GET_RX_STATS(RX_UCAST); |
112 | dev_stats->rx_frames = dev_stats->rx_bcast_frames + |
113 | dev_stats->rx_mcast_frames + |
114 | dev_stats->rx_ucast_frames; |
115 | |
116 | dev_stats->tx_bytes = OTX2_GET_TX_STATS(TX_OCTS); |
117 | dev_stats->tx_drops = OTX2_GET_TX_STATS(TX_DROP); |
118 | dev_stats->tx_bcast_frames = OTX2_GET_TX_STATS(TX_BCAST); |
119 | dev_stats->tx_mcast_frames = OTX2_GET_TX_STATS(TX_MCAST); |
120 | dev_stats->tx_ucast_frames = OTX2_GET_TX_STATS(TX_UCAST); |
121 | dev_stats->tx_frames = dev_stats->tx_bcast_frames + |
122 | dev_stats->tx_mcast_frames + |
123 | dev_stats->tx_ucast_frames; |
124 | } |
125 | |
126 | void otx2_get_stats64(struct net_device *netdev, |
127 | struct rtnl_link_stats64 *stats) |
128 | { |
129 | struct otx2_nic *pfvf = netdev_priv(dev: netdev); |
130 | struct otx2_dev_stats *dev_stats; |
131 | |
132 | otx2_get_dev_stats(pfvf); |
133 | |
134 | dev_stats = &pfvf->hw.dev_stats; |
135 | stats->rx_bytes = dev_stats->rx_bytes; |
136 | stats->rx_packets = dev_stats->rx_frames; |
137 | stats->rx_dropped = dev_stats->rx_drops; |
138 | stats->multicast = dev_stats->rx_mcast_frames; |
139 | |
140 | stats->tx_bytes = dev_stats->tx_bytes; |
141 | stats->tx_packets = dev_stats->tx_frames; |
142 | stats->tx_dropped = dev_stats->tx_drops; |
143 | } |
144 | EXPORT_SYMBOL(otx2_get_stats64); |
145 | |
146 | /* Sync MAC address with RVU AF */ |
147 | static int otx2_hw_set_mac_addr(struct otx2_nic *pfvf, u8 *mac) |
148 | { |
149 | struct nix_set_mac_addr *req; |
150 | int err; |
151 | |
152 | mutex_lock(&pfvf->mbox.lock); |
153 | req = otx2_mbox_alloc_msg_nix_set_mac_addr(mbox: &pfvf->mbox); |
154 | if (!req) { |
155 | mutex_unlock(lock: &pfvf->mbox.lock); |
156 | return -ENOMEM; |
157 | } |
158 | |
159 | ether_addr_copy(dst: req->mac_addr, src: mac); |
160 | |
161 | err = otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
162 | mutex_unlock(lock: &pfvf->mbox.lock); |
163 | return err; |
164 | } |
165 | |
166 | static int otx2_hw_get_mac_addr(struct otx2_nic *pfvf, |
167 | struct net_device *netdev) |
168 | { |
169 | struct nix_get_mac_addr_rsp *rsp; |
170 | struct mbox_msghdr *msghdr; |
171 | struct msg_req *req; |
172 | int err; |
173 | |
174 | mutex_lock(&pfvf->mbox.lock); |
175 | req = otx2_mbox_alloc_msg_nix_get_mac_addr(mbox: &pfvf->mbox); |
176 | if (!req) { |
177 | mutex_unlock(lock: &pfvf->mbox.lock); |
178 | return -ENOMEM; |
179 | } |
180 | |
181 | err = otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
182 | if (err) { |
183 | mutex_unlock(lock: &pfvf->mbox.lock); |
184 | return err; |
185 | } |
186 | |
187 | msghdr = otx2_mbox_get_rsp(mbox: &pfvf->mbox.mbox, devid: 0, msg: &req->hdr); |
188 | if (IS_ERR(ptr: msghdr)) { |
189 | mutex_unlock(lock: &pfvf->mbox.lock); |
190 | return PTR_ERR(ptr: msghdr); |
191 | } |
192 | rsp = (struct nix_get_mac_addr_rsp *)msghdr; |
193 | eth_hw_addr_set(dev: netdev, addr: rsp->mac_addr); |
194 | mutex_unlock(lock: &pfvf->mbox.lock); |
195 | |
196 | return 0; |
197 | } |
198 | |
199 | int otx2_set_mac_address(struct net_device *netdev, void *p) |
200 | { |
201 | struct otx2_nic *pfvf = netdev_priv(dev: netdev); |
202 | struct sockaddr *addr = p; |
203 | |
204 | if (!is_valid_ether_addr(addr: addr->sa_data)) |
205 | return -EADDRNOTAVAIL; |
206 | |
207 | if (!otx2_hw_set_mac_addr(pfvf, mac: addr->sa_data)) { |
208 | eth_hw_addr_set(dev: netdev, addr: addr->sa_data); |
209 | /* update dmac field in vlan offload rule */ |
210 | if (netif_running(dev: netdev) && |
211 | pfvf->flags & OTX2_FLAG_RX_VLAN_SUPPORT) |
212 | otx2_install_rxvlan_offload_flow(pfvf); |
213 | /* update dmac address in ntuple and DMAC filter list */ |
214 | if (pfvf->flags & OTX2_FLAG_DMACFLTR_SUPPORT) |
215 | otx2_dmacflt_update_pfmac_flow(pfvf); |
216 | } else { |
217 | return -EPERM; |
218 | } |
219 | |
220 | return 0; |
221 | } |
222 | EXPORT_SYMBOL(otx2_set_mac_address); |
223 | |
224 | int otx2_hw_set_mtu(struct otx2_nic *pfvf, int mtu) |
225 | { |
226 | struct nix_frs_cfg *req; |
227 | u16 maxlen; |
228 | int err; |
229 | |
230 | maxlen = otx2_get_max_mtu(pfvf) + OTX2_ETH_HLEN + OTX2_HW_TIMESTAMP_LEN; |
231 | |
232 | mutex_lock(&pfvf->mbox.lock); |
233 | req = otx2_mbox_alloc_msg_nix_set_hw_frs(mbox: &pfvf->mbox); |
234 | if (!req) { |
235 | mutex_unlock(lock: &pfvf->mbox.lock); |
236 | return -ENOMEM; |
237 | } |
238 | |
239 | req->maxlen = pfvf->netdev->mtu + OTX2_ETH_HLEN + OTX2_HW_TIMESTAMP_LEN; |
240 | |
241 | /* Use max receive length supported by hardware for loopback devices */ |
242 | if (is_otx2_lbkvf(pdev: pfvf->pdev)) |
243 | req->maxlen = maxlen; |
244 | |
245 | err = otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
246 | mutex_unlock(lock: &pfvf->mbox.lock); |
247 | return err; |
248 | } |
249 | |
250 | int otx2_config_pause_frm(struct otx2_nic *pfvf) |
251 | { |
252 | struct cgx_pause_frm_cfg *req; |
253 | int err; |
254 | |
255 | if (is_otx2_lbkvf(pdev: pfvf->pdev)) |
256 | return 0; |
257 | |
258 | mutex_lock(&pfvf->mbox.lock); |
259 | req = otx2_mbox_alloc_msg_cgx_cfg_pause_frm(mbox: &pfvf->mbox); |
260 | if (!req) { |
261 | err = -ENOMEM; |
262 | goto unlock; |
263 | } |
264 | |
265 | req->rx_pause = !!(pfvf->flags & OTX2_FLAG_RX_PAUSE_ENABLED); |
266 | req->tx_pause = !!(pfvf->flags & OTX2_FLAG_TX_PAUSE_ENABLED); |
267 | req->set = 1; |
268 | |
269 | err = otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
270 | unlock: |
271 | mutex_unlock(lock: &pfvf->mbox.lock); |
272 | return err; |
273 | } |
274 | EXPORT_SYMBOL(otx2_config_pause_frm); |
275 | |
276 | int otx2_set_flowkey_cfg(struct otx2_nic *pfvf) |
277 | { |
278 | struct otx2_rss_info * = &pfvf->hw.rss_info; |
279 | struct nix_rss_flowkey_cfg_rsp *rsp; |
280 | struct nix_rss_flowkey_cfg *req; |
281 | int err; |
282 | |
283 | mutex_lock(&pfvf->mbox.lock); |
284 | req = otx2_mbox_alloc_msg_nix_rss_flowkey_cfg(mbox: &pfvf->mbox); |
285 | if (!req) { |
286 | mutex_unlock(lock: &pfvf->mbox.lock); |
287 | return -ENOMEM; |
288 | } |
289 | req->mcam_index = -1; /* Default or reserved index */ |
290 | req->flowkey_cfg = rss->flowkey_cfg; |
291 | req->group = DEFAULT_RSS_CONTEXT_GROUP; |
292 | |
293 | err = otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
294 | if (err) |
295 | goto fail; |
296 | |
297 | rsp = (struct nix_rss_flowkey_cfg_rsp *) |
298 | otx2_mbox_get_rsp(mbox: &pfvf->mbox.mbox, devid: 0, msg: &req->hdr); |
299 | if (IS_ERR(ptr: rsp)) { |
300 | err = PTR_ERR(ptr: rsp); |
301 | goto fail; |
302 | } |
303 | |
304 | pfvf->hw.flowkey_alg_idx = rsp->alg_idx; |
305 | fail: |
306 | mutex_unlock(lock: &pfvf->mbox.lock); |
307 | return err; |
308 | } |
309 | |
310 | int (struct otx2_nic *pfvf, int ctx_id) |
311 | { |
312 | struct otx2_rss_info * = &pfvf->hw.rss_info; |
313 | const int index = rss->rss_size * ctx_id; |
314 | struct mbox *mbox = &pfvf->mbox; |
315 | struct otx2_rss_ctx *; |
316 | struct nix_aq_enq_req *aq; |
317 | int idx, err; |
318 | |
319 | mutex_lock(&mbox->lock); |
320 | rss_ctx = rss->rss_ctx[ctx_id]; |
321 | /* Get memory to put this msg */ |
322 | for (idx = 0; idx < rss->rss_size; idx++) { |
323 | aq = otx2_mbox_alloc_msg_nix_aq_enq(mbox); |
324 | if (!aq) { |
325 | /* The shared memory buffer can be full. |
326 | * Flush it and retry |
327 | */ |
328 | err = otx2_sync_mbox_msg(mbox); |
329 | if (err) { |
330 | mutex_unlock(lock: &mbox->lock); |
331 | return err; |
332 | } |
333 | aq = otx2_mbox_alloc_msg_nix_aq_enq(mbox); |
334 | if (!aq) { |
335 | mutex_unlock(lock: &mbox->lock); |
336 | return -ENOMEM; |
337 | } |
338 | } |
339 | |
340 | aq->rss.rq = rss_ctx->ind_tbl[idx]; |
341 | |
342 | /* Fill AQ info */ |
343 | aq->qidx = index + idx; |
344 | aq->ctype = NIX_AQ_CTYPE_RSS; |
345 | aq->op = NIX_AQ_INSTOP_INIT; |
346 | } |
347 | err = otx2_sync_mbox_msg(mbox); |
348 | mutex_unlock(lock: &mbox->lock); |
349 | return err; |
350 | } |
351 | |
352 | void (struct otx2_nic *pfvf) |
353 | { |
354 | struct otx2_rss_info * = &pfvf->hw.rss_info; |
355 | u64 *key = (u64 *)&rss->key[4]; |
356 | int idx; |
357 | |
358 | /* 352bit or 44byte key needs to be configured as below |
359 | * NIX_LF_RX_SECRETX0 = key<351:288> |
360 | * NIX_LF_RX_SECRETX1 = key<287:224> |
361 | * NIX_LF_RX_SECRETX2 = key<223:160> |
362 | * NIX_LF_RX_SECRETX3 = key<159:96> |
363 | * NIX_LF_RX_SECRETX4 = key<95:32> |
364 | * NIX_LF_RX_SECRETX5<63:32> = key<31:0> |
365 | */ |
366 | otx2_write64(nic: pfvf, NIX_LF_RX_SECRETX(5), |
367 | val: (u64)(*((u32 *)&rss->key)) << 32); |
368 | idx = sizeof(rss->key) / sizeof(u64); |
369 | while (idx > 0) { |
370 | idx--; |
371 | otx2_write64(nic: pfvf, NIX_LF_RX_SECRETX(idx), val: *key++); |
372 | } |
373 | } |
374 | |
375 | int (struct otx2_nic *pfvf) |
376 | { |
377 | struct otx2_rss_info * = &pfvf->hw.rss_info; |
378 | struct otx2_rss_ctx *; |
379 | int idx, ret = 0; |
380 | |
381 | rss->rss_size = sizeof(*rss->rss_ctx[DEFAULT_RSS_CONTEXT_GROUP]); |
382 | |
383 | /* Init RSS key if it is not setup already */ |
384 | if (!rss->enable) |
385 | netdev_rss_key_fill(buffer: rss->key, len: sizeof(rss->key)); |
386 | otx2_set_rss_key(pfvf); |
387 | |
388 | if (!netif_is_rxfh_configured(dev: pfvf->netdev)) { |
389 | /* Set RSS group 0 as default indirection table */ |
390 | rss->rss_ctx[DEFAULT_RSS_CONTEXT_GROUP] = kzalloc(size: rss->rss_size, |
391 | GFP_KERNEL); |
392 | if (!rss->rss_ctx[DEFAULT_RSS_CONTEXT_GROUP]) |
393 | return -ENOMEM; |
394 | |
395 | rss_ctx = rss->rss_ctx[DEFAULT_RSS_CONTEXT_GROUP]; |
396 | for (idx = 0; idx < rss->rss_size; idx++) |
397 | rss_ctx->ind_tbl[idx] = |
398 | ethtool_rxfh_indir_default(index: idx, |
399 | n_rx_rings: pfvf->hw.rx_queues); |
400 | } |
401 | ret = otx2_set_rss_table(pfvf, DEFAULT_RSS_CONTEXT_GROUP); |
402 | if (ret) |
403 | return ret; |
404 | |
405 | /* Flowkey or hash config to be used for generating flow tag */ |
406 | rss->flowkey_cfg = rss->enable ? rss->flowkey_cfg : |
407 | NIX_FLOW_KEY_TYPE_IPV4 | NIX_FLOW_KEY_TYPE_IPV6 | |
408 | NIX_FLOW_KEY_TYPE_TCP | NIX_FLOW_KEY_TYPE_UDP | |
409 | NIX_FLOW_KEY_TYPE_SCTP | NIX_FLOW_KEY_TYPE_VLAN | |
410 | NIX_FLOW_KEY_TYPE_IPV4_PROTO; |
411 | |
412 | ret = otx2_set_flowkey_cfg(pfvf); |
413 | if (ret) |
414 | return ret; |
415 | |
416 | rss->enable = true; |
417 | return 0; |
418 | } |
419 | |
420 | /* Setup UDP segmentation algorithm in HW */ |
421 | static void otx2_setup_udp_segmentation(struct nix_lso_format_cfg *lso, bool v4) |
422 | { |
423 | struct nix_lso_format *field; |
424 | |
425 | field = (struct nix_lso_format *)&lso->fields[0]; |
426 | lso->field_mask = GENMASK(18, 0); |
427 | |
428 | /* IP's Length field */ |
429 | field->layer = NIX_TXLAYER_OL3; |
430 | /* In ipv4, length field is at offset 2 bytes, for ipv6 it's 4 */ |
431 | field->offset = v4 ? 2 : 4; |
432 | field->sizem1 = 1; /* i.e 2 bytes */ |
433 | field->alg = NIX_LSOALG_ADD_PAYLEN; |
434 | field++; |
435 | |
436 | /* No ID field in IPv6 header */ |
437 | if (v4) { |
438 | /* Increment IPID */ |
439 | field->layer = NIX_TXLAYER_OL3; |
440 | field->offset = 4; |
441 | field->sizem1 = 1; /* i.e 2 bytes */ |
442 | field->alg = NIX_LSOALG_ADD_SEGNUM; |
443 | field++; |
444 | } |
445 | |
446 | /* Update length in UDP header */ |
447 | field->layer = NIX_TXLAYER_OL4; |
448 | field->offset = 4; |
449 | field->sizem1 = 1; |
450 | field->alg = NIX_LSOALG_ADD_PAYLEN; |
451 | } |
452 | |
453 | /* Setup segmentation algorithms in HW and retrieve algorithm index */ |
454 | void otx2_setup_segmentation(struct otx2_nic *pfvf) |
455 | { |
456 | struct nix_lso_format_cfg_rsp *rsp; |
457 | struct nix_lso_format_cfg *lso; |
458 | struct otx2_hw *hw = &pfvf->hw; |
459 | int err; |
460 | |
461 | mutex_lock(&pfvf->mbox.lock); |
462 | |
463 | /* UDPv4 segmentation */ |
464 | lso = otx2_mbox_alloc_msg_nix_lso_format_cfg(mbox: &pfvf->mbox); |
465 | if (!lso) |
466 | goto fail; |
467 | |
468 | /* Setup UDP/IP header fields that HW should update per segment */ |
469 | otx2_setup_udp_segmentation(lso, v4: true); |
470 | |
471 | err = otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
472 | if (err) |
473 | goto fail; |
474 | |
475 | rsp = (struct nix_lso_format_cfg_rsp *) |
476 | otx2_mbox_get_rsp(mbox: &pfvf->mbox.mbox, devid: 0, msg: &lso->hdr); |
477 | if (IS_ERR(ptr: rsp)) |
478 | goto fail; |
479 | |
480 | hw->lso_udpv4_idx = rsp->lso_format_idx; |
481 | |
482 | /* UDPv6 segmentation */ |
483 | lso = otx2_mbox_alloc_msg_nix_lso_format_cfg(mbox: &pfvf->mbox); |
484 | if (!lso) |
485 | goto fail; |
486 | |
487 | /* Setup UDP/IP header fields that HW should update per segment */ |
488 | otx2_setup_udp_segmentation(lso, v4: false); |
489 | |
490 | err = otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
491 | if (err) |
492 | goto fail; |
493 | |
494 | rsp = (struct nix_lso_format_cfg_rsp *) |
495 | otx2_mbox_get_rsp(mbox: &pfvf->mbox.mbox, devid: 0, msg: &lso->hdr); |
496 | if (IS_ERR(ptr: rsp)) |
497 | goto fail; |
498 | |
499 | hw->lso_udpv6_idx = rsp->lso_format_idx; |
500 | mutex_unlock(lock: &pfvf->mbox.lock); |
501 | return; |
502 | fail: |
503 | mutex_unlock(lock: &pfvf->mbox.lock); |
504 | netdev_info(dev: pfvf->netdev, |
505 | format: "Failed to get LSO index for UDP GSO offload, disabling\n" ); |
506 | pfvf->netdev->hw_features &= ~NETIF_F_GSO_UDP_L4; |
507 | } |
508 | |
509 | void otx2_config_irq_coalescing(struct otx2_nic *pfvf, int qidx) |
510 | { |
511 | /* Configure CQE interrupt coalescing parameters |
512 | * |
513 | * HW triggers an irq when ECOUNT > cq_ecount_wait, hence |
514 | * set 1 less than cq_ecount_wait. And cq_time_wait is in |
515 | * usecs, convert that to 100ns count. |
516 | */ |
517 | otx2_write64(nic: pfvf, NIX_LF_CINTX_WAIT(qidx), |
518 | val: ((u64)(pfvf->hw.cq_time_wait * 10) << 48) | |
519 | ((u64)pfvf->hw.cq_qcount_wait << 32) | |
520 | (pfvf->hw.cq_ecount_wait - 1)); |
521 | } |
522 | |
523 | static int otx2_alloc_pool_buf(struct otx2_nic *pfvf, struct otx2_pool *pool, |
524 | dma_addr_t *dma) |
525 | { |
526 | unsigned int offset = 0; |
527 | struct page *page; |
528 | size_t sz; |
529 | |
530 | sz = SKB_DATA_ALIGN(pool->rbsize); |
531 | sz = ALIGN(sz, OTX2_ALIGN); |
532 | |
533 | page = page_pool_alloc_frag(pool: pool->page_pool, offset: &offset, size: sz, GFP_ATOMIC); |
534 | if (unlikely(!page)) |
535 | return -ENOMEM; |
536 | |
537 | *dma = page_pool_get_dma_addr(page) + offset; |
538 | return 0; |
539 | } |
540 | |
541 | static int __otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool, |
542 | dma_addr_t *dma) |
543 | { |
544 | u8 *buf; |
545 | |
546 | if (pool->page_pool) |
547 | return otx2_alloc_pool_buf(pfvf, pool, dma); |
548 | |
549 | buf = napi_alloc_frag_align(fragsz: pool->rbsize, OTX2_ALIGN); |
550 | if (unlikely(!buf)) |
551 | return -ENOMEM; |
552 | |
553 | *dma = dma_map_single_attrs(dev: pfvf->dev, ptr: buf, size: pool->rbsize, |
554 | dir: DMA_FROM_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); |
555 | if (unlikely(dma_mapping_error(pfvf->dev, *dma))) { |
556 | page_frag_free(addr: buf); |
557 | return -ENOMEM; |
558 | } |
559 | |
560 | return 0; |
561 | } |
562 | |
563 | int otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool, |
564 | dma_addr_t *dma) |
565 | { |
566 | int ret; |
567 | |
568 | local_bh_disable(); |
569 | ret = __otx2_alloc_rbuf(pfvf, pool, dma); |
570 | local_bh_enable(); |
571 | return ret; |
572 | } |
573 | |
574 | int otx2_alloc_buffer(struct otx2_nic *pfvf, struct otx2_cq_queue *cq, |
575 | dma_addr_t *dma) |
576 | { |
577 | if (unlikely(__otx2_alloc_rbuf(pfvf, cq->rbpool, dma))) |
578 | return -ENOMEM; |
579 | return 0; |
580 | } |
581 | |
582 | void otx2_tx_timeout(struct net_device *netdev, unsigned int txq) |
583 | { |
584 | struct otx2_nic *pfvf = netdev_priv(dev: netdev); |
585 | |
586 | schedule_work(work: &pfvf->reset_task); |
587 | } |
588 | EXPORT_SYMBOL(otx2_tx_timeout); |
589 | |
590 | void otx2_get_mac_from_af(struct net_device *netdev) |
591 | { |
592 | struct otx2_nic *pfvf = netdev_priv(dev: netdev); |
593 | int err; |
594 | |
595 | err = otx2_hw_get_mac_addr(pfvf, netdev); |
596 | if (err) |
597 | dev_warn(pfvf->dev, "Failed to read mac from hardware\n" ); |
598 | |
599 | /* If AF doesn't provide a valid MAC, generate a random one */ |
600 | if (!is_valid_ether_addr(addr: netdev->dev_addr)) |
601 | eth_hw_addr_random(dev: netdev); |
602 | } |
603 | EXPORT_SYMBOL(otx2_get_mac_from_af); |
604 | |
605 | int otx2_txschq_config(struct otx2_nic *pfvf, int lvl, int prio, bool txschq_for_pfc) |
606 | { |
607 | u16 (*schq_list)[MAX_TXSCHQ_PER_FUNC]; |
608 | struct otx2_hw *hw = &pfvf->hw; |
609 | struct nix_txschq_config *req; |
610 | u64 schq, parent; |
611 | u64 dwrr_val; |
612 | |
613 | dwrr_val = mtu_to_dwrr_weight(pfvf, mtu: pfvf->tx_max_pktlen); |
614 | |
615 | req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox: &pfvf->mbox); |
616 | if (!req) |
617 | return -ENOMEM; |
618 | |
619 | req->lvl = lvl; |
620 | req->num_regs = 1; |
621 | |
622 | schq_list = hw->txschq_list; |
623 | #ifdef CONFIG_DCB |
624 | if (txschq_for_pfc) |
625 | schq_list = pfvf->pfc_schq_list; |
626 | #endif |
627 | |
628 | schq = schq_list[lvl][prio]; |
629 | /* Set topology e.t.c configuration */ |
630 | if (lvl == NIX_TXSCH_LVL_SMQ) { |
631 | req->reg[0] = NIX_AF_SMQX_CFG(schq); |
632 | req->regval[0] = ((u64)pfvf->tx_max_pktlen << 8) | OTX2_MIN_MTU; |
633 | req->regval[0] |= (0x20ULL << 51) | (0x80ULL << 39) | |
634 | (0x2ULL << 36); |
635 | /* Set link type for DWRR MTU selection on CN10K silicons */ |
636 | if (!is_dev_otx2(pdev: pfvf->pdev)) |
637 | req->regval[0] |= FIELD_PREP(GENMASK_ULL(58, 57), |
638 | (u64)hw->smq_link_type); |
639 | req->num_regs++; |
640 | /* MDQ config */ |
641 | parent = schq_list[NIX_TXSCH_LVL_TL4][prio]; |
642 | req->reg[1] = NIX_AF_MDQX_PARENT(schq); |
643 | req->regval[1] = parent << 16; |
644 | req->num_regs++; |
645 | /* Set DWRR quantum */ |
646 | req->reg[2] = NIX_AF_MDQX_SCHEDULE(schq); |
647 | req->regval[2] = dwrr_val; |
648 | } else if (lvl == NIX_TXSCH_LVL_TL4) { |
649 | parent = schq_list[NIX_TXSCH_LVL_TL3][prio]; |
650 | req->reg[0] = NIX_AF_TL4X_PARENT(schq); |
651 | req->regval[0] = parent << 16; |
652 | req->num_regs++; |
653 | req->reg[1] = NIX_AF_TL4X_SCHEDULE(schq); |
654 | req->regval[1] = dwrr_val; |
655 | } else if (lvl == NIX_TXSCH_LVL_TL3) { |
656 | parent = schq_list[NIX_TXSCH_LVL_TL2][prio]; |
657 | req->reg[0] = NIX_AF_TL3X_PARENT(schq); |
658 | req->regval[0] = parent << 16; |
659 | req->num_regs++; |
660 | req->reg[1] = NIX_AF_TL3X_SCHEDULE(schq); |
661 | req->regval[1] = dwrr_val; |
662 | if (lvl == hw->txschq_link_cfg_lvl) { |
663 | req->num_regs++; |
664 | req->reg[2] = NIX_AF_TL3_TL2X_LINKX_CFG(schq, hw->tx_link); |
665 | /* Enable this queue and backpressure |
666 | * and set relative channel |
667 | */ |
668 | req->regval[2] = BIT_ULL(13) | BIT_ULL(12) | prio; |
669 | } |
670 | } else if (lvl == NIX_TXSCH_LVL_TL2) { |
671 | parent = schq_list[NIX_TXSCH_LVL_TL1][prio]; |
672 | req->reg[0] = NIX_AF_TL2X_PARENT(schq); |
673 | req->regval[0] = parent << 16; |
674 | |
675 | req->num_regs++; |
676 | req->reg[1] = NIX_AF_TL2X_SCHEDULE(schq); |
677 | req->regval[1] = TXSCH_TL1_DFLT_RR_PRIO << 24 | dwrr_val; |
678 | |
679 | if (lvl == hw->txschq_link_cfg_lvl) { |
680 | req->num_regs++; |
681 | req->reg[2] = NIX_AF_TL3_TL2X_LINKX_CFG(schq, hw->tx_link); |
682 | /* Enable this queue and backpressure |
683 | * and set relative channel |
684 | */ |
685 | req->regval[2] = BIT_ULL(13) | BIT_ULL(12) | prio; |
686 | } |
687 | } else if (lvl == NIX_TXSCH_LVL_TL1) { |
688 | /* Default config for TL1. |
689 | * For VF this is always ignored. |
690 | */ |
691 | |
692 | /* On CN10K, if RR_WEIGHT is greater than 16384, HW will |
693 | * clip it to 16384, so configuring a 24bit max value |
694 | * will work on both OTx2 and CN10K. |
695 | */ |
696 | req->reg[0] = NIX_AF_TL1X_SCHEDULE(schq); |
697 | req->regval[0] = TXSCH_TL1_DFLT_RR_QTM; |
698 | |
699 | req->num_regs++; |
700 | req->reg[1] = NIX_AF_TL1X_TOPOLOGY(schq); |
701 | req->regval[1] = (TXSCH_TL1_DFLT_RR_PRIO << 1); |
702 | |
703 | req->num_regs++; |
704 | req->reg[2] = NIX_AF_TL1X_CIR(schq); |
705 | req->regval[2] = 0; |
706 | } |
707 | |
708 | return otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
709 | } |
710 | EXPORT_SYMBOL(otx2_txschq_config); |
711 | |
712 | int otx2_smq_flush(struct otx2_nic *pfvf, int smq) |
713 | { |
714 | struct nix_txschq_config *req; |
715 | int rc; |
716 | |
717 | mutex_lock(&pfvf->mbox.lock); |
718 | |
719 | req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox: &pfvf->mbox); |
720 | if (!req) { |
721 | mutex_unlock(lock: &pfvf->mbox.lock); |
722 | return -ENOMEM; |
723 | } |
724 | |
725 | req->lvl = NIX_TXSCH_LVL_SMQ; |
726 | req->reg[0] = NIX_AF_SMQX_CFG(smq); |
727 | req->regval[0] |= BIT_ULL(49); |
728 | req->num_regs++; |
729 | |
730 | rc = otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
731 | mutex_unlock(lock: &pfvf->mbox.lock); |
732 | return rc; |
733 | } |
734 | EXPORT_SYMBOL(otx2_smq_flush); |
735 | |
736 | int otx2_txsch_alloc(struct otx2_nic *pfvf) |
737 | { |
738 | struct nix_txsch_alloc_req *req; |
739 | struct nix_txsch_alloc_rsp *rsp; |
740 | int lvl, schq, rc; |
741 | |
742 | /* Get memory to put this msg */ |
743 | req = otx2_mbox_alloc_msg_nix_txsch_alloc(mbox: &pfvf->mbox); |
744 | if (!req) |
745 | return -ENOMEM; |
746 | |
747 | /* Request one schq per level */ |
748 | for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) |
749 | req->schq[lvl] = 1; |
750 | rc = otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
751 | if (rc) |
752 | return rc; |
753 | |
754 | rsp = (struct nix_txsch_alloc_rsp *) |
755 | otx2_mbox_get_rsp(mbox: &pfvf->mbox.mbox, devid: 0, msg: &req->hdr); |
756 | if (IS_ERR(ptr: rsp)) |
757 | return PTR_ERR(ptr: rsp); |
758 | |
759 | /* Setup transmit scheduler list */ |
760 | for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) |
761 | for (schq = 0; schq < rsp->schq[lvl]; schq++) |
762 | pfvf->hw.txschq_list[lvl][schq] = |
763 | rsp->schq_list[lvl][schq]; |
764 | |
765 | pfvf->hw.txschq_link_cfg_lvl = rsp->link_cfg_lvl; |
766 | pfvf->hw.txschq_aggr_lvl_rr_prio = rsp->aggr_lvl_rr_prio; |
767 | |
768 | return 0; |
769 | } |
770 | |
771 | void otx2_txschq_free_one(struct otx2_nic *pfvf, u16 lvl, u16 schq) |
772 | { |
773 | struct nix_txsch_free_req *free_req; |
774 | int err; |
775 | |
776 | mutex_lock(&pfvf->mbox.lock); |
777 | |
778 | free_req = otx2_mbox_alloc_msg_nix_txsch_free(mbox: &pfvf->mbox); |
779 | if (!free_req) { |
780 | mutex_unlock(lock: &pfvf->mbox.lock); |
781 | netdev_err(dev: pfvf->netdev, |
782 | format: "Failed alloc txschq free req\n" ); |
783 | return; |
784 | } |
785 | |
786 | free_req->schq_lvl = lvl; |
787 | free_req->schq = schq; |
788 | |
789 | err = otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
790 | if (err) { |
791 | netdev_err(dev: pfvf->netdev, |
792 | format: "Failed stop txschq %d at level %d\n" , schq, lvl); |
793 | } |
794 | |
795 | mutex_unlock(lock: &pfvf->mbox.lock); |
796 | } |
797 | EXPORT_SYMBOL(otx2_txschq_free_one); |
798 | |
799 | void otx2_txschq_stop(struct otx2_nic *pfvf) |
800 | { |
801 | int lvl, schq; |
802 | |
803 | /* free non QOS TLx nodes */ |
804 | for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) |
805 | otx2_txschq_free_one(pfvf, lvl, |
806 | pfvf->hw.txschq_list[lvl][0]); |
807 | |
808 | /* Clear the txschq list */ |
809 | for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { |
810 | for (schq = 0; schq < MAX_TXSCHQ_PER_FUNC; schq++) |
811 | pfvf->hw.txschq_list[lvl][schq] = 0; |
812 | } |
813 | |
814 | } |
815 | |
816 | void otx2_sqb_flush(struct otx2_nic *pfvf) |
817 | { |
818 | int qidx, sqe_tail, sqe_head; |
819 | struct otx2_snd_queue *sq; |
820 | u64 incr, *ptr, val; |
821 | |
822 | ptr = (u64 *)otx2_get_regaddr(nic: pfvf, NIX_LF_SQ_OP_STATUS); |
823 | for (qidx = 0; qidx < otx2_get_total_tx_queues(pfvf); qidx++) { |
824 | sq = &pfvf->qset.sq[qidx]; |
825 | if (!sq->sqb_ptrs) |
826 | continue; |
827 | |
828 | incr = (u64)qidx << 32; |
829 | val = otx2_atomic64_add(incr, ptr); |
830 | sqe_head = (val >> 20) & 0x3F; |
831 | sqe_tail = (val >> 28) & 0x3F; |
832 | if (sqe_head != sqe_tail) |
833 | usleep_range(min: 50, max: 60); |
834 | } |
835 | } |
836 | |
837 | /* RED and drop levels of CQ on packet reception. |
838 | * For CQ level is measure of emptiness ( 0x0 = full, 255 = empty). |
839 | */ |
840 | #define RQ_PASS_LVL_CQ(skid, qsize) ((((skid) + 16) * 256) / (qsize)) |
841 | #define RQ_DROP_LVL_CQ(skid, qsize) (((skid) * 256) / (qsize)) |
842 | |
843 | /* RED and drop levels of AURA for packet reception. |
844 | * For AURA level is measure of fullness (0x0 = empty, 255 = full). |
845 | * Eg: For RQ length 1K, for pass/drop level 204/230. |
846 | * RED accepts pkts if free pointers > 102 & <= 205. |
847 | * Drops pkts if free pointers < 102. |
848 | */ |
849 | #define RQ_BP_LVL_AURA (255 - ((85 * 256) / 100)) /* BP when 85% is full */ |
850 | #define RQ_PASS_LVL_AURA (255 - ((95 * 256) / 100)) /* RED when 95% is full */ |
851 | #define RQ_DROP_LVL_AURA (255 - ((99 * 256) / 100)) /* Drop when 99% is full */ |
852 | |
853 | static int otx2_rq_init(struct otx2_nic *pfvf, u16 qidx, u16 lpb_aura) |
854 | { |
855 | struct otx2_qset *qset = &pfvf->qset; |
856 | struct nix_aq_enq_req *aq; |
857 | |
858 | /* Get memory to put this msg */ |
859 | aq = otx2_mbox_alloc_msg_nix_aq_enq(mbox: &pfvf->mbox); |
860 | if (!aq) |
861 | return -ENOMEM; |
862 | |
863 | aq->rq.cq = qidx; |
864 | aq->rq.ena = 1; |
865 | aq->rq.pb_caching = 1; |
866 | aq->rq.lpb_aura = lpb_aura; /* Use large packet buffer aura */ |
867 | aq->rq.lpb_sizem1 = (DMA_BUFFER_LEN(pfvf->rbsize) / 8) - 1; |
868 | aq->rq.xqe_imm_size = 0; /* Copying of packet to CQE not needed */ |
869 | aq->rq.flow_tagw = 32; /* Copy full 32bit flow_tag to CQE header */ |
870 | aq->rq.qint_idx = 0; |
871 | aq->rq.lpb_drop_ena = 1; /* Enable RED dropping for AURA */ |
872 | aq->rq.xqe_drop_ena = 1; /* Enable RED dropping for CQ/SSO */ |
873 | aq->rq.xqe_pass = RQ_PASS_LVL_CQ(pfvf->hw.rq_skid, qset->rqe_cnt); |
874 | aq->rq.xqe_drop = RQ_DROP_LVL_CQ(pfvf->hw.rq_skid, qset->rqe_cnt); |
875 | aq->rq.lpb_aura_pass = RQ_PASS_LVL_AURA; |
876 | aq->rq.lpb_aura_drop = RQ_DROP_LVL_AURA; |
877 | |
878 | /* Fill AQ info */ |
879 | aq->qidx = qidx; |
880 | aq->ctype = NIX_AQ_CTYPE_RQ; |
881 | aq->op = NIX_AQ_INSTOP_INIT; |
882 | |
883 | return otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
884 | } |
885 | |
886 | int otx2_sq_aq_init(void *dev, u16 qidx, u16 sqb_aura) |
887 | { |
888 | struct otx2_nic *pfvf = dev; |
889 | struct otx2_snd_queue *sq; |
890 | struct nix_aq_enq_req *aq; |
891 | |
892 | sq = &pfvf->qset.sq[qidx]; |
893 | sq->lmt_addr = (__force u64 *)(pfvf->reg_base + LMT_LF_LMTLINEX(qidx)); |
894 | /* Get memory to put this msg */ |
895 | aq = otx2_mbox_alloc_msg_nix_aq_enq(mbox: &pfvf->mbox); |
896 | if (!aq) |
897 | return -ENOMEM; |
898 | |
899 | aq->sq.cq = pfvf->hw.rx_queues + qidx; |
900 | aq->sq.max_sqe_size = NIX_MAXSQESZ_W16; /* 128 byte */ |
901 | aq->sq.cq_ena = 1; |
902 | aq->sq.ena = 1; |
903 | aq->sq.smq = otx2_get_smq_idx(pfvf, qidx); |
904 | aq->sq.smq_rr_quantum = mtu_to_dwrr_weight(pfvf, mtu: pfvf->tx_max_pktlen); |
905 | aq->sq.default_chan = pfvf->hw.tx_chan_base; |
906 | aq->sq.sqe_stype = NIX_STYPE_STF; /* Cache SQB */ |
907 | aq->sq.sqb_aura = sqb_aura; |
908 | aq->sq.sq_int_ena = NIX_SQINT_BITS; |
909 | aq->sq.qint_idx = 0; |
910 | /* Due pipelining impact minimum 2000 unused SQ CQE's |
911 | * need to maintain to avoid CQ overflow. |
912 | */ |
913 | aq->sq.cq_limit = ((SEND_CQ_SKID * 256) / (pfvf->qset.sqe_cnt)); |
914 | |
915 | /* Fill AQ info */ |
916 | aq->qidx = qidx; |
917 | aq->ctype = NIX_AQ_CTYPE_SQ; |
918 | aq->op = NIX_AQ_INSTOP_INIT; |
919 | |
920 | return otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
921 | } |
922 | |
923 | int otx2_sq_init(struct otx2_nic *pfvf, u16 qidx, u16 sqb_aura) |
924 | { |
925 | struct otx2_qset *qset = &pfvf->qset; |
926 | struct otx2_snd_queue *sq; |
927 | struct otx2_pool *pool; |
928 | int err; |
929 | |
930 | pool = &pfvf->qset.pool[sqb_aura]; |
931 | sq = &qset->sq[qidx]; |
932 | sq->sqe_size = NIX_SQESZ_W16 ? 64 : 128; |
933 | sq->sqe_cnt = qset->sqe_cnt; |
934 | |
935 | err = qmem_alloc(dev: pfvf->dev, q: &sq->sqe, qsize: 1, entry_sz: sq->sqe_size); |
936 | if (err) |
937 | return err; |
938 | |
939 | if (qidx < pfvf->hw.tx_queues) { |
940 | err = qmem_alloc(dev: pfvf->dev, q: &sq->tso_hdrs, qsize: qset->sqe_cnt, |
941 | TSO_HEADER_SIZE); |
942 | if (err) |
943 | return err; |
944 | } |
945 | |
946 | sq->sqe_base = sq->sqe->base; |
947 | sq->sg = kcalloc(n: qset->sqe_cnt, size: sizeof(struct sg_list), GFP_KERNEL); |
948 | if (!sq->sg) |
949 | return -ENOMEM; |
950 | |
951 | if (pfvf->ptp && qidx < pfvf->hw.tx_queues) { |
952 | err = qmem_alloc(dev: pfvf->dev, q: &sq->timestamps, qsize: qset->sqe_cnt, |
953 | entry_sz: sizeof(*sq->timestamps)); |
954 | if (err) { |
955 | kfree(objp: sq->sg); |
956 | sq->sg = NULL; |
957 | return err; |
958 | } |
959 | } |
960 | |
961 | sq->head = 0; |
962 | sq->cons_head = 0; |
963 | sq->sqe_per_sqb = (pfvf->hw.sqb_size / sq->sqe_size) - 1; |
964 | sq->num_sqbs = (qset->sqe_cnt + sq->sqe_per_sqb) / sq->sqe_per_sqb; |
965 | /* Set SQE threshold to 10% of total SQEs */ |
966 | sq->sqe_thresh = ((sq->num_sqbs * sq->sqe_per_sqb) * 10) / 100; |
967 | sq->aura_id = sqb_aura; |
968 | sq->aura_fc_addr = pool->fc_addr->base; |
969 | sq->io_addr = (__force u64)otx2_get_regaddr(nic: pfvf, NIX_LF_OP_SENDX(0)); |
970 | |
971 | sq->stats.bytes = 0; |
972 | sq->stats.pkts = 0; |
973 | |
974 | err = pfvf->hw_ops->sq_aq_init(pfvf, qidx, sqb_aura); |
975 | if (err) { |
976 | kfree(objp: sq->sg); |
977 | sq->sg = NULL; |
978 | return err; |
979 | } |
980 | |
981 | return 0; |
982 | |
983 | } |
984 | |
985 | static int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx) |
986 | { |
987 | struct otx2_qset *qset = &pfvf->qset; |
988 | int err, pool_id, non_xdp_queues; |
989 | struct nix_aq_enq_req *aq; |
990 | struct otx2_cq_queue *cq; |
991 | |
992 | cq = &qset->cq[qidx]; |
993 | cq->cq_idx = qidx; |
994 | non_xdp_queues = pfvf->hw.rx_queues + pfvf->hw.tx_queues; |
995 | if (qidx < pfvf->hw.rx_queues) { |
996 | cq->cq_type = CQ_RX; |
997 | cq->cint_idx = qidx; |
998 | cq->cqe_cnt = qset->rqe_cnt; |
999 | if (pfvf->xdp_prog) |
1000 | xdp_rxq_info_reg(xdp_rxq: &cq->xdp_rxq, dev: pfvf->netdev, queue_index: qidx, napi_id: 0); |
1001 | } else if (qidx < non_xdp_queues) { |
1002 | cq->cq_type = CQ_TX; |
1003 | cq->cint_idx = qidx - pfvf->hw.rx_queues; |
1004 | cq->cqe_cnt = qset->sqe_cnt; |
1005 | } else { |
1006 | if (pfvf->hw.xdp_queues && |
1007 | qidx < non_xdp_queues + pfvf->hw.xdp_queues) { |
1008 | cq->cq_type = CQ_XDP; |
1009 | cq->cint_idx = qidx - non_xdp_queues; |
1010 | cq->cqe_cnt = qset->sqe_cnt; |
1011 | } else { |
1012 | cq->cq_type = CQ_QOS; |
1013 | cq->cint_idx = qidx - non_xdp_queues - |
1014 | pfvf->hw.xdp_queues; |
1015 | cq->cqe_cnt = qset->sqe_cnt; |
1016 | } |
1017 | } |
1018 | cq->cqe_size = pfvf->qset.xqe_size; |
1019 | |
1020 | /* Allocate memory for CQEs */ |
1021 | err = qmem_alloc(dev: pfvf->dev, q: &cq->cqe, qsize: cq->cqe_cnt, entry_sz: cq->cqe_size); |
1022 | if (err) |
1023 | return err; |
1024 | |
1025 | /* Save CQE CPU base for faster reference */ |
1026 | cq->cqe_base = cq->cqe->base; |
1027 | /* In case where all RQs auras point to single pool, |
1028 | * all CQs receive buffer pool also point to same pool. |
1029 | */ |
1030 | pool_id = ((cq->cq_type == CQ_RX) && |
1031 | (pfvf->hw.rqpool_cnt != pfvf->hw.rx_queues)) ? 0 : qidx; |
1032 | cq->rbpool = &qset->pool[pool_id]; |
1033 | cq->refill_task_sched = false; |
1034 | |
1035 | /* Get memory to put this msg */ |
1036 | aq = otx2_mbox_alloc_msg_nix_aq_enq(mbox: &pfvf->mbox); |
1037 | if (!aq) |
1038 | return -ENOMEM; |
1039 | |
1040 | aq->cq.ena = 1; |
1041 | aq->cq.qsize = Q_SIZE(cq->cqe_cnt, 4); |
1042 | aq->cq.caching = 1; |
1043 | aq->cq.base = cq->cqe->iova; |
1044 | aq->cq.cint_idx = cq->cint_idx; |
1045 | aq->cq.cq_err_int_ena = NIX_CQERRINT_BITS; |
1046 | aq->cq.qint_idx = 0; |
1047 | aq->cq.avg_level = 255; |
1048 | |
1049 | if (qidx < pfvf->hw.rx_queues) { |
1050 | aq->cq.drop = RQ_DROP_LVL_CQ(pfvf->hw.rq_skid, cq->cqe_cnt); |
1051 | aq->cq.drop_ena = 1; |
1052 | |
1053 | if (!is_otx2_lbkvf(pdev: pfvf->pdev)) { |
1054 | /* Enable receive CQ backpressure */ |
1055 | aq->cq.bp_ena = 1; |
1056 | #ifdef CONFIG_DCB |
1057 | aq->cq.bpid = pfvf->bpid[pfvf->queue_to_pfc_map[qidx]]; |
1058 | #else |
1059 | aq->cq.bpid = pfvf->bpid[0]; |
1060 | #endif |
1061 | |
1062 | /* Set backpressure level is same as cq pass level */ |
1063 | aq->cq.bp = RQ_PASS_LVL_CQ(pfvf->hw.rq_skid, qset->rqe_cnt); |
1064 | } |
1065 | } |
1066 | |
1067 | /* Fill AQ info */ |
1068 | aq->qidx = qidx; |
1069 | aq->ctype = NIX_AQ_CTYPE_CQ; |
1070 | aq->op = NIX_AQ_INSTOP_INIT; |
1071 | |
1072 | return otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
1073 | } |
1074 | |
1075 | static void otx2_pool_refill_task(struct work_struct *work) |
1076 | { |
1077 | struct otx2_cq_queue *cq; |
1078 | struct refill_work *wrk; |
1079 | struct otx2_nic *pfvf; |
1080 | int qidx; |
1081 | |
1082 | wrk = container_of(work, struct refill_work, pool_refill_work.work); |
1083 | pfvf = wrk->pf; |
1084 | qidx = wrk - pfvf->refill_wrk; |
1085 | cq = &pfvf->qset.cq[qidx]; |
1086 | |
1087 | cq->refill_task_sched = false; |
1088 | |
1089 | local_bh_disable(); |
1090 | napi_schedule(n: wrk->napi); |
1091 | local_bh_enable(); |
1092 | } |
1093 | |
1094 | int otx2_config_nix_queues(struct otx2_nic *pfvf) |
1095 | { |
1096 | int qidx, err; |
1097 | |
1098 | /* Initialize RX queues */ |
1099 | for (qidx = 0; qidx < pfvf->hw.rx_queues; qidx++) { |
1100 | u16 lpb_aura = otx2_get_pool_idx(pfvf, type: AURA_NIX_RQ, idx: qidx); |
1101 | |
1102 | err = otx2_rq_init(pfvf, qidx, lpb_aura); |
1103 | if (err) |
1104 | return err; |
1105 | } |
1106 | |
1107 | /* Initialize TX queues */ |
1108 | for (qidx = 0; qidx < pfvf->hw.non_qos_queues; qidx++) { |
1109 | u16 sqb_aura = otx2_get_pool_idx(pfvf, type: AURA_NIX_SQ, idx: qidx); |
1110 | |
1111 | err = otx2_sq_init(pfvf, qidx, sqb_aura); |
1112 | if (err) |
1113 | return err; |
1114 | } |
1115 | |
1116 | /* Initialize completion queues */ |
1117 | for (qidx = 0; qidx < pfvf->qset.cq_cnt; qidx++) { |
1118 | err = otx2_cq_init(pfvf, qidx); |
1119 | if (err) |
1120 | return err; |
1121 | } |
1122 | |
1123 | pfvf->cq_op_addr = (__force u64 *)otx2_get_regaddr(nic: pfvf, |
1124 | NIX_LF_CQ_OP_STATUS); |
1125 | |
1126 | /* Initialize work queue for receive buffer refill */ |
1127 | pfvf->refill_wrk = devm_kcalloc(dev: pfvf->dev, n: pfvf->qset.cq_cnt, |
1128 | size: sizeof(struct refill_work), GFP_KERNEL); |
1129 | if (!pfvf->refill_wrk) |
1130 | return -ENOMEM; |
1131 | |
1132 | for (qidx = 0; qidx < pfvf->qset.cq_cnt; qidx++) { |
1133 | pfvf->refill_wrk[qidx].pf = pfvf; |
1134 | INIT_DELAYED_WORK(&pfvf->refill_wrk[qidx].pool_refill_work, |
1135 | otx2_pool_refill_task); |
1136 | } |
1137 | return 0; |
1138 | } |
1139 | |
1140 | int otx2_config_nix(struct otx2_nic *pfvf) |
1141 | { |
1142 | struct nix_lf_alloc_req *nixlf; |
1143 | struct nix_lf_alloc_rsp *rsp; |
1144 | int err; |
1145 | |
1146 | pfvf->qset.xqe_size = pfvf->hw.xqe_size; |
1147 | |
1148 | /* Get memory to put this msg */ |
1149 | nixlf = otx2_mbox_alloc_msg_nix_lf_alloc(mbox: &pfvf->mbox); |
1150 | if (!nixlf) |
1151 | return -ENOMEM; |
1152 | |
1153 | /* Set RQ/SQ/CQ counts */ |
1154 | nixlf->rq_cnt = pfvf->hw.rx_queues; |
1155 | nixlf->sq_cnt = otx2_get_total_tx_queues(pfvf); |
1156 | nixlf->cq_cnt = pfvf->qset.cq_cnt; |
1157 | nixlf->rss_sz = MAX_RSS_INDIR_TBL_SIZE; |
1158 | nixlf->rss_grps = MAX_RSS_GROUPS; |
1159 | nixlf->xqe_sz = pfvf->hw.xqe_size == 128 ? NIX_XQESZ_W16 : NIX_XQESZ_W64; |
1160 | /* We don't know absolute NPA LF idx attached. |
1161 | * AF will replace 'RVU_DEFAULT_PF_FUNC' with |
1162 | * NPA LF attached to this RVU PF/VF. |
1163 | */ |
1164 | nixlf->npa_func = RVU_DEFAULT_PF_FUNC; |
1165 | /* Disable alignment pad, enable L2 length check, |
1166 | * enable L4 TCP/UDP checksum verification. |
1167 | */ |
1168 | nixlf->rx_cfg = BIT_ULL(33) | BIT_ULL(35) | BIT_ULL(37); |
1169 | |
1170 | err = otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
1171 | if (err) |
1172 | return err; |
1173 | |
1174 | rsp = (struct nix_lf_alloc_rsp *)otx2_mbox_get_rsp(mbox: &pfvf->mbox.mbox, devid: 0, |
1175 | msg: &nixlf->hdr); |
1176 | if (IS_ERR(ptr: rsp)) |
1177 | return PTR_ERR(ptr: rsp); |
1178 | |
1179 | if (rsp->qints < 1) |
1180 | return -ENXIO; |
1181 | |
1182 | return rsp->hdr.rc; |
1183 | } |
1184 | |
1185 | void otx2_sq_free_sqbs(struct otx2_nic *pfvf) |
1186 | { |
1187 | struct otx2_qset *qset = &pfvf->qset; |
1188 | struct otx2_hw *hw = &pfvf->hw; |
1189 | struct otx2_snd_queue *sq; |
1190 | int sqb, qidx; |
1191 | u64 iova, pa; |
1192 | |
1193 | for (qidx = 0; qidx < otx2_get_total_tx_queues(pfvf); qidx++) { |
1194 | sq = &qset->sq[qidx]; |
1195 | if (!sq->sqb_ptrs) |
1196 | continue; |
1197 | for (sqb = 0; sqb < sq->sqb_count; sqb++) { |
1198 | if (!sq->sqb_ptrs[sqb]) |
1199 | continue; |
1200 | iova = sq->sqb_ptrs[sqb]; |
1201 | pa = otx2_iova_to_phys(iommu_domain: pfvf->iommu_domain, dma_addr: iova); |
1202 | dma_unmap_page_attrs(dev: pfvf->dev, addr: iova, size: hw->sqb_size, |
1203 | dir: DMA_FROM_DEVICE, |
1204 | DMA_ATTR_SKIP_CPU_SYNC); |
1205 | put_page(virt_to_page(phys_to_virt(pa))); |
1206 | } |
1207 | sq->sqb_count = 0; |
1208 | } |
1209 | } |
1210 | |
1211 | void otx2_free_bufs(struct otx2_nic *pfvf, struct otx2_pool *pool, |
1212 | u64 iova, int size) |
1213 | { |
1214 | struct page *page; |
1215 | u64 pa; |
1216 | |
1217 | pa = otx2_iova_to_phys(iommu_domain: pfvf->iommu_domain, dma_addr: iova); |
1218 | page = virt_to_head_page(phys_to_virt(address: pa)); |
1219 | |
1220 | if (pool->page_pool) { |
1221 | page_pool_put_full_page(pool: pool->page_pool, page, allow_direct: true); |
1222 | } else { |
1223 | dma_unmap_page_attrs(dev: pfvf->dev, addr: iova, size, |
1224 | dir: DMA_FROM_DEVICE, |
1225 | DMA_ATTR_SKIP_CPU_SYNC); |
1226 | |
1227 | put_page(page); |
1228 | } |
1229 | } |
1230 | |
1231 | void otx2_free_aura_ptr(struct otx2_nic *pfvf, int type) |
1232 | { |
1233 | int pool_id, pool_start = 0, pool_end = 0, size = 0; |
1234 | struct otx2_pool *pool; |
1235 | u64 iova; |
1236 | |
1237 | if (type == AURA_NIX_SQ) { |
1238 | pool_start = otx2_get_pool_idx(pfvf, type, idx: 0); |
1239 | pool_end = pool_start + pfvf->hw.sqpool_cnt; |
1240 | size = pfvf->hw.sqb_size; |
1241 | } |
1242 | if (type == AURA_NIX_RQ) { |
1243 | pool_start = otx2_get_pool_idx(pfvf, type, idx: 0); |
1244 | pool_end = pfvf->hw.rqpool_cnt; |
1245 | size = pfvf->rbsize; |
1246 | } |
1247 | |
1248 | /* Free SQB and RQB pointers from the aura pool */ |
1249 | for (pool_id = pool_start; pool_id < pool_end; pool_id++) { |
1250 | iova = otx2_aura_allocptr(pfvf, aura: pool_id); |
1251 | pool = &pfvf->qset.pool[pool_id]; |
1252 | while (iova) { |
1253 | if (type == AURA_NIX_RQ) |
1254 | iova -= OTX2_HEAD_ROOM; |
1255 | |
1256 | otx2_free_bufs(pfvf, pool, iova, size); |
1257 | |
1258 | iova = otx2_aura_allocptr(pfvf, aura: pool_id); |
1259 | } |
1260 | } |
1261 | } |
1262 | |
1263 | void otx2_aura_pool_free(struct otx2_nic *pfvf) |
1264 | { |
1265 | struct otx2_pool *pool; |
1266 | int pool_id; |
1267 | |
1268 | if (!pfvf->qset.pool) |
1269 | return; |
1270 | |
1271 | for (pool_id = 0; pool_id < pfvf->hw.pool_cnt; pool_id++) { |
1272 | pool = &pfvf->qset.pool[pool_id]; |
1273 | qmem_free(dev: pfvf->dev, qmem: pool->stack); |
1274 | qmem_free(dev: pfvf->dev, qmem: pool->fc_addr); |
1275 | page_pool_destroy(pool: pool->page_pool); |
1276 | pool->page_pool = NULL; |
1277 | } |
1278 | devm_kfree(dev: pfvf->dev, p: pfvf->qset.pool); |
1279 | pfvf->qset.pool = NULL; |
1280 | } |
1281 | |
1282 | int otx2_aura_init(struct otx2_nic *pfvf, int aura_id, |
1283 | int pool_id, int numptrs) |
1284 | { |
1285 | struct npa_aq_enq_req *aq; |
1286 | struct otx2_pool *pool; |
1287 | int err; |
1288 | |
1289 | pool = &pfvf->qset.pool[pool_id]; |
1290 | |
1291 | /* Allocate memory for HW to update Aura count. |
1292 | * Alloc one cache line, so that it fits all FC_STYPE modes. |
1293 | */ |
1294 | if (!pool->fc_addr) { |
1295 | err = qmem_alloc(dev: pfvf->dev, q: &pool->fc_addr, qsize: 1, OTX2_ALIGN); |
1296 | if (err) |
1297 | return err; |
1298 | } |
1299 | |
1300 | /* Initialize this aura's context via AF */ |
1301 | aq = otx2_mbox_alloc_msg_npa_aq_enq(mbox: &pfvf->mbox); |
1302 | if (!aq) { |
1303 | /* Shared mbox memory buffer is full, flush it and retry */ |
1304 | err = otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
1305 | if (err) |
1306 | return err; |
1307 | aq = otx2_mbox_alloc_msg_npa_aq_enq(mbox: &pfvf->mbox); |
1308 | if (!aq) |
1309 | return -ENOMEM; |
1310 | } |
1311 | |
1312 | aq->aura_id = aura_id; |
1313 | /* Will be filled by AF with correct pool context address */ |
1314 | aq->aura.pool_addr = pool_id; |
1315 | aq->aura.pool_caching = 1; |
1316 | aq->aura.shift = ilog2(numptrs) - 8; |
1317 | aq->aura.count = numptrs; |
1318 | aq->aura.limit = numptrs; |
1319 | aq->aura.avg_level = 255; |
1320 | aq->aura.ena = 1; |
1321 | aq->aura.fc_ena = 1; |
1322 | aq->aura.fc_addr = pool->fc_addr->iova; |
1323 | aq->aura.fc_hyst_bits = 0; /* Store count on all updates */ |
1324 | |
1325 | /* Enable backpressure for RQ aura */ |
1326 | if (aura_id < pfvf->hw.rqpool_cnt && !is_otx2_lbkvf(pdev: pfvf->pdev)) { |
1327 | aq->aura.bp_ena = 0; |
1328 | /* If NIX1 LF is attached then specify NIX1_RX. |
1329 | * |
1330 | * Below NPA_AURA_S[BP_ENA] is set according to the |
1331 | * NPA_BPINTF_E enumeration given as: |
1332 | * 0x0 + a*0x1 where 'a' is 0 for NIX0_RX and 1 for NIX1_RX so |
1333 | * NIX0_RX is 0x0 + 0*0x1 = 0 |
1334 | * NIX1_RX is 0x0 + 1*0x1 = 1 |
1335 | * But in HRM it is given that |
1336 | * "NPA_AURA_S[BP_ENA](w1[33:32]) - Enable aura backpressure to |
1337 | * NIX-RX based on [BP] level. One bit per NIX-RX; index |
1338 | * enumerated by NPA_BPINTF_E." |
1339 | */ |
1340 | if (pfvf->nix_blkaddr == BLKADDR_NIX1) |
1341 | aq->aura.bp_ena = 1; |
1342 | #ifdef CONFIG_DCB |
1343 | aq->aura.nix0_bpid = pfvf->bpid[pfvf->queue_to_pfc_map[aura_id]]; |
1344 | #else |
1345 | aq->aura.nix0_bpid = pfvf->bpid[0]; |
1346 | #endif |
1347 | |
1348 | /* Set backpressure level for RQ's Aura */ |
1349 | aq->aura.bp = RQ_BP_LVL_AURA; |
1350 | } |
1351 | |
1352 | /* Fill AQ info */ |
1353 | aq->ctype = NPA_AQ_CTYPE_AURA; |
1354 | aq->op = NPA_AQ_INSTOP_INIT; |
1355 | |
1356 | return 0; |
1357 | } |
1358 | |
1359 | int otx2_pool_init(struct otx2_nic *pfvf, u16 pool_id, |
1360 | int stack_pages, int numptrs, int buf_size, int type) |
1361 | { |
1362 | struct page_pool_params pp_params = { 0 }; |
1363 | struct npa_aq_enq_req *aq; |
1364 | struct otx2_pool *pool; |
1365 | int err; |
1366 | |
1367 | pool = &pfvf->qset.pool[pool_id]; |
1368 | /* Alloc memory for stack which is used to store buffer pointers */ |
1369 | err = qmem_alloc(dev: pfvf->dev, q: &pool->stack, |
1370 | qsize: stack_pages, entry_sz: pfvf->hw.stack_pg_bytes); |
1371 | if (err) |
1372 | return err; |
1373 | |
1374 | pool->rbsize = buf_size; |
1375 | |
1376 | /* Initialize this pool's context via AF */ |
1377 | aq = otx2_mbox_alloc_msg_npa_aq_enq(mbox: &pfvf->mbox); |
1378 | if (!aq) { |
1379 | /* Shared mbox memory buffer is full, flush it and retry */ |
1380 | err = otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
1381 | if (err) { |
1382 | qmem_free(dev: pfvf->dev, qmem: pool->stack); |
1383 | return err; |
1384 | } |
1385 | aq = otx2_mbox_alloc_msg_npa_aq_enq(mbox: &pfvf->mbox); |
1386 | if (!aq) { |
1387 | qmem_free(dev: pfvf->dev, qmem: pool->stack); |
1388 | return -ENOMEM; |
1389 | } |
1390 | } |
1391 | |
1392 | aq->aura_id = pool_id; |
1393 | aq->pool.stack_base = pool->stack->iova; |
1394 | aq->pool.stack_caching = 1; |
1395 | aq->pool.ena = 1; |
1396 | aq->pool.buf_size = buf_size / 128; |
1397 | aq->pool.stack_max_pages = stack_pages; |
1398 | aq->pool.shift = ilog2(numptrs) - 8; |
1399 | aq->pool.ptr_start = 0; |
1400 | aq->pool.ptr_end = ~0ULL; |
1401 | |
1402 | /* Fill AQ info */ |
1403 | aq->ctype = NPA_AQ_CTYPE_POOL; |
1404 | aq->op = NPA_AQ_INSTOP_INIT; |
1405 | |
1406 | if (type != AURA_NIX_RQ) { |
1407 | pool->page_pool = NULL; |
1408 | return 0; |
1409 | } |
1410 | |
1411 | pp_params.order = get_order(size: buf_size); |
1412 | pp_params.flags = PP_FLAG_DMA_MAP; |
1413 | pp_params.pool_size = min(OTX2_PAGE_POOL_SZ, numptrs); |
1414 | pp_params.nid = NUMA_NO_NODE; |
1415 | pp_params.dev = pfvf->dev; |
1416 | pp_params.dma_dir = DMA_FROM_DEVICE; |
1417 | pool->page_pool = page_pool_create(params: &pp_params); |
1418 | if (IS_ERR(ptr: pool->page_pool)) { |
1419 | netdev_err(dev: pfvf->netdev, format: "Creation of page pool failed\n" ); |
1420 | return PTR_ERR(ptr: pool->page_pool); |
1421 | } |
1422 | |
1423 | return 0; |
1424 | } |
1425 | |
1426 | int otx2_sq_aura_pool_init(struct otx2_nic *pfvf) |
1427 | { |
1428 | int qidx, pool_id, stack_pages, num_sqbs; |
1429 | struct otx2_qset *qset = &pfvf->qset; |
1430 | struct otx2_hw *hw = &pfvf->hw; |
1431 | struct otx2_snd_queue *sq; |
1432 | struct otx2_pool *pool; |
1433 | dma_addr_t bufptr; |
1434 | int err, ptr; |
1435 | |
1436 | /* Calculate number of SQBs needed. |
1437 | * |
1438 | * For a 128byte SQE, and 4K size SQB, 31 SQEs will fit in one SQB. |
1439 | * Last SQE is used for pointing to next SQB. |
1440 | */ |
1441 | num_sqbs = (hw->sqb_size / 128) - 1; |
1442 | num_sqbs = (qset->sqe_cnt + num_sqbs) / num_sqbs; |
1443 | |
1444 | /* Get no of stack pages needed */ |
1445 | stack_pages = |
1446 | (num_sqbs + hw->stack_pg_ptrs - 1) / hw->stack_pg_ptrs; |
1447 | |
1448 | for (qidx = 0; qidx < hw->non_qos_queues; qidx++) { |
1449 | pool_id = otx2_get_pool_idx(pfvf, type: AURA_NIX_SQ, idx: qidx); |
1450 | /* Initialize aura context */ |
1451 | err = otx2_aura_init(pfvf, aura_id: pool_id, pool_id, numptrs: num_sqbs); |
1452 | if (err) |
1453 | goto fail; |
1454 | |
1455 | /* Initialize pool context */ |
1456 | err = otx2_pool_init(pfvf, pool_id, stack_pages, |
1457 | numptrs: num_sqbs, buf_size: hw->sqb_size, type: AURA_NIX_SQ); |
1458 | if (err) |
1459 | goto fail; |
1460 | } |
1461 | |
1462 | /* Flush accumulated messages */ |
1463 | err = otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
1464 | if (err) |
1465 | goto fail; |
1466 | |
1467 | /* Allocate pointers and free them to aura/pool */ |
1468 | for (qidx = 0; qidx < hw->non_qos_queues; qidx++) { |
1469 | pool_id = otx2_get_pool_idx(pfvf, type: AURA_NIX_SQ, idx: qidx); |
1470 | pool = &pfvf->qset.pool[pool_id]; |
1471 | |
1472 | sq = &qset->sq[qidx]; |
1473 | sq->sqb_count = 0; |
1474 | sq->sqb_ptrs = kcalloc(n: num_sqbs, size: sizeof(*sq->sqb_ptrs), GFP_KERNEL); |
1475 | if (!sq->sqb_ptrs) { |
1476 | err = -ENOMEM; |
1477 | goto err_mem; |
1478 | } |
1479 | |
1480 | for (ptr = 0; ptr < num_sqbs; ptr++) { |
1481 | err = otx2_alloc_rbuf(pfvf, pool, dma: &bufptr); |
1482 | if (err) |
1483 | goto err_mem; |
1484 | pfvf->hw_ops->aura_freeptr(pfvf, pool_id, bufptr); |
1485 | sq->sqb_ptrs[sq->sqb_count++] = (u64)bufptr; |
1486 | } |
1487 | } |
1488 | |
1489 | err_mem: |
1490 | return err ? -ENOMEM : 0; |
1491 | |
1492 | fail: |
1493 | otx2_mbox_reset(mbox: &pfvf->mbox.mbox, devid: 0); |
1494 | otx2_aura_pool_free(pfvf); |
1495 | return err; |
1496 | } |
1497 | |
1498 | int otx2_rq_aura_pool_init(struct otx2_nic *pfvf) |
1499 | { |
1500 | struct otx2_hw *hw = &pfvf->hw; |
1501 | int stack_pages, pool_id, rq; |
1502 | struct otx2_pool *pool; |
1503 | int err, ptr, num_ptrs; |
1504 | dma_addr_t bufptr; |
1505 | |
1506 | num_ptrs = pfvf->qset.rqe_cnt; |
1507 | |
1508 | stack_pages = |
1509 | (num_ptrs + hw->stack_pg_ptrs - 1) / hw->stack_pg_ptrs; |
1510 | |
1511 | for (rq = 0; rq < hw->rx_queues; rq++) { |
1512 | pool_id = otx2_get_pool_idx(pfvf, type: AURA_NIX_RQ, idx: rq); |
1513 | /* Initialize aura context */ |
1514 | err = otx2_aura_init(pfvf, aura_id: pool_id, pool_id, numptrs: num_ptrs); |
1515 | if (err) |
1516 | goto fail; |
1517 | } |
1518 | for (pool_id = 0; pool_id < hw->rqpool_cnt; pool_id++) { |
1519 | err = otx2_pool_init(pfvf, pool_id, stack_pages, |
1520 | numptrs: num_ptrs, buf_size: pfvf->rbsize, type: AURA_NIX_RQ); |
1521 | if (err) |
1522 | goto fail; |
1523 | } |
1524 | |
1525 | /* Flush accumulated messages */ |
1526 | err = otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
1527 | if (err) |
1528 | goto fail; |
1529 | |
1530 | /* Allocate pointers and free them to aura/pool */ |
1531 | for (pool_id = 0; pool_id < hw->rqpool_cnt; pool_id++) { |
1532 | pool = &pfvf->qset.pool[pool_id]; |
1533 | for (ptr = 0; ptr < num_ptrs; ptr++) { |
1534 | err = otx2_alloc_rbuf(pfvf, pool, dma: &bufptr); |
1535 | if (err) |
1536 | return -ENOMEM; |
1537 | pfvf->hw_ops->aura_freeptr(pfvf, pool_id, |
1538 | bufptr + OTX2_HEAD_ROOM); |
1539 | } |
1540 | } |
1541 | return 0; |
1542 | fail: |
1543 | otx2_mbox_reset(mbox: &pfvf->mbox.mbox, devid: 0); |
1544 | otx2_aura_pool_free(pfvf); |
1545 | return err; |
1546 | } |
1547 | |
1548 | int otx2_config_npa(struct otx2_nic *pfvf) |
1549 | { |
1550 | struct otx2_qset *qset = &pfvf->qset; |
1551 | struct npa_lf_alloc_req *npalf; |
1552 | struct otx2_hw *hw = &pfvf->hw; |
1553 | int aura_cnt; |
1554 | |
1555 | /* Pool - Stack of free buffer pointers |
1556 | * Aura - Alloc/frees pointers from/to pool for NIX DMA. |
1557 | */ |
1558 | |
1559 | if (!hw->pool_cnt) |
1560 | return -EINVAL; |
1561 | |
1562 | qset->pool = devm_kcalloc(dev: pfvf->dev, n: hw->pool_cnt, |
1563 | size: sizeof(struct otx2_pool), GFP_KERNEL); |
1564 | if (!qset->pool) |
1565 | return -ENOMEM; |
1566 | |
1567 | /* Get memory to put this msg */ |
1568 | npalf = otx2_mbox_alloc_msg_npa_lf_alloc(mbox: &pfvf->mbox); |
1569 | if (!npalf) |
1570 | return -ENOMEM; |
1571 | |
1572 | /* Set aura and pool counts */ |
1573 | npalf->nr_pools = hw->pool_cnt; |
1574 | aura_cnt = ilog2(roundup_pow_of_two(hw->pool_cnt)); |
1575 | npalf->aura_sz = (aura_cnt >= ilog2(128)) ? (aura_cnt - 6) : 1; |
1576 | |
1577 | return otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
1578 | } |
1579 | |
1580 | int otx2_detach_resources(struct mbox *mbox) |
1581 | { |
1582 | struct rsrc_detach *detach; |
1583 | |
1584 | mutex_lock(&mbox->lock); |
1585 | detach = otx2_mbox_alloc_msg_detach_resources(mbox); |
1586 | if (!detach) { |
1587 | mutex_unlock(lock: &mbox->lock); |
1588 | return -ENOMEM; |
1589 | } |
1590 | |
1591 | /* detach all */ |
1592 | detach->partial = false; |
1593 | |
1594 | /* Send detach request to AF */ |
1595 | otx2_sync_mbox_msg(mbox); |
1596 | mutex_unlock(lock: &mbox->lock); |
1597 | return 0; |
1598 | } |
1599 | EXPORT_SYMBOL(otx2_detach_resources); |
1600 | |
1601 | int otx2_attach_npa_nix(struct otx2_nic *pfvf) |
1602 | { |
1603 | struct rsrc_attach *attach; |
1604 | struct msg_req *msix; |
1605 | int err; |
1606 | |
1607 | mutex_lock(&pfvf->mbox.lock); |
1608 | /* Get memory to put this msg */ |
1609 | attach = otx2_mbox_alloc_msg_attach_resources(mbox: &pfvf->mbox); |
1610 | if (!attach) { |
1611 | mutex_unlock(lock: &pfvf->mbox.lock); |
1612 | return -ENOMEM; |
1613 | } |
1614 | |
1615 | attach->npalf = true; |
1616 | attach->nixlf = true; |
1617 | |
1618 | /* Send attach request to AF */ |
1619 | err = otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
1620 | if (err) { |
1621 | mutex_unlock(lock: &pfvf->mbox.lock); |
1622 | return err; |
1623 | } |
1624 | |
1625 | pfvf->nix_blkaddr = BLKADDR_NIX0; |
1626 | |
1627 | /* If the platform has two NIX blocks then LF may be |
1628 | * allocated from NIX1. |
1629 | */ |
1630 | if (otx2_read64(nic: pfvf, RVU_PF_BLOCK_ADDRX_DISC(BLKADDR_NIX1)) & 0x1FFULL) |
1631 | pfvf->nix_blkaddr = BLKADDR_NIX1; |
1632 | |
1633 | /* Get NPA and NIX MSIX vector offsets */ |
1634 | msix = otx2_mbox_alloc_msg_msix_offset(mbox: &pfvf->mbox); |
1635 | if (!msix) { |
1636 | mutex_unlock(lock: &pfvf->mbox.lock); |
1637 | return -ENOMEM; |
1638 | } |
1639 | |
1640 | err = otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
1641 | if (err) { |
1642 | mutex_unlock(lock: &pfvf->mbox.lock); |
1643 | return err; |
1644 | } |
1645 | mutex_unlock(lock: &pfvf->mbox.lock); |
1646 | |
1647 | if (pfvf->hw.npa_msixoff == MSIX_VECTOR_INVALID || |
1648 | pfvf->hw.nix_msixoff == MSIX_VECTOR_INVALID) { |
1649 | dev_err(pfvf->dev, |
1650 | "RVUPF: Invalid MSIX vector offset for NPA/NIX\n" ); |
1651 | return -EINVAL; |
1652 | } |
1653 | |
1654 | return 0; |
1655 | } |
1656 | EXPORT_SYMBOL(otx2_attach_npa_nix); |
1657 | |
1658 | void otx2_ctx_disable(struct mbox *mbox, int type, bool npa) |
1659 | { |
1660 | struct hwctx_disable_req *req; |
1661 | |
1662 | mutex_lock(&mbox->lock); |
1663 | /* Request AQ to disable this context */ |
1664 | if (npa) |
1665 | req = otx2_mbox_alloc_msg_npa_hwctx_disable(mbox); |
1666 | else |
1667 | req = otx2_mbox_alloc_msg_nix_hwctx_disable(mbox); |
1668 | |
1669 | if (!req) { |
1670 | mutex_unlock(lock: &mbox->lock); |
1671 | return; |
1672 | } |
1673 | |
1674 | req->ctype = type; |
1675 | |
1676 | if (otx2_sync_mbox_msg(mbox)) |
1677 | dev_err(mbox->pfvf->dev, "%s failed to disable context\n" , |
1678 | __func__); |
1679 | |
1680 | mutex_unlock(lock: &mbox->lock); |
1681 | } |
1682 | |
1683 | int otx2_nix_config_bp(struct otx2_nic *pfvf, bool enable) |
1684 | { |
1685 | struct nix_bp_cfg_req *req; |
1686 | |
1687 | if (enable) |
1688 | req = otx2_mbox_alloc_msg_nix_bp_enable(mbox: &pfvf->mbox); |
1689 | else |
1690 | req = otx2_mbox_alloc_msg_nix_bp_disable(mbox: &pfvf->mbox); |
1691 | |
1692 | if (!req) |
1693 | return -ENOMEM; |
1694 | |
1695 | req->chan_base = 0; |
1696 | #ifdef CONFIG_DCB |
1697 | req->chan_cnt = pfvf->pfc_en ? IEEE_8021QAZ_MAX_TCS : 1; |
1698 | req->bpid_per_chan = pfvf->pfc_en ? 1 : 0; |
1699 | #else |
1700 | req->chan_cnt = 1; |
1701 | req->bpid_per_chan = 0; |
1702 | #endif |
1703 | |
1704 | return otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
1705 | } |
1706 | EXPORT_SYMBOL(otx2_nix_config_bp); |
1707 | |
1708 | /* Mbox message handlers */ |
1709 | void mbox_handler_cgx_stats(struct otx2_nic *pfvf, |
1710 | struct cgx_stats_rsp *rsp) |
1711 | { |
1712 | int id; |
1713 | |
1714 | for (id = 0; id < CGX_RX_STATS_COUNT; id++) |
1715 | pfvf->hw.cgx_rx_stats[id] = rsp->rx_stats[id]; |
1716 | for (id = 0; id < CGX_TX_STATS_COUNT; id++) |
1717 | pfvf->hw.cgx_tx_stats[id] = rsp->tx_stats[id]; |
1718 | } |
1719 | |
1720 | void mbox_handler_cgx_fec_stats(struct otx2_nic *pfvf, |
1721 | struct cgx_fec_stats_rsp *rsp) |
1722 | { |
1723 | pfvf->hw.cgx_fec_corr_blks += rsp->fec_corr_blks; |
1724 | pfvf->hw.cgx_fec_uncorr_blks += rsp->fec_uncorr_blks; |
1725 | } |
1726 | |
1727 | void mbox_handler_npa_lf_alloc(struct otx2_nic *pfvf, |
1728 | struct npa_lf_alloc_rsp *rsp) |
1729 | { |
1730 | pfvf->hw.stack_pg_ptrs = rsp->stack_pg_ptrs; |
1731 | pfvf->hw.stack_pg_bytes = rsp->stack_pg_bytes; |
1732 | } |
1733 | EXPORT_SYMBOL(mbox_handler_npa_lf_alloc); |
1734 | |
1735 | void mbox_handler_nix_lf_alloc(struct otx2_nic *pfvf, |
1736 | struct nix_lf_alloc_rsp *rsp) |
1737 | { |
1738 | pfvf->hw.sqb_size = rsp->sqb_size; |
1739 | pfvf->hw.rx_chan_base = rsp->rx_chan_base; |
1740 | pfvf->hw.tx_chan_base = rsp->tx_chan_base; |
1741 | pfvf->hw.lso_tsov4_idx = rsp->lso_tsov4_idx; |
1742 | pfvf->hw.lso_tsov6_idx = rsp->lso_tsov6_idx; |
1743 | pfvf->hw.cgx_links = rsp->cgx_links; |
1744 | pfvf->hw.lbk_links = rsp->lbk_links; |
1745 | pfvf->hw.tx_link = rsp->tx_link; |
1746 | } |
1747 | EXPORT_SYMBOL(mbox_handler_nix_lf_alloc); |
1748 | |
1749 | void mbox_handler_msix_offset(struct otx2_nic *pfvf, |
1750 | struct msix_offset_rsp *rsp) |
1751 | { |
1752 | pfvf->hw.npa_msixoff = rsp->npa_msixoff; |
1753 | pfvf->hw.nix_msixoff = rsp->nix_msixoff; |
1754 | } |
1755 | EXPORT_SYMBOL(mbox_handler_msix_offset); |
1756 | |
1757 | void mbox_handler_nix_bp_enable(struct otx2_nic *pfvf, |
1758 | struct nix_bp_cfg_rsp *rsp) |
1759 | { |
1760 | int chan, chan_id; |
1761 | |
1762 | for (chan = 0; chan < rsp->chan_cnt; chan++) { |
1763 | chan_id = ((rsp->chan_bpid[chan] >> 10) & 0x7F); |
1764 | pfvf->bpid[chan_id] = rsp->chan_bpid[chan] & 0x3FF; |
1765 | } |
1766 | } |
1767 | EXPORT_SYMBOL(mbox_handler_nix_bp_enable); |
1768 | |
1769 | void otx2_free_cints(struct otx2_nic *pfvf, int n) |
1770 | { |
1771 | struct otx2_qset *qset = &pfvf->qset; |
1772 | struct otx2_hw *hw = &pfvf->hw; |
1773 | int irq, qidx; |
1774 | |
1775 | for (qidx = 0, irq = hw->nix_msixoff + NIX_LF_CINT_VEC_START; |
1776 | qidx < n; |
1777 | qidx++, irq++) { |
1778 | int vector = pci_irq_vector(dev: pfvf->pdev, nr: irq); |
1779 | |
1780 | irq_set_affinity_hint(irq: vector, NULL); |
1781 | free_cpumask_var(mask: hw->affinity_mask[irq]); |
1782 | free_irq(vector, &qset->napi[qidx]); |
1783 | } |
1784 | } |
1785 | |
1786 | void otx2_set_cints_affinity(struct otx2_nic *pfvf) |
1787 | { |
1788 | struct otx2_hw *hw = &pfvf->hw; |
1789 | int vec, cpu, irq, cint; |
1790 | |
1791 | vec = hw->nix_msixoff + NIX_LF_CINT_VEC_START; |
1792 | cpu = cpumask_first(cpu_online_mask); |
1793 | |
1794 | /* CQ interrupts */ |
1795 | for (cint = 0; cint < pfvf->hw.cint_cnt; cint++, vec++) { |
1796 | if (!alloc_cpumask_var(mask: &hw->affinity_mask[vec], GFP_KERNEL)) |
1797 | return; |
1798 | |
1799 | cpumask_set_cpu(cpu, dstp: hw->affinity_mask[vec]); |
1800 | |
1801 | irq = pci_irq_vector(dev: pfvf->pdev, nr: vec); |
1802 | irq_set_affinity_hint(irq, m: hw->affinity_mask[vec]); |
1803 | |
1804 | cpu = cpumask_next(n: cpu, cpu_online_mask); |
1805 | if (unlikely(cpu >= nr_cpu_ids)) |
1806 | cpu = 0; |
1807 | } |
1808 | } |
1809 | |
1810 | static u32 get_dwrr_mtu(struct otx2_nic *pfvf, struct nix_hw_info *hw) |
1811 | { |
1812 | if (is_otx2_lbkvf(pdev: pfvf->pdev)) { |
1813 | pfvf->hw.smq_link_type = SMQ_LINK_TYPE_LBK; |
1814 | return hw->lbk_dwrr_mtu; |
1815 | } |
1816 | |
1817 | pfvf->hw.smq_link_type = SMQ_LINK_TYPE_RPM; |
1818 | return hw->rpm_dwrr_mtu; |
1819 | } |
1820 | |
1821 | u16 otx2_get_max_mtu(struct otx2_nic *pfvf) |
1822 | { |
1823 | struct nix_hw_info *rsp; |
1824 | struct msg_req *req; |
1825 | u16 max_mtu; |
1826 | int rc; |
1827 | |
1828 | mutex_lock(&pfvf->mbox.lock); |
1829 | |
1830 | req = otx2_mbox_alloc_msg_nix_get_hw_info(mbox: &pfvf->mbox); |
1831 | if (!req) { |
1832 | rc = -ENOMEM; |
1833 | goto out; |
1834 | } |
1835 | |
1836 | rc = otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
1837 | if (!rc) { |
1838 | rsp = (struct nix_hw_info *) |
1839 | otx2_mbox_get_rsp(mbox: &pfvf->mbox.mbox, devid: 0, msg: &req->hdr); |
1840 | |
1841 | /* HW counts VLAN insertion bytes (8 for double tag) |
1842 | * irrespective of whether SQE is requesting to insert VLAN |
1843 | * in the packet or not. Hence these 8 bytes have to be |
1844 | * discounted from max packet size otherwise HW will throw |
1845 | * SMQ errors |
1846 | */ |
1847 | max_mtu = rsp->max_mtu - 8 - OTX2_ETH_HLEN; |
1848 | |
1849 | /* Also save DWRR MTU, needed for DWRR weight calculation */ |
1850 | pfvf->hw.dwrr_mtu = get_dwrr_mtu(pfvf, hw: rsp); |
1851 | if (!pfvf->hw.dwrr_mtu) |
1852 | pfvf->hw.dwrr_mtu = 1; |
1853 | } |
1854 | |
1855 | out: |
1856 | mutex_unlock(lock: &pfvf->mbox.lock); |
1857 | if (rc) { |
1858 | dev_warn(pfvf->dev, |
1859 | "Failed to get MTU from hardware setting default value(1500)\n" ); |
1860 | max_mtu = 1500; |
1861 | } |
1862 | return max_mtu; |
1863 | } |
1864 | EXPORT_SYMBOL(otx2_get_max_mtu); |
1865 | |
1866 | int otx2_handle_ntuple_tc_features(struct net_device *netdev, netdev_features_t features) |
1867 | { |
1868 | netdev_features_t changed = features ^ netdev->features; |
1869 | struct otx2_nic *pfvf = netdev_priv(dev: netdev); |
1870 | bool ntuple = !!(features & NETIF_F_NTUPLE); |
1871 | bool tc = !!(features & NETIF_F_HW_TC); |
1872 | |
1873 | if ((changed & NETIF_F_NTUPLE) && !ntuple) |
1874 | otx2_destroy_ntuple_flows(pf: pfvf); |
1875 | |
1876 | if ((changed & NETIF_F_NTUPLE) && ntuple) { |
1877 | if (!pfvf->flow_cfg->max_flows) { |
1878 | netdev_err(dev: netdev, |
1879 | format: "Can't enable NTUPLE, MCAM entries not allocated\n" ); |
1880 | return -EINVAL; |
1881 | } |
1882 | } |
1883 | |
1884 | if ((changed & NETIF_F_HW_TC) && !tc && |
1885 | otx2_tc_flower_rule_cnt(pfvf)) { |
1886 | netdev_err(dev: netdev, format: "Can't disable TC hardware offload while flows are active\n" ); |
1887 | return -EBUSY; |
1888 | } |
1889 | |
1890 | if ((changed & NETIF_F_NTUPLE) && ntuple && |
1891 | otx2_tc_flower_rule_cnt(pfvf) && !(changed & NETIF_F_HW_TC)) { |
1892 | netdev_err(dev: netdev, |
1893 | format: "Can't enable NTUPLE when TC flower offload is active, disable TC rules and retry\n" ); |
1894 | return -EINVAL; |
1895 | } |
1896 | |
1897 | return 0; |
1898 | } |
1899 | EXPORT_SYMBOL(otx2_handle_ntuple_tc_features); |
1900 | |
1901 | #define M(_name, _id, _fn_name, _req_type, _rsp_type) \ |
1902 | int __weak \ |
1903 | otx2_mbox_up_handler_ ## _fn_name(struct otx2_nic *pfvf, \ |
1904 | struct _req_type *req, \ |
1905 | struct _rsp_type *rsp) \ |
1906 | { \ |
1907 | /* Nothing to do here */ \ |
1908 | return 0; \ |
1909 | } \ |
1910 | EXPORT_SYMBOL(otx2_mbox_up_handler_ ## _fn_name); |
1911 | MBOX_UP_CGX_MESSAGES |
1912 | MBOX_UP_MCS_MESSAGES |
1913 | #undef M |
1914 | |