1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Marvell RVU Physical Function ethernet driver |
3 | * |
4 | * Copyright (C) 2023 Marvell. |
5 | * |
6 | */ |
7 | |
8 | #include <linux/netdevice.h> |
9 | #include <net/tso.h> |
10 | |
11 | #include "cn10k.h" |
12 | #include "otx2_reg.h" |
13 | #include "otx2_common.h" |
14 | #include "otx2_txrx.h" |
15 | #include "otx2_struct.h" |
16 | |
17 | #define OTX2_QOS_MAX_LEAF_NODES 16 |
18 | |
19 | static void otx2_qos_aura_pool_free(struct otx2_nic *pfvf, int pool_id) |
20 | { |
21 | struct otx2_pool *pool; |
22 | |
23 | if (!pfvf->qset.pool) |
24 | return; |
25 | |
26 | pool = &pfvf->qset.pool[pool_id]; |
27 | qmem_free(dev: pfvf->dev, qmem: pool->stack); |
28 | qmem_free(dev: pfvf->dev, qmem: pool->fc_addr); |
29 | pool->stack = NULL; |
30 | pool->fc_addr = NULL; |
31 | } |
32 | |
33 | static int otx2_qos_sq_aura_pool_init(struct otx2_nic *pfvf, int qidx) |
34 | { |
35 | struct otx2_qset *qset = &pfvf->qset; |
36 | int pool_id, stack_pages, num_sqbs; |
37 | struct otx2_hw *hw = &pfvf->hw; |
38 | struct otx2_snd_queue *sq; |
39 | struct otx2_pool *pool; |
40 | dma_addr_t bufptr; |
41 | int err, ptr; |
42 | u64 iova, pa; |
43 | |
44 | /* Calculate number of SQBs needed. |
45 | * |
46 | * For a 128byte SQE, and 4K size SQB, 31 SQEs will fit in one SQB. |
47 | * Last SQE is used for pointing to next SQB. |
48 | */ |
49 | num_sqbs = (hw->sqb_size / 128) - 1; |
50 | num_sqbs = (qset->sqe_cnt + num_sqbs) / num_sqbs; |
51 | |
52 | /* Get no of stack pages needed */ |
53 | stack_pages = |
54 | (num_sqbs + hw->stack_pg_ptrs - 1) / hw->stack_pg_ptrs; |
55 | |
56 | pool_id = otx2_get_pool_idx(pfvf, type: AURA_NIX_SQ, idx: qidx); |
57 | pool = &pfvf->qset.pool[pool_id]; |
58 | |
59 | /* Initialize aura context */ |
60 | err = otx2_aura_init(pfvf, aura_id: pool_id, pool_id, numptrs: num_sqbs); |
61 | if (err) |
62 | return err; |
63 | |
64 | /* Initialize pool context */ |
65 | err = otx2_pool_init(pfvf, pool_id, stack_pages, |
66 | numptrs: num_sqbs, buf_size: hw->sqb_size, type: AURA_NIX_SQ); |
67 | if (err) |
68 | goto aura_free; |
69 | |
70 | /* Flush accumulated messages */ |
71 | err = otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
72 | if (err) |
73 | goto pool_free; |
74 | |
75 | /* Allocate pointers and free them to aura/pool */ |
76 | sq = &qset->sq[qidx]; |
77 | sq->sqb_count = 0; |
78 | sq->sqb_ptrs = kcalloc(n: num_sqbs, size: sizeof(*sq->sqb_ptrs), GFP_KERNEL); |
79 | if (!sq->sqb_ptrs) { |
80 | err = -ENOMEM; |
81 | goto pool_free; |
82 | } |
83 | |
84 | for (ptr = 0; ptr < num_sqbs; ptr++) { |
85 | err = otx2_alloc_rbuf(pfvf, pool, dma: &bufptr); |
86 | if (err) |
87 | goto sqb_free; |
88 | pfvf->hw_ops->aura_freeptr(pfvf, pool_id, bufptr); |
89 | sq->sqb_ptrs[sq->sqb_count++] = (u64)bufptr; |
90 | } |
91 | |
92 | return 0; |
93 | |
94 | sqb_free: |
95 | while (ptr--) { |
96 | if (!sq->sqb_ptrs[ptr]) |
97 | continue; |
98 | iova = sq->sqb_ptrs[ptr]; |
99 | pa = otx2_iova_to_phys(iommu_domain: pfvf->iommu_domain, dma_addr: iova); |
100 | dma_unmap_page_attrs(dev: pfvf->dev, addr: iova, size: hw->sqb_size, |
101 | dir: DMA_FROM_DEVICE, |
102 | DMA_ATTR_SKIP_CPU_SYNC); |
103 | put_page(virt_to_page(phys_to_virt(pa))); |
104 | otx2_aura_allocptr(pfvf, aura: pool_id); |
105 | } |
106 | sq->sqb_count = 0; |
107 | kfree(objp: sq->sqb_ptrs); |
108 | pool_free: |
109 | qmem_free(dev: pfvf->dev, qmem: pool->stack); |
110 | aura_free: |
111 | qmem_free(dev: pfvf->dev, qmem: pool->fc_addr); |
112 | otx2_mbox_reset(mbox: &pfvf->mbox.mbox, devid: 0); |
113 | return err; |
114 | } |
115 | |
116 | static void otx2_qos_sq_free_sqbs(struct otx2_nic *pfvf, int qidx) |
117 | { |
118 | struct otx2_qset *qset = &pfvf->qset; |
119 | struct otx2_hw *hw = &pfvf->hw; |
120 | struct otx2_snd_queue *sq; |
121 | u64 iova, pa; |
122 | int sqb; |
123 | |
124 | sq = &qset->sq[qidx]; |
125 | if (!sq->sqb_ptrs) |
126 | return; |
127 | for (sqb = 0; sqb < sq->sqb_count; sqb++) { |
128 | if (!sq->sqb_ptrs[sqb]) |
129 | continue; |
130 | iova = sq->sqb_ptrs[sqb]; |
131 | pa = otx2_iova_to_phys(iommu_domain: pfvf->iommu_domain, dma_addr: iova); |
132 | dma_unmap_page_attrs(dev: pfvf->dev, addr: iova, size: hw->sqb_size, |
133 | dir: DMA_FROM_DEVICE, |
134 | DMA_ATTR_SKIP_CPU_SYNC); |
135 | put_page(virt_to_page(phys_to_virt(pa))); |
136 | } |
137 | |
138 | sq->sqb_count = 0; |
139 | |
140 | sq = &qset->sq[qidx]; |
141 | qmem_free(dev: pfvf->dev, qmem: sq->sqe); |
142 | qmem_free(dev: pfvf->dev, qmem: sq->tso_hdrs); |
143 | kfree(objp: sq->sg); |
144 | kfree(objp: sq->sqb_ptrs); |
145 | qmem_free(dev: pfvf->dev, qmem: sq->timestamps); |
146 | |
147 | memset((void *)sq, 0, sizeof(*sq)); |
148 | } |
149 | |
150 | /* send queue id */ |
151 | static void otx2_qos_sqb_flush(struct otx2_nic *pfvf, int qidx) |
152 | { |
153 | int sqe_tail, sqe_head; |
154 | u64 incr, *ptr, val; |
155 | |
156 | ptr = (__force u64 *)otx2_get_regaddr(nic: pfvf, NIX_LF_SQ_OP_STATUS); |
157 | incr = (u64)qidx << 32; |
158 | val = otx2_atomic64_add(incr, ptr); |
159 | sqe_head = (val >> 20) & 0x3F; |
160 | sqe_tail = (val >> 28) & 0x3F; |
161 | if (sqe_head != sqe_tail) |
162 | usleep_range(min: 50, max: 60); |
163 | } |
164 | |
165 | static int otx2_qos_ctx_disable(struct otx2_nic *pfvf, u16 qidx, int aura_id) |
166 | { |
167 | struct nix_cn10k_aq_enq_req *cn10k_sq_aq; |
168 | struct npa_aq_enq_req *aura_aq; |
169 | struct npa_aq_enq_req *pool_aq; |
170 | struct nix_aq_enq_req *sq_aq; |
171 | |
172 | if (test_bit(CN10K_LMTST, &pfvf->hw.cap_flag)) { |
173 | cn10k_sq_aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(mbox: &pfvf->mbox); |
174 | if (!cn10k_sq_aq) |
175 | return -ENOMEM; |
176 | cn10k_sq_aq->qidx = qidx; |
177 | cn10k_sq_aq->sq.ena = 0; |
178 | cn10k_sq_aq->sq_mask.ena = 1; |
179 | cn10k_sq_aq->ctype = NIX_AQ_CTYPE_SQ; |
180 | cn10k_sq_aq->op = NIX_AQ_INSTOP_WRITE; |
181 | } else { |
182 | sq_aq = otx2_mbox_alloc_msg_nix_aq_enq(mbox: &pfvf->mbox); |
183 | if (!sq_aq) |
184 | return -ENOMEM; |
185 | sq_aq->qidx = qidx; |
186 | sq_aq->sq.ena = 0; |
187 | sq_aq->sq_mask.ena = 1; |
188 | sq_aq->ctype = NIX_AQ_CTYPE_SQ; |
189 | sq_aq->op = NIX_AQ_INSTOP_WRITE; |
190 | } |
191 | |
192 | aura_aq = otx2_mbox_alloc_msg_npa_aq_enq(mbox: &pfvf->mbox); |
193 | if (!aura_aq) { |
194 | otx2_mbox_reset(mbox: &pfvf->mbox.mbox, devid: 0); |
195 | return -ENOMEM; |
196 | } |
197 | |
198 | aura_aq->aura_id = aura_id; |
199 | aura_aq->aura.ena = 0; |
200 | aura_aq->aura_mask.ena = 1; |
201 | aura_aq->ctype = NPA_AQ_CTYPE_AURA; |
202 | aura_aq->op = NPA_AQ_INSTOP_WRITE; |
203 | |
204 | pool_aq = otx2_mbox_alloc_msg_npa_aq_enq(mbox: &pfvf->mbox); |
205 | if (!pool_aq) { |
206 | otx2_mbox_reset(mbox: &pfvf->mbox.mbox, devid: 0); |
207 | return -ENOMEM; |
208 | } |
209 | |
210 | pool_aq->aura_id = aura_id; |
211 | pool_aq->pool.ena = 0; |
212 | pool_aq->pool_mask.ena = 1; |
213 | |
214 | pool_aq->ctype = NPA_AQ_CTYPE_POOL; |
215 | pool_aq->op = NPA_AQ_INSTOP_WRITE; |
216 | |
217 | return otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
218 | } |
219 | |
220 | int otx2_qos_get_qid(struct otx2_nic *pfvf) |
221 | { |
222 | int qidx; |
223 | |
224 | qidx = find_first_zero_bit(addr: pfvf->qos.qos_sq_bmap, |
225 | size: pfvf->hw.tc_tx_queues); |
226 | |
227 | return qidx == pfvf->hw.tc_tx_queues ? -ENOSPC : qidx; |
228 | } |
229 | |
230 | void otx2_qos_free_qid(struct otx2_nic *pfvf, int qidx) |
231 | { |
232 | clear_bit(nr: qidx, addr: pfvf->qos.qos_sq_bmap); |
233 | } |
234 | |
235 | int otx2_qos_enable_sq(struct otx2_nic *pfvf, int qidx) |
236 | { |
237 | struct otx2_hw *hw = &pfvf->hw; |
238 | int pool_id, sq_idx, err; |
239 | |
240 | if (pfvf->flags & OTX2_FLAG_INTF_DOWN) |
241 | return -EPERM; |
242 | |
243 | sq_idx = hw->non_qos_queues + qidx; |
244 | |
245 | mutex_lock(&pfvf->mbox.lock); |
246 | err = otx2_qos_sq_aura_pool_init(pfvf, qidx: sq_idx); |
247 | if (err) |
248 | goto out; |
249 | |
250 | pool_id = otx2_get_pool_idx(pfvf, type: AURA_NIX_SQ, idx: sq_idx); |
251 | err = otx2_sq_init(pfvf, qidx: sq_idx, sqb_aura: pool_id); |
252 | if (err) |
253 | goto out; |
254 | out: |
255 | mutex_unlock(lock: &pfvf->mbox.lock); |
256 | return err; |
257 | } |
258 | |
259 | void otx2_qos_disable_sq(struct otx2_nic *pfvf, int qidx) |
260 | { |
261 | struct otx2_qset *qset = &pfvf->qset; |
262 | struct otx2_hw *hw = &pfvf->hw; |
263 | struct otx2_snd_queue *sq; |
264 | struct otx2_cq_queue *cq; |
265 | int pool_id, sq_idx; |
266 | |
267 | sq_idx = hw->non_qos_queues + qidx; |
268 | |
269 | /* If the DOWN flag is set SQs are already freed */ |
270 | if (pfvf->flags & OTX2_FLAG_INTF_DOWN) |
271 | return; |
272 | |
273 | sq = &pfvf->qset.sq[sq_idx]; |
274 | if (!sq->sqb_ptrs) |
275 | return; |
276 | |
277 | if (sq_idx < hw->non_qos_queues || |
278 | sq_idx >= otx2_get_total_tx_queues(pfvf)) { |
279 | netdev_err(dev: pfvf->netdev, format: "Send Queue is not a QoS queue\n" ); |
280 | return; |
281 | } |
282 | |
283 | cq = &qset->cq[pfvf->hw.rx_queues + sq_idx]; |
284 | pool_id = otx2_get_pool_idx(pfvf, type: AURA_NIX_SQ, idx: sq_idx); |
285 | |
286 | otx2_qos_sqb_flush(pfvf, qidx: sq_idx); |
287 | otx2_smq_flush(pfvf, smq: otx2_get_smq_idx(pfvf, qidx: sq_idx)); |
288 | otx2_cleanup_tx_cqes(pfvf, cq); |
289 | |
290 | mutex_lock(&pfvf->mbox.lock); |
291 | otx2_qos_ctx_disable(pfvf, qidx: sq_idx, aura_id: pool_id); |
292 | mutex_unlock(lock: &pfvf->mbox.lock); |
293 | |
294 | otx2_qos_sq_free_sqbs(pfvf, qidx: sq_idx); |
295 | otx2_qos_aura_pool_free(pfvf, pool_id); |
296 | } |
297 | |