1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Marvell RVU Ethernet driver |
3 | * |
4 | * Copyright (C) 2021 Marvell. |
5 | * |
6 | */ |
7 | |
8 | #include "cn10k.h" |
9 | #include "otx2_reg.h" |
10 | #include "otx2_struct.h" |
11 | |
12 | static struct dev_hw_ops otx2_hw_ops = { |
13 | .sq_aq_init = otx2_sq_aq_init, |
14 | .sqe_flush = otx2_sqe_flush, |
15 | .aura_freeptr = otx2_aura_freeptr, |
16 | .refill_pool_ptrs = otx2_refill_pool_ptrs, |
17 | }; |
18 | |
19 | static struct dev_hw_ops cn10k_hw_ops = { |
20 | .sq_aq_init = cn10k_sq_aq_init, |
21 | .sqe_flush = cn10k_sqe_flush, |
22 | .aura_freeptr = cn10k_aura_freeptr, |
23 | .refill_pool_ptrs = cn10k_refill_pool_ptrs, |
24 | }; |
25 | |
26 | int cn10k_lmtst_init(struct otx2_nic *pfvf) |
27 | { |
28 | |
29 | struct lmtst_tbl_setup_req *req; |
30 | struct otx2_lmt_info *lmt_info; |
31 | int err, cpu; |
32 | |
33 | if (!test_bit(CN10K_LMTST, &pfvf->hw.cap_flag)) { |
34 | pfvf->hw_ops = &otx2_hw_ops; |
35 | return 0; |
36 | } |
37 | |
38 | pfvf->hw_ops = &cn10k_hw_ops; |
39 | /* Total LMTLINES = num_online_cpus() * 32 (For Burst flush).*/ |
40 | pfvf->tot_lmt_lines = (num_online_cpus() * LMT_BURST_SIZE); |
41 | pfvf->hw.lmt_info = alloc_percpu(struct otx2_lmt_info); |
42 | |
43 | mutex_lock(&pfvf->mbox.lock); |
44 | req = otx2_mbox_alloc_msg_lmtst_tbl_setup(mbox: &pfvf->mbox); |
45 | if (!req) { |
46 | mutex_unlock(lock: &pfvf->mbox.lock); |
47 | return -ENOMEM; |
48 | } |
49 | |
50 | req->use_local_lmt_region = true; |
51 | |
52 | err = qmem_alloc(dev: pfvf->dev, q: &pfvf->dync_lmt, qsize: pfvf->tot_lmt_lines, |
53 | LMT_LINE_SIZE); |
54 | if (err) { |
55 | mutex_unlock(lock: &pfvf->mbox.lock); |
56 | return err; |
57 | } |
58 | pfvf->hw.lmt_base = (u64 *)pfvf->dync_lmt->base; |
59 | req->lmt_iova = (u64)pfvf->dync_lmt->iova; |
60 | |
61 | err = otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
62 | mutex_unlock(lock: &pfvf->mbox.lock); |
63 | |
64 | for_each_possible_cpu(cpu) { |
65 | lmt_info = per_cpu_ptr(pfvf->hw.lmt_info, cpu); |
66 | lmt_info->lmt_addr = ((u64)pfvf->hw.lmt_base + |
67 | (cpu * LMT_BURST_SIZE * LMT_LINE_SIZE)); |
68 | lmt_info->lmt_id = cpu * LMT_BURST_SIZE; |
69 | } |
70 | |
71 | return 0; |
72 | } |
73 | EXPORT_SYMBOL(cn10k_lmtst_init); |
74 | |
75 | int cn10k_sq_aq_init(void *dev, u16 qidx, u16 sqb_aura) |
76 | { |
77 | struct nix_cn10k_aq_enq_req *aq; |
78 | struct otx2_nic *pfvf = dev; |
79 | |
80 | /* Get memory to put this msg */ |
81 | aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(mbox: &pfvf->mbox); |
82 | if (!aq) |
83 | return -ENOMEM; |
84 | |
85 | aq->sq.cq = pfvf->hw.rx_queues + qidx; |
86 | aq->sq.max_sqe_size = NIX_MAXSQESZ_W16; /* 128 byte */ |
87 | aq->sq.cq_ena = 1; |
88 | aq->sq.ena = 1; |
89 | aq->sq.smq = otx2_get_smq_idx(pfvf, qidx); |
90 | aq->sq.smq_rr_weight = mtu_to_dwrr_weight(pfvf, mtu: pfvf->tx_max_pktlen); |
91 | aq->sq.default_chan = pfvf->hw.tx_chan_base; |
92 | aq->sq.sqe_stype = NIX_STYPE_STF; /* Cache SQB */ |
93 | aq->sq.sqb_aura = sqb_aura; |
94 | aq->sq.sq_int_ena = NIX_SQINT_BITS; |
95 | aq->sq.qint_idx = 0; |
96 | /* Due pipelining impact minimum 2000 unused SQ CQE's |
97 | * need to maintain to avoid CQ overflow. |
98 | */ |
99 | aq->sq.cq_limit = ((SEND_CQ_SKID * 256) / (pfvf->qset.sqe_cnt)); |
100 | |
101 | /* Fill AQ info */ |
102 | aq->qidx = qidx; |
103 | aq->ctype = NIX_AQ_CTYPE_SQ; |
104 | aq->op = NIX_AQ_INSTOP_INIT; |
105 | |
106 | return otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
107 | } |
108 | |
109 | #define NPA_MAX_BURST 16 |
110 | int cn10k_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq) |
111 | { |
112 | struct otx2_nic *pfvf = dev; |
113 | int cnt = cq->pool_ptrs; |
114 | u64 ptrs[NPA_MAX_BURST]; |
115 | dma_addr_t bufptr; |
116 | int num_ptrs = 1; |
117 | |
118 | /* Refill pool with new buffers */ |
119 | while (cq->pool_ptrs) { |
120 | if (otx2_alloc_buffer(pfvf, cq, dma: &bufptr)) { |
121 | if (num_ptrs--) |
122 | __cn10k_aura_freeptr(pfvf, aura: cq->cq_idx, ptrs, |
123 | num_ptrs); |
124 | break; |
125 | } |
126 | cq->pool_ptrs--; |
127 | ptrs[num_ptrs] = (u64)bufptr + OTX2_HEAD_ROOM; |
128 | num_ptrs++; |
129 | if (num_ptrs == NPA_MAX_BURST || cq->pool_ptrs == 0) { |
130 | __cn10k_aura_freeptr(pfvf, aura: cq->cq_idx, ptrs, |
131 | num_ptrs); |
132 | num_ptrs = 1; |
133 | } |
134 | } |
135 | return cnt - cq->pool_ptrs; |
136 | } |
137 | |
138 | void cn10k_sqe_flush(void *dev, struct otx2_snd_queue *sq, int size, int qidx) |
139 | { |
140 | struct otx2_lmt_info *lmt_info; |
141 | struct otx2_nic *pfvf = dev; |
142 | u64 val = 0, tar_addr = 0; |
143 | |
144 | lmt_info = per_cpu_ptr(pfvf->hw.lmt_info, smp_processor_id()); |
145 | /* FIXME: val[0:10] LMT_ID. |
146 | * [12:15] no of LMTST - 1 in the burst. |
147 | * [19:63] data size of each LMTST in the burst except first. |
148 | */ |
149 | val = (lmt_info->lmt_id & 0x7FF); |
150 | /* Target address for LMTST flush tells HW how many 128bit |
151 | * words are present. |
152 | * tar_addr[6:4] size of first LMTST - 1 in units of 128b. |
153 | */ |
154 | tar_addr |= sq->io_addr | (((size / 16) - 1) & 0x7) << 4; |
155 | dma_wmb(); |
156 | memcpy((u64 *)lmt_info->lmt_addr, sq->sqe_base, size); |
157 | cn10k_lmt_flush(val, tar_addr); |
158 | |
159 | sq->head++; |
160 | sq->head &= (sq->sqe_cnt - 1); |
161 | } |
162 | |
163 | int cn10k_free_all_ipolicers(struct otx2_nic *pfvf) |
164 | { |
165 | struct nix_bandprof_free_req *req; |
166 | int rc; |
167 | |
168 | if (is_dev_otx2(pdev: pfvf->pdev)) |
169 | return 0; |
170 | |
171 | mutex_lock(&pfvf->mbox.lock); |
172 | |
173 | req = otx2_mbox_alloc_msg_nix_bandprof_free(mbox: &pfvf->mbox); |
174 | if (!req) { |
175 | rc = -ENOMEM; |
176 | goto out; |
177 | } |
178 | |
179 | /* Free all bandwidth profiles allocated */ |
180 | req->free_all = true; |
181 | |
182 | rc = otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
183 | out: |
184 | mutex_unlock(lock: &pfvf->mbox.lock); |
185 | return rc; |
186 | } |
187 | |
188 | int cn10k_alloc_leaf_profile(struct otx2_nic *pfvf, u16 *leaf) |
189 | { |
190 | struct nix_bandprof_alloc_req *req; |
191 | struct nix_bandprof_alloc_rsp *rsp; |
192 | int rc; |
193 | |
194 | req = otx2_mbox_alloc_msg_nix_bandprof_alloc(mbox: &pfvf->mbox); |
195 | if (!req) |
196 | return -ENOMEM; |
197 | |
198 | req->prof_count[BAND_PROF_LEAF_LAYER] = 1; |
199 | |
200 | rc = otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
201 | if (rc) |
202 | goto out; |
203 | |
204 | rsp = (struct nix_bandprof_alloc_rsp *) |
205 | otx2_mbox_get_rsp(mbox: &pfvf->mbox.mbox, devid: 0, msg: &req->hdr); |
206 | if (!rsp->prof_count[BAND_PROF_LEAF_LAYER]) { |
207 | rc = -EIO; |
208 | goto out; |
209 | } |
210 | |
211 | *leaf = rsp->prof_idx[BAND_PROF_LEAF_LAYER][0]; |
212 | out: |
213 | if (rc) { |
214 | dev_warn(pfvf->dev, |
215 | "Failed to allocate ingress bandwidth policer\n" ); |
216 | } |
217 | |
218 | return rc; |
219 | } |
220 | |
221 | int cn10k_alloc_matchall_ipolicer(struct otx2_nic *pfvf) |
222 | { |
223 | struct otx2_hw *hw = &pfvf->hw; |
224 | int ret; |
225 | |
226 | mutex_lock(&pfvf->mbox.lock); |
227 | |
228 | ret = cn10k_alloc_leaf_profile(pfvf, leaf: &hw->matchall_ipolicer); |
229 | |
230 | mutex_unlock(lock: &pfvf->mbox.lock); |
231 | |
232 | return ret; |
233 | } |
234 | |
235 | #define POLICER_TIMESTAMP 1 /* 1 second */ |
236 | #define MAX_RATE_EXP 22 /* Valid rate exponent range: 0 - 22 */ |
237 | |
238 | static void cn10k_get_ingress_burst_cfg(u32 burst, u32 *burst_exp, |
239 | u32 *burst_mantissa) |
240 | { |
241 | int tmp; |
242 | |
243 | /* Burst is calculated as |
244 | * (1+[BURST_MANTISSA]/256)*2^[BURST_EXPONENT] |
245 | * This is the upper limit on number tokens (bytes) that |
246 | * can be accumulated in the bucket. |
247 | */ |
248 | *burst_exp = ilog2(burst); |
249 | if (burst < 256) { |
250 | /* No float: can't express mantissa in this case */ |
251 | *burst_mantissa = 0; |
252 | return; |
253 | } |
254 | |
255 | if (*burst_exp > MAX_RATE_EXP) |
256 | *burst_exp = MAX_RATE_EXP; |
257 | |
258 | /* Calculate mantissa |
259 | * Find remaining bytes 'burst - 2^burst_exp' |
260 | * mantissa = (remaining bytes) / 2^ (burst_exp - 8) |
261 | */ |
262 | tmp = burst - rounddown_pow_of_two(burst); |
263 | *burst_mantissa = tmp / (1UL << (*burst_exp - 8)); |
264 | } |
265 | |
266 | static void cn10k_get_ingress_rate_cfg(u64 rate, u32 *rate_exp, |
267 | u32 *rate_mantissa, u32 *rdiv) |
268 | { |
269 | u32 div = 0; |
270 | u32 exp = 0; |
271 | u64 tmp; |
272 | |
273 | /* Figure out mantissa, exponent and divider from given max pkt rate |
274 | * |
275 | * To achieve desired rate HW adds |
276 | * (1+[RATE_MANTISSA]/256)*2^[RATE_EXPONENT] tokens (bytes) at every |
277 | * policer timeunit * 2^rdiv ie 2 * 2^rdiv usecs, to the token bucket. |
278 | * Here policer timeunit is 2 usecs and rate is in bits per sec. |
279 | * Since floating point cannot be used below algorithm uses 1000000 |
280 | * scale factor to support rates upto 100Gbps. |
281 | */ |
282 | tmp = rate * 32 * 2; |
283 | if (tmp < 256000000) { |
284 | while (tmp < 256000000) { |
285 | tmp = tmp * 2; |
286 | div++; |
287 | } |
288 | } else { |
289 | for (exp = 0; tmp >= 512000000 && exp <= MAX_RATE_EXP; exp++) |
290 | tmp = tmp / 2; |
291 | |
292 | if (exp > MAX_RATE_EXP) |
293 | exp = MAX_RATE_EXP; |
294 | } |
295 | |
296 | *rate_mantissa = (tmp - 256000000) / 1000000; |
297 | *rate_exp = exp; |
298 | *rdiv = div; |
299 | } |
300 | |
301 | int cn10k_map_unmap_rq_policer(struct otx2_nic *pfvf, int rq_idx, |
302 | u16 policer, bool map) |
303 | { |
304 | struct nix_cn10k_aq_enq_req *aq; |
305 | |
306 | aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(mbox: &pfvf->mbox); |
307 | if (!aq) |
308 | return -ENOMEM; |
309 | |
310 | /* Enable policing and set the bandwidth profile (policer) index */ |
311 | if (map) |
312 | aq->rq.policer_ena = 1; |
313 | else |
314 | aq->rq.policer_ena = 0; |
315 | aq->rq_mask.policer_ena = 1; |
316 | |
317 | aq->rq.band_prof_id = policer; |
318 | aq->rq_mask.band_prof_id = GENMASK(9, 0); |
319 | |
320 | /* Fill AQ info */ |
321 | aq->qidx = rq_idx; |
322 | aq->ctype = NIX_AQ_CTYPE_RQ; |
323 | aq->op = NIX_AQ_INSTOP_WRITE; |
324 | |
325 | return otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
326 | } |
327 | |
328 | int cn10k_free_leaf_profile(struct otx2_nic *pfvf, u16 leaf) |
329 | { |
330 | struct nix_bandprof_free_req *req; |
331 | |
332 | req = otx2_mbox_alloc_msg_nix_bandprof_free(mbox: &pfvf->mbox); |
333 | if (!req) |
334 | return -ENOMEM; |
335 | |
336 | req->prof_count[BAND_PROF_LEAF_LAYER] = 1; |
337 | req->prof_idx[BAND_PROF_LEAF_LAYER][0] = leaf; |
338 | |
339 | return otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
340 | } |
341 | |
342 | int cn10k_free_matchall_ipolicer(struct otx2_nic *pfvf) |
343 | { |
344 | struct otx2_hw *hw = &pfvf->hw; |
345 | int qidx, rc; |
346 | |
347 | mutex_lock(&pfvf->mbox.lock); |
348 | |
349 | /* Remove RQ's policer mapping */ |
350 | for (qidx = 0; qidx < hw->rx_queues; qidx++) |
351 | cn10k_map_unmap_rq_policer(pfvf, rq_idx: qidx, |
352 | policer: hw->matchall_ipolicer, map: false); |
353 | |
354 | rc = cn10k_free_leaf_profile(pfvf, leaf: hw->matchall_ipolicer); |
355 | |
356 | mutex_unlock(lock: &pfvf->mbox.lock); |
357 | return rc; |
358 | } |
359 | |
360 | int cn10k_set_ipolicer_rate(struct otx2_nic *pfvf, u16 profile, |
361 | u32 burst, u64 rate, bool pps) |
362 | { |
363 | struct nix_cn10k_aq_enq_req *aq; |
364 | u32 burst_exp, burst_mantissa; |
365 | u32 rate_exp, rate_mantissa; |
366 | u32 rdiv; |
367 | |
368 | /* Get exponent and mantissa values for the desired rate */ |
369 | cn10k_get_ingress_burst_cfg(burst, burst_exp: &burst_exp, burst_mantissa: &burst_mantissa); |
370 | cn10k_get_ingress_rate_cfg(rate, rate_exp: &rate_exp, rate_mantissa: &rate_mantissa, rdiv: &rdiv); |
371 | |
372 | /* Init bandwidth profile */ |
373 | aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(mbox: &pfvf->mbox); |
374 | if (!aq) |
375 | return -ENOMEM; |
376 | |
377 | /* Set initial color mode to blind */ |
378 | aq->prof.icolor = 0x03; |
379 | aq->prof_mask.icolor = 0x03; |
380 | |
381 | /* Set rate and burst values */ |
382 | aq->prof.cir_exponent = rate_exp; |
383 | aq->prof_mask.cir_exponent = 0x1F; |
384 | |
385 | aq->prof.cir_mantissa = rate_mantissa; |
386 | aq->prof_mask.cir_mantissa = 0xFF; |
387 | |
388 | aq->prof.cbs_exponent = burst_exp; |
389 | aq->prof_mask.cbs_exponent = 0x1F; |
390 | |
391 | aq->prof.cbs_mantissa = burst_mantissa; |
392 | aq->prof_mask.cbs_mantissa = 0xFF; |
393 | |
394 | aq->prof.rdiv = rdiv; |
395 | aq->prof_mask.rdiv = 0xF; |
396 | |
397 | if (pps) { |
398 | /* The amount of decremented tokens is calculated according to |
399 | * the following equation: |
400 | * max([ LMODE ? 0 : (packet_length - LXPTR)] + |
401 | * ([ADJUST_MANTISSA]/256 - 1) * 2^[ADJUST_EXPONENT], |
402 | * 1/256) |
403 | * if LMODE is 1 then rate limiting will be based on |
404 | * PPS otherwise bps. |
405 | * The aim of the ADJUST value is to specify a token cost per |
406 | * packet in contrary to the packet length that specifies a |
407 | * cost per byte. To rate limit based on PPS adjust mantissa |
408 | * is set as 384 and exponent as 1 so that number of tokens |
409 | * decremented becomes 1 i.e, 1 token per packeet. |
410 | */ |
411 | aq->prof.adjust_exponent = 1; |
412 | aq->prof_mask.adjust_exponent = 0x1F; |
413 | |
414 | aq->prof.adjust_mantissa = 384; |
415 | aq->prof_mask.adjust_mantissa = 0x1FF; |
416 | |
417 | aq->prof.lmode = 0x1; |
418 | aq->prof_mask.lmode = 0x1; |
419 | } |
420 | |
421 | /* Two rate three color marker |
422 | * With PEIR/EIR set to zero, color will be either green or red |
423 | */ |
424 | aq->prof.meter_algo = 2; |
425 | aq->prof_mask.meter_algo = 0x3; |
426 | |
427 | aq->prof.rc_action = NIX_RX_BAND_PROF_ACTIONRESULT_DROP; |
428 | aq->prof_mask.rc_action = 0x3; |
429 | |
430 | aq->prof.yc_action = NIX_RX_BAND_PROF_ACTIONRESULT_PASS; |
431 | aq->prof_mask.yc_action = 0x3; |
432 | |
433 | aq->prof.gc_action = NIX_RX_BAND_PROF_ACTIONRESULT_PASS; |
434 | aq->prof_mask.gc_action = 0x3; |
435 | |
436 | /* Setting exponent value as 24 and mantissa as 0 configures |
437 | * the bucket with zero values making bucket unused. Peak |
438 | * information rate and Excess information rate buckets are |
439 | * unused here. |
440 | */ |
441 | aq->prof.peir_exponent = 24; |
442 | aq->prof_mask.peir_exponent = 0x1F; |
443 | |
444 | aq->prof.peir_mantissa = 0; |
445 | aq->prof_mask.peir_mantissa = 0xFF; |
446 | |
447 | aq->prof.pebs_exponent = 24; |
448 | aq->prof_mask.pebs_exponent = 0x1F; |
449 | |
450 | aq->prof.pebs_mantissa = 0; |
451 | aq->prof_mask.pebs_mantissa = 0xFF; |
452 | |
453 | aq->prof.hl_en = 0; |
454 | aq->prof_mask.hl_en = 1; |
455 | |
456 | /* Fill AQ info */ |
457 | aq->qidx = profile; |
458 | aq->ctype = NIX_AQ_CTYPE_BANDPROF; |
459 | aq->op = NIX_AQ_INSTOP_WRITE; |
460 | |
461 | return otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
462 | } |
463 | |
464 | int cn10k_set_matchall_ipolicer_rate(struct otx2_nic *pfvf, |
465 | u32 burst, u64 rate) |
466 | { |
467 | struct otx2_hw *hw = &pfvf->hw; |
468 | int qidx, rc; |
469 | |
470 | mutex_lock(&pfvf->mbox.lock); |
471 | |
472 | rc = cn10k_set_ipolicer_rate(pfvf, profile: hw->matchall_ipolicer, burst, |
473 | rate, pps: false); |
474 | if (rc) |
475 | goto out; |
476 | |
477 | for (qidx = 0; qidx < hw->rx_queues; qidx++) { |
478 | rc = cn10k_map_unmap_rq_policer(pfvf, rq_idx: qidx, |
479 | policer: hw->matchall_ipolicer, map: true); |
480 | if (rc) |
481 | break; |
482 | } |
483 | |
484 | out: |
485 | mutex_unlock(lock: &pfvf->mbox.lock); |
486 | return rc; |
487 | } |
488 | |