1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* Marvell RVU Admin Function driver |
3 | * |
4 | * Copyright (C) 2020 Marvell. |
5 | * |
6 | */ |
7 | |
8 | #include <linux/bitfield.h> |
9 | #include <linux/pci.h> |
10 | #include "rvu_struct.h" |
11 | #include "rvu_reg.h" |
12 | #include "mbox.h" |
13 | #include "rvu.h" |
14 | |
15 | /* CPT PF device id */ |
16 | #define PCI_DEVID_OTX2_CPT_PF 0xA0FD |
17 | #define PCI_DEVID_OTX2_CPT10K_PF 0xA0F2 |
18 | |
19 | /* Length of initial context fetch in 128 byte words */ |
20 | #define CPT_CTX_ILEN 1ULL |
21 | |
22 | #define cpt_get_eng_sts(e_min, e_max, rsp, etype) \ |
23 | ({ \ |
24 | u64 free_sts = 0, busy_sts = 0; \ |
25 | typeof(rsp) _rsp = rsp; \ |
26 | u32 e, i; \ |
27 | \ |
28 | for (e = (e_min), i = 0; e < (e_max); e++, i++) { \ |
29 | reg = rvu_read64(rvu, blkaddr, CPT_AF_EXEX_STS(e)); \ |
30 | if (reg & 0x1) \ |
31 | busy_sts |= 1ULL << i; \ |
32 | \ |
33 | if (reg & 0x2) \ |
34 | free_sts |= 1ULL << i; \ |
35 | } \ |
36 | (_rsp)->busy_sts_##etype = busy_sts; \ |
37 | (_rsp)->free_sts_##etype = free_sts; \ |
38 | }) |
39 | |
40 | static irqreturn_t cpt_af_flt_intr_handler(int vec, void *ptr) |
41 | { |
42 | struct rvu_block *block = ptr; |
43 | struct rvu *rvu = block->rvu; |
44 | int blkaddr = block->addr; |
45 | u64 reg, val; |
46 | int i, eng; |
47 | u8 grp; |
48 | |
49 | reg = rvu_read64(rvu, block: blkaddr, CPT_AF_FLTX_INT(vec)); |
50 | dev_err_ratelimited(rvu->dev, "Received CPTAF FLT%d irq : 0x%llx" , vec, reg); |
51 | |
52 | i = -1; |
53 | while ((i = find_next_bit(addr: (unsigned long *)®, size: 64, offset: i + 1)) < 64) { |
54 | switch (vec) { |
55 | case 0: |
56 | eng = i; |
57 | break; |
58 | case 1: |
59 | eng = i + 64; |
60 | break; |
61 | case 2: |
62 | eng = i + 128; |
63 | break; |
64 | } |
65 | grp = rvu_read64(rvu, block: blkaddr, CPT_AF_EXEX_CTL2(eng)) & 0xFF; |
66 | /* Disable and enable the engine which triggers fault */ |
67 | rvu_write64(rvu, block: blkaddr, CPT_AF_EXEX_CTL2(eng), val: 0x0); |
68 | val = rvu_read64(rvu, block: blkaddr, CPT_AF_EXEX_CTL(eng)); |
69 | rvu_write64(rvu, block: blkaddr, CPT_AF_EXEX_CTL(eng), val: val & ~1ULL); |
70 | |
71 | rvu_write64(rvu, block: blkaddr, CPT_AF_EXEX_CTL2(eng), val: grp); |
72 | rvu_write64(rvu, block: blkaddr, CPT_AF_EXEX_CTL(eng), val: val | 1ULL); |
73 | |
74 | spin_lock(lock: &rvu->cpt_intr_lock); |
75 | block->cpt_flt_eng_map[vec] |= BIT_ULL(i); |
76 | val = rvu_read64(rvu, block: blkaddr, CPT_AF_EXEX_STS(eng)); |
77 | val = val & 0x3; |
78 | if (val == 0x1 || val == 0x2) |
79 | block->cpt_rcvrd_eng_map[vec] |= BIT_ULL(i); |
80 | spin_unlock(lock: &rvu->cpt_intr_lock); |
81 | } |
82 | rvu_write64(rvu, block: blkaddr, CPT_AF_FLTX_INT(vec), val: reg); |
83 | |
84 | return IRQ_HANDLED; |
85 | } |
86 | |
87 | static irqreturn_t rvu_cpt_af_flt0_intr_handler(int irq, void *ptr) |
88 | { |
89 | return cpt_af_flt_intr_handler(vec: CPT_AF_INT_VEC_FLT0, ptr); |
90 | } |
91 | |
92 | static irqreturn_t rvu_cpt_af_flt1_intr_handler(int irq, void *ptr) |
93 | { |
94 | return cpt_af_flt_intr_handler(vec: CPT_AF_INT_VEC_FLT1, ptr); |
95 | } |
96 | |
97 | static irqreturn_t rvu_cpt_af_flt2_intr_handler(int irq, void *ptr) |
98 | { |
99 | return cpt_af_flt_intr_handler(vec: CPT_10K_AF_INT_VEC_FLT2, ptr); |
100 | } |
101 | |
102 | static irqreturn_t rvu_cpt_af_rvu_intr_handler(int irq, void *ptr) |
103 | { |
104 | struct rvu_block *block = ptr; |
105 | struct rvu *rvu = block->rvu; |
106 | int blkaddr = block->addr; |
107 | u64 reg; |
108 | |
109 | reg = rvu_read64(rvu, block: blkaddr, CPT_AF_RVU_INT); |
110 | dev_err_ratelimited(rvu->dev, "Received CPTAF RVU irq : 0x%llx" , reg); |
111 | |
112 | rvu_write64(rvu, block: blkaddr, CPT_AF_RVU_INT, val: reg); |
113 | return IRQ_HANDLED; |
114 | } |
115 | |
116 | static irqreturn_t rvu_cpt_af_ras_intr_handler(int irq, void *ptr) |
117 | { |
118 | struct rvu_block *block = ptr; |
119 | struct rvu *rvu = block->rvu; |
120 | int blkaddr = block->addr; |
121 | u64 reg; |
122 | |
123 | reg = rvu_read64(rvu, block: blkaddr, CPT_AF_RAS_INT); |
124 | dev_err_ratelimited(rvu->dev, "Received CPTAF RAS irq : 0x%llx" , reg); |
125 | |
126 | rvu_write64(rvu, block: blkaddr, CPT_AF_RAS_INT, val: reg); |
127 | return IRQ_HANDLED; |
128 | } |
129 | |
130 | static int rvu_cpt_do_register_interrupt(struct rvu_block *block, int irq_offs, |
131 | irq_handler_t handler, |
132 | const char *name) |
133 | { |
134 | struct rvu *rvu = block->rvu; |
135 | int ret; |
136 | |
137 | ret = request_irq(irq: pci_irq_vector(dev: rvu->pdev, nr: irq_offs), handler, flags: 0, |
138 | name, dev: block); |
139 | if (ret) { |
140 | dev_err(rvu->dev, "RVUAF: %s irq registration failed" , name); |
141 | return ret; |
142 | } |
143 | |
144 | WARN_ON(rvu->irq_allocated[irq_offs]); |
145 | rvu->irq_allocated[irq_offs] = true; |
146 | return 0; |
147 | } |
148 | |
149 | static void cpt_10k_unregister_interrupts(struct rvu_block *block, int off) |
150 | { |
151 | struct rvu *rvu = block->rvu; |
152 | int blkaddr = block->addr; |
153 | int i; |
154 | |
155 | /* Disable all CPT AF interrupts */ |
156 | rvu_write64(rvu, block: blkaddr, CPT_AF_FLTX_INT_ENA_W1C(0), val: ~0ULL); |
157 | rvu_write64(rvu, block: blkaddr, CPT_AF_FLTX_INT_ENA_W1C(1), val: ~0ULL); |
158 | rvu_write64(rvu, block: blkaddr, CPT_AF_FLTX_INT_ENA_W1C(2), val: 0xFFFF); |
159 | |
160 | rvu_write64(rvu, block: blkaddr, CPT_AF_RVU_INT_ENA_W1C, val: 0x1); |
161 | rvu_write64(rvu, block: blkaddr, CPT_AF_RAS_INT_ENA_W1C, val: 0x1); |
162 | |
163 | for (i = 0; i < CPT_10K_AF_INT_VEC_CNT; i++) |
164 | if (rvu->irq_allocated[off + i]) { |
165 | free_irq(pci_irq_vector(dev: rvu->pdev, nr: off + i), block); |
166 | rvu->irq_allocated[off + i] = false; |
167 | } |
168 | } |
169 | |
170 | static void cpt_unregister_interrupts(struct rvu *rvu, int blkaddr) |
171 | { |
172 | struct rvu_hwinfo *hw = rvu->hw; |
173 | struct rvu_block *block; |
174 | int i, offs; |
175 | |
176 | if (!is_block_implemented(hw: rvu->hw, blkaddr)) |
177 | return; |
178 | offs = rvu_read64(rvu, block: blkaddr, CPT_PRIV_AF_INT_CFG) & 0x7FF; |
179 | if (!offs) { |
180 | dev_warn(rvu->dev, |
181 | "Failed to get CPT_AF_INT vector offsets\n" ); |
182 | return; |
183 | } |
184 | block = &hw->block[blkaddr]; |
185 | if (!is_rvu_otx2(rvu)) |
186 | return cpt_10k_unregister_interrupts(block, off: offs); |
187 | |
188 | /* Disable all CPT AF interrupts */ |
189 | for (i = 0; i < CPT_AF_INT_VEC_RVU; i++) |
190 | rvu_write64(rvu, block: blkaddr, CPT_AF_FLTX_INT_ENA_W1C(i), val: ~0ULL); |
191 | rvu_write64(rvu, block: blkaddr, CPT_AF_RVU_INT_ENA_W1C, val: 0x1); |
192 | rvu_write64(rvu, block: blkaddr, CPT_AF_RAS_INT_ENA_W1C, val: 0x1); |
193 | |
194 | for (i = 0; i < CPT_AF_INT_VEC_CNT; i++) |
195 | if (rvu->irq_allocated[offs + i]) { |
196 | free_irq(pci_irq_vector(dev: rvu->pdev, nr: offs + i), block); |
197 | rvu->irq_allocated[offs + i] = false; |
198 | } |
199 | } |
200 | |
201 | void rvu_cpt_unregister_interrupts(struct rvu *rvu) |
202 | { |
203 | cpt_unregister_interrupts(rvu, blkaddr: BLKADDR_CPT0); |
204 | cpt_unregister_interrupts(rvu, blkaddr: BLKADDR_CPT1); |
205 | } |
206 | |
207 | static int cpt_10k_register_interrupts(struct rvu_block *block, int off) |
208 | { |
209 | struct rvu *rvu = block->rvu; |
210 | int blkaddr = block->addr; |
211 | irq_handler_t flt_fn; |
212 | int i, ret; |
213 | |
214 | for (i = CPT_10K_AF_INT_VEC_FLT0; i < CPT_10K_AF_INT_VEC_RVU; i++) { |
215 | sprintf(buf: &rvu->irq_name[(off + i) * NAME_SIZE], fmt: "CPTAF FLT%d" , i); |
216 | |
217 | switch (i) { |
218 | case CPT_10K_AF_INT_VEC_FLT0: |
219 | flt_fn = rvu_cpt_af_flt0_intr_handler; |
220 | break; |
221 | case CPT_10K_AF_INT_VEC_FLT1: |
222 | flt_fn = rvu_cpt_af_flt1_intr_handler; |
223 | break; |
224 | case CPT_10K_AF_INT_VEC_FLT2: |
225 | flt_fn = rvu_cpt_af_flt2_intr_handler; |
226 | break; |
227 | } |
228 | ret = rvu_cpt_do_register_interrupt(block, irq_offs: off + i, |
229 | handler: flt_fn, name: &rvu->irq_name[(off + i) * NAME_SIZE]); |
230 | if (ret) |
231 | goto err; |
232 | if (i == CPT_10K_AF_INT_VEC_FLT2) |
233 | rvu_write64(rvu, block: blkaddr, CPT_AF_FLTX_INT_ENA_W1S(i), val: 0xFFFF); |
234 | else |
235 | rvu_write64(rvu, block: blkaddr, CPT_AF_FLTX_INT_ENA_W1S(i), val: ~0ULL); |
236 | } |
237 | |
238 | ret = rvu_cpt_do_register_interrupt(block, irq_offs: off + CPT_10K_AF_INT_VEC_RVU, |
239 | handler: rvu_cpt_af_rvu_intr_handler, |
240 | name: "CPTAF RVU" ); |
241 | if (ret) |
242 | goto err; |
243 | rvu_write64(rvu, block: blkaddr, CPT_AF_RVU_INT_ENA_W1S, val: 0x1); |
244 | |
245 | ret = rvu_cpt_do_register_interrupt(block, irq_offs: off + CPT_10K_AF_INT_VEC_RAS, |
246 | handler: rvu_cpt_af_ras_intr_handler, |
247 | name: "CPTAF RAS" ); |
248 | if (ret) |
249 | goto err; |
250 | rvu_write64(rvu, block: blkaddr, CPT_AF_RAS_INT_ENA_W1S, val: 0x1); |
251 | |
252 | return 0; |
253 | err: |
254 | rvu_cpt_unregister_interrupts(rvu); |
255 | return ret; |
256 | } |
257 | |
258 | static int cpt_register_interrupts(struct rvu *rvu, int blkaddr) |
259 | { |
260 | struct rvu_hwinfo *hw = rvu->hw; |
261 | struct rvu_block *block; |
262 | irq_handler_t flt_fn; |
263 | int i, offs, ret = 0; |
264 | |
265 | if (!is_block_implemented(hw: rvu->hw, blkaddr)) |
266 | return 0; |
267 | |
268 | block = &hw->block[blkaddr]; |
269 | offs = rvu_read64(rvu, block: blkaddr, CPT_PRIV_AF_INT_CFG) & 0x7FF; |
270 | if (!offs) { |
271 | dev_warn(rvu->dev, |
272 | "Failed to get CPT_AF_INT vector offsets\n" ); |
273 | return 0; |
274 | } |
275 | |
276 | if (!is_rvu_otx2(rvu)) |
277 | return cpt_10k_register_interrupts(block, off: offs); |
278 | |
279 | for (i = CPT_AF_INT_VEC_FLT0; i < CPT_AF_INT_VEC_RVU; i++) { |
280 | sprintf(buf: &rvu->irq_name[(offs + i) * NAME_SIZE], fmt: "CPTAF FLT%d" , i); |
281 | switch (i) { |
282 | case CPT_AF_INT_VEC_FLT0: |
283 | flt_fn = rvu_cpt_af_flt0_intr_handler; |
284 | break; |
285 | case CPT_AF_INT_VEC_FLT1: |
286 | flt_fn = rvu_cpt_af_flt1_intr_handler; |
287 | break; |
288 | } |
289 | ret = rvu_cpt_do_register_interrupt(block, irq_offs: offs + i, |
290 | handler: flt_fn, name: &rvu->irq_name[(offs + i) * NAME_SIZE]); |
291 | if (ret) |
292 | goto err; |
293 | rvu_write64(rvu, block: blkaddr, CPT_AF_FLTX_INT_ENA_W1S(i), val: ~0ULL); |
294 | } |
295 | |
296 | ret = rvu_cpt_do_register_interrupt(block, irq_offs: offs + CPT_AF_INT_VEC_RVU, |
297 | handler: rvu_cpt_af_rvu_intr_handler, |
298 | name: "CPTAF RVU" ); |
299 | if (ret) |
300 | goto err; |
301 | rvu_write64(rvu, block: blkaddr, CPT_AF_RVU_INT_ENA_W1S, val: 0x1); |
302 | |
303 | ret = rvu_cpt_do_register_interrupt(block, irq_offs: offs + CPT_AF_INT_VEC_RAS, |
304 | handler: rvu_cpt_af_ras_intr_handler, |
305 | name: "CPTAF RAS" ); |
306 | if (ret) |
307 | goto err; |
308 | rvu_write64(rvu, block: blkaddr, CPT_AF_RAS_INT_ENA_W1S, val: 0x1); |
309 | |
310 | return 0; |
311 | err: |
312 | rvu_cpt_unregister_interrupts(rvu); |
313 | return ret; |
314 | } |
315 | |
316 | int rvu_cpt_register_interrupts(struct rvu *rvu) |
317 | { |
318 | int ret; |
319 | |
320 | ret = cpt_register_interrupts(rvu, blkaddr: BLKADDR_CPT0); |
321 | if (ret) |
322 | return ret; |
323 | |
324 | return cpt_register_interrupts(rvu, blkaddr: BLKADDR_CPT1); |
325 | } |
326 | |
327 | static int get_cpt_pf_num(struct rvu *rvu) |
328 | { |
329 | int i, domain_nr, cpt_pf_num = -1; |
330 | struct pci_dev *pdev; |
331 | |
332 | domain_nr = pci_domain_nr(bus: rvu->pdev->bus); |
333 | for (i = 0; i < rvu->hw->total_pfs; i++) { |
334 | pdev = pci_get_domain_bus_and_slot(domain: domain_nr, bus: i + 1, devfn: 0); |
335 | if (!pdev) |
336 | continue; |
337 | |
338 | if (pdev->device == PCI_DEVID_OTX2_CPT_PF || |
339 | pdev->device == PCI_DEVID_OTX2_CPT10K_PF) { |
340 | cpt_pf_num = i; |
341 | put_device(dev: &pdev->dev); |
342 | break; |
343 | } |
344 | put_device(dev: &pdev->dev); |
345 | } |
346 | return cpt_pf_num; |
347 | } |
348 | |
349 | static bool is_cpt_pf(struct rvu *rvu, u16 pcifunc) |
350 | { |
351 | int cpt_pf_num = rvu->cpt_pf_num; |
352 | |
353 | if (rvu_get_pf(pcifunc) != cpt_pf_num) |
354 | return false; |
355 | if (pcifunc & RVU_PFVF_FUNC_MASK) |
356 | return false; |
357 | |
358 | return true; |
359 | } |
360 | |
361 | static bool is_cpt_vf(struct rvu *rvu, u16 pcifunc) |
362 | { |
363 | int cpt_pf_num = rvu->cpt_pf_num; |
364 | |
365 | if (rvu_get_pf(pcifunc) != cpt_pf_num) |
366 | return false; |
367 | if (!(pcifunc & RVU_PFVF_FUNC_MASK)) |
368 | return false; |
369 | |
370 | return true; |
371 | } |
372 | |
373 | static int validate_and_get_cpt_blkaddr(int req_blkaddr) |
374 | { |
375 | int blkaddr; |
376 | |
377 | blkaddr = req_blkaddr ? req_blkaddr : BLKADDR_CPT0; |
378 | if (blkaddr != BLKADDR_CPT0 && blkaddr != BLKADDR_CPT1) |
379 | return -EINVAL; |
380 | |
381 | return blkaddr; |
382 | } |
383 | |
384 | int rvu_mbox_handler_cpt_lf_alloc(struct rvu *rvu, |
385 | struct cpt_lf_alloc_req_msg *req, |
386 | struct msg_rsp *rsp) |
387 | { |
388 | u16 pcifunc = req->hdr.pcifunc; |
389 | struct rvu_block *block; |
390 | int cptlf, blkaddr; |
391 | int num_lfs, slot; |
392 | u64 val; |
393 | |
394 | blkaddr = validate_and_get_cpt_blkaddr(req_blkaddr: req->blkaddr); |
395 | if (blkaddr < 0) |
396 | return blkaddr; |
397 | |
398 | if (req->eng_grpmsk == 0x0) |
399 | return CPT_AF_ERR_GRP_INVALID; |
400 | |
401 | block = &rvu->hw->block[blkaddr]; |
402 | num_lfs = rvu_get_rsrc_mapcount(pfvf: rvu_get_pfvf(rvu, pcifunc), |
403 | blkaddr: block->addr); |
404 | if (!num_lfs) |
405 | return CPT_AF_ERR_LF_INVALID; |
406 | |
407 | /* Check if requested 'CPTLF <=> NIXLF' mapping is valid */ |
408 | if (req->nix_pf_func) { |
409 | /* If default, use 'this' CPTLF's PFFUNC */ |
410 | if (req->nix_pf_func == RVU_DEFAULT_PF_FUNC) |
411 | req->nix_pf_func = pcifunc; |
412 | if (!is_pffunc_map_valid(rvu, pcifunc: req->nix_pf_func, blktype: BLKTYPE_NIX)) |
413 | return CPT_AF_ERR_NIX_PF_FUNC_INVALID; |
414 | } |
415 | |
416 | /* Check if requested 'CPTLF <=> SSOLF' mapping is valid */ |
417 | if (req->sso_pf_func) { |
418 | /* If default, use 'this' CPTLF's PFFUNC */ |
419 | if (req->sso_pf_func == RVU_DEFAULT_PF_FUNC) |
420 | req->sso_pf_func = pcifunc; |
421 | if (!is_pffunc_map_valid(rvu, pcifunc: req->sso_pf_func, blktype: BLKTYPE_SSO)) |
422 | return CPT_AF_ERR_SSO_PF_FUNC_INVALID; |
423 | } |
424 | |
425 | for (slot = 0; slot < num_lfs; slot++) { |
426 | cptlf = rvu_get_lf(rvu, block, pcifunc, slot); |
427 | if (cptlf < 0) |
428 | return CPT_AF_ERR_LF_INVALID; |
429 | |
430 | /* Set CPT LF group and priority */ |
431 | val = (u64)req->eng_grpmsk << 48 | 1; |
432 | if (!is_rvu_otx2(rvu)) { |
433 | if (req->ctx_ilen_valid) |
434 | val |= (req->ctx_ilen << 17); |
435 | else |
436 | val |= (CPT_CTX_ILEN << 17); |
437 | } |
438 | |
439 | rvu_write64(rvu, block: blkaddr, CPT_AF_LFX_CTL(cptlf), val); |
440 | |
441 | /* Set CPT LF NIX_PF_FUNC and SSO_PF_FUNC. EXE_LDWB is set |
442 | * on reset. |
443 | */ |
444 | val = rvu_read64(rvu, block: blkaddr, CPT_AF_LFX_CTL2(cptlf)); |
445 | val &= ~(GENMASK_ULL(63, 48) | GENMASK_ULL(47, 32)); |
446 | val |= ((u64)req->nix_pf_func << 48 | |
447 | (u64)req->sso_pf_func << 32); |
448 | rvu_write64(rvu, block: blkaddr, CPT_AF_LFX_CTL2(cptlf), val); |
449 | } |
450 | |
451 | return 0; |
452 | } |
453 | |
454 | static int cpt_lf_free(struct rvu *rvu, struct msg_req *req, int blkaddr) |
455 | { |
456 | u16 pcifunc = req->hdr.pcifunc; |
457 | int num_lfs, cptlf, slot, err; |
458 | struct rvu_block *block; |
459 | |
460 | block = &rvu->hw->block[blkaddr]; |
461 | num_lfs = rvu_get_rsrc_mapcount(pfvf: rvu_get_pfvf(rvu, pcifunc), |
462 | blkaddr: block->addr); |
463 | if (!num_lfs) |
464 | return 0; |
465 | |
466 | for (slot = 0; slot < num_lfs; slot++) { |
467 | cptlf = rvu_get_lf(rvu, block, pcifunc, slot); |
468 | if (cptlf < 0) |
469 | return CPT_AF_ERR_LF_INVALID; |
470 | |
471 | /* Perform teardown */ |
472 | rvu_cpt_lf_teardown(rvu, pcifunc, blkaddr, lf: cptlf, slot); |
473 | |
474 | /* Reset LF */ |
475 | err = rvu_lf_reset(rvu, block, lf: cptlf); |
476 | if (err) { |
477 | dev_err(rvu->dev, "Failed to reset blkaddr %d LF%d\n" , |
478 | block->addr, cptlf); |
479 | } |
480 | } |
481 | |
482 | return 0; |
483 | } |
484 | |
485 | int rvu_mbox_handler_cpt_lf_free(struct rvu *rvu, struct msg_req *req, |
486 | struct msg_rsp *rsp) |
487 | { |
488 | int ret; |
489 | |
490 | ret = cpt_lf_free(rvu, req, blkaddr: BLKADDR_CPT0); |
491 | if (ret) |
492 | return ret; |
493 | |
494 | if (is_block_implemented(hw: rvu->hw, blkaddr: BLKADDR_CPT1)) |
495 | ret = cpt_lf_free(rvu, req, blkaddr: BLKADDR_CPT1); |
496 | |
497 | return ret; |
498 | } |
499 | |
500 | static int cpt_inline_ipsec_cfg_inbound(struct rvu *rvu, int blkaddr, u8 cptlf, |
501 | struct cpt_inline_ipsec_cfg_msg *req) |
502 | { |
503 | u16 sso_pf_func = req->sso_pf_func; |
504 | u8 nix_sel; |
505 | u64 val; |
506 | |
507 | val = rvu_read64(rvu, block: blkaddr, CPT_AF_LFX_CTL(cptlf)); |
508 | if (req->enable && (val & BIT_ULL(16))) { |
509 | /* IPSec inline outbound path is already enabled for a given |
510 | * CPT LF, HRM states that inline inbound & outbound paths |
511 | * must not be enabled at the same time for a given CPT LF |
512 | */ |
513 | return CPT_AF_ERR_INLINE_IPSEC_INB_ENA; |
514 | } |
515 | /* Check if requested 'CPTLF <=> SSOLF' mapping is valid */ |
516 | if (sso_pf_func && !is_pffunc_map_valid(rvu, pcifunc: sso_pf_func, blktype: BLKTYPE_SSO)) |
517 | return CPT_AF_ERR_SSO_PF_FUNC_INVALID; |
518 | |
519 | nix_sel = (blkaddr == BLKADDR_CPT1) ? 1 : 0; |
520 | /* Enable CPT LF for IPsec inline inbound operations */ |
521 | if (req->enable) |
522 | val |= BIT_ULL(9); |
523 | else |
524 | val &= ~BIT_ULL(9); |
525 | |
526 | val |= (u64)nix_sel << 8; |
527 | rvu_write64(rvu, block: blkaddr, CPT_AF_LFX_CTL(cptlf), val); |
528 | |
529 | if (sso_pf_func) { |
530 | /* Set SSO_PF_FUNC */ |
531 | val = rvu_read64(rvu, block: blkaddr, CPT_AF_LFX_CTL2(cptlf)); |
532 | val |= (u64)sso_pf_func << 32; |
533 | val |= (u64)req->nix_pf_func << 48; |
534 | rvu_write64(rvu, block: blkaddr, CPT_AF_LFX_CTL2(cptlf), val); |
535 | } |
536 | if (req->sso_pf_func_ovrd) |
537 | /* Set SSO_PF_FUNC_OVRD for inline IPSec */ |
538 | rvu_write64(rvu, block: blkaddr, CPT_AF_ECO, val: 0x1); |
539 | |
540 | /* Configure the X2P Link register with the cpt base channel number and |
541 | * range of channels it should propagate to X2P |
542 | */ |
543 | if (!is_rvu_otx2(rvu)) { |
544 | val = (ilog2(NIX_CHAN_CPT_X2P_MASK + 1) << 16); |
545 | val |= (u64)rvu->hw->cpt_chan_base; |
546 | |
547 | rvu_write64(rvu, block: blkaddr, CPT_AF_X2PX_LINK_CFG(0), val); |
548 | rvu_write64(rvu, block: blkaddr, CPT_AF_X2PX_LINK_CFG(1), val); |
549 | } |
550 | |
551 | return 0; |
552 | } |
553 | |
554 | static int cpt_inline_ipsec_cfg_outbound(struct rvu *rvu, int blkaddr, u8 cptlf, |
555 | struct cpt_inline_ipsec_cfg_msg *req) |
556 | { |
557 | u16 nix_pf_func = req->nix_pf_func; |
558 | int nix_blkaddr; |
559 | u8 nix_sel; |
560 | u64 val; |
561 | |
562 | val = rvu_read64(rvu, block: blkaddr, CPT_AF_LFX_CTL(cptlf)); |
563 | if (req->enable && (val & BIT_ULL(9))) { |
564 | /* IPSec inline inbound path is already enabled for a given |
565 | * CPT LF, HRM states that inline inbound & outbound paths |
566 | * must not be enabled at the same time for a given CPT LF |
567 | */ |
568 | return CPT_AF_ERR_INLINE_IPSEC_OUT_ENA; |
569 | } |
570 | |
571 | /* Check if requested 'CPTLF <=> NIXLF' mapping is valid */ |
572 | if (nix_pf_func && !is_pffunc_map_valid(rvu, pcifunc: nix_pf_func, blktype: BLKTYPE_NIX)) |
573 | return CPT_AF_ERR_NIX_PF_FUNC_INVALID; |
574 | |
575 | /* Enable CPT LF for IPsec inline outbound operations */ |
576 | if (req->enable) |
577 | val |= BIT_ULL(16); |
578 | else |
579 | val &= ~BIT_ULL(16); |
580 | rvu_write64(rvu, block: blkaddr, CPT_AF_LFX_CTL(cptlf), val); |
581 | |
582 | if (nix_pf_func) { |
583 | /* Set NIX_PF_FUNC */ |
584 | val = rvu_read64(rvu, block: blkaddr, CPT_AF_LFX_CTL2(cptlf)); |
585 | val |= (u64)nix_pf_func << 48; |
586 | rvu_write64(rvu, block: blkaddr, CPT_AF_LFX_CTL2(cptlf), val); |
587 | |
588 | nix_blkaddr = rvu_get_blkaddr(rvu, blktype: BLKTYPE_NIX, pcifunc: nix_pf_func); |
589 | nix_sel = (nix_blkaddr == BLKADDR_NIX0) ? 0 : 1; |
590 | |
591 | val = rvu_read64(rvu, block: blkaddr, CPT_AF_LFX_CTL(cptlf)); |
592 | val |= (u64)nix_sel << 8; |
593 | rvu_write64(rvu, block: blkaddr, CPT_AF_LFX_CTL(cptlf), val); |
594 | } |
595 | |
596 | return 0; |
597 | } |
598 | |
599 | int rvu_mbox_handler_cpt_inline_ipsec_cfg(struct rvu *rvu, |
600 | struct cpt_inline_ipsec_cfg_msg *req, |
601 | struct msg_rsp *rsp) |
602 | { |
603 | u16 pcifunc = req->hdr.pcifunc; |
604 | struct rvu_block *block; |
605 | int cptlf, blkaddr, ret; |
606 | u16 actual_slot; |
607 | |
608 | blkaddr = rvu_get_blkaddr_from_slot(rvu, blktype: BLKTYPE_CPT, pcifunc, |
609 | global_slot: req->slot, slot_in_block: &actual_slot); |
610 | if (blkaddr < 0) |
611 | return CPT_AF_ERR_LF_INVALID; |
612 | |
613 | block = &rvu->hw->block[blkaddr]; |
614 | |
615 | cptlf = rvu_get_lf(rvu, block, pcifunc, slot: actual_slot); |
616 | if (cptlf < 0) |
617 | return CPT_AF_ERR_LF_INVALID; |
618 | |
619 | switch (req->dir) { |
620 | case CPT_INLINE_INBOUND: |
621 | ret = cpt_inline_ipsec_cfg_inbound(rvu, blkaddr, cptlf, req); |
622 | break; |
623 | |
624 | case CPT_INLINE_OUTBOUND: |
625 | ret = cpt_inline_ipsec_cfg_outbound(rvu, blkaddr, cptlf, req); |
626 | break; |
627 | |
628 | default: |
629 | return CPT_AF_ERR_PARAM; |
630 | } |
631 | |
632 | return ret; |
633 | } |
634 | |
635 | static bool is_valid_offset(struct rvu *rvu, struct cpt_rd_wr_reg_msg *req) |
636 | { |
637 | u64 offset = req->reg_offset; |
638 | int blkaddr, num_lfs, lf; |
639 | struct rvu_block *block; |
640 | struct rvu_pfvf *pfvf; |
641 | |
642 | blkaddr = validate_and_get_cpt_blkaddr(req_blkaddr: req->blkaddr); |
643 | if (blkaddr < 0) |
644 | return false; |
645 | |
646 | /* Registers that can be accessed from PF/VF */ |
647 | if ((offset & 0xFF000) == CPT_AF_LFX_CTL(0) || |
648 | (offset & 0xFF000) == CPT_AF_LFX_CTL2(0)) { |
649 | if (offset & 7) |
650 | return false; |
651 | |
652 | lf = (offset & 0xFFF) >> 3; |
653 | block = &rvu->hw->block[blkaddr]; |
654 | pfvf = rvu_get_pfvf(rvu, pcifunc: req->hdr.pcifunc); |
655 | num_lfs = rvu_get_rsrc_mapcount(pfvf, blkaddr: block->addr); |
656 | if (lf >= num_lfs) |
657 | /* Slot is not valid for that PF/VF */ |
658 | return false; |
659 | |
660 | /* Translate local LF used by VFs to global CPT LF */ |
661 | lf = rvu_get_lf(rvu, block: &rvu->hw->block[blkaddr], |
662 | pcifunc: req->hdr.pcifunc, slot: lf); |
663 | if (lf < 0) |
664 | return false; |
665 | |
666 | return true; |
667 | } else if (!(req->hdr.pcifunc & RVU_PFVF_FUNC_MASK)) { |
668 | /* Registers that can be accessed from PF */ |
669 | switch (offset) { |
670 | case CPT_AF_DIAG: |
671 | case CPT_AF_CTL: |
672 | case CPT_AF_PF_FUNC: |
673 | case CPT_AF_BLK_RST: |
674 | case CPT_AF_CONSTANTS1: |
675 | case CPT_AF_CTX_FLUSH_TIMER: |
676 | return true; |
677 | } |
678 | |
679 | switch (offset & 0xFF000) { |
680 | case CPT_AF_EXEX_STS(0): |
681 | case CPT_AF_EXEX_CTL(0): |
682 | case CPT_AF_EXEX_CTL2(0): |
683 | case CPT_AF_EXEX_UCODE_BASE(0): |
684 | if (offset & 7) |
685 | return false; |
686 | break; |
687 | default: |
688 | return false; |
689 | } |
690 | return true; |
691 | } |
692 | return false; |
693 | } |
694 | |
695 | int rvu_mbox_handler_cpt_rd_wr_register(struct rvu *rvu, |
696 | struct cpt_rd_wr_reg_msg *req, |
697 | struct cpt_rd_wr_reg_msg *rsp) |
698 | { |
699 | int blkaddr; |
700 | |
701 | blkaddr = validate_and_get_cpt_blkaddr(req_blkaddr: req->blkaddr); |
702 | if (blkaddr < 0) |
703 | return blkaddr; |
704 | |
705 | /* This message is accepted only if sent from CPT PF/VF */ |
706 | if (!is_cpt_pf(rvu, pcifunc: req->hdr.pcifunc) && |
707 | !is_cpt_vf(rvu, pcifunc: req->hdr.pcifunc)) |
708 | return CPT_AF_ERR_ACCESS_DENIED; |
709 | |
710 | rsp->reg_offset = req->reg_offset; |
711 | rsp->ret_val = req->ret_val; |
712 | rsp->is_write = req->is_write; |
713 | |
714 | if (!is_valid_offset(rvu, req)) |
715 | return CPT_AF_ERR_ACCESS_DENIED; |
716 | |
717 | if (req->is_write) |
718 | rvu_write64(rvu, block: blkaddr, offset: req->reg_offset, val: req->val); |
719 | else |
720 | rsp->val = rvu_read64(rvu, block: blkaddr, offset: req->reg_offset); |
721 | |
722 | return 0; |
723 | } |
724 | |
725 | static void get_ctx_pc(struct rvu *rvu, struct cpt_sts_rsp *rsp, int blkaddr) |
726 | { |
727 | if (is_rvu_otx2(rvu)) |
728 | return; |
729 | |
730 | rsp->ctx_mis_pc = rvu_read64(rvu, block: blkaddr, CPT_AF_CTX_MIS_PC); |
731 | rsp->ctx_hit_pc = rvu_read64(rvu, block: blkaddr, CPT_AF_CTX_HIT_PC); |
732 | rsp->ctx_aop_pc = rvu_read64(rvu, block: blkaddr, CPT_AF_CTX_AOP_PC); |
733 | rsp->ctx_aop_lat_pc = rvu_read64(rvu, block: blkaddr, |
734 | CPT_AF_CTX_AOP_LATENCY_PC); |
735 | rsp->ctx_ifetch_pc = rvu_read64(rvu, block: blkaddr, CPT_AF_CTX_IFETCH_PC); |
736 | rsp->ctx_ifetch_lat_pc = rvu_read64(rvu, block: blkaddr, |
737 | CPT_AF_CTX_IFETCH_LATENCY_PC); |
738 | rsp->ctx_ffetch_pc = rvu_read64(rvu, block: blkaddr, CPT_AF_CTX_FFETCH_PC); |
739 | rsp->ctx_ffetch_lat_pc = rvu_read64(rvu, block: blkaddr, |
740 | CPT_AF_CTX_FFETCH_LATENCY_PC); |
741 | rsp->ctx_wback_pc = rvu_read64(rvu, block: blkaddr, CPT_AF_CTX_FFETCH_PC); |
742 | rsp->ctx_wback_lat_pc = rvu_read64(rvu, block: blkaddr, |
743 | CPT_AF_CTX_FFETCH_LATENCY_PC); |
744 | rsp->ctx_psh_pc = rvu_read64(rvu, block: blkaddr, CPT_AF_CTX_FFETCH_PC); |
745 | rsp->ctx_psh_lat_pc = rvu_read64(rvu, block: blkaddr, |
746 | CPT_AF_CTX_FFETCH_LATENCY_PC); |
747 | rsp->ctx_err = rvu_read64(rvu, block: blkaddr, CPT_AF_CTX_ERR); |
748 | rsp->ctx_enc_id = rvu_read64(rvu, block: blkaddr, CPT_AF_CTX_ENC_ID); |
749 | rsp->ctx_flush_timer = rvu_read64(rvu, block: blkaddr, CPT_AF_CTX_FLUSH_TIMER); |
750 | |
751 | rsp->rxc_time = rvu_read64(rvu, block: blkaddr, CPT_AF_RXC_TIME); |
752 | rsp->rxc_time_cfg = rvu_read64(rvu, block: blkaddr, CPT_AF_RXC_TIME_CFG); |
753 | rsp->rxc_active_sts = rvu_read64(rvu, block: blkaddr, CPT_AF_RXC_ACTIVE_STS); |
754 | rsp->rxc_zombie_sts = rvu_read64(rvu, block: blkaddr, CPT_AF_RXC_ZOMBIE_STS); |
755 | rsp->rxc_dfrg = rvu_read64(rvu, block: blkaddr, CPT_AF_RXC_DFRG); |
756 | rsp->x2p_link_cfg0 = rvu_read64(rvu, block: blkaddr, CPT_AF_X2PX_LINK_CFG(0)); |
757 | rsp->x2p_link_cfg1 = rvu_read64(rvu, block: blkaddr, CPT_AF_X2PX_LINK_CFG(1)); |
758 | } |
759 | |
760 | static void get_eng_sts(struct rvu *rvu, struct cpt_sts_rsp *rsp, int blkaddr) |
761 | { |
762 | u16 max_ses, max_ies, max_aes; |
763 | u32 e_min = 0, e_max = 0; |
764 | u64 reg; |
765 | |
766 | reg = rvu_read64(rvu, block: blkaddr, CPT_AF_CONSTANTS1); |
767 | max_ses = reg & 0xffff; |
768 | max_ies = (reg >> 16) & 0xffff; |
769 | max_aes = (reg >> 32) & 0xffff; |
770 | |
771 | /* Get AE status */ |
772 | e_min = max_ses + max_ies; |
773 | e_max = max_ses + max_ies + max_aes; |
774 | cpt_get_eng_sts(e_min, e_max, rsp, ae); |
775 | /* Get SE status */ |
776 | e_min = 0; |
777 | e_max = max_ses; |
778 | cpt_get_eng_sts(e_min, e_max, rsp, se); |
779 | /* Get IE status */ |
780 | e_min = max_ses; |
781 | e_max = max_ses + max_ies; |
782 | cpt_get_eng_sts(e_min, e_max, rsp, ie); |
783 | } |
784 | |
785 | int rvu_mbox_handler_cpt_sts(struct rvu *rvu, struct cpt_sts_req *req, |
786 | struct cpt_sts_rsp *rsp) |
787 | { |
788 | int blkaddr; |
789 | |
790 | blkaddr = validate_and_get_cpt_blkaddr(req_blkaddr: req->blkaddr); |
791 | if (blkaddr < 0) |
792 | return blkaddr; |
793 | |
794 | /* This message is accepted only if sent from CPT PF/VF */ |
795 | if (!is_cpt_pf(rvu, pcifunc: req->hdr.pcifunc) && |
796 | !is_cpt_vf(rvu, pcifunc: req->hdr.pcifunc)) |
797 | return CPT_AF_ERR_ACCESS_DENIED; |
798 | |
799 | get_ctx_pc(rvu, rsp, blkaddr); |
800 | |
801 | /* Get CPT engines status */ |
802 | get_eng_sts(rvu, rsp, blkaddr); |
803 | |
804 | /* Read CPT instruction PC registers */ |
805 | rsp->inst_req_pc = rvu_read64(rvu, block: blkaddr, CPT_AF_INST_REQ_PC); |
806 | rsp->inst_lat_pc = rvu_read64(rvu, block: blkaddr, CPT_AF_INST_LATENCY_PC); |
807 | rsp->rd_req_pc = rvu_read64(rvu, block: blkaddr, CPT_AF_RD_REQ_PC); |
808 | rsp->rd_lat_pc = rvu_read64(rvu, block: blkaddr, CPT_AF_RD_LATENCY_PC); |
809 | rsp->rd_uc_pc = rvu_read64(rvu, block: blkaddr, CPT_AF_RD_UC_PC); |
810 | rsp->active_cycles_pc = rvu_read64(rvu, block: blkaddr, |
811 | CPT_AF_ACTIVE_CYCLES_PC); |
812 | rsp->exe_err_info = rvu_read64(rvu, block: blkaddr, CPT_AF_EXE_ERR_INFO); |
813 | rsp->cptclk_cnt = rvu_read64(rvu, block: blkaddr, CPT_AF_CPTCLK_CNT); |
814 | rsp->diag = rvu_read64(rvu, block: blkaddr, CPT_AF_DIAG); |
815 | |
816 | return 0; |
817 | } |
818 | |
819 | #define RXC_ZOMBIE_THRES GENMASK_ULL(59, 48) |
820 | #define RXC_ZOMBIE_LIMIT GENMASK_ULL(43, 32) |
821 | #define RXC_ACTIVE_THRES GENMASK_ULL(27, 16) |
822 | #define RXC_ACTIVE_LIMIT GENMASK_ULL(11, 0) |
823 | #define RXC_ACTIVE_COUNT GENMASK_ULL(60, 48) |
824 | #define RXC_ZOMBIE_COUNT GENMASK_ULL(60, 48) |
825 | |
826 | static void cpt_rxc_time_cfg(struct rvu *rvu, struct cpt_rxc_time_cfg_req *req, |
827 | int blkaddr, struct cpt_rxc_time_cfg_req *save) |
828 | { |
829 | u64 dfrg_reg; |
830 | |
831 | if (save) { |
832 | /* Save older config */ |
833 | dfrg_reg = rvu_read64(rvu, block: blkaddr, CPT_AF_RXC_DFRG); |
834 | save->zombie_thres = FIELD_GET(RXC_ZOMBIE_THRES, dfrg_reg); |
835 | save->zombie_limit = FIELD_GET(RXC_ZOMBIE_LIMIT, dfrg_reg); |
836 | save->active_thres = FIELD_GET(RXC_ACTIVE_THRES, dfrg_reg); |
837 | save->active_limit = FIELD_GET(RXC_ACTIVE_LIMIT, dfrg_reg); |
838 | |
839 | save->step = rvu_read64(rvu, block: blkaddr, CPT_AF_RXC_TIME_CFG); |
840 | } |
841 | |
842 | dfrg_reg = FIELD_PREP(RXC_ZOMBIE_THRES, req->zombie_thres); |
843 | dfrg_reg |= FIELD_PREP(RXC_ZOMBIE_LIMIT, req->zombie_limit); |
844 | dfrg_reg |= FIELD_PREP(RXC_ACTIVE_THRES, req->active_thres); |
845 | dfrg_reg |= FIELD_PREP(RXC_ACTIVE_LIMIT, req->active_limit); |
846 | |
847 | rvu_write64(rvu, block: blkaddr, CPT_AF_RXC_TIME_CFG, val: req->step); |
848 | rvu_write64(rvu, block: blkaddr, CPT_AF_RXC_DFRG, val: dfrg_reg); |
849 | } |
850 | |
851 | int rvu_mbox_handler_cpt_rxc_time_cfg(struct rvu *rvu, |
852 | struct cpt_rxc_time_cfg_req *req, |
853 | struct msg_rsp *rsp) |
854 | { |
855 | int blkaddr; |
856 | |
857 | blkaddr = validate_and_get_cpt_blkaddr(req_blkaddr: req->blkaddr); |
858 | if (blkaddr < 0) |
859 | return blkaddr; |
860 | |
861 | /* This message is accepted only if sent from CPT PF/VF */ |
862 | if (!is_cpt_pf(rvu, pcifunc: req->hdr.pcifunc) && |
863 | !is_cpt_vf(rvu, pcifunc: req->hdr.pcifunc)) |
864 | return CPT_AF_ERR_ACCESS_DENIED; |
865 | |
866 | cpt_rxc_time_cfg(rvu, req, blkaddr, NULL); |
867 | |
868 | return 0; |
869 | } |
870 | |
871 | int rvu_mbox_handler_cpt_ctx_cache_sync(struct rvu *rvu, struct msg_req *req, |
872 | struct msg_rsp *rsp) |
873 | { |
874 | return rvu_cpt_ctx_flush(rvu, pcifunc: req->hdr.pcifunc); |
875 | } |
876 | |
877 | int rvu_mbox_handler_cpt_lf_reset(struct rvu *rvu, struct cpt_lf_rst_req *req, |
878 | struct msg_rsp *rsp) |
879 | { |
880 | u16 pcifunc = req->hdr.pcifunc; |
881 | struct rvu_block *block; |
882 | int cptlf, blkaddr, ret; |
883 | u16 actual_slot; |
884 | u64 ctl, ctl2; |
885 | |
886 | blkaddr = rvu_get_blkaddr_from_slot(rvu, blktype: BLKTYPE_CPT, pcifunc, |
887 | global_slot: req->slot, slot_in_block: &actual_slot); |
888 | if (blkaddr < 0) |
889 | return CPT_AF_ERR_LF_INVALID; |
890 | |
891 | block = &rvu->hw->block[blkaddr]; |
892 | |
893 | cptlf = rvu_get_lf(rvu, block, pcifunc, slot: actual_slot); |
894 | if (cptlf < 0) |
895 | return CPT_AF_ERR_LF_INVALID; |
896 | ctl = rvu_read64(rvu, block: blkaddr, CPT_AF_LFX_CTL(cptlf)); |
897 | ctl2 = rvu_read64(rvu, block: blkaddr, CPT_AF_LFX_CTL2(cptlf)); |
898 | |
899 | ret = rvu_lf_reset(rvu, block, lf: cptlf); |
900 | if (ret) |
901 | dev_err(rvu->dev, "Failed to reset blkaddr %d LF%d\n" , |
902 | block->addr, cptlf); |
903 | |
904 | rvu_write64(rvu, block: blkaddr, CPT_AF_LFX_CTL(cptlf), val: ctl); |
905 | rvu_write64(rvu, block: blkaddr, CPT_AF_LFX_CTL2(cptlf), val: ctl2); |
906 | |
907 | return 0; |
908 | } |
909 | |
910 | int rvu_mbox_handler_cpt_flt_eng_info(struct rvu *rvu, struct cpt_flt_eng_info_req *req, |
911 | struct cpt_flt_eng_info_rsp *rsp) |
912 | { |
913 | struct rvu_block *block; |
914 | unsigned long flags; |
915 | int blkaddr, vec; |
916 | |
917 | blkaddr = validate_and_get_cpt_blkaddr(req_blkaddr: req->blkaddr); |
918 | if (blkaddr < 0) |
919 | return blkaddr; |
920 | |
921 | block = &rvu->hw->block[blkaddr]; |
922 | for (vec = 0; vec < CPT_10K_AF_INT_VEC_RVU; vec++) { |
923 | spin_lock_irqsave(&rvu->cpt_intr_lock, flags); |
924 | rsp->flt_eng_map[vec] = block->cpt_flt_eng_map[vec]; |
925 | rsp->rcvrd_eng_map[vec] = block->cpt_rcvrd_eng_map[vec]; |
926 | if (req->reset) { |
927 | block->cpt_flt_eng_map[vec] = 0x0; |
928 | block->cpt_rcvrd_eng_map[vec] = 0x0; |
929 | } |
930 | spin_unlock_irqrestore(lock: &rvu->cpt_intr_lock, flags); |
931 | } |
932 | return 0; |
933 | } |
934 | |
935 | static void cpt_rxc_teardown(struct rvu *rvu, int blkaddr) |
936 | { |
937 | struct cpt_rxc_time_cfg_req req, prev; |
938 | int timeout = 2000; |
939 | u64 reg; |
940 | |
941 | if (is_rvu_otx2(rvu)) |
942 | return; |
943 | |
944 | /* Set time limit to minimum values, so that rxc entries will be |
945 | * flushed out quickly. |
946 | */ |
947 | req.step = 1; |
948 | req.zombie_thres = 1; |
949 | req.zombie_limit = 1; |
950 | req.active_thres = 1; |
951 | req.active_limit = 1; |
952 | |
953 | cpt_rxc_time_cfg(rvu, req: &req, blkaddr, save: &prev); |
954 | |
955 | do { |
956 | reg = rvu_read64(rvu, block: blkaddr, CPT_AF_RXC_ACTIVE_STS); |
957 | udelay(1); |
958 | if (FIELD_GET(RXC_ACTIVE_COUNT, reg)) |
959 | timeout--; |
960 | else |
961 | break; |
962 | } while (timeout); |
963 | |
964 | if (timeout == 0) |
965 | dev_warn(rvu->dev, "Poll for RXC active count hits hard loop counter\n" ); |
966 | |
967 | timeout = 2000; |
968 | do { |
969 | reg = rvu_read64(rvu, block: blkaddr, CPT_AF_RXC_ZOMBIE_STS); |
970 | udelay(1); |
971 | if (FIELD_GET(RXC_ZOMBIE_COUNT, reg)) |
972 | timeout--; |
973 | else |
974 | break; |
975 | } while (timeout); |
976 | |
977 | if (timeout == 0) |
978 | dev_warn(rvu->dev, "Poll for RXC zombie count hits hard loop counter\n" ); |
979 | |
980 | /* Restore config */ |
981 | cpt_rxc_time_cfg(rvu, req: &prev, blkaddr, NULL); |
982 | } |
983 | |
984 | #define INFLIGHT GENMASK_ULL(8, 0) |
985 | #define GRB_CNT GENMASK_ULL(39, 32) |
986 | #define GWB_CNT GENMASK_ULL(47, 40) |
987 | #define XQ_XOR GENMASK_ULL(63, 63) |
988 | #define DQPTR GENMASK_ULL(19, 0) |
989 | #define NQPTR GENMASK_ULL(51, 32) |
990 | |
991 | static void cpt_lf_disable_iqueue(struct rvu *rvu, int blkaddr, int slot) |
992 | { |
993 | int timeout = 1000000; |
994 | u64 inprog, inst_ptr; |
995 | u64 qsize, pending; |
996 | int i = 0; |
997 | |
998 | /* Disable instructions enqueuing */ |
999 | rvu_write64(rvu, block: blkaddr, CPT_AF_BAR2_ALIASX(slot, CPT_LF_CTL), val: 0x0); |
1000 | |
1001 | inprog = rvu_read64(rvu, block: blkaddr, |
1002 | CPT_AF_BAR2_ALIASX(slot, CPT_LF_INPROG)); |
1003 | inprog |= BIT_ULL(16); |
1004 | rvu_write64(rvu, block: blkaddr, |
1005 | CPT_AF_BAR2_ALIASX(slot, CPT_LF_INPROG), val: inprog); |
1006 | |
1007 | qsize = rvu_read64(rvu, block: blkaddr, |
1008 | CPT_AF_BAR2_ALIASX(slot, CPT_LF_Q_SIZE)) & 0x7FFF; |
1009 | do { |
1010 | inst_ptr = rvu_read64(rvu, block: blkaddr, |
1011 | CPT_AF_BAR2_ALIASX(slot, CPT_LF_Q_INST_PTR)); |
1012 | pending = (FIELD_GET(XQ_XOR, inst_ptr) * qsize * 40) + |
1013 | FIELD_GET(NQPTR, inst_ptr) - |
1014 | FIELD_GET(DQPTR, inst_ptr); |
1015 | udelay(1); |
1016 | timeout--; |
1017 | } while ((pending != 0) && (timeout != 0)); |
1018 | |
1019 | if (timeout == 0) |
1020 | dev_warn(rvu->dev, "TIMEOUT: CPT poll on pending instructions\n" ); |
1021 | |
1022 | timeout = 1000000; |
1023 | /* Wait for CPT queue to become execution-quiescent */ |
1024 | do { |
1025 | inprog = rvu_read64(rvu, block: blkaddr, |
1026 | CPT_AF_BAR2_ALIASX(slot, CPT_LF_INPROG)); |
1027 | |
1028 | if ((FIELD_GET(INFLIGHT, inprog) == 0) && |
1029 | (FIELD_GET(GRB_CNT, inprog) == 0)) { |
1030 | i++; |
1031 | } else { |
1032 | i = 0; |
1033 | timeout--; |
1034 | } |
1035 | } while ((timeout != 0) && (i < 10)); |
1036 | |
1037 | if (timeout == 0) |
1038 | dev_warn(rvu->dev, "TIMEOUT: CPT poll on inflight count\n" ); |
1039 | /* Wait for 2 us to flush all queue writes to memory */ |
1040 | udelay(2); |
1041 | } |
1042 | |
1043 | int rvu_cpt_lf_teardown(struct rvu *rvu, u16 pcifunc, int blkaddr, int lf, int slot) |
1044 | { |
1045 | u64 reg; |
1046 | |
1047 | if (is_cpt_pf(rvu, pcifunc) || is_cpt_vf(rvu, pcifunc)) |
1048 | cpt_rxc_teardown(rvu, blkaddr); |
1049 | |
1050 | mutex_lock(&rvu->alias_lock); |
1051 | /* Enable BAR2 ALIAS for this pcifunc. */ |
1052 | reg = BIT_ULL(16) | pcifunc; |
1053 | rvu_bar2_sel_write64(rvu, block: blkaddr, CPT_AF_BAR2_SEL, val: reg); |
1054 | |
1055 | cpt_lf_disable_iqueue(rvu, blkaddr, slot); |
1056 | |
1057 | rvu_bar2_sel_write64(rvu, block: blkaddr, CPT_AF_BAR2_SEL, val: 0); |
1058 | mutex_unlock(lock: &rvu->alias_lock); |
1059 | |
1060 | return 0; |
1061 | } |
1062 | |
1063 | #define CPT_RES_LEN 16 |
1064 | #define CPT_SE_IE_EGRP 1ULL |
1065 | |
1066 | static int cpt_inline_inb_lf_cmd_send(struct rvu *rvu, int blkaddr, |
1067 | int nix_blkaddr) |
1068 | { |
1069 | int cpt_pf_num = rvu->cpt_pf_num; |
1070 | struct cpt_inst_lmtst_req *req; |
1071 | dma_addr_t res_daddr; |
1072 | int timeout = 3000; |
1073 | u8 cpt_idx; |
1074 | u64 *inst; |
1075 | u16 *res; |
1076 | int rc; |
1077 | |
1078 | res = kzalloc(CPT_RES_LEN, GFP_KERNEL); |
1079 | if (!res) |
1080 | return -ENOMEM; |
1081 | |
1082 | res_daddr = dma_map_single(rvu->dev, res, CPT_RES_LEN, |
1083 | DMA_BIDIRECTIONAL); |
1084 | if (dma_mapping_error(dev: rvu->dev, dma_addr: res_daddr)) { |
1085 | dev_err(rvu->dev, "DMA mapping failed for CPT result\n" ); |
1086 | rc = -EFAULT; |
1087 | goto res_free; |
1088 | } |
1089 | *res = 0xFFFF; |
1090 | |
1091 | /* Send mbox message to CPT PF */ |
1092 | req = (struct cpt_inst_lmtst_req *) |
1093 | otx2_mbox_alloc_msg_rsp(mbox: &rvu->afpf_wq_info.mbox_up, |
1094 | devid: cpt_pf_num, size: sizeof(*req), |
1095 | size_rsp: sizeof(struct msg_rsp)); |
1096 | if (!req) { |
1097 | rc = -ENOMEM; |
1098 | goto res_daddr_unmap; |
1099 | } |
1100 | req->hdr.sig = OTX2_MBOX_REQ_SIG; |
1101 | req->hdr.id = MBOX_MSG_CPT_INST_LMTST; |
1102 | |
1103 | inst = req->inst; |
1104 | /* Prepare CPT_INST_S */ |
1105 | inst[0] = 0; |
1106 | inst[1] = res_daddr; |
1107 | /* AF PF FUNC */ |
1108 | inst[2] = 0; |
1109 | /* Set QORD */ |
1110 | inst[3] = 1; |
1111 | inst[4] = 0; |
1112 | inst[5] = 0; |
1113 | inst[6] = 0; |
1114 | /* Set EGRP */ |
1115 | inst[7] = CPT_SE_IE_EGRP << 61; |
1116 | |
1117 | /* Subtract 1 from the NIX-CPT credit count to preserve |
1118 | * credit counts. |
1119 | */ |
1120 | cpt_idx = (blkaddr == BLKADDR_CPT0) ? 0 : 1; |
1121 | rvu_write64(rvu, block: nix_blkaddr, NIX_AF_RX_CPTX_CREDIT(cpt_idx), |
1122 | BIT_ULL(22) - 1); |
1123 | |
1124 | otx2_mbox_msg_send(mbox: &rvu->afpf_wq_info.mbox_up, devid: cpt_pf_num); |
1125 | rc = otx2_mbox_wait_for_rsp(mbox: &rvu->afpf_wq_info.mbox_up, devid: cpt_pf_num); |
1126 | if (rc) |
1127 | dev_warn(rvu->dev, "notification to pf %d failed\n" , |
1128 | cpt_pf_num); |
1129 | /* Wait for CPT instruction to be completed */ |
1130 | do { |
1131 | mdelay(1); |
1132 | if (*res == 0xFFFF) |
1133 | timeout--; |
1134 | else |
1135 | break; |
1136 | } while (timeout); |
1137 | |
1138 | if (timeout == 0) |
1139 | dev_warn(rvu->dev, "Poll for result hits hard loop counter\n" ); |
1140 | |
1141 | res_daddr_unmap: |
1142 | dma_unmap_single(rvu->dev, res_daddr, CPT_RES_LEN, DMA_BIDIRECTIONAL); |
1143 | res_free: |
1144 | kfree(objp: res); |
1145 | |
1146 | return 0; |
1147 | } |
1148 | |
1149 | #define CTX_CAM_PF_FUNC GENMASK_ULL(61, 46) |
1150 | #define CTX_CAM_CPTR GENMASK_ULL(45, 0) |
1151 | |
1152 | int rvu_cpt_ctx_flush(struct rvu *rvu, u16 pcifunc) |
1153 | { |
1154 | int nix_blkaddr, blkaddr; |
1155 | u16 max_ctx_entries, i; |
1156 | int slot = 0, num_lfs; |
1157 | u64 reg, cam_data; |
1158 | int rc; |
1159 | |
1160 | nix_blkaddr = rvu_get_blkaddr(rvu, blktype: BLKTYPE_NIX, pcifunc); |
1161 | if (nix_blkaddr < 0) |
1162 | return -EINVAL; |
1163 | |
1164 | if (is_rvu_otx2(rvu)) |
1165 | return 0; |
1166 | |
1167 | blkaddr = (nix_blkaddr == BLKADDR_NIX1) ? BLKADDR_CPT1 : BLKADDR_CPT0; |
1168 | |
1169 | /* Submit CPT_INST_S to track when all packets have been |
1170 | * flushed through for the NIX PF FUNC in inline inbound case. |
1171 | */ |
1172 | rc = cpt_inline_inb_lf_cmd_send(rvu, blkaddr, nix_blkaddr); |
1173 | if (rc) |
1174 | return rc; |
1175 | |
1176 | /* Wait for rxc entries to be flushed out */ |
1177 | cpt_rxc_teardown(rvu, blkaddr); |
1178 | |
1179 | reg = rvu_read64(rvu, block: blkaddr, CPT_AF_CONSTANTS0); |
1180 | max_ctx_entries = (reg >> 48) & 0xFFF; |
1181 | |
1182 | mutex_lock(&rvu->rsrc_lock); |
1183 | |
1184 | num_lfs = rvu_get_rsrc_mapcount(pfvf: rvu_get_pfvf(rvu, pcifunc), |
1185 | blkaddr); |
1186 | if (num_lfs == 0) { |
1187 | dev_warn(rvu->dev, "CPT LF is not configured\n" ); |
1188 | goto unlock; |
1189 | } |
1190 | |
1191 | /* Enable BAR2 ALIAS for this pcifunc. */ |
1192 | reg = BIT_ULL(16) | pcifunc; |
1193 | rvu_bar2_sel_write64(rvu, block: blkaddr, CPT_AF_BAR2_SEL, val: reg); |
1194 | |
1195 | for (i = 0; i < max_ctx_entries; i++) { |
1196 | cam_data = rvu_read64(rvu, block: blkaddr, CPT_AF_CTX_CAM_DATA(i)); |
1197 | |
1198 | if ((FIELD_GET(CTX_CAM_PF_FUNC, cam_data) == pcifunc) && |
1199 | FIELD_GET(CTX_CAM_CPTR, cam_data)) { |
1200 | reg = BIT_ULL(46) | FIELD_GET(CTX_CAM_CPTR, cam_data); |
1201 | rvu_write64(rvu, block: blkaddr, |
1202 | CPT_AF_BAR2_ALIASX(slot, CPT_LF_CTX_FLUSH), |
1203 | val: reg); |
1204 | } |
1205 | } |
1206 | rvu_bar2_sel_write64(rvu, block: blkaddr, CPT_AF_BAR2_SEL, val: 0); |
1207 | |
1208 | unlock: |
1209 | mutex_unlock(lock: &rvu->rsrc_lock); |
1210 | |
1211 | return 0; |
1212 | } |
1213 | |
1214 | int rvu_cpt_init(struct rvu *rvu) |
1215 | { |
1216 | /* Retrieve CPT PF number */ |
1217 | rvu->cpt_pf_num = get_cpt_pf_num(rvu); |
1218 | spin_lock_init(&rvu->cpt_intr_lock); |
1219 | |
1220 | return 0; |
1221 | } |
1222 | |