1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Marvell RVU Admin Function driver |
3 | * |
4 | * Copyright (C) 2018 Marvell. |
5 | * |
6 | */ |
7 | #include <linux/bitfield.h> |
8 | #include <linux/module.h> |
9 | #include <linux/pci.h> |
10 | |
11 | #include "rvu_struct.h" |
12 | #include "rvu_reg.h" |
13 | #include "rvu.h" |
14 | |
15 | static int npa_aq_enqueue_wait(struct rvu *rvu, struct rvu_block *block, |
16 | struct npa_aq_inst_s *inst) |
17 | { |
18 | struct admin_queue *aq = block->aq; |
19 | struct npa_aq_res_s *result; |
20 | int timeout = 1000; |
21 | u64 reg, head; |
22 | |
23 | result = (struct npa_aq_res_s *)aq->res->base; |
24 | |
25 | /* Get current head pointer where to append this instruction */ |
26 | reg = rvu_read64(rvu, block: block->addr, NPA_AF_AQ_STATUS); |
27 | head = (reg >> 4) & AQ_PTR_MASK; |
28 | |
29 | memcpy((void *)(aq->inst->base + (head * aq->inst->entry_sz)), |
30 | (void *)inst, aq->inst->entry_sz); |
31 | memset(result, 0, sizeof(*result)); |
32 | /* sync into memory */ |
33 | wmb(); |
34 | |
35 | /* Ring the doorbell and wait for result */ |
36 | rvu_write64(rvu, block: block->addr, NPA_AF_AQ_DOOR, val: 1); |
37 | while (result->compcode == NPA_AQ_COMP_NOTDONE) { |
38 | cpu_relax(); |
39 | udelay(1); |
40 | timeout--; |
41 | if (!timeout) |
42 | return -EBUSY; |
43 | } |
44 | |
45 | if (result->compcode != NPA_AQ_COMP_GOOD) { |
46 | /* TODO: Replace this with some error code */ |
47 | if (result->compcode == NPA_AQ_COMP_CTX_FAULT || |
48 | result->compcode == NPA_AQ_COMP_LOCKERR || |
49 | result->compcode == NPA_AQ_COMP_CTX_POISON) { |
50 | if (rvu_ndc_fix_locked_cacheline(rvu, blkaddr: BLKADDR_NDC_NPA0)) |
51 | dev_err(rvu->dev, |
52 | "%s: Not able to unlock cachelines\n" , __func__); |
53 | } |
54 | |
55 | return -EBUSY; |
56 | } |
57 | |
58 | return 0; |
59 | } |
60 | |
61 | int rvu_npa_aq_enq_inst(struct rvu *rvu, struct npa_aq_enq_req *req, |
62 | struct npa_aq_enq_rsp *rsp) |
63 | { |
64 | struct rvu_hwinfo *hw = rvu->hw; |
65 | u16 pcifunc = req->hdr.pcifunc; |
66 | int blkaddr, npalf, rc = 0; |
67 | struct npa_aq_inst_s inst; |
68 | struct rvu_block *block; |
69 | struct admin_queue *aq; |
70 | struct rvu_pfvf *pfvf; |
71 | void *ctx, *mask; |
72 | bool ena; |
73 | |
74 | pfvf = rvu_get_pfvf(rvu, pcifunc); |
75 | if (!pfvf->aura_ctx || req->aura_id >= pfvf->aura_ctx->qsize) |
76 | return NPA_AF_ERR_AQ_ENQUEUE; |
77 | |
78 | blkaddr = rvu_get_blkaddr(rvu, blktype: BLKTYPE_NPA, pcifunc); |
79 | if (!pfvf->npalf || blkaddr < 0) |
80 | return NPA_AF_ERR_AF_LF_INVALID; |
81 | |
82 | block = &hw->block[blkaddr]; |
83 | aq = block->aq; |
84 | if (!aq) { |
85 | dev_warn(rvu->dev, "%s: NPA AQ not initialized\n" , __func__); |
86 | return NPA_AF_ERR_AQ_ENQUEUE; |
87 | } |
88 | |
89 | npalf = rvu_get_lf(rvu, block, pcifunc, slot: 0); |
90 | if (npalf < 0) |
91 | return NPA_AF_ERR_AF_LF_INVALID; |
92 | |
93 | memset(&inst, 0, sizeof(struct npa_aq_inst_s)); |
94 | inst.cindex = req->aura_id; |
95 | inst.lf = npalf; |
96 | inst.ctype = req->ctype; |
97 | inst.op = req->op; |
98 | /* Currently we are not supporting enqueuing multiple instructions, |
99 | * so always choose first entry in result memory. |
100 | */ |
101 | inst.res_addr = (u64)aq->res->iova; |
102 | |
103 | /* Hardware uses same aq->res->base for updating result of |
104 | * previous instruction hence wait here till it is done. |
105 | */ |
106 | spin_lock(lock: &aq->lock); |
107 | |
108 | /* Clean result + context memory */ |
109 | memset(aq->res->base, 0, aq->res->entry_sz); |
110 | /* Context needs to be written at RES_ADDR + 128 */ |
111 | ctx = aq->res->base + 128; |
112 | /* Mask needs to be written at RES_ADDR + 256 */ |
113 | mask = aq->res->base + 256; |
114 | |
115 | switch (req->op) { |
116 | case NPA_AQ_INSTOP_WRITE: |
117 | /* Copy context and write mask */ |
118 | if (req->ctype == NPA_AQ_CTYPE_AURA) { |
119 | memcpy(mask, &req->aura_mask, |
120 | sizeof(struct npa_aura_s)); |
121 | memcpy(ctx, &req->aura, sizeof(struct npa_aura_s)); |
122 | } else { |
123 | memcpy(mask, &req->pool_mask, |
124 | sizeof(struct npa_pool_s)); |
125 | memcpy(ctx, &req->pool, sizeof(struct npa_pool_s)); |
126 | } |
127 | break; |
128 | case NPA_AQ_INSTOP_INIT: |
129 | if (req->ctype == NPA_AQ_CTYPE_AURA) { |
130 | if (req->aura.pool_addr >= pfvf->pool_ctx->qsize) { |
131 | rc = NPA_AF_ERR_AQ_FULL; |
132 | break; |
133 | } |
134 | /* Set pool's context address */ |
135 | req->aura.pool_addr = pfvf->pool_ctx->iova + |
136 | (req->aura.pool_addr * pfvf->pool_ctx->entry_sz); |
137 | memcpy(ctx, &req->aura, sizeof(struct npa_aura_s)); |
138 | } else { /* POOL's context */ |
139 | memcpy(ctx, &req->pool, sizeof(struct npa_pool_s)); |
140 | } |
141 | break; |
142 | case NPA_AQ_INSTOP_NOP: |
143 | case NPA_AQ_INSTOP_READ: |
144 | case NPA_AQ_INSTOP_LOCK: |
145 | case NPA_AQ_INSTOP_UNLOCK: |
146 | break; |
147 | default: |
148 | rc = NPA_AF_ERR_AQ_FULL; |
149 | break; |
150 | } |
151 | |
152 | if (rc) { |
153 | spin_unlock(lock: &aq->lock); |
154 | return rc; |
155 | } |
156 | |
157 | /* Submit the instruction to AQ */ |
158 | rc = npa_aq_enqueue_wait(rvu, block, inst: &inst); |
159 | if (rc) { |
160 | spin_unlock(lock: &aq->lock); |
161 | return rc; |
162 | } |
163 | |
164 | /* Set aura bitmap if aura hw context is enabled */ |
165 | if (req->ctype == NPA_AQ_CTYPE_AURA) { |
166 | if (req->op == NPA_AQ_INSTOP_INIT && req->aura.ena) |
167 | __set_bit(req->aura_id, pfvf->aura_bmap); |
168 | if (req->op == NPA_AQ_INSTOP_WRITE) { |
169 | ena = (req->aura.ena & req->aura_mask.ena) | |
170 | (test_bit(req->aura_id, pfvf->aura_bmap) & |
171 | ~req->aura_mask.ena); |
172 | if (ena) |
173 | __set_bit(req->aura_id, pfvf->aura_bmap); |
174 | else |
175 | __clear_bit(req->aura_id, pfvf->aura_bmap); |
176 | } |
177 | } |
178 | |
179 | /* Set pool bitmap if pool hw context is enabled */ |
180 | if (req->ctype == NPA_AQ_CTYPE_POOL) { |
181 | if (req->op == NPA_AQ_INSTOP_INIT && req->pool.ena) |
182 | __set_bit(req->aura_id, pfvf->pool_bmap); |
183 | if (req->op == NPA_AQ_INSTOP_WRITE) { |
184 | ena = (req->pool.ena & req->pool_mask.ena) | |
185 | (test_bit(req->aura_id, pfvf->pool_bmap) & |
186 | ~req->pool_mask.ena); |
187 | if (ena) |
188 | __set_bit(req->aura_id, pfvf->pool_bmap); |
189 | else |
190 | __clear_bit(req->aura_id, pfvf->pool_bmap); |
191 | } |
192 | } |
193 | spin_unlock(lock: &aq->lock); |
194 | |
195 | if (rsp) { |
196 | /* Copy read context into mailbox */ |
197 | if (req->op == NPA_AQ_INSTOP_READ) { |
198 | if (req->ctype == NPA_AQ_CTYPE_AURA) |
199 | memcpy(&rsp->aura, ctx, |
200 | sizeof(struct npa_aura_s)); |
201 | else |
202 | memcpy(&rsp->pool, ctx, |
203 | sizeof(struct npa_pool_s)); |
204 | } |
205 | } |
206 | |
207 | return 0; |
208 | } |
209 | |
210 | static int npa_lf_hwctx_disable(struct rvu *rvu, struct hwctx_disable_req *req) |
211 | { |
212 | struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc: req->hdr.pcifunc); |
213 | struct npa_aq_enq_req aq_req; |
214 | unsigned long *bmap; |
215 | int id, cnt = 0; |
216 | int err = 0, rc; |
217 | |
218 | if (!pfvf->pool_ctx || !pfvf->aura_ctx) |
219 | return NPA_AF_ERR_AQ_ENQUEUE; |
220 | |
221 | memset(&aq_req, 0, sizeof(struct npa_aq_enq_req)); |
222 | aq_req.hdr.pcifunc = req->hdr.pcifunc; |
223 | |
224 | if (req->ctype == NPA_AQ_CTYPE_POOL) { |
225 | aq_req.pool.ena = 0; |
226 | aq_req.pool_mask.ena = 1; |
227 | cnt = pfvf->pool_ctx->qsize; |
228 | bmap = pfvf->pool_bmap; |
229 | } else if (req->ctype == NPA_AQ_CTYPE_AURA) { |
230 | aq_req.aura.ena = 0; |
231 | aq_req.aura_mask.ena = 1; |
232 | aq_req.aura.bp_ena = 0; |
233 | aq_req.aura_mask.bp_ena = 1; |
234 | cnt = pfvf->aura_ctx->qsize; |
235 | bmap = pfvf->aura_bmap; |
236 | } |
237 | |
238 | aq_req.ctype = req->ctype; |
239 | aq_req.op = NPA_AQ_INSTOP_WRITE; |
240 | |
241 | for (id = 0; id < cnt; id++) { |
242 | if (!test_bit(id, bmap)) |
243 | continue; |
244 | aq_req.aura_id = id; |
245 | rc = rvu_npa_aq_enq_inst(rvu, req: &aq_req, NULL); |
246 | if (rc) { |
247 | err = rc; |
248 | dev_err(rvu->dev, "Failed to disable %s:%d context\n" , |
249 | (req->ctype == NPA_AQ_CTYPE_AURA) ? |
250 | "Aura" : "Pool" , id); |
251 | } |
252 | } |
253 | |
254 | return err; |
255 | } |
256 | |
257 | #ifdef CONFIG_NDC_DIS_DYNAMIC_CACHING |
258 | static int npa_lf_hwctx_lockdown(struct rvu *rvu, struct npa_aq_enq_req *req) |
259 | { |
260 | struct npa_aq_enq_req lock_ctx_req; |
261 | int err; |
262 | |
263 | if (req->op != NPA_AQ_INSTOP_INIT) |
264 | return 0; |
265 | |
266 | memset(&lock_ctx_req, 0, sizeof(struct npa_aq_enq_req)); |
267 | lock_ctx_req.hdr.pcifunc = req->hdr.pcifunc; |
268 | lock_ctx_req.ctype = req->ctype; |
269 | lock_ctx_req.op = NPA_AQ_INSTOP_LOCK; |
270 | lock_ctx_req.aura_id = req->aura_id; |
271 | err = rvu_npa_aq_enq_inst(rvu, req: &lock_ctx_req, NULL); |
272 | if (err) |
273 | dev_err(rvu->dev, |
274 | "PFUNC 0x%x: Failed to lock NPA context %s:%d\n" , |
275 | req->hdr.pcifunc, |
276 | (req->ctype == NPA_AQ_CTYPE_AURA) ? |
277 | "Aura" : "Pool" , req->aura_id); |
278 | return err; |
279 | } |
280 | |
281 | int rvu_mbox_handler_npa_aq_enq(struct rvu *rvu, |
282 | struct npa_aq_enq_req *req, |
283 | struct npa_aq_enq_rsp *rsp) |
284 | { |
285 | int err; |
286 | |
287 | err = rvu_npa_aq_enq_inst(rvu, req, rsp); |
288 | if (!err) |
289 | err = npa_lf_hwctx_lockdown(rvu, req); |
290 | return err; |
291 | } |
292 | #else |
293 | |
294 | int rvu_mbox_handler_npa_aq_enq(struct rvu *rvu, |
295 | struct npa_aq_enq_req *req, |
296 | struct npa_aq_enq_rsp *rsp) |
297 | { |
298 | return rvu_npa_aq_enq_inst(rvu, req, rsp); |
299 | } |
300 | #endif |
301 | |
302 | int rvu_mbox_handler_npa_hwctx_disable(struct rvu *rvu, |
303 | struct hwctx_disable_req *req, |
304 | struct msg_rsp *rsp) |
305 | { |
306 | return npa_lf_hwctx_disable(rvu, req); |
307 | } |
308 | |
309 | static void npa_ctx_free(struct rvu *rvu, struct rvu_pfvf *pfvf) |
310 | { |
311 | kfree(objp: pfvf->aura_bmap); |
312 | pfvf->aura_bmap = NULL; |
313 | |
314 | qmem_free(dev: rvu->dev, qmem: pfvf->aura_ctx); |
315 | pfvf->aura_ctx = NULL; |
316 | |
317 | kfree(objp: pfvf->pool_bmap); |
318 | pfvf->pool_bmap = NULL; |
319 | |
320 | qmem_free(dev: rvu->dev, qmem: pfvf->pool_ctx); |
321 | pfvf->pool_ctx = NULL; |
322 | |
323 | qmem_free(dev: rvu->dev, qmem: pfvf->npa_qints_ctx); |
324 | pfvf->npa_qints_ctx = NULL; |
325 | } |
326 | |
327 | int rvu_mbox_handler_npa_lf_alloc(struct rvu *rvu, |
328 | struct npa_lf_alloc_req *req, |
329 | struct npa_lf_alloc_rsp *rsp) |
330 | { |
331 | int npalf, qints, hwctx_size, err, rc = 0; |
332 | struct rvu_hwinfo *hw = rvu->hw; |
333 | u16 pcifunc = req->hdr.pcifunc; |
334 | struct rvu_block *block; |
335 | struct rvu_pfvf *pfvf; |
336 | u64 cfg, ctx_cfg; |
337 | int blkaddr; |
338 | |
339 | if (req->aura_sz > NPA_AURA_SZ_MAX || |
340 | req->aura_sz == NPA_AURA_SZ_0 || !req->nr_pools) |
341 | return NPA_AF_ERR_PARAM; |
342 | |
343 | if (req->way_mask) |
344 | req->way_mask &= 0xFFFF; |
345 | |
346 | pfvf = rvu_get_pfvf(rvu, pcifunc); |
347 | blkaddr = rvu_get_blkaddr(rvu, blktype: BLKTYPE_NPA, pcifunc); |
348 | if (!pfvf->npalf || blkaddr < 0) |
349 | return NPA_AF_ERR_AF_LF_INVALID; |
350 | |
351 | block = &hw->block[blkaddr]; |
352 | npalf = rvu_get_lf(rvu, block, pcifunc, slot: 0); |
353 | if (npalf < 0) |
354 | return NPA_AF_ERR_AF_LF_INVALID; |
355 | |
356 | /* Reset this NPA LF */ |
357 | err = rvu_lf_reset(rvu, block, lf: npalf); |
358 | if (err) { |
359 | dev_err(rvu->dev, "Failed to reset NPALF%d\n" , npalf); |
360 | return NPA_AF_ERR_LF_RESET; |
361 | } |
362 | |
363 | ctx_cfg = rvu_read64(rvu, block: blkaddr, NPA_AF_CONST1); |
364 | |
365 | /* Alloc memory for aura HW contexts */ |
366 | hwctx_size = 1UL << (ctx_cfg & 0xF); |
367 | err = qmem_alloc(dev: rvu->dev, q: &pfvf->aura_ctx, |
368 | NPA_AURA_COUNT(req->aura_sz), entry_sz: hwctx_size); |
369 | if (err) |
370 | goto free_mem; |
371 | |
372 | pfvf->aura_bmap = kcalloc(NPA_AURA_COUNT(req->aura_sz), size: sizeof(long), |
373 | GFP_KERNEL); |
374 | if (!pfvf->aura_bmap) |
375 | goto free_mem; |
376 | |
377 | /* Alloc memory for pool HW contexts */ |
378 | hwctx_size = 1UL << ((ctx_cfg >> 4) & 0xF); |
379 | err = qmem_alloc(dev: rvu->dev, q: &pfvf->pool_ctx, qsize: req->nr_pools, entry_sz: hwctx_size); |
380 | if (err) |
381 | goto free_mem; |
382 | |
383 | pfvf->pool_bmap = kcalloc(NPA_AURA_COUNT(req->aura_sz), size: sizeof(long), |
384 | GFP_KERNEL); |
385 | if (!pfvf->pool_bmap) |
386 | goto free_mem; |
387 | |
388 | /* Get no of queue interrupts supported */ |
389 | cfg = rvu_read64(rvu, block: blkaddr, NPA_AF_CONST); |
390 | qints = (cfg >> 28) & 0xFFF; |
391 | |
392 | /* Alloc memory for Qints HW contexts */ |
393 | hwctx_size = 1UL << ((ctx_cfg >> 8) & 0xF); |
394 | err = qmem_alloc(dev: rvu->dev, q: &pfvf->npa_qints_ctx, qsize: qints, entry_sz: hwctx_size); |
395 | if (err) |
396 | goto free_mem; |
397 | |
398 | cfg = rvu_read64(rvu, block: blkaddr, NPA_AF_LFX_AURAS_CFG(npalf)); |
399 | /* Clear way partition mask and set aura offset to '0' */ |
400 | cfg &= ~(BIT_ULL(34) - 1); |
401 | /* Set aura size & enable caching of contexts */ |
402 | cfg |= (req->aura_sz << 16) | BIT_ULL(34) | req->way_mask; |
403 | |
404 | rvu_write64(rvu, block: blkaddr, NPA_AF_LFX_AURAS_CFG(npalf), val: cfg); |
405 | |
406 | /* Configure aura HW context's base */ |
407 | rvu_write64(rvu, block: blkaddr, NPA_AF_LFX_LOC_AURAS_BASE(npalf), |
408 | val: (u64)pfvf->aura_ctx->iova); |
409 | |
410 | /* Enable caching of qints hw context */ |
411 | rvu_write64(rvu, block: blkaddr, NPA_AF_LFX_QINTS_CFG(npalf), |
412 | BIT_ULL(36) | req->way_mask << 20); |
413 | rvu_write64(rvu, block: blkaddr, NPA_AF_LFX_QINTS_BASE(npalf), |
414 | val: (u64)pfvf->npa_qints_ctx->iova); |
415 | |
416 | goto exit; |
417 | |
418 | free_mem: |
419 | npa_ctx_free(rvu, pfvf); |
420 | rc = -ENOMEM; |
421 | |
422 | exit: |
423 | /* set stack page info */ |
424 | cfg = rvu_read64(rvu, block: blkaddr, NPA_AF_CONST); |
425 | rsp->stack_pg_ptrs = (cfg >> 8) & 0xFF; |
426 | rsp->stack_pg_bytes = cfg & 0xFF; |
427 | rsp->qints = (cfg >> 28) & 0xFFF; |
428 | if (!is_rvu_otx2(rvu)) { |
429 | cfg = rvu_read64(rvu, block: block->addr, NPA_AF_BATCH_CTL); |
430 | rsp->cache_lines = (cfg >> 1) & 0x3F; |
431 | } |
432 | return rc; |
433 | } |
434 | |
435 | int rvu_mbox_handler_npa_lf_free(struct rvu *rvu, struct msg_req *req, |
436 | struct msg_rsp *rsp) |
437 | { |
438 | struct rvu_hwinfo *hw = rvu->hw; |
439 | u16 pcifunc = req->hdr.pcifunc; |
440 | struct rvu_block *block; |
441 | struct rvu_pfvf *pfvf; |
442 | int npalf, err; |
443 | int blkaddr; |
444 | |
445 | pfvf = rvu_get_pfvf(rvu, pcifunc); |
446 | blkaddr = rvu_get_blkaddr(rvu, blktype: BLKTYPE_NPA, pcifunc); |
447 | if (!pfvf->npalf || blkaddr < 0) |
448 | return NPA_AF_ERR_AF_LF_INVALID; |
449 | |
450 | block = &hw->block[blkaddr]; |
451 | npalf = rvu_get_lf(rvu, block, pcifunc, slot: 0); |
452 | if (npalf < 0) |
453 | return NPA_AF_ERR_AF_LF_INVALID; |
454 | |
455 | /* Reset this NPA LF */ |
456 | err = rvu_lf_reset(rvu, block, lf: npalf); |
457 | if (err) { |
458 | dev_err(rvu->dev, "Failed to reset NPALF%d\n" , npalf); |
459 | return NPA_AF_ERR_LF_RESET; |
460 | } |
461 | |
462 | npa_ctx_free(rvu, pfvf); |
463 | |
464 | return 0; |
465 | } |
466 | |
467 | static int npa_aq_init(struct rvu *rvu, struct rvu_block *block) |
468 | { |
469 | u64 cfg; |
470 | int err; |
471 | |
472 | /* Set admin queue endianness */ |
473 | cfg = rvu_read64(rvu, block: block->addr, NPA_AF_GEN_CFG); |
474 | #ifdef __BIG_ENDIAN |
475 | cfg |= BIT_ULL(1); |
476 | rvu_write64(rvu, block->addr, NPA_AF_GEN_CFG, cfg); |
477 | #else |
478 | cfg &= ~BIT_ULL(1); |
479 | rvu_write64(rvu, block: block->addr, NPA_AF_GEN_CFG, val: cfg); |
480 | #endif |
481 | |
482 | /* Do not bypass NDC cache */ |
483 | cfg = rvu_read64(rvu, block: block->addr, NPA_AF_NDC_CFG); |
484 | cfg &= ~0x03DULL; |
485 | #ifdef CONFIG_NDC_DIS_DYNAMIC_CACHING |
486 | /* Disable caching of stack pages */ |
487 | cfg |= 0x10ULL; |
488 | #endif |
489 | rvu_write64(rvu, block: block->addr, NPA_AF_NDC_CFG, val: cfg); |
490 | |
491 | /* For CN10K NPA BATCH DMA set 35 cache lines */ |
492 | if (!is_rvu_otx2(rvu)) { |
493 | cfg = rvu_read64(rvu, block: block->addr, NPA_AF_BATCH_CTL); |
494 | cfg &= ~0x7EULL; |
495 | cfg |= BIT_ULL(6) | BIT_ULL(2) | BIT_ULL(1); |
496 | rvu_write64(rvu, block: block->addr, NPA_AF_BATCH_CTL, val: cfg); |
497 | } |
498 | /* Result structure can be followed by Aura/Pool context at |
499 | * RES + 128bytes and a write mask at RES + 256 bytes, depending on |
500 | * operation type. Alloc sufficient result memory for all operations. |
501 | */ |
502 | err = rvu_aq_alloc(rvu, ad_queue: &block->aq, |
503 | Q_COUNT(AQ_SIZE), inst_size: sizeof(struct npa_aq_inst_s), |
504 | ALIGN(sizeof(struct npa_aq_res_s), 128) + 256); |
505 | if (err) |
506 | return err; |
507 | |
508 | rvu_write64(rvu, block: block->addr, NPA_AF_AQ_CFG, AQ_SIZE); |
509 | rvu_write64(rvu, block: block->addr, |
510 | NPA_AF_AQ_BASE, val: (u64)block->aq->inst->iova); |
511 | return 0; |
512 | } |
513 | |
514 | int rvu_npa_init(struct rvu *rvu) |
515 | { |
516 | struct rvu_hwinfo *hw = rvu->hw; |
517 | int blkaddr; |
518 | |
519 | blkaddr = rvu_get_blkaddr(rvu, blktype: BLKTYPE_NPA, pcifunc: 0); |
520 | if (blkaddr < 0) |
521 | return 0; |
522 | |
523 | /* Initialize admin queue */ |
524 | return npa_aq_init(rvu, block: &hw->block[blkaddr]); |
525 | } |
526 | |
527 | void rvu_npa_freemem(struct rvu *rvu) |
528 | { |
529 | struct rvu_hwinfo *hw = rvu->hw; |
530 | struct rvu_block *block; |
531 | int blkaddr; |
532 | |
533 | blkaddr = rvu_get_blkaddr(rvu, blktype: BLKTYPE_NPA, pcifunc: 0); |
534 | if (blkaddr < 0) |
535 | return; |
536 | |
537 | block = &hw->block[blkaddr]; |
538 | rvu_aq_free(rvu, aq: block->aq); |
539 | } |
540 | |
541 | void rvu_npa_lf_teardown(struct rvu *rvu, u16 pcifunc, int npalf) |
542 | { |
543 | struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); |
544 | struct hwctx_disable_req ctx_req; |
545 | |
546 | /* Disable all pools */ |
547 | ctx_req.hdr.pcifunc = pcifunc; |
548 | ctx_req.ctype = NPA_AQ_CTYPE_POOL; |
549 | npa_lf_hwctx_disable(rvu, req: &ctx_req); |
550 | |
551 | /* Disable all auras */ |
552 | ctx_req.ctype = NPA_AQ_CTYPE_AURA; |
553 | npa_lf_hwctx_disable(rvu, req: &ctx_req); |
554 | |
555 | npa_ctx_free(rvu, pfvf); |
556 | } |
557 | |
558 | /* Due to an Hardware errata, in some corner cases, AQ context lock |
559 | * operations can result in a NDC way getting into an illegal state |
560 | * of not valid but locked. |
561 | * |
562 | * This API solves the problem by clearing the lock bit of the NDC block. |
563 | * The operation needs to be done for each line of all the NDC banks. |
564 | */ |
565 | int rvu_ndc_fix_locked_cacheline(struct rvu *rvu, int blkaddr) |
566 | { |
567 | int bank, max_bank, line, max_line, err; |
568 | u64 reg, ndc_af_const; |
569 | |
570 | /* Set the ENABLE bit(63) to '0' */ |
571 | reg = rvu_read64(rvu, block: blkaddr, NDC_AF_CAMS_RD_INTERVAL); |
572 | rvu_write64(rvu, block: blkaddr, NDC_AF_CAMS_RD_INTERVAL, val: reg & GENMASK_ULL(62, 0)); |
573 | |
574 | /* Poll until the BUSY bits(47:32) are set to '0' */ |
575 | err = rvu_poll_reg(rvu, block: blkaddr, NDC_AF_CAMS_RD_INTERVAL, GENMASK_ULL(47, 32), zero: true); |
576 | if (err) { |
577 | dev_err(rvu->dev, "Timed out while polling for NDC CAM busy bits.\n" ); |
578 | return err; |
579 | } |
580 | |
581 | ndc_af_const = rvu_read64(rvu, block: blkaddr, NDC_AF_CONST); |
582 | max_bank = FIELD_GET(NDC_AF_BANK_MASK, ndc_af_const); |
583 | max_line = FIELD_GET(NDC_AF_BANK_LINE_MASK, ndc_af_const); |
584 | for (bank = 0; bank < max_bank; bank++) { |
585 | for (line = 0; line < max_line; line++) { |
586 | /* Check if 'cache line valid bit(63)' is not set |
587 | * but 'cache line lock bit(60)' is set and on |
588 | * success, reset the lock bit(60). |
589 | */ |
590 | reg = rvu_read64(rvu, block: blkaddr, |
591 | NDC_AF_BANKX_LINEX_METADATA(bank, line)); |
592 | if (!(reg & BIT_ULL(63)) && (reg & BIT_ULL(60))) { |
593 | rvu_write64(rvu, block: blkaddr, |
594 | NDC_AF_BANKX_LINEX_METADATA(bank, line), |
595 | val: reg & ~BIT_ULL(60)); |
596 | } |
597 | } |
598 | } |
599 | |
600 | return 0; |
601 | } |
602 | |