1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (C) 2020 Marvell. */
3
4#include <linux/ctype.h>
5#include <linux/firmware.h>
6#include "otx2_cptpf_ucode.h"
7#include "otx2_cpt_common.h"
8#include "otx2_cptpf.h"
9#include "otx2_cptlf.h"
10#include "otx2_cpt_reqmgr.h"
11#include "rvu_reg.h"
12
13#define CSR_DELAY 30
14
15#define LOADFVC_RLEN 8
16#define LOADFVC_MAJOR_OP 0x01
17#define LOADFVC_MINOR_OP 0x08
18
19/*
20 * Interval to flush dirty data for next CTX entry. The interval is measured
21 * in increments of 10ns(interval time = CTX_FLUSH_TIMER_COUNT * 10ns).
22 */
23#define CTX_FLUSH_TIMER_CNT 0x2FAF0
24
25struct fw_info_t {
26 struct list_head ucodes;
27};
28
29static struct otx2_cpt_bitmap get_cores_bmap(struct device *dev,
30 struct otx2_cpt_eng_grp_info *eng_grp)
31{
32 struct otx2_cpt_bitmap bmap = { {0} };
33 bool found = false;
34 int i;
35
36 if (eng_grp->g->engs_num < 0 ||
37 eng_grp->g->engs_num > OTX2_CPT_MAX_ENGINES) {
38 dev_err(dev, "unsupported number of engines %d on octeontx2\n",
39 eng_grp->g->engs_num);
40 return bmap;
41 }
42
43 for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) {
44 if (eng_grp->engs[i].type) {
45 bitmap_or(dst: bmap.bits, src1: bmap.bits,
46 src2: eng_grp->engs[i].bmap,
47 nbits: eng_grp->g->engs_num);
48 bmap.size = eng_grp->g->engs_num;
49 found = true;
50 }
51 }
52
53 if (!found)
54 dev_err(dev, "No engines reserved for engine group %d\n",
55 eng_grp->idx);
56 return bmap;
57}
58
59static int is_eng_type(int val, int eng_type)
60{
61 return val & (1 << eng_type);
62}
63
64static int is_2nd_ucode_used(struct otx2_cpt_eng_grp_info *eng_grp)
65{
66 if (eng_grp->ucode[1].type)
67 return true;
68 else
69 return false;
70}
71
72static void set_ucode_filename(struct otx2_cpt_ucode *ucode,
73 const char *filename)
74{
75 strscpy(ucode->filename, filename, OTX2_CPT_NAME_LENGTH);
76}
77
78static char *get_eng_type_str(int eng_type)
79{
80 char *str = "unknown";
81
82 switch (eng_type) {
83 case OTX2_CPT_SE_TYPES:
84 str = "SE";
85 break;
86
87 case OTX2_CPT_IE_TYPES:
88 str = "IE";
89 break;
90
91 case OTX2_CPT_AE_TYPES:
92 str = "AE";
93 break;
94 }
95 return str;
96}
97
98static char *get_ucode_type_str(int ucode_type)
99{
100 char *str = "unknown";
101
102 switch (ucode_type) {
103 case (1 << OTX2_CPT_SE_TYPES):
104 str = "SE";
105 break;
106
107 case (1 << OTX2_CPT_IE_TYPES):
108 str = "IE";
109 break;
110
111 case (1 << OTX2_CPT_AE_TYPES):
112 str = "AE";
113 break;
114
115 case (1 << OTX2_CPT_SE_TYPES | 1 << OTX2_CPT_IE_TYPES):
116 str = "SE+IPSEC";
117 break;
118 }
119 return str;
120}
121
122static int get_ucode_type(struct device *dev,
123 struct otx2_cpt_ucode_hdr *ucode_hdr,
124 int *ucode_type, u16 rid)
125{
126 char ver_str_prefix[OTX2_CPT_UCODE_VER_STR_SZ];
127 char tmp_ver_str[OTX2_CPT_UCODE_VER_STR_SZ];
128 int i, val = 0;
129 u8 nn;
130
131 strscpy(tmp_ver_str, ucode_hdr->ver_str, OTX2_CPT_UCODE_VER_STR_SZ);
132 for (i = 0; i < strlen(tmp_ver_str); i++)
133 tmp_ver_str[i] = tolower(tmp_ver_str[i]);
134
135 sprintf(buf: ver_str_prefix, fmt: "ocpt-%02d", rid);
136 if (!strnstr(tmp_ver_str, ver_str_prefix, OTX2_CPT_UCODE_VER_STR_SZ))
137 return -EINVAL;
138
139 nn = ucode_hdr->ver_num.nn;
140 if (strnstr(tmp_ver_str, "se-", OTX2_CPT_UCODE_VER_STR_SZ) &&
141 (nn == OTX2_CPT_SE_UC_TYPE1 || nn == OTX2_CPT_SE_UC_TYPE2 ||
142 nn == OTX2_CPT_SE_UC_TYPE3))
143 val |= 1 << OTX2_CPT_SE_TYPES;
144 if (strnstr(tmp_ver_str, "ie-", OTX2_CPT_UCODE_VER_STR_SZ) &&
145 (nn == OTX2_CPT_IE_UC_TYPE1 || nn == OTX2_CPT_IE_UC_TYPE2 ||
146 nn == OTX2_CPT_IE_UC_TYPE3))
147 val |= 1 << OTX2_CPT_IE_TYPES;
148 if (strnstr(tmp_ver_str, "ae", OTX2_CPT_UCODE_VER_STR_SZ) &&
149 nn == OTX2_CPT_AE_UC_TYPE)
150 val |= 1 << OTX2_CPT_AE_TYPES;
151
152 *ucode_type = val;
153
154 if (!val)
155 return -EINVAL;
156
157 return 0;
158}
159
160static int __write_ucode_base(struct otx2_cptpf_dev *cptpf, int eng,
161 dma_addr_t dma_addr, int blkaddr)
162{
163 return otx2_cpt_write_af_reg(mbox: &cptpf->afpf_mbox, pdev: cptpf->pdev,
164 CPT_AF_EXEX_UCODE_BASE(eng),
165 val: (u64)dma_addr, blkaddr);
166}
167
168static int cptx_set_ucode_base(struct otx2_cpt_eng_grp_info *eng_grp,
169 struct otx2_cptpf_dev *cptpf, int blkaddr)
170{
171 struct otx2_cpt_engs_rsvd *engs;
172 dma_addr_t dma_addr;
173 int i, bit, ret;
174
175 /* Set PF number for microcode fetches */
176 ret = otx2_cpt_write_af_reg(mbox: &cptpf->afpf_mbox, pdev: cptpf->pdev,
177 CPT_AF_PF_FUNC,
178 val: cptpf->pf_id << RVU_PFVF_PF_SHIFT, blkaddr);
179 if (ret)
180 return ret;
181
182 for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) {
183 engs = &eng_grp->engs[i];
184 if (!engs->type)
185 continue;
186
187 dma_addr = engs->ucode->dma;
188
189 /*
190 * Set UCODE_BASE only for the cores which are not used,
191 * other cores should have already valid UCODE_BASE set
192 */
193 for_each_set_bit(bit, engs->bmap, eng_grp->g->engs_num)
194 if (!eng_grp->g->eng_ref_cnt[bit]) {
195 ret = __write_ucode_base(cptpf, eng: bit, dma_addr,
196 blkaddr);
197 if (ret)
198 return ret;
199 }
200 }
201 return 0;
202}
203
204static int cpt_set_ucode_base(struct otx2_cpt_eng_grp_info *eng_grp, void *obj)
205{
206 struct otx2_cptpf_dev *cptpf = obj;
207 int ret;
208
209 if (cptpf->has_cpt1) {
210 ret = cptx_set_ucode_base(eng_grp, cptpf, blkaddr: BLKADDR_CPT1);
211 if (ret)
212 return ret;
213 }
214 return cptx_set_ucode_base(eng_grp, cptpf, blkaddr: BLKADDR_CPT0);
215}
216
217static int cptx_detach_and_disable_cores(struct otx2_cpt_eng_grp_info *eng_grp,
218 struct otx2_cptpf_dev *cptpf,
219 struct otx2_cpt_bitmap bmap,
220 int blkaddr)
221{
222 int i, timeout = 10;
223 int busy, ret;
224 u64 reg = 0;
225
226 /* Detach the cores from group */
227 for_each_set_bit(i, bmap.bits, bmap.size) {
228 ret = otx2_cpt_read_af_reg(mbox: &cptpf->afpf_mbox, pdev: cptpf->pdev,
229 CPT_AF_EXEX_CTL2(i), val: &reg, blkaddr);
230 if (ret)
231 return ret;
232
233 if (reg & (1ull << eng_grp->idx)) {
234 eng_grp->g->eng_ref_cnt[i]--;
235 reg &= ~(1ull << eng_grp->idx);
236
237 ret = otx2_cpt_write_af_reg(mbox: &cptpf->afpf_mbox,
238 pdev: cptpf->pdev,
239 CPT_AF_EXEX_CTL2(i), val: reg,
240 blkaddr);
241 if (ret)
242 return ret;
243 }
244 }
245
246 /* Wait for cores to become idle */
247 do {
248 busy = 0;
249 usleep_range(min: 10000, max: 20000);
250 if (timeout-- < 0)
251 return -EBUSY;
252
253 for_each_set_bit(i, bmap.bits, bmap.size) {
254 ret = otx2_cpt_read_af_reg(mbox: &cptpf->afpf_mbox,
255 pdev: cptpf->pdev,
256 CPT_AF_EXEX_STS(i), val: &reg,
257 blkaddr);
258 if (ret)
259 return ret;
260
261 if (reg & 0x1) {
262 busy = 1;
263 break;
264 }
265 }
266 } while (busy);
267
268 /* Disable the cores only if they are not used anymore */
269 for_each_set_bit(i, bmap.bits, bmap.size) {
270 if (!eng_grp->g->eng_ref_cnt[i]) {
271 ret = otx2_cpt_write_af_reg(mbox: &cptpf->afpf_mbox,
272 pdev: cptpf->pdev,
273 CPT_AF_EXEX_CTL(i), val: 0x0,
274 blkaddr);
275 if (ret)
276 return ret;
277 }
278 }
279
280 return 0;
281}
282
283static int cpt_detach_and_disable_cores(struct otx2_cpt_eng_grp_info *eng_grp,
284 void *obj)
285{
286 struct otx2_cptpf_dev *cptpf = obj;
287 struct otx2_cpt_bitmap bmap;
288 int ret;
289
290 bmap = get_cores_bmap(dev: &cptpf->pdev->dev, eng_grp);
291 if (!bmap.size)
292 return -EINVAL;
293
294 if (cptpf->has_cpt1) {
295 ret = cptx_detach_and_disable_cores(eng_grp, cptpf, bmap,
296 blkaddr: BLKADDR_CPT1);
297 if (ret)
298 return ret;
299 }
300 return cptx_detach_and_disable_cores(eng_grp, cptpf, bmap,
301 blkaddr: BLKADDR_CPT0);
302}
303
304static int cptx_attach_and_enable_cores(struct otx2_cpt_eng_grp_info *eng_grp,
305 struct otx2_cptpf_dev *cptpf,
306 struct otx2_cpt_bitmap bmap,
307 int blkaddr)
308{
309 u64 reg = 0;
310 int i, ret;
311
312 /* Attach the cores to the group */
313 for_each_set_bit(i, bmap.bits, bmap.size) {
314 ret = otx2_cpt_read_af_reg(mbox: &cptpf->afpf_mbox, pdev: cptpf->pdev,
315 CPT_AF_EXEX_CTL2(i), val: &reg, blkaddr);
316 if (ret)
317 return ret;
318
319 if (!(reg & (1ull << eng_grp->idx))) {
320 eng_grp->g->eng_ref_cnt[i]++;
321 reg |= 1ull << eng_grp->idx;
322
323 ret = otx2_cpt_write_af_reg(mbox: &cptpf->afpf_mbox,
324 pdev: cptpf->pdev,
325 CPT_AF_EXEX_CTL2(i), val: reg,
326 blkaddr);
327 if (ret)
328 return ret;
329 }
330 }
331
332 /* Enable the cores */
333 for_each_set_bit(i, bmap.bits, bmap.size) {
334 ret = otx2_cpt_add_write_af_reg(mbox: &cptpf->afpf_mbox, pdev: cptpf->pdev,
335 CPT_AF_EXEX_CTL(i), val: 0x1,
336 blkaddr);
337 if (ret)
338 return ret;
339 }
340 return otx2_cpt_send_af_reg_requests(mbox: &cptpf->afpf_mbox, pdev: cptpf->pdev);
341}
342
343static int cpt_attach_and_enable_cores(struct otx2_cpt_eng_grp_info *eng_grp,
344 void *obj)
345{
346 struct otx2_cptpf_dev *cptpf = obj;
347 struct otx2_cpt_bitmap bmap;
348 int ret;
349
350 bmap = get_cores_bmap(dev: &cptpf->pdev->dev, eng_grp);
351 if (!bmap.size)
352 return -EINVAL;
353
354 if (cptpf->has_cpt1) {
355 ret = cptx_attach_and_enable_cores(eng_grp, cptpf, bmap,
356 blkaddr: BLKADDR_CPT1);
357 if (ret)
358 return ret;
359 }
360 return cptx_attach_and_enable_cores(eng_grp, cptpf, bmap, blkaddr: BLKADDR_CPT0);
361}
362
363static int load_fw(struct device *dev, struct fw_info_t *fw_info,
364 char *filename, u16 rid)
365{
366 struct otx2_cpt_ucode_hdr *ucode_hdr;
367 struct otx2_cpt_uc_info_t *uc_info;
368 int ucode_type, ucode_size;
369 int ret;
370
371 uc_info = kzalloc(size: sizeof(*uc_info), GFP_KERNEL);
372 if (!uc_info)
373 return -ENOMEM;
374
375 ret = request_firmware(fw: &uc_info->fw, name: filename, device: dev);
376 if (ret)
377 goto free_uc_info;
378
379 ucode_hdr = (struct otx2_cpt_ucode_hdr *)uc_info->fw->data;
380 ret = get_ucode_type(dev, ucode_hdr, ucode_type: &ucode_type, rid);
381 if (ret)
382 goto release_fw;
383
384 ucode_size = ntohl(ucode_hdr->code_length) * 2;
385 if (!ucode_size) {
386 dev_err(dev, "Ucode %s invalid size\n", filename);
387 ret = -EINVAL;
388 goto release_fw;
389 }
390
391 set_ucode_filename(ucode: &uc_info->ucode, filename);
392 memcpy(uc_info->ucode.ver_str, ucode_hdr->ver_str,
393 OTX2_CPT_UCODE_VER_STR_SZ);
394 uc_info->ucode.ver_str[OTX2_CPT_UCODE_VER_STR_SZ] = 0;
395 uc_info->ucode.ver_num = ucode_hdr->ver_num;
396 uc_info->ucode.type = ucode_type;
397 uc_info->ucode.size = ucode_size;
398 list_add_tail(new: &uc_info->list, head: &fw_info->ucodes);
399
400 return 0;
401
402release_fw:
403 release_firmware(fw: uc_info->fw);
404free_uc_info:
405 kfree(objp: uc_info);
406 return ret;
407}
408
409static void cpt_ucode_release_fw(struct fw_info_t *fw_info)
410{
411 struct otx2_cpt_uc_info_t *curr, *temp;
412
413 if (!fw_info)
414 return;
415
416 list_for_each_entry_safe(curr, temp, &fw_info->ucodes, list) {
417 list_del(entry: &curr->list);
418 release_firmware(fw: curr->fw);
419 kfree(objp: curr);
420 }
421}
422
423static struct otx2_cpt_uc_info_t *get_ucode(struct fw_info_t *fw_info,
424 int ucode_type)
425{
426 struct otx2_cpt_uc_info_t *curr;
427
428 list_for_each_entry(curr, &fw_info->ucodes, list) {
429 if (!is_eng_type(val: curr->ucode.type, eng_type: ucode_type))
430 continue;
431
432 return curr;
433 }
434 return NULL;
435}
436
437static void print_uc_info(struct fw_info_t *fw_info)
438{
439 struct otx2_cpt_uc_info_t *curr;
440
441 list_for_each_entry(curr, &fw_info->ucodes, list) {
442 pr_debug("Ucode filename %s\n", curr->ucode.filename);
443 pr_debug("Ucode version string %s\n", curr->ucode.ver_str);
444 pr_debug("Ucode version %d.%d.%d.%d\n",
445 curr->ucode.ver_num.nn, curr->ucode.ver_num.xx,
446 curr->ucode.ver_num.yy, curr->ucode.ver_num.zz);
447 pr_debug("Ucode type (%d) %s\n", curr->ucode.type,
448 get_ucode_type_str(curr->ucode.type));
449 pr_debug("Ucode size %d\n", curr->ucode.size);
450 pr_debug("Ucode ptr %p\n", curr->fw->data);
451 }
452}
453
454static int cpt_ucode_load_fw(struct pci_dev *pdev, struct fw_info_t *fw_info,
455 u16 rid)
456{
457 char filename[OTX2_CPT_NAME_LENGTH];
458 char eng_type[8] = {0};
459 int ret, e, i;
460
461 INIT_LIST_HEAD(list: &fw_info->ucodes);
462
463 for (e = 1; e < OTX2_CPT_MAX_ENG_TYPES; e++) {
464 strcpy(p: eng_type, q: get_eng_type_str(eng_type: e));
465 for (i = 0; i < strlen(eng_type); i++)
466 eng_type[i] = tolower(eng_type[i]);
467
468 snprintf(buf: filename, size: sizeof(filename), fmt: "mrvl/cpt%02d/%s.out",
469 rid, eng_type);
470 /* Request firmware for each engine type */
471 ret = load_fw(dev: &pdev->dev, fw_info, filename, rid);
472 if (ret)
473 goto release_fw;
474 }
475 print_uc_info(fw_info);
476 return 0;
477
478release_fw:
479 cpt_ucode_release_fw(fw_info);
480 return ret;
481}
482
483struct otx2_cpt_engs_rsvd *find_engines_by_type(
484 struct otx2_cpt_eng_grp_info *eng_grp,
485 int eng_type)
486{
487 int i;
488
489 for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) {
490 if (!eng_grp->engs[i].type)
491 continue;
492
493 if (eng_grp->engs[i].type == eng_type)
494 return &eng_grp->engs[i];
495 }
496 return NULL;
497}
498
499static int eng_grp_has_eng_type(struct otx2_cpt_eng_grp_info *eng_grp,
500 int eng_type)
501{
502 struct otx2_cpt_engs_rsvd *engs;
503
504 engs = find_engines_by_type(eng_grp, eng_type);
505
506 return (engs != NULL ? 1 : 0);
507}
508
509static int update_engines_avail_count(struct device *dev,
510 struct otx2_cpt_engs_available *avail,
511 struct otx2_cpt_engs_rsvd *engs, int val)
512{
513 switch (engs->type) {
514 case OTX2_CPT_SE_TYPES:
515 avail->se_cnt += val;
516 break;
517
518 case OTX2_CPT_IE_TYPES:
519 avail->ie_cnt += val;
520 break;
521
522 case OTX2_CPT_AE_TYPES:
523 avail->ae_cnt += val;
524 break;
525
526 default:
527 dev_err(dev, "Invalid engine type %d\n", engs->type);
528 return -EINVAL;
529 }
530 return 0;
531}
532
533static int update_engines_offset(struct device *dev,
534 struct otx2_cpt_engs_available *avail,
535 struct otx2_cpt_engs_rsvd *engs)
536{
537 switch (engs->type) {
538 case OTX2_CPT_SE_TYPES:
539 engs->offset = 0;
540 break;
541
542 case OTX2_CPT_IE_TYPES:
543 engs->offset = avail->max_se_cnt;
544 break;
545
546 case OTX2_CPT_AE_TYPES:
547 engs->offset = avail->max_se_cnt + avail->max_ie_cnt;
548 break;
549
550 default:
551 dev_err(dev, "Invalid engine type %d\n", engs->type);
552 return -EINVAL;
553 }
554 return 0;
555}
556
557static int release_engines(struct device *dev,
558 struct otx2_cpt_eng_grp_info *grp)
559{
560 int i, ret = 0;
561
562 for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) {
563 if (!grp->engs[i].type)
564 continue;
565
566 if (grp->engs[i].count > 0) {
567 ret = update_engines_avail_count(dev, avail: &grp->g->avail,
568 engs: &grp->engs[i],
569 val: grp->engs[i].count);
570 if (ret)
571 return ret;
572 }
573
574 grp->engs[i].type = 0;
575 grp->engs[i].count = 0;
576 grp->engs[i].offset = 0;
577 grp->engs[i].ucode = NULL;
578 bitmap_zero(dst: grp->engs[i].bmap, nbits: grp->g->engs_num);
579 }
580 return 0;
581}
582
583static int do_reserve_engines(struct device *dev,
584 struct otx2_cpt_eng_grp_info *grp,
585 struct otx2_cpt_engines *req_engs)
586{
587 struct otx2_cpt_engs_rsvd *engs = NULL;
588 int i, ret;
589
590 for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) {
591 if (!grp->engs[i].type) {
592 engs = &grp->engs[i];
593 break;
594 }
595 }
596
597 if (!engs)
598 return -ENOMEM;
599
600 engs->type = req_engs->type;
601 engs->count = req_engs->count;
602
603 ret = update_engines_offset(dev, avail: &grp->g->avail, engs);
604 if (ret)
605 return ret;
606
607 if (engs->count > 0) {
608 ret = update_engines_avail_count(dev, avail: &grp->g->avail, engs,
609 val: -engs->count);
610 if (ret)
611 return ret;
612 }
613
614 return 0;
615}
616
617static int check_engines_availability(struct device *dev,
618 struct otx2_cpt_eng_grp_info *grp,
619 struct otx2_cpt_engines *req_eng)
620{
621 int avail_cnt = 0;
622
623 switch (req_eng->type) {
624 case OTX2_CPT_SE_TYPES:
625 avail_cnt = grp->g->avail.se_cnt;
626 break;
627
628 case OTX2_CPT_IE_TYPES:
629 avail_cnt = grp->g->avail.ie_cnt;
630 break;
631
632 case OTX2_CPT_AE_TYPES:
633 avail_cnt = grp->g->avail.ae_cnt;
634 break;
635
636 default:
637 dev_err(dev, "Invalid engine type %d\n", req_eng->type);
638 return -EINVAL;
639 }
640
641 if (avail_cnt < req_eng->count) {
642 dev_err(dev,
643 "Error available %s engines %d < than requested %d\n",
644 get_eng_type_str(req_eng->type),
645 avail_cnt, req_eng->count);
646 return -EBUSY;
647 }
648 return 0;
649}
650
651static int reserve_engines(struct device *dev,
652 struct otx2_cpt_eng_grp_info *grp,
653 struct otx2_cpt_engines *req_engs, int ucodes_cnt)
654{
655 int i, ret = 0;
656
657 /* Validate if a number of requested engines are available */
658 for (i = 0; i < ucodes_cnt; i++) {
659 ret = check_engines_availability(dev, grp, req_eng: &req_engs[i]);
660 if (ret)
661 return ret;
662 }
663
664 /* Reserve requested engines for this engine group */
665 for (i = 0; i < ucodes_cnt; i++) {
666 ret = do_reserve_engines(dev, grp, req_engs: &req_engs[i]);
667 if (ret)
668 return ret;
669 }
670 return 0;
671}
672
673static void ucode_unload(struct device *dev, struct otx2_cpt_ucode *ucode)
674{
675 if (ucode->va) {
676 dma_free_coherent(dev, OTX2_CPT_UCODE_SZ, cpu_addr: ucode->va,
677 dma_handle: ucode->dma);
678 ucode->va = NULL;
679 ucode->dma = 0;
680 ucode->size = 0;
681 }
682
683 memset(&ucode->ver_str, 0, OTX2_CPT_UCODE_VER_STR_SZ);
684 memset(&ucode->ver_num, 0, sizeof(struct otx2_cpt_ucode_ver_num));
685 set_ucode_filename(ucode, filename: "");
686 ucode->type = 0;
687}
688
689static int copy_ucode_to_dma_mem(struct device *dev,
690 struct otx2_cpt_ucode *ucode,
691 const u8 *ucode_data)
692{
693 u32 i;
694
695 /* Allocate DMAable space */
696 ucode->va = dma_alloc_coherent(dev, OTX2_CPT_UCODE_SZ, dma_handle: &ucode->dma,
697 GFP_KERNEL);
698 if (!ucode->va)
699 return -ENOMEM;
700
701 memcpy(ucode->va, ucode_data + sizeof(struct otx2_cpt_ucode_hdr),
702 ucode->size);
703
704 /* Byte swap 64-bit */
705 for (i = 0; i < (ucode->size / 8); i++)
706 cpu_to_be64s(&((u64 *)ucode->va)[i]);
707 /* Ucode needs 16-bit swap */
708 for (i = 0; i < (ucode->size / 2); i++)
709 cpu_to_be16s(&((u16 *)ucode->va)[i]);
710 return 0;
711}
712
713static int enable_eng_grp(struct otx2_cpt_eng_grp_info *eng_grp,
714 void *obj)
715{
716 int ret;
717
718 /* Point microcode to each core of the group */
719 ret = cpt_set_ucode_base(eng_grp, obj);
720 if (ret)
721 return ret;
722
723 /* Attach the cores to the group and enable them */
724 ret = cpt_attach_and_enable_cores(eng_grp, obj);
725
726 return ret;
727}
728
729static int disable_eng_grp(struct device *dev,
730 struct otx2_cpt_eng_grp_info *eng_grp,
731 void *obj)
732{
733 int i, ret;
734
735 /* Disable all engines used by this group */
736 ret = cpt_detach_and_disable_cores(eng_grp, obj);
737 if (ret)
738 return ret;
739
740 /* Unload ucode used by this engine group */
741 ucode_unload(dev, ucode: &eng_grp->ucode[0]);
742 ucode_unload(dev, ucode: &eng_grp->ucode[1]);
743
744 for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) {
745 if (!eng_grp->engs[i].type)
746 continue;
747
748 eng_grp->engs[i].ucode = &eng_grp->ucode[0];
749 }
750
751 /* Clear UCODE_BASE register for each engine used by this group */
752 ret = cpt_set_ucode_base(eng_grp, obj);
753
754 return ret;
755}
756
757static void setup_eng_grp_mirroring(struct otx2_cpt_eng_grp_info *dst_grp,
758 struct otx2_cpt_eng_grp_info *src_grp)
759{
760 /* Setup fields for engine group which is mirrored */
761 src_grp->mirror.is_ena = false;
762 src_grp->mirror.idx = 0;
763 src_grp->mirror.ref_count++;
764
765 /* Setup fields for mirroring engine group */
766 dst_grp->mirror.is_ena = true;
767 dst_grp->mirror.idx = src_grp->idx;
768 dst_grp->mirror.ref_count = 0;
769}
770
771static void remove_eng_grp_mirroring(struct otx2_cpt_eng_grp_info *dst_grp)
772{
773 struct otx2_cpt_eng_grp_info *src_grp;
774
775 if (!dst_grp->mirror.is_ena)
776 return;
777
778 src_grp = &dst_grp->g->grp[dst_grp->mirror.idx];
779
780 src_grp->mirror.ref_count--;
781 dst_grp->mirror.is_ena = false;
782 dst_grp->mirror.idx = 0;
783 dst_grp->mirror.ref_count = 0;
784}
785
786static void update_requested_engs(struct otx2_cpt_eng_grp_info *mirror_eng_grp,
787 struct otx2_cpt_engines *engs, int engs_cnt)
788{
789 struct otx2_cpt_engs_rsvd *mirrored_engs;
790 int i;
791
792 for (i = 0; i < engs_cnt; i++) {
793 mirrored_engs = find_engines_by_type(eng_grp: mirror_eng_grp,
794 eng_type: engs[i].type);
795 if (!mirrored_engs)
796 continue;
797
798 /*
799 * If mirrored group has this type of engines attached then
800 * there are 3 scenarios possible:
801 * 1) mirrored_engs.count == engs[i].count then all engines
802 * from mirrored engine group will be shared with this engine
803 * group
804 * 2) mirrored_engs.count > engs[i].count then only a subset of
805 * engines from mirrored engine group will be shared with this
806 * engine group
807 * 3) mirrored_engs.count < engs[i].count then all engines
808 * from mirrored engine group will be shared with this group
809 * and additional engines will be reserved for exclusively use
810 * by this engine group
811 */
812 engs[i].count -= mirrored_engs->count;
813 }
814}
815
816static struct otx2_cpt_eng_grp_info *find_mirrored_eng_grp(
817 struct otx2_cpt_eng_grp_info *grp)
818{
819 struct otx2_cpt_eng_grps *eng_grps = grp->g;
820 int i;
821
822 for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) {
823 if (!eng_grps->grp[i].is_enabled)
824 continue;
825 if (eng_grps->grp[i].ucode[0].type &&
826 eng_grps->grp[i].ucode[1].type)
827 continue;
828 if (grp->idx == i)
829 continue;
830 if (!strncasecmp(s1: eng_grps->grp[i].ucode[0].ver_str,
831 s2: grp->ucode[0].ver_str,
832 OTX2_CPT_UCODE_VER_STR_SZ))
833 return &eng_grps->grp[i];
834 }
835
836 return NULL;
837}
838
839static struct otx2_cpt_eng_grp_info *find_unused_eng_grp(
840 struct otx2_cpt_eng_grps *eng_grps)
841{
842 int i;
843
844 for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) {
845 if (!eng_grps->grp[i].is_enabled)
846 return &eng_grps->grp[i];
847 }
848 return NULL;
849}
850
851static int eng_grp_update_masks(struct device *dev,
852 struct otx2_cpt_eng_grp_info *eng_grp)
853{
854 struct otx2_cpt_engs_rsvd *engs, *mirrored_engs;
855 struct otx2_cpt_bitmap tmp_bmap = { {0} };
856 int i, j, cnt, max_cnt;
857 int bit;
858
859 for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) {
860 engs = &eng_grp->engs[i];
861 if (!engs->type)
862 continue;
863 if (engs->count <= 0)
864 continue;
865
866 switch (engs->type) {
867 case OTX2_CPT_SE_TYPES:
868 max_cnt = eng_grp->g->avail.max_se_cnt;
869 break;
870
871 case OTX2_CPT_IE_TYPES:
872 max_cnt = eng_grp->g->avail.max_ie_cnt;
873 break;
874
875 case OTX2_CPT_AE_TYPES:
876 max_cnt = eng_grp->g->avail.max_ae_cnt;
877 break;
878
879 default:
880 dev_err(dev, "Invalid engine type %d\n", engs->type);
881 return -EINVAL;
882 }
883
884 cnt = engs->count;
885 WARN_ON(engs->offset + max_cnt > OTX2_CPT_MAX_ENGINES);
886 bitmap_zero(dst: tmp_bmap.bits, nbits: eng_grp->g->engs_num);
887 for (j = engs->offset; j < engs->offset + max_cnt; j++) {
888 if (!eng_grp->g->eng_ref_cnt[j]) {
889 bitmap_set(map: tmp_bmap.bits, start: j, nbits: 1);
890 cnt--;
891 if (!cnt)
892 break;
893 }
894 }
895
896 if (cnt)
897 return -ENOSPC;
898
899 bitmap_copy(dst: engs->bmap, src: tmp_bmap.bits, nbits: eng_grp->g->engs_num);
900 }
901
902 if (!eng_grp->mirror.is_ena)
903 return 0;
904
905 for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) {
906 engs = &eng_grp->engs[i];
907 if (!engs->type)
908 continue;
909
910 mirrored_engs = find_engines_by_type(
911 eng_grp: &eng_grp->g->grp[eng_grp->mirror.idx],
912 eng_type: engs->type);
913 WARN_ON(!mirrored_engs && engs->count <= 0);
914 if (!mirrored_engs)
915 continue;
916
917 bitmap_copy(dst: tmp_bmap.bits, src: mirrored_engs->bmap,
918 nbits: eng_grp->g->engs_num);
919 if (engs->count < 0) {
920 bit = find_first_bit(addr: mirrored_engs->bmap,
921 size: eng_grp->g->engs_num);
922 bitmap_clear(map: tmp_bmap.bits, start: bit, nbits: -engs->count);
923 }
924 bitmap_or(dst: engs->bmap, src1: engs->bmap, src2: tmp_bmap.bits,
925 nbits: eng_grp->g->engs_num);
926 }
927 return 0;
928}
929
930static int delete_engine_group(struct device *dev,
931 struct otx2_cpt_eng_grp_info *eng_grp)
932{
933 int ret;
934
935 if (!eng_grp->is_enabled)
936 return 0;
937
938 if (eng_grp->mirror.ref_count)
939 return -EINVAL;
940
941 /* Removing engine group mirroring if enabled */
942 remove_eng_grp_mirroring(dst_grp: eng_grp);
943
944 /* Disable engine group */
945 ret = disable_eng_grp(dev, eng_grp, obj: eng_grp->g->obj);
946 if (ret)
947 return ret;
948
949 /* Release all engines held by this engine group */
950 ret = release_engines(dev, grp: eng_grp);
951 if (ret)
952 return ret;
953
954 eng_grp->is_enabled = false;
955
956 return 0;
957}
958
959static void update_ucode_ptrs(struct otx2_cpt_eng_grp_info *eng_grp)
960{
961 struct otx2_cpt_ucode *ucode;
962
963 if (eng_grp->mirror.is_ena)
964 ucode = &eng_grp->g->grp[eng_grp->mirror.idx].ucode[0];
965 else
966 ucode = &eng_grp->ucode[0];
967 WARN_ON(!eng_grp->engs[0].type);
968 eng_grp->engs[0].ucode = ucode;
969
970 if (eng_grp->engs[1].type) {
971 if (is_2nd_ucode_used(eng_grp))
972 eng_grp->engs[1].ucode = &eng_grp->ucode[1];
973 else
974 eng_grp->engs[1].ucode = ucode;
975 }
976}
977
978static int create_engine_group(struct device *dev,
979 struct otx2_cpt_eng_grps *eng_grps,
980 struct otx2_cpt_engines *engs, int ucodes_cnt,
981 void *ucode_data[], int is_print)
982{
983 struct otx2_cpt_eng_grp_info *mirrored_eng_grp;
984 struct otx2_cpt_eng_grp_info *eng_grp;
985 struct otx2_cpt_uc_info_t *uc_info;
986 int i, ret = 0;
987
988 /* Find engine group which is not used */
989 eng_grp = find_unused_eng_grp(eng_grps);
990 if (!eng_grp) {
991 dev_err(dev, "Error all engine groups are being used\n");
992 return -ENOSPC;
993 }
994 /* Load ucode */
995 for (i = 0; i < ucodes_cnt; i++) {
996 uc_info = (struct otx2_cpt_uc_info_t *) ucode_data[i];
997 eng_grp->ucode[i] = uc_info->ucode;
998 ret = copy_ucode_to_dma_mem(dev, ucode: &eng_grp->ucode[i],
999 ucode_data: uc_info->fw->data);
1000 if (ret)
1001 goto unload_ucode;
1002 }
1003
1004 /* Check if this group mirrors another existing engine group */
1005 mirrored_eng_grp = find_mirrored_eng_grp(grp: eng_grp);
1006 if (mirrored_eng_grp) {
1007 /* Setup mirroring */
1008 setup_eng_grp_mirroring(dst_grp: eng_grp, src_grp: mirrored_eng_grp);
1009
1010 /*
1011 * Update count of requested engines because some
1012 * of them might be shared with mirrored group
1013 */
1014 update_requested_engs(mirror_eng_grp: mirrored_eng_grp, engs, engs_cnt: ucodes_cnt);
1015 }
1016 ret = reserve_engines(dev, grp: eng_grp, req_engs: engs, ucodes_cnt);
1017 if (ret)
1018 goto unload_ucode;
1019
1020 /* Update ucode pointers used by engines */
1021 update_ucode_ptrs(eng_grp);
1022
1023 /* Update engine masks used by this group */
1024 ret = eng_grp_update_masks(dev, eng_grp);
1025 if (ret)
1026 goto release_engs;
1027
1028 /* Enable engine group */
1029 ret = enable_eng_grp(eng_grp, obj: eng_grps->obj);
1030 if (ret)
1031 goto release_engs;
1032
1033 /*
1034 * If this engine group mirrors another engine group
1035 * then we need to unload ucode as we will use ucode
1036 * from mirrored engine group
1037 */
1038 if (eng_grp->mirror.is_ena)
1039 ucode_unload(dev, ucode: &eng_grp->ucode[0]);
1040
1041 eng_grp->is_enabled = true;
1042
1043 if (!is_print)
1044 return 0;
1045
1046 if (mirrored_eng_grp)
1047 dev_info(dev,
1048 "Engine_group%d: reuse microcode %s from group %d\n",
1049 eng_grp->idx, mirrored_eng_grp->ucode[0].ver_str,
1050 mirrored_eng_grp->idx);
1051 else
1052 dev_info(dev, "Engine_group%d: microcode loaded %s\n",
1053 eng_grp->idx, eng_grp->ucode[0].ver_str);
1054 if (is_2nd_ucode_used(eng_grp))
1055 dev_info(dev, "Engine_group%d: microcode loaded %s\n",
1056 eng_grp->idx, eng_grp->ucode[1].ver_str);
1057
1058 return 0;
1059
1060release_engs:
1061 release_engines(dev, grp: eng_grp);
1062unload_ucode:
1063 ucode_unload(dev, ucode: &eng_grp->ucode[0]);
1064 ucode_unload(dev, ucode: &eng_grp->ucode[1]);
1065 return ret;
1066}
1067
1068static void delete_engine_grps(struct pci_dev *pdev,
1069 struct otx2_cpt_eng_grps *eng_grps)
1070{
1071 int i;
1072
1073 /* First delete all mirroring engine groups */
1074 for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++)
1075 if (eng_grps->grp[i].mirror.is_ena)
1076 delete_engine_group(dev: &pdev->dev, eng_grp: &eng_grps->grp[i]);
1077
1078 /* Delete remaining engine groups */
1079 for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++)
1080 delete_engine_group(dev: &pdev->dev, eng_grp: &eng_grps->grp[i]);
1081}
1082
1083#define PCI_DEVID_CN10K_RNM 0xA098
1084#define RNM_ENTROPY_STATUS 0x8
1085
1086static void rnm_to_cpt_errata_fixup(struct device *dev)
1087{
1088 struct pci_dev *pdev;
1089 void __iomem *base;
1090 int timeout = 5000;
1091
1092 pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CN10K_RNM, NULL);
1093 if (!pdev)
1094 return;
1095
1096 base = pci_ioremap_bar(pdev, bar: 0);
1097 if (!base)
1098 goto put_pdev;
1099
1100 while ((readq(addr: base + RNM_ENTROPY_STATUS) & 0x7F) != 0x40) {
1101 cpu_relax();
1102 udelay(1);
1103 timeout--;
1104 if (!timeout) {
1105 dev_warn(dev, "RNM is not producing entropy\n");
1106 break;
1107 }
1108 }
1109
1110 iounmap(addr: base);
1111
1112put_pdev:
1113 pci_dev_put(dev: pdev);
1114}
1115
1116int otx2_cpt_get_eng_grp(struct otx2_cpt_eng_grps *eng_grps, int eng_type)
1117{
1118
1119 int eng_grp_num = OTX2_CPT_INVALID_CRYPTO_ENG_GRP;
1120 struct otx2_cpt_eng_grp_info *grp;
1121 int i;
1122
1123 for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) {
1124 grp = &eng_grps->grp[i];
1125 if (!grp->is_enabled)
1126 continue;
1127
1128 if (eng_type == OTX2_CPT_SE_TYPES) {
1129 if (eng_grp_has_eng_type(eng_grp: grp, eng_type) &&
1130 !eng_grp_has_eng_type(eng_grp: grp, eng_type: OTX2_CPT_IE_TYPES)) {
1131 eng_grp_num = i;
1132 break;
1133 }
1134 } else {
1135 if (eng_grp_has_eng_type(eng_grp: grp, eng_type)) {
1136 eng_grp_num = i;
1137 break;
1138 }
1139 }
1140 }
1141 return eng_grp_num;
1142}
1143
1144int otx2_cpt_create_eng_grps(struct otx2_cptpf_dev *cptpf,
1145 struct otx2_cpt_eng_grps *eng_grps)
1146{
1147 struct otx2_cpt_uc_info_t *uc_info[OTX2_CPT_MAX_ETYPES_PER_GRP] = { };
1148 struct otx2_cpt_engines engs[OTX2_CPT_MAX_ETYPES_PER_GRP] = { {0} };
1149 struct pci_dev *pdev = cptpf->pdev;
1150 struct fw_info_t fw_info;
1151 u64 reg_val;
1152 int ret = 0;
1153
1154 mutex_lock(&eng_grps->lock);
1155 /*
1156 * We don't create engine groups if it was already
1157 * made (when user enabled VFs for the first time)
1158 */
1159 if (eng_grps->is_grps_created)
1160 goto unlock;
1161
1162 ret = cpt_ucode_load_fw(pdev, fw_info: &fw_info, rid: eng_grps->rid);
1163 if (ret)
1164 goto unlock;
1165
1166 /*
1167 * Create engine group with SE engines for kernel
1168 * crypto functionality (symmetric crypto)
1169 */
1170 uc_info[0] = get_ucode(fw_info: &fw_info, ucode_type: OTX2_CPT_SE_TYPES);
1171 if (uc_info[0] == NULL) {
1172 dev_err(&pdev->dev, "Unable to find firmware for SE\n");
1173 ret = -EINVAL;
1174 goto release_fw;
1175 }
1176 engs[0].type = OTX2_CPT_SE_TYPES;
1177 engs[0].count = eng_grps->avail.max_se_cnt;
1178
1179 ret = create_engine_group(dev: &pdev->dev, eng_grps, engs, ucodes_cnt: 1,
1180 ucode_data: (void **) uc_info, is_print: 1);
1181 if (ret)
1182 goto release_fw;
1183
1184 /*
1185 * Create engine group with SE+IE engines for IPSec.
1186 * All SE engines will be shared with engine group 0.
1187 */
1188 uc_info[0] = get_ucode(fw_info: &fw_info, ucode_type: OTX2_CPT_SE_TYPES);
1189 uc_info[1] = get_ucode(fw_info: &fw_info, ucode_type: OTX2_CPT_IE_TYPES);
1190
1191 if (uc_info[1] == NULL) {
1192 dev_err(&pdev->dev, "Unable to find firmware for IE");
1193 ret = -EINVAL;
1194 goto delete_eng_grp;
1195 }
1196 engs[0].type = OTX2_CPT_SE_TYPES;
1197 engs[0].count = eng_grps->avail.max_se_cnt;
1198 engs[1].type = OTX2_CPT_IE_TYPES;
1199 engs[1].count = eng_grps->avail.max_ie_cnt;
1200
1201 ret = create_engine_group(dev: &pdev->dev, eng_grps, engs, ucodes_cnt: 2,
1202 ucode_data: (void **) uc_info, is_print: 1);
1203 if (ret)
1204 goto delete_eng_grp;
1205
1206 /*
1207 * Create engine group with AE engines for asymmetric
1208 * crypto functionality.
1209 */
1210 uc_info[0] = get_ucode(fw_info: &fw_info, ucode_type: OTX2_CPT_AE_TYPES);
1211 if (uc_info[0] == NULL) {
1212 dev_err(&pdev->dev, "Unable to find firmware for AE");
1213 ret = -EINVAL;
1214 goto delete_eng_grp;
1215 }
1216 engs[0].type = OTX2_CPT_AE_TYPES;
1217 engs[0].count = eng_grps->avail.max_ae_cnt;
1218
1219 ret = create_engine_group(dev: &pdev->dev, eng_grps, engs, ucodes_cnt: 1,
1220 ucode_data: (void **) uc_info, is_print: 1);
1221 if (ret)
1222 goto delete_eng_grp;
1223
1224 eng_grps->is_grps_created = true;
1225
1226 cpt_ucode_release_fw(fw_info: &fw_info);
1227
1228 if (is_dev_otx2(pdev))
1229 goto unlock;
1230
1231 /*
1232 * Ensure RNM_ENTROPY_STATUS[NORMAL_CNT] = 0x40 before writing
1233 * CPT_AF_CTL[RNM_REQ_EN] = 1 as a workaround for HW errata.
1234 */
1235 rnm_to_cpt_errata_fixup(dev: &pdev->dev);
1236
1237 otx2_cpt_read_af_reg(mbox: &cptpf->afpf_mbox, pdev, CPT_AF_CTL, val: &reg_val,
1238 blkaddr: BLKADDR_CPT0);
1239 /*
1240 * Configure engine group mask to allow context prefetching
1241 * for the groups and enable random number request, to enable
1242 * CPT to request random numbers from RNM.
1243 */
1244 reg_val |= OTX2_CPT_ALL_ENG_GRPS_MASK << 3 | BIT_ULL(16);
1245 otx2_cpt_write_af_reg(mbox: &cptpf->afpf_mbox, pdev, CPT_AF_CTL,
1246 val: reg_val, blkaddr: BLKADDR_CPT0);
1247 /*
1248 * Set interval to periodically flush dirty data for the next
1249 * CTX cache entry. Set the interval count to maximum supported
1250 * value.
1251 */
1252 otx2_cpt_write_af_reg(mbox: &cptpf->afpf_mbox, pdev, CPT_AF_CTX_FLUSH_TIMER,
1253 CTX_FLUSH_TIMER_CNT, blkaddr: BLKADDR_CPT0);
1254
1255 /*
1256 * Set CPT_AF_DIAG[FLT_DIS], as a workaround for HW errata, when
1257 * CPT_AF_DIAG[FLT_DIS] = 0 and a CPT engine access to LLC/DRAM
1258 * encounters a fault/poison, a rare case may result in
1259 * unpredictable data being delivered to a CPT engine.
1260 */
1261 if (cpt_is_errata_38550_exists(pdev)) {
1262 otx2_cpt_read_af_reg(mbox: &cptpf->afpf_mbox, pdev, CPT_AF_DIAG,
1263 val: &reg_val, blkaddr: BLKADDR_CPT0);
1264 otx2_cpt_write_af_reg(mbox: &cptpf->afpf_mbox, pdev, CPT_AF_DIAG,
1265 val: reg_val | BIT_ULL(24), blkaddr: BLKADDR_CPT0);
1266 }
1267
1268 mutex_unlock(lock: &eng_grps->lock);
1269 return 0;
1270
1271delete_eng_grp:
1272 delete_engine_grps(pdev, eng_grps);
1273release_fw:
1274 cpt_ucode_release_fw(fw_info: &fw_info);
1275unlock:
1276 mutex_unlock(lock: &eng_grps->lock);
1277 return ret;
1278}
1279
1280static int cptx_disable_all_cores(struct otx2_cptpf_dev *cptpf, int total_cores,
1281 int blkaddr)
1282{
1283 int timeout = 10, ret;
1284 int i, busy;
1285 u64 reg;
1286
1287 /* Disengage the cores from groups */
1288 for (i = 0; i < total_cores; i++) {
1289 ret = otx2_cpt_add_write_af_reg(mbox: &cptpf->afpf_mbox, pdev: cptpf->pdev,
1290 CPT_AF_EXEX_CTL2(i), val: 0x0,
1291 blkaddr);
1292 if (ret)
1293 return ret;
1294
1295 cptpf->eng_grps.eng_ref_cnt[i] = 0;
1296 }
1297 ret = otx2_cpt_send_af_reg_requests(mbox: &cptpf->afpf_mbox, pdev: cptpf->pdev);
1298 if (ret)
1299 return ret;
1300
1301 /* Wait for cores to become idle */
1302 do {
1303 busy = 0;
1304 usleep_range(min: 10000, max: 20000);
1305 if (timeout-- < 0)
1306 return -EBUSY;
1307
1308 for (i = 0; i < total_cores; i++) {
1309 ret = otx2_cpt_read_af_reg(mbox: &cptpf->afpf_mbox,
1310 pdev: cptpf->pdev,
1311 CPT_AF_EXEX_STS(i), val: &reg,
1312 blkaddr);
1313 if (ret)
1314 return ret;
1315
1316 if (reg & 0x1) {
1317 busy = 1;
1318 break;
1319 }
1320 }
1321 } while (busy);
1322
1323 /* Disable the cores */
1324 for (i = 0; i < total_cores; i++) {
1325 ret = otx2_cpt_add_write_af_reg(mbox: &cptpf->afpf_mbox, pdev: cptpf->pdev,
1326 CPT_AF_EXEX_CTL(i), val: 0x0,
1327 blkaddr);
1328 if (ret)
1329 return ret;
1330 }
1331 return otx2_cpt_send_af_reg_requests(mbox: &cptpf->afpf_mbox, pdev: cptpf->pdev);
1332}
1333
1334int otx2_cpt_disable_all_cores(struct otx2_cptpf_dev *cptpf)
1335{
1336 int total_cores, ret;
1337
1338 total_cores = cptpf->eng_grps.avail.max_se_cnt +
1339 cptpf->eng_grps.avail.max_ie_cnt +
1340 cptpf->eng_grps.avail.max_ae_cnt;
1341
1342 if (cptpf->has_cpt1) {
1343 ret = cptx_disable_all_cores(cptpf, total_cores, blkaddr: BLKADDR_CPT1);
1344 if (ret)
1345 return ret;
1346 }
1347 return cptx_disable_all_cores(cptpf, total_cores, blkaddr: BLKADDR_CPT0);
1348}
1349
1350void otx2_cpt_cleanup_eng_grps(struct pci_dev *pdev,
1351 struct otx2_cpt_eng_grps *eng_grps)
1352{
1353 struct otx2_cpt_eng_grp_info *grp;
1354 int i, j;
1355
1356 mutex_lock(&eng_grps->lock);
1357 delete_engine_grps(pdev, eng_grps);
1358 /* Release memory */
1359 for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) {
1360 grp = &eng_grps->grp[i];
1361 for (j = 0; j < OTX2_CPT_MAX_ETYPES_PER_GRP; j++) {
1362 kfree(objp: grp->engs[j].bmap);
1363 grp->engs[j].bmap = NULL;
1364 }
1365 }
1366 mutex_unlock(lock: &eng_grps->lock);
1367}
1368
1369int otx2_cpt_init_eng_grps(struct pci_dev *pdev,
1370 struct otx2_cpt_eng_grps *eng_grps)
1371{
1372 struct otx2_cpt_eng_grp_info *grp;
1373 int i, j, ret;
1374
1375 mutex_init(&eng_grps->lock);
1376 eng_grps->obj = pci_get_drvdata(pdev);
1377 eng_grps->avail.se_cnt = eng_grps->avail.max_se_cnt;
1378 eng_grps->avail.ie_cnt = eng_grps->avail.max_ie_cnt;
1379 eng_grps->avail.ae_cnt = eng_grps->avail.max_ae_cnt;
1380
1381 eng_grps->engs_num = eng_grps->avail.max_se_cnt +
1382 eng_grps->avail.max_ie_cnt +
1383 eng_grps->avail.max_ae_cnt;
1384 if (eng_grps->engs_num > OTX2_CPT_MAX_ENGINES) {
1385 dev_err(&pdev->dev,
1386 "Number of engines %d > than max supported %d\n",
1387 eng_grps->engs_num, OTX2_CPT_MAX_ENGINES);
1388 ret = -EINVAL;
1389 goto cleanup_eng_grps;
1390 }
1391
1392 for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) {
1393 grp = &eng_grps->grp[i];
1394 grp->g = eng_grps;
1395 grp->idx = i;
1396
1397 for (j = 0; j < OTX2_CPT_MAX_ETYPES_PER_GRP; j++) {
1398 grp->engs[j].bmap =
1399 kcalloc(BITS_TO_LONGS(eng_grps->engs_num),
1400 size: sizeof(long), GFP_KERNEL);
1401 if (!grp->engs[j].bmap) {
1402 ret = -ENOMEM;
1403 goto cleanup_eng_grps;
1404 }
1405 }
1406 }
1407 return 0;
1408
1409cleanup_eng_grps:
1410 otx2_cpt_cleanup_eng_grps(pdev, eng_grps);
1411 return ret;
1412}
1413
1414static int create_eng_caps_discovery_grps(struct pci_dev *pdev,
1415 struct otx2_cpt_eng_grps *eng_grps)
1416{
1417 struct otx2_cpt_uc_info_t *uc_info[OTX2_CPT_MAX_ETYPES_PER_GRP] = { };
1418 struct otx2_cpt_engines engs[OTX2_CPT_MAX_ETYPES_PER_GRP] = { {0} };
1419 struct fw_info_t fw_info;
1420 int ret;
1421
1422 mutex_lock(&eng_grps->lock);
1423 ret = cpt_ucode_load_fw(pdev, fw_info: &fw_info, rid: eng_grps->rid);
1424 if (ret) {
1425 mutex_unlock(lock: &eng_grps->lock);
1426 return ret;
1427 }
1428
1429 uc_info[0] = get_ucode(fw_info: &fw_info, ucode_type: OTX2_CPT_AE_TYPES);
1430 if (uc_info[0] == NULL) {
1431 dev_err(&pdev->dev, "Unable to find firmware for AE\n");
1432 ret = -EINVAL;
1433 goto release_fw;
1434 }
1435 engs[0].type = OTX2_CPT_AE_TYPES;
1436 engs[0].count = 2;
1437
1438 ret = create_engine_group(dev: &pdev->dev, eng_grps, engs, ucodes_cnt: 1,
1439 ucode_data: (void **) uc_info, is_print: 0);
1440 if (ret)
1441 goto release_fw;
1442
1443 uc_info[0] = get_ucode(fw_info: &fw_info, ucode_type: OTX2_CPT_SE_TYPES);
1444 if (uc_info[0] == NULL) {
1445 dev_err(&pdev->dev, "Unable to find firmware for SE\n");
1446 ret = -EINVAL;
1447 goto delete_eng_grp;
1448 }
1449 engs[0].type = OTX2_CPT_SE_TYPES;
1450 engs[0].count = 2;
1451
1452 ret = create_engine_group(dev: &pdev->dev, eng_grps, engs, ucodes_cnt: 1,
1453 ucode_data: (void **) uc_info, is_print: 0);
1454 if (ret)
1455 goto delete_eng_grp;
1456
1457 uc_info[0] = get_ucode(fw_info: &fw_info, ucode_type: OTX2_CPT_IE_TYPES);
1458 if (uc_info[0] == NULL) {
1459 dev_err(&pdev->dev, "Unable to find firmware for IE\n");
1460 ret = -EINVAL;
1461 goto delete_eng_grp;
1462 }
1463 engs[0].type = OTX2_CPT_IE_TYPES;
1464 engs[0].count = 2;
1465
1466 ret = create_engine_group(dev: &pdev->dev, eng_grps, engs, ucodes_cnt: 1,
1467 ucode_data: (void **) uc_info, is_print: 0);
1468 if (ret)
1469 goto delete_eng_grp;
1470
1471 cpt_ucode_release_fw(fw_info: &fw_info);
1472 mutex_unlock(lock: &eng_grps->lock);
1473 return 0;
1474
1475delete_eng_grp:
1476 delete_engine_grps(pdev, eng_grps);
1477release_fw:
1478 cpt_ucode_release_fw(fw_info: &fw_info);
1479 mutex_unlock(lock: &eng_grps->lock);
1480 return ret;
1481}
1482
1483/*
1484 * Get CPT HW capabilities using LOAD_FVC operation.
1485 */
1486int otx2_cpt_discover_eng_capabilities(struct otx2_cptpf_dev *cptpf)
1487{
1488 struct otx2_cptlfs_info *lfs = &cptpf->lfs;
1489 struct otx2_cpt_iq_command iq_cmd;
1490 union otx2_cpt_opcode opcode;
1491 union otx2_cpt_res_s *result;
1492 union otx2_cpt_inst_s inst;
1493 dma_addr_t rptr_baddr;
1494 struct pci_dev *pdev;
1495 u32 len, compl_rlen;
1496 int ret, etype;
1497 void *rptr;
1498
1499 /*
1500 * We don't get capabilities if it was already done
1501 * (when user enabled VFs for the first time)
1502 */
1503 if (cptpf->is_eng_caps_discovered)
1504 return 0;
1505
1506 pdev = cptpf->pdev;
1507 /*
1508 * Create engine groups for each type to submit LOAD_FVC op and
1509 * get engine's capabilities.
1510 */
1511 ret = create_eng_caps_discovery_grps(pdev, eng_grps: &cptpf->eng_grps);
1512 if (ret)
1513 goto delete_grps;
1514
1515 otx2_cptlf_set_dev_info(lfs, pdev: cptpf->pdev, reg_base: cptpf->reg_base,
1516 mbox: &cptpf->afpf_mbox, blkaddr: BLKADDR_CPT0);
1517 ret = otx2_cptlf_init(lfs, OTX2_CPT_ALL_ENG_GRPS_MASK,
1518 OTX2_CPT_QUEUE_HI_PRIO, lfs_num: 1);
1519 if (ret)
1520 goto delete_grps;
1521
1522 compl_rlen = ALIGN(sizeof(union otx2_cpt_res_s), OTX2_CPT_DMA_MINALIGN);
1523 len = compl_rlen + LOADFVC_RLEN;
1524
1525 result = kzalloc(size: len, GFP_KERNEL);
1526 if (!result) {
1527 ret = -ENOMEM;
1528 goto lf_cleanup;
1529 }
1530 rptr_baddr = dma_map_single(&pdev->dev, (void *)result, len,
1531 DMA_BIDIRECTIONAL);
1532 if (dma_mapping_error(dev: &pdev->dev, dma_addr: rptr_baddr)) {
1533 dev_err(&pdev->dev, "DMA mapping failed\n");
1534 ret = -EFAULT;
1535 goto free_result;
1536 }
1537 rptr = (u8 *)result + compl_rlen;
1538
1539 /* Fill in the command */
1540 opcode.s.major = LOADFVC_MAJOR_OP;
1541 opcode.s.minor = LOADFVC_MINOR_OP;
1542
1543 iq_cmd.cmd.u = 0;
1544 iq_cmd.cmd.s.opcode = cpu_to_be16(opcode.flags);
1545
1546 /* 64-bit swap for microcode data reads, not needed for addresses */
1547 cpu_to_be64s(&iq_cmd.cmd.u);
1548 iq_cmd.dptr = 0;
1549 iq_cmd.rptr = rptr_baddr + compl_rlen;
1550 iq_cmd.cptr.u = 0;
1551
1552 for (etype = 1; etype < OTX2_CPT_MAX_ENG_TYPES; etype++) {
1553 result->s.compcode = OTX2_CPT_COMPLETION_CODE_INIT;
1554 iq_cmd.cptr.s.grp = otx2_cpt_get_eng_grp(eng_grps: &cptpf->eng_grps,
1555 eng_type: etype);
1556 otx2_cpt_fill_inst(cptinst: &inst, iq_cmd: &iq_cmd, comp_baddr: rptr_baddr);
1557 lfs->ops->send_cmd(&inst, 1, &cptpf->lfs.lf[0]);
1558
1559 while (lfs->ops->cpt_get_compcode(result) ==
1560 OTX2_CPT_COMPLETION_CODE_INIT)
1561 cpu_relax();
1562
1563 cptpf->eng_caps[etype].u = be64_to_cpup(p: rptr);
1564 }
1565 dma_unmap_single(&pdev->dev, rptr_baddr, len, DMA_BIDIRECTIONAL);
1566 cptpf->is_eng_caps_discovered = true;
1567
1568free_result:
1569 kfree(objp: result);
1570lf_cleanup:
1571 otx2_cptlf_shutdown(lfs);
1572delete_grps:
1573 delete_engine_grps(pdev, eng_grps: &cptpf->eng_grps);
1574
1575 return ret;
1576}
1577
1578int otx2_cpt_dl_custom_egrp_create(struct otx2_cptpf_dev *cptpf,
1579 struct devlink_param_gset_ctx *ctx)
1580{
1581 struct otx2_cpt_engines engs[OTX2_CPT_MAX_ETYPES_PER_GRP] = { { 0 } };
1582 struct otx2_cpt_uc_info_t *uc_info[OTX2_CPT_MAX_ETYPES_PER_GRP] = {};
1583 struct otx2_cpt_eng_grps *eng_grps = &cptpf->eng_grps;
1584 char *ucode_filename[OTX2_CPT_MAX_ETYPES_PER_GRP];
1585 char tmp_buf[OTX2_CPT_NAME_LENGTH] = { 0 };
1586 struct device *dev = &cptpf->pdev->dev;
1587 char *start, *val, *err_msg, *tmp;
1588 int grp_idx = 0, ret = -EINVAL;
1589 bool has_se, has_ie, has_ae;
1590 struct fw_info_t fw_info;
1591 int ucode_idx = 0;
1592
1593 if (!eng_grps->is_grps_created) {
1594 dev_err(dev, "Not allowed before creating the default groups\n");
1595 return -EINVAL;
1596 }
1597 err_msg = "Invalid engine group format";
1598 strscpy(tmp_buf, ctx->val.vstr, strlen(ctx->val.vstr) + 1);
1599 start = tmp_buf;
1600
1601 has_se = has_ie = has_ae = false;
1602
1603 for (;;) {
1604 val = strsep(&start, ";");
1605 if (!val)
1606 break;
1607 val = strim(val);
1608 if (!*val)
1609 continue;
1610
1611 if (!strncasecmp(s1: val, s2: "se", n: 2) && strchr(val, ':')) {
1612 if (has_se || ucode_idx)
1613 goto err_print;
1614 tmp = strsep(&val, ":");
1615 if (!tmp)
1616 goto err_print;
1617 tmp = strim(tmp);
1618 if (!val)
1619 goto err_print;
1620 if (strlen(tmp) != 2)
1621 goto err_print;
1622 if (kstrtoint(s: strim(val), base: 10, res: &engs[grp_idx].count))
1623 goto err_print;
1624 engs[grp_idx++].type = OTX2_CPT_SE_TYPES;
1625 has_se = true;
1626 } else if (!strncasecmp(s1: val, s2: "ae", n: 2) && strchr(val, ':')) {
1627 if (has_ae || ucode_idx)
1628 goto err_print;
1629 tmp = strsep(&val, ":");
1630 if (!tmp)
1631 goto err_print;
1632 tmp = strim(tmp);
1633 if (!val)
1634 goto err_print;
1635 if (strlen(tmp) != 2)
1636 goto err_print;
1637 if (kstrtoint(s: strim(val), base: 10, res: &engs[grp_idx].count))
1638 goto err_print;
1639 engs[grp_idx++].type = OTX2_CPT_AE_TYPES;
1640 has_ae = true;
1641 } else if (!strncasecmp(s1: val, s2: "ie", n: 2) && strchr(val, ':')) {
1642 if (has_ie || ucode_idx)
1643 goto err_print;
1644 tmp = strsep(&val, ":");
1645 if (!tmp)
1646 goto err_print;
1647 tmp = strim(tmp);
1648 if (!val)
1649 goto err_print;
1650 if (strlen(tmp) != 2)
1651 goto err_print;
1652 if (kstrtoint(s: strim(val), base: 10, res: &engs[grp_idx].count))
1653 goto err_print;
1654 engs[grp_idx++].type = OTX2_CPT_IE_TYPES;
1655 has_ie = true;
1656 } else {
1657 if (ucode_idx > 1)
1658 goto err_print;
1659 if (!strlen(val))
1660 goto err_print;
1661 if (strnstr(val, " ", strlen(val)))
1662 goto err_print;
1663 ucode_filename[ucode_idx++] = val;
1664 }
1665 }
1666
1667 /* Validate input parameters */
1668 if (!(grp_idx && ucode_idx))
1669 goto err_print;
1670
1671 if (ucode_idx > 1 && grp_idx < 2)
1672 goto err_print;
1673
1674 if (grp_idx > OTX2_CPT_MAX_ETYPES_PER_GRP) {
1675 err_msg = "Error max 2 engine types can be attached";
1676 goto err_print;
1677 }
1678
1679 if (grp_idx > 1) {
1680 if ((engs[0].type + engs[1].type) !=
1681 (OTX2_CPT_SE_TYPES + OTX2_CPT_IE_TYPES)) {
1682 err_msg = "Only combination of SE+IE engines is allowed";
1683 goto err_print;
1684 }
1685 /* Keep SE engines at zero index */
1686 if (engs[1].type == OTX2_CPT_SE_TYPES)
1687 swap(engs[0], engs[1]);
1688 }
1689 mutex_lock(&eng_grps->lock);
1690
1691 if (cptpf->enabled_vfs) {
1692 dev_err(dev, "Disable VFs before modifying engine groups\n");
1693 ret = -EACCES;
1694 goto err_unlock;
1695 }
1696 INIT_LIST_HEAD(list: &fw_info.ucodes);
1697
1698 ret = load_fw(dev, fw_info: &fw_info, filename: ucode_filename[0], rid: eng_grps->rid);
1699 if (ret) {
1700 dev_err(dev, "Unable to load firmware %s\n", ucode_filename[0]);
1701 goto err_unlock;
1702 }
1703 if (ucode_idx > 1) {
1704 ret = load_fw(dev, fw_info: &fw_info, filename: ucode_filename[1], rid: eng_grps->rid);
1705 if (ret) {
1706 dev_err(dev, "Unable to load firmware %s\n",
1707 ucode_filename[1]);
1708 goto release_fw;
1709 }
1710 }
1711 uc_info[0] = get_ucode(fw_info: &fw_info, ucode_type: engs[0].type);
1712 if (uc_info[0] == NULL) {
1713 dev_err(dev, "Unable to find firmware for %s\n",
1714 get_eng_type_str(engs[0].type));
1715 ret = -EINVAL;
1716 goto release_fw;
1717 }
1718 if (ucode_idx > 1) {
1719 uc_info[1] = get_ucode(fw_info: &fw_info, ucode_type: engs[1].type);
1720 if (uc_info[1] == NULL) {
1721 dev_err(dev, "Unable to find firmware for %s\n",
1722 get_eng_type_str(engs[1].type));
1723 ret = -EINVAL;
1724 goto release_fw;
1725 }
1726 }
1727 ret = create_engine_group(dev, eng_grps, engs, ucodes_cnt: grp_idx,
1728 ucode_data: (void **)uc_info, is_print: 1);
1729
1730release_fw:
1731 cpt_ucode_release_fw(fw_info: &fw_info);
1732err_unlock:
1733 mutex_unlock(lock: &eng_grps->lock);
1734 return ret;
1735err_print:
1736 dev_err(dev, "%s\n", err_msg);
1737 return ret;
1738}
1739
1740int otx2_cpt_dl_custom_egrp_delete(struct otx2_cptpf_dev *cptpf,
1741 struct devlink_param_gset_ctx *ctx)
1742{
1743 struct otx2_cpt_eng_grps *eng_grps = &cptpf->eng_grps;
1744 struct device *dev = &cptpf->pdev->dev;
1745 char *tmp, *err_msg;
1746 int egrp;
1747 int ret;
1748
1749 err_msg = "Invalid input string format(ex: egrp:0)";
1750 if (strncasecmp(s1: ctx->val.vstr, s2: "egrp", n: 4))
1751 goto err_print;
1752 tmp = ctx->val.vstr;
1753 strsep(&tmp, ":");
1754 if (!tmp)
1755 goto err_print;
1756 if (kstrtoint(s: tmp, base: 10, res: &egrp))
1757 goto err_print;
1758
1759 if (egrp < 0 || egrp >= OTX2_CPT_MAX_ENGINE_GROUPS) {
1760 dev_err(dev, "Invalid engine group %d", egrp);
1761 return -EINVAL;
1762 }
1763 if (!eng_grps->grp[egrp].is_enabled) {
1764 dev_err(dev, "Error engine_group%d is not configured", egrp);
1765 return -EINVAL;
1766 }
1767 mutex_lock(&eng_grps->lock);
1768 ret = delete_engine_group(dev, eng_grp: &eng_grps->grp[egrp]);
1769 mutex_unlock(lock: &eng_grps->lock);
1770
1771 return ret;
1772
1773err_print:
1774 dev_err(dev, "%s\n", err_msg);
1775 return -EINVAL;
1776}
1777
1778static void get_engs_info(struct otx2_cpt_eng_grp_info *eng_grp, char *buf,
1779 int size, int idx)
1780{
1781 struct otx2_cpt_engs_rsvd *mirrored_engs = NULL;
1782 struct otx2_cpt_engs_rsvd *engs;
1783 int len, i;
1784
1785 buf[0] = '\0';
1786 for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) {
1787 engs = &eng_grp->engs[i];
1788 if (!engs->type)
1789 continue;
1790 if (idx != -1 && idx != i)
1791 continue;
1792
1793 if (eng_grp->mirror.is_ena)
1794 mirrored_engs = find_engines_by_type(
1795 eng_grp: &eng_grp->g->grp[eng_grp->mirror.idx],
1796 eng_type: engs->type);
1797 if (i > 0 && idx == -1) {
1798 len = strlen(buf);
1799 scnprintf(buf: buf + len, size: size - len, fmt: ", ");
1800 }
1801
1802 len = strlen(buf);
1803 scnprintf(buf: buf + len, size: size - len, fmt: "%d %s ",
1804 mirrored_engs ? engs->count + mirrored_engs->count :
1805 engs->count,
1806 get_eng_type_str(eng_type: engs->type));
1807 if (mirrored_engs) {
1808 len = strlen(buf);
1809 scnprintf(buf: buf + len, size: size - len,
1810 fmt: "(%d shared with engine_group%d) ",
1811 engs->count <= 0 ?
1812 engs->count + mirrored_engs->count :
1813 mirrored_engs->count,
1814 eng_grp->mirror.idx);
1815 }
1816 }
1817}
1818
1819void otx2_cpt_print_uc_dbg_info(struct otx2_cptpf_dev *cptpf)
1820{
1821 struct otx2_cpt_eng_grps *eng_grps = &cptpf->eng_grps;
1822 struct otx2_cpt_eng_grp_info *mirrored_grp;
1823 char engs_info[2 * OTX2_CPT_NAME_LENGTH];
1824 struct otx2_cpt_eng_grp_info *grp;
1825 struct otx2_cpt_engs_rsvd *engs;
1826 int i, j;
1827
1828 pr_debug("Engine groups global info");
1829 pr_debug("max SE %d, max IE %d, max AE %d", eng_grps->avail.max_se_cnt,
1830 eng_grps->avail.max_ie_cnt, eng_grps->avail.max_ae_cnt);
1831 pr_debug("free SE %d", eng_grps->avail.se_cnt);
1832 pr_debug("free IE %d", eng_grps->avail.ie_cnt);
1833 pr_debug("free AE %d", eng_grps->avail.ae_cnt);
1834
1835 for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) {
1836 grp = &eng_grps->grp[i];
1837 pr_debug("engine_group%d, state %s", i,
1838 grp->is_enabled ? "enabled" : "disabled");
1839 if (grp->is_enabled) {
1840 mirrored_grp = &eng_grps->grp[grp->mirror.idx];
1841 pr_debug("Ucode0 filename %s, version %s",
1842 grp->mirror.is_ena ?
1843 mirrored_grp->ucode[0].filename :
1844 grp->ucode[0].filename,
1845 grp->mirror.is_ena ?
1846 mirrored_grp->ucode[0].ver_str :
1847 grp->ucode[0].ver_str);
1848 if (is_2nd_ucode_used(eng_grp: grp))
1849 pr_debug("Ucode1 filename %s, version %s",
1850 grp->ucode[1].filename,
1851 grp->ucode[1].ver_str);
1852 }
1853
1854 for (j = 0; j < OTX2_CPT_MAX_ETYPES_PER_GRP; j++) {
1855 engs = &grp->engs[j];
1856 if (engs->type) {
1857 u32 mask[5] = { };
1858
1859 get_engs_info(eng_grp: grp, buf: engs_info,
1860 size: 2 * OTX2_CPT_NAME_LENGTH, idx: j);
1861 pr_debug("Slot%d: %s", j, engs_info);
1862 bitmap_to_arr32(buf: mask, bitmap: engs->bmap,
1863 nbits: eng_grps->engs_num);
1864 if (is_dev_otx2(pdev: cptpf->pdev))
1865 pr_debug("Mask: %8.8x %8.8x %8.8x %8.8x",
1866 mask[3], mask[2], mask[1],
1867 mask[0]);
1868 else
1869 pr_debug("Mask: %8.8x %8.8x %8.8x %8.8x %8.8x",
1870 mask[4], mask[3], mask[2], mask[1],
1871 mask[0]);
1872 }
1873 }
1874 }
1875}
1876

source code of linux/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c