1// SPDX-License-Identifier: GPL-2.0
2/* Marvell OcteonTX CPT driver
3 *
4 * Copyright (C) 2019 Marvell International Ltd.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11#include <linux/interrupt.h>
12#include <linux/module.h>
13#include "otx_cptvf.h"
14#include "otx_cptvf_algs.h"
15#include "otx_cptvf_reqmgr.h"
16
17#define DRV_NAME "octeontx-cptvf"
18#define DRV_VERSION "1.0"
19
20static void vq_work_handler(unsigned long data)
21{
22 struct otx_cptvf_wqe_info *cwqe_info =
23 (struct otx_cptvf_wqe_info *) data;
24
25 otx_cpt_post_process(wqe: &cwqe_info->vq_wqe[0]);
26}
27
28static int init_worker_threads(struct otx_cptvf *cptvf)
29{
30 struct pci_dev *pdev = cptvf->pdev;
31 struct otx_cptvf_wqe_info *cwqe_info;
32 int i;
33
34 cwqe_info = kzalloc(size: sizeof(*cwqe_info), GFP_KERNEL);
35 if (!cwqe_info)
36 return -ENOMEM;
37
38 if (cptvf->num_queues) {
39 dev_dbg(&pdev->dev, "Creating VQ worker threads (%d)\n",
40 cptvf->num_queues);
41 }
42
43 for (i = 0; i < cptvf->num_queues; i++) {
44 tasklet_init(t: &cwqe_info->vq_wqe[i].twork, func: vq_work_handler,
45 data: (u64)cwqe_info);
46 cwqe_info->vq_wqe[i].cptvf = cptvf;
47 }
48 cptvf->wqe_info = cwqe_info;
49
50 return 0;
51}
52
53static void cleanup_worker_threads(struct otx_cptvf *cptvf)
54{
55 struct pci_dev *pdev = cptvf->pdev;
56 struct otx_cptvf_wqe_info *cwqe_info;
57 int i;
58
59 cwqe_info = (struct otx_cptvf_wqe_info *)cptvf->wqe_info;
60 if (!cwqe_info)
61 return;
62
63 if (cptvf->num_queues) {
64 dev_dbg(&pdev->dev, "Cleaning VQ worker threads (%u)\n",
65 cptvf->num_queues);
66 }
67
68 for (i = 0; i < cptvf->num_queues; i++)
69 tasklet_kill(t: &cwqe_info->vq_wqe[i].twork);
70
71 kfree_sensitive(objp: cwqe_info);
72 cptvf->wqe_info = NULL;
73}
74
75static void free_pending_queues(struct otx_cpt_pending_qinfo *pqinfo)
76{
77 struct otx_cpt_pending_queue *queue;
78 int i;
79
80 for_each_pending_queue(pqinfo, queue, i) {
81 if (!queue->head)
82 continue;
83
84 /* free single queue */
85 kfree_sensitive(objp: (queue->head));
86 queue->front = 0;
87 queue->rear = 0;
88 queue->qlen = 0;
89 }
90 pqinfo->num_queues = 0;
91}
92
93static int alloc_pending_queues(struct otx_cpt_pending_qinfo *pqinfo, u32 qlen,
94 u32 num_queues)
95{
96 struct otx_cpt_pending_queue *queue = NULL;
97 int ret;
98 u32 i;
99
100 pqinfo->num_queues = num_queues;
101
102 for_each_pending_queue(pqinfo, queue, i) {
103 queue->head = kcalloc(n: qlen, size: sizeof(*queue->head), GFP_KERNEL);
104 if (!queue->head) {
105 ret = -ENOMEM;
106 goto pending_qfail;
107 }
108
109 queue->pending_count = 0;
110 queue->front = 0;
111 queue->rear = 0;
112 queue->qlen = qlen;
113
114 /* init queue spin lock */
115 spin_lock_init(&queue->lock);
116 }
117 return 0;
118
119pending_qfail:
120 free_pending_queues(pqinfo);
121
122 return ret;
123}
124
125static int init_pending_queues(struct otx_cptvf *cptvf, u32 qlen,
126 u32 num_queues)
127{
128 struct pci_dev *pdev = cptvf->pdev;
129 int ret;
130
131 if (!num_queues)
132 return 0;
133
134 ret = alloc_pending_queues(pqinfo: &cptvf->pqinfo, qlen, num_queues);
135 if (ret) {
136 dev_err(&pdev->dev, "Failed to setup pending queues (%u)\n",
137 num_queues);
138 return ret;
139 }
140 return 0;
141}
142
143static void cleanup_pending_queues(struct otx_cptvf *cptvf)
144{
145 struct pci_dev *pdev = cptvf->pdev;
146
147 if (!cptvf->num_queues)
148 return;
149
150 dev_dbg(&pdev->dev, "Cleaning VQ pending queue (%u)\n",
151 cptvf->num_queues);
152 free_pending_queues(pqinfo: &cptvf->pqinfo);
153}
154
155static void free_command_queues(struct otx_cptvf *cptvf,
156 struct otx_cpt_cmd_qinfo *cqinfo)
157{
158 struct otx_cpt_cmd_queue *queue = NULL;
159 struct otx_cpt_cmd_chunk *chunk = NULL;
160 struct pci_dev *pdev = cptvf->pdev;
161 int i;
162
163 /* clean up for each queue */
164 for (i = 0; i < cptvf->num_queues; i++) {
165 queue = &cqinfo->queue[i];
166
167 while (!list_empty(head: &cqinfo->queue[i].chead)) {
168 chunk = list_first_entry(&cqinfo->queue[i].chead,
169 struct otx_cpt_cmd_chunk, nextchunk);
170
171 dma_free_coherent(dev: &pdev->dev, size: chunk->size,
172 cpu_addr: chunk->head,
173 dma_handle: chunk->dma_addr);
174 chunk->head = NULL;
175 chunk->dma_addr = 0;
176 list_del(entry: &chunk->nextchunk);
177 kfree_sensitive(objp: chunk);
178 }
179 queue->num_chunks = 0;
180 queue->idx = 0;
181
182 }
183}
184
185static int alloc_command_queues(struct otx_cptvf *cptvf,
186 struct otx_cpt_cmd_qinfo *cqinfo,
187 u32 qlen)
188{
189 struct otx_cpt_cmd_chunk *curr, *first, *last;
190 struct otx_cpt_cmd_queue *queue = NULL;
191 struct pci_dev *pdev = cptvf->pdev;
192 size_t q_size, c_size, rem_q_size;
193 u32 qcsize_bytes;
194 int i;
195
196
197 /* Qsize in dwords, needed for SADDR config, 1-next chunk pointer */
198 cptvf->qsize = min(qlen, cqinfo->qchunksize) *
199 OTX_CPT_NEXT_CHUNK_PTR_SIZE + 1;
200 /* Qsize in bytes to create space for alignment */
201 q_size = qlen * OTX_CPT_INST_SIZE;
202
203 qcsize_bytes = cqinfo->qchunksize * OTX_CPT_INST_SIZE;
204
205 /* per queue initialization */
206 for (i = 0; i < cptvf->num_queues; i++) {
207 rem_q_size = q_size;
208 first = NULL;
209 last = NULL;
210
211 queue = &cqinfo->queue[i];
212 INIT_LIST_HEAD(list: &queue->chead);
213 do {
214 curr = kzalloc(size: sizeof(*curr), GFP_KERNEL);
215 if (!curr)
216 goto cmd_qfail;
217
218 c_size = (rem_q_size > qcsize_bytes) ? qcsize_bytes :
219 rem_q_size;
220 curr->head = dma_alloc_coherent(dev: &pdev->dev,
221 size: c_size + OTX_CPT_NEXT_CHUNK_PTR_SIZE,
222 dma_handle: &curr->dma_addr, GFP_KERNEL);
223 if (!curr->head) {
224 dev_err(&pdev->dev,
225 "Command Q (%d) chunk (%d) allocation failed\n",
226 i, queue->num_chunks);
227 goto free_curr;
228 }
229 curr->size = c_size;
230
231 if (queue->num_chunks == 0) {
232 first = curr;
233 queue->base = first;
234 }
235 list_add_tail(new: &curr->nextchunk,
236 head: &cqinfo->queue[i].chead);
237
238 queue->num_chunks++;
239 rem_q_size -= c_size;
240 if (last)
241 *((u64 *)(&last->head[last->size])) =
242 (u64)curr->dma_addr;
243
244 last = curr;
245 } while (rem_q_size);
246
247 /*
248 * Make the queue circular, tie back last chunk entry to head
249 */
250 curr = first;
251 *((u64 *)(&last->head[last->size])) = (u64)curr->dma_addr;
252 queue->qhead = curr;
253 }
254 return 0;
255free_curr:
256 kfree(objp: curr);
257cmd_qfail:
258 free_command_queues(cptvf, cqinfo);
259 return -ENOMEM;
260}
261
262static int init_command_queues(struct otx_cptvf *cptvf, u32 qlen)
263{
264 struct pci_dev *pdev = cptvf->pdev;
265 int ret;
266
267 /* setup command queues */
268 ret = alloc_command_queues(cptvf, cqinfo: &cptvf->cqinfo, qlen);
269 if (ret) {
270 dev_err(&pdev->dev, "Failed to allocate command queues (%u)\n",
271 cptvf->num_queues);
272 return ret;
273 }
274 return ret;
275}
276
277static void cleanup_command_queues(struct otx_cptvf *cptvf)
278{
279 struct pci_dev *pdev = cptvf->pdev;
280
281 if (!cptvf->num_queues)
282 return;
283
284 dev_dbg(&pdev->dev, "Cleaning VQ command queue (%u)\n",
285 cptvf->num_queues);
286 free_command_queues(cptvf, cqinfo: &cptvf->cqinfo);
287}
288
289static void cptvf_sw_cleanup(struct otx_cptvf *cptvf)
290{
291 cleanup_worker_threads(cptvf);
292 cleanup_pending_queues(cptvf);
293 cleanup_command_queues(cptvf);
294}
295
296static int cptvf_sw_init(struct otx_cptvf *cptvf, u32 qlen, u32 num_queues)
297{
298 struct pci_dev *pdev = cptvf->pdev;
299 u32 max_dev_queues = 0;
300 int ret;
301
302 max_dev_queues = OTX_CPT_NUM_QS_PER_VF;
303 /* possible cpus */
304 num_queues = min_t(u32, num_queues, max_dev_queues);
305 cptvf->num_queues = num_queues;
306
307 ret = init_command_queues(cptvf, qlen);
308 if (ret) {
309 dev_err(&pdev->dev, "Failed to setup command queues (%u)\n",
310 num_queues);
311 return ret;
312 }
313
314 ret = init_pending_queues(cptvf, qlen, num_queues);
315 if (ret) {
316 dev_err(&pdev->dev, "Failed to setup pending queues (%u)\n",
317 num_queues);
318 goto setup_pqfail;
319 }
320
321 /* Create worker threads for BH processing */
322 ret = init_worker_threads(cptvf);
323 if (ret) {
324 dev_err(&pdev->dev, "Failed to setup worker threads\n");
325 goto init_work_fail;
326 }
327 return 0;
328
329init_work_fail:
330 cleanup_worker_threads(cptvf);
331 cleanup_pending_queues(cptvf);
332
333setup_pqfail:
334 cleanup_command_queues(cptvf);
335
336 return ret;
337}
338
339static void cptvf_free_irq_affinity(struct otx_cptvf *cptvf, int vec)
340{
341 irq_set_affinity_hint(irq: pci_irq_vector(dev: cptvf->pdev, nr: vec), NULL);
342 free_cpumask_var(mask: cptvf->affinity_mask[vec]);
343}
344
345static void cptvf_write_vq_ctl(struct otx_cptvf *cptvf, bool val)
346{
347 union otx_cptx_vqx_ctl vqx_ctl;
348
349 vqx_ctl.u = readq(addr: cptvf->reg_base + OTX_CPT_VQX_CTL(0));
350 vqx_ctl.s.ena = val;
351 writeq(val: vqx_ctl.u, addr: cptvf->reg_base + OTX_CPT_VQX_CTL(0));
352}
353
354void otx_cptvf_write_vq_doorbell(struct otx_cptvf *cptvf, u32 val)
355{
356 union otx_cptx_vqx_doorbell vqx_dbell;
357
358 vqx_dbell.u = readq(addr: cptvf->reg_base + OTX_CPT_VQX_DOORBELL(0));
359 vqx_dbell.s.dbell_cnt = val * 8; /* Num of Instructions * 8 words */
360 writeq(val: vqx_dbell.u, addr: cptvf->reg_base + OTX_CPT_VQX_DOORBELL(0));
361}
362
363static void cptvf_write_vq_inprog(struct otx_cptvf *cptvf, u8 val)
364{
365 union otx_cptx_vqx_inprog vqx_inprg;
366
367 vqx_inprg.u = readq(addr: cptvf->reg_base + OTX_CPT_VQX_INPROG(0));
368 vqx_inprg.s.inflight = val;
369 writeq(val: vqx_inprg.u, addr: cptvf->reg_base + OTX_CPT_VQX_INPROG(0));
370}
371
372static void cptvf_write_vq_done_numwait(struct otx_cptvf *cptvf, u32 val)
373{
374 union otx_cptx_vqx_done_wait vqx_dwait;
375
376 vqx_dwait.u = readq(addr: cptvf->reg_base + OTX_CPT_VQX_DONE_WAIT(0));
377 vqx_dwait.s.num_wait = val;
378 writeq(val: vqx_dwait.u, addr: cptvf->reg_base + OTX_CPT_VQX_DONE_WAIT(0));
379}
380
381static u32 cptvf_read_vq_done_numwait(struct otx_cptvf *cptvf)
382{
383 union otx_cptx_vqx_done_wait vqx_dwait;
384
385 vqx_dwait.u = readq(addr: cptvf->reg_base + OTX_CPT_VQX_DONE_WAIT(0));
386 return vqx_dwait.s.num_wait;
387}
388
389static void cptvf_write_vq_done_timewait(struct otx_cptvf *cptvf, u16 time)
390{
391 union otx_cptx_vqx_done_wait vqx_dwait;
392
393 vqx_dwait.u = readq(addr: cptvf->reg_base + OTX_CPT_VQX_DONE_WAIT(0));
394 vqx_dwait.s.time_wait = time;
395 writeq(val: vqx_dwait.u, addr: cptvf->reg_base + OTX_CPT_VQX_DONE_WAIT(0));
396}
397
398
399static u16 cptvf_read_vq_done_timewait(struct otx_cptvf *cptvf)
400{
401 union otx_cptx_vqx_done_wait vqx_dwait;
402
403 vqx_dwait.u = readq(addr: cptvf->reg_base + OTX_CPT_VQX_DONE_WAIT(0));
404 return vqx_dwait.s.time_wait;
405}
406
407static void cptvf_enable_swerr_interrupts(struct otx_cptvf *cptvf)
408{
409 union otx_cptx_vqx_misc_ena_w1s vqx_misc_ena;
410
411 vqx_misc_ena.u = readq(addr: cptvf->reg_base + OTX_CPT_VQX_MISC_ENA_W1S(0));
412 /* Enable SWERR interrupts for the requested VF */
413 vqx_misc_ena.s.swerr = 1;
414 writeq(val: vqx_misc_ena.u, addr: cptvf->reg_base + OTX_CPT_VQX_MISC_ENA_W1S(0));
415}
416
417static void cptvf_enable_mbox_interrupts(struct otx_cptvf *cptvf)
418{
419 union otx_cptx_vqx_misc_ena_w1s vqx_misc_ena;
420
421 vqx_misc_ena.u = readq(addr: cptvf->reg_base + OTX_CPT_VQX_MISC_ENA_W1S(0));
422 /* Enable MBOX interrupt for the requested VF */
423 vqx_misc_ena.s.mbox = 1;
424 writeq(val: vqx_misc_ena.u, addr: cptvf->reg_base + OTX_CPT_VQX_MISC_ENA_W1S(0));
425}
426
427static void cptvf_enable_done_interrupts(struct otx_cptvf *cptvf)
428{
429 union otx_cptx_vqx_done_ena_w1s vqx_done_ena;
430
431 vqx_done_ena.u = readq(addr: cptvf->reg_base + OTX_CPT_VQX_DONE_ENA_W1S(0));
432 /* Enable DONE interrupt for the requested VF */
433 vqx_done_ena.s.done = 1;
434 writeq(val: vqx_done_ena.u, addr: cptvf->reg_base + OTX_CPT_VQX_DONE_ENA_W1S(0));
435}
436
437static void cptvf_clear_dovf_intr(struct otx_cptvf *cptvf)
438{
439 union otx_cptx_vqx_misc_int vqx_misc_int;
440
441 vqx_misc_int.u = readq(addr: cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0));
442 /* W1C for the VF */
443 vqx_misc_int.s.dovf = 1;
444 writeq(val: vqx_misc_int.u, addr: cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0));
445}
446
447static void cptvf_clear_irde_intr(struct otx_cptvf *cptvf)
448{
449 union otx_cptx_vqx_misc_int vqx_misc_int;
450
451 vqx_misc_int.u = readq(addr: cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0));
452 /* W1C for the VF */
453 vqx_misc_int.s.irde = 1;
454 writeq(val: vqx_misc_int.u, addr: cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0));
455}
456
457static void cptvf_clear_nwrp_intr(struct otx_cptvf *cptvf)
458{
459 union otx_cptx_vqx_misc_int vqx_misc_int;
460
461 vqx_misc_int.u = readq(addr: cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0));
462 /* W1C for the VF */
463 vqx_misc_int.s.nwrp = 1;
464 writeq(val: vqx_misc_int.u, addr: cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0));
465}
466
467static void cptvf_clear_mbox_intr(struct otx_cptvf *cptvf)
468{
469 union otx_cptx_vqx_misc_int vqx_misc_int;
470
471 vqx_misc_int.u = readq(addr: cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0));
472 /* W1C for the VF */
473 vqx_misc_int.s.mbox = 1;
474 writeq(val: vqx_misc_int.u, addr: cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0));
475}
476
477static void cptvf_clear_swerr_intr(struct otx_cptvf *cptvf)
478{
479 union otx_cptx_vqx_misc_int vqx_misc_int;
480
481 vqx_misc_int.u = readq(addr: cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0));
482 /* W1C for the VF */
483 vqx_misc_int.s.swerr = 1;
484 writeq(val: vqx_misc_int.u, addr: cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0));
485}
486
487static u64 cptvf_read_vf_misc_intr_status(struct otx_cptvf *cptvf)
488{
489 return readq(addr: cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0));
490}
491
492static irqreturn_t cptvf_misc_intr_handler(int __always_unused irq,
493 void *arg)
494{
495 struct otx_cptvf *cptvf = arg;
496 struct pci_dev *pdev = cptvf->pdev;
497 u64 intr;
498
499 intr = cptvf_read_vf_misc_intr_status(cptvf);
500 /* Check for MISC interrupt types */
501 if (likely(intr & OTX_CPT_VF_INTR_MBOX_MASK)) {
502 dev_dbg(&pdev->dev, "Mailbox interrupt 0x%llx on CPT VF %d\n",
503 intr, cptvf->vfid);
504 otx_cptvf_handle_mbox_intr(cptvf);
505 cptvf_clear_mbox_intr(cptvf);
506 } else if (unlikely(intr & OTX_CPT_VF_INTR_DOVF_MASK)) {
507 cptvf_clear_dovf_intr(cptvf);
508 /* Clear doorbell count */
509 otx_cptvf_write_vq_doorbell(cptvf, val: 0);
510 dev_err(&pdev->dev,
511 "Doorbell overflow error interrupt 0x%llx on CPT VF %d\n",
512 intr, cptvf->vfid);
513 } else if (unlikely(intr & OTX_CPT_VF_INTR_IRDE_MASK)) {
514 cptvf_clear_irde_intr(cptvf);
515 dev_err(&pdev->dev,
516 "Instruction NCB read error interrupt 0x%llx on CPT VF %d\n",
517 intr, cptvf->vfid);
518 } else if (unlikely(intr & OTX_CPT_VF_INTR_NWRP_MASK)) {
519 cptvf_clear_nwrp_intr(cptvf);
520 dev_err(&pdev->dev,
521 "NCB response write error interrupt 0x%llx on CPT VF %d\n",
522 intr, cptvf->vfid);
523 } else if (unlikely(intr & OTX_CPT_VF_INTR_SERR_MASK)) {
524 cptvf_clear_swerr_intr(cptvf);
525 dev_err(&pdev->dev,
526 "Software error interrupt 0x%llx on CPT VF %d\n",
527 intr, cptvf->vfid);
528 } else {
529 dev_err(&pdev->dev, "Unhandled interrupt in OTX_CPT VF %d\n",
530 cptvf->vfid);
531 }
532
533 return IRQ_HANDLED;
534}
535
536static inline struct otx_cptvf_wqe *get_cptvf_vq_wqe(struct otx_cptvf *cptvf,
537 int qno)
538{
539 struct otx_cptvf_wqe_info *nwqe_info;
540
541 if (unlikely(qno >= cptvf->num_queues))
542 return NULL;
543 nwqe_info = (struct otx_cptvf_wqe_info *)cptvf->wqe_info;
544
545 return &nwqe_info->vq_wqe[qno];
546}
547
548static inline u32 cptvf_read_vq_done_count(struct otx_cptvf *cptvf)
549{
550 union otx_cptx_vqx_done vqx_done;
551
552 vqx_done.u = readq(addr: cptvf->reg_base + OTX_CPT_VQX_DONE(0));
553 return vqx_done.s.done;
554}
555
556static inline void cptvf_write_vq_done_ack(struct otx_cptvf *cptvf,
557 u32 ackcnt)
558{
559 union otx_cptx_vqx_done_ack vqx_dack_cnt;
560
561 vqx_dack_cnt.u = readq(addr: cptvf->reg_base + OTX_CPT_VQX_DONE_ACK(0));
562 vqx_dack_cnt.s.done_ack = ackcnt;
563 writeq(val: vqx_dack_cnt.u, addr: cptvf->reg_base + OTX_CPT_VQX_DONE_ACK(0));
564}
565
566static irqreturn_t cptvf_done_intr_handler(int __always_unused irq,
567 void *cptvf_dev)
568{
569 struct otx_cptvf *cptvf = (struct otx_cptvf *)cptvf_dev;
570 struct pci_dev *pdev = cptvf->pdev;
571 /* Read the number of completions */
572 u32 intr = cptvf_read_vq_done_count(cptvf);
573
574 if (intr) {
575 struct otx_cptvf_wqe *wqe;
576
577 /*
578 * Acknowledge the number of scheduled completions for
579 * processing
580 */
581 cptvf_write_vq_done_ack(cptvf, ackcnt: intr);
582 wqe = get_cptvf_vq_wqe(cptvf, qno: 0);
583 if (unlikely(!wqe)) {
584 dev_err(&pdev->dev, "No work to schedule for VF (%d)\n",
585 cptvf->vfid);
586 return IRQ_NONE;
587 }
588 tasklet_hi_schedule(t: &wqe->twork);
589 }
590
591 return IRQ_HANDLED;
592}
593
594static void cptvf_set_irq_affinity(struct otx_cptvf *cptvf, int vec)
595{
596 struct pci_dev *pdev = cptvf->pdev;
597 int cpu;
598
599 if (!zalloc_cpumask_var(mask: &cptvf->affinity_mask[vec],
600 GFP_KERNEL)) {
601 dev_err(&pdev->dev,
602 "Allocation failed for affinity_mask for VF %d\n",
603 cptvf->vfid);
604 return;
605 }
606
607 cpu = cptvf->vfid % num_online_cpus();
608 cpumask_set_cpu(cpu: cpumask_local_spread(i: cpu, node: cptvf->node),
609 dstp: cptvf->affinity_mask[vec]);
610 irq_set_affinity_hint(irq: pci_irq_vector(dev: pdev, nr: vec),
611 m: cptvf->affinity_mask[vec]);
612}
613
614static void cptvf_write_vq_saddr(struct otx_cptvf *cptvf, u64 val)
615{
616 union otx_cptx_vqx_saddr vqx_saddr;
617
618 vqx_saddr.u = val;
619 writeq(val: vqx_saddr.u, addr: cptvf->reg_base + OTX_CPT_VQX_SADDR(0));
620}
621
622static void cptvf_device_init(struct otx_cptvf *cptvf)
623{
624 u64 base_addr = 0;
625
626 /* Disable the VQ */
627 cptvf_write_vq_ctl(cptvf, val: 0);
628 /* Reset the doorbell */
629 otx_cptvf_write_vq_doorbell(cptvf, val: 0);
630 /* Clear inflight */
631 cptvf_write_vq_inprog(cptvf, val: 0);
632 /* Write VQ SADDR */
633 base_addr = (u64)(cptvf->cqinfo.queue[0].qhead->dma_addr);
634 cptvf_write_vq_saddr(cptvf, val: base_addr);
635 /* Configure timerhold / coalescence */
636 cptvf_write_vq_done_timewait(cptvf, OTX_CPT_TIMER_HOLD);
637 cptvf_write_vq_done_numwait(cptvf, OTX_CPT_COUNT_HOLD);
638 /* Enable the VQ */
639 cptvf_write_vq_ctl(cptvf, val: 1);
640 /* Flag the VF ready */
641 cptvf->flags |= OTX_CPT_FLAG_DEVICE_READY;
642}
643
644static ssize_t vf_type_show(struct device *dev,
645 struct device_attribute *attr,
646 char *buf)
647{
648 struct otx_cptvf *cptvf = dev_get_drvdata(dev);
649 char *msg;
650
651 switch (cptvf->vftype) {
652 case OTX_CPT_AE_TYPES:
653 msg = "AE";
654 break;
655
656 case OTX_CPT_SE_TYPES:
657 msg = "SE";
658 break;
659
660 default:
661 msg = "Invalid";
662 }
663
664 return sysfs_emit(buf, fmt: "%s\n", msg);
665}
666
667static ssize_t vf_engine_group_show(struct device *dev,
668 struct device_attribute *attr,
669 char *buf)
670{
671 struct otx_cptvf *cptvf = dev_get_drvdata(dev);
672
673 return sysfs_emit(buf, fmt: "%d\n", cptvf->vfgrp);
674}
675
676static ssize_t vf_engine_group_store(struct device *dev,
677 struct device_attribute *attr,
678 const char *buf, size_t count)
679{
680 struct otx_cptvf *cptvf = dev_get_drvdata(dev);
681 int val, ret;
682
683 ret = kstrtoint(s: buf, base: 10, res: &val);
684 if (ret)
685 return ret;
686
687 if (val < 0)
688 return -EINVAL;
689
690 if (val >= OTX_CPT_MAX_ENGINE_GROUPS) {
691 dev_err(dev, "Engine group >= than max available groups %d\n",
692 OTX_CPT_MAX_ENGINE_GROUPS);
693 return -EINVAL;
694 }
695
696 ret = otx_cptvf_send_vf_to_grp_msg(cptvf, group: val);
697 if (ret)
698 return ret;
699
700 return count;
701}
702
703static ssize_t vf_coalesc_time_wait_show(struct device *dev,
704 struct device_attribute *attr,
705 char *buf)
706{
707 struct otx_cptvf *cptvf = dev_get_drvdata(dev);
708
709 return sysfs_emit(buf, fmt: "%d\n",
710 cptvf_read_vq_done_timewait(cptvf));
711}
712
713static ssize_t vf_coalesc_num_wait_show(struct device *dev,
714 struct device_attribute *attr,
715 char *buf)
716{
717 struct otx_cptvf *cptvf = dev_get_drvdata(dev);
718
719 return sysfs_emit(buf, fmt: "%d\n",
720 cptvf_read_vq_done_numwait(cptvf));
721}
722
723static ssize_t vf_coalesc_time_wait_store(struct device *dev,
724 struct device_attribute *attr,
725 const char *buf, size_t count)
726{
727 struct otx_cptvf *cptvf = dev_get_drvdata(dev);
728 long val;
729 int ret;
730
731 ret = kstrtol(s: buf, base: 10, res: &val);
732 if (ret != 0)
733 return ret;
734
735 if (val < OTX_CPT_COALESC_MIN_TIME_WAIT ||
736 val > OTX_CPT_COALESC_MAX_TIME_WAIT)
737 return -EINVAL;
738
739 cptvf_write_vq_done_timewait(cptvf, time: val);
740 return count;
741}
742
743static ssize_t vf_coalesc_num_wait_store(struct device *dev,
744 struct device_attribute *attr,
745 const char *buf, size_t count)
746{
747 struct otx_cptvf *cptvf = dev_get_drvdata(dev);
748 long val;
749 int ret;
750
751 ret = kstrtol(s: buf, base: 10, res: &val);
752 if (ret != 0)
753 return ret;
754
755 if (val < OTX_CPT_COALESC_MIN_NUM_WAIT ||
756 val > OTX_CPT_COALESC_MAX_NUM_WAIT)
757 return -EINVAL;
758
759 cptvf_write_vq_done_numwait(cptvf, val);
760 return count;
761}
762
763static DEVICE_ATTR_RO(vf_type);
764static DEVICE_ATTR_RW(vf_engine_group);
765static DEVICE_ATTR_RW(vf_coalesc_time_wait);
766static DEVICE_ATTR_RW(vf_coalesc_num_wait);
767
768static struct attribute *otx_cptvf_attrs[] = {
769 &dev_attr_vf_type.attr,
770 &dev_attr_vf_engine_group.attr,
771 &dev_attr_vf_coalesc_time_wait.attr,
772 &dev_attr_vf_coalesc_num_wait.attr,
773 NULL
774};
775
776static const struct attribute_group otx_cptvf_sysfs_group = {
777 .attrs = otx_cptvf_attrs,
778};
779
780static int otx_cptvf_probe(struct pci_dev *pdev,
781 const struct pci_device_id *ent)
782{
783 struct device *dev = &pdev->dev;
784 struct otx_cptvf *cptvf;
785 int err;
786
787 cptvf = devm_kzalloc(dev, size: sizeof(*cptvf), GFP_KERNEL);
788 if (!cptvf)
789 return -ENOMEM;
790
791 pci_set_drvdata(pdev, data: cptvf);
792 cptvf->pdev = pdev;
793
794 err = pci_enable_device(dev: pdev);
795 if (err) {
796 dev_err(dev, "Failed to enable PCI device\n");
797 goto clear_drvdata;
798 }
799 err = pci_request_regions(pdev, DRV_NAME);
800 if (err) {
801 dev_err(dev, "PCI request regions failed 0x%x\n", err);
802 goto disable_device;
803 }
804 err = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(48));
805 if (err) {
806 dev_err(dev, "Unable to get usable 48-bit DMA configuration\n");
807 goto release_regions;
808 }
809
810 /* MAP PF's configuration registers */
811 cptvf->reg_base = pci_iomap(dev: pdev, OTX_CPT_VF_PCI_CFG_BAR, max: 0);
812 if (!cptvf->reg_base) {
813 dev_err(dev, "Cannot map config register space, aborting\n");
814 err = -ENOMEM;
815 goto release_regions;
816 }
817
818 cptvf->node = dev_to_node(dev: &pdev->dev);
819 err = pci_alloc_irq_vectors(dev: pdev, OTX_CPT_VF_MSIX_VECTORS,
820 OTX_CPT_VF_MSIX_VECTORS, PCI_IRQ_MSIX);
821 if (err < 0) {
822 dev_err(dev, "Request for #%d msix vectors failed\n",
823 OTX_CPT_VF_MSIX_VECTORS);
824 goto unmap_region;
825 }
826
827 err = request_irq(irq: pci_irq_vector(dev: pdev, nr: CPT_VF_INT_VEC_E_MISC),
828 handler: cptvf_misc_intr_handler, flags: 0, name: "CPT VF misc intr",
829 dev: cptvf);
830 if (err) {
831 dev_err(dev, "Failed to request misc irq\n");
832 goto free_vectors;
833 }
834
835 /* Enable mailbox interrupt */
836 cptvf_enable_mbox_interrupts(cptvf);
837 cptvf_enable_swerr_interrupts(cptvf);
838
839 /* Check cpt pf status, gets chip ID / device Id from PF if ready */
840 err = otx_cptvf_check_pf_ready(cptvf);
841 if (err)
842 goto free_misc_irq;
843
844 /* CPT VF software resources initialization */
845 cptvf->cqinfo.qchunksize = OTX_CPT_CMD_QCHUNK_SIZE;
846 err = cptvf_sw_init(cptvf, OTX_CPT_CMD_QLEN, OTX_CPT_NUM_QS_PER_VF);
847 if (err) {
848 dev_err(dev, "cptvf_sw_init() failed\n");
849 goto free_misc_irq;
850 }
851 /* Convey VQ LEN to PF */
852 err = otx_cptvf_send_vq_size_msg(cptvf);
853 if (err)
854 goto sw_cleanup;
855
856 /* CPT VF device initialization */
857 cptvf_device_init(cptvf);
858 /* Send msg to PF to assign currnet Q to required group */
859 err = otx_cptvf_send_vf_to_grp_msg(cptvf, group: cptvf->vfgrp);
860 if (err)
861 goto sw_cleanup;
862
863 cptvf->priority = 1;
864 err = otx_cptvf_send_vf_priority_msg(cptvf);
865 if (err)
866 goto sw_cleanup;
867
868 err = request_irq(irq: pci_irq_vector(dev: pdev, nr: CPT_VF_INT_VEC_E_DONE),
869 handler: cptvf_done_intr_handler, flags: 0, name: "CPT VF done intr",
870 dev: cptvf);
871 if (err) {
872 dev_err(dev, "Failed to request done irq\n");
873 goto free_done_irq;
874 }
875
876 /* Enable done interrupt */
877 cptvf_enable_done_interrupts(cptvf);
878
879 /* Set irq affinity masks */
880 cptvf_set_irq_affinity(cptvf, vec: CPT_VF_INT_VEC_E_MISC);
881 cptvf_set_irq_affinity(cptvf, vec: CPT_VF_INT_VEC_E_DONE);
882
883 err = otx_cptvf_send_vf_up(cptvf);
884 if (err)
885 goto free_irq_affinity;
886
887 /* Initialize algorithms and set ops */
888 err = otx_cpt_crypto_init(pdev, THIS_MODULE,
889 pf_type: cptvf->vftype == OTX_CPT_SE_TYPES ? OTX_CPT_SE : OTX_CPT_AE,
890 engine_type: cptvf->vftype, num_queues: 1, num_devices: cptvf->num_vfs);
891 if (err) {
892 dev_err(dev, "Failed to register crypto algs\n");
893 goto free_irq_affinity;
894 }
895
896 err = sysfs_create_group(kobj: &dev->kobj, grp: &otx_cptvf_sysfs_group);
897 if (err) {
898 dev_err(dev, "Creating sysfs entries failed\n");
899 goto crypto_exit;
900 }
901
902 return 0;
903
904crypto_exit:
905 otx_cpt_crypto_exit(pdev, THIS_MODULE, engine_type: cptvf->vftype);
906free_irq_affinity:
907 cptvf_free_irq_affinity(cptvf, vec: CPT_VF_INT_VEC_E_DONE);
908 cptvf_free_irq_affinity(cptvf, vec: CPT_VF_INT_VEC_E_MISC);
909free_done_irq:
910 free_irq(pci_irq_vector(dev: pdev, nr: CPT_VF_INT_VEC_E_DONE), cptvf);
911sw_cleanup:
912 cptvf_sw_cleanup(cptvf);
913free_misc_irq:
914 free_irq(pci_irq_vector(dev: pdev, nr: CPT_VF_INT_VEC_E_MISC), cptvf);
915free_vectors:
916 pci_free_irq_vectors(dev: cptvf->pdev);
917unmap_region:
918 pci_iounmap(dev: pdev, cptvf->reg_base);
919release_regions:
920 pci_release_regions(pdev);
921disable_device:
922 pci_disable_device(dev: pdev);
923clear_drvdata:
924 pci_set_drvdata(pdev, NULL);
925
926 return err;
927}
928
929static void otx_cptvf_remove(struct pci_dev *pdev)
930{
931 struct otx_cptvf *cptvf = pci_get_drvdata(pdev);
932
933 if (!cptvf) {
934 dev_err(&pdev->dev, "Invalid CPT-VF device\n");
935 return;
936 }
937
938 /* Convey DOWN to PF */
939 if (otx_cptvf_send_vf_down(cptvf)) {
940 dev_err(&pdev->dev, "PF not responding to DOWN msg\n");
941 } else {
942 sysfs_remove_group(kobj: &pdev->dev.kobj, grp: &otx_cptvf_sysfs_group);
943 otx_cpt_crypto_exit(pdev, THIS_MODULE, engine_type: cptvf->vftype);
944 cptvf_free_irq_affinity(cptvf, vec: CPT_VF_INT_VEC_E_DONE);
945 cptvf_free_irq_affinity(cptvf, vec: CPT_VF_INT_VEC_E_MISC);
946 free_irq(pci_irq_vector(dev: pdev, nr: CPT_VF_INT_VEC_E_DONE), cptvf);
947 free_irq(pci_irq_vector(dev: pdev, nr: CPT_VF_INT_VEC_E_MISC), cptvf);
948 cptvf_sw_cleanup(cptvf);
949 pci_free_irq_vectors(dev: cptvf->pdev);
950 pci_iounmap(dev: pdev, cptvf->reg_base);
951 pci_release_regions(pdev);
952 pci_disable_device(dev: pdev);
953 pci_set_drvdata(pdev, NULL);
954 }
955}
956
957/* Supported devices */
958static const struct pci_device_id otx_cptvf_id_table[] = {
959 {PCI_VDEVICE(CAVIUM, OTX_CPT_PCI_VF_DEVICE_ID), 0},
960 { 0, } /* end of table */
961};
962
963static struct pci_driver otx_cptvf_pci_driver = {
964 .name = DRV_NAME,
965 .id_table = otx_cptvf_id_table,
966 .probe = otx_cptvf_probe,
967 .remove = otx_cptvf_remove,
968};
969
970module_pci_driver(otx_cptvf_pci_driver);
971
972MODULE_AUTHOR("Marvell International Ltd.");
973MODULE_DESCRIPTION("Marvell OcteonTX CPT Virtual Function Driver");
974MODULE_LICENSE("GPL v2");
975MODULE_VERSION(DRV_VERSION);
976MODULE_DEVICE_TABLE(pci, otx_cptvf_id_table);
977

source code of linux/drivers/crypto/marvell/octeontx/otx_cptvf_main.c