1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (C) 2016 Cavium, Inc. |
4 | */ |
5 | |
6 | #include <linux/interrupt.h> |
7 | #include <linux/module.h> |
8 | |
9 | #include "cptvf.h" |
10 | |
11 | #define DRV_NAME "thunder-cptvf" |
12 | #define DRV_VERSION "1.0" |
13 | |
14 | struct cptvf_wqe { |
15 | struct tasklet_struct twork; |
16 | void *cptvf; |
17 | u32 qno; |
18 | }; |
19 | |
20 | struct cptvf_wqe_info { |
21 | struct cptvf_wqe vq_wqe[CPT_NUM_QS_PER_VF]; |
22 | }; |
23 | |
24 | static void vq_work_handler(unsigned long data) |
25 | { |
26 | struct cptvf_wqe_info *cwqe_info = (struct cptvf_wqe_info *)data; |
27 | struct cptvf_wqe *cwqe = &cwqe_info->vq_wqe[0]; |
28 | |
29 | vq_post_process(cptvf: cwqe->cptvf, qno: cwqe->qno); |
30 | } |
31 | |
32 | static int init_worker_threads(struct cpt_vf *cptvf) |
33 | { |
34 | struct pci_dev *pdev = cptvf->pdev; |
35 | struct cptvf_wqe_info *cwqe_info; |
36 | int i; |
37 | |
38 | cwqe_info = kzalloc(size: sizeof(*cwqe_info), GFP_KERNEL); |
39 | if (!cwqe_info) |
40 | return -ENOMEM; |
41 | |
42 | if (cptvf->nr_queues) { |
43 | dev_info(&pdev->dev, "Creating VQ worker threads (%d)\n" , |
44 | cptvf->nr_queues); |
45 | } |
46 | |
47 | for (i = 0; i < cptvf->nr_queues; i++) { |
48 | tasklet_init(t: &cwqe_info->vq_wqe[i].twork, func: vq_work_handler, |
49 | data: (u64)cwqe_info); |
50 | cwqe_info->vq_wqe[i].qno = i; |
51 | cwqe_info->vq_wqe[i].cptvf = cptvf; |
52 | } |
53 | |
54 | cptvf->wqe_info = cwqe_info; |
55 | |
56 | return 0; |
57 | } |
58 | |
59 | static void cleanup_worker_threads(struct cpt_vf *cptvf) |
60 | { |
61 | struct cptvf_wqe_info *cwqe_info; |
62 | struct pci_dev *pdev = cptvf->pdev; |
63 | int i; |
64 | |
65 | cwqe_info = (struct cptvf_wqe_info *)cptvf->wqe_info; |
66 | if (!cwqe_info) |
67 | return; |
68 | |
69 | if (cptvf->nr_queues) { |
70 | dev_info(&pdev->dev, "Cleaning VQ worker threads (%u)\n" , |
71 | cptvf->nr_queues); |
72 | } |
73 | |
74 | for (i = 0; i < cptvf->nr_queues; i++) |
75 | tasklet_kill(t: &cwqe_info->vq_wqe[i].twork); |
76 | |
77 | kfree_sensitive(objp: cwqe_info); |
78 | cptvf->wqe_info = NULL; |
79 | } |
80 | |
81 | static void free_pending_queues(struct pending_qinfo *pqinfo) |
82 | { |
83 | int i; |
84 | struct pending_queue *queue; |
85 | |
86 | for_each_pending_queue(pqinfo, queue, i) { |
87 | if (!queue->head) |
88 | continue; |
89 | |
90 | /* free single queue */ |
91 | kfree_sensitive(objp: (queue->head)); |
92 | |
93 | queue->front = 0; |
94 | queue->rear = 0; |
95 | |
96 | return; |
97 | } |
98 | |
99 | pqinfo->qlen = 0; |
100 | pqinfo->nr_queues = 0; |
101 | } |
102 | |
103 | static int alloc_pending_queues(struct pending_qinfo *pqinfo, u32 qlen, |
104 | u32 nr_queues) |
105 | { |
106 | u32 i; |
107 | int ret; |
108 | struct pending_queue *queue = NULL; |
109 | |
110 | pqinfo->nr_queues = nr_queues; |
111 | pqinfo->qlen = qlen; |
112 | |
113 | for_each_pending_queue(pqinfo, queue, i) { |
114 | queue->head = kcalloc(n: qlen, size: sizeof(*queue->head), GFP_KERNEL); |
115 | if (!queue->head) { |
116 | ret = -ENOMEM; |
117 | goto pending_qfail; |
118 | } |
119 | |
120 | queue->front = 0; |
121 | queue->rear = 0; |
122 | atomic64_set(v: (&queue->pending_count), i: (0)); |
123 | |
124 | /* init queue spin lock */ |
125 | spin_lock_init(&queue->lock); |
126 | } |
127 | |
128 | return 0; |
129 | |
130 | pending_qfail: |
131 | free_pending_queues(pqinfo); |
132 | |
133 | return ret; |
134 | } |
135 | |
136 | static int init_pending_queues(struct cpt_vf *cptvf, u32 qlen, u32 nr_queues) |
137 | { |
138 | struct pci_dev *pdev = cptvf->pdev; |
139 | int ret; |
140 | |
141 | if (!nr_queues) |
142 | return 0; |
143 | |
144 | ret = alloc_pending_queues(pqinfo: &cptvf->pqinfo, qlen, nr_queues); |
145 | if (ret) { |
146 | dev_err(&pdev->dev, "failed to setup pending queues (%u)\n" , |
147 | nr_queues); |
148 | return ret; |
149 | } |
150 | |
151 | return 0; |
152 | } |
153 | |
154 | static void cleanup_pending_queues(struct cpt_vf *cptvf) |
155 | { |
156 | struct pci_dev *pdev = cptvf->pdev; |
157 | |
158 | if (!cptvf->nr_queues) |
159 | return; |
160 | |
161 | dev_info(&pdev->dev, "Cleaning VQ pending queue (%u)\n" , |
162 | cptvf->nr_queues); |
163 | free_pending_queues(pqinfo: &cptvf->pqinfo); |
164 | } |
165 | |
166 | static void free_command_queues(struct cpt_vf *cptvf, |
167 | struct command_qinfo *cqinfo) |
168 | { |
169 | int i; |
170 | struct command_queue *queue = NULL; |
171 | struct command_chunk *chunk = NULL; |
172 | struct pci_dev *pdev = cptvf->pdev; |
173 | struct hlist_node *node; |
174 | |
175 | /* clean up for each queue */ |
176 | for (i = 0; i < cptvf->nr_queues; i++) { |
177 | queue = &cqinfo->queue[i]; |
178 | if (hlist_empty(h: &cqinfo->queue[i].chead)) |
179 | continue; |
180 | |
181 | hlist_for_each_entry_safe(chunk, node, &cqinfo->queue[i].chead, |
182 | nextchunk) { |
183 | dma_free_coherent(dev: &pdev->dev, size: chunk->size, |
184 | cpu_addr: chunk->head, |
185 | dma_handle: chunk->dma_addr); |
186 | chunk->head = NULL; |
187 | chunk->dma_addr = 0; |
188 | hlist_del(n: &chunk->nextchunk); |
189 | kfree_sensitive(objp: chunk); |
190 | } |
191 | |
192 | queue->nchunks = 0; |
193 | queue->idx = 0; |
194 | } |
195 | |
196 | /* common cleanup */ |
197 | cqinfo->cmd_size = 0; |
198 | } |
199 | |
200 | static int alloc_command_queues(struct cpt_vf *cptvf, |
201 | struct command_qinfo *cqinfo, size_t cmd_size, |
202 | u32 qlen) |
203 | { |
204 | int i; |
205 | size_t q_size; |
206 | struct command_queue *queue = NULL; |
207 | struct pci_dev *pdev = cptvf->pdev; |
208 | |
209 | /* common init */ |
210 | cqinfo->cmd_size = cmd_size; |
211 | /* Qsize in dwords, needed for SADDR config, 1-next chunk pointer */ |
212 | cptvf->qsize = min(qlen, cqinfo->qchunksize) * |
213 | CPT_NEXT_CHUNK_PTR_SIZE + 1; |
214 | /* Qsize in bytes to create space for alignment */ |
215 | q_size = qlen * cqinfo->cmd_size; |
216 | |
217 | /* per queue initialization */ |
218 | for (i = 0; i < cptvf->nr_queues; i++) { |
219 | size_t c_size = 0; |
220 | size_t rem_q_size = q_size; |
221 | struct command_chunk *curr = NULL, *first = NULL, *last = NULL; |
222 | u32 qcsize_bytes = cqinfo->qchunksize * cqinfo->cmd_size; |
223 | |
224 | queue = &cqinfo->queue[i]; |
225 | INIT_HLIST_HEAD(&cqinfo->queue[i].chead); |
226 | do { |
227 | curr = kzalloc(size: sizeof(*curr), GFP_KERNEL); |
228 | if (!curr) |
229 | goto cmd_qfail; |
230 | |
231 | c_size = (rem_q_size > qcsize_bytes) ? qcsize_bytes : |
232 | rem_q_size; |
233 | curr->head = dma_alloc_coherent(dev: &pdev->dev, |
234 | size: c_size + CPT_NEXT_CHUNK_PTR_SIZE, |
235 | dma_handle: &curr->dma_addr, |
236 | GFP_KERNEL); |
237 | if (!curr->head) { |
238 | dev_err(&pdev->dev, "Command Q (%d) chunk (%d) allocation failed\n" , |
239 | i, queue->nchunks); |
240 | kfree(objp: curr); |
241 | goto cmd_qfail; |
242 | } |
243 | |
244 | curr->size = c_size; |
245 | if (queue->nchunks == 0) { |
246 | hlist_add_head(n: &curr->nextchunk, |
247 | h: &cqinfo->queue[i].chead); |
248 | first = curr; |
249 | } else { |
250 | hlist_add_behind(n: &curr->nextchunk, |
251 | prev: &last->nextchunk); |
252 | } |
253 | |
254 | queue->nchunks++; |
255 | rem_q_size -= c_size; |
256 | if (last) |
257 | *((u64 *)(&last->head[last->size])) = (u64)curr->dma_addr; |
258 | |
259 | last = curr; |
260 | } while (rem_q_size); |
261 | |
262 | /* Make the queue circular */ |
263 | /* Tie back last chunk entry to head */ |
264 | curr = first; |
265 | *((u64 *)(&last->head[last->size])) = (u64)curr->dma_addr; |
266 | queue->qhead = curr; |
267 | spin_lock_init(&queue->lock); |
268 | } |
269 | return 0; |
270 | |
271 | cmd_qfail: |
272 | free_command_queues(cptvf, cqinfo); |
273 | return -ENOMEM; |
274 | } |
275 | |
276 | static int init_command_queues(struct cpt_vf *cptvf, u32 qlen) |
277 | { |
278 | struct pci_dev *pdev = cptvf->pdev; |
279 | int ret; |
280 | |
281 | /* setup AE command queues */ |
282 | ret = alloc_command_queues(cptvf, cqinfo: &cptvf->cqinfo, CPT_INST_SIZE, |
283 | qlen); |
284 | if (ret) { |
285 | dev_err(&pdev->dev, "failed to allocate AE command queues (%u)\n" , |
286 | cptvf->nr_queues); |
287 | return ret; |
288 | } |
289 | |
290 | return ret; |
291 | } |
292 | |
293 | static void cleanup_command_queues(struct cpt_vf *cptvf) |
294 | { |
295 | struct pci_dev *pdev = cptvf->pdev; |
296 | |
297 | if (!cptvf->nr_queues) |
298 | return; |
299 | |
300 | dev_info(&pdev->dev, "Cleaning VQ command queue (%u)\n" , |
301 | cptvf->nr_queues); |
302 | free_command_queues(cptvf, cqinfo: &cptvf->cqinfo); |
303 | } |
304 | |
305 | static void cptvf_sw_cleanup(struct cpt_vf *cptvf) |
306 | { |
307 | cleanup_worker_threads(cptvf); |
308 | cleanup_pending_queues(cptvf); |
309 | cleanup_command_queues(cptvf); |
310 | } |
311 | |
312 | static int cptvf_sw_init(struct cpt_vf *cptvf, u32 qlen, u32 nr_queues) |
313 | { |
314 | struct pci_dev *pdev = cptvf->pdev; |
315 | int ret = 0; |
316 | u32 max_dev_queues = 0; |
317 | |
318 | max_dev_queues = CPT_NUM_QS_PER_VF; |
319 | /* possible cpus */ |
320 | nr_queues = min_t(u32, nr_queues, max_dev_queues); |
321 | cptvf->nr_queues = nr_queues; |
322 | |
323 | ret = init_command_queues(cptvf, qlen); |
324 | if (ret) { |
325 | dev_err(&pdev->dev, "Failed to setup command queues (%u)\n" , |
326 | nr_queues); |
327 | return ret; |
328 | } |
329 | |
330 | ret = init_pending_queues(cptvf, qlen, nr_queues); |
331 | if (ret) { |
332 | dev_err(&pdev->dev, "Failed to setup pending queues (%u)\n" , |
333 | nr_queues); |
334 | goto setup_pqfail; |
335 | } |
336 | |
337 | /* Create worker threads for BH processing */ |
338 | ret = init_worker_threads(cptvf); |
339 | if (ret) { |
340 | dev_err(&pdev->dev, "Failed to setup worker threads\n" ); |
341 | goto init_work_fail; |
342 | } |
343 | |
344 | return 0; |
345 | |
346 | init_work_fail: |
347 | cleanup_worker_threads(cptvf); |
348 | cleanup_pending_queues(cptvf); |
349 | |
350 | setup_pqfail: |
351 | cleanup_command_queues(cptvf); |
352 | |
353 | return ret; |
354 | } |
355 | |
356 | static void cptvf_free_irq_affinity(struct cpt_vf *cptvf, int vec) |
357 | { |
358 | irq_set_affinity_hint(irq: pci_irq_vector(dev: cptvf->pdev, nr: vec), NULL); |
359 | free_cpumask_var(mask: cptvf->affinity_mask[vec]); |
360 | } |
361 | |
362 | static void cptvf_write_vq_ctl(struct cpt_vf *cptvf, bool val) |
363 | { |
364 | union cptx_vqx_ctl vqx_ctl; |
365 | |
366 | vqx_ctl.u = cpt_read_csr64(hw_addr: cptvf->reg_base, CPTX_VQX_CTL(0, 0)); |
367 | vqx_ctl.s.ena = val; |
368 | cpt_write_csr64(hw_addr: cptvf->reg_base, CPTX_VQX_CTL(0, 0), val: vqx_ctl.u); |
369 | } |
370 | |
371 | void cptvf_write_vq_doorbell(struct cpt_vf *cptvf, u32 val) |
372 | { |
373 | union cptx_vqx_doorbell vqx_dbell; |
374 | |
375 | vqx_dbell.u = cpt_read_csr64(hw_addr: cptvf->reg_base, |
376 | CPTX_VQX_DOORBELL(0, 0)); |
377 | vqx_dbell.s.dbell_cnt = val * 8; /* Num of Instructions * 8 words */ |
378 | cpt_write_csr64(hw_addr: cptvf->reg_base, CPTX_VQX_DOORBELL(0, 0), |
379 | val: vqx_dbell.u); |
380 | } |
381 | |
382 | static void cptvf_write_vq_inprog(struct cpt_vf *cptvf, u8 val) |
383 | { |
384 | union cptx_vqx_inprog vqx_inprg; |
385 | |
386 | vqx_inprg.u = cpt_read_csr64(hw_addr: cptvf->reg_base, CPTX_VQX_INPROG(0, 0)); |
387 | vqx_inprg.s.inflight = val; |
388 | cpt_write_csr64(hw_addr: cptvf->reg_base, CPTX_VQX_INPROG(0, 0), val: vqx_inprg.u); |
389 | } |
390 | |
391 | static void cptvf_write_vq_done_numwait(struct cpt_vf *cptvf, u32 val) |
392 | { |
393 | union cptx_vqx_done_wait vqx_dwait; |
394 | |
395 | vqx_dwait.u = cpt_read_csr64(hw_addr: cptvf->reg_base, |
396 | CPTX_VQX_DONE_WAIT(0, 0)); |
397 | vqx_dwait.s.num_wait = val; |
398 | cpt_write_csr64(hw_addr: cptvf->reg_base, CPTX_VQX_DONE_WAIT(0, 0), |
399 | val: vqx_dwait.u); |
400 | } |
401 | |
402 | static void cptvf_write_vq_done_timewait(struct cpt_vf *cptvf, u16 time) |
403 | { |
404 | union cptx_vqx_done_wait vqx_dwait; |
405 | |
406 | vqx_dwait.u = cpt_read_csr64(hw_addr: cptvf->reg_base, |
407 | CPTX_VQX_DONE_WAIT(0, 0)); |
408 | vqx_dwait.s.time_wait = time; |
409 | cpt_write_csr64(hw_addr: cptvf->reg_base, CPTX_VQX_DONE_WAIT(0, 0), |
410 | val: vqx_dwait.u); |
411 | } |
412 | |
413 | static void cptvf_enable_swerr_interrupts(struct cpt_vf *cptvf) |
414 | { |
415 | union cptx_vqx_misc_ena_w1s vqx_misc_ena; |
416 | |
417 | vqx_misc_ena.u = cpt_read_csr64(hw_addr: cptvf->reg_base, |
418 | CPTX_VQX_MISC_ENA_W1S(0, 0)); |
419 | /* Set mbox(0) interupts for the requested vf */ |
420 | vqx_misc_ena.s.swerr = 1; |
421 | cpt_write_csr64(hw_addr: cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0), |
422 | val: vqx_misc_ena.u); |
423 | } |
424 | |
425 | static void cptvf_enable_mbox_interrupts(struct cpt_vf *cptvf) |
426 | { |
427 | union cptx_vqx_misc_ena_w1s vqx_misc_ena; |
428 | |
429 | vqx_misc_ena.u = cpt_read_csr64(hw_addr: cptvf->reg_base, |
430 | CPTX_VQX_MISC_ENA_W1S(0, 0)); |
431 | /* Set mbox(0) interupts for the requested vf */ |
432 | vqx_misc_ena.s.mbox = 1; |
433 | cpt_write_csr64(hw_addr: cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0), |
434 | val: vqx_misc_ena.u); |
435 | } |
436 | |
437 | static void cptvf_enable_done_interrupts(struct cpt_vf *cptvf) |
438 | { |
439 | union cptx_vqx_done_ena_w1s vqx_done_ena; |
440 | |
441 | vqx_done_ena.u = cpt_read_csr64(hw_addr: cptvf->reg_base, |
442 | CPTX_VQX_DONE_ENA_W1S(0, 0)); |
443 | /* Set DONE interrupt for the requested vf */ |
444 | vqx_done_ena.s.done = 1; |
445 | cpt_write_csr64(hw_addr: cptvf->reg_base, CPTX_VQX_DONE_ENA_W1S(0, 0), |
446 | val: vqx_done_ena.u); |
447 | } |
448 | |
449 | static void cptvf_clear_dovf_intr(struct cpt_vf *cptvf) |
450 | { |
451 | union cptx_vqx_misc_int vqx_misc_int; |
452 | |
453 | vqx_misc_int.u = cpt_read_csr64(hw_addr: cptvf->reg_base, |
454 | CPTX_VQX_MISC_INT(0, 0)); |
455 | /* W1C for the VF */ |
456 | vqx_misc_int.s.dovf = 1; |
457 | cpt_write_csr64(hw_addr: cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0), |
458 | val: vqx_misc_int.u); |
459 | } |
460 | |
461 | static void cptvf_clear_irde_intr(struct cpt_vf *cptvf) |
462 | { |
463 | union cptx_vqx_misc_int vqx_misc_int; |
464 | |
465 | vqx_misc_int.u = cpt_read_csr64(hw_addr: cptvf->reg_base, |
466 | CPTX_VQX_MISC_INT(0, 0)); |
467 | /* W1C for the VF */ |
468 | vqx_misc_int.s.irde = 1; |
469 | cpt_write_csr64(hw_addr: cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0), |
470 | val: vqx_misc_int.u); |
471 | } |
472 | |
473 | static void cptvf_clear_nwrp_intr(struct cpt_vf *cptvf) |
474 | { |
475 | union cptx_vqx_misc_int vqx_misc_int; |
476 | |
477 | vqx_misc_int.u = cpt_read_csr64(hw_addr: cptvf->reg_base, |
478 | CPTX_VQX_MISC_INT(0, 0)); |
479 | /* W1C for the VF */ |
480 | vqx_misc_int.s.nwrp = 1; |
481 | cpt_write_csr64(hw_addr: cptvf->reg_base, |
482 | CPTX_VQX_MISC_INT(0, 0), val: vqx_misc_int.u); |
483 | } |
484 | |
485 | static void cptvf_clear_mbox_intr(struct cpt_vf *cptvf) |
486 | { |
487 | union cptx_vqx_misc_int vqx_misc_int; |
488 | |
489 | vqx_misc_int.u = cpt_read_csr64(hw_addr: cptvf->reg_base, |
490 | CPTX_VQX_MISC_INT(0, 0)); |
491 | /* W1C for the VF */ |
492 | vqx_misc_int.s.mbox = 1; |
493 | cpt_write_csr64(hw_addr: cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0), |
494 | val: vqx_misc_int.u); |
495 | } |
496 | |
497 | static void cptvf_clear_swerr_intr(struct cpt_vf *cptvf) |
498 | { |
499 | union cptx_vqx_misc_int vqx_misc_int; |
500 | |
501 | vqx_misc_int.u = cpt_read_csr64(hw_addr: cptvf->reg_base, |
502 | CPTX_VQX_MISC_INT(0, 0)); |
503 | /* W1C for the VF */ |
504 | vqx_misc_int.s.swerr = 1; |
505 | cpt_write_csr64(hw_addr: cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0), |
506 | val: vqx_misc_int.u); |
507 | } |
508 | |
509 | static u64 cptvf_read_vf_misc_intr_status(struct cpt_vf *cptvf) |
510 | { |
511 | return cpt_read_csr64(hw_addr: cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0)); |
512 | } |
513 | |
514 | static irqreturn_t cptvf_misc_intr_handler(int irq, void *cptvf_irq) |
515 | { |
516 | struct cpt_vf *cptvf = (struct cpt_vf *)cptvf_irq; |
517 | struct pci_dev *pdev = cptvf->pdev; |
518 | u64 intr; |
519 | |
520 | intr = cptvf_read_vf_misc_intr_status(cptvf); |
521 | /*Check for MISC interrupt types*/ |
522 | if (likely(intr & CPT_VF_INTR_MBOX_MASK)) { |
523 | dev_dbg(&pdev->dev, "Mailbox interrupt 0x%llx on CPT VF %d\n" , |
524 | intr, cptvf->vfid); |
525 | cptvf_handle_mbox_intr(cptvf); |
526 | cptvf_clear_mbox_intr(cptvf); |
527 | } else if (unlikely(intr & CPT_VF_INTR_DOVF_MASK)) { |
528 | cptvf_clear_dovf_intr(cptvf); |
529 | /*Clear doorbell count*/ |
530 | cptvf_write_vq_doorbell(cptvf, val: 0); |
531 | dev_err(&pdev->dev, "Doorbell overflow error interrupt 0x%llx on CPT VF %d\n" , |
532 | intr, cptvf->vfid); |
533 | } else if (unlikely(intr & CPT_VF_INTR_IRDE_MASK)) { |
534 | cptvf_clear_irde_intr(cptvf); |
535 | dev_err(&pdev->dev, "Instruction NCB read error interrupt 0x%llx on CPT VF %d\n" , |
536 | intr, cptvf->vfid); |
537 | } else if (unlikely(intr & CPT_VF_INTR_NWRP_MASK)) { |
538 | cptvf_clear_nwrp_intr(cptvf); |
539 | dev_err(&pdev->dev, "NCB response write error interrupt 0x%llx on CPT VF %d\n" , |
540 | intr, cptvf->vfid); |
541 | } else if (unlikely(intr & CPT_VF_INTR_SERR_MASK)) { |
542 | cptvf_clear_swerr_intr(cptvf); |
543 | dev_err(&pdev->dev, "Software error interrupt 0x%llx on CPT VF %d\n" , |
544 | intr, cptvf->vfid); |
545 | } else { |
546 | dev_err(&pdev->dev, "Unhandled interrupt in CPT VF %d\n" , |
547 | cptvf->vfid); |
548 | } |
549 | |
550 | return IRQ_HANDLED; |
551 | } |
552 | |
553 | static inline struct cptvf_wqe *get_cptvf_vq_wqe(struct cpt_vf *cptvf, |
554 | int qno) |
555 | { |
556 | struct cptvf_wqe_info *nwqe_info; |
557 | |
558 | if (unlikely(qno >= cptvf->nr_queues)) |
559 | return NULL; |
560 | nwqe_info = (struct cptvf_wqe_info *)cptvf->wqe_info; |
561 | |
562 | return &nwqe_info->vq_wqe[qno]; |
563 | } |
564 | |
565 | static inline u32 cptvf_read_vq_done_count(struct cpt_vf *cptvf) |
566 | { |
567 | union cptx_vqx_done vqx_done; |
568 | |
569 | vqx_done.u = cpt_read_csr64(hw_addr: cptvf->reg_base, CPTX_VQX_DONE(0, 0)); |
570 | return vqx_done.s.done; |
571 | } |
572 | |
573 | static inline void cptvf_write_vq_done_ack(struct cpt_vf *cptvf, |
574 | u32 ackcnt) |
575 | { |
576 | union cptx_vqx_done_ack vqx_dack_cnt; |
577 | |
578 | vqx_dack_cnt.u = cpt_read_csr64(hw_addr: cptvf->reg_base, |
579 | CPTX_VQX_DONE_ACK(0, 0)); |
580 | vqx_dack_cnt.s.done_ack = ackcnt; |
581 | cpt_write_csr64(hw_addr: cptvf->reg_base, CPTX_VQX_DONE_ACK(0, 0), |
582 | val: vqx_dack_cnt.u); |
583 | } |
584 | |
585 | static irqreturn_t cptvf_done_intr_handler(int irq, void *cptvf_irq) |
586 | { |
587 | struct cpt_vf *cptvf = (struct cpt_vf *)cptvf_irq; |
588 | struct pci_dev *pdev = cptvf->pdev; |
589 | /* Read the number of completions */ |
590 | u32 intr = cptvf_read_vq_done_count(cptvf); |
591 | |
592 | if (intr) { |
593 | struct cptvf_wqe *wqe; |
594 | |
595 | /* Acknowledge the number of |
596 | * scheduled completions for processing |
597 | */ |
598 | cptvf_write_vq_done_ack(cptvf, ackcnt: intr); |
599 | wqe = get_cptvf_vq_wqe(cptvf, qno: 0); |
600 | if (unlikely(!wqe)) { |
601 | dev_err(&pdev->dev, "No work to schedule for VF (%d)" , |
602 | cptvf->vfid); |
603 | return IRQ_NONE; |
604 | } |
605 | tasklet_hi_schedule(t: &wqe->twork); |
606 | } |
607 | |
608 | return IRQ_HANDLED; |
609 | } |
610 | |
611 | static void cptvf_set_irq_affinity(struct cpt_vf *cptvf, int vec) |
612 | { |
613 | struct pci_dev *pdev = cptvf->pdev; |
614 | int cpu; |
615 | |
616 | if (!zalloc_cpumask_var(mask: &cptvf->affinity_mask[vec], |
617 | GFP_KERNEL)) { |
618 | dev_err(&pdev->dev, "Allocation failed for affinity_mask for VF %d" , |
619 | cptvf->vfid); |
620 | return; |
621 | } |
622 | |
623 | cpu = cptvf->vfid % num_online_cpus(); |
624 | cpumask_set_cpu(cpu: cpumask_local_spread(i: cpu, node: cptvf->node), |
625 | dstp: cptvf->affinity_mask[vec]); |
626 | irq_set_affinity_hint(irq: pci_irq_vector(dev: pdev, nr: vec), |
627 | m: cptvf->affinity_mask[vec]); |
628 | } |
629 | |
630 | static void cptvf_write_vq_saddr(struct cpt_vf *cptvf, u64 val) |
631 | { |
632 | union cptx_vqx_saddr vqx_saddr; |
633 | |
634 | vqx_saddr.u = val; |
635 | cpt_write_csr64(hw_addr: cptvf->reg_base, CPTX_VQX_SADDR(0, 0), val: vqx_saddr.u); |
636 | } |
637 | |
638 | static void cptvf_device_init(struct cpt_vf *cptvf) |
639 | { |
640 | u64 base_addr = 0; |
641 | |
642 | /* Disable the VQ */ |
643 | cptvf_write_vq_ctl(cptvf, val: 0); |
644 | /* Reset the doorbell */ |
645 | cptvf_write_vq_doorbell(cptvf, val: 0); |
646 | /* Clear inflight */ |
647 | cptvf_write_vq_inprog(cptvf, val: 0); |
648 | /* Write VQ SADDR */ |
649 | /* TODO: for now only one queue, so hard coded */ |
650 | base_addr = (u64)(cptvf->cqinfo.queue[0].qhead->dma_addr); |
651 | cptvf_write_vq_saddr(cptvf, val: base_addr); |
652 | /* Configure timerhold / coalescence */ |
653 | cptvf_write_vq_done_timewait(cptvf, CPT_TIMER_THOLD); |
654 | cptvf_write_vq_done_numwait(cptvf, val: 1); |
655 | /* Enable the VQ */ |
656 | cptvf_write_vq_ctl(cptvf, val: 1); |
657 | /* Flag the VF ready */ |
658 | cptvf->flags |= CPT_FLAG_DEVICE_READY; |
659 | } |
660 | |
661 | static int cptvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) |
662 | { |
663 | struct device *dev = &pdev->dev; |
664 | struct cpt_vf *cptvf; |
665 | int err; |
666 | |
667 | cptvf = devm_kzalloc(dev, size: sizeof(*cptvf), GFP_KERNEL); |
668 | if (!cptvf) |
669 | return -ENOMEM; |
670 | |
671 | pci_set_drvdata(pdev, data: cptvf); |
672 | cptvf->pdev = pdev; |
673 | err = pci_enable_device(dev: pdev); |
674 | if (err) { |
675 | dev_err(dev, "Failed to enable PCI device\n" ); |
676 | pci_set_drvdata(pdev, NULL); |
677 | return err; |
678 | } |
679 | |
680 | err = pci_request_regions(pdev, DRV_NAME); |
681 | if (err) { |
682 | dev_err(dev, "PCI request regions failed 0x%x\n" , err); |
683 | goto cptvf_err_disable_device; |
684 | } |
685 | /* Mark as VF driver */ |
686 | cptvf->flags |= CPT_FLAG_VF_DRIVER; |
687 | err = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(48)); |
688 | if (err) { |
689 | dev_err(dev, "Unable to get usable 48-bit DMA configuration\n" ); |
690 | goto cptvf_err_release_regions; |
691 | } |
692 | |
693 | /* MAP PF's configuration registers */ |
694 | cptvf->reg_base = pcim_iomap(pdev, bar: 0, maxlen: 0); |
695 | if (!cptvf->reg_base) { |
696 | dev_err(dev, "Cannot map config register space, aborting\n" ); |
697 | err = -ENOMEM; |
698 | goto cptvf_err_release_regions; |
699 | } |
700 | |
701 | cptvf->node = dev_to_node(dev: &pdev->dev); |
702 | err = pci_alloc_irq_vectors(dev: pdev, CPT_VF_MSIX_VECTORS, |
703 | CPT_VF_MSIX_VECTORS, PCI_IRQ_MSIX); |
704 | if (err < 0) { |
705 | dev_err(dev, "Request for #%d msix vectors failed\n" , |
706 | CPT_VF_MSIX_VECTORS); |
707 | goto cptvf_err_release_regions; |
708 | } |
709 | |
710 | err = request_irq(irq: pci_irq_vector(dev: pdev, nr: CPT_VF_INT_VEC_E_MISC), |
711 | handler: cptvf_misc_intr_handler, flags: 0, name: "CPT VF misc intr" , |
712 | dev: cptvf); |
713 | if (err) { |
714 | dev_err(dev, "Request misc irq failed" ); |
715 | goto cptvf_free_vectors; |
716 | } |
717 | |
718 | /* Enable mailbox interrupt */ |
719 | cptvf_enable_mbox_interrupts(cptvf); |
720 | cptvf_enable_swerr_interrupts(cptvf); |
721 | |
722 | /* Check ready with PF */ |
723 | /* Gets chip ID / device Id from PF if ready */ |
724 | err = cptvf_check_pf_ready(cptvf); |
725 | if (err) { |
726 | dev_err(dev, "PF not responding to READY msg" ); |
727 | goto cptvf_free_misc_irq; |
728 | } |
729 | |
730 | /* CPT VF software resources initialization */ |
731 | cptvf->cqinfo.qchunksize = CPT_CMD_QCHUNK_SIZE; |
732 | err = cptvf_sw_init(cptvf, CPT_CMD_QLEN, CPT_NUM_QS_PER_VF); |
733 | if (err) { |
734 | dev_err(dev, "cptvf_sw_init() failed" ); |
735 | goto cptvf_free_misc_irq; |
736 | } |
737 | /* Convey VQ LEN to PF */ |
738 | err = cptvf_send_vq_size_msg(cptvf); |
739 | if (err) { |
740 | dev_err(dev, "PF not responding to QLEN msg" ); |
741 | goto cptvf_free_misc_irq; |
742 | } |
743 | |
744 | /* CPT VF device initialization */ |
745 | cptvf_device_init(cptvf); |
746 | /* Send msg to PF to assign currnet Q to required group */ |
747 | cptvf->vfgrp = 1; |
748 | err = cptvf_send_vf_to_grp_msg(cptvf); |
749 | if (err) { |
750 | dev_err(dev, "PF not responding to VF_GRP msg" ); |
751 | goto cptvf_free_misc_irq; |
752 | } |
753 | |
754 | cptvf->priority = 1; |
755 | err = cptvf_send_vf_priority_msg(cptvf); |
756 | if (err) { |
757 | dev_err(dev, "PF not responding to VF_PRIO msg" ); |
758 | goto cptvf_free_misc_irq; |
759 | } |
760 | |
761 | err = request_irq(irq: pci_irq_vector(dev: pdev, nr: CPT_VF_INT_VEC_E_DONE), |
762 | handler: cptvf_done_intr_handler, flags: 0, name: "CPT VF done intr" , |
763 | dev: cptvf); |
764 | if (err) { |
765 | dev_err(dev, "Request done irq failed\n" ); |
766 | goto cptvf_free_misc_irq; |
767 | } |
768 | |
769 | /* Enable mailbox interrupt */ |
770 | cptvf_enable_done_interrupts(cptvf); |
771 | |
772 | /* Set irq affinity masks */ |
773 | cptvf_set_irq_affinity(cptvf, vec: CPT_VF_INT_VEC_E_MISC); |
774 | cptvf_set_irq_affinity(cptvf, vec: CPT_VF_INT_VEC_E_DONE); |
775 | |
776 | err = cptvf_send_vf_up(cptvf); |
777 | if (err) { |
778 | dev_err(dev, "PF not responding to UP msg" ); |
779 | goto cptvf_free_irq_affinity; |
780 | } |
781 | err = cvm_crypto_init(cptvf); |
782 | if (err) { |
783 | dev_err(dev, "Algorithm register failed\n" ); |
784 | goto cptvf_free_irq_affinity; |
785 | } |
786 | return 0; |
787 | |
788 | cptvf_free_irq_affinity: |
789 | cptvf_free_irq_affinity(cptvf, vec: CPT_VF_INT_VEC_E_DONE); |
790 | cptvf_free_irq_affinity(cptvf, vec: CPT_VF_INT_VEC_E_MISC); |
791 | cptvf_free_misc_irq: |
792 | free_irq(pci_irq_vector(dev: pdev, nr: CPT_VF_INT_VEC_E_MISC), cptvf); |
793 | cptvf_free_vectors: |
794 | pci_free_irq_vectors(dev: cptvf->pdev); |
795 | cptvf_err_release_regions: |
796 | pci_release_regions(pdev); |
797 | cptvf_err_disable_device: |
798 | pci_disable_device(dev: pdev); |
799 | pci_set_drvdata(pdev, NULL); |
800 | |
801 | return err; |
802 | } |
803 | |
804 | static void cptvf_remove(struct pci_dev *pdev) |
805 | { |
806 | struct cpt_vf *cptvf = pci_get_drvdata(pdev); |
807 | |
808 | if (!cptvf) { |
809 | dev_err(&pdev->dev, "Invalid CPT-VF device\n" ); |
810 | return; |
811 | } |
812 | |
813 | /* Convey DOWN to PF */ |
814 | if (cptvf_send_vf_down(cptvf)) { |
815 | dev_err(&pdev->dev, "PF not responding to DOWN msg" ); |
816 | } else { |
817 | cptvf_free_irq_affinity(cptvf, vec: CPT_VF_INT_VEC_E_DONE); |
818 | cptvf_free_irq_affinity(cptvf, vec: CPT_VF_INT_VEC_E_MISC); |
819 | free_irq(pci_irq_vector(dev: pdev, nr: CPT_VF_INT_VEC_E_DONE), cptvf); |
820 | free_irq(pci_irq_vector(dev: pdev, nr: CPT_VF_INT_VEC_E_MISC), cptvf); |
821 | pci_free_irq_vectors(dev: cptvf->pdev); |
822 | cptvf_sw_cleanup(cptvf); |
823 | pci_set_drvdata(pdev, NULL); |
824 | pci_release_regions(pdev); |
825 | pci_disable_device(dev: pdev); |
826 | cvm_crypto_exit(); |
827 | } |
828 | } |
829 | |
830 | static void cptvf_shutdown(struct pci_dev *pdev) |
831 | { |
832 | cptvf_remove(pdev); |
833 | } |
834 | |
835 | /* Supported devices */ |
836 | static const struct pci_device_id cptvf_id_table[] = { |
837 | {PCI_VDEVICE(CAVIUM, CPT_81XX_PCI_VF_DEVICE_ID), 0}, |
838 | { 0, } /* end of table */ |
839 | }; |
840 | |
841 | static struct pci_driver cptvf_pci_driver = { |
842 | .name = DRV_NAME, |
843 | .id_table = cptvf_id_table, |
844 | .probe = cptvf_probe, |
845 | .remove = cptvf_remove, |
846 | .shutdown = cptvf_shutdown, |
847 | }; |
848 | |
849 | module_pci_driver(cptvf_pci_driver); |
850 | |
851 | MODULE_AUTHOR("George Cherian <george.cherian@cavium.com>" ); |
852 | MODULE_DESCRIPTION("Cavium Thunder CPT Virtual Function Driver" ); |
853 | MODULE_LICENSE("GPL v2" ); |
854 | MODULE_VERSION(DRV_VERSION); |
855 | MODULE_DEVICE_TABLE(pci, cptvf_id_table); |
856 | |