1// SPDX-License-Identifier: GPL-2.0
2#include <linux/pci.h>
3#include <linux/printk.h>
4#include <linux/slab.h>
5
6#include "nitrox_dev.h"
7#include "nitrox_csr.h"
8#include "nitrox_common.h"
9#include "nitrox_hal.h"
10#include "nitrox_isr.h"
11#include "nitrox_mbx.h"
12
13/*
14 * One vector for each type of ring
15 * - NPS packet ring, AQMQ ring and ZQMQ ring
16 */
17#define NR_RING_VECTORS 3
18#define NR_NON_RING_VECTORS 1
19/* base entry for packet ring/port */
20#define PKT_RING_MSIX_BASE 0
21#define NON_RING_MSIX_BASE 192
22
23/**
24 * nps_pkt_slc_isr - IRQ handler for NPS solicit port
25 * @irq: irq number
26 * @data: argument
27 */
28static irqreturn_t nps_pkt_slc_isr(int irq, void *data)
29{
30 struct nitrox_q_vector *qvec = data;
31 union nps_pkt_slc_cnts slc_cnts;
32 struct nitrox_cmdq *cmdq = qvec->cmdq;
33
34 slc_cnts.value = readq(addr: cmdq->compl_cnt_csr_addr);
35 /* New packet on SLC output port */
36 if (slc_cnts.s.slc_int)
37 tasklet_hi_schedule(t: &qvec->resp_tasklet);
38
39 return IRQ_HANDLED;
40}
41
42static void clear_nps_core_err_intr(struct nitrox_device *ndev)
43{
44 u64 value;
45
46 /* Write 1 to clear */
47 value = nitrox_read_csr(ndev, NPS_CORE_INT);
48 nitrox_write_csr(ndev, NPS_CORE_INT, value);
49
50 dev_err_ratelimited(DEV(ndev), "NSP_CORE_INT 0x%016llx\n", value);
51}
52
53static void clear_nps_pkt_err_intr(struct nitrox_device *ndev)
54{
55 union nps_pkt_int pkt_int;
56 unsigned long value, offset;
57 int i;
58
59 pkt_int.value = nitrox_read_csr(ndev, NPS_PKT_INT);
60 dev_err_ratelimited(DEV(ndev), "NPS_PKT_INT 0x%016llx\n",
61 pkt_int.value);
62
63 if (pkt_int.s.slc_err) {
64 offset = NPS_PKT_SLC_ERR_TYPE;
65 value = nitrox_read_csr(ndev, offset);
66 nitrox_write_csr(ndev, offset, value);
67 dev_err_ratelimited(DEV(ndev),
68 "NPS_PKT_SLC_ERR_TYPE 0x%016lx\n", value);
69
70 offset = NPS_PKT_SLC_RERR_LO;
71 value = nitrox_read_csr(ndev, offset);
72 nitrox_write_csr(ndev, offset, value);
73 /* enable the solicit ports */
74 for_each_set_bit(i, &value, BITS_PER_LONG)
75 enable_pkt_solicit_port(ndev, port: i);
76
77 dev_err_ratelimited(DEV(ndev),
78 "NPS_PKT_SLC_RERR_LO 0x%016lx\n", value);
79
80 offset = NPS_PKT_SLC_RERR_HI;
81 value = nitrox_read_csr(ndev, offset);
82 nitrox_write_csr(ndev, offset, value);
83 dev_err_ratelimited(DEV(ndev),
84 "NPS_PKT_SLC_RERR_HI 0x%016lx\n", value);
85 }
86
87 if (pkt_int.s.in_err) {
88 offset = NPS_PKT_IN_ERR_TYPE;
89 value = nitrox_read_csr(ndev, offset);
90 nitrox_write_csr(ndev, offset, value);
91 dev_err_ratelimited(DEV(ndev),
92 "NPS_PKT_IN_ERR_TYPE 0x%016lx\n", value);
93 offset = NPS_PKT_IN_RERR_LO;
94 value = nitrox_read_csr(ndev, offset);
95 nitrox_write_csr(ndev, offset, value);
96 /* enable the input ring */
97 for_each_set_bit(i, &value, BITS_PER_LONG)
98 enable_pkt_input_ring(ndev, ring: i);
99
100 dev_err_ratelimited(DEV(ndev),
101 "NPS_PKT_IN_RERR_LO 0x%016lx\n", value);
102
103 offset = NPS_PKT_IN_RERR_HI;
104 value = nitrox_read_csr(ndev, offset);
105 nitrox_write_csr(ndev, offset, value);
106 dev_err_ratelimited(DEV(ndev),
107 "NPS_PKT_IN_RERR_HI 0x%016lx\n", value);
108 }
109}
110
111static void clear_pom_err_intr(struct nitrox_device *ndev)
112{
113 u64 value;
114
115 value = nitrox_read_csr(ndev, POM_INT);
116 nitrox_write_csr(ndev, POM_INT, value);
117 dev_err_ratelimited(DEV(ndev), "POM_INT 0x%016llx\n", value);
118}
119
120static void clear_pem_err_intr(struct nitrox_device *ndev)
121{
122 u64 value;
123
124 value = nitrox_read_csr(ndev, PEM0_INT);
125 nitrox_write_csr(ndev, PEM0_INT, value);
126 dev_err_ratelimited(DEV(ndev), "PEM(0)_INT 0x%016llx\n", value);
127}
128
129static void clear_lbc_err_intr(struct nitrox_device *ndev)
130{
131 union lbc_int lbc_int;
132 u64 value, offset;
133 int i;
134
135 lbc_int.value = nitrox_read_csr(ndev, LBC_INT);
136 dev_err_ratelimited(DEV(ndev), "LBC_INT 0x%016llx\n", lbc_int.value);
137
138 if (lbc_int.s.dma_rd_err) {
139 for (i = 0; i < NR_CLUSTERS; i++) {
140 offset = EFL_CORE_VF_ERR_INT0X(i);
141 value = nitrox_read_csr(ndev, offset);
142 nitrox_write_csr(ndev, offset, value);
143 offset = EFL_CORE_VF_ERR_INT1X(i);
144 value = nitrox_read_csr(ndev, offset);
145 nitrox_write_csr(ndev, offset, value);
146 }
147 }
148
149 if (lbc_int.s.cam_soft_err) {
150 dev_err_ratelimited(DEV(ndev), "CAM_SOFT_ERR, invalidating LBC\n");
151 invalidate_lbc(ndev);
152 }
153
154 if (lbc_int.s.pref_dat_len_mismatch_err) {
155 offset = LBC_PLM_VF1_64_INT;
156 value = nitrox_read_csr(ndev, offset);
157 nitrox_write_csr(ndev, offset, value);
158 offset = LBC_PLM_VF65_128_INT;
159 value = nitrox_read_csr(ndev, offset);
160 nitrox_write_csr(ndev, offset, value);
161 }
162
163 if (lbc_int.s.rd_dat_len_mismatch_err) {
164 offset = LBC_ELM_VF1_64_INT;
165 value = nitrox_read_csr(ndev, offset);
166 nitrox_write_csr(ndev, offset, value);
167 offset = LBC_ELM_VF65_128_INT;
168 value = nitrox_read_csr(ndev, offset);
169 nitrox_write_csr(ndev, offset, value);
170 }
171 nitrox_write_csr(ndev, LBC_INT, value: lbc_int.value);
172}
173
174static void clear_efl_err_intr(struct nitrox_device *ndev)
175{
176 int i;
177
178 for (i = 0; i < NR_CLUSTERS; i++) {
179 union efl_core_int core_int;
180 u64 value, offset;
181
182 offset = EFL_CORE_INTX(i);
183 core_int.value = nitrox_read_csr(ndev, offset);
184 nitrox_write_csr(ndev, offset, value: core_int.value);
185 dev_err_ratelimited(DEV(ndev), "ELF_CORE(%d)_INT 0x%016llx\n",
186 i, core_int.value);
187 if (core_int.s.se_err) {
188 offset = EFL_CORE_SE_ERR_INTX(i);
189 value = nitrox_read_csr(ndev, offset);
190 nitrox_write_csr(ndev, offset, value);
191 }
192 }
193}
194
195static void clear_bmi_err_intr(struct nitrox_device *ndev)
196{
197 u64 value;
198
199 value = nitrox_read_csr(ndev, BMI_INT);
200 nitrox_write_csr(ndev, BMI_INT, value);
201 dev_err_ratelimited(DEV(ndev), "BMI_INT 0x%016llx\n", value);
202}
203
204static void nps_core_int_tasklet(unsigned long data)
205{
206 struct nitrox_q_vector *qvec = (void *)(uintptr_t)(data);
207 struct nitrox_device *ndev = qvec->ndev;
208
209 /* if pf mode do queue recovery */
210 if (ndev->mode == __NDEV_MODE_PF) {
211 } else {
212 /**
213 * if VF(s) enabled communicate the error information
214 * to VF(s)
215 */
216 }
217}
218
219/*
220 * nps_core_int_isr - interrupt handler for NITROX errors and
221 * mailbox communication
222 */
223static irqreturn_t nps_core_int_isr(int irq, void *data)
224{
225 struct nitrox_q_vector *qvec = data;
226 struct nitrox_device *ndev = qvec->ndev;
227 union nps_core_int_active core_int;
228
229 core_int.value = nitrox_read_csr(ndev, NPS_CORE_INT_ACTIVE);
230
231 if (core_int.s.nps_core)
232 clear_nps_core_err_intr(ndev);
233
234 if (core_int.s.nps_pkt)
235 clear_nps_pkt_err_intr(ndev);
236
237 if (core_int.s.pom)
238 clear_pom_err_intr(ndev);
239
240 if (core_int.s.pem)
241 clear_pem_err_intr(ndev);
242
243 if (core_int.s.lbc)
244 clear_lbc_err_intr(ndev);
245
246 if (core_int.s.efl)
247 clear_efl_err_intr(ndev);
248
249 if (core_int.s.bmi)
250 clear_bmi_err_intr(ndev);
251
252 /* Mailbox interrupt */
253 if (core_int.s.mbox)
254 nitrox_pf2vf_mbox_handler(ndev);
255
256 /* If more work callback the ISR, set resend */
257 core_int.s.resend = 1;
258 nitrox_write_csr(ndev, NPS_CORE_INT_ACTIVE, value: core_int.value);
259
260 return IRQ_HANDLED;
261}
262
263void nitrox_unregister_interrupts(struct nitrox_device *ndev)
264{
265 struct pci_dev *pdev = ndev->pdev;
266 int i;
267
268 for (i = 0; i < ndev->num_vecs; i++) {
269 struct nitrox_q_vector *qvec;
270 int vec;
271
272 qvec = ndev->qvec + i;
273 if (!qvec->valid)
274 continue;
275
276 /* get the vector number */
277 vec = pci_irq_vector(dev: pdev, nr: i);
278 irq_set_affinity_hint(irq: vec, NULL);
279 free_irq(vec, qvec);
280
281 tasklet_disable(t: &qvec->resp_tasklet);
282 tasklet_kill(t: &qvec->resp_tasklet);
283 qvec->valid = false;
284 }
285 kfree(objp: ndev->qvec);
286 ndev->qvec = NULL;
287 pci_free_irq_vectors(dev: pdev);
288}
289
290int nitrox_register_interrupts(struct nitrox_device *ndev)
291{
292 struct pci_dev *pdev = ndev->pdev;
293 struct nitrox_q_vector *qvec;
294 int nr_vecs, vec, cpu;
295 int ret, i;
296
297 /*
298 * PF MSI-X vectors
299 *
300 * Entry 0: NPS PKT ring 0
301 * Entry 1: AQMQ ring 0
302 * Entry 2: ZQM ring 0
303 * Entry 3: NPS PKT ring 1
304 * Entry 4: AQMQ ring 1
305 * Entry 5: ZQM ring 1
306 * ....
307 * Entry 192: NPS_CORE_INT_ACTIVE
308 */
309 nr_vecs = pci_msix_vec_count(dev: pdev);
310 if (nr_vecs < 0) {
311 dev_err(DEV(ndev), "Error in getting vec count %d\n", nr_vecs);
312 return nr_vecs;
313 }
314
315 /* Enable MSI-X */
316 ret = pci_alloc_irq_vectors(dev: pdev, min_vecs: nr_vecs, max_vecs: nr_vecs, PCI_IRQ_MSIX);
317 if (ret < 0) {
318 dev_err(DEV(ndev), "msix vectors %d alloc failed\n", nr_vecs);
319 return ret;
320 }
321 ndev->num_vecs = nr_vecs;
322
323 ndev->qvec = kcalloc(n: nr_vecs, size: sizeof(*qvec), GFP_KERNEL);
324 if (!ndev->qvec) {
325 pci_free_irq_vectors(dev: pdev);
326 return -ENOMEM;
327 }
328
329 /* request irqs for packet rings/ports */
330 for (i = PKT_RING_MSIX_BASE; i < (nr_vecs - 1); i += NR_RING_VECTORS) {
331 qvec = &ndev->qvec[i];
332
333 qvec->ring = i / NR_RING_VECTORS;
334 if (qvec->ring >= ndev->nr_queues)
335 break;
336
337 qvec->cmdq = &ndev->pkt_inq[qvec->ring];
338 snprintf(buf: qvec->name, IRQ_NAMESZ, fmt: "nitrox-pkt%d", qvec->ring);
339 /* get the vector number */
340 vec = pci_irq_vector(dev: pdev, nr: i);
341 ret = request_irq(irq: vec, handler: nps_pkt_slc_isr, flags: 0, name: qvec->name, dev: qvec);
342 if (ret) {
343 dev_err(DEV(ndev), "irq failed for pkt ring/port%d\n",
344 qvec->ring);
345 goto irq_fail;
346 }
347 cpu = qvec->ring % num_online_cpus();
348 irq_set_affinity_hint(irq: vec, m: get_cpu_mask(cpu));
349
350 tasklet_init(t: &qvec->resp_tasklet, func: pkt_slc_resp_tasklet,
351 data: (unsigned long)qvec);
352 qvec->valid = true;
353 }
354
355 /* request irqs for non ring vectors */
356 i = NON_RING_MSIX_BASE;
357 qvec = &ndev->qvec[i];
358 qvec->ndev = ndev;
359
360 snprintf(buf: qvec->name, IRQ_NAMESZ, fmt: "nitrox-core-int%d", i);
361 /* get the vector number */
362 vec = pci_irq_vector(dev: pdev, nr: i);
363 ret = request_irq(irq: vec, handler: nps_core_int_isr, flags: 0, name: qvec->name, dev: qvec);
364 if (ret) {
365 dev_err(DEV(ndev), "irq failed for nitrox-core-int%d\n", i);
366 goto irq_fail;
367 }
368 cpu = num_online_cpus();
369 irq_set_affinity_hint(irq: vec, m: get_cpu_mask(cpu));
370
371 tasklet_init(t: &qvec->resp_tasklet, func: nps_core_int_tasklet,
372 data: (unsigned long)qvec);
373 qvec->valid = true;
374
375 return 0;
376
377irq_fail:
378 nitrox_unregister_interrupts(ndev);
379 return ret;
380}
381
382void nitrox_sriov_unregister_interrupts(struct nitrox_device *ndev)
383{
384 struct pci_dev *pdev = ndev->pdev;
385 int i;
386
387 for (i = 0; i < ndev->num_vecs; i++) {
388 struct nitrox_q_vector *qvec;
389 int vec;
390
391 qvec = ndev->qvec + i;
392 if (!qvec->valid)
393 continue;
394
395 vec = ndev->iov.msix.vector;
396 irq_set_affinity_hint(irq: vec, NULL);
397 free_irq(vec, qvec);
398
399 tasklet_disable(t: &qvec->resp_tasklet);
400 tasklet_kill(t: &qvec->resp_tasklet);
401 qvec->valid = false;
402 }
403 kfree(objp: ndev->qvec);
404 ndev->qvec = NULL;
405 pci_disable_msix(dev: pdev);
406}
407
408int nitrox_sriov_register_interupts(struct nitrox_device *ndev)
409{
410 struct pci_dev *pdev = ndev->pdev;
411 struct nitrox_q_vector *qvec;
412 int vec, cpu;
413 int ret;
414
415 /**
416 * only non ring vectors i.e Entry 192 is available
417 * for PF in SR-IOV mode.
418 */
419 ndev->iov.msix.entry = NON_RING_MSIX_BASE;
420 ret = pci_enable_msix_exact(dev: pdev, entries: &ndev->iov.msix, NR_NON_RING_VECTORS);
421 if (ret) {
422 dev_err(DEV(ndev), "failed to allocate nps-core-int%d\n",
423 NON_RING_MSIX_BASE);
424 return ret;
425 }
426
427 qvec = kcalloc(NR_NON_RING_VECTORS, size: sizeof(*qvec), GFP_KERNEL);
428 if (!qvec) {
429 pci_disable_msix(dev: pdev);
430 return -ENOMEM;
431 }
432 qvec->ndev = ndev;
433
434 ndev->qvec = qvec;
435 ndev->num_vecs = NR_NON_RING_VECTORS;
436 snprintf(buf: qvec->name, IRQ_NAMESZ, fmt: "nitrox-core-int%d",
437 NON_RING_MSIX_BASE);
438
439 vec = ndev->iov.msix.vector;
440 ret = request_irq(irq: vec, handler: nps_core_int_isr, flags: 0, name: qvec->name, dev: qvec);
441 if (ret) {
442 dev_err(DEV(ndev), "irq failed for nitrox-core-int%d\n",
443 NON_RING_MSIX_BASE);
444 goto iov_irq_fail;
445 }
446 cpu = num_online_cpus();
447 irq_set_affinity_hint(irq: vec, m: get_cpu_mask(cpu));
448
449 tasklet_init(t: &qvec->resp_tasklet, func: nps_core_int_tasklet,
450 data: (unsigned long)qvec);
451 qvec->valid = true;
452
453 return 0;
454
455iov_irq_fail:
456 nitrox_sriov_unregister_interrupts(ndev);
457 return ret;
458}
459

source code of linux/drivers/crypto/cavium/nitrox/nitrox_isr.c