1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | #include <linux/delay.h> |
3 | #include <linux/firmware.h> |
4 | #include <linux/list.h> |
5 | #include <linux/module.h> |
6 | #include <linux/mutex.h> |
7 | #include <linux/pci.h> |
8 | #include <linux/pci_ids.h> |
9 | |
10 | #include "nitrox_dev.h" |
11 | #include "nitrox_common.h" |
12 | #include "nitrox_csr.h" |
13 | #include "nitrox_hal.h" |
14 | #include "nitrox_isr.h" |
15 | #include "nitrox_debugfs.h" |
16 | |
17 | #define CNN55XX_DEV_ID 0x12 |
18 | #define UCODE_HLEN 48 |
19 | #define DEFAULT_SE_GROUP 0 |
20 | #define DEFAULT_AE_GROUP 0 |
21 | |
22 | #define DRIVER_VERSION "1.2" |
23 | #define CNN55XX_UCD_BLOCK_SIZE 32768 |
24 | #define CNN55XX_MAX_UCODE_SIZE (CNN55XX_UCD_BLOCK_SIZE * 2) |
25 | #define FW_DIR "cavium/" |
26 | /* SE microcode */ |
27 | #define SE_FW FW_DIR "cnn55xx_se.fw" |
28 | /* AE microcode */ |
29 | #define AE_FW FW_DIR "cnn55xx_ae.fw" |
30 | |
31 | static const char nitrox_driver_name[] = "CNN55XX" ; |
32 | |
33 | static LIST_HEAD(ndevlist); |
34 | static DEFINE_MUTEX(devlist_lock); |
35 | static unsigned int num_devices; |
36 | |
37 | /* |
38 | * nitrox_pci_tbl - PCI Device ID Table |
39 | */ |
40 | static const struct pci_device_id nitrox_pci_tbl[] = { |
41 | {PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0}, |
42 | /* required last entry */ |
43 | {0, } |
44 | }; |
45 | MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl); |
46 | |
47 | static unsigned int qlen = DEFAULT_CMD_QLEN; |
48 | module_param(qlen, uint, 0644); |
49 | MODULE_PARM_DESC(qlen, "Command queue length - default 2048" ); |
50 | |
51 | /** |
52 | * struct ucode - Firmware Header |
53 | * @id: microcode ID |
54 | * @version: firmware version |
55 | * @code_size: code section size |
56 | * @raz: alignment |
57 | * @code: code section |
58 | */ |
59 | struct ucode { |
60 | u8 id; |
61 | char version[VERSION_LEN - 1]; |
62 | __be32 code_size; |
63 | u8 raz[12]; |
64 | u64 code[]; |
65 | }; |
66 | |
67 | /* |
68 | * write_to_ucd_unit - Write Firmware to NITROX UCD unit |
69 | */ |
70 | static void write_to_ucd_unit(struct nitrox_device *ndev, u32 ucode_size, |
71 | u64 *ucode_data, int block_num) |
72 | { |
73 | u32 code_size; |
74 | u64 offset, data; |
75 | int i = 0; |
76 | |
77 | /* |
78 | * UCD structure |
79 | * |
80 | * ------------- |
81 | * | BLK 7 | |
82 | * ------------- |
83 | * | BLK 6 | |
84 | * ------------- |
85 | * | ... | |
86 | * ------------- |
87 | * | BLK 0 | |
88 | * ------------- |
89 | * Total of 8 blocks, each size 32KB |
90 | */ |
91 | |
92 | /* set the block number */ |
93 | offset = UCD_UCODE_LOAD_BLOCK_NUM; |
94 | nitrox_write_csr(ndev, offset, value: block_num); |
95 | |
96 | code_size = roundup(ucode_size, 16); |
97 | while (code_size) { |
98 | data = ucode_data[i]; |
99 | /* write 8 bytes at a time */ |
100 | offset = UCD_UCODE_LOAD_IDX_DATAX(i); |
101 | nitrox_write_csr(ndev, offset, value: data); |
102 | code_size -= 8; |
103 | i++; |
104 | } |
105 | |
106 | usleep_range(min: 300, max: 400); |
107 | } |
108 | |
109 | static int nitrox_load_fw(struct nitrox_device *ndev) |
110 | { |
111 | const struct firmware *fw; |
112 | const char *fw_name; |
113 | struct ucode *ucode; |
114 | u64 *ucode_data; |
115 | u64 offset; |
116 | union ucd_core_eid_ucode_block_num core_2_eid_val; |
117 | union aqm_grp_execmsk_lo aqm_grp_execmask_lo; |
118 | union aqm_grp_execmsk_hi aqm_grp_execmask_hi; |
119 | u32 ucode_size; |
120 | int ret, i = 0; |
121 | |
122 | fw_name = SE_FW; |
123 | dev_info(DEV(ndev), "Loading firmware \"%s\"\n" , fw_name); |
124 | |
125 | ret = request_firmware(fw: &fw, name: fw_name, DEV(ndev)); |
126 | if (ret < 0) { |
127 | dev_err(DEV(ndev), "failed to get firmware %s\n" , fw_name); |
128 | return ret; |
129 | } |
130 | |
131 | ucode = (struct ucode *)fw->data; |
132 | |
133 | ucode_size = be32_to_cpu(ucode->code_size) * 2; |
134 | if (!ucode_size || ucode_size > CNN55XX_MAX_UCODE_SIZE) { |
135 | dev_err(DEV(ndev), "Invalid ucode size: %u for firmware %s\n" , |
136 | ucode_size, fw_name); |
137 | release_firmware(fw); |
138 | return -EINVAL; |
139 | } |
140 | ucode_data = ucode->code; |
141 | |
142 | /* copy the firmware version */ |
143 | memcpy(&ndev->hw.fw_name[0][0], ucode->version, (VERSION_LEN - 2)); |
144 | ndev->hw.fw_name[0][VERSION_LEN - 1] = '\0'; |
145 | |
146 | /* Load SE Firmware on UCD Block 0 */ |
147 | write_to_ucd_unit(ndev, ucode_size, ucode_data, block_num: 0); |
148 | |
149 | release_firmware(fw); |
150 | |
151 | /* put all SE cores in DEFAULT_SE_GROUP */ |
152 | offset = POM_GRP_EXECMASKX(DEFAULT_SE_GROUP); |
153 | nitrox_write_csr(ndev, offset, value: (~0ULL)); |
154 | |
155 | /* write block number and firmware length |
156 | * bit:<2:0> block number |
157 | * bit:3 is set SE uses 32KB microcode |
158 | * bit:3 is clear SE uses 64KB microcode |
159 | */ |
160 | core_2_eid_val.value = 0ULL; |
161 | core_2_eid_val.ucode_blk = 0; |
162 | if (ucode_size <= CNN55XX_UCD_BLOCK_SIZE) |
163 | core_2_eid_val.ucode_len = 1; |
164 | else |
165 | core_2_eid_val.ucode_len = 0; |
166 | |
167 | for (i = 0; i < ndev->hw.se_cores; i++) { |
168 | offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i); |
169 | nitrox_write_csr(ndev, offset, value: core_2_eid_val.value); |
170 | } |
171 | |
172 | |
173 | fw_name = AE_FW; |
174 | dev_info(DEV(ndev), "Loading firmware \"%s\"\n" , fw_name); |
175 | |
176 | ret = request_firmware(fw: &fw, name: fw_name, DEV(ndev)); |
177 | if (ret < 0) { |
178 | dev_err(DEV(ndev), "failed to get firmware %s\n" , fw_name); |
179 | return ret; |
180 | } |
181 | |
182 | ucode = (struct ucode *)fw->data; |
183 | |
184 | ucode_size = be32_to_cpu(ucode->code_size) * 2; |
185 | if (!ucode_size || ucode_size > CNN55XX_MAX_UCODE_SIZE) { |
186 | dev_err(DEV(ndev), "Invalid ucode size: %u for firmware %s\n" , |
187 | ucode_size, fw_name); |
188 | release_firmware(fw); |
189 | return -EINVAL; |
190 | } |
191 | ucode_data = ucode->code; |
192 | |
193 | /* copy the firmware version */ |
194 | memcpy(&ndev->hw.fw_name[1][0], ucode->version, (VERSION_LEN - 2)); |
195 | ndev->hw.fw_name[1][VERSION_LEN - 1] = '\0'; |
196 | |
197 | /* Load AE Firmware on UCD Block 2 */ |
198 | write_to_ucd_unit(ndev, ucode_size, ucode_data, block_num: 2); |
199 | |
200 | release_firmware(fw); |
201 | |
202 | /* put all AE cores in DEFAULT_AE_GROUP */ |
203 | offset = AQM_GRP_EXECMSK_LOX(DEFAULT_AE_GROUP); |
204 | aqm_grp_execmask_lo.exec_0_to_39 = 0xFFFFFFFFFFULL; |
205 | nitrox_write_csr(ndev, offset, value: aqm_grp_execmask_lo.value); |
206 | offset = AQM_GRP_EXECMSK_HIX(DEFAULT_AE_GROUP); |
207 | aqm_grp_execmask_hi.exec_40_to_79 = 0xFFFFFFFFFFULL; |
208 | nitrox_write_csr(ndev, offset, value: aqm_grp_execmask_hi.value); |
209 | |
210 | /* write block number and firmware length |
211 | * bit:<2:0> block number |
212 | * bit:3 is set AE uses 32KB microcode |
213 | * bit:3 is clear AE uses 64KB microcode |
214 | */ |
215 | core_2_eid_val.value = 0ULL; |
216 | core_2_eid_val.ucode_blk = 2; |
217 | if (ucode_size <= CNN55XX_UCD_BLOCK_SIZE) |
218 | core_2_eid_val.ucode_len = 1; |
219 | else |
220 | core_2_eid_val.ucode_len = 0; |
221 | |
222 | for (i = 0; i < ndev->hw.ae_cores; i++) { |
223 | offset = UCD_AE_EID_UCODE_BLOCK_NUMX(i); |
224 | nitrox_write_csr(ndev, offset, value: core_2_eid_val.value); |
225 | } |
226 | |
227 | return 0; |
228 | } |
229 | |
230 | /** |
231 | * nitrox_add_to_devlist - add NITROX device to global device list |
232 | * @ndev: NITROX device |
233 | */ |
234 | static int nitrox_add_to_devlist(struct nitrox_device *ndev) |
235 | { |
236 | struct nitrox_device *dev; |
237 | int ret = 0; |
238 | |
239 | INIT_LIST_HEAD(list: &ndev->list); |
240 | refcount_set(r: &ndev->refcnt, n: 1); |
241 | |
242 | mutex_lock(&devlist_lock); |
243 | list_for_each_entry(dev, &ndevlist, list) { |
244 | if (dev == ndev) { |
245 | ret = -EEXIST; |
246 | goto unlock; |
247 | } |
248 | } |
249 | ndev->idx = num_devices++; |
250 | list_add_tail(new: &ndev->list, head: &ndevlist); |
251 | unlock: |
252 | mutex_unlock(lock: &devlist_lock); |
253 | return ret; |
254 | } |
255 | |
256 | /** |
257 | * nitrox_remove_from_devlist - remove NITROX device from |
258 | * global device list |
259 | * @ndev: NITROX device |
260 | */ |
261 | static void nitrox_remove_from_devlist(struct nitrox_device *ndev) |
262 | { |
263 | mutex_lock(&devlist_lock); |
264 | list_del(entry: &ndev->list); |
265 | num_devices--; |
266 | mutex_unlock(lock: &devlist_lock); |
267 | } |
268 | |
269 | struct nitrox_device *nitrox_get_first_device(void) |
270 | { |
271 | struct nitrox_device *ndev = NULL, *iter; |
272 | |
273 | mutex_lock(&devlist_lock); |
274 | list_for_each_entry(iter, &ndevlist, list) { |
275 | if (nitrox_ready(ndev: iter)) { |
276 | ndev = iter; |
277 | break; |
278 | } |
279 | } |
280 | mutex_unlock(lock: &devlist_lock); |
281 | if (!ndev) |
282 | return NULL; |
283 | |
284 | refcount_inc(r: &ndev->refcnt); |
285 | /* barrier to sync with other cpus */ |
286 | smp_mb__after_atomic(); |
287 | return ndev; |
288 | } |
289 | |
290 | void nitrox_put_device(struct nitrox_device *ndev) |
291 | { |
292 | if (!ndev) |
293 | return; |
294 | |
295 | refcount_dec(r: &ndev->refcnt); |
296 | /* barrier to sync with other cpus */ |
297 | smp_mb__after_atomic(); |
298 | } |
299 | |
300 | static int nitrox_device_flr(struct pci_dev *pdev) |
301 | { |
302 | int pos = 0; |
303 | |
304 | pos = pci_save_state(dev: pdev); |
305 | if (pos) { |
306 | dev_err(&pdev->dev, "Failed to save pci state\n" ); |
307 | return -ENOMEM; |
308 | } |
309 | |
310 | pcie_reset_flr(dev: pdev, PCI_RESET_DO_RESET); |
311 | |
312 | pci_restore_state(dev: pdev); |
313 | |
314 | return 0; |
315 | } |
316 | |
317 | static int nitrox_pf_sw_init(struct nitrox_device *ndev) |
318 | { |
319 | int err; |
320 | |
321 | err = nitrox_common_sw_init(ndev); |
322 | if (err) |
323 | return err; |
324 | |
325 | err = nitrox_register_interrupts(ndev); |
326 | if (err) |
327 | nitrox_common_sw_cleanup(ndev); |
328 | |
329 | return err; |
330 | } |
331 | |
332 | static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev) |
333 | { |
334 | nitrox_unregister_interrupts(ndev); |
335 | nitrox_common_sw_cleanup(ndev); |
336 | } |
337 | |
338 | /** |
339 | * nitrox_bist_check - Check NITROX BIST registers status |
340 | * @ndev: NITROX device |
341 | */ |
342 | static int nitrox_bist_check(struct nitrox_device *ndev) |
343 | { |
344 | u64 value = 0; |
345 | int i; |
346 | |
347 | for (i = 0; i < NR_CLUSTERS; i++) { |
348 | value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i)); |
349 | value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i)); |
350 | } |
351 | value += nitrox_read_csr(ndev, UCD_BIST_STATUS); |
352 | value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG); |
353 | value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG); |
354 | value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG); |
355 | value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG); |
356 | value += nitrox_read_csr(ndev, POM_BIST_REG); |
357 | value += nitrox_read_csr(ndev, BMI_BIST_REG); |
358 | value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT); |
359 | value += nitrox_read_csr(ndev, BMO_BIST_REG); |
360 | value += nitrox_read_csr(ndev, LBC_BIST_STATUS); |
361 | value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0)); |
362 | if (value) |
363 | return -EIO; |
364 | return 0; |
365 | } |
366 | |
367 | static int nitrox_pf_hw_init(struct nitrox_device *ndev) |
368 | { |
369 | int err; |
370 | |
371 | err = nitrox_bist_check(ndev); |
372 | if (err) { |
373 | dev_err(&ndev->pdev->dev, "BIST check failed\n" ); |
374 | return err; |
375 | } |
376 | /* get cores information */ |
377 | nitrox_get_hwinfo(ndev); |
378 | |
379 | nitrox_config_nps_core_unit(ndev); |
380 | nitrox_config_aqm_unit(ndev); |
381 | nitrox_config_nps_pkt_unit(ndev); |
382 | nitrox_config_pom_unit(ndev); |
383 | nitrox_config_efl_unit(ndev); |
384 | /* configure IO units */ |
385 | nitrox_config_bmi_unit(ndev); |
386 | nitrox_config_bmo_unit(ndev); |
387 | /* configure Local Buffer Cache */ |
388 | nitrox_config_lbc_unit(ndev); |
389 | nitrox_config_rand_unit(ndev); |
390 | |
391 | /* load firmware on cores */ |
392 | err = nitrox_load_fw(ndev); |
393 | if (err) |
394 | return err; |
395 | |
396 | nitrox_config_emu_unit(ndev); |
397 | |
398 | return 0; |
399 | } |
400 | |
401 | /** |
402 | * nitrox_probe - NITROX Initialization function. |
403 | * @pdev: PCI device information struct |
404 | * @id: entry in nitrox_pci_tbl |
405 | * |
406 | * Return: 0, if the driver is bound to the device, or |
407 | * a negative error if there is failure. |
408 | */ |
409 | static int nitrox_probe(struct pci_dev *pdev, |
410 | const struct pci_device_id *id) |
411 | { |
412 | struct nitrox_device *ndev; |
413 | int err; |
414 | |
415 | dev_info_once(&pdev->dev, "%s driver version %s\n" , |
416 | nitrox_driver_name, DRIVER_VERSION); |
417 | |
418 | err = pci_enable_device_mem(dev: pdev); |
419 | if (err) |
420 | return err; |
421 | |
422 | /* do FLR */ |
423 | err = nitrox_device_flr(pdev); |
424 | if (err) { |
425 | dev_err(&pdev->dev, "FLR failed\n" ); |
426 | goto flr_fail; |
427 | } |
428 | |
429 | if (!dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(64))) { |
430 | dev_dbg(&pdev->dev, "DMA to 64-BIT address\n" ); |
431 | } else { |
432 | err = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(32)); |
433 | if (err) { |
434 | dev_err(&pdev->dev, "DMA configuration failed\n" ); |
435 | goto flr_fail; |
436 | } |
437 | } |
438 | |
439 | err = pci_request_mem_regions(pdev, name: nitrox_driver_name); |
440 | if (err) |
441 | goto flr_fail; |
442 | pci_set_master(dev: pdev); |
443 | |
444 | ndev = kzalloc(size: sizeof(*ndev), GFP_KERNEL); |
445 | if (!ndev) { |
446 | err = -ENOMEM; |
447 | goto ndev_fail; |
448 | } |
449 | |
450 | pci_set_drvdata(pdev, data: ndev); |
451 | ndev->pdev = pdev; |
452 | |
453 | /* add to device list */ |
454 | nitrox_add_to_devlist(ndev); |
455 | |
456 | ndev->hw.vendor_id = pdev->vendor; |
457 | ndev->hw.device_id = pdev->device; |
458 | ndev->hw.revision_id = pdev->revision; |
459 | /* command timeout in jiffies */ |
460 | ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT); |
461 | ndev->node = dev_to_node(dev: &pdev->dev); |
462 | if (ndev->node == NUMA_NO_NODE) |
463 | ndev->node = 0; |
464 | |
465 | ndev->bar_addr = ioremap(pci_resource_start(pdev, 0), |
466 | pci_resource_len(pdev, 0)); |
467 | if (!ndev->bar_addr) { |
468 | err = -EIO; |
469 | goto ioremap_err; |
470 | } |
471 | /* allocate command queus based on cpus, max queues are 64 */ |
472 | ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus()); |
473 | ndev->qlen = qlen; |
474 | |
475 | err = nitrox_pf_sw_init(ndev); |
476 | if (err) |
477 | goto pf_sw_fail; |
478 | |
479 | err = nitrox_pf_hw_init(ndev); |
480 | if (err) |
481 | goto pf_hw_fail; |
482 | |
483 | nitrox_debugfs_init(ndev); |
484 | |
485 | /* clear the statistics */ |
486 | atomic64_set(v: &ndev->stats.posted, i: 0); |
487 | atomic64_set(v: &ndev->stats.completed, i: 0); |
488 | atomic64_set(v: &ndev->stats.dropped, i: 0); |
489 | |
490 | atomic_set(v: &ndev->state, i: __NDEV_READY); |
491 | /* barrier to sync with other cpus */ |
492 | smp_mb__after_atomic(); |
493 | |
494 | err = nitrox_crypto_register(); |
495 | if (err) |
496 | goto crypto_fail; |
497 | |
498 | return 0; |
499 | |
500 | crypto_fail: |
501 | nitrox_debugfs_exit(ndev); |
502 | atomic_set(v: &ndev->state, i: __NDEV_NOT_READY); |
503 | /* barrier to sync with other cpus */ |
504 | smp_mb__after_atomic(); |
505 | pf_hw_fail: |
506 | nitrox_pf_sw_cleanup(ndev); |
507 | pf_sw_fail: |
508 | iounmap(addr: ndev->bar_addr); |
509 | ioremap_err: |
510 | nitrox_remove_from_devlist(ndev); |
511 | kfree(objp: ndev); |
512 | pci_set_drvdata(pdev, NULL); |
513 | ndev_fail: |
514 | pci_release_mem_regions(pdev); |
515 | flr_fail: |
516 | pci_disable_device(dev: pdev); |
517 | return err; |
518 | } |
519 | |
520 | /** |
521 | * nitrox_remove - Unbind the driver from the device. |
522 | * @pdev: PCI device information struct |
523 | */ |
524 | static void nitrox_remove(struct pci_dev *pdev) |
525 | { |
526 | struct nitrox_device *ndev = pci_get_drvdata(pdev); |
527 | |
528 | if (!ndev) |
529 | return; |
530 | |
531 | if (!refcount_dec_and_test(r: &ndev->refcnt)) { |
532 | dev_err(DEV(ndev), "Device refcnt not zero (%d)\n" , |
533 | refcount_read(&ndev->refcnt)); |
534 | return; |
535 | } |
536 | |
537 | dev_info(DEV(ndev), "Removing Device %x:%x\n" , |
538 | ndev->hw.vendor_id, ndev->hw.device_id); |
539 | |
540 | atomic_set(v: &ndev->state, i: __NDEV_NOT_READY); |
541 | /* barrier to sync with other cpus */ |
542 | smp_mb__after_atomic(); |
543 | |
544 | nitrox_remove_from_devlist(ndev); |
545 | |
546 | /* disable SR-IOV */ |
547 | nitrox_sriov_configure(pdev, num_vfs: 0); |
548 | nitrox_crypto_unregister(); |
549 | nitrox_debugfs_exit(ndev); |
550 | nitrox_pf_sw_cleanup(ndev); |
551 | |
552 | iounmap(addr: ndev->bar_addr); |
553 | kfree(objp: ndev); |
554 | |
555 | pci_set_drvdata(pdev, NULL); |
556 | pci_release_mem_regions(pdev); |
557 | pci_disable_device(dev: pdev); |
558 | } |
559 | |
560 | static void nitrox_shutdown(struct pci_dev *pdev) |
561 | { |
562 | pci_set_drvdata(pdev, NULL); |
563 | pci_release_mem_regions(pdev); |
564 | pci_disable_device(dev: pdev); |
565 | } |
566 | |
567 | static struct pci_driver nitrox_driver = { |
568 | .name = nitrox_driver_name, |
569 | .id_table = nitrox_pci_tbl, |
570 | .probe = nitrox_probe, |
571 | .remove = nitrox_remove, |
572 | .shutdown = nitrox_shutdown, |
573 | .sriov_configure = nitrox_sriov_configure, |
574 | }; |
575 | |
576 | module_pci_driver(nitrox_driver); |
577 | |
578 | MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>" ); |
579 | MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " " ); |
580 | MODULE_LICENSE("GPL" ); |
581 | MODULE_VERSION(DRIVER_VERSION); |
582 | MODULE_FIRMWARE(SE_FW); |
583 | |