1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Huawei HiNIC PCI Express Linux driver |
4 | * Copyright(c) 2017 Huawei Technologies Co., Ltd |
5 | */ |
6 | |
7 | #include <linux/kernel.h> |
8 | #include <linux/types.h> |
9 | #include <linux/errno.h> |
10 | #include <linux/pci.h> |
11 | #include <linux/device.h> |
12 | #include <linux/slab.h> |
13 | #include <linux/dma-mapping.h> |
14 | #include <linux/bitops.h> |
15 | #include <linux/err.h> |
16 | #include <linux/jiffies.h> |
17 | #include <linux/delay.h> |
18 | #include <linux/log2.h> |
19 | #include <linux/semaphore.h> |
20 | #include <asm/byteorder.h> |
21 | #include <asm/barrier.h> |
22 | |
23 | #include "hinic_hw_csr.h" |
24 | #include "hinic_hw_if.h" |
25 | #include "hinic_hw_api_cmd.h" |
26 | |
27 | #define API_CHAIN_NUM_CELLS 32 |
28 | |
29 | #define API_CMD_CELL_SIZE_SHIFT 6 |
30 | #define API_CMD_CELL_SIZE_MIN (BIT(API_CMD_CELL_SIZE_SHIFT)) |
31 | |
32 | #define API_CMD_CELL_SIZE(cell_size) \ |
33 | (((cell_size) >= API_CMD_CELL_SIZE_MIN) ? \ |
34 | (1 << (fls(cell_size - 1))) : API_CMD_CELL_SIZE_MIN) |
35 | |
36 | #define API_CMD_CELL_SIZE_VAL(size) \ |
37 | ilog2((size) >> API_CMD_CELL_SIZE_SHIFT) |
38 | |
39 | #define API_CMD_BUF_SIZE 2048 |
40 | |
41 | /* Sizes of the members in hinic_api_cmd_cell */ |
42 | #define API_CMD_CELL_DESC_SIZE 8 |
43 | #define API_CMD_CELL_DATA_ADDR_SIZE 8 |
44 | |
45 | #define API_CMD_CELL_ALIGNMENT 8 |
46 | |
47 | #define API_CMD_TIMEOUT 1000 |
48 | |
49 | #define MASKED_IDX(chain, idx) ((idx) & ((chain)->num_cells - 1)) |
50 | |
51 | #define SIZE_8BYTES(size) (ALIGN((size), 8) >> 3) |
52 | #define SIZE_4BYTES(size) (ALIGN((size), 4) >> 2) |
53 | |
54 | #define RD_DMA_ATTR_DEFAULT 0 |
55 | #define WR_DMA_ATTR_DEFAULT 0 |
56 | |
57 | enum api_cmd_data_format { |
58 | SGE_DATA = 1, /* cell data is passed by hw address */ |
59 | }; |
60 | |
61 | enum api_cmd_type { |
62 | API_CMD_WRITE = 0, |
63 | }; |
64 | |
65 | enum api_cmd_bypass { |
66 | NO_BYPASS = 0, |
67 | BYPASS = 1, |
68 | }; |
69 | |
70 | enum api_cmd_xor_chk_level { |
71 | XOR_CHK_DIS = 0, |
72 | |
73 | XOR_CHK_ALL = 3, |
74 | }; |
75 | |
76 | static u8 xor_chksum_set(void *data) |
77 | { |
78 | int idx; |
79 | u8 *val, checksum = 0; |
80 | |
81 | val = data; |
82 | |
83 | for (idx = 0; idx < 7; idx++) |
84 | checksum ^= val[idx]; |
85 | |
86 | return checksum; |
87 | } |
88 | |
89 | static void set_prod_idx(struct hinic_api_cmd_chain *chain) |
90 | { |
91 | enum hinic_api_cmd_chain_type chain_type = chain->chain_type; |
92 | struct hinic_hwif *hwif = chain->hwif; |
93 | u32 addr, prod_idx; |
94 | |
95 | addr = HINIC_CSR_API_CMD_CHAIN_PI_ADDR(chain_type); |
96 | prod_idx = hinic_hwif_read_reg(hwif, reg: addr); |
97 | |
98 | prod_idx = HINIC_API_CMD_PI_CLEAR(prod_idx, IDX); |
99 | |
100 | prod_idx |= HINIC_API_CMD_PI_SET(chain->prod_idx, IDX); |
101 | |
102 | hinic_hwif_write_reg(hwif, reg: addr, val: prod_idx); |
103 | } |
104 | |
105 | static u32 get_hw_cons_idx(struct hinic_api_cmd_chain *chain) |
106 | { |
107 | u32 addr, val; |
108 | |
109 | addr = HINIC_CSR_API_CMD_STATUS_ADDR(chain->chain_type); |
110 | val = hinic_hwif_read_reg(hwif: chain->hwif, reg: addr); |
111 | |
112 | return HINIC_API_CMD_STATUS_GET(val, CONS_IDX); |
113 | } |
114 | |
115 | static void dump_api_chain_reg(struct hinic_api_cmd_chain *chain) |
116 | { |
117 | u32 addr, val; |
118 | |
119 | addr = HINIC_CSR_API_CMD_STATUS_ADDR(chain->chain_type); |
120 | val = hinic_hwif_read_reg(hwif: chain->hwif, reg: addr); |
121 | |
122 | dev_err(&chain->hwif->pdev->dev, "Chain type: 0x%x, cpld error: 0x%x, check error: 0x%x, current fsm: 0x%x\n" , |
123 | chain->chain_type, HINIC_API_CMD_STATUS_GET(val, CPLD_ERR), |
124 | HINIC_API_CMD_STATUS_GET(val, CHKSUM_ERR), |
125 | HINIC_API_CMD_STATUS_GET(val, FSM)); |
126 | |
127 | dev_err(&chain->hwif->pdev->dev, "Chain hw current ci: 0x%x\n" , |
128 | HINIC_API_CMD_STATUS_GET(val, CONS_IDX)); |
129 | |
130 | addr = HINIC_CSR_API_CMD_CHAIN_PI_ADDR(chain->chain_type); |
131 | val = hinic_hwif_read_reg(hwif: chain->hwif, reg: addr); |
132 | dev_err(&chain->hwif->pdev->dev, "Chain hw current pi: 0x%x\n" , val); |
133 | } |
134 | |
135 | /** |
136 | * chain_busy - check if the chain is still processing last requests |
137 | * @chain: chain to check |
138 | * |
139 | * Return 0 - Success, negative - Failure |
140 | **/ |
141 | static int chain_busy(struct hinic_api_cmd_chain *chain) |
142 | { |
143 | struct hinic_hwif *hwif = chain->hwif; |
144 | struct pci_dev *pdev = hwif->pdev; |
145 | u32 prod_idx; |
146 | |
147 | switch (chain->chain_type) { |
148 | case HINIC_API_CMD_WRITE_TO_MGMT_CPU: |
149 | chain->cons_idx = get_hw_cons_idx(chain); |
150 | prod_idx = chain->prod_idx; |
151 | |
152 | /* check for a space for a new command */ |
153 | if (chain->cons_idx == MASKED_IDX(chain, prod_idx + 1)) { |
154 | dev_err(&pdev->dev, "API CMD chain %d is busy, cons_idx: %d, prod_idx: %d\n" , |
155 | chain->chain_type, chain->cons_idx, |
156 | chain->prod_idx); |
157 | dump_api_chain_reg(chain); |
158 | return -EBUSY; |
159 | } |
160 | break; |
161 | |
162 | default: |
163 | dev_err(&pdev->dev, "Unknown API CMD Chain type\n" ); |
164 | break; |
165 | } |
166 | |
167 | return 0; |
168 | } |
169 | |
170 | /** |
171 | * get_cell_data_size - get the data size of a specific cell type |
172 | * @type: chain type |
173 | * |
174 | * Return the data(Desc + Address) size in the cell |
175 | **/ |
176 | static u8 get_cell_data_size(enum hinic_api_cmd_chain_type type) |
177 | { |
178 | u8 cell_data_size = 0; |
179 | |
180 | switch (type) { |
181 | case HINIC_API_CMD_WRITE_TO_MGMT_CPU: |
182 | cell_data_size = ALIGN(API_CMD_CELL_DESC_SIZE + |
183 | API_CMD_CELL_DATA_ADDR_SIZE, |
184 | API_CMD_CELL_ALIGNMENT); |
185 | break; |
186 | default: |
187 | break; |
188 | } |
189 | |
190 | return cell_data_size; |
191 | } |
192 | |
193 | /** |
194 | * prepare_cell_ctrl - prepare the ctrl of the cell for the command |
195 | * @cell_ctrl: the control of the cell to set the control value into it |
196 | * @data_size: the size of the data in the cell |
197 | **/ |
198 | static void prepare_cell_ctrl(u64 *cell_ctrl, u16 data_size) |
199 | { |
200 | u8 chksum; |
201 | u64 ctrl; |
202 | |
203 | ctrl = HINIC_API_CMD_CELL_CTRL_SET(SIZE_8BYTES(data_size), DATA_SZ) | |
204 | HINIC_API_CMD_CELL_CTRL_SET(RD_DMA_ATTR_DEFAULT, RD_DMA_ATTR) | |
205 | HINIC_API_CMD_CELL_CTRL_SET(WR_DMA_ATTR_DEFAULT, WR_DMA_ATTR); |
206 | |
207 | chksum = xor_chksum_set(data: &ctrl); |
208 | |
209 | ctrl |= HINIC_API_CMD_CELL_CTRL_SET(chksum, XOR_CHKSUM); |
210 | |
211 | /* The data in the HW should be in Big Endian Format */ |
212 | *cell_ctrl = cpu_to_be64(ctrl); |
213 | } |
214 | |
215 | /** |
216 | * prepare_api_cmd - prepare API CMD command |
217 | * @chain: chain for the command |
218 | * @dest: destination node on the card that will receive the command |
219 | * @cmd: command data |
220 | * @cmd_size: the command size |
221 | **/ |
222 | static void prepare_api_cmd(struct hinic_api_cmd_chain *chain, |
223 | enum hinic_node_id dest, |
224 | void *cmd, u16 cmd_size) |
225 | { |
226 | struct hinic_api_cmd_cell *cell = chain->curr_node; |
227 | struct hinic_api_cmd_cell_ctxt *cell_ctxt; |
228 | struct hinic_hwif *hwif = chain->hwif; |
229 | struct pci_dev *pdev = hwif->pdev; |
230 | |
231 | cell_ctxt = &chain->cell_ctxt[chain->prod_idx]; |
232 | |
233 | switch (chain->chain_type) { |
234 | case HINIC_API_CMD_WRITE_TO_MGMT_CPU: |
235 | cell->desc = HINIC_API_CMD_DESC_SET(SGE_DATA, API_TYPE) | |
236 | HINIC_API_CMD_DESC_SET(API_CMD_WRITE, RD_WR) | |
237 | HINIC_API_CMD_DESC_SET(NO_BYPASS, MGMT_BYPASS); |
238 | break; |
239 | |
240 | default: |
241 | dev_err(&pdev->dev, "unknown Chain type\n" ); |
242 | return; |
243 | } |
244 | |
245 | cell->desc |= HINIC_API_CMD_DESC_SET(dest, DEST) | |
246 | HINIC_API_CMD_DESC_SET(SIZE_4BYTES(cmd_size), SIZE); |
247 | |
248 | cell->desc |= HINIC_API_CMD_DESC_SET(xor_chksum_set(&cell->desc), |
249 | XOR_CHKSUM); |
250 | |
251 | /* The data in the HW should be in Big Endian Format */ |
252 | cell->desc = cpu_to_be64(cell->desc); |
253 | |
254 | memcpy(cell_ctxt->api_cmd_vaddr, cmd, cmd_size); |
255 | } |
256 | |
257 | /** |
258 | * prepare_cell - prepare cell ctrl and cmd in the current cell |
259 | * @chain: chain for the command |
260 | * @dest: destination node on the card that will receive the command |
261 | * @cmd: command data |
262 | * @cmd_size: the command size |
263 | * |
264 | * Return 0 - Success, negative - Failure |
265 | **/ |
266 | static void prepare_cell(struct hinic_api_cmd_chain *chain, |
267 | enum hinic_node_id dest, |
268 | void *cmd, u16 cmd_size) |
269 | { |
270 | struct hinic_api_cmd_cell *curr_node = chain->curr_node; |
271 | u16 data_size = get_cell_data_size(type: chain->chain_type); |
272 | |
273 | prepare_cell_ctrl(cell_ctrl: &curr_node->ctrl, data_size); |
274 | prepare_api_cmd(chain, dest, cmd, cmd_size); |
275 | } |
276 | |
277 | static inline void cmd_chain_prod_idx_inc(struct hinic_api_cmd_chain *chain) |
278 | { |
279 | chain->prod_idx = MASKED_IDX(chain, chain->prod_idx + 1); |
280 | } |
281 | |
282 | /** |
283 | * api_cmd_status_update - update the status in the chain struct |
284 | * @chain: chain to update |
285 | **/ |
286 | static void api_cmd_status_update(struct hinic_api_cmd_chain *chain) |
287 | { |
288 | enum hinic_api_cmd_chain_type chain_type; |
289 | struct hinic_api_cmd_status *wb_status; |
290 | struct hinic_hwif *hwif = chain->hwif; |
291 | struct pci_dev *pdev = hwif->pdev; |
292 | u64 ; |
293 | u32 status; |
294 | |
295 | wb_status = chain->wb_status; |
296 | status_header = be64_to_cpu(wb_status->header); |
297 | |
298 | status = be32_to_cpu(wb_status->status); |
299 | if (HINIC_API_CMD_STATUS_GET(status, CHKSUM_ERR)) { |
300 | dev_err(&pdev->dev, "API CMD status: Xor check error\n" ); |
301 | return; |
302 | } |
303 | |
304 | chain_type = HINIC_API_CMD_STATUS_HEADER_GET(status_header, CHAIN_ID); |
305 | if (chain_type >= HINIC_API_CMD_MAX) { |
306 | dev_err(&pdev->dev, "unknown API CMD Chain %d\n" , chain_type); |
307 | return; |
308 | } |
309 | |
310 | chain->cons_idx = HINIC_API_CMD_STATUS_GET(status, CONS_IDX); |
311 | } |
312 | |
313 | /** |
314 | * wait_for_status_poll - wait for write to api cmd command to complete |
315 | * @chain: the chain of the command |
316 | * |
317 | * Return 0 - Success, negative - Failure |
318 | **/ |
319 | static int wait_for_status_poll(struct hinic_api_cmd_chain *chain) |
320 | { |
321 | int err = -ETIMEDOUT; |
322 | unsigned long end; |
323 | |
324 | end = jiffies + msecs_to_jiffies(API_CMD_TIMEOUT); |
325 | do { |
326 | api_cmd_status_update(chain); |
327 | |
328 | /* wait for CI to be updated - sign for completion */ |
329 | if (chain->cons_idx == chain->prod_idx) { |
330 | err = 0; |
331 | break; |
332 | } |
333 | |
334 | msleep(msecs: 20); |
335 | } while (time_before(jiffies, end)); |
336 | |
337 | return err; |
338 | } |
339 | |
340 | /** |
341 | * wait_for_api_cmd_completion - wait for command to complete |
342 | * @chain: chain for the command |
343 | * |
344 | * Return 0 - Success, negative - Failure |
345 | **/ |
346 | static int wait_for_api_cmd_completion(struct hinic_api_cmd_chain *chain) |
347 | { |
348 | struct hinic_hwif *hwif = chain->hwif; |
349 | struct pci_dev *pdev = hwif->pdev; |
350 | int err; |
351 | |
352 | switch (chain->chain_type) { |
353 | case HINIC_API_CMD_WRITE_TO_MGMT_CPU: |
354 | err = wait_for_status_poll(chain); |
355 | if (err) { |
356 | dev_err(&pdev->dev, "API CMD Poll status timeout\n" ); |
357 | dump_api_chain_reg(chain); |
358 | break; |
359 | } |
360 | break; |
361 | |
362 | default: |
363 | dev_err(&pdev->dev, "unknown API CMD Chain type\n" ); |
364 | err = -EINVAL; |
365 | break; |
366 | } |
367 | |
368 | return err; |
369 | } |
370 | |
371 | /** |
372 | * api_cmd - API CMD command |
373 | * @chain: chain for the command |
374 | * @dest: destination node on the card that will receive the command |
375 | * @cmd: command data |
376 | * @cmd_size: the command size |
377 | * |
378 | * Return 0 - Success, negative - Failure |
379 | **/ |
380 | static int api_cmd(struct hinic_api_cmd_chain *chain, |
381 | enum hinic_node_id dest, u8 *cmd, u16 cmd_size) |
382 | { |
383 | struct hinic_api_cmd_cell_ctxt *ctxt; |
384 | int err; |
385 | |
386 | down(sem: &chain->sem); |
387 | if (chain_busy(chain)) { |
388 | up(sem: &chain->sem); |
389 | return -EBUSY; |
390 | } |
391 | |
392 | prepare_cell(chain, dest, cmd, cmd_size); |
393 | cmd_chain_prod_idx_inc(chain); |
394 | |
395 | wmb(); /* inc pi before issue the command */ |
396 | |
397 | set_prod_idx(chain); /* issue the command */ |
398 | |
399 | ctxt = &chain->cell_ctxt[chain->prod_idx]; |
400 | |
401 | chain->curr_node = ctxt->cell_vaddr; |
402 | |
403 | err = wait_for_api_cmd_completion(chain); |
404 | |
405 | up(sem: &chain->sem); |
406 | return err; |
407 | } |
408 | |
409 | /** |
410 | * hinic_api_cmd_write - Write API CMD command |
411 | * @chain: chain for write command |
412 | * @dest: destination node on the card that will receive the command |
413 | * @cmd: command data |
414 | * @size: the command size |
415 | * |
416 | * Return 0 - Success, negative - Failure |
417 | **/ |
418 | int hinic_api_cmd_write(struct hinic_api_cmd_chain *chain, |
419 | enum hinic_node_id dest, u8 *cmd, u16 size) |
420 | { |
421 | /* Verify the chain type */ |
422 | if (chain->chain_type == HINIC_API_CMD_WRITE_TO_MGMT_CPU) |
423 | return api_cmd(chain, dest, cmd, cmd_size: size); |
424 | |
425 | return -EINVAL; |
426 | } |
427 | |
428 | /** |
429 | * api_cmd_hw_restart - restart the chain in the HW |
430 | * @chain: the API CMD specific chain to restart |
431 | * |
432 | * Return 0 - Success, negative - Failure |
433 | **/ |
434 | static int api_cmd_hw_restart(struct hinic_api_cmd_chain *chain) |
435 | { |
436 | struct hinic_hwif *hwif = chain->hwif; |
437 | int err = -ETIMEDOUT; |
438 | unsigned long end; |
439 | u32 reg_addr, val; |
440 | |
441 | /* Read Modify Write */ |
442 | reg_addr = HINIC_CSR_API_CMD_CHAIN_REQ_ADDR(chain->chain_type); |
443 | val = hinic_hwif_read_reg(hwif, reg: reg_addr); |
444 | |
445 | val = HINIC_API_CMD_CHAIN_REQ_CLEAR(val, RESTART); |
446 | val |= HINIC_API_CMD_CHAIN_REQ_SET(1, RESTART); |
447 | |
448 | hinic_hwif_write_reg(hwif, reg: reg_addr, val); |
449 | |
450 | end = jiffies + msecs_to_jiffies(API_CMD_TIMEOUT); |
451 | do { |
452 | val = hinic_hwif_read_reg(hwif, reg: reg_addr); |
453 | |
454 | if (!HINIC_API_CMD_CHAIN_REQ_GET(val, RESTART)) { |
455 | err = 0; |
456 | break; |
457 | } |
458 | |
459 | msleep(msecs: 20); |
460 | } while (time_before(jiffies, end)); |
461 | |
462 | return err; |
463 | } |
464 | |
465 | /** |
466 | * api_cmd_ctrl_init - set the control register of a chain |
467 | * @chain: the API CMD specific chain to set control register for |
468 | **/ |
469 | static void api_cmd_ctrl_init(struct hinic_api_cmd_chain *chain) |
470 | { |
471 | struct hinic_hwif *hwif = chain->hwif; |
472 | u32 addr, ctrl; |
473 | u16 cell_size; |
474 | |
475 | /* Read Modify Write */ |
476 | addr = HINIC_CSR_API_CMD_CHAIN_CTRL_ADDR(chain->chain_type); |
477 | |
478 | cell_size = API_CMD_CELL_SIZE_VAL(chain->cell_size); |
479 | |
480 | ctrl = hinic_hwif_read_reg(hwif, reg: addr); |
481 | |
482 | ctrl = HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, RESTART_WB_STAT) & |
483 | HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_ERR) & |
484 | HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, AEQE_EN) & |
485 | HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_CHK_EN) & |
486 | HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, CELL_SIZE); |
487 | |
488 | ctrl |= HINIC_API_CMD_CHAIN_CTRL_SET(1, XOR_ERR) | |
489 | HINIC_API_CMD_CHAIN_CTRL_SET(XOR_CHK_ALL, XOR_CHK_EN) | |
490 | HINIC_API_CMD_CHAIN_CTRL_SET(cell_size, CELL_SIZE); |
491 | |
492 | hinic_hwif_write_reg(hwif, reg: addr, val: ctrl); |
493 | } |
494 | |
495 | /** |
496 | * api_cmd_set_status_addr - set the status address of a chain in the HW |
497 | * @chain: the API CMD specific chain to set in HW status address for |
498 | **/ |
499 | static void api_cmd_set_status_addr(struct hinic_api_cmd_chain *chain) |
500 | { |
501 | struct hinic_hwif *hwif = chain->hwif; |
502 | u32 addr, val; |
503 | |
504 | addr = HINIC_CSR_API_CMD_STATUS_HI_ADDR(chain->chain_type); |
505 | val = upper_32_bits(chain->wb_status_paddr); |
506 | hinic_hwif_write_reg(hwif, reg: addr, val); |
507 | |
508 | addr = HINIC_CSR_API_CMD_STATUS_LO_ADDR(chain->chain_type); |
509 | val = lower_32_bits(chain->wb_status_paddr); |
510 | hinic_hwif_write_reg(hwif, reg: addr, val); |
511 | } |
512 | |
513 | /** |
514 | * api_cmd_set_num_cells - set the number cells of a chain in the HW |
515 | * @chain: the API CMD specific chain to set in HW the number of cells for |
516 | **/ |
517 | static void api_cmd_set_num_cells(struct hinic_api_cmd_chain *chain) |
518 | { |
519 | struct hinic_hwif *hwif = chain->hwif; |
520 | u32 addr, val; |
521 | |
522 | addr = HINIC_CSR_API_CMD_CHAIN_NUM_CELLS_ADDR(chain->chain_type); |
523 | val = chain->num_cells; |
524 | hinic_hwif_write_reg(hwif, reg: addr, val); |
525 | } |
526 | |
527 | /** |
528 | * api_cmd_head_init - set the head of a chain in the HW |
529 | * @chain: the API CMD specific chain to set in HW the head for |
530 | **/ |
531 | static void api_cmd_head_init(struct hinic_api_cmd_chain *chain) |
532 | { |
533 | struct hinic_hwif *hwif = chain->hwif; |
534 | u32 addr, val; |
535 | |
536 | addr = HINIC_CSR_API_CMD_CHAIN_HEAD_HI_ADDR(chain->chain_type); |
537 | val = upper_32_bits(chain->head_cell_paddr); |
538 | hinic_hwif_write_reg(hwif, reg: addr, val); |
539 | |
540 | addr = HINIC_CSR_API_CMD_CHAIN_HEAD_LO_ADDR(chain->chain_type); |
541 | val = lower_32_bits(chain->head_cell_paddr); |
542 | hinic_hwif_write_reg(hwif, reg: addr, val); |
543 | } |
544 | |
545 | /** |
546 | * api_cmd_chain_hw_clean - clean the HW |
547 | * @chain: the API CMD specific chain |
548 | **/ |
549 | static void api_cmd_chain_hw_clean(struct hinic_api_cmd_chain *chain) |
550 | { |
551 | struct hinic_hwif *hwif = chain->hwif; |
552 | u32 addr, ctrl; |
553 | |
554 | addr = HINIC_CSR_API_CMD_CHAIN_CTRL_ADDR(chain->chain_type); |
555 | |
556 | ctrl = hinic_hwif_read_reg(hwif, reg: addr); |
557 | ctrl = HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, RESTART_WB_STAT) & |
558 | HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_ERR) & |
559 | HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, AEQE_EN) & |
560 | HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_CHK_EN) & |
561 | HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, CELL_SIZE); |
562 | |
563 | hinic_hwif_write_reg(hwif, reg: addr, val: ctrl); |
564 | } |
565 | |
566 | /** |
567 | * api_cmd_chain_hw_init - initialize the chain in the HW |
568 | * @chain: the API CMD specific chain to initialize in HW |
569 | * |
570 | * Return 0 - Success, negative - Failure |
571 | **/ |
572 | static int api_cmd_chain_hw_init(struct hinic_api_cmd_chain *chain) |
573 | { |
574 | struct hinic_hwif *hwif = chain->hwif; |
575 | struct pci_dev *pdev = hwif->pdev; |
576 | int err; |
577 | |
578 | api_cmd_chain_hw_clean(chain); |
579 | |
580 | api_cmd_set_status_addr(chain); |
581 | |
582 | err = api_cmd_hw_restart(chain); |
583 | if (err) { |
584 | dev_err(&pdev->dev, "Failed to restart API CMD HW\n" ); |
585 | return err; |
586 | } |
587 | |
588 | api_cmd_ctrl_init(chain); |
589 | api_cmd_set_num_cells(chain); |
590 | api_cmd_head_init(chain); |
591 | return 0; |
592 | } |
593 | |
594 | /** |
595 | * free_cmd_buf - free the dma buffer of API CMD command |
596 | * @chain: the API CMD specific chain of the cmd |
597 | * @cell_idx: the cell index of the cmd |
598 | **/ |
599 | static void free_cmd_buf(struct hinic_api_cmd_chain *chain, int cell_idx) |
600 | { |
601 | struct hinic_api_cmd_cell_ctxt *cell_ctxt; |
602 | struct hinic_hwif *hwif = chain->hwif; |
603 | struct pci_dev *pdev = hwif->pdev; |
604 | |
605 | cell_ctxt = &chain->cell_ctxt[cell_idx]; |
606 | |
607 | dma_free_coherent(dev: &pdev->dev, API_CMD_BUF_SIZE, |
608 | cpu_addr: cell_ctxt->api_cmd_vaddr, |
609 | dma_handle: cell_ctxt->api_cmd_paddr); |
610 | } |
611 | |
612 | /** |
613 | * alloc_cmd_buf - allocate a dma buffer for API CMD command |
614 | * @chain: the API CMD specific chain for the cmd |
615 | * @cell: the cell in the HW for the cmd |
616 | * @cell_idx: the index of the cell |
617 | * |
618 | * Return 0 - Success, negative - Failure |
619 | **/ |
620 | static int alloc_cmd_buf(struct hinic_api_cmd_chain *chain, |
621 | struct hinic_api_cmd_cell *cell, int cell_idx) |
622 | { |
623 | struct hinic_api_cmd_cell_ctxt *cell_ctxt; |
624 | struct hinic_hwif *hwif = chain->hwif; |
625 | struct pci_dev *pdev = hwif->pdev; |
626 | dma_addr_t cmd_paddr; |
627 | u8 *cmd_vaddr; |
628 | int err = 0; |
629 | |
630 | cmd_vaddr = dma_alloc_coherent(dev: &pdev->dev, API_CMD_BUF_SIZE, |
631 | dma_handle: &cmd_paddr, GFP_KERNEL); |
632 | if (!cmd_vaddr) |
633 | return -ENOMEM; |
634 | |
635 | cell_ctxt = &chain->cell_ctxt[cell_idx]; |
636 | |
637 | cell_ctxt->api_cmd_vaddr = cmd_vaddr; |
638 | cell_ctxt->api_cmd_paddr = cmd_paddr; |
639 | |
640 | /* set the cmd DMA address in the cell */ |
641 | switch (chain->chain_type) { |
642 | case HINIC_API_CMD_WRITE_TO_MGMT_CPU: |
643 | /* The data in the HW should be in Big Endian Format */ |
644 | cell->write.hw_cmd_paddr = cpu_to_be64(cmd_paddr); |
645 | break; |
646 | |
647 | default: |
648 | dev_err(&pdev->dev, "Unsupported API CMD chain type\n" ); |
649 | free_cmd_buf(chain, cell_idx); |
650 | err = -EINVAL; |
651 | break; |
652 | } |
653 | |
654 | return err; |
655 | } |
656 | |
657 | /** |
658 | * api_cmd_create_cell - create API CMD cell for specific chain |
659 | * @chain: the API CMD specific chain to create its cell |
660 | * @cell_idx: the index of the cell to create |
661 | * @pre_node: previous cell |
662 | * @node_vaddr: the returned virt addr of the cell |
663 | * |
664 | * Return 0 - Success, negative - Failure |
665 | **/ |
666 | static int api_cmd_create_cell(struct hinic_api_cmd_chain *chain, |
667 | int cell_idx, |
668 | struct hinic_api_cmd_cell *pre_node, |
669 | struct hinic_api_cmd_cell **node_vaddr) |
670 | { |
671 | struct hinic_api_cmd_cell_ctxt *cell_ctxt; |
672 | struct hinic_hwif *hwif = chain->hwif; |
673 | struct pci_dev *pdev = hwif->pdev; |
674 | struct hinic_api_cmd_cell *node; |
675 | dma_addr_t node_paddr; |
676 | int err; |
677 | |
678 | node = dma_alloc_coherent(dev: &pdev->dev, size: chain->cell_size, dma_handle: &node_paddr, |
679 | GFP_KERNEL); |
680 | if (!node) |
681 | return -ENOMEM; |
682 | |
683 | node->read.hw_wb_resp_paddr = 0; |
684 | |
685 | cell_ctxt = &chain->cell_ctxt[cell_idx]; |
686 | cell_ctxt->cell_vaddr = node; |
687 | cell_ctxt->cell_paddr = node_paddr; |
688 | |
689 | if (!pre_node) { |
690 | chain->head_cell_paddr = node_paddr; |
691 | chain->head_node = node; |
692 | } else { |
693 | /* The data in the HW should be in Big Endian Format */ |
694 | pre_node->next_cell_paddr = cpu_to_be64(node_paddr); |
695 | } |
696 | |
697 | switch (chain->chain_type) { |
698 | case HINIC_API_CMD_WRITE_TO_MGMT_CPU: |
699 | err = alloc_cmd_buf(chain, cell: node, cell_idx); |
700 | if (err) { |
701 | dev_err(&pdev->dev, "Failed to allocate cmd buffer\n" ); |
702 | goto err_alloc_cmd_buf; |
703 | } |
704 | break; |
705 | |
706 | default: |
707 | dev_err(&pdev->dev, "Unsupported API CMD chain type\n" ); |
708 | err = -EINVAL; |
709 | goto err_alloc_cmd_buf; |
710 | } |
711 | |
712 | *node_vaddr = node; |
713 | return 0; |
714 | |
715 | err_alloc_cmd_buf: |
716 | dma_free_coherent(dev: &pdev->dev, size: chain->cell_size, cpu_addr: node, dma_handle: node_paddr); |
717 | return err; |
718 | } |
719 | |
720 | /** |
721 | * api_cmd_destroy_cell - destroy API CMD cell of specific chain |
722 | * @chain: the API CMD specific chain to destroy its cell |
723 | * @cell_idx: the cell to destroy |
724 | **/ |
725 | static void api_cmd_destroy_cell(struct hinic_api_cmd_chain *chain, |
726 | int cell_idx) |
727 | { |
728 | struct hinic_api_cmd_cell_ctxt *cell_ctxt; |
729 | struct hinic_hwif *hwif = chain->hwif; |
730 | struct pci_dev *pdev = hwif->pdev; |
731 | struct hinic_api_cmd_cell *node; |
732 | dma_addr_t node_paddr; |
733 | size_t node_size; |
734 | |
735 | cell_ctxt = &chain->cell_ctxt[cell_idx]; |
736 | |
737 | node = cell_ctxt->cell_vaddr; |
738 | node_paddr = cell_ctxt->cell_paddr; |
739 | node_size = chain->cell_size; |
740 | |
741 | if (cell_ctxt->api_cmd_vaddr) { |
742 | switch (chain->chain_type) { |
743 | case HINIC_API_CMD_WRITE_TO_MGMT_CPU: |
744 | free_cmd_buf(chain, cell_idx); |
745 | break; |
746 | default: |
747 | dev_err(&pdev->dev, "Unsupported API CMD chain type\n" ); |
748 | break; |
749 | } |
750 | |
751 | dma_free_coherent(dev: &pdev->dev, size: node_size, cpu_addr: node, |
752 | dma_handle: node_paddr); |
753 | } |
754 | } |
755 | |
756 | /** |
757 | * api_cmd_destroy_cells - destroy API CMD cells of specific chain |
758 | * @chain: the API CMD specific chain to destroy its cells |
759 | * @num_cells: number of cells to destroy |
760 | **/ |
761 | static void api_cmd_destroy_cells(struct hinic_api_cmd_chain *chain, |
762 | int num_cells) |
763 | { |
764 | int cell_idx; |
765 | |
766 | for (cell_idx = 0; cell_idx < num_cells; cell_idx++) |
767 | api_cmd_destroy_cell(chain, cell_idx); |
768 | } |
769 | |
770 | /** |
771 | * api_cmd_create_cells - create API CMD cells for specific chain |
772 | * @chain: the API CMD specific chain |
773 | * |
774 | * Return 0 - Success, negative - Failure |
775 | **/ |
776 | static int api_cmd_create_cells(struct hinic_api_cmd_chain *chain) |
777 | { |
778 | struct hinic_api_cmd_cell *node = NULL, *pre_node = NULL; |
779 | struct hinic_hwif *hwif = chain->hwif; |
780 | struct pci_dev *pdev = hwif->pdev; |
781 | int err, cell_idx; |
782 | |
783 | for (cell_idx = 0; cell_idx < chain->num_cells; cell_idx++) { |
784 | err = api_cmd_create_cell(chain, cell_idx, pre_node, node_vaddr: &node); |
785 | if (err) { |
786 | dev_err(&pdev->dev, "Failed to create API CMD cell\n" ); |
787 | goto err_create_cell; |
788 | } |
789 | |
790 | pre_node = node; |
791 | } |
792 | |
793 | /* set the Final node to point on the start */ |
794 | node->next_cell_paddr = cpu_to_be64(chain->head_cell_paddr); |
795 | |
796 | /* set the current node to be the head */ |
797 | chain->curr_node = chain->head_node; |
798 | return 0; |
799 | |
800 | err_create_cell: |
801 | api_cmd_destroy_cells(chain, num_cells: cell_idx); |
802 | return err; |
803 | } |
804 | |
805 | /** |
806 | * api_chain_init - initialize API CMD specific chain |
807 | * @chain: the API CMD specific chain to initialize |
808 | * @attr: attributes to set in the chain |
809 | * |
810 | * Return 0 - Success, negative - Failure |
811 | **/ |
812 | static int api_chain_init(struct hinic_api_cmd_chain *chain, |
813 | struct hinic_api_cmd_chain_attr *attr) |
814 | { |
815 | struct hinic_hwif *hwif = attr->hwif; |
816 | struct pci_dev *pdev = hwif->pdev; |
817 | |
818 | chain->hwif = hwif; |
819 | chain->chain_type = attr->chain_type; |
820 | chain->num_cells = attr->num_cells; |
821 | chain->cell_size = attr->cell_size; |
822 | |
823 | chain->prod_idx = 0; |
824 | chain->cons_idx = 0; |
825 | |
826 | sema_init(sem: &chain->sem, val: 1); |
827 | |
828 | chain->cell_ctxt = devm_kcalloc(dev: &pdev->dev, n: chain->num_cells, |
829 | size: sizeof(*chain->cell_ctxt), GFP_KERNEL); |
830 | if (!chain->cell_ctxt) |
831 | return -ENOMEM; |
832 | |
833 | chain->wb_status = dma_alloc_coherent(dev: &pdev->dev, |
834 | size: sizeof(*chain->wb_status), |
835 | dma_handle: &chain->wb_status_paddr, |
836 | GFP_KERNEL); |
837 | if (!chain->wb_status) { |
838 | dev_err(&pdev->dev, "Failed to allocate DMA wb status\n" ); |
839 | return -ENOMEM; |
840 | } |
841 | |
842 | return 0; |
843 | } |
844 | |
845 | /** |
846 | * api_chain_free - free API CMD specific chain |
847 | * @chain: the API CMD specific chain to free |
848 | **/ |
849 | static void api_chain_free(struct hinic_api_cmd_chain *chain) |
850 | { |
851 | struct hinic_hwif *hwif = chain->hwif; |
852 | struct pci_dev *pdev = hwif->pdev; |
853 | |
854 | dma_free_coherent(dev: &pdev->dev, size: sizeof(*chain->wb_status), |
855 | cpu_addr: chain->wb_status, dma_handle: chain->wb_status_paddr); |
856 | } |
857 | |
858 | /** |
859 | * api_cmd_create_chain - create API CMD specific chain |
860 | * @attr: attributes to set the chain |
861 | * |
862 | * Return the created chain |
863 | **/ |
864 | static struct hinic_api_cmd_chain * |
865 | api_cmd_create_chain(struct hinic_api_cmd_chain_attr *attr) |
866 | { |
867 | struct hinic_hwif *hwif = attr->hwif; |
868 | struct pci_dev *pdev = hwif->pdev; |
869 | struct hinic_api_cmd_chain *chain; |
870 | int err; |
871 | |
872 | if (attr->num_cells & (attr->num_cells - 1)) { |
873 | dev_err(&pdev->dev, "Invalid number of cells, must be power of 2\n" ); |
874 | return ERR_PTR(error: -EINVAL); |
875 | } |
876 | |
877 | chain = devm_kzalloc(dev: &pdev->dev, size: sizeof(*chain), GFP_KERNEL); |
878 | if (!chain) |
879 | return ERR_PTR(error: -ENOMEM); |
880 | |
881 | err = api_chain_init(chain, attr); |
882 | if (err) { |
883 | dev_err(&pdev->dev, "Failed to initialize chain\n" ); |
884 | return ERR_PTR(error: err); |
885 | } |
886 | |
887 | err = api_cmd_create_cells(chain); |
888 | if (err) { |
889 | dev_err(&pdev->dev, "Failed to create cells for API CMD chain\n" ); |
890 | goto err_create_cells; |
891 | } |
892 | |
893 | err = api_cmd_chain_hw_init(chain); |
894 | if (err) { |
895 | dev_err(&pdev->dev, "Failed to initialize chain HW\n" ); |
896 | goto err_chain_hw_init; |
897 | } |
898 | |
899 | return chain; |
900 | |
901 | err_chain_hw_init: |
902 | api_cmd_destroy_cells(chain, num_cells: chain->num_cells); |
903 | |
904 | err_create_cells: |
905 | api_chain_free(chain); |
906 | return ERR_PTR(error: err); |
907 | } |
908 | |
909 | /** |
910 | * api_cmd_destroy_chain - destroy API CMD specific chain |
911 | * @chain: the API CMD specific chain to destroy |
912 | **/ |
913 | static void api_cmd_destroy_chain(struct hinic_api_cmd_chain *chain) |
914 | { |
915 | api_cmd_chain_hw_clean(chain); |
916 | api_cmd_destroy_cells(chain, num_cells: chain->num_cells); |
917 | api_chain_free(chain); |
918 | } |
919 | |
920 | /** |
921 | * hinic_api_cmd_init - Initialize all the API CMD chains |
922 | * @chain: the API CMD chains that are initialized |
923 | * @hwif: the hardware interface of a pci function device |
924 | * |
925 | * Return 0 - Success, negative - Failure |
926 | **/ |
927 | int hinic_api_cmd_init(struct hinic_api_cmd_chain **chain, |
928 | struct hinic_hwif *hwif) |
929 | { |
930 | enum hinic_api_cmd_chain_type type, chain_type; |
931 | struct hinic_api_cmd_chain_attr attr; |
932 | struct pci_dev *pdev = hwif->pdev; |
933 | size_t hw_cell_sz; |
934 | int err; |
935 | |
936 | hw_cell_sz = sizeof(struct hinic_api_cmd_cell); |
937 | |
938 | attr.hwif = hwif; |
939 | attr.num_cells = API_CHAIN_NUM_CELLS; |
940 | attr.cell_size = API_CMD_CELL_SIZE(hw_cell_sz); |
941 | |
942 | chain_type = HINIC_API_CMD_WRITE_TO_MGMT_CPU; |
943 | for ( ; chain_type < HINIC_API_CMD_MAX; chain_type++) { |
944 | attr.chain_type = chain_type; |
945 | |
946 | if (chain_type != HINIC_API_CMD_WRITE_TO_MGMT_CPU) |
947 | continue; |
948 | |
949 | chain[chain_type] = api_cmd_create_chain(attr: &attr); |
950 | if (IS_ERR(ptr: chain[chain_type])) { |
951 | dev_err(&pdev->dev, "Failed to create chain %d\n" , |
952 | chain_type); |
953 | err = PTR_ERR(ptr: chain[chain_type]); |
954 | goto err_create_chain; |
955 | } |
956 | } |
957 | |
958 | return 0; |
959 | |
960 | err_create_chain: |
961 | type = HINIC_API_CMD_WRITE_TO_MGMT_CPU; |
962 | for ( ; type < chain_type; type++) { |
963 | if (type != HINIC_API_CMD_WRITE_TO_MGMT_CPU) |
964 | continue; |
965 | |
966 | api_cmd_destroy_chain(chain: chain[type]); |
967 | } |
968 | |
969 | return err; |
970 | } |
971 | |
972 | /** |
973 | * hinic_api_cmd_free - free the API CMD chains |
974 | * @chain: the API CMD chains that are freed |
975 | **/ |
976 | void hinic_api_cmd_free(struct hinic_api_cmd_chain **chain) |
977 | { |
978 | enum hinic_api_cmd_chain_type chain_type; |
979 | |
980 | chain_type = HINIC_API_CMD_WRITE_TO_MGMT_CPU; |
981 | for ( ; chain_type < HINIC_API_CMD_MAX; chain_type++) { |
982 | if (chain_type != HINIC_API_CMD_WRITE_TO_MGMT_CPU) |
983 | continue; |
984 | |
985 | api_cmd_destroy_chain(chain: chain[chain_type]); |
986 | } |
987 | } |
988 | |