1 | /********************************************************************** |
2 | * Author: Cavium, Inc. |
3 | * |
4 | * Contact: support@cavium.com |
5 | * Please include "LiquidIO" in the subject. |
6 | * |
7 | * Copyright (c) 2003-2016 Cavium, Inc. |
8 | * |
9 | * This file is free software; you can redistribute it and/or modify |
10 | * it under the terms of the GNU General Public License, Version 2, as |
11 | * published by the Free Software Foundation. |
12 | * |
13 | * This file is distributed in the hope that it will be useful, but |
14 | * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty |
15 | * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or |
16 | * NONINFRINGEMENT. See the GNU General Public License for more |
17 | * details. |
18 | **********************************************************************/ |
19 | #include <linux/pci.h> |
20 | #include <linux/netdevice.h> |
21 | #include "liquidio_common.h" |
22 | #include "octeon_droq.h" |
23 | #include "octeon_iq.h" |
24 | #include "response_manager.h" |
25 | #include "octeon_device.h" |
26 | #include "octeon_main.h" |
27 | |
28 | static void oct_poll_req_completion(struct work_struct *work); |
29 | |
30 | int octeon_setup_response_list(struct octeon_device *oct) |
31 | { |
32 | int i, ret = 0; |
33 | struct cavium_wq *cwq; |
34 | |
35 | for (i = 0; i < MAX_RESPONSE_LISTS; i++) { |
36 | INIT_LIST_HEAD(list: &oct->response_list[i].head); |
37 | spin_lock_init(&oct->response_list[i].lock); |
38 | atomic_set(v: &oct->response_list[i].pending_req_count, i: 0); |
39 | } |
40 | spin_lock_init(&oct->cmd_resp_wqlock); |
41 | |
42 | oct->dma_comp_wq.wq = alloc_workqueue(fmt: "dma-comp" , flags: WQ_MEM_RECLAIM, max_active: 0); |
43 | if (!oct->dma_comp_wq.wq) { |
44 | dev_err(&oct->pci_dev->dev, "failed to create wq thread\n" ); |
45 | return -ENOMEM; |
46 | } |
47 | |
48 | cwq = &oct->dma_comp_wq; |
49 | INIT_DELAYED_WORK(&cwq->wk.work, oct_poll_req_completion); |
50 | cwq->wk.ctxptr = oct; |
51 | oct->cmd_resp_state = OCT_DRV_ONLINE; |
52 | |
53 | return ret; |
54 | } |
55 | EXPORT_SYMBOL_GPL(octeon_setup_response_list); |
56 | |
57 | void octeon_delete_response_list(struct octeon_device *oct) |
58 | { |
59 | cancel_delayed_work_sync(dwork: &oct->dma_comp_wq.wk.work); |
60 | destroy_workqueue(wq: oct->dma_comp_wq.wq); |
61 | } |
62 | EXPORT_SYMBOL_GPL(octeon_delete_response_list); |
63 | |
64 | int lio_process_ordered_list(struct octeon_device *octeon_dev, |
65 | u32 force_quit) |
66 | { |
67 | struct octeon_response_list *ordered_sc_list; |
68 | struct octeon_soft_command *sc; |
69 | int request_complete = 0; |
70 | int resp_to_process = MAX_ORD_REQS_TO_PROCESS; |
71 | u32 status; |
72 | u64 status64; |
73 | |
74 | octeon_free_sc_done_list(oct: octeon_dev); |
75 | |
76 | ordered_sc_list = &octeon_dev->response_list[OCTEON_ORDERED_SC_LIST]; |
77 | |
78 | do { |
79 | spin_lock_bh(lock: &ordered_sc_list->lock); |
80 | |
81 | if (list_empty(head: &ordered_sc_list->head)) { |
82 | spin_unlock_bh(lock: &ordered_sc_list->lock); |
83 | return 1; |
84 | } |
85 | |
86 | sc = list_first_entry(&ordered_sc_list->head, |
87 | struct octeon_soft_command, node); |
88 | |
89 | status = OCTEON_REQUEST_PENDING; |
90 | |
91 | /* check if octeon has finished DMA'ing a response |
92 | * to where rptr is pointing to |
93 | */ |
94 | status64 = *sc->status_word; |
95 | |
96 | if (status64 != COMPLETION_WORD_INIT) { |
97 | /* This logic ensures that all 64b have been written. |
98 | * 1. check byte 0 for non-FF |
99 | * 2. if non-FF, then swap result from BE to host order |
100 | * 3. check byte 7 (swapped to 0) for non-FF |
101 | * 4. if non-FF, use the low 32-bit status code |
102 | * 5. if either byte 0 or byte 7 is FF, don't use status |
103 | */ |
104 | if ((status64 & 0xff) != 0xff) { |
105 | octeon_swap_8B_data(data: &status64, blocks: 1); |
106 | if (((status64 & 0xff) != 0xff)) { |
107 | /* retrieve 16-bit firmware status */ |
108 | status = (u32)(status64 & 0xffffULL); |
109 | if (status) { |
110 | status = |
111 | FIRMWARE_STATUS_CODE(status); |
112 | } else { |
113 | /* i.e. no error */ |
114 | status = OCTEON_REQUEST_DONE; |
115 | } |
116 | } |
117 | } |
118 | } else if (unlikely(force_quit) || (sc->expiry_time && |
119 | time_after(jiffies, (unsigned long)sc->expiry_time))) { |
120 | struct octeon_instr_irh *irh = |
121 | (struct octeon_instr_irh *)&sc->cmd.cmd3.irh; |
122 | |
123 | dev_err(&octeon_dev->pci_dev->dev, "%s: " , __func__); |
124 | dev_err(&octeon_dev->pci_dev->dev, |
125 | "cmd %x/%x/%llx/%llx failed, " , |
126 | irh->opcode, irh->subcode, |
127 | sc->cmd.cmd3.ossp[0], sc->cmd.cmd3.ossp[1]); |
128 | dev_err(&octeon_dev->pci_dev->dev, |
129 | "timeout (%ld, %ld)\n" , |
130 | (long)jiffies, (long)sc->expiry_time); |
131 | status = OCTEON_REQUEST_TIMEOUT; |
132 | } |
133 | |
134 | if (status != OCTEON_REQUEST_PENDING) { |
135 | sc->sc_status = status; |
136 | |
137 | /* we have received a response or we have timed out */ |
138 | /* remove node from linked list */ |
139 | list_del(entry: &sc->node); |
140 | atomic_dec(v: &octeon_dev->response_list |
141 | [OCTEON_ORDERED_SC_LIST]. |
142 | pending_req_count); |
143 | |
144 | if (!sc->callback) { |
145 | atomic_inc(v: &octeon_dev->response_list |
146 | [OCTEON_DONE_SC_LIST]. |
147 | pending_req_count); |
148 | list_add_tail(new: &sc->node, |
149 | head: &octeon_dev->response_list |
150 | [OCTEON_DONE_SC_LIST].head); |
151 | |
152 | if (unlikely(READ_ONCE(sc->caller_is_done))) { |
153 | /* caller does not wait for response |
154 | * from firmware |
155 | */ |
156 | if (status != OCTEON_REQUEST_DONE) { |
157 | struct octeon_instr_irh *irh; |
158 | |
159 | irh = |
160 | (struct octeon_instr_irh *) |
161 | &sc->cmd.cmd3.irh; |
162 | dev_dbg |
163 | (&octeon_dev->pci_dev->dev, |
164 | "%s: sc failed: opcode=%x, " , |
165 | __func__, irh->opcode); |
166 | dev_dbg |
167 | (&octeon_dev->pci_dev->dev, |
168 | "subcode=%x, ossp[0]=%llx, " , |
169 | irh->subcode, |
170 | sc->cmd.cmd3.ossp[0]); |
171 | dev_dbg |
172 | (&octeon_dev->pci_dev->dev, |
173 | "ossp[1]=%llx, status=%d\n" , |
174 | sc->cmd.cmd3.ossp[1], |
175 | status); |
176 | } |
177 | } else { |
178 | complete(&sc->complete); |
179 | } |
180 | |
181 | spin_unlock_bh(lock: &ordered_sc_list->lock); |
182 | } else { |
183 | /* sc with callback function */ |
184 | if (status == OCTEON_REQUEST_TIMEOUT) { |
185 | atomic_inc(v: &octeon_dev->response_list |
186 | [OCTEON_ZOMBIE_SC_LIST]. |
187 | pending_req_count); |
188 | list_add_tail(new: &sc->node, |
189 | head: &octeon_dev->response_list |
190 | [OCTEON_ZOMBIE_SC_LIST]. |
191 | head); |
192 | } |
193 | |
194 | spin_unlock_bh(lock: &ordered_sc_list->lock); |
195 | |
196 | sc->callback(octeon_dev, status, |
197 | sc->callback_arg); |
198 | /* sc is freed by caller */ |
199 | } |
200 | |
201 | request_complete++; |
202 | |
203 | } else { |
204 | /* no response yet */ |
205 | request_complete = 0; |
206 | spin_unlock_bh |
207 | (lock: &ordered_sc_list->lock); |
208 | } |
209 | |
210 | /* If we hit the Max Ordered requests to process every loop, |
211 | * we quit |
212 | * and let this function be invoked the next time the poll |
213 | * thread runs |
214 | * to process the remaining requests. This function can take up |
215 | * the entire CPU if there is no upper limit to the requests |
216 | * processed. |
217 | */ |
218 | if (request_complete >= resp_to_process) |
219 | break; |
220 | } while (request_complete); |
221 | |
222 | return 0; |
223 | } |
224 | EXPORT_SYMBOL_GPL(lio_process_ordered_list); |
225 | |
226 | static void oct_poll_req_completion(struct work_struct *work) |
227 | { |
228 | struct cavium_wk *wk = (struct cavium_wk *)work; |
229 | struct octeon_device *oct = (struct octeon_device *)wk->ctxptr; |
230 | struct cavium_wq *cwq = &oct->dma_comp_wq; |
231 | |
232 | lio_process_ordered_list(oct, 0); |
233 | |
234 | if (atomic_read(v: &oct->response_list |
235 | [OCTEON_ORDERED_SC_LIST].pending_req_count)) |
236 | queue_delayed_work(wq: cwq->wq, dwork: &cwq->wk.work, delay: msecs_to_jiffies(m: 1)); |
237 | } |
238 | |