1 | // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) |
2 | /* |
3 | * Copyright(c) 2018 - 2020 Intel Corporation. |
4 | */ |
5 | |
6 | #include "hfi.h" |
7 | #include "affinity.h" |
8 | #include "sdma.h" |
9 | #include "netdev.h" |
10 | |
11 | /** |
12 | * msix_initialize() - Calculate, request and configure MSIx IRQs |
13 | * @dd: valid hfi1 devdata |
14 | * |
15 | */ |
16 | int msix_initialize(struct hfi1_devdata *dd) |
17 | { |
18 | u32 total; |
19 | int ret; |
20 | struct hfi1_msix_entry *entries; |
21 | |
22 | /* |
23 | * MSIx interrupt count: |
24 | * one for the general, "slow path" interrupt |
25 | * one per used SDMA engine |
26 | * one per kernel receive context |
27 | * one for each VNIC context |
28 | * ...any new IRQs should be added here. |
29 | */ |
30 | total = 1 + dd->num_sdma + dd->n_krcv_queues + dd->num_netdev_contexts; |
31 | |
32 | if (total >= CCE_NUM_MSIX_VECTORS) |
33 | return -EINVAL; |
34 | |
35 | ret = pci_alloc_irq_vectors(dev: dd->pcidev, min_vecs: total, max_vecs: total, PCI_IRQ_MSIX); |
36 | if (ret < 0) { |
37 | dd_dev_err(dd, "pci_alloc_irq_vectors() failed: %d\n" , ret); |
38 | return ret; |
39 | } |
40 | |
41 | entries = kcalloc(n: total, size: sizeof(*dd->msix_info.msix_entries), |
42 | GFP_KERNEL); |
43 | if (!entries) { |
44 | pci_free_irq_vectors(dev: dd->pcidev); |
45 | return -ENOMEM; |
46 | } |
47 | |
48 | dd->msix_info.msix_entries = entries; |
49 | spin_lock_init(&dd->msix_info.msix_lock); |
50 | bitmap_zero(dst: dd->msix_info.in_use_msix, nbits: total); |
51 | dd->msix_info.max_requested = total; |
52 | dd_dev_info(dd, "%u MSI-X interrupts allocated\n" , total); |
53 | |
54 | return 0; |
55 | } |
56 | |
57 | /** |
58 | * msix_request_irq() - Allocate a free MSIx IRQ |
59 | * @dd: valid devdata |
60 | * @arg: context information for the IRQ |
61 | * @handler: IRQ handler |
62 | * @thread: IRQ thread handler (could be NULL) |
63 | * @type: affinty IRQ type |
64 | * @name: IRQ name |
65 | * |
66 | * Allocated an MSIx vector if available, and then create the appropriate |
67 | * meta data needed to keep track of the pci IRQ request. |
68 | * |
69 | * Return: |
70 | * < 0 Error |
71 | * >= 0 MSIx vector |
72 | * |
73 | */ |
74 | static int msix_request_irq(struct hfi1_devdata *dd, void *arg, |
75 | irq_handler_t handler, irq_handler_t thread, |
76 | enum irq_type type, const char *name) |
77 | { |
78 | unsigned long nr; |
79 | int irq; |
80 | int ret; |
81 | struct hfi1_msix_entry *me; |
82 | |
83 | /* Allocate an MSIx vector */ |
84 | spin_lock(lock: &dd->msix_info.msix_lock); |
85 | nr = find_first_zero_bit(addr: dd->msix_info.in_use_msix, |
86 | size: dd->msix_info.max_requested); |
87 | if (nr < dd->msix_info.max_requested) |
88 | __set_bit(nr, dd->msix_info.in_use_msix); |
89 | spin_unlock(lock: &dd->msix_info.msix_lock); |
90 | |
91 | if (nr == dd->msix_info.max_requested) |
92 | return -ENOSPC; |
93 | |
94 | if (type < IRQ_SDMA || type >= IRQ_OTHER) |
95 | return -EINVAL; |
96 | |
97 | irq = pci_irq_vector(dev: dd->pcidev, nr); |
98 | ret = pci_request_irq(dev: dd->pcidev, nr, handler, thread_fn: thread, dev_id: arg, fmt: name); |
99 | if (ret) { |
100 | dd_dev_err(dd, |
101 | "%s: request for IRQ %d failed, MSIx %lx, err %d\n" , |
102 | name, irq, nr, ret); |
103 | spin_lock(lock: &dd->msix_info.msix_lock); |
104 | __clear_bit(nr, dd->msix_info.in_use_msix); |
105 | spin_unlock(lock: &dd->msix_info.msix_lock); |
106 | return ret; |
107 | } |
108 | |
109 | /* |
110 | * assign arg after pci_request_irq call, so it will be |
111 | * cleaned up |
112 | */ |
113 | me = &dd->msix_info.msix_entries[nr]; |
114 | me->irq = irq; |
115 | me->arg = arg; |
116 | me->type = type; |
117 | |
118 | /* This is a request, so a failure is not fatal */ |
119 | ret = hfi1_get_irq_affinity(dd, msix: me); |
120 | if (ret) |
121 | dd_dev_err(dd, "%s: unable to pin IRQ %d\n" , name, ret); |
122 | |
123 | return nr; |
124 | } |
125 | |
126 | static int msix_request_rcd_irq_common(struct hfi1_ctxtdata *rcd, |
127 | irq_handler_t handler, |
128 | irq_handler_t thread, |
129 | const char *name) |
130 | { |
131 | int nr = msix_request_irq(dd: rcd->dd, arg: rcd, handler, thread, |
132 | type: rcd->is_vnic ? IRQ_NETDEVCTXT : IRQ_RCVCTXT, |
133 | name); |
134 | if (nr < 0) |
135 | return nr; |
136 | |
137 | /* |
138 | * Set the interrupt register and mask for this |
139 | * context's interrupt. |
140 | */ |
141 | rcd->ireg = (IS_RCVAVAIL_START + rcd->ctxt) / 64; |
142 | rcd->imask = ((u64)1) << ((IS_RCVAVAIL_START + rcd->ctxt) % 64); |
143 | rcd->msix_intr = nr; |
144 | remap_intr(dd: rcd->dd, IS_RCVAVAIL_START + rcd->ctxt, msix_intr: nr); |
145 | |
146 | return 0; |
147 | } |
148 | |
149 | /** |
150 | * msix_request_rcd_irq() - Helper function for RCVAVAIL IRQs |
151 | * @rcd: valid rcd context |
152 | * |
153 | */ |
154 | int msix_request_rcd_irq(struct hfi1_ctxtdata *rcd) |
155 | { |
156 | char name[MAX_NAME_SIZE]; |
157 | |
158 | snprintf(buf: name, size: sizeof(name), DRIVER_NAME "_%d kctxt%d" , |
159 | rcd->dd->unit, rcd->ctxt); |
160 | |
161 | return msix_request_rcd_irq_common(rcd, handler: receive_context_interrupt, |
162 | thread: receive_context_thread, name); |
163 | } |
164 | |
165 | /** |
166 | * msix_netdev_request_rcd_irq - Helper function for RCVAVAIL IRQs |
167 | * for netdev context |
168 | * @rcd: valid netdev contexti |
169 | */ |
170 | int msix_netdev_request_rcd_irq(struct hfi1_ctxtdata *rcd) |
171 | { |
172 | char name[MAX_NAME_SIZE]; |
173 | |
174 | snprintf(buf: name, size: sizeof(name), DRIVER_NAME "_%d nd kctxt%d" , |
175 | rcd->dd->unit, rcd->ctxt); |
176 | return msix_request_rcd_irq_common(rcd, handler: receive_context_interrupt_napi, |
177 | NULL, name); |
178 | } |
179 | |
180 | /** |
181 | * msix_request_sdma_irq - Helper for getting SDMA IRQ resources |
182 | * @sde: valid sdma engine |
183 | * |
184 | */ |
185 | int msix_request_sdma_irq(struct sdma_engine *sde) |
186 | { |
187 | int nr; |
188 | char name[MAX_NAME_SIZE]; |
189 | |
190 | snprintf(buf: name, size: sizeof(name), DRIVER_NAME "_%d sdma%d" , |
191 | sde->dd->unit, sde->this_idx); |
192 | nr = msix_request_irq(dd: sde->dd, arg: sde, handler: sdma_interrupt, NULL, |
193 | type: IRQ_SDMA, name); |
194 | if (nr < 0) |
195 | return nr; |
196 | sde->msix_intr = nr; |
197 | remap_sdma_interrupts(dd: sde->dd, engine: sde->this_idx, msix_intr: nr); |
198 | |
199 | return 0; |
200 | } |
201 | |
202 | /** |
203 | * msix_request_general_irq - Helper for getting general IRQ |
204 | * resources |
205 | * @dd: valid device data |
206 | */ |
207 | int msix_request_general_irq(struct hfi1_devdata *dd) |
208 | { |
209 | int nr; |
210 | char name[MAX_NAME_SIZE]; |
211 | |
212 | snprintf(buf: name, size: sizeof(name), DRIVER_NAME "_%d" , dd->unit); |
213 | nr = msix_request_irq(dd, arg: dd, handler: general_interrupt, NULL, type: IRQ_GENERAL, |
214 | name); |
215 | if (nr < 0) |
216 | return nr; |
217 | |
218 | /* general interrupt must be MSIx vector 0 */ |
219 | if (nr) { |
220 | msix_free_irq(dd, msix_intr: (u8)nr); |
221 | dd_dev_err(dd, "Invalid index %d for GENERAL IRQ\n" , nr); |
222 | return -EINVAL; |
223 | } |
224 | |
225 | return 0; |
226 | } |
227 | |
228 | /** |
229 | * enable_sdma_srcs - Helper to enable SDMA IRQ srcs |
230 | * @dd: valid devdata structure |
231 | * @i: index of SDMA engine |
232 | */ |
233 | static void enable_sdma_srcs(struct hfi1_devdata *dd, int i) |
234 | { |
235 | set_intr_bits(dd, IS_SDMA_START + i, IS_SDMA_START + i, set: true); |
236 | set_intr_bits(dd, IS_SDMA_PROGRESS_START + i, |
237 | IS_SDMA_PROGRESS_START + i, set: true); |
238 | set_intr_bits(dd, IS_SDMA_IDLE_START + i, IS_SDMA_IDLE_START + i, set: true); |
239 | set_intr_bits(dd, IS_SDMAENG_ERR_START + i, IS_SDMAENG_ERR_START + i, |
240 | set: true); |
241 | } |
242 | |
243 | /** |
244 | * msix_request_irqs() - Allocate all MSIx IRQs |
245 | * @dd: valid devdata structure |
246 | * |
247 | * Helper function to request the used MSIx IRQs. |
248 | * |
249 | */ |
250 | int msix_request_irqs(struct hfi1_devdata *dd) |
251 | { |
252 | int i; |
253 | int ret = msix_request_general_irq(dd); |
254 | |
255 | if (ret) |
256 | return ret; |
257 | |
258 | for (i = 0; i < dd->num_sdma; i++) { |
259 | struct sdma_engine *sde = &dd->per_sdma[i]; |
260 | |
261 | ret = msix_request_sdma_irq(sde); |
262 | if (ret) |
263 | return ret; |
264 | enable_sdma_srcs(dd: sde->dd, i); |
265 | } |
266 | |
267 | for (i = 0; i < dd->n_krcv_queues; i++) { |
268 | struct hfi1_ctxtdata *rcd = hfi1_rcd_get_by_index_safe(dd, ctxt: i); |
269 | |
270 | if (rcd) |
271 | ret = msix_request_rcd_irq(rcd); |
272 | hfi1_rcd_put(rcd); |
273 | if (ret) |
274 | return ret; |
275 | } |
276 | |
277 | return 0; |
278 | } |
279 | |
280 | /** |
281 | * msix_free_irq() - Free the specified MSIx resources and IRQ |
282 | * @dd: valid devdata |
283 | * @msix_intr: MSIx vector to free. |
284 | * |
285 | */ |
286 | void msix_free_irq(struct hfi1_devdata *dd, u8 msix_intr) |
287 | { |
288 | struct hfi1_msix_entry *me; |
289 | |
290 | if (msix_intr >= dd->msix_info.max_requested) |
291 | return; |
292 | |
293 | me = &dd->msix_info.msix_entries[msix_intr]; |
294 | |
295 | if (!me->arg) /* => no irq, no affinity */ |
296 | return; |
297 | |
298 | hfi1_put_irq_affinity(dd, msix: me); |
299 | pci_free_irq(dev: dd->pcidev, nr: msix_intr, dev_id: me->arg); |
300 | |
301 | me->arg = NULL; |
302 | |
303 | spin_lock(lock: &dd->msix_info.msix_lock); |
304 | __clear_bit(msix_intr, dd->msix_info.in_use_msix); |
305 | spin_unlock(lock: &dd->msix_info.msix_lock); |
306 | } |
307 | |
308 | /** |
309 | * msix_clean_up_interrupts - Free all MSIx IRQ resources |
310 | * @dd: valid device data data structure |
311 | * |
312 | * Free the MSIx and associated PCI resources, if they have been allocated. |
313 | */ |
314 | void msix_clean_up_interrupts(struct hfi1_devdata *dd) |
315 | { |
316 | int i; |
317 | struct hfi1_msix_entry *me = dd->msix_info.msix_entries; |
318 | |
319 | /* remove irqs - must happen before disabling/turning off */ |
320 | for (i = 0; i < dd->msix_info.max_requested; i++, me++) |
321 | msix_free_irq(dd, msix_intr: i); |
322 | |
323 | /* clean structures */ |
324 | kfree(objp: dd->msix_info.msix_entries); |
325 | dd->msix_info.msix_entries = NULL; |
326 | dd->msix_info.max_requested = 0; |
327 | |
328 | pci_free_irq_vectors(dev: dd->pcidev); |
329 | } |
330 | |
331 | /** |
332 | * msix_netdev_synchronize_irq - netdev IRQ synchronize |
333 | * @dd: valid devdata |
334 | */ |
335 | void msix_netdev_synchronize_irq(struct hfi1_devdata *dd) |
336 | { |
337 | int i; |
338 | int ctxt_count = hfi1_netdev_ctxt_count(dd); |
339 | |
340 | for (i = 0; i < ctxt_count; i++) { |
341 | struct hfi1_ctxtdata *rcd = hfi1_netdev_get_ctxt(dd, ctxt: i); |
342 | struct hfi1_msix_entry *me; |
343 | |
344 | me = &dd->msix_info.msix_entries[rcd->msix_intr]; |
345 | |
346 | synchronize_irq(irq: me->irq); |
347 | } |
348 | } |
349 | |