1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/*
3 * Copyright (c) 2013-2021, Mellanox Technologies inc. All rights reserved.
4 */
5
6#include <linux/interrupt.h>
7#include <linux/notifier.h>
8#include <linux/mlx5/driver.h>
9#include <linux/mlx5/vport.h>
10#include <linux/mlx5/eq.h>
11#ifdef CONFIG_RFS_ACCEL
12#include <linux/cpu_rmap.h>
13#endif
14#include "mlx5_core.h"
15#include "lib/eq.h"
16#include "fpga/core.h"
17#include "eswitch.h"
18#include "lib/clock.h"
19#include "diag/fw_tracer.h"
20#include "mlx5_irq.h"
21#include "pci_irq.h"
22#include "devlink.h"
23#include "en_accel/ipsec.h"
24
25enum {
26 MLX5_EQE_OWNER_INIT_VAL = 0x1,
27};
28
29enum {
30 MLX5_EQ_STATE_ARMED = 0x9,
31 MLX5_EQ_STATE_FIRED = 0xa,
32 MLX5_EQ_STATE_ALWAYS_ARMED = 0xb,
33};
34
35enum {
36 MLX5_EQ_DOORBEL_OFFSET = 0x40,
37};
38
39/* budget must be smaller than MLX5_NUM_SPARE_EQE to guarantee that we update
40 * the ci before we polled all the entries in the EQ. MLX5_NUM_SPARE_EQE is
41 * used to set the EQ size, budget must be smaller than the EQ size.
42 */
43enum {
44 MLX5_EQ_POLLING_BUDGET = 128,
45};
46
47static_assert(MLX5_EQ_POLLING_BUDGET <= MLX5_NUM_SPARE_EQE);
48
49struct mlx5_eq_table {
50 struct xarray comp_eqs;
51 struct mlx5_eq_async pages_eq;
52 struct mlx5_eq_async cmd_eq;
53 struct mlx5_eq_async async_eq;
54
55 struct atomic_notifier_head nh[MLX5_EVENT_TYPE_MAX];
56
57 /* Since CQ DB is stored in async_eq */
58 struct mlx5_nb cq_err_nb;
59
60 struct mutex lock; /* sync async eqs creations */
61 struct mutex comp_lock; /* sync comp eqs creations */
62 int curr_comp_eqs;
63 int max_comp_eqs;
64 struct mlx5_irq_table *irq_table;
65 struct xarray comp_irqs;
66 struct mlx5_irq *ctrl_irq;
67 struct cpu_rmap *rmap;
68 struct cpumask used_cpus;
69};
70
71#define MLX5_ASYNC_EVENT_MASK ((1ull << MLX5_EVENT_TYPE_PATH_MIG) | \
72 (1ull << MLX5_EVENT_TYPE_COMM_EST) | \
73 (1ull << MLX5_EVENT_TYPE_SQ_DRAINED) | \
74 (1ull << MLX5_EVENT_TYPE_CQ_ERROR) | \
75 (1ull << MLX5_EVENT_TYPE_WQ_CATAS_ERROR) | \
76 (1ull << MLX5_EVENT_TYPE_PATH_MIG_FAILED) | \
77 (1ull << MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR) | \
78 (1ull << MLX5_EVENT_TYPE_WQ_ACCESS_ERROR) | \
79 (1ull << MLX5_EVENT_TYPE_PORT_CHANGE) | \
80 (1ull << MLX5_EVENT_TYPE_SRQ_CATAS_ERROR) | \
81 (1ull << MLX5_EVENT_TYPE_SRQ_LAST_WQE) | \
82 (1ull << MLX5_EVENT_TYPE_SRQ_RQ_LIMIT))
83
84static int mlx5_cmd_destroy_eq(struct mlx5_core_dev *dev, u8 eqn)
85{
86 u32 in[MLX5_ST_SZ_DW(destroy_eq_in)] = {};
87
88 MLX5_SET(destroy_eq_in, in, opcode, MLX5_CMD_OP_DESTROY_EQ);
89 MLX5_SET(destroy_eq_in, in, eq_number, eqn);
90 return mlx5_cmd_exec_in(dev, destroy_eq, in);
91}
92
93/* caller must eventually call mlx5_cq_put on the returned cq */
94static struct mlx5_core_cq *mlx5_eq_cq_get(struct mlx5_eq *eq, u32 cqn)
95{
96 struct mlx5_cq_table *table = &eq->cq_table;
97 struct mlx5_core_cq *cq = NULL;
98
99 rcu_read_lock();
100 cq = radix_tree_lookup(&table->tree, cqn);
101 if (likely(cq))
102 mlx5_cq_hold(cq);
103 rcu_read_unlock();
104
105 return cq;
106}
107
108static int mlx5_eq_comp_int(struct notifier_block *nb,
109 __always_unused unsigned long action,
110 __always_unused void *data)
111{
112 struct mlx5_eq_comp *eq_comp =
113 container_of(nb, struct mlx5_eq_comp, irq_nb);
114 struct mlx5_eq *eq = &eq_comp->core;
115 struct mlx5_eqe *eqe;
116 int num_eqes = 0;
117 u32 cqn = -1;
118
119 eqe = next_eqe_sw(eq);
120 if (!eqe)
121 goto out;
122
123 do {
124 struct mlx5_core_cq *cq;
125
126 /* Make sure we read EQ entry contents after we've
127 * checked the ownership bit.
128 */
129 dma_rmb();
130 /* Assume (eqe->type) is always MLX5_EVENT_TYPE_COMP */
131 cqn = be32_to_cpu(eqe->data.comp.cqn) & 0xffffff;
132
133 cq = mlx5_eq_cq_get(eq, cqn);
134 if (likely(cq)) {
135 ++cq->arm_sn;
136 cq->comp(cq, eqe);
137 mlx5_cq_put(cq);
138 } else {
139 dev_dbg_ratelimited(eq->dev->device,
140 "Completion event for bogus CQ 0x%x\n", cqn);
141 }
142
143 ++eq->cons_index;
144
145 } while ((++num_eqes < MLX5_EQ_POLLING_BUDGET) && (eqe = next_eqe_sw(eq)));
146
147out:
148 eq_update_ci(eq, arm: 1);
149
150 if (cqn != -1)
151 tasklet_schedule(t: &eq_comp->tasklet_ctx.task);
152
153 return 0;
154}
155
156/* Some architectures don't latch interrupts when they are disabled, so using
157 * mlx5_eq_poll_irq_disabled could end up losing interrupts while trying to
158 * avoid losing them. It is not recommended to use it, unless this is the last
159 * resort.
160 */
161u32 mlx5_eq_poll_irq_disabled(struct mlx5_eq_comp *eq)
162{
163 u32 count_eqe;
164
165 disable_irq(irq: eq->core.irqn);
166 count_eqe = eq->core.cons_index;
167 mlx5_eq_comp_int(nb: &eq->irq_nb, action: 0, NULL);
168 count_eqe = eq->core.cons_index - count_eqe;
169 enable_irq(irq: eq->core.irqn);
170
171 return count_eqe;
172}
173
174static void mlx5_eq_async_int_lock(struct mlx5_eq_async *eq, bool recovery,
175 unsigned long *flags)
176 __acquires(&eq->lock)
177{
178 if (!recovery)
179 spin_lock(lock: &eq->lock);
180 else
181 spin_lock_irqsave(&eq->lock, *flags);
182}
183
184static void mlx5_eq_async_int_unlock(struct mlx5_eq_async *eq, bool recovery,
185 unsigned long *flags)
186 __releases(&eq->lock)
187{
188 if (!recovery)
189 spin_unlock(lock: &eq->lock);
190 else
191 spin_unlock_irqrestore(lock: &eq->lock, flags: *flags);
192}
193
194enum async_eq_nb_action {
195 ASYNC_EQ_IRQ_HANDLER = 0,
196 ASYNC_EQ_RECOVER = 1,
197};
198
199static int mlx5_eq_async_int(struct notifier_block *nb,
200 unsigned long action, void *data)
201{
202 struct mlx5_eq_async *eq_async =
203 container_of(nb, struct mlx5_eq_async, irq_nb);
204 struct mlx5_eq *eq = &eq_async->core;
205 struct mlx5_eq_table *eqt;
206 struct mlx5_core_dev *dev;
207 struct mlx5_eqe *eqe;
208 unsigned long flags;
209 int num_eqes = 0;
210 bool recovery;
211
212 dev = eq->dev;
213 eqt = dev->priv.eq_table;
214
215 recovery = action == ASYNC_EQ_RECOVER;
216 mlx5_eq_async_int_lock(eq: eq_async, recovery, flags: &flags);
217
218 eqe = next_eqe_sw(eq);
219 if (!eqe)
220 goto out;
221
222 do {
223 /*
224 * Make sure we read EQ entry contents after we've
225 * checked the ownership bit.
226 */
227 dma_rmb();
228
229 atomic_notifier_call_chain(nh: &eqt->nh[eqe->type], val: eqe->type, v: eqe);
230 atomic_notifier_call_chain(nh: &eqt->nh[MLX5_EVENT_TYPE_NOTIFY_ANY], val: eqe->type, v: eqe);
231
232 ++eq->cons_index;
233
234 } while ((++num_eqes < MLX5_EQ_POLLING_BUDGET) && (eqe = next_eqe_sw(eq)));
235
236out:
237 eq_update_ci(eq, arm: 1);
238 mlx5_eq_async_int_unlock(eq: eq_async, recovery, flags: &flags);
239
240 return unlikely(recovery) ? num_eqes : 0;
241}
242
243void mlx5_cmd_eq_recover(struct mlx5_core_dev *dev)
244{
245 struct mlx5_eq_async *eq = &dev->priv.eq_table->cmd_eq;
246 int eqes;
247
248 eqes = mlx5_eq_async_int(nb: &eq->irq_nb, action: ASYNC_EQ_RECOVER, NULL);
249 if (eqes)
250 mlx5_core_warn(dev, "Recovered %d EQEs on cmd_eq\n", eqes);
251}
252
253static void init_eq_buf(struct mlx5_eq *eq)
254{
255 struct mlx5_eqe *eqe;
256 int i;
257
258 for (i = 0; i < eq_get_size(eq); i++) {
259 eqe = get_eqe(eq, entry: i);
260 eqe->owner = MLX5_EQE_OWNER_INIT_VAL;
261 }
262}
263
264static int
265create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq,
266 struct mlx5_eq_param *param)
267{
268 u8 log_eq_size = order_base_2(param->nent + MLX5_NUM_SPARE_EQE);
269 struct mlx5_cq_table *cq_table = &eq->cq_table;
270 u32 out[MLX5_ST_SZ_DW(create_eq_out)] = {0};
271 u8 log_eq_stride = ilog2(MLX5_EQE_SIZE);
272 struct mlx5_priv *priv = &dev->priv;
273 __be64 *pas;
274 u16 vecidx;
275 void *eqc;
276 int inlen;
277 u32 *in;
278 int err;
279 int i;
280
281 /* Init CQ table */
282 memset(cq_table, 0, sizeof(*cq_table));
283 spin_lock_init(&cq_table->lock);
284 INIT_RADIX_TREE(&cq_table->tree, GFP_ATOMIC);
285
286 eq->cons_index = 0;
287
288 err = mlx5_frag_buf_alloc_node(dev, size: wq_get_byte_sz(log_sz: log_eq_size, log_stride: log_eq_stride),
289 buf: &eq->frag_buf, node: dev->priv.numa_node);
290 if (err)
291 return err;
292
293 mlx5_init_fbc(frags: eq->frag_buf.frags, log_stride: log_eq_stride, log_sz: log_eq_size, fbc: &eq->fbc);
294 init_eq_buf(eq);
295
296 eq->irq = param->irq;
297 vecidx = mlx5_irq_get_index(irq: eq->irq);
298
299 inlen = MLX5_ST_SZ_BYTES(create_eq_in) +
300 MLX5_FLD_SZ_BYTES(create_eq_in, pas[0]) * eq->frag_buf.npages;
301
302 in = kvzalloc(size: inlen, GFP_KERNEL);
303 if (!in) {
304 err = -ENOMEM;
305 goto err_buf;
306 }
307
308 pas = (__be64 *)MLX5_ADDR_OF(create_eq_in, in, pas);
309 mlx5_fill_page_frag_array(frag_buf: &eq->frag_buf, pas);
310
311 MLX5_SET(create_eq_in, in, opcode, MLX5_CMD_OP_CREATE_EQ);
312 if (!param->mask[0] && MLX5_CAP_GEN(dev, log_max_uctx))
313 MLX5_SET(create_eq_in, in, uid, MLX5_SHARED_RESOURCE_UID);
314
315 for (i = 0; i < 4; i++)
316 MLX5_ARRAY_SET64(create_eq_in, in, event_bitmask, i,
317 param->mask[i]);
318
319 eqc = MLX5_ADDR_OF(create_eq_in, in, eq_context_entry);
320 MLX5_SET(eqc, eqc, log_eq_size, eq->fbc.log_sz);
321 MLX5_SET(eqc, eqc, uar_page, priv->uar->index);
322 MLX5_SET(eqc, eqc, intr, vecidx);
323 MLX5_SET(eqc, eqc, log_page_size,
324 eq->frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
325
326 err = mlx5_cmd_exec(dev, in, in_size: inlen, out, out_size: sizeof(out));
327 if (err)
328 goto err_in;
329
330 eq->vecidx = vecidx;
331 eq->eqn = MLX5_GET(create_eq_out, out, eq_number);
332 eq->irqn = pci_irq_vector(dev: dev->pdev, nr: vecidx);
333 eq->dev = dev;
334 eq->doorbell = priv->uar->map + MLX5_EQ_DOORBEL_OFFSET;
335
336 err = mlx5_debug_eq_add(dev, eq);
337 if (err)
338 goto err_eq;
339
340 kvfree(addr: in);
341 return 0;
342
343err_eq:
344 mlx5_cmd_destroy_eq(dev, eqn: eq->eqn);
345
346err_in:
347 kvfree(addr: in);
348
349err_buf:
350 mlx5_frag_buf_free(dev, buf: &eq->frag_buf);
351 return err;
352}
353
354/**
355 * mlx5_eq_enable - Enable EQ for receiving EQEs
356 * @dev : Device which owns the eq
357 * @eq : EQ to enable
358 * @nb : Notifier call block
359 *
360 * Must be called after EQ is created in device.
361 *
362 * @return: 0 if no error
363 */
364int mlx5_eq_enable(struct mlx5_core_dev *dev, struct mlx5_eq *eq,
365 struct notifier_block *nb)
366{
367 int err;
368
369 err = mlx5_irq_attach_nb(irq: eq->irq, nb);
370 if (!err)
371 eq_update_ci(eq, arm: 1);
372
373 return err;
374}
375EXPORT_SYMBOL(mlx5_eq_enable);
376
377/**
378 * mlx5_eq_disable - Disable EQ for receiving EQEs
379 * @dev : Device which owns the eq
380 * @eq : EQ to disable
381 * @nb : Notifier call block
382 *
383 * Must be called before EQ is destroyed.
384 */
385void mlx5_eq_disable(struct mlx5_core_dev *dev, struct mlx5_eq *eq,
386 struct notifier_block *nb)
387{
388 mlx5_irq_detach_nb(irq: eq->irq, nb);
389}
390EXPORT_SYMBOL(mlx5_eq_disable);
391
392static int destroy_unmap_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq)
393{
394 int err;
395
396 mlx5_debug_eq_remove(dev, eq);
397
398 err = mlx5_cmd_destroy_eq(dev, eqn: eq->eqn);
399 if (err)
400 mlx5_core_warn(dev, "failed to destroy a previously created eq: eqn %d\n",
401 eq->eqn);
402
403 mlx5_frag_buf_free(dev, buf: &eq->frag_buf);
404 return err;
405}
406
407int mlx5_eq_add_cq(struct mlx5_eq *eq, struct mlx5_core_cq *cq)
408{
409 struct mlx5_cq_table *table = &eq->cq_table;
410 int err;
411
412 spin_lock(lock: &table->lock);
413 err = radix_tree_insert(&table->tree, index: cq->cqn, cq);
414 spin_unlock(lock: &table->lock);
415
416 return err;
417}
418
419void mlx5_eq_del_cq(struct mlx5_eq *eq, struct mlx5_core_cq *cq)
420{
421 struct mlx5_cq_table *table = &eq->cq_table;
422 struct mlx5_core_cq *tmp;
423
424 spin_lock(lock: &table->lock);
425 tmp = radix_tree_delete(&table->tree, cq->cqn);
426 spin_unlock(lock: &table->lock);
427
428 if (!tmp) {
429 mlx5_core_dbg(eq->dev, "cq 0x%x not found in eq 0x%x tree\n",
430 eq->eqn, cq->cqn);
431 return;
432 }
433
434 if (tmp != cq)
435 mlx5_core_dbg(eq->dev, "corruption on cqn 0x%x in eq 0x%x\n",
436 eq->eqn, cq->cqn);
437}
438
439int mlx5_eq_table_init(struct mlx5_core_dev *dev)
440{
441 struct mlx5_eq_table *eq_table;
442 int i;
443
444 eq_table = kvzalloc_node(size: sizeof(*eq_table), GFP_KERNEL,
445 node: dev->priv.numa_node);
446 if (!eq_table)
447 return -ENOMEM;
448
449 dev->priv.eq_table = eq_table;
450
451 mlx5_eq_debugfs_init(dev);
452
453 mutex_init(&eq_table->lock);
454 for (i = 0; i < MLX5_EVENT_TYPE_MAX; i++)
455 ATOMIC_INIT_NOTIFIER_HEAD(&eq_table->nh[i]);
456
457 eq_table->irq_table = mlx5_irq_table_get(dev);
458 cpumask_clear(dstp: &eq_table->used_cpus);
459 xa_init(xa: &eq_table->comp_eqs);
460 xa_init(xa: &eq_table->comp_irqs);
461 mutex_init(&eq_table->comp_lock);
462 eq_table->curr_comp_eqs = 0;
463 return 0;
464}
465
466void mlx5_eq_table_cleanup(struct mlx5_core_dev *dev)
467{
468 struct mlx5_eq_table *table = dev->priv.eq_table;
469
470 mlx5_eq_debugfs_cleanup(dev);
471 xa_destroy(&table->comp_irqs);
472 xa_destroy(&table->comp_eqs);
473 kvfree(addr: table);
474}
475
476/* Async EQs */
477
478static int create_async_eq(struct mlx5_core_dev *dev,
479 struct mlx5_eq *eq, struct mlx5_eq_param *param)
480{
481 struct mlx5_eq_table *eq_table = dev->priv.eq_table;
482 int err;
483
484 mutex_lock(&eq_table->lock);
485 err = create_map_eq(dev, eq, param);
486 mutex_unlock(lock: &eq_table->lock);
487 return err;
488}
489
490static int destroy_async_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq)
491{
492 struct mlx5_eq_table *eq_table = dev->priv.eq_table;
493 int err;
494
495 mutex_lock(&eq_table->lock);
496 err = destroy_unmap_eq(dev, eq);
497 mutex_unlock(lock: &eq_table->lock);
498 return err;
499}
500
501static int cq_err_event_notifier(struct notifier_block *nb,
502 unsigned long type, void *data)
503{
504 struct mlx5_eq_table *eqt;
505 struct mlx5_core_cq *cq;
506 struct mlx5_eqe *eqe;
507 struct mlx5_eq *eq;
508 u32 cqn;
509
510 /* type == MLX5_EVENT_TYPE_CQ_ERROR */
511
512 eqt = mlx5_nb_cof(nb, struct mlx5_eq_table, cq_err_nb);
513 eq = &eqt->async_eq.core;
514 eqe = data;
515
516 cqn = be32_to_cpu(eqe->data.cq_err.cqn) & 0xffffff;
517 mlx5_core_warn(eq->dev, "CQ error on CQN 0x%x, syndrome 0x%x\n",
518 cqn, eqe->data.cq_err.syndrome);
519
520 cq = mlx5_eq_cq_get(eq, cqn);
521 if (unlikely(!cq)) {
522 mlx5_core_warn(eq->dev, "Async event for bogus CQ 0x%x\n", cqn);
523 return NOTIFY_OK;
524 }
525
526 if (cq->event)
527 cq->event(cq, type);
528
529 mlx5_cq_put(cq);
530
531 return NOTIFY_OK;
532}
533
534static void gather_user_async_events(struct mlx5_core_dev *dev, u64 mask[4])
535{
536 __be64 *user_unaffiliated_events;
537 __be64 *user_affiliated_events;
538 int i;
539
540 user_affiliated_events =
541 MLX5_CAP_DEV_EVENT(dev, user_affiliated_events);
542 user_unaffiliated_events =
543 MLX5_CAP_DEV_EVENT(dev, user_unaffiliated_events);
544
545 for (i = 0; i < 4; i++)
546 mask[i] |= be64_to_cpu(user_affiliated_events[i] |
547 user_unaffiliated_events[i]);
548}
549
550static void gather_async_events_mask(struct mlx5_core_dev *dev, u64 mask[4])
551{
552 u64 async_event_mask = MLX5_ASYNC_EVENT_MASK;
553
554 if (MLX5_VPORT_MANAGER(dev))
555 async_event_mask |= (1ull << MLX5_EVENT_TYPE_NIC_VPORT_CHANGE);
556
557 if (MLX5_CAP_GEN(dev, general_notification_event))
558 async_event_mask |= (1ull << MLX5_EVENT_TYPE_GENERAL_EVENT);
559
560 if (MLX5_CAP_GEN(dev, port_module_event))
561 async_event_mask |= (1ull << MLX5_EVENT_TYPE_PORT_MODULE_EVENT);
562 else
563 mlx5_core_dbg(dev, "port_module_event is not set\n");
564
565 if (MLX5_PPS_CAP(dev))
566 async_event_mask |= (1ull << MLX5_EVENT_TYPE_PPS_EVENT);
567
568 if (MLX5_CAP_GEN(dev, fpga))
569 async_event_mask |= (1ull << MLX5_EVENT_TYPE_FPGA_ERROR) |
570 (1ull << MLX5_EVENT_TYPE_FPGA_QP_ERROR);
571 if (MLX5_CAP_GEN_MAX(dev, dct))
572 async_event_mask |= (1ull << MLX5_EVENT_TYPE_DCT_DRAINED);
573
574 if (MLX5_CAP_GEN(dev, temp_warn_event))
575 async_event_mask |= (1ull << MLX5_EVENT_TYPE_TEMP_WARN_EVENT);
576
577 if (MLX5_CAP_MCAM_REG(dev, tracer_registers))
578 async_event_mask |= (1ull << MLX5_EVENT_TYPE_DEVICE_TRACER);
579
580 if (MLX5_CAP_GEN(dev, max_num_of_monitor_counters))
581 async_event_mask |= (1ull << MLX5_EVENT_TYPE_MONITOR_COUNTER);
582
583 if (mlx5_eswitch_is_funcs_handler(dev))
584 async_event_mask |=
585 (1ull << MLX5_EVENT_TYPE_ESW_FUNCTIONS_CHANGED);
586
587 if (MLX5_CAP_GEN_MAX(dev, vhca_state))
588 async_event_mask |= (1ull << MLX5_EVENT_TYPE_VHCA_STATE_CHANGE);
589
590 if (MLX5_CAP_MACSEC(dev, log_max_macsec_offload))
591 async_event_mask |= (1ull << MLX5_EVENT_TYPE_OBJECT_CHANGE);
592
593 if (mlx5_ipsec_device_caps(mdev: dev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD)
594 async_event_mask |=
595 (1ull << MLX5_EVENT_TYPE_OBJECT_CHANGE);
596
597 mask[0] = async_event_mask;
598
599 if (MLX5_CAP_GEN(dev, event_cap))
600 gather_user_async_events(dev, mask);
601}
602
603static int
604setup_async_eq(struct mlx5_core_dev *dev, struct mlx5_eq_async *eq,
605 struct mlx5_eq_param *param, const char *name)
606{
607 int err;
608
609 eq->irq_nb.notifier_call = mlx5_eq_async_int;
610 spin_lock_init(&eq->lock);
611
612 err = create_async_eq(dev, eq: &eq->core, param);
613 if (err) {
614 mlx5_core_warn(dev, "failed to create %s EQ %d\n", name, err);
615 return err;
616 }
617 err = mlx5_eq_enable(dev, &eq->core, &eq->irq_nb);
618 if (err) {
619 mlx5_core_warn(dev, "failed to enable %s EQ %d\n", name, err);
620 destroy_async_eq(dev, eq: &eq->core);
621 }
622 return err;
623}
624
625static void cleanup_async_eq(struct mlx5_core_dev *dev,
626 struct mlx5_eq_async *eq, const char *name)
627{
628 int err;
629
630 mlx5_eq_disable(dev, &eq->core, &eq->irq_nb);
631 err = destroy_async_eq(dev, eq: &eq->core);
632 if (err)
633 mlx5_core_err(dev, "failed to destroy %s eq, err(%d)\n",
634 name, err);
635}
636
637static u16 async_eq_depth_devlink_param_get(struct mlx5_core_dev *dev)
638{
639 struct devlink *devlink = priv_to_devlink(priv: dev);
640 union devlink_param_value val;
641 int err;
642
643 err = devl_param_driverinit_value_get(devlink,
644 param_id: DEVLINK_PARAM_GENERIC_ID_EVENT_EQ_SIZE,
645 val: &val);
646 if (!err)
647 return val.vu32;
648 mlx5_core_dbg(dev, "Failed to get param. using default. err = %d\n", err);
649 return MLX5_NUM_ASYNC_EQE;
650}
651
652static int create_async_eqs(struct mlx5_core_dev *dev)
653{
654 struct mlx5_eq_table *table = dev->priv.eq_table;
655 struct mlx5_eq_param param = {};
656 int err;
657
658 /* All the async_eqs are using single IRQ, request one IRQ and share its
659 * index among all the async_eqs of this device.
660 */
661 table->ctrl_irq = mlx5_ctrl_irq_request(dev);
662 if (IS_ERR(ptr: table->ctrl_irq))
663 return PTR_ERR(ptr: table->ctrl_irq);
664
665 MLX5_NB_INIT(&table->cq_err_nb, cq_err_event_notifier, CQ_ERROR);
666 mlx5_eq_notifier_register(dev, nb: &table->cq_err_nb);
667
668 param = (struct mlx5_eq_param) {
669 .irq = table->ctrl_irq,
670 .nent = MLX5_NUM_CMD_EQE,
671 .mask[0] = 1ull << MLX5_EVENT_TYPE_CMD,
672 };
673 mlx5_cmd_allowed_opcode(dev, opcode: MLX5_CMD_OP_CREATE_EQ);
674 err = setup_async_eq(dev, eq: &table->cmd_eq, param: &param, name: "cmd");
675 if (err)
676 goto err1;
677
678 mlx5_cmd_use_events(dev);
679 mlx5_cmd_allowed_opcode(dev, opcode: CMD_ALLOWED_OPCODE_ALL);
680
681 param = (struct mlx5_eq_param) {
682 .irq = table->ctrl_irq,
683 .nent = async_eq_depth_devlink_param_get(dev),
684 };
685
686 gather_async_events_mask(dev, mask: param.mask);
687 err = setup_async_eq(dev, eq: &table->async_eq, param: &param, name: "async");
688 if (err)
689 goto err2;
690
691 param = (struct mlx5_eq_param) {
692 .irq = table->ctrl_irq,
693 .nent = /* TODO: sriov max_vf + */ 1,
694 .mask[0] = 1ull << MLX5_EVENT_TYPE_PAGE_REQUEST,
695 };
696
697 err = setup_async_eq(dev, eq: &table->pages_eq, param: &param, name: "pages");
698 if (err)
699 goto err3;
700
701 return 0;
702
703err3:
704 cleanup_async_eq(dev, eq: &table->async_eq, name: "async");
705err2:
706 mlx5_cmd_use_polling(dev);
707 cleanup_async_eq(dev, eq: &table->cmd_eq, name: "cmd");
708err1:
709 mlx5_cmd_allowed_opcode(dev, opcode: CMD_ALLOWED_OPCODE_ALL);
710 mlx5_eq_notifier_unregister(dev, nb: &table->cq_err_nb);
711 mlx5_ctrl_irq_release(ctrl_irq: table->ctrl_irq);
712 return err;
713}
714
715static void destroy_async_eqs(struct mlx5_core_dev *dev)
716{
717 struct mlx5_eq_table *table = dev->priv.eq_table;
718
719 cleanup_async_eq(dev, eq: &table->pages_eq, name: "pages");
720 cleanup_async_eq(dev, eq: &table->async_eq, name: "async");
721 mlx5_cmd_allowed_opcode(dev, opcode: MLX5_CMD_OP_DESTROY_EQ);
722 mlx5_cmd_use_polling(dev);
723 cleanup_async_eq(dev, eq: &table->cmd_eq, name: "cmd");
724 mlx5_cmd_allowed_opcode(dev, opcode: CMD_ALLOWED_OPCODE_ALL);
725 mlx5_eq_notifier_unregister(dev, nb: &table->cq_err_nb);
726 mlx5_ctrl_irq_release(ctrl_irq: table->ctrl_irq);
727}
728
729struct mlx5_eq *mlx5_get_async_eq(struct mlx5_core_dev *dev)
730{
731 return &dev->priv.eq_table->async_eq.core;
732}
733
734void mlx5_eq_synchronize_async_irq(struct mlx5_core_dev *dev)
735{
736 synchronize_irq(irq: dev->priv.eq_table->async_eq.core.irqn);
737}
738
739void mlx5_eq_synchronize_cmd_irq(struct mlx5_core_dev *dev)
740{
741 synchronize_irq(irq: dev->priv.eq_table->cmd_eq.core.irqn);
742}
743
744/* Generic EQ API for mlx5_core consumers
745 * Needed For RDMA ODP EQ for now
746 */
747struct mlx5_eq *
748mlx5_eq_create_generic(struct mlx5_core_dev *dev,
749 struct mlx5_eq_param *param)
750{
751 struct mlx5_eq *eq = kvzalloc_node(size: sizeof(*eq), GFP_KERNEL,
752 node: dev->priv.numa_node);
753 int err;
754
755 if (!eq)
756 return ERR_PTR(error: -ENOMEM);
757
758 param->irq = dev->priv.eq_table->ctrl_irq;
759 err = create_async_eq(dev, eq, param);
760 if (err) {
761 kvfree(addr: eq);
762 eq = ERR_PTR(error: err);
763 }
764
765 return eq;
766}
767EXPORT_SYMBOL(mlx5_eq_create_generic);
768
769int mlx5_eq_destroy_generic(struct mlx5_core_dev *dev, struct mlx5_eq *eq)
770{
771 int err;
772
773 if (IS_ERR(ptr: eq))
774 return -EINVAL;
775
776 err = destroy_async_eq(dev, eq);
777 if (err)
778 goto out;
779
780 kvfree(addr: eq);
781out:
782 return err;
783}
784EXPORT_SYMBOL(mlx5_eq_destroy_generic);
785
786struct mlx5_eqe *mlx5_eq_get_eqe(struct mlx5_eq *eq, u32 cc)
787{
788 u32 ci = eq->cons_index + cc;
789 u32 nent = eq_get_size(eq);
790 struct mlx5_eqe *eqe;
791
792 eqe = get_eqe(eq, entry: ci & (nent - 1));
793 eqe = ((eqe->owner & 1) ^ !!(ci & nent)) ? NULL : eqe;
794 /* Make sure we read EQ entry contents after we've
795 * checked the ownership bit.
796 */
797 if (eqe)
798 dma_rmb();
799
800 return eqe;
801}
802EXPORT_SYMBOL(mlx5_eq_get_eqe);
803
804void mlx5_eq_update_ci(struct mlx5_eq *eq, u32 cc, bool arm)
805{
806 __be32 __iomem *addr = eq->doorbell + (arm ? 0 : 2);
807 u32 val;
808
809 eq->cons_index += cc;
810 val = (eq->cons_index & 0xffffff) | (eq->eqn << 24);
811
812 __raw_writel(val: (__force u32)cpu_to_be32(val), addr);
813 /* We still want ordering, just not swabbing, so add a barrier */
814 wmb();
815}
816EXPORT_SYMBOL(mlx5_eq_update_ci);
817
818static void comp_irq_release_pci(struct mlx5_core_dev *dev, u16 vecidx)
819{
820 struct mlx5_eq_table *table = dev->priv.eq_table;
821 struct mlx5_irq *irq;
822
823 irq = xa_load(&table->comp_irqs, index: vecidx);
824 if (!irq)
825 return;
826
827 xa_erase(&table->comp_irqs, index: vecidx);
828 mlx5_irq_release_vector(irq);
829}
830
831static int mlx5_cpumask_default_spread(int numa_node, int index)
832{
833 const struct cpumask *prev = cpu_none_mask;
834 const struct cpumask *mask;
835 int found_cpu = 0;
836 int i = 0;
837 int cpu;
838
839 rcu_read_lock();
840 for_each_numa_hop_mask(mask, numa_node) {
841 for_each_cpu_andnot(cpu, mask, prev) {
842 if (i++ == index) {
843 found_cpu = cpu;
844 goto spread_done;
845 }
846 }
847 prev = mask;
848 }
849
850spread_done:
851 rcu_read_unlock();
852 return found_cpu;
853}
854
855static struct cpu_rmap *mlx5_eq_table_get_pci_rmap(struct mlx5_core_dev *dev)
856{
857#ifdef CONFIG_RFS_ACCEL
858#ifdef CONFIG_MLX5_SF
859 if (mlx5_core_is_sf(dev))
860 return dev->priv.parent_mdev->priv.eq_table->rmap;
861#endif
862 return dev->priv.eq_table->rmap;
863#else
864 return NULL;
865#endif
866}
867
868static int comp_irq_request_pci(struct mlx5_core_dev *dev, u16 vecidx)
869{
870 struct mlx5_eq_table *table = dev->priv.eq_table;
871 struct cpu_rmap *rmap;
872 struct mlx5_irq *irq;
873 int cpu;
874
875 rmap = mlx5_eq_table_get_pci_rmap(dev);
876 cpu = mlx5_cpumask_default_spread(numa_node: dev->priv.numa_node, index: vecidx);
877 irq = mlx5_irq_request_vector(dev, cpu, vecidx, rmap: &rmap);
878 if (IS_ERR(ptr: irq))
879 return PTR_ERR(ptr: irq);
880
881 return xa_err(entry: xa_store(&table->comp_irqs, index: vecidx, entry: irq, GFP_KERNEL));
882}
883
884static void comp_irq_release_sf(struct mlx5_core_dev *dev, u16 vecidx)
885{
886 struct mlx5_eq_table *table = dev->priv.eq_table;
887 struct mlx5_irq *irq;
888 int cpu;
889
890 irq = xa_load(&table->comp_irqs, index: vecidx);
891 if (!irq)
892 return;
893
894 cpu = cpumask_first(srcp: mlx5_irq_get_affinity_mask(irq));
895 cpumask_clear_cpu(cpu, dstp: &table->used_cpus);
896 xa_erase(&table->comp_irqs, index: vecidx);
897 mlx5_irq_affinity_irq_release(dev, irq);
898}
899
900static int comp_irq_request_sf(struct mlx5_core_dev *dev, u16 vecidx)
901{
902 struct mlx5_eq_table *table = dev->priv.eq_table;
903 struct mlx5_irq_pool *pool = mlx5_irq_pool_get(dev);
904 struct irq_affinity_desc af_desc = {};
905 struct mlx5_irq *irq;
906
907 /* In case SF irq pool does not exist, fallback to the PF irqs*/
908 if (!mlx5_irq_pool_is_sf_pool(pool))
909 return comp_irq_request_pci(dev, vecidx);
910
911 af_desc.is_managed = 1;
912 cpumask_copy(dstp: &af_desc.mask, cpu_online_mask);
913 cpumask_andnot(dstp: &af_desc.mask, src1p: &af_desc.mask, src2p: &table->used_cpus);
914 irq = mlx5_irq_affinity_request(pool, af_desc: &af_desc);
915 if (IS_ERR(ptr: irq))
916 return PTR_ERR(ptr: irq);
917
918 cpumask_or(dstp: &table->used_cpus, src1p: &table->used_cpus, src2p: mlx5_irq_get_affinity_mask(irq));
919 mlx5_core_dbg(pool->dev, "IRQ %u mapped to cpu %*pbl, %u EQs on this irq\n",
920 pci_irq_vector(dev->pdev, mlx5_irq_get_index(irq)),
921 cpumask_pr_args(mlx5_irq_get_affinity_mask(irq)),
922 mlx5_irq_read_locked(irq) / MLX5_EQ_REFS_PER_IRQ);
923
924 return xa_err(entry: xa_store(&table->comp_irqs, index: vecidx, entry: irq, GFP_KERNEL));
925}
926
927static void comp_irq_release(struct mlx5_core_dev *dev, u16 vecidx)
928{
929 mlx5_core_is_sf(dev) ? comp_irq_release_sf(dev, vecidx) :
930 comp_irq_release_pci(dev, vecidx);
931}
932
933static int comp_irq_request(struct mlx5_core_dev *dev, u16 vecidx)
934{
935 return mlx5_core_is_sf(dev) ? comp_irq_request_sf(dev, vecidx) :
936 comp_irq_request_pci(dev, vecidx);
937}
938
939#ifdef CONFIG_RFS_ACCEL
940static int alloc_rmap(struct mlx5_core_dev *mdev)
941{
942 struct mlx5_eq_table *eq_table = mdev->priv.eq_table;
943
944 /* rmap is a mapping between irq number and queue number.
945 * Each irq can be assigned only to a single rmap.
946 * Since SFs share IRQs, rmap mapping cannot function correctly
947 * for irqs that are shared between different core/netdev RX rings.
948 * Hence we don't allow netdev rmap for SFs.
949 */
950 if (mlx5_core_is_sf(dev: mdev))
951 return 0;
952
953 eq_table->rmap = alloc_irq_cpu_rmap(size: eq_table->max_comp_eqs);
954 if (!eq_table->rmap)
955 return -ENOMEM;
956 return 0;
957}
958
959static void free_rmap(struct mlx5_core_dev *mdev)
960{
961 struct mlx5_eq_table *eq_table = mdev->priv.eq_table;
962
963 if (eq_table->rmap) {
964 free_irq_cpu_rmap(rmap: eq_table->rmap);
965 eq_table->rmap = NULL;
966 }
967}
968#else
969static int alloc_rmap(struct mlx5_core_dev *mdev) { return 0; }
970static void free_rmap(struct mlx5_core_dev *mdev) {}
971#endif
972
973static void destroy_comp_eq(struct mlx5_core_dev *dev, struct mlx5_eq_comp *eq, u16 vecidx)
974{
975 struct mlx5_eq_table *table = dev->priv.eq_table;
976
977 xa_erase(&table->comp_eqs, index: vecidx);
978 mlx5_eq_disable(dev, &eq->core, &eq->irq_nb);
979 if (destroy_unmap_eq(dev, eq: &eq->core))
980 mlx5_core_warn(dev, "failed to destroy comp EQ 0x%x\n",
981 eq->core.eqn);
982 tasklet_disable(t: &eq->tasklet_ctx.task);
983 kfree(objp: eq);
984 comp_irq_release(dev, vecidx);
985 table->curr_comp_eqs--;
986}
987
988static u16 comp_eq_depth_devlink_param_get(struct mlx5_core_dev *dev)
989{
990 struct devlink *devlink = priv_to_devlink(priv: dev);
991 union devlink_param_value val;
992 int err;
993
994 err = devl_param_driverinit_value_get(devlink,
995 param_id: DEVLINK_PARAM_GENERIC_ID_IO_EQ_SIZE,
996 val: &val);
997 if (!err)
998 return val.vu32;
999 mlx5_core_dbg(dev, "Failed to get param. using default. err = %d\n", err);
1000 return MLX5_COMP_EQ_SIZE;
1001}
1002
1003/* Must be called with EQ table comp_lock held */
1004static int create_comp_eq(struct mlx5_core_dev *dev, u16 vecidx)
1005{
1006 struct mlx5_eq_table *table = dev->priv.eq_table;
1007 struct mlx5_eq_param param = {};
1008 struct mlx5_eq_comp *eq;
1009 struct mlx5_irq *irq;
1010 int nent;
1011 int err;
1012
1013 lockdep_assert_held(&table->comp_lock);
1014 if (table->curr_comp_eqs == table->max_comp_eqs) {
1015 mlx5_core_err(dev, "maximum number of vectors is allocated, %d\n",
1016 table->max_comp_eqs);
1017 return -ENOMEM;
1018 }
1019
1020 err = comp_irq_request(dev, vecidx);
1021 if (err)
1022 return err;
1023
1024 nent = comp_eq_depth_devlink_param_get(dev);
1025
1026 eq = kzalloc_node(size: sizeof(*eq), GFP_KERNEL, node: dev->priv.numa_node);
1027 if (!eq) {
1028 err = -ENOMEM;
1029 goto clean_irq;
1030 }
1031
1032 INIT_LIST_HEAD(list: &eq->tasklet_ctx.list);
1033 INIT_LIST_HEAD(list: &eq->tasklet_ctx.process_list);
1034 spin_lock_init(&eq->tasklet_ctx.lock);
1035 tasklet_setup(t: &eq->tasklet_ctx.task, callback: mlx5_cq_tasklet_cb);
1036
1037 irq = xa_load(&table->comp_irqs, index: vecidx);
1038 eq->irq_nb.notifier_call = mlx5_eq_comp_int;
1039 param = (struct mlx5_eq_param) {
1040 .irq = irq,
1041 .nent = nent,
1042 };
1043
1044 err = create_map_eq(dev, eq: &eq->core, param: &param);
1045 if (err)
1046 goto clean_eq;
1047 err = mlx5_eq_enable(dev, &eq->core, &eq->irq_nb);
1048 if (err) {
1049 destroy_unmap_eq(dev, eq: &eq->core);
1050 goto clean_eq;
1051 }
1052
1053 mlx5_core_dbg(dev, "allocated completion EQN %d\n", eq->core.eqn);
1054 err = xa_err(entry: xa_store(&table->comp_eqs, index: vecidx, entry: eq, GFP_KERNEL));
1055 if (err)
1056 goto disable_eq;
1057
1058 table->curr_comp_eqs++;
1059 return eq->core.eqn;
1060
1061disable_eq:
1062 mlx5_eq_disable(dev, &eq->core, &eq->irq_nb);
1063clean_eq:
1064 kfree(objp: eq);
1065clean_irq:
1066 comp_irq_release(dev, vecidx);
1067 return err;
1068}
1069
1070int mlx5_comp_eqn_get(struct mlx5_core_dev *dev, u16 vecidx, int *eqn)
1071{
1072 struct mlx5_eq_table *table = dev->priv.eq_table;
1073 struct mlx5_eq_comp *eq;
1074 int ret = 0;
1075
1076 mutex_lock(&table->comp_lock);
1077 eq = xa_load(&table->comp_eqs, index: vecidx);
1078 if (eq) {
1079 *eqn = eq->core.eqn;
1080 goto out;
1081 }
1082
1083 ret = create_comp_eq(dev, vecidx);
1084 if (ret < 0) {
1085 mutex_unlock(lock: &table->comp_lock);
1086 return ret;
1087 }
1088
1089 *eqn = ret;
1090out:
1091 mutex_unlock(lock: &table->comp_lock);
1092 return 0;
1093}
1094EXPORT_SYMBOL(mlx5_comp_eqn_get);
1095
1096int mlx5_comp_irqn_get(struct mlx5_core_dev *dev, int vector, unsigned int *irqn)
1097{
1098 struct mlx5_eq_table *table = dev->priv.eq_table;
1099 struct mlx5_eq_comp *eq;
1100 int eqn;
1101 int err;
1102
1103 /* Allocate the EQ if not allocated yet */
1104 err = mlx5_comp_eqn_get(dev, vector, &eqn);
1105 if (err)
1106 return err;
1107
1108 eq = xa_load(&table->comp_eqs, index: vector);
1109 *irqn = eq->core.irqn;
1110 return 0;
1111}
1112
1113unsigned int mlx5_comp_vectors_max(struct mlx5_core_dev *dev)
1114{
1115 return dev->priv.eq_table->max_comp_eqs;
1116}
1117EXPORT_SYMBOL(mlx5_comp_vectors_max);
1118
1119static struct cpumask *
1120mlx5_comp_irq_get_affinity_mask(struct mlx5_core_dev *dev, int vector)
1121{
1122 struct mlx5_eq_table *table = dev->priv.eq_table;
1123 struct mlx5_eq_comp *eq;
1124
1125 eq = xa_load(&table->comp_eqs, index: vector);
1126 if (eq)
1127 return mlx5_irq_get_affinity_mask(irq: eq->core.irq);
1128
1129 return NULL;
1130}
1131
1132int mlx5_comp_vector_get_cpu(struct mlx5_core_dev *dev, int vector)
1133{
1134 struct cpumask *mask;
1135 int cpu;
1136
1137 mask = mlx5_comp_irq_get_affinity_mask(dev, vector);
1138 if (mask)
1139 cpu = cpumask_first(srcp: mask);
1140 else
1141 cpu = mlx5_cpumask_default_spread(numa_node: dev->priv.numa_node, index: vector);
1142
1143 return cpu;
1144}
1145EXPORT_SYMBOL(mlx5_comp_vector_get_cpu);
1146
1147#ifdef CONFIG_RFS_ACCEL
1148struct cpu_rmap *mlx5_eq_table_get_rmap(struct mlx5_core_dev *dev)
1149{
1150 return dev->priv.eq_table->rmap;
1151}
1152#endif
1153
1154struct mlx5_eq_comp *mlx5_eqn2comp_eq(struct mlx5_core_dev *dev, int eqn)
1155{
1156 struct mlx5_eq_table *table = dev->priv.eq_table;
1157 struct mlx5_eq_comp *eq;
1158 unsigned long index;
1159
1160 xa_for_each(&table->comp_eqs, index, eq)
1161 if (eq->core.eqn == eqn)
1162 return eq;
1163
1164 return ERR_PTR(error: -ENOENT);
1165}
1166
1167/* This function should only be called after mlx5_cmd_force_teardown_hca */
1168void mlx5_core_eq_free_irqs(struct mlx5_core_dev *dev)
1169{
1170 mlx5_irq_table_free_irqs(dev);
1171}
1172
1173#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
1174#define MLX5_MAX_ASYNC_EQS 4
1175#else
1176#define MLX5_MAX_ASYNC_EQS 3
1177#endif
1178
1179static int get_num_eqs(struct mlx5_core_dev *dev)
1180{
1181 struct mlx5_eq_table *eq_table = dev->priv.eq_table;
1182 int max_dev_eqs;
1183 int max_eqs_sf;
1184 int num_eqs;
1185
1186 /* If ethernet is disabled we use just a single completion vector to
1187 * have the other vectors available for other drivers using mlx5_core. For
1188 * example, mlx5_vdpa
1189 */
1190 if (!mlx5_core_is_eth_enabled(dev) && mlx5_eth_supported(dev))
1191 return 1;
1192
1193 max_dev_eqs = MLX5_CAP_GEN(dev, max_num_eqs) ?
1194 MLX5_CAP_GEN(dev, max_num_eqs) :
1195 1 << MLX5_CAP_GEN(dev, log_max_eq);
1196
1197 num_eqs = min_t(int, mlx5_irq_table_get_num_comp(eq_table->irq_table),
1198 max_dev_eqs - MLX5_MAX_ASYNC_EQS);
1199 if (mlx5_core_is_sf(dev)) {
1200 max_eqs_sf = min_t(int, MLX5_COMP_EQS_PER_SF,
1201 mlx5_irq_table_get_sfs_vec(eq_table->irq_table));
1202 num_eqs = min_t(int, num_eqs, max_eqs_sf);
1203 }
1204
1205 return num_eqs;
1206}
1207
1208int mlx5_eq_table_create(struct mlx5_core_dev *dev)
1209{
1210 struct mlx5_eq_table *eq_table = dev->priv.eq_table;
1211 int err;
1212
1213 eq_table->max_comp_eqs = get_num_eqs(dev);
1214 err = create_async_eqs(dev);
1215 if (err) {
1216 mlx5_core_err(dev, "Failed to create async EQs\n");
1217 goto err_async_eqs;
1218 }
1219
1220 err = alloc_rmap(mdev: dev);
1221 if (err) {
1222 mlx5_core_err(dev, "Failed to allocate rmap\n");
1223 goto err_rmap;
1224 }
1225
1226 return 0;
1227
1228err_rmap:
1229 destroy_async_eqs(dev);
1230err_async_eqs:
1231 return err;
1232}
1233
1234void mlx5_eq_table_destroy(struct mlx5_core_dev *dev)
1235{
1236 struct mlx5_eq_table *table = dev->priv.eq_table;
1237 struct mlx5_eq_comp *eq;
1238 unsigned long index;
1239
1240 xa_for_each(&table->comp_eqs, index, eq)
1241 destroy_comp_eq(dev, eq, vecidx: index);
1242
1243 free_rmap(mdev: dev);
1244 destroy_async_eqs(dev);
1245}
1246
1247int mlx5_eq_notifier_register(struct mlx5_core_dev *dev, struct mlx5_nb *nb)
1248{
1249 struct mlx5_eq_table *eqt = dev->priv.eq_table;
1250
1251 return atomic_notifier_chain_register(nh: &eqt->nh[nb->event_type], nb: &nb->nb);
1252}
1253EXPORT_SYMBOL(mlx5_eq_notifier_register);
1254
1255int mlx5_eq_notifier_unregister(struct mlx5_core_dev *dev, struct mlx5_nb *nb)
1256{
1257 struct mlx5_eq_table *eqt = dev->priv.eq_table;
1258
1259 return atomic_notifier_chain_unregister(nh: &eqt->nh[nb->event_type], nb: &nb->nb);
1260}
1261EXPORT_SYMBOL(mlx5_eq_notifier_unregister);
1262

source code of linux/drivers/net/ethernet/mellanox/mlx5/core/eq.c