1 | // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB |
2 | /* Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ |
3 | |
4 | #include "mlx5_core.h" |
5 | #include "mlx5_irq.h" |
6 | #include "pci_irq.h" |
7 | |
8 | static void cpu_put(struct mlx5_irq_pool *pool, int cpu) |
9 | { |
10 | pool->irqs_per_cpu[cpu]--; |
11 | } |
12 | |
13 | static void cpu_get(struct mlx5_irq_pool *pool, int cpu) |
14 | { |
15 | pool->irqs_per_cpu[cpu]++; |
16 | } |
17 | |
18 | /* Gets the least loaded CPU. e.g.: the CPU with least IRQs bound to it */ |
19 | static int cpu_get_least_loaded(struct mlx5_irq_pool *pool, |
20 | const struct cpumask *req_mask) |
21 | { |
22 | int best_cpu = -1; |
23 | int cpu; |
24 | |
25 | for_each_cpu_and(cpu, req_mask, cpu_online_mask) { |
26 | /* CPU has zero IRQs on it. No need to search any more CPUs. */ |
27 | if (!pool->irqs_per_cpu[cpu]) { |
28 | best_cpu = cpu; |
29 | break; |
30 | } |
31 | if (best_cpu < 0) |
32 | best_cpu = cpu; |
33 | if (pool->irqs_per_cpu[cpu] < pool->irqs_per_cpu[best_cpu]) |
34 | best_cpu = cpu; |
35 | } |
36 | if (best_cpu == -1) { |
37 | /* There isn't online CPUs in req_mask */ |
38 | mlx5_core_err(pool->dev, "NO online CPUs in req_mask (%*pbl)\n" , |
39 | cpumask_pr_args(req_mask)); |
40 | best_cpu = cpumask_first(cpu_online_mask); |
41 | } |
42 | pool->irqs_per_cpu[best_cpu]++; |
43 | return best_cpu; |
44 | } |
45 | |
46 | /* Creating an IRQ from irq_pool */ |
47 | static struct mlx5_irq * |
48 | irq_pool_request_irq(struct mlx5_irq_pool *pool, struct irq_affinity_desc *af_desc) |
49 | { |
50 | struct irq_affinity_desc auto_desc = {}; |
51 | u32 irq_index; |
52 | int err; |
53 | |
54 | err = xa_alloc(xa: &pool->irqs, id: &irq_index, NULL, limit: pool->xa_num_irqs, GFP_KERNEL); |
55 | if (err) |
56 | return ERR_PTR(error: err); |
57 | if (pool->irqs_per_cpu) { |
58 | if (cpumask_weight(srcp: &af_desc->mask) > 1) |
59 | /* if req_mask contain more then one CPU, set the least loadad CPU |
60 | * of req_mask |
61 | */ |
62 | cpumask_set_cpu(cpu: cpu_get_least_loaded(pool, req_mask: &af_desc->mask), |
63 | dstp: &auto_desc.mask); |
64 | else |
65 | cpu_get(pool, cpu: cpumask_first(srcp: &af_desc->mask)); |
66 | } |
67 | return mlx5_irq_alloc(pool, i: irq_index, |
68 | af_desc: cpumask_empty(srcp: &auto_desc.mask) ? af_desc : &auto_desc, |
69 | NULL); |
70 | } |
71 | |
72 | /* Looking for the IRQ with the smallest refcount that fits req_mask. |
73 | * If pool is sf_comp_pool, then we are looking for an IRQ with any of the |
74 | * requested CPUs in req_mask. |
75 | * for example: req_mask = 0xf, irq0_mask = 0x10, irq1_mask = 0x1. irq0_mask |
76 | * isn't subset of req_mask, so we will skip it. irq1_mask is subset of req_mask, |
77 | * we don't skip it. |
78 | * If pool is sf_ctrl_pool, then all IRQs have the same mask, so any IRQ will |
79 | * fit. And since mask is subset of itself, we will pass the first if bellow. |
80 | */ |
81 | static struct mlx5_irq * |
82 | irq_pool_find_least_loaded(struct mlx5_irq_pool *pool, const struct cpumask *req_mask) |
83 | { |
84 | int start = pool->xa_num_irqs.min; |
85 | int end = pool->xa_num_irqs.max; |
86 | struct mlx5_irq *irq = NULL; |
87 | struct mlx5_irq *iter; |
88 | int irq_refcount = 0; |
89 | unsigned long index; |
90 | |
91 | lockdep_assert_held(&pool->lock); |
92 | xa_for_each_range(&pool->irqs, index, iter, start, end) { |
93 | struct cpumask *iter_mask = mlx5_irq_get_affinity_mask(irq: iter); |
94 | int iter_refcount = mlx5_irq_read_locked(irq: iter); |
95 | |
96 | if (!cpumask_subset(src1p: iter_mask, src2p: req_mask)) |
97 | /* skip IRQs with a mask which is not subset of req_mask */ |
98 | continue; |
99 | if (iter_refcount < pool->min_threshold) |
100 | /* If we found an IRQ with less than min_thres, return it */ |
101 | return iter; |
102 | if (!irq || iter_refcount < irq_refcount) { |
103 | /* In case we won't find an IRQ with less than min_thres, |
104 | * keep a pointer to the least used IRQ |
105 | */ |
106 | irq_refcount = iter_refcount; |
107 | irq = iter; |
108 | } |
109 | } |
110 | return irq; |
111 | } |
112 | |
113 | /** |
114 | * mlx5_irq_affinity_request - request an IRQ according to the given mask. |
115 | * @pool: IRQ pool to request from. |
116 | * @af_desc: affinity descriptor for this IRQ. |
117 | * |
118 | * This function returns a pointer to IRQ, or ERR_PTR in case of error. |
119 | */ |
120 | struct mlx5_irq * |
121 | mlx5_irq_affinity_request(struct mlx5_irq_pool *pool, struct irq_affinity_desc *af_desc) |
122 | { |
123 | struct mlx5_irq *least_loaded_irq, *new_irq; |
124 | |
125 | mutex_lock(&pool->lock); |
126 | least_loaded_irq = irq_pool_find_least_loaded(pool, req_mask: &af_desc->mask); |
127 | if (least_loaded_irq && |
128 | mlx5_irq_read_locked(irq: least_loaded_irq) < pool->min_threshold) |
129 | goto out; |
130 | /* We didn't find an IRQ with less than min_thres, try to allocate a new IRQ */ |
131 | new_irq = irq_pool_request_irq(pool, af_desc); |
132 | if (IS_ERR(ptr: new_irq)) { |
133 | if (!least_loaded_irq) { |
134 | /* We failed to create an IRQ and we didn't find an IRQ */ |
135 | mlx5_core_err(pool->dev, "Didn't find a matching IRQ. err = %ld\n" , |
136 | PTR_ERR(new_irq)); |
137 | mutex_unlock(lock: &pool->lock); |
138 | return new_irq; |
139 | } |
140 | /* We failed to create a new IRQ for the requested affinity, |
141 | * sharing existing IRQ. |
142 | */ |
143 | goto out; |
144 | } |
145 | least_loaded_irq = new_irq; |
146 | goto unlock; |
147 | out: |
148 | mlx5_irq_get_locked(irq: least_loaded_irq); |
149 | if (mlx5_irq_read_locked(irq: least_loaded_irq) > pool->max_threshold) |
150 | mlx5_core_dbg(pool->dev, "IRQ %u overloaded, pool_name: %s, %u EQs on this irq\n" , |
151 | pci_irq_vector(pool->dev->pdev, |
152 | mlx5_irq_get_index(least_loaded_irq)), pool->name, |
153 | mlx5_irq_read_locked(least_loaded_irq) / MLX5_EQ_REFS_PER_IRQ); |
154 | unlock: |
155 | mutex_unlock(lock: &pool->lock); |
156 | return least_loaded_irq; |
157 | } |
158 | |
159 | void mlx5_irq_affinity_irq_release(struct mlx5_core_dev *dev, struct mlx5_irq *irq) |
160 | { |
161 | struct mlx5_irq_pool *pool = mlx5_irq_pool_get(dev); |
162 | int cpu; |
163 | |
164 | cpu = cpumask_first(srcp: mlx5_irq_get_affinity_mask(irq)); |
165 | synchronize_irq(irq: pci_irq_vector(dev: pool->dev->pdev, |
166 | nr: mlx5_irq_get_index(irq))); |
167 | if (mlx5_irq_put(irq)) |
168 | if (pool->irqs_per_cpu) |
169 | cpu_put(pool, cpu); |
170 | } |
171 | |