1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/* Copyright (c) 2018 Mellanox Technologies */
3
4#include <linux/mlx5/vport.h>
5#include <linux/list.h>
6#include "lib/devcom.h"
7#include "mlx5_core.h"
8
9static LIST_HEAD(devcom_dev_list);
10static LIST_HEAD(devcom_comp_list);
11/* protect device list */
12static DEFINE_MUTEX(dev_list_lock);
13/* protect component list */
14static DEFINE_MUTEX(comp_list_lock);
15
16#define devcom_for_each_component(iter) \
17 list_for_each_entry(iter, &devcom_comp_list, comp_list)
18
19struct mlx5_devcom_dev {
20 struct list_head list;
21 struct mlx5_core_dev *dev;
22 struct kref ref;
23};
24
25struct mlx5_devcom_comp {
26 struct list_head comp_list;
27 enum mlx5_devcom_component id;
28 u64 key;
29 struct list_head comp_dev_list_head;
30 mlx5_devcom_event_handler_t handler;
31 struct kref ref;
32 bool ready;
33 struct rw_semaphore sem;
34 struct lock_class_key lock_key;
35};
36
37struct mlx5_devcom_comp_dev {
38 struct list_head list;
39 struct mlx5_devcom_comp *comp;
40 struct mlx5_devcom_dev *devc;
41 void __rcu *data;
42};
43
44static bool devcom_dev_exists(struct mlx5_core_dev *dev)
45{
46 struct mlx5_devcom_dev *iter;
47
48 list_for_each_entry(iter, &devcom_dev_list, list)
49 if (iter->dev == dev)
50 return true;
51
52 return false;
53}
54
55static struct mlx5_devcom_dev *
56mlx5_devcom_dev_alloc(struct mlx5_core_dev *dev)
57{
58 struct mlx5_devcom_dev *devc;
59
60 devc = kzalloc(size: sizeof(*devc), GFP_KERNEL);
61 if (!devc)
62 return NULL;
63
64 devc->dev = dev;
65 kref_init(kref: &devc->ref);
66 return devc;
67}
68
69struct mlx5_devcom_dev *
70mlx5_devcom_register_device(struct mlx5_core_dev *dev)
71{
72 struct mlx5_devcom_dev *devc;
73
74 mutex_lock(&dev_list_lock);
75
76 if (devcom_dev_exists(dev)) {
77 devc = ERR_PTR(error: -EEXIST);
78 goto out;
79 }
80
81 devc = mlx5_devcom_dev_alloc(dev);
82 if (!devc) {
83 devc = ERR_PTR(error: -ENOMEM);
84 goto out;
85 }
86
87 list_add_tail(new: &devc->list, head: &devcom_dev_list);
88out:
89 mutex_unlock(lock: &dev_list_lock);
90 return devc;
91}
92
93static void
94mlx5_devcom_dev_release(struct kref *ref)
95{
96 struct mlx5_devcom_dev *devc = container_of(ref, struct mlx5_devcom_dev, ref);
97
98 mutex_lock(&dev_list_lock);
99 list_del(entry: &devc->list);
100 mutex_unlock(lock: &dev_list_lock);
101 kfree(objp: devc);
102}
103
104void mlx5_devcom_unregister_device(struct mlx5_devcom_dev *devc)
105{
106 if (!IS_ERR_OR_NULL(ptr: devc))
107 kref_put(kref: &devc->ref, release: mlx5_devcom_dev_release);
108}
109
110static struct mlx5_devcom_comp *
111mlx5_devcom_comp_alloc(u64 id, u64 key, mlx5_devcom_event_handler_t handler)
112{
113 struct mlx5_devcom_comp *comp;
114
115 comp = kzalloc(size: sizeof(*comp), GFP_KERNEL);
116 if (!comp)
117 return ERR_PTR(error: -ENOMEM);
118
119 comp->id = id;
120 comp->key = key;
121 comp->handler = handler;
122 init_rwsem(&comp->sem);
123 lockdep_register_key(key: &comp->lock_key);
124 lockdep_set_class(&comp->sem, &comp->lock_key);
125 kref_init(kref: &comp->ref);
126 INIT_LIST_HEAD(list: &comp->comp_dev_list_head);
127
128 return comp;
129}
130
131static void
132mlx5_devcom_comp_release(struct kref *ref)
133{
134 struct mlx5_devcom_comp *comp = container_of(ref, struct mlx5_devcom_comp, ref);
135
136 mutex_lock(&comp_list_lock);
137 list_del(entry: &comp->comp_list);
138 mutex_unlock(lock: &comp_list_lock);
139 lockdep_unregister_key(key: &comp->lock_key);
140 kfree(objp: comp);
141}
142
143static struct mlx5_devcom_comp_dev *
144devcom_alloc_comp_dev(struct mlx5_devcom_dev *devc,
145 struct mlx5_devcom_comp *comp,
146 void *data)
147{
148 struct mlx5_devcom_comp_dev *devcom;
149
150 devcom = kzalloc(size: sizeof(*devcom), GFP_KERNEL);
151 if (!devcom)
152 return ERR_PTR(error: -ENOMEM);
153
154 kref_get(kref: &devc->ref);
155 devcom->devc = devc;
156 devcom->comp = comp;
157 rcu_assign_pointer(devcom->data, data);
158
159 down_write(sem: &comp->sem);
160 list_add_tail(new: &devcom->list, head: &comp->comp_dev_list_head);
161 up_write(sem: &comp->sem);
162
163 return devcom;
164}
165
166static void
167devcom_free_comp_dev(struct mlx5_devcom_comp_dev *devcom)
168{
169 struct mlx5_devcom_comp *comp = devcom->comp;
170
171 down_write(sem: &comp->sem);
172 list_del(entry: &devcom->list);
173 up_write(sem: &comp->sem);
174
175 kref_put(kref: &devcom->devc->ref, release: mlx5_devcom_dev_release);
176 kfree(objp: devcom);
177 kref_put(kref: &comp->ref, release: mlx5_devcom_comp_release);
178}
179
180static bool
181devcom_component_equal(struct mlx5_devcom_comp *devcom,
182 enum mlx5_devcom_component id,
183 u64 key)
184{
185 return devcom->id == id && devcom->key == key;
186}
187
188static struct mlx5_devcom_comp *
189devcom_component_get(struct mlx5_devcom_dev *devc,
190 enum mlx5_devcom_component id,
191 u64 key,
192 mlx5_devcom_event_handler_t handler)
193{
194 struct mlx5_devcom_comp *comp;
195
196 devcom_for_each_component(comp) {
197 if (devcom_component_equal(devcom: comp, id, key)) {
198 if (handler == comp->handler) {
199 kref_get(kref: &comp->ref);
200 return comp;
201 }
202
203 mlx5_core_err(devc->dev,
204 "Cannot register existing devcom component with different handler\n");
205 return ERR_PTR(error: -EINVAL);
206 }
207 }
208
209 return NULL;
210}
211
212struct mlx5_devcom_comp_dev *
213mlx5_devcom_register_component(struct mlx5_devcom_dev *devc,
214 enum mlx5_devcom_component id,
215 u64 key,
216 mlx5_devcom_event_handler_t handler,
217 void *data)
218{
219 struct mlx5_devcom_comp_dev *devcom;
220 struct mlx5_devcom_comp *comp;
221
222 if (IS_ERR_OR_NULL(ptr: devc))
223 return ERR_PTR(error: -EINVAL);
224
225 mutex_lock(&comp_list_lock);
226 comp = devcom_component_get(devc, id, key, handler);
227 if (IS_ERR(ptr: comp)) {
228 devcom = ERR_PTR(error: -EINVAL);
229 goto out_unlock;
230 }
231
232 if (!comp) {
233 comp = mlx5_devcom_comp_alloc(id, key, handler);
234 if (IS_ERR(ptr: comp)) {
235 devcom = ERR_CAST(ptr: comp);
236 goto out_unlock;
237 }
238 list_add_tail(new: &comp->comp_list, head: &devcom_comp_list);
239 }
240 mutex_unlock(lock: &comp_list_lock);
241
242 devcom = devcom_alloc_comp_dev(devc, comp, data);
243 if (IS_ERR(ptr: devcom))
244 kref_put(kref: &comp->ref, release: mlx5_devcom_comp_release);
245
246 return devcom;
247
248out_unlock:
249 mutex_unlock(lock: &comp_list_lock);
250 return devcom;
251}
252
253void mlx5_devcom_unregister_component(struct mlx5_devcom_comp_dev *devcom)
254{
255 if (!IS_ERR_OR_NULL(ptr: devcom))
256 devcom_free_comp_dev(devcom);
257}
258
259int mlx5_devcom_comp_get_size(struct mlx5_devcom_comp_dev *devcom)
260{
261 struct mlx5_devcom_comp *comp = devcom->comp;
262
263 return kref_read(kref: &comp->ref);
264}
265
266int mlx5_devcom_send_event(struct mlx5_devcom_comp_dev *devcom,
267 int event, int rollback_event,
268 void *event_data)
269{
270 struct mlx5_devcom_comp_dev *pos;
271 struct mlx5_devcom_comp *comp;
272 int err = 0;
273 void *data;
274
275 if (IS_ERR_OR_NULL(ptr: devcom))
276 return -ENODEV;
277
278 comp = devcom->comp;
279 down_write(sem: &comp->sem);
280 list_for_each_entry(pos, &comp->comp_dev_list_head, list) {
281 data = rcu_dereference_protected(pos->data, lockdep_is_held(&comp->sem));
282
283 if (pos != devcom && data) {
284 err = comp->handler(event, data, event_data);
285 if (err)
286 goto rollback;
287 }
288 }
289
290 up_write(sem: &comp->sem);
291 return 0;
292
293rollback:
294 if (list_entry_is_head(pos, &comp->comp_dev_list_head, list))
295 goto out;
296 pos = list_prev_entry(pos, list);
297 list_for_each_entry_from_reverse(pos, &comp->comp_dev_list_head, list) {
298 data = rcu_dereference_protected(pos->data, lockdep_is_held(&comp->sem));
299
300 if (pos != devcom && data)
301 comp->handler(rollback_event, data, event_data);
302 }
303out:
304 up_write(sem: &comp->sem);
305 return err;
306}
307
308void mlx5_devcom_comp_set_ready(struct mlx5_devcom_comp_dev *devcom, bool ready)
309{
310 WARN_ON(!rwsem_is_locked(&devcom->comp->sem));
311
312 WRITE_ONCE(devcom->comp->ready, ready);
313}
314
315bool mlx5_devcom_comp_is_ready(struct mlx5_devcom_comp_dev *devcom)
316{
317 if (IS_ERR_OR_NULL(ptr: devcom))
318 return false;
319
320 return READ_ONCE(devcom->comp->ready);
321}
322
323bool mlx5_devcom_for_each_peer_begin(struct mlx5_devcom_comp_dev *devcom)
324{
325 struct mlx5_devcom_comp *comp;
326
327 if (IS_ERR_OR_NULL(ptr: devcom))
328 return false;
329
330 comp = devcom->comp;
331 down_read(sem: &comp->sem);
332 if (!READ_ONCE(comp->ready)) {
333 up_read(sem: &comp->sem);
334 return false;
335 }
336
337 return true;
338}
339
340void mlx5_devcom_for_each_peer_end(struct mlx5_devcom_comp_dev *devcom)
341{
342 up_read(sem: &devcom->comp->sem);
343}
344
345void *mlx5_devcom_get_next_peer_data(struct mlx5_devcom_comp_dev *devcom,
346 struct mlx5_devcom_comp_dev **pos)
347{
348 struct mlx5_devcom_comp *comp = devcom->comp;
349 struct mlx5_devcom_comp_dev *tmp;
350 void *data;
351
352 tmp = list_prepare_entry(*pos, &comp->comp_dev_list_head, list);
353
354 list_for_each_entry_continue(tmp, &comp->comp_dev_list_head, list) {
355 if (tmp != devcom) {
356 data = rcu_dereference_protected(tmp->data, lockdep_is_held(&comp->sem));
357 if (data)
358 break;
359 }
360 }
361
362 if (list_entry_is_head(tmp, &comp->comp_dev_list_head, list))
363 return NULL;
364
365 *pos = tmp;
366 return data;
367}
368
369void *mlx5_devcom_get_next_peer_data_rcu(struct mlx5_devcom_comp_dev *devcom,
370 struct mlx5_devcom_comp_dev **pos)
371{
372 struct mlx5_devcom_comp *comp = devcom->comp;
373 struct mlx5_devcom_comp_dev *tmp;
374 void *data;
375
376 tmp = list_prepare_entry(*pos, &comp->comp_dev_list_head, list);
377
378 list_for_each_entry_continue(tmp, &comp->comp_dev_list_head, list) {
379 if (tmp != devcom) {
380 /* This can change concurrently, however 'data' pointer will remain
381 * valid for the duration of RCU read section.
382 */
383 if (!READ_ONCE(comp->ready))
384 return NULL;
385 data = rcu_dereference(tmp->data);
386 if (data)
387 break;
388 }
389 }
390
391 if (list_entry_is_head(tmp, &comp->comp_dev_list_head, list))
392 return NULL;
393
394 *pos = tmp;
395 return data;
396}
397
398void mlx5_devcom_comp_lock(struct mlx5_devcom_comp_dev *devcom)
399{
400 if (IS_ERR_OR_NULL(ptr: devcom))
401 return;
402 down_write(sem: &devcom->comp->sem);
403}
404
405void mlx5_devcom_comp_unlock(struct mlx5_devcom_comp_dev *devcom)
406{
407 if (IS_ERR_OR_NULL(ptr: devcom))
408 return;
409 up_write(sem: &devcom->comp->sem);
410}
411
412int mlx5_devcom_comp_trylock(struct mlx5_devcom_comp_dev *devcom)
413{
414 if (IS_ERR_OR_NULL(ptr: devcom))
415 return 0;
416 return down_write_trylock(sem: &devcom->comp->sem);
417}
418

source code of linux/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c