1 | // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB |
2 | /* Copyright (c) 2018 Mellanox Technologies */ |
3 | |
4 | #include <linux/mlx5/vport.h> |
5 | #include <linux/list.h> |
6 | #include "lib/devcom.h" |
7 | #include "mlx5_core.h" |
8 | |
9 | static LIST_HEAD(devcom_dev_list); |
10 | static LIST_HEAD(devcom_comp_list); |
11 | /* protect device list */ |
12 | static DEFINE_MUTEX(dev_list_lock); |
13 | /* protect component list */ |
14 | static DEFINE_MUTEX(comp_list_lock); |
15 | |
16 | #define devcom_for_each_component(iter) \ |
17 | list_for_each_entry(iter, &devcom_comp_list, comp_list) |
18 | |
19 | struct mlx5_devcom_dev { |
20 | struct list_head list; |
21 | struct mlx5_core_dev *dev; |
22 | struct kref ref; |
23 | }; |
24 | |
25 | struct mlx5_devcom_comp { |
26 | struct list_head comp_list; |
27 | enum mlx5_devcom_component id; |
28 | u64 key; |
29 | struct list_head comp_dev_list_head; |
30 | mlx5_devcom_event_handler_t handler; |
31 | struct kref ref; |
32 | bool ready; |
33 | struct rw_semaphore sem; |
34 | struct lock_class_key lock_key; |
35 | }; |
36 | |
37 | struct mlx5_devcom_comp_dev { |
38 | struct list_head list; |
39 | struct mlx5_devcom_comp *comp; |
40 | struct mlx5_devcom_dev *devc; |
41 | void __rcu *data; |
42 | }; |
43 | |
44 | static bool devcom_dev_exists(struct mlx5_core_dev *dev) |
45 | { |
46 | struct mlx5_devcom_dev *iter; |
47 | |
48 | list_for_each_entry(iter, &devcom_dev_list, list) |
49 | if (iter->dev == dev) |
50 | return true; |
51 | |
52 | return false; |
53 | } |
54 | |
55 | static struct mlx5_devcom_dev * |
56 | mlx5_devcom_dev_alloc(struct mlx5_core_dev *dev) |
57 | { |
58 | struct mlx5_devcom_dev *devc; |
59 | |
60 | devc = kzalloc(size: sizeof(*devc), GFP_KERNEL); |
61 | if (!devc) |
62 | return NULL; |
63 | |
64 | devc->dev = dev; |
65 | kref_init(kref: &devc->ref); |
66 | return devc; |
67 | } |
68 | |
69 | struct mlx5_devcom_dev * |
70 | mlx5_devcom_register_device(struct mlx5_core_dev *dev) |
71 | { |
72 | struct mlx5_devcom_dev *devc; |
73 | |
74 | mutex_lock(&dev_list_lock); |
75 | |
76 | if (devcom_dev_exists(dev)) { |
77 | devc = ERR_PTR(error: -EEXIST); |
78 | goto out; |
79 | } |
80 | |
81 | devc = mlx5_devcom_dev_alloc(dev); |
82 | if (!devc) { |
83 | devc = ERR_PTR(error: -ENOMEM); |
84 | goto out; |
85 | } |
86 | |
87 | list_add_tail(new: &devc->list, head: &devcom_dev_list); |
88 | out: |
89 | mutex_unlock(lock: &dev_list_lock); |
90 | return devc; |
91 | } |
92 | |
93 | static void |
94 | mlx5_devcom_dev_release(struct kref *ref) |
95 | { |
96 | struct mlx5_devcom_dev *devc = container_of(ref, struct mlx5_devcom_dev, ref); |
97 | |
98 | mutex_lock(&dev_list_lock); |
99 | list_del(entry: &devc->list); |
100 | mutex_unlock(lock: &dev_list_lock); |
101 | kfree(objp: devc); |
102 | } |
103 | |
104 | void mlx5_devcom_unregister_device(struct mlx5_devcom_dev *devc) |
105 | { |
106 | if (!IS_ERR_OR_NULL(ptr: devc)) |
107 | kref_put(kref: &devc->ref, release: mlx5_devcom_dev_release); |
108 | } |
109 | |
110 | static struct mlx5_devcom_comp * |
111 | mlx5_devcom_comp_alloc(u64 id, u64 key, mlx5_devcom_event_handler_t handler) |
112 | { |
113 | struct mlx5_devcom_comp *comp; |
114 | |
115 | comp = kzalloc(size: sizeof(*comp), GFP_KERNEL); |
116 | if (!comp) |
117 | return ERR_PTR(error: -ENOMEM); |
118 | |
119 | comp->id = id; |
120 | comp->key = key; |
121 | comp->handler = handler; |
122 | init_rwsem(&comp->sem); |
123 | lockdep_register_key(key: &comp->lock_key); |
124 | lockdep_set_class(&comp->sem, &comp->lock_key); |
125 | kref_init(kref: &comp->ref); |
126 | INIT_LIST_HEAD(list: &comp->comp_dev_list_head); |
127 | |
128 | return comp; |
129 | } |
130 | |
131 | static void |
132 | mlx5_devcom_comp_release(struct kref *ref) |
133 | { |
134 | struct mlx5_devcom_comp *comp = container_of(ref, struct mlx5_devcom_comp, ref); |
135 | |
136 | mutex_lock(&comp_list_lock); |
137 | list_del(entry: &comp->comp_list); |
138 | mutex_unlock(lock: &comp_list_lock); |
139 | lockdep_unregister_key(key: &comp->lock_key); |
140 | kfree(objp: comp); |
141 | } |
142 | |
143 | static struct mlx5_devcom_comp_dev * |
144 | devcom_alloc_comp_dev(struct mlx5_devcom_dev *devc, |
145 | struct mlx5_devcom_comp *comp, |
146 | void *data) |
147 | { |
148 | struct mlx5_devcom_comp_dev *devcom; |
149 | |
150 | devcom = kzalloc(size: sizeof(*devcom), GFP_KERNEL); |
151 | if (!devcom) |
152 | return ERR_PTR(error: -ENOMEM); |
153 | |
154 | kref_get(kref: &devc->ref); |
155 | devcom->devc = devc; |
156 | devcom->comp = comp; |
157 | rcu_assign_pointer(devcom->data, data); |
158 | |
159 | down_write(sem: &comp->sem); |
160 | list_add_tail(new: &devcom->list, head: &comp->comp_dev_list_head); |
161 | up_write(sem: &comp->sem); |
162 | |
163 | return devcom; |
164 | } |
165 | |
166 | static void |
167 | devcom_free_comp_dev(struct mlx5_devcom_comp_dev *devcom) |
168 | { |
169 | struct mlx5_devcom_comp *comp = devcom->comp; |
170 | |
171 | down_write(sem: &comp->sem); |
172 | list_del(entry: &devcom->list); |
173 | up_write(sem: &comp->sem); |
174 | |
175 | kref_put(kref: &devcom->devc->ref, release: mlx5_devcom_dev_release); |
176 | kfree(objp: devcom); |
177 | kref_put(kref: &comp->ref, release: mlx5_devcom_comp_release); |
178 | } |
179 | |
180 | static bool |
181 | devcom_component_equal(struct mlx5_devcom_comp *devcom, |
182 | enum mlx5_devcom_component id, |
183 | u64 key) |
184 | { |
185 | return devcom->id == id && devcom->key == key; |
186 | } |
187 | |
188 | static struct mlx5_devcom_comp * |
189 | devcom_component_get(struct mlx5_devcom_dev *devc, |
190 | enum mlx5_devcom_component id, |
191 | u64 key, |
192 | mlx5_devcom_event_handler_t handler) |
193 | { |
194 | struct mlx5_devcom_comp *comp; |
195 | |
196 | devcom_for_each_component(comp) { |
197 | if (devcom_component_equal(devcom: comp, id, key)) { |
198 | if (handler == comp->handler) { |
199 | kref_get(kref: &comp->ref); |
200 | return comp; |
201 | } |
202 | |
203 | mlx5_core_err(devc->dev, |
204 | "Cannot register existing devcom component with different handler\n" ); |
205 | return ERR_PTR(error: -EINVAL); |
206 | } |
207 | } |
208 | |
209 | return NULL; |
210 | } |
211 | |
212 | struct mlx5_devcom_comp_dev * |
213 | mlx5_devcom_register_component(struct mlx5_devcom_dev *devc, |
214 | enum mlx5_devcom_component id, |
215 | u64 key, |
216 | mlx5_devcom_event_handler_t handler, |
217 | void *data) |
218 | { |
219 | struct mlx5_devcom_comp_dev *devcom; |
220 | struct mlx5_devcom_comp *comp; |
221 | |
222 | if (IS_ERR_OR_NULL(ptr: devc)) |
223 | return ERR_PTR(error: -EINVAL); |
224 | |
225 | mutex_lock(&comp_list_lock); |
226 | comp = devcom_component_get(devc, id, key, handler); |
227 | if (IS_ERR(ptr: comp)) { |
228 | devcom = ERR_PTR(error: -EINVAL); |
229 | goto out_unlock; |
230 | } |
231 | |
232 | if (!comp) { |
233 | comp = mlx5_devcom_comp_alloc(id, key, handler); |
234 | if (IS_ERR(ptr: comp)) { |
235 | devcom = ERR_CAST(ptr: comp); |
236 | goto out_unlock; |
237 | } |
238 | list_add_tail(new: &comp->comp_list, head: &devcom_comp_list); |
239 | } |
240 | mutex_unlock(lock: &comp_list_lock); |
241 | |
242 | devcom = devcom_alloc_comp_dev(devc, comp, data); |
243 | if (IS_ERR(ptr: devcom)) |
244 | kref_put(kref: &comp->ref, release: mlx5_devcom_comp_release); |
245 | |
246 | return devcom; |
247 | |
248 | out_unlock: |
249 | mutex_unlock(lock: &comp_list_lock); |
250 | return devcom; |
251 | } |
252 | |
253 | void mlx5_devcom_unregister_component(struct mlx5_devcom_comp_dev *devcom) |
254 | { |
255 | if (!IS_ERR_OR_NULL(ptr: devcom)) |
256 | devcom_free_comp_dev(devcom); |
257 | } |
258 | |
259 | int mlx5_devcom_comp_get_size(struct mlx5_devcom_comp_dev *devcom) |
260 | { |
261 | struct mlx5_devcom_comp *comp = devcom->comp; |
262 | |
263 | return kref_read(kref: &comp->ref); |
264 | } |
265 | |
266 | int mlx5_devcom_send_event(struct mlx5_devcom_comp_dev *devcom, |
267 | int event, int rollback_event, |
268 | void *event_data) |
269 | { |
270 | struct mlx5_devcom_comp_dev *pos; |
271 | struct mlx5_devcom_comp *comp; |
272 | int err = 0; |
273 | void *data; |
274 | |
275 | if (IS_ERR_OR_NULL(ptr: devcom)) |
276 | return -ENODEV; |
277 | |
278 | comp = devcom->comp; |
279 | down_write(sem: &comp->sem); |
280 | list_for_each_entry(pos, &comp->comp_dev_list_head, list) { |
281 | data = rcu_dereference_protected(pos->data, lockdep_is_held(&comp->sem)); |
282 | |
283 | if (pos != devcom && data) { |
284 | err = comp->handler(event, data, event_data); |
285 | if (err) |
286 | goto rollback; |
287 | } |
288 | } |
289 | |
290 | up_write(sem: &comp->sem); |
291 | return 0; |
292 | |
293 | rollback: |
294 | if (list_entry_is_head(pos, &comp->comp_dev_list_head, list)) |
295 | goto out; |
296 | pos = list_prev_entry(pos, list); |
297 | list_for_each_entry_from_reverse(pos, &comp->comp_dev_list_head, list) { |
298 | data = rcu_dereference_protected(pos->data, lockdep_is_held(&comp->sem)); |
299 | |
300 | if (pos != devcom && data) |
301 | comp->handler(rollback_event, data, event_data); |
302 | } |
303 | out: |
304 | up_write(sem: &comp->sem); |
305 | return err; |
306 | } |
307 | |
308 | void mlx5_devcom_comp_set_ready(struct mlx5_devcom_comp_dev *devcom, bool ready) |
309 | { |
310 | WARN_ON(!rwsem_is_locked(&devcom->comp->sem)); |
311 | |
312 | WRITE_ONCE(devcom->comp->ready, ready); |
313 | } |
314 | |
315 | bool mlx5_devcom_comp_is_ready(struct mlx5_devcom_comp_dev *devcom) |
316 | { |
317 | if (IS_ERR_OR_NULL(ptr: devcom)) |
318 | return false; |
319 | |
320 | return READ_ONCE(devcom->comp->ready); |
321 | } |
322 | |
323 | bool mlx5_devcom_for_each_peer_begin(struct mlx5_devcom_comp_dev *devcom) |
324 | { |
325 | struct mlx5_devcom_comp *comp; |
326 | |
327 | if (IS_ERR_OR_NULL(ptr: devcom)) |
328 | return false; |
329 | |
330 | comp = devcom->comp; |
331 | down_read(sem: &comp->sem); |
332 | if (!READ_ONCE(comp->ready)) { |
333 | up_read(sem: &comp->sem); |
334 | return false; |
335 | } |
336 | |
337 | return true; |
338 | } |
339 | |
340 | void mlx5_devcom_for_each_peer_end(struct mlx5_devcom_comp_dev *devcom) |
341 | { |
342 | up_read(sem: &devcom->comp->sem); |
343 | } |
344 | |
345 | void *mlx5_devcom_get_next_peer_data(struct mlx5_devcom_comp_dev *devcom, |
346 | struct mlx5_devcom_comp_dev **pos) |
347 | { |
348 | struct mlx5_devcom_comp *comp = devcom->comp; |
349 | struct mlx5_devcom_comp_dev *tmp; |
350 | void *data; |
351 | |
352 | tmp = list_prepare_entry(*pos, &comp->comp_dev_list_head, list); |
353 | |
354 | list_for_each_entry_continue(tmp, &comp->comp_dev_list_head, list) { |
355 | if (tmp != devcom) { |
356 | data = rcu_dereference_protected(tmp->data, lockdep_is_held(&comp->sem)); |
357 | if (data) |
358 | break; |
359 | } |
360 | } |
361 | |
362 | if (list_entry_is_head(tmp, &comp->comp_dev_list_head, list)) |
363 | return NULL; |
364 | |
365 | *pos = tmp; |
366 | return data; |
367 | } |
368 | |
369 | void *mlx5_devcom_get_next_peer_data_rcu(struct mlx5_devcom_comp_dev *devcom, |
370 | struct mlx5_devcom_comp_dev **pos) |
371 | { |
372 | struct mlx5_devcom_comp *comp = devcom->comp; |
373 | struct mlx5_devcom_comp_dev *tmp; |
374 | void *data; |
375 | |
376 | tmp = list_prepare_entry(*pos, &comp->comp_dev_list_head, list); |
377 | |
378 | list_for_each_entry_continue(tmp, &comp->comp_dev_list_head, list) { |
379 | if (tmp != devcom) { |
380 | /* This can change concurrently, however 'data' pointer will remain |
381 | * valid for the duration of RCU read section. |
382 | */ |
383 | if (!READ_ONCE(comp->ready)) |
384 | return NULL; |
385 | data = rcu_dereference(tmp->data); |
386 | if (data) |
387 | break; |
388 | } |
389 | } |
390 | |
391 | if (list_entry_is_head(tmp, &comp->comp_dev_list_head, list)) |
392 | return NULL; |
393 | |
394 | *pos = tmp; |
395 | return data; |
396 | } |
397 | |
398 | void mlx5_devcom_comp_lock(struct mlx5_devcom_comp_dev *devcom) |
399 | { |
400 | if (IS_ERR_OR_NULL(ptr: devcom)) |
401 | return; |
402 | down_write(sem: &devcom->comp->sem); |
403 | } |
404 | |
405 | void mlx5_devcom_comp_unlock(struct mlx5_devcom_comp_dev *devcom) |
406 | { |
407 | if (IS_ERR_OR_NULL(ptr: devcom)) |
408 | return; |
409 | up_write(sem: &devcom->comp->sem); |
410 | } |
411 | |
412 | int mlx5_devcom_comp_trylock(struct mlx5_devcom_comp_dev *devcom) |
413 | { |
414 | if (IS_ERR_OR_NULL(ptr: devcom)) |
415 | return 0; |
416 | return down_write_trylock(sem: &devcom->comp->sem); |
417 | } |
418 | |