1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Interconnect framework core driver |
4 | * |
5 | * Copyright (c) 2017-2019, Linaro Ltd. |
6 | * Author: Georgi Djakov <georgi.djakov@linaro.org> |
7 | */ |
8 | |
9 | #include <linux/debugfs.h> |
10 | #include <linux/device.h> |
11 | #include <linux/idr.h> |
12 | #include <linux/init.h> |
13 | #include <linux/interconnect.h> |
14 | #include <linux/interconnect-provider.h> |
15 | #include <linux/list.h> |
16 | #include <linux/mutex.h> |
17 | #include <linux/slab.h> |
18 | #include <linux/of.h> |
19 | #include <linux/overflow.h> |
20 | |
21 | #include "internal.h" |
22 | |
23 | #define CREATE_TRACE_POINTS |
24 | #include "trace.h" |
25 | |
26 | static DEFINE_IDR(icc_idr); |
27 | static LIST_HEAD(icc_providers); |
28 | static int providers_count; |
29 | static bool synced_state; |
30 | static DEFINE_MUTEX(icc_lock); |
31 | static DEFINE_MUTEX(icc_bw_lock); |
32 | static struct dentry *icc_debugfs_dir; |
33 | |
34 | static void icc_summary_show_one(struct seq_file *s, struct icc_node *n) |
35 | { |
36 | if (!n) |
37 | return; |
38 | |
39 | seq_printf(m: s, fmt: "%-42s %12u %12u\n" , |
40 | n->name, n->avg_bw, n->peak_bw); |
41 | } |
42 | |
43 | static int icc_summary_show(struct seq_file *s, void *data) |
44 | { |
45 | struct icc_provider *provider; |
46 | |
47 | seq_puts(m: s, s: " node tag avg peak\n" ); |
48 | seq_puts(m: s, s: "--------------------------------------------------------------------\n" ); |
49 | |
50 | mutex_lock(&icc_lock); |
51 | |
52 | list_for_each_entry(provider, &icc_providers, provider_list) { |
53 | struct icc_node *n; |
54 | |
55 | list_for_each_entry(n, &provider->nodes, node_list) { |
56 | struct icc_req *r; |
57 | |
58 | icc_summary_show_one(s, n); |
59 | hlist_for_each_entry(r, &n->req_list, req_node) { |
60 | u32 avg_bw = 0, peak_bw = 0; |
61 | |
62 | if (!r->dev) |
63 | continue; |
64 | |
65 | if (r->enabled) { |
66 | avg_bw = r->avg_bw; |
67 | peak_bw = r->peak_bw; |
68 | } |
69 | |
70 | seq_printf(m: s, fmt: " %-27s %12u %12u %12u\n" , |
71 | dev_name(dev: r->dev), r->tag, avg_bw, peak_bw); |
72 | } |
73 | } |
74 | } |
75 | |
76 | mutex_unlock(lock: &icc_lock); |
77 | |
78 | return 0; |
79 | } |
80 | DEFINE_SHOW_ATTRIBUTE(icc_summary); |
81 | |
82 | static void icc_graph_show_link(struct seq_file *s, int level, |
83 | struct icc_node *n, struct icc_node *m) |
84 | { |
85 | seq_printf(m: s, fmt: "%s\"%d:%s\" -> \"%d:%s\"\n" , |
86 | level == 2 ? "\t\t" : "\t" , |
87 | n->id, n->name, m->id, m->name); |
88 | } |
89 | |
90 | static void icc_graph_show_node(struct seq_file *s, struct icc_node *n) |
91 | { |
92 | seq_printf(m: s, fmt: "\t\t\"%d:%s\" [label=\"%d:%s" , |
93 | n->id, n->name, n->id, n->name); |
94 | seq_printf(m: s, fmt: "\n\t\t\t|avg_bw=%ukBps" , n->avg_bw); |
95 | seq_printf(m: s, fmt: "\n\t\t\t|peak_bw=%ukBps" , n->peak_bw); |
96 | seq_puts(m: s, s: "\"]\n" ); |
97 | } |
98 | |
99 | static int icc_graph_show(struct seq_file *s, void *data) |
100 | { |
101 | struct icc_provider *provider; |
102 | struct icc_node *n; |
103 | int cluster_index = 0; |
104 | int i; |
105 | |
106 | seq_puts(m: s, s: "digraph {\n\trankdir = LR\n\tnode [shape = record]\n" ); |
107 | mutex_lock(&icc_lock); |
108 | |
109 | /* draw providers as cluster subgraphs */ |
110 | cluster_index = 0; |
111 | list_for_each_entry(provider, &icc_providers, provider_list) { |
112 | seq_printf(m: s, fmt: "\tsubgraph cluster_%d {\n" , ++cluster_index); |
113 | if (provider->dev) |
114 | seq_printf(m: s, fmt: "\t\tlabel = \"%s\"\n" , |
115 | dev_name(dev: provider->dev)); |
116 | |
117 | /* draw nodes */ |
118 | list_for_each_entry(n, &provider->nodes, node_list) |
119 | icc_graph_show_node(s, n); |
120 | |
121 | /* draw internal links */ |
122 | list_for_each_entry(n, &provider->nodes, node_list) |
123 | for (i = 0; i < n->num_links; ++i) |
124 | if (n->provider == n->links[i]->provider) |
125 | icc_graph_show_link(s, level: 2, n, |
126 | m: n->links[i]); |
127 | |
128 | seq_puts(m: s, s: "\t}\n" ); |
129 | } |
130 | |
131 | /* draw external links */ |
132 | list_for_each_entry(provider, &icc_providers, provider_list) |
133 | list_for_each_entry(n, &provider->nodes, node_list) |
134 | for (i = 0; i < n->num_links; ++i) |
135 | if (n->provider != n->links[i]->provider) |
136 | icc_graph_show_link(s, level: 1, n, |
137 | m: n->links[i]); |
138 | |
139 | mutex_unlock(lock: &icc_lock); |
140 | seq_puts(m: s, s: "}" ); |
141 | |
142 | return 0; |
143 | } |
144 | DEFINE_SHOW_ATTRIBUTE(icc_graph); |
145 | |
146 | static struct icc_node *node_find(const int id) |
147 | { |
148 | return idr_find(&icc_idr, id); |
149 | } |
150 | |
151 | static struct icc_node *node_find_by_name(const char *name) |
152 | { |
153 | struct icc_provider *provider; |
154 | struct icc_node *n; |
155 | |
156 | list_for_each_entry(provider, &icc_providers, provider_list) { |
157 | list_for_each_entry(n, &provider->nodes, node_list) { |
158 | if (!strcmp(n->name, name)) |
159 | return n; |
160 | } |
161 | } |
162 | |
163 | return NULL; |
164 | } |
165 | |
166 | static struct icc_path *path_init(struct device *dev, struct icc_node *dst, |
167 | ssize_t num_nodes) |
168 | { |
169 | struct icc_node *node = dst; |
170 | struct icc_path *path; |
171 | int i; |
172 | |
173 | path = kzalloc(struct_size(path, reqs, num_nodes), GFP_KERNEL); |
174 | if (!path) |
175 | return ERR_PTR(error: -ENOMEM); |
176 | |
177 | path->num_nodes = num_nodes; |
178 | |
179 | for (i = num_nodes - 1; i >= 0; i--) { |
180 | node->provider->users++; |
181 | hlist_add_head(n: &path->reqs[i].req_node, h: &node->req_list); |
182 | path->reqs[i].node = node; |
183 | path->reqs[i].dev = dev; |
184 | path->reqs[i].enabled = true; |
185 | /* reference to previous node was saved during path traversal */ |
186 | node = node->reverse; |
187 | } |
188 | |
189 | return path; |
190 | } |
191 | |
192 | static struct icc_path *path_find(struct device *dev, struct icc_node *src, |
193 | struct icc_node *dst) |
194 | { |
195 | struct icc_path *path = ERR_PTR(error: -EPROBE_DEFER); |
196 | struct icc_node *n, *node = NULL; |
197 | struct list_head traverse_list; |
198 | struct list_head edge_list; |
199 | struct list_head visited_list; |
200 | size_t i, depth = 1; |
201 | bool found = false; |
202 | |
203 | INIT_LIST_HEAD(list: &traverse_list); |
204 | INIT_LIST_HEAD(list: &edge_list); |
205 | INIT_LIST_HEAD(list: &visited_list); |
206 | |
207 | list_add(new: &src->search_list, head: &traverse_list); |
208 | src->reverse = NULL; |
209 | |
210 | do { |
211 | list_for_each_entry_safe(node, n, &traverse_list, search_list) { |
212 | if (node == dst) { |
213 | found = true; |
214 | list_splice_init(list: &edge_list, head: &visited_list); |
215 | list_splice_init(list: &traverse_list, head: &visited_list); |
216 | break; |
217 | } |
218 | for (i = 0; i < node->num_links; i++) { |
219 | struct icc_node *tmp = node->links[i]; |
220 | |
221 | if (!tmp) { |
222 | path = ERR_PTR(error: -ENOENT); |
223 | goto out; |
224 | } |
225 | |
226 | if (tmp->is_traversed) |
227 | continue; |
228 | |
229 | tmp->is_traversed = true; |
230 | tmp->reverse = node; |
231 | list_add_tail(new: &tmp->search_list, head: &edge_list); |
232 | } |
233 | } |
234 | |
235 | if (found) |
236 | break; |
237 | |
238 | list_splice_init(list: &traverse_list, head: &visited_list); |
239 | list_splice_init(list: &edge_list, head: &traverse_list); |
240 | |
241 | /* count the hops including the source */ |
242 | depth++; |
243 | |
244 | } while (!list_empty(head: &traverse_list)); |
245 | |
246 | out: |
247 | |
248 | /* reset the traversed state */ |
249 | list_for_each_entry_reverse(n, &visited_list, search_list) |
250 | n->is_traversed = false; |
251 | |
252 | if (found) |
253 | path = path_init(dev, dst, num_nodes: depth); |
254 | |
255 | return path; |
256 | } |
257 | |
258 | /* |
259 | * We want the path to honor all bandwidth requests, so the average and peak |
260 | * bandwidth requirements from each consumer are aggregated at each node. |
261 | * The aggregation is platform specific, so each platform can customize it by |
262 | * implementing its own aggregate() function. |
263 | */ |
264 | |
265 | static int aggregate_requests(struct icc_node *node) |
266 | { |
267 | struct icc_provider *p = node->provider; |
268 | struct icc_req *r; |
269 | u32 avg_bw, peak_bw; |
270 | |
271 | node->avg_bw = 0; |
272 | node->peak_bw = 0; |
273 | |
274 | if (p->pre_aggregate) |
275 | p->pre_aggregate(node); |
276 | |
277 | hlist_for_each_entry(r, &node->req_list, req_node) { |
278 | if (r->enabled) { |
279 | avg_bw = r->avg_bw; |
280 | peak_bw = r->peak_bw; |
281 | } else { |
282 | avg_bw = 0; |
283 | peak_bw = 0; |
284 | } |
285 | p->aggregate(node, r->tag, avg_bw, peak_bw, |
286 | &node->avg_bw, &node->peak_bw); |
287 | |
288 | /* during boot use the initial bandwidth as a floor value */ |
289 | if (!synced_state) { |
290 | node->avg_bw = max(node->avg_bw, node->init_avg); |
291 | node->peak_bw = max(node->peak_bw, node->init_peak); |
292 | } |
293 | } |
294 | |
295 | return 0; |
296 | } |
297 | |
298 | static int apply_constraints(struct icc_path *path) |
299 | { |
300 | struct icc_node *next, *prev = NULL; |
301 | struct icc_provider *p; |
302 | int ret = -EINVAL; |
303 | int i; |
304 | |
305 | for (i = 0; i < path->num_nodes; i++) { |
306 | next = path->reqs[i].node; |
307 | p = next->provider; |
308 | |
309 | /* both endpoints should be valid master-slave pairs */ |
310 | if (!prev || (p != prev->provider && !p->inter_set)) { |
311 | prev = next; |
312 | continue; |
313 | } |
314 | |
315 | /* set the constraints */ |
316 | ret = p->set(prev, next); |
317 | if (ret) |
318 | goto out; |
319 | |
320 | prev = next; |
321 | } |
322 | out: |
323 | return ret; |
324 | } |
325 | |
326 | int icc_std_aggregate(struct icc_node *node, u32 tag, u32 avg_bw, |
327 | u32 peak_bw, u32 *agg_avg, u32 *agg_peak) |
328 | { |
329 | *agg_avg += avg_bw; |
330 | *agg_peak = max(*agg_peak, peak_bw); |
331 | |
332 | return 0; |
333 | } |
334 | EXPORT_SYMBOL_GPL(icc_std_aggregate); |
335 | |
336 | /* of_icc_xlate_onecell() - Translate function using a single index. |
337 | * @spec: OF phandle args to map into an interconnect node. |
338 | * @data: private data (pointer to struct icc_onecell_data) |
339 | * |
340 | * This is a generic translate function that can be used to model simple |
341 | * interconnect providers that have one device tree node and provide |
342 | * multiple interconnect nodes. A single cell is used as an index into |
343 | * an array of icc nodes specified in the icc_onecell_data struct when |
344 | * registering the provider. |
345 | */ |
346 | struct icc_node *of_icc_xlate_onecell(struct of_phandle_args *spec, |
347 | void *data) |
348 | { |
349 | struct icc_onecell_data *icc_data = data; |
350 | unsigned int idx = spec->args[0]; |
351 | |
352 | if (idx >= icc_data->num_nodes) { |
353 | pr_err("%s: invalid index %u\n" , __func__, idx); |
354 | return ERR_PTR(error: -EINVAL); |
355 | } |
356 | |
357 | return icc_data->nodes[idx]; |
358 | } |
359 | EXPORT_SYMBOL_GPL(of_icc_xlate_onecell); |
360 | |
361 | /** |
362 | * of_icc_get_from_provider() - Look-up interconnect node |
363 | * @spec: OF phandle args to use for look-up |
364 | * |
365 | * Looks for interconnect provider under the node specified by @spec and if |
366 | * found, uses xlate function of the provider to map phandle args to node. |
367 | * |
368 | * Returns a valid pointer to struct icc_node_data on success or ERR_PTR() |
369 | * on failure. |
370 | */ |
371 | struct icc_node_data *of_icc_get_from_provider(struct of_phandle_args *spec) |
372 | { |
373 | struct icc_node *node = ERR_PTR(error: -EPROBE_DEFER); |
374 | struct icc_node_data *data = NULL; |
375 | struct icc_provider *provider; |
376 | |
377 | if (!spec) |
378 | return ERR_PTR(error: -EINVAL); |
379 | |
380 | mutex_lock(&icc_lock); |
381 | list_for_each_entry(provider, &icc_providers, provider_list) { |
382 | if (provider->dev->of_node == spec->np) { |
383 | if (provider->xlate_extended) { |
384 | data = provider->xlate_extended(spec, provider->data); |
385 | if (!IS_ERR(ptr: data)) { |
386 | node = data->node; |
387 | break; |
388 | } |
389 | } else { |
390 | node = provider->xlate(spec, provider->data); |
391 | if (!IS_ERR(ptr: node)) |
392 | break; |
393 | } |
394 | } |
395 | } |
396 | mutex_unlock(lock: &icc_lock); |
397 | |
398 | if (IS_ERR(ptr: node)) |
399 | return ERR_CAST(ptr: node); |
400 | |
401 | if (!data) { |
402 | data = kzalloc(size: sizeof(*data), GFP_KERNEL); |
403 | if (!data) |
404 | return ERR_PTR(error: -ENOMEM); |
405 | data->node = node; |
406 | } |
407 | |
408 | return data; |
409 | } |
410 | EXPORT_SYMBOL_GPL(of_icc_get_from_provider); |
411 | |
412 | static void devm_icc_release(struct device *dev, void *res) |
413 | { |
414 | icc_put(path: *(struct icc_path **)res); |
415 | } |
416 | |
417 | struct icc_path *devm_of_icc_get(struct device *dev, const char *name) |
418 | { |
419 | struct icc_path **ptr, *path; |
420 | |
421 | ptr = devres_alloc(devm_icc_release, sizeof(*ptr), GFP_KERNEL); |
422 | if (!ptr) |
423 | return ERR_PTR(error: -ENOMEM); |
424 | |
425 | path = of_icc_get(dev, name); |
426 | if (!IS_ERR(ptr: path)) { |
427 | *ptr = path; |
428 | devres_add(dev, res: ptr); |
429 | } else { |
430 | devres_free(res: ptr); |
431 | } |
432 | |
433 | return path; |
434 | } |
435 | EXPORT_SYMBOL_GPL(devm_of_icc_get); |
436 | |
437 | /** |
438 | * of_icc_get_by_index() - get a path handle from a DT node based on index |
439 | * @dev: device pointer for the consumer device |
440 | * @idx: interconnect path index |
441 | * |
442 | * This function will search for a path between two endpoints and return an |
443 | * icc_path handle on success. Use icc_put() to release constraints when they |
444 | * are not needed anymore. |
445 | * If the interconnect API is disabled, NULL is returned and the consumer |
446 | * drivers will still build. Drivers are free to handle this specifically, |
447 | * but they don't have to. |
448 | * |
449 | * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned |
450 | * when the API is disabled or the "interconnects" DT property is missing. |
451 | */ |
452 | struct icc_path *of_icc_get_by_index(struct device *dev, int idx) |
453 | { |
454 | struct icc_path *path; |
455 | struct icc_node_data *src_data, *dst_data; |
456 | struct device_node *np; |
457 | struct of_phandle_args src_args, dst_args; |
458 | int ret; |
459 | |
460 | if (!dev || !dev->of_node) |
461 | return ERR_PTR(error: -ENODEV); |
462 | |
463 | np = dev->of_node; |
464 | |
465 | /* |
466 | * When the consumer DT node do not have "interconnects" property |
467 | * return a NULL path to skip setting constraints. |
468 | */ |
469 | if (!of_property_present(np, propname: "interconnects" )) |
470 | return NULL; |
471 | |
472 | /* |
473 | * We use a combination of phandle and specifier for endpoint. For now |
474 | * lets support only global ids and extend this in the future if needed |
475 | * without breaking DT compatibility. |
476 | */ |
477 | ret = of_parse_phandle_with_args(np, list_name: "interconnects" , |
478 | cells_name: "#interconnect-cells" , index: idx * 2, |
479 | out_args: &src_args); |
480 | if (ret) |
481 | return ERR_PTR(error: ret); |
482 | |
483 | of_node_put(node: src_args.np); |
484 | |
485 | ret = of_parse_phandle_with_args(np, list_name: "interconnects" , |
486 | cells_name: "#interconnect-cells" , index: idx * 2 + 1, |
487 | out_args: &dst_args); |
488 | if (ret) |
489 | return ERR_PTR(error: ret); |
490 | |
491 | of_node_put(node: dst_args.np); |
492 | |
493 | src_data = of_icc_get_from_provider(&src_args); |
494 | |
495 | if (IS_ERR(ptr: src_data)) { |
496 | dev_err_probe(dev, err: PTR_ERR(ptr: src_data), fmt: "error finding src node\n" ); |
497 | return ERR_CAST(ptr: src_data); |
498 | } |
499 | |
500 | dst_data = of_icc_get_from_provider(&dst_args); |
501 | |
502 | if (IS_ERR(ptr: dst_data)) { |
503 | dev_err_probe(dev, err: PTR_ERR(ptr: dst_data), fmt: "error finding dst node\n" ); |
504 | kfree(objp: src_data); |
505 | return ERR_CAST(ptr: dst_data); |
506 | } |
507 | |
508 | mutex_lock(&icc_lock); |
509 | path = path_find(dev, src: src_data->node, dst: dst_data->node); |
510 | mutex_unlock(lock: &icc_lock); |
511 | if (IS_ERR(ptr: path)) { |
512 | dev_err(dev, "%s: invalid path=%ld\n" , __func__, PTR_ERR(path)); |
513 | goto free_icc_data; |
514 | } |
515 | |
516 | if (src_data->tag && src_data->tag == dst_data->tag) |
517 | icc_set_tag(path, tag: src_data->tag); |
518 | |
519 | path->name = kasprintf(GFP_KERNEL, fmt: "%s-%s" , |
520 | src_data->node->name, dst_data->node->name); |
521 | if (!path->name) { |
522 | kfree(objp: path); |
523 | path = ERR_PTR(error: -ENOMEM); |
524 | } |
525 | |
526 | free_icc_data: |
527 | kfree(objp: src_data); |
528 | kfree(objp: dst_data); |
529 | return path; |
530 | } |
531 | EXPORT_SYMBOL_GPL(of_icc_get_by_index); |
532 | |
533 | /** |
534 | * of_icc_get() - get a path handle from a DT node based on name |
535 | * @dev: device pointer for the consumer device |
536 | * @name: interconnect path name |
537 | * |
538 | * This function will search for a path between two endpoints and return an |
539 | * icc_path handle on success. Use icc_put() to release constraints when they |
540 | * are not needed anymore. |
541 | * If the interconnect API is disabled, NULL is returned and the consumer |
542 | * drivers will still build. Drivers are free to handle this specifically, |
543 | * but they don't have to. |
544 | * |
545 | * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned |
546 | * when the API is disabled or the "interconnects" DT property is missing. |
547 | */ |
548 | struct icc_path *of_icc_get(struct device *dev, const char *name) |
549 | { |
550 | struct device_node *np; |
551 | int idx = 0; |
552 | |
553 | if (!dev || !dev->of_node) |
554 | return ERR_PTR(error: -ENODEV); |
555 | |
556 | np = dev->of_node; |
557 | |
558 | /* |
559 | * When the consumer DT node do not have "interconnects" property |
560 | * return a NULL path to skip setting constraints. |
561 | */ |
562 | if (!of_property_present(np, propname: "interconnects" )) |
563 | return NULL; |
564 | |
565 | /* |
566 | * We use a combination of phandle and specifier for endpoint. For now |
567 | * lets support only global ids and extend this in the future if needed |
568 | * without breaking DT compatibility. |
569 | */ |
570 | if (name) { |
571 | idx = of_property_match_string(np, propname: "interconnect-names" , string: name); |
572 | if (idx < 0) |
573 | return ERR_PTR(error: idx); |
574 | } |
575 | |
576 | return of_icc_get_by_index(dev, idx); |
577 | } |
578 | EXPORT_SYMBOL_GPL(of_icc_get); |
579 | |
580 | /** |
581 | * icc_get() - get a path handle between two endpoints |
582 | * @dev: device pointer for the consumer device |
583 | * @src: source node name |
584 | * @dst: destination node name |
585 | * |
586 | * This function will search for a path between two endpoints and return an |
587 | * icc_path handle on success. Use icc_put() to release constraints when they |
588 | * are not needed anymore. |
589 | * |
590 | * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned |
591 | * when the API is disabled. |
592 | */ |
593 | struct icc_path *icc_get(struct device *dev, const char *src, const char *dst) |
594 | { |
595 | struct icc_node *src_node, *dst_node; |
596 | struct icc_path *path = ERR_PTR(error: -EPROBE_DEFER); |
597 | |
598 | mutex_lock(&icc_lock); |
599 | |
600 | src_node = node_find_by_name(name: src); |
601 | if (!src_node) { |
602 | dev_err(dev, "%s: invalid src=%s\n" , __func__, src); |
603 | goto out; |
604 | } |
605 | |
606 | dst_node = node_find_by_name(name: dst); |
607 | if (!dst_node) { |
608 | dev_err(dev, "%s: invalid dst=%s\n" , __func__, dst); |
609 | goto out; |
610 | } |
611 | |
612 | path = path_find(dev, src: src_node, dst: dst_node); |
613 | if (IS_ERR(ptr: path)) { |
614 | dev_err(dev, "%s: invalid path=%ld\n" , __func__, PTR_ERR(path)); |
615 | goto out; |
616 | } |
617 | |
618 | path->name = kasprintf(GFP_KERNEL, fmt: "%s-%s" , src_node->name, dst_node->name); |
619 | if (!path->name) { |
620 | kfree(objp: path); |
621 | path = ERR_PTR(error: -ENOMEM); |
622 | } |
623 | out: |
624 | mutex_unlock(lock: &icc_lock); |
625 | return path; |
626 | } |
627 | |
628 | /** |
629 | * icc_set_tag() - set an optional tag on a path |
630 | * @path: the path we want to tag |
631 | * @tag: the tag value |
632 | * |
633 | * This function allows consumers to append a tag to the requests associated |
634 | * with a path, so that a different aggregation could be done based on this tag. |
635 | */ |
636 | void icc_set_tag(struct icc_path *path, u32 tag) |
637 | { |
638 | int i; |
639 | |
640 | if (!path) |
641 | return; |
642 | |
643 | mutex_lock(&icc_lock); |
644 | |
645 | for (i = 0; i < path->num_nodes; i++) |
646 | path->reqs[i].tag = tag; |
647 | |
648 | mutex_unlock(lock: &icc_lock); |
649 | } |
650 | EXPORT_SYMBOL_GPL(icc_set_tag); |
651 | |
652 | /** |
653 | * icc_get_name() - Get name of the icc path |
654 | * @path: interconnect path |
655 | * |
656 | * This function is used by an interconnect consumer to get the name of the icc |
657 | * path. |
658 | * |
659 | * Returns a valid pointer on success, or NULL otherwise. |
660 | */ |
661 | const char *icc_get_name(struct icc_path *path) |
662 | { |
663 | if (!path) |
664 | return NULL; |
665 | |
666 | return path->name; |
667 | } |
668 | EXPORT_SYMBOL_GPL(icc_get_name); |
669 | |
670 | /** |
671 | * icc_set_bw() - set bandwidth constraints on an interconnect path |
672 | * @path: interconnect path |
673 | * @avg_bw: average bandwidth in kilobytes per second |
674 | * @peak_bw: peak bandwidth in kilobytes per second |
675 | * |
676 | * This function is used by an interconnect consumer to express its own needs |
677 | * in terms of bandwidth for a previously requested path between two endpoints. |
678 | * The requests are aggregated and each node is updated accordingly. The entire |
679 | * path is locked by a mutex to ensure that the set() is completed. |
680 | * The @path can be NULL when the "interconnects" DT properties is missing, |
681 | * which will mean that no constraints will be set. |
682 | * |
683 | * Returns 0 on success, or an appropriate error code otherwise. |
684 | */ |
685 | int icc_set_bw(struct icc_path *path, u32 avg_bw, u32 peak_bw) |
686 | { |
687 | struct icc_node *node; |
688 | u32 old_avg, old_peak; |
689 | size_t i; |
690 | int ret; |
691 | |
692 | if (!path) |
693 | return 0; |
694 | |
695 | if (WARN_ON(IS_ERR(path) || !path->num_nodes)) |
696 | return -EINVAL; |
697 | |
698 | mutex_lock(&icc_bw_lock); |
699 | |
700 | old_avg = path->reqs[0].avg_bw; |
701 | old_peak = path->reqs[0].peak_bw; |
702 | |
703 | for (i = 0; i < path->num_nodes; i++) { |
704 | node = path->reqs[i].node; |
705 | |
706 | /* update the consumer request for this path */ |
707 | path->reqs[i].avg_bw = avg_bw; |
708 | path->reqs[i].peak_bw = peak_bw; |
709 | |
710 | /* aggregate requests for this node */ |
711 | aggregate_requests(node); |
712 | |
713 | trace_icc_set_bw(p: path, n: node, i, avg_bw, peak_bw); |
714 | } |
715 | |
716 | ret = apply_constraints(path); |
717 | if (ret) { |
718 | pr_debug("interconnect: error applying constraints (%d)\n" , |
719 | ret); |
720 | |
721 | for (i = 0; i < path->num_nodes; i++) { |
722 | node = path->reqs[i].node; |
723 | path->reqs[i].avg_bw = old_avg; |
724 | path->reqs[i].peak_bw = old_peak; |
725 | aggregate_requests(node); |
726 | } |
727 | apply_constraints(path); |
728 | } |
729 | |
730 | mutex_unlock(lock: &icc_bw_lock); |
731 | |
732 | trace_icc_set_bw_end(p: path, ret); |
733 | |
734 | return ret; |
735 | } |
736 | EXPORT_SYMBOL_GPL(icc_set_bw); |
737 | |
738 | static int __icc_enable(struct icc_path *path, bool enable) |
739 | { |
740 | int i; |
741 | |
742 | if (!path) |
743 | return 0; |
744 | |
745 | if (WARN_ON(IS_ERR(path) || !path->num_nodes)) |
746 | return -EINVAL; |
747 | |
748 | mutex_lock(&icc_lock); |
749 | |
750 | for (i = 0; i < path->num_nodes; i++) |
751 | path->reqs[i].enabled = enable; |
752 | |
753 | mutex_unlock(lock: &icc_lock); |
754 | |
755 | return icc_set_bw(path, path->reqs[0].avg_bw, |
756 | path->reqs[0].peak_bw); |
757 | } |
758 | |
759 | int icc_enable(struct icc_path *path) |
760 | { |
761 | return __icc_enable(path, enable: true); |
762 | } |
763 | EXPORT_SYMBOL_GPL(icc_enable); |
764 | |
765 | int icc_disable(struct icc_path *path) |
766 | { |
767 | return __icc_enable(path, enable: false); |
768 | } |
769 | EXPORT_SYMBOL_GPL(icc_disable); |
770 | |
771 | /** |
772 | * icc_put() - release the reference to the icc_path |
773 | * @path: interconnect path |
774 | * |
775 | * Use this function to release the constraints on a path when the path is |
776 | * no longer needed. The constraints will be re-aggregated. |
777 | */ |
778 | void icc_put(struct icc_path *path) |
779 | { |
780 | struct icc_node *node; |
781 | size_t i; |
782 | int ret; |
783 | |
784 | if (!path || WARN_ON(IS_ERR(path))) |
785 | return; |
786 | |
787 | ret = icc_set_bw(path, 0, 0); |
788 | if (ret) |
789 | pr_err("%s: error (%d)\n" , __func__, ret); |
790 | |
791 | mutex_lock(&icc_lock); |
792 | for (i = 0; i < path->num_nodes; i++) { |
793 | node = path->reqs[i].node; |
794 | hlist_del(n: &path->reqs[i].req_node); |
795 | if (!WARN_ON(!node->provider->users)) |
796 | node->provider->users--; |
797 | } |
798 | mutex_unlock(lock: &icc_lock); |
799 | |
800 | kfree_const(x: path->name); |
801 | kfree(objp: path); |
802 | } |
803 | EXPORT_SYMBOL_GPL(icc_put); |
804 | |
805 | static struct icc_node *icc_node_create_nolock(int id) |
806 | { |
807 | struct icc_node *node; |
808 | |
809 | /* check if node already exists */ |
810 | node = node_find(id); |
811 | if (node) |
812 | return node; |
813 | |
814 | node = kzalloc(size: sizeof(*node), GFP_KERNEL); |
815 | if (!node) |
816 | return ERR_PTR(error: -ENOMEM); |
817 | |
818 | id = idr_alloc(&icc_idr, ptr: node, start: id, end: id + 1, GFP_KERNEL); |
819 | if (id < 0) { |
820 | WARN(1, "%s: couldn't get idr\n" , __func__); |
821 | kfree(objp: node); |
822 | return ERR_PTR(error: id); |
823 | } |
824 | |
825 | node->id = id; |
826 | |
827 | return node; |
828 | } |
829 | |
830 | /** |
831 | * icc_node_create() - create a node |
832 | * @id: node id |
833 | * |
834 | * Return: icc_node pointer on success, or ERR_PTR() on error |
835 | */ |
836 | struct icc_node *icc_node_create(int id) |
837 | { |
838 | struct icc_node *node; |
839 | |
840 | mutex_lock(&icc_lock); |
841 | |
842 | node = icc_node_create_nolock(id); |
843 | |
844 | mutex_unlock(lock: &icc_lock); |
845 | |
846 | return node; |
847 | } |
848 | EXPORT_SYMBOL_GPL(icc_node_create); |
849 | |
850 | /** |
851 | * icc_node_destroy() - destroy a node |
852 | * @id: node id |
853 | */ |
854 | void icc_node_destroy(int id) |
855 | { |
856 | struct icc_node *node; |
857 | |
858 | mutex_lock(&icc_lock); |
859 | |
860 | node = node_find(id); |
861 | if (node) { |
862 | idr_remove(&icc_idr, id: node->id); |
863 | WARN_ON(!hlist_empty(&node->req_list)); |
864 | } |
865 | |
866 | mutex_unlock(lock: &icc_lock); |
867 | |
868 | if (!node) |
869 | return; |
870 | |
871 | kfree(objp: node->links); |
872 | kfree(objp: node); |
873 | } |
874 | EXPORT_SYMBOL_GPL(icc_node_destroy); |
875 | |
876 | /** |
877 | * icc_link_create() - create a link between two nodes |
878 | * @node: source node id |
879 | * @dst_id: destination node id |
880 | * |
881 | * Create a link between two nodes. The nodes might belong to different |
882 | * interconnect providers and the @dst_id node might not exist (if the |
883 | * provider driver has not probed yet). So just create the @dst_id node |
884 | * and when the actual provider driver is probed, the rest of the node |
885 | * data is filled. |
886 | * |
887 | * Return: 0 on success, or an error code otherwise |
888 | */ |
889 | int icc_link_create(struct icc_node *node, const int dst_id) |
890 | { |
891 | struct icc_node *dst; |
892 | struct icc_node **new; |
893 | int ret = 0; |
894 | |
895 | if (!node->provider) |
896 | return -EINVAL; |
897 | |
898 | mutex_lock(&icc_lock); |
899 | |
900 | dst = node_find(id: dst_id); |
901 | if (!dst) { |
902 | dst = icc_node_create_nolock(id: dst_id); |
903 | |
904 | if (IS_ERR(ptr: dst)) { |
905 | ret = PTR_ERR(ptr: dst); |
906 | goto out; |
907 | } |
908 | } |
909 | |
910 | new = krealloc(objp: node->links, |
911 | new_size: (node->num_links + 1) * sizeof(*node->links), |
912 | GFP_KERNEL); |
913 | if (!new) { |
914 | ret = -ENOMEM; |
915 | goto out; |
916 | } |
917 | |
918 | node->links = new; |
919 | node->links[node->num_links++] = dst; |
920 | |
921 | out: |
922 | mutex_unlock(lock: &icc_lock); |
923 | |
924 | return ret; |
925 | } |
926 | EXPORT_SYMBOL_GPL(icc_link_create); |
927 | |
928 | /** |
929 | * icc_node_add() - add interconnect node to interconnect provider |
930 | * @node: pointer to the interconnect node |
931 | * @provider: pointer to the interconnect provider |
932 | */ |
933 | void icc_node_add(struct icc_node *node, struct icc_provider *provider) |
934 | { |
935 | if (WARN_ON(node->provider)) |
936 | return; |
937 | |
938 | mutex_lock(&icc_lock); |
939 | mutex_lock(&icc_bw_lock); |
940 | |
941 | node->provider = provider; |
942 | list_add_tail(new: &node->node_list, head: &provider->nodes); |
943 | |
944 | /* get the initial bandwidth values and sync them with hardware */ |
945 | if (provider->get_bw) { |
946 | provider->get_bw(node, &node->init_avg, &node->init_peak); |
947 | } else { |
948 | node->init_avg = INT_MAX; |
949 | node->init_peak = INT_MAX; |
950 | } |
951 | node->avg_bw = node->init_avg; |
952 | node->peak_bw = node->init_peak; |
953 | |
954 | if (node->avg_bw || node->peak_bw) { |
955 | if (provider->pre_aggregate) |
956 | provider->pre_aggregate(node); |
957 | |
958 | if (provider->aggregate) |
959 | provider->aggregate(node, 0, node->init_avg, node->init_peak, |
960 | &node->avg_bw, &node->peak_bw); |
961 | if (provider->set) |
962 | provider->set(node, node); |
963 | } |
964 | |
965 | node->avg_bw = 0; |
966 | node->peak_bw = 0; |
967 | |
968 | mutex_unlock(lock: &icc_bw_lock); |
969 | mutex_unlock(lock: &icc_lock); |
970 | } |
971 | EXPORT_SYMBOL_GPL(icc_node_add); |
972 | |
973 | /** |
974 | * icc_node_del() - delete interconnect node from interconnect provider |
975 | * @node: pointer to the interconnect node |
976 | */ |
977 | void icc_node_del(struct icc_node *node) |
978 | { |
979 | mutex_lock(&icc_lock); |
980 | |
981 | list_del(entry: &node->node_list); |
982 | |
983 | mutex_unlock(lock: &icc_lock); |
984 | } |
985 | EXPORT_SYMBOL_GPL(icc_node_del); |
986 | |
987 | /** |
988 | * icc_nodes_remove() - remove all previously added nodes from provider |
989 | * @provider: the interconnect provider we are removing nodes from |
990 | * |
991 | * Return: 0 on success, or an error code otherwise |
992 | */ |
993 | int icc_nodes_remove(struct icc_provider *provider) |
994 | { |
995 | struct icc_node *n, *tmp; |
996 | |
997 | if (WARN_ON(IS_ERR_OR_NULL(provider))) |
998 | return -EINVAL; |
999 | |
1000 | list_for_each_entry_safe_reverse(n, tmp, &provider->nodes, node_list) { |
1001 | icc_node_del(n); |
1002 | icc_node_destroy(n->id); |
1003 | } |
1004 | |
1005 | return 0; |
1006 | } |
1007 | EXPORT_SYMBOL_GPL(icc_nodes_remove); |
1008 | |
1009 | /** |
1010 | * icc_provider_init() - initialize a new interconnect provider |
1011 | * @provider: the interconnect provider to initialize |
1012 | * |
1013 | * Must be called before adding nodes to the provider. |
1014 | */ |
1015 | void icc_provider_init(struct icc_provider *provider) |
1016 | { |
1017 | WARN_ON(!provider->set); |
1018 | |
1019 | INIT_LIST_HEAD(list: &provider->nodes); |
1020 | } |
1021 | EXPORT_SYMBOL_GPL(icc_provider_init); |
1022 | |
1023 | /** |
1024 | * icc_provider_register() - register a new interconnect provider |
1025 | * @provider: the interconnect provider to register |
1026 | * |
1027 | * Return: 0 on success, or an error code otherwise |
1028 | */ |
1029 | int icc_provider_register(struct icc_provider *provider) |
1030 | { |
1031 | if (WARN_ON(!provider->xlate && !provider->xlate_extended)) |
1032 | return -EINVAL; |
1033 | |
1034 | mutex_lock(&icc_lock); |
1035 | list_add_tail(new: &provider->provider_list, head: &icc_providers); |
1036 | mutex_unlock(lock: &icc_lock); |
1037 | |
1038 | dev_dbg(provider->dev, "interconnect provider registered\n" ); |
1039 | |
1040 | return 0; |
1041 | } |
1042 | EXPORT_SYMBOL_GPL(icc_provider_register); |
1043 | |
1044 | /** |
1045 | * icc_provider_deregister() - deregister an interconnect provider |
1046 | * @provider: the interconnect provider to deregister |
1047 | */ |
1048 | void icc_provider_deregister(struct icc_provider *provider) |
1049 | { |
1050 | mutex_lock(&icc_lock); |
1051 | WARN_ON(provider->users); |
1052 | |
1053 | list_del(entry: &provider->provider_list); |
1054 | mutex_unlock(lock: &icc_lock); |
1055 | } |
1056 | EXPORT_SYMBOL_GPL(icc_provider_deregister); |
1057 | |
1058 | static const struct of_device_id __maybe_unused ignore_list[] = { |
1059 | { .compatible = "qcom,sc7180-ipa-virt" }, |
1060 | { .compatible = "qcom,sc8180x-ipa-virt" }, |
1061 | { .compatible = "qcom,sdx55-ipa-virt" }, |
1062 | { .compatible = "qcom,sm8150-ipa-virt" }, |
1063 | { .compatible = "qcom,sm8250-ipa-virt" }, |
1064 | {} |
1065 | }; |
1066 | |
1067 | static int of_count_icc_providers(struct device_node *np) |
1068 | { |
1069 | struct device_node *child; |
1070 | int count = 0; |
1071 | |
1072 | for_each_available_child_of_node(np, child) { |
1073 | if (of_property_read_bool(np: child, propname: "#interconnect-cells" ) && |
1074 | likely(!of_match_node(ignore_list, child))) |
1075 | count++; |
1076 | count += of_count_icc_providers(np: child); |
1077 | } |
1078 | |
1079 | return count; |
1080 | } |
1081 | |
1082 | void icc_sync_state(struct device *dev) |
1083 | { |
1084 | struct icc_provider *p; |
1085 | struct icc_node *n; |
1086 | static int count; |
1087 | |
1088 | count++; |
1089 | |
1090 | if (count < providers_count) |
1091 | return; |
1092 | |
1093 | mutex_lock(&icc_lock); |
1094 | mutex_lock(&icc_bw_lock); |
1095 | synced_state = true; |
1096 | list_for_each_entry(p, &icc_providers, provider_list) { |
1097 | dev_dbg(p->dev, "interconnect provider is in synced state\n" ); |
1098 | list_for_each_entry(n, &p->nodes, node_list) { |
1099 | if (n->init_avg || n->init_peak) { |
1100 | n->init_avg = 0; |
1101 | n->init_peak = 0; |
1102 | aggregate_requests(node: n); |
1103 | p->set(n, n); |
1104 | } |
1105 | } |
1106 | } |
1107 | mutex_unlock(lock: &icc_bw_lock); |
1108 | mutex_unlock(lock: &icc_lock); |
1109 | } |
1110 | EXPORT_SYMBOL_GPL(icc_sync_state); |
1111 | |
1112 | static int __init icc_init(void) |
1113 | { |
1114 | struct device_node *root; |
1115 | |
1116 | /* Teach lockdep about lock ordering wrt. shrinker: */ |
1117 | fs_reclaim_acquire(GFP_KERNEL); |
1118 | might_lock(&icc_bw_lock); |
1119 | fs_reclaim_release(GFP_KERNEL); |
1120 | |
1121 | root = of_find_node_by_path(path: "/" ); |
1122 | |
1123 | providers_count = of_count_icc_providers(np: root); |
1124 | of_node_put(node: root); |
1125 | |
1126 | icc_debugfs_dir = debugfs_create_dir(name: "interconnect" , NULL); |
1127 | debugfs_create_file(name: "interconnect_summary" , mode: 0444, |
1128 | parent: icc_debugfs_dir, NULL, fops: &icc_summary_fops); |
1129 | debugfs_create_file(name: "interconnect_graph" , mode: 0444, |
1130 | parent: icc_debugfs_dir, NULL, fops: &icc_graph_fops); |
1131 | |
1132 | icc_debugfs_client_init(icc_dir: icc_debugfs_dir); |
1133 | |
1134 | return 0; |
1135 | } |
1136 | |
1137 | device_initcall(icc_init); |
1138 | |