1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Interconnect framework core driver
4 *
5 * Copyright (c) 2017-2019, Linaro Ltd.
6 * Author: Georgi Djakov <georgi.djakov@linaro.org>
7 */
8
9#include <linux/debugfs.h>
10#include <linux/device.h>
11#include <linux/idr.h>
12#include <linux/init.h>
13#include <linux/interconnect.h>
14#include <linux/interconnect-provider.h>
15#include <linux/list.h>
16#include <linux/mutex.h>
17#include <linux/slab.h>
18#include <linux/of.h>
19#include <linux/overflow.h>
20
21#include "internal.h"
22
23#define CREATE_TRACE_POINTS
24#include "trace.h"
25
26static DEFINE_IDR(icc_idr);
27static LIST_HEAD(icc_providers);
28static int providers_count;
29static bool synced_state;
30static DEFINE_MUTEX(icc_lock);
31static DEFINE_MUTEX(icc_bw_lock);
32static struct dentry *icc_debugfs_dir;
33
34static void icc_summary_show_one(struct seq_file *s, struct icc_node *n)
35{
36 if (!n)
37 return;
38
39 seq_printf(m: s, fmt: "%-42s %12u %12u\n",
40 n->name, n->avg_bw, n->peak_bw);
41}
42
43static int icc_summary_show(struct seq_file *s, void *data)
44{
45 struct icc_provider *provider;
46
47 seq_puts(m: s, s: " node tag avg peak\n");
48 seq_puts(m: s, s: "--------------------------------------------------------------------\n");
49
50 mutex_lock(&icc_lock);
51
52 list_for_each_entry(provider, &icc_providers, provider_list) {
53 struct icc_node *n;
54
55 list_for_each_entry(n, &provider->nodes, node_list) {
56 struct icc_req *r;
57
58 icc_summary_show_one(s, n);
59 hlist_for_each_entry(r, &n->req_list, req_node) {
60 u32 avg_bw = 0, peak_bw = 0;
61
62 if (!r->dev)
63 continue;
64
65 if (r->enabled) {
66 avg_bw = r->avg_bw;
67 peak_bw = r->peak_bw;
68 }
69
70 seq_printf(m: s, fmt: " %-27s %12u %12u %12u\n",
71 dev_name(dev: r->dev), r->tag, avg_bw, peak_bw);
72 }
73 }
74 }
75
76 mutex_unlock(lock: &icc_lock);
77
78 return 0;
79}
80DEFINE_SHOW_ATTRIBUTE(icc_summary);
81
82static void icc_graph_show_link(struct seq_file *s, int level,
83 struct icc_node *n, struct icc_node *m)
84{
85 seq_printf(m: s, fmt: "%s\"%d:%s\" -> \"%d:%s\"\n",
86 level == 2 ? "\t\t" : "\t",
87 n->id, n->name, m->id, m->name);
88}
89
90static void icc_graph_show_node(struct seq_file *s, struct icc_node *n)
91{
92 seq_printf(m: s, fmt: "\t\t\"%d:%s\" [label=\"%d:%s",
93 n->id, n->name, n->id, n->name);
94 seq_printf(m: s, fmt: "\n\t\t\t|avg_bw=%ukBps", n->avg_bw);
95 seq_printf(m: s, fmt: "\n\t\t\t|peak_bw=%ukBps", n->peak_bw);
96 seq_puts(m: s, s: "\"]\n");
97}
98
99static int icc_graph_show(struct seq_file *s, void *data)
100{
101 struct icc_provider *provider;
102 struct icc_node *n;
103 int cluster_index = 0;
104 int i;
105
106 seq_puts(m: s, s: "digraph {\n\trankdir = LR\n\tnode [shape = record]\n");
107 mutex_lock(&icc_lock);
108
109 /* draw providers as cluster subgraphs */
110 cluster_index = 0;
111 list_for_each_entry(provider, &icc_providers, provider_list) {
112 seq_printf(m: s, fmt: "\tsubgraph cluster_%d {\n", ++cluster_index);
113 if (provider->dev)
114 seq_printf(m: s, fmt: "\t\tlabel = \"%s\"\n",
115 dev_name(dev: provider->dev));
116
117 /* draw nodes */
118 list_for_each_entry(n, &provider->nodes, node_list)
119 icc_graph_show_node(s, n);
120
121 /* draw internal links */
122 list_for_each_entry(n, &provider->nodes, node_list)
123 for (i = 0; i < n->num_links; ++i)
124 if (n->provider == n->links[i]->provider)
125 icc_graph_show_link(s, level: 2, n,
126 m: n->links[i]);
127
128 seq_puts(m: s, s: "\t}\n");
129 }
130
131 /* draw external links */
132 list_for_each_entry(provider, &icc_providers, provider_list)
133 list_for_each_entry(n, &provider->nodes, node_list)
134 for (i = 0; i < n->num_links; ++i)
135 if (n->provider != n->links[i]->provider)
136 icc_graph_show_link(s, level: 1, n,
137 m: n->links[i]);
138
139 mutex_unlock(lock: &icc_lock);
140 seq_puts(m: s, s: "}");
141
142 return 0;
143}
144DEFINE_SHOW_ATTRIBUTE(icc_graph);
145
146static struct icc_node *node_find(const int id)
147{
148 return idr_find(&icc_idr, id);
149}
150
151static struct icc_node *node_find_by_name(const char *name)
152{
153 struct icc_provider *provider;
154 struct icc_node *n;
155
156 list_for_each_entry(provider, &icc_providers, provider_list) {
157 list_for_each_entry(n, &provider->nodes, node_list) {
158 if (!strcmp(n->name, name))
159 return n;
160 }
161 }
162
163 return NULL;
164}
165
166static struct icc_path *path_init(struct device *dev, struct icc_node *dst,
167 ssize_t num_nodes)
168{
169 struct icc_node *node = dst;
170 struct icc_path *path;
171 int i;
172
173 path = kzalloc(struct_size(path, reqs, num_nodes), GFP_KERNEL);
174 if (!path)
175 return ERR_PTR(error: -ENOMEM);
176
177 path->num_nodes = num_nodes;
178
179 for (i = num_nodes - 1; i >= 0; i--) {
180 node->provider->users++;
181 hlist_add_head(n: &path->reqs[i].req_node, h: &node->req_list);
182 path->reqs[i].node = node;
183 path->reqs[i].dev = dev;
184 path->reqs[i].enabled = true;
185 /* reference to previous node was saved during path traversal */
186 node = node->reverse;
187 }
188
189 return path;
190}
191
192static struct icc_path *path_find(struct device *dev, struct icc_node *src,
193 struct icc_node *dst)
194{
195 struct icc_path *path = ERR_PTR(error: -EPROBE_DEFER);
196 struct icc_node *n, *node = NULL;
197 struct list_head traverse_list;
198 struct list_head edge_list;
199 struct list_head visited_list;
200 size_t i, depth = 1;
201 bool found = false;
202
203 INIT_LIST_HEAD(list: &traverse_list);
204 INIT_LIST_HEAD(list: &edge_list);
205 INIT_LIST_HEAD(list: &visited_list);
206
207 list_add(new: &src->search_list, head: &traverse_list);
208 src->reverse = NULL;
209
210 do {
211 list_for_each_entry_safe(node, n, &traverse_list, search_list) {
212 if (node == dst) {
213 found = true;
214 list_splice_init(list: &edge_list, head: &visited_list);
215 list_splice_init(list: &traverse_list, head: &visited_list);
216 break;
217 }
218 for (i = 0; i < node->num_links; i++) {
219 struct icc_node *tmp = node->links[i];
220
221 if (!tmp) {
222 path = ERR_PTR(error: -ENOENT);
223 goto out;
224 }
225
226 if (tmp->is_traversed)
227 continue;
228
229 tmp->is_traversed = true;
230 tmp->reverse = node;
231 list_add_tail(new: &tmp->search_list, head: &edge_list);
232 }
233 }
234
235 if (found)
236 break;
237
238 list_splice_init(list: &traverse_list, head: &visited_list);
239 list_splice_init(list: &edge_list, head: &traverse_list);
240
241 /* count the hops including the source */
242 depth++;
243
244 } while (!list_empty(head: &traverse_list));
245
246out:
247
248 /* reset the traversed state */
249 list_for_each_entry_reverse(n, &visited_list, search_list)
250 n->is_traversed = false;
251
252 if (found)
253 path = path_init(dev, dst, num_nodes: depth);
254
255 return path;
256}
257
258/*
259 * We want the path to honor all bandwidth requests, so the average and peak
260 * bandwidth requirements from each consumer are aggregated at each node.
261 * The aggregation is platform specific, so each platform can customize it by
262 * implementing its own aggregate() function.
263 */
264
265static int aggregate_requests(struct icc_node *node)
266{
267 struct icc_provider *p = node->provider;
268 struct icc_req *r;
269 u32 avg_bw, peak_bw;
270
271 node->avg_bw = 0;
272 node->peak_bw = 0;
273
274 if (p->pre_aggregate)
275 p->pre_aggregate(node);
276
277 hlist_for_each_entry(r, &node->req_list, req_node) {
278 if (r->enabled) {
279 avg_bw = r->avg_bw;
280 peak_bw = r->peak_bw;
281 } else {
282 avg_bw = 0;
283 peak_bw = 0;
284 }
285 p->aggregate(node, r->tag, avg_bw, peak_bw,
286 &node->avg_bw, &node->peak_bw);
287
288 /* during boot use the initial bandwidth as a floor value */
289 if (!synced_state) {
290 node->avg_bw = max(node->avg_bw, node->init_avg);
291 node->peak_bw = max(node->peak_bw, node->init_peak);
292 }
293 }
294
295 return 0;
296}
297
298static int apply_constraints(struct icc_path *path)
299{
300 struct icc_node *next, *prev = NULL;
301 struct icc_provider *p;
302 int ret = -EINVAL;
303 int i;
304
305 for (i = 0; i < path->num_nodes; i++) {
306 next = path->reqs[i].node;
307 p = next->provider;
308
309 /* both endpoints should be valid master-slave pairs */
310 if (!prev || (p != prev->provider && !p->inter_set)) {
311 prev = next;
312 continue;
313 }
314
315 /* set the constraints */
316 ret = p->set(prev, next);
317 if (ret)
318 goto out;
319
320 prev = next;
321 }
322out:
323 return ret;
324}
325
326int icc_std_aggregate(struct icc_node *node, u32 tag, u32 avg_bw,
327 u32 peak_bw, u32 *agg_avg, u32 *agg_peak)
328{
329 *agg_avg += avg_bw;
330 *agg_peak = max(*agg_peak, peak_bw);
331
332 return 0;
333}
334EXPORT_SYMBOL_GPL(icc_std_aggregate);
335
336/* of_icc_xlate_onecell() - Translate function using a single index.
337 * @spec: OF phandle args to map into an interconnect node.
338 * @data: private data (pointer to struct icc_onecell_data)
339 *
340 * This is a generic translate function that can be used to model simple
341 * interconnect providers that have one device tree node and provide
342 * multiple interconnect nodes. A single cell is used as an index into
343 * an array of icc nodes specified in the icc_onecell_data struct when
344 * registering the provider.
345 */
346struct icc_node *of_icc_xlate_onecell(struct of_phandle_args *spec,
347 void *data)
348{
349 struct icc_onecell_data *icc_data = data;
350 unsigned int idx = spec->args[0];
351
352 if (idx >= icc_data->num_nodes) {
353 pr_err("%s: invalid index %u\n", __func__, idx);
354 return ERR_PTR(error: -EINVAL);
355 }
356
357 return icc_data->nodes[idx];
358}
359EXPORT_SYMBOL_GPL(of_icc_xlate_onecell);
360
361/**
362 * of_icc_get_from_provider() - Look-up interconnect node
363 * @spec: OF phandle args to use for look-up
364 *
365 * Looks for interconnect provider under the node specified by @spec and if
366 * found, uses xlate function of the provider to map phandle args to node.
367 *
368 * Returns a valid pointer to struct icc_node_data on success or ERR_PTR()
369 * on failure.
370 */
371struct icc_node_data *of_icc_get_from_provider(struct of_phandle_args *spec)
372{
373 struct icc_node *node = ERR_PTR(error: -EPROBE_DEFER);
374 struct icc_node_data *data = NULL;
375 struct icc_provider *provider;
376
377 if (!spec)
378 return ERR_PTR(error: -EINVAL);
379
380 mutex_lock(&icc_lock);
381 list_for_each_entry(provider, &icc_providers, provider_list) {
382 if (provider->dev->of_node == spec->np) {
383 if (provider->xlate_extended) {
384 data = provider->xlate_extended(spec, provider->data);
385 if (!IS_ERR(ptr: data)) {
386 node = data->node;
387 break;
388 }
389 } else {
390 node = provider->xlate(spec, provider->data);
391 if (!IS_ERR(ptr: node))
392 break;
393 }
394 }
395 }
396 mutex_unlock(lock: &icc_lock);
397
398 if (IS_ERR(ptr: node))
399 return ERR_CAST(ptr: node);
400
401 if (!data) {
402 data = kzalloc(size: sizeof(*data), GFP_KERNEL);
403 if (!data)
404 return ERR_PTR(error: -ENOMEM);
405 data->node = node;
406 }
407
408 return data;
409}
410EXPORT_SYMBOL_GPL(of_icc_get_from_provider);
411
412static void devm_icc_release(struct device *dev, void *res)
413{
414 icc_put(path: *(struct icc_path **)res);
415}
416
417struct icc_path *devm_of_icc_get(struct device *dev, const char *name)
418{
419 struct icc_path **ptr, *path;
420
421 ptr = devres_alloc(devm_icc_release, sizeof(*ptr), GFP_KERNEL);
422 if (!ptr)
423 return ERR_PTR(error: -ENOMEM);
424
425 path = of_icc_get(dev, name);
426 if (!IS_ERR(ptr: path)) {
427 *ptr = path;
428 devres_add(dev, res: ptr);
429 } else {
430 devres_free(res: ptr);
431 }
432
433 return path;
434}
435EXPORT_SYMBOL_GPL(devm_of_icc_get);
436
437/**
438 * of_icc_get_by_index() - get a path handle from a DT node based on index
439 * @dev: device pointer for the consumer device
440 * @idx: interconnect path index
441 *
442 * This function will search for a path between two endpoints and return an
443 * icc_path handle on success. Use icc_put() to release constraints when they
444 * are not needed anymore.
445 * If the interconnect API is disabled, NULL is returned and the consumer
446 * drivers will still build. Drivers are free to handle this specifically,
447 * but they don't have to.
448 *
449 * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned
450 * when the API is disabled or the "interconnects" DT property is missing.
451 */
452struct icc_path *of_icc_get_by_index(struct device *dev, int idx)
453{
454 struct icc_path *path;
455 struct icc_node_data *src_data, *dst_data;
456 struct device_node *np;
457 struct of_phandle_args src_args, dst_args;
458 int ret;
459
460 if (!dev || !dev->of_node)
461 return ERR_PTR(error: -ENODEV);
462
463 np = dev->of_node;
464
465 /*
466 * When the consumer DT node do not have "interconnects" property
467 * return a NULL path to skip setting constraints.
468 */
469 if (!of_property_present(np, propname: "interconnects"))
470 return NULL;
471
472 /*
473 * We use a combination of phandle and specifier for endpoint. For now
474 * lets support only global ids and extend this in the future if needed
475 * without breaking DT compatibility.
476 */
477 ret = of_parse_phandle_with_args(np, list_name: "interconnects",
478 cells_name: "#interconnect-cells", index: idx * 2,
479 out_args: &src_args);
480 if (ret)
481 return ERR_PTR(error: ret);
482
483 of_node_put(node: src_args.np);
484
485 ret = of_parse_phandle_with_args(np, list_name: "interconnects",
486 cells_name: "#interconnect-cells", index: idx * 2 + 1,
487 out_args: &dst_args);
488 if (ret)
489 return ERR_PTR(error: ret);
490
491 of_node_put(node: dst_args.np);
492
493 src_data = of_icc_get_from_provider(&src_args);
494
495 if (IS_ERR(ptr: src_data)) {
496 dev_err_probe(dev, err: PTR_ERR(ptr: src_data), fmt: "error finding src node\n");
497 return ERR_CAST(ptr: src_data);
498 }
499
500 dst_data = of_icc_get_from_provider(&dst_args);
501
502 if (IS_ERR(ptr: dst_data)) {
503 dev_err_probe(dev, err: PTR_ERR(ptr: dst_data), fmt: "error finding dst node\n");
504 kfree(objp: src_data);
505 return ERR_CAST(ptr: dst_data);
506 }
507
508 mutex_lock(&icc_lock);
509 path = path_find(dev, src: src_data->node, dst: dst_data->node);
510 mutex_unlock(lock: &icc_lock);
511 if (IS_ERR(ptr: path)) {
512 dev_err(dev, "%s: invalid path=%ld\n", __func__, PTR_ERR(path));
513 goto free_icc_data;
514 }
515
516 if (src_data->tag && src_data->tag == dst_data->tag)
517 icc_set_tag(path, tag: src_data->tag);
518
519 path->name = kasprintf(GFP_KERNEL, fmt: "%s-%s",
520 src_data->node->name, dst_data->node->name);
521 if (!path->name) {
522 kfree(objp: path);
523 path = ERR_PTR(error: -ENOMEM);
524 }
525
526free_icc_data:
527 kfree(objp: src_data);
528 kfree(objp: dst_data);
529 return path;
530}
531EXPORT_SYMBOL_GPL(of_icc_get_by_index);
532
533/**
534 * of_icc_get() - get a path handle from a DT node based on name
535 * @dev: device pointer for the consumer device
536 * @name: interconnect path name
537 *
538 * This function will search for a path between two endpoints and return an
539 * icc_path handle on success. Use icc_put() to release constraints when they
540 * are not needed anymore.
541 * If the interconnect API is disabled, NULL is returned and the consumer
542 * drivers will still build. Drivers are free to handle this specifically,
543 * but they don't have to.
544 *
545 * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned
546 * when the API is disabled or the "interconnects" DT property is missing.
547 */
548struct icc_path *of_icc_get(struct device *dev, const char *name)
549{
550 struct device_node *np;
551 int idx = 0;
552
553 if (!dev || !dev->of_node)
554 return ERR_PTR(error: -ENODEV);
555
556 np = dev->of_node;
557
558 /*
559 * When the consumer DT node do not have "interconnects" property
560 * return a NULL path to skip setting constraints.
561 */
562 if (!of_property_present(np, propname: "interconnects"))
563 return NULL;
564
565 /*
566 * We use a combination of phandle and specifier for endpoint. For now
567 * lets support only global ids and extend this in the future if needed
568 * without breaking DT compatibility.
569 */
570 if (name) {
571 idx = of_property_match_string(np, propname: "interconnect-names", string: name);
572 if (idx < 0)
573 return ERR_PTR(error: idx);
574 }
575
576 return of_icc_get_by_index(dev, idx);
577}
578EXPORT_SYMBOL_GPL(of_icc_get);
579
580/**
581 * icc_get() - get a path handle between two endpoints
582 * @dev: device pointer for the consumer device
583 * @src: source node name
584 * @dst: destination node name
585 *
586 * This function will search for a path between two endpoints and return an
587 * icc_path handle on success. Use icc_put() to release constraints when they
588 * are not needed anymore.
589 *
590 * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned
591 * when the API is disabled.
592 */
593struct icc_path *icc_get(struct device *dev, const char *src, const char *dst)
594{
595 struct icc_node *src_node, *dst_node;
596 struct icc_path *path = ERR_PTR(error: -EPROBE_DEFER);
597
598 mutex_lock(&icc_lock);
599
600 src_node = node_find_by_name(name: src);
601 if (!src_node) {
602 dev_err(dev, "%s: invalid src=%s\n", __func__, src);
603 goto out;
604 }
605
606 dst_node = node_find_by_name(name: dst);
607 if (!dst_node) {
608 dev_err(dev, "%s: invalid dst=%s\n", __func__, dst);
609 goto out;
610 }
611
612 path = path_find(dev, src: src_node, dst: dst_node);
613 if (IS_ERR(ptr: path)) {
614 dev_err(dev, "%s: invalid path=%ld\n", __func__, PTR_ERR(path));
615 goto out;
616 }
617
618 path->name = kasprintf(GFP_KERNEL, fmt: "%s-%s", src_node->name, dst_node->name);
619 if (!path->name) {
620 kfree(objp: path);
621 path = ERR_PTR(error: -ENOMEM);
622 }
623out:
624 mutex_unlock(lock: &icc_lock);
625 return path;
626}
627
628/**
629 * icc_set_tag() - set an optional tag on a path
630 * @path: the path we want to tag
631 * @tag: the tag value
632 *
633 * This function allows consumers to append a tag to the requests associated
634 * with a path, so that a different aggregation could be done based on this tag.
635 */
636void icc_set_tag(struct icc_path *path, u32 tag)
637{
638 int i;
639
640 if (!path)
641 return;
642
643 mutex_lock(&icc_lock);
644
645 for (i = 0; i < path->num_nodes; i++)
646 path->reqs[i].tag = tag;
647
648 mutex_unlock(lock: &icc_lock);
649}
650EXPORT_SYMBOL_GPL(icc_set_tag);
651
652/**
653 * icc_get_name() - Get name of the icc path
654 * @path: interconnect path
655 *
656 * This function is used by an interconnect consumer to get the name of the icc
657 * path.
658 *
659 * Returns a valid pointer on success, or NULL otherwise.
660 */
661const char *icc_get_name(struct icc_path *path)
662{
663 if (!path)
664 return NULL;
665
666 return path->name;
667}
668EXPORT_SYMBOL_GPL(icc_get_name);
669
670/**
671 * icc_set_bw() - set bandwidth constraints on an interconnect path
672 * @path: interconnect path
673 * @avg_bw: average bandwidth in kilobytes per second
674 * @peak_bw: peak bandwidth in kilobytes per second
675 *
676 * This function is used by an interconnect consumer to express its own needs
677 * in terms of bandwidth for a previously requested path between two endpoints.
678 * The requests are aggregated and each node is updated accordingly. The entire
679 * path is locked by a mutex to ensure that the set() is completed.
680 * The @path can be NULL when the "interconnects" DT properties is missing,
681 * which will mean that no constraints will be set.
682 *
683 * Returns 0 on success, or an appropriate error code otherwise.
684 */
685int icc_set_bw(struct icc_path *path, u32 avg_bw, u32 peak_bw)
686{
687 struct icc_node *node;
688 u32 old_avg, old_peak;
689 size_t i;
690 int ret;
691
692 if (!path)
693 return 0;
694
695 if (WARN_ON(IS_ERR(path) || !path->num_nodes))
696 return -EINVAL;
697
698 mutex_lock(&icc_bw_lock);
699
700 old_avg = path->reqs[0].avg_bw;
701 old_peak = path->reqs[0].peak_bw;
702
703 for (i = 0; i < path->num_nodes; i++) {
704 node = path->reqs[i].node;
705
706 /* update the consumer request for this path */
707 path->reqs[i].avg_bw = avg_bw;
708 path->reqs[i].peak_bw = peak_bw;
709
710 /* aggregate requests for this node */
711 aggregate_requests(node);
712
713 trace_icc_set_bw(p: path, n: node, i, avg_bw, peak_bw);
714 }
715
716 ret = apply_constraints(path);
717 if (ret) {
718 pr_debug("interconnect: error applying constraints (%d)\n",
719 ret);
720
721 for (i = 0; i < path->num_nodes; i++) {
722 node = path->reqs[i].node;
723 path->reqs[i].avg_bw = old_avg;
724 path->reqs[i].peak_bw = old_peak;
725 aggregate_requests(node);
726 }
727 apply_constraints(path);
728 }
729
730 mutex_unlock(lock: &icc_bw_lock);
731
732 trace_icc_set_bw_end(p: path, ret);
733
734 return ret;
735}
736EXPORT_SYMBOL_GPL(icc_set_bw);
737
738static int __icc_enable(struct icc_path *path, bool enable)
739{
740 int i;
741
742 if (!path)
743 return 0;
744
745 if (WARN_ON(IS_ERR(path) || !path->num_nodes))
746 return -EINVAL;
747
748 mutex_lock(&icc_lock);
749
750 for (i = 0; i < path->num_nodes; i++)
751 path->reqs[i].enabled = enable;
752
753 mutex_unlock(lock: &icc_lock);
754
755 return icc_set_bw(path, path->reqs[0].avg_bw,
756 path->reqs[0].peak_bw);
757}
758
759int icc_enable(struct icc_path *path)
760{
761 return __icc_enable(path, enable: true);
762}
763EXPORT_SYMBOL_GPL(icc_enable);
764
765int icc_disable(struct icc_path *path)
766{
767 return __icc_enable(path, enable: false);
768}
769EXPORT_SYMBOL_GPL(icc_disable);
770
771/**
772 * icc_put() - release the reference to the icc_path
773 * @path: interconnect path
774 *
775 * Use this function to release the constraints on a path when the path is
776 * no longer needed. The constraints will be re-aggregated.
777 */
778void icc_put(struct icc_path *path)
779{
780 struct icc_node *node;
781 size_t i;
782 int ret;
783
784 if (!path || WARN_ON(IS_ERR(path)))
785 return;
786
787 ret = icc_set_bw(path, 0, 0);
788 if (ret)
789 pr_err("%s: error (%d)\n", __func__, ret);
790
791 mutex_lock(&icc_lock);
792 for (i = 0; i < path->num_nodes; i++) {
793 node = path->reqs[i].node;
794 hlist_del(n: &path->reqs[i].req_node);
795 if (!WARN_ON(!node->provider->users))
796 node->provider->users--;
797 }
798 mutex_unlock(lock: &icc_lock);
799
800 kfree_const(x: path->name);
801 kfree(objp: path);
802}
803EXPORT_SYMBOL_GPL(icc_put);
804
805static struct icc_node *icc_node_create_nolock(int id)
806{
807 struct icc_node *node;
808
809 /* check if node already exists */
810 node = node_find(id);
811 if (node)
812 return node;
813
814 node = kzalloc(size: sizeof(*node), GFP_KERNEL);
815 if (!node)
816 return ERR_PTR(error: -ENOMEM);
817
818 id = idr_alloc(&icc_idr, ptr: node, start: id, end: id + 1, GFP_KERNEL);
819 if (id < 0) {
820 WARN(1, "%s: couldn't get idr\n", __func__);
821 kfree(objp: node);
822 return ERR_PTR(error: id);
823 }
824
825 node->id = id;
826
827 return node;
828}
829
830/**
831 * icc_node_create() - create a node
832 * @id: node id
833 *
834 * Return: icc_node pointer on success, or ERR_PTR() on error
835 */
836struct icc_node *icc_node_create(int id)
837{
838 struct icc_node *node;
839
840 mutex_lock(&icc_lock);
841
842 node = icc_node_create_nolock(id);
843
844 mutex_unlock(lock: &icc_lock);
845
846 return node;
847}
848EXPORT_SYMBOL_GPL(icc_node_create);
849
850/**
851 * icc_node_destroy() - destroy a node
852 * @id: node id
853 */
854void icc_node_destroy(int id)
855{
856 struct icc_node *node;
857
858 mutex_lock(&icc_lock);
859
860 node = node_find(id);
861 if (node) {
862 idr_remove(&icc_idr, id: node->id);
863 WARN_ON(!hlist_empty(&node->req_list));
864 }
865
866 mutex_unlock(lock: &icc_lock);
867
868 if (!node)
869 return;
870
871 kfree(objp: node->links);
872 kfree(objp: node);
873}
874EXPORT_SYMBOL_GPL(icc_node_destroy);
875
876/**
877 * icc_link_create() - create a link between two nodes
878 * @node: source node id
879 * @dst_id: destination node id
880 *
881 * Create a link between two nodes. The nodes might belong to different
882 * interconnect providers and the @dst_id node might not exist (if the
883 * provider driver has not probed yet). So just create the @dst_id node
884 * and when the actual provider driver is probed, the rest of the node
885 * data is filled.
886 *
887 * Return: 0 on success, or an error code otherwise
888 */
889int icc_link_create(struct icc_node *node, const int dst_id)
890{
891 struct icc_node *dst;
892 struct icc_node **new;
893 int ret = 0;
894
895 if (!node->provider)
896 return -EINVAL;
897
898 mutex_lock(&icc_lock);
899
900 dst = node_find(id: dst_id);
901 if (!dst) {
902 dst = icc_node_create_nolock(id: dst_id);
903
904 if (IS_ERR(ptr: dst)) {
905 ret = PTR_ERR(ptr: dst);
906 goto out;
907 }
908 }
909
910 new = krealloc(objp: node->links,
911 new_size: (node->num_links + 1) * sizeof(*node->links),
912 GFP_KERNEL);
913 if (!new) {
914 ret = -ENOMEM;
915 goto out;
916 }
917
918 node->links = new;
919 node->links[node->num_links++] = dst;
920
921out:
922 mutex_unlock(lock: &icc_lock);
923
924 return ret;
925}
926EXPORT_SYMBOL_GPL(icc_link_create);
927
928/**
929 * icc_node_add() - add interconnect node to interconnect provider
930 * @node: pointer to the interconnect node
931 * @provider: pointer to the interconnect provider
932 */
933void icc_node_add(struct icc_node *node, struct icc_provider *provider)
934{
935 if (WARN_ON(node->provider))
936 return;
937
938 mutex_lock(&icc_lock);
939 mutex_lock(&icc_bw_lock);
940
941 node->provider = provider;
942 list_add_tail(new: &node->node_list, head: &provider->nodes);
943
944 /* get the initial bandwidth values and sync them with hardware */
945 if (provider->get_bw) {
946 provider->get_bw(node, &node->init_avg, &node->init_peak);
947 } else {
948 node->init_avg = INT_MAX;
949 node->init_peak = INT_MAX;
950 }
951 node->avg_bw = node->init_avg;
952 node->peak_bw = node->init_peak;
953
954 if (node->avg_bw || node->peak_bw) {
955 if (provider->pre_aggregate)
956 provider->pre_aggregate(node);
957
958 if (provider->aggregate)
959 provider->aggregate(node, 0, node->init_avg, node->init_peak,
960 &node->avg_bw, &node->peak_bw);
961 if (provider->set)
962 provider->set(node, node);
963 }
964
965 node->avg_bw = 0;
966 node->peak_bw = 0;
967
968 mutex_unlock(lock: &icc_bw_lock);
969 mutex_unlock(lock: &icc_lock);
970}
971EXPORT_SYMBOL_GPL(icc_node_add);
972
973/**
974 * icc_node_del() - delete interconnect node from interconnect provider
975 * @node: pointer to the interconnect node
976 */
977void icc_node_del(struct icc_node *node)
978{
979 mutex_lock(&icc_lock);
980
981 list_del(entry: &node->node_list);
982
983 mutex_unlock(lock: &icc_lock);
984}
985EXPORT_SYMBOL_GPL(icc_node_del);
986
987/**
988 * icc_nodes_remove() - remove all previously added nodes from provider
989 * @provider: the interconnect provider we are removing nodes from
990 *
991 * Return: 0 on success, or an error code otherwise
992 */
993int icc_nodes_remove(struct icc_provider *provider)
994{
995 struct icc_node *n, *tmp;
996
997 if (WARN_ON(IS_ERR_OR_NULL(provider)))
998 return -EINVAL;
999
1000 list_for_each_entry_safe_reverse(n, tmp, &provider->nodes, node_list) {
1001 icc_node_del(n);
1002 icc_node_destroy(n->id);
1003 }
1004
1005 return 0;
1006}
1007EXPORT_SYMBOL_GPL(icc_nodes_remove);
1008
1009/**
1010 * icc_provider_init() - initialize a new interconnect provider
1011 * @provider: the interconnect provider to initialize
1012 *
1013 * Must be called before adding nodes to the provider.
1014 */
1015void icc_provider_init(struct icc_provider *provider)
1016{
1017 WARN_ON(!provider->set);
1018
1019 INIT_LIST_HEAD(list: &provider->nodes);
1020}
1021EXPORT_SYMBOL_GPL(icc_provider_init);
1022
1023/**
1024 * icc_provider_register() - register a new interconnect provider
1025 * @provider: the interconnect provider to register
1026 *
1027 * Return: 0 on success, or an error code otherwise
1028 */
1029int icc_provider_register(struct icc_provider *provider)
1030{
1031 if (WARN_ON(!provider->xlate && !provider->xlate_extended))
1032 return -EINVAL;
1033
1034 mutex_lock(&icc_lock);
1035 list_add_tail(new: &provider->provider_list, head: &icc_providers);
1036 mutex_unlock(lock: &icc_lock);
1037
1038 dev_dbg(provider->dev, "interconnect provider registered\n");
1039
1040 return 0;
1041}
1042EXPORT_SYMBOL_GPL(icc_provider_register);
1043
1044/**
1045 * icc_provider_deregister() - deregister an interconnect provider
1046 * @provider: the interconnect provider to deregister
1047 */
1048void icc_provider_deregister(struct icc_provider *provider)
1049{
1050 mutex_lock(&icc_lock);
1051 WARN_ON(provider->users);
1052
1053 list_del(entry: &provider->provider_list);
1054 mutex_unlock(lock: &icc_lock);
1055}
1056EXPORT_SYMBOL_GPL(icc_provider_deregister);
1057
1058static const struct of_device_id __maybe_unused ignore_list[] = {
1059 { .compatible = "qcom,sc7180-ipa-virt" },
1060 { .compatible = "qcom,sc8180x-ipa-virt" },
1061 { .compatible = "qcom,sdx55-ipa-virt" },
1062 { .compatible = "qcom,sm8150-ipa-virt" },
1063 { .compatible = "qcom,sm8250-ipa-virt" },
1064 {}
1065};
1066
1067static int of_count_icc_providers(struct device_node *np)
1068{
1069 struct device_node *child;
1070 int count = 0;
1071
1072 for_each_available_child_of_node(np, child) {
1073 if (of_property_read_bool(np: child, propname: "#interconnect-cells") &&
1074 likely(!of_match_node(ignore_list, child)))
1075 count++;
1076 count += of_count_icc_providers(np: child);
1077 }
1078
1079 return count;
1080}
1081
1082void icc_sync_state(struct device *dev)
1083{
1084 struct icc_provider *p;
1085 struct icc_node *n;
1086 static int count;
1087
1088 count++;
1089
1090 if (count < providers_count)
1091 return;
1092
1093 mutex_lock(&icc_lock);
1094 mutex_lock(&icc_bw_lock);
1095 synced_state = true;
1096 list_for_each_entry(p, &icc_providers, provider_list) {
1097 dev_dbg(p->dev, "interconnect provider is in synced state\n");
1098 list_for_each_entry(n, &p->nodes, node_list) {
1099 if (n->init_avg || n->init_peak) {
1100 n->init_avg = 0;
1101 n->init_peak = 0;
1102 aggregate_requests(node: n);
1103 p->set(n, n);
1104 }
1105 }
1106 }
1107 mutex_unlock(lock: &icc_bw_lock);
1108 mutex_unlock(lock: &icc_lock);
1109}
1110EXPORT_SYMBOL_GPL(icc_sync_state);
1111
1112static int __init icc_init(void)
1113{
1114 struct device_node *root;
1115
1116 /* Teach lockdep about lock ordering wrt. shrinker: */
1117 fs_reclaim_acquire(GFP_KERNEL);
1118 might_lock(&icc_bw_lock);
1119 fs_reclaim_release(GFP_KERNEL);
1120
1121 root = of_find_node_by_path(path: "/");
1122
1123 providers_count = of_count_icc_providers(np: root);
1124 of_node_put(node: root);
1125
1126 icc_debugfs_dir = debugfs_create_dir(name: "interconnect", NULL);
1127 debugfs_create_file(name: "interconnect_summary", mode: 0444,
1128 parent: icc_debugfs_dir, NULL, fops: &icc_summary_fops);
1129 debugfs_create_file(name: "interconnect_graph", mode: 0444,
1130 parent: icc_debugfs_dir, NULL, fops: &icc_graph_fops);
1131
1132 icc_debugfs_client_init(icc_dir: icc_debugfs_dir);
1133
1134 return 0;
1135}
1136
1137device_initcall(icc_init);
1138

source code of linux/drivers/interconnect/core.c