1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright (c) 2022, Linaro Limited, All rights reserved. |
4 | * Author: Mike Leach <mike.leach@linaro.org> |
5 | */ |
6 | #include <linux/coresight-pmu.h> |
7 | #include <linux/cpumask.h> |
8 | #include <linux/kernel.h> |
9 | #include <linux/spinlock.h> |
10 | #include <linux/types.h> |
11 | |
12 | #include "coresight-trace-id.h" |
13 | |
14 | /* Default trace ID map. Used on systems that don't require per sink mappings */ |
15 | static struct coresight_trace_id_map id_map_default; |
16 | |
17 | /* maintain a record of the mapping of IDs and pending releases per cpu */ |
18 | static DEFINE_PER_CPU(atomic_t, cpu_id) = ATOMIC_INIT(0); |
19 | static cpumask_t cpu_id_release_pending; |
20 | |
21 | /* perf session active counter */ |
22 | static atomic_t perf_cs_etm_session_active = ATOMIC_INIT(0); |
23 | |
24 | /* lock to protect id_map and cpu data */ |
25 | static DEFINE_SPINLOCK(id_map_lock); |
26 | |
27 | /* #define TRACE_ID_DEBUG 1 */ |
28 | #if defined(TRACE_ID_DEBUG) || defined(CONFIG_COMPILE_TEST) |
29 | |
30 | static void coresight_trace_id_dump_table(struct coresight_trace_id_map *id_map, |
31 | const char *func_name) |
32 | { |
33 | pr_debug("%s id_map::\n" , func_name); |
34 | pr_debug("Used = %*pb\n" , CORESIGHT_TRACE_IDS_MAX, id_map->used_ids); |
35 | pr_debug("Pend = %*pb\n" , CORESIGHT_TRACE_IDS_MAX, id_map->pend_rel_ids); |
36 | } |
37 | #define DUMP_ID_MAP(map) coresight_trace_id_dump_table(map, __func__) |
38 | #define DUMP_ID_CPU(cpu, id) pr_debug("%s called; cpu=%d, id=%d\n", __func__, cpu, id) |
39 | #define DUMP_ID(id) pr_debug("%s called; id=%d\n", __func__, id) |
40 | #define PERF_SESSION(n) pr_debug("%s perf count %d\n", __func__, n) |
41 | #else |
42 | #define DUMP_ID_MAP(map) |
43 | #define DUMP_ID(id) |
44 | #define DUMP_ID_CPU(cpu, id) |
45 | #define PERF_SESSION(n) |
46 | #endif |
47 | |
48 | /* unlocked read of current trace ID value for given CPU */ |
49 | static int _coresight_trace_id_read_cpu_id(int cpu) |
50 | { |
51 | return atomic_read(v: &per_cpu(cpu_id, cpu)); |
52 | } |
53 | |
54 | /* look for next available odd ID, return 0 if none found */ |
55 | static int coresight_trace_id_find_odd_id(struct coresight_trace_id_map *id_map) |
56 | { |
57 | int found_id = 0, bit = 1, next_id; |
58 | |
59 | while ((bit < CORESIGHT_TRACE_ID_RES_TOP) && !found_id) { |
60 | /* |
61 | * bitmap length of CORESIGHT_TRACE_ID_RES_TOP, |
62 | * search from offset `bit`. |
63 | */ |
64 | next_id = find_next_zero_bit(addr: id_map->used_ids, |
65 | CORESIGHT_TRACE_ID_RES_TOP, offset: bit); |
66 | if ((next_id < CORESIGHT_TRACE_ID_RES_TOP) && (next_id & 0x1)) |
67 | found_id = next_id; |
68 | else |
69 | bit = next_id + 1; |
70 | } |
71 | return found_id; |
72 | } |
73 | |
74 | /* |
75 | * Allocate new ID and set in use |
76 | * |
77 | * if @preferred_id is a valid id then try to use that value if available. |
78 | * if @preferred_id is not valid and @prefer_odd_id is true, try for odd id. |
79 | * |
80 | * Otherwise allocate next available ID. |
81 | */ |
82 | static int coresight_trace_id_alloc_new_id(struct coresight_trace_id_map *id_map, |
83 | int preferred_id, bool prefer_odd_id) |
84 | { |
85 | int id = 0; |
86 | |
87 | /* for backwards compatibility, cpu IDs may use preferred value */ |
88 | if (IS_VALID_CS_TRACE_ID(preferred_id) && |
89 | !test_bit(preferred_id, id_map->used_ids)) { |
90 | id = preferred_id; |
91 | goto trace_id_allocated; |
92 | } else if (prefer_odd_id) { |
93 | /* may use odd ids to avoid preferred legacy cpu IDs */ |
94 | id = coresight_trace_id_find_odd_id(id_map); |
95 | if (id) |
96 | goto trace_id_allocated; |
97 | } |
98 | |
99 | /* |
100 | * skip reserved bit 0, look at bitmap length of |
101 | * CORESIGHT_TRACE_ID_RES_TOP from offset of bit 1. |
102 | */ |
103 | id = find_next_zero_bit(addr: id_map->used_ids, CORESIGHT_TRACE_ID_RES_TOP, offset: 1); |
104 | if (id >= CORESIGHT_TRACE_ID_RES_TOP) |
105 | return -EINVAL; |
106 | |
107 | /* mark as used */ |
108 | trace_id_allocated: |
109 | set_bit(nr: id, addr: id_map->used_ids); |
110 | return id; |
111 | } |
112 | |
113 | static void coresight_trace_id_free(int id, struct coresight_trace_id_map *id_map) |
114 | { |
115 | if (WARN(!IS_VALID_CS_TRACE_ID(id), "Invalid Trace ID %d\n" , id)) |
116 | return; |
117 | if (WARN(!test_bit(id, id_map->used_ids), "Freeing unused ID %d\n" , id)) |
118 | return; |
119 | clear_bit(nr: id, addr: id_map->used_ids); |
120 | } |
121 | |
122 | static void coresight_trace_id_set_pend_rel(int id, struct coresight_trace_id_map *id_map) |
123 | { |
124 | if (WARN(!IS_VALID_CS_TRACE_ID(id), "Invalid Trace ID %d\n" , id)) |
125 | return; |
126 | set_bit(nr: id, addr: id_map->pend_rel_ids); |
127 | } |
128 | |
129 | /* |
130 | * release all pending IDs for all current maps & clear CPU associations |
131 | * |
132 | * This currently operates on the default id map, but may be extended to |
133 | * operate on all registered id maps if per sink id maps are used. |
134 | */ |
135 | static void coresight_trace_id_release_all_pending(void) |
136 | { |
137 | struct coresight_trace_id_map *id_map = &id_map_default; |
138 | unsigned long flags; |
139 | int cpu, bit; |
140 | |
141 | spin_lock_irqsave(&id_map_lock, flags); |
142 | for_each_set_bit(bit, id_map->pend_rel_ids, CORESIGHT_TRACE_ID_RES_TOP) { |
143 | clear_bit(nr: bit, addr: id_map->used_ids); |
144 | clear_bit(nr: bit, addr: id_map->pend_rel_ids); |
145 | } |
146 | for_each_cpu(cpu, &cpu_id_release_pending) { |
147 | atomic_set(v: &per_cpu(cpu_id, cpu), i: 0); |
148 | cpumask_clear_cpu(cpu, dstp: &cpu_id_release_pending); |
149 | } |
150 | spin_unlock_irqrestore(lock: &id_map_lock, flags); |
151 | DUMP_ID_MAP(id_map); |
152 | } |
153 | |
154 | static int coresight_trace_id_map_get_cpu_id(int cpu, struct coresight_trace_id_map *id_map) |
155 | { |
156 | unsigned long flags; |
157 | int id; |
158 | |
159 | spin_lock_irqsave(&id_map_lock, flags); |
160 | |
161 | /* check for existing allocation for this CPU */ |
162 | id = _coresight_trace_id_read_cpu_id(cpu); |
163 | if (id) |
164 | goto get_cpu_id_clr_pend; |
165 | |
166 | /* |
167 | * Find a new ID. |
168 | * |
169 | * Use legacy values where possible in the dynamic trace ID allocator to |
170 | * allow older tools to continue working if they are not upgraded at the |
171 | * same time as the kernel drivers. |
172 | * |
173 | * If the generated legacy ID is invalid, or not available then the next |
174 | * available dynamic ID will be used. |
175 | */ |
176 | id = coresight_trace_id_alloc_new_id(id_map, |
177 | CORESIGHT_LEGACY_CPU_TRACE_ID(cpu), |
178 | prefer_odd_id: false); |
179 | if (!IS_VALID_CS_TRACE_ID(id)) |
180 | goto get_cpu_id_out_unlock; |
181 | |
182 | /* allocate the new id to the cpu */ |
183 | atomic_set(v: &per_cpu(cpu_id, cpu), i: id); |
184 | |
185 | get_cpu_id_clr_pend: |
186 | /* we are (re)using this ID - so ensure it is not marked for release */ |
187 | cpumask_clear_cpu(cpu, dstp: &cpu_id_release_pending); |
188 | clear_bit(nr: id, addr: id_map->pend_rel_ids); |
189 | |
190 | get_cpu_id_out_unlock: |
191 | spin_unlock_irqrestore(lock: &id_map_lock, flags); |
192 | |
193 | DUMP_ID_CPU(cpu, id); |
194 | DUMP_ID_MAP(id_map); |
195 | return id; |
196 | } |
197 | |
198 | static void coresight_trace_id_map_put_cpu_id(int cpu, struct coresight_trace_id_map *id_map) |
199 | { |
200 | unsigned long flags; |
201 | int id; |
202 | |
203 | /* check for existing allocation for this CPU */ |
204 | id = _coresight_trace_id_read_cpu_id(cpu); |
205 | if (!id) |
206 | return; |
207 | |
208 | spin_lock_irqsave(&id_map_lock, flags); |
209 | |
210 | if (atomic_read(v: &perf_cs_etm_session_active)) { |
211 | /* set release at pending if perf still active */ |
212 | coresight_trace_id_set_pend_rel(id, id_map); |
213 | cpumask_set_cpu(cpu, dstp: &cpu_id_release_pending); |
214 | } else { |
215 | /* otherwise clear id */ |
216 | coresight_trace_id_free(id, id_map); |
217 | atomic_set(v: &per_cpu(cpu_id, cpu), i: 0); |
218 | } |
219 | |
220 | spin_unlock_irqrestore(lock: &id_map_lock, flags); |
221 | DUMP_ID_CPU(cpu, id); |
222 | DUMP_ID_MAP(id_map); |
223 | } |
224 | |
225 | static int coresight_trace_id_map_get_system_id(struct coresight_trace_id_map *id_map) |
226 | { |
227 | unsigned long flags; |
228 | int id; |
229 | |
230 | spin_lock_irqsave(&id_map_lock, flags); |
231 | /* prefer odd IDs for system components to avoid legacy CPU IDS */ |
232 | id = coresight_trace_id_alloc_new_id(id_map, preferred_id: 0, prefer_odd_id: true); |
233 | spin_unlock_irqrestore(lock: &id_map_lock, flags); |
234 | |
235 | DUMP_ID(id); |
236 | DUMP_ID_MAP(id_map); |
237 | return id; |
238 | } |
239 | |
240 | static void coresight_trace_id_map_put_system_id(struct coresight_trace_id_map *id_map, int id) |
241 | { |
242 | unsigned long flags; |
243 | |
244 | spin_lock_irqsave(&id_map_lock, flags); |
245 | coresight_trace_id_free(id, id_map); |
246 | spin_unlock_irqrestore(lock: &id_map_lock, flags); |
247 | |
248 | DUMP_ID(id); |
249 | DUMP_ID_MAP(id_map); |
250 | } |
251 | |
252 | /* API functions */ |
253 | |
254 | int coresight_trace_id_get_cpu_id(int cpu) |
255 | { |
256 | return coresight_trace_id_map_get_cpu_id(cpu, id_map: &id_map_default); |
257 | } |
258 | EXPORT_SYMBOL_GPL(coresight_trace_id_get_cpu_id); |
259 | |
260 | void coresight_trace_id_put_cpu_id(int cpu) |
261 | { |
262 | coresight_trace_id_map_put_cpu_id(cpu, id_map: &id_map_default); |
263 | } |
264 | EXPORT_SYMBOL_GPL(coresight_trace_id_put_cpu_id); |
265 | |
266 | int coresight_trace_id_read_cpu_id(int cpu) |
267 | { |
268 | return _coresight_trace_id_read_cpu_id(cpu); |
269 | } |
270 | EXPORT_SYMBOL_GPL(coresight_trace_id_read_cpu_id); |
271 | |
272 | int coresight_trace_id_get_system_id(void) |
273 | { |
274 | return coresight_trace_id_map_get_system_id(id_map: &id_map_default); |
275 | } |
276 | EXPORT_SYMBOL_GPL(coresight_trace_id_get_system_id); |
277 | |
278 | void coresight_trace_id_put_system_id(int id) |
279 | { |
280 | coresight_trace_id_map_put_system_id(id_map: &id_map_default, id); |
281 | } |
282 | EXPORT_SYMBOL_GPL(coresight_trace_id_put_system_id); |
283 | |
284 | void coresight_trace_id_perf_start(void) |
285 | { |
286 | atomic_inc(v: &perf_cs_etm_session_active); |
287 | PERF_SESSION(atomic_read(&perf_cs_etm_session_active)); |
288 | } |
289 | EXPORT_SYMBOL_GPL(coresight_trace_id_perf_start); |
290 | |
291 | void coresight_trace_id_perf_stop(void) |
292 | { |
293 | if (!atomic_dec_return(v: &perf_cs_etm_session_active)) |
294 | coresight_trace_id_release_all_pending(); |
295 | PERF_SESSION(atomic_read(&perf_cs_etm_session_active)); |
296 | } |
297 | EXPORT_SYMBOL_GPL(coresight_trace_id_perf_stop); |
298 | |