1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright (c) 2014, The Linux Foundation. All rights reserved. |
4 | */ |
5 | |
6 | #include <linux/acpi.h> |
7 | #include <linux/bitops.h> |
8 | #include <linux/kernel.h> |
9 | #include <linux/moduleparam.h> |
10 | #include <linux/init.h> |
11 | #include <linux/types.h> |
12 | #include <linux/device.h> |
13 | #include <linux/io.h> |
14 | #include <linux/err.h> |
15 | #include <linux/fs.h> |
16 | #include <linux/slab.h> |
17 | #include <linux/delay.h> |
18 | #include <linux/smp.h> |
19 | #include <linux/sysfs.h> |
20 | #include <linux/stat.h> |
21 | #include <linux/clk.h> |
22 | #include <linux/cpu.h> |
23 | #include <linux/cpu_pm.h> |
24 | #include <linux/coresight.h> |
25 | #include <linux/coresight-pmu.h> |
26 | #include <linux/pm_wakeup.h> |
27 | #include <linux/amba/bus.h> |
28 | #include <linux/seq_file.h> |
29 | #include <linux/uaccess.h> |
30 | #include <linux/perf_event.h> |
31 | #include <linux/platform_device.h> |
32 | #include <linux/pm_runtime.h> |
33 | #include <linux/property.h> |
34 | #include <linux/clk/clk-conf.h> |
35 | |
36 | #include <asm/barrier.h> |
37 | #include <asm/sections.h> |
38 | #include <asm/sysreg.h> |
39 | #include <asm/local.h> |
40 | #include <asm/virt.h> |
41 | |
42 | #include "coresight-etm4x.h" |
43 | #include "coresight-etm-perf.h" |
44 | #include "coresight-etm4x-cfg.h" |
45 | #include "coresight-self-hosted-trace.h" |
46 | #include "coresight-syscfg.h" |
47 | #include "coresight-trace-id.h" |
48 | |
49 | static int boot_enable; |
50 | module_param(boot_enable, int, 0444); |
51 | MODULE_PARM_DESC(boot_enable, "Enable tracing on boot" ); |
52 | |
53 | #define PARAM_PM_SAVE_FIRMWARE 0 /* save self-hosted state as per firmware */ |
54 | #define PARAM_PM_SAVE_NEVER 1 /* never save any state */ |
55 | #define PARAM_PM_SAVE_SELF_HOSTED 2 /* save self-hosted state only */ |
56 | |
57 | static int pm_save_enable = PARAM_PM_SAVE_FIRMWARE; |
58 | module_param(pm_save_enable, int, 0444); |
59 | MODULE_PARM_DESC(pm_save_enable, |
60 | "Save/restore state on power down: 1 = never, 2 = self-hosted" ); |
61 | |
62 | static struct etmv4_drvdata *etmdrvdata[NR_CPUS]; |
63 | static void etm4_set_default_config(struct etmv4_config *config); |
64 | static int etm4_set_event_filters(struct etmv4_drvdata *drvdata, |
65 | struct perf_event *event); |
66 | static u64 etm4_get_access_type(struct etmv4_config *config); |
67 | |
68 | static enum cpuhp_state hp_online; |
69 | |
70 | struct etm4_init_arg { |
71 | struct device *dev; |
72 | struct csdev_access *csa; |
73 | }; |
74 | |
75 | static DEFINE_PER_CPU(struct etm4_init_arg *, delayed_probe); |
76 | static int etm4_probe_cpu(unsigned int cpu); |
77 | |
78 | /* |
79 | * Check if TRCSSPCICRn(i) is implemented for a given instance. |
80 | * |
81 | * TRCSSPCICRn is implemented only if : |
82 | * TRCSSPCICR<n> is present only if all of the following are true: |
83 | * TRCIDR4.NUMSSCC > n. |
84 | * TRCIDR4.NUMPC > 0b0000 . |
85 | * TRCSSCSR<n>.PC == 0b1 |
86 | */ |
87 | static inline bool etm4x_sspcicrn_present(struct etmv4_drvdata *drvdata, int n) |
88 | { |
89 | return (n < drvdata->nr_ss_cmp) && |
90 | drvdata->nr_pe && |
91 | (drvdata->config.ss_status[n] & TRCSSCSRn_PC); |
92 | } |
93 | |
94 | u64 etm4x_sysreg_read(u32 offset, bool _relaxed, bool _64bit) |
95 | { |
96 | u64 res = 0; |
97 | |
98 | switch (offset) { |
99 | ETM4x_READ_SYSREG_CASES(res) |
100 | default : |
101 | pr_warn_ratelimited("etm4x: trying to read unsupported register @%x\n" , |
102 | offset); |
103 | } |
104 | |
105 | if (!_relaxed) |
106 | __io_ar(res); /* Imitate the !relaxed I/O helpers */ |
107 | |
108 | return res; |
109 | } |
110 | |
111 | void etm4x_sysreg_write(u64 val, u32 offset, bool _relaxed, bool _64bit) |
112 | { |
113 | if (!_relaxed) |
114 | __io_bw(); /* Imitate the !relaxed I/O helpers */ |
115 | if (!_64bit) |
116 | val &= GENMASK(31, 0); |
117 | |
118 | switch (offset) { |
119 | ETM4x_WRITE_SYSREG_CASES(val) |
120 | default : |
121 | pr_warn_ratelimited("etm4x: trying to write to unsupported register @%x\n" , |
122 | offset); |
123 | } |
124 | } |
125 | |
126 | static u64 ete_sysreg_read(u32 offset, bool _relaxed, bool _64bit) |
127 | { |
128 | u64 res = 0; |
129 | |
130 | switch (offset) { |
131 | ETE_READ_CASES(res) |
132 | default : |
133 | pr_warn_ratelimited("ete: trying to read unsupported register @%x\n" , |
134 | offset); |
135 | } |
136 | |
137 | if (!_relaxed) |
138 | __io_ar(res); /* Imitate the !relaxed I/O helpers */ |
139 | |
140 | return res; |
141 | } |
142 | |
143 | static void ete_sysreg_write(u64 val, u32 offset, bool _relaxed, bool _64bit) |
144 | { |
145 | if (!_relaxed) |
146 | __io_bw(); /* Imitate the !relaxed I/O helpers */ |
147 | if (!_64bit) |
148 | val &= GENMASK(31, 0); |
149 | |
150 | switch (offset) { |
151 | ETE_WRITE_CASES(val) |
152 | default : |
153 | pr_warn_ratelimited("ete: trying to write to unsupported register @%x\n" , |
154 | offset); |
155 | } |
156 | } |
157 | |
158 | static void etm_detect_os_lock(struct etmv4_drvdata *drvdata, |
159 | struct csdev_access *csa) |
160 | { |
161 | u32 oslsr = etm4x_relaxed_read32(csa, TRCOSLSR); |
162 | |
163 | drvdata->os_lock_model = ETM_OSLSR_OSLM(oslsr); |
164 | } |
165 | |
166 | static void etm_write_os_lock(struct etmv4_drvdata *drvdata, |
167 | struct csdev_access *csa, u32 val) |
168 | { |
169 | val = !!val; |
170 | |
171 | switch (drvdata->os_lock_model) { |
172 | case ETM_OSLOCK_PRESENT: |
173 | etm4x_relaxed_write32(csa, val, TRCOSLAR); |
174 | break; |
175 | case ETM_OSLOCK_PE: |
176 | write_sysreg_s(val, SYS_OSLAR_EL1); |
177 | break; |
178 | default: |
179 | pr_warn_once("CPU%d: Unsupported Trace OSLock model: %x\n" , |
180 | smp_processor_id(), drvdata->os_lock_model); |
181 | fallthrough; |
182 | case ETM_OSLOCK_NI: |
183 | return; |
184 | } |
185 | isb(); |
186 | } |
187 | |
188 | static inline void etm4_os_unlock_csa(struct etmv4_drvdata *drvdata, |
189 | struct csdev_access *csa) |
190 | { |
191 | WARN_ON(drvdata->cpu != smp_processor_id()); |
192 | |
193 | /* Writing 0 to OS Lock unlocks the trace unit registers */ |
194 | etm_write_os_lock(drvdata, csa, val: 0x0); |
195 | drvdata->os_unlock = true; |
196 | } |
197 | |
198 | static void etm4_os_unlock(struct etmv4_drvdata *drvdata) |
199 | { |
200 | if (!WARN_ON(!drvdata->csdev)) |
201 | etm4_os_unlock_csa(drvdata, csa: &drvdata->csdev->access); |
202 | } |
203 | |
204 | static void etm4_os_lock(struct etmv4_drvdata *drvdata) |
205 | { |
206 | if (WARN_ON(!drvdata->csdev)) |
207 | return; |
208 | /* Writing 0x1 to OS Lock locks the trace registers */ |
209 | etm_write_os_lock(drvdata, csa: &drvdata->csdev->access, val: 0x1); |
210 | drvdata->os_unlock = false; |
211 | } |
212 | |
213 | static void etm4_cs_lock(struct etmv4_drvdata *drvdata, |
214 | struct csdev_access *csa) |
215 | { |
216 | /* Software Lock is only accessible via memory mapped interface */ |
217 | if (csa->io_mem) |
218 | CS_LOCK(addr: csa->base); |
219 | } |
220 | |
221 | static void etm4_cs_unlock(struct etmv4_drvdata *drvdata, |
222 | struct csdev_access *csa) |
223 | { |
224 | if (csa->io_mem) |
225 | CS_UNLOCK(addr: csa->base); |
226 | } |
227 | |
228 | static int etm4_cpu_id(struct coresight_device *csdev) |
229 | { |
230 | struct etmv4_drvdata *drvdata = dev_get_drvdata(dev: csdev->dev.parent); |
231 | |
232 | return drvdata->cpu; |
233 | } |
234 | |
235 | int etm4_read_alloc_trace_id(struct etmv4_drvdata *drvdata) |
236 | { |
237 | int trace_id; |
238 | |
239 | /* |
240 | * This will allocate a trace ID to the cpu, |
241 | * or return the one currently allocated. |
242 | * The trace id function has its own lock |
243 | */ |
244 | trace_id = coresight_trace_id_get_cpu_id(cpu: drvdata->cpu); |
245 | if (IS_VALID_CS_TRACE_ID(trace_id)) |
246 | drvdata->trcid = (u8)trace_id; |
247 | else |
248 | dev_err(&drvdata->csdev->dev, |
249 | "Failed to allocate trace ID for %s on CPU%d\n" , |
250 | dev_name(&drvdata->csdev->dev), drvdata->cpu); |
251 | return trace_id; |
252 | } |
253 | |
254 | void etm4_release_trace_id(struct etmv4_drvdata *drvdata) |
255 | { |
256 | coresight_trace_id_put_cpu_id(cpu: drvdata->cpu); |
257 | } |
258 | |
259 | struct etm4_enable_arg { |
260 | struct etmv4_drvdata *drvdata; |
261 | int rc; |
262 | }; |
263 | |
264 | /* |
265 | * etm4x_prohibit_trace - Prohibit the CPU from tracing at all ELs. |
266 | * When the CPU supports FEAT_TRF, we could move the ETM to a trace |
267 | * prohibited state by filtering the Exception levels via TRFCR_EL1. |
268 | */ |
269 | static void etm4x_prohibit_trace(struct etmv4_drvdata *drvdata) |
270 | { |
271 | /* If the CPU doesn't support FEAT_TRF, nothing to do */ |
272 | if (!drvdata->trfcr) |
273 | return; |
274 | cpu_prohibit_trace(); |
275 | } |
276 | |
277 | /* |
278 | * etm4x_allow_trace - Allow CPU tracing in the respective ELs, |
279 | * as configured by the drvdata->config.mode for the current |
280 | * session. Even though we have TRCVICTLR bits to filter the |
281 | * trace in the ELs, it doesn't prevent the ETM from generating |
282 | * a packet (e.g, TraceInfo) that might contain the addresses from |
283 | * the excluded levels. Thus we use the additional controls provided |
284 | * via the Trace Filtering controls (FEAT_TRF) to make sure no trace |
285 | * is generated for the excluded ELs. |
286 | */ |
287 | static void etm4x_allow_trace(struct etmv4_drvdata *drvdata) |
288 | { |
289 | u64 trfcr = drvdata->trfcr; |
290 | |
291 | /* If the CPU doesn't support FEAT_TRF, nothing to do */ |
292 | if (!trfcr) |
293 | return; |
294 | |
295 | if (drvdata->config.mode & ETM_MODE_EXCL_KERN) |
296 | trfcr &= ~TRFCR_ELx_ExTRE; |
297 | if (drvdata->config.mode & ETM_MODE_EXCL_USER) |
298 | trfcr &= ~TRFCR_ELx_E0TRE; |
299 | |
300 | write_trfcr(val: trfcr); |
301 | } |
302 | |
303 | #ifdef CONFIG_ETM4X_IMPDEF_FEATURE |
304 | |
305 | #define HISI_HIP08_AMBA_ID 0x000b6d01 |
306 | #define ETM4_AMBA_MASK 0xfffff |
307 | #define HISI_HIP08_CORE_COMMIT_MASK 0x3000 |
308 | #define HISI_HIP08_CORE_COMMIT_SHIFT 12 |
309 | #define HISI_HIP08_CORE_COMMIT_FULL 0b00 |
310 | #define HISI_HIP08_CORE_COMMIT_LVL_1 0b01 |
311 | #define HISI_HIP08_CORE_COMMIT_REG sys_reg(3, 1, 15, 2, 5) |
312 | |
313 | struct etm4_arch_features { |
314 | void (*arch_callback)(bool enable); |
315 | }; |
316 | |
317 | static bool etm4_hisi_match_pid(unsigned int id) |
318 | { |
319 | return (id & ETM4_AMBA_MASK) == HISI_HIP08_AMBA_ID; |
320 | } |
321 | |
322 | static void etm4_hisi_config_core_commit(bool enable) |
323 | { |
324 | u8 commit = enable ? HISI_HIP08_CORE_COMMIT_LVL_1 : |
325 | HISI_HIP08_CORE_COMMIT_FULL; |
326 | u64 val; |
327 | |
328 | /* |
329 | * bit 12 and 13 of HISI_HIP08_CORE_COMMIT_REG are used together |
330 | * to set core-commit, 2'b00 means cpu is at full speed, 2'b01, |
331 | * 2'b10, 2'b11 mean reduce pipeline speed, and 2'b01 means level-1 |
332 | * speed(minimun value). So bit 12 and 13 should be cleared together. |
333 | */ |
334 | val = read_sysreg_s(HISI_HIP08_CORE_COMMIT_REG); |
335 | val &= ~HISI_HIP08_CORE_COMMIT_MASK; |
336 | val |= commit << HISI_HIP08_CORE_COMMIT_SHIFT; |
337 | write_sysreg_s(val, HISI_HIP08_CORE_COMMIT_REG); |
338 | } |
339 | |
340 | static struct etm4_arch_features etm4_features[] = { |
341 | [ETM4_IMPDEF_HISI_CORE_COMMIT] = { |
342 | .arch_callback = etm4_hisi_config_core_commit, |
343 | }, |
344 | {}, |
345 | }; |
346 | |
347 | static void etm4_enable_arch_specific(struct etmv4_drvdata *drvdata) |
348 | { |
349 | struct etm4_arch_features *ftr; |
350 | int bit; |
351 | |
352 | for_each_set_bit(bit, drvdata->arch_features, ETM4_IMPDEF_FEATURE_MAX) { |
353 | ftr = &etm4_features[bit]; |
354 | |
355 | if (ftr->arch_callback) |
356 | ftr->arch_callback(true); |
357 | } |
358 | } |
359 | |
360 | static void etm4_disable_arch_specific(struct etmv4_drvdata *drvdata) |
361 | { |
362 | struct etm4_arch_features *ftr; |
363 | int bit; |
364 | |
365 | for_each_set_bit(bit, drvdata->arch_features, ETM4_IMPDEF_FEATURE_MAX) { |
366 | ftr = &etm4_features[bit]; |
367 | |
368 | if (ftr->arch_callback) |
369 | ftr->arch_callback(false); |
370 | } |
371 | } |
372 | |
373 | static void etm4_check_arch_features(struct etmv4_drvdata *drvdata, |
374 | struct csdev_access *csa) |
375 | { |
376 | /* |
377 | * TRCPIDR* registers are not required for ETMs with system |
378 | * instructions. They must be identified by the MIDR+REVIDRs. |
379 | * Skip the TRCPID checks for now. |
380 | */ |
381 | if (!csa->io_mem) |
382 | return; |
383 | |
384 | if (etm4_hisi_match_pid(coresight_get_pid(csa))) |
385 | set_bit(ETM4_IMPDEF_HISI_CORE_COMMIT, drvdata->arch_features); |
386 | } |
387 | #else |
388 | static void etm4_enable_arch_specific(struct etmv4_drvdata *drvdata) |
389 | { |
390 | } |
391 | |
392 | static void etm4_disable_arch_specific(struct etmv4_drvdata *drvdata) |
393 | { |
394 | } |
395 | |
396 | static void etm4_check_arch_features(struct etmv4_drvdata *drvdata, |
397 | struct csdev_access *csa) |
398 | { |
399 | } |
400 | #endif /* CONFIG_ETM4X_IMPDEF_FEATURE */ |
401 | |
402 | static int etm4_enable_hw(struct etmv4_drvdata *drvdata) |
403 | { |
404 | int i, rc; |
405 | struct etmv4_config *config = &drvdata->config; |
406 | struct coresight_device *csdev = drvdata->csdev; |
407 | struct device *etm_dev = &csdev->dev; |
408 | struct csdev_access *csa = &csdev->access; |
409 | |
410 | |
411 | etm4_cs_unlock(drvdata, csa); |
412 | etm4_enable_arch_specific(drvdata); |
413 | |
414 | etm4_os_unlock(drvdata); |
415 | |
416 | rc = coresight_claim_device_unlocked(csdev); |
417 | if (rc) |
418 | goto done; |
419 | |
420 | /* Disable the trace unit before programming trace registers */ |
421 | etm4x_relaxed_write32(csa, 0, TRCPRGCTLR); |
422 | |
423 | /* |
424 | * If we use system instructions, we need to synchronize the |
425 | * write to the TRCPRGCTLR, before accessing the TRCSTATR. |
426 | * See ARM IHI0064F, section |
427 | * "4.3.7 Synchronization of register updates" |
428 | */ |
429 | if (!csa->io_mem) |
430 | isb(); |
431 | |
432 | /* wait for TRCSTATR.IDLE to go up */ |
433 | if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, value: 1)) |
434 | dev_err(etm_dev, |
435 | "timeout while waiting for Idle Trace Status\n" ); |
436 | if (drvdata->nr_pe) |
437 | etm4x_relaxed_write32(csa, config->pe_sel, TRCPROCSELR); |
438 | etm4x_relaxed_write32(csa, config->cfg, TRCCONFIGR); |
439 | /* nothing specific implemented */ |
440 | etm4x_relaxed_write32(csa, 0x0, TRCAUXCTLR); |
441 | etm4x_relaxed_write32(csa, config->eventctrl0, TRCEVENTCTL0R); |
442 | etm4x_relaxed_write32(csa, config->eventctrl1, TRCEVENTCTL1R); |
443 | if (drvdata->stallctl) |
444 | etm4x_relaxed_write32(csa, config->stall_ctrl, TRCSTALLCTLR); |
445 | etm4x_relaxed_write32(csa, config->ts_ctrl, TRCTSCTLR); |
446 | etm4x_relaxed_write32(csa, config->syncfreq, TRCSYNCPR); |
447 | etm4x_relaxed_write32(csa, config->ccctlr, TRCCCCTLR); |
448 | etm4x_relaxed_write32(csa, config->bb_ctrl, TRCBBCTLR); |
449 | etm4x_relaxed_write32(csa, drvdata->trcid, TRCTRACEIDR); |
450 | etm4x_relaxed_write32(csa, config->vinst_ctrl, TRCVICTLR); |
451 | etm4x_relaxed_write32(csa, config->viiectlr, TRCVIIECTLR); |
452 | etm4x_relaxed_write32(csa, config->vissctlr, TRCVISSCTLR); |
453 | if (drvdata->nr_pe_cmp) |
454 | etm4x_relaxed_write32(csa, config->vipcssctlr, TRCVIPCSSCTLR); |
455 | for (i = 0; i < drvdata->nrseqstate - 1; i++) |
456 | etm4x_relaxed_write32(csa, config->seq_ctrl[i], TRCSEQEVRn(i)); |
457 | if (drvdata->nrseqstate) { |
458 | etm4x_relaxed_write32(csa, config->seq_rst, TRCSEQRSTEVR); |
459 | etm4x_relaxed_write32(csa, config->seq_state, TRCSEQSTR); |
460 | } |
461 | etm4x_relaxed_write32(csa, config->ext_inp, TRCEXTINSELR); |
462 | for (i = 0; i < drvdata->nr_cntr; i++) { |
463 | etm4x_relaxed_write32(csa, config->cntrldvr[i], TRCCNTRLDVRn(i)); |
464 | etm4x_relaxed_write32(csa, config->cntr_ctrl[i], TRCCNTCTLRn(i)); |
465 | etm4x_relaxed_write32(csa, config->cntr_val[i], TRCCNTVRn(i)); |
466 | } |
467 | |
468 | /* |
469 | * Resource selector pair 0 is always implemented and reserved. As |
470 | * such start at 2. |
471 | */ |
472 | for (i = 2; i < drvdata->nr_resource * 2; i++) |
473 | etm4x_relaxed_write32(csa, config->res_ctrl[i], TRCRSCTLRn(i)); |
474 | |
475 | for (i = 0; i < drvdata->nr_ss_cmp; i++) { |
476 | /* always clear status bit on restart if using single-shot */ |
477 | if (config->ss_ctrl[i] || config->ss_pe_cmp[i]) |
478 | config->ss_status[i] &= ~TRCSSCSRn_STATUS; |
479 | etm4x_relaxed_write32(csa, config->ss_ctrl[i], TRCSSCCRn(i)); |
480 | etm4x_relaxed_write32(csa, config->ss_status[i], TRCSSCSRn(i)); |
481 | if (etm4x_sspcicrn_present(drvdata, n: i)) |
482 | etm4x_relaxed_write32(csa, config->ss_pe_cmp[i], TRCSSPCICRn(i)); |
483 | } |
484 | for (i = 0; i < drvdata->nr_addr_cmp * 2; i++) { |
485 | etm4x_relaxed_write64(csa, config->addr_val[i], TRCACVRn(i)); |
486 | etm4x_relaxed_write64(csa, config->addr_acc[i], TRCACATRn(i)); |
487 | } |
488 | for (i = 0; i < drvdata->numcidc; i++) |
489 | etm4x_relaxed_write64(csa, config->ctxid_pid[i], TRCCIDCVRn(i)); |
490 | etm4x_relaxed_write32(csa, config->ctxid_mask0, TRCCIDCCTLR0); |
491 | if (drvdata->numcidc > 4) |
492 | etm4x_relaxed_write32(csa, config->ctxid_mask1, TRCCIDCCTLR1); |
493 | |
494 | for (i = 0; i < drvdata->numvmidc; i++) |
495 | etm4x_relaxed_write64(csa, config->vmid_val[i], TRCVMIDCVRn(i)); |
496 | etm4x_relaxed_write32(csa, config->vmid_mask0, TRCVMIDCCTLR0); |
497 | if (drvdata->numvmidc > 4) |
498 | etm4x_relaxed_write32(csa, config->vmid_mask1, TRCVMIDCCTLR1); |
499 | |
500 | if (!drvdata->skip_power_up) { |
501 | u32 trcpdcr = etm4x_relaxed_read32(csa, TRCPDCR); |
502 | |
503 | /* |
504 | * Request to keep the trace unit powered and also |
505 | * emulation of powerdown |
506 | */ |
507 | etm4x_relaxed_write32(csa, trcpdcr | TRCPDCR_PU, TRCPDCR); |
508 | } |
509 | |
510 | /* |
511 | * ETE mandates that the TRCRSR is written to before |
512 | * enabling it. |
513 | */ |
514 | if (etm4x_is_ete(drvdata)) |
515 | etm4x_relaxed_write32(csa, TRCRSR_TA, TRCRSR); |
516 | |
517 | etm4x_allow_trace(drvdata); |
518 | /* Enable the trace unit */ |
519 | etm4x_relaxed_write32(csa, 1, TRCPRGCTLR); |
520 | |
521 | /* Synchronize the register updates for sysreg access */ |
522 | if (!csa->io_mem) |
523 | isb(); |
524 | |
525 | /* wait for TRCSTATR.IDLE to go back down to '0' */ |
526 | if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, value: 0)) |
527 | dev_err(etm_dev, |
528 | "timeout while waiting for Idle Trace Status\n" ); |
529 | |
530 | /* |
531 | * As recommended by section 4.3.7 ("Synchronization when using the |
532 | * memory-mapped interface") of ARM IHI 0064D |
533 | */ |
534 | dsb(sy); |
535 | isb(); |
536 | |
537 | done: |
538 | etm4_cs_lock(drvdata, csa); |
539 | |
540 | dev_dbg(etm_dev, "cpu: %d enable smp call done: %d\n" , |
541 | drvdata->cpu, rc); |
542 | return rc; |
543 | } |
544 | |
545 | static void etm4_enable_hw_smp_call(void *info) |
546 | { |
547 | struct etm4_enable_arg *arg = info; |
548 | |
549 | if (WARN_ON(!arg)) |
550 | return; |
551 | arg->rc = etm4_enable_hw(drvdata: arg->drvdata); |
552 | } |
553 | |
554 | /* |
555 | * The goal of function etm4_config_timestamp_event() is to configure a |
556 | * counter that will tell the tracer to emit a timestamp packet when it |
557 | * reaches zero. This is done in order to get a more fine grained idea |
558 | * of when instructions are executed so that they can be correlated |
559 | * with execution on other CPUs. |
560 | * |
561 | * To do this the counter itself is configured to self reload and |
562 | * TRCRSCTLR1 (always true) used to get the counter to decrement. From |
563 | * there a resource selector is configured with the counter and the |
564 | * timestamp control register to use the resource selector to trigger the |
565 | * event that will insert a timestamp packet in the stream. |
566 | */ |
567 | static int etm4_config_timestamp_event(struct etmv4_drvdata *drvdata) |
568 | { |
569 | int ctridx, ret = -EINVAL; |
570 | int counter, rselector; |
571 | u32 val = 0; |
572 | struct etmv4_config *config = &drvdata->config; |
573 | |
574 | /* No point in trying if we don't have at least one counter */ |
575 | if (!drvdata->nr_cntr) |
576 | goto out; |
577 | |
578 | /* Find a counter that hasn't been initialised */ |
579 | for (ctridx = 0; ctridx < drvdata->nr_cntr; ctridx++) |
580 | if (config->cntr_val[ctridx] == 0) |
581 | break; |
582 | |
583 | /* All the counters have been configured already, bail out */ |
584 | if (ctridx == drvdata->nr_cntr) { |
585 | pr_debug("%s: no available counter found\n" , __func__); |
586 | ret = -ENOSPC; |
587 | goto out; |
588 | } |
589 | |
590 | /* |
591 | * Searching for an available resource selector to use, starting at |
592 | * '2' since every implementation has at least 2 resource selector. |
593 | * ETMIDR4 gives the number of resource selector _pairs_, |
594 | * hence multiply by 2. |
595 | */ |
596 | for (rselector = 2; rselector < drvdata->nr_resource * 2; rselector++) |
597 | if (!config->res_ctrl[rselector]) |
598 | break; |
599 | |
600 | if (rselector == drvdata->nr_resource * 2) { |
601 | pr_debug("%s: no available resource selector found\n" , |
602 | __func__); |
603 | ret = -ENOSPC; |
604 | goto out; |
605 | } |
606 | |
607 | /* Remember what counter we used */ |
608 | counter = 1 << ctridx; |
609 | |
610 | /* |
611 | * Initialise original and reload counter value to the smallest |
612 | * possible value in order to get as much precision as we can. |
613 | */ |
614 | config->cntr_val[ctridx] = 1; |
615 | config->cntrldvr[ctridx] = 1; |
616 | |
617 | /* Set the trace counter control register */ |
618 | val = 0x1 << 16 | /* Bit 16, reload counter automatically */ |
619 | 0x0 << 7 | /* Select single resource selector */ |
620 | 0x1; /* Resource selector 1, i.e always true */ |
621 | |
622 | config->cntr_ctrl[ctridx] = val; |
623 | |
624 | val = 0x2 << 16 | /* Group 0b0010 - Counter and sequencers */ |
625 | counter << 0; /* Counter to use */ |
626 | |
627 | config->res_ctrl[rselector] = val; |
628 | |
629 | val = 0x0 << 7 | /* Select single resource selector */ |
630 | rselector; /* Resource selector */ |
631 | |
632 | config->ts_ctrl = val; |
633 | |
634 | ret = 0; |
635 | out: |
636 | return ret; |
637 | } |
638 | |
639 | static int etm4_parse_event_config(struct coresight_device *csdev, |
640 | struct perf_event *event) |
641 | { |
642 | int ret = 0; |
643 | struct etmv4_drvdata *drvdata = dev_get_drvdata(dev: csdev->dev.parent); |
644 | struct etmv4_config *config = &drvdata->config; |
645 | struct perf_event_attr *attr = &event->attr; |
646 | unsigned long cfg_hash; |
647 | int preset, cc_threshold; |
648 | |
649 | /* Clear configuration from previous run */ |
650 | memset(config, 0, sizeof(struct etmv4_config)); |
651 | |
652 | if (attr->exclude_kernel) |
653 | config->mode = ETM_MODE_EXCL_KERN; |
654 | |
655 | if (attr->exclude_user) |
656 | config->mode = ETM_MODE_EXCL_USER; |
657 | |
658 | /* Always start from the default config */ |
659 | etm4_set_default_config(config); |
660 | |
661 | /* Configure filters specified on the perf cmd line, if any. */ |
662 | ret = etm4_set_event_filters(drvdata, event); |
663 | if (ret) |
664 | goto out; |
665 | |
666 | /* Go from generic option to ETMv4 specifics */ |
667 | if (attr->config & BIT(ETM_OPT_CYCACC)) { |
668 | config->cfg |= TRCCONFIGR_CCI; |
669 | /* TRM: Must program this for cycacc to work */ |
670 | cc_threshold = attr->config3 & ETM_CYC_THRESHOLD_MASK; |
671 | if (!cc_threshold) |
672 | cc_threshold = ETM_CYC_THRESHOLD_DEFAULT; |
673 | if (cc_threshold < drvdata->ccitmin) |
674 | cc_threshold = drvdata->ccitmin; |
675 | config->ccctlr = cc_threshold; |
676 | } |
677 | if (attr->config & BIT(ETM_OPT_TS)) { |
678 | /* |
679 | * Configure timestamps to be emitted at regular intervals in |
680 | * order to correlate instructions executed on different CPUs |
681 | * (CPU-wide trace scenarios). |
682 | */ |
683 | ret = etm4_config_timestamp_event(drvdata); |
684 | |
685 | /* |
686 | * No need to go further if timestamp intervals can't |
687 | * be configured. |
688 | */ |
689 | if (ret) |
690 | goto out; |
691 | |
692 | /* bit[11], Global timestamp tracing bit */ |
693 | config->cfg |= TRCCONFIGR_TS; |
694 | } |
695 | |
696 | /* Only trace contextID when runs in root PID namespace */ |
697 | if ((attr->config & BIT(ETM_OPT_CTXTID)) && |
698 | task_is_in_init_pid_ns(current)) |
699 | /* bit[6], Context ID tracing bit */ |
700 | config->cfg |= TRCCONFIGR_CID; |
701 | |
702 | /* |
703 | * If set bit ETM_OPT_CTXTID2 in perf config, this asks to trace VMID |
704 | * for recording CONTEXTIDR_EL2. Do not enable VMID tracing if the |
705 | * kernel is not running in EL2. |
706 | */ |
707 | if (attr->config & BIT(ETM_OPT_CTXTID2)) { |
708 | if (!is_kernel_in_hyp_mode()) { |
709 | ret = -EINVAL; |
710 | goto out; |
711 | } |
712 | /* Only trace virtual contextID when runs in root PID namespace */ |
713 | if (task_is_in_init_pid_ns(current)) |
714 | config->cfg |= TRCCONFIGR_VMID | TRCCONFIGR_VMIDOPT; |
715 | } |
716 | |
717 | /* return stack - enable if selected and supported */ |
718 | if ((attr->config & BIT(ETM_OPT_RETSTK)) && drvdata->retstack) |
719 | /* bit[12], Return stack enable bit */ |
720 | config->cfg |= TRCCONFIGR_RS; |
721 | |
722 | /* |
723 | * Set any selected configuration and preset. |
724 | * |
725 | * This extracts the values of PMU_FORMAT_ATTR(configid) and PMU_FORMAT_ATTR(preset) |
726 | * in the perf attributes defined in coresight-etm-perf.c. |
727 | * configid uses bits 63:32 of attr->config2, preset uses bits 3:0 of attr->config. |
728 | * A zero configid means no configuration active, preset = 0 means no preset selected. |
729 | */ |
730 | if (attr->config2 & GENMASK_ULL(63, 32)) { |
731 | cfg_hash = (u32)(attr->config2 >> 32); |
732 | preset = attr->config & 0xF; |
733 | ret = cscfg_csdev_enable_active_config(csdev, cfg_hash, preset); |
734 | } |
735 | |
736 | /* branch broadcast - enable if selected and supported */ |
737 | if (attr->config & BIT(ETM_OPT_BRANCH_BROADCAST)) { |
738 | if (!drvdata->trcbb) { |
739 | /* |
740 | * Missing BB support could cause silent decode errors |
741 | * so fail to open if it's not supported. |
742 | */ |
743 | ret = -EINVAL; |
744 | goto out; |
745 | } else { |
746 | config->cfg |= BIT(ETM4_CFG_BIT_BB); |
747 | } |
748 | } |
749 | |
750 | out: |
751 | return ret; |
752 | } |
753 | |
754 | static int etm4_enable_perf(struct coresight_device *csdev, |
755 | struct perf_event *event) |
756 | { |
757 | int ret = 0, trace_id; |
758 | struct etmv4_drvdata *drvdata = dev_get_drvdata(dev: csdev->dev.parent); |
759 | |
760 | if (WARN_ON_ONCE(drvdata->cpu != smp_processor_id())) { |
761 | ret = -EINVAL; |
762 | goto out; |
763 | } |
764 | |
765 | /* Configure the tracer based on the session's specifics */ |
766 | ret = etm4_parse_event_config(csdev, event); |
767 | if (ret) |
768 | goto out; |
769 | |
770 | /* |
771 | * perf allocates cpu ids as part of _setup_aux() - device needs to use |
772 | * the allocated ID. This reads the current version without allocation. |
773 | * |
774 | * This does not use the trace id lock to prevent lock_dep issues |
775 | * with perf locks - we know the ID cannot change until perf shuts down |
776 | * the session |
777 | */ |
778 | trace_id = coresight_trace_id_read_cpu_id(cpu: drvdata->cpu); |
779 | if (!IS_VALID_CS_TRACE_ID(trace_id)) { |
780 | dev_err(&drvdata->csdev->dev, "Failed to set trace ID for %s on CPU%d\n" , |
781 | dev_name(&drvdata->csdev->dev), drvdata->cpu); |
782 | ret = -EINVAL; |
783 | goto out; |
784 | } |
785 | drvdata->trcid = (u8)trace_id; |
786 | |
787 | /* And enable it */ |
788 | ret = etm4_enable_hw(drvdata); |
789 | |
790 | out: |
791 | return ret; |
792 | } |
793 | |
794 | static int etm4_enable_sysfs(struct coresight_device *csdev) |
795 | { |
796 | struct etmv4_drvdata *drvdata = dev_get_drvdata(dev: csdev->dev.parent); |
797 | struct etm4_enable_arg arg = { }; |
798 | unsigned long cfg_hash; |
799 | int ret, preset; |
800 | |
801 | /* enable any config activated by configfs */ |
802 | cscfg_config_sysfs_get_active_cfg(cfg_hash: &cfg_hash, preset: &preset); |
803 | if (cfg_hash) { |
804 | ret = cscfg_csdev_enable_active_config(csdev, cfg_hash, preset); |
805 | if (ret) |
806 | return ret; |
807 | } |
808 | |
809 | spin_lock(lock: &drvdata->spinlock); |
810 | |
811 | /* sysfs needs to read and allocate a trace ID */ |
812 | ret = etm4_read_alloc_trace_id(drvdata); |
813 | if (ret < 0) |
814 | goto unlock_sysfs_enable; |
815 | |
816 | /* |
817 | * Executing etm4_enable_hw on the cpu whose ETM is being enabled |
818 | * ensures that register writes occur when cpu is powered. |
819 | */ |
820 | arg.drvdata = drvdata; |
821 | ret = smp_call_function_single(cpuid: drvdata->cpu, |
822 | func: etm4_enable_hw_smp_call, info: &arg, wait: 1); |
823 | if (!ret) |
824 | ret = arg.rc; |
825 | if (!ret) |
826 | drvdata->sticky_enable = true; |
827 | |
828 | if (ret) |
829 | etm4_release_trace_id(drvdata); |
830 | |
831 | unlock_sysfs_enable: |
832 | spin_unlock(lock: &drvdata->spinlock); |
833 | |
834 | if (!ret) |
835 | dev_dbg(&csdev->dev, "ETM tracing enabled\n" ); |
836 | return ret; |
837 | } |
838 | |
839 | static int etm4_enable(struct coresight_device *csdev, struct perf_event *event, |
840 | enum cs_mode mode) |
841 | { |
842 | int ret; |
843 | |
844 | if (!coresight_take_mode(csdev, new_mode: mode)) { |
845 | /* Someone is already using the tracer */ |
846 | return -EBUSY; |
847 | } |
848 | |
849 | switch (mode) { |
850 | case CS_MODE_SYSFS: |
851 | ret = etm4_enable_sysfs(csdev); |
852 | break; |
853 | case CS_MODE_PERF: |
854 | ret = etm4_enable_perf(csdev, event); |
855 | break; |
856 | default: |
857 | ret = -EINVAL; |
858 | } |
859 | |
860 | /* The tracer didn't start */ |
861 | if (ret) |
862 | coresight_set_mode(csdev, new_mode: CS_MODE_DISABLED); |
863 | |
864 | return ret; |
865 | } |
866 | |
867 | static void etm4_disable_hw(void *info) |
868 | { |
869 | u32 control; |
870 | struct etmv4_drvdata *drvdata = info; |
871 | struct etmv4_config *config = &drvdata->config; |
872 | struct coresight_device *csdev = drvdata->csdev; |
873 | struct device *etm_dev = &csdev->dev; |
874 | struct csdev_access *csa = &csdev->access; |
875 | int i; |
876 | |
877 | etm4_cs_unlock(drvdata, csa); |
878 | etm4_disable_arch_specific(drvdata); |
879 | |
880 | if (!drvdata->skip_power_up) { |
881 | /* power can be removed from the trace unit now */ |
882 | control = etm4x_relaxed_read32(csa, TRCPDCR); |
883 | control &= ~TRCPDCR_PU; |
884 | etm4x_relaxed_write32(csa, control, TRCPDCR); |
885 | } |
886 | |
887 | control = etm4x_relaxed_read32(csa, TRCPRGCTLR); |
888 | |
889 | /* EN, bit[0] Trace unit enable bit */ |
890 | control &= ~0x1; |
891 | |
892 | /* |
893 | * If the CPU supports v8.4 Trace filter Control, |
894 | * set the ETM to trace prohibited region. |
895 | */ |
896 | etm4x_prohibit_trace(drvdata); |
897 | /* |
898 | * Make sure everything completes before disabling, as recommended |
899 | * by section 7.3.77 ("TRCVICTLR, ViewInst Main Control Register, |
900 | * SSTATUS") of ARM IHI 0064D |
901 | */ |
902 | dsb(sy); |
903 | isb(); |
904 | /* Trace synchronization barrier, is a nop if not supported */ |
905 | tsb_csync(); |
906 | etm4x_relaxed_write32(csa, control, TRCPRGCTLR); |
907 | |
908 | /* wait for TRCSTATR.PMSTABLE to go to '1' */ |
909 | if (coresight_timeout(csa, TRCSTATR, TRCSTATR_PMSTABLE_BIT, value: 1)) |
910 | dev_err(etm_dev, |
911 | "timeout while waiting for PM stable Trace Status\n" ); |
912 | /* read the status of the single shot comparators */ |
913 | for (i = 0; i < drvdata->nr_ss_cmp; i++) { |
914 | config->ss_status[i] = |
915 | etm4x_relaxed_read32(csa, TRCSSCSRn(i)); |
916 | } |
917 | |
918 | /* read back the current counter values */ |
919 | for (i = 0; i < drvdata->nr_cntr; i++) { |
920 | config->cntr_val[i] = |
921 | etm4x_relaxed_read32(csa, TRCCNTVRn(i)); |
922 | } |
923 | |
924 | coresight_disclaim_device_unlocked(csdev); |
925 | etm4_cs_lock(drvdata, csa); |
926 | |
927 | dev_dbg(&drvdata->csdev->dev, |
928 | "cpu: %d disable smp call done\n" , drvdata->cpu); |
929 | } |
930 | |
931 | static int etm4_disable_perf(struct coresight_device *csdev, |
932 | struct perf_event *event) |
933 | { |
934 | u32 control; |
935 | struct etm_filters *filters = event->hw.addr_filters; |
936 | struct etmv4_drvdata *drvdata = dev_get_drvdata(dev: csdev->dev.parent); |
937 | struct perf_event_attr *attr = &event->attr; |
938 | |
939 | if (WARN_ON_ONCE(drvdata->cpu != smp_processor_id())) |
940 | return -EINVAL; |
941 | |
942 | etm4_disable_hw(info: drvdata); |
943 | /* |
944 | * The config_id occupies bits 63:32 of the config2 perf event attr |
945 | * field. If this is non-zero then we will have enabled a config. |
946 | */ |
947 | if (attr->config2 & GENMASK_ULL(63, 32)) |
948 | cscfg_csdev_disable_active_config(csdev); |
949 | |
950 | /* |
951 | * Check if the start/stop logic was active when the unit was stopped. |
952 | * That way we can re-enable the start/stop logic when the process is |
953 | * scheduled again. Configuration of the start/stop logic happens in |
954 | * function etm4_set_event_filters(). |
955 | */ |
956 | control = etm4x_relaxed_read32(&csdev->access, TRCVICTLR); |
957 | /* TRCVICTLR::SSSTATUS, bit[9] */ |
958 | filters->ssstatus = (control & BIT(9)); |
959 | |
960 | /* |
961 | * perf will release trace ids when _free_aux() is |
962 | * called at the end of the session. |
963 | */ |
964 | |
965 | return 0; |
966 | } |
967 | |
968 | static void etm4_disable_sysfs(struct coresight_device *csdev) |
969 | { |
970 | struct etmv4_drvdata *drvdata = dev_get_drvdata(dev: csdev->dev.parent); |
971 | |
972 | /* |
973 | * Taking hotplug lock here protects from clocks getting disabled |
974 | * with tracing being left on (crash scenario) if user disable occurs |
975 | * after cpu online mask indicates the cpu is offline but before the |
976 | * DYING hotplug callback is serviced by the ETM driver. |
977 | */ |
978 | cpus_read_lock(); |
979 | spin_lock(lock: &drvdata->spinlock); |
980 | |
981 | /* |
982 | * Executing etm4_disable_hw on the cpu whose ETM is being disabled |
983 | * ensures that register writes occur when cpu is powered. |
984 | */ |
985 | smp_call_function_single(cpuid: drvdata->cpu, func: etm4_disable_hw, info: drvdata, wait: 1); |
986 | |
987 | spin_unlock(lock: &drvdata->spinlock); |
988 | cpus_read_unlock(); |
989 | |
990 | /* |
991 | * we only release trace IDs when resetting sysfs. |
992 | * This permits sysfs users to read the trace ID after the trace |
993 | * session has completed. This maintains operational behaviour with |
994 | * prior trace id allocation method |
995 | */ |
996 | |
997 | dev_dbg(&csdev->dev, "ETM tracing disabled\n" ); |
998 | } |
999 | |
1000 | static void etm4_disable(struct coresight_device *csdev, |
1001 | struct perf_event *event) |
1002 | { |
1003 | enum cs_mode mode; |
1004 | |
1005 | /* |
1006 | * For as long as the tracer isn't disabled another entity can't |
1007 | * change its status. As such we can read the status here without |
1008 | * fearing it will change under us. |
1009 | */ |
1010 | mode = coresight_get_mode(csdev); |
1011 | |
1012 | switch (mode) { |
1013 | case CS_MODE_DISABLED: |
1014 | break; |
1015 | case CS_MODE_SYSFS: |
1016 | etm4_disable_sysfs(csdev); |
1017 | break; |
1018 | case CS_MODE_PERF: |
1019 | etm4_disable_perf(csdev, event); |
1020 | break; |
1021 | } |
1022 | |
1023 | if (mode) |
1024 | coresight_set_mode(csdev, new_mode: CS_MODE_DISABLED); |
1025 | } |
1026 | |
1027 | static const struct coresight_ops_source etm4_source_ops = { |
1028 | .cpu_id = etm4_cpu_id, |
1029 | .enable = etm4_enable, |
1030 | .disable = etm4_disable, |
1031 | }; |
1032 | |
1033 | static const struct coresight_ops etm4_cs_ops = { |
1034 | .source_ops = &etm4_source_ops, |
1035 | }; |
1036 | |
1037 | static inline bool cpu_supports_sysreg_trace(void) |
1038 | { |
1039 | u64 dfr0 = read_sysreg_s(SYS_ID_AA64DFR0_EL1); |
1040 | |
1041 | return ((dfr0 >> ID_AA64DFR0_EL1_TraceVer_SHIFT) & 0xfUL) > 0; |
1042 | } |
1043 | |
1044 | static bool etm4_init_sysreg_access(struct etmv4_drvdata *drvdata, |
1045 | struct csdev_access *csa) |
1046 | { |
1047 | u32 devarch; |
1048 | |
1049 | if (!cpu_supports_sysreg_trace()) |
1050 | return false; |
1051 | |
1052 | /* |
1053 | * ETMs implementing sysreg access must implement TRCDEVARCH. |
1054 | */ |
1055 | devarch = read_etm4x_sysreg_const_offset(TRCDEVARCH); |
1056 | switch (devarch & ETM_DEVARCH_ID_MASK) { |
1057 | case ETM_DEVARCH_ETMv4x_ARCH: |
1058 | *csa = (struct csdev_access) { |
1059 | .io_mem = false, |
1060 | .read = etm4x_sysreg_read, |
1061 | .write = etm4x_sysreg_write, |
1062 | }; |
1063 | break; |
1064 | case ETM_DEVARCH_ETE_ARCH: |
1065 | *csa = (struct csdev_access) { |
1066 | .io_mem = false, |
1067 | .read = ete_sysreg_read, |
1068 | .write = ete_sysreg_write, |
1069 | }; |
1070 | break; |
1071 | default: |
1072 | return false; |
1073 | } |
1074 | |
1075 | drvdata->arch = etm_devarch_to_arch(devarch); |
1076 | return true; |
1077 | } |
1078 | |
1079 | static bool is_devtype_cpu_trace(void __iomem *base) |
1080 | { |
1081 | u32 devtype = readl(addr: base + TRCDEVTYPE); |
1082 | |
1083 | return (devtype == CS_DEVTYPE_PE_TRACE); |
1084 | } |
1085 | |
1086 | static bool etm4_init_iomem_access(struct etmv4_drvdata *drvdata, |
1087 | struct csdev_access *csa) |
1088 | { |
1089 | u32 devarch = readl_relaxed(drvdata->base + TRCDEVARCH); |
1090 | |
1091 | if (!is_coresight_device(base: drvdata->base) || !is_devtype_cpu_trace(base: drvdata->base)) |
1092 | return false; |
1093 | |
1094 | /* |
1095 | * All ETMs must implement TRCDEVARCH to indicate that |
1096 | * the component is an ETMv4. Even though TRCIDR1 also |
1097 | * contains the information, it is part of the "Trace" |
1098 | * register and must be accessed with the OSLK cleared, |
1099 | * with MMIO. But we cannot touch the OSLK until we are |
1100 | * sure this is an ETM. So rely only on the TRCDEVARCH. |
1101 | */ |
1102 | if ((devarch & ETM_DEVARCH_ID_MASK) != ETM_DEVARCH_ETMv4x_ARCH) { |
1103 | pr_warn_once("TRCDEVARCH doesn't match ETMv4 architecture\n" ); |
1104 | return false; |
1105 | } |
1106 | |
1107 | drvdata->arch = etm_devarch_to_arch(devarch); |
1108 | *csa = CSDEV_ACCESS_IOMEM(drvdata->base); |
1109 | return true; |
1110 | } |
1111 | |
1112 | static bool etm4_init_csdev_access(struct etmv4_drvdata *drvdata, |
1113 | struct csdev_access *csa) |
1114 | { |
1115 | /* |
1116 | * Always choose the memory mapped io, if there is |
1117 | * a memory map to prevent sysreg access on broken |
1118 | * systems. |
1119 | */ |
1120 | if (drvdata->base) |
1121 | return etm4_init_iomem_access(drvdata, csa); |
1122 | |
1123 | if (etm4_init_sysreg_access(drvdata, csa)) |
1124 | return true; |
1125 | |
1126 | return false; |
1127 | } |
1128 | |
1129 | static void cpu_detect_trace_filtering(struct etmv4_drvdata *drvdata) |
1130 | { |
1131 | u64 dfr0 = read_sysreg(id_aa64dfr0_el1); |
1132 | u64 trfcr; |
1133 | |
1134 | drvdata->trfcr = 0; |
1135 | if (!cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_EL1_TraceFilt_SHIFT)) |
1136 | return; |
1137 | |
1138 | /* |
1139 | * If the CPU supports v8.4 SelfHosted Tracing, enable |
1140 | * tracing at the kernel EL and EL0, forcing to use the |
1141 | * virtual time as the timestamp. |
1142 | */ |
1143 | trfcr = (TRFCR_ELx_TS_VIRTUAL | |
1144 | TRFCR_ELx_ExTRE | |
1145 | TRFCR_ELx_E0TRE); |
1146 | |
1147 | /* If we are running at EL2, allow tracing the CONTEXTIDR_EL2. */ |
1148 | if (is_kernel_in_hyp_mode()) |
1149 | trfcr |= TRFCR_EL2_CX; |
1150 | |
1151 | drvdata->trfcr = trfcr; |
1152 | } |
1153 | |
1154 | /* |
1155 | * The following errata on applicable cpu ranges, affect the CCITMIN filed |
1156 | * in TCRIDR3 register. Software read for the field returns 0x100 limiting |
1157 | * the cycle threshold granularity, whereas the right value should have |
1158 | * been 0x4, which is well supported in the hardware. |
1159 | */ |
1160 | static struct midr_range etm_wrong_ccitmin_cpus[] = { |
1161 | /* Erratum #1490853 - Cortex-A76 */ |
1162 | MIDR_RANGE(MIDR_CORTEX_A76, 0, 0, 4, 0), |
1163 | /* Erratum #1490853 - Neoverse-N1 */ |
1164 | MIDR_RANGE(MIDR_NEOVERSE_N1, 0, 0, 4, 0), |
1165 | /* Erratum #1491015 - Cortex-A77 */ |
1166 | MIDR_RANGE(MIDR_CORTEX_A77, 0, 0, 1, 0), |
1167 | /* Erratum #1502854 - Cortex-X1 */ |
1168 | MIDR_REV(MIDR_CORTEX_X1, 0, 0), |
1169 | /* Erratum #1619801 - Neoverse-V1 */ |
1170 | MIDR_REV(MIDR_NEOVERSE_V1, 0, 0), |
1171 | {}, |
1172 | }; |
1173 | |
1174 | static void etm4_fixup_wrong_ccitmin(struct etmv4_drvdata *drvdata) |
1175 | { |
1176 | /* |
1177 | * Erratum affected cpus will read 256 as the minimum |
1178 | * instruction trace cycle counting threshold whereas |
1179 | * the correct value should be 4 instead. Override the |
1180 | * recorded value for 'drvdata->ccitmin' to workaround |
1181 | * this problem. |
1182 | */ |
1183 | if (is_midr_in_range_list(read_cpuid_id(), etm_wrong_ccitmin_cpus)) { |
1184 | if (drvdata->ccitmin == 256) |
1185 | drvdata->ccitmin = 4; |
1186 | } |
1187 | } |
1188 | |
1189 | static void etm4_init_arch_data(void *info) |
1190 | { |
1191 | u32 etmidr0; |
1192 | u32 etmidr2; |
1193 | u32 etmidr3; |
1194 | u32 etmidr4; |
1195 | u32 etmidr5; |
1196 | struct etm4_init_arg *init_arg = info; |
1197 | struct etmv4_drvdata *drvdata; |
1198 | struct csdev_access *csa; |
1199 | struct device *dev = init_arg->dev; |
1200 | int i; |
1201 | |
1202 | drvdata = dev_get_drvdata(dev: init_arg->dev); |
1203 | csa = init_arg->csa; |
1204 | |
1205 | /* |
1206 | * If we are unable to detect the access mechanism, |
1207 | * or unable to detect the trace unit type, fail |
1208 | * early. |
1209 | */ |
1210 | if (!etm4_init_csdev_access(drvdata, csa)) |
1211 | return; |
1212 | |
1213 | if (!csa->io_mem || |
1214 | fwnode_property_present(dev_fwnode(dev), propname: "qcom,skip-power-up" )) |
1215 | drvdata->skip_power_up = true; |
1216 | |
1217 | /* Detect the support for OS Lock before we actually use it */ |
1218 | etm_detect_os_lock(drvdata, csa); |
1219 | |
1220 | /* Make sure all registers are accessible */ |
1221 | etm4_os_unlock_csa(drvdata, csa); |
1222 | etm4_cs_unlock(drvdata, csa); |
1223 | |
1224 | etm4_check_arch_features(drvdata, csa); |
1225 | |
1226 | /* find all capabilities of the tracing unit */ |
1227 | etmidr0 = etm4x_relaxed_read32(csa, TRCIDR0); |
1228 | |
1229 | /* INSTP0, bits[2:1] P0 tracing support field */ |
1230 | drvdata->instrp0 = !!(FIELD_GET(TRCIDR0_INSTP0_MASK, etmidr0) == 0b11); |
1231 | /* TRCBB, bit[5] Branch broadcast tracing support bit */ |
1232 | drvdata->trcbb = !!(etmidr0 & TRCIDR0_TRCBB); |
1233 | /* TRCCOND, bit[6] Conditional instruction tracing support bit */ |
1234 | drvdata->trccond = !!(etmidr0 & TRCIDR0_TRCCOND); |
1235 | /* TRCCCI, bit[7] Cycle counting instruction bit */ |
1236 | drvdata->trccci = !!(etmidr0 & TRCIDR0_TRCCCI); |
1237 | /* RETSTACK, bit[9] Return stack bit */ |
1238 | drvdata->retstack = !!(etmidr0 & TRCIDR0_RETSTACK); |
1239 | /* NUMEVENT, bits[11:10] Number of events field */ |
1240 | drvdata->nr_event = FIELD_GET(TRCIDR0_NUMEVENT_MASK, etmidr0); |
1241 | /* QSUPP, bits[16:15] Q element support field */ |
1242 | drvdata->q_support = FIELD_GET(TRCIDR0_QSUPP_MASK, etmidr0); |
1243 | /* TSSIZE, bits[28:24] Global timestamp size field */ |
1244 | drvdata->ts_size = FIELD_GET(TRCIDR0_TSSIZE_MASK, etmidr0); |
1245 | |
1246 | /* maximum size of resources */ |
1247 | etmidr2 = etm4x_relaxed_read32(csa, TRCIDR2); |
1248 | /* CIDSIZE, bits[9:5] Indicates the Context ID size */ |
1249 | drvdata->ctxid_size = FIELD_GET(TRCIDR2_CIDSIZE_MASK, etmidr2); |
1250 | /* VMIDSIZE, bits[14:10] Indicates the VMID size */ |
1251 | drvdata->vmid_size = FIELD_GET(TRCIDR2_VMIDSIZE_MASK, etmidr2); |
1252 | /* CCSIZE, bits[28:25] size of the cycle counter in bits minus 12 */ |
1253 | drvdata->ccsize = FIELD_GET(TRCIDR2_CCSIZE_MASK, etmidr2); |
1254 | |
1255 | etmidr3 = etm4x_relaxed_read32(csa, TRCIDR3); |
1256 | /* CCITMIN, bits[11:0] minimum threshold value that can be programmed */ |
1257 | drvdata->ccitmin = FIELD_GET(TRCIDR3_CCITMIN_MASK, etmidr3); |
1258 | etm4_fixup_wrong_ccitmin(drvdata); |
1259 | |
1260 | /* EXLEVEL_S, bits[19:16] Secure state instruction tracing */ |
1261 | drvdata->s_ex_level = FIELD_GET(TRCIDR3_EXLEVEL_S_MASK, etmidr3); |
1262 | drvdata->config.s_ex_level = drvdata->s_ex_level; |
1263 | /* EXLEVEL_NS, bits[23:20] Non-secure state instruction tracing */ |
1264 | drvdata->ns_ex_level = FIELD_GET(TRCIDR3_EXLEVEL_NS_MASK, etmidr3); |
1265 | /* |
1266 | * TRCERR, bit[24] whether a trace unit can trace a |
1267 | * system error exception. |
1268 | */ |
1269 | drvdata->trc_error = !!(etmidr3 & TRCIDR3_TRCERR); |
1270 | /* SYNCPR, bit[25] implementation has a fixed synchronization period? */ |
1271 | drvdata->syncpr = !!(etmidr3 & TRCIDR3_SYNCPR); |
1272 | /* STALLCTL, bit[26] is stall control implemented? */ |
1273 | drvdata->stallctl = !!(etmidr3 & TRCIDR3_STALLCTL); |
1274 | /* SYSSTALL, bit[27] implementation can support stall control? */ |
1275 | drvdata->sysstall = !!(etmidr3 & TRCIDR3_SYSSTALL); |
1276 | /* |
1277 | * NUMPROC - the number of PEs available for tracing, 5bits |
1278 | * = TRCIDR3.bits[13:12]bits[30:28] |
1279 | * bits[4:3] = TRCIDR3.bits[13:12] (since etm-v4.2, otherwise RES0) |
1280 | * bits[3:0] = TRCIDR3.bits[30:28] |
1281 | */ |
1282 | drvdata->nr_pe = (FIELD_GET(TRCIDR3_NUMPROC_HI_MASK, etmidr3) << 3) | |
1283 | FIELD_GET(TRCIDR3_NUMPROC_LO_MASK, etmidr3); |
1284 | /* NOOVERFLOW, bit[31] is trace overflow prevention supported */ |
1285 | drvdata->nooverflow = !!(etmidr3 & TRCIDR3_NOOVERFLOW); |
1286 | |
1287 | /* number of resources trace unit supports */ |
1288 | etmidr4 = etm4x_relaxed_read32(csa, TRCIDR4); |
1289 | /* NUMACPAIRS, bits[0:3] number of addr comparator pairs for tracing */ |
1290 | drvdata->nr_addr_cmp = FIELD_GET(TRCIDR4_NUMACPAIRS_MASK, etmidr4); |
1291 | /* NUMPC, bits[15:12] number of PE comparator inputs for tracing */ |
1292 | drvdata->nr_pe_cmp = FIELD_GET(TRCIDR4_NUMPC_MASK, etmidr4); |
1293 | /* |
1294 | * NUMRSPAIR, bits[19:16] |
1295 | * The number of resource pairs conveyed by the HW starts at 0, i.e a |
1296 | * value of 0x0 indicate 1 resource pair, 0x1 indicate two and so on. |
1297 | * As such add 1 to the value of NUMRSPAIR for a better representation. |
1298 | * |
1299 | * For ETM v4.3 and later, 0x0 means 0, and no pairs are available - |
1300 | * the default TRUE and FALSE resource selectors are omitted. |
1301 | * Otherwise for values 0x1 and above the number is N + 1 as per v4.2. |
1302 | */ |
1303 | drvdata->nr_resource = FIELD_GET(TRCIDR4_NUMRSPAIR_MASK, etmidr4); |
1304 | if ((drvdata->arch < ETM_ARCH_V4_3) || (drvdata->nr_resource > 0)) |
1305 | drvdata->nr_resource += 1; |
1306 | /* |
1307 | * NUMSSCC, bits[23:20] the number of single-shot |
1308 | * comparator control for tracing. Read any status regs as these |
1309 | * also contain RO capability data. |
1310 | */ |
1311 | drvdata->nr_ss_cmp = FIELD_GET(TRCIDR4_NUMSSCC_MASK, etmidr4); |
1312 | for (i = 0; i < drvdata->nr_ss_cmp; i++) { |
1313 | drvdata->config.ss_status[i] = |
1314 | etm4x_relaxed_read32(csa, TRCSSCSRn(i)); |
1315 | } |
1316 | /* NUMCIDC, bits[27:24] number of Context ID comparators for tracing */ |
1317 | drvdata->numcidc = FIELD_GET(TRCIDR4_NUMCIDC_MASK, etmidr4); |
1318 | /* NUMVMIDC, bits[31:28] number of VMID comparators for tracing */ |
1319 | drvdata->numvmidc = FIELD_GET(TRCIDR4_NUMVMIDC_MASK, etmidr4); |
1320 | |
1321 | etmidr5 = etm4x_relaxed_read32(csa, TRCIDR5); |
1322 | /* NUMEXTIN, bits[8:0] number of external inputs implemented */ |
1323 | drvdata->nr_ext_inp = FIELD_GET(TRCIDR5_NUMEXTIN_MASK, etmidr5); |
1324 | /* TRACEIDSIZE, bits[21:16] indicates the trace ID width */ |
1325 | drvdata->trcid_size = FIELD_GET(TRCIDR5_TRACEIDSIZE_MASK, etmidr5); |
1326 | /* ATBTRIG, bit[22] implementation can support ATB triggers? */ |
1327 | drvdata->atbtrig = !!(etmidr5 & TRCIDR5_ATBTRIG); |
1328 | /* |
1329 | * LPOVERRIDE, bit[23] implementation supports |
1330 | * low-power state override |
1331 | */ |
1332 | drvdata->lpoverride = (etmidr5 & TRCIDR5_LPOVERRIDE) && (!drvdata->skip_power_up); |
1333 | /* NUMSEQSTATE, bits[27:25] number of sequencer states implemented */ |
1334 | drvdata->nrseqstate = FIELD_GET(TRCIDR5_NUMSEQSTATE_MASK, etmidr5); |
1335 | /* NUMCNTR, bits[30:28] number of counters available for tracing */ |
1336 | drvdata->nr_cntr = FIELD_GET(TRCIDR5_NUMCNTR_MASK, etmidr5); |
1337 | etm4_cs_lock(drvdata, csa); |
1338 | cpu_detect_trace_filtering(drvdata); |
1339 | } |
1340 | |
1341 | static inline u32 etm4_get_victlr_access_type(struct etmv4_config *config) |
1342 | { |
1343 | return etm4_get_access_type(config) << __bf_shf(TRCVICTLR_EXLEVEL_MASK); |
1344 | } |
1345 | |
1346 | /* Set ELx trace filter access in the TRCVICTLR register */ |
1347 | static void etm4_set_victlr_access(struct etmv4_config *config) |
1348 | { |
1349 | config->vinst_ctrl &= ~TRCVICTLR_EXLEVEL_MASK; |
1350 | config->vinst_ctrl |= etm4_get_victlr_access_type(config); |
1351 | } |
1352 | |
1353 | static void etm4_set_default_config(struct etmv4_config *config) |
1354 | { |
1355 | /* disable all events tracing */ |
1356 | config->eventctrl0 = 0x0; |
1357 | config->eventctrl1 = 0x0; |
1358 | |
1359 | /* disable stalling */ |
1360 | config->stall_ctrl = 0x0; |
1361 | |
1362 | /* enable trace synchronization every 4096 bytes, if available */ |
1363 | config->syncfreq = 0xC; |
1364 | |
1365 | /* disable timestamp event */ |
1366 | config->ts_ctrl = 0x0; |
1367 | |
1368 | /* TRCVICTLR::EVENT = 0x01, select the always on logic */ |
1369 | config->vinst_ctrl = FIELD_PREP(TRCVICTLR_EVENT_MASK, 0x01); |
1370 | |
1371 | /* TRCVICTLR::EXLEVEL_NS:EXLEVELS: Set kernel / user filtering */ |
1372 | etm4_set_victlr_access(config); |
1373 | } |
1374 | |
1375 | static u64 etm4_get_ns_access_type(struct etmv4_config *config) |
1376 | { |
1377 | u64 access_type = 0; |
1378 | |
1379 | /* |
1380 | * EXLEVEL_NS, for NonSecure Exception levels. |
1381 | * The mask here is a generic value and must be |
1382 | * shifted to the corresponding field for the registers |
1383 | */ |
1384 | if (!is_kernel_in_hyp_mode()) { |
1385 | /* Stay away from hypervisor mode for non-VHE */ |
1386 | access_type = ETM_EXLEVEL_NS_HYP; |
1387 | if (config->mode & ETM_MODE_EXCL_KERN) |
1388 | access_type |= ETM_EXLEVEL_NS_OS; |
1389 | } else if (config->mode & ETM_MODE_EXCL_KERN) { |
1390 | access_type = ETM_EXLEVEL_NS_HYP; |
1391 | } |
1392 | |
1393 | if (config->mode & ETM_MODE_EXCL_USER) |
1394 | access_type |= ETM_EXLEVEL_NS_APP; |
1395 | |
1396 | return access_type; |
1397 | } |
1398 | |
1399 | /* |
1400 | * Construct the exception level masks for a given config. |
1401 | * This must be shifted to the corresponding register field |
1402 | * for usage. |
1403 | */ |
1404 | static u64 etm4_get_access_type(struct etmv4_config *config) |
1405 | { |
1406 | /* All Secure exception levels are excluded from the trace */ |
1407 | return etm4_get_ns_access_type(config) | (u64)config->s_ex_level; |
1408 | } |
1409 | |
1410 | static u64 etm4_get_comparator_access_type(struct etmv4_config *config) |
1411 | { |
1412 | return etm4_get_access_type(config) << TRCACATR_EXLEVEL_SHIFT; |
1413 | } |
1414 | |
1415 | static void etm4_set_comparator_filter(struct etmv4_config *config, |
1416 | u64 start, u64 stop, int comparator) |
1417 | { |
1418 | u64 access_type = etm4_get_comparator_access_type(config); |
1419 | |
1420 | /* First half of default address comparator */ |
1421 | config->addr_val[comparator] = start; |
1422 | config->addr_acc[comparator] = access_type; |
1423 | config->addr_type[comparator] = ETM_ADDR_TYPE_RANGE; |
1424 | |
1425 | /* Second half of default address comparator */ |
1426 | config->addr_val[comparator + 1] = stop; |
1427 | config->addr_acc[comparator + 1] = access_type; |
1428 | config->addr_type[comparator + 1] = ETM_ADDR_TYPE_RANGE; |
1429 | |
1430 | /* |
1431 | * Configure the ViewInst function to include this address range |
1432 | * comparator. |
1433 | * |
1434 | * @comparator is divided by two since it is the index in the |
1435 | * etmv4_config::addr_val array but register TRCVIIECTLR deals with |
1436 | * address range comparator _pairs_. |
1437 | * |
1438 | * Therefore: |
1439 | * index 0 -> compatator pair 0 |
1440 | * index 2 -> comparator pair 1 |
1441 | * index 4 -> comparator pair 2 |
1442 | * ... |
1443 | * index 14 -> comparator pair 7 |
1444 | */ |
1445 | config->viiectlr |= BIT(comparator / 2); |
1446 | } |
1447 | |
1448 | static void etm4_set_start_stop_filter(struct etmv4_config *config, |
1449 | u64 address, int comparator, |
1450 | enum etm_addr_type type) |
1451 | { |
1452 | int shift; |
1453 | u64 access_type = etm4_get_comparator_access_type(config); |
1454 | |
1455 | /* Configure the comparator */ |
1456 | config->addr_val[comparator] = address; |
1457 | config->addr_acc[comparator] = access_type; |
1458 | config->addr_type[comparator] = type; |
1459 | |
1460 | /* |
1461 | * Configure ViewInst Start-Stop control register. |
1462 | * Addresses configured to start tracing go from bit 0 to n-1, |
1463 | * while those configured to stop tracing from 16 to 16 + n-1. |
1464 | */ |
1465 | shift = (type == ETM_ADDR_TYPE_START ? 0 : 16); |
1466 | config->vissctlr |= BIT(shift + comparator); |
1467 | } |
1468 | |
1469 | static void etm4_set_default_filter(struct etmv4_config *config) |
1470 | { |
1471 | /* Trace everything 'default' filter achieved by no filtering */ |
1472 | config->viiectlr = 0x0; |
1473 | |
1474 | /* |
1475 | * TRCVICTLR::SSSTATUS == 1, the start-stop logic is |
1476 | * in the started state |
1477 | */ |
1478 | config->vinst_ctrl |= TRCVICTLR_SSSTATUS; |
1479 | config->mode |= ETM_MODE_VIEWINST_STARTSTOP; |
1480 | |
1481 | /* No start-stop filtering for ViewInst */ |
1482 | config->vissctlr = 0x0; |
1483 | } |
1484 | |
1485 | static void etm4_set_default(struct etmv4_config *config) |
1486 | { |
1487 | if (WARN_ON_ONCE(!config)) |
1488 | return; |
1489 | |
1490 | /* |
1491 | * Make default initialisation trace everything |
1492 | * |
1493 | * This is done by a minimum default config sufficient to enable |
1494 | * full instruction trace - with a default filter for trace all |
1495 | * achieved by having no filtering. |
1496 | */ |
1497 | etm4_set_default_config(config); |
1498 | etm4_set_default_filter(config); |
1499 | } |
1500 | |
1501 | static int etm4_get_next_comparator(struct etmv4_drvdata *drvdata, u32 type) |
1502 | { |
1503 | int nr_comparator, index = 0; |
1504 | struct etmv4_config *config = &drvdata->config; |
1505 | |
1506 | /* |
1507 | * nr_addr_cmp holds the number of comparator _pair_, so time 2 |
1508 | * for the total number of comparators. |
1509 | */ |
1510 | nr_comparator = drvdata->nr_addr_cmp * 2; |
1511 | |
1512 | /* Go through the tally of comparators looking for a free one. */ |
1513 | while (index < nr_comparator) { |
1514 | switch (type) { |
1515 | case ETM_ADDR_TYPE_RANGE: |
1516 | if (config->addr_type[index] == ETM_ADDR_TYPE_NONE && |
1517 | config->addr_type[index + 1] == ETM_ADDR_TYPE_NONE) |
1518 | return index; |
1519 | |
1520 | /* Address range comparators go in pairs */ |
1521 | index += 2; |
1522 | break; |
1523 | case ETM_ADDR_TYPE_START: |
1524 | case ETM_ADDR_TYPE_STOP: |
1525 | if (config->addr_type[index] == ETM_ADDR_TYPE_NONE) |
1526 | return index; |
1527 | |
1528 | /* Start/stop address can have odd indexes */ |
1529 | index += 1; |
1530 | break; |
1531 | default: |
1532 | return -EINVAL; |
1533 | } |
1534 | } |
1535 | |
1536 | /* If we are here all the comparators have been used. */ |
1537 | return -ENOSPC; |
1538 | } |
1539 | |
1540 | static int etm4_set_event_filters(struct etmv4_drvdata *drvdata, |
1541 | struct perf_event *event) |
1542 | { |
1543 | int i, comparator, ret = 0; |
1544 | u64 address; |
1545 | struct etmv4_config *config = &drvdata->config; |
1546 | struct etm_filters *filters = event->hw.addr_filters; |
1547 | |
1548 | if (!filters) |
1549 | goto default_filter; |
1550 | |
1551 | /* Sync events with what Perf got */ |
1552 | perf_event_addr_filters_sync(event); |
1553 | |
1554 | /* |
1555 | * If there are no filters to deal with simply go ahead with |
1556 | * the default filter, i.e the entire address range. |
1557 | */ |
1558 | if (!filters->nr_filters) |
1559 | goto default_filter; |
1560 | |
1561 | for (i = 0; i < filters->nr_filters; i++) { |
1562 | struct etm_filter *filter = &filters->etm_filter[i]; |
1563 | enum etm_addr_type type = filter->type; |
1564 | |
1565 | /* See if a comparator is free. */ |
1566 | comparator = etm4_get_next_comparator(drvdata, type); |
1567 | if (comparator < 0) { |
1568 | ret = comparator; |
1569 | goto out; |
1570 | } |
1571 | |
1572 | switch (type) { |
1573 | case ETM_ADDR_TYPE_RANGE: |
1574 | etm4_set_comparator_filter(config, |
1575 | start: filter->start_addr, |
1576 | stop: filter->stop_addr, |
1577 | comparator); |
1578 | /* |
1579 | * TRCVICTLR::SSSTATUS == 1, the start-stop logic is |
1580 | * in the started state |
1581 | */ |
1582 | config->vinst_ctrl |= TRCVICTLR_SSSTATUS; |
1583 | |
1584 | /* No start-stop filtering for ViewInst */ |
1585 | config->vissctlr = 0x0; |
1586 | break; |
1587 | case ETM_ADDR_TYPE_START: |
1588 | case ETM_ADDR_TYPE_STOP: |
1589 | /* Get the right start or stop address */ |
1590 | address = (type == ETM_ADDR_TYPE_START ? |
1591 | filter->start_addr : |
1592 | filter->stop_addr); |
1593 | |
1594 | /* Configure comparator */ |
1595 | etm4_set_start_stop_filter(config, address, |
1596 | comparator, type); |
1597 | |
1598 | /* |
1599 | * If filters::ssstatus == 1, trace acquisition was |
1600 | * started but the process was yanked away before the |
1601 | * stop address was hit. As such the start/stop |
1602 | * logic needs to be re-started so that tracing can |
1603 | * resume where it left. |
1604 | * |
1605 | * The start/stop logic status when a process is |
1606 | * scheduled out is checked in function |
1607 | * etm4_disable_perf(). |
1608 | */ |
1609 | if (filters->ssstatus) |
1610 | config->vinst_ctrl |= TRCVICTLR_SSSTATUS; |
1611 | |
1612 | /* No include/exclude filtering for ViewInst */ |
1613 | config->viiectlr = 0x0; |
1614 | break; |
1615 | default: |
1616 | ret = -EINVAL; |
1617 | goto out; |
1618 | } |
1619 | } |
1620 | |
1621 | goto out; |
1622 | |
1623 | |
1624 | default_filter: |
1625 | etm4_set_default_filter(config); |
1626 | |
1627 | out: |
1628 | return ret; |
1629 | } |
1630 | |
1631 | void etm4_config_trace_mode(struct etmv4_config *config) |
1632 | { |
1633 | u32 mode; |
1634 | |
1635 | mode = config->mode; |
1636 | mode &= (ETM_MODE_EXCL_KERN | ETM_MODE_EXCL_USER); |
1637 | |
1638 | /* excluding kernel AND user space doesn't make sense */ |
1639 | WARN_ON_ONCE(mode == (ETM_MODE_EXCL_KERN | ETM_MODE_EXCL_USER)); |
1640 | |
1641 | /* nothing to do if neither flags are set */ |
1642 | if (!(mode & ETM_MODE_EXCL_KERN) && !(mode & ETM_MODE_EXCL_USER)) |
1643 | return; |
1644 | |
1645 | etm4_set_victlr_access(config); |
1646 | } |
1647 | |
1648 | static int etm4_online_cpu(unsigned int cpu) |
1649 | { |
1650 | if (!etmdrvdata[cpu]) |
1651 | return etm4_probe_cpu(cpu); |
1652 | |
1653 | if (etmdrvdata[cpu]->boot_enable && !etmdrvdata[cpu]->sticky_enable) |
1654 | coresight_enable_sysfs(csdev: etmdrvdata[cpu]->csdev); |
1655 | return 0; |
1656 | } |
1657 | |
1658 | static int etm4_starting_cpu(unsigned int cpu) |
1659 | { |
1660 | if (!etmdrvdata[cpu]) |
1661 | return 0; |
1662 | |
1663 | spin_lock(lock: &etmdrvdata[cpu]->spinlock); |
1664 | if (!etmdrvdata[cpu]->os_unlock) |
1665 | etm4_os_unlock(drvdata: etmdrvdata[cpu]); |
1666 | |
1667 | if (coresight_get_mode(csdev: etmdrvdata[cpu]->csdev)) |
1668 | etm4_enable_hw(drvdata: etmdrvdata[cpu]); |
1669 | spin_unlock(lock: &etmdrvdata[cpu]->spinlock); |
1670 | return 0; |
1671 | } |
1672 | |
1673 | static int etm4_dying_cpu(unsigned int cpu) |
1674 | { |
1675 | if (!etmdrvdata[cpu]) |
1676 | return 0; |
1677 | |
1678 | spin_lock(lock: &etmdrvdata[cpu]->spinlock); |
1679 | if (coresight_get_mode(csdev: etmdrvdata[cpu]->csdev)) |
1680 | etm4_disable_hw(info: etmdrvdata[cpu]); |
1681 | spin_unlock(lock: &etmdrvdata[cpu]->spinlock); |
1682 | return 0; |
1683 | } |
1684 | |
1685 | static int __etm4_cpu_save(struct etmv4_drvdata *drvdata) |
1686 | { |
1687 | int i, ret = 0; |
1688 | struct etmv4_save_state *state; |
1689 | struct coresight_device *csdev = drvdata->csdev; |
1690 | struct csdev_access *csa; |
1691 | struct device *etm_dev; |
1692 | |
1693 | if (WARN_ON(!csdev)) |
1694 | return -ENODEV; |
1695 | |
1696 | etm_dev = &csdev->dev; |
1697 | csa = &csdev->access; |
1698 | |
1699 | /* |
1700 | * As recommended by 3.4.1 ("The procedure when powering down the PE") |
1701 | * of ARM IHI 0064D |
1702 | */ |
1703 | dsb(sy); |
1704 | isb(); |
1705 | |
1706 | etm4_cs_unlock(drvdata, csa); |
1707 | /* Lock the OS lock to disable trace and external debugger access */ |
1708 | etm4_os_lock(drvdata); |
1709 | |
1710 | /* wait for TRCSTATR.PMSTABLE to go up */ |
1711 | if (coresight_timeout(csa, TRCSTATR, TRCSTATR_PMSTABLE_BIT, value: 1)) { |
1712 | dev_err(etm_dev, |
1713 | "timeout while waiting for PM Stable Status\n" ); |
1714 | etm4_os_unlock(drvdata); |
1715 | ret = -EBUSY; |
1716 | goto out; |
1717 | } |
1718 | |
1719 | state = drvdata->save_state; |
1720 | |
1721 | state->trcprgctlr = etm4x_read32(csa, TRCPRGCTLR); |
1722 | if (drvdata->nr_pe) |
1723 | state->trcprocselr = etm4x_read32(csa, TRCPROCSELR); |
1724 | state->trcconfigr = etm4x_read32(csa, TRCCONFIGR); |
1725 | state->trcauxctlr = etm4x_read32(csa, TRCAUXCTLR); |
1726 | state->trceventctl0r = etm4x_read32(csa, TRCEVENTCTL0R); |
1727 | state->trceventctl1r = etm4x_read32(csa, TRCEVENTCTL1R); |
1728 | if (drvdata->stallctl) |
1729 | state->trcstallctlr = etm4x_read32(csa, TRCSTALLCTLR); |
1730 | state->trctsctlr = etm4x_read32(csa, TRCTSCTLR); |
1731 | state->trcsyncpr = etm4x_read32(csa, TRCSYNCPR); |
1732 | state->trcccctlr = etm4x_read32(csa, TRCCCCTLR); |
1733 | state->trcbbctlr = etm4x_read32(csa, TRCBBCTLR); |
1734 | state->trctraceidr = etm4x_read32(csa, TRCTRACEIDR); |
1735 | state->trcqctlr = etm4x_read32(csa, TRCQCTLR); |
1736 | |
1737 | state->trcvictlr = etm4x_read32(csa, TRCVICTLR); |
1738 | state->trcviiectlr = etm4x_read32(csa, TRCVIIECTLR); |
1739 | state->trcvissctlr = etm4x_read32(csa, TRCVISSCTLR); |
1740 | if (drvdata->nr_pe_cmp) |
1741 | state->trcvipcssctlr = etm4x_read32(csa, TRCVIPCSSCTLR); |
1742 | state->trcvdctlr = etm4x_read32(csa, TRCVDCTLR); |
1743 | state->trcvdsacctlr = etm4x_read32(csa, TRCVDSACCTLR); |
1744 | state->trcvdarcctlr = etm4x_read32(csa, TRCVDARCCTLR); |
1745 | |
1746 | for (i = 0; i < drvdata->nrseqstate - 1; i++) |
1747 | state->trcseqevr[i] = etm4x_read32(csa, TRCSEQEVRn(i)); |
1748 | |
1749 | if (drvdata->nrseqstate) { |
1750 | state->trcseqrstevr = etm4x_read32(csa, TRCSEQRSTEVR); |
1751 | state->trcseqstr = etm4x_read32(csa, TRCSEQSTR); |
1752 | } |
1753 | state->trcextinselr = etm4x_read32(csa, TRCEXTINSELR); |
1754 | |
1755 | for (i = 0; i < drvdata->nr_cntr; i++) { |
1756 | state->trccntrldvr[i] = etm4x_read32(csa, TRCCNTRLDVRn(i)); |
1757 | state->trccntctlr[i] = etm4x_read32(csa, TRCCNTCTLRn(i)); |
1758 | state->trccntvr[i] = etm4x_read32(csa, TRCCNTVRn(i)); |
1759 | } |
1760 | |
1761 | for (i = 0; i < drvdata->nr_resource * 2; i++) |
1762 | state->trcrsctlr[i] = etm4x_read32(csa, TRCRSCTLRn(i)); |
1763 | |
1764 | for (i = 0; i < drvdata->nr_ss_cmp; i++) { |
1765 | state->trcssccr[i] = etm4x_read32(csa, TRCSSCCRn(i)); |
1766 | state->trcsscsr[i] = etm4x_read32(csa, TRCSSCSRn(i)); |
1767 | if (etm4x_sspcicrn_present(drvdata, n: i)) |
1768 | state->trcsspcicr[i] = etm4x_read32(csa, TRCSSPCICRn(i)); |
1769 | } |
1770 | |
1771 | for (i = 0; i < drvdata->nr_addr_cmp * 2; i++) { |
1772 | state->trcacvr[i] = etm4x_read64(csa, TRCACVRn(i)); |
1773 | state->trcacatr[i] = etm4x_read64(csa, TRCACATRn(i)); |
1774 | } |
1775 | |
1776 | /* |
1777 | * Data trace stream is architecturally prohibited for A profile cores |
1778 | * so we don't save (or later restore) trcdvcvr and trcdvcmr - As per |
1779 | * section 1.3.4 ("Possible functional configurations of an ETMv4 trace |
1780 | * unit") of ARM IHI 0064D. |
1781 | */ |
1782 | |
1783 | for (i = 0; i < drvdata->numcidc; i++) |
1784 | state->trccidcvr[i] = etm4x_read64(csa, TRCCIDCVRn(i)); |
1785 | |
1786 | for (i = 0; i < drvdata->numvmidc; i++) |
1787 | state->trcvmidcvr[i] = etm4x_read64(csa, TRCVMIDCVRn(i)); |
1788 | |
1789 | state->trccidcctlr0 = etm4x_read32(csa, TRCCIDCCTLR0); |
1790 | if (drvdata->numcidc > 4) |
1791 | state->trccidcctlr1 = etm4x_read32(csa, TRCCIDCCTLR1); |
1792 | |
1793 | state->trcvmidcctlr0 = etm4x_read32(csa, TRCVMIDCCTLR0); |
1794 | if (drvdata->numvmidc > 4) |
1795 | state->trcvmidcctlr0 = etm4x_read32(csa, TRCVMIDCCTLR1); |
1796 | |
1797 | state->trcclaimset = etm4x_read32(csa, TRCCLAIMCLR); |
1798 | |
1799 | if (!drvdata->skip_power_up) |
1800 | state->trcpdcr = etm4x_read32(csa, TRCPDCR); |
1801 | |
1802 | /* wait for TRCSTATR.IDLE to go up */ |
1803 | if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, value: 1)) { |
1804 | dev_err(etm_dev, |
1805 | "timeout while waiting for Idle Trace Status\n" ); |
1806 | etm4_os_unlock(drvdata); |
1807 | ret = -EBUSY; |
1808 | goto out; |
1809 | } |
1810 | |
1811 | drvdata->state_needs_restore = true; |
1812 | |
1813 | /* |
1814 | * Power can be removed from the trace unit now. We do this to |
1815 | * potentially save power on systems that respect the TRCPDCR_PU |
1816 | * despite requesting software to save/restore state. |
1817 | */ |
1818 | if (!drvdata->skip_power_up) |
1819 | etm4x_relaxed_write32(csa, (state->trcpdcr & ~TRCPDCR_PU), |
1820 | TRCPDCR); |
1821 | out: |
1822 | etm4_cs_lock(drvdata, csa); |
1823 | return ret; |
1824 | } |
1825 | |
1826 | static int etm4_cpu_save(struct etmv4_drvdata *drvdata) |
1827 | { |
1828 | int ret = 0; |
1829 | |
1830 | /* Save the TRFCR irrespective of whether the ETM is ON */ |
1831 | if (drvdata->trfcr) |
1832 | drvdata->save_trfcr = read_trfcr(); |
1833 | /* |
1834 | * Save and restore the ETM Trace registers only if |
1835 | * the ETM is active. |
1836 | */ |
1837 | if (coresight_get_mode(csdev: drvdata->csdev) && drvdata->save_state) |
1838 | ret = __etm4_cpu_save(drvdata); |
1839 | return ret; |
1840 | } |
1841 | |
1842 | static void __etm4_cpu_restore(struct etmv4_drvdata *drvdata) |
1843 | { |
1844 | int i; |
1845 | struct etmv4_save_state *state = drvdata->save_state; |
1846 | struct csdev_access tmp_csa = CSDEV_ACCESS_IOMEM(drvdata->base); |
1847 | struct csdev_access *csa = &tmp_csa; |
1848 | |
1849 | etm4_cs_unlock(drvdata, csa); |
1850 | etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET); |
1851 | |
1852 | etm4x_relaxed_write32(csa, state->trcprgctlr, TRCPRGCTLR); |
1853 | if (drvdata->nr_pe) |
1854 | etm4x_relaxed_write32(csa, state->trcprocselr, TRCPROCSELR); |
1855 | etm4x_relaxed_write32(csa, state->trcconfigr, TRCCONFIGR); |
1856 | etm4x_relaxed_write32(csa, state->trcauxctlr, TRCAUXCTLR); |
1857 | etm4x_relaxed_write32(csa, state->trceventctl0r, TRCEVENTCTL0R); |
1858 | etm4x_relaxed_write32(csa, state->trceventctl1r, TRCEVENTCTL1R); |
1859 | if (drvdata->stallctl) |
1860 | etm4x_relaxed_write32(csa, state->trcstallctlr, TRCSTALLCTLR); |
1861 | etm4x_relaxed_write32(csa, state->trctsctlr, TRCTSCTLR); |
1862 | etm4x_relaxed_write32(csa, state->trcsyncpr, TRCSYNCPR); |
1863 | etm4x_relaxed_write32(csa, state->trcccctlr, TRCCCCTLR); |
1864 | etm4x_relaxed_write32(csa, state->trcbbctlr, TRCBBCTLR); |
1865 | etm4x_relaxed_write32(csa, state->trctraceidr, TRCTRACEIDR); |
1866 | etm4x_relaxed_write32(csa, state->trcqctlr, TRCQCTLR); |
1867 | |
1868 | etm4x_relaxed_write32(csa, state->trcvictlr, TRCVICTLR); |
1869 | etm4x_relaxed_write32(csa, state->trcviiectlr, TRCVIIECTLR); |
1870 | etm4x_relaxed_write32(csa, state->trcvissctlr, TRCVISSCTLR); |
1871 | if (drvdata->nr_pe_cmp) |
1872 | etm4x_relaxed_write32(csa, state->trcvipcssctlr, TRCVIPCSSCTLR); |
1873 | etm4x_relaxed_write32(csa, state->trcvdctlr, TRCVDCTLR); |
1874 | etm4x_relaxed_write32(csa, state->trcvdsacctlr, TRCVDSACCTLR); |
1875 | etm4x_relaxed_write32(csa, state->trcvdarcctlr, TRCVDARCCTLR); |
1876 | |
1877 | for (i = 0; i < drvdata->nrseqstate - 1; i++) |
1878 | etm4x_relaxed_write32(csa, state->trcseqevr[i], TRCSEQEVRn(i)); |
1879 | |
1880 | if (drvdata->nrseqstate) { |
1881 | etm4x_relaxed_write32(csa, state->trcseqrstevr, TRCSEQRSTEVR); |
1882 | etm4x_relaxed_write32(csa, state->trcseqstr, TRCSEQSTR); |
1883 | } |
1884 | etm4x_relaxed_write32(csa, state->trcextinselr, TRCEXTINSELR); |
1885 | |
1886 | for (i = 0; i < drvdata->nr_cntr; i++) { |
1887 | etm4x_relaxed_write32(csa, state->trccntrldvr[i], TRCCNTRLDVRn(i)); |
1888 | etm4x_relaxed_write32(csa, state->trccntctlr[i], TRCCNTCTLRn(i)); |
1889 | etm4x_relaxed_write32(csa, state->trccntvr[i], TRCCNTVRn(i)); |
1890 | } |
1891 | |
1892 | for (i = 0; i < drvdata->nr_resource * 2; i++) |
1893 | etm4x_relaxed_write32(csa, state->trcrsctlr[i], TRCRSCTLRn(i)); |
1894 | |
1895 | for (i = 0; i < drvdata->nr_ss_cmp; i++) { |
1896 | etm4x_relaxed_write32(csa, state->trcssccr[i], TRCSSCCRn(i)); |
1897 | etm4x_relaxed_write32(csa, state->trcsscsr[i], TRCSSCSRn(i)); |
1898 | if (etm4x_sspcicrn_present(drvdata, n: i)) |
1899 | etm4x_relaxed_write32(csa, state->trcsspcicr[i], TRCSSPCICRn(i)); |
1900 | } |
1901 | |
1902 | for (i = 0; i < drvdata->nr_addr_cmp * 2; i++) { |
1903 | etm4x_relaxed_write64(csa, state->trcacvr[i], TRCACVRn(i)); |
1904 | etm4x_relaxed_write64(csa, state->trcacatr[i], TRCACATRn(i)); |
1905 | } |
1906 | |
1907 | for (i = 0; i < drvdata->numcidc; i++) |
1908 | etm4x_relaxed_write64(csa, state->trccidcvr[i], TRCCIDCVRn(i)); |
1909 | |
1910 | for (i = 0; i < drvdata->numvmidc; i++) |
1911 | etm4x_relaxed_write64(csa, state->trcvmidcvr[i], TRCVMIDCVRn(i)); |
1912 | |
1913 | etm4x_relaxed_write32(csa, state->trccidcctlr0, TRCCIDCCTLR0); |
1914 | if (drvdata->numcidc > 4) |
1915 | etm4x_relaxed_write32(csa, state->trccidcctlr1, TRCCIDCCTLR1); |
1916 | |
1917 | etm4x_relaxed_write32(csa, state->trcvmidcctlr0, TRCVMIDCCTLR0); |
1918 | if (drvdata->numvmidc > 4) |
1919 | etm4x_relaxed_write32(csa, state->trcvmidcctlr0, TRCVMIDCCTLR1); |
1920 | |
1921 | etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET); |
1922 | |
1923 | if (!drvdata->skip_power_up) |
1924 | etm4x_relaxed_write32(csa, state->trcpdcr, TRCPDCR); |
1925 | |
1926 | drvdata->state_needs_restore = false; |
1927 | |
1928 | /* |
1929 | * As recommended by section 4.3.7 ("Synchronization when using the |
1930 | * memory-mapped interface") of ARM IHI 0064D |
1931 | */ |
1932 | dsb(sy); |
1933 | isb(); |
1934 | |
1935 | /* Unlock the OS lock to re-enable trace and external debug access */ |
1936 | etm4_os_unlock(drvdata); |
1937 | etm4_cs_lock(drvdata, csa); |
1938 | } |
1939 | |
1940 | static void etm4_cpu_restore(struct etmv4_drvdata *drvdata) |
1941 | { |
1942 | if (drvdata->trfcr) |
1943 | write_trfcr(val: drvdata->save_trfcr); |
1944 | if (drvdata->state_needs_restore) |
1945 | __etm4_cpu_restore(drvdata); |
1946 | } |
1947 | |
1948 | static int etm4_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd, |
1949 | void *v) |
1950 | { |
1951 | struct etmv4_drvdata *drvdata; |
1952 | unsigned int cpu = smp_processor_id(); |
1953 | |
1954 | if (!etmdrvdata[cpu]) |
1955 | return NOTIFY_OK; |
1956 | |
1957 | drvdata = etmdrvdata[cpu]; |
1958 | |
1959 | if (WARN_ON_ONCE(drvdata->cpu != cpu)) |
1960 | return NOTIFY_BAD; |
1961 | |
1962 | switch (cmd) { |
1963 | case CPU_PM_ENTER: |
1964 | if (etm4_cpu_save(drvdata)) |
1965 | return NOTIFY_BAD; |
1966 | break; |
1967 | case CPU_PM_EXIT: |
1968 | case CPU_PM_ENTER_FAILED: |
1969 | etm4_cpu_restore(drvdata); |
1970 | break; |
1971 | default: |
1972 | return NOTIFY_DONE; |
1973 | } |
1974 | |
1975 | return NOTIFY_OK; |
1976 | } |
1977 | |
1978 | static struct notifier_block etm4_cpu_pm_nb = { |
1979 | .notifier_call = etm4_cpu_pm_notify, |
1980 | }; |
1981 | |
1982 | /* Setup PM. Deals with error conditions and counts */ |
1983 | static int __init etm4_pm_setup(void) |
1984 | { |
1985 | int ret; |
1986 | |
1987 | ret = cpu_pm_register_notifier(nb: &etm4_cpu_pm_nb); |
1988 | if (ret) |
1989 | return ret; |
1990 | |
1991 | ret = cpuhp_setup_state_nocalls(state: CPUHP_AP_ARM_CORESIGHT_STARTING, |
1992 | name: "arm/coresight4:starting" , |
1993 | startup: etm4_starting_cpu, teardown: etm4_dying_cpu); |
1994 | |
1995 | if (ret) |
1996 | goto unregister_notifier; |
1997 | |
1998 | ret = cpuhp_setup_state_nocalls(state: CPUHP_AP_ONLINE_DYN, |
1999 | name: "arm/coresight4:online" , |
2000 | startup: etm4_online_cpu, NULL); |
2001 | |
2002 | /* HP dyn state ID returned in ret on success */ |
2003 | if (ret > 0) { |
2004 | hp_online = ret; |
2005 | return 0; |
2006 | } |
2007 | |
2008 | /* failed dyn state - remove others */ |
2009 | cpuhp_remove_state_nocalls(state: CPUHP_AP_ARM_CORESIGHT_STARTING); |
2010 | |
2011 | unregister_notifier: |
2012 | cpu_pm_unregister_notifier(nb: &etm4_cpu_pm_nb); |
2013 | return ret; |
2014 | } |
2015 | |
2016 | static void etm4_pm_clear(void) |
2017 | { |
2018 | cpu_pm_unregister_notifier(nb: &etm4_cpu_pm_nb); |
2019 | cpuhp_remove_state_nocalls(state: CPUHP_AP_ARM_CORESIGHT_STARTING); |
2020 | if (hp_online) { |
2021 | cpuhp_remove_state_nocalls(state: hp_online); |
2022 | hp_online = 0; |
2023 | } |
2024 | } |
2025 | |
2026 | static int etm4_add_coresight_dev(struct etm4_init_arg *init_arg) |
2027 | { |
2028 | int ret; |
2029 | struct coresight_platform_data *pdata = NULL; |
2030 | struct device *dev = init_arg->dev; |
2031 | struct etmv4_drvdata *drvdata = dev_get_drvdata(dev); |
2032 | struct coresight_desc desc = { 0 }; |
2033 | u8 major, minor; |
2034 | char *type_name; |
2035 | |
2036 | if (!drvdata) |
2037 | return -EINVAL; |
2038 | |
2039 | desc.access = *init_arg->csa; |
2040 | |
2041 | if (!drvdata->arch) |
2042 | return -EINVAL; |
2043 | |
2044 | major = ETM_ARCH_MAJOR_VERSION(drvdata->arch); |
2045 | minor = ETM_ARCH_MINOR_VERSION(drvdata->arch); |
2046 | |
2047 | if (etm4x_is_ete(drvdata)) { |
2048 | type_name = "ete" ; |
2049 | /* ETE v1 has major version == 0b101. Adjust this for logging.*/ |
2050 | major -= 4; |
2051 | } else { |
2052 | type_name = "etm" ; |
2053 | } |
2054 | |
2055 | desc.name = devm_kasprintf(dev, GFP_KERNEL, |
2056 | fmt: "%s%d" , type_name, drvdata->cpu); |
2057 | if (!desc.name) |
2058 | return -ENOMEM; |
2059 | |
2060 | etm4_set_default(config: &drvdata->config); |
2061 | |
2062 | pdata = coresight_get_platform_data(dev); |
2063 | if (IS_ERR(ptr: pdata)) |
2064 | return PTR_ERR(ptr: pdata); |
2065 | |
2066 | dev->platform_data = pdata; |
2067 | |
2068 | desc.type = CORESIGHT_DEV_TYPE_SOURCE; |
2069 | desc.subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_PROC; |
2070 | desc.ops = &etm4_cs_ops; |
2071 | desc.pdata = pdata; |
2072 | desc.dev = dev; |
2073 | desc.groups = coresight_etmv4_groups; |
2074 | drvdata->csdev = coresight_register(desc: &desc); |
2075 | if (IS_ERR(ptr: drvdata->csdev)) |
2076 | return PTR_ERR(ptr: drvdata->csdev); |
2077 | |
2078 | ret = etm_perf_symlink(csdev: drvdata->csdev, link: true); |
2079 | if (ret) { |
2080 | coresight_unregister(csdev: drvdata->csdev); |
2081 | return ret; |
2082 | } |
2083 | |
2084 | /* register with config infrastructure & load any current features */ |
2085 | ret = etm4_cscfg_register(csdev: drvdata->csdev); |
2086 | if (ret) { |
2087 | coresight_unregister(csdev: drvdata->csdev); |
2088 | return ret; |
2089 | } |
2090 | |
2091 | etmdrvdata[drvdata->cpu] = drvdata; |
2092 | |
2093 | dev_info(&drvdata->csdev->dev, "CPU%d: %s v%d.%d initialized\n" , |
2094 | drvdata->cpu, type_name, major, minor); |
2095 | |
2096 | if (boot_enable) { |
2097 | coresight_enable_sysfs(csdev: drvdata->csdev); |
2098 | drvdata->boot_enable = true; |
2099 | } |
2100 | |
2101 | return 0; |
2102 | } |
2103 | |
2104 | static int etm4_probe(struct device *dev) |
2105 | { |
2106 | struct etmv4_drvdata *drvdata = dev_get_drvdata(dev); |
2107 | struct csdev_access access = { 0 }; |
2108 | struct etm4_init_arg init_arg = { 0 }; |
2109 | struct etm4_init_arg *delayed; |
2110 | |
2111 | if (WARN_ON(!drvdata)) |
2112 | return -ENOMEM; |
2113 | |
2114 | if (pm_save_enable == PARAM_PM_SAVE_FIRMWARE) |
2115 | pm_save_enable = coresight_loses_context_with_cpu(dev) ? |
2116 | PARAM_PM_SAVE_SELF_HOSTED : PARAM_PM_SAVE_NEVER; |
2117 | |
2118 | if (pm_save_enable != PARAM_PM_SAVE_NEVER) { |
2119 | drvdata->save_state = devm_kmalloc(dev, |
2120 | size: sizeof(struct etmv4_save_state), GFP_KERNEL); |
2121 | if (!drvdata->save_state) |
2122 | return -ENOMEM; |
2123 | } |
2124 | |
2125 | spin_lock_init(&drvdata->spinlock); |
2126 | |
2127 | drvdata->cpu = coresight_get_cpu(dev); |
2128 | if (drvdata->cpu < 0) |
2129 | return drvdata->cpu; |
2130 | |
2131 | init_arg.dev = dev; |
2132 | init_arg.csa = &access; |
2133 | |
2134 | /* |
2135 | * Serialize against CPUHP callbacks to avoid race condition |
2136 | * between the smp call and saving the delayed probe. |
2137 | */ |
2138 | cpus_read_lock(); |
2139 | if (smp_call_function_single(cpuid: drvdata->cpu, |
2140 | func: etm4_init_arch_data, info: &init_arg, wait: 1)) { |
2141 | /* The CPU was offline, try again once it comes online. */ |
2142 | delayed = devm_kmalloc(dev, size: sizeof(*delayed), GFP_KERNEL); |
2143 | if (!delayed) { |
2144 | cpus_read_unlock(); |
2145 | return -ENOMEM; |
2146 | } |
2147 | |
2148 | *delayed = init_arg; |
2149 | |
2150 | per_cpu(delayed_probe, drvdata->cpu) = delayed; |
2151 | |
2152 | cpus_read_unlock(); |
2153 | return 0; |
2154 | } |
2155 | cpus_read_unlock(); |
2156 | |
2157 | return etm4_add_coresight_dev(init_arg: &init_arg); |
2158 | } |
2159 | |
2160 | static int etm4_probe_amba(struct amba_device *adev, const struct amba_id *id) |
2161 | { |
2162 | struct etmv4_drvdata *drvdata; |
2163 | void __iomem *base; |
2164 | struct device *dev = &adev->dev; |
2165 | struct resource *res = &adev->res; |
2166 | int ret; |
2167 | |
2168 | /* Validity for the resource is already checked by the AMBA core */ |
2169 | base = devm_ioremap_resource(dev, res); |
2170 | if (IS_ERR(ptr: base)) |
2171 | return PTR_ERR(ptr: base); |
2172 | |
2173 | drvdata = devm_kzalloc(dev, size: sizeof(*drvdata), GFP_KERNEL); |
2174 | if (!drvdata) |
2175 | return -ENOMEM; |
2176 | |
2177 | drvdata->base = base; |
2178 | dev_set_drvdata(dev, data: drvdata); |
2179 | ret = etm4_probe(dev); |
2180 | if (!ret) |
2181 | pm_runtime_put(dev: &adev->dev); |
2182 | |
2183 | return ret; |
2184 | } |
2185 | |
2186 | static int etm4_probe_platform_dev(struct platform_device *pdev) |
2187 | { |
2188 | struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0); |
2189 | struct etmv4_drvdata *drvdata; |
2190 | int ret; |
2191 | |
2192 | drvdata = devm_kzalloc(dev: &pdev->dev, size: sizeof(*drvdata), GFP_KERNEL); |
2193 | if (!drvdata) |
2194 | return -ENOMEM; |
2195 | |
2196 | drvdata->pclk = coresight_get_enable_apb_pclk(dev: &pdev->dev); |
2197 | if (IS_ERR(ptr: drvdata->pclk)) |
2198 | return -ENODEV; |
2199 | |
2200 | if (res) { |
2201 | drvdata->base = devm_ioremap_resource(dev: &pdev->dev, res); |
2202 | if (IS_ERR(ptr: drvdata->base)) { |
2203 | clk_put(clk: drvdata->pclk); |
2204 | return PTR_ERR(ptr: drvdata->base); |
2205 | } |
2206 | } |
2207 | |
2208 | dev_set_drvdata(dev: &pdev->dev, data: drvdata); |
2209 | pm_runtime_get_noresume(dev: &pdev->dev); |
2210 | pm_runtime_set_active(dev: &pdev->dev); |
2211 | pm_runtime_enable(dev: &pdev->dev); |
2212 | |
2213 | ret = etm4_probe(dev: &pdev->dev); |
2214 | |
2215 | pm_runtime_put(dev: &pdev->dev); |
2216 | return ret; |
2217 | } |
2218 | |
2219 | static int etm4_probe_cpu(unsigned int cpu) |
2220 | { |
2221 | int ret; |
2222 | struct etm4_init_arg init_arg; |
2223 | struct csdev_access access = { 0 }; |
2224 | struct etm4_init_arg *iap = *this_cpu_ptr(&delayed_probe); |
2225 | |
2226 | if (!iap) |
2227 | return 0; |
2228 | |
2229 | init_arg = *iap; |
2230 | devm_kfree(dev: init_arg.dev, p: iap); |
2231 | *this_cpu_ptr(&delayed_probe) = NULL; |
2232 | |
2233 | ret = pm_runtime_resume_and_get(dev: init_arg.dev); |
2234 | if (ret < 0) { |
2235 | dev_err(init_arg.dev, "Failed to get PM runtime!\n" ); |
2236 | return 0; |
2237 | } |
2238 | |
2239 | init_arg.csa = &access; |
2240 | etm4_init_arch_data(info: &init_arg); |
2241 | |
2242 | etm4_add_coresight_dev(init_arg: &init_arg); |
2243 | |
2244 | pm_runtime_put(dev: init_arg.dev); |
2245 | return 0; |
2246 | } |
2247 | |
2248 | static struct amba_cs_uci_id uci_id_etm4[] = { |
2249 | { |
2250 | /* ETMv4 UCI data */ |
2251 | .devarch = ETM_DEVARCH_ETMv4x_ARCH, |
2252 | .devarch_mask = ETM_DEVARCH_ID_MASK, |
2253 | .devtype = CS_DEVTYPE_PE_TRACE, |
2254 | } |
2255 | }; |
2256 | |
2257 | static void clear_etmdrvdata(void *info) |
2258 | { |
2259 | int cpu = *(int *)info; |
2260 | |
2261 | etmdrvdata[cpu] = NULL; |
2262 | per_cpu(delayed_probe, cpu) = NULL; |
2263 | } |
2264 | |
2265 | static void etm4_remove_dev(struct etmv4_drvdata *drvdata) |
2266 | { |
2267 | bool had_delayed_probe; |
2268 | /* |
2269 | * Taking hotplug lock here to avoid racing between etm4_remove_dev() |
2270 | * and CPU hotplug call backs. |
2271 | */ |
2272 | cpus_read_lock(); |
2273 | |
2274 | had_delayed_probe = per_cpu(delayed_probe, drvdata->cpu); |
2275 | |
2276 | /* |
2277 | * The readers for etmdrvdata[] are CPU hotplug call backs |
2278 | * and PM notification call backs. Change etmdrvdata[i] on |
2279 | * CPU i ensures these call backs has consistent view |
2280 | * inside one call back function. |
2281 | */ |
2282 | if (smp_call_function_single(cpuid: drvdata->cpu, func: clear_etmdrvdata, info: &drvdata->cpu, wait: 1)) |
2283 | clear_etmdrvdata(info: &drvdata->cpu); |
2284 | |
2285 | cpus_read_unlock(); |
2286 | |
2287 | if (!had_delayed_probe) { |
2288 | etm_perf_symlink(csdev: drvdata->csdev, link: false); |
2289 | cscfg_unregister_csdev(csdev: drvdata->csdev); |
2290 | coresight_unregister(csdev: drvdata->csdev); |
2291 | } |
2292 | } |
2293 | |
2294 | static void etm4_remove_amba(struct amba_device *adev) |
2295 | { |
2296 | struct etmv4_drvdata *drvdata = dev_get_drvdata(dev: &adev->dev); |
2297 | |
2298 | if (drvdata) |
2299 | etm4_remove_dev(drvdata); |
2300 | } |
2301 | |
2302 | static void etm4_remove_platform_dev(struct platform_device *pdev) |
2303 | { |
2304 | struct etmv4_drvdata *drvdata = dev_get_drvdata(dev: &pdev->dev); |
2305 | |
2306 | if (drvdata) |
2307 | etm4_remove_dev(drvdata); |
2308 | pm_runtime_disable(dev: &pdev->dev); |
2309 | |
2310 | if (drvdata && !IS_ERR_OR_NULL(ptr: drvdata->pclk)) |
2311 | clk_put(clk: drvdata->pclk); |
2312 | } |
2313 | |
2314 | static const struct amba_id etm4_ids[] = { |
2315 | CS_AMBA_ID(0x000bb95d), /* Cortex-A53 */ |
2316 | CS_AMBA_ID(0x000bb95e), /* Cortex-A57 */ |
2317 | CS_AMBA_ID(0x000bb95a), /* Cortex-A72 */ |
2318 | CS_AMBA_ID(0x000bb959), /* Cortex-A73 */ |
2319 | CS_AMBA_UCI_ID(0x000bb9da, uci_id_etm4),/* Cortex-A35 */ |
2320 | CS_AMBA_UCI_ID(0x000bbd05, uci_id_etm4),/* Cortex-A55 */ |
2321 | CS_AMBA_UCI_ID(0x000bbd0a, uci_id_etm4),/* Cortex-A75 */ |
2322 | CS_AMBA_UCI_ID(0x000bbd0c, uci_id_etm4),/* Neoverse N1 */ |
2323 | CS_AMBA_UCI_ID(0x000bbd41, uci_id_etm4),/* Cortex-A78 */ |
2324 | CS_AMBA_UCI_ID(0x000f0205, uci_id_etm4),/* Qualcomm Kryo */ |
2325 | CS_AMBA_UCI_ID(0x000f0211, uci_id_etm4),/* Qualcomm Kryo */ |
2326 | CS_AMBA_UCI_ID(0x000bb802, uci_id_etm4),/* Qualcomm Kryo 385 Cortex-A55 */ |
2327 | CS_AMBA_UCI_ID(0x000bb803, uci_id_etm4),/* Qualcomm Kryo 385 Cortex-A75 */ |
2328 | CS_AMBA_UCI_ID(0x000bb805, uci_id_etm4),/* Qualcomm Kryo 4XX Cortex-A55 */ |
2329 | CS_AMBA_UCI_ID(0x000bb804, uci_id_etm4),/* Qualcomm Kryo 4XX Cortex-A76 */ |
2330 | CS_AMBA_UCI_ID(0x000bbd0d, uci_id_etm4),/* Qualcomm Kryo 5XX Cortex-A77 */ |
2331 | CS_AMBA_UCI_ID(0x000cc0af, uci_id_etm4),/* Marvell ThunderX2 */ |
2332 | CS_AMBA_UCI_ID(0x000b6d01, uci_id_etm4),/* HiSilicon-Hip08 */ |
2333 | CS_AMBA_UCI_ID(0x000b6d02, uci_id_etm4),/* HiSilicon-Hip09 */ |
2334 | /* |
2335 | * Match all PIDs with ETM4 DEVARCH. No need for adding any of the new |
2336 | * CPUs to the list here. |
2337 | */ |
2338 | CS_AMBA_MATCH_ALL_UCI(uci_id_etm4), |
2339 | {}, |
2340 | }; |
2341 | |
2342 | MODULE_DEVICE_TABLE(amba, etm4_ids); |
2343 | |
2344 | static struct amba_driver etm4x_amba_driver = { |
2345 | .drv = { |
2346 | .name = "coresight-etm4x" , |
2347 | .owner = THIS_MODULE, |
2348 | .suppress_bind_attrs = true, |
2349 | }, |
2350 | .probe = etm4_probe_amba, |
2351 | .remove = etm4_remove_amba, |
2352 | .id_table = etm4_ids, |
2353 | }; |
2354 | |
2355 | #ifdef CONFIG_PM |
2356 | static int etm4_runtime_suspend(struct device *dev) |
2357 | { |
2358 | struct etmv4_drvdata *drvdata = dev_get_drvdata(dev); |
2359 | |
2360 | if (drvdata->pclk && !IS_ERR(ptr: drvdata->pclk)) |
2361 | clk_disable_unprepare(clk: drvdata->pclk); |
2362 | |
2363 | return 0; |
2364 | } |
2365 | |
2366 | static int etm4_runtime_resume(struct device *dev) |
2367 | { |
2368 | struct etmv4_drvdata *drvdata = dev_get_drvdata(dev); |
2369 | |
2370 | if (drvdata->pclk && !IS_ERR(ptr: drvdata->pclk)) |
2371 | clk_prepare_enable(clk: drvdata->pclk); |
2372 | |
2373 | return 0; |
2374 | } |
2375 | #endif |
2376 | |
2377 | static const struct dev_pm_ops etm4_dev_pm_ops = { |
2378 | SET_RUNTIME_PM_OPS(etm4_runtime_suspend, etm4_runtime_resume, NULL) |
2379 | }; |
2380 | |
2381 | static const struct of_device_id etm4_sysreg_match[] = { |
2382 | { .compatible = "arm,coresight-etm4x-sysreg" }, |
2383 | { .compatible = "arm,embedded-trace-extension" }, |
2384 | {} |
2385 | }; |
2386 | |
2387 | #ifdef CONFIG_ACPI |
2388 | static const struct acpi_device_id etm4x_acpi_ids[] = { |
2389 | {"ARMHC500" , 0, 0, 0}, /* ARM CoreSight ETM4x */ |
2390 | {} |
2391 | }; |
2392 | MODULE_DEVICE_TABLE(acpi, etm4x_acpi_ids); |
2393 | #endif |
2394 | |
2395 | static struct platform_driver etm4_platform_driver = { |
2396 | .probe = etm4_probe_platform_dev, |
2397 | .remove_new = etm4_remove_platform_dev, |
2398 | .driver = { |
2399 | .name = "coresight-etm4x" , |
2400 | .of_match_table = etm4_sysreg_match, |
2401 | .acpi_match_table = ACPI_PTR(etm4x_acpi_ids), |
2402 | .suppress_bind_attrs = true, |
2403 | .pm = &etm4_dev_pm_ops, |
2404 | }, |
2405 | }; |
2406 | |
2407 | static int __init etm4x_init(void) |
2408 | { |
2409 | int ret; |
2410 | |
2411 | ret = etm4_pm_setup(); |
2412 | |
2413 | /* etm4_pm_setup() does its own cleanup - exit on error */ |
2414 | if (ret) |
2415 | return ret; |
2416 | |
2417 | ret = amba_driver_register(drv: &etm4x_amba_driver); |
2418 | if (ret) { |
2419 | pr_err("Error registering etm4x AMBA driver\n" ); |
2420 | goto clear_pm; |
2421 | } |
2422 | |
2423 | ret = platform_driver_register(&etm4_platform_driver); |
2424 | if (!ret) |
2425 | return 0; |
2426 | |
2427 | pr_err("Error registering etm4x platform driver\n" ); |
2428 | amba_driver_unregister(drv: &etm4x_amba_driver); |
2429 | |
2430 | clear_pm: |
2431 | etm4_pm_clear(); |
2432 | return ret; |
2433 | } |
2434 | |
2435 | static void __exit etm4x_exit(void) |
2436 | { |
2437 | amba_driver_unregister(drv: &etm4x_amba_driver); |
2438 | platform_driver_unregister(&etm4_platform_driver); |
2439 | etm4_pm_clear(); |
2440 | } |
2441 | |
2442 | module_init(etm4x_init); |
2443 | module_exit(etm4x_exit); |
2444 | |
2445 | MODULE_AUTHOR("Pratik Patel <pratikp@codeaurora.org>" ); |
2446 | MODULE_AUTHOR("Mathieu Poirier <mathieu.poirier@linaro.org>" ); |
2447 | MODULE_DESCRIPTION("Arm CoreSight Program Flow Trace v4.x driver" ); |
2448 | MODULE_LICENSE("GPL v2" ); |
2449 | |