1// SPDX-License-Identifier: GPL-2.0
2/*
3 * This driver enables Trace Buffer Extension (TRBE) as a per-cpu coresight
4 * sink device could then pair with an appropriate per-cpu coresight source
5 * device (ETE) thus generating required trace data. Trace can be enabled
6 * via the perf framework.
7 *
8 * The AUX buffer handling is inspired from Arm SPE PMU driver.
9 *
10 * Copyright (C) 2020 ARM Ltd.
11 *
12 * Author: Anshuman Khandual <anshuman.khandual@arm.com>
13 */
14#define DRVNAME "arm_trbe"
15
16#define pr_fmt(fmt) DRVNAME ": " fmt
17
18#include <asm/barrier.h>
19#include <asm/cpufeature.h>
20
21#include "coresight-self-hosted-trace.h"
22#include "coresight-trbe.h"
23
24#define PERF_IDX2OFF(idx, buf) ((idx) % ((buf)->nr_pages << PAGE_SHIFT))
25
26/*
27 * A padding packet that will help the user space tools
28 * in skipping relevant sections in the captured trace
29 * data which could not be decoded. TRBE doesn't support
30 * formatting the trace data, unlike the legacy CoreSight
31 * sinks and thus we use ETE trace packets to pad the
32 * sections of the buffer.
33 */
34#define ETE_IGNORE_PACKET 0x70
35
36/*
37 * Minimum amount of meaningful trace will contain:
38 * A-Sync, Trace Info, Trace On, Address, Atom.
39 * This is about 44bytes of ETE trace. To be on
40 * the safer side, we assume 64bytes is the minimum
41 * space required for a meaningful session, before
42 * we hit a "WRAP" event.
43 */
44#define TRBE_TRACE_MIN_BUF_SIZE 64
45
46enum trbe_fault_action {
47 TRBE_FAULT_ACT_WRAP,
48 TRBE_FAULT_ACT_SPURIOUS,
49 TRBE_FAULT_ACT_FATAL,
50};
51
52struct trbe_buf {
53 /*
54 * Even though trbe_base represents vmap()
55 * mapped allocated buffer's start address,
56 * it's being as unsigned long for various
57 * arithmetic and comparision operations &
58 * also to be consistent with trbe_write &
59 * trbe_limit sibling pointers.
60 */
61 unsigned long trbe_base;
62 /* The base programmed into the TRBE */
63 unsigned long trbe_hw_base;
64 unsigned long trbe_limit;
65 unsigned long trbe_write;
66 int nr_pages;
67 void **pages;
68 bool snapshot;
69 struct trbe_cpudata *cpudata;
70};
71
72/*
73 * TRBE erratum list
74 *
75 * The errata are defined in arm64 generic cpu_errata framework.
76 * Since the errata work arounds could be applied individually
77 * to the affected CPUs inside the TRBE driver, we need to know if
78 * a given CPU is affected by the erratum. Unlike the other erratum
79 * work arounds, TRBE driver needs to check multiple times during
80 * a trace session. Thus we need a quicker access to per-CPU
81 * errata and not issue costly this_cpu_has_cap() everytime.
82 * We keep a set of the affected errata in trbe_cpudata, per TRBE.
83 *
84 * We rely on the corresponding cpucaps to be defined for a given
85 * TRBE erratum. We map the given cpucap into a TRBE internal number
86 * to make the tracking of the errata lean.
87 *
88 * This helps in :
89 * - Not duplicating the detection logic
90 * - Streamlined detection of erratum across the system
91 */
92#define TRBE_WORKAROUND_OVERWRITE_FILL_MODE 0
93#define TRBE_WORKAROUND_WRITE_OUT_OF_RANGE 1
94#define TRBE_NEEDS_DRAIN_AFTER_DISABLE 2
95#define TRBE_NEEDS_CTXT_SYNC_AFTER_ENABLE 3
96#define TRBE_IS_BROKEN 4
97
98static int trbe_errata_cpucaps[] = {
99 [TRBE_WORKAROUND_OVERWRITE_FILL_MODE] = ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE,
100 [TRBE_WORKAROUND_WRITE_OUT_OF_RANGE] = ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE,
101 [TRBE_NEEDS_DRAIN_AFTER_DISABLE] = ARM64_WORKAROUND_2064142,
102 [TRBE_NEEDS_CTXT_SYNC_AFTER_ENABLE] = ARM64_WORKAROUND_2038923,
103 [TRBE_IS_BROKEN] = ARM64_WORKAROUND_1902691,
104 -1, /* Sentinel, must be the last entry */
105};
106
107/* The total number of listed errata in trbe_errata_cpucaps */
108#define TRBE_ERRATA_MAX (ARRAY_SIZE(trbe_errata_cpucaps) - 1)
109
110/*
111 * Safe limit for the number of bytes that may be overwritten
112 * when ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE is triggered.
113 */
114#define TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES 256
115
116/*
117 * struct trbe_cpudata: TRBE instance specific data
118 * @trbe_flag - TRBE dirty/access flag support
119 * @trbe_hw_align - Actual TRBE alignment required for TRBPTR_EL1.
120 * @trbe_align - Software alignment used for the TRBPTR_EL1.
121 * @cpu - CPU this TRBE belongs to.
122 * @mode - Mode of current operation. (perf/disabled)
123 * @drvdata - TRBE specific drvdata
124 * @errata - Bit map for the errata on this TRBE.
125 */
126struct trbe_cpudata {
127 bool trbe_flag;
128 u64 trbe_hw_align;
129 u64 trbe_align;
130 int cpu;
131 enum cs_mode mode;
132 struct trbe_buf *buf;
133 struct trbe_drvdata *drvdata;
134 DECLARE_BITMAP(errata, TRBE_ERRATA_MAX);
135};
136
137struct trbe_drvdata {
138 struct trbe_cpudata __percpu *cpudata;
139 struct perf_output_handle * __percpu *handle;
140 struct hlist_node hotplug_node;
141 int irq;
142 cpumask_t supported_cpus;
143 enum cpuhp_state trbe_online;
144 struct platform_device *pdev;
145};
146
147static void trbe_check_errata(struct trbe_cpudata *cpudata)
148{
149 int i;
150
151 for (i = 0; i < TRBE_ERRATA_MAX; i++) {
152 int cap = trbe_errata_cpucaps[i];
153
154 if (WARN_ON_ONCE(cap < 0))
155 return;
156 if (this_cpu_has_cap(cap))
157 set_bit(i, cpudata->errata);
158 }
159}
160
161static inline bool trbe_has_erratum(struct trbe_cpudata *cpudata, int i)
162{
163 return (i < TRBE_ERRATA_MAX) && test_bit(i, cpudata->errata);
164}
165
166static inline bool trbe_may_overwrite_in_fill_mode(struct trbe_cpudata *cpudata)
167{
168 return trbe_has_erratum(cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE);
169}
170
171static inline bool trbe_may_write_out_of_range(struct trbe_cpudata *cpudata)
172{
173 return trbe_has_erratum(cpudata, TRBE_WORKAROUND_WRITE_OUT_OF_RANGE);
174}
175
176static inline bool trbe_needs_drain_after_disable(struct trbe_cpudata *cpudata)
177{
178 /*
179 * Errata affected TRBE implementation will need TSB CSYNC and
180 * DSB in order to prevent subsequent writes into certain TRBE
181 * system registers from being ignored and not effected.
182 */
183 return trbe_has_erratum(cpudata, TRBE_NEEDS_DRAIN_AFTER_DISABLE);
184}
185
186static inline bool trbe_needs_ctxt_sync_after_enable(struct trbe_cpudata *cpudata)
187{
188 /*
189 * Errata affected TRBE implementation will need an additional
190 * context synchronization in order to prevent an inconsistent
191 * TRBE prohibited region view on the CPU which could possibly
192 * corrupt the TRBE buffer or the TRBE state.
193 */
194 return trbe_has_erratum(cpudata, TRBE_NEEDS_CTXT_SYNC_AFTER_ENABLE);
195}
196
197static inline bool trbe_is_broken(struct trbe_cpudata *cpudata)
198{
199 return trbe_has_erratum(cpudata, TRBE_IS_BROKEN);
200}
201
202static int trbe_alloc_node(struct perf_event *event)
203{
204 if (event->cpu == -1)
205 return NUMA_NO_NODE;
206 return cpu_to_node(cpu: event->cpu);
207}
208
209static inline void trbe_drain_buffer(void)
210{
211 tsb_csync();
212 dsb(nsh);
213}
214
215static inline void set_trbe_enabled(struct trbe_cpudata *cpudata, u64 trblimitr)
216{
217 /*
218 * Enable the TRBE without clearing LIMITPTR which
219 * might be required for fetching the buffer limits.
220 */
221 trblimitr |= TRBLIMITR_EL1_E;
222 write_sysreg_s(trblimitr, SYS_TRBLIMITR_EL1);
223
224 /* Synchronize the TRBE enable event */
225 isb();
226
227 if (trbe_needs_ctxt_sync_after_enable(cpudata))
228 isb();
229}
230
231static inline void set_trbe_disabled(struct trbe_cpudata *cpudata)
232{
233 u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1);
234
235 /*
236 * Disable the TRBE without clearing LIMITPTR which
237 * might be required for fetching the buffer limits.
238 */
239 trblimitr &= ~TRBLIMITR_EL1_E;
240 write_sysreg_s(trblimitr, SYS_TRBLIMITR_EL1);
241
242 if (trbe_needs_drain_after_disable(cpudata))
243 trbe_drain_buffer();
244 isb();
245}
246
247static void trbe_drain_and_disable_local(struct trbe_cpudata *cpudata)
248{
249 trbe_drain_buffer();
250 set_trbe_disabled(cpudata);
251}
252
253static void trbe_reset_local(struct trbe_cpudata *cpudata)
254{
255 trbe_drain_and_disable_local(cpudata);
256 write_sysreg_s(0, SYS_TRBLIMITR_EL1);
257 write_sysreg_s(0, SYS_TRBPTR_EL1);
258 write_sysreg_s(0, SYS_TRBBASER_EL1);
259 write_sysreg_s(0, SYS_TRBSR_EL1);
260}
261
262static void trbe_report_wrap_event(struct perf_output_handle *handle)
263{
264 /*
265 * Mark the buffer to indicate that there was a WRAP event by
266 * setting the COLLISION flag. This indicates to the user that
267 * the TRBE trace collection was stopped without stopping the
268 * ETE and thus there might be some amount of trace that was
269 * lost between the time the WRAP was detected and the IRQ
270 * was consumed by the CPU.
271 *
272 * Setting the TRUNCATED flag would move the event to STOPPED
273 * state unnecessarily, even when there is space left in the
274 * ring buffer. Using the COLLISION flag doesn't have this side
275 * effect. We only set TRUNCATED flag when there is no space
276 * left in the ring buffer.
277 */
278 perf_aux_output_flag(handle, PERF_AUX_FLAG_COLLISION);
279}
280
281static void trbe_stop_and_truncate_event(struct perf_output_handle *handle)
282{
283 struct trbe_buf *buf = etm_perf_sink_config(handle);
284
285 /*
286 * We cannot proceed with the buffer collection and we
287 * do not have any data for the current session. The
288 * etm_perf driver expects to close out the aux_buffer
289 * at event_stop(). So disable the TRBE here and leave
290 * the update_buffer() to return a 0 size.
291 */
292 trbe_drain_and_disable_local(cpudata: buf->cpudata);
293 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
294 perf_aux_output_end(handle, size: 0);
295 *this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL;
296}
297
298/*
299 * TRBE Buffer Management
300 *
301 * The TRBE buffer spans from the base pointer till the limit pointer. When enabled,
302 * it starts writing trace data from the write pointer onward till the limit pointer.
303 * When the write pointer reaches the address just before the limit pointer, it gets
304 * wrapped around again to the base pointer. This is called a TRBE wrap event, which
305 * generates a maintenance interrupt when operated in WRAP or FILL mode. This driver
306 * uses FILL mode, where the TRBE stops the trace collection at wrap event. The IRQ
307 * handler updates the AUX buffer and re-enables the TRBE with updated WRITE and
308 * LIMIT pointers.
309 *
310 * Wrap around with an IRQ
311 * ------ < ------ < ------- < ----- < -----
312 * | |
313 * ------ > ------ > ------- > ----- > -----
314 *
315 * +---------------+-----------------------+
316 * | | |
317 * +---------------+-----------------------+
318 * Base Pointer Write Pointer Limit Pointer
319 *
320 * The base and limit pointers always needs to be PAGE_SIZE aligned. But the write
321 * pointer can be aligned to the implementation defined TRBE trace buffer alignment
322 * as captured in trbe_cpudata->trbe_align.
323 *
324 *
325 * head tail wakeup
326 * +---------------------------------------+----- ~ ~ ------
327 * |$$$$$$$|################|$$$$$$$$$$$$$$| |
328 * +---------------------------------------+----- ~ ~ ------
329 * Base Pointer Write Pointer Limit Pointer
330 *
331 * The perf_output_handle indices (head, tail, wakeup) are monotonically increasing
332 * values which tracks all the driver writes and user reads from the perf auxiliary
333 * buffer. Generally [head..tail] is the area where the driver can write into unless
334 * the wakeup is behind the tail. Enabled TRBE buffer span needs to be adjusted and
335 * configured depending on the perf_output_handle indices, so that the driver does
336 * not override into areas in the perf auxiliary buffer which is being or yet to be
337 * consumed from the user space. The enabled TRBE buffer area is a moving subset of
338 * the allocated perf auxiliary buffer.
339 */
340
341static void __trbe_pad_buf(struct trbe_buf *buf, u64 offset, int len)
342{
343 memset((void *)buf->trbe_base + offset, ETE_IGNORE_PACKET, len);
344}
345
346static void trbe_pad_buf(struct perf_output_handle *handle, int len)
347{
348 struct trbe_buf *buf = etm_perf_sink_config(handle);
349 u64 head = PERF_IDX2OFF(handle->head, buf);
350
351 __trbe_pad_buf(buf, offset: head, len);
352 if (!buf->snapshot)
353 perf_aux_output_skip(handle, size: len);
354}
355
356static unsigned long trbe_snapshot_offset(struct perf_output_handle *handle)
357{
358 struct trbe_buf *buf = etm_perf_sink_config(handle);
359
360 /*
361 * The ETE trace has alignment synchronization packets allowing
362 * the decoder to reset in case of an overflow or corruption.
363 * So we can use the entire buffer for the snapshot mode.
364 */
365 return buf->nr_pages * PAGE_SIZE;
366}
367
368static u64 trbe_min_trace_buf_size(struct perf_output_handle *handle)
369{
370 u64 size = TRBE_TRACE_MIN_BUF_SIZE;
371 struct trbe_buf *buf = etm_perf_sink_config(handle);
372 struct trbe_cpudata *cpudata = buf->cpudata;
373
374 /*
375 * When the TRBE is affected by an erratum that could make it
376 * write to the next "virtually addressed" page beyond the LIMIT.
377 * We need to make sure there is always a PAGE after the LIMIT,
378 * within the buffer. Thus we ensure there is at least an extra
379 * page than normal. With this we could then adjust the LIMIT
380 * pointer down by a PAGE later.
381 */
382 if (trbe_may_write_out_of_range(cpudata))
383 size += PAGE_SIZE;
384 return size;
385}
386
387/*
388 * TRBE Limit Calculation
389 *
390 * The following markers are used to illustrate various TRBE buffer situations.
391 *
392 * $$$$ - Data area, unconsumed captured trace data, not to be overridden
393 * #### - Free area, enabled, trace will be written
394 * %%%% - Free area, disabled, trace will not be written
395 * ==== - Free area, padded with ETE_IGNORE_PACKET, trace will be skipped
396 */
397static unsigned long __trbe_normal_offset(struct perf_output_handle *handle)
398{
399 struct trbe_buf *buf = etm_perf_sink_config(handle);
400 struct trbe_cpudata *cpudata = buf->cpudata;
401 const u64 bufsize = buf->nr_pages * PAGE_SIZE;
402 u64 limit = bufsize;
403 u64 head, tail, wakeup;
404
405 head = PERF_IDX2OFF(handle->head, buf);
406
407 /*
408 * head
409 * ------->|
410 * |
411 * head TRBE align tail
412 * +----|-------|---------------|-------+
413 * |$$$$|=======|###############|$$$$$$$|
414 * +----|-------|---------------|-------+
415 * trbe_base trbe_base + nr_pages
416 *
417 * Perf aux buffer output head position can be misaligned depending on
418 * various factors including user space reads. In case misaligned, head
419 * needs to be aligned before TRBE can be configured. Pad the alignment
420 * gap with ETE_IGNORE_PACKET bytes that will be ignored by user tools
421 * and skip this section thus advancing the head.
422 */
423 if (!IS_ALIGNED(head, cpudata->trbe_align)) {
424 unsigned long delta = roundup(head, cpudata->trbe_align) - head;
425
426 delta = min(delta, handle->size);
427 trbe_pad_buf(handle, len: delta);
428 head = PERF_IDX2OFF(handle->head, buf);
429 }
430
431 /*
432 * head = tail (size = 0)
433 * +----|-------------------------------+
434 * |$$$$|$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ |
435 * +----|-------------------------------+
436 * trbe_base trbe_base + nr_pages
437 *
438 * Perf aux buffer does not have any space for the driver to write into.
439 */
440 if (!handle->size)
441 return 0;
442
443 /* Compute the tail and wakeup indices now that we've aligned head */
444 tail = PERF_IDX2OFF(handle->head + handle->size, buf);
445 wakeup = PERF_IDX2OFF(handle->wakeup, buf);
446
447 /*
448 * Lets calculate the buffer area which TRBE could write into. There
449 * are three possible scenarios here. Limit needs to be aligned with
450 * PAGE_SIZE per the TRBE requirement. Always avoid clobbering the
451 * unconsumed data.
452 *
453 * 1) head < tail
454 *
455 * head tail
456 * +----|-----------------------|-------+
457 * |$$$$|#######################|$$$$$$$|
458 * +----|-----------------------|-------+
459 * trbe_base limit trbe_base + nr_pages
460 *
461 * TRBE could write into [head..tail] area. Unless the tail is right at
462 * the end of the buffer, neither an wrap around nor an IRQ is expected
463 * while being enabled.
464 *
465 * 2) head == tail
466 *
467 * head = tail (size > 0)
468 * +----|-------------------------------+
469 * |%%%%|###############################|
470 * +----|-------------------------------+
471 * trbe_base limit = trbe_base + nr_pages
472 *
473 * TRBE should just write into [head..base + nr_pages] area even though
474 * the entire buffer is empty. Reason being, when the trace reaches the
475 * end of the buffer, it will just wrap around with an IRQ giving an
476 * opportunity to reconfigure the buffer.
477 *
478 * 3) tail < head
479 *
480 * tail head
481 * +----|-----------------------|-------+
482 * |%%%%|$$$$$$$$$$$$$$$$$$$$$$$|#######|
483 * +----|-----------------------|-------+
484 * trbe_base limit = trbe_base + nr_pages
485 *
486 * TRBE should just write into [head..base + nr_pages] area even though
487 * the [trbe_base..tail] is also empty. Reason being, when the trace
488 * reaches the end of the buffer, it will just wrap around with an IRQ
489 * giving an opportunity to reconfigure the buffer.
490 */
491 if (head < tail)
492 limit = round_down(tail, PAGE_SIZE);
493
494 /*
495 * Wakeup may be arbitrarily far into the future. If it's not in the
496 * current generation, either we'll wrap before hitting it, or it's
497 * in the past and has been handled already.
498 *
499 * If there's a wakeup before we wrap, arrange to be woken up by the
500 * page boundary following it. Keep the tail boundary if that's lower.
501 *
502 * head wakeup tail
503 * +----|---------------|-------|-------+
504 * |$$$$|###############|%%%%%%%|$$$$$$$|
505 * +----|---------------|-------|-------+
506 * trbe_base limit trbe_base + nr_pages
507 */
508 if (handle->wakeup < (handle->head + handle->size) && head <= wakeup)
509 limit = min(limit, round_up(wakeup, PAGE_SIZE));
510
511 /*
512 * There are two situation when this can happen i.e limit is before
513 * the head and hence TRBE cannot be configured.
514 *
515 * 1) head < tail (aligned down with PAGE_SIZE) and also they are both
516 * within the same PAGE size range.
517 *
518 * PAGE_SIZE
519 * |----------------------|
520 *
521 * limit head tail
522 * +------------|------|--------|-------+
523 * |$$$$$$$$$$$$$$$$$$$|========|$$$$$$$|
524 * +------------|------|--------|-------+
525 * trbe_base trbe_base + nr_pages
526 *
527 * 2) head < wakeup (aligned up with PAGE_SIZE) < tail and also both
528 * head and wakeup are within same PAGE size range.
529 *
530 * PAGE_SIZE
531 * |----------------------|
532 *
533 * limit head wakeup tail
534 * +----|------|-------|--------|-------+
535 * |$$$$$$$$$$$|=======|========|$$$$$$$|
536 * +----|------|-------|--------|-------+
537 * trbe_base trbe_base + nr_pages
538 */
539 if (limit > head)
540 return limit;
541
542 trbe_pad_buf(handle, len: handle->size);
543 return 0;
544}
545
546static unsigned long trbe_normal_offset(struct perf_output_handle *handle)
547{
548 struct trbe_buf *buf = etm_perf_sink_config(handle);
549 u64 limit = __trbe_normal_offset(handle);
550 u64 head = PERF_IDX2OFF(handle->head, buf);
551
552 /*
553 * If the head is too close to the limit and we don't
554 * have space for a meaningful run, we rather pad it
555 * and start fresh.
556 *
557 * We might have to do this more than once to make sure
558 * we have enough required space.
559 */
560 while (limit && ((limit - head) < trbe_min_trace_buf_size(handle))) {
561 trbe_pad_buf(handle, len: limit - head);
562 limit = __trbe_normal_offset(handle);
563 head = PERF_IDX2OFF(handle->head, buf);
564 }
565 return limit;
566}
567
568static unsigned long compute_trbe_buffer_limit(struct perf_output_handle *handle)
569{
570 struct trbe_buf *buf = etm_perf_sink_config(handle);
571 unsigned long offset;
572
573 if (buf->snapshot)
574 offset = trbe_snapshot_offset(handle);
575 else
576 offset = trbe_normal_offset(handle);
577 return buf->trbe_base + offset;
578}
579
580static void clr_trbe_status(void)
581{
582 u64 trbsr = read_sysreg_s(SYS_TRBSR_EL1);
583
584 WARN_ON(is_trbe_enabled());
585 trbsr &= ~TRBSR_EL1_IRQ;
586 trbsr &= ~TRBSR_EL1_TRG;
587 trbsr &= ~TRBSR_EL1_WRAP;
588 trbsr &= ~TRBSR_EL1_EC_MASK;
589 trbsr &= ~TRBSR_EL1_BSC_MASK;
590 trbsr &= ~TRBSR_EL1_S;
591 write_sysreg_s(trbsr, SYS_TRBSR_EL1);
592}
593
594static void set_trbe_limit_pointer_enabled(struct trbe_buf *buf)
595{
596 u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1);
597 unsigned long addr = buf->trbe_limit;
598
599 WARN_ON(!IS_ALIGNED(addr, (1UL << TRBLIMITR_EL1_LIMIT_SHIFT)));
600 WARN_ON(!IS_ALIGNED(addr, PAGE_SIZE));
601
602 trblimitr &= ~TRBLIMITR_EL1_nVM;
603 trblimitr &= ~TRBLIMITR_EL1_FM_MASK;
604 trblimitr &= ~TRBLIMITR_EL1_TM_MASK;
605 trblimitr &= ~TRBLIMITR_EL1_LIMIT_MASK;
606
607 /*
608 * Fill trace buffer mode is used here while configuring the
609 * TRBE for trace capture. In this particular mode, the trace
610 * collection is stopped and a maintenance interrupt is raised
611 * when the current write pointer wraps. This pause in trace
612 * collection gives the software an opportunity to capture the
613 * trace data in the interrupt handler, before reconfiguring
614 * the TRBE.
615 */
616 trblimitr |= (TRBLIMITR_EL1_FM_FILL << TRBLIMITR_EL1_FM_SHIFT) &
617 TRBLIMITR_EL1_FM_MASK;
618
619 /*
620 * Trigger mode is not used here while configuring the TRBE for
621 * the trace capture. Hence just keep this in the ignore mode.
622 */
623 trblimitr |= (TRBLIMITR_EL1_TM_IGNR << TRBLIMITR_EL1_TM_SHIFT) &
624 TRBLIMITR_EL1_TM_MASK;
625 trblimitr |= (addr & PAGE_MASK);
626 set_trbe_enabled(cpudata: buf->cpudata, trblimitr);
627}
628
629static void trbe_enable_hw(struct trbe_buf *buf)
630{
631 WARN_ON(buf->trbe_hw_base < buf->trbe_base);
632 WARN_ON(buf->trbe_write < buf->trbe_hw_base);
633 WARN_ON(buf->trbe_write >= buf->trbe_limit);
634 set_trbe_disabled(buf->cpudata);
635 clr_trbe_status();
636 set_trbe_base_pointer(buf->trbe_hw_base);
637 set_trbe_write_pointer(buf->trbe_write);
638
639 /*
640 * Synchronize all the register updates
641 * till now before enabling the TRBE.
642 */
643 isb();
644 set_trbe_limit_pointer_enabled(buf);
645}
646
647static enum trbe_fault_action trbe_get_fault_act(struct perf_output_handle *handle,
648 u64 trbsr)
649{
650 int ec = get_trbe_ec(trbsr);
651 int bsc = get_trbe_bsc(trbsr);
652 struct trbe_buf *buf = etm_perf_sink_config(handle);
653 struct trbe_cpudata *cpudata = buf->cpudata;
654
655 WARN_ON(is_trbe_running(trbsr));
656 if (is_trbe_trg(trbsr) || is_trbe_abort(trbsr))
657 return TRBE_FAULT_ACT_FATAL;
658
659 if ((ec == TRBE_EC_STAGE1_ABORT) || (ec == TRBE_EC_STAGE2_ABORT))
660 return TRBE_FAULT_ACT_FATAL;
661
662 /*
663 * If the trbe is affected by TRBE_WORKAROUND_OVERWRITE_FILL_MODE,
664 * it might write data after a WRAP event in the fill mode.
665 * Thus the check TRBPTR == TRBBASER will not be honored.
666 */
667 if ((is_trbe_wrap(trbsr) && (ec == TRBE_EC_OTHERS) && (bsc == TRBE_BSC_FILLED)) &&
668 (trbe_may_overwrite_in_fill_mode(cpudata) ||
669 get_trbe_write_pointer() == get_trbe_base_pointer()))
670 return TRBE_FAULT_ACT_WRAP;
671
672 return TRBE_FAULT_ACT_SPURIOUS;
673}
674
675static unsigned long trbe_get_trace_size(struct perf_output_handle *handle,
676 struct trbe_buf *buf, bool wrap)
677{
678 u64 write;
679 u64 start_off, end_off;
680 u64 size;
681 u64 overwrite_skip = TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES;
682
683 /*
684 * If the TRBE has wrapped around the write pointer has
685 * wrapped and should be treated as limit.
686 *
687 * When the TRBE is affected by TRBE_WORKAROUND_WRITE_OUT_OF_RANGE,
688 * it may write upto 64bytes beyond the "LIMIT". The driver already
689 * keeps a valid page next to the LIMIT and we could potentially
690 * consume the trace data that may have been collected there. But we
691 * cannot be really sure it is available, and the TRBPTR may not
692 * indicate the same. Also, affected cores are also affected by another
693 * erratum which forces the PAGE_SIZE alignment on the TRBPTR, and thus
694 * could potentially pad an entire PAGE_SIZE - 64bytes, to get those
695 * 64bytes. Thus we ignore the potential triggering of the erratum
696 * on WRAP and limit the data to LIMIT.
697 */
698 if (wrap)
699 write = get_trbe_limit_pointer();
700 else
701 write = get_trbe_write_pointer();
702
703 /*
704 * TRBE may use a different base address than the base
705 * of the ring buffer. Thus use the beginning of the ring
706 * buffer to compute the offsets.
707 */
708 end_off = write - buf->trbe_base;
709 start_off = PERF_IDX2OFF(handle->head, buf);
710
711 if (WARN_ON_ONCE(end_off < start_off))
712 return 0;
713
714 size = end_off - start_off;
715 /*
716 * If the TRBE is affected by the following erratum, we must fill
717 * the space we skipped with IGNORE packets. And we are always
718 * guaranteed to have at least a PAGE_SIZE space in the buffer.
719 */
720 if (trbe_has_erratum(cpudata: buf->cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE) &&
721 !WARN_ON(size < overwrite_skip))
722 __trbe_pad_buf(buf, offset: start_off, len: overwrite_skip);
723
724 return size;
725}
726
727static void *arm_trbe_alloc_buffer(struct coresight_device *csdev,
728 struct perf_event *event, void **pages,
729 int nr_pages, bool snapshot)
730{
731 struct trbe_buf *buf;
732 struct page **pglist;
733 int i;
734
735 /*
736 * TRBE LIMIT and TRBE WRITE pointers must be page aligned. But with
737 * just a single page, there would not be any room left while writing
738 * into a partially filled TRBE buffer after the page size alignment.
739 * Hence restrict the minimum buffer size as two pages.
740 */
741 if (nr_pages < 2)
742 return NULL;
743
744 buf = kzalloc_node(size: sizeof(*buf), GFP_KERNEL, node: trbe_alloc_node(event));
745 if (!buf)
746 return ERR_PTR(error: -ENOMEM);
747
748 pglist = kcalloc(n: nr_pages, size: sizeof(*pglist), GFP_KERNEL);
749 if (!pglist) {
750 kfree(objp: buf);
751 return ERR_PTR(error: -ENOMEM);
752 }
753
754 for (i = 0; i < nr_pages; i++)
755 pglist[i] = virt_to_page(pages[i]);
756
757 buf->trbe_base = (unsigned long)vmap(pages: pglist, count: nr_pages, VM_MAP, PAGE_KERNEL);
758 if (!buf->trbe_base) {
759 kfree(objp: pglist);
760 kfree(objp: buf);
761 return ERR_PTR(error: -ENOMEM);
762 }
763 buf->trbe_limit = buf->trbe_base + nr_pages * PAGE_SIZE;
764 buf->trbe_write = buf->trbe_base;
765 buf->snapshot = snapshot;
766 buf->nr_pages = nr_pages;
767 buf->pages = pages;
768 kfree(objp: pglist);
769 return buf;
770}
771
772static void arm_trbe_free_buffer(void *config)
773{
774 struct trbe_buf *buf = config;
775
776 vunmap(addr: (void *)buf->trbe_base);
777 kfree(objp: buf);
778}
779
780static unsigned long arm_trbe_update_buffer(struct coresight_device *csdev,
781 struct perf_output_handle *handle,
782 void *config)
783{
784 struct trbe_drvdata *drvdata = dev_get_drvdata(dev: csdev->dev.parent);
785 struct trbe_cpudata *cpudata = dev_get_drvdata(dev: &csdev->dev);
786 struct trbe_buf *buf = config;
787 enum trbe_fault_action act;
788 unsigned long size, status;
789 unsigned long flags;
790 bool wrap = false;
791
792 WARN_ON(buf->cpudata != cpudata);
793 WARN_ON(cpudata->cpu != smp_processor_id());
794 WARN_ON(cpudata->drvdata != drvdata);
795 if (cpudata->mode != CS_MODE_PERF)
796 return 0;
797
798 /*
799 * We are about to disable the TRBE. And this could in turn
800 * fill up the buffer triggering, an IRQ. This could be consumed
801 * by the PE asynchronously, causing a race here against
802 * the IRQ handler in closing out the handle. So, let us
803 * make sure the IRQ can't trigger while we are collecting
804 * the buffer. We also make sure that a WRAP event is handled
805 * accordingly.
806 */
807 local_irq_save(flags);
808
809 /*
810 * If the TRBE was disabled due to lack of space in the AUX buffer or a
811 * spurious fault, the driver leaves it disabled, truncating the buffer.
812 * Since the etm_perf driver expects to close out the AUX buffer, the
813 * driver skips it. Thus, just pass in 0 size here to indicate that the
814 * buffer was truncated.
815 */
816 if (!is_trbe_enabled()) {
817 size = 0;
818 goto done;
819 }
820 /*
821 * perf handle structure needs to be shared with the TRBE IRQ handler for
822 * capturing trace data and restarting the handle. There is a probability
823 * of an undefined reference based crash when etm event is being stopped
824 * while a TRBE IRQ also getting processed. This happens due the release
825 * of perf handle via perf_aux_output_end() in etm_event_stop(). Stopping
826 * the TRBE here will ensure that no IRQ could be generated when the perf
827 * handle gets freed in etm_event_stop().
828 */
829 trbe_drain_and_disable_local(cpudata);
830
831 /* Check if there is a pending interrupt and handle it here */
832 status = read_sysreg_s(SYS_TRBSR_EL1);
833 if (is_trbe_irq(trbsr: status)) {
834
835 /*
836 * Now that we are handling the IRQ here, clear the IRQ
837 * from the status, to let the irq handler know that it
838 * is taken care of.
839 */
840 clr_trbe_irq();
841 isb();
842
843 act = trbe_get_fault_act(handle, trbsr: status);
844 /*
845 * If this was not due to a WRAP event, we have some
846 * errors and as such buffer is empty.
847 */
848 if (act != TRBE_FAULT_ACT_WRAP) {
849 size = 0;
850 goto done;
851 }
852
853 trbe_report_wrap_event(handle);
854 wrap = true;
855 }
856
857 size = trbe_get_trace_size(handle, buf, wrap);
858
859done:
860 local_irq_restore(flags);
861
862 if (buf->snapshot)
863 handle->head += size;
864 return size;
865}
866
867
868static int trbe_apply_work_around_before_enable(struct trbe_buf *buf)
869{
870 /*
871 * TRBE_WORKAROUND_OVERWRITE_FILL_MODE causes the TRBE to overwrite a few cache
872 * line size from the "TRBBASER_EL1" in the event of a "FILL".
873 * Thus, we could loose some amount of the trace at the base.
874 *
875 * Before Fix:
876 *
877 * normal-BASE head (normal-TRBPTR) tail (normal-LIMIT)
878 * | \/ /
879 * -------------------------------------------------------------
880 * | Pg0 | Pg1 | | | PgN |
881 * -------------------------------------------------------------
882 *
883 * In the normal course of action, we would set the TRBBASER to the
884 * beginning of the ring-buffer (normal-BASE). But with the erratum,
885 * the TRBE could overwrite the contents at the "normal-BASE", after
886 * hitting the "normal-LIMIT", since it doesn't stop as expected. And
887 * this is wrong. This could result in overwriting trace collected in
888 * one of the previous runs, being consumed by the user. So we must
889 * always make sure that the TRBBASER is within the region
890 * [head, head+size]. Note that TRBBASER must be PAGE aligned,
891 *
892 * After moving the BASE:
893 *
894 * normal-BASE head (normal-TRBPTR) tail (normal-LIMIT)
895 * | \/ /
896 * -------------------------------------------------------------
897 * | | |xyzdef. |.. tuvw| |
898 * -------------------------------------------------------------
899 * /
900 * New-BASER
901 *
902 * Also, we would set the TRBPTR to head (after adjusting for
903 * alignment) at normal-PTR. This would mean that the last few bytes
904 * of the trace (say, "xyz") might overwrite the first few bytes of
905 * trace written ("abc"). More importantly they will appear in what
906 * userspace sees as the beginning of the trace, which is wrong. We may
907 * not always have space to move the latest trace "xyz" to the correct
908 * order as it must appear beyond the LIMIT. (i.e, [head..head+size]).
909 * Thus it is easier to ignore those bytes than to complicate the
910 * driver to move it, assuming that the erratum was triggered and
911 * doing additional checks to see if there is indeed allowed space at
912 * TRBLIMITR.LIMIT.
913 *
914 * Thus the full workaround will move the BASE and the PTR and would
915 * look like (after padding at the skipped bytes at the end of
916 * session) :
917 *
918 * normal-BASE head (normal-TRBPTR) tail (normal-LIMIT)
919 * | \/ /
920 * -------------------------------------------------------------
921 * | | |///abc.. |.. rst| |
922 * -------------------------------------------------------------
923 * / |
924 * New-BASER New-TRBPTR
925 *
926 * To summarize, with the work around:
927 *
928 * - We always align the offset for the next session to PAGE_SIZE
929 * (This is to ensure we can program the TRBBASER to this offset
930 * within the region [head...head+size]).
931 *
932 * - At TRBE enable:
933 * - Set the TRBBASER to the page aligned offset of the current
934 * proposed write offset. (which is guaranteed to be aligned
935 * as above)
936 * - Move the TRBPTR to skip first 256bytes (that might be
937 * overwritten with the erratum). This ensures that the trace
938 * generated in the session is not re-written.
939 *
940 * - At trace collection:
941 * - Pad the 256bytes skipped above again with IGNORE packets.
942 */
943 if (trbe_has_erratum(cpudata: buf->cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE)) {
944 if (WARN_ON(!IS_ALIGNED(buf->trbe_write, PAGE_SIZE)))
945 return -EINVAL;
946 buf->trbe_hw_base = buf->trbe_write;
947 buf->trbe_write += TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES;
948 }
949
950 /*
951 * TRBE_WORKAROUND_WRITE_OUT_OF_RANGE could cause the TRBE to write to
952 * the next page after the TRBLIMITR.LIMIT. For perf, the "next page"
953 * may be:
954 * - The page beyond the ring buffer. This could mean, TRBE could
955 * corrupt another entity (kernel / user)
956 * - A portion of the "ring buffer" consumed by the userspace.
957 * i.e, a page outisde [head, head + size].
958 *
959 * We work around this by:
960 * - Making sure that we have at least an extra space of PAGE left
961 * in the ring buffer [head, head + size], than we normally do
962 * without the erratum. See trbe_min_trace_buf_size().
963 *
964 * - Adjust the TRBLIMITR.LIMIT to leave the extra PAGE outside
965 * the TRBE's range (i.e [TRBBASER, TRBLIMITR.LIMI] ).
966 */
967 if (trbe_has_erratum(cpudata: buf->cpudata, TRBE_WORKAROUND_WRITE_OUT_OF_RANGE)) {
968 s64 space = buf->trbe_limit - buf->trbe_write;
969 /*
970 * We must have more than a PAGE_SIZE worth space in the proposed
971 * range for the TRBE.
972 */
973 if (WARN_ON(space <= PAGE_SIZE ||
974 !IS_ALIGNED(buf->trbe_limit, PAGE_SIZE)))
975 return -EINVAL;
976 buf->trbe_limit -= PAGE_SIZE;
977 }
978
979 return 0;
980}
981
982static int __arm_trbe_enable(struct trbe_buf *buf,
983 struct perf_output_handle *handle)
984{
985 int ret = 0;
986
987 perf_aux_output_flag(handle, PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW);
988 buf->trbe_limit = compute_trbe_buffer_limit(handle);
989 buf->trbe_write = buf->trbe_base + PERF_IDX2OFF(handle->head, buf);
990 if (buf->trbe_limit == buf->trbe_base) {
991 ret = -ENOSPC;
992 goto err;
993 }
994 /* Set the base of the TRBE to the buffer base */
995 buf->trbe_hw_base = buf->trbe_base;
996
997 ret = trbe_apply_work_around_before_enable(buf);
998 if (ret)
999 goto err;
1000
1001 *this_cpu_ptr(buf->cpudata->drvdata->handle) = handle;
1002 trbe_enable_hw(buf);
1003 return 0;
1004err:
1005 trbe_stop_and_truncate_event(handle);
1006 return ret;
1007}
1008
1009static int arm_trbe_enable(struct coresight_device *csdev, enum cs_mode mode,
1010 void *data)
1011{
1012 struct trbe_drvdata *drvdata = dev_get_drvdata(dev: csdev->dev.parent);
1013 struct trbe_cpudata *cpudata = dev_get_drvdata(dev: &csdev->dev);
1014 struct perf_output_handle *handle = data;
1015 struct trbe_buf *buf = etm_perf_sink_config(handle);
1016
1017 WARN_ON(cpudata->cpu != smp_processor_id());
1018 WARN_ON(cpudata->drvdata != drvdata);
1019 if (mode != CS_MODE_PERF)
1020 return -EINVAL;
1021
1022 cpudata->buf = buf;
1023 cpudata->mode = mode;
1024 buf->cpudata = cpudata;
1025
1026 return __arm_trbe_enable(buf, handle);
1027}
1028
1029static int arm_trbe_disable(struct coresight_device *csdev)
1030{
1031 struct trbe_drvdata *drvdata = dev_get_drvdata(dev: csdev->dev.parent);
1032 struct trbe_cpudata *cpudata = dev_get_drvdata(dev: &csdev->dev);
1033 struct trbe_buf *buf = cpudata->buf;
1034
1035 WARN_ON(buf->cpudata != cpudata);
1036 WARN_ON(cpudata->cpu != smp_processor_id());
1037 WARN_ON(cpudata->drvdata != drvdata);
1038 if (cpudata->mode != CS_MODE_PERF)
1039 return -EINVAL;
1040
1041 trbe_drain_and_disable_local(cpudata);
1042 buf->cpudata = NULL;
1043 cpudata->buf = NULL;
1044 cpudata->mode = CS_MODE_DISABLED;
1045 return 0;
1046}
1047
1048static void trbe_handle_spurious(struct perf_output_handle *handle)
1049{
1050 struct trbe_buf *buf = etm_perf_sink_config(handle);
1051 u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1);
1052
1053 /*
1054 * If the IRQ was spurious, simply re-enable the TRBE
1055 * back without modifying the buffer parameters to
1056 * retain the trace collected so far.
1057 */
1058 set_trbe_enabled(cpudata: buf->cpudata, trblimitr);
1059}
1060
1061static int trbe_handle_overflow(struct perf_output_handle *handle)
1062{
1063 struct perf_event *event = handle->event;
1064 struct trbe_buf *buf = etm_perf_sink_config(handle);
1065 unsigned long size;
1066 struct etm_event_data *event_data;
1067
1068 size = trbe_get_trace_size(handle, buf, wrap: true);
1069 if (buf->snapshot)
1070 handle->head += size;
1071
1072 trbe_report_wrap_event(handle);
1073 perf_aux_output_end(handle, size);
1074 event_data = perf_aux_output_begin(handle, event);
1075 if (!event_data) {
1076 /*
1077 * We are unable to restart the trace collection,
1078 * thus leave the TRBE disabled. The etm-perf driver
1079 * is able to detect this with a disconnected handle
1080 * (handle->event = NULL).
1081 */
1082 trbe_drain_and_disable_local(cpudata: buf->cpudata);
1083 *this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL;
1084 return -EINVAL;
1085 }
1086
1087 return __arm_trbe_enable(buf, handle);
1088}
1089
1090static bool is_perf_trbe(struct perf_output_handle *handle)
1091{
1092 struct trbe_buf *buf = etm_perf_sink_config(handle);
1093 struct trbe_cpudata *cpudata = buf->cpudata;
1094 struct trbe_drvdata *drvdata = cpudata->drvdata;
1095 int cpu = smp_processor_id();
1096
1097 WARN_ON(buf->trbe_hw_base != get_trbe_base_pointer());
1098 WARN_ON(buf->trbe_limit != get_trbe_limit_pointer());
1099
1100 if (cpudata->mode != CS_MODE_PERF)
1101 return false;
1102
1103 if (cpudata->cpu != cpu)
1104 return false;
1105
1106 if (!cpumask_test_cpu(cpu, cpumask: &drvdata->supported_cpus))
1107 return false;
1108
1109 return true;
1110}
1111
1112static irqreturn_t arm_trbe_irq_handler(int irq, void *dev)
1113{
1114 struct perf_output_handle **handle_ptr = dev;
1115 struct perf_output_handle *handle = *handle_ptr;
1116 struct trbe_buf *buf = etm_perf_sink_config(handle);
1117 enum trbe_fault_action act;
1118 u64 status;
1119 bool truncated = false;
1120 u64 trfcr;
1121
1122 /* Reads to TRBSR_EL1 is fine when TRBE is active */
1123 status = read_sysreg_s(SYS_TRBSR_EL1);
1124 /*
1125 * If the pending IRQ was handled by update_buffer callback
1126 * we have nothing to do here.
1127 */
1128 if (!is_trbe_irq(trbsr: status))
1129 return IRQ_NONE;
1130
1131 /* Prohibit the CPU from tracing before we disable the TRBE */
1132 trfcr = cpu_prohibit_trace();
1133 /*
1134 * Ensure the trace is visible to the CPUs and
1135 * any external aborts have been resolved.
1136 */
1137 trbe_drain_and_disable_local(cpudata: buf->cpudata);
1138 clr_trbe_irq();
1139 isb();
1140
1141 if (WARN_ON_ONCE(!handle) || !perf_get_aux(handle))
1142 return IRQ_NONE;
1143
1144 if (!is_perf_trbe(handle))
1145 return IRQ_NONE;
1146
1147 act = trbe_get_fault_act(handle, trbsr: status);
1148 switch (act) {
1149 case TRBE_FAULT_ACT_WRAP:
1150 truncated = !!trbe_handle_overflow(handle);
1151 break;
1152 case TRBE_FAULT_ACT_SPURIOUS:
1153 trbe_handle_spurious(handle);
1154 break;
1155 case TRBE_FAULT_ACT_FATAL:
1156 trbe_stop_and_truncate_event(handle);
1157 truncated = true;
1158 break;
1159 }
1160
1161 /*
1162 * If the buffer was truncated, ensure perf callbacks
1163 * have completed, which will disable the event.
1164 *
1165 * Otherwise, restore the trace filter controls to
1166 * allow the tracing.
1167 */
1168 if (truncated)
1169 irq_work_run();
1170 else
1171 write_trfcr(val: trfcr);
1172
1173 return IRQ_HANDLED;
1174}
1175
1176static const struct coresight_ops_sink arm_trbe_sink_ops = {
1177 .enable = arm_trbe_enable,
1178 .disable = arm_trbe_disable,
1179 .alloc_buffer = arm_trbe_alloc_buffer,
1180 .free_buffer = arm_trbe_free_buffer,
1181 .update_buffer = arm_trbe_update_buffer,
1182};
1183
1184static const struct coresight_ops arm_trbe_cs_ops = {
1185 .sink_ops = &arm_trbe_sink_ops,
1186};
1187
1188static ssize_t align_show(struct device *dev, struct device_attribute *attr, char *buf)
1189{
1190 struct trbe_cpudata *cpudata = dev_get_drvdata(dev);
1191
1192 return sprintf(buf, fmt: "%llx\n", cpudata->trbe_hw_align);
1193}
1194static DEVICE_ATTR_RO(align);
1195
1196static ssize_t flag_show(struct device *dev, struct device_attribute *attr, char *buf)
1197{
1198 struct trbe_cpudata *cpudata = dev_get_drvdata(dev);
1199
1200 return sprintf(buf, fmt: "%d\n", cpudata->trbe_flag);
1201}
1202static DEVICE_ATTR_RO(flag);
1203
1204static struct attribute *arm_trbe_attrs[] = {
1205 &dev_attr_align.attr,
1206 &dev_attr_flag.attr,
1207 NULL,
1208};
1209
1210static const struct attribute_group arm_trbe_group = {
1211 .attrs = arm_trbe_attrs,
1212};
1213
1214static const struct attribute_group *arm_trbe_groups[] = {
1215 &arm_trbe_group,
1216 NULL,
1217};
1218
1219static void arm_trbe_enable_cpu(void *info)
1220{
1221 struct trbe_drvdata *drvdata = info;
1222 struct trbe_cpudata *cpudata = this_cpu_ptr(drvdata->cpudata);
1223
1224 trbe_reset_local(cpudata);
1225 enable_percpu_irq(irq: drvdata->irq, type: IRQ_TYPE_NONE);
1226}
1227
1228static void arm_trbe_disable_cpu(void *info)
1229{
1230 struct trbe_drvdata *drvdata = info;
1231 struct trbe_cpudata *cpudata = this_cpu_ptr(drvdata->cpudata);
1232
1233 disable_percpu_irq(irq: drvdata->irq);
1234 trbe_reset_local(cpudata);
1235}
1236
1237
1238static void arm_trbe_register_coresight_cpu(struct trbe_drvdata *drvdata, int cpu)
1239{
1240 struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu);
1241 struct coresight_device *trbe_csdev = coresight_get_percpu_sink(cpu);
1242 struct coresight_desc desc = { 0 };
1243 struct device *dev;
1244
1245 if (WARN_ON(trbe_csdev))
1246 return;
1247
1248 /* If the TRBE was not probed on the CPU, we shouldn't be here */
1249 if (WARN_ON(!cpudata->drvdata))
1250 return;
1251
1252 dev = &cpudata->drvdata->pdev->dev;
1253 desc.name = devm_kasprintf(dev, GFP_KERNEL, fmt: "trbe%d", cpu);
1254 if (!desc.name)
1255 goto cpu_clear;
1256 /*
1257 * TRBE coresight devices do not need regular connections
1258 * information, as the paths get built between all percpu
1259 * source and their respective percpu sink devices. Though
1260 * coresight_register() expect device connections via the
1261 * platform_data, which TRBE devices do not have. As they
1262 * are not real ACPI devices, coresight_get_platform_data()
1263 * ends up failing. Instead let's allocate a dummy zeroed
1264 * coresight_platform_data structure and assign that back
1265 * into the device for that purpose.
1266 */
1267 desc.pdata = devm_kzalloc(dev, size: sizeof(*desc.pdata), GFP_KERNEL);
1268 if (IS_ERR(ptr: desc.pdata))
1269 goto cpu_clear;
1270
1271 desc.type = CORESIGHT_DEV_TYPE_SINK;
1272 desc.subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_PERCPU_SYSMEM;
1273 desc.ops = &arm_trbe_cs_ops;
1274 desc.groups = arm_trbe_groups;
1275 desc.dev = dev;
1276 trbe_csdev = coresight_register(desc: &desc);
1277 if (IS_ERR(ptr: trbe_csdev))
1278 goto cpu_clear;
1279
1280 dev_set_drvdata(dev: &trbe_csdev->dev, data: cpudata);
1281 coresight_set_percpu_sink(cpu, csdev: trbe_csdev);
1282 return;
1283cpu_clear:
1284 cpumask_clear_cpu(cpu, dstp: &drvdata->supported_cpus);
1285}
1286
1287/*
1288 * Must be called with preemption disabled, for trbe_check_errata().
1289 */
1290static void arm_trbe_probe_cpu(void *info)
1291{
1292 struct trbe_drvdata *drvdata = info;
1293 int cpu = smp_processor_id();
1294 struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu);
1295 u64 trbidr;
1296
1297 if (WARN_ON(!cpudata))
1298 goto cpu_clear;
1299
1300 if (!is_trbe_available()) {
1301 pr_err("TRBE is not implemented on cpu %d\n", cpu);
1302 goto cpu_clear;
1303 }
1304
1305 trbidr = read_sysreg_s(SYS_TRBIDR_EL1);
1306 if (!is_trbe_programmable(trbidr)) {
1307 pr_err("TRBE is owned in higher exception level on cpu %d\n", cpu);
1308 goto cpu_clear;
1309 }
1310
1311 cpudata->trbe_hw_align = 1ULL << get_trbe_address_align(trbidr);
1312 if (cpudata->trbe_hw_align > SZ_2K) {
1313 pr_err("Unsupported alignment on cpu %d\n", cpu);
1314 goto cpu_clear;
1315 }
1316
1317 /*
1318 * Run the TRBE erratum checks, now that we know
1319 * this instance is about to be registered.
1320 */
1321 trbe_check_errata(cpudata);
1322
1323 if (trbe_is_broken(cpudata)) {
1324 pr_err("Disabling TRBE on cpu%d due to erratum\n", cpu);
1325 goto cpu_clear;
1326 }
1327
1328 /*
1329 * If the TRBE is affected by erratum TRBE_WORKAROUND_OVERWRITE_FILL_MODE,
1330 * we must always program the TBRPTR_EL1, 256bytes from a page
1331 * boundary, with TRBBASER_EL1 set to the page, to prevent
1332 * TRBE over-writing 256bytes at TRBBASER_EL1 on FILL event.
1333 *
1334 * Thus make sure we always align our write pointer to a PAGE_SIZE,
1335 * which also guarantees that we have at least a PAGE_SIZE space in
1336 * the buffer (TRBLIMITR is PAGE aligned) and thus we can skip
1337 * the required bytes at the base.
1338 */
1339 if (trbe_may_overwrite_in_fill_mode(cpudata))
1340 cpudata->trbe_align = PAGE_SIZE;
1341 else
1342 cpudata->trbe_align = cpudata->trbe_hw_align;
1343
1344 cpudata->trbe_flag = get_trbe_flag_update(trbidr);
1345 cpudata->cpu = cpu;
1346 cpudata->drvdata = drvdata;
1347 return;
1348cpu_clear:
1349 cpumask_clear_cpu(cpu, dstp: &drvdata->supported_cpus);
1350}
1351
1352static void arm_trbe_remove_coresight_cpu(struct trbe_drvdata *drvdata, int cpu)
1353{
1354 struct coresight_device *trbe_csdev = coresight_get_percpu_sink(cpu);
1355
1356 if (trbe_csdev) {
1357 coresight_unregister(csdev: trbe_csdev);
1358 coresight_set_percpu_sink(cpu, NULL);
1359 }
1360}
1361
1362static int arm_trbe_probe_coresight(struct trbe_drvdata *drvdata)
1363{
1364 int cpu;
1365
1366 drvdata->cpudata = alloc_percpu(typeof(*drvdata->cpudata));
1367 if (!drvdata->cpudata)
1368 return -ENOMEM;
1369
1370 for_each_cpu(cpu, &drvdata->supported_cpus) {
1371 /* If we fail to probe the CPU, let us defer it to hotplug callbacks */
1372 if (smp_call_function_single(cpuid: cpu, func: arm_trbe_probe_cpu, info: drvdata, wait: 1))
1373 continue;
1374 if (cpumask_test_cpu(cpu, cpumask: &drvdata->supported_cpus))
1375 arm_trbe_register_coresight_cpu(drvdata, cpu);
1376 if (cpumask_test_cpu(cpu, cpumask: &drvdata->supported_cpus))
1377 smp_call_function_single(cpuid: cpu, func: arm_trbe_enable_cpu, info: drvdata, wait: 1);
1378 }
1379 return 0;
1380}
1381
1382static int arm_trbe_remove_coresight(struct trbe_drvdata *drvdata)
1383{
1384 int cpu;
1385
1386 for_each_cpu(cpu, &drvdata->supported_cpus) {
1387 smp_call_function_single(cpuid: cpu, func: arm_trbe_disable_cpu, info: drvdata, wait: 1);
1388 arm_trbe_remove_coresight_cpu(drvdata, cpu);
1389 }
1390 free_percpu(pdata: drvdata->cpudata);
1391 return 0;
1392}
1393
1394static void arm_trbe_probe_hotplugged_cpu(struct trbe_drvdata *drvdata)
1395{
1396 preempt_disable();
1397 arm_trbe_probe_cpu(info: drvdata);
1398 preempt_enable();
1399}
1400
1401static int arm_trbe_cpu_startup(unsigned int cpu, struct hlist_node *node)
1402{
1403 struct trbe_drvdata *drvdata = hlist_entry_safe(node, struct trbe_drvdata, hotplug_node);
1404
1405 if (cpumask_test_cpu(cpu, cpumask: &drvdata->supported_cpus)) {
1406
1407 /*
1408 * If this CPU was not probed for TRBE,
1409 * initialize it now.
1410 */
1411 if (!coresight_get_percpu_sink(cpu)) {
1412 arm_trbe_probe_hotplugged_cpu(drvdata);
1413 if (cpumask_test_cpu(cpu, cpumask: &drvdata->supported_cpus))
1414 arm_trbe_register_coresight_cpu(drvdata, cpu);
1415 if (cpumask_test_cpu(cpu, cpumask: &drvdata->supported_cpus))
1416 arm_trbe_enable_cpu(info: drvdata);
1417 } else {
1418 arm_trbe_enable_cpu(info: drvdata);
1419 }
1420 }
1421 return 0;
1422}
1423
1424static int arm_trbe_cpu_teardown(unsigned int cpu, struct hlist_node *node)
1425{
1426 struct trbe_drvdata *drvdata = hlist_entry_safe(node, struct trbe_drvdata, hotplug_node);
1427
1428 if (cpumask_test_cpu(cpu, cpumask: &drvdata->supported_cpus))
1429 arm_trbe_disable_cpu(info: drvdata);
1430 return 0;
1431}
1432
1433static int arm_trbe_probe_cpuhp(struct trbe_drvdata *drvdata)
1434{
1435 enum cpuhp_state trbe_online;
1436 int ret;
1437
1438 trbe_online = cpuhp_setup_state_multi(state: CPUHP_AP_ONLINE_DYN, DRVNAME,
1439 startup: arm_trbe_cpu_startup, teardown: arm_trbe_cpu_teardown);
1440 if (trbe_online < 0)
1441 return trbe_online;
1442
1443 ret = cpuhp_state_add_instance(state: trbe_online, node: &drvdata->hotplug_node);
1444 if (ret) {
1445 cpuhp_remove_multi_state(state: trbe_online);
1446 return ret;
1447 }
1448 drvdata->trbe_online = trbe_online;
1449 return 0;
1450}
1451
1452static void arm_trbe_remove_cpuhp(struct trbe_drvdata *drvdata)
1453{
1454 cpuhp_state_remove_instance(state: drvdata->trbe_online, node: &drvdata->hotplug_node);
1455 cpuhp_remove_multi_state(state: drvdata->trbe_online);
1456}
1457
1458static int arm_trbe_probe_irq(struct platform_device *pdev,
1459 struct trbe_drvdata *drvdata)
1460{
1461 int ret;
1462
1463 drvdata->irq = platform_get_irq(pdev, 0);
1464 if (drvdata->irq < 0) {
1465 pr_err("IRQ not found for the platform device\n");
1466 return drvdata->irq;
1467 }
1468
1469 if (!irq_is_percpu(irq: drvdata->irq)) {
1470 pr_err("IRQ is not a PPI\n");
1471 return -EINVAL;
1472 }
1473
1474 if (irq_get_percpu_devid_partition(irq: drvdata->irq, affinity: &drvdata->supported_cpus))
1475 return -EINVAL;
1476
1477 drvdata->handle = alloc_percpu(struct perf_output_handle *);
1478 if (!drvdata->handle)
1479 return -ENOMEM;
1480
1481 ret = request_percpu_irq(irq: drvdata->irq, handler: arm_trbe_irq_handler, DRVNAME, percpu_dev_id: drvdata->handle);
1482 if (ret) {
1483 free_percpu(pdata: drvdata->handle);
1484 return ret;
1485 }
1486 return 0;
1487}
1488
1489static void arm_trbe_remove_irq(struct trbe_drvdata *drvdata)
1490{
1491 free_percpu_irq(drvdata->irq, drvdata->handle);
1492 free_percpu(pdata: drvdata->handle);
1493}
1494
1495static int arm_trbe_device_probe(struct platform_device *pdev)
1496{
1497 struct trbe_drvdata *drvdata;
1498 struct device *dev = &pdev->dev;
1499 int ret;
1500
1501 /* Trace capture is not possible with kernel page table isolation */
1502 if (arm64_kernel_unmapped_at_el0()) {
1503 pr_err("TRBE wouldn't work if kernel gets unmapped at EL0\n");
1504 return -EOPNOTSUPP;
1505 }
1506
1507 drvdata = devm_kzalloc(dev, size: sizeof(*drvdata), GFP_KERNEL);
1508 if (!drvdata)
1509 return -ENOMEM;
1510
1511 dev_set_drvdata(dev, data: drvdata);
1512 drvdata->pdev = pdev;
1513 ret = arm_trbe_probe_irq(pdev, drvdata);
1514 if (ret)
1515 return ret;
1516
1517 ret = arm_trbe_probe_coresight(drvdata);
1518 if (ret)
1519 goto probe_failed;
1520
1521 ret = arm_trbe_probe_cpuhp(drvdata);
1522 if (ret)
1523 goto cpuhp_failed;
1524
1525 return 0;
1526cpuhp_failed:
1527 arm_trbe_remove_coresight(drvdata);
1528probe_failed:
1529 arm_trbe_remove_irq(drvdata);
1530 return ret;
1531}
1532
1533static void arm_trbe_device_remove(struct platform_device *pdev)
1534{
1535 struct trbe_drvdata *drvdata = platform_get_drvdata(pdev);
1536
1537 arm_trbe_remove_cpuhp(drvdata);
1538 arm_trbe_remove_coresight(drvdata);
1539 arm_trbe_remove_irq(drvdata);
1540}
1541
1542static const struct of_device_id arm_trbe_of_match[] = {
1543 { .compatible = "arm,trace-buffer-extension"},
1544 {},
1545};
1546MODULE_DEVICE_TABLE(of, arm_trbe_of_match);
1547
1548#ifdef CONFIG_ACPI
1549static const struct platform_device_id arm_trbe_acpi_match[] = {
1550 { ARMV8_TRBE_PDEV_NAME, 0 },
1551 { }
1552};
1553MODULE_DEVICE_TABLE(platform, arm_trbe_acpi_match);
1554#endif
1555
1556static struct platform_driver arm_trbe_driver = {
1557 .id_table = ACPI_PTR(arm_trbe_acpi_match),
1558 .driver = {
1559 .name = DRVNAME,
1560 .of_match_table = of_match_ptr(arm_trbe_of_match),
1561 .suppress_bind_attrs = true,
1562 },
1563 .probe = arm_trbe_device_probe,
1564 .remove_new = arm_trbe_device_remove,
1565};
1566
1567static int __init arm_trbe_init(void)
1568{
1569 int ret;
1570
1571 ret = platform_driver_register(&arm_trbe_driver);
1572 if (!ret)
1573 return 0;
1574
1575 pr_err("Error registering %s platform driver\n", DRVNAME);
1576 return ret;
1577}
1578
1579static void __exit arm_trbe_exit(void)
1580{
1581 platform_driver_unregister(&arm_trbe_driver);
1582}
1583module_init(arm_trbe_init);
1584module_exit(arm_trbe_exit);
1585
1586MODULE_AUTHOR("Anshuman Khandual <anshuman.khandual@arm.com>");
1587MODULE_DESCRIPTION("Arm Trace Buffer Extension (TRBE) driver");
1588MODULE_LICENSE("GPL v2");
1589

source code of linux/drivers/hwtracing/coresight/coresight-trbe.c