1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * StarFive's StarLink PMU driver
4 *
5 * Copyright (C) 2023 StarFive Technology Co., Ltd.
6 *
7 * Author: Ji Sheng Teoh <jisheng.teoh@starfivetech.com>
8 *
9 */
10
11#define STARLINK_PMU_PDEV_NAME "starfive_starlink_pmu"
12#define pr_fmt(fmt) STARLINK_PMU_PDEV_NAME ": " fmt
13
14#include <linux/bitmap.h>
15#include <linux/cpu_pm.h>
16#include <linux/io.h>
17#include <linux/irq.h>
18#include <linux/kernel.h>
19#include <linux/module.h>
20#include <linux/mod_devicetable.h>
21#include <linux/perf_event.h>
22#include <linux/platform_device.h>
23#include <linux/sysfs.h>
24
25#define STARLINK_PMU_MAX_COUNTERS 64
26#define STARLINK_PMU_NUM_COUNTERS 16
27#define STARLINK_PMU_IDX_CYCLE_COUNTER 63
28
29#define STARLINK_PMU_EVENT_SELECT 0x060
30#define STARLINK_PMU_EVENT_COUNTER 0x160
31#define STARLINK_PMU_COUNTER_MASK GENMASK_ULL(63, 0)
32#define STARLINK_PMU_CYCLE_COUNTER 0x058
33
34#define STARLINK_PMU_CONTROL 0x040
35#define STARLINK_PMU_GLOBAL_ENABLE BIT_ULL(0)
36
37#define STARLINK_PMU_INTERRUPT_ENABLE 0x050
38#define STARLINK_PMU_COUNTER_OVERFLOW_STATUS 0x048
39#define STARLINK_PMU_CYCLE_OVERFLOW_MASK BIT_ULL(63)
40
41#define STARLINK_CYCLES 0x058
42#define CACHE_READ_REQUEST 0x04000701
43#define CACHE_WRITE_REQUEST 0x03000001
44#define CACHE_RELEASE_REQUEST 0x0003e001
45#define CACHE_READ_HIT 0x00901202
46#define CACHE_READ_MISS 0x04008002
47#define CACHE_WRITE_HIT 0x006c0002
48#define CACHE_WRITE_MISS 0x03000002
49#define CACHE_WRITEBACK 0x00000403
50
51#define to_starlink_pmu(p) (container_of(p, struct starlink_pmu, pmu))
52
53#define STARLINK_FORMAT_ATTR(_name, _config) \
54 (&((struct dev_ext_attribute[]) { \
55 { .attr = __ATTR(_name, 0444, starlink_pmu_sysfs_format_show, NULL), \
56 .var = (void *)_config, } \
57 })[0].attr.attr)
58
59#define STARLINK_EVENT_ATTR(_name, _id) \
60 PMU_EVENT_ATTR_ID(_name, starlink_pmu_sysfs_event_show, _id)
61
62static int starlink_pmu_cpuhp_state;
63
64struct starlink_hw_events {
65 struct perf_event *events[STARLINK_PMU_MAX_COUNTERS];
66 DECLARE_BITMAP(used_mask, STARLINK_PMU_MAX_COUNTERS);
67};
68
69struct starlink_pmu {
70 struct pmu pmu;
71 struct starlink_hw_events __percpu *hw_events;
72 struct hlist_node node;
73 struct notifier_block starlink_pmu_pm_nb;
74 void __iomem *pmu_base;
75 cpumask_t cpumask;
76 int irq;
77};
78
79static ssize_t
80starlink_pmu_sysfs_format_show(struct device *dev,
81 struct device_attribute *attr,
82 char *buf)
83{
84 struct dev_ext_attribute *eattr = container_of(attr,
85 struct dev_ext_attribute, attr);
86
87 return sysfs_emit(buf, fmt: "%s\n", (char *)eattr->var);
88}
89
90static struct attribute *starlink_pmu_format_attrs[] = {
91 STARLINK_FORMAT_ATTR(event, "config:0-31"),
92 NULL
93};
94
95static const struct attribute_group starlink_pmu_format_attr_group = {
96 .name = "format",
97 .attrs = starlink_pmu_format_attrs,
98};
99
100static ssize_t
101starlink_pmu_sysfs_event_show(struct device *dev,
102 struct device_attribute *attr,
103 char *buf)
104{
105 struct perf_pmu_events_attr *eattr = container_of(attr,
106 struct perf_pmu_events_attr, attr);
107
108 return sysfs_emit(buf, fmt: "event=0x%02llx\n", eattr->id);
109}
110
111static struct attribute *starlink_pmu_event_attrs[] = {
112 STARLINK_EVENT_ATTR(cycles, STARLINK_CYCLES),
113 STARLINK_EVENT_ATTR(read_request, CACHE_READ_REQUEST),
114 STARLINK_EVENT_ATTR(write_request, CACHE_WRITE_REQUEST),
115 STARLINK_EVENT_ATTR(release_request, CACHE_RELEASE_REQUEST),
116 STARLINK_EVENT_ATTR(read_hit, CACHE_READ_HIT),
117 STARLINK_EVENT_ATTR(read_miss, CACHE_READ_MISS),
118 STARLINK_EVENT_ATTR(write_hit, CACHE_WRITE_HIT),
119 STARLINK_EVENT_ATTR(write_miss, CACHE_WRITE_MISS),
120 STARLINK_EVENT_ATTR(writeback, CACHE_WRITEBACK),
121 NULL
122};
123
124static const struct attribute_group starlink_pmu_events_attr_group = {
125 .name = "events",
126 .attrs = starlink_pmu_event_attrs,
127};
128
129static ssize_t
130cpumask_show(struct device *dev, struct device_attribute *attr, char *buf)
131{
132 struct starlink_pmu *starlink_pmu = to_starlink_pmu(dev_get_drvdata(dev));
133
134 return cpumap_print_to_pagebuf(list: true, buf, mask: &starlink_pmu->cpumask);
135}
136
137static DEVICE_ATTR_RO(cpumask);
138
139static struct attribute *starlink_pmu_cpumask_attrs[] = {
140 &dev_attr_cpumask.attr,
141 NULL
142};
143
144static const struct attribute_group starlink_pmu_cpumask_attr_group = {
145 .attrs = starlink_pmu_cpumask_attrs,
146};
147
148static const struct attribute_group *starlink_pmu_attr_groups[] = {
149 &starlink_pmu_format_attr_group,
150 &starlink_pmu_events_attr_group,
151 &starlink_pmu_cpumask_attr_group,
152 NULL
153};
154
155static void starlink_pmu_set_event_period(struct perf_event *event)
156{
157 struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu);
158 struct hw_perf_event *hwc = &event->hw;
159 int idx = event->hw.idx;
160
161 /*
162 * Program counter to half of it's max count to handle
163 * cases of extreme interrupt latency.
164 */
165 u64 val = STARLINK_PMU_COUNTER_MASK >> 1;
166
167 local64_set(&hwc->prev_count, val);
168 if (hwc->config == STARLINK_CYCLES)
169 writeq(val, addr: starlink_pmu->pmu_base + STARLINK_PMU_CYCLE_COUNTER);
170 else
171 writeq(val, addr: starlink_pmu->pmu_base + STARLINK_PMU_EVENT_COUNTER +
172 idx * sizeof(u64));
173}
174
175static void starlink_pmu_counter_start(struct perf_event *event,
176 struct starlink_pmu *starlink_pmu)
177{
178 struct hw_perf_event *hwc = &event->hw;
179 int idx = event->hw.idx;
180 u64 val;
181
182 /*
183 * Enable counter overflow interrupt[63:0],
184 * which is mapped as follow:
185 *
186 * event counter 0 - Bit [0]
187 * event counter 1 - Bit [1]
188 * ...
189 * cycle counter - Bit [63]
190 */
191 val = readq(addr: starlink_pmu->pmu_base + STARLINK_PMU_INTERRUPT_ENABLE);
192
193 if (hwc->config == STARLINK_CYCLES) {
194 /*
195 * Cycle count has its dedicated register, and it starts
196 * counting as soon as STARLINK_PMU_GLOBAL_ENABLE is set.
197 */
198 val |= STARLINK_PMU_CYCLE_OVERFLOW_MASK;
199 } else {
200 writeq(val: event->hw.config, addr: starlink_pmu->pmu_base +
201 STARLINK_PMU_EVENT_SELECT + idx * sizeof(u64));
202
203 val |= BIT_ULL(idx);
204 }
205
206 writeq(val, addr: starlink_pmu->pmu_base + STARLINK_PMU_INTERRUPT_ENABLE);
207
208 writeq(STARLINK_PMU_GLOBAL_ENABLE, addr: starlink_pmu->pmu_base +
209 STARLINK_PMU_CONTROL);
210}
211
212static void starlink_pmu_counter_stop(struct perf_event *event,
213 struct starlink_pmu *starlink_pmu)
214{
215 struct hw_perf_event *hwc = &event->hw;
216 int idx = event->hw.idx;
217 u64 val;
218
219 val = readq(addr: starlink_pmu->pmu_base + STARLINK_PMU_CONTROL);
220 val &= ~STARLINK_PMU_GLOBAL_ENABLE;
221 writeq(val, addr: starlink_pmu->pmu_base + STARLINK_PMU_CONTROL);
222
223 val = readq(addr: starlink_pmu->pmu_base + STARLINK_PMU_INTERRUPT_ENABLE);
224 if (hwc->config == STARLINK_CYCLES)
225 val &= ~STARLINK_PMU_CYCLE_OVERFLOW_MASK;
226 else
227 val &= ~BIT_ULL(idx);
228
229 writeq(val, addr: starlink_pmu->pmu_base + STARLINK_PMU_INTERRUPT_ENABLE);
230}
231
232static void starlink_pmu_update(struct perf_event *event)
233{
234 struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu);
235 struct hw_perf_event *hwc = &event->hw;
236 int idx = hwc->idx;
237 u64 prev_raw_count, new_raw_count;
238 u64 oldval;
239 u64 delta;
240
241 do {
242 prev_raw_count = local64_read(&hwc->prev_count);
243 if (hwc->config == STARLINK_CYCLES)
244 new_raw_count = readq(addr: starlink_pmu->pmu_base +
245 STARLINK_PMU_CYCLE_COUNTER);
246 else
247 new_raw_count = readq(addr: starlink_pmu->pmu_base +
248 STARLINK_PMU_EVENT_COUNTER +
249 idx * sizeof(u64));
250 oldval = local64_cmpxchg(l: &hwc->prev_count, old: prev_raw_count,
251 new: new_raw_count);
252 } while (oldval != prev_raw_count);
253
254 delta = (new_raw_count - prev_raw_count) & STARLINK_PMU_COUNTER_MASK;
255 local64_add(delta, &event->count);
256}
257
258static void starlink_pmu_start(struct perf_event *event, int flags)
259{
260 struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu);
261 struct hw_perf_event *hwc = &event->hw;
262
263 if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
264 return;
265
266 if (flags & PERF_EF_RELOAD)
267 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
268
269 hwc->state = 0;
270
271 starlink_pmu_set_event_period(event);
272 starlink_pmu_counter_start(event, starlink_pmu);
273
274 perf_event_update_userpage(event);
275}
276
277static void starlink_pmu_stop(struct perf_event *event, int flags)
278{
279 struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu);
280 struct hw_perf_event *hwc = &event->hw;
281
282 if (hwc->state & PERF_HES_STOPPED)
283 return;
284
285 starlink_pmu_counter_stop(event, starlink_pmu);
286 starlink_pmu_update(event);
287 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
288}
289
290static int starlink_pmu_add(struct perf_event *event, int flags)
291{
292 struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu);
293 struct starlink_hw_events *hw_events =
294 this_cpu_ptr(starlink_pmu->hw_events);
295 struct hw_perf_event *hwc = &event->hw;
296 unsigned long *used_mask = hw_events->used_mask;
297 u32 n_events = STARLINK_PMU_NUM_COUNTERS;
298 int idx;
299
300 /*
301 * Cycle counter has dedicated register to hold counter value.
302 * Event other than cycle count has to be enabled through
303 * event select register, and assigned with independent counter
304 * as they appear.
305 */
306
307 if (hwc->config == STARLINK_CYCLES) {
308 idx = STARLINK_PMU_IDX_CYCLE_COUNTER;
309 } else {
310 idx = find_first_zero_bit(addr: used_mask, size: n_events);
311 /* All counter are in use */
312 if (idx < 0)
313 return idx;
314
315 set_bit(nr: idx, addr: used_mask);
316 }
317
318 hwc->idx = idx;
319 hw_events->events[idx] = event;
320 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
321
322 if (flags & PERF_EF_START)
323 starlink_pmu_start(event, PERF_EF_RELOAD);
324
325 perf_event_update_userpage(event);
326
327 return 0;
328}
329
330static void starlink_pmu_del(struct perf_event *event, int flags)
331{
332 struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu);
333 struct starlink_hw_events *hw_events =
334 this_cpu_ptr(starlink_pmu->hw_events);
335 struct hw_perf_event *hwc = &event->hw;
336
337 starlink_pmu_stop(event, PERF_EF_UPDATE);
338 hw_events->events[hwc->idx] = NULL;
339 clear_bit(nr: hwc->idx, addr: hw_events->used_mask);
340
341 perf_event_update_userpage(event);
342}
343
344static bool starlink_pmu_validate_event_group(struct perf_event *event)
345{
346 struct perf_event *leader = event->group_leader;
347 struct perf_event *sibling;
348 int counter = 1;
349
350 /*
351 * Ensure hardware events in the group are on the same PMU,
352 * software events are acceptable.
353 */
354 if (event->group_leader->pmu != event->pmu &&
355 !is_software_event(event: event->group_leader))
356 return false;
357
358 for_each_sibling_event(sibling, leader) {
359 if (sibling->pmu != event->pmu && !is_software_event(event: sibling))
360 return false;
361
362 counter++;
363 }
364
365 return counter <= STARLINK_PMU_NUM_COUNTERS;
366}
367
368static int starlink_pmu_event_init(struct perf_event *event)
369{
370 struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu);
371 struct hw_perf_event *hwc = &event->hw;
372
373 /*
374 * Sampling is not supported, as counters are shared
375 * by all CPU.
376 */
377 if (hwc->sample_period)
378 return -EOPNOTSUPP;
379
380 /*
381 * Per-task and attach to a task are not supported,
382 * as uncore events are not specific to any CPU.
383 */
384 if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK)
385 return -EOPNOTSUPP;
386
387 if (!starlink_pmu_validate_event_group(event))
388 return -EINVAL;
389
390 hwc->idx = -1;
391 hwc->config = event->attr.config;
392 event->cpu = cpumask_first(srcp: &starlink_pmu->cpumask);
393
394 return 0;
395}
396
397static irqreturn_t starlink_pmu_handle_irq(int irq_num, void *data)
398{
399 struct starlink_pmu *starlink_pmu = data;
400 struct starlink_hw_events *hw_events =
401 this_cpu_ptr(starlink_pmu->hw_events);
402 bool handled = false;
403 int idx;
404 u64 overflow_status;
405
406 for (idx = 0; idx < STARLINK_PMU_MAX_COUNTERS; idx++) {
407 struct perf_event *event = hw_events->events[idx];
408
409 if (!event)
410 continue;
411
412 overflow_status = readq(addr: starlink_pmu->pmu_base +
413 STARLINK_PMU_COUNTER_OVERFLOW_STATUS);
414 if (!(overflow_status & BIT_ULL(idx)))
415 continue;
416
417 writeq(BIT_ULL(idx), addr: starlink_pmu->pmu_base +
418 STARLINK_PMU_COUNTER_OVERFLOW_STATUS);
419
420 starlink_pmu_update(event);
421 starlink_pmu_set_event_period(event);
422 handled = true;
423 }
424 return IRQ_RETVAL(handled);
425}
426
427static int starlink_setup_irqs(struct starlink_pmu *starlink_pmu,
428 struct platform_device *pdev)
429{
430 int ret, irq;
431
432 irq = platform_get_irq(pdev, 0);
433 if (irq < 0)
434 return -EINVAL;
435
436 ret = devm_request_irq(dev: &pdev->dev, irq, handler: starlink_pmu_handle_irq,
437 irqflags: 0, STARLINK_PMU_PDEV_NAME, dev_id: starlink_pmu);
438 if (ret)
439 return dev_err_probe(dev: &pdev->dev, err: ret, fmt: "Failed to request IRQ\n");
440
441 starlink_pmu->irq = irq;
442
443 return 0;
444}
445
446static int starlink_pmu_pm_notify(struct notifier_block *b,
447 unsigned long cmd, void *v)
448{
449 struct starlink_pmu *starlink_pmu = container_of(b, struct starlink_pmu,
450 starlink_pmu_pm_nb);
451 struct starlink_hw_events *hw_events =
452 this_cpu_ptr(starlink_pmu->hw_events);
453 int enabled = bitmap_weight(src: hw_events->used_mask,
454 STARLINK_PMU_MAX_COUNTERS);
455 struct perf_event *event;
456 int idx;
457
458 if (!enabled)
459 return NOTIFY_OK;
460
461 for (idx = 0; idx < STARLINK_PMU_MAX_COUNTERS; idx++) {
462 event = hw_events->events[idx];
463 if (!event)
464 continue;
465
466 switch (cmd) {
467 case CPU_PM_ENTER:
468 /* Stop and update the counter */
469 starlink_pmu_stop(event, PERF_EF_UPDATE);
470 break;
471 case CPU_PM_EXIT:
472 case CPU_PM_ENTER_FAILED:
473 /* Restore and enable the counter */
474 starlink_pmu_start(event, PERF_EF_RELOAD);
475 break;
476 default:
477 break;
478 }
479 }
480
481 return NOTIFY_OK;
482}
483
484static int starlink_pmu_pm_register(struct starlink_pmu *starlink_pmu)
485{
486 if (!IS_ENABLED(CONFIG_CPU_PM))
487 return 0;
488
489 starlink_pmu->starlink_pmu_pm_nb.notifier_call = starlink_pmu_pm_notify;
490 return cpu_pm_register_notifier(nb: &starlink_pmu->starlink_pmu_pm_nb);
491}
492
493static void starlink_pmu_pm_unregister(struct starlink_pmu *starlink_pmu)
494{
495 if (!IS_ENABLED(CONFIG_CPU_PM))
496 return;
497
498 cpu_pm_unregister_notifier(nb: &starlink_pmu->starlink_pmu_pm_nb);
499}
500
501static void starlink_pmu_destroy(struct starlink_pmu *starlink_pmu)
502{
503 starlink_pmu_pm_unregister(starlink_pmu);
504 cpuhp_state_remove_instance(state: starlink_pmu_cpuhp_state,
505 node: &starlink_pmu->node);
506}
507
508static int starlink_pmu_probe(struct platform_device *pdev)
509{
510 struct starlink_pmu *starlink_pmu;
511 struct starlink_hw_events *hw_events;
512 struct resource *res;
513 int cpuid, i, ret;
514
515 starlink_pmu = devm_kzalloc(dev: &pdev->dev, size: sizeof(*starlink_pmu), GFP_KERNEL);
516 if (!starlink_pmu)
517 return -ENOMEM;
518
519 starlink_pmu->pmu_base =
520 devm_platform_get_and_ioremap_resource(pdev, index: 0, res: &res);
521 if (IS_ERR(ptr: starlink_pmu->pmu_base))
522 return PTR_ERR(ptr: starlink_pmu->pmu_base);
523
524 starlink_pmu->hw_events = alloc_percpu_gfp(struct starlink_hw_events,
525 GFP_KERNEL);
526 if (!starlink_pmu->hw_events) {
527 dev_err(&pdev->dev, "Failed to allocate per-cpu PMU data\n");
528 return -ENOMEM;
529 }
530
531 for_each_possible_cpu(cpuid) {
532 hw_events = per_cpu_ptr(starlink_pmu->hw_events, cpuid);
533 for (i = 0; i < STARLINK_PMU_MAX_COUNTERS; i++)
534 hw_events->events[i] = NULL;
535 }
536
537 ret = starlink_setup_irqs(starlink_pmu, pdev);
538 if (ret)
539 return ret;
540
541 ret = cpuhp_state_add_instance(state: starlink_pmu_cpuhp_state,
542 node: &starlink_pmu->node);
543 if (ret) {
544 dev_err(&pdev->dev, "Failed to register hotplug\n");
545 return ret;
546 }
547
548 ret = starlink_pmu_pm_register(starlink_pmu);
549 if (ret) {
550 cpuhp_state_remove_instance(state: starlink_pmu_cpuhp_state,
551 node: &starlink_pmu->node);
552 return ret;
553 }
554
555 starlink_pmu->pmu = (struct pmu) {
556 .task_ctx_nr = perf_invalid_context,
557 .event_init = starlink_pmu_event_init,
558 .add = starlink_pmu_add,
559 .del = starlink_pmu_del,
560 .start = starlink_pmu_start,
561 .stop = starlink_pmu_stop,
562 .read = starlink_pmu_update,
563 .attr_groups = starlink_pmu_attr_groups,
564 };
565
566 ret = perf_pmu_register(pmu: &starlink_pmu->pmu, STARLINK_PMU_PDEV_NAME, type: -1);
567 if (ret)
568 starlink_pmu_destroy(starlink_pmu);
569
570 return ret;
571}
572
573static const struct of_device_id starlink_pmu_of_match[] = {
574 { .compatible = "starfive,jh8100-starlink-pmu" },
575 {}
576};
577MODULE_DEVICE_TABLE(of, starlink_pmu_of_match);
578
579static struct platform_driver starlink_pmu_driver = {
580 .driver = {
581 .name = STARLINK_PMU_PDEV_NAME,
582 .of_match_table = starlink_pmu_of_match,
583 .suppress_bind_attrs = true,
584 },
585 .probe = starlink_pmu_probe,
586};
587
588static int
589starlink_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
590{
591 struct starlink_pmu *starlink_pmu = hlist_entry_safe(node,
592 struct starlink_pmu,
593 node);
594
595 if (cpumask_empty(srcp: &starlink_pmu->cpumask))
596 cpumask_set_cpu(cpu, dstp: &starlink_pmu->cpumask);
597
598 WARN_ON(irq_set_affinity(starlink_pmu->irq, cpumask_of(cpu)));
599
600 return 0;
601}
602
603static int
604starlink_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
605{
606 struct starlink_pmu *starlink_pmu = hlist_entry_safe(node,
607 struct starlink_pmu,
608 node);
609 unsigned int target;
610
611 if (!cpumask_test_and_clear_cpu(cpu, cpumask: &starlink_pmu->cpumask))
612 return 0;
613
614 target = cpumask_any_but(cpu_online_mask, cpu);
615 if (target >= nr_cpu_ids)
616 return 0;
617
618 perf_pmu_migrate_context(pmu: &starlink_pmu->pmu, src_cpu: cpu, dst_cpu: target);
619
620 cpumask_set_cpu(cpu: target, dstp: &starlink_pmu->cpumask);
621 WARN_ON(irq_set_affinity(starlink_pmu->irq, cpumask_of(target)));
622
623 return 0;
624}
625
626static int __init starlink_pmu_init(void)
627{
628 int ret;
629
630 ret = cpuhp_setup_state_multi(state: CPUHP_AP_ONLINE_DYN,
631 name: "soc/starfive/starlink_pmu:online",
632 startup: starlink_pmu_online_cpu,
633 teardown: starlink_pmu_offline_cpu);
634 if (ret < 0)
635 return ret;
636
637 starlink_pmu_cpuhp_state = ret;
638
639 return platform_driver_register(&starlink_pmu_driver);
640}
641
642device_initcall(starlink_pmu_init);
643

source code of linux/drivers/perf/starfive_starlink_pmu.c