1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * StarFive's StarLink PMU driver |
4 | * |
5 | * Copyright (C) 2023 StarFive Technology Co., Ltd. |
6 | * |
7 | * Author: Ji Sheng Teoh <jisheng.teoh@starfivetech.com> |
8 | * |
9 | */ |
10 | |
11 | #define STARLINK_PMU_PDEV_NAME "starfive_starlink_pmu" |
12 | #define pr_fmt(fmt) STARLINK_PMU_PDEV_NAME ": " fmt |
13 | |
14 | #include <linux/bitmap.h> |
15 | #include <linux/cpu_pm.h> |
16 | #include <linux/io.h> |
17 | #include <linux/irq.h> |
18 | #include <linux/kernel.h> |
19 | #include <linux/module.h> |
20 | #include <linux/mod_devicetable.h> |
21 | #include <linux/perf_event.h> |
22 | #include <linux/platform_device.h> |
23 | #include <linux/sysfs.h> |
24 | |
25 | #define STARLINK_PMU_MAX_COUNTERS 64 |
26 | #define STARLINK_PMU_NUM_COUNTERS 16 |
27 | #define STARLINK_PMU_IDX_CYCLE_COUNTER 63 |
28 | |
29 | #define STARLINK_PMU_EVENT_SELECT 0x060 |
30 | #define STARLINK_PMU_EVENT_COUNTER 0x160 |
31 | #define STARLINK_PMU_COUNTER_MASK GENMASK_ULL(63, 0) |
32 | #define STARLINK_PMU_CYCLE_COUNTER 0x058 |
33 | |
34 | #define STARLINK_PMU_CONTROL 0x040 |
35 | #define STARLINK_PMU_GLOBAL_ENABLE BIT_ULL(0) |
36 | |
37 | #define STARLINK_PMU_INTERRUPT_ENABLE 0x050 |
38 | #define STARLINK_PMU_COUNTER_OVERFLOW_STATUS 0x048 |
39 | #define STARLINK_PMU_CYCLE_OVERFLOW_MASK BIT_ULL(63) |
40 | |
41 | #define STARLINK_CYCLES 0x058 |
42 | #define CACHE_READ_REQUEST 0x04000701 |
43 | #define CACHE_WRITE_REQUEST 0x03000001 |
44 | #define CACHE_RELEASE_REQUEST 0x0003e001 |
45 | #define CACHE_READ_HIT 0x00901202 |
46 | #define CACHE_READ_MISS 0x04008002 |
47 | #define CACHE_WRITE_HIT 0x006c0002 |
48 | #define CACHE_WRITE_MISS 0x03000002 |
49 | #define CACHE_WRITEBACK 0x00000403 |
50 | |
51 | #define to_starlink_pmu(p) (container_of(p, struct starlink_pmu, pmu)) |
52 | |
53 | #define STARLINK_FORMAT_ATTR(_name, _config) \ |
54 | (&((struct dev_ext_attribute[]) { \ |
55 | { .attr = __ATTR(_name, 0444, starlink_pmu_sysfs_format_show, NULL), \ |
56 | .var = (void *)_config, } \ |
57 | })[0].attr.attr) |
58 | |
59 | #define STARLINK_EVENT_ATTR(_name, _id) \ |
60 | PMU_EVENT_ATTR_ID(_name, starlink_pmu_sysfs_event_show, _id) |
61 | |
62 | static int starlink_pmu_cpuhp_state; |
63 | |
64 | struct starlink_hw_events { |
65 | struct perf_event *events[STARLINK_PMU_MAX_COUNTERS]; |
66 | DECLARE_BITMAP(used_mask, STARLINK_PMU_MAX_COUNTERS); |
67 | }; |
68 | |
69 | struct starlink_pmu { |
70 | struct pmu pmu; |
71 | struct starlink_hw_events __percpu *hw_events; |
72 | struct hlist_node node; |
73 | struct notifier_block starlink_pmu_pm_nb; |
74 | void __iomem *pmu_base; |
75 | cpumask_t cpumask; |
76 | int irq; |
77 | }; |
78 | |
79 | static ssize_t |
80 | starlink_pmu_sysfs_format_show(struct device *dev, |
81 | struct device_attribute *attr, |
82 | char *buf) |
83 | { |
84 | struct dev_ext_attribute *eattr = container_of(attr, |
85 | struct dev_ext_attribute, attr); |
86 | |
87 | return sysfs_emit(buf, fmt: "%s\n" , (char *)eattr->var); |
88 | } |
89 | |
90 | static struct attribute *starlink_pmu_format_attrs[] = { |
91 | STARLINK_FORMAT_ATTR(event, "config:0-31" ), |
92 | NULL |
93 | }; |
94 | |
95 | static const struct attribute_group starlink_pmu_format_attr_group = { |
96 | .name = "format" , |
97 | .attrs = starlink_pmu_format_attrs, |
98 | }; |
99 | |
100 | static ssize_t |
101 | starlink_pmu_sysfs_event_show(struct device *dev, |
102 | struct device_attribute *attr, |
103 | char *buf) |
104 | { |
105 | struct perf_pmu_events_attr *eattr = container_of(attr, |
106 | struct perf_pmu_events_attr, attr); |
107 | |
108 | return sysfs_emit(buf, fmt: "event=0x%02llx\n" , eattr->id); |
109 | } |
110 | |
111 | static struct attribute *starlink_pmu_event_attrs[] = { |
112 | STARLINK_EVENT_ATTR(cycles, STARLINK_CYCLES), |
113 | STARLINK_EVENT_ATTR(read_request, CACHE_READ_REQUEST), |
114 | STARLINK_EVENT_ATTR(write_request, CACHE_WRITE_REQUEST), |
115 | STARLINK_EVENT_ATTR(release_request, CACHE_RELEASE_REQUEST), |
116 | STARLINK_EVENT_ATTR(read_hit, CACHE_READ_HIT), |
117 | STARLINK_EVENT_ATTR(read_miss, CACHE_READ_MISS), |
118 | STARLINK_EVENT_ATTR(write_hit, CACHE_WRITE_HIT), |
119 | STARLINK_EVENT_ATTR(write_miss, CACHE_WRITE_MISS), |
120 | STARLINK_EVENT_ATTR(writeback, CACHE_WRITEBACK), |
121 | NULL |
122 | }; |
123 | |
124 | static const struct attribute_group starlink_pmu_events_attr_group = { |
125 | .name = "events" , |
126 | .attrs = starlink_pmu_event_attrs, |
127 | }; |
128 | |
129 | static ssize_t |
130 | cpumask_show(struct device *dev, struct device_attribute *attr, char *buf) |
131 | { |
132 | struct starlink_pmu *starlink_pmu = to_starlink_pmu(dev_get_drvdata(dev)); |
133 | |
134 | return cpumap_print_to_pagebuf(list: true, buf, mask: &starlink_pmu->cpumask); |
135 | } |
136 | |
137 | static DEVICE_ATTR_RO(cpumask); |
138 | |
139 | static struct attribute *starlink_pmu_cpumask_attrs[] = { |
140 | &dev_attr_cpumask.attr, |
141 | NULL |
142 | }; |
143 | |
144 | static const struct attribute_group starlink_pmu_cpumask_attr_group = { |
145 | .attrs = starlink_pmu_cpumask_attrs, |
146 | }; |
147 | |
148 | static const struct attribute_group *starlink_pmu_attr_groups[] = { |
149 | &starlink_pmu_format_attr_group, |
150 | &starlink_pmu_events_attr_group, |
151 | &starlink_pmu_cpumask_attr_group, |
152 | NULL |
153 | }; |
154 | |
155 | static void starlink_pmu_set_event_period(struct perf_event *event) |
156 | { |
157 | struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu); |
158 | struct hw_perf_event *hwc = &event->hw; |
159 | int idx = event->hw.idx; |
160 | |
161 | /* |
162 | * Program counter to half of it's max count to handle |
163 | * cases of extreme interrupt latency. |
164 | */ |
165 | u64 val = STARLINK_PMU_COUNTER_MASK >> 1; |
166 | |
167 | local64_set(&hwc->prev_count, val); |
168 | if (hwc->config == STARLINK_CYCLES) |
169 | writeq(val, addr: starlink_pmu->pmu_base + STARLINK_PMU_CYCLE_COUNTER); |
170 | else |
171 | writeq(val, addr: starlink_pmu->pmu_base + STARLINK_PMU_EVENT_COUNTER + |
172 | idx * sizeof(u64)); |
173 | } |
174 | |
175 | static void starlink_pmu_counter_start(struct perf_event *event, |
176 | struct starlink_pmu *starlink_pmu) |
177 | { |
178 | struct hw_perf_event *hwc = &event->hw; |
179 | int idx = event->hw.idx; |
180 | u64 val; |
181 | |
182 | /* |
183 | * Enable counter overflow interrupt[63:0], |
184 | * which is mapped as follow: |
185 | * |
186 | * event counter 0 - Bit [0] |
187 | * event counter 1 - Bit [1] |
188 | * ... |
189 | * cycle counter - Bit [63] |
190 | */ |
191 | val = readq(addr: starlink_pmu->pmu_base + STARLINK_PMU_INTERRUPT_ENABLE); |
192 | |
193 | if (hwc->config == STARLINK_CYCLES) { |
194 | /* |
195 | * Cycle count has its dedicated register, and it starts |
196 | * counting as soon as STARLINK_PMU_GLOBAL_ENABLE is set. |
197 | */ |
198 | val |= STARLINK_PMU_CYCLE_OVERFLOW_MASK; |
199 | } else { |
200 | writeq(val: event->hw.config, addr: starlink_pmu->pmu_base + |
201 | STARLINK_PMU_EVENT_SELECT + idx * sizeof(u64)); |
202 | |
203 | val |= BIT_ULL(idx); |
204 | } |
205 | |
206 | writeq(val, addr: starlink_pmu->pmu_base + STARLINK_PMU_INTERRUPT_ENABLE); |
207 | |
208 | writeq(STARLINK_PMU_GLOBAL_ENABLE, addr: starlink_pmu->pmu_base + |
209 | STARLINK_PMU_CONTROL); |
210 | } |
211 | |
212 | static void starlink_pmu_counter_stop(struct perf_event *event, |
213 | struct starlink_pmu *starlink_pmu) |
214 | { |
215 | struct hw_perf_event *hwc = &event->hw; |
216 | int idx = event->hw.idx; |
217 | u64 val; |
218 | |
219 | val = readq(addr: starlink_pmu->pmu_base + STARLINK_PMU_CONTROL); |
220 | val &= ~STARLINK_PMU_GLOBAL_ENABLE; |
221 | writeq(val, addr: starlink_pmu->pmu_base + STARLINK_PMU_CONTROL); |
222 | |
223 | val = readq(addr: starlink_pmu->pmu_base + STARLINK_PMU_INTERRUPT_ENABLE); |
224 | if (hwc->config == STARLINK_CYCLES) |
225 | val &= ~STARLINK_PMU_CYCLE_OVERFLOW_MASK; |
226 | else |
227 | val &= ~BIT_ULL(idx); |
228 | |
229 | writeq(val, addr: starlink_pmu->pmu_base + STARLINK_PMU_INTERRUPT_ENABLE); |
230 | } |
231 | |
232 | static void starlink_pmu_update(struct perf_event *event) |
233 | { |
234 | struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu); |
235 | struct hw_perf_event *hwc = &event->hw; |
236 | int idx = hwc->idx; |
237 | u64 prev_raw_count, new_raw_count; |
238 | u64 oldval; |
239 | u64 delta; |
240 | |
241 | do { |
242 | prev_raw_count = local64_read(&hwc->prev_count); |
243 | if (hwc->config == STARLINK_CYCLES) |
244 | new_raw_count = readq(addr: starlink_pmu->pmu_base + |
245 | STARLINK_PMU_CYCLE_COUNTER); |
246 | else |
247 | new_raw_count = readq(addr: starlink_pmu->pmu_base + |
248 | STARLINK_PMU_EVENT_COUNTER + |
249 | idx * sizeof(u64)); |
250 | oldval = local64_cmpxchg(l: &hwc->prev_count, old: prev_raw_count, |
251 | new: new_raw_count); |
252 | } while (oldval != prev_raw_count); |
253 | |
254 | delta = (new_raw_count - prev_raw_count) & STARLINK_PMU_COUNTER_MASK; |
255 | local64_add(delta, &event->count); |
256 | } |
257 | |
258 | static void starlink_pmu_start(struct perf_event *event, int flags) |
259 | { |
260 | struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu); |
261 | struct hw_perf_event *hwc = &event->hw; |
262 | |
263 | if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED))) |
264 | return; |
265 | |
266 | if (flags & PERF_EF_RELOAD) |
267 | WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); |
268 | |
269 | hwc->state = 0; |
270 | |
271 | starlink_pmu_set_event_period(event); |
272 | starlink_pmu_counter_start(event, starlink_pmu); |
273 | |
274 | perf_event_update_userpage(event); |
275 | } |
276 | |
277 | static void starlink_pmu_stop(struct perf_event *event, int flags) |
278 | { |
279 | struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu); |
280 | struct hw_perf_event *hwc = &event->hw; |
281 | |
282 | if (hwc->state & PERF_HES_STOPPED) |
283 | return; |
284 | |
285 | starlink_pmu_counter_stop(event, starlink_pmu); |
286 | starlink_pmu_update(event); |
287 | hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; |
288 | } |
289 | |
290 | static int starlink_pmu_add(struct perf_event *event, int flags) |
291 | { |
292 | struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu); |
293 | struct starlink_hw_events *hw_events = |
294 | this_cpu_ptr(starlink_pmu->hw_events); |
295 | struct hw_perf_event *hwc = &event->hw; |
296 | unsigned long *used_mask = hw_events->used_mask; |
297 | u32 n_events = STARLINK_PMU_NUM_COUNTERS; |
298 | int idx; |
299 | |
300 | /* |
301 | * Cycle counter has dedicated register to hold counter value. |
302 | * Event other than cycle count has to be enabled through |
303 | * event select register, and assigned with independent counter |
304 | * as they appear. |
305 | */ |
306 | |
307 | if (hwc->config == STARLINK_CYCLES) { |
308 | idx = STARLINK_PMU_IDX_CYCLE_COUNTER; |
309 | } else { |
310 | idx = find_first_zero_bit(addr: used_mask, size: n_events); |
311 | /* All counter are in use */ |
312 | if (idx < 0) |
313 | return idx; |
314 | |
315 | set_bit(nr: idx, addr: used_mask); |
316 | } |
317 | |
318 | hwc->idx = idx; |
319 | hw_events->events[idx] = event; |
320 | hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; |
321 | |
322 | if (flags & PERF_EF_START) |
323 | starlink_pmu_start(event, PERF_EF_RELOAD); |
324 | |
325 | perf_event_update_userpage(event); |
326 | |
327 | return 0; |
328 | } |
329 | |
330 | static void starlink_pmu_del(struct perf_event *event, int flags) |
331 | { |
332 | struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu); |
333 | struct starlink_hw_events *hw_events = |
334 | this_cpu_ptr(starlink_pmu->hw_events); |
335 | struct hw_perf_event *hwc = &event->hw; |
336 | |
337 | starlink_pmu_stop(event, PERF_EF_UPDATE); |
338 | hw_events->events[hwc->idx] = NULL; |
339 | clear_bit(nr: hwc->idx, addr: hw_events->used_mask); |
340 | |
341 | perf_event_update_userpage(event); |
342 | } |
343 | |
344 | static bool starlink_pmu_validate_event_group(struct perf_event *event) |
345 | { |
346 | struct perf_event *leader = event->group_leader; |
347 | struct perf_event *sibling; |
348 | int counter = 1; |
349 | |
350 | /* |
351 | * Ensure hardware events in the group are on the same PMU, |
352 | * software events are acceptable. |
353 | */ |
354 | if (event->group_leader->pmu != event->pmu && |
355 | !is_software_event(event: event->group_leader)) |
356 | return false; |
357 | |
358 | for_each_sibling_event(sibling, leader) { |
359 | if (sibling->pmu != event->pmu && !is_software_event(event: sibling)) |
360 | return false; |
361 | |
362 | counter++; |
363 | } |
364 | |
365 | return counter <= STARLINK_PMU_NUM_COUNTERS; |
366 | } |
367 | |
368 | static int starlink_pmu_event_init(struct perf_event *event) |
369 | { |
370 | struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu); |
371 | struct hw_perf_event *hwc = &event->hw; |
372 | |
373 | /* |
374 | * Sampling is not supported, as counters are shared |
375 | * by all CPU. |
376 | */ |
377 | if (hwc->sample_period) |
378 | return -EOPNOTSUPP; |
379 | |
380 | /* |
381 | * Per-task and attach to a task are not supported, |
382 | * as uncore events are not specific to any CPU. |
383 | */ |
384 | if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK) |
385 | return -EOPNOTSUPP; |
386 | |
387 | if (!starlink_pmu_validate_event_group(event)) |
388 | return -EINVAL; |
389 | |
390 | hwc->idx = -1; |
391 | hwc->config = event->attr.config; |
392 | event->cpu = cpumask_first(srcp: &starlink_pmu->cpumask); |
393 | |
394 | return 0; |
395 | } |
396 | |
397 | static irqreturn_t starlink_pmu_handle_irq(int irq_num, void *data) |
398 | { |
399 | struct starlink_pmu *starlink_pmu = data; |
400 | struct starlink_hw_events *hw_events = |
401 | this_cpu_ptr(starlink_pmu->hw_events); |
402 | bool handled = false; |
403 | int idx; |
404 | u64 overflow_status; |
405 | |
406 | for (idx = 0; idx < STARLINK_PMU_MAX_COUNTERS; idx++) { |
407 | struct perf_event *event = hw_events->events[idx]; |
408 | |
409 | if (!event) |
410 | continue; |
411 | |
412 | overflow_status = readq(addr: starlink_pmu->pmu_base + |
413 | STARLINK_PMU_COUNTER_OVERFLOW_STATUS); |
414 | if (!(overflow_status & BIT_ULL(idx))) |
415 | continue; |
416 | |
417 | writeq(BIT_ULL(idx), addr: starlink_pmu->pmu_base + |
418 | STARLINK_PMU_COUNTER_OVERFLOW_STATUS); |
419 | |
420 | starlink_pmu_update(event); |
421 | starlink_pmu_set_event_period(event); |
422 | handled = true; |
423 | } |
424 | return IRQ_RETVAL(handled); |
425 | } |
426 | |
427 | static int starlink_setup_irqs(struct starlink_pmu *starlink_pmu, |
428 | struct platform_device *pdev) |
429 | { |
430 | int ret, irq; |
431 | |
432 | irq = platform_get_irq(pdev, 0); |
433 | if (irq < 0) |
434 | return -EINVAL; |
435 | |
436 | ret = devm_request_irq(dev: &pdev->dev, irq, handler: starlink_pmu_handle_irq, |
437 | irqflags: 0, STARLINK_PMU_PDEV_NAME, dev_id: starlink_pmu); |
438 | if (ret) |
439 | return dev_err_probe(dev: &pdev->dev, err: ret, fmt: "Failed to request IRQ\n" ); |
440 | |
441 | starlink_pmu->irq = irq; |
442 | |
443 | return 0; |
444 | } |
445 | |
446 | static int starlink_pmu_pm_notify(struct notifier_block *b, |
447 | unsigned long cmd, void *v) |
448 | { |
449 | struct starlink_pmu *starlink_pmu = container_of(b, struct starlink_pmu, |
450 | starlink_pmu_pm_nb); |
451 | struct starlink_hw_events *hw_events = |
452 | this_cpu_ptr(starlink_pmu->hw_events); |
453 | int enabled = bitmap_weight(src: hw_events->used_mask, |
454 | STARLINK_PMU_MAX_COUNTERS); |
455 | struct perf_event *event; |
456 | int idx; |
457 | |
458 | if (!enabled) |
459 | return NOTIFY_OK; |
460 | |
461 | for (idx = 0; idx < STARLINK_PMU_MAX_COUNTERS; idx++) { |
462 | event = hw_events->events[idx]; |
463 | if (!event) |
464 | continue; |
465 | |
466 | switch (cmd) { |
467 | case CPU_PM_ENTER: |
468 | /* Stop and update the counter */ |
469 | starlink_pmu_stop(event, PERF_EF_UPDATE); |
470 | break; |
471 | case CPU_PM_EXIT: |
472 | case CPU_PM_ENTER_FAILED: |
473 | /* Restore and enable the counter */ |
474 | starlink_pmu_start(event, PERF_EF_RELOAD); |
475 | break; |
476 | default: |
477 | break; |
478 | } |
479 | } |
480 | |
481 | return NOTIFY_OK; |
482 | } |
483 | |
484 | static int starlink_pmu_pm_register(struct starlink_pmu *starlink_pmu) |
485 | { |
486 | if (!IS_ENABLED(CONFIG_CPU_PM)) |
487 | return 0; |
488 | |
489 | starlink_pmu->starlink_pmu_pm_nb.notifier_call = starlink_pmu_pm_notify; |
490 | return cpu_pm_register_notifier(nb: &starlink_pmu->starlink_pmu_pm_nb); |
491 | } |
492 | |
493 | static void starlink_pmu_pm_unregister(struct starlink_pmu *starlink_pmu) |
494 | { |
495 | if (!IS_ENABLED(CONFIG_CPU_PM)) |
496 | return; |
497 | |
498 | cpu_pm_unregister_notifier(nb: &starlink_pmu->starlink_pmu_pm_nb); |
499 | } |
500 | |
501 | static void starlink_pmu_destroy(struct starlink_pmu *starlink_pmu) |
502 | { |
503 | starlink_pmu_pm_unregister(starlink_pmu); |
504 | cpuhp_state_remove_instance(state: starlink_pmu_cpuhp_state, |
505 | node: &starlink_pmu->node); |
506 | } |
507 | |
508 | static int starlink_pmu_probe(struct platform_device *pdev) |
509 | { |
510 | struct starlink_pmu *starlink_pmu; |
511 | struct starlink_hw_events *hw_events; |
512 | struct resource *res; |
513 | int cpuid, i, ret; |
514 | |
515 | starlink_pmu = devm_kzalloc(dev: &pdev->dev, size: sizeof(*starlink_pmu), GFP_KERNEL); |
516 | if (!starlink_pmu) |
517 | return -ENOMEM; |
518 | |
519 | starlink_pmu->pmu_base = |
520 | devm_platform_get_and_ioremap_resource(pdev, index: 0, res: &res); |
521 | if (IS_ERR(ptr: starlink_pmu->pmu_base)) |
522 | return PTR_ERR(ptr: starlink_pmu->pmu_base); |
523 | |
524 | starlink_pmu->hw_events = alloc_percpu_gfp(struct starlink_hw_events, |
525 | GFP_KERNEL); |
526 | if (!starlink_pmu->hw_events) { |
527 | dev_err(&pdev->dev, "Failed to allocate per-cpu PMU data\n" ); |
528 | return -ENOMEM; |
529 | } |
530 | |
531 | for_each_possible_cpu(cpuid) { |
532 | hw_events = per_cpu_ptr(starlink_pmu->hw_events, cpuid); |
533 | for (i = 0; i < STARLINK_PMU_MAX_COUNTERS; i++) |
534 | hw_events->events[i] = NULL; |
535 | } |
536 | |
537 | ret = starlink_setup_irqs(starlink_pmu, pdev); |
538 | if (ret) |
539 | return ret; |
540 | |
541 | ret = cpuhp_state_add_instance(state: starlink_pmu_cpuhp_state, |
542 | node: &starlink_pmu->node); |
543 | if (ret) { |
544 | dev_err(&pdev->dev, "Failed to register hotplug\n" ); |
545 | return ret; |
546 | } |
547 | |
548 | ret = starlink_pmu_pm_register(starlink_pmu); |
549 | if (ret) { |
550 | cpuhp_state_remove_instance(state: starlink_pmu_cpuhp_state, |
551 | node: &starlink_pmu->node); |
552 | return ret; |
553 | } |
554 | |
555 | starlink_pmu->pmu = (struct pmu) { |
556 | .task_ctx_nr = perf_invalid_context, |
557 | .event_init = starlink_pmu_event_init, |
558 | .add = starlink_pmu_add, |
559 | .del = starlink_pmu_del, |
560 | .start = starlink_pmu_start, |
561 | .stop = starlink_pmu_stop, |
562 | .read = starlink_pmu_update, |
563 | .attr_groups = starlink_pmu_attr_groups, |
564 | }; |
565 | |
566 | ret = perf_pmu_register(pmu: &starlink_pmu->pmu, STARLINK_PMU_PDEV_NAME, type: -1); |
567 | if (ret) |
568 | starlink_pmu_destroy(starlink_pmu); |
569 | |
570 | return ret; |
571 | } |
572 | |
573 | static const struct of_device_id starlink_pmu_of_match[] = { |
574 | { .compatible = "starfive,jh8100-starlink-pmu" }, |
575 | {} |
576 | }; |
577 | MODULE_DEVICE_TABLE(of, starlink_pmu_of_match); |
578 | |
579 | static struct platform_driver starlink_pmu_driver = { |
580 | .driver = { |
581 | .name = STARLINK_PMU_PDEV_NAME, |
582 | .of_match_table = starlink_pmu_of_match, |
583 | .suppress_bind_attrs = true, |
584 | }, |
585 | .probe = starlink_pmu_probe, |
586 | }; |
587 | |
588 | static int |
589 | starlink_pmu_online_cpu(unsigned int cpu, struct hlist_node *node) |
590 | { |
591 | struct starlink_pmu *starlink_pmu = hlist_entry_safe(node, |
592 | struct starlink_pmu, |
593 | node); |
594 | |
595 | if (cpumask_empty(srcp: &starlink_pmu->cpumask)) |
596 | cpumask_set_cpu(cpu, dstp: &starlink_pmu->cpumask); |
597 | |
598 | WARN_ON(irq_set_affinity(starlink_pmu->irq, cpumask_of(cpu))); |
599 | |
600 | return 0; |
601 | } |
602 | |
603 | static int |
604 | starlink_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) |
605 | { |
606 | struct starlink_pmu *starlink_pmu = hlist_entry_safe(node, |
607 | struct starlink_pmu, |
608 | node); |
609 | unsigned int target; |
610 | |
611 | if (!cpumask_test_and_clear_cpu(cpu, cpumask: &starlink_pmu->cpumask)) |
612 | return 0; |
613 | |
614 | target = cpumask_any_but(cpu_online_mask, cpu); |
615 | if (target >= nr_cpu_ids) |
616 | return 0; |
617 | |
618 | perf_pmu_migrate_context(pmu: &starlink_pmu->pmu, src_cpu: cpu, dst_cpu: target); |
619 | |
620 | cpumask_set_cpu(cpu: target, dstp: &starlink_pmu->cpumask); |
621 | WARN_ON(irq_set_affinity(starlink_pmu->irq, cpumask_of(target))); |
622 | |
623 | return 0; |
624 | } |
625 | |
626 | static int __init starlink_pmu_init(void) |
627 | { |
628 | int ret; |
629 | |
630 | ret = cpuhp_setup_state_multi(state: CPUHP_AP_ONLINE_DYN, |
631 | name: "soc/starfive/starlink_pmu:online" , |
632 | startup: starlink_pmu_online_cpu, |
633 | teardown: starlink_pmu_offline_cpu); |
634 | if (ret < 0) |
635 | return ret; |
636 | |
637 | starlink_pmu_cpuhp_state = ret; |
638 | |
639 | return platform_driver_register(&starlink_pmu_driver); |
640 | } |
641 | |
642 | device_initcall(starlink_pmu_init); |
643 | |