1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Copyright(c) 2020 Intel Corporation. All rights rsvd. */ |
3 | |
4 | #include <linux/sched/task.h> |
5 | #include <linux/io-64-nonatomic-lo-hi.h> |
6 | #include "idxd.h" |
7 | #include "perfmon.h" |
8 | |
9 | static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr, |
10 | char *buf); |
11 | |
12 | static cpumask_t perfmon_dsa_cpu_mask; |
13 | static bool cpuhp_set_up; |
14 | static enum cpuhp_state cpuhp_slot; |
15 | |
16 | /* |
17 | * perf userspace reads this attribute to determine which cpus to open |
18 | * counters on. It's connected to perfmon_dsa_cpu_mask, which is |
19 | * maintained by the cpu hotplug handlers. |
20 | */ |
21 | static DEVICE_ATTR_RO(cpumask); |
22 | |
23 | static struct attribute *perfmon_cpumask_attrs[] = { |
24 | &dev_attr_cpumask.attr, |
25 | NULL, |
26 | }; |
27 | |
28 | static struct attribute_group cpumask_attr_group = { |
29 | .attrs = perfmon_cpumask_attrs, |
30 | }; |
31 | |
32 | /* |
33 | * These attributes specify the bits in the config word that the perf |
34 | * syscall uses to pass the event ids and categories to perfmon. |
35 | */ |
36 | DEFINE_PERFMON_FORMAT_ATTR(event_category, "config:0-3" ); |
37 | DEFINE_PERFMON_FORMAT_ATTR(event, "config:4-31" ); |
38 | |
39 | /* |
40 | * These attributes specify the bits in the config1 word that the perf |
41 | * syscall uses to pass filter data to perfmon. |
42 | */ |
43 | DEFINE_PERFMON_FORMAT_ATTR(filter_wq, "config1:0-31" ); |
44 | DEFINE_PERFMON_FORMAT_ATTR(filter_tc, "config1:32-39" ); |
45 | DEFINE_PERFMON_FORMAT_ATTR(filter_pgsz, "config1:40-43" ); |
46 | DEFINE_PERFMON_FORMAT_ATTR(filter_sz, "config1:44-51" ); |
47 | DEFINE_PERFMON_FORMAT_ATTR(filter_eng, "config1:52-59" ); |
48 | |
49 | #define PERFMON_FILTERS_START 2 |
50 | #define PERFMON_FILTERS_MAX 5 |
51 | |
52 | static struct attribute *perfmon_format_attrs[] = { |
53 | &format_attr_idxd_event_category.attr, |
54 | &format_attr_idxd_event.attr, |
55 | &format_attr_idxd_filter_wq.attr, |
56 | &format_attr_idxd_filter_tc.attr, |
57 | &format_attr_idxd_filter_pgsz.attr, |
58 | &format_attr_idxd_filter_sz.attr, |
59 | &format_attr_idxd_filter_eng.attr, |
60 | NULL, |
61 | }; |
62 | |
63 | static struct attribute_group perfmon_format_attr_group = { |
64 | .name = "format" , |
65 | .attrs = perfmon_format_attrs, |
66 | }; |
67 | |
68 | static const struct attribute_group *perfmon_attr_groups[] = { |
69 | &perfmon_format_attr_group, |
70 | &cpumask_attr_group, |
71 | NULL, |
72 | }; |
73 | |
74 | static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr, |
75 | char *buf) |
76 | { |
77 | return cpumap_print_to_pagebuf(list: true, buf, mask: &perfmon_dsa_cpu_mask); |
78 | } |
79 | |
80 | static bool is_idxd_event(struct idxd_pmu *idxd_pmu, struct perf_event *event) |
81 | { |
82 | return &idxd_pmu->pmu == event->pmu; |
83 | } |
84 | |
85 | static int perfmon_collect_events(struct idxd_pmu *idxd_pmu, |
86 | struct perf_event *leader, |
87 | bool do_grp) |
88 | { |
89 | struct perf_event *event; |
90 | int n, max_count; |
91 | |
92 | max_count = idxd_pmu->n_counters; |
93 | n = idxd_pmu->n_events; |
94 | |
95 | if (n >= max_count) |
96 | return -EINVAL; |
97 | |
98 | if (is_idxd_event(idxd_pmu, event: leader)) { |
99 | idxd_pmu->event_list[n] = leader; |
100 | idxd_pmu->event_list[n]->hw.idx = n; |
101 | n++; |
102 | } |
103 | |
104 | if (!do_grp) |
105 | return n; |
106 | |
107 | for_each_sibling_event(event, leader) { |
108 | if (!is_idxd_event(idxd_pmu, event) || |
109 | event->state <= PERF_EVENT_STATE_OFF) |
110 | continue; |
111 | |
112 | if (n >= max_count) |
113 | return -EINVAL; |
114 | |
115 | idxd_pmu->event_list[n] = event; |
116 | idxd_pmu->event_list[n]->hw.idx = n; |
117 | n++; |
118 | } |
119 | |
120 | return n; |
121 | } |
122 | |
123 | static void perfmon_assign_hw_event(struct idxd_pmu *idxd_pmu, |
124 | struct perf_event *event, int idx) |
125 | { |
126 | struct idxd_device *idxd = idxd_pmu->idxd; |
127 | struct hw_perf_event *hwc = &event->hw; |
128 | |
129 | hwc->idx = idx; |
130 | hwc->config_base = ioread64(CNTRCFG_REG(idxd, idx)); |
131 | hwc->event_base = ioread64(CNTRCFG_REG(idxd, idx)); |
132 | } |
133 | |
134 | static int perfmon_assign_event(struct idxd_pmu *idxd_pmu, |
135 | struct perf_event *event) |
136 | { |
137 | int i; |
138 | |
139 | for (i = 0; i < IDXD_PMU_EVENT_MAX; i++) |
140 | if (!test_and_set_bit(nr: i, addr: idxd_pmu->used_mask)) |
141 | return i; |
142 | |
143 | return -EINVAL; |
144 | } |
145 | |
146 | /* |
147 | * Check whether there are enough counters to satisfy that all the |
148 | * events in the group can actually be scheduled at the same time. |
149 | * |
150 | * To do this, create a fake idxd_pmu object so the event collection |
151 | * and assignment functions can be used without affecting the internal |
152 | * state of the real idxd_pmu object. |
153 | */ |
154 | static int perfmon_validate_group(struct idxd_pmu *pmu, |
155 | struct perf_event *event) |
156 | { |
157 | struct perf_event *leader = event->group_leader; |
158 | struct idxd_pmu *fake_pmu; |
159 | int i, ret = 0, n, idx; |
160 | |
161 | fake_pmu = kzalloc(size: sizeof(*fake_pmu), GFP_KERNEL); |
162 | if (!fake_pmu) |
163 | return -ENOMEM; |
164 | |
165 | fake_pmu->pmu.name = pmu->pmu.name; |
166 | fake_pmu->n_counters = pmu->n_counters; |
167 | |
168 | n = perfmon_collect_events(idxd_pmu: fake_pmu, leader, do_grp: true); |
169 | if (n < 0) { |
170 | ret = n; |
171 | goto out; |
172 | } |
173 | |
174 | fake_pmu->n_events = n; |
175 | n = perfmon_collect_events(idxd_pmu: fake_pmu, leader: event, do_grp: false); |
176 | if (n < 0) { |
177 | ret = n; |
178 | goto out; |
179 | } |
180 | |
181 | fake_pmu->n_events = n; |
182 | |
183 | for (i = 0; i < n; i++) { |
184 | event = fake_pmu->event_list[i]; |
185 | |
186 | idx = perfmon_assign_event(idxd_pmu: fake_pmu, event); |
187 | if (idx < 0) { |
188 | ret = idx; |
189 | goto out; |
190 | } |
191 | } |
192 | out: |
193 | kfree(objp: fake_pmu); |
194 | |
195 | return ret; |
196 | } |
197 | |
198 | static int perfmon_pmu_event_init(struct perf_event *event) |
199 | { |
200 | struct idxd_device *idxd; |
201 | int ret = 0; |
202 | |
203 | idxd = event_to_idxd(event); |
204 | event->hw.idx = -1; |
205 | |
206 | if (event->attr.type != event->pmu->type) |
207 | return -ENOENT; |
208 | |
209 | /* sampling not supported */ |
210 | if (event->attr.sample_period) |
211 | return -EINVAL; |
212 | |
213 | if (event->cpu < 0) |
214 | return -EINVAL; |
215 | |
216 | if (event->pmu != &idxd->idxd_pmu->pmu) |
217 | return -EINVAL; |
218 | |
219 | event->hw.event_base = ioread64(PERFMON_TABLE_OFFSET(idxd)); |
220 | event->cpu = idxd->idxd_pmu->cpu; |
221 | event->hw.config = event->attr.config; |
222 | |
223 | if (event->group_leader != event) |
224 | /* non-group events have themselves as leader */ |
225 | ret = perfmon_validate_group(pmu: idxd->idxd_pmu, event); |
226 | |
227 | return ret; |
228 | } |
229 | |
230 | static inline u64 perfmon_pmu_read_counter(struct perf_event *event) |
231 | { |
232 | struct hw_perf_event *hwc = &event->hw; |
233 | struct idxd_device *idxd; |
234 | int cntr = hwc->idx; |
235 | |
236 | idxd = event_to_idxd(event); |
237 | |
238 | return ioread64(CNTRDATA_REG(idxd, cntr)); |
239 | } |
240 | |
241 | static void perfmon_pmu_event_update(struct perf_event *event) |
242 | { |
243 | struct idxd_device *idxd = event_to_idxd(event); |
244 | u64 prev_raw_count, new_raw_count, delta, p, n; |
245 | int shift = 64 - idxd->idxd_pmu->counter_width; |
246 | struct hw_perf_event *hwc = &event->hw; |
247 | |
248 | prev_raw_count = local64_read(&hwc->prev_count); |
249 | do { |
250 | new_raw_count = perfmon_pmu_read_counter(event); |
251 | } while (!local64_try_cmpxchg(l: &hwc->prev_count, |
252 | old: &prev_raw_count, new: new_raw_count)); |
253 | n = (new_raw_count << shift); |
254 | p = (prev_raw_count << shift); |
255 | |
256 | delta = ((n - p) >> shift); |
257 | |
258 | local64_add(delta, &event->count); |
259 | } |
260 | |
261 | void perfmon_counter_overflow(struct idxd_device *idxd) |
262 | { |
263 | int i, n_counters, max_loop = OVERFLOW_SIZE; |
264 | struct perf_event *event; |
265 | unsigned long ovfstatus; |
266 | |
267 | n_counters = min(idxd->idxd_pmu->n_counters, OVERFLOW_SIZE); |
268 | |
269 | ovfstatus = ioread32(OVFSTATUS_REG(idxd)); |
270 | |
271 | /* |
272 | * While updating overflowed counters, other counters behind |
273 | * them could overflow and be missed in a given pass. |
274 | * Normally this could happen at most n_counters times, but in |
275 | * theory a tiny counter width could result in continual |
276 | * overflows and endless looping. max_loop provides a |
277 | * failsafe in that highly unlikely case. |
278 | */ |
279 | while (ovfstatus && max_loop--) { |
280 | /* Figure out which counter(s) overflowed */ |
281 | for_each_set_bit(i, &ovfstatus, n_counters) { |
282 | unsigned long ovfstatus_clear = 0; |
283 | |
284 | /* Update event->count for overflowed counter */ |
285 | event = idxd->idxd_pmu->event_list[i]; |
286 | perfmon_pmu_event_update(event); |
287 | /* Writing 1 to OVFSTATUS bit clears it */ |
288 | set_bit(nr: i, addr: &ovfstatus_clear); |
289 | iowrite32(ovfstatus_clear, OVFSTATUS_REG(idxd)); |
290 | } |
291 | |
292 | ovfstatus = ioread32(OVFSTATUS_REG(idxd)); |
293 | } |
294 | |
295 | /* |
296 | * Should never happen. If so, it means a counter(s) looped |
297 | * around twice while this handler was running. |
298 | */ |
299 | WARN_ON_ONCE(ovfstatus); |
300 | } |
301 | |
302 | static inline void perfmon_reset_config(struct idxd_device *idxd) |
303 | { |
304 | iowrite32(CONFIG_RESET, PERFRST_REG(idxd)); |
305 | iowrite32(0, OVFSTATUS_REG(idxd)); |
306 | iowrite32(0, PERFFRZ_REG(idxd)); |
307 | } |
308 | |
309 | static inline void perfmon_reset_counters(struct idxd_device *idxd) |
310 | { |
311 | iowrite32(CNTR_RESET, PERFRST_REG(idxd)); |
312 | } |
313 | |
314 | static inline void perfmon_reset(struct idxd_device *idxd) |
315 | { |
316 | perfmon_reset_config(idxd); |
317 | perfmon_reset_counters(idxd); |
318 | } |
319 | |
320 | static void perfmon_pmu_event_start(struct perf_event *event, int mode) |
321 | { |
322 | u32 flt_wq, flt_tc, flt_pg_sz, flt_xfer_sz, flt_eng = 0; |
323 | u64 cntr_cfg, cntrdata, event_enc, event_cat = 0; |
324 | struct hw_perf_event *hwc = &event->hw; |
325 | union filter_cfg flt_cfg; |
326 | union event_cfg event_cfg; |
327 | struct idxd_device *idxd; |
328 | int cntr; |
329 | |
330 | idxd = event_to_idxd(event); |
331 | |
332 | event->hw.idx = hwc->idx; |
333 | cntr = hwc->idx; |
334 | |
335 | /* Obtain event category and event value from user space */ |
336 | event_cfg.val = event->attr.config; |
337 | flt_cfg.val = event->attr.config1; |
338 | event_cat = event_cfg.event_cat; |
339 | event_enc = event_cfg.event_enc; |
340 | |
341 | /* Obtain filter configuration from user space */ |
342 | flt_wq = flt_cfg.wq; |
343 | flt_tc = flt_cfg.tc; |
344 | flt_pg_sz = flt_cfg.pg_sz; |
345 | flt_xfer_sz = flt_cfg.xfer_sz; |
346 | flt_eng = flt_cfg.eng; |
347 | |
348 | if (flt_wq && test_bit(FLT_WQ, &idxd->idxd_pmu->supported_filters)) |
349 | iowrite32(flt_wq, FLTCFG_REG(idxd, cntr, FLT_WQ)); |
350 | if (flt_tc && test_bit(FLT_TC, &idxd->idxd_pmu->supported_filters)) |
351 | iowrite32(flt_tc, FLTCFG_REG(idxd, cntr, FLT_TC)); |
352 | if (flt_pg_sz && test_bit(FLT_PG_SZ, &idxd->idxd_pmu->supported_filters)) |
353 | iowrite32(flt_pg_sz, FLTCFG_REG(idxd, cntr, FLT_PG_SZ)); |
354 | if (flt_xfer_sz && test_bit(FLT_XFER_SZ, &idxd->idxd_pmu->supported_filters)) |
355 | iowrite32(flt_xfer_sz, FLTCFG_REG(idxd, cntr, FLT_XFER_SZ)); |
356 | if (flt_eng && test_bit(FLT_ENG, &idxd->idxd_pmu->supported_filters)) |
357 | iowrite32(flt_eng, FLTCFG_REG(idxd, cntr, FLT_ENG)); |
358 | |
359 | /* Read the start value */ |
360 | cntrdata = ioread64(CNTRDATA_REG(idxd, cntr)); |
361 | local64_set(&event->hw.prev_count, cntrdata); |
362 | |
363 | /* Set counter to event/category */ |
364 | cntr_cfg = event_cat << CNTRCFG_CATEGORY_SHIFT; |
365 | cntr_cfg |= event_enc << CNTRCFG_EVENT_SHIFT; |
366 | /* Set interrupt on overflow and counter enable bits */ |
367 | cntr_cfg |= (CNTRCFG_IRQ_OVERFLOW | CNTRCFG_ENABLE); |
368 | |
369 | iowrite64(val: cntr_cfg, CNTRCFG_REG(idxd, cntr)); |
370 | } |
371 | |
372 | static void perfmon_pmu_event_stop(struct perf_event *event, int mode) |
373 | { |
374 | struct hw_perf_event *hwc = &event->hw; |
375 | struct idxd_device *idxd; |
376 | int i, cntr = hwc->idx; |
377 | u64 cntr_cfg; |
378 | |
379 | idxd = event_to_idxd(event); |
380 | |
381 | /* remove this event from event list */ |
382 | for (i = 0; i < idxd->idxd_pmu->n_events; i++) { |
383 | if (event != idxd->idxd_pmu->event_list[i]) |
384 | continue; |
385 | |
386 | for (++i; i < idxd->idxd_pmu->n_events; i++) |
387 | idxd->idxd_pmu->event_list[i - 1] = idxd->idxd_pmu->event_list[i]; |
388 | --idxd->idxd_pmu->n_events; |
389 | break; |
390 | } |
391 | |
392 | cntr_cfg = ioread64(CNTRCFG_REG(idxd, cntr)); |
393 | cntr_cfg &= ~CNTRCFG_ENABLE; |
394 | iowrite64(val: cntr_cfg, CNTRCFG_REG(idxd, cntr)); |
395 | |
396 | if (mode == PERF_EF_UPDATE) |
397 | perfmon_pmu_event_update(event); |
398 | |
399 | event->hw.idx = -1; |
400 | clear_bit(nr: cntr, addr: idxd->idxd_pmu->used_mask); |
401 | } |
402 | |
403 | static void perfmon_pmu_event_del(struct perf_event *event, int mode) |
404 | { |
405 | perfmon_pmu_event_stop(event, PERF_EF_UPDATE); |
406 | } |
407 | |
408 | static int perfmon_pmu_event_add(struct perf_event *event, int flags) |
409 | { |
410 | struct idxd_device *idxd = event_to_idxd(event); |
411 | struct idxd_pmu *idxd_pmu = idxd->idxd_pmu; |
412 | struct hw_perf_event *hwc = &event->hw; |
413 | int idx, n; |
414 | |
415 | n = perfmon_collect_events(idxd_pmu, leader: event, do_grp: false); |
416 | if (n < 0) |
417 | return n; |
418 | |
419 | hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; |
420 | if (!(flags & PERF_EF_START)) |
421 | hwc->state |= PERF_HES_ARCH; |
422 | |
423 | idx = perfmon_assign_event(idxd_pmu, event); |
424 | if (idx < 0) |
425 | return idx; |
426 | |
427 | perfmon_assign_hw_event(idxd_pmu, event, idx); |
428 | |
429 | if (flags & PERF_EF_START) |
430 | perfmon_pmu_event_start(event, mode: 0); |
431 | |
432 | idxd_pmu->n_events = n; |
433 | |
434 | return 0; |
435 | } |
436 | |
437 | static void enable_perfmon_pmu(struct idxd_device *idxd) |
438 | { |
439 | iowrite32(COUNTER_UNFREEZE, PERFFRZ_REG(idxd)); |
440 | } |
441 | |
442 | static void disable_perfmon_pmu(struct idxd_device *idxd) |
443 | { |
444 | iowrite32(COUNTER_FREEZE, PERFFRZ_REG(idxd)); |
445 | } |
446 | |
447 | static void perfmon_pmu_enable(struct pmu *pmu) |
448 | { |
449 | struct idxd_device *idxd = pmu_to_idxd(pmu); |
450 | |
451 | enable_perfmon_pmu(idxd); |
452 | } |
453 | |
454 | static void perfmon_pmu_disable(struct pmu *pmu) |
455 | { |
456 | struct idxd_device *idxd = pmu_to_idxd(pmu); |
457 | |
458 | disable_perfmon_pmu(idxd); |
459 | } |
460 | |
461 | static void skip_filter(int i) |
462 | { |
463 | int j; |
464 | |
465 | for (j = i; j < PERFMON_FILTERS_MAX; j++) |
466 | perfmon_format_attrs[PERFMON_FILTERS_START + j] = |
467 | perfmon_format_attrs[PERFMON_FILTERS_START + j + 1]; |
468 | } |
469 | |
470 | static void idxd_pmu_init(struct idxd_pmu *idxd_pmu) |
471 | { |
472 | int i; |
473 | |
474 | for (i = 0 ; i < PERFMON_FILTERS_MAX; i++) { |
475 | if (!test_bit(i, &idxd_pmu->supported_filters)) |
476 | skip_filter(i); |
477 | } |
478 | |
479 | idxd_pmu->pmu.name = idxd_pmu->name; |
480 | idxd_pmu->pmu.attr_groups = perfmon_attr_groups; |
481 | idxd_pmu->pmu.task_ctx_nr = perf_invalid_context; |
482 | idxd_pmu->pmu.event_init = perfmon_pmu_event_init; |
483 | idxd_pmu->pmu.pmu_enable = perfmon_pmu_enable, |
484 | idxd_pmu->pmu.pmu_disable = perfmon_pmu_disable, |
485 | idxd_pmu->pmu.add = perfmon_pmu_event_add; |
486 | idxd_pmu->pmu.del = perfmon_pmu_event_del; |
487 | idxd_pmu->pmu.start = perfmon_pmu_event_start; |
488 | idxd_pmu->pmu.stop = perfmon_pmu_event_stop; |
489 | idxd_pmu->pmu.read = perfmon_pmu_event_update; |
490 | idxd_pmu->pmu.capabilities = PERF_PMU_CAP_NO_EXCLUDE; |
491 | idxd_pmu->pmu.module = THIS_MODULE; |
492 | } |
493 | |
494 | void perfmon_pmu_remove(struct idxd_device *idxd) |
495 | { |
496 | if (!idxd->idxd_pmu) |
497 | return; |
498 | |
499 | cpuhp_state_remove_instance(state: cpuhp_slot, node: &idxd->idxd_pmu->cpuhp_node); |
500 | perf_pmu_unregister(pmu: &idxd->idxd_pmu->pmu); |
501 | kfree(objp: idxd->idxd_pmu); |
502 | idxd->idxd_pmu = NULL; |
503 | } |
504 | |
505 | static int perf_event_cpu_online(unsigned int cpu, struct hlist_node *node) |
506 | { |
507 | struct idxd_pmu *idxd_pmu; |
508 | |
509 | idxd_pmu = hlist_entry_safe(node, typeof(*idxd_pmu), cpuhp_node); |
510 | |
511 | /* select the first online CPU as the designated reader */ |
512 | if (cpumask_empty(srcp: &perfmon_dsa_cpu_mask)) { |
513 | cpumask_set_cpu(cpu, dstp: &perfmon_dsa_cpu_mask); |
514 | idxd_pmu->cpu = cpu; |
515 | } |
516 | |
517 | return 0; |
518 | } |
519 | |
520 | static int perf_event_cpu_offline(unsigned int cpu, struct hlist_node *node) |
521 | { |
522 | struct idxd_pmu *idxd_pmu; |
523 | unsigned int target; |
524 | |
525 | idxd_pmu = hlist_entry_safe(node, typeof(*idxd_pmu), cpuhp_node); |
526 | |
527 | if (!cpumask_test_and_clear_cpu(cpu, cpumask: &perfmon_dsa_cpu_mask)) |
528 | return 0; |
529 | |
530 | target = cpumask_any_but(cpu_online_mask, cpu); |
531 | |
532 | /* migrate events if there is a valid target */ |
533 | if (target < nr_cpu_ids) |
534 | cpumask_set_cpu(cpu: target, dstp: &perfmon_dsa_cpu_mask); |
535 | else |
536 | target = -1; |
537 | |
538 | perf_pmu_migrate_context(pmu: &idxd_pmu->pmu, src_cpu: cpu, dst_cpu: target); |
539 | |
540 | return 0; |
541 | } |
542 | |
543 | int perfmon_pmu_init(struct idxd_device *idxd) |
544 | { |
545 | union idxd_perfcap perfcap; |
546 | struct idxd_pmu *idxd_pmu; |
547 | int rc = -ENODEV; |
548 | |
549 | /* |
550 | * perfmon module initialization failed, nothing to do |
551 | */ |
552 | if (!cpuhp_set_up) |
553 | return -ENODEV; |
554 | |
555 | /* |
556 | * If perfmon_offset or num_counters is 0, it means perfmon is |
557 | * not supported on this hardware. |
558 | */ |
559 | if (idxd->perfmon_offset == 0) |
560 | return -ENODEV; |
561 | |
562 | idxd_pmu = kzalloc(size: sizeof(*idxd_pmu), GFP_KERNEL); |
563 | if (!idxd_pmu) |
564 | return -ENOMEM; |
565 | |
566 | idxd_pmu->idxd = idxd; |
567 | idxd->idxd_pmu = idxd_pmu; |
568 | |
569 | if (idxd->data->type == IDXD_TYPE_DSA) { |
570 | rc = sprintf(buf: idxd_pmu->name, fmt: "dsa%d" , idxd->id); |
571 | if (rc < 0) |
572 | goto free; |
573 | } else if (idxd->data->type == IDXD_TYPE_IAX) { |
574 | rc = sprintf(buf: idxd_pmu->name, fmt: "iax%d" , idxd->id); |
575 | if (rc < 0) |
576 | goto free; |
577 | } else { |
578 | goto free; |
579 | } |
580 | |
581 | perfmon_reset(idxd); |
582 | |
583 | perfcap.bits = ioread64(PERFCAP_REG(idxd)); |
584 | |
585 | /* |
586 | * If total perf counter is 0, stop further registration. |
587 | * This is necessary in order to support driver running on |
588 | * guest which does not have pmon support. |
589 | */ |
590 | if (perfcap.num_perf_counter == 0) |
591 | goto free; |
592 | |
593 | /* A counter width of 0 means it can't count */ |
594 | if (perfcap.counter_width == 0) |
595 | goto free; |
596 | |
597 | /* Overflow interrupt and counter freeze support must be available */ |
598 | if (!perfcap.overflow_interrupt || !perfcap.counter_freeze) |
599 | goto free; |
600 | |
601 | /* Number of event categories cannot be 0 */ |
602 | if (perfcap.num_event_category == 0) |
603 | goto free; |
604 | |
605 | /* |
606 | * We don't support per-counter capabilities for now. |
607 | */ |
608 | if (perfcap.cap_per_counter) |
609 | goto free; |
610 | |
611 | idxd_pmu->n_event_categories = perfcap.num_event_category; |
612 | idxd_pmu->supported_event_categories = perfcap.global_event_category; |
613 | idxd_pmu->per_counter_caps_supported = perfcap.cap_per_counter; |
614 | |
615 | /* check filter capability. If 0, then filters are not supported */ |
616 | idxd_pmu->supported_filters = perfcap.filter; |
617 | if (perfcap.filter) |
618 | idxd_pmu->n_filters = hweight8(perfcap.filter); |
619 | |
620 | /* Store the total number of counters categories, and counter width */ |
621 | idxd_pmu->n_counters = perfcap.num_perf_counter; |
622 | idxd_pmu->counter_width = perfcap.counter_width; |
623 | |
624 | idxd_pmu_init(idxd_pmu); |
625 | |
626 | rc = perf_pmu_register(pmu: &idxd_pmu->pmu, name: idxd_pmu->name, type: -1); |
627 | if (rc) |
628 | goto free; |
629 | |
630 | rc = cpuhp_state_add_instance(state: cpuhp_slot, node: &idxd_pmu->cpuhp_node); |
631 | if (rc) { |
632 | perf_pmu_unregister(pmu: &idxd->idxd_pmu->pmu); |
633 | goto free; |
634 | } |
635 | out: |
636 | return rc; |
637 | free: |
638 | kfree(objp: idxd_pmu); |
639 | idxd->idxd_pmu = NULL; |
640 | |
641 | goto out; |
642 | } |
643 | |
644 | void __init perfmon_init(void) |
645 | { |
646 | int rc = cpuhp_setup_state_multi(state: CPUHP_AP_ONLINE_DYN, |
647 | name: "driver/dma/idxd/perf:online" , |
648 | startup: perf_event_cpu_online, |
649 | teardown: perf_event_cpu_offline); |
650 | if (WARN_ON(rc < 0)) |
651 | return; |
652 | |
653 | cpuhp_slot = rc; |
654 | cpuhp_set_up = true; |
655 | } |
656 | |
657 | void __exit perfmon_exit(void) |
658 | { |
659 | if (cpuhp_set_up) |
660 | cpuhp_remove_multi_state(state: cpuhp_slot); |
661 | } |
662 | |