1 | // SPDX-License-Identifier: GPL-2.0 |
2 | // Copyright 2023 NXP |
3 | |
4 | #include <linux/bitfield.h> |
5 | #include <linux/init.h> |
6 | #include <linux/interrupt.h> |
7 | #include <linux/io.h> |
8 | #include <linux/module.h> |
9 | #include <linux/of.h> |
10 | #include <linux/platform_device.h> |
11 | #include <linux/perf_event.h> |
12 | |
13 | /* Performance monitor configuration */ |
14 | #define PMCFG1 0x00 |
15 | #define PMCFG1_RD_TRANS_FILT_EN BIT(31) |
16 | #define PMCFG1_WR_TRANS_FILT_EN BIT(30) |
17 | #define PMCFG1_RD_BT_FILT_EN BIT(29) |
18 | #define PMCFG1_ID_MASK GENMASK(17, 0) |
19 | |
20 | #define PMCFG2 0x04 |
21 | #define PMCFG2_ID GENMASK(17, 0) |
22 | |
23 | /* Global control register affects all counters and takes priority over local control registers */ |
24 | #define PMGC0 0x40 |
25 | /* Global control register bits */ |
26 | #define PMGC0_FAC BIT(31) |
27 | #define PMGC0_PMIE BIT(30) |
28 | #define PMGC0_FCECE BIT(29) |
29 | |
30 | /* |
31 | * 64bit counter0 exclusively dedicated to counting cycles |
32 | * 32bit counters monitor counter-specific events in addition to counting reference events |
33 | */ |
34 | #define PMLCA(n) (0x40 + 0x10 + (0x10 * n)) |
35 | #define PMLCB(n) (0x40 + 0x14 + (0x10 * n)) |
36 | #define PMC(n) (0x40 + 0x18 + (0x10 * n)) |
37 | /* Local control register bits */ |
38 | #define PMLCA_FC BIT(31) |
39 | #define PMLCA_CE BIT(26) |
40 | #define PMLCA_EVENT GENMASK(22, 16) |
41 | |
42 | #define NUM_COUNTERS 11 |
43 | #define CYCLES_COUNTER 0 |
44 | |
45 | #define to_ddr_pmu(p) container_of(p, struct ddr_pmu, pmu) |
46 | |
47 | #define DDR_PERF_DEV_NAME "imx9_ddr" |
48 | #define DDR_CPUHP_CB_NAME DDR_PERF_DEV_NAME "_perf_pmu" |
49 | |
50 | static DEFINE_IDA(ddr_ida); |
51 | |
52 | struct imx_ddr_devtype_data { |
53 | const char *identifier; /* system PMU identifier for userspace */ |
54 | }; |
55 | |
56 | struct ddr_pmu { |
57 | struct pmu pmu; |
58 | void __iomem *base; |
59 | unsigned int cpu; |
60 | struct hlist_node node; |
61 | struct device *dev; |
62 | struct perf_event *events[NUM_COUNTERS]; |
63 | int active_events; |
64 | enum cpuhp_state cpuhp_state; |
65 | const struct imx_ddr_devtype_data *devtype_data; |
66 | int irq; |
67 | int id; |
68 | }; |
69 | |
70 | static const struct imx_ddr_devtype_data imx93_devtype_data = { |
71 | .identifier = "imx93" , |
72 | }; |
73 | |
74 | static const struct of_device_id imx_ddr_pmu_dt_ids[] = { |
75 | {.compatible = "fsl,imx93-ddr-pmu" , .data = &imx93_devtype_data}, |
76 | { /* sentinel */ } |
77 | }; |
78 | MODULE_DEVICE_TABLE(of, imx_ddr_pmu_dt_ids); |
79 | |
80 | static ssize_t ddr_perf_identifier_show(struct device *dev, |
81 | struct device_attribute *attr, |
82 | char *page) |
83 | { |
84 | struct ddr_pmu *pmu = dev_get_drvdata(dev); |
85 | |
86 | return sysfs_emit(buf: page, fmt: "%s\n" , pmu->devtype_data->identifier); |
87 | } |
88 | |
89 | static struct device_attribute ddr_perf_identifier_attr = |
90 | __ATTR(identifier, 0444, ddr_perf_identifier_show, NULL); |
91 | |
92 | static struct attribute *ddr_perf_identifier_attrs[] = { |
93 | &ddr_perf_identifier_attr.attr, |
94 | NULL, |
95 | }; |
96 | |
97 | static struct attribute_group ddr_perf_identifier_attr_group = { |
98 | .attrs = ddr_perf_identifier_attrs, |
99 | }; |
100 | |
101 | static ssize_t ddr_perf_cpumask_show(struct device *dev, |
102 | struct device_attribute *attr, char *buf) |
103 | { |
104 | struct ddr_pmu *pmu = dev_get_drvdata(dev); |
105 | |
106 | return cpumap_print_to_pagebuf(list: true, buf, cpumask_of(pmu->cpu)); |
107 | } |
108 | |
109 | static struct device_attribute ddr_perf_cpumask_attr = |
110 | __ATTR(cpumask, 0444, ddr_perf_cpumask_show, NULL); |
111 | |
112 | static struct attribute *ddr_perf_cpumask_attrs[] = { |
113 | &ddr_perf_cpumask_attr.attr, |
114 | NULL, |
115 | }; |
116 | |
117 | static const struct attribute_group ddr_perf_cpumask_attr_group = { |
118 | .attrs = ddr_perf_cpumask_attrs, |
119 | }; |
120 | |
121 | static ssize_t ddr_pmu_event_show(struct device *dev, |
122 | struct device_attribute *attr, char *page) |
123 | { |
124 | struct perf_pmu_events_attr *pmu_attr; |
125 | |
126 | pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr); |
127 | return sysfs_emit(buf: page, fmt: "event=0x%02llx\n" , pmu_attr->id); |
128 | } |
129 | |
130 | #define IMX9_DDR_PMU_EVENT_ATTR(_name, _id) \ |
131 | (&((struct perf_pmu_events_attr[]) { \ |
132 | { .attr = __ATTR(_name, 0444, ddr_pmu_event_show, NULL),\ |
133 | .id = _id, } \ |
134 | })[0].attr.attr) |
135 | |
136 | static struct attribute *ddr_perf_events_attrs[] = { |
137 | /* counter0 cycles event */ |
138 | IMX9_DDR_PMU_EVENT_ATTR(cycles, 0), |
139 | |
140 | /* reference events for all normal counters, need assert DEBUG19[21] bit */ |
141 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_ddrc1_rmw_for_ecc, 12), |
142 | IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_rreorder, 13), |
143 | IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_wreorder, 14), |
144 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_0, 15), |
145 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_1, 16), |
146 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_2, 17), |
147 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_3, 18), |
148 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_4, 19), |
149 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_5, 22), |
150 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_6, 23), |
151 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_7, 24), |
152 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_8, 25), |
153 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_9, 26), |
154 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_10, 27), |
155 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_11, 28), |
156 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_12, 31), |
157 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_13, 59), |
158 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_15, 61), |
159 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_29, 63), |
160 | |
161 | /* counter1 specific events */ |
162 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_0, 64), |
163 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_1, 65), |
164 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_2, 66), |
165 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_3, 67), |
166 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_4, 68), |
167 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_5, 69), |
168 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_6, 70), |
169 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_7, 71), |
170 | |
171 | /* counter2 specific events */ |
172 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_0, 64), |
173 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_1, 65), |
174 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_2, 66), |
175 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_3, 67), |
176 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_4, 68), |
177 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_5, 69), |
178 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_6, 70), |
179 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_7, 71), |
180 | IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_empty, 72), |
181 | IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pm_rd_trans_filt, 73), |
182 | |
183 | /* counter3 specific events */ |
184 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_0, 64), |
185 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_1, 65), |
186 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_2, 66), |
187 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_3, 67), |
188 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_4, 68), |
189 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_5, 69), |
190 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_6, 70), |
191 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_7, 71), |
192 | IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_full, 72), |
193 | IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pm_wr_trans_filt, 73), |
194 | |
195 | /* counter4 specific events */ |
196 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_0, 64), |
197 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_1, 65), |
198 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_2, 66), |
199 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_3, 67), |
200 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_4, 68), |
201 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_5, 69), |
202 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_6, 70), |
203 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_7, 71), |
204 | IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_ld_rdq2_rmw, 72), |
205 | IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pm_rd_beat_filt, 73), |
206 | |
207 | /* counter5 specific events */ |
208 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_0, 64), |
209 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_1, 65), |
210 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_2, 66), |
211 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_3, 67), |
212 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_4, 68), |
213 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_5, 69), |
214 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_6, 70), |
215 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_7, 71), |
216 | IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_ld_rdq1, 72), |
217 | |
218 | /* counter6 specific events */ |
219 | IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_end_0, 64), |
220 | IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_ld_rdq2, 72), |
221 | |
222 | /* counter7 specific events */ |
223 | IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_1_2_full, 64), |
224 | IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_ld_wrq0, 65), |
225 | |
226 | /* counter8 specific events */ |
227 | IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_bias_switched, 64), |
228 | IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_1_4_full, 65), |
229 | |
230 | /* counter9 specific events */ |
231 | IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_ld_wrq1, 65), |
232 | IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_3_4_full, 66), |
233 | |
234 | /* counter10 specific events */ |
235 | IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_misc_mrk, 65), |
236 | IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_ld_rdq0, 66), |
237 | NULL, |
238 | }; |
239 | |
240 | static const struct attribute_group ddr_perf_events_attr_group = { |
241 | .name = "events" , |
242 | .attrs = ddr_perf_events_attrs, |
243 | }; |
244 | |
245 | PMU_FORMAT_ATTR(event, "config:0-7" ); |
246 | PMU_FORMAT_ATTR(counter, "config:8-15" ); |
247 | PMU_FORMAT_ATTR(axi_id, "config1:0-17" ); |
248 | PMU_FORMAT_ATTR(axi_mask, "config2:0-17" ); |
249 | |
250 | static struct attribute *ddr_perf_format_attrs[] = { |
251 | &format_attr_event.attr, |
252 | &format_attr_counter.attr, |
253 | &format_attr_axi_id.attr, |
254 | &format_attr_axi_mask.attr, |
255 | NULL, |
256 | }; |
257 | |
258 | static const struct attribute_group ddr_perf_format_attr_group = { |
259 | .name = "format" , |
260 | .attrs = ddr_perf_format_attrs, |
261 | }; |
262 | |
263 | static const struct attribute_group *attr_groups[] = { |
264 | &ddr_perf_identifier_attr_group, |
265 | &ddr_perf_cpumask_attr_group, |
266 | &ddr_perf_events_attr_group, |
267 | &ddr_perf_format_attr_group, |
268 | NULL, |
269 | }; |
270 | |
271 | static void ddr_perf_clear_counter(struct ddr_pmu *pmu, int counter) |
272 | { |
273 | if (counter == CYCLES_COUNTER) { |
274 | writel(val: 0, addr: pmu->base + PMC(counter) + 0x4); |
275 | writel(val: 0, addr: pmu->base + PMC(counter)); |
276 | } else { |
277 | writel(val: 0, addr: pmu->base + PMC(counter)); |
278 | } |
279 | } |
280 | |
281 | static u64 ddr_perf_read_counter(struct ddr_pmu *pmu, int counter) |
282 | { |
283 | u32 val_lower, val_upper; |
284 | u64 val; |
285 | |
286 | if (counter != CYCLES_COUNTER) { |
287 | val = readl_relaxed(pmu->base + PMC(counter)); |
288 | goto out; |
289 | } |
290 | |
291 | /* special handling for reading 64bit cycle counter */ |
292 | do { |
293 | val_upper = readl_relaxed(pmu->base + PMC(counter) + 0x4); |
294 | val_lower = readl_relaxed(pmu->base + PMC(counter)); |
295 | } while (val_upper != readl_relaxed(pmu->base + PMC(counter) + 0x4)); |
296 | |
297 | val = val_upper; |
298 | val = (val << 32); |
299 | val |= val_lower; |
300 | out: |
301 | return val; |
302 | } |
303 | |
304 | static void ddr_perf_counter_global_config(struct ddr_pmu *pmu, bool enable) |
305 | { |
306 | u32 ctrl; |
307 | |
308 | ctrl = readl_relaxed(pmu->base + PMGC0); |
309 | |
310 | if (enable) { |
311 | /* |
312 | * The performance monitor must be reset before event counting |
313 | * sequences. The performance monitor can be reset by first freezing |
314 | * one or more counters and then clearing the freeze condition to |
315 | * allow the counters to count according to the settings in the |
316 | * performance monitor registers. Counters can be frozen individually |
317 | * by setting PMLCAn[FC] bits, or simultaneously by setting PMGC0[FAC]. |
318 | * Simply clearing these freeze bits will then allow the performance |
319 | * monitor to begin counting based on the register settings. |
320 | */ |
321 | ctrl |= PMGC0_FAC; |
322 | writel(val: ctrl, addr: pmu->base + PMGC0); |
323 | |
324 | /* |
325 | * Freeze all counters disabled, interrupt enabled, and freeze |
326 | * counters on condition enabled. |
327 | */ |
328 | ctrl &= ~PMGC0_FAC; |
329 | ctrl |= PMGC0_PMIE | PMGC0_FCECE; |
330 | writel(val: ctrl, addr: pmu->base + PMGC0); |
331 | } else { |
332 | ctrl |= PMGC0_FAC; |
333 | ctrl &= ~(PMGC0_PMIE | PMGC0_FCECE); |
334 | writel(val: ctrl, addr: pmu->base + PMGC0); |
335 | } |
336 | } |
337 | |
338 | static void ddr_perf_counter_local_config(struct ddr_pmu *pmu, int config, |
339 | int counter, bool enable) |
340 | { |
341 | u32 ctrl_a; |
342 | |
343 | ctrl_a = readl_relaxed(pmu->base + PMLCA(counter)); |
344 | |
345 | if (enable) { |
346 | ctrl_a |= PMLCA_FC; |
347 | writel(val: ctrl_a, addr: pmu->base + PMLCA(counter)); |
348 | |
349 | ddr_perf_clear_counter(pmu, counter); |
350 | |
351 | /* Freeze counter disabled, condition enabled, and program event.*/ |
352 | ctrl_a &= ~PMLCA_FC; |
353 | ctrl_a |= PMLCA_CE; |
354 | ctrl_a &= ~FIELD_PREP(PMLCA_EVENT, 0x7F); |
355 | ctrl_a |= FIELD_PREP(PMLCA_EVENT, (config & 0x000000FF)); |
356 | writel(val: ctrl_a, addr: pmu->base + PMLCA(counter)); |
357 | } else { |
358 | /* Freeze counter. */ |
359 | ctrl_a |= PMLCA_FC; |
360 | writel(val: ctrl_a, addr: pmu->base + PMLCA(counter)); |
361 | } |
362 | } |
363 | |
364 | static void ddr_perf_monitor_config(struct ddr_pmu *pmu, int cfg, int cfg1, int cfg2) |
365 | { |
366 | u32 pmcfg1, pmcfg2; |
367 | int event, counter; |
368 | |
369 | event = cfg & 0x000000FF; |
370 | counter = (cfg & 0x0000FF00) >> 8; |
371 | |
372 | pmcfg1 = readl_relaxed(pmu->base + PMCFG1); |
373 | |
374 | if (counter == 2 && event == 73) |
375 | pmcfg1 |= PMCFG1_RD_TRANS_FILT_EN; |
376 | else if (counter == 2 && event != 73) |
377 | pmcfg1 &= ~PMCFG1_RD_TRANS_FILT_EN; |
378 | |
379 | if (counter == 3 && event == 73) |
380 | pmcfg1 |= PMCFG1_WR_TRANS_FILT_EN; |
381 | else if (counter == 3 && event != 73) |
382 | pmcfg1 &= ~PMCFG1_WR_TRANS_FILT_EN; |
383 | |
384 | if (counter == 4 && event == 73) |
385 | pmcfg1 |= PMCFG1_RD_BT_FILT_EN; |
386 | else if (counter == 4 && event != 73) |
387 | pmcfg1 &= ~PMCFG1_RD_BT_FILT_EN; |
388 | |
389 | pmcfg1 &= ~FIELD_PREP(PMCFG1_ID_MASK, 0x3FFFF); |
390 | pmcfg1 |= FIELD_PREP(PMCFG1_ID_MASK, cfg2); |
391 | writel(val: pmcfg1, addr: pmu->base + PMCFG1); |
392 | |
393 | pmcfg2 = readl_relaxed(pmu->base + PMCFG2); |
394 | pmcfg2 &= ~FIELD_PREP(PMCFG2_ID, 0x3FFFF); |
395 | pmcfg2 |= FIELD_PREP(PMCFG2_ID, cfg1); |
396 | writel(val: pmcfg2, addr: pmu->base + PMCFG2); |
397 | } |
398 | |
399 | static void ddr_perf_event_update(struct perf_event *event) |
400 | { |
401 | struct ddr_pmu *pmu = to_ddr_pmu(event->pmu); |
402 | struct hw_perf_event *hwc = &event->hw; |
403 | int counter = hwc->idx; |
404 | u64 new_raw_count; |
405 | |
406 | new_raw_count = ddr_perf_read_counter(pmu, counter); |
407 | local64_add(new_raw_count, &event->count); |
408 | |
409 | /* clear counter's value every time */ |
410 | ddr_perf_clear_counter(pmu, counter); |
411 | } |
412 | |
413 | static int ddr_perf_event_init(struct perf_event *event) |
414 | { |
415 | struct ddr_pmu *pmu = to_ddr_pmu(event->pmu); |
416 | struct hw_perf_event *hwc = &event->hw; |
417 | struct perf_event *sibling; |
418 | |
419 | if (event->attr.type != event->pmu->type) |
420 | return -ENOENT; |
421 | |
422 | if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK) |
423 | return -EOPNOTSUPP; |
424 | |
425 | if (event->cpu < 0) { |
426 | dev_warn(pmu->dev, "Can't provide per-task data!\n" ); |
427 | return -EOPNOTSUPP; |
428 | } |
429 | |
430 | /* |
431 | * We must NOT create groups containing mixed PMUs, although software |
432 | * events are acceptable (for example to create a CCN group |
433 | * periodically read when a hrtimer aka cpu-clock leader triggers). |
434 | */ |
435 | if (event->group_leader->pmu != event->pmu && |
436 | !is_software_event(event: event->group_leader)) |
437 | return -EINVAL; |
438 | |
439 | for_each_sibling_event(sibling, event->group_leader) { |
440 | if (sibling->pmu != event->pmu && |
441 | !is_software_event(event: sibling)) |
442 | return -EINVAL; |
443 | } |
444 | |
445 | event->cpu = pmu->cpu; |
446 | hwc->idx = -1; |
447 | |
448 | return 0; |
449 | } |
450 | |
451 | static void ddr_perf_event_start(struct perf_event *event, int flags) |
452 | { |
453 | struct ddr_pmu *pmu = to_ddr_pmu(event->pmu); |
454 | struct hw_perf_event *hwc = &event->hw; |
455 | int counter = hwc->idx; |
456 | |
457 | local64_set(&hwc->prev_count, 0); |
458 | |
459 | ddr_perf_counter_local_config(pmu, config: event->attr.config, counter, enable: true); |
460 | hwc->state = 0; |
461 | } |
462 | |
463 | static int ddr_perf_event_add(struct perf_event *event, int flags) |
464 | { |
465 | struct ddr_pmu *pmu = to_ddr_pmu(event->pmu); |
466 | struct hw_perf_event *hwc = &event->hw; |
467 | int cfg = event->attr.config; |
468 | int cfg1 = event->attr.config1; |
469 | int cfg2 = event->attr.config2; |
470 | int counter; |
471 | |
472 | counter = (cfg & 0x0000FF00) >> 8; |
473 | |
474 | pmu->events[counter] = event; |
475 | pmu->active_events++; |
476 | hwc->idx = counter; |
477 | hwc->state |= PERF_HES_STOPPED; |
478 | |
479 | if (flags & PERF_EF_START) |
480 | ddr_perf_event_start(event, flags); |
481 | |
482 | /* read trans, write trans, read beat */ |
483 | ddr_perf_monitor_config(pmu, cfg, cfg1, cfg2); |
484 | |
485 | return 0; |
486 | } |
487 | |
488 | static void ddr_perf_event_stop(struct perf_event *event, int flags) |
489 | { |
490 | struct ddr_pmu *pmu = to_ddr_pmu(event->pmu); |
491 | struct hw_perf_event *hwc = &event->hw; |
492 | int counter = hwc->idx; |
493 | |
494 | ddr_perf_counter_local_config(pmu, config: event->attr.config, counter, enable: false); |
495 | ddr_perf_event_update(event); |
496 | |
497 | hwc->state |= PERF_HES_STOPPED; |
498 | } |
499 | |
500 | static void ddr_perf_event_del(struct perf_event *event, int flags) |
501 | { |
502 | struct ddr_pmu *pmu = to_ddr_pmu(event->pmu); |
503 | struct hw_perf_event *hwc = &event->hw; |
504 | |
505 | ddr_perf_event_stop(event, PERF_EF_UPDATE); |
506 | |
507 | pmu->active_events--; |
508 | hwc->idx = -1; |
509 | } |
510 | |
511 | static void ddr_perf_pmu_enable(struct pmu *pmu) |
512 | { |
513 | struct ddr_pmu *ddr_pmu = to_ddr_pmu(pmu); |
514 | |
515 | ddr_perf_counter_global_config(pmu: ddr_pmu, enable: true); |
516 | } |
517 | |
518 | static void ddr_perf_pmu_disable(struct pmu *pmu) |
519 | { |
520 | struct ddr_pmu *ddr_pmu = to_ddr_pmu(pmu); |
521 | |
522 | ddr_perf_counter_global_config(pmu: ddr_pmu, enable: false); |
523 | } |
524 | |
525 | static void ddr_perf_init(struct ddr_pmu *pmu, void __iomem *base, |
526 | struct device *dev) |
527 | { |
528 | *pmu = (struct ddr_pmu) { |
529 | .pmu = (struct pmu) { |
530 | .module = THIS_MODULE, |
531 | .capabilities = PERF_PMU_CAP_NO_EXCLUDE, |
532 | .task_ctx_nr = perf_invalid_context, |
533 | .attr_groups = attr_groups, |
534 | .event_init = ddr_perf_event_init, |
535 | .add = ddr_perf_event_add, |
536 | .del = ddr_perf_event_del, |
537 | .start = ddr_perf_event_start, |
538 | .stop = ddr_perf_event_stop, |
539 | .read = ddr_perf_event_update, |
540 | .pmu_enable = ddr_perf_pmu_enable, |
541 | .pmu_disable = ddr_perf_pmu_disable, |
542 | }, |
543 | .base = base, |
544 | .dev = dev, |
545 | }; |
546 | } |
547 | |
548 | static irqreturn_t ddr_perf_irq_handler(int irq, void *p) |
549 | { |
550 | struct ddr_pmu *pmu = (struct ddr_pmu *)p; |
551 | struct perf_event *event; |
552 | int i; |
553 | |
554 | /* |
555 | * Counters can generate an interrupt on an overflow when msb of a |
556 | * counter changes from 0 to 1. For the interrupt to be signalled, |
557 | * below condition mush be satisfied: |
558 | * PMGC0[PMIE] = 1, PMGC0[FCECE] = 1, PMLCAn[CE] = 1 |
559 | * When an interrupt is signalled, PMGC0[FAC] is set by hardware and |
560 | * all of the registers are frozen. |
561 | * Software can clear the interrupt condition by resetting the performance |
562 | * monitor and clearing the most significant bit of the counter that |
563 | * generate the overflow. |
564 | */ |
565 | for (i = 0; i < NUM_COUNTERS; i++) { |
566 | if (!pmu->events[i]) |
567 | continue; |
568 | |
569 | event = pmu->events[i]; |
570 | |
571 | ddr_perf_event_update(event); |
572 | } |
573 | |
574 | ddr_perf_counter_global_config(pmu, enable: true); |
575 | |
576 | return IRQ_HANDLED; |
577 | } |
578 | |
579 | static int ddr_perf_offline_cpu(unsigned int cpu, struct hlist_node *node) |
580 | { |
581 | struct ddr_pmu *pmu = hlist_entry_safe(node, struct ddr_pmu, node); |
582 | int target; |
583 | |
584 | if (cpu != pmu->cpu) |
585 | return 0; |
586 | |
587 | target = cpumask_any_but(cpu_online_mask, cpu); |
588 | if (target >= nr_cpu_ids) |
589 | return 0; |
590 | |
591 | perf_pmu_migrate_context(pmu: &pmu->pmu, src_cpu: cpu, dst_cpu: target); |
592 | pmu->cpu = target; |
593 | |
594 | WARN_ON(irq_set_affinity(pmu->irq, cpumask_of(pmu->cpu))); |
595 | |
596 | return 0; |
597 | } |
598 | |
599 | static int ddr_perf_probe(struct platform_device *pdev) |
600 | { |
601 | struct ddr_pmu *pmu; |
602 | void __iomem *base; |
603 | int ret, irq; |
604 | char *name; |
605 | |
606 | base = devm_platform_ioremap_resource(pdev, index: 0); |
607 | if (IS_ERR(ptr: base)) |
608 | return PTR_ERR(ptr: base); |
609 | |
610 | pmu = devm_kzalloc(dev: &pdev->dev, size: sizeof(*pmu), GFP_KERNEL); |
611 | if (!pmu) |
612 | return -ENOMEM; |
613 | |
614 | ddr_perf_init(pmu, base, dev: &pdev->dev); |
615 | |
616 | pmu->devtype_data = of_device_get_match_data(dev: &pdev->dev); |
617 | |
618 | platform_set_drvdata(pdev, data: pmu); |
619 | |
620 | pmu->id = ida_alloc(ida: &ddr_ida, GFP_KERNEL); |
621 | name = devm_kasprintf(dev: &pdev->dev, GFP_KERNEL, DDR_PERF_DEV_NAME "%d" , pmu->id); |
622 | if (!name) { |
623 | ret = -ENOMEM; |
624 | goto format_string_err; |
625 | } |
626 | |
627 | pmu->cpu = raw_smp_processor_id(); |
628 | ret = cpuhp_setup_state_multi(state: CPUHP_AP_ONLINE_DYN, DDR_CPUHP_CB_NAME, |
629 | NULL, teardown: ddr_perf_offline_cpu); |
630 | if (ret < 0) { |
631 | dev_err(&pdev->dev, "Failed to add callbacks for multi state\n" ); |
632 | goto cpuhp_state_err; |
633 | } |
634 | pmu->cpuhp_state = ret; |
635 | |
636 | /* Register the pmu instance for cpu hotplug */ |
637 | ret = cpuhp_state_add_instance_nocalls(state: pmu->cpuhp_state, node: &pmu->node); |
638 | if (ret) { |
639 | dev_err(&pdev->dev, "Error %d registering hotplug\n" , ret); |
640 | goto cpuhp_instance_err; |
641 | } |
642 | |
643 | /* Request irq */ |
644 | irq = platform_get_irq(pdev, 0); |
645 | if (irq < 0) { |
646 | ret = irq; |
647 | goto ddr_perf_err; |
648 | } |
649 | |
650 | ret = devm_request_irq(dev: &pdev->dev, irq, handler: ddr_perf_irq_handler, |
651 | IRQF_NOBALANCING | IRQF_NO_THREAD, |
652 | DDR_CPUHP_CB_NAME, dev_id: pmu); |
653 | if (ret < 0) { |
654 | dev_err(&pdev->dev, "Request irq failed: %d" , ret); |
655 | goto ddr_perf_err; |
656 | } |
657 | |
658 | pmu->irq = irq; |
659 | ret = irq_set_affinity(irq: pmu->irq, cpumask_of(pmu->cpu)); |
660 | if (ret) { |
661 | dev_err(pmu->dev, "Failed to set interrupt affinity\n" ); |
662 | goto ddr_perf_err; |
663 | } |
664 | |
665 | ret = perf_pmu_register(pmu: &pmu->pmu, name, type: -1); |
666 | if (ret) |
667 | goto ddr_perf_err; |
668 | |
669 | return 0; |
670 | |
671 | ddr_perf_err: |
672 | cpuhp_state_remove_instance_nocalls(state: pmu->cpuhp_state, node: &pmu->node); |
673 | cpuhp_instance_err: |
674 | cpuhp_remove_multi_state(state: pmu->cpuhp_state); |
675 | cpuhp_state_err: |
676 | format_string_err: |
677 | ida_free(&ddr_ida, id: pmu->id); |
678 | dev_warn(&pdev->dev, "i.MX9 DDR Perf PMU failed (%d), disabled\n" , ret); |
679 | return ret; |
680 | } |
681 | |
682 | static void ddr_perf_remove(struct platform_device *pdev) |
683 | { |
684 | struct ddr_pmu *pmu = platform_get_drvdata(pdev); |
685 | |
686 | cpuhp_state_remove_instance_nocalls(state: pmu->cpuhp_state, node: &pmu->node); |
687 | cpuhp_remove_multi_state(state: pmu->cpuhp_state); |
688 | |
689 | perf_pmu_unregister(pmu: &pmu->pmu); |
690 | |
691 | ida_free(&ddr_ida, id: pmu->id); |
692 | } |
693 | |
694 | static struct platform_driver imx_ddr_pmu_driver = { |
695 | .driver = { |
696 | .name = "imx9-ddr-pmu" , |
697 | .of_match_table = imx_ddr_pmu_dt_ids, |
698 | .suppress_bind_attrs = true, |
699 | }, |
700 | .probe = ddr_perf_probe, |
701 | .remove_new = ddr_perf_remove, |
702 | }; |
703 | module_platform_driver(imx_ddr_pmu_driver); |
704 | |
705 | MODULE_AUTHOR("Xu Yang <xu.yang_2@nxp.com>" ); |
706 | MODULE_LICENSE("GPL v2" ); |
707 | MODULE_DESCRIPTION("DDRC PerfMon for i.MX9 SoCs" ); |
708 | |