1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (c) 2016, Fuzhou Rockchip Electronics Co., Ltd |
4 | * Author: Lin Huang <hl@rock-chips.com> |
5 | */ |
6 | |
7 | #include <linux/clk.h> |
8 | #include <linux/devfreq-event.h> |
9 | #include <linux/kernel.h> |
10 | #include <linux/err.h> |
11 | #include <linux/init.h> |
12 | #include <linux/io.h> |
13 | #include <linux/mfd/syscon.h> |
14 | #include <linux/module.h> |
15 | #include <linux/platform_device.h> |
16 | #include <linux/regmap.h> |
17 | #include <linux/slab.h> |
18 | #include <linux/list.h> |
19 | #include <linux/seqlock.h> |
20 | #include <linux/of.h> |
21 | #include <linux/of_device.h> |
22 | #include <linux/bitfield.h> |
23 | #include <linux/bits.h> |
24 | #include <linux/perf_event.h> |
25 | |
26 | #include <soc/rockchip/rockchip_grf.h> |
27 | #include <soc/rockchip/rk3399_grf.h> |
28 | #include <soc/rockchip/rk3568_grf.h> |
29 | #include <soc/rockchip/rk3588_grf.h> |
30 | |
31 | #define DMC_MAX_CHANNELS 4 |
32 | |
33 | #define HIWORD_UPDATE(val, mask) ((val) | (mask) << 16) |
34 | |
35 | /* DDRMON_CTRL */ |
36 | #define DDRMON_CTRL 0x04 |
37 | #define DDRMON_CTRL_DDR4 BIT(5) |
38 | #define DDRMON_CTRL_LPDDR4 BIT(4) |
39 | #define DDRMON_CTRL_HARDWARE_EN BIT(3) |
40 | #define DDRMON_CTRL_LPDDR23 BIT(2) |
41 | #define DDRMON_CTRL_SOFTWARE_EN BIT(1) |
42 | #define DDRMON_CTRL_TIMER_CNT_EN BIT(0) |
43 | #define DDRMON_CTRL_DDR_TYPE_MASK (DDRMON_CTRL_DDR4 | \ |
44 | DDRMON_CTRL_LPDDR4 | \ |
45 | DDRMON_CTRL_LPDDR23) |
46 | |
47 | #define DDRMON_CH0_WR_NUM 0x20 |
48 | #define DDRMON_CH0_RD_NUM 0x24 |
49 | #define DDRMON_CH0_COUNT_NUM 0x28 |
50 | #define DDRMON_CH0_DFI_ACCESS_NUM 0x2c |
51 | #define DDRMON_CH1_COUNT_NUM 0x3c |
52 | #define DDRMON_CH1_DFI_ACCESS_NUM 0x40 |
53 | |
54 | #define PERF_EVENT_CYCLES 0x0 |
55 | #define PERF_EVENT_READ_BYTES 0x1 |
56 | #define PERF_EVENT_WRITE_BYTES 0x2 |
57 | #define PERF_EVENT_READ_BYTES0 0x3 |
58 | #define PERF_EVENT_WRITE_BYTES0 0x4 |
59 | #define PERF_EVENT_READ_BYTES1 0x5 |
60 | #define PERF_EVENT_WRITE_BYTES1 0x6 |
61 | #define PERF_EVENT_READ_BYTES2 0x7 |
62 | #define PERF_EVENT_WRITE_BYTES2 0x8 |
63 | #define PERF_EVENT_READ_BYTES3 0x9 |
64 | #define PERF_EVENT_WRITE_BYTES3 0xa |
65 | #define PERF_EVENT_BYTES 0xb |
66 | #define PERF_ACCESS_TYPE_MAX 0xc |
67 | |
68 | /** |
69 | * struct dmc_count_channel - structure to hold counter values from the DDR controller |
70 | * @access: Number of read and write accesses |
71 | * @clock_cycles: DDR clock cycles |
72 | * @read_access: number of read accesses |
73 | * @write_access: number of write accesses |
74 | */ |
75 | struct dmc_count_channel { |
76 | u64 access; |
77 | u64 clock_cycles; |
78 | u64 read_access; |
79 | u64 write_access; |
80 | }; |
81 | |
82 | struct dmc_count { |
83 | struct dmc_count_channel c[DMC_MAX_CHANNELS]; |
84 | }; |
85 | |
86 | /* |
87 | * The dfi controller can monitor DDR load. It has an upper and lower threshold |
88 | * for the operating points. Whenever the usage leaves these bounds an event is |
89 | * generated to indicate the DDR frequency should be changed. |
90 | */ |
91 | struct rockchip_dfi { |
92 | struct devfreq_event_dev *edev; |
93 | struct devfreq_event_desc desc; |
94 | struct dmc_count last_event_count; |
95 | |
96 | struct dmc_count last_perf_count; |
97 | struct dmc_count total_count; |
98 | seqlock_t count_seqlock; /* protects last_perf_count and total_count */ |
99 | |
100 | struct device *dev; |
101 | void __iomem *regs; |
102 | struct regmap *regmap_pmu; |
103 | struct clk *clk; |
104 | int usecount; |
105 | struct mutex mutex; |
106 | u32 ddr_type; |
107 | unsigned int channel_mask; |
108 | unsigned int max_channels; |
109 | enum cpuhp_state cpuhp_state; |
110 | struct hlist_node node; |
111 | struct pmu pmu; |
112 | struct hrtimer timer; |
113 | unsigned int cpu; |
114 | int active_events; |
115 | int burst_len; |
116 | int buswidth[DMC_MAX_CHANNELS]; |
117 | int ddrmon_stride; |
118 | bool ddrmon_ctrl_single; |
119 | }; |
120 | |
121 | static int rockchip_dfi_enable(struct rockchip_dfi *dfi) |
122 | { |
123 | void __iomem *dfi_regs = dfi->regs; |
124 | int i, ret = 0; |
125 | |
126 | mutex_lock(&dfi->mutex); |
127 | |
128 | dfi->usecount++; |
129 | if (dfi->usecount > 1) |
130 | goto out; |
131 | |
132 | ret = clk_prepare_enable(clk: dfi->clk); |
133 | if (ret) { |
134 | dev_err(&dfi->edev->dev, "failed to enable dfi clk: %d\n" , ret); |
135 | goto out; |
136 | } |
137 | |
138 | for (i = 0; i < dfi->max_channels; i++) { |
139 | u32 ctrl = 0; |
140 | |
141 | if (!(dfi->channel_mask & BIT(i))) |
142 | continue; |
143 | |
144 | /* clear DDRMON_CTRL setting */ |
145 | writel_relaxed(HIWORD_UPDATE(0, DDRMON_CTRL_TIMER_CNT_EN | |
146 | DDRMON_CTRL_SOFTWARE_EN | DDRMON_CTRL_HARDWARE_EN), |
147 | dfi_regs + i * dfi->ddrmon_stride + DDRMON_CTRL); |
148 | |
149 | /* set ddr type to dfi */ |
150 | switch (dfi->ddr_type) { |
151 | case ROCKCHIP_DDRTYPE_LPDDR2: |
152 | case ROCKCHIP_DDRTYPE_LPDDR3: |
153 | ctrl = DDRMON_CTRL_LPDDR23; |
154 | break; |
155 | case ROCKCHIP_DDRTYPE_LPDDR4: |
156 | case ROCKCHIP_DDRTYPE_LPDDR4X: |
157 | ctrl = DDRMON_CTRL_LPDDR4; |
158 | break; |
159 | default: |
160 | break; |
161 | } |
162 | |
163 | writel_relaxed(HIWORD_UPDATE(ctrl, DDRMON_CTRL_DDR_TYPE_MASK), |
164 | dfi_regs + i * dfi->ddrmon_stride + DDRMON_CTRL); |
165 | |
166 | /* enable count, use software mode */ |
167 | writel_relaxed(HIWORD_UPDATE(DDRMON_CTRL_SOFTWARE_EN, DDRMON_CTRL_SOFTWARE_EN), |
168 | dfi_regs + i * dfi->ddrmon_stride + DDRMON_CTRL); |
169 | |
170 | if (dfi->ddrmon_ctrl_single) |
171 | break; |
172 | } |
173 | out: |
174 | mutex_unlock(lock: &dfi->mutex); |
175 | |
176 | return ret; |
177 | } |
178 | |
179 | static void rockchip_dfi_disable(struct rockchip_dfi *dfi) |
180 | { |
181 | void __iomem *dfi_regs = dfi->regs; |
182 | int i; |
183 | |
184 | mutex_lock(&dfi->mutex); |
185 | |
186 | dfi->usecount--; |
187 | |
188 | WARN_ON_ONCE(dfi->usecount < 0); |
189 | |
190 | if (dfi->usecount > 0) |
191 | goto out; |
192 | |
193 | for (i = 0; i < dfi->max_channels; i++) { |
194 | if (!(dfi->channel_mask & BIT(i))) |
195 | continue; |
196 | |
197 | writel_relaxed(HIWORD_UPDATE(0, DDRMON_CTRL_SOFTWARE_EN), |
198 | dfi_regs + i * dfi->ddrmon_stride + DDRMON_CTRL); |
199 | |
200 | if (dfi->ddrmon_ctrl_single) |
201 | break; |
202 | } |
203 | |
204 | clk_disable_unprepare(clk: dfi->clk); |
205 | out: |
206 | mutex_unlock(lock: &dfi->mutex); |
207 | } |
208 | |
209 | static void rockchip_dfi_read_counters(struct rockchip_dfi *dfi, struct dmc_count *res) |
210 | { |
211 | u32 i; |
212 | void __iomem *dfi_regs = dfi->regs; |
213 | |
214 | for (i = 0; i < dfi->max_channels; i++) { |
215 | if (!(dfi->channel_mask & BIT(i))) |
216 | continue; |
217 | res->c[i].read_access = readl_relaxed(dfi_regs + |
218 | DDRMON_CH0_RD_NUM + i * dfi->ddrmon_stride); |
219 | res->c[i].write_access = readl_relaxed(dfi_regs + |
220 | DDRMON_CH0_WR_NUM + i * dfi->ddrmon_stride); |
221 | res->c[i].access = readl_relaxed(dfi_regs + |
222 | DDRMON_CH0_DFI_ACCESS_NUM + i * dfi->ddrmon_stride); |
223 | res->c[i].clock_cycles = readl_relaxed(dfi_regs + |
224 | DDRMON_CH0_COUNT_NUM + i * dfi->ddrmon_stride); |
225 | } |
226 | } |
227 | |
228 | static int rockchip_dfi_event_disable(struct devfreq_event_dev *edev) |
229 | { |
230 | struct rockchip_dfi *dfi = devfreq_event_get_drvdata(edev); |
231 | |
232 | rockchip_dfi_disable(dfi); |
233 | |
234 | return 0; |
235 | } |
236 | |
237 | static int rockchip_dfi_event_enable(struct devfreq_event_dev *edev) |
238 | { |
239 | struct rockchip_dfi *dfi = devfreq_event_get_drvdata(edev); |
240 | |
241 | return rockchip_dfi_enable(dfi); |
242 | } |
243 | |
244 | static int rockchip_dfi_set_event(struct devfreq_event_dev *edev) |
245 | { |
246 | return 0; |
247 | } |
248 | |
249 | static int rockchip_dfi_get_event(struct devfreq_event_dev *edev, |
250 | struct devfreq_event_data *edata) |
251 | { |
252 | struct rockchip_dfi *dfi = devfreq_event_get_drvdata(edev); |
253 | struct dmc_count count; |
254 | struct dmc_count *last = &dfi->last_event_count; |
255 | u32 access = 0, clock_cycles = 0; |
256 | int i; |
257 | |
258 | rockchip_dfi_read_counters(dfi, res: &count); |
259 | |
260 | /* We can only report one channel, so find the busiest one */ |
261 | for (i = 0; i < dfi->max_channels; i++) { |
262 | u32 a, c; |
263 | |
264 | if (!(dfi->channel_mask & BIT(i))) |
265 | continue; |
266 | |
267 | a = count.c[i].access - last->c[i].access; |
268 | c = count.c[i].clock_cycles - last->c[i].clock_cycles; |
269 | |
270 | if (a > access) { |
271 | access = a; |
272 | clock_cycles = c; |
273 | } |
274 | } |
275 | |
276 | edata->load_count = access * 4; |
277 | edata->total_count = clock_cycles; |
278 | |
279 | dfi->last_event_count = count; |
280 | |
281 | return 0; |
282 | } |
283 | |
284 | static const struct devfreq_event_ops rockchip_dfi_ops = { |
285 | .disable = rockchip_dfi_event_disable, |
286 | .enable = rockchip_dfi_event_enable, |
287 | .get_event = rockchip_dfi_get_event, |
288 | .set_event = rockchip_dfi_set_event, |
289 | }; |
290 | |
291 | #ifdef CONFIG_PERF_EVENTS |
292 | |
293 | static void rockchip_ddr_perf_counters_add(struct rockchip_dfi *dfi, |
294 | const struct dmc_count *now, |
295 | struct dmc_count *res) |
296 | { |
297 | const struct dmc_count *last = &dfi->last_perf_count; |
298 | int i; |
299 | |
300 | for (i = 0; i < dfi->max_channels; i++) { |
301 | res->c[i].read_access = dfi->total_count.c[i].read_access + |
302 | (u32)(now->c[i].read_access - last->c[i].read_access); |
303 | res->c[i].write_access = dfi->total_count.c[i].write_access + |
304 | (u32)(now->c[i].write_access - last->c[i].write_access); |
305 | res->c[i].access = dfi->total_count.c[i].access + |
306 | (u32)(now->c[i].access - last->c[i].access); |
307 | res->c[i].clock_cycles = dfi->total_count.c[i].clock_cycles + |
308 | (u32)(now->c[i].clock_cycles - last->c[i].clock_cycles); |
309 | } |
310 | } |
311 | |
312 | static ssize_t ddr_perf_cpumask_show(struct device *dev, |
313 | struct device_attribute *attr, char *buf) |
314 | { |
315 | struct pmu *pmu = dev_get_drvdata(dev); |
316 | struct rockchip_dfi *dfi = container_of(pmu, struct rockchip_dfi, pmu); |
317 | |
318 | return cpumap_print_to_pagebuf(list: true, buf, cpumask_of(dfi->cpu)); |
319 | } |
320 | |
321 | static struct device_attribute ddr_perf_cpumask_attr = |
322 | __ATTR(cpumask, 0444, ddr_perf_cpumask_show, NULL); |
323 | |
324 | static struct attribute *ddr_perf_cpumask_attrs[] = { |
325 | &ddr_perf_cpumask_attr.attr, |
326 | NULL, |
327 | }; |
328 | |
329 | static const struct attribute_group ddr_perf_cpumask_attr_group = { |
330 | .attrs = ddr_perf_cpumask_attrs, |
331 | }; |
332 | |
333 | PMU_EVENT_ATTR_STRING(cycles, ddr_pmu_cycles, "event=" __stringify(PERF_EVENT_CYCLES)) |
334 | |
335 | #define DFI_PMU_EVENT_ATTR(_name, _var, _str) \ |
336 | PMU_EVENT_ATTR_STRING(_name, _var, _str); \ |
337 | PMU_EVENT_ATTR_STRING(_name.unit, _var##_unit, "MB"); \ |
338 | PMU_EVENT_ATTR_STRING(_name.scale, _var##_scale, "9.536743164e-07") |
339 | |
340 | DFI_PMU_EVENT_ATTR(read-bytes0, ddr_pmu_read_bytes0, "event=" __stringify(PERF_EVENT_READ_BYTES0)); |
341 | DFI_PMU_EVENT_ATTR(write-bytes0, ddr_pmu_write_bytes0, "event=" __stringify(PERF_EVENT_WRITE_BYTES0)); |
342 | |
343 | DFI_PMU_EVENT_ATTR(read-bytes1, ddr_pmu_read_bytes1, "event=" __stringify(PERF_EVENT_READ_BYTES1)); |
344 | DFI_PMU_EVENT_ATTR(write-bytes1, ddr_pmu_write_bytes1, "event=" __stringify(PERF_EVENT_WRITE_BYTES1)); |
345 | |
346 | DFI_PMU_EVENT_ATTR(read-bytes2, ddr_pmu_read_bytes2, "event=" __stringify(PERF_EVENT_READ_BYTES2)); |
347 | DFI_PMU_EVENT_ATTR(write-bytes2, ddr_pmu_write_bytes2, "event=" __stringify(PERF_EVENT_WRITE_BYTES2)); |
348 | |
349 | DFI_PMU_EVENT_ATTR(read-bytes3, ddr_pmu_read_bytes3, "event=" __stringify(PERF_EVENT_READ_BYTES3)); |
350 | DFI_PMU_EVENT_ATTR(write-bytes3, ddr_pmu_write_bytes3, "event=" __stringify(PERF_EVENT_WRITE_BYTES3)); |
351 | |
352 | DFI_PMU_EVENT_ATTR(read-bytes, ddr_pmu_read_bytes, "event=" __stringify(PERF_EVENT_READ_BYTES)); |
353 | DFI_PMU_EVENT_ATTR(write-bytes, ddr_pmu_write_bytes, "event=" __stringify(PERF_EVENT_WRITE_BYTES)); |
354 | |
355 | DFI_PMU_EVENT_ATTR(bytes, ddr_pmu_bytes, "event=" __stringify(PERF_EVENT_BYTES)); |
356 | |
357 | #define DFI_ATTR_MB(_name) \ |
358 | &_name.attr.attr, \ |
359 | &_name##_unit.attr.attr, \ |
360 | &_name##_scale.attr.attr |
361 | |
362 | static struct attribute *ddr_perf_events_attrs[] = { |
363 | &ddr_pmu_cycles.attr.attr, |
364 | DFI_ATTR_MB(ddr_pmu_read_bytes), |
365 | DFI_ATTR_MB(ddr_pmu_write_bytes), |
366 | DFI_ATTR_MB(ddr_pmu_read_bytes0), |
367 | DFI_ATTR_MB(ddr_pmu_write_bytes0), |
368 | DFI_ATTR_MB(ddr_pmu_read_bytes1), |
369 | DFI_ATTR_MB(ddr_pmu_write_bytes1), |
370 | DFI_ATTR_MB(ddr_pmu_read_bytes2), |
371 | DFI_ATTR_MB(ddr_pmu_write_bytes2), |
372 | DFI_ATTR_MB(ddr_pmu_read_bytes3), |
373 | DFI_ATTR_MB(ddr_pmu_write_bytes3), |
374 | DFI_ATTR_MB(ddr_pmu_bytes), |
375 | NULL, |
376 | }; |
377 | |
378 | static const struct attribute_group ddr_perf_events_attr_group = { |
379 | .name = "events" , |
380 | .attrs = ddr_perf_events_attrs, |
381 | }; |
382 | |
383 | PMU_FORMAT_ATTR(event, "config:0-7" ); |
384 | |
385 | static struct attribute *ddr_perf_format_attrs[] = { |
386 | &format_attr_event.attr, |
387 | NULL, |
388 | }; |
389 | |
390 | static const struct attribute_group ddr_perf_format_attr_group = { |
391 | .name = "format" , |
392 | .attrs = ddr_perf_format_attrs, |
393 | }; |
394 | |
395 | static const struct attribute_group *attr_groups[] = { |
396 | &ddr_perf_events_attr_group, |
397 | &ddr_perf_cpumask_attr_group, |
398 | &ddr_perf_format_attr_group, |
399 | NULL, |
400 | }; |
401 | |
402 | static int rockchip_ddr_perf_event_init(struct perf_event *event) |
403 | { |
404 | struct rockchip_dfi *dfi = container_of(event->pmu, struct rockchip_dfi, pmu); |
405 | |
406 | if (event->attr.type != event->pmu->type) |
407 | return -ENOENT; |
408 | |
409 | if (event->attach_state & PERF_ATTACH_TASK) |
410 | return -EINVAL; |
411 | |
412 | if (event->cpu < 0) { |
413 | dev_warn(dfi->dev, "Can't provide per-task data!\n" ); |
414 | return -EINVAL; |
415 | } |
416 | |
417 | return 0; |
418 | } |
419 | |
420 | static u64 rockchip_ddr_perf_event_get_count(struct perf_event *event) |
421 | { |
422 | struct rockchip_dfi *dfi = container_of(event->pmu, struct rockchip_dfi, pmu); |
423 | int blen = dfi->burst_len; |
424 | struct dmc_count total, now; |
425 | unsigned int seq; |
426 | u64 count = 0; |
427 | int i; |
428 | |
429 | rockchip_dfi_read_counters(dfi, res: &now); |
430 | |
431 | do { |
432 | seq = read_seqbegin(sl: &dfi->count_seqlock); |
433 | rockchip_ddr_perf_counters_add(dfi, now: &now, res: &total); |
434 | } while (read_seqretry(sl: &dfi->count_seqlock, start: seq)); |
435 | |
436 | switch (event->attr.config) { |
437 | case PERF_EVENT_CYCLES: |
438 | count = total.c[0].clock_cycles; |
439 | break; |
440 | case PERF_EVENT_READ_BYTES: |
441 | for (i = 0; i < dfi->max_channels; i++) |
442 | count += total.c[i].read_access * blen * dfi->buswidth[i]; |
443 | break; |
444 | case PERF_EVENT_WRITE_BYTES: |
445 | for (i = 0; i < dfi->max_channels; i++) |
446 | count += total.c[i].write_access * blen * dfi->buswidth[i]; |
447 | break; |
448 | case PERF_EVENT_READ_BYTES0: |
449 | count = total.c[0].read_access * blen * dfi->buswidth[0]; |
450 | break; |
451 | case PERF_EVENT_WRITE_BYTES0: |
452 | count = total.c[0].write_access * blen * dfi->buswidth[0]; |
453 | break; |
454 | case PERF_EVENT_READ_BYTES1: |
455 | count = total.c[1].read_access * blen * dfi->buswidth[1]; |
456 | break; |
457 | case PERF_EVENT_WRITE_BYTES1: |
458 | count = total.c[1].write_access * blen * dfi->buswidth[1]; |
459 | break; |
460 | case PERF_EVENT_READ_BYTES2: |
461 | count = total.c[2].read_access * blen * dfi->buswidth[2]; |
462 | break; |
463 | case PERF_EVENT_WRITE_BYTES2: |
464 | count = total.c[2].write_access * blen * dfi->buswidth[2]; |
465 | break; |
466 | case PERF_EVENT_READ_BYTES3: |
467 | count = total.c[3].read_access * blen * dfi->buswidth[3]; |
468 | break; |
469 | case PERF_EVENT_WRITE_BYTES3: |
470 | count = total.c[3].write_access * blen * dfi->buswidth[3]; |
471 | break; |
472 | case PERF_EVENT_BYTES: |
473 | for (i = 0; i < dfi->max_channels; i++) |
474 | count += total.c[i].access * blen * dfi->buswidth[i]; |
475 | break; |
476 | } |
477 | |
478 | return count; |
479 | } |
480 | |
481 | static void rockchip_ddr_perf_event_update(struct perf_event *event) |
482 | { |
483 | u64 now; |
484 | s64 prev; |
485 | |
486 | if (event->attr.config >= PERF_ACCESS_TYPE_MAX) |
487 | return; |
488 | |
489 | now = rockchip_ddr_perf_event_get_count(event); |
490 | prev = local64_xchg(&event->hw.prev_count, now); |
491 | local64_add(now - prev, &event->count); |
492 | } |
493 | |
494 | static void rockchip_ddr_perf_event_start(struct perf_event *event, int flags) |
495 | { |
496 | u64 now = rockchip_ddr_perf_event_get_count(event); |
497 | |
498 | local64_set(&event->hw.prev_count, now); |
499 | } |
500 | |
501 | static int rockchip_ddr_perf_event_add(struct perf_event *event, int flags) |
502 | { |
503 | struct rockchip_dfi *dfi = container_of(event->pmu, struct rockchip_dfi, pmu); |
504 | |
505 | dfi->active_events++; |
506 | |
507 | if (dfi->active_events == 1) { |
508 | dfi->total_count = (struct dmc_count){}; |
509 | rockchip_dfi_read_counters(dfi, res: &dfi->last_perf_count); |
510 | hrtimer_start(timer: &dfi->timer, tim: ns_to_ktime(NSEC_PER_SEC), mode: HRTIMER_MODE_REL); |
511 | } |
512 | |
513 | if (flags & PERF_EF_START) |
514 | rockchip_ddr_perf_event_start(event, flags); |
515 | |
516 | return 0; |
517 | } |
518 | |
519 | static void rockchip_ddr_perf_event_stop(struct perf_event *event, int flags) |
520 | { |
521 | rockchip_ddr_perf_event_update(event); |
522 | } |
523 | |
524 | static void rockchip_ddr_perf_event_del(struct perf_event *event, int flags) |
525 | { |
526 | struct rockchip_dfi *dfi = container_of(event->pmu, struct rockchip_dfi, pmu); |
527 | |
528 | rockchip_ddr_perf_event_stop(event, PERF_EF_UPDATE); |
529 | |
530 | dfi->active_events--; |
531 | |
532 | if (dfi->active_events == 0) |
533 | hrtimer_cancel(timer: &dfi->timer); |
534 | } |
535 | |
536 | static enum hrtimer_restart rockchip_dfi_timer(struct hrtimer *timer) |
537 | { |
538 | struct rockchip_dfi *dfi = container_of(timer, struct rockchip_dfi, timer); |
539 | struct dmc_count now, total; |
540 | |
541 | rockchip_dfi_read_counters(dfi, res: &now); |
542 | |
543 | write_seqlock(sl: &dfi->count_seqlock); |
544 | |
545 | rockchip_ddr_perf_counters_add(dfi, now: &now, res: &total); |
546 | dfi->total_count = total; |
547 | dfi->last_perf_count = now; |
548 | |
549 | write_sequnlock(sl: &dfi->count_seqlock); |
550 | |
551 | hrtimer_forward_now(timer: &dfi->timer, interval: ns_to_ktime(NSEC_PER_SEC)); |
552 | |
553 | return HRTIMER_RESTART; |
554 | }; |
555 | |
556 | static int ddr_perf_offline_cpu(unsigned int cpu, struct hlist_node *node) |
557 | { |
558 | struct rockchip_dfi *dfi = hlist_entry_safe(node, struct rockchip_dfi, node); |
559 | int target; |
560 | |
561 | if (cpu != dfi->cpu) |
562 | return 0; |
563 | |
564 | target = cpumask_any_but(cpu_online_mask, cpu); |
565 | if (target >= nr_cpu_ids) |
566 | return 0; |
567 | |
568 | perf_pmu_migrate_context(pmu: &dfi->pmu, src_cpu: cpu, dst_cpu: target); |
569 | dfi->cpu = target; |
570 | |
571 | return 0; |
572 | } |
573 | |
574 | static void rockchip_ddr_cpuhp_remove_state(void *data) |
575 | { |
576 | struct rockchip_dfi *dfi = data; |
577 | |
578 | cpuhp_remove_multi_state(state: dfi->cpuhp_state); |
579 | |
580 | rockchip_dfi_disable(dfi); |
581 | } |
582 | |
583 | static void rockchip_ddr_cpuhp_remove_instance(void *data) |
584 | { |
585 | struct rockchip_dfi *dfi = data; |
586 | |
587 | cpuhp_state_remove_instance_nocalls(state: dfi->cpuhp_state, node: &dfi->node); |
588 | } |
589 | |
590 | static void rockchip_ddr_perf_remove(void *data) |
591 | { |
592 | struct rockchip_dfi *dfi = data; |
593 | |
594 | perf_pmu_unregister(pmu: &dfi->pmu); |
595 | } |
596 | |
597 | static int rockchip_ddr_perf_init(struct rockchip_dfi *dfi) |
598 | { |
599 | struct pmu *pmu = &dfi->pmu; |
600 | int ret; |
601 | |
602 | seqlock_init(&dfi->count_seqlock); |
603 | |
604 | pmu->module = THIS_MODULE; |
605 | pmu->capabilities = PERF_PMU_CAP_NO_EXCLUDE; |
606 | pmu->task_ctx_nr = perf_invalid_context; |
607 | pmu->attr_groups = attr_groups; |
608 | pmu->event_init = rockchip_ddr_perf_event_init; |
609 | pmu->add = rockchip_ddr_perf_event_add; |
610 | pmu->del = rockchip_ddr_perf_event_del; |
611 | pmu->start = rockchip_ddr_perf_event_start; |
612 | pmu->stop = rockchip_ddr_perf_event_stop; |
613 | pmu->read = rockchip_ddr_perf_event_update; |
614 | |
615 | dfi->cpu = raw_smp_processor_id(); |
616 | |
617 | ret = cpuhp_setup_state_multi(state: CPUHP_AP_ONLINE_DYN, |
618 | name: "rockchip_ddr_perf_pmu" , |
619 | NULL, |
620 | teardown: ddr_perf_offline_cpu); |
621 | |
622 | if (ret < 0) { |
623 | dev_err(dfi->dev, "cpuhp_setup_state_multi failed: %d\n" , ret); |
624 | return ret; |
625 | } |
626 | |
627 | dfi->cpuhp_state = ret; |
628 | |
629 | rockchip_dfi_enable(dfi); |
630 | |
631 | ret = devm_add_action_or_reset(dfi->dev, rockchip_ddr_cpuhp_remove_state, dfi); |
632 | if (ret) |
633 | return ret; |
634 | |
635 | ret = cpuhp_state_add_instance_nocalls(state: dfi->cpuhp_state, node: &dfi->node); |
636 | if (ret) { |
637 | dev_err(dfi->dev, "Error %d registering hotplug\n" , ret); |
638 | return ret; |
639 | } |
640 | |
641 | ret = devm_add_action_or_reset(dfi->dev, rockchip_ddr_cpuhp_remove_instance, dfi); |
642 | if (ret) |
643 | return ret; |
644 | |
645 | hrtimer_init(timer: &dfi->timer, CLOCK_MONOTONIC, mode: HRTIMER_MODE_REL); |
646 | dfi->timer.function = rockchip_dfi_timer; |
647 | |
648 | switch (dfi->ddr_type) { |
649 | case ROCKCHIP_DDRTYPE_LPDDR2: |
650 | case ROCKCHIP_DDRTYPE_LPDDR3: |
651 | dfi->burst_len = 8; |
652 | break; |
653 | case ROCKCHIP_DDRTYPE_LPDDR4: |
654 | case ROCKCHIP_DDRTYPE_LPDDR4X: |
655 | dfi->burst_len = 16; |
656 | break; |
657 | } |
658 | |
659 | ret = perf_pmu_register(pmu, name: "rockchip_ddr" , type: -1); |
660 | if (ret) |
661 | return ret; |
662 | |
663 | return devm_add_action_or_reset(dfi->dev, rockchip_ddr_perf_remove, dfi); |
664 | } |
665 | #else |
666 | static int rockchip_ddr_perf_init(struct rockchip_dfi *dfi) |
667 | { |
668 | return 0; |
669 | } |
670 | #endif |
671 | |
672 | static int rk3399_dfi_init(struct rockchip_dfi *dfi) |
673 | { |
674 | struct regmap *regmap_pmu = dfi->regmap_pmu; |
675 | u32 val; |
676 | |
677 | dfi->clk = devm_clk_get(dev: dfi->dev, id: "pclk_ddr_mon" ); |
678 | if (IS_ERR(ptr: dfi->clk)) |
679 | return dev_err_probe(dev: dfi->dev, err: PTR_ERR(ptr: dfi->clk), |
680 | fmt: "Cannot get the clk pclk_ddr_mon\n" ); |
681 | |
682 | /* get ddr type */ |
683 | regmap_read(map: regmap_pmu, RK3399_PMUGRF_OS_REG2, val: &val); |
684 | dfi->ddr_type = FIELD_GET(RK3399_PMUGRF_OS_REG2_DDRTYPE, val); |
685 | |
686 | dfi->channel_mask = GENMASK(1, 0); |
687 | dfi->max_channels = 2; |
688 | |
689 | dfi->buswidth[0] = FIELD_GET(RK3399_PMUGRF_OS_REG2_BW_CH0, val) == 0 ? 4 : 2; |
690 | dfi->buswidth[1] = FIELD_GET(RK3399_PMUGRF_OS_REG2_BW_CH1, val) == 0 ? 4 : 2; |
691 | |
692 | dfi->ddrmon_stride = 0x14; |
693 | dfi->ddrmon_ctrl_single = true; |
694 | |
695 | return 0; |
696 | }; |
697 | |
698 | static int rk3568_dfi_init(struct rockchip_dfi *dfi) |
699 | { |
700 | struct regmap *regmap_pmu = dfi->regmap_pmu; |
701 | u32 reg2, reg3; |
702 | |
703 | regmap_read(map: regmap_pmu, RK3568_PMUGRF_OS_REG2, val: ®2); |
704 | regmap_read(map: regmap_pmu, RK3568_PMUGRF_OS_REG3, val: ®3); |
705 | |
706 | /* lower 3 bits of the DDR type */ |
707 | dfi->ddr_type = FIELD_GET(RK3568_PMUGRF_OS_REG2_DRAMTYPE_INFO, reg2); |
708 | |
709 | /* |
710 | * For version three and higher the upper two bits of the DDR type are |
711 | * in RK3568_PMUGRF_OS_REG3 |
712 | */ |
713 | if (FIELD_GET(RK3568_PMUGRF_OS_REG3_SYSREG_VERSION, reg3) >= 0x3) |
714 | dfi->ddr_type |= FIELD_GET(RK3568_PMUGRF_OS_REG3_DRAMTYPE_INFO_V3, reg3) << 3; |
715 | |
716 | dfi->channel_mask = BIT(0); |
717 | dfi->max_channels = 1; |
718 | |
719 | dfi->buswidth[0] = FIELD_GET(RK3568_PMUGRF_OS_REG2_BW_CH0, reg2) == 0 ? 4 : 2; |
720 | |
721 | dfi->ddrmon_stride = 0x0; /* not relevant, we only have a single channel on this SoC */ |
722 | dfi->ddrmon_ctrl_single = true; |
723 | |
724 | return 0; |
725 | }; |
726 | |
727 | static int rk3588_dfi_init(struct rockchip_dfi *dfi) |
728 | { |
729 | struct regmap *regmap_pmu = dfi->regmap_pmu; |
730 | u32 reg2, reg3, reg4; |
731 | |
732 | regmap_read(map: regmap_pmu, RK3588_PMUGRF_OS_REG2, val: ®2); |
733 | regmap_read(map: regmap_pmu, RK3588_PMUGRF_OS_REG3, val: ®3); |
734 | regmap_read(map: regmap_pmu, RK3588_PMUGRF_OS_REG4, val: ®4); |
735 | |
736 | /* lower 3 bits of the DDR type */ |
737 | dfi->ddr_type = FIELD_GET(RK3588_PMUGRF_OS_REG2_DRAMTYPE_INFO, reg2); |
738 | |
739 | /* |
740 | * For version three and higher the upper two bits of the DDR type are |
741 | * in RK3588_PMUGRF_OS_REG3 |
742 | */ |
743 | if (FIELD_GET(RK3588_PMUGRF_OS_REG3_SYSREG_VERSION, reg3) >= 0x3) |
744 | dfi->ddr_type |= FIELD_GET(RK3588_PMUGRF_OS_REG3_DRAMTYPE_INFO_V3, reg3) << 3; |
745 | |
746 | dfi->buswidth[0] = FIELD_GET(RK3588_PMUGRF_OS_REG2_BW_CH0, reg2) == 0 ? 4 : 2; |
747 | dfi->buswidth[1] = FIELD_GET(RK3588_PMUGRF_OS_REG2_BW_CH1, reg2) == 0 ? 4 : 2; |
748 | dfi->buswidth[2] = FIELD_GET(RK3568_PMUGRF_OS_REG2_BW_CH0, reg4) == 0 ? 4 : 2; |
749 | dfi->buswidth[3] = FIELD_GET(RK3588_PMUGRF_OS_REG2_BW_CH1, reg4) == 0 ? 4 : 2; |
750 | dfi->channel_mask = FIELD_GET(RK3588_PMUGRF_OS_REG2_CH_INFO, reg2) | |
751 | FIELD_GET(RK3588_PMUGRF_OS_REG2_CH_INFO, reg4) << 2; |
752 | dfi->max_channels = 4; |
753 | |
754 | dfi->ddrmon_stride = 0x4000; |
755 | |
756 | return 0; |
757 | }; |
758 | |
759 | static const struct of_device_id rockchip_dfi_id_match[] = { |
760 | { .compatible = "rockchip,rk3399-dfi" , .data = rk3399_dfi_init }, |
761 | { .compatible = "rockchip,rk3568-dfi" , .data = rk3568_dfi_init }, |
762 | { .compatible = "rockchip,rk3588-dfi" , .data = rk3588_dfi_init }, |
763 | { }, |
764 | }; |
765 | |
766 | MODULE_DEVICE_TABLE(of, rockchip_dfi_id_match); |
767 | |
768 | static int rockchip_dfi_probe(struct platform_device *pdev) |
769 | { |
770 | struct device *dev = &pdev->dev; |
771 | struct rockchip_dfi *dfi; |
772 | struct devfreq_event_desc *desc; |
773 | struct device_node *np = pdev->dev.of_node, *node; |
774 | int (*soc_init)(struct rockchip_dfi *dfi); |
775 | int ret; |
776 | |
777 | soc_init = of_device_get_match_data(dev: &pdev->dev); |
778 | if (!soc_init) |
779 | return -EINVAL; |
780 | |
781 | dfi = devm_kzalloc(dev, size: sizeof(*dfi), GFP_KERNEL); |
782 | if (!dfi) |
783 | return -ENOMEM; |
784 | |
785 | dfi->regs = devm_platform_ioremap_resource(pdev, index: 0); |
786 | if (IS_ERR(ptr: dfi->regs)) |
787 | return PTR_ERR(ptr: dfi->regs); |
788 | |
789 | node = of_parse_phandle(np, phandle_name: "rockchip,pmu" , index: 0); |
790 | if (!node) |
791 | return dev_err_probe(dev: &pdev->dev, err: -ENODEV, fmt: "Can't find pmu_grf registers\n" ); |
792 | |
793 | dfi->regmap_pmu = syscon_node_to_regmap(np: node); |
794 | of_node_put(node); |
795 | if (IS_ERR(ptr: dfi->regmap_pmu)) |
796 | return PTR_ERR(ptr: dfi->regmap_pmu); |
797 | |
798 | dfi->dev = dev; |
799 | mutex_init(&dfi->mutex); |
800 | |
801 | desc = &dfi->desc; |
802 | desc->ops = &rockchip_dfi_ops; |
803 | desc->driver_data = dfi; |
804 | desc->name = np->name; |
805 | |
806 | ret = soc_init(dfi); |
807 | if (ret) |
808 | return ret; |
809 | |
810 | dfi->edev = devm_devfreq_event_add_edev(dev: &pdev->dev, desc); |
811 | if (IS_ERR(ptr: dfi->edev)) { |
812 | dev_err(&pdev->dev, |
813 | "failed to add devfreq-event device\n" ); |
814 | return PTR_ERR(ptr: dfi->edev); |
815 | } |
816 | |
817 | ret = rockchip_ddr_perf_init(dfi); |
818 | if (ret) |
819 | return ret; |
820 | |
821 | platform_set_drvdata(pdev, data: dfi); |
822 | |
823 | return 0; |
824 | } |
825 | |
826 | static struct platform_driver rockchip_dfi_driver = { |
827 | .probe = rockchip_dfi_probe, |
828 | .driver = { |
829 | .name = "rockchip-dfi" , |
830 | .of_match_table = rockchip_dfi_id_match, |
831 | .suppress_bind_attrs = true, |
832 | }, |
833 | }; |
834 | module_platform_driver(rockchip_dfi_driver); |
835 | |
836 | MODULE_LICENSE("GPL v2" ); |
837 | MODULE_AUTHOR("Lin Huang <hl@rock-chips.com>" ); |
838 | MODULE_DESCRIPTION("Rockchip DFI driver" ); |
839 | |